diff --git a/.asf.yaml b/.asf.yaml index 848391d29d323..0657d888cb2c6 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -40,12 +40,75 @@ github: - olehborysevych - rshamunov - andreydevyatkin + - liferoad enabled_merge_buttons: squash: true merge: true rebase: false + protected_branches: + master: {} + release-2.51.0: {} + release-2.50.0: {} + release-2.49.0: {} + release-2.48.0: {} + release-2.47.0: {} + release-2.46.0: {} + release-2.45.0: {} + release-2.44.0: {} + release-2.43.0: {} + release-2.42.0: {} + release-2.41.0: {} + release-2.40.0: {} + release-2.39.0: {} + release-2.38.0: {} + release-2.37.0: {} + release-2.36.0: {} + release-2.35.0: {} + release-2.34.0: {} + release-2.33.0: {} + release-2.32.0: {} + release-2.31.0: {} + release-2.30.0: {} + release-2.29.0: {} + release-2.28.0: {} + release-2.27.0: {} + release-2.26.0: {} + release-2.25.0: {} + release-2.24.0: {} + release-2.23.0: {} + release-2.22.0: {} + release-2.21.0: {} + release-2.20.0: {} + release-2.19.0: {} + release-2.18.0: {} + release-2.17.0: {} + release-2.16.0: {} + release-2.15.0: {} + release-2.14.0: {} + release-2.13.0: {} + release-2.12.0: {} + release-2.11.0: {} + release-2.10.0: {} + release-2.8.0: {} + release-2.8.0: {} + release-2.7.0: {} + release-2.6.0: {} + release-2.5.0: {} + release-2.4.0: {} + release-2.3.0: {} + release-2.2.0: {} + release-2.1.1: {} + release-2.1.0: {} + release-0.6.0: {} + release-0.5.0: {} + release-0.4.0: {} + release-0.4.0-incubating: {} + release-0.3.0-incubating: {} + release-0.2.0-incubating: {} + release-0.1.0-incubating: {} + notifications: commits: commits@beam.apache.org issues: github@beam.apache.org diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 1ff96082c0357..67f8b21445dcc 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -63,6 +63,7 @@ body: - label: "Component: Go SDK" - label: "Component: Typescript SDK" - label: "Component: IO connector" + - label: "Component: Beam YAML" - label: "Component: Beam examples" - label: "Component: Beam playground" - label: "Component: Beam katas" diff --git a/.github/ISSUE_TEMPLATE/failing_test.yml b/.github/ISSUE_TEMPLATE/failing_test.yml index 0904719e6d2c7..44e1cd720745b 100644 --- a/.github/ISSUE_TEMPLATE/failing_test.yml +++ b/.github/ISSUE_TEMPLATE/failing_test.yml @@ -69,6 +69,7 @@ body: - label: "Component: Go SDK" - label: "Component: Typescript SDK" - label: "Component: IO connector" + - label: "Component: Beam YAML" - label: "Component: Beam examples" - label: "Component: Beam playground" - label: "Component: Beam katas" diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml index deaa14287b8a1..11234a5e1501e 100644 --- a/.github/ISSUE_TEMPLATE/feature.yml +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -57,6 +57,7 @@ body: - label: "Component: Go SDK" - label: "Component: Typescript SDK" - label: "Component: IO connector" + - label: "Component: Beam YAML" - label: "Component: Beam examples" - label: "Component: Beam playground" - label: "Component: Beam katas" diff --git a/.github/ISSUE_TEMPLATE/task.yml b/.github/ISSUE_TEMPLATE/task.yml index 4fa1b241207d5..477b91b181be0 100644 --- a/.github/ISSUE_TEMPLATE/task.yml +++ b/.github/ISSUE_TEMPLATE/task.yml @@ -58,6 +58,7 @@ body: - label: "Component: Go SDK" - label: "Component: Typescript SDK" - label: "Component: IO connector" + - label: "Component: Beam YAML" - label: "Component: Beam examples" - label: "Component: Beam playground" - label: "Component: Beam katas" diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 04c1911a430ad..44e31838aa2f5 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -8,7 +8,7 @@ Thank you for your contribution! Follow this checklist to help us incorporate yo - [ ] Update `CHANGES.md` with noteworthy changes. - [ ] If this contribution is large, please file an Apache [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.pdf). -See the [Contributor Guide](https://beam.apache.org/contribute) for more tips on [how to make review process smoother](https://beam.apache.org/contribute/get-started-contributing/#make-the-reviewers-job-easier). +See the [Contributor Guide](https://beam.apache.org/contribute) for more tips on [how to make review process smoother](https://github.com/apache/beam/blob/master/CONTRIBUTING.md#make-the-reviewers-job-easier). To check the build health, please visit [https://github.com/apache/beam/blob/master/.test-infra/BUILD_STATUS.md](https://github.com/apache/beam/blob/master/.test-infra/BUILD_STATUS.md) @@ -19,4 +19,4 @@ GitHub Actions Tests Status (on master branch) [![Java tests](https://github.com/apache/beam/workflows/Java%20Tests/badge.svg?branch=master&event=schedule)](https://github.com/apache/beam/actions?query=workflow%3A%22Java+Tests%22+branch%3Amaster+event%3Aschedule) [![Go tests](https://github.com/apache/beam/workflows/Go%20tests/badge.svg?branch=master&event=schedule)](https://github.com/apache/beam/actions?query=workflow%3A%22Go+tests%22+branch%3Amaster+event%3Aschedule) -See [CI.md](https://github.com/apache/beam/blob/master/CI.md) for more information about GitHub Actions CI. +See [CI.md](https://github.com/apache/beam/blob/master/CI.md) for more information about GitHub Actions CI or the [workflows README](https://github.com/apache/beam/blob/master/.github/workflows/README.md) to see a list of phrases to trigger workflows. diff --git a/.github/REVIEWERS.yml b/.github/REVIEWERS.yml index a38effbaeeb4b..1c260fbea5eb6 100644 --- a/.github/REVIEWERS.yml +++ b/.github/REVIEWERS.yml @@ -45,15 +45,17 @@ labels: reviewers: - chamikaramj - johnjcasey - - pabloem - Abacn - ahmedabu98 - bvolpato - - manavgarg exclusionList: [] - name: spanner reviewers: - nielm + - name: bigtable + reviewers: + - igorbernstein2 + - mutianf exclusionList: [] - name: Build reviewers: diff --git a/.github/actions/gradle-command-self-hosted-action/action.yml b/.github/actions/gradle-command-self-hosted-action/action.yml index 8fbf63aca42f7..906b35169d9db 100644 --- a/.github/actions/gradle-command-self-hosted-action/action.yml +++ b/.github/actions/gradle-command-self-hosted-action/action.yml @@ -24,12 +24,6 @@ inputs: required: false description: 'Gradle options' default: '' - default-arguments: - required: false - description: 'Default gradle switches' # Copied from CommonJobProperties.groovy' - default: | - --continue -Dorg.gradle.jvmargs=-Xms2g -Dorg.gradle.jvmargs=-Xmx6g \ - -Dorg.gradle.vfs.watch=false -Pdocker-pull-licenses max-workers: required: false description: 'Max number of workers' @@ -44,6 +38,9 @@ runs: # Removing settings.xml is a workaround to avoid a decryption issue # of Beam's gradle-command-action plugin and github's provided # maven settings.xml file - rm ~/.m2/settings.xml - ./gradlew ${{ inputs.gradle-command }} --max-workers=${{ inputs.max-workers }} ${{ inputs.arguments }} \ - ${{ inputs.default-arguments }} \ No newline at end of file + if [ -f ~/.m2/settings.xml ]; then + rm ~/.m2/settings.xml + fi + ./gradlew ${{ inputs.gradle-command }} --max-workers=${{ inputs.max-workers }} --continue \ + -Dorg.gradle.jvmargs=-Xms2g -Dorg.gradle.jvmargs=-Xmx6g -Dorg.gradle.vfs.watch=false -Pdocker-pull-licenses \ + ${{ inputs.arguments }} diff --git a/.github/actions/rerun-job-action/action.yml b/.github/actions/rerun-job-action/action.yml new file mode 100644 index 0000000000000..73679aaaf8190 --- /dev/null +++ b/.github/actions/rerun-job-action/action.yml @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +#Action used to trigger a failed check re-run within a PR using a comment. Add this action to your workflow with an if condition +#to check if the comment is present +#If the check is failed this will trigger it again. If its not failed a new instance of workflow will run which will not show in the status box or checks tab in the PR and can be found in the actions tab https://github.com/apache/beam/actions + +name: "Rerun Job Action" +description: Re-runs a job that is attached to the PR as Check Run +inputs: + pull_request_url: + description: "The URL of the PR" + required: true + github_repository: + description: "The GitHub repository" + required: true + github_token: + description: "The GitHub token" + required: true + github_job: + description: "The GitHub job" + required: true + github_current_run_id: + description: "The GitHub current run id. Not the same that is fetched in this action" + required: true + + +runs: + using: composite + steps: + - name: Get Last Commit SHA + shell: bash + run: | + URL=${{inputs.pull_request_url}}/commits + PRSHA=$(curl \ + -H 'Authorization: Bearer ${{inputs.github_token}}' \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -s $URL | jq -r '.[-1].sha' ) + echo prsha=$PRSHA >> $GITHUB_ENV + - name: Get Status and Conclusion for PR Job + shell: bash + run: | + JOB="${{inputs.github_job}}" + QUERY_JOB=${JOB// /+} + URL="${{github.api_url}}/repos/${{inputs.github_repository}}/commits/${{env.prsha}}/check-runs?check_name=$QUERY_JOB" + CHECK_RUN=$(curl \ + -H 'Authorization: Bearer ${{inputs.github_token}}' \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -s $URL | jq -r '.check_runs | .[] | select(.name=="${{inputs.github_job}}")') + if [ -z "$CHECK_RUN" ]; then + echo "No check runs found for this job" + echo skip=true >> $GITHUB_ENV + exit 0 + fi + read -r STATUS CONCLUSION CHECK_SUITE_ID<<< $(echo $CHECK_RUN | jq -r '"\(.status) \(.conclusion) \(.check_suite.id)"') + echo status=$STATUS >> $GITHUB_ENV + echo conclusion=$CONCLUSION >> $GITHUB_ENV + echo check_suite_id=$CHECK_SUITE_ID >> $GITHUB_ENV + + + - name: Disable Rerun for Success or Skipped + if: ${{(env.status == 'completed' && (env.conclusion == 'success' || env.conclusion == 'skipped')) || env.skip == 'true'}} + shell: bash + run: echo rerun=false >> $GITHUB_ENV + + - name: Get Run ID + if: ${{env.rerun != 'false' }} + shell: bash + run: | + URL="${{github.api_url}}/repos/${{inputs.github_repository}}/actions/runs?check_suite_id=${{env.check_suite_id}}" + RUN_ID=$(curl \ + -H 'Authorization: Bearer ${{inputs.github_token}}' \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -s $URL | jq -r '.workflow_runs | .[0] | .id') + echo run_id=$RUN_ID >> $GITHUB_ENV + - name: Get Job ID + if: ${{env.rerun != 'false' }} + shell: bash + run: | + URL="${{github.api_url}}/repos/${{inputs.github_repository}}/actions/runs/${{env.run_id}}/jobs" + JOB_ID=$(curl \ + -H 'Authorization: Bearer ${{inputs.github_token}}' \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -s $URL | jq -r '.jobs | .[] | select(.name=="${{inputs.github_job}}") | .id ') + echo job_id=$JOB_ID >> $GITHUB_ENV + - name: Trigger Re-run + if: ${{env.rerun != 'false' }} + shell: bash + run: | + URL="${{github.api_url}}/repos/${{inputs.github_repository}}/actions/jobs/${{env.job_id}}/rerun" + curl -X POST \ + -H 'Authorization: Bearer ${{inputs.github_token}}' \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -s $URL + - name: Install GH Cli + if: ${{env.rerun != 'false' }} + shell: bash + run: | + wget https://github.com/cli/cli/releases/download/v2.31.0/gh_2.31.0_linux_amd64.tar.gz + tar -xvf gh_2.31.0_linux_amd64.tar.gz + sudo mv gh_2.31.0_linux_amd64/bin/gh /usr/local/bin + - name: Exit rest of the run + if: ${{env.rerun != 'false' }} + shell: bash + run: | + gh run cancel ${{ inputs.github_current_run_id }} + gh run watch ${{ inputs.github_current_run_id }} + env: + GITHUB_TOKEN: ${{ inputs.github_token }} \ No newline at end of file diff --git a/.github/actions/setup-action/action.yml b/.github/actions/setup-action/action.yml new file mode 100644 index 0000000000000..da69dd9a97ddc --- /dev/null +++ b/.github/actions/setup-action/action.yml @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Setup action' +description: 'Setup repository and configure the required steps' +inputs: + comment_phrase: + description: "The comment phrase in the PR" + required: true + github_token: + description: "The GitHub token" + required: true + github_job: + description: "The GitHub job" + required: false + default: '' + +runs: + using: composite + steps: + - name: Check if the phrase is correct + shell: bash + if: github.event_name == 'issue_comment' && github.event.comment.body != inputs.comment_phrase + run: | + echo "The comment ${{ github.event.comment.body }} does not match the phrase for this instance: ${{ inputs.comment_phrase }}. Exiting." + exit 1 + - name: Check out repository code if pull request commit + shell: bash + if: ${{ github.event_name == 'pull_request_target' }} + run: | + # GitHub will automatically generate a merge commit when there are no merge conflicts. + # We first try to check that out, and fall back to checking out the tip of the pull request branch. + git fetch --depth=1 origin +refs/pull/${{ github.event.number }}/merge:refs/remotes/pull/${{ github.event.number }}/merge || \ + git fetch --depth=1 origin +refs/pull/${{ github.event.number }}/head:refs/remotes/pull/${{ github.event.number }}/head + git checkout pull/${{ github.event.number }}/merge || git checkout pull/${{ github.event.number }}/head + - name: Check out repository code if comment phrase + shell: bash + if: ${{ github.event.comment.body == inputs.comment_phrase }} + run: | + # GitHub will automatically generate a merge commit when there are no merge conflicts. + # We first try to check that out, and fall back to checking out the tip of the pull request branch. + git fetch --depth=1 origin +refs/pull/${{ github.event.issue.number }}/merge:refs/remotes/pull/${{ github.event.issue.number }}/merge || \ + git fetch --depth=1 origin +refs/pull/${{ github.event.issue.number }}/head:refs/remotes/pull/${{ github.event.issue.number }}/head + git checkout pull/${{ github.event.issue.number }}/merge || git checkout pull/${{ github.event.issue.number }}/head + - name: Rerun if comment phrase + if: ${{ github.event.comment.body == inputs.comment_phrase }} + uses: ./.github/actions/rerun-job-action + with: + github_token: ${{ inputs.github_token }} + github_job: ${{ inputs.github_job || github.job }} + github_repository: ${{ github.repository }} + github_current_run_id: ${{ github.run_id }} + pull_request_url: ${{ github.event.issue.pull_request.url }} + ## Used for jobs that spawn docker containers and need to mount gcloud config directory + - name: expose gcloud path + shell: bash + run: | + echo KUBELET_GCLOUD_CONFIG_PATH=/var/lib/kubelet/pods/$POD_UID/volumes/kubernetes.io~empty-dir/gcloud >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-environment-action diff --git a/.github/actions/setup-default-test-properties/test-properties.json b/.github/actions/setup-default-test-properties/test-properties.json index 32b449ea1ed9e..b53169ffef9c2 100644 --- a/.github/actions/setup-default-test-properties/test-properties.json +++ b/.github/actions/setup-default-test-properties/test-properties.json @@ -1,14 +1,14 @@ { "PythonTestProperties": { - "ALL_SUPPORTED_VERSIONS": ["3.7", "3.8", "3.9", "3.10", "3.11"], - "LOWEST_SUPPORTED": ["3.7"], + "ALL_SUPPORTED_VERSIONS": ["3.8", "3.9", "3.10", "3.11"], + "LOWEST_SUPPORTED": ["3.8"], "HIGHEST_SUPPORTED": ["3.11"], - "ESSENTIAL_VERSIONS": ["3.7", "3.11"], - "CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS": ["3.7", "3.11"], + "ESSENTIAL_VERSIONS": ["3.8", "3.11"], + "CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS": ["3.8", "3.11"], "CROSS_LANGUAGE_VALIDATES_RUNNER_DATAFLOW_USING_SQL_PYTHON_VERSIONS": ["3.11"], - "VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.7", "3.8", "3.9", "3.10", "3.11" ] - "LOAD_TEST_PYTHON_VERSION": "3.7", - "CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.7", + "VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.8", "3.9", "3.10", "3.11" ] + "LOAD_TEST_PYTHON_VERSION": "3.8", + "CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.8", "DEFAULT_INTERPRETER": "python3.8", "TOX_ENV": ["Cloud", "Cython"] }, @@ -18,6 +18,6 @@ "SPARK_VERSIONS": ["2", "3"] }, "GoTestProperties": { - "SUPPORTED_VERSIONS": ["1.19"] + "SUPPORTED_VERSIONS": ["1.20"] } } diff --git a/.github/actions/setup-environment-action/action.yml b/.github/actions/setup-environment-action/action.yml new file mode 100644 index 0000000000000..3452a16c132c2 --- /dev/null +++ b/.github/actions/setup-environment-action/action.yml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Setup environment action' +description: 'Setup environment to run jobs' +inputs: + python-version: + required: false + description: 'Install Python version' + default: '' + java-version: + required: false + description: 'Install Java version' + default: '' + go-version: + required: false + description: 'Install Go version' + default: '' + +runs: + using: "composite" + steps: + - name: Install Python + if: ${{ inputs.python-version != '' }} + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + - name: Install Java + if: ${{ inputs.java-version != '' }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ inputs.java-version }} + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + with: + cache-read-only: false + - name: Install Go + if: ${{ inputs.go-version != '' }} + uses: actions/setup-go@v3 + with: + go-version: ${{ inputs.go-version }} # never set patch, to get latest patch releases. diff --git a/.github/actions/setup-k8s-access/action.yml b/.github/actions/setup-k8s-access/action.yml new file mode 100644 index 0000000000000..5248fff429a0e --- /dev/null +++ b/.github/actions/setup-k8s-access/action.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +#Action used to trigger a failed check re-run within a PR using a comment. Add this action to your workflow with an if condition +#to check if the comment is present +#If the check is failed this will trigger it again. If its not failed a new instance of workflow will run which will not show in the status box or checks tab in the PR and can be found in the actions tab https://github.com/apache/beam/actions + +name: "Setup Kuberenetes Access" +description: Sets up kuberenetes access in gcp for the current workflow +inputs: + cluster_name: + description: "Name of the cluster to be created" + required: true + default: "io-datastores" + k8s_namespace: + description: "Name of the namespace to be created" + required: true + cluster_zone: + description: "Zone of the cluster to be created" + required: true + default: "us-central1-a" + + +runs: + using: composite + steps: + - name: Check if inputs were provided + shell: bash + run: | + if [ -z "${{ inputs.k8s_namespace }}" ]; then + echo "Kubernetes namespace not provided" + exit 1 + fi + - name: replace '_' with '-' in namespace + shell: bash + id: replace_namespace + run: | + TEST_NAMESPACE=$(echo "${{ inputs.k8s_namespace }}" | tr '_' '-' | tr '[:upper:]' '[:lower:]') + echo TEST_NAMESPACE=$TEST_NAMESPACE >> $GITHUB_OUTPUT + - name: Get the kubeconfig using gcloud + shell: bash + run: | + gcloud container clusters get-credentials ${{ inputs.cluster_name }} --zone ${{ inputs.cluster_zone }} --project apache-beam-testing + - name: Create namespace + shell: bash + run: | + kubectl create namespace ${{ steps.replace_namespace.outputs.TEST_NAMESPACE }} + - name: Set default namespace + shell: bash + run: | + kubectl config set-context --current --namespace=${{ steps.replace_namespace.outputs.TEST_NAMESPACE }} + - name: Post cleanup + uses: pyTooling/Actions/with-post-step@v0.4.6 + with: + main: echo "Post Cleanup" + post: | + echo "Post Cleanup" + kubectl delete namespace ${{ steps.replace_namespace.outputs.TEST_NAMESPACE }} diff --git a/.github/actions/setup-self-hosted-action/action.yml b/.github/actions/setup-self-hosted-action/action.yml deleted file mode 100644 index 0e7cc8534e9e5..0000000000000 --- a/.github/actions/setup-self-hosted-action/action.yml +++ /dev/null @@ -1,70 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: 'Setup environment for self-hosted runners' -description: 'Setup action to run jobs in a self-hosted runner' -inputs: - requires-py-37: - required: false - description: 'Set as false if does not require py37 setup' - default: 'true' - requires-py-38: - required: false - description: 'Set as false if does not require py38 setup' - default: 'true' - requires-py-39: - required: false - description: 'Set as false if does not require py39 setup' - default: 'true' - requires-java-8: - required: false - description: 'Set as false if does not require java-8 setup' - default: 'true' - requires-go: - required: false - description: 'Set as false if does not require go setup' - default: 'true' - -runs: - using: "composite" - steps: - - name: Install python 3.7 - if: ${{ inputs.requires-py-37 == 'true' }} - uses: actions/setup-python@v4 - with: - python-version: "3.7" - - name: Install python 3.8 - if: ${{ inputs.requires-py-38 == 'true' }} - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - name: Install python 3.9 - if: ${{ inputs.requires-py-39 == 'true' }} - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - name: Set Java Version - if: ${{ inputs.requires-java-8 == 'true' }} - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 8 - - name: Set Go Version - if: ${{ inputs.requires-go == 'true' }} - uses: actions/setup-go@v3 - with: - go-version: '1.20' # never set patch, to get latest patch releases. diff --git a/.github/actions/test-arguments-action/action.yml b/.github/actions/test-arguments-action/action.yml new file mode 100644 index 0000000000000..17814098980e9 --- /dev/null +++ b/.github/actions/test-arguments-action/action.yml @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Set test arguments action' +description: 'Set test arguments action to run the test' +inputs: + argument-file-paths: + required: true + description: 'List of paths to files with test arguments' + default: '' + arguments: + required: false + description: 'Test arguments generated at runtime' + default: '' + test-type: + required: true + description: 'Specify if this is a "load" or "performance" test' + test-language: + required: true + description: 'Specify if this is a "java", "python" or "go" test' + +runs: + using: composite + steps: + - name: Check if test-type was provided + shell: bash + run: | + if [ -z "${{ inputs.test-type }}" ]; then + echo "Test type was not provided" + exit 1 + fi + - name: Check if test-language was provided + shell: bash + run: | + if [ -z "${{ inputs.test-language }}" ]; then + echo "Test language was not provided" + exit 1 + fi + - name: Set common arguments + id: common_arguments + shell: bash + run: | + echo project="apache-beam-testing" >> $GITHUB_OUTPUT + echo influx_db_name="beam_test_metrics" >> $GITHUB_OUTPUT + echo influx_host="http://10.128.0.96:8086" >> $GITHUB_OUTPUT + - name: Get default ${{ inputs.test-language }} test arguments + id: default_arguments + shell: bash + run: | + DEFAULT_ARGUMENTS="" + if ${{ inputs.test-language == 'java' }}; then + DEFAULT_ARGUMENTS=$(echo " + --project=${{ steps.common_arguments.outputs.project }} + --influxDatabase=${{ steps.common_arguments.outputs.influx_db_name }} + --influxHost=${{ steps.common_arguments.outputs.influx_host }} + ") + elif ${{ inputs.test-language == 'python' || inputs.test-language == 'go' }}; then + DEFAULT_ARGUMENTS=$(echo " + --project=${{ steps.common_arguments.outputs.project }} + --influx_db_name=${{ steps.common_arguments.outputs.influx_db_name }} + --influx_hostname=${{ steps.common_arguments.outputs.influx_host }} + ") + fi + echo arguments=$DEFAULT_ARGUMENTS >> $GITHUB_OUTPUT + - name: Group arguments + shell: bash + run: | + PATHS=($(echo "${{ inputs.argument-file-paths }}" | tr '\n' ' ')) + for index in "${!PATHS[@]}"; do + CONFIG=$(grep -v "^#.*" ${PATHS[index]}) + ARGUMENTS=$(echo "${{ steps.default_arguments.outputs.arguments }} ${{ inputs.arguments }} $CONFIG" | tr '\n' ' ') + ARGUMENTS="${ARGUMENTS% }" + if ${{ inputs.test-type == 'performance' }}; then + arguments="" + read -ra args <<< "$ARGUMENTS" + for arg in "${args[@]}"; do + arguments="${arguments}\"${arg}\"," + done + ARGUMENTS="${arguments%,}" + fi + echo "${{ github.job }}_test_arguments_$((index + 1))=$ARGUMENTS" >> $GITHUB_ENV + done diff --git a/.github/autolabeler.yml b/.github/autolabeler.yml index 5a8a22044da43..57c8f65c6ac17 100644 --- a/.github/autolabeler.yml +++ b/.github/autolabeler.yml @@ -31,6 +31,7 @@ python: ["sdks/python/**/*", "learning/katas/python/**/*"] typescript: ["sdks/typescript/**/*"] vendor: ["vendor/**/*"] website: ["website/**/*"] +yaml: ["sdks/python/apache_beam/yaml/**"] # Extensions extensions: ["sdks/java/extensions/**/*", "runners/extensions-java/**/*"] @@ -68,6 +69,7 @@ io: ["sdks/go/pkg/beam/io/**/*", "sdks/java/io/**/*", "sdks/python/apache_beam/ "redis": ["sdks/java/io/redis/**/*"] "solr": ["sdks/java/io/solr/**/*"] "spanner": ["sdks/go/pkg/beam/io/spannerio/**/*", "sdks/python/apache_beam/io/gcp/spanner.py", "sdks/python/apache_beam/io/gcp/experimental/spannerio.py", "sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/**/*"] +"bigtable": ["sdks/go/pkg/beam/io/bigtableio/**/*", "sdks/go/pkg/beam/io/xlang/bigtableio/**/*", "sdks/python/apache_beam/io/gcp/bigtableio.py", "sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/**/*"] "synthetic": ["sdks/java/io/synthetic/**/*"] "tests": ["sdks/java/io/file-based-io-tests/**/*"] "thrift": ["sdks/java/io/thrift/**/*"] diff --git a/.github/codecov.yml b/.github/codecov.yml index c1c5dfb17bb48..211591768a167 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -65,6 +65,7 @@ ignore: - "**/*_microbenchmark.py" - "sdks/go/pkg/beam/register/register.go" - "sdks/python/apache_beam/testing/benchmarks/nexmark/**" + - "sdks/python/apache_beam/examples/**" # See https://docs.codecov.com/docs/flags for options. flag_management: diff --git a/.github/gh-actions-self-hosted-runners/arc/README.md b/.github/gh-actions-self-hosted-runners/arc/README.md index 5be4a93b43a3e..e5055826d00c8 100644 --- a/.github/gh-actions-self-hosted-runners/arc/README.md +++ b/.github/gh-actions-self-hosted-runners/arc/README.md @@ -38,8 +38,15 @@ All are created in the step before project_id = "PROJECT_ID" # google PROJECT_ID that you want to deploy in region = "gcp_region" # GCP region for the network zone = "europe-west3-c" # GCP zone for the nodes -min_main_node_count = "1" # Minimal and initial node count for main pool -max_main_node_count = "5" # Maximal node count for main pool +main_runner = { + name = "main-runner" # Main runner pool name + machine_type = "e2-standard-16" # Main runner pool machine type + min_node_count = "1" # Main runner pool minimal node count + max_node_count = "5" # Main runner pool maximal node count + min_replicas = "5" # Min number of runner PODs in the main pool . Do not confuse with Nodes + max_replicas = "20" # Max number of runner PODs in the main pool . Do not confuse with Nodes + webhook_scaling # Enable webhook scaling for main pool +} environment = "environment_name" # Name of the environment. Used as a prefix like dev- stag- anything- ingress_domain = "fqdn" # FQDN for webhook ingress organization = "org" # Github Organization to use runners in @@ -48,15 +55,40 @@ github_app_id_secret_name = "app_id_secret_name" # Google secret na github_app_install_id_secret_name = "install_id_secret_name" # Google secret name for install_id github_private_key_secret_name = "pem_file_secret_name" # Google secret name for pem file deploy_webhook = "false" # Terraform to deploy the scaling webhook -max_main_replicas = "2" # Max number of runner PODs . Do not confuse with Nodes -min_main_replicas = "1" # Min number of runner PODs . Do not confuse with Nodes -webhook_scaling = "false" # Enable webhook scaling. When disabled runner busy percentage is used #state_bucket_name = "state_bucket_name" # Not used by terraform. This is just to reference what bucket is used for others ``` +If you want to create additonal pools you can use the `additional_runner_pools` which is a list of objects. Example: +``` +additional_runner_pools = [ +{ +name = "test-runner" # Pool name +machine_type = "e2-standard-2" # Macihne type for the pool +min_node_count = 1 # Minimal node count +max_node_count = 2 # Maximal node count +min_replicas = 1 # Minimal replica count +min_replicas = 2 # Maximal replica count +webhook_scaling = true # Enable webhook based scaling +runner_image = "gcr.io/someimage:sometag" # Image to use +labels = ["self-hosted", "testrunner"] # Label set for runner pool. Used in `on` +enable_selector = "true" # Enables NodeSelector, forcing runners to this pool +enable_taint = "true" # Enables Taints. Prevents other runner pods to run in this pool. +requests = { # K8s cpu and memory requests + cpu = "500m" # + memory = "500mi"} # +limits = { # K8s cpu and memory limits + cpu = "2" # + memory = "2Gi"}}] # + +``` + + + 5. Make sure you set the bucket name in the comment in the environment file for documentation purposes -6. From this directory, init terraform with: +6. From this directory, login to your gcloud account that you created the bucket with and init terraform with: ``` +gcloud auth login +gcloud auth application-default login terraform init -backend-config="bucket=bucket_name" ``` 7. Terraform apply diff --git a/.github/gh-actions-self-hosted-runners/arc/config/arc_autoscaler.tpl b/.github/gh-actions-self-hosted-runners/arc/config/arc_autoscaler.tpl index 0de685c453bb0..f6da0aff038ae 100644 --- a/.github/gh-actions-self-hosted-runners/arc/config/arc_autoscaler.tpl +++ b/.github/gh-actions-self-hosted-runners/arc/config/arc_autoscaler.tpl @@ -19,12 +19,12 @@ apiVersion: actions.summerwind.dev/v1alpha1 kind: HorizontalRunnerAutoscaler metadata: - name: main-runners + name: ${name} spec: scaleDownDelaySecondsAfterScaleOut: 300 scaleTargetRef: kind: RunnerDeployment - name: main-runners + name: ${name} minReplicas: ${min_runners} maxReplicas: ${max_runners} %{~ if webhook_scaling == "true" ~} diff --git a/.github/gh-actions-self-hosted-runners/arc/config/arc_deployment.tpl b/.github/gh-actions-self-hosted-runners/arc/config/arc_deployment.tpl index 98eeb8544f781..6234571c55a30 100644 --- a/.github/gh-actions-self-hosted-runners/arc/config/arc_deployment.tpl +++ b/.github/gh-actions-self-hosted-runners/arc/config/arc_deployment.tpl @@ -19,21 +19,42 @@ apiVersion: actions.summerwind.dev/v1alpha1 kind: RunnerDeployment metadata: - name: main-runners + name: ${name} spec: template: spec: - image: summerwind/actions-runner:v2.304.0-ubuntu-20.04-30355f7 + %{~ if selector == true ~} + nodeSelector: + runner-pool: ${name} + %{~ endif ~} + %{~ if taint == true ~} + tolerations: + - key: "runner-pool" + operator: "Equal" + value: ${name} + effect: "NoSchedule" + %{~ endif ~} + image: ${image} organization: ${organization} group: "${group}" labels: - - "ubuntu-20.04" - - "self-hosted" - env: [] + %{~ for label in labels ~} + - ${label} + %{~ endfor ~} + env: + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid resources: - limits: - cpu: "4.0" - memory: "8Gi" requests: - cpu: "500m" - memory: "500Mi" + cpu: ${requests.cpu} + memory: ${requests.memory} + limits: + %{~ if limits.cpu != "" ~} + cpu: ${limits.cpu} + %{~ if limits.memory != "" ~} + memory: ${limits.memory} + %{~ endif ~} + %{~ endif ~} + diff --git a/.github/gh-actions-self-hosted-runners/arc/environments/beam.env b/.github/gh-actions-self-hosted-runners/arc/environments/beam.env index e215e4c7bd2bf..9de66b628c898 100644 --- a/.github/gh-actions-self-hosted-runners/arc/environments/beam.env +++ b/.github/gh-actions-self-hosted-runners/arc/environments/beam.env @@ -18,20 +18,64 @@ # project_id = "apache-beam-testing" -region = "us-west1" -zone = "us-west1-b" -min_main_node_count = "1" -max_main_node_count = "5" +region = "us-central1" +zone = "us-central1-b" environment = "beam" -ingress_domain = "runners.example.com" +ingress_domain = "action.beam.apache.org" organization = "apache" repository = "beam" github_app_id_secret_name = "gh-app_id" github_app_install_id_secret_name = "gh-app_installation_id" github_private_key_secret_name = "gh-pem_key" -deploy_webhook = "false" -max_main_replicas = "40" -min_main_replicas = "5" -webhook_scaling = "false" +deploy_webhook = "true" runner_group = "beam" -machine_type = "e2-standard-16" \ No newline at end of file +main_runner = { + name = "main-runner" + runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:2b20e26bb3b99d8e4f41a3d1d9d2e7080043de5c" + machine_type = "e2-standard-16" + min_node_count = "1" + max_node_count = "24" + min_replicas = "1" + max_replicas = "200" + webhook_scaling = true + disk_size_gb = 200 + requests = { + cpu = "2" + memory = "3Gi" + } +} +additional_runner_pools = [{ + name = "small-runner" + machine_type = "e2-standard-2" + runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:2b20e26bb3b99d8e4f41a3d1d9d2e7080043de5c" + min_node_count = "1" + max_node_count = "10" + min_replicas = "1" + max_replicas = "10" + webhook_scaling = "true" + requests = { + cpu = "1500m" + memory = "5Gi" + } + labels = ["self-hosted", "ubuntu-20.04", "small"] + enable_selector = true + enable_taint = true +}, +{ + name = "highmem-runner" + machine_type = "c3-highmem-8" + runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:2b20e26bb3b99d8e4f41a3d1d9d2e7080043de5c" + min_node_count = "1" + max_node_count = "10" + min_replicas = "1" + max_replicas = "10" + webhook_scaling = "true" + requests = { + cpu = "7.5" + memory = "5Gi" + } + labels = ["self-hosted", "ubuntu-20.04", "highmem"] + enable_selector = true + enable_taint = true +}] +#state_bucket_name = "beam-arc-state" diff --git a/.github/gh-actions-self-hosted-runners/arc/gke.tf b/.github/gh-actions-self-hosted-runners/arc/gke.tf index d4740b834eb17..bfb048885570a 100644 --- a/.github/gh-actions-self-hosted-runners/arc/gke.tf +++ b/.github/gh-actions-self-hosted-runners/arc/gke.tf @@ -23,30 +23,72 @@ resource "google_container_cluster" "actions-runner-gke" { initial_node_count = 1 network = google_compute_network.actions-runner-network.id subnetwork = google_compute_subnetwork.actions-runner-subnetwork.id - remove_default_node_pool = true + remove_default_node_pool = true } -resource "google_container_node_pool" "actions-runner-pool" { +resource "google_container_node_pool" "main-actions-runner-pool" { name = "main-pool" cluster = google_container_cluster.actions-runner-gke.name location = google_container_cluster.actions-runner-gke.location autoscaling { - min_node_count = var.min_main_node_count - max_node_count = var.max_main_node_count + min_node_count = var.main_runner.min_node_count + max_node_count = var.main_runner.max_node_count } + initial_node_count = var.main_runner.min_node_count management { auto_repair = "true" auto_upgrade = "true" } node_config { - machine_type = var.machine_type - service_account = google_service_account.actions_service_account.email + disk_size_gb = var.main_runner.disk_size_gb + machine_type = var.main_runner.machine_type oauth_scopes = [ "https://www.googleapis.com/auth/cloud-platform" ] tags = ["actions-runner-pool"] } } + +resource "google_container_node_pool" "additional_runner_pools" { + for_each = { + for index, runner_pool in var.additional_runner_pools : runner_pool.name => runner_pool + } + + name = each.value.name + cluster = google_container_cluster.actions-runner-gke.name + location = google_container_cluster.actions-runner-gke.location + autoscaling { + min_node_count = each.value.min_node_count + max_node_count = each.value.max_node_count + } + initial_node_count = each.value.min_node_count + management { + auto_repair = "true" + auto_upgrade = "true" + } + node_config { + disk_size_gb = each.value.disk_size_gb + machine_type = each.value.machine_type + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform" + ] + tags = ["actions-runner-pool"] + labels = { + "runner-pool" = each.value.name + } + + dynamic "taint" { + for_each = each.value.enable_taint == true ? [1] : [] + content { + key = "runner-pool" + value = each.value.name + effect = "NO_SCHEDULE" + } + } + } + } + + resource "google_compute_global_address" "actions-runner-ip" { name = "${var.environment}-actions-runner-ip" } \ No newline at end of file diff --git a/.github/gh-actions-self-hosted-runners/arc/helm.tf b/.github/gh-actions-self-hosted-runners/arc/helm.tf index 8b7d528dcc1b6..4c2badaf32398 100644 --- a/.github/gh-actions-self-hosted-runners/arc/helm.tf +++ b/.github/gh-actions-self-hosted-runners/arc/helm.tf @@ -30,7 +30,7 @@ resource "helm_release" "cert-manager" { name = "installCRDs" value = "true" } - depends_on = [ google_container_node_pool.actions-runner-pool ] + depends_on = [ google_container_node_pool.main-actions-runner-pool ] } resource "helm_release" "arc" { diff --git a/.github/gh-actions-self-hosted-runners/arc/iam.tf b/.github/gh-actions-self-hosted-runners/arc/iam.tf index 54f209e535b85..e6ba2be6545a0 100644 --- a/.github/gh-actions-self-hosted-runners/arc/iam.tf +++ b/.github/gh-actions-self-hosted-runners/arc/iam.tf @@ -16,13 +16,6 @@ # specific language governing permissions and limitations # under the License. # - -resource "google_service_account" "actions_service_account" { - account_id = "${var.environment}-runner-gke-sa" - display_name = "${var.environment}-runner-gke-sa" -} - - data "google_client_config" "provider" {} data "google_client_openid_userinfo" "provider_identity" { diff --git a/.github/gh-actions-self-hosted-runners/arc/images/Dockerfile b/.github/gh-actions-self-hosted-runners/arc/images/Dockerfile new file mode 100644 index 0000000000000..2cbfea75ab558 --- /dev/null +++ b/.github/gh-actions-self-hosted-runners/arc/images/Dockerfile @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +FROM ghcr.io/actions-runner-controller/actions-runner-controller/actions-runner:ubuntu-20.04 +#install BuildX +COPY --from=docker/buildx-bin:0.11.1 /buildx /usr/libexec/docker/cli-plugins/docker-buildx +RUN docker buildx install && docker buildx version + + +USER root +#Install Node +RUN curl -OL https://nodejs.org/dist/v18.16.0/node-v18.16.0-linux-x64.tar.xz && \ + tar -C /usr/local -xf node-v18.16.0-linux-x64.tar.xz && \ + rm node-v18.16.0-linux-x64.tar.xz && \ + mv /usr/local/node-v18.16.0-linux-x64 /usr/local/node +ENV PATH="${PATH}:/usr/local/node/bin" +#Install Go +ARG go_version=1.21.0 +RUN curl -OL https://go.dev/dl/go${go_version}.linux-amd64.tar.gz && \ + tar -C /usr/local -xzf go${go_version}.linux-amd64.tar.gz && \ + rm go${go_version}.linux-amd64.tar.gz +ENV PATH="${PATH}:/usr/local/go/bin" +#Install Java +RUN curl -OL https://cdn.azul.com/zulu/bin/zulu8.70.0.23-ca-jdk8.0.372-linux_x64.tar.gz && \ + tar -C /usr/local -xzf zulu8.70.0.23-ca-jdk8.0.372-linux_x64.tar.gz && \ + rm zulu8.70.0.23-ca-jdk8.0.372-linux_x64.tar.gz && \ + mv /usr/local/zulu8.70.0.23-ca-jdk8.0.372-linux_x64 /usr/local/java +ENV PATH="${PATH}:/usr/local/java/bin" +#Install Gradle +RUN curl -OL https://services.gradle.org/distributions/gradle-7.3.3-bin.zip && \ + unzip gradle-7.3.3-bin.zip && \ + rm gradle-7.3.3-bin.zip && \ + mv gradle-7.3.3 /usr/local/gradle +ENV PATH="${PATH}:/usr/local/gradle/bin" +#Install GitHub CLI +RUN curl -OL https://github.com/cli/cli/releases/download/v2.31.0/gh_2.31.0_linux_amd64.tar.gz && \ + tar -xvf gh_2.31.0_linux_amd64.tar.gz && \ + rm gh_2.31.0_linux_amd64.tar.gz && \ + mv gh_2.31.0_linux_amd64/bin/gh /usr/local/bin +#Install GCloud CLI and Kubectl +RUN curl -OL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-367.0.0-linux-x86_64.tar.gz && \ + tar -xvf google-cloud-sdk-367.0.0-linux-x86_64.tar.gz && \ + rm google-cloud-sdk-367.0.0-linux-x86_64.tar.gz && \ + mv google-cloud-sdk /usr/local/google-cloud-sdk && \ + /usr/local/google-cloud-sdk/install.sh --quiet && \ + /usr/local/google-cloud-sdk/bin/gcloud components install gke-gcloud-auth-plugin && \ + #revert permission + chown -R runner:runner /home/runner/.config +ENV USE_GKE_GCLOUD_AUTH_PLUGIN=True +ENV PATH="${PATH}:/usr/local/google-cloud-sdk/bin" +#Install Kubectl +RUN curl -OL https://dl.k8s.io/release/v1.28.1/bin/linux/amd64/kubectl && \ + chmod +x ./kubectl && \ + mv ./kubectl /usr/local/bin/kubectl +#Install Apache Maven +RUN curl -OL https://dlcdn.apache.org/maven/maven-3/3.9.4/binaries/apache-maven-3.9.4-bin.tar.gz && \ + tar -xvf apache-maven-3.9.4-bin.tar.gz && \ + rm apache-maven-3.9.4-bin.tar.gz && \ + mv apache-maven-3.9.4 /usr/local/maven +ENV PATH="${PATH}:/usr/local/maven/bin" +ENV MAVEN_HOME="/usr/local/maven" + +# Needed to transfer path addtitions to runner environment +RUN echo PATH=$PATH >> /runnertmp/.env +RUN echo USE_GKE_GCLOUD_AUTH_PLUGIN=$USE_GKE_GCLOUD_AUTH_PLUGIN >> /runnertmp/.env +USER runner diff --git a/.github/gh-actions-self-hosted-runners/arc/images/README.md b/.github/gh-actions-self-hosted-runners/arc/images/README.md new file mode 100644 index 0000000000000..6db908d1bc48c --- /dev/null +++ b/.github/gh-actions-self-hosted-runners/arc/images/README.md @@ -0,0 +1,36 @@ + +# Manual build and push + +First set a tag you want to use: +``` +export RUNNER_IMAGE_TAG=some_tag +``` +After which you run the build command: +``` +docker build -t us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:$RUNNER_IMAGE_TAG -t us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:$(git rev-parse --short HEAD) . +``` +This builds and tags the image with both the Git SHA and desired tag set. + +Authenticate to the docker repository in GCP with: +``` +gcloud auth configure-docker us-central1-docker.pkg.dev +``` +docker push us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:$RUNNER_IMAGE_TAG +docker push us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:$(git rev-parse --short HEAD) +``` \ No newline at end of file diff --git a/.github/gh-actions-self-hosted-runners/arc/kubernetes.tf b/.github/gh-actions-self-hosted-runners/arc/kubernetes.tf index 9622a0cab1164..bafb653896d73 100644 --- a/.github/gh-actions-self-hosted-runners/arc/kubernetes.tf +++ b/.github/gh-actions-self-hosted-runners/arc/kubernetes.tf @@ -17,17 +17,35 @@ # under the License. # resource "kubectl_manifest" "arc_deployment" { - yaml_body = templatefile("config/arc_deployment.tpl", { organization = var.organization , group = var.runner_group}) + yaml_body = templatefile("config/arc_deployment.tpl", { organization = var.organization, group = var.runner_group, name = var.main_runner.name, image = var.main_runner.runner_image, labels = var.main_runner.labels, selector = var.main_runner.enable_selector, taint = var.main_runner.enable_taint, requests = var.main_runner.requests, limits = var.main_runner.limits}) override_namespace = "arc" - depends_on = [ helm_release.arc ] + depends_on = [helm_release.arc] } resource "kubectl_manifest" "arc_autoscaler" { - yaml_body = templatefile("config/arc_autoscaler.tpl", { min_runners = var.min_main_replicas, max_runners = var.max_main_replicas , webhook_scaling = var.webhook_scaling}) + yaml_body = templatefile("config/arc_autoscaler.tpl", { name = var.main_runner.name, min_runners = var.main_runner.min_replicas, max_runners = var.main_runner.max_replicas, webhook_scaling = var.main_runner.webhook_scaling }) override_namespace = "arc" - depends_on = [ helm_release.arc ] + depends_on = [helm_release.arc] } resource "kubectl_manifest" "arc_webhook_certificate" { - yaml_body = templatefile("config/arc_certificate.tpl", { ingress_domain = var.ingress_domain }) + yaml_body = templatefile("config/arc_certificate.tpl", { ingress_domain = var.ingress_domain }) override_namespace = "arc" - depends_on = [ helm_release.arc ] + depends_on = [helm_release.arc] +} + + +resource "kubectl_manifest" "arc_deployment_additional" { + for_each = { + for index, runner_pool in var.additional_runner_pools : runner_pool.name => runner_pool + } + yaml_body = templatefile("config/arc_deployment.tpl", { organization = var.organization, group = var.runner_group, name = each.value.name, image = each.value.runner_image, labels = each.value.labels, selector = each.value.enable_selector, taint = each.value.enable_taint , requests = each.value.requests, limits = each.value.limits}) + override_namespace = "arc" + depends_on = [helm_release.arc] +} +resource "kubectl_manifest" "arc_autoscaler_additional" { + for_each = { + for index, runner_pool in var.additional_runner_pools : runner_pool.name => runner_pool + } + yaml_body = templatefile("config/arc_autoscaler.tpl", { name = each.value.name, min_runners = each.value.min_replicas, max_runners = each.value.max_replicas, webhook_scaling = each.value.webhook_scaling }) + override_namespace = "arc" + depends_on = [helm_release.arc] } diff --git a/.github/gh-actions-self-hosted-runners/arc/variables.tf b/.github/gh-actions-self-hosted-runners/arc/variables.tf index 81d6695e133f3..43f51938b7d1f 100644 --- a/.github/gh-actions-self-hosted-runners/arc/variables.tf +++ b/.github/gh-actions-self-hosted-runners/arc/variables.tf @@ -27,28 +27,6 @@ variable "region" { variable "zone" { description = "Google Zone to use for deployment" } -variable "min_main_node_count" { - description = "Minimal node count for GKE" - default = "1" -} -variable "max_main_node_count" { - description = "Maximal node count for GKE" - default = "2" -} -variable "max_main_replicas" { - description = "Maximal replicas for Action Runners" - default = "2" - -} -variable "min_main_replicas" { - description = "Minimal replicas for Action Runners" - default = "1" - -} -variable machine_type { - description = "Machine type to use for runner Node Pool" - default = "e2-standard-2" -} variable "environment" { description = "name of environment" default = "" @@ -84,8 +62,63 @@ variable "runner_group" { description = "value for the runner group label" default = "" } -variable "webhook_scaling" { - description = "Enable scaling of runners based on webhook events" - default = "false" - -} + +variable "main_runner" { + type = object({ + name = string + machine_type = optional(string, "e2-standard-2") + min_node_count = optional(number, 1) + max_node_count = optional(number, 1) + min_replicas = optional(number, 1) + max_replicas = optional(number, 1) + disk_size_gb = optional(number, 100) + webhook_scaling = optional(bool, false) + runner_image = optional(string, "summerwind/actions-runner:v2.304.0-ubuntu-20.04-30355f7") + labels = optional(list(string), ["self-hosted", "ubuntu-20.04","main"]) + enable_selector = optional(bool, false) + enable_taint = optional(bool, false) + requests = optional(object({ + cpu = string + memory = string + }), { cpu = "500m", + memory = "500Mi" + }) + limits = optional(object({ + cpu = optional(string) + memory = optional(string) + }), { + cpu = "", + memory = "" + }) + }) +} +variable "additional_runner_pools" { + type = list(object({ + name = string + machine_type = optional(string, "e2-standard-2") + min_node_count = optional(number, 1) + max_node_count = optional(number, 1) + min_replicas = optional(number, 1) + max_replicas = optional(number, 1) + disk_size_gb = optional(number, 100) + webhook_scaling = optional(bool, false) + runner_image = optional(string, "summerwind/actions-runner:v2.304.0-ubuntu-20.04-30355f7") + labels = optional(list(string), ["self-hosted", "ubuntu-20.04","changeme"]) + enable_selector = optional(bool, true) + enable_taint = optional(bool, true) + requests = optional(object({ + cpu = string + memory = string + }), { cpu = "500m", + memory = "500Mi" + }) + limits = optional(object({ + cpu = optional(string) + memory = optional(string) + }), { + cpu = "", + memory = "" + }) + })) + default = [] +} \ No newline at end of file diff --git a/.github/issue-rules.yml b/.github/issue-rules.yml index c26cb84bf6bad..b01a22dafd784 100644 --- a/.github/issue-rules.yml +++ b/.github/issue-rules.yml @@ -36,6 +36,8 @@ rules: addLabels: ['typescript'] - contains: '[x] Component: IO' addLabels: ['io'] +- contains: '[x] Component: Beam YAML' + addLabels: ['yaml'] - contains: '[x] Component: Beam examples' addLabels: ['examples'] - contains: '[x] Component: Beam playground' diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000000000..3bd7591062070 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,352 @@ + + +# Running Workflows Manually + +Most workflows will get kicked off automatically when you open a PR, push code, or on a schedule. + +If you would like to manually trigger a job, you have 2 options: + +1) Trigger Phrases: Many jobs have trigger phrases associated with them (e.g. `Run XYZ PreCommit`). These will appear in statuses of previous PR runs of that check. You can trigger the job on any PR by commenting that trigger phrase in the PR. +2) **Committers only** - Manual triggering: Any committer can start any job with a [workflow_dispatch](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_dispatch) trigger defined (all jobs should have these). To do so, navigate to the [Actions tab](https://github.com/apache/beam/actions), click on your desired workflow in the left navigation bar, and then click `Run Workflow`. + +# Guidelines for Adding or Modifying Workflows + +On top of normal Actions workflow steps, all new CI workflows (excluding release workflows or other workflow automation) should have the following: + +1) Name and phrase set via matrix for re-run to work (See below) +2) A set of specific triggers +3) An explicit checkout step +4) A set of GitHub token permissions +5) Concurrency Groups +6) Comment Triggering Support + +Each of these is described in more detail below. +## Name and Phrase +Due to specifics on how the comment triggered rerun is handled it is required that all jobs have name and phrase set via matrix elements. See the following example: +``` +jobs: + beam_job_name: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: [beam_job_name] + job_phrase: [Run Job Phrase] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Job Phrase' + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + +``` +And in case when the workflow already utilizes matrix do the following: +``` +jobs: + beam_job_with_matrix: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + job_name: ["beam_job_with_matrix"] + job_phrase: ["Run Job With Matrix"] + python_version: ['3.8','3.9','3.10','3.11'] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + startsWith(github.event.comment.body, 'Run Job With Matrix') + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) +``` +Notice how the matrix element of `python_version` is appended to the name. + +## Workflow triggers + +GitHub allows workflows to define a set of triggers that dictate when a job should be run. For more info, see [documentation here](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows). + +For the purposes of Beam, each CI workflow should define the following triggers: + +1) A `push` trigger +2) A `pull_request_target` trigger +3) An issue_comment trigger (for issue created). This is needed for comment triggering support (see section below). +4) A scheduled trigger +5) A workflow_dispatch trigger + +The `push`/`pull_request_target` triggers should only run when appropriate paths are modified. See https://github.com/apache/beam/blob/master/.github/workflows/beam_PreCommit_Go.yml#L4 for an example (you can copy and paste this into your workflow, you just need to change the paths). + +## Checkout step + +Because we use the `pull_request_target` trigger instead of `pull_request`, we need an explicit checkout of the correct commit. This can be done as a step that uses the `setup-action` action in your workflow. See https://github.com/apache/beam/blob/0ee2dc73ec6f555a5bf1a643dffd37f4927be67e/.github/workflows/beam_PreCommit_Go.yml#L65-L70 for an example (you can copy and paste this into your workflow). Please make sure that you checkout the code before using the composite action. + +## Token Permissions + +You should explicitly define the GitHub Actions token scopes needed by your job. For most jobs, this will be `actions: write` (needed for comment triggering support) and `read` permissions for all other options. See https://github.com/apache/beam/blob/907c5110163b0efe52e9e12127fd013c7fc455d7/.github/workflows/beam_PreCommit_Go.yml#L16 for an example (you can copy and paste this into your workflow). + +## Concurrency Groups + +Concurrency groups are a way of making sure that no more than one Actions run is running per job/group at any given time (previous ones can be cancelled). To reduce resource usage, you should define the following concurrency group: + +``` +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true +``` + +this defines the following groups: + +1) Each workflow will represent a different set of groups +2) Within each workflow, each PR will represent a single group +3) Within each non-PR run for a workflow, each commit will represent a set of groups +4) Within each commit, each push event, schedule event, and manual run event will represent a set of groups + +## Comment Triggering Support + +In order to make it easier for non-committers to interact with workflows, workflows should implement comment triggering support. This requires 3 additional components beyond the ones mentioned above: + +1) Each job should be gated on an if condition that maps to that workflow's comment trigger (example: https://github.com/apache/beam/blob/907c5110163b0efe52e9e12127fd013c7fc455d7/.github/workflows/beam_PreCommit_Go.yml#L38) +2) Each job should have the rerun action immediately after its checkout step. You can add a step that uses the `setup-action` action in your workflow, which encapsulates the checkout and rerun logic in one place. This should be gated on the comment trigger (example: https://github.com/apache/beam/blob/0ee2dc73ec6f555a5bf1a643dffd37f4927be67e/.github/workflows/beam_PreCommit_Go.yml#L65-L70) +3) Each job should have a descriptive name that includes the comment trigger (example: https://github.com/apache/beam/blob/ba8fc935222aeb070668fbafd588bc58e7a21289/.github/workflows/beam_PreCommit_CommunityMetrics.yml#L48) + +# Testing new workflows or workflow updates + +## Testing New Workflows + +New workflows are not added to the [UI on the Actions tab](https://github.com/apache/beam/actions) until they are merged into the repo's main branch (master for Beam). +To test new workflows, we recommend the following pattern: + +1) Fork the Beam repo +2) Add your proposed workflow to the main branch of your fork. +3) Run the workflow in the [Actions tab](https://github.com/apache/beam/actions) of your fork using the `Run workflow` button. + +Note: most workflows use [self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners) +with the main and ubuntu labels to execute ([example](https://github.com/apache/beam/blob/5a54ee6ddd8cb8444c41802929a364fe2561001e/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml#L41)). +If you are testing on a fork, you likely will not have self-hosted runners set up. +To work around this, you can start using hosted runners and then switch over when you're ready to create a PR. +You can do this by changing `runs-on: [self-hosted, ubuntu-20.04, main]` (self-hosted, use in your PR) to `runs-on: ubuntu-20.04` (GitHub hosted, use for local testing). + +## Testing Workflow Updates + +If you need to make more changes to the workflow yaml file after it has been added to the repo, you can develop normally on a branch (using your fork or the main Beam repo if you are a committer). If you are a non-committer, this flow has several caveats mentioned below. +To do this: + +1) Make your change on a development branch. +2) Navigate to your workflow in the [Actions tab](https://github.com/apache/beam/actions). If your changes are on a fork, navigate to the fork's Actions tab instead. If you don't see the correct action, make sure that your fork's main branch is up to date with Beam's master branch. +3) Click run workflow. Before clicking submit, update to run on your branch. + +Note: If you run a workflow from your fork of Beam, it will not have access to secrets stored in the Beam repository. +This will cause some things like authenticating to external services to fail, which may cause your workflow to fail. +If you run into this issue, you can either: +1) Ask for a committers help to add the workflow to a branch on the main apache/beam repo. +2) Upload secrets to your repo mirroring the secrets used in the main Beam repo. +3) Wait until the changes are merged into Beam to test (this should only be done rarely). + +Additionally, as mentioned above your fork likely will not have self-hosted runners set up. +To work around this, you can start using hosted runners and then switch over when you're ready to create a PR. +You can do this by changing runs-on: [self-hosted, ubuntu-20.04, main] (self-hosted, use in your PR) to runs-on: ubuntu-20.04 (GitHub hosted, use for local testing). + +# Workflows +Please note that jobs with matrix need to have matrix element in the comment. Example: +```Run Python PreCommit (3.8)``` +| Workflow name | Matrix | Trigger Phrase | Cron Status | +|:-------------:|:------:|:--------------:|:-----------:| +| [ Java InfluxDbIO Integration Test ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml) | N/A |`Run Java InfluxDbIO_IT`| [![.github/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml) +| [ Load Tests GBK Dataflow Batch Go ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml) | N/A |`Run Load Tests Go GBK Dataflow Batch`| [![.github/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml) +| [ Load Tests CoGBK Dataflow Streaming Java ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml) | N/A |`Run Load Tests Java CoGBK Dataflow Streaming`| [![.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml) +| [ Load Tests Combine Dataflow Batch Python ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml) | N/A |`Run Load Tests Python Combine Dataflow Batch`| [![.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml) +| [ Load Tests Combine Dataflow Batch Python ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml) | N/A |`Run Load Tests Python Combine Dataflow Batch`| [![.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml) +| [ Load Tests FnApiRunner Microbenchmark Python ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml) | N/A |`Run Python Load Tests FnApiRunner Microbenchmark`| [![.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml) +| [ Load Tests ParDo Dataflow Batch Go ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch.yml) | N/A |`Run Load Tests Go ParDo Dataflow Batch`| [![.github/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch.yml) +| [ Performance Tests AvroIOIT HDFS ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml) | N/A |`Run Java AvroIO Performance Test HDFS`| [![.github/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml) +| [ Performance Tests AvroIOIT ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_AvroIOIT.yml) | N/A |`Run Java AvroIO Performance Test`| [![.github/workflows/beam_PerformanceTests_AvroIOIT.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_AvroIOIT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_AvroIOIT.yml) +| [ Performance Tests BigQueryIO Batch Java Avro ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml) | N/A |`Run BigQueryIO Batch Performance Test Java Avro`| [![.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml) +| [ Performance Tests BigQueryIO Batch Java Json ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml) | N/A |`Run BigQueryIO Batch Performance Test Java Json`| [![.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml) +| [ Performance Tests BigQueryIO Streaming Java ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml) | N/A |`Run BigQueryIO Streaming Performance Test Java`| [![.github/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml) +| [ PostCommit BeamMetrics Publish ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_BeamMetrics_Publish.yml) | N/A |`Run Beam Metrics Deployment`| [![.github/workflows/beam_PostCommit_BeamMetrics_Publish.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_BeamMetrics_Publish.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_BeamMetrics_Publish.yml) +| [ PostCommit Go ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go.yml) | N/A |`Run Go PostCommit`| [![.github/workflows/beam_PostCommit_Go.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go.yml) | +| [ PostCommit Go Dataflow ARM](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_Dataflow_ARM.yml) | N/A |`Run Go PostCommit Dataflow ARM`| [![.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_Dataflow_ARM.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_Dataflow_ARM.yml) | +| [ PostCommit Go VR Flink](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Flink.yml) | N/A |`Run Go Flink ValidatesRunner`| [![.github/workflows/beam_PostCommit_Go_VR_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Flink.yml) | +| [ PostCommit Go VR Samza](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Samza.yml) | N/A |`Run Go Samza ValidatesRunner`| [![.github/workflows/beam_PostCommit_Go_VR_Samza.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Samza.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Samza.yml) | +| [ PostCommit Go VR Spark](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Spark.yml) | N/A |`Run Go Spark ValidatesRunner`| [![.github/workflows/beam_PostCommit_Go_VR_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Spark.yml) | +| [ PostCommit Java Avro Versions ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Avro_Versions.yml) | N/A |`Run Java Avro Versions PostCommit`| [![.github/workflows/beam_PostCommit_Java_Avro_Versions.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Avro_Versions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Avro_Versions.yml) | +| [ PostCommit Java Dataflow V1 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV1.yml) | N/A |`Run PostCommit_Java_Dataflow`| [![.github/workflows/beam_PostCommit_Java_DataflowV1.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV1.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV1.yml) | +| [ PostCommit Java Dataflow V2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV2.yml) | N/A |`Run PostCommit_Java_DataflowV2`| [![.github/workflows/beam_PostCommit_Java_DataflowV2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV2.yml) | +| [ PostCommit Java Examples Dataflow ARM ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml) | ['8','11','17'] |`Run Java_Examples_Dataflow_ARM PostCommit (matrix_element)`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml) | +| [ PostCommit Java Examples Dataflow](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow.yml) | N/A |`Run Java examples on Dataflow`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow.yml) | +| [ PostCommit Java Examples Dataflow Java ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml) | ['11','17'] |`Run Java examples on Dataflow Java (matrix_element)`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml) | +| [ PostCommit Java Examples Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Direct.yml) | N/A |`Run Java Examples_Direct`| [![.github/workflows/beam_PostCommit_Java_Examples_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Direct.yml) | +| [ PostCommit Java Examples Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Flink.yml) | N/A |`Run Java Examples_Flink`| [![.github/workflows/beam_PostCommit_Java_Examples_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Flink.yml) | +| [ PostCommit Java Examples Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Spark.yml) | N/A |`Run Java Examples_Spark`| [![.github/workflows/beam_PostCommit_Java_Examples_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Spark.yml) | +| [ PostCommit Java Hadoop Versions ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Hadoop_Versions.yml) | N/A |`Run PostCommit_Java_Hadoop_Versions`| [![.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Hadoop_Versions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Hadoop_Versions.yml) | +| [ PostCommit Java Jpms Dataflow Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml) | N/A |`Run Jpms Dataflow Java 11 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml) | +| [ PostCommit Java Jpms Dataflow Java17 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml) | N/A |`Run Jpms Dataflow Java 17 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml) | +| [ PostCommit Java Jpms Direct Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml) | N/A |`Run Jpms Direct Java 11 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java11](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml) | +| [ PostCommit Java Jpms Direct Java17 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml) | N/A |`Run Jpms Direct Java 17 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml) | +| [ PostCommit Java Jpms Flink Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml) | N/A |`Run Jpms Flink Java 11 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Flink_Java11](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml) | +| [ PostCommit Java Jpms Spark Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml) | N/A |`Run Jpms Spark Java 11 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Spark_Java11](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml) | +| [ PostCommit Java Nexmark Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml) | N/A |`Run Dataflow Runner Nexmark Tests`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml) | +| [ PostCommit Java Nexmark Dataflow V2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml) | N/A |`Run Dataflow Runner V2 Nexmark Tests`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml) | +| [ PostCommit Java Nexmark Dataflow V2 Java ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml) | ['11','17'] |`Run Dataflow Runner V2 Java (matrix) Nexmark Tests`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml) | +| [ PostCommit Java Nexmark Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Direct.yml) | N/A |`Run Direct Runner Nexmark Tests`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Direct.yml) | +| [ PostCommit Java Nexmark Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Flink.yml) | N/A |`Run Flink Runner Nexmark Tests`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Flink.yml) | +| [ PostCommit Java Nexmark Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Spark.yml) | N/A |`Run Spark Runner Nexmark Tests`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Spark.yml) | +| [ PostCommit Java PVR Flink Streaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml) | N/A |`Run Java Flink PortableValidatesRunner Streaming`| [![PostCommit Java PVR Flink Streaming](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml) | +| [ PostCommit Java PVR Samza ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Samza.yml) | N/A |`Run Java Samza PortableValidatesRunner`| [![PostCommit Java PVR Samza](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Samza.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Samza.yml) | +| [ PostCommit Java PVR Spark3 Streaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml) | N/A |`Run Java Spark v3 PortableValidatesRunner Streaming`| [![PostCommit Java PVR Spark3 Streaming](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml) | +| [ PostCommit Java PVR Spark Batch ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml) | N/A |`Run Java Spark PortableValidatesRunner Batch`| [![PostCommit Java PVR Spark Batch](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml) | +| [ PostCommit Java Sickbay ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Sickbay.yml) | N/A |`Run Java Sickbay`| [![.github/workflows/beam_PostCommit_Java_Sickbay.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Sickbay.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Sickbay.yml) | +| [ PostCommit Java Tpcds Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml) | N/A |`Run Dataflow Runner Tpcds Tests`| [![.github/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml) | +| [ PostCommit Java Tpcds Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Flink.yml) | N/A |`Run Flink Runner Tpcds Tests`| [![.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Flink.yml) | +| [ PostCommit Java Tpcds Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Spark.yml) | N/A |`Run Spark Runner Tpcds Tests`| [![.github/workflows/beam_PostCommit_Java_Tpcds_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Spark.yml) | +| [ PostCommit Java ValidatesRunner Dataflow JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Java.yml) | ['11','17'] |`Run Dataflow ValidatesRunner Java (matrix_element)`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Java.yml) | +| [ PostCommit Java ValidatesRunner Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml) | N/A |`Run Dataflow Streaming ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml) | +| [ PostCommit Java ValidatesRunner Dataflow V2 Streaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml) | N/A |`Run Java Dataflow V2 ValidatesRunner Streaming`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml) | +| [ PostCommit Java ValidatesRunner Dataflow V2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml) | N/A |`Run Java Dataflow V2 ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml) | +| [ PostCommit Java ValidatesRunner Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml) | N/A |`Run Dataflow ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml) | +| [ PostCommit Java ValidatesRunner Direct JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_Java.yml) | ['11','17'] |`Run Direct ValidatesRunner Java (matrix_element)`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_Java.yml) | +| [ PostCommit Java ValidatesRunner Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml) | N/A |`Run Direct ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml) | +| [ PostCommit Java ValidatesRunner Flink Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml) | N/A |`Run Flink ValidatesRunner Java 11`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml) | +| [ PostCommit Java ValidatesRunner Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml) | N/A |`Run Flink ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml) | +| [ PostCommit Java ValidatesRunner Samza ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml) | N/A |`Run Samza ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml) | +| [ PostCommit Java ValidatesRunner Spark Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml) | N/A |`Run Spark ValidatesRunner Java 11`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml) | +| [ PostCommit Java ValidatesRunner Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml) | N/A |`Run Spark ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml) | +| [ PostCommit Java ValidatesRunner SparkStructuredStreaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml) | N/A |`Run Spark StructuredStreaming ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml) | +| [ PostCommit Java ValidatesRunner Twister2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml) | N/A |`Run Twister2 ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml) | +| [ PostCommit Java ValidatesRunner ULR ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml) | N/A |`Run ULR Loopback ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml) | +| [ PostCommit Java ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java.yml) | N/A |`Run Java PostCommit`| [![.github/workflows/beam_PostCommit_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java.yml) | +| [ PostCommit Javadoc ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Javadoc.yml) | N/A |`Run Javadoc PostCommit`| [![.github/workflows/beam_PostCommit_Javadoc.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Javadoc.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Javadoc.yml) | +| [ PostCommit PortableJar Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Flink.yml) | N/A |`Run PortableJar_Flink PostCommit`| [![.github/workflows/beam_PostCommit_PortableJar_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Flink.yml) | +| [ PostCommit PortableJar Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Spark.yml) | N/A |`Run PortableJar_Spark PostCommit`| [![.github/workflows/beam_PostCommit_PortableJar_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Spark.yml) | +| [ PostCommit Python ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python.yml) | ['3.8','3.9','3.10','3.11'] |`Run Python PostCommit (matrix_element)`| [![.github/workflows/beam_PostCommit_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python.yml) | +| [ PostCommit Python Arm](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Arm.yml) | ['3.8','3.9','3.10','3.11'] |`Run Python PostCommit Arm (matrix_element)`| [![.github/workflows/beam_PostCommit_Python_Arm.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Arm.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Arm.yml) | +| [ PostCommit Python Examples Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Dataflow.yml) | N/A |`Run Python Examples_Dataflow`| [![.github/workflows/beam_PostCommit_Python_Examples_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Dataflow.yml) | +| [ PostCommit Python Examples Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Direct.yml) | ['3.8','3.9','3.10','3.11'] |`Run Python Examples_Direct (matrix_element)`| [![.github/workflows/beam_PostCommit_Python_Examples_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Direct.yml) | +| [ PostCommit Python Examples Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Flink.yml) | ['3.8','3.11'] |`Run Python Examples_Flink (matrix_element)`| [![.github/workflows/beam_PostCommit_Python_Examples_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Flink.yml) | +| [ PostCommit Python Examples Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Spark.yml) | ['3.8','3.11'] |`Run Python Examples_Spark (matrix_element)`| [![.github/workflows/beam_PostCommit_Python_Examples_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Spark.yml) | +| [ PostCommit Python MongoDBIO IT ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml) | N/A |`Run Python MongoDBIO_IT`| [![.github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml) | +| [ PostCommit Python Nexmark Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Nexmark_Direct.yml) | N/A |`Run Python Direct Runner Nexmark Tests`| [![.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Nexmark_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Nexmark_Direct.yml) | +| [ PostCommit Python ValidatesContainer Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml) | ['3.8','3.9','3.10','3.11'] |`Run Python Dataflow ValidatesContainer (matrix_element)`| [![.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml) | +| [ PostCommit Python ValidatesContainer Dataflow With RC ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml) | ['3.8','3.9','3.10','3.11'] |`Run Python RC Dataflow ValidatesContainer (matrix_element)`| [![.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml) | +| [ PostCommit Python ValidatesRunner Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml) | ['3.8','3.11'] |`Run Python Dataflow ValidatesRunner (matrix_element)`| [![PostCommit Python ValidatesRunner Dataflow](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml) | +| [ PostCommit Python ValidatesRunner Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml) | ['3.8','3.11'] |`Run Python Flink ValidatesRunner (matrix_element)`| [![PostCommit Python ValidatesRunner Flink](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml) | +| [ PostCommit Python ValidatesRunner Samza ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml) | ['3.8','3.11'] |`Run Python Samza ValidatesRunner (matrix_element)`| [![PostCommit Python ValidatesRunner Samza](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml) | +| [ PostCommit Python ValidatesRunner Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml) | ['3.8','3.9','3.11'] |`Run Python Spark ValidatesRunner (matrix_element)`| [![PostCommit Python ValidatesRunner Spark](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml) | +| [ PostCommit Python Xlang Gcp Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml) | N/A |`Run Python_Xlang_Gcp_Dataflow PostCommit`| [![.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml) | +| [ PostCommit Python Xlang Gcp Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml) | N/A |`Run Python_Xlang_Gcp_Direct PostCommit`| [![.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml) | +| [ PostCommit Python Xlang IO Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml) | N/A |`Run Python_Xlang_IO_Dataflow PostCommit`| [![.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml) | +| [ PostCommit Sickbay Python ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Sickbay_Python.yml) | ['3.8','3.9','3.10','3.11'] |`Run Python (matrix_element) PostCommit Sickbay`| [![.github/workflows/beam_PostCommit_Sickbay_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Sickbay_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Sickbay_Python.yml) | +| [ PostCommit SQL ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_SQL.yml) | N/A |`Run SQL PostCommit`| [![.github/workflows/beam_PostCommit_SQL.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_SQL.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_SQL.yml) | +| [ PostCommit TransformService Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_TransformService_Direct.yml) | N/A |`Run TransformService_Direct PostCommit`| [![.github/workflows/beam_PostCommit_TransformService_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_TransformService_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_TransformService_Direct.yml) +| [ PostCommit Website Publish ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Website_Publish.yml) | N/A | N/A | [![.github/workflows/beam_PostCommit_Website_Publish.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Website_Publish.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Website_Publish.yml) | +| [ PostCommit Website Test](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Website_Test.yml) | N/A |`Run Full Website Test`| [![.github/workflows/beam_PostCommit_Website_Test](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Website_Test.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Website_Test.yml) | +| [ PostCommit XVR GoUsingJava Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml) | N/A |`Run XVR_GoUsingJava_Dataflow PostCommit`| [![.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml) | +| [ PostCommit XVR Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Direct.yml) | N/A |`Run XVR_Direct PostCommit`| [![.github/workflows/beam_PostCommit_XVR_Direct](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Direct.yml) | +| [ PostCommit XVR Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Flink.yml) | N/A |`Run XVR_Flink PostCommit`| [![.github/workflows/beam_PostCommit_XVR_Flink](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Flink.yml) | +| [ PostCommit XVR JavaUsingPython Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml) | N/A |`Run XVR_JavaUsingPython_Dataflow PostCommit`| [![.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml) | +| [ PostCommit XVR PythonUsingJava Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml) | N/A |`Run XVR_PythonUsingJava_Dataflow PostCommit`| [![.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml) | +| [ PostCommit XVR PythonUsingJavaSQL Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml) | N/A |`Run XVR_PythonUsingJavaSQL_Dataflow PostCommit`| [![.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml) | +| [ PostCommit XVR Samza ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Samza.yml) | N/A |`Run XVR_Samza PostCommit`| [![.github/workflows/beam_PostCommit_XVR_Samza](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Samza.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Samza.yml) | +| [ PostCommit XVR Spark3 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Spark3.yml) | N/A |`Run XVR_Spark3 PostCommit`| [![.github/workflows/beam_PostCommit_XVR_Spark3](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Spark3.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Spark3.yml) | +| [ PreCommit Community Metrics ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_CommunityMetrics.yml) | N/A |`Run CommunityMetrics PreCommit`| [![.github/workflows/beam_PreCommit_CommunityMetrics.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_CommunityMetrics.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_CommunityMetrics.yml) | +| [ PreCommit Go ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Go.yml) | N/A |`Run Go PreCommit`| [![.github/workflows/beam_PreCommit_Go.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Go.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Go.yml) | +| [ PreCommit Java ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java.yml) | N/A |`Run Java PreCommit`| [![.github/workflows/beam_PreCommit_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java.yml) | +| [ PreCommit Java Amazon Web Services IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Amazon-Web-Services_IO_Direct.yml) | N/A |`Run Java_Amazon-Web-Services_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Amazon-Web-Services_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Amazon-Web-Services_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Amazon-Web-Services_IO_Direct.yml) | +| [ PreCommit Java Amazon Web Services2 IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct.yml) | N/A |`Run Java_Amazon-Web-Services2_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct.yml) | +| [ PreCommit Java Amqp IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Amqp_IO_Direct.yml) | N/A |`Run Java_Amqp_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Amqp_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Amqp_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Amqp_IO_Direct.yml) | +| [ PreCommit Java Azure IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Azure_IO_Direct.yml) | N/A |`Run Java_Azure_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Azure_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Azure_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Azure_IO_Direct.yml) | +| [ PreCommit Java Cassandra IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Cassandra_IO_Direct.yml) | N/A |`Run Java_Cassandra_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Cassandra_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Cassandra_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Cassandra_IO_Direct.yml) | +| [ PreCommit Java Cdap IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Cdap_IO_Direct.yml) | N/A |`Run Java_Cdap_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Cdap_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Cdap_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Cdap_IO_Direct.yml) | +| [ PreCommit Java Clickhouse IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml) | N/A |`Run Java_Clickhouse_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml) | +| [ PreCommit Java Csv IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml) | N/A |`Run Java_Csv_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml) | +| [ PreCommit Java Debezium IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml) | N/A |`Run Java_Debezium_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml) | +| [ PreCommit Java ElasticSearch IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml) | N/A |`Run Java_ElasticSearch_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml) | +| [ PreCommit Java Examples Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow.yml) | N/A |`Run Java_Examples_Dataflow PreCommit`| [![.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow.yml) | +| [ PreCommit Java Flink Versions ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Flink_Versions.yml) | N/A |`Run Java_Flink_Versions PreCommit`| [![.github/workflows/beam_PreCommit_Java_Flink_Versions.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Flink_Versions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Flink_Versions.yml) | +| [ PreCommit Java GCP IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml) | N/A |`Run Java_GCP_IO_Direct PreCommit`| [![.github\workflows\beam_PreCommit_Java_GCP_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml) | +| [ PreCommit Java Examples Dataflow Java11 ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml) | N/A | `Run Java_Examples_Dataflow_Java11 PreCommit` | [![.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml) | +| [ PreCommit Java Examples Dataflow Java17 ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java17.yml) | N/A | `Run Java_Examples_Dataflow_Java17 PreCommit` | [![.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java17.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java17.yml) | +| [ PreCommit Java File-schema-transform IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml) | N/A |`Run Java_File-schema-transform_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml) | +| [ PreCommit Java Hadoop IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Hadoop_IO_Direct.yml) | N/A |`Run Java_Hadoop_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Hadoop_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Hadoop_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Hadoop_IO_Direct.yml) | +| [ PreCommit Java HBase IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_HBase_IO_Direct.yml) | N/A |`Run Java_HBase_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_HBase_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_HBase_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_HBase_IO_Direct.yml) | +| [ PreCommit Java HCatalog IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml) | N/A |`Run Java_HCatalog_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml) | +| [ PreCommit Java Kafka IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Kafka_IO_Direct.yml) | N/A |`Run Java_Kafka_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Kafka_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Kafka_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Kafka_IO_Direct.yml) | +| [ PreCommit Java InfluxDb IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_InfluxDb_IO_Direct.yml) | N/A |`Run Java_InfluxDb_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_InfluxDb_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_InfluxDb_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_InfluxDb_IO_Direct.yml) | +| [ PreCommit Java IOs Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_IOs_Direct.yml) | N/A |`Run Java_IOs_Direct PreCommit`| N/A | +| [ PreCommit Java JDBC IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_JDBC_IO_Direct.yml) | N/A |`Run Java_JDBC_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_JDBC_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_JDBC_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_JDBC_IO_Direct.yml) | +| [ PreCommit Java Jms IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Jms_IO_Direct.yml) | N/A |`Run Java_Jms_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Jms_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Jms_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Jms_IO_Direct.yml) | +| [ PreCommit Java Kinesis IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Kinesis_IO_Direct.yml) | N/A |`Run Java_Kinesis_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Kinesis_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Kinesis_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Kinesis_IO_Direct.yml) | +| [ PreCommit Java Kudu IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Kudu_IO_Direct.yml) | N/A |`Run Java_Kudu_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Kudu_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Kudu_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Kudu_IO_Direct.yml) | +| [ PreCommit Java MongoDb IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_MongoDb_IO_Direct.yml) | N/A |`Run Java_MongoDb_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_MongoDb_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_MongoDb_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_MongoDb_IO_Direct.yml) | +| [ PreCommit Java Mqtt IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Mqtt_IO_Direct.yml) | N/A |`Run Java_Mqtt_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Mqtt_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Mqtt_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Mqtt_IO_Direct.yml) | +| [ PreCommit Java Neo4j IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Neo4j_IO_Direct.yml) | N/A |`Run Java_Neo4j_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Neo4j_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Neo4j_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Neo4j_IO_Direct.yml) | +| [ PreCommit Java Parquet IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Parquet_IO_Direct.yml) | N/A |`Run Java_Parquet_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Parquet_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Parquet_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Parquet_IO_Direct.yml) | +| [ PreCommit Java Pulsar IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Pulsar_IO_Direct.yml) | N/A |`Run Java_Pulsar_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Pulsar_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Pulsar_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Pulsar_IO_Direct.yml) | +| [ PreCommit Java PVR Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_PVR_Flink_Batch.yml) | N/A |`Run Java_PVR_Flink_Batch PreCommit`| [![.github/workflows/beam_PreCommit_Java_PVR_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_PVR_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_PVR_Flink_Batch.yml) | +| [ PreCommit Java PVR Flink Docker ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_PVR_Flink_Docker.yml) | N/A |`Run Java_PVR_Flink_Docker PreCommit`| [![.github/workflows/beam_PreCommit_Java_PVR_Flink_Docker.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_PVR_Flink_Docker.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_PVR_Flink_Docker.yml) | +| [ PreCommit Java RabbitMq IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_RabbitMq_IO_Direct.yml) | N/A |`Run Java_RabbitMq_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_RabbitMq_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_RabbitMq_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_RabbitMq_IO_Direct.yml) | +| [ PreCommit Java Redis IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Redis_IO_Direct.yml) | N/A |`Run Java_Redis_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Redis_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Redis_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Redis_IO_Direct.yml) | +| [ PreCommit Java SingleStore IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml) | N/A |`Run Java_SingleStore_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml) | +| [ PreCommit Java Snowflake IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml) | N/A |`Run Java_Snowflake_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml) | +| [ PreCommit Java Solr IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml) | N/A |`Run Java_Solr_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml) | +| [ PreCommit Java Spark3 Versions ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark3_Versions.yml) | N/A | `Run Java_Spark3_Versions PreCommit` | [![.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark3_Versions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark3_Versions.yml) | +| [ PreCommit Java Splunk IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml) | N/A |`Run Java_Splunk_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml) | +| [ PreCommit Java Thrift IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml) | N/A |`Run Java_Thrift_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml) | +| [ PreCommit Java Tika IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml) | N/A |`Run Java_Tika_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml) | +| [ PreCommit Python ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python.yml) | ['3.8','3.9','3.10','3.11'] | `Run Python PreCommit (matrix_element)` | [![.github/workflows/beam_PreCommit_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python.yml) | +| [ PreCommit Python Coverage ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Coverage.yml) | N/A | `Run Python_Coverage PreCommit`| [![.github/workflows/beam_PreCommit_Python_Coverage.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Coverage.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Coverage.yml) | +| [ PreCommit Python Dataframes ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Dataframes.yml) | ['3.8','3.9','3.10','3.11'] | `Run Python_Dataframes PreCommit (matrix_element)`| [![.github/workflows/beam_PreCommit_Python_Dataframes.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Dataframes.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Dataframes.yml) | +| [ PreCommit Python Docker ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_PythonDocker.yml) | ['3.8','3.9','3.10','3.11'] | `Run PythonDocker PreCommit (matrix_element)`| [![.github/workflows/beam_PreCommit_PythonDocker.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_PythonDocker.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_PythonDocker.yml) | +| [ PreCommit Python Docs ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_PythonDocs.yml) | N/A | `Run PythonDocs PreCommit`| [![.github/workflows/beam_PreCommit_PythonDocs.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_PythonDocs.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_PythonDocs.yml) | +| [ PreCommit Python Examples ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Examples.yml) | ['3.8','3.9','3.10','3.11'] | `Run Python_Examples PreCommit (matrix_element)` | [![.github/workflows/beam_PreCommit_Python_Examples.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Examples.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Examples.yml) | +| [ PreCommit Python Formatter ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_PythonFormatter.yml) | N/A | `Run PythonFormatter PreCommit`| [![.github/workflows/beam_PreCommit_PythonFormatter.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_PythonFormatter.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_PythonFormatter.yml) | +| [ PreCommit Python Integration](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Integration.yml) | ['3.8','3.11'] | `Run Python_Integration PreCommit (matrix_element)` | [![.github/workflows/beam_PreCommit_Python_Integration.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Integration.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Integration.yml) | +| [ PreCommit Python Lint ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_PythonLint.yml) | N/A | `Run PythonLint PreCommit` | [![.github/workflows/beam_PreCommit_PythonLint.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_PythonLint.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_PythonLint.yml) | +| [ PreCommit Python PVR Flink ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_PVR_Flink.yml) | N/A | `Run Python_PVR_Flink PreCommit` | [![.github/workflows/beam_PreCommit_Python_PVR_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_PVR_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_PVR_Flink.yml) | +| [ PreCommit Python Runners ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Runners.yml) | ['3.8','3.9','3.10','3.11'] | `Run Python_Runners PreCommit (matrix_element)`| [![.github/workflows/beam_PreCommit_Python_Runners.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Runners.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Runners.yml) | +| [ PreCommit Python Transforms ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Transforms.yml) | ['3.8','3.9','3.10','3.11'] | `Run Python_Transforms PreCommit (matrix_element)`| [![.github/workflows/beam_PreCommit_Python_Transforms.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Transforms.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Python_Transforms.yml) | +| [ PreCommit RAT ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_RAT.yml) | N/A | `Run RAT PreCommit` | [![.github/workflows/beam_PreCommit_RAT.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_RAT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_RAT.yml) | +| [ PreCommit Spotless ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Spotless.yml) | N/A | `Run Spotless PreCommit` | [![.github/workflows/beam_PreCommit_Spotless.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Spotless.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Spotless.yml) | +| [ PreCommit SQL ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL.yml) | N/A |`Run SQL PreCommit`| [![.github/workflows/beam_PreCommit_SQL.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL.yml) | +| [ PreCommit SQL Java11 ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java11.yml) | N/A |`Run SQL_Java11 PreCommit`| [![.github/workflows/beam_PreCommit_SQL_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java11.yml) | +| [ PreCommit SQL Java17 ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java17.yml) | N/A |`Run SQL_Java17 PreCommit`| [![.github/workflows/beam_PreCommit_SQL_Java17.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java17.yml) | +| [ PreCommit Typescript ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Typescript.yml) | N/A |`Run Typescript PreCommit`| [![.github/workflows/beam_PreCommit_Typescript.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Typescript.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Typescript.yml) | +| [ PreCommit Website ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website.yml) | N/A |`Run Website PreCommit`| [![.github/workflows/beam_PreCommit_Website.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website.yml) | +| [ PreCommit Website Stage GCS ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website_Stage_GCS.yml) | N/A |`Run Website_Stage_GCS PreCommit`| [![PreCommit Website Stage GCS](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website_Stage_GCS.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website_Stage_GCS.yml) | +| [ PreCommit Whitespace ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Whitespace.yml) | N/A |`Run Whitespace PreCommit`| [![.github/workflows/beam_PreCommit_Whitespace.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Whitespace.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Whitespace.yml) | +| [ Python Validates Container Dataflow ARM ](https://github.com/apache/beam/actions/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml) | ['3.8','3.9','3.10','3.11'] | `Run Python ValidatesContainer Dataflow ARM (matrix_element)`| [![.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml](https://github.com/apache/beam/actions/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml) | +| [ PreCommit GoPortable ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_GoPortable.yml) | N/A |`Run GoPortable PreCommit`| [![.github/workflows/beam_PreCommit_GoPortable.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_GoPortable.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_GoPortable.yml) | +| [ PreCommit Kotlin Examples ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Kotlin_Examples.yml) | N/A | `Run Kotlin_Examples PreCommit` | [![.github/workflows/beam_PreCommit_Kotlin_Examples.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Kotlin_Examples.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Kotlin_Examples.yml) | +| [ PreCommit Portable Python ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Portable_Python.yml) | ['3.8','3.11'] | `Run Portable_Python PreCommit` | [![.github/workflows/beam_PreCommit_Portable_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Portable_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Portable_Python.yml) | +| [ Cancel Stale Dataflow Jobs ](https://github.com/apache/beam/actions/workflows/beam_CancelStaleDataflowJobs.yml) | N/A | `Run Cancel Stale Dataflow Jobs` | [![.github/workflows/beam_CancelStaleDataflowJobs.yml](https://github.com/apache/beam/actions/workflows/beam_CancelStaleDataflowJobs.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_CancelStaleDataflowJobs.yml) | +| [ Clean Up GCP Resources ](https://github.com/apache/beam/actions/workflows/beam_CleanUpGCPResources.yml) | N/A | `Run Clean GCP Resources` | [![.github/workflows/beam_CleanUpGCPResources.yml](https://github.com/apache/beam/actions/workflows/beam_CleanUpGCPResources.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_CleanUpGCPResources.yml) | +| [ Clean Up Prebuilt SDK Images ](https://github.com/apache/beam/actions/workflows/beam_CleanUpPrebuiltSDKImages.yml) | N/A | `Run Clean Prebuilt Images` | [![.github/workflows/beam_beam_CleanUpPrebuiltSDKImages.yml](https://github.com/apache/beam/actions/workflows/beam_CleanUpPrebuiltSDKImages.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_CleanUpPrebuiltSDKImages.yml) | diff --git a/.github/workflows/assign_milestone.yml b/.github/workflows/assign_milestone.yml index 9be68ecd3d0f1..17cc167ebcbbb 100644 --- a/.github/workflows/assign_milestone.yml +++ b/.github/workflows/assign_milestone.yml @@ -31,7 +31,7 @@ jobs: issues: write runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 2 diff --git a/.github/workflows/beam_CancelStaleDataflowJobs.yml b/.github/workflows/beam_CancelStaleDataflowJobs.yml new file mode 100644 index 0000000000000..63e780c2fef53 --- /dev/null +++ b/.github/workflows/beam_CancelStaleDataflowJobs.yml @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Cancel Stale Dataflow Jobs + +on: + schedule: + - cron: '0 */4 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +jobs: + beam_CancelStaleDataflowJobs: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + strategy: + matrix: + job_name: [beam_CancelStaleDataflowJobs] + job_phrase: [Run Cancel Stale Dataflow Jobs] + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Cancel Stale Dataflow Jobs' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: run cancel stale dataflow jobs + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :beam-test-tools:cancelStaleDataflowJobs + diff --git a/.github/workflows/beam_CleanUpDataprocResources.yml b/.github/workflows/beam_CleanUpDataprocResources.yml new file mode 100644 index 0000000000000..b6081e1891e6e --- /dev/null +++ b/.github/workflows/beam_CleanUpDataprocResources.yml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Cleanup Dataproc Resources + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_CleanUpDataprocResources: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: "beam_CleanUpDataprocResources" + steps: + - uses: actions/checkout@v3 + - name: Delete leaked resources for all the jobs that generates flink clusters + run: | + cd ${{ github.workspace }}/.test-infra/dataproc; ./cleanup.sh -xe \ No newline at end of file diff --git a/.github/workflows/beam_CleanUpGCPResources.yml b/.github/workflows/beam_CleanUpGCPResources.yml new file mode 100644 index 0000000000000..9aa92f0003c42 --- /dev/null +++ b/.github/workflows/beam_CleanUpGCPResources.yml @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Clean Up GCP Resources + +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +jobs: + beam_CleanUpGCPResources: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + strategy: + matrix: + job_name: [beam_CleanUpGCPResources] + job_phrase: [Run Clean GCP Resources] + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Clean GCP Resources' + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: run cleanup GCP resources + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :beam-test-tools:cleanupOtherStaleResources \ No newline at end of file diff --git a/.github/workflows/beam_CleanUpPrebuiltSDKImages.yml b/.github/workflows/beam_CleanUpPrebuiltSDKImages.yml new file mode 100644 index 0000000000000..345624f063bb2 --- /dev/null +++ b/.github/workflows/beam_CleanUpPrebuiltSDKImages.yml @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Clean Up Prebuilt SDK Images + +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +jobs: + beam_CleanUpPrebuiltSDKImages: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + strategy: + matrix: + job_name: [beam_CleanUpPrebuiltSDKImages] + job_phrase: [Run Clean Prebuilt Images] + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Clean Prebuilt Images' + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: run remove stale sdk container images + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :beam-test-tools:removeStaleSDKContainerImages \ No newline at end of file diff --git a/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml b/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml new file mode 100644 index 0000000000000..c34d9b5118e64 --- /dev/null +++ b/.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: CloudML Benchmarks Dataflow + +on: + issue_comment: + types: [created] + schedule: + - cron: '10 21 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.body || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_CloudML_Benchmarks_Dataflow: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run TFT Criteo Benchmarks' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 360 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_CloudML_Benchmarks_Dataflow"] + job_phrase: ["Run TFT Criteo Benchmarks"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup Python environment + uses: ./.github/actions/setup-environment-action + with: + python-version: | + 3.8 + 3.9 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/beam_CloudML_Benchmarks_Dataflow_arguments.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run TFT Criteo Benchmarks + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:dataflow:tftTests + arguments: | + -PpythonVersion=3.9 \ + -Prunner=DataflowRunner \ + '-Popts=${{ env.beam_CloudML_Benchmarks_Dataflow_test_arguments_1 }}' \ No newline at end of file diff --git a/.github/workflows/beam_IODatastoresCredentialsRotation.yml b/.github/workflows/beam_IODatastoresCredentialsRotation.yml new file mode 100644 index 0000000000000..36e6b238cdfc5 --- /dev/null +++ b/.github/workflows/beam_IODatastoresCredentialsRotation.yml @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Rotate IO-Datastores Cluster Credentials + +on: + schedule: + - cron: '0 2 1 * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_IODatastoresCredentialsRotation: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} + strategy: + matrix: + job_name: ["beam_IODatastoresCredentialsRotation"] + job_phrase: ["N/A"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} + - name: Starting credential rotation + run: | + gcloud container clusters update io-datastores --start-credential-rotation --zone=us-central1-a --quiet + - name: Rebuilding the nodes + run: | + gcloud container clusters upgrade io-datastores --node-pool=pool-1 --zone=us-central1-a --quiet + - name: Completing the rotation + run: | + gcloud container clusters update io-datastores --complete-credential-rotation --zone=us-central1-a --quiet +# TODO: Send email to dev@beam.apache.org if something went wrong during credentials rotation \ No newline at end of file diff --git a/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml b/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml new file mode 100644 index 0000000000000..97099ec525f13 --- /dev/null +++ b/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml @@ -0,0 +1,146 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Inference Python Benchmarks Dataflow + +on: + issue_comment: + types: [created] + schedule: + - cron: '50 3 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} + INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} + +jobs: + beam_Inference_Python_Benchmarks_Dataflow: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Inference Benchmarks' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 900 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_Inference_Python_Benchmarks_Dataflow"] + job_phrase: ["Run Inference Benchmarks"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup Python environment + uses: ./.github/actions/setup-environment-action + with: + python-version: '3.8' + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Vision_Classification_Resnet_101.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Base_Uncased.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Large_Uncased.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152_Tesla_T4_GPU.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: get current time + run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV + - name: run Pytorch Vision Classification with Resnet 101 + uses: ./.github/actions/gradle-command-self-hosted-action + timeout-minutes: 180 + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + -PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \ + '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_1 }} --job_name=benchmark-tests-pytorch-imagenet-python-101-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet101-${{env.NOW_UTC}}.txt' \ + - name: run Pytorch Imagenet Classification with Resnet 152 + uses: ./.github/actions/gradle-command-self-hosted-action + timeout-minutes: 180 + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + -PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \ + '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_2 }} --job_name=benchmark-tests-pytorch-imagenet-python-152-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet152-${{env.NOW_UTC}}.txt' \ + - name: run Pytorch Language Modeling using Hugging face bert-base-uncased model + uses: ./.github/actions/gradle-command-self-hosted-action + timeout-minutes: 180 + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_language_modeling_benchmarks \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + -PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \ + '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_3 }} --job_name=benchmark-tests-pytorch-language-modeling-bert-base-uncased-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_bert_base_uncased-${{env.NOW_UTC}}.txt' \ + - name: run Pytorch Langauge Modeling using Hugging Face bert-large-uncased model + uses: ./.github/actions/gradle-command-self-hosted-action + timeout-minutes: 180 + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_language_modeling_benchmarks \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + -PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \ + '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_4 }} --job_name=benchmark-tests-pytorch-language-modeling-bert-large-uncased-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_bert_large_uncased-${{env.NOW_UTC}}.txt' \ + - name: run Pytorch Imagenet Classification with Resnet 152 with Tesla T4 GPU + uses: ./.github/actions/gradle-command-self-hosted-action + timeout-minutes: 180 + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + -PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \ + '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_5 }} --job_name=benchmark-tests-pytorch-imagenet-python-gpu-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet152_gpu-${{env.NOW_UTC}}.txt' \ No newline at end of file diff --git a/.github/workflows/beam_Java_JMH.yml b/.github/workflows/beam_Java_JMH.yml new file mode 100644 index 0000000000000..07beb1dadba1a --- /dev/null +++ b/.github/workflows/beam_Java_JMH.yml @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Java JMH + +on: + schedule: + - cron: '0 0 * * 0' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + INFLUXDB_HOST: http://10.128.0.96:8086 + INFLUXDB_DATABASE: beam_test_metrics + +jobs: + beam_Java_JMH: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 900 + name: "beam_Java_JMH" + steps: + - uses: actions/checkout@v3 + - name: run the Java JMH micro-benchmark harness suite + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:harness:jmh:jmh + - name: run the Java JMH micro-benchmark core suite + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:core:jmh:jmh \ No newline at end of file diff --git a/.github/workflows/beam_Java_LoadTests_Combine_Smoke.yml b/.github/workflows/beam_Java_LoadTests_Combine_Smoke.yml new file mode 100644 index 0000000000000..6660caab2e3f2 --- /dev/null +++ b/.github/workflows/beam_Java_LoadTests_Combine_Smoke.yml @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java Combine Smoke + +on: + # issue_comment: + # types: [created] + # schedule: + # - cron: '1 1 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_Java_LoadTests_Combine_Smoke: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java Load Tests Combine Smoke' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_Java_LoadTests_Combine_Smoke"] + job_phrase: ["Run Java Load Tests Combine Smoke"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-1.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-2.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-3.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run CombineLoadTest load test Dataflow-1 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CombineLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_Java_LoadTests_Combine_Smoke_test_arguments_1 }}' \ + - name: run CombineLoadTest load test Dataflow-2 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CombineLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_Java_LoadTests_Combine_Smoke_test_arguments_2 }}' \ + - name: run CombineLoadTest load test Dataflow-3 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CombineLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_Java_LoadTests_Combine_Smoke_test_arguments_3 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml new file mode 100644 index 0000000000000..f50408e09b439 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Load Tests CoGBK Dataflow Batch Go + +on: + schedule: + - cron: '40 23 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Go_CoGBK_Dataflow_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Go CoGBK Dataflow Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Go_CoGBK_Dataflow_Batch"] + job_phrase: ["Run Load Tests Go CoGBK Dataflow Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: go + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_SingleKey.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_MultipleKey.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_Reiteration_10KB.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_Reiteration_2MB.txt + arguments: | + --job_name=load-tests-go-dataflow-batch-cogbk-$(date '+%m%d%H%M%S' --utc) + - name: run CoGBK Dataflow Batch Go Load Test 1 (single key) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=cogbk \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_CoGBK_Dataflow_Batch_test_arguments_1 }}' \ + - name: run CoGBK Dataflow Batch Go Load Test 2 (multiple keys) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=cogbk \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_CoGBK_Dataflow_Batch_test_arguments_2 }}' \ + - name: run CoGBK Dataflow Batch Go Load Test 3 (reiterate 10KB) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=cogbk \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_CoGBK_Dataflow_Batch_test_arguments_3 }}' \ + - name: run CoGBK Dataflow Batch Go Load Test 4 (reiterate 2MB) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=cogbk \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_CoGBK_Dataflow_Batch_test_arguments_4 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml b/.github/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml new file mode 100644 index 0000000000000..49894879a915e --- /dev/null +++ b/.github/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml @@ -0,0 +1,127 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Go CoGBK Flink Batch + +on: + schedule: + - cron: '10 14 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GCLOUD_ZONE: us-central1-a + CLUSTER_NAME: beam-loadtests-go-cogbk-flink-batch-${{ github.run_id }} + GCS_BUCKET: gs://beam-flink-cluster + FLINK_DOWNLOAD_URL: https://archive.apache.org/dist/flink/flink-1.15.0/flink-1.15.0-bin-scala_2.12.tgz + HADOOP_DOWNLOAD_URL: https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar + FLINK_TASKMANAGER_SLOTS: 1 + DETACHED_MODE: true + HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest + JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink1.15_job_server:latest + ARTIFACTS_DIR: gs://beam-flink-cluster/beam-loadtests-go-cogbk-flink-batch-${{ github.run_id }} + +jobs: + beam_LoadTests_Go_CoGBK_Flink_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Go CoGBK Flink Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Go_CoGBK_Flink_Batch"] + job_phrase: ["Run Load Tests Go CoGBK Flink Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: go + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_MultipleKey.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_Reiteration_10KB.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_Reiteration_2MB.txt + arguments: | + --job_name=load-tests-go-flink-batch-cogbk-$(date '+%m%d%H%M%S' --utc) + - name: Start Flink with parallelism 5 + env: + FLINK_NUM_WORKERS: 5 + run: | + cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh create + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run CoGBK Flink Batch Go Load Test 1 (multiple keys) + timeout-minutes: 240 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=cogbk \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_CoGBK_Flink_Batch_test_arguments_1 }}' \ + - name: run CoGBK Flink Batch Go Load Test 2 (reiterate 4 times 10KB values) + timeout-minutes: 240 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=cogbk \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_CoGBK_Flink_Batch_test_arguments_2 }}' \ + - name: run CoGBK Flink Batch Go Load Test 3 (reiterate 4 times 2MB values) + timeout-minutes: 240 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=cogbk \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_CoGBK_Flink_Batch_test_arguments_3 }}' + - name: Teardown Flink + if: always() + run: | + ${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Go_Combine_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Go_Combine_Dataflow_Batch.yml new file mode 100644 index 0000000000000..365f48fa1be24 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Go_Combine_Dataflow_Batch.yml @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Load Tests Combine Dataflow Batch Go + +on: + schedule: + - cron: '40 23 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Go_Combine_Dataflow_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Go Combine Dataflow Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Go_Combine_Dataflow_Batch"] + job_phrase: ["Run Load Tests Go Combine Dataflow Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare configs + #Reads config files, excludes comments, appends current date to the job_name parameter + id: set_configs + shell: bash + run: | + CURDATE=$(date '+%m%d%H%M%S' --utc) + CONFIG_ARR=('config_Combine_Go_Batch_10b.txt' 'config_Combine_Go_Batch_Fanout_4.txt' 'config_Combine_Go_Batch_Fanout_8.txt') + for INDEX in ${!CONFIG_ARR[@]} + do + CURCONFIG=$(grep -v "^#.*" ./.github/workflows/load-tests-job-configs/${CONFIG_ARR[INDEX]} | tr '\n' ' ') + CURCONFIG=$(echo "${CURCONFIG/load-tests-go-dataflow-batch-combine-$((INDEX + 1))-/load-tests-go-dataflow-batch-combine-$((INDEX + 1))-$CURDATE}") + echo "prepared_config_$((INDEX + 1))=$CURCONFIG" >> $GITHUB_OUTPUT + done + - name: run Combine Dataflow Batch Go Load Test 1 (single key) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=combine \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_1 }}' \ + - name: run Combine Dataflow Batch Go Load Test 2 (multiple keys) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=combine \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_2 }}' \ + - name: run Combine Dataflow Batch Go Load Test 3 (reiterate 10KB) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=combine \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_3 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Go_Combine_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Go_Combine_Flink_Batch.yml new file mode 100644 index 0000000000000..7cab058f5eeb4 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Go_Combine_Flink_Batch.yml @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Go Combine Flink Batch + +on: + schedule: + - cron: '40 6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GCLOUD_ZONE: us-central1-a + CLUSTER_NAME: beam-loadtests-go-combine-flink-batch-${{ github.run_id }} + GCS_BUCKET: gs://beam-flink-cluster + FLINK_DOWNLOAD_URL: https://archive.apache.org/dist/flink/flink-1.15.0/flink-1.15.0-bin-scala_2.12.tgz + HADOOP_DOWNLOAD_URL: https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar + FLINK_TASKMANAGER_SLOTS: 1 + DETACHED_MODE: true + HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest + JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink1.15_job_server:latest + ARTIFACTS_DIR: gs://beam-flink-cluster/beam-loadtests-go-combine-flink-batch-${{ github.run_id }} + +jobs: + beam_LoadTests_Go_Combine_Flink_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Go Combine Flink Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Go_Combine_Flink_Batch"] + job_phrase: ["Run Load Tests Go Combine Flink Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: go + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_10b.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_Fanout_8.txt + arguments: | + --job_name=load-tests-go-flink-batch-combine-$(date '+%m%d%H%M%S' --utc) + - name: Start Flink with parallelism 5 + env: + FLINK_NUM_WORKERS: 5 + run: | + cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh create + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run Combine Flink Batch Go Load Test 1 (10b records) + timeout-minutes: 240 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=combine \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_Combine_Flink_Batch_test_arguments_1 }}' \ + - name: Restart Flink with parallelism 16 + env: + FLINK_NUM_WORKERS: 16 + run: | + cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh restart + - name: run Combine Flink Batch Go Load Test 2 (fanout 4) + timeout-minutes: 240 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=combine \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_Combine_Flink_Batch_test_arguments_2 }}' \ + - name: run Combine Flink Batch Go Load Test 3 (fanout 8) + timeout-minutes: 240 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=combine \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_Combine_Flink_Batch_test_arguments_3 }}' + - name: Teardown Flink + if: always() + run: | + ${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml new file mode 100644 index 0000000000000..9656c14ed999f --- /dev/null +++ b/.github/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml @@ -0,0 +1,141 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Load Tests GBK Dataflow Batch Go + +on: + schedule: + - cron: '40 23 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Go_GBK_Dataflow_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Go GBK Dataflow Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Go_GBK_Dataflow_Batch"] + job_phrase: ["Run Load Tests Go GBK Dataflow Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: go + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_10b.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100b.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100kb.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_8.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_10KB.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_2MB.txt + - name: get current time + run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run GBK Dataflow Batch Go Load Test 1 (10 b records) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=group_by_key \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_GBK_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-go-dataflow-batch-gbk-1-${{env.NOW_UTC}}' \ + - name: run GBK Dataflow Batch Go Load Test 2 (100 b records) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=group_by_key \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_GBK_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-go-dataflow-batch-gbk-2-${{env.NOW_UTC}}' \ + - name: run GBK Dataflow Batch Go Load Test 3 (100 kb records) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=group_by_key \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_GBK_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-go-dataflow-batch-gbk-3-${{env.NOW_UTC}}' \ + - name: run GBK Dataflow Batch Go Load Test 4 (fanout 4) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=group_by_key \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_GBK_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-go-dataflow-batch-gbk-4-${{env.NOW_UTC}}' \ + - name: run GBK Dataflow Batch Go Load Test 5 (fanout 8) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=group_by_key \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_GBK_Dataflow_Batch_test_arguments_5 }} --job_name=load-tests-go-dataflow-batch-gbk-5-${{env.NOW_UTC}}' \ + - name: run GBK Dataflow Batch Go Load Test 6 (reiterate 4 times 10 kb) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=group_by_key \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_GBK_Dataflow_Batch_test_arguments_6 }} --job_name=load-tests-go-dataflow-batch-gbk-6-${{env.NOW_UTC}}' \ + - name: run GBK Dataflow Batch Go Load Test 7 (reiterate 4 times 2 mb) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=group_by_key \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_GBK_Dataflow_Batch_test_arguments_7 }} --job_name=load-tests-go-dataflow-batch-gbk-7-${{env.NOW_UTC}}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml new file mode 100644 index 0000000000000..41eb61efc580e --- /dev/null +++ b/.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml @@ -0,0 +1,162 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Go GBK Flink Batch + +on: + schedule: + - cron: '20 1 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GCLOUD_ZONE: us-central1-a + CLUSTER_NAME: beam-loadtests-go-gbk-flink-batch-${{ github.run_id }} + GCS_BUCKET: gs://beam-flink-cluster + FLINK_DOWNLOAD_URL: https://archive.apache.org/dist/flink/flink-1.15.0/flink-1.15.0-bin-scala_2.12.tgz + HADOOP_DOWNLOAD_URL: https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar + FLINK_TASKMANAGER_SLOTS: 1 + DETACHED_MODE: true + HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest + JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink1.15_job_server:latest + ARTIFACTS_DIR: gs://beam-flink-cluster/beam-loadtests-go-gbk-flink-batch-${{ github.run_id }} + +jobs: + beam_LoadTests_Go_GBK_Flink_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Go GBK Flink Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Go_GBK_Flink_Batch"] + job_phrase: ["Run Load Tests Go GBK Flink Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: go + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_10b.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_100b.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_100kb.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Fanout_8.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Reiteration_10KB.txt + arguments: | + --job_name=load-tests-go-flink-batch-gbk-$(date '+%m%d%H%M%S' --utc) + - name: Start Flink with parallelism 5 + env: + FLINK_NUM_WORKERS: 5 + run: | + cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh create + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run GBK Flink Batch Go Load Test 1 (10 b records) + timeout-minutes: 120 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=group_by_key \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_GBK_Flink_Batch_test_arguments_1 }}' \ + - name: run GBK Flink Batch Go Load Test 2 (100 b records) + timeout-minutes: 120 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=group_by_key \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_GBK_Flink_Batch_test_arguments_2 }}' \ + - name: run GBK Flink Batch Go Load Test 3 (100 kb records) + timeout-minutes: 120 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=group_by_key \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_GBK_Flink_Batch_test_arguments_3 }}' \ + - name: run GBK Flink Batch Go Load Test 6 (reiterate 4 times 10 kb) + timeout-minutes: 120 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=group_by_key \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_GBK_Flink_Batch_test_arguments_6 }}' \ + - name: Restart Flink with parallelism 16 + env: + FLINK_NUM_WORKERS: 16 + run: | + cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh restart + - name: run GBK Flink Batch Go Load Test 4 (fanout 4) + timeout-minutes: 120 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=group_by_key \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_GBK_Flink_Batch_test_arguments_4 }}' \ + - name: run GBK Flink Batch Go Load Test 5 (fanout 8) + timeout-minutes: 120 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=group_by_key \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_GBK_Flink_Batch_test_arguments_5 }}' \ + - name: Teardown Flink + if: always() + run: | + ${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch.yml new file mode 100644 index 0000000000000..c1aa013e6ede3 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch.yml @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Load Tests ParDo Dataflow Batch Go + +on: + schedule: + - cron: '15 18 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Go_ParDo_Dataflow_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Go ParDo Dataflow Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Go_ParDo_Dataflow_Batch"] + job_phrase: ["Run Load Tests Go ParDo Dataflow Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: go + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_10_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_200_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_10_Counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_100_Counters.txt + arguments: | + --job_name=load-tests-go-dataflow-batch-pardo-$(date '+%m%d%H%M%S' --utc) + - name: run ParDo Dataflow Batch Go Load Test 1 (10 iterations) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=pardo \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_ParDo_Dataflow_Batch_test_arguments_1 }}' \ + - name: run ParDo Dataflow Batch Go Load Test 2 (200 iterations) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=pardo \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_ParDo_Dataflow_Batch_test_arguments_2 }}' \ + - name: run ParDo Dataflow Batch Go Load Test 3 (10 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=pardo \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_ParDo_Dataflow_Batch_test_arguments_3 }}' \ + - name: run ParDo Dataflow Batch Go Load Test 4 (100 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=pardo \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_ParDo_Dataflow_Batch_test_arguments_4 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Go_ParDo_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Go_ParDo_Flink_Batch.yml new file mode 100644 index 0000000000000..a99422287a5b9 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Go_ParDo_Flink_Batch.yml @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Go ParDo Flink Batch + +on: + schedule: + - cron: '40 2 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GCLOUD_ZONE: us-central1-a + CLUSTER_NAME: beam-loadtests-go-pardo-flink-batch-${{ github.run_id }} + GCS_BUCKET: gs://beam-flink-cluster + FLINK_DOWNLOAD_URL: https://archive.apache.org/dist/flink/flink-1.15.0/flink-1.15.0-bin-scala_2.12.tgz + HADOOP_DOWNLOAD_URL: https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar + FLINK_TASKMANAGER_SLOTS: 1 + DETACHED_MODE: true + HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest + JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink1.15_job_server:latest + ARTIFACTS_DIR: gs://beam-flink-cluster/beam-loadtests-go-pardo-flink-batch-${{ github.run_id }} + +jobs: + beam_LoadTests_Go_ParDo_Flink_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Go ParDo Flink Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Go_ParDo_Flink_Batch"] + job_phrase: ["Run Load Tests Go ParDo Flink Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: go + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_10_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_200_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_10_counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_100_counters.txt + arguments: | + --job_name=load-tests-go-flink-batch-pardo-$(date '+%m%d%H%M%S' --utc) + - name: Start Flink with parallelism 5 + env: + FLINK_NUM_WORKERS: 5 + run: | + cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh create + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run ParDo Flink Batch Go Load Test 1 (10 times) + timeout-minutes: 180 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=pardo \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_ParDo_Flink_Batch_test_arguments_1 }}' \ + - name: run ParDo Flink Batch Go Load Test 2 (200 times) + timeout-minutes: 180 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=pardo \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_ParDo_Flink_Batch_test_arguments_2 }}' \ + - name: run ParDo Flink Batch Go Load Test 3 (10 counters) + timeout-minutes: 180 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=pardo \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_ParDo_Flink_Batch_test_arguments_3 }}' \ + - name: run ParDo Flink Batch Go Load Test 4 (100 counters) + timeout-minutes: 180 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=pardo \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_ParDo_Flink_Batch_test_arguments_4 }}' \ + - name: Teardown Flink + if: always() + run: | + ${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Go_SideInput_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Go_SideInput_Dataflow_Batch.yml new file mode 100644 index 0000000000000..38373e98a4ccb --- /dev/null +++ b/.github/workflows/beam_LoadTests_Go_SideInput_Dataflow_Batch.yml @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Load Tests SideInput Dataflow Batch Go + +on: + schedule: + - cron: '40 23 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Go_SideInput_Dataflow_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Go SideInput Dataflow Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Go_SideInput_Dataflow_Batch"] + job_phrase: ["Run Load Tests Go SideInput Dataflow Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: go + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_SideInput_Go_Batch_Dataflow_First_Iterable.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_SideInput_Go_Batch_Dataflow_Iterable.txt + arguments: | + --job_name=load-tests-go-dataflow-batch-sideinput-$(date '+%m%d%H%M%S' --utc) + - name: run SideInput Dataflow Batch Go Load Test 1 (first iterable) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=sideinput \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_SideInput_Dataflow_Batch_test_arguments_1 }}' \ + - name: run SideInput Dataflow Batch Go Load Test 2 (iterable) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=sideinput \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_SideInput_Dataflow_Batch_test_arguments_2 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Go_SideInput_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Go_SideInput_Flink_Batch.yml new file mode 100644 index 0000000000000..079aac5ad7f33 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Go_SideInput_Flink_Batch.yml @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Go SideInput Flink Batch + +on: + schedule: + - cron: '40 23 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GCLOUD_ZONE: us-central1-a + CLUSTER_NAME: beam-loadtests-go-sideinput-flink-batch-${{ github.run_id }} + GCS_BUCKET: gs://beam-flink-cluster + FLINK_DOWNLOAD_URL: https://archive.apache.org/dist/flink/flink-1.15.0/flink-1.15.0-bin-scala_2.12.tgz + HADOOP_DOWNLOAD_URL: https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar + FLINK_TASKMANAGER_SLOTS: 1 + DETACHED_MODE: true + HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest + JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink1.15_job_server:latest + ARTIFACTS_DIR: gs://beam-flink-cluster/beam-loadtests-go-sideinput-flink-batch-${{ github.run_id }} + +jobs: + beam_LoadTests_Go_SideInput_Flink_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Go SideInput Flink Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Go_SideInput_Flink_Batch"] + job_phrase: ["Run Load Tests Go SideInput Flink Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: go + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_SideInput_Flink_Batch_First_Iterable.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_SideInput_Flink_Batch_Iterable.txt + arguments: | + --job_name=load-tests-go-flink-batch-sideinput-$(date '+%m%d%H%M%S' --utc) + - name: Start Flink with parallelism 10 + env: + FLINK_NUM_WORKERS: 10 + run: | + cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh create + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run SideInput Flink Batch Go Load Test 1 (first iterable) + timeout-minutes: 240 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=sideinput \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_SideInput_Flink_Batch_test_arguments_1 }}' \ + - name: run SideInput Flink Batch Go Load Test 2 (iterable) + timeout-minutes: 240 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:load:run + arguments: | + -PloadTest.mainClass=sideinput \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Go_SideInput_Flink_Batch_test_arguments_2 }}' + - name: Teardown Flink + if: always() + run: | + ${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Batch.yml new file mode 100644 index 0000000000000..8c05e76a0fddd --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Batch.yml @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java CoGBK Dataflow Batch + +on: + issue_comment: + types: [created] + schedule: + - cron: '30 9 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Java_CoGBK_Dataflow_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Java CoGBK Dataflow Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Java_CoGBK_Dataflow_Batch"] + job_phrase: ["Run Load Tests Java CoGBK Dataflow Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_100b_Single_Key.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_10kB.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_2MB.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run CoGBK 2GB 100 byte records - single key + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_Batch_test_arguments_1 }} --appName=load_tests_Java_Dataflow_batch_CoGBK_1' \ + - name: run CoGBK 2GB 100 byte records - multiple keys + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_Batch_test_arguments_2 }} --appName=load_tests_Java_Dataflow_batch_CoGBK_2' \ + - name: run CoGBK 2GB reiteration 10kB value + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_Batch_test_arguments_3 }} --appName=load_tests_Java_Dataflow_batch_CoGBK_3' \ + - name: run CoGBK 2GB reiteration 2MB value + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_Batch_test_arguments_4 }} --appName=load_tests_Java_Dataflow_batch_CoGBK_4' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml new file mode 100644 index 0000000000000..91ba9639c9c8a --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Load Tests CoGBK Dataflow Streaming Java + +on: + schedule: + - cron: '50 10 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Java_CoGBK_Dataflow_Streaming: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Java CoGBK Dataflow Streaming' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: [ "beam_LoadTests_Java_CoGBK_Dataflow_Streaming" ] + job_phrase: ["Run Load Tests Java CoGBK Dataflow Streaming"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare configs + id: set_configs + shell: bash + run: | + CURCONFIG=$(grep -v "^#.*" ./.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_SingleKey.txt | tr '\n' ' ') + echo "prepared_config_1=$CURCONFIG" >> $GITHUB_OUTPUT + CURCONFIG=$(grep -v "^#.*" ./.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_MultipleKey.txt | tr '\n' ' ') + echo "prepared_config_2=$CURCONFIG" >> $GITHUB_OUTPUT + CURCONFIG=$(grep -v "^#.*" ./.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_Reiteration_10KB.txt | tr '\n' ' ') + echo "prepared_config_3=$CURCONFIG" >> $GITHUB_OUTPUT + CURCONFIG=$(grep -v "^#.*" ./.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_Reiteration_2MB.txt | tr '\n' ' ') + echo "prepared_config_4=$CURCONFIG" >> $GITHUB_OUTPUT + - name: run CoGBK Dataflow Streaming Java Load Test 1 (single key) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_1 }}' \ + - name: run CoGBK Dataflow Streaming Java Load Test 2 (multiple key) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_2 }}' \ + - name: run CoGBK Dataflow Streaming Java Load Test 3 (reiteration 10KB value) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_3 }}' \ + - name: run CoGBK Dataflow Streaming Java Load Test 4 (reiteration 2MB value) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_4 }}' \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml new file mode 100644 index 0000000000000..2960fafe6b32e --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java CoGBK Dataflow V2 Batch JavaVersions + +on: + issue_comment: + types: [created] + schedule: + - cron: '0 10 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + (contains(github.event.comment.body, 'Run Load Tests Java') && + contains(github.event.comment.body, 'CoGBK Dataflow V2 Batch')) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase_1 }} ${{ matrix.java_version }} ${{ matrix.job_phrase_2 }}) + strategy: + fail-fast: false + matrix: + job_name: ["beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions"] + job_phrase_1: ["Run Load Tests Java"] + job_phrase_2: ["CoGBK Dataflow V2 Batch"] + java_version: ['11','17'] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase_1 }} ${{ matrix.java_version }} ${{ matrix.job_phrase_2 }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase_1 }} ${{ matrix.java_version }} ${{ matrix.job_phrase_2 }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: ${{ matrix.java_version }} + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_100b_Single_Key.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_10kB.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_2MB.txt + arguments: | + --influxTags={\"runnerVersion\":\"v2\",\"jdk\":\"java${{ matrix.java_version }}\"} + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run CoGBK 2GB 100 byte records - single key + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions_test_arguments_1 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_CoGBK_1' \ + - name: run CoGBK 2GB 100 byte records - multiple keys + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions_test_arguments_2 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_CoGBK_2' \ + - name: run CoGBK 2GB reiteration 10kB value + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions_test_arguments_3 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_CoGBK_3' \ + - name: run CoGBK 2GB reiteration 2MB value + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions_test_arguments_4 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_CoGBK_4' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml new file mode 100644 index 0000000000000..113927dba3402 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java CoGBK Dataflow V2 Streaming JavaVersions + +on: + issue_comment: + types: [created] + schedule: + - cron: '30 10 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + (contains(github.event.comment.body, 'Run Load Tests Java') && + contains(github.event.comment.body, 'CoGBK Dataflow V2 Streaming')) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase_1 }} ${{ matrix.java_version }} ${{ matrix.job_phrase_2 }}) + strategy: + fail-fast: false + matrix: + job_name: ["beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions"] + job_phrase_1: ["Run Load Tests Java"] + job_phrase_2: ["CoGBK Dataflow V2 Streaming"] + java_version: ['11','17'] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase_1 }} ${{ matrix.java_version }} ${{ matrix.job_phrase_2 }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase_1 }} ${{ matrix.java_version }} ${{ matrix.job_phrase_2 }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: ${{ matrix.java_version }} + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Single_Key.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Multiple_Keys.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_10kB.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_2MB.txt + arguments: | + --influxTags={\"runnerVersion\":\"v2\",\"jdk\":\"java${{ matrix.java_version }}\"} + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run CoGBK 2GB 100 byte records - single key + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions_test_arguments_1 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_CoGBK_1' \ + - name: run CoGBK 2GB 100 byte records - multiple keys + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions_test_arguments_2 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_CoGBK_2' \ + - name: run CoGBK 2GB reiteration 10kB value + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions_test_arguments_3 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_CoGBK_3' \ + - name: run CoGBK 2GB reiteration 2MB value + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions_test_arguments_4 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_CoGBK_4' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch.yml b/.github/workflows/beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch.yml new file mode 100644 index 0000000000000..1c39a54e7ca1c --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch.yml @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java CoGBK SparkStructuredStreaming Batch + +on: + issue_comment: + types: [created] + schedule: + - cron: '0 11 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Java CoGBK SparkStructuredStreaming Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch"] + job_phrase: ["Run Load Tests Java CoGBK SparkStructuredStreaming Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_100b_Single_Key.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_100b_Multiple_Keys.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_10kB.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_2MB.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run CoGBK 2GB 100 byte records - single key + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch_test_arguments_1 }}' \ + - name: run CoGBK 2GB 100 byte records - multiple keys + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch_test_arguments_2 }}' \ + - name: run CoGBK 2GB reiteration 10kB value + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch_test_arguments_3 }}' \ + - name: run CoGBK 2GB reiteration 2MB value + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch_test_arguments_4 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_Combine_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Java_Combine_Dataflow_Batch.yml new file mode 100644 index 0000000000000..de317b4a87ca2 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_Combine_Dataflow_Batch.yml @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Load Tests Combine Dataflow Batch Java + +on: + schedule: + - cron: '35 7 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Java_Combine_Dataflow_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Java Combine Dataflow Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Java_Combine_Dataflow_Batch"] + job_phrase: ["Run Load Tests Java Combine Dataflow Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_10b.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_Fanout_8.txt + - name: run Combine Dataflow Batch Java Load Test 1 (10 b records) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CombineLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_Combine_Dataflow_Batch_test_arguments_1 }}' \ + - name: run Combine Dataflow Batch Java Load Test 2 (fanout 4) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CombineLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_Combine_Dataflow_Batch_test_arguments_2 }}' \ + - name: run Combine Dataflow Batch Java Load Test 3 (fanout 8) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CombineLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_Combine_Dataflow_Batch_test_arguments_3 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_Combine_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Java_Combine_Dataflow_Streaming.yml new file mode 100644 index 0000000000000..13b07f16b259a --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_Combine_Dataflow_Streaming.yml @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java Combine Dataflow Streaming + +on: + schedule: + - cron: '25 14 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_LoadTests_Java_Combine_Dataflow_Streaming_test_arguments_1: '' + beam_LoadTests_Java_Combine_Dataflow_Streaming_test_arguments_2: '' + beam_LoadTests_Java_Combine_Dataflow_Streaming_test_arguments_3: '' + +jobs: + beam_LoadTests_Java_Combine_Dataflow_Streaming: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Java Combine Dataflow Streaming' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Java_Combine_Dataflow_Streaming"] + job_phrase: ["Run Load Tests Java Combine Dataflow Streaming"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_10b.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_Fanout_8.txt + - name: run Combine Dataflow Streaming Java Load Test 1 (10 b records) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CombineLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_Combine_Dataflow_Streaming_test_arguments_1 }}' \ + - name: run Combine Dataflow Streaming Java Load Test 2 (fanout 4) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CombineLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_Combine_Dataflow_Streaming_test_arguments_2 }}' \ + - name: run Combine Dataflow Streaming Java Load Test 3 (fanout 8) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CombineLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_Combine_Dataflow_Streaming_test_arguments_3 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml b/.github/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml new file mode 100644 index 0000000000000..bae659d14d09e --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java Combine SparkStructuredStreaming Batch + +on: + schedule: + - cron: '15 18 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch_test_arguments_1: '' + beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch_test_arguments_2: '' + beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch_test_arguments_3: '' + +jobs: + beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Java Combine SparkStructuredStreaming Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch"] + job_phrase: ["Run Load Tests Java Combine SparkStructuredStreaming Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_10b.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_Fanout_8.txt + - name: run Combine SparkStructuredStreaming Batch Java Load Test 1 (10b records) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CombineLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch_test_arguments_1 }}' \ + - name: run Combine SparkStructuredStreaming Batch Java Load Test 2 (fanout 4) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CombineLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch_test_arguments_2 }}' \ + - name: run Combine SparkStructuredStreaming Batch Java Load Test 3 (fanout 8) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CombineLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch_test_arguments_3 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml new file mode 100644 index 0000000000000..e3aca91c1e769 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml @@ -0,0 +1,139 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java GBK Dataflow Batch + +on: + schedule: + - cron: '10 6 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Java_GBK_Dataflow_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Java GBK Dataflow Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Java_GBK_Dataflow_Batch"] + job_phrase: ["Run Load Tests Java GBK Dataflow Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_reiterate_4_times_2MB_values.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run Load test 2GB of 10B records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_Batch_test_arguments_1 }}' \ + - name: run Load test 2GB of 100B records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_Batch_test_arguments_2 }}' \ + - name: run Load test 2GB of 100kB records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_Batch_test_arguments_3 }}' \ + - name: run Load test fanout 4 times with 2GB 10-byte records total + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_Batch_test_arguments_4 }}' \ + - name: run Load test fanout 8 times with 2GB 10-byte records total + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_Batch_test_arguments_5 }}' \ + - name: run Load test reiterate 4 times 10kB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_Batch_test_arguments_6 }}' \ + - name: run Load test reiterate 4 times 2MB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_Batch_test_arguments_7 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml new file mode 100644 index 0000000000000..f28c7a147b947 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml @@ -0,0 +1,139 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java GBK Dataflow Streaming + +on: + schedule: + - cron: '50 6 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Java_GBK_Dataflow_Streaming: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Java GBK Dataflow Streaming' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Java_GBK_Dataflow_Streaming"] + job_phrase: ["Run Load Tests Java GBK Dataflow Streaming"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_reiterate_4_times_2MB_values.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run Load test 2GB of 10B records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_Streaming_test_arguments_1 }}' \ + - name: run Load test 2GB of 100B records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_Streaming_test_arguments_2 }}' \ + - name: run Load test 2GB of 100kB records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_Streaming_test_arguments_3 }}' \ + - name: run Load test fanout 4 times with 2GB 10-byte records total + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_Streaming_test_arguments_4 }}' \ + - name: run Load test fanout 8 times with 2GB 10-byte records total + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_Streaming_test_arguments_5 }}' \ + - name: run Load test reiterate 4 times 10kB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_Streaming_test_arguments_6 }}' \ + - name: run Load test reiterate 4 times 2MB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_Streaming_test_arguments_7 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml new file mode 100644 index 0000000000000..a7f60be0a2ce1 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml @@ -0,0 +1,164 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java GBK Dataflow V2 Batch Java11 + +on: + schedule: + - cron: '10 7 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Java 11 GBK Dataflow V2 Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11"] + job_phrase: ["Run Load Tests Java 11 GBK Dataflow V2 Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 11 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_2MB_values.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run Load test 2GB of 10B records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava11 \ + -Pjava11Home=$JAVA_HOME_11_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_1 }}' \ + - name: run Load test 2GB of 100B records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava11 \ + -Pjava11Home=$JAVA_HOME_11_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_2 }}' \ + - name: run Load test 2GB of 100kB records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava11 \ + -Pjava11Home=$JAVA_HOME_11_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_3 }}' \ + - name: run Load test fanout 4 times with 2GB 10-byte records total + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava11 \ + -Pjava11Home=$JAVA_HOME_11_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_4 }}' \ + - name: run Load test fanout 8 times with 2GB 10-byte records total + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava11 \ + -Pjava11Home=$JAVA_HOME_11_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_5 }}' \ + - name: run Load test reiterate 4 times 10kB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava11 \ + -Pjava11Home=$JAVA_HOME_11_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_6 }}' \ + - name: run Load test reiterate 4 times 2MB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava11 \ + -Pjava11Home=$JAVA_HOME_11_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_7 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml new file mode 100644 index 0000000000000..88de449d97291 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml @@ -0,0 +1,166 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java GBK Dataflow V2 Batch Java17 + +on: + schedule: + - cron: '50 7 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Java 17 GBK Dataflow V2 Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17"] + job_phrase: ["Run Load Tests Java 17 GBK Dataflow V2 Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: | + 17 + 8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_2MB_values.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run Load test 2GB of 10B records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava17 \ + -Pjava17Home=$JAVA_HOME_17_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_1 }}' \ + - name: run Load test 2GB of 100B records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava17 \ + -Pjava17Home=$JAVA_HOME_17_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_2 }}' \ + - name: run Load test 2GB of 100kB records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava17 \ + -Pjava17Home=$JAVA_HOME_17_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_3 }}' \ + - name: run Load test fanout 4 times with 2GB 10-byte records total + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava17 \ + -Pjava17Home=$JAVA_HOME_17_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_4 }}' \ + - name: run Load test fanout 8 times with 2GB 10-byte records total + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava17 \ + -Pjava17Home=$JAVA_HOME_17_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_5 }}' \ + - name: run Load test reiterate 4 times 10kB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava17 \ + -Pjava17Home=$JAVA_HOME_17_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_6 }}' \ + - name: run Load test reiterate 4 times 2MB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava17 \ + -Pjava17Home=$JAVA_HOME_17_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_7 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml new file mode 100644 index 0000000000000..39166855e39f7 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml @@ -0,0 +1,164 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java GBK Dataflow V2 Streaming Java11 + +on: + schedule: + - cron: '50 8 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Java 11 GBK Dataflow V2 Streaming' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11"] + job_phrase: ["Run Load Tests Java 11 GBK Dataflow V2 Streaming"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 11 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_2MB_values.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run Load test 2GB of 10B records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava11 \ + -Pjava11Home=$JAVA_HOME_11_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_1 }}' \ + - name: run Load test 2GB of 100B records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava11 \ + -Pjava11Home=$JAVA_HOME_11_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_2 }}' \ + - name: run Load test 2GB of 100kB records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava11 \ + -Pjava11Home=$JAVA_HOME_11_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_3 }}' \ + - name: run Load test fanout 4 times with 2GB 10-byte records total + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava11 \ + -Pjava11Home=$JAVA_HOME_11_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_4 }}' \ + - name: run Load test fanout 8 times with 2GB 10-byte records total + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava11 \ + -Pjava11Home=$JAVA_HOME_11_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_5 }}' \ + - name: run Load test reiterate 4 times 10kB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava11 \ + -Pjava11Home=$JAVA_HOME_11_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_6 }}' \ + - name: run Load test reiterate 4 times 2MB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava11 \ + -Pjava11Home=$JAVA_HOME_11_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_7 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml new file mode 100644 index 0000000000000..4f76b3f45eb95 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml @@ -0,0 +1,166 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java GBK Dataflow V2 Streaming Java17 + +on: + schedule: + - cron: '50 9 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Java 17 GBK Dataflow V2 Streaming' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17"] + job_phrase: ["Run Load Tests Java 17 GBK Dataflow V2 Streaming"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: | + 17 + 8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_2MB_values.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run Load test 2GB of 10B records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava17 \ + -Pjava17Home=$JAVA_HOME_17_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_1 }}' \ + - name: run Load test 2GB of 100B records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava17 \ + -Pjava17Home=$JAVA_HOME_17_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_2 }}' \ + - name: run Load test 2GB of 100kB records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava17 \ + -Pjava17Home=$JAVA_HOME_17_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_3 }}' \ + - name: run Load test fanout 4 times with 2GB 10-byte records total + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava17 \ + -Pjava17Home=$JAVA_HOME_17_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_4 }}' \ + - name: run Load test fanout 8 times with 2GB 10-byte records total + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava17 \ + -Pjava17Home=$JAVA_HOME_17_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_5 }}' \ + - name: run Load test reiterate 4 times 10kB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava17 \ + -Pjava17Home=$JAVA_HOME_17_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_6 }}' \ + - name: run Load test reiterate 4 times 2MB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava17 \ + -Pjava17Home=$JAVA_HOME_17_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_7 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Smoke.yml b/.github/workflows/beam_LoadTests_Java_GBK_Smoke.yml new file mode 100644 index 0000000000000..59cdad972d780 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_GBK_Smoke.yml @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java GBK Smoke + +on: + issue_comment: + types: [created] + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.comment.body || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Java_GBK_Smoke: + if: | + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java Load Tests GBK Smoke' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Java_GBK_Smoke"] + job_phrase: ["Run Java Load Tests GBK Smoke"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Direct.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Dataflow.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Flink.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Spark.txt + - name: Set current datetime + id: datetime + run: | + echo "datetime=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_OUTPUT + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run GroupByKey load test Direct + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:direct-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Smoke_test_arguments_1 }}' \ + - name: run GroupByKey load test Dataflow + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Smoke_test_arguments_2 }}' \ + - name: run GroupByKey load test Flink + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + --info \ + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:flink:1.15 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Smoke_test_arguments_3 }}' \ + - name: run GroupByKey load test Spark + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Smoke_test_arguments_4 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml b/.github/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml new file mode 100644 index 0000000000000..13fe4df9906d9 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml @@ -0,0 +1,139 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java GBK SparkStructuredStreaming Batch + +on: + schedule: + - cron: '10 10 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Java GBK SparkStructuredStreaming Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch"] + job_phrase: ["Run Load Tests Java GBK SparkStructuredStreaming Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_2MB_values.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run Load test 2GB of 10B records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch_test_arguments_1 }}' \ + - name: run Load test 2GB of 100B records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch_test_arguments_2 }}' \ + - name: run Load test 2GB of 100kB records + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch_test_arguments_3 }}' \ + - name: run Load test fanout 4 times with 2GB 10-byte records total + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch_test_arguments_4 }}' \ + - name: run Load test fanout 8 times with 2GB 10-byte records total + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch_test_arguments_5 }}' \ + - name: run Load test reiterate 4 times 10kB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch_test_arguments_6 }}' \ + - name: run Load test reiterate 4 times 2MB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch_test_arguments_7 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_Batch.yml new file mode 100644 index 0000000000000..1434521f7c28a --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_Batch.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java ParDo Dataflow Batch + +on: + schedule: + - cron: '55 9 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_LoadTests_Java_ParDo_Dataflow_Batch_test_arguments_1: '' + beam_LoadTests_Java_ParDo_Dataflow_Batch_test_arguments_2: '' + beam_LoadTests_Java_ParDo_Dataflow_Batch_test_arguments_3: '' + beam_LoadTests_Java_ParDo_Dataflow_Batch_test_arguments_4: '' + +jobs: + beam_LoadTests_Java_ParDo_Dataflow_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Java ParDo Dataflow Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Java_ParDo_Dataflow_Batch"] + job_phrase: ["Run Load Tests Java ParDo Dataflow Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_10_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_200_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_10_counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_100_counters.txt + - name: run ParDo Dataflow Batch Java Load Test 1 (10 times) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_Batch_test_arguments_1 }}' \ + - name: run ParDo Dataflow Batch Java Load Test 2 (200 times) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_Batch_test_arguments_2 }}' \ + - name: run ParDo Dataflow Batch Java Load Test 3 (10 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_Batch_test_arguments_3 }}' \ + - name: run ParDo Dataflow Batch Java Load Test 4 (100 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_Batch_test_arguments_4 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_Streaming.yml new file mode 100644 index 0000000000000..aabc12c366b51 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_Streaming.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java ParDo Dataflow Streaming + +on: + schedule: + - cron: '10 11 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_LoadTests_Java_ParDo_Dataflow_Streaming_test_arguments_1: '' + beam_LoadTests_Java_ParDo_Dataflow_Streaming_test_arguments_2: '' + beam_LoadTests_Java_ParDo_Dataflow_Streaming_test_arguments_3: '' + beam_LoadTests_Java_ParDo_Dataflow_Streaming_test_arguments_4: '' + +jobs: + beam_LoadTests_Java_ParDo_Dataflow_Streaming: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Java ParDo Dataflow Streaming' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Java_ParDo_Dataflow_Streaming"] + job_phrase: ["Run Load Tests Java ParDo Dataflow Streaming"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_10_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_200_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_10_counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_100_counters.txt + - name: run ParDo Dataflow Streaming Java Load Test 1 (10 times) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_Streaming_test_arguments_1 }}' \ + - name: run ParDo Dataflow Streaming Java Load Test 2 (200 times) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_Streaming_test_arguments_2 }}' \ + - name: run ParDo Dataflow Streaming Java Load Test 3 (10 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_Streaming_test_arguments_3 }}' \ + - name: run ParDo Dataflow Streaming Java Load Test 4 (100 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_Streaming_test_arguments_4 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml new file mode 100644 index 0000000000000..eebc96a63959b --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java ParDo Dataflow V2 Batch JavaVersions + +on: + issue_comment: + types: [created] + schedule: + - cron: '40 16 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.body || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + (contains(github.event.comment.body, 'Run Load Tests Java') && + contains(github.event.comment.body, 'ParDo Dataflow V2 Batch')) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase_1 }} ${{ matrix.java_version }} ${{ matrix.job_phrase_2 }}) + strategy: + fail-fast: false + matrix: + job_name: ["beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions"] + job_phrase_1: ["Run Load Tests Java"] + job_phrase_2: ["ParDo Dataflow V2 Batch"] + java_version: ['11','17'] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase_1 }} ${{ matrix.java_version }} ${{ matrix.job_phrase_2 }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase_1 }} ${{ matrix.java_version }} ${{ matrix.job_phrase_2 }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: ${{ matrix.java_version }} + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_200_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_100_counters.txt + arguments: | + --influxTags={\"runnerVersion\":\"v2\",\"jdk\":\"java${{ matrix.java_version }}\"} + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run ParDo Dataflow V2 Batch Java Load Test 1 (10 times) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions_test_arguments_1 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_ParDo_1' \ + - name: run ParDo Dataflow V2 Batch Java Load Test 2 (200 times) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions_test_arguments_2 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_ParDo_2' \ + - name: run ParDo Dataflow V2 Batch Java Load Test 3 (10 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions_test_arguments_3 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_ParDo_3' \ + - name: run ParDo Dataflow V2 Batch Java Load Test 4 (100 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions_test_arguments_4 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_ParDo_4' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml new file mode 100644 index 0000000000000..dcec86f8b4e00 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java ParDo Dataflow V2 Streaming JavaVersions + +on: + issue_comment: + types: [created] + schedule: + - cron: '30 21 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.body || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + (contains(github.event.comment.body, 'Run Load Tests Java') && + contains(github.event.comment.body, 'ParDo Dataflow V2 Streaming')) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase_1 }} ${{ matrix.java_version }} ${{ matrix.job_phrase_2 }}) + strategy: + fail-fast: false + matrix: + job_name: ["beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions"] + job_phrase_1: ["Run Load Tests Java"] + job_phrase_2: ["ParDo Dataflow V2 Streaming"] + java_version: ['11','17'] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase_1 }} ${{ matrix.java_version }} ${{ matrix.job_phrase_2 }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase_1 }} ${{ matrix.java_version }} ${{ matrix.job_phrase_2 }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: ${{ matrix.java_version }} + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_200_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_100_counters.txt + arguments: | + --influxTags={\"runnerVersion\":\"v2\",\"jdk\":\"java${{ matrix.java_version }}\"} + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run ParDo Dataflow V2 Streaming Java Load Test 1 (10 times) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions_test_arguments_1 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_ParDo_1' \ + - name: run ParDo Dataflow V2 Streaming Java Load Test 2 (200 times) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions_test_arguments_2 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_ParDo_2' \ + - name: run ParDo Dataflow V2 Streaming Java Load Test 3 (10 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions_test_arguments_3 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_ParDo_3' \ + - name: run ParDo Dataflow V2 Streaming Java Load Test 4 (100 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:google-cloud-dataflow-java \ + -Prunner.version=V2 \ + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions_test_arguments_4 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_ParDo_4' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch.yml b/.github/workflows/beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch.yml new file mode 100644 index 0000000000000..b5fc5792e0ffd --- /dev/null +++ b/.github/workflows/beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Java ParDo SparkStructuredStreaming Batch + +on: + schedule: + - cron: '25 8 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch_test_arguments_1: '' + beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch_test_arguments_2: '' + beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch_test_arguments_3: '' + beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch_test_arguments_4: '' + +jobs: + beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Java ParDo SparkStructuredStreaming Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch"] + job_phrase: ["Run Load Tests Java ParDo SparkStructuredStreaming Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_10_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_200_times.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_10_counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_100_counters.txt + - name: run ParDo SparkStructuredStreaming Batch Java Load Test 1 (10 times) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch_test_arguments_1 }}' \ + - name: run ParDo SparkStructuredStreaming Batch Java Load Test 2 (200 times) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch_test_arguments_2 }}' \ + - name: run ParDo SparkStructuredStreaming Batch Java Load Test 3 (10 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch_test_arguments_3 }}' \ + - name: run ParDo SparkStructuredStreaming Batch Java Load Test 4 (100 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:load-tests:run + arguments: | + -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ + -Prunner=:runners:spark:3 \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch_test_arguments_4 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml new file mode 100644 index 0000000000000..5f63bdb9c3b5d --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml @@ -0,0 +1,127 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Python CoGBK Dataflow Batch + +on: + issue_comment: + types: [created] + schedule: + - cron: '30 11 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Python_CoGBK_Dataflow_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Python CoGBK Dataflow Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_CoGBK_Dataflow_Batch"] + job_phrase: ["Run Load Tests Python CoGBK Dataflow Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Batch_100b_Single_Key.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Batch_10kB.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Batch_2MB.txt + - name: Set current datetime + id: datetime + run: | + echo "datetime=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_OUTPUT + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run CoGBK 2GB of 100B records with a single key + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + --info \ + -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-cogbk-1-${{ steps.datetime.outputs.datetime }}' \ + - name: run CoGBK 2GB of 100B records with multiple keys + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-cogbk-2-${{ steps.datetime.outputs.datetime }}' \ + - name: run CoGBK reiterate 4 times 10kB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-cogbk-3-${{ steps.datetime.outputs.datetime }}' \ + - name: run CoGBK reiterate 4 times 2MB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-cogbk-4-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml new file mode 100644 index 0000000000000..4e25d81bfcbc6 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Python CoGBK Dataflow Streaming + +on: + issue_comment: + types: [created] + schedule: + - cron: '30 11 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Python_CoGBK_Dataflow_Streaming: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Python CoGBK Dataflow Streaming' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_CoGBK_Dataflow_Streaming"] + job_phrase: ["Run Load Tests Python CoGBK Dataflow Streaming"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_100b_Single_Key.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_100b_Multiple_Keys.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_10kB.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_2MB.txt + - name: Set current datetime + id: datetime + run: | + echo "datetime=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_OUTPUT + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run CoGBK 2GB of 100B records with a single key + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-cogbk-1-${{ steps.datetime.outputs.datetime }}' \ + - name: run CoGBK 2GB of 100B records with multiple keys + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-cogbk-2-${{ steps.datetime.outputs.datetime }}' \ + - name: run CoGBK reiterate 4 times 10kB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_3 }} --job_name=load-tests-python-dataflow-streaming-cogbk-3-${{ steps.datetime.outputs.datetime }}' \ + - name: run CoGBK reiterate 4 times 2MB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Dataflow_Streaming_test_arguments_4 }} --job_name=load-tests-python-dataflow-streaming-cogbk-4-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml new file mode 100644 index 0000000000000..17851d5e57e13 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml @@ -0,0 +1,135 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Python CoGBK Dataflow Flink Batch + +on: + issue_comment: + types: [created] + schedule: + - cron: '40 12 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GCLOUD_ZONE: us-central1-a + CLUSTER_NAME: beam-loadtests-python-cogbk-flink-batch-${{ github.run_id }} + GCS_BUCKET: gs://beam-flink-cluster + FLINK_DOWNLOAD_URL: https://archive.apache.org/dist/flink/flink-1.15.0/flink-1.15.0-bin-scala_2.12.tgz + HADOOP_DOWNLOAD_URL: https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar + FLINK_TASKMANAGER_SLOTS: 1 + DETACHED_MODE: true + HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest + JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink1.15_job_server:latest + ARTIFACTS_DIR: gs://beam-flink-cluster/beam-loadtests-python-cogbk-flink-batch-${{ github.run_id }} + +jobs: + beam_LoadTests_Python_CoGBK_Flink_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Python CoGBK Flink Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_CoGBK_Flink_Batch"] + job_phrase: ["Run Load Tests Python CoGBK Flink Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Flink_Batch_100b_Single_Key.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Flink_Batch_100b_Multiple_Keys.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Flink_Batch_10kB.txt + - name: Start Flink with parallelism 5 + env: + FLINK_NUM_WORKERS: 5 + run: | + cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh create + - name: Set current datetime + id: datetime + run: | + echo "datetime=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_OUTPUT + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run CoGBK 2GB of 100B records with a single key + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + --info \ + -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Flink_Batch_test_arguments_1 }} --job_name=load-tests-python-flink-batch-cogbk-1-${{ steps.datetime.outputs.datetime }}' \ + - name: run CoGBK 2GB of 100B records with multiple keys + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + --info \ + -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-cogbk-2-${{ steps.datetime.outputs.datetime }}' \ + - name: run CoGBK reiterate 4 times 10kB values + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + --info \ + -PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \ + -Prunner=FlinkRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-cogbk-3-${{ steps.datetime.outputs.datetime }}' \ + - name: Teardown Flink + if: always() + run: | + ${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml new file mode 100644 index 0000000000000..15143708c365c --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Load Tests Combine Dataflow Batch Python + +on: + schedule: + - cron: '40 5 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Python_Combine_Dataflow_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Python Combine Dataflow Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_Combine_Dataflow_Batch"] + job_phrase: ["Run Load Tests Python Combine Dataflow Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name: Prepare configs + #Reads config files, excludes comments, appends current date to the job_name parameter + id: set_configs + shell: bash + run: | + CURDATE=$(date '+%m%d%H%M%S' --utc) + CONFIG_ARR=('config_Combine_Python_Batch_2GB_10b.txt' 'config_Combine_Python_Batch_2GB_Fanout_4.txt' 'config_Combine_Python_Batch_2GB_Fanout_8.txt') + for INDEX in ${!CONFIG_ARR[@]} + do + CURCONFIG=$(grep -v "^#.*" ./.github/workflows/load-tests-job-configs/${CONFIG_ARR[INDEX]} | tr '\n' ' ') + CURCONFIG=$(echo "${CURCONFIG/load-tests-python-dataflow-batch-combine-$((INDEX + 1))-/load-tests-python-dataflow-batch-combine-$((INDEX + 1))-$CURDATE}") + echo "prepared_config_$((INDEX + 1))=$CURCONFIG" >> $GITHUB_OUTPUT + done + - name: run Combine Dataflow Batch Python Load Test 1 (10 bytes records) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_1 }}' \ + - name: run Combine Dataflow Batch Python Load Test 2 (fanout 4) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_2 }}' \ + - name: run Combine Dataflow Batch Python Load Test 3 (fanout 8) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_3 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml new file mode 100644 index 0000000000000..005bddfa49ac7 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Python Combine Dataflow Streaming + +on: + issue_comment: + types: [created] + schedule: + - cron: '10 9 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.body || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Python_Combine_Dataflow_Streaming: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Python Combine Dataflow Streaming' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_Combine_Dataflow_Streaming"] + job_phrase: ["Run Load Tests Python Combine Dataflow Streaming"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_10_byte_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_Fanout_8.txt + - name: get current time + run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run 2GB 10 byte records test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-combine-1-${{env.NOW_UTC}}' \ + - name: run 2GB Fanout 4 test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-combine-4-${{env.NOW_UTC}}' \ + - name: run 2GB Fanout 8 test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Dataflow_Streaming_test_arguments_3 }} --job_name=load-tests-python-dataflow-streaming-combine-5-${{env.NOW_UTC}}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml new file mode 100644 index 0000000000000..6f7d2a8caa0b2 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml @@ -0,0 +1,138 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Python Combine Flink Batch + +on: + schedule: + - cron: '10 6 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GCLOUD_ZONE: us-central1-a + CLUSTER_NAME: beam-loadtests-py-cmb-flink-batch-${{ github.run_id }} + GCS_BUCKET: gs://beam-flink-cluster + FLINK_DOWNLOAD_URL: https://archive.apache.org/dist/flink/flink-1.15.0/flink-1.15.0-bin-scala_2.12.tgz + HADOOP_DOWNLOAD_URL: https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar + FLINK_TASKMANAGER_SLOTS: 1 + DETACHED_MODE: true + HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest + JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink1.15_job_server:latest + ARTIFACTS_DIR: gs://beam-flink-cluster/beam-loadtests-py-cmb-flink-batch-${{ github.run_id }} + +jobs: + beam_LoadTests_Python_Combine_Flink_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Python Combine Flink Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_Combine_Flink_Batch"] + job_phrase: ["Run Load Tests Python Combine Flink Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_10_byte_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_Fanout_8.txt + - name: Start Flink with parallelism 5 + env: + FLINK_NUM_WORKERS: 5 + run: | + cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh create + - name: get current time + run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run Load test 2GB 10 byte records + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PpythonVersion=3.8 \ + -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ + -Prunner=PortableRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Batch_test_arguments_1 }} --job_name=load-tests-python-flink-batch-combine-1-${{env.NOW_UTC}}' \ + - name: Restart Flink with parallelism 16 + env: + FLINK_NUM_WORKERS: 16 + run: | + cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh restart + - name: run Load test 2GB Fanout 4 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PpythonVersion=3.8 \ + -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ + -Prunner=PortableRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-combine-4-${{env.NOW_UTC}}' \ + - name: run Load test 2GB Fanout 8 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PpythonVersion=3.8 \ + -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ + -Prunner=PortableRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Batch_test_arguments_3 }} --job_name=load-tests-python-flink-batch-combine-5-${{env.NOW_UTC}}' \ + - name: Teardown Flink + if: always() + run: | + ${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml new file mode 100644 index 0000000000000..636dd20c148b4 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Python Combine Flink Streaming + +on: + schedule: + - cron: '10 7 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GCLOUD_ZONE: us-central1-a + CLUSTER_NAME: beam-loadtests-py-cmb-flink-streaming-${{ github.run_id }} + GCS_BUCKET: gs://beam-flink-cluster + FLINK_DOWNLOAD_URL: https://archive.apache.org/dist/flink/flink-1.15.0/flink-1.15.0-bin-scala_2.12.tgz + HADOOP_DOWNLOAD_URL: https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar + FLINK_TASKMANAGER_SLOTS: 1 + DETACHED_MODE: true + HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest + JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink1.15_job_server:latest + ARTIFACTS_DIR: gs://beam-flink-cluster/beam-loadtests-py-cmb-flink-streaming-${{ github.run_id }} + +jobs: + beam_LoadTests_Python_Combine_Flink_Streaming: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Python Combine Flink Streaming' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_Combine_Flink_Streaming"] + job_phrase: ["Run Load Tests Python Combine Flink Streaming"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_Fanout_4.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_Fanout_8.txt + - name: Start Flink with parallelism 16 + env: + FLINK_NUM_WORKERS: 16 + run: | + cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh create + - name: get current time + run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run Load test 2GB Fanout 4 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PpythonVersion=3.8 \ + -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ + -Prunner=PortableRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Streaming_test_arguments_1 }} --job_name=load-tests-python-flink-streaming-combine-4-${{env.NOW_UTC}}' \ + - name: run Load test 2GB Fanout 8 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PpythonVersion=3.8 \ + -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ + -Prunner=PortableRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Streaming_test_arguments_2 }} --job_name=load-tests-python-flink-streaming-combine-5-${{env.NOW_UTC}}' \ + - name: Teardown Flink + if: always() + run: | + ${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete + + # // TODO(https://github.com/apache/beam/issues/20402). Skipping some cases because they are too slow: + # load-tests-python-flink-streaming-combine-1' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml b/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml new file mode 100644 index 0000000000000..3077376a4da9f --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Load Tests FnApiRunner Microbenchmark Python + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Python_FnApiRunner_Microbenchmark: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Python Load Tests FnApiRunner Microbenchmark' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_FnApiRunner_Microbenchmark"] + job_phrase: ["Run Python Load Tests FnApiRunner Microbenchmark"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/config_FnApiRunner_Python_Microbenchmark.txt + - name: run FnApiRunner Microbenchmark Python Load Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.microbenchmarks_test \ + -Prunner=DirectRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_FnApiRunner_Microbenchmark_test_arguments_1 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml new file mode 100644 index 0000000000000..d9d9564d3bfe7 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Python GBK Dataflow Batch + +on: + schedule: + - cron: '10 2 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Python_GBK_Dataflow_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Python GBK Dataflow Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_GBK_Dataflow_Batch"] + job_phrase: ["Run Load Tests Python GBK Dataflow Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt + - name: get current time + run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run 2GB of 10B records test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-gbk-1-${{env.NOW_UTC}}' \ + - name: run 2GB of 100B records test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-gbk-2-${{env.NOW_UTC}}' \ + - name: run 2GB of 100kB records test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-gbk-3-${{env.NOW_UTC}}' \ + - name: run fanout 4 times with 2GB 10-byte records test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-gbk-4-${{env.NOW_UTC}}' \ + - name: run fanout 8 times with 2GB 10-byte records total test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Batch_test_arguments_5 }} --job_name=load-tests-python-dataflow-batch-gbk-5-${{env.NOW_UTC}}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml new file mode 100644 index 0000000000000..cf252f0758e07 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Python GBK Dataflow Streaming + +on: + schedule: + - cron: '10 4 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Python_GBK_Dataflow_Streaming: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Python GBK Dataflow Streaming' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_GBK_Dataflow_Streaming"] + job_phrase: ["Run Load Tests Python GBK Dataflow Streaming"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt + - name: get current time + run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV + # The env variable is created and populated in the test-arguments-action as "_test_arguments_" + - name: run 2GB of 100kB records test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-gbk-3-${{env.NOW_UTC}}' \ + + # // TODO(https://github.com/apache/beam/issues/20403). Skipping some cases because they are too slow: + # load-tests-python-dataflow-streaming-gbk-1 + # load-tests-python-dataflow-streaming-gbk-2 + # load-tests-python-dataflow-streaming-gbk-4 + # load-tests-python-dataflow-streaming-gbk-5 \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_GBK_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Python_GBK_Flink_Batch.yml new file mode 100644 index 0000000000000..cba71c0bd5263 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_GBK_Flink_Batch.yml @@ -0,0 +1,155 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Python GBK Flink Batch + +on: + # issue_comment: + # types: [created] + # schedule: + # - cron: '1 1 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GCLOUD_ZONE: us-central1-a + CLUSTER_NAME: beam-loadtests-py-gbk-flk-batch-${{ github.run_id }} + GCS_BUCKET: gs://beam-flink-cluster + FLINK_DOWNLOAD_URL: https://archive.apache.org/dist/flink/flink-1.15.0/flink-1.15.0-bin-scala_2.12.tgz + HADOOP_DOWNLOAD_URL: https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar + FLINK_TASKMANAGER_SLOTS: 1 + DETACHED_MODE: true + HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest + JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink1.15_job_server:latest + ARTIFACTS_DIR: gs://beam-flink-cluster/beam-loadtests-py-gbk-flk-batch-${{ github.run_id }} + +jobs: + beam_LoadTests_Python_GBK_Flink_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Python GBK Flink Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_GBK_Flink_Batch"] + job_phrase: ["Run Load Tests Python GBK Flink Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt + - name: get current time + run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV + - name: Start Flink with parallelism 5 + env: + FLINK_NUM_WORKERS: 5 + run: | + cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh create + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run Flink Batch 2GB of 10B records test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run --info + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ + -Prunner=PortableRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Flink_Batch_test_arguments_1 }} --job_name=load-tests-python-flink-batch-gbk-1-${{env.NOW_UTC}}' \ + - name: run Flink Batch 2GB of 100B records test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ + -Prunner=PortableRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-gbk-2-${{env.NOW_UTC}}' \ + - name: run reiterate 4 times 10kB values test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ + -Prunner=PortableRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Flink_Batch_test_arguments_5 }} --job_name=load-tests-python-flink-batch-gbk-6-${{env.NOW_UTC}}' \ + - name: Restart Flink with parallelism 16 + env: + FLINK_NUM_WORKERS: 16 + run: | + cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh restart + - name: run fanout 4 times with 2GB 10-byte records total test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ + -Prunner=PortableRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Flink_Batch_test_arguments_3 }} --job_name=load-tests-python-flink-batch-gbk-4-${{env.NOW_UTC}}' \ + - name: run fanout 8 times with 2GB 10-byte records total test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ + -Prunner=PortableRunner \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_Flink_Batch_test_arguments_4 }} --job_name=load-tests-python-flink-batch-gbk-5-${{env.NOW_UTC}}' \ + - name: Teardown Flink + if: always() + run: | + ${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete + + # TODO(https://github.com/apache/beam/issues/20146) Re-enable auto builds after these tests pass. \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml new file mode 100644 index 0000000000000..febc54c0c23e3 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Python GBK reiterate Dataflow Batch + +on: + issue_comment: + types: [created] + schedule: + - cron: '10 1 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.body || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Python GBK reiterate Dataflow Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch"] + job_phrase: ["Run Load Tests Python GBK reiterate Dataflow Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_2MB_values.txt + - name: get current time + run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run reiterate 4 times 10kB values test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-gbk-6-${{env.NOW_UTC}}' \ + - name: run reiterate 4 times 2MB values test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-gbk-7-${{env.NOW_UTC}}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml new file mode 100644 index 0000000000000..a904c74d24b44 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Python GBK reiterate Dataflow Streaming + +on: + issue_comment: + types: [created] + schedule: + - cron: '10 5 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.body || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Python GBK reiterate Dataflow Streaming' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming"] + job_phrase: ["Run Load Tests Python GBK reiterate Dataflow Streaming"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_2MB_values.txt + - name: get current time + run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run reiterate 4 times 10kB values test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-gbk-6-${{env.NOW_UTC}}' \ + - name: run reiterate 4 times 2MB values test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-gbk-7-${{env.NOW_UTC}}' \ diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml new file mode 100644 index 0000000000000..c16bafd34b2c1 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Python ParDo Dataflow Batch + +on: + issue_comment: + types: [created] + schedule: + - cron: '30 13 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Python_ParDo_Dataflow_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Python ParDo Dataflow Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 200 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_ParDo_Dataflow_Batch"] + job_phrase: ["Run Load Tests Python ParDo Dataflow Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_10_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_200_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_10_Counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_100_Counters.txt + - name: Set current datetime + id: datetime + run: | + echo "datetime=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_OUTPUT + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run ParDo Dataflow Batch Python Load Test 1 (10 iterations) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-pardo-1-${{ steps.datetime.outputs.datetime }}' \ + - name: run ParDo Dataflow Batch Python Load Test 2 (200 iterations) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-pardo-2-${{ steps.datetime.outputs.datetime }}' \ + - name: run ParDo Dataflow Batch Python Load Test 3 (10 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-pardo-3-${{ steps.datetime.outputs.datetime }}' \ + - name: run ParDo Dataflow Batch Python Load Test 4 (100 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-pardo-4-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml new file mode 100644 index 0000000000000..9d4bcdba60929 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Python ParDo Dataflow Streaming + +on: + issue_comment: + types: [created] + schedule: + - cron: '30 16 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Python_ParDo_Dataflow_Streaming: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Python Load Tests ParDo Dataflow Streaming' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 200 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_ParDo_Dataflow_Streaming"] + job_phrase: ["Run Python Load Tests ParDo Dataflow Streaming"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-pardo/python_ParDo_Dataflow_Streaming_10_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-pardo/python_ParDo_Dataflow_Streaming_200_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-pardo/python_ParDo_Dataflow_Streaming_10_Counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-pardo/python_ParDo_Dataflow_Streaming_100_Counters.txt + - name: Set current datetime + id: datetime + run: | + echo "datetime=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_OUTPUT + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run ParDo Dataflow Streaming Python Load Test 1 (10 iterations) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Streaming_test_arguments_1 }} --job_name=load-tests-python-dataflow-streaming-pardo-1-${{ steps.datetime.outputs.datetime }}' \ + - name: run ParDo Dataflow Streaming Python Load Test 2 (200 iterations) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Streaming_test_arguments_2 }} --job_name=load-tests-python-dataflow-streaming-pardo-2-${{ steps.datetime.outputs.datetime }}' \ + - name: run ParDo Dataflow Streaming Python Load Test 3 (10 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Streaming_test_arguments_3 }} --job_name=load-tests-python-dataflow-streaming-pardo-3-${{ steps.datetime.outputs.datetime }}' \ + - name: run ParDo Dataflow Streaming Python Load Test 4 (100 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Dataflow_Streaming_test_arguments_4 }} --job_name=load-tests-python-dataflow-streaming-pardo-4-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml new file mode 100644 index 0000000000000..7a450fadbdb2d --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml @@ -0,0 +1,131 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Python ParDo Flink Batch + +on: + issue_comment: + types: [created] + schedule: + - cron: '10 14 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GCLOUD_ZONE: us-central1-a + CLUSTER_NAME: beam-loadtests-python-pardo-flink-batch-${{ github.run_id }} + GCS_BUCKET: gs://beam-flink-cluster + FLINK_DOWNLOAD_URL: https://archive.apache.org/dist/flink/flink-1.15.0/flink-1.15.0-bin-scala_2.12.tgz + HADOOP_DOWNLOAD_URL: https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar + FLINK_TASKMANAGER_SLOTS: 1 + DETACHED_MODE: true + HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest + JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink1.15_job_server:latest + ARTIFACTS_DIR: gs://beam-flink-cluster/beam-loadtests-python-pardo-flink-batch-${{ github.run_id }} + +jobs: + beam_LoadTests_Python_ParDo_Flink_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Python ParDo Flink Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_ParDo_Flink_Batch"] + job_phrase: ["Run Load Tests Python ParDo Flink Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-pardo/python_ParDo_Flink_Batch_10_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-pardo/python_ParDo_Flink_Batch_200_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-pardo/python_ParDo_Flink_Batch_10_Counters.txt + - name: Start Flink with parallelism 5 + env: + FLINK_NUM_WORKERS: 5 + run: | + cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh create + - name: Set current datetime + id: datetime + run: | + echo "datetime=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_OUTPUT + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run ParDo Flink Batch Python Load Test 1 (10 iterations) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ + -Prunner=PortableRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Batch_test_arguments_1 }} --job_name=load-tests-python-flink-batch-pardo-1-${{ steps.datetime.outputs.datetime }}' \ + - name: run ParDo Flink Batch Python Load Test 2 (200 iterations) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ + -Prunner=PortableRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-pardo-3-${{ steps.datetime.outputs.datetime }}' \ + - name: run ParDo Flink Batch Python Load Test 3 (10 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ + -Prunner=PortableRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Batch_test_arguments_3 }} --job_name=load-tests-python-flink-batch-pardo-4-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml new file mode 100644 index 0000000000000..37824b28ce0c4 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml @@ -0,0 +1,155 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Python ParDo Flink Streaming + +on: + issue_comment: + types: [created] + schedule: + - cron: '15 15 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GCLOUD_ZONE: us-central1-a + CLUSTER_NAME: beam-loadtests-python-pardo-flink-stream-${{ github.run_id }} + GCS_BUCKET: gs://beam-flink-cluster + FLINK_DOWNLOAD_URL: https://archive.apache.org/dist/flink/flink-1.15.0/flink-1.15.0-bin-scala_2.12.tgz + HADOOP_DOWNLOAD_URL: https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar + FLINK_TASKMANAGER_SLOTS: 1 + DETACHED_MODE: true + HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest + JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink1.15_job_server:latest + ARTIFACTS_DIR: gs://beam-flink-cluster/beam-loadtests-python-pardo-flink-stream-${{ github.run_id }} + +jobs: + beam_LoadTests_Python_ParDo_Flink_Streaming: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Python ParDo Flink Streaming' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_ParDo_Flink_Streaming"] + job_phrase: ["Run Load Tests Python ParDo Flink Streaming"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-pardo/python_ParDo_Flink_Streaming_10_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-pardo/python_ParDo_Flink_Streaming_200_Iterations.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-pardo/python_ParDo_Flink_Streaming_10_Counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-pardo/python_ParDo_Flink_Streaming_100_Counters.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python-pardo/python_ParDo_Flink_Streaming_5_Iterations.txt + - name: Start Flink with parallelism 5 + env: + FLINK_NUM_WORKERS: 5 + run: | + cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh create + - name: Set current datetime + id: datetime + run: | + echo "datetime=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_OUTPUT + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run ParDo Flink Streaming Python Load Test 1 (10 iterations) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ + -Prunner=PortableRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_1 }} --job_name=load-tests-python-flink-streaming-pardo-1-${{ steps.datetime.outputs.datetime }}' \ + - name: run ParDo Flink Streaming Python Load Test 2 (200 iterations) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ + -Prunner=PortableRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_2 }} --job_name=load-tests-python-flink-streaming-pardo-2-${{ steps.datetime.outputs.datetime }}' \ + - name: run ParDo Flink Streaming Python Load Test 3 (10 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ + -Prunner=PortableRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_3 }} --job_name=load-tests-python-flink-streaming-pardo-3-${{ steps.datetime.outputs.datetime }}' \ + - name: run ParDo Flink Streaming Python Load Test 4 (100 counters) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ + -Prunner=PortableRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_4 }} --job_name=load-tests-python-flink-streaming-pardo-4-${{ steps.datetime.outputs.datetime }}' \ + - name: run ParDo Flink Streaming Python Load Test 5 (5 iterations) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \ + -Prunner=PortableRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_ParDo_Flink_Streaming_test_arguments_5 }} --job_name=load-tests-python-flink-streaming-pardo-6-${{ steps.datetime.outputs.datetime }}' \ + - name: Teardown Flink + if: always() + run: | + ${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml b/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml new file mode 100644 index 0000000000000..91e4c6a105140 --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml @@ -0,0 +1,186 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Python SideInput Dataflow Batch + +on: + issue_comment: + types: [created] + schedule: + - cron: '30 13 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Python_SideInput_Dataflow_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Load Tests Python SideInput Dataflow Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_SideInput_Dataflow_Batch"] + job_phrase: ["Run Load Tests Python SideInput Dataflow Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_1key_percent_dict.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_99key_percent_dict.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1window_first_iterable.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1window_iterable.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_first_list.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_list.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1000window_1key_percent_dict.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1000window_99key_percent_dict.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1000window_first_iterable.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1000window_iterable.txt + - name: Set current datetime + id: datetime + run: | + echo "datetime=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_OUTPUT + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run SideInput Dataflow Batch Python Load Test 1 (1gb-1kb-10workers-1window-1key-percent-dict) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_1 }} --job_name=load-tests-python-dataflow-batch-sideinput-1-${{ steps.datetime.outputs.datetime }}' \ + - name: run SideInput Dataflow Batch Python Load Test 2 (1gb-1kb-10workers-1window-99key-percent-dict) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-sideinput-2-${{ steps.datetime.outputs.datetime }}' \ + - name: run SideInput Dataflow Batch Python Load Test 3 (10gb-1kb-10workers-1window-first-iterable) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_3 }} --job_name=load-tests-python-dataflow-batch-sideinput-3-${{ steps.datetime.outputs.datetime }}' \ + - name: run SideInput Dataflow Batch Python Load Test 4 (10gb-1kb-10workers-1window-iterable) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_4 }} --job_name=load-tests-python-dataflow-batch-sideinput-4-${{ steps.datetime.outputs.datetime }}' \ + - name: run SideInput Dataflow Batch Python Load Test 5 (1gb-1kb-10workers-1window-first-list) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_5 }} --job_name=load-tests-python-dataflow-batch-sideinput-5-${{ steps.datetime.outputs.datetime }}' \ + - name: run SideInput Dataflow Batch Python Load Test 6 (1gb-1kb-10workers-1window-list) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_6 }} --job_name=load-tests-python-dataflow-batch-sideinput-6-${{ steps.datetime.outputs.datetime }}' \ + - name: run SideInput Dataflow Batch Python Load Test 7 (1gb-1kb-10workers-1000window-1key-percent-dict) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_7 }} --job_name=load-tests-python-dataflow-batch-sideinput-7-${{ steps.datetime.outputs.datetime }}' \ + - name: run SideInput Dataflow Batch Python Load Test 8 (1gb-1kb-10workers-1000window-99key-percent-dict) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_8 }} --job_name=load-tests-python-dataflow-batch-sideinput-8-${{ steps.datetime.outputs.datetime }}' \ + - name: run SideInput Dataflow Batch Python Load Test 9 (10gb-1kb-10workers-1000window-first-iterable) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_9 }} --job_name=load-tests-python-dataflow-batch-sideinput-9-${{ steps.datetime.outputs.datetime }}' \ + - name: run SideInput Dataflow Batch Python Load Test 10 (10gb-1kb-10workers-1000window-iterable) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.sideinput_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_SideInput_Dataflow_Batch_test_arguments_10 }} --job_name=load-tests-python-dataflow-batch-sideinput-10-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Python_Smoke.yml b/.github/workflows/beam_LoadTests_Python_Smoke.yml new file mode 100644 index 0000000000000..720561d09e06e --- /dev/null +++ b/.github/workflows/beam_LoadTests_Python_Smoke.yml @@ -0,0 +1,103 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: LoadTests Python Smoke + +on: + issue_comment: + types: [created] + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.comment.body || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_LoadTests_Python_Smoke: + if: | + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Python Load Tests Smoke' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_LoadTests_Python_Smoke"] + job_phrase: ["Run Python Load Tests Smoke"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Smoke_GroupByKey_Direct.txt + ${{ github.workspace }}/.github/workflows/load-tests-job-configs/python_Smoke_GroupByKey_Dataflow.txt + - name: Set current datetime + id: datetime + run: | + echo "datetime=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_OUTPUT + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: run GroupByKey Python load test Direct + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ + -Prunner=DirectRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_Smoke_test_arguments_1 }} --job_name=load-tests-python-direct-batch-gbk-smoke-${{ steps.datetime.outputs.datetime }}' \ + - name: run GroupByKey Python load test Dataflow + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.load_tests.group_by_key_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_LoadTests_Python_Smoke_test_arguments_2 }} --job_name=load-tests-python-dataflow-batch-gbk-smoke-${{ steps.datetime.outputs.datetime }}' \ No newline at end of file diff --git a/.github/workflows/beam_MetricsCredentialsRotation.yml b/.github/workflows/beam_MetricsCredentialsRotation.yml new file mode 100644 index 0000000000000..9bd795f0c2a49 --- /dev/null +++ b/.github/workflows/beam_MetricsCredentialsRotation.yml @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Rotate Metrics Cluster Credentials + +on: + schedule: + - cron: '0 2 1 * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_MetricsCredentialsRotation: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} + strategy: + matrix: + job_name: ["beam_MetricsCredentialsRotation"] + job_phrase: ["N/A"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} + - name: Starting credential rotation + run: | + gcloud container clusters update metrics --start-credential-rotation --zone=us-central1-a --quiet + - name: Rebuilding the nodes + run: | + gcloud container clusters upgrade metrics --node-pool=default-pool --zone=us-central1-a --quiet + - name: Completing the rotation + run: | + gcloud container clusters update metrics --complete-credential-rotation --zone=us-central1-a --quiet +# TODO: Send email to dev@beam.apache.org if something went wrong during credentials rotation \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_AvroIOIT.yml b/.github/workflows/beam_PerformanceTests_AvroIOIT.yml new file mode 100644 index 0000000000000..cc3799cd59d3a --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_AvroIOIT.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Performance Tests AvroIOIT + +on: + schedule: + - cron: '10 1/13 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_AvroIOIT_test_arguments_1: '' + +jobs: + beam_PerformanceTests_AvroIOIT: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java AvroIO Performance Test' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_AvroIOIT"] + job_phrase: ["Run Java AvroIO Performance Test"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_AvroIOIT.txt + arguments: | + --filenamePrefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}}/ + - name: run integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:file-based-io-tests:integrationTest + arguments: | + --tests org.apache.beam.sdk.io.avro.AvroIOIT \ + --info \ + -Dfilesystem=gcs \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_AvroIOIT_test_arguments_1 }}]' \ diff --git a/.github/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml b/.github/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml new file mode 100644 index 0000000000000..3bf97098213ec --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Performance Tests AvroIOIT HDFS + +on: + schedule: + - cron: '10 1/13 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_AvroIOIT_HDFS_test_arguments_1: '' + +jobs: + beam_PerformanceTests_AvroIOIT_HDFS: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java AvroIO Performance Test HDFS' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_AvroIOIT_HDFS"] + job_phrase: ["Run Java AvroIO Performance Test HDFS"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Set k8s access + uses: ./.github/actions/setup-k8s-access + with: + cluster_name: io-datastores + k8s_namespace: ${{ matrix.job_name }}-${{ github.run_id }} + - name: Install Hadoop + id: install_hadoop + run: | + kubectl apply -f ${{ github.workspace }}/.test-infra/kubernetes/hadoop/LargeITCluster/hdfs-multi-datanode-cluster.yml + kubectl wait svc/hadoop --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=60s + loadbalancer_IP=$(kubectl get svc hadoop -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo hadoop_IP=$loadbalancer_IP >> $GITHUB_OUTPUT + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_AvroIOIT_HDFS.txt + arguments: | + --filenamePrefix=hdfs://${{ steps.install_hadoop.outputs.hadoop_IP }}:9000/TEXTIO_IT_ + --hdfsConfiguration=[{\\\"fs.defaultFS\\\":\\\"hdfs:${{ steps.install_hadoop.outputs.hadoop_IP }}:9000\\\",\\\"dfs.replication\\\":1}] + - name: run integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:file-based-io-tests:integrationTest + arguments: | + --tests org.apache.beam.sdk.io.avro.AvroIOIT \ + --info \ + -Dfilesystem=hdfs \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_AvroIOIT_HDFS_test_arguments_1 }}]' \ diff --git a/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml b/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml new file mode 100644 index 0000000000000..a79c35704a8fe --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Performance Tests BigQueryIO Batch Java Avro + +on: + schedule: + - cron: '10 1,13 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PerformanceTests_BigQueryIO_Batch_Java_Avro: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run BigQueryIO Batch Performance Test Java Avro' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_BigQueryIO_Batch_Java_Avro"] + job_phrase: ["Run BigQueryIO Batch Performance Test Java Avro"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare config + id: set_config + shell: bash + run: | + CURDATE=$(date '+%m%d%H%M%S' --utc) + CURCONFIG=$(grep -v "^#.*" ./.github/workflows/performance-tests-job-configs/config_BigQueryIO_Batch_Java_Avro.txt | tr '\n' ' ') + CONFIGWITHDATE=$(echo "${CURCONFIG/bqio_write_10GB_java_avro_/bqio_write_10GB_java_avro_$CURDATE}") + echo "prepared_config=$CONFIGWITHDATE" >> $GITHUB_OUTPUT + - name: run Java BigQueryIO Batch Avro Performance Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:bigquery-io-perf-tests:integrationTest + arguments: | + --tests org.apache.beam.sdk.bigqueryioperftests.BigQueryIOIT \ + --info \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions=${{ steps.set_config.outputs.prepared_config }} \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml b/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml new file mode 100644 index 0000000000000..d59b942539912 --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Performance Tests BigQueryIO Batch Java Json + +on: + schedule: + - cron: '30 8,20 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PerformanceTests_BigQueryIO_Batch_Java_Json: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run BigQueryIO Batch Performance Test Java Json' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_BigQueryIO_Batch_Java_Json"] + job_phrase: ["Run BigQueryIO Batch Performance Test Java Json"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare config + id: set_config + shell: bash + run: | + CURDATE=$(date '+%m%d%H%M%S' --utc) + CURCONFIG=$(grep -v "^#.*" ./.github/workflows/performance-tests-job-configs/config_BigQueryIO_Batch_Java_Json.txt | tr '\n' ' ') + CONFIGWITHDATE=$(echo "${CURCONFIG/bqio_write_10GB_java_json_/bqio_write_10GB_java_json_$CURDATE}") + echo "prepared_config=$CONFIGWITHDATE" >> $GITHUB_OUTPUT + - name: run Java BigQueryIO Batch Json Performance Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:bigquery-io-perf-tests:integrationTest + arguments: | + --tests org.apache.beam.sdk.bigqueryioperftests.BigQueryIOIT \ + --info \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions=${{ steps.set_config.outputs.prepared_config }} \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml b/.github/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml new file mode 100644 index 0000000000000..775fe5abe9387 --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Performance Tests BigQueryIO Streaming Java + +on: + schedule: + - cron: '20 15,22 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PerformanceTests_BigQueryIO_Streaming_Java: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run BigQueryIO Streaming Performance Test Java' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_BigQueryIO_Streaming_Java"] + job_phrase: ["Run BigQueryIO Streaming Performance Test Java"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare config + id: set_config + shell: bash + run: | + CURDATE=$(date '+%m%d%H%M%S' --utc) + CURCONFIG=$(grep -v "^#.*" ./.github/workflows/performance-tests-job-configs/config_BigQueryIO_Streaming_Java.txt | tr '\n' ' ') + CONFIGWITHDATE=$(echo "${CURCONFIG/bqio_write_10GB_java_stream_/bqio_write_10GB_java_stream_$CURDATE}") + echo "prepared_config=$CONFIGWITHDATE" >> $GITHUB_OUTPUT + - name: run Java BigQueryIO Streaming Performance Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:bigquery-io-perf-tests:integrationTest + arguments: | + --tests org.apache.beam.sdk.bigqueryioperftests.BigQueryIOIT \ + --info \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions=${{ steps.set_config.outputs.prepared_config }} \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml new file mode 100644 index 0000000000000..c51e9c7d74424 --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests BiqQueryIO Read Python + +on: + schedule: + - cron: '0 2 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_BiqQueryIO_Read_Python_test_arguments_1: '' + +jobs: + beam_PerformanceTests_BiqQueryIO_Read_Python: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run BigQueryIO Read Performance Test Python' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_BiqQueryIO_Read_Python"] + job_phrase: ["Run BigQueryIO Read Performance Test Python"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/biqQueryIO_Read_Python.txt + arguments: | + --job_name=performance-tests-bqio-read-python-10gb$(date '+%m%d%H%M%S' --utc) + - name: Run BigQueryIO Read Performance Test Python + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.io.gcp.bigquery_read_perf_test \ + -PpythonVersion=3.8 \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{env.beam_PerformanceTests_BiqQueryIO_Read_Python_test_arguments_1}}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml new file mode 100644 index 0000000000000..c1d25b21e14e9 --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests BiqQueryIO Write Python Batch + +on: + schedule: + - cron: '0 1 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_BiqQueryIO_Write_Python_Batch_test_arguments_1: '' + +jobs: + beam_PerformanceTests_BiqQueryIO_Write_Python_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run BigQueryIO Write Performance Test Python' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_BiqQueryIO_Write_Python_Batch"] + job_phrase: ["Run BigQueryIO Write Performance Test Python"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/biqQueryIO_Write_Python_Batch.txt + arguments: | + --job_name=performance-tests-bqio-write-python-batch-10gb$(date '+%m%d%H%M%S' --utc) + - name: run BigQueryIO Write Batch Python Performance Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.io.gcp.bigquery_write_perf_test \ + -PpythonVersion=3.8 \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{env.beam_PerformanceTests_BiqQueryIO_Write_Python_Batch_test_arguments_1}}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_Cdap.yml b/.github/workflows/beam_PerformanceTests_Cdap.yml new file mode 100644 index 0000000000000..d8cd8f8fdbe5a --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_Cdap.yml @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests Cdap + +on: + schedule: + - cron: '13 4/12 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_Cdap_test_arguments_1: '' + +jobs: + beam_PerformanceTests_Cdap: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java CdapIO Performance Test' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_Cdap"] + job_phrase: ["Run Java CdapIO Performance Test"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Set k8s access + uses: ./.github/actions/setup-k8s-access + with: + cluster_name: io-datastores + k8s_namespace: ${{ matrix.job_name }}-${{ github.run_id }} + - name: Install postgres + id: install_postgres + run: | + kubectl apply -f ${{ github.workspace }}/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml + kubectl wait svc/postgres-for-dev --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=60s + loadbalancer_IP=$(kubectl get svc postgres-for-dev -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo postgres_IP=$loadbalancer_IP >> $GITHUB_OUTPUT + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/cdap.txt + arguments: | + --postgresServerName=${{ steps.install_postgres.outputs.postgres_IP }} + - name: run integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:cdap:integrationTest + arguments: | + --info \ + --tests org.apache.beam.sdk.io.cdap.CdapIOIT \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_Cdap_test_arguments_1 }}]' \ diff --git a/.github/workflows/beam_PerformanceTests_Compressed_TextIOIT.yml b/.github/workflows/beam_PerformanceTests_Compressed_TextIOIT.yml new file mode 100644 index 0000000000000..cb1b77c84db49 --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_Compressed_TextIOIT.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests Compressed TextIOIT + +on: + schedule: + - cron: '10 1/12 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PerformanceTests_Compressed_TextIOIT: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java CompressedTextIO Performance Test' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_Compressed_TextIOIT"] + job_phrase: ["Run Java CompressedTextIO Performance Test"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) +# The env variable is created and populated in the test-arguments-action as "_test_arguments_" + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_Compressed_TextIOIT.txt + arguments: | + --filenamePrefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}}/ + - name: run integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:file-based-io-tests:integrationTest + arguments: | + --tests org.apache.beam.sdk.io.text.TextIOIT \ + --info \ + -Dfilesystem=gcs \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_Compressed_TextIOIT_test_arguments_1 }}]' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_Compressed_TextIOIT_HDFS.yml b/.github/workflows/beam_PerformanceTests_Compressed_TextIOIT_HDFS.yml new file mode 100644 index 0000000000000..bea11acf259a6 --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_Compressed_TextIOIT_HDFS.yml @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests Compressed TextIOIT HDFS + +on: + schedule: + - cron: '50 1/12 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PerformanceTests_Compressed_TextIOIT_HDFS: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java CompressedTextIO Performance Test HDFS' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_Compressed_TextIOIT_HDFS"] + job_phrase: ["Run Java CompressedTextIO Performance Test HDFS"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Set k8s access + uses: ./.github/actions/setup-k8s-access + with: + k8s_namespace: ${{ matrix.job_name }}-${{ github.run_id }} + - name: Install Hadoop + id: install_hadoop + run: | + kubectl apply -f ${{ github.workspace }}/.test-infra/kubernetes/hadoop/LargeITCluster/hdfs-multi-datanode-cluster.yml + kubectl wait svc/hadoop --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=120s + loadbalancer_IP=$(kubectl get svc hadoop -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo hadoop_IP=$loadbalancer_IP >> $GITHUB_OUTPUT +# The env variable is created and populated in the test-arguments-action as "_test_arguments_" + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_Compressed_TextIOIT_HDFS.txt + arguments: | + --filenamePrefix=hdfs://${{ steps.install_hadoop.outputs.hadoop_IP }}:9000/TEXTIO_IT_ + --hdfsConfiguration=[{\\\"fs.defaultFS\\\":\\\"hdfs:${{ steps.install_hadoop.outputs.hadoop_IP }}:9000\\\",\\\"dfs.replication\\\":1}] + - name: run integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:file-based-io-tests:integrationTest + arguments: | + --tests org.apache.beam.sdk.io.text.TextIOIT \ + --info \ + -Dfilesystem=hdfs \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_Compressed_TextIOIT_HDFS_test_arguments_1 }}]' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_HadoopFormat.yml b/.github/workflows/beam_PerformanceTests_HadoopFormat.yml new file mode 100644 index 0000000000000..b65387a857ee9 --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_HadoopFormat.yml @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests HadoopFormat + +on: + schedule: + - cron: '16 7/12 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_HadoopFormat_test_arguments_1: '' + +jobs: + beam_PerformanceTests_HadoopFormat: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java HadoopFormatIO Performance Test' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_HadoopFormat"] + job_phrase: ["Run Java HadoopFormatIO Performance Test"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Set k8s access + uses: ./.github/actions/setup-k8s-access + with: + cluster_name: io-datastores + k8s_namespace: ${{ matrix.job_name }}-${{ github.run_id }} + - name: Install postgres + id: install_postgres + run: | + kubectl apply -f ${{ github.workspace }}/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml + kubectl wait svc/postgres-for-dev --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=60s + loadbalancer_IP=$(kubectl get svc postgres-for-dev -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo postgres_IP=$loadbalancer_IP >> $GITHUB_OUTPUT + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/hadoopFormat.txt + arguments: | + --postgresServerName=${{ steps.install_postgres.outputs.postgres_IP }} + - name: run integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:hadoop-format:integrationTest + arguments: | + --info \ + --tests org.apache.beam.sdk.io.hadoop.format.HadoopFormatIOIT \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_HadoopFormat_test_arguments_1 }}]' \ diff --git a/.github/workflows/beam_PerformanceTests_JDBC.yml b/.github/workflows/beam_PerformanceTests_JDBC.yml new file mode 100644 index 0000000000000..897fc461fd199 --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_JDBC.yml @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests JDBC + +on: + schedule: + - cron: '30 1,13 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_JDBC_test_arguments_1: '' + +jobs: + beam_PerformanceTests_JDBC: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java JdbcIO Performance Test' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_JDBC"] + job_phrase: ["Run Java JdbcIO Performance Test"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Set k8s access + uses: ./.github/actions/setup-k8s-access + with: + cluster_name: io-datastores + k8s_namespace: ${{ matrix.job_name }}-${{ github.run_id }} + - name: Install Postgres for dev + id: postgres_for_dev + run: | + kubectl apply -f ${{ github.workspace }}/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml + kubectl wait svc/postgres-for-dev --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=120s + loadbalancer_ip=$(kubectl get svc postgres-for-dev -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo "pfd_ip=$loadbalancer_ip" >> $GITHUB_OUTPUT + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/JDBC.txt + arguments: | + --postgresServerName=${{ steps.postgres_for_dev.outputs.pfd_ip }} + - name: Run Java JdbcIO Performance Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:jdbc:integrationTest + arguments: | + --tests org.apache.beam.sdk.io.jdbc.JdbcIOIT \ + --info \ + -Dfilesystem=hdfs \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{env.beam_PerformanceTests_JDBC_test_arguments_1}}]' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_Kafka_IO.yml b/.github/workflows/beam_PerformanceTests_Kafka_IO.yml new file mode 100644 index 0000000000000..149be8587237d --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_Kafka_IO.yml @@ -0,0 +1,120 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests Kafka IO + +on: + schedule: + - cron: '30 2,14 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_Kafka_IO_test_arguments_1: '' + beam_PerformanceTests_Kafka_IO_test_arguments_2: '' + +jobs: + beam_PerformanceTests_Kafka_IO: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java KafkaIO Performance Test' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_Kafka_IO"] + job_phrase: ["Run Java KafkaIO Performance Test"] + env: + KAFKA_SERVICE_PORT: 32400 + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Set k8s access + uses: ./.github/actions/setup-k8s-access + with: + cluster_name: io-datastores + k8s_namespace: ${{ matrix.job_name }}-${{ github.run_id }} + - name: Install Kafka + id: install_kafka + run: | + cd ${{ github.workspace }}/.test-infra/kubernetes/kafka-cluster/ + kubectl apply -R -f ${{ github.workspace }}/.test-infra/kubernetes/kafka-cluster/ + - name: Set up Kafka brokers + id: set_brokers + run: | + declare -a kafka_service_brokers + for INDEX in {0..2}; do + kubectl wait svc/outside-${INDEX} --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=120s + kafka_service_brokers[$INDEX]=$(kubectl get svc outside-${INDEX} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo "KAFKA_SERVICE_BROKER_${INDEX}=${kafka_service_brokers[$INDEX]}" >> $GITHUB_OUTPUT + done + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/kafka_IO_Streaming.txt + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/kafka_IO_Batch.txt + arguments: | + --kafkaBootstrapServerAddresses=${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_0 }}:${{ env.KAFKA_SERVICE_PORT }},${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_1 }}:${{ env.KAFKA_SERVICE_PORT }},${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_2 }}:${{ env.KAFKA_SERVICE_PORT }} + - name: Run Java KafkaIO Performance Streaming Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:kafka:integrationTest + arguments: | + --tests org.apache.beam.sdk.io.kafka.KafkaIOIT.testKafkaIOReadsAndWritesCorrectlyInStreaming \ + --info \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{env.beam_PerformanceTests_Kafka_IO_test_arguments_1}}]' \ + - name: Run Java KafkaIO Performance Batch Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:kafka:integrationTest + arguments: | + --tests org.apache.beam.sdk.io.kafka.KafkaIOIT.testKafkaIOReadsAndWritesCorrectlyInBatch \ + --info \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{env.beam_PerformanceTests_Kafka_IO_test_arguments_2}}]' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_ManyFiles_TextIOIT.yml b/.github/workflows/beam_PerformanceTests_ManyFiles_TextIOIT.yml new file mode 100644 index 0000000000000..0736b9d55b0ca --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_ManyFiles_TextIOIT.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests ManyFiles TextIOIT + +on: + schedule: + - cron: '10 2/12 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_ManyFiles_TextIOIT_test_arguments_1: '' + +jobs: + beam_PerformanceTests_ManyFiles_TextIOIT: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java ManyFilesTextIO Performance Test' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_ManyFiles_TextIOIT"] + job_phrase: ["Run Java ManyFilesTextIO Performance Test"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_ManyFiles_TextIOIT.txt + arguments: | + --filenamePrefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}}/ + - name: run integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:file-based-io-tests:integrationTest + arguments: | + --tests org.apache.beam.sdk.io.text.TextIOIT \ + --info \ + -Dfilesystem=gcs \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_ManyFiles_TextIOIT_test_arguments_1 }}]' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml b/.github/workflows/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml new file mode 100644 index 0000000000000..90a6dbdd76dfd --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests ManyFiles TextIOIT HDFS + +on: + schedule: + - cron: '50 2/12 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_ManyFiles_TextIOIT_HDFS_test_arguments_1: '' + +jobs: + beam_PerformanceTests_ManyFiles_TextIOIT_HDFS: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java ManyFilesTextIO Performance Test HDFS' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_ManyFiles_TextIOIT_HDFS"] + job_phrase: ["Run Java ManyFilesTextIO Performance Test HDFS"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Set k8s access + uses: ./.github/actions/setup-k8s-access + with: + k8s_namespace: ${{ matrix.job_name }}-${{ github.run_id }} + - name: Install Hadoop + id: install_hadoop + run: | + kubectl apply -f ${{ github.workspace }}/.test-infra/kubernetes/hadoop/LargeITCluster/hdfs-multi-datanode-cluster.yml + kubectl wait svc/hadoop --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=120s + loadbalancer_IP=$(kubectl get svc hadoop -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo hadoop_IP=$loadbalancer_IP >> $GITHUB_OUTPUT + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_ManyFiles_TextIOIT_HDFS.txt + arguments: | + --filenamePrefix=hdfs://${{ steps.install_hadoop.outputs.hadoop_IP }}:9000/TEXTIO_IT_ + --hdfsConfiguration=[{\\\"fs.defaultFS\\\":\\\"hdfs:${{ steps.install_hadoop.outputs.hadoop_IP }}:9000\\\",\\\"dfs.replication\\\":1}] + - name: run integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:file-based-io-tests:integrationTest + arguments: | + --tests org.apache.beam.sdk.io.text.TextIOIT \ + --info \ + -Dfilesystem=hdfs \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_ManyFiles_TextIOIT_HDFS_test_arguments_1 }}]' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_MongoDBIO_IT.yml b/.github/workflows/beam_PerformanceTests_MongoDBIO_IT.yml new file mode 100644 index 0000000000000..187706e6c0710 --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_MongoDBIO_IT.yml @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests MongoDBIO IT + +on: + schedule: + - cron: '14 5/12 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_MongoDBIO_IT_test_arguments_1: '' + +jobs: + beam_PerformanceTests_MongoDBIO_IT: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java MongoDBIO Performance Test' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_MongoDBIO_IT"] + job_phrase: ["Run Java MongoDBIO Performance Test"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Set k8s access + uses: ./.github/actions/setup-k8s-access + with: + cluster_name: io-datastores + k8s_namespace: ${{ matrix.job_name }}-${{ github.run_id }} + - name: Install mongo + id: install_mongo + run: | + kubectl apply -f ${{ github.workspace }}/.test-infra/kubernetes/mongodb/load-balancer/mongo.yml + kubectl wait svc/mongo-load-balancer-service --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=60s + loadbalancer_IP=$(kubectl get svc mongo-load-balancer-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo mongo_IP=$loadbalancer_IP >> $GITHUB_OUTPUT + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/mongoDBIO_IT.txt + arguments: | + --mongoDBHostName=${{ steps.install_mongo.outputs.mongo_IP }} + - name: run integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:mongodb:integrationTest + arguments: | + --info \ + --tests org.apache.beam.sdk.io.mongodb.MongoDBIOIT \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_MongoDBIO_IT_test_arguments_1 }}]' \ diff --git a/.github/workflows/beam_PerformanceTests_ParquetIOIT.yml b/.github/workflows/beam_PerformanceTests_ParquetIOIT.yml new file mode 100644 index 0000000000000..df0f9137acf20 --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_ParquetIOIT.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests ParquetIOIT + +on: + schedule: + - cron: '10 3/12 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PerformanceTests_ParquetIOIT: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java ParquetIO Performance Test' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_ParquetIOIT"] + job_phrase: ["Run Java ParquetIO Performance Test"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) +# The env variable is created and populated in the test-arguments-action as "_test_arguments_" + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_ParquetIOIT.txt + arguments: | + --filenamePrefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}}/ + - name: run integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:file-based-io-tests:integrationTest + arguments: | + --tests org.apache.beam.sdk.io.parquet.ParquetIOIT \ + --info \ + -Dfilesystem=gcs \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_ParquetIOIT_test_arguments_1 }}]' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_ParquetIOIT_HDFS.yml b/.github/workflows/beam_PerformanceTests_ParquetIOIT_HDFS.yml new file mode 100644 index 0000000000000..2cd7ac021df9a --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_ParquetIOIT_HDFS.yml @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests ParquetIOIT HDFS + +on: + schedule: + - cron: '50 3/12 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PerformanceTests_ParquetIOIT_HDFS: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java ParquetIO Performance Test HDFS' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_ParquetIOIT_HDFS"] + job_phrase: ["Run Java ParquetIO Performance Test HDFS"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Set k8s access + uses: ./.github/actions/setup-k8s-access + with: + k8s_namespace: ${{ matrix.job_name }}-${{ github.run_id }} + - name: Install Hadoop + id: install_hadoop + run: | + kubectl apply -f ${{ github.workspace }}/.test-infra/kubernetes/hadoop/LargeITCluster/hdfs-multi-datanode-cluster.yml + kubectl wait svc/hadoop --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=120s + loadbalancer_IP=$(kubectl get svc hadoop -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo hadoop_IP=$loadbalancer_IP >> $GITHUB_OUTPUT +# The env variable is created and populated in the test-arguments-action as "_test_arguments_" + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_ParquetIOIT_HDFS.txt + arguments: | + --filenamePrefix=hdfs://${{ steps.install_hadoop.outputs.hadoop_IP }}:9000/TEXTIO_IT_ + --hdfsConfiguration=[{\\\"fs.defaultFS\\\":\\\"hdfs:${{ steps.install_hadoop.outputs.hadoop_IP }}:9000\\\",\\\"dfs.replication\\\":1}] + - name: run integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:file-based-io-tests:integrationTest + arguments: | + --tests org.apache.beam.sdk.io.parquet.ParquetIOIT \ + --info \ + -Dfilesystem=hdfs \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_ParquetIOIT_HDFS_test_arguments_1 }}]' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml b/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml new file mode 100644 index 0000000000000..4f652caa74c19 --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests PubsubIOIT Python Streaming + +on: + schedule: + - cron: '11 2 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_PubsubIOIT_Python_Streaming_test_arguments_1: '' + +jobs: + beam_PerformanceTests_PubsubIOIT_Python_Streaming: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run PubsubIO Performance Test Python' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_PubsubIOIT_Python_Streaming"] + job_phrase: ["Run PubsubIO Performance Test Python"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/pubsubIOIT_Python_Streaming.txt + arguments: | + --job_name=performance-tests-psio-python-2gb$(date '+%m%d%H%M%S' --utc) + - name: Run test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.io.gcp.pubsub_io_perf_test \ + -Prunner=TestDataflowRunner \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_PerformanceTests_PubsubIOIT_Python_Streaming_test_arguments_1 }}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml b/.github/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml new file mode 100644 index 0000000000000..370e038a6bcad --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests SQLBigQueryIO Batch Java + +on: + schedule: + - cron: '0 7,19 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_SQLBigQueryIO_Batch_Java_test_arguments_1: '' + +jobs: + beam_PerformanceTests_SQLBigQueryIO_Batch_Java: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run SQLBigQueryIO Batch Performance Test Java' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_SQLBigQueryIO_Batch_Java"] + job_phrase: ["Run SQLBigQueryIO Batch Performance Test Java"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/SQLBigQueryIO_Batch_Java.txt + - name: Run SQLBigQueryIO Batch Performance Test Java + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:extensions:sql:perf-tests:integrationTest + arguments: | + --tests org.apache.beam.sdk.extensions.sql.meta.provider.bigquery.BigQueryIOPushDownIT \ + -DintegrationTestRunner=dataflow \ + '-DintegrationTestPipelineOptions=[${{env.beam_PerformanceTests_SQLBigQueryIO_Batch_Java_test_arguments_1}}]' \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml b/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml new file mode 100644 index 0000000000000..4a951e720e374 --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests SpannerIO Read 2GB Python + +on: + schedule: + - cron: '30 4 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_SpannerIO_Read_2GB_Python_test_arguments_1: '' + +jobs: + beam_PerformanceTests_SpannerIO_Read_2GB_Python: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run SpannerIO Read 2GB Performance Test Python' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 480 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_SpannerIO_Read_2GB_Python"] + job_phrase: ["Run SpannerIO Read 2GB Performance Test Python"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/spannerIO_Read_2GB_Python.txt + arguments: | + --job_name=performance-tests-spanner-read-python-2gb$(date '+%m%d%H%M%S' --utc) + - name: run Performance SpannerIO Read 2GB Test Python + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.io.gcp.experimental.spannerio_read_perf_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + -PloadTest.args='${{env.beam_PerformanceTests_SpannerIO_Read_2GB_Python_test_arguments_1}}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml b/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml new file mode 100644 index 0000000000000..17d4b1cb9072f --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests SpannerIO Write 2GB Python Batch + +on: + schedule: + - cron: '0 5 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch_test_arguments_1: '' + +jobs: + beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run SpannerIO Write 2GB Performance Test Python Batch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 480 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch"] + job_phrase: ["Run SpannerIO Write 2GB Performance Test Python Batch"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/spannerIO_Write_2GB_Python.txt + arguments: | + --job_name=performance-tests-spannerio-write-python-batch-2gb$(date '+%m%d%H%M%S' --utc) + - name: run Performance SpannerIO Write 2GB Test Python + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.io.gcp.experimental.spannerio_write_perf_test \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.8 \ + -PloadTest.args='${{env.beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch_test_arguments_1}}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_SparkReceiver_IO.yml b/.github/workflows/beam_PerformanceTests_SparkReceiver_IO.yml new file mode 100644 index 0000000000000..5a8a26ca1f61f --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_SparkReceiver_IO.yml @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests SparkReceiver IO + +on: + schedule: + - cron: '15 6/12 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_SparkReceiver_IO_test_arguments_1: '' + +jobs: + beam_PerformanceTests_SparkReceiver_IO: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java SparkReceiverIO Performance Test' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_SparkReceiver_IO"] + job_phrase: ["Run Java SparkReceiverIO Performance Test"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Set k8s access + uses: ./.github/actions/setup-k8s-access + with: + cluster_name: io-datastores + k8s_namespace: ${{ matrix.job_name }}-${{ github.run_id }} + - name: Install rabbitmq + id: install_rabbitmq + run: | + kubectl apply -f ${{ github.workspace }}/.test-infra/kubernetes/rabbit/rabbitmq.yaml + kubectl wait svc/rabbitmq --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=60s + loadbalancer_IP=$(kubectl get svc rabbitmq -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo rabbitmq_IP=$loadbalancer_IP >> $GITHUB_OUTPUT + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/sparkReceiver_IO.txt + arguments: | + --rabbitMqBootstrapServerAddress=amqp://guest:guest@${{ steps.install_rabbitmq.outputs.rabbitmq_IP }}:5672 + - name: run integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:sparkreceiver:2:integrationTest + arguments: | + --info \ + --tests org.apache.beam.sdk.io.sparkreceiver.SparkReceiverIOIT \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_SparkReceiver_IO_test_arguments_1 }}]' \ diff --git a/.github/workflows/beam_PerformanceTests_TFRecordIOIT.yml b/.github/workflows/beam_PerformanceTests_TFRecordIOIT.yml new file mode 100644 index 0000000000000..8c1f89b999a5b --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_TFRecordIOIT.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests TFRecordIOIT + +on: + schedule: + - cron: '10 4/12 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_TFRecordIOIT_test_arguments_1: '' + +jobs: + beam_PerformanceTests_TFRecordIOIT: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java TFRecordIO Performance Test' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_TFRecordIOIT"] + job_phrase: ["Run Java TFRecordIO Performance Test"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_TFRecordIOIT.txt + arguments: | + --filenamePrefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}}/ + - name: run integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:file-based-io-tests:integrationTest + arguments: | + --tests org.apache.beam.sdk.io.tfrecord.TFRecordIOIT \ + --info \ + -Dfilesystem=gcs \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_TFRecordIOIT_test_arguments_1 }}]' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_TFRecordIOIT_HDFS.yml b/.github/workflows/beam_PerformanceTests_TFRecordIOIT_HDFS.yml new file mode 100644 index 0000000000000..0e0da80e62a84 --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_TFRecordIOIT_HDFS.yml @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests TFRecordIOIT HDFS + +on: + # TODO(https://github.com/apache/beam/issues/18796) TFRecord performance test is failing only when running on hdfs. + # We need to fix this before enabling this job on jenkins. + # schedule: + # - cron: '17 8/20 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_TFRecordIOIT_HDFS_test_arguments_1: '' + +jobs: + beam_PerformanceTests_TFRecordIOIT_HDFS: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java TFRecordIO Performance Test HDFS' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_TFRecordIOIT_HDFS"] + job_phrase: ["Run Java TFRecordIO Performance Test HDFS"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Set k8s access + uses: ./.github/actions/setup-k8s-access + with: + cluster_name: io-datastores + k8s_namespace: ${{ matrix.job_name }}-${{ github.run_id }} + - name: Install Hadoop + id: install_hadoop + run: | + kubectl apply -f ${{ github.workspace }}/.test-infra/kubernetes/hadoop/LargeITCluster/hdfs-multi-datanode-cluster.yml + kubectl wait svc/hadoop --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=60s + loadbalancer_IP=$(kubectl get svc hadoop -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo hadoop_IP=$loadbalancer_IP >> $GITHUB_OUTPUT + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/TFRecordIOIT_HDFS.txt + arguments: | + --filenamePrefix=hdfs://${{ steps.install_hadoop.outputs.hadoop_IP }}:9000/TEXTIO_IT_ + --hdfsConfiguration=[{\\\"fs.defaultFS\\\":\\\"hdfs:${{ steps.install_hadoop.outputs.hadoop_IP }}:9000\\\",\\\"dfs.replication\\\":1}] + - name: run integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:file-based-io-tests:integrationTest + arguments: | + --info \ + --tests org.apache.beam.sdk.io.tfrecord.TFRecordIOIT \ + -Dfilesystem=hdfs \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_TFRecordIOIT_HDFS_test_arguments_1 }}]' \ \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_TextIOIT.yml b/.github/workflows/beam_PerformanceTests_TextIOIT.yml new file mode 100644 index 0000000000000..32bb4821208cb --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_TextIOIT.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests TextIOIT + +on: + schedule: + - cron: '0 7,19 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PerformanceTests_TextIOIT: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java TextIO Performance Test' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_TextIOIT"] + job_phrase: ["Run Java TextIO Performance Test"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/textIOIT.txt + arguments: | + --filenamePrefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}} + # The env variable is created and populated in the test-arguments-action as "beam_PerformanceTests_TextIOIT_test_arguments_1" + - name: Run Java TextIO Performance Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:file-based-io-tests:integrationTest + arguments: | + --tests org.apache.beam.sdk.io.text.TextIOIT \ + --info \ + -Dfilesystem=gcs \ + -DintegrationTestRunner=dataflow \ + '-DintegrationTestPipelineOptions=[${{env.beam_PerformanceTests_TextIOIT_test_arguments_1}}]' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_TextIOIT_HDFS.yml b/.github/workflows/beam_PerformanceTests_TextIOIT_HDFS.yml new file mode 100644 index 0000000000000..94cda3343d5ee --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_TextIOIT_HDFS.yml @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests TextIOIT HDFS + +on: + schedule: + - cron: '30 7,19 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PerformanceTests_TextIOIT_HDFS: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java TextIO Performance Test HDFS' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_TextIOIT_HDFS"] + job_phrase: ["Run Java TextIO Performance Test HDFS"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Set k8s access + uses: ./.github/actions/setup-k8s-access + with: + cluster_name: io-datastores + k8s_namespace: ${{ matrix.job_name }}-${{ github.run_id }} + - name: Install Hadoop + id: install_hadoop + run: | + kubectl apply -f ${{ github.workspace }}/.test-infra/kubernetes/hadoop/LargeITCluster/hdfs-multi-datanode-cluster.yml + kubectl wait svc/hadoop --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=120s + loadbalancer_IP=$(kubectl get svc hadoop -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo hadoop_IP=$loadbalancer_IP >> $GITHUB_OUTPUT + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/textIOIT_HDFS.txt + arguments: | + --filenamePrefix=hdfs://${{ steps.install_hadoop.outputs.hadoop_IP }}:9000/TEXTIO_IT_ + --hdfsConfiguration=[{\\\"fs.defaultFS\\\":\\\"hdfs:${{ steps.install_hadoop.outputs.hadoop_IP }}:9000\\\",\\\"dfs.replication\\\":1}] + # The env variable is created and populated in the test-arguments-action as "beam_PerformanceTests_TextIOIT_HDFS_test_arguments_1" + - name: Run Java TextIO Performance Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:file-based-io-tests:integrationTest + arguments: | + --tests org.apache.beam.sdk.io.text.TextIOIT \ + --info \ + -Dfilesystem=hdfs \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{env.beam_PerformanceTests_TextIOIT_HDFS_test_arguments_1}}]' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml b/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml new file mode 100644 index 0000000000000..f49a573c3d911 --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests TextIOIT Python + +on: + schedule: + - cron: '0 8,20 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PerformanceTests_TextIOIT_Python: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Python TextIO Performance Test' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_TextIOIT_Python"] + job_phrase: ["Run Python TextIO Performance Test"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/textIOIT_Python.txt + arguments: | + --filename_prefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}} + # The env variable is created and populated in the test-arguments-action as "beam_PerformanceTests_TextIOIT_Python_test_arguments_1" + - name: Run Python TextIO Performance Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PpythonVersion=3.8 \ + -PloadTest.mainClass=apache_beam.io.filebasedio_perf_test \ + -Prunner=DataflowRunner \ + '-PloadTest.args=${{env.beam_PerformanceTests_TextIOIT_Python_test_arguments_1}}' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml b/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml new file mode 100644 index 0000000000000..9e901cc819263 --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests WordCountIT PythonVersions + +on: + schedule: + - cron: '12 3 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_WordCountIT_PythonVersions_test_arguments_1: '' + +jobs: + beam_PerformanceTests_WordCountIT_PythonVersions: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + (startswith(github.event.comment.body, 'Run Python') && + endswith(github.event.comment.body, 'WordCountIT Performance Test')) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase_1 }} ${{ matrix.python_version }} ${{ matrix.job_phrase_2 }}) + strategy: + fail-fast: false + matrix: + job_name: ["beam_PerformanceTests_WordCountIT_PythonVersions"] + job_phrase_1: [Run Python] + job_phrase_2: [WordCountIT Performance Test] + python_version: ['3.8'] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase_1 }} ${{ matrix.python_version }} ${{ matrix.job_phrase_2 }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase_1 }} ${{ matrix.python_version }} ${{ matrix.job_phrase_2 }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: ${{ matrix.python_version }} + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/wordCountIT_Python.txt + arguments: | + --job_name=performance-tests-wordcount-python${{steps.set_py_ver_clean.outputs.py_ver_clean}}-batch-1gb$(date '+%m%d%H%M%S' --utc) + --metrics_table=wordcount_py${{steps.set_py_ver_clean.outputs.py_ver_clean}}_pkb_results + --influx_measurement=wordcount_py${{steps.set_py_ver_clean.outputs.py_ver_clean}}_results + - name: Run test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:dataflow:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:runPerformanceTest + arguments: | + --info \ + -Ptest=apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it \ + "-Ptest-pipeline-options=${{ env.beam_PerformanceTests_WordCountIT_PythonVersions_test_arguments_1 }}" + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_XmlIOIT.yml b/.github/workflows/beam_PerformanceTests_XmlIOIT.yml new file mode 100644 index 0000000000000..43451bc095eb4 --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_XmlIOIT.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests XmlIOIT + +on: + schedule: + - cron: '30 4/12 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_XmlIOIT_test_arguments_1: '' + +jobs: + beam_PerformanceTests_XmlIOIT: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java XmlIO Performance Test' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_XmlIOIT"] + job_phrase: ["Run Java XmlIO Performance Test"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_XmlIOIT.txt + arguments: | + --filenamePrefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}}/ + - name: run integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:file-based-io-tests:integrationTest + arguments: | + --tests org.apache.beam.sdk.io.xml.XmlIOIT \ + --info \ + -Dfilesystem=gcs \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_XmlIOIT_test_arguments_1 }}]' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_XmlIOIT_HDFS.yml b/.github/workflows/beam_PerformanceTests_XmlIOIT_HDFS.yml new file mode 100644 index 0000000000000..32ffbd63cc794 --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_XmlIOIT_HDFS.yml @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests XmlIOIT HDFS + +on: + schedule: + - cron: '50 4/12 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + beam_PerformanceTests_XmlIOIT_HDFS_test_arguments_1: '' + +jobs: + beam_PerformanceTests_XmlIOIT_HDFS: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java XmlIO Performance Test HDFS' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_XmlIOIT_HDFS"] + job_phrase: ["Run Java XmlIO Performance Test HDFS"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Set k8s access + uses: ./.github/actions/setup-k8s-access + with: + k8s_namespace: ${{ matrix.job_name }}-${{ github.run_id }} + - name: Install Hadoop + id: install_hadoop + run: | + kubectl apply -f ${{ github.workspace }}/.test-infra/kubernetes/hadoop/LargeITCluster/hdfs-multi-datanode-cluster.yml + kubectl wait svc/hadoop --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=120s + loadbalancer_IP=$(kubectl get svc hadoop -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo hadoop_IP=$loadbalancer_IP >> $GITHUB_OUTPUT + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: performance + test-language: java + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/config_XmlIOIT_HDFS.txt + arguments: | + --filenamePrefix=hdfs://${{ steps.install_hadoop.outputs.hadoop_IP }}:9000/TEXTIO_IT_ + --hdfsConfiguration=[{\\\"fs.defaultFS\\\":\\\"hdfs:${{ steps.install_hadoop.outputs.hadoop_IP }}:9000\\\",\\\"dfs.replication\\\":1}] + - name: run integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:file-based-io-tests:integrationTest + arguments: | + --tests org.apache.beam.sdk.io.xml.XmlIOIT \ + --info \ + -Dfilesystem=hdfs \ + -DintegrationTestRunner=dataflow \ + -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_XmlIOIT_HDFS_test_arguments_1 }}]' \ No newline at end of file diff --git a/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml b/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml new file mode 100644 index 0000000000000..4d830223a5fa2 --- /dev/null +++ b/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PerformanceTests xlang KafkaIO Python + +on: + schedule: + - cron: '10 5 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PerformanceTests_xlang_KafkaIO_Python: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Python xlang KafkaIO Performance Test' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PerformanceTests_xlang_KafkaIO_Python"] + job_phrase: ["Run Python xlang KafkaIO Performance Test"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Set k8s access + uses: ./.github/actions/setup-k8s-access + with: + cluster_name: io-datastores-for-tests + k8s_namespace: ${{ matrix.job_name }}-${{ github.run_id }} + - name: Install Kafka + id: install_kafka + run: | + cd ${{ github.workspace }}/.test-infra/kubernetes/kafka-cluster/ + kubectl apply -R -f . + - name: Get Kafka IP + id: kafka_ip + run: | + kubectl wait svc/outside-0 --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=120s + kubectl wait svc/outside-1 --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=120s + kubectl wait svc/outside-2 --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=120s + KAFKA_BROKER_0_IP=$(kubectl get svc outside-0 -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + KAFKA_BROKER_1_IP=$(kubectl get svc outside-1 -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + KAFKA_BROKER_2_IP=$(kubectl get svc outside-2 -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo KAFKA_BROKER_0=$KAFKA_BROKER_0_IP >> $GITHUB_OUTPUT + echo KAFKA_BROKER_1=$KAFKA_BROKER_1_IP >> $GITHUB_OUTPUT + echo KAFKA_BROKER_2=$KAFKA_BROKER_2_IP >> $GITHUB_OUTPUT + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/performance-tests-job-configs/xlang_KafkaIO_Python.txt + arguments: | + --filename_prefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}}/ + --bootstrap_servers=${{ steps.kafka_ip.outputs.KAFKA_BROKER_0 }}:32400,${{ steps.kafka_ip.outputs.KAFKA_BROKER_1 }}:32400,${{ steps.kafka_ip.outputs.KAFKA_BROKER_2 }}:32400 + - name: run shadowJar + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:expansion-service:shadowJar + # The env variable is created and populated in the test-arguments-action as "_test_arguments_" + - name: run integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -Prunner=DataflowRunner \ + -PloadTest.mainClass=apache_beam.io.external.xlang_kafkaio_perf_test \ + -PpythonVersion=3.8 \ + '-PloadTest.args=${{ env.beam_PerformanceTests_xlang_KafkaIO_Python_test_arguments_1 }}' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_BeamMetrics_Publish.yml b/.github/workflows/beam_PostCommit_BeamMetrics_Publish.yml new file mode 100644 index 0000000000000..177704a179a17 --- /dev/null +++ b/.github/workflows/beam_PostCommit_BeamMetrics_Publish.yml @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit BeamMetrics Publish + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: ['.github/workflows/beam_PostCommit_BeamMetrics_Publish.yml'] + schedule: + - cron: '24 2 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_BeamMetrics_Publish: + if: | + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Beam Metrics Deployment' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_BeamMetrics_Publish"] + job_phrase: ["Run Beam Metrics Deployment"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Setup credential helper + run: | + gcloud auth configure-docker + - name: Install gcloud Kubectl + run: gcloud components install kubectl + - name: run Beam Metrics deployment script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :beam-test-infra-metrics:getClusterCredentials :beam-test-infra-metrics:deploy + arguments: | + -PKUBE_CONFIG_PATH='$HOME/.kube/config' diff --git a/.github/workflows/beam_PostCommit_Go.yml b/.github/workflows/beam_PostCommit_Go.yml new file mode 100644 index 0000000000000..e373d4f623bc8 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Go.yml @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Go + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Go: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Go PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 300 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Go"] + job_phrase: ["Run Go PostCommit"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Authenticate on GCP + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + export_default_credentials: true + - name: GCloud Docker credential helper + run: | + gcloud auth configure-docker us.gcr.io + - name: run Go PostCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + env: + USER: github-actions + with: + gradle-command: :goPostCommit + arguments: | + --no-parallel \ + -Pdocker-repository-root=us.gcr.io/apache-beam-testing/github-actions \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml new file mode 100644 index 0000000000000..4a2f00a646686 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Go Dataflow ARM + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: ['model/**', 'sdks/go.**', 'release/**', '.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml'] + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Go_Dataflow_ARM: + if: | + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Go PostCommit Dataflow ARM' + runs-on: [self-hosted, ubuntu-20.04, main] + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Go_Dataflow_ARM"] + job_phrase: ["Run Go PostCommit Dataflow ARM"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + go-version: 1.21 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Authenticate on GCP + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + export_default_credentials: true + - name: GCloud Docker credential helper + run: | + gcloud auth configure-docker us.gcr.io + - name: run goPostCommitDataflowARM script + run: ./gradlew :goPostCommitDataflowARM + env: + USER: github-actions diff --git a/.github/workflows/beam_PostCommit_Go_VR_Flink.yml b/.github/workflows/beam_PostCommit_Go_VR_Flink.yml new file mode 100644 index 0000000000000..6f43cd461f25d --- /dev/null +++ b/.github/workflows/beam_PostCommit_Go_VR_Flink.yml @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Go VR Flink + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Go_VR_Flink: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Go Flink ValidatesRunner' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Go_VR_Flink"] + job_phrase: ["Run Go Flink ValidatesRunner"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Go Flink ValidatesRunner script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:flinkValidatesRunner \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Go_VR_Samza.yml b/.github/workflows/beam_PostCommit_Go_VR_Samza.yml new file mode 100644 index 0000000000000..f317615914ce4 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Go_VR_Samza.yml @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Go VR Samza + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Go_VR_Samza: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Go Samza ValidatesRunner' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Go_VR_Samza"] + job_phrase: ["Run Go Samza ValidatesRunner"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Go Samza ValidatesRunner script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:samzaValidatesRunner \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Go_VR_Spark.yml b/.github/workflows/beam_PostCommit_Go_VR_Spark.yml new file mode 100644 index 0000000000000..98abb1004b91b --- /dev/null +++ b/.github/workflows/beam_PostCommit_Go_VR_Spark.yml @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Go VR Spark + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Go_VR_Spark: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Go Spark ValidatesRunner' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Go_VR_Spark"] + job_phrase: ["Run Go Spark ValidatesRunner"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Go Spark ValidatesRunner script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:go:test:sparkValidatesRunner + \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java.yml b/.github/workflows/beam_PostCommit_Java.yml new file mode 100644 index 0000000000000..fb029e8402fa1 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java.yml @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + strategy: + matrix: + job_name: [beam_PostCommit_Java] + job_phrase: [Run Java PostCommit] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java PostCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit Java script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :javaPostCommit + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Avro_Versions.yml b/.github/workflows/beam_PostCommit_Java_Avro_Versions.yml new file mode 100644 index 0000000000000..ef451b85f7289 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Avro_Versions.yml @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Avro Versions + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_Avro_Versions: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + strategy: + matrix: + job_name: [beam_PostCommit_Java_Avro_Versions] + job_phrase: [Run Java Avro Versions PostCommit] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java Avro Versions PostCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit Java Avro Versions script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :javaAvroVersionsTest + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml b/.github/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml new file mode 100644 index 0000000000000..b281e550e562b --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java BigQueryEarlyRollout + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: write + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_BigQueryEarlyRollout: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + strategy: + matrix: + job_name: [beam_PostCommit_Java_BigQueryEarlyRollout] + job_phrase: [Run Java BigQueryEarlyRollout PostCommit] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java BigQueryEarlyRollout PostCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + export_default_credentials: true + - name: run PostCommit Java BigQueryEarlyRollout script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:google-cloud-platform:bigQueryEarlyRolloutIntegrationTest + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_DataflowV1.yml b/.github/workflows/beam_PostCommit_Java_DataflowV1.yml new file mode 100644 index 0000000000000..7515da1cad7bb --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_DataflowV1.yml @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Dataflow V1 + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_DataflowV1: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + strategy: + matrix: + job_name: [beam_PostCommit_Java_DataflowV1] + job_phrase: [Run PostCommit_Java_Dataflow] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run PostCommit_Java_Dataflow' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Set up Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'temurin' + java-version: | + 11 + 8 + - name: run PostCommit Java Dataflow V1 script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:postCommit + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_DataflowV2.yml b/.github/workflows/beam_PostCommit_Java_DataflowV2.yml new file mode 100644 index 0000000000000..2360feeb7d3f2 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_DataflowV2.yml @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Dataflow V2 + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_DataflowV2: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + strategy: + matrix: + job_name: [beam_PostCommit_Java_DataflowV2] + job_phrase: [Run PostCommit_Java_DataflowV2] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run PostCommit_Java_DataflowV2' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit Java Dataflow V2 script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:postCommitRunnerV2 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow.yml new file mode 100644 index 0000000000000..a1e2a00890294 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Examples Dataflow + +on: + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_Examples_Dataflow_V2: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 180 + strategy: + matrix: + job_name: [beam_PostCommit_Java_Examples_Dataflow_V1] + job_phrase: [Run Java examples on Dataflow] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java Examples on Dataflow' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit Java Examples Dataflow script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:examples:javaPostCommit + max-workers: 12 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml new file mode 100644 index 0000000000000..c4d414fc32e2c --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml @@ -0,0 +1,135 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Examples Dataflow ARM + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - 'model/**' + - 'sdks/java/**' + - 'runners/google-cloud-dataflow-java/**' + - 'examples/java/**' + - 'examples/kotlin/**' + - 'release/**' + - '.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml' + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +jobs: + beam_PostCommit_Java_Examples__Dataflow_ARM: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.java_version }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + job_name: [beam_PostCommit_Java_Examples__Dataflow_ARM] + job_phrase: [Run Java_Examples_Dataflow_ARM PostCommit] + java_version: ['8','11','17'] + if: | + github.event_name == 'push' || + github.event_name == 'schedule' || + github. event_name == 'workflow_dispatch' || + startswith(github.event.comment.body, 'Run Java_Examples_Dataflow_ARM PostCommit') + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.java_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.java_version }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: ${{ matrix.java_version }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Authenticate on GCP + id: auth + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + export_default_credentials: true + - name: GCloud Docker credential helper + run: | + gcloud auth configure-docker us.gcr.io + - name: Generate TAG unique variable based on timestamp + id: set_tag + run: echo "TAG=$(date +'%Y%m%d-%H%M%S%N')" >> $GITHUB_OUTPUT + - name: run javaExamplesDataflowARMPostCommit script for Java${{ matrix.java_version }} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:arm:examplesJavaRunnerV2IntegrationTestARM + max-workers: 12 + arguments: | + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -Pcontainer-architecture-list=arm64,amd64 \ + -Ppush-containers \ + -Pdocker-repository-root=us.gcr.io/apache-beam-testing/github-actions \ + -Pdocker-tag=${{ steps.set_tag.outputs.TAG }} \ + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + -PskipCheckerFramework \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml new file mode 100644 index 0000000000000..f9a87b26ab83f --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Examples Dataflow Java + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_Examples_Dataflow_Java: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.java_version }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + job_name: [beam_PostCommit_Java_Examples_Dataflow_Java] + job_phrase: [Run Java examples on Dataflow Java] + java_version: ['11','17'] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + startswith(github.event.comment.body, 'Run Java examples on Dataflow Java') + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.java_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.java_version }}) + - name: Set up Java${{ matrix.java_version }} + uses: actions/setup-java@v3.8.0 + with: + distribution: 'temurin' + java-version: | + ${{ matrix.java_version }} + 8 + - name: run java${{ matrix.java_version }}PostCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:examples:java${{ matrix.java_version }}PostCommit + max-workers: 12 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml new file mode 100644 index 0000000000000..28f6ae4d20539 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Examples Dataflow V2 + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_Examples_Dataflow_V2: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 180 + strategy: + matrix: + job_name: [beam_PostCommit_Java_Examples_Dataflow_V2] + job_phrase: [Run Java Examples on Dataflow Runner V2] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java Examples on Dataflow Runner V2' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit Java Examples Dataflow V2 script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:examplesJavaRunnerV2IntegrationTest + max-workers: 12 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml new file mode 100644 index 0000000000000..16f48df032925 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml @@ -0,0 +1,103 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Examples Dataflow V2 Java + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_Examples_Dataflow_V2_Java: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase_1 }} ${{ matrix.java_version }} ${{ matrix.job_phrase_2 }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + job_name: [beam_PostCommit_Java_Examples_Dataflow_V2_Java] + job_phrase_1: [Run Java ] + job_phrase_2: [Examples on Dataflow Runner V2] + java_version: ['11', '17'] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + (contains(github.event.comment.body, 'Run Java') && + contains(github.event.comment.body, 'Examples on Dataflow Runner V2')) + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase_1 }} ${{ matrix.java_version }} ${{ matrix.job_phrase_2 }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase_1 }} ${{ matrix.java_version }} ${{ matrix.job_phrase_2 }}) + - name: Set up Java${{ matrix.java_version }} + uses: actions/setup-java@v3.8.0 + with: + distribution: 'temurin' + java-version: ${{ matrix.java_version }} + - name: run PostCommit Java Examples Dataflow V2 Java${{ matrix.java_version }} script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:examplesJavaRunnerV2IntegrationTest + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + -PskipCheckerFramework \ + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Direct.yml b/.github/workflows/beam_PostCommit_Java_Examples_Direct.yml new file mode 100644 index 0000000000000..3290937312733 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Examples_Direct.yml @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Examples Direct + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_Examples_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + strategy: + matrix: + job_name: [beam_PostCommit_Java_Examples_Direct] + job_phrase: [Run Java Examples_Direct] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java Examples_Direct' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'zulu' + java-version: '8' + - name: run examplesIntegrationTest script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:direct:examplesIntegrationTest + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Flink.yml b/.github/workflows/beam_PostCommit_Java_Examples_Flink.yml new file mode 100644 index 0000000000000..db368031d9f05 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Examples_Flink.yml @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Examples Flink + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_Examples_Flink: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + strategy: + matrix: + job_name: [beam_PostCommit_Java_Examples_Flink] + job_phrase: [Run Java Examples_Flink] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java Examples_Flink' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: 3.8 + - name: run examplesIntegrationTest script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:flink:1.15:examplesIntegrationTest + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Spark.yml b/.github/workflows/beam_PostCommit_Java_Examples_Spark.yml new file mode 100644 index 0000000000000..f7d12a9860da3 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Examples_Spark.yml @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Examples Spark + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_Examples_Spark: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + strategy: + matrix: + job_name: [beam_PostCommit_Java_Examples_Spark] + job_phrase: [Run Java Examples_Spark] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java Examples_Spark' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'zulu' + java-version: '8' + - name: run examplesIntegrationTest script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:spark:3:examplesIntegrationTest + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml b/.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml new file mode 100644 index 0000000000000..65ade4bf63d4e --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java Hadoop Versions + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_Hadoop_Versions: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + strategy: + matrix: + job_name: [beam_PostCommit_Java_Hadoop_Versions] + job_phrase: [Run PostCommit_Java_Hadoop_Versions] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run PostCommit_Java_Hadoop_Versions' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'zulu' + java-version: '8' + - name: run validatesRunner script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :javaHadoopVersionsTest + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml b/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml new file mode 100644 index 0000000000000..649e48fb889db --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java IO Performance Tests + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: ['it/google-cloud-platform/**','.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml'] + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_IO_Performance_Tests: + if: | + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java PostCommit IO Performance Tests' + runs-on: [self-hosted, ubuntu-20.04, main] + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.test_case }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Java_IO_Performance_Tests"] + job_phrase: ["Run Java PostCommit IO Performance Tests"] + test_case: ["GCSPerformanceTest", "BigTablePerformanceTest", "BigQueryStorageApiStreamingPerformanceTest"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.test_case }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.test_case }}) + - name: Checkout release branch + if: github.event_name == 'schedule' #This has scheduled runs run against the latest release + uses: actions/checkout@v4 + with: + ref: v2.50.0 #TODO(https://github.com/apache/beam/issues/28330) automate updating this + repository: apache/beam + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + - name: Authenticate on GCP + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + export_default_credentials: true + - name: run scheduled javaPostcommitIOPerformanceTests script + if: github.event_name == 'schedule' #This ensures only scheduled runs publish metrics publicly by changing which exportTable is configured + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :it:${{ matrix.test_case }} + env: + exportDataset: performance_tests + exportTable: io_performance_metrics + - name: run triggered javaPostcommitIOPerformanceTests script + if: github.event_name != 'schedule' + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :it:${{ matrix.test_case }} + env: + exportDataset: performance_tests + exportTable: io_performance_metrics_test + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml b/.github/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml new file mode 100644 index 0000000000000..8d834a247e6bd --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Java InfluxDbIO Integration Test + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +jobs: + beam_PostCommit_Java_InfluxDbIO_IT: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + strategy: + matrix: + job_name: [beam_PostCommit_Java_InfluxDbIO_IT] + job_phrase: [Run Java InfluxDbIO_IT] + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java InfluxDbIO_IT' + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Set k8s access + uses: ./.github/actions/setup-k8s-access + with: + cluster_name: io-datastores + k8s_namespace: ${{ matrix.job_name }}-${{ github.run_id }} + - name: Install InfluxDB + id: install_influxdb + run: | + kubectl apply -f ${{ github.workspace }}/.test-infra/kubernetes/influxdb/influxdb.yml + kubectl wait svc/influxdb-load-balancer-service --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' + loadbalancer_IP=$(kubectl get svc influxdb-load-balancer-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo influxdb_IP=$loadbalancer_IP >> $GITHUB_OUTPUT + - name: Run Java InfluxDbIO_IT + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:influxdb:integrationTest --tests org.apache.beam.sdk.io.influxdb.InfluxDbIOIT + arguments: --info -DintegrationTestRunner=direct -DintegrationTestPipelineOptions='["--influxDBURL=http://${{ steps.install_influxdb.outputs.influxdb_IP }}:8086","--influxDBUserName=superadmin","--influxDBPassword=supersecretpassword","--databaseName=db1"]' diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml new file mode 100644 index 0000000000000..4c52984bcaa39 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java Jpms Dataflow Java11 + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_Jpms_Dataflow_Java11: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Jpms Dataflow Java 11 PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Java_Jpms_Dataflow_Java11"] + job_phrase: ["Run Jpms Dataflow Java 11 PostCommit"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Set up Java 11 + uses: actions/setup-java@v3.11.0 + with: + distribution: 'temurin' + java-version: '11' + - name: run PostCommit Java Jpms Dataflow Java11 script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:jpms-tests:dataflowRunnerIntegrationTest + arguments: -Dorg.gradle.java.home=$JAVA_HOME_11_X64 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml new file mode 100644 index 0000000000000..404292a834026 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java Jpms Dataflow Java17 + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_Jpms_Dataflow_Java17: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Jpms Dataflow Java 17 PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Java_Jpms_Dataflow_Java17"] + job_phrase: ["Run Jpms Dataflow Java 17 PostCommit"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Set up Java + uses: actions/setup-java@v3.11.0 + with: + distribution: 'temurin' + java-version: | + 17 + 8 + - name: run PostCommit Java Jpms Dataflow Java17 script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:jpms-tests:dataflowRunnerIntegrationTest + arguments: + -PskipCheckerFramework + -PcompileAndRunTestsWithJava17 + -Pjava17Home=$JAVA_HOME_17_X64 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml new file mode 100644 index 0000000000000..5046430d31b67 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java Jpms Direct Java11 + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_Jpms_Direct_Java11: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Jpms Direct Java 11 PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Java_Jpms_Direct_Java11"] + job_phrase: ["Run Jpms Direct Java 11 PostCommit"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Set up Java 11 + uses: actions/setup-java@v3.11.0 + with: + distribution: 'temurin' + java-version: '11' + - name: run PostCommit Java Jpms Direct Java11 script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:jpms-tests:directRunnerIntegrationTest + arguments: -Dorg.gradle.java.home=$JAVA_HOME_11_X64 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml new file mode 100644 index 0000000000000..8c4a78aea6639 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java Jpms Direct Java17 + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_Jpms_Direct_Java17: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Jpms Direct Java 17 PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Java_Jpms_Direct_Java17"] + job_phrase: ["Run Jpms Direct Java 17 PostCommit"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Set up Java + uses: actions/setup-java@v3.11.0 + with: + distribution: 'temurin' + java-version: | + 17 + 8 + - name: run PostCommit Java Jpms Direct Java17 script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:jpms-tests:directRunnerIntegrationTest + arguments: + -PskipCheckerFramework + -PcompileAndRunTestsWithJava17 + -Pjava17Home=$JAVA_HOME_17_X64 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml new file mode 100644 index 0000000000000..2dcf6a4444f5e --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java Jpms Flink Java11 + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_Jpms_Flink_Java11: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Jpms Flink Java 11 PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Java_Jpms_Flink_Java11"] + job_phrase: ["Run Jpms Flink Java 11 PostCommit"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Set up Java 11 + uses: actions/setup-java@v3.11.0 + with: + distribution: 'temurin' + java-version: '11' + - name: run PostCommit Java Jpms Flink Java11 script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:jpms-tests:flinkRunnerIntegrationTest + arguments: -Dorg.gradle.java.home=$JAVA_HOME_11_X64 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml new file mode 100644 index 0000000000000..01680a55026a8 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java Jpms Spark Java11 + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_Jpms_Spark_Java11: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Jpms Spark Java 11 PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Java_Jpms_Spark_Java11"] + job_phrase: ["Run Jpms Spark Java 11 PostCommit"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Set up Java 11 + uses: actions/setup-java@v3.11.0 + with: + distribution: 'temurin' + java-version: '11' + - name: run PostCommit Java Jpms Spark Java11 script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:jpms-tests:sparkRunnerIntegrationTest + arguments: -Dorg.gradle.java.home=$JAVA_HOME_11_X64 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml b/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml new file mode 100644 index 0000000000000..99e5fb8e677be --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Nexmark Dataflow + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GRADLE_COMMAND_ARGUMENTS: | + -Pnexmark.args=--manageResources=false + --monitorJobs=true + --bigQueryTable=nexmark + --bigQueryDataset=nexmark + --project=apache-beam-testing + --resourceNameMode=QUERY_RUNNER_AND_MODE + --exportSummaryToBigQuery=true + --tempLocation=gs://temp-storage-for-perf-tests/nexmark + --influxDatabase=beam_test_metrics + --influxHost=http://10.128.0.96:8086 + --baseInfluxMeasurement=nexmark + --exportSummaryToInfluxDB=true + --influxRetentionPolicy=forever + --region=us-central1 + --suite=STRESS + --numWorkers=4 + --maxNumWorkers=4 + --autoscalingAlgorithm=NONE + --nexmarkParallel=16 + --enforceEncodability=true + --enforceImmutability=true + --runner=DataflowRunner + +jobs: + beam_PostCommit_Java_Nexmark_Dataflow: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: + job_name: [beam_PostCommit_Java_Nexmark_Dataflow] + job_phrase: [Run Dataflow Runner Nexmark Tests] + streaming: [false, true] + queryLanguage: [sql, zetasql, none] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Dataflow Runner Nexmark Tests' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit Java Nexmark Dataflow (${{ matrix.streaming }} ${{ matrix.queryLanguage }}) script + if: matrix.queryLanguage != 'none' + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:nexmark:run + arguments: | + -Pnexmark.runner=:runners:google-cloud-dataflow-java \ + "${{ env.GRADLE_COMMAND_ARGUMENTS }}--streaming=${{ matrix.streaming }} --queryLanguage=${{ matrix.queryLanguage }}" \ + - name: run PostCommit Java Nexmark Dataflow (${{ matrix.streaming }}) script + if: matrix.queryLanguage == 'none' + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:nexmark:run + arguments: | + -Pnexmark.runner=:runners:google-cloud-dataflow-java \ + "${{ env.GRADLE_COMMAND_ARGUMENTS }}--streaming=${{ matrix.streaming }}" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml b/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml new file mode 100644 index 0000000000000..aba92fd8bb07a --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Nexmark Dataflow V2 + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GRADLE_COMMAND_ARGUMENTS: | + -Pnexmark.args=--manageResources=false + --monitorJobs=true + --bigQueryTable=nexmark + --bigQueryDataset=nexmark + --project=apache-beam-testing + --resourceNameMode=QUERY_RUNNER_AND_MODE + --exportSummaryToBigQuery=false + --tempLocation=gs://temp-storage-for-perf-tests/nexmark + --influxDatabase=beam_test_metrics + --influxHost=http://10.128.0.96:8086 + --baseInfluxMeasurement=nexmark + --exportSummaryToInfluxDB=true + --influxRetentionPolicy=forever + --influxTags={"runnerVersion":"V2","javaVersion":"8"} + --region=us-central1 + --suite=STRESS + --numWorkers=4 + --maxNumWorkers=4 + --autoscalingAlgorithm=NONE + --nexmarkParallel=16 + --enforceEncodability=true + --enforceImmutability=true + --runner=DataflowRunner + +jobs: + beam_PostCommit_Java_Nexmark_Dataflow_V2: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: + job_name: [beam_PostCommit_Java_Nexmark_Dataflow_V2] + job_phrase: [Run Dataflow Runner V2 Nexmark Tests] + streaming: [false, true] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Dataflow Runner V2 Nexmark Tests' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit Java Nexmark Dataflow V2 (streaming = ${{ matrix.streaming }}) script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:nexmark:run + arguments: | + -Pnexmark.runner=:runners:google-cloud-dataflow-java \ + '${{ env.GRADLE_COMMAND_ARGUMENTS }}--streaming=${{ matrix.streaming }}' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml b/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml new file mode 100644 index 0000000000000..72e1f736e917a --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Nexmark Dataflow V2 Java + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GRADLE_COMMAND_ARGUMENTS: | + -Pnexmark.args=--manageResources=false + --monitorJobs=true + --bigQueryTable=nexmark + --bigQueryDataset=nexmark + --project=apache-beam-testing + --resourceNameMode=QUERY_RUNNER_AND_MODE + --exportSummaryToBigQuery=false + --tempLocation=gs://temp-storage-for-perf-tests/nexmark + --influxDatabase=beam_test_metrics + --influxHost=http://10.128.0.96:8086 + --baseInfluxMeasurement=nexmark + --exportSummaryToInfluxDB=true + --influxRetentionPolicy=forever + --region=us-central1 + --suite=STRESS + --numWorkers=4 + --maxNumWorkers=4 + --autoscalingAlgorithm=NONE + --nexmarkParallel=16 + --enforceEncodability=true + --enforceImmutability=true + --runner=DataflowRunner + +jobs: + beam_PostCommit_Java_Nexmark_Dataflow_V2_Java: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase_1 }} ${{ matrix.java_version }} ${{ matrix.job_phrase_2 }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: + job_name: [beam_PostCommit_Java_Nexmark_Dataflow_V2_Java] + job_phrase_1: [Run Dataflow Runner V2 Java] + job_phrase_2: [Nexmark Tests] + streaming: [false, true] + java_version: ['11','17'] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + (contains(github.event.comment.body, 'Run Dataflow Runner V2 Java') && + contains(github.event.comment.body, 'Nexmark Tests')) + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase_1 }} ${{ matrix.java_version }} ${{ matrix.job_phrase_2 }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase_1 }} ${{ matrix.java_version }} ${{ matrix.job_phrase_2 }}) + - name: Set up Java${{ matrix.java_version }} + uses: actions/setup-java@v3.8.0 + with: + distribution: 'temurin' + java-version: ${{ matrix.java_version }} + - name: run PostCommit Java ${{ matrix.java_version }} Nexmark Dataflow V2 (streaming = ${{ matrix.streaming }}) script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:nexmark:run + arguments: | + -PcompileAndRunTestsWithJava${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -Pnexmark.runner.version=V2 \ + -Pnexmark.runner=:runners:google-cloud-dataflow-java \ + '${{ env.GRADLE_COMMAND_ARGUMENTS }}--influxTags={"runnerVersion":"V2","javaVersion":"${{ matrix.java_version }}"}--streaming=${{ matrix.streaming }}' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Nexmark_Direct.yml b/.github/workflows/beam_PostCommit_Java_Nexmark_Direct.yml new file mode 100644 index 0000000000000..c2e7433f04789 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Nexmark_Direct.yml @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Nexmark Direct + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GRADLE_COMMAND_ARGUMENTS: | + -Pnexmark.args=--manageResources=false + --monitorJobs=true + --bigQueryTable=nexmark + --bigQueryDataset=nexmark + --project=apache-beam-testing + --resourceNameMode=QUERY_RUNNER_AND_MODE + --exportSummaryToBigQuery=true + --tempLocation=gs://temp-storage-for-perf-tests/nexmark + --influxDatabase=beam_test_metrics + --influxHost=http://10.128.0.96:8086 + --baseInfluxMeasurement=nexmark + --exportSummaryToInfluxDB=true + --influxRetentionPolicy=forever + --suite=SMOKE + --enforceEncodability=true + --enforceImmutability=true + --runner=DirectRunner + +jobs: + beam_PostCommit_Java_Nexmark_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: + job_name: [beam_PostCommit_Java_Nexmark_Direct] + job_phrase: [Run Direct Runner Nexmark Tests] + streaming: [false, true] + queryLanguage: [sql, zetasql, none] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Direct Runner Nexmark Tests' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit Java Nexmark Direct (${{ matrix.streaming }} ${{ matrix.queryLanguage }}) script + if: matrix.queryLanguage != 'none' + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:nexmark:run + arguments: | + -Pnexmark.runner=:runners:direct-java \ + "${{ env.GRADLE_COMMAND_ARGUMENTS }} --streaming=${{ matrix.streaming }} --queryLanguage=${{ matrix.queryLanguage }}" \ + - name: run PostCommit Java Nexmark Direct (${{ matrix.streaming }}) script + if: matrix.queryLanguage == 'none' + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:nexmark:run + arguments: | + -Pnexmark.runner=:runners:direct-java \ + "${{ env.GRADLE_COMMAND_ARGUMENTS }}--streaming=${{ matrix.streaming }}" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Nexmark_Flink.yml b/.github/workflows/beam_PostCommit_Java_Nexmark_Flink.yml new file mode 100644 index 0000000000000..e80c5a5b2802a --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Nexmark_Flink.yml @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Nexmark Flink + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GRADLE_COMMAND_ARGUMENTS: | + -Pnexmark.args=--manageResources=false + --monitorJobs=true + --bigQueryTable=nexmark + --bigQueryDataset=nexmark + --project=apache-beam-testing + --resourceNameMode=QUERY_RUNNER_AND_MODE + --exportSummaryToBigQuery=true + --tempLocation=gs://temp-storage-for-perf-tests/nexmark + --influxDatabase=beam_test_metrics + --influxHost=http://10.128.0.96:8086 + --baseInfluxMeasurement=nexmark + --exportSummaryToInfluxDB=true + --influxRetentionPolicy=forever + --suite=SMOKE + --streamTimeout=60 + --runner=FlinkRunner + +jobs: + beam_PostCommit_Java_Nexmark_Flink: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: + job_name: [beam_PostCommit_Java_Nexmark_Flink] + job_phrase: [Run Flink Runner Nexmark Tests] + streaming: [false, true] + queryLanguage: [sql, zetasql, none] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Flink Runner Nexmark Tests' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit Java Nexmark Flink (${{ matrix.streaming }} ${{ matrix.queryLanguage }}) script + if: matrix.queryLanguage != 'none' + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:nexmark:run + arguments: | + -Pnexmark.runner=:runners:flink:1.15 \ + "${{ env.GRADLE_COMMAND_ARGUMENTS }} --streaming=${{ matrix.streaming }} --queryLanguage=${{ matrix.queryLanguage }}" \ + - name: run PostCommit Java Nexmark Flink (${{ matrix.streaming }}) script + if: matrix.queryLanguage == 'none' + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:nexmark:run + arguments: | + -Pnexmark.runner=:runners:flink:1.15 \ + "${{ env.GRADLE_COMMAND_ARGUMENTS }}--streaming=${{ matrix.streaming }}" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Nexmark_Spark.yml b/.github/workflows/beam_PostCommit_Java_Nexmark_Spark.yml new file mode 100644 index 0000000000000..0ccd751ac980b --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Nexmark_Spark.yml @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Nexmark Spark + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GRADLE_COMMAND_ARGUMENTS: | + -Pnexmark.args=--manageResources=false + --monitorJobs=true + --bigQueryTable=nexmark + --bigQueryDataset=nexmark + --project=apache-beam-testing + --resourceNameMode=QUERY_RUNNER_AND_MODE + --exportSummaryToBigQuery=true + --tempLocation=gs://temp-storage-for-perf-tests/nexmark + --influxDatabase=beam_test_metrics + --influxHost=http://10.128.0.96:8086 + --baseInfluxMeasurement=nexmark + --exportSummaryToInfluxDB=true + --influxRetentionPolicy=forever + --suite=SMOKE + --streamTimeout=60 + --streaming=false + +jobs: + beam_PostCommit_Java_Nexmark_Spark: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: + job_name: [beam_PostCommit_Java_Nexmark_Spark] + job_phrase: [Run Spark Runner Nexmark Tests] + runner: [SparkRunner, SparkStructuredStreamingRunner --skipQueries=3] + queryLanguage: [sql, zetasql, none] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Spark Runner Nexmark Tests' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit Java Nexmark Spark (runner = ${{ matrix.runner }} queryLanguage = ${{ matrix.queryLanguage }}) script + if: matrix.queryLanguage != 'none' + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:nexmark:run + arguments: | + -Pnexmark.runner=:runners:spark:3 \ + "${{ env.GRADLE_COMMAND_ARGUMENTS }} --runner=${{ matrix.runner }} --queryLanguage=${{ matrix.queryLanguage }}" \ + - name: run PostCommit Java Nexmark Spark (${{ matrix.runner }}) script + if: matrix.queryLanguage == 'none' + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:nexmark:run + arguments: | + -Pnexmark.runner=:runners:spark:3 \ + "${{ env.GRADLE_COMMAND_ARGUMENTS }}--runner=${{ matrix.runner }}" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml b/.github/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml new file mode 100644 index 0000000000000..209fb2acf23ae --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java PVR Flink Streaming + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_PVR_Flink_Streaming: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + strategy: + matrix: + job_name: [beam_PostCommit_Java_PVR_Flink_Streaming] + job_phrase: [Run Java Flink PortableValidatesRunner Streaming] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java Flink PortableValidatesRunner Streaming' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit Java Flink PortableValidatesRunner Streaming script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: runners:flink:1.15:job-server:validatesPortableRunnerStreaming + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_PVR_Samza.yml b/.github/workflows/beam_PostCommit_Java_PVR_Samza.yml new file mode 100644 index 0000000000000..f6f0140a6ec7f --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_PVR_Samza.yml @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java PVR Samza + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_PVR_Samza: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + strategy: + matrix: + job_name: [beam_PostCommit_Java_PVR_Samza] + job_phrase: [Run Java Samza PortableValidatesRunner] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java Samza PortableValidatesRunner' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit Java Samza script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:samza:job-server:validatesPortableRunner + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml b/.github/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml new file mode 100644 index 0000000000000..4fff363d989a3 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java PVR Spark3 Streaming + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_PVR_Spark3_Streaming: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + strategy: + matrix: + job_name: [beam_PostCommit_Java_PVR_Spark3_Streaming] + job_phrase: [Run Java Spark v3 PortableValidatesRunner Streaming] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java Spark v3 PortableValidatesRunner Streaming' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit Java PortableValidatesRunner Spark3 Streaming script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:spark:3:job-server:validatesPortableRunnerStreaming + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml b/.github/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml new file mode 100644 index 0000000000000..d4db6f01518e7 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java PVR Spark Batch + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_PVR_Spark_Batch: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + strategy: + matrix: + job_name: [beam_PostCommit_Java_PVR_Spark_Batch] + job_phrase: [Run Java Spark PortableValidatesRunner Batch] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + - name: run PostCommit Java PortableValidatesRunner Spark Batch script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: | + :runners:spark:3:job-server:validatesPortableRunnerBatch \ + :runners:spark:3:job-server:validatesPortableRunnerDocker \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + large_files: true + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + with: + name: SpotBugs Results + path: "**/build/reports/spotbugs/*.html" diff --git a/.github/workflows/beam_PostCommit_Java_Sickbay.yml b/.github/workflows/beam_PostCommit_Java_Sickbay.yml new file mode 100644 index 0000000000000..3478e8902e086 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Sickbay.yml @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java Sickbay + +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_Sickbay: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + strategy: + matrix: + job_name: [beam_PostCommit_Java_Sickbay] + job_phrase: [Run Java Sickbay] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java Sickbay' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit Java Sickbay script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :javaPostCommitSickbay + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml b/.github/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml new file mode 100644 index 0000000000000..43b7d0bb1d812 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml @@ -0,0 +1,103 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java SingleStoreIO IT + +on: + schedule: + - cron: '0 */23 * * *' + workflow_dispatch: +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_SingleStoreIO_IT: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + strategy: + fail-fast: false + matrix: + job_name: [beam_PostCommit_Java_SingleStoreIO_IT] + job_phrase: [Run Java SingleStoreIO_IT] + if: | + github.event_name == 'push' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java SingleStoreIO_IT' + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Set k8s access + uses: ./.github/actions/setup-k8s-access + with: + cluster_name: io-datastores + k8s_namespace: ${{ matrix.job_name }}-${{ github.run_id }} + - name: Install Singlestore operator + run: | + kubectl apply -f ${{github.workspace}}/.test-infra/kubernetes/singlestore/sdb-rbac.yaml + kubectl apply -f ${{github.workspace}}/.test-infra/kubernetes/singlestore/sdb-cluster-crd.yaml + kubectl apply -f ${{github.workspace}}/.test-infra/kubernetes/singlestore/sdb-operator.yaml + kubectl wait --for=condition=Ready pod -l name=sdb-operator --timeout=120s + - name: Install Singlestore cluster + id: install_singlestore + run: | + kubectl apply -f ${{github.workspace}}/.test-infra/kubernetes/singlestore/sdb-cluster.yaml + kubectl wait --for=jsonpath='{.status.phase}'=Running memsqlclusters.memsql.com --all --timeout=120s + kubectl wait svc/svc-sdb-cluster-ddl --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=120s + loadbalancer_IP=$(kubectl get svc svc-sdb-cluster-ddl -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo lb_ip=$loadbalancer_IP >> $GITHUB_OUTPUT + - name: Run Java SingleStore IO IT + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:singlestore:integrationTest --tests org.apache.beam.sdk.io.singlestore.SingleStoreIODefaultMapperIT + arguments: --info -DintegrationTestRunner=dataflow -DintegrationTestPipelineOptions='["--tempRoot=gs://temp-storage-for-perf-tests","--project=apache-beam-testing","--runner=DataflowRunner","--singleStoreUsername=admin","--singleStorePassword=secretpass","--singleStorePort=3306","--numberOfRecords=1000", "--singleStoreServerName=${{ steps.install_singlestore.outputs.lb_ip }}"]' diff --git a/.github/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml b/.github/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml new file mode 100644 index 0000000000000..9cb67bbdf3c20 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml @@ -0,0 +1,101 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java Tpcds Dataflow + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GRADLE_COMMAND_ARGUMENTS: | + --runner=DataflowRunner + --region=us-central1 + --numWorkers=4 + --maxNumWorkers=4 + --autoscalingAlgorithm=NONE + --dataSize=1GB + --sourceType=PARQUET + --dataDirectory=gs://beam-tpcds/datasets/parquet/nonpartitioned + --resultsDirectory=gs://beam-tpcds/results/dataflow/ + --tpcParallel=1 + tpcdsBigQueryArgs: | + --bigQueryTable=tpcds + --bigQueryDataset=tpcds + --project=apache-beam-testing + --resourceNameMode=QUERY_RUNNER_AND_MODE + --exportSummaryToBigQuery=true + --tempLocation=gs://temp-storage-for-perf-tests/tpcds + tpcdsInfluxDBArgs: | + --influxDatabase=beam_test_metrics + --influxHost=http://10.128.0.96:8086 + --baseInfluxMeasurement=tpcds + --exportSummaryToInfluxDB=true + --influxRetentionPolicy=forever + tpcdsQueriesArg: 3,7,10,25,26,29,35,38,40,42,43,52,55,69,79,83,84,87,93,96 + +jobs: + beam_PostCommit_Java_Tpcds_Dataflow: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Dataflow Runner Tpcds Tests' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Java_Tpcds_Dataflow"] + job_phrase: ["Run Dataflow Runner Tpcds Tests"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit Java Tpcds Dataflow script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:tpcds:run + arguments: | + -Ptpcds.runner=:runners:google-cloud-dataflow-java \ + "-Ptpcds.args=${{env.tpcdsBigQueryArgs}} ${{env.tpcdsInfluxDBArgs}} ${{ env.GRADLE_COMMAND_ARGUMENTS }} --queries=${{env.tpcdsQueriesArg}}" \ diff --git a/.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml b/.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml new file mode 100644 index 0000000000000..6864576f651e4 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java Tpcds Flink + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GRADLE_COMMAND_ARGUMENTS: | + --runner=FlinkRunner + --parallelism=4 + --dataSize=1GB + --sourceType=PARQUET + --dataDirectory=gs://beam-tpcds/datasets/parquet/nonpartitioned + --resultsDirectory=gs://beam-tpcds/results/flink/ + --tpcParallel=1 + tpcdsBigQueryArgs: | + --bigQueryTable=tpcds + --bigQueryDataset=tpcds + --project=apache-beam-testing + --resourceNameMode=QUERY_RUNNER_AND_MODE + --exportSummaryToBigQuery=true + --tempLocation=gs://temp-storage-for-perf-tests/tpcds + tpcdsInfluxDBArgs: | + --influxDatabase=beam_test_metrics + --influxHost=http://10.128.0.96:8086 + --baseInfluxMeasurement=tpcds + --exportSummaryToInfluxDB=true + --influxRetentionPolicy=forever + tpcdsQueriesArg: 3,7,10,25,26,29,35,38,40,42,43,52,55,69,79,83,84,87,93,96 + +jobs: + beam_PostCommit_Java_Tpcds_Flink: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Flink Runner Tpcds Tests' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Java_Tpcds_Flink"] + job_phrase: ["Run Flink Runner Tpcds Tests"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit Java Tpcds Flink script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:tpcds:run + arguments: | + -Ptpcds.runner=:runners:flink:1.13 \ + "-Ptpcds.args=${{env.tpcdsBigQueryArgs}} ${{env.tpcdsInfluxDBArgs}} ${{ env.GRADLE_COMMAND_ARGUMENTS }} --queries=${{env.tpcdsQueriesArg}}" \ diff --git a/.github/workflows/beam_PostCommit_Java_Tpcds_Spark.yml b/.github/workflows/beam_PostCommit_Java_Tpcds_Spark.yml new file mode 100644 index 0000000000000..be4aef55f7143 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_Tpcds_Spark.yml @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java Tpcds Spark + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GRADLE_COMMAND_ARGUMENTS: | + --runner=SparkRunner + --dataSize=1GB + --sourceType=PARQUET + --dataDirectory=gs://beam-tpcds/datasets/parquet/nonpartitioned + --resultsDirectory=gs://beam-tpcds/results/spark3-rdd/ + --tpcParallel=1 + tpcdsBigQueryArgs: | + --bigQueryTable=tpcds + --bigQueryDataset=tpcds + --project=apache-beam-testing + --resourceNameMode=QUERY_RUNNER_AND_MODE + --exportSummaryToBigQuery=true + --tempLocation=gs://temp-storage-for-perf-tests/tpcds + tpcdsInfluxDBArgs: | + --influxDatabase=beam_test_metrics + --influxHost=http://10.128.0.96:8086 + --baseInfluxMeasurement=tpcds + --exportSummaryToInfluxDB=true + --influxRetentionPolicy=forever + tpcdsQueriesArg: 3,7,10,25,26,29,35,38,40,42,43,52,55,69,79,83,84,87,93,96 + +jobs: + beam_PostCommit_Java_Tpcds_Spark: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Spark Runner Tpcds Tests' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Java_Tpcds_Spark"] + job_phrase: ["Run Spark Runner Tpcds Tests"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit Java Tpcds Spark script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:tpcds:run + arguments: | + -Ptpcds.runner=:runners:spark:3 \ + "-Ptpcds.args=${{env.tpcdsBigQueryArgs}} ${{env.tpcdsInfluxDBArgs}} ${{ env.GRADLE_COMMAND_ARGUMENTS }} --queries=${{env.tpcdsQueriesArg}}" \ diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml new file mode 100644 index 0000000000000..8d8609ad96d6f --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java ValidatesRunner Dataflow + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_ValidatesRunner_Dataflow: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 480 + strategy: + matrix: + job_name: [beam_PostCommit_Java_ValidatesRunner_Dataflow] + job_phrase: [Run Dataflow ValidatesRunner] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Dataflow ValidatesRunner' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'zulu' + java-version: '8' + - name: run validatesRunner script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:validatesRunner + max-workers: 12 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml new file mode 100644 index 0000000000000..5b151e535f97e --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java ValidatesRunner Dataflow JavaVersions + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.java_version }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 480 + strategy: + fail-fast: false + matrix: + job_name: [beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions] + job_phrase: [Run Dataflow ValidatesRunner Java] + java_version: ['11','17'] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + startswith(github.event.comment.body, 'Run Dataflow ValidatesRunner Java') + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.java_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) ${{ matrix.java_version }} + - name: Set up Java${{ matrix.java_version }} + uses: actions/setup-java@v3.8.0 + with: + distribution: 'temurin' + java-version: | + ${{ matrix.java_version }} + 8 + - name: run jar Java${{ matrix.java_version }} script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:testJar :runners:google-cloud-dataflow-java:worker:shadowJar + arguments: | + -Dorg.gradle.java.home=$JAVA_HOME_8_X64 \ + - name: run validatesRunner Java${{ matrix.java_version }} script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:validatesRunner + arguments: | + -x shadowJar \ + -x shadowTestJar \ + -x compileJava \ + -x compileTestJava \ + -x jar \ + -x testJar \ + -x classes \ + -x testClasses \ + -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + max-workers: 12 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml new file mode 100644 index 0000000000000..2a61431c81af0 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java ValidatesRunner Dataflow Streaming + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 720 + strategy: + matrix: + job_name: [beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming] + job_phrase: [Run Dataflow Streaming ValidatesRunner] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Dataflow Streaming ValidatesRunner' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'zulu' + java-version: '8' + - name: run validatesRunnerStreaming script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:validatesRunnerStreaming + max-workers: 12 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml new file mode 100644 index 0000000000000..4d3664618d9e9 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java ValidatesRunner Dataflow V2 + +on: + schedule: + - cron: '0 */8 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_ValidatesRunner_Dataflow_V2: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 390 + strategy: + matrix: + job_name: [beam_PostCommit_Java_ValidatesRunner_Dataflow_V2] + job_phrase: [Run Java Dataflow V2 ValidatesRunner] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java Dataflow V2 ValidatesRunner' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'zulu' + java-version: '8' + - name: run validatesRunnerV2 script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:validatesRunnerV2 + max-workers: 12 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml new file mode 100644 index 0000000000000..10725d6a72d9f --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java ValidatesRunner Dataflow V2 Streaming + +on: + schedule: + - cron: '0 */8 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 510 + strategy: + matrix: + job_name: [beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming] + job_phrase: [Run Java Dataflow V2 ValidatesRunner Streaming] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java Dataflow V2 ValidatesRunner Streaming' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'zulu' + java-version: '8' + - name: run validatesRunnerV2Streaming script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:validatesRunnerV2Streaming + max-workers: 12 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml new file mode 100644 index 0000000000000..5bf14340e0d18 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java ValidatesRunner Direct + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_ValidatesRunner_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 180 + strategy: + matrix: + job_name: [beam_PostCommit_Java_ValidatesRunner_Direct] + job_phrase: [Run Direct ValidatesRunner] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Direct ValidatesRunner' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'zulu' + java-version: '8' + - name: run validatesRunner script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:direct-java:validatesRunner + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml new file mode 100644 index 0000000000000..e71c47266eac6 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java ValidatesRunner Direct JavaVersions + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.java_version }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 480 + strategy: + fail-fast: false + matrix: + job_name: [beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions] + job_phrase: [Run Direct ValidatesRunner Java] + java_version: ['11','17'] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + startswith(github.event.comment.body, 'Run Direct ValidatesRunner Java') + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.java_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) ${{ matrix.java_version }} + - name: Set up Java${{ matrix.java_version }} + uses: actions/setup-java@v3.8.0 + with: + distribution: 'temurin' + java-version: | + ${{ matrix.java_version }} + 8 + - name: run jar Java${{ matrix.java_version }} script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:direct-java:shadowJar :runners:direct-java:shadowTestJar + arguments: | + -Dorg.gradle.java.home=$JAVA_HOME_8_X64 \ + - name: run validatesRunner Java${{ matrix.java_version }} script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:direct-java:validatesRunner + arguments: | + -x shadowJar \ + -x shadowTestJar \ + -x compileJava \ + -x compileTestJava \ + -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml new file mode 100644 index 0000000000000..6df33f411b8d1 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java ValidatesRunner Flink + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_ValidatesRunner_Flink: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + strategy: + matrix: + job_name: [beam_PostCommit_Java_ValidatesRunner_Flink] + job_phrase: [Run Flink ValidatesRunner] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Flink ValidatesRunner' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: 3.8 + - name: run validatesRunner script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:flink:1.15:validatesRunner + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml new file mode 100644 index 0000000000000..eaeef09bb7812 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java ValidatesRunner Flink Java11 + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_ValidatesRunner_Flink_Java11: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 270 + strategy: + matrix: + job_name: [beam_PostCommit_Java_ValidatesRunner_Flink_Java11] + job_phrase: [Run Flink ValidatesRunner Java 11] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + startswith(github.event.comment.body, 'Run Flink ValidatesRunner Java 11') + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Set up Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'temurin' + java-version: | + 11 + 8 + - name: run jar Java8 script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:flink:1.15:jar :runners:flink:1.15:testJar + arguments: | + -Dorg.gradle.java.home=$JAVA_HOME_8_X64 \ + - name: run validatesRunner Java11 script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:flink:1.15:validatesRunner + arguments: | + -x shadowJar \ + -x shadowTestJar \ + -x compileJava \ + -x compileTestJava \ + -x jar \ + -x testJar \ + -x classes \ + -x testClasses \ + -Dorg.gradle.java.home=$JAVA_HOME_11_X64 \ + max-workers: 12 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml new file mode 100644 index 0000000000000..cc68b3c7cc671 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java ValidatesRunner Samza + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_ValidatesRunner_Samza: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + strategy: + matrix: + job_name: [beam_PostCommit_Java_ValidatesRunner_Samza] + job_phrase: [Run Samza ValidatesRunner] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Samza ValidatesRunner' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'zulu' + java-version: '8' + - name: run validatesRunner script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:samza:validatesRunner + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml new file mode 100644 index 0000000000000..b1c0395dc87aa --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java ValidatesRunner Spark + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_ValidatesRunner_Spark: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + strategy: + matrix: + job_name: [beam_PostCommit_Java_ValidatesRunner_Spark] + job_phrase: [Run Spark ValidatesRunner] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Spark ValidatesRunner' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'zulu' + java-version: '8' + - name: run validatesRunner script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:spark:3:validatesRunner + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml new file mode 100644 index 0000000000000..2047e5a448a2e --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java ValidatesRunner SparkStructuredStreaming + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + strategy: + matrix: + job_name: [beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming] + job_phrase: [Run Spark StructuredStreaming ValidatesRunner] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Spark StructuredStreaming ValidatesRunner' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'zulu' + java-version: '8' + - name: run validatesStructuredStreamingRunnerBatch script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:spark:3:validatesStructuredStreamingRunnerBatch + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml new file mode 100644 index 0000000000000..3456f24594a6f --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Java ValidatesRunner Spark Java11 + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_ValidatesRunner_Spark_Java11: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 270 + strategy: + matrix: + job_name: [beam_PostCommit_Java_ValidatesRunner_Spark_Java11] + job_phrase: [Run Spark ValidatesRunner Java 11] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + startswith(github.event.comment.body, 'Run Spark ValidatesRunner Java 11') + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Set up Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'temurin' + java-version: | + 11 + 8 + - name: run jar Java8 script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:spark:3:jar :runners:spark:3:testJar + arguments: | + -Dorg.gradle.java.home=$JAVA_HOME_8_X64 \ + - name: run validatesRunner Java11 script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:spark:3:validatesRunner + arguments: | + -x shadowJar \ + -x shadowTestJar \ + -x compileJava \ + -x compileTestJava \ + -x jar \ + -x testJar \ + -x classes \ + -x testClasses \ + -Dorg.gradle.java.home=$JAVA_HOME_11_X64 \ + max-workers: 12 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml new file mode 100644 index 0000000000000..72ad389087f13 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java ValidatesRunner Twister2 + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_ValidatesRunner_Twister2: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + strategy: + matrix: + job_name: [beam_PostCommit_Java_ValidatesRunner_Twister2] + job_phrase: [Run Twister2 ValidatesRunner] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Twister2 ValidatesRunner' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'zulu' + java-version: '8' + - name: run validatesRunner script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:twister2:validatesRunner + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml new file mode 100644 index 0000000000000..3296edfd3e62a --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java ValidatesRunner ULR + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_ValidatesRunner_ULR: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 180 + strategy: + matrix: + job_name: [beam_PostCommit_Java_ValidatesRunner_ULR] + job_phrase: [Run ULR Loopback ValidatesRunner] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run ULR Loopback ValidatesRunner' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'zulu' + java-version: '8' + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name: run ulrLoopbackValidatesRunner script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:portability:java:ulrLoopbackValidatesRunner + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Javadoc.yml b/.github/workflows/beam_PostCommit_Javadoc.yml new file mode 100644 index 0000000000000..deab0f879f523 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Javadoc.yml @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Javadoc + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Javadoc: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + strategy: + matrix: + job_name: [beam_PostCommit_Javadoc] + job_phrase: [Run Javadoc PostCommit] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Javadoc PostCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run aggregateJavadoc script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:javadoc:aggregateJavadoc + - name: Upload Javadoc Results + uses: actions/upload-artifact@v3 + with: + name: Javadoc Results + path: '**/sdks/java/javadoc/build/docs/javadoc/**' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_PortableJar_Flink.yml b/.github/workflows/beam_PostCommit_PortableJar_Flink.yml new file mode 100644 index 0000000000000..854371af6ef72 --- /dev/null +++ b/.github/workflows/beam_PostCommit_PortableJar_Flink.yml @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit PortableJar Flink + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_PortableJar_Flink: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run PortableJar_Flink PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + strategy: + matrix: + job_name: ["beam_PostCommit_PortableJar_Flink"] + job_phrase: ["Run PortableJar_Flink PostCommit"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: run testPipelineJarFlinkRunner script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:portable:py38:testPipelineJarFlinkRunner + arguments: | + -PpythonVersion=3.8 \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_PortableJar_Spark.yml b/.github/workflows/beam_PostCommit_PortableJar_Spark.yml new file mode 100644 index 0000000000000..fbafbbda46085 --- /dev/null +++ b/.github/workflows/beam_PostCommit_PortableJar_Spark.yml @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit PortableJar Spark + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_PortableJar_Spark: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run PortableJar_Spark PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + strategy: + matrix: + job_name: ["beam_PostCommit_PortableJar_Spark"] + job_phrase: ["Run PortableJar_Spark PostCommit"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: run testPipelineJarSparkRunner script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:portable:py38:testPipelineJarSparkRunner + arguments: | + -PpythonVersion=3.8 \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python.yml b/.github/workflows/beam_PostCommit_Python.yml new file mode 100644 index 0000000000000..e6e5ebd93d302 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Python.yml @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Python + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Python: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: + job_name: [beam_PostCommit_Python] + job_phrase: [Run Python PostCommit] + python_version: ['3.8', '3.9', '3.10', '3.11'] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + startswith(github.event.comment.body, 'Run Python PostCommit 3.') + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python_version }} + - name: Install docker compose + run: | + sudo curl -L https://github.com/docker/compose/releases/download/1.22.0/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: run PostCommit Python ${{ matrix.python_version }} script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :python${{steps.set_py_ver_clean.outputs.py_ver_clean}}PostCommit + arguments: | + -PuseWheelDistribution \ + -PpythonVersion=${{ matrix.python_version }} \ + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python_Arm.yml b/.github/workflows/beam_PostCommit_Python_Arm.yml new file mode 100644 index 0000000000000..f3d2b2463ef2e --- /dev/null +++ b/.github/workflows/beam_PostCommit_Python_Arm.yml @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Python Arm + +on: + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Python_Arm: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: + job_name: [beam_PostCommit_Python_Arm] + job_phrase: [Run Python PostCommit Arm] + python_version: ['3.8', '3.9', '3.10', '3.11'] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + startsWith(github.event.comment.body, 'Run Python PostCommit Arm') + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python_version }} + - name: Install docker compose + run: | + sudo curl -L https://github.com/docker/compose/releases/download/1.22.0/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: GCloud Docker credential helper + run: | + gcloud auth configure-docker us.gcr.io + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: Generate TAG unique variable based on timestamp + id: set_tag + run: echo "TAG=$(date +'%Y%m%d-%H%M%S%N')" >> $GITHUB_OUTPUT + - name: run PostCommit Python ${{ matrix.python_version }} script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:dataflow:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:postCommitArmIT + arguments: | + -PuseWheelDistribution \ + -PpythonVersion=${{ matrix.python_version }} \ + -Pcontainer-architecture-list=arm64,amd64 \ + -Pdocker-repository-root=us.gcr.io/apache-beam-testing/github-actions \ + -Pdocker-tag=${{ steps.set_tag.outputs.TAG }} \ + -Ppush-containers \ + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + MULTIARCH_TAG: ${{ steps.set_tag.outputs.TAG }} + USER: github-actions + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Examples_Dataflow.yml new file mode 100644 index 0000000000000..941ee99944366 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Python_Examples_Dataflow.yml @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Python Examples Dataflow + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Python_Examples_Dataflow: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Python Examples_Dataflow' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 180 + strategy: + matrix: + job_name: ["beam_PostCommit_Python_Examples_Dataflow"] + job_phrase: ["Run Python Examples_Dataflow"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: 3.11 + - name: Run examplesPostCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:dataflow:examplesPostCommit + arguments: | + -PuseWheelDistribution \ + -PpythonVersion=3.11 \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml b/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml new file mode 100644 index 0000000000000..a54981bde8876 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Python Examples Direct + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Python_Examples_Direct: + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + startsWith(github.event.comment.body, 'Run Python Examples_Direct') + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + strategy: + fail-fast: false + matrix: + job_name: ["beam_PostCommit_Python_Examples_Direct"] + job_phrase: ["Run Python Examples_Direct"] + python_version: ['3.8','3.9','3.10','3.11'] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: ${{ matrix.python_version }} + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: Run examplesPostCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:direct:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:examples + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml b/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml new file mode 100644 index 0000000000000..2cc2ff69aed96 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Python Examples Flink + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Python_Examples_Flink: + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + startsWith(github.event.comment.body, 'Run Python Examples_Flink') + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + strategy: + fail-fast: false + matrix: + job_name: ["beam_PostCommit_Python_Examples_Flink"] + job_phrase: ["Run Python Examples_Flink"] + python_version: ['3.8', '3.11'] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: ${{ matrix.python_version }} + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: Run examplesPostCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:portable:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:flinkExamples + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml b/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml new file mode 100644 index 0000000000000..4c8b39f5f116c --- /dev/null +++ b/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Python Examples Spark + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Python_Examples_Spark: + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + startsWith(github.event.comment.body, 'Run Python Examples_Spark') + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + strategy: + fail-fast: false + matrix: + job_name: ["beam_PostCommit_Python_Examples_Spark"] + job_phrase: ["Run Python Examples_Spark"] + python_version: ['3.8', '3.11'] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: ${{ matrix.python_version }} + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: Run examplesPostCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:portable:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:sparkExamples + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml b/.github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml new file mode 100644 index 0000000000000..f58632bad758c --- /dev/null +++ b/.github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Python MongoDBIO IT + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Python_MongoDBIO_IT: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Python MongoDBIO_IT' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + strategy: + matrix: + job_name: ["beam_PostCommit_Python_MongoDBIO_IT"] + job_phrase: ["Run Python MongoDBIO_IT"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: 3.11 + - name: Run mongodbioIT script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:direct:py311:mongodbioIT + arguments: | + -PpythonVersion=3.11 \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml b/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml new file mode 100644 index 0000000000000..1fbe9a47296aa --- /dev/null +++ b/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Python Nexmark Direct + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + GRADLE_JAVA_COMMAND_ARGUMENTS: | + --manageResources=false + --monitorJobs=true + --bigQueryTable=nexmark + --bigQueryDataset=nexmark + --project=apache-beam-testing + --resourceNameMode=QUERY_RUNNER_AND_MODE + --exportSummaryToBigQuery=true + --tempLocation=gs://temp-storage-for-perf-tests/nexmark + --influxDatabase=beam_test_metrics + --influxHost=http://10.128.0.96:8086 + --baseInfluxMeasurement=nexmark + --exportSummaryToInfluxDB=true + --influxRetentionPolicy=forever + --suite=SMOKE + --enforceEncodability=true + --enforceImmutability=true + --runner=DirectRunner + --numEvents=100000 + GRADLE_PYTHON_COMMAND_ARGUMENTS: | + --monitor_jobs + --big_query_table=nexmark + --big_query_dataset=nexmark + --project=apache-beam-testing + --resource_name_mode=QUERY_RUNNER_AND_MODE + --export_summary_to_big_query + --temp_location=gs://temp-storage-for-perf-tests/nexmark + --influx_database=beam_test_metrics + --influx_host=http://10.128.0.96:8086 + --base_influx_measurement=nexmark + --export_summary_to_influx_db + --influx_retention_policy=forever + --suite=SMOKE + --enforce_encodability + --enforce_immutability + --runner=DirectRunner + --num_events=100000 + +jobs: + beam_PostCommit_Python_Nexmark_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: + job_name: [beam_PostCommit_Python_Nexmark_Direct] + job_phrase: [Run Python Direct Runner Nexmark Tests] + # Query numbers are listed explicitly due to issues with Python Nexmark Query: + # query = 1 - https://github.com/apache/beam/issues/24678 + # query = 4,6,9 - https://github.com/apache/beam/issues/24679 + # query = 12 - https://github.com/apache/beam/issues/24680 + query: [0, 2, 3, 5, 7, 8, 10, 11] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Python Direct Runner Nexmark Tests' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: run Java Testing Nexmark (query ${{ matrix.query }}) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:testing:nexmark:run + arguments: | + -Pnexmark.runner=:runners:direct-java \ + "-Pnexmark.args=${{ env.GRADLE_JAVA_COMMAND_ARGUMENTS }} \ + --query=${{ matrix.query }} \ + --generateEventFilePathPrefix=gs://temp-storage-for-perf-tests/nexmark/eventFiles/beam_PostCommit_Python_Nexmark_Direct/query${{ matrix.query }}-" \ + - name: run Python Testing Benchmarks Nexmark (query ${{ matrix.query }}) + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:apache_beam:testing:benchmarks:nexmark:run + arguments: | + -PpythonVersion=3.8 \ + "-Pnexmark.args=${{ env.GRADLE_PYTHON_COMMAND_ARGUMENTS }} \ + --query=${{ matrix.query }} \ + --input=gs://temp-storage-for-perf-tests/nexmark/eventFiles/beam_PostCommit_Python_Nexmark_Direct/query${{ matrix.query }}-\*" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml new file mode 100644 index 0000000000000..364f1455e955a --- /dev/null +++ b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Python ValidatesContainer Dataflow + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Python_ValidatesContainer_Dataflow: + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + startsWith(github.event.comment.body, 'Run Python Dataflow ValidatesContainer') + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + strategy: + fail-fast: false + matrix: + job_name: ["beam_PostCommit_Python_ValidatesContainer_Dataflow"] + job_phrase: ["Run Python Dataflow ValidatesContainer"] + python_version: ['3.8','3.9','3.10','3.11'] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: ${{ matrix.python_version }} + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: Run validatesContainer script + env: + USER: github-actions + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:dataflow:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:validatesContainer + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml new file mode 100644 index 0000000000000..164577228e4f6 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Python ValidatesContainer Dataflow With RC + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC: + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + startsWith(github.event.comment.body, 'Run Python RC Dataflow ValidatesContainer') + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + strategy: + fail-fast: false + matrix: + job_name: ["beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC"] + job_phrase: ["Run Python RC Dataflow ValidatesContainer"] + python_version: ['3.8','3.9','3.10','3.11'] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: ${{ matrix.python_version }} + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: Run validatesContainer script + env: + USER: github-actions + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:dataflow:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:validatesContainer + arguments: | + -PtestRCDependencies=true \ + -PpythonVersion=${{ matrix.python_version }} \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml new file mode 100644 index 0000000000000..c9256c538594a --- /dev/null +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Python ValidatesRunner Dataflow + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Python_ValidatesRunner_Dataflow: + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + startsWith(github.event.comment.body, 'Run Python Dataflow ValidatesRunner') + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 200 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + strategy: + fail-fast: false + matrix: + job_name: ["beam_PostCommit_Python_ValidatesRunner_Dataflow"] + job_phrase: ["Run Python Dataflow ValidatesRunner"] + python_version: ['3.8', '3.11'] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: ${{ matrix.python_version }} + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: Run validatesRunnerBatchTests script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:dataflow:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:validatesRunnerBatchTests + arguments: | + -PuseWheelDistribution \ + -PpythonVersion=${{ matrix.python_version }} \ + - name: Run validatesRunnerStreamingTests script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:dataflow:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:validatesRunnerStreamingTests + arguments: | + -PuseWheelDistribution \ + -PpythonVersion=${{ matrix.python_version }} \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml new file mode 100644 index 0000000000000..f6e6b3a182ef4 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Python ValidatesRunner Flink + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Python_ValidatesRunner_Flink: + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + startsWith(github.event.comment.body, 'Run Python Flink ValidatesRunner') + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + strategy: + fail-fast: false + matrix: + job_name: ["beam_PostCommit_Python_ValidatesRunner_Flink"] + job_phrase: ["Run Python Flink ValidatesRunner"] + python_version: ['3.8', '3.11'] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: ${{ matrix.python_version }} + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: Run flinkValidatesRunner script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:portable:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:flinkValidatesRunner + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml new file mode 100644 index 0000000000000..805640999a576 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Python ValidatesRunner Samza + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Python_ValidatesRunner_Samza: + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + startsWith(github.event.comment.body, 'Run Python Samza ValidatesRunner') + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + strategy: + fail-fast: false + matrix: + job_name: ["beam_PostCommit_Python_ValidatesRunner_Samza"] + job_phrase: ["Run Python Samza ValidatesRunner"] + python_version: ['3.8', '3.11'] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: ${{ matrix.python_version }} + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: Run samzaValidatesRunner script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:portable:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:samzaValidatesRunner + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml new file mode 100644 index 0000000000000..fb80210301133 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Python ValidatesRunner Spark + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Python_ValidatesRunner_Spark: + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + startsWith(github.event.comment.body, 'Run Python Spark ValidatesRunner') + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + strategy: + fail-fast: false + matrix: + job_name: ["beam_PostCommit_Python_ValidatesRunner_Spark"] + job_phrase: ["Run Python Spark ValidatesRunner"] + python_version: ['3.8', '3.9', '3.11'] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: ${{ matrix.python_version }} + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: Run sparkValidatesRunner script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:portable:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:sparkValidatesRunner + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml new file mode 100644 index 0000000000000..cdfae228a53e9 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Python Xlang Gcp Dataflow + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Python_Xlang_Gcp_Dataflow: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Python_Xlang_Gcp_Dataflow PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 180 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Python_Xlang_Gcp_Dataflow"] + job_phrase: ["Run Python_Xlang_Gcp_Dataflow PostCommit"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: | + 3.8 + 3.11 + - name: run PostCommit Python Xlang Gcp Dataflow script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:dataflow:gcpCrossLanguagePostCommit + arguments: -PuseWheelDistribution + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml new file mode 100644 index 0000000000000..d58098ed3c721 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Python Xlang Gcp Direct + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Python_Xlang_Gcp_Direct: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Python_Xlang_Gcp_Direct PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Python_Xlang_Gcp_Direct"] + job_phrase: ["Run Python_Xlang_Gcp_Direct PostCommit"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: | + 3.8 + 3.11 + - name: run PostCommit Python Xlang Gcp Direct script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:direct:gcpCrossLanguagePostCommit + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml new file mode 100644 index 0000000000000..c0b8b2c993c40 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Python Xlang IO Dataflow + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Python_Xlang_IO_Dataflow: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Python_Xlang_IO_Dataflow PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 180 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Python_Xlang_IO_Dataflow"] + job_phrase: ["Run Python_Xlang_IO_Dataflow PostCommit"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: | + 3.8 + 3.11 + - name: run PostCommit Python Xlang IO Dataflow script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:dataflow:ioCrossLanguagePostCommit + arguments: | + -PuseWheelDistribution \ + -PkafkaBootstrapServer=10.128.0.40:9094,10.128.0.28:9094,10.128.0.165:9094 \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: archiveJunit + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_SQL.yml b/.github/workflows/beam_PostCommit_SQL.yml new file mode 100644 index 0000000000000..776f7ce1210b8 --- /dev/null +++ b/.github/workflows/beam_PostCommit_SQL.yml @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit SQL + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_SQL: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 240 + strategy: + matrix: + job_name: [beam_PostCommit_SQL] + job_phrase: [Run SQL PostCommit] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run SQL PostCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit SQL script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sqlPostCommit + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' diff --git a/.github/workflows/beam_PostCommit_Sickbay_Python.yml b/.github/workflows/beam_PostCommit_Sickbay_Python.yml new file mode 100644 index 0000000000000..99effb7513573 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Sickbay_Python.yml @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PostCommit Sickbay Python + +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Sickbay_Python: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase_1 }} ${{ matrix.python_version }} ${{ matrix.job_phrase_2 }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + job_name: [beam_PostCommit_Sickbay_Python] + job_phrase_1: [Run Python] + job_phrase_2: [PostCommit Sickbay] + python_version: ['3.8', '3.9', '3.10', '3.11'] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + (startswith(github.event.comment.body, 'Run Python') && + endswith(github.event.comment.body, 'PostCommit Sickbay')) + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase_1 }} ${{ matrix.python_version }} ${{ matrix.job_phrase_2 }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase_1 }} ${{ matrix.python_version }} ${{ matrix.job_phrase_2 }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python_version }} + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: run PostCommit Python ${{ matrix.python_version }} script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:dataflow:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:postCommitSickbay + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_TransformService_Direct.yml b/.github/workflows/beam_PostCommit_TransformService_Direct.yml new file mode 100644 index 0000000000000..4cc1ccf99c225 --- /dev/null +++ b/.github/workflows/beam_PostCommit_TransformService_Direct.yml @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit TransformService Direct + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_TransformService_Direct: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run TransformService_Direct PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + strategy: + matrix: + job_name: ["beam_PostCommit_TransformService_Direct"] + job_phrase: ["Run TransformService_Direct PostCommit"] + python_version: ['3.8','3.11'] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Set up Java 11 + uses: actions/setup-java@v3.11.0 + with: + distribution: 'temurin' + java-version: '11' + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: | + 3.8 + 3.11 + - name: run TransformService Direct script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:direct:xlang:transformServicePythonUsingJava + arguments: | + -PcompileAndRunTestsWithJava11 \ + -Pjava11Home=$JAVA_HOME_11_X64 \ + -PuseWheelDistribution \ + -PpythonVersion=${{ matrix.python_version }} \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Website_Publish.yml b/.github/workflows/beam_PostCommit_Website_Publish.yml new file mode 100644 index 0000000000000..834c3d043afe2 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Website_Publish.yml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Website Publish + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: write + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Website_Publish: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 30 + name: beam_PostCommit_Website_Publish + steps: + - uses: actions/checkout@v4 + - name: run PostCommit Website Publish script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :website:clean :website:publishWebsite + arguments: -PgitPublishRemote="https://github.com/apache/beam.git" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Website_Test.yml b/.github/workflows/beam_PostCommit_Website_Test.yml new file mode 100644 index 0000000000000..6f2229ff76ff7 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Website_Test.yml @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Website Test + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Website_Test: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Full Website Test' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 60 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Website_Test"] + job_phrase: ["Run Full Website Test"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PostCommit Website Test script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :website:testWebsite + arguments: -PdisableExternal=false \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_XVR_Direct.yml b/.github/workflows/beam_PostCommit_XVR_Direct.yml new file mode 100644 index 0000000000000..51e36053ea968 --- /dev/null +++ b/.github/workflows/beam_PostCommit_XVR_Direct.yml @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit XVR Direct + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_XVR_Direct: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run XVR_Direct PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + strategy: + matrix: + job_name: ["beam_PostCommit_XVR_Direct"] + job_phrase: ["Run XVR_Direct PostCommit"] + python_version: ['3.8','3.11'] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: | + 3.8 + 3.11 + - name: run PostCommit XVR Direct script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + if: ${{ matrix.python_version != '3.8' }} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:direct:xlang:validatesCrossLanguageRunner + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + -PskipNonPythonTask=true \ + - name: run PostCommit XVR Direct script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + if: ${{ matrix.python_version == '3.8' }} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:direct:xlang:validatesCrossLanguageRunner + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + -PskipNonPythonTask=false \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_XVR_Flink.yml b/.github/workflows/beam_PostCommit_XVR_Flink.yml new file mode 100644 index 0000000000000..9df8b1c8e7769 --- /dev/null +++ b/.github/workflows/beam_PostCommit_XVR_Flink.yml @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit XVR Flink + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + FlinkVersion: 1.15 + +jobs: + beam_PostCommit_XVR_Flink: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run XVR_Flink PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + strategy: + matrix: + job_name: ["beam_PostCommit_XVR_Flink"] + job_phrase: ["Run XVR_Flink PostCommit"] + python_version: ['3.8','3.11'] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: | + 3.8 + 3.11 + - name: run PostCommit XVR Flink script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + if: ${{ matrix.python_version != '3.8' }} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:flink:${{ env.FlinkVersion }}:job-server:validatesCrossLanguageRunner + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + -PskipNonPythonTask=true \ + - name: run PostCommit XVR Flink script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + if: ${{ matrix.python_version == '3.8' }} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:flink:${{ env.FlinkVersion }}:job-server:validatesCrossLanguageRunner + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + -PskipNonPythonTask=false \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml new file mode 100644 index 0000000000000..439453364e22d --- /dev/null +++ b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit XVR GoUsingJava Dataflow + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_XVR_GoUsingJava_Dataflow: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run XVR_GoUsingJava_Dataflow PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_XVR_GoUsingJava_Dataflow"] + job_phrase: ["Run XVR_GoUsingJava_Dataflow PostCommit"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name: run XVR GoUsingJava Dataflow script + env: + USER: github-actions + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerGoUsingJava + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml new file mode 100644 index 0000000000000..93e495c5eb153 --- /dev/null +++ b/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit XVR JavaUsingPython Dataflow + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_XVR_JavaUsingPython_Dataflow: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run XVR_JavaUsingPython_Dataflow PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + strategy: + matrix: + job_name: ["beam_PostCommit_XVR_JavaUsingPython_Dataflow"] + job_phrase: ["Run XVR_JavaUsingPython_Dataflow PostCommit"] + python_version: ['3.8','3.11'] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: | + 3.8 + 3.11 + - name: run PostCommit XVR JavaUsingPython Dataflow script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerJavaUsingPython + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml new file mode 100644 index 0000000000000..8fdade48da413 --- /dev/null +++ b/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit XVR PythonUsingJavaSQL Dataflow + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run XVR_PythonUsingJavaSQL_Dataflow PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow"] + job_phrase: ["Run XVR_PythonUsingJavaSQL_Dataflow PostCommit"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: 3.11 + - name: run PostCommit XVR PythonUsingJavaSQL Dataflow script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerPythonUsingSql + arguments: | + -PpythonVersion=3.11 \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml new file mode 100644 index 0000000000000..9284c36988249 --- /dev/null +++ b/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit XVR PythonUsingJava Dataflow + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_XVR_PythonUsingJava_Dataflow: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run XVR_PythonUsingJava_Dataflow PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + strategy: + matrix: + job_name: ["beam_PostCommit_XVR_PythonUsingJava_Dataflow"] + job_phrase: ["Run XVR_PythonUsingJava_Dataflow PostCommit"] + python_version: ['3.8','3.11'] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: | + 3.8 + 3.11 + - name: run PostCommit XVR PythonUsingJava Dataflow script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerPythonUsingJava + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_XVR_Samza.yml b/.github/workflows/beam_PostCommit_XVR_Samza.yml new file mode 100644 index 0000000000000..f053f28ac1907 --- /dev/null +++ b/.github/workflows/beam_PostCommit_XVR_Samza.yml @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit XVR Samza + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_XVR_Samza: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run XVR_Samza PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + strategy: + matrix: + job_name: ["beam_PostCommit_XVR_Samza"] + job_phrase: ["Run XVR_Samza PostCommit"] + python_version: ['3.8','3.11'] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: | + 3.8 + 3.11 + - name: run PostCommit XVR Samza script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + if: ${{ matrix.python_version != '3.8' }} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:samza:job-server:validatesCrossLanguageRunner + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + -PskipNonPythonTask=true \ + - name: run PostCommit XVR Samza script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + if: ${{ matrix.python_version == '3.8' }} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:samza:job-server:validatesCrossLanguageRunner + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + -PskipNonPythonTask=false \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_XVR_Spark3.yml b/.github/workflows/beam_PostCommit_XVR_Spark3.yml new file mode 100644 index 0000000000000..3b6d5bdad5511 --- /dev/null +++ b/.github/workflows/beam_PostCommit_XVR_Spark3.yml @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit XVR Spark3 + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_XVR_Spark3: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run XVR_Spark3 PostCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + strategy: + matrix: + job_name: ["beam_PostCommit_XVR_Spark3"] + job_phrase: ["Run XVR_Spark3 PostCommit"] + python_version: ['3.8','3.11'] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: | + 3.8 + 3.11 + - name: run PostCommit XVR Spark3 script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + if: ${{ matrix.python_version != '3.8' }} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:spark:3:job-server:validatesCrossLanguageRunner + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + -PskipNonPythonTask=true \ + - name: run PostCommit XVR Spark3 script + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + if: ${{ matrix.python_version == '3.8' }} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:spark:3:job-server:validatesCrossLanguageRunner + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + -PskipNonPythonTask=false \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_CommunityMetrics.yml b/.github/workflows/beam_PreCommit_CommunityMetrics.yml new file mode 100644 index 0000000000000..9341b1e8877fc --- /dev/null +++ b/.github/workflows/beam_PreCommit_CommunityMetrics.yml @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Community Metrics + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: ['.test-infra/metrics/**', 'buildSrc/build.gradle.kts', '.github/workflows/beam_PreCommit_CommunityMetrics.yml'] + pull_request_target: + branches: ['master', 'release-*'] + paths: ['.test-infra/metrics/**', 'buildSrc/build.gradle.kts'] + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_CommunityMetrics: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + timeout-minutes: 120 + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_CommunityMetrics] + job_phrase: [Run CommunityMetrics PreCommit] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run CommunityMetrics PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Remove default github maven configuration + run: rm ~/.m2/settings.xml + - name: Install docker compose + run: | + sudo curl -L https://github.com/docker/compose/releases/download/1.22.0/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + - name: Authenticate on GCP + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Install gcloud Kubectl + run: gcloud components install kubectl + - name: run Community Metrics PreCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :communityMetricsPreCommit + arguments: | + -PKUBE_CONFIG_PATH='$HOME/.kube/config' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Go.yml b/.github/workflows/beam_PreCommit_Go.yml index 1239b0644b614..c202c30d5a4b7 100644 --- a/.github/workflows/beam_PreCommit_Go.yml +++ b/.github/workflows/beam_PreCommit_Go.yml @@ -1,37 +1,89 @@ -name: beam_PreCommit_Go +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Go on: push: tags: ['v*'] branches: ['master', 'release-*'] - pull_request: - branches: ['master'] + paths: ['model/**', 'sdks/go.**', 'release/**', '.github/workflows/beam_PreCommit_Go.yml'] + pull_request_target: + branches: ['master', 'release-*' ] paths: ['model/**', 'sdks/go.**', 'release/**'] issue_comment: types: [created] schedule: - - cron: '* */6 * * *' + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} jobs: beam_PreCommit_Go: - if: ${{github.event.issue.pull_request}} || ${{github.event.comment.body == 'Run Go PreCommit'}} || ${{github.event.schedule}} - runs-on: [self-hosted, ubuntu-20.04] - name: beam_PreCommit_Go - steps: - - name: Git checkout - uses: actions/checkout@v3 - - name: Install Java - uses: actions/setup-java@v3.8.0 - with: - distribution: 'zulu' - java-version: '8' - - name: Install Go - uses: actions/setup-go@v4 - with: - go-version: '1.20' - - name: Setup Gradle - uses: gradle/gradle-build-action@v2 - with: - cache-read-only: false - - name: run goPreCommit script - run: ./gradlew :goPreCommit \ No newline at end of file + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_Go] + job_phrase: [Run Go PreCommit] + timeout-minutes: 120 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Go PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + go-version: 1.21 + - name: run goPreCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :goPreCommit \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_GoPortable.yml b/.github/workflows/beam_PreCommit_GoPortable.yml new file mode 100644 index 0000000000000..3034285731a55 --- /dev/null +++ b/.github/workflows/beam_PreCommit_GoPortable.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit GoPortable + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: ['model/**', 'sdks/go.**', 'release/**','.github/workflows/beam_PreCommit_GoPortable.yml'] + pull_request_target: + branches: ['master', 'release-*'] + paths: ['model/**', 'sdks/go.**', 'release/**'] + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +jobs: + beam_PreCommit_GoPortable: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_GoPortable] + job_phrase: [Run GoPortable PreCommit] + timeout-minutes: 120 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run GoPortable PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + java-version: 8 + - name: Run goPortablePreCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :goPortablePreCommit \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_GoPrism.yml b/.github/workflows/beam_PreCommit_GoPrism.yml new file mode 100644 index 0000000000000..9e403d13a2efc --- /dev/null +++ b/.github/workflows/beam_PreCommit_GoPrism.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit GoPrism + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: ['model/**', 'sdks/go.**', 'release/**','.github/workflows/beam_PreCommit_GoPrism.yml'] + pull_request_target: + branches: ['master', 'release-*'] + paths: ['model/**', 'sdks/go.**', 'release/**'] + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +jobs: + beam_PreCommit_GoPrism: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_GoPrism] + job_phrase: [Run GoPrism PreCommit] + timeout-minutes: 120 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run GoPrism PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + java-version: 8 + - name: Run goPrismPreCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :goPrismPreCommit \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_ItFramework.yml b/.github/workflows/beam_PreCommit_ItFramework.yml new file mode 100644 index 0000000000000..ef384def0d5f7 --- /dev/null +++ b/.github/workflows/beam_PreCommit_ItFramework.yml @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Integration and Load Test Framework + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - 'it/**' + - '.github/workflows/beam_PreCommit_ItFramework.yml' + pull_request_target: + branches: ['master', 'release-*'] + paths: + - 'it/**' + issue_comment: + types: [created] + schedule: + - cron: '10 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read +jobs: + beam_PreCommit_ItFramework: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_ItFramework] + job_phrase: [Run It_Framework PreCommit] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run It_Framework PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + - name: run ItFrameworkPrecommit script + run: ./gradlew -p it build + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java.yml b/.github/workflows/beam_PreCommit_Java.yml new file mode 100644 index 0000000000000..0b971462f3340 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java.yml @@ -0,0 +1,204 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - 'model/**' + - 'sdks/java/**' + - 'runners/**' + - 'examples/java/**' + - 'examples/kotlin/**' + - 'release/**' + - '.github/workflows/beam_PreCommit_Java.yml' + - '!sdks/java/extensions/avro/**' + - '!sdks/java/extensions/sql/**' + - '!sdks/java/io/amazon-web-services/**' + - '!sdks/java/io/amazon-web-services2/**' + - '!sdks/java/io/amqp/**' + - '!sdks/java/io/azure/**' + - '!sdks/java/io/cassandra/**' + - '!sdks/java/io/cdap/**' + - '!sdks/java/io/clickhouse/**' + - '!sdks/java/io/csv/**' + - '!sdks/java/io/debezium/**' + - '!sdks/java/io/elasticsearch/**' + - '!sdks/java/io/elasticsearch-tests/**' + - '!sdks/java/io/file-schema-transform/**' + - '!sdks/java/io/google-cloud-platform/**' + - '!sdks/java/io/hadoop-common/**' + - '!sdks/java/io/hadoop-file-system/**' + - '!sdks/java/io/hadoop-format/**' + - '!sdks/java/io/hbase/**' + - '!sdks/java/io/hcatalog/**' + - '!sdks/java/io/influxdb/**' + - '!sdks/java/io/jdbc/**' + - '!sdks/java/io/jms/**' + - '!sdks/java/io/kafka/**' + - '!sdks/java/io/kinesis/**' + - '!sdks/java/io/kudu/**' + - '!sdks/java/io/mqtt/**' + - '!sdks/java/io/mongodb/**' + - '!sdks/java/io/neo4j/**' + - '!sdks/java/io/parquet/**' + - '!sdks/java/io/pulsar/**' + - '!sdks/java/io/rabbitmq/**' + - '!sdks/java/io/redis/**' + - '!sdks/java/io/singlestore/**' + - '!sdks/java/io/snowflake/**' + - '!sdks/java/io/solr/**' + - '!sdks/java/io/splunk/**' + - '!sdks/java/io/thrift/**' + - '!sdks/java/io/tika/**' + + pull_request_target: + branches: ['master', 'release-*'] + paths: + - 'model/**' + - 'sdks/java/**' + - 'runners/**' + - 'examples/java/**' + - 'examples/kotlin/**' + - 'release/**' + - '!sdks/java/extensions/avro/**' + - '!sdks/java/extensions/sql/**' + - '!sdks/java/io/amazon-web-services/**' + - '!sdks/java/io/amazon-web-services2/**' + - '!sdks/java/io/amqp/**' + - '!sdks/java/io/azure/**' + - '!sdks/java/io/cassandra/**' + - '!sdks/java/io/cdap/**' + - '!sdks/java/io/clickhouse/**' + - '!sdks/java/io/csv/**' + - '!sdks/java/io/debezium/**' + - '!sdks/java/io/elasticsearch/**' + - '!sdks/java/io/elasticsearch-tests/**' + - '!sdks/java/io/file-schema-transform/**' + - '!sdks/java/io/google-cloud-platform/**' + - '!sdks/java/io/hadoop-common/**' + - '!sdks/java/io/hadoop-file-system/**' + - '!sdks/java/io/hadoop-format/**' + - '!sdks/java/io/hbase/**' + - '!sdks/java/io/hcatalog/**' + - '!sdks/java/io/influxdb/**' + - '!sdks/java/io/jdbc/**' + - '!sdks/java/io/jms/**' + - '!sdks/java/io/kafka/**' + - '!sdks/java/io/kinesis/**' + - '!sdks/java/io/kudu/**' + - '!sdks/java/io/mqtt/**' + - '!sdks/java/io/mongodb/**' + - '!sdks/java/io/neo4j/**' + - '!sdks/java/io/parquet/**' + - '!sdks/java/io/pulsar/**' + - '!sdks/java/io/rabbitmq/**' + - '!sdks/java/io/redis/**' + - '!sdks/java/io/singlestore/**' + - '!sdks/java/io/snowflake/**' + - '!sdks/java/io/solr/**' + - '!sdks/java/io/splunk/**' + - '!sdks/java/io/thrift/**' + - '!sdks/java/io/tika/**' + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + timeout-minutes: 180 + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + fail-fast: false + matrix: + job_name: [beam_PreCommit_Java] + job_phrase: [Run Java PreCommit] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Java PreCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :javaPreCommit + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct.yml new file mode 100644 index 0000000000000..67b27a06462da --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct.yml @@ -0,0 +1,140 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Amazon-Web-Services2 IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/amazon-web-services2/**" + - "sdks/java/io/common/**" + - "sdks/java/core/src/main/**" + - "build.gradle" + - "buildSrc/**" + - "gradle/**" + - "gradle.properties" + - "gradlew" + - "gradle.bat" + - "settings.gradle.kts" + - ".github/workflows/beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/amazon-web-services2/**" + - "sdks/java/io/common/**" + - "sdks/java/core/src/main/**" + - "build.gradle" + - "buildSrc/**" + - "gradle/**" + - "gradle.properties" + - "gradlew" + - "gradle.bat" + - "settings.gradle.kts" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct"] + job_phrase: ["Run Java_Amazon-Web-Services2_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Amazon-Web-Services2_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Amazon-Web-Services2 IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:amazon-web-services2:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: run Amazon-Web-Services2 IO IT script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:amazon-web-services2:integrationTest + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Amazon-Web-Services_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Amazon-Web-Services_IO_Direct.yml new file mode 100644 index 0000000000000..c8e56e928cb87 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Amazon-Web-Services_IO_Direct.yml @@ -0,0 +1,140 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Amazon-Web-Services IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/amazon-web-services/**" + - "sdks/java/io/common/**" + - "sdks/java/core/src/main/**" + - "build.gradle" + - "buildSrc/**" + - "gradle/**" + - "gradle.properties" + - "gradlew" + - "gradle.bat" + - "settings.gradle.kts" + - ".github/workflows/beam_PreCommit_Java_Amazon-Web-Services_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/amazon-web-services/**" + - "sdks/java/io/common/**" + - "sdks/java/core/src/main/**" + - "build.gradle" + - "buildSrc/**" + - "gradle/**" + - "gradle.properties" + - "gradlew" + - "gradle.bat" + - "settings.gradle.kts" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Amazon-Web-Services_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Amazon-Web-Services_IO_Direct"] + job_phrase: ["Run Java_Amazon-Web-Services_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Amazon-Web-Services_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Amazon-Web-Services IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:amazon-web-services:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: run Amazon-Web-Services IO IT script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:amazon-web-services:integrationTest + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Amqp_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Amqp_IO_Direct.yml new file mode 100644 index 0000000000000..050ff91493a00 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Amqp_IO_Direct.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Amqp IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/amqp/**" + - ".github/workflows/beam_PreCommit_Java_Amqp_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/amqp/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Amqp_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Amqp_IO_Direct"] + job_phrase: ["Run Java_Amqp_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Amqp_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Amqp IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:amqp:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Azure_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Azure_IO_Direct.yml new file mode 100644 index 0000000000000..cbd33a766a992 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Azure_IO_Direct.yml @@ -0,0 +1,133 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Azure IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/azure/**" + - "sdks/java/io/common/**" + - "sdks/java/core/src/main/**" + - "build.gradle" + - "buildSrc/**" + - "gradle/**" + - "gradle.properties" + - "gradlew" + - "gradle.bat" + - "settings.gradle.kts" + - ".github/workflows/beam_PreCommit_Java_Azure_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/azure/**" + - "sdks/java/io/common/**" + - "sdks/java/core/src/main/**" + - "build.gradle" + - "buildSrc/**" + - "gradle/**" + - "gradle.properties" + - "gradlew" + - "gradle.bat" + - "settings.gradle.kts" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Azure_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Azure_IO_Direct"] + job_phrase: ["Run Java_Azure_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Azure_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Azure IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:azure:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Cassandra_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Cassandra_IO_Direct.yml new file mode 100644 index 0000000000000..2b9066c1d27aa --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Cassandra_IO_Direct.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Cassandra IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/cassandra/**" + - ".github/workflows/beam_PreCommit_Java_Cassandra_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/cassandra/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Cassandra_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Cassandra_IO_Direct"] + job_phrase: ["Run Java_Cassandra_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Cassandra_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Cassandra IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:cassandra:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Cdap_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Cdap_IO_Direct.yml new file mode 100644 index 0000000000000..29447d880deed --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Cdap_IO_Direct.yml @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Cdap IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/cdap/**" + - "sdks/java/io/hadoop-common/**" + - "sdks/java/io/hadoop-format/**" + - ".github/workflows/beam_PreCommit_Java_Cdap_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/cdap/**" + - "sdks/java/io/hadoop-common/**" + - "sdks/java/io/hadoop-format/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Cdap_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Cdap_IO_Direct"] + job_phrase: ["Run Java_Cdap_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Cdap_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Cdap IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:cdap:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml new file mode 100644 index 0000000000000..4086f690af4e7 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Clickhouse IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/clickhouse/**" + - ".github/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/clickhouse/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Clickhouse_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Clickhouse_IO_Direct"] + job_phrase: ["Run Java_Clickhouse_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Clickhouse_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Clickhouse IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:clickhouse:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml new file mode 100644 index 0000000000000..878951b971866 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Csv IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/csv/**" + - ".github/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/csv/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Csv_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Csv_IO_Direct"] + job_phrase: ["Run Java_Csv_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Csv_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Csv IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:csv:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml new file mode 100644 index 0000000000000..281a9ce4a0ad2 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Debezium IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/debezium/**" + - ".github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/debezium/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Debezium_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Debezium_IO_Direct"] + job_phrase: ["Run Java_Debezium_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Debezium_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Debezium IO build task + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:debezium:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: run Debezium IO additional tasks + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: | + :sdks:java:io:debezium:expansion-service:build \ + :sdks:java:io:debezium:integrationTest \ + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml new file mode 100644 index 0000000000000..f275d6276e009 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java ElasticSearch IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/elasticsearch/**" + - "sdks/java/io/elasticsearch-tests/**" + - ".github/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/elasticsearch/**" + - "sdks/java/io/elasticsearch-tests/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_ElasticSearch_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_ElasticSearch_IO_Direct"] + job_phrase: ["Run Java_ElasticSearch_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_ElasticSearch_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run ElasticSearch IO build task + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:elasticsearch:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: run ElasticSearch IO additional tasks + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: | + :sdks:java:io:elasticsearch-tests:elasticsearch-tests-5:build \ + :sdks:java:io:elasticsearch-tests:elasticsearch-tests-7:build \ + :sdks:java:io:elasticsearch-tests:elasticsearch-tests-8:build \ + :sdks:java:io:elasticsearch-tests:elasticsearch-tests-common:build \ + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml new file mode 100644 index 0000000000000..c68a85a0bc218 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Java Examples Dataflow + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - 'model/**' + - 'sdks/java/**' + - 'runners/google-cloud-dataflow-java/**' + - 'examples/java/**' + - 'examples/kotlin/**' + - 'release/**' + - '.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml' + pull_request_target: + branches: ['master', 'release-*'] + paths: + - 'model/**' + - 'sdks/java/**' + - 'runners/google-cloud-dataflow-java/**' + - 'examples/java/**' + - 'examples/kotlin/**' + - 'release/**' + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read +jobs: + beam_PreCommit_Java_Examples_Dataflow: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + timeout-minutes: 60 + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_Java_Examples_Dataflow] + job_phrase: [Run Java_Examples_Dataflow PreCommit] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Examples_Dataflow PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + - name: Authenticate on GCP + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + export_default_credentials: true + - name: run javaExamplesDataflowPrecommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :javaExamplesDataflowPrecommit + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml new file mode 100644 index 0000000000000..032d13152ebac --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Java Examples Dataflow Java11 + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - 'model/**' + - 'sdks/java/**' + - 'runners/google-cloud-dataflow-java/**' + - 'examples/java/**' + - 'examples/kotlin/**' + - 'release/**' + - '.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml' + pull_request_target: + branches: ['master', 'release-*'] + paths: + - 'model/**' + - 'sdks/java/**' + - 'runners/google-cloud-dataflow-java/**' + - 'examples/java/**' + - 'examples/kotlin/**' + - 'release/**' + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +jobs: + beam_PreCommit_Java_Examples_Dataflow_Java11: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_Java_Examples_Dataflow_Java11] + job_phrase: [Run Java_Examples_Dataflow_Java11 PreCommit] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Examples_Dataflow_Java11 PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + export_default_credentials: true + # The workflow installs java 11 and as default jvm. This is different from + # PreCommit_Java_Examples_Dataflow_Java17 where the build system and sources are compiled with Java8 + - name: Set up Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'temurin' + java-version: '11' + - name: run javaExamplesDataflowPrecommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:examples:preCommit + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + -PskipCheckerFramework \ + -PcompileAndRunTestsWithJava11 \ + -Pjava11Home=$JAVA_HOME_11_X64 \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java17.yml b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java17.yml new file mode 100644 index 0000000000000..64809809dbd8a --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java17.yml @@ -0,0 +1,139 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Examples Dataflow Java17 + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - 'model/**' + - 'sdks/java/**' + - 'runners/google-cloud-dataflow-java/**' + - 'examples/java/**' + - 'examples/kotlin/**' + - 'release/**' + - '.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java17.yml' + pull_request_target: + branches: ['master', 'release-*'] + paths: + - 'model/**' + - 'sdks/java/**' + - 'runners/google-cloud-dataflow-java/**' + - 'examples/java/**' + - 'examples/kotlin/**' + - 'release/**' + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Examples_Dataflow_Java17: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Examples_Dataflow_Java17"] + job_phrase: ["Run Java_Examples_Dataflow_Java17 PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Examples_Dataflow_Java17 PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + # The test requires Java 17 and Java 8 versions. + # Java 8 is installed second because JAVA_HOME needs to point to Java 8. + - name: Set up Java 17 and 8 + uses: actions/setup-java@v3.11.0 + with: + distribution: 'temurin' + java-version: | + 17 + 8 + - name: Clean + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :clean + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + -PskipCheckerFramework \ + - name: Build and Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:examples:preCommit + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + -PcompileAndRunTestsWithJava17 \ + -PskipCheckerFramework \ + -Pjava17Home=$JAVA_HOME_17_X64 \ + max-workers: 12 + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml new file mode 100644 index 0000000000000..2ad4de1332282 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java File-schema-transform IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/file-schema-transform/**" + - ".github/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/file-schema-transform/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_File-schema-transform_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_File-schema-transform_IO_Direct"] + job_phrase: ["Run Java_File-schema-transform_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_File-schema-transform_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run File-schema-transform IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:file-schema-transform:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + -Dfile.encoding=UTF-8 \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Flink_Versions.yml b/.github/workflows/beam_PreCommit_Java_Flink_Versions.yml new file mode 100644 index 0000000000000..6590583925e1c --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Flink_Versions.yml @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Flink Versions + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - 'model/**' + - 'runners/flink/**' + - 'release/**' + - '.github/workflows/beam_PreCommit_Java_Flink_Versions.yml' + pull_request_target: + branches: ['master', 'release-*'] + paths: + - 'model/**' + - 'runners/flink/**' + - 'release/**' + issue_comment: + types: [created] + schedule: + - cron: '20 */6 * * *' + workflow_dispatch: + +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Flink_Versions: + name: beam_PreCommit_Java_Flink_Versions (Run Java_Flink_Versions PreCommit) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 120 + if: | + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java_Flink_Versions PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: 'Run Java_Flink_Versions PreCommit' + github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: 3.8 + - name: run Java Flink Versions PreCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :flinkPreCommit + arguments: | + -PdisableSpotlessCheck=true -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml new file mode 100644 index 0000000000000..0c832aecf7491 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java GCP IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "runners/core-construction-java/**" + - "runners/core-java/**" + - "sdks/java/core/src/main/**" + - "sdks/java/extensions/arrow/**" + - "sdks/java/extensions/google-cloud-platform-core/**" + - "sdks/java/extensions/protobuf/**" + - "sdks/java/testing/test-utils/**" + - "sdks/java/io/common/**" + - "sdks/java/io/expansion-service/**" + - "sdks/java/io/google-cloud-platform/**" + - ".github/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "runners/core-construction-java/**" + - "runners/core-java/**" + - "sdks/java/core/src/main/**" + - "sdks/java/extensions/arrow/**" + - "sdks/java/extensions/google-cloud-platform-core/**" + - "sdks/java/extensions/protobuf/**" + - "sdks/java/testing/test-utils/**" + - "sdks/java/io/common/**" + - "sdks/java/io/expansion-service/**" + - "sdks/java/io/google-cloud-platform/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_GCP_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_GCP_IO_Direct"] + job_phrase: ["Run Java_GCP_IO_Direct PreCommit"] + timeout-minutes: 120 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_GCP_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PreCommit Java GCP IO Direct script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: | + :sdks:java:io:google-cloud-platform:build \ + :sdks:java:io:google-cloud-platform:expansion-service:build \ + :sdks:java:io:google-cloud-platform:postCommit \ + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + -PenableJacocoReport \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_HBase_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_HBase_IO_Direct.yml new file mode 100644 index 0000000000000..4e1e300bb9254 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_HBase_IO_Direct.yml @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java HBase IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/hbase/**" + - "sdks/java/io/hadoop-common/**" + - ".github/workflows/beam_PreCommit_Java_HBase_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/hbase/**" + - "sdks/java/io/hadoop-common/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_HBase_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_HBase_IO_Direct"] + job_phrase: ["Run Java_HBase_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_HBase_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run HBase IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:hbase:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml new file mode 100644 index 0000000000000..93eacb82cf2bc --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java HCatalog IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/hcatalog/**" + - "sdks/java/io/hadoop-common/**" + - ".github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/hcatalog/**" + - "sdks/java/io/hadoop-common/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_HCatalog_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_HCatalog_IO_Direct"] + job_phrase: ["Run Java_HCatalog_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_HCatalog_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run HCatalog IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:hcatalog:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Hadoop_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Hadoop_IO_Direct.yml new file mode 100644 index 0000000000000..2a61bcfb38096 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Hadoop_IO_Direct.yml @@ -0,0 +1,155 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Hadoop IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/hadoop-file-system/**" + - "sdks/java/io/common/**" + - "sdks/java/core/src/main/**" + - "build.gradle" + - "buildSrc/**" + - "gradle/**" + - "gradle.properties" + - "gradlew" + - "gradle.bat" + - "settings.gradle.kts" + - "examples/java/**" + - "sdks/java/testing/test-utils/**" + - "sdks/java/io/hadoop-common/**" + - "sdks/java/io/hadoop-format/**" + - ".github/workflows/beam_PreCommit_Java_Hadoop_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/hadoop-file-system/**" + - "sdks/java/io/common/**" + - "sdks/java/core/src/main/**" + - "build.gradle" + - "buildSrc/**" + - "gradle/**" + - "gradle.properties" + - "gradlew" + - "gradle.bat" + - "settings.gradle.kts" + - "examples/java/**" + - "sdks/java/testing/test-utils/**" + - "sdks/java/io/hadoop-common/**" + - "sdks/java/io/hadoop-format/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Hadoop_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Hadoop_IO_Direct"] + job_phrase: ["Run Java_Hadoop_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Hadoop_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Hadoop IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:hadoop-file-system:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: run Hadoop Common IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:hadoop-common:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: run Hadoop Format IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:hadoop-format:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml b/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml new file mode 100644 index 0000000000000..7936c391a644b --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java IOs Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/common/**" + - "sdks/java/core/src/main/**" + - ".github/workflows/beam_PreCommit_Java_IOs_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/common/**" + - "sdks/java/core/src/main/**" + issue_comment: + types: [created] + workflow_dispatch: +# disable cron run because the tasks are covered by the single IO precommits below + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_IOs_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_IOs_Direct"] + job_phrase: ["Run Java_IOs_Direct PreCommit"] + timeout-minutes: 120 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_IOs_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Java IOs PreCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :javaioPreCommit + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + -Dfile.encoding=UTF-8 \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_InfluxDb_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_InfluxDb_IO_Direct.yml new file mode 100644 index 0000000000000..607a202b44944 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_InfluxDb_IO_Direct.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java InfluxDb IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/influxdb/**" + - ".github/workflows/beam_PreCommit_Java_InfluxDb_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/influxdb/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_InfluxDb_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_InfluxDb_IO_Direct"] + job_phrase: ["Run Java_InfluxDb_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_InfluxDb_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run InfluxDb IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:influxdb:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_JDBC_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_JDBC_IO_Direct.yml new file mode 100644 index 0000000000000..eab48ce8a0f37 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_JDBC_IO_Direct.yml @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java JDBC IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/jdbc/**" + - ".github/workflows/beam_PreCommit_Java_JDBC_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/jdbc/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_JDBC_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_JDBC_IO_Direct"] + job_phrase: ["Run Java_JDBC_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_JDBC_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run JDBC IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:jdbc:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: run JDBC IO IT script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:jdbc:integrationTest + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Jms_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Jms_IO_Direct.yml new file mode 100644 index 0000000000000..aebe9984ab735 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Jms_IO_Direct.yml @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Jms IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/jms/**" + - ".github/workflows/beam_PreCommit_Java_Jms_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/jms/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Jms_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Jms_IO_Direct"] + job_phrase: ["Run Java_Jms_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Jms_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Jms IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:jms:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: run Jms IO IT script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:jms:integrationTest + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Kafka_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Kafka_IO_Direct.yml new file mode 100644 index 0000000000000..8b7b35922d372 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Kafka_IO_Direct.yml @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Kafka IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/kafka/**" + - "sdks/java/testing/test-utils/**" + - "sdks/java/expansion-service/**" + - "sdks/java/io/synthetic/**" + - "sdks/java/io/expansion-service/**" + - ".github/workflows/beam_PreCommit_Java_Kafka_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/kafka/**" + - "sdks/java/testing/test-utils/**" + - "sdks/java/expansion-service/**" + - "sdks/java/io/synthetic/**" + - "sdks/java/io/expansion-service/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Kafka_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Kafka_IO_Direct"] + job_phrase: ["Run Java_Kafka_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Kafka_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Kafka IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:kafka:build :sdks:java:io:kafka:kafkaVersionsCompatibilityTest + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + --no-parallel \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Kinesis_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Kinesis_IO_Direct.yml new file mode 100644 index 0000000000000..be29e3b874950 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Kinesis_IO_Direct.yml @@ -0,0 +1,147 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Kinesis IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/kinesis/**" + - "sdks/java/io/common/**" + - "sdks/java/core/src/main/**" + - "build.gradle" + - "buildSrc/**" + - "gradle/**" + - "gradle.properties" + - "gradlew" + - "gradle.bat" + - "settings.gradle.kts" + - ".github/workflows/beam_PreCommit_Java_Kinesis_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/kinesis/**" + - "sdks/java/io/common/**" + - "sdks/java/core/src/main/**" + - "build.gradle" + - "buildSrc/**" + - "gradle/**" + - "gradle.properties" + - "gradlew" + - "gradle.bat" + - "settings.gradle.kts" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Kinesis_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Kinesis_IO_Direct"] + job_phrase: ["Run Java_Kinesis_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Kinesis_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Kinesis IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:kinesis:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: run Kinesis expansion service script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:kinesis:expansion-service:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: run Kinesis IO IT script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:kinesis:integrationTest + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Kudu_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Kudu_IO_Direct.yml new file mode 100644 index 0000000000000..12cb1b2aa2306 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Kudu_IO_Direct.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Kudu IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/kudu/**" + - ".github/workflows/beam_PreCommit_Java_Kudu_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/kudu/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Kudu_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Kudu_IO_Direct"] + job_phrase: ["Run Java_Kudu_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Kudu_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Kudu IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:kudu:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_MongoDb_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_MongoDb_IO_Direct.yml new file mode 100644 index 0000000000000..19ca7a24a27a5 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_MongoDb_IO_Direct.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java MongoDb IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/mongodb/**" + - ".github/workflows/beam_PreCommit_Java_MongoDb_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/mongodb/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_MongoDb_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_MongoDb_IO_Direct"] + job_phrase: ["Run Java_MongoDb_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_MongoDb_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run MongoDb IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:mongodb:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Mqtt_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Mqtt_IO_Direct.yml new file mode 100644 index 0000000000000..5e9c8ecf9a48f --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Mqtt_IO_Direct.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Mqtt IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/mqtt/**" + - ".github/workflows/beam_PreCommit_Java_Mqtt_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/mqtt/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Mqtt_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Mqtt_IO_Direct"] + job_phrase: ["Run Java_Mqtt_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Mqtt_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Mqtt IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:mqtt:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Neo4j_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Neo4j_IO_Direct.yml new file mode 100644 index 0000000000000..0dc3f2c946adb --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Neo4j_IO_Direct.yml @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Neo4j IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/neo4j/**" + - "sdks/java/testing/test-utils/**" + - ".github/workflows/beam_PreCommit_Java_Neo4j_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/neo4j/**" + - "sdks/java/testing/test-utils/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Neo4j_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: [ "beam_PreCommit_Java_Neo4j_IO_Direct" ] + job_phrase: ["Run Java_Neo4j_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Neo4j_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Neo4j IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:neo4j:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: run Neo4j IO IT script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:neo4j:integrationTest + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_PVR_Flink_Batch.yml b/.github/workflows/beam_PreCommit_Java_PVR_Flink_Batch.yml new file mode 100644 index 0000000000000..283e0c959d098 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_PVR_Flink_Batch.yml @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Java PVR Flink Batch + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - 'runners/flink/**' + - 'runners/java-fn-execution/**' + - 'sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/**' + - '.github/workflows/beam_PreCommit_Java_PVR_Flink_Batch.yml' + pull_request_target: + branches: ['master', 'release-*'] + paths: + - 'runners/flink/**' + - 'runners/java-fn-execution/**' + - 'sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/**' + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_PVR_Flink_Batch: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_PVR_Flink_Batch"] + job_phrase: ["Run Java_PVR_Flink_Batch PreCommit"] + timeout-minutes: 240 + runs-on: [self-hosted, ubuntu-20.04, highmem] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_PVR_Flink_Batch PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run validatesPortableRunnerBatch script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:flink:1.15:job-server:validatesPortableRunnerBatch + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH }} + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Upload test report + uses: actions/upload-artifact@v3 + with: + name: java-code-coverage-report + path: "**/build/test-results/**/*.xml" +# TODO: Investigate 'Max retries exceeded' issue with EnricoMi/publish-unit-test-result-action@v2. \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_PVR_Flink_Docker.yml b/.github/workflows/beam_PreCommit_Java_PVR_Flink_Docker.yml new file mode 100644 index 0000000000000..e41f59f67205d --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_PVR_Flink_Docker.yml @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Java PVR Flink Docker + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - 'sdks/java/core/src/test/java/org/apache/beam/sdk/**' + - 'sdks/java/container/**' + - 'sdks/java/harness/**' + - 'runners/flink/**' + - 'runners/java-fn-execution/**' + - '.github/workflows/beam_PreCommit_Java_PVR_Flink_Docker.yml' + pull_request_target: + branches: ['master', 'release-*'] + paths: + - 'sdks/java/core/src/test/java/org/apache/beam/sdk/**' + - 'sdks/java/container/**' + - 'sdks/java/harness/**' + - 'runners/flink/**' + - 'runners/java-fn-execution/**' + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_PVR_Flink_Docker: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + fail-fast: false + matrix: + job_name: [ beam_PreCommit_Java_PVR_Flink_Docker ] + job_phrase: [ Run Java_PVR_Flink_Docker PreCommit ] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_PVR_Flink_Docker PreCommit' + timeout-minutes: 240 + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run PreCommit Java PVR Flink Docker script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:flink:1.15:job-server:validatesPortableRunnerDocker + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Parquet_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Parquet_IO_Direct.yml new file mode 100644 index 0000000000000..a0f021bb82d56 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Parquet_IO_Direct.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Parquet IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/parquet/**" + - ".github/workflows/beam_PreCommit_Java_Parquet_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/parquet/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Parquet_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Parquet_IO_Direct"] + job_phrase: ["Run Java_Parquet_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Parquet_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Parquet IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:parquet:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Pulsar_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Pulsar_IO_Direct.yml new file mode 100644 index 0000000000000..ef22e19ca514a --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Pulsar_IO_Direct.yml @@ -0,0 +1,133 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Pulsar IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/pulsar/**" + - "sdks/java/io/common/**" + - "sdks/java/core/src/main/**" + - "build.gradle" + - "buildSrc/**" + - "gradle/**" + - "gradle.properties" + - "gradlew" + - "gradle.bat" + - "settings.gradle.kts" + - ".github/workflows/beam_PreCommit_Java_Pulsar_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/pulsar/**" + - "sdks/java/io/common/**" + - "sdks/java/core/src/main/**" + - "build.gradle" + - "buildSrc/**" + - "gradle/**" + - "gradle.properties" + - "gradlew" + - "gradle.bat" + - "settings.gradle.kts" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Pulsar_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Pulsar_IO_Direct"] + job_phrase: ["Run Java_Pulsar_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Pulsar_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Pulsar IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:pulsar:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_RabbitMq_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_RabbitMq_IO_Direct.yml new file mode 100644 index 0000000000000..f03b38b18257d --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_RabbitMq_IO_Direct.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java RabbitMq IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/rabbitmq/**" + - ".github/workflows/beam_PreCommit_Java_RabbitMq_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/rabbitmq/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_RabbitMq_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_RabbitMq_IO_Direct"] + job_phrase: ["Run Java_RabbitMq_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_RabbitMq_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run RabbitMq IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:rabbitmq:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Redis_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Redis_IO_Direct.yml new file mode 100644 index 0000000000000..21efdf4b574c4 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Redis_IO_Direct.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Redis IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/redis/**" + - ".github/workflows/beam_PreCommit_Java_Redis_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/redis/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Redis_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Redis_IO_Direct"] + job_phrase: ["Run Java_Redis_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Redis_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Redis IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:redis:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml new file mode 100644 index 0000000000000..4e98c1fee7082 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java SingleStore IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/singlestore/**" + - "sdks/java/testing/test-utils/**" + - ".github/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/singlestore/**" + - "sdks/java/testing/test-utils/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_SingleStore_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_SingleStore_IO_Direct"] + job_phrase: ["Run Java_SingleStore_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_SingleStore_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run SingleStore IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:singlestore:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml new file mode 100644 index 0000000000000..4f7cfb6b8316a --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Snowflake IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/snowflake/**" + - "sdks/java/extensions/google-cloud-platform-core/**" + - "sdks/java/testing/test-utils/**" + - ".github/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/snowflake/**" + - "sdks/java/extensions/google-cloud-platform-core/**" + - "sdks/java/testing/test-utils/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Snowflake_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Snowflake_IO_Direct"] + job_phrase: ["Run Java_Snowflake_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Snowflake_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Snowflake IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:snowflake:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: run Snowflake expansion service script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:snowflake:expansion-service:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml new file mode 100644 index 0000000000000..4a30cfbf3f40d --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Solr IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/solr/**" + - ".github/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/solr/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Solr_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Solr_IO_Direct"] + job_phrase: ["Run Java_Solr_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Solr_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Solr IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:solr:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml b/.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml new file mode 100644 index 0000000000000..1809f11d8c7c3 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Java Spark3 Versions + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - 'runners/spark/**' + - '.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml' + pull_request_target: + branches: ['master', 'release-*'] + paths: + - 'runners/spark/**' + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Spark3_Versions: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_Java_Spark3_Versions] + job_phrase: [Run Java_Spark3_Versions PreCommit] + timeout-minutes: 120 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Spark3_Versions PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + export_default_credentials: true + - name: Install Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'zulu' + java-version: '8' + cache: 'gradle' + check-latest: true + - name: run sparkVersionsTest script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:spark:3:sparkVersionsTest + arguments: -PdisableSpotlessCheck=true + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml new file mode 100644 index 0000000000000..786276aeb885c --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Splunk IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/splunk/**" + - ".github/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/splunk/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Splunk_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Splunk_IO_Direct"] + job_phrase: ["Run Java_Splunk_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Splunk_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Splunk IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:splunk:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml new file mode 100644 index 0000000000000..64cd4a755e731 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Thrift IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/thrift/**" + - ".github/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/thrift/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Thrift_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Thrift_IO_Direct"] + job_phrase: ["Run Java_Thrift_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Thrift_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Thrift IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:thrift:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml new file mode 100644 index 0000000000000..e6bed237a8086 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Tika IO Direct + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/tika/**" + - ".github/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/tika/**" + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Tika_IO_Direct: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Tika_IO_Direct"] + job_phrase: ["Run Java_Tika_IO_Direct PreCommit"] + timeout-minutes: 60 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Tika_IO_Direct PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Tika IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:tika:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Kotlin_Examples.yml b/.github/workflows/beam_PreCommit_Kotlin_Examples.yml new file mode 100644 index 0000000000000..8fe1673d9c558 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Kotlin_Examples.yml @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Kotlin Examples + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - 'model/**' + - 'sdks/java/**' + - 'runners/flink/**' + - 'runners/spark/**' + - 'runners/direct-java/**' + - 'examples/kotlin/**' + - 'release/**' + - '.github/workflows/beam_PreCommit_Kotlin_Examples.yml' + pull_request_target: + branches: ['master', 'release-*'] + paths: + - 'model/**' + - 'sdks/java/**' + - 'runners/flink/**' + - 'runners/spark/**' + - 'runners/direct-java/**' + - 'examples/kotlin/**' + - 'release/**' + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Kotlin_Examples: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + timeout-minutes: 120 + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_Kotlin_Examples] + job_phrase: [Run Kotlin_Examples PreCommit] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Kotlin_Examples PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'zulu' + java-version: '8' + cache: 'gradle' + check-latest: true + - name: run Kotlin Examples script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :examples:kotlin:preCommit \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Portable_Python.yml b/.github/workflows/beam_PreCommit_Portable_Python.yml new file mode 100644 index 0000000000000..e8221f7f23f5e --- /dev/null +++ b/.github/workflows/beam_PreCommit_Portable_Python.yml @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Portable Python + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - 'model/**' + - 'runners/core-construction-java/**' + - 'runners/core-java/**' + - 'runners/extensions-java/**' + - 'runners/flink/**' + - 'runners/java-fn-execution/**' + - 'runners/reference/**' + - 'sdks/python/**' + - 'release/**' + - '.github/workflows/beam_PreCommit_Portable_Python.yml' + pull_request_target: + branches: ['master', 'release-*'] + paths: + - 'model/**' + - 'runners/core-construction-java/**' + - 'runners/core-java/**' + - 'runners/extensions-java/**' + - 'runners/flink/**' + - 'runners/java-fn-execution/**' + - 'runners/reference/**' + - 'sdks/python/**' + - 'release/**' + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Portable_Python: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + timeout-minutes: 120 + runs-on: ['self-hosted', ubuntu-20.04, main] + strategy: + fail-fast: false + matrix: + job_name: ['beam_PreCommit_Portable_Python'] + job_phrase: ['Run Portable_Python PreCommit'] + python_version: ['3.8', '3.11'] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + startsWith(github.event.comment.body, 'Run Portable_Python PreCommit') + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Install Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'zulu' + java-version: '8' + cache: 'gradle' + check-latest: true + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: | + ${{ matrix.python_version }} + 3.8 + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: run Portable Python script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:portable:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:preCommitPy${{steps.set_py_ver_clean.outputs.py_ver_clean}} \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Python.yml b/.github/workflows/beam_PreCommit_Python.yml new file mode 100644 index 0000000000000..d65a8e643eb48 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Python.yml @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Python +on: + pull_request_target: + branches: [ "master", "release-*" ] + paths: [ "model/**","sdks/python/**","release/**"] + issue_comment: + types: [created] + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: [ "model/**","sdks/python/**","release/**",".github/workflows/beam_PreCommit_Python.yml"] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Python: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + job_name: ['beam_PreCommit_Python'] + job_phrase: ['Run Python PreCommit'] + python_version: ['3.8','3.9','3.10','3.11'] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + startsWith(github.event.comment.body, 'Run Python PreCommit') + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: ${{ matrix.python_version }} + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: Run pythonPreCommit + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:preCommitPy${{steps.set_py_ver_clean.outputs.py_ver_clean}} + arguments: | + -Pposargs="--ignore=apache_beam/dataframe/ --ignore=apache_beam/examples/ --ignore=apache_beam/runners/ --ignore=apache_beam/transforms/" \ + -PpythonVersion=${{ matrix.python_version }} \ + -PuseWheelDistribution + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_PythonDocker.yml b/.github/workflows/beam_PreCommit_PythonDocker.yml new file mode 100644 index 0000000000000..dc5ee73e198d6 --- /dev/null +++ b/.github/workflows/beam_PreCommit_PythonDocker.yml @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Python Docker +on: + pull_request_target: + branches: [ "master", "release-*" ] + paths: [ "model/**","sdks/python/**","release/**"] + issue_comment: + types: [created] + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: [ "model/**","sdks/python/**","release/**",".github/workflows/beam_PreCommit_PythonDocker.yml"] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_PythonDocker: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + job_name: ["beam_PreCommit_PythonDocker"] + job_phrase: ["Run PythonDocker PreCommit"] + python_version: ['3.8','3.9','3.10','3.11'] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + startsWith(github.event.comment.body, 'Run PythonDocker PreCommit') + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: ${{ matrix.python_version }} + go-version: 1.16 + - name: Setup Buildx + uses: docker/setup-buildx-action@v2 + with: + install: true + driver: 'docker' + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: Run pythonDockerBuildPreCommit + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:container:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:docker + arguments: | + -Pposargs=apache_beam/dataframe/ \ + -PpythonVersion=${{ matrix.python_version }} \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_PythonDocs.yml b/.github/workflows/beam_PreCommit_PythonDocs.yml new file mode 100644 index 0000000000000..896d954386b51 --- /dev/null +++ b/.github/workflows/beam_PreCommit_PythonDocs.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Python Docs + +on: + pull_request_target: + branches: [ "master", "release-*" ] + paths: ["sdks/python/**"] + issue_comment: + types: [created] + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: ["sdks/python/**",".github/workflows/beam_PreCommit_PythonDocs.yml"] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_PythonDocs: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_PythonDocs] + job_phrase: [Run PythonDocs PreCommit] + timeout-minutes: 30 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run PythonDocs PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: 3.8 + - name: run pythonDocsPreCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :pythonDocsPreCommit diff --git a/.github/workflows/beam_PreCommit_PythonFormatter.yml b/.github/workflows/beam_PreCommit_PythonFormatter.yml new file mode 100644 index 0000000000000..e9c6787f6ee90 --- /dev/null +++ b/.github/workflows/beam_PreCommit_PythonFormatter.yml @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Python Formatter +on: + pull_request_target: + branches: [ "master", "release-*" ] + paths: [ "sdks/python/apache_beam/**"] + issue_comment: + types: [created] + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: [ "sdks/python/apache_beam/**",".github/workflows/beam_PreCommit_PythonFormatter.yml"] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_PythonFormatter: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_PythonFormatter] + job_phrase: [Run PythonFormatter PreCommit] + timeout-minutes: 120 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run PythonFormatter PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: 3.8 + - name: run pythonFormatterPreCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :pythonFormatterPreCommit diff --git a/.github/workflows/beam_PreCommit_PythonLint.yml b/.github/workflows/beam_PreCommit_PythonLint.yml new file mode 100644 index 0000000000000..2a1f186f2418e --- /dev/null +++ b/.github/workflows/beam_PreCommit_PythonLint.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Python Lint +on: + pull_request_target: + branches: [ "master", "release-*" ] + paths: ["sdks/python/**","release/**"] + issue_comment: + types: [created] + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: ["sdks/python/**","release/**",".github/workflows/beam_PreCommit_PythonLint.yml"] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_PythonLint: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_PythonLint] + job_phrase: [Run PythonLint PreCommit] + timeout-minutes: 120 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run PythonLint PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: 3.8 + go-version: 1.16 + - name: run pythonLintPreCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :pythonLintPreCommit diff --git a/.github/workflows/beam_PreCommit_Python_Coverage.yml b/.github/workflows/beam_PreCommit_Python_Coverage.yml new file mode 100644 index 0000000000000..ef8f9ee1bf1d0 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Python_Coverage.yml @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Python Coverage +on: + pull_request_target: + branches: [ "master", "release-*" ] + paths: [ "model/**","sdks/python/**","release/**"] + issue_comment: + types: [created] + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: [ "model/**","sdks/python/**","release/**", ".github/workflows/beam_PreCommit_Python_Coverage.yml"] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Python_Coverage: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_Python_Coverage] + job_phrase: [Run Python_Coverage PreCommit] + timeout-minutes: 180 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Python_Coverage PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: 3.8 + - name: Run preCommitPyCoverage + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py38:preCommitPyCoverage + arguments: | + -PuseWheelDistribution + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Python_Dataframes.yml b/.github/workflows/beam_PreCommit_Python_Dataframes.yml new file mode 100644 index 0000000000000..b2292e3484a3c --- /dev/null +++ b/.github/workflows/beam_PreCommit_Python_Dataframes.yml @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Python Dataframes +on: + pull_request_target: + branches: [ "master", "release-*" ] + paths: [ "model/**","sdks/python/**","release/**"] + issue_comment: + types: [created] + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: [ "model/**","sdks/python/**","release/**",".github/workflows/beam_PreCommit_Python_Dataframes.yml"] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Python_Dataframes: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase}} ${{ matrix.python_version}}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + job_name: ['beam_PreCommit_Python_Dataframes'] + job_phrase: ['Run Python_Dataframes PreCommit'] + python_version: ['3.8','3.9','3.10','3.11'] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + startsWith(github.event.comment.body, 'Run Python_Dataframes PreCommit') + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase}} ${{ matrix.python_version}} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name}} (${{ matrix.job_phrase}} ${{ matrix.python_version}}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: ${{ matrix.python_version }} + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: Run pythonPreCommit + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:preCommitPy${{steps.set_py_ver_clean.outputs.py_ver_clean}} + arguments: | + -Pposargs=apache_beam/dataframe/ \ + -PpythonVersion=${{ matrix.python_version }} \ + -PuseWheelDistribution + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Python_Examples.yml b/.github/workflows/beam_PreCommit_Python_Examples.yml new file mode 100644 index 0000000000000..96d7149736103 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Python_Examples.yml @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Python Examples +on: + pull_request_target: + branches: [ "master", "release-*" ] + paths: [ "model/**","sdks/python/**","release/**"] + issue_comment: + types: [created] + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: [ "model/**","sdks/python/**","release/**",".github/workflows/beam_PreCommit_Python_Examples.yml"] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Python_Examples: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + job_name: ['beam_PreCommit_Python_Examples'] + job_phrase: ['Run Python_Examples PreCommit'] + python_version: ['3.8','3.9','3.10','3.11'] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + startsWith(github.event.comment.body, 'Run Python_Examples PreCommit') + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: ${{ matrix.python_version }} + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: Run pythonPreCommit + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:preCommitPy${{steps.set_py_ver_clean.outputs.py_ver_clean}} + arguments: | + -Pposargs=apache_beam/examples/ \ + -PpythonVersion=${{ matrix.python_version }} \ + -PuseWheelDistribution + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Python_Integration.yml b/.github/workflows/beam_PreCommit_Python_Integration.yml new file mode 100644 index 0000000000000..afb11abaabc91 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Python_Integration.yml @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Python Integration +on: + pull_request_target: + branches: [ "master", "release-*" ] + paths: ["model/**", "sdks/python/**", "release/**"] + issue_comment: + types: [created] + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: ["model/**", "sdks/python/**", "release/**", ".github/workflows/beam_PreCommit_Python_Integration.yml"] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Python_Integration: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + job_name: ['beam_PreCommit_Python_Integration'] + job_phrase: ['Run Python_Integration PreCommit'] + python_version: ['3.8', '3.11'] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + startsWith(github.event.comment.body, 'Run Python_Integration PreCommit') + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: ${{ matrix.python_version }} + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: run Python Integration PreCommit batch script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:dataflow:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:preCommitIT_batch + arguments: | + -PuseWheelDistribution \ + -PpythonVersion=${{ matrix.python_version }} \ + - name: run Python Integration PreCommit streaming script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:dataflow:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:preCommitIT_streaming + arguments: | + -PuseWheelDistribution \ + -PpythonVersion=${{ matrix.python_version }} \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" diff --git a/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml b/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml new file mode 100644 index 0000000000000..94655f959fd77 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Python PVR Flink + +on: + pull_request_target: + branches: ['master', 'release-*'] + paths: + - 'model/**' + - 'sdks/python/**' + - 'release/**' + - 'sdks/java/io/kafka/**' + - 'runners/core-construction-java/**' + - 'runners/core-java/**' + - 'runners/extensions-java/**' + - 'runners/flink/**' + - 'runners/java-fn-execution/**' + - 'runners/reference/**' + issue_comment: + types: [created] + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - 'model/**' + - 'sdks/python/**' + - 'release/**' + - 'sdks/java/io/kafka/**' + - 'runners/core-construction-java/**' + - 'runners/core-java/**' + - 'runners/extensions-java/**' + - 'runners/flink/**' + - 'runners/java-fn-execution/**' + - 'runners/reference/**' + - '.github/workflows/beam_PreCommit_Python_PVR_Flink.yml' + schedule: + - cron: '* */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Python_PVR_Flink: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Python_PVR_Flink"] + job_phrase: ["Run Python_PVR_Flink PreCommit"] + timeout-minutes: 120 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Python_PVR_Flink PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: 3.11 + - name: run Python PVR Flink PreCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + env: + CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} + with: + gradle-command: :sdks:python:test-suites:portable:py311:flinkValidatesRunner + arguments: | + -PpythonVersion=3.11 \ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: '**/pytest*.xml' diff --git a/.github/workflows/beam_PreCommit_Python_Runners.yml b/.github/workflows/beam_PreCommit_Python_Runners.yml new file mode 100644 index 0000000000000..e3ad7675e9834 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Python_Runners.yml @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Python Runners +on: + pull_request_target: + branches: [ "master", "release-*" ] + paths: [ "model/**","sdks/python/**","release/**"] + issue_comment: + types: [created] + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: [ "model/**","sdks/python/**","release/**",".github/workflows/beam_PreCommit_Python_Runners.yml"] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Python_Runners: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + job_name: ['beam_PreCommit_Python_Runners'] + job_phrase: ['Run Python_Runners PreCommit'] + python_version: ['3.8','3.9','3.10','3.11'] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + startsWith(github.event.comment.body, 'Run Python_Runners PreCommit') + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: ${{ matrix.python_version }} + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: Run pythonPreCommit + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:preCommitPy${{steps.set_py_ver_clean.outputs.py_ver_clean}} + arguments: | + -Pposargs=apache_beam/runners/ \ + -PpythonVersion=${{ matrix.python_version }} \ + -PuseWheelDistribution + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Python_Transforms.yml b/.github/workflows/beam_PreCommit_Python_Transforms.yml new file mode 100644 index 0000000000000..b62d4b4665ef6 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Python_Transforms.yml @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Python Transforms +on: + pull_request_target: + branches: [ "master", "release-*" ] + paths: [ "model/**","sdks/python/**","release/**"] + issue_comment: + types: [created] + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: [ "model/**","sdks/python/**","release/**",".github/workflows/beam_PreCommit_Python_Transforms.yml"] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Python_Transforms: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + job_name: ['beam_PreCommit_Python_Transforms'] + job_phrase: ['Run Python_Transforms PreCommit'] + python_version: ['3.8','3.9','3.10','3.11'] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + startsWith(github.event.comment.body, 'Run Python_Transforms PreCommit') + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: ${{ matrix.python_version }} + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: Run pythonPreCommit + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:preCommitPy${{steps.set_py_ver_clean.outputs.py_ver_clean}} + arguments: | + -Pposargs=apache_beam/transforms/ \ + -PpythonVersion=${{ matrix.python_version }} \ + -PuseWheelDistribution + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_RAT.yml b/.github/workflows/beam_PreCommit_RAT.yml new file mode 100644 index 0000000000000..3d907410b5cd7 --- /dev/null +++ b/.github/workflows/beam_PreCommit_RAT.yml @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit RAT + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + pull_request_target: + branches: ['master', 'release-*'] + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_RAT: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_RAT] + job_phrase: [Run RAT PreCommit] + timeout-minutes: 120 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run RAT PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + - name: run RAT script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :rat diff --git a/.github/workflows/beam_PreCommit_SQL.yml b/.github/workflows/beam_PreCommit_SQL.yml new file mode 100644 index 0000000000000..3dd36ce050af2 --- /dev/null +++ b/.github/workflows/beam_PreCommit_SQL.yml @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit SQL + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: ['sdks/java/extensions/sql/**','.github/workflows/beam_PreCommit_SQL.yml'] + pull_request_target: + branches: ['master', 'release-*'] + paths: [sdks/java/extensions/sql/**] + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_SQL: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_SQL] + job_phrase: [Run SQL PreCommit] + timeout-minutes: 120 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run SQL PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Set up Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'temurin' + java-version: '11' + - name: Build and Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sqlPreCommit + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + -PenableJacocoReport \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_SQL_Java11.yml b/.github/workflows/beam_PreCommit_SQL_Java11.yml new file mode 100644 index 0000000000000..0f8c032a598f2 --- /dev/null +++ b/.github/workflows/beam_PreCommit_SQL_Java11.yml @@ -0,0 +1,133 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit SQL Java11 + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: ['sdks/java/extensions/sql/**','.github/workflows/beam_PreCommit_SQL_Java11.yml'] + pull_request_target: + branches: ['master', 'release-*'] + paths: [sdks/java/extensions/sql/**] + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +jobs: + beam_PreCommit_SQL_Java11: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_SQL_Java11] + job_phrase: [Run SQL_Java11 PreCommit] + timeout-minutes: 120 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run SQL_Java11 PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Set up Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'temurin' + java-version: '11' + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: '1.21' + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name: Install Flutter + uses: subosito/flutter-action@v2 + with: + flutter-version: '3.10.4' + channel: 'stable' + - name: Build and Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sqlPreCommit + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + -PcompileAndRunTestsWithJava11 \ + -PskipCheckerFramework \ + -Pjava11Home=$JAVA_HOME_11_X64 \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_SQL_Java17.yml b/.github/workflows/beam_PreCommit_SQL_Java17.yml new file mode 100644 index 0000000000000..4b67db10f5fc2 --- /dev/null +++ b/.github/workflows/beam_PreCommit_SQL_Java17.yml @@ -0,0 +1,131 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit SQL Java17 + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: ['sdks/java/extensions/sql/**','.github/workflows/beam_PreCommit_SQL_Java17.yml'] + pull_request_target: + branches: ['master', 'release-*'] + paths: [sdks/java/extensions/sql/**] + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_SQL_Java17: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_SQL_Java17"] + job_phrase: ["Run SQL_Java17 PreCommit"] + timeout-minutes: 120 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run SQL_Java17 PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) +# The test requires Java 17 and Java 8 versions. +# Java 8 is installed second because JAVA_HOME needs to point to Java 8. + - name: Set up Java 17 + uses: actions/setup-java@v3.11.0 + with: + distribution: 'temurin' + java-version: '17' + - name: Set up Java 8 + uses: actions/setup-java@v3.11.0 + with: + distribution: 'temurin' + java-version: '8' + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name: Build and Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sqlPreCommit + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + -PcompileAndRunTestsWithJava17 \ + -PskipCheckerFramework \ + -Pjava17Home=$JAVA_HOME_17_X64 \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v3 + if: failure() + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v3 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Spotless.yml b/.github/workflows/beam_PreCommit_Spotless.yml new file mode 100644 index 0000000000000..f3fc3794bdf3d --- /dev/null +++ b/.github/workflows/beam_PreCommit_Spotless.yml @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Spotless + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - 'buildSrc/**' + - 'sdks/java/**' + - 'runners/**' + - 'examples/java/**' + - 'examples/kotlin/**' + - '.test-infra/jenkins/' + - '.github/workflows/beam_PreCommit_Spotless.yml' + pull_request_target: + branches: ['master', 'release-*'] + paths: + - 'buildSrc/**' + - 'sdks/java/**' + - 'runners/**' + - 'examples/java/**' + - 'examples/kotlin/**' + - '.test-infra/jenkins/' + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: write + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Spotless: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + timeout-minutes: 120 + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + fail-fast: false + matrix: + job_name: [beam_PreCommit_Spotless] + job_phrase: [Run Spotless PreCommit] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Spotless PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: run Spotless PreCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: spotlessCheck checkStyleMain checkStyleTest + - name: Upload test report + uses: actions/upload-artifact@v3 + with: + name: java-code-coverage-report + path: "**/build/reports/checkstyle/*.xml" + - name: Publish checkstyle check + uses: jwgmeligmeyling/checkstyle-github-action@v1 + if: always() + with: + path: '**/build/reports/checkstyle/*.xml' \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Typescript.yml b/.github/workflows/beam_PreCommit_Typescript.yml new file mode 100644 index 0000000000000..803d51e7e9a42 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Typescript.yml @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Typescript + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: ['sdks/python/apache_beam/runners/interactive/extensions/**', '.github/workflows/beam_PreCommit_Typescript.yml'] + pull_request_target: + branches: ['master', 'release-*'] + paths: ['sdks/python/apache_beam/runners/interactive/extensions/**'] + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + + # This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read +jobs: + beam_PreCommit_Typescript: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + timeout-minutes: 120 + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_Typescript] + job_phrase: [Run Typescript PreCommit] + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Typescript PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + java-version: 8 + - name: run typescriptPreCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :typescriptPreCommit \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Website.yml b/.github/workflows/beam_PreCommit_Website.yml index b7a4067f25499..bce51e0c929a1 100644 --- a/.github/workflows/beam_PreCommit_Website.yml +++ b/.github/workflows/beam_PreCommit_Website.yml @@ -13,37 +13,76 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: beam_PreCommit_Website +name: PreCommit Website on: push: tags: ['v*'] branches: ['master', 'release-*'] - paths: ['website/**'] - pull_request: + paths: ['website/**','.github/workflows/beam_PreCommit_Website.yml'] + pull_request_target: branches: ['master', 'release-*'] paths: ['website/**'] issue_comment: types: [created] schedule: - - cron: '* */6 * * *' + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} jobs: - beam_PreCommit_Website: - if: ${{github.event.issue.pull_request}} || ${{github.event.comment.body == 'Run Website PreCommit'}} || ${{github.event.schedule}} - runs-on: [self-hosted, ubuntu-20.04] - name: beam_PreCommit_Website - steps: - - name: Git checkout - uses: actions/checkout@v3 - - name: Install Java - uses: actions/setup-java@v3.8.0 - with: - distribution: 'zulu' - java-version: '8' - - name: Setup Gradle - uses: gradle/gradle-build-action@v2 - with: - cache-read-only: false - - name: run websitePreCommit script - run: ./gradlew :websitePreCommit \ No newline at end of file + beam_PreCommit_Website: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase}}) + runs-on: [self-hosted, ubuntu-20.04, small] + strategy: + matrix: + job_name: [beam_PreCommit_Website] + job_phrase: [Run Website PreCommit] + timeout-minutes: 120 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Website PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + - name: run websitePreCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :websitePreCommit \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Website_Stage_GCS.yml b/.github/workflows/beam_PreCommit_Website_Stage_GCS.yml new file mode 100644 index 0000000000000..19d96eb8a1ce2 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Website_Stage_GCS.yml @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Website Stage GCS + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: ['website/**','.github/workflows/beam_PreCommit_Website_Stage_GCS.yml'] + pull_request_target: + branches: ['master', 'release-*'] + paths: ['website/**'] + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + gcsbucket: apache-beam-website-pull-requests + ghprbPullId: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +jobs: + beam_PreCommit_Website_Stage_GCS: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_Website_Stage_GCS] + job_phrase: [Run Website_Stage_GCS PreCommit] + timeout-minutes: 120 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Website_Stage_GCS PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Echo PR number + run: echo "ghprbPullId=${{ github.event.pull_request.number || github.event.issue.number }}" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + java-version: 8 + - name: Authenticate on GCP + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + export_default_credentials: true + - name: Run website_stageWebsite script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :website:stageWebsite + arguments: -PwebsiteBucket=$gcsbucket + - name: Add website link to the summary if pr + if: env.ghprbPullId != null + run: echo "Website published to http://$gcsbucket.storage.googleapis.com/$ghprbPullId/index.html" >> $GITHUB_STEP_SUMMARY + - name: Add website link to the summary + if: env.ghprbPullId == null + run: | + USERNAME="$(whoami)" + echo "Website published to http://$gcsbucket.storage.googleapis.com/$USERNAME-${GITHUB_REF##*/}/index.html" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/beam_PreCommit_Whitespace.yml b/.github/workflows/beam_PreCommit_Whitespace.yml index a47531180bc97..148de272b622a 100644 --- a/.github/workflows/beam_PreCommit_Whitespace.yml +++ b/.github/workflows/beam_PreCommit_Whitespace.yml @@ -13,41 +13,76 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: beam_PreCommit_Whitespace +name: PreCommit Whitespace on: push: tags: ['v*'] branches: ['master', 'release-*'] - paths: ['*.md', '*.build.gradle'] - pull_request: + paths: ['**.md', '**.build.gradle', 'build.gradle.kts', '.github/workflows/beam_PreCommit_Whitespace.yml'] + pull_request_target: branches: ['master', 'release-*'] - paths: ['*.md', '*.build.gradle'] + paths: ['**.md', '**.build.gradle', 'build.gradle.kts'] issue_comment: types: [created] schedule: - - cron: '* */6 * * *' + - cron: '0 */6 * * *' + workflow_dispatch: + +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} jobs: - beam_PreCommit_Whitespace: - if: ${{github.event.issue.pull_request}} || ${{github.event.comment.body == 'Run Whitespace PreCommit'}} || ${{github.event.schedule}} - runs-on: [self-hosted, ubuntu-20.04] - name: beam_PreCommit_Whitespace - steps: - - name: Git checkout - uses: actions/checkout@v3 - - name: Install Java - uses: actions/setup-java@v3.8.0 - with: - distribution: 'zulu' - java-version: '8' - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - name: Setup Gradle - uses: gradle/gradle-build-action@v2 - with: - cache-read-only: false - - name: run whitespacePreCommit script - run: ./gradlew :whitespacePreCommit \ No newline at end of file + beam_PreCommit_Whitespace: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_PreCommit_Whitespace] + job_phrase: [Run Whitespace PreCommit] + timeout-minutes: 120 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Whitespace PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: 3.8 + - name: run whitespacePreCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :whitespacePreCommit diff --git a/.github/workflows/beam_Prober_CommunityMetrics.yml b/.github/workflows/beam_Prober_CommunityMetrics.yml new file mode 100644 index 0000000000000..d22b7db8309f8 --- /dev/null +++ b/.github/workflows/beam_Prober_CommunityMetrics.yml @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Community Metrics Prober + +on: + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_Prober_CommunityMetrics: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Community Metrics Prober' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_Prober_CommunityMetrics"] + job_phrase: ["Run Community Metrics Prober"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Health check probes for the Community Metrics infrastructure + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :communityMetricsProber + arguments: | + --rerun-tasks \ No newline at end of file diff --git a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml new file mode 100644 index 0000000000000..9bc268bbe9c2a --- /dev/null +++ b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Publish Beam SDK Snapshots + +on: + schedule: + - cron: '0 */4 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + docker_registry: gcr.io + +jobs: + beam_Publish_Beam_SDK_Snapshots: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.container_task }}) + strategy: + fail-fast: false + matrix: + job_name: ["beam_Publish_Beam_SDK_Snapshots"] + job_phrase: ["N/A"] + container_task: ["go:container", "java:container:java8", "java:container:java11", "java:container:java17", "python:container:py38", "python:container:py39", "python:container:py310", "python:container:py311"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.container_task }}) + - name: Authenticate on GCP + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + export_default_credentials: true + - name: GCloud Docker credential helper + run: | + gcloud auth configure-docker ${{ env.docker_registry }} + - name: Setup Java environment + if: ${{ startsWith(matrix.container_task, 'java') }} + uses: ./.github/actions/setup-environment-action + with: + java-version: 11 + - name: Setup Python environment + if: ${{ startsWith(matrix.container_task, 'python') }} + uses: ./.github/actions/setup-environment-action + with: + python-version: '3.8' + - name: run Publish Beam SDK Snapshots script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:${{ matrix.container_task }}:dockerTagPush + arguments: | + -Pjava11Home=$JAVA_HOME_11_X64 \ + -Pdocker-repository-root=gcr.io/apache-beam-testing/beam-sdk \ + -Pdocker-tag-list=${{ github.sha }},latest \ No newline at end of file diff --git a/.github/workflows/beam_Publish_Docker_Snapshots.yml b/.github/workflows/beam_Publish_Docker_Snapshots.yml new file mode 100644 index 0000000000000..cf2520f4635dd --- /dev/null +++ b/.github/workflows/beam_Publish_Docker_Snapshots.yml @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Publish Docker Snapshots + +on: + schedule: + - cron: '0 13 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + docker_registry: gcr.io + +jobs: + beam_Publish_Docker_Snapshots: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Publish Docker Snapshots' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 100 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_Publish_Docker_Snapshots"] + job_phrase: ["Publish Docker Snapshots"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Authenticate on GCP + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + export_default_credentials: true + - name: GCloud Docker credential helper + run: | + gcloud auth configure-docker ${{ env.docker_registry }} + - name: run Publish Docker Snapshots script for Spark + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:spark:3:job-server:container:dockerPush + arguments: | + -Pdocker-repository-root=gcr.io/apache-beam-testing/beam_portability \ + -Pdocker-tag-list=latest \ + - name: run Publish Docker Snapshots script for Flink + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:flink:1.15:job-server-container:dockerPush + arguments: | + -Pdocker-repository-root=gcr.io/apache-beam-testing/beam_portability \ + -Pdocker-tag-list=latest \ No newline at end of file diff --git a/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml b/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml new file mode 100644 index 0000000000000..eaf3f2c5ca1a6 --- /dev/null +++ b/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml @@ -0,0 +1,121 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Python ValidatesContainer Dataflow ARM + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: ["sdks/python/**",".github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml"] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_Python_ValidatesContainer_Dataflow_ARM: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + strategy: + fail-fast: false + matrix: + job_name: [beam_Python_ValidatesContainer_Dataflow_ARM] + job_phrase: [Run Python ValidatesContainer Dataflow ARM] + python_version: ['3.8','3.9','3.10','3.11'] + if: | + github.event_name == 'push' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + startsWith(github.event.comment.body, 'Run Python ValidatesContainer Dataflow ARM') + + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: ${{ matrix.python_version }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Authenticate on GCP + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + export_default_credentials: true + - name: GCloud Docker credential helper + run: | + gcloud auth configure-docker us.gcr.io + - name: Set PY_VER_CLEAN + id: set_py_ver_clean + run: | + PY_VER=${{ matrix.python_version }} + PY_VER_CLEAN=${PY_VER//.} + echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT + - name: Generate TAG unique variable based on timestamp + id: set_tag + run: echo "TAG=$(date +'%Y%m%d-%H%M%S%N')" >> $GITHUB_OUTPUT + - name: run Python validatesContainerARM script + id: run_script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:dataflow:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:validatesContainerARM + arguments: | + -PpythonVersion=${{ matrix.python_version }} \ + -Pcontainer-architecture-list=arm64,amd64 \ + -Pdocker-repository-root=us.gcr.io/apache-beam-testing/github-actions \ + -Pdocker-tag=${{ steps.set_tag.outputs.TAG }} \ + -Ppush-containers \ + env: + MULTIARCH_TAG: ${{ steps.set_tag.outputs.TAG }} + USER: github-actions + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + if: always() + with: + name: python-code-coverage-report + path: "**/pytest*.xml" diff --git a/.github/workflows/beam_Release_NightlySnapshot.yml b/.github/workflows/beam_Release_NightlySnapshot.yml new file mode 100644 index 0000000000000..bf22344c4199b --- /dev/null +++ b/.github/workflows/beam_Release_NightlySnapshot.yml @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Release Nightly Snapshot + +on: + schedule: + - cron: '15 12 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_Release_NightlySnapshot: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_Release_NightlySnapshot] + job_phrase: [Release Nightly Snapshot] + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + github_token: ${{ secrets.GITHUB_TOKEN }} + comment_phrase: ${{ matrix.job_phrase }} + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + - name: Auth on snapshot repository + run: | + mkdir -p ${HOME}/.m2 + echo " + + + apache.snapshots.https + ${{ secrets.NEXUS_USER }} + ${{ secrets.NEXUS_PW }} + + + " > ${HOME}/.m2/settings.xml + - name: run Publish script + run: | + ./gradlew publish --max-workers=8 -Ppublishing -PskipCheckerFramework \ + --continue -Dorg.gradle.jvmargs=-Xms2g -Dorg.gradle.jvmargs=-Xmx6g \ + -Dorg.gradle.vfs.watch=false -Pdocker-pull-licenses diff --git a/.github/workflows/beam_Release_Python_NightlySnapshot.yml b/.github/workflows/beam_Release_Python_NightlySnapshot.yml new file mode 100644 index 0000000000000..a9f4ac5d2d145 --- /dev/null +++ b/.github/workflows/beam_Release_Python_NightlySnapshot.yml @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Release Nightly Snapshot Python + +on: + schedule: + - cron: '15 12 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_Release_Python_NightlySnapshot: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + matrix: + job_name: [beam_Release_Python_NightlySnapshot] + job_phrase: [Release Nightly Snapshot Python] + if: github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' + + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + github_token: ${{ secrets.GITHUB_TOKEN }} + comment_phrase: ${{ matrix.job_phrase }} + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + python-version: 3.8 + - name: Authenticate on GCP + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + export_default_credentials: true + - name: run Cleanup script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:clean + - name: run Snapshot script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:buildSnapshot + - name: Publish Python Artifacts + run: | + bash sdks/python/scripts/run_snapshot_publish.sh + env: + WORKSPACE: ${{ github.workspace }} \ No newline at end of file diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml index 5a05c1a098cb8..12f1537dac181 100644 --- a/.github/workflows/build_release_candidate.yml +++ b/.github/workflows/build_release_candidate.yml @@ -19,6 +19,9 @@ on: APACHE_PASSWORD: description: Your Apache password. Required if you want to stage artifacts into https://dist.apache.org/repos/dist/dev/beam/ required: false + BEAM_SITE_TOKEN: + description: Github Personal Access Token with apache/beam-site repo permission if you want to create the beam-site docs PR. See https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens. + default: '' PUBLISH_JAVA_ARTIFACTS: description: Whether to publish java artifacts to https://repository.apache.org/#stagingRepositories (yes/no) required: true @@ -27,14 +30,31 @@ on: description: Whether to stage the java source into https://dist.apache.org/repos/dist/dev/beam/ required: true default: 'no' + STAGE_DOCKER_ARTIFACTS: + description: Whether to stage SDK docker images to docker hub Apache organization + required: true + default: 'no' + STAGE_PYTHON_ARTIFACTS: + description: Whether to stage the python artifacts into https://dist.apache.org/repos/dist/dev/beam/ + required: true + default: 'no' + CREATE_BEAM_SITE_PR: + description: Whether to create the documentation update PR against apache/beam-site. + required: true + default: 'no' + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} jobs: publish_java_artifacts: if: ${{github.event.inputs.PUBLISH_JAVA_ARTIFACTS == 'yes'}} - runs-on: ubuntu-latest + runs-on: [self-hosted, ubuntu-20.04, main] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" repository: apache/beam @@ -105,7 +125,7 @@ jobs: RC_ZIP="${RC_DIR}.zip" RELEASE_DIR="beam-${{ github.event.inputs.RELEASE }}" RC_TAG="v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" - SOURCE_RELEASE_ZIP="apache-beam-${RELEASE}-source-release.zip" + SOURCE_RELEASE_ZIP="apache-beam-${{ github.event.inputs.RELEASE }}-source-release.zip" # Check whether there is an existing dist dir if (svn ls "${SOURCE_RELEASE_ZIP}"); then echo "Removing existing ${SOURCE_RELEASE_ZIP}." @@ -122,7 +142,7 @@ jobs: rm -r "$RELEASE_DIR" echo "----Signing Source Release ${SOURCE_RELEASE_ZIP}-----" - gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --detach-sig "${SOURCE_RELEASE_ZIP}" + gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --batch --yes --detach-sig "${SOURCE_RELEASE_ZIP}" echo "----Creating Hash Value for ${SOURCE_RELEASE_ZIP}----" sha512sum ${SOURCE_RELEASE_ZIP} > ${SOURCE_RELEASE_ZIP}.sha512 @@ -130,3 +150,219 @@ jobs: svn add --force . svn status svn commit -m "Staging Java artifacts for Apache Beam ${{ github.event.inputs.RELEASE }} RC${{ github.event.inputs.RC }}" --non-interactive --username ${{ github.event.inputs.APACHE_ID }} --password ${{ github.event.inputs.APACHE_PASSWORD }} + + stage_python_artifacts: + if: ${{github.event.inputs.STAGE_PYTHON_ARTIFACTS == 'yes'}} + runs-on: ubuntu-latest + steps: + - name: Validate and mask apache id/password + run: | + echo "::add-mask::${{ github.event.inputs.APACHE_PASSWORD }}" + if [ "${{ github.event.inputs.APACHE_ID }}" == "" ] + then + echo "Must provide an apache id to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/" + fi + if [ "${{ github.event.inputs.APACHE_PASSWORD }}" == "" ] + then + echo "Must provide an apache password to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/" + fi + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.8 + - name: Import GPG key + id: import_gpg + uses: crazy-max/ghaction-import-gpg@111c56156bcc6918c056dbef52164cfa583dc549 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + - name: stage python artifacts + env: + RC_TAG: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" + GIT_REPO_BASE_URL: https://github.com/apache/beam + RELEASE_DIR: "beam/${{ github.event.inputs.RELEASE }}" + SCRIPT_DIR: release/src/main/scripts + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + svn co https://dist.apache.org/repos/dist/dev/beam + mkdir -p "${SVN_ARTIFACTS_DIR}" + + RELEASE_COMMIT=$(git rev-list -n 1 "tags/${RC_TAG}") + + python "${SCRIPT_DIR}/download_github_actions_artifacts.py" \ + --github-token-var GITHUB_TOKEN \ + --repo-url "${GIT_REPO_BASE_URL}" \ + --rc-tag "${RC_TAG}" \ + --release-commit "${RELEASE_COMMIT}" \ + --artifacts_dir "${RELEASE_DIR}/python" + + cd "${RELEASE_DIR}"/python + + echo "------Checking Hash Value for apache-beam-${RELEASE}.zip-----" + sha512sum -c "apache-beam-${RELEASE}.zip.sha512" + + echo "------Signing Source Release apache-beam-${RELEASE}.zip------" + gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --detach-sig "apache-beam-${RELEASE}.zip" + + for artifact in *.whl; do + echo "----------Checking Hash Value for ${artifact} wheel-----------" + sha512sum -c "${artifact}.sha512" + done + + for artifact in *.whl; do + echo "------------------Signing ${artifact} wheel-------------------" + gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --batch --yes --detach-sig "${artifact}" + done + + cd .. + svn add --force python + svn status + svn commit --no-auth-cache -m "Staging Python artifacts for Apache Beam ${RELEASE} RC${RC_NUM}" + + + stage_docker: + if: ${{github.event.inputs.STAGE_DOCKER_ARTIFACTS == 'yes'}} + # Note: if this ever changes to self-hosted, remove the "Remove default github maven configuration" step + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" + repository: apache/beam + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@v1.3.0 + - name: Install Java 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + - name: Install Python 3.8 + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - run: echo $JAVA_HOME + - run: echo "JAVA11_HOME=${JAVA_HOME}" >> "$GITHUB_OUTPUT" + id: export-java11 + - name: Install Java 8 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '8' + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Remove default github maven configuration + # This step is a workaround to avoid a decryption issue of Beam's + # net.linguica.gradle.maven.settings plugin and github's provided maven + # settings.xml file + run: rm ~/.m2/settings.xml || true + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Push docker images + run: ./gradlew :pushAllDockerImages -PisRelease -Pdocker-pull-licenses -Pprune-images -Pdocker-tag=${{ github.event.inputs.RELEASE }}rc${{ github.event.inputs.RC }} -Pjava11Home=${{steps.export-java11.outputs.JAVA11_HOME}} --no-daemon --no-parallel + + beam_site_pr: + if: ${{github.event.inputs.CREATE_BEAM_SITE_PR == 'yes'}} + # Note: if this ever changes to self-hosted, remove the "Remove default github maven configuration" step + runs-on: ubuntu-latest + env: + RC_TAG: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" + BRANCH_NAME: updates_release_${{ github.event.inputs.RELEASE }} + BEAM_ROOT_DIR: ${{ github.workspace }}/beam + SITE_ROOT_DIR: ${{ github.workspace }}/beam-site + steps: + - name: Checkout Beam Repo + uses: actions/checkout@v4 + with: + ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" + repository: apache/beam + path: beam + - name: Checkout Beam Site Repo + uses: actions/checkout@v4 + with: + repository: apache/beam-site + path: beam-site + token: ${{ github.event.inputs.BEAM_SITE_TOKEN }} + ref: release-docs + - name: Install Python 3.8 + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name: Install node + uses: actions/setup-node@v3 + with: + node-version: '16' + - name: Install Java 8 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '8' + - name: Remove default github maven configuration + # This step is a workaround to avoid a decryption issue of Beam's + # net.linguica.gradle.maven.settings plugin and github's provided maven + # settings.xml file + run: rm ~/.m2/settings.xml || true + - name: Create documentation commit for website + run: | + echo "OK!" + - name: Build Python Docs + working-directory: beam/sdks/python + run: | + pip install --upgrade pip setuptools wheel + pip install -U pip + pip install tox + # TODO(https://github.com/apache/beam/issues/20209): Don't hardcode py version in this file. + tox -e py38-docs + rm -rf target/docs/_build/.doctrees + - name: Build Typescript Docs + working-directory: beam/sdks/typescript + run: | + npm ci && npm run docs + - name: Build Java Docs + working-directory: beam + run: | + ./gradlew :sdks:java:javadoc:aggregateJavadoc -PisRelease --no-daemon --no-parallel + - name: Consolidate Release Docs to beam-site branch with symlinks + working-directory: beam-site + run: | + git checkout -b $BRANCH_NAME release-docs + + echo "..........Copying generated javadoc into beam-site.........." + cp -r ${BEAM_ROOT_DIR}/sdks/java/javadoc/build/docs/javadoc/ javadoc/${{ github.event.inputs.RELEASE }} + # Update current symlink to point to the latest release + unlink javadoc/current + ln -s ${{ github.event.inputs.RELEASE }} javadoc/current + + echo "............Copying generated pydoc into beam-site.........." + cp -r ${BEAM_ROOT_DIR}/sdks/python/target/docs/_build pydoc/${{ github.event.inputs.RELEASE }} + # Update current symlink to point to the latest release + unlink pydoc/current + ln -s ${{ github.event.inputs.RELEASE }} pydoc/current + + echo "............Copying generated typedoc into beam-site.........." + mkdir -p typedoc + cp -r ${BEAM_ROOT_DIR}/sdks/typescript/docs typedoc/${{ github.event.inputs.RELEASE }} + # Update current symlink to point to the latest release + unlink typedoc/current | true + ln -s ${{ github.event.inputs.RELEASE }} typedoc/current + - name: Create commit on beam-site branch + working-directory: beam + run: | + # Get the commit from the beam repo, not the beam-site repo. + RELEASE_COMMIT=$(git rev-list -n 1 "tags/${RC_TAG}") + cd ${SITE_ROOT_DIR} + git config user.name $GITHUB_ACTOR + git config user.email actions@"$RUNNER_NAME".local + git add -A + git commit -m "Update beam-site for release ${{ github.event.inputs.RELEASE }}." -m "Content generated from commit ${RELEASE_COMMIT}." + git push -f --set-upstream origin $BRANCH_NAME + - name: Create beam-site PR + working-directory: beam-site + env: + GH_TOKEN: ${{ github.event.inputs.BEAM_SITE_TOKEN }} + PR_TITLE: "Publish docs for ${{ github.event.inputs.RELEASE }} release" + PR_BODY: "Content generated from https://github.com/apache/beam/tree/${{ env.RC_TAG }}." + run: | + gh pr create -t "$PR_TITLE" -b "$PR_BODY" --base release-docs diff --git a/.github/workflows/build_runner_image.yml b/.github/workflows/build_runner_image.yml new file mode 100644 index 0000000000000..6071d936958ad --- /dev/null +++ b/.github/workflows/build_runner_image.yml @@ -0,0 +1,71 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: Build and Version Runner Docker Image + +on: + push: + branches: ['master'] + paths: ['.github/gh-actions-self-hosted-runners/arc/images/**','.github/workflows/build_runner_image.yml'] + pull_request: + branches: ['master'] + paths: ['.github/gh-actions-self-hosted-runners/arc/images/**'] +env: + docker_registry: us-central1-docker.pkg.dev + docker_repo: apache-beam-testing/beam-github-actions/beam-arc-runner +jobs: + build-and-version-runner: + env: + working-directory: .github/gh-actions-self-hosted-runners/arc/images/ + runs-on: [self-hosted, ubuntu-20.04] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Authenticate on GCP + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + export_default_credentials: true + - name: GCloud Docker credential helper + run: | + gcloud auth configure-docker ${{env.docker_registry}} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Build and Load to docker + uses: docker/build-push-action@v4 + with: + context: ${{ env.working-directory }} + load: true + tags: | + ${{env.docker_registry}}/${{env.docker_repo}}:latest + ${{env.docker_registry}}/${{env.docker_repo}}:${{ github.sha }} + - name: Push Docker image + if: github.event_name == 'push' + id: docker_build + uses: docker/build-push-action@v4 + with: + context: ${{ env.working-directory }} + push: true + tags: | + ${{env.docker_registry}}/${{env.docker_repo}}:latest + ${{env.docker_registry}}/${{env.docker_repo}}:${{ github.sha }} diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index ead302975d686..f4ccf368bacb8 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -33,7 +33,7 @@ on: # This allows a subsequently queued workflow run to interrupt previous runs concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login}}' cancel-in-progress: true env: @@ -48,13 +48,13 @@ jobs: runs-on: ubuntu-latest env: EVENT_NAME: ${{ github.event_name }} - PY_VERSIONS_FULL: "cp37-* cp38-* cp39-* cp310-* cp311-*" + PY_VERSIONS_FULL: "cp38-* cp39-* cp310-* cp311-*" outputs: gcp-variables-set: ${{ steps.check_gcp_variables.outputs.gcp-variables-set }} py-versions-full: ${{ steps.set-py-versions.outputs.py-versions-full }} py-versions-test: ${{ steps.set-py-versions.outputs.py-versions-test }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: "Check are GCP variables set" run: "./scripts/ci/ci_check_are_gcp_variables_set.sh" id: check_gcp_variables @@ -87,16 +87,11 @@ jobs: rc_num: ${{ steps.get_rc_version.outputs.RC_NUM }} steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install python uses: actions/setup-python@v4 with: - python-version: 3.7 - - name: Get build dependencies - working-directory: ./sdks/python - run: python -m pip install -r build-requirements.txt - - name: Install wheels - run: python -m pip install wheel + python-version: 3.8 - name: Get tag id: get_tag run: | @@ -117,15 +112,15 @@ jobs: echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_OUTPUT - name: Build source working-directory: ./sdks/python - run: python setup.py sdist --formats=zip + run: pip install -U build && python -m build --sdist - name: Add checksums working-directory: ./sdks/python/dist run: | - file=$(ls | grep .zip | head -n 1) + file=$(ls | grep .tar.gz | head -n 1) sha512sum $file > ${file}.sha512 - name: Unzip source working-directory: ./sdks/python - run: unzip dist/$(ls dist | grep .zip | head -n 1) + run: tar -xzvf dist/$(ls dist | grep .tar.gz | head -n 1) - name: Rename source directory working-directory: ./sdks/python run: mv $(ls | grep apache-beam) apache-beam-source @@ -155,17 +150,17 @@ jobs: - name: Build RC source if: steps.is_rc.outputs.is_rc == 1 working-directory: ./sdks/python - run: python setup.py sdist --formats=zip + run: pip install -U build && pythom -m build --sdist - name: Add RC checksums if: steps.is_rc.outputs.is_rc == 1 working-directory: ./sdks/python/dist run: | - file=$(ls | grep .zip | head -n 1) + file=$(ls | grep .tar.gz | head -n 1) sha512sum $file > ${file}.sha512 - name: Unzip RC source if: steps.is_rc.outputs.is_rc == 1 working-directory: ./sdks/python - run: unzip dist/$(ls dist | grep .zip | head -n 1) + run: tar -xzvf dist/$(ls dist | grep .tar.gz | head -n 1) - name: Rename RC source directory if: steps.is_rc.outputs.is_rc == 1 working-directory: ./sdks/python @@ -255,7 +250,7 @@ jobs: - name: Install Python uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.8 - uses: docker/setup-qemu-action@v1 if: ${{matrix.arch == 'aarch64'}} name: Set up QEMU @@ -269,7 +264,7 @@ jobs: # TODO: https://github.com/apache/beam/issues/23048 CIBW_SKIP: "*-musllinux_*" CIBW_ENVIRONMENT: "SETUPTOOLS_USE_DISTUTILS=stdlib" - CIBW_BEFORE_BUILD: pip install cython numpy && pip install --upgrade setuptools + CIBW_BEFORE_BUILD: pip install cython==0.29.36 numpy --config-settings=setup-args="-Dallow-noblas=true" && pip install --upgrade setuptools run: cibuildwheel --print-build-identifiers && cibuildwheel --output-dir wheelhouse shell: bash - name: install sha512sum on MacOS @@ -295,7 +290,7 @@ jobs: # TODO: https://github.com/apache/beam/issues/23048 CIBW_SKIP: "*-musllinux_*" CIBW_ENVIRONMENT: "SETUPTOOLS_USE_DISTUTILS=stdlib" - CIBW_BEFORE_BUILD: pip install cython numpy && pip install --upgrade setuptools + CIBW_BEFORE_BUILD: pip install cython==0.29.36 numpy --config-settings=setup-args="-Dallow-noblas=true" && pip install --upgrade setuptools run: cibuildwheel --print-build-identifiers && cibuildwheel --output-dir wheelhouse shell: bash - name: Add RC checksums @@ -385,7 +380,7 @@ jobs: if: github.repository_owner == 'apache' && github.event_name == 'schedule' steps: - name: Checkout code on master branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false submodules: recursive diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml index df9172473f987..f826b22e043b4 100644 --- a/.github/workflows/cancel.yml +++ b/.github/workflows/cancel.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false submodules: recursive diff --git a/.github/workflows/choose_rc_commit.yml b/.github/workflows/choose_rc_commit.yml index d4de9d454a5c1..0e51e5284a76a 100644 --- a/.github/workflows/choose_rc_commit.yml +++ b/.github/workflows/choose_rc_commit.yml @@ -55,7 +55,7 @@ jobs: DEBUG: "" steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: release-${{ github.event.inputs.RELEASE }} - name: Set git config diff --git a/.github/workflows/code_completion_plugin_tests.yml b/.github/workflows/code_completion_plugin_tests.yml new file mode 100644 index 0000000000000..38ffd2fbd3f4f --- /dev/null +++ b/.github/workflows/code_completion_plugin_tests.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +# The workflow is triggered on the following events: +# - pull requests that contain changes in plugins/beam-code-completion-plugin/ directory +# - pushes to any branch except master. the push must contain changes in plugins/beam-code-completion-plugin/ directory + +# To learn more about GitHub Actions in Apache Beam check the CI.md + +name: Code Completion Plugin Tests +on: + push: + branches-ignore: + - 'master' + paths: + - 'plugins/beam-code-completion-plugin/**' + - '.github/workflows/code_completion_plugin_tests.yml' + pull_request: + paths: + - 'plugins/beam-code-completion-plugin/**' + - '.github/workflows/code_completion_plugin_tests.yml' +env: + INTELLIJ_IDEA_SOURCES: /home/runner/work/beam/beam/intellij + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} +jobs: + # Run Gradle Wrapper Validation Action to verify the wrapper's checksum + # Run verifyPlugin, IntelliJ Plugin Verifier, and test Gradle tasks + # Build plugin and provide the artifact for the next workflow jobs + test: + name: Build and run model-level tests + runs-on: ubuntu-latest + steps: + # Free GitHub Actions Environment Disk Space + - name: Maximize Build Space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + + # Check out beam repository + - name: Fetch beam Sources + uses: actions/checkout@v4 + with: + path: main + + # Check out intellij community repository for tests + - name: Fetch intellij-community Sources + uses: actions/checkout@v4 + with: + repository: JetBrains/intellij-community + path: intellij + + # Validate wrapper + - name: Gradle Wrapper Validation + uses: gradle/wrapper-validation-action@v1.0.6 + + # Setup Java environment for the next steps + - name: Setup Java + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + + # Set environment variables + - name: Export Properties + id: properties + shell: bash + run: | + pwd + cd main/plugins/beam-code-completion-plugin/ + pwd + PROPERTIES="$(./gradlew properties --console=plain -q)" + VERSION="$(echo "$PROPERTIES" | grep "^version:" | cut -f2- -d ' ')" + NAME="$(echo "$PROPERTIES" | grep "^name:" | cut -f2- -d ' ')" + + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "name=$NAME" >> $GITHUB_OUTPUT + echo "pluginVerifierHomeDir=~/.pluginVerifier" >> $GITHUB_OUTPUT + + echo "changelog<> $GITHUB_OUTPUT + echo "$CHANGELOG" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + # Run tests + - name: Run Tests + run: | + pwd + cd main/plugins/beam-code-completion-plugin/ + pwd + ./gradlew test --info + + # Collect Tests Result of failed tests + - name: Collect Tests Result + if: ${{ failure() }} + uses: actions/upload-artifact@v3 + with: + name: tests-result + path: ${{ github.workspace }}/build/reports/tests diff --git a/.github/workflows/cut_release_branch.yml b/.github/workflows/cut_release_branch.yml index f026c41ca9c3a..4201d6018c608 100644 --- a/.github/workflows/cut_release_branch.yml +++ b/.github/workflows/cut_release_branch.yml @@ -68,7 +68,7 @@ jobs: exit 1 fi - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set git config run: | git config user.name $GITHUB_ACTOR @@ -100,6 +100,7 @@ jobs: MASTER_BRANCH: master NEXT_RELEASE: ${{ github.event.inputs.NEXT_VERSION }} SCRIPT_DIR: ./release/src/main/scripts + RELEASE: ${{ github.event.inputs.RELEASE_VERSION }} steps: - name: Mask Jenkins token run: | @@ -115,7 +116,7 @@ jobs: exit 1 fi - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set git config run: | git config user.name $GITHUB_ACTOR @@ -146,10 +147,13 @@ jobs: fi done - cat /tmp/result | sort | uniq | grep -i -E 'precommit|postcommit|validates|vr|example|test|gradle build' | grep -v -i -E 'load|perf|website' >> release/src/main/scripts/jenkins_jobs.txt + cat /tmp/result | sort | uniq | grep -i -E 'precommit|postcommit|validates|vr|example|test' | grep -v -i -E 'load|perf|website' >> release/src/main/scripts/jenkins_jobs.txt env: JENKINS_USERNAME: ${{ github.event.inputs.JENKINS_USERNAME }} JENKINS_TOKEN: ${{ github.event.inputs.JENKINS_TOKEN }} + - name: Update .asf.yaml to protect new release branch from force push + run: | + sed -i -e "s/master: {}/master: {}\n release-${RELEASE}: {}/g" .asf.yaml - name: Update master branch run: | bash "${SCRIPT_DIR}/set_version.sh" "${NEXT_VERSION_IN_BASE_BRANCH}" @@ -159,6 +163,7 @@ jobs: - name: Commit and Push to master branch files with Next Version run: | git add * + git add .asf.yaml git commit -m "Moving to ${NEXT_VERSION_IN_BASE_BRANCH}-SNAPSHOT on master branch." git push origin ${MASTER_BRANCH} @@ -170,6 +175,7 @@ jobs: REMOTE_NAME: remote_repo REMOTE_URL: ${{ github.server_url }}/${{ github.repository }} BRANCH_NAME: snapshot_build-${{ github.event.inputs.RELEASE_VERSION }} + RELEASE_BRANCH: release-${{ github.event.inputs.RELEASE_VERSION }} steps: - name: Install Hub run: | @@ -178,7 +184,7 @@ jobs: tar zvxvf hub-linux-amd64-2.14.2.tgz sudo ./hub-linux-amd64-2.14.2/install echo "eval "$(hub alias -s)"" >> ~/.bashrc - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set git config run: | git config user.name $GITHUB_ACTOR @@ -187,7 +193,7 @@ jobs: run: | git remote add ${REMOTE_NAME} ${REMOTE_URL} git checkout -b ${BRANCH_NAME} - touch empty_file.txt + touch empty_file.json git add -A git commit -m "Add empty file in order to create PR" git push -f ${REMOTE_NAME} @@ -195,7 +201,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - hub pull-request -F- <<<"[DO NOT MERGE]Start snapshot build for release process + hub pull-request -b apache:${RELEASE_BRANCH} -F- <<<"[DO NOT MERGE]Start snapshot build for release process Run Gradle Publish" diff --git a/.github/workflows/dask_runner_tests.yml b/.github/workflows/dask_runner_tests.yml index 33d0575e2c8c6..35c320086992d 100644 --- a/.github/workflows/dask_runner_tests.yml +++ b/.github/workflows/dask_runner_tests.yml @@ -29,7 +29,7 @@ on: # This allows a subsequently queued workflow run to interrupt previous runs concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login}}' cancel-in-progress: true jobs: @@ -39,17 +39,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install python uses: actions/setup-python@v4 with: - python-version: 3.7 - - name: Get build dependencies - working-directory: ./sdks/python - run: pip install pip setuptools --upgrade && pip install -r build-requirements.txt + python-version: 3.8 - name: Build source working-directory: ./sdks/python - run: python setup.py sdist + run: pip install -U build && python -m build --sdist - name: Rename source file working-directory: ./sdks/python/dist run: mv $(ls | grep "apache-beam.*tar\.gz") apache-beam-source.tar.gz @@ -67,21 +64,17 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] params: [ - {"py_ver": "3.7", "tox_env": "py37"}, {"py_ver": "3.8", "tox_env": "py38"}, {"py_ver": "3.9", "tox_env": "py39"}, {"py_ver": "3.10", "tox_env": "py310" }, ] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install python uses: actions/setup-python@v4 with: python-version: ${{ matrix.params.py_ver }} - - name: Get build dependencies - working-directory: ./sdks/python - run: pip install -r build-requirements.txt - name: Install tox run: pip install tox - name: Install SDK with dask diff --git a/.github/workflows/git_tag_released_version.yml b/.github/workflows/git_tag_released_version.yml index 871149bd26a18..0c67826038563 100644 --- a/.github/workflows/git_tag_released_version.yml +++ b/.github/workflows/git_tag_released_version.yml @@ -37,7 +37,7 @@ jobs: VERSION_PATH: ${{ github.event.inputs.VERSION_TAG }} steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set git config run: | git config user.name $GITHUB_ACTOR diff --git a/.github/workflows/go_tests.yml b/.github/workflows/go_tests.yml index 6ea03675a1e91..02947eff5ca03 100644 --- a/.github/workflows/go_tests.yml +++ b/.github/workflows/go_tests.yml @@ -29,9 +29,10 @@ on: branches: ['master', 'release-*'] tags: ['v*'] paths: ['sdks/go/pkg/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/container/*', 'sdks/java/container/*', 'sdks/python/container/*', 'sdks/typescript/container/*'] + workflow_dispatch: # This allows a subsequently queued workflow run to interrupt previous runs concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login}}' cancel-in-progress: true jobs: build: @@ -39,12 +40,14 @@ jobs: name: Go Build steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 2 - uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' + cache-dependency-path: | + sdks/go.sum - name: Delete old coverage run: "cd sdks && rm -rf .coverage.txt || :" - name: Run coverage diff --git a/.github/workflows/issue-tagger.yml b/.github/workflows/issue-tagger.yml index 39f92d87f7884..dbfe2e996d5ef 100644 --- a/.github/workflows/issue-tagger.yml +++ b/.github/workflows/issue-tagger.yml @@ -24,7 +24,7 @@ jobs: permissions: issues: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: damccorm/tag-ur-it@6fa72bbf1a2ea157b533d7e7abeafdb5855dbea5 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/java_tests.yml b/.github/workflows/java_tests.yml index 3a94a6bf9ddfc..1132ba1c196b6 100644 --- a/.github/workflows/java_tests.yml +++ b/.github/workflows/java_tests.yml @@ -37,8 +37,12 @@ on: 'examples/kotlin/**', 'release/**', 'buildSrc/**'] # This allows a subsequently queued workflow run to interrupt previous runs concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login}}' cancel-in-progress: true +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} jobs: check_gcp_variables: timeout-minutes: 5 @@ -48,7 +52,7 @@ jobs: gcp-variables-set: ${{ steps.check_gcp_variables.outputs.gcp-variables-set }} steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: "Check are GCP variables set" run: "./scripts/ci/ci_check_are_gcp_variables_set.sh" id: check_gcp_variables @@ -69,15 +73,15 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false submodules: recursive - - name: Setup self-hosted - uses: ./.github/actions/setup-self-hosted-action + - name: Setup environment + uses: ./.github/actions/setup-environment-action with: - requires-py-38: false - requires-py-39: false + java-version: 8 + go-version: 1.21 - name: Remove default github maven configuration # This step is a workaround to avoid a decryption issue of Beam's # net.linguica.gradle.maven.settings plugin and github's provided maven @@ -128,16 +132,15 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false submodules: recursive - - name: Setup self-hosted - uses: ./.github/actions/setup-self-hosted-action + - name: Setup environment + uses: ./.github/actions/setup-environment-action with: - requires-py-38: false - requires-py-39: false - + java-version: 8 + go-version: 1.21 - name: Remove default github maven configuration # This step is a workaround to avoid a decryption issue of Beam's # net.linguica.gradle.maven.settings plugin and github's provided maven @@ -172,15 +175,15 @@ jobs: ) steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false submodules: recursive - - name: Setup self-hosted - uses: ./.github/actions/setup-self-hosted-action - with: - requires-py-38: false - requires-py-39: false + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + java-version: 8 + go-version: 1.21 - name: Authenticate on GCP uses: google-github-actions/setup-gcloud@v0 with: diff --git a/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_100B_records.txt b/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_100B_records.txt new file mode 100644 index 0000000000000..244483d336f8c --- /dev/null +++ b/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_100B_records.txt @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--appName=load_tests_Java_SparkStructuredStreaming_batch_GBK_2 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_2 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=1 +--iterations=1 +--streaming=false +--runner=SparkStructuredStreamingRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_100kB_records.txt b/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_100kB_records.txt new file mode 100644 index 0000000000000..e0a51802e708c --- /dev/null +++ b/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_100kB_records.txt @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--appName=load_tests_Java_SparkStructuredStreaming_batch_GBK_3 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_3 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000,"keySizeBytes":10000,"valueSizeBytes":90000} +--fanout=1 +--iterations=1 +--streaming=false +--runner=SparkStructuredStreamingRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_10B_records.txt b/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_10B_records.txt new file mode 100644 index 0000000000000..1003f5ad4cf1b --- /dev/null +++ b/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_2GB_of_10B_records.txt @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--appName=load_tests_Java_SparkStructuredStreaming_batch_GBK_1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_1 +--publishToInfluxDB=true +--sourceOptions={"numRecords":200000000,"keySizeBytes":1,"valueSizeBytes":9} +--fanout=1 +--iterations=1 +--streaming=false +--runner=SparkStructuredStreamingRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..424d44570877d --- /dev/null +++ b/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--appName=load_tests_Java_SparkStructuredStreaming_batch_GBK_4 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_4 +--publishToInfluxDB=true +--sourceOptions={"numRecords":5000000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=4 +--iterations=1 +--streaming=false +--runner=SparkStructuredStreamingRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..f07fc70a9e0f6 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--appName=load_tests_Java_SparkStructuredStreaming_batch_GBK_5 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_5 +--publishToInfluxDB=true +--sourceOptions={"numRecords":2500000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=8 +--iterations=1 +--streaming=false +--runner=SparkStructuredStreamingRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_10kB_values.txt new file mode 100644 index 0000000000000..ecbe7dd534a28 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_10kB_values.txt @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--appName=load_tests_Java_SparkStructuredStreaming_batch_GBK_6 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_6 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":200,"hotKeyFraction":1} +--fanout=1 +--iterations=4 +--streaming=false +--runner=SparkStructuredStreamingRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_2MB_values.txt new file mode 100644 index 0000000000000..ace93b3a5b07c --- /dev/null +++ b/.github/workflows/load-tests-job-configs/Java_GBK_SparkStructuredStreaming_Batch_reiterate_4_times_2MB_values.txt @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--appName=load_tests_Java_SparkStructuredStreaming_batch_GBK_7 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_7 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":10,"hotKeyFraction":1} +--fanout=1 +--iterations=4 +--streaming=false +--runner=SparkStructuredStreamingRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/beam_CloudML_Benchmarks_Dataflow_arguments.txt b/.github/workflows/load-tests-job-configs/beam_CloudML_Benchmarks_Dataflow_arguments.txt new file mode 100644 index 0000000000000..b1b45c4cc9e44 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/beam_CloudML_Benchmarks_Dataflow_arguments.txt @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--metrics_dataset=beam_cloudml +--publish_to_big_query=true +--region=us-central1 +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--runner=DataflowRunner +--requirements_file=apache_beam/testing/benchmarks/cloudml/requirements.txt \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152.txt b/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152.txt new file mode 100644 index 0000000000000..c65317b495731 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152.txt @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--machine_type=n1-standard-2 +--num_workers=75 +--disk_size_gb=50 +--autoscaling_algorithm=NONE +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--requirements_file=apache_beam/ml/inference/torch_tests_requirements.txt +--publish_to_big_query=true +--metrics_dataset=beam_run_inference +--metrics_table=torch_inference_imagenet_results_resnet152 +--input_options={} +--influx_measurement=torch_inference_imagenet_resnet152 +--pretrained_model_name=resnet152 +--device=CPU +--input_file=gs://apache-beam-ml/testing/inputs/openimage_50k_benchmark.txt +--model_state_dict_path=gs://apache-beam-ml/models/torchvision.models.resnet152.pth +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152_Tesla_T4_GPU.txt b/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152_Tesla_T4_GPU.txt new file mode 100644 index 0000000000000..e79369befad72 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152_Tesla_T4_GPU.txt @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--machine_type=n1-standard-2 +--num_workers=30 +--disk_size_gb=50 +--autoscaling_algorithm=NONE +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--requirements_file=apache_beam/ml/inference/torch_tests_requirements.txt +--publish_to_big_query=true +--metrics_dataset=beam_run_inference +--metrics_table=torch_inference_imagenet_results_resnet152_tesla_t4 +--input_options={} +--influx_measurement=torch_inference_imagenet_resnet152_tesla_t4 +--pretrained_model_name=resnet152 +--device=GPU +--experiments=worker_accelerator=type:nvidia-tesla-t4;count:1;install-nvidia-driver +--sdk_container_image=us.gcr.io/apache-beam-testing/python-postcommit-it/tensor_rt:latest +--input_file=gs://apache-beam-ml/testing/inputs/openimage_50k_benchmark.txt +--model_state_dict_path=gs://apache-beam-ml/models/torchvision.models.resnet152.pth +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Base_Uncased.txt b/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Base_Uncased.txt new file mode 100644 index 0000000000000..66aca5fdbcd7a --- /dev/null +++ b/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Base_Uncased.txt @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--machine_type=n1-standard-2 +--num_workers=250 +--disk_size_gb=50 +--autoscaling_algorithm=NONE +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--requirements_file=apache_beam/ml/inference/torch_tests_requirements.txt +--publish_to_big_query=true +--metrics_dataset=beam_run_inference +--metrics_table=torch_language_modeling_bert_base_uncased +--input_options={} +--influx_measurement=torch_language_modeling_bert_base_uncased +--device=CPU +--input_file=gs://apache-beam-ml/testing/inputs/sentences_50k.txt +--bert_tokenizer=bert-base-uncased +--model_state_dict_path=gs://apache-beam-ml/models/huggingface.BertForMaskedLM.bert-base-uncased.pth +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Large_Uncased.txt b/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Large_Uncased.txt new file mode 100644 index 0000000000000..d6406271685bc --- /dev/null +++ b/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Large_Uncased.txt @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--machine_type=n1-standard-2 +--num_workers=250 +--disk_size_gb=50 +--autoscaling_algorithm=NONE +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--requirements_file=apache_beam/ml/inference/torch_tests_requirements.txt +--publish_to_big_query=true +--metrics_dataset=beam_run_inference +--metrics_table=torch_language_modeling_bert_large_uncased +--input_options={} +--influx_measurement=torch_language_modeling_bert_large_uncased +--device=CPU +--input_file=gs://apache-beam-ml/testing/inputs/sentences_50k.txt +--bert_tokenizer=bert-large-uncased +--model_state_dict_path=gs://apache-beam-ml/models/huggingface.BertForMaskedLM.bert-large-uncased.pth +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Vision_Classification_Resnet_101.txt b/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Vision_Classification_Resnet_101.txt new file mode 100644 index 0000000000000..5a0d250439097 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Vision_Classification_Resnet_101.txt @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--machine_type=n1-standard-2 +--num_workers=75 +--disk_size_gb=50 +--autoscaling_algorithm=NONE +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--requirements_file=apache_beam/ml/inference/torch_tests_requirements.txt +--publish_to_big_query=true +--metrics_dataset=beam_run_inference +--metrics_table=torch_inference_imagenet_results_resnet101 +--input_options={} +--influx_measurement=torch_inference_imagenet_resnet101 +--pretrained_model_name=resnet101 +--device=CPU +--input_file=gs://apache-beam-ml/testing/inputs/openimage_50k_benchmark.txt +--model_state_dict_path=gs://apache-beam-ml/models/torchvision.models.resnet101.pth +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_MultipleKey.txt b/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_MultipleKey.txt new file mode 100644 index 0000000000000..b60f617b21a65 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_MultipleKey.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_measurement=go_batch_cogbk_2 +--influx_namespace=dataflow +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":5,\"hot_key_fraction\":1}'' +--co_input_options=''{\"num_records\":2000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}'' +--iterations=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_Reiteration_10KB.txt b/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_Reiteration_10KB.txt new file mode 100644 index 0000000000000..e413ea295dd34 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_Reiteration_10KB.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_measurement=go_batch_cogbk_3 +--influx_namespace=dataflow +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":200000,\"hot_key_fraction\":1}'' +--co_input_options=''{\"num_records\":2000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}'' +--iterations=4 +--num_workers=5 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_Reiteration_2MB.txt b/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_Reiteration_2MB.txt new file mode 100644 index 0000000000000..d75f2dc5a5c22 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_Reiteration_2MB.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_measurement=go_batch_cogbk_4 +-influx_namespace=dataflow +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}'' +--co_input_options=''{\"num_records\":2000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}'' +--iterations=4 +--num_workers=5 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_SingleKey.txt b/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_SingleKey.txt new file mode 100644 index 0000000000000..73a4a5f0cb7cd --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_CoGBK_Go_Batch_SingleKey.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_measurement=go_batch_cogbk_1 +--influx_namespace=dataflow +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1,\"hot_key_fraction\":1}'' +--co_input_options=''{\"num_records\":2000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}'' +--iterations=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_MultipleKey.txt b/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_MultipleKey.txt new file mode 100644 index 0000000000000..5fd9518bc8d09 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_MultipleKey.txt @@ -0,0 +1,34 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### +--project=apache-beam-testing +--region=us-central1 +--appName=load_tests_Java_Dataflow_streaming_CoGBK_2 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_cogbk_2 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":5} +--coSourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--coInputWindowDurationSec=1200 +--influxDatabase=beam_test_metrics +--influxHost=http://10.128.0.96:8086 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_Reiteration_10KB.txt b/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_Reiteration_10KB.txt new file mode 100644 index 0000000000000..2840fe75d5af2 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_Reiteration_10KB.txt @@ -0,0 +1,34 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### +--project=apache-beam-testing +--region=us-central1 +--appName=load_tests_Java_Dataflow_streaming_CoGBK_3 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_cogbk_3 +--publishToInfluxDB=true +--sourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":200000} +--coSourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--iterations=4 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--coInputWindowDurationSec=1200 +--influxDatabase=beam_test_metrics +--influxHost=http://10.128.0.96:8086 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_Reiteration_2MB.txt b/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_Reiteration_2MB.txt new file mode 100644 index 0000000000000..bcc8a36cf31fb --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_Reiteration_2MB.txt @@ -0,0 +1,34 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### +--project=apache-beam-testing +--region=us-central1 +--appName=load_tests_Java_Dataflow_streaming_CoGBK_4 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_cogbk_4 +--publishToInfluxDB=true +--sourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--coSourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--iterations=4 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--coInputWindowDurationSec=1200 +--influxDatabase=beam_test_metrics +--influxHost=http://10.128.0.96:8086 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_SingleKey.txt b/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_SingleKey.txt new file mode 100644 index 0000000000000..afae1a1bd6bf8 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_CoGBK_Java_Streaming_2GB_SingleKey.txt @@ -0,0 +1,34 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### +--project=apache-beam-testing +--region=us-central1 +--appName=load_tests_Java_Dataflow_streaming_CoGBK_1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_cogbk_1 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1} +--coSourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--coInputWindowDurationSec=1200 +--influxDatabase=beam_test_metrics +--influxHost=http://10.128.0.96:8086 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_Combine_Go_Batch_10b.txt b/.github/workflows/load-tests-job-configs/config_Combine_Go_Batch_10b.txt new file mode 100644 index 0000000000000..b9ad28105903b --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_Combine_Go_Batch_10b.txt @@ -0,0 +1,34 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### +--job_name=load-tests-go-dataflow-batch-combine-1- +--project=apache-beam-testing +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_namespace=dataflow +--influx_measurement=go_batch_combine_1 +--input_options=''{\"num_records\":200000000,\"key_size\":1,\"value_size\":9}'' +--fanout=1 +--top_count=20 +--num_workers=5 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--influx_db_name=beam_test_metrics +--influx_hostname=http://10.128.0.96:8086 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_Combine_Go_Batch_Fanout_4.txt b/.github/workflows/load-tests-job-configs/config_Combine_Go_Batch_Fanout_4.txt new file mode 100644 index 0000000000000..5f3a185832703 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_Combine_Go_Batch_Fanout_4.txt @@ -0,0 +1,34 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### +--job_name=load-tests-go-dataflow-batch-combine-2- +--project=apache-beam-testing +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_namespace=dataflow +--influx_measurement=go_batch_combine_2 +--input_options=''{\"num_records\":5000000,\"key_size\":10,\"value_size\":90}'' +--fanout=4 +--top_count=20 +--num_workers=16 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--influx_db_name=beam_test_metrics +--influx_hostname=http://10.128.0.96:8086 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_Combine_Go_Batch_Fanout_8.txt b/.github/workflows/load-tests-job-configs/config_Combine_Go_Batch_Fanout_8.txt new file mode 100644 index 0000000000000..eba65b666a257 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_Combine_Go_Batch_Fanout_8.txt @@ -0,0 +1,34 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### +--job_name=load-tests-go-dataflow-batch-combine-3- +--project=apache-beam-testing +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_namespace=dataflow +--influx_measurement=go_batch_combine_3 +--input_options=''{\"num_records\":2500000,\"key_size\":10,\"value_size\":90}'' +--fanout=8 +--top_count=20 +--num_workers=16 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--influx_db_name=beam_test_metrics +--influx_hostname=http://10.128.0.96:8086 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_10b.txt b/.github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_10b.txt new file mode 100644 index 0000000000000..be6edd016dee1 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_10b.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_batch_Combine_1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_combine_1 +--publishToInfluxDB=true +--sourceOptions={"numRecords":200000000,"keySizeBytes":1,"valueSizeBytes":9} +--fanout=1 +--iterations=1 +--topCount=20 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--perKeyCombiner=TOP_LARGEST +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_Fanout_4.txt b/.github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_Fanout_4.txt new file mode 100644 index 0000000000000..4f6216c1d8d24 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_Fanout_4.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_batch_Combine_2 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_combine_2 +--publishToInfluxDB=true +--sourceOptions={"numRecords":5000000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=4 +--iterations=1 +--topCount=20 +--numWorkers=16 +--autoscalingAlgorithm=NONE +--perKeyCombiner=TOP_LARGEST +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_Fanout_8.txt b/.github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_Fanout_8.txt new file mode 100644 index 0000000000000..68bed0bae78a4 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_Combine_Java_Dataflow_Batch_Fanout_8.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_batch_Combine_3 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_combine_3 +--publishToInfluxDB=true +--sourceOptions={"numRecords":2500000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=8 +--iterations=1 +--topCount=20 +--numWorkers=16 +--autoscalingAlgorithm=NONE +--perKeyCombiner=TOP_LARGEST +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_Combine_Python_Batch_2GB_10b.txt b/.github/workflows/load-tests-job-configs/config_Combine_Python_Batch_2GB_10b.txt new file mode 100644 index 0000000000000..a6dabb5e50868 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_Combine_Python_Batch_2GB_10b.txt @@ -0,0 +1,32 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### +--job_name=load-tests-python-dataflow-batch-combine-1- +--project=apache-beam-testing +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/smoketests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_combine_1 +--influx_measurement=python_batch_combine_1 +--input_options=''{\\"num_records\\":200000000,\\"key_size\\":1,\\"value_size\\":9,\\"algorithm\\":\\"lcg\\"}'' +--num_workers=5 +--autoscaling_algorithm=NONE +--top_count=20 +--influxDatabase=beam_test_metrics +--influxHost=http://10.128.0.96:8086 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_Combine_Python_Batch_2GB_Fanout_4.txt b/.github/workflows/load-tests-job-configs/config_Combine_Python_Batch_2GB_Fanout_4.txt new file mode 100644 index 0000000000000..7639456296b6d --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_Combine_Python_Batch_2GB_Fanout_4.txt @@ -0,0 +1,33 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### +--job_name=load-tests-python-dataflow-batch-combine-2- +--project=apache-beam-testing +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/smoketests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_combine_2 +--influx_measurement=python_batch_combine_2 +--input_options=''{\\"num_records\\":5000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--num_workers=16 +--autoscaling_algorithm=NONE +--fanout=4 +--top_count=20 +--influxDatabase=beam_test_metrics +--influxHost=http://10.128.0.96:8086 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_Combine_Python_Batch_2GB_Fanout_8.txt b/.github/workflows/load-tests-job-configs/config_Combine_Python_Batch_2GB_Fanout_8.txt new file mode 100644 index 0000000000000..e5d46791a83c1 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_Combine_Python_Batch_2GB_Fanout_8.txt @@ -0,0 +1,33 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### +--job_name=load-tests-python-dataflow-batch-combine-3- +--project=apache-beam-testing +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/smoketests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_combine_3 +--influx_measurement=python_batch_combine_3 +--input_options=''{\\"num_records\\":2500000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--num_workers=16 +--autoscaling_algorithm=NONE +--fanout=8 +--top_count=20 +--influxDatabase=beam_test_metrics +--influxHost=http://10.128.0.96:8086 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_FnApiRunner_Python_Microbenchmark.txt b/.github/workflows/load-tests-job-configs/config_FnApiRunner_Python_Microbenchmark.txt new file mode 100644 index 0000000000000..bc228b700a31c --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_FnApiRunner_Python_Microbenchmark.txt @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--influx_measurement=python_direct_microbenchmarks +--project=apache-beam-testing +--metrics_dataset=load_test +--metrics_table=python_direct_microbenchmarks +--input_options={} +--runner=DirectRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_100_Counters.txt b/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_100_Counters.txt new file mode 100644 index 0000000000000..52edf69824deb --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_100_Counters.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_measurement=go_batch_pardo_4 +--influx_namespace=dataflow +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90}'' +--iterations=1 +--number_of_counter_operations=100 +--number_of_counters=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_10_Counters.txt b/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_10_Counters.txt new file mode 100644 index 0000000000000..c2fa9edb2795a --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_10_Counters.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_measurement=go_batch_pardo_3 +--influx_namespace=dataflow +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90}'' +--iterations=1 +--number_of_counter_operations=10 +--number_of_counters=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_10_Iterations.txt b/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_10_Iterations.txt new file mode 100644 index 0000000000000..60aca27fda5fc --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_10_Iterations.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_measurement=go_batch_pardo_1 +--influx_namespace=dataflow +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90}'' +--iterations=10 +--number_of_counter_operations=0 +--number_of_counters=0 +--num_workers=5 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_200_Iterations.txt b/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_200_Iterations.txt new file mode 100644 index 0000000000000..abd115c37668e --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_ParDo_Go_Batch_200_Iterations.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_measurement=go_batch_pardo_2 +--influx_namespace=dataflow +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90}'' +--iterations=200 +--number_of_counter_operations=0 +--number_of_counters=0 +--num_workers=5 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_SideInput_Go_Batch_Dataflow_First_Iterable.txt b/.github/workflows/load-tests-job-configs/config_SideInput_Go_Batch_Dataflow_First_Iterable.txt new file mode 100644 index 0000000000000..51575d6468220 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_SideInput_Go_Batch_Dataflow_First_Iterable.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_namespace=dataflow +--influx_measurement=go_batch_sideinput_1 +--input_options=''{\"num_records\":10000000,\"key_size\":100,\"value_size\":900}'' +--access_percentage=1 +--num_workers=10 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/config_SideInput_Go_Batch_Dataflow_Iterable.txt b/.github/workflows/load-tests-job-configs/config_SideInput_Go_Batch_Dataflow_Iterable.txt new file mode 100644 index 0000000000000..4dac5e8d4ebb8 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/config_SideInput_Go_Batch_Dataflow_Iterable.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_namespace=dataflow +--influx_measurement=go_batch_sideinput_2 +--input_options=''{\"num_records\":10000000,\"key_size\":100,\"value_size\":900}'' +--num_workers=10 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_MultipleKey.txt b/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_MultipleKey.txt new file mode 100644 index 0000000000000..e508c47bbaa81 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_MultipleKey.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--influx_measurement=go_batch_cogbk_1 +--influx_namespace=flink +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":5,\"hot_key_fraction\":1}'' +--co_input_options=''{\"num_records\":2000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}'' +--iterations=1 +--parallelism=5 +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_Reiteration_10KB.txt b/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_Reiteration_10KB.txt new file mode 100644 index 0000000000000..b399b283f300f --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_Reiteration_10KB.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--influx_measurement=go_batch_cogbk_2 +--influx_namespace=flink +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":200000,\"hot_key_fraction\":1}'' +--co_input_options=''{\"num_records\":2000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}'' +--iterations=4 +--parallelism=5 +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_Reiteration_2MB.txt b/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_Reiteration_2MB.txt new file mode 100644 index 0000000000000..957f0b4c02725 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_Reiteration_2MB.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--influx_measurement=go_batch_cogbk_3 +--influx_namespace=flink +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}'' +--co_input_options=''{\"num_records\":2000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}'' +--iterations=4 +--parallelism=5 +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_10b.txt b/.github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_10b.txt new file mode 100644 index 0000000000000..6e13d4e185cb0 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_10b.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--influx_namespace=flink +--influx_measurement=go_batch_combine_1 +--input_options=''{\"num_records\":200000000,\"key_size\":1,\"value_size\":9}'' +--fanout=1 +--top_count=20 +--parallelism=5 +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_Fanout_4.txt b/.github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_Fanout_4.txt new file mode 100644 index 0000000000000..154076a639806 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_Fanout_4.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--influx_namespace=flink +--influx_measurement=go_batch_combine_2 +--input_options=''{\"num_records\":5000000,\"key_size\":10,\"value_size\":90}'' +--fanout=4 +--top_count=20 +--parallelism=16 +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_Fanout_8.txt b/.github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_Fanout_8.txt new file mode 100644 index 0000000000000..05e4dccf74deb --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_Combine_Flink_Batch_Fanout_8.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--influx_namespace=flink +--influx_measurement=go_batch_combine_3 +--fanout=8 +--top_count=20 +--parallelism=16 +--input_options=''{\"num_records\":2500000,\"key_size\":10,\"value_size\":90}'' +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100b.txt b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100b.txt new file mode 100644 index 0000000000000..b5266fa83047d --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100b.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_namespace=dataflow +--influx_measurement=go_batch_gbk_2 +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90}'' +--iterations=1 +--fanout=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100kb.txt b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100kb.txt new file mode 100644 index 0000000000000..072ab494515ca --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100kb.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_namespace=dataflow +--influx_measurement=go_batch_gbk_3 +--input_options=''{\"num_records\":20000,\"key_size\":10000,\"value_size\":90000}'' +--iterations=1 +--fanout=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_10b.txt b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_10b.txt new file mode 100644 index 0000000000000..2e69dd0b457b7 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_10b.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_namespace=dataflow +--influx_measurement=go_batch_gbk_1 +--input_options=''{\"num_records\":200000000,\"key_size\":1,\"value_size\":9}'' +--iterations=1 +--fanout=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_4.txt b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_4.txt new file mode 100644 index 0000000000000..6371123142d82 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_4.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_namespace=dataflow +--influx_measurement=go_batch_gbk_4 +--input_options=''{\"num_records\":5000000,\"key_size\":10,\"value_size\":90}'' +--iterations=1 +--fanout=4 +--num_workers=16 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_8.txt b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_8.txt new file mode 100644 index 0000000000000..77d5f2e0162bc --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_8.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_namespace=dataflow +--influx_measurement=go_batch_gbk_5 +--input_options=''{\"num_records\":2500000,\"key_size\":10,\"value_size\":90}'' +--iterations=1 +--fanout=8 +--num_workers=16 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_10KB.txt b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_10KB.txt new file mode 100644 index 0000000000000..7b27693ed8cc9 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_10KB.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_namespace=dataflow +--influx_measurement=go_batch_gbk_6 +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":200,\"hot_key_fraction\":1}'' +--iterations=4 +--fanout=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_2MB.txt b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_2MB.txt new file mode 100644 index 0000000000000..40971e5464547 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_2MB.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--influx_namespace=dataflow +--influx_measurement=go_batch_gbk_7 +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":10,\"hot_key_fraction\":1}'' +--iterations=4 +--fanout=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_100b.txt b/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_100b.txt new file mode 100644 index 0000000000000..78f99fb4e9cc0 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_100b.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--influx_namespace=flink +--influx_measurement=go_batch_gbk_2 +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90}'' +--iterations=1 +--fanout=1 +--parallelism=5 +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_100kb.txt b/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_100kb.txt new file mode 100644 index 0000000000000..5772a27ca184c --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_100kb.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--influx_namespace=flink +--influx_measurement=go_batch_gbk_3 +--iterations=1 +--fanout=1 +--parallelism=5 +--input_options=''{\"num_records\":20000,\"key_size\":10000,\"value_size\":90000}'' +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_10b.txt b/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_10b.txt new file mode 100644 index 0000000000000..e826fbe10dae2 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_10b.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--influx_namespace=flink +--influx_measurement=go_batch_gbk_1 +--input_options=''{\"num_records\":200000000,\"key_size\":1,\"value_size\":9}'' +--iterations=1 +--fanout=1 +--parallelism=5 +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Fanout_4.txt b/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Fanout_4.txt new file mode 100644 index 0000000000000..a5f6b09345218 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Fanout_4.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--influx_namespace=flink +--influx_measurement=go_batch_gbk_4 +--iterations=1 +--fanout=4 +--parallelism=16 +--input_options=''{\"num_records\":5000000,\"key_size\":10,\"value_size\":90}'' +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Fanout_8.txt b/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Fanout_8.txt new file mode 100644 index 0000000000000..b261ff58c5bfe --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Fanout_8.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--influx_namespace=flink +--influx_measurement=go_batch_gbk_5 +--iterations=1 +--fanout=8 +--parallelism=16 +--input_options=''{\"num_records\":2500000,\"key_size\":10,\"value_size\":90}'' +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Reiteration_10KB.txt b/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Reiteration_10KB.txt new file mode 100644 index 0000000000000..7e8d1eaa60d99 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_GBK_Flink_Batch_Reiteration_10KB.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--influx_namespace=flink +--influx_measurement=go_batch_gbk_6 +--iterations=4 +--fanout=1 +--parallelism=5 +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":200,\"hot_key_fraction\":1}'' +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_100_counters.txt b/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_100_counters.txt new file mode 100644 index 0000000000000..1b60c5380609f --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_100_counters.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--influx_measurement=go_batch_pardo_4 +--influx_namespace=flink +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90}'' +--iterations=1 +--number_of_counter_operations=100 +--number_of_counters=1 +--parallelism=5 +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_10_counters.txt b/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_10_counters.txt new file mode 100644 index 0000000000000..7179f2d469f23 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_10_counters.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--influx_measurement=go_batch_pardo_3 +--influx_namespace=flink +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90}'' +--iterations=1 +--number_of_counter_operations=10 +--number_of_counters=1 +--parallelism=5 +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_10_times.txt b/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_10_times.txt new file mode 100644 index 0000000000000..04821e3b6c9a1 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_10_times.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--influx_measurement=go_batch_pardo_1 +--influx_namespace=flink +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90}'' +--iterations=10 +--number_of_counter_operations=0 +--number_of_counters=0 +--parallelism=5 +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_200_times.txt b/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_200_times.txt new file mode 100644 index 0000000000000..a2606f097f4e7 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_ParDo_Flink_Batch_200_times.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--influx_measurement=go_batch_pardo_2 +--influx_namespace=flink +--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90}'' +--iterations=200 +--number_of_counter_operations=0 +--number_of_counters=0 +--parallelism=5 +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_SideInput_Flink_Batch_First_Iterable.txt b/.github/workflows/load-tests-job-configs/go_SideInput_Flink_Batch_First_Iterable.txt new file mode 100644 index 0000000000000..0d50916b8bd68 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_SideInput_Flink_Batch_First_Iterable.txt @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--influx_namespace=flink +--influx_measurement=go_batch_sideinput_1 +--input_options=''{\"num_records\":400000,\"key_size\":100,\"value_size\":900}'' +--access_percentage=1 +--parallelism=10 +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/go_SideInput_Flink_Batch_Iterable.txt b/.github/workflows/load-tests-job-configs/go_SideInput_Flink_Batch_Iterable.txt new file mode 100644 index 0000000000000..ca11bc55faa94 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/go_SideInput_Flink_Batch_Iterable.txt @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--influx_namespace=flink +--influx_measurement=go_batch_sideinput_2 +--input_options=''{\"num_records\":400000,\"key_size\":100,\"value_size\":900}'' +--parallelism=10 +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt b/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt new file mode 100644 index 0000000000000..6402c43d1567d --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_cogbk_2 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":5} +--coSourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_100b_Single_Key.txt b/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_100b_Single_Key.txt new file mode 100644 index 0000000000000..5d53c3b3d7ef0 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_100b_Single_Key.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_cogbk_1 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1} +--coSourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_10kB.txt b/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_10kB.txt new file mode 100644 index 0000000000000..501fa6b3a57f7 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_10kB.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_cogbk_3 +--publishToInfluxDB=true +--sourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":200000} +--coSourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--iterations=4 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_2MB.txt b/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_2MB.txt new file mode 100644 index 0000000000000..0d5f57fc47b8d --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_Batch_2MB.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_cogbk_4 +--publishToInfluxDB=true +--sourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--coSourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--iterations=4 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Multiple_Keys.txt b/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Multiple_Keys.txt new file mode 100644 index 0000000000000..24aff12bad79d --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Multiple_Keys.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_cogbk_2 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":5} +--coSourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--coInputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Single_Key.txt b/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Single_Key.txt new file mode 100644 index 0000000000000..eead04aeb7e4e --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_100b_Single_Key.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_cogbk_1 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1} +--coSourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--coInputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_10kB.txt b/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_10kB.txt new file mode 100644 index 0000000000000..4b45c7df3f9e9 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_10kB.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_cogbk_3 +--publishToInfluxDB=true +--sourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":200000} +--coSourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--iterations=4 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--coInputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_2MB.txt b/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_2MB.txt new file mode 100644 index 0000000000000..f522fd7ec36dc --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_CoGBK_Dataflow_V2_Streaming_Java_2MB.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_cogbk_4 +--publishToInfluxDB=true +--sourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--coSourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--iterations=4 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--coInputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_100b_Multiple_Keys.txt b/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_100b_Multiple_Keys.txt new file mode 100644 index 0000000000000..747f495b144d2 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_100b_Multiple_Keys.txt @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--appName=load_tests_Java_SparkStructuredStreaming_batch_CoGBK_2 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_cogbk_2 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":5} +--coSourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--iterations=1 +--streaming=false +--runner=SparkStructuredStreamingRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_100b_Single_Key.txt b/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_100b_Single_Key.txt new file mode 100644 index 0000000000000..b17549a702f35 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_100b_Single_Key.txt @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--appName=load_tests_Java_SparkStructuredStreaming_batch_CoGBK_1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_cogbk_1 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1} +--coSourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--iterations=1 +--streaming=false +--runner=SparkStructuredStreamingRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_10kB.txt b/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_10kB.txt new file mode 100644 index 0000000000000..84f53ee120a2e --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_10kB.txt @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--appName=load_tests_Java_SparkStructuredStreaming_batch_CoGBK_3 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_cogbk_3 +--publishToInfluxDB=true +--sourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":200000} +--coSourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--iterations=4 +--streaming=false +--runner=SparkStructuredStreamingRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_2MB.txt b/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_2MB.txt new file mode 100644 index 0000000000000..8f8bdb8995516 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_CoGBK_SparkStructuredStreaming_Batch_2MB.txt @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--appName=load_tests_Java_SparkStructuredStreaming_batch_CoGBK_4 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_cogbk_4 +--publishToInfluxDB=true +--sourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--coSourceOptions={"numRecords":2000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":1000} +--iterations=4 +--streaming=false +--runner=SparkStructuredStreamingRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_10b.txt b/.github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_10b.txt new file mode 100644 index 0000000000000..f52b737aaa94c --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_10b.txt @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_streaming_Combine_1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_combine_1 +--publishToInfluxDB=true +--sourceOptions={"numRecords":200000000,"keySizeBytes":1,"valueSizeBytes":9} +--fanout=1 +--iterations=1 +--topCount=20 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--perKeyCombiner=TOP_LARGEST +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_Fanout_4.txt b/.github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_Fanout_4.txt new file mode 100644 index 0000000000000..0a6424d3becc3 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_Fanout_4.txt @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_streaming_Combine_2 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_combine_2 +--publishToInfluxDB=true +--sourceOptions={"numRecords":5000000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=4 +--iterations=1 +--topCount=20 +--numWorkers=16 +--autoscalingAlgorithm=NONE +--perKeyCombiner=TOP_LARGEST +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_Fanout_8.txt b/.github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_Fanout_8.txt new file mode 100644 index 0000000000000..9425a442cda2c --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_Combine_Dataflow_Streaming_Fanout_8.txt @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_streaming_Combine_3 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_combine_3 +--publishToInfluxDB=true +--sourceOptions={"numRecords":2500000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=8 +--iterations=1 +--topCount=20 +--numWorkers=16 +--autoscalingAlgorithm=NONE +--perKeyCombiner=TOP_LARGEST +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_10b.txt b/.github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_10b.txt new file mode 100644 index 0000000000000..a8e93ca4c702a --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_10b.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--appName=load_tests_Java_SparkStructuredStreaming_batch_Combine_1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_combine_1 +--publishToInfluxDB=true +--sourceOptions={"numRecords":200000000,"keySizeBytes":1,"valueSizeBytes":9} +--fanout=1 +--iterations=1 +--topCount=20 +--perKeyCombiner=TOP_LARGEST +--streaming=false +--runner=SparkStructuredStreamingRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_Fanout_4.txt b/.github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_Fanout_4.txt new file mode 100644 index 0000000000000..c0240410a356e --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_Fanout_4.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--appName=load_tests_Java_SparkStructuredStreaming_batch_Combine_2 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_combine_2 +--publishToInfluxDB=true +--sourceOptions={"numRecords":5000000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=4 +--iterations=1 +--topCount=20 +--perKeyCombiner=TOP_LARGEST +--streaming=false +--runner=SparkStructuredStreamingRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_Fanout_8.txt b/.github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_Fanout_8.txt new file mode 100644 index 0000000000000..17daad210ed95 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_Combine_SparkStructuredStreaming_Batch_Fanout_8.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--appName=load_tests_Java_SparkStructuredStreaming_batch_Combine_3 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_combine_3 +--publishToInfluxDB=true +--sourceOptions={"numRecords":2500000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=8 +--iterations=1 +--topCount=20 +--perKeyCombiner=TOP_LARGEST +--streaming=false +--runner=SparkStructuredStreamingRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_100B_records.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_100B_records.txt new file mode 100644 index 0000000000000..29d0ded11fac8 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_100B_records.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_batch_GBK_2 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_2 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=1 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_100kB_records.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_100kB_records.txt new file mode 100644 index 0000000000000..920f0a60d198a --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_100kB_records.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_batch_GBK_3 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_3 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000,"keySizeBytes":10000,"valueSizeBytes":90000} +--fanout=1 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_10B_records.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_10B_records.txt new file mode 100644 index 0000000000000..7ccacc6c791cd --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_10B_records.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_batch_GBK_1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_1 +--publishToInfluxDB=true +--sourceOptions={"numRecords":200000000,"keySizeBytes":1,"valueSizeBytes":9} +--fanout=1 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..8c6f6f1c89a08 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_batch_GBK_4 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_4 +--publishToInfluxDB=true +--sourceOptions={"numRecords":5000000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=4 +--iterations=1 +--numWorkers=16 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..43f04dac1d02a --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_batch_GBK_5 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_5 +--publishToInfluxDB=true +--sourceOptions={"numRecords":2500000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=8 +--iterations=1 +--numWorkers=16 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_reiterate_4_times_10kB_values.txt new file mode 100644 index 0000000000000..c8f9ecc34770f --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_reiterate_4_times_10kB_values.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_batch_GBK_6 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_6 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":200,"hotKeyFraction":1} +--fanout=1 +--iterations=4 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_reiterate_4_times_2MB_values.txt new file mode 100644 index 0000000000000..653004c6a04e7 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_reiterate_4_times_2MB_values.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_batch_GBK_7 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_7 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":10,"hotKeyFraction":1} +--fanout=1 +--iterations=4 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_100B_records.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_100B_records.txt new file mode 100644 index 0000000000000..a6523c64edbc1 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_100B_records.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_streaming_GBK_2 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_2 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=1 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt new file mode 100644 index 0000000000000..422557e84a664 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_streaming_GBK_3 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_3 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000,"keySizeBytes":10000,"valueSizeBytes":90000} +--fanout=1 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_10B_records.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_10B_records.txt new file mode 100644 index 0000000000000..da6c04d0d284e --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_10B_records.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_streaming_GBK_1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_1 +--publishToInfluxDB=true +--sourceOptions={"numRecords":200000000,"keySizeBytes":1,"valueSizeBytes":9} +--fanout=1 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..33181a9397009 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_streaming_GBK_4 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_4 +--publishToInfluxDB=true +--sourceOptions={"numRecords":5000000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=4 +--iterations=1 +--numWorkers=16 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..32b59df2a2e0b --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_streaming_GBK_5 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_5 +--publishToInfluxDB=true +--sourceOptions={"numRecords":2500000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=8 +--iterations=1 +--numWorkers=16 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_reiterate_4_times_10kB_values.txt new file mode 100644 index 0000000000000..376d7e3244a8c --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_reiterate_4_times_10kB_values.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_streaming_GBK_6 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_6 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":200,"hotKeyFraction":1} +--fanout=1 +--iterations=4 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_reiterate_4_times_2MB_values.txt new file mode 100644 index 0000000000000..b7bf9db40e825 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_reiterate_4_times_2MB_values.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_streaming_GBK_7 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_7 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":10,"hotKeyFraction":1} +--fanout=1 +--iterations=4 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100B_records.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100B_records.txt new file mode 100644 index 0000000000000..d9b1918383b26 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100B_records.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java11_Dataflow_V2_batch_GBK_2 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_2 +--influxTags={"runnerVersion":"v2","jdk":"java11"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=1 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100kB_records.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100kB_records.txt new file mode 100644 index 0000000000000..66911feb7d173 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100kB_records.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java11_Dataflow_V2_batch_GBK_3 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_3 +--influxTags={"runnerVersion":"v2","jdk":"java11"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000,"keySizeBytes":10000,"valueSizeBytes":90000} +--fanout=1 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_10B_records.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_10B_records.txt new file mode 100644 index 0000000000000..e265724ec1c16 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_10B_records.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java11_Dataflow_V2_batch_GBK_1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_1 +--influxTags={"runnerVersion":"v2","jdk":"java11"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":200000000,"keySizeBytes":1,"valueSizeBytes":9} +--fanout=1 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..8610a9dff9f98 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java11_Dataflow_V2_batch_GBK_4 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_4 +--influxTags={"runnerVersion":"v2","jdk":"java11"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":5000000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=4 +--iterations=1 +--numWorkers=16 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..65aa569932945 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java11_Dataflow_V2_batch_GBK_5 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_5 +--influxTags={"runnerVersion":"v2","jdk":"java11"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":2500000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=8 +--iterations=1 +--numWorkers=16 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_10kB_values.txt new file mode 100644 index 0000000000000..aaf66ef03505c --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_10kB_values.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java11_Dataflow_V2_batch_GBK_6 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_6 +--influxTags={"runnerVersion":"v2","jdk":"java11"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":200,"hotKeyFraction":1} +--fanout=1 +--iterations=4 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_2MB_values.txt new file mode 100644 index 0000000000000..8698ac90cac11 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_2MB_values.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java11_Dataflow_V2_batch_GBK_7 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_7 +--influxTags={"runnerVersion":"v2","jdk":"java11"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":10,"hotKeyFraction":1} +--fanout=1 +--iterations=4 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100B_records.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100B_records.txt new file mode 100644 index 0000000000000..f5a432b6d1403 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100B_records.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java17_Dataflow_V2_batch_GBK_2 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_2 +--influxTags={"runnerVersion":"v2","jdk":"java17"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=1 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100kB_records.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100kB_records.txt new file mode 100644 index 0000000000000..976bd20f7dff2 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100kB_records.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java17_Dataflow_V2_batch_GBK_3 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_3 +--influxTags={"runnerVersion":"v2","jdk":"java17"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000,"keySizeBytes":10000,"valueSizeBytes":90000} +--fanout=1 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_10B_records.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_10B_records.txt new file mode 100644 index 0000000000000..89d356201a76a --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_10B_records.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java17_Dataflow_V2_batch_GBK_1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_1 +--influxTags={"runnerVersion":"v2","jdk":"java17"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":200000000,"keySizeBytes":1,"valueSizeBytes":9} +--fanout=1 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..0735c9bafc45b --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java17_Dataflow_V2_batch_GBK_4 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_4 +--influxTags={"runnerVersion":"v2","jdk":"java17"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":5000000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=4 +--iterations=1 +--numWorkers=16 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..4ea7f402cc581 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java17_Dataflow_V2_batch_GBK_5 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_5 +--influxTags={"runnerVersion":"v2","jdk":"java17"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":2500000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=8 +--iterations=1 +--numWorkers=16 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_10kB_values.txt new file mode 100644 index 0000000000000..881640714df2d --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_10kB_values.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java17_Dataflow_V2_batch_GBK_6 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_6 +--influxTags={"runnerVersion":"v2","jdk":"java17"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":200,"hotKeyFraction":1} +--fanout=1 +--iterations=4 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_2MB_values.txt new file mode 100644 index 0000000000000..a09c16d1e66a5 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_2MB_values.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java17_Dataflow_V2_batch_GBK_7 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_gbk_7 +--influxTags={"runnerVersion":"v2","jdk":"java17"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":10,"hotKeyFraction":1} +--fanout=1 +--iterations=4 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100B_records.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100B_records.txt new file mode 100644 index 0000000000000..84349e3dc06ea --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100B_records.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java11_Dataflow_V2_streaming_GBK_2 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_2 +--influxTags={"runnerVersion":"v2","jdk":"java11"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=1 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100kB_records.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100kB_records.txt new file mode 100644 index 0000000000000..761fad3d11ba3 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100kB_records.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java11_Dataflow_V2_streaming_GBK_3 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_3 +--influxTags={"runnerVersion":"v2","jdk":"java11"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000,"keySizeBytes":10000,"valueSizeBytes":90000} +--fanout=1 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_10B_records.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_10B_records.txt new file mode 100644 index 0000000000000..1b6624e52482c --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_10B_records.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java11_Dataflow_V2_streaming_GBK_1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_1 +--influxTags={"runnerVersion":"v2","jdk":"java11"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":200000000,"keySizeBytes":1,"valueSizeBytes":9} +--fanout=1 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..cf173fa845c3b --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java11_Dataflow_V2_streaming_GBK_4 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_4 +--influxTags={"runnerVersion":"v2","jdk":"java11"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":5000000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=4 +--iterations=1 +--numWorkers=16 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..8e751d638ceb0 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java11_Dataflow_V2_streaming_GBK_5 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_5 +--influxTags={"runnerVersion":"v2","jdk":"java11"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":2500000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=8 +--iterations=1 +--numWorkers=16 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_10kB_values.txt new file mode 100644 index 0000000000000..2e9c847cb8327 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_10kB_values.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java11_Dataflow_V2_streaming_GBK_6 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_6 +--influxTags={"runnerVersion":"v2","jdk":"java11"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":200,"hotKeyFraction":1} +--fanout=1 +--iterations=4 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_2MB_values.txt new file mode 100644 index 0000000000000..dd410a81487c2 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_2MB_values.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java11_Dataflow_V2_streaming_GBK_7 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_7 +--influxTags={"runnerVersion":"v2","jdk":"java11"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":10,"hotKeyFraction":1} +--fanout=1 +--iterations=4 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100B_records.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100B_records.txt new file mode 100644 index 0000000000000..05d93f213ec7e --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100B_records.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java17_Dataflow_V2_streaming_GBK_2 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_2 +--influxTags={"runnerVersion":"v2","jdk":"java17"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=1 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100kB_records.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100kB_records.txt new file mode 100644 index 0000000000000..2391bfe4e416c --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100kB_records.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java17_Dataflow_V2_streaming_GBK_3 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_3 +--influxTags={"runnerVersion":"v2","jdk":"java17"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000,"keySizeBytes":10000,"valueSizeBytes":90000} +--fanout=1 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_10B_records.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_10B_records.txt new file mode 100644 index 0000000000000..3c4e04e02adfa --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_10B_records.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java17_Dataflow_V2_streaming_GBK_1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_1 +--influxTags={"runnerVersion":"v2","jdk":"java17"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":200000000,"keySizeBytes":1,"valueSizeBytes":9} +--fanout=1 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..d43ab9d84688c --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java17_Dataflow_V2_streaming_GBK_4 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_4 +--influxTags={"runnerVersion":"v2","jdk":"java17"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":5000000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=4 +--iterations=1 +--numWorkers=16 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..fd5233180ff62 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java17_Dataflow_V2_streaming_GBK_5 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_5 +--influxTags={"runnerVersion":"v2","jdk":"java17"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":2500000,"keySizeBytes":10,"valueSizeBytes":90} +--fanout=8 +--iterations=1 +--numWorkers=16 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_10kB_values.txt new file mode 100644 index 0000000000000..50d17886b7c91 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_10kB_values.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java17_Dataflow_V2_streaming_GBK_6 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_6 +--influxTags={"runnerVersion":"v2","jdk":"java17"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":200,"hotKeyFraction":1} +--fanout=1 +--iterations=4 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_2MB_values.txt new file mode 100644 index 0000000000000..e28d4d895ada7 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_2MB_values.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java17_Dataflow_V2_streaming_GBK_7 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_gbk_7 +--influxTags={"runnerVersion":"v2","jdk":"java17"} +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90,"numHotKeys":10,"hotKeyFraction":1} +--fanout=1 +--iterations=4 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-1.txt b/.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-1.txt new file mode 100644 index 0000000000000..7cd503ad44992 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-1.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=smoke-dsl-java +--tempLocation=gs://temp-storage-for-perf-tests/smoketests +--sourceOptions={"numRecords":100000,"splitPointFrequencyRecords":1} +--stepOptions={"outputRecordsPerInputRecord":1,"preservesInputKeyDistribution":true} +--fanout=10 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-2.txt b/.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-2.txt new file mode 100644 index 0000000000000..bccffadea8d7f --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-2.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=smoke-dsl-java +--tempLocation=gs://temp-storage-for-perf-tests/smoketests +--sourceOptions={"numRecords":100000,"keySizeBytes":1,"valueSizeBytes":1} +--fanout=1 +--iterations=1 +--numWorkers=3 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-3.txt b/.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-3.txt new file mode 100644 index 0000000000000..44bd342b462c3 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-3.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=smoke-dsl-java +--tempLocation=gs://temp-storage-for-perf-tests/smoketests +--sourceOptions={"numRecords":20000,"keySizeBytes":1,"valueSizeBytes":1} +--fanout=10 +--iterations=1 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_100_counters.txt b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_100_counters.txt new file mode 100644 index 0000000000000..b18650a8e8a68 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_100_counters.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_batch_ParDo_4 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_pardo_4 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=1 +--numberOfCounters=1 +--numberOfCounterOperations=100 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_10_counters.txt b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_10_counters.txt new file mode 100644 index 0000000000000..c6a7785e02c4b --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_10_counters.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_batch_ParDo_3 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_pardo_3 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=1 +--numberOfCounters=1 +--numberOfCounterOperations=10 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_10_times.txt b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_10_times.txt new file mode 100644 index 0000000000000..8e692050c25ce --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_10_times.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_batch_ParDo_1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_pardo_1 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=10 +--numberOfCounters=1 +--numberOfCounterOperations=0 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_200_times.txt b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_200_times.txt new file mode 100644 index 0000000000000..795990ee7ba06 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Batch_200_times.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_batch_ParDo_2 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_pardo_2 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=200 +--numberOfCounters=1 +--numberOfCounterOperations=0 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_100_counters.txt b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_100_counters.txt new file mode 100644 index 0000000000000..adcfb461913d9 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_100_counters.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_streaming_ParDo_4 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_pardo_4 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=1 +--numberOfCounters=1 +--numberOfCounterOperations=100 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_10_counters.txt b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_10_counters.txt new file mode 100644 index 0000000000000..a954c0b8180ec --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_10_counters.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_streaming_ParDo_3 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_pardo_3 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=1 +--numberOfCounters=1 +--numberOfCounterOperations=10 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_10_times.txt b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_10_times.txt new file mode 100644 index 0000000000000..9b579c428be13 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_10_times.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_streaming_ParDo_1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_pardo_1 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=10 +--numberOfCounters=1 +--numberOfCounterOperations=0 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_200_times.txt b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_200_times.txt new file mode 100644 index 0000000000000..6b08284cd06e8 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_Streaming_200_times.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--appName=load_tests_Java_Dataflow_streaming_ParDo_2 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_pardo_2 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=200 +--numberOfCounters=1 +--numberOfCounterOperations=0 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_100_counters.txt b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_100_counters.txt new file mode 100644 index 0000000000000..7f05aedab7636 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_100_counters.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_pardo_4 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=1 +--numberOfCounters=1 +--numberOfCounterOperations=100 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_counters.txt b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_counters.txt new file mode 100644 index 0000000000000..e9c158698c57a --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_counters.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_pardo_3 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=1 +--numberOfCounters=1 +--numberOfCounterOperations=10 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_times.txt b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_times.txt new file mode 100644 index 0000000000000..cd6b006d8f3ae --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_times.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_pardo_1 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=10 +--numberOfCounters=1 +--numberOfCounterOperations=0 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_200_times.txt b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_200_times.txt new file mode 100644 index 0000000000000..d4b6bef42f787 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_200_times.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_pardo_2 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=200 +--numberOfCounters=1 +--numberOfCounterOperations=0 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=false +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_100_counters.txt b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_100_counters.txt new file mode 100644 index 0000000000000..1dcb6f4a7cbb8 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_100_counters.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_pardo_4 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=1 +--numberOfCounters=1 +--numberOfCounterOperations=100 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_counters.txt b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_counters.txt new file mode 100644 index 0000000000000..34fb4b4658d83 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_counters.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_pardo_3 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=1 +--numberOfCounters=1 +--numberOfCounterOperations=10 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_times.txt b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_times.txt new file mode 100644 index 0000000000000..9d4d918f2e6d2 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_times.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_pardo_1 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=10 +--numberOfCounters=1 +--numberOfCounterOperations=0 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_200_times.txt b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_200_times.txt new file mode 100644 index 0000000000000..c7d0e3e4835ac --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_200_times.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_streaming_pardo_2 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=200 +--numberOfCounters=1 +--numberOfCounterOperations=0 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--streaming=true +--inputWindowDurationSec=1200 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_100_counters.txt b/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_100_counters.txt new file mode 100644 index 0000000000000..5c2ccf18f3cbd --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_100_counters.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--appName=load_tests_Java_SparkStructuredStreaming_batch_ParDo_4 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_pardo_4 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=1 +--numberOfCounters=1 +--numberOfCounterOperations=100 +--streaming=false +--runner=SparkStructuredStreamingRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_10_counters.txt b/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_10_counters.txt new file mode 100644 index 0000000000000..9d0b1c65a2eda --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_10_counters.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--appName=load_tests_Java_SparkStructuredStreaming_batch_ParDo_3 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_pardo_3 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=1 +--numberOfCounters=1 +--numberOfCounterOperations=10 +--streaming=false +--runner=SparkStructuredStreamingRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_10_times.txt b/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_10_times.txt new file mode 100644 index 0000000000000..d3a82cf3caaed --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_10_times.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--appName=load_tests_Java_SparkStructuredStreaming_batch_ParDo_1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_pardo_1 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=10 +--numberOfCounters=1 +--numberOfCounterOperations=0 +--streaming=false +--runner=SparkStructuredStreamingRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_200_times.txt b/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_200_times.txt new file mode 100644 index 0000000000000..e5642d9b32462 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_ParDo_SparkStructuredStreaming_Batch_200_times.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--appName=load_tests_Java_SparkStructuredStreaming_batch_ParDo_2 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--influxMeasurement=java_batch_pardo_2 +--publishToInfluxDB=true +--sourceOptions={"numRecords":20000000,"keySizeBytes":10,"valueSizeBytes":90} +--iterations=200 +--numberOfCounters=1 +--numberOfCounterOperations=0 +--streaming=false +--runner=SparkStructuredStreamingRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Dataflow.txt b/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Dataflow.txt new file mode 100644 index 0000000000000..411aeaeaf5869 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Dataflow.txt @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--sourceOptions={"numRecords":100000,"splitPointFrequencyRecords":1} +--stepOptions={"outputRecordsPerInputRecord":1,"preservesInputKeyDistribution":true} +--fanout=10 +--iterations=1 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Direct.txt b/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Direct.txt new file mode 100644 index 0000000000000..a062dbc4846a1 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Direct.txt @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--sourceOptions={"numRecords":100000,"splitPointFrequencyRecords":1} +--stepOptions={"outputRecordsPerInputRecord":1,"preservesInputKeyDistribution":true} +--fanout=10 +--iterations=1 \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Flink.txt b/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Flink.txt new file mode 100644 index 0000000000000..bfa07b5f2dcd3 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Flink.txt @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--sourceOptions={"numRecords":100000,"splitPointFrequencyRecords":1} +--stepOptions={"outputRecordsPerInputRecord":1,"preservesInputKeyDistribution":true} +--fanout=10 +--iterations=1 +--runner=FlinkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Spark.txt b/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Spark.txt new file mode 100644 index 0000000000000..9eed902195c71 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Spark.txt @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--sparkMaster=local[4] +--sourceOptions={"numRecords":100000,"splitPointFrequencyRecords":1} +--stepOptions={"outputRecordsPerInputRecord":1,"preservesInputKeyDistribution":true} +--fanout=10 +--iterations=1 +--runner=SparkRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt new file mode 100644 index 0000000000000..d5ba43180738e --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Batch_100b_Multiple_Keys.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_cogbk_2 +--influx_measurement=python_batch_cogbk_2 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":5,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--co_input_options=''{\\"num_records\\":2000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":1000,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Batch_100b_Single_Key.txt b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Batch_100b_Single_Key.txt new file mode 100644 index 0000000000000..47ebf22dc8354 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Batch_100b_Single_Key.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_cogbk_1 +--influx_measurement=python_batch_cogbk_1 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":1,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--co_input_options=''{\\"num_records\\":2000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":1000,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Batch_10kB.txt b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Batch_10kB.txt new file mode 100644 index 0000000000000..13161125b570e --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Batch_10kB.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_cogbk_3 +--influx_measurement=python_batch_cogbk_3 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":200000,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--co_input_options=''{\\"num_records\\":2000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":1000,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--iterations=4 +--num_workers=5 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Batch_2MB.txt b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Batch_2MB.txt new file mode 100644 index 0000000000000..052c2464a1cc3 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Batch_2MB.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_cogbk_4 +--influx_measurement=python_batch_cogbk_4 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":1000,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--co_input_options=''{\\"num_records\\":2000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":1000,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--iterations=4 +--num_workers=5 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Flink_Batch_100b_Multiple_Keys.txt b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Flink_Batch_100b_Multiple_Keys.txt new file mode 100644 index 0000000000000..4b8a2f72010b5 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Flink_Batch_100b_Multiple_Keys.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_batch_cogbk_2 +--influx_measurement=python_batch_cogbk_2 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":5,\\"hot_key_fraction\\":1}'' +--co_input_options=''{\\"num_records\\":2000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":5,\\"hot_key_fraction\\":1}'' +--iterations=1 +--parallelism=5 +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Flink_Batch_100b_Single_Key.txt b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Flink_Batch_100b_Single_Key.txt new file mode 100644 index 0000000000000..3aeb927f04eed --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Flink_Batch_100b_Single_Key.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_batch_cogbk_1 +--influx_measurement=python_batch_cogbk_1 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":1,\\"hot_key_fraction\\":1}'' +--co_input_options=''{\\"num_records\\":2000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":1000,\\"hot_key_fraction\\":1}'' +--iterations=1 +--parallelism=5 +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Flink_Batch_10kB.txt b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Flink_Batch_10kB.txt new file mode 100644 index 0000000000000..e350e2d29944d --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Flink_Batch_10kB.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_batch_cogbk_3 +--influx_measurement=python_batch_cogbk_3 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":200000,\\"hot_key_fraction\\":1}'' +--co_input_options=''{\\"num_records\\":2000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":1000,\\"hot_key_fraction\\":1}'' +--iterations=4 +--parallelism=5 +--endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_100b_Multiple_Keys.txt b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_100b_Multiple_Keys.txt new file mode 100644 index 0000000000000..a687f0cf5de7d --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_100b_Multiple_Keys.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_streaming_cogbk_2 +--influx_measurement=python_streaming_cogbk_2 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":5,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--co_input_options=''{\\"num_records\\":2000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":1000,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--streaming +--worker_machine_type=n1-highmem-4 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_100b_Single_Key.txt b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_100b_Single_Key.txt new file mode 100644 index 0000000000000..9141182b90fc1 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_100b_Single_Key.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_streaming_cogbk_1 +--influx_measurement=python_streaming_cogbk_1 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":1,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--co_input_options=''{\\"num_records\\":2000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":1000,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--streaming +--worker_machine_type=n1-highmem-4 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_10kB.txt b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_10kB.txt new file mode 100644 index 0000000000000..7250f073f25ea --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_10kB.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_streaming_cogbk_3 +--influx_measurement=python_streaming_cogbk_3 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":200000,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--co_input_options=''{\\"num_records\\":2000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":1000,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--iterations=4 +--num_workers=5 +--autoscaling_algorithm=NONE +--streaming +--worker_machine_type=n1-highmem-4 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_2MB.txt b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_2MB.txt new file mode 100644 index 0000000000000..59723107d53c8 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python-cogbk/python_CoGBK_Dataflow_Streaming_2MB.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_streaming_cogbk_4 +--influx_measurement=python_streaming_cogbk_4 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":1000,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--co_input_options=''{\\"num_records\\":2000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":1000,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--iterations=4 +--num_workers=5 +--autoscaling_algorithm=NONE +--streaming +--worker_machine_type=n1-highmem-4 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_10_byte_records.txt b/.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_10_byte_records.txt new file mode 100644 index 0000000000000..8535f85c737d8 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_10_byte_records.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/smoketests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_streaming_combine_1 +--influx_measurement=python_streaming_combine_1 +--input_options=''{\\"num_records\\":200000000,\\"key_size\\":1,\\"value_size\\":9,\\"algorithm\\":\\"lcg\\"}'' +--num_workers=5 +--autoscaling_algorithm=NONE +--top_count=20 +--streaming +--experiments=use_runner_v2 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_Fanout_4.txt b/.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_Fanout_4.txt new file mode 100644 index 0000000000000..03b26b9d4e13e --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_Fanout_4.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/smoketests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_streaming_combine_4 +--influx_measurement=python_streaming_combine_4 +--input_options=''{\\"num_records\\":5000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--num_workers=16 +--autoscaling_algorithm=NONE +--fanout=4 +--top_count=20 +--streaming +--experiments=use_runner_v2 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_Fanout_8.txt b/.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_Fanout_8.txt new file mode 100644 index 0000000000000..46d68261342f7 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_Fanout_8.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/smoketests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_streaming_combine_5 +--influx_measurement=python_streaming_combine_5 +--input_options=''{\\"num_records\\":2500000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--num_workers=16 +--autoscaling_algorithm=NONE +--fanout=8 +--top_count=20 +--streaming +--experiments=use_runner_v2 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_10_byte_records.txt b/.github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_10_byte_records.txt new file mode 100644 index 0000000000000..8295d1c8aa860 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_10_byte_records.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_batch_combine_1 +--influx_measurement=python_batch_combine_1 +--input_options=''{\\"num_records\\":200000000,\\"key_size\\":1,\\"value_size\\":9,\\"algorithm\\":\\"lcg\\"}'' +--parallelism=5 +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--top_count=20 +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_Fanout_4.txt b/.github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_Fanout_4.txt new file mode 100644 index 0000000000000..82f8bcc7c0aee --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_Fanout_4.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_batch_combine_4 +--influx_measurement=python_batch_combine_4 +--input_options=''{\\"num_records\\":5000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--parallelism=16 +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--fanout=4 +--top_count=20 +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_Fanout_8.txt b/.github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_Fanout_8.txt new file mode 100644 index 0000000000000..45425b6bf1536 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_Combine_Flink_Batch_2GB_Fanout_8.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_batch_combine_5 +--influx_measurement=python_batch_combine_5 +--input_options=''{\\"num_records\\":2500000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--parallelism=16 +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--fanout=8 +--top_count=20 +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_10_byte_records.txt b/.github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_10_byte_records.txt new file mode 100644 index 0000000000000..12ffc1790e46f --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_10_byte_records.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_streaming_combine_1 +--influx_measurement=python_streaming_combine_1 +--input_options=''{\\"num_records\\":200000000,\\"key_size\\":1,\\"value_size\\":9,\\"algorithm\\":\\"lcg\\"}'' +--parallelism=5 +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--top_count=20 +--streaming +--use_stateful_load_generator +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_Fanout_4.txt b/.github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_Fanout_4.txt new file mode 100644 index 0000000000000..c7d5552a03bd6 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_Fanout_4.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_streaming_combine_4 +--influx_measurement=python_streaming_combine_4 +--input_options=''{\\"num_records\\":5000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--parallelism=16 +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--fanout=4 +--top_count=20 +--streaming +--use_stateful_load_generator +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_Fanout_8.txt b/.github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_Fanout_8.txt new file mode 100644 index 0000000000000..bffdeab2cb11f --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_Combine_Flink_Streaming_2GB_Fanout_8.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_streaming_combine_5 +--influx_measurement=python_streaming_combine_5 +--input_options=''{\\"num_records\\":2500000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--parallelism=16 +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--fanout=8 +--top_count=20 +--streaming +--use_stateful_load_generator +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_100B_records.txt b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_100B_records.txt new file mode 100644 index 0000000000000..ad05bf1e85d30 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_100B_records.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_gbk_2 +--influx_measurement=python_batch_gbk_2 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--fanout=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_100kB_records.txt b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_100kB_records.txt new file mode 100644 index 0000000000000..8d3358a12f98e --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_100kB_records.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_gbk_3 +--influx_measurement=python_batch_gbk_3 +--input_options=''{\\"num_records\\":20000,\\"key_size\\":10000,\\"value_size\\":90000,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--fanout=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_10B_records.txt b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_10B_records.txt new file mode 100644 index 0000000000000..885c5ca61954c --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_2GB_of_10B_records.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_gbk_1 +--influx_measurement=python_batch_gbk_1 +--input_options=''{\\"num_records\\":200000000,\\"key_size\\":1,\\"value_size\\":9,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--fanout=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..1663e646f542a --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_gbk_4 +--influx_measurement=python_batch_gbk_4 +--input_options=''{\\"num_records\\":5000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--fanout=4 +--num_workers=16 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..4a1768c9d17dd --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_gbk_5 +--influx_measurement=python_batch_gbk_5 +--input_options=''{\\"num_records\\":2500000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--fanout=8 +--num_workers=16 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_2GB_of_100B_records.txt b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_2GB_of_100B_records.txt new file mode 100644 index 0000000000000..057f71d5627c8 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_2GB_of_100B_records.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_streaming_gbk_2 +--influx_measurement=python_streaming_gbk_2 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--fanout=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--streaming +--experiments=use_runner_v2 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt new file mode 100644 index 0000000000000..57c1be11d592e --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_streaming_gbk_3 +--influx_measurement=python_streaming_gbk_3 +--input_options=''{\\"num_records\\":20000,\\"key_size\\":10000,\\"value_size\\":90000,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--fanout=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--streaming +--experiments=use_runner_v2 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_2GB_of_10B_records.txt b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_2GB_of_10B_records.txt new file mode 100644 index 0000000000000..64d224a4663f1 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_2GB_of_10B_records.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_streaming_gbk_1 +--influx_measurement=python_streaming_gbk_1 +--input_options=''{\\"num_records\\":200000000,\\"key_size\\":1,\\"value_size\\":9,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--fanout=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--streaming +--experiments=use_runner_v2 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..8e38713cc66de --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_streaming_gbk_4 +--influx_measurement=python_streaming_gbk_4 +--input_options=''{\\"num_records\\":5000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--fanout=4 +--num_workers=16 +--autoscaling_algorithm=NONE +--streaming +--experiments=use_runner_v2 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..35508480662c3 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_streaming_gbk_5 +--influx_measurement=python_streaming_gbk_5 +--input_options=''{\\"num_records\\":2500000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--fanout=8 +--num_workers=16 +--autoscaling_algorithm=NONE +--streaming +--experiments=use_runner_v2 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_2GB_of_100B_records.txt b/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_2GB_of_100B_records.txt new file mode 100644 index 0000000000000..4cb5bfb0d9885 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_2GB_of_100B_records.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_batch_GBK_2 +--influx_measurement=python_batch_gbk_2 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90}'' +--iterations=1 +--fanout=4 +--parallelism=5 +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_2GB_of_10B_records.txt b/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_2GB_of_10B_records.txt new file mode 100644 index 0000000000000..2427e21cde454 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_2GB_of_10B_records.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_batch_GBK_1 +--influx_measurement=python_batch_gbk_1 +--input_options=''{\\"num_records\\":200000000,\\"key_size\\":1,\\"value_size\\":9}'' +--iterations=1 +--fanout=1 +--parallelism=5 +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..bf9085141eab8 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_batch_GBK_4 +--influx_measurement=python_batch_gbk_4 +--input_options=''{\\"num_records\\":5000000,\\"key_size\\":10,\\"value_size\\":90}'' +--iterations=1 +--fanout=4 +--parallelism=16 +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt new file mode 100644 index 0000000000000..a59f873eb775e --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_batch_GBK_5 +--influx_measurement=python_batch_gbk_5 +--input_options=''{\\"num_records\\":2500000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--fanout=8 +--parallelism=16 +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt new file mode 100644 index 0000000000000..0e5d00b961519 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_batch_GBK_6 +--influx_measurement=python_batch_gbk_6 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":200,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--iterations=4 +--fanout=1 +--parallelism=5 +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_10kB_values.txt new file mode 100644 index 0000000000000..fb5a3db9e6bb3 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_10kB_values.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_gbk_6 +--influx_measurement=python_batch_gbk_6 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":200,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--iterations=4 +--fanout=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_2MB_values.txt new file mode 100644 index 0000000000000..b4b46682869ae --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_2MB_values.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_gbk_7 +--influx_measurement=python_batch_gbk_7 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":10,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--iterations=4 +--fanout=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_10kB_values.txt new file mode 100644 index 0000000000000..6cb1e68aeafc5 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_10kB_values.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_streaming_gbk_6 +--influx_measurement=python_streaming_gbk_6 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":200,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--iterations=4 +--fanout=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--streaming +--experiments=use_runner_v2 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_2MB_values.txt new file mode 100644 index 0000000000000..712749090aec7 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_2MB_values.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_streaming_gbk_7 +--influx_measurement=python_streaming_gbk_7 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":10,\\"hot_key_fraction\\":1,\\"algorithm\\":\\"lcg\\"}'' +--iterations=4 +--fanout=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--streaming +--experiments=use_runner_v2 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_100_Counters.txt b/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_100_Counters.txt new file mode 100644 index 0000000000000..a5bb7979b86b4 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_100_Counters.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_pardo_4 +--influx_measurement=python_batch_pardo_4 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--number_of_counter_operations=100 +--number_of_counters=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_10_Counters.txt b/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_10_Counters.txt new file mode 100644 index 0000000000000..7e35ef74dfa1e --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_10_Counters.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_pardo_3 +--influx_measurement=python_batch_pardo_3 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--number_of_counter_operations=10 +--number_of_counters=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_10_Iterations.txt b/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_10_Iterations.txt new file mode 100644 index 0000000000000..734360397c9b4 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_10_Iterations.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_pardo_1 +--influx_measurement=python_batch_pardo_1 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=10 +--number_of_counter_operations=0 +--number_of_counters=0 +--num_workers=5 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_200_Iterations.txt b/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_200_Iterations.txt new file mode 100644 index 0000000000000..825fee427a31a --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_200_Iterations.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_pardo_2 +--influx_measurement=python_batch_pardo_2 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=200 +--number_of_counter_operations=0 +--number_of_counters=0 +--num_workers=5 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_100_Counters.txt b/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_100_Counters.txt new file mode 100644 index 0000000000000..71fc818d0e070 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_100_Counters.txt @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_streaming_pardo_4 +--influx_measurement=python_streaming_pardo_4 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--number_of_counter_operations=100 +--number_of_counters=1 +--num_****s=5 +--autoscaling_algorithm=NONE +--streaming +--experiments=use_runner_v2 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_10_Counters.txt b/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_10_Counters.txt new file mode 100644 index 0000000000000..fe48dedfa8373 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_10_Counters.txt @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_streaming_pardo_3 +--influx_measurement=python_streaming_pardo_3 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--number_of_counter_operations=10 +--number_of_counters=1 +--num_workers=5 +--autoscaling_algorithm=NONE +--streaming +--experiments=use_runner_v2 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_10_Iterations.txt b/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_10_Iterations.txt new file mode 100644 index 0000000000000..84d2cdb7a85bd --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_10_Iterations.txt @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_streaming_pardo_1 +--influx_measurement=python_streaming_pardo_1 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=10 +--number_of_counter_operations=0 +--number_of_counters=0 +--num_workers=5 +--autoscaling_algorithm=NONE +--streaming +--experiments=use_runner_v2 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_200_Iterations.txt b/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_200_Iterations.txt new file mode 100644 index 0000000000000..02636335cb372 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_200_Iterations.txt @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_streaming_pardo_2 +--influx_measurement=python_streaming_pardo_2 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=200 +--number_of_counter_operations=0 +--number_of_counters=0 +--num_workers=5 +--autoscaling_algorithm=NONE +--streaming +--experiments=use_runner_v2 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_10_Counters.txt b/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_10_Counters.txt new file mode 100644 index 0000000000000..4d8bda8ac2f8c --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_10_Counters.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_batch_pardo_4 +--influx_measurement=python_batch_pardo_4 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--number_of_counter_operations=100 +--number_of_counters=1 +--parallelism=5 +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_10_Iterations.txt b/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_10_Iterations.txt new file mode 100644 index 0000000000000..e84cee2f50cf2 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_10_Iterations.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_batch_pardo_1 +--influx_measurement=python_batch_pardo_1 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=10 +--number_of_counter_operations=0 +--number_of_counters=0 +--parallelism=5 +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_200_Iterations.txt b/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_200_Iterations.txt new file mode 100644 index 0000000000000..4d8bda8ac2f8c --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_200_Iterations.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_batch_pardo_4 +--influx_measurement=python_batch_pardo_4 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--number_of_counter_operations=100 +--number_of_counters=1 +--parallelism=5 +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_100_Counters.txt b/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_100_Counters.txt new file mode 100644 index 0000000000000..b17e2cecc2c80 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_100_Counters.txt @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_streaming_pardo_4 +--influx_measurement=python_streaming_pardo_4 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--number_of_counter_operations=100 +--number_of_counters=1 +--parallelism=5 +--streaming +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--use_stateful_load_generator +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_10_Counters.txt b/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_10_Counters.txt new file mode 100644 index 0000000000000..957bc6c086d82 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_10_Counters.txt @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_streaming_pardo_3 +--influx_measurement=python_streaming_pardo_3 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=1 +--number_of_counter_operations=10 +--number_of_counters=1 +--parallelism=5 +--streaming +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--use_stateful_load_generator +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_10_Iterations.txt b/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_10_Iterations.txt new file mode 100644 index 0000000000000..baa34ec455b50 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_10_Iterations.txt @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_streaming_pardo_5 +--influx_measurement=python_streaming_pardo_1 +--input_options=''{\\"num_records\\":2000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=10 +--number_of_counter_operations=0 +--number_of_counters=0 +--parallelism=5 +--streaming +--stateful +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--use_stateful_load_generator +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_200_Iterations.txt b/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_200_Iterations.txt new file mode 100644 index 0000000000000..44483a6e51ccf --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_200_Iterations.txt @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_streaming_pardo_2 +--influx_measurement=python_streaming_pardo_2 +--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=200 +--number_of_counter_operations=0 +--number_of_counters=0 +--parallelism=5 +--streaming +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--use_stateful_load_generator +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_5_Iterations.txt b/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_5_Iterations.txt new file mode 100644 index 0000000000000..571b33fb7a490 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_5_Iterations.txt @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_flink_streaming_pardo_6 +--influx_measurement=python_streaming_pardo_6 +--input_options=''{\\"num_records\\":2000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--iterations=5 +--number_of_counter_operations=10 +--number_of_counters=3 +--parallelism=5 +--streaming +--stateful +--checkpointing_interval=10000 +--report_checkpoint_duration=python_flink_streaming_pardo_6 +--shutdown_sources_after_idle_ms=300000 +--job_endpoint=localhost:8099 +--environment_type=DOCKER +--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.8_sdk:latest +--use_stateful_load_generator +--runner=PortableRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1000window_first_iterable.txt b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1000window_first_iterable.txt new file mode 100644 index 0000000000000..204c07bc16a28 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1000window_first_iterable.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_sideinput_9 +--influx_measurement=python_batch_sideinput_9 +--num_workers=10 +--autoscaling_algorithm=NONE +--experiments=use_runner_v2 +--input_options=''{\\"num_records\\":10000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' +--side_input_type=iter +--access_percentage=1 +--window_count=1000 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1000window_iterable.txt b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1000window_iterable.txt new file mode 100644 index 0000000000000..1ae64bb4a3697 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1000window_iterable.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_sideinput_10 +--influx_measurement=python_batch_sideinput_10 +--num_workers=10 +--autoscaling_algorithm=NONE +--experiments=use_runner_v2 +--input_options=''{\\"num_records\\":10000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' +--side_input_type=iter +--window_count=1000 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1window_first_iterable.txt b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1window_first_iterable.txt new file mode 100644 index 0000000000000..0759517d9c2ed --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1window_first_iterable.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_sideinput_3 +--influx_measurement=python_batch_sideinput_3 +--num_workers=10 +--autoscaling_algorithm=NONE +--experiments=use_runner_v2 +--input_options=''{\\"num_records\\":10000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' +--side_input_type=iter +--access_percentage=1 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1window_iterable.txt b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1window_iterable.txt new file mode 100644 index 0000000000000..c555c0d32d4c3 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_10gb_1window_iterable.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_sideinput_4 +--influx_measurement=python_batch_sideinput_4 +--num_workers=10 +--autoscaling_algorithm=NONE +--experiments=use_runner_v2 +--input_options=''{\\"num_records\\":10000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' +--side_input_type=iter +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1000window_1key_percent_dict.txt b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1000window_1key_percent_dict.txt new file mode 100644 index 0000000000000..4b3cee817f430 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1000window_1key_percent_dict.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_sideinput_7 +--influx_measurement=python_batch_sideinput_7 +--num_workers=10 +--autoscaling_algorithm=NONE +--experiments=use_runner_v2 +--input_options=''{\\"num_records\\":1000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' +--side_input_type=dict +--access_percentage=1 +--window_count=1000 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1000window_99key_percent_dict.txt b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1000window_99key_percent_dict.txt new file mode 100644 index 0000000000000..00ba6feef50cc --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1000window_99key_percent_dict.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_sideinput_8 +--influx_measurement=python_batch_sideinput_8 +--num_workers=10 +--autoscaling_algorithm=NONE +--experiments=use_runner_v2 +--input_options=''{\\"num_records\\":1000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' +--side_input_type=dict +--access_percentage=99 +--window_count=1000 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_1key_percent_dict.txt b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_1key_percent_dict.txt new file mode 100644 index 0000000000000..07e4a5ecba621 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_1key_percent_dict.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_sideinput_1 +--influx_measurement=python_batch_sideinput_1 +--num_workers=10 +--autoscaling_algorithm=NONE +--experiments=use_runner_v2 +--input_options=''{\\"num_records\\":1000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' +--side_input_type=dict +--access_percentage=1 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_99key_percent_dict.txt b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_99key_percent_dict.txt new file mode 100644 index 0000000000000..b565598c1a167 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_99key_percent_dict.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_sideinput_2 +--influx_measurement=python_batch_sideinput_2 +--num_workers=10 +--autoscaling_algorithm=NONE +--experiments=use_runner_v2 +--input_options=''{\\"num_records\\":1000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' +--side_input_type=dict +--access_percentage=99 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_first_list.txt b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_first_list.txt new file mode 100644 index 0000000000000..de5b02198177b --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_first_list.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_sideinput_5 +--influx_measurement=python_batch_sideinput_5 +--num_workers=10 +--autoscaling_algorithm=NONE +--experiments=use_runner_v2 +--input_options=''{\\"num_records\\":1000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' +--side_input_type=list +--access_percentage=1 +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_list.txt b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_list.txt new file mode 100644 index 0000000000000..078ce2cc74fec --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_SideInput_Dataflow_Batch_1gb_1window_list.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test +--metrics_table=python_dataflow_batch_sideinput_6 +--influx_measurement=python_batch_sideinput_6 +--num_workers=10 +--autoscaling_algorithm=NONE +--experiments=use_runner_v2 +--input_options=''{\\"num_records\\":1000000,\\"key_size\\":100,\\"value_size\\":900,\\"algorithm\\":\\"lcg\\"}'' +--side_input_type=list +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_Smoke_GroupByKey_Dataflow.txt b/.github/workflows/load-tests-job-configs/python_Smoke_GroupByKey_Dataflow.txt new file mode 100644 index 0000000000000..9a069df2bd1c5 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_Smoke_GroupByKey_Dataflow.txt @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test_SMOKE +--metrics_table=python_dataflow_gbk +--input_options=''{\\"num_records\\":100000,\\"key_size\\":1,\\"value_size\\":1}'' +--max_num_workers=1 \ No newline at end of file diff --git a/.github/workflows/load-tests-job-configs/python_Smoke_GroupByKey_Direct.txt b/.github/workflows/load-tests-job-configs/python_Smoke_GroupByKey_Direct.txt new file mode 100644 index 0000000000000..7490675e43831 --- /dev/null +++ b/.github/workflows/load-tests-job-configs/python_Smoke_GroupByKey_Direct.txt @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=load_test_SMOKE +--metrics_table=python_direct_gbk +--input_options=''{\\"num_records\\":100000,\\"key_size\\":1,\\"value_size\\":1}'' +--max_num_workers=1 \ No newline at end of file diff --git a/.github/workflows/local_env_tests.yml b/.github/workflows/local_env_tests.yml index a9c9741194089..7dbff4feeb45d 100644 --- a/.github/workflows/local_env_tests.yml +++ b/.github/workflows/local_env_tests.yml @@ -31,19 +31,24 @@ on: # This allows a subsequently queued workflow run to interrupt previous runs concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login}}' cancel-in-progress: true +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + jobs: run_local_env_install_ubuntu: timeout-minutes: 25 name: "Ubuntu run local environment shell script" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' - uses: actions/setup-python@v4 with: python-version: '3.8' @@ -58,10 +63,10 @@ jobs: name: "Mac run local environment shell script" runs-on: macos-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' - uses: actions/setup-python@v4 with: python-version: '3.8' diff --git a/.github/workflows/performance-tests-job-configs/JDBC.txt b/.github/workflows/performance-tests-job-configs/JDBC.txt new file mode 100644 index 0000000000000..74fd85145c591 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/JDBC.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--tempRoot=gs://temp-storage-for-perf-tests +--runner=DataflowRunner +--numberOfRecords=5000000 +--bigQueryDataset=beam_performance +--bigQueryTable=jdbcioit_results +--influxMeasurement=jdbcioit_results +--postgresUsername=postgres +--postgresPassword=uuinkks +--postgresDatabaseName=postgres +--postgresSsl=false +--postgresPort=5432 +--autoscalingAlgorithm=NONE +--numWorkers=5 \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/SQLBigQueryIO_Batch_Java.txt b/.github/workflows/performance-tests-job-configs/SQLBigQueryIO_Batch_Java.txt new file mode 100644 index 0000000000000..2411e62886806 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/SQLBigQueryIO_Batch_Java.txt @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--tempLocation=gs://temp-storage-for-perf-tests/loadtests +--tempRoot=gs://temp-storage-for-perf-tests/loadtests +--metricsBigQueryDataset=beam_performance +--metricsBigQueryTable=sql_bqio_read_java_batch +--runner=DataflowRunner +--maxNumWorkers=5 +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/TFRecordIOIT_HDFS.txt b/.github/workflows/performance-tests-job-configs/TFRecordIOIT_HDFS.txt new file mode 100644 index 0000000000000..608e11c7847af --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/TFRecordIOIT_HDFS.txt @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--runner=DataflowRunner +--tempRoot=gs://temp-storage-for-perf-tests +--numberOfRecords=18000000 +--expectedHash=543104423f8b6eb097acb9f111c19fe4 +--datasetSize=1019380000 +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/biqQueryIO_Read_Python.txt b/.github/workflows/performance-tests-job-configs/biqQueryIO_Read_Python.txt new file mode 100644 index 0000000000000..12428da8d0916 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/biqQueryIO_Read_Python.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--input_dataset=beam_performance +--input_table=bqio_read_10GB +--publish_to_big_query=true +--metrics_dataset=beam_performance +--metrics_table=bqio_read_10GB_results +--influx_measurement=python_bqio_read_10GB_results +--input_options=''{\\"num_records\\":10485760,\\"key_size\\":1,\\"value_size\\":1024,\\"algorithm\\":\\"lcg\\"}'' +--num_workers=5 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/biqQueryIO_Write_Python_Batch.txt b/.github/workflows/performance-tests-job-configs/biqQueryIO_Write_Python_Batch.txt new file mode 100644 index 0000000000000..9edec48d3d23d --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/biqQueryIO_Write_Python_Batch.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--output_dataset=beam_performance +--output_table=bqio_write_10GB +--publish_to_big_query=true +--metrics_dataset=beam_performance +--metrics_table=bqio_write_10GB_results +--influx_measurement=python_bqio_write_10GB_results +--input_options=''{\\"num_records\\":10485760,\\"key_size\\":1,\\"value_size\\":1024,\\"algorithm\\":\\"lcg\\"}'' +--num_workers=5 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/cdap.txt b/.github/workflows/performance-tests-job-configs/cdap.txt new file mode 100644 index 0000000000000..60c98a1065d55 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/cdap.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--tempRoot=gs://temp-storage-for-perf-tests +--runner=DataflowRunner +--numberOfRecords=5000000 +--bigQueryDataset=beam_performance +--bigQueryTable=cdapioit_results +--influxMeasurement=cdapioit_results +--postgresUsername=postgres +--postgresPassword=uuinkks +--postgresDatabaseName=postgres +--postgresSsl=false +--postgresPort=5432 +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/config_AvroIOIT.txt b/.github/workflows/performance-tests-job-configs/config_AvroIOIT.txt new file mode 100644 index 0000000000000..12ae78de6cc0a --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/config_AvroIOIT.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--runner=DataflowRunner +--tempRoot=gs://temp-storage-for-perf-tests +--bigQueryDataset=beam_performance +--bigQueryTable=avroioit_results +--influxMeasurement=avroioit_results +--numberOfRecords=225000000 +--expectedHash=2f9f5ca33ea464b25109c0297eb6aecb +--datasetSize=1089730000 +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/config_AvroIOIT_HDFS.txt b/.github/workflows/performance-tests-job-configs/config_AvroIOIT_HDFS.txt new file mode 100644 index 0000000000000..6d27c85393c32 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/config_AvroIOIT_HDFS.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--runner=DataflowRunner +--tempRoot=gs://temp-storage-for-perf-tests +--bigQueryDataset=beam_performance +--bigQueryTable=avroioit_hdfs_results +--influxMeasurement=avroioit_hdfs_results +--numberOfRecords=225000000 +--expectedHash=2f9f5ca33ea464b25109c0297eb6aecb +--datasetSize=1089730000 +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/config_BigQueryIO_Batch_Java_Avro.txt b/.github/workflows/performance-tests-job-configs/config_BigQueryIO_Batch_Java_Avro.txt new file mode 100644 index 0000000000000..5e7e53821231c --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/config_BigQueryIO_Batch_Java_Avro.txt @@ -0,0 +1,38 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### +'["--tempLocation=gs://temp-storage-for-perf-tests/loadtests", +"--project=apache-beam-testing", +"--tempRoot=gs://temp-storage-for-perf-tests/loadtests", +"--writeMethod=FILE_LOADS", +"--writeFormat=AVRO", +"--testBigQueryDataset=beam_performance", +"--testBigQueryTable=bqio_write_10GB_java_avro_", +"--metricsBigQueryDataset=beam_performance", +"--metricsBigQueryTable=bqio_10GB_results_java_batch_avro", +"--influxMeasurement=bqio_10GB_results_java_batch_avro", +"--sourceOptions={ +\"numRecords\":\"10485760\", +\"keySizeBytes\":\"1\", +\"valueSizeBytes\":\"1024\" +}", +"--runner=DataflowRunner", +"--maxNumWorkers=5", +"--numWorkers=5", +"--autoscalingAlgorithm=NONE", +"--influxDatabase=beam_test_metrics", +"--influxHost=http://10.128.0.96:8086"]' \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/config_BigQueryIO_Batch_Java_Json.txt b/.github/workflows/performance-tests-job-configs/config_BigQueryIO_Batch_Java_Json.txt new file mode 100644 index 0000000000000..7bd9c30ae7380 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/config_BigQueryIO_Batch_Java_Json.txt @@ -0,0 +1,38 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### +'["--tempLocation=gs://temp-storage-for-perf-tests/loadtests", +"--project=apache-beam-testing", +"--tempRoot=gs://temp-storage-for-perf-tests/loadtests", +"--writeMethod=FILE_LOADS", +"--writeFormat=JSON", +"--testBigQueryDataset=beam_performance", +"--testBigQueryTable=bqio_write_10GB_java_json_", +"--metricsBigQueryDataset=beam_performance", +"--metricsBigQueryTable=bqio_10GB_results_java_batch_json", +"--influxMeasurement=bqio_10GB_results_java_batch_json", +"--sourceOptions={ +\"numRecords\":\"10485760\", +\"keySizeBytes\":\"1\", +\"valueSizeBytes\":\"1024\" +}", +"--runner=DataflowRunner", +"--maxNumWorkers=5", +"--numWorkers=5", +"--autoscalingAlgorithm=NONE", +"--influxDatabase=beam_test_metrics", +"--influxHost=http://10.128.0.96:8086"]' \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/config_BigQueryIO_Streaming_Java.txt b/.github/workflows/performance-tests-job-configs/config_BigQueryIO_Streaming_Java.txt new file mode 100644 index 0000000000000..8bddea5fcb8bd --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/config_BigQueryIO_Streaming_Java.txt @@ -0,0 +1,39 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### +'["--tempLocation=gs://temp-storage-for-perf-tests/loadtests", +"--project=apache-beam-testing", +"--tempRoot=gs://temp-storage-for-perf-tests/loadtests", +"--writeMethod=STREAMING_INSERTS", +"--writeFormat=JSON", +"--pipelineTimeout=1200", +"--testBigQueryDataset=beam_performance", +"--testBigQueryTable=bqio_write_10GB_java_stream_", +"--metricsBigQueryDataset=beam_performance", +"--metricsBigQueryTable=bqio_10GB_results_java_stream", +"--influxMeasurement=bqio_10GB_results_java_stream", +"--sourceOptions={ +\"numRecords\":\"10485760\", +\"keySizeBytes\":\"1\", +\"valueSizeBytes\":\"1024\" +}", +"--runner=DataflowRunner", +"--maxNumWorkers=5", +"--numWorkers=5", +"--autoscalingAlgorithm=NONE", +"--influxDatabase=beam_test_metrics", +"--influxHost=http://10.128.0.96:8086"]' \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/config_Compressed_TextIOIT.txt b/.github/workflows/performance-tests-job-configs/config_Compressed_TextIOIT.txt new file mode 100644 index 0000000000000..137eb56354a38 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/config_Compressed_TextIOIT.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--runner=DataflowRunner +--tempRoot=gs://temp-storage-for-perf-tests +--bigQueryDataset=beam_performance +--bigQueryTable=compressed_textioit_results +--influxMeasurement=compressed_textioit_results +--numberOfRecords=450000000 +--expectedHash=8a3de973354abc6fba621c6797cc0f06 +--datasetSize=1097840000 +--compressionType=GZIP +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/config_Compressed_TextIOIT_HDFS.txt b/.github/workflows/performance-tests-job-configs/config_Compressed_TextIOIT_HDFS.txt new file mode 100644 index 0000000000000..9ad5137cefec8 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/config_Compressed_TextIOIT_HDFS.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--runner=DataflowRunner +--tempRoot=gs://temp-storage-for-perf-tests +--bigQueryDataset=beam_performance +--bigQueryTable=compressed_textioit_hdfs_results +--influxMeasurement=compressed_textioit_hdfs_results +--numberOfRecords=450000000 +--expectedHash=8a3de973354abc6fba621c6797cc0f06 +--datasetSize=1097840000 +--compressionType=GZIP +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/config_ManyFiles_TextIOIT.txt b/.github/workflows/performance-tests-job-configs/config_ManyFiles_TextIOIT.txt new file mode 100644 index 0000000000000..dcb6e31526fff --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/config_ManyFiles_TextIOIT.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--runner=DataflowRunner +--tempRoot=gs://temp-storage-for-perf-tests +--bigQueryDataset=beam_performance +--bigQueryTable=many_files_textioit_results +--influxMeasurement=many_files_textioit_results +--reportGcsPerformanceMetrics=true +--gcsPerformanceMetrics=true +--numberOfRecords=25000000 +--expectedHash=f8453256ccf861e8a312c125dfe0e436 +--datasetSize=1062290000 +--numberOfShards=1000 +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/config_ManyFiles_TextIOIT_HDFS.txt b/.github/workflows/performance-tests-job-configs/config_ManyFiles_TextIOIT_HDFS.txt new file mode 100644 index 0000000000000..f01a4f4883128 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/config_ManyFiles_TextIOIT_HDFS.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--runner=DataflowRunner +--tempRoot=gs://temp-storage-for-perf-tests +--bigQueryDataset=beam_performance +--bigQueryTable=many_files_textioit_hdfs_results +--influxMeasurement=many_files_textioit_hdfs_results +--reportGcsPerformanceMetrics=true +--gcsPerformanceMetrics=true +--numberOfRecords=25000000 +--expectedHash=f8453256ccf861e8a312c125dfe0e436 +--datasetSize=1062290000 +--numberOfShards=1000 +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/config_ParquetIOIT.txt b/.github/workflows/performance-tests-job-configs/config_ParquetIOIT.txt new file mode 100644 index 0000000000000..10dc0eba73da2 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/config_ParquetIOIT.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--runner=DataflowRunner +--tempRoot=gs://temp-storage-for-perf-tests +--bigQueryDataset=beam_performance +--bigQueryTable=parquetioit_results +--influxMeasurement=parquetioit_results +--numberOfRecords=225000000 +--expectedHash=2f9f5ca33ea464b25109c0297eb6aecb +--datasetSize=1087370000 +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/config_ParquetIOIT_HDFS.txt b/.github/workflows/performance-tests-job-configs/config_ParquetIOIT_HDFS.txt new file mode 100644 index 0000000000000..ecb725cc12f70 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/config_ParquetIOIT_HDFS.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--runner=DataflowRunner +--tempRoot=gs://temp-storage-for-perf-tests +--bigQueryDataset=beam_performance +--bigQueryTable=parquetioit_hdfs_results +--influxMeasurement=parquetioit_hdfs_results +--numberOfRecords=225000000 +--expectedHash=2f9f5ca33ea464b25109c0297eb6aecb +--datasetSize=1087370000 +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/config_TFRecordIOIT.txt b/.github/workflows/performance-tests-job-configs/config_TFRecordIOIT.txt new file mode 100644 index 0000000000000..bb263609019ad --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/config_TFRecordIOIT.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--runner=DataflowRunner +--tempRoot=gs://temp-storage-for-perf-tests +--bigQueryDataset=beam_performance +--bigQueryTable=tfrecordioit_results +--influxMeasurement=tfrecordioit_results +--numberOfRecords=18000000 +--expectedHash=543104423f8b6eb097acb9f111c19fe4 +--datasetSize=1019380000 +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/config_XmlIOIT.txt b/.github/workflows/performance-tests-job-configs/config_XmlIOIT.txt new file mode 100644 index 0000000000000..5847dc34aa58c --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/config_XmlIOIT.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--runner=DataflowRunner +--tempRoot=gs://temp-storage-for-perf-tests +--bigQueryDataset=beam_performance +--bigQueryTable=xmlioit_results +--influxMeasurement=xmlioit_results +--numberOfRecords=12000000 +--expectedHash=b3b717e7df8f4878301b20f314512fb3 +--datasetSize=1076590000 +--charset=UTF-8 +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/config_XmlIOIT_HDFS.txt b/.github/workflows/performance-tests-job-configs/config_XmlIOIT_HDFS.txt new file mode 100644 index 0000000000000..0de45091d484b --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/config_XmlIOIT_HDFS.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--runner=DataflowRunner +--tempRoot=gs://temp-storage-for-perf-tests +--bigQueryDataset=beam_performance +--bigQueryTable=xmlioit_hdfs_results +--influxMeasurement=xmlioit_hdfs_results +--numberOfRecords=12000000 +--expectedHash=b3b717e7df8f4878301b20f314512fb3 +--datasetSize=1076590000 +--charset=UTF-8 +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/hadoopFormat.txt b/.github/workflows/performance-tests-job-configs/hadoopFormat.txt new file mode 100644 index 0000000000000..712f29e3d52ca --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/hadoopFormat.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--tempRoot=gs://temp-storage-for-perf-tests +--runner=DataflowRunner +--numberOfRecords=5000000 +--bigQueryDataset=beam_performance +--bigQueryTable=hadoopformatioit_results +--influxMeasurement=hadoopformatioit_results +--postgresUsername=postgres +--postgresPassword=uuinkks +--postgresDatabaseName=postgres +--postgresSsl=false +--postgresPort=5432 +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/kafka_IO_Batch.txt b/.github/workflows/performance-tests-job-configs/kafka_IO_Batch.txt new file mode 100644 index 0000000000000..bad323daacd51 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/kafka_IO_Batch.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--tempRoot=gs://temp-storage-for-perf-tests +--runner=DataflowRunner +--sourceOptions={\"numRecords\":\"100000000\",\"keySizeBytes\":\"10\",\"valueSizeBytes\":\"90\"} +--bigQueryDataset=beam_performance +--bigQueryTable=kafkaioit_results +--influxMeasurement=kafkaioit_results +--kafkaTopic=beam-batch +--readTimeout=1800 +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/kafka_IO_Streaming.txt b/.github/workflows/performance-tests-job-configs/kafka_IO_Streaming.txt new file mode 100644 index 0000000000000..c44ce0180c2d6 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/kafka_IO_Streaming.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--tempRoot=gs://temp-storage-for-perf-tests +--runner=DataflowRunner +--sourceOptions={\"numRecords\":\"100000000\",\"keySizeBytes\":\"10\",\"valueSizeBytes\":\"90\"} +--bigQueryDataset=beam_performance +--bigQueryTable=kafkaioit_results_runner_v2 +--influxMeasurement=kafkaioit_results_runner_v2 +--kafkaTopic=beam-sdf +--readTimeout=1500 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--experiments=use_runner_v2,use_unified_worker \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/mongoDBIO_IT.txt b/.github/workflows/performance-tests-job-configs/mongoDBIO_IT.txt new file mode 100644 index 0000000000000..67a60f6407f9f --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/mongoDBIO_IT.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--tempRoot=gs://temp-storage-for-perf-tests +--runner=DataflowRunner +--numberOfRecords=10000000 +--bigQueryDataset=beam_performance +--bigQueryTable=mongodbioit_results +--influxMeasurement=mongodbioit_results +--mongoDBDatabaseName=beam +--mongoDBPort=27017 +--mongoDBUsername=root +--mongoDBPassword=uuinkkS +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/pubsubIOIT_Python_Streaming.txt b/.github/workflows/performance-tests-job-configs/pubsubIOIT_Python_Streaming.txt new file mode 100644 index 0000000000000..257fc196de7e9 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/pubsubIOIT_Python_Streaming.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--publish_to_big_query=true +--metrics_dataset=beam_performance +--metrics_table=psio_io_2GB_results +--influx_measurement=python_psio_2GB_results +--input_options=''{\\"num_records\\":2097152,\\"key_size\\":1,\\"value_size\\":1024,\\"algorithm\\":\\"lcg\\"}'' +--num_workers=5 +--autoscaling_algorithm=NONE +--pubsub_namespace_prefix=pubsub_io_performance_ +--wait_until_finish_duration=720000 +--runner=TestDataflowRunner \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/spannerIO_Read_2GB_Python.txt b/.github/workflows/performance-tests-job-configs/spannerIO_Read_2GB_Python.txt new file mode 100644 index 0000000000000..8ce05ac6a4f33 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/spannerIO_Read_2GB_Python.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-west1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--spanner_instance=beam-test +--spanner_database=pyspanner_read_2gb +--publish_to_big_query=true +--metrics_dataset=beam_performance +--metrics_table=pyspanner_read_2GB_results +--influx_measurement=python_spannerio_read_2GB_results +--input_options=''{\\"num_records\\":2097152,\\"key_size\\":1,\\"value_size\\":1024,\\"algorithm\\":\\"lcg\\"}'' +--num_workers=5 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/spannerIO_Write_2GB_Python.txt b/.github/workflows/performance-tests-job-configs/spannerIO_Write_2GB_Python.txt new file mode 100644 index 0000000000000..de723a9509e1e --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/spannerIO_Write_2GB_Python.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-west1 +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--spanner_instance=beam-test +--spanner_database=pyspanner_write_2gb +--publish_to_big_query=true +--metrics_dataset=beam_performance +--metrics_table=pyspanner_write_2GB_results +--influx_measurement=python_spanner_write_2GB_results +--input_options=''{\\"num_records\\":2097152,\\"key_size\\":1,\\"value_size\\":1024,\\"algorithm\\":\\"lcg\\"}'' +--num_workers=5 +--autoscaling_algorithm=NONE +--runner=DataflowRunner \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/sparkReceiver_IO.txt b/.github/workflows/performance-tests-job-configs/sparkReceiver_IO.txt new file mode 100644 index 0000000000000..12d2bde673273 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/sparkReceiver_IO.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--tempRoot=gs://temp-storage-for-perf-tests +--runner=DataflowRunner +--sourceOptions=''{\"numRecords\":\"5000000\",\"keySizeBytes\":\"1\",\"valueSizeBytes\":\"90\"}'' +--bigQueryDataset=beam_performance +--bigQueryTable=sparkreceiverioit_results +--influxMeasurement=sparkreceiverioit_results +--streamName=rabbitMqTestStream +--readTimeout=1800 +--numWorkers=1 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/textIOIT.txt b/.github/workflows/performance-tests-job-configs/textIOIT.txt new file mode 100644 index 0000000000000..dab9f5b082bd2 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/textIOIT.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--runner=DataflowRunner +--tempRoot=gs://temp-storage-for-perf-tests +--bigQueryDataset=beam_performance +--bigQueryTable=textioit_results +--influxMeasurement=textioit_results +--numberOfRecords=25000000 +--expectedHash=f8453256ccf861e8a312c125dfe0e436 +--datasetSize=1062290000 +--numWorkers=5 +--autoscalingAlgorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/textIOIT_HDFS.txt b/.github/workflows/performance-tests-job-configs/textIOIT_HDFS.txt new file mode 100644 index 0000000000000..0344ed1a57d23 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/textIOIT_HDFS.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--bigQueryDataset=beam_performance +--bigQueryTable=textioit_hdfs_results +--influxMeasurement=textioit_hdfs_results +--numberOfRecords=25000000 +--expectedHash=f8453256ccf861e8a312c125dfe0e436 +--datasetSize=1062290000 +--numWorkers=5 +--autoscalingAlgorithm=NONE +--runner=DataflowRunner +--tempRoot=gs://temp-storage-for-perf-tests \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/textIOIT_Python.txt b/.github/workflows/performance-tests-job-configs/textIOIT_Python.txt new file mode 100644 index 0000000000000..58434aa618475 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/textIOIT_Python.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--runner=DataflowRunner +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/ +--publish_to_big_query=true +--metrics_dataset=beam_performance +--metrics_table=python_textio_1GB_results +--influx_measurement=python_textio_1GB_results +--test_class=TextIOPerfTest +--input_options=''{\\"num_records\\":25000000,\\"key_size\\":9,\\"value_size\\":21,\\"algorithm\\":\\"lcg\\"}'' +--dataset_size=1050000000 +--num_workers=5 +--autoscaling_algorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/wordCountIT_Python.txt b/.github/workflows/performance-tests-job-configs/wordCountIT_Python.txt new file mode 100644 index 0000000000000..9b9abeeb092a0 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/wordCountIT_Python.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--staging_location=gs://temp-storage-for-end-to-end-tests/staging-it +--temp_location=gs://temp-storage-for-end-to-end-tests/temp-it +--runner=TestDataflowRunner +--publish_to_big_query=true +--metrics_dataset=beam_performance +--input=gs://apache-beam-samples/input_small_files/ascii_sort_1MB_input.0000* +--output=gs://temp-storage-for-end-to-end-tests/py-it-cloud/output +--expect_checksum=ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710 +--num_workers=10 +--autoscaling_algorithm=NONE \ No newline at end of file diff --git a/.github/workflows/performance-tests-job-configs/xlang_KafkaIO_Python.txt b/.github/workflows/performance-tests-job-configs/xlang_KafkaIO_Python.txt new file mode 100644 index 0000000000000..81702220d3b19 --- /dev/null +++ b/.github/workflows/performance-tests-job-configs/xlang_KafkaIO_Python.txt @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--runner=DataflowRunner +--region=us-central1 +--temp_location=gs://temp-storage-for-perf-tests/ +--sdk_harness_container_image_overrides=.*java.*,gcr.io/apache-beam-testing/beam-sdk/beam_java8_sdk:latest +--publish_to_big_query=true +--metrics_dataset=beam_performance +--metrics_table=python_kafkaio_results +--influx_measurement=python_kafkaio_results +--test_class=KafkaIOPerfTest +--input_options=''{\\"num_records\\":100000000,\\"key_size\\":10,\\"value_size\\":90,\\"algorithm\\":\\"lcg\\"}'' +--kafka_topic=beam +--read_timeout=1500 +--num_workers=5 +--autoscaling_algorithm=NONE \ No newline at end of file diff --git a/.github/workflows/playground_backend_precommit.yml b/.github/workflows/playground_backend_precommit.yml index d32ba3801fcf6..114ca4aac1cb3 100644 --- a/.github/workflows/playground_backend_precommit.yml +++ b/.github/workflows/playground_backend_precommit.yml @@ -21,6 +21,12 @@ on: paths: - .github/workflows/playground_backend_precommit.yml - playground/backend/** + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + jobs: precommit_check: name: precommit-check @@ -31,20 +37,13 @@ jobs: JAVA_VERSION: '11' steps: - name: Check out the repo - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: '${{ env.PYTHON_VERSION }}' - - uses: actions/setup-java@v3.8.0 + uses: actions/checkout@v4 + + - name: Setup environment + uses: ./.github/actions/setup-environment-action with: - distribution: 'zulu' java-version: '${{ env.JAVA_VERSION }}' - - - name: Setup Gradle - uses: gradle/gradle-build-action@v2 - with: - cache-read-only: false + python-version: '${{ env.PYTHON_VERSION }}' - name: Add GOPATH/bin to PATH run: echo "PATH=$PATH:$(go env GOPATH)/bin" >> $GITHUB_ENV diff --git a/.github/workflows/playground_frontend_test.yml b/.github/workflows/playground_frontend_test.yml index 7d5287ea5a11a..4d70197a11f74 100644 --- a/.github/workflows/playground_frontend_test.yml +++ b/.github/workflows/playground_frontend_test.yml @@ -28,9 +28,14 @@ on: # This allows a subsequently queued workflow run to interrupt previous runs concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login}}' cancel-in-progress: true +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + jobs: playground_frontend_test: name: Playground Frontend Test @@ -40,7 +45,7 @@ jobs: FLUTTER_VERSION: '3.10.4' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: 'Cache Flutter Dependencies' uses: actions/cache@v3 @@ -84,7 +89,7 @@ jobs: working-directory: playground/frontend run: flutter test - - uses: nanasess/setup-chromedriver@v1 + - uses: nanasess/setup-chromedriver@v2 - name: 'Integration tests' run: | diff --git a/.github/workflows/pr-bot-new-prs.yml b/.github/workflows/pr-bot-new-prs.yml index 8ba27fbec3dce..ef825e067b7d9 100644 --- a/.github/workflows/pr-bot-new-prs.yml +++ b/.github/workflows/pr-bot-new-prs.yml @@ -33,7 +33,7 @@ jobs: if: github.repository == 'apache/beam' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Node uses: actions/setup-node@v3 with: diff --git a/.github/workflows/pr-bot-pr-updates.yml b/.github/workflows/pr-bot-pr-updates.yml index d96a11368cb83..f3734e0adcbf7 100644 --- a/.github/workflows/pr-bot-pr-updates.yml +++ b/.github/workflows/pr-bot-pr-updates.yml @@ -18,6 +18,7 @@ on: pull_request_target: types: ["synchronize"] # Synchronize is the action that runs after pushes to the user branch issue_comment: + types: [created] permissions: read-all jobs: process-pr-update: @@ -35,7 +36,7 @@ jobs: steps: # Pin to master so users can't do anything malicious on their own branch and run it here. - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: 'master' - name: Setup Node diff --git a/.github/workflows/pr-bot-prs-needing-attention.yml b/.github/workflows/pr-bot-prs-needing-attention.yml index e96d3983746b8..9dff7c8565a46 100644 --- a/.github/workflows/pr-bot-prs-needing-attention.yml +++ b/.github/workflows/pr-bot-prs-needing-attention.yml @@ -33,7 +33,7 @@ jobs: if: github.repository == 'apache/beam' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Node uses: actions/setup-node@v3 with: diff --git a/.github/workflows/pr-bot-update-reviewers.yml b/.github/workflows/pr-bot-update-reviewers.yml index f3d343b12fb92..b4c41b66f9d63 100644 --- a/.github/workflows/pr-bot-update-reviewers.yml +++ b/.github/workflows/pr-bot-update-reviewers.yml @@ -33,7 +33,7 @@ jobs: if: github.repository == 'apache/beam' runs-on: [self-hosted, ubuntu-20.04] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Node uses: actions/setup-node@v3 with: diff --git a/.github/workflows/publish_github_release_notes.yml b/.github/workflows/publish_github_release_notes.yml index 246ce690f8b10..473e0deef83db 100644 --- a/.github/workflows/publish_github_release_notes.yml +++ b/.github/workflows/publish_github_release_notes.yml @@ -36,7 +36,7 @@ jobs: properties: ${{ steps.test-properties.outputs.properties }} steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -49,7 +49,7 @@ jobs: name: Publish Github Release Notes steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Publish github release notes run: | POST_PATH="website/www/site/content/en/blog/beam-${{env.RELEASE_VERSION}}.md" diff --git a/.github/workflows/python_dependency_tests.yml b/.github/workflows/python_dependency_tests.yml index daf30b4bae9c0..166899df90cbc 100644 --- a/.github/workflows/python_dependency_tests.yml +++ b/.github/workflows/python_dependency_tests.yml @@ -7,11 +7,11 @@ on: branches: ['master', 'release-*'] tags: 'v*' # paths where Beam Python's dependencies are configured. - paths: ['sdks/python/setup.py', 'sdks/python/build-requirements.txt', 'sdks/python/container/base_image_requirements_manual.txt'] + paths: ['sdks/python/setup.py', 'sdks/python/pyproject.toml', 'sdks/python/container/base_image_requirements_manual.txt'] # This allows a subsequently queued workflow run to interrupt previous runs concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login}}' cancel-in-progress: true jobs: @@ -26,7 +26,6 @@ jobs: matrix: os: [ubuntu-latest] params: [ - {"py_ver": "3.7", "py_env": "py37"}, {"py_ver": "3.8", "py_env": "py38"}, {"py_ver": "3.9", "py_env": "py39"}, {"py_ver": "3.10", "py_env": "py310" }, @@ -34,14 +33,11 @@ jobs: ] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install python uses: actions/setup-python@v4 with: python-version: ${{ matrix.params.py_ver }} - - name: Install Build dependencies - working-directory: ./sdks/python - run: pip install -r build-requirements.txt - name: Install base_image_requirements.txt working-directory: ./sdks/python run: pip install --no-deps -r container/${{ matrix.params.py_env }}/base_image_requirements.txt diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index e47d3fe13cd81..0309329e84e13 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -37,7 +37,7 @@ on: # This allows a subsequently queued workflow run to interrupt previous runs concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login}}' cancel-in-progress: true jobs: @@ -49,7 +49,7 @@ jobs: outputs: gcp-variables-set: ${{ steps.check_gcp_variables.outputs.gcp-variables-set }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: "Check are GCP variables set" run: "./scripts/ci/ci_check_are_gcp_variables_set.sh" id: check_gcp_variables @@ -73,17 +73,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install python uses: actions/setup-python@v4 with: - python-version: 3.7 - - name: Get build dependencies - working-directory: ./sdks/python - run: pip install pip setuptools --upgrade && pip install -r build-requirements.txt + python-version: 3.8 - name: Build source working-directory: ./sdks/python - run: python setup.py sdist + run: pip install -U build && python -m build --sdist - name: Rename source file working-directory: ./sdks/python/dist run: mv $(ls | grep "apache-beam.*tar\.gz") apache-beam-source.tar.gz @@ -99,9 +96,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [macos-latest, windows-latest] params: [ - {"py_ver": "3.7", "tox_env": "py37"}, {"py_ver": "3.8", "tox_env": "py38"}, {"py_ver": "3.9", "tox_env": "py39"}, {"py_ver": "3.10", "tox_env": "py310" }, @@ -109,14 +105,11 @@ jobs: ] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install python uses: actions/setup-python@v4 with: python-version: ${{ matrix.params.py_ver }} - - name: Get build dependencies - working-directory: ./sdks/python - run: pip install -r build-requirements.txt --use-pep517 - name: Install tox run: pip install tox - name: Run tests basic unix @@ -141,17 +134,14 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python: ["3.8", "3.9", "3.10", "3.11"] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install python uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - - name: Get build dependencies - working-directory: ./sdks/python - run: pip install -r build-requirements.txt - name: Install requirements working-directory: ./sdks/python run: pip install setuptools --upgrade && pip install -e . @@ -169,10 +159,10 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python: ["3.8", "3.9", "3.10", "3.11"] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install python uses: actions/setup-python@v4 with: @@ -180,7 +170,7 @@ jobs: - name: Install go uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' - name: Download source from artifacts uses: actions/download-artifact@v3 with: @@ -193,9 +183,6 @@ jobs: service_account_key: ${{ secrets.GCP_SA_KEY }} project_id: ${{ secrets.GCP_PROJECT_ID }} export_default_credentials: true - - name: Get build dependencies - working-directory: ./sdks/python - run: pip install -r build-requirements.txt - name: Install requirements working-directory: ./sdks/python run: pip install setuptools --upgrade && pip install -e ".[gcp]" diff --git a/.github/workflows/reportGenerator.yml b/.github/workflows/reportGenerator.yml index 44055cd563102..8f6bccddcfad0 100644 --- a/.github/workflows/reportGenerator.yml +++ b/.github/workflows/reportGenerator.yml @@ -26,7 +26,7 @@ jobs: name: Generate issue report runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Node uses: actions/setup-node@v3 with: diff --git a/.github/workflows/run_perf_alert_tool.yml b/.github/workflows/run_perf_alert_tool.yml index 510db83f86908..bc59bd945fe23 100644 --- a/.github/workflows/run_perf_alert_tool.yml +++ b/.github/workflows/run_perf_alert_tool.yml @@ -23,27 +23,29 @@ on: workflow_dispatch: schedule: - cron: '5 22 * * *' + pull_request: + branches: ['master'] + tags: 'v*' + paths: ['sdks/python/apache_beam/testing/**'] jobs: python_run_change_point_analysis: name: Run Change Point Analysis. - runs-on: ubuntu-latest + runs-on: [self-hosted, ubuntu-20.04, main] permissions: issues: write steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install python uses: actions/setup-python@v4 with: python-version: 3.8 - name: Authenticate on GCP + if: github.event_name != 'pull_request' uses: google-github-actions/setup-gcloud@v0 with: service_account_key: ${{ secrets.GCP_SA_KEY }} export_default_credentials: true - - name: Get Apache Beam Build dependencies - working-directory: ./sdks/python - run: pip install pip setuptools --upgrade && pip install -r build-requirements.txt - name: Install Apache Beam working-directory: ./sdks/python run: pip install -e .[gcp,test] @@ -54,7 +56,15 @@ jobs: - name: Run Change Point Analysis. working-directory: ./sdks/python/apache_beam/testing/analyzers shell: bash - run: python perf_analysis.py + run: python perf_analysis.py --config_file_path=./tests_config.yaml --save_alert_metadata + if: github.event_name != 'pull_request' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Run Change Point Analysis. + working-directory: ./sdks/python/apache_beam/testing/analyzers + shell: bash + run: python perf_analysis.py --config_file_path=./tests_config.yaml + if: github.event_name == 'pull_request' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Run change point analysis tests. diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml index 7a98942eed1fa..4902fee810160 100644 --- a/.github/workflows/run_rc_validation.yml +++ b/.github/workflows/run_rc_validation.yml @@ -78,7 +78,7 @@ jobs: WORKING_BRANCH: "v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}_validations" steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ env.RC_TAG }} - name: Setup GitHub CLI @@ -92,8 +92,8 @@ jobs: - name: Create Pull Request run: | git checkout -b ${{env.WORKING_BRANCH}} ${{ env.RC_TAG }} --quiet - touch empty_file.txt - git add empty_file.txt + touch empty_file.json + git add empty_file.json git commit -m "Add empty file in order to create PR" --quiet git push origin ${{env.WORKING_BRANCH}} --quiet GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t"[DO NOT MERGE] Run Python RC Validation Tests" -b "PR to run Python ReleaseCandidate Jenkins Job.") @@ -109,7 +109,7 @@ jobs: py_version: [3.8] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{env.RC_TAG}} @@ -174,7 +174,7 @@ jobs: py_version: [3.8] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{env.RC_TAG}} - name: Verify ENV values @@ -280,7 +280,7 @@ jobs: needs: generate_shared_pubsub steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{env.RC_TAG}} - name: Install Python @@ -355,7 +355,7 @@ jobs: needs: generate_shared_pubsub steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{env.RC_TAG}} @@ -403,7 +403,7 @@ jobs: needs: [generate_shared_pubsub] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{env.RC_TAG}} @@ -456,7 +456,7 @@ jobs: needs: [generate_shared_pubsub] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{env.RC_TAG}} - name: Install Python @@ -505,7 +505,7 @@ jobs: needs: [generate_shared_pubsub] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{env.RC_TAG}} - name: Install Python diff --git a/.github/workflows/self-assign.yml b/.github/workflows/self-assign.yml index c6b7cc69ce978..29ad240cd0ddb 100644 --- a/.github/workflows/self-assign.yml +++ b/.github/workflows/self-assign.yml @@ -16,7 +16,7 @@ name: Assign or close an issue on: issue_comment: - + types: [created] jobs: assign: permissions: diff --git a/.github/workflows/tour_of_beam_backend.yml b/.github/workflows/tour_of_beam_backend.yml index f54277318cd4c..3585bd0e670fc 100644 --- a/.github/workflows/tour_of_beam_backend.yml +++ b/.github/workflows/tour_of_beam_backend.yml @@ -31,7 +31,7 @@ on: # This allows a subsequently queued workflow run to interrupt previous runs concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login}}' cancel-in-progress: true jobs: @@ -41,7 +41,7 @@ jobs: run: working-directory: ./learning/tour-of-beam/backend steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v4 with: # pin to the biggest Go version supported by Cloud Functions runtime diff --git a/.github/workflows/tour_of_beam_backend_integration.yml b/.github/workflows/tour_of_beam_backend_integration.yml index a566cb2aa52b0..c96611b3215b3 100644 --- a/.github/workflows/tour_of_beam_backend_integration.yml +++ b/.github/workflows/tour_of_beam_backend_integration.yml @@ -37,7 +37,7 @@ on: # This allows a subsequently queued workflow run to interrupt previous runs concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login}}' cancel-in-progress: true env: @@ -61,6 +61,11 @@ env: PORT_POST_USER_CODE: 8806 PORT_POST_DELETE_PROGRESS: 8807 + # Gradle Enterprise + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + jobs: integration: @@ -69,17 +74,14 @@ jobs: run: working-directory: ./learning/tour-of-beam/backend steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 + - uses: actions/checkout@v4 + + - name: Setup environment + uses: ./.github/actions/setup-environment-action with: # pin to the biggest Go version supported by Cloud Functions runtime go-version: '1.16' - - name: Setup Gradle - uses: gradle/gradle-build-action@v2 - with: - cache-read-only: false - - name: Build Playground router image run: ./gradlew -i playground:backend:containers:router:docker working-directory: ${{ env.GITHUB_WORKSPACE }} diff --git a/.github/workflows/tour_of_beam_frontend_test.yml b/.github/workflows/tour_of_beam_frontend_test.yml index 1bd416e89e4c9..49bc9ef357d60 100644 --- a/.github/workflows/tour_of_beam_frontend_test.yml +++ b/.github/workflows/tour_of_beam_frontend_test.yml @@ -30,9 +30,14 @@ on: # This allows a subsequently queued workflow run to interrupt previous runs concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login}}' cancel-in-progress: true +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + jobs: tour_of_beam_test: name: Tour of Beam Frontend Test @@ -42,7 +47,7 @@ jobs: FLUTTER_VERSION: '3.10.4' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: 'Cache Flutter Dependencies' uses: actions/cache@v3 @@ -62,7 +67,7 @@ jobs: cd playground/frontend/playground_components_dev && flutter pub get && cd - cd learning/tour-of-beam/frontend && flutter pub get && cd - - - uses: nanasess/setup-chromedriver@v1 + - uses: nanasess/setup-chromedriver@v2 - name: 'Integration tests' run: | diff --git a/.github/workflows/typescript_tests.yml b/.github/workflows/typescript_tests.yml index 1df0010d12972..edbe8399e7d85 100644 --- a/.github/workflows/typescript_tests.yml +++ b/.github/workflows/typescript_tests.yml @@ -33,7 +33,7 @@ on: # This allows a subsequently queued workflow run to interrupt previous runs concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login}}' cancel-in-progress: true jobs: typescript_unit_tests: @@ -45,7 +45,7 @@ jobs: os: [[self-hosted, ubuntu-20.04], macos-latest, [self-hosted, windows-server-2019]] steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false submodules: recursive @@ -74,7 +74,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false submodules: recursive @@ -89,10 +89,8 @@ jobs: - name: Setup Beam Python working-directory: ./sdks/python run: | - pip install pip setuptools --upgrade - pip install -r build-requirements.txt pip install 'pandas>=1.0,<1.5' - python setup.py develop + pip install -e . - run: npm ci working-directory: ./sdks/typescript - run: npm run build @@ -109,7 +107,7 @@ jobs: outputs: gcp-variables-set: ${{ steps.check_gcp_variables.outputs.gcp-variables-set }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: "Check are GCP variables set" run: "./scripts/ci/ci_check_are_gcp_variables_set.sh" id: check_gcp_variables @@ -131,7 +129,7 @@ jobs: fail-fast: false steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false submodules: recursive @@ -146,10 +144,7 @@ jobs: - name: Setup Beam Python working-directory: ./sdks/python run: | - pip install pip setuptools --upgrade - pip install -r build-requirements.txt pip install 'pandas>=1.0,<1.5' - python setup.py develop pip install -e ".[gcp]" - name: Authenticate on GCP uses: google-github-actions/setup-gcloud@v0 diff --git a/.github/workflows/update_python_dependencies.yml b/.github/workflows/update_python_dependencies.yml index 5d6d4dde92275..b4b839c3204c9 100644 --- a/.github/workflows/update_python_dependencies.yml +++ b/.github/workflows/update_python_dependencies.yml @@ -26,6 +26,11 @@ on: workflow_dispatch: permissions: read-all +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + jobs: set-properties: runs-on: [self-hosted, ubuntu-20.04] @@ -33,7 +38,7 @@ jobs: properties: ${{ steps.test-properties.outputs.properties }} steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -43,9 +48,15 @@ jobs: name: Update Python Dependencies steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup environment - uses: ./.github/actions/setup-self-hosted-action + uses: ./.github/actions/setup-environment-action + with: + python-version: | + 3.8 + 3.9 + java-version: 8 + go-version: 1.21 - name: Update Python Dependencies uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml deleted file mode 100644 index 2ab76079e1d8c..0000000000000 --- a/.github/workflows/verify_release_build.yml +++ /dev/null @@ -1,81 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# To learn more about GitHub Actions in Apache Beam check the CI.md - -name: Verify Release Build -on: - workflow_dispatch: - inputs: - RELEASE_VER: - description: Beam version of current release - required: true - COMMENT_TRIGGER_PHRASES: - description: Flag to comment the Jenkins Trigger Phrases in the new PR - required: true -env: - WORKING_BRANCH: postcommit_validation_pr -jobs: - verify_release: - runs-on: [self-hosted, ubuntu-20.04] - permissions: - pull-requests: write - contents: write - env: - RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }} - steps: - - name: Verify branch name - run: ./scripts/ci/ci_check_git_branch.sh $WORKING_BRANCH - - - name: Set RELEASE_BRANCH env variable - run: | - RELEASE_BRANCH=release-${{env.RELEASE_VER}} - echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV - - name: Check out code - uses: actions/checkout@v3 - with: - ref: ${{ env.RELEASE_BRANCH }} - - name: Install gh cli - uses: ./.github/actions/setup-gh-cli-linux - - name: Set git config - run: | - git config user.name $GITHUB_ACTOR - git config user.email actions@"$RUNNER_NAME".local - - name: Set Release Version - run: | - git checkout -b ${{env.WORKING_BRANCH}} --quiet - /bin/bash release/src/main/scripts/set_version.sh "$RELEASE_VER" --git-add - # In case the version string was not changed, append a newline to CHANGES.md - echo "" >> CHANGES.md - - name: Push Changes in ${{ env.WORKING_BRANCH }} Branch - run: | - git add CHANGES.md - git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet - git push origin $WORKING_BRANCH --quiet - - name: Create new Release PR - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t "[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch" -b "Run Gradle Release Build and Jenkins PreCommit and PostCommits tests automatically in this PR (refer to scripts/ci/release/mass_comment.txt). Please run the missing tests manually.") - # Run Gradle Release Build and PostCommit/PreCommit Tests against Release Branch on Jenkins. - - name: Comment Jenkins Trigger Phrases in PR - if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} - run: | - sh ./comment_pr_trigger_phrases.sh $GITHUB_PR_URL - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - working-directory: 'scripts/ci/release' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 72f7ec9642eb1..a6c0faae0823e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: - repo: https://github.com/pycqa/pylint # this rev is a release tag in the repo above and corresponds with a pylint # version. make sure this matches the version of pylint in tox.ini. - rev: v2.11.1 + rev: v2.17.5 hooks: - id: pylint args: ["--rcfile=sdks/python/.pylintrc"] diff --git a/.test-infra/OWNERS b/.test-infra/OWNERS deleted file mode 100644 index 7d9c8cae4d7f6..0000000000000 --- a/.test-infra/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lgajowy diff --git a/.test-infra/jenkins/CommonJobProperties.groovy b/.test-infra/jenkins/CommonJobProperties.groovy index 4cb7f330b8b06..93c81ff9af30c 100644 --- a/.test-infra/jenkins/CommonJobProperties.groovy +++ b/.test-infra/jenkins/CommonJobProperties.groovy @@ -108,6 +108,7 @@ class CommonJobProperties { credentialsBinding { string("CODECOV_TOKEN", "beam-codecov-token") string("COVERALLS_REPO_TOKEN", "beam-coveralls-token") + usernamePassword("GRADLE_ENTERPRISE_CACHE_USERNAME", "GRADLE_ENTERPRISE_CACHE_PASSWORD", "beam_cache_node_credentials") } timestamps() colorizeOutput() diff --git a/.test-infra/jenkins/LoadTestsBuilder.groovy b/.test-infra/jenkins/LoadTestsBuilder.groovy index b5b0b664acfff..060a2ea654243 100644 --- a/.test-infra/jenkins/LoadTestsBuilder.groovy +++ b/.test-infra/jenkins/LoadTestsBuilder.groovy @@ -24,10 +24,10 @@ import InfluxDBCredentialsHelper import static PythonTestProperties.LOAD_TEST_PYTHON_VERSION class LoadTestsBuilder { - final static String DOCKER_CONTAINER_REGISTRY = 'gcr.io/apache-beam-testing/beam_portability' - final static String DOCKER_CONTAINER_REGISTRY_SNAPSHOTS = 'gcr.io/apache-beam-testing/beam-sdk' - final static String GO_SDK_CONTAINER = "${DOCKER_CONTAINER_REGISTRY_SNAPSHOTS}/beam_go_sdk:latest" + final static String DOCKER_CONTAINER_REGISTRY = 'gcr.io/apache-beam-testing/beam-sdk' + final static String GO_SDK_CONTAINER = "${DOCKER_CONTAINER_REGISTRY}/beam_go_sdk:latest" final static String DOCKER_BEAM_SDK_IMAGE = "beam_python${LOAD_TEST_PYTHON_VERSION}_sdk:latest" + final static String DOCKER_BEAM_JOBSERVER = 'gcr.io/apache-beam-testing/beam_portability' static void loadTests(scope, CommonTestProperties.SDK sdk, List testConfigurations, String test, String mode, List jobSpecificSwitches = null) { diff --git a/.test-infra/jenkins/OWNERS b/.test-infra/jenkins/OWNERS deleted file mode 100644 index ca45477af958c..0000000000000 --- a/.test-infra/jenkins/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lgajowy - - kkucharc - - echauchot diff --git a/.test-infra/jenkins/PythonTestProperties.groovy b/.test-infra/jenkins/PythonTestProperties.groovy index 1ebf7cc84a87f..98257a6e1c288 100644 --- a/.test-infra/jenkins/PythonTestProperties.groovy +++ b/.test-infra/jenkins/PythonTestProperties.groovy @@ -20,7 +20,6 @@ class PythonTestProperties { // Indicates all supported Python versions. // This must be sorted in ascending order. final static List ALL_SUPPORTED_VERSIONS = [ - '3.7', '3.8', '3.9', '3.10', @@ -38,9 +37,9 @@ class PythonTestProperties { final static List CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS = ESSENTIAL_VERSIONS final static List CROSS_LANGUAGE_VALIDATES_RUNNER_DATAFLOW_USING_SQL_PYTHON_VERSIONS = [HIGHEST_SUPPORTED] final static List VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS = ALL_SUPPORTED_VERSIONS - final static String LOAD_TEST_PYTHON_VERSION = '3.7' + final static String LOAD_TEST_PYTHON_VERSION = '3.8' final static String RUN_INFERENCE_TEST_PYTHON_VERSION = '3.8' - final static String CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION = '3.7' + final static String CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION = '3.8' // Use for various shell scripts triggered by Jenkins. // Gradle scripts should use project.ext.pythonVersion defined by PythonNature/BeamModulePlugin. final static String DEFAULT_INTERPRETER = 'python3.8' diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index 5f9b55d4e2366..aae5842d274d8 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -148,7 +148,6 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PostCommit_PortableJar_Spark | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_PortableJar_Spark/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_PortableJar_Spark_PR/) | `Run PortableJar_Spark PostCommit` | [![Build Status](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_PortableJar_Spark/badge/icon)](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_PortableJar_Spark/) | | beam_PostCommit_Java_Sickbay | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Sickbay/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Sickbay_PR/) | `Run Java Sickbay` | [![Build Status](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Sickbay/badge/icon)](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Sickbay/) | | beam_PostCommit_Py_VR_Dataflow | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow_PR/) | `Run Python Dataflow ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow) | -| beam_PostCommit_Py_VR_Dataflow_V2 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow_V2/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow_V2_PR/) | `Run Python Dataflow V2 ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow_V2/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow_V2) | | beam_PostCommit_Py_ValCont | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont_PR/) | `Run Python Dataflow ValidatesContainer` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont) | | beam_PostCommit_Python_VR_Flink | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Flink/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Flink_PR/) | `Run Python Flink ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Flink/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Flink) | | beam_PostCommit_Python_VR_Samza | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Samza/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Samza_PR/) | `Run Python Samza ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Samza/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Samza) | @@ -163,12 +162,11 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PostCommit_Python_VR_Spark | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark/) | `Run Python Spark ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark) | | beam_PostCommit_Python_Xlang_Gcp_Direct | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Xlang_Gcp_Direct/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Xlang_Gcp_Direct_PR/) | `Run Python_Xlang_Gcp_Direct PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Xlang_Gcp_Direct/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Xlang_Gcp_Direct/) | | beam_PostCommit_Python_Xlang_Gcp_Dataflow | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Xlang_Gcp_Dataflow/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Xlang_Gcp_Dataflow_PR/) | `Run Python_Xlang_Gcp_Dataflow PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Xlang_Gcp_Dataflow/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Xlang_Gcp_Dataflow/) | -| beam_PostCommit_Python37 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python37), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python37_PR/) | `Run Python 3.7 PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python37/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python37) | +| beam_PostCommit_Python_Xlang_IO_Dataflow | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Xlang_IO_Dataflow/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Xlang_IO_Dataflow_PR/) | `Run Python_Xlang_IO_Dataflow PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Xlang_IO_Dataflow/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Xlang_IO_Dataflow/| | beam_PostCommit_Python38 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python38), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python38_PR/) | `Run Python 3.8 PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python38/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python38) | | beam_PostCommit_Python39 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python39), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python39_PR/) | `Run Python 3.9 PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python39/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python39) | | beam_PostCommit_Python310 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python310), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python310_PR/) | `Run Python 3.10 PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python310/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python310) | | beam_PostCommit_Python311 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python311), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python311_PR/) | `Run Python 3.11 PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python311/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python311) | -| beam_PostCommit_Sickbay_Python37 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python37), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SickBay_Python37_PR/) | `Run Python 3.7 PostCommit Sickbay tests` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python37/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python37) | | beam_PostCommit_Sickbay_Python38 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python38), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SickBay_Python38_PR/) | `Run Python 3.8 PostCommit Sickbay tests` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python38/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python38) | | beam_PostCommit_Sickbay_Python39 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python39), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SickBay_Python39_PR/) | `Run Python 3.9 PostCommit Sickbay tests` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python39/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python39) | | beam_PostCommit_Sickbay_Python310 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python310), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SickBay_Python310_PR/) | `Run Python 3.10 PostCommit Sickbay tests` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python310/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python310) | @@ -202,7 +200,7 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PerformanceTests_SparkReceiver_IO | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_SparkReceiver_IO/) | `Run Java SparkReceiverIO Performance Test` | [![Build Status](https://ci-beam.apache.org/job/beam_PerformanceTests_SparkReceiver_IO/badge/icon)](https://ci-beam.apache.org/job/beam_PerformanceTests_SparkReceiver_IO) | | beam_PerformanceTests_TFRecordIOIT | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_TFRecordIOIT/) | `Run Java TFRecordIO Performance Test` | [![Build Status](https://ci-beam.apache.org/job/beam_PerformanceTests_TFRecordIOIT/badge/icon)](https://ci-beam.apache.org/job/beam_PerformanceTests_TFRecordIOIT) | | beam_PerformanceTests_TextIOIT | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_TextIOIT/), [hdfs_cron](https://ci-beam.apache.org/job/beam_PerformanceTests_TextIOIT_HDFS/) | `Run Java TextIO Performance Test` | [![Build Status](https://ci-beam.apache.org/job/beam_PerformanceTests_TextIOIT/badge/icon)](https://ci-beam.apache.org/job/beam_PerformanceTests_TextIOIT) [![Build Status](https://ci-beam.apache.org/job/beam_PerformanceTests_TextIOIT_HDFS/badge/icon)](https://ci-beam.apache.org/job/beam_PerformanceTests_TextIOIT_HDFS) | -| beam_PerformanceTests_WordCountIT_Py37 | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/) | `Run Python37 WordCountIT Performance Test` | [![Build Status](https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/badge/icon)](https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37) | +| beam_PerformanceTests_WordCountIT_Py38 | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py38/) | `Run Python38 WordCountIT Performance Test` | [![Build Status](https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py38/badge/icon)](https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py38) | | beam_PerformanceTests_XmlIOIT | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_XmlIOIT/), [hdfs_cron](https://ci-beam.apache.org/job/beam_PerformanceTests_XmlIOIT_HDFS/) | `Run Java XmlIO Performance Test` | [![Build Status](https://ci-beam.apache.org/job/beam_PerformanceTests_XmlIOIT/badge/icon)](https://ci-beam.apache.org/job/beam_PerformanceTests_XmlIOIT) [![Build Status](https://ci-beam.apache.org/job/beam_PerformanceTests_XmlIOIT_HDFS/badge/icon)](https://ci-beam.apache.org/job/beam_PerformanceTests_XmlIOIT_HDFS) | | beam_PerformanceTests_SQLBigQueryIO_Batch_Java | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_SQLBigQueryIO_Batch_Java/) | `Run SQLBigQueryIO Batch Performance Test Java` | [![Build Status](https://ci-beam.apache.org/job/beam_PerformanceTests_SQLBigQueryIO_Batch_Java/badge/icon)](https://ci-beam.apache.org/job/beam_PerformanceTests_SQLBigQueryIO_Batch_Java/) | | beam_Java_JMH | [cron](https://ci-beam.apache.org/job/beam_Java_JMH/) | | [![Build Status](https://ci-beam.apache.org/job/beam_Java_JMH/badge/icon)](https://ci-beam.apache.org/job/beam_Java_JMH/) | @@ -298,7 +296,6 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_Publish_Beam_SDK_Snapshots | [cron](https://ci-beam.apache.org/job/beam_Publish_Beam_SDK_Snapshots/)| N/A | [![Build Status](https://ci-beam.apache.org/job/beam_Publish_Beam_SDK_Snapshots/badge/icon)](https://ci-beam.apache.org/job/beam_Publish_Beam_SDK_Snapshots/) | | beam_Publish_Docker_Snapshots | [cron](https://ci-beam.apache.org/job/beam_Publish_Docker_Snapshots/)| N/A | [![Build Status](https://ci-beam.apache.org/job/beam_Publish_Docker_Snapshots/badge/icon)](https://ci-beam.apache.org/job/beam_Publish_Docker_Snapshots/) | | beam_PostRelease_Python_Candidate | [cron](https://ci-beam.apache.org/job/beam_PostRelease_Python_Candidate/)| `Run Python ReleaseCandidate` | [![Build Status](https://ci-beam.apache.org/job/beam_PostRelease_Python_Candidate/badge/icon)](https://ci-beam.apache.org/job/beam_PostRelease_Python_Candidate/) | -| beam_Release_Gradle_Build | [cron](https://ci-beam.apache.org/job/beam_Release_Gradle_Build/) | `Run Release Gradle Build` | [![Build Status](https://ci-beam.apache.org/job/beam_Release_Gradle_Build/badge/icon)](https://ci-beam.apache.org/job/beam_Release_Gradle_Build/) ### Notes: @@ -308,4 +305,4 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) retest this please ``` -* Last update (mm/dd/yyyy): 04/04/2022 +* Last update (mm/dd/yyyy): 06/20/2022 diff --git a/.test-infra/jenkins/job_LoadTests_Combine_Flink_Go.groovy b/.test-infra/jenkins/job_LoadTests_Combine_Flink_Go.groovy index e161ba0c0deee..9b8adc732f98a 100644 --- a/.test-infra/jenkins/job_LoadTests_Combine_Flink_Go.groovy +++ b/.test-infra/jenkins/job_LoadTests_Combine_Flink_Go.groovy @@ -23,7 +23,7 @@ import PhraseTriggeringPostCommitBuilder import Flink import InfluxDBCredentialsHelper -import static LoadTestsBuilder.DOCKER_CONTAINER_REGISTRY +import static LoadTestsBuilder.DOCKER_BEAM_JOBSERVER import static LoadTestsBuilder.GO_SDK_CONTAINER @@ -107,7 +107,7 @@ def loadTestJob = { scope, triggeringContext, mode -> GO_SDK_CONTAINER ], initialParallelism, - "${DOCKER_CONTAINER_REGISTRY}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") + "${DOCKER_BEAM_JOBSERVER}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") // Execute all scenarios connected with initial parallelism. loadTestsBuilder.loadTests(scope, CommonTestProperties.SDK.GO, initialScenarios, 'combine', mode) diff --git a/.test-infra/jenkins/job_LoadTests_Combine_Flink_Python.groovy b/.test-infra/jenkins/job_LoadTests_Combine_Flink_Python.groovy index ff7cc5a9e7957..54b92fdade264 100644 --- a/.test-infra/jenkins/job_LoadTests_Combine_Flink_Python.groovy +++ b/.test-infra/jenkins/job_LoadTests_Combine_Flink_Python.groovy @@ -24,6 +24,7 @@ import Flink import InfluxDBCredentialsHelper import static LoadTestsBuilder.DOCKER_CONTAINER_REGISTRY +import static LoadTestsBuilder.DOCKER_BEAM_JOBSERVER import static LoadTestsBuilder.DOCKER_BEAM_SDK_IMAGE String now = new Date().format("MMddHHmmss", TimeZone.getTimeZone('UTC')) @@ -135,7 +136,7 @@ def loadTestJob = { scope, triggeringContext, mode -> "${DOCKER_CONTAINER_REGISTRY}/${DOCKER_BEAM_SDK_IMAGE}" ], initialParallelism, - "${DOCKER_CONTAINER_REGISTRY}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") + "${DOCKER_BEAM_JOBSERVER}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") // Execute all scenarios connected with initial parallelism. loadTestsBuilder.loadTests(scope, CommonTestProperties.SDK.PYTHON, initialScenarios, 'Combine', mode) diff --git a/.test-infra/jenkins/job_LoadTests_GBK_Flink_Go.groovy b/.test-infra/jenkins/job_LoadTests_GBK_Flink_Go.groovy index 59fbeb75bc279..d5a6910b2a0d1 100644 --- a/.test-infra/jenkins/job_LoadTests_GBK_Flink_Go.groovy +++ b/.test-infra/jenkins/job_LoadTests_GBK_Flink_Go.groovy @@ -23,7 +23,7 @@ import PhraseTriggeringPostCommitBuilder import Flink import InfluxDBCredentialsHelper -import static LoadTestsBuilder.DOCKER_CONTAINER_REGISTRY +import static LoadTestsBuilder.DOCKER_BEAM_JOBSERVER import static LoadTestsBuilder.GO_SDK_CONTAINER String now = new Date().format('MMddHHmmss', TimeZone.getTimeZone('UTC')) @@ -199,7 +199,7 @@ def loadTestJob = { scope, triggeringContext, mode -> GO_SDK_CONTAINER ], initialParallelism, - "${DOCKER_CONTAINER_REGISTRY}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") + "${DOCKER_BEAM_JOBSERVER}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") // Execute all scenarios connected with initial parallelism. loadTestsBuilder.loadTests(scope, CommonTestProperties.SDK.GO, initialScenarios, 'group_by_key', mode) diff --git a/.test-infra/jenkins/job_LoadTests_GBK_Flink_Python.groovy b/.test-infra/jenkins/job_LoadTests_GBK_Flink_Python.groovy index f4714a2d0e309..25e2647ebf32f 100644 --- a/.test-infra/jenkins/job_LoadTests_GBK_Flink_Python.groovy +++ b/.test-infra/jenkins/job_LoadTests_GBK_Flink_Python.groovy @@ -24,6 +24,7 @@ import Flink import InfluxDBCredentialsHelper import static LoadTestsBuilder.DOCKER_CONTAINER_REGISTRY +import static LoadTestsBuilder.DOCKER_BEAM_JOBSERVER import static LoadTestsBuilder.DOCKER_BEAM_SDK_IMAGE String now = new Date().format("MMddHHmmss", TimeZone.getTimeZone('UTC')) @@ -146,7 +147,7 @@ def loadTest = { scope, triggeringContext -> "${DOCKER_CONTAINER_REGISTRY}/${DOCKER_BEAM_SDK_IMAGE}" ], numberOfWorkers, - "${DOCKER_CONTAINER_REGISTRY}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") + "${DOCKER_BEAM_JOBSERVER}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") def configurations = testScenarios.findAll { it.pipelineOptions?.parallelism?.value == numberOfWorkers } loadTestsBuilder.loadTests(scope, sdk, configurations, "GBK", "batch") diff --git a/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Go.groovy b/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Go.groovy index c5af6cc42d123..df20312f27b5a 100644 --- a/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Go.groovy +++ b/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Go.groovy @@ -23,7 +23,7 @@ import PhraseTriggeringPostCommitBuilder import Flink import InfluxDBCredentialsHelper -import static LoadTestsBuilder.DOCKER_CONTAINER_REGISTRY +import static LoadTestsBuilder.DOCKER_BEAM_JOBSERVER import static LoadTestsBuilder.GO_SDK_CONTAINER String now = new Date().format("MMddHHmmss", TimeZone.getTimeZone('UTC')) @@ -127,7 +127,7 @@ def loadTestJob = { scope, triggeringContext, mode -> GO_SDK_CONTAINER ], numberOfWorkers, - "${DOCKER_CONTAINER_REGISTRY}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") + "${DOCKER_BEAM_JOBSERVER}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") loadTestsBuilder.loadTests(scope, CommonTestProperties.SDK.GO, batchScenarios(), 'ParDo', mode) } diff --git a/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Python.groovy b/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Python.groovy index 2979a16300cd1..4af2efd1be6a6 100644 --- a/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Python.groovy +++ b/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Python.groovy @@ -24,6 +24,7 @@ import Flink import InfluxDBCredentialsHelper import static LoadTestsBuilder.DOCKER_CONTAINER_REGISTRY +import static LoadTestsBuilder.DOCKER_BEAM_JOBSERVER import static LoadTestsBuilder.DOCKER_BEAM_SDK_IMAGE String now = new Date().format("MMddHHmmss", TimeZone.getTimeZone('UTC')) @@ -329,7 +330,7 @@ def loadTestJob = { scope, triggeringContext, mode -> "${DOCKER_CONTAINER_REGISTRY}/${DOCKER_BEAM_SDK_IMAGE}" ], numberOfWorkers, - "${DOCKER_CONTAINER_REGISTRY}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") + "${DOCKER_BEAM_JOBSERVER}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") loadTestsBuilder.loadTests(scope, CommonTestProperties.SDK.PYTHON, testScenarios, 'ParDo', mode) } diff --git a/.test-infra/jenkins/job_LoadTests_SideInput_Flink_Go.groovy b/.test-infra/jenkins/job_LoadTests_SideInput_Flink_Go.groovy index f2c008b4f25ae..bd0eaa4f23e6b 100644 --- a/.test-infra/jenkins/job_LoadTests_SideInput_Flink_Go.groovy +++ b/.test-infra/jenkins/job_LoadTests_SideInput_Flink_Go.groovy @@ -22,7 +22,7 @@ import LoadTestsBuilder as loadTestsBuilder import PhraseTriggeringPostCommitBuilder import InfluxDBCredentialsHelper -import static LoadTestsBuilder.DOCKER_CONTAINER_REGISTRY +import static LoadTestsBuilder.DOCKER_BEAM_JOBSERVER import static LoadTestsBuilder.GO_SDK_CONTAINER def now = new Date().format("MMddHHmmss", TimeZone.getTimeZone('UTC')) @@ -79,7 +79,7 @@ def loadTestJob = { scope, triggeringContext, mode -> GO_SDK_CONTAINER ], numberOfWorkers, - "${DOCKER_CONTAINER_REGISTRY}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") + "${DOCKER_BEAM_JOBSERVER}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") loadTestsBuilder.loadTests(scope, CommonTestProperties.SDK.GO, batchScenarios(), 'SideInput', mode) diff --git a/.test-infra/jenkins/job_LoadTests_coGBK_Flink_Go.groovy b/.test-infra/jenkins/job_LoadTests_coGBK_Flink_Go.groovy index 95468a9d2f28c..8c7a60e724f9a 100644 --- a/.test-infra/jenkins/job_LoadTests_coGBK_Flink_Go.groovy +++ b/.test-infra/jenkins/job_LoadTests_coGBK_Flink_Go.groovy @@ -23,7 +23,7 @@ import PhraseTriggeringPostCommitBuilder import Flink import InfluxDBCredentialsHelper -import static LoadTestsBuilder.DOCKER_CONTAINER_REGISTRY +import static LoadTestsBuilder.DOCKER_BEAM_JOBSERVER import static LoadTestsBuilder.GO_SDK_CONTAINER String now = new Date().format("MMddHHmmss", TimeZone.getTimeZone('UTC')) @@ -159,7 +159,7 @@ def loadTestJob = { scope, triggeringContext, mode -> GO_SDK_CONTAINER ], numberOfWorkers, - "${DOCKER_CONTAINER_REGISTRY}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") + "${DOCKER_BEAM_JOBSERVER}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") loadTestsBuilder.loadTests(scope, CommonTestProperties.SDK.GO, batchScenarios(), 'CoGBK', mode) } diff --git a/.test-infra/jenkins/job_LoadTests_coGBK_Flink_Python.groovy b/.test-infra/jenkins/job_LoadTests_coGBK_Flink_Python.groovy index a0a06ea4f14b4..9a0798f8107b2 100644 --- a/.test-infra/jenkins/job_LoadTests_coGBK_Flink_Python.groovy +++ b/.test-infra/jenkins/job_LoadTests_coGBK_Flink_Python.groovy @@ -25,6 +25,7 @@ import InfluxDBCredentialsHelper import static LoadTestsBuilder.DOCKER_CONTAINER_REGISTRY import static LoadTestsBuilder.DOCKER_BEAM_SDK_IMAGE +import static LoadTestsBuilder.DOCKER_BEAM_JOBSERVER String now = new Date().format("MMddHHmmss", TimeZone.getTimeZone('UTC')) @@ -137,7 +138,7 @@ def loadTest = { scope, triggeringContext -> "${DOCKER_CONTAINER_REGISTRY}/${DOCKER_BEAM_SDK_IMAGE}" ], numberOfWorkers, - "${DOCKER_CONTAINER_REGISTRY}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") + "${DOCKER_BEAM_JOBSERVER}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") loadTestsBuilder.loadTests(scope, CommonTestProperties.SDK.PYTHON, testScenarios, 'CoGBK', 'batch') } diff --git a/.test-infra/jenkins/job_PerformanceTests_Python.groovy b/.test-infra/jenkins/job_PerformanceTests_Python.groovy index a2544a098cd4f..04c8fc9995307 100644 --- a/.test-infra/jenkins/job_PerformanceTests_Python.groovy +++ b/.test-infra/jenkins/job_PerformanceTests_Python.groovy @@ -30,7 +30,7 @@ def dataflowPipelineArgs = [ ] testConfigurations = [] -pythonVersions = ['37'] +pythonVersions = ['38'] for (pythonVersion in pythonVersions) { testConfigurations.add([ diff --git a/.test-infra/jenkins/job_PerformanceTests_xlang_KafkaIO_Python.groovy b/.test-infra/jenkins/job_PerformanceTests_xlang_KafkaIO_Python.groovy deleted file mode 100644 index bbab65ed8e4ab..0000000000000 --- a/.test-infra/jenkins/job_PerformanceTests_xlang_KafkaIO_Python.groovy +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as common -import Kubernetes -import LoadTestsBuilder as loadTestsBuilder -import InfluxDBCredentialsHelper - -def jobs = [ - [ - name : 'beam_PerformanceTests_xlang_KafkaIO_Python', - description : 'Runs performance tests for xlang Python KafkaIO', - test : 'apache_beam.io.external.xlang_kafkaio_perf_test', - githubTitle : 'Python xlang KafkaIO Performance Test', - githubTriggerPhrase: 'Run Python xlang KafkaIO Performance Test', - pipelineOptions : [ - publish_to_big_query : true, - metrics_dataset : 'beam_performance', - metrics_table : 'python_kafkaio_results', - influx_measurement : 'python_kafkaio_results', - test_class : 'KafkaIOPerfTest', - input_options : """'{ - "num_records": 100000000, - "key_size": 10, - "value_size": 90, - "algorithm": "lcg" - }'""".trim().replaceAll("\\s", ""), - kafka_topic : 'beam', - read_timeout : '1500', - num_workers : '5', - autoscaling_algorithm: 'NONE' - ] - ] -] - -jobs.findAll { - it.name in [ - // all tests that enabled - 'beam_PerformanceTests_xlang_KafkaIO_Python', - ] -}.forEach { testJob -> createKafkaIOTestJob(testJob) } - -private void createKafkaIOTestJob(testJob) { - job(testJob.name) { - description(testJob.description) - common.setTopLevelMainJobProperties(delegate) - common.enablePhraseTriggeringFromPullRequest(delegate, testJob.githubTitle, testJob.githubTriggerPhrase) - common.setAutoJob(delegate, 'H H * * *') - InfluxDBCredentialsHelper.useCredentials(delegate) - - // Setup kafka k8s pods - String namespace = common.getKubernetesNamespace(testJob.name) - String kubeconfig = common.getKubeconfigLocationForNamespace(namespace) - Kubernetes k8s = Kubernetes.create(delegate, kubeconfig, namespace) - String kafkaDir = common.makePathAbsolute("src/.test-infra/kubernetes/kafka-cluster") - String kafkaTopicJob = "job.batch/kafka-config-eff079ec" - - /** - * Specifies steps to avoid port collisions when the Kafka outside services (1,2,3) are created. - Function k8s.availablePort finds unused ports in the Kubernetes cluster in a range from 32400 - to 32767 by querying used ports, those ports are stored in env vars like KAFKA_SERVICE_PORT_${service}, - which are used to replace default ports for outside-${service}.yml files, before the apply command. - */ - steps { - String[] configuredPorts = ["32400", "32401", "32402"] - String HIGH_RANGE_PORT = "32767" - (0..2).each { service -> - k8s.availablePort(service == 0 ? configuredPorts[service] : "\$KAFKA_SERVICE_PORT_${service-1}", - HIGH_RANGE_PORT, "KAFKA_SERVICE_PORT_$service") - shell("sed -i -e s/${configuredPorts[service]}/\$KAFKA_SERVICE_PORT_$service/ \ - ${kafkaDir}/04-outside-services/outside-${service}.yml") - } - gradle { - rootBuildScriptDir(common.checkoutDir) - tasks(':sdks:java:io:expansion-service:shadowJar') - } - } - k8s.apply(kafkaDir) - (0..2).each { k8s.loadBalancerIP("outside-$it", "KAFKA_BROKER_$it") } - k8s.waitForJob(kafkaTopicJob,"40m") - - additionalPipelineArgs = [ - influx_db_name: InfluxDBCredentialsHelper.InfluxDBDatabaseName, - influx_hostname: InfluxDBCredentialsHelper.InfluxDBHostUrl, - bootstrap_servers: "\$KAFKA_BROKER_0:\$KAFKA_SERVICE_PORT_0,\$KAFKA_BROKER_1:\$KAFKA_SERVICE_PORT_1," + - "\$KAFKA_BROKER_2:\$KAFKA_SERVICE_PORT_2", //KAFKA_BROKER_ represents IP and KAFKA_SERVICE_ port of outside services - ] - testJob.pipelineOptions.putAll(additionalPipelineArgs) - - def dataflowSpecificOptions = [ - runner : 'DataflowRunner', - project : 'apache-beam-testing', - region : 'us-central1', - temp_location : 'gs://temp-storage-for-perf-tests/', - filename_prefix : "gs://temp-storage-for-perf-tests/${testJob.name}/\${BUILD_ID}/", - sdk_harness_container_image_overrides: '.*java.*,gcr.io/apache-beam-testing/beam-sdk/beam_java8_sdk:latest' - ] - - Map allPipelineOptions = dataflowSpecificOptions << testJob.pipelineOptions - - loadTestsBuilder.loadTest( - delegate, - testJob.name, - CommonTestProperties.Runner.DATAFLOW, - CommonTestProperties.SDK.PYTHON, - allPipelineOptions, - testJob.test) - } -} diff --git a/.test-infra/jenkins/job_PostCommit_BeamMetrics_Publish.groovy b/.test-infra/jenkins/job_PostCommit_BeamMetrics_Publish.groovy deleted file mode 100644 index 34912d2825f14..0000000000000 --- a/.test-infra/jenkins/job_PostCommit_BeamMetrics_Publish.groovy +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, eit her express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import Kubernetes -import PostcommitJobBuilder - -PostcommitJobBuilder.postCommitJob('beam_PostCommit_BeamMetrics_Publish', 'Run Beam Metrics deployment', - 'Beam Metrics deployment', this) { - - description('Applies new configuration to Beam Metrics infrastructure.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - - String kubeconfig = commonJobProperties.getKubeconfigLocationForNamespace('default') - Kubernetes.create(delegate, kubeconfig, '', 'metrics') - - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-test-infra-metrics:deploy') - commonJobProperties.setGradleSwitches(delegate) - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_Java_SingleStoreIO_IT.groovy b/.test-infra/jenkins/job_PostCommit_Java_SingleStoreIO_IT.groovy index f43da5a0531a3..901b76364333a 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_SingleStoreIO_IT.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_SingleStoreIO_IT.groovy @@ -81,6 +81,7 @@ PostcommitJobBuilder.postCommitJob(jobName, switches("-DintegrationTestRunner=dataflow") tasks(":sdks:java:io:singlestore:integrationTest --tests org.apache.beam.sdk.io.singlestore.SingleStoreIODefaultMapperIT") tasks(":sdks:java:io:singlestore:integrationTest --tests org.apache.beam.sdk.io.singlestore.SingleStoreIOSchemaTransformIT") + tasks(":sdks:java:io:singlestore:integrationTest --tests org.apache.beam.sdk.io.singlestore.SingleStoreIOConnectionAttributesIT") } } } diff --git a/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy b/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy index 1332b61ccb04e..0c6f51f8be543 100644 --- a/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy +++ b/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy @@ -31,7 +31,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_PortableJar_Flink', steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:test-suites:portable:py37:testPipelineJarFlinkRunner') + tasks(':sdks:python:test-suites:portable:py38:testPipelineJarFlinkRunner') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy b/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy index 93e58af8979ac..1f1963a9b2e4e 100644 --- a/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy +++ b/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy @@ -31,7 +31,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_PortableJar_Spark', steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:test-suites:portable:py37:testPipelineJarSparkRunner') + tasks(':sdks:python:test-suites:portable:py38:testPipelineJarSparkRunner') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/.test-infra/jenkins/job_PostCommit_Python_Chicago_Taxi_Example_Flink.groovy b/.test-infra/jenkins/job_PostCommit_Python_Chicago_Taxi_Example_Flink.groovy index c1b4a4496f2ea..6cf852a168939 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_Chicago_Taxi_Example_Flink.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_Chicago_Taxi_Example_Flink.groovy @@ -22,6 +22,7 @@ import Flink import LoadTestsBuilder import PhraseTriggeringPostCommitBuilder +import static LoadTestsBuilder.DOCKER_BEAM_JOBSERVER import static LoadTestsBuilder.DOCKER_CONTAINER_REGISTRY import static PythonTestProperties.CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION @@ -38,7 +39,7 @@ def chicagoTaxiJob = { scope -> "${DOCKER_CONTAINER_REGISTRY}/${beamSdkDockerImage}" ], numberOfWorkers, - "${DOCKER_CONTAINER_REGISTRY}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") + "${DOCKER_BEAM_JOBSERVER}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest") def pipelineOptions = [ parallelism : numberOfWorkers, diff --git a/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_Gcp_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_Gcp_Dataflow.groovy index d1676fcae46c7..1280fcb4e2339 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_Gcp_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_Gcp_Dataflow.groovy @@ -28,11 +28,11 @@ import static PythonTestProperties.CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIO // Collects tests with the @pytest.mark.uses_gcp_java_expansion_service decorator PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Xlang_Gcp_Dataflow', 'Run Python_Xlang_Gcp_Dataflow PostCommit', 'Python_Xlang_Gcp_Dataflow (\"Run Python_Xlang_Gcp_Dataflow PostCommit\")', this) { - description('Runs end-to-end cross language GCP IO tests on the Dataflow runner.') + description('Runs end-to-end cross language GCP IO tests on the Dataflow runner. \"Run Python_Xlang_Gcp_Dataflow PostCommit\"') // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) + commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180) // Publish all test results to Jenkins diff --git a/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_Gcp_Direct.groovy b/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_Gcp_Direct.groovy index 438b735fba7ff..e4bf771be1ae9 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_Gcp_Direct.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_Gcp_Direct.groovy @@ -28,7 +28,7 @@ import static PythonTestProperties.CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIO // Collects tests with the @pytest.mark.uses_gcp_java_expansion_service decorator PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Xlang_Gcp_Direct', 'Run Python_Xlang_Gcp_Direct PostCommit', 'Python_Xlang_Gcp_Direct (\"Run Python_Xlang_Gcp_Direct PostCommit\")', this) { - description('Runs end-to-end cross language GCP IO tests on the Direct runner.') + description('Runs end-to-end cross language GCP IO tests on the Direct runner. \"Run Python_Xlang_Gcp_Direct PostCommit\"') // Set common parameters. commonJobProperties.setTopLevelMainJobProperties(delegate) diff --git a/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_IO_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_IO_Dataflow.groovy new file mode 100644 index 0000000000000..9a37e76cbde81 --- /dev/null +++ b/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_IO_Dataflow.groovy @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import CommonJobProperties as commonJobProperties +import PostcommitJobBuilder + + +import static PythonTestProperties.CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS + + +// This job runs end-to-end cross language GCP IO tests with DataflowRunner. +// Collects tests with the @pytest.mark.uses_io_java_expansion_service decorator +PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Xlang_IO_Dataflow', + 'Run Python_Xlang_IO_Dataflow PostCommit', 'Python_Xlang_IO_Dataflow (\"Run Python_Xlang_IO_Dataflow PostCommit\")', this) { + description('Runs end-to-end cross language non-GCP IO tests on the Dataflow runner.') + + + // Set common parameters. + commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180) + + + // Publish all test results to Jenkins + publishers { + archiveJunit('**/pytest*.xml') + } + + + // Gradle goals for this job. + + steps { + gradle { + rootBuildScriptDir(commonJobProperties.checkoutDir) + tasks(":sdks:python:test-suites:dataflow:ioCrossLanguagePostCommit") + commonJobProperties.setGradleSwitches(delegate) + switches("-PuseWheelDistribution") + switches("-PkafkaBootstrapServer=10.128.0.40:9094,10.128.0.28:9094,10.128.0.165:9094") + } + } + } diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy index d93a2fb0984c0..db052a0046ce4 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy @@ -20,7 +20,7 @@ import CommonJobProperties as commonJobProperties import PostcommitJobBuilder // This job runs the suite of Python ValidatesRunner tests against the -// Dataflow runner. +// Dataflow runner V2. PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_VR_Dataflow', 'Run Python Dataflow ValidatesRunner', 'Google Cloud Dataflow Runner Python ValidatesRunner Tests', this) { description('Runs Python ValidatesRunner suite on the Dataflow runner.') @@ -37,7 +37,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_VR_Dataflow', 'Run Python gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) tasks(':sdks:python:test-suites:dataflow:validatesRunnerBatchTests') - switches('-Pdisable_runner_v2_until_v2.50') + tasks(':sdks:python:test-suites:dataflow:validatesRunnerStreamingTests') + switches('-PuseWheelDistribution') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow_V2.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow_V2.groovy deleted file mode 100644 index b4667b9080843..0000000000000 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow_V2.groovy +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// This job runs the suite of Python ValidatesRunner tests against the -// Dataflow runner V2. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_VR_Dataflow_V2', 'Run Python Dataflow V2 ValidatesRunner', - 'Google Cloud Dataflow Runner V2 Python ValidatesRunner Tests', this) { - description('Runs Python ValidatesRunner suite on the Dataflow runner v2.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 200) - - publishers { - archiveJunit('**/pytest*.xml') - } - - // Execute gradle task to test Python SDK. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:test-suites:dataflow:validatesRunnerBatchTestsV2') - tasks(':sdks:python:test-suites:dataflow:validatesRunnerStreamingTestsV2') - switches('-PuseRunnerV2') - switches('-PuseWheelDistribution') - commonJobProperties.setGradleSwitches(delegate) - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_TransformService_Direct.groovy b/.test-infra/jenkins/job_PostCommit_TransformService_Direct.groovy new file mode 100644 index 0000000000000..0d7f58e717064 --- /dev/null +++ b/.test-infra/jenkins/job_PostCommit_TransformService_Direct.groovy @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import CommonJobProperties as commonJobProperties +import PostcommitJobBuilder + +import static PythonTestProperties.CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS + +// This job runs multi-language pipelines using the Docker Compose based TransformService against the Direct runner. +// Collects tests with the @pytest.mark.uses_transform_service decorator +PostcommitJobBuilder.postCommitJob('beam_PostCommit_TransformService_Direct', + 'Run TransformService_Direct PostCommit', 'Direct TransformService Tests', this) { + description('Runs the TransformService suite on the Direct runner.') + + // Set common parameters. + commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120) + + // Publish all test results to Jenkins + publishers { + archiveJunit('**/pytest*.xml') + } + + // Gradle goals for this job. + steps { + CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS.each { pythonVersion -> + shell("echo \"*** RUN TRANSFORM SERVICE Python SDK TESTS USING THE DIRECT RUNNER AND THE PYTHON VERSION ${pythonVersion} ***\"") + gradle { + rootBuildScriptDir(commonJobProperties.checkoutDir) + tasks(':sdks:python:test-suites:direct:xlang:transformServicePythonUsingJava') + commonJobProperties.setGradleSwitches(delegate) + switches '-PcompileAndRunTestsWithJava11' + switches "-Pjava11Home=${commonJobProperties.JAVA_11_HOME}" + switches("-PuseWheelDistribution") + switches("-PpythonVersion=${pythonVersion}") + } + } + } + } diff --git a/.test-infra/jenkins/job_PostCommit_Website_Publish.groovy b/.test-infra/jenkins/job_PostCommit_Website_Publish.groovy deleted file mode 100644 index fcf22e2b98983..0000000000000 --- a/.test-infra/jenkins/job_PostCommit_Website_Publish.groovy +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - - -// This job builds and publishes the website into the asf-site branch of the beam repo. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Website_Publish', '', - 'Website Publish', this) { - - description('Publish generated website content into asf-site branch for hosting.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 30, true, 'git-websites') - - // Gradle goals for this job. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':website:clean') - tasks(':website:publishWebsite') - commonJobProperties.setGradleSwitches(delegate) - } - } - } diff --git a/.test-infra/jenkins/job_PreCommit_CommunityMetrics.groovy b/.test-infra/jenkins/job_PreCommit_CommunityMetrics.groovy deleted file mode 100644 index 2bf63df9cd5a2..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_CommunityMetrics.groovy +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'CommunityMetrics', - gradleTask: ':communityMetricsPreCommit', - triggerPathPatterns: ['^.test-infra/metrics/.*$']) -builder.build() - diff --git a/.test-infra/jenkins/job_PreCommit_Go.groovy b/.test-infra/jenkins/job_PreCommit_Go.groovy deleted file mode 100644 index 73b6cf4c93843..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_Go.groovy +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Go', - gradleTask: ':goPreCommit', - triggerPathPatterns: [ - '^model/.*$', - '^sdks/go.*$', - '^release/.*$', - ] - ) -builder.build() diff --git a/.test-infra/jenkins/job_PreCommit_Go_Portable.groovy b/.test-infra/jenkins/job_PreCommit_Go_Portable.groovy deleted file mode 100644 index 12c762e5eb37f..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_Go_Portable.groovy +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'GoPortable', - gradleTask: ':goPortablePreCommit', - triggerPathPatterns: [ - '^model/.*$', - '^sdks/go.*$', - '^release/.*$', - ] - ) -builder.build() diff --git a/.test-infra/jenkins/job_PreCommit_Java.groovy b/.test-infra/jenkins/job_PreCommit_Java.groovy index ddeb05506cb8d..41a3b418a015a 100644 --- a/.test-infra/jenkins/job_PreCommit_Java.groovy +++ b/.test-infra/jenkins/job_PreCommit_Java.groovy @@ -34,6 +34,7 @@ def excludePaths = [ 'io/elasticsearch', 'io/elasticsearch-tests', 'io/file-schema-transform', + 'io/google-ads', 'io/google-cloud-platform', 'io/hadoop-common', 'io/hadoop-file-system', diff --git a/.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow.groovy b/.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow.groovy deleted file mode 100644 index 109456b3bc4fc..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow.groovy +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Java_Examples_Dataflow', - gradleTask: ':javaExamplesDataflowPreCommit', - gradleSwitches: [ - '-PdisableSpotlessCheck=true', - '-PdisableCheckStyle=true' - ], // spotless checked in separate pre-commit - triggerPathPatterns: [ - '^model/.*$', - '^sdks/java/.*$', - '^runners/google-cloud-dataflow-java/.*$', - '^examples/java/.*$', - '^examples/kotlin/.*$', - '^release/.*$', - ], - timeoutMins: 60, - ) -builder.build { - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow_Java11.groovy b/.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow_Java11.groovy deleted file mode 100644 index adeb7f14c0eb2..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow_Java11.groovy +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder -import CommonJobProperties as properties - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Java_Examples_Dataflow_Java11', - gradleTask: ':clean', - gradleSwitches: [ - '-PdisableSpotlessCheck=true', - '-PdisableCheckStyle=true', - '-PskipCheckerFramework' // Gradle itself is running under JDK8 so plugin configures wrong for JDK11 - ], // spotless checked in separate pre-commit - triggerPathPatterns: [ - '^model/.*$', - '^sdks/java/.*$', - '^runners/google-cloud-dataflow-java/.*$', - '^examples/java/.*$', - '^examples/kotlin/.*$', - '^release/.*$', - ], - timeoutMins: 60, - ) -builder.build { - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } - - steps { - gradle { - rootBuildScriptDir(properties.checkoutDir) - tasks 'javaExamplesDataflowPreCommit' - switches '-PdisableSpotlessCheck=true' - switches '-PdisableCheckStyle=true' - switches '-PskipCheckerFramework' // Gradle itself is running under JDK8 so plugin configures wrong for JDK11 - switches '-PcompileAndRunTestsWithJava11' - switches "-Pjava11Home=${properties.JAVA_11_HOME}" - properties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors()) - } - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Java_IOs.groovy b/.test-infra/jenkins/job_PreCommit_Java_IOs.groovy index f707be2674c5f..09bf1982d1271 100644 --- a/.test-infra/jenkins/job_PreCommit_Java_IOs.groovy +++ b/.test-infra/jenkins/job_PreCommit_Java_IOs.groovy @@ -73,33 +73,34 @@ def ioModulesMap = [ // These projects are also covered by 'Java_IOs_Direct', and won't trigger on default patterns. false: [ - 'amqp', - 'cassandra', - 'cdap', - 'clickhouse', - 'csv', - 'debezium', - 'elasticsearch', - 'file-schema-transform', - 'hbase', - 'hcatalog', - 'influxdb', - 'jdbc', - 'jms', - 'kafka', - 'kudu', - 'mongodb', - 'mqtt', - 'neo4j', - 'parquet', - 'rabbitmq', - 'redis', - 'singlestore', - 'snowflake', - 'solr', - 'splunk', - 'thrift', - 'tika' + // 'amqp', + // 'cassandra', + // 'cdap', + // 'clickhouse', + // 'csv', + // 'debezium', + // 'elasticsearch', + // 'file-schema-transform', + 'google-ads', + // 'hbase', + // 'hcatalog', + // 'influxdb', + // 'jdbc', + // 'jms', + // 'kafka', + // 'kudu', + // 'mongodb', + // 'mqtt', + // 'neo4j', + // 'parquet', + // 'rabbitmq', + // 'redis', + // 'singlestore', + // 'snowflake', + // 'solr', + // 'splunk', + // 'thrift', + // 'tika' ] ] diff --git a/.test-infra/jenkins/job_PreCommit_Java_PortableValidatesRunner_Flink_Docker.groovy b/.test-infra/jenkins/job_PreCommit_Java_PortableValidatesRunner_Flink_Docker.groovy deleted file mode 100644 index bb14a792291cd..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_Java_PortableValidatesRunner_Flink_Docker.groovy +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonTestProperties -import PrecommitJobBuilder - -// This job runs a limited subset of ValidatesRunner tests against the Flink runner in the docker environment. -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Java_PVR_Flink_Docker', - gradleTask: ":runners:flink:${CommonTestProperties.getFlinkVersion()}:job-server:validatesPortableRunnerDocker", - timeoutMins: 240, - triggerPathPatterns: [ - '^sdks/java/core/src/test/java/org/apache/beam/sdk/.*$', - '^sdks/java/container/.*$', - '^sdks/java/harness/.*$', - '^runners/flink/.*$', - '^runners/java-fn-execution/.*$', - ], - ) -builder.build { - // Publish all test results to Jenkins. - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Java_Spark3_Versions.groovy b/.test-infra/jenkins/job_PreCommit_Java_Spark3_Versions.groovy deleted file mode 100644 index f13c4c0a1e2b6..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_Java_Spark3_Versions.groovy +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Java_Spark3_Versions', - gradleTask: ':runners:spark:3:sparkVersionsTest', - gradleSwitches: [ - '-PdisableSpotlessCheck=true' - ], // spotless checked in separate pre-commit - triggerPathPatterns: [ - '^runners/spark/.*$', - ], - timeoutMins: 120, - ) -builder.build { - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } -} \ No newline at end of file diff --git a/.test-infra/jenkins/job_PreCommit_Kotlin_Examples.groovy b/.test-infra/jenkins/job_PreCommit_Kotlin_Examples.groovy deleted file mode 100644 index c3ce31f1f6d16..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_Kotlin_Examples.groovy +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Kotlin_Examples', - gradleTask: ':examples:kotlin:preCommit', - triggerPathPatterns: [ - '^model/.*$', - '^sdks/java/.*$', - '^runners/flink/.*$', - '^runners/spark/.*$', - '^runners/direct-java/.*$', - '^examples/kotlin/.*$', - '^release/.*$', - ] - ) -builder.build() diff --git a/.test-infra/jenkins/job_PreCommit_Portable_Python.groovy b/.test-infra/jenkins/job_PreCommit_Portable_Python.groovy deleted file mode 100644 index 2992cbbd0d06d..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_Portable_Python.groovy +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PrecommitJobBuilder -import static PythonTestProperties.LOWEST_SUPPORTED -import static PythonTestProperties.HIGHEST_SUPPORTED - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Portable_Python', - gradleTask: ':clean', // Do nothing here. Add test configs below. - triggerPathPatterns: [ - '^model/.*$', - '^runners/core-construction-java/.*$', - '^runners/core-java/.*$', - '^runners/extensions-java/.*$', - '^runners/flink/.*$', - '^runners/java-fn-execution/.*$', - '^runners/reference/.*$', - '^sdks/python/.*$', - '^release/.*$', - ] - ) - -builder.build { - // Due to BEAM-7993, run multiple Python version of portable precommit - // tests in parallel could lead python3 container crash. We manually - // config gradle steps here to run tests in sequential. - def lowestSupported = LOWEST_SUPPORTED.replace('.', '') - def highestSupported = HIGHEST_SUPPORTED.replace('.', '') - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(":sdks:python:test-suites:portable:py${lowestSupported}:preCommitPy${lowestSupported}") - commonJobProperties.setGradleSwitches(delegate) - } - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(":sdks:python:test-suites:portable:py${highestSupported}:preCommitPy${highestSupported}") - commonJobProperties.setGradleSwitches(delegate) - } - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Python.groovy b/.test-infra/jenkins/job_PreCommit_Python.groovy index 0e439d7888773..9c9740e3c97ee 100644 --- a/.test-infra/jenkins/job_PreCommit_Python.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python.groovy @@ -29,6 +29,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( '^release/.*$', ], gradleSwitches: [ + '-PuseWheelDistribution', '-Pposargs=\"--ignore=apache_beam/dataframe/ --ignore=apache_beam/examples/ --ignore=apache_beam/runners/ --ignore=apache_beam/transforms/\"' // All these tests are covered by different jobs. ], numBuildsToRetain: 40 diff --git a/.test-infra/jenkins/job_PreCommit_PythonAutoformatter.groovy b/.test-infra/jenkins/job_PreCommit_PythonAutoformatter.groovy deleted file mode 100644 index 90e037edf2af6..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_PythonAutoformatter.groovy +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'PythonFormatter', - gradleTask: ':pythonFormatterPreCommit', - triggerPathPatterns: [ - '^sdks/python/apache_beam/.*$', - ] - ) -builder.build() diff --git a/.test-infra/jenkins/job_PreCommit_PythonDocs.groovy b/.test-infra/jenkins/job_PreCommit_PythonDocs.groovy deleted file mode 100644 index 17202263493c6..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_PythonDocs.groovy +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder -import CommonJobProperties as common - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'PythonDocs', - gradleTask: ':pythonDocsPreCommit', - timeoutMins: 30, - triggerPathPatterns: [ - '^sdks/python/.*$', - ] - ) -builder.build { - publishers {} -} diff --git a/.test-infra/jenkins/job_PreCommit_PythonLint.groovy b/.test-infra/jenkins/job_PreCommit_PythonLint.groovy deleted file mode 100644 index 118ca7b412b71..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_PythonLint.groovy +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'PythonLint', - gradleTask: ':pythonLintPreCommit', - triggerPathPatterns: [ - '^sdks/python/.*$', - '^release/.*$', - ] - ) -builder.build() diff --git a/.test-infra/jenkins/job_PreCommit_Python_Coverage.groovy b/.test-infra/jenkins/job_PreCommit_Python_Coverage.groovy index c0cb48cf62319..43a204fd7cfc8 100644 --- a/.test-infra/jenkins/job_PreCommit_Python_Coverage.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python_Coverage.groovy @@ -22,6 +22,9 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( scope: this, nameBase: 'Python_Coverage', gradleTask: ':sdks:python:test-suites:tox:py38:preCommitPyCoverage', + gradleSwitches: [ + '-PuseWheelDistribution' + ], timeoutMins: 180, triggerPathPatterns: [ '^model/.*$', diff --git a/.test-infra/jenkins/job_PreCommit_Python_Dataframes.groovy b/.test-infra/jenkins/job_PreCommit_Python_Dataframes.groovy index e2914e9bdb8e0..dea034f613a58 100644 --- a/.test-infra/jenkins/job_PreCommit_Python_Dataframes.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python_Dataframes.groovy @@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Python_Dataframes', gradleTask: ':pythonPreCommit', gradleSwitches: [ - '-Pposargs=apache_beam/dataframe/' + '-Pposargs=apache_beam/dataframe/', + '-PuseWheelDistribution' ], timeoutMins: 180, triggerPathPatterns: [ diff --git a/.test-infra/jenkins/job_PreCommit_Python_Examples.groovy b/.test-infra/jenkins/job_PreCommit_Python_Examples.groovy index f4ef9f51d7fbe..3dd7bf6f6f47c 100644 --- a/.test-infra/jenkins/job_PreCommit_Python_Examples.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python_Examples.groovy @@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Python_Examples', gradleTask: ':pythonPreCommit', gradleSwitches: [ - '-Pposargs=apache_beam/examples/' + '-Pposargs=apache_beam/examples/', + '-PuseWheelDistribution' ], timeoutMins: 180, triggerPathPatterns: [ diff --git a/.test-infra/jenkins/job_PreCommit_Python_Runners.groovy b/.test-infra/jenkins/job_PreCommit_Python_Runners.groovy index e80dba6cf5cd8..4ae1d283b7a9b 100644 --- a/.test-infra/jenkins/job_PreCommit_Python_Runners.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python_Runners.groovy @@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Python_Runners', gradleTask: ':pythonPreCommit', gradleSwitches: [ - '-Pposargs=apache_beam/runners/' + '-Pposargs=apache_beam/runners/', + '-PuseWheelDistribution' ], timeoutMins: 180, triggerPathPatterns: [ diff --git a/.test-infra/jenkins/job_PreCommit_Python_Transforms.groovy b/.test-infra/jenkins/job_PreCommit_Python_Transforms.groovy index dd16d48b1731a..ccd3f08b78ab0 100644 --- a/.test-infra/jenkins/job_PreCommit_Python_Transforms.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python_Transforms.groovy @@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Python_Transforms', gradleTask: ':pythonPreCommit', gradleSwitches: [ - '-Pposargs=apache_beam/transforms/' + '-Pposargs=apache_beam/transforms/', + '-PuseWheelDistribution' ], timeoutMins: 180, triggerPathPatterns: [ diff --git a/.test-infra/jenkins/job_PreCommit_RAT.groovy b/.test-infra/jenkins/job_PreCommit_RAT.groovy deleted file mode 100644 index 613caa9af0de6..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_RAT.groovy +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'RAT', - gradleTask: ':rat' - ) -builder.build() diff --git a/.test-infra/jenkins/job_PreCommit_SQL.groovy b/.test-infra/jenkins/job_PreCommit_SQL.groovy deleted file mode 100644 index 3cd81e330cac4..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_SQL.groovy +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'SQL', - gradleTask: ':sqlPreCommit', - gradleSwitches: [ - '-PdisableSpotlessCheck=true', - '-PdisableCheckStyle=true', - '-PenableJacocoReport' - ], // spotless checked in job_PreCommit_Spotless - triggerPathPatterns: [ - '^sdks/java/extensions/sql.*$', - ], - numBuildsToRetain: 40 - ) -builder.build { - publishers { - archiveJunit('**/build/test-results/**/*.xml') - recordIssues { - tools { - errorProne() - spotBugs { - pattern('**/build/reports/spotbugs/*.xml') - } - } - enabledForFailure(true) - } - jacocoCodeCoverage { - execPattern('**/build/jacoco/*.exec') - exclusionPattern('**/AutoValue_*') - inclusionPattern("**/org/apache/beam/sdk/extensions/sql/**") - } - } -} diff --git a/.test-infra/jenkins/job_PreCommit_SQL_Java11.groovy b/.test-infra/jenkins/job_PreCommit_SQL_Java11.groovy deleted file mode 100644 index 9742ab756cf70..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_SQL_Java11.groovy +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder -import CommonJobProperties as properties - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'SQL_Java11', - gradleTask: ':sqlPreCommit', - gradleSwitches: [ - '-PdisableSpotlessCheck=true', - '-PdisableCheckStyle=true', - '-PcompileAndRunTestsWithJava11', - '-PskipCheckerFramework', - // Gradle itself is running under JDK8 so plugin configures wrong for JDK11 - "-Pjava11Home=${properties.JAVA_11_HOME}" - ], // spotless checked in job_PreCommit_Spotless - triggerPathPatterns: [ - '^sdks/java/extensions/sql.*$', - ] - ) -builder.build { - publishers { - archiveJunit('**/build/test-results/**/*.xml') - recordIssues { - tools { - errorProne() - java() - spotBugs { - pattern('**/build/reports/spotbugs/*.xml') - } - } - enabledForFailure(true) - } - } -} diff --git a/.test-infra/jenkins/job_PreCommit_SQL_Java17.groovy b/.test-infra/jenkins/job_PreCommit_SQL_Java17.groovy deleted file mode 100644 index 158fa683c1a8a..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_SQL_Java17.groovy +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder -import CommonJobProperties as properties - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'SQL_Java17', - gradleTask: ':sqlPreCommit', - gradleSwitches: [ - '-PdisableSpotlessCheck=true', - '-PdisableCheckStyle=true', - '-PcompileAndRunTestsWithJava17', - '-PskipCheckerFramework', - // Gradle itself is running under JDK8 so plugin configures wrong for JDK17 - "-Pjava17Home=${properties.JAVA_17_HOME}" - ], // spotless checked in job_PreCommit_Spotless - triggerPathPatterns: [ - '^sdks/java/extensions/sql.*$', - ] - ) -builder.build { - publishers { - archiveJunit('**/build/test-results/**/*.xml') - recordIssues { - tools { - java() - spotBugs { - pattern('**/build/reports/spotbugs/*.xml') - } - } - enabledForFailure(true) - } - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Spotless.groovy b/.test-infra/jenkins/job_PreCommit_Spotless.groovy deleted file mode 100644 index a9da1ad5491a8..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_Spotless.groovy +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Spotless', - gradleTask: 'spotlessCheck checkStyleMain checkStyleTest', - triggerPathPatterns: [ - '^buildSrc/.*$', - '^sdks/java/.*$', - '^runners/.*$', - '^examples/java/.*$', - '^examples/kotlin/.*$', - '^.test-infra/jenkins/.*$', - ] - ) -builder.build { - publishers { - recordIssues { - tools { - checkStyle { - pattern('**/build/reports/checkstyle/*.xml') - } - } - enabledForFailure(true) - } - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Typescript.groovy b/.test-infra/jenkins/job_PreCommit_Typescript.groovy deleted file mode 100644 index 29cb93ede8b6d..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_Typescript.groovy +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Typescript', - gradleTask: ':typescriptPreCommit', - triggerPathPatterns: [ - '^sdks/python/apache_beam/runners/interactive/extensions/.*$', - ] - ) -builder.build() diff --git a/.test-infra/jenkins/job_PreCommit_Website.groovy b/.test-infra/jenkins/job_PreCommit_Website.groovy deleted file mode 100644 index 73014819ed004..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_Website.groovy +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Website', - gradleTask: ':websitePreCommit', - triggerPathPatterns: ['^website/.*$']) -builder.build() - diff --git a/.test-infra/jenkins/job_PreCommit_Website_Stage_GCS.groovy b/.test-infra/jenkins/job_PreCommit_Website_Stage_GCS.groovy deleted file mode 100644 index e2f7202d14eb4..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_Website_Stage_GCS.groovy +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Website_Stage_GCS', - gradleTask: ':website:stageWebsite', - triggerPathPatterns: ['^website/.*$']) -builder.build { - publishers { - buildDescription(/Website published to (http:\/\/.+\/index.html)/) - } -} - diff --git a/.test-infra/jenkins/job_PreCommit_Whitespace.groovy b/.test-infra/jenkins/job_PreCommit_Whitespace.groovy deleted file mode 100644 index 1b8341395a684..0000000000000 --- a/.test-infra/jenkins/job_PreCommit_Whitespace.groovy +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Whitespace', - gradleTask: ':whitespacePreCommit', - triggerPathPatterns: [ - '.*\\.md$', - '.*build\\.gradle$', - ] - ) -builder.build() diff --git a/.test-infra/jenkins/job_Precommit_Java_Examples_Dataflow_Java17.groovy b/.test-infra/jenkins/job_Precommit_Java_Examples_Dataflow_Java17.groovy deleted file mode 100644 index 29a37cf4b5211..0000000000000 --- a/.test-infra/jenkins/job_Precommit_Java_Examples_Dataflow_Java17.groovy +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder -import CommonJobProperties as properties - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Java_Examples_Dataflow_Java17', - gradleTask: ':clean', - gradleSwitches: [ - '-PdisableSpotlessCheck=true', - '-PdisableCheckStyle=true', - '-PskipCheckerFramework' // Gradle itself is running under JDK8 so plugin configures wrong for JDK17 - ], // spotless checked in separate pre-commit - triggerPathPatterns: [ - '^model/.*$', - '^sdks/java/.*$', - '^runners/google-cloud-dataflow-java/.*$', - '^examples/java/.*$', - '^examples/kotlin/.*$', - '^release/.*$', - ], - timeoutMins: 60, - ) -builder.build { - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } - - steps { - gradle { - rootBuildScriptDir(properties.checkoutDir) - tasks 'javaExamplesDataflowPreCommit' - switches '-PdisableSpotlessCheck=true' - switches '-PdisableCheckStyle=true' - switches '-PskipCheckerFramework' // Gradle itself is running under JDK8 so plugin configures wrong for JDK17 - switches '-PcompileAndRunTestsWithJava17' - switches "-Pjava17Home=${properties.JAVA_17_HOME}" - properties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors()) - } - } -} diff --git a/.test-infra/jenkins/job_Publish_Docker_Snapshots.groovy b/.test-infra/jenkins/job_Publish_Docker_Snapshots.groovy index e313b93925528..510acd8f37d8e 100644 --- a/.test-infra/jenkins/job_Publish_Docker_Snapshots.groovy +++ b/.test-infra/jenkins/job_Publish_Docker_Snapshots.groovy @@ -41,9 +41,7 @@ job('beam_Publish_Docker_Snapshots') { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) commonJobProperties.setGradleSwitches(delegate) - SUPPORTED_CONTAINER_TASKS.each { taskVer -> - tasks(":sdks:python:container:${taskVer}:dockerPush") - } + tasks(":runners:spark:${CommonTestProperties.getSparkVersion()}:job-server:container:dockerPush") tasks(":runners:flink:${CommonTestProperties.getFlinkVersion()}:job-server-container:dockerPush") switches("-Pdocker-repository-root=gcr.io/apache-beam-testing/beam_portability") switches("-Pdocker-tag=latest") diff --git a/.test-infra/jenkins/job_Release_Gradle_Build.groovy b/.test-infra/jenkins/job_Release_Gradle_Build.groovy deleted file mode 100644 index ba3efeca85fe1..0000000000000 --- a/.test-infra/jenkins/job_Release_Gradle_Build.groovy +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties - -// This job runs complete cycle of Gradle build against the official release -// version. Release manager should use this job to verify a release branch -// after cut. -job('beam_Release_Gradle_Build') { - description('Verify Gradle build against the official release version.') - - // Set common parameters. - commonJobProperties - .setTopLevelMainJobProperties( - delegate, - defaultBranch='master', - defaultTimeout=300) - - // Allows triggering this build against pull requests. - commonJobProperties.enablePhraseTriggeringFromPullRequest( - delegate, - 'Release_Build ("Run Release Gradle Build")', - 'Run Release Gradle Build') - - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks('build') - commonJobProperties.setGradleSwitches(delegate) - switches('-PisRelease') - switches('--stacktrace') - } - } -} diff --git a/.test-infra/jenkins/job_Release_Python_NightlySnapshot.groovy b/.test-infra/jenkins/job_Release_Python_NightlySnapshot.groovy deleted file mode 100644 index 19e84d19a67bb..0000000000000 --- a/.test-infra/jenkins/job_Release_Python_NightlySnapshot.groovy +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties - -// This creates the Python nightly snapshot build. Into gs://beam-python-nightly-snapshots. -job('beam_Release_Python_NightlySnapshot') { - description('Publish a nightly snapshot for Python SDK.') - - // Execute concurrent builds if necessary. - concurrentBuild() - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - - // This is a post-commit job that runs once per day, not for every push. - commonJobProperties.setAutoJob( - delegate, - '@daily', - 'builds@beam.apache.org') - - // Allows triggering this build against pull requests. - commonJobProperties.enablePhraseTriggeringFromPullRequest( - delegate, - 'Create Python SDK Nightly Snapshot', - 'Run Python Publish') - - steps { - // Cleanup Python directory. - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:clean') - commonJobProperties.setGradleSwitches(delegate) - } - // Build snapshot. - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:buildSnapshot') - commonJobProperties.setGradleSwitches(delegate) - } - // Publish snapshot to a public accessible GCS directory. - shell('cd ' + commonJobProperties.checkoutDir + - ' && bash sdks/python/scripts/run_snapshot_publish.sh') - } -} diff --git a/.test-infra/jenkins/job_sonarqube_report.groovy b/.test-infra/jenkins/job_sonarqube_report.groovy deleted file mode 100644 index 515b7e43061e6..0000000000000 --- a/.test-infra/jenkins/job_sonarqube_report.groovy +++ /dev/null @@ -1,55 +0,0 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties - -job('beam_sonarqube_report') { - commonJobProperties.setTopLevelMainJobProperties( - delegate, 'master', 120, - true) - - /** - * https://issues.jenkins-ci.org/browse/JENKINS-42741 - */ - wrappers { - withSonarQubeEnv { - installationName('ASF Sonar Analysis') - } - } - - - // TODO(https://github.com/apache/beam/issues/24768) remove or fix this job. - // commonJobProperties.setAutoJob delegate - - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } - - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks("test") - tasks("jacocoTestReport") - tasks("sonarqube") - switches("--continue") - switches("-PdisableSpotlessCheck=true") - switches("-PdisableCheckStyle=true") - // disable parallelization to avoid output collisions - switches("--no-parallel") - } - } -} diff --git a/.test-infra/jenkins/metrics_report/tox.ini b/.test-infra/jenkins/metrics_report/tox.ini index 6ea39e0ab3927..dbf68016c57b2 100644 --- a/.test-infra/jenkins/metrics_report/tox.ini +++ b/.test-infra/jenkins/metrics_report/tox.ini @@ -27,10 +27,10 @@ commands_pre = [testenv:py38-test] deps = -r requirements.txt -passenv = WORKSPACE INFLUXDB_USER INFLUXDB_USER_PASSWORD +passenv = WORKSPACE,INFLUXDB_USER,INFLUXDB_USER_PASSWORD commands = python -m unittest dashboards_parser.py [testenv:py38-generate-report] deps = -r requirements.txt -passenv = WORKSPACE INFLUXDB_USER INFLUXDB_USER_PASSWORD +passenv = WORKSPACE,INFLUXDB_USER,INFLUXDB_USER_PASSWORD commands = python report_generator.py {posargs} diff --git a/.test-infra/jupyter/OWNERS b/.test-infra/jupyter/OWNERS deleted file mode 100644 index 586139172cea3..0000000000000 --- a/.test-infra/jupyter/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - udim diff --git a/.test-infra/kafka/bitnami/README.md b/.test-infra/kafka/bitnami/README.md new file mode 100644 index 0000000000000..0e9ae249d6462 --- /dev/null +++ b/.test-infra/kafka/bitnami/README.md @@ -0,0 +1,113 @@ + + +# Overview + +This module provisions a Bitnami Kafka Cluster based on its +[helm chart](https://github.com/bitnami/charts/tree/main/bitnami/kafka). +It uses the [terraform helm provider](https://registry.terraform.io/providers/hashicorp/helm/latest/docs) +Therefore, you DO NOT need helm to apply this module. + +# Requirements + +- [terraform](https://terraform.io) +- Connection to a kubernetes cluster (See: [.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/google-kubernetes-engine](../../terraform/google-cloud-platform/google-kubernetes-engine) in this repository) +- [kubectl](https://kubernetes.io/docs/reference/kubectl/) cli + +# Usage + +Simply follow standard terraform workflow to apply this module. + +``` +terraform init +terraform apply +``` + +# Special note about GKE Autopilot + +When applying this module to +[Google Kubernetes Engine (GKE) Autopilot](https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-overview) +you will see an "Unschedulable" status. This is because, GKE Autopilot +needs time to scale up the node. After some time, the kubernetes cluster +will provision the kafka cluster when these compute resources are available. + +# Debugging and Troubleshooting + +This module deploys a kafka client on the cluster to help with debugging and +troubleshooting. + +## Query the kafka client pod name + +Run the following command to query the pod name. + +``` +kubectl get po -l app=kafka-client +``` + +You should see something similar to the following: +``` +NAME READY STATUS RESTARTS AGE +kafka-client-cdc7c8885-nmcjc 1/1 Running 0 4m12s +``` + +## Get a shell to the running container + +Run the following command to shell into the running container. + +``` +kubectl exec --stdin --tty kafka-client-cdc7c8885-nmcjc -- /bin/bash +``` + +## Execute kafka commands + +The container executes using the latest [bitnami/kafka](https://hub.docker.com/r/bitnami/kafka/) +image. It has installed all the necessary `kafka-*.sh` scripts in its path. + +In all of the commands, you can use the flag: `--bootstrap-server kafka:9092` +because the pod is in the same kubernetes cluster and takes advantage +of its domain name service (DNS). The bitnami helm operator creates a Kubernetes +service called `kafka` that exposes port `9092`. + +### Get the cluster-id + +The following command gives you the cluster ID and validates you can connect to +the cluster. + +``` +kafka-cluster.sh cluster-id --bootstrap-server kafka:9092 +``` + +### Create a topic + +The following command creates a Kafka topic. + +``` +kafka-topics.sh --create --topic some-topic --partitions 3 --replication-factor 3 --bootstrap-server kafka:9092 +``` + +### Get information about a topic + +The following command queries information about a topic +(assuming the name `some-topic`). + +``` +kafka-topics.sh --describe --topic some-topic --bootstrap-server kafka:9092 +``` + +See https://kubernetes.io/docs/tasks/debug/debug-application/get-shell-running-container/ diff --git a/.test-infra/kafka/bitnami/kafka.tf b/.test-infra/kafka/bitnami/kafka.tf new file mode 100644 index 0000000000000..d9330f19ec150 --- /dev/null +++ b/.test-infra/kafka/bitnami/kafka.tf @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Provision the kafka cluster using the Bitnami helm chart. +resource "helm_release" "kafka" { + wait = false + repository = "https://charts.bitnami.com/bitnami" + chart = "kafka" + name = "kafka" + set { + name = "listeners.client.protocol" + value = "PLAINTEXT" + } + set { + name = "listeners.interbroker.protocol" + value = "PLAINTEXT" + } + set { + name = "listeners.external.protocol" + value = "PLAINTEXT" + } + set { + name = "externalAccess.enabled" + value = "true" + } + set { + name = "externalAccess.autoDiscovery.enabled" + value = "true" + } + set { + name = "rbac.create" + value = "true" + } + set { + name = "service.annotations" + value = yamlencode({"networking.gke.io/load-balancer-type": "Internal" }) + } + set { + name = "externalAccess.service.broker.ports.external" + value = "9094" + } + set { + name = "externalAccess.service.controller.containerPorts.external" + value = "9094" + } + set_list { + name = "externalAccess.controller.service.loadBalancerAnnotations" + value = [ + yamlencode({"networking.gke.io/load-balancer-type": "Internal" }), + yamlencode({"networking.gke.io/load-balancer-type": "Internal" }), + yamlencode({"networking.gke.io/load-balancer-type": "Internal" }), + ] + } + set_list { + name = "externalAccess.broker.service.loadBalancerAnnotations" + value = [ + yamlencode({"networking.gke.io/load-balancer-type": "Internal" }), + yamlencode({"networking.gke.io/load-balancer-type": "Internal" }), + yamlencode({"networking.gke.io/load-balancer-type": "Internal" }), + ] + } +} + +// Provision a kafka client to validate and debug the cluster. +resource "kubernetes_deployment" "kafka_client" { + wait_for_rollout = false + metadata { + name = "kafka-client" + labels = { + app = "kafka-client" + } + } + spec { + selector { + match_labels = { + app = "kafka-client" + } + } + template { + metadata { + labels = { + app = "kafka-client" + } + } + spec { + container { + name = "kafka-client" + image = "bitnami/kafka:latest" + image_pull_policy = "IfNotPresent" + command = ["/bin/bash"] + args = [ + "-c", + "while true; do sleep 2; done", + ] + } + } + } + } +} \ No newline at end of file diff --git a/.test-infra/kafka/bitnami/provider.tf b/.test-infra/kafka/bitnami/provider.tf new file mode 100644 index 0000000000000..8bb92dcf219a8 --- /dev/null +++ b/.test-infra/kafka/bitnami/provider.tf @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "kubernetes" { + config_path = "~/.kube/config" +} + +provider "helm" { + kubernetes { + config_path = "~/.kube/config" + } +} diff --git a/.test-infra/kubernetes/OWNERS b/.test-infra/kubernetes/OWNERS deleted file mode 100644 index 2a140107225d9..0000000000000 --- a/.test-infra/kubernetes/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lgajowy - - kkucharc diff --git a/.test-infra/kubernetes/kafka-cluster/04-outside-services/outside-0.yml b/.test-infra/kubernetes/kafka-cluster/04-outside-services/outside-0.yml index e7513ec1b89a9..b5dd678c2a23d 100644 --- a/.test-infra/kubernetes/kafka-cluster/04-outside-services/outside-0.yml +++ b/.test-infra/kubernetes/kafka-cluster/04-outside-services/outside-0.yml @@ -25,5 +25,4 @@ spec: - protocol: TCP targetPort: 9094 port: 32400 - nodePort: 32400 type: LoadBalancer diff --git a/.test-infra/kubernetes/kafka-cluster/04-outside-services/outside-1.yml b/.test-infra/kubernetes/kafka-cluster/04-outside-services/outside-1.yml index 50e5fb0650bd4..af079b7fdf757 100644 --- a/.test-infra/kubernetes/kafka-cluster/04-outside-services/outside-1.yml +++ b/.test-infra/kubernetes/kafka-cluster/04-outside-services/outside-1.yml @@ -24,6 +24,5 @@ spec: ports: - protocol: TCP targetPort: 9094 - port: 32401 - nodePort: 32401 + port: 32400 type: LoadBalancer diff --git a/.test-infra/kubernetes/kafka-cluster/04-outside-services/outside-2.yml b/.test-infra/kubernetes/kafka-cluster/04-outside-services/outside-2.yml index 87c324b8eea99..e5c3090673f99 100644 --- a/.test-infra/kubernetes/kafka-cluster/04-outside-services/outside-2.yml +++ b/.test-infra/kubernetes/kafka-cluster/04-outside-services/outside-2.yml @@ -24,6 +24,5 @@ spec: ports: - protocol: TCP targetPort: 9094 - port: 32402 - nodePort: 32402 + port: 32400 type: LoadBalancer diff --git a/.test-infra/kubernetes/singlestore/sdb-cluster.yaml b/.test-infra/kubernetes/singlestore/sdb-cluster.yaml index 655a8bfb5240c..8acb0b7a7f747 100644 --- a/.test-infra/kubernetes/singlestore/sdb-cluster.yaml +++ b/.test-infra/kubernetes/singlestore/sdb-cluster.yaml @@ -22,7 +22,7 @@ spec: adminHashedPassword: "*9177CC8207174BDBB5ED66B2140C75171283F15D" nodeImage: repository: singlestore/node - tag: alma-7.5.22-8b82f8a84e + tag: latest redundancyLevel: 1 diff --git a/.test-infra/kubernetes/singlestore/sdb-operator.yaml b/.test-infra/kubernetes/singlestore/sdb-operator.yaml index 5c6bb1f76e133..fc2b2585b24a0 100644 --- a/.test-infra/kubernetes/singlestore/sdb-operator.yaml +++ b/.test-infra/kubernetes/singlestore/sdb-operator.yaml @@ -32,7 +32,7 @@ spec: serviceAccountName: sdb-operator containers: - name: sdb-operator - image: singlestore/operator:3.0.32-db8f5aff + image: singlestore/operator:3.0.98-156a0090 imagePullPolicy: Always args: [ # Cause the operator to merge rather than replace annotations on services diff --git a/.test-infra/metrics/OWNERS b/.test-infra/metrics/OWNERS deleted file mode 100644 index dcdc81963674a..0000000000000 --- a/.test-infra/metrics/OWNERS +++ /dev/null @@ -1,8 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - Ardagan - - alanmyrvold - - markflyhigh - - yifanzou - diff --git a/.test-infra/metrics/build.gradle b/.test-infra/metrics/build.gradle index 679ecd35735d5..f1ecba05f84df 100644 --- a/.test-infra/metrics/build.gradle +++ b/.test-infra/metrics/build.gradle @@ -27,7 +27,7 @@ ext { cluster = 'metrics' zone='us-central1-a' projectName='apache-beam-testing' - kubeConfigPath='/home/jenkins/.kube/config' + kubeConfigPath = project.properties['KUBE_CONFIG_PATH'] ?: '/home/jenkins/.kube/config' } applyGroovyNature() @@ -69,7 +69,7 @@ task removeKubeConfig(type: Exec) { task validateK8sConfiguration(type: Exec) { - commandLine 'sh', '-c', 'kubectl apply --dry-run=client -Rf kubernetes' + commandLine 'sh', '-c', 'kubectl --namespace default apply --dry-run=client -Rf kubernetes' } task validateConfiguration { @@ -102,11 +102,11 @@ task deploy { def stdout = new ByteArrayOutputStream() exec { executable 'sh' - args '-c', 'kubectl get deployment beamgrafana -o jsonpath="{..image}"' + args '-c', 'kubectl --namespace default get deployment beamgrafana -o jsonpath="{..image}"' standardOutput = stdout } - // All images have the same tag, it doesn't matter which we choose. + // All images have the same tag, it doesn't matter which we choose. String image = (stdout.toString().split(' ') as List)[0] String currentImageTag = (image.split(':') as List)[1] println "Current image tag: ${currentImageTag}" @@ -122,7 +122,7 @@ task deploy { exec { executable 'sh' - args '-c', "kubectl set image deployment/beamgrafana \ + args '-c', "kubectl --namespace default set image deployment/beamgrafana \ beamgrafana=gcr.io/apache-beam-testing/beamgrafana:${project.lastCommitId} \ beammetricssyncjenkins=gcr.io/apache-beam-testing/beammetricssyncjenkins:${project.lastCommitId} \ beammetricssyncgithub=gcr.io/apache-beam-testing/beammetricssyncgithub:${project.lastCommitId}" diff --git a/.test-infra/metrics/docker-compose.yml b/.test-infra/metrics/docker-compose.yml index 77f07bad13fa6..3d847ff796761 100644 --- a/.test-infra/metrics/docker-compose.yml +++ b/.test-infra/metrics/docker-compose.yml @@ -85,7 +85,10 @@ services: - DB_DBNAME=beam_metrics - DB_DBUSERNAME=admin - DB_DBPWD= - - GH_ACCESS_TOKEN= + - GH_APP_ID= + - GH_APP_INSTALLATION_ID= + - GH_PEM_KEY= + - GH_NUMBER_OF_WORKFLOW_RUNS_TO_FETCH=30 syncjenkins: image: syncjenkins container_name: beamsyncjenkins diff --git a/.test-infra/metrics/grafana/Dockerfile b/.test-infra/metrics/grafana/Dockerfile index bfff4ec519582..1f02e460dbd95 100644 --- a/.test-infra/metrics/grafana/Dockerfile +++ b/.test-infra/metrics/grafana/Dockerfile @@ -16,7 +16,7 @@ # limitations under the License. ################################################################################ -FROM grafana/grafana:8.1.8 +FROM grafana/grafana:10.0.3 RUN grafana-cli plugins install marcusolsson-json-datasource diff --git a/.test-infra/metrics/grafana/dashboards/perftests_metrics/Python_WordCount_IT_Benchmarks.json b/.test-infra/metrics/grafana/dashboards/perftests_metrics/Python_WordCount_IT_Benchmarks.json index ef49855314535..02e707b68bdf0 100644 --- a/.test-infra/metrics/grafana/dashboards/perftests_metrics/Python_WordCount_IT_Benchmarks.json +++ b/.test-infra/metrics/grafana/dashboards/perftests_metrics/Python_WordCount_IT_Benchmarks.json @@ -224,7 +224,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "WordCountIT Batch 1Gb Files - py37", + "title": "WordCountIT Batch 1Gb Files - py38", "tooltip": { "shared": true, "sort": 0, diff --git a/.test-infra/metrics/src/test/groovy/ProberTests.groovy b/.test-infra/metrics/src/test/groovy/ProberTests.groovy index 4f6cce7f49da9..c5de9ca64c8a3 100644 --- a/.test-infra/metrics/src/test/groovy/ProberTests.groovy +++ b/.test-infra/metrics/src/test/groovy/ProberTests.groovy @@ -27,7 +27,7 @@ import static groovy.test.GroovyAssert.shouldFail */ class ProberTests { // TODO: Make this configurable - def grafanaEndpoint = 'http://104.154.241.245' + def grafanaEndpoint = 'http://metrics.beam.apache.org' @Test void PingGrafanaHttpApi() { diff --git a/.test-infra/metrics/sync/github/requirements.txt b/.test-infra/metrics/sync/github/requirements.txt index 14a64686e55a4..5b231565459fd 100644 --- a/.test-infra/metrics/sync/github/requirements.txt +++ b/.test-infra/metrics/sync/github/requirements.txt @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -requests +aiohttp +backoff psycopg2-binary -ddt - +PyGithub \ No newline at end of file diff --git a/.test-infra/metrics/sync/github/sync_workflows.py b/.test-infra/metrics/sync/github/sync_workflows.py index a83d518af4fb2..25065878c4bdc 100644 --- a/.test-infra/metrics/sync/github/sync_workflows.py +++ b/.test-infra/metrics/sync/github/sync_workflows.py @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -6,182 +5,267 @@ # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + ''' -This module queries GitHub to collect Beam-related workflows metrics and put them in -PostgreSQL. -This Script is running every 3 hours in a cloud function in apache-beam-testing project. -This cloud function is triggered by a pubsub topic. -You can find the cloud function in the next link +This module queries GitHub API to collect Beam-related workflows metrics and +put them in PostgreSQL. +This script is running every 3 hours as a cloud function +"github_actions_workflows_dashboard_sync" in apache-beam-testing project: https://console.cloud.google.com/functions/details/us-central1/github_actions_workflows_dashboard_sync?env=gen1&project=apache-beam-testing -Pub sub topic : https://console.cloud.google.com/cloudpubsub/topic/detail/github_actions_workflows_sync?project=apache-beam-testing -Cron Job : https://console.cloud.google.com/cloudscheduler/jobs/edit/us-central1/github_actions_workflows_dashboard_sync?project=apache-beam-testing -Writing the latest 10 runs of every postcommit workflow in master branch in a beammetrics database +This cloud function is triggered by a pubsub topic: +https://console.cloud.google.com/cloudpubsub/topic/detail/github_actions_workflows_sync?project=apache-beam-testing +Cron Job: +https://console.cloud.google.com/cloudscheduler/jobs/edit/us-central1/github_actions_workflows_dashboard_sync?project=apache-beam-testing ''' +import asyncio +import aiohttp +import backoff +import math import os import sys import time import re -import requests import psycopg2 - -from datetime import datetime -from github import GithubIntegration +from github import GithubIntegration DB_HOST = os.environ['DB_HOST'] DB_PORT = os.environ['DB_PORT'] DB_NAME = os.environ['DB_DBNAME'] DB_USER_NAME = os.environ['DB_DBUSERNAME'] DB_PASSWORD = os.environ['DB_DBPWD'] -GH_WORKFLOWS_TABLE_NAME = "github_workflows" -# Number of workflows that fetch github API -GH_NUMBER_OF_WORKFLOWS = 100 -GH_WORKFLOWS_NUMBER_EXECUTIONS = 100 -WORKFLOWS_OBJECT_LIST = [] +GH_APP_ID = os.environ['GH_APP_ID'] +GH_APP_INSTALLATION_ID = os.environ['GH_APP_INSTALLATION_ID'] +GH_PEM_KEY = os.environ['GH_PEM_KEY'] +GH_NUMBER_OF_WORKFLOW_RUNS_TO_FETCH =\ + os.environ['GH_NUMBER_OF_WORKFLOW_RUNS_TO_FETCH'] class Workflow: - def __init__(self,id,name,filename): - self.id = id - self.name = name - self.filename = filename - self.listOfRuns = [] - self.runUrl = [] - -# The table will save the latest ten run of every workflow -GH_WORKFLOWS_CREATE_TABLE_QUERY = f""" -CREATE TABLE IF NOT EXISTS {GH_WORKFLOWS_TABLE_NAME} ( - job_name text PRIMARY KEY, - job_yml_filename text""" -for i in range(0,GH_WORKFLOWS_NUMBER_EXECUTIONS): - i = i + 1 - GH_WORKFLOWS_CREATE_TABLE_QUERY += """,\n run{} text, - run{}Id text""".format(str(i),str(i)) -GH_WORKFLOWS_CREATE_TABLE_QUERY += ")\n" - -def githubWorkflowsGrafanaSync(data,context): - print('Started') - print('Updating table with recent workflow runs') - databaseOperations(initDbConnection(),fetchWorkflowData()) - print('Done') - return "Completed" - -def initDbConnection(): - '''Init connection with the Database''' - connection = None - maxRetries = 3 - i = 0 - while connection == None and i < maxRetries: - try: - connection = psycopg2.connect( - f"dbname='{DB_NAME}' user='{DB_USER_NAME}' host='{DB_HOST}'" - f" port='{DB_PORT}' password='{DB_PASSWORD}'") - except Exception as e: - print('Failed to connect to DB; retrying in 1 minute') - print(e) - time.sleep(60) - i = i + 1 - if i >= maxRetries: - print("Number of retries exceded ") - sys.exit(1) - return connection - -def getToken(): - git_integration = GithubIntegration( - os.environ["GH_APP_ID"], - os.environ["GH_PEM_KEY"]) - token=git_integration.get_access_token( - os.environ["GH_APP_INSTALLATION_ID"] - ).token - return token - -def retriesRequest(request): - requestSucceeded = False - retryFactor = 1 - while not requestSucceeded: - retryTime = 60 * retryFactor - if request.status_code != 200: - print('Failed to get the request with code {}'.format(request.status_code)) - time.sleep(retryTime) - retryFactor = retryFactor + retryFactor - if retryFactor * 60 >= 3600: - print("Error: The request take more than an hour") - sys.exit(1) - else: - requestSucceeded = True -def fetchWorkflowData(): - '''Return a json with all the workflows and the latests - ten executions''' - completed = False - page = 1 - workflows = [] + def __init__(self, id, name, filename): + self.id = id + self.name = name + self.filename = filename + self.runs = [] + +async def github_workflows_dashboard_sync(): + print('Started') + print('Updating table with recent workflow runs') + + if not GH_NUMBER_OF_WORKFLOW_RUNS_TO_FETCH or \ + not GH_NUMBER_OF_WORKFLOW_RUNS_TO_FETCH.isdigit(): + raise ValueError( + 'The number of workflow runs to fetch is not specified or not an integer' + ) + + database_operations(init_db_connection(), await fetch_workflow_data()) + + print('Done') + return "Completed" + +def init_db_connection(): + '''Init connection with the Database''' + connection = None + maxRetries = 3 + i = 0 + while connection is None and i < maxRetries: try: - while not completed: - url = "https://api.github.com/repos/apache/beam/actions/workflows" - queryOptions = { 'branch' : 'master', 'page': page, 'per_page' : GH_NUMBER_OF_WORKFLOWS } - response = requests.get(url = url, params = queryOptions) - retriesRequest(response) - jsonResponse = response.json() - if jsonResponse['total_count'] >= GH_NUMBER_OF_WORKFLOWS: - page = page + 1 - workflowsPage = jsonResponse['workflows'] - workflows.append(workflowsPage) - else: - completed = True - workflowsPage = jsonResponse['workflows'] - workflows.append(workflowsPage) - for pageItem in workflows: - for item in pageItem: - path =item['path'] - isPostCommit = re.search('(.*)postcommit(.*)',path) - if isPostCommit: - result = re.search('/(.*).yml', path) - path =(result.group(1)) + ".yml" - workflowObject = Workflow(item['id'],item['name'],path) - WORKFLOWS_OBJECT_LIST.append(workflowObject) - url = "https://api.github.com/repos/apache/beam/actions/workflows/" - queryOptions = { 'branch' : 'master', 'per_page' : GH_WORKFLOWS_NUMBER_EXECUTIONS, - 'page' :'1', 'exclude_pull_request':True } - for workflow in WORKFLOWS_OBJECT_LIST: - response = requests.get(url = "{}{}/runs".format(url,workflow.id), - params=queryOptions) - retriesRequest(response) - responseJson = response.json() - workflowsRuns = responseJson['workflow_runs'] - for item in workflowsRuns: - if item['status'] == 'completed': - workflow.runUrl.append(item['html_url']) - workflow.listOfRuns.append(item['conclusion']) - else: - workflow.listOfRuns.append(item['status']) - workflow.runUrl.append(item['html_url']) - for i in range(0,GH_WORKFLOWS_NUMBER_EXECUTIONS): - if i >= len(workflow.listOfRuns): - workflow.listOfRuns.append('None') - workflow.runUrl.append('None') + connection = psycopg2.connect( + f"dbname='{DB_NAME}' user='{DB_USER_NAME}' host='{DB_HOST}'" + f" port='{DB_PORT}' password='{DB_PASSWORD}'") except Exception as e: - print('Failed to get GHA workflows') - print(e) - -def databaseOperations(connection,fetchWorkflows): - '''Create the table if not exist and update the table with the latest runs - of the workflows ''' - queryInsert = "INSERT INTO {} VALUES ".format(GH_WORKFLOWS_TABLE_NAME) - cursor = connection.cursor() - cursor.execute(GH_WORKFLOWS_CREATE_TABLE_QUERY) - cursor.execute("DELETE FROM {};".format(GH_WORKFLOWS_TABLE_NAME)) - query = "" - for workflow in WORKFLOWS_OBJECT_LIST: - rowInsert = "(\'{}\',\'{}\'".format(workflow.name,workflow.filename) - for run, runUrl in zip(workflow.listOfRuns,workflow.runUrl): - rowInsert += ",\'{}\',\'{}\'".format(run,runUrl) - query = query + rowInsert - query += ")," - query = query[:-1] + ";" - query = queryInsert + query - cursor.execute(query) - cursor.close() - connection.commit() - connection.close() \ No newline at end of file + print('Failed to connect to DB; retrying in 1 minute') + print(e) + time.sleep(60) + i = i + 1 + if i >= maxRetries: + print("Number of retries exceded ") + sys.exit(1) + return connection + +def get_token(): + git_integration = GithubIntegration(GH_APP_ID, GH_PEM_KEY) + token = git_integration.get_access_token(GH_APP_INSTALLATION_ID).token + return f'Bearer {token}' + +@backoff.on_exception(backoff.constant, aiohttp.ClientResponseError, max_tries=5) +async def fetch(url, semaphore, params=None, headers=None, request_id=None): + async with semaphore: + async with aiohttp.ClientSession() as session: + async with session.get(url, params=params, headers=headers) as response: + if response.status == 200: + result = await response.json() + if request_id: + return request_id, result + return result + elif response.status == 403: + print(f'Retry for: {url}') + headers['Authorization'] = get_token() + raise aiohttp.ClientResponseError( + response.request_info, + response.history, + status=response.status, + message=response.reason, + headers=response.headers + ) + +async def fetch_workflow_data(): + def append_workflow_runs(workflow, runs): + for run in runs: + # Getting rid of all runs with a "skipped" status to display + # only actual runs + if run['conclusion'] != 'skipped': + status = '' + if run['status'] == 'completed': + status = run['conclusion'] + elif run['status'] != 'cancelled': + status = run['status'] + workflow.runs.append((int(run['id']), status, run['html_url'])) + + url = "https://api.github.com/repos/apache/beam/actions/workflows" + headers = {'Authorization': get_token()} + page = 1 + number_of_entries_per_page = 100 # The number of results per page (max 100) + params =\ + {'branch': 'master', 'page': page, 'per_page': number_of_entries_per_page} + concurrent_requests = 30 # Number of requests to send simultaneously + semaphore = asyncio.Semaphore(concurrent_requests) + + print("Start fetching recent workflow runs") + workflow_tasks = [] + response = await fetch(url, semaphore, params, headers) + pages_to_fetch =\ + math.ceil(response['total_count'] / number_of_entries_per_page) + while pages_to_fetch >= page: + params = { + 'branch': 'master', + 'page': page, + 'per_page': number_of_entries_per_page + } + workflow_tasks.append(fetch(url, semaphore, params, headers)) + page += 1 + + workflow_run_tasks = [] + for completed_task in asyncio.as_completed(workflow_tasks): + response = await completed_task + workflows = response.get('workflows', []) + for workflow in workflows: + runs_url = f"{url}/{workflow['id']}/runs" + page = 1 + pages_to_fetch = math.ceil( + int(GH_NUMBER_OF_WORKFLOW_RUNS_TO_FETCH) / number_of_entries_per_page + ) + while pages_to_fetch >= page: + params = { + 'branch': 'master', + 'page': page, + 'per_page': number_of_entries_per_page, + 'exclude_pull_requests': 'true' + } + workflow_run_tasks.append(fetch(runs_url, semaphore, params, headers)) + page += 1 + print("Successfully fetched workflow runs") + + print("Start fetching workflow runs details") + workflows = {} + workflow_ids_to_fetch_extra_runs = {} + for completed_task in asyncio.as_completed(workflow_run_tasks): + response = await completed_task + workflow_runs = response.get('workflow_runs') + if workflow_runs: + workflow_id = workflow_runs[0]['workflow_id'] + workflow = workflows.get(workflow_id) + if not workflow: + workflow_name = workflow_runs[0]['name'] + workflow_path = workflow_runs[0]['path'] + result = re.search(r'(workflows\/.*)$', workflow_path) + if result: + workflow_path = result.group(1) + workflow = Workflow(workflow_id, workflow_name, workflow_path) + + append_workflow_runs(workflow, workflow_runs) + workflows[workflow_id] = workflow + if len(workflow.runs) < int(GH_NUMBER_OF_WORKFLOW_RUNS_TO_FETCH): + workflow_ids_to_fetch_extra_runs[workflow_id] = workflow_id + else: + workflow_ids_to_fetch_extra_runs.pop(workflow_id, None) + print(f"Successfully fetched details for: {workflow.filename}") + + page = math.ceil( + int(GH_NUMBER_OF_WORKFLOW_RUNS_TO_FETCH) / number_of_entries_per_page + ) + 1 + # Fetch extra workflow runs if the specified number of runs is not reached + while workflow_ids_to_fetch_extra_runs: + extra_workflow_runs_tasks = [] + for workflow_id in list(workflow_ids_to_fetch_extra_runs.values()): + runs_url = f"{url}/{workflow_id}/runs" + params = { + 'branch': 'master', + 'page': page, + 'per_page': number_of_entries_per_page, + 'exclude_pull_requests': 'true' + } + extra_workflow_runs_tasks.append(fetch(runs_url, semaphore, params, headers, workflow_id)) + for completed_task in asyncio.as_completed(extra_workflow_runs_tasks): + workflow_id, response = await completed_task + workflow = workflows[workflow_id] + print(f"Fetching extra workflow runs for: {workflow.filename}") + workflow_runs = response.get('workflow_runs') + if workflow_runs: + append_workflow_runs(workflow, workflow_runs) + else: + number_of_runs_to_add =\ + int(GH_NUMBER_OF_WORKFLOW_RUNS_TO_FETCH) - len(workflow.runs) + workflow.runs.extend([(0, 'None', 'None')] * number_of_runs_to_add) + if len(workflow.runs) >= int(GH_NUMBER_OF_WORKFLOW_RUNS_TO_FETCH): + workflow_ids_to_fetch_extra_runs.pop(workflow_id, None) + print(f"Successfully fetched extra workflow runs for: {workflow.filename}") + page += 1 + print("Successfully fetched workflow runs details") + + for workflow in list(workflows.values()): + runs = sorted(workflow.runs, key=lambda r: r[0], reverse=True) + workflow.runs = runs[:int(GH_NUMBER_OF_WORKFLOW_RUNS_TO_FETCH)] + + return list(workflows.values()) + +def database_operations(connection, workflows): + # Create the table and update it with the latest workflow runs + if not workflows: + return + cursor = connection.cursor() + workflows_table_name = "github_workflows" + cursor.execute(f"DROP TABLE IF EXISTS {workflows_table_name};") + create_table_query = f""" + CREATE TABLE IF NOT EXISTS {workflows_table_name} ( + workflow_id integer NOT NULL PRIMARY KEY, + job_name text NOT NULL, + job_yml_filename text NOT NULL""" + for i in range(int(GH_NUMBER_OF_WORKFLOW_RUNS_TO_FETCH)): + create_table_query += f""", + run{i+1} text, + run{i+1}Id text""" + create_table_query += ")\n" + cursor.execute(create_table_query) + insert_query = f"INSERT INTO {workflows_table_name} VALUES " + for workflow in workflows: + row_insert =\ + f"(\'{workflow.id}\',\'{workflow.name}\',\'{workflow.filename}\'" + for _, status, url in workflow.runs: + row_insert += f",\'{status}\',\'{url}\'" + insert_query += f"{row_insert})," + insert_query = insert_query[:-1] + ";" + cursor.execute(insert_query) + cursor.close() + connection.commit() + connection.close() + +if __name__ == '__main__': + asyncio.run(github_workflows_dashboard_sync()) diff --git a/.test-infra/metrics/sync/jenkins/syncjenkins.py b/.test-infra/metrics/sync/jenkins/syncjenkins.py index d421094c0456d..32bbf1fff2e93 100644 --- a/.test-infra/metrics/sync/jenkins/syncjenkins.py +++ b/.test-infra/metrics/sync/jenkins/syncjenkins.py @@ -62,7 +62,7 @@ def fetchJobs(): url = ('https://ci-beam.apache.org/api/json' '?tree=jobs[name,url,lastCompletedBuild[id]]&depth=1') r = requests.get(url) - jobs = r.json()[u'jobs'] + jobs = r.json()['jobs'] result = map(lambda x: (x['name'], int(x['lastCompletedBuild']['id']) if x['lastCompletedBuild'] is not None @@ -122,31 +122,31 @@ def fetchBuildsForJob(jobUrl): f'estimatedDuration,fullDisplayName,actions[{durFields}]') url = f'{jobUrl}api/json?depth=1&tree=builds[{fields}]' r = requests.get(url) - return r.json()[u'builds'] + return r.json()['builds'] def buildRowValuesArray(jobName, build): timings = next((x - for x in build[u'actions'] - if (u'_class' in x) - and (x[u'_class'] == u'jenkins.metrics.impl.TimeInQueueAction')), + for x in build['actions'] + if ('_class' in x) + and (x['_class'] == 'jenkins.metrics.impl.TimeInQueueAction')), None) values = [jobName, - int(build[u'id']), - build[u'url'], - build[u'result'], - datetime.fromtimestamp(build[u'timestamp'] / 1000), - build[u'builtOn'], - build[u'duration'], - build[u'estimatedDuration'], - build[u'fullDisplayName'], - timings[u'blockedDurationMillis'] if timings is not None else -1, - timings[u'buildableDurationMillis'] if timings is not None else -1, - timings[u'buildingDurationMillis'] if timings is not None else -1, - timings[u'executingTimeMillis'] if timings is not None else -1, - timings[u'queuingDurationMillis'] if timings is not None else -1, - timings[u'totalDurationMillis'] if timings is not None else -1, - timings[u'waitingDurationMillis'] if timings is not None else -1] + int(build['id']), + build['url'], + build['result'], + datetime.fromtimestamp(build['timestamp'] / 1000), + build['builtOn'], + build['duration'], + build['estimatedDuration'], + build['fullDisplayName'], + timings['blockedDurationMillis'] if timings is not None else -1, + timings['buildableDurationMillis'] if timings is not None else -1, + timings['buildingDurationMillis'] if timings is not None else -1, + timings['executingTimeMillis'] if timings is not None else -1, + timings['queuingDurationMillis'] if timings is not None else -1, + timings['totalDurationMillis'] if timings is not None else -1, + timings['waitingDurationMillis'] if timings is not None else -1] return values @@ -168,16 +168,16 @@ def fetchNewData(): syncedJobId = syncedJobs[newJobName] if newJobName in syncedJobs else -1 if newJobLastBuildId > syncedJobId: builds = fetchBuildsForJob(newJobUrl) - builds = [x for x in builds if int(x[u'id']) > syncedJobId] + builds = [x for x in builds if int(x['id']) > syncedJobId] connection = initConnection() cursor = connection.cursor() for build in builds: - if build[u'building']: + if build['building']: continue; rowValues = buildRowValuesArray(newJobName, build) - print("inserting", newJobName, build[u'id']) + print("inserting", newJobName, build['id']) insertRow(cursor, rowValues) cursor.close() diff --git a/.test-infra/pipelines/build.gradle b/.test-infra/pipelines/build.gradle index 84b41b642615f..a073b6c087ba7 100644 --- a/.test-infra/pipelines/build.gradle +++ b/.test-infra/pipelines/build.gradle @@ -53,7 +53,7 @@ dependencies { implementation library.java.jackson_annotations implementation library.java.jackson_core implementation library.java.jackson_databind - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.google_auth_library_credentials implementation library.java.grpc_auth implementation library.java.protobuf_java diff --git a/.test-infra/pipelines/go.mod b/.test-infra/pipelines/go.mod new file mode 100644 index 0000000000000..0204d037a4cca --- /dev/null +++ b/.test-infra/pipelines/go.mod @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This module contains all Go code for internal use by .test-infra pipelines. +module github.com/apache/beam/test-infra/pipelines + +go 1.20 + +require github.com/google/go-cmp v0.5.9 // indirect diff --git a/.test-infra/pipelines/go.sum b/.test-infra/pipelines/go.sum new file mode 100644 index 0000000000000..62841cdb151d8 --- /dev/null +++ b/.test-infra/pipelines/go.sum @@ -0,0 +1,2 @@ +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= diff --git a/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/README.md b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/README.md new file mode 100644 index 0000000000000..9ea3a98ee777d --- /dev/null +++ b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/README.md @@ -0,0 +1,46 @@ + + +# Overview + +This directory provisions a redis cluster in Kubernetes. + +# Usage + +Follow terraform workflow convention to apply this module. It assumes the +working directory is at +[.test-infra/pipelines/infrastructure/03.io/api-overuse-study](..). + +## Terraform Init + +Initialize the terraform workspace. + +``` +DIR=02.redis +terraform -chdir=$DIR init +``` + +## Terraform Apply + +Apply the terraform module. + +``` +DIR=02.redis +terraform -chdir=$DIR apply -var-file=common.tfvars +``` \ No newline at end of file diff --git a/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/common.tfvars b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/common.tfvars new file mode 100644 index 0000000000000..f71b496b5a20a --- /dev/null +++ b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/common.tfvars @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace = "api-overuse-study" \ No newline at end of file diff --git a/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/data.tf b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/data.tf new file mode 100644 index 0000000000000..32cb7434d3006 --- /dev/null +++ b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/data.tf @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Query the Kubernetes namespace to verify existence. +data "kubernetes_namespace" "default" { + metadata { + name = var.namespace + } +} \ No newline at end of file diff --git a/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/provider.tf b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/provider.tf new file mode 100644 index 0000000000000..20bdbac74ea2f --- /dev/null +++ b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/provider.tf @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "kubernetes" { + config_path = "~/.kube/config" +} + +provider "helm" { + kubernetes { + config_path = "~/.kube/config" + } +} \ No newline at end of file diff --git a/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/redis.tf b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/redis.tf new file mode 100644 index 0000000000000..93b74a28782e2 --- /dev/null +++ b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/redis.tf @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Spin up a Redis cluster within the Kubernetes cluster using the bitami +// helm chart. +resource "helm_release" "redis" { + wait = false + repository = "https://charts.bitnami.com/bitnami" + chart = "redis" + name = "redis" + namespace = data.kubernetes_namespace.default.metadata[0].name + set { + name = "auth.enabled" + value = false + } + set { + name = "auth.sentinel" + value = false + } +} \ No newline at end of file diff --git a/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/variables.tf b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/variables.tf new file mode 100644 index 0000000000000..b55ce279654a3 --- /dev/null +++ b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/02.redis/variables.tf @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "namespace" { + type = string + description = "The Kubernetes namespace" +} \ No newline at end of file diff --git a/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/README.md b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/README.md new file mode 100644 index 0000000000000..19c1a5fe7db63 --- /dev/null +++ b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/README.md @@ -0,0 +1,46 @@ + + +# Overview + +This directory sets up the Kubernetes environment for subsequent modules. + +# Usage + +Follow terraform workflow convention to apply this module. +The following assumes the working directory is at +[.test-infra/pipelines/infrastructure/03.io/api-overuse-study](..). + +## Terraform Init + +Initialize the terraform workspace. + +``` +DIR=01.setup +terraform -chdir=$DIR init +``` + +## Terraform Apply + +Apply the terraform module. + +``` +DIR=01.setup +terraform -chdir=$DIR apply -var-file=common.tfvars +``` \ No newline at end of file diff --git a/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/common.tfvars b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/common.tfvars new file mode 100644 index 0000000000000..f71b496b5a20a --- /dev/null +++ b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/common.tfvars @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace = "api-overuse-study" \ No newline at end of file diff --git a/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/namespace.tf b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/namespace.tf new file mode 100644 index 0000000000000..f72e08dc08db9 --- /dev/null +++ b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/namespace.tf @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Provisions the namespace shared by all resources. +resource "kubernetes_namespace" "default" { + metadata { + name = var.namespace + } +} \ No newline at end of file diff --git a/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/provider.tf b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/provider.tf new file mode 100644 index 0000000000000..1846a87174697 --- /dev/null +++ b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/provider.tf @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "kubernetes" { + config_path = "~/.kube/config" +} \ No newline at end of file diff --git a/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/variables.tf b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/variables.tf new file mode 100644 index 0000000000000..2ae6cc65410d2 --- /dev/null +++ b/.test-infra/pipelines/infrastructure/03.io/api-overuse-study/variables.tf @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "namespace" { + type = string + description = "The Kubernetes namespace to provision resources" +} \ No newline at end of file diff --git a/.test-infra/pipelines/src/main/go/internal/environment/variable.go b/.test-infra/pipelines/src/main/go/internal/environment/variable.go new file mode 100644 index 0000000000000..1e81a2e2ce83b --- /dev/null +++ b/.test-infra/pipelines/src/main/go/internal/environment/variable.go @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package environment provides helpers for interacting with environment variables. +package environment + +import ( + "fmt" + "os" + "strings" +) + +// Variable defines an environment variable via a string type alias. +// Variable's string defaultValue assigns the system environment variable key. +type Variable string + +// Default a defaultValue to the system environment. +func (v Variable) Default(value string) error { + if v.Missing() { + return os.Setenv((string)(v), value) + } + return nil +} + +// Missing reports whether the system environment variable is an empty string. +func (v Variable) Missing() bool { + return v.Value() == "" +} + +// Key returns the system environment variable key. +func (v Variable) Key() string { + return (string)(v) +} + +// Value returns the system environment variable defaultValue. +func (v Variable) Value() string { + return os.Getenv((string)(v)) +} + +// KeyValue returns a concatenated string of the system environment variable's +// =. +func (v Variable) KeyValue() string { + return fmt.Sprintf("%s=%s", (string)(v), v.Value()) +} + +// Missing reports as an error listing all Variable among vars that are +// not assigned in the system environment. +func Missing(vars ...Variable) error { + var missing []string + for _, v := range vars { + if v.Missing() { + missing = append(missing, v.KeyValue()) + } + } + if len(missing) > 0 { + return fmt.Errorf("variables empty but expected from environment: %s", strings.Join(missing, "; ")) + } + return nil +} + +// Map converts a slice of Variable into a map. +// Its usage is for logging purposes. +func Map(vars ...Variable) map[string]interface{} { + result := map[string]interface{}{} + for _, v := range vars { + result[(string)(v)] = v.Value() + } + return result +} diff --git a/.test-infra/pipelines/src/main/go/internal/environment/variable_test.go b/.test-infra/pipelines/src/main/go/internal/environment/variable_test.go new file mode 100644 index 0000000000000..6675b39ab26cf --- /dev/null +++ b/.test-infra/pipelines/src/main/go/internal/environment/variable_test.go @@ -0,0 +1,358 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package environment + +import ( + "errors" + "github.com/google/go-cmp/cmp" + "os" + "testing" +) + +func TestMap(t *testing.T) { + type args struct { + vars []Variable + values []string + } + tests := []struct { + name string + args args + want map[string]interface{} + }{ + { + name: "{}", + args: args{}, + want: map[string]interface{}{}, + }, + { + name: "{A=1; B=2; C=3}", + args: args{ + vars: []Variable{ + "A", + "B", + "C", + }, + values: []string{ + "1", + "2", + "3", + }, + }, + want: map[string]interface{}{ + "A": "1", + "B": "2", + "C": "3", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clear(tt.args.vars...) + set(t, tt.args.vars, tt.args.values) + got := Map(tt.args.vars...) + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Errorf("Map() = %v, want %v, diff:\n%v", got, tt.want, diff) + } + }) + } +} + +func TestMissing(t *testing.T) { + type args struct { + vars []Variable + values []string + } + tests := []struct { + name string + args args + want error + }{ + { + name: "{}", + args: args{}, + }, + { + name: "{A=}", + args: args{ + vars: []Variable{ + "A", + }, + values: []string{ + "", + }, + }, + want: errors.New("variables empty but expected from environment: A="), + }, + { + name: "{A=1}", + args: args{ + vars: []Variable{ + "A", + }, + values: []string{ + "1", + }, + }, + want: nil, + }, + { + name: "{A=; B=}", + args: args{ + vars: []Variable{ + "A", + "B", + }, + values: []string{ + "", + "", + }, + }, + want: errors.New("variables empty but expected from environment: A=; B="), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var got, want string + clear(tt.args.vars...) + set(t, tt.args.vars, tt.args.values) + err := Missing(tt.args.vars...) + if err != nil { + got = err.Error() + } + if tt.want != nil { + want = tt.want.Error() + } + if diff := cmp.Diff(got, want); diff != "" { + t.Errorf("Missing() error = %v, want %v, diff:\n%s", err, tt.want, diff) + } + }) + } +} + +func TestVariable_Default(t *testing.T) { + type args struct { + setValue string + defaultValue string + } + tests := []struct { + name string + v Variable + args args + want string + }{ + { + name: "environment variable not set", + v: "A", + args: args{ + defaultValue: "1", + }, + want: "1", + }, + { + name: "environment variable default is overridden by set value", + v: "A", + args: args{ + setValue: "2", + defaultValue: "1", + }, + want: "2", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clear(tt.v) + if tt.args.setValue != "" { + set(t, []Variable{tt.v}, []string{tt.args.setValue}) + } + if err := tt.v.Default(tt.args.defaultValue); err != nil { + t.Fatalf("could not set default environment variable value during test execution: %v", err) + } + got := os.Getenv(tt.v.Key()) + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Errorf("Default() = %s, want %s, diff:\n%s", got, tt.want, diff) + } + }) + } +} + +func TestVariable_KeyValue(t *testing.T) { + tests := []struct { + name string + v Variable + value string + want string + }{ + { + name: "environment variable not set", + v: "A", + want: "A=", + }, + { + name: "environment variable is set", + v: "A", + value: "1", + want: "A=1", + }, + } + for _, tt := range tests { + clear(tt.v) + t.Run(tt.name, func(t *testing.T) { + set(t, []Variable{tt.v}, []string{tt.value}) + got := tt.v.KeyValue() + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Errorf("KeyValue() = %v, want %v, diff:\n%s", got, tt.want, diff) + } + }) + } +} + +func TestVariable_Missing(t *testing.T) { + type args struct { + setValue string + defaultValue string + } + tests := []struct { + name string + args args + v Variable + want bool + }{ + { + name: "no default and not set", + args: args{}, + v: "A", + want: true, + }, + { + name: "has default but not set", + args: args{ + defaultValue: "1", + }, + v: "A", + want: false, + }, + { + name: "no default but set", + args: args{ + setValue: "1", + }, + v: "A", + want: false, + }, + { + name: "has default and set", + args: args{ + setValue: "2", + defaultValue: "1", + }, + v: "A", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clear(tt.v) + if tt.args.defaultValue != "" { + if err := tt.v.Default(tt.args.defaultValue); err != nil { + t.Fatalf("could not set default environment variable value during test execution: %v", err) + } + } + if tt.args.setValue != "" { + set(t, []Variable{tt.v}, []string{tt.args.setValue}) + } + if got := tt.v.Missing(); got != tt.want { + t.Errorf("Missing() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestVariable_Value(t *testing.T) { + type args struct { + setValue string + defaultValue string + } + tests := []struct { + name string + args args + v Variable + want string + }{ + { + name: "no default and not set", + args: args{}, + v: "A", + want: "", + }, + { + name: "has default but not set", + args: args{ + defaultValue: "1", + }, + v: "A", + want: "1", + }, + { + name: "no default but set", + args: args{ + setValue: "1", + }, + v: "A", + want: "1", + }, + { + name: "has default and set", + args: args{ + setValue: "2", + defaultValue: "1", + }, + v: "A", + want: "2", + }, + } + for _, tt := range tests { + clear(tt.v) + if tt.args.defaultValue != "" { + if err := tt.v.Default(tt.args.defaultValue); err != nil { + t.Fatalf("could not set default environment variable value during test execution: %v", err) + } + } + if tt.args.setValue != "" { + set(t, []Variable{tt.v}, []string{tt.args.setValue}) + } + t.Run(tt.name, func(t *testing.T) { + if got := tt.v.Value(); got != tt.want { + t.Errorf("Value() = %v, want %v", got, tt.want) + } + }) + } +} + +func clear(vars ...Variable) { + for _, k := range vars { + _ = os.Setenv(k.Key(), "") + } +} + +func set(t *testing.T, vars []Variable, values []string) { + if len(vars) != len(values) { + t.Fatalf("test cases should be configured with matching args.vars and args.values: len(tt.args.vars): %v != len(tt.args.values): %v", len(vars), len(values)) + } + for i := range vars { + key := vars[i].Key() + value := values[i] + _ = os.Setenv(key, value) + } +} diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/ReadDataflowApiWriteBigQuery.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/ReadDataflowApiWriteBigQuery.java index f12f285ec4732..fc1699cc44f2a 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/ReadDataflowApiWriteBigQuery.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/ReadDataflowApiWriteBigQuery.java @@ -63,7 +63,7 @@ import org.apache.beam.testinfra.pipelines.dataflow.JobMetricsWithAppendedDetails; import org.apache.beam.testinfra.pipelines.dataflow.StageSummaryWithAppendedDetails; import org.apache.beam.testinfra.pipelines.pubsub.PubsubReadOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.NonNull; /** diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/bigquery/BigQueryWrites.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/bigquery/BigQueryWrites.java index 37dcb99f8bdd4..4f60d03efc32c 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/bigquery/BigQueryWrites.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/bigquery/BigQueryWrites.java @@ -34,7 +34,7 @@ import org.apache.beam.testinfra.pipelines.dataflow.JobMetricsWithAppendedDetails; import org.apache.beam.testinfra.pipelines.dataflow.StageSummaryWithAppendedDetails; import org.apache.beam.testinfra.pipelines.dataflow.WorkerDetailsWithAppendedDetails; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/bigquery/DatasetReferenceOptionValue.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/bigquery/DatasetReferenceOptionValue.java index 855dcef769a28..7988a11d3adb3 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/bigquery/DatasetReferenceOptionValue.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/bigquery/DatasetReferenceOptionValue.java @@ -18,7 +18,7 @@ package org.apache.beam.testinfra.pipelines.bigquery; import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.services.bigquery.model.DatasetReference; import java.io.Serializable; diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/ConversionError.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/ConversionError.java index 353d066788010..f3f42e33440d7 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/ConversionError.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/ConversionError.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.CaseFormat; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CaseFormat; import org.joda.time.Instant; /** Stores errors related to conversions. */ diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/EventarcConversions.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/EventarcConversions.java index f5191af603954..856f64d0da723 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/EventarcConversions.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/EventarcConversions.java @@ -18,7 +18,7 @@ package org.apache.beam.testinfra.pipelines.conversions; import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -31,7 +31,7 @@ import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; import org.joda.time.Instant; /** Methods for converting from Eventarc JSON payloads. */ diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/JobsToRow.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/JobsToRow.java index 0fcbfdc96edf6..5bbe9e04ac69a 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/JobsToRow.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/JobsToRow.java @@ -34,7 +34,7 @@ import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.testinfra.pipelines.schemas.DescriptorSchemaRegistry; import org.apache.beam.testinfra.pipelines.schemas.GeneratedMessageV3RowBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; import org.checkerframework.checker.nullness.qual.NonNull; import org.joda.time.Instant; diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/RowConversionResult.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/RowConversionResult.java index 32c2d0307a9be..0f9bafff4d3e1 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/RowConversionResult.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/RowConversionResult.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** Convenience class for bundling {@link Row} conversion successes and failures. */ @Internal diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/WithAppendedDetailsToRow.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/WithAppendedDetailsToRow.java index 4be29db2057b1..c62de6deb80a4 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/WithAppendedDetailsToRow.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/WithAppendedDetailsToRow.java @@ -43,7 +43,7 @@ import org.apache.beam.testinfra.pipelines.dataflow.WorkerDetailsWithAppendedDetails; import org.apache.beam.testinfra.pipelines.schemas.DescriptorSchemaRegistry; import org.apache.beam.testinfra.pipelines.schemas.GeneratedMessageV3RowBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; import org.checkerframework.checker.nullness.qual.NonNull; import org.joda.time.Instant; diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowFilterEventarcJobs.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowFilterEventarcJobs.java index 8daa7adef650e..1fe03ed04d682 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowFilterEventarcJobs.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowFilterEventarcJobs.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.transforms.Filter; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.NonNull; /** Filters Eventarc {@link Job}s. */ diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetJobExecutionDetails.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetJobExecutionDetails.java index 3df99c447bfe0..fcb5518a11053 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetJobExecutionDetails.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetJobExecutionDetails.java @@ -36,8 +36,8 @@ import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.NonNull; import org.joda.time.Duration; diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetJobMetrics.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetJobMetrics.java index c8900d123532f..3429f88949a18 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetJobMetrics.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetJobMetrics.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.NonNull; import org.joda.time.Duration; diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetJobs.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetJobs.java index e7a59560a7ec5..728c6bd6d6b9a 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetJobs.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetJobs.java @@ -34,7 +34,7 @@ import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.NonNull; import org.joda.time.Duration; diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetStageExecutionDetails.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetStageExecutionDetails.java index 1663577783b61..650631533a0bd 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetStageExecutionDetails.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetStageExecutionDetails.java @@ -36,8 +36,8 @@ import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.NonNull; import org.joda.time.Duration; diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowReadResult.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowReadResult.java index 5ea13eb1fa004..af01eb6abd437 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowReadResult.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowReadResult.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.UnknownKeyFor; diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowRequestError.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowRequestError.java index a23843afa6550..5d7260c473590 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowRequestError.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowRequestError.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.CaseFormat; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CaseFormat; import org.joda.time.Instant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/JobMetricsWithAppendedDetails.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/JobMetricsWithAppendedDetails.java index 24ab889c7af5d..e2472d2e42f16 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/JobMetricsWithAppendedDetails.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/JobMetricsWithAppendedDetails.java @@ -21,7 +21,7 @@ import com.google.dataflow.v1beta3.JobMetrics; import java.io.Serializable; import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/StageSummaryWithAppendedDetails.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/StageSummaryWithAppendedDetails.java index 4f454301bf52c..0cdca9accb9c0 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/StageSummaryWithAppendedDetails.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/StageSummaryWithAppendedDetails.java @@ -21,7 +21,7 @@ import com.google.dataflow.v1beta3.StageSummary; import java.io.Serializable; import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/WorkerDetailsWithAppendedDetails.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/WorkerDetailsWithAppendedDetails.java index 5b1542fbe30da..4b6d0a32d3679 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/WorkerDetailsWithAppendedDetails.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/WorkerDetailsWithAppendedDetails.java @@ -21,7 +21,7 @@ import com.google.dataflow.v1beta3.WorkerDetails; import java.io.Serializable; import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/schemas/DependencyDrivenDescriptorQueue.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/schemas/DependencyDrivenDescriptorQueue.java index 13458da742d07..94007e29650b5 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/schemas/DependencyDrivenDescriptorQueue.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/schemas/DependencyDrivenDescriptorQueue.java @@ -18,7 +18,7 @@ package org.apache.beam.testinfra.pipelines.schemas; import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.FieldDescriptor; diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/schemas/DescriptorSchemaRegistry.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/schemas/DescriptorSchemaRegistry.java index 44fd4d668a389..a8e60f4030584 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/schemas/DescriptorSchemaRegistry.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/schemas/DescriptorSchemaRegistry.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.Schema.Field; import org.apache.beam.sdk.schemas.Schema.FieldType; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.NonNull; /** Registers and builds {@link Schema}s of {@link Descriptor} based types. */ diff --git a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/schemas/GeneratedMessageV3RowBuilder.java b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/schemas/GeneratedMessageV3RowBuilder.java index f8f8af3e06d47..666c787efb80c 100644 --- a/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/schemas/GeneratedMessageV3RowBuilder.java +++ b/.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/schemas/GeneratedMessageV3RowBuilder.java @@ -20,7 +20,7 @@ import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; import static org.apache.beam.testinfra.pipelines.schemas.DescriptorSchemaRegistry.KEY_FIELD_NAME; import static org.apache.beam.testinfra.pipelines.schemas.DescriptorSchemaRegistry.VALUE_FIELD_NAME; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.protobuf.AbstractMessage; import com.google.protobuf.Any; @@ -46,10 +46,10 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.Schema.TypeName; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.checkerframework.checker.nullness.qual.NonNull; import org.joda.time.Instant; diff --git a/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/conversions/JobMetricsWithAppendedDetailsTest.java b/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/conversions/JobMetricsWithAppendedDetailsTest.java index afa792958fbff..ec0e8e7d2832a 100644 --- a/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/conversions/JobMetricsWithAppendedDetailsTest.java +++ b/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/conversions/JobMetricsWithAppendedDetailsTest.java @@ -41,7 +41,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.testinfra.pipelines.dataflow.JobMetricsWithAppendedDetails; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.NonNull; import org.joda.time.Instant; import org.joda.time.ReadableDateTime; diff --git a/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/conversions/StageSummaryWithAppendedDetailsTest.java b/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/conversions/StageSummaryWithAppendedDetailsTest.java index a1bf2709ff667..1f40f78ca63d4 100644 --- a/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/conversions/StageSummaryWithAppendedDetailsTest.java +++ b/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/conversions/StageSummaryWithAppendedDetailsTest.java @@ -41,7 +41,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.testinfra.pipelines.dataflow.StageSummaryWithAppendedDetails; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.NonNull; import org.joda.time.Instant; import org.joda.time.ReadableDateTime; diff --git a/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/conversions/WorkerDetailsWithAppendedDetailsTest.java b/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/conversions/WorkerDetailsWithAppendedDetailsTest.java index bd0a900780aee..b466343837bc2 100644 --- a/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/conversions/WorkerDetailsWithAppendedDetailsTest.java +++ b/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/conversions/WorkerDetailsWithAppendedDetailsTest.java @@ -42,7 +42,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.testinfra.pipelines.dataflow.WorkerDetailsWithAppendedDetails; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.NonNull; import org.joda.time.Instant; import org.joda.time.ReadableDateTime; diff --git a/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/AbstractGeneratedMessageV3RowBuilderTest.java b/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/AbstractGeneratedMessageV3RowBuilderTest.java index eb7feacb0aef1..95e041419ddf0 100644 --- a/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/AbstractGeneratedMessageV3RowBuilderTest.java +++ b/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/AbstractGeneratedMessageV3RowBuilderTest.java @@ -45,8 +45,8 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.commons.lang3.tuple.Pair; import org.checkerframework.checker.nullness.qual.NonNull; import org.joda.time.ReadableDateTime; diff --git a/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/EnvironmentRowBuilderTest.java b/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/EnvironmentRowBuilderTest.java index 001b07fcc75e8..a0be8b29568f2 100644 --- a/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/EnvironmentRowBuilderTest.java +++ b/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/EnvironmentRowBuilderTest.java @@ -29,7 +29,7 @@ import java.util.Collections; import java.util.Set; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.jupiter.api.Test; diff --git a/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/ExecutionStageSummaryTest.java b/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/ExecutionStageSummaryTest.java index 16830e17180c0..a0b489cf57459 100644 --- a/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/ExecutionStageSummaryTest.java +++ b/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/ExecutionStageSummaryTest.java @@ -21,7 +21,7 @@ import com.google.protobuf.Descriptors.Descriptor; import java.util.Set; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.NonNull; /** diff --git a/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/TransformSummaryRowBuilderTest.java b/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/TransformSummaryRowBuilderTest.java index b07700c18ceed..19afba8033361 100644 --- a/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/TransformSummaryRowBuilderTest.java +++ b/.test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/TransformSummaryRowBuilderTest.java @@ -21,7 +21,7 @@ import com.google.protobuf.Descriptors.Descriptor; import java.util.Set; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.NonNull; /** diff --git a/.test-infra/pubsub/README.md b/.test-infra/pubsub/README.md new file mode 100644 index 0000000000000..509c1006e00ed --- /dev/null +++ b/.test-infra/pubsub/README.md @@ -0,0 +1,37 @@ + + +# Overview + +This folder contains Python scripts to create a Pub/Sub topic under +the GCP project `apache-beam-testing` and test the topic. +The created topic is `projects/apache-beam-testing/topics/Imagenet_openimage_50k_benchmark`. + +# Create the topic `Imagenet_openimage_50k_benchmark` + +- Create one VM to run `gcs_image_looper.py`. + The VM `pubsub-test-do-not-delete` was already created under `apache-beam-testing`. + Keep the script running to continuously publish data. +- You might run `gcloud auth application-default login` to get the auth. +- You might run `pip install google-cloud-core google-cloud-pubsub google-cloud-storage`. +- Must make `Imagenet_openimage_50k_benchmark` public by adding `allAuthenticatedUsers` to the Pub/Sub Subscriber role. + +# Tes the topic by subscribing it + +- Run `test_image_looper.py` to check whether you could get any data. diff --git a/.test-infra/pubsub/gcs_image_looper.py b/.test-infra/pubsub/gcs_image_looper.py new file mode 100644 index 0000000000000..a00e36f9ae72b --- /dev/null +++ b/.test-infra/pubsub/gcs_image_looper.py @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""This executable loops image filepaths from a gcs bucket file.""" +import random +import time + +from google.api_core.exceptions import AlreadyExists +from google.cloud import pubsub_v1 +from google.cloud import storage + +# use the eou project and gcs to run the word looper +project_id = "apache-beam-testing" +gcs_bucket = "apache-beam-ml" +num_images_per_second = 5 + +publisher = pubsub_v1.PublisherClient() +image_file_path = "testing/inputs/openimage_50k_benchmark.txt" +topic_name = "Imagenet_openimage_50k_benchmark" +topic_path = publisher.topic_path(project_id, topic_name) + + +class ImageLooper(object): + """Loop the images in a gcs bucket file and publish them to a pubsub topic. + """ + content = "" + cursor = 0 + + def __init__(self, filename): + self._read_gcs_file(filename) + + def get_next_image(self): + """Returns the next image randomly.""" + next_image = "" + while not next_image: + image_id = random.randint(0, len(self.content) - 1) + next_image = self.content[image_id] + return next_image + + def _read_gcs_file(self, filename): + client = storage.Client() + bucket = client.get_bucket(gcs_bucket) + blob = bucket.get_blob(filename) + self.content = blob.download_as_string().decode("utf-8").split('\n') + + +try: + publisher.create_topic(request={"name": topic_path}) +except AlreadyExists: + pass + +looper = ImageLooper(image_file_path) +while True: + image = looper.get_next_image() + publisher.publish(topic_path, data=image.encode("utf-8")) + time.sleep(1 / num_images_per_second) diff --git a/.test-infra/pubsub/test_image_looper.py b/.test-infra/pubsub/test_image_looper.py new file mode 100644 index 0000000000000..c5fcf9f652c76 --- /dev/null +++ b/.test-infra/pubsub/test_image_looper.py @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""This executable test the pub/sub topic created by gcs_image_looper.py""" + +from concurrent.futures import TimeoutError + +from google.cloud import pubsub_v1 +from google.api_core.exceptions import AlreadyExists + +project_id = "apache-beam-testing" +subscription_id = "test-image-looper" +topic_id = "Imagenet_openimage_50k_benchmark" + +publisher = pubsub_v1.PublisherClient() +subscriber = pubsub_v1.SubscriberClient() +topic_path = publisher.topic_path(project_id, topic_id) +subscription_path = subscriber.subscription_path(project_id, subscription_id) + +try: + subscription = subscriber.create_subscription(request={ + "name": subscription_path, + "topic": topic_path + }) + print(f"Subscription created: {subscription}") +except AlreadyExists: + subscriber.delete_subscription(request={"subscription": subscription_path}) + subscription = subscriber.create_subscription(request={ + "name": subscription_path, + "topic": topic_path + }) + print(f"Subscription recreated: {subscription}") + +timeout = 3.0 + +total_images = [] + + +def callback(message: pubsub_v1.subscriber.message.Message) -> None: + total_images.append(message.data.decode()) + message.ack() + + +streaming_pull_future = subscriber.subscribe(subscription_path, + callback=callback) +print(f"Listening for messages on {subscription_path}..\n") + +try: + # When `timeout` is not set, result() will block indefinitely, + # unless an exception is encountered first. + streaming_pull_future.result(timeout=timeout) +except TimeoutError: + streaming_pull_future.cancel() # Trigger the shutdown. + streaming_pull_future.result() # Block until the shutdown is complete. +print("Results: \n", total_images) + +subscriber.delete_subscription(request={"subscription": subscription_path}) + +print(f"Subscription deleted: {subscription_path}.") \ No newline at end of file diff --git a/.test-infra/terraform/OWNERS b/.test-infra/terraform/OWNERS deleted file mode 100644 index 2ff287d847355..0000000000000 --- a/.test-infra/terraform/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - damondouglas - diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/README.md b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/README.md index 80db3c8a77f14..cd9558e7739fb 100644 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/README.md +++ b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/README.md @@ -19,38 +19,52 @@ # Overview -This module provisions a private Google Kubernetes Engine cluster. +This module provisions a private Google Kubernetes Engine cluster in the +Google Cloud Platform (GCP). # Requirements and Usage See [Google Cloud Platform requirements](../../google-cloud-platform/README.md) -for details on requirements -and usage. +for details on requirements and usage. -## 1. Create vars.tfvars +# Prerequisites + +This module assumes the following pre-existing resources: + +- [Cloud Resource Manager API Enabled](https://console.cloud.google.com/apis/library/cloudresourcemanager.googleapis.com) +- [Virtual Private Cloud (VPC) network and subnetwork](https://cloud.google.com/vpc/docs/create-modify-vpc-networks) +- [GCP Service Account](https://cloud.google.com/iam/docs/service-accounts-create) + +# Step 1. Create vars.tfvars + +## If you are provisioning in `apache-beam-testing`: + +You can skip this step and follow the next instruction. For security reasons, +the `service_account_id` was omitted. + +## If you are provisioning in a custom GCP project: Create a `vars.tfvars` file in [.test-infra/terraform/google-cloud-platform/google-kubernetes-engine](.). Edit with your IDE terraform plugin installed and it will autocomplete the variable names. -## 2. Initialize and apply the terraform module. +# Step 2. Initialize and apply the terraform module. + +## If you are provisioning in `apache-beam-testing`: ``` terraform init -terraform plan -var-file=vars.tfvars -terraform apply -var-file=vars.tfvars +terraform apply -var-file=apache-beam-testing.tfvars ``` -# Special Instructions - -This module also provisions a bastion host needed to connect to the private -cluster. To connect to the kubernetes -cluster, do so through the bastion host by following directions starting at -[Connect to your cluster from the remote client](https://cloud.google.com/kubernetes-engine/docs/tutorials/private-cluster-bastion#connect). +You will be prompted for any remaining variables. -To find the bastion host, run: +## If you are provisioning in a custom GCP project: ``` -gcloud compute instances list --filter=name:bastion +terraform init +terraform apply -var-file=vars.tfvars ``` + +You will be prompted for any remaining variables. diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/apache-beam-testing.tfvars b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/apache-beam-testing.tfvars new file mode 100644 index 0000000000000..f1105e180f708 --- /dev/null +++ b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/apache-beam-testing.tfvars @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +project = "apache-beam-testing" +network = "default" +subnetwork = "default" +region = "us-central1" diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/cluster.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/cluster.tf new file mode 100644 index 0000000000000..90c6ac793389b --- /dev/null +++ b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/cluster.tf @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "postfix" { + length = 6 + upper = false + special = false +} + +resource "google_container_cluster" "default" { + depends_on = [google_project_service.required] + name = "${var.cluster_name_prefix}-${random_string.postfix.result}" + location = var.region + enable_autopilot = true + network = data.google_compute_network.default.id + subnetwork = data.google_compute_subnetwork.default.id + master_authorized_networks_config {} + private_cluster_config { + enable_private_nodes = true + enable_private_endpoint = false + } +} diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/main.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/main.tf deleted file mode 100644 index 4a4713de48eb9..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/main.tf +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "random_string" "postfix" { - length = 8 - upper = false - special = false -} - -locals { - common_resource_name = "${var.resource_name_prefix}-${random_string.postfix.result}" -} - -// Provision minimally necessary environment to provision a Google Kubernetes engine. -module "setup" { - source = "./modules/01-setup" - - project = var.project - kubernetes_node_service_account_id = local.common_resource_name - region = var.region -} - -// Provision Google Cloud Virtual Provide Cloud (VPC) network and related resources. -module "network" { - source = "./modules/02-network" - - kubernetes_node_service_account = module.setup.kubernetes_node_service_account - network_base_name = local.common_resource_name - project = var.project - region = var.region - subnetwork_cidr_range = var.subnetwork_cidr_range -} - -// Provision Google Kubernetes Engine cluster. -module "cluster" { - source = "./modules/03-cluster" - - cluster_name = local.common_resource_name - kubernetes_node_service_account = module.setup.kubernetes_node_service_account - network = module.network.network - subnetwork = module.network.subnetwork - project = var.project - region = var.region -} - -// Provision bastion host for remote private GKE cluster connectivity. -module "bastion" { - source = "./modules/04-bastion" - - bastion_compute_machine_type = var.bastion_compute_machine_type - kubernetes_node_service_account = module.setup.kubernetes_node_service_account - network = module.network.network - subnetwork = module.network.subnetwork - project = var.project - region = var.region - router = module.network.router - nat = module.network.nat -} \ No newline at end of file diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/01-setup/README.md b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/01-setup/README.md deleted file mode 100644 index 018b30f897488..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/01-setup/README.md +++ /dev/null @@ -1,26 +0,0 @@ - - -# Overview - -This module sets up the Google Cloud minimally necessary environment to -provision a Google Kubernetes engine. - -- Service Account with Kubernetes Engine Node Service Account role -- Required Project service APIs diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/01-setup/iam.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/01-setup/iam.tf deleted file mode 100644 index a23c2180ce98d..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/01-setup/iam.tf +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Provision a service account to bind to the Kubernetes cluster node -resource "google_service_account" "kubernetes_node_service_account" { - account_id = var.kubernetes_node_service_account_id - display_name = var.kubernetes_node_service_account_id - description = "The service account bound to the Kubernetes node" -} - -// Provision IAM roles for the kubernetes node service account -resource "google_project_iam_member" "kubernetes_node_service_account_roles" { - depends_on = [google_project_service.required_services] - - // provision as toset to make it easier to add new IAM roles in the future - for_each = toset([ - "roles/container.nodeServiceAccount", - ]) - - role = each.key - member = "serviceAccount:${google_service_account.kubernetes_node_service_account.email}" - project = var.project - -} \ No newline at end of file diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/01-setup/output.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/01-setup/output.tf deleted file mode 100644 index 3030c9a250eb4..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/01-setup/output.tf +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Export the provisioned service account. -output "kubernetes_node_service_account" { - value = google_service_account.kubernetes_node_service_account -} diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/01-setup/services.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/01-setup/services.tf deleted file mode 100644 index 2d2d398ee56a1..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/01-setup/services.tf +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Provision the required Google Cloud services -resource "google_project_service" "required_services" { - for_each = toset([ - "container", - "compute", - ]) - - service = "${each.key}.googleapis.com" - disable_on_destroy = false -} \ No newline at end of file diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/01-setup/variables.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/01-setup/variables.tf deleted file mode 100644 index 291bf0a8bf26d..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/01-setup/variables.tf +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project" { - type = string - description = "The Google Cloud Platform (GCP) project within which resources are provisioned" -} - -variable "region" { - type = string - description = "The Google Cloud Platform (GCP) region in which to provision resources" -} - -variable "kubernetes_node_service_account_id" { - type = string - description = "The Google Cloud Platform Service Account to be used by the node VMs created by GKE Autopilot" -} diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/README.md b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/README.md deleted file mode 100644 index 15b39251434ba..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/README.md +++ /dev/null @@ -1,26 +0,0 @@ - - -# Overview - -This module sets up the Google Cloud Virtual Provide Cloud (VPC) network and -related resources. - -- Provision a custom network and subnetwork -- Provision Cloud NAT and Router diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/nat.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/nat.tf deleted file mode 100644 index 7f573a9d31843..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/nat.tf +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// For security, provision the Kubernetes node using private IP addresses. -// The recommendation in this context is to additionally provision -// a Cloud NAT and Router so that private IP only compute engine instances -// (created as a result of Google Kubernetes Engine node pool) can access -// resources outside the Google Virtual Private Cloud -// See https://cloud.google.com/nat/docs/overview -resource "google_compute_router_nat" "default" { - name = "${google_compute_router.default.name}-nat" - nat_ip_allocate_option = "AUTO_ONLY" - region = google_compute_router.default.region - router = google_compute_router.default.name - source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" - log_config { - enable = true - filter = "ERRORS_ONLY" - } -} - -// Required by the Cloud NAT. -// See https://cloud.google.com/network-connectivity/docs/router. -resource "google_compute_router" "default" { - name = "${google_compute_subnetwork.default.name}-${var.region}-router" - network = google_compute_subnetwork.default.network - region = google_compute_subnetwork.default.region -} \ No newline at end of file diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/network.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/network.tf deleted file mode 100644 index 6a8a78bf80da3..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/network.tf +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Provision virtual custom 02-network -resource "google_compute_network" "default" { - depends_on = [google_project_service.compute] - name = var.network_base_name - auto_create_subnetworks = false -} - -// Provision subnetwork of the virtual custom 02-network -resource "google_compute_subnetwork" "default" { - name = google_compute_network.default.name - ip_cidr_range = var.subnetwork_cidr_range - network = google_compute_network.default.name - private_ip_google_access = true - region = var.region -} - -// Provision firewall rule for internal 02-network traffic only -resource "google_compute_firewall" "default" { - name = "allow-${google_compute_network.default.name}-internal" - network = google_compute_network.default.name - - allow { - protocol = "tcp" - } - - source_service_accounts = [ - var.kubernetes_node_service_account.email - ] -} - -// Provision firewall rule for SSH ingress from identity aware proxy -// See https://cloud.google.com/iap/docs/using-tcp-forwarding#create-firewall-rule -resource "google_compute_firewall" "iap" { - name = "allow-${google_compute_network.default.name}-ssh-ingress-from-iap" - network = google_compute_network.default.name - - allow { - protocol = "tcp" - ports = [ - 22, - ] - } - - source_ranges = [ - "35.235.240.0/20", - ] -} diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/output.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/output.tf deleted file mode 100644 index b48c1fa0c2432..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/output.tf +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Export the provisioned VPC network. -output "network" { - value = google_compute_network.default -} - -// Export the provisioned VPC subnetwork. -output "subnetwork" { - value = google_compute_subnetwork.default -} - -// Export the provisioned VPC NAT. -output "nat" { - value = google_compute_router_nat.default -} - -// Export the provisioned VPC NAT router. -output "router" { - value = google_compute_router.default -} diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/provider.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/provider.tf deleted file mode 100644 index e070d8408d963..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/provider.tf +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Setup Google Cloud provider -provider "google" { - project = var.project -} diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/services.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/services.tf deleted file mode 100644 index 5ec40dc6a4292..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/services.tf +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Required for depends_on implicit declaration -resource "google_project_service" "compute" { - service = "compute.googleapis.com" - disable_on_destroy = false -} diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/variables.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/variables.tf deleted file mode 100644 index 9157a478e3e08..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network/variables.tf +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project" { - type = string - description = "The Google Cloud Platform (GCP) project within which resources are provisioned" -} - -variable "region" { - type = string - description = "The Google Cloud Platform (GCP) region in which to provision resources" -} - -variable "subnetwork_cidr_range" { - type = string - description = "The address range for this subnet, in CIDR notation. Use a standard private VPC network address range: for example, 10.0.0.0/9." -} - -variable "network_base_name" { - type = string - description = "The name basis for network resources." -} - -variable "kubernetes_node_service_account" { - type = object({ - email = string - }) - description = "The Google Cloud Platform Service Account bound to the GKE node" -} diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/03-cluster/README.md b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/03-cluster/README.md deleted file mode 100644 index 2e60183f105a8..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/03-cluster/README.md +++ /dev/null @@ -1,22 +0,0 @@ - - -# Overview - -This module provisions the private Google Kubernetes Engine cluster. diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/03-cluster/cluster.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/03-cluster/cluster.tf deleted file mode 100644 index db4f3954d9f40..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/03-cluster/cluster.tf +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Provisions regional private Google Kubernetes Engine cluster -resource "google_container_cluster" "default" { - depends_on = [google_project_service.container] - name = var.cluster_name - location = var.region - enable_autopilot = true - network = var.network.id - subnetwork = var.subnetwork.id - cluster_autoscaling { - auto_provisioning_defaults { - service_account = var.kubernetes_node_service_account.email - oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"] - } - } - master_authorized_networks_config {} - private_cluster_config { - enable_private_endpoint = true - enable_private_nodes = true - master_global_access_config { - enabled = true - } - } - ip_allocation_policy {} -} diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/03-cluster/provider.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/03-cluster/provider.tf deleted file mode 100644 index e070d8408d963..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/03-cluster/provider.tf +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Setup Google Cloud provider -provider "google" { - project = var.project -} diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/03-cluster/services.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/03-cluster/services.tf deleted file mode 100644 index 1cde6ccf4d410..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/03-cluster/services.tf +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Required for depends_on implicit declaration -resource "google_project_service" "container" { - service = "container.googleapis.com" - disable_on_destroy = false -} diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/03-cluster/variables.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/03-cluster/variables.tf deleted file mode 100644 index 55b2bc8cf67fc..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/03-cluster/variables.tf +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project" { - type = string - description = "The Google Cloud Platform (GCP) project within which resources are provisioned" -} - -variable "region" { - type = string - description = "The Google Cloud Platform (GCP) region in which to provision resources" -} - -variable "kubernetes_node_service_account" { - type = object({ - email = string - }) - description = "The Google Cloud Platform Service Account bound to the GKE node" -} - -variable "network" { - type = object({ - id = string - }) - description = "The Google Cloud Platform Virtual Cloud network within which we provision the kubernetes node" -} - -variable "subnetwork" { - type = object({ - id = string - }) - description = "The Google Cloud Platform Virtual Cloud subnetwork within which we provision the kubernetes node" -} - -variable "cluster_name" { - type = string - description = "The name of the Google Kubernetes engine cluster." -} diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/README.md b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/README.md deleted file mode 100644 index 2e60183f105a8..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/README.md +++ /dev/null @@ -1,22 +0,0 @@ - - -# Overview - -This module provisions the private Google Kubernetes Engine cluster. diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/compute.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/compute.tf deleted file mode 100644 index bfdf8ce31dee4..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/compute.tf +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generate a postfix for resource naming. -resource "random_string" "postfix" { - length = 6 - upper = false - special = false -} - -locals { - tinyproxy_content = file("${path.module}/tinyproxy.conf") -} - -// Query available zones in the region. -// Provision bastion host for private cluster connectivity. -// See https://cloud.google.com/kubernetes-engine/docs/tutorials/private-cluster-bastion -resource "google_compute_instance" "bastion" { - depends_on = [data.google_compute_router.default] - machine_type = var.bastion_compute_machine_type - name = "bastion-${random_string.postfix.result}" - zone = data.google_compute_zones.available.names[0] - service_account { - scopes = [ - "https://www.googleapis.com/auth/cloud-platform" - ] - email = var.kubernetes_node_service_account.email - } - boot_disk { - initialize_params { - image = "debian-cloud/debian-11" - } - } - network_interface { - network = data.google_compute_network.default.id - subnetwork = data.google_compute_subnetwork.default.id - } - - metadata_startup_script = < /etc/tinyproxy/tinyproxy.conf -${local.tinyproxy_content} -TP - -sudo service tinyproxy restart -EOF -} \ No newline at end of file diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/data.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/data.tf deleted file mode 100644 index 12163f9453b4d..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/data.tf +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_compute_zones" "available" { - depends_on = [google_project_service.compute] - region = var.region -} - -// Query the Virtual Private Cloud (VPC) network. -data "google_compute_network" "default" { - depends_on = [google_project_service.compute] - name = var.network.name -} - -// Query the Virtual Private Cloud (VPC) subnetwork. -data "google_compute_subnetwork" "default" { - depends_on = [google_project_service.compute] - name = var.subnetwork.name - region = var.region -} - -// Query the Virtual Private Cloud (VPC) router NAT. -// The bastion host requires this. -data "google_compute_router" "default" { - depends_on = [google_project_service.compute] - name = var.router.name - network = data.google_compute_network.default.name - region = data.google_compute_subnetwork.default.region -} diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/provider.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/provider.tf deleted file mode 100644 index e070d8408d963..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/provider.tf +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Setup Google Cloud provider -provider "google" { - project = var.project -} diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/services.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/services.tf deleted file mode 100644 index 5ec40dc6a4292..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/services.tf +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Required for depends_on implicit declaration -resource "google_project_service" "compute" { - service = "compute.googleapis.com" - disable_on_destroy = false -} diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/tinyproxy.conf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/tinyproxy.conf deleted file mode 100644 index 032c2dadb0cf1..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/tinyproxy.conf +++ /dev/null @@ -1,368 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -## -## tinyproxy.conf -- tinyproxy daemon configuration file -## -## This example tinyproxy.conf file contains example settings -## with explanations in comments. For decriptions of all -## parameters, see the tinproxy.conf(5) manual page. -## - -# -# User/Group: This allows you to set the user and group that will be -# used for tinyproxy after the initial binding to the port has been done -# as the root user. Either the user or group name or the UID or GID -# number may be used. -# -User tinyproxy -Group tinyproxy - -# -# Port: Specify the port which tinyproxy will listen on. Please note -# that should you choose to run on a port lower than 1024 you will need -# to start tinyproxy using root. -# -Port 8888 - -# -# Listen: If you have multiple interfaces this allows you to bind to -# only one. If this is commented out, tinyproxy will bind to all -# interfaces present. -# -#Listen 192.168.0.1 - -# -# Bind: This allows you to specify which interface will be used for -# outgoing connections. This is useful for multi-home'd machines where -# you want all traffic to appear outgoing from one particular interface. -# -#Bind 192.168.0.1 - -# -# BindSame: If enabled, tinyproxy will bind the outgoing connection to the -# ip address of the incoming connection. -# -#BindSame yes - -# -# Timeout: The maximum number of seconds of inactivity a connection is -# allowed to have before it is closed by tinyproxy. -# -Timeout 600 - -# -# ErrorFile: Defines the HTML file to send when a given HTTP error -# occurs. You will probably need to customize the location to your -# particular install. The usual locations to check are: -# /usr/local/share/tinyproxy -# /usr/share/tinyproxy -# /etc/tinyproxy -# -#ErrorFile 404 "/usr/share/tinyproxy/404.html" -#ErrorFile 400 "/usr/share/tinyproxy/400.html" -#ErrorFile 503 "/usr/share/tinyproxy/503.html" -#ErrorFile 403 "/usr/share/tinyproxy/403.html" -#ErrorFile 408 "/usr/share/tinyproxy/408.html" - -# -# DefaultErrorFile: The HTML file that gets sent if there is no -# HTML file defined with an ErrorFile keyword for the HTTP error -# that has occured. -# -DefaultErrorFile "/usr/share/tinyproxy/default.html" - -# -# StatHost: This configures the host name or IP address that is treated -# as the stat host: Whenever a request for this host is received, -# Tinyproxy will return an internal statistics page instead of -# forwarding the request to that host. The default value of StatHost is -# tinyproxy.stats. -# -#StatHost "tinyproxy.stats" -# - -# -# StatFile: The HTML file that gets sent when a request is made -# for the stathost. If this file doesn't exist a basic page is -# hardcoded in tinyproxy. -# -StatFile "/usr/share/tinyproxy/stats.html" - -# -# LogFile: Allows you to specify the location where information should -# be logged to. If you would prefer to log to syslog, then disable this -# and enable the Syslog directive. These directives are mutually -# exclusive. If neither Syslog nor LogFile are specified, output goes -# to stdout. -# -LogFile "/var/log/tinyproxy/tinyproxy.log" - -# -# Syslog: Tell tinyproxy to use syslog instead of a logfile. This -# option must not be enabled if the Logfile directive is being used. -# These two directives are mutually exclusive. -# -#Syslog On - -# -# LogLevel: Warning -# -# Set the logging level. Allowed settings are: -# Critical (least verbose) -# Error -# Warning -# Notice -# Connect (to log connections without Info's noise) -# Info (most verbose) -# -# The LogLevel logs from the set level and above. For example, if the -# LogLevel was set to Warning, then all log messages from Warning to -# Critical would be output, but Notice and below would be suppressed. -# -LogLevel Info - -# -# PidFile: Write the PID of the main tinyproxy thread to this file so it -# can be used for signalling purposes. -# If not specified, no pidfile will be written. -# -PidFile "/run/tinyproxy/tinyproxy.pid" - -# -# XTinyproxy: Tell Tinyproxy to include the X-Tinyproxy header, which -# contains the client's IP address. -# -#XTinyproxy Yes - -# -# Upstream: -# -# Turns on upstream proxy support. -# -# The upstream rules allow you to selectively route upstream connections -# based on the host/domain of the site being accessed. -# -# Syntax: upstream type (user:pass@)ip:port ("domain") -# Or: upstream none "domain" -# The parts in parens are optional. -# Possible types are http, socks4, socks5, none -# -# For example: -# # connection to test domain goes through testproxy -# upstream http testproxy:8008 ".test.domain.invalid" -# upstream http testproxy:8008 ".our_testbed.example.com" -# upstream http testproxy:8008 "192.168.128.0/255.255.254.0" -# -# # upstream proxy using basic authentication -# upstream http user:pass@testproxy:8008 ".test.domain.invalid" -# -# # no upstream proxy for internal websites and unqualified hosts -# upstream none ".internal.example.com" -# upstream none "www.example.com" -# upstream none "10.0.0.0/8" -# upstream none "192.168.0.0/255.255.254.0" -# upstream none "." -# -# # connection to these boxes go through their DMZ firewalls -# upstream http cust1_firewall:8008 "testbed_for_cust1" -# upstream http cust2_firewall:8008 "testbed_for_cust2" -# -# # default upstream is internet firewall -# upstream http firewall.internal.example.com:80 -# -# You may also use SOCKS4/SOCKS5 upstream proxies: -# upstream socks4 127.0.0.1:9050 -# upstream socks5 socksproxy:1080 -# -# The LAST matching rule wins the route decision. As you can see, you -# can use a host, or a domain: -# name matches host exactly -# .name matches any host in domain "name" -# . matches any host with no domain (in 'empty' domain) -# IP/bits matches network/mask -# IP/mask matches network/mask -# -#Upstream http some.remote.proxy:port - -# -# MaxClients: This is the absolute highest number of threads which will -# be created. In other words, only MaxClients number of clients can be -# connected at the same time. -# -MaxClients 100 - -# -# MinSpareServers/MaxSpareServers: These settings set the upper and -# lower limit for the number of spare servers which should be available. -# -# If the number of spare servers falls below MinSpareServers then new -# server processes will be spawned. If the number of servers exceeds -# MaxSpareServers then the extras will be killed off. -# -MinSpareServers 5 -MaxSpareServers 20 - -# -# StartServers: The number of servers to start initially. -# -StartServers 10 - -# -# MaxRequestsPerChild: The number of connections a thread will handle -# before it is killed. In practise this should be set to 0, which -# disables thread reaping. If you do notice problems with memory -# leakage, then set this to something like 10000. -# -MaxRequestsPerChild 0 - -# -# Allow: Customization of authorization controls. If there are any -# access control keywords then the default action is to DENY. Otherwise, -# the default action is ALLOW. -# -# The order of the controls are important. All incoming connections are -# tested against the controls based on order. -# -Allow 127.0.0.1 -Allow localhost -#Allow 192.168.0.0/16 -#Allow 172.16.0.0/12 -#Allow 10.0.0.0/8 - -# BasicAuth: HTTP "Basic Authentication" for accessing the proxy. -# If there are any entries specified, access is only granted for authenticated -# users. -#BasicAuth user password - -# -# AddHeader: Adds the specified headers to outgoing HTTP requests that -# Tinyproxy makes. Note that this option will not work for HTTPS -# traffic, as Tinyproxy has no control over what headers are exchanged. -# -#AddHeader "X-My-Header" "Powered by Tinyproxy" - -# -# ViaProxyName: The "Via" header is required by the HTTP RFC, but using -# the real host name is a security concern. If the following directive -# is enabled, the string supplied will be used as the host name in the -# Via header; otherwise, the server's host name will be used. -# -ViaProxyName "tinyproxy" - -# -# DisableViaHeader: When this is set to yes, Tinyproxy does NOT add -# the Via header to the requests. This virtually puts Tinyproxy into -# stealth mode. Note that RFC 2616 requires proxies to set the Via -# header, so by enabling this option, you break compliance. -# Don't disable the Via header unless you know what you are doing... -# -#DisableViaHeader Yes - -# -# Filter: This allows you to specify the location of the filter file. -# -#Filter "/etc/tinyproxy/filter" - -# -# FilterURLs: Filter based on URLs rather than domains. -# -#FilterURLs On - -# -# FilterExtended: Use POSIX Extended regular expressions rather than -# basic. -# -#FilterExtended On - -# -# FilterCaseSensitive: Use case sensitive regular expressions. -# -#FilterCaseSensitive On - -# -# FilterDefaultDeny: Change the default policy of the filtering system. -# If this directive is commented out, or is set to "No" then the default -# policy is to allow everything which is not specifically denied by the -# filter file. -# -# However, by setting this directive to "Yes" the default policy becomes -# to deny everything which is _not_ specifically allowed by the filter -# file. -# -#FilterDefaultDeny Yes - -# -# Anonymous: If an Anonymous keyword is present, then anonymous proxying -# is enabled. The headers listed are allowed through, while all others -# are denied. If no Anonymous keyword is present, then all headers are -# allowed through. You must include quotes around the headers. -# -# Most sites require cookies to be enabled for them to work correctly, so -# you will need to allow Cookies through if you access those sites. -# -#Anonymous "Host" -#Anonymous "Authorization" -#Anonymous "Cookie" - -# -# ConnectPort: This is a list of ports allowed by tinyproxy when the -# CONNECT method is used. To disable the CONNECT method altogether, set -# the value to 0. If no ConnectPort line is found, all ports are -# allowed. -# -# The following two ports are used by SSL. -# -ConnectPort 443 -ConnectPort 563 - -# -# Configure one or more ReversePath directives to enable reverse proxy -# support. With reverse proxying it's possible to make a number of -# sites appear as if they were part of a single site. -# -# If you uncomment the following two directives and run tinyproxy -# on your own computer at port 8888, you can access Google using -# http://localhost:8888/google/ and Wired News using -# http://localhost:8888/wired/news/. Neither will actually work -# until you uncomment ReverseMagic as they use absolute linking. -# -#ReversePath "/google/" "http://www.google.com/" -#ReversePath "/wired/" "http://www.wired.com/" - -# -# When using tinyproxy as a reverse proxy, it is STRONGLY recommended -# that the normal proxy is turned off by uncommenting the next directive. -# -#ReverseOnly Yes - -# -# Use a cookie to track reverse proxy mappings. If you need to reverse -# proxy sites which have absolute links you must uncomment this. -# -#ReverseMagic Yes - -# -# The URL that's used to access this reverse proxy. The URL is used to -# rewrite HTTP redirects so that they won't escape the proxy. If you -# have a chain of reverse proxies, you'll need to put the outermost -# URL here (the address which the end user types into his/her browser). -# -# If not set then no rewriting occurs. -# -#ReverseBaseURL "http://localhost:8888/" diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/variables.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/variables.tf deleted file mode 100644 index 66a65720a831f..0000000000000 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/04-bastion/variables.tf +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project" { - type = string - description = "The Google Cloud Platform (GCP) project within which resources are provisioned" -} - -variable "region" { - type = string - description = "The Google Cloud Platform (GCP) region in which to provision resources" -} - -variable "kubernetes_node_service_account" { - type = object({ - email = string - }) - description = "The Google Cloud Platform Service Account bound to the GKE node" -} - -variable "network" { - type = object({ - name = string - }) - description = "The Google Cloud Platform Virtual Cloud network within which we provision the kubernetes node" -} - -variable "subnetwork" { - type = object({ - name = string - }) - description = "The Google Cloud Platform Virtual Cloud subnetwork within which we provision the kubernetes node" -} - -variable "bastion_compute_machine_type" { - type = string - description = "The machine type of the Bastion host. See gcloud compute machine-types list for available types" -} - -variable "router" { - type = object({ - name = string - }) - description = "The Google Cloud Platform Virtual Cloud router" -} - -variable "nat" { - type = object({ - name = string - }) - description = "The Google Cloud Platform Virtual Cloud router NAT" -} diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/prerequisites.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/prerequisites.tf new file mode 100644 index 0000000000000..d17998fa9188b --- /dev/null +++ b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/prerequisites.tf @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_project_service" "required" { + for_each = toset([ + "container", + "iam", + ]) + service = "${each.key}.googleapis.com" + disable_on_destroy = false +} + +// Query the VPC network to make sure it exists. +data "google_compute_network" "default" { + depends_on = [google_project_service.required] + name = var.network +} + +// Query the VPC subnetwork to make sure it exists in the region specified. +data "google_compute_subnetwork" "default" { + depends_on = [google_project_service.required] + name = var.subnetwork + region = var.region +} + +// Query the Service Account. +data "google_service_account" "default" { + depends_on = [google_project_service.required] + account_id = var.service_account_id +} diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/01-setup/provider.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/provider.tf similarity index 100% rename from .test-infra/terraform/google-cloud-platform/google-kubernetes-engine/modules/01-setup/provider.tf rename to .test-infra/terraform/google-cloud-platform/google-kubernetes-engine/provider.tf diff --git a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/variables.tf b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/variables.tf index aa43c4ff95968..40dc0f0a8d5a6 100644 --- a/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/variables.tf +++ b/.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/variables.tf @@ -26,17 +26,22 @@ variable "region" { description = "The Google Cloud Platform (GCP) region in which to provision resources" } -variable "resource_name_prefix" { +variable "cluster_name_prefix" { type = string - description = "The basis to name all provisioned resources i.e. service account, network, cluster, etc." + description = "The prefix to assign the provisioned Google Kubernetes Engine (GKE) cluster; a random string is appended to this value" } -variable "subnetwork_cidr_range" { +variable "network" { type = string - description = "The address range for this subnet, in CIDR notation. Use a standard private VPC network address range: for example, 10.128.0.0/20" + description = "The Google Cloud Virtual Private Cloud (VPC) network name" } -variable "bastion_compute_machine_type" { +variable "subnetwork" { type = string - description = "The machine type of the Bastion host. See gcloud compute machine-types list for available types, for example e2-standard-2" + description = "The Google Cloud Virtual Private Cloud (VPC) subnetwork name" +} + +variable "service_account_id" { + type = string + description = "The ID of the service account bound to the Google Kubernetes Engine (GKE) cluster" } diff --git a/.test-infra/terraform/google-cloud-platform/vertex-ai-featurestore/README.md b/.test-infra/terraform/google-cloud-platform/vertex-ai-featurestore/README.md new file mode 100644 index 0000000000000..a84e5f62383ba --- /dev/null +++ b/.test-infra/terraform/google-cloud-platform/vertex-ai-featurestore/README.md @@ -0,0 +1,74 @@ + + +# Overview + +This module provisions a +[Vertex AI Featurestore](https://cloud.google.com/vertex-ai/docs/featurestore). + +# Requirements and Usage + +See [Google Cloud Platform requirements](../../../google-cloud-platform/README.md) +for details on requirements +and usage. + +## 1. Initialize the terraform module + +``` +cd .test-infra/terraform/google-cloud-platform/vertex-ai-featurestore +terraform init +``` + +## 2. Create a *.tfvars file + +Create a `*.tfvars` file in the same directory as this module. + +``` +cd .test-infra/terraform/google-cloud-platform/vertex-ai-featurestore +touch vars.tfvars +``` + +See [Examples](#examples) below for some example `*.tfvars` files. + +## 3. Apply the terraform module. + +``` +cd .test-infra/terraform/google-cloud-platform/vertex-ai-featurestore +terraform apply -var-file=vars.tfvars +``` + +# Examples + +## synthea.tfvars + +This directory holds a [synthea.tfvars](synthea.tfvars) to generate an +example Vertex AI Featurestore based on data generated from +https://github.com/synthetichealth/synthea +and stored in Google Cloud FHIR Store with BigQuery streaming. +See: https://cloud.google.com/healthcare-api/docs/how-tos/fhir-bigquery-streaming +for more details. + +To apply using this `*.tfvars` file: + +``` +cd .test-infra/terraform/google-cloud-platform/vertex-ai-featurestore +terraform apply -var-file=synthea.tfvars +``` + +You will be prompted for any remaining unset variables. diff --git a/.test-infra/terraform/google-cloud-platform/vertex-ai-featurestore/featurestore.tf b/.test-infra/terraform/google-cloud-platform/vertex-ai-featurestore/featurestore.tf new file mode 100644 index 0000000000000..e7f0d9d563cd1 --- /dev/null +++ b/.test-infra/terraform/google-cloud-platform/vertex-ai-featurestore/featurestore.tf @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "google" { + project = var.project +} + +resource "google_project_service" "required" { + service = "aiplatform.googleapis.com" + disable_on_destroy = false +} + +resource "random_string" "postfix" { + length = 6 + upper = false + special = false +} + +resource "google_vertex_ai_featurestore" "default" { + depends_on = [google_project_service.required] + name = "${var.featurestore.name_prefix}_${random_string.postfix.result}" + region = var.region + online_serving_config { + fixed_node_count = var.featurestore.fixed_node_count + } +} + +resource "google_vertex_ai_featurestore_entitytype" "entities" { + depends_on = [google_project_service.required] + for_each = var.featurestore.entity_types + name = each.key + featurestore = google_vertex_ai_featurestore.default.id + description = each.value.description + monitoring_config { + + categorical_threshold_config { + value = 0.3 + } + + numerical_threshold_config { + value = 0.3 + } + + snapshot_analysis { + disabled = false + monitoring_interval_days = 1 + staleness_days = 21 + } + } +} + +locals { + features = flatten([ + for entitytype_name, entitytype in var.featurestore.entity_types : [ + for feature_name, feature_type in entitytype.features : { + entitytype_name = entitytype_name + feature_name = feature_name + feature_type = feature_type + } + ] + ]) + features_map = tomap({ + for feature in local.features : + "${feature["entitytype_name"]}.${feature["feature_name"]}" => feature + }) +} + +resource "google_vertex_ai_featurestore_entitytype_feature" "features" { + depends_on = [google_project_service.required] + for_each = local.features_map + name = each.value["feature_name"] + entitytype = google_vertex_ai_featurestore_entitytype.entities[each.value["entitytype_name"]].id + value_type = each.value["feature_type"] +} diff --git a/.test-infra/terraform/google-cloud-platform/vertex-ai-featurestore/synthea.tfvars b/.test-infra/terraform/google-cloud-platform/vertex-ai-featurestore/synthea.tfvars new file mode 100644 index 0000000000000..b3b39edbbda41 --- /dev/null +++ b/.test-infra/terraform/google-cloud-platform/vertex-ai-featurestore/synthea.tfvars @@ -0,0 +1,621 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This tfvars file represents a Vertex AI Featurestore based on data generated + * from https://github.com/synthetichealth/synthea and stored in Google Cloud + * FHIR Store with BigQuery streaming. + * See: https://cloud.google.com/healthcare-api/docs/how-tos/fhir-bigquery-streaming + * for more details. + */ + +region = "us-central1" + +featurestore = { + name_prefix = "synthea" + fixed_node_count = 1 + entity_types = { + // Flags whether patient has condition identified by the Snomed code. + // Featurestore indexes features by an ID and a timestamp to reflect + // what is known about a feature at a given time. A tuple of a patient's + // id, timestamp and condition flags represents known active conditions + // of a patient at that point in time. + // See https://browser.ihtsdotools.org/ for more information about + // Snomed codes. + conditions = { + description = "Flags whether patient has condition identified by the Snomed code." + features = { + snomed_10509002 = "BOOL" + snomed_105531004 = "BOOL" + snomed_10939881000119105 = "BOOL" + snomed_109838007 = "BOOL" + snomed_110030002 = "BOOL" + snomed_110359009 = "BOOL" + snomed_111282000 = "BOOL" + snomed_111287006 = "BOOL" + snomed_1121000119107 = "BOOL" + snomed_1231000119100 = "BOOL" + snomed_124171000119105 = "BOOL" + snomed_125601008 = "BOOL" + snomed_125605004 = "BOOL" + snomed_126906006 = "BOOL" + snomed_127013003 = "BOOL" + snomed_127295002 = "BOOL" + snomed_128188000 = "BOOL" + snomed_128613002 = "BOOL" + snomed_132281000119108 = "BOOL" + snomed_14760008 = "BOOL" + snomed_1501000119109 = "BOOL" + snomed_1551000119108 = "BOOL" + snomed_156073000 = "BOOL" + snomed_157141000119108 = "BOOL" + snomed_15724005 = "BOOL" + snomed_157265008 = "BOOL" + snomed_15777000 = "BOOL" + snomed_15802004 = "BOOL" + snomed_160701002 = "BOOL" + snomed_160903007 = "BOOL" + snomed_160904001 = "BOOL" + snomed_160968000 = "BOOL" + snomed_16114001 = "BOOL" + snomed_161665007 = "BOOL" + snomed_161679004 = "BOOL" + snomed_162573006 = "BOOL" + snomed_162864005 = "BOOL" + snomed_171131006 = "BOOL" + snomed_1734006 = "BOOL" + snomed_183996000 = "BOOL" + snomed_185086009 = "BOOL" + snomed_1871000124103 = "BOOL" + snomed_19169002 = "BOOL" + snomed_192127007 = "BOOL" + snomed_195662009 = "BOOL" + snomed_195967001 = "BOOL" + snomed_196416002 = "BOOL" + snomed_197927001 = "BOOL" + snomed_198992004 = "BOOL" + snomed_201834006 = "BOOL" + snomed_203082005 = "BOOL" + snomed_213150003 = "BOOL" + snomed_221360009 = "BOOL" + snomed_22298006 = "BOOL" + snomed_224295006 = "BOOL" + snomed_224299000 = "BOOL" + snomed_224355006 = "BOOL" + snomed_230265002 = "BOOL" + snomed_230690007 = "BOOL" + snomed_232353008 = "BOOL" + snomed_233604007 = "BOOL" + snomed_233678006 = "BOOL" + snomed_234466008 = "BOOL" + snomed_235595009 = "BOOL" + snomed_235919008 = "BOOL" + snomed_236077008 = "BOOL" + snomed_237602007 = "BOOL" + snomed_239720000 = "BOOL" + snomed_239872002 = "BOOL" + snomed_239873007 = "BOOL" + snomed_24079001 = "BOOL" + snomed_241929008 = "BOOL" + snomed_246677007 = "BOOL" + snomed_248595008 = "BOOL" + snomed_249497008 = "BOOL" + snomed_25064002 = "BOOL" + snomed_254632001 = "BOOL" + snomed_254637007 = "BOOL" + snomed_254837009 = "BOOL" + snomed_25675004 = "BOOL" + snomed_262521009 = "BOOL" + snomed_262574004 = "BOOL" + snomed_263102004 = "BOOL" + snomed_266934004 = "BOOL" + snomed_266948004 = "BOOL" + snomed_267020005 = "BOOL" + snomed_267036007 = "BOOL" + snomed_267060006 = "BOOL" + snomed_267102003 = "BOOL" + snomed_26929004 = "BOOL" + snomed_271737000 = "BOOL" + snomed_271825005 = "BOOL" + snomed_274531002 = "BOOL" + snomed_275272006 = "BOOL" + snomed_278860009 = "BOOL" + snomed_283371005 = "BOOL" + snomed_283385000 = "BOOL" + snomed_283545005 = "BOOL" + snomed_284549007 = "BOOL" + snomed_284551006 = "BOOL" + snomed_288959006 = "BOOL" + snomed_300916003 = "BOOL" + snomed_301011002 = "BOOL" + snomed_302297009 = "BOOL" + snomed_302870006 = "BOOL" + snomed_307731004 = "BOOL" + snomed_30832001 = "BOOL" + snomed_312157006 = "BOOL" + snomed_312608009 = "BOOL" + snomed_314529007 = "BOOL" + snomed_315268008 = "BOOL" + snomed_32911000 = "BOOL" + snomed_33737001 = "BOOL" + snomed_359817006 = "BOOL" + snomed_35999006 = "BOOL" + snomed_361055000 = "BOOL" + snomed_363406005 = "BOOL" + snomed_367498001 = "BOOL" + snomed_368581000119106 = "BOOL" + snomed_36923009 = "BOOL" + snomed_36955009 = "BOOL" + snomed_36971009 = "BOOL" + snomed_370143000 = "BOOL" + snomed_370247008 = "BOOL" + snomed_384709000 = "BOOL" + snomed_386661006 = "BOOL" + snomed_38822007 = "BOOL" + snomed_389087006 = "BOOL" + snomed_398152000 = "BOOL" + snomed_398254007 = "BOOL" + snomed_39848009 = "BOOL" + snomed_39898005 = "BOOL" + snomed_399211009 = "BOOL" + snomed_399261000 = "BOOL" + snomed_399912005 = "BOOL" + snomed_40055000 = "BOOL" + snomed_401303003 = "BOOL" + snomed_401314000 = "BOOL" + snomed_40275004 = "BOOL" + snomed_403190006 = "BOOL" + snomed_403191005 = "BOOL" + snomed_403192003 = "BOOL" + snomed_408512008 = "BOOL" + snomed_409089005 = "BOOL" + snomed_414545008 = "BOOL" + snomed_414667000 = "BOOL" + snomed_422034002 = "BOOL" + snomed_422587007 = "BOOL" + snomed_422650009 = "BOOL" + snomed_423315002 = "BOOL" + snomed_424132000 = "BOOL" + snomed_424393004 = "BOOL" + snomed_427419006 = "BOOL" + snomed_428251008 = "BOOL" + snomed_428915008 = "BOOL" + snomed_429280009 = "BOOL" + snomed_431855005 = "BOOL" + snomed_431856006 = "BOOL" + snomed_431857002 = "BOOL" + snomed_433144002 = "BOOL" + snomed_43724002 = "BOOL" + snomed_43878008 = "BOOL" + snomed_44054006 = "BOOL" + snomed_443165006 = "BOOL" + snomed_444448004 = "BOOL" + snomed_444470001 = "BOOL" + snomed_44465007 = "BOOL" + snomed_444814009 = "BOOL" + snomed_446096008 = "BOOL" + snomed_446654005 = "BOOL" + snomed_449868002 = "BOOL" + snomed_4557003 = "BOOL" + snomed_45816000 = "BOOL" + snomed_46177005 = "BOOL" + snomed_47505003 = "BOOL" + snomed_47693006 = "BOOL" + snomed_48333001 = "BOOL" + snomed_48724000 = "BOOL" + snomed_49436004 = "BOOL" + snomed_49727002 = "BOOL" + snomed_49915006 = "BOOL" + snomed_5251000175109 = "BOOL" + snomed_53827007 = "BOOL" + snomed_55680006 = "BOOL" + snomed_55822004 = "BOOL" + snomed_56018004 = "BOOL" + snomed_5602001 = "BOOL" + snomed_56786000 = "BOOL" + snomed_57676002 = "BOOL" + snomed_58150001 = "BOOL" + snomed_59621000 = "BOOL" + snomed_60234000 = "BOOL" + snomed_60573004 = "BOOL" + snomed_6072007 = "BOOL" + snomed_62106007 = "BOOL" + snomed_62564004 = "BOOL" + snomed_62718007 = "BOOL" + snomed_64859006 = "BOOL" + snomed_6525002 = "BOOL" + snomed_65275009 = "BOOL" + snomed_65363002 = "BOOL" + snomed_65710008 = "BOOL" + snomed_65966004 = "BOOL" + snomed_66857006 = "BOOL" + snomed_67782005 = "BOOL" + snomed_67811000119102 = "BOOL" + snomed_68235000 = "BOOL" + snomed_68496003 = "BOOL" + snomed_68566005 = "BOOL" + snomed_68962001 = "BOOL" + snomed_698306007 = "BOOL" + snomed_698754002 = "BOOL" + snomed_69896004 = "BOOL" + snomed_703151001 = "BOOL" + snomed_706870000 = "BOOL" + snomed_706893006 = "BOOL" + snomed_70704007 = "BOOL" + snomed_713197008 = "BOOL" + snomed_713458007 = "BOOL" + snomed_7200002 = "BOOL" + snomed_72892002 = "BOOL" + snomed_73430006 = "BOOL" + snomed_73438004 = "BOOL" + snomed_73595000 = "BOOL" + snomed_741062008 = "BOOL" + snomed_74400008 = "BOOL" + snomed_75498004 = "BOOL" + snomed_76571007 = "BOOL" + snomed_770349000 = "BOOL" + snomed_78275009 = "BOOL" + snomed_79586000 = "BOOL" + snomed_79619009 = "BOOL" + snomed_80394007 = "BOOL" + snomed_80583007 = "BOOL" + snomed_82423001 = "BOOL" + snomed_83664006 = "BOOL" + snomed_840539006 = "BOOL" + snomed_840544004 = "BOOL" + snomed_84229001 = "BOOL" + snomed_84757009 = "BOOL" + snomed_86406008 = "BOOL" + snomed_86849004 = "BOOL" + snomed_87433001 = "BOOL" + snomed_88805009 = "BOOL" + snomed_90460009 = "BOOL" + snomed_90560007 = "BOOL" + snomed_90781000119102 = "BOOL" + snomed_91302008 = "BOOL" + snomed_91434003 = "BOOL" + snomed_91861009 = "BOOL" + snomed_92691004 = "BOOL" + snomed_93761005 = "BOOL" + snomed_94260004 = "BOOL" + snomed_95417003 = "BOOL" + snomed_97331000119101 = "BOOL" + } + } + + // Flags whether patient has active medication identified by its RxNorm code. + // Featurestore indexes features by an ID and a timestamp to reflect + // what is known about a feature at a given time. A tuple of a patient's + // id, timestamp and medication flags represents known active medications + // of a patient at that point in time. + // See https://www.nlm.nih.gov/research/umls/rxnorm/index.html for more + // details about RxNorm codes. + medications = { + description = "Flags whether patient has active medication identified by its RxNorm code" + features = { + rxnorm_1000126 = "BOOL" + rxnorm_108515 = "BOOL" + rxnorm_1116635 = "BOOL" + rxnorm_1234995 = "BOOL" + rxnorm_1361048 = "BOOL" + rxnorm_1361226 = "BOOL" + rxnorm_1601380 = "BOOL" + rxnorm_1605257 = "BOOL" + rxnorm_1659131 = "BOOL" + rxnorm_1659149 = "BOOL" + rxnorm_1659263 = "BOOL" + rxnorm_1664986 = "BOOL" + rxnorm_1665060 = "BOOL" + rxnorm_1719286 = "BOOL" + rxnorm_1723208 = "BOOL" + rxnorm_1728805 = "BOOL" + rxnorm_1729584 = "BOOL" + rxnorm_1732136 = "BOOL" + rxnorm_1732186 = "BOOL" + rxnorm_1734919 = "BOOL" + rxnorm_1735006 = "BOOL" + rxnorm_1736776 = "BOOL" + rxnorm_1736854 = "BOOL" + rxnorm_1737466 = "BOOL" + rxnorm_1740467 = "BOOL" + rxnorm_1790099 = "BOOL" + rxnorm_1796676 = "BOOL" + rxnorm_1803932 = "BOOL" + rxnorm_1804799 = "BOOL" + rxnorm_1807510 = "BOOL" + rxnorm_1808217 = "BOOL" + rxnorm_1809104 = "BOOL" + rxnorm_1856546 = "BOOL" + rxnorm_1860480 = "BOOL" + rxnorm_1873983 = "BOOL" + rxnorm_1946840 = "BOOL" + rxnorm_198039 = "BOOL" + rxnorm_198240 = "BOOL" + rxnorm_198440 = "BOOL" + rxnorm_199224 = "BOOL" + rxnorm_1997015 = "BOOL" + rxnorm_200243 = "BOOL" + rxnorm_200349 = "BOOL" + rxnorm_205923 = "BOOL" + rxnorm_2119714 = "BOOL" + rxnorm_212033 = "BOOL" + rxnorm_2123111 = "BOOL" + rxnorm_226719 = "BOOL" + rxnorm_238100 = "BOOL" + rxnorm_242969 = "BOOL" + rxnorm_309845 = "BOOL" + rxnorm_310261 = "BOOL" + rxnorm_311034 = "BOOL" + rxnorm_311700 = "BOOL" + rxnorm_312617 = "BOOL" + rxnorm_313212 = "BOOL" + rxnorm_389221 = "BOOL" + rxnorm_542347 = "BOOL" + rxnorm_562366 = "BOOL" + rxnorm_583214 = "BOOL" + rxnorm_597195 = "BOOL" + rxnorm_727711 = "BOOL" + rxnorm_727762 = "BOOL" + rxnorm_749196 = "BOOL" + rxnorm_752899 = "BOOL" + rxnorm_807283 = "BOOL" + rxnorm_854235 = "BOOL" + rxnorm_854252 = "BOOL" + rxnorm_854255 = "BOOL" + rxnorm_855812 = "BOOL" + rxnorm_979485 = "BOOL" + rxnorm_999999 = "BOOL" + } + } + + // Flags as 'LOW', 'MID', 'HIGH' patient's observations and measurements + // identified by its Loinc Code. + // Featurestore indexes features by an ID and a timestamp to reflect + // what is known about a feature at a given time. A tuple of a patient's + // id, timestamp and observations represents the latest known value + // of a patient at that point in time. + // See https://loinc.org/ for more information about Loinc codes. + observations = { + description = "Flags whether patient has active medication identified by its Loinc code" + features = { + loinc_10230_1 = "STRING" + loinc_10480_2 = "STRING" + loinc_10834_0 = "STRING" + loinc_14627_4 = "STRING" + loinc_14804_9 = "STRING" + loinc_14959_1 = "STRING" + loinc_1742_6 = "STRING" + loinc_1751_7 = "STRING" + loinc_17861_6 = "STRING" + loinc_18262_6 = "STRING" + loinc_18752_6 = "STRING" + loinc_19123_9 = "STRING" + loinc_1920_8 = "STRING" + loinc_1960_4 = "STRING" + loinc_1975_2 = "STRING" + loinc_1988_5 = "STRING" + loinc_19926_5 = "STRING" + loinc_19994_3 = "STRING" + loinc_2019_8 = "STRING" + loinc_2021_4 = "STRING" + loinc_2027_1 = "STRING" + loinc_2028_9 = "STRING" + loinc_20447_9 = "STRING" + loinc_20454_5 = "STRING" + loinc_20505_4 = "STRING" + loinc_20565_8 = "STRING" + loinc_20570_8 = "STRING" + loinc_2069_3 = "STRING" + loinc_2075_0 = "STRING" + loinc_2085_9 = "STRING" + loinc_2093_3 = "STRING" + loinc_21000_5 = "STRING" + loinc_21377_7 = "STRING" + loinc_2157_6 = "STRING" + loinc_2160_0 = "STRING" + loinc_21905_5 = "STRING" + loinc_21906_3 = "STRING" + loinc_21907_1 = "STRING" + loinc_21908_9 = "STRING" + loinc_21924_6 = "STRING" + loinc_2276_4 = "STRING" + loinc_2339_0 = "STRING" + loinc_2345_7 = "STRING" + loinc_24467_3 = "STRING" + loinc_2498_4 = "STRING" + loinc_2500_7 = "STRING" + loinc_2502_3 = "STRING" + loinc_2514_8 = "STRING" + loinc_2532_0 = "STRING" + loinc_25428_4 = "STRING" + loinc_2571_8 = "STRING" + loinc_26453_1 = "STRING" + loinc_26464_8 = "STRING" + loinc_26515_7 = "STRING" + loinc_2703_7 = "STRING" + loinc_2705_2 = "STRING" + loinc_2708_6 = "STRING" + loinc_2713_6 = "STRING" + loinc_2744_1 = "STRING" + loinc_2746_6 = "STRING" + loinc_2777_1 = "STRING" + loinc_2823_3 = "STRING" + loinc_28245_9 = "STRING" + loinc_2857_1 = "STRING" + loinc_2885_2 = "STRING" + loinc_29463_7 = "STRING" + loinc_2947_0 = "STRING" + loinc_2951_2 = "STRING" + loinc_29554_3 = "STRING" + loinc_3016_3 = "STRING" + loinc_3024_7 = "STRING" + loinc_30385_9 = "STRING" + loinc_30428_7 = "STRING" + loinc_3094_0 = "STRING" + loinc_3173_2 = "STRING" + loinc_3184_9 = "STRING" + loinc_32167_9 = "STRING" + loinc_32207_3 = "STRING" + loinc_32465_7 = "STRING" + loinc_32623_1 = "STRING" + loinc_32693_4 = "STRING" + loinc_33037_3 = "STRING" + loinc_33728_7 = "STRING" + loinc_33756_8 = "STRING" + loinc_33762_6 = "STRING" + loinc_33914_3 = "STRING" + loinc_33959_8 = "STRING" + loinc_34533_0 = "STRING" + loinc_38208_5 = "STRING" + loinc_38265_5 = "STRING" + loinc_38483_4 = "STRING" + loinc_39156_5 = "STRING" + loinc_42719_5 = "STRING" + loinc_44261_6 = "STRING" + loinc_44667_4 = "STRING" + loinc_44963_7 = "STRING" + loinc_4544_3 = "STRING" + loinc_4548_4 = "STRING" + loinc_46240_8 = "STRING" + loinc_46288_7 = "STRING" + loinc_48065_7 = "STRING" + loinc_49765_1 = "STRING" + loinc_55277_8 = "STRING" + loinc_55758_7 = "STRING" + loinc_5767_9 = "STRING" + loinc_5770_3 = "STRING" + loinc_5778_6 = "STRING" + loinc_57905_2 = "STRING" + loinc_5792_7 = "STRING" + loinc_5794_3 = "STRING" + loinc_5797_6 = "STRING" + loinc_5799_2 = "STRING" + loinc_5802_4 = "STRING" + loinc_5803_2 = "STRING" + loinc_5804_0 = "STRING" + loinc_5811_5 = "STRING" + loinc_5902_2 = "STRING" + loinc_5905_5 = "STRING" + loinc_59408_5 = "STRING" + loinc_59460_6 = "STRING" + loinc_59461_4 = "STRING" + loinc_59557_9 = "STRING" + loinc_59576_9 = "STRING" + loinc_6075_6 = "STRING" + loinc_6082_2 = "STRING" + loinc_6085_5 = "STRING" + loinc_6095_4 = "STRING" + loinc_6106_9 = "STRING" + loinc_6158_0 = "STRING" + loinc_6189_5 = "STRING" + loinc_6206_7 = "STRING" + loinc_6246_3 = "STRING" + loinc_6248_9 = "STRING" + loinc_6273_7 = "STRING" + loinc_6276_0 = "STRING" + loinc_6298_4 = "STRING" + loinc_6299_2 = "STRING" + loinc_6301_6 = "STRING" + loinc_63513_6 = "STRING" + loinc_65750_2 = "STRING" + loinc_66519_0 = "STRING" + loinc_66524_0 = "STRING" + loinc_66529_9 = "STRING" + loinc_66534_9 = "STRING" + loinc_6690_2 = "STRING" + loinc_6768_6 = "STRING" + loinc_6833_8 = "STRING" + loinc_6844_5 = "STRING" + loinc_70006_2 = "STRING" + loinc_70274_6 = "STRING" + loinc_704_7 = "STRING" + loinc_706_2 = "STRING" + loinc_711_2 = "STRING" + loinc_713_8 = "STRING" + loinc_71425_3 = "STRING" + loinc_718_7 = "STRING" + loinc_71802_3 = "STRING" + loinc_72091_2 = "STRING" + loinc_72106_8 = "STRING" + loinc_72166_2 = "STRING" + loinc_72514_3 = "STRING" + loinc_7258_7 = "STRING" + loinc_731_0 = "STRING" + loinc_736_9 = "STRING" + loinc_74006_8 = "STRING" + loinc_742_7 = "STRING" + loinc_751_8 = "STRING" + loinc_75325_1 = "STRING" + loinc_75443_2 = "STRING" + loinc_75626_2 = "STRING" + loinc_76504_0 = "STRING" + loinc_76690_7 = "STRING" + loinc_770_8 = "STRING" + loinc_77606_2 = "STRING" + loinc_777_3 = "STRING" + loinc_785_6 = "STRING" + loinc_786_4 = "STRING" + loinc_787_2 = "STRING" + loinc_788_0 = "STRING" + loinc_789_8 = "STRING" + loinc_80271_0 = "STRING" + loinc_80382_5 = "STRING" + loinc_80383_3 = "STRING" + loinc_82667_7 = "STRING" + loinc_8289_1 = "STRING" + loinc_8302_2 = "STRING" + loinc_8310_5 = "STRING" + loinc_8331_1 = "STRING" + loinc_84215_3 = "STRING" + loinc_8478_0 = "STRING" + loinc_85318_4 = "STRING" + loinc_85319_2 = "STRING" + loinc_85337_4 = "STRING" + loinc_85339_0 = "STRING" + loinc_85343_2 = "STRING" + loinc_85344_0 = "STRING" + loinc_85352_3 = "STRING" + loinc_85354_9 = "STRING" + loinc_86923_0 = "STRING" + loinc_88020_3 = "STRING" + loinc_88021_1 = "STRING" + loinc_88040_1 = "STRING" + loinc_88262_1 = "STRING" + loinc_8867_4 = "STRING" + loinc_89204_2 = "STRING" + loinc_89579_7 = "STRING" + loinc_91148_7 = "STRING" + loinc_92130_4 = "STRING" + loinc_92131_2 = "STRING" + loinc_92134_6 = "STRING" + loinc_92138_7 = "STRING" + loinc_92139_5 = "STRING" + loinc_92140_3 = "STRING" + loinc_92141_1 = "STRING" + loinc_92142_9 = "STRING" + loinc_9279_1 = "STRING" + loinc_93025_5 = "STRING" + loinc_94040_3 = "STRING" + loinc_94531_1 = "STRING" + loinc_9843_4 = "STRING" + loinc_99999_0 = "STRING" + loinc_x9999_0 = "STRING" + } + } + } +} diff --git a/.test-infra/terraform/google-cloud-platform/vertex-ai-featurestore/variables.tf b/.test-infra/terraform/google-cloud-platform/vertex-ai-featurestore/variables.tf new file mode 100644 index 0000000000000..6a87bcc1e5673 --- /dev/null +++ b/.test-infra/terraform/google-cloud-platform/vertex-ai-featurestore/variables.tf @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project" { + type = string + description = "The Google Cloud Platform (GCP) project within which resources are provisioned" +} + +variable "region" { + type = string + description = "The Google Cloud Platform (GCP) region in which to provision resources" +} + +variable "featurestore" { + type = object({ + // The prefix of the Featurestore name. + name_prefix = string + + // The number of nodes to configure the Featurestore. + fixed_node_count = number + + // The Featurestore's Entity Type configuration where the map key configures + // the Entity Type name. + entity_types = map(object({ + + // The Entity Type's features configuration where the map key configures + // the Feature name and the value configures its data type such as + // BOOL, STRING, INT64, etc. + // See https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.featurestores.entityTypes.features#ValueType + // for allowed data types. + description = string + features = map(string) + })) + }) + description = "The Vertex AI Featurestore configuration" +} diff --git a/.test-infra/tools/README.md b/.test-infra/tools/README.md index df6ab9525e9a2..758c3885a8141 100644 --- a/.test-infra/tools/README.md +++ b/.test-infra/tools/README.md @@ -47,7 +47,7 @@ Example: - Original ```bash -python_versions_arr=("3.8.16" "3.7.16" "3.9.16" "3.10.10") +python_versions_arr=("3.8.16" "3.9.16" "3.10.10" "3.11.4") ``` - Change diff --git a/.test-infra/tools/build.gradle b/.test-infra/tools/build.gradle index ea4a12a24fc69..5ed6ebfc18f6d 100644 --- a/.test-infra/tools/build.gradle +++ b/.test-infra/tools/build.gradle @@ -24,6 +24,16 @@ task removeStaleSDKContainerImages(type: Exec) { commandLine './stale_dataflow_prebuilt_image_cleaner.sh' } -task cleanupOtherStaleResources(type: Exec) { +task removeStaleBqDatasets(type: Exec) { commandLine './stale_bq_datasets_cleaner.sh' } + +task removeStaleK8sWorkload(type: Exec) { + commandLine './stale_k8s_workload_cleaner.sh' +} + +task cleanupOtherStaleResources { + // declared as finalizedBy dependency so that other task continue even if one dep task fails + finalizedBy tasks.removeStaleBqDatasets + finalizedBy tasks.removeStaleK8sWorkload +} diff --git a/.test-infra/tools/python_installer.sh b/.test-infra/tools/python_installer.sh index 0b40eb12b993c..b1b05e597cb3b 100644 --- a/.test-infra/tools/python_installer.sh +++ b/.test-infra/tools/python_installer.sh @@ -20,7 +20,7 @@ set -euo pipefail # Variable containing the python versions to install -python_versions_arr=("3.8.16" "3.7.16" "3.9.16" "3.10.10") +python_versions_arr=("3.8.16" "3.9.16" "3.10.10" "3.11.4") # Install pyenv dependencies. pyenv_dep(){ diff --git a/.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh b/.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh index a9f7be945ffc6..eb3acb809ff51 100755 --- a/.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh +++ b/.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh @@ -22,8 +22,8 @@ set -euo pipefail # Clean up private registry (us.gcr.io) # Images more than 5 day old and not the latest (either has latest label or newest) -PUBLIC_REPOSITORIES=(beam-sdk beam_portability) -PRIVATE_REPOSITORIES=(java-postcommit-it python-postcommit-it jenkins) +PUBLIC_REPOSITORIES=(beam-sdk beam_portability beamgrafana beammetricssyncjenkins beammetricssyncgithub) +PRIVATE_REPOSITORIES=(java-postcommit-it python-postcommit-it jenkins github-actions) # set as the same as 6-week release period DELETE_BEFORE_DAY=$(date --iso-8601=s -d '6 weeks ago') @@ -51,6 +51,7 @@ done for image_name in ${IMAGE_NAMES[@]}; do echo IMAGES FOR image ${image_name} + FAILED_TO_DELETE="" # get the newest image without latest label LATEST_IN_TIME=$(gcloud container images list-tags \ ${image_name} --sort-by="~TIMESTAMP" --filter="NOT tags:latest " --format="get(digest)" --limit=1) @@ -70,10 +71,18 @@ for image_name in ${IMAGE_NAMES[@]}; do # this make sure we leave at least one container under each image name, either labelled "latest" or not if [ "$LATEST_IN_TIME" != "$current" ]; then echo "Deleting image. Command: gcloud container images delete ${image_name}@"${current}" --force-delete-tags -q" - gcloud container images delete ${image_name}@"${current}" --force-delete-tags -q + gcloud container images delete ${image_name}@"${current}" --force-delete-tags -q || FAILED_TO_DELETE+="${current} " fi done fi + + # Some images may not be successfully deleted the first time due to flakiness or having a dependency that hasn't been deleted yet. + RETRY_DELETE=("${FAILED_TO_DELETE[@]}") + echo "Failed to delete the following images: ${FAILED_TO_DELETE}. Retrying each of them." + for current in $RETRY_DELETE; do + echo "Trying again to delete image ${image_name}@"${current}". Command: gcloud container images delete ${image_name}@"${current}" --force-delete-tags -q" + gcloud container images delete ${image_name}@"${current}" --force-delete-tags -q + done done if [[ ${STALE_IMAGES} ]]; then diff --git a/.test-infra/tools/stale_k8s_workload_cleaner.sh b/.test-infra/tools/stale_k8s_workload_cleaner.sh new file mode 100755 index 0000000000000..f4d6f9b88324c --- /dev/null +++ b/.test-infra/tools/stale_k8s_workload_cleaner.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Deletes stale and old BQ datasets that are left after tests. +# + +set -euo pipefail + +# Clean up the stale kubernetes workload of given cluster + +PROJECT=apache-beam-testing +LOCATION=us-central1-a +CLUSTER=io-datastores + +function should_teardown() { + if [[ $1 =~ ^([0-9]+)([a-z]) ]]; then + local time_scale=${BASH_REMATCH[1]} + local time_unit=${BASH_REMATCH[2]} + # cutoff = 8 h + if [ $time_unit == y ] || [ $time_unit == d ]; then + return 0 + elif [ $time_unit == h ] && [ $time_scale -ge 8 ]; then + return 0 + fi + fi + return 1 +} + +gcloud container clusters get-credentials io-datastores --zone us-central1-a --project apache-beam-testing + +while read NAME STATUS AGE; do + if [[ $NAME =~ ^beam-.+test ]] && should_teardown $AGE; then + kubectl delete namespace $NAME + fi +done < <( kubectl get namespaces --context=gke_${PROJECT}_${LOCATION}_${CLUSTER} ) diff --git a/CHANGES.md b/CHANGES.md index 117274880d537..0f1530a78fe9c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -45,16 +45,22 @@ * Fixed X (Java/Python) ([#X](https://github.com/apache/beam/issues/X)). +## Security Fixes +* Fixed (CVE-YYYY-NNNN)[https://www.cve.org/CVERecord?id=CVE-YYYY-NNNN] (Java/Python/Go) ([#X](https://github.com/apache/beam/issues/X)). + ## Known Issues * ([#X](https://github.com/apache/beam/issues/X)). --> -# [2.49.0] - Unreleased + +# [2.52.0] - Unreleased ## Highlights -* New highly anticipated feature X added to Python SDK ([#X](https://github.com/apache/beam/issues/X)). -* New highly anticipated feature Y added to Java SDK ([#Y](https://github.com/apache/beam/issues/Y)). +* Previously deprecated Avro-dependent code (Beam Release 2.46.0) has been finally removed from Java SDK "core" package. +Please, use `beam-sdks-java-extensions-avro` instead. This will allow to easily update Avro version in user code without +potential breaking changes in Beam "core" since the Beam Avro extension already supports the latest Avro versions and +should handle this. ([#25252](https://github.com/apache/beam/issues/25252)). ## I/Os @@ -62,13 +68,15 @@ ## New Features / Improvements -* Allow prebuilding large images when using `--prebuild_sdk_container_engine=cloud_build`, like images depending on `tensorflow` or `torch` ([#27023](https://github.com/apache/beam/pull/27023)). -* Disabled `pip` cache when installing packages on the workers. This reduces the size of prebuilt Python container images ([#27035](https://github.com/apache/beam/pull/27035)). -* Select dedicated avro datum reader and writer (Java) ([#18874](https://github.com/apache/beam/issues/18874)). +* X feature added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)). ## Breaking Changes * X behavior was changed ([#X](https://github.com/apache/beam/issues/X)). +* `org.apache.beam.sdk.io.CountingSource.CounterMark` uses custom `CounterMarkCoder` as a default coder since all Avro-dependent +classes finally moved to `extensions/avro`. In case if it's still required to use `AvroCoder` for `CounterMark`, then, +as a workaround, a copy of "old" `CountingSource` class should be placed into a project code and used directly +([#25252](https://github.com/apache/beam/issues/25252)). ## Deprecations @@ -76,13 +84,136 @@ ## Bugfixes -* Fixed X (Java/Python) ([#X](https://github.com/apache/beam/issues/X)). -* Fixed KinesisIO `NullPointerException` when a progress check is made before the reader is started (IO) ([#23868](https://github.com/apache/beam/issues/23868)) +* Fixed "Desired bundle size 0 bytes must be greater than 0" in Java SDK's BigtableIO.BigtableSource when you have more cores than bytes to read (Java) [#28793](https://github.com/apache/beam/issues/28793). +* `watch_file_pattern` arg of the [RunInference](https://github.com/apache/beam/blob/104c10b3ee536a9a3ea52b4dbf62d86b669da5d9/sdks/python/apache_beam/ml/inference/base.py#L997) arg had no effect prior to 2.52.0. To use the behavior of arg `watch_file_pattern` prior to 2.52.0, follow the documentation at https://beam.apache.org/documentation/ml/side-input-updates/ and use `WatchFilePattern` PTransform as a SideInput. ([#28948](https://github.com/apache/beam/pulls/28948)) + +## Security Fixes +* Fixed (CVE-YYYY-NNNN)[https://www.cve.org/CVERecord?id=CVE-YYYY-NNNN] (Java/Python/Go) ([#X](https://github.com/apache/beam/issues/X)). ## Known Issues * ([#X](https://github.com/apache/beam/issues/X)). +# [2.51.0] - 2023-10-03 + +## New Features / Improvements + +* In Python, [RunInference](https://beam.apache.org/documentation/sdks/python-machine-learning/#why-use-the-runinference-api) now supports loading many models in the same transform using a [KeyedModelHandler](https://beam.apache.org/documentation/sdks/python-machine-learning/#use-a-keyed-modelhandler) ([#27628](https://github.com/apache/beam/issues/27628)). +* In Python, the [VertexAIModelHandlerJSON](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.inference.vertex_ai_inference.html#apache_beam.ml.inference.vertex_ai_inference.VertexAIModelHandlerJSON) now supports passing in inference_args. These will be passed through to the Vertex endpoint as parameters. +* Added support to run `mypy` on user pipelines ([#27906](https://github.com/apache/beam/issues/27906)) + + +## Breaking Changes + +* Removed fastjson library dependency for Beam SQL. Table property is changed to be based on jackson ObjectNode (Java) ([#24154](https://github.com/apache/beam/issues/24154)). +* Removed TensorFlow from Beam Python container images [PR](https://github.com/apache/beam/pull/28424). If you have been negatively affected by this change, please comment on [#20605](https://github.com/apache/beam/issues/20605). +* Removed the parameter `t reflect.Type` from `parquetio.Write`. The element type is derived from the input PCollection (Go) ([#28490](https://github.com/apache/beam/issues/28490)) +* Refactor BeamSqlSeekableTable.setUp adding a parameter joinSubsetType. [#28283](https://github.com/apache/beam/issues/28283) + + +## Bugfixes + +* Fixed exception chaining issue in GCS connector (Python) ([#26769](https://github.com/apache/beam/issues/26769#issuecomment-1700422615)). +* Fixed streaming inserts exception handling, GoogleAPICallErrors are now retried according to retry strategy and routed to failed rows where appropriate rather than causing a pipeline error (Python) ([#21080](https://github.com/apache/beam/issues/21080)). +* Fixed a bug in Python SDK's cross-language Bigtable sink that mishandled records that don't have an explicit timestamp set: [#28632](https://github.com/apache/beam/issues/28632). + + +## Security Fixes +* Python containers updated, fixing [CVE-2021-30474](https://nvd.nist.gov/vuln/detail/CVE-2021-30474), [CVE-2021-30475](https://nvd.nist.gov/vuln/detail/CVE-2021-30475), [CVE-2021-30473](https://nvd.nist.gov/vuln/detail/CVE-2021-30473), [CVE-2020-36133](https://nvd.nist.gov/vuln/detail/CVE-2020-36133), [CVE-2020-36131](https://nvd.nist.gov/vuln/detail/CVE-2020-36131), [CVE-2020-36130](https://nvd.nist.gov/vuln/detail/CVE-2020-36130), and [CVE-2020-36135](https://nvd.nist.gov/vuln/detail/CVE-2020-36135) +* Used go 1.21.1 to build, fixing [CVE-2023-39320](https://security-tracker.debian.org/tracker/CVE-2023-39320) + +## Known Issues + +* Python pipelines using BigQuery Storage Read API must pin `fastavro` + dependency to 1.8.3 or earlier: [#28811](https://github.com/apache/beam/issues/28811) + +# [2.50.0] - 2023-08-30 + +## Highlights + +* Spark 3.2.2 is used as default version for Spark runner ([#23804](https://github.com/apache/beam/issues/23804)). +* The Go SDK has a new default local runner, called Prism ([#24789](https://github.com/apache/beam/issues/24789)). +* All Beam released container images are now [multi-arch images](https://cloud.google.com/kubernetes-engine/docs/how-to/build-multi-arch-for-arm#what_is_a_multi-arch_image) that support both x86 and ARM CPU architectures. + +## I/Os + +* Java KafkaIO now supports picking up topics via topicPattern ([#26948](https://github.com/apache/beam/pull/26948)) +* Support for read from Cosmos DB Core SQL API ([#23604](https://github.com/apache/beam/issues/23604)) +* Upgraded to HBase 2.5.5 for HBaseIO. (Java) ([#27711](https://github.com/apache/beam/issues/19554)) +* Added support for GoogleAdsIO source (Java) ([#27681](https://github.com/apache/beam/pull/27681)). + +## New Features / Improvements + +* The Go SDK now requires Go 1.20 to build. ([#27558](https://github.com/apache/beam/issues/27558)) +* The Go SDK has a new default local runner, Prism. ([#24789](https://github.com/apache/beam/issues/24789)). + * Prism is a portable runner that executes each transform independantly, ensuring coders. + * At this point it supercedes the Go direct runner in functionality. The Go direct runner is now deprecated. + * See https://github.com/apache/beam/blob/master/sdks/go/pkg/beam/runners/prism/README.md for the goals and features of Prism. +* Hugging Face Model Handler for RunInference added to Python SDK. ([#26632](https://github.com/apache/beam/pull/26632)) +* Hugging Face Pipelines support for RunInference added to Python SDK. ([#27399](https://github.com/apache/beam/pull/27399)) +* Vertex AI Model Handler for RunInference now supports private endpoints ([#27696](https://github.com/apache/beam/pull/27696)) +* MLTransform transform added with support for common ML pre/postprocessing operations ([#26795](https://github.com/apache/beam/pull/26795)) +* Upgraded the Kryo extension for the Java SDK to Kryo 5.5.0. This brings in bug fixes, performance improvements, and serialization of Java 14 records. ([#27635](https://github.com/apache/beam/issues/27635)) +* All Beam released container images are now [multi-arch images](https://cloud.google.com/kubernetes-engine/docs/how-to/build-multi-arch-for-arm#what_is_a_multi-arch_image) that support both x86 and ARM CPU architectures. ([#27674](https://github.com/apache/beam/issues/27674)). The multi-arch container images include: + * All versions of Go, Python, Java and Typescript SDK containers. + * All versions of Flink job server containers. + * Java and Python expansion service containers. + * Transform service controller container. + * Spark3 job server container. +* Added support for batched writes to AWS SQS for improved throughput (Java, AWS 2).([#21429](https://github.com/apache/beam/issues/21429)) + +## Breaking Changes + +* Python SDK: Legacy runner support removed from Dataflow, all pipelines must use runner v2. +* Python SDK: Dataflow Runner will no longer stage Beam SDK from PyPI in the `--staging_location` at pipeline submission. Custom container images that are not based on Beam's default image must include Apache Beam installation.([#26996](https://github.com/apache/beam/issues/26996)) + +## Deprecations + +* The Go Direct Runner is now Deprecated. It remains available to reduce migration churn. + * Tests can be set back to the direct runner by overriding TestMain: `func TestMain(m *testing.M) { ptest.MainWithDefault(m, "direct") }` + * It's recommended to fix issues seen in tests using Prism, as they can also happen on any portable runner. + * Use the generic register package for your pipeline DoFns to ensure pipelines function on portable runners, like prism. + * Do not rely on closures or using package globals for DoFn configuration. They don't function on portable runners. + +## Bugfixes + +* Fixed DirectRunner bug in Python SDK where GroupByKey gets empty PCollection and fails when pipeline option `direct_num_workers!=1`.([#27373](https://github.com/apache/beam/pull/27373)) +* Fixed BigQuery I/O bug when estimating size on queries that utilize row-level security ([#27474](https://github.com/apache/beam/pull/27474)) + +## Known Issues + +* Long-running Python pipelines might experience a memory leak: [#28246](https://github.com/apache/beam/issues/28246). +* Python Pipelines using BigQuery IO or `orjson` dependency might experience segmentation faults or get stuck: [#28318](https://github.com/apache/beam/issues/28318). +* Beam Python containers rely on a version of Debian/aom that has several security vulnerabilities: [CVE-2021-30474](https://nvd.nist.gov/vuln/detail/CVE-2021-30474), [CVE-2021-30475](https://nvd.nist.gov/vuln/detail/CVE-2021-30475), [CVE-2021-30473](https://nvd.nist.gov/vuln/detail/CVE-2021-30473), [CVE-2020-36133](https://nvd.nist.gov/vuln/detail/CVE-2020-36133), [CVE-2020-36131](https://nvd.nist.gov/vuln/detail/CVE-2020-36131), [CVE-2020-36130](https://nvd.nist.gov/vuln/detail/CVE-2020-36130), and [CVE-2020-36135](https://nvd.nist.gov/vuln/detail/CVE-2020-36135) +* Python SDK's cross-language Bigtable sink mishandles records that don't have an explicit timestamp set: [#28632](https://github.com/apache/beam/issues/28632). To avoid this issue, set explicit timestamps for all records before writing to Bigtable. + + +# [2.49.0] - 2023-07-17 + + +## I/Os + +* Support for Bigtable Change Streams added in Java `BigtableIO.ReadChangeStream` ([#27183](https://github.com/apache/beam/issues/27183)) + +## New Features / Improvements + +* Allow prebuilding large images when using `--prebuild_sdk_container_engine=cloud_build`, like images depending on `tensorflow` or `torch` ([#27023](https://github.com/apache/beam/pull/27023)). +* Disabled `pip` cache when installing packages on the workers. This reduces the size of prebuilt Python container images ([#27035](https://github.com/apache/beam/pull/27035)). +* Select dedicated avro datum reader and writer (Java) ([#18874](https://github.com/apache/beam/issues/18874)). +* Timer API for the Go SDK (Go) ([#22737](https://github.com/apache/beam/issues/22737)). + +## Deprecations + +* Removed Python 3.7 support. ([#26447](https://github.com/apache/beam/issues/26447)) + +## Bugfixes + +* Fixed KinesisIO `NullPointerException` when a progress check is made before the reader is started (IO) ([#23868](https://github.com/apache/beam/issues/23868)) + +## Known Issues + +* Long-running Python pipelines might experience a memory leak: [#28246](https://github.com/apache/beam/issues/28246). + # [2.48.0] - 2023-05-31 @@ -118,16 +249,17 @@ ## Deprecations -* X behavior is deprecated and will be removed in X versions ([#X](https://github.com/apache/beam/issues/X)). ## Bugfixes -* Fixed X (Java/Python) ([#X](https://github.com/apache/beam/issues/X)). * Fixed Java bootloader failing with Too Long Args due to long classpaths, with a pathing jar. (Java) ([#25582](https://github.com/apache/beam/issues/25582)). ## Known Issues -* ([#X](https://github.com/apache/beam/issues/X)). +* PubsubIO writes will throw *SizeLimitExceededException* for any message above 100 bytes, when used in batch (bounded) mode. (Java) ([#27000](https://github.com/apache/beam/issues/27000)). +* Long-running Python pipelines might experience a memory leak: [#28246](https://github.com/apache/beam/issues/28246). +* Python SDK's cross-language Bigtable sink mishandles records that don't have an explicit timestamp set: [#28632](https://github.com/apache/beam/issues/28632). To avoid this issue, set explicit timestamps for all records before writing to Bigtable. + # [2.47.0] - 2023-05-10 @@ -171,6 +303,7 @@ * The google-cloud-profiler dependency was accidentally removed from Beam's Python Docker Image [#26998](https://github.com/apache/beam/issues/26698). [Dataflow Docker images](https://cloud.google.com/dataflow/docs/concepts/sdk-worker-dependencies) still preinstall this dependency. +* Long-running Python pipelines might experience a memory leak: [#28246](https://github.com/apache/beam/issues/28246). # [2.46.0] - 2023-03-10 @@ -224,7 +357,6 @@ ## Bugfixes -* Fixed X (Java/Python) ([#X](https://github.com/apache/beam/issues/X)). # [2.45.0] - 2023-02-15 diff --git a/CI.md b/CI.md index 0b6075c24d5bb..200e1208e4d79 100644 --- a/CI.md +++ b/CI.md @@ -90,6 +90,9 @@ These variables are: Service Account shall have following permissions ([IAM roles](https://cloud.google.com/iam/docs/understanding-roles)): * Storage Admin (roles/storage.admin) * Dataflow Admin (roles/dataflow.admin) + * Artifact Registry writer (roles/artifactregistry.createOnPush) + * Big Query Data Editor (roles/bigquery.dataEditor) + * Service Account User (roles/iam.serviceAccountUser) ### Workflows diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 686c1982303e6..7eefa8ae8cbf0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,13 +17,314 @@ under the License. --> -## Contributing to Beam +# Contributing to Beam -*Before opening a pull request*, review the -[Beam contribution guide](https://beam.apache.org/contribute/). -It lists steps that are required before creating a PR. In particular, consider +There are many ways to contribute to Beam, just one of which is by contributing code. +For a full list of ways to contribute and get plugged into Beam, see the +[Beam Contribution Guide](https://beam.apache.org/contribute/) + +## Code Contributions + +*Before opening a pull request*, review the Beam contribution guide below. +It lists steps that are required before creating a PR and provides tips for +getting started. In particular, consider the following: - Have you searched for existing, related Issues and pull requests? - Have you shared your intent by creating an issue and commenting that you plan to take it on? - If the change is large, have you discussed it on the dev@ mailing list? - Is the change being proposed clearly explained and motivated? + +These steps and instructions on getting started are outlined below as well. + +### Prerequisites + +- A [GitHub](https://github.com/) account. +- A Linux, macOS, or Microsoft Windows development environment. +- Java JDK 8 installed. +- [Go](https://golang.org) 1.16.0 or later installed. +- [Docker](https://www.docker.com/) installed for some tasks including building worker containers and testing changes to this website locally. +- For SDK Development: + - Python 3.x interpreters. You will need Python interpreters for all Python versions supported by Beam. + Interpreters should be installed and available in shell via `python3.x` commands. For more information, see: + Python installation tips in [Developer Wiki](https://cwiki.apache.org/confluence/display/BEAM/Python+Tips#PythonTips-InstallingPythoninterpreters). +- For large contributions, a signed [Individual Contributor License. + Agreement](https://www.apache.org/licenses/icla.pdf) (ICLA) to the Apache + Software Foundation (ASF). + +### Share Your Intent +1. Find or create an issue in the [Beam repo](https://github.com/apache/beam/issues/new/choose). + Tracking your work in an issue will avoid duplicated or conflicting work, and provide + a place for notes. Later, your pull request will be linked to the issue as well. +2. Comment ".take-issue" on the issue. This will cause the issue to be assigned to you. + When you've completed the issue, you can close it by commenting ".close-issue". + If you are a committer and would like to assign an issue to a non-committer, they must comment + on the issue first; please tag the user asking them to do so or to comment "\`.take-issue\`". + The command will be ignored if it is surrounded by `\`` markdown characters. +3. If your change is large or it is your first change, it is a good idea to + [discuss it on the dev@beam.apache.org mailing list](https://beam.apache.org/community/contact-us/). +4. For large changes create a design doc + ([template](https://s.apache.org/beam-design-doc-template), + [examples](https://s.apache.org/beam-design-docs)) and email it to the [dev@beam.apache.org mailing list](https://beam.apache.org/community/contact-us/). + +### Setup Your Environment and Learn About Language Specific Setup + +Before you begin, check out the Wiki pages. There are many useful tips about [Git](https://cwiki.apache.org/confluence/display/BEAM/Git+Tips), [Go](https://cwiki.apache.org/confluence/display/BEAM/Go+Tips), [Gradle](https://cwiki.apache.org/confluence/display/BEAM/Gradle+Tips), [Java](https://cwiki.apache.org/confluence/display/BEAM/Java+Tips), [Python](https://cwiki.apache.org/confluence/display/BEAM/Python+Tips), etc. + +#### Configuration Options +You have two options for configuring your development environment: +- Local: + - Manually installing the [prerequisites](https://beam.apache.org/contribute/#prerequisites). + - Using the automated script for Linux and macOS. +- Container-based: using a [Docker](https://www.docker.com/) image. + +##### Local: Debian-based Distribution + +###### Manual steps + +To install these in a Debian-based distribution: +1. Execute: + ``` + sudo apt-get install \ + openjdk-8-jdk \ + python-setuptools \ + python-pip \ + virtualenv \ + tox \ + docker-ce + ``` +2. On some systems, like Ubuntu 20.04, install these: + ``` + pip3 install grpcio-tools mypy-protobuf + ``` +3. If you develop in GO: + 1. Install [Go](https://golang.org/doc/install). + 2. Check BEAM repo is in: `$GOPATH/src/github.com/apache/` + 3. At the end, it should look like this: `$GOPATH/src/github.com/apache/beam` +4. Once Go is installed, install goavro: + ``` + $ export GOPATH=`pwd`/sdks/go/examples/.gogradle/project_gopath + $ go get github.com/linkedin/goavro/v2 + ``` +**Important**: gLinux users should configure their machines for sudoless Docker. + +###### Automated script for Linux and macOS + +You can install these in a Debian-based distribution for Linux or macOs using the [local-env-setup.sh](https://github.com/apache/beam/blob/master/local-env-setup.sh) script, which is part of the Beam repo. It contains: + +* pip3 packages +* go packages +* goavro +* JDK 8 +* Python +* Docker + +To install execute: +``` +./local-env-setup.sh +``` + +##### Container: Docker-based + +Alternatively, you can use the Docker based local development environment to wrap your clone of the Beam repo +into a container meeting the requirements above. + +You can start this container using the [start-build-env.sh](https://github.com/apache/beam/blob/master/start-build-env.sh) script which is part of the Beam repo. + +Execute: +``` +./start-build-env.sh +``` + +#### Development Setup {#development-setup} + +1. Check [Git workflow tips](https://cwiki.apache.org/confluence/display/BEAM/Git+Tips) if you need help with git forking, cloning, branching, committing, pull requests, and squashing commits. + +2. Make a fork of https://github.com/apache/beam repo. + +3. Clone the forked repository. You can download it anywhere you like. + ``` + $ mkdir -p ~/path/to/your/folder + $ cd ~/path/to/your/folder + $ git clone https://github.com/forked/apache/beam + $ cd beam + ``` + For **Go development**: + + We recommend putting it in your `$GOPATH` (`$HOME/go` by default on Unix systems). + + Clone the repo, and update your branch as normal: + ``` + $ git clone https://github.com/apache/beam.git + $ cd beam + $ git remote add git@github.com:/beam.git + $ git fetch --all + ``` + + Get or Update all the Go SDK dependencies: + ``` + $ go get -u ./... + ``` + +4. Check the environment was set up correctly. + + **Option 1**: validate the Go, Java, and Python environments: + + **Important**: Make sure you have activated Python development. + ``` + ./gradlew :checkSetup + ``` + **Option 2**: Run independent checks: + - For **Go development**: + ``` + export GOLANG_PROTOBUF_REGISTRATION_CONFLICT=ignore./gradlew :sdks:go:examples:wordCount + ``` + - For **Python development**: + ``` + ./gradlew :sdks:python:wordCount + ``` + - For **Java development**: + ``` + ./gradlew :examples:java:wordCount + ``` + +5. Familiarize yourself with gradle and the project structure. + + At the root of the git repository, run: + ``` + $ ./gradlew projects + ``` + Examine the available tasks in a project. For the default set of tasks, use: + ``` + $ ./gradlew tasks + ``` + For a given module, use: + ``` + $ ./gradlew -p sdks/java/io/cassandra tasks + ``` + For an exhaustive list of tasks, use: + ``` + $ ./gradlew tasks --all + ``` + +6. Make sure you can build and run tests. + + Since Beam is a large project, usually, you will want to limit testing to the particular module you are working on. Gradle will build just the necessary things to run those tests. For example: + ``` + $ ./gradlew -p sdks/go check + $ ./gradlew -p sdks/java/io/cassandra check + $ ./gradlew -p runners/flink check + ``` + +7. Now you may want to set up your preferred IDE and other aspects of your development + environment. See the Developers' wiki for tips, guides, and FAQs on: + - [IntelliJ](https://cwiki.apache.org/confluence/display/BEAM/Using+IntelliJ+IDE) + - [Java](https://cwiki.apache.org/confluence/display/BEAM/Java+Tips) + - [Python](https://cwiki.apache.org/confluence/display/BEAM/Python+Tips) + - [Go](https://cwiki.apache.org/confluence/display/BEAM/Go+Tips) + - [Website](https://cwiki.apache.org/confluence/display/BEAM/Website+Tips) + - [Gradle](https://cwiki.apache.org/confluence/display/BEAM/Gradle+Tips) + - [Jenkins](https://cwiki.apache.org/confluence/display/BEAM/Jenkins+Tips) + - [FAQ](https://cwiki.apache.org/confluence/display/BEAM/Contributor+FAQ) + +### Create a Pull Request + +1. Make your code change. Every source file needs to include the Apache license header. Every new dependency needs to + have an open source license [compatible](https://www.apache.org/legal/resolved.html#criteria) with Apache. + +2. Add unit tests for your change. + +3. Use descriptive commit messages that make it easy to identify changes and provide a clear history. + +4. When your change is ready to be reviewed and merged, create a pull request. + +5. Link to the issue you are addressing in your pull request. + +6. The pull request and any changes pushed to it will trigger [pre-commit + jobs](https://cwiki.apache.org/confluence/display/BEAM/Contribution+Testing+Guide#ContributionTestingGuide-Pre-commit). If a test fails and appears unrelated to your + change, you can cause tests to be re-run by adding a single line comment on your + PR: + ``` + retest this please + ``` +Pull request template has a link to a [catalog of trigger phrases](https://github.com/apache/beam/blob/master/.test-infra/jenkins/README.md) +that start various post-commit tests suites. Use these sparingly because post-commit tests consume shared development resources. + +### Review Process and Releases + +#### Get Reviewed + +Your pull requests should automatically have reviewers assigned within a few hours of opening it. +If that doesn't happen for some reason, you can also request a review yourself. + +1. Pull requests can only be merged by a + [Beam committer](https://home.apache.org/phonebook.html?pmc=beam). + To find a committer for your area, either: + - look for similar code merges, or + - ask on [dev@beam.apache.org](https://beam.apache.org/community/contact-us/) + + Use `R: @username` in the pull request to notify a reviewer. + +2. If you don't get any response in 3 business days, email the [dev@beam.apache.org mailing list](https://beam.apache.org/community/contact-us/) to ask for someone to look at your pull request. + +#### Make the Reviewer’s Job Easier + +1. Provide context for your changes in the associated issue and/or PR description. + +2. Avoid huge mega-changes. + +3. Review feedback typically leads to follow-up changes. It is easier to review follow-up changes when they are added as additional "fixup" commits to the + existing PR/branch. This allows reviewer(s) to track the incremental progress and focus on new changes, + and keeps comment threads attached to the code. + Please refrain from squashing new commits into reviewed commits before review is completed. + Because squashing reviewed and unreviewed commits often makes it harder to + see the difference between the review iterations, reviewers may ask you to unsquash new changes. + +4. After review is complete and the PR is accepted, fixup commits should be squashed (see [Git workflow tips](https://cwiki.apache.org/confluence/display/BEAM/Git+Tips)). + Beam committers [can squash](https://beam.apache.org/contribute/committer-guide/#merging-it) + all commits in the PR during merge, however if a PR has a mixture of independent changes that should not be squashed, and fixup commits, + then the PR author should help squashing fixup commits to maintain a clean commit history. + +#### Apache Beam Releases + +Apache Beam makes minor releases every 6 weeks. Apache Beam has a +[calendar](https://calendar.google.com/calendar/embed?src=0p73sl034k80oob7seouanigd0%40group.calendar.google.com) for +cutting the next release branch. Your change needs to be checked into master before the release branch is cut +to make the next release. + +#### Stale Pull Requests + +The community will close stale pull requests in order to keep the project +healthy. A pull request becomes stale after its author fails to respond to +actionable comments for 60 days. Author of a closed pull request is welcome to +reopen the same pull request again in the future. + +### Troubleshooting + +If you run into any issues, check out the [contribution FAQ](https://cwiki.apache.org/confluence/display/BEAM/Contributor+FAQ) or ask on the [dev@ mailing list](https://beam.apache.org/community/contact-us/) or [#beam channel of the ASF Slack](https://beam.apache.org/community/contact-us/). + +If you didn't find the information you were looking for in this guide, please +[reach out to the Beam community](https://beam.apache.org/community/contact-us/). + + + +## Find Efforts to Contribute to +A great way to contribute is to join an existing effort. If you want to get involved but don’t have a project in mind, check our [list of open starter tasks](https://s.apache.org/beam-starter-tasks). +For the most intensive efforts, check out the [roadmap](https://beam.apache.org/roadmap/). + +## Contributing to the Developer Documentation + +New contributors are often best equipped to find gaps in the developer documentation. +If you'd like to contribute to our documentation, either open a PR in the Beam repo with +the proposed changes or make edits to the [Beam wiki](https://cwiki.apache.org/confluence/display/BEAM/Apache+Beam). + +By default, everyone has access to the wiki. If you wish to contribute changes, +please create an account and request edit access on the dev@beam.apache.org mailing list (include your Wiki account user ID). + +## Additional Resources +Please see Beam developers’ [Wiki Contributor FAQ](https://cwiki.apache.org/confluence/display/BEAM/Contributor+FAQ) for more information. + +If you are contributing a ```PTransform``` to Beam, we have an extensive [PTransform Style Guide](https://beam.apache.org/contribute/ptransform-style-guide). + +If you are contributing a Runner to Beam, refer to the [Runner authoring guide](https://beam.apache.org/contribute/runner-guide/). + +Review [design documents](https://s.apache.org/beam-design-docs). diff --git a/OWNERS b/OWNERS deleted file mode 100644 index 6f6d2a478b288..0000000000000 --- a/OWNERS +++ /dev/null @@ -1,22 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -filters: - # Per-language reviewers. - "\\.go$": - reviewers: - - herohde - "\\.java$": - reviewers: - - lukecwik - - kennknowles - - aaltay - "\\.py$": - reviewers: - - pabloem - - robertwb - - aaltay - - charlesccychen - - "build\\.gradle$": - reviewers: - - lukecwik diff --git a/README.md b/README.md index a039e5dec4509..0bd0e8b83cc44 100644 --- a/README.md +++ b/README.md @@ -70,32 +70,63 @@ Beam supports executing programs on multiple distributed processing backends thr - The `DirectRunner` runs the pipeline on your local machine. - The `DataflowRunner` submits the pipeline to the [Google Cloud Dataflow](http://cloud.google.com/dataflow/). - The `FlinkRunner` runs the pipeline on an Apache Flink cluster. The code has been donated from [dataArtisans/flink-dataflow](https://github.com/dataArtisans/flink-dataflow) and is now part of Beam. -- The `SparkRunner` runs the pipeline on an Apache Spark cluster. The code has been donated from [cloudera/spark-dataflow](https://github.com/cloudera/spark-dataflow) and is now part of Beam. +- The `SparkRunner` runs the pipeline on an Apache Spark cluster. - The `JetRunner` runs the pipeline on a Hazelcast Jet cluster. The code has been donated from [hazelcast/hazelcast-jet](https://github.com/hazelcast/hazelcast-jet) and is now part of Beam. - The `Twister2Runner` runs the pipeline on a Twister2 cluster. The code has been donated from [DSC-SPIDAL/twister2](https://github.com/DSC-SPIDAL/twister2) and is now part of Beam. Have ideas for new Runners? See the [runner-ideas label](https://github.com/apache/beam/issues?q=is%3Aopen+is%3Aissue+label%3Arunner-ideas). -## Getting Started -To learn how to write Beam pipelines, read the Quickstart for [[Java](https://beam.apache.org/get-started/quickstart-java), [Python](https://beam.apache.org/get-started/quickstart-py), or -[Go](https://beam.apache.org/get-started/quickstart-go)] available on our website. +Instructions for building and testing Beam itself +are in the [contribution guide](./CONTRIBUTING.md). + +## 📚 Learn More + +Here are some resources actively maintained by the Beam community to help you get started: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ResourceDetails
Apache Beam WebsiteOur website discussing the project, and it's specifics.
Java QuickstartA guide to getting started with the Java SDK.
Python QuickstartA guide to getting started with the Python SDK.
Go Quickstart A guide to getting started with the Go SDK.
Tour of Beam A comprehensive, interactive learning experience covering Beam concepts in depth.
Beam Quest A certification granted by Google Cloud, certifying proficiency in Beam.
Community Metrics Beam's Git Community Metrics.
## Contact Us -To get involved in Apache Beam: +To get involved with Apache Beam: -* [Subscribe](mailto:user-subscribe@beam.apache.org) or [mail](mailto:user@beam.apache.org) the [user@beam.apache.org](http://mail-archives.apache.org/mod_mbox/beam-user/) list. -* [Subscribe](mailto:dev-subscribe@beam.apache.org) or [mail](mailto:dev@beam.apache.org) the [dev@beam.apache.org](http://mail-archives.apache.org/mod_mbox/beam-dev/) list. +* [Subscribe to](https://beam.apache.org/community/contact-us/#:~:text=Subscribe%20and%20Unsubscribe) or e-mail the [user@beam.apache.org](http://mail-archives.apache.org/mod_mbox/beam-user/) list. +* [Subscribe to](https://beam.apache.org/community/contact-us/#:~:text=Subscribe%20and%20Unsubscribe) or e-mail the [dev@beam.apache.org](http://mail-archives.apache.org/mod_mbox/beam-dev/) list. * [Join ASF Slack](https://s.apache.org/slack-invite) on [#beam channel](https://s.apache.org/beam-slack-channel) * [Report an issue](https://github.com/apache/beam/issues/new/choose). - -Instructions for building and testing Beam itself -are in the [contribution guide](https://beam.apache.org/contribute/). - -## More Information - -* [Apache Beam](https://beam.apache.org) -* [Overview](https://beam.apache.org/use/beam-overview/) -* Quickstart: [Java](https://beam.apache.org/get-started/quickstart-java), [Python](https://beam.apache.org/get-started/quickstart-py), [Go](https://beam.apache.org/get-started/quickstart-go) -* [Community metrics](https://s.apache.org/beam-community-metrics) diff --git a/build.gradle.kts b/build.gradle.kts index 77520c3ce7233..ea1b4e6784e36 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -19,7 +19,7 @@ plugins { base // Apply one top level rat plugin to perform any required license enforcement analysis - id("org.nosphere.apache.rat") version "0.8.0" + id("org.nosphere.apache.rat") version "0.8.1" // Enable gradle-based release management id("net.researchgate.release") version "2.8.1" id("org.apache.beam.module") @@ -241,16 +241,10 @@ tasks.register("javaPreCommit") { dependsOn(":runners:core-java:build") dependsOn(":runners:direct-java:build") dependsOn(":runners:extensions-java:metrics:build") - dependsOn(":runners:flink:1.12:build") - dependsOn(":runners:flink:1.12:job-server:build") - dependsOn(":runners:flink:1.13:build") - dependsOn(":runners:flink:1.13:job-server:build") - dependsOn(":runners:flink:1.14:build") - dependsOn(":runners:flink:1.14:job-server:build") - dependsOn(":runners:flink:1.15:build") - dependsOn(":runners:flink:1.15:job-server:build") - dependsOn(":runners:flink:1.16:build") - dependsOn(":runners:flink:1.16:job-server:build") + // lowest supported flink version + var flinkVersions = project.ext.get("allFlinkVersions") as Array<*> + dependsOn(":runners:flink:${flinkVersions[0]}:build") + dependsOn(":runners:flink:${flinkVersions[0]}:job-server:build") dependsOn(":runners:google-cloud-dataflow-java:build") dependsOn(":runners:google-cloud-dataflow-java:examples-streaming:build") dependsOn(":runners:google-cloud-dataflow-java:examples:build") @@ -316,6 +310,8 @@ tasks.register("javaPreCommit") { dependsOn(":sdks:java:testing:test-utils:build") dependsOn(":sdks:java:testing:tpcds:build") dependsOn(":sdks:java:testing:watermarks:build") + dependsOn(":sdks:java:transform-service:build") + dependsOn(":sdks:java:transform-service:launcher:build") dependsOn(":examples:java:preCommit") dependsOn(":examples:java:twitter:preCommit") @@ -341,6 +337,7 @@ tasks.register("javaioPreCommit") { dependsOn(":sdks:java:io:elasticsearch-tests:elasticsearch-tests-common:build") dependsOn(":sdks:java:io:elasticsearch:build") dependsOn(":sdks:java:io:file-schema-transform:build") + dependsOn(":sdks:java:io:google-ads:build") dependsOn(":sdks:java:io:hbase:build") dependsOn(":sdks:java:io:hcatalog:build") dependsOn(":sdks:java:io:influxdb:build") @@ -361,6 +358,16 @@ tasks.register("javaioPreCommit") { dependsOn(":sdks:java:io:tika:build") } +// a precommit task testing additional supported flink versions not covered by +// the main Java PreCommit (lowest supported version) +tasks.register("flinkPreCommit") { + var flinkVersions = project.ext.get("allFlinkVersions") as Array<*> + for (version in flinkVersions.slice(1..flinkVersions.size - 1)) { + dependsOn(":runners:flink:${version}:build") + dependsOn(":runners:flink:${version}:job-server:build") + } +} + tasks.register("sqlPreCommit") { dependsOn(":sdks:java:extensions:sql:runBasicExample") dependsOn(":sdks:java:extensions:sql:runPojoExample") @@ -380,11 +387,9 @@ tasks.register("javaPostCommit") { tasks.register("javaPostCommitSickbay") { dependsOn(":runners:samza:validatesRunnerSickbay") - dependsOn(":runners:flink:1.12:validatesRunnerSickbay") - dependsOn(":runners:flink:1.13:validatesRunnerSickbay") - dependsOn(":runners:flink:1.14:validatesRunnerSickbay") - dependsOn(":runners:flink:1.15:validatesRunnerSickbay") - dependsOn(":runners:flink:1.16:validatesRunnerSickbay") + for (version in project.ext.get("allFlinkVersions") as Array<*>) { + dependsOn(":runners:flink:${version}:validatesRunnerSickbay") + } dependsOn(":runners:spark:3:job-server:validatesRunnerSickbay") dependsOn(":runners:direct-java:validatesRunnerSickbay") dependsOn(":runners:portability:java:validatesRunnerSickbay") @@ -440,19 +445,16 @@ tasks.register("goPortablePreCommit") { dependsOn(":sdks:go:test:ulrValidatesRunner") } -tasks.register("goPostCommit") { - dependsOn(":goIntegrationTests") +tasks.register("goPrismPreCommit") { + dependsOn(":sdks:go:test:prismValidatesRunner") } -tasks.register("goIntegrationTests") { - doLast { - exec { - executable("sh") - args("-c", "./sdks/go/test/run_validatesrunner_tests.sh --runner dataflow") - } - } - dependsOn(":sdks:go:test:build") - dependsOn(":runners:google-cloud-dataflow-java:worker:shadowJar") +tasks.register("goPostCommitDataflowARM") { + dependsOn(":sdks:go:test:dataflowValidatesRunnerARM64") +} + +tasks.register("goPostCommit") { + dependsOn(":sdks:go:test:dataflowValidatesRunner") } tasks.register("playgroundPreCommit") { @@ -463,7 +465,6 @@ tasks.register("playgroundPreCommit") { tasks.register("pythonPreCommit") { dependsOn(":sdks:python:test-suites:tox:pycommon:preCommitPyCommon") - dependsOn(":sdks:python:test-suites:tox:py37:preCommitPy37") dependsOn(":sdks:python:test-suites:tox:py38:preCommitPy38") dependsOn(":sdks:python:test-suites:tox:py39:preCommitPy39") dependsOn(":sdks:python:test-suites:tox:py310:preCommitPy310") @@ -473,7 +474,6 @@ tasks.register("pythonPreCommit") { tasks.register("pythonPreCommitIT") { dependsOn(":sdks:python:test-suites:tox:pycommon:preCommitPyCommon") dependsOn(":sdks:python:test-suites:dataflow:preCommitIT") - dependsOn(":sdks:python:test-suites:dataflow:preCommitIT_V2") } tasks.register("pythonDocsPreCommit") { @@ -481,7 +481,6 @@ tasks.register("pythonDocsPreCommit") { } tasks.register("pythonDockerBuildPreCommit") { - dependsOn(":sdks:python:container:py37:docker") dependsOn(":sdks:python:container:py38:docker") dependsOn(":sdks:python:container:py39:docker") dependsOn(":sdks:python:container:py310:docker") @@ -490,25 +489,13 @@ tasks.register("pythonDockerBuildPreCommit") { tasks.register("pythonLintPreCommit") { // TODO(https://github.com/apache/beam/issues/20209): Find a better way to specify lint and formatter tasks without hardcoding py version. - dependsOn(":sdks:python:test-suites:tox:py37:lint") + dependsOn(":sdks:python:test-suites:tox:py38:lint") } tasks.register("pythonFormatterPreCommit") { dependsOn("sdks:python:test-suites:tox:py38:formatter") } -tasks.register("python37PostCommit") { - dependsOn(":sdks:python:test-suites:dataflow:py37:postCommitIT") - dependsOn(":sdks:python:test-suites:direct:py37:postCommitIT") - dependsOn(":sdks:python:test-suites:direct:py37:directRunnerIT") - dependsOn(":sdks:python:test-suites:direct:py37:hdfsIntegrationTest") - dependsOn(":sdks:python:test-suites:direct:py37:azureIntegrationTest") - dependsOn(":sdks:python:test-suites:portable:py37:postCommitPy37") - dependsOn(":sdks:python:test-suites:dataflow:py37:spannerioIT") - dependsOn(":sdks:python:test-suites:direct:py37:spannerioIT") - dependsOn(":sdks:python:test-suites:portable:py37:xlangSpannerIOIT") -} - tasks.register("python38PostCommit") { dependsOn(":sdks:python:test-suites:dataflow:py38:postCommitIT") dependsOn(":sdks:python:test-suites:direct:py38:postCommitIT") @@ -547,12 +534,11 @@ tasks.register("python311PostCommit") { } tasks.register("portablePythonPreCommit") { - dependsOn(":sdks:python:test-suites:portable:py37:preCommitPy37") + dependsOn(":sdks:python:test-suites:portable:py38:preCommitPy38") dependsOn(":sdks:python:test-suites:portable:py311:preCommitPy311") } tasks.register("pythonSparkPostCommit") { - dependsOn(":sdks:python:test-suites:portable:py37:sparkValidatesRunner") dependsOn(":sdks:python:test-suites:portable:py38:sparkValidatesRunner") dependsOn(":sdks:python:test-suites:portable:py39:sparkValidatesRunner") dependsOn(":sdks:python:test-suites:portable:py311:sparkValidatesRunner") @@ -573,6 +559,7 @@ tasks.register("communityMetricsProber") { tasks.register("javaExamplesDataflowPrecommit") { dependsOn(":runners:google-cloud-dataflow-java:examples:preCommit") dependsOn(":runners:google-cloud-dataflow-java:examples-streaming:preCommit") + dependsOn(":runners:google-cloud-dataflow-java:examplesJavaRunnerV2PreCommit") } tasks.register("whitespacePreCommit") { @@ -588,18 +575,65 @@ tasks.register("typescriptPreCommit") { dependsOn(":sdks:python:test-suites:tox:py38:jest") } -tasks.register("pushAllDockerImages") { - dependsOn(":runners:spark:3:job-server:container:dockerPush") +tasks.register("pushAllRunnersDockerImages") { + dependsOn(":runners:spark:3:job-server:container:docker") + for (version in project.ext.get("allFlinkVersions") as Array<*>) { + dependsOn(":runners:flink:${version}:job-server-container:docker") + } + + doLast { + if (project.hasProperty("prune-images")) { + exec { + executable("docker") + args("system", "prune", "-a", "--force") + } + } + } +} + +tasks.register("pushAllSdkDockerImages") { + // Enforce ordering to allow the prune step to happen between runs. + // This will ensure we don't use up too much space (especially in CI environments) + mustRunAfter(":pushAllRunnersDockerImages") + dependsOn(":sdks:java:container:pushAll") dependsOn(":sdks:python:container:pushAll") dependsOn(":sdks:go:container:pushAll") dependsOn(":sdks:typescript:container:pushAll") - for (version in project.ext.get("allFlinkVersions") as Array<*>) { - dependsOn(":runners:flink:${version}:job-server-container:dockerPush") + + doLast { + if (project.hasProperty("prune-images")) { + exec { + executable("docker") + args("system", "prune", "-a", "--force") + } + } + } +} + +tasks.register("pushAllXlangDockerImages") { + // Enforce ordering to allow the prune step to happen between runs. + // This will ensure we don't use up too much space (especially in CI environments) + mustRunAfter(":pushAllSdkDockerImages") + + dependsOn(":sdks:java:expansion-service:container:docker") + dependsOn(":sdks:java:transform-service:controller-container:docker") + dependsOn(":sdks:python:expansion-service-container:docker") + + doLast { + if (project.hasProperty("prune-images")) { + exec { + executable("docker") + args("system", "prune", "-a", "--force") + } + } } - dependsOn(":sdks:java:expansion-service:container:dockerPush") - dependsOn(":sdks:java:transform-service:controller-container:dockerPush") - dependsOn(":sdks:python:expansion-service-container:dockerPush") +} + +tasks.register("pushAllDockerImages") { + dependsOn(":pushAllRunnersDockerImages") + dependsOn(":pushAllSdkDockerImages") + dependsOn(":pushAllXlangDockerImages") } // Use this task to validate the environment set up for Go, Python and Java diff --git a/buildSrc/OWNERS b/buildSrc/OWNERS deleted file mode 100644 index af32c311923a2..0000000000000 --- a/buildSrc/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lukecwik diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 6df462a335de3..d99a1003c3964 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -30,6 +30,11 @@ repositories { url = uri("https://repo.spring.io/plugins-release/") content { includeGroup("io.spring.gradle") } } + // For obsolete Avro plugin + maven { + url = uri("https://jitpack.io") + content { includeGroup("com.github.davidmc24.gradle-avro-plugin") } + } } // Dependencies on other plugins used when this plugin is invoked @@ -39,21 +44,19 @@ dependencies { implementation("gradle.plugin.com.github.johnrengelman:shadow:7.1.1") implementation("com.github.spotbugs.snom:spotbugs-gradle-plugin:5.0.14") - runtimeOnly("com.google.protobuf:protobuf-gradle-plugin:0.8.13") // Enable proto code generation - runtimeOnly("com.commercehub.gradle.plugin:gradle-avro-plugin:0.11.0") // Enable Avro code generation - runtimeOnly("com.diffplug.spotless:spotless-plugin-gradle:5.6.1") // Enable a code formatting plugin - runtimeOnly("com.palantir.gradle.docker:gradle-docker:0.34.0") // Enable building Docker containers - runtimeOnly("gradle.plugin.com.dorongold.plugins:task-tree:1.5") // Adds a 'taskTree' task to print task dependency tree - runtimeOnly("gradle.plugin.com.github.johnrengelman:shadow:7.1.1") // Enable shading Java dependencies - runtimeOnly("ca.coglinc:javacc-gradle-plugin:2.4.0") // Enable the JavaCC parser generator + runtimeOnly("com.google.protobuf:protobuf-gradle-plugin:0.8.13") // Enable proto code generation + runtimeOnly("com.github.davidmc24.gradle-avro-plugin:gradle-avro-plugin:0.16.0") // Enable Avro code generation + runtimeOnly("com.diffplug.spotless:spotless-plugin-gradle:5.6.1") // Enable a code formatting plugin + runtimeOnly("gradle.plugin.com.dorongold.plugins:task-tree:1.5") // Adds a 'taskTree' task to print task dependency tree + runtimeOnly("gradle.plugin.com.github.johnrengelman:shadow:7.1.1") // Enable shading Java dependencies runtimeOnly("net.linguica.gradle:maven-settings-plugin:0.5") runtimeOnly("gradle.plugin.io.pry.gradle.offline_dependencies:gradle-offline-dependencies-plugin:0.5.0") // Enable creating an offline repository - runtimeOnly("net.ltgt.gradle:gradle-errorprone-plugin:1.2.1") // Enable errorprone Java static analysis + runtimeOnly("net.ltgt.gradle:gradle-errorprone-plugin:3.1.0") // Enable errorprone Java static analysis runtimeOnly("org.ajoberstar.grgit:grgit-gradle:4.1.1") // Enable website git publish to asf-site branch - runtimeOnly("com.avast.gradle:gradle-docker-compose-plugin:0.16.12") // Enable docker compose tasks + runtimeOnly("com.avast.gradle:gradle-docker-compose-plugin:0.16.12") // Enable docker compose tasks runtimeOnly("ca.cutterslade.gradle:gradle-dependency-analyze:1.8.3") // Enable dep analysis runtimeOnly("gradle.plugin.net.ossindex:ossindex-gradle-plugin:0.4.11") // Enable dep vulnerability analysis - runtimeOnly("org.checkerframework:checkerframework-gradle-plugin:0.6.19") // Enable enhanced static checking plugin + runtimeOnly("org.checkerframework:checkerframework-gradle-plugin:0.6.34") // Enable enhanced static checking plugin } // Because buildSrc is built and tested automatically _before_ gradle diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamDockerPlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamDockerPlugin.groovy new file mode 100644 index 0000000000000..442b35439cae5 --- /dev/null +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamDockerPlugin.groovy @@ -0,0 +1,325 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.gradle + +import java.util.regex.Pattern +import org.gradle.api.GradleException +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.Task +import org.gradle.api.file.CopySpec +import org.gradle.api.logging.LogLevel +import org.gradle.api.logging.Logger +import org.gradle.api.logging.Logging +import org.gradle.api.tasks.Copy +import org.gradle.api.tasks.Delete +import org.gradle.api.tasks.Exec + +/** + * A gradle plug-in interacting with docker. Originally replicated from + * com.palantir.docker plugin. + */ +class BeamDockerPlugin implements Plugin { + private static final Logger logger = Logging.getLogger(BeamDockerPlugin.class) + private static final Pattern LABEL_KEY_PATTERN = Pattern.compile('^[a-z0-9.-]*$') + + static class DockerExtension { + Project project + + private static final String DEFAULT_DOCKERFILE_PATH = 'Dockerfile' + String name = null + File dockerfile = null + String dockerComposeTemplate = 'docker-compose.yml.template' + String dockerComposeFile = 'docker-compose.yml' + Set dependencies = [] as Set + Set tags = [] as Set + Map namedTags = [:] + Map labels = [:] + Map buildArgs = [:] + boolean pull = false + boolean noCache = false + String network = null + boolean buildx = false + Set platform = [] as Set + boolean load = false + boolean push = false + String builder = null + + File resolvedDockerfile = null + File resolvedDockerComposeTemplate = null + File resolvedDockerComposeFile = null + + // The CopySpec defining the Docker Build Context files + final CopySpec copySpec + + DockerExtension(Project project) { + this.project = project + this.copySpec = project.copySpec() + } + + void resolvePathsAndValidate() { + if (dockerfile != null) { + resolvedDockerfile = dockerfile + } else { + resolvedDockerfile = project.file(DEFAULT_DOCKERFILE_PATH) + } + resolvedDockerComposeFile = project.file(dockerComposeFile) + resolvedDockerComposeTemplate = project.file(dockerComposeTemplate) + } + + void dependsOn(Task... args) { + this.dependencies = args as Set + } + + Set getDependencies() { + return dependencies + } + + void files(Object... files) { + copySpec.from(files) + } + + void tags(String... args) { + this.tags = args as Set + } + + Set getTags() { + return this.tags + project.getVersion().toString() + } + + Set getPlatform() { + return platform + } + + void platform(String... args) { + this.platform = args as Set + } + } + + @Override + void apply(Project project) { + DockerExtension ext = project.extensions.create('docker', DockerExtension, project) + + Delete clean = project.tasks.create('dockerClean', Delete, { + group = 'Docker' + description = 'Cleans Docker build directory.' + }) + + Copy prepare = project.tasks.create('dockerPrepare', Copy, { + group = 'Docker' + description = 'Prepares Docker build directory.' + dependsOn clean + }) + + Exec exec = project.tasks.create('docker', Exec, { + group = 'Docker' + description = 'Builds Docker image.' + dependsOn prepare + }) + + Task tag = project.tasks.create('dockerTag', { + group = 'Docker' + description = 'Applies all tags to the Docker image.' + dependsOn exec + }) + + Task pushAllTags = project.tasks.create('dockerTagsPush', { + group = 'Docker' + description = 'Pushes all tagged Docker images to configured Docker Hub.' + }) + + project.tasks.create('dockerPush', { + group = 'Docker' + description = 'Pushes named Docker image to configured Docker Hub.' + dependsOn pushAllTags + }) + + project.afterEvaluate { + ext.resolvePathsAndValidate() + String dockerDir = "${project.buildDir}/docker" + clean.delete dockerDir + + prepare.with { + with ext.copySpec + from(ext.resolvedDockerfile) { + rename { fileName -> + fileName.replace(ext.resolvedDockerfile.getName(), 'Dockerfile') + } + } + into dockerDir + } + + exec.with { + workingDir dockerDir + commandLine buildCommandLine(ext) + dependsOn ext.getDependencies() + logging.captureStandardOutput LogLevel.INFO + logging.captureStandardError LogLevel.ERROR + } + + Map tags = ext.namedTags.collectEntries { taskName, tagName -> + [ + generateTagTaskName(taskName), + [ + tagName: tagName, + tagTask: { + -> tagName } + ] + ] + } + + if (!ext.tags.isEmpty()) { + ext.tags.each { unresolvedTagName -> + String taskName = generateTagTaskName(unresolvedTagName) + + if (tags.containsKey(taskName)) { + throw new IllegalArgumentException("Task name '${taskName}' is existed.") + } + + tags[taskName] = [ + tagName: unresolvedTagName, + tagTask: { + -> computeName(ext.name, unresolvedTagName) } + ] + } + } + + tags.each { taskName, tagConfig -> + Exec tagSubTask = project.tasks.create('dockerTag' + taskName, Exec, { + group = 'Docker' + description = "Tags Docker image with tag '${tagConfig.tagName}'" + workingDir dockerDir + commandLine 'docker', 'tag', "${-> ext.name}", "${-> tagConfig.tagTask()}" + dependsOn exec + }) + tag.dependsOn tagSubTask + + Exec pushSubTask = project.tasks.create('dockerPush' + taskName, Exec, { + group = 'Docker' + description = "Pushes the Docker image with tag '${tagConfig.tagName}' to configured Docker Hub" + workingDir dockerDir + commandLine 'docker', 'push', "${-> tagConfig.tagTask()}" + dependsOn tagSubTask + }) + pushAllTags.dependsOn pushSubTask + } + } + } + + private List buildCommandLine(DockerExtension ext) { + List buildCommandLine = ['docker'] + if (ext.buildx) { + buildCommandLine.addAll(['buildx', 'build']) + if (!ext.platform.isEmpty()) { + buildCommandLine.addAll('--platform', String.join(',', ext.platform)) + } + if (ext.load) { + buildCommandLine.add '--load' + } + if (ext.push) { + buildCommandLine.add '--push' + if (ext.load) { + throw new Exception("cannot combine 'push' and 'load' options") + } + } + if (ext.builder != null) { + buildCommandLine.addAll('--builder', ext.builder) + } + } else { + buildCommandLine.add 'build' + } + if (ext.noCache) { + buildCommandLine.add '--no-cache' + } + if (ext.getNetwork() != null) { + buildCommandLine.addAll('--network', ext.network) + } + if (!ext.buildArgs.isEmpty()) { + for (Map.Entry buildArg : ext.buildArgs.entrySet()) { + buildCommandLine.addAll('--build-arg', "${buildArg.getKey()}=${buildArg.getValue()}" as String) + } + } + if (!ext.labels.isEmpty()) { + for (Map.Entry label : ext.labels.entrySet()) { + if (!label.getKey().matches(LABEL_KEY_PATTERN)) { + throw new GradleException(String.format("Docker label '%s' contains illegal characters. " + + "Label keys must only contain lowercase alphanumberic, `.`, or `-` characters (must match %s).", + label.getKey(), LABEL_KEY_PATTERN.pattern())) + } + buildCommandLine.addAll('--label', "${label.getKey()}=${label.getValue()}" as String) + } + } + if (ext.pull) { + buildCommandLine.add '--pull' + } + buildCommandLine.addAll(['-t', "${-> ext.name}", '.']) + logger.debug("${buildCommandLine}" as String) + return buildCommandLine + } + + private static String computeName(String name, String tag) { + int firstAt = tag.indexOf("@") + + String tagValue + if (firstAt > 0) { + tagValue = tag.substring(firstAt + 1, tag.length()) + } else { + tagValue = tag + } + + if (tagValue.contains(':') || tagValue.contains('/')) { + // tag with ':' or '/' -> force use the tag value + return tagValue + } else { + // tag without ':' and '/' -> replace the tag part of original name + int lastColon = name.lastIndexOf(':') + int lastSlash = name.lastIndexOf('/') + + int endIndex; + + // image_name -> this should remain + // host:port/image_name -> this should remain. + // host:port/image_name:v1 -> v1 should be replaced + if (lastColon > lastSlash) endIndex = lastColon + else endIndex = name.length() + + return name.substring(0, endIndex) + ":" + tagValue + } + } + + private static String generateTagTaskName(String name) { + String tagTaskName = name + int firstAt = name.indexOf("@") + + if (firstAt > 0) { + // Get substring of task name + tagTaskName = name.substring(0, firstAt) + } else if (firstAt == 0) { + // Task name must not be empty + throw new GradleException("Task name of docker tag '${name}' must not be empty.") + } else if (name.contains(':') || name.contains('/')) { + // Tags which with repo or name must have a task name + throw new GradleException("Docker tag '${name}' must have a task name.") + } + + StringBuffer sb = new StringBuffer(tagTaskName) + // Uppercase the first letter of task name + sb.replace(0, 1, tagTaskName.substring(0, 1).toUpperCase()); + return sb.toString() + } +} diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamDockerRunPlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamDockerRunPlugin.groovy new file mode 100644 index 0000000000000..5297c70181396 --- /dev/null +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamDockerRunPlugin.groovy @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.gradle + +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.tasks.Exec + +/** + * A gradle plug-in handling 'docker run' command. Originally replicated from + * com.palantir.docker-run plugin. + */ +class BeamDockerRunPlugin implements Plugin { + + /** A class defining the configurations of dockerRun task. */ + static class DockerRunExtension { + String name + String image + Set ports = [] as Set + Map env = [:] + List arguments = [] + Map volumes = [:] + boolean daemonize = true + boolean clean = false + + public String getName() { + return name + } + + public void setName(String name) { + this.name = name + } + } + + @Override + void apply(Project project) { + DockerRunExtension ext = project.extensions.create('dockerRun', DockerRunExtension) + + Exec dockerRunStatus = project.tasks.create('dockerRunStatus', Exec, { + group = 'Docker Run' + description = 'Checks the run status of the container' + }) + + Exec dockerRun = project.tasks.create('dockerRun', Exec, { + group = 'Docker Run' + description = 'Runs the specified container with port mappings' + }) + + Exec dockerStop = project.tasks.create('dockerStop', Exec, { + group = 'Docker Run' + description = 'Stops the named container if it is running' + ignoreExitValue = true + }) + + Exec dockerRemoveContainer = project.tasks.create('dockerRemoveContainer', Exec, { + group = 'Docker Run' + description = 'Removes the persistent container associated with the Docker Run tasks' + ignoreExitValue = true + }) + + project.afterEvaluate { + /** Inspect status of docker. */ + dockerRunStatus.with { + standardOutput = new ByteArrayOutputStream() + commandLine 'docker', 'inspect', '--format={{.State.Running}}', ext.name + doLast { + if (standardOutput.toString().trim() != 'true') { + println "Docker container '${ext.name}' is STOPPED." + return 1 + } else { + println "Docker container '${ext.name}' is RUNNING." + } + } + } + + /** + * Run a docker container. See {@link DockerRunExtension} for supported + * arguments. + * + * Replication of dockerRun task of com.palantir.docker-run plugin. + */ + dockerRun.with { + List args = new ArrayList() + args.addAll(['docker', 'run']) + + if (ext.daemonize) { + args.add('-d') + } + if (ext.clean) { + args.add('--rm') + } else { + finalizedBy dockerRunStatus + } + for (String port : ext.ports) { + args.add('-p') + args.add(port) + } + for (Map.Entry volume : ext.volumes.entrySet()) { + File localFile = project.file(volume.key) + + if (!localFile.exists()) { + logger.error("ERROR: Local folder ${localFile} doesn't exist. Mounted volume will not be visible to container") + throw new IllegalStateException("Local folder ${localFile} doesn't exist.") + } + args.add('-v') + args.add("${localFile.absolutePath}:${volume.value}") + } + args.addAll(ext.env.collect{ k, v -> ['-e', "${k}=${v}"]}.flatten()) + args.add('--name') + args.add(ext.name) + if (!ext.arguments.isEmpty()) { + args.addAll(ext.arguments) + } + args.add(ext.image) + + commandLine args + } + + dockerStop.with { + commandLine 'docker', 'stop', ext.name + } + + dockerRemoveContainer.with { + commandLine 'docker', 'rm', ext.name + } + } + } +} diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 546c0d651288a..46968e0e5ad6a 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -18,23 +18,23 @@ package org.apache.beam.gradle +import static java.util.UUID.randomUUID + import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar import groovy.json.JsonOutput import groovy.json.JsonSlurper +import java.util.logging.Logger import org.gradle.api.attributes.Category import org.gradle.api.GradleException import org.gradle.api.JavaVersion import org.gradle.api.Plugin import org.gradle.api.Project -import org.gradle.api.Task import org.gradle.api.artifacts.Configuration import org.gradle.api.artifacts.ProjectDependency import org.gradle.api.file.FileCollection import org.gradle.api.file.FileTree -import org.gradle.api.plugins.quality.Checkstyle import org.gradle.api.publish.maven.MavenPublication import org.gradle.api.tasks.Copy -import org.gradle.api.tasks.Delete import org.gradle.api.tasks.Exec import org.gradle.api.tasks.JavaExec import org.gradle.api.tasks.TaskProvider @@ -43,11 +43,10 @@ import org.gradle.api.tasks.compile.CompileOptions import org.gradle.api.tasks.compile.JavaCompile import org.gradle.api.tasks.javadoc.Javadoc import org.gradle.api.tasks.testing.Test -import org.gradle.api.tasks.PathSensitive import org.gradle.api.tasks.PathSensitivity import org.gradle.testing.jacoco.tasks.JacocoReport -import java.net.ServerSocket + /** * This plugin adds methods to configure a module with Beam's defaults, called "natures". * @@ -65,6 +64,8 @@ import java.net.ServerSocket */ class BeamModulePlugin implements Plugin { + static final Logger logger = Logger.getLogger(BeamModulePlugin.class.getName()) + /** Licence header enforced by spotless */ static final String javaLicenseHeader = """/* * Licensed to the Apache Software Foundation (ASF) under one @@ -331,6 +332,8 @@ class BeamModulePlugin implements Plugin { TaskProvider startJobServer // Job server cleanup task. TaskProvider cleanupJobServer + // any additional environment variables specific to the suite of tests + Map additionalEnvs } // A class defining the configuration for CrossLanguageUsingJavaExpansion. @@ -356,6 +359,8 @@ class BeamModulePlugin implements Plugin { String expansionProjectPath // Collect Python pipeline tests with this marker String collectMarker + // any additional environment variables to be exported + Map additionalEnvs } // A class defining the configuration for CrossLanguageValidatesRunner. @@ -398,10 +403,33 @@ class BeamModulePlugin implements Plugin { FileCollection classpath } + // A class defining the configuration for createTransformServiceTask. + static class TransformServiceConfiguration { + // Task name TransformService case. + String name = 'transformService' + + List pythonPipelineOptions = [] + + List javaPipelineOptions = [] + + // Additional pytest options + List pytestOptions = [] + // Job server startup task. + TaskProvider startJobServer + // Job server cleanup task. + TaskProvider cleanupJobServer + // Number of parallel test runs. + Integer numParallelTests = 1 + // Whether the pipeline needs --sdk_location option + boolean needsSdkLocation = false + + // Collect Python pipeline tests with this marker + String collectMarker + } + def isRelease(Project project) { return parseBooleanProperty(project, 'isRelease'); } - /** * Parses -Pprop as true for use as a flag, and otherwise uses Groovy's toBoolean */ @@ -421,6 +449,38 @@ class BeamModulePlugin implements Plugin { return 'beam' + p.path.replace(':', '-') } + /* + * Set compile args for compiling and running in different java version by modifying the compiler args in place. + * + * Replace `-source X` and `-target X` or `--release X` options if already existed in compilerArgs. + */ + static def setCompileAndRuntimeJavaVersion(List compilerArgs, String ver) { + boolean foundS = false, foundT = false + int foundR = -1 + logger.fine("set java ver ${ver} to compiler args") + for (int i = 0; i < compilerArgs.size()-1; ++i) { + if (compilerArgs.get(i) == '-source') { + foundS = true + compilerArgs.set(i+1, ver) + } else if (compilerArgs.get(i) == '-target') { + foundT = true + compilerArgs.set(i+1, ver) + } else if (compilerArgs.get(i) == '--release') { + foundR = i + } + } + if (foundR != -1) { + compilerArgs.removeAt(foundR + 1) + compilerArgs.removeAt(foundR) + } + if (!foundS) { + compilerArgs.addAll('-source', ver) + } + if (!foundT) { + compilerArgs.addAll('-target', ver) + } + } + void apply(Project project) { /** ***********************************************************************************************/ @@ -428,13 +488,6 @@ class BeamModulePlugin implements Plugin { project.ext.mavenGroupId = 'org.apache.beam' - // Automatically use the official release version if we are performing a release - // otherwise append '-SNAPSHOT' - project.version = '2.49.0' - if (!isRelease(project)) { - project.version += '-SNAPSHOT' - } - // Default to dash-separated directories for artifact base name, // which will also be the default artifactId for maven publications project.apply plugin: 'base' @@ -490,9 +543,13 @@ class BeamModulePlugin implements Plugin { project.ext.containerArchitectures = { if (isRelease(project)) { // Ensure we always publish the expected containers. - return ["amd64"]; + return ["amd64", "arm64"]; } else if (project.rootProject.findProperty("container-architecture-list") != null) { - return project.rootProject.findProperty("container-architecture-list").split(',') + def containerArchitectures = project.rootProject.findProperty("container-architecture-list").split(',') + if (containerArchitectures.size() > 1 && !project.rootProject.hasProperty("push-containers")) { + throw new GradleException("A multi-arch image can't be saved in the local image store, please append the -Ppush-containers flag and specify a repository to push in the -Pdocker-repository-root flag."); + } + return containerArchitectures } else { return [project.nativeArchitecture()] } @@ -522,20 +579,21 @@ class BeamModulePlugin implements Plugin { def cassandra_driver_version = "3.10.2" def cdap_version = "6.5.1" def checkerframework_version = "3.27.0" - def classgraph_version = "4.8.104" + def classgraph_version = "4.8.162" def dbcp2_version = "2.9.0" def errorprone_version = "2.10.0" // Try to keep gax_version consistent with gax-grpc version in google_cloud_platform_libraries_bom - def gax_version = "2.28.1" + def gax_version = "2.33.0" + def google_ads_version = "26.0.0" def google_clients_version = "2.0.0" - def google_cloud_bigdataoss_version = "2.2.6" + def google_cloud_bigdataoss_version = "2.2.16" // Try to keep google_cloud_spanner_version consistent with google_cloud_spanner_bom in google_cloud_platform_libraries_bom - def google_cloud_spanner_version = "6.42.3" - def google_code_gson_version = "2.9.1" + def google_cloud_spanner_version = "6.47.0" + def google_code_gson_version = "2.10.1" def google_oauth_clients_version = "1.34.1" // Try to keep grpc_version consistent with gRPC version in google_cloud_platform_libraries_bom - def grpc_version = "1.55.1" - def guava_version = "31.1-jre" + def grpc_version = "1.56.1" + def guava_version = "32.1.2-jre" def hadoop_version = "2.10.2" def hamcrest_version = "2.1" def influxdb_version = "2.19" @@ -548,18 +606,19 @@ class BeamModulePlugin implements Plugin { def kafka_version = "2.4.1" def log4j2_version = "2.20.0" def nemo_version = "0.1" - def netty_version = "4.1.77.Final" + // Try to keep netty_version consistent with the netty version in grpc_bom (includes grpc_netty) in google_cloud_platform_libraries_bom + def netty_version = "4.1.87.Final" def postgres_version = "42.2.16" def powermock_version = "2.0.9" // Try to keep protobuf_version consistent with the protobuf version in google_cloud_platform_libraries_bom - def protobuf_version = "3.21.12" + def protobuf_version = "3.23.2" def qpid_jms_client_version = "0.61.0" def quickcheck_version = "1.0" def sbe_tool_version = "1.25.1" def singlestore_jdbc_version = "1.1.4" def slf4j_version = "1.7.30" def spark2_version = "2.4.8" - def spark3_version = "3.1.2" + def spark3_version = "3.2.2" def spotbugs_version = "4.0.6" def testcontainers_version = "1.17.3" def arrow_version = "5.0.0" @@ -612,6 +671,7 @@ class BeamModulePlugin implements Plugin { aws_java_sdk2_regions : "software.amazon.awssdk:regions:$aws_java_sdk2_version", aws_java_sdk2_utils : "software.amazon.awssdk:utils:$aws_java_sdk2_version", aws_java_sdk2_profiles : "software.amazon.awssdk:profiles:$aws_java_sdk2_version", + azure_sdk_bom : "com.azure:azure-sdk-bom:1.2.14", bigdataoss_gcsio : "com.google.cloud.bigdataoss:gcsio:$google_cloud_bigdataoss_version", bigdataoss_util : "com.google.cloud.bigdataoss:util:$google_cloud_bigdataoss_version", byte_buddy : "net.bytebuddy:byte-buddy:1.12.14", @@ -633,46 +693,53 @@ class BeamModulePlugin implements Plugin { commons_collections : "commons-collections:commons-collections:3.2.2", commons_compress : "org.apache.commons:commons-compress:1.21", commons_csv : "org.apache.commons:commons-csv:1.8", - commons_io : "commons-io:commons-io:2.7", + commons_io : "commons-io:commons-io:2.13.0", commons_lang3 : "org.apache.commons:commons-lang3:3.9", commons_logging : "commons-logging:commons-logging:1.2", commons_math3 : "org.apache.commons:commons-math3:3.6.1", dbcp2 : "org.apache.commons:commons-dbcp2:$dbcp2_version", error_prone_annotations : "com.google.errorprone:error_prone_annotations:$errorprone_version", - flogger_system_backend : "com.google.flogger:flogger-system-backend:0.7.3", + failsafe : "dev.failsafe:failsafe:3.3.0", + flogger_system_backend : "com.google.flogger:flogger-system-backend:0.7.4", gax : "com.google.api:gax", // google_cloud_platform_libraries_bom sets version gax_grpc : "com.google.api:gax-grpc", // google_cloud_platform_libraries_bom sets version gax_grpc_test : "com.google.api:gax-grpc:$gax_version:testlib", // google_cloud_platform_libraries_bom sets version gax_httpjson : "com.google.api:gax-httpjson", // google_cloud_platform_libraries_bom sets version + google_ads : "com.google.api-ads:google-ads:$google_ads_version", + google_ads_stubs_v14 : "com.google.api-ads:google-ads-stubs-v14:$google_ads_version", google_api_client : "com.google.api-client:google-api-client:$google_clients_version", // for the libraries using $google_clients_version below. google_api_client_jackson2 : "com.google.api-client:google-api-client-jackson2:$google_clients_version", google_api_client_java6 : "com.google.api-client:google-api-client-java6:$google_clients_version", google_api_common : "com.google.api:api-common", // google_cloud_platform_libraries_bom sets version - google_api_services_bigquery : "com.google.apis:google-api-services-bigquery:v2-rev20220924-$google_clients_version", - google_api_services_cloudresourcemanager : "com.google.apis:google-api-services-cloudresourcemanager:v1-rev20220828-$google_clients_version", + // Keep version consistent with the version in google_cloud_bigquery, managed by google_cloud_platform_libraries_bom + google_api_services_bigquery : "com.google.apis:google-api-services-bigquery:v2-rev20230812-$google_clients_version", + // Keep version consistent with the version in google_cloud_resourcemanager, managed by google_cloud_platform_libraries_bom + google_api_services_cloudresourcemanager : "com.google.apis:google-api-services-cloudresourcemanager:v1-rev20230806-$google_clients_version", google_api_services_dataflow : "com.google.apis:google-api-services-dataflow:v1b3-rev20220920-$google_clients_version", - google_api_services_healthcare : "com.google.apis:google-api-services-healthcare:v1-rev20220818-$google_clients_version", + google_api_services_healthcare : "com.google.apis:google-api-services-healthcare:v1-rev20231003-$google_clients_version", google_api_services_pubsub : "com.google.apis:google-api-services-pubsub:v1-rev20220904-$google_clients_version", - google_api_services_storage : "com.google.apis:google-api-services-storage:v1-rev20230301-$google_clients_version", + // Keep version consistent with the version in google_cloud_nio, managed by google_cloud_platform_libraries_bom + google_api_services_storage : "com.google.apis:google-api-services-storage:v1-rev20230907-$google_clients_version", google_auth_library_credentials : "com.google.auth:google-auth-library-credentials", // google_cloud_platform_libraries_bom sets version google_auth_library_oauth2_http : "com.google.auth:google-auth-library-oauth2-http", // google_cloud_platform_libraries_bom sets version google_cloud_bigquery : "com.google.cloud:google-cloud-bigquery", // google_cloud_platform_libraries_bom sets version google_cloud_bigquery_storage : "com.google.cloud:google-cloud-bigquerystorage", // google_cloud_platform_libraries_bom sets version google_cloud_bigtable : "com.google.cloud:google-cloud-bigtable", // google_cloud_platform_libraries_bom sets version google_cloud_bigtable_client_core_config : "com.google.cloud.bigtable:bigtable-client-core-config:1.28.0", - google_cloud_bigtable_emulator : "com.google.cloud:google-cloud-bigtable-emulator:0.147.3", + google_cloud_bigtable_emulator : "com.google.cloud:google-cloud-bigtable-emulator", // google_cloud_platform_libraries_bom sets version google_cloud_core : "com.google.cloud:google-cloud-core", // google_cloud_platform_libraries_bom sets version google_cloud_core_grpc : "com.google.cloud:google-cloud-core-grpc", // google_cloud_platform_libraries_bom sets version google_cloud_datacatalog_v1beta1 : "com.google.cloud:google-cloud-datacatalog", // google_cloud_platform_libraries_bom sets version google_cloud_dataflow_java_proto_library_all: "com.google.cloud.dataflow:google-cloud-dataflow-java-proto-library-all:0.5.160304", - google_cloud_datastore_v1_proto_client : "com.google.cloud.datastore:datastore-v1-proto-client:2.9.0", + // Keep version consistent with the version in google_cloud_datastore, managed by google_cloud_platform_libraries_bom + google_cloud_datastore_v1_proto_client : "com.google.cloud.datastore:datastore-v1-proto-client:2.17.1", google_cloud_firestore : "com.google.cloud:google-cloud-firestore", // google_cloud_platform_libraries_bom sets version google_cloud_pubsub : "com.google.cloud:google-cloud-pubsub", // google_cloud_platform_libraries_bom sets version google_cloud_pubsublite : "com.google.cloud:google-cloud-pubsublite", // google_cloud_platform_libraries_bom sets version // The release notes shows the versions set by the BOM: - // https://github.com/googleapis/java-cloud-bom/releases/tag/v26.16.0 + // https://github.com/googleapis/java-cloud-bom/releases/tag/v26.21.0 // Update libraries-bom version on sdks/java/container/license_scripts/dep_urls_java.yaml - google_cloud_platform_libraries_bom : "com.google.cloud:libraries-bom:26.16.0", + google_cloud_platform_libraries_bom : "com.google.cloud:libraries-bom:26.23.0", google_cloud_spanner : "com.google.cloud:google-cloud-spanner", // google_cloud_platform_libraries_bom sets version google_cloud_spanner_test : "com.google.cloud:google-cloud-spanner:$google_cloud_spanner_version:tests", google_code_gson : "com.google.code.gson:gson:$google_code_gson_version", @@ -793,7 +860,7 @@ class BeamModulePlugin implements Plugin { slf4j_jul_to_slf4j : "org.slf4j:jul-to-slf4j:$slf4j_version", slf4j_log4j12 : "org.slf4j:slf4j-log4j12:$slf4j_version", slf4j_jcl : "org.slf4j:slf4j-jcl:$slf4j_version", - snappy_java : "org.xerial.snappy:snappy-java:1.1.10.0", + snappy_java : "org.xerial.snappy:snappy-java:1.1.10.4", spark_core : "org.apache.spark:spark-core_2.11:$spark2_version", spark_streaming : "org.apache.spark:spark-streaming_2.11:$spark2_version", spark3_core : "org.apache.spark:spark-core_2.12:$spark3_version", @@ -802,17 +869,26 @@ class BeamModulePlugin implements Plugin { spark3_streaming : "org.apache.spark:spark-streaming_2.12:$spark3_version", stax2_api : "org.codehaus.woodstox:stax2-api:4.2.1", tephra : "org.apache.tephra:tephra-api:0.15.0-incubating", + testcontainers_azure : "org.testcontainers:azure:$testcontainers_version", testcontainers_base : "org.testcontainers:testcontainers:$testcontainers_version", + testcontainers_cassandra : "org.testcontainers:cassandra:$testcontainers_version", testcontainers_clickhouse : "org.testcontainers:clickhouse:$testcontainers_version", testcontainers_elasticsearch : "org.testcontainers:elasticsearch:$testcontainers_version", + testcontainers_gcloud : "org.testcontainers:gcloud:$testcontainers_version", + testcontainers_jdbc : "org.testcontainers:jdbc:$testcontainers_version", testcontainers_kafka : "org.testcontainers:kafka:$testcontainers_version", testcontainers_localstack : "org.testcontainers:localstack:$testcontainers_version", - testcontainers_postgresql : "org.testcontainers:postgresql:$testcontainers_version", + testcontainers_mongodb : "org.testcontainers:mongodb:$testcontainers_version", + testcontainers_mssqlserver : "org.testcontainers:mssqlserver:$testcontainers_version", testcontainers_mysql : "org.testcontainers:mysql:$testcontainers_version", - testcontainers_gcloud : "org.testcontainers:gcloud:$testcontainers_version", + testcontainers_neo4j : "org.testcontainers:neo4j:$testcontainers_version", + testcontainers_oracle : "org.testcontainers:oracle-xe:$testcontainers_version", + testcontainers_postgresql : "org.testcontainers:postgresql:$testcontainers_version", testcontainers_rabbitmq : "org.testcontainers:rabbitmq:$testcontainers_version", + truth : "com.google.truth:truth:1.1.5", + threetenbp : "org.threeten:threetenbp:1.6.8", vendored_grpc_1_54_0 : "org.apache.beam:beam-vendor-grpc-1_54_0:0.1", - vendored_guava_26_0_jre : "org.apache.beam:beam-vendor-guava-26_0-jre:0.1", + vendored_guava_32_1_2_jre : "org.apache.beam:beam-vendor-guava-32_1_2-jre:0.1", vendored_calcite_1_28_0 : "org.apache.beam:beam-vendor-calcite-1_28_0:0.2", woodstox_core_asl : "org.codehaus.woodstox:woodstox-core-asl:4.4.1", zstd_jni : "com.github.luben:zstd-jni:1.5.2-5", @@ -1000,9 +1076,9 @@ class BeamModulePlugin implements Plugin { project.tasks.withType(JavaCompile).configureEach { options.encoding = "UTF-8" - // Use --release 8 when targeting Java 8 and running on JDK > 8 + // Use -source 8 -target 8 when targeting Java 8 and running on JDK > 8 // - // Consider migrating compilation and testing to use JDK 9+ and setting '--release=8' as + // Consider migrating compilation and testing to use JDK 9+ and setting '--release 8' as // the default allowing 'applyJavaNature' to override it for the few modules that need JDK 9+ // artifacts. See https://stackoverflow.com/a/43103038/4368200 for additional details. if (JavaVersion.VERSION_1_8.compareTo(JavaVersion.toVersion(project.javaVersion)) == 0 @@ -1029,20 +1105,6 @@ class BeamModulePlugin implements Plugin { preserveFileTimestamps(false) } - if (project.hasProperty("compileAndRunTestsWithJava11")) { - def java11Home = project.findProperty("java11Home") - project.tasks.compileTestJava { - options.fork = true - options.forkOptions.javaHome = java11Home as File - options.compilerArgs += ['-Xlint:-path'] - options.compilerArgs.addAll(['--release', '11']) - } - project.tasks.withType(Test) { - useJUnit() - executable = "${java11Home}/bin/java" - } - } - // Configure the default test tasks set of tests executed // to match the equivalent set that is executed by the maven-surefire-plugin. // See http://maven.apache.org/components/surefire/maven-surefire-plugin/test-mojo.html @@ -1179,12 +1241,10 @@ class BeamModulePlugin implements Plugin { } } - - if (configuration.shadowClosure) { // Ensure that tests are packaged and part of the artifact set. project.task('packageTests', type: Jar) { - classifier = 'tests-unshaded' + archiveClassifier = 'tests-unshaded' from project.sourceSets.test.output } project.artifacts.archives project.packageTests @@ -1197,7 +1257,7 @@ class BeamModulePlugin implements Plugin { "com.google.auto.service:auto-service-annotations:$autoservice_version", "com.google.auto.value:auto-value-annotations:$autovalue_version", "com.google.code.findbugs:jsr305:$jsr305_version", - "com.google.j2objc:j2objc-annotations:1.3", + "com.google.j2objc:j2objc-annotations:2.8", // These dependencies are needed to avoid error-prone warnings on package-info.java files, // also to include the annotations to suppress warnings. // @@ -1254,13 +1314,13 @@ class BeamModulePlugin implements Plugin { // Configures a checkstyle plugin enforcing a set of rules and also allows for a set of // suppressions. project.apply plugin: 'checkstyle' - project.tasks.withType(Checkstyle) { - configFile = project.project(":").file("sdks/java/build-tools/src/main/resources/beam/checkstyle.xml") - configProperties = ["checkstyle.suppressions.file": project.project(":").file("sdks/java/build-tools/src/main/resources/beam/suppressions.xml")] + project.checkstyle { + configDirectory = project.rootProject.layout.projectDirectory.dir("sdks/java/build-tools/src/main/resources/beam/checkstyle") + configFile = project.rootProject.layout.projectDirectory.file("sdks/java/build-tools/src/main/resources/beam/checkstyle/checkstyle.xml").asFile showViolations = true maxErrors = 0 + toolVersion = "8.23" } - project.checkstyle { toolVersion = "8.23" } // CheckStyle can be removed from the 'check' task by passing -PdisableCheckStyle=true on the Gradle // command-line. This is useful for pre-commit which runs checkStyle separately. def disableCheckStyle = project.hasProperty('disableCheckStyle') && @@ -1432,17 +1492,30 @@ class BeamModulePlugin implements Plugin { options.errorprone.errorproneArgs.add("-Xep:Slf4jLoggerShouldBeNonStatic:OFF") } - if (project.hasProperty("compileAndRunTestsWithJava17")) { + if (project.hasProperty("compileAndRunTestsWithJava11")) { + def java11Home = project.findProperty("java11Home") + project.tasks.compileTestJava { + options.fork = true + options.forkOptions.javaHome = java11Home as File + options.compilerArgs += ['-Xlint:-path'] + setCompileAndRuntimeJavaVersion(options.compilerArgs, '11') + } + project.tasks.withType(Test).configureEach { + useJUnit() + executable = "${java11Home}/bin/java" + } + } else if (project.hasProperty("compileAndRunTestsWithJava17")) { def java17Home = project.findProperty("java17Home") project.tasks.compileTestJava { - options.compilerArgs.addAll(['--release', '17']) + setCompileAndRuntimeJavaVersion(options.compilerArgs, '17') project.ext.setJava17Options(options) } - project.tasks.withType(Test) { + project.tasks.withType(Test).configureEach { useJUnit() executable = "${java17Home}/bin/java" } } + if (configuration.shadowClosure) { // Enables a plugin which can perform shading of classes. See the general comments // above about dependency management for Java projects and how the shadow plugin @@ -1479,13 +1552,13 @@ class BeamModulePlugin implements Plugin { } } - // Always configure the shadowJar classifier and merge service files. + // Always configure the shadowJar archiveClassifier and merge service files. if (configuration.shadowClosure) { // Only set the classifer on the unshaded classes if we are shading. - project.jar { classifier = "unshaded" } + project.jar { archiveClassifier = "unshaded" } project.shadowJar({ - classifier = null + archiveClassifier = null mergeServiceFiles() zip64 true into("META-INF/") { @@ -1494,11 +1567,11 @@ class BeamModulePlugin implements Plugin { } } << configuration.shadowClosure) - // Always configure the shadowTestJar classifier and merge service files. + // Always configure the shadowTestJar archiveClassifier and merge service files. project.task('shadowTestJar', type: ShadowJar, { group = "Shadow" description = "Create a combined JAR of project and test dependencies" - classifier = "tests" + archiveClassifier = "tests" from project.sourceSets.test.output configurations = [ project.configurations.testRuntimeMigration @@ -1558,7 +1631,7 @@ class BeamModulePlugin implements Plugin { project.tasks.register("testJar", Jar) { group = "Jar" description = "Create a JAR of test classes" - classifier = "tests" + archiveClassifier = "tests" from project.sourceSets.test.output zip64 true exclude "META-INF/INDEX.LIST" @@ -1713,18 +1786,18 @@ class BeamModulePlugin implements Plugin { project.task('sourcesJar', type: Jar) { from project.sourceSets.main.allSource - classifier = 'sources' + archiveClassifier = 'sources' } project.artifacts.archives project.sourcesJar project.task('testSourcesJar', type: Jar) { from project.sourceSets.test.allSource - classifier = 'test-sources' + archiveClassifier = 'test-sources' } project.artifacts.archives project.testSourcesJar project.task('javadocJar', type: Jar, dependsOn: project.javadoc) { - classifier = 'javadoc' + archiveClassifier = 'javadoc' from project.javadoc.destinationDir } project.artifacts.archives project.javadocJar @@ -1834,8 +1907,8 @@ class BeamModulePlugin implements Plugin { def dependencyNode = dependenciesNode.appendNode('dependency') def appendClassifier = { dep -> dep.artifacts.each { art -> - if (art.hasProperty('classifier')) { - dependencyNode.appendNode('classifier', art.classifier) + if (art.hasProperty('archiveClassifier')) { + dependencyNode.appendNode('archiveClassifier', art.archiveClassifier) } } } @@ -2081,7 +2154,7 @@ class BeamModulePlugin implements Plugin { def goRootDir = "${project.rootDir}/sdks/go" // This sets the whole project Go version. - project.ext.goVersion = "go1.20.4" + project.ext.goVersion = "go1.21.2" // Minor TODO: Figure out if we can pull out the GOCMD env variable after goPrepare script // completion, and avoid this GOBIN substitution. @@ -2129,7 +2202,7 @@ class BeamModulePlugin implements Plugin { /** ***********************************************************************************************/ project.ext.applyDockerNature = { - project.apply plugin: "com.palantir.docker" + project.apply plugin: BeamDockerPlugin project.docker { noCache true } project.tasks.create(name: "copyLicenses", type: Copy) { from "${project.rootProject.projectDir}/LICENSE" @@ -2141,7 +2214,7 @@ class BeamModulePlugin implements Plugin { } project.ext.applyDockerRunNature = { - project.apply plugin: "com.palantir.docker-run" + project.apply plugin: BeamDockerRunPlugin } /** ***********************************************************************************************/ @@ -2322,7 +2395,20 @@ class BeamModulePlugin implements Plugin { // TODO: Decide whether this should be inlined into the one project that relies on it // or be left here. - project.ext.applyAvroNature = { project.apply plugin: "com.commercehub.gradle.plugin.avro" } + project.ext.applyAvroNature = { + project.apply plugin: "com.commercehub.gradle.plugin.avro" + + // add dependency BeamModulePlugin defined custom tasks + // they are defined only when certain flags are provided (e.g. -Prelease; -Ppublishing, etc) + def sourcesJar = project.tasks.findByName('sourcesJar') + if (sourcesJar != null) { + sourcesJar.dependsOn project.tasks.getByName('generateAvroJava') + } + def testSourcesJar = project.tasks.findByName('testSourcesJar') + if (testSourcesJar != null) { + testSourcesJar.dependsOn project.tasks.getByName('generateTestAvroJava') + } + } project.ext.applyAntlrNature = { project.apply plugin: 'antlr' @@ -2333,6 +2419,17 @@ class BeamModulePlugin implements Plugin { generatedSourceDirs += project.generateTestGrammarSource.outputDirectory } } + + // add dependency BeamModulePlugin defined custom tasks + // they are defined only when certain flags are provided (e.g. -Prelease; -Ppublishing, etc) + def sourcesJar = project.tasks.findByName('sourcesJar') + if (sourcesJar != null) { + sourcesJar.mustRunAfter project.tasks.getByName('generateGrammarSource') + } + def testSourcesJar = project.tasks.findByName('testSourcesJar') + if (testSourcesJar != null) { + testSourcesJar.dependsOn project.tasks.getByName('generateTestGrammarSource') + } } // Creates a task to run the quickstart for a runner. @@ -2502,6 +2599,9 @@ class BeamModulePlugin implements Plugin { project.exec { environment "EXPANSION_JAR", expansionJar environment "EXPANSION_PORT", javaExpansionPort + for (envs in config.additionalEnvs){ + environment envs.getKey(), envs.getValue() + } executable 'sh' args '-c', ". ${project.ext.envdir}/bin/activate && cd $pythonDir && ./scripts/run_integration_test.sh $cmdArgs" } @@ -2711,6 +2811,117 @@ class BeamModulePlugin implements Plugin { /** ***********************************************************************************************/ + // Method to create the createTransformServiceTask. + // The method takes TransformServiceConfiguration as parameter. + project.ext.createTransformServiceTask = { + // This task won't work if the python build file doesn't exist. + if (!project.project(":sdks:python").buildFile.exists()) { + System.err.println 'Python build file not found. Skipping createTransformServiceTask.' + return + } + def config = it ? it as TransformServiceConfiguration : new TransformServiceConfiguration() + + project.evaluationDependsOn(":sdks:python") + project.evaluationDependsOn(":runners:core-construction-java") + project.evaluationDependsOn(":sdks:java:extensions:python") + project.evaluationDependsOn(":sdks:java:transform-service:launcher") + + def usesDataflowRunner = config.pythonPipelineOptions.contains("--runner=TestDataflowRunner") || config.pythonPipelineOptions.contains("--runner=DataflowRunner") + + // Task for launching transform services + def envDir = project.project(":sdks:python").envdir + def pythonDir = project.project(":sdks:python").projectDir + def externalPort = getRandomPort() + def launcherJar = project.project(':sdks:java:transform-service:launcher').shadowJar.archivePath + def groupId = project.name + randomUUID().toString() + def transformServiceOpts = [ + "transform_service_launcher_jar": launcherJar, + "group_id": groupId, + "external_port": externalPort, + "beam_version": project.version + ] + def serviceArgs = project.project(':sdks:python').mapToArgString(transformServiceOpts) + def pythonContainerSuffix = project.project(':sdks:python').pythonVersion.replace('.', '') + def javaContainerSuffix + if (JavaVersion.current() == JavaVersion.VERSION_1_8) { + javaContainerSuffix = 'java8' + } else if (JavaVersion.current() == JavaVersion.VERSION_11) { + javaContainerSuffix = 'java11' + } else if (JavaVersion.current() == JavaVersion.VERSION_17) { + javaContainerSuffix = 'java17' + } else { + String exceptionMessage = "Your Java version is unsupported. You need Java version of 8 or 11 or 17 to get started, but your Java version is: " + JavaVersion.current(); + throw new GradleException(exceptionMessage) + } + + // Transform service delivers transforms that refer to SDK harness containers with following sufixes. + def transformServiceJavaContainerSuffix = 'java11' + def transformServicePythonContainerSuffix = '38' + + def setupTask = project.tasks.register(config.name+"Setup", Exec) { + // Containers for main SDKs when running tests. + dependsOn ':sdks:java:container:'+javaContainerSuffix+':docker' + dependsOn ':sdks:python:container:py'+pythonContainerSuffix+':docker' + // Containers for external SDKs used through the transform service. + dependsOn ':sdks:java:container:'+transformServiceJavaContainerSuffix+':docker' + dependsOn ':sdks:python:container:py'+transformServicePythonContainerSuffix+':docker' + dependsOn ':sdks:java:transform-service:controller-container:docker' + dependsOn ':sdks:python:expansion-service-container:docker' + dependsOn ':sdks:java:expansion-service:container:docker' + dependsOn ":sdks:python:installGcpTest" + dependsOn project.project(':sdks:java:transform-service:launcher').shadowJar.getPath() + + if (usesDataflowRunner) { + dependsOn ":sdks:python:test-suites:dataflow:py${project.ext.pythonVersion.replace('.', '')}:initializeForDataflowJob" + } + + // setup test env + executable 'sh' + args '-c', "$pythonDir/scripts/run_transform_service.sh stop $serviceArgs && $pythonDir/scripts/run_transform_service.sh start $serviceArgs" + } + + if (config.needsSdkLocation) { + setupTask.configure {dependsOn ':sdks:python:sdist'} + } + + def pythonTask = project.tasks.register(config.name+"PythonUsingJava") { + group = "Verification" + description = "Runs Python SDK pipeline tests that use transform service" + dependsOn setupTask + dependsOn config.startJobServer + doLast { + def beamPythonTestPipelineOptions = [ + "pipeline_opts": config.pythonPipelineOptions + (usesDataflowRunner ? [ + "--sdk_location=${project.ext.sdkLocation}"] + : []), + "test_opts": config.pytestOptions, + "suite": config.name, + "collect": config.collectMarker, + ] + def cmdArgs = project.project(':sdks:python').mapToArgString(beamPythonTestPipelineOptions) + + project.exec { + environment "EXPANSION_PORT", externalPort + executable 'sh' + args '-c', ". $envDir/bin/activate && cd $pythonDir && ./scripts/run_integration_test.sh $cmdArgs" + } + } + } + + def cleanupTask = project.tasks.register(config.name+'Cleanup', Exec) { + // teardown test env + executable 'sh' + args '-c', "$pythonDir/scripts/run_transform_service.sh stop $serviceArgs" + } + setupTask.configure {finalizedBy cleanupTask} + config.startJobServer.configure {finalizedBy config.cleanupJobServer} + + cleanupTask.configure{mustRunAfter pythonTask} + config.cleanupJobServer.configure{mustRunAfter pythonTask} + } + + /** ***********************************************************************************************/ + project.ext.applyPythonNature = { // Define common lifecycle tasks and artifact types @@ -2752,9 +2963,12 @@ class BeamModulePlugin implements Plugin { } project.exec { executable 'sh' + // TODO: https://github.com/apache/beam/issues/29022 + // pip 23.3 is failing due to Hash mismatch between expected SHA of the packaged and actual SHA. + // until it is resolved on pip's side, don't use pip's cache. args '-c', ". ${project.ext.envdir}/bin/activate && " + - "pip install --pre --retries 10 --upgrade pip && " + - "pip install --pre --retries 10 --upgrade tox -r ${project.rootDir}/sdks/python/build-requirements.txt" + "pip install --pre --retries 10 --upgrade pip --no-cache-dir && " + + "pip install --pre --retries 10 --upgrade tox --no-cache-dir" } } // Gradle will delete outputs whenever it thinks they are stale. Putting a @@ -2837,30 +3051,40 @@ class BeamModulePlugin implements Plugin { } return argList.join(' ') } - project.ext.toxTask = { name, tox_env, posargs='' -> project.tasks.register(name) { dependsOn setupVirtualenv dependsOn ':sdks:python:sdist' - - doLast { - // Python source directory is also tox execution workspace, We want - // to isolate them per tox suite to avoid conflict when running - // multiple tox suites in parallel. - project.copy { from project.pythonSdkDeps; into copiedSrcRoot } - - def copiedPyRoot = "${copiedSrcRoot}/sdks/python" - def distTarBall = "${pythonRootDir}/build/apache-beam.tar.gz" - project.exec { - executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedPyRoot} && scripts/run_tox.sh $tox_env $distTarBall '$posargs'" + if (project.hasProperty('useWheelDistribution')) { + def pythonVersionNumber = project.ext.pythonVersion.replace('.', '') + dependsOn ":sdks:python:bdistPy${pythonVersionNumber}linux" + doLast { + project.copy { from project.pythonSdkDeps; into copiedSrcRoot } + def copiedPyRoot = "${copiedSrcRoot}/sdks/python" + def collection = project.fileTree(project.project(':sdks:python').buildDir){ + include "**/apache_beam-*cp${pythonVersionNumber}*manylinux*.whl" + } + String packageFilename = collection.singleFile.toString() + project.exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedPyRoot} && scripts/run_tox.sh $tox_env ${packageFilename} '$posargs' " + } + } + } else { + // tox task will run in editable mode, which is configured in the tox.ini file. + doLast { + project.copy { from project.pythonSdkDeps; into copiedSrcRoot } + def copiedPyRoot = "${copiedSrcRoot}/sdks/python" + project.exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedPyRoot} && scripts/run_tox.sh $tox_env '$posargs'" + } } } inputs.files project.pythonSdkDeps outputs.files project.fileTree(dir: "${pythonRootDir}/target/.tox/${tox_env}/log/") } } - // Run single or a set of integration tests with provided test options and pipeline options. project.ext.enablePythonPerformanceTest = { @@ -2909,10 +3133,10 @@ class BeamModulePlugin implements Plugin { mustRunAfter = [ ":runners:flink:${project.ext.latestFlinkVersion}:job-server:shadowJar", ':runners:spark:3:job-server:shadowJar', - ':sdks:python:container:py37:docker', ':sdks:python:container:py38:docker', ':sdks:python:container:py39:docker', ':sdks:python:container:py310:docker', + ':sdks:python:container:py311:docker', ] doLast { // TODO: Figure out GCS credentials and use real GCS input and output. diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/IoPerformanceTestUtilities.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/IoPerformanceTestUtilities.groovy new file mode 100644 index 0000000000000..844afd75f008b --- /dev/null +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/IoPerformanceTestUtilities.groovy @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.gradle + +import org.gradle.api.Project +import org.gradle.api.tasks.testing.Test + +import javax.inject.Inject + +class IoPerformanceTestUtilities { + abstract static class IoPerformanceTest extends Test { + @Inject + IoPerformanceTest(Project runningProject, String module, String testClass, Map systemProperties){ + group = "Verification" + description = "Runs IO Performance Test for $testClass" + outputs.upToDateWhen { false } + testClassesDirs = runningProject.findProject(":it:${module}").sourceSets.test.output.classesDirs + classpath = runningProject.sourceSets.test.runtimeClasspath + runningProject.findProject(":it:${module}").sourceSets.test.runtimeClasspath + + include "**/${testClass}.class" + + systemProperty 'exportDataset', System.getenv('exportDataset') + systemProperty 'exportTable', System.getenv('exportTable') + + for (entry in systemProperties){ + systemProperty entry.key, entry.value + } + } + } +} diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/VendorJavaPlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/VendorJavaPlugin.groovy index 8f2fbdc8f1837..97d96e6cf1ebd 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/VendorJavaPlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/VendorJavaPlugin.groovy @@ -36,11 +36,11 @@ import org.gradle.api.publish.maven.MavenPublication *
  • Increment the vendored artifact version only if we need to release a new version. * * - *

    Example for com.google.guava:guava:26.0-jre: + *

    Example for com.google.guava:guava:32.1.2-jre: *

      *
    • groupId: org.apache.beam - *
    • artifactId: guava-26_0-jre - *
    • namespace: org.apache.beam.vendor.guava.v26_0_jre + *
    • artifactId: guava-32_1_2-jre + *
    • namespace: org.apache.beam.vendor.guava.v32_1_2_jre *
    • version: 0.1 *
    * @@ -126,7 +126,7 @@ artifactId=${project.name} } config.exclusions.each { exclude it } - classifier = null + archiveClassifier = null mergeServiceFiles() zip64 true exclude "META-INF/INDEX.LIST" diff --git a/contributor-docs/committer-guide.md b/contributor-docs/committer-guide.md new file mode 100644 index 0000000000000..9e63679776c73 --- /dev/null +++ b/contributor-docs/committer-guide.md @@ -0,0 +1,156 @@ + + +# Committer Guide + +This guide is for +[committers](https://www.apache.org/foundation/how-it-works.html#committers) +and covers Beam's guidelines for reviewing and merging code. + +## Pull request review objectives + +The review process aims for: + + - Review iterations should be efficient, timely and of quality (avoid tiny or + out-of-context changes or huge mega-changes) + - Support efficiency of authoring (don't want to wait on a review for a tiny + bit because GitHub makes it very hard to stack up reviews in sequence / + don't want to have major changes blocked because of difficulty of review) + - Ease of first-time contribution (encourage to follow [contribution + guildelines](/contribute/#contributing-code) but committer may absorb some + extra effort for new contributors) + - Pull requests and commit messages establish a clear history with purpose and + origin of changes + - Ability to perform a granular rollback, if necessary (also see + [policies](/contribute/postcommits-policies/)) + +Granularity of changes: + + - We prefer small independent, incremental PRs with descriptive, isolated + commits. Each commit is a single clear change + - It is OK to keep separate commits for different logical pieces of the code, + if they make reviewing and revisiting code easier + - Making commits isolated is a good practice, authors should be able to + relatively easily split the PR upon reviewer's request + - Generally, every commit should compile and pass tests. + - Avoid keeping in history formatting messages such as checkstyle or spotless + fixes. Squash such commits with previous one. + +## Always get to LGTM ("Looks good to me!") + +After a pull request goes through rounds of reviews and revisions, it will +become ready for merge. A reviewer signals their approval either by GitHub +"approval" or by a comment such as "Looks good to me!" (LGTM). + + - If the author of the pull request is not a committer, a committer must be + the one to approve the change. + - If the author of the pull request is a committer, approval from their chosen + reviewer is enough. A committer is trusted to choose an appropriate + reviewer, even if the reviewer is not a committer. + +Once a pull request is approved, any committer can merge it. + +Exceptions to this rule are rare and made on a case-by-case basis. A committer +may use their discretion for situations such as when the build is broken on the +main branch, blocking all code contributions. In this case, you should still +seek a review on the pull request! A common acronym you may see is "TBR" -- +"to be reviewed". + +**Always go through a pull request, even if you won’t wait for the code +review.** Committers should never commit anything without going through a pull +request, even when it is an urgent fix or rollback due to build breakage. +Skipping pull request bypasses test coverage and could potentially cause the +build to fail, or fail to fix breakage. In addition, pull requests ensure that +changes are communicated properly and potential flaws or improvements can be +spotted, even after the merge happens. + +## Contributor License Agreement + +If you are merging a larger contribution, please make sure that the contributor +has an ICLA on file with the Apache Secretary. You can view the list of +committers [here](https://home.apache.org/phonebook.html?unix=committers), as +well as [ICLA-signers who aren’t yet +committers](https://home.apache.org/unlistedclas.html). + +For smaller contributions, however, this is not required. In this case, we rely +on [clause five](https://www.apache.org/licenses/LICENSE-2.0#contributions) of +the Apache License, Version 2.0, describing licensing of intentionally +submitted contributions. + +## Tests + +Before merging, please make sure that Jenkins tests pass, as visible in the +GitHub pull request. Do not merge the pull request if there are test failures. + +If the pull request contains changes that call for extra test coverage, you can +ask Jenkins to run an extended test suite. For example, if the pull request +modifies a runner, you can run the full `ValidatesRunner` suite with a comment +such as "Run Spark ValidatesRunner". You can run the examples and some IO +integration tests with "Run Java PostCommit". + +## Finishing touches + +At some point in the review process, the change to the codebase will be +complete. However, the pull request may have a collection of review-related +commits that are not meaningful to preserve in the history. The reviewer should +give the LGTM and then request that the author of the pull request rebase, +squash, split, etc, the commits, so that the history is most useful: + + - Favor commits that do just one thing. The commit is the smallest unit of + easy rollback; it is easy to roll back many commits, or a whole pull + request, but harder to roll back part of a commit. + - Commit messages should be descriptive and should reference the issue number + that they address. It should later not be necessary to find a merge commit or + first PR commit to find out what caused a change. + - Pull request descriptions should contain a link to the issue being addressed by the changes. + - `CHANGES.md` file should be updated with noteworthy changes (e.g. new features, backward + incompatible changes, dependency changes, etc.). + - Squash the "Fixup!", "Address comments" type of commits that resulted from review iterations. + +## Merging it! + +While it is preferred that authors squash commits after review is complete, +there may be situations where it is more practical for the committer to handle +this (such as when the action to be taken is obvious or the author isn't +available). The committer may use the "Squash and merge" option in Github (or +modify the PR commits in other ways). The committer is ultimately responsible +and we "trust the committer's judgment"! + +After all the tests pass, there should be a green merge button at the bottom of +the pull request. There are multiple choices. Unless you want to squash commits +as part of the merge (see above) you should choose "Merge pull request" and +ensure "Create a merge commit" is selected from the drop down. This preserves +the commit history and adds a merge commit, so be sure the commit history has +been curated appropriately. + +Do _not_ use the default GitHub commit message, which looks like this: + + Merge pull request #1234 from some_user/transient_branch_name + + [BEAM-7873] Fix the foo bizzle bazzle + +Instead, pull it all into the subject line: + + Merge pull request #1234: [BEAM-7873] Fix the foo bizzle bazzle + +If you have comments to add, put them in the body of the commit message. + +## Seed jobs + +As a committer, you can now run seed jobs! These are used to update our Jenkins +configuration and can be run to test PRs modifying Groovy files before they are +merged. + +To make sure you have these permissions, put up a PR adding yourself to +https://github.com/apache/beam/blob/master/.test-infra/jenkins/Committers.groovy diff --git a/contributor-docs/committer-onboarding.md b/contributor-docs/committer-onboarding.md new file mode 100644 index 0000000000000..bf4dd6ed1c028 --- /dev/null +++ b/contributor-docs/committer-onboarding.md @@ -0,0 +1,37 @@ + + +# Apache Beam Committer Onboarding Guide + +A brief checklist of things to know and do for new committers. + +## For you, the new committer + + - [ ] Sign an ICLA and send it to secretary@apache.org if you haven't before. + - [ ] Read the ASF new committer guide: + https://www.apache.org/dev/new-committers-guide.html + - [ ] Read the [Beam committer guide](committer-guide.md) + - [ ] Log in to https://whimsy.apache.org; that will confirm a working ASF account + you can edit email routing for the account, and add other emails that + you own you can directly edit mailing list subscriptions (for example, + you might switch them to your ASF account - you can still post from any + of your registered emails)\ + - [ ] Link your GitHub account with your ASF account at + https://gitbox.apache.org/. Once you see the big green "Merge" button on pull + requests, this is working. + +## For the PMC + + - [ ] (PMC Chair) Add the new committer to https://whimsy.apache.org/roster/committee/beam + - [ ] Announce the new committer diff --git a/website/www/site/static/images/cut-release-branch.png b/contributor-docs/images/cut-release-branch.png similarity index 100% rename from website/www/site/static/images/cut-release-branch.png rename to contributor-docs/images/cut-release-branch.png diff --git a/website/www/site/static/images/release-guide-1.png b/contributor-docs/images/release-guide-1.png similarity index 100% rename from website/www/site/static/images/release-guide-1.png rename to contributor-docs/images/release-guide-1.png diff --git a/website/www/site/static/images/tag-rc-commit.png b/contributor-docs/images/tag-rc-commit.png similarity index 100% rename from website/www/site/static/images/tag-rc-commit.png rename to contributor-docs/images/tag-rc-commit.png diff --git a/contributor-docs/java-dependency-upgrades.md b/contributor-docs/java-dependency-upgrades.md new file mode 100644 index 0000000000000..21d9c8f741a2b --- /dev/null +++ b/contributor-docs/java-dependency-upgrades.md @@ -0,0 +1,130 @@ + + +# Upgrading a Java Dependency + +To perform a dependency upgrade we want to ensure that the PR is not +introducing any new [linkage errors](https://jlbp.dev/glossary). We do this by +combining successful Jenkins test runs with analysis performed using a linkage +checker. This allows us to gain confidence that we are minimizing the number of +linkage issues that will arise for users. To perform a dependency upgrade: + + - Find all Gradle subprojects that are impacted by the dependency change. + - For each Gradle subproject impacted by a dependency change: + - Perform the before and after linkage checker analysis. + - Provide the results as part of your PR. + - For each Gradle subproject: + - Find and run relevant Jenkins test suites. + +See the following sections for how step-by-step instructions. + +## How to find all Gradle subprojects that are impacted by the dependency change + +Execute the command below will print out a dependency report in a text file for +each project: + + ./gradlew dependencyReport + +Grep for a specific maven artifact identifier such as guava in all the +dependency reports with: + + grep -l "guava" `find ./ -name dependencies.txt` + +## Linkage checker analysis + +> :warning: This step relies on modifying your local maven repository, +> typically found in ~/.m2/. + +Use the shell script to do this on your behalf (note that it will run the +manual command below on your current workspace and also on HEAD): + + /bin/bash sdks/java/build-tools/beam-linkage-check.sh origin/master "artifactId1,artifactId2,..." + +> :warning: If you omit the artifactIds, it uses beam-sdks-java-core +> beam-sdks-java-io-google-cloud-platform +> beam-runners-google-cloud-dataflow-java beam-sdks-java-io-hadoop-format; +> these artifacts often suffer dependency conflicts. + +Copy and paste the output to the PR. If it is large, you may want to use a GitHub gist. For example PRs (1, 2, 3, 4, and 5). + +Note that you can manually run the linkage checker on your current workspace by invoking: + + ./gradlew -Ppublishing -PjavaLinkageArtifactIds=artifactId1,artifactId2,... :checkJavaLinkage + +Check the example output is: + +``` +Class org.brotli.dec.BrotliInputStream is not found; + referenced by 1 class file + org.apache.beam.repackaged.core.org.apache.commons.compress.compressors.brotli.BrotliCompressorInputStream (beam-sdks-java-core-2.20.0-SNAPSHOT.jar) +Class com.github.luben.zstd.ZstdInputStream is not found; + referenced by 1 class file + org.apache.beam.repackaged.core.org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream (beam-sdks-java-core-2.20.0-SNAPSHOT.jar) +Class com.github.luben.zstd.ZstdOutputStream is not found; + referenced by 1 class file + org.apache.beam.repackaged.core.org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream (beam-sdks-java-core-2.20.0-SNAPSHOT.jar) +Class org.apache.beam.vendor.bytebuddy.v1_9_3.net.bytebuddy.jar.asm.commons.ModuleHashesAttribute is not found; + referenced by 1 class file + org.apache.beam.vendor.bytebuddy.v1_9_3.net.bytebuddy.jar.asm.commons.ClassRemapper (beam-vendor-bytebuddy-1_9_3-0.1.jar) +``` + +Delete any installed Apache Beam SNAPSHOT artifacts: + + rm -rf ~/.m2/repository/org/apache/beam + +## Run relevant Jenkins test suites and GitHub Actions + +You can find all Jenkins job configurations within +https://github.com/apache/beam/tree/master/.test-infra/jenkins and request that +the reviewer run the relevant test suites by providing them with a list of all +the relevant trigger phrases. You can perform this request directly on your PR +or on the dev mailing list, for [example](https://lists.apache.org/thread/jgjdt52jm6rk0ndrjjnvk1nn65dl9358). + +# Google Cloud-related dependency upgrades + +To provide the consistent dependencies to Beam users, follow the following steps when upgrading Google Cloud-related dependencies: + + - [ ] Set the Libraries BOM version. Find the latest release in + https://github.com/googleapis/java-cloud-bom/releases and set libraries-bom + value in BeamModulePlugin.groovy + - [ ] Find core Google Java library versions. + - Such as gRPC, Protobuf, Guava, Google Auth Library in the release note + of the Libraries BOM and set them in BeamModulePlugin.groovy + - [ ] Find appropriate Netty version by checking io.grpc:grpc-netty's + dependency declaration. For example, you can tell gRPC version 1.49.0 + was built with Netty "4.1.77.Final" by reading + https://search.maven.org/artifact/io.grpc/grpc-netty/1.49.0/jar: + ``` + netty-codec-http2 + 4.1.77.Final + ``` + - [ ] Update netty_version in BeamModulePlugin.groovy + - [ ] Find netty-tcnative version via netty-parent artifact. For example, you + can tell Netty 4.1.77.Final was built with netty-tcnative "2.0.52.Final". + https://search.maven.org/artifact/io.netty/netty-parent/4.1.77.Final/jar: + ``` + 2.0.52.Final + ``` + - [ ] Update netty_tcnative_boringssl_static version in BeamModulePlugin.groovy + + +The following script may be useful to identify matching/consistent dependency overrides. + + export BOM_VERSION=26.22.0 ; \ + cd /tmp; \ + wget https://repo1.maven.org/maven2/com/google/cloud/libraries-bom/$BOM_VERSION/libraries-bom-$BOM_VERSION.pom -O base.pom && \ + mvn help:effective-pom -f base.pom -Doutput=effective.pom && cat effective.pom | \ + grep -v 'dependencyManagement' > cleanup.pom && \ + mvn dependency:tree -f cleanup.pom + diff --git a/contributor-docs/release-guide.md b/contributor-docs/release-guide.md new file mode 100644 index 0000000000000..189bc61e15e6a --- /dev/null +++ b/contributor-docs/release-guide.md @@ -0,0 +1,1405 @@ + + +# Apache Beam Release Guide + +## Introduction + +The Beam community treats releases with great importance. They are a public +face of the project and most users interact with the project only through the +releases. Releases are signed off by the entire Beam community in a public +vote. + +Each release is executed by a *Release Manager*, who is selected among the Beam +committers. This document describes the process that the Release Manager +follows to perform a release. + +Please remember that publishing software has legal consequences. This guide +complements the foundation-wide guides: + + - [Product Release Policy](https://www.apache.org/dev/release.html) + - [Release Distribution + Policy](https://www.apache.org/dev/release-distribution). + +### What is in a Beam release + +A Beam release consists of the following: + + - ASF source zips archived on + [dist.apache.org](https://dist.apache.org/release/beam) (later archived to + [archive.apache.org](https://archive.apache.org/dist/beam) + - Java jars and poms published to [Maven + Central](https://mvnrepository.com/artifact/org.apache.beam) + - Python wheels published to [pypi](https://pypi.org/project/apache-beam/) + - Go artifacts published to + [pkg.go.dev](https://pkg.go.dev/github.com/apache/beam) + - Docker images published to + [dockerhub](https://hub.docker.com/search?q=apache%2Fbeam&type=image) + - A tag on GitHub indicating the commit from which the release was built + +In addition, each release is accompanied by: + + - A blog post announcing the release and describing the changes + - An update to the webpage to indicate the latest version + +### Phases of the release process + +The release process consists of several phases: + +1. Prepare for release +2. Stabilize the release branch / burn down release-blocking issues +3. Build a release candidate +4. Validate and approve the release candidate +5. Finalize the release +6. Promote the release +7. Post-release tasks + +------------ + +## Prepare for release (~1 week before release cut) + +The following steps take place before the release branch is cut. + +### Decide to release + +Deciding to release and selecting a Release Manager is the first step of the +release process. This is a consensus-based decision of the entire community. +Anybody can propose a release on the `dev@` list. There is no formal process, +no vote requirements, and no timing requirements. A committer must be +identified to be the Release Manager. In practice, most often a committer both +proposes to release and volunteers themselves as Release Manager. + +------- + +### Create a new milestone in GitHub for the next release + +When contributors resolve an issue in GitHub, they are tagging it with a +release that will contain their changes. With the release currently underway, +new issues should be resolved against a subsequent future release. Therefore, +you should create a release item for this subsequent release, as follows: + +In GitHub, navigate to [`Issues > Milestones > New +Milestone`](https://github.com/apache/beam/milestones) and add a new +release for the next minor version after the version you are preparing +to release. + +---- + +### Prepare accounts, keys, etc + +Before your first release, you need to make sure you have all the necessary +accounts, keys, and access for publishing the release. The release process also +requires a variety of API tokens, which you can generate now or later when they +are needed. + +These are the credentials you will need: + + - Apache ID and Password + - GitHub ID, Password, and Personal Access Token + - PyPi account with beam maintainer access and API Token + - GPG pass phrase & 16-digit key ID + - Access to Beam's Apache Nexus repository + - Account to access to apache-beam-testing Google Cloud Platform project. The + account must have permissions to start Cloud Build triggers. Required for + Playground environment update. (E-mail at dev@ mailing list to request + access) + +#### Apache ID and Password + +This is your Apache committer user name and password. You selected these when +you became an Apache Beam Committer. + +#### Github ID, Password, and Personal Access Token + + - [ ] If you are using [GitHub two-factor + authentication](https://help.github.com/articles/securing-your-account-with-two-factor-authentication-2fa/) + and haven't configure HTTPS access, please follow [the + guide](https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/) + to configure command line access. + - [ ] Generate a Personal Access Token with `repo` and `workflow` permissions. + They can be generated from this page: https://github.com/settings/tokens. + See + https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens + for details. + +#### PyPI account and API token + + - [ ] [Create an account with PyPI](https://pypi.python.org/account/register/) + if you don't have one already. + - [ ] Become a maintainer (or an owner) of the + [apache-beam](https://pypi.python.org/pypi/apache-beam) package. + - [ ] Generate a [PyPI APIToken](https://pypi.org/help/#apitoken) for use + during the release. + +#### GPG Key + +You need to have a GPG key to sign the release artifacts. Please be aware of +the ASF-wide [release signing +guidelines](https://www.apache.org/dev/release-signing.html). If you don’t +have a GPG key associated with your Apache account, you must now create one +according to the guidelines. + +Run the following helper script, or you can open it and run the commands +individually (helpful if it doesn't work as intended or if you already are +partially set up) + + ./release/src/main/scripts/preparation_before_release.sh + +> **__NOTE__**: +> When generating the key, please make sure you choose the key type as +> __RSA and RSA (default)__ and key size as __4096 bit__. + +Now you should have: + + - [ ] A GPG key meeting ASF guidelines + - [ ] The key added to + [dev KEYS](https://dist.apache.org/repos/dist/dev/beam/KEYS) and [release KEYS](https://dist.apache.org/repos/dist/release/beam/KEYS) + **NOTE**: Only PMC can write into [release repo](https://dist.apache.org/repos/dist/release/beam/). + - [ ] The `user.signingkey` set in your `.gitconfig` + - [ ] `gpg-agent` with the key loaded + +##### Key ID + +You may need your Key ID for future steps. Determine your Apache GPG Key and +Key ID as follows: + + gpg --list-sigs --keyid-format LONG + +This will list your GPG keys. One of these should reflect your Apache account, +for example: + + -------------------------------------------------- + pub rsa4096/845E6689845E6689 2016-02-23 + uid Nomen Nescio + sub rsa4096/BA4D50BEBA4D50BE 2016-02-23 + +Here, the key ID is the 16-digit hex string in the `pub` line: `845E6689845E6689`. + +##### Submit your GPG public key into Ubuntu OpenPGP Key Server + +In order to make yourself have right permission to stage java artifacts in +Apache Nexus staging repository, please submit your GPG public key into the +[Ubuntu OpenPGP Key Server](https://keyserver.ubuntu.com/). + +You will need to use an ascii-armored version of your key. This can be +obtained by running: + + gpg --export --armor + +Copying the whole block including `-----START PGP PUBLIC KEY BLOCK-----` and +`-----END PGP PUBLIC KEY BLOCK-----` + +#### Access to Apache Nexus repository + +Configure access to the [Apache Nexus +repository](https://repository.apache.org/), which enables final deployment of +releases to the Maven Central Repository. + +1. Log in with your Apache account. +2. Confirm you have appropriate access by finding `org.apache.beam` under + `Staging Profiles`. +3. Navigate to your `Profile` (top right dropdown menu of the page). +4. Choose `User Token` from the dropdown, then click `Access User Token`. Copy + a snippet of the Maven XML configuration block. +5. Insert this snippet + twice into your global Maven `settings.xml` file, typically + `${HOME}/.m2/settings.xml`. The end result should look like this, where + `TOKEN_NAME` and `TOKEN_PASSWORD` are your secret tokens: + + + + + + apache.releases.https + TOKEN_NAME + TOKEN_PASSWORD + + + apache.snapshots.https + TOKEN_NAME + TOKEN_PASSWORD + + + + +---- + +### Dependency checks + +Each language has routine dependency maintenance that you should check now. + +#### Update base image dependencies for Python container images + +The Python base container images have pinned `requirements.txt` that are +compatible with our dependency constraints, and design to avoid run-time +installs, since run-time installs cause large delays at start-up time. Ideally, +we this should happen regularly when dependencies update, but it is important +to ensure that they are fully up to date for each release. + +Follow the instructions at +https://s.apache.org/beam-python-requirements-generate + +#### Update Go version used for container builds + +Go makes security patch releases of their tooling. This potentially affects +container bootloader security, and at the least can cause false positives when +an default-configuration scanner is pointed at our containers. Ideally, we +upgrade as soon as possible, but it is also good to ensure we are up to date +for each release. + +See if https://go.dev/doc/devel/release has a newer release. Update throughout +Beam. See example at https://github.com/apache/beam/pull/27900/files + +#### Update the Java BOM + +Google releases a BOM that pins compatible versions of their Java libraries. +Ideally, this update happens as far in advance of the release as possible, such +as just after a release. But if that was not done, consider doing it before +cutting the release branch. + +To do so, follow instructions at +https://github.com/apache/beam/blob/master/contributor-docs/java-dependency-upgrades.md. + +### Cut the release branch + +> **Note** +> Wait until the proposed branch cut day! + +We cut the release branch on time and do not block/delay branch cut for incoming +fixes. This is because bugs are always being introduced as part of normal +development. We cut the branch to prevent new bugs being introduced and _then_ +we fix and cherrypick any truly release-blocking problems. + +In order to run this workflow, you will need to provide a Apache ID and Jenkins +API token. Your Jenkins API token can be generated by visiting +https://ci-beam.apache.org, signing in with your Apache credentials, then going +to `https://ci-beam.apache.org/user//configure` and clicking +`Add new token` in the API token section. + +- [ ] Run + [cut_release_branch](https://github.com/apache/beam/actions/workflows/cut_release_branch.yml) + (click `run workflow`) + +The final state of the repository after release branch is cut should match this +diagram: + +Increment minor version on master branch and set Dataflow container version on release branch + +This should be accomplished by the +[cut_release_branch](https://github.com/apache/beam/actions/workflows/cut_release_branch.yml) +workflow. This workflow will also update +[mass_comment.py](https://github.com/apache/beam/blob/master/release/src/main/scripts/mass_comment.py) +to contain all of the active Jenkins jobs. + +The following must be manually done or confirmed: + +- [ ] The `master` branch has the SNAPSHOT/dev version incremented. +- [ ] The release branch has the SNAPSHOT/dev version to be released. +- [ ] The Dataflow container image should be modified to the version to be released. +- [ ] Due to a bug/limitation in the workflow, you must navigate to the pull + request found in the logs and comment `Run Gradle Publish`. +- [ ] After publish, close the PR. +- [ ] Manually update `CHANGES.md` on `master` by adding a new section for the + next release + ([example](https://github.com/apache/beam/commit/96ab1fb3fe07acf7f7dc9d8c829ae36890d1535c)). + +#### Inform the mailing list + +The dev@ mailing list should be informed about the release branch being cut. +Alongside with this note, a list of pending issues and to-be-triaged issues +should be included. Afterwards, this list can be refined and updated by the +release manager and the Beam community. + + +### Checklist to proceed to the next phase + +- [ ] Community agrees to release +- [ ] Community selects a committer (you) as Release Manager +- [ ] Next release has a milestone in github. +- [ ] You have your various account credentials prepared. +- [ ] You checked the dependency maintenance for each language. +- [ ] The release branch is created. +- [ ] The `master` branch is moved along to the next release. +- [ ] You have informed `dev@beam.apache.org` that you have cut the branch and + are proceeding to stabilization + +------- + +## Stabilize the release branch + +Once the release branch is cut, your job is to make sure tests pass, fix bugs, +confirm performance, defer feature requests, etc, until the branch is ready for +the work of building a release candidate. + +### Verify release branch + +After the release branch is cut, make sure it builds and has no significant +issues that would block the creation of the release candidate. + +> **NOTE** +> Dataflow tests will fail if the Dataflow worker container is not created and +> published by this time. Should be done by Google, in response to the +> creation of the release branch, and docker images are hosted. This should +> not block creation of the first release candidate, but should block approval +> of the release. + +- **Script:** + [verify_release_build.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/verify_release_build.sh) + +- **Usage** + 1. Create a personal access token from your Github account. + See instruction [here](https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line). + It'll be used by the script for accessing Github API. + You need to enable `repo` and `workflow` permissions for this token. + 2. Update required configurations listed in `RELEASE_BUILD_CONFIGS` in [script.config](https://github.com/apache/beam/blob/master/release/src/main/scripts/script.config) + 3. Then run + ``` + (cd release/src/main/scripts && ./verify_release_build.sh) + ``` + 4. Trigger all Jenkins PostCommit jobs from the PR created by the previous step. + You can run [mass_comment.py](https://github.com/apache/beam/blob/master/release/src/main/scripts/mass_comment.py) to do that. + Or manually add one trigger phrase per PR comment. + See [jenkins_jobs.txt](https://github.com/apache/beam/blob/master/release/src/main/scripts/jenkins_jobs.txt) + for a full list of phrases. + +- **Tasks included in the script** + - Installs `hub` with your agreement and setup local git repo; + - Create a test PR against release branch; + +There are some projects that don't produce the artifacts, e.g. +`beam-test-tools`, you may be able to ignore failures there. + +To triage the failures and narrow things down you may want to look at +`settings.gradle.kts` and run the build only for the projects you're interested +at the moment, e.g. `./gradlew :runners:java-fn-execution`. + +The `verify_release_build.sh` script may include failing or flaky tests. For +each of the failing tests create a GitHub Issue with the following properties: + +* **Issue Type:** Bug + +* **Summary:** Name of failing gradle task and name of failing test (where applicable) in form of :MyGradleProject:SomeGradleTask NameOfFailedTest: Short description of failure + +* **Priority:** P1 + +* **Component:** "test-failures" + +* **Milestone:** Release number of verified release branch + +* **Description:** Description of failure + +### Investigate performance regressions + +Check the Beam load tests for possible performance regressions. Measurements +are available on [metrics.beam.apache.org](http://metrics.beam.apache.org). + +All Runners which publish data should be checked for the following, in both +*batch* and *streaming* mode: + +- [ParDo](http://metrics.beam.apache.org/d/MOi-kf3Zk/pardo-load-tests) and + [GBK](http://metrics.beam.apache.org/d/UYZ-oJ3Zk/gbk-load-test): Runtime, + latency, checkpoint duration +- [Nexmark](http://metrics.beam.apache.org/d/ahudA_zGz/nexmark): Query runtime + for all queries +- [IO](http://metrics.beam.apache.org/d/bnlHKP3Wz/java-io-it-tests-dataflow): Runtime + +If regressions are found, the release branch can still be created, but the +regressions should be investigated and fixed as part of the release process. +The role of the release manager is to file GitHub issues for each regression +with the milestone set to the to-be-released version. The release manager +oversees these just like any other issue marked with the milestone of the +release. + +The mailing list should be informed to allow fixing the regressions in the +course of the release. Issues should be filed and tagged with the milestone. + +### Triage release-blocking issues in GitHub + +There could be outstanding release-blocking issues, which should be triaged +before proceeding to build a release candidate. We track them by assigning the +blocked release to the issue's milestone before the issue is resolved. + +The release manager should triage what does and does not block a release. The +list of release-blocking issues is available at the [milestone status +page](https://github.com/apache/beam/milestones). Triage each unresolved issue +with one of the following resolutions: + + - An issue should not block the release if the problem exists in the current + released version or is a bug in new functionality that does not exist in the + current released version. + - An issue should be a blocker if the problem is a regression between the + currently released version and the release in progress and has no easy + workaround. + +For all GitHub issues: + + - If the issue has been resolved and the GitHub issue was not updated, + resolve it accordingly. + +For issues with type "Bug" or labeled "flaky": + + - If the issue is a known continuously failing test, it is not acceptable to + defer this until the next release. Please work with the Beam community to + resolve the issue. + - If the issue is a known flaky test, make an attempt to delegate a fix. + However, if the issue may take too long to fix (to the discretion of the + release manager): + - Delegate manual testing of the flaky issue to ensure no release blocking issues. + - Update the milestone to the version of the next release. + Please consider discussing this with stakeholders and the dev@ mailing + list, as appropriate. + +For all other GitHub issues: + + - If the issue has not been resolved and it is acceptable to defer this until the next release, update the milestone to the new version you just created. + Please consider discussing this with stakeholders and the dev@ mailing list, as appropriate. + - If the issue has not been resolved and it is not acceptable to release until it is fixed, the release cannot proceed. + Instead, work with the Beam community to resolve the issue. + +If there is a bug found in the RC creation process/tools, those issues should +be considered high priority and fixed in 7 days. + +### Review cherry-picks + +The release manager is empowered to triage issues, and accept or reject +cherry-picks to the release branch. Cherry picks are necessary if there are +outstanding issues at time of the release branch cut, or issues were found in +verification. + +Check if there are outstanding cherry-picks into the release branch, [e.g. for +`2.14.0`](https://github.com/apache/beam/pulls?utf8=%E2%9C%93&q=is%3Apr+base%3Arelease-2.14.0). +Make sure they have blocker Issues attached and are OK to get into the release +by checking with community if needed. + +You are encouraged to ask the following questions to be answered on each +cherry-pick PR and you can choose to reject cherry-pick requests if these +questions are not satisfactorily answered: + + - Is this a regression from a previous release? (If no, fix could go to a + newer version.) + - Is this a new feature or related to a new feature? (If yes, fix could go to + a new version.) + - Would this impact production workloads for users? (E.g. if this is a direct + runner only fix it may not need to be a cherry pick.) + - What percentage of users would be impacted by this issue if it is not fixed? + (E.g. If this is predicted to be a small number it may not need to be a + cherry pick.) + - Would it be possible for the impacted users to skip this version? (If users + could skip this version, fix could go to a newer version.) + +It is important to accept major/blocking fixes to isolated issues to make a +higher quality release. However, beyond that each cherry pick will increase +the time required for the release and add more last minute code to the release +branch. Neither late releases nor not fully tested code will provide positive +user value. + +> **Tip**: Another tool in your toolbox is the known issues section of the +> release blog. Consider adding known issues there for minor issues instead of +> accepting cherry picks to the release branch. + +## Build a release candidate + +From the release branch, building a candidate involves selecting a commit, +tagging that commit, and building the various artifacts against that commit. +You can also run verifications against the RC commit (verification will also +occur during voting phase). + +#### Checklist before proceeding + +- [ ] There are no release blocking GitHub issues. +- [ ] There are no open pull requests to release branch. +- [ ] Release Manager’s GPG key is published to `dist.apache.org`. +- [ ] Release Manager’s GPG key is configured in `git` configuration. +- [ ] Set `SIGNING_KEY` to the public key of the Manager's GPG key. +- [ ] Release Manager has `org.apache.beam` listed under `Staging Profiles` in Nexus. +- [ ] Release Manager’s Nexus User Token is configured in `settings.xml`. +- [ ] Set `JAVA_HOME` to JDK 8 (Example: `export JAVA_HOME=/example/path/to/java/jdk8`). +- [ ] Have Java 11 installed. + +### Tag a chosen commit for the RC + +Release candidates are built from single tagged commits off the release branch. +When you have identified a good commit on the release branch, run +[choose_rc_commit.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/choose_rc_commit.sh) +to set it up correctly. + + ./release/src/main/scripts/choose_rc_commit.sh \ + --release "${RELEASE_VERSION}" \ + --rc "${RC_NUM}" \ + --commit "${COMMIT_REF}" \ + --clone \ + --push-tag + +You can do a dry run by omitting the `--push-tag` flag. Then it will only clone +the repo, adjust the version, and add the tag locally. If it looks good, run it +again with `--push-tag`. If you already have a clone that includes the +`${COMMIT_REF}` then you can omit `--clone`. This is perfectly safe since the +script does not depend on the current working tree. + +See the source of the script for more details, or to run commands manually in +case of a problem. + +The final state of the repository after an RC commit is chosen should match +this diagram: + +Set version to non-SNAPSHOT, non-dev, on tagged RC commit + +The following should be confirmed: + +- [ ] The release branch is unchanged. +- [ ] There is a commit not on the release branch with the version adjusted. +- [ ] The RC tag points to that commit. + +### Run build_release_candidate GitHub Action to create a release candidate + +Note: This step is partially automated (in progress), so part of the RC +creation is done by GitHub Actions and the rest is done by a script. You don't +need to wait for the action to complete to start running the script. + +**Action** [build_release_candidate](https://github.com/apache/beam/actions/workflows/build_release_candidate.yml) (click `run workflow`) + +**The action will:** + +1. Clone the repo at the selected RC tag. +2. Run gradle publish to push java artifacts into Maven staging repo. +3. Stage SDK docker images to [docker hub Apache + organization](https://hub.docker.com/search?q=apache%2Fbeam&type=image). +4. Build javadoc, pydoc, typedocs for a PR to update beam-site. + - **NOTE**: Do not merge this PR until after an RC has been approved (see + "Finalize the Release"). + +### Verify source distributions + + - [ ] Verify that the source zip of the whole project is present in [dist.apache.org](https://dist.apache.org/repos/dist/dev/beam). + - [ ] Verify that the Python binaries are present in [dist.apache.org](https://dist.apache.org/repos/dist/dev/beam). + +### Verify docker images + +At +[https://hub.docker.com/u/apache](https://hub.docker.com/search?q=apache%2Fbeam&type=image), +visit each repository and navigate to "tags" tab. Verify images are pushed +with tags: `${RELEASE_VERSION}_rc{RC_NUM}` + +Verify that third party licenses are included in Docker. You can do this with a simple script: + + for pyver in 3.8 3.9 3.10 3.11; do + docker run --rm --entrypoint sh \ + apache/beam_python${pyver}_sdk:2.51.0rc1 \ + -c 'ls -al /opt/apache/beam/third_party_licenses/ | wc -l' + done + + for javaver in 8 11 17; do + docker run --rm --entrypoint sh \ + apache/beam_java${pyver}_sdk:2.51.0rc1 \ + -c 'ls -al /opt/apache/beam/third_party_licenses/ | wc -l' + done + +And you may choose to log in to the containers and inspect: + + docker run --rm -it --entrypoint=/bin/bash \ + apache/beam_java${ver}_sdk:${RELEASE_VERSION}rc${RC_NUM} + ls -al /opt/apache/beam/third_party_licenses/ + +### Publish Java staging artifacts (manual) + + 1. Log in to the [Apache Nexus](https://repository.apache.org/#stagingRepositories) website. + 2. Navigate to Build Promotion -> Staging Repositories (in the left sidebar). + 3. Select repository `orgapachebeam-NNNN`. + 4. Click the Close button. + 5. When prompted for a description, enter “Apache Beam, version X, release candidate Y”. + 6. Review all staged artifacts on `https://repository.apache.org/content/repositories/orgapachebeam-NNNN/`. + They should contain all relevant parts for each module, including `pom.xml`, jar, test jar, javadoc, etc. + Artifact names should follow [the existing format](https://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.beam%22) in which artifact name mirrors directory structure, e.g., `beam-sdks-java-io-kafka`. + Carefully review any new artifacts. + Some additional validation should be done during the rc validation step. + + +### Upload `rc` artifacts to PyPI + +This step uploads artifacts such as `apache-beam-${RELEASE_VERSION}rc${RC_NUM}` +to PyPI, so the RC artifacts can be depended upon directly by consumers, for +ease of RC verification. + +**Script:** [deploy_release_candidate_pypi.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/deploy_release_candidate_pypi.sh) + +**Usage** + + ./release/src/main/scripts/deploy_release_candidate_pypi.sh \ + --release "${RELEASE_VERSION}" \ + --rc "${RC_NUM}" \ + --user "${GITHUB_USER}" \ + --deploy + +**The script will:** + +Download previously build python binary artifacts Deploy release candidate +to PyPI with an `rc` suffix. + +__Attention:__ Verify that: +- [ ] The File names version include ``rc-#`` suffix +- [ ] [Download Files](https://pypi.org/project/apache-beam/#files) have: + - [ ] All wheels uploaded as artifacts + - [ ] Release source's zip published + - [ ] Signatures and hashes do not need to be uploaded + +You can do a dry run by omitting the `--deploy` flag. Then it will only +download the release candidate binaries. If it looks good, rerun it with +`--deploy`. + +See the source of the script for more details or to run commands manually in +case of a problem. + +### Propose pull requests for website updates + +Beam publishes API reference manuals for each release on the website. For Java +and Python SDKs, that’s Javadoc and PyDoc, respectively. The final step of +building the candidate is to propose website pull requests that update these +manuals. + +Merge the pull requests only after finalizing the release. To avoid invalid +redirects for the 'current' version, merge these PRs in the order listed. Once +the PR is merged, the new contents will get picked up automatically and served +to the Beam website, usually within an hour. A committer can manually trigger +the +[beam_PostCommit_Website_Publish](https://ci-beam.apache.org/job/beam_PostCommit_Website_Publish/) +task in Jenkins to avoid waiting. + +**PR 1: apache/beam-site** + +This pull request is against the `apache/beam-site` repo, on the `release-docs` +branch ([example](https://github.com/apache/beam-site/pull/603)). It is +created by the `build_release_candidate` workflow (see above). + +**PR 2: apache/beam** + +This pull request is against the `apache/beam` repo, on the `master` branch +([example](https://github.com/apache/beam/pull/17378)). + +- Update `CHANGES.md` to update release date and remove template. +- Update release version in `website/www/site/config.toml`. +- Add new release in `website/www/site/content/en/get-started/downloads.md`. + - Download links will not work until the release is finalized. +- Update links to prior releases to point to https://archive.apache.org (see + example PR). +- Create the Blog post: + +#### Blog post + +Use the template below to write a blog post for the release. See +[beam-2.31.0.md](https://github.com/apache/beam/commit/a32a75ed0657c122c6625aee1ace27994e7df195#diff-1e2b83a4f61dce8014a1989869b6d31eb3f80cb0d6dade42fb8df5d9407b4748) +as an example. + +- Copy the changes for the current release from `CHANGES.md` to the blog post + and edit as necessary. +- Be sure to add yourself to + [authors.yml](https://github.com/apache/beam/blob/master/website/www/site/data/authors.yml) + if necessary. + +> **TIP** +> Use git log to find contributors to the releases. (e.g: `git fetch +> origin --tags; git log --pretty='%aN' ^v2.10.0 v2.11.0-RC1 | sort | uniq`). +> Make sure to clean it up, as there may be duplicate or incorrect user names. + +> **NOTE** +> Make sure to include any breaking changes, even to `@Experimental` +> features, all major features and bug fixes, and all known issues. + +**Template:** + + --- + title: "Apache Beam {$RELEASE_VERSION}" + date: YYYY-MM-DD H:MM:00 Z + categories: + - blog + - release + authors: + - {$RELEASE_MANAGER} + --- + + + We are happy to present the new {$RELEASE_VERSION} release of Beam. + This release includes both improvements and new functionality. + See the [download page](/get-started/downloads/{$DOWNLOAD_ANCHOR}) for this release. + + <{$REMOVE_FOR_VALID_SUMMARY_BREAK}!--more--> + + For more information on changes in {$RELEASE_VERSION}, check out the [detailed release notes]({$LINK_TO_GITHUB_MILESTONE}). + + ## Highlights + + * New highly anticipated feature X added to Python SDK ([#X](https://github.com/apache/beam/issues/X)). + * New highly anticipated feature Y added to Java SDK ([#Y](https://github.com/apache/beam/issues/Y)). + + {$TOPICS e.g.:} + ### I/Os + * Support for X source added (Java) ([#X](https://github.com/apache/beam/issues/X)). + {$TOPICS} + + ### New Features / Improvements + + * X feature added (Python) ([#X](https://github.com/apache/beam/issues/X)). + * Y feature added (Java) [#Y](https://github.com/apache/beam/issues/Y). + + ### Breaking Changes + + * X behavior was changed ([#X](https://github.com/apache/beam/issues/X)). + * Y behavior was changed ([#Y](https://github.com/apache/beam/issues/Y)). + + ### Deprecations + + * X behavior is deprecated and will be removed in X versions ([#X](https://github.com/apache/beam/issues/X)). + + ### Bugfixes + + * Fixed X (Python) ([#X](https://github.com/apache/beam/issues/X)). + * Fixed Y (Java) ([#Y](https://github.com/apache/beam/issues/Y)). + + ### Known Issues + + * {$KNOWN_ISSUE_1} + * {$KNOWN_ISSUE_2} + + ## List of Contributors + + According to git shortlog, the following people contributed to the {$RELEASE_VERSION} release. Thank you to all contributors! + + ${CONTRIBUTORS} + + +### Checklist to proceed to the next phase + +- [ ] Maven artifacts deployed to the staging repository of + [repository.apache.org](https://repository.apache.org/content/repositories/) +- [ ] Source distribution deployed to the dev repository of + [dist.apache.org](https://dist.apache.org/repos/dist/dev/beam/) +- [ ] Website pull request proposed to list the + [release](/get-started/downloads/), publish the [Java API reference + manual](https://beam.apache.org/releases/javadoc/), and publish the [Python + API reference manual](https://beam.apache.org/releases/pydoc/). +- [ ] Docker images are published to + [DockerHub](https://hub.docker.com/search?q=apache%2Fbeam&type=image) with + tags: `{RELEASE_VERSION}_rc{RC_NUM}`. + +You can (optionally) also do additional verification by: + +- [ ] Check that Python zip file contains the `README.md`, `NOTICE`, and + `LICENSE` files. +- [ ] Check hashes (e.g. `md5sum -c *.md5` and `sha1sum -c *.sha1`. Note that + signature/checksum files of Java artifacts may not contain filenames. Hence + you might need to compare checksums/signatures manually or modify the files by + appending the filenames.) +- [ ] Check signatures (e.g. `gpg --verify apache-beam-1.2.3-python.zip.asc + apache-beam-1.2.3-python.zip`) +- [ ] `grep` for legal headers in each file. +- [ ] Run all jenkins suites and include links to passing tests in the voting + email. +- [ ] Pull docker images to make sure they are pullable. (e.g. `docker pull apache/beam_python3.7_sdk:2.39.0rc1` + +********** + +## Vote and validate the release candidate + +Once you have built and individually reviewed the release candidate, please +share it for the community-wide review. Please review foundation-wide [voting +guidelines](https://www.apache.org/foundation/voting.html) for more +information. + +Start the review-and-vote thread on the dev@ mailing list. Here’s an email +template; please adjust as you see fit. + + From: Release Manager + To: dev@beam.apache.org + Subject: [VOTE] Release 1.2.3, release candidate #3 + + Hi everyone, + Please review and vote on the release candidate #3 for the version 1.2.3, as follows: + [ ] +1, Approve the release + [ ] -1, Do not approve the release (please provide specific comments) + + + Reviewers are encouraged to test their own use cases with the release candidate, and vote +1 if + no issues are found. Only PMC member votes will count towards the final vote, but votes from all + community members is encouraged and helpful for finding regressions; you can either test your own + use cases or use cases from the validation sheet [10]. + + The complete staging area is available for your review, which includes: + * GitHub Release notes [1], + * the official Apache source release to be deployed to dist.apache.org [2], which is signed with the key with fingerprint FFFFFFFF [3], + * all artifacts to be deployed to the Maven Central Repository [4], + * source code tag "v1.2.3-RC3" [5], + * website pull request listing the release [6], the blog post [6], and publishing the API reference manual [7]. + * Java artifacts were built with Gradle GRADLE_VERSION and OpenJDK/Oracle JDK JDK_VERSION. + * Python artifacts are deployed along with the source release to the dist.apache.org [2] and PyPI[8]. + * Go artifacts and documentation are available at pkg.go.dev [9] + * Validation sheet with a tab for 1.2.3 release to help with validation [10]. + * Docker images published to Docker Hub [11]. + * PR to run tests against release branch [12]. + + The vote will be open for at least 72 hours. It is adopted by majority approval, with at least 3 PMC affirmative votes. + + For guidelines on how to try the release in your projects, check out our blog post at /blog/validate-beam-release/. + + Thanks, + Release Manager + + [1] https://github.com/apache/beam/milestone/1... + [2] https://dist.apache.org/repos/dist/dev/beam/1.2.3/ + [3] https://dist.apache.org/repos/dist/release/beam/KEYS + [4] https://repository.apache.org/content/repositories/orgapachebeam-NNNN/ + [5] https://github.com/apache/beam/tree/v1.2.3-RC3 + [6] https://github.com/apache/beam/pull/... + [7] https://github.com/apache/beam-site/pull/... + [8] https://pypi.org/project/apache-beam/1.2.3rc3/ + [9] https://pkg.go.dev/github.com/apache/beam/sdks/v2@v1.2.3-RC3/go/pkg/beam + [10] https://docs.google.com/spreadsheets/d/1qk-N5vjXvbcEk68GjbkSZTR8AGqyNUM-oLFo_ZXBpJw/edit#gid=... + [11] https://hub.docker.com/search?q=apache%2Fbeam&type=image + [12] https://github.com/apache/beam/pull/... + +If there are any issues found in the release candidate, reply on the vote +thread to cancel the vote. There’s no need to wait 72 hours. Go back to +["Stabilize the Release Branch"](#stabilize-the-release-branch) and address the problem. However, some issues +don’t require cancellation. For example, if an issue is found in the website +pull request, just correct it on the spot and the vote can continue as-is. + +### Run validation tests + +The community is responsible for performing validation, but as release manager +you are expected to contribute as well. + +Before accepting an RC, as a community we try to exercise most (if not all) of +the tests listed in this +[spreadsheet](https://s.apache.org/beam-release-validation), and those are good +validations for you to try out as release manager. The goal of these tests is +to validate that we're able to run basic pipelines from a variety of +environments (not just our CI environment). + +Since there are many tests, we recommend you running some validations using an +automation script. In case of script failure, you can still run all of them +manually. + +You may need to have Python interpreters for all supported Python minor +versions to run all of the tests. See Python installation tips in [Developer +Wiki](https://cwiki.apache.org/confluence/display/BEAM/Python+Tips#PythonTips-InstallingPythoninterpreters). + +> **Note** +> The community's validation means more than just running the tests +> that we have already run. It includes users trying out the RC on their own +> downstream tests. It also includes double checking that our human-language +> instructions actually still correspond to the automation that we have built. + +#### Run validations using run_rc_validation.sh + +**Script:** [run_rc_validation.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/run_rc_validation.sh) + +**Usage** + 1. First update required configurations listed in `RC_VALIDATE_CONFIGS` in + [script.config](https://github.com/apache/beam/blob/master/release/src/main/scripts/script.config) + 2. Then run + ``` + ./release/src/main/scripts/run_rc_validation.sh + ``` + +**Note:** running the validations requires the ability to do the following in your GCP account: start pipelines, +write to BigQuery, and create a cluster of machines for running containers (for x-lang validation). + +**Tasks included** + 1. Create a PR to trigger Python validation job, including + * Python quickstart in batch and streaming mode with direct runner and Dataflow runner. + * Python Mobile Games(UserScore, HourlyTeamScore) with direct runner and Dataflow runner. + 2. Run Python Streaming MobileGames, includes + * Start a new terminal to run Java Pubsub injector. + * Start a new terminal to run Python LeaderBoard with Direct Runner. + * Start a new terminal to run Python LeaderBoard with Dataflow Runner. + * Start a new terminal to run Python GameStats with Direct Runner. + * Start a new terminal to run Python GameStats with Dataflow Runner. + 3. Multi-language pipelines validation, includes + * Running the Python quickstart example using Python portable DirectRunner. This will start a new terminal for the Java expansion service. + * Running the Java quickstart example using Python portable DirectRunner. This will start new terminals for the Python expansion service and the job server. + * Start a new terminal to run Python multi-language Java kafka validation with Dataflow Runner. + * Start a new terminal to run Python multi-language Java sql validation with Dataflow Runner. + +* **Tasks you need to do manually**. + +- [ ] Check whether validations succeed by following console output instructions. +- [ ] Terminate streaming jobs and java injector. +- [ ] Run Java quickstart (wordcount) and mobile game examples with the staged artifacts. The easiest way to do this is by running the tests on Jenkins. + +- Other manual validation will follow, but this will at least validate that the staged artifacts can be used. + * Log in to Jenkins. + * Go to https://ci-beam.apache.org/job/beam_PostRelease_NightlySnapshot/. + * Click "Build with Parameters". + * Set `snapshot_version` to `2.xx.0`, and set `snapshot_url` to point to the staged artifacts in Maven central (https://repository.apache.org/content/repositories/orgapachebeam-NNNN/). + * Click "Build". +- [ ] Sign up [spreadsheet](https://s.apache.org/beam-release-validation). +- [ ] Vote in the release thread. + +#### Run validations manually + +> **Note** +> `-Prepourl` and `-Pver` can be found in the RC vote email sent by Release Manager. + +* **Java Quickstart Validation** + + **Direct Runner** + ``` + ./gradlew :runners:direct-java:runQuickstartJavaDirect \ + -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-${KEY} \ + -Pver=${RELEASE_VERSION} + ``` + **Flink Local Runner** + ``` + ./gradlew :runners:flink:1.13:runQuickstartJavaFlinkLocal \ + -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-${KEY} \ + -Pver=${RELEASE_VERSION} + ``` + **Spark Local Runner** + ``` + ./gradlew :runners:spark:3:runQuickstartJavaSpark \ + -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-${KEY} \ + -Pver=${RELEASE_VERSION} + ``` + **Dataflow Runner** + ``` + ./gradlew :runners:google-cloud-dataflow-java:runQuickstartJavaDataflow \ + -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-${KEY} \ + -Pver=${RELEASE_VERSION} \ + -PgcpProject=${YOUR_GCP_PROJECT} \ + -PgcsBucket=${YOUR_GCP_BUCKET} + ``` +* **Java Mobile Game(UserScore, HourlyTeamScore, Leaderboard)** + + **Prerequisites** + * **Create your own BigQuery dataset** + ``` + bq mk --project_id=${YOUR_GCP_PROJECT} ${YOUR_DATASET} + ``` + * **Create your PubSub topic** + ``` + gcloud alpha pubsub topics create --project=${YOUR_GCP_PROJECT} ${YOUR_PROJECT_PUBSUB_TOPIC} + ``` + * **Setup your service account** + + Goto IAM console in your project to create a service account as `project owner`, then run + + ``` + gcloud iam service-accounts keys create ${YOUR_KEY_JSON} --iam-account ${YOUR_SERVICE_ACCOUNT_NAME}@${YOUR_PROJECT_NAME} + export GOOGLE_APPLICATION_CREDENTIALS=${PATH_TO_YOUR_KEY_JSON} + ``` + **Run** + ``` + ./gradlew :runners:google-cloud-dataflow-java:runMobileGamingJavaDataflow \ + -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-${KEY} \ + -Pver=${RELEASE_VERSION} \ + -PgcpProject=${YOUR_GCP_PROJECT} \ + -PgcsBucket=${YOUR_GCP_BUCKET} \ + -PbqDataset=${YOUR_DATASET} -PpubsubTopic=${YOUR_PROJECT_PUBSUB_TOPIC} + ``` +* **Python Quickstart(batch & streaming), MobileGame(UserScore, HourlyTeamScore)** + + Create a new PR in apache/beam. + + In comment area, type in `Run Python ReleaseCandidate` to trigger validation. + +* **Python Leaderboard & GameStats** + * **Get staging RC** `wget https://dist.apache.org/repos/dist/dev/beam/2.5.0/* ` + * **Verify the hashes** + + ``` + sha512sum -c apache-beam-2.5.0-python.zip.sha512 + sha512sum -c apache-beam-2.5.0-source-release.zip.sha512 + ``` + * **Build SDK** + + ``` + sudo apt-get install unzip + unzip apache-beam-2.5.0-source-release.zip + python setup.py sdist + ``` + * **Setup virtual environment** + + ``` + python3 -m venv beam_env + . ./beam_env/bin/activate + pip install --upgrade pip setuptools wheel + ``` + * **Install SDK** + + ``` + pip install dist/apache-beam-2.5.0.tar.gz + pip install dist/apache-beam-2.5.0.tar.gz[gcp] + ``` + * **Setup GCP** + + Please repeat following steps for every following test. + + ``` + bq rm -rf --project=${YOUR_PROJECT} ${USER}_test + bq mk --project_id=${YOUR_PROJECT} ${USER}_test + gsutil rm -rf ${YOUR_GS_STORAGE] + gsutil mb -p ${YOUR_PROJECT} ${YOUR_GS_STORAGE} + gcloud alpha pubsub topics create --project=${YOUR_PROJECT} ${YOUR_PUBSUB_TOPIC} + ``` + Setup your service account as described in ```Java Mobile Game``` section above. + + * **Produce data by using java injector:** + + Configure your ~/.m2/settings.xml as following: + ``` + + + + release-repo + + true + + + + Release 2.4.0 RC3 + Release 2.4.0 RC3 + https://repository.apache.org/content/repositories/orgapachebeam-1031/ + + + + + + ``` + __Note__: You can found the latest ```id```, ```name``` and ```url``` for one RC in the vote email thread sent out by Release Manager. + + Run + ``` + mvn archetype:generate \ + -DarchetypeGroupId=org.apache.beam \ + -DarchetypeArtifactId=beam-sdks-java-maven-archetypes-examples \ + -DarchetypeVersion=${RELEASE_VERSION} \ + -DgroupId=org.example \ + -DartifactId=word-count-beam \ + -Dversion="0.1" \ + -Dpackage=org.apache.beam.examples \ + -DinteractiveMode=false + -DarchetypeCatalog=internal + + mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.complete.game.injector.Injector \ + -Dexec.args="${YOUR_PROJECT} ${YOUR_PUBSUB_TOPIC} none" + ``` + * **Run Leaderboard with Direct Runner** + ``` + python -m apache_beam.examples.complete.game.leader_board \ + --project=${YOUR_PROJECT} \ + --topic projects/${YOUR_PROJECT}/topics/${YOUR_PUBSUB_TOPIC} \ + --dataset ${USER}_test + ``` + Inspect results: + * Check whether there is any error messages in console. + * Goto your BigQuery console and check whether your ${USER}_test has leader_board_users and leader_board_teams table. + * bq head -n 10 ${USER}_test.leader_board_users + * bq head -n 10 ${USER}_test.leader_board_teams + + * **Run Leaderboard with Dataflow Runner** + ``` + python -m apache_beam.examples.complete.game.leader_board \ + --project=${YOUR_PROJECT} \ + --region=${GCE_REGION} \ + --topic projects/${YOUR_PROJECT}/topics/${YOUR_PUBSUB_TOPIC} \ + --dataset ${USER}_test \ + --runner DataflowRunner \ + --temp_location=${YOUR_GS_BUCKET}/temp/ \ + --sdk_location dist/* + ``` + Inspect results: + * Goto your Dataflow job console and check whether there is any error. + * Goto your BigQuery console and check whether your ${USER}_test has leader_board_users and leader_board_teams table. + * bq head -n 10 ${USER}_test.leader_board_users + * bq head -n 10 ${USER}_test.leader_board_teams + + * **Run GameStats with Direct Runner** + ``` + python -m apache_beam.examples.complete.game.game_stats \ + --project=${YOUR_PROJECT} \ + --topic projects/${YOUR_PROJECT}/topics/${YOUR_PUBSUB_TOPIC} \ + --dataset ${USER}_test \ + --fixed_window_duration ${SOME_SMALL_DURATION} + ``` + Inspect results: + * Check whether there is any error messages in console. + * Goto your BigQuery console and check whether your ${USER}_test has game_stats_teams and game_stats_sessions table. + * bq head -n 10 ${USER}_test.game_stats_teams + * bq head -n 10 ${USER}_test.game_stats_sessions + + * **Run GameStats with Dataflow Runner** + ``` + python -m apache_beam.examples.complete.game.game_stats \ + --project=${YOUR_PROJECT} \ + --region=${GCE_REGION} \ + --topic projects/${YOUR_PROJECT}/topics/${YOUR_PUBSUB_TOPIC} \ + --dataset ${USER}_test \ + --runner DataflowRunner \ + --temp_location=${YOUR_GS_BUCKET}/temp/ \ + --sdk_location dist/* \ + --fixed_window_duration ${SOME_SMALL_DURATION} + ``` + Inspect results: + * Goto your Dataflow job console and check whether there is any error. + * Goto your BigQuery console and check whether your ${USER}_test has game_stats_teams and game_stats_sessions table. + * bq head -n 10 ${USER}_test.game_stats_teams + * bq head -n 10 ${USER}_test.game_stats_sessions + + +### Finalize the vote + +Reply on the vote thread to close the voting once following conditions are met +for the current release candidate. + +- [ ] At least 72 hours has passed since the voting email. +- [ ] No release blocking issues have been identified. +- [ ] Voting thread has at least three approving PMC votes. + +Then, tally the votes in a separate email thread. Here’s an email template; +please adjust as you see fit. + + From: Release Manager + To: dev@beam.apache.org + Subject: [RESULT] [VOTE] Release 1.2.3, release candidate #3 + + I'm happy to announce that we have unanimously approved this release. + + There are XXX approving votes, XXX of which are binding: + * approver 1 + * approver 2 + * approver 3 + * approver 4 + + There are no disapproving votes. + + Thanks everyone! + +### Checklist to proceed to the next step + +- [ ] Issues identified during vote have been resolved, with fixes committed to the release branch. +- [ ] All issues in the current release's milestone should be closed. +- [ ] Community votes to release the proposed candidate, with at least three approving PMC votes. + + +---- + +## Finalize the release + +Once the release candidate has been reviewed and approved by the community, the release should be finalized. +This involves the final deployment of the release candidate to the release repositories, merging of the website changes, etc. + +This does not take very long, and can be accomplished within hours of the vote being finalized. + +### Deploy artifacts to Maven Central Repository + +Use the [Apache Nexus repository manager](https://repository.apache.org/#stagingRepositories) to release the staged binary artifacts to the Maven Central repository. +In the `Staging Repositories` section, find the relevant release candidate `orgapachebeam-XXX` entry and click `Release`. +Drop all other release candidates that are not being released. + +__NOTE__: If you are using [GitHub two-factor authentication](https://help.github.com/articles/securing-your-account-with-two-factor-authentication-2fa/) and haven't configure HTTPS access, +please follow [the guide](https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/) to configure command line access. + +### Deploy Python artifacts to PyPI + +* **Script:** [deploy_pypi.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/deploy_pypi.sh) +* **Usage** +``` +./release/src/main/scripts/deploy_pypi.sh +``` +* Verify that the files at https://pypi.org/project/apache-beam/#files are correct. +All wheels should be published, in addition to the zip of the release source. +(Signatures and hashes do _not_ need to be uploaded.) + +### Deploy docker images to DockerHub + +Note: if you are not a member of the [beam DockerHub team](https://hub.docker.com/orgs/apache/teams/beam), +you will need help with this step. Please email dev@ mailing list and ask a member of the beam DockerHub team for help. + +* **Script:** [publish_docker_images.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/publish_docker_images.sh) +* **Usage** +``` +./release/src/main/scripts/publish_docker_images.sh +``` +* **Verify that:** + * Images are published at [DockerHub](https://hub.docker.com/search?q=apache%2Fbeam&type=image) with tags {RELEASE_VERSION} and *latest*. + * Images with *latest* tag are pointing to current release by confirming the digest of the image with *latest* tag is the same as the one with {RELEASE_VERSION} tag. + +(Optional) Clean up any unneeded local images afterward to save disk space. + +### Merge Website pull requests + +Merge all of the website pull requests +- [listing the release](/get-started/downloads/) +- publishing the [Python API reference manual](https://beam.apache.org/releases/pydoc/) and the [Java API reference manual](https://beam.apache.org/releases/javadoc/), and +- adding the release blog post. + +### Git tag + +Create and push a new signed tag for the released version by copying the tag for the final release candidate, as follows: + +``` +# Optional: unlock the signing key by signing an arbitrary file. +gpg --output ~/doc.sig --sign ~/.bashrc + +VERSION_TAG="v${RELEASE_VERSION}" +RC_TAG="${VERSION_TAG}-RC${RC_NUM}" + +# Ensure local tags are in sync. If there's a mismatch, it will tell you. +git fetch --all --tags + +# If the tag exists, a commit number is produced, otherwise there's an error. +git rev-list $RC_TAG -n 1 + +# Tag for Go SDK +git tag -s "sdks/$VERSION_TAG" "$RC_TAG" +git push https://github.com/apache/beam "sdks/$VERSION_TAG" + +# Tag for repo root. +git tag -s "$VERSION_TAG" "$RC_TAG" +git push https://github.com/apache/beam "$VERSION_TAG" +``` + +After pushing the tag, the tag should be visible on Github's [Tags](https://github.com/apache/beam/tags) page. + +### Publish release to Github + +Once the tag is uploaded, publish the release notes to Github. From the [Beam release page on Github](https://github.com/apache/beam/releases) select +"Draft a new release." Title the release "Beam ${RELEASE_VERSION} release" and set the release at the version tag created above. Use the content of the +release blog post as the body of the release notes, set this version as the latest release, and publish it. + +The release notes should now be visible on Github's [Releases](https://github.com/apache/beam/releases) page. + +### Mark the version as released in GitHub + +In GitHub, in the [milestone page](https://github.com/apache/beam/milestones), click close on the current release. + +### PMC-Only Finalization +There are a few release finalization tasks that only PMC members have permissions to do. +Ping [dev@](mailto:dev@beam.apache.org) mailing list for assistance if you need it. + +#### Deploy source release to dist.apache.org + +Copy the source release from the `dev` repository to the `release` repository at `dist.apache.org` using Subversion. + +Make sure the last release's artifacts have been copied from `dist.apache.org` to `archive.apache.org`. +This should happen automatically: [dev@ thread](https://lists.apache.org/thread.html/39c26c57c5125a7ca06c3c9315b4917b86cd0e4567b7174f4bc4d63b%40%3Cdev.beam.apache.org%3E) with context. + +#### Recordkeeping with ASF + +Use [reporter.apache.org](https://reporter.apache.org/addrelease.html?beam) to seed the information about the release into future project reports. + +### Checklist to proceed to the next step + +- [ ] Maven artifacts released and indexed in the [Maven Central Repository](https://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.beam%22) +- [ ] Source distribution available in the release repository of [dist.apache.org](https://dist.apache.org/repos/dist/release/beam/) +- [ ] Source distribution removed from the dev repository of [dist.apache.org](https://dist.apache.org/repos/dist/dev/beam/) +- [ ] Website pull request to [list the release](/get-started/downloads/) and publish the [API reference manual](https://beam.apache.org/releases/javadoc/) merged. +- [ ] The release is tagged on Github's [Tags](https://github.com/apache/beam/tags) page. +- [ ] The release notes are published on Github's [Releases](https://github.com/apache/beam/releases) page. +- [ ] Release version finalized in GitHub. +- [ ] Release version is listed at reporter.apache.org + + +********** + +## Promote the release + +Once the release has been finalized, the last step of the process is to promote the release within the project and beyond. + +### Apache mailing lists + +Announce on the dev@ mailing list that the release has been finished. + +Announce on the release on the user@ mailing list, listing major improvements and contributions. + +Announce the release on the announce@apache.org mailing list. +__NOTE__: This can only be done from `@apache.org` email address. This email has to be in plain text (no HTML tags). + +### Social media + +Tweet, post on Facebook, LinkedIn, and other platforms. +Ask other contributors to do the same. + +Also, update [the Wikipedia article on Apache Beam](https://en.wikipedia.org/wiki/Apache_Beam). + +### Checklist to declare the process completed + +- [ ] Release announced on the user@ mailing list. +- [ ] Blog post published, if applicable. +- [ ] Release recorded in reporter.apache.org. +- [ ] Release announced on social media. +- [ ] Completion declared on the dev@ mailing list. +- [ ] Update Wikipedia Apache Beam article. + +********** + +## Post-Release Tasks + +At the end of the release, go to the GitHub milestones page and mark the recently released version as closed. + +### Update Beam Playground + +After new Beam Release is published, Beam Playground can be updated following the steps below: + +1. Open the [Cloud Build triggers in apache-beam-testing](https://console.cloud.google.com/cloud-build/triggers?project=apache-beam-testing) GCP project. +2. Find the trigger "Deploy-Update-Playground-environment-stg": + 1. Click on the trigger name to open its settings + 2. Change the value for _SDK_TAG variable (Advanced -> Substitution Variables) to the actual version of Beam SDK (e.g. 2.47.0) + 3. Click the Save button. The settings window should close without any errors + 4. Click the RUN button next to the trigger name + 5. In the panel that opened, set the value for the _CONTAINER_TAG variable in format DD-MM-vXX (DD - day, MM - month, XX - version, e.g., 20-12-v01) + 6. Click the Run Trigger button + 7. Open the [Trigger History](https://console.cloud.google.com/cloud-build/builds?project=apache-beam-testing) and wait for the job completion. Ensure that the job completed successfully (Status field shows a green tick) +3. Find the trigger "Playground-CD-stable-manual-stg", it will be run twice, once with default variables, and once with some overridden: + 8. Click the RUN button next to the trigger name + 9. In the panel that opened, click the Run Trigger button (with default variable values) + 10. Open the [Trigger History](https://console.cloud.google.com/cloud-build/builds?project=apache-beam-testing) and wait for the job completion. Ensure that the job completed successfully (Status field shows a green tick) + 11. Click the RUN button next to the trigger name + 12. In the panel that opened, change values for the variables: + * _ORIGIN = PG_BEAMDOC + * _SUBDIRS = ./learning/beamdoc + 13. Click the Run Trigger button + 14. Open the [Trigger History](https://console.cloud.google.com/cloud-build/builds?project=apache-beam-testing) and wait for the job completion. Ensure that the job completed successfully (Status field shows a green tick) +4. Test updated [staging Playground](https://play-dev.beam.apache.org/) in a browser + 15. Open the menu (represented by '...' in the right top corner) and click on Versions. Validate that commit is the same for all listed containers, and the hash belongs to a [recent master branch commit](https://github.com/apache/beam/commits/master) + 16. For each of the supported SDKs (Java, Python, Go, SCIO): + * Switch to the SDK + * Make any changes to the loaded default example + * Click the Run button + * Wait for successful completion + * Click "Share My Code" to ensure that the link is generated +5. Repeat the same steps for "Deploy-Update-Playground-environment-prod" trigger as for "Deploy-Update-Playground-environment-stg" trigger +6. Repeat the same steps for "Playground-CD-stable-manual-prod" trigger as for "Playground-CD-stable-manual-stg" trigger +7. Test updated [prod Playground](https://play.beam.apache.org/) in a browser. The process is similar to the staging environment. +8. Find the trigger "Playground-CI-stable" + 1. Click on the trigger name to open its settings + 2. Set the value for the _BEAM_VERSION variable (Advanced -> Substitution Variables) to the actual version of Beam SDK (e.g., 2.47.0) + 3. Click the Save button. Click the Save button. The settings window should close without any errors + +### Improve the process + +It is important that we improve the release processes over time. +Once you’ve finished the release, please take a step back and look what areas of this process and be improved. Perhaps some part of the process can be simplified. +Perhaps parts of this guide can be clarified. + +If we have specific ideas, please start a discussion on the dev@ mailing list and/or propose a pull request to update this guide. +Thanks! diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 1301baa041f41..c118ad0983342 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -78,7 +78,7 @@ RUN pip3 install distlib==0.3.1 yapf==0.29.0 pytest ### # Install Go ### -ENV DOWNLOAD_GO_VERSION=1.19.6 +ENV DOWNLOAD_GO_VERSION=1.21.0 RUN wget https://golang.org/dl/go${DOWNLOAD_GO_VERSION}.linux-amd64.tar.gz && \ tar -C /usr/local -xzf go${DOWNLOAD_GO_VERSION}.linux-amd64.tar.gz ENV GOROOT /usr/local/go diff --git a/examples/java/OWNERS b/examples/java/OWNERS deleted file mode 100644 index 38112463e09a4..0000000000000 --- a/examples/java/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lukecwik - - aaltay diff --git a/examples/java/build.gradle b/examples/java/build.gradle index 9b5421cd6110e..2e262e8de795a 100644 --- a/examples/java/build.gradle +++ b/examples/java/build.gradle @@ -53,7 +53,7 @@ configurations.sparkRunnerPreCommit { dependencies { implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.kafka_clients implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(":sdks:java:extensions:avro") @@ -83,7 +83,7 @@ dependencies { implementation library.java.commons_csv runtimeOnly project(path: ":runners:direct-java", configuration: "shadow") implementation library.java.vendored_grpc_1_54_0 - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation "com.google.api.grpc:proto-google-cloud-language-v1:1.81.4" implementation ("io.confluent:kafka-avro-serializer:5.3.2") { // It depends on "spotbugs-annotations:3.1.9" which clashes with current diff --git a/examples/java/cdap/hubspot/build.gradle b/examples/java/cdap/hubspot/build.gradle index 4097842d1cc0d..8f560c73cf967 100644 --- a/examples/java/cdap/hubspot/build.gradle +++ b/examples/java/cdap/hubspot/build.gradle @@ -73,7 +73,7 @@ dependencies { implementation library.java.google_code_gson implementation library.java.hadoop_common implementation library.java.slf4j_api - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre runtimeOnly project(path: ":runners:direct-java", configuration: "shadow") // Add dependencies for the PreCommit configurations diff --git a/examples/java/cdap/hubspot/src/main/java/org/apache/beam/examples/complete/cdap/hubspot/utils/GetOffsetUtils.java b/examples/java/cdap/hubspot/src/main/java/org/apache/beam/examples/complete/cdap/hubspot/utils/GetOffsetUtils.java index e59323ddaa3a9..0b212d646e574 100644 --- a/examples/java/cdap/hubspot/src/main/java/org/apache/beam/examples/complete/cdap/hubspot/utils/GetOffsetUtils.java +++ b/examples/java/cdap/hubspot/src/main/java/org/apache/beam/examples/complete/cdap/hubspot/utils/GetOffsetUtils.java @@ -24,7 +24,7 @@ import java.util.HashMap; import org.apache.beam.sdk.io.cdap.Plugin; import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.reflect.TypeToken; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.reflect.TypeToken; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/examples/java/cdap/hubspot/src/main/java/org/apache/beam/examples/complete/cdap/hubspot/utils/PluginConfigOptionsConverter.java b/examples/java/cdap/hubspot/src/main/java/org/apache/beam/examples/complete/cdap/hubspot/utils/PluginConfigOptionsConverter.java index ddcdd3004a8d1..0f4625d69d0bb 100644 --- a/examples/java/cdap/hubspot/src/main/java/org/apache/beam/examples/complete/cdap/hubspot/utils/PluginConfigOptionsConverter.java +++ b/examples/java/cdap/hubspot/src/main/java/org/apache/beam/examples/complete/cdap/hubspot/utils/PluginConfigOptionsConverter.java @@ -24,7 +24,7 @@ import java.util.Map; import org.apache.beam.examples.complete.cdap.hubspot.options.CdapHubspotOptions; import org.apache.beam.examples.complete.cdap.hubspot.options.CdapHubspotStreamingSourceOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * Class for converting CDAP {@link org.apache.beam.sdk.options.PipelineOptions} to map for {@link diff --git a/examples/java/cdap/salesforce/build.gradle b/examples/java/cdap/salesforce/build.gradle index d1038af997eba..1abe19d6403b7 100644 --- a/examples/java/cdap/salesforce/build.gradle +++ b/examples/java/cdap/salesforce/build.gradle @@ -69,7 +69,7 @@ dependencies { implementation library.java.google_code_gson implementation library.java.hadoop_common implementation library.java.slf4j_api - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre runtimeOnly project(path: ":runners:direct-java", configuration: "shadow") // Add dependencies for the PreCommit configurations diff --git a/examples/java/cdap/salesforce/src/main/java/org/apache/beam/examples/complete/cdap/salesforce/utils/GetOffsetUtils.java b/examples/java/cdap/salesforce/src/main/java/org/apache/beam/examples/complete/cdap/salesforce/utils/GetOffsetUtils.java index 1f6b7a280d3ae..4ca7ca81db4fa 100644 --- a/examples/java/cdap/salesforce/src/main/java/org/apache/beam/examples/complete/cdap/salesforce/utils/GetOffsetUtils.java +++ b/examples/java/cdap/salesforce/src/main/java/org/apache/beam/examples/complete/cdap/salesforce/utils/GetOffsetUtils.java @@ -25,7 +25,7 @@ import java.util.HashMap; import org.apache.beam.sdk.io.cdap.Plugin; import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.reflect.TypeToken; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.reflect.TypeToken; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/examples/java/cdap/salesforce/src/main/java/org/apache/beam/examples/complete/cdap/salesforce/utils/PluginConfigOptionsConverter.java b/examples/java/cdap/salesforce/src/main/java/org/apache/beam/examples/complete/cdap/salesforce/utils/PluginConfigOptionsConverter.java index 2063a803c5309..d22287af65a64 100644 --- a/examples/java/cdap/salesforce/src/main/java/org/apache/beam/examples/complete/cdap/salesforce/utils/PluginConfigOptionsConverter.java +++ b/examples/java/cdap/salesforce/src/main/java/org/apache/beam/examples/complete/cdap/salesforce/utils/PluginConfigOptionsConverter.java @@ -26,7 +26,7 @@ import org.apache.beam.examples.complete.cdap.salesforce.options.CdapSalesforceSinkOptions; import org.apache.beam.examples.complete.cdap.salesforce.options.CdapSalesforceSourceOptions; import org.apache.beam.examples.complete.cdap.salesforce.options.CdapSalesforceStreamingSourceOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * Class for converting CDAP {@link org.apache.beam.sdk.options.PipelineOptions} to map for {@link diff --git a/examples/java/cdap/servicenow/build.gradle b/examples/java/cdap/servicenow/build.gradle index 1ae88b39230fc..f113677bae532 100644 --- a/examples/java/cdap/servicenow/build.gradle +++ b/examples/java/cdap/servicenow/build.gradle @@ -65,7 +65,7 @@ dependencies { implementation library.java.cdap_plugin_service_now implementation library.java.hadoop_common implementation library.java.slf4j_api - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre runtimeOnly project(path: ":runners:direct-java", configuration: "shadow") // Add dependencies for the PreCommit configurations diff --git a/examples/java/cdap/servicenow/src/main/java/org/apache/beam/examples/complete/cdap/servicenow/utils/PluginConfigOptionsConverter.java b/examples/java/cdap/servicenow/src/main/java/org/apache/beam/examples/complete/cdap/servicenow/utils/PluginConfigOptionsConverter.java index 32c7b6b9ce042..daa4c15241cae 100644 --- a/examples/java/cdap/servicenow/src/main/java/org/apache/beam/examples/complete/cdap/servicenow/utils/PluginConfigOptionsConverter.java +++ b/examples/java/cdap/servicenow/src/main/java/org/apache/beam/examples/complete/cdap/servicenow/utils/PluginConfigOptionsConverter.java @@ -21,7 +21,7 @@ import io.cdap.plugin.servicenow.source.util.ServiceNowConstants; import java.util.Map; import org.apache.beam.examples.complete.cdap.servicenow.options.CdapServiceNowOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * Class for converting CDAP {@link org.apache.beam.sdk.options.PipelineOptions} to map for {@link diff --git a/examples/java/cdap/zendesk/build.gradle b/examples/java/cdap/zendesk/build.gradle index 289faecbf8e5f..ef99fe8306ed2 100644 --- a/examples/java/cdap/zendesk/build.gradle +++ b/examples/java/cdap/zendesk/build.gradle @@ -65,7 +65,7 @@ dependencies { implementation library.java.cdap_plugin_zendesk implementation library.java.hadoop_common implementation library.java.slf4j_api - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre runtimeOnly project(path: ":runners:direct-java", configuration: "shadow") // Add dependencies for the PreCommit configurations diff --git a/examples/java/cdap/zendesk/src/main/java/org/apache/beam/examples/complete/cdap/zendesk/utils/PluginConfigOptionsConverter.java b/examples/java/cdap/zendesk/src/main/java/org/apache/beam/examples/complete/cdap/zendesk/utils/PluginConfigOptionsConverter.java index a7b18cacf05d5..53552753ad6f5 100644 --- a/examples/java/cdap/zendesk/src/main/java/org/apache/beam/examples/complete/cdap/zendesk/utils/PluginConfigOptionsConverter.java +++ b/examples/java/cdap/zendesk/src/main/java/org/apache/beam/examples/complete/cdap/zendesk/utils/PluginConfigOptionsConverter.java @@ -22,7 +22,7 @@ import io.cdap.plugin.zendesk.source.common.config.BaseZendeskSourceConfig; import java.util.Map; import org.apache.beam.examples.complete.cdap.zendesk.options.CdapZendeskOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * Class for converting CDAP {@link org.apache.beam.sdk.options.PipelineOptions} to map for {@link diff --git a/examples/java/src/main/java/org/apache/beam/examples/common/ExampleUtils.java b/examples/java/src/main/java/org/apache/beam/examples/common/ExampleUtils.java index a8901aa64113b..0d52baefb8463 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/common/ExampleUtils.java +++ b/examples/java/src/main/java/org/apache/beam/examples/common/ExampleUtils.java @@ -50,10 +50,10 @@ import org.apache.beam.sdk.util.BackOffUtils; import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.joda.time.Duration; /** diff --git a/examples/java/src/main/java/org/apache/beam/examples/common/WriteOneFilePerWindow.java b/examples/java/src/main/java/org/apache/beam/examples/common/WriteOneFilePerWindow.java index b15eff640dc8a..902ac06c19695 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/common/WriteOneFilePerWindow.java +++ b/examples/java/src/main/java/org/apache/beam/examples/common/WriteOneFilePerWindow.java @@ -17,7 +17,7 @@ */ package org.apache.beam.examples.common; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; import javax.annotation.Nullable; import org.apache.beam.sdk.io.FileBasedSink; diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java b/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java index 99f509f9f7b87..1393aa79ccc9d 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java @@ -19,7 +19,7 @@ import static com.google.datastore.v1.client.DatastoreHelper.makeKey; import static com.google.datastore.v1.client.DatastoreHelper.makeValue; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.services.bigquery.model.TableFieldSchema; import com.google.api.services.bigquery.model.TableReference; @@ -68,7 +68,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.joda.time.Duration; /** diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java b/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java index 309834a4b6c31..b3e5fd04fa7e8 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java @@ -73,7 +73,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java b/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java index 75d7e2febb5b1..b06cd8da9d439 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java @@ -61,7 +61,7 @@ import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ComparisonChain; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ComparisonChain; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficRoutes.java b/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficRoutes.java index 8a9cdcecf4ef4..d389819062b24 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficRoutes.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficRoutes.java @@ -71,7 +71,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.Instant; import org.joda.time.format.DateTimeFormat; diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/DataTokenization.java b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/DataTokenization.java index e4d2d6342e608..b27479e7fc9bc 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/DataTokenization.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/DataTokenization.java @@ -19,7 +19,7 @@ import static org.apache.beam.examples.complete.datatokenization.utils.DurationUtils.parseDuration; import static org.apache.beam.examples.complete.datatokenization.utils.SchemasUtils.DEADLETTER_SCHEMA; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.nio.charset.StandardCharsets; diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/DataProtectors.java b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/DataProtectors.java index 8af71372e306b..287f8be71d5c2 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/DataProtectors.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/DataProtectors.java @@ -51,7 +51,7 @@ import org.apache.beam.vendor.grpc.v1p54p0.com.google.gson.Gson; import org.apache.beam.vendor.grpc.v1p54p0.com.google.gson.JsonArray; import org.apache.beam.vendor.grpc.v1p54p0.com.google.gson.JsonObject; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; import org.apache.commons.io.IOUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpStatus; diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/JsonToBeamRow.java b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/JsonToBeamRow.java index 6ae023e692cd3..96a27e2b181f9 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/JsonToBeamRow.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/JsonToBeamRow.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; /** The {@link JsonToBeamRow} converts jsons string to beam rows. */ public class JsonToBeamRow extends PTransform, PCollection> { diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/SerializableFunctions.java b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/SerializableFunctions.java index 83417d19ee063..437388e7dc480 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/SerializableFunctions.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/SerializableFunctions.java @@ -21,7 +21,7 @@ import java.util.ArrayList; import org.apache.beam.examples.complete.datatokenization.utils.FailsafeElement; import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; /** The {@link SerializableFunctions} class to store static Serializable functions. */ public class SerializableFunctions { diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/CsvConverters.java b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/CsvConverters.java index b570d1b0c6ba4..76eb486e21b02 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/CsvConverters.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/CsvConverters.java @@ -18,7 +18,7 @@ package org.apache.beam.examples.complete.datatokenization.utils; import static org.apache.beam.examples.complete.datatokenization.utils.SchemasUtils.getGcsFileAsString; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import com.google.gson.JsonArray; @@ -52,7 +52,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.slf4j.Logger; diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/DurationUtils.java b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/DurationUtils.java index 83d3aea3acb5e..a70e44b060239 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/DurationUtils.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/DurationUtils.java @@ -17,8 +17,8 @@ */ package org.apache.beam.examples.complete.datatokenization.utils; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.Locale; import org.joda.time.DateTime; diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/ErrorConverters.java b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/ErrorConverters.java index e91ab6c97216a..01377add08580 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/ErrorConverters.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/ErrorConverters.java @@ -17,7 +17,7 @@ */ package org.apache.beam.examples.complete.datatokenization.utils; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.bigquery.model.TableRow; import com.google.auto.value.AutoValue; @@ -39,7 +39,7 @@ import org.apache.beam.sdk.values.PCollection.IsBounded; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTimeZone; import org.joda.time.Duration; diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/FailsafeElement.java b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/FailsafeElement.java index a4e22e8b463ab..daf719a8f830c 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/FailsafeElement.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/FailsafeElement.java @@ -19,7 +19,7 @@ import java.util.Objects; import org.apache.beam.sdk.coders.DefaultCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/SchemasUtils.java b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/SchemasUtils.java index 4543aa623b835..4f14eca53352e 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/SchemasUtils.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/utils/SchemasUtils.java @@ -18,7 +18,7 @@ package org.apache.beam.examples.complete.datatokenization.utils; import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils.fromTableSchema; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.services.bigquery.model.TableSchema; import java.io.IOException; @@ -37,8 +37,8 @@ import org.apache.beam.sdk.io.fs.ResourceId; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers; import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CharStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CharStreams; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java b/examples/java/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java index e0f56c81231e9..b8d81056a3e89 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java @@ -44,7 +44,7 @@ import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/game/StatefulTeamScore.java b/examples/java/src/main/java/org/apache/beam/examples/complete/game/StatefulTeamScore.java index 5e70d85ec6947..2315f1b811b69 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/game/StatefulTeamScore.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/game/StatefulTeamScore.java @@ -17,7 +17,7 @@ */ package org.apache.beam.examples.complete.game; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; import java.util.HashMap; import java.util.Map; @@ -42,7 +42,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.Instant; /** diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/Injector.java b/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/Injector.java index c46440517dfce..c6eb55d497b40 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/Injector.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/Injector.java @@ -30,7 +30,7 @@ import java.util.List; import java.util.Random; import org.apache.beam.examples.complete.game.utils.GameConstants; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * This is a generator that simulates usage data from a mobile game, and either publishes the data diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/InjectorUtils.java b/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/InjectorUtils.java index dbefed2b7cc30..d0a1f2872d5bb 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/InjectorUtils.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/InjectorUtils.java @@ -17,7 +17,7 @@ */ package org.apache.beam.examples.complete.game.injector; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; import com.google.api.client.googleapis.json.GoogleJsonResponseException; @@ -47,7 +47,7 @@ public static Pubsub getClient(final HttpTransport httpTransport, final JsonFact } if (credential.getClientAuthentication() != null) { System.out.println( - "\n***Warning! You are not using service account credentials to " + "\n***Error! You are not using service account credentials to " + "authenticate.\nYou need to use service account credentials for this example," + "\nsince user-level credentials do not have enough pubsub quota,\nand so you will run " + "out of PubSub quota very quickly.\nSee " diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/RetryHttpInitializerWrapper.java b/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/RetryHttpInitializerWrapper.java index c487157a798e1..d9fae936465fc 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/RetryHttpInitializerWrapper.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/RetryHttpInitializerWrapper.java @@ -17,7 +17,7 @@ */ package org.apache.beam.examples.complete.game.injector; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.client.auth.oauth2.Credential; import com.google.api.client.http.HttpBackOffIOExceptionHandler; diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/game/utils/WriteToText.java b/examples/java/src/main/java/org/apache/beam/examples/complete/game/utils/WriteToText.java index 68b077288715c..330769e0c79e1 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/game/utils/WriteToText.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/game/utils/WriteToText.java @@ -17,7 +17,7 @@ */ package org.apache.beam.examples.complete.game.utils; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.Serializable; import java.util.ArrayList; diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsub.java b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsub.java index 649b41d240eca..49aaeb5ac577a 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsub.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsub.java @@ -22,7 +22,7 @@ import static org.apache.beam.examples.complete.kafkatopubsub.kafka.consumer.Utils.getKafkaCredentialsFromVault; import static org.apache.beam.examples.complete.kafkatopubsub.kafka.consumer.Utils.isSslSpecified; import static org.apache.beam.examples.complete.kafkatopubsub.kafka.consumer.Utils.parseKafkaConsumerConfig; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.ArrayList; import java.util.Arrays; diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/transforms/FormatTransform.java b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/transforms/FormatTransform.java index 1dcf9e196221b..2d9089fcd29af 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/transforms/FormatTransform.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/transforms/FormatTransform.java @@ -37,8 +37,8 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.kafka.common.serialization.StringDeserializer; /** Different transformations over the processed data in the pipeline. */ diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/BigQueryTornadoes.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/BigQueryTornadoes.java index 4a7ea4a469398..b7ef94338d741 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/BigQueryTornadoes.java +++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/BigQueryTornadoes.java @@ -36,7 +36,7 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,15 +63,15 @@ * * See examples/java/README.md for instructions about how to configure different runners. * - *

    The BigQuery input table defaults to {@code clouddataflow-readonly:samples.weather_stations} - * and can be overridden with {@code --input}. + *

    The BigQuery input table defaults to {@code apache-beam-testing.samples.weather_stations} and + * can be overridden with {@code --input}. */ public class BigQueryTornadoes { private static final Logger LOG = LoggerFactory.getLogger(BigQueryTornadoes.class); // Default to using a 1000 row subset of the public weather station table publicdata:samples.gsod. private static final String WEATHER_SAMPLES_TABLE = - "clouddataflow-readonly:samples.weather_stations"; + "apache-beam-testing.samples.weather_stations"; /** * Examines each row in the input table. If a tornado was recorded in that sample, the month in diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/FilterExamples.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/FilterExamples.java index 1baefbcda101f..9187bb83d7da5 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/FilterExamples.java +++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/FilterExamples.java @@ -71,13 +71,13 @@ * * See examples/java/README.md for instructions about how to configure different runners. * - *

    The BigQuery input table defaults to {@code clouddataflow-readonly:samples.weather_stations} - * and can be overridden with {@code --input}. + *

    The BigQuery input table defaults to {@code apache-beam-testing.samples.weather_stations} and + * can be overridden with {@code --input}. */ public class FilterExamples { // Default to using a 1000 row subset of the public weather station table publicdata:samples.gsod. private static final String WEATHER_SAMPLES_TABLE = - "clouddataflow-readonly:samples.weather_stations"; + "apache-beam-testing.samples.weather_stations"; static final Logger LOG = Logger.getLogger(FilterExamples.class.getName()); static final int MONTH_TO_FILTER = 7; diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/JoinExamples.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/JoinExamples.java index 66980f4ce9f8c..f78df0c094616 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/JoinExamples.java +++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/JoinExamples.java @@ -58,7 +58,7 @@ public class JoinExamples { // A 1000-row sample of the GDELT data here: gdelt-bq:full.events. - private static final String GDELT_EVENTS_TABLE = "clouddataflow-readonly:samples.gdelt_sample"; + private static final String GDELT_EVENTS_TABLE = "apache-beam-testing.samples.gdelt_sample"; // A table that maps country codes to country names. private static final String COUNTRY_CODES = "gdelt-bq:full.crosswalk_geocountrycodetohuman"; diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/MaxPerKeyExamples.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/MaxPerKeyExamples.java index dec3b70a66675..8760d562d0403 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/MaxPerKeyExamples.java +++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/MaxPerKeyExamples.java @@ -59,13 +59,13 @@ * * See examples/java/README.md for instructions about how to configure different runners. * - *

    The BigQuery input table defaults to {@code clouddataflow-readonly:samples.weather_stations } - * and can be overridden with {@code --input}. + *

    The BigQuery input table defaults to {@code apache-beam-testing.samples.weather_stations } and + * can be overridden with {@code --input}. */ public class MaxPerKeyExamples { // Default to using a 1000 row subset of the public weather station table publicdata:samples.gsod. private static final String WEATHER_SAMPLES_TABLE = - "clouddataflow-readonly:samples.weather_stations"; + "apache-beam-testing.samples.weather_stations"; /** * Examines each row (weather reading) in the input table. Output the month of the reading, and diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/MinimalBigQueryTornadoes.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/MinimalBigQueryTornadoes.java index 43654ea4a05ba..60b5c02a5f460 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/MinimalBigQueryTornadoes.java +++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/MinimalBigQueryTornadoes.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,14 +58,14 @@ * *

    Concepts: Reading/writing BigQuery; counting a PCollection; user-defined PTransforms * - *

    The BigQuery input is taken from {@code clouddataflow-readonly:samples.weather_stations} + *

    The BigQuery input is taken from {@code apache-beam-testing.samples.weather_stations} */ public class MinimalBigQueryTornadoes { private static final Logger LOG = LoggerFactory.getLogger(MinimalBigQueryTornadoes.class); // Use a 1000 row subset of the public weather station table publicdata:samples.gsod. private static final String WEATHER_SAMPLES_TABLE = - "clouddataflow-readonly:samples.weather_stations"; + "apache-beam-testing.samples.weather_stations"; /** * Examines each row in the input table. If a tornado was recorded in that sample, the month in diff --git a/examples/java/src/main/java/org/apache/beam/examples/multilanguage/SklearnMnistClassification.java b/examples/java/src/main/java/org/apache/beam/examples/multilanguage/SklearnMnistClassification.java index 4668ec1b41efe..9840830bd7b15 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/multilanguage/SklearnMnistClassification.java +++ b/examples/java/src/main/java/org/apache/beam/examples/multilanguage/SklearnMnistClassification.java @@ -37,7 +37,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; /** * An example Java Multi-language pipeline that Performs image classification on handwritten digits diff --git a/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java b/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java index fe719bf80118c..3ca431455a06e 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java +++ b/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java @@ -108,7 +108,7 @@ import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; import org.apache.beam.sdk.values.ValueInSingleWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.joda.time.Instant; import org.joda.time.format.DateTimeFormat; @@ -172,7 +172,7 @@ public static void modelBigQueryIO( Pipeline p, String writeProject, String writeDataset, String writeTable) { { // [START BigQueryTableSpec] - String tableSpec = "clouddataflow-readonly:samples.weather_stations"; + String tableSpec = "apache-beam-testing.samples.weather_stations"; // [END BigQueryTableSpec] } @@ -212,7 +212,7 @@ public static void modelBigQueryIO( } { - String tableSpec = "clouddataflow-readonly:samples.weather_stations"; + String tableSpec = "apache-beam-testing.samples.weather_stations"; // [START BigQueryReadTable] PCollection maxTemperatures = p.apply(BigQueryIO.readTableRows().from(tableSpec)) @@ -224,7 +224,7 @@ public static void modelBigQueryIO( } { - String tableSpec = "clouddataflow-readonly:samples.weather_stations"; + String tableSpec = "apache-beam-testing.samples.weather_stations"; // [START BigQueryReadFunction] PCollection maxTemperatures = p.apply( @@ -242,7 +242,7 @@ public static void modelBigQueryIO( BigQueryIO.read( (SchemaAndRecord elem) -> (Double) elem.getRecord().get("max_temperature")) .fromQuery( - "SELECT max_temperature FROM [clouddataflow-readonly:samples.weather_stations]") + "SELECT max_temperature FROM [apache-beam-testing.samples.weather_stations]") .withCoder(DoubleCoder.of())); // [END BigQueryReadQuery] } @@ -280,7 +280,7 @@ public static void modelBigQueryIO( // [END BigQuerySchemaJson] { - String tableSpec = "clouddataflow-readonly:samples.weather_stations"; + String tableSpec = "apache-beam-testing.samples.weather_stations"; if (!writeProject.isEmpty() && !writeDataset.isEmpty() && !writeTable.isEmpty()) { tableSpec = writeProject + ":" + writeDataset + "." + writeTable; } @@ -403,7 +403,7 @@ public WeatherData(long year, long month, long day, double maxTemp) { }) .fromQuery( "SELECT year, month, day, max_temperature " - + "FROM [clouddataflow-readonly:samples.weather_stations] " + + "FROM [apache-beam-testing.samples.weather_stations] " + "WHERE year BETWEEN 2007 AND 2009") .withCoder(AvroCoder.of(WeatherData.class))); @@ -461,7 +461,7 @@ public TableSchema getSchema(Long destination) { .withWriteDisposition(WriteDisposition.WRITE_TRUNCATE)); // [END BigQueryWriteDynamicDestinations] - String tableSpec = "clouddataflow-readonly:samples.weather_stations"; + String tableSpec = "apache-beam-testing.samples.weather_stations"; if (!writeProject.isEmpty() && !writeDataset.isEmpty() && !writeTable.isEmpty()) { tableSpec = writeProject + ":" + writeDataset + "." + writeTable + "_partitioning"; } @@ -1357,27 +1357,28 @@ private Service initializeService() { } public interface Service { - List readNextRecords(long position) throws ThrottlingException; + List readNextRecords(long position) throws ThrottlingException; } - public interface Record { + public interface RecordPosition { long getPosition(); } // [START SDF_UserInitiatedCheckpoint] @ProcessElement public ProcessContinuation processElement( - RestrictionTracker tracker, OutputReceiver outputReceiver) { + RestrictionTracker tracker, + OutputReceiver outputReceiver) { long currentPosition = tracker.currentRestriction().getFrom(); Service service = initializeService(); try { while (true) { - List records = service.readNextRecords(currentPosition); + List records = service.readNextRecords(currentPosition); if (records.isEmpty()) { // Return a short delay if there is no data to process at the moment. return ProcessContinuation.resume().withResumeDelay(Duration.standardSeconds(10)); } - for (Record record : records) { + for (RecordPosition record : records) { if (!tracker.tryClaim(record.getPosition())) { return ProcessContinuation.stop(); } diff --git a/examples/java/src/main/java/org/apache/beam/examples/subprocess/kernel/SubProcessCommandLineArgs.java b/examples/java/src/main/java/org/apache/beam/examples/subprocess/kernel/SubProcessCommandLineArgs.java index aa06a631c03a4..3ae76cc67ec0d 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/subprocess/kernel/SubProcessCommandLineArgs.java +++ b/examples/java/src/main/java/org/apache/beam/examples/subprocess/kernel/SubProcessCommandLineArgs.java @@ -18,7 +18,7 @@ package org.apache.beam.examples.subprocess.kernel; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; /** Parameters to the sub-process, has tuple of ordinal position and the value. */ @SuppressWarnings({ diff --git a/examples/java/src/main/java/org/apache/beam/examples/subprocess/utils/CallingSubProcessUtils.java b/examples/java/src/main/java/org/apache/beam/examples/subprocess/utils/CallingSubProcessUtils.java index 3acd1eed33662..0d5f978fab9a0 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/subprocess/utils/CallingSubProcessUtils.java +++ b/examples/java/src/main/java/org/apache/beam/examples/subprocess/utils/CallingSubProcessUtils.java @@ -22,7 +22,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Semaphore; import org.apache.beam.examples.subprocess.configuration.SubProcessConfiguration; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/examples/java/src/test/java/org/apache/beam/examples/DebuggingWordCountTest.java b/examples/java/src/test/java/org/apache/beam/examples/DebuggingWordCountTest.java index a7675603ed366..280c684b34660 100644 --- a/examples/java/src/test/java/org/apache/beam/examples/DebuggingWordCountTest.java +++ b/examples/java/src/test/java/org/apache/beam/examples/DebuggingWordCountTest.java @@ -21,7 +21,7 @@ import java.nio.charset.StandardCharsets; import org.apache.beam.examples.DebuggingWordCount.WordCountOptions; import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -44,10 +44,8 @@ private String getFilePath(String filePath) { public void testDebuggingWordCount() throws Exception { File inputFile = tmpFolder.newFile(); File outputFile = tmpFolder.newFile(); - Files.write( - "stomach secret Flourish message Flourish here Flourish", - inputFile, - StandardCharsets.UTF_8); + Files.asCharSink(inputFile, StandardCharsets.UTF_8) + .write("stomach secret Flourish message Flourish here Flourish"); WordCountOptions options = TestPipeline.testingPipelineOptions().as(WordCountOptions.class); options.setInputFile(getFilePath(inputFile.getAbsolutePath())); options.setOutput(getFilePath(outputFile.getAbsolutePath())); diff --git a/examples/java/src/test/java/org/apache/beam/examples/MinimalWordCountTest.java b/examples/java/src/test/java/org/apache/beam/examples/MinimalWordCountTest.java index 8f867481e0c63..28bffefd5d185 100644 --- a/examples/java/src/test/java/org/apache/beam/examples/MinimalWordCountTest.java +++ b/examples/java/src/test/java/org/apache/beam/examples/MinimalWordCountTest.java @@ -34,7 +34,7 @@ import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java index bfb11ae014624..fa25d72cb22a6 100644 --- a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java +++ b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java @@ -45,9 +45,9 @@ import org.apache.beam.sdk.util.NumberedShardedFile; import org.apache.beam.sdk.util.ShardedFile; import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.hamcrest.Description; import org.hamcrest.TypeSafeMatcher; import org.joda.time.Duration; diff --git a/examples/java/src/test/java/org/apache/beam/examples/complete/datatokenization/DataTokenizationTest.java b/examples/java/src/test/java/org/apache/beam/examples/complete/datatokenization/DataTokenizationTest.java index e0ba847cfb847..2a00d53358ab3 100644 --- a/examples/java/src/test/java/org/apache/beam/examples/complete/datatokenization/DataTokenizationTest.java +++ b/examples/java/src/test/java/org/apache/beam/examples/complete/datatokenization/DataTokenizationTest.java @@ -46,8 +46,8 @@ import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Resources; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Resources; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; diff --git a/examples/java/src/test/java/org/apache/beam/examples/complete/game/LeaderBoardTest.java b/examples/java/src/test/java/org/apache/beam/examples/complete/game/LeaderBoardTest.java index fc449909e139f..f6293f6931334 100644 --- a/examples/java/src/test/java/org/apache/beam/examples/complete/game/LeaderBoardTest.java +++ b/examples/java/src/test/java/org/apache/beam/examples/complete/game/LeaderBoardTest.java @@ -36,7 +36,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/examples/java/src/test/java/org/apache/beam/examples/complete/game/UserScoreTest.java b/examples/java/src/test/java/org/apache/beam/examples/complete/game/UserScoreTest.java index 035c71047fa62..04aa122054bd4 100644 --- a/examples/java/src/test/java/org/apache/beam/examples/complete/game/UserScoreTest.java +++ b/examples/java/src/test/java/org/apache/beam/examples/complete/game/UserScoreTest.java @@ -33,7 +33,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/examples/java/src/test/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsubE2ETest.java b/examples/java/src/test/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsubE2ETest.java index 8e44fc2258dd1..7639064a27d88 100644 --- a/examples/java/src/test/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsubE2ETest.java +++ b/examples/java/src/test/java/org/apache/beam/examples/complete/kafkatopubsub/KafkaToPubsubE2ETest.java @@ -34,7 +34,7 @@ import org.apache.beam.sdk.io.gcp.pubsub.TestPubsub; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerConfig; diff --git a/examples/java/src/test/java/org/apache/beam/examples/cookbook/BigQueryTornadoesTest.java b/examples/java/src/test/java/org/apache/beam/examples/cookbook/BigQueryTornadoesTest.java index a67273d158216..2bd37a3caa52b 100644 --- a/examples/java/src/test/java/org/apache/beam/examples/cookbook/BigQueryTornadoesTest.java +++ b/examples/java/src/test/java/org/apache/beam/examples/cookbook/BigQueryTornadoesTest.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/examples/java/src/test/java/org/apache/beam/examples/cookbook/MaxPerKeyExamplesTest.java b/examples/java/src/test/java/org/apache/beam/examples/cookbook/MaxPerKeyExamplesTest.java index 82e31f1a60560..5c32f36660d65 100644 --- a/examples/java/src/test/java/org/apache/beam/examples/cookbook/MaxPerKeyExamplesTest.java +++ b/examples/java/src/test/java/org/apache/beam/examples/cookbook/MaxPerKeyExamplesTest.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/examples/java/src/test/java/org/apache/beam/examples/cookbook/MinimalBigQueryTornadoesTest.java b/examples/java/src/test/java/org/apache/beam/examples/cookbook/MinimalBigQueryTornadoesTest.java index 6591f03adb370..7e922bc879652 100644 --- a/examples/java/src/test/java/org/apache/beam/examples/cookbook/MinimalBigQueryTornadoesTest.java +++ b/examples/java/src/test/java/org/apache/beam/examples/cookbook/MinimalBigQueryTornadoesTest.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/examples/java/src/test/java/org/apache/beam/examples/cookbook/TriggerExampleTest.java b/examples/java/src/test/java/org/apache/beam/examples/cookbook/TriggerExampleTest.java index 3108c476c983b..8f076a9d8d891 100644 --- a/examples/java/src/test/java/org/apache/beam/examples/cookbook/TriggerExampleTest.java +++ b/examples/java/src/test/java/org/apache/beam/examples/cookbook/TriggerExampleTest.java @@ -36,8 +36,8 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/examples/java/src/test/java/org/apache/beam/examples/snippets/SnippetsTest.java b/examples/java/src/test/java/org/apache/beam/examples/snippets/SnippetsTest.java index bbcd3a827adef..1791c361ad0e4 100644 --- a/examples/java/src/test/java/org/apache/beam/examples/snippets/SnippetsTest.java +++ b/examples/java/src/test/java/org/apache/beam/examples/snippets/SnippetsTest.java @@ -39,7 +39,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Assert; diff --git a/examples/java/src/test/java/org/apache/beam/examples/subprocess/ExampleEchoPipelineTest.java b/examples/java/src/test/java/org/apache/beam/examples/subprocess/ExampleEchoPipelineTest.java index 87b919aaca415..01055d9658a9e 100644 --- a/examples/java/src/test/java/org/apache/beam/examples/subprocess/ExampleEchoPipelineTest.java +++ b/examples/java/src/test/java/org/apache/beam/examples/subprocess/ExampleEchoPipelineTest.java @@ -44,7 +44,7 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/examples/kotlin/build.gradle b/examples/kotlin/build.gradle index 6fb3ef64e2827..98258401d5881 100644 --- a/examples/kotlin/build.gradle +++ b/examples/kotlin/build.gradle @@ -50,7 +50,7 @@ def kotlin_version = "1.6.10" dependencies { implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") // Add the dependency that sdks:java:core that is marked as provided implementation library.java.hamcrest diff --git a/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/common/ExampleUtils.kt b/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/common/ExampleUtils.kt index 7fd9aeb8549db..dc304f6a029fd 100644 --- a/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/common/ExampleUtils.kt +++ b/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/common/ExampleUtils.kt @@ -38,10 +38,10 @@ import org.apache.beam.sdk.options.PipelineOptions import org.apache.beam.sdk.util.BackOffUtils import org.apache.beam.sdk.util.FluentBackoff import org.apache.beam.sdk.util.Sleeper -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles import org.joda.time.Duration import java.io.IOException import java.util.concurrent.TimeUnit diff --git a/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/common/WriteOneFilePerWindow.kt b/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/common/WriteOneFilePerWindow.kt index 601fe2191d0b5..94c5022b51339 100644 --- a/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/common/WriteOneFilePerWindow.kt +++ b/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/common/WriteOneFilePerWindow.kt @@ -30,7 +30,7 @@ import org.apache.beam.sdk.transforms.windowing.IntervalWindow import org.apache.beam.sdk.transforms.windowing.PaneInfo import org.apache.beam.sdk.values.PCollection import org.apache.beam.sdk.values.PDone -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull import org.joda.time.format.ISODateTimeFormat /** diff --git a/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/cookbook/BigQueryTornadoes.kt b/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/cookbook/BigQueryTornadoes.kt index 59637d8002c2f..ec56bc6599704 100644 --- a/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/cookbook/BigQueryTornadoes.kt +++ b/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/cookbook/BigQueryTornadoes.kt @@ -31,7 +31,7 @@ import org.apache.beam.sdk.transforms.PTransform import org.apache.beam.sdk.transforms.ParDo import org.apache.beam.sdk.values.KV import org.apache.beam.sdk.values.PCollection -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists /** @@ -60,12 +60,12 @@ import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists * See examples/java/README.md for instructions about how to configure different runners. * * - * The BigQuery input table defaults to `clouddataflow-readonly:samples.weather_stations` + * The BigQuery input table defaults to `apache-beam-testing.samples.weather_stations` * and can be overridden with `--input`. */ object BigQueryTornadoes { // Default to using a 1000 row subset of the public weather station table publicdata:samples.gsod. - private const val WEATHER_SAMPLES_TABLE = "clouddataflow-readonly:samples.weather_stations" + private const val WEATHER_SAMPLES_TABLE = "apache-beam-testing.samples.weather_stations" /** * Examines each row in the input table. If a tornado was recorded in that sample, the month in diff --git a/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/cookbook/FilterExamples.kt b/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/cookbook/FilterExamples.kt index e8c670e4d0fe2..2625f5bfec107 100644 --- a/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/cookbook/FilterExamples.kt +++ b/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/cookbook/FilterExamples.kt @@ -65,12 +65,12 @@ import java.util.logging.Logger * See examples/kotlin/README.md for instructions about how to configure different runners. * * - * The BigQuery input table defaults to `clouddataflow-readonly:samples.weather_stations` + * The BigQuery input table defaults to `apache-beam-testing.samples.weather_stations` * and can be overridden with `--input`. */ object FilterExamples { // Default to using a 1000 row subset of the public weather station table publicdata:samples.gsod. - private const val WEATHER_SAMPLES_TABLE = "clouddataflow-readonly:samples.weather_stations" + private const val WEATHER_SAMPLES_TABLE = "apache-beam-testing.samples.weather_stations" internal val LOG = Logger.getLogger(FilterExamples::class.java.name) internal const val MONTH_TO_FILTER = 7 diff --git a/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/cookbook/JoinExamples.kt b/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/cookbook/JoinExamples.kt index 3b7f3c4c35821..2f2215e1d96ac 100644 --- a/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/cookbook/JoinExamples.kt +++ b/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/cookbook/JoinExamples.kt @@ -60,7 +60,7 @@ import org.apache.beam.sdk.values.TupleTag object JoinExamples { // A 1000-row sample of the GDELT data here: gdelt-bq:full.events. - private const val GDELT_EVENTS_TABLE = "clouddataflow-readonly:samples.gdelt_sample" + private const val GDELT_EVENTS_TABLE = "apache-beam-testing.samples.gdelt_sample" // A table that maps country codes to country names. private const val COUNTRY_CODES = "gdelt-bq:full.crosswalk_geocountrycodetohuman" diff --git a/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/cookbook/MaxPerKeyExamples.kt b/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/cookbook/MaxPerKeyExamples.kt index 74d392de4e29d..11418d3933cf0 100644 --- a/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/cookbook/MaxPerKeyExamples.kt +++ b/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/cookbook/MaxPerKeyExamples.kt @@ -60,12 +60,12 @@ import java.util.ArrayList * See examples/java/README.md for instructions about how to configure different runners. * * - * The BigQuery input table defaults to `clouddataflow-readonly:samples.weather_stations ` + * The BigQuery input table defaults to `apache-beam-testing.samples.weather_stations ` * and can be overridden with `--input`. */ object MaxPerKeyExamples { // Default to using a 1000 row subset of the public weather station table publicdata:samples.gsod. - private const val WEATHER_SAMPLES_TABLE = "clouddataflow-readonly:samples.weather_stations" + private const val WEATHER_SAMPLES_TABLE = "apache-beam-testing.samples.weather_stations" /** * Examines each row (weather reading) in the input table. Output the month of the reading, and diff --git a/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/snippets/Snippets.kt b/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/snippets/Snippets.kt index 2ba7b3742e166..d2f58c215a561 100644 --- a/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/snippets/Snippets.kt +++ b/examples/kotlin/src/main/java/org/apache/beam/examples/kotlin/snippets/Snippets.kt @@ -84,7 +84,7 @@ object Snippets { pipeline: Pipeline, writeProject: String = "", writeDataset: String = "", writeTable: String = "") { run { // [START BigQueryTableSpec] - val tableSpec = "clouddataflow-readonly:samples.weather_stations" + val tableSpec = "apache-beam-testing.samples.weather_stations" // [END BigQueryTableSpec] } @@ -104,7 +104,7 @@ object Snippets { } run { - val tableSpec = "clouddataflow-readonly:samples.weather_stations" + val tableSpec = "apache-beam-testing.samples.weather_stations" // [START BigQueryReadTable] val maxTemperatures = pipeline.apply(BigQueryIO.readTableRows().from(tableSpec)) // Each row is of type TableRow @@ -118,7 +118,7 @@ object Snippets { } run { - val tableSpec = "clouddataflow-readonly:samples.weather_stations" + val tableSpec = "apache-beam-testing.samples.weather_stations" // [START BigQueryReadFunction] val maxTemperatures = pipeline.apply( BigQueryIO.read { it.record["max_temperature"] as Double? } @@ -132,7 +132,7 @@ object Snippets { val maxTemperatures = pipeline.apply( BigQueryIO.read { it.record["max_temperature"] as Double? } .fromQuery( - "SELECT max_temperature FROM [clouddataflow-readonly:samples.weather_stations]") + "SELECT max_temperature FROM [apache-beam-testing.samples.weather_stations]") .withCoder(DoubleCoder.of())) // [END BigQueryReadQuery] } @@ -167,7 +167,7 @@ object Snippets { // [END BigQuerySchemaJson] run { - var tableSpec = "clouddataflow-readonly:samples.weather_stations" + var tableSpec = "apache-beam-testing.samples.weather_stations" if (writeProject.isNotEmpty() && writeDataset.isNotEmpty() && writeTable.isNotEmpty()) { tableSpec = "$writeProject:$writeDataset.$writeTable" } @@ -259,7 +259,7 @@ object Snippets { } .fromQuery(""" SELECT year, month, day, max_temperature - FROM [clouddataflow-readonly:samples.weather_stations] + FROM [apache-beam-testing.samples.weather_stations] WHERE year BETWEEN 2007 AND 2009 """.trimIndent()) .withCoder(AvroCoder.of(WeatherData::class.java))) @@ -297,7 +297,7 @@ object Snippets { .withWriteDisposition(WriteDisposition.WRITE_TRUNCATE)) // [END BigQueryWriteDynamicDestinations] - var tableSpec = "clouddataflow-readonly:samples.weather_stations" + var tableSpec = "apache-beam-testing.samples.weather_stations" if (writeProject.isNotEmpty() && writeDataset.isNotEmpty() && writeTable.isNotEmpty()) { tableSpec = "$writeProject:$writeDataset.${writeTable}_partitioning" } diff --git a/examples/kotlin/src/test/java/org/apache/beam/examples/kotlin/cookbook/MaxPerKeyExamplesTest.kt b/examples/kotlin/src/test/java/org/apache/beam/examples/kotlin/cookbook/MaxPerKeyExamplesTest.kt index c9f4ec3e0b05f..7995d9c1c795e 100644 --- a/examples/kotlin/src/test/java/org/apache/beam/examples/kotlin/cookbook/MaxPerKeyExamplesTest.kt +++ b/examples/kotlin/src/test/java/org/apache/beam/examples/kotlin/cookbook/MaxPerKeyExamplesTest.kt @@ -24,7 +24,7 @@ import org.apache.beam.sdk.testing.ValidatesRunner import org.apache.beam.sdk.transforms.Create import org.apache.beam.sdk.transforms.ParDo import org.apache.beam.sdk.values.KV -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList import org.junit.Rule import org.junit.Test import org.junit.experimental.categories.Category diff --git a/examples/multi-language/build.gradle b/examples/multi-language/build.gradle index b266faeb8f171..71e2ba543a197 100644 --- a/examples/multi-language/build.gradle +++ b/examples/multi-language/build.gradle @@ -32,13 +32,13 @@ description = "Apache Beam :: Examples :: Multi Language" ext.summary = "Java Classes for Multi-language Examples" dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") runtimeOnly project(path: ":examples:java") runtimeOnly project(path: ":runners:direct-java", configuration: "shadow") runtimeOnly project(path: ":runners:google-cloud-dataflow-java") runtimeOnly project(path: ":runners:portability:java") - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(":sdks:java:expansion-service") permitUnusedDeclared project(":sdks:java:expansion-service") // BEAM-11761 } diff --git a/examples/multi-language/src/main/java/org/apache/beam/examples/multilanguage/JavaCountRegistrar.java b/examples/multi-language/src/main/java/org/apache/beam/examples/multilanguage/JavaCountRegistrar.java index 43e4a396f4cf9..f40856f4c5e2f 100644 --- a/examples/multi-language/src/main/java/org/apache/beam/examples/multilanguage/JavaCountRegistrar.java +++ b/examples/multi-language/src/main/java/org/apache/beam/examples/multilanguage/JavaCountRegistrar.java @@ -21,7 +21,7 @@ import java.util.Map; import org.apache.beam.sdk.expansion.ExternalTransformRegistrar; import org.apache.beam.sdk.transforms.ExternalTransformBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; @AutoService(ExternalTransformRegistrar.class) public class JavaCountRegistrar implements ExternalTransformRegistrar { diff --git a/examples/multi-language/src/main/java/org/apache/beam/examples/multilanguage/JavaPrefixRegistrar.java b/examples/multi-language/src/main/java/org/apache/beam/examples/multilanguage/JavaPrefixRegistrar.java index 45d5e44bf3acb..ed11f28283c41 100644 --- a/examples/multi-language/src/main/java/org/apache/beam/examples/multilanguage/JavaPrefixRegistrar.java +++ b/examples/multi-language/src/main/java/org/apache/beam/examples/multilanguage/JavaPrefixRegistrar.java @@ -21,7 +21,7 @@ import java.util.Map; import org.apache.beam.sdk.expansion.ExternalTransformRegistrar; import org.apache.beam.sdk.transforms.ExternalTransformBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; @AutoService(ExternalTransformRegistrar.class) public class JavaPrefixRegistrar implements ExternalTransformRegistrar { diff --git a/examples/notebooks/README.md b/examples/notebooks/README.md index 34505d2824218..381e7d640c6f4 100644 --- a/examples/notebooks/README.md +++ b/examples/notebooks/README.md @@ -19,15 +19,18 @@ # Interactive Python Notebooks -## Adding a notebooks -1. Import an .ipynb file or create a new notebook on [Google Colab](https://colab.research.google.com). -1. In the upper-left corner menu, go to *File* -> *Save a copy in Github...* +Thank you for choosing to contribute an example! This will help both the Beam Developer community as well as the Beam User +community. + +## Adding a notebook +1. Import an .ipynb file or create a new notebook on [Google Colab](https://colab.research.google.com). +2. In the upper-left corner menu, go to *File* -> *Save a copy in Github...* > You'll be asked to authenticate on GitHub. -1. Select the *Repository* as `your-username/beam` and the *Branch* you're working on, not master! -1. Set the file path to somewhere inside `examples/notebooks/`, e.g. `examples/notebooks/get-started/try-apache-beam-py.ipynb`. +3. Select the *Repository* as `your-username/beam` and the *Branch* you're working on, not master! +4. Set the file path to somewhere inside `examples/notebooks/`, e.g. `examples/notebooks/get-started/try-apache-beam-py.ipynb`. > You can leave the commit message as the default value, we'll squash all the commits into a single one at the end anyways. -1. Make sure the *Include a link to Colaboratory* is checked. -1. Pull the latest changes from the remote branch. +5. Make sure the *Include a link to Colaboratory* is checked. +6. Pull the latest changes from the remote branch. ```sh git pull ``` @@ -36,12 +39,12 @@ git checkout -- file1 file2 .. fileN git pull ``` -1. Repeat for all the notebooks you want to add. -1. From the project root directory, patch the Notebooks to point to the `master` branch instead of the local branch. +7. Repeat for all the notebooks you want to add. +8. From the project root directory, patch the Notebooks to point to the `master` branch instead of the local branch. ```sh python examples/notebooks/patch.py ``` -1. Squash all the commits into a single commit. +9. Squash all the commits into a single commit. ```sh git commit --all --amend ``` @@ -63,4 +66,13 @@ ``` > NOTE: in vim you can do this with `:2,$s/^pick/s/g` - Finally, your editor will open again. All the commit messages will be visible, delete and reword as necessary to leave only one uncommented commit message. After closing your editor all your commits should be squashed :) \ No newline at end of file + Finally, your editor will open again. All the commit messages will be visible, delete and reword as necessary to leave only one uncommented commit message. After closing your editor all your commits should be squashed :) + +## Notebook Example Guidelines + +1. Use [InteractiveRunner](https://cloud.google.com/dataflow/docs/guides/interactive-pipeline-development) and [DirectRunner](https://beam.apache.org/documentation/runners/direct/) as much as possible, to maintain Beam's vision of being runner-agnostic. +2. If you must use another runner or external technologies, document all commands and instructions needed for authentication, etc. +3. Document steps to import external files onto Colab, if needed. +4. Refrain from having too many code cells without explanation. Remember, this code should be able to be understood by someone with none or very limited experience in Beam! + +*Thanks again for choosing to contribute an example!* diff --git a/examples/notebooks/beam-ml/README.md b/examples/notebooks/beam-ml/README.md index cc9e9401b3df4..0ae937e9e284f 100644 --- a/examples/notebooks/beam-ml/README.md +++ b/examples/notebooks/beam-ml/README.md @@ -54,15 +54,25 @@ This section contains the following example notebooks. * [Apache Beam RunInference for scikit-learn](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/run_inference_sklearn.ipynb) * [Apache Beam RunInference with TensorFlow](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/run_inference_tensorflow.ipynb) * [Use RunInference with a model from TensorFlow Hub](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/run_inference_with_tensorflow_hub.ipynb) +* [Apache Beam RunInference with Hugging Face](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/run_inference_huggingface.ipynb) * [Apache Beam RunInference with XGBoost](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/run_inference_xgboost.ipynb) * [Use RunInference with TFX](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/run_inference_tensorflow_with_tfx.ipynb) +* [Use RunInference with a remotely deployed Vertex AI endpoint](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/run_inference_vertex_ai.ipynb) * [Use RunInference in Apache Beam](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/run_inference_pytorch_tensorflow_sklearn.ipynb) +* [Use RunInference with a LLM](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/run_inference_generative_ai.ipynb) +* [Use RunInference with Beam's windowing semantics](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/run_inference_windowing.ipynb) ### Custom inference * [Bring your own machine learning (ML) model to Apache Beam RunInference](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/run_custom_inference.ipynb) * [Remote inference](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/custom_remote_inference.ipynb) +### Machine Learning Use Cases + +* [Image processing with Apache Beam](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/image_processing_tensorflow.ipynb) +* [Natural language processing with Apache Beam](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/nlp_tensorflow_streaming.ipynb) +* [Speech emotion recognition with Apache Beam](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/speech_emotion_tensorflow.ipynb) + ### Automatic Model Refresh * [Update your model in production](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/automatic_model_refresh.ipynb) @@ -70,6 +80,7 @@ This section contains the following example notebooks. ### Multi-model pipelines * [Ensemble model using an image captioning and ranking](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/run_inference_multi_model.ipynb) +* [Run ML inference with multiple differently-trained models](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/per_key_models.ipynb) ### Model Evaluation @@ -77,4 +88,5 @@ This section contains the following example notebooks. ### Data processing +* [Preprocess data with MLTransform](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/mltransform_basic.ipynb) * [Preprocessing with the Apache Beam DataFrames API](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/dataframe_api_preprocessing.ipynb) diff --git a/examples/notebooks/beam-ml/automatic_model_refresh.ipynb b/examples/notebooks/beam-ml/automatic_model_refresh.ipynb index 67fe51af12530..3bafa4f078878 100644 --- a/examples/notebooks/beam-ml/automatic_model_refresh.ipynb +++ b/examples/notebooks/beam-ml/automatic_model_refresh.ipynb @@ -1,605 +1,530 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, - "cells": [{ - "cell_type": "code", - "source": [ - "# @title ###### Licensed to the Apache Software Foundation (ASF), Version 2.0 (the \"License\")\n", - "\n", - "# Licensed to the Apache Software Foundation (ASF) under one\n", - "# or more contributor license agreements. See the NOTICE file\n", - "# distributed with this work for additional information\n", - "# regarding copyright ownership. The ASF licenses this file\n", - "# to you under the Apache License, Version 2.0 (the\n", - "# \"License\"); you may not use this file except in compliance\n", - "# with the License. You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing,\n", - "# software distributed under the License is distributed on an\n", - "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n", - "# KIND, either express or implied. See the License for the\n", - "# specific language governing permissions and limitations\n", - "# under the License" - ], - "metadata": { - "cellView": "form", - "id": "OsFaZscKSPvo" - }, - "execution_count": null, - "outputs": [{ - "output_type": "stream", - "name": "stdout", - "text": [ - "\n" - ] - }] - }, - { - "cell_type": "markdown", - "source": [ - "# Update ML models in running pipelines\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    \n" - ], - "metadata": { - "id": "ZUSiAR62SgO8" - } - }, - { - "cell_type": "markdown", - "source": [ - "This notebook demonstrates how to perform automatic model updates without stopping your Apache Beam pipeline.\n", - "You can use side inputs to update your model in real time, even while the Apache Beam pipeline is running. The side input is passed in a `ModelHandler` configuration object. You can update the model either by leveraging one of Apache Beam's provided patterns, such as the `WatchFilePattern`, or by configuring a custom side input `PCollection` that defines the logic for the model update.\n", - "\n", - "The pipeline in this notebook uses a RunInference `PTransform` with TensorFlow machine learning (ML) models to run inference on images. To update the model, it uses a side input `PCollection` that emits `ModelMetadata`.\n", - "For more information about side inputs, see the [Side inputs](https://beam.apache.org/documentation/programming-guide/#side-inputs) section in the Apache Beam Programming Guide.\n", - "\n", - "This example uses `WatchFilePattern` as a side input. `WatchFilePattern` is used to watch for file updates that match the `file_pattern` based on timestamps. It emits the latest `ModelMetadata`, which is used in the RunInference `PTransform` to automatically update the ML model without stopping the Apache Beam pipeline.\n" - ], - "metadata": { - "id": "tBtqF5UpKJNZ" - } - }, - { - "cell_type": "markdown", - "source": [ - "## Before you begin\n", - "Install the dependencies required to run this notebook.\n", - "\n", - "To use RunInference with side inputs for automatic model updates, use Apache Beam version 2.46.0 or later." - ], - "metadata": { - "id": "SPuXFowiTpWx" - } - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "1RyTYsFEIOlA", - "outputId": "0e6b88a7-82d8-4d94-951c-046a9b8b7abb", - "colab": { - "base_uri": "https://localhost:8080/" - } - }, - "outputs": [{ - "output_type": "stream", - "name": "stdout", - "text": [ - "\n" - ] - }], - "source": [ - "!pip install apache_beam[gcp]>=2.46.0 --quiet\n", - "!pip install tensorflow\n", - "!pip install tensorflow_hub" - ] - }, - { - "cell_type": "code", - "source": [ - "# Imports required for the notebook.\n", - "import logging\n", - "import time\n", - "from typing import Iterable\n", - "from typing import Tuple\n", - "\n", - "import apache_beam as beam\n", - "from apache_beam.examples.inference.tensorflow_imagenet_segmentation import PostProcessor\n", - "from apache_beam.examples.inference.tensorflow_imagenet_segmentation import read_image\n", - "from apache_beam.ml.inference.base import PredictionResult\n", - "from apache_beam.ml.inference.base import RunInference\n", - "from apache_beam.ml.inference.tensorflow_inference import TFModelHandlerTensor\n", - "from apache_beam.ml.inference.utils import WatchFilePattern\n", - "from apache_beam.options.pipeline_options import GoogleCloudOptions\n", - "from apache_beam.options.pipeline_options import PipelineOptions\n", - "from apache_beam.options.pipeline_options import SetupOptions\n", - "from apache_beam.options.pipeline_options import StandardOptions\n", - "from apache_beam.transforms.periodicsequence import PeriodicImpulse\n", - "import numpy\n", - "from PIL import Image\n", - "import tensorflow as tf" - ], - "metadata": { - "id": "Rs4cwwNrIV9H" - }, - "execution_count": 2, - "outputs": [{ - "output_type": "stream", - "name": "stdout", - "text": [ - "\n" - ] - }] - }, - { - "cell_type": "code", - "source": [ - "# Authenticate to your Google Cloud account.\n", - "from google.colab import auth\n", - "auth.authenticate_user()" - ], - "metadata": { - "id": "jAKpPcmmGm03" - }, - "execution_count": 3, - "outputs": [{ - "output_type": "stream", - "name": "stdout", - "text": [ - "\n" - ] - }] - }, - { - "cell_type": "markdown", - "source": [ - "## Configure the runner\n", - "\n", - "This pipeline uses the Dataflow Runner. To run the pipeline, you need to complete the following tasks:\n", - "\n", - "* Ensure that you have all the required permissions to run the pipeline on Dataflow.\n", - "* Configure the pipeline options for the pipeline to run on Dataflow. Make sure the pipeline is using streaming mode.\n", - "\n", - "In the following code, replace `BUCKET_NAME` with the the name of your Cloud Storage bucket." - ], - "metadata": { - "id": "ORYNKhH3WQyP" - } - }, - { - "cell_type": "code", - "source": [ - "options = PipelineOptions()\n", - "options.view_as(StandardOptions).streaming = True\n", - "\n", - "# Provide required pipeline options for the Dataflow Runner.\n", - "options.view_as(StandardOptions).runner = \"DataflowRunner\"\n", - "\n", - "# Set the project to the default project in your current Google Cloud environment.\n", - "options.view_as(GoogleCloudOptions).project = 'your-project'\n", - "\n", - "# Set the Google Cloud region that you want to run Dataflow in.\n", - "options.view_as(GoogleCloudOptions).region = 'us-central1'\n", - "\n", - "# IMPORTANT: Replace BUCKET_NAME with the the name of your Cloud Storage bucket.\n", - "dataflow_gcs_location = \"gs://BUCKET_NAME/tmp/\"\n", - "\n", - "# The Dataflow staging location. This location is used to stage the Dataflow pipeline and the SDK binary.\n", - "options.view_as(GoogleCloudOptions).staging_location = '%s/staging' % dataflow_gcs_location\n", - "\n", - "# The Dataflow temp location. This location is used to store temporary files or intermediate results before outputting to the sink.\n", - "options.view_as(GoogleCloudOptions).temp_location = '%s/temp' % dataflow_gcs_location\n", - "\n" - ], - "metadata": { - "id": "wWjbnq6X-4uE" - }, - "execution_count": 4, - "outputs": [{ - "output_type": "stream", - "name": "stdout", - "text": [ - "\n" - ] - }] - }, - { - "cell_type": "markdown", - "source": [ - "Install the `tensorflow` and `tensorflow_hub` dependencies on Dataflow. Use the `requirements_file` pipeline option to pass these dependencies." - ], - "metadata": { - "id": "HTJV8pO2Wcw4" - } - }, - { - "cell_type": "code", - "source": [ - "# In a requirements file, define the dependencies required for the pipeline.\n", - "deps_required_for_pipeline = ['tensorflow>=2.12.0', 'tensorflow-hub>=0.10.0', 'Pillow>=9.0.0']\n", - "requirements_file_path = './requirements.txt'\n", - "# Write the dependencies to the requirements file.\n", - "with open(requirements_file_path, 'w') as f:\n", - " for dep in deps_required_for_pipeline:\n", - " f.write(dep + '\\n')\n", - "\n", - "# Install the pipeline dependencies on Dataflow.\n", - "options.view_as(SetupOptions).requirements_file = requirements_file_path" - ], - "metadata": { - "id": "lEy4PkluWbdm" - }, - "execution_count": 5, - "outputs": [{ - "output_type": "stream", - "name": "stdout", - "text": [ - "\n" - ] - }] - }, - { - "cell_type": "markdown", - "source": [ - "## Use the TensorFlow model handler\n", - " This example uses `TFModelHandlerTensor` as the model handler and the `resnet_101` model trained on [ImageNet](https://www.image-net.org/).\n", - "\n", - " Download the model from [Google Cloud Storage](https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet101_weights_tf_dim_ordering_tf_kernels.h5) (link downloads the model), and place it in the directory that you want to use to update your model.\n", - "\n", - "In the following code, replace `BUCKET_NAME` with the the name of your Cloud Storage bucket." - ], - "metadata": { - "id": "_AUNH_GJk_NE" - } - }, - { - "cell_type": "code", - "source": [ - "model_handler = TFModelHandlerTensor(\n", - " model_uri=\"gs://BUCKET_NAME/resnet101_weights_tf_dim_ordering_tf_kernels.h5\")" - ], - "metadata": { - "id": "kkSnsxwUk-Sp" - }, - "execution_count": 6, - "outputs": [{ - "output_type": "stream", - "name": "stdout", - "text": [ - "\n" - ] - }] - }, - { - "cell_type": "markdown", - "source": [ - "## Preprocess images\n", - "\n", - "Use `preprocess_image` to run the inference, read the image, and convert the image to a TensorFlow tensor." - ], - "metadata": { - "id": "tZH0r0sL-if5" - } - }, - { - "cell_type": "code", - "source": [ - "def preprocess_image(image_name, image_dir):\n", - " img = tf.keras.utils.get_file(image_name, image_dir + image_name)\n", - " img = Image.open(img).resize((224, 224))\n", - " img = numpy.array(img) / 255.0\n", - " img_tensor = tf.cast(tf.convert_to_tensor(img[...]), dtype=tf.float32)\n", - " return img_tensor" - ], - "metadata": { - "id": "dU5imgTt-8Ne" - }, - "execution_count": 7, - "outputs": [{ - "output_type": "stream", - "name": "stdout", - "text": [ - "\n" - ] - }] - }, - { - "cell_type": "code", - "source": [ - "class PostProcessor(beam.DoFn):\n", - " \"\"\"Process the PredictionResult to get the predicted label.\n", - " Returns predicted label.\n", - " \"\"\"\n", - " def process(self, element: PredictionResult) -> Iterable[Tuple[str, str]]:\n", - " predicted_class = numpy.argmax(element.inference, axis=-1)\n", - " labels_path = tf.keras.utils.get_file(\n", - " 'ImageNetLabels.txt',\n", - " 'https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt' # pylint: disable=line-too-long\n", - " )\n", - " imagenet_labels = numpy.array(open(labels_path).read().splitlines())\n", - " predicted_class_name = imagenet_labels[predicted_class]\n", - " yield predicted_class_name.title(), element.model_id" - ], - "metadata": { - "id": "6V5tJxO6-gyt" - }, - "execution_count": 8, - "outputs": [{ - "output_type": "stream", - "name": "stdout", - "text": [ - "\n" - ] - }] - }, - { - "cell_type": "code", - "source": [ - "# Define the pipeline object.\n", - "pipeline = beam.Pipeline(options=options)" - ], - "metadata": { - "id": "GpdKk72O_NXT", - "outputId": "bcbaa8a6-0408-427a-de9e-78a6a7eefd7b", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 400 - } - }, - "execution_count": 9, - "outputs": [{ - "output_type": "stream", - "name": "stdout", - "text": [ - "\n" - ] - }] - }, - { - "cell_type": "markdown", - "source": [ - "Next, review the pipeline steps and examine the code.\n", - "\n", - "### Pipeline steps\n" - ], - "metadata": { - "id": "elZ53uxc_9Hv" - } - }, - { - "cell_type": "markdown", - "source": [ - "1. Create a `PeriodicImpulse` transform, which emits output every `n` seconds. The `PeriodicImpulse` transform generates an infinite sequence of elements with a given runtime interval.\n", - "\n", - " In this example, `PeriodicImpulse` mimics the Pub/Sub source. Because the inputs in a streaming pipeline arrive in intervals, use `PeriodicImpulse` to output elements at `m` intervals.\n", - "To learn more about `PeriodicImpulse`, see the [`PeriodicImpulse` code](https://github.com/apache/beam/blob/9c52e0594d6f0e59cd17ee005acfb41da508e0d5/sdks/python/apache_beam/transforms/periodicsequence.py#L150)." - ], - "metadata": { - "id": "305tkV2sAD-S" - } - }, - { - "cell_type": "code", - "source": [ - "start_timestamp = time.time() # start timestamp of the periodic impulse\n", - "end_timestamp = start_timestamp + 60 * 20 # end timestamp of the periodic impulse (will run for 20 minutes).\n", - "main_input_fire_interval = 60 # interval in seconds at which the main input PCollection is emitted.\n", - "side_input_fire_interval = 60 # interval in seconds at which the side input PCollection is emitted.\n", - "\n", - "periodic_impulse = (\n", - " pipeline\n", - " | \"MainInputPcoll\" >> PeriodicImpulse(\n", - " start_timestamp=start_timestamp,\n", - " stop_timestamp=end_timestamp,\n", - " fire_interval=main_input_fire_interval))" - ], - "metadata": { - "id": "vUFStz66_Tbb", - "outputId": "39f2704b-021e-4d41-fce3-a2fac90a5bad", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 133 - } - }, - "execution_count": 10, - "outputs": [{ - "output_type": "stream", - "name": "stdout", - "text": [ - "\n" - ] - }] - }, - { - "cell_type": "markdown", - "source": [ - "2. To read and preprocess the images, use the `read_image` function. This example uses `Cat-with-beanie.jpg` for all inferences.\n", - "\n", - " **Note**: Image used for prediction is licensed in CC-BY. The creator is listed in the [LICENSE.txt](https://storage.googleapis.com/apache-beam-samples/image_captioning/LICENSE.txt) file." - ], - "metadata": { - "id": "8-sal2rFAxP2" - } - }, - { - "cell_type": "markdown", - "source": [ - "![download.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAOAAAADgCAIAAACVT/22AAAKMWlDQ1BJQ0MgUHJvZmlsZQAAeJydlndUU9kWh8+9N71QkhCKlNBraFICSA29SJEuKjEJEErAkAAiNkRUcERRkaYIMijggKNDkbEiioUBUbHrBBlE1HFwFBuWSWStGd+8ee/Nm98f935rn73P3Wfvfda6AJD8gwXCTFgJgAyhWBTh58WIjYtnYAcBDPAAA2wA4HCzs0IW+EYCmQJ82IxsmRP4F726DiD5+yrTP4zBAP+flLlZIjEAUJiM5/L42VwZF8k4PVecJbdPyZi2NE3OMErOIlmCMlaTc/IsW3z2mWUPOfMyhDwZy3PO4mXw5Nwn4405Er6MkWAZF+cI+LkyviZjg3RJhkDGb+SxGXxONgAoktwu5nNTZGwtY5IoMoIt43kA4EjJX/DSL1jMzxPLD8XOzFouEiSniBkmXFOGjZMTi+HPz03ni8XMMA43jSPiMdiZGVkc4XIAZs/8WRR5bRmyIjvYODk4MG0tbb4o1H9d/JuS93aWXoR/7hlEH/jD9ld+mQ0AsKZltdn6h21pFQBd6wFQu/2HzWAvAIqyvnUOfXEeunxeUsTiLGcrq9zcXEsBn2spL+jv+p8Of0NffM9Svt3v5WF485M4knQxQ143bmZ6pkTEyM7icPkM5p+H+B8H/nUeFhH8JL6IL5RFRMumTCBMlrVbyBOIBZlChkD4n5r4D8P+pNm5lona+BHQllgCpSEaQH4eACgqESAJe2Qr0O99C8ZHA/nNi9GZmJ37z4L+fVe4TP7IFiR/jmNHRDK4ElHO7Jr8WgI0IABFQAPqQBvoAxPABLbAEbgAD+ADAkEoiARxYDHgghSQAUQgFxSAtaAYlIKtYCeoBnWgETSDNnAYdIFj4DQ4By6By2AE3AFSMA6egCnwCsxAEISFyBAVUod0IEPIHLKFWJAb5AMFQxFQHJQIJUNCSAIVQOugUqgcqobqoWboW+godBq6AA1Dt6BRaBL6FXoHIzAJpsFasBFsBbNgTzgIjoQXwcnwMjgfLoK3wJVwA3wQ7oRPw5fgEVgKP4GnEYAQETqiizARFsJGQpF4JAkRIauQEqQCaUDakB6kH7mKSJGnyFsUBkVFMVBMlAvKHxWF4qKWoVahNqOqUQdQnag+1FXUKGoK9RFNRmuizdHO6AB0LDoZnYsuRlegm9Ad6LPoEfQ4+hUGg6FjjDGOGH9MHCYVswKzGbMb0445hRnGjGGmsVisOtYc64oNxXKwYmwxtgp7EHsSewU7jn2DI+J0cLY4X1w8TogrxFXgWnAncFdwE7gZvBLeEO+MD8Xz8MvxZfhGfA9+CD+OnyEoE4wJroRIQiphLaGS0EY4S7hLeEEkEvWITsRwooC4hlhJPEQ8TxwlviVRSGYkNimBJCFtIe0nnSLdIr0gk8lGZA9yPFlM3kJuJp8h3ye/UaAqWCoEKPAUVivUKHQqXFF4pohXNFT0VFysmK9YoXhEcUjxqRJeyUiJrcRRWqVUo3RU6YbStDJV2UY5VDlDebNyi/IF5UcULMWI4kPhUYoo+yhnKGNUhKpPZVO51HXURupZ6jgNQzOmBdBSaaW0b2iDtCkVioqdSrRKnkqNynEVKR2hG9ED6On0Mvph+nX6O1UtVU9Vvuom1TbVK6qv1eaoeajx1UrU2tVG1N6pM9R91NPUt6l3qd/TQGmYaYRr5Grs0Tir8XQObY7LHO6ckjmH59zWhDXNNCM0V2ju0xzQnNbS1vLTytKq0jqj9VSbru2hnaq9Q/uE9qQOVcdNR6CzQ+ekzmOGCsOTkc6oZPQxpnQ1df11Jbr1uoO6M3rGelF6hXrtevf0Cfos/ST9Hfq9+lMGOgYhBgUGrQa3DfGGLMMUw12G/YavjYyNYow2GHUZPTJWMw4wzjduNb5rQjZxN1lm0mByzRRjyjJNM91tetkMNrM3SzGrMRsyh80dzAXmu82HLdAWThZCiwaLG0wS05OZw2xljlrSLYMtCy27LJ9ZGVjFW22z6rf6aG1vnW7daH3HhmITaFNo02Pzq62ZLde2xvbaXPJc37mr53bPfW5nbse322N3055qH2K/wb7X/oODo4PIoc1h0tHAMdGx1vEGi8YKY21mnXdCO3k5rXY65vTW2cFZ7HzY+RcXpkuaS4vLo3nG8/jzGueNueq5clzrXaVuDLdEt71uUnddd457g/sDD30PnkeTx4SnqWeq50HPZ17WXiKvDq/XbGf2SvYpb8Tbz7vEe9CH4hPlU+1z31fPN9m31XfKz95vhd8pf7R/kP82/xsBWgHcgOaAqUDHwJWBfUGkoAVB1UEPgs2CRcE9IXBIYMj2kLvzDecL53eFgtCA0O2h98KMw5aFfR+OCQ8Lrwl/GGETURDRv4C6YMmClgWvIr0iyyLvRJlESaJ6oxWjE6Kbo1/HeMeUx0hjrWJXxl6K04gTxHXHY+Oj45vipxf6LNy5cDzBPqE44foi40V5iy4s1licvvj4EsUlnCVHEtGJMYktie85oZwGzvTSgKW1S6e4bO4u7hOeB28Hb5Lvyi/nTyS5JpUnPUp2Td6ePJninlKR8lTAFlQLnqf6p9alvk4LTduf9ik9Jr09A5eRmHFUSBGmCfsytTPzMoezzLOKs6TLnJftXDYlChI1ZUPZi7K7xTTZz9SAxESyXjKa45ZTk/MmNzr3SJ5ynjBvYLnZ8k3LJ/J9879egVrBXdFboFuwtmB0pefK+lXQqqWrelfrry5aPb7Gb82BtYS1aWt/KLQuLC98uS5mXU+RVtGaorH1futbixWKRcU3NrhsqNuI2ijYOLhp7qaqTR9LeCUXS61LK0rfb+ZuvviVzVeVX33akrRlsMyhbM9WzFbh1uvb3LcdKFcuzy8f2x6yvXMHY0fJjpc7l+y8UGFXUbeLsEuyS1oZXNldZVC1tep9dUr1SI1XTXutZu2m2te7ebuv7PHY01anVVda926vYO/Ner/6zgajhop9mH05+x42Rjf2f836urlJo6m06cN+4X7pgYgDfc2Ozc0tmi1lrXCrpHXyYMLBy994f9Pdxmyrb6e3lx4ChySHHn+b+O31w0GHe4+wjrR9Z/hdbQe1o6QT6lzeOdWV0iXtjusePhp4tLfHpafje8vv9x/TPVZzXOV42QnCiaITn07mn5w+lXXq6enk02O9S3rvnIk9c60vvG/wbNDZ8+d8z53p9+w/ed71/LELzheOXmRd7LrkcKlzwH6g4wf7HzoGHQY7hxyHui87Xe4Znjd84or7ldNXva+euxZw7dLI/JHh61HXb95IuCG9ybv56Fb6ree3c27P3FlzF3235J7SvYr7mvcbfjT9sV3qID0+6j068GDBgztj3LEnP2X/9H686CH5YcWEzkTzI9tHxyZ9Jy8/Xvh4/EnWk5mnxT8r/1z7zOTZd794/DIwFTs1/lz0/NOvm1+ov9j/0u5l73TY9P1XGa9mXpe8UX9z4C3rbf+7mHcTM7nvse8rP5h+6PkY9PHup4xPn34D94Tz+6TMXDkAAQAASURBVHichP3Xt2xJeh+IfV9EbJs+8/hzvalbt3x1tQe70WgSAAnQDClySGo4miXxbV70oLX0b0hr6UEPkkZrRmuRHJrhiAAJEoYwDaIb7dDl63pzzr3Hn/SZ20TEp4cwe+e5RSq7+p4020R88YvfZyM2/rN/9zsMERAAgAgAABEBAADcH/t39Ut/MCABIBIRgL0OIhIRIgIRIgAgubPMO3MUoX1P5kJE1V0QQWtg6C9KRBcaYP+PSEQEhHChnQSABAQEgIgIQIT2eEB3O+1awMwJ/nxC00CC//8vc1bVPgJ/Ipo2ABARs6Kyn22/AAgIEY1k3O3Inou+D/Y3K9gvawTCRVEQuS4wrLXKtsi33bZKmxPJDgfU5I5V27B2Cd8OTVZkHhdE2ouydiBBbSjt+1Upk/vB/BVGMmgHnGzLDODMsPqPtfFAspgjcPK4iGvzHpFIuzv4MXCDY2/nYe0HAAEI0c4D/6sTKVENjgTMSXlFHnYc0UCVABmiF3c1Ydz40MooVB+qMas65GaWux4gkiZEtOCwLQRw6ERE8211LS8tRNDmW6yBD8HInqpbmIbY8UIrPCsef6wfXyAgI3QAIIsgK2QkM9DaSd53zQqPwLcEEQE0EFY0U8nCDiAB1qQO1a0MTJlFnAcUMjKD4MZ/RS5oIWxELTwy0FEa1DBOfgo5jJKdI4YgbfcJ7YXRCsl1zw6UQ4oXMZHrC/l54W9UcWrVNqBao9HPInBU5KYu1eRMrrNYURzWoWbpybUKyE2SCql2kLz47Im0Ipn6qNUmT4XBSuLVqDg025G9OPqORGsaDP0x5KBG6IjTH2PbSVBrl1Vz4JkT3fRZuZvneys4cNIwI7cqHKxRVG1WVw1ZYU2wDcKKIiwFoMEnICPSgCunIyKrtbDSUQjAABkBA2CrOp4BMsd/tdlee2deBEjEEMlCzoPeAdad4KYa2RNrbbBvCMxVGCIAMEMClhgJALVlZXSzYHWeIVYD7fSv/89+rBG5GQD0TbCnIFZIc+A17SVws6zqnZWQ66vnJY8Nx4U1jWwJr5oDQNWguFPs0NQEVaETERkgq3Wypp+djqxdzmCsrhsBfAvAmW019UzGgvAz58I5Ncl7QToGcx3XRhV4kwYAPMUiYk3otoNsFT1VOwmtfMndiqG/rMcSrp7lB8sKjFzH6iDwl6gI6UuuUGu/w0DtG4sL5i5DBscAjKzSR6ia6AYVEIChRTw4c8LJDo2OxNW2WCPJcXBl7dSGhyGaMWJ2grguERhTyCLP/0Z+9Kz0EYDVeL4aXrTWQdVpOzxeSismqYGPvVmNxqoJWZuZUB+Xumirc+qnrwiGKmvCWlDVvfwkdV/WXBwPGnO+QZmG+ssSn0Ukq+upisIqAgGOyFYnWgUXrJ1iIeJAzBz+aq32H/3twLLIxQasYhrN6FUidqNnNaZrngZS3iCqAYgANGntNQTWpofnYSBk9tJ1TNSuY8zIWjdq4+UsGqiEazkU681GNzLOdKumnZkJiMzCnGEFBqjo0A+Bx5GXniNjWuFdcy5WcPFS8K/qSwcuzwf1Qb9gNKxcyp5BNYVPABqcMVqHZiXBmuKtdcga60YoosYOq0aVtfB8c1bHxBmK5sW8MYe2gXXLFZmf+mS42YKPVgyl+ssZtbY1K9Kxk9fbwn4QrMeqrZvlrSkvddMCc5gb1pp2JyIEBpW5y6xBaZFMTpOvkqzzCqp56uRLTom4A9C2k/nW1k6z56wwaG0M7W211hZUXvwV8/pjV9w4g1qnaMFRwoXxRCIC7ZtcXRj8vZz/QLXWVoEC8AyP9U9o5zwQMYQqMOImVTVBViaDa5aAyouvjCBy6s9JcGWWgcOhJ27vXWG9LW7swPlADJCM2UtQ98l8c6GmdiuIr0qp1iEfgbH2KzmWZYCOL11YgBzHOF+ypqexGjE3sM6tQCsEz9++PWYK4IVGec+ZauC2SHFNNW8RVnmoenl311JZxY6VqnGMwLzbsaKjKvb1DSATs7pgV6LDiGs2eLD6sSB/oO945dza+9QvB9oaOwBE2uk7BjbmgH5GW22+QtXoohOWHpnviTV2KvMIkIDXBsTadVS11MCXMXZBzI7SsS4JRLQxCERC0CuTpzY2F6/jfqkrW/uNu4kdQj+RUJtfGTorwnXLdhGsCrCGkJ916K4LjsTRqW/ndtSaWtOkNaPMDicyrNxLE2IxHffidFJG31Nng3ioOWPSXdtLYGUaW0l/Cda9MWrOXiHZVWiRNtY2klUg1S2qLoDzkwDAxZjQAcO1zzRF+wFBZIjcdquunA3EaLWNKz8iAoiqM/a+tckH4AXqNYUNctcutjKDa7EJ86WdIXZeQCVuhAtT35zAvNCdiC2B+fiEH+GaO+mJ34LVaQFnTlrNhZVycghw4q8UGkPSNrLt4vxewyAjy40rto8lW9MjYoZcrUK0POx64Qfcwh2p0qoWqz7wSXRhUNz9nD1qcWdMCh+/gy9LW1zsdKUhLZPVLuvGzjWz+saLxKgCN6CWC8kHRKsGG8TrSuujuxR5htJfpkdsX4RVjlXTyPN+TQtaIjF6AgCZ0eyvqoNX57E7wrET1rBQYyQHYAepSjDoEj+V2nOdrIte12TvXWcrympkVsOQWL+5HWhy0RwN2t3Qm6FARMZjW417VPepUbQzvmoWoe2XYStHMdWEdKoP/HxzXfSZOh/5r+59AbXGeKpM/BWXyd6uzvUrP7tb1O0qrExFGzdwTcVqNvmsIfPsVHXFslLdEvDviFZasNJUcEThCLHS7g4iNcohx4jM36PGczUX114daxqhjkj06SqyQR8A67QyP0iGh4x0GVKlwoyRt8oPNU0F4IOFYJzy2pFkjRQ3jk43oTPprMZyagGt+nQhN9+FmhTdl4DMRWqrcUArTJ9W8m11t8XVy6ATpj/fDS0iAOgVR57cXK6NN9p0gxuzGgM64gRwwS+y2qyKpNpZ6uaSmYy1/tbEYBR0TSPZ3lZQrOYB1c6n6gh8BZyI9T8AAKKmD9AzXO0M52QAAoB28LHIsdqkAqBpen2ae3qw/OS8GXsNp0BruRDbM9TE0Sejq9/QBSMd0Xg95Nvm3WWnRV2a2ArJd5+qP479UFfBkUoEULu0U/KV6oNVcw1rJwEAECGgrvl8tr8rg+0ZF6imZ8gRmLt3pcfrkwOq36nGpeD6i0RU5THtMNiuX3AELjLlBS1fM4tesQqs0q4JrlID5BSBgzV4lWHvYnNGK3RGRKI+XLUqA0AiYnaO+ytWt6GaCqvBxxmgNdQ7M7QSi0sokzsbYWXW2ps6m94lVw1xIlgHq67vap1y/1pjzBCzs+XAWn3kZvuqiLyh4gRIVir22uTi+37G2hdDIPIFE2TJsqYM0TceEcEk7ldVTjWTPStWVloNgx6vaO+8gpJVX95fuvrtohPv54YJGFXM61T4xUvR6qe6mbBid5jGEhEgA9KeJmoNsoKpCBdqdzd4Jc3sQZ55EEyG0waKzYT2YPNE467FAH1I2SPFyxSppmfRKkHjV+Mrwje2o+VEBLAgRjchABCIISESrBSIVPzmbk4ezXRhSNALcsULr4ubAMgOGNQNa2/DOt1c6acVVWeu4SI8K/UfAACga+G1uv706MRaY+oGmLt07Ve40IdX4yI1qay6VmiHsQ7ui69Xvof6ePnbXRQjuFFAQOZu6rujbatthQg5KazC3JkBorqgs3JYfSJQpW78Bew/lp5qsTZnXTPP32hiYisT0ZCiV1Xo/JVKC62wEyEAdwl3G5ZAJG2igACrI+rnIDilie6CF+jKcZj3Ro1N4NMxrr9OHVRcUaUdnHlDfgSweuubVOlAZziad1QdVJnsTiDOu6prrsp/8a6tlznW7lgHUDUPV5FVfeUUXV04vq/1l6eDCgnk4r71bjuh+p77QIUxbCofAPwQ2Pa493buM0ThjU4TSK9ZPwSANsRJxrolBrZMxjGGLZmstDhYlFteNQ3yjXDq3N4OkQi0uWzFIC5XAd79RjRBeT9dqSZa329wISrHkxV7XeRKsLzsZpqLuPtP1WiaD1UcwqGEHLH6SVYB1dl3KxNtldouxHesjGslIljLHrhv3Mx5hSYNXrXSF+9Sh1pNBAgVlNHJz7d2BdarMvOjAwCgq7Pc0XaukfMxLWM7Dc0Y0+TI04q7GlU7ALUbE4DQldVlhepiv15X2hGx3qizlS4YIzWHtKJPcl0Cr+UulJbWJ5q5lyM/p4Yr0DGHhopRfD/ANm/FyfNdcCzlm1u5q/U2+0m2ah3WvIXKlPDMVrXKIG7FFFiNDXqS9WlPp838B6w3x4p21TR0XhSsvrT+EnSuQtlpFSJH5EAARBoqVodXX+hqzcw4sKrOrvJg3XBVpzhDwt6W7DiAK1oCXAk8+SGtcTMAEYmqDxVMfdKwrsKdTVLZ4dY3NCZlBdlKy1paR6xoHWoq3BzPTFn7ilqtTRKn9SrFWnWHfMvgIl+B9wncEXXZVxMSHHVZOXptZOMHqwzsu2475+ewvzr6fA1WCsuOjFN4tUbAynS9ACnvwXhKqyPVf3+RUJ20Lxy8ckylD6zYvM1jBeJrAF2g0LIMVQDCWu02eYr2sqhg4Fy7ipaZayf6Y5xgrEIC1zXjqDvCcgMGKydWB6xOdKN0kBGgdm6MA7EZS+ZPNO4XVnks6xIBkC05XXGi7d1ddT06UTnh2pwhWfJ0QjJmADq9VXP+6yrWNMrXwmJFcUZq5q09x5kPXiN4saD/tzK1jOZf8f8qrJi7uuGqtK5v2avodBem+psa7mus409xDm/d+7FOur+i18v+zFq62rTvS1tVNchb+eiJAXFVesYFcpFU1/PVqxHU7Sn72c8KAGSrVrVRWlUSuebMgbeQ6vaMMSjc5LOpT+bTJOTQ4sRUHzQvAxehA49R9FekSuExqCpMnRFoNbKXOpqlHd4CuOBmOlOdKm/Dj6sbt1VAeCUF1vKs7uQMLNdARMYsN5ATXcUdVf2/u8KKIGtDgBVFrQ7NxdJPU88AlqFXzMqLp1eYISedSpWBo7bqfJfErqYWwAVD1pv7VHuZ08kreGfXkDXDmL+NWbTkmmjzKibMY7GPCEAMrRqtKQV7J6dZai3yxzicWHFr2yUbNTAFqEgucWBZqqYpVl6OwKA2+Rzuq3nsZ4G/O1q+9DyEANoPk9c1XjsjIFQrCMyksUq58nFq96zcoMq2Ii9Uj4aaiFYzPW4OuLmHYIN3hrMtj7tX1aj61VfE5DBR00O1H1d65tpbI3iXP3Etqt5ZavcGja7BvUbD5gQLxAopXmqreqreNKehicjHeKAKNiIAOb6qjR4BIIqa3lztK75C7oh1bq+pM4DKlkJXEvKK0VeLOjDjWtSq0+sHoHWqEQz0ybr5RizMpkur0fISJHdkHVTgRtTTrXtPzltGJw7bPquf7aT3g1ybLOCMbrJXMHRkqyJqp2rDWL4idpWqzBfeNSR/WSunFZRWCueC2eoU1MUTVoQPlnHcLPT5sIqGWDVHvYnpPYG6MKu2X3R5V039WsPJ1toCgY2Dej0HFVycTnLfoikWqfcWoKYyyY2aw79BgWuxuawfbj+UK/asuZsCDVb5gvcevI7Q9SF3bSOqFj35yFRlezhp+EY4uYM1RK1jUdnT5LtGADWZO6MC6sPjpoO3n4zxg+B8BqiKOhyuSVu4VVdHrLwmB0E3AyqsYjXwFt8rKc1KJhUOakPmD6op+BV68KKg2lkXbk1ElRzd5az/pPUFw7F2DPnTa9qmIkxwQ+ksnKorPgLgZr+TCQIAc0FFEK+6eBeYyePN3xMdD1XyWJ3TlaJ2ROeNB3S6vFrJ6kspbHIfyCwt8oyLzkqoId9cnrmWubkP1veqW07Wd6rG12YTmF/uUo2UBQdU2DEKyPbb9qVmjFVThhBqJdhkKrOp0hpA6GJS6JgCnbnm4bhSmuS5ocaaVB/kVbCuGp2VpF6Fuzf0/U/oxnDFKnCX9YFYkx/xP5gBrYjDOXHeHvBWTsUP9eSGE40DazWVrTpCFL65vkHsFbyaUarPXy8uN1fAfnthqiFAtc7QDr5vnGkOA7xwRs1yBfCAqheNmFt7oiIA5iI/tUOs9C1sa2e5X1eaCbU2vaJnKwHUPtcmEVVXIQACZMwMle1Q7farI+Mv6zFa1XTWKjw9bqi+DgEv5NBXm10P5l0k44u2WwWymmtfb6qbpWiVD67Axk1rL4NV2brEksGcFxiBcwhXNYGLJiMC1JwG31Xj6fjjV9EKQKSdarYH2JynP74mQavk/Ky3NYJ+7nvmswxVK5jytrR2PeeA9WXQxqm22fkKzmi9AQSnVmoRUqrE6iRCta5UMnIcDPWWGvlWysoitBZPcCxoLobOCwNEs1uBu5rTM26kV6jXt6Cuyv0QXtDvK15szZFftTL9l5Ukav6WVXHkvNzapPPjWHtfN0Jr4jMjiL7pVawJHAwNFIicU26vsDJ5bBfd5HJLPjzmvKVQV+JERKSNHL1KcwNPNYvrQm+shvdHEgGCttFL5x/VMOqrRTx32gahNxaMFFxAEaz/zQCYn97GIHIzAW0b3ayo1I2b/dWV0V2V3Kx3CqNyLiutbgYXXQS2ukI1qH40fUiaavfy08DwsSdCN8ArPOSIFi6+XtXydVr1JoTx2KpBraz5utKujeBFVK5onrpercbFCMWYs1Zr4kW95ia87aphHkCwa/otnfkyBwtQ5u21SklYFq1nUXxrPIUS2dhKzdbyU7Am31pNgI8Je+WOZOHlpoanIDQ/2Wag5S9mDE1HUWbiWkI17VxR9DVtgIZQWPWxzlXuSGcuufCi740RKlVaE2sNtgNjruLfO4cLyIT00NGM722N3V/hEg9716RVZf2K4qrmHmKVz3SvV/Vv/XVB718wlpxfRX5kXm2qlfwFB9S/BRsTcL5TnczNZVfsX/NXGA9BW4fXmHLerDbmgzGlK/PIX1o7oKK9v+dtu4DTfVztSN31czLWCGgKNwlWnA2oT/AKwU412C5hRf0rwnNmqFuLvKLWqU4GVFWSr2b/rPdDNW3kysN8gsHalq4S1BqxHtG2Zdqrb9+1uklQp6gap10A1Kp9UiM2x5SWuhyW6p7QhVOsUn6lDLQaKYDqQr499RGpCbyaX37+umnvRw9sl8n10hhj5Gax1zaeGZFZ3e0miOk9EGiqxo/ckLr8kLkc+WSqKVGprDqvx4jcR2N42LbXZOwNn8qgIbfiH/xccxkG1xgCIiSbx3fGluPv2pSw3GML+Wvz8yJZ+Ex0pTRqNoDlL0f4Fz1ia0qYA1cBRH4CVzrolevV/oPK0LDTG9D99GUp9S9lshqPejXoJeKPctPnFXSuaH+yd0H4khs5RHmrsmq/+b7SMx4b1jAwhqEG0tVMAvSmEgEY9S5s+xEAkEgTugSvAbC5AQJpS3Lmsyt7RABysaGqEAkA0OYjagrS6keobghAzLhsrj++BMMfi64prjyOnEfFfDoObHfdncmbMk6MjhGhkik4drkwPABQHVYzb8zNwXrXNZokr7PsjHVhaAAy2zq69qObnA43WFV4+lG3zhNhrba/dhBRbbLVSGTFE6pfz+uFeocuzlCopFGnbK+VfLXkyglOvFgNvVNZda/DKQy7N1g1xGjULOnaLCY3ZPYvc6rLmHBAREgayVl0hibde/dRW7+HnIOEgKTRagyjzmrDa/4lBAIN5O/nceHJ1azl8GLxzfSdNE10BqsLzlnxomsfeHQ6nxvQmL61YanQuUoOjmepJi/bWu3bZTdQQ388YlV7X4WpEG0tgTEVnJIy7XQ2ghWDd7NoFVXkzsRV5q4gBMAY88C6cMyXMN9/5oXO6fTvjWK4cIBv8IV/fQO8T+J0JPiou5uermukLXVZmK6YdkREpAVWBKZtNNNaPnbDLbIkWs2sFceWkECb5cg2SoeWVhj5CDgikptW6NzeWtfQB6zrc86ebaqkzSw2R0NNIVZIBANp9PUdXiSGcR3fO9FDNWntHIOV+zp2cFUEPqFZ1xYmt4AVKVJVlI/1YcKq2Ssr5qqoUoUrK8RatmPldZHsoTZGr77oYkLyP/eq3F9HmXVevHCwvxdjTGtdk+bqbLcdqc6xGs9OWfM1AGlwYncAsjAQlXNX67PxGMjxW4267a8X+1YVR5oiOiRwaWhzTbKmkRGXsynAarAqfl9lNbyPr62B6MwKBzg3uwlMagxsUt21xR5B9urmpu52nupq4rR2e/UZ7PwGBFczapvukr727qYozDIk+Ru7s+uRf2b50Q6CBza4eCrYyeQA7eysVzyY2rh4WNCrduorqL3gZlUm1n/B5kFXwfAKWFeuU2uDwV8Feqd6nEjrtsYFPNlOG+AJfw83HOj64F3WlXaQb65T7uDu5t9aB5YIEN1ax0pYRITuf4AE2hp3hMhspM5ONa+pKx/IrSQhNzENrZmW+CVK2ql6oz7s8uhXyn5fHbkaFV3IirkRIjdCteHzDgB4ovUwtarK31Q7I9NJzImOagXOXl0a/sba5ITqVuD7Uj/Rf3Q2d2XJOBOsxotVrOG/+NKuDTUQ/5csCqrnzMjSkLZCtA1FdNuEW1E57nOWEBARCdAK3C5CRiGx6sp2fS05Brbt0y6uY+To4gz1lqJFp2tnFTfWnoXRF/LYZpvF42jNGPSDh4jgksCaKik4/8iBF4zjh8Dq0TcAQDT7770KTXJtqA+w7UG1pIGYLaomyyNQG1lcGY2aLVSjXqgUIDk8e15lUN3JN9j6XNWYr4DDHWbbUNUVOIvK//uqx4NV3Z1VIl/Cu5WAnMfpf/dBPa+OV+VpNZx1sqGqqGXMCrhCtuU0gx+HFjczAQBA6No5XozV8BIoZ1hUVGr1lzOnzEy2MKvW/1SD5jDqsOgCjE58bjaTE1dFUYioSXsM2oll/Vj0Q2VdJOdmVz2og6PqWPXOXsRbVDbTa4KmDvHAKhl5bK4UZa7MTfd/x6P1e7pgRk0uHjkeOm4CuJiMNVIc7KjWIW+AuS9r7rMRC658dBi6CKxqvC98U+t3bfRda10Gvn61Sjms/CE/92qsbXFRc3D8SURWxZOu8YJrvXFINKHPX/kZX8+21e1oo6S8biS7gJ385gLoM+4Ort62cSqwYiKn1ezmu5os3drHcph4PjMhUtMd26zKEndAMZKoPlWK6aJ+9EDxuoicejJjagREhHa5N9Z53k4MN5o1xnE/fhkJvEpC1cvVfLmTatErj3U3aq6/7nDvaNYogJxX4MDiW23TMPVz6yLyUvpyI9VpMts28AY6WIm4G1uGAoDarsrmFj4c5wNB5nvh5oibr34uVqzjw2PGHjJCQQDS2ulY1zr71k0yMl5eFTsyAQVA96ZqLAFjCGAel2FxAGBdHgsVUmaQ7MukZhCBMQC77WhN7lWays/4C+7VhbEEWEVK3TS3Ss4vgXCArcw7gBUGdy4Ueu6vq1fn29aYg1ysEd1MA3D7/FTfVA2tq2T0zqXjBt/n+sUJPCtYE2+l81hxR/1enjRIVxu0ebl5G4msR1gz0y9afpbinAI26t+FDKsIFPnpBHbZcZ26jNTMKDKs5GAeplTjWqiAau0OvyDVwMrcQCmjmD2JuPwokSbyMkEAZfvpLoHM7UNid+Wyjq/W5lvGGCIjAoaGMO3mPtqqZGdqVO5dXffXMPHlH6l2rMUSVOJmfjZ4jqX6mVYtMqpRu6Nc9EJ0NoNtqnc78BWjfqVtVq1Wprj/FT0OXqFS27Ia8znlAwSEjDkt5OC7cpIfb3BNttxYCbbePHKNdicanVyFhowEELVNEFfnVHdEBCIBXBipIGOOqJzRgd70NjF0Aqv+DVdbY8BckYiYk5fjXSs9czuTDTdJ/yrLhCC1BkclWmvrWiETnHPBOWOCMc4YcqZBo1ZGFsgYETFGjHHQhNzOZmYqQ1dTl25ymIhgDXernsGFjysc7DjLO8jVWebAGlW6UXLvHI5th91Y+AHxs8gw6KujXqnyVcX5qkJ3N1nRCeiH9EsDogY32kUi0ZGoOx1qgK5+81h13Fy7v7ODoZo5rArKVcrd2Xg1S916jE7zAAhVKjsa9kwCk4i3ZgGRX7pOdrJ6Oge/vgpA24y+9X+UUjVvirS2q9nMUVpr86/WWmmSWhtLxQCUMRaGYRAEURhGQRAFQSg4h4BAkyzQApRzLoiIkJGt5ajK1bwAqllZCdjHny6Mk51etVH+Eu3nTqupafexUqsVBH1iE3zLLiDEfFXxtgdkjZc9MtDbEDUurJuMF3SCw0s9gfwlx1RMVwnrS14XVD84f+kC6D0xO4vDuNB+LrHaQ+h07ZJGnuQmhZkFKObzBQGYImQyiXUiItCkwdWAGvCBVdlau9JQTaS1VlprAiBSWkmlEUBrJZVWpEmT1koRaa2kUlIpg0JFpElLJUmR0koRKSIA0lJqgEacdJutfrfbbjTbjYRRzCAEhgSaVElaA2OccwLiEIBWZtg0EUPGnJTr3i6Ck5gdBled5ATqe+fH78IoO2u3JkkEb+pRzWq3I1LdF83WHZ7DLcOt6DSXU6lFcNwlPNjJq9SaQqysuaovuFJtWPObLvo9dSL/z70ukHfVPKfE607eRSuFmN8qjaxtYztqL0FeKCst9d0nIHF4eqqJSJPSSlkgWn6TSmvQWpOSysAINUklldYladBaAWmtldIGjUprTZq0UkoTkDKRLQKlNQNCBK0tzxkAmC3kydpwIKUsZamknAZhmWUcMQ4CGQQ60GQmBihd5KA1IKcwAGCIjBEn0pqYzUe7YUXPGZXzawMBUE/2XBiPCpjODiQCt9ae6JURssis2JQcbNxlPOGhHQJ7nDXnoX5B0ySsobbGlNW4OUPMVxf4eB9Ahc7K79b/GSS6pNqXqH1ErbU1TAl8s2n1YzVZX+FRqBSYUwhA7umL5KXh7KKaTJ0yNH6c+IvPPlOktdbSlN0ppbQyjCKJFGnUWluKNY9DJQJQiAjELVUh5wztw7IY5wwJOOchIuM8YAw5RwDBuODmxRAZ40wwzu3WJoyIpCxny2yZLTVQGMWIjDSVSinTJIZaS1mWWkrOOJBGAsZQK8aY2ye6St/UctneC4aawCqIVGWaflCdYe1iG17KLvtSw2ltABwTOEh66kBnETjpV1WjFg2OiN2Yr7QTwJWr1p2h2hnWliB3NUMzVvXri0bzRRhdSLsbO9Gk170cTHPqhilU0Ux78cr4uKCAPMOS9dWNsQgMzVbylqG1k8bKfBMvjl8CczAB5IiaMYYoAELGGBMUMMZQMM4Y45wFXDDOGWLAOGPczFHOOeOcMyY4t941Y/ZXxgLOOGMaUDDGjW/ODHsyhmD8dNJUynKZ57PFYplnSymBMQVEQEprpTSg1qSkLFVRaMaACDnnSmhOmohpAm5IBd3WobWcvNfm3vyq2z6vDJj7vsaL5ELCZljoyx2O2mesX9iF78AbBubZZORrWQzIqhv40h034RBXblG7tYWUb3HNVrnwpmraqnnjZ9wFB3/1xEp3V5ijCtNVBM3WuGF1M3Q6xFbPMXc3++1FjVO7r7hz5ZrhM4aMIWOccWSMc8ZQMAbIiSFnDJEJhgyZ4Cg4BxcEMgJx33BEYAZ8BnsI6DeEcfEiRHNBZIiu3Aq01qWUQZgJEbClKOczBQSaSGkg0kohggZSUpZloRkHRC4DCjRpTVoR48Ye0mi8Om8K2j92eBkD68lZ4dVs1FcGcXVcycMCCejL3eGaWYn+LuDcT7COl0v2VosPnS9sdzJatRTrd8cauMz3nv/cBHIHuxZdPGPloyM/589e6FCVWa31EWyEa0UCDmtE3sjyxIsmaoPIjD4yc89mIaqQ5Uoc37dU3L5+gwA5IgCabClniGiZzizfMTFH5vddAIM931rkzGysY5G5kj90tQ5+3yKDTdd4sIar1sBYqSQXQgguEECTIq1ISyk554wxjVqRKqXUqBFRBaVSkmsbuTKz1xqZlUXoYt0AAOZhzL6itmZMeRtA/2ez0s7YW40xoR9cIlctZe4DaJW7T4IQ82ZFzfkwDXPjqX1hg8U6OUlWBkTVYIsEK9tawyqGBHv9L+P7evyoupG/qZ9Rq1rF0179ItafN5dy85Tqixc1gLaocUOENhpei4DU5oIhXjHoDmx00LAdmKQ+GvZjCIzZ+mDmaceZHxo0kOFK8M5P3QV2g+aF5QrJKj0LyDiBZgDEmOCcM64ANAERlaXUoTIxKamUZqQJlFIEWnCujXlKittZa29V1Wq6+2o7kljjpdVaAagNTDX0F1fzAFj146jXkl39flj3GzQhQxeItvnZuifrTMlajKC6mbtPjVNXScvGfSpqdwL31ii5ca3FLC4SKq7gr24ZrDRnxU4186xmkddVv+diJ0DzzpxONV7zt7GGgTvTm14EACJNG+TN3hW7yTGmITxrL5uLaCMTRjasiojA7EN/q+fOVdMCAVETWE/bJZycyQyaMaUUU8r6UMA0oo1REZlgFjA00VklpQaUQkitAyLQJsRkfR2sZaidbVPPeRos262ia4NtRV8HQT1iYgiSwC/EsMTnj6xsxBqNoRegCyMgoDeOLbNa/qxOdjTvFLSBGtR64gas1pTarb19XKnvFV6qRhhr+W+HVLTWkY9ouQmwMgeqnK6bRN6aMrVvnuEr46DiAMeZq1wMdevLhxcED5gpF3GGA6GjT6uyjUK0T1a1IvZPtNBam0chIkNknBt2dKG4+hYPhIxbBwa9dDSR0hq15pxLw9wAJnVEJJXDCJjnfJLW1h5FZUxPE2PV5GRBBLbFQCuKnhm9gs4Q+LJXHWc+vIc1vYHgRg9tssN71nWCcQOBhCiJSqXnpSIgqfSoKENr7iNn0A1EzFkoOBg5eHuUIfjgqysaomowTU7O+kW+M/WQ5EVQojdzbWFEld+yqtweq0kjOi+whuVagMny3MrtKiaqGyy2yhGs0jWttRh0ee4LM6ZSTGYmCIYINoJIYPdpRLNFqDMWV0pdjOCYQx4xxhCJMWRMIPpKOMM4HBxKLHAMl7jhRBeCI1JEzMSLrIvFCMHYpkor52QgIhKCVlqZ8KvWSilhph6ZJ0BY9XNhlBxp2VX2tW8qfq0FvK0qv2DZ+fY6FeDlXrsZESCWAOeZPCvkSVZIpV8ucwYwLItny+z1VmstDJ4tFuOs3EjChMHlNHq73xrEETP13eiKtrxt6ywQbxxo10nH5Gj+rOhlwzfgZ5EjtJVwmGu6m5+e9cnpzBqyqPK4qxiq41Gyyr5S3ZWfamdD9Wgh8MWu9taVavelw0RAJAQPHMd7i4V57kRE6zbUuJe7xI+ZWWicJEQAE2JyE4wUQ9SARJoZHBO5XZerdhs4Kq2ZQSgiMiYELxhqK3NGWiulgKEpvzD5KSWVVgrIZk7N0xlNAQqSsTeo7tPolUyF92+8PVl5AKb+1wxUXdlVgvfoQKdSwT5kWxOM8vJgoX4xXe7n5Wtp/GJZ/tHB2d+8vMMAn40mCpdf6YZnRfHT4fCG7r3RiP74ZPiz02ES8K+vde902pHggd2f0sKKKhihtTOqSWOa7cDg2mg1qOuy01dud0jfo5ozvmKlIK5eCirU1V5klXJl9PgfrDFji9uwbhzXo1D+3qyGh4pEAYTZU9Bk261zZOjT23FY2UJeZbKqQtR776ZPtrWmDWRntxWDcTYrNVrzlxmiAkCzAYrRAwRaa6k12YpVICDiDBC1Mll8pU0+n+z/TaQLyC8M8ZrL7Thfk2yd+KoPq06r0QMXvqyMeDMBEM6yMlMkNQHC0bJ4upTExJ+P57OiuJWmz5fFrVYzFfywoGku201xuZFKUqdK306SqVInhA0hNNGnw/lZrkNEjriRRruNOBGstuUuOKfMzpyVgIVtk1epNbeGISIoRUopzrkpaLzocrkhAa8bV3/03zqL3JUXrMz66joGZGgnjtM4hndr6HQ2qzuiNmbmhsLoOO2SMM7kQl+fx40NxGqVLAxtOIQ8PNEiEam2PR6iCSK4XtSMPMttruFgzWAExhhDBiYuD9oSu5tohps1aa2Mc6Q1aCCNNZbxD8y02sr/BRfPcErJDzm4GpsVNPgxc+LzysuVYEMm6dks35tn+8v8+TK/mcYa8EVRPsqKrSD42qB7fzrth0E/bq7F0X62vNPpXG00/uRsuFB0NW48Xi4Zsq93B52Ac9BPZvMfHJy/2UhezmdLpW42G391e3Cr2+LMrMujWqOdJ1U1zprH9aYSQFEWk8n45Ox0Op0mSYLILu9eardadeuxQsQF9e2MGPQ/ei70hFdX4vaCPodXv6iuyc+TKTqTtzr9glkmLNUwZ1YhmuyQUwJ+dMERKJrV6wx8N4CIbLmxScjVXDbmGlRvMtSnSQVy5meI+wxSKqmUIs21tqkBBBOd12TqWWoPEAULP+fLO6+0GkbyITHw2sTGSiuLvVLfgC6GVJ2liealmhZqoek4108X+cez+b3Dk6+s9UeKSlWOpD6dTIjzb3aai4D/fDonhC+U/MHx+Vf63QDhZZZ1gvDr/cFhtvjXe/tNxt/ptglxK0kvp8nxYnmz2eQIZ1n+o9PJ80JLwMtJeKMRBojCZE2AGDIC5986Kw2cp6K0yuez8dnRdDKcj89OTo7LUgVJIhiWs9Mbd95tNdvWG6gpblzVHjXYVuq/ApeNDVUAtiIj8FEj8H6pw/HKZKjF/ZzkLZz8FBAm2ehs4Qo3zHozoLX1ujz5+1VfjKFG4D6rROgvYAK0WgMwby7bhyvWnWXTDJsaJRKMG4s2YJyAFJk6FucEASEyYIwIbBWVUkbFO1CRBmKakNlyXnebCl7WiDRSXrHNV1Q3+GnphpAQl1KNc3mwKM6LcizpSV40uXhRSAHwm1cvZYg/G082wnCsqRFFqeDHUj8r5PU03Y7CoywPpLqWJNea6UFRTkspEa63Wv/w+lVQaiELjTjRuBbHaUSfnJ0d5vKtXvtUqqcno4NSDkT4rU6cCNZLEs44J1VqnQq2mYTdgHNnr2uCIs/zxXg2PJ6Mz8siAxbEcXrl6g2ttdIglUSZjw+fNa7fFSKoOMKPBlgoYW0hHq0ypEEsWTbSK9cgiz2H8HqMz4q5fpzHpfeuagreFCw7T8H+4w0rj24XtvOmmLml1lozEzdyKqHiWyJyQVoX6vMa6cKLuTCK/cgYmBQooiyVUorsniXGLjXsTsYGVUq5v1KEIeeccVZdx6ooqpQ7IFAtC14Tra1Xssay5RJNOJf6PC+Hhcylmin9YpFnSsecPVpkzxbLd9qtsZR3koRINxnrBPyoLG7G8d1BNyf6s9PhdhJfTZMux+Ns+cs76w2OPz073w6CS1Hw+Xg0kvp2q7WdJjpb/M9P9jbSxq2WjgVrJqkupy8X+bv9LmbZ3Wb6YDJ7kKnNNJAlU6DH2fLPjk97Ufhmu7mbhJtJ0Ap4isCWk3xyfHz4YjqbIxNRnMg8V1KJMMrzXBPkpQwD0dVKFpkBKFWYpAofiLWUhFd4VSmfobP6BIcqfIL2gZzWACCP75XsSC18BhVtgz/AkKbQCOgD12h4DsF6sgZAFdTJcgmgjwZXNmulBtBW56DddsLYqG7WVOkytLF3v1WYMRy5iyAoIKW1sVBtbIvZ+aC1zmVJi3khNVssgY04E4ssWywzpZSWstlu7Wxt97vddqtFYAIz5KkTELkLJeZKj7JyWsqlVO0wGJdKk95IotNc/fTkfJiXW40kZmwk1X5e5poEw6KUSymjMPx0PDs6PV+7uvtut3W0XA4Q16MgQfpkPOmF0XYcxYi/82Tvdr+znSbdMPxoMmkwvp4mcyn3hmNVyLjbTYVYRsnNgD85PN6N49NMjYpiLU4I8c9Hs0BTP2ncHgx++/GzQZz+rZtX8rL4i+mCR/GlNPro+OQjxot8eVvQr241OpBPxpOSRKc7KJcLqSQRLPNseHiU5wVyzrhotdt5USpdZ74Vf4nsGFUj4o+oRxRW8+ZW9TtF5wvoTa2f110VF1vDb8Utq13QfS/cvr8e4PalzdIf+zUCaO0XalS8XNnYzPkSFT2hjUFgbZcRtmIQg6kX8QEG43wwxjmwqu5Fa9sYIgSbONCkkUgr/dkXnz978kwpmeXlcpmVZQmAUqoSIGk01zc2vv7BB++89dZg0I9EoAlM6RYBHGby49FiUcqCKNT6YLmcSxkxcaR0qakjxHFe7u29vLo5SJLk0TJ7NptvxmEvDDnnL2QZhdFQqs2Qp+s9xvl/Op+cTGedZvP9XvuHwzHXcq4UAd+bTlRRrsXJN9b696ezo2V+pdFgjA2i8Eaa/Nns7Acn56+1Gg/my93B+neuXCatn0xnqMVGI2VIf/Ly6JsbG3d77YSx0e7Wz84nT+fLUqujXA7K5R//xV8U52fddvKd3f6vv7aN+WiumAiTBggpS8nEYrGYTmdZXsyzPF8uAUBpKKTsdTobeU5Nb15axemw4j0IN96v6D6bgrH8WLcNLQq9f+KDfQiepGq4ruH1IjqJEEB4uxacpqOac2R6YJxDi1MA+8B2RN8Td4qzPQHcNPLTjlbSW1iFwBnachPS2t4XgTPGEKV57hECEGmluOAAwETAkJli3TSKv/bee99+/71QYLZcaFmGYaiByrw4Pj394uHTn33x5J88evwf/uAPdi/vrvX7nW43iRMWp0eK/8nZ/GQpsdNm3c41rXdb6SBOzsvys8kEcsmD6Ea78e3XbxZaP1vKe0/2W3H8lUu7PxsOl3lZar0s85SxD7bXP5tM9ubLsdK/sr15UpY/n8wWSv/1zY1mwP+XJ/s7ofilqzdPsuLH55OlUkEYbUfBh+ej02X+br+FAf/sxcH6jau/tt7544PTf3F4+pev7WrGQ17+dDj8eq97o9V4vszeKPX96fgP7+3dubqNoH/2+PFkPNHD03gyvMKKv37r8ru3rz1/8UJK2eoMgjAOWcCV0oBFUY6n89FkEogAkGnSSqv5bCGVRO72jXMPr6+zoBkIH4r34ZaqsMIhEcGFgVfY0PmfgATKfOswfbFsqrJXa34/uYCa8EBHRKNAnaGADswO38ZZcfmsGt5NyNcarc7WrNL/jke9rWubYeLhPh1qVAZHxl31HwEoTcr4Ssb+AEacE0NyMVBEFjbbrThWOJxOJxzCVqfNGe+t7bz/3ge/cvjyP/zHP/mLB3ufjsZRFERJHAjOkJEQDalDScsghptv7DX6nz8tO+uD1zd7b7Waw0Qj4wutn07mo6xAwTe31iLGPz4ZZlINWo290SgbT1+/ujspJSPsR2JbiJOiGEupFf2d7bV7kzkCAuc3eu1lkf/7oxOYL5uE37117c1ee/Li8NNHz9/ovnW91VwsM6XVsNC3e53z5bIdCEySHKh4cTxpNG/2eg8nsz8+OT04n6go+s7Wxh/d++Lw44+bctmnrFku/8Gvfvfa7ubRyZkIEuRqMpk2GzoIQy7C9a1eb1DOFvnp2VmZZY1GEnFEwlmWl2UJvmbF/TFFyh6aKwxnQ0s1a69W8kI2a+n50cfIvQvv4vn2GBeKX3UD3I0qdJF/ErZN4tjISh3f4AKUAJ7MwNsnKzwKjvap7sVVN3fUT7WPAE7BGA1vU53IGHKGDEmTVtrsUmviSSYWZbP4WmtSSslCU5o2tNbHpyej4ZCIAPlsWfbXtv63/5u/8Xe+/0EvjXSpymVeFlKWCpVKApZyakzOwp/8UfqLHzQoG8/mP/3wi5PzaYzseLGcKV0w1mmlJVDAg1Gp749nu61mNwzm86zf6dxstRZSf3Z8dr4selH8cDh9r9O+0WpMNTyYzv7wky+y83FPhM20MRCiuP/gW1uD1zvNo6zsJMm3blxZSyJkvMgKPZoQgx8dnrJF/vx0xBgfxLGIo8eTGUh9cD7++fODnX7n8lrrf713//H9BxvL0eXl6QZkf/d7X799/dLZcEzIC6lyqaXWIk7iRrvV7THGo7R56dKltUFfkZZlQUBKKhHwOEnSpGkItD5YdVDWIy1+lGsaGS06/UhaU8GHP8wPzluqxanI+TIWMzVAmFONKWlDjT7s6ljScJsxSohcqY25LlttguvgCuzq8wEATMUeVZEsHzNykclVZWGSSQatZqWUTxeZ85BzYEiuyomIiLQGCOLm1sZW2mwWZTkdj4UIGs02D2ISjV/61rf/q19+v9+OC6nyLFdK5YssL0ogEEEQhEEyPmn/7E/ig6da8P3ZfCHlWhTebSRrUYCE+eHx8Wiy00rbafysKJ8u8tuXtjd6ncfzvET+j+7e2mq3fvj0YLfRuN5uX0+Te8PJ+eno9uXd/8PX3w04+3S6yB8+SvNlnjYbYfB4vvizFydXO+1C6d1mM8iyX+y9RGDv9zvnWfbnL4+HpfqT/WPJgv3R7F/9xedBFP/GzvpXOw18+fzwP/77wd5nV2LVbkZ/6f23X7t5czKZZ3mZlTJJG51Od2NrhwXhIl8OpxOplSyy5XIZxvGg12EcNcE8K0ql02abB6FFnTPT6tyJF1iqBlX71423c3TAsaP5T5N9KqXVzn6sL+LjwsVdXZVDPQiD+9r/apaEK10jUpaZHeTRwdZAyVycwFbemeXv6PS7bZq9FgBW6+YAwCxPNv4+AXCzWIQxk3GVSiljJBCgJptFsLaBJtKgyGa9GCadQbcoRsPzuNWJWx3ORZnnRCgl3X3znUYj/eGH9754fjJfLAVjXKowChWRJCBkLFt2P/qhvvra6Pqbe6XaXOs90jCV6mw4ub6zs9lMZ0WxjthP4stpdH883R/PfvXapWEhrzTSPzs+H8Qhj6OhlM0o/Pzo5FYzvbOzmRBcbiTHUuk0vvz+ezHC6WKpBH+a56+V8nQyHc6z9NKV0enZv77/bDMJi15/VJb//smLv3p1a7GYH7caj6dxJ4nf3N0a51mM1B4eru0MGo1oPYq/8vbbZ+enWlEUJ8V8UZRqs7++WCzCMMnL5Xw6ZWQWIOSqyIi04EJrUlqv9bob69uMcY8LeqWi2dNqhVTnTtiojVvg4egTKgL0lmC1SLFyVKy3tMpnHrXW4fLeC5FA5ySZiCZaRV7FjKByyVfAjz5XSWSr5SurwrXbTSi0hgn5yaSJfATU+PLovHizqQgjsLuEABn+RCBuwlCMEUMCWwSq7R6inIfRzuUba+vbwBgQlHmGZu8dFhBrXL319s7ulSdPHv7gF/efHg6JSAOVpJVSoDUxZEyw5w/ak9Hk6t3DNKFlWU7nDOnF2SibzDe2N2Yo1gi+GE6vd9v3xvOzvHw+nl5Pwm6aTjQ0EQ/mixKI0uTxIlsMJ3/88Nn1tf5RUc46a41AnGbLH7w4FEm8EQanZdlJ40+OT5iOwjR9Y2Pweqf1cDT8yXiaPTscb/S/c2nn/mgssvmTZ8//jFQrTXZvvrb4eLPTFDHAW6/dnExGRVlGcTNMmlu7V4MgQsYG61ukdZ4tVWegSZ+dHIVhNBgMnj2dSSlLSbnUrUYzbTTBaS1HiBVisF5O6iHr/1S5X/LHw0pxI9aQYpx3unjNV10lspVKzgy1VxZmkZnZVttSofV2NBGzsSJvWyKYQiY7Xep+FIDzlMB13J7J7PNAEMyWoOjMl5olXrMJqgirJgIyRSGkAbjLNzKw6U4FpJQ0G0Bo0JwxHoSNIFCl1EpiEAAAaaW0lnEhyzII0zff7W3vXPr43sM//cX9WVYKzoHZjf+AQDEmhsfd+bhYH3Ru3u5t9I6VHp+NFkl4mhdX0+gP9g+CIDgATNP0pCieFvn/5+mLnAevt9KdJJqXRab18nR4ab2fav2VW5f/zcf3ck3fv3ntvbXeVhp3AvE7v/tHrV6/tTb41lpv/+x8/9GTN7a33u20fvL85aX1XqIndH58NtqEnfX1JAl2d+/tHd7qdw7m8x+fnq+laSuCdhRtrq89e/6Mi2iwllKWvZwvNrZ20iRhnCMXiFiWeSnLOE0n08lskbMglEVRSinCYPfq1bTRrFtZjlIqrBBUQEQHniqcVOGsFj4iAl+D5QHnrghYjXi9ntmPexW0XHXRBAOosoA26WPylszkOc0P2mhZ53FXTOlor7KCtc3jOyOaTOBCE3AG9ikNjkqNLLwv7xIRIJAxxjQAkS5Lu+rDGMIMGXAOQGSynVqZzLwpSWYMgyAMw6gsC600ApRlQUoiY4wzpbFUvNW/9Je+uXHtypXf+dOfPD04C8JQSqlLCWADEIKx2dFxfOX6r1zZ/pOjs5dPnl+6vPsP7t4aBLwTit/9wY+//s2v7AXil9d6XY6//Xs/uHn71q/evLSZROfLbKzkj5I444It80QE5csDJnV+5fLpMr/daYYigDC8tbN5rRn/7pP9NIoHzVQge3e992I2/539I5EtMUlUWeZET6bzF1mOefbpwcmdy9v9gxdqMh0TXt3aIeRKY7/bHQzWCHkUJ0naCIKAALLlYjI+f/r0yXA2LfPl/t7L44PDIs8E50zwt95//627byPjX+I3rGh58t84twbrP7n0v/PcV71vMFrRee7GOoSV+tHavaqleStzxrpDfomE+c8kkJSDhMsTAXM7Z6IDZtU4e4/6dYlIawI3cQAdRRH4HXZqxGlX5ZkFyWaxMoIpgzfAJLNRjrkqAuOAqLV2G+5ocy3SWkolpVREKIQCKlSptVKIUmsNCEEYJg1AXkjc2bn8N7//S2/cugKchWEYhIFdGsiYGGyEb7x5OJysJfFkNIHz07VG/OB8kkt9KQxZXo4KdX50KoT46b0nSZ595/rOw/Hsf7n3LOIs04BRPJ/M51n2yWiqW+13b9/49u7G4XT+py9PPj6bRIO1uNe7M+iCkh8+3jt/+ODl4ctPx4uvbq31QWWHR+XLl51G+nKZX+40Xy7meZErhi/Hs9cuXc2v3N4b50sl86I4GQ73j49m2ZILxjiTspiOh2cnh/svnh+eHi/L/PHjJx/df3h0epoXRVGUCuDt99771b/y6420WaGz5q1QxZAmSISkfeVXVTHngt6Ol6C2kIicQQguUF5zhWHVzLW39HrfO2fevdEE1kmy8we9c1Zz4oyuJ22YFarl9cyayWQdcFvN5EnQGwcOmQjKWZwVrP288QFWH19AxhHNZk4m1VnNVcZN36T26XhZFEXAAwh80Ax4GE3nhVaSEZVFXpRlqbUQIgwEY+kyW66tbf+176azP/iTvWeHTAiBTGU5aJLd/vuba6WCvWUxB7z19a/95ntv/mj/6CAvf+/B81/6+vvv7qwLwf7s+Pzw5OybX33/g8u7+5Ppp0enaRz/9MMvysn8O3euPpjOPn70vNVIw36/1HSp1/7dR3vroeg1k79398bz4eRXblyJovAXxYTC+POz4U/PR7fbHX0nGvW7T8az5+WLcDQuk1heuvrkxREFZ4Moij/49otO74vpYePk5PHJSXl4MCuKa5evdjq9KAgF50UhgfGk0QrTZtruPnn6ZO/Z86NFxqNo48135O23KYjspgw2W2O1qkeJ2yTuIpDQ4bHyeIynYcfx1bOoslj9BHCQAq8w60Reu4r3rEVNv3vYoAM3ObSRS1qBrUElAPug4iq+5U60H8nRpJ2E7gD9ZR4cY0wp5cwHZMg45yCYLv22ZJaABWOMM20iZHZ/KEVEoHUhS8pRA0AQBoFggne7fQIqsyyKYqVkWRQSSGlFHCLOVbZoN5pvXrt8eHSuSsU5E0JEb73f++Z3Z8ienZ0/G03V6fHVW7cSxiTn/+r+0ybAd67unGT5jw5Op5NJ7+zlla++9YuT8zu9VtpI/69/+vO21v/Hb78fCnYljf/li8PtOPpLV3YDoJv9zheT+Z+dnK8R/+nJ6IeP945Ozr/91u32G+88f/T09NHTq/1eS2nJeLC+dnw8/MrG2no3/eHpmItoXGSUZTwKW7NpHqSfdbaXhyezklGmP/zi0eP9l+04EVHEBUdNQoggCKRSs+l8OZ/P5gsVis6tOweNtecvT+9ubtzptokqOxIvQAds0bNTxwR2IaTVZl6Tm/p861v4wn9vC1TDXvtYMyQu+PJOnVb+u6Er4VyiFXSu4tU8/tXu+Wkeu+J8M9srH0+rB9LQl8LXwhb2mNXb1Btt2Q/ttiO+np501VOzSZgmUqQ1kSJSWmktzaxSZZkTLAo5lZRGQRqIRpxEYSiVIq2zMl/mWV4Wi2w5G48GrealS7vRR18sdC6JSsQrN26/s7a2HgW/V6ifffrZ37515Vt3rs/y4loz+eG98d2ttTQMbkbBt9Z7P1jMkktXbm5v/+zobA30tVbj58PJBsCTefafHjzd6nfY1StPXhz/84/vh+2mPDvvdzu3+90XPNBad5upSkKlJGXZdn/wl1+7EnP+dq/5//rki48++jxsd7qN9O1ea6PX+6Nf/OLlF/f0u+9/9fqVGxH7BZMPT+U9GRXNHb7VvDzo6pP9h88e6dmcsmXQ7QftTsCQFblOGjpIZKK6V2+xze3ifHI0nf+Hx88Hb9weRKHN3VUJnosq2EdtnBFKDn3eUvS1oeji7WiTiXXMEUF9/Z0lvi+p5/8S3iISjrpNjGllJXj9Kmj2fbJGJHFCRNRoV4m4xaquYS7HQKuzBJz6tu/rFnFVr4cm1RkwxhkDxkyg3tZ6AjBAYynqUmqpSVkzlDQpIk5EQEqrZ4viRU4fjo6+1e8ca70WikEkEo4NHgQRB025VjNZtIke7b/Ms0Igiig+275+XyT3Hr+4FIpmp1X0Nvfj1h8enH3+4vDNy1vrN6/96MXx/XvP9Hj62nqvc/Pm3icP/vjpizIQv/34RdBItzrNNrDradR/6+aPXh6dHw05gztrnTc2+3+ky0ir6dHJV9fX/vLO2t1OQwL93//t758+2fvub/x6X4jf+/mn7//lb8Jsyc5O4/7g4XD06U9+/o9/4/stpSGKQtAfH5+Fm4PF6dns8/saUW/vKKX3gkZvsD3OKNrlMeK4VFkpRZr0moku5HyZdS9vnyslCorb7XLvxdPp4hcn59+9tMVtTBG8H25QZiOXThGjz3xXbpLRcuTYrgYRYyNcMOEMImrq/kvReeHl4gAgvCdk5kTd1PCAM61htr3Vii1m9+02WsEbl4gAJururQUfob3QCB9MZYhS+4fQGLGgN8G1RSFppXnAiXHidstvk+w0u0WadZ4AUEq1LPWS2EjqAtnPRrMNwX56OryWxO+t9Q6L8koieo2dU2qPqMiam+9/99ep29dJ40qURCI4ni3+4vlBvMh22o12EAgOT8+Gs9PzvNv9+2/d7At2rxUfzxeT4fh779zeTqK73VbIdh/PF//0j3/y1TduMYBbzcZrb9w6/K3fPz487dy8cqvV3H3j1j/9wQ+fffT5L/+jv3+yyH/nsyd/9yuv77aT4WL81Uubd/utjzb7f3R0PnyxH4fRP/zG+yWp//Hx4//13uOT8yEfrH/3+uX/9PnDPwQaPHkM52fq/Q+SjTX56Gk/ChcZUiFnaTvrdZL5rDw4WQQJRK0tmg9Ph0W3y0s1PDjevHa50+0eHpz8OE06YfD+xqAGn5ptZkfcfqtJGVPO6l+LHLt7j1PmGvz41gKcFWdhnUJrnEU2RLpySnUWgHsON/qZQC6uUKc6e1Gbf3W7c9U4b+VQcCit2b/abeC7Yu448mbVe+IMGWMcmbYhU1TalYuA2amXkDFiTCOQtIs7takc1ZqASGsk6gbIQA9CETJ8Iw2/PuhsCEFF8e+Oz3/rowfPxvlBRvtK/D8fj0927m5+8K0fp4PPWPy9zfVf3Rj8tWs7t0JBp8O/fm3ne7sbf2Nn/VvXdl6enuHZ2UYgbrebVxrptSSmh4/6ef52v/One0eHk/lbvU633fz86GxG9O8e7k2z4u7Na6zZ3Gk1vhhOf3hwdnNnKx6fqPm8FQVxt/l/+fnnj18eX711c1jKw1l2Y2vj9x48Pn3wRSvkDycLhvyDOzc//dnPijDavX6tH0bdRhw8ubd8+UL1ByyMmUaxs71/eJyfDyGKIc+ns6wcrHciwc+HmaIiiUPBTk7PdbsVhsHJ+ShupE2G2WL5o+cvJ3nhuIB5tDhWQmujGV+35lEAEIB23jmRSZsjOq7FOor8WPuPGsgZDB4EUB3nj6zhiLkIwCocaxetx49s8BI8z5nWo9bVkmp65ZZQY3W7i1BNibglm7QaRmCCm+0btNa6MFtCutYgIGPM6HRt/CSllCliMUvsEQTovuAfdNtzRU3OTws5HU42onAjivTp8OnB8Rvdxgfr3WA8+fMPP8ey+DubvQ86zX/6F18cLbPrafLffuu9S5e2/ulPP/39Fyf3zydf3dncefv1MghGhQwYy5ReRDEUy2eHxzd67buDzr85PD3Pyw+uX7qxvX6rlV4dtH9wcJakabvdRsCvbHTbcfCLpeSt5uPHTxDZB+u9FodZEG698VYnEA9H0504utrr5m99Zbl1WYH+83uPu2F89a03eLMzHo1+609/ErQ7jIvpxra6dpNzoYgKWYrTo+z0PBusi80NkGq+zKnTjjiGAStns7DfbbQaAaLotAVgpmHz0hZjLOS81PoCHlbYyBR5uPo7qP1nQeyeweUvYQOT3uV3cNK1KgtvIzjPujIQ6hjDGvz8PgYrYVj7lVuupMEsbwezkzdzS+bdXKP62lhyzYSay1WDuw2VoSvy8wUo5ktmQ7LAGeOME2Kp/SI552khklnbSaSUIqXIxURNcNRs6LyUshkEy7Jci4JRIZ8uSwqj97vN6NqlPaV+cjJaD8NvvXY1L+XhPHuz29puJE9OR/uzzOzs/+2dDRTiu9uDRIhJXkrGkzReiwJFdK2RnM/menQ2fnz/3z15OVb6H9zYPc/ynz0/EFm2P8/u9DrdQPzw6Yu3ttcJ4MFoPpplc2D0ze99zJOPT4f3J/MsCPibb/z4fPI//OLeh8PpvclsbzjS62tTJf/oT3706cPHv/vDn015fE44kTrttN5O02+8/0H/8jU4OysOXs4fPFbjGUax7K2p6aIoqNWIwpPjJbH48m5jNhnPFhPk0Ou24jAWotVrM6XGy4IYC8PggsazKMMKhmbuu81T/ZA6Jl3Vy+Doxptz/lcXQAfvalV4ALdxQQ0bmgg1OSIkYU09gytnEtdhbWKV9V1KqFYsooEYcG9rugZVvFiDOxprw+zuQGYD35oZ6tHsthdlHJHbBeCklXI7PwMCQ8Y0gEkjKa0lUWgr87RSSmp9nqmXpWA8OMrKlLMn84w3kmYUbiXxRr9zI4mutxp708Wnhf7gK2++M+g8HM22mumVmL34/PPi8ub+bLkWBn/tzpUfHp3/+uWtJC/WABln24342Xj2jd11CSR/5ftqOFwq1Q+DTiAixGBjcF/Rdl7uTedzgGBtcIrQEOxfPno+kXK5zMIo/vruhiL66qCzPzx/+vBJp9//W3evbSfxOM+CNEpGIxWKxt3Xfv3aztPheKrViIoG8J3r14so3Iyjm7duHokAGbvUbEqVv5zO2OmIQGLajHudxXRWnJ6VRyetyztJuz1bLEe5ZL2WOB8WgGvNdO/F4fpr1zXRIi/7UbjKSMYvd0Co+ezmLQJUxW0+mujRWSMjuPhaiZK6uVABwwcuAXws3T70lXlG9VPBX9/qc7soA2yWvE6HtVv6r7WbRjWvfsUA0P6JzbXr1IIJAGbzXUQT6zSekKkrZEDc1tvbpW9aa2kwqZRWsijL+WIxXMwl0W4ScaBBHHHGp1pfX+vtpDEHWA8DTrop+EkhoSx/fbMfMDzI8i/Gs4OiPC3yAPF6u1EofavT/tpG75Pz8c128x/eusw2N39v7/jH55P/8ZNHEeNfv3YVrl7/0dl4f5H9Pz57/J8OzkINnTQ+zPI7vXagFQ0n393dvNJM3lzr/sqV7a2ylM/3N8KwLGWWFXe7bXj4sByNOeK/f7T3clnkivD8nB8c9ZrpRhB8Z3tjlGf8yePZeDSaL/74/sNPDk9CxtT5SC6yNhNfu3ypV+QciXZ3eKez1u50Ww3otIMb13ivv7G5HirZioM4DHcGPcqzRVk2Qw5Ex9PFy9lcO78CaqNvi9PIu0CWusyxhggv2G810iX/zYUIY3V87V42OGWVradxz9JmxD2KvYdUXQ6rq6FJ5+g60SOAezpRlYKqgkc1WvXtcwFfwkoidur54xCRM/MIGrMdmV0IL5Wy9a8EnDEE0EorqVSpZFkWZbnMs8lkfDI8O11kPIi7YVCSjhljyP7qRv+vb6/1w0AA/M2dtW+v9wFgLQq+sdE/LMphXt7ttbZD8UvvvrW/feXfPNr7ydnkrChOi/JsWfzgbPI/PdzvpBGQ/snJsBEGC8Y/PDpLSV8PxC+vd9/tt7+10f+DDz8Tx6e/sbv+w8d7y/Hk1y9vlk+e5tP5J2fTm83GdhB8cOvqZDwr8+J2p/Hw+GwqFaSN33jn7tfWe4M0+rdfPM7ORpmCb9y5+c217j//6N5plpcoFBfrg36nmawF7On58NHey7VGozsanp0cP11k0dkZFAUuFtn5+dl4mknFsiwHGGfZQaaS9XVotobz7GWYXrt9c8nErZvX1pJwt5X0IqPlSVtr0ngw2i7zBk2g3ZJij0BFpDwF+kH1IKu7H1jzklcIjZzbpZ3i95lCR0/kaqAIiDlr44KfZC5H1RsCZvi0Os6jHLyn5VEI3qZkrB5YRbtDpMcw1qEJlW1qK+cRmV1YrEkTSU3SuvPMhKC01lLJLM/PxqMnBy8/fnjv40cPh7liIiyJSsJc67lSD2aLmZS/fTw8XGQbcTSS+ufDWQDw7qDz09Px77w8+1dPD59N51/ttb/a75xr/ZPhBJXeisO3+u3//vbu+73GP/tPf/FeFPyf3n3tb1za+Md3rkxL+X/7F/9OHx0nnO8NJzf7HQj47d31QRzuNqL/90f3//hwOMmzZVG8Nmg/HM8ejmdrnXbUTg9m8xzZMon+/cf3UZaPTs6fjmfrrWY3CbOz4+DJoyxtLAjTfuefffYw23/eyhYTLl7MlrLdUwHb+/iTrFSD996TcfRk78WSC9XvQ6PByuzs2f6Mc0piNpnAoyezrJCjcf70ebn/cjhdCCFUUUitx6UMhIgCAWgtf7LLZv36xYor/PqJilSdF1FB1J3wqsNxwQdy7pV9TIyJZnmqskhyH5AINAnXjlVnitzVLKCdIvbGgHHKHPfVjNLKgibfW5dltwu0XKmrPxgdrGsbhJvtyBEAzENqtNZIxICQiAEA4xRw0lpqtciWo/nyfLY4H03yxTJoNvnmtZ5UgWDnUv3obPi3Lm19uH/8Vzb7gygYST1WWnC2UOogL+fT+W9e2mgG4miZ/dbDvXxd/t2rW7nWv7139K9/+tEO0d/5/rc3InG33/nnjx4zJS/1Or/38PnfuH3laqcRhCxot17rtvZOFSP65huvncwXB5P5u1d294Lwn/34I0a0zPPtNBZB8NPTEWolFvP3d9Y/Ozj5leu7x+fDnxzsiyh6ucxn09n17fXzkyM1WNdpqvLia4PuT4r8cHJWdgd6mV1VmifBpw9eBpyJZmPv+Ly9vk5PH8oXL4CFEKei39WnZ3g+Ut1OO42LvGDZfCFl3EzLZSbiYDmdtgIhlWbEpFKzotRE3EACcXVraXRekUafcXLf+iG/wCweQnVN6N9UC57qUSF7G0ddTrVaBBIQWIBaiHl6rkGzNkuICIhh5bG7s8CWz9VMZnKumdtEh5zpXQEYa1tMoTufyO6IgnaFMJpdwH1ZiKFWAFSKFkWRLbNlKeezxXQ6L/OCNDUQR+Ph2WLRSNIbSbjfbIzzfBAFz2aLv3Jl63/67Mnv/+hn//vvffN6K/39e0+ePX72j3/tO39yePrtrcFoNP3dp8+3W43rrTQGKtJkPjwLArE/XbQbydvvvrWVxHf77YPR5D/uHyHnbDF7rZUMorC52f/5yXC8zJ49ePy13c020D+6tPGHQL9IgkfjxZ/uHd1KwmSr/zuP9qjd+3g4+9EvPr3/bG+xu8Nff+OzJ88PlTx+/mLz6mWxfamUcO/ho881KME7a2tFo6tPR/zzTx/tPWev3YZGozgbrbUa5Wx2Pp1yTXpjhzY2BGJZFuF0BFKFyKaAvWa82Ntb9vsyinvdjp6Nz4m2+p3FbNHvtYHg5XRxs9dhjPmxqVwiAL+U0iBgBbl1LluFKdbR45BX94+8ljdHO4YCJG2fJu/oi9zNhL9abSG8jxSAe76XW+RfD7JahW51/mqzrVaoJ2a9Y+jSAMb2dqf4qJPT3eDMU4EImqSUeVkIjkzKpSzOx+OD4Xg8nJgqJSklaWKADEEvs/zk8EW73+hvXum0302jaam6SfTZePb+dP7Weu+jZ/ufn493uu3rW4N7z/eeHp3s9rpfnA5Zv5PleRoGL+fLWSEL4NMo7SNdWuv90cHpkdTH9x78xuvX39teezlfPp0u7rz37pKxo0UWM3yr1wKiA33zt1+c8Cx/vxn/6rVL7W73Z/tHyeHZt7cG7/Y7R8vsoNduBuK/+85X//jgJBHs8qBbIF3rtr9zZefT8eTFcDzY2exzHCKPkuR6HAaCnutd7LYkIZUSAoECjj75JGykFCdquYT5XNy/B2EMl7YDpeXOjiAqigKUZlqzIOw0kjTL5kW5jONCiLVOGIaCEANe3wUJHJV4Zeuw5FVijfLqoKzDl8APtz/Gesyv8i6aZ2lbyJstUQ06tYsYESIKchvUuFbaIuXaRKkZumAJz6wYJjCbjqH18WstqJgXAc2CDSRA9E924ASIdkMHIw8fOK7NV2SMCeSAUBQFap0tF4UsT0bnJ6fH0/FUliUQMG4fOmsWN4HWw71nZVHwN74yThJkPOH4TqvxdJn/4dn4q93WndvXP53Ov7I5eLoUjStXPjse/p/fuPlsuvjTk/FiMn90Nvz69nojjjd77Ww+eTmZXY3CX7u0MSzLB430//v08O1e82ojaQveiG79wcnwlqT3uq1L7WYzyvqM3Uzi33zrllTlH3/+8MXp+dvbGxvN5NHZ6Z9//vlH9+5f3hzI61f+3cHJ7OQ4DgTb3u13msvx+Wd7T54/fNhKkmgwkGkSZhl9cVT2252y2BRx0oqXo5E6HeciDDYGRRgiF4HgshHxQBAXGkAUWUFQHJ9AGLSaDX52StMJIuYcIlmWp6fTZnuWxkkoGpwFgsdCcES0WUpPJaveTy2O7YbEbe+4aoZqupjHNuPnnSTnU1viM/FMUw9l4oh2h1BA//gd24R/9lv/wSwPAkeKZqGFybdWUX1to6jmxuaxG0ZBMMadUnahAADSimwUyHl5aLfXNUYCQ0TG7N7J5gwCG9Qsy0W2HE8n5+Px+XQ4GU9kWTICJcs8LybTyWQ6zpYZAgRC8EBwzrTSSmsETJIYGUPOB5ubV+++HW1dL0SERP0oWgBqgLIoIiH2lvl6FGrGJlJdMo9u4Ow8K07GU57ElxrJIAoXpbw3W76cZ3c7zff6raXSPzuffDGe//1r22tp9Mnp+ZPh5FK/myAKWfz48wd/sf/itUGn3Ujm58Ozg/3xwUss8ka7qRtpORrHHJMyu7Q5SDvNaVFmo8nLg+NZpuL1tZjxG1e2H9z7gqTSWqfNxmKxWB8MeuuD8fERaRqNZ9liITgvEHnS4GFIUi9bHb51ydS/8bgxyTIkAMExCBLBaTZfMCaSeBBHyzxXgI1QpEkYiGA7Dtth8Ob64Eavw6CSf50eyVdlQAWyeuiwBuNqcQRetBUvHukvaBCozVIgswlZdbvaM8SIRD2F6FW4Jm3dbfsYdq38VvBuXtnZhUCgmd0s20ZMNWkCrbQmIjSPmHHARQBVm2GGeJXSRk3LUipNUpbz5XIxm56en52OzmfjUZ7lpSyLIi+yXJYlaQ2c8UCIQACilFJpigKRNpIgjgCx0+5sXrume1tLHp3m+XC57AueBmEYJS3BR1KlgqdhECBsRkE3jv7j4Vkm5ZvN9N2d9eeL/Omi6IZBrmQC+o1+eyMKfnY+3B+OSqW/2u/eOzn6n/f2954/S89PppvrOk17XDenk6/K6fTzz5+PJnme72yubbZ4JJqv3bp67/GzIgXB+Vff+WpZSk302cNHe/svl1mezbOgEe5ub1watIOrW6dn40GvNRpP2kn/+u2rf/6DH22u9+OQt9bbx6dqluV6mXOSLYgKQjocZk/uMQRgjHe6zWxJpUTGIQyJCyjydrMVrK2HYSDPz4QGHcWzKOJpQ29uta/spmGAXnnBRQiiz6pXg7USbyGXOvIj61m2ArQ/y0HTRF5tZZ790u0pV8UQVl5CSmlvbc+ynoqpoWNIiKjcE2CV1syt4eREyu4Uh9p562QedAB+vZBSSkqzua/WpZJKSgD7pFqyTwWlvCzHs+loOs2zLC9LJaUs5Gw+G03GZbZUZSlLaTcE1YQAnDMRCMa5JI0auGBpGvabKQ8FRvHm2nrjymt78fpPDs4wf9nncEmNh6OzrNnLm102HZdRkwAeI1zdujRTellmAOzW5tY0n//kYP9yq/F+v/v49OiTg0NdZO/221+MRgfTKVvOscgeB2EjYleXc3r+LEK8EzRPz44H7dbbb91+vLf34ejs8o3do9Pz/mBAUr37+vWP7j86OD67fuXy6zevPN9/eT6f7e7uChEwzsusiMPg6s5Gq9GMEAJkoRBa6vffeW9UFE/v3Rv02u1mmhfl1UtbGpDOhtcubWV5cXw+4owrpbjAbjNdZDnMRpyxrMg5w3wx4QCMcyoWej5UaRKSzhfLQlOcNhZhYzIcz5b5WVZ8bXv9jV6bu/gLw2oLLefgVkuIVuJPZC3EFUh5C7YCcVX4bBKe5kTz/ENy+ScX90RnG7odmEkTkVBaItgHWDGG2vvfWnNEs7rKPT1bgdmdnuyGvyanaUpFAQg0mOL2UpayLAslVVlmebbMllmeZXmelfksW0opEUiTlmVZSCmLcr5YTheL5WyuitJuCWqeO0vE3Tp50sC0RgTGUAjOuV1HH8VxpxF306TdaoIQFARs4/LPoXPvi/tXhnuXIiXzJSjZYkzIZXt5Op0uWFEGAZMaFiePG1HcScJ2q4MHI5VlrxeZOi7uPdBRlO6enRXnpyeCzUaTq4MecX4+HCPATKrRaLLe771x5/Ynn95b3+xvb27MlpmSspHGh2dD0nBtd/unn37623/4n5SWTHApi2VRPHr+8uXhyecPnvV6nWKRkVJhGm31exu9XgDUabeOTs+nWXgyWzz47LN2I+m125e3Nw9Pzg5Phy9eHAVxyJCtd9vHJ2el0o1mYz6exEkcp/FwNI0QW712WUpcZKVWQnCOoMsim+luI02bDS0VIgSQqXJS7D39Qspn08VfvXn562tdtgowS4dWQ37JEmRwURrHrUj2gdeV1epVpFf97mrWojDhKyRbSufKUip/yDNoCQACEEWgNBBo/yBRbeo2NCmlbHDVTh2zolKhL00iAAClZVnKoizmy/lsNpvNZ6Usy6KYLmfTxVIWWZnn88VimWdaE2gt8zKXUpZKuwfOIENun2pjFijblDworQk0IhcoOAejLBiLwqCRhJ00bjcbcRxjHMet7iLtHcyzG7S4kuhyPGZaFYTEWTfBSOswwDBOwig4G8+CfK5UTjLMF7OlpkajIUqZRmK+mE6OD3Y2NkaylU2nd65dns3nnWa6v/dyssw5MMZYp9v5gz/5s2aa/Nrr3yLS//K3/8Novmg10q2NdRHHAYNOKI6ns29+8FZAsNDqeDjKtC7yXM7noGmt32k1kkYjPj4bCeQ6mz1/efTe++932q1///v/sdtMMylzonkuMQyhKN9+87W8KOaT6cvxZDbPOt1Wr5UEoJeLZRSF/Vaz12k1kvj0fBgGQVEWy0IWUqVRAKRJSS1BE3CBXKqCyTjSfDaccv4XrcblNN5JogpQKxissaFLRroPFX3W9h686MmjLdp3gDZWBVK1nY2PbAISahO5r7xkxoSxQIgbl98WUxLZBymCW1ZhlvdWZgq5DTdriUopZZZnk+n4+Pzk7PxkPh7LItdS5UW+yPIAmdRqkRfLotSlVFIVUhERN1vbMcY5E8yn7BEQTWYTNSEQR+QcQs5LAKlUzMNmFMVR0IzjOIyAYCFlg4s0TuaM3e6kR8t+fjoJgiDP1SCNoyDoJMlkOonDcGtj/fh8FAouhBBax4jLPG9FkSiyzfUNEYTzZSnl7MEXD7bWN7a2t8psqaRaLLO0kbxz906/1/3wi3s/+NHPVFn+5vf/0nA4BMYODo46ve5av9PpdS9vbGitHj58HoRBwNjZ+agoihfHZwcvj5SU/UH3vbfu7G5tkFT7B0dT4vcePSkWs1/79V9f73V/8eM/DzhLk+Qsy8Mw+OLJ8wihncbns/k8y5IwjMJwrdduN9Px+QgB0zgSnIdROJnOkGG71Yzj+PR8hJDzKJRKkgYlVZrGhdRmG2ulyuloWC6L+WjO4ni6u0FpjK5g14MTfbWaNzXrxOjIk9w+oy40qV2U08cjrdfjntMBmuwGCcavB7cxGWoAl3x3XpsWBv5ak0b7hHcwViSCBs0INZFyRUNKSSK7gI4h5lyAtSeoLEtZlnmZn4+Hh4cvp2cni+lMlbKUKtOqVIoRlER5UUql0S6yJ0JQBESaA9MAGkAAlQSAIAAER47IGBM+ikEEQAEP4igIo6ARRUkS8VBoLngYxs3mUbr2uxOVsPmNiG80wvOJ7idpu5EA6aPTMyC9PuiGcdxpNvIiX+/3EPH5/kvGeWdjrSjKo7OzxTIDxP76xrVr11SeSa02L106Ozl+svditlhub25c3tl+ur8PSEqqLx4921objIcjALx+4+pX3rjz4aefvQTYXuvPpjMeBVmWX7+0+/OPP3vx4sDE7zhjr1290u92P/nii9licZbJl0+efft73+l3upgvRcB7G5vTvJiNJ/PpvNNIllqNxpNlXiDptX53o9+5vrsxns6KPF9mRZbl/W47y/OAB6BBar27tbG1Nnj0fP/g+KwZx6WS86IslGokcRSKvJAh6AhUPhljL5Bau0eZO2Ss+jpQ24XG56Vx9RFt9lcHYiQDwirxCWjMTASt/fJfN/7WAYIq1O7PQgAU1jTVWqPSQFJKX3nKAEgIEzMqykIqleV5WRaFLMsicwuFCBBLpbI8K6XU2aLIFrPJZD6eyaKUpEutpNK51mUptdJSSjCLQ2zk0hUWasUYAHGJKIkYAucsCYTg3CyOY+bZx0ARMhGIOAyiKGo1m2kc5UBBGK5tbNDgyk9l+mL/SWM5/d7lpIk6C4JIcK1pMp1NZ4s3Xruxu7P9/OXx8xcvACgSnAiajXR90Ot2Wo+e7J0NR8dnw1YjVYCdXu/uzduz+fzTz+9P8/LF8XC93WilycuDw0iIACAnGk6ng7XBeDpdFsV0PN0/OrmyvfXhvUcPn+31d7azxWyyWLx1++a1q5eeHBzNJtO007i0u9VI0zCKAs7zZbb3+Pna7s5wOnv++OEH77/7jW99689/+vM//ejTfrcZtVMgkpJarcbV3S3OUANs9jvtdmuZ5VKqUspGmkZRIAIRx7FWMi9VnhetZrqzPsgLeXQ+jAMRcM4Qi1Ii55yxXEoFiKBQq3KxfDlbXmulfFWlYw2XUENh/ScPJheJtD/bA43z4+qaq+oN0ohu32W3sSi5oqpVU5eAQJgrS01EpEgXZVmWpVKyLAsOKIKAcU5EWsk8yxZ5Ns8Xy/l0OhmPZvM8y4hIA+ZFCVpHDJhSAoAxLMqyVNKkJ0mTlrKUCjUhgEbQRNzlOYmII0NSqJEYA4KQs4CzNAjSKNKkF3kBpEUgEJgG4pyHYRAGIk2TTrMxnc8ni2UUha20wQc0EOLNG5dffvHJ2cFep5220uT47DyNwlazubU+2FjfLDVurg0Q8Xh43u/1sqKIw4Ahe/7iKI7CVqs9ycvj8Uwg3Ly00wjZoLt9Np09+vQLu202UiONbl/d3b919Ucf3d/d2uAAHDGJgs8/f7C+sUZRMFvmB0fH77z/Xgh6d9A+GY4eHRyR1qR0Mwg/uPva9s7u6dEhZ7hYZFEUh4y9fPLkm2+/OZ8vijIfnp6urfeOjs/W1npr3W6v01GkdjbWpVKCiwBJK73W743Gk8PTYasZa61ms2WeFXmeh1H08Nl+lhWB4K1GMo/DeZZLwEgIpZQmSqNIKlUQKQVMSV2Wv/9k/7VuczuOtOdI7Vc9OgjWV1vYx6rYWOmq8iewD4Eh/xBEq/cqe7By7cHZojXnDNyzu215gEBkZJ65XqpSyflivsiW2XIxmc1Aa855EAaIyBFBlfNsOZmMFpPpbDafLpZZlhdKm4amnLMwCDjPQZOmoiikCTMBSEVmgy6bbQVAjvYhBwTIMGBccKbRBF5ZFIoQkTMslMrKclkUseABF0EQEJIGUAg5QBPZYpnNF0upVIJMypKVy+3GIKdg0Yjzs+zxZHQ8GseBaDXSjbW1Qb+viRaLRb5c7u3v9zutMs8E0Gg4C8OIAeRZnmfLVIg8CsIoWd/a1oTHJ8fZbNzttUfDYVmUYRA0m+12q722tsbZg/l8gchK0pzzLJs/Pzi+vLuZpM2rV5KtVnN9vT88OX7w4vH58Wk2XQjO37x5dWttsJhOy7IMgiAMuS7L4XB4+85NhtTrdD7++NPZfJmEwZ0bV9+8+1qRl81m4/TsfDJfXN+91EyTxWJeFFlR5M00aaSZVroEtczLl8fn82VmiQxRcB7HYTOJI8aXpczLJSKmOpJSIWeNOC2UEkmiFJ2PJqO83E1iiw/wGtmqcxPkAXBL6WoZbKOj3bN8zOoL7crsCBHJFSNTVaVELvtvyXole2V0qtulg4iEJm1L0ZUsyqIoisV8djY6Pz87z4tMgw4QQsYF50A6z/LFbJ5n+VLKZVGWUhFAwDAWQgDIUpZKlVqTUlIprZQiUkpLbbOnAMamRPNoWq21WQAPnCvGjD2Sci4QNUBWllJlpdICkTNknAWcl/bxSCiISMplKYGglSSDwaC9uf2UtX/3fCqnw/fzUTPkJ8P5bJFRGj05OBr0unEULbLFaHTOGV7d3Tg+Oc+y5dbG4Ma1q51uf75YTqbTHaL5fPH4+Yvz+WKZl5sbW8D5YG3t4d5LIhJCcM7DKFkqdWVnfa3Tevni4Hw8vHH1+htvj1/u7R/s7W9vbrz/ztvNZnN0crqzsdOJoiBgj56/DJP47ddvfv1rHwRxslzOwygw9f8sEMBwdnraa7WIi/7G2ht377QajVa3vZzOF/n40uXLN65fX2ZZyHkjbTSW8+l8JpV6tn80WWRFUc4Wy7KUSmkbrAEATVLpuVJBEAjOdZFrgABgmWcMIEzifDbXPEz7g3kUJXGUCqFddKcK3tvMYG2jjVoyp+7gk4vuAwJoDW6xsg2tG9iRKbcgALelPKBGV0f8SsDfx/YFIiolS1nKspjN54v5fD6bz8aT6WyilkuulSKSnCFjUsM8z8s8L5WWSmmtOQEiJIxzBKUVIJIGqaSSqtSkpDYBeUQMze4TyMCWh4LZlNY8m9Nk581i0dI9d8asfwqY27fJJHwJEICTJg3zPO+kjUaaNBuNKAohavTaXRyfyNMTORo9PjvMizxIYgVwY3vr1vUbSdJQWidxEgVikWXzLNte722ubwzWt4KoofT5YpGJMCo03r3z2vOXByfHR8046ne7rTRpthpnp8ODo9PpbNbvbybNzqA/uHNj96cf3X/+4uDWtfh73/zmk63HP/rFJ6DZ22+81Ww05WsyYKzsd5+9fLm+tRlE43feegOIsmXGGD84OjobjstS5vMFIF55685ar3d8cDAdnl3b2TGLsLWSN65cKvP86PhwMp4ss6JQdH5+Jjju9FsMdTsJIQnXuq2iKKeLbJ7lpVJKKkQw0z0rinaa9JtNwwpFKRd5vsxLGXDNGJcq7SffvX5pJw61UlWu6BXrE2pOknGRwBKp3/5I2yXJaBeyuWwmeZiaOJTP2ZPWzDpolUWLtQ3hDSsLrXUp5SJbzuazk/Oz8Xi0mM0Ws2mxmDOlCFFqPc9yM8OyvFBSEgBjTJitHAAJoJCq1PZBpdzEoZQtkUPGwkAEAdeMmXVt0ixf18SQcbMPqNbkHjuqiEqAkEHAGRMCAZAxpfSyKAutBeMEUBApUjFApOSg2Y2TJFeywdi4KEUcJoKdHp8kJKMoTuJod9C5feUyY/zw5Gh4fjYaTUqlwoALzqezpdLAg0QE0WBtu9vbmC/mWSkDzoXgw+FwNh1vbW5eu3R5b//Fw3uPs7KczTNCAk2Dta2rly//7OMH9548H4+n3//er6RhuLm5vrE50LJc6w8QkQueL9ud9kCww7/yy7+UBmI8HDa7nZCJsizmi2VeSgDY3lq/de2K0qqZJts77+ZF/mJ/LwyDQb+7zPL9/WehwHYj2dvfe/D8aDpfvP3atcvb671WNF8sc6nDMN7Z3l4sl5/df/T4+UtZlou8kEov80IWUsUUhKKUCgA0gCJQIqZGlxifa4o1dQLBkCqv24eWHFJ9vMnkkmw9vFXYJvLoiNvbl+DQqXX9OUYm6e4rSyrCdHUkzmytdvEUWqqiyCfT6dHZ6cnJ8Xg8KpZzyjKlFUPkyEops7LUAKXSpEkAcM5CzgCYVkoTFFLaPecEF4HgjEEhZV6WUgJpJMoBCCEOOTGmSlkqBYicc84YMvPgTRfGReSBMOWljLGAcc6ZMVOkVKVWjHFimGsCwEaatJqNfrcbxkkhJWTTgtJZrqVI+OBSSsvs5GWe5be2N+IoODw+/OiL+8/3D5ZlIaVuN9JGFCRx3Nw/EFEjjeOsKPO8BFCNMHjx8mCjP3jt2nUiPZ/P0yT94O23tJI/+/BTU8NbFosyX671un/7r/7Kh/cefvT5/WtXLokovnvrxkZ/kERhnMSMMc54HEVf/8Y3ci1J6vagNZJqeD48G40Xi3m3074bh5e3t25ev8IZ29/fD6KoVPKLR0/2X7ycL5ff/cZXEHUzDZMk3hh0XhwczMaTXr/7/hu3Nza34jjpKsVFoJQSjMWd5nq/vVhmjSQ6PR8fnY1Kpcu8GI4mCICCAwAyBkzwuKFEQI32RERc6ZYQdaa0oXVjHDrutBCygNKueNRp+1rRKLkaJVDOeaqVnoB90AC5bJF2BRmgNdmqKp9lNTboZDo6m4xfHh+fnp7MxqMiy3RRgJKkiTiXpAuptCJNGjVxhmZHGmOjmOdoEwBnwIUIozDkopBykeVFURrTGoGAtJQyB+DIlCaOLImiKBSylKXJjUoNSMQ4cIYIkWCMcxGIThIrgKJUpZQaKEIRBEGBwJQmZI0gEIhSK6bkMlu2kvhrnfjjvLgn4vHNN1U5vy2gODncOx91uq295/tP9w6yUs3meV4UZ+djxlgrjbVWy9lw0Gkdnk0Vss21te2trTt3bkkpJ7NlXmgmlFSy2+197b13yix7vr937crlIAyJdCONte589xtf/Y9/+kPB2WI66Xa7jSju9QacQZJEnHEA2Nzcvnvz5tOnjz+7/zCMo/lkqkm1GunN6zeDIFhmy70XL37+0Sc33rj7+qXLy+n4z3/+F8+PhhGot1/f3d3cSONeGEbtduvsfKy1/qX333znrbd5EDMRj0dDAAaolC6Pjs8+ufe00UhbrdZskQvOG3G40LooSs0QlQICBIXtpg4izYVqd7e31v/K9Z231zrGdkVbiWkxQ3YLGQI0z7T0vrvFrtP3jget2kfn/dfj/LVNGWv2g0/vO+MWwFoI1j8BBPHR558OJ5PpZCzzpZaSa+IIKAJAVECylIjEOSJxwYg545e5QDtjLA4Dbda1KT0vl4tllmW5if8zzgQyY5dIQM4w4oJxnkZhIESBqPNcmVAYY1xwwZmZe4ngjThinC/zfF4UUlHAMYmjNIkyqRZSKg15URZFOZ3NE0IexXGQrCXRb261QITPs3w8kk9Ys8NCHE32T4Y8ihfLfDqbc87jMOitdTuN5LVru2+8dqvf64og6Owd/JPf+r0g3rt77fIH7761tj746Onhzz96EPHiv/lb39cgGu3OX/r2t2bTyWw2XdvYjuJYnqs8z8Io+d7X30ta7aKQO5tbrV4/SVPOuSu+hm63u7W9OxufHR8dTGazZhJvrW2sr69nWfbpJ5/de/pccNgYdGfD04NnIo6Cv/rLX//F/ecaYHNtjTOBjDUb8f0Hj+492v+V73zre9/9Xqe3rrUu8mI4HMZRMFsUw/PR/UfPJ7P52qArlUyTqNVI0zgadFpKU1GWi6KYzpeax7y7RlECLJDA3t0Y/PLOhoGYtRstKtHuZmM4VWuwTwM0Vpgr/TXVbX7rMWdZevhWVigg1OMADt0O0DaGiqu75Zvovbj/+KHKc6F1yJgABEQym4JzxqQSnAkec7M1KEFeFHlRaOeREQITQSAEcpaXZZ4XeZFneVFK++wmJAIOXJg9VpkQvBFHURAqrRExiaKAs0VR5lJKAhOnIgTBWCMIQiFMrACIkKEQXAiuCMwDaqSSGjRyliRxHMfLPBueH7W77dvt7W/IJo7pxTI6ba2L6ai1PMry7Gw0KxR1uu1eM9FSfvP9N69dudzrDcIoVqoMg/DWrVvXdj/++N6j/SQKv7j3lfDNm1e2fvzhgzQQg8GAsRgAdacnZfHk8cMvfvzzr7z3/tvvfWs8OhsPT7O80MA2tna7vXXGeBTHrljWJpL7a+tJowUMtVJbmxvr65vno/Mf/vwXn3z4edzpNNstfXjSCNgkZHmccM7eu32Jc2SIUhX5ovz4sy9++tHDv/Ir3/2173+fMT6bzQBIiCBJG0cnJ7LIHu+9fHk6FEEwXxZxFANgEkdm6/RSaUCc5AWLG9jdhCBm7a7iYavbeWOtIwxyXBQdXRTI1GqC20jGZIdMYQcZ1BnvxkGTLE2iO9aF6O3WjOSNWqt7fTmfy1c5UcFKuTKgkIulAC3sIzq96kapCYhCLjjnJtOUF0VRytIWdiDnLIrDVpoKzhFQMKGlmtvqT9Jamx0cgEEYiEgYpS0EF4o0ADDERhIpHQJbUgakNAcQCIJzzplGKksZMJYEIZGWRIr02XzeCMIoCnkgBOPdVuvmlSv93mCR50maJnES8iAW+m7KZ6rxbrf127qM8v6VsLXZagh2QqQ2W+nl9X6v19tYXwvDSBEAchEEhBAE4d/7m785/2f/6uGzvdl8cXx8evvGlW+8sb6zscbsA/hYFHGZT8fDsw+/eNpb382z7M7dN9e3Lj1/+jiMk6TRFkJEURxHEeecITJAjaSkCsO41e4209Q8u0RJeXh0/Gz/QJLO5nMAGBfF6Wi2Nmhv9LutRhwHvFR6scxHs8Vksuj3Bv/dP/iv775+lwfxYjpSslwu53t7z/YPj569PMjyfD6bTybTXquVRqGUsihlo5FIqc6H42VZLotCKSUaHaW1DkQep81u52/fvf56t1FKiYyZCJHDUBWHh1qwqbI17XuPTct+9pkYFkMuFK+dEkdHujaw6jKb6DwudDEtF1k3jRECNDdBHLfujpmNQpQWDAVnGmhZFFlelEWplEY7A0gI3m21Ntf6RDQcT4ssF0LEQaCk1JqYyQYwxhnjCAhIWiFx4+4IzsNAMMYLVWgixpkg0giFVpyzNIqiOMzLEhHSJEKAolRZWRaktFLLPIdAsCDCMARCSbrRaCCy8WSUL5dbQqRB53Yz/Xg8XhMh27n28/sfbu49v7LWu355O0G2vr6ZpOn+i5e9bqvXG0RRLMKItC4V9QYb/7v/+u/8i9/+nU8+u392cn5wdHLzylY7YM+ePm61e81mK5PZx5988uG9p61WtyyW//b3fvzg4cO/9/f+wdUbt8+Oj0LGojjt9PpBEAjGzMIBk0ojgk6332l3ZFmkaTqfTfdfvMznyzAMpVSz0QiI5kCn56MHfF8wxjiP4mi9371x9eqv/co7b73xVhSnJydHe3ufnp4Pn+/vLcsyWy6fvjweNBMGsFgsu83GlUtbjTSWspRKDcfTk8m0yEulZBnEMNiUSavIchGlcx6sp8k7G/0AmfJbHxlXxlKd5UsXUoeaCieya9vs4g1vkvrYqFnDSVjVgRhAe1fJl0Q5LrbBArJxKLSPWAIgIKGkVIgISEiC8ZAHjHMA4EwTQVaURVlmRZGV0i38RSaY4DwUIuAMkTjnQSCyLAsED4MgK0smlQlbcIaCSJaSNEWhMMF5xtBEeUoli1JyZGkYlkIXWmuiQARRGABjQRCEnMdRpLRWtBQKAwxaaRJGYa5pXsrZdDocnXNOSlMcJePxKA7EcjBYUDJXwcEie342Srvd3btfhUc/bTfi3c3NWZa1u/1mms5m8z//6UeXdrc21tcbaSOK4m5/g4uw1Rn8t3/373x+//7PPv70+YuDn3366LNHz9vtZpqmoRDn56PjszGxoN/Lnj1/lufZaDx69+13Xnv9zVk05EHQ6nTSNDY2iR94RCzLQmmVNBp8eEZaTaeTg+PTQukwjERARZEHQdBK404r7baa/W4nTRs729uv37nb6fbm8/mDh/dPT09Hw7PDk5Nllk9n81lZAuPrnSZTajiadFvpG6/diKPw4Oh4/+B0sVwi40iaVIlRqnuX8iDWRKwVLqOYgvCDrUFDMLtEAlxhkSYA0OjyR+i8bOeMg9PgBGZz5QqeF7LzVsVXBSTg0KixngL1J5CvKfF1JNYJE7lUGiBENJvGRoEIAqGVLgoqyjLP86IoCynd/olIDAWKNArDKCzK8nw4jqJAK2X6RESkNAdExs3m3DlpxnjKeBSEXAhkGHBu1tAFXGRZwRDTOCq1XshSaR2HASIulzkHxIRzzuIo0gCKdFnK4WIZKcUAGJFZx7LMsqIoAVicpmmcKk3rrCCt3mrGzy5tb8fJrTTak4vR9LCb55PZdDqbbm3u7u7SycnJTz7+ott4trnWXet1WieH3e6A8YCQ3755+83X747Ho4OT46OT0/lyeXR6tpgvgzDZXONBIDhjKkrCra2drc37D+/duv1amqaL+Xw+m8RJHLrAjYkYSqnLstRKN5O00+loXYIqTFkjB9AIAedX1jtvv3b19dduDza246ShlJ7NF188vE8ERZmPR6OyLONAbK/3F8uciPphGAb85eHJ6WTa6zTfvntrc63/yecPv3i0BwzjKGrEkVJqHMezsCUBismIdbq6O9jttb9zbeeblzZMEZyBoS2udKGjClPkYkuVUehodCWf5DJGfim5NxGoxppk1qXZcGdVPuLpuYZUv/xJMGQCgQPEIoiDUDCmlVpkxSLLZF4URVm65w0CAeMsCEQSBlEUxVFIQMu8WGY5YywKhFSKI3BEYkwASKlyJRljwOxDwky6qCjKQpYI2GrwRhopqZFhgLzFmAnpGUMCOQNEwThnjCPjyErGllmeSdlqJN1GI4lCxoRUMJ0viIvu2nocJ9lizri41GhebTY30vgX87IkosHOsNFqqglNZ4fD8635NIziwWDQbaSCY16o+09eaHqOhBpZVpaMi521QavZGAzWu+3Ozsb6+3fvtNrtKAyBdLPZ5pw/e/rwj3/8IRAxVKAKxrlWeb6YzecN3mpxt+rVpCzKImdcrO1cSRuNxXQEpJMoVFopJYNQ9Nrt7fXe1uZalKR5nj14/HRZSMFwNl92O61eM+Wkx5MpAyqKcp4V/U4zK9XR2ehsNNla79+6emm5zP7khz8fjqcgBBdcE02XWSGCIu3LuMWJxSJcNNuNVvPvvfPa24OWts/F8LFO6747l8ijzmpdr5LNr7YcqQqLmrVrNude51FwgSpnD9i4fc30rEHfpuy9WUEAIBKzQJMzxpiUSipVFHKRLfOizEupTZEHACAILsJQxGEkhACGZu2Hto/owjAMmZQiECIQZHZFNAs47QNLqCiLrAQkEIwjw6KU56NpFIZxFJrWFaUkoHYYaKK81JyY4BwZkq1W5rIsA864EAqw4Gyt2RgMBgQggfqDdWBcI2s1mygEMZ4myfsxa0fF42UhARut9uNRHNJxYzhuPn/ywZvv337tjSAMHzx62Gs12u32+WR6dHiyKPIwCEopi+VCKl0qNV/mUlO/09pe61+/cmVne+ta2mp1O1euXH97OPzxZ49PBZyeHDTb/SCMhAi0UmUpMQg4Z1KZ5zyiCAIRBDQbHx8dnhy9jKKw0UjNyqxWI71yaTOOAkXs6fO90TSbZ3kplVQqTeL5fCYJDk/Ollm2zItBu1USHZ+cTsfTNIluX92+cmn7fDR7+vKIEBuNlCMCQ5HEQ+JjasxEpDUIpEyEZdL4jWs7d3pNpezTZRjaLb4McWinbpHQrn90j0YgW7hJFd25Pbh9QZOPENmNh92xRABGszNCMFstmZXyPk7lZ4UDNyJp8/RNJoihCEQzjUqlZ8tMKkVSyUKWZVlK6ScK54wLlibJ+qDXTOOiVOaAQDDGWByFYSgAKImjspSkNZAG4L7RpZRJFEVhsMxyAkriWHBuqvQ5xyQOT0fTWbZMomhRlouiEARJxBhH8+iPOAozKbks0yTBQDDGm41G0kgbjWar3Vkoef/h/fW1tfDS1SBthWEYx2kaJw0ukijuz7NhHh8tM6m7i8uvNYvpXM2/ePTF9cvX37j7zs727pNnT6aHB1EQ7GytS6nSJELGxpPZ0dm54GJr0Jgvl9li+ej5i+f7L/u99uWtjffffqvT6TaaLUbqP/zpj+JQ/Pqv/bUgbDTa3SCMyrIkojiKzJOdGGNRlJRFLou8LMvpfPH85cH5eIqcpa1m3Gwo4MDD0TRrNptbm51SytPTsyd7+804VArSJO60GuPZfDJfjMfTVjPZWetfeusOIpVlOctV1Om8s76OnJdSglRBFA7ny2f7w6kGPZ/xRhMaretrve/cvPyt7QE3qCACArMPmNFt2kXTbVCSmXXrZLFrF0Rax4WY21KZ/JcubumIcBVyaJU3+WfBgLbEbbzp2i4SCKBtCh1ACx9r1ZpAkyxKVUpTzImamA07oeA8DII4DJMoSpKYIDfPfmPIk1CEUSilBC0FxygMi6IEpavNShyNB5xJIRhiyJmJkHHE5TJfLPNlngsCXcrJZJpJRQBSSq2p0UjjCKMwCAMRl4EG4IxHSRInaZw2lmUZ5jko1Wq1ev21brvfSJsiCKIgYpyLgG9E4XY7PZ5l/8P98U/Pxq2g0dncGBfT2fDw/NGD13Z21vtrX1vbHA7PHjy8d3Z6XBTli+MzZCwU/Prl3WWWCcZuXb00mU4WWW60Uhjw0+GQiVBr2txYOx7PHz7f/+rJ8fa1O0EYg6sSV0qZrfUZwzAI0rTR7q21z06KstSAZSmTNO11O0kcD9YGu5ubWlMYiIPDw363szYYrA36Usmzs1EYCNB6d2Pt2tbabL6QSm2tD6I4ZoyHTZamjaTRjMJQCBEEotT044fPPj3NZixhACXwZdJs9/v/zVfu3O6kUmm3IsOqVJfStAkkF9m0jyFgTvGiY04bM9VOudtfccUUdTFOs9bXFX+a/UIQzO7AJrBktxAnv1sjeXsDzHVQJEEQBFyRVrJkSAhUSqm0AgNsRMZYGIpGo9FKkzAMlFaT6VxrzRCSOOIMw4AzhELrUirQmnMQgmnFUGvBuUkpcM4LKfPpjDPebqSc86Isy1Lm2jzNjIBIa5JSMc6QQBItizIuyjhWSimgoJMmnPFRlsWcN4MgkGU2mpwul7rIhRBr6xth3DBPQfaPQDMPA+GcDxrJN9Z7B4tsP5fXuz2gzo+1SJbTfJKvzZ53A7bd77/15lvHh3v37z+4cWmLiWCZZS+PToui6LYay+Xi9du3AHC+WDKkJEmbrc50kSuFUbP7/lvpjZ2dwfbVKE6iMOQctdbmvsiACq0BASAIo0arc/XGbRGIz7/4vMwyFoSFNlngQkkVCN5I4sFgsFjM+73BxtpgPJ3O8jLLsyjgj/YOllm+3usMep0gjNJGK24kcZKKMIqCQARBGARpEP3RvUd/9uwsU6AZQy0LQBknX9lZ323EZSn9okskF9OxhGdD9I7rwJdtvLoLQ5UNclamJrdpElZ4dlFL912tGMXfUbsKEp/uJxdh9ZcXaRqbBdFKk1IkpVZKAyBnLBQMAJCzRhy3G400jQMhgkCYwATnLBAcEaRSUkoE4AxKRRwxYLwAaboXBgHj5rEIyBA5Z0IwAiqVUloLxgRjRFSWUgNJJakEHohIiJBzEy4lIE2UxkkUJ+W5WmbLWbbsNZvpYIDIsjzvdDqETKkiYFwwIXjAAyGECIUQXCBiGolfvb7zznr36WSeMn5/NHsskVjzWr8/oTxEdV4qmc0VRrfu3M2y5fD8/PD4NA74Zq+fxmGr2cyyRac7SBvNQATIeRhGZVEIIQbrG+ubu5vrm0GchoGIwsCg02wdYB5Hhpo0IwKKoki1ujtXbk6X2dPn+71Wc9DvP3m+LxgOh+e9brcsy267vTEYKKVe7O+dj8ZxGGQMO+3219/pLpZLBGQi6G9t9drdMAgwEFyIkIkwDE9mix88ePCn957OxtMoSYK0PQui9W7rl6/tfP/yugCtwe164BBWcWkNg1U5nMv0XMSo2+HLsW3tdBsBUB5gRGif82qPMgVw1oNHxtzTFldaUd2USJSlYghFWZalKopSSUlaAxBnLOA8CAQXIgpDHgiNEEVBHEetNGmk0Xi6mM4WjSTkis2KQnDebKST2VKqAt22KUrpLC+5wDAMm3EiBEcAKdV0nmmto1A4W8fk1YAjk+i2M2VMCM4519rmscIg3FlbW5ZFTsS5oEBESdJqtjqdPiCejofnozMUQgRBxCJu8w22poUxdrnb2u0081Ii4neW+bSUgzQBikZSPl9Ot6L+7e3rnTjI5tOjg30M0pBjK42iQORSLubL0+EoEOLq5ctJo5PEyWQ8bLQ661uXe4N1Yedq9fKCNirerOIXQjAAQtzY3P7at/4SkCpm492NQRxHjUaj1UgbaQMZV7Icj0bLPAsCZsyDRqPBGaSNZpAkzWY7abYE4xwZFyIKglme//aHn/3kycuTyYITYRyp9c0pD9rt5n//3s3X20mplNlmwCfdq4Q4QR0ZxrmhVx7ZUYcsur0U7ctlgcgyY5Vk8rGnyktf5Ul0G+RSVUxSJQQsgy7zXEpVFEUpldndXQNxxIAzxtBYB4yxgPM0EEkUCs4JSCqSSgWCM4b5oljmRRoFSRwJIUaT2TIr7OITTZokAYtjJCLSVCqV53lRSkRMQtFI4rJURb4EAEWglUa3mC4RnHE2XWStNBGCL/OCcx7HsSY9nkwyrUnwZpJmSi2yBRCAkkoVSpZSlrIouQlMuIcy2NAbYhQGdze7m610nOVlqf7J88OfnZzvRmEqdNqg87JoBfHV196+eecttZxl80mRL7gItKKz8ZiLsNVpB2EcRclWu9dsdZqtVhIFRHY3NT9y5Ha7MMoO7eNNkDGWLeZK69u3X0uS9MWTB0mcaFVMpzPGxXyZtZqN8XisVTGbjJZ5SURRFDXTWCpM291mpxuHIRdhFISc80zKF5PpD+89/MGnD2WhkzSlNC2TRrvVvNZq/N07V262wlxKR2UWMpqIedq6sBWtz0z6b5zC9Y9hJecYgUelx1dlJfhtkMDFmHzk0yWrajEpD0p7fMXmKMwazbKUiohzzpnJypvHwHDGWKfZ6HU7jKHWWkq1zPLzoWy3Gv1ui3OuNCVpI4rj8WQ6mmVpyFuNJMuK5TIDIrO7PCJxZER6Os+WeaGUCoQIuQBAxrjSpVkehVpLpYxzKBii4IJxRRBFYRRFWmcAtMwWw8kUlN7udiMRnI9Gs8WCtO51uuvrWyJMSOvFcg4MwSpaBAROYJ6hDAAILA3DJAg3W/E8K7+fF5+ejz4cTr53aTvH8P5k8uenZ3ca6dfWumtR0G302o02B5RKNjt9IijypeA8jJNGo9Fstkw4QrnHibo13NVScbtyABFAm122GQBqzRgDoss371y5+fpyMjw6Otg/OjkfnXXH55PpjDMYjsen5+Oz88mdm5caEev0NgbdfqPZYkIwwNPZ/OcvDp+eT6bz7Gg0K1mkG0L3BnGv/feu/f/o+rMlyZIkSxDjRUTupott7h7uHhG5VVZWVVNXD6iHgBnQPIBo3gEC5g34BvwM/gGfgEcQDdEsBAwGQC9V1VWdlZWRkRG+26aqdxMRZsaD3KtmkdUwD4rwMFPT5V4WXg4fPvzVby63r9oqIEiS9eaDmZZJtMXqcO2jL4sCz5s5F2P5qWtdqypbS6G14F8fDwa6eoInU376zvk5f5oUnL8DzxOJ818QnIhIFjNTs0DUhjrllLNUVajrqqnrF9f73XYbY+z7fhjHw6mPMaOmytN2u62rqmubKoTHY98Pp6E377jUirLMlhISZRFEFDNEcMQIqAgFUlYDdq5sMiRGWUIFRrXDNAHYPrZVCNuu8T48nPrb02lb1QTWD0dE2m02ZspIJprGU0ZsmdM8ReYybksMZeaPCNakHAHRgdvU9F+8vm48/4eH056pYni92VxO+RZgBP9/+3h/ezr8V9fbb7rmZ7uNxWkae4QyEs1t1zVN45gZ0NCyloKddP0qZZ/kDIgqkkUkpTgODjE4V4y18lVVN5dXN9dffX3z6cPf/Nv/+e7T+2GaP9097rb7f/WvfnE6HrZtuHlx02yvBoPvP36eRFOW//jpy8e70+NxyEiz97rZpN1l1TX/21+9/a+/voKyurTcZX0yBV3ReAB4GkxbvSKep92f5G7OXPlnCOjq7J5DRmuc/4krPLtMePZry/fXTtEZXIVnwef54x2AiSkCsFkZ4wRGw+S93282222nhnePB0fETHlSUfEes8rh2BvA5QX1g51Oxya40XHfD+OkCxRa8ClA77z33gePzI6dihASe1cXWhoiOTYBImYkyQIAjGgpZQBm6scxZgnBv7i63LTtpqpCcOTctt2GUDVVU9WNCyFUlfM+isxTX9eNI1LNBm7dbA6FUbMm6GYITLRv6v/q65f/xVdXWXXMcjvEl7X/3eMJQP9st/t3D4f/6x8+/+cvb/5Pl9dX2yrnzM5V7abbbB17Wq8yEeGzNU5FNa3cGCJWMBWdx+n2y6fD423p28XH++bVW2YHgCbK7Jqmy+S7y5ff/Hzz1z5c7ff7/X6O0zRNYvBpmv/m999/Oc1D1MeYT1GOx5O1Lex2v7y5+K+/+UoRX7Xh266SmP/UJnAdVl/xyZ/8+MlOFgqIlYJqJcHhefao6CSWeLBu9LRzzf+svD9nq+dS60986tkuz+ntT9/RU4MU/+pf/2c5JiYyBO/c5cW2bmoiRMNhTrV3SbKZtHUVHKuqc9y0TUwionUVNm1jAHePh3GY+2Ho+2GcYoEARQSJC2Llq+Cdd84BgKkiADM7z6I6z6l4NRGZVdXAOdfWFTNVzjd1aLrWISFTHcK2bes6XOwvjN2YIhEz8367vb56udtdOKJxGoi56XZVVbVt530IvmJHRMjEZwNVW+5ByYzLPTKzJDLFPMQ4xKRAd1P8/eH4bVv9q9cvGMQMkNAAq1D5wgssI6qIolo0L8qstYquECOISBynT58+/n//5t9++PTx19+8veyaerO/fP1t8JVzzsyyZCISyTnO89j3p+NxOM2nByb7YdB/8/EYqtrU/vD+8w+Pj7jZNCFQ8JuXL9/suv/ml6/f1F5SFFEp87PnvO7ssZ6F1zOQRM8IdIsRPWHvBbSk1XHqc2dXhnIXaPOnVCZ4eob15VZ5nMX6n9niUm+cS66fBv6C3jsCLLl9XfuuK+0ev9+0TdM8noZxGAwElFUkqjim4KpN2zZ1gwgx53kaJaUUo2pi1F1XNZWTnOc5ncZY3piqxikmSiXfR3a2Fl5N8KYwx5iLwiIAOmbHSOSd984F7755cSkKv3v3IaesKlGaUNUxZwDYb3fsmENQYjH17HeXN95XROwcEzsih+eW7/r5yxxp+aYhlk0CpYBlotr7XVPFlGeRbeWvKhdTfJimi6aufEkVClP+qVwoUr9cemNyFrOyJTE1SPP48e//X7d/92/VwH/zutle7l++9t4jURl7LLO1atb3p9PDl9PxkCRfv/rmwxDf3304jPnd5499TC+ur/7Ft9/sN81fv9i/6OoXTbP1bCoxRhF53tdZzcbwJzd+ocqXHy5nSHUZdysx2sDOwNCyz1rPiaeZrH3IpTyHM5z6HMuEp9rombU+Xf9ztPmJZf8zVwoATiQnySBgYESsBsd+fDz0Vxe77Xaz23aS0zBM0zzN0zTFpKpJtKlHQ1DReR4rR7vWU+sB2pRlnOLjcZxjVkAFJUTnnBGqaYxxRGyauqoCIqpaElsELYiyqWNum7oIh1TEjsv7mUqqAGYxZdPps936ENq27XP2RNL3OUvgV5tu50MFxETsXXDkaBlQwMUsVw+6gEErOL36FaOl9c8VEHEOTJvKl4iDVPaewRnVFhE7a/QjYvmkZjFGU5Wph+lR5lHjIMPjXj/u6/z6z//zb37zL0PbMTsEMtOcc3knagqI7XbXdJuLsf90OPy3v/3+//39+3q333399n9/uctgPvhf77tvL3YOLcdoBiJalpg991tncOccggFWJQZTPa9dsfWzI61UtMKvo3WebEWOFttRe25ATz96+tLnNvccpTprkDxXznkOfP4JDQ8XSVCX8qKxkFIepgkASgU/9P23377pmqv9xe5yv308HL58kfEY72NOd0cz846DZ+/Z2uqi6rZdE0J4PI2H4TaLFIUwAiBmdmwAqiYqZWJE1Zo6AELOOaU0pVQkx5tQX2833jskSillk02oTsN0HMZ+HMcs+67dbToX2NfVwzTaPF1d3wR2TG7OklL0VeXY+TX4nj/tchWe2eh6J88/X442AhoCOfTK3rkCo5R9dilLVq29R1wWlqrJOW1YdH2ZQwiZSMbD9Om3px9/9/7LXR/zv/njY/fmN/+rv/5ft123CAIAGigVGFi15ImE+DjFv789pZR7F/76z3/19dU+OPdmU99sujYEzy7OUxIlHyQmXbEtldXPrUofsPpme/bZy1b1J5+3atTDsytiyz7jc320xnAzwJ9UXWcwAM4z9ef04lkqCQBL5loSvAJ0rK+x/tbyeFsxWlM1RGeiTOSYpKzEEgFEXwUzBdE5xpRTzHmeJmJu60BE45ymOXrvura53G+JcUpJhxiSjtMEZUAewExs4cIiOzYzzJhyUtMpRgCrghvneZ7nLALMRJTATjFuEKtAiuCRN03Tj9M0z/00j0kc04urfVVV4N1ls/E+7Npt8CGKfL6/q6qmabeeHRKKqYoxMCIy0Pkq/gnA8bxNcjZeW7zpMhZgBqpo2ZjAg0s5JZGCyzt2pfnMzFYmxBEBkMi53Qv38/9y5lcP+fcc/P/mX3/9s5//AgBNFRxIzmpFLVBzFjMTk2FO/3D3aEg/TvGvr/f/y59/TQWmAphE5iwofUQKvvLBm6mrUEcxMwNDwpVe/FMYp9xwNABewaDF2tYEAJ9V6XbGJ88t+OfX6yz6dfZ/9icQ0k/h0jPXCcpg8bNffMqPn52f9VeWH2HZ8uEdt7VPYqfTwGaGVFV12zZZdZrGmCXGRGg5ZUKovGubOmZzjKI2xtQ1gYiGcYoTxJRiytOc5pimKABAqsEbm9ZV1bW1qU0xBc8IeOwnUa2r0BJlNTXrnHeAKee6Cl9dXY7T/HDozbQO4fbYm0jtvahNKVUDfH/3eJqnfde9ur4JVT2mNEyjmb568bqqayT2zpV1dZIVmJAUFjbikxN9ss4ycrAe65VSbss1ZPToTDVlISRHJiIAEHMyfXq24reZCMBE1MjdfPvr/auv52EAg/F0ZOeWBjRYTinOkwHY0moWVf1q01SM/+U3r4go5bTyiC1QYdMyqsVpBEQXAjnPdQ1xKpsqFmVqhCXXXM+gLYNrslok6MpFUoCVybSCSvj0uz8x8sUB/7QGen7az50nXSdCi8iSrvSZf+Yd1uPzU1+7Pn+5I66E/zHKPM9JBAG9c+wYAULw3vuH4ynH6JhjKuFYg/cGIIRiMIxzP4ybtmKmrJYVH09zP8aYxMBKnatmKcasmiQTUhGzGeY4zjMRBl8jUZaY1Vow59gAKu+C9+OcssygOsWoYNttVznu+8EBTTDdno51VZ/6McDd5X6bU/qcZ5QkOb1587Ou3SCgmGpKhAjgGdBIi6h5cRN/0pb8yd9XCHAx3GJUSyWUAICZk2QoU+OmKUvKOaYcswDiRddsqkBoKpkBqiqkmFRyShEAytCgmJUiScHMhBBaz7vGm9nYH6umY3YiYqZEzAgqeTktiAYWx9Fs4KqqNjs5PKhKoamVxPOc0i1TFiWs/0n+95wyt4AbS2IA//zrmRn9NBNdje9pz9ZPSqP1kPynn/L5v89/ef4qLqacsxiAqjCRD2G/7TabRszu7h9TakwtiyCAqExTfDgOjOS9qypfV5UBJDWiZrvZpKxTfDRAVSOipmlg3XNvavMcU4zBeefdlFKMkRCDrwxwmOMcUx1C19R1HXKW0zAOc3JMSKhIWa1mDoSFGHCcJu/4zeVVUzfvT4/34+Cb6mevvjLmKcXD2L+Ic/SO1ZUYXSSYy709ry1dGymFX4uwEnDWS1pu6PMEoCwsKQ4TdV1wSoimwoxmnFUt5ynGL6ee2P3m5XXwFYdwru6ziKjYIseiAkk1g2RclpOjipRIG8eTbzbsHIBJnJmdcz6ntBbLiIg5x3gY0bmq3Uz9oUDITzf7nGqDncP6E1b0nAXyVPuvlfXT7z5Z0p8Y0/l4P9n3swv4E2N99n179rpPYf1cUK1GD4tLB7eoQSJ67+u6Ct61XV1V1TzncZol5xB8FeoquHg4nYZJsjjHAJpFUspV8G1dEWJKaYoppdw1VYoxiTJRUwUgGscxQraUVCwDFpkxQHDeOe9hcfJIRGoQYxpjUpFN21ShBbBxijGnnEWdtCH4uvrx3YfrrmWik6S2aVTs/vHovPuLX/z6Zbc9jv0/ffcPX7/52f7iCpC888ysYJBVCQtCdMaqEZD/tMW8LHt8usSLk1lqj+W/ts5IFAo5oGOuAUBVVP7N3eGHMX11e/jXl1002FeVI9xXTGZieppmBCPNc85q0PJyYM7VNxJZzpojgnnndZ41ZyNi5zUnWyoMW1DXcc4p1t12Hk45zmcDWe7xWiTBuj4LdNlZ9RRe4dkne2ay+J+quP/ETS5G9vxsPy/V7Se9ejv/FgA8yTTj+QiVN4DP2vSufKtMZgbvEbE/DcfjKGpMOCC8vL7Ybbo5ZyTynsvT5Gzeg6jGlEx1inHTNuQcEW26RtQAUUQNQHImxMp7QkwxqSqTg7KHE7BspSGzxlPFfOz7snq3qiomFlm22agIM9VVcMEzQMscQuhTvOz2L/b7u/7YbS7fvv4659Q/3k45TYCB3t3df3n56u3F/jrGRJSpiEgWimpBU5jOF3ptsix1/p/6jKcjjcvC6me3BwGQEM3YqPb+b+6O//Oh90T/+Hh4xfr9FBX5v388/pfb6v/w9gbRpjkxU60xiCA7pIAKBEiAyCQigIjOaRZEyoAUvOUMZiqZvJeUVESl7K5SAMhxHnKuuo2Z5TQv1c5P4/jySc8+71nyd/aKf4KcP/eUPwEvn2OZy3qPhS5z9n/nFupT8vDc0FcsFs7Txk+3oPxweRtODcCUVFRojlHVUs7MXFUB0AFAVmWmCh0SNnXtKM4x5ZwIzZTBOyIGoKyGWUSECTdd44NDs3GK4zSnqGZlVcgCthXV3PJeU85ZNXhfVX7KmR233hPgNM2gIqJTTF1dt22zbdu2CneHY+29ZiEEVL3aX/zFr//ix88f//7v/v2ubX/17c+96qfbL/3D/as3X19fv5qnYSZyzjdVretlLUwSXdZFPiVIhShz3jyGz6KkWfETVnrramWcwyGAmBVWChL8zcPx//JPP/51G+7NDlG/3rV/cbX9NMt71e/j9MMw/aL1O4ZCMDRENCWwlShgZlbu9NJKIFLJ7JyAGJS9P8Leq6lGUS1NAUNEVZmHU2g6U805PYvuTyX2k6vDn9aK/0nk/HmOuJjPfyIHff7M9uxXllPx01ThbNn2TGvkWSkKz3xBeaQ6RhOFnDVJpDkBoqFtgru62LVNHTyHEGLO85xyygjqHBfQNKs5gizq1LZVqOsq5ixJh3E2haapnHMGs5mVTiCYLSo/qp658N3LB2UiAFC1yrlN2wKCaFl1h0l0ivPN5cXLywsxm2OapjmpxpxCcC9vXjSb7T/8/rcPDw9MBETv775c7y9ev3rddFvR/Ic//Pbm5evrq1d1qHNOMUUicqX1icDEBaImwmXh5wqYn1O05YIut2BZn6cqktMiwAQIVNaRAiK83jT/x29fvQ58Efz7ab5pm5ppV6X/c3s5ioGJrt2dxVkgoxkSnVsva263JMdIBGbIbJKt1CIGRFSq/udmJDlP/bFqOjU9Q/e4OrY/MaxzWY3PCEf/3PLKd/SntOXnR/d8ifSnRmk/bWL95JntKcdYar4/qczMEMpaV3DnF1109wi9D0Qcgt/tt4FJVO4eDsdTn1M2FdWi5oPM1DbVHGWe4/3DQbRjQkfo2naeYozJnOaUwZQIELng82qaUo6iVfCOfYwp5VzsJYvsmrYK7jTNlXNNVT2cTuM0eefrphHVEHwWNQBP1Hbt22++3u92f/junwjx1cVV17ZieuoHR3R9efPhy/v+eHzx4tU4jff3n0+Hu+BDaNqqarNkMHPOMxNTEThjWIC6Mz0NaCmI1z2OCudpXVVBRCKGBa0D1AWp/qpr/3c/r2LOIvLtrs0iamaKBNCSLroyZb+UKi2eA1cMdgUOVo9ipoisJsxcFqmBQc7RsFRsT1VR+YuKTMOpbrppONlPgzucyyBYq/pnEdz+U2X4swACf2Jnz//yp988pxPPDH1F4J8euqYNcnamuLQMbM2jABDc+vaW9xB8aJqanTv1Y9vU3LUx6zTFOCciZOcsC6oF75goxtSfBhGt6goYL3abtq6c901dDcMwxwQI3jvvuESWmJJkFVEtRSuloj8OZoTIxMMcYxZX+kDEu64jJMd0s992XWuAn77cmhoEt7m8PPVj//iYRXzw1xcXTHh7f5hVfFXffvngiH/1q9+4qvnx3R//zbvvX7/86te//ksFNDXvfZYMgN65um7qwGAoubD40QyQjAgXPbbiHmRZAl0KbVVl9lg2qpSiCklsWUdmSMSsYAWHN8kgGQqOCbIM+C5ZAcIiH702s57d0eI/EUFFDQnLqkIEKEGc6LkHPduf5jyPg6uqOI5PdnNO+NZHPuXW///tcgm16/88fy148r7rz/75V7HdgsuuvBRcCqeVFbA8/RpU7CyKvLycqyqfs2jZjVTOlSoaSJbbu8eYEhEBYt02lXdt2xDRPI2HY386DZIFCduqqqvQeN9WdajCNM0q4p2TnHdd470Dg893j9OcVMqlpuBdCL7yzsxSFlObcpbj0Xned93Fxa5tmxhzziCqlXfsuGnqw3E49H1m3G12P75//2q7/ebtm8/39xfdZrvpTv345uXLm5tX4zwhNGZ693D/3bu/ufv04e3bt7uLq7svn9i5bbcLdU3OO18hoimoWRkUJuaiRBy8N3zyNabPTz5kk1ITMJKBZRUEICRFxKJVbFr0xtY7sCwJBwAgWgzCAEyRGJjOFrkUYmaABEiw2C8qgErGspDKVERyTshsa+b3VHQDGEAxXx9CnOcla1yhh+dJ5BKF4ZmbLAH9mcnCszLo3Jz8yZIk/BPztBU7sNUfnst8xKcmp9kqg7OEnnOGQ7gUS6aA4Lz3gGh6pt8Alu2DoNM0pRQ3XXtztd9tuxBCU4WU8+3dwxzTiupp8ME5J2qH44l6RMKymogdq9g0pSmmw3EYp5glI1IdQlUHJEQEWYhAmFXnaa4tBBdP/TjOaY5RRMukxDzHUz9mkV99++0k+o/ffQcASNQPQ/But93WVY1I7WafVU3leHzo2s398XD7+cMvfvHLN1+9/fD+B0kpSmqr+vLi6ubm5eWLr3KmmCMiiuSStIhKW7dmy+7dP7l5VpryOT3P/XPORGRPHnC5E8GxY55jMY9VMw5gqYfKUB0TIi0M6oWvoaZGCFguEK76mrYM8Zb9vJJTGbnS8zv7aUSO8xTqmoj1zDwqkO0/QydW612LFvhJunnuaCxpYvGChGtg/6n80tmnmhmu45qr08ZlKRIuj1yxvifnamsf6+ltoWu7ulz1OeaSJprqtq1D8LcPh2makXC3bRihDc4AjqdhGoeS5xJzFcK2a4eYYsxNHerghnnuh2gAqtYP8xxTznkYx5SzAniCUnk6IAbKkMt+PAZMoimlw0mGYfSO67q+vLzo6qqua+e9iDrvFPHh/g7MJMs4Tv0wusrv2o4QQ1XFnIbTQXPabvauasLh4X/x1/+ZIf0//6f/cRrHum1eXl7P0/zv/+5vfv6LX445vXj5FgFOhwcFaJsu5bhpd6aljWmIBPh86LtsQxEwOCvbqMhiAbgkfLisMUFCzIpMWSEX+KZw+emcz3FRElyW/5bKHckB6vMArypLhgqmAMV9iirquvZgNZaz5yv/TvPsQtCoiya3mf6pr9OnxPeMjuvq+nBFJp8kZg2QymPO5rt0n3A1cQOAZ0UPnh21mpbjtiAUxZSfoCU9E/cKVLtEfbdtAiGa6mmIc4wiQgRq1rX1OM0jWOUYNR9PhxxHx6wpgWkd2Ll6jknEbh8OKQsRT3MkRFObYxYRAyiiWVNMKWezMpkEWS0vxFoBM0dUh2rKcY6gak1bFWYTMgfvnPPOcUppu9lcXVz88cN7NGDmvh9yzlHFJPXjeH152bXd8Xg4PD68ffMN180PP3y32+67uv3db//+ctMN3h2n+ePtF0d0eXGlCGL46dO7dDoS49ff/ll/ug9VVzTzUYt5aBlqOk9xmC5pJYADMFERlRKZVcW7gIi67ODFcqOXmT01tFX0dTW+NYiDwbLaGaDQn/mc262RDgFW9TmRnJJIXlb5LNT9n3j6c7sop0RMGvMadRcLXbHes/tciO5qRgC2KC0v7Bo4O0GDwhzFp3pr9ZAGy/rDAmuu45rL1g9bX6BYvhoiLU9sBgiGeB5GWf1qWbMEjtillMtnzCLjlBCyqJU6pq5CcCiSEbya5iRMtN20O9B+nB8lDykP46yiVRViygAQvKvqKqUEgDGbyawqRUFHwdCU1yn4kp+54EUFVStedm9Xwfvg99tt3TaO0Mz2uwvv/e3d7b5tQXTO6dgP/TgR4nbTVt61TTPM84/v37VNQ879+Mffd93m1YtXd7cf97vdfnfhvD+ejuPQf/funQb/6tXb4+HxH//hb9M8/+qXv9psvzDi5usLW+jwGgBKwnQe1zSzLLlI8SAWxExKTV6mPNTUMZuirtt3yx94qpiXe1lo+biMJwJqWT9ugIrIT9j1spUFz65pdWUl2cgiWsYTnu12sXKiYDFBU1Xisk3oyX8aPmWOCzVkhYMWuYViOSXaLvSi8jA8H4enHGEtXlbDXtBiWIujMztpSTRxbW0iAFKhv5XsG5ac1QxBTMGgsMUwpTzFPIwppaRmpdSum6YJbs7qxFqgqqpTzqhKYCnb4TQ9nsac1TEpEhFLlj6mXIfgPRIP45xSyqkUFQtrwS/zzMhEazqiKaYk2QCaurrc7oCgaWoAPB5PhHh9eZEl398PouqcyyLBe+94itEHf+yH3/34fkr54vLCVC8urv7p9/8Yc7q8vLn7/FFNAen7H38AhJuLSzP75S9+WW13t18+f/jxu6ZpLi4uI8C7d38M7FzVtE2LzE23sWZjZt45Ji7WqaYxpUIzWLM+IEIClGWUv2x4fAq7Kc2LldiiGgyICFRi8AK7qqy3r9w/QeSnnK/cZVNb3wPiUy/2yb5K/H1WERdvVbykFZBVZdGkJVp7Rc9tbX0xfIrQa2pamKZPFVUhNNia/xTLfNYNwqdGkT0p6BWPWV5DdS2zSule0tmnrHRx6mbm5pjAICbJWbxnx6hmIjpPc+UdVt4MuLCO2c1JYlJTGcY55cyIGUBFDDCmOIxCCETglqkIK8a3MM8BHbF3jpmYmZ3LWaA0UQC9c9McY8oxRwyBs+Q0A8DlxRYMPn6+jTGG4AGw8h4BArHv6ix5irHThrxDw8vrm4+fP8U4v337zd/+/d+Nx9Nm2xnCOIw+BMt5f3n1+uVX96fTH7/7XV1XQz98+fLl17/5y5TTOM23/+7/4xG+evP2xatvDv5xv79SCYSYVyGgkuQtbgnRyrYcRFEEQJHs2CEgM+esxWmBKgEoMZqpChEXB1xIXmaiOWnO5D0RQ7loy7xlua1lcGjZdyo55ZSW8FeOBDwVFksNshbnZ1DIRBbB/GJuqrAKKp01vNbzsox+PquknhCf5VydE8dzYbg693ORtuTThmWOdAkBq+9dnhPxfC7O/twQwBTPTXkwxz7klMuZ8Y6JfEoiElU1paRgnjllnWOifhBTUTVVJuyampGkH89oHKEhUk55xsjOMaIZJpEygwtgIpJyxgLBSFYzx+ydA8CSpKac5yxM0k9T63zd1I/9+HgcmjrEnKecdm0nWaY4dV3TVvVp6KcYk1ntQ1XVHz59+u4P371+8+bd+x8Dgdbhy5fb4zDGlK4uLm5uXoSmiXP89PHd5Wbb1HWeY319w4C//aff37x49bd/8+9f3Vy7UP3dP/zdX/z6L+ebPlRN1bSIxMwhVLR0vJQdn+//Ot5UdAMyswOwLJJyWu9kyUORic20JJqw4FDFnSiqAvPKGIH1flrxj5qlCFLM0xjjbGoGhkaL5xFd8wAzKN3bpRhbK2wrKxFWwGyt/JDOQfrML1JYhOXxTBxerPVcSz0VW2cY9AwB4NnMivktaTTg+owAZ3r0E3awAvzFrksaUJ4QXdt1x+OxFGwpSXmrTCiCc8zTOHnuxjmmlLabpgqOmHxw6r2oVlVV1ZXkHLOM05xSVjUkMjBRyTmrCiGWJxVVAJQ455x8Dt45dIwOF5U8R66umIgJGawJoWKXY+rnSbKodE3dAKOqzpr7GBsfvHdJNac8TdPd4/HDw+nv/ubfv7i5dsGneQ7OHY+37z5+BsQ6eCYapylnvX+4y9MUmsbMfvbNt7vdxfc/fG+it18+OUQR+b//9//dz968vf386eOHHy8vrtpu02z3l9evgg/2VJlClAwAniBpzpKYHBhkSSK5jFUgErrFJhQFiZdweXYPhEhc8FPNCZDYLbtBwNCQSxFpalaYpGnOklU0a8Yy0KxqJuUtrcjmGoqX/yx3XmHtkRWzL4iCySo3QmVH0rMVXUuye0aFDADLiponn7omjKV0On89++vK+i5+8qy2eH6Y6TM4dUEfdHXYAADgXl5fmsg4TM4bO5UsKWY1IGZE6vtJRDdtzVVIKRECCQpReWNE2LWBsD710xxz8OgcI2LhmKYsVoI7MSFhSkW3PIuKzoBQOzZRNW2qGj1TFlhTEwLo5ylnGebIiI/9OKVc12Gapl23ud7vg3Mh+K5rs5lmOfXDu8+fVPLP3ryZ+z7PM+93v/j5z/e73e3D48PD4fv37yNY03af3v+432xjjI7dj+/fnaZ5mqdfffv1x0+fLvbb4+Hw4mJfef/+4wfHdDw8MuLNzQvLOV1eV1VjAN4FE02aHbEQZ82mqiClrBbNZ0wRlq6IIdHCMTc833qThVpK7CRHS8kQFRwVfsBaPpeSSCTHeVqTJVBTNCqJqYE9MZUBDHTF/FeqK6LpQrrGRfprjcugVlQQbdnDsVq4rdH8yaDU7Mw8xuVUrMZoAAAKdtYggLM12+pX7VlDy85w/pKYPund2E8M3PXDmLJ47wAw5xycY6JpmhHNOQKAnHWaEzOfevXBVz6ktLAumqZqKp/FQuVfXu+KUZ7GeZzigkQQEaGI0uK6nxcQsQ5BTeMsaujUERKATTE2REwcswxTFMlKdNbDceyOpz6myETe+Vc319f7C+f83f3dzX7XEL37/Onjx89dXX9NvO82X79544Nnoq+Cv/7q9b/727853h/2m+7icq+q+/32TVV9/DiMw+C9+/Nf/uJ3f/je1Mbh+Ob1W+/dH77/Pkve7XaHu89//MM/fvXV21DVdbclYnauabeDSJJYh2ZJrtZUzJ6xKwzQ+VBcqaiZChgsfA4AAEUm0lUxgKQUX7AgB6o5l7nNlauiALrkkiq2SM2fsaaFS28ABEuHfXGLRYV2ibrP5RdtdbdwLnpsIcoAgKEaPBU9qGC0ppJrNQ7FjBfuz3OEoER5BSRAfXLpq9z4OUeys5ozrP6/fAw3p5QlL5mAihKGyiNBirnsPDJQIvRZ6qbyvmLnppjnrJ4UoSrLgOq6Ct6PU/z45Z6Jm7qOqdRGwMw5C1iZ6CU1LZ5D1KYYg/dokHJCRF9z27ZVFdqqhiW82ThKzoominjqh1I4b7t2s+kIoAr+5YuXMaVtW2fVx+NxmqY0zxcXF/v9TsGY+XK/u7m6urq8fv/p83g4jeOYyvQSIjPfPd4fjkdT/dW33xCiiDw8Hl6/vLm82H+5vWWmHz/effN2bJoBTaahlxQPD18Acbe9lJzqpkOirIKIiIxUGCRWEr6zUKgVEQQELHPlCFDUhnSBFYtmTvFJarks1pWcJGfNKeU0p0UxGBFVwExtAbkWrpGB4TKVtqCturi/kmguosVrP9UWSsC5vwolSVzWKZg+ddLOmWxxo2gA9KxFCSuqD2e/f3aF51T1qWlUMktcIfw1xqyyPKtnXio+QNc2NQMM3vXjpKpgSgjOOREtA8kIBGbB0aapxznOMYJBTjmBNXVu6ooQx5jnmEUtpkxmITCgiZbN2yriRAUkkyoYnrnXMSU1YyQQUJGYU1a5dpel7eoc++D6EdQ0CUA0MCOm4EIUeXg4pG13HMemP+23G2badJv9fv/u/TsQ9d41VfDMMUUD3HQbQ3q4v/3zX/383afPn7/cEyI693A4uo+fReTmcj/N0+Px0NYhuMu6rgqmO07zpm23bX08Pswxtu0WEKrgRfI8nZCpblrvAwCknAByQXKLTFcp1WHp1WBeEH7IOcOKqpSVZ7jeYBWFs+tVlRjnNOcUc86Sc85pwXdK+DRd88mSZS4pxNpKOtfpdvZYxZ2XDNcWMPQpzyy1O61Li5dAfi5h1r75Cvg/83bnBy4fakl8n2eoZ7D2J+XQ+hJrqHkq7lf/C65t6qaqnHdlGKJoVDChGaQskkU0xRQBAMmFyoNqzFlzruoqqw1z9s4R4ek0MHNThxRT2SxjANbYFLzcH2QWWMDAp/kVMyNE9AgAIoKqWVQVLrab7abz7Jhy5TgwE7NjIoBd1znnHk9HMUCDz59vD8fDVzcv6rqAtm6/2e67TkRPp36co3fsvdteXH1490PbNiGEKc4Xm+7hcDwceyQCkctNV4fq/vEQvM+iZvrh05eu3f7+D98/PD4iwHd//GHT1k3TgGmOU1XtiB0Ano6HZnvhfAVYHKLmZM75pde5msqSc6qmnBaitCgYIDGVsXNVMy3cZFAoSnwqZTvIlGJUEQNVlbLg+sl4ylc5AWujG88LiwtOuVZPz1ua6xDdancGRsumw2clzLNhkTPiDmiwHBJYnxEBlzO5ErWW/AFxwUefg/p4zjaXRPWMiRWaCK4VfxnIdwgYKj/GOQRP2MwTD+M8z7OqFsQPzAiJmFJKpVt1GmdTBaS6aVmAHAFQzNliQiLvHSMGRwaYxLIXcmyTCZxR3+X9qoElZRVmt2areZhmDJ68I+eAqKqqrqouLnZJpAm+Cv7Uj4YbSXI8nk6E33792rHLOX/6/JmdQ4DPd/fjODKzAjim3WbXHw/7rmubGsF2m/bvf/u7vp9ELTgHYA+Hx3mevPfH03BztVcVIgJNc5xVJSV9eDyo6X5/pap395+nmK8uL8dpHMbeOc7TWLcbKrJTgCKZ2BXRKGIHBqpZl9TTJGV2zN5pXsY1i12W6geITKRMHeWcUoqSs4pITstocOGWPsGEz4J8ufOEQOuylzNgeW7hrKZ3JhQtRlh8KoIZnFuQ5ycv40IE+ATWL65vVRhbSiQ711SI+IT72/qEBbh66tsvD17fJpgV7VIDWPcuA7i7h8cXl7vah8izCSPzdtMh4jQnYfHgiLmqAiL1w9SDOWYfvCrPcybTTVsz0ZBSXdcxikj2zJuuEZFhnKY5TXNyzjFRybtyVitI4HI+IYuAgXMumxkii8qcBjftd7vdtkveXe33b1++OI6Dc3y13zl2D4fT6dRHScfTMCb5cHurWU7zlHPe1LUB5pya4Ku6mlM69KcUZxWbU2yrepymLNpt2kbyxXYzTdP9wwEM1OzXv/r5m5fXovrjhw/9ODimnOTL3UPK8g3xZnfx+eH++x/et93xuh+OD/fj0P8qi4lImhGZiFyoyHtyys6LZsrJl12jOSOAqgCgqjIW78kGqqLE3gBVcrFByTmLzNMoOYmKgiosY/gLYgMLfWNJDkzXOgNFtRTrstAAFoB2NaMFzCle8Mx2IwDQ83DSmm/YatxmaKBriVtwAFh75etbOhf7TzATLm7IbBlBO9t9yS8AziO25av0QRZ/utRPrj/1cZqK0qepwhzZcagqdkNNVU650CXnecpZidk7V6ux9yr57uHYT9FUg3dVFarAwyRZVdSISAEAiZkZ0TmHiIVejkWNSu3MNCvnuPTysmQ1lSyaUl2F0DZVFbKKY3bEaMjE2+2mbRvN8vv5h9M4xml+PB2naUazT6qO+XK/e3l9dbXfV1X17tOnH959GMZxGqcpxs1mc3Ox7do6xnT38LjtWlV7PPX/8i/+7Nc//yZJ/nJ39/s/vMtqwXERsDud+v/4u9+/fvPNTHA3zH/8dPtmnOd++N0//f7h0P+Lf/GXu92pcd774EPl62azu1yKJMkSah+qMqFR3F4pPBGBijwTqJosBZNIQftTimqaJUtKpcu5AqIFk8en0LuEzadypfjFUjTj+ohFDRPXqLo6UzrDkKuV6FK2r89cCCCLn1vLdizmpE91efG0uKpEPOWWa1qwUpFLurxWTctvnmcVzxZ+fofucOwRkZ3bbjehrjeAMaWYkndeJINBzjmmVAZ3QEUEY85d8ORDyhJPvWNOIkDUNaEKDgBENKoycxVIRZBwKTOXiqFcbQAQQkQiJSRgIlRRRqir0FbVNM0GVoUw0BgcAeAUo6l556qqIsLD6ThM09QP4zAeTv0cIwIQU11Vl7utqX748tkUvOP9pr29uz+eeud407XHcVK1u8cjItZVYObf/Opn37x9lVX7/jSOU9s20zSpalWHn3/z1ePhlFL+7rt/+vWv/+z1zXXvXR5Or66vQWZCPTw+quTBeQS4vLpxOTG7qttKnGGdDXI+lFOoJmBYpmZIlZgRwETLP5JzyjHnJJI155K72iKV94SFLzQ5WPpOVhL6c1xagRxcCpW1Vl6yghUtX/PjJ/LA8s3FQa6Gu/yIFnMtAfgpjUQAKmqMT4CnmgEtT7u8UTsH8lIuPWvCrVIrT19rgQ8A4C4udmYmIsd+qELYdM2ph6oK3rmc8zQnG0a30BwppVy0FOdpZudCVQXvCE2TpJTDflNVVYzJzBoXqiocjv04jEzkvXPOaZaY5QlaKDnVgtkSoTMEQyTmdrfpp/n21Hsam354PBx98HVdBWIi6odhjvHj7e2HT1/iNCPAZtN12K2QDc4x/fHDx92m27TtOM9meLHfFRIwGXji+2M/TXPbNsMw5ZxSSh8+fd51fcy5CuHNq+vjqf98+2gizru//LOfuVCJ5q3XVz//um1+8/B4C0D7XdvUVdd1VdUBkqo2XRdCVS6tr2qRhEhZMwiBmWp+DkOrgmVcrAELyJdzjDFFyVlywem0cMll1YkodcxSzdjzruMTxoxr4vjMFa1hdGXCwTlJffKAK3wL51D/pDuiazuyhLtzeSSqRnp2eM/B1vIiq09VtIXpjE/nxOA55o9wttq1HWauqsIcU1bLcwQDFQak66urOSbJqZqjqmAi7xiJYZxySmXpAswREL1z45xEJGdhpqqqADDnxIRzyqdhGueUk4TgHbsZ4notjBANsaDRSzWmQkQG1o/jdd7dXOxzzsfHw+HY18G/uNzXIUwx5SzjPKmB937bNBLCzdXFVy9fxpSmeQ5VuH849McTAoro4XgChMv97mLXfazC7eHYz1OcYxX8zdVFP8XPd/c5JQEL3n/4+GXTdduuIYK2DteXm4vt5jRM96fx5rq52F82bRecC1Xz6tXbaRpD1RFR1zXeVz7UWYS8D3UDCCKJfe1DU6aZJMUzplNOJROtJfJiZCnFeZ5yTinOWfLC7Suw8XqMAQHLHCkUKchFXn7BiZ5F1hX3gZWuX6x3VWWCBQJavO0y6AwGIEs9DmcjO2eZZ5s7HzMzW6ux5bsrkADnR5UibMlxVdfE5KliKz+n5+gVLpwqBXBfvtwX6K4KgRDLCJvkNKecY0pZnQ/OOWYGM8mLuLo6ZmIQ7U89EGYRJsxZ5zQwYfC+9IHGOYpoFoljTHOUp+0CS9auxXmUTTMZkAwAR+2/Tz++fHndOj8hSZ7NLGdFxNMwHIdBTV9eXKRpnqa5qcOc8vtPnzdt5Z0jtcttGxinOToCIqqrSrMchzGrff3VKwNQyXUIj4c+xlxX4WGacpYvtw93D4df/2JTRMlF01cvLt+++urz7e39f/zu8XBwRDlLyrcvbubLy6uUzYUgpmIIgM75qm7jPKZ5qrqtgcV5cL5iZjPIKSKUFbumqpqTEbNzCGgGOc4xznGeYpxFpFTutvjNglCVuLoMnVpxoIqIQLROL2HZ/XkGbQyfxM/PsXaxicV+Vj+p68rG59+FMxp1xrJWu7GnX129/9k/r8H8jGqd04EltVv94zn9XTz36pLPlo2lSLp/OLRNBUhZlByLmpo6orZpBrOqot2uVbWc0jRH0QBgCuYBJEvOqeQ+KiohnPpxTtkxbdraOY4xoYGYzTHGORaWl5oRoq3Y7FNubgaqiECqHjGLHA595R2IJJE5JT+ObgieMHjHzJu2Gab58mL/9tWN815UxxjfffycYu7aZk4JweZ5ZuK80cv9znl3Xe03m1ZE+3749OXu4dg7x8H7m+vLeYrvjp/3280U09bo4Xh88+r6+upaALJq3VSmmtIc4yhG//j77/D7P75586aqq+DcjDEAzIC82YWmi8OJ5yk0XZY5zgORK+PzKmIay9U3tSxJRdh5IATElOI4DnGeYs7FenC5m1qwQTUjpNV/FUcMJfIyGhHpMhp65rydgaTVONdBElsodcsD1yQBV+7egkQ+C/5P7lCfynA4t9SfQU0FSyhHyM5euCQNYmAL4HD+3urjn3lcRCwCz+XzuJzzMFNTV4QUY1K1ENz+4qJtG2Ym1N1mk7I8HI7kfFXXh8MRwBLxbHNK2cxMFh6dqG27pm6qlPIwziK5TNHTAh0rITgmM1DVYqnlzZZ1RmoGZRmI6q7pri8vMthpmmIWFT0+HEikaZqL7Wbb1Z65q6td11R1NU2zmKWY2rpJHO8eHvt+KNhKVVf7lKtQpZSHND8cjllknOZQ+b/+q18Tu3fv3t3dH+aUuqaOKZ/6sR+GaU43lxfHfvhyf59S8j7c3h8u99s5SYrx4XD6/t2H+8fji5c3F5vN9eVlcEFyStPgQx1cmKcBCR25lEQtRxNids6X1vpCvlvmmRSZUoyScwkvjssoMzAZAIGKSjYreBMCMpArK1TtfEtLHFomTc7R3tbYuth7yQfWNvqS9i1i5uXYlEJ29R3FVlbyGwKYLK5x0T5eQACzsoDG1mhNhnbuRuFakRlgoRCUiqPwVdYhu5KcnNNoOD8MwJlZnEYH5p0rWgSqELxXEQCLMQ/j1DT1brOZY5rnOE2zmbJXAxQdC26SJZ81VVSUEERV1VKSmFIxU1szKaRS1Zdc6Xy9lgyKCrFScn/q1UxyNsmaZeyzY2q69hTj4dOn2/uqCt4xA+ChH+7uHx4eDzkmBGDvysXt2padG6f5+z/+yLxsjClMqz//5c9evbh+9+HjtutE9OXNRds0x37YbjY5p4sdiOT3Hz7fH45NXTFxCH6Omcl9fHy4eziM0/xPv/99jLHfbhxgW7fBeVVNcSq8u3kYoKpNNUsyU5mn4AM7j4CGRkhnEDFN4zT2c5wJlIJDMC4SlQAm2cSySrQsksyQ2NkSuBdsbknxCtKJiMXLrpno+bqW0snW4I2rB3sGkp9L7OdpACwgFyCu436rokLJ0Jb/eSZsuy5OXvnNuAjrruG7hHRd84zFwJfS6uyKz87LYUGFUqpyruttacE/HE+OKcZ0PJ2mOXYxHvspptzWoW3rUHlHNIzzPcI0zRqFkGLK0zhpzv2i407OOchyrtPVQAxMDVTWDKN8FDhf2uINRDVOcYDBED1yW9UnHUz0dOoR4fLiotk0fYxzSpumxhHuHx4fjsftpiuicG3bjvNUlj/d3d3NKRFi0zRmqmKIuOmaH9+9/3J/3zVN1zZNU7+8vnCOQ1Vd7PZm+k/ffTdHff/5lgjfvLz2jkNgJP/4eArOgxoZSMy3n28t57ap9/ttXQWwMig0W8k0cyRya3RGydHMnHOEfHZRojnGKaVZJRWGPhESIROqKJhoTpKipBlVaQGmzFZl01L02KIYa7zQ5sH0+TDqkvHbOadcSylEOI/4LsTNNXdcy6IVxcSlsb7a8FLvrdXQalII5yUjT8nkGX1aC6sSNLW8h7WHUI4BPU9YAQDRVVUoW89MlUodENPj4fjier9p64fjaZrjbtshoedl5crFdlN5R3g8nbyKmBo7z46z5Kyy6dqmrQlJ1UQtx5whQ1EgkDPndb1F5Qroyj1bpNUWzAERxKSufExuzDPEeDyail7InoJ37ACwH+fb+/vdZvOLb79x3h37/vHhMAzTaRglppwTMHn2cV42iLJ3SPTjx89fv35FiA+P8a//6s8v99txnrfdtm0aM3375qthmP/ww485yeHYbzdtXfm23d09HKYYv3r1IqbkGLfbdretm9qlFEXEOVs6OqXmMwU2QCgk7JJpmZloJnJLLqdqKqYCmkspC8CgIKo5JU3TPPXTNKlmACAkYkNEIzV0WhpAuojIF20INSurpwDPFQwUJfZSbi9bFQCWSgB04evgYnDnGuicGCxhzbQoQeMZJTzDorbknraIvwMtNXjJMBcm6Yqwltuu5ZlLfxSXE2Nr6xPOR8AhQJG9iCk9PByy6ByjY3e931Dt6+CHcZpj3jTN3f3jaRhNtR/GTdfKwrsBZhLRnLKIFNLUpmvbtp3mKKLa1OWNapaMshSaz5gLiFhS0sL/MUSYiySONW0NhQyKVD5MznI89dM0b7q2auq7xwfLUp7l3fuPx74fpinO8zRFyQkVyBECDFNvIuxcCKG28Jjz61cvguMvdw83lxfbrjv2PRiEEMTADC4url9c0bsPH/7w/buPn28/fbl7/dVN225fXl89+Idv3ry5utyd+tN+t7u62O/3F03TMTM8u7trI7d4NTAs1Fheot7iXiTlmNI8T4PmCCCBUAkFUFVTitM8zTGKKpqAWRlkgqzeBSVRK7RxBMDiYpTIMS1usHBSy0/hp2o2WCLVApESLCZki7jf01dxgGfwH5ah5CUZhRV4R4RC7afVny5eeWXzrYo9oAhPNCszxDO7dC29znDX0ogwV1c+JhymGLOcTj0RBe+cd5/vDs6dALAKYbE9xMq7LGoG/TAhQlNXkiXGJBJTSipCRNM0/uEPP4QqOO+rKtQhVI4fAFJKwXyClMSYyQdvBiK5iLMvufQyLqiSNKt657pNpyLOcVkYXCj6GezUD8McvePgXRXCMExfhrv+1BfB0VIpm5klAwAi8t674Nn7Kcab66vLy4vH43HbNt5zP/QAVldVERA9nfrNZkfO/ebPfqkqp+Px05f7FGOM88vr/X7bbLfbFzfXx9Oprpu6akJVBx8KVAfPv5aKBRdqE5GBETECIaKq5JzSNE7DaRgOKBlNlAwMRHLKaY5pSsuwHq2VSMlCk0cgBWQDVDVEcs45ZkQUNUYzKFL1CGcwfJlORrVi17hCAmcbKpXKgrgv9m2ABFqmSZ+sfCnRC7BQyCZlBmABws7wPuAa4RGL7hWU8aiSDyMY8FJAL2+3GPg5IzAAZ0jEzjlNKY0pEbNqBYBjnszMOb/fdo758XAcx8kH75xjxL4fYozOORd8VnXqYozFf8SYRWWc5q5r99sNIk0pOuamaUacsxqD1HW13+1EpO/7mBIIruSpBYBDNTM5HY5FWGae5mUAHEALImWJRXLE6H2Y0xzjNE3lUBZ421YADgm9dyEE7zlLQqS2ae4eHtGgl3Ga59cvrq+vLh+Px4uL6+1mezjczfOYstvvL3/182/jNLbN+8dTXyCI3c2NITd123Z7ACPi0ujLOS83dVn+fTbLsscBl5mPp7TNUpofD7e3t58kjpAjqjgw1JzFprJ52kAAkZgJiR2wA3Jski0hKnPhvSPzqrhZcgRaQCiiMv1WOA+LEdDqHBcLWRuU5dqKLFjAcsitNIBWsGhFjojwJ2b0VKcvhlf+FITnjCzo2u4ssZtwmUQ/Aw8r/Ln2HswAwJUqpq4bQxz7QWOa5rnsCTbCbdfmXMUYqyqI6BzjNEUmSCnnnLNoXdelSxRzjlmKClfpMLHjKaby5bgMLmdiUtOU8zhOTVtvNpspzillSWmeY2k5AJGYMVoWmGNk50SfWhBqCgKFCI2ISXJCLnvEbf0qRo4AzAxExFRXfo5pnueqqt5/+Nh1m66p1ezt6xfMZAbOeSJMKW67/eF0PPWfr69e7PeXqe1Cu/ub//C3265pmrqqKnIVs/fOWyljVwlZK3xXeMJ3ibjIJgMRLNW3AnAJ+ExODWOKp4fHNE8oCSRbFgETRSAUIGXnmBw7YGVnzgGgslNmR2zeO2ZHVNZSo5qgQtn6DQaiBgDFDkqeVGyjEJ0WV1lo/kvpsoA+a7fp3M2Bc0JWCBVl+zzC8wnntY+12nGxxMX4DIpYry47P+BcTj0dl3VZyvmZiz92m+1mGMbgXV373abtx2nsRzNj79i5rmsN8dOX+xBcCKHt2hxjylKchKj1w1BOmveubZuUUk4JkIo+o5l1bTPOpKJoggjBOUJUUSYMzicUyskxo0HOIks9alBwXVRMqUiAr5XHCqcgEhMTmYGUr6KRuyLNpcvimJ3juqpSSnNKRpSLDDT7fhic4/1u808/fLi+GF6/vL5/uD/54367HafheDzst1vv64fD8XJ/9c3bt1Vdh1BhMS0kMyNiRDaRdTYI1y8oeQUgItHTfYClE5NklhTH8SQiTdXmNinQPPQx55yWvrshKRqiOmbvzcgoWxUMiUmNSJ0zREMEpmWrYVnnp1IolUtqsTCTDEUXUJwJrbQ0AQqdTBdjWqr2pUBZYKIzBL/8t0D/+GRb68+xwE1rG+n8W2t/vpwJwnPjdkVs16przd/h7IZNzeWcg3dNXTvPAKTwOI0TiAXvEanw5It8vZltKs9NdewHxOrUj9M05Tkuu7+c224753x/PB1Px5QTR8eurFxlJhYil5JlSWZECITTVFSbivSIOeaVPLMc3DKbz6p1FRAwznNSQUDHjr3zzi3D+yKmiiXpXvsiZZ0XM4vY8dQjknO8CF0RH09HU+u23cfPXx6Pp6EfwaypQxZ5+/p1XTUA+PhwD8j9NKvq26/eYMkg2RGxAZZhYCyrjvGcQS0V3QKcreopiIiFSwyQ8jScDsfj4zCcSvkfQiVSpG/IKKIsGm4lAOecUxbDDMijS84xO0fEwYeUXF1VWgVRVRe8d6svt3NvxgAJQeFp9ZGuE54GZdoTABbyUSlnVxTgGTj67IytBrloTZbwX6buyaiwLM4oTXmiUi+qAZ/j+YKDL667nOpl09cZDTVTMDcM4+V+d3mxH6bp0I8AEOp6HkYRIbY8ZzFlwipUwPzw8Ng2NSzPguycD5aHcZqmk0o1VdeXl+2my6ZxmmKKNFMIgYl16d4zB984NjNQjbnMziuY5YV3boBIRCKLoiQgECIQIyAHj+rK+u4SrpjZQJkpxlTaWk/HVoXJkXOBnUimBc5TNJCUsgp7j4D39w+FVgIAbVNnydOU/uLPfl5X9T/8/vsXVxddt5Wc5mm4uHyhgGZISLZQLSyLQEkkzplTwcdwEbl7MtBS7ZqJ5HEaHw8P0zRIWnOglGLMomjkAIyxiJRp0dMp4AgSZNGYiDiFEJiJjRVMDQiZCEWtnINyBdbeh5mZ5nOfU3Hl4S4AT0HG1ppmgTmXuaU1Pq9/oXNnaHGm628uHxlMQdath7ikp0tqscKquHbrF+9Lq/Nex51/0iZ1bV23TV2Gg2vvQURCdoRV8N4HVRnH6XTqcxqqugohlAaQijBzW9fZ+5Qz5wyIaY6fPn6qqsoHj8SQc8556HvnPRGBllm2SsxATSTHlJJkWxp5Aqti2RIW194SMzsiAUMjzwucR0TddrPpmqEfHh+PAKl8BFyPLQAyc9e1Taicd4QQY4opDf2QUyZEx06ylCGklNIwTqqKCN/98V0InhC+3B1ykr/6ixdd10hOMcW2u4BlldGCv5TUU02Zym5EMCvq+7S60dU4kYrcvHceF5AX1ExMk+gU8xSzKx0OAgUC5lLiOEQHAMSGTMSAWOAR74N3ntGZgUpWAiQnsDQwF3LQk5kiFEp4wSuhvJt1+LiEeIIiTrhy6Qvw9CdPhavBLvG4YOjwfBhkSQ7OOaYVrAuBVO1MeVk9CRhaliUxXds1CwiFAO50Og3jVFWhpGuA4IMn5pxzSqMPjp1j5yRLjFFVYyQEqOtAaCKSVcvAmkiWLPM8j+M0znPZRpTmmHN2zjFSUSZiJkRi76qqUgBOCcg0ZStjuGC2rLJ6ymxKsKtCUK8i6gCRuW6qtmmc91VTXyD1Y5jXKr74g65rry8v2bGqbruOGdVgHGfv3IFOOUuZKwcDEzsdTz74/W47x/jp9g4Ad9tOlU7DPE39zc01AJqh5BxCXZCgM+RpsHTY0bC4f1ySv2XestxGJFpq0zINl6OUoWRywQOYOWZYKjwzMzBrVKSUzwAA66aaEh/VRCQCZBEimh3XIYQQmBlpWaRbxgFWi1m9KqyHo8hKWonVpcO8QHLPQvozJVsAU80ATFTCMaxaOgvsClC6LeXt4dIseHqGwlx7KreeJHDO2Wtpi0KR/St+1h1PAxOlVNd1oIR1XbdN/Xg4ZREVTSKIGOqKgKZpHMdpnmd23OWmaZrC09l2Tds2p364u39ERPbujOIyMTKpyBCnsv7HETtmcux2O+9YqjpzFoNxGJd+nFkWwTUyAqD3vttunPcxxpyFwAxMVU/9oCervKur0HQtgKFaKZUMoGqqTdc1oRrGQVRR6auXNynLp+C6bXc69sMwlfbPlNLh8egrH95+dRrldOqnYazrutl0VxcX7z9+2e8urm++YhcQIOdYIratze51P5CtIX5B6UusffKhBoAgItM8HE/HYZhMs2d2zokCsqKCSjZAIvKOiYhpDdCqWXROOueckqSkZaifHXt23vlQOQAEJKfmPAIgE6rqUtEXGUe0M8t9yYJWMoQuNCV8yl9XJmTxEPhk5nYmSZfIvSp2L2nl2fqWMH6ePF4h0jUleMKVCjRTsl4pqmx6dtfgYoyFPDuNIzG/uHZNU1dVDYDGOsfEjJcX+7ZppmmepvH+/nEch5zySQfnuKlrBX/sx9MwZlVyblNVOeXCCmXvETHlPI5jnOeUEiLmlDWlTdddXF5cEt3dP07TjFjU/J8CQ/nQzOyrOlRB1nwsiw7zlFJm4q5ti9txhJuuDcxmFkplxuyD2293zNdgNs1z1zRf7u69921dN3UlYo+Ph3mamCnGrFkOh1OOiRD7YUw5i9nrVy+ato0pppR8aFyockoqWVWLCvOStC2hcyVglJoDCYmw/FmRxSxpmqZpGud50pwyEzMjgGdCY2VCAMfkHTMioRGqqmQpUgk2RxmneZpTUkMi77hyvmsbH7rF+NY3Y7bARes0CBVDNANaS8lzwrrkhmsyivTUOoJnDm+trZfwVip9Lfsd1rGTUhKVfSlECGtdf75Q53xjQbbsHNwXX6uLR14AcWeqQCg5IyEjz3HGExX1wGma1KzyjWQZ57mq3HZ7fXmxu729/3x7P/QDmE1hqurKee+8++rFjfcOAI7H0+HYRxEHgER1CLvNJub0+fZuGkdVaZrGAMZ+IMdF/kpXbwRqsAKZzByqwAj9sTfQlCTFGQAIIDCHqvKeYVHzQDLwTN57JCoMnKaqEGyepqap37x6iQgpp5xzjOnFzY3m7BnnubnYb6c5fv5y//n2vqqqEKpxnlXUIXaBX7140XU7YlfKSheCZsqQACBLWsH5pewsi2myZAfOsXtyKsVPIBCiqmURFUmiKWVCgDIpgcBEROgIVvKxJs1lNE2WYZCUYoxziiKGaHW9aZqubdqm9j5474nZMRNS8VNJFBEcczG9BWrEMsX5NDEHS3WyVtxPE55WcgpbqylY4UnEFapcLLeUtlIaEfQshVgvzrn0WZ5HVns/5xKm5b6X5sGyZcVpFkV0zvu6IqTDcbh/OJYyuSBH3jkwSbMeHw/eu8uL3cuXN8j88cPnaZrmGHNKoQreezSrm1pEnONQBTUlYjAz1aqubjZXSPTly21/OhV4KIswYt00oQrjOK54w/IxHFHwftO2zrlhGKdhLBVVoTUG7x1RkUuAnCvvqxBCCNM0IlLXdV3XMNE0z/Mc1bRrm65tX1xfVp6dr6qq+nJ35xjJuYvd7uHxUNfh9vYRifb7bZmrRDQi9M6FqsaV/6U5mRkzgREbq503Fa29uaVaV5HM4IAYgGDBtJGQQwhV8EsdgyQqKcWUpTgSzxgdES40ZQQtLKEi4EpgnjARAFBVN9eX+6vLy65r6yoQO8eMS0wH0Gc7txft+iVNVls2/gkAAwAUKjSoWqmTnhzbiggsQX/NX5dLYWZarH11jUimKmvgXrzgs448IJTXWoGwdRh55Vkv4JSBrWssXCpnkWIScd4557qma5tWVUSyrZV1TGmcZxU5nfrdbnt9dQFmX77cl0gXU5pjmubYzrEKgZkckyn64FW0H8cxxg+fPhFQ1zRFZX2aJnbOORdUDWzZBFj6WgAIRs6Ftokip2FIMWph7gEQEhAZQFKriGrvfPBqdup7s4UC1o+jqrZNNc+RiB4PJwO4vkhVVbdtJ6qq0g+9qe27jhGPp1PlfV1X7Fzwvu8Hx1RVNTuva1Ve9GIRUFUk5wWct+X2LJo0ZT0Nu2X5UWksFXkDREQUyVkkppyyppyhCCESlQEFM9UEEyzlFyM4QiYEYkBgIiarA3vfuarebrcXu31T184xlm7S2iQoyh9rzAcRMUJCgpVTVMBzpsXmVBUBDQt+hmuGWlJDKDXeikYt7nRtiJRyTp87VFxnoJ8wPztDK2CmgMtIdHlWXeP+uV+1Ml4BAEprfUlmEdAxe+9TSkmSiuYspfh3TMGxmn78fHc69TfXV6pWVpoSczkxkvPpeBqY6rpGIiaqEZHRAIa+Tyk5dtc3N7s9jcMIZkRUhVCc5ZKp2DLwh0xMbGJjnHLZnarGRIwEjN57JvSePfESIAlNCxk5jcNoiNeXFzFGJAzB13Wlau8/fqmCB4C2qXbbXV3V9/ePP7z/xzcvr8tNuNhtx2l++9WrLHkcxhB8FSrnvAEsx7WEJDWRnKVMXUqpLXipaNAWZSxaoPD1sq9RTCTnQn5bq3oDQueY0HK24hRK+1TAMqJzzKxIXHzUZts1TVe3XdNsmNgARK103WHFtmzx6LYAQFjE5stWkKWTtLxyGRp+FnlXZZ3nRmaotLI/zUpEWEqcxSjPqNDaKQV4srznF2F9agPAkr+u8PCaBz+lHWYG4JquNTMmBgQkjCmZQRUCGJhomTrJEhM+QQ6H0zBMMyM758ocZhWCimYRx7zdds7xMEXnnJg4dteXF3NM4zSaQRUYknG3IQTHTEybrgXV27uHRcC/pDvEQCR54d2pmiOq6trMFDQE31Y1EBWjAUQ2E8RDjNM8qSgifkiprarNtvPeV1U1DFPOWQ3meXbOp5y7pn5xsZWcfv+HH0vNXIUQk1xc7HebTYqxbaqqqgpSAQCqYklK/00k55xNZalGS/FROptES9JmRcjWFJQIkDCnPMd5mKYpJjNzXGyFAMAxxag5WUqiIqBSArT3DsuoFmIIoa6but003ab0uuaUSyuOEN2yFIWR2GwVU16rt5I5rdUPIQCcx0oLE6kc86LcXswUccF6V31aW0eS1gpqrcbObSsoYuWGAMtQgC2WVox1IR8uQrmrehmeSSa2NEp1ZUUBOO89GjjnRBUQnOOyAfviYi8q93f30zQXvFBVJWfvXQh+nqQAwq6cfTAlc94XHlOZD4wpxZybGqsQ1Gn2vq1rAGDm64vtHNPd4+M0joExeD53BQ2AiOq2qdqNSpJBHTtw3rFDQgKoHTNREiEzds57n2IcYoICkZgCuyx5enione+H7fHUI5GpXWy3vGmrur7Y7w6nU07p4eGh6DOOczycTkm0qcLxeCTiU9+XzNI7t6ZlRXRDRUUkiyQwO7c0ccXt1wTLcBm2XMBtUxXN8zyfTofj6TBNk2fk4IJjIlQmBJ3mFMVkTmaKCHUVgiNfusXOhaqu267bdCE0YDjOsZ/GApYRQ+V929SVeS5COrbgU2fnhYigi8hCSaiLuxUtK/LQAAqlgdcGREHH1kFkNLBz1X8GgYvpPqt2llC8+kpb6qTzFuc1A3jiJp6FzVY28xktRURXVVXOuWkrIhrG2cxc8Ju23W46M/XO9f1wOBxTSrTsldK+H4vYFTOHEABBshhYVdXsuOxUMFNGaOo6eBdTTDHWwVd11TYVAHZN049TTDFOU8r58nKPf/zRkhaYm733oUIiSUrO1XUNZR5a1FSkrNIgMoQ4p2kcUxYog9VgubSuRQzAN/T4eHg8nqrgzSzn/On+7qub67sqTCldbDdTnNnxy1c3CDCneH9/qCp/fbkfpth1m03XFEJCqc2hcJeWFa6GAEVMmcjR+U6t0anYZXERJWcpGfYwDqfTqe+HYRiDI4SKqWJiJqq9l7Y2kQkkpVwElACM0AJD8BQCegZSmcdhmPNpGMseFeewCgwMmjkvK0aX3cl/stoLDASFkBQAlzZBUc43QaRnB2ydVILzx1JVe95jx2Wnk1mpulYlvYXdaQBl6mqZiDz3mFafuuSwazBfsodSjZ1fBQzcfr83zd65lAXnhACXu+1u2wGBGXnvmqYuxA4wkJTGcSqfIaakoqYaqsCOmcgxVcGbOTFIMZqoGey23a5rDsNoprvtlomHYTicTjnn3abbb1tVu9xvf/bN19//4YcSetA5ruumaUNd5ZQgSxZhIMeWhQpqLSlLzlb2GqkhAJbtOUspBT6EJMoEdVWpKjGdTifn3f3D4fb+4c2rF3cim7bedt0U0+Vuy4QpppRyU1ebTVdVvGmb7bYtTQFmlsJaAss5F+9C5MyMaFkqe3Yr63V/lk2BmRbZkDnnWPRw1HSOyfFCzchqCOYYPXPZzEmgKiqE6ha37fM85jRFvevjnLIVAqggKpiKqqhktXPBsyy3KD6Scdl/XogjxfEr0ULFQxJEJHRET+gYPstDYdEQXz0gPCeL2ApqrqAbFDirLIFeGSnLA21F9Q20KE8U1kXpmp4helMDAueZfN2WOayri13BgcfpbrvtLi/2L64vd5sup/THD59jTKfj4fGRzCyLDv2gqnUd6roqSbMP3gCHfkgpzTGpaE65rcNXLy598HeHUxVC27bf//h+ntNm23775qZr24fD8eF4MoC6rlJMyNy0zdXl5W6z7cfx8Pg4xZTjrCKjiuZsa5timV5acnIzBTBBRCRkYgQUVXIMBC+vb64v9uW6DOPonDOAcY5EtNu4oR+2XUvsri53wzAZyPXl1W7bimRmMtUsUc0hoKioSokeJcU692bWlsgaAZc7eq6SDABSzvM898OUUkYERkCElHIpTHJKKSXJsuKV5lBBxRQkGZkKaB/jGGXKEAUNzDnHTLVDLmurwUTVs8Kic3jev2oAoFj2WayZcoEtzRCBkUqWSEAGkBHLB3ySOT1bGBaCvRGCyaq2gFhWJC4Z5EqJtDUrhVU3qlyeJROFFYGCFTpdW6ELpI8Iqm4YBvY+54ymTVNf7rfTFE/DAADOuU1bi+r98dTUgRGOJ+y6lhCO/dDUAQm994u5gOWcAVBUc1aE5Soc+/Hv/uN3dRUuL3ei+eFw3Gy64/ET4+by8qLttnMW5/zt7f2CZRC2TXNzdRXqKkp2jst1AijZn5oIwJPw3zIys+xEI1vYT+g8Bx9KzgAAPvhffPPm8dSf+rEKXlVD8Mdj/+7D59Mw1nVV1/XjYSCCUz+1zenm6pLYqaGa5pwQpOjDSE5lWXwxvTJBIWKAsJZTCE9Ts08VLgOC6uF0fHh8HMbRMXpytmx8N1NNKaeSAjJ7RgITQ2L0rmxN0WGYslq2wv6h4F1VcVfXPnjvGNk5dmKoBkxlXLgEFF1bkWVr3mo6q6vLGYAXGMzMSnOfzr0HW7n3sJZ9z5qU8LSgttjb8mC1Yr4L/LomrE8Zgy1lOwAA4wLJLWnnuco3AwB3d/+w6TZ1UyFSFu28u9l27dCkGA/HU6mKzFRFiHC36fp+VFMfqpJalFXe7JiZ4xxjEgRyjpwLRNS0bRHo+nz7cDgNL24uQqhev3q57br7x+P7D59fvLBt12mjl/vtl89f2LHfbC6/enV1c01IKaY4TvMc8zyLyOIP1kjxFDsQC4Nsyf+ICoQxp8jOVRjGOJ+G8f2nL01bf/Xyep5jGQkZpul0OmXRlHIV9NAPY9+/+eplynLs+8v9pffBlk2F8RzwVGXN64u/hjLHLuukh63wYBmjLc5DVIjIsYNSPs6as6uzD56dK9W+sYHzbCulgwAITAxMdU5SVoUTZCZ0zgcOm9pt2hCCRyIpI8tAWVF0aQqnnM/+HlciSzFTVaJis2vtwsxY8Jx1Azetqq5gcEZZl9q+2Fk5rCsmtQaQJX1d2vFrXnvOD0qHae3+rzmnlbwWzhSAAgE4RIwpMaP3PKf8+HiQ3KkaOvd4ON3fPVzstyEEBdtvWu8dO5YsOethmr1j7z0i+uB3m/Z4Gvq+B6BpNmZXVSGlHOdY7KYfhvoYXr9qHdmbV1fb7ebdpy9ZPn395vW267569fLz3cPx1Fd13Xjf1cGHahz6j2hzjDEWreEzjF/QNWLPBTMmIlMpooMqYiKi5h07A0Qkg9NpyFmuL7aaJKtuu3aSVFfe8/7T7f3j8RRCeHl9daxCSokRqhDmOHehcezmec650OaLyt+512K69GOYtJACFHGhjBGWaaSlri1Oixm99545qUrOmZAZUYAJ0TSJgCktjGcgMFQVkZg1GywUIjMkrIkdagBFzRINwLJhQmbnEZ0ZpKzTnOZ5nqa5GMfCGyd2jokZkFxZH+icqhYBVyIEoLN3LB+yMAxU4Tx2b0tBsxhSacevQFKh7q92CWVIfw36pa5XKLgBLV53wT9X77OKmxgAmquaJqV0OA2qaip1FSTrpmu2bcfE4zhOc5xTUrUUZ88MAHNMYsbMxBScA4QQfEqZHG93u5TSME0ppeAdmA7DmESmcWy7ZrvtRHWc5inGTdt2Tf3p9q5pmov9/vrq+uu3r999+ARMofJ1XV1fXV9fXbndxvi3h8+c+n6eJhEpnsuHyjHbKjKMgFkFwUA0RpBShaQ8i6YUTye32W3die/u7pu63m/boW8Ld+bVzeV+t5li3nTdHKdf//LbYZqiSDkEw9B37YaZNceYo4oyoxHTmkURAREvgt/ETA4KOYCWYdqCg5aStyT6wVMVWDSDFehOsJBcEZkwJpslg6krZsrMzlceagQzUEBH4GmZaYpqGrNaVjMgBjJnRIwiNk5pGMd+mPphlJQBrMwXOOaq8iEEdhzKDKRzxKyqwTtjXpC1QouHwpVHXuc0114RnuuedWPzMi1ccP6lolqxzhKun0DUdVxOn1WSxSJlzecXFMzAtU0DdfV4PPVDjwop5XGKt85tu3p/sa+rKqdU9qjePx4QoK6qzaa9vtzfml3utm3XTNNUenebribCH999FBHnXFZrm4adi/MsOeck4zTvdtvjaXCOHKIjAKQ/vnv/9ds3N69fzznNcxzH2aY4joOjFzeXV5f7i6tu8x9++49fPnw8HQ5xHDVnAHDeNXXdVMHMspqKOCYDO51OAgCZVLKplMS/ClUc43GeEZEc3z9WF9tNEmnbpvIupkzMnum2H+I8v371YpynOaamcobSDz2W/cEGZpbFQJSf5jYL/5KXze8/BfQWwOlZyVLgd8ccfCBUhwAAOWsZYxMAYvKrriwxe0fOccWEoFDm1s1ETZGQyJCSARM7Ju88MhsyIuWc5jiP03Tqh1M/pHkq8bocIu+9d857R0zeOb8sPfdVVdVVVVUBQIkcEZ3ppMuuxRW0KslWaUrpOfVZu6ylT7mw6kzLaJqt4MaSpCOWPTq0WnlJBmytLBhpGZqL88zOeeeD95IyMYlpnud5nh+PQ1WFtm32u00IPsXkvNt03W7blvp9t9s0dXU4nD59vq2bum0qM1C1qqqc45ur/Xa7yWIq0m66z1/u7u7uL7bdy5dXMemU0qaprrftw3H4fHt7c/PCeb/dbcVs6vsvnz7uu2az2Ww3u7/61Z81df0fu/b9Dz88PDzmafaEvq6NCiccm8AhhHIMtt2mrrKUPwCqxgDTNI/z7J2rQpVFjqd+GCdAeEn0+e4wTVMIXkT7YVgmNAkeHk9vXr2s64a5SE1JycsK/aWYGjM5ZMJlCKRgpEwOyia4Qr0lOkMzi1cycwx1IAACMFVLOUczptJ5J+fJOy6rugkNzUiTqZYOpQAoEDISOWQXgg+LPiYhUTZOOReGSE4pzlOcxnmeUaVM0TEzMlNx8lwmI533vqpC29Rz3bRNs2lbH0yJjblIQiwZZylo1rrz3EfiYlhqpR6Sxa0uWJusZq1rQ2lpMsGZSrKSa5/lb1r6+4DueDwhESMwsWvctm0FIOUiO5z7YYwxxhjbtmna2jMHR455Ur25vmya6njqvWNAHMYpHE9zTOM0Vd4H5yRLCOGXr199ur33pz6nfOqHz3eP37x99e3bF5++3B5Pp21X13X98PD423/87RCn4MPlfp9i8sTHx8Px4b4Nzabbvn756svD7Xw6BaJxnsFMY0opiTko/e2iCWaGAMxUheCCr9pWpvnxcDyqkBQyvzla7o9jqrwv7uHh8TiM8xRnRjyehqv9lphSlm9ef1XX9TkKFW9ScileSh+ErAbiEFUZCpy/gNNnhKlwiCyLmGRTUYPKMyEpWBbNKWdREVMxIlRzxFx7xwRQyHlZRFKRVzVidr5yyExNFXZt40MooLkYgFjKBAAIKlkkZclJcwaVUrcbrPMoi4myY3bBzXOwMl9F5BwbgA/IRCIFHi3EDlQDybl8PCIs43dSahxYkgKzsjF3CdxSli3COa+FonTG5xbXOq+9QgV2/pGZuXEYAJGYQgie3TDNarrdbrY3V4dTfzgc52lOMSJAVQUkvH14TDm/uLnqNpsY5+1207WN9+724RhT7vshjhMBsOOH07C7SCH4n33z5re/+66qw+HUG8D7j7evXr64udzPMY7j5MjGcfzhhx/atlNCQuy6hgDv7h5MNQT/6tXby93u9c2Lx9vbY99LSnmeQbToOjvvyHFgbupqTpnBCLBq6zevXjh2X77cMeH11X4cp34YPLskeY6JEczg9vauqirn3P39PSI1dX0/nNq6lZy3281ut51S9N4T01LWApDZuTsPBqqCCCLFUz6BjrZ2QWE1U0JidlVV1VVIKaJpYUMyITpCsIJSSFa1TEgelsoJc0oxDSklEQNgdlWNVeVqh23FPriq8kQIarNoEiO0lLNkycXsRVXFRNZeF6KqFE4goiILUc5sat67kKosOcZExLR+TEQsi6zxzPVcNywv2czyuHMHH/AZc54WT7oQmYlWqGrZMbLSROCpcaVLegSA4BAJCb3ziNiPI5h1bVv2Zu26jtn1x2PKGZBijGAuJQnep5Rvbx9Siiml66vLqq62G4kxHQ6HpmtD8DHlNM+3n2+/a8Lbr17udtuPX+6r4DdtfXc4PRxOlxcXVQjTOBZN4X6cRQyZRPXF9bUPPuU8zfHx8WG7u9hu9t++fptzDN7fO344nXKWGqBI3Q7DoBBdCGKmgMwURIfTgIhlXoCYri52++1WVac5TvMcvD8cj48Pj23bXlxcIGJKkdgR8uPxgI6TCjNt2rr2IVA4RyWAIvDLS5GEi6oblcGeEqnUgIsM1TJxVrIrIvTe+xBCCDmnMx0MAIr/K6BjykaQQRTAQAQkx5SnLGU1cPBGjJK9SgbLYNmUDagkuWiGpqZaCJAp5aLOvlKiYSnenpYciKqiQKmAVFVWPcKy+GaF9E0AHRMRmloRn4BlkGON13YWMoFnlNAClC4VUklyFBCXtc+2etelNFrnR8r/GQI6XwcAIGbnHDtnqqZ2PJ6yyPXlbu+dSsZproIHw2GMTPR4ON3ePRpCCN4xl/H2yntE7DaditZ19fHj55zyoR++fHkIobq82P3s69cPjwdTy2a3d/ffvn2bLvafb+9ubx/6YR7nuQqx67o5p007brt267oQguU8DaftZn95+eLPQqib7vu20/cfjg8PMs6FA2pmQ8o6TioCiOT9cRjv7x9LJOo2m6YKpbk0DsOifQI2z7OKxhinadZlcasgsyENw9jVNSGNU9x0mRMqETOvedgCvkJRpyPjJXxhqUNwaTItzFwod0MlpZhzLuMoomKS5yjDNEvKK4Vy8RyaJBGoWhJRkUJOMTMiqEUNyPvkfHRjJHKESOSyLbRTQkOErGpgxX+W5I/PZwhsAXnKxyAkdrSo1xqYZpGU85KflEX2RLAoZ67wvgEuumKll1foJvYUOgrAtLTdbQWsy4UyWqceoDSl7Jwi4DpEurhZV3QZGNkAVHWKMcZYBV9VYZxiQcjalolpTqmsRCrn0nlXhdC1TRbxzg/jBGb77abwmolwf7G72O+qJjgmEdnvti+uL999/BxjOg3j4fD48ubm8Xj68d2n4/GUUp7GqdzPh4fH/eUFImiMh8MRAJq6vX755sXli227vby4cm3zH/727x8fjjnmMjBUSAnMpGpxGBBwJHREoa58XdGEVFVTjKdTryLe+1M/ZjVDnFN+OJ00ZRERyuyYEVTy8XjabDomkiyZkI1XqJnKTA8qANhZN6SwnEiE2T2bqi1egUpDf44xxjnlbAaO2EiL85mTTHNcFIDBGNEzBbduvipebuUKC6AiipmKxBhHYgB0voBfCGXQouz/hGXAlxFhHYsuGQcTQZk/sXUAuTQ7DHJWp1r6SYgIzCbCAGVG2Vby3mqRZ/BywaHWLtOThN1aDOnZPMEgi66ud5F5MgNFKIt412lPUDCHZo6dc26a52mezdQ5Zu+K5pZ3DAApSco5WEEASUQNLHiHAMM4dm3T1LWI9MP48vpCzT58uq2b5s1XN23X7rcbVfvjD++7Tfvt29cXu+3d/cN+24hkNQvOGeAcY5qjqkqS0FQxp83n21AFU5VNF1Os6qYK1f76Zddtfl7XL65uXlxd/3f/w//j/R++xwTMDAbeMRKlnBAcmJFjDj6Kfvly54O/ubz0zAUR1Cwpp8qHnCKYaYxFb2xOsfNt1zTZ1HknotMc67pwTRacE4nMsNC0zYCJHLOacWkdFTdYeoLPaiRcun4mlj0B1U4ylqctNYtzLqdU/B0wMoKpeUeO0AC9Z1VX8FXvKHhGIDFMApANBWsHXApn1Zg1xmSSQZXAmACZzIxLMki0IOQlA8bSrSmsYU0irmi1qoFZFilbfYnQlMonRESwUiyspbgVd1mm5J7mjHnBbhefWnrFZcSq4BoLy3+1Wjgrh69oKyI6FUkpMVFb18xsZpXzxIwIcZ6D7+qqApvMrEwbIlFdhQJ7qVpK+XQaVVQBNl272XT9MKScry92bdPkJF9u76sqPB5OADDNcds1Kc3brg2e+1P/3Q/v+34owI2IxDir5FylL19ut7utqngmZjw+Pn5pPnKo9hfXdagqH/71X/5L7/x/+z/8j+9+/wfpe43zlBOxI+fIsqpYsphFzbxzdVWJSI6JmBxzVmFmIKtDJSLsuPQr0xy9d03beu9e3lxdXV7UdX1GAgUEoAhkoiqQWrFXOM/TFHIGmKrxsgeqXPlSk6BnrrwfmSVnXcIYIiJTGXgBoqfGtJqBISMQsQAqY1F+V0UzMqCshMaVq5grI1ZEE4hJxzmOc0opS15asqVmh0JtWXyoYdlju5CqSwZb2HIqqlky5YJJgRXxRirJI4oKIYKVDW24wqAABaVHW/v4RebgCdXHpRFma5vDZNG6XzT3cGm/L/ZdcAMnYpjyBNA0zcV+t+0aUe37YZrmcZKmCqPpHBMilgaDmbVtc3WxnebYD9OpHwhximmcpk3X3d4/Hk99OWgiYqpTTOM8A2IWAdPLi0tEmOb5y/3h/v7xh/efU8yExMxF2EPA8jSNw7DdbT27VXTTDodHHz5WoWq3F8zUtdt/+eu/AID/qa6///6P0+lkScoOVslJRbSkRwgK4KaoIsH7y8tLFbl/ePDOtU09V5WBtW0bQnCOh2FUkbZtmrqqq1AFf7HdOO/HccwplsLVIXrHziEXljygKqgYOjiDfcVSCQUQEFdhWwAids4D0BxlTnmc0zRHyQX21sLsoDKqU5TTJCdznm1tOoIYAAExsAIahZJKEgGwiKacj/30eBxPw9xPMeeMYExF8Q4IkLFMrBQneE5OIOsSAVQsZaUsRLlw82BJO4rntVULYrEkfd5kX7nJBTXlc/WOiCvzb4kqixkuoiVSvOw54KyggKoRmRNVVGW1nFLOeY6pwNCErKYfPn6JOXdte3190TZVGQgvcR8ARTITEnHsx2EYS8ibpjmL6KRVXe02HTEP09i1NTNPU5znJCIPj6eY8u+/++HUj8TkwK/NCairWk1Pfd8dT69eXjvvUsrTPIUQ7u9v67qu6oZdS4Sbzf6vf/NXu83mP/zHf/jdP313eDyMj8dxnrzjFFNOidi1bRu8F8nDlASwypkAtrtdW9XOk6myc5cXu912o6rzHMdprKtwud/VdV2CoZnVdZOZYpxzkR0yFikkUcgAZtHMiJDYOSQAKIkQEtPTyKNpwT+N1CyJjnOa5jTFmGKOsUhUWfkPIhAgM3lHCZSX51hLKNKsmhRb4OBDP6akAyGA6hjTcZiPx9PpNEzjVAi7YKXiXmaV8Lm+kkFxq4Rn2P1JwlJUEUnJcEXaiWjZu0aA5/K9IJq2Up/WLxErsSAvMX0ZIzyPdzACPBtFWrNYK52Bwo8jBWcqkiEjeO9yln6cJOdxnGOcY0oIcLHf3dxcXu53zFhCUzGlrg59TzFnQk051XXlmad5VlMTTWbznNqXTaiC907Xaax+mO4eT/ePx3mOwzghqCOXzZwr6xyEENqmnXM+HI7bTeO9c8xf7u6nOe62u+3mmGIMdYPEnvBie/EXv2xeXb14/erl3/7933/84X0YpjTNRIOvKt+2dVUVTKPIJ2Uzk4yio1mLVfCha1tmRsBt13318kVKkRDrqmqaJoTKAB8Oh9OpF0Mt60cyMBMjAljKgoDOeUA085JzKoU8Cy2Rs3iKJX4BQBYxUVWNMZsYQREIMUBgxy67Ocac85wTZYiJvRMm0rXqKjFymqkf4zCnOYn3wxKzAVK2aY7jOKYYJSXJmVau+1KLEBLAcgNxUaa1VYhgsdJ1sYfZMtJbyu9zik1Ey1KFIvVTxIYX4tKahK/D44Qrrrmw+59kSdciHp6p45QsFlZzBQNz5Yp67w2ggAumVrpH5NzVfvfi+qJpGu9YAWKcDscTIg5D1zR1COHqwo3DFLyP83zq+3GcmCirEmKc53EYr6+vjgD9MJpZP/Smev9w6PthnucqODB3sd8C4uPhFGNSkZQyEm67dp7j/cOhrjwTeucOpxMRD9MwTUPVdoGpkMSbugkvXrVNc7G7+Mfr33789OX+/jGXESUwIt50zaatifBwOFqZaidqmsYzg+mu28CicMRNs7m+anJOzBR8xc7N80SApjpNU5aMpaYWJRQgAgAkdEzOPfFxEU3VBISxeCtEOC/h0ZTyOKc5ZlXNZoAUPAXvRDUX+V9YFJ0KMxpMMz1Rh8tWxCzqsuacx3Fynpl96T5m0RijpKSaVaR0DhhtzTqXCkYLYPlUfy9qVnregQBwFn9U1TLnU6xcoQgzCSAScVEgxHUhaOEenjU4zkNwKwj7BKWdlZ7X7xReyfkcrkUSgLu8viyJ0zhNwzDUIQBAjDMivXpxc3mxTzn3w5RFihzSPCdEQCIXwnZTT+N4O47jOB8Oh2me26ZGtDLp4hz74K8v9oR4PPWEMEcZp2Pf90M/DMO42bQF5ri+vCjFLBM+HE79qd9uNvv9rh/GOUbnfcoCgHf3D0i46bZtu3HeEy4pOTm3313+5Z+1b796e39/++7jxz+++/H27m7sRxCZ4uyDe7nb7OoqNM3N9Yuuazdth2bj0JeCwBFXzhNRCKFcl3Kx5mk69sfj6RhjArAijZAKW4UBmDwXgkhpHZaq9KnjBypAvCTRZqo6pzzFPMecRXPOIooGa3O79ACXDmT5pXMxa2eUlNCxc75M2RAgRlGNySTnlMq0foGZFhmwFRQq70nO+jTPxjAUlh6DrIVSCfZudatgpgpFuhEWCYYyUmIASISqi0bVmYas69rLc9FjtkxaP0sGFnB0JSmYnhforGQ8h4BznEVVRQkoZjHTum6cc6Y2T1MuG/tirirftU1p5DLR8TikmCTnOaaHh8M0TT4UnVtYHAriy5urly9eTCmZWdNU3ofTMKUsh+MppUxEu91GVA+nHlfN4q9eXn8AIIJffvt6mOLd42PK2RGrChI/PDz+7vvvQte+9Z6ajnEhDDlHznVN017tr169evPN19+8+/DuD9//cP943227b796/ermRdM028120+0WEA+5LKlMaZKcoQD1gDnFnCIimshUzSI6TqPKIqirjOyYEIiQbZ3bRCvoI7GpKS/PvwCB610yBHLes3PFCJ2Dog6dRUubsNgnsitEOzAoQmLl8UXVh5kZSU1zloUwkXLKWUVKP6Cg8aWuIgIuk+eFf7EilbgO9JXcgxa6arHWkgoubQJf6n9bXKCaiepKgNflZCmULGoRTVvU1IqZPXXVAUB0ZZnimsICrMjU8m9Yl9CVpMGBQfB+nGbnedNtQggiEmPMIqdTP05T2zSd95uuqUKY5nma0xyTY/PBplm9D1nUwPYX+7atNMvx1AOY926aYZomZmJ22+2mqYJjPh57WE/vMIzeu8vmYp6jiHablpnmKb68uZzm2TFf7LY5y3EYsggCpDmq2u3nLx93P1xuLr0PhStUrjISMpBv27quL7b7b15//Zs/+02McwhVW7chBGbnuMyEcYlTRR8hS1X0wMqMmk8+zU5FlWVj29ev3w7j+HD/YIsXRLd0OYjo/1fWlzVJciTn+RGRmXX1NTM4uFxSF2W2K5ErM+n/v+uND6KkXRDkClhgMcBMd1dXVR4R4e568IisWglmAHp6uqqzMj38+Nz9+1idn1zQWEpRJAmhdlP8OftWk5mllOdlllI8x0VkKRKCqFoRLSIkAkgamIOKKVyHs91HQy6FAJMlU3XuQdEqOysiCEBemTumBQQI7LI+jfXzNtK669JqGSqqwaskqG2BZlIu0MhgtdejToSMykxaQU5rkRnQ84HW4L01U6tbRvVv1x+whoRWd65wfR1AcPVSJNoOQ9/1S8mlVMYbVWPmzWZ4erjfDN1lmqd5MdVSihO/Oxf9MPTvnx4B0aSM05RTZkZjTql8+vw6zfN2Oxx2m5KLiuRSxnHsukiE0zTNc9KisevLPG/6+Hi3/+Hjs6o93t+fp7mLHSJGotp5Q7hcLudxkpxU9Le//YeHD18hddxGN7w7gszM3HX9brev9wXqsHgtN82s0QICGHpbEsHQQohFjYOYZgQk5qHv7w53aUnTNJqpAZmq6/ENnfkABBiIKl5LYOe+8yfjqjAyT+Pn1+c///zL68txGqciDjeKk6aoqKhTpNZ3qdRHVbLFAA3N54mMauTVai6ga0mCzQIRkZkMkQCL2G1W556p9i49eiOymvhIlROnmgDwWtiLmvtJIoA2fK2qa9LpDrH6TjOvnHTd8GwZurYRE2g+3tYtu+ZzYXXYZgAQ3o7HGFtLvZRh6O/2OxF9O51yTiJlnufz5ZxyPy/L29u55GSAOZOo7Laboe/6vgPT4/H0w48fc8rMpFLmccIQlmVZ5nE/DPvd9vPnl6UUr+aJKWIU1SXnl+Pb0+NDCHwe568/vNvvt88vx83Qh8BLSkSwpDROc4yhjzGGcFnS88vx+z99/3j/sNnuAj8iYOCay9hNWIkc2u1YCwJoGb0nr6xKCFiggLgcHnMIkhEJQQARh34YhgERiVlFVKGYGUhnUkRCjNbc91ooYcXFEb1iMjVVkTKN8/l0fj2epmnKjc/H7dLLlrZ0Ve0T2qav+yesOiG26mFQm568AvBQc8/A6ImWeJG+Tk4BGJDnkYIGBmKgIkgC6rMr4lTsFfRZf6kBqgGajyJoLanEqAHvZiJCiC7S0owNWhy4LjJdO24tC/KkuDXuYW2YAkAIxKaaVZGZCWMgKdkMDrvtdhgATIo8v55jnLxNstvtzOWMnE4CcdP3IllVY9+B2WWap3EG02GzSTl/en65v79/ergvuXx+eSPip8e7nPI0L0hoqpfLqGp3d3sV/dNPv9ztd2h2PB6ZaLfbiGDX96/H03gZN5thM/QuIvrp0+efPv707sOXsd/2Qy/1Pq5OAqhlgQAVAGlnst4XJTJVInMw3b2R184q6uALEYUQu9gZQEoZfBSXsYAtuVAoUSSEQOi7H0De+AarDIZmagqmueRxPL+d3pZpKsuUl1mKz69Wxl2/t2ujUOvwbt1+YGifytvooCsg4+nlLcaNCIM3JwFc9kUdpScI5DcBtXLZeHPK1LCospqoUx2YqIooIXmI57oMCKZeBiliPZFXG8QmdLUS1xi0lYL6ceg6GVoN0LtcazIDdSTP3w3MLABiKYUCMWEfY8kyp0REXYw+R7Lfb0MIOWcA6AIV0Wle7u8O758ei5Tz+XyZplJK3/dfvH/69Mun17eTqCEAMavZ5+cXJuqHoeviZogiQwj8nN92203fdy5DP47jMAwcwsdPL2/n0VSnRZBOSLjZ9M4abgDjOKtoiNHMTufL9z/88PDuabPdc3hX0y5ne/zLzHw1WWtbyv7hGzRt0HSaPaAhAsWAhCKEIhx1v9v1fX86n52d3gyJTYXqHp96YlmI2Ut1NrMmpuYdRSnldD69HY+XyyWnZCpoyqAG5gsRRlghyfbItbJU1b0BbGABIJhhRTRbp7FB78hM3n8l8uFiNXBQBSITNXhStdgKdXp17WqctR9gzbZuzknzSczV7RVRpnUx8KYcXKWMW37piBys43kV62xYl7WEWGsIFFXPygAsUOAhUAjBzJ7fTmCKhkRUYjkcdtutSydE3G4A9Hy6HN/OMYanh7uH+8Pr8fh2PuecD/vdbrv55dPz+TyaqsvuDl3UUn76+dM8z09PTyHGIroZBgBMOe0326fHu58/PTNzP/S7bT+nwkgIQEy7GBDhMs4xhE3fxxhUShEdpznkEiMvRb/7/gdTtSz/4T/+5v7dB7AOIzrFu5/mFQJuRnnNyn1F08lCEFthQYjAaOjxrmZViJvN9v7ubhwv8zyr+Bqw525WREQMUJlNxYRVVEkNyZrFGwAUkbQsaZnzsoCUgCaEhAwVbgQDqNTlqutz9cul1nQxrOhhhSuxwuBQh4YBCTvGyAhIapDFzFtchMHFnRAV0GWDwMWV3DE2tQ7fVwb3kQ0tU1M0VFUAAjK6mZzzH0b0ee3q9lazXuv0m8B17ba3L2ogcNIGz3Bw3ZtFCA93ewMspcw5iyoDdF1AgBh5uxnAUEUCD10Xp2lKuQxD/8X7p4e7nWp5PZ5EZDP0h21/nufTecwixIQIMYau786XUaWMl5E5/NXXX0oX53kexzEQbYZutxmeHh/MIAa+vz/Qeco5dTEQQgguRCmXad5vN/vdxic6xmlx+DCGYGAfP/5cyj9eLpff/Pbv3335NeAWQsWr1nB/a6YtiGij3FAEcHYh8fBmLh0qJTuvTlHVwGEY+hACE5takYLkQhlkZiLKEQFBzLhKZNwsjps6WsQhumvSym5cGVyl+W6wSuB5kz+1SgKguk/w1LM2LeufwQDBZ1EdlzVA5y8lAmaMTIHaOryjN17yAAIYke8V14gjaiKmCiLAbP4ZDE1rBQ9QkInwirRjQ46qoa833k1wRVrMfbOfqNZeu75LS1bd8B3+N8MwjbMSDsOwDSGGAACBedN3+8N+6LpSJCX5/PLmyf7Qd/f3u/vDYUnpx58+vb6dnAT55e3y/PnldDqbKiFRF+72u76L4zQ59QO0RH5Z0ul8GS/TF+/fpyJdjH/7619N8xwDH3bDPMNhv0ulmFpOi+MvXeDArKrD0HdddxnnlAsiRGJDez0ef//NN1nKfwZ7/9WvzIYQAt2YI1xDjyG6Fps3bNVVNNR1F1TNtOSU05KXRaXUghiBOOx3hxi7k15KKWoaiVrCVyFPJ2VgDkxc/UR1f6amIcT7+4d3797P8zL3s5QCKqWISMEiRQTMyCmTbgGgmpkhNoYSJlwVDQzASbrALBBFxsAElZRBTTUiMFEgjETtfQEN1FX8GgZaTcopSwkQzKVOTU3VPKCDl2a8+mtPmB1sa7lTm51b+5a+Ue8zTOhQEa6Aa2WE8bH566vWFNU5DsDCOI4xRuw6ZBpih2ix67549wiI85xqbtvY8D03mZd0vlwu49jHsN304zT9/Mvn09sJEZ32qO/idrsRKYgQ2CcmZZrmXBVXMRV5PZ7u7ndE2HcRwAihlDIvqe/C4/1BDVKKx9NlSfP5TbvY5ZReU9ptht2mI4J5XpJACCwmb6fTH7/7bhiGjsPjF19Rpb+sYkt2Q0ForRIyAxHnGpOa+bkWr6mWoipLWtCLAyJk2u52T48Pb29vS81ZDcCHmdANXW/ZCsxUFEyaIg0S0n63+/qrL4ksLYsvNqioqlzGaZ6XeZrnaSpLKqWQKoAFdBQIGI3b4g9C1Qx22fM6gk4YAwYmYgSArCZi7JA/YSAicn5iED+S9VKB1ll2aPpETeLDTyY28NIfELtCcsud6gWtZnU9sjUFsAa/39ZSUAv2inf5oamG5ZVuS3v9r0M/9EM/IMA0zqJyt993MV7GKfbdYb81Uzf81+NbSnkz9OM4//zLc8o5MMUuppxTyjHwbrdlDt5q2++2fcfjuLgmZwisIs/PL0S0rmPPKQ1Lt+kjaNkOXZFSSpmmaZ5HAnh4uNvvhqGP07y8HM9pOauUlHJKadP3wzAMfX8Zp2UuMUYA+/zp8//I/6Sl/P3v/uvDh6+67trMWH1nM9BKoVNDjohlM8JVAAAU0klEQVQrvZqolmyiOaWcZibKOad5jl1HHIZ++PDui7e3t1LyspiqNXHJCg/lIlEkWhsG97holVWGiGKM2+3m4W6/LAEQCTEwRWZ3iCmVcZrf3k7H49vlfJnGUdLi9IDNkRpUGseKwjuvCSEGxuCDz0h+4hyAgNY+1iZG6KCStQwB0Nj/Cq6dr9XkzEzBGKyIsoFVeWFTA18guX5OJCKHe625hVoJ3DyCa24KrU/QlBoqCtaK2mazAIgY+mEAAG+YbTeb7WaQUhBx33XELAJD3xsYB96Fbd91x7fTOI5mlgmnKQGCSjFVV84khN7FN8w3zDGECICvxzMAPNwdfEp/M3RoWkqeTYjg3eNDypazT/rJp+eXnJM3n97d7fuue3l5ndKSMuVctBSVMgzDZujnJaWUWIRDPL29/f6bf0bmf/gv/+3h3ZfO/nzrO/0fUZ/WUfBSybF7KVKSqJaS1URUfTt5LsnA+h4Dh81mc3d3fzydVM2joWuEurn7ZISocuU0WNP8+gUhdTEG5sLeyHTRTQ4hEtHhQO8M8ldf5JzGcXp9Pb28vr6+Hi+ns6YEph5IoaV+DUqjwBhDna9KIimrgSFSdAYyX1ABUDVXAyX0IU5AF+9qdYqDkQAgarG6TwfjIXBYz7rBKg2vWNeDV/j1mne2fytE2tgc1vdABGCsaBo2LMKal8WbWxfSkjwJ22yGu/1OVHNKRLTMy+UyLkvaboa7w/7x4T4wPb+8juOcixRnU0FERBFx3h816yIPm6GLQUWYYxchMJqJFEHmYehcE5sQRcsyT8qMaGPf+XOMgU3AVMZ5joGYsKiGEB7u73IpqjbPKiKXaU6lDH3fdbH49EaBjPD6+vLNt/887Ha/4e7w8BhjXA+iNXTYP3xt/YmoFKtTOwUAfcyAiUwEwPq+rxQHgfuu326Gvot5SX43vdwpIszk3SMRp0EXqgU2rc7AzFJK85JySj73TQhMAbFGW0Qahj7QQR7t668kp/R8PP7y8y8f//zx9eW1LKkOVACuOyRdpMiIxISYRKesZtYxxUAxUDWOlte1C0aDRo8P5hu+qwFds1t16/GMYuXmNwKDtm/kS/FNCWSt3GGt8lcss/72ViN52ipqDbwGaA/Jv1rzB0MI5/M5MHd9L0VO5wsTiRlSEpVlXsQgMOciqioERHQ4bJeUx3H26biUMjP1fURERo0h7LfDdtOfx7mIhIAxhi4ENT0cDofd5sePn0rOgYmQci59F/vA87Lst/22C6UPS/bDDSKiJRFGUNh0PAz9ZZz6LoyLOgBNYF2kwj0nzKWAaWQoy/Ttt38Ydrvf7H7Xdb3vErWehqeL2cSk5JyWZbzMy1RymecplZxTulwu0+U8TvNlHKd5STlpKSpKIbhevKn1fTS1yBzIwRfMRTopMXSe4NYUoiVStgY0oixlmhcmCrEO+/SddV0k32NzbXeiXb/B3e5wODzd3x/22z/+8fvnnz+ntIDnnWgGFKOTgRERLWJzElGL7AeMAyIhFKgmogBeqRCBKDQYx9bGWgvX0K7XmhHXf1SNqxRj7QtVf+m912piCP/PxmBNWf3+rzm6451XIKBG+YoDXAslAAsi4tG5SOm6oKq+u+XEYPu+N7Pj8Q0A+j5uh76PztqDSwrjZSKSYdOHGEwNI3dDT8zTtKRlJoTtbui76NFov9vMaVly9hnJ2MWHIW6HIVdRoqHrY99Hn7rBSgBquWRE6EL3dLc9n08FYb87FHGiIjrs+t12k4ucx7kU3Qw9MlrKnz5+/D/dN3eHu1Jknqd5msbxMo7j+Xyaxul0OU/jNLrk2zy7Y1MEyaWUDIjEDKrMrAbe+0OEEHjouoe7u6en+4fDduBoToPp6aC2Hnq742YOULuMhnEI93f3l8tpmWeVUkrxOXaHG82gw87bWqoCGfqu32w2zCEXOZ/H6TJKya7KbIDR1eh8lUp0TiWLej3KgYNXRoBkVgB1PSV1J7giTf57nV1RTU2JfdXMTGpC1BDZFnDV1h+t5FRt2xOc56FBntU8bd3ywCv1/FpI2fV7a+SH1ZXXEN910flPOAbRK6Y2dN3Qx1JkmiZiGvqeENSsFCHEh4c7MJgOyzLPuRRABAZVNrW386Qli8pmGDbD0HV+t5AQUi5oQkxD3x12w9BHM8tZN32362PH0IG8XfA8zmYKpqbFFKdc5iUzcd/15/PRpyoQaMrLp9cLM5nBksu8ZFH10XH77/8YkBgxO5qj9TarFxzXxAjqkSfutwN2Xcm5C7zZDGYQQ4yBY+RI9OMvz7mUKRc5HqeUXt82D4fdX33x2K0TVQamUkphLoGDEjuICC3CMtF2s/vw/ot5mY8vn6WU5Gv12lAEr2gYzCDlUkQGHQKH/XZz2O+7vgNE393pAm0iIyIB5iznJKUIB45MHVMM3LEPF4MaYsVcnSyy1s8ebREAQdFFPAy5Si9ALZEcuIHKxlvb5dV0gAwIkW59J9xaFzTErNke4K1vrT93BQpuav+bFwJAIGIzW1LCXJjR5w4J0VTO55xTJqZh6D3zm5dEYBy7/XbbdbEUSWl2AF9FUhFULVIA8eFuv9/vNpshBEKRcZ6nSS7jlHMhMJMyT/M4TiaSckGiTy9v05J1WS5LHuekRZFIAXIWABAz8WxJ69SPj3m120yE6BpnDdhBW7sUBsgUQyBmcNhTxZ+CAAQwJCZmy0UAus1GU0pzssBzLuxxvGgMrKWAWVablgWILks6nS9/92//+uHurqiIlFzIAEMouWRiFqnL5j5LogYU+O5w//WXX0vJ57ejw3ArDsUUQlBELKLTNOWS+3642x9UrY+hC9HxUCbqmAlRDHKROUkRDUx9oL4LQ+AQAiCKT0itjsjvk88p+7QIgrpUMQAYtLUNU1UPCBU2p1r0GxgBrrdX0XvuWOEqwLrCVZVr3LPC+vv/vy9uc8/qpKtZ37hY8CIJwYpqYy7H0MUuRwQTNSbaxk3OZV4SghHRdjv8+ssP7x7vmclUcu7PfZ9ymqZFL+NlSSllM8spv7xeFKGksizzkkopJeUiClbhx3ppdX6mHqu6boitaXbTCmqwBDa7A+v7jpjHcfJDSE5I4LEMcf/01B/2m74//vSxlLx/fNrttv12+Paf/rfj8GQmZqRqpkKBCEEEkUrO5CR3mgWgeNXffq2ITcvCRJ9Tmv7Xv/zm7/72w9N9kQKZkIJHM5FCRCLA7PHNPwrGrru/fyqSfzSYLqfWLii5MC5zylkNRORymVLJm83Gw/C8LFnFAALjEJkIFUBFp1SKKBP2kTdd6COHwNAcJCGwuZKx943AsClyuHX6NEkbdyqiQK71B0T1IBOCoSkh15ebI/dOzF5dI1FFNFuqXQe5V3d6Dd14PRPXx3vbC71iyf6tUHLy5Al9Y8EZ9IoQIjEjwGW8iKgUQcS+6/a77dvr6XQ8zSm5bsU4ZxE5T9M8LUWtqFopVmn1aibTnH514h5dvPVgNXmutSZWYgH1DAZUCakWmW5+oqYCxIioCiLZcy4T9SBpRZylo8zz01cfdveHy/PLdLmcxwsEvFxO5lNkhABYZUGKACr5vTUwU9IqKaBtrZ2Q0JCYDbGIFjUEyHn8n3/419/9p39/t9+JFNMiJQsHQhLMAIgoCGrkhIcIiH3fPz68E7XnzyG5uA8REqVi83yelrTMaVkWIALDE5/Ssnz85WWeZkJwvnA1SKJLKqkoEXaB+8Aup+SR3Wyl7qqYe+X2rF4R18UkA59RMbvRj3Oks9qKGxgSVO7/6u+rHdY/tI3N6voaRXRD8HHNAW4zgBs3uXaQbssydzSuduqqJb6SYmVJvl4j1mZbmkTZZZqej2+ilfy/fmTC2lpoJQJWVj4lbAwpWHtdgCDSph0QsfaOgYmM6s42EiNCSdlXPZG5nTIjZslFTJ3iAwhFPBkidRUT36pBBIBlGv/0zbcqlVQ7ny6fz2cRicQYgvkcbt0R9t1bD7VcJ46d1c33exAICAjNVPyB+Z0xe7mMv//2T7/7zb+LwdKyeoyq6mCmzExgQAZWPQ1z2O/uum5AxMDRZ2tSyqfz6eX19Zfl87RkMNAiry+v8zgdT6cyzQGRyTvizvUiRLSJFAPHQFWLEcBXLg3RAVkD1FaA0NWJAiBBZchBbfi8udxUq+9xHQiogZtW4hv/ayc7gSY5fq1+/H0rHEa1adRstMX0vzDF1XGu7+/1QvCRQZeq0gYZqpo6S4CPmdSBNChiAE2/o5Ua0LABu7p09AYxUWMd8DlwIkV0CW0MoePgFYK7F4C6mswtlDiJMIhKyVKkpkn105m07BsQjcldgVNZYtVGA1N1/9RSPehCrFmA4xwtDtXM1dPbGLz1ylqRca2llvPUARMBB4rR1VOeX0+vx9PT450VQsoGUKT0XSche6EVYgQItfAQUZG+73fbfQwxhFCLNpGnp3fv34/v3j1/9/33f/rhxx8/firLwqBsQqaAoICmNmXJRZgxBu4jd13sYqSG5DhCpAZSR+zB5wZ8mUIrY0MNWW4lAUGrAyEidI7bhiBDS5QNXSCs3kqqVdJNZolVMQLEqq/0b1q7t6s1ri7zLxwnrq+oxGMAEFyVR1Swvq7CqvWtoYqGae00GFRx3CpgX/1oiyX+cg4hdhGRvOpUNS3ZT23d0DbQItOSKmq8TgqQ3xcFAAKqd0FN/fIIpW2ZWz0O6IeePG1vC0cYWK4pbFuaRIC6OuZsa2ae+SMGZKe1r8hKQmLWXFwecm1DOQRpAAURuZBqiB0xTSn//Hz86sOTc9XHGGIIoiZpURWLilh5AXzXMYSw6frA0Yc369YDABKGwLvt5vH+/unp6Ztvv/3lp5/yNEYCBhKDopZLyUUNoAu06UIXA4fg5Q9jZRfzR1H5u7V6eh+sqv6krntAnbOzSqy/XgggNk4cvHGlNTez9Ut/4Zo1ErVhwesNtxpTbQ33t2DcreOE1pLFduGIEJD4WgFUC7WVuKyaf8Vi1y0T0yY9iIDrgun6i31QB8FW2vaVMkpbc78+b0Q0tDVNrhJEgDXBUKzTWw51oI9F+bVRQxCb0/ZLVIXKKOd4HiG6rwVfloPKCOzllBYLTP5snTLJ3bKJBGJPNOv8uidrqGbq6bMVEcwAQcwuWTwKeoOVQzA1p05ELGZKRBy6GHo/vYEjVa5Or63FzCU+LIbw8PBwOBy+/uKLf/3jv3z3x2/H1xfJUkRTKbmYmXWRQwgUmCudrgJQMdCGuzeXYT4jpw3P/EvPtaZ+BFUIgpl8lg4JyYeu1dwH11vdkKgaM2m1O2gWA1Wy+9Ykrj1QuO4bXd0qrtkqXOslNQMIfoIcFHA/4Y8EW+BTbaOyiCvBxNoZa1F9nT8ARwOaL/KOAbpktyGtVJrNE3qeBC1/bR+hJkHg5Fa3rn1NGLQ11xBx3QX09EDluhpYf+paollTP6nfN1W1AoimRlWWDwCsqCATuC6Lj3+hVdm06h8MDdCMmNOcUirUU8qFQzAtuRTnIo0x9v1mt9v3/cYNlOoGRXsHADMrLu7BrhGDzPzh/Ye7w/7Du8dv/vD7H/70Yz6PToDXRRpiiDF2zA5OO/zu8TG0FhGs8xCoq/34HSGAutJqoADMFJidv5KZXfpbnaaHzNePSK3uK5iBmrfSDUypqsyoGQFBW5harRXbI7uxSAODdZMJrwG/rot5IxqJQDVwDKZa1XEBWy9r3SgADq3DBUbEiOsZrWRU9QJW4/c82we/Ddb2N7QawVq6oLCOfpEUBwV9VNHM5Si6vjvsp5cXLeLKj+t5rElxjVGK2OgrbnqMyEziCJGfKN9Qg7prVtMEBEABIzNP9LniDz6dWdc13ZzNFOsX6MvhaErABJCWdJnmGNgAci6qMs9zEYkh9F3Xd7ELMbDLDVQc11o9a35hKgZiVlFFzx77fvvll7+6jOPHz8fl9SSqMfIQOUYOTB7Trfmu5ubqLVh5ofysqvojaoPrHsF8x44CkEsmVTkGf8wG5s7K6tq7Qht2r12g5tFAa/sDKmiDayMNWm/dh0wBoA0A6C26szr9a0dU1cBC4GDkV9+yzBXz9yQFVwqK1ZFdE4LqJAFahmotCBgYmppJqb/MBZkBEZCYQgzU93le+s32/a//5vP3352eP/nkm98/DhyGAZlXuoHmQ/2Wo62p/urHGxeeX7yWQmui0k5KayZXdwrY+hmInjzUil4rhQG1zNMqveD67mCEqqiqhJRULuOy7bsQ9ZxzkYKEfey7GBFRREQzC4OBS87VigRJwVRFck55RgAIaIDcVkSRHDp93O52hhRi6ALFyEhciRPBRF1WGOlKKltp3hXWR2k1ufSPpVp/DpGIkchtE4m9dFQzMedchIpauas29CE8n1wz5JrOX2MgoK9IA1R2xTYnbpW/pBXWWjmhK2TQ7iy0+RJ/r9CYxFo9ezPLX61hHUg0xXYckdZThboiKus0YU05tCqPQisIyTcfgTnsHx9ou3n788+B+fD4eH55eXv5bG1AC8yAKA5DWhYTqQOD1bdd8d0GSHjMaFnP6rO19javiVe7EbWkb6mtf2y8Ge+u1r/CBgBonrbadZxOGwsWqojlnEpOl1HUlJm2my1VsFZLkZQyGDEbMleCFwaj6ziJFEHweX8hysxBW7L3eH//b/7mr89vx+V4JIDaO1YpAMVA1Vu8GJmtldV+gNvYKtQWsHtvtSKVS9DpJzgEInZtEiBSF7kz8HWkla+UjMzZmKAen+Y/cS1IoO3lwU0BtP5XVrTJy6D66JzpxLHv+hRW6wsVWGnfamlus83V/66bMrjaH4A76ht7lurq6oPFa+FVk0VCNB8QE6MlA6CJ5mXG1kZyX4vMHAIA2rI45o/12lrN6G/rw93tTNj16AEAEhN4U98BPzPTumpOgOZdaPVHxL5eiTXpqEmVrSxbjfN//WDr/xTBEFR0zmVKScWQMDCbWS4lFAakEDRnAcgBkFwD07nNlCpMiBS7IS2jSVYzJEk5OTMCMyPS4/3h3ePdz9MIIoC1/WMAYi7dC4YE4Pp0gK5600YLPexXIohKKA5iQOs4FDMyM3GFPG+Ki3rP1JiaMmqlGSOX4VLzHN3M72ZlnKwv9lu5GkmN0c3D113VenbcsNHaipQ/7v8LuhRpugWd6nYAAAAASUVORK5CYII=)" - ], - "metadata": { - "id": "gW4cE8bhXS-d" - } - }, - { - "cell_type": "code", - "source": [ - "image_data = (periodic_impulse | beam.Map(lambda x: \"Cat-with-beanie.jpg\")\n", - " | \"ReadImage\" >> beam.Map(lambda image_name: read_image(\n", - " image_name=image_name, image_dir='https://storage.googleapis.com/apache-beam-samples/image_captioning/')))" - ], - "metadata": { - "id": "dGg11TpV_aV6", - "outputId": "a57e8197-6756-4fd8-a664-f51ef2fea730", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 204 - } - }, - "execution_count": 11, - "outputs": [{ - "output_type": "stream", - "name": "stdout", - "text": [ - "\n" - ] - }] - }, - { - "cell_type": "markdown", - "source": [ - "3. Pass the images to the RunInference `PTransform`. RunInference takes `model_handler` and `model_metadata_pcoll` as input parameters.\n", - " * `model_metadata_pcoll` is a side input `PCollection` to the RunInference `PTransform`. This side input is used to update the `model_uri` in the `model_handler` without needing to stop the Apache Beam pipeline\n", - " * Use `WatchFilePattern` as side input to watch a `file_pattern` matching `.h5` files. In this case, the `file_pattern` is `'gs://BUCKET_NAME/*.h5'`.\n", - "\n" - ], - "metadata": { - "id": "eB0-ewd-BCKE" - } - }, - { - "cell_type": "code", - "source": [ - " # The side input used to watch for the .h5 file and update the model_uri of the TFModelHandlerTensor.\n", - "file_pattern = 'gs://BUCKET_NAME/*.h5'\n", - "side_input_pcoll = (\n", - " pipeline\n", - " | \"WatchFilePattern\" >> WatchFilePattern(file_pattern=file_pattern,\n", - " interval=side_input_fire_interval,\n", - " stop_timestamp=end_timestamp))\n", - "inferences = (\n", - " image_data\n", - " | \"ApplyWindowing\" >> beam.WindowInto(beam.window.FixedWindows(10))\n", - " | \"RunInference\" >> RunInference(model_handler=model_handler,\n", - " model_metadata_pcoll=side_input_pcoll))" - ], - "metadata": { - "id": "_AjvvexJ_hUq", - "outputId": "291fcc38-0abb-4b11-f840-4a850097a56f", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 133 - } - }, - "execution_count": 12, - "outputs": [{ - "output_type": "stream", - "name": "stdout", - "text": [ - "\n" - ] - }] - }, - { - "cell_type": "markdown", - "source": [ - "4. Post-process the `PredictionResult` object.\n", - "When the inference is complete, RunInference outputs a `PredictionResult` object that contains the fields `example`, `inference`, and `model_id`. The `model_id` field identifies the model used to run the inference. The `PostProcessor` returns the predicted label and the model ID used to run the inference on the predicted label." - ], - "metadata": { - "id": "lTA4wRWNDVis" - } - }, - { - "cell_type": "code", - "source": [ - "post_processor = (\n", - " inferences\n", - " | \"PostProcessResults\" >> beam.ParDo(PostProcessor())\n", - " | \"LogResults\" >> beam.Map(logging.info))" - ], - "metadata": { - "id": "9TB76fo-_vZJ", - "outputId": "3e12d482-1bdf-4136-fbf7-9d5bb4bb62c3", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 222 - } - }, - "execution_count": 13, - "outputs": [{ - "output_type": "stream", - "name": "stdout", - "text": [ - "\n" - ] - }] - }, - { - "cell_type": "markdown", - "source": [ - "### Watch for the model update\n", - "\n", - "After the pipeline starts processing data and when you see output emitted from the RunInference `PTransform`, upload a `resnet152` model saved in `.h5` format to a Google Cloud Storage bucket location that matches the `file_pattern` you defined earlier. You can [download a copy of the model](https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet152_weights_tf_dim_ordering_tf_kernels.h5) (link downloads the model). RunInference uses `WatchFilePattern` as a side input to update the `model_uri` of `TFModelHandlerTensor`." - ], - "metadata": { - "id": "wYp-mBHHjOjA" - } - }, - { - "cell_type": "markdown", - "source": [ - "## Run the pipeline\n", - "\n", - "Use the following code to run the pipeline." - ], - "metadata": { - "id": "_ty03jDnKdKR" - } - }, - { - "cell_type": "code", - "source": [ - "# Run the pipeline.\n", - "result = pipeline.run().wait_until_finish()" - ], - "metadata": { - "id": "wd0VJLeLEWBU", - "outputId": "3489c891-05d2-4739-d693-1899cfe78859", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 186 - } - }, - "execution_count": 14, - "outputs": [{ - "output_type": "stream", - "name": "stdout", - "text": [ - "\n" - ] - }] - } - ] -} + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "include_colab_link": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "code", + "source": [ + "# @title ###### Licensed to the Apache Software Foundation (ASF), Version 2.0 (the \"License\")\n", + "\n", + "# Licensed to the Apache Software Foundation (ASF) under one\n", + "# or more contributor license agreements. See the NOTICE file\n", + "# distributed with this work for additional information\n", + "# regarding copyright ownership. The ASF licenses this file\n", + "# to you under the Apache License, Version 2.0 (the\n", + "# \"License\"); you may not use this file except in compliance\n", + "# with the License. You may obtain a copy of the License at\n", + "#\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing,\n", + "# software distributed under the License is distributed on an\n", + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n", + "# KIND, either express or implied. See the License for the\n", + "# specific language governing permissions and limitations\n", + "# under the License" + ], + "metadata": { + "cellView": "form", + "id": "OsFaZscKSPvo" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "# Update ML models in running pipelines\n", + "\n", + "\n", + " \n", + " \n", + "
    \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + "
    \n" + ], + "metadata": { + "id": "ZUSiAR62SgO8" + } + }, + { + "cell_type": "markdown", + "source": [ + "This notebook demonstrates how to perform automatic model updates without stopping your Apache Beam pipeline.\n", + "You can use side inputs to update your model in real time, even while the Apache Beam pipeline is running. The side input is passed in a `ModelHandler` configuration object. You can update the model either by leveraging one of Apache Beam's provided patterns, such as the `WatchFilePattern`, or by configuring a custom side input `PCollection` that defines the logic for the model update.\n", + "\n", + "The pipeline in this notebook uses a RunInference `PTransform` with TensorFlow machine learning (ML) models to run inference on images. To update the model, it uses a side input `PCollection` that emits `ModelMetadata`.\n", + "For more information about side inputs, see the [Side inputs](https://beam.apache.org/documentation/programming-guide/#side-inputs) section in the Apache Beam Programming Guide.\n", + "\n", + "This example uses `WatchFilePattern` as a side input. `WatchFilePattern` is used to watch for file updates that match the `file_pattern` based on timestamps. It emits the latest `ModelMetadata`, which is used in the RunInference `PTransform` to automatically update the ML model without stopping the Apache Beam pipeline.\n" + ], + "metadata": { + "id": "tBtqF5UpKJNZ" + } + }, + { + "cell_type": "markdown", + "source": [ + "## Before you begin\n", + "Install the dependencies required to run this notebook.\n", + "\n", + "To use RunInference with side inputs for automatic model updates, use Apache Beam version 2.46.0 or later." + ], + "metadata": { + "id": "SPuXFowiTpWx" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1RyTYsFEIOlA" + }, + "outputs": [], + "source": [ + "!pip install apache_beam[gcp]>=2.46.0 --quiet\n", + "!pip install tensorflow --quiet\n", + "!pip install tensorflow_hub --quiet" + ] + }, + { + "cell_type": "code", + "source": [ + "# Imports required for the notebook.\n", + "import logging\n", + "import time\n", + "from typing import Iterable\n", + "from typing import Tuple\n", + "\n", + "import apache_beam as beam\n", + "from apache_beam.ml.inference.base import PredictionResult\n", + "from apache_beam.ml.inference.base import RunInference\n", + "from apache_beam.ml.inference.tensorflow_inference import TFModelHandlerTensor\n", + "from apache_beam.ml.inference.utils import WatchFilePattern\n", + "from apache_beam.options.pipeline_options import GoogleCloudOptions\n", + "from apache_beam.options.pipeline_options import PipelineOptions\n", + "from apache_beam.options.pipeline_options import SetupOptions\n", + "from apache_beam.options.pipeline_options import StandardOptions\n", + "from apache_beam.options.pipeline_options import WorkerOptions\n", + "from apache_beam.transforms.periodicsequence import PeriodicImpulse\n", + "import numpy\n", + "from PIL import Image\n", + "import tensorflow as tf" + ], + "metadata": { + "id": "Rs4cwwNrIV9H" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# Authenticate to your Google Cloud account.\n", + "def auth_to_colab():\n", + " from google.colab import auth\n", + " auth.authenticate_user()\n", + "\n", + "auth_to_colab()" + ], + "metadata": { + "id": "jAKpPcmmGm03" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Configure the runner\n", + "\n", + "This pipeline uses the Dataflow Runner. To run the pipeline, you need to complete the following tasks:\n", + "\n", + "* Ensure that you have all the required permissions to run the pipeline on Dataflow.\n", + "* Configure the pipeline options for the pipeline to run on Dataflow. Make sure the pipeline is using streaming mode.\n", + "\n", + "In the following code, replace `BUCKET_NAME` with the the name of your Cloud Storage bucket." + ], + "metadata": { + "id": "ORYNKhH3WQyP" + } + }, + { + "cell_type": "code", + "source": [ + "options = PipelineOptions()\n", + "options.view_as(StandardOptions).streaming = True\n", + "\n", + "BUCKET_NAME = '' # Replace with your bucket name.\n", + "\n", + "# Provide required pipeline options for the Dataflow Runner.\n", + "options.view_as(StandardOptions).runner = \"DataflowRunner\"\n", + "\n", + "# Set the project to the default project in your current Google Cloud environment.\n", + "options.view_as(GoogleCloudOptions).project = ''\n", + "\n", + "# Set the Google Cloud region that you want to run Dataflow in.\n", + "options.view_as(GoogleCloudOptions).region = 'us-central1'\n", + "\n", + "# IMPORTANT: Replace BUCKET_NAME with the the name of your Cloud Storage bucket.\n", + "dataflow_gcs_location = \"gs://%s/dataflow\" % BUCKET_NAME\n", + "\n", + "# The Dataflow staging location. This location is used to stage the Dataflow pipeline and the SDK binary.\n", + "options.view_as(GoogleCloudOptions).staging_location = '%s/staging' % dataflow_gcs_location\n", + "\n", + "\n", + "# The Dataflow staging location. This location is used to stage the Dataflow pipeline and the SDK binary.\n", + "options.view_as(GoogleCloudOptions).staging_location = '%s/staging' % dataflow_gcs_location\n", + "\n", + "# The Dataflow temp location. This location is used to store temporary files or intermediate results before outputting to the sink.\n", + "options.view_as(GoogleCloudOptions).temp_location = '%s/temp' % dataflow_gcs_location\n", + "\n", + "options.view_as(SetupOptions).save_main_session = True\n", + "\n", + "# Launching Dataflow with only one worker might result in processing delays due to\n", + "# initial input processing. This could further postpone the side input model updates.\n", + "# To expedite the model update process, it's recommended to set num_workers>1.\n", + "# https://github.com/apache/beam/issues/28776\n", + "options.view_as(WorkerOptions).num_workers = 5" + ], + "metadata": { + "id": "wWjbnq6X-4uE" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "Install the `tensorflow` and `tensorflow_hub` dependencies on Dataflow. Use the `requirements_file` pipeline option to pass these dependencies." + ], + "metadata": { + "id": "HTJV8pO2Wcw4" + } + }, + { + "cell_type": "code", + "source": [ + "# In a requirements file, define the dependencies required for the pipeline.\n", + "!printf 'tensorflow>=2.12.0\\ntensorflow_hub>=0.10.0\\nPillow>=9.0.0' > ./requirements.txt\n", + "# Install the pipeline dependencies on Dataflow.\n", + "options.view_as(SetupOptions).requirements_file = './requirements.txt'" + ], + "metadata": { + "id": "lEy4PkluWbdm" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Use the TensorFlow model handler\n", + " This example uses `TFModelHandlerTensor` as the model handler and the `resnet_101` model trained on [ImageNet](https://www.image-net.org/).\n", + "\n", + "\n", + "For the Dataflow runner, you need to store the model in a remote location that the Apache Beam pipeline can access. For this example, download the `ResNet101` model, and upload it to the Google Cloud Storage bucket.\n" + ], + "metadata": { + "id": "_AUNH_GJk_NE" + } + }, + { + "cell_type": "code", + "source": [ + "model = tf.keras.applications.resnet.ResNet101()\n", + "model.save('resnet101_weights_tf_dim_ordering_tf_kernels.keras')\n", + "# After saving the model locally, upload the model to GCS bucket and provide that gcs bucket `URI` as `model_uri` to the `TFModelHandler`\n", + "# Replace `BUCKET_NAME` value with actual bucket name.\n", + "!gsutil cp resnet101_weights_tf_dim_ordering_tf_kernels.keras gs:///dataflow/resnet101_weights_tf_dim_ordering_tf_kernels.keras" + ], + "metadata": { + "id": "ibkWiwVNvyrn" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "model_handler = TFModelHandlerTensor(\n", + " model_uri=dataflow_gcs_location + \"/resnet101_weights_tf_dim_ordering_tf_kernels.keras\")" + ], + "metadata": { + "id": "kkSnsxwUk-Sp" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Preprocess images\n", + "\n", + "Use `preprocess_image` to run the inference, read the image, and convert the image to a TensorFlow tensor." + ], + "metadata": { + "id": "tZH0r0sL-if5" + } + }, + { + "cell_type": "code", + "source": [ + "def preprocess_image(image_name, image_dir):\n", + " img = tf.keras.utils.get_file(image_name, image_dir + image_name)\n", + " img = Image.open(img).resize((224, 224))\n", + " img = numpy.array(img) / 255.0\n", + " img_tensor = tf.cast(tf.convert_to_tensor(img[...]), dtype=tf.float32)\n", + " return img_tensor" + ], + "metadata": { + "id": "dU5imgTt-8Ne" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "class PostProcessor(beam.DoFn):\n", + " \"\"\"Process the PredictionResult to get the predicted label.\n", + " Returns predicted label.\n", + " \"\"\"\n", + " def process(self, element: PredictionResult) -> Iterable[Tuple[str, str]]:\n", + " predicted_class = numpy.argmax(element.inference, axis=-1)\n", + " labels_path = tf.keras.utils.get_file(\n", + " 'ImageNetLabels.txt',\n", + " 'https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt' # pylint: disable=line-too-long\n", + " )\n", + " imagenet_labels = numpy.array(open(labels_path).read().splitlines())\n", + " predicted_class_name = imagenet_labels[predicted_class]\n", + " yield predicted_class_name.title(), element.model_id" + ], + "metadata": { + "id": "6V5tJxO6-gyt" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# Define the pipeline object.\n", + "pipeline = beam.Pipeline(options=options)" + ], + "metadata": { + "id": "GpdKk72O_NXT" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "Next, review the pipeline steps and examine the code.\n", + "\n", + "### Pipeline steps\n" + ], + "metadata": { + "id": "elZ53uxc_9Hv" + } + }, + { + "cell_type": "markdown", + "source": [ + "1. Create a `PeriodicImpulse` transform, which emits output every `n` seconds. The `PeriodicImpulse` transform generates an infinite sequence of elements with a given runtime interval.\n", + "\n", + " In this example, `PeriodicImpulse` mimics the Pub/Sub source. Because the inputs in a streaming pipeline arrive in intervals, use `PeriodicImpulse` to output elements at `m` intervals.\n", + "To learn more about `PeriodicImpulse`, see the [`PeriodicImpulse` code](https://github.com/apache/beam/blob/9c52e0594d6f0e59cd17ee005acfb41da508e0d5/sdks/python/apache_beam/transforms/periodicsequence.py#L150)." + ], + "metadata": { + "id": "305tkV2sAD-S" + } + }, + { + "cell_type": "code", + "source": [ + "start_timestamp = time.time() # start timestamp of the periodic impulse\n", + "end_timestamp = start_timestamp + 60 * 20 # end timestamp of the periodic impulse (will run for 20 minutes).\n", + "main_input_fire_interval = 60 # interval in seconds at which the main input PCollection is emitted.\n", + "side_input_fire_interval = 60 # interval in seconds at which the side input PCollection is emitted.\n", + "\n", + "periodic_impulse = (\n", + " pipeline\n", + " | \"MainInputPcoll\" >> PeriodicImpulse(\n", + " start_timestamp=start_timestamp,\n", + " stop_timestamp=end_timestamp,\n", + " fire_interval=main_input_fire_interval))" + ], + "metadata": { + "id": "vUFStz66_Tbb" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "2. To read and preprocess the images, use the `preprocess_image` function. This example uses `Cat-with-beanie.jpg` for all inferences.\n", + "\n", + " **Note**: The image used for prediction is licensed in CC-BY. The creator is listed in the [LICENSE.txt](https://storage.googleapis.com/apache-beam-samples/image_captioning/LICENSE.txt) file." + ], + "metadata": { + "id": "8-sal2rFAxP2" + } + }, + { + "cell_type": "markdown", + "source": [ + "![download.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAOAAAADgCAIAAACVT/22AAAKMWlDQ1BJQ0MgUHJvZmlsZQAAeJydlndUU9kWh8+9N71QkhCKlNBraFICSA29SJEuKjEJEErAkAAiNkRUcERRkaYIMijggKNDkbEiioUBUbHrBBlE1HFwFBuWSWStGd+8ee/Nm98f935rn73P3Wfvfda6AJD8gwXCTFgJgAyhWBTh58WIjYtnYAcBDPAAA2wA4HCzs0IW+EYCmQJ82IxsmRP4F726DiD5+yrTP4zBAP+flLlZIjEAUJiM5/L42VwZF8k4PVecJbdPyZi2NE3OMErOIlmCMlaTc/IsW3z2mWUPOfMyhDwZy3PO4mXw5Nwn4405Er6MkWAZF+cI+LkyviZjg3RJhkDGb+SxGXxONgAoktwu5nNTZGwtY5IoMoIt43kA4EjJX/DSL1jMzxPLD8XOzFouEiSniBkmXFOGjZMTi+HPz03ni8XMMA43jSPiMdiZGVkc4XIAZs/8WRR5bRmyIjvYODk4MG0tbb4o1H9d/JuS93aWXoR/7hlEH/jD9ld+mQ0AsKZltdn6h21pFQBd6wFQu/2HzWAvAIqyvnUOfXEeunxeUsTiLGcrq9zcXEsBn2spL+jv+p8Of0NffM9Svt3v5WF485M4knQxQ143bmZ6pkTEyM7icPkM5p+H+B8H/nUeFhH8JL6IL5RFRMumTCBMlrVbyBOIBZlChkD4n5r4D8P+pNm5lona+BHQllgCpSEaQH4eACgqESAJe2Qr0O99C8ZHA/nNi9GZmJ37z4L+fVe4TP7IFiR/jmNHRDK4ElHO7Jr8WgI0IABFQAPqQBvoAxPABLbAEbgAD+ADAkEoiARxYDHgghSQAUQgFxSAtaAYlIKtYCeoBnWgETSDNnAYdIFj4DQ4By6By2AE3AFSMA6egCnwCsxAEISFyBAVUod0IEPIHLKFWJAb5AMFQxFQHJQIJUNCSAIVQOugUqgcqobqoWboW+godBq6AA1Dt6BRaBL6FXoHIzAJpsFasBFsBbNgTzgIjoQXwcnwMjgfLoK3wJVwA3wQ7oRPw5fgEVgKP4GnEYAQETqiizARFsJGQpF4JAkRIauQEqQCaUDakB6kH7mKSJGnyFsUBkVFMVBMlAvKHxWF4qKWoVahNqOqUQdQnag+1FXUKGoK9RFNRmuizdHO6AB0LDoZnYsuRlegm9Ad6LPoEfQ4+hUGg6FjjDGOGH9MHCYVswKzGbMb0445hRnGjGGmsVisOtYc64oNxXKwYmwxtgp7EHsSewU7jn2DI+J0cLY4X1w8TogrxFXgWnAncFdwE7gZvBLeEO+MD8Xz8MvxZfhGfA9+CD+OnyEoE4wJroRIQiphLaGS0EY4S7hLeEEkEvWITsRwooC4hlhJPEQ8TxwlviVRSGYkNimBJCFtIe0nnSLdIr0gk8lGZA9yPFlM3kJuJp8h3ye/UaAqWCoEKPAUVivUKHQqXFF4pohXNFT0VFysmK9YoXhEcUjxqRJeyUiJrcRRWqVUo3RU6YbStDJV2UY5VDlDebNyi/IF5UcULMWI4kPhUYoo+yhnKGNUhKpPZVO51HXURupZ6jgNQzOmBdBSaaW0b2iDtCkVioqdSrRKnkqNynEVKR2hG9ED6On0Mvph+nX6O1UtVU9Vvuom1TbVK6qv1eaoeajx1UrU2tVG1N6pM9R91NPUt6l3qd/TQGmYaYRr5Grs0Tir8XQObY7LHO6ckjmH59zWhDXNNCM0V2ju0xzQnNbS1vLTytKq0jqj9VSbru2hnaq9Q/uE9qQOVcdNR6CzQ+ekzmOGCsOTkc6oZPQxpnQ1df11Jbr1uoO6M3rGelF6hXrtevf0Cfos/ST9Hfq9+lMGOgYhBgUGrQa3DfGGLMMUw12G/YavjYyNYow2GHUZPTJWMw4wzjduNb5rQjZxN1lm0mByzRRjyjJNM91tetkMNrM3SzGrMRsyh80dzAXmu82HLdAWThZCiwaLG0wS05OZw2xljlrSLYMtCy27LJ9ZGVjFW22z6rf6aG1vnW7daH3HhmITaFNo02Pzq62ZLde2xvbaXPJc37mr53bPfW5nbse322N3055qH2K/wb7X/oODo4PIoc1h0tHAMdGx1vEGi8YKY21mnXdCO3k5rXY65vTW2cFZ7HzY+RcXpkuaS4vLo3nG8/jzGueNueq5clzrXaVuDLdEt71uUnddd457g/sDD30PnkeTx4SnqWeq50HPZ17WXiKvDq/XbGf2SvYpb8Tbz7vEe9CH4hPlU+1z31fPN9m31XfKz95vhd8pf7R/kP82/xsBWgHcgOaAqUDHwJWBfUGkoAVB1UEPgs2CRcE9IXBIYMj2kLvzDecL53eFgtCA0O2h98KMw5aFfR+OCQ8Lrwl/GGETURDRv4C6YMmClgWvIr0iyyLvRJlESaJ6oxWjE6Kbo1/HeMeUx0hjrWJXxl6K04gTxHXHY+Oj45vipxf6LNy5cDzBPqE44foi40V5iy4s1licvvj4EsUlnCVHEtGJMYktie85oZwGzvTSgKW1S6e4bO4u7hOeB28Hb5Lvyi/nTyS5JpUnPUp2Td6ePJninlKR8lTAFlQLnqf6p9alvk4LTduf9ik9Jr09A5eRmHFUSBGmCfsytTPzMoezzLOKs6TLnJftXDYlChI1ZUPZi7K7xTTZz9SAxESyXjKa45ZTk/MmNzr3SJ5ynjBvYLnZ8k3LJ/J9879egVrBXdFboFuwtmB0pefK+lXQqqWrelfrry5aPb7Gb82BtYS1aWt/KLQuLC98uS5mXU+RVtGaorH1futbixWKRcU3NrhsqNuI2ijYOLhp7qaqTR9LeCUXS61LK0rfb+ZuvviVzVeVX33akrRlsMyhbM9WzFbh1uvb3LcdKFcuzy8f2x6yvXMHY0fJjpc7l+y8UGFXUbeLsEuyS1oZXNldZVC1tep9dUr1SI1XTXutZu2m2te7ebuv7PHY01anVVda926vYO/Ner/6zgajhop9mH05+x42Rjf2f836urlJo6m06cN+4X7pgYgDfc2Ozc0tmi1lrXCrpHXyYMLBy994f9Pdxmyrb6e3lx4ChySHHn+b+O31w0GHe4+wjrR9Z/hdbQe1o6QT6lzeOdWV0iXtjusePhp4tLfHpafje8vv9x/TPVZzXOV42QnCiaITn07mn5w+lXXq6enk02O9S3rvnIk9c60vvG/wbNDZ8+d8z53p9+w/ed71/LELzheOXmRd7LrkcKlzwH6g4wf7HzoGHQY7hxyHui87Xe4Znjd84or7ldNXva+euxZw7dLI/JHh61HXb95IuCG9ybv56Fb6ree3c27P3FlzF3235J7SvYr7mvcbfjT9sV3qID0+6j068GDBgztj3LEnP2X/9H686CH5YcWEzkTzI9tHxyZ9Jy8/Xvh4/EnWk5mnxT8r/1z7zOTZd794/DIwFTs1/lz0/NOvm1+ov9j/0u5l73TY9P1XGa9mXpe8UX9z4C3rbf+7mHcTM7nvse8rP5h+6PkY9PHup4xPn34D94Tz+6TMXDkAAQAASURBVHichP3Xt2xJeh+IfV9EbJs+8/hzvalbt3x1tQe70WgSAAnQDClySGo4miXxbV70oLX0b0hr6UEPkkZrRmuRHJrhiAAJEoYwDaIb7dDl63pzzr3Hn/SZ20TEp4cwe+e5RSq7+p4020R88YvfZyM2/rN/9zsMERAAgAgAABEBAADcH/t39Ut/MCABIBIRgL0OIhIRIgIRIgAgubPMO3MUoX1P5kJE1V0QQWtg6C9KRBcaYP+PSEQEhHChnQSABAQEgIgIQIT2eEB3O+1awMwJ/nxC00CC//8vc1bVPgJ/Ipo2ABARs6Kyn22/AAgIEY1k3O3Inou+D/Y3K9gvawTCRVEQuS4wrLXKtsi33bZKmxPJDgfU5I5V27B2Cd8OTVZkHhdE2ouydiBBbSjt+1Upk/vB/BVGMmgHnGzLDODMsPqPtfFAspgjcPK4iGvzHpFIuzv4MXCDY2/nYe0HAAEI0c4D/6sTKVENjgTMSXlFHnYc0UCVABmiF3c1Ydz40MooVB+qMas65GaWux4gkiZEtOCwLQRw6ERE8211LS8tRNDmW6yBD8HInqpbmIbY8UIrPCsef6wfXyAgI3QAIIsgK2QkM9DaSd53zQqPwLcEEQE0EFY0U8nCDiAB1qQO1a0MTJlFnAcUMjKD4MZ/RS5oIWxELTwy0FEa1DBOfgo5jJKdI4YgbfcJ7YXRCsl1zw6UQ4oXMZHrC/l54W9UcWrVNqBao9HPInBU5KYu1eRMrrNYURzWoWbpybUKyE2SCql2kLz47Im0Ipn6qNUmT4XBSuLVqDg025G9OPqORGsaDP0x5KBG6IjTH2PbSVBrl1Vz4JkT3fRZuZvneys4cNIwI7cqHKxRVG1WVw1ZYU2wDcKKIiwFoMEnICPSgCunIyKrtbDSUQjAABkBA2CrOp4BMsd/tdlee2deBEjEEMlCzoPeAdad4KYa2RNrbbBvCMxVGCIAMEMClhgJALVlZXSzYHWeIVYD7fSv/89+rBG5GQD0TbCnIFZIc+A17SVws6zqnZWQ66vnJY8Nx4U1jWwJr5oDQNWguFPs0NQEVaETERkgq3Wypp+djqxdzmCsrhsBfAvAmW019UzGgvAz58I5Ncl7QToGcx3XRhV4kwYAPMUiYk3otoNsFT1VOwmtfMndiqG/rMcSrp7lB8sKjFzH6iDwl6gI6UuuUGu/w0DtG4sL5i5DBscAjKzSR6ia6AYVEIChRTw4c8LJDo2OxNW2WCPJcXBl7dSGhyGaMWJ2grguERhTyCLP/0Z+9Kz0EYDVeL4aXrTWQdVpOzxeSismqYGPvVmNxqoJWZuZUB+Xumirc+qnrwiGKmvCWlDVvfwkdV/WXBwPGnO+QZmG+ssSn0Ukq+upisIqAgGOyFYnWgUXrJ1iIeJAzBz+aq32H/3twLLIxQasYhrN6FUidqNnNaZrngZS3iCqAYgANGntNQTWpofnYSBk9tJ1TNSuY8zIWjdq4+UsGqiEazkU681GNzLOdKumnZkJiMzCnGEFBqjo0A+Bx5GXniNjWuFdcy5WcPFS8K/qSwcuzwf1Qb9gNKxcyp5BNYVPABqcMVqHZiXBmuKtdcga60YoosYOq0aVtfB8c1bHxBmK5sW8MYe2gXXLFZmf+mS42YKPVgyl+ssZtbY1K9Kxk9fbwn4QrMeqrZvlrSkvddMCc5gb1pp2JyIEBpW5y6xBaZFMTpOvkqzzCqp56uRLTom4A9C2k/nW1k6z56wwaG0M7W211hZUXvwV8/pjV9w4g1qnaMFRwoXxRCIC7ZtcXRj8vZz/QLXWVoEC8AyP9U9o5zwQMYQqMOImVTVBViaDa5aAyouvjCBy6s9JcGWWgcOhJ27vXWG9LW7swPlADJCM2UtQ98l8c6GmdiuIr0qp1iEfgbH2KzmWZYCOL11YgBzHOF+ypqexGjE3sM6tQCsEz9++PWYK4IVGec+ZauC2SHFNNW8RVnmoenl311JZxY6VqnGMwLzbsaKjKvb1DSATs7pgV6LDiGs2eLD6sSB/oO945dza+9QvB9oaOwBE2uk7BjbmgH5GW22+QtXoohOWHpnviTV2KvMIkIDXBsTadVS11MCXMXZBzI7SsS4JRLQxCERC0CuTpzY2F6/jfqkrW/uNu4kdQj+RUJtfGTorwnXLdhGsCrCGkJ916K4LjsTRqW/ndtSaWtOkNaPMDicyrNxLE2IxHffidFJG31Nng3ioOWPSXdtLYGUaW0l/Cda9MWrOXiHZVWiRNtY2klUg1S2qLoDzkwDAxZjQAcO1zzRF+wFBZIjcdquunA3EaLWNKz8iAoiqM/a+tckH4AXqNYUNctcutjKDa7EJ86WdIXZeQCVuhAtT35zAvNCdiC2B+fiEH+GaO+mJ34LVaQFnTlrNhZVycghw4q8UGkPSNrLt4vxewyAjy40rto8lW9MjYoZcrUK0POx64Qfcwh2p0qoWqz7wSXRhUNz9nD1qcWdMCh+/gy9LW1zsdKUhLZPVLuvGzjWz+saLxKgCN6CWC8kHRKsGG8TrSuujuxR5htJfpkdsX4RVjlXTyPN+TQtaIjF6AgCZ0eyvqoNX57E7wrET1rBQYyQHYAepSjDoEj+V2nOdrIte12TvXWcrympkVsOQWL+5HWhy0RwN2t3Qm6FARMZjW417VPepUbQzvmoWoe2XYStHMdWEdKoP/HxzXfSZOh/5r+59AbXGeKpM/BWXyd6uzvUrP7tb1O0qrExFGzdwTcVqNvmsIfPsVHXFslLdEvDviFZasNJUcEThCLHS7g4iNcohx4jM36PGczUX114daxqhjkj06SqyQR8A67QyP0iGh4x0GVKlwoyRt8oPNU0F4IOFYJzy2pFkjRQ3jk43oTPprMZyagGt+nQhN9+FmhTdl4DMRWqrcUArTJ9W8m11t8XVy6ATpj/fDS0iAOgVR57cXK6NN9p0gxuzGgM64gRwwS+y2qyKpNpZ6uaSmYy1/tbEYBR0TSPZ3lZQrOYB1c6n6gh8BZyI9T8AAKKmD9AzXO0M52QAAoB28LHIsdqkAqBpen2ae3qw/OS8GXsNp0BruRDbM9TE0Sejq9/QBSMd0Xg95Nvm3WWnRV2a2ArJd5+qP479UFfBkUoEULu0U/KV6oNVcw1rJwEAECGgrvl8tr8rg+0ZF6imZ8gRmLt3pcfrkwOq36nGpeD6i0RU5THtMNiuX3AELjLlBS1fM4tesQqs0q4JrlID5BSBgzV4lWHvYnNGK3RGRKI+XLUqA0AiYnaO+ytWt6GaCqvBxxmgNdQ7M7QSi0sokzsbYWXW2ps6m94lVw1xIlgHq67vap1y/1pjzBCzs+XAWn3kZvuqiLyh4gRIVir22uTi+37G2hdDIPIFE2TJsqYM0TceEcEk7ldVTjWTPStWVloNgx6vaO+8gpJVX95fuvrtohPv54YJGFXM61T4xUvR6qe6mbBid5jGEhEgA9KeJmoNsoKpCBdqdzd4Jc3sQZ55EEyG0waKzYT2YPNE467FAH1I2SPFyxSppmfRKkHjV+Mrwje2o+VEBLAgRjchABCIISESrBSIVPzmbk4ezXRhSNALcsULr4ubAMgOGNQNa2/DOt1c6acVVWeu4SI8K/UfAACga+G1uv706MRaY+oGmLt07Ve40IdX4yI1qay6VmiHsQ7ui69Xvof6ePnbXRQjuFFAQOZu6rujbatthQg5KazC3JkBorqgs3JYfSJQpW78Bew/lp5qsTZnXTPP32hiYisT0ZCiV1Xo/JVKC62wEyEAdwl3G5ZAJG2igACrI+rnIDilie6CF+jKcZj3Ro1N4NMxrr9OHVRcUaUdnHlDfgSweuubVOlAZziad1QdVJnsTiDOu6prrsp/8a6tlznW7lgHUDUPV5FVfeUUXV04vq/1l6eDCgnk4r71bjuh+p77QIUxbCofAPwQ2Pa493buM0ThjU4TSK9ZPwSANsRJxrolBrZMxjGGLZmstDhYlFteNQ3yjXDq3N4OkQi0uWzFIC5XAd79RjRBeT9dqSZa329wISrHkxV7XeRKsLzsZpqLuPtP1WiaD1UcwqGEHLH6SVYB1dl3KxNtldouxHesjGslIljLHrhv3Mx5hSYNXrXSF+9Sh1pNBAgVlNHJz7d2BdarMvOjAwCgq7Pc0XaukfMxLWM7Dc0Y0+TI04q7GlU7ALUbE4DQldVlhepiv15X2hGx3qizlS4YIzWHtKJPcl0Cr+UulJbWJ5q5lyM/p4Yr0DGHhopRfD/ANm/FyfNdcCzlm1u5q/U2+0m2ah3WvIXKlPDMVrXKIG7FFFiNDXqS9WlPp838B6w3x4p21TR0XhSsvrT+EnSuQtlpFSJH5EAARBoqVodXX+hqzcw4sKrOrvJg3XBVpzhDwt6W7DiAK1oCXAk8+SGtcTMAEYmqDxVMfdKwrsKdTVLZ4dY3NCZlBdlKy1paR6xoHWoq3BzPTFn7ilqtTRKn9SrFWnWHfMvgIl+B9wncEXXZVxMSHHVZOXptZOMHqwzsu2475+ewvzr6fA1WCsuOjFN4tUbAynS9ACnvwXhKqyPVf3+RUJ20Lxy8ckylD6zYvM1jBeJrAF2g0LIMVQDCWu02eYr2sqhg4Fy7ipaZayf6Y5xgrEIC1zXjqDvCcgMGKydWB6xOdKN0kBGgdm6MA7EZS+ZPNO4XVnks6xIBkC05XXGi7d1ddT06UTnh2pwhWfJ0QjJmADq9VXP+6yrWNMrXwmJFcUZq5q09x5kPXiN4saD/tzK1jOZf8f8qrJi7uuGqtK5v2avodBem+psa7mus409xDm/d+7FOur+i18v+zFq62rTvS1tVNchb+eiJAXFVesYFcpFU1/PVqxHU7Sn72c8KAGSrVrVRWlUSuebMgbeQ6vaMMSjc5LOpT+bTJOTQ4sRUHzQvAxehA49R9FekSuExqCpMnRFoNbKXOpqlHd4CuOBmOlOdKm/Dj6sbt1VAeCUF1vKs7uQMLNdARMYsN5ATXcUdVf2/u8KKIGtDgBVFrQ7NxdJPU88AlqFXzMqLp1eYISedSpWBo7bqfJfErqYWwAVD1pv7VHuZ08kreGfXkDXDmL+NWbTkmmjzKibMY7GPCEAMrRqtKQV7J6dZai3yxzicWHFr2yUbNTAFqEgucWBZqqYpVl6OwKA2+Rzuq3nsZ4G/O1q+9DyEANoPk9c1XjsjIFQrCMyksUq58nFq96zcoMq2Ii9Uj4aaiFYzPW4OuLmHYIN3hrMtj7tX1aj61VfE5DBR00O1H1d65tpbI3iXP3Etqt5ZavcGja7BvUbD5gQLxAopXmqreqreNKehicjHeKAKNiIAOb6qjR4BIIqa3lztK75C7oh1bq+pM4DKlkJXEvKK0VeLOjDjWtSq0+sHoHWqEQz0ybr5RizMpkur0fISJHdkHVTgRtTTrXtPzltGJw7bPquf7aT3g1ybLOCMbrJXMHRkqyJqp2rDWL4idpWqzBfeNSR/WSunFZRWCueC2eoU1MUTVoQPlnHcLPT5sIqGWDVHvYnpPYG6MKu2X3R5V039WsPJ1toCgY2Dej0HFVycTnLfoikWqfcWoKYyyY2aw79BgWuxuawfbj+UK/asuZsCDVb5gvcevI7Q9SF3bSOqFj35yFRlezhp+EY4uYM1RK1jUdnT5LtGADWZO6MC6sPjpoO3n4zxg+B8BqiKOhyuSVu4VVdHrLwmB0E3AyqsYjXwFt8rKc1KJhUOakPmD6op+BV68KKg2lkXbk1ElRzd5az/pPUFw7F2DPnTa9qmIkxwQ+ksnKorPgLgZr+TCQIAc0FFEK+6eBeYyePN3xMdD1XyWJ3TlaJ2ROeNB3S6vFrJ6kspbHIfyCwt8oyLzkqoId9cnrmWubkP1veqW07Wd6rG12YTmF/uUo2UBQdU2DEKyPbb9qVmjFVThhBqJdhkKrOp0hpA6GJS6JgCnbnm4bhSmuS5ocaaVB/kVbCuGp2VpF6Fuzf0/U/oxnDFKnCX9YFYkx/xP5gBrYjDOXHeHvBWTsUP9eSGE40DazWVrTpCFL65vkHsFbyaUarPXy8uN1fAfnthqiFAtc7QDr5vnGkOA7xwRs1yBfCAqheNmFt7oiIA5iI/tUOs9C1sa2e5X1eaCbU2vaJnKwHUPtcmEVVXIQACZMwMle1Q7farI+Mv6zFa1XTWKjw9bqi+DgEv5NBXm10P5l0k44u2WwWymmtfb6qbpWiVD67Axk1rL4NV2brEksGcFxiBcwhXNYGLJiMC1JwG31Xj6fjjV9EKQKSdarYH2JynP74mQavk/Ky3NYJ+7nvmswxVK5jytrR2PeeA9WXQxqm22fkKzmi9AQSnVmoRUqrE6iRCta5UMnIcDPWWGvlWysoitBZPcCxoLobOCwNEs1uBu5rTM26kV6jXt6Cuyv0QXtDvK15szZFftTL9l5Ukav6WVXHkvNzapPPjWHtfN0Jr4jMjiL7pVawJHAwNFIicU26vsDJ5bBfd5HJLPjzmvKVQV+JERKSNHL1KcwNPNYvrQm+shvdHEgGCttFL5x/VMOqrRTx32gahNxaMFFxAEaz/zQCYn97GIHIzAW0b3ayo1I2b/dWV0V2V3Kx3CqNyLiutbgYXXQS2ukI1qH40fUiaavfy08DwsSdCN8ArPOSIFi6+XtXydVr1JoTx2KpBraz5utKujeBFVK5onrpercbFCMWYs1Zr4kW95ia87aphHkCwa/otnfkyBwtQ5u21SklYFq1nUXxrPIUS2dhKzdbyU7Am31pNgI8Je+WOZOHlpoanIDQ/2Wag5S9mDE1HUWbiWkI17VxR9DVtgIZQWPWxzlXuSGcuufCi740RKlVaE2sNtgNjruLfO4cLyIT00NGM722N3V/hEg9716RVZf2K4qrmHmKVz3SvV/Vv/XVB718wlpxfRX5kXm2qlfwFB9S/BRsTcL5TnczNZVfsX/NXGA9BW4fXmHLerDbmgzGlK/PIX1o7oKK9v+dtu4DTfVztSN31czLWCGgKNwlWnA2oT/AKwU412C5hRf0rwnNmqFuLvKLWqU4GVFWSr2b/rPdDNW3kysN8gsHalq4S1BqxHtG2Zdqrb9+1uklQp6gap10A1Kp9UiM2x5SWuhyW6p7QhVOsUn6lDLQaKYDqQr499RGpCbyaX37+umnvRw9sl8n10hhj5Gax1zaeGZFZ3e0miOk9EGiqxo/ckLr8kLkc+WSqKVGprDqvx4jcR2N42LbXZOwNn8qgIbfiH/xccxkG1xgCIiSbx3fGluPv2pSw3GML+Wvz8yJZ+Ex0pTRqNoDlL0f4Fz1ia0qYA1cBRH4CVzrolevV/oPK0LDTG9D99GUp9S9lshqPejXoJeKPctPnFXSuaH+yd0H4khs5RHmrsmq/+b7SMx4b1jAwhqEG0tVMAvSmEgEY9S5s+xEAkEgTugSvAbC5AQJpS3Lmsyt7RABysaGqEAkA0OYjagrS6keobghAzLhsrj++BMMfi64prjyOnEfFfDoObHfdncmbMk6MjhGhkik4drkwPABQHVYzb8zNwXrXNZokr7PsjHVhaAAy2zq69qObnA43WFV4+lG3zhNhrba/dhBRbbLVSGTFE6pfz+uFeocuzlCopFGnbK+VfLXkyglOvFgNvVNZda/DKQy7N1g1xGjULOnaLCY3ZPYvc6rLmHBAREgayVl0hibde/dRW7+HnIOEgKTRagyjzmrDa/4lBAIN5O/nceHJ1azl8GLxzfSdNE10BqsLzlnxomsfeHQ6nxvQmL61YanQuUoOjmepJi/bWu3bZTdQQ388YlV7X4WpEG0tgTEVnJIy7XQ2ghWDd7NoFVXkzsRV5q4gBMAY88C6cMyXMN9/5oXO6fTvjWK4cIBv8IV/fQO8T+J0JPiou5uermukLXVZmK6YdkREpAVWBKZtNNNaPnbDLbIkWs2sFceWkECb5cg2SoeWVhj5CDgikptW6NzeWtfQB6zrc86ebaqkzSw2R0NNIVZIBANp9PUdXiSGcR3fO9FDNWntHIOV+zp2cFUEPqFZ1xYmt4AVKVJVlI/1YcKq2Ssr5qqoUoUrK8RatmPldZHsoTZGr77oYkLyP/eq3F9HmXVevHCwvxdjTGtdk+bqbLcdqc6xGs9OWfM1AGlwYncAsjAQlXNX67PxGMjxW4267a8X+1YVR5oiOiRwaWhzTbKmkRGXsynAarAqfl9lNbyPr62B6MwKBzg3uwlMagxsUt21xR5B9urmpu52nupq4rR2e/UZ7PwGBFczapvukr727qYozDIk+Ru7s+uRf2b50Q6CBza4eCrYyeQA7eysVzyY2rh4WNCrduorqL3gZlUm1n/B5kFXwfAKWFeuU2uDwV8Feqd6nEjrtsYFPNlOG+AJfw83HOj64F3WlXaQb65T7uDu5t9aB5YIEN1ax0pYRITuf4AE2hp3hMhspM5ONa+pKx/IrSQhNzENrZmW+CVK2ql6oz7s8uhXyn5fHbkaFV3IirkRIjdCteHzDgB4ovUwtarK31Q7I9NJzImOagXOXl0a/sba5ITqVuD7Uj/Rf3Q2d2XJOBOsxotVrOG/+NKuDTUQ/5csCqrnzMjSkLZCtA1FdNuEW1E57nOWEBARCdAK3C5CRiGx6sp2fS05Brbt0y6uY+To4gz1lqJFp2tnFTfWnoXRF/LYZpvF42jNGPSDh4jgksCaKik4/8iBF4zjh8Dq0TcAQDT7770KTXJtqA+w7UG1pIGYLaomyyNQG1lcGY2aLVSjXqgUIDk8e15lUN3JN9j6XNWYr4DDHWbbUNUVOIvK//uqx4NV3Z1VIl/Cu5WAnMfpf/dBPa+OV+VpNZx1sqGqqGXMCrhCtuU0gx+HFjczAQBA6No5XozV8BIoZ1hUVGr1lzOnzEy2MKvW/1SD5jDqsOgCjE58bjaTE1dFUYioSXsM2oll/Vj0Q2VdJOdmVz2og6PqWPXOXsRbVDbTa4KmDvHAKhl5bK4UZa7MTfd/x6P1e7pgRk0uHjkeOm4CuJiMNVIc7KjWIW+AuS9r7rMRC658dBi6CKxqvC98U+t3bfRda10Gvn61Sjms/CE/92qsbXFRc3D8SURWxZOu8YJrvXFINKHPX/kZX8+21e1oo6S8biS7gJ385gLoM+4Ort62cSqwYiKn1ezmu5os3drHcph4PjMhUtMd26zKEndAMZKoPlWK6aJ+9EDxuoicejJjagREhHa5N9Z53k4MN5o1xnE/fhkJvEpC1cvVfLmTatErj3U3aq6/7nDvaNYogJxX4MDiW23TMPVz6yLyUvpyI9VpMts28AY6WIm4G1uGAoDarsrmFj4c5wNB5nvh5oibr34uVqzjw2PGHjJCQQDS2ulY1zr71k0yMl5eFTsyAQVA96ZqLAFjCGAel2FxAGBdHgsVUmaQ7MukZhCBMQC77WhN7lWays/4C+7VhbEEWEVK3TS3Ss4vgXCArcw7gBUGdy4Ueu6vq1fn29aYg1ysEd1MA3D7/FTfVA2tq2T0zqXjBt/n+sUJPCtYE2+l81hxR/1enjRIVxu0ebl5G4msR1gz0y9afpbinAI26t+FDKsIFPnpBHbZcZ26jNTMKDKs5GAeplTjWqiAau0OvyDVwMrcQCmjmD2JuPwokSbyMkEAZfvpLoHM7UNid+Wyjq/W5lvGGCIjAoaGMO3mPtqqZGdqVO5dXffXMPHlH6l2rMUSVOJmfjZ4jqX6mVYtMqpRu6Nc9EJ0NoNtqnc78BWjfqVtVq1Wprj/FT0OXqFS27Ia8znlAwSEjDkt5OC7cpIfb3BNttxYCbbePHKNdicanVyFhowEELVNEFfnVHdEBCIBXBipIGOOqJzRgd70NjF0Aqv+DVdbY8BckYiYk5fjXSs9czuTDTdJ/yrLhCC1BkclWmvrWiETnHPBOWOCMc4YcqZBo1ZGFsgYETFGjHHQhNzOZmYqQ1dTl25ymIhgDXernsGFjysc7DjLO8jVWebAGlW6UXLvHI5th91Y+AHxs8gw6KujXqnyVcX5qkJ3N1nRCeiH9EsDogY32kUi0ZGoOx1qgK5+81h13Fy7v7ODoZo5rArKVcrd2Xg1S916jE7zAAhVKjsa9kwCk4i3ZgGRX7pOdrJ6Oge/vgpA24y+9X+UUjVvirS2q9nMUVpr86/WWmmSWhtLxQCUMRaGYRAEURhGQRAFQSg4h4BAkyzQApRzLoiIkJGt5ajK1bwAqllZCdjHny6Mk51etVH+Eu3nTqupafexUqsVBH1iE3zLLiDEfFXxtgdkjZc9MtDbEDUurJuMF3SCw0s9gfwlx1RMVwnrS14XVD84f+kC6D0xO4vDuNB+LrHaQ+h07ZJGnuQmhZkFKObzBQGYImQyiXUiItCkwdWAGvCBVdlau9JQTaS1VlprAiBSWkmlEUBrJZVWpEmT1koRaa2kUlIpg0JFpElLJUmR0koRKSIA0lJqgEacdJutfrfbbjTbjYRRzCAEhgSaVElaA2OccwLiEIBWZtg0EUPGnJTr3i6Ck5gdBled5ATqe+fH78IoO2u3JkkEb+pRzWq3I1LdF83WHZ7DLcOt6DSXU6lFcNwlPNjJq9SaQqysuaovuFJtWPObLvo9dSL/z70ukHfVPKfE607eRSuFmN8qjaxtYztqL0FeKCst9d0nIHF4eqqJSJPSSlkgWn6TSmvQWpOSysAINUklldYladBaAWmtldIGjUprTZq0UkoTkDKRLQKlNQNCBK0tzxkAmC3kydpwIKUsZamknAZhmWUcMQ4CGQQ60GQmBihd5KA1IKcwAGCIjBEn0pqYzUe7YUXPGZXzawMBUE/2XBiPCpjODiQCt9ae6JURssis2JQcbNxlPOGhHQJ7nDXnoX5B0ySsobbGlNW4OUPMVxf4eB9Ahc7K79b/GSS6pNqXqH1ErbU1TAl8s2n1YzVZX+FRqBSYUwhA7umL5KXh7KKaTJ0yNH6c+IvPPlOktdbSlN0ppbQyjCKJFGnUWluKNY9DJQJQiAjELVUh5wztw7IY5wwJOOchIuM8YAw5RwDBuODmxRAZ40wwzu3WJoyIpCxny2yZLTVQGMWIjDSVSinTJIZaS1mWWkrOOJBGAsZQK8aY2ye6St/UctneC4aawCqIVGWaflCdYe1iG17KLvtSw2ltABwTOEh66kBnETjpV1WjFg2OiN2Yr7QTwJWr1p2h2hnWliB3NUMzVvXri0bzRRhdSLsbO9Gk170cTHPqhilU0Ux78cr4uKCAPMOS9dWNsQgMzVbylqG1k8bKfBMvjl8CczAB5IiaMYYoAELGGBMUMMZQMM4Y45wFXDDOGWLAOGPczFHOOeOcMyY4t941Y/ZXxgLOOGMaUDDGjW/ODHsyhmD8dNJUynKZ57PFYplnSymBMQVEQEprpTSg1qSkLFVRaMaACDnnSmhOmohpAm5IBd3WobWcvNfm3vyq2z6vDJj7vsaL5ELCZljoyx2O2mesX9iF78AbBubZZORrWQzIqhv40h034RBXblG7tYWUb3HNVrnwpmraqnnjZ9wFB3/1xEp3V5ijCtNVBM3WuGF1M3Q6xFbPMXc3++1FjVO7r7hz5ZrhM4aMIWOccWSMc8ZQMAbIiSFnDJEJhgyZ4Cg4BxcEMgJx33BEYAZ8BnsI6DeEcfEiRHNBZIiu3Aq01qWUQZgJEbClKOczBQSaSGkg0kohggZSUpZloRkHRC4DCjRpTVoR48Ye0mi8Om8K2j92eBkD68lZ4dVs1FcGcXVcycMCCejL3eGaWYn+LuDcT7COl0v2VosPnS9sdzJatRTrd8cauMz3nv/cBHIHuxZdPGPloyM/589e6FCVWa31EWyEa0UCDmtE3sjyxIsmaoPIjD4yc89mIaqQ5Uoc37dU3L5+gwA5IgCabClniGiZzizfMTFH5vddAIM931rkzGysY5G5kj90tQ5+3yKDTdd4sIar1sBYqSQXQgguEECTIq1ISyk554wxjVqRKqXUqBFRBaVSkmsbuTKz1xqZlUXoYt0AAOZhzL6itmZMeRtA/2ez0s7YW40xoR9cIlctZe4DaJW7T4IQ82ZFzfkwDXPjqX1hg8U6OUlWBkTVYIsEK9tawyqGBHv9L+P7evyoupG/qZ9Rq1rF0179ItafN5dy85Tqixc1gLaocUOENhpei4DU5oIhXjHoDmx00LAdmKQ+GvZjCIzZ+mDmaceZHxo0kOFK8M5P3QV2g+aF5QrJKj0LyDiBZgDEmOCcM64ANAERlaXUoTIxKamUZqQJlFIEWnCujXlKittZa29V1Wq6+2o7kljjpdVaAagNTDX0F1fzAFj146jXkl39flj3GzQhQxeItvnZuifrTMlajKC6mbtPjVNXScvGfSpqdwL31ii5ca3FLC4SKq7gr24ZrDRnxU4186xmkddVv+diJ0DzzpxONV7zt7GGgTvTm14EACJNG+TN3hW7yTGmITxrL5uLaCMTRjasiojA7EN/q+fOVdMCAVETWE/bJZycyQyaMaUUU8r6UMA0oo1REZlgFjA00VklpQaUQkitAyLQJsRkfR2sZaidbVPPeRos262ia4NtRV8HQT1iYgiSwC/EsMTnj6xsxBqNoRegCyMgoDeOLbNa/qxOdjTvFLSBGtR64gas1pTarb19XKnvFV6qRhhr+W+HVLTWkY9ouQmwMgeqnK6bRN6aMrVvnuEr46DiAMeZq1wMdevLhxcED5gpF3GGA6GjT6uyjUK0T1a1IvZPtNBam0chIkNknBt2dKG4+hYPhIxbBwa9dDSR0hq15pxLw9wAJnVEJJXDCJjnfJLW1h5FZUxPE2PV5GRBBLbFQCuKnhm9gs4Q+LJXHWc+vIc1vYHgRg9tssN71nWCcQOBhCiJSqXnpSIgqfSoKENr7iNn0A1EzFkoOBg5eHuUIfjgqysaomowTU7O+kW+M/WQ5EVQojdzbWFEld+yqtweq0kjOi+whuVagMny3MrtKiaqGyy2yhGs0jWttRh0ee4LM6ZSTGYmCIYINoJIYPdpRLNFqDMWV0pdjOCYQx4xxhCJMWRMIPpKOMM4HBxKLHAMl7jhRBeCI1JEzMSLrIvFCMHYpkor52QgIhKCVlqZ8KvWSilhph6ZJ0BY9XNhlBxp2VX2tW8qfq0FvK0qv2DZ+fY6FeDlXrsZESCWAOeZPCvkSVZIpV8ucwYwLItny+z1VmstDJ4tFuOs3EjChMHlNHq73xrEETP13eiKtrxt6ywQbxxo10nH5Gj+rOhlwzfgZ5EjtJVwmGu6m5+e9cnpzBqyqPK4qxiq41Gyyr5S3ZWfamdD9Wgh8MWu9taVavelw0RAJAQPHMd7i4V57kRE6zbUuJe7xI+ZWWicJEQAE2JyE4wUQ9SARJoZHBO5XZerdhs4Kq2ZQSgiMiYELxhqK3NGWiulgKEpvzD5KSWVVgrIZk7N0xlNAQqSsTeo7tPolUyF92+8PVl5AKb+1wxUXdlVgvfoQKdSwT5kWxOM8vJgoX4xXe7n5Wtp/GJZ/tHB2d+8vMMAn40mCpdf6YZnRfHT4fCG7r3RiP74ZPiz02ES8K+vde902pHggd2f0sKKKhihtTOqSWOa7cDg2mg1qOuy01dud0jfo5ozvmKlIK5eCirU1V5klXJl9PgfrDFji9uwbhzXo1D+3qyGh4pEAYTZU9Bk261zZOjT23FY2UJeZbKqQtR776ZPtrWmDWRntxWDcTYrNVrzlxmiAkCzAYrRAwRaa6k12YpVICDiDBC1Mll8pU0+n+z/TaQLyC8M8ZrL7Thfk2yd+KoPq06r0QMXvqyMeDMBEM6yMlMkNQHC0bJ4upTExJ+P57OiuJWmz5fFrVYzFfywoGku201xuZFKUqdK306SqVInhA0hNNGnw/lZrkNEjriRRruNOBGstuUuOKfMzpyVgIVtk1epNbeGISIoRUopzrkpaLzocrkhAa8bV3/03zqL3JUXrMz66joGZGgnjtM4hndr6HQ2qzuiNmbmhsLoOO2SMM7kQl+fx40NxGqVLAxtOIQ8PNEiEam2PR6iCSK4XtSMPMttruFgzWAExhhDBiYuD9oSu5tohps1aa2Mc6Q1aCCNNZbxD8y02sr/BRfPcErJDzm4GpsVNPgxc+LzysuVYEMm6dks35tn+8v8+TK/mcYa8EVRPsqKrSD42qB7fzrth0E/bq7F0X62vNPpXG00/uRsuFB0NW48Xi4Zsq93B52Ac9BPZvMfHJy/2UhezmdLpW42G391e3Cr2+LMrMujWqOdJ1U1zprH9aYSQFEWk8n45Ox0Op0mSYLILu9eardadeuxQsQF9e2MGPQ/ei70hFdX4vaCPodXv6iuyc+TKTqTtzr9glkmLNUwZ1YhmuyQUwJ+dMERKJrV6wx8N4CIbLmxScjVXDbmGlRvMtSnSQVy5meI+wxSKqmUIs21tqkBBBOd12TqWWoPEAULP+fLO6+0GkbyITHw2sTGSiuLvVLfgC6GVJ2liealmhZqoek4108X+cez+b3Dk6+s9UeKSlWOpD6dTIjzb3aai4D/fDonhC+U/MHx+Vf63QDhZZZ1gvDr/cFhtvjXe/tNxt/ptglxK0kvp8nxYnmz2eQIZ1n+o9PJ80JLwMtJeKMRBojCZE2AGDIC5986Kw2cp6K0yuez8dnRdDKcj89OTo7LUgVJIhiWs9Mbd95tNdvWG6gpblzVHjXYVuq/ApeNDVUAtiIj8FEj8H6pw/HKZKjF/ZzkLZz8FBAm2ehs4Qo3zHozoLX1ujz5+1VfjKFG4D6rROgvYAK0WgMwby7bhyvWnWXTDJsaJRKMG4s2YJyAFJk6FucEASEyYIwIbBWVUkbFO1CRBmKakNlyXnebCl7WiDRSXrHNV1Q3+GnphpAQl1KNc3mwKM6LcizpSV40uXhRSAHwm1cvZYg/G082wnCsqRFFqeDHUj8r5PU03Y7CoywPpLqWJNea6UFRTkspEa63Wv/w+lVQaiELjTjRuBbHaUSfnJ0d5vKtXvtUqqcno4NSDkT4rU6cCNZLEs44J1VqnQq2mYTdgHNnr2uCIs/zxXg2PJ6Mz8siAxbEcXrl6g2ttdIglUSZjw+fNa7fFSKoOMKPBlgoYW0hHq0ypEEsWTbSK9cgiz2H8HqMz4q5fpzHpfeuagreFCw7T8H+4w0rj24XtvOmmLml1lozEzdyKqHiWyJyQVoX6vMa6cKLuTCK/cgYmBQooiyVUorsniXGLjXsTsYGVUq5v1KEIeeccVZdx6ooqpQ7IFAtC14Tra1Xssay5RJNOJf6PC+Hhcylmin9YpFnSsecPVpkzxbLd9qtsZR3koRINxnrBPyoLG7G8d1BNyf6s9PhdhJfTZMux+Ns+cs76w2OPz073w6CS1Hw+Xg0kvp2q7WdJjpb/M9P9jbSxq2WjgVrJqkupy8X+bv9LmbZ3Wb6YDJ7kKnNNJAlU6DH2fLPjk97Ufhmu7mbhJtJ0Ap4isCWk3xyfHz4YjqbIxNRnMg8V1KJMMrzXBPkpQwD0dVKFpkBKFWYpAofiLWUhFd4VSmfobP6BIcqfIL2gZzWACCP75XsSC18BhVtgz/AkKbQCOgD12h4DsF6sgZAFdTJcgmgjwZXNmulBtBW56DddsLYqG7WVOkytLF3v1WYMRy5iyAoIKW1sVBtbIvZ+aC1zmVJi3khNVssgY04E4ssWywzpZSWstlu7Wxt97vddqtFYAIz5KkTELkLJeZKj7JyWsqlVO0wGJdKk95IotNc/fTkfJiXW40kZmwk1X5e5poEw6KUSymjMPx0PDs6PV+7uvtut3W0XA4Q16MgQfpkPOmF0XYcxYi/82Tvdr+znSbdMPxoMmkwvp4mcyn3hmNVyLjbTYVYRsnNgD85PN6N49NMjYpiLU4I8c9Hs0BTP2ncHgx++/GzQZz+rZtX8rL4i+mCR/GlNPro+OQjxot8eVvQr241OpBPxpOSRKc7KJcLqSQRLPNseHiU5wVyzrhotdt5USpdZ74Vf4nsGFUj4o+oRxRW8+ZW9TtF5wvoTa2f110VF1vDb8Utq13QfS/cvr8e4PalzdIf+zUCaO0XalS8XNnYzPkSFT2hjUFgbZcRtmIQg6kX8QEG43wwxjmwqu5Fa9sYIgSbONCkkUgr/dkXnz978kwpmeXlcpmVZQmAUqoSIGk01zc2vv7BB++89dZg0I9EoAlM6RYBHGby49FiUcqCKNT6YLmcSxkxcaR0qakjxHFe7u29vLo5SJLk0TJ7NptvxmEvDDnnL2QZhdFQqs2Qp+s9xvl/Op+cTGedZvP9XvuHwzHXcq4UAd+bTlRRrsXJN9b696ezo2V+pdFgjA2i8Eaa/Nns7Acn56+1Gg/my93B+neuXCatn0xnqMVGI2VIf/Ly6JsbG3d77YSx0e7Wz84nT+fLUqujXA7K5R//xV8U52fddvKd3f6vv7aN+WiumAiTBggpS8nEYrGYTmdZXsyzPF8uAUBpKKTsdTobeU5Nb15axemw4j0IN96v6D6bgrH8WLcNLQq9f+KDfQiepGq4ruH1IjqJEEB4uxacpqOac2R6YJxDi1MA+8B2RN8Td4qzPQHcNPLTjlbSW1iFwBnachPS2t4XgTPGEKV57hECEGmluOAAwETAkJli3TSKv/bee99+/71QYLZcaFmGYaiByrw4Pj394uHTn33x5J88evwf/uAPdi/vrvX7nW43iRMWp0eK/8nZ/GQpsdNm3c41rXdb6SBOzsvys8kEcsmD6Ea78e3XbxZaP1vKe0/2W3H8lUu7PxsOl3lZar0s85SxD7bXP5tM9ubLsdK/sr15UpY/n8wWSv/1zY1mwP+XJ/s7ofilqzdPsuLH55OlUkEYbUfBh+ej02X+br+FAf/sxcH6jau/tt7544PTf3F4+pev7WrGQ17+dDj8eq97o9V4vszeKPX96fgP7+3dubqNoH/2+PFkPNHD03gyvMKKv37r8ru3rz1/8UJK2eoMgjAOWcCV0oBFUY6n89FkEogAkGnSSqv5bCGVRO72jXMPr6+zoBkIH4r34ZaqsMIhEcGFgVfY0PmfgATKfOswfbFsqrJXa34/uYCa8EBHRKNAnaGADswO38ZZcfmsGt5NyNcarc7WrNL/jke9rWubYeLhPh1qVAZHxl31HwEoTcr4Ssb+AEacE0NyMVBEFjbbrThWOJxOJxzCVqfNGe+t7bz/3ge/cvjyP/zHP/mLB3ufjsZRFERJHAjOkJEQDalDScsghptv7DX6nz8tO+uD1zd7b7Waw0Qj4wutn07mo6xAwTe31iLGPz4ZZlINWo290SgbT1+/ujspJSPsR2JbiJOiGEupFf2d7bV7kzkCAuc3eu1lkf/7oxOYL5uE37117c1ee/Li8NNHz9/ovnW91VwsM6XVsNC3e53z5bIdCEySHKh4cTxpNG/2eg8nsz8+OT04n6go+s7Wxh/d++Lw44+bctmnrFku/8Gvfvfa7ubRyZkIEuRqMpk2GzoIQy7C9a1eb1DOFvnp2VmZZY1GEnFEwlmWl2UJvmbF/TFFyh6aKwxnQ0s1a69W8kI2a+n50cfIvQvv4vn2GBeKX3UD3I0qdJF/ErZN4tjISh3f4AKUAJ7MwNsnKzwKjvap7sVVN3fUT7WPAE7BGA1vU53IGHKGDEmTVtrsUmviSSYWZbP4WmtSSslCU5o2tNbHpyej4ZCIAPlsWfbXtv63/5u/8Xe+/0EvjXSpymVeFlKWCpVKApZyakzOwp/8UfqLHzQoG8/mP/3wi5PzaYzseLGcKV0w1mmlJVDAg1Gp749nu61mNwzm86zf6dxstRZSf3Z8dr4selH8cDh9r9O+0WpMNTyYzv7wky+y83FPhM20MRCiuP/gW1uD1zvNo6zsJMm3blxZSyJkvMgKPZoQgx8dnrJF/vx0xBgfxLGIo8eTGUh9cD7++fODnX7n8lrrf713//H9BxvL0eXl6QZkf/d7X799/dLZcEzIC6lyqaXWIk7iRrvV7THGo7R56dKltUFfkZZlQUBKKhHwOEnSpGkItD5YdVDWIy1+lGsaGS06/UhaU8GHP8wPzluqxanI+TIWMzVAmFONKWlDjT7s6ljScJsxSohcqY25LlttguvgCuzq8wEATMUeVZEsHzNykclVZWGSSQatZqWUTxeZ85BzYEiuyomIiLQGCOLm1sZW2mwWZTkdj4UIGs02D2ISjV/61rf/q19+v9+OC6nyLFdK5YssL0ogEEEQhEEyPmn/7E/ig6da8P3ZfCHlWhTebSRrUYCE+eHx8Wiy00rbafysKJ8u8tuXtjd6ncfzvET+j+7e2mq3fvj0YLfRuN5uX0+Te8PJ+eno9uXd/8PX3w04+3S6yB8+SvNlnjYbYfB4vvizFydXO+1C6d1mM8iyX+y9RGDv9zvnWfbnL4+HpfqT/WPJgv3R7F/9xedBFP/GzvpXOw18+fzwP/77wd5nV2LVbkZ/6f23X7t5czKZZ3mZlTJJG51Od2NrhwXhIl8OpxOplSyy5XIZxvGg12EcNcE8K0ql02abB6FFnTPT6tyJF1iqBlX71423c3TAsaP5T5N9KqXVzn6sL+LjwsVdXZVDPQiD+9r/apaEK10jUpaZHeTRwdZAyVycwFbemeXv6PS7bZq9FgBW6+YAwCxPNv4+AXCzWIQxk3GVSiljJBCgJptFsLaBJtKgyGa9GCadQbcoRsPzuNWJWx3ORZnnRCgl3X3znUYj/eGH9754fjJfLAVjXKowChWRJCBkLFt2P/qhvvra6Pqbe6XaXOs90jCV6mw4ub6zs9lMZ0WxjthP4stpdH883R/PfvXapWEhrzTSPzs+H8Qhj6OhlM0o/Pzo5FYzvbOzmRBcbiTHUuk0vvz+ezHC6WKpBH+a56+V8nQyHc6z9NKV0enZv77/bDMJi15/VJb//smLv3p1a7GYH7caj6dxJ4nf3N0a51mM1B4eru0MGo1oPYq/8vbbZ+enWlEUJ8V8UZRqs7++WCzCMMnL5Xw6ZWQWIOSqyIi04EJrUlqv9bob69uMcY8LeqWi2dNqhVTnTtiojVvg4egTKgL0lmC1SLFyVKy3tMpnHrXW4fLeC5FA5ySZiCZaRV7FjKByyVfAjz5XSWSr5SurwrXbTSi0hgn5yaSJfATU+PLovHizqQgjsLuEABn+RCBuwlCMEUMCWwSq7R6inIfRzuUba+vbwBgQlHmGZu8dFhBrXL319s7ulSdPHv7gF/efHg6JSAOVpJVSoDUxZEyw5w/ak9Hk6t3DNKFlWU7nDOnF2SibzDe2N2Yo1gi+GE6vd9v3xvOzvHw+nl5Pwm6aTjQ0EQ/mixKI0uTxIlsMJ3/88Nn1tf5RUc46a41AnGbLH7w4FEm8EQanZdlJ40+OT5iOwjR9Y2Pweqf1cDT8yXiaPTscb/S/c2nn/mgssvmTZ8//jFQrTXZvvrb4eLPTFDHAW6/dnExGRVlGcTNMmlu7V4MgQsYG61ukdZ4tVWegSZ+dHIVhNBgMnj2dSSlLSbnUrUYzbTTBaS1HiBVisF5O6iHr/1S5X/LHw0pxI9aQYpx3unjNV10lspVKzgy1VxZmkZnZVttSofV2NBGzsSJvWyKYQiY7Xep+FIDzlMB13J7J7PNAEMyWoOjMl5olXrMJqgirJgIyRSGkAbjLNzKw6U4FpJQ0G0Bo0JwxHoSNIFCl1EpiEAAAaaW0lnEhyzII0zff7W3vXPr43sM//cX9WVYKzoHZjf+AQDEmhsfd+bhYH3Ru3u5t9I6VHp+NFkl4mhdX0+gP9g+CIDgATNP0pCieFvn/5+mLnAevt9KdJJqXRab18nR4ab2fav2VW5f/zcf3ck3fv3ntvbXeVhp3AvE7v/tHrV6/tTb41lpv/+x8/9GTN7a33u20fvL85aX1XqIndH58NtqEnfX1JAl2d+/tHd7qdw7m8x+fnq+laSuCdhRtrq89e/6Mi2iwllKWvZwvNrZ20iRhnCMXiFiWeSnLOE0n08lskbMglEVRSinCYPfq1bTRrFtZjlIqrBBUQEQHniqcVOGsFj4iAl+D5QHnrghYjXi9ntmPexW0XHXRBAOosoA26WPylszkOc0P2mhZ53FXTOlor7KCtc3jOyOaTOBCE3AG9ikNjkqNLLwv7xIRIJAxxjQAkS5Lu+rDGMIMGXAOQGSynVqZzLwpSWYMgyAMw6gsC600ApRlQUoiY4wzpbFUvNW/9Je+uXHtypXf+dOfPD04C8JQSqlLCWADEIKx2dFxfOX6r1zZ/pOjs5dPnl+6vPsP7t4aBLwTit/9wY+//s2v7AXil9d6XY6//Xs/uHn71q/evLSZROfLbKzkj5I444It80QE5csDJnV+5fLpMr/daYYigDC8tbN5rRn/7pP9NIoHzVQge3e992I2/539I5EtMUlUWeZET6bzF1mOefbpwcmdy9v9gxdqMh0TXt3aIeRKY7/bHQzWCHkUJ0naCIKAALLlYjI+f/r0yXA2LfPl/t7L44PDIs8E50zwt95//627byPjX+I3rGh58t84twbrP7n0v/PcV71vMFrRee7GOoSV+tHavaqleStzxrpDfomE+c8kkJSDhMsTAXM7Z6IDZtU4e4/6dYlIawI3cQAdRRH4HXZqxGlX5ZkFyWaxMoIpgzfAJLNRjrkqAuOAqLV2G+5ocy3SWkolpVREKIQCKlSptVKIUmsNCEEYJg1AXkjc2bn8N7//S2/cugKchWEYhIFdGsiYGGyEb7x5OJysJfFkNIHz07VG/OB8kkt9KQxZXo4KdX50KoT46b0nSZ595/rOw/Hsf7n3LOIs04BRPJ/M51n2yWiqW+13b9/49u7G4XT+py9PPj6bRIO1uNe7M+iCkh8+3jt/+ODl4ctPx4uvbq31QWWHR+XLl51G+nKZX+40Xy7meZErhi/Hs9cuXc2v3N4b50sl86I4GQ73j49m2ZILxjiTspiOh2cnh/svnh+eHi/L/PHjJx/df3h0epoXRVGUCuDt99771b/y6420WaGz5q1QxZAmSISkfeVXVTHngt6Ol6C2kIicQQguUF5zhWHVzLW39HrfO2fevdEE1kmy8we9c1Zz4oyuJ22YFarl9cyayWQdcFvN5EnQGwcOmQjKWZwVrP288QFWH19AxhHNZk4m1VnNVcZN36T26XhZFEXAAwh80Ax4GE3nhVaSEZVFXpRlqbUQIgwEY+kyW66tbf+176azP/iTvWeHTAiBTGU5aJLd/vuba6WCvWUxB7z19a/95ntv/mj/6CAvf+/B81/6+vvv7qwLwf7s+Pzw5OybX33/g8u7+5Ppp0enaRz/9MMvysn8O3euPpjOPn70vNVIw36/1HSp1/7dR3vroeg1k79398bz4eRXblyJovAXxYTC+POz4U/PR7fbHX0nGvW7T8az5+WLcDQuk1heuvrkxREFZ4Moij/49otO74vpYePk5PHJSXl4MCuKa5evdjq9KAgF50UhgfGk0QrTZtruPnn6ZO/Z86NFxqNo48135O23KYjspgw2W2O1qkeJ2yTuIpDQ4bHyeIynYcfx1bOoslj9BHCQAq8w60Reu4r3rEVNv3vYoAM3ObSRS1qBrUElAPug4iq+5U60H8nRpJ2E7gD9ZR4cY0wp5cwHZMg45yCYLv22ZJaABWOMM20iZHZ/KEVEoHUhS8pRA0AQBoFggne7fQIqsyyKYqVkWRQSSGlFHCLOVbZoN5pvXrt8eHSuSsU5E0JEb73f++Z3Z8ienZ0/G03V6fHVW7cSxiTn/+r+0ybAd67unGT5jw5Op5NJ7+zlla++9YuT8zu9VtpI/69/+vO21v/Hb78fCnYljf/li8PtOPpLV3YDoJv9zheT+Z+dnK8R/+nJ6IeP945Ozr/91u32G+88f/T09NHTq/1eS2nJeLC+dnw8/MrG2no3/eHpmItoXGSUZTwKW7NpHqSfdbaXhyezklGmP/zi0eP9l+04EVHEBUdNQoggCKRSs+l8OZ/P5gsVis6tOweNtecvT+9ubtzptokqOxIvQAds0bNTxwR2IaTVZl6Tm/p861v4wn9vC1TDXvtYMyQu+PJOnVb+u6Er4VyiFXSu4tU8/tXu+Wkeu+J8M9srH0+rB9LQl8LXwhb2mNXb1Btt2Q/ttiO+np501VOzSZgmUqQ1kSJSWmktzaxSZZkTLAo5lZRGQRqIRpxEYSiVIq2zMl/mWV4Wi2w5G48GrealS7vRR18sdC6JSsQrN26/s7a2HgW/V6ifffrZ37515Vt3rs/y4loz+eG98d2ttTQMbkbBt9Z7P1jMkktXbm5v/+zobA30tVbj58PJBsCTefafHjzd6nfY1StPXhz/84/vh+2mPDvvdzu3+90XPNBad5upSkKlJGXZdn/wl1+7EnP+dq/5//rki48++jxsd7qN9O1ea6PX+6Nf/OLlF/f0u+9/9fqVGxH7BZMPT+U9GRXNHb7VvDzo6pP9h88e6dmcsmXQ7QftTsCQFblOGjpIZKK6V2+xze3ifHI0nf+Hx88Hb9weRKHN3VUJnosq2EdtnBFKDn3eUvS1oeji7WiTiXXMEUF9/Z0lvi+p5/8S3iISjrpNjGllJXj9Kmj2fbJGJHFCRNRoV4m4xaquYS7HQKuzBJz6tu/rFnFVr4cm1RkwxhkDxkyg3tZ6AjBAYynqUmqpSVkzlDQpIk5EQEqrZ4viRU4fjo6+1e8ca70WikEkEo4NHgQRB025VjNZtIke7b/Ms0Igiig+275+XyT3Hr+4FIpmp1X0Nvfj1h8enH3+4vDNy1vrN6/96MXx/XvP9Hj62nqvc/Pm3icP/vjpizIQv/34RdBItzrNNrDradR/6+aPXh6dHw05gztrnTc2+3+ky0ir6dHJV9fX/vLO2t1OQwL93//t758+2fvub/x6X4jf+/mn7//lb8Jsyc5O4/7g4XD06U9+/o9/4/stpSGKQtAfH5+Fm4PF6dns8/saUW/vKKX3gkZvsD3OKNrlMeK4VFkpRZr0moku5HyZdS9vnyslCorb7XLvxdPp4hcn59+9tMVtTBG8H25QZiOXThGjz3xXbpLRcuTYrgYRYyNcMOEMImrq/kvReeHl4gAgvCdk5kTd1PCAM61htr3Vii1m9+02WsEbl4gAJururQUfob3QCB9MZYhS+4fQGLGgN8G1RSFppXnAiXHidstvk+w0u0WadZ4AUEq1LPWS2EjqAtnPRrMNwX56OryWxO+t9Q6L8koieo2dU2qPqMiam+9/99ep29dJ40qURCI4ni3+4vlBvMh22o12EAgOT8+Gs9PzvNv9+2/d7At2rxUfzxeT4fh779zeTqK73VbIdh/PF//0j3/y1TduMYBbzcZrb9w6/K3fPz487dy8cqvV3H3j1j/9wQ+fffT5L/+jv3+yyH/nsyd/9yuv77aT4WL81Uubd/utjzb7f3R0PnyxH4fRP/zG+yWp//Hx4//13uOT8yEfrH/3+uX/9PnDPwQaPHkM52fq/Q+SjTX56Gk/ChcZUiFnaTvrdZL5rDw4WQQJRK0tmg9Ph0W3y0s1PDjevHa50+0eHpz8OE06YfD+xqAGn5ptZkfcfqtJGVPO6l+LHLt7j1PmGvz41gKcFWdhnUJrnEU2RLpySnUWgHsON/qZQC6uUKc6e1Gbf3W7c9U4b+VQcCit2b/abeC7Yu448mbVe+IMGWMcmbYhU1TalYuA2amXkDFiTCOQtIs7takc1ZqASGsk6gbIQA9CETJ8Iw2/PuhsCEFF8e+Oz3/rowfPxvlBRvtK/D8fj0927m5+8K0fp4PPWPy9zfVf3Rj8tWs7t0JBp8O/fm3ne7sbf2Nn/VvXdl6enuHZ2UYgbrebVxrptSSmh4/6ef52v/One0eHk/lbvU633fz86GxG9O8e7k2z4u7Na6zZ3Gk1vhhOf3hwdnNnKx6fqPm8FQVxt/l/+fnnj18eX711c1jKw1l2Y2vj9x48Pn3wRSvkDycLhvyDOzc//dnPijDavX6tH0bdRhw8ubd8+UL1ByyMmUaxs71/eJyfDyGKIc+ns6wcrHciwc+HmaIiiUPBTk7PdbsVhsHJ+ShupE2G2WL5o+cvJ3nhuIB5tDhWQmujGV+35lEAEIB23jmRSZsjOq7FOor8WPuPGsgZDB4EUB3nj6zhiLkIwCocaxetx49s8BI8z5nWo9bVkmp65ZZQY3W7i1BNibglm7QaRmCCm+0btNa6MFtCutYgIGPM6HRt/CSllCliMUvsEQTovuAfdNtzRU3OTws5HU42onAjivTp8OnB8Rvdxgfr3WA8+fMPP8ey+DubvQ86zX/6F18cLbPrafLffuu9S5e2/ulPP/39Fyf3zydf3dncefv1MghGhQwYy5ReRDEUy2eHxzd67buDzr85PD3Pyw+uX7qxvX6rlV4dtH9wcJakabvdRsCvbHTbcfCLpeSt5uPHTxDZB+u9FodZEG698VYnEA9H0504utrr5m99Zbl1WYH+83uPu2F89a03eLMzHo1+609/ErQ7jIvpxra6dpNzoYgKWYrTo+z0PBusi80NkGq+zKnTjjiGAStns7DfbbQaAaLotAVgpmHz0hZjLOS81PoCHlbYyBR5uPo7qP1nQeyeweUvYQOT3uV3cNK1KgtvIzjPujIQ6hjDGvz8PgYrYVj7lVuupMEsbwezkzdzS+bdXKP62lhyzYSay1WDuw2VoSvy8wUo5ktmQ7LAGeOME2Kp/SI552khklnbSaSUIqXIxURNcNRs6LyUshkEy7Jci4JRIZ8uSwqj97vN6NqlPaV+cjJaD8NvvXY1L+XhPHuz29puJE9OR/uzzOzs/+2dDRTiu9uDRIhJXkrGkzReiwJFdK2RnM/menQ2fnz/3z15OVb6H9zYPc/ynz0/EFm2P8/u9DrdQPzw6Yu3ttcJ4MFoPpplc2D0ze99zJOPT4f3J/MsCPibb/z4fPI//OLeh8PpvclsbzjS62tTJf/oT3706cPHv/vDn015fE44kTrttN5O02+8/0H/8jU4OysOXs4fPFbjGUax7K2p6aIoqNWIwpPjJbH48m5jNhnPFhPk0Ou24jAWotVrM6XGy4IYC8PggsazKMMKhmbuu81T/ZA6Jl3Vy+Doxptz/lcXQAfvalV4ALdxQQ0bmgg1OSIkYU09gytnEtdhbWKV9V1KqFYsooEYcG9rugZVvFiDOxprw+zuQGYD35oZ6tHsthdlHJHbBeCklXI7PwMCQ8Y0gEkjKa0lUWgr87RSSmp9nqmXpWA8OMrKlLMn84w3kmYUbiXxRr9zI4mutxp708Wnhf7gK2++M+g8HM22mumVmL34/PPi8ub+bLkWBn/tzpUfHp3/+uWtJC/WABln24342Xj2jd11CSR/5ftqOFwq1Q+DTiAixGBjcF/Rdl7uTedzgGBtcIrQEOxfPno+kXK5zMIo/vruhiL66qCzPzx/+vBJp9//W3evbSfxOM+CNEpGIxWKxt3Xfv3aztPheKrViIoG8J3r14so3Iyjm7duHokAGbvUbEqVv5zO2OmIQGLajHudxXRWnJ6VRyetyztJuz1bLEe5ZL2WOB8WgGvNdO/F4fpr1zXRIi/7UbjKSMYvd0Co+ezmLQJUxW0+mujRWSMjuPhaiZK6uVABwwcuAXws3T70lXlG9VPBX9/qc7soA2yWvE6HtVv6r7WbRjWvfsUA0P6JzbXr1IIJAGbzXUQT6zSekKkrZEDc1tvbpW9aa2kwqZRWsijL+WIxXMwl0W4ScaBBHHHGp1pfX+vtpDEHWA8DTrop+EkhoSx/fbMfMDzI8i/Gs4OiPC3yAPF6u1EofavT/tpG75Pz8c128x/eusw2N39v7/jH55P/8ZNHEeNfv3YVrl7/0dl4f5H9Pz57/J8OzkINnTQ+zPI7vXagFQ0n393dvNJM3lzr/sqV7a2ylM/3N8KwLGWWFXe7bXj4sByNOeK/f7T3clnkivD8nB8c9ZrpRhB8Z3tjlGf8yePZeDSaL/74/sNPDk9CxtT5SC6yNhNfu3ypV+QciXZ3eKez1u50Ww3otIMb13ivv7G5HirZioM4DHcGPcqzRVk2Qw5Ex9PFy9lcO78CaqNvi9PIu0CWusyxhggv2G810iX/zYUIY3V87V42OGWVradxz9JmxD2KvYdUXQ6rq6FJ5+g60SOAezpRlYKqgkc1WvXtcwFfwkoidur54xCRM/MIGrMdmV0IL5Wy9a8EnDEE0EorqVSpZFkWZbnMs8lkfDI8O11kPIi7YVCSjhljyP7qRv+vb6/1w0AA/M2dtW+v9wFgLQq+sdE/LMphXt7ttbZD8UvvvrW/feXfPNr7ydnkrChOi/JsWfzgbPI/PdzvpBGQ/snJsBEGC8Y/PDpLSV8PxC+vd9/tt7+10f+DDz8Tx6e/sbv+w8d7y/Hk1y9vlk+e5tP5J2fTm83GdhB8cOvqZDwr8+J2p/Hw+GwqFaSN33jn7tfWe4M0+rdfPM7ORpmCb9y5+c217j//6N5plpcoFBfrg36nmawF7On58NHey7VGozsanp0cP11k0dkZFAUuFtn5+dl4mknFsiwHGGfZQaaS9XVotobz7GWYXrt9c8nErZvX1pJwt5X0IqPlSVtr0ngw2i7zBk2g3ZJij0BFpDwF+kH1IKu7H1jzklcIjZzbpZ3i95lCR0/kaqAIiDlr44KfZC5H1RsCZvi0Os6jHLyn5VEI3qZkrB5YRbtDpMcw1qEJlW1qK+cRmV1YrEkTSU3SuvPMhKC01lLJLM/PxqMnBy8/fnjv40cPh7liIiyJSsJc67lSD2aLmZS/fTw8XGQbcTSS+ufDWQDw7qDz09Px77w8+1dPD59N51/ttb/a75xr/ZPhBJXeisO3+u3//vbu+73GP/tPf/FeFPyf3n3tb1za+Md3rkxL+X/7F/9OHx0nnO8NJzf7HQj47d31QRzuNqL/90f3//hwOMmzZVG8Nmg/HM8ejmdrnXbUTg9m8xzZMon+/cf3UZaPTs6fjmfrrWY3CbOz4+DJoyxtLAjTfuefffYw23/eyhYTLl7MlrLdUwHb+/iTrFSD996TcfRk78WSC9XvQ6PByuzs2f6Mc0piNpnAoyezrJCjcf70ebn/cjhdCCFUUUitx6UMhIgCAWgtf7LLZv36xYor/PqJilSdF1FB1J3wqsNxwQdy7pV9TIyJZnmqskhyH5AINAnXjlVnitzVLKCdIvbGgHHKHPfVjNLKgibfW5dltwu0XKmrPxgdrGsbhJvtyBEAzENqtNZIxICQiAEA4xRw0lpqtciWo/nyfLY4H03yxTJoNvnmtZ5UgWDnUv3obPi3Lm19uH/8Vzb7gygYST1WWnC2UOogL+fT+W9e2mgG4miZ/dbDvXxd/t2rW7nWv7139K9/+tEO0d/5/rc3InG33/nnjx4zJS/1Or/38PnfuH3laqcRhCxot17rtvZOFSP65huvncwXB5P5u1d294Lwn/34I0a0zPPtNBZB8NPTEWolFvP3d9Y/Ozj5leu7x+fDnxzsiyh6ucxn09n17fXzkyM1WNdpqvLia4PuT4r8cHJWdgd6mV1VmifBpw9eBpyJZmPv+Ly9vk5PH8oXL4CFEKei39WnZ3g+Ut1OO42LvGDZfCFl3EzLZSbiYDmdtgIhlWbEpFKzotRE3EACcXVraXRekUafcXLf+iG/wCweQnVN6N9UC57qUSF7G0ddTrVaBBIQWIBaiHl6rkGzNkuICIhh5bG7s8CWz9VMZnKumdtEh5zpXQEYa1tMoTufyO6IgnaFMJpdwH1ZiKFWAFSKFkWRLbNlKeezxXQ6L/OCNDUQR+Ph2WLRSNIbSbjfbIzzfBAFz2aLv3Jl63/67Mnv/+hn//vvffN6K/39e0+ePX72j3/tO39yePrtrcFoNP3dp8+3W43rrTQGKtJkPjwLArE/XbQbydvvvrWVxHf77YPR5D/uHyHnbDF7rZUMorC52f/5yXC8zJ49ePy13c020D+6tPGHQL9IgkfjxZ/uHd1KwmSr/zuP9qjd+3g4+9EvPr3/bG+xu8Nff+OzJ88PlTx+/mLz6mWxfamUcO/ho881KME7a2tFo6tPR/zzTx/tPWev3YZGozgbrbUa5Wx2Pp1yTXpjhzY2BGJZFuF0BFKFyKaAvWa82Ntb9vsyinvdjp6Nz4m2+p3FbNHvtYHg5XRxs9dhjPmxqVwiAL+U0iBgBbl1LluFKdbR45BX94+8ljdHO4YCJG2fJu/oi9zNhL9abSG8jxSAe76XW+RfD7JahW51/mqzrVaoJ2a9Y+jSAMb2dqf4qJPT3eDMU4EImqSUeVkIjkzKpSzOx+OD4Xg8nJgqJSklaWKADEEvs/zk8EW73+hvXum0302jaam6SfTZePb+dP7Weu+jZ/ufn493uu3rW4N7z/eeHp3s9rpfnA5Zv5PleRoGL+fLWSEL4NMo7SNdWuv90cHpkdTH9x78xuvX39teezlfPp0u7rz37pKxo0UWM3yr1wKiA33zt1+c8Cx/vxn/6rVL7W73Z/tHyeHZt7cG7/Y7R8vsoNduBuK/+85X//jgJBHs8qBbIF3rtr9zZefT8eTFcDzY2exzHCKPkuR6HAaCnutd7LYkIZUSAoECjj75JGykFCdquYT5XNy/B2EMl7YDpeXOjiAqigKUZlqzIOw0kjTL5kW5jONCiLVOGIaCEANe3wUJHJV4Zeuw5FVijfLqoKzDl8APtz/Gesyv8i6aZ2lbyJstUQ06tYsYESIKchvUuFbaIuXaRKkZumAJz6wYJjCbjqH18WstqJgXAc2CDSRA9E924ASIdkMHIw8fOK7NV2SMCeSAUBQFap0tF4UsT0bnJ6fH0/FUliUQMG4fOmsWN4HWw71nZVHwN74yThJkPOH4TqvxdJn/4dn4q93WndvXP53Ov7I5eLoUjStXPjse/p/fuPlsuvjTk/FiMn90Nvz69nojjjd77Ww+eTmZXY3CX7u0MSzLB430//v08O1e82ojaQveiG79wcnwlqT3uq1L7WYzyvqM3Uzi33zrllTlH3/+8MXp+dvbGxvN5NHZ6Z9//vlH9+5f3hzI61f+3cHJ7OQ4DgTb3u13msvx+Wd7T54/fNhKkmgwkGkSZhl9cVT2252y2BRx0oqXo5E6HeciDDYGRRgiF4HgshHxQBAXGkAUWUFQHJ9AGLSaDX52StMJIuYcIlmWp6fTZnuWxkkoGpwFgsdCcES0WUpPJaveTy2O7YbEbe+4aoZqupjHNuPnnSTnU1viM/FMUw9l4oh2h1BA//gd24R/9lv/wSwPAkeKZqGFybdWUX1to6jmxuaxG0ZBMMadUnahAADSimwUyHl5aLfXNUYCQ0TG7N7J5gwCG9Qsy0W2HE8n5+Px+XQ4GU9kWTICJcs8LybTyWQ6zpYZAgRC8EBwzrTSSmsETJIYGUPOB5ubV+++HW1dL0SERP0oWgBqgLIoIiH2lvl6FGrGJlJdMo9u4Ow8K07GU57ElxrJIAoXpbw3W76cZ3c7zff6raXSPzuffDGe//1r22tp9Mnp+ZPh5FK/myAKWfz48wd/sf/itUGn3Ujm58Ozg/3xwUss8ka7qRtpORrHHJMyu7Q5SDvNaVFmo8nLg+NZpuL1tZjxG1e2H9z7gqTSWqfNxmKxWB8MeuuD8fERaRqNZ9liITgvEHnS4GFIUi9bHb51ydS/8bgxyTIkAMExCBLBaTZfMCaSeBBHyzxXgI1QpEkYiGA7Dtth8Ob64Eavw6CSf50eyVdlQAWyeuiwBuNqcQRetBUvHukvaBCozVIgswlZdbvaM8SIRD2F6FW4Jm3dbfsYdq38VvBuXtnZhUCgmd0s20ZMNWkCrbQmIjSPmHHARQBVm2GGeJXSRk3LUipNUpbz5XIxm56en52OzmfjUZ7lpSyLIi+yXJYlaQ2c8UCIQACilFJpigKRNpIgjgCx0+5sXrume1tLHp3m+XC57AueBmEYJS3BR1KlgqdhECBsRkE3jv7j4Vkm5ZvN9N2d9eeL/Omi6IZBrmQC+o1+eyMKfnY+3B+OSqW/2u/eOzn6n/f2954/S89PppvrOk17XDenk6/K6fTzz5+PJnme72yubbZ4JJqv3bp67/GzIgXB+Vff+WpZSk302cNHe/svl1mezbOgEe5ub1watIOrW6dn40GvNRpP2kn/+u2rf/6DH22u9+OQt9bbx6dqluV6mXOSLYgKQjocZk/uMQRgjHe6zWxJpUTGIQyJCyjydrMVrK2HYSDPz4QGHcWzKOJpQ29uta/spmGAXnnBRQiiz6pXg7USbyGXOvIj61m2ArQ/y0HTRF5tZZ790u0pV8UQVl5CSmlvbc+ynoqpoWNIiKjcE2CV1syt4eREyu4Uh9p562QedAB+vZBSSkqzua/WpZJKSgD7pFqyTwWlvCzHs+loOs2zLC9LJaUs5Gw+G03GZbZUZSlLaTcE1YQAnDMRCMa5JI0auGBpGvabKQ8FRvHm2nrjymt78fpPDs4wf9nncEmNh6OzrNnLm102HZdRkwAeI1zdujRTellmAOzW5tY0n//kYP9yq/F+v/v49OiTg0NdZO/221+MRgfTKVvOscgeB2EjYleXc3r+LEK8EzRPz44H7dbbb91+vLf34ejs8o3do9Pz/mBAUr37+vWP7j86OD67fuXy6zevPN9/eT6f7e7uChEwzsusiMPg6s5Gq9GMEAJkoRBa6vffeW9UFE/v3Rv02u1mmhfl1UtbGpDOhtcubWV5cXw+4owrpbjAbjNdZDnMRpyxrMg5w3wx4QCMcyoWej5UaRKSzhfLQlOcNhZhYzIcz5b5WVZ8bXv9jV6bu/gLw2oLLefgVkuIVuJPZC3EFUh5C7YCcVX4bBKe5kTz/ENy+ScX90RnG7odmEkTkVBaItgHWDGG2vvfWnNEs7rKPT1bgdmdnuyGvyanaUpFAQg0mOL2UpayLAslVVlmebbMllmeZXmelfksW0opEUiTlmVZSCmLcr5YTheL5WyuitJuCWqeO0vE3Tp50sC0RgTGUAjOuV1HH8VxpxF306TdaoIQFARs4/LPoXPvi/tXhnuXIiXzJSjZYkzIZXt5Op0uWFEGAZMaFiePG1HcScJ2q4MHI5VlrxeZOi7uPdBRlO6enRXnpyeCzUaTq4MecX4+HCPATKrRaLLe771x5/Ynn95b3+xvb27MlpmSspHGh2dD0nBtd/unn37623/4n5SWTHApi2VRPHr+8uXhyecPnvV6nWKRkVJhGm31exu9XgDUabeOTs+nWXgyWzz47LN2I+m125e3Nw9Pzg5Phy9eHAVxyJCtd9vHJ2el0o1mYz6exEkcp/FwNI0QW712WUpcZKVWQnCOoMsim+luI02bDS0VIgSQqXJS7D39Qspn08VfvXn562tdtgowS4dWQ37JEmRwURrHrUj2gdeV1epVpFf97mrWojDhKyRbSufKUip/yDNoCQACEEWgNBBo/yBRbeo2NCmlbHDVTh2zolKhL00iAAClZVnKoizmy/lsNpvNZ6Usy6KYLmfTxVIWWZnn88VimWdaE2gt8zKXUpZKuwfOIENun2pjFijblDworQk0IhcoOAejLBiLwqCRhJ00bjcbcRxjHMet7iLtHcyzG7S4kuhyPGZaFYTEWTfBSOswwDBOwig4G8+CfK5UTjLMF7OlpkajIUqZRmK+mE6OD3Y2NkaylU2nd65dns3nnWa6v/dyssw5MMZYp9v5gz/5s2aa/Nrr3yLS//K3/8Novmg10q2NdRHHAYNOKI6ns29+8FZAsNDqeDjKtC7yXM7noGmt32k1kkYjPj4bCeQ6mz1/efTe++932q1///v/sdtMMylzonkuMQyhKN9+87W8KOaT6cvxZDbPOt1Wr5UEoJeLZRSF/Vaz12k1kvj0fBgGQVEWy0IWUqVRAKRJSS1BE3CBXKqCyTjSfDaccv4XrcblNN5JogpQKxissaFLRroPFX3W9h686MmjLdp3gDZWBVK1nY2PbAISahO5r7xkxoSxQIgbl98WUxLZBymCW1ZhlvdWZgq5DTdriUopZZZnk+n4+Pzk7PxkPh7LItdS5UW+yPIAmdRqkRfLotSlVFIVUhERN1vbMcY5E8yn7BEQTWYTNSEQR+QcQs5LAKlUzMNmFMVR0IzjOIyAYCFlg4s0TuaM3e6kR8t+fjoJgiDP1SCNoyDoJMlkOonDcGtj/fh8FAouhBBax4jLPG9FkSiyzfUNEYTzZSnl7MEXD7bWN7a2t8psqaRaLLO0kbxz906/1/3wi3s/+NHPVFn+5vf/0nA4BMYODo46ve5av9PpdS9vbGitHj58HoRBwNjZ+agoihfHZwcvj5SU/UH3vbfu7G5tkFT7B0dT4vcePSkWs1/79V9f73V/8eM/DzhLk+Qsy8Mw+OLJ8wihncbns/k8y5IwjMJwrdduN9Px+QgB0zgSnIdROJnOkGG71Yzj+PR8hJDzKJRKkgYlVZrGhdRmG2ulyuloWC6L+WjO4ni6u0FpjK5g14MTfbWaNzXrxOjIk9w+oy40qV2U08cjrdfjntMBmuwGCcavB7cxGWoAl3x3XpsWBv5ak0b7hHcwViSCBs0INZFyRUNKSSK7gI4h5lyAtSeoLEtZlnmZn4+Hh4cvp2cni+lMlbKUKtOqVIoRlER5UUql0S6yJ0JQBESaA9MAGkAAlQSAIAAER47IGBM+ikEEQAEP4igIo6ARRUkS8VBoLngYxs3mUbr2uxOVsPmNiG80wvOJ7idpu5EA6aPTMyC9PuiGcdxpNvIiX+/3EPH5/kvGeWdjrSjKo7OzxTIDxP76xrVr11SeSa02L106Ozl+svditlhub25c3tl+ur8PSEqqLx4921objIcjALx+4+pX3rjz4aefvQTYXuvPpjMeBVmWX7+0+/OPP3vx4sDE7zhjr1290u92P/nii9licZbJl0+efft73+l3upgvRcB7G5vTvJiNJ/PpvNNIllqNxpNlXiDptX53o9+5vrsxns6KPF9mRZbl/W47y/OAB6BBar27tbG1Nnj0fP/g+KwZx6WS86IslGokcRSKvJAh6AhUPhljL5Bau0eZO2Ss+jpQ24XG56Vx9RFt9lcHYiQDwirxCWjMTASt/fJfN/7WAYIq1O7PQgAU1jTVWqPSQFJKX3nKAEgIEzMqykIqleV5WRaFLMsicwuFCBBLpbI8K6XU2aLIFrPJZD6eyaKUpEutpNK51mUptdJSSjCLQ2zk0hUWasUYAHGJKIkYAucsCYTg3CyOY+bZx0ARMhGIOAyiKGo1m2kc5UBBGK5tbNDgyk9l+mL/SWM5/d7lpIk6C4JIcK1pMp1NZ4s3Xruxu7P9/OXx8xcvACgSnAiajXR90Ot2Wo+e7J0NR8dnw1YjVYCdXu/uzduz+fzTz+9P8/LF8XC93WilycuDw0iIACAnGk6ng7XBeDpdFsV0PN0/OrmyvfXhvUcPn+31d7azxWyyWLx1++a1q5eeHBzNJtO007i0u9VI0zCKAs7zZbb3+Pna7s5wOnv++OEH77/7jW99689/+vM//ejTfrcZtVMgkpJarcbV3S3OUANs9jvtdmuZ5VKqUspGmkZRIAIRx7FWMi9VnhetZrqzPsgLeXQ+jAMRcM4Qi1Ii55yxXEoFiKBQq3KxfDlbXmulfFWlYw2XUENh/ScPJheJtD/bA43z4+qaq+oN0ohu32W3sSi5oqpVU5eAQJgrS01EpEgXZVmWpVKyLAsOKIKAcU5EWsk8yxZ5Ns8Xy/l0OhmPZvM8y4hIA+ZFCVpHDJhSAoAxLMqyVNKkJ0mTlrKUCjUhgEbQRNzlOYmII0NSqJEYA4KQs4CzNAjSKNKkF3kBpEUgEJgG4pyHYRAGIk2TTrMxnc8ni2UUha20wQc0EOLNG5dffvHJ2cFep5220uT47DyNwlazubU+2FjfLDVurg0Q8Xh43u/1sqKIw4Ahe/7iKI7CVqs9ycvj8Uwg3Ly00wjZoLt9Np09+vQLu202UiONbl/d3b919Ucf3d/d2uAAHDGJgs8/f7C+sUZRMFvmB0fH77z/Xgh6d9A+GY4eHRyR1qR0Mwg/uPva9s7u6dEhZ7hYZFEUh4y9fPLkm2+/OZ8vijIfnp6urfeOjs/W1npr3W6v01GkdjbWpVKCiwBJK73W743Gk8PTYasZa61ms2WeFXmeh1H08Nl+lhWB4K1GMo/DeZZLwEgIpZQmSqNIKlUQKQVMSV2Wv/9k/7VuczuOtOdI7Vc9OgjWV1vYx6rYWOmq8iewD4Eh/xBEq/cqe7By7cHZojXnDNyzu215gEBkZJ65XqpSyflivsiW2XIxmc1Aa855EAaIyBFBlfNsOZmMFpPpbDafLpZZlhdKm4amnLMwCDjPQZOmoiikCTMBSEVmgy6bbQVAjvYhBwTIMGBccKbRBF5ZFIoQkTMslMrKclkUseABF0EQEJIGUAg5QBPZYpnNF0upVIJMypKVy+3GIKdg0Yjzs+zxZHQ8GseBaDXSjbW1Qb+viRaLRb5c7u3v9zutMs8E0Gg4C8OIAeRZnmfLVIg8CsIoWd/a1oTHJ8fZbNzttUfDYVmUYRA0m+12q722tsbZg/l8gchK0pzzLJs/Pzi+vLuZpM2rV5KtVnN9vT88OX7w4vH58Wk2XQjO37x5dWttsJhOy7IMgiAMuS7L4XB4+85NhtTrdD7++NPZfJmEwZ0bV9+8+1qRl81m4/TsfDJfXN+91EyTxWJeFFlR5M00aaSZVroEtczLl8fn82VmiQxRcB7HYTOJI8aXpczLJSKmOpJSIWeNOC2UEkmiFJ2PJqO83E1iiw/wGtmqcxPkAXBL6WoZbKOj3bN8zOoL7crsCBHJFSNTVaVELvtvyXole2V0qtulg4iEJm1L0ZUsyqIoisV8djY6Pz87z4tMgw4QQsYF50A6z/LFbJ5n+VLKZVGWUhFAwDAWQgDIUpZKlVqTUlIprZQiUkpLbbOnAMamRPNoWq21WQAPnCvGjD2Sci4QNUBWllJlpdICkTNknAWcl/bxSCiISMplKYGglSSDwaC9uf2UtX/3fCqnw/fzUTPkJ8P5bJFRGj05OBr0unEULbLFaHTOGV7d3Tg+Oc+y5dbG4Ma1q51uf75YTqbTHaL5fPH4+Yvz+WKZl5sbW8D5YG3t4d5LIhJCcM7DKFkqdWVnfa3Tevni4Hw8vHH1+htvj1/u7R/s7W9vbrz/ztvNZnN0crqzsdOJoiBgj56/DJP47ddvfv1rHwRxslzOwygw9f8sEMBwdnraa7WIi/7G2ht377QajVa3vZzOF/n40uXLN65fX2ZZyHkjbTSW8+l8JpV6tn80WWRFUc4Wy7KUSmkbrAEATVLpuVJBEAjOdZFrgABgmWcMIEzifDbXPEz7g3kUJXGUCqFddKcK3tvMYG2jjVoyp+7gk4vuAwJoDW6xsg2tG9iRKbcgALelPKBGV0f8SsDfx/YFIiolS1nKspjN54v5fD6bz8aT6WyilkuulSKSnCFjUsM8z8s8L5WWSmmtOQEiJIxzBKUVIJIGqaSSqtSkpDYBeUQMze4TyMCWh4LZlNY8m9Nk581i0dI9d8asfwqY27fJJHwJEICTJg3zPO+kjUaaNBuNKAohavTaXRyfyNMTORo9PjvMizxIYgVwY3vr1vUbSdJQWidxEgVikWXzLNte722ubwzWt4KoofT5YpGJMCo03r3z2vOXByfHR8046ne7rTRpthpnp8ODo9PpbNbvbybNzqA/uHNj96cf3X/+4uDWtfh73/zmk63HP/rFJ6DZ22+81Ww05WsyYKzsd5+9fLm+tRlE43feegOIsmXGGD84OjobjstS5vMFIF55685ar3d8cDAdnl3b2TGLsLWSN65cKvP86PhwMp4ss6JQdH5+Jjju9FsMdTsJIQnXuq2iKKeLbJ7lpVJKKkQw0z0rinaa9JtNwwpFKRd5vsxLGXDNGJcq7SffvX5pJw61UlWu6BXrE2pOknGRwBKp3/5I2yXJaBeyuWwmeZiaOJTP2ZPWzDpolUWLtQ3hDSsLrXUp5SJbzuazk/Oz8Xi0mM0Ws2mxmDOlCFFqPc9yM8OyvFBSEgBjTJitHAAJoJCq1PZBpdzEoZQtkUPGwkAEAdeMmXVt0ixf18SQcbMPqNbkHjuqiEqAkEHAGRMCAZAxpfSyKAutBeMEUBApUjFApOSg2Y2TJFeywdi4KEUcJoKdHp8kJKMoTuJod9C5feUyY/zw5Gh4fjYaTUqlwoALzqezpdLAg0QE0WBtu9vbmC/mWSkDzoXgw+FwNh1vbW5eu3R5b//Fw3uPs7KczTNCAk2Dta2rly//7OMH9548H4+n3//er6RhuLm5vrE50LJc6w8QkQueL9ud9kCww7/yy7+UBmI8HDa7nZCJsizmi2VeSgDY3lq/de2K0qqZJts77+ZF/mJ/LwyDQb+7zPL9/WehwHYj2dvfe/D8aDpfvP3atcvb671WNF8sc6nDMN7Z3l4sl5/df/T4+UtZlou8kEov80IWUsUUhKKUCgA0gCJQIqZGlxifa4o1dQLBkCqv24eWHFJ9vMnkkmw9vFXYJvLoiNvbl+DQqXX9OUYm6e4rSyrCdHUkzmytdvEUWqqiyCfT6dHZ6cnJ8Xg8KpZzyjKlFUPkyEops7LUAKXSpEkAcM5CzgCYVkoTFFLaPecEF4HgjEEhZV6WUgJpJMoBCCEOOTGmSlkqBYicc84YMvPgTRfGReSBMOWljLGAcc6ZMVOkVKVWjHFimGsCwEaatJqNfrcbxkkhJWTTgtJZrqVI+OBSSsvs5GWe5be2N+IoODw+/OiL+8/3D5ZlIaVuN9JGFCRx3Nw/EFEjjeOsKPO8BFCNMHjx8mCjP3jt2nUiPZ/P0yT94O23tJI/+/BTU8NbFosyX671un/7r/7Kh/cefvT5/WtXLokovnvrxkZ/kERhnMSMMc54HEVf/8Y3ci1J6vagNZJqeD48G40Xi3m3074bh5e3t25ev8IZ29/fD6KoVPKLR0/2X7ycL5ff/cZXEHUzDZMk3hh0XhwczMaTXr/7/hu3Nza34jjpKsVFoJQSjMWd5nq/vVhmjSQ6PR8fnY1Kpcu8GI4mCICCAwAyBkzwuKFEQI32RERc6ZYQdaa0oXVjHDrutBCygNKueNRp+1rRKLkaJVDOeaqVnoB90AC5bJF2BRmgNdmqKp9lNTboZDo6m4xfHh+fnp7MxqMiy3RRgJKkiTiXpAuptCJNGjVxhmZHGmOjmOdoEwBnwIUIozDkopBykeVFURrTGoGAtJQyB+DIlCaOLImiKBSylKXJjUoNSMQ4cIYIkWCMcxGIThIrgKJUpZQaKEIRBEGBwJQmZI0gEIhSK6bkMlu2kvhrnfjjvLgn4vHNN1U5vy2gODncOx91uq295/tP9w6yUs3meV4UZ+djxlgrjbVWy9lw0Gkdnk0Vss21te2trTt3bkkpJ7NlXmgmlFSy2+197b13yix7vr937crlIAyJdCONte589xtf/Y9/+kPB2WI66Xa7jSju9QacQZJEnHEA2Nzcvnvz5tOnjz+7/zCMo/lkqkm1GunN6zeDIFhmy70XL37+0Sc33rj7+qXLy+n4z3/+F8+PhhGot1/f3d3cSONeGEbtduvsfKy1/qX333znrbd5EDMRj0dDAAaolC6Pjs8+ufe00UhbrdZskQvOG3G40LooSs0QlQICBIXtpg4izYVqd7e31v/K9Z231zrGdkVbiWkxQ3YLGQI0z7T0vrvFrtP3jget2kfn/dfj/LVNGWv2g0/vO+MWwFoI1j8BBPHR558OJ5PpZCzzpZaSa+IIKAJAVECylIjEOSJxwYg545e5QDtjLA4Dbda1KT0vl4tllmW5if8zzgQyY5dIQM4w4oJxnkZhIESBqPNcmVAYY1xwwZmZe4ngjThinC/zfF4UUlHAMYmjNIkyqRZSKg15URZFOZ3NE0IexXGQrCXRb261QITPs3w8kk9Ys8NCHE32T4Y8ihfLfDqbc87jMOitdTuN5LVru2+8dqvf64og6Owd/JPf+r0g3rt77fIH7761tj746Onhzz96EPHiv/lb39cgGu3OX/r2t2bTyWw2XdvYjuJYnqs8z8Io+d7X30ta7aKQO5tbrV4/SVPOuSu+hm63u7W9OxufHR8dTGazZhJvrW2sr69nWfbpJ5/de/pccNgYdGfD04NnIo6Cv/rLX//F/ecaYHNtjTOBjDUb8f0Hj+492v+V73zre9/9Xqe3rrUu8mI4HMZRMFsUw/PR/UfPJ7P52qArlUyTqNVI0zgadFpKU1GWi6KYzpeax7y7RlECLJDA3t0Y/PLOhoGYtRstKtHuZmM4VWuwTwM0Vpgr/TXVbX7rMWdZevhWVigg1OMADt0O0DaGiqu75Zvovbj/+KHKc6F1yJgABEQym4JzxqQSnAkec7M1KEFeFHlRaOeREQITQSAEcpaXZZ4XeZFneVFK++wmJAIOXJg9VpkQvBFHURAqrRExiaKAs0VR5lJKAhOnIgTBWCMIQiFMrACIkKEQXAiuCMwDaqSSGjRyliRxHMfLPBueH7W77dvt7W/IJo7pxTI6ba2L6ai1PMry7Gw0KxR1uu1eM9FSfvP9N69dudzrDcIoVqoMg/DWrVvXdj/++N6j/SQKv7j3lfDNm1e2fvzhgzQQg8GAsRgAdacnZfHk8cMvfvzzr7z3/tvvfWs8OhsPT7O80MA2tna7vXXGeBTHrljWJpL7a+tJowUMtVJbmxvr65vno/Mf/vwXn3z4edzpNNstfXjSCNgkZHmccM7eu32Jc2SIUhX5ovz4sy9++tHDv/Ir3/2173+fMT6bzQBIiCBJG0cnJ7LIHu+9fHk6FEEwXxZxFANgEkdm6/RSaUCc5AWLG9jdhCBm7a7iYavbeWOtIwxyXBQdXRTI1GqC20jGZIdMYQcZ1BnvxkGTLE2iO9aF6O3WjOSNWqt7fTmfy1c5UcFKuTKgkIulAC3sIzq96kapCYhCLjjnJtOUF0VRytIWdiDnLIrDVpoKzhFQMKGlmtvqT9Jamx0cgEEYiEgYpS0EF4o0ADDERhIpHQJbUgakNAcQCIJzzplGKksZMJYEIZGWRIr02XzeCMIoCnkgBOPdVuvmlSv93mCR50maJnES8iAW+m7KZ6rxbrf127qM8v6VsLXZagh2QqQ2W+nl9X6v19tYXwvDSBEAchEEhBAE4d/7m785/2f/6uGzvdl8cXx8evvGlW+8sb6zscbsA/hYFHGZT8fDsw+/eNpb382z7M7dN9e3Lj1/+jiMk6TRFkJEURxHEeecITJAjaSkCsO41e4209Q8u0RJeXh0/Gz/QJLO5nMAGBfF6Wi2Nmhv9LutRhwHvFR6scxHs8Vksuj3Bv/dP/iv775+lwfxYjpSslwu53t7z/YPj569PMjyfD6bTybTXquVRqGUsihlo5FIqc6H42VZLotCKSUaHaW1DkQep81u52/fvf56t1FKiYyZCJHDUBWHh1qwqbI17XuPTct+9pkYFkMuFK+dEkdHujaw6jKb6DwudDEtF1k3jRECNDdBHLfujpmNQpQWDAVnGmhZFFlelEWplEY7A0gI3m21Ntf6RDQcT4ssF0LEQaCk1JqYyQYwxhnjCAhIWiFx4+4IzsNAMMYLVWgixpkg0giFVpyzNIqiOMzLEhHSJEKAolRZWRaktFLLPIdAsCDCMARCSbrRaCCy8WSUL5dbQqRB53Yz/Xg8XhMh27n28/sfbu49v7LWu355O0G2vr6ZpOn+i5e9bqvXG0RRLMKItC4V9QYb/7v/+u/8i9/+nU8+u392cn5wdHLzylY7YM+ePm61e81mK5PZx5988uG9p61WtyyW//b3fvzg4cO/9/f+wdUbt8+Oj0LGojjt9PpBEAjGzMIBk0ojgk6332l3ZFmkaTqfTfdfvMznyzAMpVSz0QiI5kCn56MHfF8wxjiP4mi9371x9eqv/co7b73xVhSnJydHe3ufnp4Pn+/vLcsyWy6fvjweNBMGsFgsu83GlUtbjTSWspRKDcfTk8m0yEulZBnEMNiUSavIchGlcx6sp8k7G/0AmfJbHxlXxlKd5UsXUoeaCieya9vs4g1vkvrYqFnDSVjVgRhAe1fJl0Q5LrbBArJxKLSPWAIgIKGkVIgISEiC8ZAHjHMA4EwTQVaURVlmRZGV0i38RSaY4DwUIuAMkTjnQSCyLAsED4MgK0smlQlbcIaCSJaSNEWhMMF5xtBEeUoli1JyZGkYlkIXWmuiQARRGABjQRCEnMdRpLRWtBQKAwxaaRJGYa5pXsrZdDocnXNOSlMcJePxKA7EcjBYUDJXwcEie342Srvd3btfhUc/bTfi3c3NWZa1u/1mms5m8z//6UeXdrc21tcbaSOK4m5/g4uw1Rn8t3/373x+//7PPv70+YuDn3366LNHz9vtZpqmoRDn56PjszGxoN/Lnj1/lufZaDx69+13Xnv9zVk05EHQ6nTSNDY2iR94RCzLQmmVNBp8eEZaTaeTg+PTQukwjERARZEHQdBK404r7baa/W4nTRs729uv37nb6fbm8/mDh/dPT09Hw7PDk5Nllk9n81lZAuPrnSZTajiadFvpG6/diKPw4Oh4/+B0sVwi40iaVIlRqnuX8iDWRKwVLqOYgvCDrUFDMLtEAlxhkSYA0OjyR+i8bOeMg9PgBGZz5QqeF7LzVsVXBSTg0KixngL1J5CvKfF1JNYJE7lUGiBENJvGRoEIAqGVLgoqyjLP86IoCynd/olIDAWKNArDKCzK8nw4jqJAK2X6RESkNAdExs3m3DlpxnjKeBSEXAhkGHBu1tAFXGRZwRDTOCq1XshSaR2HASIulzkHxIRzzuIo0gCKdFnK4WIZKcUAGJFZx7LMsqIoAVicpmmcKk3rrCCt3mrGzy5tb8fJrTTak4vR9LCb55PZdDqbbm3u7u7SycnJTz7+ott4trnWXet1WieH3e6A8YCQ3755+83X747Ho4OT46OT0/lyeXR6tpgvgzDZXONBIDhjKkrCra2drc37D+/duv1amqaL+Xw+m8RJHLrAjYkYSqnLstRKN5O00+loXYIqTFkjB9AIAedX1jtvv3b19dduDza246ShlJ7NF188vE8ERZmPR6OyLONAbK/3F8uciPphGAb85eHJ6WTa6zTfvntrc63/yecPv3i0BwzjKGrEkVJqHMezsCUBismIdbq6O9jttb9zbeeblzZMEZyBoS2udKGjClPkYkuVUehodCWf5DJGfim5NxGoxppk1qXZcGdVPuLpuYZUv/xJMGQCgQPEIoiDUDCmlVpkxSLLZF4URVm65w0CAeMsCEQSBlEUxVFIQMu8WGY5YywKhFSKI3BEYkwASKlyJRljwOxDwky6qCjKQpYI2GrwRhopqZFhgLzFmAnpGUMCOQNEwThnjCPjyErGllmeSdlqJN1GI4lCxoRUMJ0viIvu2nocJ9lizri41GhebTY30vgX87IkosHOsNFqqglNZ4fD8635NIziwWDQbaSCY16o+09eaHqOhBpZVpaMi521QavZGAzWu+3Ozsb6+3fvtNrtKAyBdLPZ5pw/e/rwj3/8IRAxVKAKxrlWeb6YzecN3mpxt+rVpCzKImdcrO1cSRuNxXQEpJMoVFopJYNQ9Nrt7fXe1uZalKR5nj14/HRZSMFwNl92O61eM+Wkx5MpAyqKcp4V/U4zK9XR2ehsNNla79+6emm5zP7khz8fjqcgBBdcE02XWSGCIu3LuMWJxSJcNNuNVvPvvfPa24OWts/F8LFO6747l8ijzmpdr5LNr7YcqQqLmrVrNude51FwgSpnD9i4fc30rEHfpuy9WUEAIBKzQJMzxpiUSipVFHKRLfOizEupTZEHACAILsJQxGEkhACGZu2Hto/owjAMmZQiECIQZHZFNAs47QNLqCiLrAQkEIwjw6KU56NpFIZxFJrWFaUkoHYYaKK81JyY4BwZkq1W5rIsA864EAqw4Gyt2RgMBgQggfqDdWBcI2s1mygEMZ4myfsxa0fF42UhARut9uNRHNJxYzhuPn/ywZvv337tjSAMHzx62Gs12u32+WR6dHiyKPIwCEopi+VCKl0qNV/mUlO/09pe61+/cmVne+ta2mp1O1euXH97OPzxZ49PBZyeHDTb/SCMhAi0UmUpMQg4Z1KZ5zyiCAIRBDQbHx8dnhy9jKKw0UjNyqxWI71yaTOOAkXs6fO90TSbZ3kplVQqTeL5fCYJDk/Ollm2zItBu1USHZ+cTsfTNIluX92+cmn7fDR7+vKIEBuNlCMCQ5HEQ+JjasxEpDUIpEyEZdL4jWs7d3pNpezTZRjaLb4McWinbpHQrn90j0YgW7hJFd25Pbh9QZOPENmNh92xRABGszNCMFstmZXyPk7lZ4UDNyJp8/RNJoihCEQzjUqlZ8tMKkVSyUKWZVlK6ScK54wLlibJ+qDXTOOiVOaAQDDGWByFYSgAKImjspSkNZAG4L7RpZRJFEVhsMxyAkriWHBuqvQ5xyQOT0fTWbZMomhRlouiEARJxBhH8+iPOAozKbks0yTBQDDGm41G0kgbjWar3Vkoef/h/fW1tfDS1SBthWEYx2kaJw0ukijuz7NhHh8tM6m7i8uvNYvpXM2/ePTF9cvX37j7zs727pNnT6aHB1EQ7GytS6nSJELGxpPZ0dm54GJr0Jgvl9li+ej5i+f7L/u99uWtjffffqvT6TaaLUbqP/zpj+JQ/Pqv/bUgbDTa3SCMyrIkojiKzJOdGGNRlJRFLou8LMvpfPH85cH5eIqcpa1m3Gwo4MDD0TRrNptbm51SytPTsyd7+804VArSJO60GuPZfDJfjMfTVjPZWetfeusOIpVlOctV1Om8s76OnJdSglRBFA7ny2f7w6kGPZ/xRhMaretrve/cvPyt7QE3qCACArMPmNFt2kXTbVCSmXXrZLFrF0Rax4WY21KZ/JcubumIcBVyaJU3+WfBgLbEbbzp2i4SCKBtCh1ACx9r1ZpAkyxKVUpTzImamA07oeA8DII4DJMoSpKYIDfPfmPIk1CEUSilBC0FxygMi6IEpavNShyNB5xJIRhiyJmJkHHE5TJfLPNlngsCXcrJZJpJRQBSSq2p0UjjCKMwCAMRl4EG4IxHSRInaZw2lmUZ5jko1Wq1ev21brvfSJsiCKIgYpyLgG9E4XY7PZ5l/8P98U/Pxq2g0dncGBfT2fDw/NGD13Z21vtrX1vbHA7PHjy8d3Z6XBTli+MzZCwU/Prl3WWWCcZuXb00mU4WWW60Uhjw0+GQiVBr2txYOx7PHz7f/+rJ8fa1O0EYg6sSV0qZrfUZwzAI0rTR7q21z06KstSAZSmTNO11O0kcD9YGu5ubWlMYiIPDw363szYYrA36Usmzs1EYCNB6d2Pt2tbabL6QSm2tD6I4ZoyHTZamjaTRjMJQCBEEotT044fPPj3NZixhACXwZdJs9/v/zVfu3O6kUmm3IsOqVJfStAkkF9m0jyFgTvGiY04bM9VOudtfccUUdTFOs9bXFX+a/UIQzO7AJrBktxAnv1sjeXsDzHVQJEEQBFyRVrJkSAhUSqm0AgNsRMZYGIpGo9FKkzAMlFaT6VxrzRCSOOIMw4AzhELrUirQmnMQgmnFUGvBuUkpcM4LKfPpjDPebqSc86Isy1Lm2jzNjIBIa5JSMc6QQBItizIuyjhWSimgoJMmnPFRlsWcN4MgkGU2mpwul7rIhRBr6xth3DBPQfaPQDMPA+GcDxrJN9Z7B4tsP5fXuz2gzo+1SJbTfJKvzZ53A7bd77/15lvHh3v37z+4cWmLiWCZZS+PToui6LYay+Xi9du3AHC+WDKkJEmbrc50kSuFUbP7/lvpjZ2dwfbVKE6iMOQctdbmvsiACq0BASAIo0arc/XGbRGIz7/4vMwyFoSFNlngQkkVCN5I4sFgsFjM+73BxtpgPJ3O8jLLsyjgj/YOllm+3usMep0gjNJGK24kcZKKMIqCQARBGARpEP3RvUd/9uwsU6AZQy0LQBknX9lZ323EZSn9okskF9OxhGdD9I7rwJdtvLoLQ5UNclamJrdpElZ4dlFL912tGMXfUbsKEp/uJxdh9ZcXaRqbBdFKk1IkpVZKAyBnLBQMAJCzRhy3G400jQMhgkCYwATnLBAcEaRSUkoE4AxKRRwxYLwAaboXBgHj5rEIyBA5Z0IwAiqVUloLxgRjRFSWUgNJJakEHohIiJBzEy4lIE2UxkkUJ+W5WmbLWbbsNZvpYIDIsjzvdDqETKkiYFwwIXjAAyGECIUQXCBiGolfvb7zznr36WSeMn5/NHsskVjzWr8/oTxEdV4qmc0VRrfu3M2y5fD8/PD4NA74Zq+fxmGr2cyyRac7SBvNQATIeRhGZVEIIQbrG+ubu5vrm0GchoGIwsCg02wdYB5Hhpo0IwKKoki1ujtXbk6X2dPn+71Wc9DvP3m+LxgOh+e9brcsy267vTEYKKVe7O+dj8ZxGGQMO+3219/pLpZLBGQi6G9t9drdMAgwEFyIkIkwDE9mix88ePCn957OxtMoSYK0PQui9W7rl6/tfP/yugCtwe164BBWcWkNg1U5nMv0XMSo2+HLsW3tdBsBUB5gRGif82qPMgVw1oNHxtzTFldaUd2USJSlYghFWZalKopSSUlaAxBnLOA8CAQXIgpDHgiNEEVBHEetNGmk0Xi6mM4WjSTkis2KQnDebKST2VKqAt22KUrpLC+5wDAMm3EiBEcAKdV0nmmto1A4W8fk1YAjk+i2M2VMCM4519rmscIg3FlbW5ZFTsS5oEBESdJqtjqdPiCejofnozMUQgRBxCJu8w22poUxdrnb2u0081Ii4neW+bSUgzQBikZSPl9Ot6L+7e3rnTjI5tOjg30M0pBjK42iQORSLubL0+EoEOLq5ctJo5PEyWQ8bLQ661uXe4N1Yedq9fKCNirerOIXQjAAQtzY3P7at/4SkCpm492NQRxHjUaj1UgbaQMZV7Icj0bLPAsCZsyDRqPBGaSNZpAkzWY7abYE4xwZFyIKglme//aHn/3kycuTyYITYRyp9c0pD9rt5n//3s3X20mplNlmwCfdq4Q4QR0ZxrmhVx7ZUYcsur0U7ctlgcgyY5Vk8rGnyktf5Ul0G+RSVUxSJQQsgy7zXEpVFEUpldndXQNxxIAzxtBYB4yxgPM0EEkUCs4JSCqSSgWCM4b5oljmRRoFSRwJIUaT2TIr7OITTZokAYtjJCLSVCqV53lRSkRMQtFI4rJURb4EAEWglUa3mC4RnHE2XWStNBGCL/OCcx7HsSY9nkwyrUnwZpJmSi2yBRCAkkoVSpZSlrIouQlMuIcy2NAbYhQGdze7m610nOVlqf7J88OfnZzvRmEqdNqg87JoBfHV196+eecttZxl80mRL7gItKKz8ZiLsNVpB2EcRclWu9dsdZqtVhIFRHY3NT9y5Ha7MMoO7eNNkDGWLeZK69u3X0uS9MWTB0mcaFVMpzPGxXyZtZqN8XisVTGbjJZ5SURRFDXTWCpM291mpxuHIRdhFISc80zKF5PpD+89/MGnD2WhkzSlNC2TRrvVvNZq/N07V262wlxKR2UWMpqIedq6sBWtz0z6b5zC9Y9hJecYgUelx1dlJfhtkMDFmHzk0yWrajEpD0p7fMXmKMwazbKUiohzzpnJypvHwHDGWKfZ6HU7jKHWWkq1zPLzoWy3Gv1ui3OuNCVpI4rj8WQ6mmVpyFuNJMuK5TIDIrO7PCJxZER6Os+WeaGUCoQIuQBAxrjSpVkehVpLpYxzKBii4IJxRRBFYRRFWmcAtMwWw8kUlN7udiMRnI9Gs8WCtO51uuvrWyJMSOvFcg4MwSpaBAROYJ6hDAAILA3DJAg3W/E8K7+fF5+ejz4cTr53aTvH8P5k8uenZ3ca6dfWumtR0G302o02B5RKNjt9IijypeA8jJNGo9Fstkw4QrnHibo13NVScbtyABFAm122GQBqzRgDoss371y5+fpyMjw6Otg/OjkfnXXH55PpjDMYjsen5+Oz88mdm5caEev0NgbdfqPZYkIwwNPZ/OcvDp+eT6bz7Gg0K1mkG0L3BnGv/feu/f/o+rMlyZIkSxDjRUTupott7h7uHhG5VVZWVVNXD6iHgBnQPIBo3gEC5g34BvwM/gGfgEcQDdEsBAwGQC9V1VWdlZWRkRG+26aqdxMRZsaD3KtmkdUwD4rwMFPT5V4WXg4fPvzVby63r9oqIEiS9eaDmZZJtMXqcO2jL4sCz5s5F2P5qWtdqypbS6G14F8fDwa6eoInU376zvk5f5oUnL8DzxOJ818QnIhIFjNTs0DUhjrllLNUVajrqqnrF9f73XYbY+z7fhjHw6mPMaOmytN2u62rqmubKoTHY98Pp6E377jUirLMlhISZRFEFDNEcMQIqAgFUlYDdq5sMiRGWUIFRrXDNAHYPrZVCNuu8T48nPrb02lb1QTWD0dE2m02ZspIJprGU0ZsmdM8ReYybksMZeaPCNakHAHRgdvU9F+8vm48/4eH056pYni92VxO+RZgBP9/+3h/ezr8V9fbb7rmZ7uNxWkae4QyEs1t1zVN45gZ0NCyloKddP0qZZ/kDIgqkkUkpTgODjE4V4y18lVVN5dXN9dffX3z6cPf/Nv/+e7T+2GaP9097rb7f/WvfnE6HrZtuHlx02yvBoPvP36eRFOW//jpy8e70+NxyEiz97rZpN1l1TX/21+9/a+/voKyurTcZX0yBV3ReAB4GkxbvSKep92f5G7OXPlnCOjq7J5DRmuc/4krPLtMePZry/fXTtEZXIVnwef54x2AiSkCsFkZ4wRGw+S93282222nhnePB0fETHlSUfEes8rh2BvA5QX1g51Oxya40XHfD+OkCxRa8ClA77z33gePzI6dihASe1cXWhoiOTYBImYkyQIAjGgpZQBm6scxZgnBv7i63LTtpqpCcOTctt2GUDVVU9WNCyFUlfM+isxTX9eNI1LNBm7dbA6FUbMm6GYITLRv6v/q65f/xVdXWXXMcjvEl7X/3eMJQP9st/t3D4f/6x8+/+cvb/5Pl9dX2yrnzM5V7abbbB17Wq8yEeGzNU5FNa3cGCJWMBWdx+n2y6fD423p28XH++bVW2YHgCbK7Jqmy+S7y5ff/Hzz1z5c7ff7/X6O0zRNYvBpmv/m999/Oc1D1MeYT1GOx5O1Lex2v7y5+K+/+UoRX7Xh266SmP/UJnAdVl/xyZ/8+MlOFgqIlYJqJcHhefao6CSWeLBu9LRzzf+svD9nq+dS60986tkuz+ntT9/RU4MU/+pf/2c5JiYyBO/c5cW2bmoiRMNhTrV3SbKZtHUVHKuqc9y0TUwionUVNm1jAHePh3GY+2Ho+2GcYoEARQSJC2Llq+Cdd84BgKkiADM7z6I6z6l4NRGZVdXAOdfWFTNVzjd1aLrWISFTHcK2bes6XOwvjN2YIhEz8367vb56udtdOKJxGoi56XZVVbVt530IvmJHRMjEZwNVW+5ByYzLPTKzJDLFPMQ4xKRAd1P8/eH4bVv9q9cvGMQMkNAAq1D5wgssI6qIolo0L8qstYquECOISBynT58+/n//5t9++PTx19+8veyaerO/fP1t8JVzzsyyZCISyTnO89j3p+NxOM2nByb7YdB/8/EYqtrU/vD+8w+Pj7jZNCFQ8JuXL9/suv/ml6/f1F5SFFEp87PnvO7ssZ6F1zOQRM8IdIsRPWHvBbSk1XHqc2dXhnIXaPOnVCZ4eob15VZ5nMX6n9niUm+cS66fBv6C3jsCLLl9XfuuK+0ev9+0TdM8noZxGAwElFUkqjim4KpN2zZ1gwgx53kaJaUUo2pi1F1XNZWTnOc5ncZY3piqxikmSiXfR3a2Fl5N8KYwx5iLwiIAOmbHSOSd984F7755cSkKv3v3IaesKlGaUNUxZwDYb3fsmENQYjH17HeXN95XROwcEzsih+eW7/r5yxxp+aYhlk0CpYBlotr7XVPFlGeRbeWvKhdTfJimi6aufEkVClP+qVwoUr9cemNyFrOyJTE1SPP48e//X7d/92/VwH/zutle7l++9t4jURl7LLO1atb3p9PDl9PxkCRfv/rmwxDf3304jPnd5499TC+ur/7Ft9/sN81fv9i/6OoXTbP1bCoxRhF53tdZzcbwJzd+ocqXHy5nSHUZdysx2sDOwNCyz1rPiaeZrH3IpTyHM5z6HMuEp9rombU+Xf9ztPmJZf8zVwoATiQnySBgYESsBsd+fDz0Vxe77Xaz23aS0zBM0zzN0zTFpKpJtKlHQ1DReR4rR7vWU+sB2pRlnOLjcZxjVkAFJUTnnBGqaYxxRGyauqoCIqpaElsELYiyqWNum7oIh1TEjsv7mUqqAGYxZdPps936ENq27XP2RNL3OUvgV5tu50MFxETsXXDkaBlQwMUsVw+6gEErOL36FaOl9c8VEHEOTJvKl4iDVPaewRnVFhE7a/QjYvmkZjFGU5Wph+lR5lHjIMPjXj/u6/z6z//zb37zL0PbMTsEMtOcc3knagqI7XbXdJuLsf90OPy3v/3+//39+3q333399n9/uctgPvhf77tvL3YOLcdoBiJalpg991tncOccggFWJQZTPa9dsfWzI61UtMKvo3WebEWOFttRe25ATz96+tLnNvccpTprkDxXznkOfP4JDQ8XSVCX8qKxkFIepgkASgU/9P23377pmqv9xe5yv308HL58kfEY72NOd0cz846DZ+/Z2uqi6rZdE0J4PI2H4TaLFIUwAiBmdmwAqiYqZWJE1Zo6AELOOaU0pVQkx5tQX2833jskSillk02oTsN0HMZ+HMcs+67dbToX2NfVwzTaPF1d3wR2TG7OklL0VeXY+TX4nj/tchWe2eh6J88/X442AhoCOfTK3rkCo5R9dilLVq29R1wWlqrJOW1YdH2ZQwiZSMbD9Om3px9/9/7LXR/zv/njY/fmN/+rv/5ft123CAIAGigVGFi15ImE+DjFv789pZR7F/76z3/19dU+OPdmU99sujYEzy7OUxIlHyQmXbEtldXPrUofsPpme/bZy1b1J5+3atTDsytiyz7jc320xnAzwJ9UXWcwAM4z9ef04lkqCQBL5loSvAJ0rK+x/tbyeFsxWlM1RGeiTOSYpKzEEgFEXwUzBdE5xpRTzHmeJmJu60BE45ymOXrvura53G+JcUpJhxiSjtMEZUAewExs4cIiOzYzzJhyUtMpRgCrghvneZ7nLALMRJTATjFuEKtAiuCRN03Tj9M0z/00j0kc04urfVVV4N1ls/E+7Npt8CGKfL6/q6qmabeeHRKKqYoxMCIy0Pkq/gnA8bxNcjZeW7zpMhZgBqpo2ZjAg0s5JZGCyzt2pfnMzFYmxBEBkMi53Qv38/9y5lcP+fcc/P/mX3/9s5//AgBNFRxIzmpFLVBzFjMTk2FO/3D3aEg/TvGvr/f/y59/TQWmAphE5iwofUQKvvLBm6mrUEcxMwNDwpVe/FMYp9xwNABewaDF2tYEAJ9V6XbGJ88t+OfX6yz6dfZ/9icQ0k/h0jPXCcpg8bNffMqPn52f9VeWH2HZ8uEdt7VPYqfTwGaGVFV12zZZdZrGmCXGRGg5ZUKovGubOmZzjKI2xtQ1gYiGcYoTxJRiytOc5pimKABAqsEbm9ZV1bW1qU0xBc8IeOwnUa2r0BJlNTXrnHeAKee6Cl9dXY7T/HDozbQO4fbYm0jtvahNKVUDfH/3eJqnfde9ur4JVT2mNEyjmb568bqqayT2zpV1dZIVmJAUFjbikxN9ss4ycrAe65VSbss1ZPToTDVlISRHJiIAEHMyfXq24reZCMBE1MjdfPvr/auv52EAg/F0ZOeWBjRYTinOkwHY0moWVf1q01SM/+U3r4go5bTyiC1QYdMyqsVpBEQXAjnPdQ1xKpsqFmVqhCXXXM+gLYNrslok6MpFUoCVybSCSvj0uz8x8sUB/7QGen7az50nXSdCi8iSrvSZf+Yd1uPzU1+7Pn+5I66E/zHKPM9JBAG9c+wYAULw3vuH4ynH6JhjKuFYg/cGIIRiMIxzP4ybtmKmrJYVH09zP8aYxMBKnatmKcasmiQTUhGzGeY4zjMRBl8jUZaY1Vow59gAKu+C9+OcssygOsWoYNttVznu+8EBTTDdno51VZ/6McDd5X6bU/qcZ5QkOb1587Ou3SCgmGpKhAjgGdBIi6h5cRN/0pb8yd9XCHAx3GJUSyWUAICZk2QoU+OmKUvKOaYcswDiRddsqkBoKpkBqiqkmFRyShEAytCgmJUiScHMhBBaz7vGm9nYH6umY3YiYqZEzAgqeTktiAYWx9Fs4KqqNjs5PKhKoamVxPOc0i1TFiWs/0n+95wyt4AbS2IA//zrmRn9NBNdje9pz9ZPSqP1kPynn/L5v89/ef4qLqacsxiAqjCRD2G/7TabRszu7h9TakwtiyCAqExTfDgOjOS9qypfV5UBJDWiZrvZpKxTfDRAVSOipmlg3XNvavMcU4zBeefdlFKMkRCDrwxwmOMcUx1C19R1HXKW0zAOc3JMSKhIWa1mDoSFGHCcJu/4zeVVUzfvT4/34+Cb6mevvjLmKcXD2L+Ic/SO1ZUYXSSYy709ry1dGymFX4uwEnDWS1pu6PMEoCwsKQ4TdV1wSoimwoxmnFUt5ynGL6ee2P3m5XXwFYdwru6ziKjYIseiAkk1g2RclpOjipRIG8eTbzbsHIBJnJmdcz6ntBbLiIg5x3gY0bmq3Uz9oUDITzf7nGqDncP6E1b0nAXyVPuvlfXT7z5Z0p8Y0/l4P9n3swv4E2N99n179rpPYf1cUK1GD4tLB7eoQSJ67+u6Ct61XV1V1TzncZol5xB8FeoquHg4nYZJsjjHAJpFUspV8G1dEWJKaYoppdw1VYoxiTJRUwUgGscxQraUVCwDFpkxQHDeOe9hcfJIRGoQYxpjUpFN21ShBbBxijGnnEWdtCH4uvrx3YfrrmWik6S2aVTs/vHovPuLX/z6Zbc9jv0/ffcPX7/52f7iCpC888ysYJBVCQtCdMaqEZD/tMW8LHt8usSLk1lqj+W/ts5IFAo5oGOuAUBVVP7N3eGHMX11e/jXl1002FeVI9xXTGZieppmBCPNc85q0PJyYM7VNxJZzpojgnnndZ41ZyNi5zUnWyoMW1DXcc4p1t12Hk45zmcDWe7xWiTBuj4LdNlZ9RRe4dkne2ay+J+quP/ETS5G9vxsPy/V7Se9ejv/FgA8yTTj+QiVN4DP2vSufKtMZgbvEbE/DcfjKGpMOCC8vL7Ybbo5ZyTynsvT5Gzeg6jGlEx1inHTNuQcEW26RtQAUUQNQHImxMp7QkwxqSqTg7KHE7BspSGzxlPFfOz7snq3qiomFlm22agIM9VVcMEzQMscQuhTvOz2L/b7u/7YbS7fvv4659Q/3k45TYCB3t3df3n56u3F/jrGRJSpiEgWimpBU5jOF3ptsix1/p/6jKcjjcvC6me3BwGQEM3YqPb+b+6O//Oh90T/+Hh4xfr9FBX5v388/pfb6v/w9gbRpjkxU60xiCA7pIAKBEiAyCQigIjOaRZEyoAUvOUMZiqZvJeUVESl7K5SAMhxHnKuuo2Z5TQv1c5P4/jySc8+71nyd/aKf4KcP/eUPwEvn2OZy3qPhS5z9n/nFupT8vDc0FcsFs7Txk+3oPxweRtODcCUVFRojlHVUs7MXFUB0AFAVmWmCh0SNnXtKM4x5ZwIzZTBOyIGoKyGWUSECTdd44NDs3GK4zSnqGZlVcgCthXV3PJeU85ZNXhfVX7KmR233hPgNM2gIqJTTF1dt22zbdu2CneHY+29ZiEEVL3aX/zFr//ix88f//7v/v2ubX/17c+96qfbL/3D/as3X19fv5qnYSZyzjdVretlLUwSXdZFPiVIhShz3jyGz6KkWfETVnrramWcwyGAmBVWChL8zcPx//JPP/51G+7NDlG/3rV/cbX9NMt71e/j9MMw/aL1O4ZCMDRENCWwlShgZlbu9NJKIFLJ7JyAGJS9P8Leq6lGUS1NAUNEVZmHU2g6U805PYvuTyX2k6vDn9aK/0nk/HmOuJjPfyIHff7M9uxXllPx01ThbNn2TGvkWSkKz3xBeaQ6RhOFnDVJpDkBoqFtgru62LVNHTyHEGLO85xyygjqHBfQNKs5gizq1LZVqOsq5ixJh3E2haapnHMGs5mVTiCYLSo/qp658N3LB2UiAFC1yrlN2wKCaFl1h0l0ivPN5cXLywsxm2OapjmpxpxCcC9vXjSb7T/8/rcPDw9MBETv775c7y9ev3rddFvR/Ic//Pbm5evrq1d1qHNOMUUicqX1icDEBaImwmXh5wqYn1O05YIut2BZn6cqktMiwAQIVNaRAiK83jT/x29fvQ58Efz7ab5pm5ppV6X/c3s5ioGJrt2dxVkgoxkSnVsva263JMdIBGbIbJKt1CIGRFSq/udmJDlP/bFqOjU9Q/e4OrY/MaxzWY3PCEf/3PLKd/SntOXnR/d8ifSnRmk/bWL95JntKcdYar4/qczMEMpaV3DnF1109wi9D0Qcgt/tt4FJVO4eDsdTn1M2FdWi5oPM1DbVHGWe4/3DQbRjQkfo2naeYozJnOaUwZQIELng82qaUo6iVfCOfYwp5VzsJYvsmrYK7jTNlXNNVT2cTuM0eefrphHVEHwWNQBP1Hbt22++3u92f/junwjx1cVV17ZieuoHR3R9efPhy/v+eHzx4tU4jff3n0+Hu+BDaNqqarNkMHPOMxNTEThjWIC6Mz0NaCmI1z2OCudpXVVBRCKGBa0D1AWp/qpr/3c/r2LOIvLtrs0iamaKBNCSLroyZb+UKi2eA1cMdgUOVo9ipoisJsxcFqmBQc7RsFRsT1VR+YuKTMOpbrppONlPgzucyyBYq/pnEdz+U2X4swACf2Jnz//yp988pxPPDH1F4J8euqYNcnamuLQMbM2jABDc+vaW9xB8aJqanTv1Y9vU3LUx6zTFOCciZOcsC6oF75goxtSfBhGt6goYL3abtq6c901dDcMwxwQI3jvvuESWmJJkFVEtRSuloj8OZoTIxMMcYxZX+kDEu64jJMd0s992XWuAn77cmhoEt7m8PPVj//iYRXzw1xcXTHh7f5hVfFXffvngiH/1q9+4qvnx3R//zbvvX7/86te//ksFNDXvfZYMgN65um7qwGAoubD40QyQjAgXPbbiHmRZAl0KbVVl9lg2qpSiCklsWUdmSMSsYAWHN8kgGQqOCbIM+C5ZAcIiH702s57d0eI/EUFFDQnLqkIEKEGc6LkHPduf5jyPg6uqOI5PdnNO+NZHPuXW///tcgm16/88fy148r7rz/75V7HdgsuuvBRcCqeVFbA8/RpU7CyKvLycqyqfs2jZjVTOlSoaSJbbu8eYEhEBYt02lXdt2xDRPI2HY386DZIFCduqqqvQeN9WdajCNM0q4p2TnHdd470Dg893j9OcVMqlpuBdCL7yzsxSFlObcpbj0Xned93Fxa5tmxhzziCqlXfsuGnqw3E49H1m3G12P75//2q7/ebtm8/39xfdZrvpTv345uXLm5tX4zwhNGZ693D/3bu/ufv04e3bt7uLq7svn9i5bbcLdU3OO18hoimoWRkUJuaiRBy8N3zyNabPTz5kk1ITMJKBZRUEICRFxKJVbFr0xtY7sCwJBwAgWgzCAEyRGJjOFrkUYmaABEiw2C8qgErGspDKVERyTshsa+b3VHQDGEAxXx9CnOcla1yhh+dJ5BKF4ZmbLAH9mcnCszLo3Jz8yZIk/BPztBU7sNUfnst8xKcmp9kqg7OEnnOGQ7gUS6aA4Lz3gGh6pt8Alu2DoNM0pRQ3XXtztd9tuxBCU4WU8+3dwxzTiupp8ME5J2qH44l6RMKymogdq9g0pSmmw3EYp5glI1IdQlUHJEQEWYhAmFXnaa4tBBdP/TjOaY5RRMukxDzHUz9mkV99++0k+o/ffQcASNQPQ/But93WVY1I7WafVU3leHzo2s398XD7+cMvfvHLN1+9/fD+B0kpSmqr+vLi6ubm5eWLr3KmmCMiiuSStIhKW7dmy+7dP7l5VpryOT3P/XPORGRPHnC5E8GxY55jMY9VMw5gqYfKUB0TIi0M6oWvoaZGCFguEK76mrYM8Zb9vJJTGbnS8zv7aUSO8xTqmoj1zDwqkO0/QydW612LFvhJunnuaCxpYvGChGtg/6n80tmnmhmu45qr08ZlKRIuj1yxvifnamsf6+ltoWu7ulz1OeaSJprqtq1D8LcPh2makXC3bRihDc4AjqdhGoeS5xJzFcK2a4eYYsxNHerghnnuh2gAqtYP8xxTznkYx5SzAniCUnk6IAbKkMt+PAZMoimlw0mGYfSO67q+vLzo6qqua+e9iDrvFPHh/g7MJMs4Tv0wusrv2o4QQ1XFnIbTQXPabvauasLh4X/x1/+ZIf0//6f/cRrHum1eXl7P0/zv/+5vfv6LX445vXj5FgFOhwcFaJsu5bhpd6aljWmIBPh86LtsQxEwOCvbqMhiAbgkfLisMUFCzIpMWSEX+KZw+emcz3FRElyW/5bKHckB6vMArypLhgqmAMV9iirquvZgNZaz5yv/TvPsQtCoiya3mf6pr9OnxPeMjuvq+nBFJp8kZg2QymPO5rt0n3A1cQOAZ0UPnh21mpbjtiAUxZSfoCU9E/cKVLtEfbdtAiGa6mmIc4wiQgRq1rX1OM0jWOUYNR9PhxxHx6wpgWkd2Ll6jknEbh8OKQsRT3MkRFObYxYRAyiiWVNMKWezMpkEWS0vxFoBM0dUh2rKcY6gak1bFWYTMgfvnPPOcUppu9lcXVz88cN7NGDmvh9yzlHFJPXjeH152bXd8Xg4PD68ffMN180PP3y32+67uv3db//+ctMN3h2n+ePtF0d0eXGlCGL46dO7dDoS49ff/ll/ug9VVzTzUYt5aBlqOk9xmC5pJYADMFERlRKZVcW7gIi67ODFcqOXmT01tFX0dTW+NYiDwbLaGaDQn/mc262RDgFW9TmRnJJIXlb5LNT9n3j6c7sop0RMGvMadRcLXbHes/tciO5qRgC2KC0v7Bo4O0GDwhzFp3pr9ZAGy/rDAmuu45rL1g9bX6BYvhoiLU9sBgiGeB5GWf1qWbMEjtillMtnzCLjlBCyqJU6pq5CcCiSEbya5iRMtN20O9B+nB8lDykP46yiVRViygAQvKvqKqUEgDGbyawqRUFHwdCU1yn4kp+54EUFVStedm9Xwfvg99tt3TaO0Mz2uwvv/e3d7b5tQXTO6dgP/TgR4nbTVt61TTPM84/v37VNQ879+Mffd93m1YtXd7cf97vdfnfhvD+ejuPQf/funQb/6tXb4+HxH//hb9M8/+qXv9psvzDi5usLW+jwGgBKwnQe1zSzLLlI8SAWxExKTV6mPNTUMZuirtt3yx94qpiXe1lo+biMJwJqWT9ugIrIT9j1spUFz65pdWUl2cgiWsYTnu12sXKiYDFBU1Xisk3oyX8aPmWOCzVkhYMWuYViOSXaLvSi8jA8H4enHGEtXlbDXtBiWIujMztpSTRxbW0iAFKhv5XsG5ac1QxBTMGgsMUwpTzFPIwppaRmpdSum6YJbs7qxFqgqqpTzqhKYCnb4TQ9nsac1TEpEhFLlj6mXIfgPRIP45xSyqkUFQtrwS/zzMhEazqiKaYk2QCaurrc7oCgaWoAPB5PhHh9eZEl398PouqcyyLBe+94itEHf+yH3/34fkr54vLCVC8urv7p9/8Yc7q8vLn7/FFNAen7H38AhJuLSzP75S9+WW13t18+f/jxu6ZpLi4uI8C7d38M7FzVtE2LzE23sWZjZt45Ji7WqaYxpUIzWLM+IEIClGWUv2x4fAq7Kc2LldiiGgyICFRi8AK7qqy3r9w/QeSnnK/cZVNb3wPiUy/2yb5K/H1WERdvVbykFZBVZdGkJVp7Rc9tbX0xfIrQa2pamKZPFVUhNNia/xTLfNYNwqdGkT0p6BWPWV5DdS2zSule0tmnrHRx6mbm5pjAICbJWbxnx6hmIjpPc+UdVt4MuLCO2c1JYlJTGcY55cyIGUBFDDCmOIxCCETglqkIK8a3MM8BHbF3jpmYmZ3LWaA0UQC9c9McY8oxRwyBs+Q0A8DlxRYMPn6+jTGG4AGw8h4BArHv6ix5irHThrxDw8vrm4+fP8U4v337zd/+/d+Nx9Nm2xnCOIw+BMt5f3n1+uVX96fTH7/7XV1XQz98+fLl17/5y5TTOM23/+7/4xG+evP2xatvDv5xv79SCYSYVyGgkuQtbgnRyrYcRFEEQJHs2CEgM+esxWmBKgEoMZqpChEXB1xIXmaiOWnO5D0RQ7loy7xlua1lcGjZdyo55ZSW8FeOBDwVFksNshbnZ1DIRBbB/GJuqrAKKp01vNbzsox+PquknhCf5VydE8dzYbg693ORtuTThmWOdAkBq+9dnhPxfC7O/twQwBTPTXkwxz7klMuZ8Y6JfEoiElU1paRgnjllnWOifhBTUTVVJuyampGkH89oHKEhUk55xsjOMaIZJpEygwtgIpJyxgLBSFYzx+ydA8CSpKac5yxM0k9T63zd1I/9+HgcmjrEnKecdm0nWaY4dV3TVvVp6KcYk1ntQ1XVHz59+u4P371+8+bd+x8Dgdbhy5fb4zDGlK4uLm5uXoSmiXP89PHd5Wbb1HWeY319w4C//aff37x49bd/8+9f3Vy7UP3dP/zdX/z6L+ebPlRN1bSIxMwhVLR0vJQdn+//Ot5UdAMyswOwLJJyWu9kyUORic20JJqw4FDFnSiqAvPKGIH1flrxj5qlCFLM0xjjbGoGhkaL5xFd8wAzKN3bpRhbK2wrKxFWwGyt/JDOQfrML1JYhOXxTBxerPVcSz0VW2cY9AwB4NnMivktaTTg+owAZ3r0E3awAvzFrksaUJ4QXdt1x+OxFGwpSXmrTCiCc8zTOHnuxjmmlLabpgqOmHxw6r2oVlVV1ZXkHLOM05xSVjUkMjBRyTmrCiGWJxVVAJQ455x8Dt45dIwOF5U8R66umIgJGawJoWKXY+rnSbKodE3dAKOqzpr7GBsfvHdJNac8TdPd4/HDw+nv/ubfv7i5dsGneQ7OHY+37z5+BsQ6eCYapylnvX+4y9MUmsbMfvbNt7vdxfc/fG+it18+OUQR+b//9//dz968vf386eOHHy8vrtpu02z3l9evgg/2VJlClAwAniBpzpKYHBhkSSK5jFUgErrFJhQFiZdweXYPhEhc8FPNCZDYLbtBwNCQSxFpalaYpGnOklU0a8Yy0KxqJuUtrcjmGoqX/yx3XmHtkRWzL4iCySo3QmVH0rMVXUuye0aFDADLiponn7omjKV0On89++vK+i5+8qy2eH6Y6TM4dUEfdHXYAADgXl5fmsg4TM4bO5UsKWY1IGZE6vtJRDdtzVVIKRECCQpReWNE2LWBsD710xxz8OgcI2LhmKYsVoI7MSFhSkW3PIuKzoBQOzZRNW2qGj1TFlhTEwLo5ylnGebIiI/9OKVc12Gapl23ud7vg3Mh+K5rs5lmOfXDu8+fVPLP3ryZ+z7PM+93v/j5z/e73e3D48PD4fv37yNY03af3v+432xjjI7dj+/fnaZ5mqdfffv1x0+fLvbb4+Hw4mJfef/+4wfHdDw8MuLNzQvLOV1eV1VjAN4FE02aHbEQZ82mqiClrBbNZ0wRlq6IIdHCMTc833qThVpK7CRHS8kQFRwVfsBaPpeSSCTHeVqTJVBTNCqJqYE9MZUBDHTF/FeqK6LpQrrGRfprjcugVlQQbdnDsVq4rdH8yaDU7Mw8xuVUrMZoAAAKdtYggLM12+pX7VlDy85w/pKYPund2E8M3PXDmLJ47wAw5xycY6JpmhHNOQKAnHWaEzOfevXBVz6ktLAumqZqKp/FQuVfXu+KUZ7GeZzigkQQEaGI0uK6nxcQsQ5BTeMsaujUERKATTE2REwcswxTFMlKdNbDceyOpz6myETe+Vc319f7C+f83f3dzX7XEL37/Onjx89dXX9NvO82X79544Nnoq+Cv/7q9b/727853h/2m+7icq+q+/32TVV9/DiMw+C9+/Nf/uJ3f/je1Mbh+Ob1W+/dH77/Pkve7XaHu89//MM/fvXV21DVdbclYnauabeDSJJYh2ZJrtZUzJ6xKwzQ+VBcqaiZChgsfA4AAEUm0lUxgKQUX7AgB6o5l7nNlauiALrkkiq2SM2fsaaFS28ABEuHfXGLRYV2ibrP5RdtdbdwLnpsIcoAgKEaPBU9qGC0ppJrNQ7FjBfuz3OEoER5BSRAfXLpq9z4OUeys5ozrP6/fAw3p5QlL5mAihKGyiNBirnsPDJQIvRZ6qbyvmLnppjnrJ4UoSrLgOq6Ct6PU/z45Z6Jm7qOqdRGwMw5C1iZ6CU1LZ5D1KYYg/dokHJCRF9z27ZVFdqqhiW82ThKzoominjqh1I4b7t2s+kIoAr+5YuXMaVtW2fVx+NxmqY0zxcXF/v9TsGY+XK/u7m6urq8fv/p83g4jeOYyvQSIjPfPd4fjkdT/dW33xCiiDw8Hl6/vLm82H+5vWWmHz/effN2bJoBTaahlxQPD18Acbe9lJzqpkOirIKIiIxUGCRWEr6zUKgVEQQELHPlCFDUhnSBFYtmTvFJarks1pWcJGfNKeU0p0UxGBFVwExtAbkWrpGB4TKVtqCturi/kmguosVrP9UWSsC5vwolSVzWKZg+ddLOmWxxo2gA9KxFCSuqD2e/f3aF51T1qWlUMktcIfw1xqyyPKtnXio+QNc2NQMM3vXjpKpgSgjOOREtA8kIBGbB0aapxznOMYJBTjmBNXVu6ooQx5jnmEUtpkxmITCgiZbN2yriRAUkkyoYnrnXMSU1YyQQUJGYU1a5dpel7eoc++D6EdQ0CUA0MCOm4EIUeXg4pG13HMemP+23G2badJv9fv/u/TsQ9d41VfDMMUUD3HQbQ3q4v/3zX/383afPn7/cEyI693A4uo+fReTmcj/N0+Px0NYhuMu6rgqmO07zpm23bX08Pswxtu0WEKrgRfI8nZCpblrvAwCknAByQXKLTFcp1WHp1WBeEH7IOcOKqpSVZ7jeYBWFs+tVlRjnNOcUc86Sc85pwXdK+DRd88mSZS4pxNpKOtfpdvZYxZ2XDNcWMPQpzyy1O61Li5dAfi5h1r75Cvg/83bnBy4fakl8n2eoZ7D2J+XQ+hJrqHkq7lf/C65t6qaqnHdlGKJoVDChGaQskkU0xRQBAMmFyoNqzFlzruoqqw1z9s4R4ek0MHNThxRT2SxjANbYFLzcH2QWWMDAp/kVMyNE9AgAIoKqWVQVLrab7abz7Jhy5TgwE7NjIoBd1znnHk9HMUCDz59vD8fDVzcv6rqAtm6/2e67TkRPp36co3fsvdteXH1490PbNiGEKc4Xm+7hcDwceyQCkctNV4fq/vEQvM+iZvrh05eu3f7+D98/PD4iwHd//GHT1k3TgGmOU1XtiB0Ano6HZnvhfAVYHKLmZM75pde5msqSc6qmnBaitCgYIDGVsXNVMy3cZFAoSnwqZTvIlGJUEQNVlbLg+sl4ylc5AWujG88LiwtOuVZPz1ua6xDdancGRsumw2clzLNhkTPiDmiwHBJYnxEBlzO5ErWW/AFxwUefg/p4zjaXRPWMiRWaCK4VfxnIdwgYKj/GOQRP2MwTD+M8z7OqFsQPzAiJmFJKpVt1GmdTBaS6aVmAHAFQzNliQiLvHSMGRwaYxLIXcmyTCZxR3+X9qoElZRVmt2areZhmDJ68I+eAqKqqrqouLnZJpAm+Cv7Uj4YbSXI8nk6E33792rHLOX/6/JmdQ4DPd/fjODKzAjim3WbXHw/7rmubGsF2m/bvf/u7vp9ELTgHYA+Hx3mevPfH03BztVcVIgJNc5xVJSV9eDyo6X5/pap395+nmK8uL8dpHMbeOc7TWLcbKrJTgCKZ2BXRKGIHBqpZl9TTJGV2zN5pXsY1i12W6geITKRMHeWcUoqSs4pITstocOGWPsGEz4J8ufOEQOuylzNgeW7hrKZ3JhQtRlh8KoIZnFuQ5ycv40IE+ATWL65vVRhbSiQ711SI+IT72/qEBbh66tsvD17fJpgV7VIDWPcuA7i7h8cXl7vah8izCSPzdtMh4jQnYfHgiLmqAiL1w9SDOWYfvCrPcybTTVsz0ZBSXdcxikj2zJuuEZFhnKY5TXNyzjFRybtyVitI4HI+IYuAgXMumxkii8qcBjftd7vdtkveXe33b1++OI6Dc3y13zl2D4fT6dRHScfTMCb5cHurWU7zlHPe1LUB5pya4Ku6mlM69KcUZxWbU2yrepymLNpt2kbyxXYzTdP9wwEM1OzXv/r5m5fXovrjhw/9ODimnOTL3UPK8g3xZnfx+eH++x/et93xuh+OD/fj0P8qi4lImhGZiFyoyHtyys6LZsrJl12jOSOAqgCgqjIW78kGqqLE3gBVcrFByTmLzNMoOYmKgiosY/gLYgMLfWNJDkzXOgNFtRTrstAAFoB2NaMFzCle8Mx2IwDQ83DSmm/YatxmaKBriVtwAFh75etbOhf7TzATLm7IbBlBO9t9yS8AziO25av0QRZ/utRPrj/1cZqK0qepwhzZcagqdkNNVU650CXnecpZidk7V6ux9yr57uHYT9FUg3dVFarAwyRZVdSISAEAiZkZ0TmHiIVejkWNSu3MNCvnuPTysmQ1lSyaUl2F0DZVFbKKY3bEaMjE2+2mbRvN8vv5h9M4xml+PB2naUazT6qO+XK/e3l9dbXfV1X17tOnH959GMZxGqcpxs1mc3Ox7do6xnT38LjtWlV7PPX/8i/+7Nc//yZJ/nJ39/s/vMtqwXERsDud+v/4u9+/fvPNTHA3zH/8dPtmnOd++N0//f7h0P+Lf/GXu92pcd774EPl62azu1yKJMkSah+qMqFR3F4pPBGBijwTqJosBZNIQftTimqaJUtKpcu5AqIFk8en0LuEzadypfjFUjTj+ohFDRPXqLo6UzrDkKuV6FK2r89cCCCLn1vLdizmpE91efG0uKpEPOWWa1qwUpFLurxWTctvnmcVzxZ+fofucOwRkZ3bbjehrjeAMaWYkndeJINBzjmmVAZ3QEUEY85d8ORDyhJPvWNOIkDUNaEKDgBENKoycxVIRZBwKTOXiqFcbQAQQkQiJSRgIlRRRqir0FbVNM0GVoUw0BgcAeAUo6l556qqIsLD6ThM09QP4zAeTv0cIwIQU11Vl7utqX748tkUvOP9pr29uz+eeud407XHcVK1u8cjItZVYObf/Opn37x9lVX7/jSOU9s20zSpalWHn3/z1ePhlFL+7rt/+vWv/+z1zXXvXR5Or66vQWZCPTw+quTBeQS4vLpxOTG7qttKnGGdDXI+lFOoJmBYpmZIlZgRwETLP5JzyjHnJJI155K72iKV94SFLzQ5WPpOVhL6c1xagRxcCpW1Vl6yghUtX/PjJ/LA8s3FQa6Gu/yIFnMtAfgpjUQAKmqMT4CnmgEtT7u8UTsH8lIuPWvCrVIrT19rgQ8A4C4udmYmIsd+qELYdM2ph6oK3rmc8zQnG0a30BwppVy0FOdpZudCVQXvCE2TpJTDflNVVYzJzBoXqiocjv04jEzkvXPOaZaY5QlaKDnVgtkSoTMEQyTmdrfpp/n21Hsam354PBx98HVdBWIi6odhjvHj7e2HT1/iNCPAZtN12K2QDc4x/fHDx92m27TtOM9meLHfFRIwGXji+2M/TXPbNsMw5ZxSSh8+fd51fcy5CuHNq+vjqf98+2gizru//LOfuVCJ5q3XVz//um1+8/B4C0D7XdvUVdd1VdUBkqo2XRdCVS6tr2qRhEhZMwiBmWp+DkOrgmVcrAELyJdzjDFFyVlywem0cMll1YkodcxSzdjzruMTxoxr4vjMFa1hdGXCwTlJffKAK3wL51D/pDuiazuyhLtzeSSqRnp2eM/B1vIiq09VtIXpjE/nxOA55o9wttq1HWauqsIcU1bLcwQDFQak66urOSbJqZqjqmAi7xiJYZxySmXpAswREL1z45xEJGdhpqqqADDnxIRzyqdhGueUk4TgHbsZ4notjBANsaDRSzWmQkQG1o/jdd7dXOxzzsfHw+HY18G/uNzXIUwx5SzjPKmB937bNBLCzdXFVy9fxpSmeQ5VuH849McTAoro4XgChMv97mLXfazC7eHYz1OcYxX8zdVFP8XPd/c5JQEL3n/4+GXTdduuIYK2DteXm4vt5jRM96fx5rq52F82bRecC1Xz6tXbaRpD1RFR1zXeVz7UWYS8D3UDCCKJfe1DU6aZJMUzplNOJROtJfJiZCnFeZ5yTinOWfLC7Suw8XqMAQHLHCkUKchFXn7BiZ5F1hX3gZWuX6x3VWWCBQJavO0y6AwGIEs9DmcjO2eZZ5s7HzMzW6ux5bsrkADnR5UibMlxVdfE5KliKz+n5+gVLpwqBXBfvtwX6K4KgRDLCJvkNKecY0pZnQ/OOWYGM8mLuLo6ZmIQ7U89EGYRJsxZ5zQwYfC+9IHGOYpoFoljTHOUp+0CS9auxXmUTTMZkAwAR+2/Tz++fHndOj8hSZ7NLGdFxNMwHIdBTV9eXKRpnqa5qcOc8vtPnzdt5Z0jtcttGxinOToCIqqrSrMchzGrff3VKwNQyXUIj4c+xlxX4WGacpYvtw93D4df/2JTRMlF01cvLt+++urz7e39f/zu8XBwRDlLyrcvbubLy6uUzYUgpmIIgM75qm7jPKZ5qrqtgcV5cL5iZjPIKSKUFbumqpqTEbNzCGgGOc4xznGeYpxFpFTutvjNglCVuLoMnVpxoIqIQLROL2HZ/XkGbQyfxM/PsXaxicV+Vj+p68rG59+FMxp1xrJWu7GnX129/9k/r8H8jGqd04EltVv94zn9XTz36pLPlo2lSLp/OLRNBUhZlByLmpo6orZpBrOqot2uVbWc0jRH0QBgCuYBJEvOqeQ+KiohnPpxTtkxbdraOY4xoYGYzTHGORaWl5oRoq3Y7FNubgaqiECqHjGLHA595R2IJJE5JT+ObgieMHjHzJu2Gab58mL/9tWN815UxxjfffycYu7aZk4JweZ5ZuK80cv9znl3Xe03m1ZE+3749OXu4dg7x8H7m+vLeYrvjp/3280U09bo4Xh88+r6+upaALJq3VSmmtIc4yhG//j77/D7P75586aqq+DcjDEAzIC82YWmi8OJ5yk0XZY5zgORK+PzKmIay9U3tSxJRdh5IATElOI4DnGeYs7FenC5m1qwQTUjpNV/FUcMJfIyGhHpMhp65rydgaTVONdBElsodcsD1yQBV+7egkQ+C/5P7lCfynA4t9SfQU0FSyhHyM5euCQNYmAL4HD+3urjn3lcRCwCz+XzuJzzMFNTV4QUY1K1ENz+4qJtG2Ym1N1mk7I8HI7kfFXXh8MRwBLxbHNK2cxMFh6dqG27pm6qlPIwziK5TNHTAh0rITgmM1DVYqnlzZZ1RmoGZRmI6q7pri8vMthpmmIWFT0+HEikaZqL7Wbb1Z65q6td11R1NU2zmKWY2rpJHO8eHvt+KNhKVVf7lKtQpZSHND8cjllknOZQ+b/+q18Tu3fv3t3dH+aUuqaOKZ/6sR+GaU43lxfHfvhyf59S8j7c3h8u99s5SYrx4XD6/t2H+8fji5c3F5vN9eVlcEFyStPgQx1cmKcBCR25lEQtRxNids6X1vpCvlvmmRSZUoyScwkvjssoMzAZAIGKSjYreBMCMpArK1TtfEtLHFomTc7R3tbYuth7yQfWNvqS9i1i5uXYlEJ29R3FVlbyGwKYLK5x0T5eQACzsoDG1mhNhnbuRuFakRlgoRCUiqPwVdYhu5KcnNNoOD8MwJlZnEYH5p0rWgSqELxXEQCLMQ/j1DT1brOZY5rnOE2zmbJXAxQdC26SJZ81VVSUEERV1VKSmFIxU1szKaRS1Zdc6Xy9lgyKCrFScn/q1UxyNsmaZeyzY2q69hTj4dOn2/uqCt4xA+ChH+7uHx4eDzkmBGDvysXt2padG6f5+z/+yLxsjClMqz//5c9evbh+9+HjtutE9OXNRds0x37YbjY5p4sdiOT3Hz7fH45NXTFxCH6Omcl9fHy4eziM0/xPv/99jLHfbhxgW7fBeVVNcSq8u3kYoKpNNUsyU5mn4AM7j4CGRkhnEDFN4zT2c5wJlIJDMC4SlQAm2cSySrQsksyQ2NkSuBdsbknxCtKJiMXLrpno+bqW0snW4I2rB3sGkp9L7OdpACwgFyCu436rokLJ0Jb/eSZsuy5OXvnNuAjrruG7hHRd84zFwJfS6uyKz87LYUGFUqpyruttacE/HE+OKcZ0PJ2mOXYxHvspptzWoW3rUHlHNIzzPcI0zRqFkGLK0zhpzv2i407OOchyrtPVQAxMDVTWDKN8FDhf2uINRDVOcYDBED1yW9UnHUz0dOoR4fLiotk0fYxzSpumxhHuHx4fjsftpiuicG3bjvNUlj/d3d3NKRFi0zRmqmKIuOmaH9+9/3J/3zVN1zZNU7+8vnCOQ1Vd7PZm+k/ffTdHff/5lgjfvLz2jkNgJP/4eArOgxoZSMy3n28t57ap9/ttXQWwMig0W8k0cyRya3RGydHMnHOEfHZRojnGKaVZJRWGPhESIROqKJhoTpKipBlVaQGmzFZl01L02KIYa7zQ5sH0+TDqkvHbOadcSylEOI/4LsTNNXdcy6IVxcSlsb7a8FLvrdXQalII5yUjT8nkGX1aC6sSNLW8h7WHUI4BPU9YAQDRVVUoW89MlUodENPj4fjier9p64fjaZrjbtshoedl5crFdlN5R3g8nbyKmBo7z46z5Kyy6dqmrQlJ1UQtx5whQ1EgkDPndb1F5Qroyj1bpNUWzAERxKSufExuzDPEeDyail7InoJ37ACwH+fb+/vdZvOLb79x3h37/vHhMAzTaRglppwTMHn2cV42iLJ3SPTjx89fv35FiA+P8a//6s8v99txnrfdtm0aM3375qthmP/ww485yeHYbzdtXfm23d09HKYYv3r1IqbkGLfbdretm9qlFEXEOVs6OqXmMwU2QCgk7JJpmZloJnJLLqdqKqYCmkspC8CgIKo5JU3TPPXTNKlmACAkYkNEIzV0WhpAuojIF20INSurpwDPFQwUJfZSbi9bFQCWSgB04evgYnDnGuicGCxhzbQoQeMZJTzDorbknraIvwMtNXjJMBcm6Yqwltuu5ZlLfxSXE2Nr6xPOR8AhQJG9iCk9PByy6ByjY3e931Dt6+CHcZpj3jTN3f3jaRhNtR/GTdfKwrsBZhLRnLKIFNLUpmvbtp3mKKLa1OWNapaMshSaz5gLiFhS0sL/MUSYiySONW0NhQyKVD5MznI89dM0b7q2auq7xwfLUp7l3fuPx74fpinO8zRFyQkVyBECDFNvIuxcCKG28Jjz61cvguMvdw83lxfbrjv2PRiEEMTADC4url9c0bsPH/7w/buPn28/fbl7/dVN225fXl89+Idv3ry5utyd+tN+t7u62O/3F03TMTM8u7trI7d4NTAs1Fheot7iXiTlmNI8T4PmCCCBUAkFUFVTitM8zTGKKpqAWRlkgqzeBSVRK7RxBMDiYpTIMS1usHBSy0/hp2o2WCLVApESLCZki7jf01dxgGfwH5ah5CUZhRV4R4RC7afVny5eeWXzrYo9oAhPNCszxDO7dC29znDX0ogwV1c+JhymGLOcTj0RBe+cd5/vDs6dALAKYbE9xMq7LGoG/TAhQlNXkiXGJBJTSipCRNM0/uEPP4QqOO+rKtQhVI4fAFJKwXyClMSYyQdvBiK5iLMvufQyLqiSNKt657pNpyLOcVkYXCj6GezUD8McvePgXRXCMExfhrv+1BfB0VIpm5klAwAi8t674Nn7Kcab66vLy4vH43HbNt5zP/QAVldVERA9nfrNZkfO/ebPfqkqp+Px05f7FGOM88vr/X7bbLfbFzfXx9Oprpu6akJVBx8KVAfPv5aKBRdqE5GBETECIaKq5JzSNE7DaRgOKBlNlAwMRHLKaY5pSsuwHq2VSMlCk0cgBWQDVDVEcs45ZkQUNUYzKFL1CGcwfJlORrVi17hCAmcbKpXKgrgv9m2ABFqmSZ+sfCnRC7BQyCZlBmABws7wPuAa4RGL7hWU8aiSDyMY8FJAL2+3GPg5IzAAZ0jEzjlNKY0pEbNqBYBjnszMOb/fdo758XAcx8kH75xjxL4fYozOORd8VnXqYozFf8SYRWWc5q5r99sNIk0pOuamaUacsxqD1HW13+1EpO/7mBIIruSpBYBDNTM5HY5FWGae5mUAHEALImWJRXLE6H2Y0xzjNE3lUBZ421YADgm9dyEE7zlLQqS2ae4eHtGgl3Ga59cvrq+vLh+Px4uL6+1mezjczfOYstvvL3/182/jNLbN+8dTXyCI3c2NITd123Z7ACPi0ujLOS83dVn+fTbLsscBl5mPp7TNUpofD7e3t58kjpAjqjgw1JzFprJ52kAAkZgJiR2wA3Jski0hKnPhvSPzqrhZcgRaQCiiMv1WOA+LEdDqHBcLWRuU5dqKLFjAcsitNIBWsGhFjojwJ2b0VKcvhlf+FITnjCzo2u4ssZtwmUQ/Aw8r/Ln2HswAwJUqpq4bQxz7QWOa5rnsCTbCbdfmXMUYqyqI6BzjNEUmSCnnnLNoXdelSxRzjlmKClfpMLHjKaby5bgMLmdiUtOU8zhOTVtvNpspzillSWmeY2k5AJGYMVoWmGNk50SfWhBqCgKFCI2ISXJCLnvEbf0qRo4AzAxExFRXfo5pnueqqt5/+Nh1m66p1ezt6xfMZAbOeSJMKW67/eF0PPWfr69e7PeXqe1Cu/ub//C3265pmrqqKnIVs/fOWyljVwlZK3xXeMJ3ibjIJgMRLNW3AnAJ+ExODWOKp4fHNE8oCSRbFgETRSAUIGXnmBw7YGVnzgGgslNmR2zeO2ZHVNZSo5qgQtn6DQaiBgDFDkqeVGyjEJ0WV1lo/kvpsoA+a7fp3M2Bc0JWCBVl+zzC8wnntY+12nGxxMX4DIpYry47P+BcTj0dl3VZyvmZiz92m+1mGMbgXV373abtx2nsRzNj79i5rmsN8dOX+xBcCKHt2hxjylKchKj1w1BOmveubZuUUk4JkIo+o5l1bTPOpKJoggjBOUJUUSYMzicUyskxo0HOIks9alBwXVRMqUiAr5XHCqcgEhMTmYGUr6KRuyLNpcvimJ3juqpSSnNKRpSLDDT7fhic4/1u808/fLi+GF6/vL5/uD/54367HafheDzst1vv64fD8XJ/9c3bt1Vdh1BhMS0kMyNiRDaRdTYI1y8oeQUgItHTfYClE5NklhTH8SQiTdXmNinQPPQx55yWvrshKRqiOmbvzcgoWxUMiUmNSJ0zREMEpmWrYVnnp1IolUtqsTCTDEUXUJwJrbQ0AQqdTBdjWqr2pUBZYKIzBL/8t0D/+GRb68+xwE1rG+n8W2t/vpwJwnPjdkVs16przd/h7IZNzeWcg3dNXTvPAKTwOI0TiAXvEanw5It8vZltKs9NdewHxOrUj9M05Tkuu7+c224753x/PB1Px5QTR8eurFxlJhYil5JlSWZECITTVFSbivSIOeaVPLMc3DKbz6p1FRAwznNSQUDHjr3zzi3D+yKmiiXpXvsiZZ0XM4vY8dQjknO8CF0RH09HU+u23cfPXx6Pp6EfwaypQxZ5+/p1XTUA+PhwD8j9NKvq26/eYMkg2RGxAZZhYCyrjvGcQS0V3QKcreopiIiFSwyQ8jScDsfj4zCcSvkfQiVSpG/IKKIsGm4lAOecUxbDDMijS84xO0fEwYeUXF1VWgVRVRe8d6svt3NvxgAJQeFp9ZGuE54GZdoTABbyUSlnVxTgGTj67IytBrloTZbwX6buyaiwLM4oTXmiUi+qAZ/j+YKDL667nOpl09cZDTVTMDcM4+V+d3mxH6bp0I8AEOp6HkYRIbY8ZzFlwipUwPzw8Ng2NSzPguycD5aHcZqmk0o1VdeXl+2my6ZxmmKKNFMIgYl16d4zB984NjNQjbnMziuY5YV3boBIRCKLoiQgECIQIyAHj+rK+u4SrpjZQJkpxlTaWk/HVoXJkXOBnUimBc5TNJCUsgp7j4D39w+FVgIAbVNnydOU/uLPfl5X9T/8/vsXVxddt5Wc5mm4uHyhgGZISLZQLSyLQEkkzplTwcdwEbl7MtBS7ZqJ5HEaHw8P0zRIWnOglGLMomjkAIyxiJRp0dMp4AgSZNGYiDiFEJiJjRVMDQiZCEWtnINyBdbeh5mZ5nOfU3Hl4S4AT0HG1ppmgTmXuaU1Pq9/oXNnaHGm628uHxlMQdath7ikp0tqscKquHbrF+9Lq/Nex51/0iZ1bV23TV2Gg2vvQURCdoRV8N4HVRnH6XTqcxqqugohlAaQijBzW9fZ+5Qz5wyIaY6fPn6qqsoHj8SQc8556HvnPRGBllm2SsxATSTHlJJkWxp5Aqti2RIW194SMzsiAUMjzwucR0TddrPpmqEfHh+PAKl8BFyPLQAyc9e1Taicd4QQY4opDf2QUyZEx06ylCGklNIwTqqKCN/98V0InhC+3B1ykr/6ixdd10hOMcW2u4BlldGCv5TUU02Zym5EMCvq+7S60dU4kYrcvHceF5AX1ExMk+gU8xSzKx0OAgUC5lLiOEQHAMSGTMSAWOAR74N3ntGZgUpWAiQnsDQwF3LQk5kiFEp4wSuhvJt1+LiEeIIiTrhy6Qvw9CdPhavBLvG4YOjwfBhkSQ7OOaYVrAuBVO1MeVk9CRhaliUxXds1CwiFAO50Og3jVFWhpGuA4IMn5pxzSqMPjp1j5yRLjFFVYyQEqOtAaCKSVcvAmkiWLPM8j+M0znPZRpTmmHN2zjFSUSZiJkRi76qqUgBOCcg0ZStjuGC2rLJ6ymxKsKtCUK8i6gCRuW6qtmmc91VTXyD1Y5jXKr74g65rry8v2bGqbruOGdVgHGfv3IFOOUuZKwcDEzsdTz74/W47x/jp9g4Ad9tOlU7DPE39zc01AJqh5BxCXZCgM+RpsHTY0bC4f1ySv2XestxGJFpq0zINl6OUoWRywQOYOWZYKjwzMzBrVKSUzwAA66aaEh/VRCQCZBEimh3XIYQQmBlpWaRbxgFWi1m9KqyHo8hKWonVpcO8QHLPQvozJVsAU80ATFTCMaxaOgvsClC6LeXt4dIseHqGwlx7KreeJHDO2Wtpi0KR/St+1h1PAxOlVNd1oIR1XbdN/Xg4ZREVTSKIGOqKgKZpHMdpnmd23OWmaZrC09l2Tds2p364u39ERPbujOIyMTKpyBCnsv7HETtmcux2O+9YqjpzFoNxGJd+nFkWwTUyAqD3vttunPcxxpyFwAxMVU/9oCervKur0HQtgKFaKZUMoGqqTdc1oRrGQVRR6auXNynLp+C6bXc69sMwlfbPlNLh8egrH95+dRrldOqnYazrutl0VxcX7z9+2e8urm++YhcQIOdYIratze51P5CtIX5B6UusffKhBoAgItM8HE/HYZhMs2d2zokCsqKCSjZAIvKOiYhpDdCqWXROOueckqSkZaifHXt23vlQOQAEJKfmPAIgE6rqUtEXGUe0M8t9yYJWMoQuNCV8yl9XJmTxEPhk5nYmSZfIvSp2L2nl2fqWMH6ePF4h0jUleMKVCjRTsl4pqmx6dtfgYoyFPDuNIzG/uHZNU1dVDYDGOsfEjJcX+7ZppmmepvH+/nEch5zySQfnuKlrBX/sx9MwZlVyblNVOeXCCmXvETHlPI5jnOeUEiLmlDWlTdddXF5cEt3dP07TjFjU/J8CQ/nQzOyrOlRB1nwsiw7zlFJm4q5ti9txhJuuDcxmFkplxuyD2293zNdgNs1z1zRf7u69921dN3UlYo+Ph3mamCnGrFkOh1OOiRD7YUw5i9nrVy+ato0pppR8aFyockoqWVWLCvOStC2hcyVglJoDCYmw/FmRxSxpmqZpGud50pwyEzMjgGdCY2VCAMfkHTMioRGqqmQpUgk2RxmneZpTUkMi77hyvmsbH7rF+NY3Y7bARes0CBVDNANaS8lzwrrkhmsyivTUOoJnDm+trZfwVip9Lfsd1rGTUhKVfSlECGtdf75Q53xjQbbsHNwXX6uLR14AcWeqQCg5IyEjz3HGExX1wGma1KzyjWQZ57mq3HZ7fXmxu729/3x7P/QDmE1hqurKee+8++rFjfcOAI7H0+HYRxEHgER1CLvNJub0+fZuGkdVaZrGAMZ+IMdF/kpXbwRqsAKZzByqwAj9sTfQlCTFGQAIIDCHqvKeYVHzQDLwTN57JCoMnKaqEGyepqap37x6iQgpp5xzjOnFzY3m7BnnubnYb6c5fv5y//n2vqqqEKpxnlXUIXaBX7140XU7YlfKSheCZsqQACBLWsH5pewsi2myZAfOsXtyKsVPIBCiqmURFUmiKWVCgDIpgcBEROgIVvKxJs1lNE2WYZCUYoxziiKGaHW9aZqubdqm9j5474nZMRNS8VNJFBEcczG9BWrEMsX5NDEHS3WyVtxPE55WcgpbqylY4UnEFapcLLeUtlIaEfQshVgvzrn0WZ5HVns/5xKm5b6X5sGyZcVpFkV0zvu6IqTDcbh/OJYyuSBH3jkwSbMeHw/eu8uL3cuXN8j88cPnaZrmGHNKoQreezSrm1pEnONQBTUlYjAz1aqubjZXSPTly21/OhV4KIswYt00oQrjOK54w/IxHFHwftO2zrlhGKdhLBVVoTUG7x1RkUuAnCvvqxBCCNM0IlLXdV3XMNE0z/Mc1bRrm65tX1xfVp6dr6qq+nJ35xjJuYvd7uHxUNfh9vYRifb7bZmrRDQi9M6FqsaV/6U5mRkzgREbq503Fa29uaVaV5HM4IAYgGDBtJGQQwhV8EsdgyQqKcWUpTgSzxgdES40ZQQtLKEi4EpgnjARAFBVN9eX+6vLy65r6yoQO8eMS0wH0Gc7txft+iVNVls2/gkAAwAUKjSoWqmTnhzbiggsQX/NX5dLYWZarH11jUimKmvgXrzgs448IJTXWoGwdRh55Vkv4JSBrWssXCpnkWIScd4557qma5tWVUSyrZV1TGmcZxU5nfrdbnt9dQFmX77cl0gXU5pjmubYzrEKgZkckyn64FW0H8cxxg+fPhFQ1zRFZX2aJnbOORdUDWzZBFj6WgAIRs6Ftokip2FIMWph7gEQEhAZQFKriGrvfPBqdup7s4UC1o+jqrZNNc+RiB4PJwO4vkhVVbdtJ6qq0g+9qe27jhGPp1PlfV1X7Fzwvu8Hx1RVNTuva1Ve9GIRUFUk5wWct+X2LJo0ZT0Nu2X5UWksFXkDREQUyVkkppyyppyhCCESlQEFM9UEEyzlFyM4QiYEYkBgIiarA3vfuarebrcXu31T184xlm7S2iQoyh9rzAcRMUJCgpVTVMBzpsXmVBUBDQt+hmuGWlJDKDXeikYt7nRtiJRyTp87VFxnoJ8wPztDK2CmgMtIdHlWXeP+uV+1Ml4BAEprfUlmEdAxe+9TSkmSiuYspfh3TMGxmn78fHc69TfXV6pWVpoSczkxkvPpeBqY6rpGIiaqEZHRAIa+Tyk5dtc3N7s9jcMIZkRUhVCc5ZKp2DLwh0xMbGJjnHLZnarGRIwEjN57JvSePfESIAlNCxk5jcNoiNeXFzFGJAzB13Wlau8/fqmCB4C2qXbbXV3V9/ePP7z/xzcvr8tNuNhtx2l++9WrLHkcxhB8FSrnvAEsx7WEJDWRnKVMXUqpLXipaNAWZSxaoPD1sq9RTCTnQn5bq3oDQueY0HK24hRK+1TAMqJzzKxIXHzUZts1TVe3XdNsmNgARK103WHFtmzx6LYAQFjE5stWkKWTtLxyGRp+FnlXZZ3nRmaotLI/zUpEWEqcxSjPqNDaKQV4srznF2F9agPAkr+u8PCaBz+lHWYG4JquNTMmBgQkjCmZQRUCGJhomTrJEhM+QQ6H0zBMMyM758ocZhWCimYRx7zdds7xMEXnnJg4dteXF3NM4zSaQRUYknG3IQTHTEybrgXV27uHRcC/pDvEQCR54d2pmiOq6trMFDQE31Y1EBWjAUQ2E8RDjNM8qSgifkiprarNtvPeV1U1DFPOWQ3meXbOp5y7pn5xsZWcfv+HH0vNXIUQk1xc7HebTYqxbaqqqgpSAQCqYklK/00k55xNZalGS/FROptES9JmRcjWFJQIkDCnPMd5mKYpJjNzXGyFAMAxxag5WUqiIqBSArT3DsuoFmIIoa6but003ab0uuaUSyuOEN2yFIWR2GwVU16rt5I5rdUPIQCcx0oLE6kc86LcXswUccF6V31aW0eS1gpqrcbObSsoYuWGAMtQgC2WVox1IR8uQrmrehmeSSa2NEp1ZUUBOO89GjjnRBUQnOOyAfviYi8q93f30zQXvFBVJWfvXQh+nqQAwq6cfTAlc94XHlOZD4wpxZybGqsQ1Gn2vq1rAGDm64vtHNPd4+M0joExeD53BQ2AiOq2qdqNSpJBHTtw3rFDQgKoHTNREiEzds57n2IcYoICkZgCuyx5enione+H7fHUI5GpXWy3vGmrur7Y7w6nU07p4eGh6DOOczycTkm0qcLxeCTiU9+XzNI7t6ZlRXRDRUUkiyQwO7c0ccXt1wTLcBm2XMBtUxXN8zyfTofj6TBNk2fk4IJjIlQmBJ3mFMVkTmaKCHUVgiNfusXOhaqu267bdCE0YDjOsZ/GApYRQ+V929SVeS5COrbgU2fnhYigi8hCSaiLuxUtK/LQAAqlgdcGREHH1kFkNLBz1X8GgYvpPqt2llC8+kpb6qTzFuc1A3jiJp6FzVY28xktRURXVVXOuWkrIhrG2cxc8Ju23W46M/XO9f1wOBxTSrTsldK+H4vYFTOHEABBshhYVdXsuOxUMFNGaOo6eBdTTDHWwVd11TYVAHZN049TTDFOU8r58nKPf/zRkhaYm733oUIiSUrO1XUNZR5a1FSkrNIgMoQ4p2kcUxYog9VgubSuRQzAN/T4eHg8nqrgzSzn/On+7qub67sqTCldbDdTnNnxy1c3CDCneH9/qCp/fbkfpth1m03XFEJCqc2hcJeWFa6GAEVMmcjR+U6t0anYZXERJWcpGfYwDqfTqe+HYRiDI4SKqWJiJqq9l7Y2kQkkpVwElACM0AJD8BQCegZSmcdhmPNpGMseFeewCgwMmjkvK0aX3cl/stoLDASFkBQAlzZBUc43QaRnB2ydVILzx1JVe95jx2Wnk1mpulYlvYXdaQBl6mqZiDz3mFafuuSwazBfsodSjZ1fBQzcfr83zd65lAXnhACXu+1u2wGBGXnvmqYuxA4wkJTGcSqfIaakoqYaqsCOmcgxVcGbOTFIMZqoGey23a5rDsNoprvtlomHYTicTjnn3abbb1tVu9xvf/bN19//4YcSetA5ruumaUNd5ZQgSxZhIMeWhQpqLSlLzlb2GqkhAJbtOUspBT6EJMoEdVWpKjGdTifn3f3D4fb+4c2rF3cim7bedt0U0+Vuy4QpppRyU1ebTVdVvGmb7bYtTQFmlsJaAss5F+9C5MyMaFkqe3Yr63V/lk2BmRbZkDnnWPRw1HSOyfFCzchqCOYYPXPZzEmgKiqE6ha37fM85jRFvevjnLIVAqggKpiKqqhktXPBsyy3KD6Scdl/XogjxfEr0ULFQxJEJHRET+gYPstDYdEQXz0gPCeL2ApqrqAbFDirLIFeGSnLA21F9Q20KE8U1kXpmp4helMDAueZfN2WOayri13BgcfpbrvtLi/2L64vd5sup/THD59jTKfj4fGRzCyLDv2gqnUd6roqSbMP3gCHfkgpzTGpaE65rcNXLy598HeHUxVC27bf//h+ntNm23775qZr24fD8eF4MoC6rlJMyNy0zdXl5W6z7cfx8Pg4xZTjrCKjiuZsa5timV5acnIzBTBBRCRkYgQUVXIMBC+vb64v9uW6DOPonDOAcY5EtNu4oR+2XUvsri53wzAZyPXl1W7bimRmMtUsUc0hoKioSokeJcU692bWlsgaAZc7eq6SDABSzvM898OUUkYERkCElHIpTHJKKSXJsuKV5lBBxRQkGZkKaB/jGGXKEAUNzDnHTLVDLmurwUTVs8Kic3jev2oAoFj2WayZcoEtzRCBkUqWSEAGkBHLB3ySOT1bGBaCvRGCyaq2gFhWJC4Z5EqJtDUrhVU3qlyeJROFFYGCFTpdW6ELpI8Iqm4YBvY+54ymTVNf7rfTFE/DAADOuU1bi+r98dTUgRGOJ+y6lhCO/dDUAQm994u5gOWcAVBUc1aE5Soc+/Hv/uN3dRUuL3ei+eFw3Gy64/ET4+by8qLttnMW5/zt7f2CZRC2TXNzdRXqKkp2jst1AijZn5oIwJPw3zIys+xEI1vYT+g8Bx9KzgAAPvhffPPm8dSf+rEKXlVD8Mdj/+7D59Mw1nVV1/XjYSCCUz+1zenm6pLYqaGa5pwQpOjDSE5lWXwxvTJBIWKAsJZTCE9Ts08VLgOC6uF0fHh8HMbRMXpytmx8N1NNKaeSAjJ7RgITQ2L0rmxN0WGYslq2wv6h4F1VcVfXPnjvGNk5dmKoBkxlXLgEFF1bkWVr3mo6q6vLGYAXGMzMSnOfzr0HW7n3sJZ9z5qU8LSgttjb8mC1Yr4L/LomrE8Zgy1lOwAA4wLJLWnnuco3AwB3d/+w6TZ1UyFSFu28u9l27dCkGA/HU6mKzFRFiHC36fp+VFMfqpJalFXe7JiZ4xxjEgRyjpwLRNS0bRHo+nz7cDgNL24uQqhev3q57br7x+P7D59fvLBt12mjl/vtl89f2LHfbC6/enV1c01IKaY4TvMc8zyLyOIP1kjxFDsQC4Nsyf+ICoQxp8jOVRjGOJ+G8f2nL01bf/Xyep5jGQkZpul0OmXRlHIV9NAPY9+/+eplynLs+8v9pffBlk2F8RzwVGXN64u/hjLHLuukh63wYBmjLc5DVIjIsYNSPs6as6uzD56dK9W+sYHzbCulgwAITAxMdU5SVoUTZCZ0zgcOm9pt2hCCRyIpI8tAWVF0aQqnnM/+HlciSzFTVaJis2vtwsxY8Jx1Azetqq5gcEZZl9q+2Fk5rCsmtQaQJX1d2vFrXnvOD0qHae3+rzmnlbwWzhSAAgE4RIwpMaP3PKf8+HiQ3KkaOvd4ON3fPVzstyEEBdtvWu8dO5YsOethmr1j7z0i+uB3m/Z4Gvq+B6BpNmZXVSGlHOdY7KYfhvoYXr9qHdmbV1fb7ebdpy9ZPn395vW267569fLz3cPx1Fd13Xjf1cGHahz6j2hzjDEWreEzjF/QNWLPBTMmIlMpooMqYiKi5h07A0Qkg9NpyFmuL7aaJKtuu3aSVFfe8/7T7f3j8RRCeHl9daxCSokRqhDmOHehcezmec650OaLyt+512K69GOYtJACFHGhjBGWaaSlri1Oixm99545qUrOmZAZUYAJ0TSJgCktjGcgMFQVkZg1GywUIjMkrIkdagBFzRINwLJhQmbnEZ0ZpKzTnOZ5nqa5GMfCGyd2jokZkFxZH+icqhYBVyIEoLN3LB+yMAxU4Tx2b0tBsxhSacevQFKh7q92CWVIfw36pa5XKLgBLV53wT9X77OKmxgAmquaJqV0OA2qaip1FSTrpmu2bcfE4zhOc5xTUrUUZ88MAHNMYsbMxBScA4QQfEqZHG93u5TSME0ppeAdmA7DmESmcWy7ZrvtRHWc5inGTdt2Tf3p9q5pmov9/vrq+uu3r999+ARMofJ1XV1fXV9fXbndxvi3h8+c+n6eJhEpnsuHyjHbKjKMgFkFwUA0RpBShaQ8i6YUTye32W3die/u7pu63m/boW8Ld+bVzeV+t5li3nTdHKdf//LbYZqiSDkEw9B37YaZNceYo4oyoxHTmkURAREvgt/ETA4KOYCWYdqCg5aStyT6wVMVWDSDFehOsJBcEZkwJpslg6krZsrMzlceagQzUEBH4GmZaYpqGrNaVjMgBjJnRIwiNk5pGMd+mPphlJQBrMwXOOaq8iEEdhzKDKRzxKyqwTtjXpC1QouHwpVHXuc0114RnuuedWPzMi1ccP6lolqxzhKun0DUdVxOn1WSxSJlzecXFMzAtU0DdfV4PPVDjwop5XGKt85tu3p/sa+rKqdU9qjePx4QoK6qzaa9vtzfml3utm3XTNNUenebribCH999FBHnXFZrm4adi/MsOeck4zTvdtvjaXCOHKIjAKQ/vnv/9ds3N69fzznNcxzH2aY4joOjFzeXV5f7i6tu8x9++49fPnw8HQ5xHDVnAHDeNXXdVMHMspqKOCYDO51OAgCZVLKplMS/ClUc43GeEZEc3z9WF9tNEmnbpvIupkzMnum2H+I8v371YpynOaamcobSDz2W/cEGZpbFQJSf5jYL/5KXze8/BfQWwOlZyVLgd8ccfCBUhwAAOWsZYxMAYvKrriwxe0fOccWEoFDm1s1ETZGQyJCSARM7Ju88MhsyIuWc5jiP03Tqh1M/pHkq8bocIu+9d857R0zeOb8sPfdVVdVVVVUBQIkcEZ3ppMuuxRW0KslWaUrpOfVZu6ylT7mw6kzLaJqt4MaSpCOWPTq0WnlJBmytLBhpGZqL88zOeeeD95IyMYlpnud5nh+PQ1WFtm32u00IPsXkvNt03W7blvp9t9s0dXU4nD59vq2bum0qM1C1qqqc45ur/Xa7yWIq0m66z1/u7u7uL7bdy5dXMemU0qaprrftw3H4fHt7c/PCeb/dbcVs6vsvnz7uu2az2Ww3u7/61Z81df0fu/b9Dz88PDzmafaEvq6NCiccm8AhhHIMtt2mrrKUPwCqxgDTNI/z7J2rQpVFjqd+GCdAeEn0+e4wTVMIXkT7YVgmNAkeHk9vXr2s64a5SE1JycsK/aWYGjM5ZMJlCKRgpEwOyia4Qr0lOkMzi1cycwx1IAACMFVLOUczptJ5J+fJOy6rugkNzUiTqZYOpQAoEDISOWQXgg+LPiYhUTZOOReGSE4pzlOcxnmeUaVM0TEzMlNx8lwmI533vqpC29Rz3bRNs2lbH0yJjblIQiwZZylo1rrz3EfiYlhqpR6Sxa0uWJusZq1rQ2lpMsGZSrKSa5/lb1r6+4DueDwhESMwsWvctm0FIOUiO5z7YYwxxhjbtmna2jMHR455Ur25vmya6njqvWNAHMYpHE9zTOM0Vd4H5yRLCOGXr199ur33pz6nfOqHz3eP37x99e3bF5++3B5Pp21X13X98PD423/87RCn4MPlfp9i8sTHx8Px4b4Nzabbvn756svD7Xw6BaJxnsFMY0opiTko/e2iCWaGAMxUheCCr9pWpvnxcDyqkBQyvzla7o9jqrwv7uHh8TiM8xRnRjyehqv9lphSlm9ef1XX9TkKFW9ScileSh+ErAbiEFUZCpy/gNNnhKlwiCyLmGRTUYPKMyEpWBbNKWdREVMxIlRzxFx7xwRQyHlZRFKRVzVidr5yyExNFXZt40MooLkYgFjKBAAIKlkkZclJcwaVUrcbrPMoi4myY3bBzXOwMl9F5BwbgA/IRCIFHi3EDlQDybl8PCIs43dSahxYkgKzsjF3CdxSli3COa+FonTG5xbXOq+9QgV2/pGZuXEYAJGYQgie3TDNarrdbrY3V4dTfzgc52lOMSJAVQUkvH14TDm/uLnqNpsY5+1207WN9+724RhT7vshjhMBsOOH07C7SCH4n33z5re/+66qw+HUG8D7j7evXr64udzPMY7j5MjGcfzhhx/atlNCQuy6hgDv7h5MNQT/6tXby93u9c2Lx9vbY99LSnmeQbToOjvvyHFgbupqTpnBCLBq6zevXjh2X77cMeH11X4cp34YPLskeY6JEczg9vauqirn3P39PSI1dX0/nNq6lZy3281ut51S9N4T01LWApDZuTsPBqqCCCLFUz6BjrZ2QWE1U0JidlVV1VVIKaJpYUMyITpCsIJSSFa1TEgelsoJc0oxDSklEQNgdlWNVeVqh23FPriq8kQIarNoEiO0lLNkycXsRVXFRNZeF6KqFE4goiILUc5sat67kKosOcZExLR+TEQsi6zxzPVcNywv2czyuHMHH/AZc54WT7oQmYlWqGrZMbLSROCpcaVLegSA4BAJCb3ziNiPI5h1bVv2Zu26jtn1x2PKGZBijGAuJQnep5Rvbx9Siiml66vLqq62G4kxHQ6HpmtD8DHlNM+3n2+/a8Lbr17udtuPX+6r4DdtfXc4PRxOlxcXVQjTOBZN4X6cRQyZRPXF9bUPPuU8zfHx8WG7u9hu9t++fptzDN7fO344nXKWGqBI3Q7DoBBdCGKmgMwURIfTgIhlXoCYri52++1WVac5TvMcvD8cj48Pj23bXlxcIGJKkdgR8uPxgI6TCjNt2rr2IVA4RyWAIvDLS5GEi6oblcGeEqnUgIsM1TJxVrIrIvTe+xBCCDmnMx0MAIr/K6BjykaQQRTAQAQkx5SnLGU1cPBGjJK9SgbLYNmUDagkuWiGpqZaCJAp5aLOvlKiYSnenpYciKqiQKmAVFVWPcKy+GaF9E0AHRMRmloRn4BlkGON13YWMoFnlNAClC4VUklyFBCXtc+2etelNFrnR8r/GQI6XwcAIGbnHDtnqqZ2PJ6yyPXlbu+dSsZproIHw2GMTPR4ON3ePRpCCN4xl/H2yntE7DaditZ19fHj55zyoR++fHkIobq82P3s69cPjwdTy2a3d/ffvn2bLvafb+9ubx/6YR7nuQqx67o5p007brt267oQguU8DaftZn95+eLPQqib7vu20/cfjg8PMs6FA2pmQ8o6TioCiOT9cRjv7x9LJOo2m6YKpbk0DsOifQI2z7OKxhinadZlcasgsyENw9jVNSGNU9x0mRMqETOvedgCvkJRpyPjJXxhqUNwaTItzFwod0MlpZhzLuMoomKS5yjDNEvKK4Vy8RyaJBGoWhJRkUJOMTMiqEUNyPvkfHRjJHKESOSyLbRTQkOErGpgxX+W5I/PZwhsAXnKxyAkdrSo1xqYZpGU85KflEX2RLAoZ67wvgEuumKll1foJvYUOgrAtLTdbQWsy4UyWqceoDSl7Jwi4DpEurhZV3QZGNkAVHWKMcZYBV9VYZxiQcjalolpTqmsRCrn0nlXhdC1TRbxzg/jBGb77abwmolwf7G72O+qJjgmEdnvti+uL999/BxjOg3j4fD48ubm8Xj68d2n4/GUUp7GqdzPh4fH/eUFImiMh8MRAJq6vX755sXli227vby4cm3zH/727x8fjjnmMjBUSAnMpGpxGBBwJHREoa58XdGEVFVTjKdTryLe+1M/ZjVDnFN+OJ00ZRERyuyYEVTy8XjabDomkiyZkI1XqJnKTA8qANhZN6SwnEiE2T2bqi1egUpDf44xxjnlbAaO2EiL85mTTHNcFIDBGNEzBbduvipebuUKC6AiipmKxBhHYgB0voBfCGXQouz/hGXAlxFhHYsuGQcTQZk/sXUAuTQ7DHJWp1r6SYgIzCbCAGVG2Vby3mqRZ/BywaHWLtOThN1aDOnZPMEgi66ud5F5MgNFKIt412lPUDCHZo6dc26a52mezdQ5Zu+K5pZ3DAApSco5WEEASUQNLHiHAMM4dm3T1LWI9MP48vpCzT58uq2b5s1XN23X7rcbVfvjD++7Tfvt29cXu+3d/cN+24hkNQvOGeAcY5qjqkqS0FQxp83n21AFU5VNF1Os6qYK1f76Zddtfl7XL65uXlxd/3f/w//j/R++xwTMDAbeMRKlnBAcmJFjDj6Kfvly54O/ubz0zAUR1Cwpp8qHnCKYaYxFb2xOsfNt1zTZ1HknotMc67pwTRacE4nMsNC0zYCJHLOacWkdFTdYeoLPaiRcun4mlj0B1U4ylqctNYtzLqdU/B0wMoKpeUeO0AC9Z1VX8FXvKHhGIDFMApANBWsHXApn1Zg1xmSSQZXAmACZzIxLMki0IOQlA8bSrSmsYU0irmi1qoFZFilbfYnQlMonRESwUiyspbgVd1mm5J7mjHnBbhefWnrFZcSq4BoLy3+1Wjgrh69oKyI6FUkpMVFb18xsZpXzxIwIcZ6D7+qqApvMrEwbIlFdhQJ7qVpK+XQaVVQBNl272XT9MKScry92bdPkJF9u76sqPB5OADDNcds1Kc3brg2e+1P/3Q/v+34owI2IxDir5FylL19ut7utqngmZjw+Pn5pPnKo9hfXdagqH/71X/5L7/x/+z/8j+9+/wfpe43zlBOxI+fIsqpYsphFzbxzdVWJSI6JmBxzVmFmIKtDJSLsuPQr0xy9d03beu9e3lxdXV7UdX1GAgUEoAhkoiqQWrFXOM/TFHIGmKrxsgeqXPlSk6BnrrwfmSVnXcIYIiJTGXgBoqfGtJqBISMQsQAqY1F+V0UzMqCshMaVq5grI1ZEE4hJxzmOc0opS15asqVmh0JtWXyoYdlju5CqSwZb2HIqqlky5YJJgRXxRirJI4oKIYKVDW24wqAABaVHW/v4RebgCdXHpRFma5vDZNG6XzT3cGm/L/ZdcAMnYpjyBNA0zcV+t+0aUe37YZrmcZKmCqPpHBMilgaDmbVtc3WxnebYD9OpHwhximmcpk3X3d4/Hk99OWgiYqpTTOM8A2IWAdPLi0tEmOb5y/3h/v7xh/efU8yExMxF2EPA8jSNw7DdbT27VXTTDodHHz5WoWq3F8zUtdt/+eu/AID/qa6///6P0+lkScoOVslJRbSkRwgK4KaoIsH7y8tLFbl/ePDOtU09V5WBtW0bQnCOh2FUkbZtmrqqq1AFf7HdOO/HccwplsLVIXrHziEXljygKqgYOjiDfcVSCQUQEFdhWwAids4D0BxlTnmc0zRHyQX21sLsoDKqU5TTJCdznm1tOoIYAAExsAIahZJKEgGwiKacj/30eBxPw9xPMeeMYExF8Q4IkLFMrBQneE5OIOsSAVQsZaUsRLlw82BJO4rntVULYrEkfd5kX7nJBTXlc/WOiCvzb4kqixkuoiVSvOw54KyggKoRmRNVVGW1nFLOeY6pwNCErKYfPn6JOXdte3190TZVGQgvcR8ARTITEnHsx2EYS8ibpjmL6KRVXe02HTEP09i1NTNPU5znJCIPj6eY8u+/++HUj8TkwK/NCairWk1Pfd8dT69eXjvvUsrTPIUQ7u9v67qu6oZdS4Sbzf6vf/NXu83mP/zHf/jdP313eDyMj8dxnrzjFFNOidi1bRu8F8nDlASwypkAtrtdW9XOk6myc5cXu912o6rzHMdprKtwud/VdV2CoZnVdZOZYpxzkR0yFikkUcgAZtHMiJDYOSQAKIkQEtPTyKNpwT+N1CyJjnOa5jTFmGKOsUhUWfkPIhAgM3lHCZSX51hLKNKsmhRb4OBDP6akAyGA6hjTcZiPx9PpNEzjVAi7YKXiXmaV8Lm+kkFxq4Rn2P1JwlJUEUnJcEXaiWjZu0aA5/K9IJq2Up/WLxErsSAvMX0ZIzyPdzACPBtFWrNYK52Bwo8jBWcqkiEjeO9yln6cJOdxnGOcY0oIcLHf3dxcXu53zFhCUzGlrg59TzFnQk051XXlmad5VlMTTWbznNqXTaiC907Xaax+mO4eT/ePx3mOwzghqCOXzZwr6xyEENqmnXM+HI7bTeO9c8xf7u6nOe62u+3mmGIMdYPEnvBie/EXv2xeXb14/erl3/7933/84X0YpjTNRIOvKt+2dVUVTKPIJ2Uzk4yio1mLVfCha1tmRsBt13318kVKkRDrqmqaJoTKAB8Oh9OpF0Mt60cyMBMjAljKgoDOeUA085JzKoU8Cy2Rs3iKJX4BQBYxUVWNMZsYQREIMUBgxy67Ocac85wTZYiJvRMm0rXqKjFymqkf4zCnOYn3wxKzAVK2aY7jOKYYJSXJmVau+1KLEBLAcgNxUaa1VYhgsdJ1sYfZMtJbyu9zik1Ey1KFIvVTxIYX4tKahK/D44Qrrrmw+59kSdciHp6p45QsFlZzBQNz5Yp67w2ggAumVrpH5NzVfvfi+qJpGu9YAWKcDscTIg5D1zR1COHqwo3DFLyP83zq+3GcmCirEmKc53EYr6+vjgD9MJpZP/Smev9w6PthnucqODB3sd8C4uPhFGNSkZQyEm67dp7j/cOhrjwTeucOpxMRD9MwTUPVdoGpkMSbugkvXrVNc7G7+Mfr33789OX+/jGXESUwIt50zaatifBwOFqZaidqmsYzg+mu28CicMRNs7m+anJOzBR8xc7N80SApjpNU5aMpaYWJRQgAgAkdEzOPfFxEU3VBISxeCtEOC/h0ZTyOKc5ZlXNZoAUPAXvRDUX+V9YFJ0KMxpMMz1Rh8tWxCzqsuacx3Fynpl96T5m0RijpKSaVaR0DhhtzTqXCkYLYPlUfy9qVnregQBwFn9U1TLnU6xcoQgzCSAScVEgxHUhaOEenjU4zkNwKwj7BKWdlZ7X7xReyfkcrkUSgLu8viyJ0zhNwzDUIQBAjDMivXpxc3mxTzn3w5RFihzSPCdEQCIXwnZTT+N4O47jOB8Oh2me26ZGtDLp4hz74K8v9oR4PPWEMEcZp2Pf90M/DMO42bQF5ri+vCjFLBM+HE79qd9uNvv9rh/GOUbnfcoCgHf3D0i46bZtu3HeEy4pOTm3313+5Z+1b796e39/++7jxz+++/H27m7sRxCZ4uyDe7nb7OoqNM3N9Yuuazdth2bj0JeCwBFXzhNRCKFcl3Kx5mk69sfj6RhjArAijZAKW4UBmDwXgkhpHZaq9KnjBypAvCTRZqo6pzzFPMecRXPOIooGa3O79ACXDmT5pXMxa2eUlNCxc75M2RAgRlGNySTnlMq0foGZFhmwFRQq70nO+jTPxjAUlh6DrIVSCfZudatgpgpFuhEWCYYyUmIASISqi0bVmYas69rLc9FjtkxaP0sGFnB0JSmYnhforGQ8h4BznEVVRQkoZjHTum6cc6Y2T1MuG/tirirftU1p5DLR8TikmCTnOaaHh8M0TT4UnVtYHAriy5urly9eTCmZWdNU3ofTMKUsh+MppUxEu91GVA+nHlfN4q9eXn8AIIJffvt6mOLd42PK2RGrChI/PDz+7vvvQte+9Z6ajnEhDDlHznVN017tr169evPN19+8+/DuD9//cP943227b796/ermRdM028120+0WEA+5LKlMaZKcoQD1gDnFnCIimshUzSI6TqPKIqirjOyYEIiQbZ3bRCvoI7GpKS/PvwCB610yBHLes3PFCJ2Dog6dRUubsNgnsitEOzAoQmLl8UXVh5kZSU1zloUwkXLKWUVKP6Cg8aWuIgIuk+eFf7EilbgO9JXcgxa6arHWkgoubQJf6n9bXKCaiepKgNflZCmULGoRTVvU1IqZPXXVAUB0ZZnimsICrMjU8m9Yl9CVpMGBQfB+nGbnedNtQggiEmPMIqdTP05T2zSd95uuqUKY5nma0xyTY/PBplm9D1nUwPYX+7atNMvx1AOY926aYZomZmJ22+2mqYJjPh57WE/vMIzeu8vmYp6jiHablpnmKb68uZzm2TFf7LY5y3EYsggCpDmq2u3nLx93P1xuLr0PhStUrjISMpBv27quL7b7b15//Zs/+02McwhVW7chBGbnuMyEcYlTRR8hS1X0wMqMmk8+zU5FlWVj29ev3w7j+HD/YIsXRLd0OYjo/1fWlzVJciTn+RGRmXX1NTM4uFxSF2W2K5ErM+n/v+uND6KkXRDkClhgMcBMd1dXVR4R4e568IisWglmAHp6uqqzMj38+Nz9+1idn1zQWEpRJAmhdlP8OftWk5mllOdlllI8x0VkKRKCqFoRLSIkAkgamIOKKVyHs91HQy6FAJMlU3XuQdEqOysiCEBemTumBQQI7LI+jfXzNtK669JqGSqqwaskqG2BZlIu0MhgtdejToSMykxaQU5rkRnQ84HW4L01U6tbRvVv1x+whoRWd65wfR1AcPVSJNoOQ9/1S8mlVMYbVWPmzWZ4erjfDN1lmqd5MdVSihO/Oxf9MPTvnx4B0aSM05RTZkZjTql8+vw6zfN2Oxx2m5KLiuRSxnHsukiE0zTNc9KisevLPG/6+Hi3/+Hjs6o93t+fp7mLHSJGotp5Q7hcLudxkpxU9Le//YeHD18hddxGN7w7gszM3HX9brev9wXqsHgtN82s0QICGHpbEsHQQohFjYOYZgQk5qHv7w53aUnTNJqpAZmq6/ENnfkABBiIKl5LYOe+8yfjqjAyT+Pn1+c///zL68txGqciDjeKk6aoqKhTpNZ3qdRHVbLFAA3N54mMauTVai6ga0mCzQIRkZkMkQCL2G1W556p9i49eiOymvhIlROnmgDwWtiLmvtJIoA2fK2qa9LpDrH6TjOvnHTd8GwZurYRE2g+3tYtu+ZzYXXYZgAQ3o7HGFtLvZRh6O/2OxF9O51yTiJlnufz5ZxyPy/L29u55GSAOZOo7Laboe/6vgPT4/H0w48fc8rMpFLmccIQlmVZ5nE/DPvd9vPnl6UUr+aJKWIU1SXnl+Pb0+NDCHwe568/vNvvt88vx83Qh8BLSkSwpDROc4yhjzGGcFnS88vx+z99/3j/sNnuAj8iYOCay9hNWIkc2u1YCwJoGb0nr6xKCFiggLgcHnMIkhEJQQARh34YhgERiVlFVKGYGUhnUkRCjNbc91ooYcXFEb1iMjVVkTKN8/l0fj2epmnKjc/H7dLLlrZ0Ve0T2qav+yesOiG26mFQm568AvBQc8/A6ImWeJG+Tk4BGJDnkYIGBmKgIkgC6rMr4lTsFfRZf6kBqgGajyJoLanEqAHvZiJCiC7S0owNWhy4LjJdO24tC/KkuDXuYW2YAkAIxKaaVZGZCWMgKdkMDrvtdhgATIo8v55jnLxNstvtzOWMnE4CcdP3IllVY9+B2WWap3EG02GzSTl/en65v79/ergvuXx+eSPip8e7nPI0L0hoqpfLqGp3d3sV/dNPv9ztd2h2PB6ZaLfbiGDX96/H03gZN5thM/QuIvrp0+efPv707sOXsd/2Qy/1Pq5OAqhlgQAVAGlnst4XJTJVInMw3b2R184q6uALEYUQu9gZQEoZfBSXsYAtuVAoUSSEQOi7H0De+AarDIZmagqmueRxPL+d3pZpKsuUl1mKz69Wxl2/t2ujUOvwbt1+YGifytvooCsg4+nlLcaNCIM3JwFc9kUdpScI5DcBtXLZeHPK1LCospqoUx2YqIooIXmI57oMCKZeBiliPZFXG8QmdLUS1xi0lYL6ceg6GVoN0LtcazIDdSTP3w3MLABiKYUCMWEfY8kyp0REXYw+R7Lfb0MIOWcA6AIV0Wle7u8O758ei5Tz+XyZplJK3/dfvH/69Mun17eTqCEAMavZ5+cXJuqHoeviZogiQwj8nN92203fdy5DP47jMAwcwsdPL2/n0VSnRZBOSLjZ9M4abgDjOKtoiNHMTufL9z/88PDuabPdc3hX0y5ne/zLzHw1WWtbyv7hGzRt0HSaPaAhAsWAhCKEIhx1v9v1fX86n52d3gyJTYXqHp96YlmI2Ut1NrMmpuYdRSnldD69HY+XyyWnZCpoyqAG5gsRRlghyfbItbJU1b0BbGABIJhhRTRbp7FB78hM3n8l8uFiNXBQBSITNXhStdgKdXp17WqctR9gzbZuzknzSczV7RVRpnUx8KYcXKWMW37piBys43kV62xYl7WEWGsIFFXPygAsUOAhUAjBzJ7fTmCKhkRUYjkcdtutSydE3G4A9Hy6HN/OMYanh7uH+8Pr8fh2PuecD/vdbrv55dPz+TyaqsvuDl3UUn76+dM8z09PTyHGIroZBgBMOe0326fHu58/PTNzP/S7bT+nwkgIQEy7GBDhMs4xhE3fxxhUShEdpznkEiMvRb/7/gdTtSz/4T/+5v7dB7AOIzrFu5/mFQJuRnnNyn1F08lCEFthQYjAaOjxrmZViJvN9v7ubhwv8zyr+Bqw525WREQMUJlNxYRVVEkNyZrFGwAUkbQsaZnzsoCUgCaEhAwVbgQDqNTlqutz9cul1nQxrOhhhSuxwuBQh4YBCTvGyAhIapDFzFtchMHFnRAV0GWDwMWV3DE2tQ7fVwb3kQ0tU1M0VFUAAjK6mZzzH0b0ee3q9lazXuv0m8B17ba3L2ogcNIGz3Bw3ZtFCA93ewMspcw5iyoDdF1AgBh5uxnAUEUCD10Xp2lKuQxD/8X7p4e7nWp5PZ5EZDP0h21/nufTecwixIQIMYau786XUaWMl5E5/NXXX0oX53kexzEQbYZutxmeHh/MIAa+vz/Qeco5dTEQQgguRCmXad5vN/vdxic6xmlx+DCGYGAfP/5cyj9eLpff/Pbv3335NeAWQsWr1nB/a6YtiGij3FAEcHYh8fBmLh0qJTuvTlHVwGEY+hACE5takYLkQhlkZiLKEQFBzLhKZNwsjps6WsQhumvSym5cGVyl+W6wSuB5kz+1SgKguk/w1LM2LeufwQDBZ1EdlzVA5y8lAmaMTIHaOryjN17yAAIYke8V14gjaiKmCiLAbP4ZDE1rBQ9QkInwirRjQ46qoa833k1wRVrMfbOfqNZeu75LS1bd8B3+N8MwjbMSDsOwDSGGAACBedN3+8N+6LpSJCX5/PLmyf7Qd/f3u/vDYUnpx58+vb6dnAT55e3y/PnldDqbKiFRF+72u76L4zQ59QO0RH5Z0ul8GS/TF+/fpyJdjH/7619N8xwDH3bDPMNhv0ulmFpOi+MvXeDArKrD0HdddxnnlAsiRGJDez0ef//NN1nKfwZ7/9WvzIYQAt2YI1xDjyG6Fps3bNVVNNR1F1TNtOSU05KXRaXUghiBOOx3hxi7k15KKWoaiVrCVyFPJ2VgDkxc/UR1f6amIcT7+4d3797P8zL3s5QCKqWISMEiRQTMyCmTbgGgmpkhNoYSJlwVDQzASbrALBBFxsAElZRBTTUiMFEgjETtfQEN1FX8GgZaTcopSwkQzKVOTU3VPKCDl2a8+mtPmB1sa7lTm51b+5a+Ue8zTOhQEa6Aa2WE8bH566vWFNU5DsDCOI4xRuw6ZBpih2ix67549wiI85xqbtvY8D03mZd0vlwu49jHsN304zT9/Mvn09sJEZ32qO/idrsRKYgQ2CcmZZrmXBVXMRV5PZ7u7ndE2HcRwAihlDIvqe/C4/1BDVKKx9NlSfP5TbvY5ZReU9ptht2mI4J5XpJACCwmb6fTH7/7bhiGjsPjF19Rpb+sYkt2Q0ForRIyAxHnGpOa+bkWr6mWoipLWtCLAyJk2u52T48Pb29vS81ZDcCHmdANXW/ZCsxUFEyaIg0S0n63+/qrL4ksLYsvNqioqlzGaZ6XeZrnaSpLKqWQKoAFdBQIGI3b4g9C1Qx22fM6gk4YAwYmYgSArCZi7JA/YSAicn5iED+S9VKB1ll2aPpETeLDTyY28NIfELtCcsud6gWtZnU9sjUFsAa/39ZSUAv2inf5oamG5ZVuS3v9r0M/9EM/IMA0zqJyt993MV7GKfbdYb81Uzf81+NbSnkz9OM4//zLc8o5MMUuppxTyjHwbrdlDt5q2++2fcfjuLgmZwisIs/PL0S0rmPPKQ1Lt+kjaNkOXZFSSpmmaZ5HAnh4uNvvhqGP07y8HM9pOauUlHJKadP3wzAMfX8Zp2UuMUYA+/zp8//I/6Sl/P3v/uvDh6+67trMWH1nM9BKoVNDjohlM8JVAAAU0klEQVQrvZqolmyiOaWcZibKOad5jl1HHIZ++PDui7e3t1LyspiqNXHJCg/lIlEkWhsG97holVWGiGKM2+3m4W6/LAEQCTEwRWZ3iCmVcZrf3k7H49vlfJnGUdLi9IDNkRpUGseKwjuvCSEGxuCDz0h+4hyAgNY+1iZG6KCStQwB0Nj/Cq6dr9XkzEzBGKyIsoFVeWFTA18guX5OJCKHe625hVoJ3DyCa24KrU/QlBoqCtaK2mazAIgY+mEAAG+YbTeb7WaQUhBx33XELAJD3xsYB96Fbd91x7fTOI5mlgmnKQGCSjFVV84khN7FN8w3zDGECICvxzMAPNwdfEp/M3RoWkqeTYjg3eNDypazT/rJp+eXnJM3n97d7fuue3l5ndKSMuVctBSVMgzDZujnJaWUWIRDPL29/f6bf0bmf/gv/+3h3ZfO/nzrO/0fUZ/WUfBSybF7KVKSqJaS1URUfTt5LsnA+h4Dh81mc3d3fzydVM2joWuEurn7ZISocuU0WNP8+gUhdTEG5sLeyHTRTQ4hEtHhQO8M8ldf5JzGcXp9Pb28vr6+Hi+ns6YEph5IoaV+DUqjwBhDna9KIimrgSFSdAYyX1ABUDVXAyX0IU5AF+9qdYqDkQAgarG6TwfjIXBYz7rBKg2vWNeDV/j1mne2fytE2tgc1vdABGCsaBo2LMKal8WbWxfSkjwJ22yGu/1OVHNKRLTMy+UyLkvaboa7w/7x4T4wPb+8juOcixRnU0FERBFx3h816yIPm6GLQUWYYxchMJqJFEHmYehcE5sQRcsyT8qMaGPf+XOMgU3AVMZ5joGYsKiGEB7u73IpqjbPKiKXaU6lDH3fdbH49EaBjPD6+vLNt/887Ha/4e7w8BhjXA+iNXTYP3xt/YmoFKtTOwUAfcyAiUwEwPq+rxQHgfuu326Gvot5SX43vdwpIszk3SMRp0EXqgU2rc7AzFJK85JySj73TQhMAbFGW0Qahj7QQR7t668kp/R8PP7y8y8f//zx9eW1LKkOVACuOyRdpMiIxISYRKesZtYxxUAxUDWOlte1C0aDRo8P5hu+qwFds1t16/GMYuXmNwKDtm/kS/FNCWSt3GGt8lcss/72ViN52ipqDbwGaA/Jv1rzB0MI5/M5MHd9L0VO5wsTiRlSEpVlXsQgMOciqioERHQ4bJeUx3H26biUMjP1fURERo0h7LfDdtOfx7mIhIAxhi4ENT0cDofd5sePn0rOgYmQci59F/vA87Lst/22C6UPS/bDDSKiJRFGUNh0PAz9ZZz6LoyLOgBNYF2kwj0nzKWAaWQoy/Ttt38Ydrvf7H7Xdb3vErWehqeL2cSk5JyWZbzMy1RymecplZxTulwu0+U8TvNlHKd5STlpKSpKIbhevKn1fTS1yBzIwRfMRTopMXSe4NYUoiVStgY0oixlmhcmCrEO+/SddV0k32NzbXeiXb/B3e5wODzd3x/22z/+8fvnnz+ntIDnnWgGFKOTgRERLWJzElGL7AeMAyIhFKgmogBeqRCBKDQYx9bGWgvX0K7XmhHXf1SNqxRj7QtVf+m912piCP/PxmBNWf3+rzm6451XIKBG+YoDXAslAAsi4tG5SOm6oKq+u+XEYPu+N7Pj8Q0A+j5uh76PztqDSwrjZSKSYdOHGEwNI3dDT8zTtKRlJoTtbui76NFov9vMaVly9hnJ2MWHIW6HIVdRoqHrY99Hn7rBSgBquWRE6EL3dLc9n08FYb87FHGiIjrs+t12k4ucx7kU3Qw9MlrKnz5+/D/dN3eHu1Jknqd5msbxMo7j+Xyaxul0OU/jNLrk2zy7Y1MEyaWUDIjEDKrMrAbe+0OEEHjouoe7u6en+4fDduBoToPp6aC2Hnq742YOULuMhnEI93f3l8tpmWeVUkrxOXaHG82gw87bWqoCGfqu32w2zCEXOZ/H6TJKya7KbIDR1eh8lUp0TiWLej3KgYNXRoBkVgB1PSV1J7giTf57nV1RTU2JfdXMTGpC1BDZFnDV1h+t5FRt2xOc56FBntU8bd3ywCv1/FpI2fV7a+SH1ZXXEN910flPOAbRK6Y2dN3Qx1JkmiZiGvqeENSsFCHEh4c7MJgOyzLPuRRABAZVNrW386Qli8pmGDbD0HV+t5AQUi5oQkxD3x12w9BHM8tZN32362PH0IG8XfA8zmYKpqbFFKdc5iUzcd/15/PRpyoQaMrLp9cLM5nBksu8ZFH10XH77/8YkBgxO5qj9TarFxzXxAjqkSfutwN2Xcm5C7zZDGYQQ4yBY+RI9OMvz7mUKRc5HqeUXt82D4fdX33x2K0TVQamUkphLoGDEjuICC3CMtF2s/vw/ot5mY8vn6WU5Gv12lAEr2gYzCDlUkQGHQKH/XZz2O+7vgNE393pAm0iIyIB5iznJKUIB45MHVMM3LEPF4MaYsVcnSyy1s8ebREAQdFFPAy5Si9ALZEcuIHKxlvb5dV0gAwIkW59J9xaFzTErNke4K1vrT93BQpuav+bFwJAIGIzW1LCXJjR5w4J0VTO55xTJqZh6D3zm5dEYBy7/XbbdbEUSWl2AF9FUhFULVIA8eFuv9/vNpshBEKRcZ6nSS7jlHMhMJMyT/M4TiaSckGiTy9v05J1WS5LHuekRZFIAXIWABAz8WxJ69SPj3m120yE6BpnDdhBW7sUBsgUQyBmcNhTxZ+CAAQwJCZmy0UAus1GU0pzssBzLuxxvGgMrKWAWVablgWILks6nS9/92//+uHurqiIlFzIAEMouWRiFqnL5j5LogYU+O5w//WXX0vJ57ejw3ArDsUUQlBELKLTNOWS+3642x9UrY+hC9HxUCbqmAlRDHKROUkRDUx9oL4LQ+AQAiCKT0itjsjvk88p+7QIgrpUMQAYtLUNU1UPCBU2p1r0GxgBrrdX0XvuWOEqwLrCVZVr3LPC+vv/vy9uc8/qpKtZ37hY8CIJwYpqYy7H0MUuRwQTNSbaxk3OZV4SghHRdjv8+ssP7x7vmclUcu7PfZ9ymqZFL+NlSSllM8spv7xeFKGksizzkkopJeUiClbhx3ppdX6mHqu6boitaXbTCmqwBDa7A+v7jpjHcfJDSE5I4LEMcf/01B/2m74//vSxlLx/fNrttv12+Paf/rfj8GQmZqRqpkKBCEEEkUrO5CR3mgWgeNXffq2ITcvCRJ9Tmv7Xv/zm7/72w9N9kQKZkIJHM5FCRCLA7PHNPwrGrru/fyqSfzSYLqfWLii5MC5zylkNRORymVLJm83Gw/C8LFnFAALjEJkIFUBFp1SKKBP2kTdd6COHwNAcJCGwuZKx943AsClyuHX6NEkbdyqiQK71B0T1IBOCoSkh15ebI/dOzF5dI1FFNFuqXQe5V3d6Dd14PRPXx3vbC71iyf6tUHLy5Al9Y8EZ9IoQIjEjwGW8iKgUQcS+6/a77dvr6XQ8zSm5bsU4ZxE5T9M8LUWtqFopVmn1aibTnH514h5dvPVgNXmutSZWYgH1DAZUCakWmW5+oqYCxIioCiLZcy4T9SBpRZylo8zz01cfdveHy/PLdLmcxwsEvFxO5lNkhABYZUGKACr5vTUwU9IqKaBtrZ2Q0JCYDbGIFjUEyHn8n3/419/9p39/t9+JFNMiJQsHQhLMAIgoCGrkhIcIiH3fPz68E7XnzyG5uA8REqVi83yelrTMaVkWIALDE5/Ssnz85WWeZkJwvnA1SKJLKqkoEXaB+8Aup+SR3Wyl7qqYe+X2rF4R18UkA59RMbvRj3Oks9qKGxgSVO7/6u+rHdY/tI3N6voaRXRD8HHNAW4zgBs3uXaQbssydzSuduqqJb6SYmVJvl4j1mZbmkTZZZqej2+ilfy/fmTC2lpoJQJWVj4lbAwpWHtdgCDSph0QsfaOgYmM6s42EiNCSdlXPZG5nTIjZslFTJ3iAwhFPBkidRUT36pBBIBlGv/0zbcqlVQ7ny6fz2cRicQYgvkcbt0R9t1bD7VcJ46d1c33exAICAjNVPyB+Z0xe7mMv//2T7/7zb+LwdKyeoyq6mCmzExgQAZWPQ1z2O/uum5AxMDRZ2tSyqfz6eX19Zfl87RkMNAiry+v8zgdT6cyzQGRyTvizvUiRLSJFAPHQFWLEcBXLg3RAVkD1FaA0NWJAiBBZchBbfi8udxUq+9xHQiogZtW4hv/ayc7gSY5fq1+/H0rHEa1adRstMX0vzDF1XGu7+/1QvCRQZeq0gYZqpo6S4CPmdSBNChiAE2/o5Ua0LABu7p09AYxUWMd8DlwIkV0CW0MoePgFYK7F4C6mswtlDiJMIhKyVKkpkn105m07BsQjcldgVNZYtVGA1N1/9RSPehCrFmA4xwtDtXM1dPbGLz1ylqRca2llvPUARMBB4rR1VOeX0+vx9PT450VQsoGUKT0XSche6EVYgQItfAQUZG+73fbfQwxhFCLNpGnp3fv34/v3j1/9/33f/rhxx8/firLwqBsQqaAoICmNmXJRZgxBu4jd13sYqSG5DhCpAZSR+zB5wZ8mUIrY0MNWW4lAUGrAyEidI7bhiBDS5QNXSCs3kqqVdJNZolVMQLEqq/0b1q7t6s1ri7zLxwnrq+oxGMAEFyVR1Swvq7CqvWtoYqGae00GFRx3CpgX/1oiyX+cg4hdhGRvOpUNS3ZT23d0DbQItOSKmq8TgqQ3xcFAAKqd0FN/fIIpW2ZWz0O6IeePG1vC0cYWK4pbFuaRIC6OuZsa2ae+SMGZKe1r8hKQmLWXFwecm1DOQRpAAURuZBqiB0xTSn//Hz86sOTc9XHGGIIoiZpURWLilh5AXzXMYSw6frA0Yc369YDABKGwLvt5vH+/unp6Ztvv/3lp5/yNEYCBhKDopZLyUUNoAu06UIXA4fg5Q9jZRfzR1H5u7V6eh+sqv6krntAnbOzSqy/XgggNk4cvHGlNTez9Ut/4Zo1ErVhwesNtxpTbQ33t2DcreOE1pLFduGIEJD4WgFUC7WVuKyaf8Vi1y0T0yY9iIDrgun6i31QB8FW2vaVMkpbc78+b0Q0tDVNrhJEgDXBUKzTWw51oI9F+bVRQxCb0/ZLVIXKKOd4HiG6rwVfloPKCOzllBYLTP5snTLJ3bKJBGJPNOv8uidrqGbq6bMVEcwAQcwuWTwKeoOVQzA1p05ELGZKRBy6GHo/vYEjVa5Or63FzCU+LIbw8PBwOBy+/uKLf/3jv3z3x2/H1xfJUkRTKbmYmXWRQwgUmCudrgJQMdCGuzeXYT4jpw3P/EvPtaZ+BFUIgpl8lg4JyYeu1dwH11vdkKgaM2m1O2gWA1Wy+9Ykrj1QuO4bXd0qrtkqXOslNQMIfoIcFHA/4Y8EW+BTbaOyiCvBxNoZa1F9nT8ARwOaL/KOAbpktyGtVJrNE3qeBC1/bR+hJkHg5Fa3rn1NGLQ11xBx3QX09EDluhpYf+paollTP6nfN1W1AoimRlWWDwCsqCATuC6Lj3+hVdm06h8MDdCMmNOcUirUU8qFQzAtuRTnIo0x9v1mt9v3/cYNlOoGRXsHADMrLu7BrhGDzPzh/Ye7w/7Du8dv/vD7H/70Yz6PToDXRRpiiDF2zA5OO/zu8TG0FhGs8xCoq/34HSGAutJqoADMFJidv5KZXfpbnaaHzNePSK3uK5iBmrfSDUypqsyoGQFBW5harRXbI7uxSAODdZMJrwG/rot5IxqJQDVwDKZa1XEBWy9r3SgADq3DBUbEiOsZrWRU9QJW4/c82we/Ddb2N7QawVq6oLCOfpEUBwV9VNHM5Si6vjvsp5cXLeLKj+t5rElxjVGK2OgrbnqMyEziCJGfKN9Qg7prVtMEBEABIzNP9LniDz6dWdc13ZzNFOsX6MvhaErABJCWdJnmGNgAci6qMs9zEYkh9F3Xd7ELMbDLDVQc11o9a35hKgZiVlFFzx77fvvll7+6jOPHz8fl9SSqMfIQOUYOTB7Trfmu5ubqLVh5ofysqvojaoPrHsF8x44CkEsmVTkGf8wG5s7K6tq7Qht2r12g5tFAa/sDKmiDayMNWm/dh0wBoA0A6C26szr9a0dU1cBC4GDkV9+yzBXz9yQFVwqK1ZFdE4LqJAFahmotCBgYmppJqb/MBZkBEZCYQgzU93le+s32/a//5vP3352eP/nkm98/DhyGAZlXuoHmQ/2Wo62p/urHGxeeX7yWQmui0k5KayZXdwrY+hmInjzUil4rhQG1zNMqveD67mCEqqiqhJRULuOy7bsQ9ZxzkYKEfey7GBFRREQzC4OBS87VigRJwVRFck55RgAIaIDcVkSRHDp93O52hhRi6ALFyEhciRPBRF1WGOlKKltp3hXWR2k1ufSPpVp/DpGIkchtE4m9dFQzMedchIpauas29CE8n1wz5JrOX2MgoK9IA1R2xTYnbpW/pBXWWjmhK2TQ7iy0+RJ/r9CYxFo9ezPLX61hHUg0xXYckdZThboiKus0YU05tCqPQisIyTcfgTnsHx9ou3n788+B+fD4eH55eXv5bG1AC8yAKA5DWhYTqQOD1bdd8d0GSHjMaFnP6rO19javiVe7EbWkb6mtf2y8Ge+u1r/CBgBonrbadZxOGwsWqojlnEpOl1HUlJm2my1VsFZLkZQyGDEbMleCFwaj6ziJFEHweX8hysxBW7L3eH//b/7mr89vx+V4JIDaO1YpAMVA1Vu8GJmtldV+gNvYKtQWsHtvtSKVS9DpJzgEInZtEiBSF7kz8HWkla+UjMzZmKAen+Y/cS1IoO3lwU0BtP5XVrTJy6D66JzpxLHv+hRW6wsVWGnfamlus83V/66bMrjaH4A76ht7lurq6oPFa+FVk0VCNB8QE6MlA6CJ5mXG1kZyX4vMHAIA2rI45o/12lrN6G/rw93tTNj16AEAEhN4U98BPzPTumpOgOZdaPVHxL5eiTXpqEmVrSxbjfN//WDr/xTBEFR0zmVKScWQMDCbWS4lFAakEDRnAcgBkFwD07nNlCpMiBS7IS2jSVYzJEk5OTMCMyPS4/3h3ePdz9MIIoC1/WMAYi7dC4YE4Pp0gK5600YLPexXIohKKA5iQOs4FDMyM3GFPG+Ki3rP1JiaMmqlGSOX4VLzHN3M72ZlnKwv9lu5GkmN0c3D113VenbcsNHaipQ/7v8LuhRpugWd6nYAAAAASUVORK5CYII=)" + ], + "metadata": { + "id": "gW4cE8bhXS-d" + } + }, + { + "cell_type": "code", + "source": [ + "image_data = (periodic_impulse | beam.Map(lambda x: \"Cat-with-beanie.jpg\")\n", + " | \"ReadImage\" >> beam.Map(lambda image_name: preprocess_image(\n", + " image_name=image_name, image_dir='https://storage.googleapis.com/apache-beam-samples/image_captioning/')))" + ], + "metadata": { + "id": "dGg11TpV_aV6" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "3. Pass the images to the RunInference `PTransform`. RunInference takes `model_handler` and `model_metadata_pcoll` as input parameters.\n", + " * `model_metadata_pcoll` is a side input `PCollection` to the RunInference `PTransform`. This side input updates the `model_uri` in the `model_handler` while the Apache Beam pipeline runs.\n", + " * Use `WatchFilePattern` as side input to watch a `file_pattern` matching `.keras` files. In this case, the `file_pattern` is `'gs://BUCKET_NAME/dataflow/*keras'`.\n", + "\n" + ], + "metadata": { + "id": "eB0-ewd-BCKE" + } + }, + { + "cell_type": "code", + "source": [ + " # The side input used to watch for the .keras file and update the model_uri of the TFModelHandlerTensor.\n", + "file_pattern = dataflow_gcs_location + '/*.keras'\n", + "side_input_pcoll = (\n", + " pipeline\n", + " | \"WatchFilePattern\" >> WatchFilePattern(file_pattern=file_pattern,\n", + " interval=side_input_fire_interval,\n", + " stop_timestamp=end_timestamp))\n", + "inferences = (\n", + " image_data\n", + " | \"ApplyWindowing\" >> beam.WindowInto(beam.window.FixedWindows(10))\n", + " | \"RunInference\" >> RunInference(model_handler=model_handler,\n", + " model_metadata_pcoll=side_input_pcoll))" + ], + "metadata": { + "id": "_AjvvexJ_hUq" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "4. Post-process the `PredictionResult` object.\n", + "When the inference is complete, RunInference outputs a `PredictionResult` object that contains the fields `example`, `inference`, and `model_id`. The `model_id` field identifies the model used to run the inference. The `PostProcessor` returns the predicted label and the model ID used to run the inference on the predicted label." + ], + "metadata": { + "id": "lTA4wRWNDVis" + } + }, + { + "cell_type": "code", + "source": [ + "post_processor = (\n", + " inferences\n", + " | \"PostProcessResults\" >> beam.ParDo(PostProcessor())\n", + " | \"LogResults\" >> beam.Map(logging.info))" + ], + "metadata": { + "id": "9TB76fo-_vZJ" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### Watch for the model update\n", + "\n", + "After the pipeline starts processing data, when you see output emitted from the RunInference `PTransform`, upload a `resnet152` model saved in the `.keras` format to a Google Cloud Storage bucket location that matches the `file_pattern` you defined earlier.\n" + ], + "metadata": { + "id": "wYp-mBHHjOjA" + } + }, + { + "cell_type": "code", + "source": [ + "model = tf.keras.applications.resnet.ResNet152()\n", + "model.save('resnet152_weights_tf_dim_ordering_tf_kernels.keras')\n", + "# Replace the `BUCKET_NAME` with the actual bucket name.\n", + "!gsutil cp resnet152_weights_tf_dim_ordering_tf_kernels.keras gs:///resnet152_weights_tf_dim_ordering_tf_kernels.keras" + ], + "metadata": { + "id": "FpUfNBSWH9Xy" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Run the pipeline\n", + "\n", + "Use the following code to run the pipeline." + ], + "metadata": { + "id": "_ty03jDnKdKR" + } + }, + { + "cell_type": "code", + "source": [ + "# Run the pipeline.\n", + "result = pipeline.run().wait_until_finish()" + ], + "metadata": { + "id": "wd0VJLeLEWBU" + }, + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/examples/notebooks/beam-ml/image_processing_tensorflow.ipynb b/examples/notebooks/beam-ml/image_processing_tensorflow.ipynb new file mode 100644 index 0000000000000..c9cd2de2a3134 --- /dev/null +++ b/examples/notebooks/beam-ml/image_processing_tensorflow.ipynb @@ -0,0 +1,918 @@ +{ + "cells": [ + { + "cell_type": "code", + "source": [ + "# @title ###### Licensed to the Apache Software Foundation (ASF), Version 2.0 (the \"License\")\n", + "\n", + "# Licensed to the Apache Software Foundation (ASF) under one\n", + "# or more contributor license agreements. See the NOTICE file\n", + "# distributed with this work for additional information\n", + "# regarding copyright ownership. The ASF licenses this file\n", + "# to you under the Apache License, Version 2.0 (the\n", + "# \"License\"); you may not use this file except in compliance\n", + "# with the License. You may obtain a copy of the License at\n", + "#\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing,\n", + "# software distributed under the License is distributed on an\n", + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n", + "# KIND, either express or implied. See the License for the\n", + "# specific language governing permissions and limitations\n", + "# under the License" + ], + "metadata": { + "id": "NsNImDL8TGM1" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "# Image Processing using Apache Beam\n", + "\n", + "\n", + " \n", + " \n", + "
    \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + "
    \n", + "\n" + ], + "metadata": { + "id": "SwN0Rj4cJSg5" + } + }, + { + "cell_type": "markdown", + "source": [ + "Image Processing is a machine learning technique to read, analyze and extract meaningful information from images. It involves multiple steps such as applying various preprocessing fuctions, getting predictions from a model, storing the predictions in a useful format, etc. Apache Beam is a suitable tool to handle these tasks and build a structured workflow. This notebook demonstrates the use of Apache Beam in image processing and performs the following:\n", + "* Import and preprocess the CIFAR-10 dataset\n", + "* Train a TensorFlow model to classify images\n", + "* Store the model in Google Cloud and create a model handler\n", + "* Build a Beam pipeline to:\n", + " 1. Create a [PCollection]('https://beam.apache.org/documentation/programming-guide/#pcollections') of input images\n", + " 2. Perform preprocessing [transforms]('https://beam.apache.org/documentation/programming-guide/#transforms')\n", + " 3. RunInference to get predictions from the previously trained model\n", + " 4. Store the results\n", + "\n", + "For more information on using Apache Beam for machine learning, have a look at [AI/ML Pipelines using Beam]('https://beam.apache.org/documentation/ml/overview/')." + ], + "metadata": { + "id": "yxLoBQxocAOv" + } + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OSZrRmHl9NQY" + }, + "source": [ + "## Installing Apache Beam" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "MO7iNmvkBdA5", + "outputId": "6c76e29d-3c70-4c3e-aca2-7cc1dcd167a1" + }, + "outputs": [], + "source": [ + "!pip install apache_beam --quiet\n", + "!pip install apache-beam[interactive] --quiet" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "45mf7oHu9XbI" + }, + "source": [ + "## Importing necessary libraries\n", + "Here is a brief overview of the uses of each library imported:\n", + "* **NumPy**: Multidimensional numpy arrays are used to store images, and the library also allows performing various operations on them.\n", + "* **Matplotlib**: Displays images stored in numpy array format.\n", + "* **TensorFlow**: Trains a machine learning model.\n", + "* **TFModelHandlerNumpy**: Defines the configuration used to load/use the model that we train. We use `TFModelHandlerNumpy` because the model was trained with TensorFlow and takes numpy arrays as input.\n", + "* **RunInference**: Loads the model and obtains predictions as part of the Apache Beam pipeline. For more information, see [docs on prediction and inference](https://beam.apache.org/documentation/ml/inference-overview/).\n", + "* **Apache Beam**: Builds a pipeline for Image Processing." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "z5_PUeZgOygU" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import tensorflow as tf\n", + "from apache_beam.ml.inference.tensorflow_inference import TFModelHandlerNumpy\n", + "from apache_beam.ml.inference.base import RunInference\n", + "import apache_beam as beam" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "x3tSAqP7R2rZ" + }, + "source": [ + "## CIFAR-10 Dataset\n", + "CIFAR-10 is a popular dataset used for multiclass object classification.\n", + "It has 60,000 images of the following 10 categories:\n", + "\n", + "* airplane\n", + "* automobile\n", + "* bird\n", + "* cat\n", + "* deer\n", + "* dog\n", + "* frog\n", + "* horse\n", + "* ship\n", + "* truck\n", + "\n", + "The dataset can be directly imported from the TensorFlow library." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "MqylmjBhPCOW", + "outputId": "9d9f5854-80f2-4a4f-a52b-2b81d6295639" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Downloading data from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz\n", + "170498071/170498071 [==============================] - 4s 0us/step\n" + ] + } + ], + "source": [ + "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar10.load_data()" + ] + }, + { + "cell_type": "code", + "source": [ + "x_test.shape" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pfzkgryZUV8P", + "outputId": "79bc798f-f93b-4d7b-8783-c5defa6a2322" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(10000, 32, 32, 3)" + ] + }, + "metadata": {}, + "execution_count": 4 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "The labels in y_train and y_test are numeric, with each number representing a class. The labels list defined below contains the various classes, and their positions in the list represent the corresponding number used to refer to them." + ], + "metadata": { + "id": "6hEHIHPsVxw4" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3uImFIBXv0My" + }, + "outputs": [], + "source": [ + "labels = ['Airplane', 'Automobile', 'Bird', 'Cat', 'Deer', 'Dog', 'Frog', 'Horse','Ship', 'Truck']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 447 + }, + "id": "zeE81PNOcGfZ", + "outputId": "d2a08cb5-4fdc-47af-c2b2-5602e7600f09" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 6 + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "

    " + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGdCAYAAABU0qcqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvsklEQVR4nO3df3Bc5Xn3/8/Z1e5KsqSVZVuShWVjG2Pzy85TBxwNCSXYxXanDARPB5LM1KR8YaAyU3DTJO4kEGg7SslMQpJxzB+luHkmhoQ+MQx8GyiYWDStTWsHPw5QHOwYbGJLBtv6rf2hPff3D76oFdhwX7bk2xLv18zOWNrLl+5zzu5eOtrdz0bOOScAAM6wROgFAAA+nhhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgykIv4P3iONahQ4dUXV2tKIpCLwcAYOScU29vr5qampRInPw856wbQIcOHVJzc3PoZQAATtPBgwc1Y8aMk14/ZgNo/fr1+va3v62Ojg4tWrRIP/jBD3TZZZd95P+rrq6WJP3R//NHSqVTXj+r+UL/gRWXFb1rJSlpOAlLRra/aCaU9K6NPuS3iBP2Npw9WrOYnItN9aXYv75kXExs+A+leMjU27JuOdvxcbKd3VsSs1xs24mx4Xhak7uGTMfetg+T6bR3bWVVjal3ZXayqT5K+B/PQq7X1Hugu8u7tpQrmHonDIfT8hBUyBX0v//q4eHH85MZkwH0k5/8RGvXrtWDDz6oJUuW6IEHHtDy5cu1Z88e1dfXf+j/fe/Pbql0SqmM3wDKVGS81xaXGW/kYzmAIgbQB2tta7ENIP/9/W59ybvWOVvvsR1Atp04lgMoMaYDyP9+n6ksN/XOVFaY6iPDA0WUsP0SPFTw386S8VkL2wCyPyXyUU+jjMmLEL7zne/olltu0Ze+9CVdeOGFevDBB1VZWal/+Id/GIsfBwAYh0Z9ABUKBe3cuVPLli377x+SSGjZsmXatm3bB+rz+bx6enpGXAAAE9+oD6B33nlHpVJJDQ0NI77f0NCgjo6OD9S3tbUpm80OX3gBAgB8PAR/H9C6devU3d09fDl48GDoJQEAzoBRfxHC1KlTlUwm1dnZOeL7nZ2damxs/EB9JpNRJuP/JBsAYGIY9TOgdDqtxYsXa8uWLcPfi+NYW7ZsUUtLy2j/OADAODUmL8Neu3atVq9erU9+8pO67LLL9MADD6i/v19f+tKXxuLHAQDGoTEZQDfccIPefvtt3X333ero6NAnPvEJPf300x94YQIA4ONrzJIQ1qxZozVr1pzy/y8NlT40Q+h/coY3uEcJ2yYnk/5/pUw4vzfOvseV/HsX8rY3rxUM74hOGP8Sm0zb3nSZKPd/x3osW1pBXPLfTssbLiXJ8p5L6xs0rW//jQz1LjL2trzJVbZ9mLDUO/83/kpSPOh/7PtyA6beg319pvrKyXXeteWTKk29s9P87z8547oHe/1TGUpD/vdN3/eHB38VHADg44kBBAAIggEEAAiCAQQACIIBBAAIggEEAAiCAQQACIIBBAAIggEEAAiCAQQACGLMonhOl4uScgm/yBdX8t+Mw7/94IfifWjvgn9MSUV5jan3gTcPedfu+81vTb0tHw4/KV1laj15StZUf+78Gd6155w7xdQ79k8pUUmDpt4f8XH2I2tNnU+F/09IGFcTG9JyEiVbDFNsiBAqyRbFEyWt8Uf+XN4/okaSBo75R2UVS7bbeHlFtX9t5WRT70Tk/1E4+XzOuzZO5P1+vndHAABGEQMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABDEWZsFN1jIa0h+IVXd/V3efTve9s9fk6RCj38+VZyvMPU+bMiCSydsuVfHjh71ru2LbblXcdGWNdbxu1951y78xAJT7zkXNXvXpqptv2/lXJ93bTJhCFSTpMiWqSbnv/aEIX9NkspMeW22Yz9kKffMfhzm/Pd5bAm8k6SE7bZi6Z/r9b9dSdJQzr93WcL4kB4Zjr2hdykx5FXHGRAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIIizNoqnPJ1SKpPyq53kP0cv+l/zbOtIVnvXZsvPNfV+8zeHvWtf3bXT1HvmzJnetVXV5abe6cwkU/1rr77hX/t/99rWkkh7157/ybm23qmMd21B/abepcgvquQ9LvKPhIpiW1xOQv4ROHFki7RJGJaScLZ1m3J+jL1LxqXIEGeUGMrbOg/5H/uSMc7IlfnXl1X43x+SnrcTzoAAAEEwgAAAQTCAAABBMIAAAEEwgAAAQTCAAABBMIAAAEEwgAAAQTCAAABBMIAAAEEwgAAAQZy1WXDOleRiv/kYGXKeXOSf2SRJQ27Qu7arcMDUu35eo3dt7+AnTL3f3LPfu/bYgU5T74F+W15beabKuzY3YMtUe3nnr71rB3r7TL1nzZ/lXVvXPM3Uu5jqMdXnXJd3rUvafq+M5Z+nZ7z7KCH/HDNn7F2W8t/OMkPm2btrsWXeWbLgJGNvVzQU2zIGS85/HyYNj7NJzzVzBgQACGLUB9A3v/lNRVE04rJgwYLR/jEAgHFuTP4Ed9FFF+m555777x9Sdtb+pQ8AEMiYTIaysjI1Nvo/vwEA+PgZk+eAXn/9dTU1NWnOnDn64he/qAMHTv7kfD6fV09Pz4gLAGDiG/UBtGTJEm3cuFFPP/20NmzYoP379+szn/mMent7T1jf1tambDY7fGlubh7tJQEAzkKjPoBWrlypP/7jP9bChQu1fPly/fM//7O6urr005/+9IT169atU3d39/Dl4MGDo70kAMBZaMxfHVBbW6vzzz9fe/ee+L0jmUxGmYz/Z40DACaGMX8fUF9fn/bt26fp06eP9Y8CAIwjoz6AvvzlL6u9vV1vvPGG/v3f/12f+9znlEwm9fnPf360fxQAYBwb9T/BvfXWW/r85z+vo0ePatq0afr0pz+t7du3a9o0W1RJvr+gUtEv3mLIkD6RMMR3SFIp8o/NKLmjtt5D/r3PmTPT1Hugxz8C5a3X/eOGJCkeskXaxMm8d+3AoO1VkN1d/vv8+DHb8el8yz+i6LwL5pl6z77kHFN9de1U79r+0olf8HMyLvK/rcSG+4Nk+w03NibURJF/NIxZbO09hmsxxfwY84wM9UM5/8eJobzffX7UB9Cjjz462i0BABMQWXAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCDG/OMYTlUhFysueWZUFVL+jY1bHCf9g+aSyaStufPP4Kqqti18cl2Nd23fZP9aSaqfYtvOQt4/m6z7+DFT79xgv3ftQM4WNvbmmyf/JN/3O9Lxtql3Z+f5pvrFV3zCu7aqvs7Uu6/kv3brb6xx5P8/ooQtx8xSHTlbVptLGrfU0D+y5rVF1nw3fwlD5p2L/fdJwrOWMyAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBBnbxTPYFHxkF8ERfOUud59Z82bZVrHnjde9q493n/E1LuU8I+RKUVHTb2nTpvsXdv39jRT79KQLdaksmqSd+1FmUpT79/u2eNdWyzaonhyQ/5RSf1D/sdSkt7Y96apPpnwjz9a+JkFpt5lk/2PTykaNPW2/Ioby3Z8IucfUZMw1EoyReu8uxhj/RiJnG0fWrjI//6Q8KzlDAgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQxFmbBVcs5RQn/Jb3q50vefd1LmNax/xzL/OuLSb6TL33/s5/3YMDx029yyr8t7Oq1j8LTJJ6evwzoSSpp7/XuzabbTD1Pv+Scu/awwd/a+ptiffq6R4w9Y6LRVN9X5f/8X/7zbdNvWeUN3rXugrbukuJvHdtQra8Nkv6WtL5Z+lJkotsmWq2eltunDPUO2c7pygZdvmQ4XxlKPKr5QwIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEMRZmwVXXTtJZWm/5cWZQe++z/3r/2taR/pf/XPS5s+7wNR77oK53rX5clvW2PHj/hlcqeoqU+/qshpTfVH+2WTHertMvcucfxacS9vWPZTr966tqKg29Y4qTeXKFfyP/zuH3jH1rs1O9q7Nzsmaepfibu9al7DlzEmG/DVLsJ8kyZgdl/DvHxtz6WLDw3RsXHexzL930bCOwlDOq44zIABAEOYB9MILL+iaa65RU1OToijS448/PuJ655zuvvtuTZ8+XRUVFVq2bJlef/310VovAGCCMA+g/v5+LVq0SOvXrz/h9ffff7++//3v68EHH9SLL76oSZMmafny5crl/E7JAAAfD+bngFauXKmVK1ee8DrnnB544AF9/etf17XXXitJ+tGPfqSGhgY9/vjjuvHGG09vtQCACWNUnwPav3+/Ojo6tGzZsuHvZbNZLVmyRNu2bTvh/8nn8+rp6RlxAQBMfKM6gDo6OiRJDQ0jP9WyoaFh+Lr3a2trUzabHb40NzeP5pIAAGep4K+CW7dunbq7u4cvBw8eDL0kAMAZMKoDqLHx3c+W7+zsHPH9zs7O4eveL5PJqKamZsQFADDxjeoAmj17thobG7Vly5bh7/X09OjFF19US0vLaP4oAMA4Z34VXF9fn/bu3Tv89f79+7Vr1y7V1dVp5syZuvPOO/U3f/M3mjdvnmbPnq1vfOMbampq0nXXXTea6wYAjHPmAbRjxw599rOfHf567dq1kqTVq1dr48aN+spXvqL+/n7deuut6urq0qc//Wk9/fTTKi/3j0yRpKpshVLplFdt42z/eJD62bbIlJ6j/nEsu17+V1Pv1/b8X+/azyy7ytT7/S8E+TCDuSOm3oP+6SqSpGmJE//59URSqQrbWgb6vGur6/xuT+/J93V51yZdwdS7ttY/4kmSBgb9Xx06OOgfTSVJv3vjkHdttv4iU+9s1n+f95T8I5skKU6W/GuNETUlZ7utlAzxOiXjWoZi/4fpfMF/n0hSPucf2VUs+N/xi559zQPoyiuvlHPupNdHUaT77rtP9913n7U1AOBjJPir4AAAH08MIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBDmKJ4zJY5ixVHsVVtyfnWSVJY8eYzQiUxp9M+Zy2b9ayXpzd8c8K594mebTL0vv3y5d+3c+bZ8rzfefMdUPzRkOD5pW2agG8h5107K1pt6T67zz9NzcdHUu7am0lTvSgPetQN9tky1/GCvd+3RN4+bep970TnetbmEf+6iJPUb9slQyZa/VixFpvo48q8vxLa8tv4+/wy2Qt6WAxgl/NddljLk4yX8tpEzIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEGdtFE8yVaZk2m95lnAdV7JF8ZTckH9x2jbPZ1/oH/WilH8chyT92/b/4107ONBn6n3e/EtN9cWif3xLT58h7kNS/fTzvGtzg7btrKz0X0sist2VypK2qJekCv69M1Wm3sWC/22re9AWl3Po8FHv2swUWzxRvugfw1RK+u8/SVKZLbrHJQ2329h27Ctq/fdLbbra1DuT8Y++Sjj/fZL3jMjiDAgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQxFmbBVdeVaFUxi9fyUWxd1/n/GslyZLaVJIhN06SUkXv0lkX1JhaZyb1eNdu277Z1Pvw7zpM9Rf9ryXetVVZWxZcIumfk9XU3GzqnfHMIpSkfMF27Af7bZlqlRnDLTGuNfUuFf2z4Hr7bce+q2fQu3ZyusLUu7au3ru2P+mfRyhJiUr/jDRJSqb8M9jihO02Hjv/Yx+XbLfDpPN/DIrz/rfZKPbL3uMMCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQxFkbxZOZVK50edqrNpYhiidypnVEhugeS60kRYmkd20i7V8rSdkG/+1snmfr/dtXdtjWUlPlXdt03gWm3n15/xiZfNF2c0+npnjXRkn/SBNJKgz1murl/I9nRcbWOnYl79rJWf/4G0lS0X+/OGeLkSkv94/uKZTb7vcu7ffYM1wv/50e2x4mFA/578PIUCtJruh/O4wHu/z7Dua96jgDAgAEwQACAARhHkAvvPCCrrnmGjU1NSmKIj3++OMjrr/pppsURdGIy4oVK0ZrvQCACcI8gPr7+7Vo0SKtX7/+pDUrVqzQ4cOHhy+PPPLIaS0SADDxmF+EsHLlSq1cufJDazKZjBobG095UQCAiW9MngPaunWr6uvrNX/+fN1+++06evToSWvz+bx6enpGXAAAE9+oD6AVK1boRz/6kbZs2aK/+7u/U3t7u1auXKlS6cQv9Wxra1M2mx2+NBs/tRIAMD6N+vuAbrzxxuF/X3LJJVq4cKHmzp2rrVu3aunSpR+oX7dundauXTv8dU9PD0MIAD4Gxvxl2HPmzNHUqVO1d+/eE16fyWRUU1Mz4gIAmPjGfAC99dZbOnr0qKZPnz7WPwoAMI6Y/wTX19c34mxm//792rVrl+rq6lRXV6d7771Xq1atUmNjo/bt26evfOUrOu+887R8+fJRXTgAYHwzD6AdO3bos5/97PDX7z1/s3r1am3YsEG7d+/WP/7jP6qrq0tNTU26+uqr9dd//dfKZGwBVcmyhJJlfidozhCu5CLTMgwpc1KUsIU8uZL/CWjNpDmm3jVT/DO7Eql9pt79vYdN9b/a+Qvv2qIrmHrPPN9/v5QZM9KGSoa7h7M1T5elTPVvdxzwrh3K2V5Jmoz893ldrX+unyTNOn+Wd20ubTv2fSm/vDFJitP+uXGSLV/y3f+Q868dsm1nsjjgX2yplRQV/dcSxf6ZgSXPx2TzALryyivlPiQY8ZlnnrG2BAB8DJEFBwAIggEEAAiCAQQACIIBBAAIggEEAAiCAQQACIIBBAAIggEEAAiCAQQACIIBBAAIYtQ/D2i0RIl3L1618g94iyJjGJxBHNnyvaKSf95Uf9+gqfc5hvTxSVW2fTI5Y/vIjKNN/jlp+157y9S7PJ30rq2uP27q3Zd707u2snyKqfc7R98x1R87fvJPFX6/qZNta8mkyr1rB8tsWWPdyV7v2qjafx2SpKT//S0y5q8ljPVxwX+/REOG3DhJUcl/Lcm4aOqtD4lVe7+S4bHTt5QzIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEGdvFI/84xxs/KMnhhfi3do2z6OE/1qKBVsUz9Cgf8xP0+SLTL0nT59nqj+3rtK7drBnm6n3b179tXdt9lCjqXcuLnnXNp3vHzckSZOm2Oqz9f7xR9Mb/WOYJKmsrMq7ts/1mHonqv2jkpxsMTJRbsi7dqin39S7zNnub2XOEFHkbDE/kfxvh5ZYMklypnr/xxRfnAEBAIJgAAEAgmAAAQCCYAABAIJgAAEAgmAAAQCCYAABAIJgAAEAgmAAAQCCYAABAIJgAAEAgjh7s+AS7168OEu+my0Lzjn//KM48s+mMivaflfoO+q/lpde2WPqffxop6k+Ve2fe1ZW6Z8dJkn9vf75YXHhmKl3zbRy79pMyj/vTpKqs/69Jang8t61fTpiW4shry1dljL1LuX9c8/eOWBb9+u/es27dsYUWz7elGmTTPXl1f6PK1GZ7TEoTvo/BjnjOYXpodP02OlXyxkQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACCIszeKxzlFnjkRUcIQg5G0xWCUpfxjSsoythiZhPOf/0OD/nEcktR3zD8u560DtiieeRfOM9Wnsv6xJsfe6jD1rq33jxw6duSoqXeqf4p3ba7XduwzVaZyTZpS411bVWeLkVFZv3dpYdA/EkiScsdy3rVH3zho6v27V/Z518ZZ/22UpNRFc0z1FRXV3rXJtO33/thF3rUu8q+VJEX+j4eJMajlDAgAEIRpALW1tenSSy9VdXW16uvrdd1112nPnpG/PedyObW2tmrKlCmqqqrSqlWr1NlpC68EAEx8pgHU3t6u1tZWbd++Xc8++6yKxaKuvvpq9ff/9+ntXXfdpSeffFKPPfaY2tvbdejQIV1//fWjvnAAwPhmeg7o6aefHvH1xo0bVV9fr507d+qKK65Qd3e3HnroIW3atElXXXWVJOnhhx/WBRdcoO3bt+tTn/rU6K0cADCundZzQN3d3ZKkuro6SdLOnTtVLBa1bNmy4ZoFCxZo5syZ2rZt2wl75PN59fT0jLgAACa+Ux5AcRzrzjvv1OWXX66LL75YktTR0aF0Oq3a2toRtQ0NDeroOPGrm9ra2pTNZocvzc3Np7okAMA4csoDqLW1VS+//LIeffTR01rAunXr1N3dPXw5eND2UkwAwPh0Su8DWrNmjZ566im98MILmjFjxvD3GxsbVSgU1NXVNeIsqLOzU42NjSfslclklMn4f2QzAGBiMJ0BOee0Zs0abd68Wc8//7xmz5494vrFixcrlUppy5Ytw9/bs2ePDhw4oJaWltFZMQBgQjCdAbW2tmrTpk164oknVF1dPfy8TjabVUVFhbLZrG6++WatXbtWdXV1qqmp0R133KGWlhZeAQcAGME0gDZs2CBJuvLKK0d8/+GHH9ZNN90kSfrud7+rRCKhVatWKZ/Pa/ny5frhD384KosFAEwcpgHkPLLZysvLtX79eq1fv/6UFyVJmaqEMhV+fyGcVFXu3bc6618rSRWVae/ayoztNR0lz6w7Scr12zLs3t57yLt2siGrTZLmnn+BqT5f8M8De+e3r5t6nzuv3ru2GPeaeufz/jlz6Um2p1OzU237fFJdrXdtvmjLDRzsOeZdm+uzZapFQ/73icbGrKl34rJLvGv73/E/lpJUiG2ZagOFQe/ayirb8XGGLDhDXJskKTYsxRmesYk9a8mCAwAEwQACAATBAAIABMEAAgAEwQACAATBAAIABMEAAgAEwQACAATBAAIABMEAAgAEcUofx3AmzLpwiiqq/D6mobK8wrtv5GwxGPFQybt2qFg09e7uHfCuzfUWTL17ev1jZ6ZOOfFHZZxMearSVP/6a3u9a/v7bPuwqm6ad+30WbYYptf/yz8WyMk/bkiSamtqTfXvHPOPy+nt7TP1jkv+t/FEZNuHqaR/73S1LUdm1nz/GKaBev9ILUkqk+0jYmprkt61/YVOU28lLfvFf39LkpN/79j5j4vYsy1nQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgztosOBc7xZ6BQseP9Xj3zedsmWqDA/4ZX4WcLYcpjvzzo+pqaky9U2n/fdLfY8sxe+2V10z173T4Z1+lM7acuVLSfx9m6237cL5metcee+OQqffLfbbcs0nTqr1r48h2G08mUt61kSmXTJLzz/aLE0Om1lEy8q6tnGzLghs8blxL5H+7TZXZbof5obz/Ooz70JX87/tR7H8so5JfLWdAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgztoongO/OaZMhV98RrHkH4GTKLNtcjqT8a5NZWzzvLzKfy3lxt654lHv2uqaClPvVHLAVJ8f8I8FylTbYkok/xgZpfxjeyQpO22ad23n4f2m3sd+122qv6R5lndtMmOMY4n8Y2oSCds+jF3sX2zsbQm+ssQNSVJmUpWpfjBtWHtmiqm3DI9vceQflyNJKhpiuIr+x9JFfvFBnAEBAIJgAAEAgmAAAQCCYAABAIJgAAEAgmAAAQCCYAABAIJgAAEAgmAAAQCCYAABAIJgAAEAgjhrs+CGhhJKDPnNx4oa/9ymiupy0zrKDPlhUcKWwaWo4F1aGjJkNkmKh/yymCQpU5M19a6u9M/Hk6RyQ56eq5hs6p0on+TfW7asMVdR51077/Lppt65AVueXt+A//EsV6WpdzITedc6SwCbJMX+v+MOyZAbJymKnP8yItvCXZl/b0nq6/K/75cl/bP3JKmyxv945uJBU+/IkKXoEv7rHor8+nIGBAAIwjSA2tradOmll6q6ulr19fW67rrrtGfPnhE1V155paIoGnG57bbbRnXRAIDxzzSA2tvb1draqu3bt+vZZ59VsVjU1Vdfrf7+/hF1t9xyiw4fPjx8uf/++0d10QCA8c/0HNDTTz894uuNGzeqvr5eO3fu1BVXXDH8/crKSjU2No7OCgEAE9JpPQfU3f3uh2rV1Y18svbHP/6xpk6dqosvvljr1q3TwIc84ZrP59XT0zPiAgCY+E75VXBxHOvOO+/U5Zdfrosvvnj4+1/4whc0a9YsNTU1affu3frqV7+qPXv26Gc/+9kJ+7S1tenee+891WUAAMapUx5Ara2tevnll/XLX/5yxPdvvfXW4X9fcsklmj59upYuXap9+/Zp7ty5H+izbt06rV27dvjrnp4eNTc3n+qyAADjxCkNoDVr1uipp57SCy+8oBkzZnxo7ZIlSyRJe/fuPeEAymQyyhjeJwIAmBhMA8g5pzvuuEObN2/W1q1bNXv27I/8P7t27ZIkTZ9ue6MeAGBiMw2g1tZWbdq0SU888YSqq6vV0dEhScpms6qoqNC+ffu0adMm/eEf/qGmTJmi3bt366677tIVV1yhhQsXjskGAADGJ9MA2rBhg6R332z6Pz388MO66aablE6n9dxzz+mBBx5Qf3+/mpubtWrVKn39618ftQUDACYG85/gPkxzc7Pa29tPa0HvqW3IKuOZORZl/DO+UinbK8/Lkv45WaW8LW+qUPDPgitL2tY9VPKvP3LU9tL3dNo/e0+S6qb5Z6r1GPa3JE2q9N/nccl2fI539XnXppO25zHf/u2bpvqXtr3iXfvJyy4z9Z49v967dsjZssYSCf/jacl2k6TIcJeIbDcrJRL+GWmSlO/0vy8X4ymm3rMX+b8o68DRI6befSX/jMFi5H+/Lw769SULDgAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQxCl/HtBYc8mSXJlfdErsYu++ZTJGbAz4R2z0H+8y9U4YYoHKKipMvVPl/rEZA122eJUjb3eZ6tMV/jez2qx/rJIkNTb475ehgSFT7+Nvd3jX5ou241NTU26qn1TpX18YsB3PdOR/fIrGOKM44X8bN6blKPqIaLD/yZAIJEkaGiqa6uvqpnnX1lbPN/UuK/OPsiqp2tS733CfiOW/v4s5v1rOgAAAQTCAAABBMIAAAEEwgAAAQTCAAABBMIAAAEEwgAAAQTCAAABBMIAAAEEwgAAAQTCAAABBnLVZcKl0Rul0xqs2MqRIubwty+rYoXe8a6PIPytJkjLpSu/awYItx6yQ919LnLMFZR0bPGaqr23yz6WbUl1j6l2Ie7xrB3P+uX6SlCjzv3vkCrbbVVWtLZNw/kXneNce+d3bpt79Ped616aqbZl3ueKAd23SGthmuL+5yNa7lPDPl5SkoSjnX1y0HZ/Ow8e9a7uKtnXHKf+MwYQhBzCR8KvlDAgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEMRZG8XjiiW5omecQ5T07psu86+VpOnn1HvXRsb4jnzRP77jeEeXqXfPsX7vWtdni/mpqLbFmlRnp3rXZtJTTL2P9XZ41xbyeVPv8gr/2JlC/6Cpdylh2+d1U/33y5G3/I+9JL21b7937fyF80y9c8Vu79pS0nb/SUZ+UV3/f3dTb5ey/W5eKvOPeep3R229Y/+H6UlJ430zXfSujWP/WKVC5Lc/OAMCAATBAAIABMEAAgAEwQACAATBAAIABMEAAgAEwQACAATBAAIABMEAAgAEwQACAATBAAIABHHWZsEdeO2g0uUpr9pkwj/fLZWxZcFlMv7ZStbekyr8tk+SqhL+tZLUNM0/wy6f7jH1jlP+uVeSlJ3qvxYlK029ayY3edeW19v2Yb7bPz/sWOXbpt6DA7bsuGKvf+5Zuef95j0dh970rp1cV2vqPeWcKu/annyXqbcz3N0SkS1nLoqcqT5p2OVRZMsB7Os67l2bssUdqqEp613bG/vfZuOYLDgAwFnMNIA2bNighQsXqqamRjU1NWppadHPf/7z4etzuZxaW1s1ZcoUVVVVadWqVers7Bz1RQMAxj/TAJoxY4a+9a1vaefOndqxY4euuuoqXXvttXrllVckSXfddZeefPJJPfbYY2pvb9ehQ4d0/fXXj8nCAQDjm+k5oGuuuWbE13/7t3+rDRs2aPv27ZoxY4Yeeughbdq0SVdddZUk6eGHH9YFF1yg7du361Of+tTorRoAMO6d8nNApVJJjz76qPr7+9XS0qKdO3eqWCxq2bJlwzULFizQzJkztW3btpP2yefz6unpGXEBAEx85gH061//WlVVVcpkMrrtttu0efNmXXjhhero6FA6nVZtbe2I+oaGBnV0nPxTK9va2pTNZocvzc3N5o0AAIw/5gE0f/587dq1Sy+++KJuv/12rV69Wq+++uopL2DdunXq7u4evhw8ePCUewEAxg/z+4DS6bTOO+88SdLixYv1n//5n/re976nG264QYVCQV1dXSPOgjo7O9XY2HjSfplMRpmM5bPdAQATwWm/DyiOY+XzeS1evFipVEpbtmwZvm7Pnj06cOCAWlpaTvfHAAAmGNMZ0Lp167Ry5UrNnDlTvb292rRpk7Zu3apnnnlG2WxWN998s9auXau6ujrV1NTojjvuUEtLC6+AAwB8gGkAHTlyRH/yJ3+iw4cPK5vNauHChXrmmWf0B3/wB5Kk7373u0okElq1apXy+byWL1+uH/7wh6e0sEJPLFfwi8+wnMblk0XTOvoThtgMY8RGddY/v6OpYZqtd5N/pE3HQJep91AmbapXynCEkrYskYT8I1Pyg7Zjn6rwz3ppmlVn6p3rtkXDbHv+V9612aztL+uzzpntXXv4rTdMvatr5nvXZrO2fdiV6/KuLau2xRNFCVsUT6nY6987zpl6Vxsivmqra029yxL+981I/rFkvrWmW+pDDz30odeXl5dr/fr1Wr9+vaUtAOBjiCw4AEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEOY07LHm3LsRGMW8f2xKZJijUdIWgRKNYRRPwZBokxsomHqX5f33ST5ni6gpOdvvLbl+Q7xO0da7JP/j6Yb8o0QkqZT0j0BJGo99bsB2Oyzm/fsPFWxrsdzXhorG23je/3Yb5WzxNwXD7TYqs/W2RvHEhsMZOVvvRNH/dpgv2aKsLPdly7Es5N6tdR+xrZH7qIoz7K233uJD6QBgAjh48KBmzJhx0uvPugEUx7EOHTqk6upqRdF//8ba09Oj5uZmHTx4UDU1NQFXOLbYzonj47CNEts50YzGdjrn1Nvbq6amJiU+JPD0rPsTXCKR+NCJWVNTM6EP/nvYzonj47CNEts50Zzudmaz2Y+s4UUIAIAgGEAAgCDGzQDKZDK65557lMlkQi9lTLGdE8fHYRsltnOiOZPbeda9CAEA8PEwbs6AAAATCwMIABAEAwgAEAQDCAAQxLgZQOvXr9e5556r8vJyLVmyRP/xH/8Rekmj6pvf/KaiKBpxWbBgQehlnZYXXnhB11xzjZqamhRFkR5//PER1zvndPfdd2v69OmqqKjQsmXL9Prrr4dZ7Gn4qO286aabPnBsV6xYEWaxp6itrU2XXnqpqqurVV9fr+uuu0579uwZUZPL5dTa2qopU6aoqqpKq1atUmdnZ6AVnxqf7bzyyis/cDxvu+22QCs+NRs2bNDChQuH32za0tKin//858PXn6ljOS4G0E9+8hOtXbtW99xzj371q19p0aJFWr58uY4cORJ6aaPqoosu0uHDh4cvv/zlL0Mv6bT09/dr0aJFWr9+/Qmvv//++/X9739fDz74oF588UVNmjRJy5cvVy6XO8MrPT0ftZ2StGLFihHH9pFHHjmDKzx97e3tam1t1fbt2/Xss8+qWCzq6quvVn9//3DNXXfdpSeffFKPPfaY2tvbdejQIV1//fUBV23ns52SdMstt4w4nvfff3+gFZ+aGTNm6Fvf+pZ27typHTt26KqrrtK1116rV155RdIZPJZuHLjssstca2vr8NelUsk1NTW5tra2gKsaXffcc49btGhR6GWMGUlu8+bNw1/HcewaGxvdt7/97eHvdXV1uUwm4x555JEAKxwd799O55xbvXq1u/baa4OsZ6wcOXLESXLt7e3OuXePXSqVco899thwzX/91385SW7btm2hlnna3r+dzjn3+7//++7P//zPwy1qjEyePNn9/d///Rk9lmf9GVChUNDOnTu1bNmy4e8lEgktW7ZM27ZtC7iy0ff666+rqalJc+bM0Re/+EUdOHAg9JLGzP79+9XR0THiuGazWS1ZsmTCHVdJ2rp1q+rr6zV//nzdfvvtOnr0aOglnZbu7m5JUl1dnSRp586dKhaLI47nggULNHPmzHF9PN+/ne/58Y9/rKlTp+riiy/WunXrNDAwEGJ5o6JUKunRRx9Vf3+/WlpazuixPOvCSN/vnXfeUalUUkNDw4jvNzQ06LXXXgu0qtG3ZMkSbdy4UfPnz9fhw4d177336jOf+YxefvllVVdXh17eqOvo6JCkEx7X966bKFasWKHrr79es2fP1r59+/RXf/VXWrlypbZt26ak4TOHzhZxHOvOO+/U5ZdfrosvvljSu8cznU6rtrZ2RO14Pp4n2k5J+sIXvqBZs2apqalJu3fv1le/+lXt2bNHP/vZzwKu1u7Xv/61WlpalMvlVFVVpc2bN+vCCy/Url27ztixPOsH0MfFypUrh/+9cOFCLVmyRLNmzdJPf/pT3XzzzQFXhtN14403Dv/7kksu0cKFCzV37lxt3bpVS5cuDbiyU9Pa2qqXX3553D9H+VFOtp233nrr8L8vueQSTZ8+XUuXLtW+ffs0d+7cM73MUzZ//nzt2rVL3d3d+qd/+ietXr1a7e3tZ3QNZ/2f4KZOnapkMvmBV2B0dnaqsbEx0KrGXm1trc4//3zt3bs39FLGxHvH7uN2XCVpzpw5mjp16rg8tmvWrNFTTz2lX/ziFyM+NqWxsVGFQkFdXV0j6sfr8TzZdp7IkiVLJGncHc90Oq3zzjtPixcvVltbmxYtWqTvfe97Z/RYnvUDKJ1Oa/HixdqyZcvw9+I41pYtW9TS0hJwZWOrr69P+/bt0/Tp00MvZUzMnj1bjY2NI45rT0+PXnzxxQl9XKV3P/X36NGj4+rYOue0Zs0abd68Wc8//7xmz5494vrFixcrlUqNOJ579uzRgQMHxtXx/KjtPJFdu3ZJ0rg6nicSx7Hy+fyZPZaj+pKGMfLoo4+6TCbjNm7c6F599VV36623utraWtfR0RF6aaPmL/7iL9zWrVvd/v373b/927+5ZcuWualTp7ojR46EXtop6+3tdS+99JJ76aWXnCT3ne98x7300kvuzTffdM45961vfcvV1ta6J554wu3evdtde+21bvbs2W5wcDDwym0+bDt7e3vdl7/8Zbdt2za3f/9+99xzz7nf+73fc/PmzXO5XC700r3dfvvtLpvNuq1bt7rDhw8PXwYGBoZrbrvtNjdz5kz3/PPPux07driWlhbX0tIScNV2H7Wde/fudffdd5/bsWOH279/v3viiSfcnDlz3BVXXBF45TZf+9rXXHt7u9u/f7/bvXu3+9rXvuaiKHL/8i//4pw7c8dyXAwg55z7wQ9+4GbOnOnS6bS77LLL3Pbt20MvaVTdcMMNbvr06S6dTrtzzjnH3XDDDW7v3r2hl3VafvGLXzhJH7isXr3aOffuS7G/8Y1vuIaGBpfJZNzSpUvdnj17wi76FHzYdg4MDLirr77aTZs2zaVSKTdr1ix3yy23jLtfnk60fZLcww8/PFwzODjo/uzP/sxNnjzZVVZWus997nPu8OHD4RZ9Cj5qOw8cOOCuuOIKV1dX5zKZjDvvvPPcX/7lX7ru7u6wCzf60z/9Uzdr1iyXTqfdtGnT3NKlS4eHj3Nn7ljycQwAgCDO+ueAAAATEwMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEMT/B5JVDaVYFj2wAAAAAElFTkSuQmCC\n" + }, + "metadata": {} + } + ], + "source": [ + "plt.imshow(x_train[800])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "4arvJDYwfsAj", + "outputId": "a355a4e2-c1a7-461e-bff9-059daaa6a9f7" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(32, 32, 3)" + ] + }, + "metadata": {}, + "execution_count": 7 + } + ], + "source": [ + "x_train[0].shape" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ndeZ_RH32Upu" + }, + "source": [ + "(32, 32, 3) represents an image of size 32x32 in the RGB scale" + ] + }, + { + "cell_type": "markdown", + "source": [ + "### Preprocessing" + ], + "metadata": { + "id": "L2pg1uxSXPHn" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Standardization** is the process of transforming the pixel values of an image to have zero mean and unit variance. This brings the pixel values to a similar scale and makes them easier to work with." + ], + "metadata": { + "id": "Hwwm-EHhW0rC" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ZlInmab9MD-N" + }, + "outputs": [], + "source": [ + "x_train = x_train/255.0" + ] + }, + { + "cell_type": "markdown", + "source": [ + "**Normalization** is the process of scaling the pixel values to a specified range, typically between 0 and 1. This improves the consistency of images." + ], + "metadata": { + "id": "6GFdU-HZWztg" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 447 + }, + "id": "TLmsgV9_Wij5", + "outputId": "03fb00c5-efb9-421c-ef55-bbd36679dfbe" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 9 + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
    " + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGdCAYAAABU0qcqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvsklEQVR4nO3df3Bc5Xn3/8/Z1e5KsqSVZVuShWVjG2Pzy85TBxwNCSXYxXanDARPB5LM1KR8YaAyU3DTJO4kEGg7SslMQpJxzB+luHkmhoQ+MQx8GyiYWDStTWsHPw5QHOwYbGJLBtv6rf2hPff3D76oFdhwX7bk2xLv18zOWNrLl+5zzu5eOtrdz0bOOScAAM6wROgFAAA+nhhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgykIv4P3iONahQ4dUXV2tKIpCLwcAYOScU29vr5qampRInPw856wbQIcOHVJzc3PoZQAATtPBgwc1Y8aMk14/ZgNo/fr1+va3v62Ojg4tWrRIP/jBD3TZZZd95P+rrq6WJP3R//NHSqVTXj+r+UL/gRWXFb1rJSlpOAlLRra/aCaU9K6NPuS3iBP2Npw9WrOYnItN9aXYv75kXExs+A+leMjU27JuOdvxcbKd3VsSs1xs24mx4Xhak7uGTMfetg+T6bR3bWVVjal3ZXayqT5K+B/PQq7X1Hugu8u7tpQrmHonDIfT8hBUyBX0v//q4eHH85MZkwH0k5/8RGvXrtWDDz6oJUuW6IEHHtDy5cu1Z88e1dfXf+j/fe/Pbql0SqmM3wDKVGS81xaXGW/kYzmAIgbQB2tta7ENIP/9/W59ybvWOVvvsR1Atp04lgMoMaYDyP9+n6ksN/XOVFaY6iPDA0WUsP0SPFTw386S8VkL2wCyPyXyUU+jjMmLEL7zne/olltu0Ze+9CVdeOGFevDBB1VZWal/+Id/GIsfBwAYh0Z9ABUKBe3cuVPLli377x+SSGjZsmXatm3bB+rz+bx6enpGXAAAE9+oD6B33nlHpVJJDQ0NI77f0NCgjo6OD9S3tbUpm80OX3gBAgB8PAR/H9C6devU3d09fDl48GDoJQEAzoBRfxHC1KlTlUwm1dnZOeL7nZ2damxs/EB9JpNRJuP/JBsAYGIY9TOgdDqtxYsXa8uWLcPfi+NYW7ZsUUtLy2j/OADAODUmL8Neu3atVq9erU9+8pO67LLL9MADD6i/v19f+tKXxuLHAQDGoTEZQDfccIPefvtt3X333ero6NAnPvEJPf300x94YQIA4ONrzJIQ1qxZozVr1pzy/y8NlT40Q+h/coY3uEcJ2yYnk/5/pUw4vzfOvseV/HsX8rY3rxUM74hOGP8Sm0zb3nSZKPd/x3osW1pBXPLfTssbLiXJ8p5L6xs0rW//jQz1LjL2trzJVbZ9mLDUO/83/kpSPOh/7PtyA6beg319pvrKyXXeteWTKk29s9P87z8547oHe/1TGUpD/vdN3/eHB38VHADg44kBBAAIggEEAAiCAQQACIIBBAAIggEEAAiCAQQACIIBBAAIggEEAAiCAQQACGLMonhOl4uScgm/yBdX8t+Mw7/94IfifWjvgn9MSUV5jan3gTcPedfu+81vTb0tHw4/KV1laj15StZUf+78Gd6155w7xdQ79k8pUUmDpt4f8XH2I2tNnU+F/09IGFcTG9JyEiVbDFNsiBAqyRbFEyWt8Uf+XN4/okaSBo75R2UVS7bbeHlFtX9t5WRT70Tk/1E4+XzOuzZO5P1+vndHAABGEQMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABDEWZsFN1jIa0h+IVXd/V3efTve9s9fk6RCj38+VZyvMPU+bMiCSydsuVfHjh71ru2LbblXcdGWNdbxu1951y78xAJT7zkXNXvXpqptv2/lXJ93bTJhCFSTpMiWqSbnv/aEIX9NkspMeW22Yz9kKffMfhzm/Pd5bAm8k6SE7bZi6Z/r9b9dSdJQzr93WcL4kB4Zjr2hdykx5FXHGRAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIIizNoqnPJ1SKpPyq53kP0cv+l/zbOtIVnvXZsvPNfV+8zeHvWtf3bXT1HvmzJnetVXV5abe6cwkU/1rr77hX/t/99rWkkh7157/ybm23qmMd21B/abepcgvquQ9LvKPhIpiW1xOQv4ROHFki7RJGJaScLZ1m3J+jL1LxqXIEGeUGMrbOg/5H/uSMc7IlfnXl1X43x+SnrcTzoAAAEEwgAAAQTCAAABBMIAAAEEwgAAAQTCAAABBMIAAAEEwgAAAQTCAAABBMIAAAEEwgAAAQZy1WXDOleRiv/kYGXKeXOSf2SRJQ27Qu7arcMDUu35eo3dt7+AnTL3f3LPfu/bYgU5T74F+W15beabKuzY3YMtUe3nnr71rB3r7TL1nzZ/lXVvXPM3Uu5jqMdXnXJd3rUvafq+M5Z+nZ7z7KCH/HDNn7F2W8t/OMkPm2btrsWXeWbLgJGNvVzQU2zIGS85/HyYNj7NJzzVzBgQACGLUB9A3v/lNRVE04rJgwYLR/jEAgHFuTP4Ed9FFF+m555777x9Sdtb+pQ8AEMiYTIaysjI1Nvo/vwEA+PgZk+eAXn/9dTU1NWnOnDn64he/qAMHTv7kfD6fV09Pz4gLAGDiG/UBtGTJEm3cuFFPP/20NmzYoP379+szn/mMent7T1jf1tambDY7fGlubh7tJQEAzkKjPoBWrlypP/7jP9bChQu1fPly/fM//7O6urr005/+9IT169atU3d39/Dl4MGDo70kAMBZaMxfHVBbW6vzzz9fe/ee+L0jmUxGmYz/Z40DACaGMX8fUF9fn/bt26fp06eP9Y8CAIwjoz6AvvzlL6u9vV1vvPGG/v3f/12f+9znlEwm9fnPf360fxQAYBwb9T/BvfXWW/r85z+vo0ePatq0afr0pz+t7du3a9o0W1RJvr+gUtEv3mLIkD6RMMR3SFIp8o/NKLmjtt5D/r3PmTPT1Hugxz8C5a3X/eOGJCkeskXaxMm8d+3AoO1VkN1d/vv8+DHb8el8yz+i6LwL5pl6z77kHFN9de1U79r+0olf8HMyLvK/rcSG+4Nk+w03NibURJF/NIxZbO09hmsxxfwY84wM9UM5/8eJobzffX7UB9Cjjz462i0BABMQWXAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCDG/OMYTlUhFysueWZUFVL+jY1bHCf9g+aSyaStufPP4Kqqti18cl2Nd23fZP9aSaqfYtvOQt4/m6z7+DFT79xgv3ftQM4WNvbmmyf/JN/3O9Lxtql3Z+f5pvrFV3zCu7aqvs7Uu6/kv3brb6xx5P8/ooQtx8xSHTlbVptLGrfU0D+y5rVF1nw3fwlD5p2L/fdJwrOWMyAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBBnbxTPYFHxkF8ERfOUud59Z82bZVrHnjde9q493n/E1LuU8I+RKUVHTb2nTpvsXdv39jRT79KQLdaksmqSd+1FmUpT79/u2eNdWyzaonhyQ/5RSf1D/sdSkt7Y96apPpnwjz9a+JkFpt5lk/2PTykaNPW2/Ioby3Z8IucfUZMw1EoyReu8uxhj/RiJnG0fWrjI//6Q8KzlDAgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQxFmbBVcs5RQn/Jb3q50vefd1LmNax/xzL/OuLSb6TL33/s5/3YMDx029yyr8t7Oq1j8LTJJ6evwzoSSpp7/XuzabbTD1Pv+Scu/awwd/a+ptiffq6R4w9Y6LRVN9X5f/8X/7zbdNvWeUN3rXugrbukuJvHdtQra8Nkv6WtL5Z+lJkotsmWq2eltunDPUO2c7pygZdvmQ4XxlKPKr5QwIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEMRZmwVXXTtJZWm/5cWZQe++z/3r/2taR/pf/XPS5s+7wNR77oK53rX5clvW2PHj/hlcqeoqU+/qshpTfVH+2WTHertMvcucfxacS9vWPZTr966tqKg29Y4qTeXKFfyP/zuH3jH1rs1O9q7Nzsmaepfibu9al7DlzEmG/DVLsJ8kyZgdl/DvHxtz6WLDw3RsXHexzL930bCOwlDOq44zIABAEOYB9MILL+iaa65RU1OToijS448/PuJ655zuvvtuTZ8+XRUVFVq2bJlef/310VovAGCCMA+g/v5+LVq0SOvXrz/h9ffff7++//3v68EHH9SLL76oSZMmafny5crl/E7JAAAfD+bngFauXKmVK1ee8DrnnB544AF9/etf17XXXitJ+tGPfqSGhgY9/vjjuvHGG09vtQCACWNUnwPav3+/Ojo6tGzZsuHvZbNZLVmyRNu2bTvh/8nn8+rp6RlxAQBMfKM6gDo6OiRJDQ0jP9WyoaFh+Lr3a2trUzabHb40NzeP5pIAAGep4K+CW7dunbq7u4cvBw8eDL0kAMAZMKoDqLHx3c+W7+zsHPH9zs7O4eveL5PJqKamZsQFADDxjeoAmj17thobG7Vly5bh7/X09OjFF19US0vLaP4oAMA4Z34VXF9fn/bu3Tv89f79+7Vr1y7V1dVp5syZuvPOO/U3f/M3mjdvnmbPnq1vfOMbampq0nXXXTea6wYAjHPmAbRjxw599rOfHf567dq1kqTVq1dr48aN+spXvqL+/n7deuut6urq0qc//Wk9/fTTKi/3j0yRpKpshVLplFdt42z/eJD62bbIlJ6j/nEsu17+V1Pv1/b8X+/azyy7ytT7/S8E+TCDuSOm3oP+6SqSpGmJE//59URSqQrbWgb6vGur6/xuT+/J93V51yZdwdS7ttY/4kmSBgb9Xx06OOgfTSVJv3vjkHdttv4iU+9s1n+f95T8I5skKU6W/GuNETUlZ7utlAzxOiXjWoZi/4fpfMF/n0hSPucf2VUs+N/xi559zQPoyiuvlHPupNdHUaT77rtP9913n7U1AOBjJPir4AAAH08MIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBDmKJ4zJY5ixVHsVVtyfnWSVJY8eYzQiUxp9M+Zy2b9ayXpzd8c8K594mebTL0vv3y5d+3c+bZ8rzfefMdUPzRkOD5pW2agG8h5107K1pt6T67zz9NzcdHUu7am0lTvSgPetQN9tky1/GCvd+3RN4+bep970TnetbmEf+6iJPUb9slQyZa/VixFpvo48q8vxLa8tv4+/wy2Qt6WAxgl/NddljLk4yX8tpEzIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEGdtFE8yVaZk2m95lnAdV7JF8ZTckH9x2jbPZ1/oH/WilH8chyT92/b/4107ONBn6n3e/EtN9cWif3xLT58h7kNS/fTzvGtzg7btrKz0X0sist2VypK2qJekCv69M1Wm3sWC/22re9AWl3Po8FHv2swUWzxRvugfw1RK+u8/SVKZLbrHJQ2329h27Ctq/fdLbbra1DuT8Y++Sjj/fZL3jMjiDAgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQxFmbBVdeVaFUxi9fyUWxd1/n/GslyZLaVJIhN06SUkXv0lkX1JhaZyb1eNdu277Z1Pvw7zpM9Rf9ryXetVVZWxZcIumfk9XU3GzqnfHMIpSkfMF27Af7bZlqlRnDLTGuNfUuFf2z4Hr7bce+q2fQu3ZyusLUu7au3ru2P+mfRyhJiUr/jDRJSqb8M9jihO02Hjv/Yx+XbLfDpPN/DIrz/rfZKPbL3uMMCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQxFkbxZOZVK50edqrNpYhiidypnVEhugeS60kRYmkd20i7V8rSdkG/+1snmfr/dtXdtjWUlPlXdt03gWm3n15/xiZfNF2c0+npnjXRkn/SBNJKgz1murl/I9nRcbWOnYl79rJWf/4G0lS0X+/OGeLkSkv94/uKZTb7vcu7ffYM1wv/50e2x4mFA/578PIUCtJruh/O4wHu/z7Dua96jgDAgAEwQACAARhHkAvvPCCrrnmGjU1NSmKIj3++OMjrr/pppsURdGIy4oVK0ZrvQCACcI8gPr7+7Vo0SKtX7/+pDUrVqzQ4cOHhy+PPPLIaS0SADDxmF+EsHLlSq1cufJDazKZjBobG095UQCAiW9MngPaunWr6uvrNX/+fN1+++06evToSWvz+bx6enpGXAAAE9+oD6AVK1boRz/6kbZs2aK/+7u/U3t7u1auXKlS6cQv9Wxra1M2mx2+NBs/tRIAMD6N+vuAbrzxxuF/X3LJJVq4cKHmzp2rrVu3aunSpR+oX7dundauXTv8dU9PD0MIAD4Gxvxl2HPmzNHUqVO1d+/eE16fyWRUU1Mz4gIAmPjGfAC99dZbOnr0qKZPnz7WPwoAMI6Y/wTX19c34mxm//792rVrl+rq6lRXV6d7771Xq1atUmNjo/bt26evfOUrOu+887R8+fJRXTgAYHwzD6AdO3bos5/97PDX7z1/s3r1am3YsEG7d+/WP/7jP6qrq0tNTU26+uqr9dd//dfKZGwBVcmyhJJlfidozhCu5CLTMgwpc1KUsIU8uZL/CWjNpDmm3jVT/DO7Eql9pt79vYdN9b/a+Qvv2qIrmHrPPN9/v5QZM9KGSoa7h7M1T5elTPVvdxzwrh3K2V5Jmoz893ldrX+unyTNOn+Wd20ubTv2fSm/vDFJitP+uXGSLV/y3f+Q868dsm1nsjjgX2yplRQV/dcSxf6ZgSXPx2TzALryyivlPiQY8ZlnnrG2BAB8DJEFBwAIggEEAAiCAQQACIIBBAAIggEEAAiCAQQACIIBBAAIggEEAAiCAQQACIIBBAAIYtQ/D2i0RIl3L1618g94iyJjGJxBHNnyvaKSf95Uf9+gqfc5hvTxSVW2fTI5Y/vIjKNN/jlp+157y9S7PJ30rq2uP27q3Zd707u2snyKqfc7R98x1R87fvJPFX6/qZNta8mkyr1rB8tsWWPdyV7v2qjafx2SpKT//S0y5q8ljPVxwX+/REOG3DhJUcl/Lcm4aOqtD4lVe7+S4bHTt5QzIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEGdvFI/84xxs/KMnhhfi3do2z6OE/1qKBVsUz9Cgf8xP0+SLTL0nT59nqj+3rtK7drBnm6n3b179tXdt9lCjqXcuLnnXNp3vHzckSZOm2Oqz9f7xR9Mb/WOYJKmsrMq7ts/1mHonqv2jkpxsMTJRbsi7dqin39S7zNnub2XOEFHkbDE/kfxvh5ZYMklypnr/xxRfnAEBAIJgAAEAgmAAAQCCYAABAIJgAAEAgmAAAQCCYAABAIJgAAEAgmAAAQCCYAABAIJgAAEAgjh7s+AS7168OEu+my0Lzjn//KM48s+mMivaflfoO+q/lpde2WPqffxop6k+Ve2fe1ZW6Z8dJkn9vf75YXHhmKl3zbRy79pMyj/vTpKqs/69Jang8t61fTpiW4shry1dljL1LuX9c8/eOWBb9+u/es27dsYUWz7elGmTTPXl1f6PK1GZ7TEoTvo/BjnjOYXpodP02OlXyxkQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACCIszeKxzlFnjkRUcIQg5G0xWCUpfxjSsoythiZhPOf/0OD/nEcktR3zD8u560DtiieeRfOM9Wnsv6xJsfe6jD1rq33jxw6duSoqXeqf4p3ba7XduwzVaZyTZpS411bVWeLkVFZv3dpYdA/EkiScsdy3rVH3zho6v27V/Z518ZZ/22UpNRFc0z1FRXV3rXJtO33/thF3rUu8q+VJEX+j4eJMajlDAgAEIRpALW1tenSSy9VdXW16uvrdd1112nPnpG/PedyObW2tmrKlCmqqqrSqlWr1NlpC68EAEx8pgHU3t6u1tZWbd++Xc8++6yKxaKuvvpq9ff/9+ntXXfdpSeffFKPPfaY2tvbdejQIV1//fWjvnAAwPhmeg7o6aefHvH1xo0bVV9fr507d+qKK65Qd3e3HnroIW3atElXXXWVJOnhhx/WBRdcoO3bt+tTn/rU6K0cADCundZzQN3d3ZKkuro6SdLOnTtVLBa1bNmy4ZoFCxZo5syZ2rZt2wl75PN59fT0jLgAACa+Ux5AcRzrzjvv1OWXX66LL75YktTR0aF0Oq3a2toRtQ0NDeroOPGrm9ra2pTNZocvzc3Np7okAMA4csoDqLW1VS+//LIeffTR01rAunXr1N3dPXw5eND2UkwAwPh0Su8DWrNmjZ566im98MILmjFjxvD3GxsbVSgU1NXVNeIsqLOzU42NjSfslclklMn4f2QzAGBiMJ0BOee0Zs0abd68Wc8//7xmz5494vrFixcrlUppy5Ytw9/bs2ePDhw4oJaWltFZMQBgQjCdAbW2tmrTpk164oknVF1dPfy8TjabVUVFhbLZrG6++WatXbtWdXV1qqmp0R133KGWlhZeAQcAGME0gDZs2CBJuvLKK0d8/+GHH9ZNN90kSfrud7+rRCKhVatWKZ/Pa/ny5frhD384KosFAEwcpgHkPLLZysvLtX79eq1fv/6UFyVJmaqEMhV+fyGcVFXu3bc6618rSRWVae/ayoztNR0lz6w7Scr12zLs3t57yLt2siGrTZLmnn+BqT5f8M8De+e3r5t6nzuv3ru2GPeaeufz/jlz6Um2p1OzU237fFJdrXdtvmjLDRzsOeZdm+uzZapFQ/73icbGrKl34rJLvGv73/E/lpJUiG2ZagOFQe/ayirb8XGGLDhDXJskKTYsxRmesYk9a8mCAwAEwQACAATBAAIABMEAAgAEwQACAATBAAIABMEAAgAEwQACAATBAAIABMEAAgAEcUofx3AmzLpwiiqq/D6mobK8wrtv5GwxGPFQybt2qFg09e7uHfCuzfUWTL17ev1jZ6ZOOfFHZZxMearSVP/6a3u9a/v7bPuwqm6ad+30WbYYptf/yz8WyMk/bkiSamtqTfXvHPOPy+nt7TP1jkv+t/FEZNuHqaR/73S1LUdm1nz/GKaBev9ILUkqk+0jYmprkt61/YVOU28lLfvFf39LkpN/79j5j4vYsy1nQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgztosOBc7xZ6BQseP9Xj3zedsmWqDA/4ZX4WcLYcpjvzzo+pqaky9U2n/fdLfY8sxe+2V10z173T4Z1+lM7acuVLSfx9m6237cL5metcee+OQqffLfbbcs0nTqr1r48h2G08mUt61kSmXTJLzz/aLE0Om1lEy8q6tnGzLghs8blxL5H+7TZXZbof5obz/Ooz70JX87/tR7H8so5JfLWdAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgGEAAgCAYQACAIBhAAIAgztoongO/OaZMhV98RrHkH4GTKLNtcjqT8a5NZWzzvLzKfy3lxt654lHv2uqaClPvVHLAVJ8f8I8FylTbYkok/xgZpfxjeyQpO22ad23n4f2m3sd+122qv6R5lndtMmOMY4n8Y2oSCds+jF3sX2zsbQm+ssQNSVJmUpWpfjBtWHtmiqm3DI9vceQflyNJKhpiuIr+x9JFfvFBnAEBAIJgAAEAgmAAAQCCYAABAIJgAAEAgmAAAQCCYAABAIJgAAEAgmAAAQCCYAABAIJgAAEAgjhrs+CGhhJKDPnNx4oa/9ymiupy0zrKDPlhUcKWwaWo4F1aGjJkNkmKh/yymCQpU5M19a6u9M/Hk6RyQ56eq5hs6p0on+TfW7asMVdR51077/Lppt65AVueXt+A//EsV6WpdzITedc6SwCbJMX+v+MOyZAbJymKnP8yItvCXZl/b0nq6/K/75cl/bP3JKmyxv945uJBU+/IkKXoEv7rHor8+nIGBAAIwjSA2tradOmll6q6ulr19fW67rrrtGfPnhE1V155paIoGnG57bbbRnXRAIDxzzSA2tvb1draqu3bt+vZZ59VsVjU1Vdfrf7+/hF1t9xyiw4fPjx8uf/++0d10QCA8c/0HNDTTz894uuNGzeqvr5eO3fu1BVXXDH8/crKSjU2No7OCgEAE9JpPQfU3f3uh2rV1Y18svbHP/6xpk6dqosvvljr1q3TwIc84ZrP59XT0zPiAgCY+E75VXBxHOvOO+/U5Zdfrosvvnj4+1/4whc0a9YsNTU1affu3frqV7+qPXv26Gc/+9kJ+7S1tenee+891WUAAMapUx5Ara2tevnll/XLX/5yxPdvvfXW4X9fcsklmj59upYuXap9+/Zp7ty5H+izbt06rV27dvjrnp4eNTc3n+qyAADjxCkNoDVr1uipp57SCy+8oBkzZnxo7ZIlSyRJe/fuPeEAymQyyhjeJwIAmBhMA8g5pzvuuEObN2/W1q1bNXv27I/8P7t27ZIkTZ9ue6MeAGBiMw2g1tZWbdq0SU888YSqq6vV0dEhScpms6qoqNC+ffu0adMm/eEf/qGmTJmi3bt366677tIVV1yhhQsXjskGAADGJ9MA2rBhg6R332z6Pz388MO66aablE6n9dxzz+mBBx5Qf3+/mpubtWrVKn39618ftQUDACYG85/gPkxzc7Pa29tPa0HvqW3IKuOZORZl/DO+UinbK8/Lkv45WaW8LW+qUPDPgitL2tY9VPKvP3LU9tL3dNo/e0+S6qb5Z6r1GPa3JE2q9N/nccl2fI539XnXppO25zHf/u2bpvqXtr3iXfvJyy4z9Z49v967dsjZssYSCf/jacl2k6TIcJeIbDcrJRL+GWmSlO/0vy8X4ymm3rMX+b8o68DRI6befSX/jMFi5H+/Lw769SULDgAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQxCl/HtBYc8mSXJlfdErsYu++ZTJGbAz4R2z0H+8y9U4YYoHKKipMvVPl/rEZA122eJUjb3eZ6tMV/jez2qx/rJIkNTb475ehgSFT7+Nvd3jX5ou241NTU26qn1TpX18YsB3PdOR/fIrGOKM44X8bN6blKPqIaLD/yZAIJEkaGiqa6uvqpnnX1lbPN/UuK/OPsiqp2tS733CfiOW/v4s5v1rOgAAAQTCAAABBMIAAAEEwgAAAQTCAAABBMIAAAEEwgAAAQTCAAABBMIAAAEEwgAAAQTCAAABBnLVZcKl0Rul0xqs2MqRIubwty+rYoXe8a6PIPytJkjLpSu/awYItx6yQ919LnLMFZR0bPGaqr23yz6WbUl1j6l2Ie7xrB3P+uX6SlCjzv3vkCrbbVVWtLZNw/kXneNce+d3bpt79Ped616aqbZl3ueKAd23SGthmuL+5yNa7lPDPl5SkoSjnX1y0HZ/Ow8e9a7uKtnXHKf+MwYQhBzCR8KvlDAgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEMRZG8XjiiW5omecQ5T07psu86+VpOnn1HvXRsb4jnzRP77jeEeXqXfPsX7vWtdni/mpqLbFmlRnp3rXZtJTTL2P9XZ41xbyeVPv8gr/2JlC/6Cpdylh2+d1U/33y5G3/I+9JL21b7937fyF80y9c8Vu79pS0nb/SUZ+UV3/f3dTb5ey/W5eKvOPeep3R229Y/+H6UlJ430zXfSujWP/WKVC5Lc/OAMCAATBAAIABMEAAgAEwQACAATBAAIABMEAAgAEwQACAATBAAIABMEAAgAEwQACAATBAAIABHHWZsEdeO2g0uUpr9pkwj/fLZWxZcFlMv7ZStbekyr8tk+SqhL+tZLUNM0/wy6f7jH1jlP+uVeSlJ3qvxYlK029ayY3edeW19v2Yb7bPz/sWOXbpt6DA7bsuGKvf+5Zuef95j0dh970rp1cV2vqPeWcKu/annyXqbcz3N0SkS1nLoqcqT5p2OVRZMsB7Os67l2bssUdqqEp613bG/vfZuOYLDgAwFnMNIA2bNighQsXqqamRjU1NWppadHPf/7z4etzuZxaW1s1ZcoUVVVVadWqVers7Bz1RQMAxj/TAJoxY4a+9a1vaefOndqxY4euuuoqXXvttXrllVckSXfddZeefPJJPfbYY2pvb9ehQ4d0/fXXj8nCAQDjm+k5oGuuuWbE13/7t3+rDRs2aPv27ZoxY4Yeeughbdq0SVdddZUk6eGHH9YFF1yg7du361Of+tTorRoAMO6d8nNApVJJjz76qPr7+9XS0qKdO3eqWCxq2bJlwzULFizQzJkztW3btpP2yefz6unpGXEBAEx85gH061//WlVVVcpkMrrtttu0efNmXXjhhero6FA6nVZtbe2I+oaGBnV0nPxTK9va2pTNZocvzc3N5o0AAIw/5gE0f/587dq1Sy+++KJuv/12rV69Wq+++uopL2DdunXq7u4evhw8ePCUewEAxg/z+4DS6bTOO+88SdLixYv1n//5n/re976nG264QYVCQV1dXSPOgjo7O9XY2HjSfplMRpmM5bPdAQATwWm/DyiOY+XzeS1evFipVEpbtmwZvm7Pnj06cOCAWlpaTvfHAAAmGNMZ0Lp167Ry5UrNnDlTvb292rRpk7Zu3apnnnlG2WxWN998s9auXau6ujrV1NTojjvuUEtLC6+AAwB8gGkAHTlyRH/yJ3+iw4cPK5vNauHChXrmmWf0B3/wB5Kk7373u0okElq1apXy+byWL1+uH/7wh6e0sEJPLFfwi8+wnMblk0XTOvoThtgMY8RGddY/v6OpYZqtd5N/pE3HQJep91AmbapXynCEkrYskYT8I1Pyg7Zjn6rwz3ppmlVn6p3rtkXDbHv+V9612aztL+uzzpntXXv4rTdMvatr5nvXZrO2fdiV6/KuLau2xRNFCVsUT6nY6987zpl6Vxsivmqra029yxL+981I/rFkvrWmW+pDDz30odeXl5dr/fr1Wr9+vaUtAOBjiCw4AEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEOY07LHm3LsRGMW8f2xKZJijUdIWgRKNYRRPwZBokxsomHqX5f33ST5ni6gpOdvvLbl+Q7xO0da7JP/j6Yb8o0QkqZT0j0BJGo99bsB2Oyzm/fsPFWxrsdzXhorG23je/3Yb5WzxNwXD7TYqs/W2RvHEhsMZOVvvRNH/dpgv2aKsLPdly7Es5N6tdR+xrZH7qIoz7K233uJD6QBgAjh48KBmzJhx0uvPugEUx7EOHTqk6upqRdF//8ba09Oj5uZmHTx4UDU1NQFXOLbYzonj47CNEts50YzGdjrn1Nvbq6amJiU+JPD0rPsTXCKR+NCJWVNTM6EP/nvYzonj47CNEts50Zzudmaz2Y+s4UUIAIAgGEAAgCDGzQDKZDK65557lMlkQi9lTLGdE8fHYRsltnOiOZPbeda9CAEA8PEwbs6AAAATCwMIABAEAwgAEAQDCAAQxLgZQOvXr9e5556r8vJyLVmyRP/xH/8Rekmj6pvf/KaiKBpxWbBgQehlnZYXXnhB11xzjZqamhRFkR5//PER1zvndPfdd2v69OmqqKjQsmXL9Prrr4dZ7Gn4qO286aabPnBsV6xYEWaxp6itrU2XXnqpqqurVV9fr+uuu0579uwZUZPL5dTa2qopU6aoqqpKq1atUmdnZ6AVnxqf7bzyyis/cDxvu+22QCs+NRs2bNDChQuH32za0tKin//858PXn6ljOS4G0E9+8hOtXbtW99xzj371q19p0aJFWr58uY4cORJ6aaPqoosu0uHDh4cvv/zlL0Mv6bT09/dr0aJFWr9+/Qmvv//++/X9739fDz74oF588UVNmjRJy5cvVy6XO8MrPT0ftZ2StGLFihHH9pFHHjmDKzx97e3tam1t1fbt2/Xss8+qWCzq6quvVn9//3DNXXfdpSeffFKPPfaY2tvbdejQIV1//fUBV23ns52SdMstt4w4nvfff3+gFZ+aGTNm6Fvf+pZ27typHTt26KqrrtK1116rV155RdIZPJZuHLjssstca2vr8NelUsk1NTW5tra2gKsaXffcc49btGhR6GWMGUlu8+bNw1/HcewaGxvdt7/97eHvdXV1uUwm4x555JEAKxwd799O55xbvXq1u/baa4OsZ6wcOXLESXLt7e3OuXePXSqVco899thwzX/91385SW7btm2hlnna3r+dzjn3+7//++7P//zPwy1qjEyePNn9/d///Rk9lmf9GVChUNDOnTu1bNmy4e8lEgktW7ZM27ZtC7iy0ff666+rqalJc+bM0Re/+EUdOHAg9JLGzP79+9XR0THiuGazWS1ZsmTCHVdJ2rp1q+rr6zV//nzdfvvtOnr0aOglnZbu7m5JUl1dnSRp586dKhaLI47nggULNHPmzHF9PN+/ne/58Y9/rKlTp+riiy/WunXrNDAwEGJ5o6JUKunRRx9Vf3+/WlpazuixPOvCSN/vnXfeUalUUkNDw4jvNzQ06LXXXgu0qtG3ZMkSbdy4UfPnz9fhw4d177336jOf+YxefvllVVdXh17eqOvo6JCkEx7X966bKFasWKHrr79es2fP1r59+/RXf/VXWrlypbZt26ak4TOHzhZxHOvOO+/U5ZdfrosvvljSu8cznU6rtrZ2RO14Pp4n2k5J+sIXvqBZs2apqalJu3fv1le/+lXt2bNHP/vZzwKu1u7Xv/61WlpalMvlVFVVpc2bN+vCCy/Url27ztixPOsH0MfFypUrh/+9cOFCLVmyRLNmzdJPf/pT3XzzzQFXhtN14403Dv/7kksu0cKFCzV37lxt3bpVS5cuDbiyU9Pa2qqXX3553D9H+VFOtp233nrr8L8vueQSTZ8+XUuXLtW+ffs0d+7cM73MUzZ//nzt2rVL3d3d+qd/+ietXr1a7e3tZ3QNZ/2f4KZOnapkMvmBV2B0dnaqsbEx0KrGXm1trc4//3zt3bs39FLGxHvH7uN2XCVpzpw5mjp16rg8tmvWrNFTTz2lX/ziFyM+NqWxsVGFQkFdXV0j6sfr8TzZdp7IkiVLJGncHc90Oq3zzjtPixcvVltbmxYtWqTvfe97Z/RYnvUDKJ1Oa/HixdqyZcvw9+I41pYtW9TS0hJwZWOrr69P+/bt0/Tp00MvZUzMnj1bjY2NI45rT0+PXnzxxQl9XKV3P/X36NGj4+rYOue0Zs0abd68Wc8//7xmz5494vrFixcrlUqNOJ579uzRgQMHxtXx/KjtPJFdu3ZJ0rg6nicSx7Hy+fyZPZaj+pKGMfLoo4+6TCbjNm7c6F599VV36623utraWtfR0RF6aaPmL/7iL9zWrVvd/v373b/927+5ZcuWualTp7ojR46EXtop6+3tdS+99JJ76aWXnCT3ne98x7300kvuzTffdM45961vfcvV1ta6J554wu3evdtde+21bvbs2W5wcDDwym0+bDt7e3vdl7/8Zbdt2za3f/9+99xzz7nf+73fc/PmzXO5XC700r3dfvvtLpvNuq1bt7rDhw8PXwYGBoZrbrvtNjdz5kz3/PPPux07driWlhbX0tIScNV2H7Wde/fudffdd5/bsWOH279/v3viiSfcnDlz3BVXXBF45TZf+9rXXHt7u9u/f7/bvXu3+9rXvuaiKHL/8i//4pw7c8dyXAwg55z7wQ9+4GbOnOnS6bS77LLL3Pbt20MvaVTdcMMNbvr06S6dTrtzzjnH3XDDDW7v3r2hl3VafvGLXzhJH7isXr3aOffuS7G/8Y1vuIaGBpfJZNzSpUvdnj17wi76FHzYdg4MDLirr77aTZs2zaVSKTdr1ix3yy23jLtfnk60fZLcww8/PFwzODjo/uzP/sxNnjzZVVZWus997nPu8OHD4RZ9Cj5qOw8cOOCuuOIKV1dX5zKZjDvvvPPcX/7lX7ru7u6wCzf60z/9Uzdr1iyXTqfdtGnT3NKlS4eHj3Nn7ljycQwAgCDO+ueAAAATEwMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEAQDCAAQBAMIABAEAwgAEMT/B5JVDaVYFj2wAAAAAElFTkSuQmCC\n" + }, + "metadata": {} + } + ], + "source": [ + "x_train = (x_train - np.min(x_train)) / (np.max(x_train) - np.min(x_train))\n", + "plt.imshow(x_train[800])" + ] + }, + { + "cell_type": "markdown", + "source": [ + "**Grayscale Conversion** refers to the conversion of a colored image in RGB scale into a grayscale image. It represents the pixel intensities without considering colors, which makes calculations easier." + ], + "metadata": { + "id": "bfgy0Z_gX_lH" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "P2oPvZkbfEPo" + }, + "outputs": [], + "source": [ + "grayscale = []\n", + "for i in x_train:\n", + " grayImage = 0.07 * i[:,:,2] + 0.72 * i[:,:,1] + 0.21 * i[:,:,0]\n", + " grayscale.append(grayImage)\n", + "x_train_gray = np.asarray(grayscale)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jzQ2Zulg99NU" + }, + "source": [ + "## Defining DoFns for Image Preprocessing\n", + "\n", + "[DoFn](https://beam.apache.org/releases/typedoc/current/interfaces/transforms_pardo.DoFn) stands for \"Do Function\". In Apache Beam, it is a set of operations that can be applied to individual elements of a PCollection (a collection of data). It is similar to a function in Python, except that it is used in Beam Pipelines to apply various transformations. DoFns can be used in various Apache Beam transforms, such as ParDo, Map, Filter, and FlatMap." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cqm-m0cONsZS" + }, + "outputs": [], + "source": [ + "class StandardizeImage(beam.DoFn):\n", + " def process(self, element: np.ndarray):\n", + " element = element/255.0\n", + " return [element]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mZhFCgPxPEwm" + }, + "outputs": [], + "source": [ + "class NormalizeImage(beam.DoFn):\n", + " def process(self, element: np.ndarray):\n", + " element = (element-element.min())/(element.max()-element.min())\n", + " return [element]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gv23KPt5NyXT" + }, + "outputs": [], + "source": [ + "class GrayscaleImage(beam.DoFn):\n", + " def process(self, element: np.ndarray):\n", + " element = 0.07 * element[:,:,2] + 0.72 * element[:,:,1] + 0.21 * element[:,:,0]\n", + " return [element]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8gz7_SvN-P2L" + }, + "source": [ + "## Training a Convolutional Neural Network\n", + "\n", + "A Convolutional Neural Network (CNN) is one of the most popular model types for image processing. Here is a brief description of the convolutional layers used in the model.\n", + "* **Reshape**: Changes the shape of the input data to the desired size.\n", + "The CIFAR-10 images are of 32x32 pixels in grayscale. We will train our model using these images and thus, all images fed into the model need to be reshaped to the required size, that is (32,32,1).\n", + "* **Conv2D**: Applies a set of filters to extract features from the input image, producing a feature map as the output.\n", + "This layer is used as it is an essential component of a CNN, and does the major task of finding patterns in images.\n", + "* **MaxPooling2D**: Reduces the spatial dimensions of the input while retaining the most prominent features.\n", + "We use this layer to downsample the images and preserve only the important features.\n", + "* **Flatten**: Flattens the input data or feature maps into a 1-dimensional vector.\n", + "The input images are 2-dimensional. However in the end we require our results in a 1-D array. Flatten layer is used for this.\n", + "* **Dense**: Connects every neuron in the current layer to every neuron in the subsequent layer.\n", + "The CIFAR-10 dataset contains images belonging to 10 different classes. This is why the last dense layer gives 10 outputs, where each output corresponds to the probability of an image belonging to one of the 10 classes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "chYD9y7cH4Td", + "outputId": "b91eed4e-c383-40b8-96dc-8cf330d11a09" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Model: \"sequential\"\n", + "_________________________________________________________________\n", + " Layer (type) Output Shape Param # \n", + "=================================================================\n", + " reshape (Reshape) (None, 32, 32, 1) 0 \n", + " \n", + " conv2d (Conv2D) (None, 30, 30, 32) 320 \n", + " \n", + " max_pooling2d (MaxPooling2D (None, 15, 15, 32) 0 \n", + " ) \n", + " \n", + " conv2d_1 (Conv2D) (None, 13, 13, 64) 18496 \n", + " \n", + " max_pooling2d_1 (MaxPooling (None, 6, 6, 64) 0 \n", + " 2D) \n", + " \n", + " conv2d_2 (Conv2D) (None, 4, 4, 64) 36928 \n", + " \n", + " flatten (Flatten) (None, 1024) 0 \n", + " \n", + " dense (Dense) (None, 64) 65600 \n", + " \n", + " dense_1 (Dense) (None, 10) 650 \n", + " \n", + "=================================================================\n", + "Total params: 121,994\n", + "Trainable params: 121,994\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n" + ] + } + ], + "source": [ + "def create_model():\n", + " model = tf.keras.Sequential([\n", + " tf.keras.layers.Reshape((32,32,1),input_shape=x_train_gray.shape[1:]),\n", + " tf.keras.layers.Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 1)),\n", + " tf.keras.layers.MaxPooling2D((2, 2)),\n", + " tf.keras.layers.Conv2D(64, (3, 3), activation='relu'),\n", + " tf.keras.layers.MaxPooling2D((2, 2)),\n", + " tf.keras.layers.Conv2D(64, (3, 3), activation='relu'),\n", + " tf.keras.layers.Flatten(),\n", + " tf.keras.layers.Dense(64, activation='relu'),\n", + " tf.keras.layers.Dense(10)\n", + " ])\n", + " model.compile(optimizer='adam',\n", + " loss='sparse_categorical_crossentropy',\n", + " metrics=['accuracy'])\n", + " return model\n", + "\n", + "model = create_model()\n", + "model.summary()" + ] + }, + { + "cell_type": "markdown", + "source": [ + "The input shape is changed to (32,32,1) as our input images are of 32 x 32 pixels and 1 represents grayscale. In the final dense layer, there are 10 outputs as there are 10 possible classes in the CIFAR-10 dataset." + ], + "metadata": { + "id": "UpO090vNbHir" + } + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3D1SdCC94DQi" + }, + "source": [ + "## Fitting the model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "KuMP46UVXkof", + "outputId": "5340424c-7a33-45c9-b773-3ff625c65290" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Epoch 1/10\n", + "1563/1563 [==============================] - 87s 55ms/step - loss: 1.6511 - accuracy: 0.4054\n", + "Epoch 2/10\n", + "1563/1563 [==============================] - 84s 54ms/step - loss: 1.2737 - accuracy: 0.5540\n", + "Epoch 3/10\n", + "1563/1563 [==============================] - 80s 51ms/step - loss: 1.1204 - accuracy: 0.6095\n", + "Epoch 4/10\n", + "1563/1563 [==============================] - 79s 51ms/step - loss: 1.0184 - accuracy: 0.6461\n", + "Epoch 5/10\n", + "1563/1563 [==============================] - 80s 51ms/step - loss: 0.9430 - accuracy: 0.6724\n", + "Epoch 6/10\n", + "1563/1563 [==============================] - 81s 52ms/step - loss: 0.8810 - accuracy: 0.6946\n", + "Epoch 7/10\n", + "1563/1563 [==============================] - 80s 51ms/step - loss: 0.8299 - accuracy: 0.7135\n", + "Epoch 8/10\n", + "1563/1563 [==============================] - 80s 51ms/step - loss: 0.7904 - accuracy: 0.7248\n", + "Epoch 9/10\n", + "1563/1563 [==============================] - 80s 51ms/step - loss: 0.7504 - accuracy: 0.7385\n", + "Epoch 10/10\n", + "1563/1563 [==============================] - 84s 54ms/step - loss: 0.7150 - accuracy: 0.7498\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 22 + } + ], + "source": [ + "model.fit(x_train_gray, y_train, epochs=10)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3obkK5fQ4KG6" + }, + "source": [ + "## Authenticating from Google Cloud\n", + "\n", + "We need to store our trained model in Google Cloud. For running inferences, we will load our model from cloud into the notebook using a Model Handler." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Jj_NX568mhsE" + }, + "outputs": [], + "source": [ + "from google.colab import auth\n", + "auth.authenticate_user()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7jn7HYPi4hH9" + }, + "source": [ + "Saving the trained model in a Google Cloud Storage bucket" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CDqLPkzLfrt1" + }, + "outputs": [], + "source": [ + "save_model_dir = '' # Add the link to you GCS bucket here\n", + "model.save(save_model_dir)" + ] + }, + { + "cell_type": "markdown", + "source": [ + "A model handler is used to save, load and manage trained ML models. Here we used TFModelHandlerNumpy as our input images are in the form of numpy arrays." + ], + "metadata": { + "id": "ilYg15uOcZSY" + } + }, + { + "cell_type": "code", + "source": [ + "model_handler = TFModelHandlerNumpy(save_model_dir)" + ], + "metadata": { + "id": "RqefS6I_c1kc" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### Saving predictions\n", + "RunInference returns the predictions for each class. In the below DoFn, the maximum predicion is selected (which refers to the class the input image most probably belongs to) and is stored in a list of predictions." + ], + "metadata": { + "id": "4vJZEXFOboUi" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9T_YW19P46dW" + }, + "outputs": [], + "source": [ + "from tensorflow.python.ops.numpy_ops import np_config\n", + "np_config.enable_numpy_behavior()\n", + "predictions = []\n", + "class SavePredictions(beam.DoFn):\n", + " def process(self, element, *args, **kwargs):\n", + " list_of_predictions = element.inference.tolist()\n", + " highest_prediction = max(list_of_predictions)\n", + " ans = labels[list_of_predictions.index(highest_prediction)]\n", + " predictions.append(ans)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "c5x1HD_V4-RS" + }, + "source": [ + "## Building a Beam Pipeline\n", + "\n", + "A Pipeline represents the workflow of a series of computations. Here we are performing the following tasks in our pipeline:\n", + "* Creating a PCollection of the data on which we need to run inference\n", + "* Appying the Image Preprocessing DoFns we defined earlier
    \n", + " These include:\n", + " 1. Standardization\n", + " 2. Normalization\n", + " 3. Converting to grayscale\n", + "* Running Inference by using the trained model stored in Google Cloud.\n", + "* Displaying the output of the model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "KxYdNXefTFcD" + }, + "outputs": [], + "source": [ + "with beam.Pipeline() as p:\n", + " _ = (p | beam.Create(x_test)\n", + " | beam.ParDo(StandardizeImage())\n", + " | beam.ParDo(NormalizeImage())\n", + " | beam.ParDo(GrayscaleImage())\n", + " | RunInference(model_handler)\n", + " | beam.ParDo(SavePredictions())\n", + " )" + ] + }, + { + "cell_type": "markdown", + "source": [ + "So we got our predictions! Let us verify one of them." + ], + "metadata": { + "id": "yCADuo_yk1sK" + } + }, + { + "cell_type": "code", + "source": [ + "index = 5000\n", + "#You can change this index value to see and verify any image\n", + "predictions[index]" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 35 + }, + "id": "Zj2p_V2qmYMM", + "outputId": "f45c4f9f-3fa2-49f9-8471-602eea8faf68" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "'Horse'" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" + } + }, + "metadata": {}, + "execution_count": 40 + } + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 447 + }, + "id": "kd5fXda1yr6G", + "outputId": "51589888-2986-4d99-a6a3-c89ff24ebd9e" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 30 + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
    " + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGdCAYAAABU0qcqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAxWElEQVR4nO3dfXTU9Z33/9fMZGZynxBC7iQgNwqigIqCuawuAstNr+NPK2d/2na72Hr06EavVbbblp5Wq7t7xbVnW9teiNfvrCvb31W0ulfR6rZaRQl1BSpUimiLgEFAkiA3uZtkksnM9/rDy7RRkM8bEj5JfD7OmXMk8/adz3e+M9/3fDMzrwkFQRAIAIAzLOx7AQCATycGEADACwYQAMALBhAAwAsGEADACwYQAMALBhAAwAsGEADAiyzfC/ioTCajgwcPqqCgQKFQyPdyAABGQRCovb1dVVVVCodPfJ4z5AbQwYMHVV1d7XsZAIDTtH//fo0dO/aE1w/aAFq5cqW++93vqqmpSTNnztSPfvQjzZ49+6T/X0FBgSRpes1cRbLcllc5psx5XT29vc61kqSY+0102SVzTK2L8guda48dazH1nj79fOfastLRpt652VFTfTQcca4tKi4w9X777V3Otel0xtR77FlVzrU//4+fm3q/07DTVN/b2+lc251JmXonetLOtXlJ220YNuz7cDxm6p04fMS5tjPZauptPU5kMu63yyedERy33hCWlhXYeqej7vVpuS+ktzetzZt+13c8P5FBGUA//elPtXz5cj388MOaM2eOHnzwQS1atEg7d+5UWdknD4sP/+wWycpSJMvtQBeNut9xM9Y/60XdD7bZ2Tmm1jk5uc61XV3dpt65eXnOtXn5+abeedm2A0U04n4QKihwH8qSlJfnvvZ02v1AK0n5J3nw/Kns7GxT71jMdhuGw+4HRMOxUJIUNbwUHDUOcdMAihmf2ETdD19ZvbZDXcZwsJWkTMb9uDKUBlAoy70+ZLxNJJ30ZZRBeRPC9773Pd1888368pe/rGnTpunhhx9Wbm6u/vVf/3Uwfh0AYBga8AHU09OjrVu3asGCBX/8JeGwFixYoI0bN36svru7W21tbf0uAICRb8AH0OHDh5VOp1VeXt7v5+Xl5WpqavpYfV1dnYqKivouvAEBAD4dvH8OaMWKFWptbe277N+/3/eSAABnwIC/CaG0tFSRSETNzc39ft7c3KyKioqP1cfjccXj8YFeBgBgiBvwM6BYLKZZs2Zp3bp1fT/LZDJat26dampqBvrXAQCGqUF5G/by5cu1bNkyXXLJJZo9e7YefPBBJRIJffnLXx6MXwcAGIYGZQBdf/31ev/993X33XerqalJF154oZ577rmPvTEBAPDpFQqCwP7pokHU1tamoqIiffObX1d2tttrQwvmzXPuv/G135jW885B9zdFXD1/ial3ssv9g5H7G5tPXvQnLB8Wzc+2/SU2mbS9Vb61tcW5NjvH9oHOg+8ddK5tPNho6p0Vcn9tsvmQbf+0dBwz1fdmepxr80YXmXoHhg8Kh450mHq7pplIUlev+zZKUtDp/uHsXtnSIUJh2wfWLR+4tX4gOki7H6KDjO2xnHJfttKGm6S3N62tr72p1tZWFRae+MPl3t8FBwD4dGIAAQC8YAABALxgAAEAvGAAAQC8YAABALxgAAEAvGAAAQC8YAABALxgAAEAvBiULLiB8Jtfv6ioY4zHb19d79y3S7YYjKSh/s1Xfm3qHY2Pcq6ddvFsU+/db73hXJtqs8XIZEK22zCVyTjXxmLuEUKSJEM8SCwrampdWlDqXNvV2mrq3d2dNNWnDU8VYylbulZOtuF2idgOGaGw+8IjhkggSQpF3dedMsTZSFLGlsSj3rT7fby31xrF4947ZMnLkZQybGg65H4busYNcQYEAPCCAQQA8IIBBADwggEEAPCCAQQA8IIBBADwggEEAPCCAQQA8IIBBADwggEEAPCCAQQA8GLIZsE1HTyoSMRtPvYG7llJuWWjTeuIF+Y516aTnabeQeDeW6G4qXc8nutc25HsNvVW3JbXFooY1h6y3SVdM6ckqSvZa+pddnaZc20Qtq279eB7pvog7J6T1ivbfSUdcd+fGeN2RgL3rLHuVMrUO5Nyf9x3u5dKsh1TJCljqE8bc+lCve718ZAtCy6Tce+dMWTBufblDAgA4AUDCADgBQMIAOAFAwgA4AUDCADgBQMIAOAFAwgA4AUDCADgBQMIAOAFAwgA4MWQjeLJCmcpEnacjyH3ORqEbTElkWz3uJysaIepd5YhRibLEMchSdnxfOfa4opKU+9ET8JU39PmXh+KGOKJJLUfa3OuLeqxPd9qeeegc+2xkG3fZ9Rjqg/1usfl9KTc71eSJMP9UIGtd07SEN+StEXx9PS6RyuF0ra4qYxs29lruGuF07aYn6yMe7xOxnAslKSM6zFWkiKWceF2+3EGBADwggEEAPCCAQQA8IIBBADwggEEAPCCAQQA8IIBBADwggEEAPCCAQQA8IIBBADwggEEAPBiyGbBBUFIQeCWgRQY5mjgHh8lSQor6lzbGXPPbJIkpd3zwLrUaWp99jnVzrVjiiebev/+je2m+j1H33auTaVtz4mqJpzrXHtZ9TRT7/bDLc617zbuMPUuyC8w1We63e+4iVSXqXc0UuhcW1Babuqd2+WeqZY86J69J0k9YcPhy5iRFjJmwWVF3PvHYxFbb0tWX8aWMRjIPZcuZckBdKzlDAgA4MWAD6DvfOc7CoVC/S5Tp04d6F8DABjmBuVPcOeff75efPHFP/6SrCH7lz4AgCeDMhmysrJUUVExGK0BACPEoLwGtGvXLlVVVWnixIn64he/qH379p2wtru7W21tbf0uAICRb8AH0Jw5c7R69Wo999xzWrVqlRoaGnTFFVeovb39uPV1dXUqKirqu1RXu797CwAwfA34AFqyZIn+4i/+QjNmzNCiRYv0i1/8Qi0tLXriiSeOW79ixQq1trb2Xfbv3z/QSwIADEGD/u6A4uJinXvuudq9e/dxr4/H44rH44O9DADAEDPonwPq6OjQnj17VFlZOdi/CgAwjAz4APrqV7+q+vp67d27V6+++qo+97nPKRKJ6POf//xA/yoAwDA24H+CO3DggD7/+c/ryJEjGjNmjD7zmc9o06ZNGjNmjKlPKpxSJuw2H6MR9wic3CxbXE6uAufaaGCL2Ah63GMzco69b+o9tcQ9XmeUMf5m4vmzTPXbs0qca599fZup956md51rJ5bNNPW+cPZVzrV/+HWrqfe0c2x/EehsP+Jcu7Xh+H/uPpEc5TrXThx3tql3QcT9EJPsdY+FkaREs3t0T/k4W9yU9WWBjlb3d+9OPWeKqfeRJvftfG/fH0y949nu2xnLc4+PSqVSeuP135+0bsAH0OOPPz7QLQEAIxBZcAAALxhAAAAvGEAAAC8YQAAALxhAAAAvGEAAAC8YQAAALxhAAAAvGEAAAC8YQAAALwb96xhO1QX5eYpG3LLVCmL5zn2Li9xzySQpJ+reO1Jouzmz3Fsrv73D1DvvnQbn2lRnl6l3OmrLyRqXV+hcm+nqNvV+r8k9I23bW3tMvXvD7tvZG46aeucY7rOSVD3J/SvuGw7bvlW4/VDSufaNw2+beocy7nmHzY227wLrTrv3Lkr1mnpPnWTLaxt/sXu2X/VZY02943nuWX29huxKSeq2RGPGc5xLOxMJ/eoXL5y0jjMgAIAXDCAAgBcMIACAFwwgAIAXDCAAgBcMIACAFwwgAIAXDCAAgBcMIACAFwwgAIAXQzaKZ0GsSDlZblE8QZd7zEbWIVtMSV6h2xokKVo4xtQ7k+Me35IMOk29u4+871zbe7TV1DsesT1v6cjJc67NdLWbepeMco+0SStl6v3ypnrn2tLKAlPviWefa6ofXeAeZ5S/bZepd8Pu3c61jY0HTL2TPe6Pt97uhKl32HD02rXvXVPvmZNtUTwXnj/Tufbo0WOm3qPHuEf3tKVjpt7bf/eGc+3h1ibn2u6k2/GKMyAAgBcMIACAFwwgAIAXDCAAgBcMIACAFwwgAIAXDCAAgBcMIACAFwwgAIAXDCAAgBcMIACAF0M2Cy7o6lIQccthSxv6ZkVtmxyKZZxrSyttWXDHQoFzbV5Rhal39yH3vKnD779p6h1Eekz1DYc7nGuPtB8x9T7S6Z4f9mbClqc36+L/4lxbWV5i6n2gqdFU39jofrvsevMdU++9+/7gXNveYcsxS/e631cipkeyFDI8fe7Nsj3X3rjd9pgoqJzuXNt8xJZ5d/TVt51rS0aPNvU+9P5h59r3j7jfB3tT3U51nAEBALxgAAEAvGAAAQC8YAABALxgAAEAvGAAAQC8YAABALxgAAEAvGAAAQC8YAABALxgAAEAvBiyWXDVsz6jvFjMqTaWl+fcN5Jxz1+TpIPN7pld6cmlpt6RnqRzbWkk29S7xZDbdDDca+r9+uEuU/22LvfnOR29cVPvTE+Lc21LypYz19Cwx7l2dEm+qfd/PP+cqb7laItz7bv79pt6t3a557uFwra8tqywe5ZiJGRqrVgs6lybTNvW3dNlu4+PGzfOuXZ/43ZT77jhhhk7OtfUu6r4LOfa0LlVzrXJZKc2/O+T13EGBADwwjyANmzYoKuvvlpVVVUKhUJ66qmn+l0fBIHuvvtuVVZWKicnRwsWLNCuXbsGar0AgBHCPIASiYRmzpyplStXHvf6Bx54QD/84Q/18MMPa/PmzcrLy9OiRYuUTLr/uQkAMPKZXwNasmSJlixZctzrgiDQgw8+qG9961u65pprJEk//vGPVV5erqeeeko33HDD6a0WADBiDOhrQA0NDWpqatKCBQv6flZUVKQ5c+Zo48aNx/1/uru71dbW1u8CABj5BnQANTU1SZLKy8v7/by8vLzvuo+qq6tTUVFR36W6unoglwQAGKK8vwtuxYoVam1t7bvs3297CykAYHga0AFUUVEhSWpubu738+bm5r7rPioej6uwsLDfBQAw8g3oAJowYYIqKiq0bt26vp+1tbVp8+bNqqmpGchfBQAY5szvguvo6NDu3bv7/t3Q0KBt27appKRE48aN05133ql/+Id/0DnnnKMJEybo29/+tqqqqnTttdcO5LoBAMOceQBt2bJFV111Vd+/ly9fLklatmyZVq9era997WtKJBK65ZZb1NLSos985jN67rnnlJ1tjJJpbFBP1C1qIxS4n8jFZcv7SCU6nGsbf2f7rFM84hY1JElNKdvJ6sFW93iVdYcPmnrvC59tqp/92c8716be+LWp96//82nn2vPPm2nqfdHFn3GuTad7TL1TmYipfs9e9w9zRyLu8TeSFIu6PyZ6e21RVmG518ez3KN1JCmW5f6YKB5VbOr9/17zX0318y5zv2+dU11m6t3c7P74HFNSYOodd4w7k6TAsOs7HI+b5gE0d+5cBZ+wklAopPvuu0/33XeftTUA4FPE+7vgAACfTgwgAIAXDCAAgBcMIACAFwwgAIAXDCAAgBcMIACAFwwgAIAXDCAAgBcMIACAF+YonjOl893fSxG3+djdnXLuW2z8uoc8S4bdIVNrlVZPdq7d1d5l6v3vb+51rn0z6Z4HJUlnTb/IVH+kJ9+59pKL5pl6t7cc/4sOjyeVtuWY5RdUOdfmZOeYeh892m6qD8K/da5NptxzACUpK+KeSxcObFmK8Sz3+9aFM8439T5y9H3n2rQxZ660rMhUf2D/751rU73dpt7xeMK59mire3blB9z3ZzqTdq7t7Ox0quMMCADgBQMIAOAFAwgA4AUDCADgBQMIAOAFAwgA4AUDCADgBQMIAOAFAwgA4AUDCADgxZCN4inKLlCuY0RIazjp3LczsMWxJOVeH+TZ4j52HD3oXPsf2/eaev9mn3scS9FZ00y99za3mer3v/Oqc+3ci6aaep8zabpz7a83bzT1fuONd51r/+tnrzb1nnWJezyRJL225dfOtYmEWwzKh4oL3WOEOjuMUS+G+JazqipNrYNQxrn2vRZbPNFrv3vNVL/9ze3OtZGI+7olSWH3Y1BHj611qrfXudYSwtTT7bYQzoAAAF4wgAAAXjCAAABeMIAAAF4wgAAAXjCAAABeMIAAAF4wgAAAXjCAAABeMIAAAF4wgAAAXgzZLLi90Zhystyy4Nqy3OdowpgFlw6757s1Hjhs6v3qrgbn2r1ttpCndLjQubbtaJOp9zkV55jqp06b5Vwbzoqbevd2tTrXlpVPMPXOLih1rt2z/z1T7/37f2eq70y6Z3ZlMjFT75jheWhR2ShT76y422NYkoKo7XBUWFrhXBsb7f54kKQjxxpN9SFDUlpnZ8LUu7vH/bGfnZNr6p3OuOfSJbu6nGt7U273V86AAABeMIAAAF4wgAAAXjCAAABeMIAAAF4wgAAAXjCAAABeMIAAAF4wgAAAXjCAAABeDNkonm1VYxWLusXgdGfc43W6etwjTSRJIfcokb279ptav3vMfS1BLNvUO0innGuTiUOm3l3H9pjqy0tmO9cWFZWbeoej7lEiuw/Y9s9bu3/rXLtj5yum3j0dtvij7q6jzrWRTKep96FD7lEv0ZjtOWt+UZ5zbcg9zUaSNP2C85xru7raTL0LCvJN9YWF7lE/GUP8jSSFQu63eU627ThhSBBSyLCDurqSWvcfG05axxkQAMALBhAAwAvzANqwYYOuvvpqVVVVKRQK6amnnup3/Y033qhQKNTvsnjx4oFaLwBghDAPoEQioZkzZ2rlypUnrFm8eLEaGxv7Lo899thpLRIAMPKY34SwZMkSLVmy5BNr4vG4Kircv6sDAPDpMyivAa1fv15lZWWaMmWKbrvtNh05cuSEtd3d3Wpra+t3AQCMfAM+gBYvXqwf//jHWrdunf7pn/5J9fX1WrJkidLp9HHr6+rqVFRU1Heprq4e6CUBAIagAf8c0A033ND339OnT9eMGTM0adIkrV+/XvPnz/9Y/YoVK7R8+fK+f7e1tTGEAOBTYNDfhj1x4kSVlpZq9+7dx70+Ho+rsLCw3wUAMPIN+gA6cOCAjhw5osrKysH+VQCAYcT8J7iOjo5+ZzMNDQ3atm2bSkpKVFJSonvvvVdLly5VRUWF9uzZo6997WuaPHmyFi1aNKALBwAMb+YBtGXLFl111VV9//7w9Ztly5Zp1apV2r59u/7t3/5NLS0tqqqq0sKFC/X3f//3isfjpt/TnOlR1DHjLZV0z1TLzrJlJbW2djjXNhxuMfXOxHKca8Mh97w7SRo1yj2DKyxb7717Npvqf7qm0bl26vmXm3rnFrpnx3V0tZh6dyTeca7NUtLUu7f7mKk+K2h3rg0F7jmAkhRE3DO+unuP/2aiE0k0tzrXjq8ea+r9lb+83rm2N2nbP/GY7XgVDrvfhtFozNQ7FnPLxPxgHbY/alke+YEhc7O9vUNf/bv7TlpnHkBz585VEJx4Ic8//7y1JQDgU4gsOACAFwwgAIAXDCAAgBcMIACAFwwgAIAXDCAAgBcMIACAFwwgAIAXDCAAgBcMIACAFwP+fUAD5fqr5io3xy23LaSIc9+SohLTOh5/4mfOtVs7j5p658Xdc+niIdtzhVGF7jlzZWdVmXq//fbbpvqjR951rt30myZT7+xs96/vKB41ytS7OMs9BzDZ6Z7VJkkJ432lN+Oe7xYxZLtJUuYTorU+XmxqrVjU/bFZXmp7bBbmu2ekdVrz8QL3fElJCoXctzPV22XqbanPitgO6b297tv5SRFsH9WRSDjVcQYEAPCCAQQA8IIBBADwggEEAPCCAQQA8IIBBADwggEEAPCCAQQA8IIBBADwggEEAPBiyEbx/D9XX63CggKn2t7AfY7u3feeaR2/37vXubans9XUe9q0s5xrjx05Zuq9951dzrX7m2y3SSQaM9VffsXFzrV5cdtzogPv7nWuTfccMvXOi+Q51yYj7rFKklSQO8ZUf6zdPernWJvtflg62j0CZ1Rhsan3vj17nWvbW23r7uxsca491u4WDfNHtvthfp77fSWTseUZGYKSlJU1eFE8ltpEkigeAMAQxgACAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4wQACAHgxZLPgjh1rUm/KLf+qK5N27vu/nnjctI7Xd7zhXJtfkGvqfeMtNzrX7jFkaknS//gfDzvXphLuOWOSNGb0aFP9bV+6wbn2vIlVpt6t7S3OtT29llQtqast6VwbhGz5XrFcW3ZcY/NR59p77/uuqffs2Rc51y5cONfU+xtfu8+5NtHpnjUmSaGI+/PnwpwcU28FIVt92v0YZOxs+h/SqR5T61g06lybHXOvDRzz7jgDAgB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4MWSjeDKZlDKZlFPt7rd3Ofd9/rnnTOvo6XGP2Ljk4imm3hddNMO5duOrm029KyvKnGvLyspNvTs7OmxrKRvjXFs6qtjUu6gwz7m2x5aWo6yQIXok5H4/kaRQxBbIUlk51rm2rNQWlVRa4l4/6+KZpt6TJo13rk2nbTsoGo279zbunyBji20Kh92fy6cNsT2SFMkyHKZDtnV3J93jpnIscUaB2zo4AwIAeGEaQHV1dbr00ktVUFCgsrIyXXvttdq5c2e/mmQyqdraWo0ePVr5+flaunSpmpubB3TRAIDhzzSA6uvrVVtbq02bNumFF15QKpXSwoULlUgk+mruuusuPfPMM3ryySdVX1+vgwcP6rrrrhvwhQMAhjfTa0DPfeT1k9WrV6usrExbt27VlVdeqdbWVj3yyCNas2aN5s2bJ0l69NFHdd5552nTpk267LLLBm7lAIBh7bReA2ptbZUklZSUSJK2bt2qVCqlBQsW9NVMnTpV48aN08aNG4/bo7u7W21tbf0uAICR75QHUCaT0Z133qnLL79cF1xwgSSpqalJsVhMxcXF/WrLy8vV1NR03D51dXUqKirqu1RXV5/qkgAAw8gpD6Da2lrt2LFDjz9u+4bRj1qxYoVaW1v7Lvv37z+tfgCA4eGUPgd0++2369lnn9WGDRs0duwfP59QUVGhnp4etbS09DsLam5uVkVFxXF7xeNxxePu7+cHAIwMpjOgIAh0++23a+3atXrppZc0YcKEftfPmjVL0WhU69at6/vZzp07tW/fPtXU1AzMigEAI4LpDKi2tlZr1qzR008/rYKCgr7XdYqKipSTk6OioiLddNNNWr58uUpKSlRYWKg77rhDNTU1vAMOANCPaQCtWrVKkjR37tx+P3/00Ud14403SpK+//3vKxwOa+nSperu7taiRYv00EMPDchiAQAjh2kABQ75PtnZ2Vq5cqVWrlx5youSpHgsW/F4tlPtjh07nPu+9957p7qkk5pz6UWm+q2b3fPdnv/FL029v/SlLzrXNjTsM/Xef8B2GxbkFzjX9vS45f99KC33XK3Orh5T74jc6yNR2/t50oExmE7uuXRjz7K9kzTZ7X6b5+S6PSY/dN60c51rjxw5bOrdleg21Haaeodky+qLRCLOtX/6wX0X2YYMtlDEeD/MuN8Pe3rcHw8px1qy4AAAXjCAAABeMIAAAF4wgAAAXjCAAABeMIAAAF4wgAAAXjCAAABeMIAAAF4wgAAAXpzS1zGcCYnODoXDJ4/+kaT3D7/v3LcnZYtjKRnlHiNz9rixJy/6Ey//SWr4yVxy0QxT76XXXO1c+7/W/NTUu2Hvu6b6o8eOOteWFpabeqd73WNk8vJyTb0zvYZ1pG33q54e9xgZSYpnu39lyYm++uREjh5rca7NWG4USbPnXOxc+/xz6029E4YonuLCIlPvrk5bdI9DSlmffEM0lSRFY+4xTL3GiKdoyD1yyBI3FMlyGy2cAQEAvGAAAQC8YAABALxgAAEAvGAAAQC8YAABALxgAAEAvGAAAQC8YAABALxgAAEAvGAAAQC8GLJZcNFoXNGoW/5VkHHPM0p2GkKbJM1fcIlzbfVYWxbc0aNHnGu/9Jd/Zeo9dmyVc+2Xl/2lqXfaVC2tedw9a+6/3foVU+/CQvd8t0jUdncPZ7k/P+vpdb8PSlI8L89Uv3dfk3PtO+82mHpfcXmNc21OtnsumSRdOmu6c+2v61819d7zzl7n2jkXnW/qbYg9kySFwu77P5Ox5bV1dLQ618ZzbHmHPSn3bD/LuhMJtyw9zoAAAF4wgAAAXjCAAABeMIAAAF4wgAAAXjCAAABeMIAAAF4wgAAAXjCAAABeMIAAAF4M2SiedCaktGPETumo0c59L7v4XNM6bvrLpc61+w8cMPXOzslxrr3wQluUSE93h3NtYb7tbrDsi+63iST96KGHnWtXPvr/m3rX3nyjc+2oPFuIUDpwrw+H3felJL3x1jum+h+s+v+ca/MKbWu5at6FzrVdLYdNvfOi7pk2ebkxU+9Nv/mtc+2MaRNMvZPdbaZ6yT2mJtndZeocj2c71/b22CKhMmn3dVuieDK93U51nAEBALxgAAEAvGAAAQC8YAABALxgAAEAvGAAAQC8YAABALxgAAEAvGAAAQC8YAABALxgAAEAvBiyWXDhrLTCWW5ZXP/l8kuc+150kS1TLRR2n9H/8sjPTb0njBvvXJsVsuWY9XS2OtcGmZSpd2VR3FS/eG6Nc+1//+cfmnoX57nnZH3p87YMu1Co17k20emevSdJDz30P031W7a655498M/3mXpHs9zv44lEwtQ7NzvXuTY725YF96sXXnSuvXjaJFPvqVMmm+ozhsdQb8qW1xbNcq+Pxd2z9yQpyATOtV3JpHNtd5IsOADAEGYaQHV1dbr00ktVUFCgsrIyXXvttdq5c2e/mrlz5yoUCvW73HrrrQO6aADA8GcaQPX19aqtrdWmTZv0wgsvKJVKaeHChR87Lb/55pvV2NjYd3nggQcGdNEAgOHP9BrQc8891+/fq1evVllZmbZu3aorr7yy7+e5ubmqqKgYmBUCAEak03oNqLX1gxe6S0pK+v38Jz/5iUpLS3XBBRdoxYoV6uzsPGGP7u5utbW19bsAAEa+U34XXCaT0Z133qnLL79cF1xwQd/Pv/CFL2j8+PGqqqrS9u3b9fWvf107d+7Uz372s+P2qaur07333nuqywAADFOnPIBqa2u1Y8cOvfLKK/1+fsstt/T99/Tp01VZWan58+drz549mjTp42+FXLFihZYvX97377a2NlVXV5/qsgAAw8QpDaDbb79dzz77rDZs2KCxY8d+Yu2cOXMkSbt37z7uAIrH44rHbZ8rAQAMf6YBFASB7rjjDq1du1br16/XhAkTTvr/bNu2TZJUWVl5SgsEAIxMpgFUW1urNWvW6Omnn1ZBQYGampokSUVFRcrJydGePXu0Zs0affazn9Xo0aO1fft23XXXXbryyis1Y8aMQdkAAMDwZBpAq1atkvTBh03/1KOPPqobb7xRsVhML774oh588EElEglVV1dr6dKl+ta3vjVgCwYAjAzmP8F9kurqatXX15/Wgv74u1IKArd8pfLyUue+4Yjtnec/f/oZ59pdu9429b7wwouca3t73XPJJKkn1eNcmxOz5Ud1tBw21Z937kTn2gXz55t6v7zhlZMX/V8LF80z9Z404Wzn2u60LQsunbbtz3Mmur8x59wJ7hmDkhQOuWewZQL3+5UkpTMZ59qLLp5u6v3qa1uda996c7ep96yLLjXVK+S+nUXGD7+kUu4ZbCFbzJwKCwuda4uLi51r2zvcMgPJggMAeMEAAgB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeMEAAgB4wQACAHjBAAIAeHHK3wc06ILoBxcHqZ5u97aBLQJl4sSpzrWzLrvC1HvXuweda99tOmrqnZvtHq9y+J0mU++eHlscS1faPR+kuPyTv97jo86d5r7v32nYZ+rd2t7uXGtIPpIk1dTMMdVPmuQerzOmZJSpd+vRFufaguxcU+9YzP057uRz3CObJKnu/r93rh1bMsbUO2qMp7KwHoOiUffDdDhsO6dIJNwicySZvjbH9RjBGRAAwAsGEADACwYQAMALBhAAwAsGEADACwYQAMALBhAAwAsGEADACwYQAMALBhAAwAsGEADAiyGbBbdlyxvKzcl2qj34nnumWjqdNq2jraPDubY34555JklHW9qca5/43z839Y4YnlrEsmx3g/yiYlN9EHLvnzKGqo0ZXeJc23rUlqcXZNzvK/n5tvy1P58/z1Q/prTYubazyz3DTpLy8/Kda+PmI4Z77llpifu+lKScgiLn2lDSPTNQkro6W031gSzHlYypdzKZdC8O2TLsOjs7nWvjMfcsuI6EW1/OgAAAXjCAAABeMIAAAF4wgAAAXjCAAABeMIAAAF4wgAAAXjCAAABeMIAAAF4wgAAAXgzZKJ633npT8XjMqTbZ7R6zkR13i/c5FZdMP9dUn5OT61wbCmzPFTKBeyxQ9dhqU++8nKipflSxe2SKKXZEUldnwrk2Py/P1Dscdr/NE4ZIE0nqbD1iqt/X5h4j1N1juw1LS90jcJJhW9xUOu0exZPOBKbepWn3+rAhEkiSsiK27QxH3B8T6V7bWuJRt+OgJKUD221YUOD+mGhrc48O6+7pcqrjDAgA4AUDCADgBQMIAOAFAwgA4AUDCADgBQMIAOAFAwgA4AUDCADgBQMIAOAFAwgA4AUDCADgxZDNgrvi8ouVl5vjVJuV5b4Z4Yht5sZicefaUMiWH5UyZELlZOebeh8+fMy5dveuPabezT0pU31xkXsWXE6uez6eJLW3tzvX7jXmtQVBxrk2Y6iVJNs9RcrOcXssSNKxo+65cZJUPW6sc200assBtOSHjRkzxtS7uLDAuTbd65ZN9ke2TLXsbPeMyZaWFlPv3nTaubag0P2xJknJpHuOZigUca+VWy1nQAAAL0wDaNWqVZoxY4YKCwtVWFiompoa/fKXv+y7PplMqra2VqNHj1Z+fr6WLl2q5ubmAV80AGD4Mw2gsWPH6v7779fWrVu1ZcsWzZs3T9dcc43efPNNSdJdd92lZ555Rk8++aTq6+t18OBBXXfddYOycADA8GZ6Dejqq6/u9+9//Md/1KpVq7Rp0yaNHTtWjzzyiNasWaN58+ZJkh599FGdd9552rRpky677LKBWzUAYNg75deA0um0Hn/8cSUSCdXU1Gjr1q1KpVJasGBBX83UqVM1btw4bdy48YR9uru71dbW1u8CABj5zAPojTfeUH5+vuLxuG699VatXbtW06ZNU1NTk2KxmIqLi/vVl5eXq6mp6YT96urqVFRU1HeprrZ9OycAYHgyD6ApU6Zo27Zt2rx5s2677TYtW7ZMb7311ikvYMWKFWptbe277N+//5R7AQCGD/PngGKxmCZPnixJmjVrll577TX94Ac/0PXXX6+enh61tLT0Owtqbm5WRUXFCfvF43HF4+6ftQEAjAyn/TmgTCaj7u5uzZo1S9FoVOvWreu7bufOndq3b59qampO99cAAEYY0xnQihUrtGTJEo0bN07t7e1as2aN1q9fr+eff15FRUW66aabtHz5cpWUlKiwsFB33HGHampqeAccAOBjTAPo0KFD+qu/+is1NjaqqKhIM2bM0PPPP68///M/lyR9//vfVzgc1tKlS9Xd3a1FixbpoYceOqWF5ccD5We7xWFkAvdomEzGPdZCktJJ9wiPVMYW3xHPdo+dyYnbTlarz3KPNYmGbesOh2Km+sDQPj/fFjmUMcSUpHptEULxbPeHR06OexSLJL333num+ljM/TbPZMabeo8aVeJcG4nY/mpvebxZY34SHe4xTJnepKl3OGwLS8pk3KOYLPtSkiKG3slkj6n3EUNkV1Gxe8xPyPGPa6Z70yOPPPKJ12dnZ2vlypVauXKlpS0A4FOILDgAgBcMIACAFwwgAIAXDCAAgBcMIACAFwwgAIAXDCAAgBcMIACAFwwgAIAX5jTswRb839yWRKd7dIYlAScwRvFI7pEcKUvmjKRU2r13JNxp6m15bpHodI8bkqRwyHYbmm6WkO05kSWKp9cYxZNKuz880oa4FMl+m6d63bfTEgsjSdGY+31rSEXxJNzXnentNvW2RvFEsiKmeou04QCXCWzrthxns6LuEUIf3r+Dkzz4Q8HJKs6wAwcO8KV0ADAC7N+/X2PHjj3h9UNuAGUyGR08eFAFBQUKhf44zdva2lRdXa39+/ersLDQ4woHF9s5cnwatlFiO0eagdjOIAjU3t6uqqoqhcMn/qvGkPsTXDgc/sSJWVhYOKJ3/ofYzpHj07CNEts50pzudhYVnTw9mzchAAC8YAABALwYNgMoHo/rnnvuUTwe972UQcV2jhyfhm2U2M6R5kxu55B7EwIA4NNh2JwBAQBGFgYQAMALBhAAwAsGEADAi2EzgFauXKmzzz5b2dnZmjNnjn7zm9/4XtKA+s53vqNQKNTvMnXqVN/LOi0bNmzQ1VdfraqqKoVCIT311FP9rg+CQHfffbcqKyuVk5OjBQsWaNeuXX4WexpOtp033njjx/bt4sWL/Sz2FNXV1enSSy9VQUGBysrKdO2112rnzp39apLJpGprazV69Gjl5+dr6dKlam5u9rTiU+OynXPnzv3Y/rz11ls9rfjUrFq1SjNmzOj7sGlNTY1++ctf9l1/pvblsBhAP/3pT7V8+XLdc889+u1vf6uZM2dq0aJFOnTokO+lDajzzz9fjY2NfZdXXnnF95JOSyKR0MyZM7Vy5crjXv/AAw/ohz/8oR5++GFt3rxZeXl5WrRokZJJ94DEoeBk2ylJixcv7rdvH3vssTO4wtNXX1+v2tpabdq0SS+88IJSqZQWLlyoRCLRV3PXXXfpmWee0ZNPPqn6+nodPHhQ1113ncdV27lspyTdfPPN/fbnAw884GnFp2bs2LG6//77tXXrVm3ZskXz5s3TNddcozfffFPSGdyXwTAwe/bsoLa2tu/f6XQ6qKqqCurq6jyuamDdc889wcyZM30vY9BICtauXdv370wmE1RUVATf/e53+37W0tISxOPx4LHHHvOwwoHx0e0MgiBYtmxZcM0113hZz2A5dOhQICmor68PguCDfReNRoMnn3yyr+b3v/99ICnYuHGjr2Weto9uZxAEwZ/92Z8Ff/M3f+NvUYNk1KhRwb/8y7+c0X055M+Aenp6tHXrVi1YsKDvZ+FwWAsWLNDGjRs9rmzg7dq1S1VVVZo4caK++MUvat++fb6XNGgaGhrU1NTUb78WFRVpzpw5I26/StL69etVVlamKVOm6LbbbtORI0d8L+m0tLa2SpJKSkokSVu3blUqleq3P6dOnapx48YN6/350e380E9+8hOVlpbqggsu0IoVK9TZaf26lKEjnU7r8ccfVyKRUE1NzRndl0MujPSjDh8+rHQ6rfLy8n4/Ly8v1x/+8AdPqxp4c+bM0erVqzVlyhQ1Njbq3nvv1RVXXKEdO3aooKDA9/IGXFNTkyQdd79+eN1IsXjxYl133XWaMGGC9uzZo29+85tasmSJNm7cqEhk8L5HZrBkMhndeeeduvzyy3XBBRdI+mB/xmIxFRcX96sdzvvzeNspSV/4whc0fvx4VVVVafv27fr617+unTt36mc/+5nH1dq98cYbqqmpUTKZVH5+vtauXatp06Zp27ZtZ2xfDvkB9GmxZMmSvv+eMWOG5syZo/Hjx+uJJ57QTTfd5HFlOF033HBD339Pnz5dM2bM0KRJk7R+/XrNnz/f48pOTW1trXbs2DHsX6M8mRNt5y233NL339OnT1dlZaXmz5+vPXv2aNKkSWd6madsypQp2rZtm1pbW/Xv//7vWrZsmerr68/oGob8n+BKS0sViUQ+9g6M5uZmVVRUeFrV4CsuLta5556r3bt3+17KoPhw333a9qskTZw4UaWlpcNy395+++169tln9fLLL/f72pSKigr19PSopaWlX/1w3Z8n2s7jmTNnjiQNu/0Zi8U0efJkzZo1S3V1dZo5c6Z+8IMfnNF9OeQHUCwW06xZs7Ru3bq+n2UyGa1bt041NTUeVza4Ojo6tGfPHlVWVvpeyqCYMGGCKioq+u3XtrY2bd68eUTvV+mDb/09cuTIsNq3QRDo9ttv19q1a/XSSy9pwoQJ/a6fNWuWotFov/25c+dO7du3b1jtz5Nt5/Fs27ZNkobV/jyeTCaj7u7uM7svB/QtDYPk8ccfD+LxeLB69ergrbfeCm655ZaguLg4aGpq8r20AfO3f/u3wfr164OGhobgP//zP4MFCxYEpaWlwaFDh3wv7ZS1t7cHr7/+evD6668HkoLvfe97weuvvx68++67QRAEwf333x8UFxcHTz/9dLB9+/bgmmuuCSZMmBB0dXV5XrnNJ21ne3t78NWvfjXYuHFj0NDQELz44ovBxRdfHJxzzjlBMpn0vXRnt912W1BUVBSsX78+aGxs7Lt0dnb21dx6663BuHHjgpdeeinYsmVLUFNTE9TU1Hhctd3JtnP37t3BfffdF2zZsiVoaGgInn766WDixInBlVde6XnlNt/4xjeC+vr6oKGhIdi+fXvwjW98IwiFQsGvfvWrIAjO3L4cFgMoCILgRz/6UTBu3LggFosFs2fPDjZt2uR7SQPq+uuvDyorK4NYLBacddZZwfXXXx/s3r3b97JOy8svvxxI+thl2bJlQRB88Fbsb3/720F5eXkQj8eD+fPnBzt37vS76FPwSdvZ2dkZLFy4MBgzZkwQjUaD8ePHBzfffPOwe/J0vO2TFDz66KN9NV1dXcFf//VfB6NGjQpyc3ODz33uc0FjY6O/RZ+Ck23nvn37giuvvDIoKSkJ4vF4MHny5ODv/u7vgtbWVr8LN/rKV74SjB8/PojFYsGYMWOC+fPn9w2fIDhz+5KvYwAAeDHkXwMCAIxMDCAAgBcMIACAFwwgAIAXDCAAgBcMIACAFwwgAIAXDCAAgBcMIACAFwwgAIAXDCAAgBcMIACAF/8HQQxmwY8SIeIAAAAASUVORK5CYII=\n" + }, + "metadata": {} + } + ], + "source": [ + "plt.imshow(x_test[index])" + ] + }, + { + "cell_type": "code", + "source": [ + "labels[y_test[index][0]]" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 35 + }, + "id": "O32stbnNkTLh", + "outputId": "c6dc9b40-290d-4029-e626-da88efc6e4e3" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "'Horse'" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" + } + }, + "metadata": {}, + "execution_count": 32 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "Let us make a dictionary to see how many predictions belong to each class" + ], + "metadata": { + "id": "qT00-He7pfNi" + } + }, + { + "cell_type": "code", + "source": [ + "aggregate_results = dict()\n", + "for i in range(len(predictions)):\n", + " if predictions[i] in aggregate_results:\n", + " aggregate_results[predictions[i]] += 1\n", + " else:\n", + " aggregate_results[predictions[i]] = 1" + ], + "metadata": { + "id": "x744tmPnowMr" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "aggregate_results" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "evQnjvDdqbuG", + "outputId": "69c6d8d6-7986-444b-c66b-76c2fda98553" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "{'Dog': 641,\n", + " 'Automobile': 3387,\n", + " 'Deer': 793,\n", + " 'Horse': 1030,\n", + " 'Truck': 392,\n", + " 'Frog': 290,\n", + " 'Airplane': 179,\n", + " 'Cat': 3175,\n", + " 'Bird': 91,\n", + " 'Ship': 22}" + ] + }, + "metadata": {}, + "execution_count": 37 + } + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/examples/notebooks/beam-ml/mltransform_basic.ipynb b/examples/notebooks/beam-ml/mltransform_basic.ipynb new file mode 100644 index 0000000000000..e44be91fe1cd5 --- /dev/null +++ b/examples/notebooks/beam-ml/mltransform_basic.ipynb @@ -0,0 +1,667 @@ +{ + "cells": [ + { + "cell_type": "code", + "source": [ + "# @title ###### Licensed to the Apache Software Foundation (ASF), Version 2.0 (the \"License\")\n", + "\n", + "# Licensed to the Apache Software Foundation (ASF) under one\n", + "# or more contributor license agreements. See the NOTICE file\n", + "# distributed with this work for additional information\n", + "# regarding copyright ownership. The ASF licenses this file\n", + "# to you under the Apache License, Version 2.0 (the\n", + "# \"License\"); you may not use this file except in compliance\n", + "# with the License. You may obtain a copy of the License at\n", + "#\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing,\n", + "# software distributed under the License is distributed on an\n", + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n", + "# KIND, either express or implied. See the License for the\n", + "# specific language governing permissions and limitations\n", + "# under the License" + ], + "metadata": { + "id": "34gTXZ7BIArp" + }, + "id": "34gTXZ7BIArp", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "# Preprocess data with MLTransform\n", + "\n", + "\n", + " \n", + " \n", + "
    \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + "
    \n" + ], + "metadata": { + "id": "0n0YAd-0KQyi" + }, + "id": "0n0YAd-0KQyi" + }, + { + "cell_type": "markdown", + "id": "d3b81cf2-8603-42bd-995e-9e14631effd0", + "metadata": { + "id": "d3b81cf2-8603-42bd-995e-9e14631effd0" + }, + "source": [ + "This notebook demonstrates how to use `MLTransform` to preprocess your data for machine learning models. `MLTransform` is a `PTransform` that wraps multiple Apache Beam data processing transforms. With `MLTransform`, you can preprocess different types of data in multiple ways with one transform.\n", + "\n", + "This notebook uses data processing transforms defined in the [apache_beam/ml/transforms/tft](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.transforms.tft.html) module." + ] + }, + { + "cell_type": "markdown", + "id": "f0097dbd-2657-4cbe-a334-e0401816db01", + "metadata": { + "id": "f0097dbd-2657-4cbe-a334-e0401816db01" + }, + "source": [ + "## Import the required modules\n", + "\n", + "To use `MLTransfrom`, install `tensorflow_transform` and the Apache Beam SDK version 2.50.0 or later.\n" + ] + }, + { + "cell_type": "code", + "source": [ + "!pip install tensorflow_transform --quiet\n", + "!pip install apache_beam>=2.50.0 --quiet" + ], + "metadata": { + "id": "MRWkC-n2DmjM" + }, + "id": "MRWkC-n2DmjM", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "88ddd3a4-3643-4731-b99e-a5d697fbc165", + "metadata": { + "id": "88ddd3a4-3643-4731-b99e-a5d697fbc165" + }, + "outputs": [], + "source": [ + "import apache_beam as beam\n", + "from apache_beam.ml.transforms.base import MLTransform\n", + "from apache_beam.ml.transforms.tft import ComputeAndApplyVocabulary\n", + "from apache_beam.options.pipeline_options import PipelineOptions\n", + "from apache_beam.ml.transforms.utils import ArtifactsFetcher" + ] + }, + { + "cell_type": "markdown", + "source": [ + "Artifacts are additional data elements created by data transformations. Examples of artifacts are the `minimum` and `maximum` values from a `ScaleTo01` transformation, or the `mean` and `variance` from a `ScaleToZScore` transformation. For more information about artifacts, see [Artifacts](https://beam.apache.org/documentation/ml/preprocess-data/#artifacts).\n", + "\n", + "\n", + "\n" + ], + "metadata": { + "id": "90nXXc_A4Bmf" + }, + "id": "90nXXc_A4Bmf" + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bdabbc57-ec98-4113-b37e-61962f488d61", + "metadata": { + "id": "bdabbc57-ec98-4113-b37e-61962f488d61" + }, + "outputs": [], + "source": [ + "# Store artifacts generated by MLTransform.\n", + "# Each MLTransform instance requires an empty artifact location.\n", + "# This method deletes and refreshes the artifact location for each example.\n", + "artifact_location = './my_artifacts'\n", + "def delete_artifact_location(artifact_location):\n", + " import shutil\n", + " import os\n", + " if os.path.exists(artifact_location):\n", + " shutil.rmtree(artifact_location)" + ] + }, + { + "cell_type": "markdown", + "id": "28b1719c-7287-4cec-870b-9fabc4c4a4ef", + "metadata": { + "id": "28b1719c-7287-4cec-870b-9fabc4c4a4ef" + }, + "source": [ + "## Compute and map the vocabulary\n", + "\n", + "\n", + "[ComputeAndApplyVocabulary](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.transforms.tft.html#apache_beam.ml.transforms.tft.ComputeAndApplyVocabulary) is a data processing transform that computes a unique vocabulary from a dataset and then maps each word or token to a distinct integer index. It facilitates transforming textual data into numerical representations for machine learning tasks.\n", + "\n", + "Use `ComputeAndApplyVocabulary` with `MLTransform`.\n", + "\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "56d6d09a-8d34-444f-a1e4-a75624b36932", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "56d6d09a-8d34-444f-a1e4-a75624b36932", + "outputId": "2eb99e87-fb23-498c-ed08-775befa3a823" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Row(x=array([1, 0, 4]))\n", + "Row(x=array([1, 0, 6, 2, 3, 5]))\n" + ] + } + ], + "source": [ + "delete_artifact_location(artifact_location)\n", + "\n", + "data = [\n", + " {'x': ['I', 'love', 'pie']},\n", + " {'x': ['I', 'love', 'going', 'to', 'the', 'park']}\n", + "]\n", + "options = PipelineOptions()\n", + "with beam.Pipeline(options=options) as p:\n", + " data = (\n", + " p\n", + " | 'CreateData' >> beam.Create(data)\n", + " | 'MLTransform' >> MLTransform(write_artifact_location=artifact_location).with_transform(ComputeAndApplyVocabulary(columns=['x']))\n", + " | 'PrintResults' >> beam.Map(print)\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "1e133002-7229-459d-8e3c-b41f4d65e76d", + "metadata": { + "id": "1e133002-7229-459d-8e3c-b41f4d65e76d" + }, + "source": [ + "### Fetch vocabulary artifacts\n", + "\n", + "This example generates a file with all the vocabulary in the dataset, referred to in `MLTransform` as an artifact. To fetch artifacts generated by the `ComputeAndApplyVocabulary` transform, use the `ArtifactsFetcher` class. This class fetches both a vocabulary list and a path to the vocabulary file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9c5fe46a-c718-4a82-bad8-aa091c0b0538", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "9c5fe46a-c718-4a82-bad8-aa091c0b0538", + "outputId": "cd8b6cf3-6093-4b1b-a063-ff327c090a92" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "['love', 'I', 'to', 'the', 'pie', 'park', 'going']\n", + "./my_artifacts/transform_fn/assets/compute_and_apply_vocab\n", + "7\n" + ] + } + ], + "source": [ + "fetcher = ArtifactsFetcher(artifact_location=artifact_location)\n", + "# get vocab list\n", + "vocab_list = fetcher.get_vocab_list()\n", + "print(vocab_list)\n", + "# get vocab file path\n", + "vocab_file_path = fetcher.get_vocab_filepath()\n", + "print(vocab_file_path)\n", + "# get vocab size\n", + "vocab_size = fetcher.get_vocab_size()\n", + "print(vocab_size)" + ] + }, + { + "cell_type": "markdown", + "id": "5f955f3d-3192-42f7-aa55-48249223418d", + "metadata": { + "id": "5f955f3d-3192-42f7-aa55-48249223418d" + }, + "source": [ + "## Use TD-IDF to weight terms\n", + "\n", + "[TF-IDF](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.transforms.tft.html#apache_beam.ml.transforms.tft.ComputeAndApplyVocabulary) (Term Frequency-Inverse Document Frequency) is a numerical statistic used in text processing to reflect how important a word is to a document in a collection or corpus. It balances the frequency of a word in a document against its frequency in the entire corpus, giving higher value to more specific terms.\n", + "\n", + "Use `TF-IDF` with `MLTransform`.\n", + "\n", + "1. Compute the vocabulary of the dataset by using `ComputeAndApplyVocabulary`.\n", + "2. Use the output of `ComputeAndApplyVocabulary` to calculate the `TF-IDF` weights.\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8a8cb94b-57eb-4c4c-aa4c-22cf3193ea85", + "metadata": { + "id": "8a8cb94b-57eb-4c4c-aa4c-22cf3193ea85" + }, + "outputs": [], + "source": [ + "from apache_beam.ml.transforms.tft import TFIDF" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "970d7222-194e-460e-b698-a00f1fcafb95", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "970d7222-194e-460e-b698-a00f1fcafb95", + "outputId": "e87409ed-5e33-43fa-d3b6-a0c012636cef" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Row(x=array([1, 0, 4]), x_tfidf_weight=array([0.33333334, 0.33333334, 0.4684884 ], dtype=float32), x_vocab_index=array([0, 1, 4]))\n", + "Row(x=array([1, 0, 6, 2, 3, 5]), x_tfidf_weight=array([0.16666667, 0.16666667, 0.2342442 , 0.2342442 , 0.2342442 ,\n", + " 0.2342442 ], dtype=float32), x_vocab_index=array([0, 1, 2, 3, 5, 6]))\n" + ] + } + ], + "source": [ + "data = [\n", + " {'x': ['I', 'love', 'pie']},\n", + " {'x': ['I', 'love', 'going', 'to', 'the', 'park']}\n", + "]\n", + "delete_artifact_location(artifact_location)\n", + "options = PipelineOptions()\n", + "with beam.Pipeline(options=options) as p:\n", + " data = (\n", + " p\n", + " | beam.Create(data)\n", + " | MLTransform(write_artifact_location=artifact_location\n", + " ).with_transform(ComputeAndApplyVocabulary(columns=['x'])\n", + " ).with_transform(TFIDF(columns=['x']))\n", + " )\n", + " _ = data | beam.Map(print)" + ] + }, + { + "cell_type": "markdown", + "id": "7b1feb4f-bb0b-4f61-8349-e1ba411858cf", + "metadata": { + "id": "7b1feb4f-bb0b-4f61-8349-e1ba411858cf" + }, + "source": [ + "### TF-IDF output\n", + "\n", + "`TF-IDF` produces two output columns for a given input. For example, if you input `x`, the output column names in the dictionary are `x_vocab_index` and `x_tfidf_weight`.\n", + "\n", + "- `vocab_index`: indices of the words computed in the `ComputeAndApplyVocabulary` transform.\n", + "- `tfidif_weight`: the weight for each vocabulary index. The weight represents how important the word present at that `vocab_index` is to the document.\n" + ] + }, + { + "cell_type": "markdown", + "id": "d3b5b9dd-ed35-460b-9fb3-0ffb5c3633db", + "metadata": { + "id": "d3b5b9dd-ed35-460b-9fb3-0ffb5c3633db" + }, + "source": [ + "## Scale the data\n", + "\n", + "The following examples show two ways to scale data:\n", + "\n", + "* Scale data between 0 and 1.\n", + "* Scale data using z-score.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "3bd20692-6d14-4ece-a2e7-69a2a6fac5d4", + "metadata": { + "id": "3bd20692-6d14-4ece-a2e7-69a2a6fac5d4" + }, + "source": [ + "### Scale the data between 0 and 1\n", + "\n", + "Scale the data so that it's in the range of 0 and 1. To scale the data, the transform calculates `minimum` and `maximum` values on the whole dataset, and then performs the following calculation:\n", + "\n", + "`x = (x - x_min) / (x_max)`\n", + "\n", + "To scale the data, use the [ScaleTo01](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.transforms.tft.html#apache_beam.ml.transforms.tft.ScaleTo01) data processing transform in `MLTransform`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "841a8e1f-2f5b-4fd9-bb35-12a2393922de", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "841a8e1f-2f5b-4fd9-bb35-12a2393922de", + "outputId": "efcae38d-96f6-4394-e5f5-c36644d3a9ff" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Row(x=array([0. , 0.01010101, 0.02020202], dtype=float32), x_max=array([100.], dtype=float32), x_min=array([1.], dtype=float32))\n", + "Row(x=array([0.03030303, 0.04040404, 0.06060606], dtype=float32), x_max=array([100.], dtype=float32), x_min=array([1.], dtype=float32))\n", + "Row(x=array([0.09090909, 0.01010101, 0.09090909, 0.33333334, 1. ,\n", + " 0.53535354, 0.1919192 , 0.09090909, 0.01010101, 0.02020202,\n", + " 0.1010101 , 0.11111111], dtype=float32), x_max=array([100.], dtype=float32), x_min=array([1.], dtype=float32))\n" + ] + } + ], + "source": [ + "delete_artifact_location(artifact_location)\n", + "\n", + "from apache_beam.ml.transforms.tft import ScaleTo01\n", + "data = [\n", + " {'x': [1, 2, 3]}, {'x': [4, 5, 7]}, {'x': [10, 2, 10, 34, 100, 54, 20, 10, 2, 3, 11, 12]}]\n", + "\n", + "with beam.Pipeline() as p:\n", + " _ = (\n", + " p\n", + " | 'CreateData' >> beam.Create(data)\n", + " | 'MLTransform' >> MLTransform(write_artifact_location=artifact_location).with_transform(ScaleTo01(columns=['x']))\n", + " | 'PrintResults' >> beam.Map(print)\n", + " )\n" + ] + }, + { + "cell_type": "markdown", + "id": "b1838ecb-2168-45f8-bdf2-41ae0007cb71", + "metadata": { + "id": "b1838ecb-2168-45f8-bdf2-41ae0007cb71" + }, + "source": [ + "The output contains artifacts such as `x_max` and `x_min`, which represent the maximum and minimum values of the entire dataset.\n" + ] + }, + { + "cell_type": "markdown", + "source": [ + "### Scale the data by using the z-score\n", + "\n", + "Similar to `ScaleTo01`, use [ScaleToZScore](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.transforms.tft.html#apache_beam.ml.transforms.tft.ScaleToZScore) to scale the values by using the [z-score]([z-score](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/scale_to_z_score#:~:text=Scaling%20to%20z%2Dscore%20subtracts%20out%20the%20mean%20and%20divides%20by%20standard%20deviation.%20Note%20that%20the%20standard%20deviation%20computed%20here%20is%20based%20on%20the%20biased%20variance%20(0%20delta%20degrees%20of%20freedom)%2C%20as%20computed%20by%20analyzers.var.).\n" + ], + "metadata": { + "id": "_bHdYkuF74Fe" + }, + "id": "_bHdYkuF74Fe" + }, + { + "cell_type": "code", + "source": [ + "delete_artifact_location(artifact_location)\n", + "\n", + "from apache_beam.ml.transforms.tft import ScaleToZScore\n", + "data = [\n", + " {'x': [1, 2, 3]}, {'x': [4, 5, 7]}, {'x': [10, 2, 10, 34, 100, 54, 20, 10, 2, 3, 11, 12]}]\n", + "\n", + "# delete_artifact_location(artifact_location)\n", + "with beam.Pipeline() as p:\n", + " _ = (\n", + " p\n", + " | 'CreateData' >> beam.Create(data)\n", + " | 'MLTransform' >> MLTransform(write_artifact_location=artifact_location).with_transform(ScaleToZScore(columns=['x']))\n", + " | 'PrintResults' >> beam.Map(print)\n", + " )\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "aHK6zdfE732A", + "outputId": "8b4f5082-35a2-42c4-9342-a77f99338e17" + }, + "id": "aHK6zdfE732A", + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Row(x=array([-0.62608355, -0.5846515 , -0.54321957], dtype=float32), x_mean=array([16.11111], dtype=float32), x_var=array([582.5432], dtype=float32))\n", + "Row(x=array([-0.50178754, -0.46035555, -0.37749153], dtype=float32), x_mean=array([16.11111], dtype=float32), x_var=array([582.5432], dtype=float32))\n", + "Row(x=array([-0.25319555, -0.5846515 , -0.25319555, 0.7411725 , 3.4756844 ,\n", + " 1.5698125 , 0.16112447, -0.25319555, -0.5846515 , -0.54321957,\n", + " -0.21176355, -0.17033154], dtype=float32), x_mean=array([16.11111], dtype=float32), x_var=array([582.5432], dtype=float32))\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Use multiple transforms on a single MLTransform\n", + "\n", + "Apply the same transform on multiple columns. For example, columns `x` and\n", + "`y` require scaling by 0 and 1. For column `s`, compute vocabulary. You can use a single `MLTransform` for both of these tasks.\n", + "\n", + "When using multiple data processing transforms, either pass the transforms as chained transforms or directly as a list." + ], + "metadata": { + "id": "FNoWfyMR8JI-" + }, + "id": "FNoWfyMR8JI-" + }, + { + "cell_type": "markdown", + "source": [ + "### Use multiple data processing transforms in a single MLTransform\n", + "\n", + "The following example shows multiple data processing transforms chained to `MLTransform`." + ], + "metadata": { + "id": "Mj6hd3jZ9-nr" + }, + "id": "Mj6hd3jZ9-nr" + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6e382cca-cfd3-4ac1-956a-16480603dd5b", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "6e382cca-cfd3-4ac1-956a-16480603dd5b", + "outputId": "7f185d92-ad91-4067-c11f-66150968ec97" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Row(s=array([1, 0, 4]), x=array([0. , 0.16666667, 0.33333334], dtype=float32), x_max=array([7.], dtype=float32), x_min=array([1.], dtype=float32), y=array([0. , 0.8910891, 1. ], dtype=float32), y_max=array([111.], dtype=float32), y_min=array([10.], dtype=float32))\n", + "Row(s=array([1, 0, 6, 2, 3, 5]), x=array([0.5 , 0.6666667, 1. ], dtype=float32), x_max=array([7.], dtype=float32), x_min=array([1.], dtype=float32), y=array([0.00990099, 0.10891089, 0.3960396 ], dtype=float32), y_max=array([111.], dtype=float32), y_min=array([10.], dtype=float32))\n" + ] + } + ], + "source": [ + "delete_artifact_location(artifact_location)\n", + "\n", + "from apache_beam.ml.transforms.tft import ScaleTo01\n", + "from apache_beam.ml.transforms.tft import ComputeAndApplyVocabulary\n", + "\n", + "data = [\n", + " {'x': [1, 2, 3], 'y': [10, 100, 111], 's': ['I', 'love', 'pie']},\n", + " {'x': [4, 5, 7], 'y': [11, 21, 50], 's': ['I', 'love', 'going', 'to', 'the', 'park']}\n", + "]\n", + "\n", + "# delete_artifact_location(artifact_location)\n", + "with beam.Pipeline() as p:\n", + " _ = (\n", + " p\n", + " | 'CreateData' >> beam.Create(data)\n", + " | 'MLTransform' >> MLTransform(write_artifact_location=artifact_location).with_transform(\n", + " ScaleTo01(columns=['x', 'y'])).with_transform(ComputeAndApplyVocabulary(columns=['s']))\n", + " | 'PrintResults' >> beam.Map(print)\n", + " )" + ] + }, + { + "cell_type": "markdown", + "source": [ + "The following example shows multiple data processing transforms passed in as a list to `MLTransform`." + ], + "metadata": { + "id": "IIrL13uEG3mH" + }, + "id": "IIrL13uEG3mH" + }, + { + "cell_type": "code", + "source": [ + "delete_artifact_location(artifact_location)\n", + "\n", + "from apache_beam.ml.transforms.tft import ScaleTo01\n", + "from apache_beam.ml.transforms.tft import ComputeAndApplyVocabulary\n", + "\n", + "data = [\n", + " {'x': [1, 2, 3], 'y': [10, 100, 111], 's': ['I', 'love', 'pie']},\n", + " {'x': [4, 5, 7], 'y': [11, 21, 50], 's': ['I', 'love', 'going', 'to', 'the', 'park']}\n", + "]\n", + "\n", + "transforms = [\n", + " ScaleTo01(columns=['x', 'y']),\n", + " ComputeAndApplyVocabulary(columns=['s'])\n", + "]\n", + "\n", + "with beam.Pipeline() as p:\n", + " _ = (\n", + " p\n", + " | 'CreateData' >> beam.Create(data)\n", + " | 'MLTransform' >> MLTransform(write_artifact_location=artifact_location,\n", + " transforms=transforms)\n", + " | 'PrintResults' >> beam.Map(print)\n", + " )" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "equV7ptY-FKL", + "outputId": "9c3f9461-31e9-41de-cc5d-96e7ff5a3600" + }, + "id": "equV7ptY-FKL", + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Row(s=array([1, 0, 4]), x=array([0. , 0.16666667, 0.33333334], dtype=float32), x_max=array([7.], dtype=float32), x_min=array([1.], dtype=float32), y=array([0. , 0.8910891, 1. ], dtype=float32), y_max=array([111.], dtype=float32), y_min=array([10.], dtype=float32))\n", + "Row(s=array([1, 0, 6, 2, 3, 5]), x=array([0.5 , 0.6666667, 1. ], dtype=float32), x_max=array([7.], dtype=float32), x_min=array([1.], dtype=float32), y=array([0.00990099, 0.10891089, 0.3960396 ], dtype=float32), y_max=array([111.], dtype=float32), y_min=array([10.], dtype=float32))\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "### MLTransform for inference workloads\n", + "\n", + "The previous examples show how to preprocess data for model training. This example uses the same preprocessing steps on the inference data. By using the same steps on the inference data, you can maintain consistent results.\n", + "\n", + "Preprocess the data used by the inference by using the same preprocessing steps that you used on the data prior to training. When using `MLTransform`, pass the artifact location from the previous transforms to the parameter `read_artifact_location`. `MLTransform` uses the values and artifacts produced in the previous steps. You don't need to provide the transforms, because they are saved with the artifacts in the artifact location.\n" + ], + "metadata": { + "id": "kcnQSwkA-eSA" + }, + "id": "kcnQSwkA-eSA" + }, + { + "cell_type": "code", + "source": [ + "data = [\n", + " {'x': [2], 'y': [59, 91, 85], 's': ['love']},\n", + " {'x': [4, 5, 7], 'y': [111, 26, 30], 's': ['I', 'love', 'parks', 'and', 'dogs']}\n", + "]\n", + "\n", + "with beam.Pipeline() as p:\n", + " _ = (\n", + " p\n", + " | 'CreateData' >> beam.Create(data)\n", + " | 'MLTransform' >> MLTransform(read_artifact_location=artifact_location)\n", + " | 'PrintResults' >> beam.Map(print)\n", + " )" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "m0HpQ-Ff-Xmz", + "outputId": "1631e0f6-ee58-4c90-f90d-0e183aaaf3c2" + }, + "id": "m0HpQ-Ff-Xmz", + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Row(s=array([0]), x=array([0.16666667], dtype=float32), x_max=array([7.], dtype=float32), x_min=array([1.], dtype=float32), y=array([0.48514852, 0.8019802 , 0.7425743 ], dtype=float32), y_max=array([111.], dtype=float32), y_min=array([10.], dtype=float32))\n", + "Row(s=array([ 1, 0, -1, -1, -1]), x=array([0.5 , 0.6666667, 1. ], dtype=float32), x_max=array([7.], dtype=float32), x_min=array([1.], dtype=float32), y=array([1. , 0.15841584, 0.1980198 ], dtype=float32), y_max=array([111.], dtype=float32), y_min=array([10.], dtype=float32))\n" + ] + } + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.5" + }, + "colab": { + "provenance": [], + "include_colab_link": true + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/examples/notebooks/beam-ml/nlp_tensorflow_streaming.ipynb b/examples/notebooks/beam-ml/nlp_tensorflow_streaming.ipynb new file mode 100644 index 0000000000000..99461071ae71e --- /dev/null +++ b/examples/notebooks/beam-ml/nlp_tensorflow_streaming.ipynb @@ -0,0 +1,1085 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "id": "CbCjxaCTpTI7" + }, + "outputs": [], + "source": [ + "# @title ###### Licensed to the Apache Software Foundation (ASF), Version 2.0 (the \"License\")\n", + "\n", + "# Licensed to the Apache Software Foundation (ASF) under one\n", + "# or more contributor license agreements. See the NOTICE file\n", + "# distributed with this work for additional information\n", + "# regarding copyright ownership. The ASF licenses this file\n", + "# to you under the Apache License, Version 2.0 (the\n", + "# \"License\"); you may not use this file except in compliance\n", + "# with the License. You may obtain a copy of the License at\n", + "#\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing,\n", + "# software distributed under the License is distributed on an\n", + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n", + "# KIND, either express or implied. See the License for the\n", + "# specific language governing permissions and limitations\n", + "# under the License" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MQzPKKsd-LvV" + }, + "source": [ + "# Natural Language Processing using Streaming Beam Pipeline\n", + "\n", + "\n", + " \n", + " \n", + "
    \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + "
    \n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "m0xDYq_X-M18" + }, + "source": [ + "Natural Language Processing or NLP is a field of Artifical Intelligence that enables computers to interpret and understand human language. It involves multiple steps such as applying various preprocessing fuctions, getting predictions from a model, storing the predictions in a useful format, etc.\n", + "Sentiment Analysis is a popular use case of NLP, which allows computers to analyze the sentiment of a text. This notebook demonstrates the use of streaming pipelines in NLP.\n", + "* Extracts comments using [Youtube API](https://developers.google.com/youtube/v3) and publishing them to Pub/Sub\n", + "* Trains a TensorFlow model to predict the sentiment of text\n", + "* Stores the model in Google Cloud and creates a model handler\n", + "* Builds a Beam pipeline to:\n", + " 1. Read data from Pub/Sub\n", + " 2. Create a [PCollection](https://beam.apache.org/documentation/programming-guide/#pcollections) of input text\n", + " 3. Perform preprocessing [transforms](https://beam.apache.org/documentation/programming-guide/#transforms)\n", + " 4. RunInference to get predictions from the previously trained model\n", + " 5. Store the results\n", + "\n", + "For more information on using Apache Beam for machine learning, have a look at [AI/ML Pipelines using Beam](https://beam.apache.org/documentation/ml/overview/)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DH64zgK17BlJ" + }, + "source": [ + "## Installing necessary libraries" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "id": "3CflkK6F2Zot" + }, + "outputs": [], + "source": [ + "!pip install apache-beam[interactive,gcp] --quiet\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2eRsXCuq7Fv-" + }, + "source": [ + "## Importing libraries\n", + "\n", + "Here's a brief overview of the libraries we have imported and what they do:\n", + "* **NumPy**: It provides support for multi-dimensional arrays, along with many mathematical functions to operate on these arrays efficiently.\n", + "* **Pandas**: It allows us to work efficiently with structured or tabular data. Here we have used pandas to import a dataset from a csv file and manipulate it.\n", + "* **TextBlob**: It is a library for processing textual data and common NLP tasks. Here we have used it to analyze comments and find their sentiment polarity.\n", + "* **Apache Beam**: It is used to build and execute data processing pipelines.\n", + "* **RunInference**: It uses a pretrained model to predict results for new unseen data.\n", + "* **TFModelHandlerNumpy**: It is used to manage trained TensorFlow models that take NumPy arrays as input." + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": { + "id": "Zj50mA55ULJ5" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import tensorflow as tf\n", + "\n", + "from textblob import TextBlob\n", + "\n", + "import apache_beam as beam\n", + "from apache_beam.ml.inference.base import RunInference\n", + "from apache_beam.ml.inference.tensorflow_inference import TFModelHandlerNumpy\n", + "from apache_beam.options import pipeline_options\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BeyZU5fa6b1C" + }, + "source": [ + "## Sentiment Analysis\n", + "\n", + "Sentiment analysis is an NLP technique used to determine the sentiment or emotion expressed in a piece of text. The goal of sentiment analysis is to identify whether the text expresses a positive, negative, or neutral sentiment towards a particular subject or topic.\n", + "\n", + "Our goal is to build a streaming pipeline that ultimately tells us the sentiment of YouTube comments. For that, we need to have a pretrained model that can predict the sentiment of text." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pEnsIzu-3o_-" + }, + "source": [ + "## Training a model with labeled youtube comments dataset\n", + "The dataset can be found on [Kaggle](https://www.kaggle.com/datasets/datasnaek/youtube?select=UScomments.csv). It contains various statistics for comments on trending videos on YouTube. Since our goal is to perform a sentiment analysis on comments, we only have to consider the text.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uJMnkxSnByeL" + }, + "source": [ + "#### Reading the data from a csv file\n", + "Pandas allows us to load data from a CSV file and convert it into a DataFrame, which makes it easy to perform data analysis and manipulation." + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "id": "WPzS1qRV2ft1" + }, + "outputs": [], + "source": [ + "comm = pd.read_csv('UScomments.csv',encoding='utf8',nrows = 1000, error_bad_lines=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "W1veygZRCTA0" + }, + "source": [ + "The dataset has no labels, so we have used [TextBlob](https://textblob.readthedocs.io/en/dev/) to assign the appropriate labels by finding sentiment polarity. Polarity is a number between -1 to 1, which depicts the sentiment of a text. -1 represents most negative and 1 represents most positive sentiment." + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": { + "id": "h7tsSISv23F-" + }, + "outputs": [], + "source": [ + "pol=[]\n", + "for i in comm.comment_text.values:\n", + " try:\n", + " analysis =TextBlob(i)\n", + " pol.append(analysis.sentiment.polarity)\n", + "\n", + " except:\n", + " pol.append(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bLTIrypMCVmf" + }, + "source": [ + "We need to convert the continuous numerical values of sentiment polarity into categorical values. We will add a new column 'pol' to the DataFrame which contains the categorical labels.\n", + "* pol = 0 means positive comment (Sentiment polarity should be 0 or more)\n", + "* pol = 1 means negative comment (Sentiment polarity should be less than 0)" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": { + "id": "LrKXREKh284D" + }, + "outputs": [], + "source": [ + "comm['pol']=pol\n", + "comm['pol'][comm.pol >= 0]= 0 #positive\n", + "comm['pol'][comm.pol < 0]= 1 #negative" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9eoFlczACsRR" + }, + "source": [ + "Next, we can drop unnecessary columns from the dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": { + "id": "neZAKUkk2_Kp" + }, + "outputs": [], + "source": [ + "comm = comm.drop(['video_id','likes','replies'],axis=1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8smzuTjUC0yE" + }, + "source": [ + "### Preprocessing\n", + "\n", + "Preprocessing refers to the series of steps taken to clean, transform, and prepare raw text data. This preprocessed data can easily be fed into our ML framework and provide better results." + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": { + "id": "fOG1MWhsIQW0" + }, + "outputs": [], + "source": [ + "#Dropping null values\n", + "comm = comm.dropna()" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": { + "id": "EJ1lxVNP3vIz" + }, + "outputs": [], + "source": [ + "#Removing unnecessary characters\n", + "def remove_symbols(text):\n", + " return text.replace(\"[^a-zA-Z#]\", \" \")\n", + "comm['comment_text'] = comm['comment_text'].map(remove_symbols)" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": { + "id": "cS9hhcrG4EVP" + }, + "outputs": [], + "source": [ + "#Removing words of length 3 or less\n", + "def remove_short_words(text):\n", + " return ' '.join([str(w) for w in text.split() if len(str(w))>3])\n", + "comm['comment_text'] = comm['comment_text'].map(remove_short_words)" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": { + "id": "wopFqXpN4aBL" + }, + "outputs": [], + "source": [ + "#Converting to lowercase\n", + "def lower_case(text):\n", + " return text.lower()\n", + "comm['comment_text'] = comm['comment_text'].map(lower_case)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1GwE_Uufdrq9" + }, + "source": [ + "Next, we will divide our dataset into 2 parts:\n", + "* X = Array of string comments\n", + "* Y = Polarity Category (0 or 1)\n", + "\n", + "Here X is the unlabeled data which we will use for training and testing our model, and Y contains the corresponding labels." + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": { + "id": "i3zX3oLs5jU_" + }, + "outputs": [], + "source": [ + "X = np.array(comm['comment_text'])\n", + "Y = np.array(comm['pol'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NRAyKOXHfEpU" + }, + "source": [ + "Now we need to split the data into training and testing splits. The train split will be used for training the model, and the test split will be used to check it's performance." + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": { + "id": "2oI-gFhs7RMf" + }, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.2, random_state=0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3mgtFKPHJqHm" + }, + "source": [ + "TensorFlow's [Tokenizer](https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/text/Tokenizer) is used to convert text into an array of numbers that represent it based on the frequency of each word. This is done because an ML model can't process text directly, it can only process vectors of numbers. The [fit_on_texts](https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/text/Tokenizer#fit_on_texts) method updates the vocabulary of the tokenizer based on the words present in the data passed to it." + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": { + "id": "Oi210fpELrnf" + }, + "outputs": [], + "source": [ + "tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=10000, oov_token='')\n", + "tokenizer.fit_on_texts(comm['comment_text'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8p39hoJKM0wZ" + }, + "source": [ + "Here we have defined a function that takes a tokenizer and array of strings as input, and returns an array of tokenized strings.\n", + "* [texts_to_sequences](https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/text/Tokenizer#texts_to_sequences): Converts text to a sequence of numbers, or tokens.\n", + "* [pad_sequences](https://www.tensorflow.org/api_docs/python/tf/keras/utils/pad_sequences): Transforms tokens of different sizes to the same size." + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": { + "id": "XxtYDlejLrrA" + }, + "outputs": [], + "source": [ + "maxlen=100\n", + "def get_sequences(comments):\n", + " sequences = tokenizer.texts_to_sequences(comments)\n", + " padded = tf.keras.utils.pad_sequences(sequences, truncating = 'post', padding='post', maxlen=maxlen)\n", + " return padded" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DF2qMnof37J3" + }, + "source": [ + "Using the function defined aboved, now we will tokenize the X_train and X_test datasets." + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": { + "id": "WLZd3QhHMyZQ" + }, + "outputs": [], + "source": [ + "padded_seq_train = get_sequences(X_train)\n", + "padded_seq_test = get_sequences(X_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1mT_Js8pUbs-" + }, + "source": [ + "### Building a simple TensorFlow model to predict polarity of comments\n", + "\n", + "Now that we have our preprocessed training and testing data, we need to build a model. This model will take input as strings and predict which category (positive or negative) the string belongs to. Here is a brief description of the layers we have used to build this model:\n", + "* **Embedding Layer**: It converts tokens into dense vector representations. This allows a neural network to capture semantic relationships between words and generalize better on unseen data.\n", + "* **Bidirectional Layer**: It enhances the information flow by processing input sequences in both forward and backward directions. It is used for sequential data like natural language.\n", + "* **Dense Layer**: It connects every neuron in the previous layer to the neurons in the next layer. The comments can either be positive or negative, so there are two categories. Thus, the Dense layer provides two outputs at the end, both of which correspond to the probabibility of the text belonging to that category." + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": { + "id": "5HuyQgQBNzaB" + }, + "outputs": [], + "source": [ + "model = tf.keras.models.Sequential([\n", + "tf.keras.layers.Embedding(10000,16,input_length=maxlen),\n", + "tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(20, return_sequences=True)),\n", + "tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(10)),\n", + "tf.keras.layers.Dense(2, activation='softmax')\n", + "])\n", + "model.compile(\n", + " loss='sparse_categorical_crossentropy',\n", + " optimizer='adam',\n", + " metrics=['accuracy']\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "u1cOz6wVXKwG" + }, + "source": [ + "Next, we will create a checkpoint to save the model with the best validation accuracy." + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": { + "id": "XrosBmBqX0yO" + }, + "outputs": [], + "source": [ + "checkpoint_acc = tf.keras.callbacks.ModelCheckpoint(\"weights_acc\", monitor=\"val_accuracy\",\n", + "save_best_only=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "O-lZlCWQb6ut" + }, + "source": [ + "Now we will train our model by fitting it with the training data and using testing data for validation." + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "aQwWsnwEPY82", + "outputId": "1d0e5578-04b7-42e6-8b17-b53f18e6f8c9" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Epoch 1/10\n", + "25/25 [==============================] - ETA: 0s - loss: 0.5931 - accuracy: 0.7650" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "WARNING:absl:Found untraced functions such as _update_step_xla, lstm_cell_7_layer_call_fn, lstm_cell_7_layer_call_and_return_conditional_losses, lstm_cell_8_layer_call_fn, lstm_cell_8_layer_call_and_return_conditional_losses while saving (showing 5 of 9). These functions will not be directly callable after loading.\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r25/25 [==============================] - 60s 2s/step - loss: 0.5931 - accuracy: 0.7650 - val_loss: 0.3625 - val_accuracy: 0.8900\n", + "Epoch 2/10\n", + "25/25 [==============================] - 4s 169ms/step - loss: 0.4663 - accuracy: 0.8163 - val_loss: 0.3644 - val_accuracy: 0.8900\n", + "Epoch 3/10\n", + "25/25 [==============================] - 3s 120ms/step - loss: 0.4550 - accuracy: 0.8163 - val_loss: 0.3663 - val_accuracy: 0.8900\n", + "Epoch 4/10\n", + "25/25 [==============================] - 3s 112ms/step - loss: 0.4268 - accuracy: 0.8163 - val_loss: 0.3683 - val_accuracy: 0.8900\n", + "Epoch 5/10\n", + "25/25 [==============================] - 3s 111ms/step - loss: 0.3700 - accuracy: 0.8238 - val_loss: 0.3266 - val_accuracy: 0.8900\n", + "Epoch 6/10\n", + "25/25 [==============================] - 4s 179ms/step - loss: 0.2280 - accuracy: 0.9237 - val_loss: 0.3199 - val_accuracy: 0.8450\n", + "Epoch 7/10\n", + "25/25 [==============================] - 4s 152ms/step - loss: 0.1034 - accuracy: 0.9825 - val_loss: 0.4166 - val_accuracy: 0.8000\n", + "Epoch 8/10\n", + "25/25 [==============================] - 3s 108ms/step - loss: 0.0597 - accuracy: 0.9900 - val_loss: 0.4530 - val_accuracy: 0.8500\n", + "Epoch 9/10\n", + "25/25 [==============================] - 3s 114ms/step - loss: 0.0311 - accuracy: 0.9962 - val_loss: 0.5967 - val_accuracy: 0.8000\n", + "Epoch 10/10\n", + "25/25 [==============================] - 3s 106ms/step - loss: 0.0142 - accuracy: 0.9987 - val_loss: 0.8104 - val_accuracy: 0.7800\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 57 + } + ], + "source": [ + "model.fit(\n", + " padded_seq_train, y_train,\n", + " validation_data=(padded_seq_test, y_test),\n", + " epochs=10,\n", + " callbacks=checkpoint_acc\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ow06ihbv7JkW" + }, + "source": [ + "## Authenticating for Google cloud\n", + "\n", + "We need to authenticate our google account for the following:\n", + "* Saving the model in Google cloud\n", + "* Publishing messages in Pub/Sub\n", + "* Accessing previously published messages using a subscription" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": { + "id": "POlTUDj8D0_L" + }, + "outputs": [], + "source": [ + "from google.colab import auth\n", + "auth.authenticate_user()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TzpdiU39gA9v" + }, + "source": [ + "## Saving the model in Google cloud\n", + "\n", + "We will save the model in Google Cloud so that we can easily load it using RunInference. Then it can be used to predict results for the input data in our Beam pipeline." + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "2fpLQvm_ZEze", + "outputId": "35b6d356-c86d-43e5-e882-ddef6bc6d214" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 59 + } + ], + "source": [ + "model.load_weights('weights_acc')" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": { + "id": "-3H9m7NrZTs9" + }, + "outputs": [], + "source": [ + "save_model_dir = '' # Add the link to your GCS bucket here" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "cS5wxvxTZgcp", + "outputId": "999c6366-d887-49aa-8e40-0ebb995006d6" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "WARNING:absl:Found untraced functions such as _update_step_xla, lstm_cell_7_layer_call_fn, lstm_cell_7_layer_call_and_return_conditional_losses, lstm_cell_8_layer_call_fn, lstm_cell_8_layer_call_and_return_conditional_losses while saving (showing 5 of 9). These functions will not be directly callable after loading.\n" + ] + } + ], + "source": [ + "model.save(save_model_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "a-lXoeTqo7UG" + }, + "source": [ + "## Creating a model handler\n", + "\n", + "A model handler is used to save, load and manage trained ML models. Here we used TFModelHandlerNumpy as our input text is in the form of numpy arrays." + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": { + "id": "mTR2SonmZiBQ" + }, + "outputs": [], + "source": [ + "model_handler = TFModelHandlerNumpy(save_model_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "94aXKrR0Cnvw" + }, + "source": [ + "## Understanding Pub/Sub\n", + "\n", + "Google Cloud [Pub/Sub](https://cloud.google.com/pubsub/docs/overview) is a messaging service provided by Google Cloud Platform (GCP). It is designed to enable scalable, reliable, and real-time messaging between independent applications. Pub/Sub follows the publish-subscribe model, where messages are published by senders (publishers) to a topic, and then delivered to multiple receivers (subscribers) who have expressed interest in that topic.

    \n", + "Pub/Sub acts as an unbounded source, as it's constantly receiving and sending messages in real time. In such cases, we need to build a [Streaming Pipeline](https://beam.apache.org/documentation/sdks/python-streaming/)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QMPdmWS57Oww" + }, + "source": [ + "## Creating a publisher for a pubsub topic in Google Cloud Console\n", + "A publisher is a component that allows us to create and send messages to Google Cloud Pub/Sub. Learn more about publishing and received messages from Pub/Sub [here](https://cloud.google.com/pubsub/docs/publish-receive-messages-client-library)." + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": { + "id": "2UyYvEcz9IAx" + }, + "outputs": [], + "source": [ + "import os\n", + "from google.cloud import pubsub_v1\n", + "PROJECT_ID = '' # Add your project ID here\n", + "TOPIC = '' # Add your topic name here\n", + "publisher = pubsub_v1.PublisherClient()\n", + "topic_name = 'projects/{project_id}/topics/{topic}'.format(\n", + " project_id = PROJECT_ID,\n", + " topic = TOPIC,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2wAnY8zz7cIu" + }, + "source": [ + "## Extracting and sending comments to Pub/Sub\n", + "YouTube API provides an interface for accessing YouTube data. First, we need to enable YouTube API on the Google Cloud Console project. After that, we need to create a credential, which will further provide an API key. This API key, along with a video ID can be used to access data of that YouTube video. The Publisher created earlier is used to publish each comment to Pub/Sub.\n", + "\n", + "See examples of using the YouTube API [here](https://developers.google.com/youtube/v3/code_samples/code_snippets)." + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "frKiefXGHXve", + "outputId": "565d73f5-ec69-498b-91fc-211ce4c495e7" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Can’t wait to watch you guys grow . Harmonies are on point and the oversized early 90’s blazers are a great touch.\n", + "Amazing performance! Such an inspiring group ❤\n", + "Love the vibe\n", + "Your telling me this has less than 100 views???? Unreal\n", + "I'm happy that I lived long enough to see and hear music that tells the truth that millions of men and boys live every day! WELL DONE!\n", + "Love the unity of sound\n" + ] + } + ], + "source": [ + "from googleapiclient.discovery import build\n", + "\n", + "api_key = '' #Add your API key here\n", + "\n", + "def video_comments(video_id):\n", + " # Creating youtube resource object\n", + " youtube = build('youtube', 'v3',\n", + " developerKey=api_key)\n", + "\n", + " # Retrieve youtube video results\n", + " video_response=youtube.commentThreads().list(\n", + " part='snippet,replies',\n", + " videoId=video_id\n", + " ).execute()\n", + "\n", + " # Iterate video response\n", + " while video_response:\n", + "\n", + " # extracting required info from each object\n", + " for item in video_response['items']:\n", + "\n", + " # Extracting comments\n", + " comment = item['snippet']['topLevelComment']['snippet']['textDisplay']\n", + "\n", + " # Print comment\n", + " print(comment, end = '\\n')\n", + " data = comment.encode(\"utf-8\")\n", + "\n", + " # Publishing the comment to Pub/Sub\n", + " publisher.publish(topic_name, data)\n", + "\n", + " # Repeat until there are no next pages\n", + " if 'nextPageToken' in video_response:\n", + " video_response = youtube.commentThreads().list(\n", + " part = 'snippet,replies',\n", + " videoId = video_id\n", + " ).execute()\n", + " else:\n", + " return\n", + "\n", + "# The video ID can be extracted from the video URL, which can be represented like this\n", + "# https://www.youtube.com/watch?v=VIDEO_ID\n", + "# Enter here the desired video ID\n", + "video_id = \"fCXYrAH2gQI\"\n", + "\n", + "# Call function\n", + "video_comments(video_id)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "P51E6640pbw4" + }, + "source": [ + "## Defining utility functions\n", + "\n", + "Below we have defined some functions for our Beam pipeline to perform the following tasks:\n", + "* Print the messages received from Pub/Sub\n", + "* Tokenize the strings\n", + "* Save the predictions in a list\n", + "\n", + "These functions can be used in our pipeline by using [Map](https://beam.apache.org/documentation/transforms/python/elementwise/map/), which essentially calls the function on each element in the PCollection." + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": { + "id": "kdzXM_ooco9-" + }, + "outputs": [], + "source": [ + "# Index 0 corresponds to positive comment while index 1 corresponds to negative comment\n", + "labels = ['positive','negative']" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": { + "id": "HKbvU544Cd6v" + }, + "outputs": [], + "source": [ + "# Printing values\n", + "def print_values(element):\n", + " print(element)\n", + " return element\n", + "\n", + "# Here along with printing, we have also returned the element.\n", + "# This is done so that the element is passed into the next functions or transforms after printing." + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": { + "id": "gNpyjcEwLny9" + }, + "outputs": [], + "source": [ + "# Tokenizing the strings\n", + "def tokenize(element):\n", + " padded_seq = get_sequences([element])\n", + " return padded_seq[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": { + "id": "58xSecy4Jfn8" + }, + "outputs": [], + "source": [ + "# Saving predictions in a list\n", + "predictions = []\n", + "from tensorflow.python.ops.numpy_ops import np_config\n", + "np_config.enable_numpy_behavior()\n", + "def save_predictions(element):\n", + " list_of_predictions = element.inference.tolist()\n", + " highest_prediction = max(list_of_predictions)\n", + " ans = labels[list_of_predictions.index(highest_prediction)]\n", + " predictions.append([list_of_predictions,ans])\n", + " print(ans)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YaqDqk9w0LlA" + }, + "source": [ + "## Building an Apache Beam Pipeline\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OgH15nbK0Whw" + }, + "source": [ + "We need to build a streaming pipeline that takes data from Pub/Sub. A [Runner](https://beam.apache.org/documentation/#runners) is used to execute Beam pipelines in a distributed manner. We need to use a streaming runner to run a streaming pipeline. [InteractiveRunner](https://beam.apache.org/releases/pydoc/2.10.0/apache_beam.runners.interactive.interactive_runner.html) is suitable for this and allows developing and running Beam pipelines interactively in notebooks.\n", + "\n", + "See more details on how to use InteractiveRunner [here](https://cloud.google.com/dataflow/docs/guides/interactive-pipeline-development)." + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": { + "id": "Nq1N7JGR9Vwa" + }, + "outputs": [], + "source": [ + "# path to the topic\n", + "TOPIC_PATH = '' # Add the path to your topic here" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "metadata": { + "id": "5-06sxMtwyWN" + }, + "outputs": [], + "source": [ + "# path to the subscription\n", + "SUBS_PATH = '' # Add the path to your subscription here" + ] + }, + { + "cell_type": "markdown", + "source": [ + "Importing InteractiveRunner" + ], + "metadata": { + "id": "UliBhojEfxhq" + } + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": { + "id": "Nr6lPbQMhywM" + }, + "outputs": [], + "source": [ + "from apache_beam.runners.interactive.interactive_runner import InteractiveRunner\n", + "import apache_beam.runners.interactive.interactive_beam as ib" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": { + "id": "nuHyErBPZDVY" + }, + "outputs": [], + "source": [ + "ib.options.recording_duration = '2m' # This is how long Interactive Runner will listen to data from Pub/Sub\n", + "ib.options.recording_size_limit = 1e9 # This is the recording size limit set to 1 GB\n", + "options = pipeline_options.PipelineOptions()\n", + "options.view_as(pipeline_options.StandardOptions).streaming = True # Streaming mode is set True" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Wfo0UqQmz821" + }, + "source": [ + "The pipeline performs the following tasks:\n", + "* Reads messages from Cloud Pub Sub\n", + "* Prints the messages\n", + "* Performs preprocessing. We can reuse all of our previously defined preprocessing functions for training using [beam.Map](https://beam.apache.org/documentation/transforms/python/elementwise/map/).\n", + "* RunInference on the preprocessed data\n", + "* Prints the result and store in a list" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 627 + }, + "id": "unDFnXzcLkvF", + "outputId": "b20179e0-6bbd-407b-8752-1f282ff2c2ed" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Can’t wait to watch you guys grow . Harmonies are on point and the oversized early 90’s blazers are a great touch.\n", + "Amazing performance! Such an inspiring group ❤\n", + "Love the vibe\n", + "Your telling me this has less than 100 views???? Unreal\n", + "I'm happy that I lived long enough to see and hear music that tells the truth that millions of men and boys live every day! WELL DONE!\n", + "Love the unity of sound\n", + "positive\n", + "positive\n", + "positive\n", + "positive\n", + "positive\n", + "positive\n" + ] + } + ], + "source": [ + "with beam.Pipeline(options=options) as p:\n", + " _ = (p | \"Read From Pub/Sub\" >> beam.io.ReadFromPubSub(subscription=SUBS_PATH)\n", + " | \"Convert to String\" >> beam.Map(lambda element: element.decode('utf-8'))\n", + " | \"Print\" >> beam.Map(print_values)\n", + " | \"Remove Symbols\" >> beam.Map(remove_symbols)\n", + " | \"Remove Short Words\" >> beam.Map(remove_short_words)\n", + " | \"Lower Case\" >> beam.Map(lower_case)\n", + " | \"Tokenize\" >> beam.Map(tokenize)\n", + " | \"RunInference\" >> RunInference(model_handler)\n", + " | \"Store Predictions\" >> beam.Map(save_predictions)\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bIhqhtyvzMXR" + }, + "source": [ + "The above pipeline is a streaming pipeline, which means it will run continuously, unless we stop it manually. This is why a keyboard interrupt can be seen here." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vEch9GkVzp9S" + }, + "source": [ + "Let us print the predictions made by the model and the corresponding sentiment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "YqNoxnd4qJAD", + "outputId": "bde9cf5b-97b6-4197-9629-41d0b53c043a" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[[[0.852806806564331, 0.14719319343566895], 'positive'],\n", + " [[0.8602035045623779, 0.13979655504226685], 'positive'],\n", + " [[0.8670042753219604, 0.13299570977687836], 'positive'],\n", + " [[0.8574993014335632, 0.14250065386295319], 'positive'],\n", + " [[0.8401712775230408, 0.15982864797115326], 'positive'],\n", + " [[0.8648154735565186, 0.13518451154232025], 'positive']]" + ] + }, + "metadata": {}, + "execution_count": 38 + } + ], + "source": [ + "predictions" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/examples/notebooks/beam-ml/per_key_models.ipynb b/examples/notebooks/beam-ml/per_key_models.ipynb new file mode 100644 index 0000000000000..53845c0b3e191 --- /dev/null +++ b/examples/notebooks/beam-ml/per_key_models.ipynb @@ -0,0 +1,603 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "code", + "source": [ + "# @title ###### Licensed to the Apache Software Foundation (ASF), Version 2.0 (the \"License\")\n", + "\n", + "# Licensed to the Apache Software Foundation (ASF) under one\n", + "# or more contributor license agreements. See the NOTICE file\n", + "# distributed with this work for additional information\n", + "# regarding copyright ownership. The ASF licenses this file\n", + "# to you under the Apache License, Version 2.0 (the\n", + "# \"License\"); you may not use this file except in compliance\n", + "# with the License. You may obtain a copy of the License at\n", + "#\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing,\n", + "# software distributed under the License is distributed on an\n", + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n", + "# KIND, either express or implied. See the License for the\n", + "# specific language governing permissions and limitations\n", + "# under the License" + ], + "metadata": { + "id": "OsFaZscKSPvo" + }, + "execution_count": 1, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "# Run ML inference with multiple differently-trained models\n", + "\n", + "\n", + " \n", + " \n", + "
    \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + "
    \n" + ], + "metadata": { + "id": "ZUSiAR62SgO8" + } + }, + { + "cell_type": "markdown", + "source": [ + "Running inference with multiple differently-trained models performing the same task is useful in many scenarios, including the following examples:\n", + "\n", + "* You want to compare the performance of multiple different models.\n", + "* You have models trained on different datasets that you want to use conditionally based on additional metadata.\n", + "\n", + "In Apache Beam, the recommended way to run inference is to use the `RunInference` transform. By using a `KeyedModelHandler`, you can efficiently run inference with O(100s) of models without having to manage memory yourself.\n", + "\n", + "This notebook demonstrates how to use a `KeyedModelHandler` to run inference in an Apache Beam pipeline with multiple different models on a per-key basis. This notebook uses pretrained pipelines from Hugging Face. Before continuing with this notebook, it is recommended that you walk through the [Use RunInference in Apache Beam](https://colab.sandbox.google.com/github/apache/beam/blob/master/examples/notebooks/beam-ml/run_inference_pytorch_tensorflow_sklearn.ipynb) notebook." + ], + "metadata": { + "id": "ZAVOrrW2An1n" + } + }, + { + "cell_type": "markdown", + "source": [ + "## Install dependencies\n", + "\n", + "Install both Apache Beam and the dependencies needed by Hugging Face." + ], + "metadata": { + "id": "_fNyheQoDgGt" + } + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "B-ENznuJqArA", + "outputId": "f72963fc-82db-4d0d-9225-07f6b501e256" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "" + ] + } + ], + "source": [ + "!pip install apache_beam[gcp]>=2.51.0 --quiet\n", + "!pip install torch --quiet\n", + "!pip install transformers --quiet\n", + "\n", + "# To use the newly installed versions, restart the runtime.\n", + "exit()" + ] + }, + { + "cell_type": "code", + "source": [ + "from typing import Dict\n", + "from typing import Iterable\n", + "from typing import Tuple\n", + "\n", + "from transformers import pipeline\n", + "\n", + "import apache_beam as beam\n", + "from apache_beam.ml.inference.base import KeyedModelHandler\n", + "from apache_beam.ml.inference.base import KeyModelMapping\n", + "from apache_beam.ml.inference.base import PredictionResult\n", + "from apache_beam.ml.inference.huggingface_inference import HuggingFacePipelineModelHandler\n", + "from apache_beam.ml.inference.base import RunInference" + ], + "metadata": { + "id": "wUmBEglvsOYW" + }, + "execution_count": 1, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Define the model configurations\n", + "\n", + "A model handler is the Apache Beam method used to define the configuration needed to load and invoke models. Because this example uses two models, we define two model handlers, one for each model. Because both models are incapsulated within Hugging Face pipelines, we use the model handler `HuggingFacePipelineModelHandler`.\n", + "\n", + "For this example, load the models using Hugging Face, and then run them against an example. The models produce different outputs." + ], + "metadata": { + "id": "uEqljVgCD7hx" + } + }, + { + "cell_type": "code", + "source": [ + "distilbert_mh = HuggingFacePipelineModelHandler('text-classification', model=\"distilbert-base-uncased-finetuned-sst-2-english\")\n", + "roberta_mh = HuggingFacePipelineModelHandler('text-classification', model=\"roberta-large-mnli\")\n", + "\n", + "distilbert_pipe = pipeline('text-classification', model=\"distilbert-base-uncased-finetuned-sst-2-english\")\n", + "roberta_large_pipe = pipeline(model=\"roberta-large-mnli\")" + ], + "metadata": { + "id": "v2NJT5ZcxgH5", + "outputId": "3924d72e-5c49-477d-c50f-6d9098f5a4b2" + }, + "execution_count": 2, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Downloading (…)lve/main/config.json: 0%| | 0.00/629 [00:00-`." + ], + "metadata": { + "id": "r6GXL5PLFBY7" + } + }, + { + "cell_type": "code", + "source": [ + "class FormatExamples(beam.DoFn):\n", + " \"\"\"\n", + " Map each example to a tuple of ('-', 'example').\n", + " Use these keys to map our elements to the correct models.\n", + " \"\"\"\n", + " def process(self, element: Tuple[str, str]) -> Iterable[Tuple[str, str]]:\n", + " yield (f'distilbert-{element[1]}', element[0])\n", + " yield (f'roberta-{element[1]}', element[0])" + ], + "metadata": { + "id": "p2uVwws8zRpg" + }, + "execution_count": 6, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "Use the formatted keys to define a `KeyedModelHandler` that maps keys to the `ModelHandler` used for those keys. The `KeyedModelHandler` method lets you define an optional `max_models_per_worker_hint`, which limits the number of models that can be held in a single worker process at one time. If your worker might run out of memory, use this option. For more information about managing memory, see [Use a keyed ModelHandler](https://beam.apache.org/documentation/sdks/python-machine-learning/index.html#use-a-keyed-modelhandler)." + ], + "metadata": { + "id": "IP65_5nNGIb8" + } + }, + { + "cell_type": "code", + "source": [ + "per_key_mhs = [\n", + " KeyModelMapping(['distilbert-positive', 'distilbert-neutral', 'distilbert-negative'], distilbert_mh),\n", + " KeyModelMapping(['roberta-positive', 'roberta-neutral', 'roberta-negative'], roberta_mh)\n", + "]\n", + "mh = KeyedModelHandler(per_key_mhs, max_models_per_worker_hint=2)" + ], + "metadata": { + "id": "DZpfjeGL2hMG" + }, + "execution_count": 7, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Postprocess the results\n", + "\n", + "The `RunInference` transform returns a tuple that contains the following objects:\n", + "* the original key\n", + "* a `PredictionResult` object containing the original example and the inference\n", + "Use those outputs to extract the relevant data. Then, to compare each model's prediction, group this data by the original example." + ], + "metadata": { + "id": "_a4ZmnD5FSeG" + } + }, + { + "cell_type": "code", + "source": [ + "class ExtractResults(beam.DoFn):\n", + " \"\"\"\n", + " Extract the relevant data from the PredictionResult object.\n", + " \"\"\"\n", + " def process(self, element: Tuple[str, PredictionResult]) -> Iterable[Tuple[str, Dict[str, str]]]:\n", + " actual_sentiment = element[0].split('-')[1]\n", + " model = element[0].split('-')[0]\n", + " result = element[1]\n", + " example = result.example\n", + " predicted_sentiment = result.inference[0]['label']\n", + "\n", + " yield (example, {'model': model, 'actual_sentiment': actual_sentiment, 'predicted_sentiment': predicted_sentiment})" + ], + "metadata": { + "id": "FOwFNQA053TG" + }, + "execution_count": 8, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "Finally, print the results produced by each model." + ], + "metadata": { + "id": "JVnv4gGbFohk" + } + }, + { + "cell_type": "code", + "source": [ + "class PrintResults(beam.DoFn):\n", + " \"\"\"\n", + " Print the results produced by each model along with the actual sentiment.\n", + " \"\"\"\n", + " def process(self, element: Tuple[str, Iterable[Dict[str, str]]]):\n", + " example = element[0]\n", + " actual_sentiment = element[1][0]['actual_sentiment']\n", + " predicted_sentiment_1 = element[1][0]['predicted_sentiment']\n", + " model_1 = element[1][0]['model']\n", + " predicted_sentiment_2 = element[1][1]['predicted_sentiment']\n", + " model_2 = element[1][1]['model']\n", + "\n", + " if model_1 == 'distilbert':\n", + " distilbert_prediction = predicted_sentiment_1\n", + " roberta_prediction = predicted_sentiment_2\n", + " else:\n", + " roberta_prediction = predicted_sentiment_1\n", + " distilbert_prediction = predicted_sentiment_2\n", + "\n", + " print(f'Example: {example}\\nActual Sentiment: {actual_sentiment}\\n'\n", + " f'Distilbert Prediction: {distilbert_prediction}\\n'\n", + " f'Roberta Prediction: {roberta_prediction}\\n------------')" + ], + "metadata": { + "id": "kUQJNYOa9Q5-" + }, + "execution_count": 9, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Run the pipeline\n", + "\n", + "To run a single Apache Beam pipeline, combine the previous steps." + ], + "metadata": { + "id": "-LrpmM2PGAkf" + } + }, + { + "cell_type": "code", + "source": [ + "with beam.Pipeline() as beam_pipeline:\n", + "\n", + " formatted_examples = (\n", + " beam_pipeline\n", + " | \"ReadExamples\" >> beam.Create(examples)\n", + " | \"FormatExamples\" >> beam.ParDo(FormatExamples()))\n", + " inferences = (\n", + " formatted_examples\n", + " | \"Run Inference\" >> RunInference(mh)\n", + " | \"ExtractResults\" >> beam.ParDo(ExtractResults())\n", + " | \"GroupByExample\" >> beam.GroupByKey()\n", + " )\n", + "\n", + " inferences | beam.ParDo(PrintResults())\n", + "\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 463 + }, + "id": "B9Wti3XH0Iqe", + "outputId": "528ad732-ecf8-4877-ab6a-badad7944fed" + }, + "execution_count": 10, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "\n", + " if (typeof window.interactive_beam_jquery == 'undefined') {\n", + " var jqueryScript = document.createElement('script');\n", + " jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n", + " jqueryScript.type = 'text/javascript';\n", + " jqueryScript.onload = function() {\n", + " var datatableScript = document.createElement('script');\n", + " datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n", + " datatableScript.type = 'text/javascript';\n", + " datatableScript.onload = function() {\n", + " window.interactive_beam_jquery = jQuery.noConflict(true);\n", + " window.interactive_beam_jquery(document).ready(function($){\n", + " \n", + " });\n", + " }\n", + " document.head.appendChild(datatableScript);\n", + " };\n", + " document.head.appendChild(jqueryScript);\n", + " } else {\n", + " window.interactive_beam_jquery(document).ready(function($){\n", + " \n", + " });\n", + " }" + ] + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Example: This restaurant is awesome\n", + "Actual Sentiment: positive\n", + "Distilbert Prediction: POSITIVE\n", + "Roberta Prediction: NEUTRAL\n", + "------------\n", + "Example: This restaurant is bad\n", + "Actual Sentiment: negative\n", + "Distilbert Prediction: NEGATIVE\n", + "Roberta Prediction: NEUTRAL\n", + "------------\n", + "Example: I love chocolate\n", + "Actual Sentiment: positive\n", + "Distilbert Prediction: POSITIVE\n", + "Roberta Prediction: NEUTRAL\n", + "------------\n", + "Example: I feel fine\n", + "Actual Sentiment: neutral\n", + "Distilbert Prediction: POSITIVE\n", + "Roberta Prediction: ENTAILMENT\n", + "------------\n" + ] + } + ] + } + ] +} diff --git a/examples/notebooks/beam-ml/run_inference_generative_ai.ipynb b/examples/notebooks/beam-ml/run_inference_generative_ai.ipynb index 92d185f0a4c76..40b283982b684 100644 --- a/examples/notebooks/beam-ml/run_inference_generative_ai.ipynb +++ b/examples/notebooks/beam-ml/run_inference_generative_ai.ipynb @@ -168,7 +168,7 @@ { "cell_type": "markdown", "source": [ - "## Define utitlity functions\n", + "## Define utility functions\n", "The input and output for the [`google/flan-t5-small`](https://huggingface.co/google/flan-t5-small) model are token tensors. These utility functions are used for the conversion of text to token tensors and then back to text.\n" ], "metadata": { @@ -264,13 +264,6 @@ }, "execution_count": 7, "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "WARNING:apache_beam.runners.interactive.interactive_environment:Dependencies required for Interactive Beam PCollection visualization are not available, please use: `pip install apache-beam[interactive]` to install necessary dependencies to enable all data visualization features.\n" - ] - }, { "output_type": "display_data", "data": { @@ -302,14 +295,6 @@ }, "metadata": {} }, - { - "output_type": "stream", - "name": "stderr", - "text": [ - "WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['-f', '/root/.local/share/jupyter/runtime/kernel-f8e31a84-7d91-44b3-8c53-59a03657a833.json']\n", - "WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['-f', '/root/.local/share/jupyter/runtime/kernel-f8e31a84-7d91-44b3-8c53-59a03657a833.json']\n" - ] - }, { "output_type": "stream", "name": "stdout", @@ -320,4 +305,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/examples/notebooks/beam-ml/run_inference_huggingface.ipynb b/examples/notebooks/beam-ml/run_inference_huggingface.ipynb new file mode 100644 index 0000000000000..71f7e3f0a3fb9 --- /dev/null +++ b/examples/notebooks/beam-ml/run_inference_huggingface.ipynb @@ -0,0 +1,534 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "code", + "source": [ + "# @title ###### Licensed to the Apache Software Foundation (ASF), Version 2.0 (the \"License\")\n", + "\n", + "# Licensed to the Apache Software Foundation (ASF) under one\n", + "# or more contributor license agreements. See the NOTICE file\n", + "# distributed with this work for additional information\n", + "# regarding copyright ownership. The ASF licenses this file\n", + "# to you under the Apache License, Version 2.0 (the\n", + "# \"License\"); you may not use this file except in compliance\n", + "# with the License. You may obtain a copy of the License at\n", + "#\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing,\n", + "# software distributed under the License is distributed on an\n", + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n", + "# KIND, either express or implied. See the License for the\n", + "# specific language governing permissions and limitations\n", + "# under the License" + ], + "metadata": { + "id": "SGjEjVxwudf2" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "# Apache Beam RunInference with Hugging Face\n", + "\n", + "\n", + " \n", + " \n", + "
    \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + "
    " + ], + "metadata": { + "id": "BJ0mTOhFucGg" + } + }, + { + "cell_type": "markdown", + "source": [ + "This notebook shows how to use models from [Hugging Face](https://huggingface.co/) and [Hugging Face pipeline](https://huggingface.co/docs/transformers/main_classes/pipelines) in Apache Beam pipelines that uses the [RunInference](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.inference.base.html#apache_beam.ml.inference.base.RunInference) transform.\n", + "\n", + "Apache Beam has built-in support for Hugging Face model handlers. Hugging Face has three model handlers:\n", + "\n", + "\n", + "\n", + "* Use the [`HuggingFacePipelineModelHandler`](https://github.com/apache/beam/blob/926774dd02be5eacbe899ee5eceab23afb30abca/sdks/python/apache_beam/ml/inference/huggingface_inference.py#L567) model handler to run inference with [Hugging Face pipelines](https://huggingface.co/docs/transformers/main_classes/pipelines#pipelines).\n", + "* Use the [`HuggingFaceModelHandlerKeyedTensor`](https://github.com/apache/beam/blob/926774dd02be5eacbe899ee5eceab23afb30abca/sdks/python/apache_beam/ml/inference/huggingface_inference.py#L208) model handler to run inference with models that uses keyed tensors as inputs. For example, you might use this model handler with language modeling tasks.\n", + "* Use the [`HuggingFaceModelHandlerTensor`](https://github.com/apache/beam/blob/926774dd02be5eacbe899ee5eceab23afb30abca/sdks/python/apache_beam/ml/inference/huggingface_inference.py#L392) model handler to run inference with models that uses tensor inputs, such as `tf.Tensor` or `torch.Tensor`. \n", + "\n", + "\n", + "For more information about using RunInference, see [Get started with AI/ML pipelines](https://beam.apache.org/documentation/ml/overview/) in the Apache Beam documentation." + ], + "metadata": { + "id": "GBloorZevCXC" + } + }, + { + "cell_type": "markdown", + "source": [ + "## Install dependencies" + ], + "metadata": { + "id": "xpylrB7_2Jzk" + } + }, + { + "cell_type": "markdown", + "source": [ + "Install both Apache Beam and the required dependencies for Hugging Face." + ], + "metadata": { + "id": "IBQLg8on2S40" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yrqNSBB3qsI1" + }, + "outputs": [], + "source": [ + "!pip install torch --quiet\n", + "!pip install tensorflow --quiet\n", + "!pip install transformers==4.30.0 --quiet\n", + "!pip install apache-beam[gcp]>=2.50 --quiet" + ] + }, + { + "cell_type": "code", + "source": [ + "from typing import Dict\n", + "from typing import Iterable\n", + "from typing import Tuple\n", + "\n", + "import tensorflow as tf\n", + "import torch\n", + "from transformers import AutoTokenizer\n", + "from transformers import TFAutoModelForMaskedLM\n", + "\n", + "import apache_beam as beam\n", + "from apache_beam.ml.inference.base import KeyedModelHandler\n", + "from apache_beam.ml.inference.base import PredictionResult\n", + "from apache_beam.ml.inference.base import RunInference\n", + "from apache_beam.ml.inference.huggingface_inference import HuggingFacePipelineModelHandler\n", + "from apache_beam.ml.inference.huggingface_inference import HuggingFaceModelHandlerKeyedTensor\n", + "from apache_beam.ml.inference.huggingface_inference import HuggingFaceModelHandlerTensor\n", + "from apache_beam.ml.inference.huggingface_inference import PipelineTask\n" + ], + "metadata": { + "id": "BIDZLGFRrAmF" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Use RunInference with Hugging Face pipelines" + ], + "metadata": { + "id": "OQ1wv6xk3UeV" + } + }, + { + "cell_type": "markdown", + "source": [ + "You can use [Hugging Face pipelines](https://huggingface.co/docs/transformers/main_classes/pipelines#pipelines) with `RunInference` by using the `HuggingFacePipelineModelHandler` model handler. Similar to the Hugging Face pipelines, to instantiate the model handler, the model handler needs either the pipeline `task` or the `model` that defines the task. To pass any optional arguments to load the pipeline, use `load_pipeline_args`. To pass the optional arguments for inference, use `inference_args`.\n", + "\n", + "\n", + "\n", + "You can define the pipeline task in one of the following two ways:\n", + "\n", + "\n", + "\n", + "* In the form of string, for example `\"translation\"`. This option is similar to how the pipeline task is defined when using Hugging Face.\n", + "* In the form of a [`PipelineTask`](https://github.com/apache/beam/blob/ac936b0b89a92d836af59f3fc04f5733ad6819b3/sdks/python/apache_beam/ml/inference/huggingface_inference.py#L75) enum object defined in Apache Beam, such as `PipelineTask.Translation`.\n" + ], + "metadata": { + "id": "hWZQ49Pt3ojg" + } + }, + { + "cell_type": "markdown", + "source": [ + "### Create a model handler\n", + "\n", + "This example demonstrates a task that translates text from English to Spanish." + ], + "metadata": { + "id": "pVVg9RfET86L" + } + }, + { + "cell_type": "code", + "source": [ + "model_handler = HuggingFacePipelineModelHandler(\n", + " task=PipelineTask.Translation_XX_to_YY,\n", + " model = \"google/flan-t5-small\",\n", + " load_pipeline_args={'framework': 'pt'},\n", + " inference_args={'max_length': 200}\n", + ")" + ], + "metadata": { + "id": "aF_BDPXk3UG4" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### Define the input examples\n", + "\n", + "Use this code to define the input examples." + ], + "metadata": { + "id": "lxTImFGJUBIw" + } + }, + { + "cell_type": "code", + "source": [ + "text = [\"translate English to Spanish: How are you doing?\",\n", + " \"translate English to Spanish: This is the Apache Beam project.\"]" + ], + "metadata": { + "id": "POAuIFS_UDgE" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### Postprocess the results\n", + "\n", + "The output from the `RunInference` transform is a `PredictionResult` object. Use that output to extract inferences, and then format and print the results." + ], + "metadata": { + "id": "Ay6-7DD5TZLn" + } + }, + { + "cell_type": "code", + "source": [ + "class FormatOutput(beam.DoFn):\n", + " \"\"\"\n", + " Extract the results from PredictionResult and print the results.\n", + " \"\"\"\n", + " def process(self, element):\n", + " example = element.example\n", + " translated_text = element.inference[0]['translation_text']\n", + " print(f'Example: {example}')\n", + " print(f'Translated text: {translated_text}')\n", + " print('-' * 80)\n" + ], + "metadata": { + "id": "74I3U1JsrG0R" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### Run the pipeline\n", + "\n", + "Use the following code to run the pipeline." + ], + "metadata": { + "id": "Ve2cpHZ_UH0o" + } + }, + { + "cell_type": "code", + "source": [ + "with beam.Pipeline() as beam_pipeline:\n", + " examples = (\n", + " beam_pipeline\n", + " | \"CreateExamples\" >> beam.Create(text)\n", + " )\n", + " inferences = (\n", + " examples\n", + " | \"RunInference\" >> RunInference(model_handler)\n", + " | \"Print\" >> beam.ParDo(FormatOutput())\n", + " )" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "_xStwO3qubqr", + "outputId": "5aeef601-c3e5-4b0f-e982-183ff36dc56e" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Example: translate English to Spanish: How are you doing?\n", + "Translated text: Cómo está acerca?\n", + "--------------------------------------------------------------------------------\n", + "Example: translate English to Spanish: This is the Apache Beam project.\n", + "Translated text: Esto es el proyecto Apache Beam.\n", + "--------------------------------------------------------------------------------\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## RunInference with a pretrained model from Hugging Face Hub\n" + ], + "metadata": { + "id": "KJEsPkXnUS5y" + } + }, + { + "cell_type": "markdown", + "source": [ + "To use pretrained models directly from [Hugging Face Hub](https://huggingface.co/docs/hub/models), use either the `HuggingFaceModelHandlerTensor` model handler or the `HuggingFaceModelHandlerKeyedTensor` model handler. Which model handler you use depends on your input type:\n", + "\n", + "\n", + "* Use `HuggingFaceModelHandlerKeyedTensor` to run inference with models that uses keyed tensors as inputs.\n", + "* Use `HuggingFaceModelHandlerTensor` to run inference with models that uses tensor inputs, such as `tf.Tensor` or `torch.Tensor`.\n", + "\n", + "When you construct your pipeline, you might also need to use the following items:\n", + "\n", + "\n", + "* Use the `load_model_args` to provide optional arguments to load the model.\n", + "* Use the `inference_args` argument to do the inference.\n", + "* For TensorFlow models, specify the `framework='tf'`.\n", + "* For PyTorch models, specify the `framework='pt'`.\n", + "\n", + "\n", + "\n", + "The following language modeling task predicts the masked word in a sentence." + ], + "metadata": { + "id": "mDcpG78tWcBN" + } + }, + { + "cell_type": "markdown", + "source": [ + "### Create a model handler\n", + "\n", + "This example shows a masked language modeling task. These models take keyed tensors as inputs." + ], + "metadata": { + "id": "dU6NDE4DaRuF" + } + }, + { + "cell_type": "code", + "source": [ + "model_handler = HuggingFaceModelHandlerKeyedTensor(\n", + " model_uri=\"stevhliu/my_awesome_eli5_mlm_model\",\n", + " model_class=TFAutoModelForMaskedLM,\n", + " framework='tf',\n", + " load_model_args={'from_pt': True},\n", + " max_batch_size=1\n", + ")" + ], + "metadata": { + "id": "Zx1ep1UXWYrq" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### Define the input examples\n", + "\n", + "Use this code to define the input examples." + ], + "metadata": { + "id": "D18eQZfgcIM6" + } + }, + { + "cell_type": "code", + "source": [ + "text = ['The capital of France is Paris .',\n", + " 'It is raining cats and dogs .',\n", + " 'He looked up and saw the sun and stars .',\n", + " 'Today is Monday and tomorrow is Tuesday .',\n", + " 'There are 5 coconuts on this palm tree .']" + ], + "metadata": { + "id": "vWI_A6VrcH-m" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### Preprocess the input\n", + "\n", + "Edit the given input to replace the last word with a ``. Then, tokenize the input for doing inference." + ], + "metadata": { + "id": "-62nvMSbeNBy" + } + }, + { + "cell_type": "code", + "source": [ + "def add_mask_to_last_word(text: str) -> Tuple[str, str]:\n", + " \"\"\"Replace the last word of sentence with and return\n", + " the original sentence and the masked sentence.\"\"\"\n", + " text_list = text.split()\n", + " masked = ' '.join(text_list[:-2] + ['' + text_list[-1]])\n", + " return text, masked\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"stevhliu/my_awesome_eli5_mlm_model\")\n", + "\n", + "def tokenize_sentence(\n", + " text_and_mask: Tuple[str, str],\n", + " tokenizer) -> Tuple[str, Dict[str, tf.Tensor]]:\n", + " \"\"\"Convert string examples to tensors.\"\"\"\n", + " text, masked_text = text_and_mask\n", + " tokenized_sentence = tokenizer.encode_plus(\n", + " masked_text, return_tensors=\"tf\")\n", + "\n", + " # Workaround to manually remove batch dim until we have the feature to\n", + " # add optional batching flag.\n", + " # TODO(https://github.com/apache/beam/issues/21863): Remove when optional\n", + " # batching flag added\n", + " return text, {\n", + " k: tf.squeeze(v)\n", + " for k, v in dict(tokenized_sentence).items()\n", + " }" + ], + "metadata": { + "id": "d6TXVfWhWRzz" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### Postprocess the results\n", + "\n", + "Extract the result from the `PredictionResult` object. Then, format the output to print the actual sentence and the word predicted for the last word in the sentence." + ], + "metadata": { + "id": "KnLtuYyTfC-g" + } + }, + { + "cell_type": "code", + "source": [ + "class PostProcessor(beam.DoFn):\n", + " \"\"\"Processes the PredictionResult to get the predicted word.\n", + "\n", + " The logits are the output of the BERT Model. To get the word with the highest\n", + " probability of being the masked word, take the argmax.\n", + " \"\"\"\n", + " def __init__(self, tokenizer):\n", + " super().__init__()\n", + " self.tokenizer = tokenizer\n", + "\n", + " def process(self, element: Tuple[str, PredictionResult]) -> Iterable[str]:\n", + " text, prediction_result = element\n", + " inputs = prediction_result.example\n", + " logits = prediction_result.inference['logits']\n", + " mask_token_index = tf.where(inputs[\"input_ids\"] == self.tokenizer.mask_token_id)[0]\n", + " predicted_token_id = tf.math.argmax(logits[mask_token_index[0]], axis=-1)\n", + " decoded_word = self.tokenizer.decode(predicted_token_id)\n", + " print(f\"Actual Sentence: {text}\\nPredicted last word: {decoded_word}\")\n", + " print('-' * 80)" + ], + "metadata": { + "id": "DnWlNV1kelnq" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### Run the pipeline\n", + "\n", + "Use the following code to run the pipeline." + ], + "metadata": { + "id": "scepcVUpgD63" + } + }, + { + "cell_type": "code", + "source": [ + "with beam.Pipeline() as beam_pipeline:\n", + " tokenized_examples = (\n", + " beam_pipeline\n", + " | \"CreateExamples\" >> beam.Create(text)\n", + " | 'AddMask' >> beam.Map(add_mask_to_last_word)\n", + " | 'TokenizeSentence' >>\n", + " beam.Map(lambda x: tokenize_sentence(x, tokenizer)))\n", + "\n", + " result = (\n", + " tokenized_examples\n", + " | \"RunInference\" >> RunInference(KeyedModelHandler(model_handler))\n", + " | \"PostProcess\" >> beam.ParDo(PostProcessor(tokenizer))\n", + " )" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "IEPrQGEWgBCo", + "outputId": "218cc1f4-2613-4bf1-9666-782df020536b" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Actual Sentence: The capital of France is Paris .\n", + "Predicted last word: Paris\n", + "--------------------------------------------------------------------------------\n", + "Actual Sentence: It is raining cats and dogs .\n", + "Predicted last word: dogs\n", + "--------------------------------------------------------------------------------\n", + "Actual Sentence: He looked up and saw the sun and stars .\n", + "Predicted last word: stars\n", + "--------------------------------------------------------------------------------\n", + "Actual Sentence: Today is Monday and tomorrow is Tuesday .\n", + "Predicted last word: Tuesday\n", + "--------------------------------------------------------------------------------\n", + "Actual Sentence: There are 5 coconuts on this palm tree .\n", + "Predicted last word: tree\n", + "--------------------------------------------------------------------------------\n" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/examples/notebooks/beam-ml/run_inference_multi_model.ipynb b/examples/notebooks/beam-ml/run_inference_multi_model.ipynb index 9a99ad2cf475f..7cd144223cae0 100644 --- a/examples/notebooks/beam-ml/run_inference_multi_model.ipynb +++ b/examples/notebooks/beam-ml/run_inference_multi_model.ipynb @@ -47,8 +47,7 @@ { "cell_type": "markdown", "source": [ - "# Ensemble model using an image captioning and ranking example", - "\n", + "# Ensemble model using an image captioning and ranking example\n", "\n", " + + + + + + + + + + +
    \n", " Run in Google Colab\n", @@ -65,12 +64,12 @@ { "cell_type": "markdown", "source": [ - "A single machine learning model might not be the right solution for your task. Often, machine learning model tasks involve aggregating mutliple models together to produce one optimal predictive model and to boost performance. \n", - " \n", + "When performing complex tasks like image captioning, using a single ML model may not be the best solution.\n", + "\n", "\n", "This notebook shows how to implement a cascade model in Apache Beam using the [RunInference API](https://beam.apache.org/documentation/sdks/python-machine-learning/). The RunInference API enables you to run your Beam transforms as part of your pipeline for optimal machine learning inference.\n", "\n", - "For more information about the RunInference API, review the [RunInference notebook](https://colab.research.google.com/drive/111USL4VhUa0xt_mKJxl5nC1YLOC8_yF4?usp=sharing#scrollTo=746b67a7-3562-467f-bea3-d8cd18c14927).\n", + "For more information about the RunInference API, review the [RunInference notebook](https://colab.research.google.com/drive/111USL4VhUa0xt_mKJxl5nC1YLOC8_yF4?usp=sharing#scrollTo=746b67a7-3562-467f-bea3-d8cd18c14927) or the [Beam ML documentation](https://beam.apache.org/documentation/ml/overview/).\n", "\n", "**Note:** All images are licensed CC-BY, and creators are listed in the [LICENSE.txt](https://storage.googleapis.com/apache-beam-samples/image_captioning/LICENSE.txt) file." ], @@ -94,7 +93,7 @@ "\n", "This example shows how to generate captions on a a large set of images. Apache Beam is the ideal tool to handle this workflow. We use two models for this task:\n", "\n", - "* [BLIP](https://github.com/salesforce/BLIP): Generates a set of candidate captions for a given image. \n", + "* [BLIP](https://github.com/salesforce/BLIP): Generates a set of candidate captions for a given image.\n", "* [CLIP](https://github.com/openai/CLIP): Ranks the generated captions based on accuracy." ], "metadata": { @@ -119,7 +118,7 @@ "* Run inference with BLIP to generate a list of caption candidates.\n", "* Aggregate the generated captions with their source image.\n", "* Preprocess the aggregated image-caption pairs to rank them with CLIP.\n", - "* Run inference with CLIP to generate the caption ranking. \n", + "* Run inference with CLIP to generate the caption ranking.\n", "* Print the image names and the captions sorted according to their ranking.\n", "\n", "\n", @@ -139,13 +138,13 @@ "metadata": { "colab": { "base_uri": "https://localhost:8080/", - "height": 440 + "height": 460 }, "id": "3suC5woJLW_N", - "outputId": "d2f9f67b-361b-4ae9-f9db-ce2ff9abd509", + "outputId": "2b5e78bf-f212-4a77-9325-8808ef024c2e", "cellView": "form" }, - "execution_count": null, + "execution_count": 1, "outputs": [ { "output_type": "execute_result", @@ -158,7 +157,7 @@ ] }, "metadata": {}, - "execution_count": 3 + "execution_count": 1 } ] }, @@ -184,68 +183,34 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 1, "metadata": { - "id": "tTUZpG9_q-OW", - "colab": { - "base_uri": "https://localhost:8080/" - }, - "outputId": "9ee6407a-8e4b-4520-fe5d-54a886b6e0b1" + "id": "tTUZpG9_q-OW" }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "\u001b[K |████████████████████████████████| 2.1 MB 7.0 MB/s \n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m47.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m90.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m182.4/182.4 kB\u001b[0m \u001b[31m21.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m880.6/880.6 kB\u001b[0m \u001b[31m69.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m377.0/377.0 kB\u001b[0m \u001b[31m10.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25h\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.1/53.1 kB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25h\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.5/6.5 MB\u001b[0m \u001b[31m60.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.8/12.8 MB\u001b[0m \u001b[31m91.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25h\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m235.4/235.4 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25h Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", - " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", - " Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n", - " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", - " Building wheel for fairscale (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", - "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[0m" - ] - } - ], + "outputs": [], "source": [ "!pip install --upgrade pip --quiet\n", - "!pip install transformers==4.15.0 --quiet\n", + "!pip install transformers==4.30.2 --quiet\n", "!pip install timm==0.4.12 --quiet\n", "!pip install ftfy==6.1.1 --quiet\n", "!pip install spacy==3.4.1 --quiet\n", "!pip install fairscale==0.4.4 --quiet\n", - "!pip install apache_beam[gcp]>=2.40.0 \n", + "!pip install apache_beam[gcp]>=2.48.0\n", "\n", "# To use the newly installed versions, restart the runtime.\n", - "exit() " + "exit()" ] }, { "cell_type": "code", "source": [ "import requests\n", - "import os \n", + "import os\n", "import urllib\n", - "import json \n", + "import json\n", "import io\n", "from io import BytesIO\n", + "from typing import Sequence\n", "from typing import Iterator\n", "from typing import Iterable\n", "from typing import Tuple\n", @@ -303,7 +268,7 @@ "base_uri": "https://localhost:8080/" }, "id": "Ud4sUXV2x8LO", - "outputId": "9e12ea04-a347-426f-8145-280a5676e78b" + "outputId": "cc814ff8-d424-4880-e006-56803e0508aa" }, "execution_count": 2, "outputs": [ @@ -311,7 +276,6 @@ "output_type": "stream", "name": "stdout", "text": [ - "Error: Failed to call git rev-parse --git-dir --show-toplevel: \"fatal: not a git repository (or any of the parent directories): .git\\n\"\n", "Git LFS initialized.\n", "Cloning into 'clip-vit-base-patch32'...\n", "remote: Enumerating objects: 51, done.\u001b[K\n", @@ -362,7 +326,7 @@ "base_uri": "https://localhost:8080/" }, "id": "g4-6WwqUtxea", - "outputId": "3b04b933-aab0-4f5b-c967-ed784125bc6a" + "outputId": "29112ca0-f111-48b7-d8cc-a4e04fb7a02b" }, "execution_count": 4, "outputs": [ @@ -388,8 +352,8 @@ "from BLIP.models.blip import blip_decoder\n", "\n", "!gdown 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model*_base_caption.pth'\n", - "# The blip model is saved as a checkoint, load it and save it as a state dict since RunInference required \n", - "# a state dict for model instantiation \n", + "# The blip model is saved as a checkpoint, load it and save it as a state dict since RunInference required\n", + "# a state dict for model instantiation\n", "blip_state_dict_path = '/content/BLIP/blip_state_dict.pth'\n", "torch.save(torch.load('/content/BLIP/model*_base_caption.pth')['model'], blip_state_dict_path)" ], @@ -398,7 +362,7 @@ "base_uri": "https://localhost:8080/" }, "id": "GCvOP_iZh41c", - "outputId": "224c22b1-eda6-463c-c926-1341ec9edef8" + "outputId": "a96f0ff5-cdf7-4394-be6e-d5bfca2f3a1f" }, "execution_count": 5, "outputs": [ @@ -409,7 +373,7 @@ "Downloading...\n", "From: https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model*_base_caption.pth\n", "To: /content/BLIP/model*_base_caption.pth\n", - "100% 896M/896M [00:04<00:00, 198MB/s] \n" + "100% 896M/896M [00:04<00:00, 198MB/s]\n" ] } ] @@ -500,9 +464,9 @@ "\n", " \"\"\"\n", " Process the raw image input to a format suitable for BLIP inference. The processed\n", - " images are duplicated to the number of desired captions per image. \n", + " images are duplicated to the number of desired captions per image.\n", "\n", - " Preprocessing transformation taken from: \n", + " Preprocessing transformation taken from:\n", " https://github.com/salesforce/BLIP/blob/d10be550b2974e17ea72e74edc7948c9e5eab884/predict.py\n", " \"\"\"\n", "\n", @@ -510,7 +474,7 @@ " self._captions_per_image = captions_per_image\n", "\n", " def setup(self):\n", - " \n", + "\n", " # Initialize the image transformer.\n", " self._transform = transforms.Compose([\n", " transforms.Resize((384, 384),interpolation=InterpolationMode.BICUBIC),\n", @@ -519,7 +483,7 @@ " ])\n", "\n", " def process(self, element):\n", - " image_url, image = element \n", + " image_url, image = element\n", " # The following lines provide a workaround to turn off BatchElements.\n", " preprocessed_img = self._transform(image).unsqueeze(0)\n", " preprocessed_img = preprocessed_img.repeat(self._captions_per_image, 1, 1, 1)\n", @@ -533,7 +497,7 @@ " Process the PredictionResult to get the generated image captions\n", " \"\"\"\n", " def process(self, element : Tuple[str, Iterable[PredictionResult]]):\n", - " image_url, prediction = element \n", + " image_url, prediction = element\n", "\n", " return [(image_url, prediction.inference)]" ], @@ -546,7 +510,7 @@ { "cell_type": "markdown", "source": [ - "### Define CLIP functions \n", + "### Define CLIP functions\n", "\n", "Define the preprocessing and postprocessing functions for CLIP." ], @@ -560,9 +524,9 @@ "class PreprocessCLIPInput(beam.DoFn):\n", "\n", " \"\"\"\n", - " Process the image-caption pair to a format suitable for CLIP inference. \n", + " Process the image-caption pair to a format suitable for CLIP inference.\n", "\n", - " After grouping the raw images with the generated captions, we need to \n", + " After grouping the raw images with the generated captions, we need to\n", " preprocess them before passing them to the ranking stage (CLIP model).\n", " \"\"\"\n", "\n", @@ -572,12 +536,12 @@ " merges_file_config_path: str):\n", "\n", " self._feature_extractor_config_path = feature_extractor_config_path\n", - " self._tokenizer_vocab_config_path = tokenizer_vocab_config_path \n", + " self._tokenizer_vocab_config_path = tokenizer_vocab_config_path\n", " self._merges_file_config_path = merges_file_config_path\n", "\n", "\n", " def setup(self):\n", - " \n", + "\n", " # Initialize the CLIP feature extractor.\n", " feature_extractor_config = CLIPConfig.from_pretrained(self._feature_extractor_config_path)\n", " feature_extractor = CLIPFeatureExtractor(feature_extractor_config)\n", @@ -585,14 +549,14 @@ " # Initialize the CLIP tokenizer.\n", " tokenizer = CLIPTokenizer(self._tokenizer_vocab_config_path,\n", " self._merges_file_config_path)\n", - " \n", + "\n", " # Initialize the CLIP processor used to process the image-caption pair.\n", " self._processor = CLIPProcessor(feature_extractor=feature_extractor,\n", " tokenizer=tokenizer)\n", "\n", " def process(self, element: Tuple[str, Dict[str, List[Any]]]):\n", "\n", - " image_url, image_captions_pair = element \n", + " image_url, image_captions_pair = element\n", " # Unpack the image and captions after grouping them with 'CoGroupByKey()'.\n", " image = image_captions_pair['image'][0]\n", " captions = image_captions_pair['captions'][0]\n", @@ -600,7 +564,7 @@ " text = captions,\n", " return_tensors=\"pt\",\n", " padding=True)\n", - " \n", + "\n", " image_url_caption_pair = (image_url, captions)\n", " return [(image_url_caption_pair, preprocessed_clip_input)]\n", "\n", @@ -612,7 +576,7 @@ " The logits are the output of the CLIP model. Here, we apply a softmax activation\n", " function to the logits to get the probabilistic distribution of the relevance\n", " of each caption to the target image. After that, we sort the captions in descending\n", - " order with respect to the probabilities as a caption-probability pair. \n", + " order with respect to the probabilities as a caption-probability pair.\n", " \"\"\"\n", "\n", " def process(self, element : Tuple[Tuple[str, List[str]], Iterable[PredictionResult]]):\n", @@ -642,7 +606,9 @@ { "cell_type": "markdown", "source": [ - "Use a `KeyedModelHandler` for both models to attach a key to the general `ModelHandler`.\n", + "A `ModelHandler` is Beam's method for defining the configuration needed to load and invoke your model. Since both the BLIP and CLIP models use Pytorch and take KeyedTensors as inputs, we will use `PytorchModelHandlerKeyedTensor` for both.\n", + "\n", + "We will use a `KeyedModelHandler` for both models to attach a key to the general `ModelHandler`.\n", "The key is used for the following purposes:\n", "* To keep a reference to the image that the inference is associated with.\n", "* To aggregate transforms of different inputs.\n", @@ -654,36 +620,6 @@ "id": "BTmSPnjj8M2m" } }, - { - "cell_type": "code", - "source": [ - "class PytorchNoBatchModelHandlerKeyedTensor(PytorchModelHandlerKeyedTensor):\n", - " \"\"\"Wrapper to PytorchModelHandler to limit batch size to 1.\n", - " The caption strings generated from the BLIP tokenizer might have different\n", - " lengths. Different length strings don't work with torch.stack() in the current RunInference\n", - " implementation, because stack() requires tensors to be the same size.\n", - " Restricting max_batch_size to 1 means there is only 1 example per `batch`\n", - " in the run_inference() call.\n", - " \"\"\"\n", - " # The following lines provide a workaround to turn off BatchElements.\n", - " def batch_elements_kwargs(self):\n", - " return {'max_batch_size': 1}" - ], - "metadata": { - "id": "OaR02_wxTMpc" - }, - "execution_count": 9, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "Note that we use a `KeyedModelHandler` for both models to attach a key to the general `ModelHandler`. The key is used for aggregation transforms of different inputs." - ], - "metadata": { - "id": "gNLRO0EwvcGP" - } - }, { "cell_type": "markdown", "source": [ @@ -713,48 +649,36 @@ { "cell_type": "code", "source": [ - "class BLIPWrapper(torch.nn.Module):\n", - " \"\"\"\n", - " Wrapper around the BLIP model to overwrite the default \"forward\" method with the \"generate\" method, because BLIP uses the \n", - " \"generate\" method to produce the image captions.\n", - " \"\"\"\n", - " \n", - " def __init__(self, base_model: blip_decoder, num_beams: int, max_length: int,\n", - " min_length: int):\n", - " super().__init__()\n", - " self._model = base_model()\n", - " self._num_beams = num_beams\n", - " self._max_length = max_length\n", - " self._min_length = min_length\n", - "\n", - " def forward(self, inputs: torch.Tensor):\n", - " # Squeeze because RunInference adds an extra dimension, which is empty.\n", - " # The following lines provide a workaround to turn off BatchElements.\n", - " inputs = inputs.squeeze(0)\n", - " captions = self._model.generate(inputs,\n", - " sample=True,\n", - " num_beams=self._num_beams,\n", - " max_length=self._max_length,\n", - " min_length=self._min_length)\n", - " return [captions]\n", - "\n", - " def load_state_dict(self, state_dict: dict):\n", - " self._model.load_state_dict(state_dict)\n", - "\n", - "\n", - "BLIP_model_handler = PytorchNoBatchModelHandlerKeyedTensor(\n", + "def blip_keyed_tensor_inference_fn(\n", + " batch: Sequence[Dict[str, torch.Tensor]],\n", + " model: torch.nn.Module,\n", + " device: str,\n", + " inference_args: Optional[Dict[str, Any]] = None,\n", + " model_id: Optional[str] = None,\n", + ") -> Iterable[PredictionResult]:\n", + " # By default, Beam batches inputs for bulk inference and calls model(batch)\n", + " # Since we want to call model.generate on a single unbatched input (BLIP/CLIP\n", + " # don't handle batched inputs), we define a custom inference function.\n", + " captions = model.generate(batch[0]['inputs'],\n", + " sample=True,\n", + " num_beams=NUM_BEAMS,\n", + " max_length=MAX_CAPTION_LENGTH,\n", + " min_length=MIN_CAPTION_LENGTH)\n", + " return [PredictionResult(batch[0], captions, model_id)]\n", + "\n", + "\n", + "BLIP_model_handler = PytorchModelHandlerKeyedTensor(\n", " state_dict_path=blip_state_dict_path,\n", - " model_class=BLIPWrapper,\n", - " model_params={'base_model': blip_decoder, 'num_beams': NUM_BEAMS,\n", - " 'max_length': MAX_CAPTION_LENGTH, 'min_length': MIN_CAPTION_LENGTH},\n", - " device='GPU')\n", + " model_class=blip_decoder,\n", + " inference_fn=blip_keyed_tensor_inference_fn,\n", + " max_batch_size=1)\n", "\n", "BLIP_keyed_model_handler = KeyedModelHandler(BLIP_model_handler)" ], "metadata": { "id": "RCKBJjujVw4q" }, - "execution_count": 11, + "execution_count": 10, "outputs": [] }, { @@ -771,29 +695,33 @@ { "cell_type": "code", "source": [ - "class CLIPWrapper(CLIPModel):\n", - "\n", - " def forward(self, **kwargs: Dict[str, torch.Tensor]):\n", - " # Squeeze because RunInference adds an extra dimension, which is empty.\n", - " # The following lines provide a workaround to turn off BatchElements.\n", - " kwargs = {key: tensor.squeeze(0) for key, tensor in kwargs.items()}\n", - " output = super().forward(**kwargs)\n", - " logits = output.logits_per_image\n", - " return logits\n", - "\n", - "\n", - "CLIP_model_handler = PytorchNoBatchModelHandlerKeyedTensor(\n", + "def clip_keyed_tensor_inference_fn(\n", + " batch: Sequence[Dict[str, torch.Tensor]],\n", + " model: torch.nn.Module,\n", + " device: str,\n", + " inference_args: Optional[Dict[str, Any]] = None,\n", + " model_id: Optional[str] = None,\n", + ") -> Iterable[PredictionResult]:\n", + " # By default, Beam batches inputs for bulk inference and calls model(batch)\n", + " # Since we want to call model on a single unbatched input (BLIP/CLIP don't\n", + " # handle batched inputs), we define a custom inference function.\n", + " output = model(**batch[0], **inference_args)\n", + " return [PredictionResult(batch[0], output.logits_per_image[0], model_id)]\n", + "\n", + "\n", + "CLIP_model_handler = PytorchModelHandlerKeyedTensor(\n", " state_dict_path=clip_state_dict_path,\n", - " model_class=CLIPWrapper,\n", + " model_class=CLIPModel,\n", " model_params={'config': CLIPConfig.from_pretrained(clip_model_config_path)},\n", - " device='GPU')\n", + " inference_fn=clip_keyed_tensor_inference_fn,\n", + " max_batch_size=1)\n", "\n", "CLIP_keyed_model_handler = KeyedModelHandler(CLIP_model_handler)\n" ], "metadata": { "id": "EJw_OnZ1ZfuH" }, - "execution_count": 12, + "execution_count": 11, "outputs": [] }, { @@ -817,7 +745,7 @@ "metadata": { "id": "VJwE0bquoXOf" }, - "execution_count": 13, + "execution_count": 12, "outputs": [] }, { @@ -834,7 +762,7 @@ "source": [ "#@title\n", "license_txt_url = 'https://storage.googleapis.com/apache-beam-samples/image_captioning/LICENSE.txt'\n", - "license_dict = json.loads(urllib.request.urlopen(license_txt_url).read().decode(\"utf-8\")) \n", + "license_dict = json.loads(urllib.request.urlopen(license_txt_url).read().decode(\"utf-8\"))\n", "\n", "for image_url in images_url:\n", " response = requests.get(image_url)\n", @@ -855,7 +783,7 @@ "outputId": "6e771e4e-a76a-4855-b466-976cdf35b506", "cellView": "form" }, - "execution_count": 16, + "execution_count": null, "outputs": [ { "output_type": "display_data", @@ -918,7 +846,7 @@ "metadata": { "id": "Dcz_M9GW0Kan" }, - "execution_count": 14, + "execution_count": 13, "outputs": [] }, { @@ -947,13 +875,13 @@ "with beam.Pipeline() as pipeline:\n", "\n", " read_images = (\n", - " pipeline \n", + " pipeline\n", " | \"ReadUrl\" >> beam.Create(images_url)\n", " | \"ReadImages\" >> beam.ParDo(ReadImagesFromUrl()))\n", "\n", " blip_caption_generation = (\n", " read_images\n", - " | \"PreprocessBlipInput\" >> beam.ParDo(PreprocessBLIPInput(NUM_CAPTIONS_PER_IMAGE)) \n", + " | \"PreprocessBlipInput\" >> beam.ParDo(PreprocessBLIPInput(NUM_CAPTIONS_PER_IMAGE))\n", " | \"GenerateCaptions\" >> RunInference(BLIP_keyed_model_handler)\n", " | \"PostprocessCaptions\" >> beam.ParDo(PostprocessBLIPOutput()))\n", "\n", @@ -966,19 +894,21 @@ " clip_tokenizer_vocab_config_path,\n", " clip_merges_config_path))\n", " | \"GetRankingLogits\" >> RunInference(CLIP_keyed_model_handler)\n", - " | \"RankClipOutput\" >> beam.ParDo(RankCLIPOutput()))\n", + " | \"RankClipOutput\" >> beam.ParDo(RankCLIPOutput())\n", + " )\n", "\n", " clip_captions_ranking | \"FormatCaptions\" >> beam.ParDo(FormatCaptions(NUM_TOP_CAPTIONS_TO_DISPLAY))\n", - " " + "" ], "metadata": { "colab": { - "base_uri": "https://localhost:8080/" + "base_uri": "https://localhost:8080/", + "height": 428 }, "id": "002e-FNbmuB8", - "outputId": "49c646f1-8612-433f-b134-ea8af0ff5591" + "outputId": "1b540b1e-b146-45d6-f8d3-ccaf461a87b7" }, - "execution_count": 18, + "execution_count": 14, "outputs": [ { "output_type": "stream", @@ -986,29 +916,41 @@ "text": [ "Image: Paris-sunset\n", "\tTop 3 captions ranked by CLIP:\n", - "\t\t1: the eiffel tower in paris is silhouetted at sunset. (Caption probability: 0.23)\n", - "\t\t2: the sun sets over the city of paris, with the eiffel tower in the distance. (Caption probability: 0.19)\n", - "\t\t3: the sun sets over the eiffel tower in paris. (Caption probability: 0.17)\n", + "\t\t1: the setting sun is reflected in an orange setting sky over paris. (Caption probability: 0.28)\n", + "\t\t2: the sun rising above the eiffel tower over paris. (Caption probability: 0.23)\n", + "\t\t3: the sun setting over the eiffel tower and rooftops. (Caption probability: 0.15)\n", "\n", "\n", "Image: Wedges\n", "\tTop 3 captions ranked by CLIP:\n", - "\t\t1: a basket of baked fries with a sauce in it. (Caption probability: 0.60)\n", - "\t\t2: cooked french fries with ketchup and dip sitting in napkin. (Caption probability: 0.16)\n", - "\t\t3: some french fries with dipping sauce on the side. (Caption probability: 0.08)\n", + "\t\t1: sweet potato fries with ketchup served in bowl. (Caption probability: 0.73)\n", + "\t\t2: this is a plate of sweet potato fries with ketchup. (Caption probability: 0.16)\n", + "\t\t3: sweet potato fries and a dipping sauce are on the tray. (Caption probability: 0.06)\n", "\n", "\n", "Image: Hamsters\n", "\tTop 3 captions ranked by CLIP:\n", - "\t\t1: a person petting two small hamsters while in their home. (Caption probability: 0.51)\n", - "\t\t2: a woman holding two small white baby animals. (Caption probability: 0.23)\n", - "\t\t3: a hand holding a small mouse that looks tiny. (Caption probability: 0.09)\n", + "\t\t1: person holding two small animals in their hands. (Caption probability: 0.62)\n", + "\t\t2: a person's hand holding a small hamster in front of them. (Caption probability: 0.20)\n", + "\t\t3: a person holding a small animal in their hands. (Caption probability: 0.09)\n", "\n", "\n" ] } ] }, + { + "cell_type": "markdown", + "source": [ + "# Conclusion\n", + "\n", + "After running the pipeline, you can see the captions generated by the BLIP model and ranked by the CLIP model with all of our pre/postprocessing logic applied.\n", + "As you can see, running multi-model inference is easy with the power of Beam.\n" + ], + "metadata": { + "id": "gPCMXWgOtM_0" + } + }, { "cell_type": "markdown", "source": [ diff --git a/examples/notebooks/beam-ml/run_inference_pytorch_tensorflow_sklearn.ipynb b/examples/notebooks/beam-ml/run_inference_pytorch_tensorflow_sklearn.ipynb index 2ec05801e9bf9..a314f6cd71169 100644 --- a/examples/notebooks/beam-ml/run_inference_pytorch_tensorflow_sklearn.ipynb +++ b/examples/notebooks/beam-ml/run_inference_pytorch_tensorflow_sklearn.ipynb @@ -376,83 +376,6 @@ "outputId": "2a17a966-fe2d-45d8-b6b9-02534f40c9a8" }, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", - "Requirement already satisfied: pip in /usr/local/lib/python3.7/dist-packages (22.3)\n", - "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", - "Collecting apache_beam[gcp]==2.41.0\n", - " Downloading apache_beam-2.41.0-cp37-cp37m-manylinux2010_x86_64.whl (10.9 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.9/10.9 MB\u001b[0m \u001b[31m42.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: proto-plus<2,>=1.7.1 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (1.22.1)\n", - "Requirement already satisfied: pydot<2,>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (1.3.0)\n", - "Requirement already satisfied: numpy<1.23.0,>=1.14.3 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (1.21.6)\n", - "Requirement already satisfied: pyarrow<8.0.0,>=0.15.1 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (6.0.1)\n", - "Requirement already satisfied: fastavro<2,>=0.23.6 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (1.6.1)\n", - "Requirement already satisfied: hdfs<3.0.0,>=2.1.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (2.7.0)\n", - "Requirement already satisfied: dill<0.3.2,>=0.3.1.1 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (0.3.1.1)\n", - "Requirement already satisfied: requests<3.0.0,>=2.24.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (2.28.1)\n", - "Requirement already satisfied: python-dateutil<3,>=2.8.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (2.8.2)\n", - "Requirement already satisfied: pytz>=2018.3 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (2022.4)\n", - "Requirement already satisfied: crcmod<2.0,>=1.7 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (1.7)\n", - "Requirement already satisfied: protobuf<4,>=3.12.2 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (3.20.3)\n", - "Requirement already satisfied: cloudpickle<3,>=2.1.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (2.1.0)\n", - "Requirement already satisfied: orjson<4.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (3.8.0)\n", - "Requirement already satisfied: pymongo<4.0.0,>=3.8.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (3.12.3)\n", - "Requirement already satisfied: grpcio<2,>=1.33.1 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (1.49.1)\n", - "Requirement already satisfied: typing-extensions>=3.7.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (4.1.1)\n", - "Requirement already satisfied: httplib2<0.21.0,>=0.8 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (0.17.4)\n", - "Requirement already satisfied: google-cloud-language<2,>=1.3.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (1.3.2)\n", - "Requirement already satisfied: google-cloud-pubsub<3,>=2.1.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (2.13.10)\n", - "Requirement already satisfied: google-apitools<0.5.32,>=0.5.31 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (0.5.31)\n", - "Requirement already satisfied: google-cloud-recommendations-ai<0.8.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (0.7.1)\n", - "Requirement already satisfied: cachetools<5,>=3.1.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (4.2.4)\n", - "Requirement already satisfied: google-cloud-bigtable<2,>=0.31.1 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (1.7.2)\n", - "Requirement already satisfied: google-cloud-dlp<4,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (3.9.2)\n", - "Requirement already satisfied: google-auth-httplib2<0.2.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (0.1.0)\n", - "Requirement already satisfied: google-cloud-datastore<2,>=1.8.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (1.8.0)\n", - "Requirement already satisfied: google-cloud-spanner<2,>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (1.19.3)\n", - "Requirement already satisfied: google-cloud-bigquery-storage<2.14,>=2.6.3 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (2.13.2)\n", - "Requirement already satisfied: google-cloud-vision<2,>=0.38.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (1.0.2)\n", - "Requirement already satisfied: google-cloud-core<3,>=0.28.1 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (1.7.3)\n", - "Requirement already satisfied: google-cloud-videointelligence<2,>=1.8.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (1.16.3)\n", - "Requirement already satisfied: grpcio-gcp<1,>=0.2.2 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (0.2.2)\n", - "Requirement already satisfied: google-cloud-pubsublite<2,>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (1.6.0)\n", - "Requirement already satisfied: google-auth<3,>=1.18.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (1.35.0)\n", - "Requirement already satisfied: google-cloud-bigquery<3,>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (1.21.0)\n", - "Requirement already satisfied: google-api-core!=2.8.2,<3 in /usr/local/lib/python3.7/dist-packages (from apache_beam[gcp]==2.41.0) (1.32.0)\n", - "Requirement already satisfied: six>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core!=2.8.2,<3->apache_beam[gcp]==2.41.0) (1.15.0)\n", - "Requirement already satisfied: packaging>=14.3 in /usr/local/lib/python3.7/dist-packages (from google-api-core!=2.8.2,<3->apache_beam[gcp]==2.41.0) (21.3)\n", - "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core!=2.8.2,<3->apache_beam[gcp]==2.41.0) (1.56.4)\n", - "Requirement already satisfied: setuptools>=40.3.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core!=2.8.2,<3->apache_beam[gcp]==2.41.0) (57.4.0)\n", - "Requirement already satisfied: fasteners>=0.14 in /usr/local/lib/python3.7/dist-packages (from google-apitools<0.5.32,>=0.5.31->apache_beam[gcp]==2.41.0) (0.18)\n", - "Requirement already satisfied: oauth2client>=1.4.12 in /usr/local/lib/python3.7/dist-packages (from google-apitools<0.5.32,>=0.5.31->apache_beam[gcp]==2.41.0) (4.1.3)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.18.0->apache_beam[gcp]==2.41.0) (4.9)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.18.0->apache_beam[gcp]==2.41.0) (0.2.8)\n", - "Requirement already satisfied: google-resumable-media!=0.4.0,<0.5.0dev,>=0.3.1 in /usr/local/lib/python3.7/dist-packages (from google-cloud-bigquery<3,>=1.6.0->apache_beam[gcp]==2.41.0) (0.4.1)\n", - "Requirement already satisfied: grpc-google-iam-v1<0.13dev,>=0.12.3 in /usr/local/lib/python3.7/dist-packages (from google-cloud-bigtable<2,>=0.31.1->apache_beam[gcp]==2.41.0) (0.12.4)\n", - "Requirement already satisfied: grpcio-status>=1.16.0 in /usr/local/lib/python3.7/dist-packages (from google-cloud-pubsub<3,>=2.1.0->apache_beam[gcp]==2.41.0) (1.48.2)\n", - "Requirement already satisfied: overrides<7.0.0,>=6.0.1 in /usr/local/lib/python3.7/dist-packages (from google-cloud-pubsublite<2,>=1.2.0->apache_beam[gcp]==2.41.0) (6.5.0)\n", - "Requirement already satisfied: docopt in /usr/local/lib/python3.7/dist-packages (from hdfs<3.0.0,>=2.1.0->apache_beam[gcp]==2.41.0) (0.6.2)\n", - "Requirement already satisfied: pyparsing>=2.1.4 in /usr/local/lib/python3.7/dist-packages (from pydot<2,>=1.2.0->apache_beam[gcp]==2.41.0) (3.0.9)\n", - "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.24.0->apache_beam[gcp]==2.41.0) (1.24.3)\n", - "Requirement already satisfied: charset-normalizer<3,>=2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.24.0->apache_beam[gcp]==2.41.0) (2.1.1)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.24.0->apache_beam[gcp]==2.41.0) (2.10)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.24.0->apache_beam[gcp]==2.41.0) (2022.9.24)\n", - "Requirement already satisfied: pyasn1>=0.1.7 in /usr/local/lib/python3.7/dist-packages (from oauth2client>=1.4.12->google-apitools<0.5.32,>=0.5.31->apache_beam[gcp]==2.41.0) (0.4.8)\n", - "Installing collected packages: apache_beam\n", - " Attempting uninstall: apache_beam\n", - " Found existing installation: apache-beam 2.42.0\n", - " Uninstalling apache-beam-2.42.0:\n", - " Successfully uninstalled apache-beam-2.42.0\n", - "Successfully installed apache_beam-2.41.0\n", - "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[0m" - ] - }, { "data": { "application/vnd.colab-display-data+json": { @@ -466,213 +389,6 @@ "metadata": {}, "output_type": "display_data" }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", - "Collecting tensorflow==2.8\n", - " Downloading https://us-python.pkg.dev/colab-wheels/public/tensorflow/tensorflow-2.8.0%2Bzzzcolab20220506162203-cp37-cp37m-linux_x86_64.whl (668.3 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m668.3/668.3 MB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: libclang>=9.0.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8) (14.0.6)\n", - "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8) (3.1.0)\n", - "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8) (0.2.0)\n", - "Collecting keras<2.9,>=2.8.0rc0\n", - " Downloading keras-2.8.0-py2.py3-none-any.whl (1.4 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.4/1.4 MB\u001b[0m \u001b[31m17.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: flatbuffers>=1.12 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8) (1.12)\n", - "Requirement already satisfied: gast>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8) (0.4.0)\n", - "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8) (3.3.0)\n", - "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8) (1.15.0)\n", - "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8) (1.49.1)\n", - "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8) (4.1.1)\n", - "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8) (0.27.0)\n", - "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8) (57.4.0)\n", - "Requirement already satisfied: keras-preprocessing>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8) (1.1.2)\n", - "Requirement already satisfied: absl-py>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8) (1.3.0)\n", - "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8) (3.20.3)\n", - "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8) (1.6.3)\n", - "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8) (2.0.1)\n", - "Collecting tf-estimator-nightly==2.8.0.dev2021122109\n", - " Downloading tf_estimator_nightly-2.8.0.dev2021122109-py2.py3-none-any.whl (462 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m462.5/462.5 kB\u001b[0m \u001b[31m19.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting tensorboard<2.9,>=2.8\n", - " Downloading tensorboard-2.8.0-py3-none-any.whl (5.8 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.8/5.8 MB\u001b[0m \u001b[31m62.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8) (1.14.1)\n", - "Requirement already satisfied: numpy>=1.20 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.8) (1.21.6)\n", - "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.7/dist-packages (from astunparse>=1.6.0->tensorflow==2.8) (0.37.1)\n", - "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py>=2.9.0->tensorflow==2.8) (1.5.2)\n", - "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow==2.8) (3.4.1)\n", - "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow==2.8) (2.28.1)\n", - "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow==2.8) (0.4.6)\n", - "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow==2.8) (1.0.1)\n", - "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow==2.8) (1.8.1)\n", - "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow==2.8) (1.35.0)\n", - "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow==2.8) (0.6.1)\n", - "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow==2.8) (4.2.4)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow==2.8) (0.2.8)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow==2.8) (4.9)\n", - "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow==2.8) (1.3.1)\n", - "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow==2.8) (4.13.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow==2.8) (2.10)\n", - "Requirement already satisfied: charset-normalizer<3,>=2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow==2.8) (2.1.1)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow==2.8) (2022.9.24)\n", - "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow==2.8) (1.24.3)\n", - "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow==2.8) (3.9.0)\n", - "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow==2.8) (0.4.8)\n", - "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow==2.8) (3.2.1)\n", - "Installing collected packages: tf-estimator-nightly, keras, tensorboard, tensorflow\n", - " Attempting uninstall: keras\n", - " Found existing installation: keras 2.9.0\n", - " Uninstalling keras-2.9.0:\n", - " Successfully uninstalled keras-2.9.0\n", - " Attempting uninstall: tensorboard\n", - " Found existing installation: tensorboard 2.9.1\n", - " Uninstalling tensorboard-2.9.1:\n", - " Successfully uninstalled tensorboard-2.9.1\n", - " Attempting uninstall: tensorflow\n", - " Found existing installation: tensorflow 2.9.2\n", - " Uninstalling tensorflow-2.9.2:\n", - " Successfully uninstalled tensorflow-2.9.2\n", - "Successfully installed keras-2.8.0 tensorboard-2.8.0 tensorflow-2.8.0+zzzcolab20220506162203 tf-estimator-nightly-2.8.0.dev2021122109\n", - "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", - "Collecting tfx_bsl\n", - " Downloading tfx_bsl-1.10.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (21.6 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.6/21.6 MB\u001b[0m \u001b[31m49.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: tensorflow-metadata<1.11.0,>=1.10.0 in /usr/local/lib/python3.7/dist-packages (from tfx_bsl) (1.10.0)\n", - "Collecting tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5\n", - " Downloading tensorflow-2.10.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (578.0 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m578.0/578.0 MB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: google-api-python-client<2,>=1.7.11 in /usr/local/lib/python3.7/dist-packages (from tfx_bsl) (1.12.11)\n", - "Collecting tensorflow-serving-api!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15\n", - " Downloading tensorflow_serving_api-2.10.0-py2.py3-none-any.whl (37 kB)\n", - "Requirement already satisfied: numpy<2,>=1.16 in /usr/local/lib/python3.7/dist-packages (from tfx_bsl) (1.21.6)\n", - "Requirement already satisfied: apache-beam[gcp]<3,>=2.40 in /usr/local/lib/python3.7/dist-packages (from tfx_bsl) (2.41.0)\n", - "Requirement already satisfied: absl-py<2.0.0,>=0.9 in /usr/local/lib/python3.7/dist-packages (from tfx_bsl) (1.3.0)\n", - "Requirement already satisfied: protobuf<3.21,>=3.13 in /usr/local/lib/python3.7/dist-packages (from tfx_bsl) (3.20.3)\n", - "Requirement already satisfied: pyarrow<7,>=6 in /usr/local/lib/python3.7/dist-packages (from tfx_bsl) (6.0.1)\n", - "Requirement already satisfied: pandas<2,>=1.0 in /usr/local/lib/python3.7/dist-packages (from tfx_bsl) (1.3.5)\n", - "Requirement already satisfied: requests<3.0.0,>=2.24.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (2.28.1)\n", - "Requirement already satisfied: dill<0.3.2,>=0.3.1.1 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (0.3.1.1)\n", - "Requirement already satisfied: pymongo<4.0.0,>=3.8.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (3.12.3)\n", - "Requirement already satisfied: cloudpickle<3,>=2.1.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (2.1.0)\n", - "Requirement already satisfied: fastavro<2,>=0.23.6 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (1.6.1)\n", - "Requirement already satisfied: pydot<2,>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (1.3.0)\n", - "Requirement already satisfied: pytz>=2018.3 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (2022.4)\n", - "Requirement already satisfied: grpcio<2,>=1.33.1 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (1.49.1)\n", - "Requirement already satisfied: crcmod<2.0,>=1.7 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (1.7)\n", - "Requirement already satisfied: httplib2<0.21.0,>=0.8 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (0.17.4)\n", - "Requirement already satisfied: hdfs<3.0.0,>=2.1.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (2.7.0)\n", - "Requirement already satisfied: proto-plus<2,>=1.7.1 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (1.22.1)\n", - "Requirement already satisfied: typing-extensions>=3.7.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (4.1.1)\n", - "Requirement already satisfied: orjson<4.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (3.8.0)\n", - "Requirement already satisfied: python-dateutil<3,>=2.8.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (2.8.2)\n", - "Requirement already satisfied: cachetools<5,>=3.1.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (4.2.4)\n", - "Requirement already satisfied: google-cloud-spanner<2,>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (1.19.3)\n", - "Requirement already satisfied: grpcio-gcp<1,>=0.2.2 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (0.2.2)\n", - "Requirement already satisfied: google-cloud-videointelligence<2,>=1.8.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (1.16.3)\n", - "Requirement already satisfied: google-cloud-language<2,>=1.3.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (1.3.2)\n", - "Requirement already satisfied: google-cloud-pubsub<3,>=2.1.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (2.13.10)\n", - "Requirement already satisfied: google-cloud-core<3,>=0.28.1 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (1.7.3)\n", - "Requirement already satisfied: google-cloud-dlp<4,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (3.9.2)\n", - "Requirement already satisfied: google-auth<3,>=1.18.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (1.35.0)\n", - "Requirement already satisfied: google-auth-httplib2<0.2.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (0.1.0)\n", - "Requirement already satisfied: google-cloud-bigtable<2,>=0.31.1 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (1.7.2)\n", - "Requirement already satisfied: google-cloud-bigquery-storage<2.14,>=2.6.3 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (2.13.2)\n", - "Requirement already satisfied: google-api-core!=2.8.2,<3 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (1.32.0)\n", - "Requirement already satisfied: google-cloud-datastore<2,>=1.8.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (1.8.0)\n", - "Requirement already satisfied: google-cloud-recommendations-ai<0.8.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (0.7.1)\n", - "Requirement already satisfied: google-apitools<0.5.32,>=0.5.31 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (0.5.31)\n", - "Requirement already satisfied: google-cloud-vision<2,>=0.38.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (1.0.2)\n", - "Requirement already satisfied: google-cloud-bigquery<3,>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (1.21.0)\n", - "Requirement already satisfied: google-cloud-pubsublite<2,>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from apache-beam[gcp]<3,>=2.40->tfx_bsl) (1.6.0)\n", - "Requirement already satisfied: six<2dev,>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client<2,>=1.7.11->tfx_bsl) (1.15.0)\n", - "Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client<2,>=1.7.11->tfx_bsl) (3.0.1)\n", - "Collecting tensorflow-estimator<2.11,>=2.10.0\n", - " Downloading tensorflow_estimator-2.10.0-py2.py3-none-any.whl (438 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m438.7/438.7 kB\u001b[0m \u001b[31m31.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (3.3.0)\n", - "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (3.1.0)\n", - "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (21.3)\n", - "Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (0.4.0)\n", - "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (1.6.3)\n", - "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (2.0.1)\n", - "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (57.4.0)\n", - "Collecting protobuf<3.21,>=3.13\n", - " Downloading protobuf-3.19.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m24.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (0.2.0)\n", - "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (1.14.1)\n", - "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (0.27.0)\n", - "Collecting flatbuffers>=2.0\n", - " Downloading flatbuffers-22.9.24-py2.py3-none-any.whl (26 kB)\n", - "Collecting tensorboard<2.11,>=2.10\n", - " Downloading tensorboard-2.10.1-py3-none-any.whl (5.9 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.9/5.9 MB\u001b[0m \u001b[31m58.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting keras<2.11,>=2.10.0\n", - " Downloading keras-2.10.0-py2.py3-none-any.whl (1.7 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m38.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (14.0.6)\n", - "Requirement already satisfied: keras-preprocessing>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (1.1.2)\n", - "Requirement already satisfied: googleapis-common-protos<2,>=1.52.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-metadata<1.11.0,>=1.10.0->tfx_bsl) (1.56.4)\n", - "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.7/dist-packages (from astunparse>=1.6.0->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (0.37.1)\n", - "Requirement already satisfied: fasteners>=0.14 in /usr/local/lib/python3.7/dist-packages (from google-apitools<0.5.32,>=0.5.31->apache-beam[gcp]<3,>=2.40->tfx_bsl) (0.18)\n", - "Requirement already satisfied: oauth2client>=1.4.12 in /usr/local/lib/python3.7/dist-packages (from google-apitools<0.5.32,>=0.5.31->apache-beam[gcp]<3,>=2.40->tfx_bsl) (4.1.3)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.18.0->apache-beam[gcp]<3,>=2.40->tfx_bsl) (0.2.8)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.18.0->apache-beam[gcp]<3,>=2.40->tfx_bsl) (4.9)\n", - "Requirement already satisfied: google-resumable-media!=0.4.0,<0.5.0dev,>=0.3.1 in /usr/local/lib/python3.7/dist-packages (from google-cloud-bigquery<3,>=1.6.0->apache-beam[gcp]<3,>=2.40->tfx_bsl) (0.4.1)\n", - "Requirement already satisfied: grpc-google-iam-v1<0.13dev,>=0.12.3 in /usr/local/lib/python3.7/dist-packages (from google-cloud-bigtable<2,>=0.31.1->apache-beam[gcp]<3,>=2.40->tfx_bsl) (0.12.4)\n", - "Requirement already satisfied: grpcio-status>=1.16.0 in /usr/local/lib/python3.7/dist-packages (from google-cloud-pubsub<3,>=2.1.0->apache-beam[gcp]<3,>=2.40->tfx_bsl) (1.48.2)\n", - "Requirement already satisfied: overrides<7.0.0,>=6.0.1 in /usr/local/lib/python3.7/dist-packages (from google-cloud-pubsublite<2,>=1.2.0->apache-beam[gcp]<3,>=2.40->tfx_bsl) (6.5.0)\n", - "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py>=2.9.0->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (1.5.2)\n", - "Requirement already satisfied: docopt in /usr/local/lib/python3.7/dist-packages (from hdfs<3.0.0,>=2.1.0->apache-beam[gcp]<3,>=2.40->tfx_bsl) (0.6.2)\n", - "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (3.0.9)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.24.0->apache-beam[gcp]<3,>=2.40->tfx_bsl) (2022.9.24)\n", - "Requirement already satisfied: charset-normalizer<3,>=2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.24.0->apache-beam[gcp]<3,>=2.40->tfx_bsl) (2.1.1)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.24.0->apache-beam[gcp]<3,>=2.40->tfx_bsl) (2.10)\n", - "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.24.0->apache-beam[gcp]<3,>=2.40->tfx_bsl) (1.24.3)\n", - "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.11,>=2.10->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (1.0.1)\n", - "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.11,>=2.10->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (0.4.6)\n", - "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.11,>=2.10->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (1.8.1)\n", - "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.11,>=2.10->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (0.6.1)\n", - "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.11,>=2.10->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (3.4.1)\n", - "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.11,>=2.10->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (1.3.1)\n", - "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.11,>=2.10->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (4.13.0)\n", - "Requirement already satisfied: pyasn1>=0.1.7 in /usr/local/lib/python3.7/dist-packages (from oauth2client>=1.4.12->google-apitools<0.5.32,>=0.5.31->apache-beam[gcp]<3,>=2.40->tfx_bsl) (0.4.8)\n", - "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.11,>=2.10->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (3.9.0)\n", - "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.11,>=2.10->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5->tfx_bsl) (3.2.1)\n", - "Installing collected packages: keras, flatbuffers, tensorflow-estimator, protobuf, tensorboard, tensorflow, tensorflow-serving-api, tfx_bsl\n", - " Attempting uninstall: keras\n", - " Found existing installation: keras 2.8.0\n", - " Uninstalling keras-2.8.0:\n", - " Successfully uninstalled keras-2.8.0\n", - " Attempting uninstall: flatbuffers\n", - " Found existing installation: flatbuffers 1.12\n", - " Uninstalling flatbuffers-1.12:\n", - " Successfully uninstalled flatbuffers-1.12\n", - " Attempting uninstall: tensorflow-estimator\n", - " Found existing installation: tensorflow-estimator 2.9.0\n", - " Uninstalling tensorflow-estimator-2.9.0:\n", - " Successfully uninstalled tensorflow-estimator-2.9.0\n", - " Attempting uninstall: protobuf\n", - " Found existing installation: protobuf 3.20.3\n", - " Uninstalling protobuf-3.20.3:\n", - " Successfully uninstalled protobuf-3.20.3\n", - " Attempting uninstall: tensorboard\n", - " Found existing installation: tensorboard 2.8.0\n", - " Uninstalling tensorboard-2.8.0:\n", - " Successfully uninstalled tensorboard-2.8.0\n", - " Attempting uninstall: tensorflow\n", - " Found existing installation: tensorflow 2.8.0+zzzcolab20220506162203\n", - " Uninstalling tensorflow-2.8.0+zzzcolab20220506162203:\n", - " Successfully uninstalled tensorflow-2.8.0+zzzcolab20220506162203\n", - "Successfully installed flatbuffers-22.9.24 keras-2.10.0 protobuf-3.19.6 tensorboard-2.10.1 tensorflow-2.10.0 tensorflow-estimator-2.10.0 tensorflow-serving-api-2.10.0 tfx_bsl-1.10.1\n", - "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[0m" - ] - }, { "data": { "application/vnd.colab-display-data+json": { @@ -685,89 +401,6 @@ }, "metadata": {}, "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", - "Collecting tensorflow-text==2.8.1\n", - " Downloading tensorflow_text-2.8.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (4.9 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.9/4.9 MB\u001b[0m \u001b[31m39.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: tensorflow-hub>=0.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-text==2.8.1) (0.12.0)\n", - "Collecting tensorflow<2.9,>=2.8.0\n", - " Downloading tensorflow-2.8.3-cp37-cp37m-manylinux2010_x86_64.whl (497.9 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m497.9/497.9 MB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (2.0.1)\n", - "Requirement already satisfied: protobuf<3.20,>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (3.19.6)\n", - "Requirement already satisfied: flatbuffers>=1.12 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (22.9.24)\n", - "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (4.1.1)\n", - "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (57.4.0)\n", - "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (3.3.0)\n", - "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (1.49.1)\n", - "Requirement already satisfied: absl-py>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (1.3.0)\n", - "Collecting tensorboard<2.9,>=2.8\n", - " Using cached tensorboard-2.8.0-py3-none-any.whl (5.8 MB)\n", - "Collecting tensorflow-estimator<2.9,>=2.8\n", - " Downloading tensorflow_estimator-2.8.0-py2.py3-none-any.whl (462 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m462.3/462.3 kB\u001b[0m \u001b[31m25.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: numpy>=1.20 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (1.21.6)\n", - "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (0.27.0)\n", - "Requirement already satisfied: libclang>=9.0.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (14.0.6)\n", - "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (1.14.1)\n", - "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (0.2.0)\n", - "Collecting keras<2.9,>=2.8.0rc0\n", - " Using cached keras-2.8.0-py2.py3-none-any.whl (1.4 MB)\n", - "Requirement already satisfied: gast>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (0.4.0)\n", - "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (3.1.0)\n", - "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (1.15.0)\n", - "Requirement already satisfied: keras-preprocessing>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (1.1.2)\n", - "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (1.6.3)\n", - "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.7/dist-packages (from astunparse>=1.6.0->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (0.37.1)\n", - "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py>=2.9.0->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (1.5.2)\n", - "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (3.4.1)\n", - "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (0.4.6)\n", - "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (1.8.1)\n", - "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (2.28.1)\n", - "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (1.35.0)\n", - "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (1.0.1)\n", - "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (0.6.1)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (4.9)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (0.2.8)\n", - "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (4.2.4)\n", - "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (1.3.1)\n", - "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (4.13.0)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (2022.9.24)\n", - "Requirement already satisfied: charset-normalizer<3,>=2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (2.1.1)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (2.10)\n", - "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (1.24.3)\n", - "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (3.9.0)\n", - "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (0.4.8)\n", - "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow-text==2.8.1) (3.2.1)\n", - "Installing collected packages: tensorflow-estimator, keras, tensorboard, tensorflow, tensorflow-text\n", - " Attempting uninstall: tensorflow-estimator\n", - " Found existing installation: tensorflow-estimator 2.10.0\n", - " Uninstalling tensorflow-estimator-2.10.0:\n", - " Successfully uninstalled tensorflow-estimator-2.10.0\n", - " Attempting uninstall: keras\n", - " Found existing installation: keras 2.10.0\n", - " Uninstalling keras-2.10.0:\n", - " Successfully uninstalled keras-2.10.0\n", - " Attempting uninstall: tensorboard\n", - " Found existing installation: tensorboard 2.10.1\n", - " Uninstalling tensorboard-2.10.1:\n", - " Successfully uninstalled tensorboard-2.10.1\n", - " Attempting uninstall: tensorflow\n", - " Found existing installation: tensorflow 2.10.0\n", - " Uninstalling tensorflow-2.10.0:\n", - " Successfully uninstalled tensorflow-2.10.0\n", - "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", - "tfx-bsl 1.10.1 requires tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5, but you have tensorflow 2.8.3 which is incompatible.\n", - "tensorflow-serving-api 2.10.0 requires tensorflow<3,>=2.10.0, but you have tensorflow 2.8.3 which is incompatible.\u001b[0m\u001b[31m\n", - "\u001b[0mSuccessfully installed keras-2.8.0 tensorboard-2.8.0 tensorflow-2.8.3 tensorflow-estimator-2.8.0 tensorflow-text-2.8.1\n", - "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[0m" - ] } ], "source": [ diff --git a/examples/notebooks/beam-ml/run_inference_tensorflow.ipynb b/examples/notebooks/beam-ml/run_inference_tensorflow.ipynb index 2cb0eac860ded..ad5bb671cce27 100644 --- a/examples/notebooks/beam-ml/run_inference_tensorflow.ipynb +++ b/examples/notebooks/beam-ml/run_inference_tensorflow.ipynb @@ -1,22 +1,13 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - }, - "accelerator": "GPU" - }, "cells": [ { "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "fFjof1NgAJwu" + }, + "outputs": [], "source": [ "# @title ###### Licensed to the Apache Software Foundation (ASF), Version 2.0 (the \"License\")\n", "\n", @@ -36,13 +27,7 @@ "# KIND, either express or implied. See the License for the\n", "# specific language governing permissions and limitations\n", "# under the License" - ], - "metadata": { - "cellView": "form", - "id": "fFjof1NgAJwu" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -64,6 +49,9 @@ }, { "cell_type": "markdown", + "metadata": { + "id": "HrCtxslBGK8Z" + }, "source": [ "This notebook shows how to use the Apache Beam [RunInference](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.inference.base.html#apache_beam.ml.inference.base.RunInference) transform for [TensorFlow](https://www.tensorflow.org/).\n", "Apache Beam has built-in support for two TensorFlow model handlers: [`TFModelHandlerNumpy`](https://github.com/apache/beam/blob/ca0787642a6b3804a742326147281c99ae8d08d2/sdks/python/apache_beam/ml/inference/tensorflow_inference.py#L91) and [`TFModelHandlerTensor`](https://github.com/apache/beam/blob/ca0787642a6b3804a742326147281c99ae8d08d2/sdks/python/apache_beam/ml/inference/tensorflow_inference.py#L184).\n", @@ -86,84 +74,84 @@ " * saved weights\n", "\n", "For more information about using RunInference, see [Get started with AI/ML pipelines](https://beam.apache.org/documentation/ml/overview/) in the Apache Beam documentation." - ], - "metadata": { - "id": "HrCtxslBGK8Z" - } + ] }, { "cell_type": "markdown", + "metadata": { + "id": "gVCtGOKTHMm4" + }, "source": [ "## Before you begin\n", "Set up your environment and download dependencies." - ], - "metadata": { - "id": "gVCtGOKTHMm4" - } + ] }, { "cell_type": "markdown", + "metadata": { + "id": "YDHPlMjZRuY0" + }, "source": [ "### Install Apache Beam\n", "To use RunInference with the built-in Tensorflow model handler, install Apache Beam version 2.46.0 or later." - ], - "metadata": { - "id": "YDHPlMjZRuY0" - } + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "jBakpNZnAhqk" }, + "outputs": [], "source": [ "!pip install protobuf --quiet\n", - "!pip install apache_beam==2.46.0 --quiet" - ], - "execution_count": null, - "outputs": [] + "!pip install apache_beam==2.46.0 --quiet\n", + "\n", + "# To use the newly installed versions, restart the runtime.\n", + "exit()" + ] }, { "cell_type": "markdown", + "metadata": { + "id": "X80jy3FqHjK4" + }, "source": [ "### Authenticate with Google Cloud\n", "This notebook relies on saving your model to Google Cloud. To use your Google Cloud account, authenticate this notebook." - ], - "metadata": { - "id": "X80jy3FqHjK4" - } + ] }, { "cell_type": "code", + "execution_count": 2, "metadata": { "id": "Kz9sccyGBqz3" }, + "outputs": [], "source": [ "from google.colab import auth\n", "auth.authenticate_user()" - ], - "execution_count": 2, - "outputs": [] + ] }, { "cell_type": "markdown", + "metadata": { + "id": "40qtP6zJuMXm" + }, "source": [ "### Import dependencies and set up your bucket\n", "Use the following code to import dependencies and to set up your Google Cloud Storage bucket.\n", "\n", - "Replace `PROJECT_ID` and `BUCKET_NAME` with the ID of your project and the name of your bucket.\n", - "\n", - "**Important**: If an error occurs, restart your runtime." - ], - "metadata": { - "id": "40qtP6zJuMXm" - } + "Replace `PROJECT_ID` and `BUCKET_NAME` with the ID of your project and the name of your bucket." + ] }, { "cell_type": "code", + "execution_count": 22, "metadata": { "id": "eEle839_Akqx" }, + "outputs": [], "source": [ "import argparse\n", "from typing import Dict, Text, Any, Tuple, List\n", @@ -185,20 +173,18 @@ "\n", "save_model_dir_multiply = f'gs://{bucket}/tf-inference/model/multiply_five/v1/'\n", "save_weights_dir_multiply = f'gs://{bucket}/tf-inference/weights/multiply_five/v1/'\n" - ], - "execution_count": 22, - "outputs": [] + ] }, { "cell_type": "markdown", + "metadata": { + "id": "YzvZWEv-1oiK" + }, "source": [ "## Create and test a simple model\n", "\n", "This step creates and tests a model that predicts the 5 times table." - ], - "metadata": { - "id": "YzvZWEv-1oiK" - } + ] }, { "cell_type": "markdown", @@ -212,6 +198,7 @@ }, { "cell_type": "code", + "execution_count": 16, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -219,29 +206,10 @@ "id": "SH7iq3zeBBJ-", "outputId": "5a3d3ce4-f9d8-4d87-a1bc-05afc3c9b06e" }, - "source": [ - "# Create training data that represents the 5 times multiplication table for the numbers 0 to 99.\n", - "# x is the data and y is the labels.\n", - "x = numpy.arange(0, 100) # Examples\n", - "y = x * 5 # Labels\n", - "\n", - "# Use create_model to build a simple linear regression model.\n", - "# Note that the model has a shape of (1) for its input layer and expects a single int64 value.\n", - "def create_model():\n", - " input_layer = keras.layers.Input(shape=(1), dtype=tf.float32, name='x')\n", - " output_layer= keras.layers.Dense(1)(input_layer)\n", - " model = keras.Model(input_layer, output_layer)\n", - " model.compile(optimizer=tf.optimizers.Adam(), loss='mean_absolute_error')\n", - " return model\n", - "\n", - "model = create_model()\n", - "model.summary()" - ], - "execution_count": 16, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "Model: \"model_1\"\n", "_________________________________________________________________\n", @@ -258,21 +226,40 @@ "_________________________________________________________________\n" ] } + ], + "source": [ + "# Create training data that represents the 5 times multiplication table for the numbers 0 to 99.\n", + "# x is the data and y is the labels.\n", + "x = numpy.arange(0, 100) # Examples\n", + "y = x * 5 # Labels\n", + "\n", + "# Use create_model to build a simple linear regression model.\n", + "# Note that the model has a shape of (1) for its input layer and expects a single int64 value.\n", + "def create_model():\n", + " input_layer = keras.layers.Input(shape=(1), dtype=tf.float32, name='x')\n", + " output_layer= keras.layers.Dense(1)(input_layer)\n", + " model = keras.Model(input_layer, output_layer)\n", + " model.compile(optimizer=tf.optimizers.Adam(), loss='mean_absolute_error')\n", + " return model\n", + "\n", + "model = create_model()\n", + "model.summary()" ] }, { "cell_type": "markdown", + "metadata": { + "id": "O_a0-4Gb19cy" + }, "source": [ "### Test the model\n", "\n", "This step tests the model that you created." - ], - "metadata": { - "id": "O_a0-4Gb19cy" - } + ] }, { "cell_type": "code", + "execution_count": 17, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -280,20 +267,10 @@ "id": "5XkIYXhJBFmS", "outputId": "ad2ff8a9-522c-41f4-e5d8-dbdcb53b0ded" }, - "source": [ - "model.fit(x, y, epochs=500, verbose=0)\n", - "test_examples =[20, 40, 60, 90]\n", - "value_to_predict = numpy.array(test_examples, dtype=numpy.float32)\n", - "predictions = model.predict(value_to_predict)\n", - "\n", - "print('Test Examples ' + str(test_examples))\n", - "print('Predictions ' + str(predictions))" - ], - "execution_count": 17, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "1/1 [==============================] - 0s 38ms/step\n", "Test Examples [20, 40, 60, 90]\n", @@ -303,32 +280,44 @@ " [91.544655]]\n" ] } + ], + "source": [ + "model.fit(x, y, epochs=500, verbose=0)\n", + "test_examples =[20, 40, 60, 90]\n", + "value_to_predict = numpy.array(test_examples, dtype=numpy.float32)\n", + "predictions = model.predict(value_to_predict)\n", + "\n", + "print('Test Examples ' + str(test_examples))\n", + "print('Predictions ' + str(predictions))" ] }, { "cell_type": "markdown", + "metadata": { + "id": "Y3BC2aY8cIMI" + }, "source": [ "### Save the model\n", "\n", "This step shows how to save your model." - ], - "metadata": { - "id": "Y3BC2aY8cIMI" - } + ] }, { "cell_type": "code", - "source": [ - "model.save(save_model_dir_multiply)" - ], + "execution_count": 18, "metadata": { "id": "2JbE7WkGcAkK" }, - "execution_count": 18, - "outputs": [] + "outputs": [], + "source": [ + "model.save(save_model_dir_multiply)" + ] }, { "cell_type": "markdown", + "metadata": { + "id": "g_qVtXPeUcMS" + }, "source": [ "Instead of saving the entire model, you can [save the model weights for inference](https://www.tensorflow.org/guide/keras/save_and_serialize#saving_loading_only_the_models_weights_values). You can use this method when you need the model for inference but don't need any compilation information or optimizer state. In addition, when using transfer learning applications, you can use this method to load the weights with new models.\n", "\n", @@ -340,49 +329,32 @@ "model_handler = TFModelHandlerNumpy(path_to_weights, model_type=ModelType.SAVED_WEIGHTS, create_model_fn=build_tensorflow_model)\n", "```\n", "\n" - ], - "metadata": { - "id": "g_qVtXPeUcMS" - } + ] }, { "cell_type": "code", - "source": [ - "model.save_weights(save_weights_dir_multiply)" - ], + "execution_count": 19, "metadata": { "id": "Kl1C_NwaUbiv" }, - "execution_count": 19, - "outputs": [] + "outputs": [], + "source": [ + "model.save_weights(save_weights_dir_multiply)" + ] }, { "cell_type": "markdown", + "metadata": { + "id": "0a1zerXycQ0z" + }, "source": [ "## Run the pipeline\n", "Use the following code to run the pipeline by specifying path to the trained TensorFlow model." - ], - "metadata": { - "id": "0a1zerXycQ0z" - } + ] }, { "cell_type": "code", - "source": [ - "class FormatOutput(beam.DoFn):\n", - " def process(self, element, *args, **kwargs):\n", - " yield \"example is {example} prediction is {prediction}\".format(example=element.example, prediction=element.inference)\n", - "\n", - "\n", - "examples = numpy.array([20, 40, 60, 90], dtype=numpy.float32)\n", - "model_handler = TFModelHandlerNumpy(save_model_dir_multiply)\n", - "with beam.Pipeline() as p:\n", - " _ = (p | beam.Create(examples)\n", - " | RunInference(model_handler)\n", - " | beam.ParDo(FormatOutput())\n", - " | beam.Map(print)\n", - " )" - ], + "execution_count": 20, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -391,49 +363,24 @@ "id": "St07XoibcQSb", "outputId": "d36f77f2-d07e-4868-f4cb-d120ab54e653" }, - "execution_count": 20, "outputs": [ { - "output_type": "stream", "name": "stderr", + "output_type": "stream", "text": [ "WARNING:apache_beam.runners.interactive.interactive_environment:Dependencies required for Interactive Beam PCollection visualization are not available, please use: `pip install apache-beam[interactive]` to install necessary dependencies to enable all data visualization features.\n" ] }, { - "output_type": "display_data", "data": { - "application/javascript": [ - "\n", - " if (typeof window.interactive_beam_jquery == 'undefined') {\n", - " var jqueryScript = document.createElement('script');\n", - " jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n", - " jqueryScript.type = 'text/javascript';\n", - " jqueryScript.onload = function() {\n", - " var datatableScript = document.createElement('script');\n", - " datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n", - " datatableScript.type = 'text/javascript';\n", - " datatableScript.onload = function() {\n", - " window.interactive_beam_jquery = jQuery.noConflict(true);\n", - " window.interactive_beam_jquery(document).ready(function($){\n", - " \n", - " });\n", - " }\n", - " document.head.appendChild(datatableScript);\n", - " };\n", - " document.head.appendChild(jqueryScript);\n", - " } else {\n", - " window.interactive_beam_jquery(document).ready(function($){\n", - " \n", - " });\n", - " }" - ] + "application/javascript": "\n if (typeof window.interactive_beam_jquery == 'undefined') {\n var jqueryScript = document.createElement('script');\n jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n jqueryScript.type = 'text/javascript';\n jqueryScript.onload = function() {\n var datatableScript = document.createElement('script');\n datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n datatableScript.type = 'text/javascript';\n datatableScript.onload = function() {\n window.interactive_beam_jquery = jQuery.noConflict(true);\n window.interactive_beam_jquery(document).ready(function($){\n \n });\n }\n document.head.appendChild(datatableScript);\n };\n document.head.appendChild(jqueryScript);\n } else {\n window.interactive_beam_jquery(document).ready(function($){\n \n });\n }" }, - "metadata": {} + "metadata": {}, + "output_type": "display_data" }, { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "example is 20.0 prediction is [21.896107]\n", "example is 40.0 prediction is [41.795692]\n", @@ -441,32 +388,37 @@ "example is 90.0 prediction is [91.544655]\n" ] } + ], + "source": [ + "class FormatOutput(beam.DoFn):\n", + " def process(self, element, *args, **kwargs):\n", + " yield \"example is {example} prediction is {prediction}\".format(example=element.example, prediction=element.inference)\n", + "\n", + "\n", + "examples = numpy.array([20, 40, 60, 90], dtype=numpy.float32)\n", + "model_handler = TFModelHandlerNumpy(save_model_dir_multiply)\n", + "with beam.Pipeline() as p:\n", + " _ = (p | beam.Create(examples)\n", + " | RunInference(model_handler)\n", + " | beam.ParDo(FormatOutput())\n", + " | beam.Map(print)\n", + " )" ] }, { "cell_type": "markdown", + "metadata": { + "id": "0lbPamYGV8E6" + }, "source": [ "Use the following code to run the pipeline with the saved weights of a TensorFlow model.\n", "\n", "To load the model with saved weights, the `TFModelHandlerNumpy` class requires a `create_model` function that builds and returns a TensorFlow model that is compatible with the saved weights." - ], - "metadata": { - "id": "0lbPamYGV8E6" - } + ] }, { "cell_type": "code", - "source": [ - "from apache_beam.ml.inference.tensorflow_inference import ModelType\n", - "examples = numpy.array([20, 40, 60, 90], dtype=numpy.float32)\n", - "model_handler = TFModelHandlerNumpy(save_weights_dir_multiply, model_type=ModelType.SAVED_WEIGHTS, create_model_fn=create_model)\n", - "with beam.Pipeline() as p:\n", - " _ = (p | beam.Create(examples)\n", - " | RunInference(model_handler)\n", - " | beam.ParDo(FormatOutput())\n", - " | beam.Map(print)\n", - " )" - ], + "execution_count": 21, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -474,11 +426,10 @@ "id": "QQam0O4cWG42", "outputId": "d2ab8603-cdc7-4cd6-b909-e6edfeaa5422" }, - "execution_count": 21, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "example is 20.0 prediction is [21.896107]\n", "example is 40.0 prediction is [41.795692]\n", @@ -486,10 +437,24 @@ "example is 90.0 prediction is [91.544655]\n" ] } + ], + "source": [ + "from apache_beam.ml.inference.tensorflow_inference import ModelType\n", + "examples = numpy.array([20, 40, 60, 90], dtype=numpy.float32)\n", + "model_handler = TFModelHandlerNumpy(save_weights_dir_multiply, model_type=ModelType.SAVED_WEIGHTS, create_model_fn=create_model)\n", + "with beam.Pipeline() as p:\n", + " _ = (p | beam.Create(examples)\n", + " | RunInference(model_handler)\n", + " | beam.ParDo(FormatOutput())\n", + " | beam.Map(print)\n", + " )" ] }, { "cell_type": "markdown", + "metadata": { + "id": "tRLArcjOcYuO" + }, "source": [ "## Use a keyed model handler\n", "To use a keyed model handler, use `KeyedModelHandler` with TensorFlow by using `TFModelHandlerNumpy`.\n", @@ -498,30 +463,11 @@ "\n", "* If you know that keys are associated with your examples, use `beam.KeyedModelHandler` to wrap the model handler.\n", "* If you don't know whether keys are associated with your examples, use `beam.MaybeKeyedModelHandler`." - ], - "metadata": { - "id": "tRLArcjOcYuO" - } + ] }, { "cell_type": "code", - "source": [ - "class FormatOutputKeyed(FormatOutput):\n", - " # To simplify, inherit from FormatOutput.\n", - " def process(self, tuple_in: Tuple):\n", - " key, element = tuple_in\n", - " output = super().process(element)\n", - " yield \"{} : {}\".format(key, [op for op in output])\n", - "\n", - "examples = numpy.array([(1,20), (2,40), (3,60), (4,90)], dtype=numpy.float32)\n", - "keyed_model_handler = KeyedModelHandler(TFModelHandlerNumpy(save_model_dir_multiply))\n", - "with beam.Pipeline() as p:\n", - " _ = (p | 'CreateExamples' >> beam.Create(examples)\n", - " | RunInference(keyed_model_handler)\n", - " | beam.ParDo(FormatOutputKeyed())\n", - " | beam.Map(print)\n", - " )" - ], + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -529,11 +475,10 @@ "id": "P6l9RwL2cAW3", "outputId": "03459fea-7d0a-4501-93cb-18bbad915d13" }, - "execution_count": null, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "1.0 : ['example is 20.0 prediction is [51.815357]']\n", "2.0 : ['example is 40.0 prediction is [101.63492]']\n", @@ -541,7 +486,39 @@ "4.0 : ['example is 90.0 prediction is [226.18384]']\n" ] } + ], + "source": [ + "class FormatOutputKeyed(FormatOutput):\n", + " # To simplify, inherit from FormatOutput.\n", + " def process(self, tuple_in: Tuple):\n", + " key, element = tuple_in\n", + " output = super().process(element)\n", + " yield \"{} : {}\".format(key, [op for op in output])\n", + "\n", + "examples = numpy.array([(1,20), (2,40), (3,60), (4,90)], dtype=numpy.float32)\n", + "keyed_model_handler = KeyedModelHandler(TFModelHandlerNumpy(save_model_dir_multiply))\n", + "with beam.Pipeline() as p:\n", + " _ = (p | 'CreateExamples' >> beam.Create(examples)\n", + " | RunInference(keyed_model_handler)\n", + " | beam.ParDo(FormatOutputKeyed())\n", + " | beam.Map(print)\n", + " )" ] } - ] -} \ No newline at end of file + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/examples/notebooks/beam-ml/run_inference_vertex_ai.ipynb b/examples/notebooks/beam-ml/run_inference_vertex_ai.ipynb new file mode 100644 index 0000000000000..46bfc0f2fc003 --- /dev/null +++ b/examples/notebooks/beam-ml/run_inference_vertex_ai.ipynb @@ -0,0 +1,396 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "fFjof1NgAJwu" + }, + "outputs": [], + "source": [ + "# @title ###### Licensed to the Apache Software Foundation (ASF), Version 2.0 (the \"License\")\n", + "\n", + "# Licensed to the Apache Software Foundation (ASF) under one\n", + "# or more contributor license agreements. See the NOTICE file\n", + "# distributed with this work for additional information\n", + "# regarding copyright ownership. The ASF licenses this file\n", + "# to you under the Apache License, Version 2.0 (the\n", + "# \"License\"); you may not use this file except in compliance\n", + "# with the License. You may obtain a copy of the License at\n", + "#\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing,\n", + "# software distributed under the License is distributed on an\n", + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n", + "# KIND, either express or implied. See the License for the\n", + "# specific language governing permissions and limitations\n", + "# under the License" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "A8xNRyZMW1yK" + }, + "source": [ + "# Apache Beam RunInference with Vertex AI\n", + "\n", + "\n", + " \n", + " \n", + "
    \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + "
    \n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HrCtxslBGK8Z" + }, + "source": [ + "This notebook shows how to use the Apache Beam [RunInference](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.inference.base.html#apache_beam.ml.inference.base.RunInference) transform for image classification with [Vertex AI](https://cloud.google.com/vertex-ai).\n", + "Apache Beam has built-in support for sending requests to a remotely deployed Vertex AI endpoint by using the [`VertexAIModelHandlerJSON`](https://github.com/apache/beam/blob/395c4d15bb74351b0aa020dc7463de8d85766e07/sdks/python/apache_beam/ml/inference/vertex_ai_inference.py#L61) class. The input for this class is a JSON-serializable type, like a list.\n", + "\n", + "When you use remote inference with Vertex AI, consider the following factors:\n", + "1. Public endpoints have a maximum request size of 1.5 MB. If you want to send larger requests, you must configure a [private endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints) and run your pipeline within the same VPC network. You might want to send larger requests if you send batches of requests.\n", + "2. Inputs to the Vertex AI model handler must be JSON serializable. If the inputs aren't JSON serializable, the request to the endpoint fails.\n", + "3. Hosting a model on Vertex AI and deploying it to an endpoint incurs cost from Google Cloud.\n", + "\n", + "This notebook demonstrates the following steps:\n", + "- Configure access to a public Vertex AI endpoint.\n", + "- Set up example data.\n", + "- Run those examples with the built-in model handlers and get a prediction inside an Apache Beam pipeline.\n", + "\n", + "For more information about using RunInference, see [Get started with AI/ML pipelines](https://beam.apache.org/documentation/ml/overview/) in the Apache Beam documentation." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gVCtGOKTHMm4" + }, + "source": [ + "## Before you begin\n", + "Set up your environment and download dependencies." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ryC1AntVGtLi" + }, + "source": [ + "### Prerequisites\n", + "To run this notebook, first follow the steps for training a custom model in the Vertex AI [\"Hello Custom Training\"](https://cloud.google.com/vertex-ai/docs/tutorials/image-recognition-custom) tutorial. At minimum, you need to have a trained image classification model deployed to an endpoint within your Google Cloud project." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YDHPlMjZRuY0" + }, + "source": [ + "### Install Apache Beam\n", + "To use RunInference with the built-in Vertex AI model handler, install the Apache Beam SDK version 2.50.0 or later.\n", + "\n", + "**Note:** The Apache Beam 2.50.0 SDK is currently in the release process and is not available. This notebook uses a release candidate build." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "jBakpNZnAhqk" + }, + "outputs": [], + "source": [ + "!pip install protobuf --quiet\n", + "!pip install apache_beam[gcp,interactive]==2.50.0 --quiet\n", + "# Enforce shapely < 2.0.0 to avoid an issue with google.aiplatform\n", + "!pip install shapely==1.7.1 --quiet\n", + "\n", + "# To use the newly installed versions, restart the runtime.\n", + "exit()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "X80jy3FqHjK4" + }, + "source": [ + "### Authenticate with Google Cloud\n", + "This notebook relies on having a Vertex AI endpoint deployed to Google Cloud. To use your Google Cloud account, authenticate this notebook." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "Kz9sccyGBqz3" + }, + "outputs": [], + "source": [ + "from google.colab import auth\n", + "auth.authenticate_user()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "40qtP6zJuMXm" + }, + "source": [ + "### Import dependencies and set up your bucket\n", + "Use the following code to import dependencies and to set up your Google Cloud Storage bucket.\n", + "\n", + "Replace `PROJECT_ID`, `LOCATION_NAME`, and `ENDPOINT_ID` with the ID of your project, the GCP region where your model is deployed, and the ID of your Vertex AI endpoint." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "O_a0-4Gb19cy" + }, + "source": [ + "### Query your Endpoint\n", + "\n", + "Verify that your model is deployed to your Vertex AI endpoint. If you encounter errors, make sure that your endpoint is live and accessible from your current account." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 35 + }, + "id": "5XkIYXhJBFmS", + "outputId": "46b4c3ca-3428-465e-8c24-bfef525813fd" + }, + "outputs": [ + { + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" + }, + "text/plain": [ + "'hello_custom'" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "aiplatform.init(project=project, location=location)\n", + "endpoint = aiplatform.Endpoint(endpoint_name=endpoint_id)\n", + "# To get more metadata, remove [0].display_name\n", + "endpoint.list_models()[0].display_name" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Y3BC2aY8cIMI" + }, + "source": [ + "### Preprocess an example image\n", + "\n", + "Preprocess an input to match what your model expects. Use the following code to complete these steps:\n", + "\n", + "1. Test the model by using an image of sunflowers.\n", + "2. Trim the image to 128x128 pixels to match the model's expectations.\n", + "3. Output the image as a list." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 381 + }, + "id": "wEXucyi2liij", + "outputId": "636baf08-55dc-4e68-c58b-236882dc3786" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAFbCAYAAABI7o1QAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOz9eax9WXbfh33W3ufce997v6Gmrq6uqu7qeZ7YZDdHkU22JJKSJZGSbEuyNdiWYTtIkH8SB0gQxAGMGMgfQTwEAawgQgJbiCHZsQVFoiVLJGWZImmJVHezyWY3u5s9V3dNv+EN955z9l75Y62197mvqskfAQH+g79TePV77w7n7L322mt917hFVZWH18Pr4fXweng9vB5ev2+v9D/1AB5eD6+H18Pr4fXwenj9T3s9BAMPr4fXw+vh9fB6eP0+vx6CgYfXw+vh9fB6eD28fp9fD8HAw+vh9fB6eD28Hl6/z6+HYODh9fB6eD28Hl4Pr9/n10Mw8PB6eD28Hl4Pr4fX7/PrIRh4eD28Hl4Pr4fXw+v3+fUQDDy8Hl4Pr4fXw+vh9fv8eggGHl4Pr4fXw+vh9fD6fX4ND/rB5z78bpRCSoKIUBF/R6laEQRFEex9BBSotaJVQUFVUVUqlaoLtS5AQe029r8kiACi/qIiIiRRQFAEVLHHa3uW5OyjiYaKQhJhfVVAS/wodVbwsVV/vFTsGVWp9dXNGUWk/ZDEhiE2DkkDkhNpUHKGnDMpJbIkUhZ7L4Fkm2P85BT3TPaCPwc6zdr1qr+FqoJWRUtFi81n0cqiyqIVSoWilKQkhaSCJmEAVIQhGxtIEkg4vbM9R4KaoMnmnJ1eCuQKRSqCkqQN39a5VkCo1V6staKqzkOrOUplGJLRS0BIqPOVPVuoS6VWv3eFUgpaKyq0hxqtjCaKUmqllkpZKmVR6oJ9x19HjWMRISdbhyo4D4rxF8k5Sp0ldcX7giQazQQBu63RrlZKCf4PXlN/f8VPwe4Y/RFBFBKvwQMSFHH6Danzo6itkwBZSCmRkpJzas9RVarfr5QCNVGrokuhLj5e7fOJ56SUqLU2vl8xoO381RiNTJ2PRYz3ZRDSILY3BrFxDbYfck6QnZap+GZcr23x0RgZ49mIgmaOyWNyCBWkGA8q0vijxnrEOmjcrY8/6N7mG/tA+n3igRpr5JuiFpMdIoKo+oKs9lK2lxKVnFIXBLGm/jy9Ln+afFJqyMcmSCCLIlSTuwWSy5Pqc5OqpGr7R4HqPBl71dY98VpNacUEFTlLk9HJeSrlzqeokhAkuayutP1YRUErKXhEVvT0CebkS1NDDkDKQIqFKr6+CWULqbLRmZsnA3/5h0/5tz5RKfmc/+KXhf/4bxcuLkdqhlKNDnUuvPWREz7w9pt89puv8Cu/eYnoBmplWQqahLQUSrH9pFpABwYVNoMy5wGGxfaWZCQLmiBlNdkuYrwqGRE1vXLEOwKlssxQZt9vpVJLMd5cBJ2gaDH+619v8tL2VLxGlxkJk61JGF0Px7781q/fedWaXr8eGAzUWlEqKqkJyrreSbbcMWyoxlQxyXYPre3zx5dvGNWuUVzZrxWMfU9W70nfjE68NoS4l8R3pQviEL6266B2oRFAYD2UJhjWl+IfkK4wRBtgEvHFybZhxQGNqMawDES58DU8dEwXVaUu2sBRcYW6Fu4aAr3UNu6qDmaqrRkKuiiK2ibslGIpC4KQMaGakphg9uGoA4CYs6jP2IVi0YqowbpGiZVyKEXb+huDprZxAtTFapjgiC0goAUpXfjaUlUDGi4sdTVWKk0Z5GSSpfpGEVHUeSbl5J83Id3GEsI1OZjUTqumcNZLFPwR7zcF4Mq/GhAL5dP+XYFpDaXbgLa+anccXT7O2AlGtq48U+o/koyaGec1/3xFSSJODx+3HAuQ9VUDia2HIdIUnUjt+0PpnLACV7EnDDgnNOECzICAii+gdn5H3XlZ+9g75WJNnBLxe2OW9V4P+tNB2fo7QR8BceWUnBeErnBj3hpj9EkqNtRaK5LU8ZKB8zZnCVBgdBjzcCRTFKNVFdun4vwXa5HVvld8rDg/r7ZyF6PJx+XPTNX3ra9X/EcS488+AjMK2qxiXGbgpMHkg31XkSQkqd0+87uvNgSOyww4O/8naGNvdHGDQtXuTVHnCTFgnMDQYvDzTNooqpXHbibe9thE0gkZBp55auGJR4WXLzJ5AKmVcVFmMlcMcHZGvrWQt0q6qix6xZgrWjcUEfKmMtaB+3XDYUmcZFBdyKmSJJEkYexvPGvskAzoyuBGBgbGYm+q8U2VRKrVQJkKRU2jS4GUcKMtHYGBNZ+IgAwGNswQBcnSgEAzTGKlH/DEgQcGA3OdQWAIZnILI+as0oFAsFAonhBtlbDGbOGDBdbcolrbDhZXFKrx6SDGagP5Zq7O0BISUqAKfj8aEyrJEPxKuITc7qglrDdxBBoC6hoY6ANqTG3W2Eog59TGa7fQlYDy5wO1lGsLntvvBqiCljRAFZ6PEqhSa5N3zTpFqEtpVnkIHMEFCTBIsnurroRWrKbvQb+x+LpXlIKSK40X1kAgngemwNc31OQ6WKvTDNYeD3t8cv8ALFoI3d8NbHtacjAVz0W1Ccq4UgItxg/AERhqgk+0KVLaprqGzVZKJkRfU7AaylkbqFMXvtJ0jh7xsVmHnd7xbwoB6vdv3wiwwkq5iLg1Le0ni5AyDOLepjZoB0KqlBWtZfWLsCbdMSi5Ps5jWqYV1AjKYMrGwaXtCdwDoN2qSStNsiZ31T6YV4HxRPNPaVDKB4S9ZUopiL/eE+v1aFDiGNixuuf6Sva5hJiV7l9UlXZ/aV5N0KR9TI2O9u6CYuQwQyEAYZJk66OrFViPm5Cyqc2DkCt+/5yc91ShhlxM1BR7SNHaPblrXooHCQ6i3WNoho3tJ0lQfGllJRob1qKaF0I5lpv+YbUHGD/0N8FBKggpC110xPonB2iJTVIkw5ROuH2m3Ly5MG+ESuXWI8ITTyqf/6Yw1kxNUHOhLMpUzYp/dISdHLgaMkO9aYZUnkl1hGXmsk7c2ow896zyjfsH7l5t2abR5txooI1GaXCZL2IGWHhAQ+OpQhUyCtm9BI2pTD+KCnlQfy91sOlKIweoHuh7Npn3RLLtf6NvX5ByJMS+8/V78AwUJJsbVdwlVuOR4hvABWJ3+mhzUWldzD1bw/VrDGMKsjoQsDe6B0CbMOuKQo42dNu0Ab5CaLhgqyGqampeAQ3F6Jt3JftM0bpvR6tS3QX+WsKqWzu28GmwRUkpkXOyBXJgspLj/qyG+VANSvY5Gv4P2hvdwq0G3ZsQt1S9brl1IdJohysv37Hq1nlxwqTkks7pqKqGgl3lFcWFjrn8mo9HBS3FwGEIsRVdQykEEGnDW80HTS78U1dggZhRlEKt4eKO6YkDvu6mDt0hsYG0CytxjkB8bGLzDeGtUkEykpJbcyuA6/c4xoSBOGnAF2hGrTiAixCUoRKaEjJypO6VWAnzFDEXI9rRyq69GCHDZUgt/JQkgMCx4o55aBK3EIPGwR/S7vkgl2L3anOmT6NxpgsoyS4sMyuFYoK/4ZUItzUZQBeYq5ARKuhqDRWbj++M7nzRFTzRNZ/YnV7LA2Pr2OWLSFd+DRk7YmpAzT0kFlb0cGXsH99TaQ2OxBRGcaGl4nSwmVDVLec2VhtMk7dBcEkkKt0EiBVwntDa9gAVVKSFiBr/EayZGquZohayJvdq0mWZAzoxBG8kcQ9cPN+kV/cAxxDWYNHY99UyHbp3qrLmX5uEJDMRFFh8O1MXbo2Fxx8ZGdKMojw1Km+6NfPLO2VZTpEKY1V0mUmqDGOF05G6HdhUEAqlQl02iMzutRj5cz+w5c/9aOLvfGbk//F3z7m4Uk7z1kBegsHDWyIWLpGQc7kj6wgvNkM1JTOkVBHn64qQcA9Z7FPnU7QbDSlZyBmp3QgY4tnO17Xa2mE8JtdVw3e4HhwMUEkItdrmWCO2NkvpSqnW1eajWIze8wdSm2MwZXKPW7FNK9UVhd/e1zyEhASMb7veGKaud47EZjZBocU2f7dYpQmItsu1trhfc0EfWauG6PqlPc6eaa5oCddNY+SVpL92OXA/fj8seo1whXosV8meGyGYHgrd2kFCaKLVowFEyeDuZKcXlkMQdLM1Di+JaTFJUBBSVcsNEPG16s/QYp4Hi/FLe26ADxMI2pcspCyhjMVJ7tkCquYZMeRGbZaXuV2b4ndlWUPo1SNNTbjImlB1FO+oCnGXo0i/n7hrPaBmEjket0+uOwk6+OnIrIOh9pUkNPKs1s0kcbd6Td4mJCcD3iUAt3TSHnkHaAIi52QWnHTBGzFL0zsGclKMr5oFItqVxDFv9nFd/z2FSdfGEl4aXS2BtK2VEiQXoE25Rq6Frj+/VoCp0URJfbs3mscM04rW/v0qvrdioWKfd37UWD/o4ZLVuI+AWsunUbrxE/M3PmxhEASpchxakWNaiShpAJHUXON2Swu5hVx6TckRlo9pwiZrYr+pWjg2acbD9Ed7wRRTBxYNwLlskABqJEKTiHTLN/ZG910e876o85ioWcdB+uvTcN4N+vfXYy1Wa6cGtBqsSgUZIeURysKjpyMnWakVMiNn28qjtwq7NJJSYpEDurEcmmHYcbK9zeOPCbdO4GKq1ASbKTEMsNSBe1Plxz8o/IUfVt78yMKPvu+Kf/Z1+KVPb4xGuaIizSJH7F/xvIrjvWT0kiQNoGYxoJvcK5vVAFQYnlqFa9vR1qAB6Ywmk2cpwAB9nya156aqBlwe4HpgMNBX1BO5gvGTTSJrKCMaw4UnYO2qb2E26Ohe3Yfg7qPQZ4pZABIE9X0ZcbA1sUNQhOAP3hIXHGGdmaCIZ0pTei1PoNRmNUkbG5CSvV6JgFVzMTcSCSuB4neoxXCGxPO1bQL1F2vq4zRL3lB3gJelqsX7VaGGa90SV8IFpGrKpoVafc3UEXV1hlsvZw2F3VzM3QsTJK61GpNWfGNrAwMVo2XRAAI0Y9bWyYRxLj6vwFzNPWbKqLorV9z2kWpjtmS/whJ8tOjKq9MXOQDo9cTKzg8BBkAiQ4kuQE0HeZJdTuYZMO5r8zHSN+RGAwSxptpldORtxDNibD0PTWKITT3h9M8iLUywTgpcW/gppWY9ZY+599BUQlK3/FTDqqCNY1Cb36JLB+0lPGVHLH0EAnqOx0r5SAZijIA29eseMzXLKa8AQCQ8OUgxU9iFWYiKxkyCrk2P2I/aBW4DPrqSFepu2I4gutbzj7yWpaptXkbLWFSLx8I6BEDGvQvFn2f3yQiaFC3SPtq8iARYqr4fu0lUJXXgt8ozsVC58a0pC2m8Y3SM/braV9U8XZEjo2r7VBWkVqSqubJXQMLCOdo9TCvrXrKSUnYPgXtxtY+9kT3QgK4Muja0HgPPLixjH4vLU8NWvllTQmvI48G8G4Fua0YXW2fJmVGUbU7oZkOZCkMqPH4ibIfKxVIZxGUHMI7w3MnMn3puzxv2hf/sFzJ3zhMn25FaKsvFnvc/q/wbH9/xptdVXtzDzZOBH/2g8NXnL/nGy6fklEi+N5PS5KfxuLjVb0MtwaURNkmQqvFnIYAyvjnULenwZDuPe/hBHKQyiOdsYGvmeypCM+EoLvLqPf2drgcGA+u4t6FRd1dgySmmxGoXKLWHBjQoUENQSBOix+G066NWt6Z6MpdK3yRd7VgGb3gD1ncSTxAzhev38JwBez8EfTGhVEy5SCAPf4ZtrhA2xyi2efzXz5VEJMPFbda2YsvMlq6wupvg2FfgCblNOWd3pYs2XMP15ElRYxZVJY+m5NOi7v4SMlCSx6e1C/qq2sOGYT1mZ3z1OKGqM3alVG2xz+6+9ty5EGaBc1aUUxfYtVaSGP+bUMsemsGybsvCPJdmCAkKpYng1VJoG69IQqQ6mDp2Da/BWriAm4pungFapnSAmFqlz8+BgBwppBW7dCRgwl+bilqtqgn5IWcLa7vgTCG8BdKQ3Wo+BhdDyu4lg+yx98ErVyR1nlQ1z4+IQCRe6mpdizpi08a8AX7iWWBjaqMOPg4hHnRbAatOYJBB3Ko2JRGGtPh+lPDSeMa5MYntWWnqOmHVBOsx+DhqJa3za0IIrwfhu7AZIjHd+MOVkqiYx8bBVgtf2J2dr3zc4mA61tvDBhrGtOeKhHOgWbtBJ8IzpSvQamC4VmFZakv6DCNJfM7BC0PKZlBJuDNbvMV4YLXbYgOKmttYKpTMav2wNfIwyDo5zTxPsT8xD5Tvgaq1h8Fib2rIuOL8k7ucbBaZIDUZ/E/q4Cas2EopcHVheSHjkNiOJoM0FSIBOFNZ6sQwDNzYjNzIghwWlMKNnHjr48Kzj8987vkNDCfIoiQVnnl05v2PfIk3jge+920zv/3SzC98dsd0daDkyslN+Fe+d8uH3lK5Vy6ZS2LUxHe9fuBX337gxU8v1GUkuV4ahkQeBsgwa2GZZ3YpE4mRlhQMOiSvZAm8E2Fmt+bDTnE5I1jVXgDTwcOXSUwOkxykDRFKdZmq3YhpYeEHuB4cDPh/2rnKfkuAx5VLKehSHBi41Vfj44sxujNns+IUVMRij7U060rw7HTfQCWE2npilWZKNHctNGbTtZZe7DPiSiHgk7piE7eQQjkffb9ZA77hvfKAlNpnJcx/F7alzs2Ck6pULVS1csMhZcymqJ5pGmES3/gtgdFi5EmLW+DKDCxUNhUSiSrqSUK2Hib3PZltSYzggkotfosJvkWVvHhsGpt/nQuIl9cRC6FkD+0k3AsQJXjFhH1WWBkRUHsGOyjVn1F9bUotXnbpJaoVsmRCwMyo3TwEDVCKCdZarTTVynfEv+Pxy+ruT89BMaDkeSqSurIh8gSqW84gKZFGt/6Ca1zZJ0lNzqqol3wpiM1Bi3b7TmgAv+E7L+8UcYXtyh/f1Oss+5rNhWjVFqUnOGrsQE8Oc+FJTiaUUm7hAlkBzFqNTpoMOdqYElUTpcgKrJuwb2DECR/bNMXa0veheYwXLJspJp9NNjjDpGQCPuLwDTBVtYqOFABkBao0uTJVhMWsb43wFfQIfHbluRqcW2DHsWYHiXSwE96k+FQVz9JXA7uCQk1IykhWRAowGwCIXBN1Hg9h73tGXagLSk5RXhbgKYChYDXOuKfT5YeXBpuMiDXwiTdFHbxY2iLZOkWekSIOXi3Pxva3uLdBBSuhWyO3JCYfRA34rypAmocmm5u6htvZLXXbSt0TEzIx6cgyZmSeSRXKaWKphc28hTSxZ2bHiMpITbPJmGJg99bNmR/86Cnl7JJf/2rlc59LpAlOT4VJB5Y6kqSwk4LWhSGLlXaLkqpQJuG5R5QPv2nDl1+qbKaZywWee2Tkz3xAed8Te+7s4cmzke9/Z+G3Xla+/u1MuUz88NsTP/yhyiyFu3tlrMKgG548m/nEu5TPfnXh+Rd3nKDMY2XYTFwehI0mPvweYZPgV35bKSWzKQszO8h7MpV8JSw7oWZlo4m5TGhKjIMwcYABhmkLHICtyc40kWom54ym2aoNBLLveej7p2TnQbEqNCtrXiP073w9MBgIVGdITvyBXtUqkGoIFGkxaBPYoUjpyV9xz3BBqrb6eNu0oYhc2Befrrs/jyyw6ni53TfCDJ0A4SYLorWkmhDYK/dvfC5q5HtCU1jz3RKy8eLCzzd7bFR3n4uApmS9ANxSK6E1wpTS1DBHZOKXZtGGBMez+N1jWJSixdz/a8xCt+TMm6DHIRLWsUOfZ/RL0C4cHdS6YSXHr6FuyQUMkq4s29p3sCdRc92mLVYmVf2eaWVhxzNciZZSqMWSFw0QaLOOolRT1R1x7t0J93rLPVuDwsYWHWWHRyARvSl6qCoAiqpabwK5Nse6mmtnwe5GVXX65PXDV14GmvUVHok8iPen6NnUDSSLeIJ+6uNOyQRDdss27ISWALNWjJGQeuTg9Xt5+AHMq+TuyvjmOlxg4zW3ZDMVmvVr4IqkprBiri13oG3tRkeRPr7YREbe/nsjb1M4/e/r9G+87HtuNX3AQE9T4P5ZU+a22QNQoWagWPggtX0Slvr6Ct5tvDwkz5HynaIYcCe8Zr42jZ/or0FLLGmJgE5jUEuk09pRJWt6SPP+BBCoAYaCnkJLOAsLNTwCUe7ZkgrhKLSjbuE1slfXDC5PFMupyKkyLTMFqIsw34UNMAx7hlwZ6o5JALliqJXNsGWvwo1B+Dd+9IS/8EcqeUlcHEb+/ucr/+nPHPj0r1fOdspuUPIysufA7TRyaxw5GZVKcQBfeWKz8PG3JH7zhcw//lLi9jbzh95b+L43g8gAUhi18M4nhA8/q7zwgnB2qvzEhzK3bxXOzxd0EdKJgeo0K+98/SnvfHbmW3cnDnnDbhDuLDuefUT4d38cfugdhRunG/6jf3zFX//5ypI2vo+2qMyMu5GlTKgKVzox5IFMpcjCsGwos5C2C1pHtCYLA1ppCFUXxjGzLMtq/ydEtVWfGNtIr45TaV7w3+16YDCQnOEGgkGNyZMLwxJlg+5eqlLd5eNMU7G60WZ1x0bAMypDC3cLl6QNWBD3iE/GBq5+n2vKPK7mMqW/3xVVmAnSNng379ZgpSsIDXnBOkYZm6++iu4piwteU5hLNQRhFqi727ykJJRILdqS4iqgS7WSPsyDotVzKarVsKbs4lj6Zg90GOAn3OUxr6NEk0YzSz5KKTXFEylMRubUaL8OSzRA95qcI63vQvusl5mtx9oHIc1S1ILPFU9A7f0f1nkHobmF4JHqysc1kJufEqCnJXjGeFJz+UW4o2IWUICeiKk3AKly1Exrfa2BltaI2Xee8lnShHsrZ+yKchCj0SCrveKfzxLhgC60U/LXXVmYV8QUea3hzVixvK7p3cMAPR8ncki6r/xoDZMgYsLKrIHkJXLGd2THEe4G7dZx3Cu157z66g2nIka85q6jLPsVvYMd2njB80867Y8BUOjeAATmxbOIphAN0ZRk+yS1O3Xwu35Ykwb+hnX46kKZlcIPObi+xxrg+OTDM9rAdICJWo/+tl9DjkEL/yh97Xk1tcMp8WpgavIqrde8PS48orXxf3jAbCnMo3cuAjWxK5Uf+9CW971T+AefvOBXvrChXgi3bsyW5HgYSUPikCYkbXnmhvIH36mMzOgCN7eVn/pA5SNv3PBX/v4V//XPCxf3hdPT+yQ2SBJOs6K1kBbhsGy5miuyKB94PfzUeypf/0bmiV3hY2+qnA7CXCcKlTRnntooH30L/MqXZt51e8N3vXFhWvZMJZPJjEkhF+qivOFM+dCbK7/+21uu6oapLDw1LPx7fyLzvd9T2C4TQ4GPvwf+3v+ofOteYjMKlIFCZRmKgfdFKcPI4WpmBBZdSClzuh3Z6wEZNjDBKBVypfo+r8Bms1nxuIND3DB3Pqb0UGDthWm/4/V78AwYYmxANElPXlsJZKUQ2axHaFUwy94ZNN6KpI61QJXYWNUTdKoeubRCXPXQgPS4bBPOLnTXdRWyHhFNCMQ9j3b2tV0j/vGmDKqhT8mW9W4K15WYgwdTNhaYq4K50sT8aoO7Uy3+Uel1ypHoo014GbJ05axg5pi7xkWpZKs/pYuj7p2J+H4HBatVBRFK6cox7tlUhYctukUczLdW3wY6pH+phx+kK7GeJXy00EedIttva2+EeDqfLs1zEr0gYh4BXFSjKVOfH2AVAhGL9p+Eoe5o2JHc6jZ96LwqUQUgRx4mnK5dGPobK09Bs85Yg6GeABgD6WGCeF+au3+lXhBJLcyQkzQg0ABCG+OqR6LnT7S65Uhnis96HXyENWIVbCxpFX8/CqCvltCAgLSyVMUiPm69Oi8loRfB+e+JtKo07Xt2vX/xTpjxmo2xA5Y1H3TeoX3WrPprYE1WH4t1ijUVkyVRUdLp4nk3MU5VSur3MPbqnoPOFx6O8I5yqg0CvyZ4XlV6mhfWAeoKMTTwEw151jcLORKvrRO2A4CYeF3tuY6i+618DyCrTq7hCYs/a3zNQ5uelxLPH6fEG59M/Pi7hD/xkZkPvXfgp993xc9+KfFXfzbxmW8ceGw44WYWDrKw6BUnS+Wtjw889eSWsih5W6lFmWfh2Vs7/rd/HD7x4cr/62cXfuFXN5Qq7JjQISMnGwojwgW7tKHKCZsMH33DzA+/rTJo5ulHBjKZQSGVAyLKkJXHbxTe+YzyB54+4WRz4OoA240wCJy6PXGZBcrMu19XefL2XT7/7UdIw8i/+y9s+IEPzoz7+8zTSPbwxe4UuD8wqILMJIWDKktdEF2QOjIMG+q8MLAFKRQuGeUGhQvSsAFRN84MfGstDENeGcluaBoa6/qpEJHWV3ey/A7Xg1cTOIOHwjnO3I7fo7GNdhdufKnQEXxwW2v5e22zuvKBLmjDYulJRRwJhqONfTSerpAC2h9bsroSogEg4r4dEYfwsRi+fd+6jSWOSpCQ9sxmgR9lyfolsaEjWdH+bj0FmvI2t5sZWe7CU/u8rpUu63ikfzcQu5eQ1abMO4MUUQ9nrGKElNa1L2HdsKz+NSoGIjnK/45qiBWG6vcSFKOPhS/0aLnDC3Hda2EtXSNPoHqiklUXGN/UJuwiHhtRYGsF64l7yTxUKaoFQuE67dKQyONgwEBqS3YU54sjxeQArAHazoRt7F0ov5Yr25OJJLKp8VCx0BwESRiiKmDllTJaxU/y0EAHAyky0VEsh2cVZooxNP5zPk+p1bOvm0SBg1eJMEC/wiPTQiqtuYp04eB9BTStU9hqa9Sy5qE2Ponqj3hSn7c2gbdSsGsFteKf429Gnoq0NyIZfb0uDjOO52lt/jowVHUnkzavjC237zHxdsDezlddOLcqIQfTlltoc2lFwtLH1xAM3qBJYn6xEL0Ph6XuBvDzmH3T/l0utuRktT16RKkmG0P2deJET5mgcYDh2nhLkdJvHfs1DLKlzvzgO874Fz5yxqOnL1Fq5dlbN/kzH4EPvAn+7z+742d/sXKfK3bbHUO9QdoUPvD2xPa2tcpmEiQvBojmAydZ+fg7R97/JuWvv0v4K39z5u7e57JXhs2AyAklHcgjTFeJjVY+8MYDdT+SC8zF8hO0KJVEziO3dwufeFflbTcnKnBzHJFBGXPiNIl5GtRynJ6+lXjTE1u+8PzAX/5+5Uc+dh/mTJp3lLJQ88jVXkmDMuSKyIalWshTPMFUa0EOQtpsmbKYF2AcURYGndmdJs4vLaE6pYx4UksaR9MHHlptfSNc1kdoSIuaV9XzOh7k+r0lEOqqPCrQqSuYumYy7egQ3zDVM/UDOXfXtWPVhvyP48GEFRpCtkdR2z+mD1ZCoX1WsO3fY222afwD4Tq/hkdeW4jTvnNUOBumQ1XA+q+Lt+UK4R7x1OoSRCSEoYdEQshVj9NGvgBKr1Xu7mJ7PQRV2Cx941cX7rXYuPTIija6aI0cjoq0msMAMIJINQbMivXnNgFvOqInHyn0+dDL0Ih1Xa1WE1r05zUoI68GAqVE7oQpobWV42rZ59XXMBRKZOeLf/jI8s6mSEOZNnesCzobUgc89tN57mjzrQSttOeveDGUtB7zbbTKNoXaQWi3/lc8gzhYSa8CAjnnRvNQ0wZYah9D35idYbK0RE/BEhCVEPStG7U1UVntlxSWs4glgSq9RjzAqSt9TSswkiwzXIRVe+5E1YUA0E2BaehH68QX4bjYwcX3ijhdGr1XrBAUqav3mxw4okfw0XrPt+mYDElKr/83MB5u9LVirazadsd+1Z5FX8OzFK+FnIoJr8Io6obVdTd9l7ltki2xkiYfOJLPEcIUFU/K7vzSdqhKr0Vo76frBG3AvRTrHUN4xwjPpj2/VjibKm+7ccrpjYU5FzZpYFHh6mrmw4/e4P/wkxc89+TAX/+5xN37cHaSeMNp4iNvUaYxMV/NbAZlUwdGLVi5nVCnKx7Nwp//frixrfy/f2bDxcUZFEHzOWixxE8VloOimnnPrR2Hmwtlmbi7t312tWBnLZTM604rr3vjgi4HTmXgbFMpaSBnoZSFqQi1GMedbYTbNyd+7Lsv+Bd+8IQz3fLit2a2Y2KaN7wyzFTgbBBU9hQd/GwEZawjmcxBLAQwTxNpwLy99Yxpf4cPf/BRyuaET/7aV9iMN3HfrxnCtaLRAMrXfB32Vg9NFg+xtk3xANeDewZcwUUP9DWiDoGoeOa2douqtZFdnDlLMChHOQDSmMlfc+/A2to1Ru1DWgveUD5HUQGFaB12nRyxEVrSHnhTCG0EXOEHImnKxuAC27sNrgfV3ICqrYzPgJQr7ZUiRd1NqhDu7XDpXx9wxAPrEt3KzDVZCRes0z9AFq5/Sv+730t7LLYCFHPzCqukO3+AgmglREVUH7Ui0SDSyooVF9MtjBMZ+LKCBwHiqof2V+vaxqhrS8O/JObqVrV4WAz2eH7BN64uvCSs1eKLnVuQs2e+u9I2Jdq8be254QLFvQ6tPCieF/RaC+o2TWnenwAmuhL6IpYX0M+r6R6OAJH+wQYQcngNVrSKz8ROqSGway9vax8RsQZArFnXFqQlda6f7cqXZPe1nAXzCAxeja6+V8Proq3FsCUSSttInW9aG4AmsDojhMdw3TEvFFGAsMZfK2WP7zWcEuvViO/H7y3JtIuQo99jzexQK2+P6wnL4vWgSnj9jJhL8II/pWGcvlqNd1YiEBzwIN3cUbWi3dQqbWwPHQn/eL312g5e1BZCqY2h+7i6vFUo6qDPRxy8jDYvJHDE23aomjGMag8Rxpoiwjzu2F8unM0zu2FLrQN5gJMMF/Wcx7fwb333wA8+d8V/8ctX/NynK6+/ueOdj2/YLDObMsFB0ZpgGNFkFWmZM7QUbnDgj71bqHf3vPT8wkU9YyOVzMCwJEiFUifmKXFrSEwqSMmoJhaBQiIzk9KeWwJLGVkq3DitDKMyqHCoyvlcOSzioFeQonzPmxPvOcu8Yas8//wFy6GwXwa0KrvNwC5P3Nxt0UUY0oG3PH2Tb90X7t6dqAWqty3MWqEs6GZkutzzxFnmx3/o/fztf/plSvUDwrRQs5JToiyVIW1c5pVuVGrXxSazVmt9TbZ+p+uBwUBWW2k788YT3lYNhQrVJEn15DcfTAhPr+RyxoWjXUgwrB5r+4h1NBPPJtfQ6urP1ADIWiAESJG++/yKzX506fU/OoJOKwGckp80uAoRqKwsAC81i977Fud3dypunnrsrcn6KoiX5+h6Lt5mMISmha9NCFQRi6vldSKPOjAQC82UuhI4ATQCYFkLTAMQILJKvHPFrV5BoaqvppdgJaHXpGjcPbmVMHszB4lDZ/zLIVwGLAEOV/AtgKP9NDUDMJ4bUkNB+5haUHVVNbJmJQVacaQrEKe/WdpBU8tGDnpY9UDktKwVUbxAf9iK/tDBo/tLmp6TpvXXyX9moURlw6uQ0atZ8og+8bz4t3suYjxHmsCAwPXHtDyXuI8nLGntuS7xTL+fiPffklAknUNaCe9qP6+ndqzAY3h9DZtwuxbvb/fyDX0MvmIdYrYxl04ff/haLzYgF1eNPIdVfwjrX0FLLow9EVO023cated7uKrdSfu/GiC6v7ueoJWLusetlU9q91Kq1zBq1eOWs2pltjHu4PcAeS5EsDBqzIWGTjTh7mZt/SDUrTfDQNIaZNVlsfLoBpA7D13mK144KDpu2KRK0gNTVtJSuVl2XBTY5ImPveGMd/z4FR9+Y+Hb35gpeYvOCvPWTNw0UNJMloW0DCgHSMoiW0628H1vm/n1VLl7UXnq0UxNCwwLospWYVKYtLApA1kzJc3UXKmTMEnmkSGxyYnDDDs3eieUrJWLuXC1DNRlMk8YCV0OfOCxxGM3dty/u2fUzKID0yJsB2GXF7aa2KVCWna84fGRP/T9b+G//R+/xLdfukMaT2GBJSVSKQwi7Kswlzv81B/5MZ64fZPPfemrDNkqEXIWZi1UqeRt9k5Cr+H7v/ZaC/3+884ZMIsho7KwUEl1ZKgjhSumoZIO2pG4M50WRRZpjX3sqpRqbufW07q4NaFRmhIasv/T1VebOYLEoXMroXAtpuhXxIKPXPyshOiaXkduW3MJJs84j2YRuAu3umC3+LopdPuuxfIqhSJDa07SWsXGcz3bOU4qM1eqNlqAoHOvqpBqB3GoiB2JTDJXeqB3MOTpR/z2E8kczfsahJIo+Lq64kipV3F3p4tbDwoMpsxaP/SyWOc0tVIigVY3rWKApJZwoXucn7DQ7V6tEyXRMtrBh1brsRDu4lTRpXgli5jQwtzn63wPaXczxrUsfTs6NzZIGnLrr44YGC3VMm8NWVe0FANm4QtRc7f2bsUBNCMRbwUw2/vlKGFQ3YK0nuIKuaDZXMytW5zgcM6EcG58Zxm8teBNYHqi6FF6n67c6l7dg9ePq9AUh7W2tjh3KASbfwcWuVoTqCoGDMQbyNgRrsKBSkqDxTCd6JJBcvCkdEYS4wNJvXSvlUDFivnv1WVIkjgoLFxEeD5Em+Ga3B2+t1v29Vj96r8I1GSle3QlG2XS6kAwQqPWfVOaV6L6G9J+Ggz25wlosYNpDDk3oC/hmdE+olbe5zOr4sre5YYBiNKAbvI+LuobQRrKhSYYY8+5ArcRVmSpBuRjjURRTdSakFlaL4shLWgdQBOSZmtNLgM5DaSaqMMVF7KjTgc2qnZkdNmwSwVhYJbEjSGzOdtSyxWDbpg5pww7hnJgWTITB26PhX/5fSP3nsPa69Y9KiOlgJSFQUzIqjOqqClrSZWnHy9spmTSbMnW3a8YgC2aocKweD9FseZRVo1cyCrUmpmqWh+ArKSSOahyvhSW2fq8pDwwKpinMTNo4nA453Q7cEgbzqcJWSo3TzbsBjgBdpsR1St+4KNv5/SRE7515xVujVuuSqLKBNtEKgN5C/sX7vOHvvdt/KGPv4P/6P/zc1zcLex2W4YsqCY2gpWnzwbWEuFhTm7bFdZJ5ZshgLXwgFjgwcHAMoplQFLt5MIyo7onoQwHYam1KVGNmLfHlEK4hHJLkmkujSqtN7c2IRabJzawC92wsvC3XXB0IdCvtYUWl0DrC2AfonXJQ3vCG9DK4aLMxnqIRz4AvQbMx2mWpllNSbHOMB5njA0YtLA9Lq12N2L86wOU+oih1nI0nyZ03CpsbkX/n1ZrOOFtAFctF/rJiOoWRqdJzNnXQrpiM+eEEKnfobqjYiWFZUx4R3p9chPW67Vqc/HnhDRVVhb4ysPRQifaBGYIhb64PZ9BXcFbKEdaVz5wN/bKyxM06+7ovhZWkheemuTr494J00qN+cIzdWyNx9C6hM9OY8EUS8LvJ65sJPod9JbDlWtHYmD8GsrQZL8nTfp3SvGDndbJqBpAOYASTlNt9Ek5QiIeEvPySsJrtNJWpijNTelLYFUQUU0Qz3DrltVn1Qlfoj8Jq32/4pdORO+y1srXjDeiQLF5bcBzIdYEC0Zf/e007Pst5Mp6fE7dGuTo87AQxfXw48pY0T6H4gIaDdBdXZL19Qt2Ft93em29e15AyASXC9XTENXB10p+aIsPGXVrhOQcZNZqZ5FITrSSXHOXmRdzhINWM2zSiOqWkwRlmZAtiBZO9ZS9zpylHUtJyMk5zz265UvPnyPllLuXSs4Du+HclFlakHKKVnP5i6obLCPjkHnysZFdFnQeYcGEjwosiibraS2u/ISFVBPbnLl5mpEFdC5k6w9sVTxoA0eqBuBqeG6Cng4crAOlGSTzVEgijNn5TkAK1BnKnNkOwq2zDWjl3v0JLTCMguRKSplN3lD0Pm955mk++pF38gu//pvcv5PQOrBNC9uyoSyZsrngG3eU97z5dfzbf+L7+PJXPs2nPvUZxvwY2Y2cWmvrO1JQal1ank4Lz6Vk3QidbwYFSJZ39R28a9evBwYD22qH1tZSUUksIpS6QaWylCvihLa4tHjSVQ2ia9tha8XQooFWAwQEM3dGFiIT3a8UQEGOyiauOQPiwzieaDaEXHufa1taPO25HUGMuPUUJgDOlK5gMIvRegra+JIKcUxyc3V6BYCqx6+x0quqHhuvscl7rC9q1Y9Kknx/SLWzr/sbPs/qB/pURWqz4W2s/jxQL1+TZoWG0FTpt7Qktl5xAdLOAzLPTAggu4uuzjdoaxmCUQOtdoXfwIB/r7GMW+QqpR9EpCu2WF1dxq8sb+hArmXgy6tcxqFIqlp4y36UWgXVvCqX6i5pJbjS1jSsQ9Rccq0HeSSaNfp1kBnYyjZ5591EB2VZvCueBDjotDxyszdcFAC7tvBcA5qrsUdbbbOGO00sQzkdhaVKxY+9dQ+OQiS3qSiDDPb9FImxUYLsgj6JeRRslxBVATH2aDegbd+vwfAxaD1a84Zi+706M/TkPdL6HscQ4SgJ1AEkvj+Vtc1gL5iulb6fVjQNb4xekyfqZzW4Cu/rF4DFhVM7KClZmCIdoQFp3+vzo3kZ8PuZB8UhkvpR7Suvj83Xnlmc5tJonJrhIgqpJrhSZNgwpMKSDsiYWUik7RkTsK0D0+4VbrKlTHv2V/BnPnTGn/74Df5Xf/WCLz5fef6Vhbvnpzz2BOiUWFhQNsxcgSZrba7KdDDP1e5MkRmo5omqM92I8jMuEDw+ZcbWmISbp4myd4YVnxPmsZQIQUeeR8XKiUVNRlfL30uSyEmoFOsCmgemYqFwreaRW5YZ0cxWElkS51d7pr2ScgaZKbWS2XG6UU53wse/77u4eXaLT37+G7DsyZuE1pEicH9+iUW2fPxdlf/Nnzzlw8/9CieffZ7Hz27y7ftx+iWtB0pwVRJBs+dwtWRz5yM/9Cx5q/GV2v1drwcGA3MRMoaElnygpsJyKIgOQO0byBeiep1jdxi4OpIeK25WjEYfAX3V4GMj50DkYc34BulCOaD/OqO9C4iI78VmF2nqz+P92iVyeABSpEX7raOUeu16jPvH02VVcOBJf6WuPiy9LGQt0AMIlEj4i4YR1csKQ2AG2vXntbbkq9es4YQxcWVlJbkACQ2tWGZ9kzsN5Gh3BUc/7JU1bcmQ3vAIcTerWy+1Wzm1/UFf81hXekyzK+c4AMWfE+1+pThvmK+gub7T8XK0BY01TVHWGQqvA4HwQuEeiAAcDYQVR0UVU/BrhWFc1hZfVsq+CXUX2uRVIh6uiFuToezlY/a3CaCwnuMQk15Z0P4LL9UK/MQGDCVRK3ZuhJajsyEs21QbvdYlpa38U9ctfn0yYifGtbK3a6Aq8JwdSCTt/jUtR+VpIaiDB16F7fQYDKym5msdnhV/fl3de33FgjdPQ1fRkTQs1xknNrN0WdQUdsgOoYe1/KYNKIu278XD4nPXZRorZNscY8nloIgBsKItfGoCglDfrc27bzRaQmUblwax/akGUopWl7WGPMKNnFSJus/ktM3J9uNIJsuCMnNgJGviifGS/XbDS/stMidGtjx6lvij3zPzng+f88H3KJ9/fuHlS/jWHXjLZgv7PVKUKjPFalcRrBVvmRbqrJTLyWJgm9E6sS6Wg5XJjCIm8KIZmCRUTeGPudipimLVQXjSszQaqYec+uIEHyalNdNLmPcgp0zVyjQXDsUXqSykogzZSoOvLicuLwtJtgzedjppImlis7zCm598HR/82Pv4ja//Fl/6yovstqeUMnDnYuEkT3ziIyf82e8TfujdG8b0EuwP1LLw8lWlys7aYgthNZjnItW2b8HkhfoeUC0MWOg651bDe7RPf6frwZsOnUwwZXSGZfZM+nJA60xJuZXUaEgkBwLryoLjEii6C4ewhleMu0LG4gLa65haEC9QLPR9Ya/WpgCPGi70DwXYPhKEca2buYQYup7YdWxd+py7pusgSG08DYAIRO/0GhNwZF90oS6RAa6NidsURDzWKIQl0m7RJkhTlsH2JuPWG0E7Q7V/pfVD6DShWVTpml0Vz40t7bhndVBM2B20+a0vC21GnsLgysdzJFYyLdZDgxYh38L7slpLoNFHkudU5DjRbxUaCCvO2WmdgWseFXoFQXvDlT4rn1UDh0Enp3/tvBI16VFOJiJ+KpwYmBHpHosk1mHSqHC8iV1HSeKowVANDox5+PxaB8uqrXmNJMs9SCk17489YqVgfY90S72hn/a5NQ9EouvR60cgQYh20Ufr7/8LBdarJrpODh5YN1GK58bRLP27nb96InHj/te8uoeyC5FGB3oeT5RLtjM7iHDa8Zzba2GtxRB0DQA6v1l+UIQjsQTiALvVFJ+GBeIbrLNjKIA+9UhWDh64TjPc+KqqzdK0/Wr3NkDQc1Yu5ZLdeMp+SaRl4Cwr5FNOuMf/7icf411vfp5PvrjlH30Wfuk3CpfnynOP3uBE4Q9/CP72PxH208DVXCh+vHadhFlmUhlMyS6VIWNdScvYKjdscIlSi+UNZBAyka0U/Ryo4D5Wz4mIvAw6fSKBvQHL2jyBSrYcpDAwEZZSmUrlsCiXh0JVIQ8W+kyDMEii1sK0r8zzSJZE0gXREWom1cKlJt775vez2w78s8/9NtwfuDvDo5sDf/r7F/78xxJvf2di3G5Jd7/NpQxsN4/xX/7CJeeXcHJjXAm1ANjh5YnqnDXvGXiLg+Yir9u6gh7L3u90PTgYuFSWYuhLDgUOW1JZmGXPVGC7el6Ug7VYL4G011Z7Z+p1AlN7c/VPC9mpZ/i6JBHEAe9KG6xtjdiEYsxz3YI4EkDXBNjRAJpsV8ISYCV42yc1yv5sh1rrYBftCnHql7gl2AcJcdxzAwGuQIOpIZTi9TH2IR5ZH9cUidIRpY2lx9PDMj12ZfvLfvcOBpRu3QUKEZdX5pHIKyVtwsaF9tpK0ybKV14Sids7bT0ZKoklMIYy7dlUFp5qa0gbP8lDnysgEDSrtbaE6pbfEm71AnVRyuKCpnlRuuKRFRJaex6CR/oheunIIxEH3CBOyURrfmS8oVhgIPt6W/JTWGqB8lveRABQjVi6AwCnrwHsegSCYg6tCASfA9KqPzq6MqFjU1tZnrFI4tAoUIoDH7WNanNQT3aNtcdDZf5fy79YKeTG04qJ/vB6rNa/g7m1p9GlQoQemjbuk208HKPRNqrVs7snps0z1le03VPausQ9+nXM077nmgLqvNEBiY+rrNCQ0Kxg30zG7w64I/lS3YsjPs3glRT0e41L2zMx8IIEwa1yJGe2+QR0QTYjpMw0XTEtC29/SvjjH5rg8Zu866n7/KkP3eBTz8Pf+B+EO5fCW5Pykbed8eYnr3jhq5mlTlAzdRDqMsBUyaVQPYeiTAsRu58YOCzCIIsl+VWxs0lmtYMNEpaQXnwvu/7AO7sSilK7AWGnUIXuUXQBHZxJqiUlpyExDAOlVvaHysWsHGqlqDvT6oJ4WKx1mFarLnCiWxfKKtRcGMcbnNx6gm/cf4Ff/dwLLGXkJz9a+Td/cMPb3rKwlQNMFaYrqCec7k74h5/5Nn/rN4XTkxtNRiRX5uox92T17M27BXSwL9Jyi9a21wM6Bh4cDLz9scyXvg1lgFz3lHlmscbrnCygkZjRzkUPEBDbpLurI/bVLE23uLoO1tX3/PJjW9dKtDYh64JMS2fwEGbSVHOzchuKb2bImnArON8lgKOxtZfj2PLWlhyx+kdAWVnkIZRFWie2uIcxbs+Ybq4+pWUGRyLJ8dUt8K6D7YhiU5Ix9ngvXL20ZMg4i9s68hnzJVYtbrvHqdGiP52jv9eTbx4Ln4PSgZGu6NFaJDfA5BtBLN5srhQFsQZJduxyerWyjha9GT90p3cd7CuFP7Oufu8goJaKltqtu5X1mPKK9rK6XcsbWIE96bxlyjf7n2F/2L/DWs+gbX2sTkT6o0RWoQFzgQZt1x6fnij4GkogvrBSoB3FhEemPxMRxuyHRAlHvQmML/y5Lave8xq89XN4BKIfxur2bT1kjQNCqTvvV7V17GGj2J99EY4wbwDUZnDo8Wcb+lmRI+bUBmZ7Ozp7HgOBNX2uAYXk6xAegaDxyuKsnhL7WldUKrSUA8FARDHZErmquvagKvQmD9qmvjaqMlizowDPK2B4FHpqBHESqrUTLiOQZ1QS48mGq0vDfWW6j06JRKZMl3zXk4n3/6RydTiwXD7CE48on3hf4a9+9YqXzs9Ii+UAJBnJJAp7MpklCVo2Fj4oCzNwdZXZzGpWbcpQC3Wu1KUyJPXkYXG94YN2L150ZNQqXh4slAXqEl0/vXLHtrjlLqowOvA+zJWrgzJVvOGXuecphRCqWZNn83u1k2RrCIay3WbSVvjmfMoruy1f/PLX+ejjr/Cv/OGRt79nJk8Lw2HLBIzjKXM9sNkKF8uB/8vfG7g6nHKqwkKB8FATzbkgRQhXoiFXhE6xkKTnlsU5BYocl53+DtcDg4H/+Z+Y+St/a+GTz2/twRs41AGlsi0NitMSyTQs5A4Ihk1mMySmw8yywJDs+9YUpzQ0RKDUYExJZi8JhHNwtV9QUSsHcmZ/tSB0C0ACzbsgik3SLumfZbXh/a2+CS2J0n4PYaI9dtOfaovibo2wYkVW7uCV4GrWSQjzpuygq/P1d8ylpnGwULOEfPQx5tzXwb7qJSitvl6axRUJbtmTadYn561zHK6LNXW6NuBFxzO0BLYuiEIxmLsv3onSxNRlsLSVgJza/e2Bka7ldQerfv3txDf/tgSwauTWLlxrpRSlLLU1xzLy9k6a4hPSKq09X1iVaxCTojuYtIiwtwrufJKqNZvKfub5sffA1jpCKFHKGsp5HcJBhRq1kEdX50CRruaC/466eMa86DZzczi4WDFt3kERDnqAds59ALKWELoKDaw5xcq7nGZBWH9weNxaLkDQIp612gMqfU1M0Uvz7gQFWtXONRDQadRQiNMiVmy9V6SRqXt/2rK/aoZH+TerZZHj/3XaX7+c+KqW0FbVEuCqv9cSb9tdpHl2rt2I6+sfADzYJazaFDyIEFUOgiCjQs5skr1aZ2VkZH+555XpBk+kc+bhjDGfM0+ZcSek3UK6uIvsNvzBD4z8tV/Y88LdDcwDsiykukGGyj1V6qGYt7nuEBVyqpQ6sz8oOsN2EMatt9YuxXOhOt3CQ1c1qraML82TAFUTZVHKXO2k7RTASLwNuzBKZhgGhMTVfubi6sBCRgdPRCyAnWaAiJ27I6miNaN1RtS8jKUoKSvb3ZYlVb5x+RyyHfjhx7/OX/qpG4wnd9jPmbQUlIUNIyUpG4X9Fv5vf2PPb372jLMT2OfEKJ4joKy62LrOKdWr1Ww/1ra2RpOEn2gZCvI7ANDr1wODgT/8zgH5xIF//2++wldfvkUdJ24cEiqZK7F6TU0V5krdQJl2bMvBy10WFOUjH9zwR959wj/8tT1//1cy2zIwjZfs65YkMwyZPFdk2aKDUuQSlsw45NYe1ZjAvQEQgbzG/v1DjoqaS7GHKFq0fRVe0GuWXJRuSTVLrSs4dZdfJdy+UYrWxnFkmZU2skW7EBNHctbYJcTJCvE7qsMFYz+ut4db2vtijYFSSi3BWkQoHhhOGKoM13g8JDLszaImDs/zmnxHmGpBmvCo9LMkjpnM+m2H5UrLYo8uaOoemjhTAMRPtTMLIAkWM8dQbcSEo9FZI091VSlq3OvKJ0rx1r36tShFYMn+DCxrmGqbvNZqxyIv6m08HdC1pDSX+J5db0341HWGtv4FtuaYoFBvEexz1QCgsVkRdFQHsYV4F7HqiYKAJrJYdXqOapb14UaYkmvgJBgeUO/OGCGHzCrjPMi4UmSp2hqoaGuXHZ66QZRFq1sdlZawEUHJWDMHt2RFc4NrqAoLtfWDMD6vXfFLQrUcA8dGclfIgUxckSorhQtmka98otFEtClr58V1fB7sO+JhqOtyw6pnKpKTt1X2ua5jrzUAbT/WPHsCbHEgFxUg1cuFGsCSDh7bblwZJuGBzNo6XLAO/9j6a5P1Ktryd1KTcascrbblugGTJJNQZqwbpTJAstg5aWDSBKUgubAkQZZCYsNmvOBbdwc+8zXhR56JQ3gSoxRYNuS6tS4/TLz/0cz/6V8auPfCnjsvbji9qSTNFCY2bCwcUBILlVkLo6q1K9ZCzRtGBNJCGoR5gmWvSMroIOSqaBaQwWRx7ImazMM3OyUFqiR0qgwKNaufRWknUQ6Dkquyv1Lu7eGyJkqyEuLijJRrQVJ1OexyNBdUN1YiKQmpmbObV3Ba+fL9p7mZX+Yty89za/MysKdeDOxkQ6mQ8owOUM8z+fSUf/CL5/zVnxsYbm4oy8yWgo5KPgyIWov7MheKZpJs2dUD5JlRt9RSuJGEaYbDJqG5MlShkvtevV6n+h2uBwYDG+Dd7zrhX/v+Df/X/+7AxUGY00ItGyQvyAQpZ9IAukxYTXtiloVUz3jijXv+0k9s+NjrC9/97A1OxgN/65cu2dTMLk3UckqpFfKC6kLVbCgyq+cMdhdtB+h9ktfdh2G1haBoIP/IhRjCwbdhQ+UdnRctTYgcKewQsoQU41WHDLVs1bAL/fkRAkwqxniuDNawxr76GqVTdAsSohd8SIL+sTiExhIH7T5rQAWRWNe9AWsrwgR2agcjNRDSydQtFA1+W/uCYr7HQCcdKRxTfKBINm/EymRsgjr646uqy7N1/oAPIKwgX98kYAf2WKZwVGN0K1pWhyBpU5jXD+xR1IWAtESrDhojftLXLDr2ZenAqI2vuTajYUh30VoM2MGBBNdImyd0RWDzdHBRWY33O4QH4t31wjhxu1LSWPSV9WsL28bgyj8wyVEPjpbU271DwfPxSNVyPD53fWrsV0cpR6Kr8XofWtjvMec4092Zzc62iPiDh5fis+t9u041khX/JC9HlZb96f+E50bxg7tKH0fbCHGoWDpar3Yf/1+TKSt6yNFnZOXqDRode9hijRoPiTRQoKG1VjIurH57TalUxtGb8ogBXc3mEYQLqCcGavaJIVVKnpAB7txTfv4zB37gvRXZVER35LJHh4KiltU/LdxMG376bYnz1y3UVHlxv+WsTJydVpY0MqbCVArZY+8LfhZJgSs/fzBPVl+vOVFqZZhdboonQibfRYp5UEplWZRlsZCbYnQsqqTi5d9JWWqxmv2cKBVKWZgnf23oJBY3PnovEPthEaZlpuhI1cLZduIs3+Tq6oxHx3MekbsM0wTbkcqGnDJ6tVhFXj6BUkkU6v2ZL35lZru9xUxiW2dmGdE6wpggZy7mS7Y5M9bEMl9Qxxl0y1yFpQxUmdBdJS3KSdowcQV520qMo8HV73Y9eGlh3XN7WPjEhx/hSy/t+Wv/CBhHE7reQbB4QxBB2GZBcqaUhZwrP/0HTvmeZ+B0mHju6YF/6xMj9+8f+Nlft5OmxprMnYywyEzWhGg2MBDVBU2wBkOv1GdkkMbGcsZfuyiPkiqd+X0/dSG29iBIR+lHpXFh4WBbzNove+xmDQT8eYtbINaKVrsbudIs85Cwa8HX5uPvS5Lm0ls3RYraUlnFtFMSY2QRVIdjBafaBGtalak1gd8U8bHgaXMKwCNhsayUpNrgVeiu+h5nIFwXfS3qUawr0n7DkrbJhJ6QI0DX3Zs9/BJAIdZK1O5d1BrcFLforJww0jSOGOfYVHTk1ta2tbxk5SXqz7S+WrUlZAbotH+0LayitD0qckQ+SxRSsmjLEYlETlmvFZYk2SxGjZ1hV06JpfYUMmnP0gY61vdrfOH3rj5XcX7qYML4Koua1dws2jYddyIEr3vuRwqXfT22VlRbUkHb4m0B455yRKcA5wEimhoVvFGS0wsspNdLbo6Xtq17gJpwy/bHh6MwDAbxIUc+UruxAxLJDtLcAm3HWDeWDnAsR7zsHN7WorOlAYE+cMB9Bmv6BO2OwZRZwCLS1iQlg5pL9pwQMkM2+hTJJFGSmmFWWUhpx0xhXxO7CmnI/NIXKl/72glveduV5YklmLWwKUAekXqCqFKSMtxWpv1EPYd7y0BOGzZaqFmY5ooUSGQWjb56lakUzqsw1sRJSuQRtMwwm4esYopbc2JMyRsnWGdBXQpa3MPp0f2lVHDluODHT7vush4CYg3wkKjipIUtnc+kurdULAdBF2GSwjZVtiSuDspBL5B0RUUZdzskKaVOVJnIo/Fh0Zm8L8wloYfEj7x75O98euFrd0HljJ1UypIZhwOlTpwMI6ILWi6RtGEZtuRp5FDv8e43Pc7VlHj+xftsJLOXiuiGXVSkHVXp/c7XA4OBCSFp5fQR+Bc/suXXnr/gn3xhw6ZWUpohDVT8YI2s6DIxiUJOvOPNwo9+cGS7LBQGls2WNz51wb/+Eze4Nx34J5+ryOZATsriMRpzXfoWD9T/mlds2diQNC16VL0Ax7+zft1/CUW++ryuN/pqg7aYs9I6BIp0ERL3X4OC+NcSo1h9/tqM2gZfeUOStJ8Uv4v4CXzmpo0sdstDEnN1avfuhlKy59aVrF0BJroVXb31bs29E5ZUTMFoROoVJFscK0kXkGYmN8sFpJU8HkE0oYGRI8tdrFTGOoJBJO12FKFH415RjwAENk1tVSvRUCjOf4hV6nfsllz3hPhnVglKsUBhfVr9r/j8fdotXGNQIgUgUBNS0cYg444dsUkeAVCsk6C4ojqyHH2NJErS9HjsNqFkmeHfce/gIDMaUNd2j/hXEi1fILxR4B0R19pHAgAHqHMU4WlzOB3aaaS+L2q9toOvKbNjkLQadgCNJP3AKvr+FO9BER4fZ57efe5VZHBvC4HfAhyHkdDn2/NNAvsKr6JwDt8OtjeLZc5LPS4zbs9fjerYe3J93v2N5KC7ATrt90mYNS3SgbolIYrRDjt1UmohDZmaCrkkhuzZ+ekMqIzLwKBC3RzY1lOYhLSZ+fy9C37uN27x5mewkMswIGNBk1J1JktmAtAdulRksbDEK2ViOYend1uGVBgks3hOxCCJItZIrS6VywqbPJJqZVPNuV/oXrdFIRdPqiuWdEsVqxZwxkoeYyyLLZgOFq5dBge4kbMd+0cNXdcwcoCSBDXHO5oStYrlF5WEZGuVXKYte5kodW/Lf1JJwwbqyHabKXmiloVBlTwJEwUtI5InEltIB1huUbdXHGQk5crERJYdOg02l3GgpMx+f8VJhn/nxz/A7SfP+G9+9tcpdWQaNmxr5ZAm9jp0UP4a/P5a1wODgVFGDmmk6MQbnxL+pY8mfuOrhfkwkGVgCffnYPHSNGyYWdgOlR/8noE3PT6TRZmHyikL09mW7/qA8u9cVv79by988U5ltxVYKiPZK03UUKN4vA8TXEeWum/Qrlj1aHHtM9/ZJXfkyoOeBChQ6rIyC7T9RAy7amRtmjSvVY82+Dq+L02Q+N7UY2v8usBuHsoQzO6ORVzxJztFzcAAzTNgZ5eYiyzkLqxp1q2OQL1h5YanAfcM1BrxNfeCVGyj1BUQaG+oW69GiyaEMcs8kpe6kujWVpY2kmZpuZz3De1DWq0xaFO415k9+taHq7aKmlsxnhlr4lUU10Hg0b3shm09W/zVRxknXTaw5cI3OkbWHPmjTotkbVKTelYzyY7EpbNao5HAkLSPg2jORaNhrF2f+woUowgZ9cbRr7UHguZhkfbXe3Osrq5jff3eQvNctF4CLkTV82w6rdYhr+DD1yD2q9bBLNrXgs0BuzVhTc64Nrf13pbGpvGBbqlLV6LhY1AHOBa4ahlDbW2PK01WY4o1dKWF2F5PRH5HepWR0ud/jSDaX5fXfOMYhDQ5E/IuX5N7YkAgHAqZZJZzTsy5MrAlcc6iW3RJpJOR7e099+/NTFeZ06qwq4gs6HLG3//sPf7E997isUcTh8PMsCgyZJIO1FxAZzbASGHPwFhMaZ9fzLw0J85OM4NsKLJYtn+yA++qo6xlFu6XCrNyW4VhHDgUC8NJylQtZqVjZy1Y46DUwIUK5sWsylwctHjOUK1K9RLK6HAYJ+oWNe+BEoneQqnV84MMhCzzQkkjp8sOWWYO25lUhVoH0hZubQQpM2wWB2WJXDbWq+d8IesJOsC9FyZeOsCyVMrmHlvZ+AqPyLRBh4TkAvNCSRum6Yo33hj58z/9w7z+yQ3/z//vz/O5r+zZDpmcFnQS8kb9+Hp6aPMBrgcHA0A9VNI4IycDP/DmE37w3Qf+208WdpwiaSHpYIehFCEPAyzw+kf3/OC7E6dpopBQ3bBl4WSzYV/P+eEPjvy5b1X+k/9aOZ+FlBZyzcwRS9YB0kR1BBtx1M78weyhGbrSa0LAJYlIiFIXgS4EajccjjZpgAp1+GgtROkJbf57jwvj7rxjV25YE0iP30epZFP4bYxdeBEKKz7YQgNCTg4EchwaAzJEC0ufUCtD1GbZA73dsT8mNVe5mvJqX7M5tf7xYQ0FvcLaj3BES+TqytO+XzxsgAMaF2MrCb/2ggSB48gqV1FN8FkQIUBFJcpJj4S/uWwMmJhENlpWbeslCXPphqdApIUkVux0ja9c4FdPJnR+DFVS1d3EAV5qVyZrwew9ZUhqCWfR7iHGpnYcIBWxw1SwA4u6Imk9OxtPqyeK2nOcHgIiuU+InsPSKao9Ox9fJ59RXnv0nfmbEsT9JtEhMojTLiXUa8vZaJIpxnNcrrkG8qq+Plxb39VzOuCOtdLV6x06BuI5Agnx2FDavmbtLA7FksgiL8Uzs9UVarT2bYPFPiJR7B0BZkmN37Mcg6DrLtwQJeu5+CAbcIlbB73XsiaOlm65Dw4AVNdgXD3uDmm7Nfe7CGmwUO1cKk9sZv5nP3nC976v8uKdws98ofKPP3XON56/ST1kTjYDv/p85pd/feYnvqdS80CVkbQsCAtLFjZDYt5b6d04Co9JQtOOu1W5fzhQdWCTNhR1wNqSk5WsiaLCfjae3rDhjJFZJ3SubCyeQ8EMluR52ha1ksiEBq1UKtNs+y0noaqVeUrCauVLtiTi2bi+eDltVVvL4vbRplroe66FstjR4xkh5YGDHKgibPOWmgtJdui2wHZB6x49ZJKeUS4n9lrYbZRXvl0py446D8zLHnREFQY9cEllm0eKHmAQRDOH+3d5/9sf4y//2Z9gmw/8H//zn+NLv3XF7bOR7VChVM7HxFi31jArqYevH+x6YDBQq7BJM6UomgYef7Typz4i/MoXFl6+XNhYV2Kz4sdE1SvKAb77gxve//rEZtzBtDAI7DYDykzdnCKD8BMfueDTnx/5mV+vLCmzAajWenFQoaw2a2QvN0+Bb4zIcm9xyrBQbcd1Bdv2WN+8a6QdUmWdlGW3WFlo4vdsgEOPBN310EStYZn1jOacrWVkKKDYpNHKdQ0INEGrYXcQEOcKWKc9szhzzk3pV4U4MVjWFRcN4ITFB6XWXr5VlXXHuBYOsBKBoxhUuCi7V4Gj7zWaHllsq9bTrjxD8K5TNJurPpS+9ynoh0WFCpSVgop2pBw1JkJpvntRU+JhOVjOxeqjfnjLOkn/yNW8yhmo4TESkOqWdHLrI3m837ChlYhFzDwQjloSl6hl8nfLvK/fkio1mQVn+sjPwPNz5JPbd1UKfoiAgyU/KUP6ROJXVT3qWdDXJn5ZGqAI/aErBNxWU6KHvrTzZBBZdd4LwL1y4ayxgr/Y95QcvRrpFyIBChpz+DxsXK1vewo+8fm2wfpr10pv2+FHaAtFhgudKOcT+oFwziTRGCy8SdllkYWzDChINoWTHLBaoxjvOVId0DY6He+SdfCq0TrGtzqxSqq/1k4jPAYD4sl1uCejeRmxXKFSFqa0sFlgVCFJRceBZQ8feXPi3/zRBDcmdD/yo991xlc/fsHf/vTEf/eryqe/MnPvnvB3PjXwo+8cyLcP3Juv2A0jp7kgh0LVLaMIpWQOVMaNskU50YXzkrmcCkuaSVinvOp5VxmxFu4VtFSuqnC3FAbNyJg4VPMeaBZKjsqL5B0UjVSLKlqM/mWG+QBZzQAoxcINKQELlH1hmmbmg1JTYqmwOMvWWg0cW42f5R8titTMFrFTR8eESKakQpLKUEF258jJhlIG8nQLOVTqPLM/FDjd8dLdiXK1JZ0s3Lu7cEe35Lqj6sT9YtUeJQljHZgOmYMUvveDT/Kv/5kf4vl7L/Cf/NX/nm9/K/G62ztGueBy3qCyYyuzleHnZHpA5Ojk2t/pemAwwKCkesqmnDPXmXF7wgeeLfzoexN/41dh2CjMMKTCY48/Qq2XnNxQvu8jJzyWr1gmkO0AY0WHhVEy0zwjbHjqySf4xMfu8Utfrrw4DaYUi5BYLLFE7EjNnLODgVD4KxdYEnPhOmRuGyv50a3ahdvRJhOaUGClKF+F2CVi2nFf4cicWrn+Vi/5JmxmiSnNIerh7e+aunvvyCXeXCAQXoLkSj9FjwDB2u56fX2tIT1X8dOmzK7nPGi3ol3IetWceTjc0q8ApbRTKIMepgA8VwB323cTrc09+VwsScp+IsEzwFxYkLF44c3RaL9M5B6sdMqK3rq6h+BlXb4GVY0Gqbrlp+7CFumubndvc20NWa+HAw6tljAVIROL3Sc0J+u4p040TwYQrDNYAxVBB7XJq4j3ITega0re8YsVR1uGdxuftCV7VXLQmocb5daMfPyvrF9v0w/+t3nFejdndSSB0oFvy63wexiNC1o9tEVt4KCtlISS7mOQJI7broHw6rklwNFGZgWesXWSFU3a/dvHhah1XDsxNO67BhHSMUrzeqlavbuDbhEP20mUNbqRUrWV7wS3B0+Hx0C8/MfyGx1Ur0B0HxPHsqDBndSAceQtiJ8lEp9PblQYySIZ1UIYaRjZD5VRM1bZMaM5My/K+54VyMrVVNlm08xvejzzb/+Y8Be/T/gfPrfwX/1i4dNfy/zTb2S+59aGm+XAwhkXS2GTJ7IqhUJKym4UytXMNm2REWDmYoG9FrZVGCRbxr9aVX8t5t6nKJMq9+aFXRVu3hztJL5DJWWljtL4pthGp6pymGd0D5sxM0+F/dVClkweLJxbE9RUWQ6V/aBcTRPznNBBmEpt5+UUBZKf3D4YwChzYccGxNreX5WJnAvbYWSp8PiojDLCBPUwoYdELYmL8715Di5n0n1Yyp6tKs/fOTBfnDHkA687nXnD05U3n53z6O3Epii/9e3M8sST/PEfewdf/eoX+Q//m9/k7reUx24Lk14yccpuHFAWyJmhGi2juZKmFaP/DtcDgwFRyPmCpZ6wqwtS9zxxW/jD74d/+PnCnQqaB0rZ8obnTnj9zTO+5113+SNvnVkqSM4MFHZa2chAQthWWEphu1n40NszH39P4b/6lYqOIFLIuTJXZZhGy2yVakK+xSkNtVVgVgtlFBUGT+xKOfFd7x556kbhH39q4dsHOKFSdENOCxVYXHil6omA3j64KX8JZSUcdf9za8TkotWXRwJVlzyxaQc/nMbkEDkhQ2q12g3su+fAFEhYxSa8IofAxtDHKAi52hhLLVQiSQ6yCqV41L8l0nklQOlCJrnyrygsnptA79SVRD3rtoMkSwrLzYJPa2WBIBKKkJYUh2ZU0qpVZiWODHUqEuorlJnWQqq5eUY0C9F3QZKXFkk/kCSOyTbq55a8Zoa1zV1SYggvhHgiuwrW98rd/qH7zXNqVQIlXLwrCz/AS437YJ4Y/17GAGnJroCqPSM7GFAf/+Ljz1joR0VYELK6MtU4l1xW/IlX8LjHSQPU9hyBqAggciaOTvrTpr0Vwy4Ins9tQCdH1jWEtiOq9gCWvBBhKYdKtE3jLpfu8u5JlPFJdR6gf8QTtQy0JacLBdsDrQx2nbMSvBlTFetopz15sYF47a2YrZOjulXfk2vj/lEd1cIDpQNqw7m+BsUqYtoJp77GFA+JeSIsWNKpJHy/eP4APjf1yoPiQCwAaHhbgkQBrFcyvrfcvgaUAxzEpnP5UjBPwK5WroYdm7qwyMywjGxG5UYSYGKWzEa21slwOYDc4GR34Mfee8aPvvcun/z8wH6/44Itm+EK1T21QKkzmjbuDYGhgm5HuFqQlLi9PYW052JfONSBJImNWnLxnJQxwVIz6TBzUgYuSbw8FW7kgSyJO1czqSQ2YyZtoW4KqsW9egPTkjlMhdtqSe5Xc4WaGE6ww4+WwjIK+1w4SZlStiy1MmHPBctHQBNpqOZ5SHBxuEKnDTKY568opJOBlLfUw4HTk0I6Gy2R8R7ovGM+KMvFFTIOzGlgeeXAslfq5pQr3fPSJXzv2we+//XKOx6Hpx/ZsxmFTRmZZeLz73gDLz3xRv7pZz/Pf/nfvsTlS1tuni5Mk5LHnXPFYrJ0sf4YA0DKLMmOEXiQ68HDBCRrJSkjSmFWGEZ46zMbfugdB/7Wr+24sZm4NxeG3S1+7MdP+ZNPzsxXE4eriUImJ2HImSFbs9U5AX4k8hOPbPnQO+AffW7m5ctsJ2alDSMV2VWOquRdKZpiKiQ195PmDeMAsizUknjzs/C/+Kkzblzt+dbzhW9/qaCbLZFhmxnMkguXIN2Vun6WuDJr1oe/rqqWYFKuexLWpogLmGQKnYxb9v1IXa5t7PB6RLzRFHgHGbaf+xdWxzK1mnlVc//XOO1MXakp1FJbpzbxjLju+lcvGaxEf+/iLunWDTGkYSoGAPyAldrmsSr1UiEas4QJX8MV67ooIJM0RWT3cEPtyJVtHpjV591bIhoJhetM79rK7q4nigoehhAgqYEMTS6I7TnxY8DaQdiqwU2bR8ylWoBdk7QEUU1KrYJlLbkgT1CTtxuO7zXT3L0NYnularG5ISBxQqg24zg8XuqC9CiZ0JnFPFrHiXnhtTFYok7n0HJx9HNuNzJ9qBYOCf+2wrKUFYDFnxO0i0G0X0z5RihHhSXouhpXr+Sglae2g6aOtsox6I5l68X2q9fav907ZvMyHjW+6N6ntsnbYuNg08fYkhEtPNTWOPk5HlmOxEBUAaTVBFSj62rQzqVQAANV8EOlEvRSVB/nupRYGhjo+6MG/ZwesQKs9mlldF4aQBKbDBwuuVNOAbhFompB54W0GSl6ZUbTckUeTvnQe2eWeo+0nLCcb5H5YGW8esq0HBhSMsntMnTIibpU5rkidUOd9ixLJQ+FcUjkpCQqO8nMFK40+XkncNDCy+eJWycbpCpXVwvLVMlF2ZBJMjBVQbIiLuOGDOMusZTC5SXcyImKciiWVluKoEWos+VKHASW8GRSGJOdHkqGRYVpL+wURhlYmJk2M8OwQa9mNifCjVunJJ0pV5X9RUUPFQoG7FOGWZmXyrkosjkw18qPPT3wU29buHG65zALpW64mBOTVi5ubvm6bPlnn/wyP/ePv8HFncRuB4qtl+UDecK4WIUIKRmfokjKK775na8HDxM40s9iyqeoZWm/6fHEH3qP8k+/OvPCVWUoO5567g4/8a7Ko+cHLg9wXgfGDYxZGAU2kppys/iMcjIKH3l74t1PV37xC5lxHJiAQYudhxBWBc7LrZbYjtxNCld1YqMjU1Y2N/b8gR94hD/wXOXXPnWFlomUBzSHRox4aaXVra3dhG3eK8Xrv9u2D02lnrHZFWXLPF4p+dbcpTX66YfY9DL9sKibakDDbq6KNgDUR2hKvsf818K0VqUUHAz463UtyGix0WNvc/FwQQAEhxsrARvfVbG4NSni/e461VWiWlRZhELSZrg0nRvB9SangybuUo2QCh4nJUM7e8CZwugZh/VEH4GIazQo1ZdWrP5dcAseccEemd+uRKt7Plp3QrPICatuNeg+FVvH6qAlWgwEDSNnIauHBvCeAdKrMlTEDisSnOOKn0+RadUzKl3JhTXZoWEDCn3S8XckunrjHBGPbKivvo2irr6jbqlb3kUHhdHl0zjHeeyoqUdjGEBayqC1TJXOS8oR2FotlAFNLxdMq1UMQOR3bP+s/AZG8qOmAvEqK6QQr2tL+mw8o0opDsKq9FNFXVYEjW2XRK5BdQveXJfX6dFymwJkqat4oXnmitodMzSgm4UGNMmp5baELAmQHEDKIgMJ707VaQYtIRhma6KkCXRhkzOffb4ynW/Y3C4eZp2gWBO4lDOkylKvGA+JTGVJ52zPhHpvQcvIoVrcvibbW4sqg8AgYnkEtVLOC8uFUiRzWCrDppDGRCaDCjtdmHTgshbyYmHKl5fCmJSRzN39gata2CzZ9vlombtDqdbqeFbkTNiM5mWarpRlZxL1ACzAPAslQ51hnpUD1ohIU2IUyFIZBgvVXc2VZcncTIIuhWlX0U1GS+E0CzfOtuSUKJfCfB/KkmF2Y2tM6CKUqXBYYDOcIcykWXl6NzCnhXuXC1krSz7hLC9sT4VPXuz47798xWc+eZeLlzdsgb0qO6WdT5CbgQPRkycOprPTXVce7d/hevBTC2UxIZGFDTBrppTKboD3v2nku567x3/zqSd56tl7/Isf2fGMvIKWystXB3RIDBmGbG7HcLVV30BFlbosvOEx5SNvH/hnX59RNhY/GrLFYldWsckMm2y0sJ1KZRxHko4MOfHBD275V34ksc0HPvdS5rfvQN4lRGazhhIUFko2V2wqgop3MlsrW1fWiXAvattIobIj8a2WUMr9Hq2ZiMjKxbmOH0K0IrVPukUk/vDIsF17Alrc3p41r1y/60zsOAWxzh2oVI9z9zMPXmOtV5Zmsyyd7kk6XIpSu7D81JOsmg3YNH9Yic08ab6DCK1EAmgLzaaeMGkdCFefB9MrXg7R8mWrdu/HdSUYNG3WnwO5pqm1hWIsH0KbSzvOlFeR1mgpqRBZc0JXlmu0I+7qJko/U+QxiIVdxGudCaXm+8KFZgOhiiXREip6zTw0UBaLp26dq9bGKw1YNS9JJGR6T4+gV5xy6Fo38krU771gYS1tCtO9QkGDoxJCd4sLZu7qCqbUAK0S7NO9AdDBp48l+iXY3lsJv3UpEJ3/+ifkVXzQuwOuM/st9NTAqZ+2aGylRwewaQEz6s2zpMn5I7rVObjqVQohA+L3ntAXHSRr6zhp/5pTIMKh2mQMEl6S1GWNGHnD6A+a5dR5qJK6F9ABW1BoyIsZSWxYpsrJ7jaf/PI53/rGjqdvTQjF5PCcYcbCuCmR84hOCljYrVIYNqPF+a8mqMkSzvFjtTTW1nqjjMtMmiyUswhczAV2sBtMDm8K7KpwpYlpWch14KLAyxcTT+zswKO7FxNbtTBiLoVxTFRRlklZDlAmyy3IWTiUxDJDTcpclAnhsLN25TpDmWCqio6BeIUBtf4Hi3B5uXAybBlEKWlBVUh1IOvM6S6zyXC4d0XdA1Oyo45FKFJJmpADlEOlkEgcGAZBLxOHw8S8GRi4TSpXpHHLLiW+dnWPn/t84TO/NfHNVwZSEqb9wnaIXJHYIuFdihdoDedA1x37f8frwXMGHHFqKiSFTREWNug88+TjW37snWf849+64M/9sRt84k3AoVDKjGZlLNZcYhBhcIVRve2kbSJD/dtd5kNvSTz1qZmvvFAZd8bMYzouxxEs4Yrg+ySM4wBFmLnPm55J/Ns/fsrHnhW+/MXEZ59feGXZMQyVrBkdBirFEFMyRk0SlnO3rGAVm/M+CpHsFkKiV8Mk1ylrwRNjlNa6laOFMQETiXV2FGVvPANmPYWykBBG2pOX7FrlUNCtpei5X4q5d1ulQfRJX7taGnG1PSes7VWrPEvQahaoi16N+H51wW1CPeZDkUiqb2RRaONJARBreBccZAziVlUHMkHP7Mpl7frVos1ia5CkhVn6ujTAFbNyASpRVugUrbXZfr4AniwZPEu0NF4pWV0B1lotecciLoZQ0rFyqrWSs/QFZnU6oSp1AZKQMwY8/GAnNLweDjgI5VpXYKDBNBoh/XO18ZiFoNoBP65J45CmFvqHhj0Wf8OUVCjz8GE1KEhLiJSuyKLpUxgEEZlY5x30dQnq9xVrcgijp4EB48m2vm0sfT2PdmUDzOvw0TF41NrvYXzm4DG8Py7DRKxqoPFRAzb0Kh53pcT3GtKqBk3RKD00cIl4WaCfSIcINamH4wwsRvpSRqxePg73WgHalkPQRJp7e4pXmlRTzAk/ECllA0BJeP4c/u5nZv6Nt1fmcSaXU0tgni05PUuispDz4i6tgVqUtBEGYKcKl4llKe65tGcsqtbJT+HGbmCZ4O6FMhWYc+JyUdha6aDMwlgrJxXulcS8WL+QV5aFXUmMaaAsE+fnC5u04VRAssHRumTKXpkuFk5uCbucuTtX5llJA9SpMlVlOhTKmNDFAMRUqvf9qMypcpqFoQj3LxamfeXWme3hOUGSgWFWbtzckLJyOF9YDpaobMeBF+ZU0JzYlkydClMFxoTsCmWeuZgzlS0se0q+ZCRxyhV1TPz8F5Vf+pLw4jcn6iKILGy3G0Y1mg4ZPPuoAYEkDsDRpruO4fl3vh4cDMiGMS3Mou5eL4ySqQrjBt7/VOIv/1H409+9Zcc5Op+w32c2y4zmEbSSMAS33pBgB2UUVVQTb3pd5l1PD3z1JWXcZApKloLK0JOEBKIdiCmvirBQNyOvu7XjX/0jmT/23Rk9jLx055IvvVhhEIaU7Bxs32y5DCafszZLMI5l7UqAjrij5Vjo0JYxRivjseYiXQgZ7Tzrf2WzrLP5wyJKTfjRBEo0r8Fr4c2Kpm9yjhVdo21xd3ep6FKbZael+KE8qy80QUW3AqHNP0KwTQhLgyr+pa4Ual9WCx1ocVpYyR6S/JyJ1QdlDWLiyaZBLQmOZrW1GvuwzJMRX9UTvvzbayHfHhWgRTuYCUkZCY+2ib2XeSjysMA0WC6FkwGwmuTwytFuK61M1B9tm1RD53dFEb0X7GXtWfOEwo65BviCVSu6dv9Q/KvpOi2K/22fKPSy2OJulLriYdEoTcQP9JEWu1890ccXp6at+KGvgvPMayhm8d0SyGZFuz6nbmXHaclH5aeBMFSP7h1rHDc9phS8WjweAwkDnsfjOB6eb4g1iHEPh0QOh4cGjNfWbloHNWLhBvH8jBRhjCZvpLOc30Mz/aAv1ZZ/kBwBXO9BH+dIgCcpr8ZL9K5AQEdq8bBrHpnLOWmX+eufvs+f/oFTbr8R9JBhM0OekTrAUqxFsGxgVkQqQypoymhKbDbmbb2qhWkxkFpKPDtBgQ3KzbMt82Gi7BfqnJnmCnOBTSLXBLJwUu2E3HvTbPJA4cXpwK3NwC6P3LucuEyJm8OIjhgAXRIcKtOFcuvmhrMxUZY9V1PiNFvIbz4o86TURaAoywzTAgNegpwh70YOV8rdOxPbPJCXypKEKVXOUM62wpDtYCSdBnJRcq4sqsylUAYlD9kM1amwpEoaBsZhYD6H/TyTZCDJlqyF2xthujHyd78+8TOfGrnzdaEcKgPm0U6yZ6o7RqkWsoqwcoB7F3hejAwpkfWfcwKhJMjuI9ScQAs1HxgkM8+FR56c+bNvPuPZs0vKxUgphXkqDOMZ83RF9vI3SdqsXY24JaEQK4/cyHzgTYlf/OLM5HH1pFiClx/E1DdXbNIB9ITNVvnjP1b4Cz96yi5fcv/yHr/5tcwXXqpsRkNyiwCUdvCR4AlA5COrPpRtKAmwjS61Z+anSDVwy6UrbejuWDwrultE0amwuQyr+kYOwe9d3FZCTjxbPantJUmr0EC4gnVl1bsHgEr3BNRq4KC5Vo0CISR6+EJXiizMitX78ffRKtjcWzm63ydoZ3H82jK5xZVBzjQpayBTwyQ3kCdyJEzdmUok9GnFTpcTbUATf5blX8bNryuJ4+u4cFGbED1WUH3eGmEftX4Ya0I0HbVa49ZCOkF3IRsQbj0kwqLzmyQVBwvmuZKwAKmoegB9FWOPXImuom2PXfdEFOd5cJ7wVsoWKlNz+YqFvNYlb5akpk2XtiV25HoUlnfhlAMsKb5hwkUdkf/Yw92bc/2Kvg3hXlpDjqNjjenKPFjwNe/YUOdqrT0kokckXQFFlEg0tH4hwdt6ZIXHcOqKdlGmJhwD0WhJnolQWABYR4zJZEN1C6gl7gaDrfZXINGOcb2xjoM+A08R+rL5z2IKWbVYhYPOVLHy5F3OfPZF5b/8pcxfvH0DOblA6kI0fKh1Qe9vSFvstFpNiIzGwxnYCKMan4rCvLe8i7A8lqUwH4SNVE42wjIIV3tlnoTDoSLbynZjYx9QThQuK0xLIcnARa2kpXCSB5a0MB2U8z1sB2UzCDoVylRY5kSqiRsba4d8uV8YNyOZAV0q86FQZ8sJKAssk1eVqHC6FWQR7p4vnF9Uzm5mKHAoBRlgs4HtaaLWGV0SQ01IrVSBQ1WKCGMaSVNiv1+4qpW0SQyDMmjilStgyezGhaFUzk523NPMP/jSzH/+SwNf+LKwmwtpnLhEGYYdSzlnyKcMRc1z+qpcGOOxpVh4A3qI9Xe7fg9gwE4iHGSkjtniELVaK+L5wO72lmdPrFymXu7Y37vHRQbSFbthsCzRyGxVSxqsbhHbUBPDAMMpvPXJxO2bC9+8VLZ5A2pCE3p/9tgB0tD3xPd8MPOXPrHh9ZuCTjvu3dnzyd+eeWUSxu3GskJLZXT3Xc2LH9s6GHJedcDCy38sNOL7roa72zZhNEQpWN5Dt3hWYs2BQYz3ta7rXoIAAdH2NBLyqLXBp3DPA+btCOCBbfpaa6tyCPd7HNRxbBj5eysBGhJXUrhfe08Em7s2QdwE8yrDPMw2rTjIccVFzNOFhHiLT3U3sAjqCX0IpJTd6lkhX1aCN6jqMTN1K9S6IpY2thDWBopivjS6alt3msu6Z+wH+AgQQ9+ANZS1+DkFroACoIsrKxf6KbvQd74VMTAUyaWtPMyfmyShtRBJpeCJg8YMDqBzG/uan2L2pfGeNtCnagc21Wy8LO7VqLjHt9rUImyh6iWcRxaye8FWXB1VjPGZVjaqq+oSPMu95QzoiguP4WUAqVjr9TZq8VHtvoKjPJROjX6P8Hr5+5YL0pW44jJJaPkCIZs6Y3ssv4ESjck2QHecv9G9POv1EbGzNzpiCQDu+7gWSLk/NWilzuPqcE+NFolOHDkiY6dsUe3JxCiSoZZCkYWcFIqd4DfKYN346sBf/YUrfujNI2/7cCGVREkDOVfrHZwKdSpoSuScUClW451ApSCpMG4zSTJSKxSl1Oz5JYmFgWl/xUhmuxk47Cs6VWZVqDOVxHa0UO5OlLMhcZiqHToEXC6Fsy3c3IzcmeFyX9nlStpldKks88KybNFFOR0GNuPAYYF5UTKZVMwzMM3FWuAXayhUq3mxh22iznD33sI8KwOCVmGulVvbgbMbIzVy0BZBygICh6Ww5ISO2fImrhYuDjN1mzjdZvKgTJewvziQl53tuS18A+VXz/f86vOF9z6+I80Tv/E14aQqOzL1ag+7AfSKyta8OZhXL4ufdxKNm7KXaS6VUv85ewY2IiybAShscrEOUTpSWBi2ymOjdYdiUS7nK+4WK9PakNERKpWsSi5q5RzVGGIUi/8XgUJiOyw8+9iGZ28Vnr9MDIO5tDMLJe1IuSK6IMOGmydbDvsr7s/w3FOZf+1HEu99ckCXhbLf87WvC5/8hrLIyJiFuhRGRlKGG2eZnIU7F5U47zwXj8FRrfuTjCDWsVLmxQRnTjCIzWEOy6gLhQYnIl08XHUuiKkmIJNoGErgTNZlSMR+u0AMVYiXr1G60Gu14x5G6eWFagcIuTtWI8ZKf04IELNw1ICPx6HaJbgZo94cxy0VV0yp9FhmR3dOjKpuEXZg08rEnG7mLai9wVCrtjBXUI7mNf69hLk9U0p24qE/KmVze9fFVKE1qKJ5S+zQDnE61OZjaB4htRwQO9nQNGRuFl0zuRrAUxfgGqvjHwk/RopOickUhEWoLBE1ElID2Kx7WIh3pixUZPBjWP07WipKVKLYM9fJYN0y9vWXY34onlNgWe7VsbmtZ/WfknrJZiUO4YrDqrolXDQ8VXKUpCQOFhdvBasry1oRDx852F3xYTtFtHGkhVba9hIr28uElewK/BoYSmFFByjz+fkmJPoyIK3tTwOzIuYFibyXSDCNPYmDhxT7Vo/BH1jGf+u7sUris/ub4M6eP+Rb42gvRiSg1tL2YSllVZHSE1PTCqyKyxT79mL5UaoENAz4Fk/alokiN5hrYVMql7plWyeWcmAznLAbC1+aTvkP/n/3+Q9uPcFTz95FRkuck5xhrOTiJ//U4tUN/jvGM2lQRAunO8g1M+1hWtTkvlT2MnBvX5AKw5hIh8IyFWa1ioO627BLAylNnA7KMo7cOxTmUsgpM5XK7iRxmoCysMwDe5RlTkz7xNVFYbpc2Gnl9mbkxaVymAvbDLthYFG4mAo3MpRZ0HmBlJEhsc3CcrlwdSUMm8xWYDrsyZsNtx7Zsj2pFCr7y8QwbcgCV+y5n2CTCrlkruaFi4uJccjc2A4Mo7BQeOU8cdjD7dPK7UcH5JHEdlf4gyP81Pec8uhjG/7Dv77wm5+buXnjMeZyh329yXJ1sBLPslAHP565JhhNjolMPPvMTd79zClffn7mN798l7n8c64mSCmZV0KVlLGFt3wqhjEz5ME2xtIbvywoaEE1M4rFsIsLZeNq7z6fgAIjmSzCIzcrz7x+5FdfLJRSkTSAbKmjMrEgqfDmJ3e845kn+OJvf53xcuKnf2THJz54Ql0uWHTk4nzkU1858MX7yniyQevCmDMlV9IAb3r2Bq+7kfnC1+/x4r3CYRkgZ89LyNw4GXjsRJj2M+cFzh4feduzlXe9BR6TgU99pvAPP1t4ZZ9MdXp2dTOMXfGAhRUS5uIl9mrLK1AkD02wQHgEgvI9Kz/chc3NuBLy60SsiPPGaYrNclx1dTuyniQUniHL5Ifp5Fib5NJPQhEYWjdFnW3edWXZuSeiW3b+k3rL4uYSlwTVa9VJpNS9JLi6TpFgZ0kHDjhMkq7jomCJTRrxGwlB7+1LHSypH17SLVoNOe4Kyt2pIk3Qu04A9UoWqnubXYEkjvMw3FpEaquIsBbF4br18yRWFSZBl1ddRmoLszgQ6Ounr/GFfqUSSXb2uawrYKlRIuceJVdOUT1Raz06iGmdeNcymGW1JNpzbRRWXmxp3+toOdbrGBC09QjQ1ypnpBnhoK1Apb3nNFqT44gyjaec7wg26orUbQJ3mR/fxQ5tWpUgi4cGY46J5sGJCqR1mWc80I4hTw2oWvmoAwDf5wGcrRHYiu7SgVIHORZibAebreYS/UKicoLga7/fMg287pGZn/4YPLFTvrG/z1e+teG3X87cPYf7FyO3pfL3vjpw9jfu8e/9y7d4/A33qJqBhbzYgXIMg91zqaDFDKaU0OjjnIBR2GwhVaAszEsFHdmIudWvJmV/WFgESs7WPnjGvJm7mUESWeH2qKSTxN2LgqBcpB2Hq8pmC+OpMGytJHPZKuW0kk+FO/vK+X5mGRauVCkqyLai08J8pVwNmXGbmbQwLZkR5eREGatw72LPrHBbThnKwoHK47vMdpOY9wt1HsmTUOvERYIrgCGhsrAcEud3D7DNjDcFbk6QTzh/ISNl4rm3b7j5eOb0xhbJBu40KewS3/jKFX/jl+5yWW/zvrcpL3HKr3zqPmdjhlk4nBZ0UuvbM8BUYZcnPv7R5/jo+5/iN77yNe6/8jK6z2zyjge5HjxMkNUySLULDRWLAY/jYCWDtTKXwlLdZQHMVSlL4WQcKPQ4eFgYKoZXt8k8BCklbuwWnjybyQI5j5QEs1oSUZaBJx7Z8rEPPse2jLx89i0++v6FP/sDwglXFvPcV155IfFrXx+4XITNRqm6ZakLSOX07AZvfdvreOvrdzz6xAmf/dwdvv6NPZMulG3lrc/AT33XyLNb4Z98eaHcSvyBDya+/7mBJ3eJ5ZXCf37Y89nfTlxeaXfFhsDUdRKX9k1YbdOymr+EFl5dR1nNGhnLvg6r90pIkGZJxC90IaDdc3DUNCnuLxF37ZK0ZUCE6zNjwi5Js1rBxVztAidcpNFSN541iCXjdHfDegQxtt5Uaq0Y1yWZaysrhO3a6kLdSo7ETheIVa2rm65c0z2+7t+ucYvudF4NCLfzjqzWlqPgQhnpIC6sNPX8izQEGDKtECGolv1NU9Ht7zWgAomlbZU0axodgb715XQNT0EoHA1auSaP+VqyZlS2xME6csyv0Ne7qvXuj9elOIRTDxtECMdzGLQrYoUemtPVGIMzQvnLmvL+vcZL3Q0v4NUbq3wap1WP669415Vz0MLyh+LdHrLrPLbK49FY/1WeDdIAc0rhyaKvc/LqpAAOwUMC4klt6xBeA0mvsayRdGw3abAoqNPBwupmAYaD1veWyl98L/yvf3KkbippHrmcr7izwDe+MfHZl+Cb377Pb75wg0++sOc//UcL/8sfu83Zo/copYO+IuY1zQq5JKgZlYwOPXSaMrDJZBVShUMtXM5qlQfDwJwqS51ZSFRJzFopsyUdLsXOccklsRM4HSrpxpbxxsCMcnlnYnc2sLsxsstb0qgkCnnc8OjNjG4EkS31xZlhJzz6+AlDmtmXglZhGjKXWEuAq4OSdpVxW7lzUblclDxAEmUqiUMdWDRxfn4wXl6su+K+Fu7qwjIKt053nJ9n9vuJcTdw+1Hh9BGlysDl5RXbXeJ1z46cPqpINvm5zAp5Ie1OefHr9/nf/2cXfO2bj/CTHx74S594hv/wZ77IXCu5JGbZI1cZ2W1Z6kThwBO34Ec++hxPP32Lv/kPfo1Pfu4uhwrD2YZpefHVDPQa1++htNCYNnq+M4iVPUllGMMirczFSk/WG3aZKstY0GGDYi1ya4m6WgMOkioyFxInnI0bnjpTtnlBxdyNmwy6FM4e2fHRD7+V5558hC/99jd453MH/vUfO+WZG1csZKQmpovCl75e+PS3Js+C3iIU8ihoUs4ey7z5rU/zntc/yjh9gW985ZIvn+zZbYQf/PCWv/Bx5fseS7z8wiXve8fAU09lHr9d7PzquXD3vPCFb2VevhhAK0lry8qOEr5IHoqdqOuM+GjCkzAVc03odOHed3IUkESugKq2nADDE+rAwO9X+7PX97F1aVJ2JUgteSxJ8gTFZI19sie/ZewMhMEzl0NwZXOrJ092OrLS8OOJXVF2b8CKr0TsGFFWMs2vOI3RWuIegxDBLGWpK+DRcIArOyzEZD/qHQK1DW5tXGuAqhXvStJea96pRysJEAdnGuvbrTN1lB9nRqQcLaUj9yLCBDQv0tH8m5WNI5/+7xFQeQ1gAOYBWNBmfXrgo+Mk04AMMnjsued/mCcLq36rqzGslA308XXQYkmGsUDm8NNuOodFrfHtDhjXU0+rNeghgEBMq/nSlgFd06vxiQZxj8BAgC3E2hB7QpADgs5j6oGLjvkcRMZJcN5sKYyaoAtJ0GQ5ARIJop48GoZUS0NwD0MAdj2i8fF17M2jn/XhLN33lf3i/UEb7TWSiaNnxyFxQ3ZoKdS5UhBOhzNOTq54w/u3fPeiyEG5PMz81t0NX/vyBc+/lHnbjTNy2qPZs9ljwQR0EpgFSl3jNU9SrUgWxiGTNyYv9sVCSdsNnJ1kLveZw1xZdKKWxFi8Q+ZW2S8LF4s1r9vsMieniUEq47DldY+fIrqgS2XcDKRZGMeJky3sbo9stnD/3sKNRzM3bg5cXAqXsyCpkjeZecjcP69c1pknbp1xuc+8fOdAzpUswrKbOZeBuRyoVTicD8zzhGhmKfDK1UzdCSe3lZdfXpgPhZtPZG69Xrn1qJVeXl4sjAM88nQmn3mmxzyjmwqbwiB77p2P/Pwv73nfjR3f/0czTz73GP/VL3+VX/jsOTflhEVG5rSwqwNXhws2mx1vfeoGP/CDj1HLzF/7W5/kmy8tnAwguw2HeW4H5f1u14N3IAz5l8St02QZzqskmKLCrOYNqFXb8aO1VqZJGXJhzJFAhZ8tbZm5iuUHMFSGSXh8l9ig7KuQ0wadrzi7ueVDH3sj73nbk9z72ivcuvFN/uWPCe9/wx6q1Z7XQ+WVVyq/9IWJb97bWDtJFUrODCyIJG7cOOHNTz/J7dOB+/OBb1/uuXGj8lM/NPKXfmTguZuFenngsddveHSXGTaJMmV0OZCWhV//WuKXv5q4U2agokWaUmgdb2psynA52wl1Cbp7ELz+vHYBtgICx8qvC16N56y9EP5ad9drE7VxLn0TqybBTKGaJDVrRrIfumKu+ZQSkq2FqIgdcZpcccd59dEZsdYaRy0YqGjhALfGpMfG49hlwA9X8s+iXanE53HXf3iRpIvrCIkE7ZpNb/KxGVndKg4gFXT0ubvVFEc7A+1wJcX4J0hMo2qfH9o9PgitmZD95CMgEImYemze9vE3xeNrWMwqifna544t9DbeFc+EkhA1Glb1hlrq3Y7UmM8zH9rzjhN041pZuCulp/5cdV3f/dTrcoNOu8gxDZ6NNQqAvCZsrzXo+8fOJPEckbU3Loa50scBdntFpPPOOtFvDcCqh3K0z9NCTCs6Bz2jx0Vr/CLtMxqhNAcCKTxqHssXr9y47tVaJ1K23lxrfr3mPbT9lBFHGNoAfqx98yXSciK8t0tsmYU9v/FNqFcj6aawcAl6ynwxoSlRRuE0D2xOJj54OvCBW7e4PBxYlnNnqpE8WN4Xg5gnQCqqM1Irsgy+EB7asgwSksuXs03y8uzFwNCcSRlkqNQxcygjS5koV4WTuqFmYRIoyQ7+KiUhqTIkGLVyepqYNXG4N1HuVXY3MuOYSFcL41x43eM78mbg/GrP/YuZW7c2lEVYFjtWeJbK6dnAQZWpKOONgXoonO0Gzm5teOnFA9shs0nCfFmYy8B+qdw5WGjjNFUO92F/tef1z+x4/TMDNx9RtGy4d28hDcLtJ89IpzMMW6qcUyWRh8RAZbl/g7tf2PPdT5zwg09l7qWF//PPfZO/9vOXnN04QVlAM+Ms7POBJx475QPv3fDh953y/Lfv8PO/eIeX7mbOTjK1HpBpgCoMsnn1ln6N68HBgDO71BAfut7rgB07uRT7KbUwZsiSURkoFaapIlsY0sCQBa2FqpUhCUVHYGZIFR0Tj54mbpyMXFxmJFV2t0758A+8gz/wrue4981vcXLjt/mz7yu88+lMqQdS3lEnZTmvfPaLhZ//Elwsie0wAAtpnKlaOB0H3v6GR3j6kRsc7n6bO/e/xeO37vOnv+uUP/sDA2+4NVHLQB4GhlRggLoUmCDVgfkq86Vv7XnxrjWjEEksIlTvoHRkta1pFxve3bB41jayijtqj7UfuTRXt4wM/roS3C1ngJAHPo71Aq0MjhBIdj97s8WuU3dxpsEUdxybnHMPTBqQUDcY7bs1ucRfWWm2uYemHMPTQBtrKDbjpwAJYYeaizUdC9GVQu9T6xaryx+vl3cwhB9G42cPNAeBA4HwILTb+Pet3YHSvAc9OaPRe+2mb2GNZHNP7TAl6W5iCUAdytUtuTCyNJLpTKhHDoTxUNxjZSFLV0pJvbIBsYxprMsn+AFWi4eonAYllEgoNadJJNah6sdlO2nTCuC28a9ov+J3ywiK5KU1QMAqXoL2amdIBACwj2tL4os1tQQp7bzTTPz+PYm1WwGt2Ce9BbhGQ0TnHVP6SdMRT4G0Ey+1RiVC92gcha1YAQYvFU3JDKLWSjtARgoe6PuXAGUxqtjQrwUE6GAlZIk4npN2Jye0o7BIoo2QiAKn4ym//KWZz3wx8YH3j47OZjbjGcthYjlMzOOWrFDSDJsNuzGzlJlh3qGzUg+WOCabBBubm47OXAc1zx2xd62CyORgJZWB0yGbFrpy31teYJMYZOBKhf1iyYCXl4XxNHPj0RPyRi1OPi2cDspjr9tZ4vfeThc87BemC2V7Y2QYhOUKlmlE6oIeJrYiPHY2QBp46c7Myy9P6CZx62Rge2vDCy9fcetk5MaNxHKReOLslFfuHjgUePz2yHJIlDpzeVAuinDIld0NoU6ZZV95+pkdzzwzMg6Z5cXK+dUVcpa4+eRAOvHWz9MlKW1Io6KHA8u9xNULB6uCODnlYn9BqZXD/cJuPGE3JOq0cK5X3L4x8NbnRj703hPe9EziM5/7Nr/8y8p+Stw8OVCWDcgp0zIzpIXK8ir+ea3r93BqoVtwkVBUFdS6V2kxxLLMlXkRFu0xbsSTxjSzlIU0VdIoDCkzJFNaKQtD9dgQBgZ2m4poYa8LT7/+Fh/7vnfzg+94M/nOfW697sv80HPnvPHRrSW8DVsr7ZPC898U/s6nlM/dH7j1qDAdZnSxiownHhl41zPwhz94wZuHf8YsL/Ovvu8+u+/a8obXZR7dLrAoMi1IhrKdEU2ksqVQqQlePl/4/DcKr1xmFj/SraHyMGWrWw1d3bkbsMdszS3vmyaEcGzyNRAQacomjEn7imeDR9VCCAAXls1tGfI9JF/LYu73x+vHTeF6vnZOTXmTaLkCIt2dHMfNtlwCV1bRTEbE75MHS0xza6lpIsIKs4EmVievRdik0VV6SVczMe1K7noOXW7TlE6rSjvXoVQ5AhLdo0ITviGTI3MfwvMTXF0Dbq3us1pvcSAw+FiitbIYTaDSXESteyTNCkT9cBQH3Wi0krWT00KpRMZ8rHPjIQDJpGR9JaIcNiB8A6bEd7Qp1/BgB+gyoy9jfXMrSXPgvT5X7fF9SQGSvC1YNILy+cnawg61GnP3G0qQhq4IzYrvAXVji9T7SHRc0AYmjR6hjABRa3Wd3OOicuRJUvuIs6i2pFFt69X5lr7krE+DjD0iUZUhvQNplQ5OrGDFf8/J/jZUYp3k1jL8GKWsH+0hHae6mEyoqqgjnloKFLwJmXqukbCl8pV78Fd+aeY/eP2OzetgmTNMCRmVPG85P4dUtmxEGW8uZIGN7tCTjFzN1KWyFEEmtSPmMRjaCZNg8ZLhLA4O1Q5LO1hoJu0hLQa+x5TRLKSqyFjYbQa2pyccZuV8f2DIytZDlVfzzLTANmcO+5nzlyuPnI7s0oblhjVFKofCvTsTF5dCkgGpmdNbyRooqTJn4ZsXcHF35o3PjTxya8OgSgbu3J0Z08gr9/bcOT9w8/aOSRfuHQpZM1fznrkkbp5ldJmo04ZnnjrhqScS9f7M/anYmQy34eZjmTRUS7CfFnI6QYfFjlB+OXF5TzifC5oz7Ct7HRhTYr85JydruCenI+94/AZvedcl73n7Dl0WfvEXD/zqr++RvGXMBS1b5nTJUEeGNKJ19ITn3/36PeQMZC8ZCbS8RvHidZ3Wzan3kY/vmgCo1bwH87wgQ5yyZMyRZEHT6N0GhSELJ6cDb3/29fzwD72VD7/lGbaHl3n89f+MD7zunJvbHUUnhsFyVpFEuZv49G/u+YdfUJ55+ine/Gjma996kUfP9nzkLVs+/BbhXU8tPPnEN7mVXkBuAWc7JGeUucWUZYaaBThBFtD5YAK9bvnWK4kvvpCZl4FB7QhmA99iRxOrl8h5EXDEMqMfAhhChlD0q1i4C49exSarWKC0+uUmNEUsvl+jfROuACM50z5d2+chStvM4nYX4gAi2YRj9JBP7uqUeK/HW5sVI4LGgUFozyZPXYhHbX0/d70xVKdF0/ndrWuHsfgxoX6f9n1XmuJflBWfxX2astRCJQ4uCgUYuRtGzGsO1QYIDEhUWgtajbm7dyzASgMXPZteoj2S06uBmva05BUI2gGLDde+L6Zgh9Y0yeGHhrXcbd+1YogXxBXsgv1ULM+vioeG2re6oGiRDlnfiKPLOZgUACJQwJpmK/q0XJa1wlf/njfFaomdR8++/twOIs1VLwYCI0/ltb/RQXcQWAIYBUigreOrTnernbciw7+xR8Nx3csCPbxlVcXeNC11IJ1XRkLkT0R1Q0ruxalmUbePOTg+OpSoAc81Sggu9Lbc3uujFqXM1nekFmWpVqa4ORv4kx9KvP9thTtMPHk5c2e5wVYvSWrgfDcql+XAVBP6SuF0gDFV8kZJg9rhb0XhIHBZqTNuBGxg4yGp4t7VZJ7iOA69lEo5CHUSdCnMh8JUEnlMbMZKVWGTtzzy+MjwyJavf0u5vDdzdS8x7UGyUHXmpa/PnIzAMnN5z06FvfVERgeYz6FcCts6Ml8cuNoX8uaM3cY6H8psJYpbTTw2jGyl8PgjA3fvLqRFmNPMC3dnXvfEGac3My/f2ZPyhk2dmWa4+YglNs6Hgdc/mXj0dmU5X5iWER2V4dbC9pb1yan7BRkywimLzkwHOL+nTFNir5U5jw7U76ObzGEamGfY3hKefRKeekp44zMzjzw+cnlv5p9+pvC5L1yxTQODG9GLFjacMKc9UoVlXeX1u1wPflBREaSIZ4/HBhNYsPPmF6XMiWUJRG4nKg0qNlAW9yLk1tQkD352eSgRmSGNbFQZd1s+8JHbPPuu9/Gup0du6W/zjse/xtO7u4yjsMiCJDtMRBiQCb79Ivzd3yy8lDb88fc9yofeIGz2d3j25oY3vy7x+KPKbjeYFVuWlu1MKauYuiezIVajv6gdbzUmahG+/kLlG3fhioRQ3BK2DmM1ZTQtCItbq5405cokUZE0omTQ4vXIQMokKiUXkg6teYSkwe6fzNErasl6oRBMKAnVM6CrSyrVsEi8v4AWF0brZCa3VvC1yOIhHdMkaXBSJNhouLUtL6Noac+2BD8HA8WTqyQ316XgpYFunUXnx9DL1oZAXXBKAzSmOARNmZQNFNjKdNVd//+s/VmsbUt2nol9IyJms9buTnf77G727JNMiiySaihKgrqyCirLsqpgAzL8UOUSUIZdNmC44CrDDwb06ie3TwZcgFGdxJJAVRkW1YAlJXsySSYzmd3NvN3p9tnNauaMZvhhRMy19s20dQ1wXZx79ll7rTljxowZ4x9j/OMfzZOuk2G96j2L+iLNt9aFqS3tnld+hQVWZDknYDfFVYEpY12Zn7u4gxb+aeduAKWJMYlY9zRVAXGGzLPgw+H6ly+KHEBo87CrcljKxc7vDp3JaJ7lQjORQ7RcrIWpqqNoan639XHAmNvmeTe5IFvaFhGoJX5i8FG1lYfVkkYpSG2niwh4j6iVhRWUhjBtCus9quuhvZ8VW89qqoSLw3KkN/BBvswCpu9M2RHYa7OpDXAeAjxmkGvoukZzWl2+qzH1OyWV6hYunCIUVxaNBkSPeohQn2epwLJ11XR3xmbptYPxP5TL2jtZDkjWLxjU5qZoobjmDNgAXT2XSIvYBQu7C4ia+JQSK2jxy/NIgZjSAY+mRB8C/+6fFf72vyac3nckhRjBTRO3oaPbJDP43vZpO1Zgu6eWHxeGAH0HXeeQvqCujiV6mEyi13mtICigMaOpIMXjCZQukaNjkkSkEBWmmDlZBXyXOT3vCW4gsWfMyusPB5474e23Z7Y75dUHPS+frfFTRjbK2K2JmqEkdNcxz4U0KfM+2TMRA0UzL57P3DsdcLnQR8fr9wOXu0hXYCU9KUb2txPbvVKS8srDgU9/auCdy8TNBlzYk6Ny/6SnRCHlxEvnIy+5Dr2diQRkpYQTwQ8Bn0Bmjx89cc7s95G4cyaKlAuzFCYnpCKIZlwBnwJXeeKlB8KPPhx54+EtZyeCy57Hbxe+8jXhK2/vWcvAgCcFR/EzQx6IFES7ugYzmv+YIwOttakui7baigxEe9RKypQsiLpFrAVq3Xp1f5rHmnNCnMc5Z5uyQO+V4ASJhbOXT/nh19/g4XDNJ8MT3jx9zKmPhiwRPBEnSsFusG6U734j8c2bzM/+4Mhf+MQtn394i5PC4HtOTgr9WMVWkiMnQ8hBHD5jDNhkohkSClL1szVZ4bEg3N4mvv5u4vmNYlK5DQRgxr0ozjsyvtaEm0a0CgQZTapSZ+MhiLecqo8UDWZAspI040PAZSuvccF0Gmo4oHpWd5kJztm4Gxu65SnBQoPhyABJZbgfavQdoRlpoYKESnqrm2WuLYx98yqXvKlYPrGGHZ0LQFnkZ5046+FdoxvHnSebXW2kusUlbHa5/eDMzRKlsreProUWBYHFK/o++VU+8NbiqFaPa/mI1u/XCk2p57Yx362HFzn8fPAKOTJqVIBSzYbD1lQ1jNpCPqU2nVoIp7VEtF5nMWQIyKIBUKoWhM33sXd494IXL/XYaMqxB7mc8vtOVPtkS5Z88Hft3O1+Ci0gVgEYZvyb0Faz0qVGBo4FkpZTa7tHR9774sl/z8mbc47x48rhl3V6D20wauptCVcejt74EY1AyDKu4zk7/HyI+NhxW+arAZXlXN6qCtpHtaYutI59SfMcTfvhVtXnNB/221a14L1bpKGlnnhJJTYkVJ2D1KppML6MpgQFur7nL3+u4+LejhnHWJTNqKxdRDbKFGdS8Xjp8KGrnIPmVHhygu2kTFJqR1roOk9wjuILJSc0OvzcnCCHSEA9FhFK2TT9B61rrIfJmi71Jz0Xj3qGoUdIbDaOKWW6LnDvbGR6GPjG7TVvvbtntxkIyaPbmfEkMq4Dp+PAZgt5Ui76nvNHI1ebLTdpZhgELZG8B9f3wA63z5zqCNEzvZiYbzPztef2VnnzI2d89vNw+kB452lkN0HKhTfeWHO2cvT7zGkf6DVhJPiOHDJ+CMbdcUI4BR1n8o1jvnZMmqEUYincpMLsFOmN4JuTkrVnzEIpa8YHGz4Sdtx3BcrA5ePIuy8cbz/JOC2c9OCLEENEJOPoCWHCJ0hOiCok+WOuJsgloiJVt+ZQGyOtyQye5l+IUNskNFeHithrTrnuLNZ5z+QqOzGFQPqIdoV+LTya3+MHxplPn2wYXUbxqKQaovN1M3e4yfHsWebbl8Iv/NDAT38+8ur526wlgR8YR2G9FpwvkIQyQUym3O0caCykjRiaDko4F4La+6WAeJOBfXw58/vfhRfbUDcW2wGleFNQ9NnG5gVJltP3pVCkgqXq9dn2kFFxqO9wmpmd4FxH3zY0LSAZxN9p0yofAAJAzWc603qov21FHvjmwbQIQAMBdVPzB6lhnD8SwTkAvhbKX853/EPzXMTKNq2Jjx52NVVEanmda+mSejiFVkGwlJkpR+kJO2nDCq0xyxJ5vmP45WhQd99d8r7tz/HcLeNgSQk349R6yLdPfh+YwZ0hNG/9aFO2yFEFM9UoHojeB+Ki9a8woGBCQC2yYV0Ky3Idx4CqWZKDcbpDZDwy/DaRIMW81ZIbCfWDU/YBIyiwVAcs55I7F96MIfWjyxyLGPGuzkeuY1s0HfQACOxeyZExO7o/9XqXnw/TdwAPDX5oSyXZmioqRtRbroWqhXEwsCy/OkCPu8DvDmJaDHAz0newbMOPcjDiyAEQHPJgUtd+i5Qp5GZsDeD7yj2ylJlFLcoHztVaFy8z03goKrXgSCzKUMOxgqOUzH4buXpPkdcu6NfvQnfKSRiJ3cS9vrC5XrPZFaZYEMl0waJhxtuwKgHnrTPqnIQZ8GJN5ZrmQMjOylRzVVz1DbjbwLVA1xlwzE457wW/hygTly8KFyeKXwWk6025dnLcXO2Yt5FH65FNznzt7Q1BA5+4L6zWI6jn+nJmfWI39fGLDcNqZJ8sPt0Fx3oQnGYSsI+Z7D0XF45t2hN3cELh4y/3hGHLS2eRl89XXG4iZwQ+97HA+f0TPvXJNbtnW7bvg9wqq7MVfvCIy/hOwWeGU094CCUo+bkn35j6oFC43Su3OTEFkOAJFfQXcWTv6SRy9aLjtRB4eJZ4NSjaOa7yDn2j55cl8St/6Ji9YxwdQQc6GU0yv2T2lbxfSib+cZcWxuwtJFVLBm1DsdyTU9u8vTi85MUDNJ+oGYa6gXjFO0dwHlULYTjnQIu1Ki7emizsnvFjJ/DJU09QKF4pEkGFoJ3lodQ8pTwltteZl+4pX3glcHHPgXrUB/rBsT5RgqubbirECDljXnw2kkneiWn8D4XVmcOXUlvyioWMI3zj3czXn3qi6xEKJeviuXq85RxLFbjB5JWLE7JPiCrr3nO6Vk5OC2cncP/EMfpMQnjrKXzrCagmRD3FB5yGxbAILKHv45d5rBbOk+oGqWaalx/wy6YWnOA7f/B460ZqkQLzZI57qJtkrnmvtlErTSrN5KR1CYc2zjpe8FoJhIuxN7LWAQA0zxpM4rXRoVk8peMN7w4T/chTVLQa8GZE7zblMbt88MSbt9fM3XEoGiySQSXALdV3dwzT4XuHYx4bZpZig6bO2ax4KQaEFiNYDbjWMWtpm/fRveWwz7foh8ohEqKaaMb52KNsY1xAXbVYoq3k02SG3aItcNfYNUPevFu7d2UZ3B3BG3vjoGux6GsYWa5Vb5Tjz2D33Uh3R8bwgLsW8KtHi6Fmw+w47gjoVYOu7ctVoOxw3Bqad9VBkbLcswYGl/XEEYuiGvE76+Toe0vKos1gXe9GUpRKEDy6h3d+Pjxry5y0ckQKpRi/QCuIdC1yUR+gY/xmFSOHc6C1tXZm4WSUJUXmTUJ3jnzpD3o+//nC8OgUr6YoSB4QgdMzGC6UzW1ke5OYpoDH42tKUzRjtHhqVYQjpcKsBVccXhwhJ0Sx7xXF19SoeOMjKR4XEmB7Zhhh1cF+33H9IiIpcdY74ixsryO+CPOtMt8oOWZSnBk7OB08jx6O3D/v2MXIk01kc2nk5Re3e1zeM3QBxdO5RPfSChS2tzMuBToVdpuJOSUuVmf0K0dMhVUJDCro5Hj63ZmHq55XXzbH8vqdmSfvKLubPfe6jpfOPCIzvkDuBvpHhXCvA2B+OrHfWPOvmDPbTeF5hNwJoRN8fTYyDu0sPZV3mfuD8idHYd15JCop73jttYGwyvzyd4Xzey9x/tIJq6Hw7OmenPZ0o2Omg41aRR9K6D8Y1/v+rw9PICyKd5bH9NnV9iilhpc9JWdTI/RirSoXrH1wLrx3eLGwdd95SgbNGdTyzCoJnzuEwqvrxHrdozpZ3jgaGxUNUAyBiWQ0Q9oKPgU+8UpiXBdiUoJPBN9xOjqCNyCgEUpSotaHoxRSdsS9QKxNKpzio+Jz3bCcIFLYbOD3vqO8e+0sUqAK1QtWD0EFic5EVjyUkuiksO5GHjyAj35E+KFPB37g455PPFRePrdmFx6HTpnf/p3I3/3FmS+/P6DeUdyekAKSQm2vXI3gB8LgqgrOPAzbnBStYWlfc74tlO689WMQ71BL9tq99TXfWSMg9cCL5yXZjBCV1JSP2KkLQYwFF9VfmLW/q4Z+PG6zlM2IO+eWfL7t7Y1AaOusAZ1jQNDO07zpxeCXph0AS8h3+QPfZzhLePdohAcv5vulHj5wLQeDUR1p0cpFKIvhL3Lw2BfPsxiYLtq6ut316Au5psYqGBGLpoEukbXD+Nu4ZXlPjo2mvYtz5rkZHbhRAfWghlidS2T5awE9Ddi30sfFiy66HKelCZrUcQORB2lc7tyLZQlV7f9mYAznVYMoSnFikYfjksMaaWx9Dtq9ygtgqWNyskQpW5ndUtpX/98IphX6HAx9Q65y8PwV0z1wy/Nizx5LlEaWe91igUcnwwl80F9rhOBGPqY+BzbHLRXXmqW1+6I1EtMm0jgopRiHJ6dkvV3UODYpZxCHBOH/+jt7Vo8i//2/sSKsshFlQ6YUh/OF0FkX2dWJcPs8MW8COXkjuYrRdU2vSREyIqU+7wXNjjk7JCuuZEKpNgBH8A4fhLAG6aHvA3MRrq4npn1c5NRjEdI+WSfDWQmaOV87xjDy+NnM1ZXw2snI2WnH5TbzYjuTc+LJk0LnlVde6jg7HdlfR1ZdR/LCWQedg2892aP7wtm6Zxczfeh4dG9gPQaezIkpJS5Oe87vB262E3GKrB8MfOf9RJoi+1y4vYW8z9x7bSD4BHOivz/AqyvkwYzOM/FFomzBFbjaFZ5dF/YpMXvHOvQ4VVKKZomDrf0uKa54ThnZ7rfMU0a7AWTG+8jjCA8/8uP8uY/e5/Q0czsl3ntvh8Yt3XDCs+mKJ+/sOF8PvPbojAfn6w9uWd/39aHBwPWl42RdKVxOkFwQNyMloLlDVAliB3RSUPVkA8l4ZyGk4D3BK0Gs4U1wRjJMyYhIng7XORwzvgsogblEQgaXRorLIBlxqZLRPDILfptZ+Wy6kNoziCC+sFpBPxjsL7MzsgegGKkrpkKZhTRZbi6pkqrqlvSOEGpZkDree1r4/beF60nwroY6K3EuSELF15BkJIhycdbxuTeVL/7AzJ/6bM8PfKzn5QfGxKUYgKJAjgXJHf5TntcfRb78RNHZ8vaFDnXJ6p+PwqmLMas7jHcemse4iN0YmUlLsZIf0YrIbYPy3tMU1KxlI9UTkmoETOO7lEITCm4bfGmNXmib6rKd2UbbasGFg6Jhc51om//iui3NaJruvn3XPCNxGBg88qva5qdqHueSfy4sIehSIxe51L4EbdPU46lr6ZFC22EPUyvVO67HpUUEjkPbx+Dh6OfmXh4bjloZsbAtDpe/GFttbioHKL3M80Jgawf/fmj/yPhyHB0womeTzW6/98cGqVZvHF9KwZ5TdzxprV+BHEOyw7WbQy81msQy96Uc9YaQNocVkNb7KXbS5d4cpuMoOlZPZ82EzDA6NWB/LNzTIgsN11jkqgEHq2xaFJloY+Bo/R3m36IKchgTh5bE7Xm5OwM1YXoUOWg4tg6rnqsCqnpiaeeXeqUtWuZYgMudu13MEbCW6vVOtrmtkU0tNYKpHCIEdY7+aJv5j/8RPLpw/JmfXXPvIhG6GYjgrCueD9CfwLkIcYR5q+Q9aLECQo+zx6fUxmo4Co6shewdLmdIhZxtzrVYQ7gghS4rffb40TGMyioKXRjwvquyzUqaE7dXGwJr4gT7NCNFuDjp0JeE66vE5XXk2VWiXwVc8GwyfOJh4NWHPVe3kc0tXM2Je+eBcei4fJFI0dF1PS/mGemU87MOyZnr6w1d8Dy48DifGDrHsxewvx3Y7DMqhT54PvbmCZfbmW9+pSBzID0XVg/W6EMPp3ur3LhO5NvMfidcXsKz68ItSupNZGjez1Z2fRLIQ0KkECZPmBxxSmy3e0rnkSGDV1ZjfZ5Wn+bHPvcJbuIl1/srvCt84hNn3Fs9ACdc7VasPnuPNz/6Cq88XBGYvmftfL/XhwYDX/2K497Dzso+Bji9EM5OAqtaKiLeCGYBY8na5mbslyBC6I3w0ndWwqYp4UKHk45SZmIprL0jp4L3HeIL6makOPw8kiUagnKKHyrZLY7oTknzhPqAEFHJZITV0LEaBE0Wgs05mRJhDaFla2GHpoLO3nJfZFIG9kI3WN49IDAL33q/8NZlIEuHVzEQ521TyOLQ7Bj6wMuvO77wucQv/GjHz3wu8JFHM33nQJ3JfqqRE8tk5ZjoTIzK19+PvPccIOECaBZ7iL1xG5oBPXg79dmv3ltRXcR5GinP5HyrJKpzC1fDOasckOqFFXQhE5qtOXjCpXl+sHxWtRgbm/rZ1sClbm6VLVe9JZPihWMH++A1ilQdfG2BB6ksbBbVvmWHrpt4KUou1bhpNQxNQXABAw24HP40b/ywsTb9h4N3uJyrTfNSW67L+Fs64lgPwv7tjr7avFjqvJuBVDkAKeRQStmU/LTOqdb7dAwImgJfE+jx3ycqsMzwESC4cznaPPTjyzy+7g+CnA+ae6DOpbRbX68z693PtChCM/zt74MqtBzl/eVobR9523fO28yYoKWSZL93qHC8Zji6txzEeLRWNLFEGQ7DPuS177yxfGA5vNZUUE0lLmmzxpOq7alV6lqgPQMVQLS0Xrt2EWv126JHWh/u3J6/oyvRSjDUw7/b1TZ+wPLsqmnww6FWquTMaSfcRPjf/L0t/3EK/PWfCezPJ9ZDAHpcN1NmheIJnac786xWhTgXSky4ucNFIEqtEvNoMYAZxJGyldiaZLmVOMa9kqdkXQo3wurCs7rwlD5yctoTYyHOmXmf6P0K4ohuZ7IU9ikzTaCzcjp4uiDkMDNNIC6yWnvLt6fI/QejXb9k7j/qyMUc0a8/veX2Gt54aaAfhMsbxzgWskvs95l7JyNjD2mv3E6O6+vMdrfl3tBz72zk/gOr/OjFtBtuHjhuN4l3nyfefP0M7RI+O9xNZnvVcfPU8/jpzHs3M8kr3dpRZofrDPA5p/jOGX8sFytrnwrsO0K3x4eeXKxxXCeeFz6yP73HtT7jZrok5YILM85lpIucjJ6Pvfwybzz8CEkmnl59kyeXT/nTP82/8vWhwcCXv/wCXa1InbBer3np5cAPf3rkI/cTTiKemieSuhCFJdTnvWcItmCDmLedKGhuPAPPpGbUpVvbw5UVUU+XHbqfiVpIKlau4UCyUKbMblvYJGEvnuKNqLbqlfXKKhpKyqSSKbmQ1FiwcVZSVtu9ZoXkyVmItZwnxsI0OUKwhlzTXvnmu4knG6uFlFzwCrF6sqch8MrLyk//QOLP/6TwU58OPFxXL9wPJBfRlPFF8MVRUkKLENWz33e8eK7849+f+PazYACgPv3KjEZnnmt9LRQnOXgl0HoAtByxIM4fiHgu3/2dOPsDlumXGjqkbiBqRiNXby6Xg0fTdrSDAuIhMlBKJS4KgFuiE8dGarmOaviskVBZyt+Ag8aBo3o6YgZIzRCXfAQsqs3RZdy6hElL9YSa4TmQ8vRg5Jfvt/k9vk4OG7B+4P16L+weHIXmOQCYxdttBqh9WQ8eo7Rwq1TjppAlL59Z9B0WH7eGgcXUIRsRbrGfulDpbF2oMyY/5aCUrQfj0UpJF2JiM0yljlW4o2xnv9JDFUY2E2slvg0wfWDO6pHNiNcUSZEFlDR9iDtr7Oi1GMv2iwrEWkShFBPpaeRaPUr5aB2rETiPsO7R80AFBYv3LixEv4Xn0WagnrORWYvqwpHiaF4X0mqd3tLmuAEJkWrnD2vHumc2PQHj8WiuqbScl5WkR2vN5k/uLOKafaotu1vDKLvwtidT24OveuWt55mvPrMKq/JkIK+Bs6nqhjjrQiiKEhEPfRcgO9h7dJNIycjdTq3MzsrNTaFVVSnq0KymIzA1+fZAulVkyJSQmW8SJ+crYk6kWJimwlw2yGRVVlMppOLpuo7b25n3rzaszxxeld0s7Iow32aCKGdBuLnZ8/7OQv0ng2ezKbxzGXnrcsKLx++FU3V46dlH5cVmDwkymTMf6INjs91TZmFwwiv3O0qaSXvTFog7uH+6ZvgMvPdEcV3AXWSCE8oN7N9Vnjyb+e6TzPVGiR7ymJmmwmrocA5WY0fXC5oTXoU8O+KUkVlx9EgIkIS+L/h1ZO9WXK3e4KaMnMzPOZfELJ63pEc142Jhte4pfuZ33/sDLl88Y9rvcKHjw7w+NBh4cj3ze99NzGPHyapn+NaG9emKl0Yz9iuB4B29qjH+cctG69ThqbXkcy2TIph3qIlOhG5ObEPPuRNgB7qiZI9kU7maVHBDpht6k7iMyrxRNptMUsG5gulgJrpVoA9aPXHIxZHVUSLM+8I81zrqGGBf8MX4AtbYRSgk3OwYe4+sHO9eZf7wXcfVzuMkVfG/jt7Bqy8nfubz8Oe+IPzsZ+DBKZaTd1ZG6FTwSa3ksoYZUxLmGeas7DaF3/v6xK/8vnK5NwTdaSBLpmhEXbAHuM5lXiyRWO8HbfX37mCkjjoLWs28tRlGjOiImlw0YDkcsY3ZUuw1JVFFQZY+CwqHTmv180sOfBlS3aFqGLvlzY9cRq1AwggGZYkKWLTCLdK3WjdELboQpAwIVK+26QQceZ+lhUVLqWIrlTVfKdjmuR0iAW1OG+nNNknbtBcDuFxUyxVYWqb5c9KAQONAurr7y8ErdHIcgpfF6JlKosWtTaq4hXGrIXNYJAFnKbHSjFcBSgt8V47iwQAv4KhaNqeVRU6x1vPKEh0pi4dblvujGeOhVGBRai774CQXA9o0j5W2AhcvtbTvYtGkQjZghkV2TLtBQRr7QJZI1cGU2w96sKM0vQx3pGS5PFnSDHa1xEfVNQsOU3smLDdv4Xzn7ZdNS2O5R7QIiK21WiVpPA8xj1elGXEbi6sgzr4u9VE4BoP1stTSPjOOHg9EsgqhZJIHTf4AbloErEWiluMrFEt5CmWp3CjqrAy8VLBfqwtyznbf1N5XH+mko+sKX/xkoHu5Q68i023Cz0JYBVwP9FrRjemQiFdzIILCCrw3ZT8SlE0mz4W4UyNkV22aPEOKQi7OutG6TLdyjA8dhILeZPJsfIWcwLvOIk4DXD01/ZVh5XBBOV117EohJwH19KN1S8wR1qOtwmcvrAx9OzvmlHn78Y4nt5GH93q2Sfm1b21wGT796IzX7huJ72oDTy+33D/vOB8c+zlzejqioeP5TcHtlXuPPPNc6HzP+p5jlQcePezMcXSO+XLi5v3Ii/cy37qa2cy2P7zwCQ2e+31gNTiGkPCoASfnTYMnRvK24HNA/UzZOcroORk9fZ/5xi5yo4GP5Pc4ye/TaccLN/PlyxMGWVHOz9js4CtX3+Vme8MgPevQW0nzh3h9aDCQTgbuh4nxXIxJmhJ9sNLCLkVwIxQIeIqYKqAhWtCkJGpBYmmbhXmSqJr4jnpj+UfwwSoNECFPhTIproMhdHg18Yq4y+y2ypyq4cgFnFUkDD6YASmZgkNViFGJ+8JuayQa7xSdHG4GV2YUJXshFYdPQqqCIy47vvvezDefF3IWvATUK/fOCj/6ceUvf9HxCz+gvHxPkVBtZ6wCHzjT2nbgi0lWpihMkzAXmDaer31j4u9/SfnKe0LMJsNZxFotSw5mg1zr3XZEREKR2suwEfA48iTr7oFKDQ3WiABQ3QaqjTvy4hALKaqSc65eSQ19Nt/UGUWokadE2iiOWNhq/9Lax+KDVRBa7ZmI4KV5xRaZcNU4aCnH2YFDuL/+rmQl5UIueTHoqN1z1FJDWlMKdv5KbKobd+tb0KoiWlXGghKWMLH9e/G+G/DhkI+3/blyNOR4JuwLzcNv39ej7y4fO0p5IMW8Zan3WOBAG2uA7GDkmmfdrN1SNWHi/9TC1pr7L6CWSnOwsO6XiBMVcKqRRl2pfId66upUL69SDW4DF63gTqrnm44b5sgh1QKNGteiJHLw9vUw33rEG21NhmhzWddje68Bk8MstZD+0YCb4mfl2CxNFpuHjRzugx1hcbqV5olr5cUs31gwx2Fsh7THQerg4NELMMmMK309T8LlDnUDolurLMgNcFKrJ9rYj85lg63e/xGgq/F6X0kROdvzWCohV4GQV5QYuTcIn37FUcKG7qSg0pGvZ5gLpXe4DiNFS009+oIJBoAEqX5IQZKjkw7JBVJhdwvzXq2BHMnAiPQ4Cr5LhJOeYQh0J47OZ/Y3W/Ls2e4KUxTby9cd0plnrwIyKyUKTj1Dp+AGrq+2nAzC+RDYzIXnO9Ox6ULkRZkRLdxE2BYHe8d2iuwnxfcBWQsnp4F9csRp5mYb8ZMSXc+8cxR2dGNhfdERNOPp8J1nOHNs9jNlH9A0wzazip7nl1uevQhc7vdsvHClkeQTnRcuQuCiE1aS6aRjdIGx87jenpMtSva2tjJK9oVHJ8rqvPA4edzNxJvdN3g4BYpPnF0E3rlWnl0GXr//kMdlj3u+R4Pwyr2HpnOj6cAn+Ve8PjQY+KGPdfy5hyOnJ4k0J1IZ+cg9a92rMlq+I5sGfYeJz6RibP+sSszGKZAC1PdyriFiJ3QSoBTSXvEr2wA1K2WCEgXfQ+8CTEqeCnFS4gwpWzjZ1Xxs3wVrd1ysZC9nYdoXpgnmSYmpeWOCJoFsZVYzSqpei49QnCLi2W8d33i78PZ1Rz90vHQ288oD4ac/6fizP1z4/CuFk7XaPEzBiFopk0uxblQ9FsnI1sp5mh376NnvEn/wR1v+/n9b+H//vuf924J4I7JlV6zKotg8tv20eWrN6JRs7TWbUv4hVVA3jKaLvhifg9ff7M+iKFm9qZYiOAYCx+6aAQHbjRaZ1bpZumasjxLS5UgN8PilIktDH6lKiAeRnAoY5WBIjCdQbFOr+uptnpurrQqHNsU151p38AYEljw2ugCIOrl3UhpSBUOWhkIsQ1lMbzNOrgEtEUQLuhjGxt+opqEat6aCh1gJXKFdt23ivg5rkXf4HlGhZRa5Y/oaEFCWqEkD3tTogMeZoVCxzX0hnlXDu3jWZYlgGPBwVDmLemY7Z2me8BGuMSDglvy5iBytgbp+neJwS9BIFytu17TceWFR96uw0O5VA6NtflpUoEWxOACc5VXH2XQGnGBlXYY0lrVwrHlx+NdhzuuTYkmbFnanyX4fAFPjJ4ous3kYRwWso8KkE9GZsI/LiULAuYRmV8HqB5bAEUg+oFStz74uERY0UhlFALV7aVkAGC5CyfTeM+8hpR7vZ4Z1oKTIfqPkXUR39ZHKFr1xQeh8QIZCcL4GPmxj197TnQfoIKaJ7ZSZJiV48/hVlGEMdGeBmcKLFztObgZCgnmb2EZHzp4HJ56SEvuriXFw7HeZeaOshoKsCsyeNBXwytANvLjdcb2f2SXYxsIwOjofrHdCKszFs9PC7TbSe8+jhwP3VoHTzvHepWkjvLTyDPScnI6Mo2MzTnR9R6eOvSr7rITsOBFlfxV58s6GtOmZ5j1xMlXby11BfWTuPbEkuiCsx57T3nNv6AmhMA6e4cQxjMKwFkKvZFfo1570kkecp8ue1HWsOsW5DZoCZzLQy57J95z1geG+8vhZYfBrNii6u+b1e/c5P+1RZ2qPWgTkjzlN8EMfX/H6RcF1GYbOkKEriCoxJSuva4z1WtZCJY3klNEuUMSMNrmWg6mY9KaAcxb0nDaJLgRcJ8hs4aWc1QR1VNHZIgVEh+ZKBKobu3gYRod3uYaTPdME+21hnh1TEvDKGBw+WpkkRZgJ7ClECp0rNWQulCQ8vcm8dznz8n3hT7ya+YmPFD7/CnzsInO2Bsnm6RdXTAehOFJ2EByjd3QqOBIpKWkeud0nttvEt79V+C9+ZeIf/E7g2a2JebiiKN5Cf7UUC5dxxS3enu0jhx2vZEHCEfmqgQF3qGM2Ul7lP2tZNqyFgV+N0HFPibvVC7b9FVrN+bFX3HgGzZPSxcO28KUcjlMNBIBIwVkc3DyLYsa/RTiokYD2/VSBQM6ZkowTkI8ByxGhcQEwdd7QfEcXgbZebCB3hJYW49Fq6j7gWbYSRIWq1ihLjwdgKdk0Z65FAw7HbyCo3Zd615aXc0dz1LzUlvCGGumy6xQptZXEcZXDgTyZi2LJLw4RBy3LsQ6Wi8Wrbi+tVSPt+hdNgDpHZYliuOVQIEvaoR3z+Ho+GFWwqNJxCFNNsKvdS8HSFdi1tjebzG/z6HFNXlmW8UkFHS1icDcSUyMD1PVFAyOyjGMBE9qOUZYhtrep51/C/w3EHV1kg1Wms1AOOh4IHcKcEj7APOd6PRmRQMwzXr3tRbkpQOqd+7REs9pAvEUPjM9v8uhiLEnjVDgh5mSM/uCYusxQHE+uI//HX1b+ty8rJ68VXNngTgd6FfIUSbPtc5qFNFk0bk+iw9MHwXWC66y5kQTAC/0awque/kS4fRaZtpkS1T67Fvozz7rzTHFmt8nGAetWgCNgsrubDbzY7OkGuFh7snjWK2WbCu+/mNlNhdAr21y4TpldFrKH8cJxvu7pKqeraGDaZ85GYZetbn/dK3NyPNklisz0Ar3vWY8j1/vIXDwPVgMqzkR7tNBTuHq84dk7wjgOPL9WNs+3vPxGx9R5vvvujttN4eV7gXUv3Ou9Gf7e4ZzSraG7EFb3A35V6MZMWGf8oPiucNo5fO10KhjosvUzcl8sUqTzwJTAh0KSwE+9Casw8y82wqsv3ePibGROkSGZaVdXEM+Hen1oMPCRl4RuVubcId7Rk8nRWI65JBO5KIUQqvvfSoYy5GSLvKAt3WnCOtTclYAjIb4jbjNz7xnVkbaFuCtkD75g6oFR0SSULMSYSUVRpxQvjJ3gO9vEcob9pGxuC2kvxGTe/riClQTyVklpJuOYiycWyB34qlfvRJj3hc1G+eTrHZ//pPDZlyIPVhCCgi9knDFpi50/tbwqJq3cuYQUE9fIk7C5TWz2ka+9NfP3/3nkl37H8fgm4N20GPxCsV4PGXCFUnkPh43HjIPC0ebSuoQ1o3Rc29wCt4cwZ6vdttC7EfI4zqc377J5FNo81YPRN48ejvamw/E5bJDNUKvqYUOlkQ4LTvwR+ICcMw5/BAQs+pRLqZEBtU582Tqf2eZoJ22e/3LdxTbgtoG2hjjLG1KN+QcMlN4xQo7jxlIcfd4qJTBj3+aNdo+qdyzN82zfOfytNIOjd85/+ELjJLSbf5RekEP4eqlEr6V7VhJqYKzQ+BCVvKcOVQfFynO1AY66ao7vj3OH46OmJkeWKvFrfBLnjyTGmscqd1NGqoc5ENudlyiMk8MHtd2XhjmatF9dpw7sfC010OaxHVerK65HhajCnTmzlMSB66I1UlAnueHEBSgc1reNd8GLejzzR8/BUSqg/a61DFYOKb0ikLKy8soXPuL5jW9MpOKJRckuoqWjaWgs0R0jKyDlALKOeQi0cbfUWzDinmRjmLQbI5X8EPBIFk7vnfL1zcTzPdzzHTkGvM6EYazCZ1b67bwDB2UWYspMe5Bdwit4qoBOByE4QnD4fsXJaWY1Dsw75foycbuJbLd7xvU5/WCgnh7602A9Bm7tOd/Fmd2UccGx2WR8LvSD592rmfcv92Q6IsJ+vycMnu7BijglVpK5OPWcnjg6L4hYr45574hzwU+KxxxCnSB45WI0qfgETPNsQCp4vnK1Z95k3lgNnL08cLvJXL3Y8mjV8+jeQImB9UuBVz95xvvXe15yKx5Gz/0+cW/wjGcdSgJ1ZCfoGrozQdZW8r469fTngu8KGhzeBRCh5ALqmPcJoeDKSM4ZCRnnBkYXSUHoI/zIqdC91vP05g3Ozno8M5kAYwIxe1K+R9Hi+78+NBgYT2Y0Kj4Ppm9NxJeBRCYJ+ISlCtRYzoKFrb1YuFbnbOQqs0LVg2vlPUJCCGJKVPO24KMwbwvTrMha0eSNPzCbNxyzMufMlAp00PeOvg84hJSFGB3bTWa3zagGCon1ynG28sjOMe0TKVmveCM7mQRL1GZIA2lWei/8+GsjF30hBAvZu+RI4omSKa11qDGJUCzkVLJA8kBvuajbzM1t5jvvKL/0q4H/6vcKT64KHdH8mBzIrjYASqVuwMY1yA0F0MLJUollR1am7bZHG4hU1bsisjSr0CUtUL1s1QMYaOkIWlbRjl5qTv1QG7+YMZoRaFvSIZV9iAg4kcVQH5jp9bilkIvUcGs16oUlumDEQGP55loloBUIlNw8zgWBHLm4eiSFW0FAA6h6SHUAy+ZpuVm587fxGCoIcNRSzQq7jvO39bSKhd+pHqMu1qPNZh1bi8I0I8RR6ma5p1QDdmSr9HAUsHvXIuj2aJUK7DiQ+BovpH2vPnuSDzdsgVD1u64ORl0reTSD2Qid2sbbSuGWrpJmTVtTn6aG6SwmT7XOy/O2lAuqIK1qpqVsFsDE0fo79voP96/9vUShSjsXdyM/DSwuHr8uc2uRjgOgvvs6pIzqjauRLzv+cQVDw25WtXGUIKjfL7ngnHB1W/i5Tzn+J399zf/8/7Th/RfmXJWi1nzGNKNBtaZT2jNtoMK0Qup6qOmBtmYNCNk+4Xwhp2jcpUAtUU2EvfKJjwn/9l8947/zk4XXzieWdAsO6WfEO3wApkKOaj1nisfhyZ11qp1iIUXQreIy9Aidc7jeGp51K0/XCfceBrpBef4skfaw65TdvjBvhb1GvFdy8ex2M0PoUAe7TebZ00hwPaf3xEjWXWA99mSNjGFAgmNXPFGUdee4dzFwes8TXK6CXsJ+r+x3gsxCmiN7FVzvjdiujpeHka4Tnl3NeJTruYA4Viewnya2e2EVOm7CzPp0xdlJz812w+sfP6E7UebLLZ/+1DnPns68/e3I/iTzsPOsxkDJhWmXzGVbCW72jOvAuu9xXsk5kufCHK2tc4mZGD0zhXHMDGFGgsd7q3orBcreUabEFPd89+Y+F+EB91cryInZ3ZKHNVrE1ssBmv//fH1oMBD6QJJCnoEZcvAMCJRCzo5S0TjusLl6LBSmqtbz2lePNANZG7gGp0QRwFj++9tMkkyZhAmlG53liGIy4hgVDBQlCXS9o18JXXCUjC2orbLfFWKG4jKrNVxceAYVbm+NSFjUcvxRE1l0MWB9gT4JxSnBKSuxfGnKJi+Mt/SC1YbXBzUpooUkBjC6mqub9zsurz23t4V3nxb+X789849+S3h83ePdDLmQS6jbj6LFSIM5GKoLWmv2OSJIVWuw7HvFNrSWXyxtI6TVtLe8cfPSKxCojO5mNMly8HxarHtxwerNXdygyj0QPQoRNKMsd8bXSISHUO7BWxUtkKiVEUdkyLahVa+oNM83W0VBSYWGAWxTdsv5S1MJooGJcmdMrYTLVXnaO5oBR3ag5fuXVMBRJEFQXF3Ph3MtJr0CMll0BexqjQTYuBtUkyomFAHHhqkCzOMmRO1nLXK4Fcu1N6DNEv2w1tYWtWu3tfEBRBVX+2DoEupv4OX4Nh/u73FFvojHU0hiEQTnWzVeJddp1aITC+nbdx0U4ws0MKUt0S4A/mCc69tFa0fABuKOfnfMrF/eboCgUQwWoFDvcf20qkVKbN3Vude6d1WDrMvhKzCugLW0m32MFyroaIhL62L7oKaDai1FVAdz4ec+Bz/4iSu++KnCf/klpQ+FmD0xR+MatTbPDf61sUpLk9SqhvbIq9CeHlHzMgUTMUvJuuQ5dRTgbPD83b91wc/8aQiyg5iJMdF1HlUjcUsQgjhc8Lgpo3sl5oQvdl6nSigYIToK+7lwO2dyUqb9Hoqj88rFRc+qD+xTsu6uQyGEQFgFLp/P7J8nxlHo1x0324jLhdtbYT9li4KEwq5EztYdF9IRJSFDz9BDCVamvZ0F7WA896xOA50LdOpIWuhXjn7tGGdlNzmKwoqAT4rMM66HbZzwouySsNknXj4ZORsdDzrH6ZmwmTK3yeNWcHk1sdtaVcrV84mzcIp38PX3rvnDbysv3Sv8UHE8OvVMycolRwZKL+QMV1Nmvk0QlGlKpNlS6sWBDA7xmdMzx8l6hYRI0cJ+67nZRHI2AT+Nkae7ge3mkgdn/4yz23uonDD3SsodsaxIuqbIhzPzH16OWDzdibCbEvNOkZNAdmoiPaVDS6q2xlUJX+MHuLqXlFmtnwBYGV8yIQznnOU0nCFbhzBNiSkLGh2pK7hs6YFS2cgZC/snBRc8fR8IPlJyIk6OOTo2t5m4sz3SD8rZReBkgPlSublMzBN0g6HvmC0VIVVMpyuKy1aG5bSQKWRnLYVdExcpVjUjVGlfVcARcYQOehH8LNxcZ65uCleXyr/88swvfbnw1guPL8UEkKQRBJPNVTVUUMO8jkV9z3TpLYQk2Dxr1hrSPtqbinkKre7fSpwO+fJSjIfRZHsXUfwW5m5goJ7TLV70YVfTuuFZ1OBIYa5u7kt6gMqMrWDAuTbSVgZlUYtWiuqaAZBDlKGUyocvBgJKLsv16LLxlg9swixRgAVFKa3qb3lP63ybF+8WI7MIN9UOkK6mRZbySmp5mzaQps0Ws/AkKmeiATKrIrDzNqPmGsO+Xq/ZL3vDyg0bga6hhBrMbwa65KO78oF7g6LZQFLhcH/BBKd0YaexTEyLTJiIFZZSwoCN04MRRxR11hzLeal5zqP1Uc/X9KcWQ7lwTioA8VIBbh1bnRjXcv0cpQ84AmR1tS/2vt7vxdizTNdyw82Ld0d2vNjYKpA6zhY03LpERVQoTeeBRsw7pEgEh7QcP7k2b2qpKjuhdS01EbCpwL0Q+fxnR9wU+Ys/Av/gSwktJtmexeEb0NQDDDvEGWztlmTlgiJSNQvs866o6VVIIkdrT9yHwHmvPDjvWJ/Bz35G+ZnPCZL3aFREOrRT0+sNCbnpKXMxKfoKCEqfLC2KVboY7Ufx3sTleu+ZJJOkELKw3WZu9pFJneXOvef+w47OFy4vJ3J2aFJygt3lTJc9np6bXNjFZM9ADyUotzGh0dIIvoOXzs44ueeRAZTMvbom+8HTj84k4RFcKRalGJVuznTJQOmpgiRlt4UpC7N65m7iajMT8Ky1EKeIv1iRvHJ9O9EVR9aZb19Gnt84zlXIxZO0cH29AzrOzyPn54GzlUejKQmmPDJJhqnQebjqBe8Tvsv0q45h9AxjzzgWpEuEzrHyQtr23O6UaTuxjZlJMhIcySXyJWQVPtFFNH6H7vYdXDciDBSndKf3+O7+Ad+6eciHeX1oMEDM0GWGE2F+4U397cz070NJzMW8+liZ+J13uGKhkRAsZzJNFsLPxcL7WmpKIRWyz3jtCVJIGdLkyLPiVSmTMoWMdtX9LVZeIioEL9YZcO+YO0Pt00aZb0wNa1wrZxcd90+Bbeb6Gdze1HLGktmpEtXq2SVXY4QnaU19YIQFqWIpc6bW7lcD6iwft9TCu8A6KIHM7a3n8lq5fp75ta/CL/0mfPVt4dGF47UHPY+f7Xl8JUtNsNZSRKUQkkk8J58JybyqEqxnQnaOXmzDRq31pR3AL15u2yW1epwt3L5kaWpVRwsL3yEM0rywA4GvsfMtHWJksuZFOrQqADYS4iFH2kyniRo1F7XuumopH8RKaxrLXLIeEdGsIkS1AQFLJZW2yaI1DdHkaOUABmqIVWkchRoeF1dr0I9IZeZKGxBYQAB4X4GAWCTASSNtVaJjdWr1KJAitTSQumHSZkTaeOrnqpy1VKa/laKYSqUe1/0X6xvRAEYzmOa9KkUzjexWaiRFVcla2cToolLZ5mnpolnnpQGkJacPFUn75U6KQifmDRWBLEontgfYF/KdCJIFHmo6q6ZXqERgV8FAEUy/vp4uC4tgmUdI1HtNu2xZjL6AVZwsHIsPEOyWa1ruMA0R2qMry3pplr99xlWDbNwPW0wKaK16spLNmh6QjKKELGRRiitG/iti1S4FioOQC6UEXJe53hd+6hXHx1/vyFcTX/yBgY892vFH758SmMlAKXfnvuiBe9EiUKKCxoLrgvXDK4Jm20MyypyFdVC++KmOz7yZ+NTLA5/6WOLlVz2fPHXEYcdKsXbLBbrsUafgFQ2K7hXdWwVPQdDi0SjkCYrmquFhkuO+zzhvnQhLdmiE89mR85oYIYo5VddbhZ01ZOqHkTAE3EpYnwSGznH1bIfQcTqC6zqGHpwPPL3eUsSTvMMNjvWpsDq16I7rPH4wSRMVT9aIC4rvR+LeQK2zZosMo4n5pCkhpeB6j6r1t5GSePnhihThepO4CB39ypN9omhhO2W2LwK7udD31lvmnSfXKHAxjLx6T7jXBU5Wjtv9zOVVYXeTuTjxnK0zQwaHR1Ji1QcuXvacvbYisyFfBzqENAspeq42iTnfENWTg6eEjCbH7kaIU6ZTjyejroAOlOIIpTDMG05OekZ5wn/9lff4w3deg7/Bv/L1ocFAqS0nfacMfWZ7mxE8q3OrKgiuQ1NmjtF6EXhvaBJ76HrnmHMm7jOpblJFBU0JcYXihW1KeGeiGGnK5KkQnKOLJhhke4zic6ZTYRQhFss7iQtcIEwps7vKyLajWwlnDxxn98CpcPMCnj+ZydEWzpQzsTjL8kuNYihIUTQdSr5K7fyXciPSVI8F2/iKlKXueNXBSXCkrfLsqvD0ReI3v5b4L3678DvvCHMWfvpTgb/6hRVf/nrm//6lPc/3kZ6enAtJ1dQDVei8MA4wdpnTUzOS6bbneSrs5kAXZxjrJtWMeY0KLAT0GqI2SdKmyldoegLmwB0xvj8AChquqI5etWd2jKbXLsv3Dt83clP7/YEP0HKahzOU2pa1Gma1QGwTgBEsrGZiQtZ0pUUhWiSgLOeEFuGgNC+3vY5CzNj5XG3nbF5/5QT4+ncLfbdSQKkCNS00rQcW+oHIdQSClnk8WCahcSaat1usukAs3SBH96D9XerckxuQqox92r3T5T4JBgxUhdzUFxeiRMNGdbxH0aAWGICWAjnM0XKnqttsGYpaVocZAa2pFD0CiC1l0yICrt7jpcxRDuurleghte5/MfbWSjzLMbCsPKM6p3esP8fzYUc4gDf7gqrc+dn0/SuQaHNRB+cR68EiYA2nZoqOJqFOopRsJXMqRtRyM0LBq8NpwJrkmgOj2cibpVh6Lk/Kj77pOVlHpidw/5WBL3zyhq+8LbhQcLm3yGBdW+3ZqkEKMMxG75y1ly8Y8MyFmBJ01gK57OHP/4Tnf/G3z/n465f0gyP0mRJ7yLW6JGeaQFKL7Il0iJ+RQaA48mQ57RwdeYI8CbMWSgTN5mFTwBeHFAPwORe77z5TEswz7FJhWAknZz1eM7sZXrwodKkwrnr2u0JJVUZdPPvtjNOO4rZ4r6zHgbB2+HVmdZLohoJ6T78acYOQRWw/nwpBnKWpViMqwjztcF1HcB15zswVAPbOkXKiaOZkdJwNPS+uEvOUuXc2Mgbl+Tayycr5wxWPLjxProTnVzuevO/YPAXtMmevFGSeWZ/07ObMu+8X3n0aeXjqeePMcf800zmP9p7T+457b6xYf9ThzwpMjqtvFNJNh+utyiY7hxsTXV+YsuP2NrC7zaQ50mer4Mg+kbUwqqdHGHvH6rTDi+cXf+2Gv/dPIj/w6Lt8mNeHTxOAGW9J+FHRjbC5VYrLDCtLGBaMyWyegLdNr1TUXDfbnApzVkOTLVQpiiTHtpZKCVAmJe0znRf6ZB3tSJZPDs5BMdQ7zYVJM9JB8p6yzaSbRI9wfq/n/L4nhMj+mfD8vcL22pArzrz8ORVyzQmX3Nw7LI/uoIiwzxbtaIjZqdSSnbpJCSCB0AvjkMjZcbtRnjxL/M7XlL//W/Abbym75Di/EH74056f+1zk1Am/+PvK7c5Cl94Jp4Pj4aPAgwfw0kXi1fuO1++d8sbLisaZ66cdv/lHt/zSbxV2O0eSmY5+2QibdG+rjbMfLbKA5hpF0GX/1Dvb7Pe763XHPQYD9VyugaUWhm5jKHpkDGustVnnxX12R+cwb1dFqlhVAwI1054PyoKLkWnz3ioVGhBo6ZB2uiYy0+5W00ZoIKDadvGHfggtLeCkVhM0SoS0egipDqW5v8cgqHzg3wZC7HpahGLpumfDPUoNyQJsbL5LJSPa5Ltj75cDWKhm2PzXNo7l+N9LHqqrYVEcrqeumgCH295GbfNl6bzWL8KELR3Z1VLV76ks0aryV9MjYFGfVgN/vMKankO7F7QkABWsVLAlBypUI6LWQMMdoLeY/hpJaIDWPleBV72HRaFVLdptqmDCgxIt4qeDpQQ1oUQQNVKvGHD1KktVjBdHLoVUpvpMOKhRrVwcRTJZBRczP/7mwOgLG2ca9X/yhzv+01/ZWyrwMAPfAzC1LlrnlZxmM4A502kyLhO2DyeEoYd/84sDn/9YQqNQdsBWkUnImmp5b1fX0HL5qHo0ZANzwdVCBj3qNp4J0RNTtmqfqORo16lJSNFSkWmvTFNmTopfOdbnPauzDtdBSh6fMxejEveRJ1cz88bjpdCHzO2mMPuCS3VdBqG4zPrU49cwJ9PpCL2QXKJzHauTkTknVHriLiO9Y1yv2M57et/jtVZqdY6uV/L2IHmfRRkHi8g6nzk7CbzYbNnsYTcVnB9Yr3pSzNxczfQ5MN3Cg/MR18OLZ4m1Mx7EH7674+Y68cZLHT/26Z6X7gW6fsCzh/PAxRuO7kGBU4vgORG60VO2QugK0mXG0VO6npv9zP5G2V5bI70QbL+LkpEu0Imj18L5yhPGwDwXfuUPL/nSVwt/6XNn/MQn0vfsAd/v9aHBgMuZokIqoF7ozjpuXkS2z/c8evmErlPoLGzvOo96b6IHbVE7q20V7ygxU9qGWmQhOBUBFeuSV2Yz9EMorKKiyZis3nZp5hTZz5mINUYKChNQboQyB7r7jpOHni5AvHZcvpu4fJwgedwAuRRiFmJSirf2zIeEstSQpfVQSBQ0gOvUGLlgZWPFmMHqC12A0Xs6hWlTeO9d4Vf/KPNLv134ze94bufAg4vMz35O+MJHAxcngpNEn8GVnn4tfPLjji9+Dn7ok8LrLzleOvG8vIZHZx2nRG4nzzx3/MgrHV95a+b33vZ47Q/7htacfX1o24ZYyCY+1BqbOCoQwyIE2Ta2oy24ftdyuwvLXxWtbZtpRkpk6Vi27LjVa2/s7CVvX70aIx0eGOHSvocFY9uebH+XBWA0U3jcEIg2lsX413moJkGW62jnblEA+4dUi3IcCZAasT9yma0qS6tZq66ZGfe74Kc1S2qYC3QZl0noVlZ8S5PUuXF1Ao8jM6qY5HO9pyJ+uUe6eI0VBhyXhR58++95jheCnBjgMY9Slku1YVVL643AuIAj3wiVNoetN8JCcF0IeroY1nIUXRDzKO5AzwYu2/1Z1nG9/8fCSYb9FhR7iEgdXdvy/XrcpUMQR2ux/nMZxxJCsu8fZB3smopYXtjLQHLgstlH6cwYStNq1kIRb6FzS/5ZiqfY/lVUwWVyDpx28KnXha5Y4zdK4sc+PXC+3rLdr8FlDjjuiN9Rx6auEEuhd46Sawi8ZLJ4I+gRcUHYpcjTbY9E0GRjLpPHlUzZQ0m2vrwz3odiUdvGSWq5m0Oa0YC76yA4BSf43qEjpqWSrLKgzBmZlJiVOEEicXZ+yvn9kUJEfCDHxPvv73HJ0ddeJKtzT+c6ZFbWq8LZ2DHFSOgHwokjdJkuZIoISQQ3jPgBCpFuWNPAX7camSQScybFiW7o6RTSfkakduycE3NJFO1ITlFfyFKY5kw/Gvh9uplwW8/905ESHO892XA+OjLw8P6azZwY1j0xRr779i1vvnHK1dOZ+Srx5hs9P/HD9zhl4v69gW4U9vuIPBT8mVgqJqe6NB1Z9viwYvCZuROmJGyulRdXhd3WIjDOGRh0yTH0hWH0dIMndEpyiXefR7759pbHVwM//1nH6Wkkfh/Rt+/3+vCcATGDmSWQNdMPynDasbmc2Ca4f+pwAYSAOkcqmRQt/19E8EXx3jH03pQAc825FaVEmErG9RaujxnSLOxnyF3mNCbmSVnlgVSUHYlbTexyIYknJcvV6Vhwe6Hznv6+x69twd8+Fp6+U9hvYOyN1b2flSlXFeN6fc47XBHTx8aMqIqiXq0SgupRFGfNeyy2S9cFhlEZPExbx/uPE7/xVfhHvwu/9pZjn4SXH8Cf/uGBf/2HIp96JRJKz5OnsNl5Lh4qP/Xjnr/y0x1f/Khw1kMJmd47zkPAucyLG+EmezwTbhA0BGJRVq2k0elhc1Tz8szzdShH5V1gbOKi1g9Jsd2t7ZeLsdHD5nr8qhutYItz2bgLZrAbO7wcjOCyXS9Mr7bBtQgDh026fu5wLRVYyNH8Uw0WR/yIlrYuVNZkOXi87uizS1SgMvVFDmWDLU0gBwDRbM/BU23/04PR0aP0ywIEjqIqNfpxh9ApzZhZznuJurQoBxyiHYshKzXUXjUptFmGclTC2P7UePuhUGCZdyowbLoWBsZqHKbd85Y6Eb0bLWnRlPr7JmbTQHTTQwCLZLSmOi0Er3K4h4txc0drpK2N+p3cQFK7vJYekrJ8XaGKVXH3uLU6YqkSqRGnhdeih68sS7OmFcw3d+AVIeMQPIGXzhXNhRc3kOdoZXpacCVbZz5xFUDU09WGaKq13TLKHJWPnTsePhTIheCtRPGlB8JrZ5Gv3gTUpaOUTmtHXCuFDCWieIpkxGstHVZS8ZydwmdfG/nmk4ntTcZ1DkZP2YDrIO+F4mbUBcwtU3zw1iMgm7y3qJVQ23zWCoXiTOcl2xzaz2oS8widFIIX055xgnbK0CmrE0V1ICnst3vOzweCd7z7+Jonjz0Xp56HL40EX5hLIe0dpMhrr67ZpUx5EVl1hTAIrgvWOrmDexcdq9Oe7GYQTywmXtSvRlBLJ/su0IWePgzkkpn3E0om5WKOrVi6Q4HQmxqt9QmAOBdk6EiqbKZIyNCNBXcirLzJKmvOvP/4ijEETvrAPEcuXyQejQM/+SP32ZXEH/3+xI9/ZuDsvJC8Z+UFyZgTVqDMhbSz1EqKgmbH9fXEzQaub+09EaPvOC9QPN3gOD0NDKfK7HbcRM/772aeXs6U7PnIQ8j9nustDN9nG/9+rw8PBoLlgrrOoZrJJXF274SJNaUXSmjEqw4E5sk0kQtCcY4cDUX2vWPoPKUk20CTNbDY50TnHOKVNMPtNrOPICOknEjRM0dhM83caCQOdtxShBSNwBd9NibpWhjvKU4Kuyvl8buZ6xdqzV68UoowR2WfzWNGTdrUO4dmiGpptCyKeAvheLU0iah199KScU7pR8fYCSsRShSePC786h/BP/gq/Pp3HFMR3ngF/tQPFn7+B+HNB4FV6HjvSeR335145bWRn/kJ4c98oeeTjyK9h1kDPhsJM86w20eu90J0Bcnwm9+ZefdpRnQFJcMi/NI2wqpSV+O9robKm2fdmPfNMC8qcXVDlCUp2zy6gwFeIgDNY86V1FR0+V1Z8vlyx0tvxzq8b+ORFpeuL3fkSaoNEBavfDkSLVpBLT0zUaDqqTs52uyPLHqLDhz/aZ7w0RjvAKHSiFuyBEQW3QQ46pCoRxGXI9ezCgc14NLs9vE9a+DrTmSgyGJjTaGx2oHFZTwW7bn7xDenrvUUaEtjmdijOdFlott1trmpQEAO4KkBgcYid0t6QO+eow3p+N/LOA6AcFlpDTRqA0+Hz7Uoi7Z1Wyx1kJvNF4EKlOToZCqYd3s8rmZg7wyyXfNhXdoRolVMiKIJLnr4m1/oefkMfuUbkX/xDeXJJtD5gCsJ1DNpRFzCZY9SS/uSnUBr2ilOyqtvCOsL0BIJPiAo4zrx6VcDv/9dwVeCcsm6SHCrWvlZRUp4D0UTKr6Cfs+82/MnfuQh/96fv8f/40vfwnGfv/jnFZU9bgw4mREnzNnSACE4HM5SACiIQ9T2Qefbc5zxAn1noGCeCillcrHW7xqxyMdcO4omR84Qi7PIA0LoPCVt6UOg946bqz3zvufjr/ZcnHo2mx2bHBAtdIAEuE3K88tEYE3XC6HL0Clh7Ahrh/bKbrclOmtO5z24HvZ5xzRZM6HVekUuJjAXUzb5HbH+OaWzSrBU08NdCJSUURcofcar8sj3XD3fI6lw73yNSrSmS9PM25d7VqsTNjcbJsl04tnsDSSeXnievLvl+X5PLh3764QnM77UQelqT2eTet49dcxbQfzAN76xQSdlmiNzERg6ulEIXnG+sFoH+lVBRocbHZebyJPbws0uM01CFzq6lWNWcCWwlsJbuz/myEBOBZGA66APnjlDHxznYhoDEWPgdl0tOE4J6aytMV7YzQXvFUaHn0FyQmOmSLBQUnaQLewVY+bFbSGJ48wFy4/MMKXE5TaxC0rnPU4KSq75Xnuo01DoHnpWK9Ct8OIxPH1ciFFZrWwzn2dhPyuRQnDWVtiJgrOcTC71XtX0QbV15knXEjkv4HsDNyuvpJ3wztPMr/5h5r/5feHX3wXp4Qtvws99zvFTn3A8OlN879lNytefRYZHHf/WFzp+8uOO83WGZJ0MRSJDF8gxcLufuMmJJAMhKM9eCP/tH0Yubwzll5KtxaiawcDLsgdbB0brVGiGsukK1D1QFF+9R9ss3cGDq8bB9kc9qL5pa/bTPPW6k2qdm1bPX9+v2/nBLugBYJRGblsY98u+vNgrqzBo42peJwvYaBEDd2QImi5/vcQlNG2bnS5naimDJUoALORIORxbaV74QXN/KdUsh/OVFi2qY7Bw7iHCctBPaGH02ka63J1rrROoWqjtKmo753aBlcsgrj4D7dothVNqNYOxjBcIeIjUNPC0eO4V5EkN9YsaQ9tbKsA7Zzr+7UzSXEd3LOlf0xYfwABN8GgBSC1U3+5jXS9i19s4Lcvnj8iih3V2mKMWWWjcCeEIzDVs4Zr9bGjksM6WdJKYrLZFg2w+nZjmhzhrLHY6Fv61T2a+8EbkZz9T+Ed/oPxnX9rz1ccrkgpWRQLEgmRPdgYUndp5VBLeB7Qojy7A9x3Mewvxz0I48fzIZzv+y9/as3Yde0ptfcwSfVuec7X+Lk6gRBNEE+8pwbPyMz/+muP+Xw68/vGeswe3sAGVCdIKkcSw8uTsCE6AjCYz6Ki1HrfipDaBCtmMRVsL01SqJoitp1yrc3KCpEos9ie3qEiaQDI+Dzx9OpEn5aMvrViPjmdPZq6eZy7WwsOLkViUd57tePp4QhReeXhiwmmlo3OFMBQ0CFfbPXP2hLX1venHjoJydXkN0nNyNjDpzJQim83EvM/0LjCGjlkL29l63Ig4wgAxWwvi3IGqY/RCXzwuDeacucLVrVJ24F1gvHBcjA7XdXz3nS2vPzzBjcJm49nvI08vHZ/8zBnjBH6bLT2TTM5ZslI2gc1l4fpy5t6rJ1xNO775rVvOyxo/BvwoDBLoBIYxc/Ew0J15tmnmxW1i8zSznYSpmACUHxTvLM0uzhGz5+0Xkf/0V+Hf/V/xr3x9aDCw3YF4ZVw7/BhwycpmusGcUwOshSD2/GcUCd68cQrau6UcKXcKoTOEKUIKFubvvDBluNorl9tEPxgLdIqFTSxs58zlZGEal4FsuXxZOXAm3rObJ14dOjoRbq+UZ4+V7dbhQ0ZFiBl2u8KU6qboLVdMFmIppFJIKmgjmGElNRQx5iwZH6BbdQyj0HfK9R6+9h3hn/9+5le+mXn7KvDShfCFNz0/9abymZfhdMg4r6wCeKc8fMXzl14JPDgrhJBJtfaVAJ2HXJRd3DMnM9ZjMCP87bfhq99yJAmEAJIPnk/zrJqf3QwxRZdwcXuJo+o7HGq+tZVWmfrP4cNt8xagbpSNobb4qFpqXr+6lcfnErds8vXTd4GEfWg5xeLBH5L8B/W3Rjr9gAmU5m4vX6+GTrUuzg+gY6m5t6pz38oHj1/Ng1wY3TSN+QqojnK4rfxN22bdyhqrMcbVUlBpjYcPERBgAWg2zCM2nhzur71tMydLMOUYQh1sJrSoQAMtNSesVT+g5rldPVf7nmVNxAD2EU+gpQrU1/tU2WatWsAq7esaavyF0ix1A0/L1C9gs51cCgdg0HgX5cDBsOvICB1aPGg6AmZAzV8vGKOtA9WjtXN8/+sKauCtRkKaFoQlB6R6/B1RC4/OldNVYXJwv3P8rR8vvPko8H/7p3t+7RsdKVv7cdxoRrw+j+oO0TapugrnK8E3FUbEZIMTfPYjjj5kVDvU23x68WSxSKRm0zhFreuEUxhcQHMELyTteX5zQz5d8wOvOIq7pmxGHBNCD8zsZm9CaqNQXKxcjyqtXYoJqBVBxVnpYq7PUXJIcXR4nAv0wcoXsxq/KrtCSNZtL3aQihDnQoxCzo7VemAcRnJK+FPPqLDbRsiF115ecTJ6us4j0QiHT28SL58PBF91aaIynJj+wnaX2CdlPO1Nyngo7KcdcVKC9qzOBopEdrtMpJBKIiD0ApSJeZq53iRycZytHL52hlUg5ZngheCEslf6PjDfbHG5cLHy5JUgAV5cJ8ZVIOG5dzbw6MEp2zix6gP9GPjM5+9x78xz+UfPmLaOYd9x+nBA8EyXE9urzPWt0l0ETt4IPP76nov1ivMO/GBCeuOJ0I2m5oiDZ5eRd59FbuKEcyN+8IQx0ksG6RAnJNnj/cDX38/811/K/PPf/mNuVDTlHi0RVwJdsVzVnGekC/gOdCokakOZVIi50Lmeoo6UEsMQyAWSRNxKkNlz9XRHcYUUIDkhh8DzF5G3bxMvUubekHm2zcTiuXCF22iVA2erQJ6VXNW3vCpBQJ1nFTxr78l7ePEk8ezpjCZH6BwUxzZmNvtMzA7vHdpZWWCcnZFNUOvB1FemvEAqGZcU70wXYQjCyeDwnWNXMl/ddvzTdwf+5duendvxE5+HL7zu+KHXlEenGe9N4Lh3wsnYMfaO9ZiYYmFOjjmplRRWN1HFEetmOgSPC4EgyrNN4be/Ibx32eP6Uj0OdzCgcCe8LQ0YLNLFNR3SPN4aURCtjHt1Vn++6AGwSN0eH7spti2elsgRkGhe/eF39VDHtq3axyOWoxg7+qAqd/hjNq+SmsgVbbbQu6v+cP2vMeIPs1AJ3bXEUQRxVQzHy1JF0NDTcnosHH+QaDZgaIb1kBI5pAU4RAwWAiUsev81SlCq0WmhbZtPqT9Wk7p4wS1sX0Ge1miFg8Y1aGNraYzjELgXj2qqx6tjatGUChJKvXBHi2DoQhB0zuFcAVeqLr0J5qhrYKAJHjUhJMuLlw94r9SIwAE5Qgvrgx7SUnUu74hgaQbtsYiU9buvBUh4NXUPpd2PJix0AJWVEnLAUW1q2/LQBqDrh7xboiVOW1GsIDlzsYaxU1znmHYFnYQ/+Wk4OXX8H/5h4le/2hvgK7Lk4h22hzhRK3Wr6oungzUws3M7q1TawUdOC+e9crOrq10gaMGFwDRFOjzWEN6DFoqz57X3nmlWPnUx8W/8pTWnH5nQfUBjIoRMyZmsK1wvfOv9PbI55XOfDuR1Rnyx0kDvKL4gnTMPT+xaJDmW8KFaXj74mjoVqwDKDpOlr2uqE9gnh3hTUS3esV55+i5bVcBcePHMRJZeejTifWGz8WhUbjYzz68LGeFiGHACe1VOO6UfTGdlnwv9YKQ8VcjJ8WK7I+4irz68x8OLNbt5RlTpnaMnMKNMaSblxJQyIUDXOboOfFbEO+Y5WiS18sp0dvhsugnRQeg8ncDTm4n5utDdc2yfF85PHSqJy8tIP8DLH+04PS3s9pFnN4nb58JJlzl/5ZTunR2b2Yiaq4vE6Rs9ssroDt54MBJWida0dpLCNmW2u8zt48w+KhEPgyesHM6bk9v1HVngZh+5iY4rPL/93cDTdMIbn/hjThO4rupmYzmsohYm8sEIdFI88zYTNZoRTULpMl2wENPgHZMqmmFYd8w5MrnIlAXXCyV5bmbPsznyNGbyKEyD8tZlZHSBj95zxBQZPPgYiJopnYVRQjHZ4LFXHt5fc6LK1ZPMu29lbm+UcTBBkH0qXE+JacqIFobekYOwm7U+LAmc4iWYo0EleiCIFoKHMHjG0TF44TYL38qOPyoD8nDNT/wgvNzt+MwD5ZUT6PtaR14gdMIwCOIKc1L2kzIliFnR4il9pGhCsjW36TRz2nnGUVHfozHz1vuJ3/rGxDb2NSJTkBAW29mU4Q753irIswAFrd53tfA1vO2bqaterqUUWtWAbZi2eZbF02z29hCBaHvrEUhgwSh38r8LB0GPSX0GDsS7OtYDELDNXha1uHJIVC/RgVZmaJyB712/S5dFJ+adOxadgaWQqxqJFtZt0WnRw3Vp4ejvxvJmASINJCx/aHNOi2XfqRO30+oB7Czz1HgELb1wuI52P2wujiof6jW0rn8qteoFRZ0e5l7a2LVGJ4QW8RG5W0Fg+grH0ZejgMRxKL6OuSgHjQO1zolNiVI4gJC7rP7jCMnR71VNfXQZq6cU61HhipJK7Q0i1fNWm8smHiYC6s1gLYM+AqMtkuaO1mS7cY0YKVWyGlVOR0fw9gyEXpgmz26j/ImPCH/nrxT+7i7yW9/1dGrPOcUMrGi2NJszwI1m7p96nHTAjPgE3pP3jpcfBl6+t+HmaqSTRKSDoORt4aWh8Bf+5AP+4S8/5cVU3QBRpBf2G/jMazP/wb+z4q//WUH3M1rUukKqVYK5DHSZjPKH72RevghcvNbjhx3FJcSFSsA18p9rgNerMd97RbIgyfZxl6xMzyWHTxCSoySx5nQR3FUieYfmDtd7ulNrvBNvHPOkDOuOMARC9uxubsjFEefM9Ys952PHuILgIierkRIKqx5UHDcxsyUz9iM7lLzN4OByk+mlMGtit03M84ymZJwOevZiFWSerhL+En1nqY2sMGUrC0eElJQyK6Mk1DtOz1fcXm85mQt959lfR1zxdCWxCvD4ReTyxTWxKB95beTB6Lh+vudmr3znfeFml7gIwvlbt7z2EWV8PdCfePr7gn/kmKeZuPEUN3F903GVErNmZlFy5ehIh8lDd4rznQn7+UwU5cWsvPU885W3Ct95X7lyBSRw/7zj5YfHO/T/99eHBgN+9Dg1RaxWSnaQWc2EriO4QIozw9DjXSalRNd3+N4jGnHOs98mxEPXBc4uevRWEB/YF2E7ZTQ4urEnuEx/0nObYRsT96NHZg8Utl1mWDuGwTE6pfeK7wJnJ8q9AeYr5ZvfnHnnSULEU1C2+8KLuXA7JZwq61o5kCdhrkxw5y08E6qcMgpkqx3uOosuhFDwXWESuJwyz2Y4yVt+7F7m/HTmVAuD9+DNKwoiDF4YeiEEI6vMM0xRicWRqvUQk84i+x7RzHoUzkbLZfU5850r+Ge/4fnm+4HuVNDcoRrxddM3A85dCV3BAIE7MrBq5Skt0H5Us2WbY7WK9tfC2beNtRw28HY8bXFyjn/XNntZHPwDgDhywZeXLoZ6aShD89KPSHVqEQixntcsugpL+Z9wkC7kcB6RIzZ9vUa3/IqWojgOMR+GVg1baXnsmkFpZVYLUGAZ4+F7uojttHskrZqgPjlOavqggPlUd2FWM9p35qEBm/b8LXN5CGO0SIY4WY7Rfme3qFQAUEGCHr5oxy41qHJEplyiIRy8fQ7g5Vj9cEmpHH1++bscvnNIHS3TdgQWBMRTsK6eriiiybQoBCyO5yqTXvDOLymvpacBYmSxdr+XWMbhfIfFRoNGiwS0Hn3nYi2EgDU4E7FS4+iYtsLPfkr4H/185H//9zLXNx05WNOeJOCK4LTY2IpFpt54EAjSnlFQzcS9cnHu+ZFPwFe+lmCAXpTr2PP6+Za/89c6/nt/tePysfKL/3LP2biiOM+8L/zga57/8N9b8xf/QgePJ1sXPqLaI7Pi6ZBSYMp88mJkeCUz7/ZwE9DZSsEJgqRkf2s4zFStFqCmFdVj1Qw7V9M4giYgOojW5njaFW6eFm63Eb+Ci0cr8qSU1DHvM8OpZ0Xg8iby7PkO3c90QXn8InE29Lz8MPDNZ1uKE077QNYJIbDbK1kT3VCVNyvw3CWrqhmGnt0UmV5c2tovSpCM+oyKY+y9qXyS6bw1KprzxGafyNHRdZ7dLlGmwug9U8xITTE4B6dr4WT0vLg0LlOJyj4q2z28fLaGAW6vJ9g57t3vef9myyTC/fs9r6wKq0fC2ZtrwitCDhE3CgyOZ9+ZeOfbke31zH4amZziguB7CL3igyAdEBzFC13I4DI3UfnOi8zvvQdffhveuQzE6FifwPoi4tzE/YsLPszrwzcqGsRydTktXdCakcml4CgMK0+6FZMZdoWclJhsc9kk2E+RRGHeZ6QI3g2sV9ZmdTslVAtdJ5z0gZgKgczqJDCrMkkhT46bOcJKefXeyEnvGHymC4XOWy5r86zwztuZt76dibGwHmA3wWaXLDSDsApGWItRmfdKVMV10AVP8HIoRVND30EgOKHrAmMoyJy4TZlpFs4iPMx7RmYLWQfAmXEMrrAaPevBiJQlwy4m9rP1PTAOBeAzPR1OCuKtJWhwgX6E0Suba88//vWZX/7dwib3BEmoC2ipPdC9LB6fHP1phsg7tzip7WWy+qU5dxZurXvzwq5vaYFqzBeEVP9aTG4DNHXzUG3AQVnE6anHlvbJA7FOKsuaBgSOwMDhc9pcdCwPfSd+b2NeIhh62NU5zI11kKtGbilFW1z4eq52TVTDdbdcUCsQaCnhshg5qlAGRwaQO+x8nByrCQMHRT7XvPll4If2zCIGwl0TTKoTZB0qDsa6kSwXD//o/KWq/mm7l/UatWr1q7J4yCa9XHURlkluo2vkvbYA7DNNLe/Ar2hA0ZtWRM0fLKTLozk/ri44rKcDwLCae2+iU1CboinOWagcqSXKuZgnfwzqlqHqkj6QA36sy8fmsEXRvMOItWIEPhGrSz9bm7R6quBDPbhkYC7ulL/0Y2u+8t0t/8k/L+wRgrNGal4CkjNKwkkgeOXBqSCSMAnnAc0RzQXdwc99zvOf/XJG/MDmZsfHTgP/wd8+4W/+XE833PI3/2zPL32p0HfC7Vb40Y/Cf/jvj/zCzwnyzgYdrHudpZwMLIlA1oi6kcvnmRMP97xQdjPswBVP6U113juw9umW3rTqggoK0pG40LaQ9oW0VeZdIU1CnGG7K2x3iatNZJ4T/Up5ceXo1omxd+Rp5vy05/mLLU8fT5wMgZP1wO1WKHmmX6957/kt17eR+34k7gqr0x4cJEyVdhgc+1SIKRIjSAh4EeN7+Z5SZpx3bEtmMznWoePeaC2Wb/bGSUjRFE1vbhPb28zpEMg6M6VMJz3iPHPO9AhZlYdnA+ve6vtff6ljVwZe3CaeXyvrwfPSReDFbs+7VxlOLOrw4Fx45fURuc186nXH+tNrwplAl9HeQZfAF977VuHynVxbGcOqz/g+IL2neEhkSlFytoqInDNXW8fXXgT+4LHn6QbWp8IX3xAuToXxpOC7TNcFVCMf5vX/l86AuJrOS0Zm811nm0uxEhfnBO89MRact/TBtC90PczJsbmdGNYd0hVuryI+dJythWme6XeR0nVs9lYDqgAl03nrG7ApsEmFIJ6XB2s6NAalCx7vHHlfePe5cHsZefxeYrv39KNws5+53Ts2u4T3ymqwS94l65GQSsF1nqEprAlQM6leHH1wDB10HXSSkUnYb2EzJUoR1jiCd4grqHfk7PAUVr2yHh3jyoPAPBf2E+wn67aIV6yZVMu3epzLODcjJZBzpGPF7hr+q19X/pN/kvjuBlxwMI84n+hCT0Zp0mGL0URZmt20zY66AR/ZPvcBz44GBNouab/4vsth2bDl6MOVRFjKwYO12vgPLKV61oVlz+ED7Wzuez57OOfizddvNA9uiVwshuhwxAYyClVLoEV/6r0+KK8dGco2V40XUD4YFWgEwfq5one/13zKhqPqD0skh8P8L3Ojh7ROk+yyyMAhutMAgRMrGz20y7H8+fKZCoIKxxEKPQIc7doMmR5Paysp/MCtW67Vlo6ykAXbfLS5qzkWV+ftDqmyRQYqsDwssQ9oW6ggxSNSSBiBzpeOgcofyOHo/teKEDHGix3HLVyNti5YzqxVhqGSSOt3nBU/WXWRHK7HSWbdC0EEbZEpSRSsQY+bPeOJ59/6uZ7fezfyG18fwYOU2tbdBXKtfhpc4WyNcQYEVHpcznQe5r3wE5/oef3Rjq+/3fPmeeJ/9rfX/Bu/kHC3Eb1J/KkfXvHKwz1Pr5Qf+bjwH/2dNX/6ZxSezjg8Wjaod0gJ1OYPgJIp+KHjH3x5x3vvr/jJj8LpibJyykmYOF8pJ6PDS6ZMM3nXWUQsF6s0SGpVB7E+C1FNaXCGGDMxKrFYRVQsyslqsJbx4ggUHt5fsVp7XjyObC4zu32hD4V7JwNREttSuHd2xtXNxIubzIn3JvXuTOQpFjOE0VtbYgFi9hQKQ+fZZ3sefOfZTw7Jwq54rqaRQU7oZc9cNswxE+dCTlaOd3Njoke+AMkxBGeVb7mw6oTRe3rv6DXT9x0dVm3z/HJGovDowvPiunC7n7m9nXl4b6DrE8kVPv7GCZebmXefzbx+/5yTvmrW9BnfgXSBx2/v+NYfZLaxJ3uFkimzoikTd4lJlZlM9qDBuATPXebpTni2g9PR89lXlI/dV+6fFAaXyGGEFHEucLXffb+n+HteHxoMpJRwrbzJO0oqNR9lIfYgQkq1c6FaTrYLHdM8gTpWAfbi6PH0o1UdaLFipd1O0POe643w5HpiP0ckONR3BHHc3s7sS+A6FUaB7SzsJwHNMDmmqGw2icsXM5tdtjIYB5uNcr0V9pPig2PllFSUzZRJGVQKoYd1b50Gm0dg3beEoXP0nf0dUOIucrNVdpOwVyFWg64acGQCESkwBsfFumMYrN52M83spsKUquKhWAMcj3lfFtLPeCmoeFzfgWbeeV/40u8W/s//NPK1pyPeR7x65s5RXESSx0sy1u8SEZA7xtfytuWQ9qjWarFNKmQtFvarRmHZlJuFXIys3jF69R2q61mbH5Vld1+24KMB6ZE3eUfYqFnRo1Izabnxo9JCMzRyx4C0l+ESOfIMj5GPGV8n7igE3Nx3PY54V0vnaJUR6qRqDUhFKQVK4lgVbglvl6PJr3HnpQizYOe8457aplZUj1IZsITrOQAAV/8sPIsWJahz6uRQn7BwKdq/5HBPbcodS0GetmkTmmLfIhZEu77D4LTOUdHMcQXGkolYqgI4lAoWPQJLB0Dy/dbrYQ4UkYwkZ7Q5dcSkuJxo3TQ9oYKgVlxYUwSW46CVwbY5kQZiWJbJndeBV0LlDzlSVjqnrDula/e6KNkVXOghR4Ir6E5482HHv/Pzwv/uKvLtZz2hRS/UI+LJpbDuPCcrwauVa2tRyIITZTMpD0fhr/xY4D9/nvlf/o9H/tqf9cj1DXQrdAMPXvG8+mjPSen5j/79NX/yiyCP9wTvIQQkT+BGJBXbIz2UZM6NJ/HffD3x218T/vFbgY89UH7wJeUHXlE+f+FZ9Ypox/VNQmK2yFimtp2vfRe8swhMbxUQPhnJrktVMlwsPlWyfS5n6EfPgxNri9ypkfXO7q+YRsf11ZaER7LybBfxOXG68lzeRroOcDMZzxwLKTjUCzEXSB4pwjgECokYE2fjCkkJJdC7DomZaRvZsOGZ33Gz3xG3gk8WvZpnK70enCPNCadCipk5FU67nk6c9QnImRebCKNHJPDiGuLkOBuEHK0qJznBhcCwDtxsEqsx8N7TzO9864bH782kfc8PjonzB4HunhJ6eL6FX/+1a7779Y4SlN0e8BNJrEKuOeF4KJ2jdEKMSgqBBxQ+dj7z8NRxf62sBqlKuo4ugfhA8ZmHp3/MLYxzyqirndtCRWZN3rYU8EJOnlwSzkMqyiBK3wtxzvRdR+gc+12iHwMna0fJjt1mYhWge7AGtpxde7ZZ2RVljtB5JUaYSyb7wrN9Yn5Puc2e0UU2m8z1jRm7oI6UwUkh58j1VpmzZ9XDeSfs1THvC6LW9rPvhRA8nRc6b6WNvXeMwTMER++VIAVNhc0kbHfKbjL9g9xbJUMnJm4kdTf0zhGCRwW2U2I3Kfu9hd+Kw/JtYnNnim4FcZngO8TVpk/ieXrl+We/u+UXfy3w9Sc2Ti2KisP52SLSweM1NhSwGJm24bZ6+aLW08HMWw3j1k2tLKS3Srw7LilseVZtRsHC5s34mVcn1UPUOyBBqLnw+i+Of9TjjbgS5YqrXmtjmcvylVbG2Ih7TdTIDJd5cDQvts6BcDBo1ONizwlIbcFaZQBbasQY9B7nHF46XK07zUVIubZhLhhyT5GSIyUmy1k37/c4ItA8++MIQDW85q1Wb/aIcnFwz+2NJhjlnFskpe94/s2sNqCilTBY37vDA5EGgup5qaVjKOLbICzkbr1DdBn90f/qZZR6T1rUphrcYhwF6lpqvRqW28ihj0UNe9iPcuA1fK8egPFETp3y4z/mOB0cv/dN5dtPI2X2hN7AeKuNCC3qUtfoAfhqXcuynL/1XaBKVItvLdVtnTjxIIVV7zhfdax8BpHKjfCV4CVkifgiyCT86c/2/LUfv+b/8sueXDqKREwf3JFSZvCBYQSXFS0B8RHEEeNMVPB74W9+ceAHP5n51//igHs84dwJMBvJXyI/+tGRn/+r5/z8T0XKexEnPRoSFIf4nkWVNBWKdpAV1wt5ynzridB3HZ99aeILnx75zEsTb74mPDxP+Cmz2wi7Xri/Ni2LQ1Sq1GZuYmBwEqZ9Iu2zybyrt+ssFjHKmqAE3OSAVJvb9eQZVoNjHBz7neAlsF51xJs9N9e3vProjDxv2EfFFQF1JPE1sGSqttMuMUdnDZnE82K3Y06CnsEuJcAxF+X59Y7vvLvn+SC8euEBT0mFtYdOHJREELWUcSXxzrvqdOWCK9URnoS0L/Q5MqyFNFu64mq7pwuBk8HaMt+76Pn64z3bbWbeTTz9TiQX4eOvr3EPHb/7zVvcl4V7F4F1B092mavHjgejZw57TqRD6JDe2f4bzJFVD5MIey3sS2GVLQ21CrB2YuqQqYJg71A/450QnLf1/CFeHz5NgD1EuVhezocAInjvSCkSc8F3HS6bkK8rUFKh84HsCqoe7zybuMd3jiErJSamOdV0wcB+u+dscGQ38nxb2GwmWMFqHAlEzrue959PPN0m8uXECuXFVeJqm1mfdqzFcXuT2ZdM6JR5stzKCeAGTyAhCUZvpT2r3rEOjsHB4JQxePrO0wVnbdazMmdrrLSbhWk2rXWTyLIyGleyPTAWuAV15Aibm0LSbPKaufZqCGqRABTvrNTIuYBIrKWRBn5ur5Vf/j34e79e+NYzoQsBspC0o4gSnLPSea8IfmGpL3Jwh3108YIMsVuuvZGzSs2zHkLFB09qYefXY7Q/h336QPJq+eBmCFv4+zj8L0fjWULDWqB2hDTNc442nqOFV447M3IQ/FkMlVTg0krvjkBJ9cTvDO1oUE3GV7wQupE+rAjeJEyDHxDXo2obwjRNpBzRUkgpMs17VPeoWr63RU3uXvBRpGIxnC3oYuBmCca7aqjbN5U7pEqthty5w8xaVEgPkQJpRL4jUKctGtA+45DWIrkaYal6zoZNai8AsRlu57U1RoWUQtN8aNEHRZY22aJAbfd9IBMeYjUNLLkaVfLiSK6uq2I8ok6DgZngyDHz5svC//rfXvHgtY7f+oOJf/qbE//4VxPfftzTu0DnhYQR9IKYh1qWKIxFZVyNiqnUjpWNEOCkaivUiEKdc3GKqmfs4HQlVgefPJoTJQmZTHaOVEygSYpQcuLf/ImB3/jGzL/4ZqgttmF0gYcPB4ZpTyeKJEcRS+FkFVKye7/fz7z58orPfWEg3cz2fK88uqvpsMuJ/8GfOeMnf1yQ9xNdDmhvZbeObEAu5rrGApSEZI/2wrceJz7SFf7yXw78uR+65SMPBvyJ3fgXV3v6oefmckYHOPvYCueg4AgKpEycbIlPGfQG8I5xVQwIVS2LtFfivjDP3qIKxYR2MgM+9JSQSKLMKVFEOD1fo2VHAAYCm10h4lEpzFoQsQ64ONsnbI06GKyk8XZvcu1j35NdIOaZzc2OlIXLzcztPjHFnpPOs/Z2rTmYkF5KoDnhpND1o5Whh56QM04Lqo45Qd4pREFHR7/yIMo0bygaOD9x5CK4KKRBeP9mj9eBq13m/J7j0x874dGDjiel8Bu/E7l+r/DaeeCkE9wgPDzrGXrH7MywFAJFhEhhFiFilXC3MbHPhQSoj/iVowtVnlnBO29N+DTje0cnkU5WJJ/5MK8PX00QTMWutZH13pnnicnyxKishp4ueNKc6Trr4pWLiTZQEl6E1RDog5J2M0k9LnTEUtCYKAU8hUcnPZApWbj/oIPk2aTCEEZcFlbjRE6JHHvUF/ozJRV4b5u53VuK4t4q0CPc7uHJNhF6OB2UUT1rbxLC686x6mDVe04C9Bg/oSTLf2WFqCZUNJdShZRqqLYoQqm5OWeI2QHOPMiShOKsyUlwmdJ5soeuypsWp7gAvSp4oQQ1KeKN40t/qPyj3y689WKk88FKir3gGvIj1BgsoJ0RAbFNXmup3GK4K39PGrHrOM9boGTbII/j/ksItYbtj7BA/Uj1PStJsJXbiR4b4eZMHlnC438v7+kSZaBGF3A2Lto4OBpDAyELgbCJCx0dVswY3mlQVEO/gi5ja16zONsA1uMZq/6cruvp+5GhOyX4sXbbjGw3t2z3G2KaiWm26hgtxKKQakltu3Y9moOjKzlGBipyWDc1HdE8ZFHTg7BLlIPn37zZo8MKtaunWhmcaDE+SEMfHELlULkA1fO3+XF3RyhlKTk03aADj8HVFdA+W6oxb5GDRTJYpa4HgaPoQFsCyz8FivM1TWJleEUwEZjkEbEeuR7llXvCj3zMcfoIPv1yx5/8UcdP/WjmH/6TzL/83ZnrrSDBWxgVpQ+C7xUJlgbKWSkqeCk4UbxIJctVMOlBfEYoeBcMdNXy+vUAYy0VVpNeQLMzh8AXxAfTsnCOtJ95/cLzP/zZma89jjzbdHiveJ358U9/nNfOlMG/jwk/JcgBEJKCJFiNnnASKHvrXx9WBdHIlAT1nrSb+dGPQ9nt8bFGNSy+AyRIDrK3PcoLFEep3VZjdPxP/7un/NwXJuv+mjO5c0w3W7p+zXSTmIvQBaHvPJTI7T6TE8zbzDxb9DeqJ8fCGBwn44CfCjkKOULcF6aNY7crpCmRJ3sQhpVnWCnDqSPuM3lSUlRubydimknJ8v3rIUPveLqrJNZi3RhTyvjOpHlPVj06Fm5z4kYjIQjjGKx0e0rcbmz/dgj3Rk9BmEtCHHgVyragKePFIhoiHpJSphmn5hSOzsG+rlOfOQmBVW9zsxo75gixQBDHvQuH7uAqZjoHFyvhM58crfW8KO8+2/HW08yzJ0pOyqQZ5zp8VqYYcZhew1xMoydjWgoRJYujSKGQkR6GznN+6lmvA6uVrylu02mx5RxwOaD9zFwmfPpjThMY0cbhpENVSTHa3p1NPAGFlA2BWE2tGdZ5TlXgJZIF1oMh5edzxHnP6UnHdt6y3c7MuXBxfyR4T5HEcDrw8KU18+3Me1fKtJ8YBuXRycj1dWQ7Z05OO86GwO4ms0+J8wfKvdGzDlZikV1hd5N5sjWpjkeD0g2Fcd0ROo8LthnuZ7UOVl5xweMQkhobuDTjVz0jE7Y5Mm5qHoi0jdCZR6oOexiDbRhdCIawJVOCVg6AWnTAC3kf+I2vFX7ptzxvPa25bTrw+yPVNiPkSKWJq9p8N4fUtVxvU+mrn1ua2sDRfTObKouxbaHreq7cgtCH0O0dYNDAwZIiONrxD77r3bSAHsLFFDEw4uUAXlrZ6jEoOQ5P1/OanZM73r5qi3ooTUcfkUoXqMa0euSHoxu5azWccro+Z+zP6bqR1XjCOJwz9CeE0FFyZHey4cXVJZvtFXPcAlgL2ZgpYk2tcMd+8p2hL/yIFsVuwzquOGhjNjBRJaDrRHsq2VRrlGf5VeUDFD2a57oOjnL6d01+XaMNL7RbskSOjr6zKEHqAhZpoEytV8kCrpaoUj1VORD1WkRIqQLUUj/rClOJ9DrS4dgLBE2oM+KbUHBe2O0Tz64Tp2vrFPjauedv/SnPn/hk5Bf/ReQPv575+EtrXmwiX/42fPu9zNUW0EAXoPNWb6/0eEkEr7X6oBgg8LbwvBgQc7V9pWTldN0xDB40L02BtATilHC9YwgeMZ8Nh2PaFn7uUyO/8LmZ/+dvedQJ25R55+kL/saf+wnOTp9h8jHeLIoVCeNd4fR8wI1qgNMHRKwDbJqr1Kw45piZo7IeBBdsRq0ccDZBLrr6oESUgASFveMTL3V8+uN7QtyRkpVKyjbRYyJq718KGkxG+PrdhNPM7WR3POPI6vCpSlV5Zb32XD+Z0WuIU2KePPtdIc3CbN2eCUAXHM5HxpUjZmF7m1h5wWvGTcrZyYptTpyu4GOvr3h2veHxe4pOkErHbmfNj8Qb0M2qxL0zAZ4SGXs4PRWmuGO/U1wJzDmDChej43bKTDMkHKEIQxZcMYdMnBBTYp8ikk1MyRWhE4fMCSmFs1XHmXOcrzxSMkOo6orJUUalHxO3KVHmjo8/WrGbC9vNxLOrDOpY9Y40WY+e+6vAvbVJ2edSmEoxYnlW/j+s/WnMLWt234f91jNU1d77nc45d+zbzW5SJEVSAzWRoijJcCyZkmxJVgLEshHHUZwg0zcHSRwECOAvDpAAyYckSJAAMeREimAkkRwIkuCAkCJKlCjOZItisyf2eMdzzjvtoaqeYeXDemrvfW6T1g2gIvue875nD7Wrnv2stf7r//+vqZr1dWnyb7zdoxA8sRO6lRA6x80guC6iHUjIRnhddggP4gpeB+PF9f+CHQihHslRWpWciiEFxb7UwSlaUhsGZIvGicFE+8eZuAn44CgpI87TbXrmKQGBYdXx/L5QtfL0yYp5LlzXwPXgbd5B1/F0yLx7PzOqUmrHPFfUZy4uOvpOGIJDY6ToxLO1h+LZponqTBb4eDD4cd3B5D13ubCdlXgQvMsEtelOoYOuN3JfxkxNpBnDO2/ERM7ZyXLmvNdgWLT1pJ2x/2t0BA+dMyMP704wrcRIFCgH4Ze/Uvibv6z8+oc26wExPaxUd8S5m5iMZea5xe9XiXgOjO3MGSlNT+XY0e7VOjpHczEWyP8VJttZkJdzzf/p90dL199m5Swa9rP8ob2fEcsWUxhpkrOjT3zjCizV8BL5T0mLNGRqec3TOywkwSU5WoLZ+eVYEongI6vhgtVwSfQrYhhYDVesh2v7XehRrayGEe8Ggo/sDneoQpoTs9hwGrScXW895h3nYMjx7+29j6hF+8fzhMHZf46GPyIcP+3x2i8PPkNtlt8vl+/8BM5O5fjYY5uove5JtdHkdscnnPAFBbRtykeTo7N/O37w1nZyZ0DFqYFhSU9XEqOzxD2iJBySewJmWiPV5th/4w5+6UuZz77TwaFAFWJVfuhT8Naf7Xj5QnjrumPvHF94f+Y3vir86pcyv/JF4esfGNTaBaELxglRr6gzX3vf+DHqAlXcUXpqpE1l0zu6CMeenDpyqtQq1FkJAXxnXyQv5vzng/Bv/kHPP/564t3bnuADv/qVlyA2PdVGptqF8UAUCOtIGATVyXTm4qBAGaFmG2DknKGyeRb6YM+zq2n7npYK3iY+ehWoE86vAOhCMgLT2IMUMhNBA85Hbm93lLRBhwlVpSQrhLwXMoUuBDREynYPXrh+rWf8MPPetya63JNnKFmhzcsIg6NznqDK0AtD54g+MHTKMECsnqzC5kLpBsEf7DFSI/sHx7hTaslcPlEug822ES8kUba7xAiUGNASkCDMKVtVfnnJ83nP+FDoQ8fQJcaslOrQbBMoex/xHkN3m0Q7BpuDU9sl2qXCRqx1O8+wq5mnm0g5QM6OVDJPLi64WHvGaeRhb+jgqvd8dDvzeKvEKLz2zHP5NHKbDlxtIs9WgWcbR9ebEqK2dtaUra1cVZnEfBGqN8Kk8+CjI3QO3zmGzgzH6AQfHK5Wc8jtheImolO6sKIGs+L/JMcnTgbEWRVH1eYrIHgfmGYbeelCsF6qVlJNOKcMfUfsIvd3O5LA5VXP435i5TfcXA188MHIw7bQryOJRIw9tWZ8dKxXHYc5QRE6F7gIgSCWpU2HTFW4uIr0QdGSiZ3ZBG/3SlGD3R72icPsqc4shbez5/nOoyq4nCEpFx1cryxbi31HqcI4AWqzrQk0SLF5bvgmyVOMgY9VX+IUp42k56xvjbPJYjhPcBUnCe8C4gzaD0HpvGeelc9/zfH//pXCr77vyTiUhBmSWtUgy/+JbfOvhN9WybRiGzDmsKq5tC2ghrIE87YRA0o9QvILsmyV/LHsP4N09eM7/qtB6bdaN2f/1Y890NB+q/w4ktpa4FeOwOeSrJykkwsqIMdA9wqJTU7yOtcGTRU5IQaKnghiTui6nqHfEH2P4Ii+Y7O65GJzQxdXNlxGoessEXBNlqgVpmlm7/cIE+dMwIXb/p3X5ZRA2mc7uuEcoYLjc0SOicIyG+DcKU/k6B/YgvPZtXklNTsx9s9yoEbzkO9IAM7JfN+RSKitv6qGl1W11XlM7j6eEb5yvmcqkPaXWqGqJ2ikeOstR0Krax2iZr8rCB8eOn7y5+FP/CHPJszWc3cCs3DtPE/edIjbsYmR1550/Pj3Od7/Uc8//bryC1+En/t1+OI3C/c7h8PTe+hagLHkuq0bb6RBZJlPUuljJXhtvhOeUoRSckO3YNplXBUrJJy1A+cJfvDtwE/8wMz/9Z+AL4F9Ucq4p+KQNnBJAGpm6DxuiOArlILzwW5SgjxV8wJoa7Zm0Gr8pFqaAVKpkAOiZkiDWsLvXI/WbInYWAnRw2QDh+o6gwiHXNk/FcrnIwABAABJREFUCqLFADStrKNlq7E6srPAXFcRv3GsrtaQCu99faTqQApW5EnB0I5a8SHR0bEOgS6AqODUbIn1OrB7roTOUzvhcTsyeM9YK/NdIquQe8W7TJ1nVsERe0cJzuD+ZJ4Pmk3FVLwYwuIDqRbu9iNTrlx4Gzf8xnVlNxYoQlQFzfhgfjigUDw1O+ac8RIIDnyxc6aApMLQmfrt/jDxeKhsVisu1w7xHokR6ZQyZT78MIMob3xqTdDC1c3APs/Mh8SmW3E/TmwOjjU2injoHCEGNrYCEYG5WttQg4AXfHTNdAjUV0N6vELnzDEXMaJgX9EugHqGUCkeyv63K9NePT45MtD2z1IdNReTxolV7rnURpUXxsOeUjOxF5IkSlVccOSS0ZYJTkXoU6UUZSwwP2ZChNitmMYdqxiYK8xp5smzDUGFl88zqwvHddfz/vORcRW5voocHmbuNLH2A73LPFZlnwveWQ8uFWU1OIIPPB6Uh7FSiidPoKXwbCOEHp40BnGuFhwFQX1ont2mKz32cLWhACx/GnznXQvKsnjQNwOPBnUKrQp0xRKBEJh3mV/+uvI3P6/86ge9rWdVousoUo7CdpWF2du6g40QYCSn06YbGxhcW2nomn/7ApFraTcTjoHlWBWe7dMLB+CVYOZ+6woSeFWad1oy9j7Lv38c+D9DKkBPtslnr3G0JRbboM5bA1bJQWO1GfnrY++v7fpY8FsSglMA9U7ou0gMHdoq367rWa8vWa8vCC40S1q7rjFGUk5M80Qulf1hbwneUdanHwuufMdxSq5YFtMZYrAkV7Jg9MfkYZlPuLQOlhTraPBzdsHPMJBTFqaWMNnL6ql4/9j1XpKBxeRHl8cfX3EhA54QmmObZ7mfeuIpLPdhObeF0d9I6UYW00yZjSDbuUBwe6ooWh25yR2zwj/8QuYf/OLEn/lxTx1nvAtAsFG/tRK9oQ06FbzLfPoKPv17hT/6A8IX/7DwS1+s/PyvJ371a5X3bh1l9vTR9gCc4tX+R/McWQzIhigEQ/KpRY0TpAtfxuDf5Gw9hU4IopRiu8Nf+H2Ov/+lA1973/YyL3urSFWbIMYSD98FS4hrRZ23blDFaAWzrXPVZK3aYihUnjFDIISaM5KC8Rd0xBex1xGH1AI01DYV/D5DcITLgVIT20MlTx1dmA2JdDbVtQfWKhQJlBA4xAGvmXmfef71g81PWSulWItCyowrPYMzaHvwA7HAbntAFS4ujFc6PSgrcXRvdUyTElwkRs/9dqRKJq6VzSqSZytAY/WEYrwBzZXoQWIFp/RamXJlUuHl48TDYcf+ULjsTFpX1TG4ykELWmDTG9Kbs3FRnFh7rPPR5uyUyiDCqnfWTpiUVe9482agXwvbubJeKTp7XClst5V3X45MY2UQz+VKuY6RfhPY7yr7qXKnmduDox4Sc0mMc6AfTBibcqJoM8xqiC5q0L5UU5xFUUIUfOeQKEg0fxzr0Dt8VNRnCpVaIyl7Xuz2fPN5IN1Xfuw7t6HvOD5xMlCzDcPVCqWYtWZtPRnBU2s2uLFCdB19GJinwu6QcL6jD/bcoR/IWXh5N1Ek0PWOcTfT9R4fC74E87YuldWmY1hV6t7Yo++803PdBeph5tB7nj3p+fYuU4qn6z0XA7y4K9w9ZtYrR4ieDcLrVwEtygf3M7G3CXj7VBkTMMFq8twUI+jZUlWicwRxBsuJEVm8iHVlFOs3ObHF5JUQHDHYRm3DWapBjAji1ewsJRppxGWiOO4fA7/+tcTf+jXhV94TxmTtgJLtudVrKzGrbRJGLz/FkAU6d2cmPW2K3NF2xzdctmHGZgjYQqG0EaQqZ8HjdMjH/jwlC/qdD1RasnCCqI/kvfPH8CqZ7cgBYAlgLVifR9FjkXrKOJQmszwL9hyRgZOkTkWPich5dbycgoghXM4ZmuS8ox8GVsOavlvZyFbnj6HXe8/Qr1n1F6Q508Xh6MSpZ7wMWYKhLHH0Y4Y6r1yA8wutJwTlzJmxNnmka+vPkpty4rAgx4zs2OoRq2RPFbmdTK0WwOvHWkTHJPF4jeQ7VsW5ikTEEmRhSQAWcOPUyjmSNF+5lSekwszCPdVV3ugco1Z2h5FdMnh+CMWMtsQRtPLBbeb/8fczf+D7nvLGTaamgo8R7wsmZDLcXCRDbgZBtXDhHH/w05Hf8+nAn/j9hV//uvKLXy784pcrX35X2I6eVRcIoakzMF6GOhBxdKHiPdQq5GxweKmurV/7rDkpI4W1s2kfwRks/d1vBP6V78/85Q8nxAWcL7iSqAWCd5SScb1NKUQt2BF9g02EPBuZGqGRxBSqkZhrUbR4qGazK02hU6sAFrgpWFKABTA9gG4L8tRRi1CrcNgVyI7aG/JZxDFevMXdB9/kBiGVYkjEZs3tuy9JU2Y8KN0QqPOMSE9wkRCUgcDKCTl5xm3h7uHAdCgMqxXTWChq44SHPrC6cvCYmO8cOguXFx3bVHg8gC+Rl/cH7obCa1eBVQQZlO5KudkYD2tbHPNtYU6QM+zHwjxVojNS927KPB4SXXDskhKB6+CpWslJ0dnWh3eVrptsoB1CLEKvwlXncc6mUcbe410lIFyvHYcCcQjkh5ndQdkMgXWATiJ328RvvjeynzNXMbF54nj9WWDjN6S9crMKXF5E+gEkGLF1SjC3osgV29eLKnNS9iUjSZDO46Ij9oKPlRqUx6ny0S5zd4DdKBwOM3eHyocvZr75buH3f/ZfMGdARGwaYbIFKEFIc6IW05bmYv1pL9F6F3icWGrrYiBEuwGbXtjmxO2u4rpAKJl5rsTgkWIXmlBQV4muh1zIGS6vIs82PZTEZhVYD55BKppBZtPTIpUkJvVbdwFdK6XAuvfMU2GzDvQrR9pXolOblljNeGI8QEgF5wohWvXkteIEojMzh+BtboGnEtrFc07w0eMDBs9VQxFC8ISwJBNC9NUYnw6ceOap8tHtxAc7x/WF8Ac/E9hOmYdd5n4PDylQklKLMM+0DdUkid67o9ujQ6jNHAU42uM6KwPNH78pHMCShWM1wtJ/PMHWp9pdXwnCCscuwZHxLq8mEXp6cnvib72WPs5lOFXSegzeR5Oc9kKnlxJeHcZk7QV3Fm3Etbp5Ca4LV+CsDLZgZ+9ZW8WMOLwPdF1P3w8Mw2CscvFHz4Rpngixo+tXhMMB5z1Lw0VbtbfEd4clx4r5PCw981cq8Y9fuPagowyvsSkXQNm3qtotueHylNZKQjgSXI8cw7NzMuOj5b3OkoSz81nuqhObTd9StEa6X/wgpKlYluSvhXnnG9LTOB0fWxTHeQSyOE8a7+CtTeR//OcCaXB848PKl78R+KWvJN57WUnJAuvGObLz/MxXMv/5P5n4r//EQKkTaMVjA7e0GgOf4JBGysNXc82bre//uafwuSfwR36g8qXnhV/8svCPPi/82lfhYRT6wbH23kjQQpNyWjs0Z9Oklww52bUO3hC6WiChzJM5pwan5ntSlH/jdw/8558/8G69ZN335kxYBXUOXGjfqdomUhqcvkznzHNGiIZcufbdLcZn8JjrpybM21ys/12biRE1o76nuoqbE+W2p86OOk2sN725CI6Jcih0QTjURlr1Hr14yvNvfo2klZwLJIibyu6x2NUOATQTA4Qw0/keV1bomLl/qOweM/NecVLoYiBlZbsbubhxbG4cda4cXhQOjxN3t5mr60s6D/u7iWkfeNiPZFcZbjzd2xn/THBXDvqCBKXTgN46eJHRooRQuAiCK55UC7hKURtJP84WVKsIL3Yzjsy6X+E0ULLtg3muuFLxOZAyTFIYK3RN0z+Phe1D4f0PZ5wEswc+JHIWLq86NoOSRri9TdylxNM31txUM2S62Cidd2y8x68Cl+uBy4tItyo4v6gHYKyVVAupmiLKtC2OIp6kMGpmTol58tZE9pWXs/CFDwuf/3rlGx8I+72w2xdyEmSa+Nd/dPitN+KPHf9/tAkiWjO1FoJ4yJDmwmIxWkUJKFGFkgrFZauWffO1LhlFmJKZUXRdIBfh4TAiLiLRk3OlSsDHynzYs30p5Dc96x7Wa+t3ZRWG3nGYlI/uJ8Ypo3NGtKOKDXW4uAw8WTvyDLejcktCqzNCRjb5i6OZDgnghf0EOmXWvfXxY1AG7xiCSUl89DhvBJ7QfheiMT0RkFoNxm2TDX3n8MGSCO+sAvBtcEbOymHMdK7yfW9GPvdWppTKflYe9oXbg+fFqDyMwpwcdw+Ol7vMyy08jtavBE+pymKhWlx7by+ExtBKopRcoHpUTMLkRMnOGOAeG/wkTo6T+Wx7lmNgaITxs2BxSgRsXZyiiNEZ9Pj7E6R9KoCXFssib7RKth5f49VgbS96rOQFI286PbnEtYfhTu91hNBbslLPz2/5dKqgQi2Fcdoz9SNh6PE+EnwkhEgXe7zrWIRbqtYSC6E3/kCIhvyosowuXgYaiRoJUlvgWxIeI9y23rSKyUZPl9B0+i15qLVF/KN0D0qrpC3EOlyxv9XFrOn4UEMBxC2Ve7t2DV05Xf/TfTEOgB6dFs/AnEZoPCUzC2awcCdO1b4lDQVObZ2WyJwjChVLRIIkcu249pU/+yPK9Rs9D4+Z9++UX/u65/NfzvzMFwu/9lXHw9aj0TEfHH/9n+z4ke+75Ps/G0nJWO8agiGVuSVFUmxS38K9UKUmRZOpCG46zx/6LuV3vS380d/p+PkvJv7hr018/hsDYx5YeXMGFVdJxUhlKdmGX6oVR0uyI7jjuO80VzR4us4GpZWc+R1vRf7EDyb+xq8K15cecjbPfxb5Hzhv2nJT14AUoeZKzkt6Xs3RsNqsBvs+OEsCZpNLElv7YlZcGEE6pJoMVnYRn6upd9cdMgT0/sBuX3DJ4zxUKl6ceZnMW7zMJF1DmclzoHeVZ68PbO8n8DaquE+BeB3YvTdx+DCaY2z2RCq+8/Rdb22XUAhrz/qyp+sL2+cTd8+F/YMFxKH3vHg+QRbefFuQtyLrp4FPvRMJ68Shg7kpEDp1TFqNP5CVrgQEa0tPWLtmtfJ00eYVPOwmalZCiOCE6B2r6NCUzJ6cQM0FSUrO4KuiAVKqhOKJXSCPc5OaV7xUnqx7Pnp85FAiqy5ytYZ3txNd53jnU2vW157x0RG9QK+8/0HhxWHHW089F2tL5voQcJ0NUhqcp+AZq/FjpwyzVmpwSPAUEVZFKAR2cyElYcqwksLn3gokMpDZ7ntut4nf/Gbkcqj8/u/+FywtnKfZkmzxeDxpTKSpEtp0vFJr65mbZWh2Fe8DFEcelf5COMyF3TjR9R2X657HbcKJsL70hA7GsaAuMI+FNAu7Q4XHytB35LGwS5nrJxtcndltM34deHrjWZUV1xcd39pPdA3C6TuT+NSUyT7T9Q4ZYRy9+QYUYxKHTvDRGKqiyso7+s5xMXjWvScGcEHx3oa3+CCE6PDeMm+nYvMvveCiw/c21CnEepoe2L6w06TMqTKnwpQrXuB6qEjzLK8bqDfSMkJFnTfNvXru944XW3j+oNztle2k3B9gzMIuVfaTYzvCYVZyddQcQAuDg1KTtR/EoToRaSRFb3Pi0XyCcc+JY3IWCTg+4Dvg7vPkoLWPOQbkVwpDWcrI0xyBRuQ7hbBTEEMWk5z2vKbkEEONj/MFxC1SQjlW0e3MGtqux2xEjkGJFpwr47hjH++JoWctV/hg6NaSGEkz9slNjulwBBcIPlBLpRSbF1/ryXhouYz1DEFZLHOPn0eNSSwfu74GUrf+ibNzP+U5Yl57TXboX0EbLMCbS2GT9LXK/9gikeV/Foxr+/siS6ycUKLanmC+BI34vkA62uRlKmdTTBfY/IQeLUjHwnNQlnuxcCsCJKHUiZp66h4u8QxPMu9cBf6l7wv8qT+c+PyXMj/1a/BPfqPwzdvKz37Z89f+3sT/6L86sB7M/dQ1R1RaMtbkKCxzFxaHQtGGjo0Gn29i5fe8XfieNz0/+kOOn/2Nwt/7pQNf+Eag1og45TBDLg6J2m7L4synywDIlmcZGqk2FIHYe2opaC38hd/b8+UXM+tOkL0VJ27hhTQUyXJcNdJaNvVVzoZKqha8OMQVu98VSnGk2czdJNhekUulziDOSJlCwGVHneHuYPbCz757TZkTOidKdkj1VFdMzimOEBzT7S1hhuw9IXVsVspr3Y5DULpYbOT6JMyjsnrNMVLZ3++RtbD5/reZ3v02nVc6MfRtNXjWK/NccEO1FkFyHB6FwfdULeh65s3PruheL9x0HW5dKPXAfnaUYtLO6LzZIHtHdILOBafmKKhUindodbgZtFacOAaxVlIfofeOTpVYjeKbFErKuFzoxdN5abJHkFJYd44hKvtkmfbNZsVqHbnohe0+EHxkc+PY5cKUMlfXA8NFgJSZneMbL/bkDNNe2QyKxIHkMw9kqno2YqoLeofrATIBR1ccUzUjrSJKqQXUvGZWl55SHXOqXM7Ka87x5tOez73h2I6OGC74x/+08OG7iTf/RdsR15ob7OaZkzIeMlIw20SFMkGNQvGOlBrvzQlpVnaHTIgdmgUpDqeC5oyncLWKxODIpSBFmVNinCfC0LFRpaQdufSUyTFRcA5yqawvhE9/14rtveOQrVL6djkQnSe6QE6ZKjYk6WKIhFCZD4VDMpg09iaJ2kRH5ypVC9U7iAHXCaE3EwcbAbxIjKxH6lRsaIdAdULoIHTGdvWd4qLiwykRyFUtARhN61paT6603qRrG6kETwiCBPC+En0lRlh1I9F5Om/QVKIyIWxnGOfCfg4cDoWXj5lv3cHXX/R85T3h/Q+F/R7wkcFHJilkCYQCUnuqM5tpJ9IAnlbFLpvTAlmeEQ1fnTOgr6yRMzzh+O8qTX1xDAanw3KGRbq5/Ob0iMUrQLyVqccEoPH5ZGmeLy6D7T2OSEJ7yZMygvb5OEEVFXLK7A8PRN9zub5mCVq5JGPLe/OPn6bENI2knKmqFK3MaSbNE6Vkg+eXj6C6eDIdr+mCyL9yHZsTnpwnBVY6o9SWTJxOd7nmrl2f42sdkzFLiI5TENu1f9XC2E7EuXanBUMo1DUmeUucLMdtlTXN2bG9Z0uqjhbNyCvneHyvJTFA23CZevS8EKAWB2RyFbYH5XpVmAlocozOCowf/Ezgd73p+eO/t/LLX+75B78y83c/X/nrPzPyA9/l+bf/S854NllxVNQLWqQ1DsoxwTpKI1vQllpxEtAsaEkM3vFDrzs+9xR+72cTf+tnMj/5z2B67M3IaKq4QYmdwyVPgkaybNelnpKsUipSA+KUGJSale+7Cfw7P/6EJwPoztal4E7kWuWY0NhwIEM3pRpjHc2W4Emg1oRzEa3CPNo0QBtXHZjHiYijhkgpe8J6BYeZ//inZ37q1wv/wz/5lNeeKPPLg/EEkp13VhDxCPYZ76YHsnq6PnHd91w+ySiVb99NkIGpEsdC2XviB4X+tcjji5E3f/fvYt8l5EUhDoFQwY2VOEK6n5ldoL8wnlfxiZuLQPckIm9NPL10dNdCMotVxrlSqiVevVc6J2iyQrPvPA+HhMzgY6UTTy8eXyv7BHkyC2YViGJTDWNVJGcE83fJ6ii5UJK1bcQ7I+shSIGAFYc+VCTD/XbmfqdcXQ6oz2xnWHvIY+X9x8TjrqCSkLVw1QuPAt/8qLJWx7OrzOWlY+hgWAeGi8rmSeBiEwkY7YMeNERqV6guMhdld0gcUqUU7H+54L2Z18UeYnbMjSi/WQW2h4nXukt+3+cC3/ym8PqzT+ZH/MlNhzQ007eKNomhwxNKR8mFUBwShHEq7B4nViuHE2U+FEoRtvc2ttf1PaIVLTOdc1QcXgGNNuNmNj+DfnDErpLpbCMKDq/C4dGkMK+/3nMRPXcjzCXhgnC5cTyWwP22UFMmpUoclM1VwCe4J1uQ9kIn0DnH2gs1FbbTsgebJlOTYxNtVGYITS4U7csfI9YiiI6+98TOhk644FAxNUJtEL73gqMSFZz3xM7ZHGxp89jFrFdpBkXO212xTb0iVLO6pCK+sO4h9kKIepwcJ1qtr1dgd/C8fBz5xovCL/6m41e+ovzTd2H34M1dTYycJNUROJhb6XwWSOAIiS+BVuUswOoJOmYJFHWBgs9AAE568mOdviAGZwHiVdZ9SyKsJOfjw3UMBWhV5pIEALVpImWpqk4veHzYyVl5uW72HFXQkhkPW7zzXE43jNOecZoQ6fC+4spMzpVxnJmmmZQzuRbmZkmccmqoQHs9pak26rHlspBrbWy0nAVcaT3+s2B9TABOzn/LsUDtx/u1PPccSXHCcZoiyyyD5WnL9dEjUCPHm3sK5OeeDajacKh2P5w4tCyJwSm54OMtpJaY6bGNAsqpercRxQUV4aMH+PufL/zX/uXAPCd8sA05p8phL1xK4Lufzrz1o/BHfrDnX/oDE3/35zJ/6xf2fO+n1vyh73fkBD4an4Kmmln4LnZTsODbbpJV18aiFBdwuVJSIQbh978jvPETwtX1yP/r7yv3O3Occ822uBzse27J1IIOyGlIE0pymTo3+adz4Co//NbIunxI1kxHmwrYuC/LdZSilsxUpaaKlx4t2mSOghaH1rkliwGPM618NagnTYXVOoIqXh2yLvyNn9zzv/xbwjurwFu/I1DKDlJlmgN5NjJ0lmpW3FLpO8HNmSc3keuNY+MrDD2/dr/mIe1ZzRMdDpGMH4S88wxPHdff+xruc5+h/OrP8GyI4DukVLyaVz7ezJEkOXxR4g3Edzz+aWJeJ5COaSqkWpirIWvBBWJQRCo5mWJrsxbylBkfEiE3xFY9M5UuFxRH6ez3uUDOhhC4XHHVEZ1jvy8cJivGeu/pvbevRTV/Bp2EVRAGiTiv+BDYjTNelZoTD7vMY3KkWOE2M+8y61VH9aBBmEV57+WBOSvf+7rwOz7VkX20ws45LqPnZtMTngrZz9jW76nVISuQIBQRhk3gfpd43FVK8kjo6FTNBydAFz0+K7064ioQekMqP3fj+d1vrXjz+l+0z0B2lDkZBFkta2pEVajBvMCTklIizxWJQk1QZsURKFnY7g4MlwMheLwH5x2pJCPnCbg+MM8jRcxjOYgnlxXTQUllYpsKeVqRquNwUL56u+fbH+3pes8bTy94clV5cZh4vp3pomPoI9UVus4kuGDQfnAF542w652ABlwVUq48zBW/t57b3MNFEVa9MgyC84EuCl0P3QBdJ8TOvAdAbYxnG99Z1EiQIVS63mQhrnet0jJ+gWubVRXXZGMV5yrOuwbNNn+B1jcvxRKHnCuTqsFhIaChkuuIdx3DEHmrm3nj9cT3fDrwR75f+Edfcvy9X3R89YPanN8y+NiSO2tFyJnznMM2/iUW1EX2smygSxThRJ073+BVz/r0y/p5JYlo1eoJvT8dR+i/BakzGJWz3y0jmo9ARKu4tZ3/CQxQFutmlTPjo+P5Cmgl14lx3LLd3XP/eMt6uKEUm7wJpvCY50zKxeYUzCOH8cA8j6akwT6LLMqNY05yFiCM2XnspYssiUDLTUTb2GA5VouvICmLY2D70NIQBHsP+9M5h1Zpao5mt8sRwz6e09Er4ePXv32Qlmcez2A596OCRYEqTbG6ZBq/1YsdX9KOJXGsLdip9bgfJ+Gv/tSWP/rD17x9oxyK2br64ChaSLMlHq4rvHHd8ef+UOT3f7fnZ74IX/8w891vOG7WkPHEaG0jl80u3N6XY5A+ZmwLmqGYd78YMdpngSq8syn8xT9SSePI3/3FzMMhWC89CNlZUSTijm2U5Qrocr3U4cQxp0TQQhmEvtzi7yY02T3RqAvM03I1tdetNvPF0AYlpUzXWeU+jgVVodimRhX7u++ElDJQTYGxq9TrnpfPC//rv5242z/hP/wzlWefq+hdxdOxH8sp0QsmQdysI4OHd2JgFRKx8xxq5v0XPR/GC67jB/RVuPRCCJEUHXJITNvM9RtX7F58mctDohsCejAEyF0YG39YBbrBQ0jIJjNcQO6U7LAe+KRMWS2guhmnYusAYc72nRlWDhcK86NS9mISRQ1oqSQyvbNZBmNN1DkTxOPUEgCPEMw5jrvHiZo9Q/CsKHSu0qmnI+CSocGrtZn27HaV24MQO8d6bRMUP3qYzC46wsO+QhEurgY+mCa++dU9n33ScekCn33dcXVVud50DGuPi4HVxuFjpuYEs8OtgU6tWMjF/DVywgVY9cE4ZYc298c5epcIFao4Uxxgpn1BhU3fM9XEQw5crUwW+UmOT04gbDOtc4WclHmv+ApOinlel4LEQnCeVbfGaSBN1eYyY5pIh0PaF0SkUksmOiGIEWm89wQndNUxhI40zTzejszFRgl79dBFSp148dG+2WNW5jFy+1jYjjNlymwE1r3jcdIGAwm5GjEn1MLlGrresd0W5gx98JaMxDZsJFqWLL2yuYSrNaxXjtXg6KIjBMUHI+5RzBZ0ytXkLYq1G7y5VmRXKeqatLAYtO3aLAHM6Sy05KBWJaug2VoSwZl7YiyF4L2pF4oFgFyVqYfZV5iNqIQqpSZKrQQPm1Xm+96pfOpm4HOvTfytny/84hd7tnNFXEKCRybTFIsugWSRxsHHg9GRDHYW6F7R1Osp2BztmvXVynyJI0vwWvqs7Yks/eVm+vjKsbDlj+2DBZNfAt2CYOi5a95ZotKUCsei9xhYbchLTonxsGW7u+N2eEnKSt/1Ji1U05eXoszzzDjuORy2zGk6QsULKVE4e//WTz5mIHX5DO0zLeTAY4zV4z8eEZXlWrMgD7UNNGp8FD1d4AWufiX4n17llXvZavlX3uv493NkYLn39fyRzlQq5TzL49g+OH78Y4JYqbW0KYZna6Z40BkvGz7/9cxf/smR/+DPrwiMlKI2ewSPdI5SlG5y1JLBwWdvet78g46PbmcbhqOemNt1NGYwlDY/5fhx6pnB5lkaI4aw2fdLKK7CrLw1BP6tf1mZx5F53pCzmZ+VvpEzlzbaMatoS0uEWgOZ2RLcDP1k5jgyJ0IyIyuJ4PpToi1NRVCSWuvU2QyAkit0dt132wmqsxaSh3lOVA0ggfEwEXyACoe5MgDf+KryS9/y/L43Mj/xx3pEM252zNkzTwmRQMEcZQOJ9aojUokqSAe3k7DfZz7IUDYD6w6uo29kRiGVzNgrFzcdKo/o8zsuJII6is+ElSeuAp0T+qjoJpPW2SR1ITNlYdw75lxJZaQ4ixVU7LOIY85KRLlYWVs2FZj3gtspFy7gXSSVA70o1XnmqtQZqvOItzDXi0Hr0r5DG6eot6l/PUrXErqUCj3Kk+vI+spmLezuM0MMXF4HXjzO1KlDiyAp89GHM65zzALvPYyEVWDtK3NJrELguy4GVr3iu0gXJ57cROKVkJ3iQkWKEEZv1vC9QjQkyVC0TJoSTuF63TF0UDVTXSLnQM6enAvOe1OR52yKPPVsa+a9feXZqueKf/7xiZOBlIyRX4vBUGRPCAO1VOY5UZIgqRCGCEXYzWaAMI+2G5YkSA3UZNOVQhD2h4xIJM2w3c3spsISR1NI5KxMh0ayCoFBfBu1mhCpvHbVE1C++dHEy/1IGJTgIaqzPlVj5oZO6TrHfqrIHi5WHSF6dvd784ZGjxly5zH1Q4BnBFZ95KL5SHsn1GKKhlpaz7RUDsXYxkXVHAubhFAChpA4a7GJ0jThTZ8tDq8WiIqaoVOzlcGLEsTOZyOmf1ZXEEy5YR5PSi2JrIr3vQ3a0AnVypyNlSoO/DDxBz6XuB5WDD7x078szBrJFMSHNh/+HO5d5GSnQv50fIzwdtpXj5DsqynE2QPOXuM8UC30OAMbpKECLaFoweQIn7fwfdYm5yRs09Ofx6q/QfEt6TjaEJ9/QABsCO48z0zzgXHc4aSnqtmUOnFH4mOtiVJmSp4opi+z9zjzOF6SoeN/9QTVn18+bfPfl0Rg+dxHdUczUViIe+czIsxvaYG7MVY6Z5/7OIaY4z3TJTg3AqcTYZlc+FvdLW2PWWAEbYijypJEpeXOfez87E/jVJpMajExsutuZEMnimYFSYxF+Ot/b+RH3on8qT8e2d4nVCMeRUOlqvkQxALem+d772a+66nJm6XBUTUX40OIQ4q8stJeyTCX/KbWdh3MPt06UxnUUSfHOxfKf+VHAw8fFKbc0XvFh2Tcp9p0/y2HW4yUqoglmLUSCFQV+lFIKyFrJeTmntcb0VnVVD1US7DqrNTijAOQShtdbEPgpp0phErVI4eqlkDJgWk60MfI/rFwmGeG0uEOmZu+8N/68Z7Xv7cnPW5xGtnvsikRFAiC1szFRSAOECajW37h3ZFhiFyEDjeIDfUaAlfieKw2xdCJEm9WrJ+tub3/kLkG8+dQ6K4C3coRFrRzJZR1JftC7z3FOZu+ujKjn43rCcEjWinZZH/bg33OTS+sPUylkLNnvi2sZofzFhCjwBpHrtCJY9N3qBfmbPtBp22gXEsOQ7A9O6qZGg1Yu0CjsBZhvfF0Ubl/qG2/rRwOcDH0xHXkhSorn/nw3T15n5HoWHVwtQp89vUNz/cH3n++57ULR0mOb46ZJ13haRDCs0BYCTokanC44pBS0Ump68ZXEwEJ+Gzf8Y2HCwfiYV8C+1l4OGRyVaiGemSnpDqDH6AceJwDX7/f83v45x+fOBkoJRyHmKhWcFaVp7mQ5konKygw7TP7qVBCRLwnJ8GppyZlzjBOCR8a1LJ3xuaeM8/vZ/YFrq8ivRSD97xHsSE+L/eZQ81sglBGBSIutPG/q8Kb15eIE97LB7ITrp9EnkTlMFdqVfalIlXpIgxDAHUMncd7oVTHYW8knzELu4PyssI8ZSrKGzvHEBxeKiFA33u6PqCNYZ4xeZCPDh/BxYrE5iUdjYEszjaIaqw863OL4iiUFpWauyQrb9bK0UHw4L25n5UGkVe1Td6LWWt655s9bMWLHO1SwTTcFNPFfu4N5ce+f+ZLX4/85qMps70uA1ZO/d5GiaK2zdJjmfRpI1ULBAuZbKlaW4AxZvWZ4UxZitdlWz4PGOeJhbzy5wJTL4GR86Th7KmnGLNIJOWEBGt7v7OAukS689dVdWi1pDfnmVwTpWRzzqRZ7lYj7+ScbEBRnpvx1llycnyL9tnPSHTHn1nOu77yiV/hULRzc9Je3H0Mhm9gwBJgBVMQ6EIqbHrQI9LzCqoir9xJ4EgiPb68nlI6UzguF/Os1cIxR3jlqPUkXVSFUov9bplNn2EeKzmZ9a04j/OZ4IVv3gn/yd9/5Ed+7xVr50lFyZLM9ZSIF0P6hIqGRuwlIRIN2WrnuCQG5yjE4lB5PN3q2ucqrWfvjTQrpf2b3RjdCd/7uuO5ZmhTQdXJ0mFDVFvbxciABVMmOc3gTDLtvDPt+GSkY+MFtNZrdkiw8625UFOlZCEVszqe54JmxbuB+TBRizt+E+w75xn3lZQTKdtnn2dwfYRd4GaC/+hf6/k3/qxCzbhsQ9nGXW4T+5S5+aFsNj2uV8o283/42xN/558V/tj3Hnh6mXnj9zzj+mlHjJGSZkqp9OKQLnLxbM04H6hThwuCXsIqRtxa8N1C3CwQZyRAJx7fV9s3xbOSashx7aipUOaMjIE8Qu8KcV3ph8A0Cal48gH8ziOlkGtL/oKpKnofCBiqVEQYsemIAVNwmbzbVEHVFaapUkuklox0Bd8JtfNoEh7vC2kUNtcdHz2O5NzTe3gYtxycZ8LjNCIxoeJ4YxO4uHZ8627Ht1+AS0K/nrjUgd1u5uW+8v4HM+88q8RVB1HQkNAuWzuxqu2XUhuPrLWYozTPGCBUvAZWgxJiZuthu6scJrXirjqSMyvoEeXF/lUZ8W93fPIRxtnkO9NYcMWqpClP5GzGOFlnitj0pVxANpngHHkU9mmmrgJXXtACH36wZXUxEBKUMhMvPRfrwNAsXcckRIlUTeATw3DB4SGxfay46Khq8sBptt7rzXXPa1eB+11mPShvPF3x5MazGxMf3poEb5qErJn12uxHt4fELJlBPBHYdIJ2nlyM+U+BbSp88f3Ee+J4MnhuLuDJhcGIudZGBmtogKepCqoFf7/MnXc05tTJpnWBgzHORXVKEEsCVk4YvDkbRjE2t7glwPpjoBa0GSL5o5QtZSj4ZjhYSalymAp1Sjykygf3kd/4pmfKbbMXpcqBulj4tcqEBfk1oOoUWM8isCwPOq4zbcH2DIpuZbA6+/0yuc6iiS6vYgtfFpvls2p+qem0BTHk7GSERSq/6LXt/6Vda07JCdjrtyziFAhbD16FBV0opZJyoeRMzRMFKNKSi2plXy6JlGbbPLRYEKYlQNIq+kUh8LFKdNHvHxMEeCVg2TnJK+e4XIPlsctLKRxbAnK8NOYVgFac+WNSa7W5GQtPgGrM/pbB2K19BSZpSYzYHJIzrsBxKiRLO8eCnLbfUbHWB1Z1SYHklFA8Ra01GNTxPZ8qvPEUPvMs8pk3C0+uHV2nTMVzHQtOlX4DIdX2+YKpeRAjGtcA2WDyhTNxWpeClPbFMTiHilLKqfdkH8fWjBPXevM0Qy5TIigW6FWVALx+E6hJKWO1te7bGlVpduGn2yyqjVBliXTNBiOXrNTRGPy5VpictUr6ipYMsyMfzCWypEBOiToZEiOuMCeH1mw24y0ZFDXPl7FasWHkzkrvI/NtYu2Ev/jHL+jfFHiZ8c7zeBhJJYCaXXyl8nQV6FeJ0l/yP/urL/l//h3ho1D5wq87vudzhX/rBx2vaUTSwJi3RNfTraG7FiiJ8XaPqzaIqBsc3bViMxLE1puvyACumcUpAaeKr0rFMx1s0l/JZnKkBfCBVedYR6HMgSllDhXKDsLBsUvtGkTH4HuCs3VetFKKa9P8oAZvUsFabYTDrEQHORWYAylXRCs+RtZe2Kw9h3Hm7nZiWHvut5WHR/B1zwejY66wGgorHM8uAqsnA1/9cMfzjzJ3Y+Lz704cRsdrKyFdVC6uhbde64nrSs4z+w8da1eIXUORo8URqiVltdr32LW2lzTjO/XaSJgzPT3PVo6rlfJiSHzz/YlSBlQzviWSzinpY8n6b3d8cjVBEVxSdFSmuVBLxYdA8J6pZl7sppbJBkotTCXR9QVVz+OcSc5xfdkRPcy7A+WQ6HPESWQVe258IZVCqrCrydj40TFcDOQqKI5UK2NqbYOVY9bCdlTuUqGLiZV3fPbNNa/fDBzmxPvPJw5JeON64GaupGSWpfvHzPPbxEcH4dlV4LqzwQ8lG+QmUuj6iHeBlDMp2FCh9SratDFV5tnMKmLvkM5QARfEGv3OgkwpFpTR0gxJrIIIIRx717V9GYbgWDtPJ3qEsqo6u6GqZC0UtfnyuZq1QZWKOmeJQbGKqDaf9YsYGPeVaZf55a8O/OwXC7/5Unj+0LPfK742d6uaUOlZep3wsQpV5Nj35ZgQLAHsvNJdgrceA4f91qraIzpvDLhGWmzvdyRP2ZfhGJbakxZlorrTu6An7sKxGl8SrXpKBo6vY1H//OFAQxVqNai3EbFyLqR5YmJP9W3Ai7oGv1syME8jtZZjsF4SjkXcv5D8lkC99DaO1sDyWwT4hiIsvv82qOoU4F75rMf7cCaXe+VYoqMR9Kpa5bX8U5WTwdTR/fHsXD7+gsdrdkZ0s9exCy7tnM+XiVRTBIRZKU1lI1oIrvDv/4VrfvD37bkJys06MLSqvUpGkiNkcL4SJBzXzkK4j2qfu7Y2yjLV84heLc6Kok0yaZvrctoigLeAaS6Ry1ptxM42sdBmj7ZrU83QSwEaxL8kmKYqaKogWZaxLfJaDXUq1doWVYzvI8UCvg+WMFbU5qAkmwiLevKIyVhrwHkjGU9jQ1iqgLOWoVT7MaslxrU0STSQ9jBsYPVWQO8TmYomYR4doURKHJnxXDvl6sLjVj2/8pXM//k/KxB6dBL6teff+7M3zGv46EF5TZSboTKsHf71QBiU57++g2QQu0jCSYBVpmaQIEgnlKQwYmikLxAy5N74EXkiJyVN5ggr0uM7JQ6ZPkZqsbiTczXUKwu7+0RNvo0dNlkpScleqM7WwhACUTNTsna1D85m6WDtGV8jA+DXyvWl8SG8NFmhRN58I5BQpr3y5hsdty+3rFc994dKzROHVHhvW7goldu7mXqxoq9wtep5bZW5DCZffEiZC3E8edLjN0CFtEvwvhDfDuhQUUm4TsBZENcMpNP68sEdzbMcFWomuEoQ5Y0bR6mRb3z7AOKhJPOqEc/8yo732x+fOBnQUvFJCVnYj4VxsirbRXO6064Qy0CpZi40KsxUhgtPlI4pFW7Hkc55stpEqFQyJSfcXhjWA77rGXfbNucgklLhMBdyBY3KxXXAeZvrLa6wn5S7bWVb4TPe9J4zhTErjw+Fh/vC+mLg008G9ruZr70HL+4znSuklLkYIhebSCmJMRlsN+VCKo4cFBcrNyvHGxeO1zeRTQxQK1kLLnhC7wid4DrsTw+LhEycQbzOWS/Qe2cmRN61eQBGGCzVmMNBHVogY5IR1zwFVJU6GRSWa9OutznrBhtUQgyIr1Rf8INnMwjXoSB9z2VY4afCl7+W+QfvVu5yILqM1IJKpBJBzdKUYxDlqB2nwWq1IQZuCQxH0l/DSs/Xip7pzFkSBj1uxMeAIUtecMpEjrIvbb1q9NiHtZhoL3AMX6+gE6dqf5G+0c53gfA/zmdYkpyFc6elUrL5CUj2JN8UNM6xGOykJiksJbes3TKb1ka3QHh2RmCP+Q6uRUtSjpMJj1XrWUR1NLTjt/pCn9o033mcUIdXzkbOLgYcSZ/2mNOFlJZhuLPXORE+OSYOuRYoredeLTlFxPz1K0zFkiNXtVkpB+ad8vI288OfvuTh9oG1G4iu2AYuAUcg+dZKq9nWopqhmWucCm1om9HO9WTNfEy8TpdfVcyBdLmsTo1Y2FojR+IrrZXQ8saj82Zry1VXcGI240Xq8b4vSJe21VW1JQFVmteEuUkqijpIC2pQBBkUJRvKUSKpyeJqEua5klu/uw+GJkz7Qs3O1FEBIz4uHbyGRNRSCL0zK+M9XHx3RENF7idYeaZdQFNnS3Xl8ZNwvfEQK/qk5//+f3zB7kG5+JSSPwj8T/6bK/7VH3H85Psj3fScp+sdV5sV4bJDhwN57zhsM4SINqv2DNBH8DMyV7hvVto9SPRU9WidzZm2NnSGk0IidpW+V4Y+IAV2JVNE6WNg7XputwfqWLm+7m37CdaKypPA5Mg14zpDtqZUyFXoYiA2dY1mk6D20ZKszWXH09cj4gu7h8r2oRAHz5ihJsfNWjlk2O2F3WFkTDDNM3cHZRbY7uFm3fOZT624uFY+9SaEMXMVIod5JktlO1XCbeJJ7Lh4CrUE5qnAi0LwoDdKjYIrjSDdCQSbFiy1LhBpKxCiyeEcUBydE9565rm9m/jonmaaVpGiR6Tun3d8cmSAiiNArmgJVIHtNFNSxa06VhcRnyOPu5m48lyuLph1RHzmUiLT48xeYVQlxMhF7wmdMlWY0x7GQBeDmRmFSMrCLhXGVOj7yLOLgXUtPB4S93vIQdkzsQeG68CzJyv2DxMv7wolV7aPmTlBV5XHlHjYzaSS6FeBECK+M4OGviu8fEzEGAgrJe0q48768y4qfQerzuMxp68ilTjIcTBRXFxBq1mfOszHwAcjLrrOIE7xziAzV47DgmpxaDbOQWobVudtwy1U0qykWSCB4tsMZTXVQwAXrdcoPiExtGEkpmp4LCA646LwO17v+e63ld7P6BzJCCShumR9z+pftdM9kxkqWOug9Z5ri+LaEgPxHOcewHcmAsBxYz4W6K8Wuc2n4JQIcE5POHudE5Ity/5rSUx5tSe2wOVWNesxiVgShVcPC7SC2shQVUopzGlCsxAkIg16XOD83P4957kFfjlyJJYgagh842G0n/XsMy0tj4+fzlFKeoY5yyvR7fya6nc81hKfpQpuuILafS16lredXYijFv/s52WQkWutnvbKljiePdc1GWNtgdU1uD1VbY5FpgKwUcdKyB3FJf7KTz3yp//IwKfe9GzvHS7scX1gFa1Odl3CFbHevDHyrM++JMQtQ6wF67NSLUeOzloHTWRBNbdAS1z0lO6IXSfX0JhjxliX1WVrxxC8tj5rtYFl3oM0LxGVIxehlNISfKjVBrMdJz+25M5VQwlQ8BlqprkNQp2gZCMNprFQk6NWk5VtLjwlJeqskKGWCq6tuWXNl9bpq+DEowcliRJed9THAxIdOkLJZqVc454YntLNI3FToXM8f+H4a397y+WlZ/9y5s/9yIb/8p8acDrxx5/M9KvMdUzE60DSgivK7nkjNEqhZCjOkhU9OPTgqFKRlYNgRNuqSqESFHBm7bzbtp63CMMAMVb64KAEck5IqKx9oHeOWmY2EojXRiBPc2a97slzpkxWqI1bIDnWvaFWriqxK1yte/aHTEmZy8uOblVJo33DglRyhjoWNmvFBWV3MGfZ7TzypW/sEe3povL6jecL747sDsrTGw9aeftTHetOeXyY+egh8dZ6zfU1PHnieDzYespZ2D4coHYMTz3dkwDjRP5QCdrBUyAk24sAghqf5MiDWoqXZn8taiOU1Uzq3nqj5/ZuR9YVlQy1niX8/8XHJ04GIoFZ1HgC1VE0sM8TroeLLrC+SGhWtFSi86xXhY0LzApZK7H3qK9cXvR0tdAXZRUiF65nnBP73cQhj6S5EDY9RSLpMNP7SF8EHSulTkz7ymGvxHXAOeWyE643A1Icj/vKy8cZkY6pmQ4dRuXbH47c3c5kFa4vAzkJ++RREaa5oFlYr6CWjBdltfJEL6hWcoL9qMRaTI/aOzpnvciaBdVmVKGmAPBe8dIUBWIzyyUUkNKCmY0/tTnZAgV8Vyx5cOZ7UCuUom1zcYSAcRA6j0QLwLi22XohUYjOlA0qkJ1jzIEXD8KX39/y618K/MNfK9wlyxa1BoM9HdTUYKWlx96qLD3bOF0zmVp20SMycLR/PREPT8dZRXoeIPWcud4qsiWInVXugqLaymwFqqBNp2f79hGPPU7MWwKfnV7DDhbE4fx8Pn60oOzEXq/kTJ5naxtIOVXELIiO9f1ymVuLoQECZ5+ThTC4BPLlUjS3xWNi9HFujxixSdpnNPD5hCp83NTHOdfshwVpjpenHv9JVaFqxD4zJKos7oWu2S7zHa8vxwTo7CZyXmQ454gVZjkaLZ9V6dbTDkz4GEjFU8uEo+B6x5e+Df/7/+wl/6v/zmsMm0f2c0X3Qi6evs/0na15xJl3vGtQvADFtb6+mJHWoj5phkA4WuvJ4GBp64SyAC5iravGxbG1p8f2DUrjgZxlrmfrza5roSRpn3pZH3JC/HJpyYAlg8bjkIagKAW792WCfBAinmmXmHJkVkeakjkDOg9kYrTRw2VWQsWQl+CtZWAuvbaWmtzRqyc/THRvdZASrjqyQJ6EfCi4MOMvAvPLLa+93ZNDpn9z4D/9j+949xuBi97xY7/T8R/+uyueIIxj4u1OwG8JFxtqytALenDs7tQkYJoI9Ka4ygWpCTZqUyTVlCgLydfncpwTMe2VmjCjo6A4bwPDxBV8cxkdQsfgHCrCYZcoj5VV7Dgcso1ADpneC/7Csy+Z8a4g2XPVe66emKusjzaFcns/0q8jN08DvlcOe7uuKQu7Q7F2rxMebiv7lNmrsNvO6Fi5fM2z6YTVxhNvFd173n+ReXaz4mK9YqpbHg+go2PqlMeUeftmxdW1sD3srZjzcBgr+cWBiyeRMJgUVG6hpoI+cejljPPmHJi1Jbpi0zClaEtQbcKuSAIJiFZeuwx815PI+7eFKkLywvxx6Pa3OT5xMpCzZ9LExMw4wzY50uC4uRzoV7DuAqMkhgvB9z1e9uACTjtqKAxeiFK4uTCG5f1tZj8qN+s1PhmMdpgKhwmoZiWsKvS9UNOBnZge1wMxmLQwqedum3jIM9sXM/vDAQZPcA7xA0jlw9vERw+VOhe6VaBH2G4n9gn6VbDJWTsjCHlxSMEy7yokJxyKcFeU3CvXa8VHpavgMk0O2KA63wKQ+YWCWN9Q1eGqosUqBusfFrwkNpvI5SbSrzxdsN7/NFZ2e0jV473Hh4r3DS72i8TFeoRgQVt9MKeuGKjimdLMb3544Kc/7/mpXx34ynszuzFYkMgjzg0kZ9Mgo0bK4n9fT9X2khhwJk+DJcieqp1z2Z8TOPHj4RwdOHfd+/hxGoRklZa0RMdsh21TN3LaERo4yuG0IQkLh2G5Jqeq+wyGOFbmryIU1jevNhMiFdI4k/xsPgyeo+zSUAaz4C1lppZkU/KqHgPqkhCcOh8LUe9jVb8aAoOXV2Bq+7RL1LLkwcmrX+bz10BOPy+mQ8dkwJ0CuykMlmt0xvY/QxbO74+IoQLVCVLPEAx5VUVSg8fNtT1GyBgXBSdoVtLsmGYHoRCjUl02b/W44q//zI4f+vSe/8afcwwlIl1m2sJu9BwOSghK34t52btmBe5NClmyae8FJTh/auXIgiDZupACJKumDSFSQxnUNZXEcj2WBHRZKvadExvV2K63PcepyenmslThS3KnLYmntQnausQS/Jae0YYO4rWSJ0g7m0B62B9I1chppU2HVV0QDCPd1bmtsYo5wYqibSKmVyzYOI+kSsqOqyeC7hXpHGlvQSTPievryC5XhgjDUKgr2D/A//Yv3xGC8N/7Uyv+zZ/o+YG34GGbiV4QH9BrRcaR0q9wcWb+IFPGQO7smvps+1t4Fig3ihwUlxyGCxXIFUkeyoCoI4+VNO7pBofvPFltiBC+IJ2zFkPJRDcAldEpL96H/G3haSh0IdA/7RjHAzI7ghfKXFl56H3lsuvY9KAxUxXGbWFQx3odcVEsiIogUrl7OZEpbC5WPH9pI7CH3rhfY/CsLxzTXCmqfLArvPlkw6ovfPFbE++/PHDz9cjmieetTcWtHPsyk2XFVJTr1x2bNwfuHwuaAv2qElxk91jRMbHZOILz6A7SoeKeOdwmIGvB+wySEUyxUBz4RVomULUgRGrxBAevXQ4MJeM7x1gy+5R+y33348cnTgZ2U2bOUOgY68gkhW7Vs9p40Il3X3j8EOg2jvUqoPRotQUfumrs0uJZBRgLZOdJWXicEmk38qLBYmNS5nnk8TDT9x35MnB54bi66ui6QIyJORdmdTwcCo+HmXQobH3EUbkchLv9zN1j5n7KpKz44lhddLxxYwMuPiiVlBWXLdNabSJXF5FcJvzB5D/pkBkRau8gmvxlPTjTTdcK6q0iiYrvldBb5inOiEE1g5ZKXbTyFXCeED1dgGEFT28cmwuIznTseapMtXlQ0xjpTinepr7XZtyy2BxboLCNL7jAOCnffJ75hS/BP/q88hvfUl48gs8rNOwoKRBYMedEcYp3hZIM4lp2waVFcOw1H4PSSat/wvfPKsIWaaWhyguZbwmE5/r48z8Rg5hda7QLtuFbhed41Uv4XLygR+hs4RgsCIG2632OA+irjfxj0FgY9dIq55Iz8zSSupkg0XJv4fgJtdpgopIncprIJVPUlAji3TEoiTPm+qIqUGwo0lI5A22s8SnQHsmFTdWhcsaSPyNlNn5bQ7dbgvSxxOt4M9yppUKzqz0+VE5IBwunof2ba3c1OE/VckR/BDGfi3qmAGkkQo7okBi6UWb+5A+veXo189O/PvPllxVlYHdX8U4pU+D/9P/Z8f2fvebHfyiw32+5uOgY26j0/Q6maSYERxe8OZe6tq68I3qD0Eup1FzbMDHPYpCkGeok1GQW56Y8MDKjoRhyXC/LBfGIOUHawkR86722dWgfW/HRPnspLVlsLYtaOKpOTB3jKapHRGXpBydVgoKbIe+V0Tum0Ybv1JTte1kqqgEN2sZJi41ibveotraketPCeOvX4AZHmQsMgTrNyHDJtL9HZKCUiWEdiDHg7wsXN5VEoftMx//mf/HIN75W+N/9d5/wr//xyFUo7PcrpJ9h1RPWUFNGe8FfTOij8vJ+pvpoUr4i4JtQuiTcPlriI8XOLSk1CaIV0YImz3yoiEakISohBkJnBNfDJLCHoe/wwZGkcvc48e7XMv6FcPWOZ+itaBjVsx8zq1Wk6yPP3vQ2V8DZhaqzMs0JKlwOHXPK7B/NVGK3q8yT2r2Njrv7yjxl+l652zseHwuPY2acK28/M+ng4XZi4yM3b/Vs55kXt4ntvOPhm47f+WbP268F7vNM9oVUEvPccXHT8XQD49YxDEIIwiHPjFPlfl+Y+8p61UESDt8S5j4TLyHegFsHtLPvmBNnPSbvyFnxviIVxEeyKC8OibuHzBtP1zxZBV5fRT7J8YmTgcf9ljx2TGNk0szmqeP6Wcdm5UlT4E4LmyFwdQO9n6hERD0rzUxVbaznHKlU/AouEfaPM+Ncuc/KR7sRrwEv0TJ2sZnSRR2rzZrrtU1tehSzuZz2iquFt54ENlcrXC48fxS8BNJccKHw3W9FVhJ4sZ1h8GwGx+E+s+49JRTSnPEiXFx4nKscHpWUHTE6NCf2s+Ji4dnasd441uvKZi1s1oFuFYkdhFhwweB9LTbrvCocZw0EwXfmmR56R+yVEJX12nFx4eiCSUgOh8r2Ubl7qExaLPtzNt+25mJMem/B0gtHcybn4DF5vvKtAz//hcTPfKnnC98K3N0HpNkcez+TqiUqUs2LOxRDNrT1W0/RAc4j9ZEE2ODkWuuxvWrHGbaq5x7rC1JyJpM723Td8XHWqwdTUSDapj26Jt06eQYsKMDyGuc4w/ExzQP/9GanJEQbAWcpqpdgKCxEucYJyIk0TwSJ1GNW0Vo7FUrN5JTIaTbb6dYb98HjYyTGjhg7C0xtbHUpiXmaSSktMpOPnd9JRaFLS+UVYuCSbC0STV2u3ilT4WOJwFmSUGnDi3S5r8stO0MlGtR+9m6WbLakZUF36gKrqJGTVIz4Ks4UR3YOjiIzf/7HOv7wv7Liz/584j/4v8DtDv69f+ea2/de8vNfFb75fuE/+r/t+Z//pYEf/mzkYW/V6mUfGDzsD8o8ZcaDach9MKQgRjONcYJZ7xZjXee5TZZsRiyahDrbuqJzaBBLqJHjtMFlyVtMV3yTOIta4DpzWkKL4NuocNcGNy0cAK1mBW1zBYRC43dpS5oVaq448RQs6ZeilEO1OfazY3URmOZMLg5KYbE7jjEy7Q/mH1ILXeepWY9rWVHj3ojixVPSTF1BHQMlTsgckU2iJOXiIrB9nFivI7LJuDc2/KOfKfzVv7Llr/z7r/Fnfl/PrC0ghkzslM5lI2t6h6wrdS9Mz4Ux9ahAzIlKaBbCHueCDVfKgsyGCDjFfudMCpl3pjAJ0REHcJ2Ch1xMNZFyZt33hJWS3cR2crz3m473vww3RbifhYftSCmQxdOvhWEQdK5EMTfb0Kvp+auj6wuOwv6xINUTvSNJ4qCFXYFVH3m435O18tqzFS8fDnx0N1EqfPQwM+eK6wJ0HesItTg+uC3kHfyez6x4542Bhymx2TsurjpWHu52E6uwIk9CnTz9jSOuBGmzFjbq6GZhe4B9FuaxcBGEKMI8wuFQ0VshrmC4dAwXgnSK9EKtgtTYyKiOPGZux8RXvpXZKuweC0+myiZ4LvjnH5/cdGgOPMwjc404F223CAnfOcR5XlsH+kGJsRi7nWg2mrUCGVVPCWajGlzHxhm54eAcflS6tV3cqdkpPhkG9oeZ/Tjz4tHMR9YxEGZHHCv1kPHiuLqwiYT7qtxcdUCgUnn7quNTlwPzwaBLRdk9FO62hatVzxMKLx5Ns8pU2R0y97sMMXA5eAiVflRCr6xXjs3Gc30Rudk41r2NPpbQ+or5VBGJE9sooh6TgG5wuGiEROeV4JUudLgSmaeZx23l7qGym1oXKBi7tmQlBMfGeyQorq/GEhUPXrnbwbe/nfmp34Rf/KeZL37Dcz8OlFQJwWY+lDJQbZ6pGR9p6zPVauYWGHqxQNVLUBK/9FxN1mmbY8WGJ7W+d6Nwl7ZZurPAbw9YAqk7IgYGlCxJRkWdDR454gvHZABrE6DHnrdWaZPzFn13Q1xaPKxnP1NbG4EzUl9LNuwnGlN/OVoiU2mumiMeodMB5+Pp2qiS88w0j8wpHStj7z3DsGK13jD0K7o4GFzrzPgpl8J42DHutqTpQNJkDGq1BMudXbkFibCzOskBj6jLkja4RXFxet5iuWzSMrXhYLLcMWVhZy4yOGvv2DxSG9W8tDUEJx4bpi3moqyWELmWEFRVUqk4VaDgateyCsX5hFf43BuRdzaJJz828Kd/6YAPgf/BX3JMH0bev4X3v6W8+1LYbg/sxo7gMg8zdM5zMTjWl8I0CtPBDM7y5HFOyRFiqMTO9NfOGXHXATkrh4OZ97gaaKN8QASPo7ZgbEmApYL2CZbFU6jibK5DNY5PUftZtBqRUAK9zzZMpzZ5H84GL2U14692p5YZFKU2t8EKqpVUnbUwpspchFI8uczkBJIKmiohQhUbBf/44KAmTFbZoTVTKPji8QWrvqMlySVBHqBMAfUHwhApk2O1cuT5wIqOuFHoOkQ8n/8bt/xP/8w1f+5HO+4eJiQ4hk7pY0dyI5lM1wvuUqgHhzzCOFe0OMzlw/wEHB5cZnBrdJ9gqkgWy9yCKTjSQUl7uw5xgDgIhNammyvTZJNahoue3mUQx1yEDz5KfPDFytPa8Zm3A7u9SaovLh19EII6Dg/VpovmRC2OJ1drNlemAPLOBA4ZKMHZbBcfGa4dcS189NHEi+cF18M+73h8UK4vVzyfDow+8uk3AzVnbvcTm5Xw3nbkGx9kLl3lU08jn/2+Aa560t0BP81E6fDRkBHXNcSoBPxaqK7JYlG6vnJ1IWzHymGbuM1CLx3RBXpm9nPh7rFD34cYZ8Kg6AbjNlCoxXEY92z3wmMeiXPg0xu4WHu8TKZe+QTHJzcdCuBXSi9ALhRfSAWq2+Cvr3gWd5SaiV2Pd76NmRQSQlXPpELKiWFYGeO0FLphoAqsciLWgWkSJk1E7whayWRyhqtReX4YDRapsD3M4IW+d6Tq+Oi5ZYerlcf5ymbl2MSeD18kHrcTIQq4jo8eDiZfcpWXD4nnW/sM+7Ei3nF51bMZhADsEMiVUj372bHPwm4qXIbM9bpj1UkbJm8sf1k2o8BxvoGLdsPUaWM6G6RVq+Owg/GhMO8zD9tKrkINAt5MRKIKXS9cXEaue6F2B1zsCXlF2iX+2dcO/M1fUP7Rl9Z87d3M4+GC5AMuuma80VElWW9Y/amPLBjlWy1gLAY5Tk/9SZtxbwtVVFEJGHnP2NDHAIUegyEc40DrQ7cqVjBUY6lA5SyoNYWkIbaWTjlp1/EY6OQYMBdmt/1sYU3bfIKqytEwqXLkEcApqQCORkQLIWyZgrikI4I0gmBihub6tfTc2oZVEvM8k0tm4Up0sePi4orLq2uGfkV0vVV1fvGGqOxDj0c4YEmyit0KwIimlq0dg721F+TYzz6SGJeLDUfdccUqTpw29QM4bUFcWqK2OPC15Ghp5djlaIncmXHUwlRfQIyj2mN5mILXRM0B9T0+G25dZGZfHOku8Hd+Yc8f/OGB9XTgv/0TPWHteVa3uE759Os99e3EzkVS8myqUpKwBQ65UnfgoklnhzUkqeSpUKuQJ6HMMI9iE0SDEnwhBEvUnfOkUcmjTU3VqjZBVD2izlAesTVfKm1khCVuSktGq2WZVYQ6KURDAHM1SyKzqa6U2kYLY1W8C45aHGMqbXop1uJrTHprHwil+Qow2ZjeUoVpysyT+RiYuZsSfOTh9pGHh0KoineOXKBSrM+eEqFJGYd1JI0zebb7Px9mNhcel4UaFZHMXCLrTpChR1cTH/7KPX/8u1ZcvdbzYjcjGugwBKa6jNOAWznYZHQS9DGSa2G7y4047ajaN46PQ0qGVJGHTI2e0ouNAE6VvIMym8Ta955ck6kSqjeOZ0M9YhcQMZJkofC4rezec/S3mSdDRyhCPYxsLnuubwYetyPvf3CgiN2jwywMvW/FXib2BYjsdzZE72438dHzytAJYbDzdlp5852BbSnc3ieePe3wofLBtydwkZX3PO4mJHrUeV7eHyA5bp45Li4i3UWPf1oZrgXdFlyOXPTCPE02TTPQCpCGgi6cKFGCKBfekO3dvpBSZc42t8cFJejMOFsrSWbI93bdZ1eY04hIYKyKlg6VRDoMdKvEvV9xe1jxOz9BjP/kswkQ+q4jrD0rV9FOWHWOGDy5d3RSSaWyHxXtBNFsVZQ69pNQQtfGUCZ8b3DQYZ9IatnhTYkcpDB0ln2RbQ74EHsGAqNGm2+glbX3rK827HPmdjfiSgRRUq1GPimOD5/PfPvFxDB4Pne1Zhwzq3Xg01c94w4+vHe4VeFZFyAJY67E3py4DofMdDCbytKCIrUw72DcWdXwmngu1sK697hoBEIXnN103260aLM3NUiyTjBnmKdKng2yq0nNqCOCd4r3gSF6Vl1lcwHrS6HbdOgE44PjV74+83d/JfNT/9Tzz77tuJ3Ba0R8IHilSkaDp7Q2zRBtsJM/IsQt6FBbpW5tAu9av3qBqJ20DU5R71GtaG2z4as0Nms169YlMrV++FJ7C8LiArcE7YWMJc4dEYAlIi4V/JEEd1x91qtdRkMfJbft/WrzSaitJ64L3t6OM+VjO04tjUV+qNCSifaapTJrIudiqAoWRCu18QayVUNidtDr1Zqb6xuurp/RdwPBRfu4y7jqamiQV5snMc2JXHNrUbgTYtHQCY4kQTvf47U4R19az56GoogrjURoZFjX2i/iGrekkU/tOqtV9MckYdFbyPFvoG0NWEsJTrLTxdDJS1N55EppGx15xaefFv7VPxb5zOtb0j7RV+F73nBorNQJJHZonpAMUUZWztM7T4nAnHnYwTg7XFH8nM2SOzicePJkTn5URWoi5zZiPIgpbzA5YeigH6AehDSa8xx5QbdsXZnk0PoFrgX+LBCLKTm8awqAYgiJdq4pA5TgPZ2vlBnAZpc4PDF40liYis0yOXZnMN6IfeccqmbHnmdr41RgniBP5r7Xi637Pgi7xz3jLrDWgEbMBluUioOSbc14bwnA/UhwazqtpBLb/bZrOI0wdJ4cCnI5MX01kX6z8tqNMpXEPBY2wxqvW5yLzDJxIWvcpWWu+cNEGCK3d4V5VmL0aBaECD4ZgbE6gge6YCQ9VdLOlAMem/SatLIryQayiSenjPeOLkbEK16MV1DF2xCqF8rNrfDs2cZ83ah0G8/zWfnwvYm6S5SxcnUdlpXLk6uOWjLvvpd4+jQyDJUPbidWvWPdqbV5g8eJMM3K5qLnkCv7D2dqssLzKx9sud8LP/hOz/4w88Fd5s0YIc4MEV57u+PtJ8rVTYfrlRqKuSyuHGTjyvnHaElo5Ogb4L0lh2YwZCTdGAS/MQSlpJkaPPt9YhUjT1cdL28nHu6UTjwxCKPuzDTJKUJHrQeqOjyOWAu3O8df/1biN144/uRf+u0i++n4xMnAwx5WV571ytOvA1kqZc6QJ+qUuJ0TpdoHwwm1WDKg0jEVh8SAp+J9wXv70mkUSNVIHqVCLqwksp8dD2XC9R2+eu73E/fjjOn1MzFWahIOO2W7n+gvB5vUFIW+c+zHwuO+sF4F3rzpSGni+ePE1apjHZTbeWLVwWev1vQeXjxkdnOFmjnsbYZ68R4fA5ILacrsXGAljgepvP98JKc257svDL0QorMZ7FGQIK0/vZiQWEVbaiMcLZWcw0Y1dwb7O9+CQSmkDIdpxd1D5asfzXzxGzO/9o3Ar3w987XnME5r5lzwfbIq0Cmow1ePC+6o5/VaDWZskLDQzkvk6FJGdQahCqi0QOxOWnh7kEerb9BqY6VrRQmn/nyD683XozYCZNMCOGm/M9TBeyMK2kAOPZH0jqz4s8UnIKX1uXMLSu3Caru2ZwryliB8DBpTXgmitErwSB6Q5fct2LaWgDH7LSFYNvTFmc7O1dF1PReXl1xfPeFyc0XXrQhtRLRrr1uqEn2HaGWeJ8ZxpE6V0ySI07mdUwVOf5eFmWHHcoGkoU7OECm3bC6uJRnUIyJg6EFDX1qiUdGmpACbKLpII5fLZoO07J6byqHUYtW0CloCxRkkW9QCTp5G/vt/sudP/2tKPKwIEVKp+DTiZo8Gh/qKOI9PnpgzZcokFOeFtYu4XtlTmZPa9zEJpcmr8IKjUvPCvbCZ9TVDDc22tRZ0tA12iDBc2vrNqZKzJaciYr4IC+tfpA0YUpKaUkG9EFuLRJ0Rg48THL3SeSFrG0RWaxvNDkK2qr8YGU5be6c073kvzXWwQpls/VUv5CTUZN9lie3r5yquRvJsLY5MBm/JbMmKbf8V8RXNwnSfGV4LkGb8ypNzxl0kfOnonMMRkRtl9+3MR19Q1m7FYapItNZizRPDyhRKPjh0k8A75peNqFxmHm+t1y9qHAzrUyRQx6oPhF4pseJrhF2FpIR1j0olzTM5V6qT1vZUhs4dUTrnjXyr4lGZqFu4uAs8uXTMEeZDRjvPvIfHfTIPi1QIKoz7ShEYvOfF/cxH+xkSXFx3pKkwXEVWK+MvPIi9h6fg5srhIfPysTKsPe/vJj741kjx8MZN5OnG89FeOVThw5eZN53nM6+v+dTrPV2XGa4E+r3ptaO2dmbGeU9QB5MVfDUox4qpVIvA0hwZqfggbKKgrkODMmwisThiX8gebpPjax/Ap68iw/VInAsahYmZkIWkFbInpcLnX0R+7muJ26l8Z0D/LY5PzhmQgnSB4DMdnsOkPBwqlxEkJ7ZTwjvPzeaKwzyzOxjbsY9C7AdyKNQE+7EQoqOoJ5VMXHk6p7giPIwFyYEogSozGisxWda7nSvzYUsMnmHtKe6Ai/Bk3eMQioKUjvkx83I78ZiU68vA9jDzuB0ZZ8/T1YpxZ6Nqn1wE1qvA3eMIUXnzqme7zbgEYR15LIVkjiYgyrByPL3peHtVedoLV0MkiFBLYT4I9SBNQ7pUanIMICrFft82BRcd3vvW9xbyGCg6mZ92gy3VW9b//Hbi//tV5We/1POFdyGXQHRCZSJ0AiUabO5NeikKkmrTpDpwiy2ZZZ9LkFvaztIijlv6+C1ayBIsEYqYF/pCIlx656pL+d560kukEqtAl3jlxNoPtoHRHBmbIsJB818+xWPXHk9rY+iSmLSebm6eCipN2bG0D5p5kJ5aBMfj7EdDLE6NgfOG/KIEWPrmtHaHipwcEXWRNVq0H4aBm+sn3Fw/Zb26IoQe7z2l1KMsUGslhoDWwn6/Y7t7IKUJ/Zhh0ivnKSc55nKfBLs+Sy+6NpMq8RjzvQWj5X09znToR+jf/ivtgtdF3aDaWkVL2+ckzVwq29pQmFJtjRZVslRqisYjiJGglVoSPnjevIGpZjPKDAFmm/nh1DW+ja3PbijUrOTkmSbFUxiC4AfHg1aqemoWcjZo30lLapxDm9Z28a9IszkZxujx3lHmynZ0dLEwDIV+5SAFDqNJxASOjnm1MQqXdgEiZnWLXU+dTe7nghxRok7EDMMUpHlS1GJJkRTb8/D+yPWoLflA1QoGVSRVxCuKJydDDxymqCml0vnAdjJ0Yw6WVDsHASHnQk8AV4jBLHo1r6gyIz7i6kR2jsFFitp14xL2t8qLnytUGRjjbEmQKusS2HjHzIzgWAWHPq2kByXMDrlwvPzaiKoF73SoODFHWHE2hfVy1Rkct3ccRrOWj70gvlJyoRaPqsdpJnobC49mW3de2kA8zNxpDrBTVsMKNp77Lz5QJyGguKnwtO8JvSISyKPiHTymkf1UWN8MvPPaik5t/PFhtMmHeV/ZV0fxwrqD3UG4v888v9uzisLblxd88OjI3hO9Zxxn3nv5yOWTge9+x1P2iQFhLYnXrgfCesCvA9KZVBgtdj/FhuTJOqK1Is2q/ojYYaRznKLBU8X8aHxn31ycEPqMNLfGoS88faPn3YeJv/Yre96+vuD1C2FYJ0YHh3FFyoH3Z+H5Hr708pLbObN2H9tjfpvjEycDV1eRYQW1JsZJ2Gtg7BwXlxeErnC5SXiJzFm4282odGiBoomhj1SX8N2Kwyg8Wa0Z4sB2/9wIJaIc1HSstUDOiYvOrI7XKnTZUzRz7zwurhgGIYqSc2GWylgT3gXmvfA4zqQWZN/7KPHBixlcZRgcd4fJZrA7T6eZfMhsesdbV5E8FV7cVXKAy1Ug1kBXlbuHzKgF13l8J/hO2WwCV+tA3zUeAI1p3jJ/+65bNagVcos3VZWsCrMFl1y0uXElg5+pCAEvnkpC4sRmqPzY5yKXQ2K9Hvhn34rc7zO9d3g6y969IzGj6gjiQAu1FCrBqo9mmSpL0KynjV6s9LCKvkXvegYTI63j0argRZ8OyjLG9hRSlwxDWtB3R9IgjVOxDJZxHry3/h7BqgOBlkhpSwbal6kuhJtCaWYIJReEzMn5T07wrzNp2CsGRGfV9oJCWAw8QcaLYFxogd+1oCotu2FJDk6BE2C1WnF1dcXlxSWr4YIQemsPtE0NNY6M94GcMrFb4X1kkRJ+jHZ5Dgec1AEc3/6V31slxcl46JjUnD3n+LMhBcs9Pr/nv+0hC/IizYiyDVGpSlGlZA814YNCTuRwAbHyc19+4N/NN3QVIKO+oG7AqVLSiCvgYzDjLYl454ysHgtalDmZ3dLQReaaDXXEJLelGDIRxOF8heJai8jWXlXPVLER4KFDvJLnwuNY6XolDBCjteyctrWrLTlQS2yr2Mvldr2Cc3bpiuKroX+5YkZjVQh4xCuii8wx4KWavHj5OrU1RxVKUUMQMbmg85XqzMegpoqLZjgjWogSeXiRSJNQUGrwaG5k4mIEmbjukOx4eDEyDGtcnMyVFCV2FakrVA/UjVDzzAc/v4cU6WPlMCVKcKwl0gVBXaCELZ1b2dji7BinRLxc8fL5yLT1uJVae6Q6CIoLbRKsdwSE9Ag52/e2SKHOlhzPaSZXcD6y7j3ROShqpGgVqNWSLYq1ZnaRIA7/dmD//sTjy8LFxcAwwPWFTYmdFR7vKkPoubrqcAfHR3NCZ0/YwvPHAzkrgws8PC/MObPZRJ681hNd4VAOXFwo93vPix188OWJx22hauZq7Xh2EYkh4fTAxdCjUXj7ycBll20+wxOFVTHES0qbG9D4YgFkZb4RvmL7rZzaddJaRy5E2xClmNMsE+RIFaWETKg9MQuvrxI/+kOBn31+4D/9x1vcamAdHBJ6snZ43/GgQiZQ/A2rboeW6b/4O96OT5wMrAZH7BxaKvvk0HUgdpkSM8Se680FU4F3v/U+RHjj6RXzOLIf9+wV3BzYz5W5Bur+gGfi/nGiFk8pM7uDp3O2eGvncRUuhohLZuTx+mXPdYyIOAozU3Y8bgtzJ1zcdLjZxliy7tn4TEHZJUcRDwXSlHiY9lw96YgFSIXQRV677BAqX39UUu4YVp6slTnlloE7VkEJoXK/z5R9Ik3KfixcDp4hujai1XbO2qRGpbZAX9u26wRtI05dwHSi0vqXrg05cgoyIz4QvUddxkXhMyt460r5gTcTP/NN+OkvZL71bjDNbsw2lcpVnJbj/HhxvkmOElIblF9pVbY7EcFQBH8WXI5pwhILrD0jRrxCjJdQayOo4WzxL336pch2DaVYUoVzuFvk7PdqJlELMuBOfeyq1RIDwL5dHlxBvSVANXgMyK44KQb9tn52PZtNUGs967/Dkf12hCJ4lfTYnPzULZD9CUmwP+sx8PoQWK0HVqsNq9Wadb8xpYhzqCxGU5WSMykX5jQTfMCJx7f18qpr4ulSLdLM03FmpNTaN8JZpAFoboXi7TPK+TS9lrydkyrMylSPn+dIJGxogbUImgyvWiWjpck3Ubo6UxGKZqpAzJkB4We/GHnv3QNvvaHUvXFPaKZOsorMhy1D7ZAOaJP2gmSIlVkhJXdcH30wrf6cC6XWRkMxiLzQHCDFG1mzNvc7cSiVNBZiF+g7C85pKkyHbLbFYv1/8xUSfKUNGFVbd9qmiorJhQMO13IOMw+t+Ape2nxxB1SPywpR6EJgj1m2G0/HgiMi5iEggs/ApIQgaDMrcuoIIVDzRD/0zGNle1cpKpRqqgMwNVBvRukMwzUv33ukJMFdV3zvKXVmdRUIDlItZtzTJ77x0zB/lFhvAmOeyPR0koixUEJHkgMrjfioaOept4n+wrE7ZB7fT0j0UM0RMbahQDY6VwlFmLeJmpXaCSULWgOCWmLle7rerNqphVJyIww3naioEVQE5iTUsdJ3Hhcy6f7AfCis31B05Rknx8Yrfe8oz5TVs8rMjsdvwkE983biYTexm5XrdeD6qefyKnA7GRJQHhU3d4hfsZeRsU48FmHKFbcSdKpcr5Rn1x63fsJ+P5J3hSeXnqdP4GoT8SsIm4pbZ4gOdbl9v5x9F51CV/Gdg0PTsrpie71zLHNhNBdDn6K3dtyyLxchSMbFgoojxp5rEf7iH73k5794y1c+EHo8RRKhXyGh0MlADhDKniyJ0K1++8B+dnziZGAYbCb3mBz7VAkBVj4xPc6keE1Nno8OW26nPd/1zms8e+LY3ydmLeyr4GblkITdmJkpbGLkkApOA6UGXIGdTFx4z8Z3HLLNk9ckPG5nPNA7oY82JfGQC6tBuOocN6uenU8kZjrnGQususibr8F0KHz7ReUhK4N6roLHRyF729zfe1F48TLzwa5wdem5lJnHQ2Y7VRLgN47LPhIqbB8TRdsUtn3i1hWCBKuWRa2KtO+8iQyCIM7TeYMafSMKutBgXRfw3oHLiPNICLiQUZ1xXgCPD9Zm8KvA91wVnj6d+cG31vz0Fxz/8Dd2fHSIrMQ2EJ9aNzuYzEmLHG1IXbX+qGobyXu8s7YJ6WLPugDJxzYBJ+RjqY5pQaX9XrC1rc1KUBqULkBzabGnLL1APYtHCtJkSYI0rbS9l1ULDY5vicgCdpv1rjbbNYPk1NWT+6A/kd1ePfSEFLRAr2KKDxc8znvcUdt4Fovl+J8TRUKN8BljR9f8BYa+t2q0Qe25IRTFJSPPet+upbVqdMEoPnaeuhAZT494lUtwRgY8Rw+W27R0OcyVxiDJpfVzLoPVthacnJK0xdXmNCuikTJb18zc76xlM4sS6wB5woVorbVV5ZsfZX7y5y75t/98wc/e5r1jqh2KJ7oVu9GIwqshkudEH3qCq+YWVyDPABlfHE6NWa3eiKRqbD+MuG5ks+AdwXvE+5YUW4CeUqIU6yN3LpAL7B4yVZS49qhvxD/V1ua3Xr5TKGJoXqXgEJy28biiSDDULzpPmUzKGMQhVZizGeiIm6mp4tVZUtEgOi3GmahGVcLlSi2OVAouF0SFnAqXVyvunt+SR0/2pq4SpwTXSJ25srmJ7Pd7Hl8WhmFFd6lUEaLzbFbmDVAZ4XrFh79Q2f7mzGqzoWZhqgmRSPVb8DfNfhtz31w5dA/aF9CeF+9uKc7akpLNOEpcxceAC5XLPlJfFkr2+F6ZUqZWT8muFTqF6BWdxcYLD8FM2kxzhjUrbWGnWZhGT0iK3jjm+8zju4XN6oKLNyLbD/eEsXDxPT2r71H0TYcMwv4hswXyoSBzpet6PrNxrIcmq8yZ8Q5qDlQ18p3fFF7sFa+B736r5+J1ZYieD58HXn6Y4BaeuR1XMeKjcHkBXS+srj1h09n8GZ9tz3GgGlFfW1uxAAVxgVIE74M5roo2FEAseZwLaLW16xWpjsqEi6F9mSdqH9Eq+DLzudcDP/67B776tQm/3pB9Rt0MaSKvBLTgpTJ7D4x8kuMTJwNEgWy9sGHdU5hZ4djNld12x37a8tFU8N2aToS5mgRDVKmlMEZHp5lLCXSrnquNZz/O3N9WVqFHugOPO6WsAskpuXo0ZVZiX/CcFcikBNEPfOapp/jEWAvdEFmvHc5V3t/OzNkW8+MUOIwKJbOJjoetsp0Tmy6wcsphTtw+Zg4HmKqSJbN7BO8jm6eOECuaHPt94eVjpS+Vq+YaOHQmNfGtsve+zc6WJptZBqZ4CFFtsFCA/x9rf/q02Xae92G/e01772d8p5779BlwJuDgYCIIguAAUiJImkNCTVZiO1YkR5VEsZ18SVX+g6TiqtgpK1ZSFZdcqXJIx6XQckmkOCgyCQqkCIKYiPngDOjTp+d3eoY9rCkf1n7eboiJCx/4olDdp9/52Xuvda/7vq7fpZygtHpiN9SCiC33hsooZUr7fSyStRJ6dJkfZjiyiv1Fz/XLmedvTvi9Lye+9k4imnIpc/Aj6EahZIvyzVit7g57mV2WwgXoxshYocpFalvx9Asohc47ffk4IshjbPK4fz1RoO8+SrMbjpb29LjJFIv7xQYV8+jPjnr8QkVAVjYkilo8C3hN0D0xZHSyY1aCL3PGNCqzxRAFkDB2FzJkjRZNlics/ryrXMafWO1kdgJKa5w1GKNLJ2SXLyGKJwr78fSZQazQuAqrDQqF05rKWexYDIhANdIkW8kYymkox4GYPMOocYg5jaOZhIllSYxSAl8kPxEXljTM8vc0XtAoBaVdao908ftcnJaVuXCBXDRDdmChcVzD7vXO46uSn3QpCrshX4BNimjzacujEBnG0/14Ao4FkvP//O8if+VHFbWNkA3ZgY490juoLG7q6TeFL680tGFAAGs0jQUfI9uuCP6UODQaYiSkQKAsvMoIEoToUwkiCuPs2uiRhx/LohxLmuogEacttVWEkKCnBPioUjjqsQCKFMuXzpmUyqbtodzTugBkVEj0LuMcqE2m70DVo3AzCDknqmRIfSIZIeRCNBA8KZXiViMMQyxhNE4IXeI8Rq7XQl1VxCHw4O4A1PQpcpg0PQM5WeqUaFTCGMODN1qyCNUkjwcIzdRpdAVZEnZZcf9r5xx/cYY0G0yYkzXEocXUPaIaQgzUudizxQhDN9DUBlOVLk+/MdjakYYBlRK4MudeNprpXKM7WHeKYAXfJ1IozrKsIjobJOfSJTFQVxktoXQtdYWSsdKMEKNmaAf8OlE15VlqH0e2Z5GbVx1JJ4YJHPykYvZqRuYBHaQUqE4ziQNvP+pIorh22eBMuY6F5lqAPRUZpxPzpSWIZ56FoBJ913LgJ8xqy3tDYPCJq4fClUON0xqVM2aaqWaC2XOoiYBJFDCXvxhxXsDXVAJtQUl5zYIB+p3KuuDIjYYhQR9KsSwGkUDIBmU1QW3QaV7yHCz4JJgMP/fRmt/4wpbNmaepA4aGVlqy6gBNIKKl+oG3+B88wrhTBPEEWxZTFR1DCPiYCdkTozB1Rci2HgJqG+nansdDIKkKJw3b2LFuA3EdOOonrELmJA0Ep4lUnPmIahRDDDhRzGqDzomhgVppjqxhfR45H0JJH1TgTKRuise2cjBtavpt4Oy4pfVgGs3yQLPIivUqELJHcubhcWDVJTZBcGNxkEKkA6ZWl9OyU2il0AScibhKmM41R3uavYngLBhjURqMHbPJZVSHq1TEXE7KiXMsBkq3MHExe1ZFjCRZLlr06UKgVmZMKkFUvpx0U41Omhv7gfmH4ZnLid/7cuBffGVgNcyYKQVbQ3ItAz0mT8lhhALx9Km2bOx5XPETjHN2daF8LhtEIu9EcOO9ULbEp9vJT+bvuzZ03i3WsHMXjl+jfMzupLuzrBW+QemglNM7Y1eA8vN7RQyGgTCekDVKLFkSOm0IuTDZM+PrNaQxoyKQkCcMeSmWHhHNDprE6A/XSqOURSlTFPmiCjhIZDxdl18m7ax6oqlshVZ2fC3Kx2g9ft44Z9ejFXIdE95HujgQgkeFVCKAyePrY0hKF6IdsWRayJONeffqPz0U0LILMdp1VEo7PqXSSlYqfZ8QsYwWxmJTqYsLeFFw7DoWFyLRTBrjtdOY37CDOUE54aqiEBzV0xkjJcHuO3e2/OrvVfx7v6Q5PxmoJmCtKoLD4Mt1dJGuC/hBqJuaGENhkOiC7q7RdF0gx0RmHD0lGQvA0RFjxjbJaD/NsQgS9didI0KIqVxzhLYPGMBoTUqROASUs4joC8cFoZRnSBkNhJgZciZphQGSH1BGMEoTVF/IfJ1l0J7KJFJr6VYeyYaUPTFEEEPwYGwp7kWEwSeCKp29oR8NhlK4I7WDB/c35JOaZDybx471oTBxHpMNlR9QN+cc3+5ozwzzI4NrPFk3VJOInTmC22IODCdf99z+rMdVWwgNYls2nQLtQIoOQ7AYrcmq2AZjNsyuatrjwOqBYCpQfbHWatOADsynhtmhoJLm+OEanSvoS+GUtSH3AWM1STqSKrHvVSX4OHYUx+cuxkAMglKRoY8MrSJnjVtownpgfbvH1RX1VSFe6Vm+TzAHNVJlGDJxiIizuEHT+IxTEasVTYLzlcc0mnrPsjfR7M2F9SqxPU+0K0/Iiak2hCbwaDvQnnmU93SrnkYbLu1XHB4Ui6YRcHuKeq5QE6DKo+5InozhVFnnduPSlHIJmHOQQixpnEZGYW+5LzEKyRrfD0h2YCK6ckBGKynakG4ge43FICnwgas1n/rwhN/4nMJqS0oDTV2ThojXpaeosvCDbvM/cDGw3UKqDF0KkAemE4X3LcEW+bKuGy5Vidun5zxcKaypOGs7jnthMavwuuCKu63my2+eE7XnYM9x2FTMJ5ouddywDislvCPmMpyrrWXlB7CKBsO573l43tGmwMHUYuoaHwwhxxKNHIRTH9i0mQDMKphMNH4d2Z9q9vem9CFxV285VBWS4OzMc9ZldA1HexadhbNNIG4S01oxm2gmE007ZI47xX6nONxXLOa6AE8sOKfKwjO2hbOiCEKNwjLavmxGdC7eWSmMdxEh5HixxO8sW4kndDoTCwlOKbVTNZFiYK/q+OAtxeFSuLFX84//yPO94465qUd07gSVWnwuN+rOYHIxm961l+MOcJMvLHmZUu1KEpI8VTjwZEL99J8XjP3dPB5KStyuWBgBL6XlHRkBDKDKfF9GlW26SIWTJxtS0qiYUDoSooxEuDJO0Ri8NEQi5YpDTuVkmEiF+qbGkUP5DS80EIlQRhTjQytPi/B2857M922+5T2lY6KUwlpb+P0xMQwDwzCMhUbZePK4ecYUabtz1tszBp9BLJISOZeUT8llIygdgUTKGSv5YvTyr79d6DAu5gKle5FkF6aTL+7F8WJ//0W7uP5P2RrHi7zLK8i7AmgUnOadAl4KdKcUYOqi2NjpDXIuYJycLf+330586oPw0g3F2VmGWlM3QuwCBovTHj2p6XvFdt1jrCEnYcieytrRfquJUorCwpRgV30+sQiaUeBHJoXCPRhCQiVVCjZLSaOMuRjxcmbwHqsFpzQpFAFfHq0aWgrLII2FKikR8hjXnMeQoQGiaPyop5JNwPex3H8JTFJsYwBtCP0wCmBzWReyxnpoO2h9QJQlm566mlPJBq22JBxf/VaPqTxXlhXKbcE2JcbYbtmbNWzONty765lWc0S1yJ5lVoGpA2o/EpeO+3/qefP3e+bNgpTPi0AtKboQyiIVM9YVKE6SiDaa1abl0rWadAbv3c44VWicPp1jlAMVmM8t02VCu8z5d1viuSl2wlw2vyJhFlROhFzojE0yMAjNLJJ0Q9d3GBXxfWmrZ5+L8DwKk6WAgS4Y4rUly+uedKNH5hG9V6LccyuwzaSmPG9sEiZmnju07F+qCT7QPypgaB8LA2GuM6rO5E7YbiJoxXJesm36lLFKmDhhrzFEY5lPFNNpWRKqWuEODKoBqRLYIgBMjIC1Cw3S2BUViuPJCdII4XzABUOudqeERIylI0xjCJsBugFdTRHbE1VAUyF2jAKPCR0D0WQq4/n5D2k++9UOEyt66chodF32GKMTmUDW/z+XkD/39gMXA23qyNnQ50xNoq4N4oQhWUKr0EvNfFpjj4/ZbM+J1ZzkS4vfotFN4nBmCUPhuQ9R0xgDwfPwpGextBwZzZRI3wmb80gcBlaD4iwIe07T+sS2HRA0TtfkJKxWA2EY2N+vmE8bot/StgMrn1gcaOZTS9xWnJ50eDWwKh4iGmc5mFf0fWSzCUxdYrG0NLWh3YSSbR2h3SRcUhij6beJe9uBLgjzacN+baiNUOlIbRhJh2UWix7RuZJJRkg6g0pkvevV6tH+J7ikyCESxzls3m3M5HGTSuSoSws4AzrhVWFTO2u5ealn/1OGS/sd/+3nKv7k7Q0SYZZnJPFl5ivff6Lc7Qw7z/Rur8gjUT2PbeeCDh47A0+dUHeFhELITyvS81MnzQyknWgqjWlrChUhqrFgGtv2u+QwJRfr/FMbWUSiIQ8BSRpRA0ImJY/EiORSLcekygZvFTEqcoqILnF1Me66MgWSVGyXeRwh6Cf6gdLJfzKL340cLn7vJydmdkCalBmGga5rsWZLSuW0A3KBuB38wOlmw+A9ErfE/pxEIsgO5yxj+TGMtkBVpHE7ncSTC1Ty4Z/SADzZ70t3IF90W0qBoEYU9ZNL9H3bP09XCLtScPc77u65HQ0hj9XReDYvkJ4YSoFgSncrpoDTCm0sdx8H/pP/euA//XtTrBnY9iB2oKqFuEkY0WjJVC4golhvB5QYhDI+MKqIYYXyOsVcCsXkU2Gzj8VTOYWV7pwRU65rMsSg6UOx3WmtSERSGAWg2RKG8nsoA8YIqF33Y1RrZEGPYSMpQ/CRJAWXrIBtHxE0dW1Rrid1isHD1IBVijzaDMvoQJdWsjEok5FsiaajU0IOsD+fojS4LPRKc/v2li/dHXj2/RX7IfLy8wdIo3l0e8VLM/BW+PxXz1HbCe+7lJgtNZUVqknAPTuHpHj7jzY8+sOIiobOdaRNzWIJfU74JEguBZMoi9IRpTXbzYBrarJ0vPdOU+4KG1FeoXNhQOztG+qlR2aa/rbC3ytrPdEgJHKbi7gwl3s8GY9rDKoWvAQChn61pmkMvR9QqgZVOkS+GzDOUe9psijc3DGtBuocSIcKVztEQdwGckiYWekg00L0htpGbt5qqPYMx+8NWJvJjnEzVQRdhLGNK500XQn7eyCi2PSGZqI4PLT4pDg58ViraCYWKoOZKPQkQx3BUjptsisYi+YpS0ZcfrIcCmQL1EJaRXJbiI6RAbGZkugoiEnoWhHXmc22o7JgTFVOiFpQziGxjOtzADrFJ641vHqj5a07mf3JlK23iPPF9SAaH+XPM1f+/7z94JoBBGd2ATma9TqzzYpoBCEQ+jXvnVhOhwyV5e4m0oaErhxx2KBbxaoKdNLx0gsNk6ZhvzGcHvecrzX7Bi5NhbkyDAa6ZIkrONsWJdEQMmdotsoRBdqux+SMGIVvI8cMTA8UXkdc7Vgky8FCYURYtx1SB4wSNuuI7yLNVDNExfFJR+sDlw4ss8pwfJo4Wacy884RrzTbVcKRqcRwuRL2jSKvYaU8vhGaCkKjqWqFrcA4hbYZbPEkJ51RdgQSjWOWwnwvZ/WcaoIXhsEjIY/t8VFMN3YZjNKoXBaTJBmjDVGDyk86NZ98PXG4n5h8NvKHXzZsuw5TxTLrZbeg7/aVJ+p5ueAF7LaDNIrQRk7CuJHI950uZfxogWjK33YigouTf9k8duhdkYykQqoTkTGGWcbQjnFhf+p+u5hbSyKngE1QKnxBp4JC7VLG0hOzI2e5YD+IKMRIKQKwpdVGpkwLMxDKLLyQnp64DUY1+UXXHSHtBD9jXVB+rLJR5AQxJHw/0G5btHLklFHKljFBFBIe7zu2q3Pa9Zo+eLIqXQOLIiZPZOwGxYSzqkySJILo73/tL7QLXAgILq7ErmhTu4JzHAWIXNQSu1HCk2TKJ6Ojp++Ai2IvaUqpsmMTjN9tV/SNOoOdJLXoEMr4IUbPTFt+9+sbfu13G/7tnzc8PDsnR4WtG6TKpMGSpEckUVUapR2bdbESCkXwRR4dDXkUPVJgWZIgpVIsi9l1pGRswQMBxJcWeAiALSAx5Yp6O4UIURNzKgx9yWirMZUmalVsrL5Aguz4vHRkQsj0CE4pYkj0IRGWnlwJcavIDGw7qEzCGVuS/pKFJDQ6ETtQ84RpoF0rNiFxuj2nuXSJae1pTwRmDXd7WK23vP7cdU7fOefVlya0D87pp0LSEz771XMOb1xme+eYalExu2JZ3HKYy472zTPe+obn/HZi5TMT66njnG2MKBSrQeNzwI63hDEabQJt3yGmRivLu98bwASmNqFxdLmnsQ37l8vPLgeZ/k7i5GselRxRBsCT65IxsziosYtEMAW4lcVwmjqyCaS2CA6zSoQA87kqwLcUycmymE3AZ7oQyL5naj1uYUpID/ECR60cxBgwvcCg8X3CTKCeO0LIxCy4SmGmClVDVYF25V5WUaNdpp7DfC/jo6I5VzibqWpYzBTr8yJ0UpVBTQxqKlCX2X3JBtiZAopgWZKM89gLcQ/C2FFzgqqE7iRQzx1SCejSqSJlUhoKgK4Rho0nrCu0ZHLdo6yFZEgSS1iSh+g1EwWvPW/5zkPPraMFEcfDTYeKvghhs8aqH6w18INrBrJGRcUQAufbwGkf8Xqg3svlhz9PPBx6TvpIIzWPWs8mRKZROA89fhWYVVMao7l2fUEYAibBtYXjaBJYGo9I4nidePg4sToDM5RqC1FsNj3ZWqKxTLVl4aCuIWah3w5se880GpwyTCuYpszQK+6tOlSO7M8tUWlOe49tBG2Fto8kkznYc1TW8OBsYL0O9Mmw7QSdhbrJdD5hIlyaJI4mgSNXokEfn8B0bVmJYE3C1RFjM2YMKNIVmEqYzoXJzGBntjx8LoFOZdFWuVhJnJAbiw+ZkDIxKYJPeB/YeMg5kCWQbUC0xmRd4CUhk7RBBYWthQ+8lPjbbsbSnvHbnxf6rUMb/+dOg09v7GWQ8a+BKS4KhvHjRsFLgQkp9Pi+snGMG0PKox1Pl9bYqJ8orewxGCZSMhGe+N3ISV0UA0//ABf/KRmVTQkYCR7EknVhpr96U3jhmubN+x3f/V4gJYcQir4CiyhDcSqMzoIdrVCNKonRsUAqo4wyElElkQ4uoEbfvxmXtzQWAsEH+r6nbVsETfCxOBKkuDl86Nn6DavtGZv2nN53eMl0FsJ5RPKAEo0xlkjREeScUSqML/0TK+guV+Dip8mRTMHQXpAhE6U1+Oe0BuV/pU4r16zAm57ufIzXQo3F2EXkcSnYNEKUAs9BZGRUqaeKg7IhJ8qps8B0HP/gX/T88AcNL16vOV1Ftp1nPpmB8+QgZDSSBKMz85lhvR5GWmfRA138DjKKblMerY7F4mpy6QTlXZtLBOMolrS+OBOGNiJdwjlFVTmUTgxdT/RSuA9BCJtYAASNLguzKrRQYkIL1CL0CL1PxUmhhKHPdC6hrHDabphVDUOOZZxQKVQsr+dm21G7Qmq0YnjsI3/yjS3PvbrAG+E8es5Oe9LWcelq5uGm5/krFTdvzLn31n2u3LzKP/ncHZpkedxv6HPg9Vf2eTTpePEI7A3Hw7OOz/3TDfff3PLBI4udaHzXQm049adYZiTR9ENLUiUCt7KKykEMhaY4mWhWqy05aQ5ri1YwnMF8ISyeAasjeSL072oe/GEk+wa3jMwPHPYoE6qIraZkDeuuOENUViQZUCKErWLmitui77Y0tSPmnn4AbS3EyOrsvDhDnKCnGrtwpLpDe0V3rvAxMJmWTVRlU+ioyaCSR6YW1WQ47ck6U880rhF0o5jODVYFolb0SmOAZiGYucFtI3VdSI5KaSYzhasUIWlSHpkWDrINZJ3ISpciWMoaRUiQShcrBYoYcHwaUgZlFXYK3fGAXincpCLFYXzOQnnOVYFtVc7QPQ4MJ0I9qVG+R6YZ7UqBo7XQ5zVa19w8MPQ+MnNTrj9zhYPtkgf3N2x9ZNOXDvcP8vaDWwsnxcrVt4nVkOgjSFacng505xmyYTJ1XFtYmkbYhoHjIXOy9by37umzcHnieeZAMUGxOR+wuWZaO9rQcX9VkZMndcLD48CDTSpEqZyZzBoqnalNpMmZqWgOlg2b0HL/fItxNVf3KvbrhrONJ/gNZyvPo07wEfamo6sgD2UjbRSbFEkpcbQ35WCh6dqBVdYYI+jBY0SwjcVpIQ4epQOVERZGsVSWSWWKgFAUVgmiIYRYlM+DKuFFGpQVzleCqzN1E7EVaJsxTrBOFeHhpMe4wjR3Zvx6Y7s3BqiDYjskeg9d0GQUKhXl9iCCKEN2Ec2EHDa87xr8Wz9TU7ktv/77A9GXxfSpbjcXoB4gSXjy7+OJ8cnsnLKpx93GP2JZkQs7mlLx4utJ1rvpABeJQagn3z/HUVswjlREFwrXhZiPizb0RcGQdWE56EilSwiMD4lPvlbxd/8NePkw8+37ml/9XeF3vtwTncZGQw4C2hOzLh0CiUUpjS4dgpzIMZbURLl4VUrhkuUiBfD7xwRjGz4l0DAIdNpitEWJgVwAQ0XMCGm8z87OzthsV8UC58GfBbqgeOZyIKvMyWkkhKp0KrKQki7tRvJFiNOuBFBShImyCztSO9//eEKxpSMRVfk59e5kcFH/PZ0/8OS6J6Go6dmVDpBVYJxilB8tj9RCBQkhpzAWS+M4QnYakPI1fE6orHjrMfxf/pnnP/27S6Qa2Kw2ONNTuWKjin487UsRTM1mFe26I0g5acnosc+5qPkVipgLyEVlsFLQuGHkHSRV5t+ioZo4tC7PUgyJofNkH5k0lsmkxvuM73fhUZnYJ1QANzFop8lW4fFIKjNwpNAHU5LSdcuG8zPP0aUaM1O8d3/DlcMp622mrhLZeMTAkCODb0gqUDvN/XuRZnrAzZs1f/on9/jQK9c535xz72TD+xZ71NNjfvJTh5yfPOatB5GHtyNv3VN87Kbjd75wyqd/9ganfcfv/3HH144mvPfbp6zaLdtOsd1ELu875j4RkmboYRsbbl13RO9JOhGDUNUl9pfk8R4me0tWq1Mwmv2JMJXEsDbYWWZxrWCQ0yyz+Z7m0ed6lnsNzYcC5rCM5k5TJPYKfdYVoh6GqnYo27PeePoB9uYOB3RtjxaNc4qu2yG+LaJ6RDS60qhZwkwS2hh8azleRfptZrGoyMGjkhD0GDw1hpDZacG75yEV1LQVUhWZTDKTqS0KfV0ipLVRuEnpIiqTqCtBaYdxmsYFmpkmxOLkcpZyiJBx3r8T4OpUdDW75W5kzpQDxzj+pHBHpEmoKWxWA3pSo1XBeZNSIS5GwGScWMz+QE9idQwP3os89oEXrjVcXSakEVxlilMhd6Q4sGpbaqd4ZnLI/t4BJ+uO+6drTs82/z07+5O3H7gYCBtP1VQYbahcYG4rHJnHbc/DPtNPhBg3kCx9rtBOMVGZs0HwMXPkKmYTjTXFj5uVYp0HaA3tmaPte1SlWZ13dIOQshpFgRofWyocKkbmzZShzZyuerJRxYOsE5sucXy64txHsrMcLooq9nwT0SnhKk2lDbYup+AchCElYjfQZc1skqkPM+8+EjatYjkxHMwUXTcgBhaN4fpScWQdS6eYVZm6UrgdunYHyxFTxEyyO4FCCDB0meBjaaXr0mJSZpx1mgLhqFzC1RlbQ1ULzoHRlkWVmTlhCJqVF/ohEYeu0L0sqJjQWTPIFq8F8Fw/qvnln7Cs+xW/84fFGrULH8qjj/zJaTdd/H2ngn26Ha3Ro4Zh3DjSaJ3Ju41kFO5l9eQByXkMMUqjKBAkxe8rNtQFprWowmUXliQgsvuZyhgjKaizkIeAV8IHXtD8B/+jzI89P2Bt4vpVRQyOtx50fPNOhZjSJw4eJrPA3r5j6BJK5uzvz2n7NcePW9p2/H6733YUoeVcxF4ieVwAuLDSlZZgIeX1GTQGLQY1xlfGVKBIGYUKgdMuUFUdn3ousF2d86++fc5smfj0Ryyfur7gnUeRf/TZc776vYDPodiX1CjczGp8vdRYqjwR6+X8lNgxK0ZcQ4H6jYWMGkkF/7quI4+neCVPkvrKRp+edHwowJ24gyyNL1ROmTjmWogyo9iwoHhFlzS1IihN5AhBZYwo/vnnE7/68oZ/86cjj4Nh1ZfPcc6Mo8aIVrYUcTkymTo6D771kAWjNVFUuf8o3TdyLp73PArIsiaMttWEKphdSehql4MRC+FzgPX5gLGKutZM56bETG88bAW2mb7rsTONmdoCjfGlKLFasJWw6RPRJ7TqCVEz9Jrp0nHuHevzHhHhbK2Zz2ravsVMLW7P8N3vrpn2c3p/yv4lSG0B+MR+TeN6VueB80c98+mCzUnH6x8+5FtvPuSf/949pjYxWMtPfmzJD71c81//8/ucMrB+EFnYCZ/5uZt8/StnfPZrJ7xxtubZfsYmZLqV8NxBwGjYEkitJcZCZrVWE8IACjabDltNsJNM7BSnbUc9dSyvWJgJYuDRFzKr2z1XXp0yHGw59UJ7F4bck4dEZSx2WlHVHmM1vY+cPAoYNHtNTWgTPRmVIpNpgUt1bSLEBLpg7V2l0fOEnRmIie605ezM4Dea6QHEPLDZaCqdSdLTuAoZQU40mjCMqbpGECM0E818z6GcIRvA9xgbMROFnYzrlhRHmnVQT4VsEs00M7ShEE+VIY8Fr1IBkVCUpjqVJMyk2JmDREpmg8huTFfgdMpG3J5jcxzYHPfMrUNPFWnUgCk1BmobjziNk4CTQFrW/KPffcR/82eZK/uafZuwlSX2K77xwFCrGevtlnYYWE7mVNWWadOwd9Bwejb7gfb4HzybICQ6DEo8l6YFu9gNZZE5PJwzOEfIkfvtOeF4xeKKIQdYPfT0KiE17DUl3KI7T3SbRNCORjLKbdGmQntojBBrcDlQoahHdbHNhqaCRaU4bQN9yNhsGPrEyXaNPxPqWUNjK6YuUcuAM7C3J0ycY9IYhtDRbj05Ql1ZVNQcn29ZBcWe1MROsRkGJlOFs5p1KPjbyUyxrDRHU8tBY2gUOF3SxIyWkrJlBecsZoyqzKosRrEovkiJ4oMGUEWVHIqvroSs9LFsnlI+V0zGWoVzBl1n5sYwsVBpCBm2SZNSpjEKGofKnjaDyZqUIyEorl4L/M0ftzw+6fjcN/aZqggyELCQNyQmTHOmVerJVp9HKA7jSTQr0u4uybEc4Hddg4vOgbs4MYuM5L5UCqOcE45IL5qkwcQEI1AjJ0XKAVGepDTKl9Op0YIJFVECkiNR9ySjCF4wUXP58sDf/qWKH3/V00gg28jcan70A5m/8sOOf3Dccz4YnA28/4blFz5h+eDNwKZPvPPI8d655Z2HFZv1QNeVE18aFwMo8+qMQlRAEVHREb3gaVFG40aBXlQZCZ5tvy0Y0pghFXCMsQYVA6su874bPb/8oyvePzvj7fsnfOxDmQ/dnPHM5S1NCLx3D37/qxluD0i2oCJRJdRAsaypjKRIUBXaKmzqSdngVUJlXSx1Ku+GAMWuHcqfIZdCIendKCZfZCzs7B2Ff5IhlfCYi2uZC5NCP1UUkMv9q2XMJci5eKVzCd5RogoDYKexIFGGSnAeNP/xr7e878qcj34wcX6c2HYRrQLKZQyK4IuupHzvEjHrtKPrEsOQCohyvPWSTlCVZawfBJNKESVS3AQhFEhRVolsFOiMqTU6a5ItqaSDzwxniaqCpkns7RnSTNGuE+1KsT1LqE1PPTOYWhe3DwGjLLMqsZWBuC0UPLMw7JlMUInWCItFQ+97VC+ceLh+veHkODNxFXtXLc15Yr0Vovc898Il8kzxR3/QY5XwZ392ynMv71EFjwqGZ2YN1WHF8e2BG1cm3Hqx4YtfPuGVy5Yvrg1z1fDXf/mI4XiLaTzGWE7OJ/zsp+f87m/f4/JzU5orNbeP1xwt9+iHR9g8ZToNDN6SE2Wsoih6qi4im4HZwRR3bUCOBtg03P7dM9Km4uC1mlMGhhMFqXRjJCv2lpblXsZUxdly92Hi0eOBibXMJpa2HUBpjEo4m7BVxWo14AeKDTOCm4HbDyjniAOcPRxYrzS9F6oq4wdF2BbWSS+K+VKDySUYypgSH39O+TidMCbgXKEFiu0Bg9IaXSXUTJGmBtl6tM9oq1BVRCqFc4p6Zmi3heCIHotxgaQUehxtBjJiSw5JPvGInxAXbRGIx9IhSOJL5ywqXBOxC+H8XJHvR+p5wh6a8bkcwJR9Q48jVF1rDo3w0rM1X3x0leOm4o3jDY/vrjg96bkx3edwf8s6t+jBgUxGW3diJjDZd3+xxUC1sGQ6KutQEuhSpFsPXHr1ZVbnZ9RK056safpTDm4+S9+v2A4t6y5xvC1JTNXKU3eK9cnA6ZnHxw03ZjXXJw2TlAtV0E6oomBlQ6zA6Ix4aKrMsqpoksargJlrlESMcdTB0SvYPyrRjscb4c5p5sEmYmuLxEiIPd0Q8Z2m6yN96Bm8YjHRXJlokh843kayNcyt4uRs4PEqcmm/4WBuWaiBiclM3G6BAusyxoI2pcW/sxlqQ/GQjiesqH0hAyZVWpE5j+KvcaGWQMxqFJEJIQlDTHRDol17ej1wLznQYOpImw3fui18493M+VBxY0+4ddmxqCNJEpOJYjlLuNrz7LMT/tpPwf0HHe88cFTTCaSEyg0qG3oVxs1fPdnweeoUyZP5sWZH5ZOnxgiqBOSoca6tSoRoUdkqUoooyTSxIWVFUluS1iiZFDhRFGxKhNRSxylBPDFFvPQkYrHtEVEtZANBZz76IfjZjxqaqivisQxZNMvL8NOfED7/duSffxleet7xt/+y4idfDdw4CIRc8dbjxG98ZcuXvjNwvop4VayksrM0ymg/zFAcEJmND1xaZm4eBR6tE/ePpRAdlSIkIQyRkDuScvi1YiqerjWkkPjYB874n/wMfODaBrVecW2pYZGZTgJKB9RQcfZOy7unCnBYCmAphYSRaaGKocqpVkVi70laiLr4kkf80MXGDiU3oJzkcxHOIRiKlTXnOI538m60XrIGdkyBMThJIWWxy8Ius4FU0L1QuhJanpAsx95QCfsZ/16MGMUOqsVhVODBo8x/9KuBv/8fKi7tRdp1YjNoFtkhJmLEk70m+6oUigG0SkynCqMT3TZBMuRc5rWKSNYDYoXQK8KgMcphKK3bEISgAjHFMfp2zLJwito5iJnQRvw2st4aBpdxk8R0KcyXin6lWJ9FNseFEmprRT0tGqEoII2hJxO6xMk7G6pDy56pUWbDYlYATJtz4fKB48GDDmaKmQj9ek0YoPMNh5caLovn2o2KrzUlKM1L4H0vV7zz1Rafe15+ZY+73+uwtqKZKdanA2d3I5/66QPeeTfyEx+tOTqCk/OB2aTmhUuRv/FXDnjY93z7nuLjH3J8470VN6/OyH6g7QU9TUyrKZvjHjfXuGVCKUNYD9CXLmVzRaGOLJt3B77328doZbn2ccPKr9AizF3FEBqShsmiY3m5IifNo+MtJ48D26F0yYxNrNstkoWmBiUeZ2tCULT9QMYW23AaaIxDO0O/7Th5GNmsyu1tbNEgbddhHPVpTK1xTUaGkmSbnEZSJviA1wNBBYzTRJNJpqCjd4A3UwmqFrIDaUcstRmdN1KEpraOZAnEkMdE17JOiFCgAzDme4xiHQG/6bDOIhNIyo/kytGxJIkk0BxETjeGbeu5u+1ZdAdcezaRjCGrDm0rYtejTUVKAZUizx8oHsU5zeFNhmuGx2dnvHn/LrWHPbPHo+0Ku5jTTC2b7XR8PjJK/WCigR9cM+ChtoY+eFLyrFtP/ZFP0i1r8ltvcR4gL25w9OKrTPDke8cMOeI07NewXxlMguPjgXfvbrGmwUii6yK2qvH9wPFmQJSmTpoSTF4Qjcum4tBAo2DdB1YhcWAnLKYKzJpJbbCNYCrFuvNMh8iVfUEmjpMt3D3uWVaaS4sJQ5UYukCKwjaWjW+GgRTIKtF1mfVpT99HrECDp4pQqeK1nVSaxdxQN6rMO/V4E+mMsRmxRR1fBKLj4piKUE2UYEWBKlnkBSmbsKOILabiL4iZcU6lSUnjB4ji6Bjz0JNwram4axPv3ev5za86TgZHoyxaRWaLwDNXLM9dz3zklubWtSmf+eiK/+qPerbDnEnK+FyNjoRMGjeVsobvxGBP3nbCsR09SESP7H3KbqLNRaKhkqdzDhQhp4Iw7SmWMT1BR0ZRUYetNSkbTJwjY+BJzOBUIsQpWUUcDnQminDpIPBLn6q4sb8FI4Qhl2DGesBO4YVb8OlX4HsPO37xE4a//CHh8tLjqkidLc+kLUd1YHUSiGtNVZVZnpLRk5tVsaXpiGTFJmhee1/gb31a8cFrmj/9bua//H3hu/cNLgVS2iJaE4Kw7hJVs2V13GDqGZ/50ci/89MtL147RdPBXKhzQqaGbIbSVvQ9f/hdz+3jgBKFN8VeWaHYWkU/lLgpJRapdcG5KlWS+cKOfMZodthdtxHFLE9ChS74DfL0xz3heaRUMkDiOPjUUgq8LLso5LEA/L57Ql1YJ3Uu486dkLAUU+N4IkY0kSEFnLX8yXcj/9GvZf6P/94E51a0g2cbEhM1Q+lxLi09cVAjcjiCBJq6xhjYnHWl+yIWJ0I2bhzVFgHo0EVMtqOVcEAGTeyEVFEcDCpiKJkH2glMNHEv07eRoctsNtCuBWcCVWXYv2yY+cTm3HB+NvD4vkcby2JpmO0Lk+uWdh2IqwHB4dDYkOj7Dba27N9ynKx6Gt9w9Jxluwnc+d6KV167zB9/9QFvvBH51CeP6E5OuXE447kPWYaNYaYCB7PIyXsnHNzc59KBx3eJZQN9u+bTn9gnOM9PfWLJ8elDvvKlKf0ZHL8X+Bu/cIX9aw3/8X/yHj/2sTmx0Zw8qLiuHd9955QYHVf2M+f3E3ZpqfYTjUxY9StSslyZOOwrBplG7v3xwL0/8iRtuPxSxfGDc1Dzkh6bEriEsYHJtGJ1PHD3UWDbKYyxGBVRKpXxl9LUYtEJZpWjcoqTsw7EMqRICCXSOHjFw7sd/SbTb8uhSVQPKRGTQ+WSiljpwGwBMoH8KJGChQqk9/g20ecxXCtkVLYoqUFFMsUloczO3VV0PVkJylJa/wBWYSdCVBE/ZBLF1ozKCIlRGPPEiZQz2hqGoSc9ylSXHNKMULlRiJuNASL1IjNpAuuHNTSaz337Ma91B7z8SsLUNTmBMpbkizVd6Qpne0KC7JZISuwvLS84hYqRly69wOkmMqjA0G8REo4S/hV/QNDAD1wMzA8mdH6N6Vo2i6vk97/CnWVAvvoV9OQKy9d+iIfuDT7/ztdI72354b05237Fa4dLnKmINtKmRIjC1DmsU8yAuROQgYGINiASCHFA6YxVCpMSThekqzZF7HK+jZyuWjpf8+hswDSR5Z5G95o+FlWnjULl4Uhn5pcaPNAHT+cTzcyyrOGgD2zbyNnZgCbh8Gx6WHfC3n7Dcq7JXc/ZqmU2M8XjTgmXmtTCpFEoA1ontBNUrZFainhFRlFWzqQcSamo7RNCTBofMzEVbcRGdm3VAntJOZJFRryuwHyCCp5piGhj0Vq4Ovd8+JoifNTxu19U/Ge/3/Fw1RCNZVhHvnk70riaP1haPvzawGI25dKh572HGhtL6EXJU9elPSU7kdmf5/mL7E7OUIA948ePLWVU4cGXh7bAeIRRSxETMWt0tSFmS44OLR5jHEFqAoEQNJXJdNGTxGFsEdBpbcr8PIN3AylZXn9J8RMvOpTqSShM0mODpVTAi5nloy9GWhF+/uOJq0cB7SJiNDkEZrXw3JHn0kHPO/cthkgi4aPFKMFQBHNDMGz7zOvvD/wHv2T5+dcjiypx0Ah/+vWON+8bYiz0yeS7Yq0aAg/OFLee2/I//je2/MonPTcvedAtMQYEhzFDCeeREiTzzqnmn32+o+8cRjdEO8Bg2YSBSj3mhZsNizpx/3jgzkqTVSKlQirUeGJ2o0AAGMFOO81DSk+AQCWsaXdByx/l3xlFjxBGsSMwzkGlwHdGbcH3QYWkQKxyGk9JlFnnE3JhEZFGBJsrAlucaDop4tl/9LnAjUst/5u/XmPoyD6zac+ZVpasFOIyRjI5JMCQotBtPNoo5suavvMMbU/0RZuB6BINPnZ2QjsUCqXTiIY+JfwqM6yKIltPIZky09VisSZiFxHmQgyq4IIT44hPqCeGZqZYXI70Q8P9hz3feuecw3bBC0c1y8NArBJt29O6yPKZA25/+4Sj5zV6r6Z1axY3FTZFLl12aL/PjT3NT/zIjHvf7dmcthyfR/avTjnad9x5dM7j7/ZM947YOwjk+wNv322ZG43ynkd3LeaZDQu1QF3qeOtbnldf07ztE6/cEq6+b85nf+dNrmrFT74m1M/vM5z3fPWNM3II3DvpOZzOmF2xHCx6jIJ2W07rzXWobs5Y3dtw57c3rG4LORvqZWDDwLYz6L6lHxzTyx60p28tm7c1Pg7oWphODEOMmAzaVEW3kgJaIo3V1M7QbrfEWMLk2r4bQVWwWvsCb+o1JfdF0LoaxaqFJUHOmErjlkXLq3qDigpCZlgFhpiwc2G6N6Geg6syWTbkGFDaFmuvoRQDkgipdHqwpSjIlNGSbgSMMAwQUy6jwx20QxKFKp2LuyUrMAZtBlbHHUo07tCSm4GsEsRRI6UzSVU0lwIP7/csfMPeNPJrf/CAn7w35Uc+skdzOaCdIFbhadEY9qeG2lE6Bbl0x2Z2Ci5Q24pbl474zuoeMQXIHaMa64mA+C+qGPjCzRnXw02M2+cbsy13uzcYHmg+/szHeTRN/OnJl7i7usO9U8dz1ZzJqgcmXFvMkeGcrs+4yjCrhb1DgzZg/Gj1yhCSUElgYTVWWdZrTxcKScmT8ZXGJ0sYEjkk7Biao7VmvR649wiOu45WKaaNpmlq+jAwsSVm9PEm8Oh8QGfNfO6Iscye0jDaiiTiU2A5U1zZVxgb2faBdhuotcJpg9GGLBCiB0m4icPWJYMAU+ZGuGLrEjWeorKQxZQFOI9q/VQCdVIqC2cvRSkdQplhBl8W5zh+fAwbtNJkAz4nsjLU2WJtpHKBw8PCOs5aF0UsFTZC23u++WjNW3/gOJhZ2m2FEuFcK2prUSHipULHMUt8J1GTJzZDNS7q+cmRsNjxRlxvKQbMiN/V6DHoZ/c+pRJzm7gynzFEuHvuiUPZTCRqYo68eMvyQ88H7hxnPv+dRJLJGDKU0NqRCVhlmU/hk687ru0VK2ZpYXuImhyEWBnEwrO3EgcHmltXFKbajjNmVcApSXjfVc3HX3Z8645ivalxJLT2pAjrHlLWLBeRH/6E59/9acdn3p9Z7gtGKQ4OhCuXoDKxFJ7Rlo2xzwQd+dEfbvlf/LLhU6959qahnDTEkING2aEMZjOQMjpW/NYXN3z93QlWCzH74kYh8aHXLD/zSsWrzzrmLvCn30j8l59reXim0aII3hSRnM4Xp3VSvhjpPA1teroYUEo92azHRM0dajinJ64CNX6+EvV9nInd98pjq9RIWUKUKn7+wpWIT+51pTA6joWGQgIECRjj+Qe/FXju+h5/5adquscZZweG3KNpio0rD4iqEV9QlhpLHBJZAo0rWRDtFoIvWGRSWcTFZFQNsRN8l1A247TCVZo4ZHwb2baeTmWqRmMnPaa2ReehMq4u2OOsbBGPloceybGwRJrM5KUJ9SXF+QPP0G2o54qEQ3vP5GhGu+kLWGgIDI/OOJxm+tRh+jlSG6rFKSfnltnVPeaTR8TNwOXrNd3plsdvdiz2aiRY1F7ivA1w3jHLMw5vOb7zzhkvvzLnzp89Zrk35Ztf6pjsLbj2Ic3Jf6eYzIXvfuEd9qs5v/CXAjRT9maG77078GDQeATRLc0scfVmizIOoxKz/Yg+mLBuE9/+jfs8fqsAyCQKWQKqcWy3HT4pmoVmcpDwCdpHmoAwWyaW+w3r9ZbYefTYNfW+B9EsXc1MMkYGumFD10eMnXJ2viWhECnBdCmOYryUxyjwsfslEaWEfmip5zC5NUNPIJ4mhhVoHdGTjJlqli8bmJasAJWFuImkNcXlo8o9LU4KOGgUVWc9Ei2VlM3banStwELXB2JMGDWKmkWNTqNRWTwWwkLCNhYlMKwztpIyQm2KmDz5iNZA8Mwbg9pXPLrb8cx+ZHEk/MM/POe33vR88pbihecbXrykaZYK0YawKeFW3hgmAXqRwjNxikfbx1zf32NW1XSDp5NIjBFSwjy1nv+FFANf7T33loqz1bdQrmW98VybvMg3+vd46503WaUNp+eO7Vbx4qGhHs6YBjg+vQ/rzKYbUJdmJAxaBKuh6yNDG1HOEbaK3mes6TBmig+ax9stVilspVkoxcm2ZdsmQrTo84G03aIXmevXphw/CBw/Gjg+T/isaWaK/QOHUYYHjzpCL+zPG8Rnzs4GTodMFyhRrzqQvGAUXDm0VBPN4+OB+48jE6O5caCZOBlpaMU7XC0sdqFQjYB2o4q7jBqKG6oIqAjjDNqocqoe3QSjLJuYEiaUdnovjLz1Emeixsx4pUquNQpsVsxR1CQigg/CaVyR1CUyDTaVBDCthFpVhDQhBMW9046pFAukiB69f4XJf1Hoyjgr54lyfTeHHneHsqmOccyFtljQvkpM4Q9oi1KFza9UQWia/cQHn4n4fk3/juJkdYi2A3HwHC4zf+eXEp+5lbi3Ufznvy/85h9GrK1ResSD2hrrNc9eG/iRlzO23o6qt3K6VtEU+45ViI1cuQSX9xKuFlA1WZmiPxgBTodz4VPvU3z5O4YvvAlWOfygyRZefaXmU68N/KUPbPnQs5ajJTQ1RakfEmaSObicmFhNv9VgezadYn8v8dd+xvM//XnLSzcH6nmx+MkQSUNGxymiM9EUz7UMmTduR37zjxJtgmg0bRu5sq/49McVP/sRxUtHlkvTHhMTjXZ89g3h/klJ3dTJj7STXU7Azg2RL5wP+en3jbt4SomdD7G8X42F1+5zRhEiu0VvJxTdCQvzjn5cfODKlII1FodODmPLNRVLqY6GQVpMaGhRNAA5EJNjO1T8n37N88xyysdfC/TnoHRd2rpOcG4GaSgGFV02ZCWaMAT6TcRWlulcGNpEt4kULaMuxEtLOWm2mdiVQkfbhHXFlTC00G1gc5KBUJw7BsSA2ICuMjIW6BsvJFFcutRgdIGPhSFzcDhhajp0p9BDA0Mi5MTxnTWyVLil8K2vB/ZfcFy6PiM9CPQnHdv3thw8M6WqE/FRS105ZgcL9g4SD897Zu+b8uhxT3/c8syh5Y375zx+Cz72s1cJ5ye876UFbtry7IeX/NmXT1hv4LWPVTx8a+DuGy2v/9A18qzl6MByvO34wh885q+/NGV5lLEDDNmj/QSvHWe9YR4zqzPFyVpzenxO+7gn50xtHKgSgd5MEjkNGO9wLrOoHOt1R041ogPzvYypHScnPTFonFWokiuFxEBtMjMdcQIoQzt4ch6Li5AwrmRbSDbkoAjeI1kTiYgpnS4tCj945nuGa89P0fOe2CokJvSRwi5qqDvEFJFgCoqYYNNHhnZgqhwmCYqIuKKFwmZyWXhHsWmx8qZRJ+MaQdeZbefpu0yt7IjALgWqjM/JriRPMaCcpq6rUlDoXNwxvZQcBQMERTI1SrXsHwpv39si25oPX5nw5uOBr70befNBZvHdE67MLQeXaixbBp+pnlvQUKKmAwY9KIyJbPuebXfG0s3oNxucnRJTYdPIXzSB8Cw/4vajB0yV4+p6xvY88d3wDo9VR201g69YPUj4beA+DXf0AdePjlhcnpA2matxxcnt+5y2HahMvc0MQ0YljUhkpi1OW3IKbGKPdpojZZkoOKgVi0lFHjKzWjEEw/H5il4XBaZrGl59Ea4947h/33Pv2PPgvQ2PVOLKUcPexKBr6IfEKgtbpahquLJwGAvbfoCQmRlTXAceuk5KBLAuuoVECSoJGLAGVRUcc9KBbPOFxLmgX8fW6Wixlw7SNhVwEGr0QBdXQEgQPfiQGWLGj+t5yIkQAzkXlXWtYG4dLjtSHEiqY8iKda9ZTB2XjzwPuxUie5ATvS0YXpMVkZ7KelJUZAwT7cg+krVgiGD0eIosmoByk4+bixQ5YR5tgQVIM7adVLHkKFFjR8CUP7UZw34UoiKHJvDKLUPXDrxxDPdPywza5oofedXwCx9ecXPhuZ4cXQ7cfifzjUcN1m2ZppokCmWF157VvHLljKTK/DHHVCKgw8irDxFlE7YqsznRodARUyDLMNLqHPUk84HnhZ98XXPn2PLw3PLSS4G/8unEz3644+alQDMVrConFEnlHkALVmsuN5mjWrNeOYZg+PCrkb/7i5bPfCJzdDlhXEY0BX1bfAmgWkRDRBc6WIj8+p8KX79TGPZ9By8/Z/mbP6r48dc11xZQu0BOipAU1TSyPzdlBopHxvapYItVVP7cI1s2fHgy14SSgTBu/CkVzQo5F/8+T32ZXNITYewwyPibSLnsxlm0syAFdxp9opCvCwvAh0jOAasDgwjOZaZZaPMGGzRONKiB750L//tfW/F//981zI80+nHB/fZdyRioGwW6KLezSmTlqbQmDpa+DYhEnDW4uaXfJPoujx2q8jtlW0S5cShFgUrFW26cZkKi22a2G8fxWQTlSSj6oSQjijbcPe64cxZoY+KTP3qNW1dhEXtyylgNc1cgYauTNdO5o96v2d5pmTY1toootSVsHVoys2lkK5rzoBgmSzbHj3nn6/fZe3Gfw33DnTceM5wrps9rql6x7ivufXOL6WuqZyskrPnWN7Zc3q/ol4pLRzOO2we8/pFrHFy3nDzacOt9Zf6/mCpUrOh0z/s/DKuzjsq3nOcpV5cVv/OlFae+5je/3LBdnzEJcHVPcWWmuTSzrNc9sjCo3tMNLZOjA7zf0jihnlvOui0pJJTr2D+s8Xi2Jx1aB6YzQ/CKtg3kbFEJJjbjTMSHxDAofFZ0fShAN1chxBEfrYr1fGRKRMoaaI2DKCwOJ9x4YYrSHWld1n+ZCVSBgTVIQo2CYCERtpnhLOOYYNBoLSTtyeRS8LlI3ha7sNLFQRVJhJHl4RqhmSvOusjmLDC74jDTMsYdLRQXXVNhlFXphG4UuASTjOzyB8IotJaEagNMFXsThWpqhrVhMmt5YZk53UBQFVsnPLJ7rHvHZFazf3mBWVgknSFyjRqFVoYUOnwSjjen7Dcz9HQB/UO0ooyD019wMfD4vMaFnjAzvB025NATQsJPHDl6Tt/VbILB58Cv3z9Di+LK9pijfsuHn7/GoVpgvnuPaYZBJ2ZZCKaiTbDJHiOppNOJJqCoJXN1UrFsNHWlaGLgUd+Sdc3B5QWTxUCbArquMKHj/nnLuhe2naIXYFLR9j2nPrIUIfXw+MSzTZH9qWHZWKSCPgemxuJ9ZtsPrM57/JCJEpjOFZZiicuVQdUV2WhChOATOZbA2EwsrXUt5dR/IcZXpZ0thrgKtNthjC5VpCyEXE63PgtDEIYQGVKpgo2M+etWM9VPIVajxwJ9cJx2kfs+gO64Pj/gTamIosH0uFgXZKXrkVQxZItVBk2LlwiVRutISg6VIqJ2p0Uh5zGfgLKJ7IyH8jSMSD35/y7mV7Qq8c3jqABdrDRBT1CTlpcvW25e8bz7eM2X34vcfqfhEx+KXD0c8JWnIvGR9xl+/OOKN38nY/IcZcuWWlWKF2/0LGfrUceoEKXJehRi5oyOhTZYLECRlAWVc0n8EkdUYGPCKrh+yfFjryq++D3hOYG/90vwqfcH6qlH6tJOF5+QYcBHhSjB5IwjcLQM2EYTJ5Ff/BHD//wXKz7yUoubB7QRMo6ERw0J8YV24yuNNoIrOzDffqfm9/5kxbCZ0KXEJz+W+JUft3z6WuLgwBDMmtxBzLZAdPKaxhisxNHKuSBzXjZyAXKBUJV26u7hL3a/lNPoYR4P+WMxUOATI2lQdpXDUwsbYydBpNzbo4ZFW001qakaR4wBkIL8HTTRFJaG5BIM1A0QvGKTPSl4QrBl/ICgdUaS4V+cZP7X/+c1/9f/8JDZ9BQ6S5Mt202P7z2TaY12xd8tlHx3LYmJKOLW4DexPGq1QRlPt2khGJRYovVlkcuKlAuutu9T4ewbUDXIGGjU+8CQWuZH+6w2HeenW2JOPPPMjBhhqTrqTujOMn0nzA9mbNKANgOTxqIqzWbdU88TftPy9nd7bry45NG7Pfe+U7F32bA4qvjGV8+4dnnNYjGBD4BdWmppeft8ywvvu0Rm4MGdDYfPGG5/z/LRD+2zyg+482eRb3zLE18Y+JEfnvOtP7nN9fmMK88v2b75iNMHPamz3Ptey/1ty9HzATN3PDsc8uh24PrREV/52imN2uND7wtcmh3y3IcNv/WPTvn5X7jBt756Tpiu+e6Z5fNvwS99pCf3jsb5Ei6lasRluvWWHBu0g/lBZqAjBWFWR5bLOetNYr3dkixk75k7YeYs3TawDZaAIoaOKELO9qJQ9T34bV8SXJMeBX0VykLOifm+5cp1Q9s9RmVNoxWpjkSrUUnh8OVZiCUettsEtmeRIWZS7nHGYrSgNWVMUHuyiQXwlaSsX6IJGZIock7YCUwWmtPHmc2ZZ9gazHSnvyktuKzKmE2LRikIKaAmpqxDLiA2IwPkXhCjibpHKovqNbZJLI+Euyct1fmUw3nLlSsNs+mM+kAx3T+gqS1YTZQlSjT0NWoimOjBJNo4kE1m67fM+3OWtuF7m4S1Fq00Ev6CxwTb84LorASMZNYxI0yJp4bTs5ZucBg8IoLVtmBa+4H3v/AqoT/m0Re+yWRouXHlqAh7CGy7gWEQBgwhJ4beYlzGSiBLxdxOmPpIxBBjx6aFVENT9+isaFcVmz5w3g0cPxbunEakylSzzDN7hjpVZBSPTwbWXaZqYGosSoSND6RB0acMqqdrI2fnBWF6MNdMmwYlmYVLHCqYKGFSZSaTTFXvWAKpUM5QRSiQi7VExlhiSCTJpGnZFPMp+C6SRI2CFU/wRRizVaXtPlGhgIycpTKKCYX01qtQ2nSp4tTDvWHgcasYfKIfLNs2oJiSCSQcyRi0DkgqCl1JpQ0muUKnAFIwtyKJrJ8SBciuQzBqCHatZHkyJx67xmUsoPS4QO9Eg7sRgUGUxuiCId2bCp98Zct80ZPywOa84u07x9y8vsS5RLCJIJrlXPiFD0Y+9/WO2w8vodQ5Kjv2pplbl0obF8ArMDkVH28eA5zI5QEMGZImx1BERyhUGj3COiAKnBhevjnw739mS7Oc8IGXE83cj+uIoEJxPJAUNukSuJLAiWbqag6Pev6dTzj+zmcir9zqMfUuEjkVolgQZCisDcSipfwcUQTdZj73lZavvdvg5om/+jHFz7+eefFaYjYTIi2xL/dmlogiU+cpYegIqgIiRpcoZj3Cf4CxfTlePlUCU562S6axnZl2VAmtRuFTHqFUTwqBnDND9IjSTBN0WlHFQj2bzCbU0x5dT3ARsni6EDgjsfGZx11gfbqhEsVeE7m8p9mbWqa1R8tAZRWahE4amzWDjgxdxz/+7Ia/+TOHWDkhG4ObaNLa0vmAmyRsI2SXRuhVqbp31uMwBHxfOA/WNPhRFCklcLgUdzaMMcylzR984YNMJglJkRQzqTE018AExzI3vNBMkDxwcGRQsWPz7oY2G5gommdAe0d3L3N+7tGTTLfN5D4yP6x45hpM9wXxHZtcM53VqKZjdljRmEi1lzhmycM7Z+wtNWeryN3HG6aDJeOZzC/z8quPIfQM7R7JnfFDrwZe/cwe99454e3vZH7spw/50uffY+nhmY9c43P/n9vMUJiV8Mrrhzz8swd866HlA89XfPvbAzMMzz+jefNhw7Jveem1CdXphI/90JwvffERD9c1n/1Gx0efVzxYC33b8/pRTWUTXjyhywypwtjAweUarTzeJ/b3Z5ga7j08pxsqlHb4INQyUFeZx9sWL4acBkgKYxxDjNST0avfG7ZnLSorjMpoWw4syhi0BesE5wLdNlA3FttYsAlNRsdy82ZlCh0zZ4aN5+y4WIdTiKiQ0Ga04uqE2BIjLOKKWJAMJo9Cv1IgxxwLgnoJqUncPc0szzz1vkPVEXKBYamkSwdREgnQ4sgmQaWIOqKxKJMIQ6Q/jdTzqugWco+Kgp0IqlGEeM6+NNy8MmN5+ZA0UYiaI6SSbyOJmCKZitj3JBPJPmCyocuK1m9w5yuO9iZYHKTCfKnqv+AI44OrHSJVUTJ3NZYJbRcYNgO2GKdQBvqQCcmjjKJNgdt373BYCx9+8aOkt98sQRsm0swMXYik7MAKLmSCa6mTJSmHRlAx8t7pij5qLi9q9qZ7mGUkhUAvmnqaccnQOsOlOmCXkeQVgiZlzduPt5zHxNRori8cYioebHveOw6cnnuaRnM4U+QobFdlcdmbOubOE2KZTy6sZlnolqSh7DfVRFPPSuBFUkUcWOxo8lRXQMYNt9gOkyjSYPEdeIRAHgl3kawiSxtw1mCNLchdJXgiPnpstISc2QyazTpw7DMnUehDwDnHWec4HTJ+TExU4wmxnPRGmMVYfWekbP7ACJS9+PfdXONJq5gnmgF5qlgYuwNKdLG8KbmI/0UK9U0rPeoHBFJH3UQuXclY10P2LKaa5aHB6LYQGSWjJIIzPHtT8+nX4P/xez3aLFEklsueZ/ZCodsljR03/qzGTSxDjhmV0hiDWoqEnJ6I6RSFeggZbeDgyPGRZbEP1ZUHSkqfxFJkSEzk7BGpxt854Y1iumj5Oz875bUXE7eue4xTT47SjJTHlCFKiWdwEUml2Euh5/aDit/4inB1Cf+DT2p+7APw3H7G6hLC7AdIXghRMFVG58JHX3UBUTVKA3lkmYu6GOeUg3tR/8k4GpCd7189pQ2JT4kOy7vGiOlduVBuHqUUVYRswXqDbjqmk0OMDTTNdWI1YUVZ0Jzx3Jyec9RsuDaJ3Jppnr+seP5IFSaHBmstaIvoolMgCToVjkRmD9qB7ckxRtXjSQ289qgoDOcQzgRtLW5WTl3JFAaDVrpoBcjEQRFyRkvBPIcM2FKUSxacFkwV6ETRdZF2G4sFrnGoGLh8dYqtAvtzw2ZIYNbUSrApM6lr8oEwm01Jqw25C0wPHcYLt78bOV533HzpgPNVj94kFi/MWHcBMQueuTzl4dma977S8uoH5sS1x1Jzue6oF5nKGV58fkr2ipPbG65/8AqLqxUPv5E56U+48r59Tt8YuPLyPhJb7nyrZVgsabNw+s6aD/0Pb9E/fMj1peXx2Skf//FbbB9teeuLLdO9yBvfWnHvuOXSYcV7jxN37njOm8gHzTXcdOCdL53yzOHASS9cvZa4v8lcO4Chzbi6JmtPDJmha8im49rNiqCEzQouX5ugrPDOW2uytqgq0rca0pY8qXh8LtTGYEzG56LdyNIzmQhkT+4M64cBHWpcDW5SoEXKRbLqS6dRili8cgY3SYgNEMsmnlVB5RMUSmDbRk4eDWhV4WNiu9oync6K5dumIojdFcx5fFa0lPVOSRFNq1BGvKqmqQKuFr76sKX9euSHj2DRRMxuzcsFTV6WSSkjNXEkBrQISSkkeYzLvLMR9taK/WuRqEvez9z0VJNM6CzzXDGZzKnsJWCD145oJgzDGmtcGc3swGF9Io7jDN0nKoQ+tMSu5WCyZN1uMFjS8BfcGViG62yHQJcG2rBCbCJXGe0h9ZqgEkMcqCtLypohFIX6W7cfcb6Y88ic8uqnfohrD87pv/EV+rMWmxsIQjg7I5uaWy8/Szh+hPUZt408HDq2A7hZTb9Xc6gCthZC0vh2zamPJByGolLeF8fQKO6erAnJoZRGD5FsE2dsOTs13Dnu2XQZp2G6UIhTnJ8OYC0HM0tTJYIvYwCnQTnNZK6ZVeVG67tE8LvBaUKZcgLLqvi6d6r8nbikWAY1KhuiH4hdRNnMtLJoVWGMIltLVuBQ6CS0bQk5iaI4FaHrMqvo6QZDHwZ6BrQY5jrQbiPfur/k4QaSthiVUejC/0FQ2RFzGFvIo+Al72bBQiKhKO3/lHZ7/9PiQYBw0Ra72GRG6Ifs/OdjR+SJbkKhtKC0QasG63ps3SM6jkWGZ9JA8j0pu5HrH0k2sneQ+cyHMn/0nZrvPJhRTTOX9jIHI7o04YvaGCEpwVC44BJLimDqIYQinVOj8ld2+3Ue5+QErBNsnQr3ASDZcYOlwHcCqOTIURGzRymPKM8rzwivu0SzHFAujY2U8npJLl8uD5kcx/wHKYtMFFC95bf/NDGkxF//S47PPB84XBacbzuUBzumMr9XOuOUwSVD5zPH68IvNwoiAYO9GAnsRjffH/iUC6yIglUuwJ3CVjfj2sc4YtmNFXfsCKVAaQE7Y5AtlbU09SX6ZuDS/pzZPHLtsOWV/S03llueXa7ZNxsaG9G6CGNBsR0oYVpDmRWn8QUqOOpMnwZMrArQSDv6lBlSh0SF8Y5kxhmwKFJQdNvE5qyjmRrqmStRslGXbkEKWJ0xUeE7jfQBlYrjGhvJkohoYhoDqbQhRcVmiLgJTK5Z8ixw/7Tn6GhGszfhvTdPOdzTLA4Ev4V+0+FUYLmwbM4zDAGrAjeebzg9n5B8zf6h0PeRoU3I2RZ8QiSgEa7c3GN+WPP4+B5p7tBd5vf++JxfuXKJra6486XH/NBP3cTpAb8543Nf7fnkj30AvXmHs5SROxtO36t47WMHvKIEsSsOn5vz7a/2/Ff/9JS/+qNHvPic5XvvwtnmhC/cDfzVF/f4J3/8gJdu3CD4wNZEfupHGv6b3zpFx4wZWv7bLwqJmvffmLPdPOTf+qWr3PM9w5fW7M8npH5Du7FI3HLpxqzE6nYtTRU4eyAMQdHMLOIS602G2KNry9BGGudQkvFpwM0cSjRWijgwDhOOH28R0cz3FaouVkXjHGRTBM86o3Qk0hOZkIIhElGVIMEiMRH9gFKKfqU5vx/RqmYwmdXa09iquFMkls3eAqbcA5LHInnM6RZGdgeBlIpYsG6ESWM48YE//pPE1VsVrx8MZb2jrGVlhKvHLnFAxYG4CWQLqkqQLVEnfDY8vLelqjPT/Rnb4Fm1Cq0qsk0EOacd5jw+azmcZOb7FeeDwqi6fF0zAUZXGRCVkHJBlxtlCDly3J9w+dINtn1iiF2xZ/5FFgPn+iFBg9VTJNcMXcT3gZgtGIvpNyglVFOHT4rUFSV/tzlnazou3djn3v17HNc1Ry88g/7Gd1Ap8PC0Ze/Gcxx+8jm+fXbCzffucV2ErXYchwwmsb+wHEwz7VngeJVZb9e8t/X0dUMYTpnvNTTO8t6jgFJSqHWSeeZwzp4WTtYdt+/2tB4WE8v+EqwI00rRh0w3BlF4Hzn1HoVCReiTZ2UjUjUs9y21LfaWrvVszkrCmplGxI5+7l1bdtSYlpGSAD1iEkNsOT1P2Noyk4ytFEYy1QBDymxyYiAwRE8XDNtgGFJmnRIxCdL7sjXnivUgHHczvnNf8YW3DKcbW+haMZJUJpuMxIzLJXN+N/8HRqV3cRVIkgLcyeUkWFoK/5oaTfRoN3wSNXzBqgcKiMg86RCM2gE1Vs5KLEZDCD1WKTC2qPOTL7kHetedKIQvbQZevO755Y9N+Id/0LJOUw4ahXXldKdyIIopxY4yBeMcQUKxycUh0fcFBy1ktMlYOx7dR7dG+QRPipk8KJSxJVI0+fIaRIVkM3Z6ErYTyK6o3Pc0ySayLkP43am6wMmk0HdC6Qoku6PeQc6Gu/cC7x3X/NJHez7+SsulyhCBPo0FXC6Y6iyCVYJTmUrB2RZOWoUzlM6PpvD4LyzET4dLPTn5lzjh8dqpXOyHAruwxuL/LtoKRkaEfvraWc0Ne8BpXfHxW2f8/MvCs1ePuTXJeN1RqeECkNUloe0giSmiy9STjUUBxho0CpVj8UmT0EnQqRRf2fjSScuWFCOoSEgeOxhC8ni6kkBpFLFNbB71DCcGOzVUS5BJItYRfEZ7jdMW6yp8XzbwGAEp8KpshxL05XeoImHVBppK058IX/vaQD0748d/YsmtaxV+7RmOI0MOVPszYutpB4uZeo7fWdE9Vlx7bYkbNpzdOeH6LcvJqmVYR648X7E+8/z+b97lgx8+YHLN8PnfeMhLL814dKbpHmg++qEDqvmG7/3RCnVpHz1P/KvfO+VHfvYq7782QT24zee+E6mXFfMbDZvzSJ971MPMmw+Fk9PM5vwB73//AffaM974ygC6p1aZv/33XuFf/b/f4APXGj76Y4r/4j9/j1/5Wy/xzh/d41d+5IizbuC9256f/UDFjedm2MpydPkmH32/4R/86jHzap+33oXlYsLxieL6JaFZtPQtiCj6c4WZehZ7Qh80D97rcI3GVkIXIjVTGgXRDUwmDlcVUFgMQFIcP27RpqZaZLJqQTkUU5KPmDKDwiiLMYm61kxmhjJ7C+ikCV0gDImqmbLZ9jx4uEZLhXOwXg+QwSiDs7Z0JLQvaYMyVu6MXQE9Mjqk0DmzMZANmYiqIpjATGvePh2483bHa69W6HkCKR2ElNI4Li3PlyeRO40MlrbeoqLD1ZYHbc8739N0KOzdLUNWdJtYUi4rA0NgGuC7w4TT9WP0vTPspObWzTnW1oQ48hiGCNZcZKh5pWiUxhjFNgfW6xYRTbvdUk8nP9Ae/4NrBlqobU1KgW4YCB6MqRlSwucWqTO1TIidIgyRad2ATmxDIjjN/bvfQ88bfuRoRvX5b5LXcHo455W//GnSi44/fvfzPP7WOc+wwBwuWH3nHY72ZkwXc7IEtr3mdO1ZhcDeXLOcKNZRoZRjb1ozN8JKEl++t8LHzKLRrJYD504TQ+Ks7lHJcOAsOFgNcNKWBXQ2cbSt53ztMUaYmEz0sO7AkumuFMRw1US0iWhb5snZG1IXSufcqe+3cMjYcJWEJAfaoCewGgKblUKOe1JOKGtoXKbWBtskogv0PQwx0ilPmwQZOtCWky7wxoOa2+cLTs4SZ2c19zdTzvtSCZcY+xEGg4xFgBpP/k//aE/+Q4kab+CdPz2PNLsn1WTOatz4EztSoYxpfgVrqy7GB0oVTK8aWQtKAeIxlnETTkAEk0ghoMSBFKcC2UJSaDKzaeLT79/y5smE3/t6hVURZfux6eJIOhVxIMXJkWNZI7IvUaw+UOh5eVQKSxqdeEWcVFREoNDEqEgpoetSyuUwUg1HGFTKofzWYor+QiCbVLoCuQgrS0RzGU0QGHULFFW0Ls6UeO65dxb58DPCs/uKS8oRcmBIBUxlculsKBGcMUVAmj05Ddw7VbSDoS4Jryhdkt+SerogezIKUDvL01iVqp01cEyHVJROUUgJMYX/L0ZhtMHYUsyJCLFOZAvLKPTNgh97+R3ah1s2WsAaBmsIEXJSZJVLRyDHkbMh2FjsqloV3odSgihb7isSuUoQE85YwjCgpPiwpVC7RiaBhZ1dUSXEMdpqE8Ojgc1JxjWGZlFjpwIVJJcKCjsm6ibTn4JvAzqXwi/liFKRpIW0gaEXzu71bPvIssoslpb1vcjJduDwmqPrSsG5uCIcP8ysNysO31cz2VRYIPqe5QQOrxp0bZgvKvw6Q23RveHWcwPTmbAd1iS1ZpgsCPfuk9sZz97S9GHCD31koH52zunXHnPyOPDONzdMrxnu3E5MkuaDH1pwe3XMIs2xWWj7E775puPHP73k5N6GFz9xyL/8J6cc3rjMS69OGTYDd+7c4/PfGvj3/2fX+dxnb/Pqa/v0OfLOqeZr37vLT9mr/Pwv3OB3/ultmqrmyhXNcweGf/jr77JXV7zz4JSUa2ZniakxvHgorFeu5F4YyGLpO2H1IJAkMls2Rf/VRmbWYVxHNhVVo8l09GcKXVfoKnN6J2Byha0MOXUlq8QLzkZcnUsHwarRHgd+yPTbRNXoQjFti0ZHWWG12vLovgflyBN4vN0yRIVVBmtMEfaqQojdjcwuup1KISYjqTw3SUFSGsSj0KAUsUpcmmt+4hVAVqzPFHuVRtlReJ3GoCwyWUri6HdPe/Kx4n0vNJhFIMrAEGr+2XtCLee8/0axwUaBvUYQo2htDVnR+CPCquPuuuNATZnEiEjL1h1BWtE0lqSE4NPYEczYrCBHNn2L45TpfEZIoNXwA+3xP3AxMIlTvB8YCERXMIeihIlVLMQSppGz846BRFSRc3+OipbJdI++dzx6fMLz+3sMjzUsDpi//yWeefUljg/f5O33/hjbzJjduIp+7kd4OLfkV26xEM2l4wdsH7zNcT8r86LaMFhhYSNVcLx7HmnvDbwVAo/PQzkR6Mhiarm1rJg3hlXv6Qbh7dPMvXVHXQt7lWHiMlIJ1tXEuWG1LcxuiYKzmYm2NJYxsSqgrSonHJOxFowxBdjS+THYTo3ACsZWuIzYXgEbmR5G6v2B9kSDgW4T2Z5HzjvhoQetM88uIpcX4FG0QRONZhCPKMVWVbz10PKn361pvQMywUQUCmcVPgRQGpc1CY3PgagN5J5dyFDZyJ8UAyI8iQp+6t92yvIiNNdlY7zoGpQOiEgqD6MyT51KgTFxUKtdu9mSgkLjSKklJY/OGaXKRpAiaC2l9Zhr8BGtLdeOen7xw2u6VlNbi04Z6MmqRmIpV1QOpfsSFWQZef3jz5JlHGOU0BstZciQY4RUNCIxx9JiFNBRg9GoKBBzcYkAqRdan7EmYmJArKBzLqMANY5K2EUbl8NCTqU40kqTfItKBt1H5krx8qU1exOLM4aQxva9MoXjnwWlhdpmJrnQzk585M5JJqsKq8eTtNaYXMKSnnQ7GMcBu7+XjheMp59x7lpgQ6VgFC0457C1RRuD0qCNYIxBK413AatqFsby3kbztftLXj8auL3WVDmgsqZykdp6FA6iLoZKVXQ7NhdNQMqREH0Zo2QKpyIJeijq7jiUnBGjDT4WW6xyZY3RSZVOi1KISiQDYoTox8pnyPjjhH/QkxXomaLeN7hpcQww15iZIraefhXwa1NGXHpMC60i0mXiOnD98pybL8xoluCPO07vBpqqQU0UeRvRSWGaQNSamGq062kOi20xbT26dqx6S7vtyNvA+T3Dne9ueeH1Pd74xjnLieZjP3WVfsiIXzDEsoXc/9Jdptdm+ActXYKX39/QrgeuPTNnc3rOogVVQVjv8Tv/8h6//NNTHh7X/Mq/vcfjr9zl6pVDHnzlhJuziulLM/puxTe/Gbl9J/KLP7fk1/7xfV7/6DOks4d8649P+dBzjtniCj/xY5f5sy9+ndc/fJPDaeS/+PU7fPN7wt/4mSn66pQb55pHbym+8+Yjbj0zod1ouj5jp4azcM7B4ZLZfBT5GU2IAd9FDuZzsrS4ucXOA3Vl2KwdMi144PvvdIQu4FxFWg9o5UAnXJMwNqEtKK1ROqJNAQGJLrbPzTqglcEYQxJP1ydWpxFJFdYY/DajkkWLpxD4xhGBCCkoVKWKribJhT2w5HaMseUk8kg2ZcxucVqznAo/+kJF7QzHx4HJwlHr0g3QYw7LzuWbDWjt+N23Br52JvzcaxWLo455ZXnvtOcfPlD8m0nz6k3PlauWZw8suoJBwaXhjJubP+Fr926yTjN6v+Ht84FJV3F0OGNQgQ6P6cE4RysDEsALDFmwVUPbByYzxWyy4HzzFxxhnCYQ2kwOoANUyuCsI0lmGDxZZxaHwhyIg6bdVpycBDbthiQbZvM5337jAfcWNfPlZfZcx972s1R+DVmjXWL/RcPd+uvce3AboeHsrOJFu8/r1TUevfcm9WyP/aZm1XtONz1T0zFzPTEKoQ84FEeNYv/qBFsZ1tuIjlBbzbKuuLzoSOtcNtqoONlEzh/01FXk0p5BpYzvElHDtMosLcyrMSs91tR2gqkSSXpQkSweMUWJHX0qZCmtGY/oZWfIZcatlGJ24Lh2TfBdzzAEdCM0DTTJMO963rir+ZcnM25dSzx/1CMq0oeBLBVaCcsmc/NKz7fvbdmuaozSxDiUDT0OJCnAmEwefba7k+Hod5Ryspe8GweUvT2Nm8duzFG2CT1qDBjHHU/Nv3efOOoPZLQW7jagMhoYiwVUCT/xqmzYolBJAF1ihkfkaxn2R0gDKWRCimidefXSOT/3wcz9boYKI/CDCCGN33Nsi2eKejaXDPMLEV9+IjLKUV0UByWATwpgjtLyTEMZQxA1kkrbOuXCKtCDRcVcNqEcRm2FGeFNT0YrOeYygciQlZSWuBhyLPnph0rorKCkZUhlQdJiS45AVKisS7FJpJSXhuMt3DkFZTRaXAldIaAw5bQjFKEmXAhCd9eiiJr02Ll5Iv4kl1O6c466rnGNw9qyQWo9WiuVMMmW3EDUGqoFv/We5/n9+zT1AnyHSYFubVhnjXYZ0SWy2mgz0uMGsspoo1GiMCKoVPgQpegMo8UWBu/x3qOVIZBJPl9keeRR7KrGqOILB+94MEk6EvpI6hPDQ9je8eQMk6WhOdJ42xcXiTNkF/A+kYOlGyI4x+FNi3+04eys5XBrC2M+K8zEYFPh2KeJ5vHxmu2Jp5po8nqF0QrBYbQm41Ezhxz3zBvL4+OItg3TRQHx+BQI60w69pzcaYnrwLOfuMG376+59zDw6uUpl5Vne6rpvea5H5rT+8DVg8hvf3ngKB4gDx7ygVfn6IMZzxwkjh/1/ObnVnz09SX/2a8+4O//H17i3Af+6I8f8c0veP7W/+pV3rj9Notrh6jDwFc+u+Xf/d9+kM/+s+/QLB1f/FfvcevKHFtHxHtevNFw83nFl+/3/C9/6QZ/8DtrLl21fPmO5c3HAy/cM9w6rNi/JszjPnUTSHmNTY69SYVoj7UVkAjWYqeBRCIlWF6rGDY973ylQ1aOqqmJw4DVDmuF5VHF/FKGqoxekVDEuxnQpcuackSSLevq4OnaTL/OxOBQLrEZOqIvB5yYI1UdMLqCrIsVGV9EeLvRXSqY4TQSQcrqktGSi6bEJILPxAF0LlhzkuPkZKCaDFy5brCTMGp3ZBTjBkiaWZ1I2fP/+pLn3RPFX/vQhME35Bz59p0lf//kIb/4Mcvf3BeMCtBUuGB5buG5sfeYF6Yrfr9+ja+d3OKb797F6C0v8ZhnmjksPE4SBk3CF4t9NuWwEhLeJdrQMnE1PLUm/Pe9/cDFQGh7ajUFW5FTomrA+xVZBqZLi6sM25Wl7RzkzGyScS4zhEDXBzbtGjt1DNuWtX2Ls42Qziqe3avYqjn7UtLr2v4x80nDw1MYYs8XHt7mbPosf+l9L7B+cIetUkg7sD0byNevM5/V6L0FB41w+eyMb/6rd3j3m2sOjjTbteexUlw5qItkug5Ibzg/SQQGjBG6XugC9N4XH7lVLOeaSmuGNuCMYAdNt8r4ecA1mclcM5lplClqdq0sUPC5OaQivBqzBpCCwCRkhIGDK5bTs8ijRx7EkQrBGGfhtelAdZx569GE47bh5WsDyyqxHguLKkZevOR453LmbOUJUaHyhKgj+ISxoyc9jcIxAZU7Uqp2pICLVtZO5Jh3gRvkf+2eeUr3wDDudRcWhafGIOPp7KnPVbvvk0obL2WPdZSHm0DMGY0hp4SkhBZHiFsMBuKabCbEtuBtK5t55fKaw9gi2iI0SJDxzvWAHYWBCmIJcorBk6Mi5x16N5HEEn0pXsqEoBQK5fMUIWasAZWkAJbISFLk4Oh8Yoj9yES3uKjAxzJ337Uad9twklIQ5DIxQCL0BhFDChGfPCZqQq5QJoMplkPIkAKGRGP+v6z9d7Bm6X3fB36edNIbbux7O0/35IQwwAAYDCIjwEyJlEWuJGhdWlnrLclrlbRr2a6VVCVrnVayVbaKlrSmAilSDBIVKEIkEQgCRB4MMIPJ090znW/f+MYTnrR/POf2UPrH2Cp0VU+o6vC+7znveX7h+/18IyZ3SFFiF3Awl+zNkiUqeoU0ghBaFDmin8Kkdcxb2QB3r+KxapJE6BOqT5UUqfsfDguyIqcoc0ymkSqp+I+vd6sEZZT4XFKZBa/7Nb5ycA8f3X6F/TiAIBhUaawvoiCGgIsWYsB7gUXirSC4iPCgjpGtJmG8c6nTiFWldEshE4Ss1BInIQTRswxARoWzCUojg0eRsj18SAJQqWWKgBZpYhJc5GjiqBeB8chgvedo0SZIlpDMljW7R4FsPODeezLGUTDdWdDsLclq8N6wenLI5HBJe9uzeb5isJ6zMsrJdMZib4EqUhJpCI4yM7TeIqYWlStGZwfMpgvKVQO5ZwVYu6ciDg1dt8ArSbtYsvf8nNFwwOB0xjefuQW7nsc/fo75YsnkjqOoSh58ZMDCWjoXefdHz/Crv/oy4TDy2L0DgtPcvNXxV//iOUI+pb6yoJgq/uifPkO+4bj2r+BDf2SFF756had+6CJf+uotXr/V8eVnO372x9d4+XLDrdsTTo492xsF+5OatbHh1duC//YXJmzfoyiU5z1nBpzbHrJ9MoLuMDoHq1BCs7puKIYqfQdUQkZnWgIGVQiEEey/6bj5Qk0mM4yJEARFZRB5R7kiWTlZIrRLwr4UIYgQHeBT595nniRqakANBQbD4HQg1J56X7KcSrwJTBtLJGUgKBnQOomMIxIp+g4AcSyn4W5zI1I7hXJIFRAyI4SA9R4fZAoKyxbMO7hxp2OwtsJKCVJYpMzSsyC2aO1RSEZ5RGRjfvNVyZWJIVrP1OaE4ojLR4Z/+9XI5qqkMjlbYonJ03coty0XBwF55iWUj3zdrTN1M3YnLdPJlMWbu5xbP8mZE5uYXLOkIy46CpNhVUR7ha8twRtW8uF3dMZ/5xHGtqTJQMYZ1TAwdx6dlRRqwCzMEV3qRMYjQ+damq5mdaukcR4pMhSawhTMlkusMBA1Y1mgs4yNMhKspDKeI2uwTcZYB2LhOFKWZ46ukp1+kKeqAc3Vq0yLdd72H/8YO9k+bXdEMHAw2SGIESfufw+6XkdYR7E/4/YrOzx35RVObxeUpmJr3BGV4NYk0gXHyQ1DWSiOZi3TpSBqzWKR7JFV9MQqI8s1wVqaaWRllCd1bA4x62FDwaWbSCV61bGKPHQCF5NgRplUQOkysr5taC3MZp5cB7xUNGiUVzy8Fcm840uXNLtzyfvuyylWLDEKaiFYGUoeuWfGSzczJss8oTjRRCX6+FgJQkOUZHgCkkwkQVoQ4Pu9NKLfkUlBJKFAtTwOuEmHRYgh7ci9AVyfUHfc/UtkkoSl0bjsswI47kTT60ihHj7p2GUSKiqn+52/J6KT7SaYPvYuQwWHluAQOAGrQ8uK6Ci1JQYBxvbFlkmWPREIOnUfwUYICtlH96YQm4SVDTEk+pxOACDZ6yBEH/DjnILokX2Mc4wqJfn5AFETukDIHCjSuDqSRo0SEIlW6NuA8KQ9Hv1qRkSwaX0QIlh8YpxHUNHgA4lQpgRlFRgOFFoXeB+wJvL6keD2RFIIQ8g6CjKiXgEcqmelp3cr+uIt3HUNqJAkcjGmQB8fk64jLzOqqqCqMsqywlSaLNfpssl0v7joMPYYzRoopCIODb93cIGLowWnR9eZzQ11LBg2nqwI5KMMFQuMCHQxokJiakBaDcQo6ILA+oizgdobjvkcaRIS+t1t4ico4XsxYxq4JZaVgKBTVHMvdjwWcQbp0/UJ/WooSLpOcHTUsbKmWF0zHO63LJYQ0GjdYWPNtdchdh3jocR1hpg7ZOio54ZipUo5A0EiW4sYwQKPzxT5CYlvFM0tiyNQDldY2IjdXbB6foSfLajW1zi4PWPaeDZDxv6rU1ZPrbF3UDObldjlHVaqkmx3xq03LSfuW0MWksNvT9mdj3jkUXjIdHz5kxPOPVTxtU+9Qek0N6cd7bmTfO+mZTzKWDmlee7bC1755pQVkfGvf3fB+y4KHvu+LX7xFy7zU+9c4/Ziib1+xIbJ+MGPGp5+eo1/8Ldu8/6PPMD6/R2/8g/e5APvOsMvf2qH127e5Ec/MmTWQZY7qszjfE0uR0QcHqjKwHA8QlZLfEjXQWVAnuHzGhlLpjdg5+Uli0nAFAUij0QVMMoDAaM1ayvjxHqwPkGGhE/e+n5FkFZfb/mIo4N2CW1tsYtIPXe45DbENiIFj5UaqUGIgNSAWiAaRTQ5Qid7q48WGxQuKpwSKRXwWESrAkhH5wWNkwgT8fRuLSXZmTTkl5Y8oFYYrLZpWhEswqjUEOQKgYVM4UTg0q4gZ4Vh6TjSCwqT88Z8zq9+cUkuJO99GFbXI6NcIlR63o7llAfGL7I7f5CJOYWk5NX9W8zrlp2DPa7e2OeJR7apTGQaAjPnkbJDyRHRe1wMrOb5d7cYuN14RDdjZbDFtJ6gM02pIxkdudDUUiLzQJ4na09eDFFSo7JU6Y/1CC1hXJRY4SjygkIqVOxQOVjTYZ1KCVSlI3YOOfdsrw+JwnHFv8zN7DTveu/3c/LUA1zTz7O38wzOZxzMWopBoJKKbKVCbt7Ge4W4P+Psu0+TfWmNZz/9JU5uGIpCsbmWsVIJbu22tNZRDQXbm0NOtIGua6g7z840YpxE4hkUEW0yTO0ZzjrGawotdHrgKk00Cd4iXUCGmFYFOnlfhQ80QTGbeOKRI88EJteMNyWddCw6iQqeIgSWEmwXOH96zkKP+NJLAz7rPB98sGNz7HBR4Gk5uyl49MKCL7+8ipPp4Xlsb5PiLVJAiCQFNsd1b//Pu+KB/gEskijtroCwXwuk/jn0//4PRk2xF3P14sO7mfdAjJ4YJWk9Ed4azcfYD3hjP5/7Q3+cTzXVMUAoijTdUCT3gkRg1Fv+hfgWTD+NwiEJ+ULfAUTu8vGFTF279wHnHFYFjJYYnYRtf0gX2YcfJU5DiA7nBN6mpEnhBdHKNJCIqXs5fl8BgXDpwJfxuEDQhNgigyJ0Atv2A6pCkmUSLZIFMuAwGVTDjLJM2pHo0yRnXsPNOw2LukCItEKI3kL0aCPwx6+712sI2TP543GCYdKUhB4ZbYzB5JqyzKkGBVWVUwwKssKgjUqOAwJeBjoraKkJMZD5JHaqmhlNsc6v3n6UP6UOqKqM1na40uCDZDlzGB0YyIwsVyjl0KoXn8YkFK104j1IqXEuUeO8jzgn8T499GNIUx7bqZQlH4CQQpwkieUgANFDv6ROO2WtwbekKGYnkC6mQyYGDvc9g3HF1pmCplswO7L4LCcMNScvZESncYeO1VWNpcQGhWLJcrfBVBq9WuDmglhLMimJTaC5E6lnlnpnSbFV0dxYMJ20DO8bgYzkGxV6K2N4q6POBbUL3Lqx5LRUFIVl/+UdqizHjnJeu1Hzwae2OchLLn+749TZNTjskIvA3/y1Q1Qm+S//yBnqF9/kkbjC9rbn5FZLvFnSjSo+/9t3uCEEe+0KH/uTG/yjv/gmf+JHN9k+LVDvWmGpPB96ao1f+6UDvDD85A+f5crzbyDyyKn7HG9cqzl9wrAIcy6csHzfe3KijFx6Oacxltf2Gq4cLHnHLcE7zuc8/gGQqxs4OUVlGoVOPIemIzSWozc1N1+bsNgTZNokwFDwuEbhlSaqjtFIs7IlUeMFUbUIrfFSgdBJI2RVPx3yeO/S97eFaCXSgosRp6FaLVjMA4d3PD5GTL7EGIEPGUoplJK4MGC+aDEmMChUWptGjw0RrwApUDbd/0r02QUiTY29S1MqpVVCxcdIUJrnbtVM55p3vScnX3VoLxBLATpNYHXI8V1EY5EFrKiMxdIRvMLGI7SOvHw94+99quWlHcWH3iZ4eF2QraSCQnnNUEXOnfAUbcRHweCooQ2CeXOEkes8d+0maytHDM0II09TDsf4MidYx9JZaGbf0Rn/HRcDmystZShZ6AmtylF+jshyGjwyZuRF2q2KKChVidEqkZ9EhVSaKFucrdEZaC3ITdpRdZ0Fq1jQkckVKqmYi465UKxsZCBbnJZEW5CLI9rqm7y+9y1aWZPlijIaTlcRLSNGK6wPzF2NEp5upnHthGJdE4Xmys0OlQk2VjWjQiGzyO4B7E4dq8OOtTwiY1Kjay0pM+is4+qtBV2bY0+kkI7hwJFlEjWGIANBJ0ujpBekSPpuM4nDBgWEoDicRw4WkRA7nPNEBZrk8Y65QERPqQqitzx8JkX+fuZbht/D872PNqyWOVZoxrnibafg5Tcb9iYRT55UryQlOr21LKV+6XTw9muBu+ItRLLkkW7+u0J/0t49xtAXBcf2q7dWBG+NotM4EKlJ+d7poL9rYSTRFqVIXHlxnH8gfI/36yfYLhIcWCt6SFDPlUegdOr8VUzdoghp90/vekjThNjrDmQqBI6V56F/Hb1ICJlcA75fJ0QvyHTfWfcjxNh3oD56fPQ4L3BBEFyAGHCdJnYq7atV72Q4nvK3EdGSrJJZv6IPGoLG20j0nioX5FWyYQYb8XgKI9CVpBhEZJ5ei3dgbaTuJLN5JKLurjUS+jmlsbm7D63+wOUtLcVxAqEgIrVAG0VeaooiYzAoqQYFxSinLEuyPKFU07VOWOwuBqKG1rfEmOEkiDBgYCdcbyt+bf9B/vTGq2xqz2GIZD5HC0dEMVk2KCvINGgtybXCaIHq782U3hkxSqYCzAhk1ge+hFQMhSCIo+PrJYguFYzRpbcaAjif9Dpt5wkeZO/mCFGASWWsdAoTBKH1zO4scI1h/cyQahgZLDsOpy13XmgRsmBjaMgqgygded0ABic00jgWd5bYecRbz3hjSJN7BltryPGSvdsOXSriQOD3YNVodi7PUIVgtZxjDx2nTo+ZioYXLlvkRs56bnnuUuDpD404tW341gtTumyL2aUrjDcrdtqKvatTbKa4sKn4gQ+epRplIHL+/r+9zX/xfz3HILMs7lvn93/rGg+tCz761Fnaw5qrb8z5+399G7mR8/M/f5WxMfyRP3U//+svPoe4M2IiD/nv/qeWpx/KaWzOme0T/K1f2KO50fH/+J5z3LSes5sj/uHn9rgolvzs91/gf//MAb/0mTk37pvx1Ws1j7ypePzcgmxQMPSO2k85tAUrwGzR0rWBgYKqkETdYL3ECMhLQbmRMTxbMViRqMonMJeVqeBziq51zFtJ27Z0rUu0TS0xmSTLQGWpYNVR4xaR3V3L5DBNsYoyxRRP8y1sUXLEnM3pHE3H4cyxrgPlOCAKSYrH7iO6dcT6SPRJMyN1mu51Lt2DCfClAI8XLSOdcc1J/t6XJ/zZ6hTvfXeXuDEFaBnItSdIhSaJERtnCapjfRC4dz3wah1omwJtJFd2lkwmkit3NE8+LLm4FdhejYyIuCIwKCWuKCiGI47mqxBatDyByA+ZtVfRexrvDthY7yiGT+BjSxCBNlqarvnuFgP/zbkOi+STBy2/PwkMS4Xyglq1jKvAAIkqStpWJI+5DojK0vk9WmcJVOg8dWJGKILtIHi0ybFdJFc5Tiyx2iK9Z6NcZSQUy6YlKEWRZQgp0TowGtSgSiKSjWpEs2xYeEUjJIeTGWaZY6xmOC5QEr797GvQOpRJCbLzZRqFV0Vgc6TZnUpmyw4tJLlO8crbVWTFOHIU0isOZx3EgBQVgogNlhMyI9MR4TpQAm+y1BEHR7COrgXbqWQT9JGmCyxqR9d5RBQEn8bgQhq8aMnQaCVpMQyV4z33znFNwW+9uMZXiowPPjRlmEdsUJzcCjx6YcpXnl+nDqkzjL0QRot+l3+8/xL9VOA4W6A/P6PoUwfucu0DoofupGKgP5zfWqzxhxSEd1dtd4uDGO8KCd8qGoAQ8b4X5yFJCwD6hXsaoftO0LYCF1LRIUWKOdY9dAQPwXtU17sFVALN0Efz4kUfbpIO/xiOKYzirnwjDSskqLTD7nxKFJP9y5IiEoPARghRYr3A2oh3oT9ooKsdnRbkAogeYVTalVsQy4hwIqndBeAkwiVhYrB9Ml6WsKW2S1MgrSKm1JhKILIUyhKdxHlB23msj0ilMZnGWYePKQ0SkQ5IoblrKbw7zYnHOgTIijKxA4xEZxlFmVGWBdUwp6qqfl1QYrRECE/E40LAeZA24CVknUMIg+kiyCnLRjPMPC/tVfzv3T38mZNXWdOOI5lhlaeMgmgkqEhjc2TnaGRAKZ80OCY92KUE75OAUkZB8BZilyBDZEgM0SUtj0CA7i2sMtwtGkMUdEKwaFPw2XHADcGmFU+Q+HR7QJ4KzOUkcrQ4YvX0gGzNMIyRIEuu701Qw45xM8ZgqLt0bwxdA2PDoRfYqWBjvaReNgRVcDSBw9ct0UiMiBQh49rBhOmBJDiBWdlk/9Yen//CnMcfKCFf8MjFERveYy/nfO+HHW9OLJdfnvPoE6eJYU4mDeNTY9ZLxUtfVHxlJ/Anf+Y+4qVbfPYXbhFEwbuerpDGIDvL7uUd7ntkjXP3V/yDv32Jv/A33sHXP/1Vrl43/OW/eC9vTiMfeNzw3//dS5zaOscT75/xua+tMNqCnaVlMBry2ud3edfJlmZjkxMPrvK1f3rIx9+r+dD9I/7F7x/xkVpy77mWT/yA4NzqGkeN48qtFuM7NrXlTRvYryP35op5NcdrwUpWonODLIDcoXJNURpWTkbWzwQo0wnULgPdkaRZKnwn0FLTeYtVHUqDKARSpCmY84G2SQLB1irmk47F0uJscpsoARMxxq5uU2nFRnubVTOlygyXJhmv7eU8vhbYjKJ3JCkIER0kSgdaAh6T9CcqPfta1wfKibS6So2TZBEFMWq+cVXwv/7rjv9xvMnF+3doJeigaaPgza7DOYfHYTrHiWHNOx8xDO6X/PYLkV//6pJGGfIiZ+IjX30FXr9d88CW4dzJwJkRrG4qRudKhusapVq2V0tOrWwy6fZ5Y+86wS4IXcnacMiFtQGj/BBZjAGo25Ti+10tBvRGyaB0zG6ugXJsrm3hG4vUBafGY5pG0toFXi5RJul9m6Whbkvycg2parTUeOfIdY4QntZZOmnJyhznPUVesmwrhtkA7T2tiBTlGqWGrgYfFJ1WSL9KrD0DV9CGhsmeZba3i60VK03AH0yZ1o56MGZuYXqzY2M9Y7QKw7LAdVB3yyQMyQSj3KfDMEjqzrEy0AxzhfEeYwRVKdEhw8XA9f0aGzLaoKmt58TJnNFGgSwCUgfQIHSyVdkuJs+tg7YNNK0leJEsWD3oxUifUq5IgAsnakymsb1Q5qlHPLuzGV+6PKI0iqfun2JW0i7t3lNzLr0x4PpR2dMD1V2q4LFwLAXXpFM7hQ6lg9EfK+CPCwUUx2yB5AZIojgfj3fSx2Pn45H0f7CrJnV4qQAQd3+KANZHFrWEmBFpOJ5cJH+qIEaFc46uTbx9qdNYD0hVu+TuaDi4cHxyp785kjQDHqKPhB4/TEyrh157eZe/T0gCnbRbh+jSWkQdk/qUJERou4i1AttJvOv/LilxjaOVEa0MSqZiiS4gaqBN3UNMdQLKBugUsXMYGZHG4EOkrh0uRJTSZCWoAoTpJxfO07SBthZYp+i6LnUj0aJNKh5DjITgUUIfL4Sgf0jF/vom5bwhL0pMphLGtzDkRUZR9HqBMicrMoosJ9OAUDjX4vsY46TeB0GOjB3CSzrfEcWIaKdURvHC0vD33ljlExcWnB3M2WkCygXMYIjoAko3fYZFvMs6EF4cG20wWoBP7zGholUSgfoAscP442lP6vpjT5kkyH631CBdWuGEfmJAFInF4LMUlBXaXq+giBKidlQrBUVhyHNFdVaxngnuFev41rNzu6O+HSmNolpVZJkhO5AMnKeRka5IB0R9u2Zt23Jqq6ZSq+hBwavXdlhb0xgB0raE3Ql5UbFVTojSsjEY8slnD3hwy1O1HfNuRDXOOfSgO/jMp6/z7vef5oVnl2x0Na9fWfDo29Y52N/h9z5/wNMfeZRvvnqVn/qxB/nyV17noccvcnl3n5tvWsLtjnpjlZ/7J5dZH27yPT8Ger3k0YsFzz7X8eH35zz9sQH/8G/d5BN/9gl+97OvsnfZ8q7HDP/4+eusOseHnl7nU7/7An/zT1/k01+/xBdecLz/XMVD7y75lc/OeOVNTX1xn1/4suDhdQP35/zyi6ko+I+eKjg9FPh2zHgoGBuJLBpMmQEVVnqWpqNbKI6uCPLME3VKfbUxILXCZBIvWmIe0TIk3U4/8TvW2bY+0iws8yOLbTNiHBCMI4hIrcbIYo2BO2Ctu4MOAqUKFq3k2wcFl6eGDTHl/DQwXE0OheNVqxIiGZ6CTE2ITBj5pvU4D0EEwrFVSEQ8LYeNhlrwzGtLfv5za/yF4Wm2Tu5hneWwCdxs07RCieR/GWaRt292nC0ca1nBG/sdX7oUWdoaZQyxE9zYgdu3PMUrjtNrJY8/uMoHTw44OcpQao3y7JCmvoPfu8kqS6pslVMbQ7ZW1jG2QkxrtJVIKfDNgizY724xMDgZufS8pdsRnNmucHcaolWMK4WZLan6PYrKhiyOajIFoyhxwmCUol1qBuP1tAdc1MisYmA26dqOPGiilYSJo5IWFRpiTHae1hlebgXrK46qgo1WIJ69Rll3LBvPpVtzZguYhtTdnjtV0iqDKxUVM5oOKKAaKsYD8KHDekXbag7nEecDK0NNbhTOBtrGI2zKDhBKMMwyVgqJio42gCMw6To4cDSdZnIYGK4pxmuK1RVBXgq6CIsmYkNMCW9EbADp88TM7x/gSkSUkrQ4okoJcIm6n/zfUQmUhI8+bnlzuuRz3xyx7BqefKzmZAnn1wMXtxpuHymcj6BMXwwAwfdj/nRAiCj/0NEd+0O9tx/ePeR1LyAM/SF73Mn3B0yPtRSI/s+LiCjvFg7Hq5EQAiEEtEj7emsjRzORqH4q7fmOATuJSZC6XS2TzafXJKeDXaQvqdKyVxtE8LEnBPYfpeh1BserA/r1QyAd1ohew5AKgrt0TpHKkmOIkJAQFdgQqZuAa9MBpEVMI24BLohkPy1iyiQIgegEoQHpUgCK8IJYR7Ax0fQiCJHhOmg7jxeKvJDILKIygTSxT1oUBKvo2kDbpSKqtSK5Ufr0wCBdT0qLaJkeTBH6i5zWIUKAzhR5nmOMRuearMjIylQI5HlOVmZkeU6WmVQsCFKID4mmRkirmTJ6GpNscWmbUmJiS+Om2LygCJ5Lfsj/52XNj29P+Mh5gwyeuZuh8g10rNM0UKYAGi3eskHST6dELomdo556CJrcFBhtibHDx7f4CQGbpkQipjyLoJC+QmSQFzaJ2rzAO4mMAhs9HpvWdc4ifYZ1kmJVsX4up3M1vpOwEBxebslKxfCkYnW7Issb3O05Uhn06Qp9YsTulUNmuzNWoyJbyQiDmmJthclU8eLzh9z/tlOMRqt85rk93qcc62XG3g3BPK/ZXzqKSeT2zoQLW2tsbvkEcjshWBMZlAue+/YOB4djpruG2d6SF28viXrME2/f4vK373DhwTWK4Zz3PHmKr3z5Nr/zmRl7hwe8+6E1zq+0HN2Z8+MfHPLJzxzxgT92ns2NdX79l1/k9huB731qzIUPbPP/+mvf5j/9o4/zheff5CvPHDFvNV95OfBnP77GX/nlHb72xk3+9l/Y5kvPXeeozhnQ8OT7TjA9OOSbrwciqxS55f/+sQbXCJ7bcewddvyZHyy5uJpWO9WGIHMKqyLWC2Tr0DogtUFmWcrbKCKhEKDSdy47jjwngbCIAroi6ZJkah6chWYZmU0syyVEP0hBVaYjwzGXGl90rMtbjGyDI2MZBcpFLjWClw5XWNgBb3aeB+ZLRkMJpgPj6Zo0ZYKAwSVyp0qY5eUypMmSoA/9iniraGLg9q6mdg6vHF+8Jjnz2hYfH7eM/YKbhy2+MWgBpossrYWgWDESXWjuX1M8dSHwwhuWhevInMG7ljY3SOtoWs3RG5aT24aq0IyLkrXxfRwsL3N5do1uOePseI01sc5ADSjlGKNLVNQMdOLgSGtpejfO/9GP77gYYNdjX8v4cN4xP9pnpdB0dc5kf8L2qOKgPiKqnLIa4zpLJgXBBhoHQua0yyVLO2E4GCBCw7y2NL6gcx2rI0V0DmkMLjQIHxmXq+zfvoVf3+Rq1/L6qZYLGxvwzBHV3FIPS2JUTFzOQdeC0eAFV+50DIeesytDgim5tjgCKbGx4/otmCwCnQusjDJWVwt87WiXloWzVEayXhVoLyhw5FlgkAnWBxlKCJZdSxdB55KoJbM2xWMeLCP5nmdURMpSILTCBXDRo2TACoHr82Mz4YgicdKdErS9Rc9GT1SJIqikQ8gs5Ttoz8bWkI+/e8Y/2lF86pkR82nJhx/v2NxecH7riGffzJgvXc+VjwQpCd6htQYfEFLe5VMLofqpQI/PReCiv+tVPybIIUKfg50O3b7fJd4VBPYirrsUwrSCUAJkDMiYePuIhNc9OAxEpxEmxSPjeitaT6ExRkAu8D4dzkHEu+I46KFE/SEfExmE6PrOv098FCIiZHJ4RFIxEHwviuyd6ZGQksmA0E9KBGmt4AAXBTaEBMFBUpqEnRbS9+uh5EzoljY5E4IgWo1tQPgOo0B2Eu9SdLD1jhglPgY6m3YN1VCnBD7p0zfQpALNd6QgqzalUXbBYwM9zz/tNn1MVLS01umQCcp893MSQqGMTBHDeU5RZEk0WGUUVZ6YArlJoCGdRINSp2lJiBGPw0WX0t8QmL66WooSI2s0Ha3sUEoT6lmirSnJZDnl515QPHsD/sw7JCfHOV13SDAarRRKCrRQfdx1T4yMHrQn9uuWojK008Bsb46RmrKqUHmvMk3jrPS5uv66ukgMTSpgYkCZ2HMSklhSI+mkxjtJCI7QQaghG1Wo3OB8jUYx2tDka4bgNfWNlvZgyvhUSfnoJvV8ye5rR2SrDWSO4kxOsIGd11pGWxmLawfY3ZZ710eIxnLt5TnvfnRMhefKDUFVRvb2lozzEVmE2bxhc9twOJEslxm3X92jOtkS6oZXJx3veeQEl1+/RlQVFzcyPvhD2/zGb16GNgn+PveZN3j/95zjH//KId//9hUefqRAbzX4qw1b7z/Plz7/Jv/Jn3+cv/G33+AjJ+/wyZcWPHZqnaeeWuFXP3mLdZdzZbLgG89MePu5VX7/0pLHH5xw6bDgT3/vJhv3Rr70bMP/9Msz/tRPbXOjnvP3/t0t5Oci77hY8a6Lmq/eWjI5rLh0e8bbzg45/XDGeWVYHAq60jJqFa4MCB3JcwVZJOiIqRyoSNdGQsiIYYnJkx06NxmEgIspjlqJiCatzWyTCLjLeaStI96DRqCVo5NpNTfvBLKE09qSy0BwCk9S8d9ewsv7q0y7DXyRcznWvDzrWB0HqqHE6MgigO/6Z0tfbCIF00nHYpaSW4UM6dkdk5XXO3hzNmOxrECP8Z3huRs5+3YLk9/kjTtzrs4EMgObSRa+Zj8sWYgBJ41iOGx49HTk9Ibn5esF0Qc675DkEB1SCLTy3H/fmPOnT7C6uooT++wefZv6aIf1fEQh1xHNAElFHgpWsyw5GITCIMllTv+g/+4VA7euSMy6ZywCzAqid3TBglhh33okJwg4Gp/wiD6knethvcCGBhNyrt/a58LFjM21DFcvmc46qjIjywRNJlgbaiZHhrnrkIWgK1dYLhb86GZGPV3j9stT3rizZJaDsSmffDhwFFXFSpVRFA13jiJdLFMyYA2bgzHlWupKDpeWYS7Ym7Q9DCaSDwAki3mgc55CCypjKPPkT21di42SqtDkmcKFQIcnZiAzSSDi6FKISi0xTlCaQKZiAlBIjVIeIRyBhAoOKJDm7gxbiMggQKc6Wjw5RaKzZSl611Jz36nI9z9V8+ufNvzBcxX784YPvH2ArgJFDvNlh/cCZI7E4L1HHwvJkjouHecydcrieJ5MEst4f1wceDIjkNJQ1xbXOoxJwJf0PUlFgYi9kyBC9La3/yQRmpTi7iiNEPAR5jW4IDHCv3UwHwsbpUdIidGCTIJUaRzeutgz5dOfdfc1oMCRuuWQuotjV4SUIiWFpVdJDBEfubvC6JOKQKb9MoG7ECJPohh6QCGockNlMiQe77v0oMDgvaBZWCKRIg84G1jWAiUdFZB1WVIfS/BSYW2ksw6poCoFJvcpZU1LookgkkDTe4G1AmeT8yHEZOZ0sQZhkEInK6vv1zquQ/QRsFEkEaHUijxPIKE8N1SDApNrTJFRVCV52RcHSqOU6rv9ALiUixE9vi/khICQpfATZScoo4ltSyU6GqEgC4RgwUbmwTMNin/+csFnXznkr39/yUfuE3RtwKhUJCICAZduRJ1WHMJpIEtTJ9NRrEeygaadwOFRexcRK/u1TyoCkvsl+NRBWp/ipVPIk+vdJTIVSjKmKG8CmQCTe5qlZXenxNWCrA1UJxI0LWjH8EJORsHR9QmLSUN1bhWxYVjuL9neLIkjwTwoBktPWEI8UzFcMXzjq4cMwwajkxlvvDTj3vs3ub03I1Q5VZ7z2NlNOj/jkYfv4cbNI9yuZ/tCxdGZ02wKwTdfnPL4mQ10pckHYx4+NaQeOr7w/BR5EDn7+BovPb/Lg0+s8hufvcEnfvoMZrjEaM+nPum4MwkML8/4yjOWh95eY+oJ5vR5/taH1tgRgk9/dp+DO3PKoeDd71vjldu3eP7ZjnNbgYsnNnju9QP+q7/0KP/kX1/h+ut3+F/+yoPYquHFy4Lm0HPxZETaAV97eR+rDYQ5jw4H3P/uEc/+fs0LOzOqbMC9g8jGakQORDrge8usNy5FSnuDMJGgPLkeoqVASI+MDm0MA1Uwr2t8iHResVx2LGuHs+kQTiLfXtdDh4wwbTWRktNDQyFrOpdi7wWO4Byvzga8Oh2xjIpcCo7sFq/NOy7MD7hoFMZJggt4l1a8XnmCTM+Uo2lNXUuiMKh+PSeFIcbIxHp2djxdYxnkR5wyktWs5MgOuLo/Yn7YIGkYhYBHUwJ17Wk7iQyaIDz3rRoePb3k21cFSxEQeYFYLiDLwcGDF0/yofc/wtmtszijeOHyF9k/uEasC2gMZlAyKDOqCENqsgDRDJBY8AEZHAPxXS4GdvchigqBT+A1rVnLFcIqjpYduVY4YakXjlE+wLqUlLeal+wtHRqJKw1RCSrvsE2LVSUxiyhhCE3DEYnoNzCauW8ZjxRu4nlld87pFcVKJRjkGXMSfantIuWgIpeWReiItWQZLLd2p9xWCp1LyrJAaYHtHKXyqHHGfCHp2khWKopcUI0V26VBW0spIoM8UGQaRE6MMG1aMB3rVUEeNcqDEz6N+aVAyRyBTOr5GKhDxCtBLgHlMcqnXaVMSmkd04EmoAdpKERmEc5hyPFCEWSbfP+AjoGgM564X7KzG/jtLy55/vUBtyeRCxsL2i759/EWaSTBJRa1DwIpdJ8gl35E168I/lA3GYLoKW8S52vOnTS841TqTn/n1SXNnsI5kEL23bpHKk9eBnTu0GKUlP792DpEENGkMiSCjJo7R5rGBlQcIUObssRjjfMFOvTrQBGTIUFHNBERAjGCQieFuYi9cj8FmIiecYBLd3IQEY3AI4+NEQl33AqgJQkkVMoK6CmRundZHEcBx9ihFGTakEkg2ORftooQJS4GLCHZPGtJa3vQjksgJeEUlhSRLEQkWJ9AOV6Q5YK8kkgdEwFCiiRoDB6cxNWetoPGCVqX7IF1F5jNhkw6ixaCUigavURRIZXCC4uKydbVKUulFcJ4tNaUlcZUhqIqMbmmKHLKPEvjw95ZoaREEPDRJ1GvS5kPnvR56yiJwlPnA6SbslQld7wlqwWZacmVQnSR1ah5bCvy6BMN7zhX8sC2R2WRclIkCJSLdw9o7uKvPTGLyJ7kKJyBkIScpgJEpDlIn3UCxgtkSK4bKTzIQPASIwUqpgLBO5lS3HpdTJQBYVOGSZCC2jqGm0Va50nL+ETBy68uObzdcGo1Y/WMYfWeDDkuCdOOcGOK8qA2NJPOkxvFYKTxqxqhJHYBh9daNscZG2sZrhY89kDJ9esHnDi7SSc6rr3YcPn2LXIVGO+3ZC5ihGUcM8ST63z6M1dQ2jDbPWRlHvnJ9z/MtavXGW+f4KRtefTpTX7nq4dcudPwiR8/wQ/eO2QW4ctfWXLl1ZpydcFTj6xx+eaEv/yz59B+wl/9yw9wfW/Bp39vylql+Yu/esD7z5T8bz/3dn7xn77KtVc6Xp9JxvPIKSPYsxn/6Fcvs3c052d+/Bxi0PIP/r/X+Gv/8UV+8/U9Xv1qpBg2TEXBmop86w3B//NnB5y6Z8Bv3PbYUvGxJ+HCqYqi1CgZwTQIlSE1qEKnzpoAxuCEw+FZtAkuNsw0RUjaEm01B5OaWePRIlLmOUELFo3Fu4AROn3DhaC2Aang5JYnyzuakNgqx4Fmh7Xh8nSTeTvA6Ah0DHJPJQuO5hm7pibvV7VN56mERmUejGHRwOGewllFzJKQGFlgZYvTgqvXFHs3cpY+ct844x2nh1zcDrwWFarT5GUgdgbEAK0bhIrMMcyatNKsDIzKyNPnIl95XXF5x1F0kkZqpLNsDQ0fePoCa2dPU8uKq9e+yvVbL7LVDdGLgGef02rBpizJ1ICYbWAjiM6jI1TlnOrEHZTc/+4WA/tLj+2WjKuCDkGeBwoVKTzkwTLKMzwwP5LsTTqGqwpt4GjaMKlj6qQ93L4z40gZjhYZGycqhK2ZLgPGFLRLz3TRYPKcQRFomyUuCrQsUQrmIVD7wMHC4kYC5ySTwzmrKznTOo1tBypjYzWNyo1QTBcN1+9YrBCslkkYt+jSDsqGgGihkJEyk0koFB1KBpTxZDqF/PigcLWmU4KsiowKSecFDhKlStveZt934kFg20g0AiMEXZ4CeDIvyVXEK49LDGGM8wSZvNVaZzjv6YggNNFGpFZYMUV5zdZK4KOPWQ5vGv7g1cjeruZoJ0dlDc7Tj45tIniJpKJ20aXq+z/IH4iyV5jG1GWqELHB0XUlq4Xi7Q8sOD22fN/bFM9eFVzZbdiZQmMl2kOz9MytoImKI+sR1pHLwKAcU2ZFIs15lZTwwbE/8+zVjmKc9ofBdiifo6MiBovwqs8VAGwv+LN9kp9K8dEQEuQnir6rF8lbdswn6HkC+JA6/ti/75CCpwCESt3l8ZnkSeN8ZNJnyJ7cm7pRiXcB3yXlfggRF9OkIZIOHGLqSINP/+N8Wo0IIYlErIs4l4J7TCbvRqkm0p8nepXscjZibQpj8S6JIUEw1oLHz1te3BO8eUfQFg3al2TRI7XFBUUswXgP2uAKKPIheQGyKHvbYFoLFJkhz7K7iYReCDQiZRUcOy5CeGs9I8AKgRSWKiw4CDmb1RF/9MyScycLzo0DG0VAqpRSp3OQWYJbCyuIXY7YCohWEtukocCSgFAxEoNEtKR9bRPBSoJN4Cjv0jXUIuJFDxPy/VQoSkQwCAVGJNtXjAGheriRj7ieN5HsagofXU/NU3jvqFZWMK4FaTn/gGYwkLjY4nGIZYdbRjqrOGoj+7s10cP6liEbROyioZ1ZwhKGI0FTBG7sRooVx+7+BNVETpwvuHrlgK2TJ3lz4FjJ5xTRYbxge72iWYI2He7Z17n1yhHvvrDB3s0Z29vr/IsvvMJ4uMrbBkuaieXTb874xtWOn/rhC7zpLV/9+pzrV+7wo0+v89QTmryq8CLygQ+e4MIDBT/3C9fZekFw49Yu/81nLX/to2P+3MfXefC+Nf7a33yeb15f8tPfs8KVL0z46Z84T9nNOLCR199s+f4nx/zLbyzwb7b8F//pPTx7pePmVYuTNadPDNA1vPJGwygzFKMh/8u/OmBkAhvjyPLAcBgb1ChnpRCozJCNEpCNJkOqiNcNznts0LQddE5gvaSWggPpsH5BUJGsUGxuRiIZy9aznLfpGarTtCc4wX47YlcNuX9tgZQNNspUKAaPyAWTLuPV+Qq3lkOiVkQjGQnFw3nDY4OGk5mEVjKJjsPa0jUe4RTr5QBDxLeOigjG44UgaIFxHq0UC1tTiJxT24KDq56HT3nedaZla1Vy0DiKGbRS4WiIssPoBNSyfsky5CAhDxkbpeXB7TXefnbK9Z007SAGlIZ3P/kIDzx6L22b8cLNb3DrxoucaQIXijus5jWolo1qSKVKXFyyoMHpMSYbUJw4wqh5Xwh8l7MJ5rOIc0sGfQAJSJoQmDQdUqVxZ4ai6xzThWVlY4ySCudqYhuZ+kCuCkaFpm0FVZHsc4ul57Cr2RgX6SaJELqOYaZZtI6bew3R5bhoMSpjezPn1JahqCTTmefmoUPnhhNScfXI0QF5rjEysJprpA/MJh3jUnNyVNA2HWQBFGQ6VVAqAI1F6IR61UZhtCQ3kGuJiKq3egW0gWEmqAy4CB7di8+OgSh9Z+IFAYGNsk9lS4zuSYTgBYNck8tIKwXepfUAyqBiRuEcQVmsTHtSlRVEJ7DecfqU5qn3ZlyaNVy7rhB5hat7xbm0iOBTBS5TCA+ofg+ffgbRuwF82sWm0XlP3RGWGDRXrja4JzXnNlvMtuGhe+fUjcZbxVDC2iAgleSw1tyZwqU9x81dweUdx+3ZjElrqZ1JB3kIaFMwOdIcHGZc2K4JKFTQIAqsn6N7mly0SSQUQsS5XkEue99uSAwF0fsiRUhTgejA1wHhAsqk6Uy0aZQuEGglsdHjWkEIEqUj6JRVIHuB5fHaQMiINgkGJGNyLgSbCgnv0r7aRfAx4vvVhVRp9ZJ0DAEHCCmRKokWnRXE6DGFxOTHwKckrEx5y4nBb9uIs6T37dLJrLTkwpbmsU3Bk6c8//hrNX/wZo7TGuEVU6HRXmAXAl05pBozVg6tZghz+i7gyhhFlmnyLCPXKTEwiFQZqV7/4YJN9wyp6AkxgIgY4+iiYOEqfvTslD/zVESNArFfs0Xt7jpDYhMIE4ftAsZniGiItiXNcJJSLPb6zxj7AquRNMsO2zpUiGiRApMUJn083tJLVjkGW6JiKiADRJc+UykjZAmErQKonjwVrKfpGjqbE2PF9mmDUoJLX9lDSs3JMyXFILJ+ekCxntHOG2wTObFtWIYUzT06LVnMAu1Ow51XA2cfHbJ9QfDclybcbCXnV0fsckQ7KNl/c8GZkWSxJ7ATjR/PuLgu0HKL5/aP2BgGXtuZMa0j791aId6RXDxR8a1XJnzwqS3sYsHCDFBZxzOvLTh/cpNaHvIfvWvI4w9lfOZrN1jcsXzk/es8d2vOuXFBN8m4/8ObfPrzV9l6ZcTnnm34yScnrGUj/sn/rcLtRH7kpzb4E3/1ed53X85v/p0P8Hf+7td54v4RP/70Cf7Yf36Vj79rm9Vizu88s+D7Pzhk+10lv/4vD/jS1Tnfc986403BKK/Ymy+YtIFHNuF//ieXaLxhdTPj/vOKzRVDkS8ZKI+1kpnIYeIJwiMIKKXxKNpgSWxqiTpupiSoXFEVmqJUFEXGbGnZ3W+pG4kUOUonLooPiS1RKsfWEArVBwtZkaapJqXSXjpY5ZXDs7ShZJg5MgkbuefMwHJPtWRTt4ig2OsitzvJ1Goykye6audQneT0cMDucs7EJyamCMmWnrkB7zsl2frQjC+9WPK2Tc3WoGU1m1M2kAuBEhlLlgSRrLlaRJztmLRJWKmjpFJwdkXx5NmOL7+QsbNIzqhTp1d49J3nyMdrvLLzCrfuvMCDesE7R44NOUeUEmkkpe4QxqKyBmVmiOERKi9B7KeHY2j59+hu341ioJkrhisVQgeUkOgc3Cwx9otxjswgI8PpFlOkXfJk2RG1pNBwuLD4EBiNYJj3fk0yTFmQq+PxMIxGOV3dUPsA2ZiVtSGhCVy5NmOQO86dGDEuJUUeUD5w47ZlMnVsVBHXtOw2LSulgmA5yh2DqkBXGbWPNMGRFZqqFXjbsaI0a5lGuCSe0iKSK0lpFJlJB4fUkOu0M7c+4ryiaSVVBUWeDpIORRQCpRP3PvQ3bOxjc42EID2tECynhls7lqNmxspIcm6jZHWoGWQZXQg46XElWJtW4uDomoghkfCKaHjsXvjQncBvXg8su0XaJ9NDfdJZ07O8FUpm+GO+wF31/PGUII2itVA00aMiFCJw50Dzy789pxxIHru/5nSm0WNPhqfIcvIiQ5mW83qZ7Gydo3OexVJxsMi5Pqu5tK955Zbh+p2c3TsLdmvJt24r3v2gQcSWIFOCoAlZsvr5xJKPXaSzAdsli6DSOk1ddOIfHOceiOMdsgXXkOx9RRISRtdvEu6+71RcOBtRHoRP6n0pBKIPYwpEohLkhUKFdPwE7wkOnBXJJmrTZMD1W/aEOZVEKZLiWaT7QQoJQWKjxTuBNBpj0tSB43Cl0DsnokzvuY04mzIspEhuCyMUZQmDPOM9Q8nJrQW//DXDv3wZTq5LLgwFxkc6I9g9FLzRLolhjHMzikKgTZUY7jpNPbQA03MkvJB4IlqkYun4pI0yaUxC8EgcnhpnM06Oan72/iVKtrhlInNGKRC7Ba7zveUTlDCImBgOwXV40U93YkyhRaGPyyZhnltrMTrH6DxxKEJyDRAbRBQUMSP4Y43A3UoCFQICUKIvvmTKqPABrPc4F/FO4lzCfGc+0LqO23csUhlWz2eUpWZYJUeInQXibkOWRRZLS91GvDK0WMpKcPb8gMNcsKxT9vxiP1KWORvS0LWWbBw4vLTHO04MWF9R3Nh1PPrkJqMtwfWdBZMbC95+ruTE1oivXNljc6Pk33z9OhtnBjSq4+2PVtTzKZsrBU+tw699fo/7HlzhtfaIvXnHC2KVx0aRosk4dVpyc6fhjTtLXr7d8IELQ65/fYdvPjPnwXs7/sqfvsCzz9zhgx/b5Pc/e4uP/dA9/I1ffIE/+YObvOP9m/ziLz1P60d87089yD/8+W/wtrev8MT7Kn79X88Y6sh4TfMvfnvBUw/CT/zEOd5485DP/l7Lzhyu7zWcHa3z+demrA4LtG442gN/tuLMCpy6eJrxKWiWHXcOOrzP8NFj+8AqlCGIHJWR0L0yIpTCHUPIlKZZRg72lxzNPco4yiqiTEawmmbhoYtkRjDOamRY4OYSZxTSaNo+QdC5nGUnyYxjSGARMxY+ZyxSgqv2yQouMsmKjpwLgs2sZFwNyWUHrUMIxcYw48RiyXwe8TGCAZzg5DBy1ijuzYc8MA6YGBgODKWUrJeK0lRkWYNulgTXTzKjpO0EkwaWVpPpDm0yVirLO05VXDzjufmiRUnBe951LyfuWeNWc8DLN77JueWED9w75nQ2JViFqHJMDgiLx6GMQxUBkTngAEGbvtMR4nebM7BwSzYGQ4yOtC7iEHTSp9Q0F1ketUx8zmwhyI3kzkHDnWbG5uoAnQnEwiGN4GARWR1ISiVokATjyTRM5h0uwCgrUEJw66Bl3nlObhYMxi2agqOm4fbBlOsEioFBOYmQmrq1THVkYywoCsEgSyl5084ipWdjYNg/sNSLjpAZahtY1hFw5GPBeiUZlTm5EqiYOnyVgcwEQkWCEBgBVU/pC17QNRKlImWZonJclAQRUtacEmSZSLY5HFF2SD3CRMPaWoMoBFdfVvzuFxsO5pbT24J3Xii577RguOqpxoZBpohdwEeNVM1du9/UdigTeOe98O1zLc9eKdCidwwcWwaV7LMIAlGBILyFqxWid6Ed3yCid601RKmJYYIh57k3BvzKpwN/vO1w52B7FYrcU0hHRuwV/El8JoojilBR5JH10YKLJwRPn6tYPCCYLh3XDkc8u1OwkUX8MkMMbIovjst+r28QEbSQiVERY3Ij+JRup+hHy8kLAIiE/O1V5d4lnC2AygT49Nm5EPsDOFkKuy4VHdIfrx5SThkRXIgYoMhkSqlUSVnsPbguwYacFwQhcDEVW/LY4SBIAkopentUQug6H578tXYAAQAASURBVIkikOUq5bNL0heUNGkIQhBdoOsShMi51HWIkA5PSXKiRGXRuefUGvzgo3ucXc146GzFPSvQuo6bTeRb1wz/6tWOoybyrgsVp9cb5r7giAwtBUolQqaQESk0Uqrk+ghJefcWVTI5RmL0OBySgswsuHVQMw8VmYfMFIj2CHckaL1LtDaSQLN1DuchxvQ5dNEm+2lPiVRR9Q4IgQyRUX68tiDBko59n14nwmCMSJM0JFIkYaeMEhE00QucTZnubWdxnU/3gQeiJBBouoBQihOnhmSlZDpLgJrBUCDbJfawJsiSwdBQrlQpqjyfE2eBtmvJhiUOS1d3DKqSpj5E2ZqhVqhVx2LSsvnIKebPtdx485BrkyXffNaw4xre8WTG5d+7w2efn2JKzaPnR6zf3OGJd4+4tLPgR3/sNGfPDvnFX72MqTTrOrB9n+Ibz0z48Ec2ue/+Lb7w6TcZrqwimfDlLyk+/3xLk0kmTPmZp87BbE5xeo07b9zivovr/NiPn+Pv//JL/ImfvZdnX9vlt15YsnV+wSODETLf4Od/6RKf+6Llz//0BsXVazx6RjM2Jf/s373JzlQzmzj4nX0efnSFg1nDz/3KLl/41pIfeLJAe80PvG+T7vCQT/zxszzz0oR//i8c5y+s8alX93j3/eep1sBO5uRyzPm1iO0CzpXUNtDh8Dql6fkOjNR0XuF9hgueEB2ddLQ2IYU3tzKi0HRtpJ5LbK3AC3JlkVJQhwIRPNoEMtJ9LKRCNZ5NveSjZ5Z8OO4TXUmnBzQiJwrFemyQwmORCBfJpOJMVaT7ztRJhGoFmEQbPFMUHNYthzbggqfQBZtVxUqeCJrVsCOTBUXWIsUelRphTIYxkqyPOA8y4IRHeMXSClyQCJUCl8pScmF9wDvum/PqTc3ZM2d47B0XEVnFlVvfpNuf8oGzcJ+Z01hBiBHTzJA6h1ykpLtSJqeWToRbmVKZerH2d/bjOy4G6iLDa5GS3RaRaQv13KF0ATIyrRfMZrA/iQzzpDDvjKDMDMoq1rclo2LArZ2Gw0WLXs8ohMfiOJgFlvOYwCDWMl/U7CwcTRPJVSTfNqyMBXpQMa09dZuQkQWOjXWIc8vNHYE3iqaOKNGwXmqEFRzuW4wUaGUJIdI2HmUFK0qRBzBBUCpJVcC4NBgRcd4ldbZRoJLK3EWJioFMBaKM+JgEjMLAoEoqVRugDaH3tINWabTt1QAlA4KWNsDawPHxd+Xcs1Hyya8s+OQzlt/4/JTTGzkPndY8fNZy8bxiYyOwsiqoVIaWAWFSpGb0HWfWSu4973nussO5AEqlNUAIqGD6fPs0ghYq/HuagRRFnG6SgETKJiFelcT5AhmW5MBXn2mwteLHnhY8di5SnYLVArzwWOvQQaBljg8q7cLxRKWQ0iP1lJGRDEYZ92zt8N6HM4LPUNpiZYO0mugTg1wcuwqICDRG9sl5SqL7/TYx4pzD9QpyAgk005EoczZghSeKHnbkwFtP6A9spZJ/3tkUQ6x0OpxVzx5oXMAEyE0gQyJNUubjBcE5nO3H2uJYnNYXWirR/2Ti5QCC6CI+JIeAqTx5mVYEUUZ8SHz+SBrVx0bStSnMynuRAnaOBZ8hYltwMpAVGV2InN0I3HeqZKQMTs5oWmiUJGYgouQHHur4+ANLlJ/wvIfnjk4l6E/PIDi+/hJBFGk6E6PHe4v3/m5RkAqCyHLp6UzL95yr0UWknNfYRU1QEofDSI21bdrxk0KDctVPTaRnVaRHjJQyOTjoDdukz0KLiOsC1gK9ZZKYPkutQei08oleEJ2k7RKtMnYR7xKvPqlKJUroFHlMKhKD66BziFzgXctyLlFBI7rA4voy8SBcRuuXzGdgJi1BeWQmyKKiXnjaxZJQFNy+ZtnIG4ZSEiYdQmime5GbO6DGkcXSMBEDzg8LwLKaVezcvM2zr7Y8eHbAhz+4xrdf22WwvUJtHaHLWe47fu1TL/MzP3uRX/vda8z8mPk3Wn7yw/fwjcs3+NIXbrK7sGQbkTduKi4/v+D0Q4ZvXGv5nns2+eYr+1yfNbjXWt643fAnP7bOf/u/fZPtoearX5zx2S/v8f57S4aN47mDmi/99mV+4oMbfM9756yVkuAa/vpvLPj4+ww/8qHzXH15l6Lc5NKNKb/07w6wQnCqUjz96EkeeyBw5ZpjceD52I88yKc+9yaf+f05Dz2SMZ/tszU0/J0/2Odjr1r+2J89B17g6ogsoMJREnH9xDSEMulpZIeTGk+azGYqQchar3Bo2i6ys79kvghIK9G+JVM9ErhzZLLFZwoZRJrYtQ5pcjoVCW2LriGTnjyDk8WSlVGHNhohKmTQ4BTYAE1AoJODQDqM1qnps0m3spblnMoijWuwMaJDBOuoM4UZG3QXyXDoLAHLMtWmezC0EG3SOElJxOKjZBl8oqkGD6ah8CUnhpZ3nIm8/uCQx558nJXtdXaOJhzdvsP7B4H3r0hcmNH4Aik6CpkiU4USiEJBcewaS7RDfC/KDZE/9Oj/7hQD1/P7aH3LSbWgEy2mjWRRoU2KBnUodO6p8kDTgkOysTom7yJdSDa86Bomi0P2nUcVYwrb4MjZnzmESJjS2rYczRuiyigHir1JQ+MMQxPRWlDojKZucPMAucbhqIoh47xjFhTRe2ato1SwkmmsCzgLi1bQWMlaodioAoVKq4BBETDak+uCPBeUeSKVWZ+ESVJKglFY0kTASYERDumTAKprJEoEBpViVOaMdaRzlsY5vJAIY6hC8v2HTGAKydwpfID7L2r+7GrBe+8f8a++MOFrr0U+c7Xjs8JRjXO2T0rOn2l59wOSC+ckmyNJrtIBtWwEShm0nuJt6oISzS91tFKptDOKgRj+fQGhQCGEuzsd6PQSGQe4dkoIjswYZKwJHXzxm9MECjEFF7clQguEURiVrDu4DokiWN0f0sfYP4XWAZG3dGqEEXOCtUCBEUOiFhBbhHOklMX+ILKe4BNfXgDCpLF+7NcC3kusiykS2ov+oIDgkuNO6gQn1EEifcD5JKZUUiDxuJTxg/IRrdLhbnsehguCpYloAk6n+y0GSWggdr03XqTfA2n8HYLvyX9pJHc8afDBgUwxwRHoXB+JHCMhJTKBMHR1oO0CLsh0nSLIY/4CEhkNOnj80lIoj84rtPIIOg5rT92W3Jp7Xt/LuXcr44fvnXG+OGKqRsz2k81S8tYB72NERHeXRBmjI0TX5zl4Ykh72ZQd7yiywF99quPRLUlYBJws0VWb1iuiICpPVpm7osQ0TUkKcR/BedHzJkIvzOyLESGRCrooiAakTisC5wLBJgcKCLqlBScT2Cn0as205UEpUCbhZIMPiakRIcj0+oWI5BKC9UxvRGx0WOdZPzHi1AMnmE2XzPdqfKMZrFSsbEPEs9zrkHaJ0B6VDanKQDzqIGY4pdm/WpPlghdueV54Y0lj97gw1lzvlnz9huTiyYyttY5LL1geeGhAJlpeen2CzjX3nM3ZeX3O09+7zbNfucGPf+IxPvlbr3GqGvGet2/w+s4+3745wYcVHr4Aj7xrzNVrcyYHlpVNz3Qm+b63ZfzG5/Z4+vFNfNZyvirYHFqGNrIxjHzkow/xqd/6Nn/uh89ya1nzD373JmdFxv/7P1th5nMeOoI3JvClV5f8l//Jw4RwyDPP3OYL32z42PvX+DcvzNk+kXF0y/Lo+ZyPfE9Orme89OaM37kMX/u7Hc++VPMjHx3yw28b8POfj1SDyCP3Sl6+Yfmv/vot/rM/NWLrbSOkaqFdEmqJEkV/4DiECASdp+eUd8QeauW6QFN76tbRWomJYELS8ORSoIQnKgd5hiRDCo9UOUrDSHqGBQxGBdWwRMiACBoroW0sh60mLlP8tVYWnQfMQJINBLoB0YlEv2x8v0LsReExsJFpDq1iFiNFAI2FNhXVRpcI5sm1FRRDLEVvHfciEoNHyopcJ1bIUWtpOkEse4SykYwH8MCJnCce22T14jqNcNzefZN7uo7vP20ZoZgFg4wWBSih6X3qHMe3CS/BaQw5USUuCjEQXeA7qQe+42LgBQx+VqBCztBbHs0bntyWVMEx3VsSZMXWuqcqNJeuLVl6OCEMh0cTZl5gkDRa01Sacu0i0+i4cesKskkPBTmIuMYz6aBxhjUjWVvRTOuMvUlHLQXjwpNnGUpHFstIJxTTuqP1Fq06VmTGcBzYncikDgXyEEEISiMJnaSMsD2QVP1naTLdp8gFVBbQpUQpQeYFvhPg0yTAmoDPkkshOo2xEWGTlamtkx2qyAOrA400ioV1HNWBpU2dtBQS4QIZgrEwzH1MI9Sh57HHLGdPa16+EvnCS5ZnLsHBQcOlVwRXXtF87g8sZ04WbGwv2BoU5FJyOD/k1nVNa3U/Bg93uy4hPN6nm0GJZB0UvQ9dQJpDC0XshSXBFyjvEdKgVUvoFJNYcvbMIX/+h7b5vscazp41jCqHqgRoB7IDq8DnPS+g6ykKKRIZkQA+eE3ml0COGLokJuo6RMwgSlwQKeArmeVJ36CkKE/ugf499O8jhrQ/djYgfZoAyH6wEH0gujR+0yhUv7sWJEiIEAn3GwSk0J8k8mxdwLlkL2gbaPA4GZBGoJDELhDbSAiyz1pPzaiIkihdDygBSDhkQboeyhgikqZx4JKmJJU4EiEheEnbOFqX3n8MJOtdUk4ikBjR4BnQyEOgAOVooqBuPTMcLTWvHmbMYuR9FxxrpWRZVlzbDRwt0yMgESE9PihCSPkGsS8OvXfp+sV+XRDCXc1ACB3/84dmnBhPsEc5Ku+Q0hBzhVUqrYtCsnvGPkSJ2E9XVfrvQK/qj0mT4F2asNgeK2y9S8VDG3tbYTLdet8RnQA0iog0/X3bT4XwEK0guBSLHHxIwsueXieV7sNnwItANjBUKzmLhcW3keXRhHJgKO4bMz+oCZ3F1RKVScqVAcGP0EctSkeKMawOR9h5RwwCvVYwaQIf+OAJPvzknNdfmvP5NztWqorH7h2zHg1HBwtOr1pqA0fLwMbIsHHa8PpzB9x/bsgbV/boVMWr1w4xKxWPP1rywpt3KGTHDjn3npbkYcJht8nBzoL3v22b6zf2Ke4RvPqK5fGHV1k7ZxgvAwO9ZG11jUu3Jhx0hr/xSy/y9g3Jl19rcK7jP/+Je1l9WPBvPr3D53//Nme34RvXM37kMYNqD5juWlYZ4bOG2Dbce1IyOfS8/x6BGguu7Fk+/UzDzg2LkmBLxR/74yfYef2Qf/mVJZP5HKmH3Hi+wR56XpjBp//Sgj/yriU/8KMDHnsioxgbMBHEjBAlwo+QsUZ4mwo3qZK4udQMVjVm6WhuCcK0Ic4V0mR46dJzy+n0FTGeTCgELZ5Ikwd0lKiFx7YSk4POOrSMrJQGleVpOiZsEnfbjOhAqkgoO3yVgGqyiYiFIraCaB1ORLIYGSFpUFRCIRwQJC44Yt5RSkHmDKowbEjHuMgosiFTM8PbllxWDAwsXcesDjQ2rSycz8mER2aa7fWc+4qz7I80d/Z22Du6wQdWGu4rJMvYEO0AHRtMrnGu/54JgZcR4W0it/q0nnRWIEXsgxkl30mI8XfuJmhn5KrCYTjyguVwjcOs4JRdkIldxtxmywhK56iXFl8OmETDLN+iM5KgPbv5Kp1pGPqO5ZUZbmawwuOD56QqiSrHHu1DMBx2Gq09G6UhU57p0rEXFNFajDJAoJBLzqwrmk5yMM24edhhgyRXmhMlaB1ZNB4RA5mISOnQUpGVJVURkXi0joxKSVkqZBXRQ0kpk0/V+0Cwnq4LGKmRIuJkIOQihd1IT8wcMmYs20Cz11F7z9aGZq3QDCXM5nBgU/iL6uOEXfBJJAa0QdDFnGIceddj8MjFjI/f7vjWlcA3LgUu3YLlUnHjWse1NyP6LqRb4UQHGFSUBOGJRYYWEd3VxEzRRsHQJ1GYj31nJkUfJpQQlVJKvOwweKw3iMwQ2wVZUfOXfmqD/8sPQjUwRB3TyDYGQgClchCBqC1EibIGnAHhwBqgRUmNEwKvI0rYZPUL6aCLPjkdlGgJ0aT6JPTiPp+CaKxTNEi0jkhcCi4KEL1ABElnHTEIZACtkrXQCImUkSg9GYrQpY7QREUuI7UM1Ba814hoQASaJXQWymHAushyrtAi6UXuMg169wJAVIKgIsGktUaMASVSyJMIIvnbVRLqWZf0AcpECC5x/4VKJEjv05QjgMCj0YiYwnsiIE0gixnOdQiREfAEH6k7kUJ0QsUdl7OoNe8+6XhkbUJQgaZW7DNkLhWu9QylQXiLk6nrL3SBcw01FulSrLP3iZZoXYePAesFdReZzHNOrBhmU8msLonigEIOkTmYPBKEQckOTUPsSkQo8CwJCKLTBPxbjo2QHlYh+KT0jxClJpOQG0FWpKwCIZJWw4ekrA5OJI1FmjH340+RBKXSQxBkUiJMWid4F7EuaTuiDOg8ogcBvVKwumHojhz+yFHvWtSqoBzoFFM7jahxIKpIrB0bJyq8B5VpDg4b3nz9EDWP3PfoNvPdlq//wQ5PPnWKctXzYCc5d6rgmRf3+Z2dlnvOFxRCgFR84EMbHOxN6KxisFZRjHNefvOQ977nFF/55k1+8H2b/OZv3cavjxmVgjOrq1zfm/Fbv9fw1COHyFwyqmrO3jfk6y9NeOrD56hv7XHj2pRz64rFYclwaNmdNpweFQyj4D3vGPBvPr9H2BzAN64Sviw5vb3NJ35QM1or+MvnNG9MI91iyr95dcnXvxH57z9xloNqiXsh8MNPneDF6w1feqnlxM09nn0dTqwWfOJHS0KmuPTqgtd2YDRsKUXOYlYz3YeFFWxET3Wh4Is7c577JzPu+w3BI+dzLl40nLnHMN4QUNQpBl47lE5Tn8Ussr/rONh1zKce7wSm0GgjCKSo60jiTESfwGdWp+5dRLBBcFAHDoTlGKUu+vWYFC1GteQmhdFVhaDMI1pLrAKpTP9rIWgBG0ksrOZgpoIuCtajohISjUKG1PEjPLIzeBkxWuJ1ZF15VpWnUwIlFUJUCG/xQBcCs06x3waCjwgjQRQQDhmOVhHFGFkbbk13ONXNeWKksdLTklOKFmkkMjoEBd555NISlKDzksaLlJzapZRShSRGT9N0nP5uFgOb1QliFGhl0EVJ7TpeP7K8rjLKExc5322wzDxyZcrizB57A8OVSiEUSDylK+kO93goOAZN5Mr+jI5IpjVVoVgtI6GrWRYaK3Ia23H9wHE0yCi1oNWRSnmicxweNMwbQTnMGRQS4QXLhSPY0EdHRopMkRtDsAJnOwaZIDeCtYFkPBCMS4UUCiEFWSHIC8gKRWEEhUk2MaUkmcpZLiyzhaVvHtP4xaQRj/QB4RdoJfDesHfgOZo6VsYZq2NDXgTWdWTaWeoQEVEjnSBzIWWtaIUJ7V0B3HAkeHRVce+9GR94Z+TOHcsLtyK3DwM7R5Kj1jFbQOgiVYRSWbY2crZPFNyzabh92PHFVxRHB4rMBWpt0crcHSWFmMa/ISQdgeqDk9oYIXjMUtAFqDLDwxcKqoEnDvqY3KgBnRKICamR1yBDhKjApCo/5gHpc4ge6ejFLAkaE0U/sgoR4YHW9ON+marYmF5j7EMERAh0tkcryz6J8Lj7ixrnkqDQ96NEH/vAIUmPEPakZ4OgEBkZgcZ1BOFwosMTsV6CeAvEs+wCsg8rCcIRASMTSjdG9xa90YM0KkG4fPq7u+jvAo1iX1BGEdK91jMPQkwdubU2qYwRPZsgJL6BFkQV7wYhOZtobK3vsQo+YnF4ETC246GNjNWVjJVM44Jn6RXLztDNLZ2y3IkOEwRDmxG7wEI2CCBzMO+nSN5bOtfRdQ0+eiIBjeKvfg3+69Lw2IkF0ksOZhntMu1auxAwtsHlkmlTkRmJZ4GPkkwJ8PauYDWQwFsARksymaiEhfao44S46NMqIKZDVAuR3Ba6r8NEuvekFD2ECBBp8hB9yhnpoks4bi3SllpHdm9p9r7pQB4wGGdoLRhWhq37BqgioEKgWYV2GpndDuRrBpkVzO/UaeecF2Sd49yFM9zeafjCl484sao4c7pkutuiihzX1LzwrRkxVjz1hAEvuHkQefxsxrpQbJ4Y8cVLU8bDVYKs2T65zktvTKBV3HltycbJkh/8mQt88atzLq4P+Wefvs07P7zOxlhy56sNa6sj9GzO2x4Y8qlnX2PvWuTpR85yp62pBjVvtjBaHzI/sLz/iZyvXJlx5kLg5UtTHnjoFNOFY//wDS4dDth9cUaIgYvrGV+/VHNmc4Vf/+9O8yu/+SpvzDL+3B89w0e+b4MXX9gnns756//jTe4/63nqYsHL1zP+5e/u8FMfGvGT74XLhwVfen2GnMDT7xxz8kzgypWO1y85VgaelQIoBDePHHe+IeCbgSpzGK/AdHdDyY5R4tpAUWiywiTNSH9tk8okXfLjWHVM+h7KPpn0WLMiZXpeibf2UkSREl3r6Kmt46hLEDWtBEomIZ/qMzNijOQCRCbIlaQoNN4rRKsxbW/TE73KKaQEVKFByAgykmkY5YFKR1okjU/5B54GJxzLaNlpT7LsalYyQMxBCuZhmzpU3K7fwO/t8t61gq0S8JFCeIyIqKAQMtmDo5d0C0ndRGoZccojZUDLCE4ge9GwfIs5990pBpx/HYundR2ykRgxYGV8GlWewImcq3qFq/YAVzU0J0sKG8l2p5yKUFjP7p055QgGm2NQkqyUqEzRtBrXWo66VGhMmyWemkobqixjrRqg/BKtAutlRakkOTVdl3Kuu7rFd4lJXhWaYaXTWNk2dM5SCk1eKNYLTW4Eq2PB6hCGZbIkRSnQKqC1IFMiAYckKB3Ic0lVCPJME3DMbQCSdSroSNQR6SD6nBhiUmzHSOcCt47mHC0UK0XBoEhBLV6BBZz0qWNBULaejgInBY6ADRHvknL67CacWZe840GNjYpZ4ziadbg2cnq94N61kjWZURSOaT3nygR+4ysGueygs2RjgyvHGNXevdG9S8Kr2B/Q6YDqI2tDuolCVMyXjqVdAgobA3lQRB+Jokvrbpc6XKQAmyYUQSpiUIk+hgPRIk3RCyoNIAk+EDqLdokNgFPQOKID51Rq/EJvIIweQiBGgeskXkjyTKK1S+jhKPAupOY9pIRI6yMyUygDORohJaJJFkGNIi80WfAJuqRFb0MDfMCHSNOBbxP+1qhj90XEySQIFTEgSDnqxyp4ekthONZsyIjJ+sqxBxolq2IvagzpIZKyFVQ6DEWafWspe7Fd+q1dm3DOSx/pXBJ8RpFqL2TkRJWxKWqUbOmcTHG+wTBpC5ZWIWLq/pGCpbRoadAyTdbmztL2wVTeW6y1WNclFLEEJSSLGPmvP3WCn71/hT/+wAEj1XErV9StQzuJ1CqNJKXBhTmoiIkC6SOiyKBt0UKQSYUSCeIldFrlJAZ9er74GEFolDRpCuNJkwThEUomXuexI8amlUB0SeQVSPeI9QJHsrKhLSIEdq56dnYctfN0vuP8Vs49D46x7ZLp4YzMBXShkJVENh2iFdiDDpUZloc1h41g7aRgWEqWBwua2jKZw3LpOH1WszysWS1yHr5vi+pkwa3bB7x2ueO12y2Pv3OFNnTszD1ffmnOynDI2x9W/OtPHXDxbJpUPv3D5/nMb73O2584wxtff4XmWqRWA95xJufBi2t86/JN3vnuDaISvHpoufLCgo89cY74rgWrleBbzwbm0jA9qBlkgQy4cdjy5Tc6toeSzXLAuS3J1652XLmdceHCCqv5jKWFS3sL3vvgmCffUfFzv3qZV99Q/MxPjvnUtw745S9OOLduefVV+LH3jFhfa/ncG4EXvr7D/+mjJWas+a0ve979QOSHnljl668vuOdUxvM3DpnfLqndknUhOX1yQGgsSitKFcjKFGxW4EDJ5OmXKmVJaIlRMYHcAun498cDuWNmauJ7JHL4cUrqW1HpaVyUdDJpWtc7fKRItl6R/oyoJEEIvHeEGMiUxGQ6AUGDo3Eau4y0tiPgKIRilCtKK4hNwp0LpRHBvSWKFTLNbEWk0qpfYST9UC5zpNSAo7OOmesQFmgNaMdEDXkprDLxcPtwn03b8J4VzVhYvBCUpsepO3rEfXLpeAfCpjW1ziVZJchMxKhUdCshMeo7qwa+cxzxbIGKjjIzyZbhjtipF4xXjyhHq2S+xR/uksUFY5kRJh69N6fSGtsEQgPrVYmbpLCH7RXN3MF8t6NwlvksjedW18aJ++5hVARWsiVaF8hmQd00NEIjC8nWiXT4LBvF/kGHVnkSb1mPVBIjNbmBYaYY6shIK7SODLJIrkJyRRgJOjEOtBEYkxTMOo9UA02Wg9KOyihs0Lipo7UxdfSq5/NrCEYmS5hLudjaaGzI6Kzl5nSBmGvKTDPO0+TBS7CFSLszJxChS0WF1YS+SwxY2t7TjuqQQrJeSNbGFUZLTq0FTg8DQwe73rPfCn7/ascX3wz4PGd4DuRYsp5LXEiZ9gDee4JNFqzg0hfJ4ci9xNmUBZC1hvnc8bmXWn7k3Scwdgk6AhoRmlRI+DypZEPEaYvxSbwZNX0OQEkQaV2A9AjnEB1pEhF7UWATaJqI7ELvww0c04F7yT4hpJ2wtR4hPEKmBEFjTG8bDPhOEqNMuOJcIjKRrIMGRCEQw4jrItIKbGZoDdReYqOgCQErAxJPG0ndNxGiRVuP9AJDupekOtYZSHyI6CxgokAF0e/HU9ciDUilEgshpKIzBEn0x/yEJKBTQvWZKAGlemulSMI56zzWe2wQeBHTOk2m2GMhBSnbUZGjaBG0QSW0tXQcNZHby5y5yCmDhRjxURKcSxkZYZaimkWKKA7e41zXCxuPdyEBKSWlMoio+cevBr6xt8UnLky5d3XCTCuki8y8I3fgVEuUioG1qGiJqkJYz3hQJDtjQnSmh/lxnxdD393LlKPRP9iP3SURiE6Bl0kL6yLRJhaFt30kNQkGE2ISlkkRIWTMppKda3Mmhx6fw3BLcf7e0xQqUO/PyIVOyvQsY9pEsipn/fE1lG+we0sIAjMcsDxcInzEhYxmcohxGVv3FFy/smB5pDi1bjh1Eq4dHPDaVx2v7wSuzD1ve/sGOjQMBiOarubj7ztD20258rrlyQcGDLcKuoni5t4cUyi2ckct1vm+7xnwy//sRbbOnaBSNY+f3GY36/jKMzdYW1nnwx/XxGnH0hf8D796nfuGA86dh/P3Zty66ShOah5dG/ITDuadonYdr1zvuH4l8raHVnjHI/AH33asbuScvW+Dzi65fGnJT//QWYZuwb99fsFrlyyNtsSlYf2047X9Od/6csPlq4L/80+f4L7zki9+dZ+T5zJWNzW3bjnec6/m2den/PrnDZt5wx/9voz7zuVsr5RceukOuVZoQIYFKg6TAlToNGZXAaHaPpmy15fEPn/k+BBP26GeEprWtKk2FG/dRjEV68SIVxBVinKXHrTsO+Wexhp8L/BVCoHEt45l15BlhiLP2Kos5ZogqJJFEziYBCY+PZYGPvZTvLeEeSKNAhHBIWKXkMkqQxmN1hKlcjKRkcdI9D4lGMYc7w9RQnLTneBlmzOrZ3TzmkdXMk4WLdCisgyhAi6qtF5LcmUQAaUiWezfp9bpuScjWoWUPRK/0xP+/49iQKgW4dIH3wiIRTpkt8M+a3s7rHWWsLcg9xFVDdiZJLHNijHMmg6cZNpYZu2C1VHOuZWKeGBRnWX9REGewc29BWpYUA40thOgOxoC7bLFtoLDeWDRLRgPDVWuUUFgtCZEx2TWsjZQCJO800WWVgGrlWKUhaQDEJEiixTGpDCeHEQOQgu0TGNhk0nyoaYYqrvoWFRgMFI4G5mESNs/goxI++OITYx3lQQpOI8ykTzTCDT1LHIwt0yWHWUuyUyymQkt0ZlBZWmXKjKQ1ifPdCAJ8SgQpsW2EhvpD1ZPOIRZLTBxya16wO+9Ivn91z1TFVl/MEfng543cIRRyUMr+kq66xxd3eFtnyAYC4QPdLWjlR1ZLambnE9+0fKJD3neeVHj4xIlcyIZMXYEIRHOIV1EhyFQoLxm6SquziVDfcjmqEkSxZilYOY+a0HECF5gg8C6VEj14X0oEYk+0vWgmc4K2iZBfvJS4YJLVEYlMZlAm2PxXw8lkim1LkSfRHBaIDNJnku0S0jRRgbqWWA598zbiPVpvdS2yR4aRUpFw0p84xhmsGY0WRTooPvAlIgwxysPCLGfChCTLVEm/Khw9MLInpHgRFqZ9IFKSlq0Aq102vEFQdNGlp3HE2klSTzXF61Ij+x/nQ9go6MzhiALpLS0IbC/LDisS7zw1NEnu533tLbBaJU6DJvWPEomC2GInohHqBRLfRfY5CJCWkZrGc8uJK8+J3jbhuD9G44nxjPOFRKRVyjV0jmHHKyljix0kGcQbXJGHNMDOXaGpPRBafNeK0LCE4feHeLTXli2Ad87DIJL90b0aUTqg6eLDVJl6TZOlCqWy8DeTkOwsLmVka/kbJyTrJ5saOaC3Vtp9XBiu8D5SDZZ4PYtr742YbhWkq9UTPamaASDrRPs7tZc+/KUzZMVk/mMsTF8/0c2mewr5rtTnn3Bc2tm2Dt05GuaJ8/lZHLJxnrG2obhW89PWM88z7045dwjK+gscHRrweq2YnkNPvqBbb7wrVs88rZ7+fl//hKnHjzLD/7INn/nf3iGt91zD1sn5zx6asTCw83bNadGFXd29vFO8q5HKs4+lvO5zy7YHnmczHn20gH7h57NoeL2bclG1tKFGdlwjVs3lry4W6NuKlQzZ7II5KcUujjin35+j9mk4sI5SVd37CA5nQ+4Pm3ZawxPXMx449qUX/m0Z4Dmpz8cWO4sOKg9X/5Sxr6z/MjThrZ2XN+v2J8vWWtrHtouKBBkeUFmAkvX0OY5uV+itO45GFla7wmBUCqJdMVbUcJ9451AV6I/+fuTLqWSyn7Vdnw8SxJsmx5lnsSxvg8ri8eY8pBEz6IvwrsuMm275JbREoTFdY7GQdVpKpvSC+nXF8eR4bF3ukTviM6RBU+uMlSWoTJNjDHZe3uZ9V4jmLqcQbC0doXLPsXTTye30E3DQ9sgVMRoAzo1KAGFkx4ZIxHTVz/p+ytCRHaxt4cn/YTo0ka1aWDw3SwGKlWhTWQInM805xGcIpB3C1T0sFC8OVdMXccwb1k1lhqNjZKgUvTqbCkYSYNQmm4YaX3kqLbIVrKpDLbTXL3VsLZWsGIEyyXYWGKbhq4NKFWQG1BSEr1kOamxFirSgV4ZycoABplkUCiqQjDQglEuqbJ0ITIt0k4qA5GFvhgA1WN5lVJkRdphIkMKF5LJgjJ0mi6kMb0gmcs9iXhlI3TBQQYqS7tuiSTHkA0EMzwHdcOBVyiVeAa5cpTGo5RK8avaYnJQmaHo9+Fd19IE0thbCULwuCiZe7iziExmkW9e73jhdkE3XuGe8zoVDF1IE41M9R7wpOwOIfz/WPuvWN3S/LwT+/3ftNb60k4nhzqVq7qrI7ub3c1mk80gWqREUaM0hseCNaAwMAwDmoGNge25MGDAczHQxQCGPZYwkkYzI41ACWNaIsUgxiab3SQ7d3WoXCeHfXb60gpv8sW7TrUu20AfoIA6p07t8O31ve8/PM/vwfeavisHrxIDXaYPnnkGOwm0qwEG4Z13en7tz054dm+H6SKONjpNHke5arQDide8xQWO2OWoz9x62PLB5ojzOmH0orCEReilZpNhpjqs2pAsSFW4DCmXh1ohI6YUcp+IITO04HNCOUvjJmgFMQhZIq7RKFscI4niqyUXul1MI1wo61G0lqhsZmf6ZB1RRvDdeLGuV4HeU7zISeg3CadhOjWkSjEMnuwTCs34WJQVxrimKFFHCUYro1AuuZQzKRcPMlEwRrAUa5xzusQzZyEHoesim87T5bKnSDaCUyirCxMgFWGdQEln1EKUhNBCjBx1FXfbXba+wqlyISYVi34DX9jnI28gK2HwbYEhUfadT3DJSnQBQwlIarDtisvG0CrFnx5O+dZRxVRqrjcrblxQvFwFfvL5BpmsaZXCSkBij84N2fvSifnR2aEtolxxVwSBIRDaCAPoVOyc3kdiSKhk3rOOFtNDhlR4GiSNyxVD7wuiNhm6IZMks7trCQ7sjmV3v8Z74e1vdzRN5uKFmugz26MVoUssH0YCCnYNfWjhwRo1KKgUw1lLbAdM1aJD4oUXL/DGuy1m5WiXx9y/t+Zoo9m/pJgvPCdnicfZ8Is/fYUHj5f86Tduc/3yHm9vH8IljW48b97c8NL7LvH1b56yezClOV3z8jMXGKzj2o3z/MXPPc//6x9+jRffd4l+nvmXf7rh7KjjFz57wCJpfu/PH/PqI+HnPnGBZ1+qWN9Z8eCxp1YVZ9sj/uYnbvCFN054dm7YW3ici3zlHce3fv8eH7w24dq1c3zo4g63bp5w5YbmD76z4r/+7RWzecX5Ky2zWcVaOT5z3fHKB+Drb9Sca1Y4N/D2fc8U+OSLc2499tw6CfigeLhu+dR1zdHGELueZrXia481V8zA+Z3A+UVFUGtUathrNDgNoosLKpWJoDYa9WT3DqVrV6o4d8bpUrHXl/f4ExppHlHi6ommS8qeXMVSCTzZ5WdVnC8yVvBlsDCu3YCUYxGNk1Gp7HSDJLQI55LmfDBUWdgkjaRY5AihFCU5FnibSoocoWGgkaY4nnIi0qJsWQm0WTG0PZs2k+yMN9QFXkuO6D2nqzMumYGnmgIHM4ZSkEiJSfc+sGMUWpXXL4wOqxwjaEGF8lqFpBliIqAYftgEwr9VKRYC8zSwMB4rmSFFuhgQ7Uh7gluDPYNZ1MwniptHiYenA12I2KjoBuEU6IaBIAMpF1Qv28TQQ7+MbDZQq8T5y5bWZx493GJcAUo01tOYhI2hKMgTWJWZ71gGn6lNZLdWTB00VWRSK2aNoamh0UUwYjU0dUKX7KAiHLFFL9Do4irQFtChVJBalwdHEm6hmUZFnzx91KVQyIlILF2WLbkEIZUdlVYZTcDrjDUwVTWbVuijYdCZLgZWwTPNBiN2HC8XVrd2A1YLpjJkrehjzzpFOq/ZDhUP2si7K+HO0YSzOGf/YM4lm9E2Y6p9kgKfOhqVQZURcEpFMR59wPclSS9nQceEC4asBqZOE93A5NSz7CP/8g9O+YsfmvOxnQbFuMYNeQybEZLW3OUqXz15lkcDrDdvc3Lacj8u+OLdhq5y7M0nVCRapixU5CcPTrihOyoSRvUE40r3l2BAlzWDFJ+/i+BtYugjPnjEWIzRhM7jY0RCuRRzSqMPHVRWZA8pl0tNyAVbPLoUKq3YaYqi31WRbUgsN+X51HXRa3TbSMBzbq9hb79B+qGM+O0Yt6rKOFOChpgxqOJAUHmE/JSlvxr3fDEWUZ4WwSJUBqwFawqOeRgyQxcZujGO1wKuJBCKGUfrWY+71LG0k8yQi/DOKVhHx9vrBbfXNUMM2FT4AesY0FYxtdCFFZvcI2JosiXQoSlMd1Gl2CtEQY3RFh0EtCfOKnxM5LZlYQ0DLUfR8/B0xu/e37LpdvjE6y3/+U9bXj7w5eDLvqx4GwNiym51gNhC3iTSNuLPRjtqlu9PXAC0Ay34MJTO0Mj3NwuprLPSEKGjODoKxgZXZ3zO9D4TtpbuVFjdXdL5SLOw7O/PCX1AvMeKpro4obki9JtIc6BKaNim6Ha6rsd3HZcvV1x65jLf+fYR995o2a0EdXjIo3sdV549z899cIfP/8lt7i4rzp03zHfhNz7/kIsXdvgLH3uaw8MlSSyXJzU3HyRWww6r056nDqY8+8ln+dbrGzYn77LYTTz7vh3+0X//B7z4ygtcOpjwb3/tXf76T13j228+5nv3FO+80/O+5w+43S5ZXLR8/gv3+PEP7fG4W/OLP7rPa4cbXru75Hi94k0z587NjsuLGRerDX/7Z67wtdc7lrd7NvUDTraOr/7Bku8eed6/Y7l8OfP2o8zmpOPZSxfITcd/8U9O+cUfu8gvfXzGpq/565+CxTn4l3/ymP/n/xT4m5+t2LGJycRwbm/ON/78hPP7DtkVusPASsPrjzIHuxN2zQoTwfdbKiUYW6OMQp6kgz1p48cVvNGMfwaiSyGAouz/lS5FwOgWKMJuebIrKB9HlcKhcCfGqeS4ZlBx5JQ8cRDkAglTCEZpshGCjhiJ7A+aC73FDJZOQUUg5rGZykWjkFJxvGStiLHMJIqTQJNIaGWxymKNoveZzieOe6jTPq/mPR7FinbbEreBD84Gzo05G0NMKJFC/A2RnIomjhRIUs6KkAre34qU10xn2pRph0hPxqcfbFfwAxcDH6UljJGjVaWYNhO2rQetSMoyGSJ+apCtgqSprFCHM9QKjFgQxwzPWRpY48iPoe0HvGgu+wqjA67ONL3QoKltpqoTYZk4WwU2HrbSc+HAoSWiBSqn8TngXGavymgN+9Oa2iiaKjFfGFwtGB2pjcEosCZSVwUlK0aD1WgHlVNMjNA0qqSfad4TfqGFrIsivBksTRsZhnLokCEoQ5KEynEMfikXeIG8eASorMHXwqof2HZbYtRYK1gx9MYDHSaBwxbF6FAKlSg9Eg2D1CyBo63ntIXjtuF0aPB6wsX9hkljsG5CVVWIdGgDOdeEHMmUGOOYPCGWbPC+D3Sbnq4d6M0SZ+csWULU6Ilw7saU+d4eN293fOXmkg+8b4/G9CRVWjTty876LDu+cjzj7vaEpAMuDRxMM9+9H3n38JRazzg4J8wqIVYN12zLJ+0SMWuymiF5jmWLUkJMgveZFALEMnGZ1roI+VpN0BEfW6w1KCeo8RDJUb2XvCelmSUFIcQMRJSKZDEo5Rh8YOgDPhSrZWMTSgvbDqpasPWocrea3b0plxYVDYlNiIhzVBODVrHs+pVlaAd8F6iVKlGt6smocrQMZshjJy+qBCdVVnAWrMkEHwruuI8M/aiB0gpxudhWdbG58t7kZCRHiiamEbiiBwKew63h1qrhJGqsyvQ+0ErEktHRs8ktiZZrizkpwr2jY8zClbx5AkZltEllZZEErQJmqpHU4lPCUTGTik2C3m8wwwbRhkuVwk077q4V/9mvDvyVDxj+8nNwdSoYVywQKhdRYG4FaRV5m8kD4Ewxmqg8EhEBIkRfDBtBlSLP55H5UcazRZMqMM2EASQo1Kh7SUNZMXUx0npP8pmQ4NFR4Oh4zbWrhrkFQiauelJtsFbY3uroYubxcWJ3z3Kwr9hsMqd3tuRZj8uW1988YjkxmHlDc+ECq5Mt/+63H/DGMdja8ODM02fDYuH46PsMtw6P+epbLecOKo4k8dbdY2586IDvHfYYiXzlN7/ES89f5dV7if/0U5f5H373azz78U8S2jP+3//su/yd/+AK37j1gDduB37iZcczF86xWR3ykUtCkza8eG2f/+6PHvPCwQ7fePcBP3Z1l3tZeGV2kcOjDZ/94D7zecfv/Klw5j2vPxr4yHnNzqzhD18/5KPXHf+7Ty54vDH8899a88u/dIm/9Av7rJdnvHEn8ctXD/j8l7b8t7+95ec+uOJv/Mw5/tGvrGhXiv/VT0wxw4apafiZl/b4wt0zbh0LTZ04XcNf/pGGVRBu39xy4/SUxa5lsisoN0XFBMmjRaMwpDzme6iMMqMLYBQ4lyJg1JEkRqeA/HsrA/VeQYCMzicFWTyo0X0y1hkaQQdIokiaolORkUb6hMUigsm5JBhuHbutIY00syoGWhVJvjREMlobCxZdkQT6nMnOgW3QdYUeHCo7VGywucNK4mGO3B/26cN5HkhFnbY87u5Th5b3V1OQNRop7hhvGYYBHyPOTgl4TBwJpmUoihaNVaY43WJZpfou0gPDD7sYSKLBlZ2EtoadOSwaxcnacdINZGdYOMfJbMlsDyod2dQVG2AwA9oqBrFMM5imjFQYyhvy3rZFeUXfSskEsJ6+NRgjNFNFSAadUhn767LzNzFgciqebSPYSjHRholLTCYRVylcLTR1GbGLTlRiqLQeO/CMtSAWnDVYB9hAcAljVcH5SkJU8ZlHZRAbcJPEpNZsurIFHlQmS+luAnacTXlGinqRQSXNYDRUHdoM3O3g3a5kGixEFfWnKtZHrUrClzEldtZVDTYX5XdCM/SK07ZjiSY6oakzs3nFbDqlrjRNVaH1hCEEQh7I2bOVAUkK6SNqKNV0FzRD7IGzYgjIK2wugCQkwq7Q7Cpu7Cd++63I34pQTUdAUNxn2Flhes3pSc3tw8hxTmQDV7Ji31Ys656jWiOhR1pLp2qsWpJV5p12ztzCTl5jTEBng4kDvVSIN5gYGBjIPhBrizaaycRC7BFvSJVGnIesceMuPQxjZc737X0OW0SIWZN1BgmorFG9RsUBZT2N1RA1dS8MYyKZ1pZLexZrhJQCy22HqjS7U8duY4g5MSSh6zoGI3gBnSI6WpIU+6OQiigujqImq1FaoU2kMhmjNcMg+A1jKmLBsg42EqpYmOOisCnhRZERaikoVk8kykAUBXrAGDhtF7zeDhxvK6Y0tOoxqIzNkUq1xFijrOXxcs3feUrxiz/Z8Z1vw3/5tTMGvcOuHui1ULlA28HVmWYbWnxwVDPDpC/ageUA5AFjA35YQE6s1RkMRYtw4uF//GambzO//IplZwEp6pGCOfIWVEKmhpwMU19GxAVmmfB9KgWhpyRPxhJLiwgpm3EdMEYWU4SoKclocxWyFrTNNHWiGjITr2DqaPYsaUjMHJy7aBmSELOmb7ccP+pJvqLvNTv7hpffJ4S4xXthudJEpdgebjGt43MfOkem4+23t4TTnvv03DzLqGkFQ8tsNmE2tbz84pRf/dIDmlrzgRt7nGrP6cmSq1ctJmQ+9dKcnZ2Kb91cYpsZf+GzO7y+2fLS+9/PuWsT/pt//GX+0mdvEKTlN7+64cLehDZnjk4e8OyFc7zvJeFf/c5d9qoJl885PvK84w++rvjasufBSY9dVmjXcnWi+N6rPR++tEP/CKabNenijFVb8X/+m89x6+gBr73T8fBUcdB0fOlbZ7x6v6WutuzZTN9PSHnNz3xUI7Hm//JPH3J81/BzP2q4dbZh4Q2f/eyCf/hrD5CQ+Gs/DUf3Ez/9YxcR1fKvfr/l7ibi39DMXjHs2AGjKtys2L6VFKiYHsf1iVTsowLBUBIphaKXGacElAn5iFyHMvKX8czOo75AvXexl3XAE11MJo4lpxrj8grNMpN0saJaFHvJMB0cZCHkRO6H8jyKokpCzJagAjF5qgFyNvQ5oDAESiE9V4I2lkEpJHu0ivQqk/xAH+CNMOU8hpNhYJmEo03PcxXs2jJR8yETgyHERPQZpxUTHZBkih4Kg489BI81cySCpEwfhNgGdFfW31Y/eZ1+SMWAuKK+T7mME6US6ol5L6Gv2wZiFGqnmc4ctYad3cTizLNNilmj2fiMjSXtr6otC6tYrj1bH1m1kQpKjoBPDCsPSmGysNsYZJppKkMzUcV/6ov9T6mCyK0s1EYzaxRNk1Eu4ypwlZSgpJFDb7RgtIx58mVCUDT7GWt12WPKkwfv+xXne55Vq3AThd0G/JDRKmNyuXBUkcAwAvGI5OKNByQMqGyoG8eFSeLOOvLd9egVT4LOoxVGymMqOlNVQl1rareDqiucdiy3G+6vV3RZMZlU7CwOqCcV0/mESeWKH5+iLB0iDCFSZ8NWBrCCFkN3smH5+DHt0FNVFpVSCYRBFS+9lAhbLYmmVnzz7cS3vpP47KfL95/9KWYQxCY6e57DHOnbFmHgvgRWdUWwFTMTWXZwMsBOE9irGmwz45Z2rMTTpJaq75nZNU+tTnC0xKpipR2xFfZ8wmVFcqWoo3EjdtZjtMEkS9oGgi77smyEGIQ4lPGAtkW3kfsyUla6eI1zLTitqOumrHO2kc6AqTRJZYwVGlOCknzUTKuG+VQxbTTRe0iCyhofLRARpzndREwfmVaZSRdIvSEoRYqlUCSUDqXSGqcVyUO/CvRtASdFFMGUHWUhJJa9aG8UdYIqlkLHS2DIGaKjdgZhy1nnud1q3j49zzonRI4gtWhTIop9ckQ2qDggNrGZnaCM8OFrE/7+1PB//PyStLvDRLXg4b/4qOb9F3vCpOZff63lS/ctyjkigapOWB3o2p5l6lm2GWcy113FZ656fuZFeOGKxVQ94MmDRjtdAAlZoIvFUjmqsYMFjQYHqlbYqFGhRDnzxKURys+CIn0o7oKkyQIx65JIGVKhEI6aAskKU4NNEa0GKhIdgVt3PMebBc++MkeSp10rtpsNg/ZcfXlC4yIhC5u2Yn0Ume9l5JIjPIbV3YHubEOOhsW1A9rTFTubmmf2hZ39SDAz7h72nK4zD+5lLtRzLl5v+Or3Tvj5v/IMh5sZr333Mb/5h494/4sHPHWt5d03jjh/3VLtB14+d56Xrhr+9e/e4ic/2qB2Bq69fJW/9zcCt+5nblyEL70qmNCxl7Ys1/vcf7jh2ac03/jmmkvNhNtHwl/7xEVmacM7y6e4f3JIPBByW/H5O0e8cG7G3/i55/n26/fZtlv+6FstTStc2odzn7nIiwvPM8/O+N2vR/7se2s+96kZn73kON7C4bbn5bOK608JgYG//PPPcf3pzNvvLHnmqXMQMo/uPaaXKZ9/teXVN045XwufuVJzfSfR6IFHUXBnkXpINM6SbSbKgKjExFpqW+ilSVJJsx05AUko1l1dMkOygqzCexhsGYuETCqrBHmif3niX5GiNctFe2BUmfaiE05KdLdPBt8mupgxTYLFltnFwifxm0S3yaiVwhxBGhLGO5LWeBtQKLS2+O0GmzIyOUWlmiolZqki+jy2hhExloTwOHvi4Ms90bfoYcsVq9i3o0MilfWFjOJDNZorCyMmE1IojIPR1xBCRnVCSoZw4jEZpqrwbH64xUA9isd8RNlUqu+JRqcEXrHOMFiYV5bZ1KBJzBaBxcyTNolqvBgrLdQ2UbtAPclUCba9Zm6fgCcSVglONA7BONCmWP6sDqWbN0VgZtI4RjJQO6FSUNcZW5eO31bQTDXWlR+DE4XJGUuJJs4mj2lzJe3OVCWgplzjuZD6JI97zFHpahXVVNFshK0XrNIYnYuCM+ZClNMalEEkIbp0l/hIzAqMcGkSeWmmWbXwOEQGLB5DF/34ECRC6mAzYIwh5I7FbM6umRRhoJpgXEZUQMlmpN0lUBqNK1+nCUQ1lLom9GgVyVno2szpyYa+70GV1YHKukxmUy6WH0aBlglYp5jszvjC7chP/khJ1ouTROUzQV3le6sFZ95TyQYrEMWxzApbOWau50QGptqwmDrms4baNWyk4jBE+kGjWXCgLnNa3eKK6ujyHq9vN1yQQy7ME8knNILVidgUYV/qiic/5ETXp9HJUWwA4steOmfINhFNySwolLJCPvQ5FpYE5ec1DAGthZ2dCt0UbHAeEttlKSAWc8O0zvghs94WHYCYQDWtMANsgeUyErsSlLTvC5MBVXgOwWe0EWwlWBFyD90msV2GYmNUkC0MpmQijGnCGMq6Q5mETzDESDBD2UXoilQN+FTxaD3lneWMYGfUKeG9p9YdwUMUjxo0QwU6CP2jjFIWnS0Pkuf6BcV/8tGK//atFacE/pNnrvDKB9dwFjFNx1/7nOIjr/X8P/6k5XZeYENg23XQCS8dBD573vGJ647nrjiYeKJK6Chkr+nJVNmSel9EYgPoPIZoqYxIGMWTBeSi1BMcMVSZUkTlRB5AkoWkSV2iXUe6bSZ6hY8DOamxGCgCwzwmJCo0OVnaIbBtwc4s072KFDV3v3eKCgMay7XLu6wRNqtESEJcxhLLrBJdMOggLNIEPxM6VXFuUsJiVnmfZmeLP9xw935i94pwYaL53Od2+eZbLSennnRf8/z79/j2d+8jGD774au8vb3Lh1+e8YFLEw5cBTv7LLcr2nXN6uyQ42Xm4nyPudPcqBz/9187Y7vSfPwZQ57U3PQ9/80/7/hff85wfmfK118d0NPM4VL48HMzvvDqIQ/PAmZ1wt61fX7uouGT75vwwncjd49bHhze49XXj1kPjuVSMV9oNoPl55+rONp2nDeKH7lW8WOv7PIPfvUObx/C+w4cP/mxCdVPHHD7/pI//sbA7muH/Ivf2XB96tirAvfPAjuLKXf7DXqo+Nn317hpx1kXeNQ7lvc1O9OOc3Wm7jI7s55ZXVHZiiiZdcxsiVS1ph4t7GU6K2ijeK/BHcWDShmQMaBn5BKUQvr7kd2iRiYAT/IwFFprggp0OTL0ieUQ8V2i9QmvNSjF/TZxfj5huow47dnfnzCZReyFTLoumGPPcDcQjjUqOHIckKgIovHGINJgqynUQwkwSn1xEKlcVoI6czZEdA/eeJbdCdXQ8+KswRpPQpXuP+YyY1Yao2U8m8dhWhByLM1mmegK0kK/jRRvpSAxFl3DD7MYUBoSuqR6VQpdKbRKGB3ZnTusymzriDhF7RIhZCZTYX9hivULoa4VxilMlRCJJJOZThSTiSvM6zgUdCuK2licAdEBJYHKWLSkYrEyCas1ljLOzxqcZIwCZ8AZAZMxKuOcYF0mUfa1LoMjYZ0eR7EJ0blkEhjGS5VxOjAuoCmK0EyxEkqVsC6/Z5t2hoLU7BJDKPtKcrFpaVFY1YATMoX3TpPZ38nc6ITtWSpjUKUIAkkJGYWJY95A9Lg0MKx6lhgsDtmZ0UxnOFsz1TMqM0GrCk0JjGEMwyn4YU1UBpMKvep0ueX0bEtHKujXoNG2WFYKia7sn7REYmnhqM7Dnx43fPfRHu+7eopuDZ06xx8enuPNpWcm0ClANLvVDNFF0HfsWrIZqJxmMZ9QVxojngpFzp6oI6auET3h/s4n2FCxXq+5ObxLdo7TaVMgOALn2LLnWyRHehnjfoOQRZPHQif7SOrKmFGMGhMGMzkacpJCqBsCIWaMOPC2jOD6MqpsXBHghCS0QbFOGj0G9qxbODuJrM48Ta1Y7Bc9xtAnep/BgJ5mVKMQp4ixHGR+KHtvPVoJUyiX2WaTGbyQbHmmokpEVTCKSikshdFfx4TPQs84tZCMqoSsSn7AbT9w1z/FWTiP5RHZdwSEKmZcVHQ54BHUGlQVCygq1KS44Vzccrqp+YkriqNjw7+5rfncRzzpUUJcQ9cPmA6evjHwn9qKL7++wcwUFzE8v5uYL2rqKoKDpDqS96gOJJYshjoKKfVjOBNYWxVLlqSyA9YgQUrlTvGS51TAWMkXO6EMlughh0ROPSlHBl/sUjE+4TcIKWlSKOFiMeQyIYhjJLRR7B0YZvsN+yahk+bwVqTLDedvOCbTTDhK9EcDUmf2LgvBw63Djq53nD5cc/fNnnlTodIZZ02xS8tQgstOJcK04WK9y8HTie/e93z+a2d88MMX0HnDy68c8Bu/+Zir1+ec+hPO15pvf+cRN/bnzHZ3eWd7xI0X9rn1RoMKlnmzJBvLdGr4F7/2dW5cFp7/kCb1mhuN46vfPeV//9eu8Qd/fpeHZ0t+4iMzXn2758cuT3m2joRZoppZ7r8VeObCml/5WsuHn3+FX/myB3XMz3/m4/y++iIP11twO1y7kvmpj0+J2vNPf2PDf7Xc8NKe4fHxMbs7sNdYLjxlOB4CD24/5mCq+fgrls4bLu/WXHlKYURj3IR5YxkeerZtx7225uxmTeo7nrkU2T1wzOcZqTUbMqlXbHNkmgKLxjCrDFoLIQS6FGi1YJzCGcEYUBLRkjHKlBRYiWNtkNG6XPRFjMh7qwKlRtCQPMFcJ7Zdy7ZNhK44FtK4Et5bOKY17OnE3BpOu8j9tWdp4eRkSW0ze5Vl1xrCdcN2MuBfC7hjQctAVIKoCd0wEDuPzASvNJ0UMXLtalytiW0J1+q6wNCDJ7DqVswErlSWIC0qKVQs9kctRagosXyvUUq+R8n/0mTAJ0+SQmHNXQlIGkIkiqDMD3lNYIA+C1ELdmJorAMfyETcvEETONkpfHFjYZOFeqJZzBPbjSagCmt9IgXhmhVaJ9woBJPc0XuIwaKA2kJTlYpQlMWJw2ghm0zWT8QgRRmaTen4DCX50LpyIJfnYsysl4KFRSucMbhaI/XYUetcQjRUek+J+oRTXeTb5enKosZEwIi15X/ruljS8lyZLlgp4Tk+hWI1wSI6ERWQEpIUKddYlzm3K8y9YrmOxKQIo1de8hiL+wTEJuMDEAI6RaQXat8wbSY09S4qCwqD1bpcZqEnR0/Zh9UkVQhVUcF69Yj1qsPODZWuscoR8grRpQiIMRMpK4MnWOBoFHfONvxXv5/4W5+Zc3n3RW62Ux71QuQWxg/4FEBZzu8tmFQO7zPaCtoa1KJhPp0w02Wk5RVUWlNLppkqos4kXXO/FUL0zOwep/oGvxcXeD2wDS1Pp7v8ZHWffToMltxnejRiysfM2RBSwkdf9mTZINGUCyEOpKDwsQh/chI2uSdIhyRd0r4MBJ0ZfKZfR3KfcSkScuYslxyEbumxxiFWs2wTfZ/JfZlCTCeKemKYOM02JUyrmWih86UorHWRwvdtYr3OdH0GC0lLoQlqxtfL4rQa69GEN4mhL8+eqiNiFUZBFMsqTDha7XB/JXQcE3LAN5GQV8RjS280aihe/14JXQtzLSxcQCnhNFYIHcdi+dGrFdd3K+x8hZw4wryljonsDGq14Omrlqf3TmBlShF9akkBht2IDYq0DYCFmMlDiRXOQFKaro0sT1v2dyq0jqASwZdESe0rJGdyDiXpETBKl24mJTo9YF2FGcWi2ljqrJC1p2sDJqr3cMQpyXhIFnU3QZAhk6Nm/Qge3lmhGsE2lsWOYiaR1eOObq2Z79dUc83qKBB8Q7dcU28MZpNpasP0OcNCN+wu4GvfOyHamo9//DJnj4/58uunDL3ni28+wtwZaJoJf/VzT/Gdt49o6syjtw756AsXuPyc40+/esj5hWU5FVpd0Z7d5sEtw87+M7x270sczKdcuzjl/O6Ef/Bv32QqO/zSh2d86fCUzZHmYN6j55rN8oRPvO8c37p3xL6bcW6352t3O95cbiBNuXnY8+zBDr4PrEn8l79zh1cOMs1kyv/i//aH/OIrc/7uz51ndu483zl7Fz3fZd60/L2/eZ2bvue//+f3+Ku/8BLnD044uyN89/CU//HXBz70tOOl64ZbD3oenK64sPB86+aE1GXO2p637ni2a8N+pbl4vkcZw+VzYK2hC4LdlPfC/lxRTQsdMMfASd+zHnqmtaFuDMZoPJkQMnHc8mubR5JqRFPySOSJ4E+VJe3YupWArByJw1A651xsOFobkgaZKiY7Bqc0jWSsAqs8FYaE4p3TjpMhkJ0iqcR2yJx1mqPTwFRFLk5hslORntG0nceeaEwniBkD79QZtduU1Xcy+KCJyaE1iAkoEdrBs+1DcZz5wNRqKgF6xqCwUghoGDNbCoupvDUyKeZxEgbDELAixSLnBT8MqEbjdhVm/oPf8T/Qr3Ewi7GZqi42vTimrVEJeZlopgmZlzdnGyJWhKbWVCZgJGHqgKspaW5J4W1+76MPQ0ZhS+wvmWkjTBpVwD0ikARjc+mIjKA1EMrlnW0uo6FUxkNK5YJ1LTh76rp47zOFPKhrVUJ3VCwj2pFkV1Soo+hvLAGAkW0x/k5ATMZVZeTT9xmSwiZBVRnjKF7ZCH4QQpeRSaRSQjZlX5oSxNxz0GiemcKyUyxDwmhbOtnwROIiGBG6UMh+zhpczoTYMqwfsaSn9yfs7+4TK0jB4qNh8AXNiwah2O98tmx6AWbM633qWgi5Z90tkRIgUOxgMPL0y7hMC2idyDnw6qMNb/1Wz/M39njpas3laUOVJ7ROM88LztqeNx+fIhYmakqfhPl0yuVzFQtXU43FZEfCaY2lIvkaHQ6I6pihP2Pjt2gMbRaOuo4hRbYnxxj7mHB+RdaRLhWLXTZSopiTIURhiFK6YFX263SR6BMhCDElfI5kT9lFk6isRjUVfszQQBKhS/jg0TpRTyBGTT+UfMfZXkNtNJHI1heVk23KLi8kjTWW3gur5UCqBN0oYhYqW6hgfR/ot5lhkLLHM6lEQpuSrmeNLnyClBnTHzhWCWs1dR2oZgFnACx3tplvrz3f2vac9lMCkS5qhqww/YJKJ9ZhTXu0S9MIk1zGsD9y0fAiAXwiJI9TDr+JiI18cHaMfTwF3TGkCVZA+kioe9gcY5gTKiGeanwIoMA9Lqs9I6GM8YNmGwaCT0SfmBzMWPcRsXNOznqmTRGN5QxGV0jVARpJBS9M1sU7HgvDwYouK7YYSaEIN6MX/LZQJdtUNBw55vdoaznJOI0oxTdDWdPsna842rQoIrWZ4TzENuJyZHPzhHsPAyo6KuWxk8Tu1ZoYNNvDnhC33H1sePNuZm+2x4efEY79EX/y+pKrl3dZVAEzN5y75Dg6ifzB1+/w8gvnefYZ+OKfnfGTH58QH614/kLDpZeu8kdfu8e3H5/nO2+e8vbtLfvX3+AXPvmTfPm1lodH91iqIx4eG27M4N028Dt/OPBXPl7x1p2O/WrCRie+8Z0NLz5zif2nha/+duY/+MycV+9u+fFnDX/9IxVvHHl+44s9L13b4/TRlt1rM/7BH69432V4+bnz/OM/XDH4b/OTH7zMP/qtN/ij7wV29mB34Xjl+oJuveYrd1d87Jl9fvzajGcvbznpGv7xbxzx/I7w0587YNX1zDeZ7z3umVbwyz87IboZb51t2d7L3Lq/oeoNJ5tM3O2ZVBWHJ4luO7BXWaa1oZlobK3BKbIR+ijEPiNVGmPWNXnI9H5cG2gNqlzwTyBDOQfieG6qJ1Haqei9jP0+00BrUGLoU0RJz+WpYm+mmWpNRHN/7bnXBZRLmEkkUKK6J0lh84A0ib7R3I5wcBiZHyfyxhP9hKQ1Q+pRwRFVhdIOpw3WOqzeIGlAJDMoEBUwKRL8gI8ePFytLZOYkK4AykQKU0eNt0GmNIoIpODLCi0pgi8FEKZi6BMhDOhpxeyKZnJZYN7/cIsBrxMpCfOpMKnLrpkMxlmEQB8iZgJ2DnGTUSojRjObKmaTMU54qjFOwBQik82FppY8EIq/2WvGsApomgIJAhlxixRVuC3jnRxGJKlVQEQHhSaVcZKm7JlsZjIz1K5YscQIxpZXVnLZ8RbalbwnPEHeO1feqwYyBZwz8jAwtqCFU474Xggpo30sljdrygNIpIuJfgPGCEkMwY7Z6xhsVpyfRuYbYb3y5PSEiFH0CjlDzNBkzZBhGHdigUzuW7p+iwicPLbs7lzkwrkb7O0dYIxFawsqkWLA2h02m0QYIhcuXuRgz3G2PmS5WWN1RClHHzqyhDJuo1i7NYLWQlQe63aRvKUdIm++9g3a04e8felZzueKWCmUWA6XR7z7+A6+79if7lBZx85Ow/nqClVJ/cEog8vD6J115OjQ5oQkA1NRoBVdGqiyYAmklAuxMTWs2oGFXZLTgJFEFT3aCwMG0w1UQ8DEok2RXC68glxWOEnUKtFXltbUbCrBN5qhNmStmXaZ+bbFhYiR8bmuMp0PWCXU2RLTE5igeq/DVSqjMNBpuj4xDMWyGBVECajxNZQsDG2m6wqECJvBlEtKm6KL0TkjocChgirujIVkmoUwnYOrDN4Lt5eaVx9nvnbkOe0UOXeEONBFwHiSUpz0Lc0w48XLWx48Nrx5rHnhUs1PPL+msh34GkVgExpS2uBMwyNRbI4jF+uKqgswUzAkTBQk75JjhzBlkC2VS3gyWQnduhxXfUyowRQXhxJkZsh4UD2LvQmnjzdoO8OmXOydfWLTTgqDIWR8jEBCjaIplRlHu7EciFkVh1ko+GqTTREWjrHGTwqA0jGV2UqIZcFa1RX1nnDj0gzlDdvHA+ulZ2IMMSg6LSwuOJwIqR3o+sSDu8LZpsNGjZMZ52/Ykl9xtOFwPaNbTrk88SzmircfDaTTSHxwyrya8cKlBmM6fvX3e37u43M2beZffqHnL/7UZX71977DPDfcWNwkPLWHWPjiF5f86Ef3Oe7/jBo4v/McP/2ht3n/cw1tV/GXfnyDDsLFC8LyOPCnXz/jg1dqvvGtY774p4mf+/guj1LkaDvw6psNXe/5V39+zNPna77waMnnJvu8fbRi3xiqSvNbX7vNTl2S9B5tN/zIsw175xPxuOMv/83r3Dk+5uHxit/8Ss8f/9ldXnjWsJhXfPSFgb/z83NiFP71v93SNYmPPwWx9fTU3FkbHj484u4qctFYXnl6gfQdVdBcXzjO7QwlpyIZsilOsJAE0wquK41WspnkSs5LkoDoMiE22pVxcCzTAlQc7XVPdE55XAuU/APrComUPE4WtIIcySkws4rrOw1XdoSgAoebyMOzwDaBtRpjHCkn5roUr961MNGkTmMeCxceaiaHIF2mC7DOAyEHxEVM3bBpIc7KM5hNLoW/RCyRnCMDgZ0keN/hxePE8LTKVN4TSGN4UymK42ipHMfGRS/gC0VzSJnYCzqX/+aJSJ2ZPGeZXNSovVCEkj/MYiBYUDEzm1lqC6H1CIa6cuTUY6uaWQVmlmiHiFItaMFWlmZSstmrRoNWZGNIeHRW6Kzp/AhqKWwSnAHryiVgDTAqRI0BnEJc+aOYigBQW0FQGFUOXT0KTmQczaISyiaUQFaZqMIT2ypZFcCNSOks8kimUGPSVcGzjX26FOwjo9VFqTFq10dCCEgA8basC+qSZGWrROgtfQnhLBWfLqJLiYZJndmfOs7awDZFCtlQypgsjF+kiwWCEUCSwihXHA4qk0bx4uOjR5yulswfz5lOp0xnuyxm57FmwaZ39K2nXT3A6FN8fswqnKKNY396wJACp6sjUo6IJMyoQi+sbVBR0dOjKW6PVtd858F97NExu27O7qKido4HJw9pty02Cqf+lIQnpgXEF8DWkAdUBuUtfUgoJUxGmIzOe2TTMZk2NDGToqfzPVVOpInmLO7xHW2wBi6ljE+au1KxyRWuW+HygCvAX4ak8GSwCV2DbWp8s0eYHHDmdlknxbY9ZL28iXp4xnQAhyIzFJLVRIhaFXtPL+SYSKkfI3GrstNG06WIMVAZg66EHDtqG9ipJyxEIbFDjQIgxt02KSM6oh3gSpFqNGiVUVLijwOgrUEmlgszQeZDeW43cPc08of3E3/+aMYqnCfaLT4VT/RAix2EODjMcsXf/4/hmWcdNij+v//miK6d8vELmdhrhjhQacO2XWNEiJstzSTit5kjZhzseXLIBCIojZWOvoeKCHGA+R72wZJ8IbMaMouFMHW+xB8PQvaZ2UKxetwxsa4ImcSw3YRCGYxllKlzKuJcA1VlQSCGhMSEymV1JdoCgh8KwhhVpmcxvpdiU/4sPVkFjgLMrIqVuFGYKo8Okcybb644WyWuXpywd75gqRks+zPB1olmd8FmDV/602OOH8DeXNh0PepBptsO3D+OYNY89dQUNdH8yVeOub8ZeOqpho88uwPAg63n+kXNp1+s+c3vbNmeLPmLn73OannCz77/HI9D5nAZubrQXL98wD+9d48vffk3OX3U8qlPfJzTw/t0Xcfnv7zm1dseFyPXDzRfeyOQCPzsh8/zW18+5Ln9KZ9+paKrhHrjmVaeu8cbTgKIN1yvDR/b2+Otx/f54hvwv/zRPe483PCN+5Gfff8VBhX59qsnfHOI/MJnz/PufsP/9R++xdMXGq6+JPznf2nB0uzz777+Lmfvtkg75fU318gcqv3M8zPh0qXzXHvOYTHcfNDz0WcrLhxlptMK/Bl7u3Ncl2iPBzadA5eophWVKnwXVCArD5WlciNUJyR8OyZeqiL2TjaQXX5P+G2SedKmlQZAZxhXuW3XATCZ1oiUMCtF4QgordhvMo0T7iwLG2IdPXWlmNflfokyYKTYeAcSRI29Kew+0hysFXZIdMaytaFMrYZE5xXSW2S9RYXMZH6FabPAKsc2G1LUxcaIYHw5a7uwxWnHRGoWtPjkqckUeTOEnEuxPE6nc5YS9pYd0XT4DJIdqR9o/QY1FS5fXVBd36AmDtGQ0g92zf/AxUDMFc2so24qTBLa6BEXydPAcCKEScDWUCdDsArNhugMuda4LAy6XPamyQy5RKgaZcldxlDoc8oVIUBFptKWpCxYEAaMEkxtSDYiNiOpeEsRwEYIhqx82RNLQbcqAGWKakByUVcoSjSsKh0j+cmuKX9/PZDLQ1NALyWnXokqcbaSSxKeTiV9MJcsbYIQfCS6gMSSFKecwypHNKXDlSTYUHzXMo7wHYnLbsvtyuC3qmQreE3MGnRAojDoknNvRquNUIR9Phc/thVBMOQusPYPWS+3mHrKZHIday/RDS2bzSHd8JgkLYFMjIpKwTr2+LhFXIkQllTY/kophhiI+GInTR1Zj1z7BFZZ6ANn/Rmn3QhnyoKzNYzWGLzm8eOer7z5Di889xznd3eQlAiho9IVVtlS/dONeF5LEMsAZD0l4xn8MY0zKFPzmtllo26wFzoO+47vnlnudzXT7Jm5nh2JWFeRq5qmnrCzmDGbLZhUUxAhxg1peRt19Daz43tcPFkifekubJVJjYExYyDFjDYBV0e6NiNaMbGavoeQNcEPOB2LziIFnNHUVjFxE0wul75HM5kUOyshM/iOoAzKCNYmTK2oZSxKgy5aGJeYNkI9z5jal1uth9Ne+LP7md+5Y3m7bciqImfP4DPXm8Cz5xILMnuzRLc65dN/dZ8XXlwyPAxkpfmlvzElvBYZlg0nastmSOhgScmSdSTbTPYV2mT6bgvtDBYdtm9Iw5aYa6oUSbkjK4fLhS7XZAVRWJ4IxlhyUGR6dvYbhnVCcBitsDbgTMRqIfYOUkTEjjZchU+Q+vI7m8FQCvKgMnoMn9FZUBayTlSuWNBCTGy6TMwWSRkzwpOMEbTTiM1EEeg90luO1rDOcON9c5650bDZDiSV6VaJuOlohsyjh4VL/6EXG+zLmdsPMt/8dsvF8/s8+2LDzu2HPDrqefOdNfNzFmccf/UnL5DVmj/+1gnzPcNf/qkLXD0345/827c5tzvlpZevcOfWGeevzvjKm2s+/YlrHG/XrLuWm7cin3ne8KMfeppl60jxKb737ld485ZieSrUJnGwq3n9/sAQMp96/5TffvWITz9zgVurE750R7g0HXjr7RXXXp5ya71h4h1XzgnfuZl4/kcyW5nzuQ94vvTGiozwM89U/NF37jGZ7/PyBybcfRXeuLVh6YVpA995t+PLr2ee2e958aWBZ/YrvnPW8c4RPPXShBsXM6/f3nIUDW9/5RBRwmRqufMw8OwVg1Mw9B0X9ya88Y5H5ZbLU8u2D5ANg/TkRlBemFqD066EgOWErjXVbs3UphJo1geGYShOpkFDr0koQiUlZE4/ObuLcDqmhNGquAnytuyBk8abHpM1JhhOBrj3aEufPdNasTer0BNF1J4+ZlJQhBDJgzBbOy6cCrtrkKjolNAmod8m2i4To6EPgeXQE7IaJ4E1rWkYTEPAASOQzNSYtCGj2aZA1VqGqmPXZsRYKrbE3IBf0w1CCKaIkOOIT1YAJdTPKUXMmS5vyX3CRMPe+QZ7UUrzpROZ4T1L4g+tGKimW5rKUTvLug3EXrNwFrqEk4jNIBNF8okuwGw6IdSJMPRM5pohZepK0FXGZkUcu/AkRURijBBtkUMqypveulwEYjmVMapW30dSolFalzGRSohL5FACZ4y2KF3WAYoIMYyFg7xXEAjjhfUeo1rG9UAik97bzZSFQSLzfdqVjKznlEJ5QBGy0sRkSKEICAcRvAU1DThVmOsxZ7zORBJ2yJisCbphf5I4t8kcDyWKNUk5ALMUcZ3K49eby9eTx2HFE5ZBpEdhMOJKtG6YEbbCsn1Izsf0uS3WLkDrcjFKDqTY0W3XRKnKhEWX/VQkjwLEsVvVZY2ilLw3MVG5TE2MaAbpxj1dYXRnSr64MuXjvXXnFkfLM/Z3FuwvFlzY3+fSxXMYZ0k+odIVhtjT+zXrds3JeknXB7Rq0KYBVbC4sU/czpoUp2x6xSZ06NiyNY6HscLZioOdPS7sTlnUDq0NOQ3k4zeZLB8yPbuHPn1A3qxQETQOZQ22Vmht8BF6X4oDbQQ9goKkMsUiKkPxx+uEcUXk1CXohogympQs6x5SnzF5YGehsZMJOrclNjpatAgTZaiyRYcOqTXiBNNEVF0OQnEGRBFyZr2OfOUk8Lv3Mu+c7JLUHGM8QdZoE/n0geKl/SUHdspMInvKoy5Zrj6zpr+fsTNNOFGkw4HqnEa0J91PpK4mylDsR1J2rsXlWBLdYt8X5kQeEAxpk8BC6MBqQyAQcyEnOtew2bRlgCcZV1VkYLtdM60n+OgRaSBb2jYTel9WdBEwBi0JJ0Xoaww4a7HWjN7qQMoZ7YXgawbp0cbQdobQB1TS1M6T8RitS24EhihFhJX7ccoVDd/+7pbXbm64/MyM7ixx9I7n4aMNDx9GXvnkHheuNDx6o0OZxO6eQvc1X7+55eHxwHzheNSe8ea3j1mkmpA1jWvZNRUrs+HNu0cctYbOGJ7aM/zPv3WHvQsLfuwj17h//4Tvvb1i2iju3d3y1bfOYBJ4uAw4cUhVc/O0ZffBDr3c5l/9my9zJpo77wy87znD9QtzjpZwbhb42Y+c47vv3mfSCi5vcNnRbnpubgJ9mmEeD1S9ZW+n4s3NwP5C86uvPeDBI/hb79uhA/ppxWuHS2Z2ggqnfPGtOR+7mmhSptnNPP/CHv0y8M3jnm/8qeLSZcv5c4G/8inLg2PBoJk08B99+gJvryLfeWvDl77juaAHXr6sqHXEk3jzJhzeTew6zf6eYiKBncmM6TxSmYreJ3wQOlGIVTg1xpb7nsBACApjwUwVbj5BJUUcAtEPqH7A9ppYehMS5XzSRqF0mRgElYiDRWNIGVSweLH0Q0KpiJ5kdnYqzMLRSyIOHrsRpuuKWQvNNjNpFdYrQoYuU4rWMcul9bDuoU+ZdZ84bYtzx2pFTCvCwUDdCI3TrHWmDwGrDVIbYpuIKrP1Azr2mGmFeRKlLp5tW3G2HtgOHu+LYFCrcgdqI1jn8UqNYWUlwMulQKCEyGk0Q9cXZDv1DzeoSC5PCEx425/STA37lxxmliBY+pNIlrI+iKuOISfsRDjYsyUz+iSy3m6YzWqauTCkTNcluk0kxJKShs5U1iI2o3JAdMI6cDUosWgB6yzZxDLKjaNIyEA2kRQiZeCYiyUuC84YJlOo7PcFGE/ufWQUmvx7wqORTjFe/uXClzRSrUoqzfc/TlbEpPAx0AcAjTJCIhakrIfcerqUSU0s/PxoUNlgs2FQmUEnQgxMHBy4QK3BR4MfV2M6a1Icvejj161R4yE5vnFyRqQgZfXICIhJFb596ol5A2IKZEkgUdS1oorvluzIBJSYshHJZdzqcxrBHTJG5mayfN91+eRFzEIR54wPZJJMocXBk3xxp4TT9THb7TGHjx3Hx/sMw7Oc37tAZStaf8b9R+9y+/EtTjYb1n1LjIlZvc9ieonZ9DKNneBNxqsOSCRjmOp9jNME1bFQHefdikvqlIvbjtnpGXa9wqxamnXLJAQmKGIwdHGKspaqEpQt4swYYLPtS2S01aSxElem7PiC12QSKZeu1FUKZTKhLcLV1MPQ9aSQcMphqlER3WtyEHRfcKHZDJgmoac1bl5j5wY1NzATsotkSSWxr+9RIcHEcslrXtpv8Lni3nrLcQcX5xWfuhB5xp2xHw17bkNTNXS9YW+nQ+KUymWyGLQKWFXTs6JaTNmLjmGTaXUp2rRA0uAQ6soQQyAMAZsqYEBFhxqEONHk1UBVOzZ+DeJwVrE+2+Jcw9BvqCvNtKnoug2z6YQU8nuj/CyMSGZXqHKVEPBFYJHLdElRhF9hGAr3YqgxKePVAPOeSeMY6GmuZKQHsxXiMGGzbEk+EQfBp248IDW1S3gVeedmz52jnpc+sMcLH5gzxIHX31ny9e+uOF9PuPPqitvfTcx3Gqo68dWvH3PrZmDH1ly6YtAHiuvnF7z29gkPT9dU24Zrl/c4PwdtI68fBzYxc+mCZlrN+IkP7PLO+pSv3HzAtemUp89F5o3lD7/V8nM/sc+ffSdwutY8Wg+cbAYOV8IXvvGHfPCFmuhmSIAQV7zzduDxfc/N445r5ycEOeK1tzSffHbCW/0MM9lyeQbfu+t59kpgtdnn4x+e8Qd/dptnLglGOuzS8Xc/veDv//pjfunFPa5dmPEH3zjjxz+cmFU7/Mmvn/D0+yqe+djTfPXuTf7t7/Zcnzc894E5Dy8ccbzqGNZTbp8EHt5bswXu3kncuNoxqTwvXMzc2DesO3iwjcwaYXe/4rln4eHtnmuXK2qKa2DYDhiE+RxsnZnOLNZBjAM+Z6xYcqrJPiM6FEpnBHERZSMyARsdqaHguKMUC6oXCAVQhR/R1qJJkvA2FtR2EvZE4yowM3DOoT3Yu5p6LdjWoJPFRY+N4LNii7DWlCCgkMlDYugzyyFyuA0criMrnznzibMu4ZNCE6l8xc46o86BMYWrEXMRtqPVeG4qYg6oEKio0D4zDKUZPToTjk47tn0mFj/cmGdSigEVExYhZ016EqNOpt9GuuOebIs4WWlHwv9Ad/wPXAxc3O3YqTSNNSQxtG3meDmwHjp6o8mhZaevmWaLqjKLWqg15ErT1D2TaFjMXSkGwshOpjDejSr7f2cpzoSoEB2xDuwTIiCFFmUdiFXEIROCL5MCJ+SsUGKKnSTmcdwtTCYWbYoPvagGx9tMjbd6LpaN8u+FXPYkGlJGVer3f41FRRq1A5RwIpMVMUQYPEFBZyqMzTQ5YQL0rcEbXWwiqVDwgmSSikiKaKO5NNVc3mYebIrYPaiADRoTIkFAxtG9lQI6Egp0yeTiq1aUNLuQhhKny2hTNGPkpjLkDCHLe/G0SRJRIk7sCB4qtsKYR3iHvBfRAFqXYdOYOV8O9wRZo33BdAoFAJJIhcdgKCPyNO7ARLMeApuH9zncnDKfNlTOMrQd7WZF3/YoXdPYmiF7+rblqL/Habtmb3IVM7+Iq89RaYdTPYv4mHk85GI65LIfOLfpmfcD9RBIPjJsIsNWMySLVJpUq/L9xUwjkUorkmTaBH7wQKJpBHFC18PQg8pCjMIwQBxDgWyjUVaolMapEgUciLgGqtoUnUUqCYRBrVBtoE6aSR1pZobJRKGrUkQhoYw4k4xQKosoWyypTtidCDv78Mp1uPlwy2++lXgY4LkduGwzO7phYSNTySi/RYtjsagheVLWEBxiN6Qhg61JKTI7b7j/eIlM5lhdRJBWaxptmFSaPP69HASxhnAC2oQxpjmjTCGEqmxJsXRkKQacrVksHN63VLVFacXZsqOZVIjEAhkyhiFEht4Xn7SORS+kdYGJWcHa8jlKQM2AIpN2hWQ1J4dr7MSibUsVFPSWHCJhI4S+vKljFqrGUO8YTk88b3675+REuHT5gIzi9379iLNV4uK1CT/zE9e4cr3m+FHP0WtLbFtsw7tTRb7hePHZOV4Mr33nmN/8/NtYqXhqYUAFTtZb/t1XN9y4MOP5/ZqjvOH1Ry1XLzUkM/D4bs/P/sVrtMuBb76+wgi8u+p4ejNhdbrGu4qnzyus6Xn68pwuRr59D2rVM4TE0+csFy85ohau3Zhw7bywXguP7ICeGZb3WlJ/yklb88xOxbl94ct31/RvPeDqRcMbdxxP7wbmKF69ecZf/9geb21bvv3qCZ983y4uNNR7mf/TL51nYyf89ldv8/Qly4+9YvnW2yvaQ8fHP3CeP/r6IcfVlr/788+wcGfceiT82ddb7p5kTAycLitS76kNnGyFj1za49puZhO2mF3D/tTTkKjnNaISfptYdR111PR06JlhMnWIKW1Yjr64yoIBIioUF1YYnXPFVyhkq1AOXJ2RmEoDFiD7Aq1rxmh6lRMuZRYTx1wUOgrhNEOb0YMQM/QK+pwYoqePZTuXxrO1MLAjISW2IXMWM4/7xO1l4NaZ53GfOe0zZ35EZZPYVZFnzgJ78cn9oUqYGRodhGFcThulmUiFw3K82fCwixgS95eJ4/VA50FEY1SmT0KVS7YCyWKykHNCZ00jiiSK/lg4eycyyYr5QUM988gk/nCLAXsyYdnD2+vMVvdsYy4AiEkiaKizcKJKvKXzAxd3p2ACaIMzFc3EUzcKoz1hKLsgoexXUIJ1CWPiGFoClbXMG4uoSMyJbDJKJ9CFQf0ktjqn0pGW4Iry7YgkJAkhFSpTlvdafv699r+M28eOllgU0e99XCD/+/9PaWvG36dCtVIZUwFVIsRMHoTeC4db4VGbmRnLroO5Kx1mtAmMoLPB+YJ2XqmEJbNoFBcmmaM2MiTAKJxopKqopeB2n4hjchYCmaAgIOQUIUNM5eJCFQWsyiBx7OzziLdMCkXBKGdVdjU6qfG/FwzxCK8qTg8RTCqvr1IjTGds9bIogkqgQhE2IijlMGJGQALF1kgJA4Iea8rHGdqOo7Yvn8hFtHIwcUVhT2IWFBObmeieOh+zk1v2/B0WMmGfCTsx0uTH7KZjziWNbGHYRPpVT5szndacDorNNhBij2oFYzU6C05gPikHgPgiwMxkTGWxthQMwRdgUGpLCmWMuQRaTWu0Lq/HMCQ6P5TXyGhcbdAq4zvYbDObdaBvBw6mlqt7DjvV1FOLSpG4CsjS4B9DUoFkEmZqcQcGmQrZ+mKZnSQkVpA0Ny4K/9EicXRaUvtm00SjIyZqltspbtGzZzQGTY4epQOSerIRhi7jbIXQk3OgaSpaAtZYjI6IAquBHFFSuvQ4ZNTMMKwG3LlUgE5aQepJXTnYyrsjoJTB+56cy9Qkp0gMARFVKJqhJ8RQIFzWMZtYhIIyFpcQU8iOClMsih5CCGAj6kJFzls2D9OohRHCSUKyYTiNBF/CnLIZqJqKZlLTtp63Xl/z8F2PGgwLaTi9taYNA4OH6cRy8YIid55f+WePefd24Gd+Yp9nXq6oqsxumhKD5WuvnvDWt+8RO4NShrM2IJf2uHBe8+obR7x0acrjbcfDB1t2dy0v79WoPvK9R1teujZne2/JzTPP3sKC0bzia/pN5MaNXZ67ssOX316yeZB5+1FLVWXOTeDkyHDnbMuFCzVna08bMlpr3roTmFQVH/mQsDoe0O3A+5+Z8+brKx4Fx/2Hmueuw+GDKbsXAmEWsCpxHAyfubzD+643/JPfOeHadcc7Dzv+wvt3ONyseea5BSfHntXxlreCRqkFn37ZsXYNC9Pz9/72df4/f/CA//q/u8sH32+4MYdPf7jhW7e2VN0U3UTe95F9jh+0fOnVlgebJVVT8+4DRbcNmPs1H3vaYtMaUwn7lxt6Iro3hYq6zPSbgXpmmcw0VSOgEsPAe1NMozUiJSfAp8IBEVJJ9CyqvIIchNGJ0rObK/ZzjUNQuuIwCbd9y7k0YLVitTNjGwWz6TDbTBoUKpbTPYRE5zPdGIBmgBRh2WUO28jtVeLWWeTuFo574WSApS8rAzIcsaU+WrObdInsLsuqIsLWhpaBHCO1WGbW0A+JNx6vOesGJsDjLrPuICaFllyi1EXhGZuzIaIokeZWFGI0IobhWHHWB5ow4VljME1A/2C1wA9eDMwuGFargTC0gCGsDcsNzA8KpzpkGNTAJhpMMEzWAzeuVqS1Z4iRZq6om4y1ijCUzj2F0Q40kv6UKpdlTgVYY10qh2sUknHFX5kKfEeUoa4tykAgMMQ4dllFpZliot1m+t7Q1IUh/94t/+RXLuMkhAJLyYwVwvgXRx9hyrmAgJ5MCSQTQyDmcWUhiew0UjvqAUIrvPrYsYw1F+vIXt2xV8H5iWJvAklH2pQIWaiSI4aIUYmFy1QOUlRESgZDVnoEaJTOtMTwxvLvjJoLykhMSeEwiMqj/lEhUmMpIreUBmzuxoQuA7mCrBlCzzjdH389cU+Uj20RwhPftozgD1U0EBnBKVuKMYq2IBFJMq4wxpQRrQSSxVIU9LhQ7J8qs4iKSeXYaxQXp8I5q9hHc84YZjqhpWPae9zQotJRiR5Fl0yHzuJ78F7oO8PxOrP0kWgVPmdCEoLSBCnhNpVKTAXoE92g3vuZZhFMBXUSksps2wL0iCkQ4oCrNYvdimpqaLuB5VmLD+XNaLSmmRhUVvTbnhwMkhOVS4iaUk0Lg8OK4Jd96R58ph86ciz+55gDYgN63mN3pPwzV6S5oKuIssV2OjOZ2X4sz6lyoFeQM2oOm2FGqyIiG3RukOSLQ0IZtLWkjUfPLD4MuNqCz/gUipVSG4xAyAEjBkFDn/BJyAG0saQukNF0XY8EgzV5FDYZfPLEmNhsBnZ3NKKEvh1KHkQqECKnJyV6OEWsS5A9MdSkDjAJq4r1i+TLGmum0BcrQuoYTgLZOHZ2NZvHZ2gzp48J2dXUJjIRYeih23bcu9Xx+E6i3QQmU42ZC6I7FlpIwWKnM/ZfmLL1G4Y28PO/cAEXM0F7jk47ukNP7Rx3Hh2yWkd+4kf2OQwQv7VhbybcuKp4/OiUzSpw52ygcYkXL9fc2wR2DxpsyuzMZ9xr4Ytf7bhxdUIlkS9974Snnp7yx3+4Zu+c5u7RwPfubphF+MVPKGR/ly98oSPrFZcuGeZNi6mmbE7KyuhgHpg2c+4+Vjy8G/jRlwOv3zujtY6LDRzd7rhZB87vNTy473nmqTnfO1pjNpmNBP7FH9/nx95/g+tXIv+Hf3KPVVjzy585zz/93be4cz/zt/7CHps2sXGWi9UMNSi+tun5n//ZXX7hY47rVzO//msbHj1veKYfuDydo6ZL7j0IfP4d4elLmnN7kccDPH63R4KnBu4fbTncmXDtYILxgao31HMD04BKgk4KPdpEh1VARc1kVuGmkT7qInjWlD0/oJItblIJZRWghCjjKnOcTfZklhIxoWMnZlKV+DyZ4+j40fqDtLOXeWw0V0+/zfMnbzLbRIZgIAdCCmxjZNsnuiEVfZooupR5tPE8WAXuryIPtpljr2iTKkJzKVbGGCNdG2m3HjAo06C0I8jAILFc3EqKCF5lVFacbHruPFgx23omSRiSJyZwqgCRplbog2Ao+TVWDJlALFcmbQW1scigyG0kPR7YnHW88KGGS8/8IIqB/z+Kge1pz4OjHm8d4jI7i8yeGHLMqFTjgzAJlmFjOUuKmydbLp2vsWmgnibMfsNsMQKjs2W17IE48gNGcZ/O6ErhvUdZQWuP1eXcS2kUC+qAMhmnEhNlsJUQiKw3xYaRM8Qhk7IQe81yGbETVTr4MfzlSVBqcWvIWGqN0JNU3ASZzHuxmDwpFL4/XwixMI/KHytSViTKOHVnZmmaCe8udzndBma9QVnPZLvhqXrNc9OBczU4Kbx5nwUVAlOj2K0VQ19a80ElUs5sU1ENSJSyLaFQt7Q82XaU+E81Jg4WO4lCRKG1I8XNWGwplLgyZKCgOcOoDzCoUruOsaDjamvE4qrRA15CPrRm/PxCFqHJGZGSTDfEhI9lyzWtHdO64rzJNAoWRlhIYq4yLmYmuoQ72RBYmMDluePyvmPelOiNTE/wARM1PifWYsi9oGN5y8cEHYoQEz4PbFLkyPcc9gmjDLNGmFYW7QxRlemSNQqTgC4xdAlCGTHGWNS4Y6VEDgqtM3WjyZJoZo7ZzNAOA+26Z9t6jK2wRhGCZxiEuqoKF0MFJpUgSnOyHkAalm1kUD0maaQTGAZmzQw1z+jKYkY+e0gJ35bpgj8UzERYVys2M9CNZs9ktFiiMThOCMmSQsbise0xdldhdQUx4CWV7j0qkvixQ1BIMvTDFmMW9KlFj5kdUFY7KSc0JTCoPfaYSkPnaDcBpU1RhI/Pvu9CwW+jqGtbIoSLG5EYNG5SwmGCj+QRnyapTAyqWgOBlEHEkYMuQkuTsfuGuFC0uaUKjpws04PIsFkRB4PdS3TtlvmBxSRLf5o5fFfx6E7BPSsS1cQRpCjUc86EqHALh3eZr335mAf3B5xRTHcHrlyvuHReM2x6xAln/YpcWS7uzBmcYtcIn/qg487dwO996Zhnrsz58Y/s8YXXTtjbs9w+3vKBG7uYFNhbTHh8vKbPmRcPhMyKN488Fy5Yjh4EUqUZti0ng2PXGH7mU3vcvO35zvfOOHnkWWP5qU9U3LuveXi/5dKFKf1qQOuKx90ZoYWnLk+5/ziw6/b45rfXfPDjjhsvZU67irNVT65nfPX2wHkslw/gf/rCms++MuXS5Zo37p/yH39mwa9/+4hvfLdlvdV84sM133i1ZRDP1x50qBT5+POK2sIHL1W8eypcm1mefcFztM18/dvC1d1T5hdnXLyaeW468Myu5ZPPnSMbePPmlnAaOTetmDSZeR2YN1J0MwPIMuAW4ByYKpMljCerKgFW7YAVjTUgOuNTpu9L8a1NyZZRRhD9JMCnTD9zypTmXJNjRFJiVWli3PB8n/muD/yecdhqjxsccmF7yHS9pe0167CC2NFGSxuEbRdZd4k+JQbgbAjcbyMPtom2E9ZB6FKx+mlgojLKCskKE2dxutwjWhmcrgiqL02SlOkbCkJO9FEYtoF7J4mwzLgYqTVoLVQaKgvTUOyMxHIG2xxHZ0EJXZtGT6MHrLZoA3tGc/O1juHM448TT/30D7EYePPBhj7NsUbQbU8OkU32BG8JfUeIiiQRVyV05bm2P6FCCMGhK8tkHqAKZQypXdkRCu9138o4RA+ITqXTmtfMdw3WJIaYyTGUUbk1RX0dBRNLcqHTQu0qfEqEaPB9IlvLOgrLsw3VPLFbGb5/lT8pBMZueOx0i5G8qPXHtXgRevAkClOeXL1lPTFeuCoHXIaAIcRIrQbOu5Y7ZspWNbQqoJkztAes2y0P22Nu7Ky4OhmYyMCAQ3UeiVCJA+PIOeNNyXxX4ooGQI+v1zjNiGkUF5pYpiYxotHUVKVGTh7FhqCGwvDHjBoIKbZJyaNg7omAq4gqi0+7RHna8lfLKzfijgtGMWK1xlUVUzFUJBYOLs40V3c0lyeZC01mtyoXTdh6qgCqHQU+vRD6Hh8i0VXUk8xiCpXN9H5AxGOxSHD0vcd3EbpSSEIgx0TvhbQVttvEcR853WZWUUElzOeG3R2Dm2p0VZII9RhNPXSZ9rQIPa3SqGxIMY6BKIVsFgaPcgojClsZjNK062G8CA2LRYV10AVhCEK/hXboaOpM5YqIbLtuyX3PveOW/inDhz7XYCcDulXoTU047tEbi4SSp5ADOCXUjSbZhJcW31fc2wpn3cDUZDaVoW62GGAIDUF36DRjG3uUGJ7fMzB05InCZFOCWnqDqyBtWpIkJNVY42iTL+FAY54FOZWsj8LARgXI64Scc/THkbP1QF1VbL2lsgMxBHRTBIfKlAAV3wVydgUBHUslnyn6nhgj1lSEqPBDpKocIcQiXI1FP2MrDVNDbHpiDtSDpdsMuLrGDD1tZ4lKIz7Q+Jq01jy8FTi9E9ksPVll6pkmYos4MkGXM23oCb44K0LOrPqeK1ccH/vwZU63PWenG/rTRFPPqSeONvSoxy233jljpgz3Hne8etjz9MLxkacb1m3Pl7+2QZpMauFjl+fcO9ngK81b3zvh3M4UQotMHUNf8dc+vOD19YZvvO755FOJ63v7/P43D+mWHf/mtY7JlRn5eMlLTxsebi3feGPL4WHCZ0Pf91w60Gy3PfOpQA2r3qNDIKrAM88I3zxespMrhuzLpeQ2zDaOQzoO1IyXn868c7dFN0d8/Y9P+Us/tcfPvLzH0xcO2DvXEnVLu9dQ5Rm32jPaw8TJqeHqJcXlA0/INa3OfPKVBQ/vB955tGXdC49e37L/UcfFCxVfvdny7p8PvHKuuMN2FhMwgVpHrK0QydTWkWabsvseHCorQIF9chvlYp3WJdo7RymcC5Ux41pyiOAj6EHQLqNdQAyF30JZj+YEJEUMQu8Hgs7UxvLxvKXnz0jL15luA7PbS7aPYClwqsAPNcEnlj5y1iWWHax8Yhk9R23iqIOzQWFUJokQJCM5omJpHGtjsHWFMUVQHsgoY6mco20TSudiBc8lMdeHRBch+MTaC2etkIeMshoj5fNYiTTGYyjftzXCLGXEFsbJVCumNpd/qsAUg1VbZm7C0WPFt/6046kf4I7/gYuBbmtJKbBdemIqdiMfLVIrUpWoraaZTtgOHdeN8OylGfQ9A55mX+PmqRCi4pN9oC+hQwlcguwSrjJkFWhc4ty+ZtoUiIhFUzKUi72JXJ6fJHGkThU8ZYXCmoQR8DHhJpB7w/oEnFM0szjaqMZbXhIl1YYRuWeRnElDgOBgmsAFlGgYaYVBpBgLkpSRplLkrMpBmEpcpLORc5Oe3W1HLxUGjaiBrAYimcO8w2Y95XhYc6U+Y4eemGqOAqzIZBOKUluEofJILAZCpdQYdhOQBFbrImd4UqCIRRL0pJI1YIrFRg2WbFX5MzKVQAyRIWdUpWlyhdYBxJOyIivBmSJ4UdEQJDCtLDMHs0qYVRX7jeH6NHJ1Ejlwmol1zCfCbpWY2iIKzSERQ6DfJAYN2zbQdZlu48EX0WNOoGuNrUpgSNcXV4nOQhhKRGfXgxoUrANEwStF10e8j/RD5ngzsO4Sy6745Xf3LQf7joXTGJ3QO4ZqolGSGHwkqYxKprgr+symjbQhYxCmvkBxYoTtIGAtlQIrqUx/gqJqoJ4JQ/BYbXAG1m1LFAuDRelMH7eg4GyTCP1AU++hQ0TFAsEKu4I6yOQRVpUB5YV0Zlg/DmxTIDaQm55J0jhqcm4JRA7XJTpc51XZ75uOIRumSlDGkvIT9XAeRVglMS0phfEVKQdytGXHnqTsHCUQlKCCQWcPDrbrTOcN8dSz2ihitrS5JXuN1oagMrEd9TqxZA0oVfCoWglaeYyZ4Fe+aAfEQoyFlpY1LIoALWUhhzKbSsYT5hmlDS5YfOsRY7Eq4TthHSI6Fw3PVgIndwbW7wQqa3ALVaxzaVSWZykiXa2Z2imqKUROnOXATjHO8OY7Z9y7vWW2b2AnI4+32GnPfNeyPu3o+oCbwCp6rswNVy8e8HC1ZXk2cP1ajTOR003m3nJAqUQcEruTmpgH5lPLNx72fOS5hq++fcb37nume5HjM89bb265cqFmiFClwDtfu8MUwzvBszyN7C1qzk8g2C0H1ZR+6Ll4TnHzfmYnZgbds4nCzK35xLMNt+4Ip0OFiy34iu2q5UPPKd66pdjbUbSnwu2QuJY6Zh+o+eYbHT/2oQmh6jg73LBer7ndCp/6wDn+w8/O+OMvbHnYDbx44QL7Tcfv/PmKs3XFL//8jHkFz7w850uvduyFyB9/rUM54cc/7HhqoTnbTji+vcTFniMDg4MD75ke1ETxTGyDmIxKCVKPpdj/ihE/loI0JZRyKB1I2RMGhU8RTKZxlpQV3meGs0iOGVtZpAKvBobscZWhroomzSWDTuVnHzWYFMnbQ3zKPL5syBNh+9Bz8ihyuI4cDWU91vvIJiXOejjphWWv2PapWMMVOF3utOLK1QSTcCRUSOiZI+UNLiayt3hJKKNhUESlSFa9twbQSrFKmiiOkHqyaEJfGDe5dKul/8slUE2kAEx7rdnXmT3X4azm8kwTxEPONNQMEtFuYNP+kNcEZ0MkJ0HpGmqHqgdUblHJ0qQpXgX8wxWX5g3PX9oF7+m6jsmihkq/d/FHD+t2oA3xCRaoXPDWo5xDO8NkkmkqIZsnR9qT3AC+39yPo+yEkKWMAWMqSviYIMTiKrC1JufI6XGHsQ1Gd8gT/nkuwjeR8rkklJIyJlged9BZdi7W4whL0DFhs0DUhJCIuVizZBQlluG1wmjLQSNcmfb0fkuLLuYQFYkqIzTkPONsqNh0GWeWJEl0WWgRfDYkJWgyjSiCEcKIadUiGKvQlCLAx4hJMoYbKcR8X/SoRDBKs7XjyywKq3RZyxrNfjNGMusBjUbLFItGK6glM6syU5c5txAuVMJVmzloYK9OTMwWaxTaWQhbUgQfDEMnbFtVBHhdxHeerhO220C7SQyt0HeJHCjpY1ph+y0hWWbiqKrixS2VQqns8ZHgy7hO8oBOmpzTmFJXLlddaSqjqZzQ1IqmVqgpqEZRTTSmqUoEdx9wFHRoshnfJqIuz1AcesiKxlZYND6GYuFJBjM65MSAVobsFZI0re/pE4jU1NZSG0UOnjBAN3hC1iAWO9fIbCAPoE0kJaHtM7VkvDVEBGlAXfVUIeJvwuaOpTUQ2WIwpFjhpUdVQhwSpq7QFEWxyXHM8Rito1khaSx4M5CkENres/qFEtJiNSlHjDJYUxF9j2iBCN1qQGWD78pzB0XIlXz5XOm99/BoU9VC9Jm+H9CqpKp0fV+cEkqIWTBj8JfKmtwrpJoi0SNOk3woPnFVDsDsA1kU1hpiPxCT4AdP1TQkEsZpbB2hEryUyyFLwpmIrsvhq7Umh0AKiZQ0Xa9YrT2nJ0LOHX0auPUIrvoJT+1bhvnAO2+uqZPh6ssznv/ILu3QUX3vlCvn5tx90HN6t+XylV0qnXj28pyj0zN2tWUTeo42cOFcxa2zDSe94sbMcvfhknceG1wT6NY1h4cwu2j47v0t1EUgvJCKYbBcrITpriC7HVNxrNspRnsUM05XHWFosbOdAuFZJsyu8PnXPDp5JhKx+8Ki9nBW8fuv9vyHH9vhi6+vaU8zf/tnz/Mn31jx4ZcbVumMm3c1tdly+yRyYgw69Pz2lx/xc+/b5z/7377Er/zOW/zGv37IL/2Y43/zMwd88d1H/A+/2zMTxcVzjgM7sNibk6VnfRZYHgpDG1nUHbOJcHEC5w4arA2klHjcDiRxVAZ2NNAYlJ4w+AFMYFZXZIS2L280JaFoV7JgVOGWRDLRa3zM9N6PU8siFtdRY0yFiCXHSN+VZyBLER8aq5ARiBZyIgCh0ehdqK8pds80q1uB9q2Bb70TuXsU8b7sS5ONZJ2JCDFbTAATCxsjKYg+YYMim4hJnrjWhB58HzELh7UV/Zgz4JzD9pHsSxsXYyiT4FQYJykVDdMTA1wJjHuiXSvf7zrl8p5CWPWReZUJA4SFMNkNtKqsBSfRsH/uh0wgdDaXA0gg+lDCQJKlTYlOdeyYxKXpDucaRTX6GquqLt+Z9eMlbvB95mzr6QbGXWYGK4hNYEtU5XyiaFwq4Kgn978uF9x7lYAwikiKcAOECPig6EOk95E2RCaVQueSN69U4txVizFlPyVPluPj+B9dxlKiyuj58aOWi6I4f7GIq4gFBpMp9L8oJRRJchHMpVwmBgrNfh25MVvTtXA/TshjAFPIYEg0dIhsWOU1QxKCLp2qgUJcCxGtEvX4w0+j1kGpEl6kVTnok0hZV2QZhYOZzJiuOG4UKm1Ge0wZZbnaMdOBq3Xm+sJSzwVJ5Q1Xu46JFeZGs1crFg5qqahtRFRJQsyi8dEQY0BCYJ2mtG1gtfJ0m0AaMjJ6clMfCx54yGVM6wXfK/o+EsKACOzvV8yyG0NJivZDIqQoxEGgT8RUCh3JCSUKk8qURrKgdUJbRe0UYjOzmaKaZZo9i61KdHV25RkyoqizAZ0Z3BjHPdUM20zeBmZZM9UVvvW02xK2VWk1xmLDEAOiISuh7wf6LtF3IDnQOJjUms06cHIK620JsjIpc/9+x3Uy0+whV2iVqWtNSAMmB2w00ApsI3lXs/esMD8/8PBOxG8qfEwEl0ipTK+cGe1UCFYSxgRmtSujfpFS8KZE0gGVTLHDWkUeEioJSmeM0hgol78yrI5XzBcNmQgR0jYjVRH7mtFMY0SRbWTaWFZnvhxasYC0SFJSOT3FPZKE2AY0tlA3E/z/WPuzYN3O+7wT+73TGr5hz3ufCWfAPBMkCA4SKcqaW7Iky3bbjjtOx0mq0p3kMjep6qqk73LlpJJ0pypJu5N0u7udcuyyLEuWrFkUJYoCCJIgAGLGmYd99vRNa613+ufi/Q6Vm1S5UrxAgSgU9uH+vrXe9z88z+9RtiCOUzQsz2Kxd9pENYKsE7ou06zQ9WSvqI0jxoDCMPQFONQ0lqFfoG1L1gpqha5hjMVqyDojREIs+iEjFonC2azj5DQi2XBuz7G5a7h75Hjq2REHuxM+/KRD0Dz5xAb1hoHe8MG3HzKqLFe3G27NPFZHvvzilJMuM92qmR/PMJ1GHYx5cD0wVIbrtzq2W8O948B0a8yD447LBw13zyInYcA4gyGziIqpBB7cDZhK8fgVx9a04s0PlxzfVNwNHU9eGJP1wHJZM2kjV89t8u33VjyxFzh33rCrLGlkiXPNZ56Ab3474Q8Sh4ea584bfv2dFS+00DxhqLPj08MFG6OGqd7gjffmbE08v/QTj3HpquONGx3//E/u8Rtvn3Ey6XjxyoRL/3CHP/mjmwQLh6eOazuauqm5cbrgw08Mz12MfOUzI3qvWIWIxbK7OeHhvWPyqkzFJg1s7lXoWlA5sYyZFAwbtaZthNEUcoBu2VNVNeOxQTvwfiD4VPbgThczlyqNUZBM0g5JRTSYJCMqYRsHWhNiJkqx7eZctDilZSvUWauhUYpoIJERExifh5cuOF54reILJ5n3Ph1498PEe5/ArXtFkO60MK49sTJF1AjkoeCCo6KA5CRT94HT4zMWZ0s297ax1q3vDo0xmhgjEqFtKhwOWxUibsrgfUL9f9/Ma0tvWk+0RRnUGuEctKVP4GNGac8owWKp2BlNufBE5sUXN7l0+d8ttvDfPagoZCrR6OhJZLxoYlBs1XD1oGGkKhZHC2zlsKPSIZAVZlSBy0iMEA2LpaePlJFrUkAmm4xTEKRchFXl1lOBsgMqve7azlcQALCulFLx1JFTsYiFAEOAIYNH0RihMaCU5uhwwNQN2+c0ti6BFkqV/59KSQlBkhLfW7mak5M587ygrhp2Nyi+cB0g+LWCsJhetVI/1CFKLmhloyLbbeaigj4I82RR1EzMBJcVaXWCj2cY46mUo8qCJq9HoIUsaKREImspRcujdUhWkLUg2qDWFKpHIoe8/lxCokCAUmaZElUuQI+uj/Sy4NpuzVMbLXtj4UITaG3FqNY0pqKyqgjtdAIJZJ1JWdH7itNUcZwdSxyD1ERpyBKwIVHlFbXuqVVf0gRRKGMwoYy5jAKz/j2VhkGt5ZfaoLQDKcFPEsvLLlGR4hoaqTW2cagUkZDpfGbWRXwCnC3R2WOFmWhcW0Kx2lpT1+5RHVQ+HJVwlWCMo4o1kov+oaszndPEPrP0A7IW6MQo+C4jFECI2GJBjSniU0RLYlRZxpOWjU2FUhkTNatl4PAsUmtF2yiWDxNp1WLakqCIgNElGIqgyi/ZZLJPMLMwcdgqc/6phuU9z+KBMIjFtULfJRQW0QmnYGwd1hkqrYsWJJeJlUhRVht02clXoAYgroORKJs3HKQkzM46pptjyKXDM7kGlTDWESWuCWiK0cjiagozXVuguABykhK2JEDSayBK0SXEBEYURmni2hK1Oo345GkOihvFtppcgYSIEYVtG6QLQEksNNqw0dRAxNryTLVTg9/M2CqV1YCHGASywWCQnFmtMssF9N2I7W3Dzp7Q2MQwCE89PuUsCF//9jGTpuZLrx5gmkg/dFRj6EJifjzw/nd77hwOXNxtOc2aqanZShrTjDA7FQ+PZpzf87z1ccelCxtMty1XjOFuWtK2hvks8HA2sD92xM2OB0dwrrIwhZYRs27g4X145/2B0bRBcubqxRHOdnx4BzZHiU8fRg4eO2PHKCYbm1y/d8aVqxUbc8PHR4nbbWYyEpxq2Rp5lsDTU8vcJO4fKjYmPZ+92FLrwGuf3ePzr1r+7M37/MbrM+xfzHn1+T3+y//lk/wf/9kx0ilm9RlbO9tcuey4EWYssnDj08DPfr7hpy5P+YVXNb/7es+3Pjzh2t4GrclMxw0yX/LMZJPFaIkzlhQ0fpbZ3gU3FnRlSRiG6MmDMNKGyWaNbEb6vmcYFLU0TCajsvP3AyF0WFOhjMUajzWZgEFsORuUsWSVyRRIkbUK/QhFr01RIqRITKmsj7Ii5wQErAYlljwYhlyky+c34NprFb/wJaGLcP/Y8L0fZL7xesf7H2e600wwGmsSkxqSLhNoJ4LzBqmE5WJBGDzGlOlGjBEjFqVtEUg/0utk8N7jYyBJuUhiKsXLI2291qW5y2vNl6ZCJQ8pkpUhBk2ni3bq0kjx6itjvvwLe0zOO+SHTfSPqBhANcxSh7WZxjh2qshTT084v9eyPDnl+HYiu0SzOQaVsMYilUFcRCShgaEX5qvyCxut8bkIRYoitCruv1pQVRHLZSnKfqV+KBNAKFUSjyxhuQzpUypFQIwaHzNDhGUoo5xxragbw6qL3L/dI8qye16jdbEbKlinFua1qFDRNFDZitu3e8YTRfvCmMYK2QeUt+RUFOohlYwEpUuSnUpF+ZqUwRphx3k61VDlhpU4SAMprfCyKNkCuaLKGUuZuoiGZIpwDaOJqkRHa10ujKzWYCBKB6jXnxOoUhCtC4GYyueQemHQChUyOnvE1AyqZfGg52w58P5+zY/vZM5tCBesYUMVmx6pwFt8diwzHHm4FTe5EXaZDS0TBdu10KjIAUtUjugoWB+RIRJ9CaxJSZX0wCzkWIA/CqhM+bs2FmMjIgN+DdgQpbFak8mk9fQjR0GTkJhYLTzHs8jJKpKMoXaOaqQZ7RrcSKgag1EFwtSHCNYW7nkUfCzscmMqnK0gaVIYcLWQosanRMoJ25SRuHhNQBF7yEojuRSx2YCuana2CyglActOSLFY5zaniqYa0a2EjVYxrjXKG2RSg1kSokbHhNKOXDelWkmxiDl9Is88ZtsioWdj21Lbik9uLunOhNZqahEm9QRVx7JKkkjfJTZw5ZCTjLZrZntBT6Lq4haQDtyaPsjazpd9Yntjs7xIStMvYxFrqnUYmCr5DG3rqJpiu4kpUriF6xTRnEvmeioHnSRBlCZbiDEXQeg6Qa4Ich2j3RGuDSQ7YGzJqig9gEL6UqhZXQ5B5wzaQkoBY0wJn3EZZwzZQxgErRzOKVT0dMvAbJZZ+UjWir3HKjYmiuQTPhrEKm4fDdw5G9g+EK5eq3n9g7vEo8RXXt3lcNnzp385Z7udcO3KDtceV2zuNzy4fcbN68LJMOeSrfn0w4ds7W9w90hzYTLh2at7vH/vkCTQPwzU04yNwuUtw4BwuoJruxsse88nN2HklpgRSErMlgnXZEa1IfdzTpVmf6fhzqlnNteMjw0bU4PrPcSyLunmkf29SJ8cewcZfweubFUcDYFoElfqTeQg8DuvH/Kf/PI1fuftB3zj7YdcOZjy4rPn2bp7l8tPP4WrF/yv/7MPuXEMLz5d89bHiaeurtjb3+W8W5LNNqe7c947OuMb74xJsmRUVxyfwSgueXxbERaGjQlUG5GnNhuGXvC+QK1UqjE+0TqwjSBTQaeKlDSzRaSZKMZbDSZr0lDWTVobxpOKGIW+j+SssKbGGWhMmRLEnEjEtYi6vINhLewVJWgraFfWDLUzKKNLHocIZAcZoo+EOKzxxgbfwdFx0S2MNhznDwYeeyzzc18bcXokfHI38t6n8J03eu4+EJQWKl1i35OO+EHQIdN1AyoLxhTce/YeyfmvyK4i+JQYvMfHQuQtcK8CvxNZa6vWrraSWphJyWK1RiXBaYXGMu/K2vi5JxxXnpkw2nCI0az6EyY/ymJgXBsu7I452NLs71WMmgQowsnA6tCyRLCNpdJFDYlN4CLZSLGg9YblKtINj671wgxQRmHXICHnMuOxwlUGVOHZIT+MD6Ko+teCOcXaB74Wb+RMv54+xKTpfGTRZ0wQ9ozFuoirMotV4v5thWssm/trd8AaPoQuB57kjLEZWwu913x4MzLdyzxxsYBWUqxY+Z55DCT1CCexTl9buxMeUQBrEjumJ+WGMARWwxzSDCWGSlWIBJItNjSMIhkhlnhFogheZYIUnCY/dNI+En8LNj/y+wsSBR+EEMvOPoays5LsCt+9AmcSrVisnXCYIndurXjjuOXCOPP4OHGhNtRGCCoxF8MqV6wGx72uATPiykbmiemMi03knO6YpjmOFSKl/c6xdIgpKwRHFkWQ0nnmRFmX6BIwkwQaZ9jYckymFmsVzhic0TglJTzHR7ocSb4IhbIPLBaJWQeDsehWkxrBTTTtWNG2hWkfBZYxoi3URpfdd9L0qajbTQpllJxKTGhhIkBlFKYG11jEZ8Tn9RRKYZRliB4fympKicInAWPpfGLoE5V2bG2MGNUwP10hqcCMpiMhdwExgiRXyJm2OFy07VDGk7JB5RYlFoZAfCDYCxP6+YJmLDzz+AZ3b3aYDOORQmQgAH7IdCGwypn93KJ0RuUy2VDKgEmoXCK1xZV1k1GuzNrWl6+KMGpHkDsUhtgnzFrMhcT1BE3jnEHpgRQowlrK5f8oEa7MTUvGgYgmJdDmEWlUo1NGEuugMIXohDGCNLYIO1eRqAVb16hlKUwkl6lgVglryrulMpA0fuWIfU/VCNXEMvSJ2VnAzxMxQFKG8caY0WakrgKxL97veqw4HuD+cY908NjlPaLviD08+dIBc5s4Osl89jPnOXdJ06jA/Fjx7e/dg0Vma7Pm4NwGh6eZDw8TR+8f8/SFiice2+af/9lN+plie5L43HP7/OC050Ec6I4D++ccI9dwFAbuH2cONiMpGrzSzOfCK89UnPYBM47MzjLtDozUiMlMce2ljkppTo4c7UZkJwbu3A+cdY6DTc1q5gmxYrPVzPOCq26MnYz5+NaMLvXU1jDeafjZL27xL79+hz943zNuO37mM/u8e+OY554a8x//x9f45ncX/OafPOTitOLP3uz4O7/ccPN65p13TjnsPU8/bjl3peLevSV2U3G8HPHmBwPDlYrzWysmxiI0oDV7exYI+OQZokeyxfqqsEmcoNoypZJYYFOpz9QbimYKvhOqSlPVoI0jBsdillguEsNKYauEqYpjRuuSiKtVueRTApULB2PownqNIKX4VbrAs9aWcpC1/bQQ71JQ5CEVumydGHKim5fn1ahI3SQ+86zjc0/ASy8e8KffqXnjz+4RkiHHniErmrphNNomR0UOZTKudV43v+VSTymTdMIqizUWrQuIrmgGSgBeFinhePJXLnhFYSwEpSCvw5lsxDrHcYB37zR899Nj+lFgCGfcuXPKr33hR1gM/MxLDia2iP3IyBLyUrM8K9G8ISk2xo7a6OLxdGtfTy4ks9jB6TIx5CJsinFtixOFQ5OdYjIWdjY0lVuPOtdxwSKPTH2UL06Vyy9SvP4pl0uwGxImaoYsdF5YBo3xuvDZbcJVijppui7x8H6gHjnaaVr3dQrR5QERVWwfqJIpf3YqfPphz4Zu2N3QhDgwXy3oV5q6ast/IwEB4rpVd6xFXCK0uqOJA3pZQVSIVMRIiQw2ELRi0IJa4y99FmISUs7knNG67EJRBR2rrf4h9yAj9DlBUqQoxFj+2xCkiO9y4WLbypZLQQdqW+w7CaiaFr+yfLoKfPJQEJvWaGGHFYrgSyzXzlV85UrHl3cOuVgFYg9xyNQqEmMkK020tgjm9Dph0RcUsrKaEBTxEXs7lYc8a4WpLRvThs2NFlQqFj9ZTzuSIkZFSq6Ms1PGR5h1mYUX1MTiJpp6oqnGiqpS5cXA0GchGcFYjbGmFCJGqIzFokkxslgL/CpnUWiST+VlfRRhLUUoqpXCaY2QUE7hsiVSRpExK7oehqFY5OpaIxnmC89sEfFBkSTgV+A/coyuFlKjSgmsKal8IZfphbaQAuKK2J5VZnV7Rrs/JYWerFZculaxPImcHXfUzrDKCgmaIcFAWQ0YKFCvDMZYEgmjHSl5lCtWVZXKxZ0kF655FhI9xghxUQKndE1R+BPLARQ1KfqSyeAVWhxKaeK622ftstGq2MVSEhBdwp0AKwoVHy39UinKjIJmrZaOFOGYBh8ixESlK9J6J1q1FmUKJhllydljjacZtcQIs+NA30HKDqzFNsLmhqKphX5pWJ7AeFLssHfurLj+4Yz7J4l2a8RyFphuwXNPT/n0bsc3vnXCpgv8zI9vsj0Z8fFNz1++HiEZKiuEpebmDxasvPDKy9t8+smccxcm3FuccbYE5xKXHxuxElg8nNF1hu2tzNaucO9DYbWCva1ENzjGNZio6PXA6V2YbIxYeWE5wJNOeP0HnvO7QnpgWbbQzVYMW4r5LDDaHNNUEa2EaTVBSCTlGULDjWHg6o4njRJf2N7inXsz/s23PuLa3pj/5H/yPP+Xf3qdlx7bZSmKf/UXx/x3//aE//nfepz9HcdT22NeenaDD2/f5w/eOGKrnvLUKx1bn2g+9+KU0xiZVdt0dwN/9yfGPPxcxbs/WDBWBU5FA7oBXQlOa0a2wUvCp0SUnlW2VIOjrU0RBVeZxjlyEJYLoZlUbO4V+64QEEk0U8dou2V+7Dl5OJCjA6/K9EklxJSzq1IacRFFIiaFjYYYNXFNaA1JiLnQVYVElrLaFUCrhFYl9VAZTZbivMoUK7mIJVOTlwMnap/Z41e5/f4HnAyRNgacSvTAvOswszNiiihtMD9cp2k0Dcb0Rce7BrNVdYNWa2KCNYQsSBm0oeURWI61JqhMzPus0UqoXdEpqJzpevjuRwv+V/+F8MoXOl597SoPbo/5tX+HO/7fuRjITUKkRyKo0KCiRVRilQYWSdMaGNkIRKRuiKbDSUmGkwSrlWe+CkSx6JgIIZDFFvU/YFymbhKTtkKrWF74dQEQVUk6FMo0Na/340kgJkVKZUKQculyQkz4FPHBsOwjfTTUrtiujIuYlJifGean0E7XZYW2kMvOWilTxHgGREWi1nx4u+PuWWB/r6Exwtlpxdwr2tAzrQ21LSuMhClQCQJWNCiHFcNYJ1oZeLiyzIMt4kcj+KiwvihVZa1f8EkY1jujWjRZMs5KCSJyGVWxLkBKfLFPlhQzKZZdWEwQ1+AlowzoHiOGKmmsshgBseXL1yqjR3NEHAYDUsQ4ORuUJKwe+OLjwi8+4fnivlDFQBKDNplgM6vkyMO6ww0l0U9iKlHMVcJGQffl47WVKb93okxDgoBJ5BRIyYEKpFAgNEYMqYd+4Rk6RS0WRBOSYhWEVUy0ymAroZ1omgq0CJ1kulzEna3WtMZhpJDBUInaKdry7RS9QYZKJbCJ7IQYIfSJoY+owaD6MoLWJlPVrIFOGW0MOmlIkbQSjGhsVVI6TzvFagiEqBFx+FXgsPfcPYRLr43Y3VqipFrvBAtdU1JES4kszlLiXG0ruLkwHC6pphU6Qw6eZtPQM+H2zUzTeNwPbVkDMXnMmggkqVyyUYqdL8WEtqqMHcspA5i/ImjVZRE3dBGJgh6VF7BMKDVGWaIfcK0qWQdJFU2CaCpriTmTYyGKGq1IsSRviiTIpZCFMlEQKQrzqDVqbCH0OF/sVUYMru/BOmLMDD6hTIFHZQbQZQJgqRlWipOTUCyjgxQ9hMuYVmHHpcg5PQRDZjxRZIQ7dwdufjLgveXy/pTxjmNIgTiDuycz3np7yW5b8eUvXmDnXMXtO2ec3Ol57YUNkl5x476hxXG5Fq7stbz54ZJ+ANut2MTzY8/A0XGD1ZrV8YKNvS1WhwsubI64+zDRWo8fW2RumEw1u5uWH9zoOZi0DNsd924ObE0Vz+4IH9+BjSZxsDXiGx8MfHlvjFp05LkwVlAHYX8seISdzUy7Spy7MsZ3C751p+WgqvjgwRHXpecrnz3Pw5MT7pw6vvHWXS6dF6Y7LV0/8D/7lSm//kcLvnP7iC9VwgsvwfZo4GvnzvGDk2Pm9ztkgE+94ea/WfGzPzNir51zqx7x5qcPePxgk/3tmtx5NkZT8jhQCTRVQ1UrcgolodE0iIEQB4SOsDJUdYVramIOuLo0hypleh+oRjXG1RCFnBLQs7Grsc5x8jAjuTRHzqh1KkrB12ME7YrNtTQUlCYplWlqXjdaOQsKgxhd0mfXDVc2lqQT6FjEtlImgUZnkIGjuuLd1nN89B5mPvCFrzzJ+Ys1v/dHn5CPEru7DT7MMVYgJbQyODUiaoMkg9GGqNbniDI49wiaVN45kVwE6VLEykoVbUHJy8moLGRr0TqCBt+VufFTT2/w/IuP8exL1/jZX/4Kg3j+t/+b//O/0x3/71wM3D3MbG07xmMFEsmo0vl4VbqusaJxNaRhvT83azVbYjkk7s4CIRb/rw+pdMcZhpzQutDJjGmKOp8MpkJWvuCJFfQpk/RaTIYthUAQgk/EdfVGtvQqkaSodSc604Qykm6doXYUf7WCLnuOemHqa0a1Jqm0FrmZNX8AotZ4WzCYaMNiCMxuB5wz6/1+w5ACXQxsSk2tNcZ4kokkGYHPmDQgkhnbmtZlVnO4fxYRMSjjibZ4R40uVagoVdwaUrq7gCBVsS1aDa7WOEuZumiFNwoVNUMYyKolUTC3WibFmuUidgDVapIph6LYQg40CkyRYJJsxmTDOAlnMrCysJsH/vrLB/wPnu4ZV4mRK0AmnRLJaOrKUEskRs3gE/2Q8IMQlCHrQg1MMZGMJadS5tbGlAuPVEKSqkhW0IeeSoPOBiXFbhNSQlWqAD1CIERHt9LkriQzLmOizYJKlqgsqwDKRkatxeqK2jmsLRxwL6GILcWyQDHkYlW0qnS/IZiyypCMz0V7En1G9R4TwDpLUoXU2KWMtz0TaTBVQzYrhpBBKozAEBNZlRz6vk60Q2azGXPzo5733xNe+VLFWJUphsKgdXFoiOj1P0MBOwmyCfq4Ic0iZiehemGYKaaN46nnMzc+VCWiu24KMlpnyA7JJXgJr6mVA+moxJCUYWkC474qDAqKnsCm0o0Y1ZDPurK6y0LUZWxpRYPx9EnRmobkV+jalNjqHEjREmNZl7k1kTCmMhqNoUKGjIx7Yq4he7QpoKdoNKOckWTLOzEZiKcKmwxCInblz68mIFVAVhndZbQK9Ek4vBXJJ4KZWrYvVWw8PqLairAYSDgOP+oYHgZGBxXVANUYplcrqg04vr5EN4FUQ+czNz4NfHw98NxTG3zpMyNWi575Ag5nkEeb3F8ltDgubFsOjzo+uNHzu9+cs9k6/tavXmB+5KntmLdv3+PmasHgWg62WqohkPvMvbsrKmuZM+Kqy9zLnrO5YjGLHFSWYRFYJcVT5wTfKW4eKVoH9abj/Y8Sn9kVfH/KYqFoRi0bRrNV9/R9xWrhYTviV5aL1Yi3Pzrh9j3D+YOOl1/cZbWY863v3uOJq/t8fPchWTTaNexMF4z3HB++r3nh6W329hu+ffMBbmmY78Hbd2f0nbA7HfPs85FfujLmD79+wjdfX7C1C5v7huufCG3q2Zkaqq2WZerZcoZqUq8ZJQ1NW+FjQXGPqoqmHnO6WDIEIRnFKkUMgjUG48CoBF4TFoF625AbKC9mqV/bHQO14ei2Z1gpqsqUcCtTooGNLaFpyilMKkVhIZVCWIePSTaIrIOKQkBiwmKwypB1JOVIWk+sTDagDdEsWNlzfF9bzlZnNHabz//qBdx4yeKO8DMbj3Ph8pQ3vnmDfDrl85/7LCdNj/gNlJrhwzGNLud3TJEoIwYdeLg4o1sGjFJ0IRGTxhlNpKw48lo4X5u1/slERpLIVCz7nsefavibv/bjPPvqM1y5eIXx7g5/8vbv8I/+039Bel/9/7rW//8rBs6fH5cvKEVyLjnm3it6n8kSGZlx2StWDtbiNkSTvGI2i/QpE1Px6osIQWW8VmUXmARtQHQukxQAKTa8wk/PJUzGlvGtJpeDL60V0/mHYvqyh4SiOVD6h3YN4wRTlyqtwWKCwS+ERSeMGoNJpajJOaJsJmjoc6TD4EWTYkarBpMUZ8uBqEB8TyDhWtgbRTaainFd0SiNiYJWgneCpETwPX2vWDxUzA8FsYGkfeFWk1E6k9b5CDkXRgBSpgOiyl5NO4WrpOCbjawtYeWXTzlhVEeKkDwoF8g5g89oV9FHjW0ydqLoh4who2tDzjViDNWgISdOrdBlzbUtxX/vxTF/66kFWrm1tbLY/lLx6Kx3dBprM2IMoi2ZIqoMQybEMn5Tce1GN7L+vnP5snK5tJMXlDWFvy+qjLCjXj8rGlWBhB7fDfQ+0SXPkDOtrnFOg0sMIRKxNK6irkdoYxArBCP4YOilFI6SBkhCCAEoHIY+RuIQiwcf1jHMJYchaEOPkPuIGypGuhw6bVtRVRolxYrpB8PpzNMrwZOxJHYnI7ZIVOOWw1Vgbntu/mCT51+taCUU98mjGOpSAZBTKGBoVXDROlbYDWE1i9RnFWaUqCeOxVnP6MDy1CsV1787EGJATIVkQ0oKEzK6tuSyAUOhUWvMsC13NpJZs7cKdVChwZfnEF0mcbL+TErQlC4sjj6Qg2CtKQRLzJr0XdZSWeW1k2Vt3xUpo4g1uvXR6q+EZFFC4jsF44xe2bLvqBVxUCRTHDyVy6iYIery/JqavAq0u4qdz07Yfryh3kiE00BcRNhqyNpyrhG097Q6MH2uZeNcaT6GaLl/w7A8UZxpxfGxZ1PDv/e1baZXN/nozsCT51oWC8+n9wam40yedTz+2Aic58HRkslGxUUCz1za4P1PlkytoTkI1IvA49s1H93zxDl0xlPbxOefOsf2fs0ffvcmZx58UFwaJbRYbgweM3bsKUPnhRGKy5PAYZc4ObM8c85wlB2z+wueOGjZ2DAch46mmaLswP7WNu1WxQ9uzPnWJ4JrDRc2e5o4JtmB7YNdHpydssoDemoYt1vs7QX+H9885anJiJ/7yV1++xs3mA9jnr/YMhmPWC6XPGM0D+9OODla8L03E4eLDtActDUHVeB7t2dgYFILulsVP/3KcNoqrp1XjCY1yZUV1OaGpg8GJZnNaWZjb8yDe57lEDBOUMqRxJQisrYoV9I9ZSgjejOy0Hcl3lg5xnUknbd8+tGc7m6h1o43La5RZJXQFPtfqmQtCTPoLOhU1srex9KsRIXNrqDbtbCiNLum1WRfpgjNKKDzgjOe4Mbmc5ycfEAvmsEdE1cPYJ5wdcX05RE/uHGbt95e8B/83b9N9ZhmvIqEXNE3NZtul5VEZHEP5yzWVNQkRsaxuzvlwd1TnIJcCTGHdSQcwHqNLMWOHZVj6BTWDvzsVy/zN/+Hv8D2lQsYYzis5/y3//q3+H/+p19nYwWXd6ofbTFg3ABBIV5BKmO6wZfdrqk1lVmLjZxCVCrpY6Loes3JXOiDKsmApf0lICQNQTLRa1ROKKsRXaJwS3iLoGStFE7FNuKMIuuy8RaKRTHmtaBLFWFGsZiUB8AnofNFVOKcUDe2XEgh0i0yp4vE3k5CdJlW2OwgRs6OPW+9E/jDDwKnAwyD4vR0QGtHYw3zlWfw4MYV5y6NuXxBM51GJk3ksW3DxREYH1nNM/dnFUdnlvc/XXH3pmGIjpAjkYLflTUXPmeKIGTN3s+5CBKNNVirMY0mOMBmlAVtZA1iihhaUiojd60MSRV4knIKcYmq0aTsWIWMawTlE7FXiAmMxJHbjmW0pB5+/Fnhf/yy4UsHI6AHk8lrR2VIFE2FikQpvnKnSl5CCRBT6FCKOBNVGdGbEv6Tc1HdGguSFDmXyNqcY8kf1w5SSWVMOeNTJvpEjAJDRJLCx0zQquA6nWArA0ZIkspIGqHzw3qKpHGq+PILFKroQawpwCClNCErFkMk9BErJR4arQqTwIHvy3NqFGhJGJcwNhN6sNph2gLl6X3PbOiIqUw+6rHFElj1msOh59480CvFnXdP6M/GbO161DBBiS8TNLsmaigpNzUFhkUqwtZ2XNGdBVozQamBdqQggKmEJ15s+MEHpwyLKatOMZpkSGNyLKmQhRdULDmSKfClJCht1jocW1TLoohDWielFRjRD9cEP5zKKRgUhEK0lCCkmNd6l6I/EGXKhCsqjC0MC/3oXc4RtQ4j0wI6CgxrCFhO+KO1vNg8Wv1l6olB6SLmLIWupZ5UjM4Z9j6Ti37g4YzVez0yGUHbcnq/p20FrQxb5ybsXK0xWwPDSYC6oTmA3gYO73uwll0zZv9CxbxLfPu3b5LRfDxpmJ0M/LVXtznYdng3IayE+WHHz/zkZYJJPLgd2NqxdKsVW7Wnz56XXtnnz99+wHNP1Hzm6S3+/K373DupGdqWP377BqtBM18IF6YjTn2Hcj3PbFX4qHg4r2ltz/l9y427kVfPVXS+4dDPOL2ruHq+Zk6imneM6jG3H65YLgXbnNCeWeJM0alDnnlqm3vNgvPnz/Hm27c5r5ZsbgqXNg3jMfzJt4/4yssjXnlqm9e/eczf/5lzXHlmwuk9x/c/mnG8CLxwvmU8bsjVQy4/MeL8xpTbpwO3Z4FPD+eYrmV7XNHYCldXNHbMcnHGxtYmopecziKjLc1obAukDc10YolhwDYWKuHCFcfpKZycdOhKo7QhoRhCQIkjjMG2GhMEWQZUXZXnx2eyMYynmstPt7z3ceTuwzn1HCo3oq01rgGvE4NKRCAlXc6TUFwJKhlELOhMMn4d714ySRRC6DMtMNm0XD+1fPOdKedfeQ0lC1Q4weRAToKjQhvwMXJyOGMz7vG3/sFjXLvm0D6infDB7TeYWEWvGo58QLIDZcg5EFOidjXKRETPEa3QEvAJrHGklH4IxosKfJ9wNvP44w0/9e//BF/6iVcYVw0ojdfCP/tXv8Wv/5++QzVYxq1wtBp+tMWAJP/DDy8G6PrEqk8lXc84jIAyZm1cLm1H8nA28yw7RU7FypTzIytWBlMO/xDLBViMz5TuIkUyRcEsqYjPsi5+S8vaYkjhA6QEZm2alnV3LapoESQJvQetKyrj8QakApU0w9JzciKki21BRSpfftdc8elNxW9+XfGHHxh0TDhxvPTq8/z0rz7HpUvb/P5vvM0f/OEbVGLYjZssemERZwziuTXLfO7CmF0Zcf1Wz+ufwL0z4WSmEF+iW8vn1jDMh3Ui4HqPm4tyVNYfYxFsJ4xTmMqgK4Wygq00xhYIUZCIIpbKV7ek3BHJVPUULYKljPKrVhWVebAMvkaNLblecKoH4lHF7qbiV79c89//7JRLdSimdFOAIH79V6b8+Zqy+5cE3gt+iPgBotfEWAQuWq0nPSIlXMmoR6aNwiRfk8WMUUXYq4SkMjELIST6QRi6Ai9qxRJCLHoQyeA0TevQrEd+TmNqhzjoc0eMCmcanFJYB0YJWmmKCQciieiL13wIQkxldfJD25suLAfjChuhVZrWKJyDrCx+mVnMBkZdBdpQuZpNakQCbQNKW06WkWFISApYNEjF4b2B91/fZOMXE7VZYWNdOvQgpcBTer0zLMKiTEDHCq2FaqNmedIx3tCYKhM7hzIKU0Wef3af9z8+4uT+mN2xJuceFSq0SUUIiy5BXL5M7bKUkXdGflgkGIE4lKLN8sixsp7yUbQSgib7AoSSJMRY3BaJTI7lP7Jl81CKW1Hrn60w2aCkFJGSir24EksKGWOFeJpJ3mDHmtB3P1z3VU1dOAeqoppoXAU4X86Ah8Jw0hGypjm/iW4NZ58u0asyQXF6xdYLkOuB4ViQVGOGjMqacTvm/JZjY1dhq0ggc7bKfOaZDbocuX+/58WXWkZjxVs3OwieTSOcdZmbJydsWIWxlsVZonYZd2Dxq5qjs6IjefknrvKH379Pd2qxUbj+yU2qAP2RZ9JaVsMZO3VL7Srmy8S9k8ArzwhxpXn9gyWjacW9mSKnjq6reO0qLFPiha2GqavxLrB/V9PtV+xNKm5+ckJ1vmK33WLlFJ9/cZtv/uV1khHO74750zsz7j70/PTn93nuyhG//Z2Ov/5Vzcsva/7o3QdUjcE7eO0ze/zW7z3ggR/x/IHmgmm4cWvgwdKzNx1xcWp4/2PHJ0nzKz85Zn424979M7Z1zcG2pmmWbI5qZh42l4rpNNGMa/oALgqbWxW5LRNNbRTbe466tZyeFGR7mUIpPImxV0hW5ErQHujW2pamTLGMimxuaJ57ouam1ZydCouQOFkK6bQ82MXXpUGZkm6oDVkX3RRGIeLQSbCmwONC0jgr7DSOIPBbfzHwm/+248UvfZbn9nYZcqBLDV4yjV2f01pjVU3C01zoGeSEd+4fcnHrGYIZcenyRWLv+eDDm9R5gnUVq77HiKVxFjGZPvdEDT7pokFyECVQa0f0Qlw3gFef3OGLP/MyX/3ac1y+coCnwpN464Pv86//1V/y/d+4zjhW2KnnOGhqrX+0xYCiLlqBmFgtFX2fSWtePuIwKuJcOWa10uQozFeZ47mnjxadSppUWvOWlSr8+vSI5pfLi6U0RZiUU4l6TUKIgo/lsjfFgQeS1umCuvxM1uI0IFAmA0qXqcFyyPicGTmF8hljoWmKkO7oFM6Gil0dyKE4DyQpbjzw3DoTdDbUo4p/+L/4Ff79//An+YM//j3+4qN3aa9t8djzBzT1wLlLgtYDQ4hocQxdzUcPGk5bzaGqWFlQ48jm2MDgyDETJZO8p9ETfIh4H4lkckzl4ZJSyCh5lPomZZTtihivpAqq4jWtGkQPZHGkaLHOIFlYHJ9hrKLa21wLFBUqN/QpkQR8gpwr2mbM1z7f8w++UvOlgwqTekAjWRhywi+EKJosJVzKZFUsjIMiek0fS/Z48LIuYqSo4QuFCTQY7TBZl5F4ThjJpSDIgvK2RDhqBVERek/XC8MAwSucWLJYjNbE6MlK0zaOyhaut0ERJOPXLgRBI0qv9+FlbN+lTBcL78JIIvpI1wt+oCCNKfbBkNY/JBXccUyRUeWYVhU2aXoPXUx0XcZaSxortseZkU/cOUoFJe0UygnVYNEu4cYK2yWWw8Dogqa/HpBuijWhAK9yUdcr1DrVMyMxo9CIceAHRFcoDc0Y4rLDhBEhC+2gyYNGmcwzj9fc/FjwS0c98kU3oMtngC4x3BILwTEBJpV5SamaS6GQ+lQ+j5zK2kxysb0CQkaSZlhFohd0XQSvWgpdUNZ/ocuqq1g2dSFfipCiIZGIqVQK1mjUoMjTTIpCmoNuAUqGvEFjK41KusTWZoNqFGhP6hNqgDBPJBztOYNpAssHwtFDqEdCa3t0o6HSpFOPVopoEqtlxCnLwQF0uca1pejxy0hUNX6AuoKf/rEDdrfG/M437vHG2zOevuLolGLVKa49MeK9T8+4+YHnyXOGixcc7/z5glWv2GgyYz3lO9++xY+/so1+cZM3v/OAF5+4wu/98afsHrS8/OyUO/c6ju9nxK7Y27Z8+YmGly9d4J/88Yf8wmcv8L2HR3z8YbH+3X+wonYrDufC7hNXePOduzzsFZOpZauqeBAsvak42Nvk3U+PaaQhpRF3+4pqGEhnFZd2LC88vc10ZGhnlteeEd78dMUHHza88tyKf/iL5/jP//Uxt3994O/97X1+/ZsP+S9/w/ALXxuzOwp8+2bmu+/M+dnXtji/3zFbBYZesbdT42pDf7iinUxoq8TYKti0HJ9ldiYV4zqixoUXYBoN0wIV813AWcN4y2Cc5eSkAxzKGLQVMBVqAF1FqBUMkRwK7Mq4iCSBYNkcacIlwRuPWSUkxYJH9+V5K2dCRlFIhSi1zhTIBTOsNVEMjbbsNokuKV7/KPMbfxL47vcST1za5KXXztGpe9w5+ojz1QjlRpyFI5KhBJuRqSvwPpC1wemem913mS0C8xPDrVsrxmxy7eIe81XHuHLoXNgr8z6wWg74PhNVKmtzKY3YoBPNRPPs0/t87svP8ONffYWDx/ZYGeHmyRlvvPVtvvGH3+XOX96hO8nUjaU2wqwH6wx4+dEWA4+IYn7ILFaRPkLSYHWJkLQOtFWIJJS2DH1mtoBlVzQCktbEMindodKgdS6XHpBEijBubZlDly6pj4plyGWdK6qEGq2Hm8WTKWvhE2s7mFoXB+WfgcIcSH3RBgSBQbBK0VjL9x9E6tsDX3u2wChCgPmJ4v3DiiMRzET46i9+jmc/v8kf/9lv8MHHN/j4xiHjzQOeeGGXPLtHt7pLFwZMrTi3uc2kOs/hXHESNFIrLl4qo6B+WNJ3qSjWB4XvBmgScajwPpKTolt6+t6Tw6P1AWsAjxTPaRByLNOVAl3NYIf1mDkBA6rXhKXlV3/1p3niiU3+r//kNzh3uUFig9YTmlHPwvekvuPJi8I/+Mman3tlkz03Q2WNNoYQEvOgWAYpo1wpKxiyID4RByH0azsjUuBCUtIdRVHcGOs9stIFp0wqeeRGKSy6RB4PEUJkVBtMraH4MaiMQlmNskLyMKTE4NfRt1YxbgwbE0vlCoY5pkS/WudnuOK8MCmjcyBIphsiMcOotlRolLP0Pv2Q+y25wILiWqijSl42qnFrcE4BVC27wLzrUViyEppeWFlDbwwXX2iY7gdSl7l/I3J36ZmfGYxONBXsb9ac35pwgYj/uCdf9eRo0KpY9B4VAAU/Wg4ujQWnwAe0tUhrwFvoMmltcUtmnXjZ1Fy+pBmWCnqHapaQ6jVvAFBlZypKIRoKp0VKimTWZfQzlL1/AQhptLYFGFUIYFgsoU9EL9j8aBqwnvrFkj+SrUJi4V7kXIoQjWHw4K0w5IwTTWUh+bJeDCeZFKDaEHKfcW0F3oFk4jygRIFRxLQGsfSaHAy5dox3HDQD/Zlndj8yahy7e6BtQowhLiI+Omxb0x8F+qHYH5MqaXYqalaryOEs0mVho9I8/8QWroE/fuMe3/vBiompWBwqchN48tqYD96f8cm9gS+8NOWlF6dMXI1MVty6/ZCr+xtIbbh0oDlb9Pz+Nw+xakzwp+xsTvilv7bP1797n9ffn/PkuU3+9s9d4r1Pz/gnf7jg5764IEbHU4/v8b0PZnzxCeEwwFkyXGunbJiBB8eH3B9WOF1xoc6keeLY1+xsNgyLyLAKPH5FMR21XHss83t/mvni05Gnnxjz+28esVsbnnlyip5k/PuB5UbP7fvCf/2vT/i5lyv+63+z5J/+s4HPvdBSv6h4672OxyYbPH+l56OscDryq69s8jvfn/Gnf7HixctjJm1EmwpbKxqrUfjiRhrVHJ6sqJqWqg24DSFVFi1SCuXKEJcJg6LZgB1bc3YsxFCaxewC1A4VTOnyW4tKkPqIcgpdOZRN5AxTZ5gmw0d31PqcKGLvWkxpEtNamK1K/oW2YFwBY+QMi0G481D46FbiW29G3novMutg0iqe/fxF1OYI7zsu7j9OouLuyQ2SOSGu5oyqLXCO+XCC0YqJa/HR8OBu4votYbXK7G9uU+uOY26xNd1GuzHDTJOVoW4jexdHXLy4xSefnrA8GqitZnu/4TNfe45Xv3iNxy8fMN3dJo0mvHP3Fn/0O3/GG1//kPkHfXnXTKZpNTkrllJcBikGUvoRZxOQEtkbui7jg8YLpYuWjNYDTauLBzgXkc9iBcfzyBBNubTXmFKNIilFthkv5aBQWRAr2EddkSi0Ll+Y98LZKoKFypUxT/GBa0KOhCzEXLCxlVDY9+voS7WmE8YMIZVD1jwa/1bCaGR5+P6SG3+ROVdt8dR5DTHzwW14+7ZjEeCpzxzw479wjXrcs7rXsYo9gmHVrZjuTbh89Uts2Yrl8oigA88/+xRH944I148LXlkFRpNQACnDmD56fPDklSYtPIvTM7JVLB+5IMSV3Wn2hX4goNSaTqjWIzPgEbgJFBIV2WeMMhiTsK4i68D/6D/6G/zv/tH/ga2NlmY6IonFqYR2sKnGvHS54e/+TM9PXkzU2oFUJBk4XmWGoSYOJSXMiiUlKXHDQxElltHyet0T1yI4pYoTZA1fUuvCD4rtD1VSEw0lgMivNLOzsjSyoxrbFlGZMtBYg9WGnArNcBUKuUty6SQmlWZUKbTTDOJBIOVEWo8V0ZloMo01+BhZ9pGmrmldTaUSOQwgpaBMUlwEORV1vU5FOOeUxopCS0Z0oZiNxmXllMVw/rkRrkqc3IM4DFx93nBwKaFyzcs/7nltUXN8P3Pv5sCDT+D0MHP31gLVCun7I5orG1RuBTnyCDAmsewHjSrTHUUGKboISQKxWGShCKG6vsc2NbVVEBNJEs4I2Q9oaaApys8EhdGfA7LOIpBYBITiSxFdeN4l5jrHUMiga9mCWov/lNGIB6IheSH4giAGSnS4UjgpNMxCes5oU1Yfqy4yjDIDoJWUn60Vdq7pTyLVhsEULCIyKHKIxeGRC9raOo0ZWK8eBNUIox2LmEA4S3R3LVu1od3IqAZoDTIYYg/GWUIfWc270r2pkmUyeGERIl0UBleVYlIyp7MVx3eEW7cW7E4UmyPF3uaIZuT48NYJuVX8vV+8wFOPOSZbjvmwJN4ZeOXpHa4+2dKnxD/7zSO6wXHt4gjpPecOND/x+af5/W/e5ptvnnFlq+Yf/t0nuXF4xO9+a1WEtIs5o134L/71W7z6xDnGJvBP33jIFy8fEI1HRjWbB4rD1w0/9nTLeF+x3Yz47tfvES5NeH7fsh9bdFtzupiznAf+2nMOO9ZkLD/9+S3+8b+d8fu3z3jtBcff+Owef+wesjyt2d9SfPdO5K//2Bbfv7Hg999a8vhBzdZI8f6DE7YWir4XvvOJpqpqPnfVgptgokfPM2PdMo49+5stngo1BKo6Qmt4eOa52DbUmx7dJnI0qEdR6WOKlVwc9TSw5eDsCCRmbBZEZ7LTmGSQLqIrhW3q0vEOiVxnlIq0NOxvO95edHzrrsaqEW4QRlkYOahchbMNwWvu311x78Gc5cJzNhfuPlDcvC0cnWr8IFQm0bbCRuP40lev8drPvoDZtqSQqEzF/cVQNFnG0WxsISrR9zNaa5nUY87mkY/vzTl9mGmsY+9CjdMRrKJ1K/q0YHvnCc5ffhxjE91geJltZNlw/598iyevXeCv/9pPIeOO/asHVNWEppoyOMOf/sWf8d/9336L7oMIocEYoR0rVoE12TSApYios8a7RymmP6JiIKXIMAjLRSQkjVIaZxtU6nEuU48q1BrgEEJiMU+cLQI+udLlaI1NBYiSlSKr0sFrpdFrVrozgKSiBdAWJULXBWYLj24UrV2DKSq7xjKWFYFPUtYKZPpgCEOxsWkpgS0hZJYrgbZE2oa6GOpcI7R1zR99JNyYr3jt8QlbTPiz7w+8cRypD1o+++NPcGG34ebNW9w5POboeM586GHwnC2F9+/OuHBhnycv7jKdTnnn3gIWA9ONCkRj3IikA05Zgo/Mlz05DwzHc2bLJZf291BZ8eD+A5Z9hlaRxYAyxFAEXJYiekMZUhZCzKQsZMo6ReWatgKJFoUjesXP/83Pc79/l2++8wkvPHUBbxXb2yNqgATndnp++WXDqyPFoCGGBX3SrPpSgVcqk3oNuqIPoVyYGXRSJYqWEi8dGTCmEA5RqYyFWWsCEESBE0VdOZxWkGBYeRZLz9mxcHw8QGWhDUhlikhPhNquv9vgUdoWaJJkxslgNbQOnM6Fb25KboPTGqWluEC8sJSCBF32HT4ImxNHZTM2eeazgdlK6AcpgT5J1pS88lymEJBsaFUuWNFeF2R1FEzWDCExORe49IJhWAjvfUc4uZXY2nC4HFCS2Bobtp6veeIVBxHmx57rP0jc+cHA3U962jcMT75m0TYUNb/WZbiT14WflOQ/lRPKVUgvmFAKHamgHjRRYLUS6s0EakDCFON8waz6CuoMORQEM+s1DoWmlkKxp5LKtC0niq5AyuEstsRla2tRSpN8RqVMHEqxTcxFCCpqvVZRZX9F+fcppnXwlyELDCETckn7zKZUuj5GqlmLEYcbW3IY0EqIZxkaKaFKlWMQIQ2JOoLWEbVrUZuKbHqiT/iTkgw5aiKihdxaVIbYRYyKIJHlCSTvoKqIwdOviiMKq1BERjEUPYgSPvikxwRFVVnEKcYTw9a+sHPO0RzscXjTM9ke8ZcfnLG7YXn7/TPuPDA8HMP1ewOTDcuozZzbU4SUmTYjsrf83p++zXFn+MrXdqmqCf/VH9xgefuUL1/cZOdzjsuXdvhH//wTJm3Ni5eED081j523vHV0xMaq4bWrEz66OSd52JnAO9c9m03ip56rqDY19496Lu6Mefv+gp2m5WDScHCtZdGu+L//y1N+/vNb/MpXpvzpt2ZsLoS3by9QdeZsNePFrRHH9zJbB4pkA5tNGfO/+MyIJx+3fPShZzV4lLIcPtCMmsy4WqF8QpsIJhGTpRlX6OSxqUXnzMCAa0asZoqd8y1ah/JMhIw261WWycSYMMpQTYQt0zN/qMliUcGgdCKaEgGuo0ZVGhkJqi9rRqmLlXDbdXzhkuL+mfDGUUArVybESXC5YkedZ2L26BrL9dkd3vzWBzy835EjKALOBDYnisoYhj7z/PMH/PSvfol6a8LCHyIKlqslZNgbj/GqZdYtycmyW21StTWHy8Dd48AER71dxMvtGsLk6hqyYtxkqI45zSco6ZAKYnSsQuIrv3ier37h5znTt7k3e4f5fYNLFzg73eLOrY5/8y9/F+4IpjJUTcbHwNKXiaaoSHbrQW4CbWJZ1/1Ii4FYM1/2eDLRKZyBKiX6VaDdnJLtAP0EBE6WgaMjQYUa0cXCJUojau0eMJkk6+hdZfDGsDNusXUixAKc0bH45eernrNTjZ4o2lqQmBiSwdiIzg4vAzEaguiSNihFxIZEkk2oqDEJhg6WCmpTVhNOHK0Txi1UNnHjSHM8hyQ1Z8ttxJ3x5GMbPPvEBeIysFXt8fjuiJM7A4t0jMcQQw8xMD/ynLQ9i7MSgLM13WN3ss+4muDqDVL0NFVicfqQ0wcPmJ96zLlzXPvqz3Lw2AXe/PofcfabN2hoCCiMs9QuYpXAoDBG4cx6v15begd9FGIf0Ou1iPcGYwasdfy9v/9r/NJ/8Az/4td/nZ09g51apiOFGyWU0+y4zJcfq3hyM0AVWXhDGNYwHV2mPovUl5WOL4LNLMXuqKpieePRHk4sWYSoLFnp4mwIgsEypA7jhGkzYuIiOhvOFpp5Dw/mmYfzyDILqo+YU8/WpmVrLNSqkMiiZLRWJBWonMXaisZFsmSsUujBoqpE0rmIFlWmcRWVrtDiiw1yiKRVZt5lZos5J7MVGkiDRgaDSpnaahqncVqIMbAaIikI1gqDgI/FMVDZyNiUPydSKJJpNVA3mle+bHhwzxN8TT2xRJUKuOewI63dD9OR4aWvWF76iRp/ljm7H0vHXbywZZKVihdQ6SJylZxAO2TIaBHErvfyUWO0wdlI7AZWiwrnWqoqkbqMThYlGZYCoxpjAsSEpikKZjUQe4tzroQJCWRVE1KEOCBVVSZ2GiQreh9Jg0PbEkmddMCGChMy2UAOCeMdrm0YFqeY8SaSVqQ0otEDi3nFQoUyHg4UpoQY8A4fEm4HtAqkIdOdOWpbkdMSUQrtFdYXcExvE2Zc1pKGTO6BY8uoLpaPmB1mCioG/DxDEGxrmR0Lw1yhnCHYrmgOFNDIOkRJo2zkaB6RBWxNYHO/4cee2eUvv3vKR3czF3ZbLu5X3D085KlLI1oCu1PDxrgB0/HxyZynn9riwo7jwVnP9kRx1Efu349s1SuUqeg8vPLqBh+/u+Lh/Ttc3TL8rZ+9yIe3z/jNT1dMb0Z85/n7P3aFb71/n7M+8+LlEf/sd1e8+BMt57cUJ37M3v7AzrWW2ceBnc2a8Z7m9ZtzXDK4uTDZ1nzneyd85bN7vHn9Lj/2whbPPa64cXfO+a1tXn3OcuPBwJ5u2N8e8d1v97zx7ornnp7w+6+veP7pbe5OOm590nHj9oLLl8Y8dWVKfX3GhXMGtexotcF5zUZVF5S1jsScWfSBvR2NkYFeaqpUYWIg1pl+2TBuNdQDqtYQ17ZUk7G6CJwFsE3F5j4lcS4FdDZo7QppUASVIugEjUElyrRSZ9AVj00NP3ZZODpLPFh5tFOlyI8rTuKnML7PY1fP8eTj+3zltQPe++CYT2+ccufWIb6fs1wpVieZS5dqfurXXmF6ccqJPyPngRw6jFTELAxxQKmKRnl0K+ACs75nORuwyjAkRQiB0Thh20y1VhtqV3Jvyvo1068hRHVK1BugDnrud99CyYqNLHz00ZKv/+6fcv/9BCuDFSHXlJhm0hqjXObFSilULs2uloywDg77URYDC9+z7DPKNEyagCTHcpaJMpBxpA6S65h1iftHmWMvxDzgDKTkEDVgTUOE9cXtcKaEWBgXqbc6JDa4rBBdct+HwXAaK3wXSAJnI8241rTJolNgiSfGEVoNeN+D9NShQppNVA5IP5ArTegTp50QsmVKxKiEGI93UEuk8tA6Q6sizUjR2hG+Nzz7zDa75wwuQ4xwspxx1g8sh4C1BmdKNbqcBeanwqVzG9hK4fNAZ3r6OGd18g6iDUdHMzbrLTbGW5zbucyTTz3O0fyY/+rX/xvOjj0X9i4xe+8W7XodgKvIklnpSE0ZcfugOOsKbrdpR2yf38A5cLYl6BUXLl/hZ3/xJ7j6pOE3f+ufc/39rihVG6g2MuPaonJFYwaymdPlSNcZfOrJeU13jCV3XolFr+2CSmK5fPUjeIWQFSQlpDUPQRGKrTApVBSyjmjnME1DajK9tXReuC2Z+1QM1qCagQ0ctQ7sjB3jcU0zLtjabMvI2TjH0AkpBEzSKFvjQyqBQ4MgypBTYEDo9CP7n8FpQ0UJdRqsRatATJSLUgq8RLIhxVjscNKQtcIPmhCK/XQQjVGCNSX0p7KK6ciiUyYsBFUpnDUIPSlpzp1rCos8LlCmLRlAY4Ndd8MhgOnLweZcxd6VBIuyejGiQBuUseWLCOsJmmR0jGstzXo1lCjwFREqZRiUxg8RJa6Q/0LJy3BZw2qA2pYDQcqkRrJQ26ogrwvWbB3+JSVaNT9if5RxP2uMe8mWUMRYMNoSc8Fg565Q1YAUe0KvkSaQUkmd7KOmHwKMQGIRF1PptVbBE2rLdKqJfaI/UWiVQBVrGUpIvRSoWAVuXJNMsYfZYGGeqEZF85BFUJs9DIbYgRoUpjZ0C2E1K4s1a0KZNgSLTamEG9UZlOb4tCITeeycZXti2dia8vBoxWrV8/lnGjY2l3z9z05ANZx7peWtm3Me3u+pdcfpQvj5VzepW+Hd6wuauuI0jrj7YM5nrtXUWxt873tnjLTm9T87xljLwXbkP/w7L/D+/Z7//LePePKxis8/r/nSc09wnBbc6TPntyqu3wtIrfjykxt8eHLIncPIT70wYfFAeOllYVge8fqNCZw5ti+0fO/WnBefm3C217E6m9Nlx+xE+MLTu/zg0zlHD3rOPzXm9pHlg/dnXHis4vlnAvt7m9w8WvJgFRneCbiNGuvh/qC59WDGlz834uozI/A925tbNNKjY2RcJ9oq07i6XLorj2yMMO3A1GpWORSqHyW90TQWl4sYHFX4Fygh6cLBKPYwwTj3V06bUjGjjQEJRZOSDWJ1scHoRMrF9q618OxuYnZe880PJpyK4kwWZFUw8cezjlV/l43xKaPdTV47d5FnTq9x585DNnZqvvkXP+C9N67z1V/+PBdevsoyJ7KKHHYnCFDlwNliIOeB7c0Vk8qTVMXMF5uy94qu6zCmom0N9UhRNYrWGnQGq4SIBh0wtqKWikrBSEGNw1UTVn3PRx8OvP5Hx3zy7Rk6NaACpo2oVBWYTLn+y+RdPdIFCUniWmS/XjPnH7GbYLUE2zqsjtQYzs4SxycJcRMmuWGCYlgmTuYDR8ti+9NG4UPJjnbKMeQeTECFEQpFoCfhGNeOjabB5DWZDyGayFkAH4TU6pLotdLYdkBGmZ16zGg4ZuUjMiRUs8eiOc+iukDav4Q7uwtv/zaVBPqtLUYjx9nK0G8/hpeaqbpD3K6YXNtgcutTFrMZzfYmrjbMT0648vh5vvj5a2xUmts3b/Pg+Jib9+9x1i9QFnIKxABaa/oUee/TG5wNY87vT2irwk2fjiY01QhlPftPPk1rdtnb3mRro+bmnQ/51rffwHcD0hjiS1eZbl1Ef3jI/PZ1cu6pUUzHGkVF3w0Y1/K1n/kSL3z1JWQDAgHnKnYOLJWeoAzcPf6I/+y/+Tqb9SZHyxm6balG5aEr9j3PmU+8dd8x7+HaFM43mVHjaBpAZaxh7SvzQMbmLeSRXDzFsoJBPxKpE60hk9bOAUtyQrCWJROS3cWHCu0rFlFYWIPeVEztERN1m1oN1NoxcmC0YKv1WM9EDAm9jqr1fUSFRM6KEAYWvcekGqtKIEqSQNCJFBLKaLQpk6IuRU5jIgVDZR1Og+SSy5Bj8R6nCGkYsAghFU1MjjCkwKAzjUCbDdY0jCuLyonT1ZpbkEBR4VpN7iPGGCQ1pKiLt743UCWMK5hTcYWuqD2oWYEZWFvWDyK5ZEmEwhxQrpAb5ZGo8VGVvx7RkxVGKVpXczybU5uKbhYwtcH3hdNu64Y085hNSw4e3VhY62nIxXWjdZkKZYpWovz8IlUIaxRzzkIehJiFPGR0vSY0ZleImcFSVyVIajVTTDcheV0yLAKsYsKMDTaudS8CKqxFv2NFJtEfJfJKU0/LKSaxCIvJufAkRopBDVhtaJRFZj1O1UWfEHr0pkI6yhqjtyiV8AMsTgvN0VoQyVhb0w+Jk2VCNUU3MZsFDMK5Lc25XUvlEt995zbX7yueurbJU1dGfHyz5+O7CcUSP0T+/J0Vo6x45mnHpPY8uGdY5cxylrnylCV6z2euOfRWxR9954z+oeepq2MubWl2dytSMLz74Ihf/827/OSzhoM9x7mNDjPa5R//+n3+5md3OMkrvve9gb/xYw13T4/50zdLEFDfDLx145QRYy5fGvP9T3quXjBMryrssrDfRGnarZZLKfHu7QWPXx1xaWIRawk+cOVSzeGx4f5hZuwiNkcuTkZ89qcyZzOKJuRiw4cfR96/Ax/c6nnpkkMrT8JBToxGDucCo5FhVGWcsShjGELP5vYIUZEtVzMETyLQ9UK7MEVArCCPE8Zq8AWqlo0vDrVYdDLKrZ+XJOhUmBei1qvIpJG19Uq5YlWVWJ7rpjE8v5+Jb53wyYOGm67iehc5G3maDcPQ9iz6wLgJTNpTmmaLF159Et1sMpfAiy89zgsvPwkuM6yWnBzfg27B/vY5lDPU1Q5xCCVCfNRw1t0hrm6jgiVrX9aFRuEqRVVbjClaI4UBJYzQaBWxRuOqwhvRonDmHCezzJ/+0U3e+rf3SYtMLY7AQFNncoIgHq1s0VCxLgKQH6b8rkMMitC+4EV+tMWAtUJNoKkdZ4Pm9tmSxdIw3anoc8eQNGZVcfLQsgqCJkAaMK5GBUs3JLKuUQ3kWshxRV05JDhsmxk1Gu8EnR2xKzYfHRWVEeywYuUq1KLj4eYeJ5OfJkfLpnuLeO6AE9nhVFrOZEIXOuz9gbbeZfzs38RKJEwu8pCK7/qOeCboqNDqGtk5RldbfuLyZ7C2Ylw3HN15wNOnwvmreziz5PRkwfF8ye3Dh5yeLYlDLrZKH0uOttY07QhbtSwXEDYtO9MR2sAqrBi3jlE1wbrEov8Q229weKK5d++YmB3TzZadaDj2d3ns1cf5/C+/yuGdU26/+RGf/Pn3WHYDjXVc/czj/Op/9He48spl/uKNP+at73yHB3dOcNUGG3ubSMr42RESFqg4YaYSegRVn7DSkbMlSsYoz1Iy9x84Pn2YOJj2PNO0nNtSHEwDG3VmZBVOO4y2aKuIaii877QO9EimHPA+FQxv0ESxDNIQXMtCWxZmzCxv42WbqCsq1+KqmsZGdtQRbnkfkwOoQEgwDJ6UKsCUBDKrSoiTDwQp4j2DLuE7tozIY0jkLjCYYpKvrCIZRecTZ71nGaFLmV4yOWvayuOrYm8NQZOGUhAopZg6zaRZUxIHT066BGhJScTMucRWD4NATHSd0K9USeU0Qkoe02jUEFHaoUIGHVA5k7xF5yJ8zUajqlQyGaKG9SpE1k4NhaCMKR1RAqN+iOT8obukOA5Kt2wwKCWMqpawCjhnGbqMToo+BcbKFDFOUe0heR3dSsH9qrxuIaRkfChVctklQw6ZFAoDxOT1/6b8XbuyGtBikGgJPmNHlpMHQuhLLkaKusTOrgyhEYzJEAwqa1JYh5GNDfVEE44jw4lQj0vxl7IpoU1Ko1yGGhIlAVVLgqAx4kg5kr3CNDXSR8JcMXiNrQsIaTkzJF+vpw2CMpkIrKJAkzHec3yq6LJhb0Ozv12xPa64ez9weM/SKM240hzNB/Ki47xJ+Kbio7ueazsVz73QcLzoCSvLZNtx/+MVn31+zEmo6Rc99TnFdz9YMOoDz312i2lrmT309DExIvPn334AdeKrP36erIXvvLPgnRvvs1Elnn1ywm+8c0QjCZUDi9Dw8BR+9lcOUCny6Tc8L13I7O6POe179J3AtXOJzY2a8xfGvPHxnPc/XPDVH9vg7etLzM0FL724w427Hapr+PaNE6ad4vzj+xw9LJCbT24u+Ph2RTXt2d+qyTiOV5ndDXjq0jaTaYfrW1bzxEZrqesRru6pGoWzQuUyrnX0saPvE+24RF3p1hIkkWNGK4POkbjIGK1JbQHGaR+h0mRdSLPkgKi1l1wDSZCoi/32EdkySykcSnUAegAsKcP2RcPjL1jufPCQ/nbm7spyxzqm5xr2L1Vs7jqGfsHZacdjFyboumbuF1y4OmV7egAYFmHByfwBKQuPXXiSUdMyX3rssGKIgdVgWYWBbljRR0+UxLgGVwuj1lHbtVA3SqHi6jVTg4RzNTpMcWmCUQOhz2TT0Gj40stP8exjV/jw3bvc+uABh7dg8AltE5YWka4cC/kRxrz8bKUyRTJcYU0gxoxZW4N/ZMXAb3df5ucvZWT2Jh99knjrbc+18zU6F1hHHMYcx8BptyJ3AVNtoVxLUkvmK8Onc8/+yLLv6iIqQhhWPWIy9QSq2FDVDVFW5NjwMLbc2n0Gv/8Evf8GRnn6J75I11zkaLYghYHcfJax24C+Y7k6QvtDmnHLoJZ0Q0OyE4yZEFdHrMKcFBKjeozoQMzgokKliqSnoCrOuhW7j13i4pMTlsslft4TVoHj056TRWJYZkxyVFrjK42xNUZbkiqFj4sGkzIpJcIQqF3FSLd08575bEFTO07Pes5Ojrh//yEZSzvWiHZMu8StW99nFe7x+KXL/NL/9KsM/+BrDEuNxvPkc5c4PL3J7//O/4vDW4cwS+xt7dFJYrY4Y6+t2N5+jFt3b7LsBrabmrDs2NzbJHtPMLmo1lMg+0RYZead5/gefEd17E4s58fCQWPZaQwbTtFWkVGdUdGXcbxoYtL0XjFbJU4WgZXPLMRi6w2kmqDaKTRj6tEGo2ZE2xj2bGTsVrRmTpOXtP1dbHeIDIkQLV0YkEqB0mgEvRb0gSWkQIiByliUU1TKsKEt2njOuoGhH5BxjcZBCiQgxJJg2SVFQJNCKqEgWqFzxqLQuYjdsgprJ0RVbHRSLkeFxjpDpQSXFNmDD5nBgIrC0AdCX5NNGfuppMpBVltyF7C2kNKUBttLYe9bUF1CLw2MLKmKqDWhsegAyrpAdBFx5mFAOVcSA9f1vUCxXGaIISFD+Z1rU9FlXw6HNTdBlGa1WDLe2iKvBvRGwzD0aKMwtkJU4FGKsVYFSawxyBo5TiyBRKLLdIKiRSz/LkGMCa0ieFemHmKYHQtWZbzPJK8IOZKDQk+LPoNQLgBiWVnkSmPEsHiQMcaha0UMGdGxiBpNRhvFQMSoEjxFiAiKgCKuJya+D6SFwvtEOzaknOkWQhoK8Elj0AaqJhMAoqA8dLMWnRVbk8jWhmZ719EPiXc+XRHIXNgwaOc5mSn6Hp55Ype70XPSzXnl+R2W2fODG4rptOHevRnGTVjUDadHp7gNIeptpmrJeHuD586P+YuPDtloLCFWfPgw8PzzexymBR8f90zVwJ+/1/NLX93h9HjOIg0c3bQcPFZz8ckdbt45YWsaSQvPUgV2bOLpz2zywa1DPvtiw8oP3I49YaVJvmO7zbx3DxDHw6XmsT1DNdK88dGSn/tMzeceH/O9b8/x8Yxzu4pb11c8/9yEBw/mfPuDCRdeBttEdnc0i7lm4gbiACkqWnp2N8e4emAysVibi0jYKtDCZDIhpEC7ztpQSrCNI/hMP6yYTIvWyC4sKgppnIpOxlcoXQrpIqaRgj1/RMHKee0WUoiW8vymjMoK5QBXwqjK9Mtw8KJj72bg6Oacu0eZ62cB83Hg4aWB/SsN566OePKpJ9jdf456tEcKHWGokNQRUuRsfh9re7b2pyQSD+YPOOsWRJ/JOTFtDsjiEX2GrTpa0zCqDU7GaGNJORNECtyIhFaJylrO6e1CWay28MFztDhlNhOgYnNrk619x7nL+1x77hJHDx5y/b0j3vneHe58copf9muom1r/3PVLzCOcPRjrAcFoihD5R1kM/IvvH7E0v8A4vsDLP/Yktv9NQv0B0d4nLYR782NWV/8292YnhOZ9rnz277J8EDl575/yg0/vcTON+fIzO0zCPcYVnCnHxoVX6I8eMEn3sHXHnMf5MD/HR33DiZswVw3EQPXiz1NXZcfrF4e0uvjPdUr03SEpD1TWseqXhEFTN1tYnej7FaNmjIqaNrdkfPEoo2DIVM0I5zaKAlp6GtuA71gtO0IAtDALZyziKbbJbGyN6BZLwrBCrCOmjunGJls7U0wFW5OKjXFVaHfZUCXH/GgFraGaVHRD4OzUM/jIeGePEMq4dDjtUT1s1i0Xp2OsX/Hdd/+czd0ttpqWhcCb736AjYEL21NmZ2cMFkKMbKnMpfNXGTrPhx/f47RfcO3y85zceYjG0edAlR2SMkEise/JXhGXGd8JPpWX69Zp4HqCnCyCwmawKmJ0plkjdmMWUkoMPrDoBjwa7Wo2d6bs7G2yebDJzuY226OW3Vpxrjpi0xwzUcLYL3DzgdR7xC/Jq0D2JTvCtOUSEFVcK1qZdTFQCgTrNCKZmIp1NYnClpySkpKXc0kZSwU/nbLCkmhUxulMrS3aWCqz7khCyR0wOjJtFVpbiMJq5fERfMnHohJBSUKJoCSjFbiqAmWRPNAvFT4VMZBRkeQNOEHVNfQeRCOpBru+TEWhq8JbyCuDFYM0Ak6vcybSel1QAETFXFN0GloMSWLpLtafi6jy/Sk00XuquqyTqqpiGAZ0U5EV+KXHNhp6oXI1qSAiUQ5kVd5vYwxx/a7nKOhYoqd9KpbGShcAVkiZnNQaL1v2RDEI7WbFg6OBrtfUNtGHRAyUvA0ttKZoBAigbIKowQqutqQT8IvMxl7hG6RUGCQojTYJQaNE41CkRSAFBU4T8iP0NoRlZug99ahGOxjOMjlYahVReLS1aFMOvNBplg8i8+NISorKajZby/62wbnIe58MnIbM1oZjMjXMjz1ihd2thqM+cutmx6WthiGsWJwlnrmksCYxnxWF/+xoyRMXR9iR5cangews154wfPPdezz77B6Lk8DWxhhBY1zik0+W/NoXd/mtP1vwxcuZcxdb7txI/NZfHNJrzSvP1jx8OOP+4QRvVuxtwuXdmre/4zi3UXPzYcX25jYvHBzxv/9vM1PT8/iFio3JmC+8AHpSXEcPHxhu3O+gV7z+4YwvPFfjk+LBdcern4f3Php49/pAXAnkOd+9rjl/TaFNTeozq2XC5IjSmu3phGkbUcYzaaYYY7BVxrlynFqnSZUQc6RxFaKLTaWqDSkWiJYxVbFE+3VS4KhoW3Qsrz7KlHdPCWILlVDiXxUAmOLA0ZIgJyRawKJsABVRXmgqy2MvO3Z+MObiqGUaM6dnkc39MefPb/Pcs09w7amLmNrhw4LsC/PA2ZoUVmxVW9TNAcYY/ODRRtOMHdIIVbVFlxP3u49QTrNlNplYgdQyaMeQO3IecMagRdOYEZWaQKj49PiM+UogBdQA3aLl44+PeHD/fbY2x1x7cZtnngRXCRu7Wzz7+SmXn9rhxnsz3v3+Te58eEoIkRRjaXLMI+KnpqoqJA+l0ZA1nfdHWQycfPI9viG3Obf7Be73W2y//O/xIH+NO4ffZRKuMz2/z+HhPm8eH3P58q9wfCOR3Rnu8V/D5A/53NUnaSctp/NjFqbHt2P81hMwvsmsepfbJP7g7kvcHyAMK4xaYs19fMwkV+FkAx977MixXKyoTcNIO7phiTGGXhZghNaCSx2Dt7T1CJszngHVOGpVE6Ujq5rxdktlwOS0Rj+2CA5PwdKm3HN8+pCzkwUSEo11hMpiq4HKWppJxdZ2xe7eiI3JqByikku4ClBXBmzGTUdYsSzPVoQUcJWjalqM0nTisabibByZ7O4y2RqjjSCVZhIrRkrz8PgEZ8EZRxDheH7Mxv4Is3KMqg0m7TbMTqANfO2nX+PJcxc4vbPiH3///41G02LwDIgvscNdSPSdYjETfMjE3NPaulw+toBaQoQhKFZLR+qFqFJhRcS0HmWX/fJoMmJrb4fzB7ts7k+Y7tRs7DbsNsL56pSLcpPNfI+8UjQpIX1ChgIqMkmomhJ24xpDWymMLmE2Yko+QEoZazWVqUvuApGYUrHEiWLkLJUrArp5jMxDLEmJqJKyh8KXNwSXFTWQokAqKYStAaMK0Mqvpwk+FdSutRqnNMUBl6msYTI2jGrDEDMpKx4cDjyZHLXtMVljbEK0QZqIpFRsekohJiMhYwZdKGqVLiLGZMldQg2CtrZ4/7WUDloAY8hrfHBhOGhSlJIZEKXsdMViQ0ECR1Xit2VIWDH0IbDVWIb5gNUN0q2jgBEkRVzjGJZp3U1kSsLEXxUESgs5aWJaT1WAGCMpGiQosA4lGq0CaOHWrZ4Wi/eGsdbkMBBVxkzKVDd7hYoAGastCo1JmtXDDlOVHT++rG3IGeU0yhpyFgwGiTB0AR8o6ObKklQEKnwHVVPhnNDNIfhSPLhcY6wh61ROO6vBwmTfcG2zZmerYncXxtsKnTJHd4XF9xKjNHBpXPQcOhs2ty2TxnHvdM6OKPQAqzNDDJm2rrh52KE7R7URufr4mNsPe+58XPZQB+dHfP2NM558bMS1cw2/9YNTdrcqnr/seOejMw42Fe/eOGZ/f5P53PP++yfcfZgYtYavfGmX6x/NqdtMskt2R4ZvH/Wcmyle+9Iml65YPvzEcuf+fS5d2Oa1K8fsn5vwwd2eyagh9x16FXnxJcOHrw88czZh/1LLnesrllfGPP3kBqs4I+gpFyeO3QsN79wRln3iK9c20GrOB0cD53cqukFz4YJgjoR+2VFdHFM1BmvAjnQp8lwq+R4pYJ2AMgzBMx5XRZOiIuOdUcn+0Jo8Ku+zjhbpBbUOkyPbtUCgEEwxBTX3Q/1M+qspGaZYZlMqIliVyyohW5AYuLqv+JW//SxP51fJlYPQ0Y4mWOfwcYGP0A9luW51iYHvw0C0CmOaYj2OgawTYhJTPUZXE1Y5kroFlyYTJlXDrFuylBU9gvEzRpWhchMMDX4Qhi5z73jByUlmNgRmywGTlmzbFjU4lg8c9z445dPFjHAauXb5EmY8IvvApIaNC/vsH+zx7Cvn+eTdQz7+8A7XP3pIP1vnt6hEzpkh9BitUYBrNZ/57LkfbTHQtru8/96ch+f/LdWzI8LqKdrJAjO6xkke4Y4VSp9xeX9MXt2j0xWro4doazFXdkiccevsFqgNdGwZR8/i3juYesqpfpl3TloGf0wrCzQdfegh1UyqLagrvE4oDNJnTFPT9Sek1ICqmS0f4nBM2ymjegutaloFK98TVGBST+h8Imuhj5ocMqOpkHSFVBa0I6SMtcUGiQgxLVn2CwbxaGWYNCPqnYadc2MQYTppsVqxXM45XfSFr54VtjJU05rRpMKgWM48Ws9AGzJCn/qy5yURKOAKt+mY1HWJe1WJEDXGOGb9rCRXiaNfBSrV0lbbjCYj0sSTfcf2NHPwyhe4sPMYD+963nz9DT5+7xar2YrdyxtEZ7BVg4oGGSD0gcEHJEdsFnRqCKpHYdBrUY/KqeQF1IXp74wqHXhU60RFg3aOjZ0x++c2aPcc04MRG5tbbG9M2Gk7Dsx9zsUFG8NATA1ZLENdGNR1Vtg04BwY56gItJWmcWAqvYYNlZQ66xwSymVkqqp0diEWQmOxJ6Nyok+BKiUqtQ6vkmJFy7nsoAcdSVogltE8ovG6CMpiLvkYIQlKaYxWGMmoIGWVkAyVc4ydxQJdFLJoTo4Dq4Vhu7ElL0rK5YnNaFeT+1KAEQGdkFEgDQYdHaqCpAeMOPCJNHhwugC3cgkNyrLWNFCInahC7Mux6DTykElSIyFhlWLoPbaqyKEI7hbek1wBGPnFUBgdnUc3GhGoasugimg3pYjConJGsb6AkxTx1v+HtT97tjVLz3uh3zu6r5nN6nebfWZVZlWpVJJKsiTLko1txLEBG4jjCyAguOGGa/4BLgiCK4jgggtOEAFcgH3igOP4gKWwsY8syWpKp1RSVWY12e7cufvVzu77vtFyMebOkiGIY4JaEZmVmVF7r7nnmt8Yb/M8vycWQqgjxzhVPLg2BWMroMg2ipvNjqvrjFl6oKv5GznX7m6PRC4eXpK0ZJ8PH9cJv/XMTmbkOKKyYY+xIOVEDoXWanQUctIEvw+1ElvZCVozrQNGK1RR+G1dIWirsFPCSsa4jMxayryg+0DvDMdfcyQNeSqE68TqaarZCY0wbwTdNNhZJEvh0M6YW4+2kdNjw7BKbFMkryZc01VxZYicHjlM0/KH37nkZpMoYnjvzZZnFzfcrDOL5Zz3P9/Qu8TxieHf/vg566HD9ZrHT3a88bbhsxcj4hz9aeFwqYjTxE8+3/DO2wt+7ZuG/+O/2nF4rrjsPecXNzgUK6oV6Ml1ws4yhzOHqJHvfLDhzhFsrtec2DnXh1vuva548WnhVtfSHBR+9IOBe8sZl88mpi6RtiN/46sdZrvms093vPOK4le+1bHyGX+55eqBcOvYsTQBMRrXO7KeSFphOl0ZD0VobEvWAcEQ00DMGWs06EjbG5ShAuqkkFtNigkXqPKVptpXyl5Iq/ZDwiJ13E4GkkYKpJDIUldJRtfMDb8DP2VKySSlWEdF0UKrhWDmeNvwfHpBWD3DGYV1C2LWSPb09hClluzSBp3BZfB5ZJJM8AOttWBnRCK2NNxbfo2QLsj6BWkYSGFL2zac9Mck7zh/DikfokzL5c0znj55jJaWuVhiCaiaV4TWDbOTBc38Cj96Pv3wgovPLrjnRi43mfnccfvEcTS/ze5gyyuv3OJXfuNdLp6tefjZEx5+9pynj9esrib8lDi+ZTi7c8hb78546+vNz7YYUClweDhjezGxvvwxs7Oex08sx6fV9uF6y6w5xidLToUhZYhnTPGGZjNRuobj/jalBEJa0+qOzVjoW8V2tcHMtyjjyUUwNHS6RTC0fU8MvsZa5nrIZ5+gdJRcfZuHs9t0zSGFQiJSTLV2hDyQcwTVEhkwBRYGospkE2iaFhUVJWW00fWD6wOS6zohlcSA5+jWKfOmpZTI5SpwM0Q2l1eUEBhjwGTFopujrMO2iqYVYhjYTJkiGq0LJYZ60WhTeQtxxDQOKRZjajSlyo7WVMzwzWZL5zwzDphCZtnMOZ4tGaYNYxmZHRzw6v1vcrA45vt//gX/+//V/5Yffe9D/Gai7xpu3TtCe4UhUZQiiSJQH1SJdSwcSqaYgmKGWKkPZqnxoBX+VEBTUbU5Y5RmipGsNfOjJYdnByyPZhwcHbBYLFnODAuTud0FXrU3nPgdyjSMbSGHhE0VcCOlYKWtqYvZY0VYdprlzNBZRc4TJQqhaKZcGQclV3tdziCtoE0hDzXmdvKZkDXWOBYCkj2hhDpmR0h5T6wzqmYkFLAi5CgMoTDEzBBjJf+R0aqAq6CckhKmWFQolCHjtWIdErqPiHfEraBugahIyQ7Jea92TnXXvo/xLVkhaLSVCjQaatdRNNAKZQqEjcfpBqUNSKbkjNGWMIUa9qQNJjumoKq1UhKuBCYtGK/oETYp02JJY6axlqvzwMHSMO0MzWBqIEEJlMmhiuC6QrzuyeoCrRpMqeyKUKAkS8wbZNdBGVELQ0muhoFRPx9TShzODD94f6inmmhGEzDJskuZ3jVYiRA0YawrIK3NHoiS2J1nXGcheKJYsq3UTq0tUTKWgkzVfhY8pGzIkjB5om8Um10VQyqlkBhQGiiKxkPfgpwUWGiiG2tgjWjyGAnnmfV5ZnsjmEbTHBWiyuyuE69/RRNOFLttwodC22aapgLLxtKAjpg00CqNKZlhs+POrMUdO97/9JqLq8jp/IgXN1vioBh3ll941zAMOz79bODv/tot/uCH1/zlDzx//7eXfP+7G85uH7CLhd124PtPJ/7Ot29zs9uw24zcPTng/HLg2a3MGzPDbhw4nLXYxrBlRC+F27MZf/7nF4QV3LmdePO1A/74/Rv6mUF6x1/8mzW37gtPX9xwZDtO34DdbuBrbzZcPIskElISD54qNmHi3plmtdZcXQ+EKTFsC1Y7umVhE7fM2gUSMspGGlMhcOKgKYYmZ4L2+2d8BCv4IdIfOWaHHVkFinH1Ui+p6gxE6uQoqao5a6G46lgxWVcyqGSKziTJTLuIv6qFplK5fp6UoxhFKJ4xgQ+K5Atj1FyWwGN5wQu/5vnFY3J5wPFxxFhFiANSAo0sOF28i9OnjPEaKATmKDJGIn1/hNULShlwVhMp3Iw/Zrt9SIgbTKOYt4eUYc4PH058908/4fKp4q99+1d446s9KWdc4zB7/RBTqemFytXiOYQamiSa3S7xu//sx2A8ORe6ueHrv3SPr3zjLrYxnB3fwbVz7r92wutvHxHSe+y2kfXNDTGuWB73ZL3Cc8k4PvnZFgOPPxDufVPTdMIPfvgRW3/B3Tu/RGu/imyPCHkiovFJUPuDOjSJg/6Adt5TIlhrGMaAlhnTmIjRM/k1zgjTOuCcraEnSqBUtGooO6Y81LjVkqFYLD29tXTWYGzDGBKTnximFUUEZwxkT6uqcjz7DcoYchYsmrbp0K5FciFnjzEGLZXkJElQSTOOA7vdBpc0pgjXN9dcrba8uN4QplA99xYa09HPF4irSEgpljjtiNFTpMZjlqIh126yhEzTaYw0zF3P5G/IEkFatPVIFGa65+RgyeHiCI1GqYlgB6Ja0c4X3O6/yq3T+3z2k0/43/0v/9f85Z88QJfC8bKlv7VAaUuJhTgWjGkpY704UhESCmyDpIjZZ0MEW/GwUkCXyq7PxhLbUqNmAxUhXTJKWlzfsDg4ZL5YMJ/PWC46ZsYgZBZ24P7sCbfKBX12THqii4psVA21oXaEqpTa8WZY2Jo10DZVB6tKRVNbpwi5sIvVwpdjomSFKoLRGi+5hlKVfdhPKEySyBR0qTCTUGqKg1KChUopTOCnxBgyY8pMOTHtu+BWW6yrfmApef+aBW0hTJ4sgtaWxbxjMwRWT4XxtVhXTrHiVUkWVEQTKVGDruP3GANaV8BIii+nAaCtw7iWElPdsYfKUDe6rRkPaOJUI2CNEYyp48AYFFEyjbKMUtC+INHjO3CisdnVEJRp4uykY/1iYtYqyjhDMeK3E3rZEC8HlOpg52ukawhEA6V4HJZVCvRJUCFUgmDWdFikBLROrDdw8UJzvNjTEqUwhUiImdAFWtHksdoQK4TMoVViutFMKWD7lqQyOVcMsp5lfJpopIFYByupGKZpBATXamYHhrRPNtUmY0ShrUa1GdcUTCdkZ6oTAcFmS86BsBHGC83NOqBaxfE9i+sUuzFz+SxA7njnvcSs94y7nvOniaunkZwSWMMwZHIamfWWg7lGa8GtLKpTbKLnzBpe/0bHwxdrDlKm7yPjk4S9b7i+2dbkR4kUv+Mbr/W4ULNVnl1Grp8P9Nrx1tcsK7niOz8O3D3NfPPbJ/zeH6w43zlymzk67LkO1zx4DrfOlpy/uOS06elPFddjTbtcjxPOBUIBpKWbw7hSdG9aPny64WxmOI6R9c7wfBU5uNNz+8Dj0UxTZhUzWo/M5i3KjxzMIEePioa2b9nEgXWccWYblM4UK9XurwGjaLUhM9aitkCxmilFerVP0iwJtAKpPA1lLSK5YrSVIQ6gLFjjKGEiS403LknQWhFs4fcuBv7o08JB61jYjNUTptRu3grEXnFjM5dJ2KjCzfQJ6/UlF6s1qEwWy8FSYRsQ6Ymq8Hj7PSgRg8Kqfk/PTIg4GC2pRGJWLGyPVokQJhSGlBXPP4s8/DDwoz9/yhc/uSGMkXbWsPvKClEtRQeSrohRyaB0FbYaoxlWW549e8bmegu5TuGePxgqh0EihciTjx/wnX/1Bfdft7z+rWNUr4mSaduexmiOZy3LpaGEHS/WD4l4lBTMl3yYn1Ex8PmjiWljefuvOYZuxqNHWz5/8p/y7tf/a3z1tXfZPB8x9obeLNiGa7qmZdxsGHTBOWE9vEBE0/dzcsiEUGNqoY6ejW7riIhETKGuinS1sJEtWhTKWFyzwNmOkgI5TqyngSllkEjRgg8TiR1tWy//pNI+nr7QNAVrCyFOJC9EJpzVtKIoqbBNGqMEnSfiNFCiYdkumPUNz8LAGCesUmgxTCnXnexRS2c0TQso2A2e4CGEhDOOrjcEdigR1H4qUCSSygCN0OgZKWWOFifM254yTiz7GWNcM+UXFAcn8yOafICoBbPFATlE/un/+T/m//6P/w15lzidO6w4is9MPqKbREmeohrSTjC9IUsNkzGl9stFFZKunbOVQlF1xJbqFV3/MHtOvyih6GqD00qYHyzpl3OavsM0tQhIOTAHvna25q3mM5ZhC9iqbJWKti2lAn9UqbAwncFqzcIqWrvfV8caHGRE45TgFVAqKhcxaGtIMbKdqhAQMRSlwEdKiqQcK1MfiCky7UeLusg+vU+RU2E7RrY+AwqjLG4f/eQk06LostBmRVsUnSnMm0JrqkhRScEpsAYuH2WGjcMdBjBVbJf8hOrVXqG/N/oKKDHV1F4KRmlSyozbiLG+ahQajR8K4wiSNRMjTglWW1CaYZfJumCagtY1sTKkGrGcVEQ7g9rCLk7Ico4bPDPnuB5GBhMwzlJuGuRgouiCQ0PU5NmWtLJV7X1ZKKNC9YGYFSoKAY/3joWzqLAjKUHUFsuMkjTPLz0he5QThp1wtOjxY8GUhiwTlAXJJxR19ZFiQopj2mXcoSVJIESDkkijFSHUDHiTLMF7bGtYbwIZjWoKXa8pOTFsAK9ZukQzz7DQ4BRFJTJqH2O+F2VO4C8Uu8tEaSJHdwyuU4SQ2YyZy8vIOBWWs4kwwfUoWDdx7y3L4Ynh/FFhdQ7DNtMYx7zTzFxmzInFSeVk2JUmLAvNURULnx1Z/NXEnSZz0jTkhebywnM+WcQ5ZgV2Y+HOiWW927JJitf6hrOzGY/PrylJeO3E4eyWIQtffD5w0LZ4H4CGs7sW3QycX2iWtw1ndsG4mBAZWK+g1QrXCLOZZyqWzdago8Up4UcPdvzCV1tmM+H2LeH66YrcOIoPLJYNs07YXSWGm5HDheVgLjXAbIjMjFCayPkUeDM7dFvjdU2p7ploCqZABZIolFhSyIxTZLMqzA511YcQUaapOqAy1tTbrEgBtBjwgRJHVFPXVjEmjLFQIss+8413LN/ZeH7/ImCnggqCT8LG1ynifG45XGqW88isecGBDiybwsFBx/lVYLetAsPq/q+gsaIdudhK6gQKE0LCSW1gSgGxQkPAlhnPLg0ffbTiR395wYMPNoxX1QJsReG0pXO20i1Lh2ZOStdI8UjWNY02ekKxiAiNc1jn9xOCmt1gHXtrsZCmwuVDz3AZePoocvqGQx8ndKtRFPoGlr1h5hq6vlrfra40x59pMbA4cjy78YQ/SvzN355xNRXGeJc/+5Pf4SZe8sv3/xqXl49pb71VgSVNYm57nm+vWcyOaJoGkX0EbUpoY+h0RwiRprGEmGu2Qa7WIx8CTTH0swVBIk2r0aojobi4PifmgNZ1TZDyRE0nMlgDWSI+gFaOoqvCdNkcouU5L24eMe/ewkggmoy1wmrzI0pasly+wmr3ES+u1uR8h/u3X2E2G/js4WM++PHnrIaBxrXVMlImbh0ccnjU08wUuUzcXG0J04QSS86KpEaKtJioySXjGuFofoTWMzpnOF3OEenIwbPdbchpA13hRnncXNPrI4xojBxwdHyL9XbLJ+8/5D/+j36Hn3znc46OHW5eE/tS3EIstLqtkcAI2iVgg9dzMLqm7+U6fi6qQikyVLynVAIekvekuVoEvFSvW2PrB8Ya2vkM0xpEy36dUvAl8vVbnl+89YIDvyOJZlMKjSgCey97qumSlppHoSI4rXFKVcJYLqRYd9aoffFCxu0V5cY1pAg+F0L0KFFoLRSriBGaXFBKkaRaCMdcI7J9qa8zqlKjQVMVyJHqhW0pOFFYq7G60FM4UIqZEpZiOJ5blnNFjpnVJqBywmpNb1vWTwObF3B42AATYmqcr4RAbhx5KKTRo40BXTtfo+r3VUpoXcdmPYBOzGYFpQvaGIZdtdKNY4EGrFO4kBi21bVgW4sziZgEnwLOFErOiHGoXWHUkc4WNj7QjY6nU+DuqWG8qHhUVKQoR1l72sMF16sbRDq8mZhEaLsjyuaajEIlhQ/COA+QFToVGn2I11dsLxrCYLAOxDRkhN5AmKrQ1JRKHYw+YE2mMS1GF7brjDGepmmIORNywLal8pawqCKsh5Gut+xuRkoQbG8wLaQciWvBjpaZK5hFguNQwVijAmVQ80wxGRkNfjWxfZZIopm9YnFzoUTFPqmbm41nM9UEu1QsJQtGjcSdYrWNdAvHrXuW80c7Ysy0fWE20yij6UQwfUCrRDNZrNecv4jMdcdmHRmvE++8s2Rxpnnx4QaVPQ8ebLi6FDSB3AonhzOy3nH+fKL0mqvViITIr3295+Lqhs004/ZSMYXKZ1gshctr+OZbDZ+eX7LeKqYwcGAN19PI7cVdHj5/TmsK253w4lzj1I7zceJ732+4/6aw2WaePw0IgbvHc6zR+GkiDuC3I8eHjvlSsQuBvlFoE8jRM+86VBG0dvgsDL4wO1SgEkprRMUq6ksZpy0xVd+7bepkZrv1uKal6Q2FSJGMVvXnUHJE24aYMjGGPfXU1NUigqLU6YQWVFF85UDzP/hmhr+IfLhy9E5onSKpjvUojGMgj4rSGULO+FIFqa4R7jVzbq4LN+sBpR0HM0FKhJhpi0M1qeqXSovK0NkZWjr8mPA7jw8Lrq5aPvrhlj/7zsizhxNprKLI5DIl5cpm0ZamtbROY0aNyVCCR9muYtxDIEqka1rOTk+5fl5Y7W6IuYbz+QgkQRkHMqEdjFPBP0iEm4nbr8/o7zuYe0IInF8Fpq7lOGlmc0tWge1u+tkWA6KE5qgwbS1Pf/CCN9475vEIp4s7PHn/O3zR3uKwOePJ6hnH3YJnz5/Qdh3OZLwKyOQY4watIaZM0xSMg3GYUHGBMYZxnLCmweoOaRJ926KtkImUIng/sN1tmMYNRReM6bGlXjBBQ4hTvdSkoWRFYx1GWxpt6Mw5lzc/QCtL342E+DFjeE4qPSFuad0Rl+uP+fDR53T9m7x99w4mP+bhxce8/+CGTt3m7Vfuc7G94iqOHPaHfPtr75HChsdXz5mC4nA+JzaazIRIQ2OWOCcs+gXH8yNs01BsZhpuEMmcry5x1iNtQ+48J4sTUqzqdtEdSrf0zZKWhkcPL/jH/6ff4d/+Zz9ATZnFXLPdjLSxRasKBHJSyGVAikfrFgZD8UIIO1RryVYzleohL0Whq02VQTJW6f3lC6gaJFOkrgdsFKi6eoyxGFN9w0hGSt233zoc+euvbrijXlAaxxgmsrKYGjO3F8EpGq1plEF8oUiqo0VqumHJpY7FlFAk1xGXzvT9XjBWFDGNFMnV4ldApOYpJDSogtMVruRiwlpFHzLrkEhSCWVaVaiOM4aZqtZGKGhJWJ1xWugQ2mxwxdBY6Gx9HWMGmgr8KSSU0aSrwhefwdHrmbner1K0Q4aItFXfkBOM04RxDmsM3k81WElblGSapmGz9qxCpm0FZSOuqRY4XRS77UTfGZpWEbww7ur6xjqFkXqBZl/waURMj1GKmxcr5ndndRqiQQ3w7GrL7fmC6y8yh1/tKaowbSf68ZD5QSS8GJiftZz7LVPYkncJ1QmDV/THQpZMHkBmAX244PqhZT1VAp2Q0BJwM0vfFh7fCLGMLNWcNGVSymhVgUfZC4lEf2DwwROLRZwnJY0oRV8Mu+uI6oTNKjANmeWRQ9uEREijQFQ0LqIOQZ9omAziFalXlNmESYF0YRjOYdwausOG/k4h60T2lWeBSTx/mlgPBu0yYQg8fjbwzHc0DpzKWDE0TaTtIE0J6xKzpabp6/i76zXKaooUshaubxKbTabXiU6gu6NJpvDsPHH1EM6s4quvzvjQex5fBYi1wDkS4fyF8PB8hVoZjmYGtSj4xnGzuuF0prlZCy8GcDnhmsynHw+ofoGd3fDZ08Tf+JbBvIBHTz3OaQ6ONJ898ty7bfjFd1sOF4UXLwJHN5q3b2nixhGzcPFsw/07C27NNTSZ1XpCbKR3ji6DbRStazA6Vf2TU6RQHS43O88t1WIkVRypFKyqQmTItI0ipgll6krAWkcpmTDun2HCfkxo93qlhHZ1dTLlggr1XNK2rp9LDjUwTddpwZtLx3/nbcW/+EgxlJbceYKOHC0c20lxsYpcrwuLrOl7U9fAQYMNdAcRf51ZryKCYt472lajFTgRXCmoqCmxZRw01+uRq8sdo1/T2xsOmzu89eZrFE74I/kOLx5dkMY6XclSiYC7XSJUGAZIdZN5X8W1OaVqi6koL1IKjNNIiLnqhsoelGUVhXpuW1tpr0VF1muYfuxZXDScveFYvGIZGbjYbSArvBiKAj/9jFMLcSNqpzBdy8cPD7i6Omf22oL27jF3+8zq6iH3v/Itnj/9PqNrOV2ecbl5Sk6eq8vHnM7voqT6u4OPaKXxfqJg2G0yd88WHM6OGMYRpRTOWULyeF9xlapkNnFNsYVjd8hBa8isGKYrNkMAexvUhlIcrTuoytLosThaG9gOnxAIaBW53v0Fg9/UJMCSmchshyfkCc5OT8Ce82j7GXiF9y2/+e23OTtZMpQd680ZNs04nLWsthdcXSW+/spXcG0k5ZHd+pSYOtrGMe/nOFXYjivW43PER/p5T5ARY4V536HMITlHuvkxMUTa7g4zN6OdOXZXgg6ef/J/+Wf8s3/yp0wbaAyoVu0FmpB9RJmI1ZokipgzrklIGIlRoVwHkrCqIElhcyQLiDYkBbkIrRJEakDIy8Wf01JDLigU2ccTl739UOf6Ac6BKYIZJ759f8cb/QriuI+MLjhd95emOsWx2mCl7uOVqcmUak/Co1Tfu0g1xZac0SXhFGQMYQI/TXhStaj3jqISKgo6V3GXKMXM2LpvD4EuZNoSaUWIAlhVM76lAocKiilmYowv8eY0YmhFo2Mml0BUQsyGKQhDyGQjOGuIPrLeDUwkPvx+4fZ7Lf2bCTMZUhzrDnTIiLJoWzAJcsx11E3DNHqSKeg9OaztHNtNIoyZflYP0poNkMhJsdskuhb63qFyZrf1xEEwvcIqy2Y3UbKh5FgxxB6uLhO3Tw1bHznQwsUVxD4gMbP6NHHwmqE/OiBsLpAjd3k2AAEAAElEQVS+Q4xGk6G1uGT4i2cbTo7qz0SU4vCO5clHA/fuzLnzVsuffSfy7i+fcv3sOepxD77QLjTRwE2ySNLc0pbVdodrFLaxhBjxo+L0nmVMmTgKYjKNVUhWqGLYrnboYtDZsFoF+mVXp1wZ4i5VEagRTJ8RrYnPwLSG0kWkTMiVYbjRbK8iSGJ23+FO6npCZ4duCmHyXF5lpiR0fSQFQwqORmd0W8E6u22mNSNOdVxcecZVpmsV84VFTKF10HYFXcAny7MXG6ZV4thYul5oZw4fC+vVjpt15tApmh5KueTO7Zbz5xO7S8Ph4Yi1iZNeMc00jy4NIXo2UQg+0ZqMtBrNjulSmK5n3D3T/Pj5yK2+QRcgan74SMgjDMMOkcyjJ5F3Xu2Iccv1jeOVk0KOmqurglEOWY3YXjN4gSnStQZUpDuxBBGObEuahrqaMoa21YSywzYOJZpIYr2JeN/Qzkx9xmpVTzFgJLNYNtUFkyLKGayrXUYJheA9KoFuKgdFlCJRA7m0rbqeHBJOV+2I5H0cNtS9uhS0gW/c0RwMii+uEsFoQk4EMzFax01TVz4rEcYgtBpuNZnWRhqVcbOW8+vAmEaW0TGPHlVGtrHl8abnwXXh82crrh5PtJI4OImUJjDOE1Iecm8BX3vzPtvVV/julLm5uKl24f0sI4fEtPP7Px+EEphKRfpro9FiMMbUwLL9VausYEpFgGutSHsiqNI1ideKAaNI2TMFyM8C4ypxZ9tx+PqM1CZS8qw3I6loYrY/22IgEzG9kAfNQOHTi47hIXzlWxOLXzhh2l1yfvmAZX/G06dPefPeaxzPzljtbpj8BuMyozegFbPeMY2erjugbxcczA8ZphW73UhRVUdQssIYRS6KFBNDTGRahJF5v0KVpzx98QWfPB44Ov46t84c09bS9wucc3g/olTdu4zek+ICZKCYKwaf8MlRJDOEgTHuM+oRdBr2hLoDilLMT0Cbpzxaf4wvlrapVfDjcUdQisUrpxR5xrPLNUv9Sxwc3GbwG7b+gmF8ymLWovoDzk5eI5cBrTVH9BXpqgrzbg5aSGWiXSxpm0O+970P+Nf/4k/5wXc/5uLJGoaMM9B3mmnvLzeloJXgVcQERci50vsiuGyRCbCBRjJd26F9roFDJJRWFAvRCF4pWtEoLSQFvtRbUfa58npPelMFyKYiisnkEgk5sRsnvnGq+LXbG3QaKVqR44goQ+cmQGFdHZ0bpVFlDw7Z++WKSKX3FfY+4j1itIAxGpUghMJ2HPG+dgTWqRqXa0C0ED11rWA1RYNOhbmu42y0oEXhgVRkH9GrYZ9nb1WhaDBJV9Z/gt4WFgtF2ypKUkwBhikyRkF1uu5DR4P2GlU802rio+84jk4Nx31GF01WBYJA3K8IrMXvItNUI5mRhvXG0xqHsYBEnBOmAdY3mbY1GKPJEhCt8GO9zJZzQz+ruerjpoazSAvKGLY3lU+gckA3lvV14fSo0InB28J8yjz5/JJXXr/P+osVy/uH4HdE1aEp6PuKaZ3R00R3YLj1c2c8/GDktVuZH33niuXha7zyKw3iR86/WPGtv3XA2UlC2xPOV1u2T9e4M0s61Lz1xj1+8C+es02OzThx+6DqS56dbzk4dOjOsLquE6pZlxBt0GIZr0eizxwfOq7ON3RdR98JacqUWAVXIjXkJm4VaV1wLRTnietC3hnGLQxDpjuAxasOZpFSMto5pikyXcNuBWNQHBw6ckpc7zIpTWByZRzEgusUs87SdZntlcaIol8K1kV0Z2h7wYTAzLU8eJaYJs1JD8dzC4tCVgm3bTAJ+mVg14A+6AhjJg0jb99veH65QYVDCi3z2ci42qG8ZxeEswPDNgqH/RE35xvOJ03XaXrleX4VuN7Aq6bFItx+vbDoDduVcH4VsaFwdmhpc6E3jo+fFV4xhdfu9Pz4RxuuhoHXDyzrm8CRbbh36nBmR6d6AgNgaNSWYgs+ZozV4ITeduhUMI0lE8mxsNp6+sMZioSRitJWVjBWI8ZjrMaUBqyQyojWdRmeU8CP0GSpkwNTkP2UUCno2gavEsNuQKj2XoOgVSHGjKgaaiRGwVFk+2JEj5pSBJ2hZU1P4ZWiiLYuRU/Fcr9paFwiImxC4NyMbIIn3CjON4aPn0e+9zjw408uefaFZ9zW9d2dV+eIPaY/KYRhxeUUIH7B7WPD179xinYdf/5n3+fy8TUp5T0qpNDYhNERVX6qaSql7O23iZQqNCHlBEpQqjI1cqnPs6AqOGiPIFeqgZBxWlFMZYxsV4UHH6zxo+PdX7iN6XdsJl/XMM3PGEecMORcx7pOZdo2AJmP/iKx6BfceReePfsu6rW/RWtbPn/ygFfvfRUbhCSR69UNr9z7Cp9+8gm29cy6jrOje1hnubp8SLQJozSlmCrSKEJMhVwCORfG6HECR7NEnp7w4OYBL3aJdc6cdYVb8xm+OWQME9O0Q2uLVhrRGfJE0Tu0HZmyEEvdbe58JkRHQqNLpCmKnBqmHGvVqSJTKKx3oNUCowJ+t2ErGqTFFGG9u0KKZzk7xurHXI+fUJodpk+Ma42176D0lqvhOdvtxNHi6yyXjqCvKCWz3mz55NNPef7kCesnjk+//4zHn7xg3FS6m3EK21fxm8o19jljqjCOiA3VMZFMJQVoKZQSasQ0ak/wKygJVVWvC0ap2j0CFiErsMYgWjHlQEyRVGKdHIhCKUtJBSFW3UGpS75cIKTIu7c1d9RIKZGUEk4K0qu689MCSdfYx1w94KKEPRNoH9Fb2dqVqpnruqAIShSiFXYM6AgNhqwMocAUKyuBmPGTkFNGiTDEiAoJk+r00VXFJi6XigUtqmKHAZ8zEuq4/vROw+JIo3RAk2iNqQ9lW7BdDdvZDImsA8ZCP2nMrjAfhJNpxnAVufykYf5zqSY9xgS9gkGIQ0DT0FjLbhoIPlfBZtBsp4HGOZwziIzYRvCDZdgIxiV60+zpbVJfw2Zi3jvm8x7iwO6qhhi0reP5boehxRmNl2rFffLMc+f1BbLxHN1fUjYd407Tv9ngtyPNIPhtoDGFsRjsXBE/L/xnf3nOr/83b3H3t1oefDJxfMvw/h9+wa/91++ytQsuVp6Hj0aez9Z88GnkW3/9lIe7HZtc2F05jucTs2bG6St36JY7nN4yDZW9fnSnZz2u2a4Vi6VFu6ooHVeBuIsc3jpgnAIGje18LQSCIZaI6yzGOvy0I0wZ3WqaQ8e08uyuI34QxBb624nZbQsttcpUws3lyOo5TAFoCwfHDlUSV9eRlAu6sTC1JJ9RKWJacJ0jl4j3ifmRMD8F43qkmareRFm2vvDs2Y42KxaHCjevrwEs487TzITe9ZhVYkiZeJNgm+nmhrMjyxcP1vRNQ3uQMMrw9jF89Nhz53TG+Srz5PGauYvcO2i5iRMXY8/zR3B2CEl7bh8Z7p8u6WSEVyznjwso4a07he2oKNlycLLm6gp2uxtMa7m6TNw7zRwuLMkHnCocz1y11apqeRNT14PNNtXGqlfENOCMpuRIJxo/FlYbzyldJWdqQVTBWoVrDUpHIIPYOu1ze1cYiWbRAoHV9RbnWtqZqXkeCBQDKdBai5obNruJ1W7CiNA4TeOEWDIlaWwpHDYwazKf3xSMUehskKLJNqGKgBeMgvMh88V6R1YZsmaIhcHDw2eav/g88sMHE8+fFsJNtcxbJ8yaCipbP/U80WvuG4s+6tiVQJ5G1PYLjvrCO28dc3l5RmTi+nxT+ShGs1gsOJwf8XzT0NBTSqzo5b1DK+dISYVxmmpqKBX0JloT9xxwVZ3fNUHUTlAKqhhiUiQSyQgpwIsnnrdez9jDJWN8DgaU5P/36/z/v2JAVMRkBcqTlCV5jRZFawrf+70Lfq05xb2z4uLqQ75y55s8ffJDnlw+4LWzt3ny7JxdmTi/+oI3775OZMd8phiHNddhwFiF63p03FFKi+hEToGEpQBNsmjWHNiC0wOfX3/CLsPNpeXyYovIhxxpRbRLxCwAhS6Vemb3XvbdNJBNJmVFypEUC6U4rDW0ew190eHLIJeiq/iMomm0JYfCLkDMDdrUqNhYArOux7Y9PuxY7S7QtqFkSxmFbilc+x+QSkCkxbiG65s/49FDz4uHEx9+f+TjD0Y2TzM5yl50XtBW6HqNNlV4F5KQKPgCee9GUAlKkYqQlVJTP1UV6+gItgCmRhDHXcbpTKvq6F+j0K0lKFAOtO0RDcmASRVxaXJBF7UHdwip1BG7dQXY1QcsFJpeeGMesUaBTZisyMbQCJRsaixpTmD3Xv/MPg93Dx2hsrVr6O5L2f2eLCZVbKNVxQojwiSFIUyMhZqYN2XGcSJKojUNKQrJVyCBFXBO0yqFK4Vdjqx8nTx0VlhkITWKo7cNr/+mYfZaqamBUq2gNSLMUIZMNcA7oni0KegsxKQhp4pR94ZxN6FXtQMSXfMD0II4TZ4SShW6zjHsIttdIHiF6hq2m5od4bqOIgmlIxKFtBO2ujCbKZKp2N88ac4vE50TZiealBVyIYRZ4t5Jz3bIbLYF7T3NouHihefWWeB663lxEXnz24dcPl9xfLgkXg0Ebfj4Dy/56q8do7OHecsbv3rKX25e8PyzLY//9Ibf/Eff4EEYWRbLi++vaN6a8cffWXEzGf7ePzzmT/7pJ2x3hd/++6fYmeV3/vkaY0Z+9W/f5Xt//ITf+Aev8NlPRj78sPDXf/s21x8+ZkwaVTL9vEbRpWtHvJlY3rW4Bm4utyznXXUDTBobFGINVoGUijru2ga3hDhGhueFMTjoE8v7itmthoQnS6GguHiYuHpcVy6mE5ZtgxXYbAIp68rUiJkSasKhbRuaFtq9Q+LwTYM5rimYlIT1lpwCWinOnyXiTnNwLLiuJlQ6Yxi2Cdcp5kvHNEbsQpiCx0fDRmtSUQwYOrWjc5abm0ibMieLjs1R4PLGElZbWhGyCAeHmu0LRbgJLFVgvjA0wVA2I9PNQHdiMQWOjgqj3zH5jr7TXN+scdnSt5FhLeSxslg2WXF3Juww2NIwt4rceDQtgpBsQZdClGoJ77QmaFDKICIo4n5l1hH9iJlrUhFODwTXF2hrgQ7UXJQikHUVC0pAsuDmipQcNxcTu0FoOodtM0olVFakmFFGOFw2eJ9YrQIX1xFjNU2rqkYsepRRLE+XPA+ZFQWrCo12OCm4AI6GLBByqm6ideHR08BHn0Z+8lHgycMJP6Rq3bWVf1PqkVMniiKEGJieXuP6ezRWY2eFcee5ZEQtn9C1nl/61qvMZ0d854//nO20I2FQRdHZBuUykjSiC0oJIrGCu2I9k1vnMMYQdYYkVRBc7/8adkSdmISoqg0TTS4TSsmeJwN+C9eXnsXrLRpPiDPKl6Dxn1ExkHP1pCtRZKn5yUWBMgplhQ/+MnJweogPn/LCvsXi6D02u8+4Xn3GcnGLF5tP2E2Wr771Jp9+8QEv1s9wjcMWjViDzl3lLLsJfMvcbsg8JzGnmIfoeIHPhcvLkSQN6w28uLjBaMdmFfmjH/+A27fe45237+F3I40RZu0EecXV+jnr7RrTA6ZiGnMKGBWQPBJJ1VsfBVEVDCRZk3wh54IvnkhC77nbSiW0KKzSFMlkb3BliVZLcoYQAilkdlvDsHWsV5nL88jzhyPPHnoun+7YXo3ksOfdK1MtUfWdBp3re1xkn1MtvHymoH5Ai5QvLScUVYFLe8BNzmXva4cgmSbvk/9EKl61zRjJYAxF710BquwvY1BKVeDwXgQj1DQ82b8+Re3eiy5Y03LUWbQZyU0VxOSianBQHKoKUdcPdt0DUA9e1JeWuwo2qt0Mam/fqQZk8Ip4XfAbTTCFYj2uQEYzlUIxBddYWqtf6peqwyFGcgGjLYvW0Dpwo7CbAmMuzIxCl0LuWw6/2dLe3pEuAqLqASOqrjJKzBW9y14wGSNytR9FygSpJRVBVGRmDUwRdlRokcpgqnNAVK4dQVFYJTSqgpNCCBTl2E2Z6D2zXqO1wlvIUfDTBq0t1kEJijSAM8L6wrN5IZy9U9iWkemyr/S5BvKYGLaODmhtx/nnAy+y4/c/GPmH3cDZ63NWL0bmVpF2hTtvLMhO0Z+2/NlfbvnWu46/+9+9xfDBNaXM+eT3P+a1X7nNeCX8P3/3Ob91r+WrX0388E9W/PjHiV/5hRM++OKSf/rPhL/9myd87d0Ff/gvP2GZzrj13iE/+rdXHN4yPH024GY9t1+f8b0/eM7b37yDURAvt2zHgcOzntkycXMzopWhKI0eFHYnKJXq5WIsYarI2yyw2yrGtadMQtMXuntCdwwpBfLckn3m6kPP1SNFtBrVZ9re0plCCQG1t5MGb8mpwZeA1pFZl5nNhNRG3K06pRlWmaY0zPtI2xoktUw3inVY0c6Epi0Y67C2pikqXZgvZM/z0Din0KMm7TzNDNaDR1Lh+MDhfUFi4t7tDhsLp4eWDz+74NBZLGCI2BQ5cJkyD3SHCtsl0jZwMjcwRjbnwu5my3zRsGg1m9XEZjNgDLhGQw4ErVj0dY26LTBTQrB1ndZ0haAVjdIkBUobTBGcKSAWcmY2bxh93u+5FcpEBM16m7m1rLZrcQkxrqKEobo8XjqW97t0xf4Mo9DNFVIsFy+2TN6hdoJtoGkUxkVSqK4AYyzLw5a2z2zXkeurTJFMY4RQMkNJ6GXDFB2dNiit8NkRGlsdRePE9nLg2cMtH/xw4IMPd1xdBmKsU8yZU1gp6KxIAkkqcwX2m8sijEPm80+fIXrJq19pkS4wjIGVtqj5huWtJXd2jlt3ljxNiTB5xnGFNgFrEkkH8Jms63kfY8IUhTaKg4MFfb/FD4GU92cxPz3/X+ICRKrFe/IT1gox7mFyUoXXu61HUFhtiUGhys+YQMg+lEQUiK0HdqHuK5QpZFuQdEDcXDCUp6xuZtxazLm6PueVewteXb7J3bM3+PTpB1yla7p5z+QLzrXoeEGjNlxMj9mMjpuba2bNOa0LFBTOBg7sAT47xHRkP3F5cU0qic5ZrNZsp8DNsGacRs5mLb32oIQpB4q+wcxg8oXodc1kByYSWhuUaEzSOHEYUZRciKl6ZwMBGws2WvIg+FHY7oTtJhNGQRVXffYpEr1is/Hshsh2k1ldjqyuC8MQqrc613GcFThYONIUST5VZWne0/9Ufa8LdfxfL9CfXlBS6iez/DuTH9n/mvrEFagWvJIQn+j7GvRSRNUEMNgHgFTvcNorVdNLH62yKBXrXktVx4ExthaEEhEpiHhAUDpzlSG7BpUKKVR/KyqR4oAxC4rir0TlVpEsJSN1VkaRn/55iqqfq5IFCYq0KQyXmWnjyY1FzRTW7fUBRLIV3Mwiksg+kBVgDWVSFF+qNUkgkPE5sxkSa59Q1tK3inAzcPNZ4OAtTdcIEitCuCbm6aqgDwmZFDpQkaunijyCjJai9yV7Eii2Fj4qQ8kodD3IMuhsa7SxjjgHuliGbSKPiSQJhWUYR1KI9LMGKVUcpUrDbp2xptBZg8dDKfRNy+rc8/j7mXtvz1F3Gjaf3+AWhrNFx0UYuVxvOTo9JnaZxfkV//3/8dt88gef8Uf/jwt++2+eMr3X0TVbnnlYsGMTet66b/n+f37OZVjyN//bdzA/+QgdW55/fM0bP9/zW7+xZPXwnJ//WyeU6xX/5B9/wf/0f/ZV5A8sH73/jP/Nf7Tmf/I/fJ2m6/m//WvP3/22xqJ55Ws9d5sbPviDJ/zyL89ZD+e0Zx0MOy6fRO7em2HmBb8rsMl0jUatEzpZYk7ILDGfW+q6pI6Ip2mEQdFpRzlM6CW0c0vWEXGK8SJw/pPI7qaitU2XkbaAjojWVeRVBJVqYRZT4nShODrSuAPBWIvRsUJfjEbdbqAp5KDZXQU2TycQxcFrFh7WiG3rEtpExjGzPHC4pj7jIoZprNZpa6krD6847hU5JiYMSjmOZhWaE4vhRZ+ZG4sh0PSGJhYOncPOJwgZIxYfFH1fd2IpgeoVQsRkwXWG3RiJVig6VcCYLsQY6RvYpoAqhr7N5JIwTlO0ptWGAY8yGivV3qt1ZT1Y7ZCm2uZEVypgYmTnGxQG19TfB+qYva4D5cuLX16eAQKoWEf5IdH0iuWh4+oqMQ6asK4TgfnC0TQ1+TAmS9gmREXamdC3hnGXuRk8IYDLgdejQtsOReWu5CmitgP2cg0v1vjnEf8wMT3JdJv6yE57XYTPUu2HSmNyrLXLvjmpyZ4Vvz2sE48+2+Fazb03O5TL7KYM+gYxwtnte/ytv/kL/OG//SGPPnvMctHSOIWQUQ7YRXKqk4eca/ZCKTWnRSSTYkBXV/f+jaLeu+zhcAJGavbClxe5UeScSAFitCjAYlj7gEo/Y81AfWfK/oeaKXpf+BVAZ7SJiJ1xPHuFm/UXLI6+wZOLJxwdNpRQ+NX3fp2ffPo9rtJj7Myh0hXPXnzAJh+wGb7gYG5ZWE/rOqKfeLwz5JQ5XTiWumU7s+ziNfNli0lC32pM39C0lZh2UBSJL9hebmi7BaNu2IQ1Pg2UlIk6UEzdLVsndf/sGxhdHQ96RVAKL4nd6BmHTApV0DWtetbnmvV1YLsVxgmGKbEdt6QSEVNhMqXU+OK8t8ihCsYqeldxmsRMngJpKJX1X8reglL2dr/9BwCpqX174azaD9LZ++XLl//fWg3+OwXB/tfHktGpmgNGD05VMYvs/f7T5ClmTwyziiw1QCjsCwKg5gEoqXoCqf+slEU0NG2DtoK1HRcF1kPhYD8+L6pAU8Bqiq/rhJwKJZb9dGlfAJCrlZC9oBD2/31fHIyZuKkj4cYoUsrEEZSzaBtxuq56KJXhn8mopiYUSq4hVCCVNLjxbMaMTwqfC5tYORRZCucfwvwtzd2vgR6FIook5aeUMF0v9pwV8SYgTaF0gj7IqJDJoVCPzD3XP5YakZwzqlhEq9papBoGRK7WzLbVlcW/m5iKB9uwmzzTxcTBrEOZ6rcmWaZRo7pC4xR+SlgTaPrCuO559MMNh68n5ncbdjee5a2OOweaZ5+NbM7X3P+NY/K7Cx786QXv/uabHM2fsL264U7SlLnj44/WvPNOQ/p8zfGv3uX5teeTPwz45ytuvbvkZl1I14rf+acv+Ed/7w7/i/9kzau/cMZrP9/z91zHH//Tn/Cb/+Ad1LTmkyc7Pn6w5m///Vs8/fCacT1wGVt+pYu8cscQpoGrmxlnRw0Xn1/zxlcXHE4tuoOyyfghI8mhgieEmkSoD6BZWgTNsBoZPBWJXKoFrPQaWSr6s4S0EJVi+9hz8cNMjAq9KATJONE4yfSd0HYav1FILDQ9HJ4qumVBtXV/q7Ora7pGka0hFmHYBraPItvLTPbguo5d2TAi6MbgqGl9MXhcA8fHDTFALIUpRapOLGMbYUyK2cwiJjCNCnGJ2cJwOGvA7ziZW/p2wiTPshP6uWEcIlZZjmYNeVfBbV0jzGeamKd939DVCVQ25BJxqmWSSLB17592mt3K07aaTUpceeH4UOrESxtcU91uJtXu1ViFSjBzhckLccr0R4bdEFGm7haVC/gIpnTMF7uq+M9xb9v9K8fTvh94OREsKiFFVw1PjMyWhlTgelXBW0+f76AolgctfZ/RbiIXIWVFUdA0gdZG+l4xTMLTnebaa8J6wk6BZjMwuw64VSTvAt5H8iTcdrBd1HPn8QA+VliSolByIu+FfFoLWfbnr9RwLyO1YNrcRB5+tKVtjji8KwR24KGdBk5OIycHp7y4vE/0G/q2QeW6bsg51ERO1H7EC0WkJj0aqeLKIijZv0n7812ktnn71OLaKKL3v08k54QCUgFrNEonRBWM8ijd/myLAa0rTlZM7SpfXlIikFMh5jWTZF45PeP56nO2N2uM6Xh+fc5vfes/4Pn5F6yHR3x482esH274yu0DnJuQ7YDsLNdDYTvrQXlaYzjrG5at0No62h2SJxvh/Hpk2kWWhwrTGg5mR+xuVrTOcbxsKD4wDFdsU6w7f632K4Aea0BFi79RrG4Sz5/ueHH+lM1uIAWFtYJxEGLGj4HeGmZ2Th4cu62QsqFtFKaJ2ARNaYjUN7rkVC01uSEGRQiJTKAQKzvAlzpuDrqO/yn7vflPL949pq56sfeXshKpKOBUC4acy18pGl5+lS//KvufTf0o1SnBLhSSSWRJxKiQEbJolCpoEoIDZerrzNXbqjVoVUeGylbRj9YapTXGKNq+RTuLto4wJcaUWeiAShOpCCootDIwFEKM5Jxr+JjW9XKUWg3nnNBJvnzFqghSNBIUcVvw2wSi0bYWXKkU4hhAV7fJGALDKEQfafdY6Zhj1TgQKFoYJ2G9LWymjNKWVglRFEMSii1cPx6Y/nNDd+o4OsrgDcUIQqyVuW1IqkZmG+MoO6GMmdIFiqvTFYmQYwTtEVOFiuQKTKkqYX463ZFKSBSfsUHRNwZ2ddyoiyGlxM1qonGGzmpEJYZU2G0zujcoK4zbhDSKPGxRk2H70PPZTeLwtGOSwq33HHeOAp/9fka2Adk1zGxkfHDFe984ZDc40qrQjPB3f6shbCa2YYla97zzVstX7yc++P4Ncvs24p+yeO+A1+YnXGP5R3//mOHBiodj5jd++YT/+b8+J//eNW/fM/zCewu++70bDmaaX/7VU558csPlky15ajl4bck//r8+4R8edsyOZ+yuYfu84qO3m4l5tyCGHWUKlK1mMwjNIjGba1qnmVaRcSekVJAEGEM4ADkIzI4t0oIPE9efZW4+KeRiSItMptC2lRnROmHR2QqOksSt2wbmmZgK400i3UTapUHPMlEKYVcY1pHLy8BuLDVYq2mYHxm224AfNNFbtEpoOyLKYtAsDjSzvrBbq7qr9tWH77oKcFIq41rIuQplUZ7lQcO8cwRJZIGm0bigaFtwBnTjKFYQS52WFKHt6g5/n+yL0hbRCSl1UpBSQjeKqDKlCJPTNJ1mizCz8DjAkS6YIkCD6yZEFZpkmErBNIocDEoFZkvLdi2IFJpWEfcOoSgB0ZGrVeTgftVSyJejy2qyeznxhCoOLwhkVUWHItVSXBKLZSVIKlUYQuL8aWbcZkSFve3ZVqtdA37hWItDsmEahSEa5r7w9s2O2TrCAMlHpgxeWZw1zFPmVhtJfSb5zBBhQEhRk1Mildro5v0NVy/gUlPu854vIgpJhdXlxMOPN2i3YH5HCCkwTJ7z1ecczTJvvrFkHF+lb6vVXVuFKuAzUHJdJydFJEFT3x+ja1JqPeCl1k8i+9fxcm28X2HuVwmyB8OlmJktDK9/ZUYzV5gi9PwV3cZ/ydf/D5OB8qVSVIuQVCWMQUFsIsWOwXuCbjnsT3j8/DO6k9egOELastpe8fTyfT7+7HNuv3qXp1eRtm/IacWt45b+wLAehSePAtFqQgebLSxmjs04cmc5QyFMfodpe7SKtErQo+es75h1DVebgZQEKz22RFKB7ZAYpsg0DsSsSZMijprdOrC+WTONI8FnfKrRK62tpZftDK5t6ZoFuekpRjFNNe7ViCYIqBzruH3/IU9JyKWgVMQaoNgKRoyenECiosSMpIgqHlT1kkcp+zXBl3di/VuBlPc7q8yXBUH9ANSq++XlX62AL4vv+vdEQbJQqESwQKk+6lgFKcbX1xqLYFSproQU0FbVi19JDXAyIKrgbK1ctTU0ncK42oV/fmP50cpgzgInNmN1nQLkoMkx46caCKKsQWn5qd6h1MetdsxVrChZIVGR14VpnRiGRIyZYhXFQOMUndX4BLtUo5VTKrXo04aYE6Ovv8YJjFLIFlJrSJLIORAjjDGy20VCFLwqDB8PmN8Vfu4/sBwuI2rKVSMgQtIZZgWuQ53+dJq8i+iVq0CQpoDLiKlUO+UVRQnFSt3zRb6cHOUU6p/ZNUjRYCsvorENbCISE5MWppjxQ4BO0zQtWm8Zt4V1yPSdrpQ9pVHWM+0Ktu9QYeLqcsWTn3QU33N023L6cwvUvLC58rz51TlX33vO9RIYO9rxki9uGu68OWP9kwlnI3x6zoOrAd9ZvvKtW/zxH11wljQnr2vu3hr5/M/X3Pn2nD/9ned8+ETzK1/zfOtVw61DxdGtjq+9NvJHY4++iDz6ZMfRvTkf/BiiHLOaPuL+ieH0/ozVOvDkzy94++vHvHioaPKW5UIRBijrRJgy7qDl4EAzaxVpKkzrggSF5EBSGZoG1SX6Hkzn2d5kLj8Sdk8FaTWhLxgMTme0DehO0c8tnc2kXJCFxsfM5tPEOEJzIizuNWglbK4y19ee60thGmoTpDtTaYm7wOZ6wtePCBIixkHTOnKq++7jU4cKmSCRXchkFMbW83KYBOcE22RKrMUeTlgeGlSZsMaRZMSohLPgbM05FQOqKdhO4WONaHZtJCdL53oSHmVytbKmQozCOCToFLsEJQvWJNpec7NNzJ3mOkVWHuZOVfKig6QzXdKE4CsPRGt83NEuGnyAECbauSHtEmIEjcVaxcVqw/LccXbWUHJVvLMfsX9ZELw8r1CAgxIpeZ9JXaqmaTmv7qLtbs603DFNAR8LMWUoE7mAHzVXRZG6jpOQON1OLLcTszERQ2IXAxOKrAtaNC5Tk1BzYoGQG800z6yAzS5Vimra669S1Tnkl5cudV8v5mVMcrW2ppR59viG2YFmeTJD2Q3TOLLVCaOfcHLS8/PfepN7h3cRHSkyYhSgLKR9g6Sq6y3myG43EoLHGlWbx3rCI2T0fjKrVS0ec0p1dZJruFHJGdcqvv5Lt3j1Pcd1HCEbrGrxw89YMyAq7Wn1++6GvPeDViGWMgqRSwJw+/CA65sLLi+fcP/kDa7XA8+ff8ZisaBrDVcvRsa+RcaBow6GaIllIGTLK7dnHM4tQXtyUGRvq1+2ydxcB+atxfVU0EzQzJseZzPr6xuG7InF4deRycPlcMN28sSsyQZas6ThAKNauhZSbAgxYXOElDDKQsnElPG+KvCLCyQ8EUOUREFQtuokCLFO8rPULpCESCSGiDaVS2+0JisFOlF0qA++EgQLkkikOnVJhcreqal3ShQp18s+7keiXwoG/52ZG//Ov+/rcGAf0kTBUYgIkgpSBFMqr51QYRexRIquXXoioqytISKaL/0s1hoaZ1Ba4xpbbUOmjtF/soLFFyN3DoSTmaUQIAViNIx5wlqFdhrlKu2PVDUB7IsSSarmj5d9bO4WpuvIsKmhIMmAsoH50tF3FiUVSawmKKFQWsiqPrXjuqYUiikgmrgvXNUC+tYyDolhqu/pbqw6j1YUrbV88f5EzMLX/07H2elE8RkVQaeJrCE7g1ortMkoJ4RtoAwK5RTSgbSCam3NVp8Sst0XZntfkBINqj7oaYpIKsQs1DFMxloFCeIYMChChuurQNtomn1nu76aSLvCwYElel//TDbz4mrgnb9+gj5qkdgzmwr/+p8/5itfPyDMGg5ez+yGLentGX2O7Mgoa5gnhW0D653hwQvPL7wb2G0TH393xZ2/3fHGG4b1BzuePNzxla8f8sf/yY+578+4+/qM27c0l8ly775l9JE/+Z7nv/cP7hLDZ7x2e87j54pXvia4PvHdP7ng3bfu8OP2CZ0SunvCjz/IbELi+Mzz2fuBcLmiUzVS2rSK2clE27SQNXEdyAMMPhFsgrmi62E+N6glXD/ynP+w4EdgkaApNFpjTUY3Ftsquj7RGIPfFVLKbMdKn2tmhuOvKOwctheZmyeJy+vCdqq7b60jOmvStpAl1NWWLhRlMCPYnGlaIRcBpVkcQdcV4pRpbE0tNba6cnIQlE5Yu9+Dx4hzCmug76XGWceMM8LMaRqE1pqK3S4J0xRcY9BNodMGbffiZylgociE0Q2qGDabCWfBWAHRpFRwNmGNIm3LPvHPczE0nDZCDAFlqAVqoOZ5SEG5DKVC8pt5ZpwEZcA1VR/jtIKiSE3m4ZMty1mHa1QFmeUaACaZmtQHdf2572pzqfAxcvky6loLHCw0WQrbbeV8mEZRYqFkjdEKUyKtH1jEzO0xo8ZICJoQM1OCIgal6nltikYXQymJyYRKv5XIKHAdFCtf48uv92FyJAPEShNF6lpzr7EqUvHrld5exdrX557h6oilnRHThp0RtFlzODvn9q2e5dzgyw2qVKGfDx4rFcRWlK5CwhAIMVB3SfX7VF1XwajqijN/ZVog2VBKREkl+hqbeffnD/jat+8wqhcM402NCQ+a8893P+NiQCsgo0XtnQSZtmvodIOi0PaO0xOY9yO2nTg+Ujz4+AGHb3+DH/zwfZ7Ea751/xZCz7KZcWobXkwX7OhpF5ljO+dmE5g1msYmWqMIBXKpUb+bTWA5a2i7QkmGkjSKyDSt2QwZxZKZuo02t9mqLVcXP2JbQiXGYbHKErxwfeP33nJFzhoRTVZVbS8ZTC64UjtqiZmcfCXRUchaIWIIkgiiiMZUf7JSkCtkokTQzqIKJMmUnKrXPksNeDGAZEpKFXKjm0qn0pkcyx5HXEd8ubycCNQLX+0Fd//Oz4VqeynsbXvycocgNV9gXxSoUjW8OdVxEpJgALQhESimIjHFFZQWxEotaIxgjKExGm0qztcYUyNj99+uF+HusuPMBQgTOU+oWAV3LRrd2arOJ+9BGlJXJKFqJ0SrvZiLmi+/TQybgA8FjEY74eDAMV/avby2oFImpEIIimQUURd2u8QUq/jPukrvKqEmHopO9MqgsqnLGx8IvlpuiosMuhAnzaffC/jryHv/Fcu9NxuULRWTGzLi9r71baDouqaKkhi3HrYVrmTajHIF0wi5DaRBMa0iRIVzTRVXpgJZE/z+Z+5jXQfpQsgZHQSbSxUvAuv1jmkyzDtFZ2eMq4jkDX3Xokzh9jun/PgPzxleJA4PFeuna5rjjld/7pRmvEEnzYf/dsd913PwXsvwxYqtseySoSmBqx8nbr3RcRky6nVNez3x8+8e8tmjLU9uPK/2GusVLy4Sp2d1+tEetlw/vuD9v4jcsTPaRc8ff/ecv/z+ksNGcetOy3p4wfjsFgOeMNT1kymJD39wztd+6YhupglXK0yf8CFz/JUZw0VhN2TunQqdzigNfihM14lhm9iaBDNd1fSzhHSW5z8JXD/weKPQM6FRVfWqm4Tpq8q/c4Lyit12qITMwRARju61zG7VMf6znySeP4dhUpBHnNScg6SEUAoajVGmBkoR2O48y9zQNRptoJQGYwKzhVB8IU6BxvQ4B5InpFgIgm3iHiKj0L0CMUiTcC2krDB9Jo+1dzZKoZVglKA06AacK9i+xm0rY0hpQhuNcgbXC34sOKMJwdJawTpBa8N2mmgaxRyDspGhwEEH6yRscyEGj1IOpQM5lX20cEKZjLKOFBKuK4SoaqR2J8RUBYs5G6QTxknz8NGGN95oMS+bxlw1TyrWQDT2iaVQkFLzUrTda54CSFFAZL6A4zPLZvA1vMwayv77IRGVDcV7hmjQRZhydUCJUpAUVoTGZqzOCAFfajw8UnBacaAtd3NhTWZdEqtQkdlCJpeK+TZq732QhOwdWUXVWHVyXSNcvtjw4EPHW2bJ/LRnmhKNzayGFxwsF9jFyC7uyGPFcucS6jrJGmKUKirVmvnM0XYja6a6gqdUXowWrN5PiFOdkEjp9u87GA1vv7fk69++TW42XG1Hgk/kyXHxyPPxDzY/42Jgf+EorcFAt+h4+/br3Jl39Ac7it4ymwfm+prSbJnPz1B6xoTnxfMVP3z+BQf969w7vs/5xWMu9RbVGlSEm9VAToneVGqVaxpCSOy2I8615Elx2Nm6S9kVpm0g5UjGos0JbXeXw8M3sP0pqUz4F+8jOaKKrurykile2F4aLi53SB45WegaGKMKqihcqQ+bsgoteh81CrFoAgqv9qxsFUByrQ6VRlOqwCSXKtiwem/DrOS72u1mJKmK290Hb+SsKulQGWLIxJiQmGtn4Cuet+w1BXWfVrUaVWBXv166AtJLscmXUZV1jJBzFRQa6ojwpeJYQp3xpFzHfBRNcQUaoW8butahyC9rCjCRXCwpg9ax7tdUS+tqMfDNk8h/9a0Nh+WarFXleRcLrtrg0IVSUn3NWchJyKHij5UWilJIFNiBv87s1jUHoDRgTKFdZGaLFlpNNqqKndYJSUIq9b2LXhNDIZJqEeV1Lb5KwfuAzploLaPOJO3331eI+7jQXAojEdsYnj0X/O96tr9YuP/zVaBlcgEmSq8gadKmUAiYRqPnLbtNYNwUZL0/tLuE6TW608yUJdxkxusqIlO2HvBCpahlyXX9IwrXaKL3MIGOiqCqPXK9CYSpcNRqOlfYXBbKMtM0mYMhc3xk2U6B7qLF9sJmGLn9SgPrY2SeuLdbMD2YOP/ulrO7ilvv9Pyz/8MVry8TJ+8e0Dxe0+8yL/4Ibn3rmKfvn3N2esD6YiTpwhQ87eC43XcMKoLtCcsL8tMN/RtnXOSJuweWf/67L/gf/Yf3uNgO/Nn3Rn79Kw1vz2Hl4c+/d8X5OrP6zNPfnrh1q2NVBH/uaGcZdaDoOouLE+rJjmIPUE0iXCYuN5l1jNiDCipqVSDtFM8+mbi4iOheV0ErGusKqim4VtM1YG0hDYIfFTkZEol2qZnfB/rI1Xnm6cPIelWLUisZsFWMq+v0U1TBNBnbGIKv9McwKrDVqlsKIIn5HBpRDNuMM5YQEjF7dNMQUkJCorWWolINs7ECkmk7qYJRhLZN+KjRtqrExUK194JYg3EaG+sEMeWEsQ50wTYKozXSBrQOWFsntlZndBZSL6i15sBoGpf5Yqe5c6LZxch5UGyi5SQoipnALGgkECWBWMRmUq4oaNMGfFTouYJNQqmWrApZZVSneXLpaV3DvXsOpTeULJTS1Gal1CnGS71Zoepp6vrzpSsKNAZX4HAReWz3qN5UqY4CSKqNgTeZdcrMBoXk2mhoAXTCKUWDrgmmdq+gMpmiFY1tmE+F09YzzTXrWWT1bMRu63plyKk2G0XV7rxUQJvw0i2keDmuTRkef76m7XveXMwwsw17RBs34wV68xNyUQxTTV/VZp+ImBMlJVJlNwN1jQiC1vU/tVZqwmEuTKnGfKeyJ87qKpx+4805v/gbr2CPI+erc/yQIViun0Z+/L0t4+pn7Cb4ez93yhgVOw9rnZmd3eLdV77J7d4g6nNG9WO2ccRnS2SkpMxJd5fGzjm//AGtLbz/0VN+4c0zFv2GzbRDNZnFLehvDDtf0AvLbOnojCXvCifukGGYKE6YxlD3x2WGtrdp+1Ncd4RzJ7RuiWnAEJAUMboD1xPHayQXVDKsb4TrK8/NdcRay8G8w2pVu2Al++o11y5b9vwEXfaVJhVClFONAlbly2JBmZrjTWFf9VLxuntrrdKQjUJRP6wlVotPtY5U5ai4goqRNO1RwUDOqe4jRWox89I6+LLd31/5ef8P/x8akQJ7XAVZ6sQfFCXX6rTISzdCRV0qKzR9S9s6jLV750HtKCiKkAKUhiSakiJajwwx8voS/s5XPGdmjfhIQtfipK0MAARK2I++MpRUrZtqn0EgRZCkKFMhDIlxSsR912FdvRybmYEWst07EgbFeCWcX0Yuk6o6PQWhbtfwsSYL1vZLMSZqyRZgDIVhqrs/UbXyj7EWoX1r0A5sKwwF3v8vIk8fGd7+duDeGwpLDf5QXcWg5kkYx3rw0O0/R2PGx0K+UZgbg2ojzQzsQuhaRbgRwrbCT/aWh3rgI8SYUVrTupZxyuQIaprAJFqdGHeFqylxeNCgvWJ17bG9wT0aeeVdx+f/xTW7LyyihOfryJ1Tgyw0zz6NvPZay8mblg9+3/PBjwJ/twz8rf9Gw/bCcf7RwNnXWn7vXz3n1VuO9k3DDR3HqiCpYJuWYRr54I+uquJdCg+/d06aCdc3AS/V9ncG/OLfOOGLRxOffbLGK0PvM2qhef/9LbePHccnsDhsOX+y4y++v+M3f+sWFy8Ct99q6PrCVRTGUUiT4XgGw0Xg8vnEucp0J4bjeUMOivNLz/rck0rGLOoay9mCdtB0hrYv2FawCtJaVdVWyZhecXCvQR97xmHi2UfCsydSIWoqYjS0jaU1wlAiu1RDY9ysICYSp4TfGaapRUi0TdXWCNV10rWOvAto0Yi2rDZbPBWuRcm4vgZ+lVS7buMMuWS6po61lRJca2EsWKOwRmGUYETVgDCtUGqfsJjrhWqdIUtEmYKx9XOoddUiWFMQU+rFIgpna27AohM+eybc2jbc6ycGDavBE6aGZu72DiRB6SqANFaRQnU4GacJPmCzIRsh6lTPp1jpf9nBg2fXNP0pZ2d6r9SHkDM1E92SJGGL/JXjquoLhKp/eumWss7g3ETyNR+jCg6p2TMlkjSMElBB0aW6cjClhpxpFDZTp6CqXnVFO+KtU4bbd0k5Mt8NvJYU4SoxffCcz39yznjjSaGQc52AlFRIf2U7+1JD8FJGr/aTxhdPrzi5qzlrW7REKJr1dmA3PQSpGSG5OELy5PxyhZIQZYil7AOKBGOl6kBsQqVELJGkNNtRoXSiLq88kuH4rOXnfu0uh3cUz6+uSWOkFMNwrvjo+5dsL8vemfBf/vXvXQz8h7+omVQiZ8vNbsnjOKc9dDRdIIw7ZBCKDTWeOBuuUqFpIYzPuRwusEcL7s4dzy52fP3Vr/Lg8fewodCjKCpwcrDg9NBCiGxWiVbPGdLAZgyEjaVp76DbJfPlfVxzTEERcyCkQN6cEzYThwczendIqxtaU3AmId4wbTTrG8VuCzkbtHYYY2tQhKkFAXEv1yj7kfve1aG1VHBMpiaTUTMFSqqrBKMUqtT9NFn2lo9a4SpqZR9L/T1KzvUQQO09pYqc68hbgqCtkEIFaaSSvhz9v1wXSP5Sj1sPlJcVx//Xr/1eLlPtfrAHDymkEUxvMV31DevG0Cw6TKuxjcVpva8nFDl5UnEkyYQSaFVhUpGF1fyNtwa+crxG4haUQpn9mG8fsoOvBLiSqTHF+z2c7NuuAnu+QkE5aBaCnStE16Qy4/Y7eamWzTzCdFl4/sjz+GJiqw1Nr8EZkpEq8AoRgyDGopTCOPCh2j4lZgi1eup7xeQzl5tM8bCcUbMYxoxuHe72gk8fbfjkoy2/9DdnvPctg9MBkseZ6kmOu0Txioym6AxNJRLqWPeAfg3TFqzLNH3BzBWqFeJKETdCSrEWlcqgnau40ZywqRBzoMEwTTXYa9bAOAmXF5GDhWPaRoabxK7bMhuX3HjN6asFddhyqFumyy3li4k7C0U5T2yD8PO3M8+OFJ+9v+ayg/e+NuPwNcXH33mB7QvrrFh9dM3F1QvuvPIW995u6eYt158W5GDH5ap2SjOz497xbYjXfP5i4vXc8vk68yuvJj76wUSIkTuzzL1+5C8+bVB5i2sW/OpvHiNmztNPbthu4fDYkmaF2WHhgz9ZkQ8KMzUye6XDu5FPLiLX68zhK5rZXHE1DmwuhGkTaNqGbp4xRmhdonEKt9S4RaabGXQWhheBzVUkSsLdEeavW2I3cfUo8eQBXK8iiMJqTSOK03nVw9yMEZ8Vrq0TgSTgR0WZMilmpijMGoW1fFmRz3qNkAihjpN3PrNLCZxGklSMr65niKQ6Qcy60LmaORBjwjZVjxRzwpp6flj9V55bVTBUTDmxeoxtI6QCxhaMq2eZ0ZUt0DhNkoRuC00WGgdRRRadA/F898nE9kSYzwvNTebVbUYfKdClHon7cxBVg8RyqgVMCB6VCl3v2IYRozUKU4t7B5vJ8ic/vOavqRnHJwo9JYzeo8xjIIqQ9V+5XfdddpVFSR3nl4LRhr6rjiCjFUXSfmdfULm6kqISfBSWHmwuFW+uFUoqNMlrz6537NySqZmzPb3H9ug+u2IgjJgSOLm+5vUMm82WZ5NHZ4VO7PVaVdydpAqC91FqlU64dwkqrVitRp5+sWNxcEg3S6Si8T4xrlcUZQiTIfhMHAeSmPozypFcFEV3GGvomoat3vASzFSTWgvTPqtARIh70frBQcMv/uprnLyhGdKWGCbKVLg61zz6/pqbJxktVf/w7/P1710MbLcXRGVRWFKAWXeCqBfE8JgxPcSb6s9MqtCVhu1Gs1jeZXe5JgVwu0C6M+OoFWbzlrtnp1izg10htDNOlh3xOjJuE6kknoxrkj5gefhNDto76P6MYZoYCqx3nuA9FLDK4VRCwgMuxpFbh++RS4OODS5bQrIMO0vwGgEaB21bkcJKGayxWJ2YJP70YpWXVhj2YB5B9F4ZK9SOUguSa5BPZervO/p9XVC7+YxoUFIBJyXXv5SqFTSiSDlDTOgsEKSmEGqDsQU/JpLPxCREv/fqZ6mX+0/LaF6uBX5aKFAnGvt/VfsHWqTu0bVRVVTVW5qZo5lZbN9ie0fRNTHSGEOMkRA8JEUkE2xAxUy2VTH/rbuKX39lgysTFINYhahUhT6pkIKQvSGVhFZ7YAaVrJfiflSohaIz2gmuEWza245eimWUIquCjBm1E4aryMVzz8V1ZByFrDNZgUEgFXSuCYHGGZQ1xBTxMeJjRheh0XqPNE5gConEECNRFGUaYcyoYnDGc+u45fCNBU/f9/zef7rl+bOWX/qNhtM+kYPgBGwHwzYRfAEl5Gzq7sTUqZIK1YYUt4qwAXEF2+YaC9to0tox7Tw++CqeUtWtkdpaKuWdQpKqKWZJaF0NLBq2VcAlhzP0PUdsMl//9WPGF1s2H3lOX1OspoJ9peUoeZ5+MbC5GPjaX7/D8x9ekZ3i2aMR/9xz9u1X6O9kboVLJh85er1jF5b8yz+64PVjYbW9xm6FVYksWscQhXffucdfPljxeBPoh8TV+ciNhufPe956TbEZBobB8PEXhvVl5OmTQFpO3PMtae3pm8hf+7UOMy/8/neu6RvFvG34k98557e+1XJ0Ynh8XfjsyTW3DpdYE7l8FlitNUUCR4eZThWsg0WvaRcF0xvMgSCzQtoKF58lLs89qlecvt2weFUz+cSTjwLPniUmX1BOYTB0ynDUCzF5rm8Ck1RBaFEQQqaY2tGHkBmnSBahbyvLv+SCNdA6RfQTBksqsPaxJmSa2mBUIV9ClCL6qiSPqk6itK4kOWMUttHEHHAGjK6BZKJAm/3/SgXyGFVdQsbWYrppNMpWoBpqLx5uhB1gLEgQGlPPhs5Ai2IdFR+vNGUz8VlreeNu4pfvaooqxL0vSUudimqriSWjpGYMeJ/pZwUVK+9EmUz2uSaVuobPn+743T9b89/69SWzuSJmIakJHTVWKVIpXxL24CXfpE4y60C0WvpmM+Hqqp5xsqeZVtdWtSKjwPfgp2qVFaUpCowKWG3YLW9xefoqN4vbeDPD2xmjFkKKFGmZfGAtnjLL9MtKMw2SCQJpfyXUs72erTpXPHwNFaI2iLqQoub6PLC5VhyczikoUh7YjiMxe3IwRK9QuYbCpVwhVEYVrKq27UYXeqvIJSG2apJ83Gcw6OpYQ2eMFF5755Q7b/f4fMX2yjNOBT9oHnx/5MUDj1J1kpD+/WqBf/9i4DLf5lIUXt/j8NY9grwgpi8o40igYcdAkxpWasNMzSlbS39mefjoAqXrfufJg3OOXj1A2xW3TwQZHTc+0GrH1bMbhqHQ6AWiWw7O7uJmr6DtEeuQWF2vGcdA3n9SfAiknFDa07lMx8DV9Q95ennBvH+dKRtMnnGz8eymmgTVtRWu4xqDqOptNUpjdEXA5pL3so06is97iFBRaX83ZVB1VGaNRheNLlCyImpVu86YKst6v1ISAXF1R19va113yhqQ6sSQpKtNMyaSB9tWW16cImGMjJuqkg5j7UrILyvnlz7Uv+oqeFkSvOQRVvueSEFpqj3JqEoXcxbXNcwOlvTLBaZtqqJXqgdWqNGjkh1J7dBS9/E3JfDNI8ffe3vNLV35AahKBKQkStaMITJMIL7Q2ZcrDUUOkEKtepXLiMuIrQLLkqhJfy9JiPuiQFKmDDBdZtYvIuvrSCyGfiZECThb0wehQmh2SlOUIqTIMAV2Q81Hhzo6XPnM1c6jfZXp+KJBFDEUQhaGnFFjYPvDc17/pVd55Vff4dM/e8K/+ZcXXD6d+I3f7rhzW9MMCjUp2j6iVMFv6+kRBTIGJbGCS14Wb2TCBNNYg5ut9pimivHSzhC3geKpPH4B3Sp8FHTOVSyVCpmAbhzTUKlzt06WbNlxfcX/i7X/etLtStM7sd+y23wu/fEHtlAAqrrascluskn2METFjDQzohSh0KUiJP1nupkrjUJBjUzQdJNik22qu1kOBaBgjz8n7ee2WU4X7040564vKhGIg8BBHmR+ub+1XvM8v4fDg8CAJa48292W888TBw8g95lvr7d0a8vDTeGT54VDp/jDf/KQL7+95Os/f8bea6q25tFpZl4r7NwRvrriywuP1z1l1rJWmiYFfnnuCd0F3257inG82uy531p++y3L7tWaeCdxeZF5+PCILz69ZKZ7To4KHz7wjJ1Ce80XFwP7vmXxxcDhvjBb1tz5wLN9fc3iZM6+y3z10w0lew4ezvn6l69wxrA6UMwaj/eGqim0c02zqMlzyLUijIHwNZx/NbDbJtp7hpPvO+pDx+tvep5/O7LdGjAOfIGQqVyk9XC1zVz3Eb8w1DNDUQkQHVBRin4b2WwzqShco6isTLqUTszaWtaAWZO1ph8TQSthdKiCnrr81leUktmrTEBYHsZNAWLWolTBecsYE85KuI6QKKXJsFrhtLhZGEXvYiox/9haLHGTIw40GK8gaIxV5CTnllKRyghuWJMxKmItvAmJf/+k53vvL1nMOyQMVCaQiYI1RiYGKUnBMhZSGKmMl65VJ9ENYNEqcLpq+OpNz189Cfy9D6TBsSGRcGQkiCgjVnWlBameyZOB6dawnZnNLNYF+lAk90NpmdJqpIMnEXxh0xp80sxIWItchLVmc3LK9cm7rN1qwgoXSgqYPBLzSD9s2ex2bPoRbxR3W43dR8G5a1ll6iK6hVuqumjoJtPVZJW21jDuR86fX3NyfEZqFUyToxgiKSZScWRlp5WHrKCsdUJ5NYqT4yV3D+e4KpOVJerMy1eXnL8cePVsRCuHUpkHDxY8+nBOr9dsrtbEAWKsePW05/zpXtgnKlIKkhvz6ywG/mJ/H1Mbjt0x1+s1dpbphxWvnmX8fMnhoYUUcd4wbFpgxaKteX31hi5vWfi7/NH7P+Tx6RXJdszikmebV+ggIgtbHbE4fMTy4D18tQSt2YfE682afSwMnZnS6RKlRGISWEkmMYaO5K4xuWO9/Zzr/jmtW6FxqKAoxaK8oq4seVLkS0ruVNFOHs4yeXELyEi/3Hr/pRLNiIBPa7BGBIhWzCfEBOMQhMGvRBQH4tE1dtKGygRrSqya7ClCsxDV7a3SP8nDF4dEtxsxNmK2AnmKPYRcJofBBH3ifz4buP3Qk7xATxW41mLRcbXHty3tfMZiNadermgWM2xVgcqM40DXi9gNpYmIt9dmy5gu8V7zT78Pv7FMaGsmSIiS1MJgGPaIbS9q3ETVygnGsZADGF1wtUY3WfbtaFnPJE3OelohyEeJhbTPhK3m+qJncy1kM+sUxiZqW6i8oXZib9oOmZAS+yFIGFWCylhQmT4mxhJJJsv+VYkm4siJ6E+R2IRM7CIlQB4jn/ynJ/zWP/9NfvjPf4Nf/puf8fO/esnmJvOP/9mMdx9lnLEQLL7JcmDtFLkXXQVFsiGUERsRo4VYpOgshTEouqFgTaB2Br+0lBApQdHvM3o0gMTqpjFhjMGqmhgVuMDBaY0/Tbz5ouPkeMH66x1Xbywnb0eMd3zv9w/ZjTv2rx0fPJqxebzk5z9+zVzVXK6voJzw3u+c0Q8veP5q5P/4z9/h26fnPLkI2LsLyt+85mevMx8/8MRUWBvNUVGMm4HPx4Hn+8jjKvH2B0tef9vx1vEZ7z6s+Tc/fcm3naJ5MfDgsaO2nmATm2/3uMOa1Znlk88UZ82ekw9aXq0S7d0Zn/9yx3rrOHun5ttfbrh83fPub53hTg3+K8PbDy3aKxrr8K1CteLu6UMhPdPstlv6PrG9KqQCx+/WHDzSlJj44q+3PH+dSErhqkJIgXSTWbSWYh1PX4/0JTI/tBgvF741GleJn3xzldhtNQmPqQveZsQYlPG1ovKK3CcoRjj5KRG1wmhNzlGCcyrDrLLsu46mkWlPKDCbFRHFToVAzokUJflPU3B6Oj+0jP+tVhQLJWaZ9Dl5s2snY2tjtZwhGrEbjmCNpstSnFqtcTZTVQObrVjvfPFUaH5+ofj0eeHvf+woKVAopOlMjLlMEeURaw0GTYwiqhzCgLZTE5Q0MFJcz9HS8IvngaaO/PY7DqVqjNYC95nOrEJGl+lcRtZQcn3JOVB5R9NEdn2W7JjJyZSnbssVhdKZUBf6WGhUwjSe0c+5aFa8nt9lq2pCkpWlKgWdQRVDyAJ+IihMhKVXuIXC7CRmJOQp1j2L2DEXpswTjUbEivpWS6jiVLiN5FBIgxRQlfZEnSg6k3UhqSROFaVQygr+fuw4nB/z+O4Djg8PWN0xjGPB2ZZnr77m//F//w+k57KmqL3n/R+d0hwG1sOObp/Q2XDxPPLtZ3uJdXaJHAUK5f3f7Y7/u08G1luOcHQFOmvJ20R/M/D06Q29yXzw6Izlao1VG7p+Tnt4wti9Yb3Z8tFbj/hf/N5v8L37ke1mQ7dp+OrNM266PVV7yOLoQ+6c/hbe1/Q5su629PvEPnk2nWEMkVLCZG8sEl1bMiqBKgFST+QGcsE7Q1+29GOgMguWhzNi8WyHLII4rcQF4C1aCbjFaI1Rk2J1QgmXKfpWfp0sKXpC9JqpIjRyYGhl0LGIyjAXybKf2ABaa6y5BQTJBS4dR5Z1g7UolZmeNtmtFzDKMPYZ3B7xMwCpoKfkv1TSRCiUUdbtiuL2428nAmDRIk4yYL2jns1ol0sWhwcsDhe4pRQCyhpiHMkEAVtMjoKsmS4vEbP8wf2aP3ykcbaAn4RMpZCjot9rdrsIpeCVpak1KcHYZ2IoYpNpMqbJElBUDGqEEhGGO3KAgpZVw5gZ1pHNpWKzg5hFl+Bs/k4sVnmL1QKFyn0gTpAXnUVoVVxBj4U4ZpSyLBtH4xUxgzLTJCQVYkw0RmNsRVKJ2cxgSmH/i8/4rX/xOzz+373Fp3/Sc/lkx8ufDBwGj2vF69s24GeK+lBh9oVxDWlUEGTalI0CL9MZBk0aEiUHVHZEXVgzyIjYwaypWDjFsBsIV5ZNlwgRGe9WhWph2V5rrjrF8GaDmSuOH1u+epk5eVzYPdnTNZavmw3H20g/nxE3mfPNjt3ZAc3Ta/xhwyefbTh6UzBnMz6yni8+O+dXyTC+uMAtF/zkhSFUhn/3ecd//Yf3OT11/OQv12ST+exlx+PTmquo+G8+OuI/7p/x9TcDL55s+cWv9nz/N1d8/vkVf/3K8P2zhgf3PJfPB0IfeP408/fuau6+s+SLbwI//ybTXFxwNoscLBv6tWKb4U00/HBm6dcdd+5ZmrkhZqDO5MqRNoXhcqC/iagIOMXNvqBaOHnsaA4V5y8D3347sN5nqtagq0w/9pikqYpmtwlcbIOkC64qKi8j37oB4wpjhM1Nod9Il5yrRDaJ2nm8thgd8bU8Q2TpHMdcSFoCisCgs8EZScrMRSJ928phrSIqS7sUe2oYC8Y7yTFRBu/kjW2MnuBiagKBKZKRZ9dVBm0VWUlcbkkFlGWICePkrCnIWZXIaG2praKqFH4OegODkZRBHwY6Wn552fLRGLAuiEATCebKt4Jmo+RscIUYLAaZnuYkGSgxS25BbWBroATDT55mTivNW/dqst1PnmlDRr63W9ueUtI0JdmyCYhMK9qZgatJc5VlXQATX0FJdkupCr0xXPszrpZ32flDzu2StT0kIXZvmdpl5IGRhFqFFm1C5ciNpW0MqtF0Y2Eoiv3UBBpg1EBSouZXWhgzMkZmdVzx0W++xXvff0i1ailuzZguCQSycmiKZAfoUaagWhpSrWRNdOfeER9872PmqzlJBZngas2Ti9e8eTMIRKixvPfBCYcPDf24pusChYrdWvH1J+dsz6fMFySq3qtCrTN/l4+/czEQ4shucGQGGHq2/chus2cfA69fBrbb1/zgeyvO7p0RaVjNK/avzvn4rQ/4v/yLP0CP3/LVt59yuRl4ebWhc4V2OYPVHer2Hfpg2AyXlJLoe8V2tHRxpMSI1fKGijlPo3zIqhAVkBU6asbBMRbwbkarLSEPpGqDaxKzeMj+jaharTNUbUNbOXRIpLD9Du8oN7YSK5FSEwQIsYBNcn1tRGGrtZ5Y0n87XdB4vPFyrye53GVcw6QcFZdAKpHCdDlbMyXlZRG+FCsTg2JQJhCygzGTgsH3WS6XmIUseKsfnIqA9F/YDs0tuhKFUWryIQJa4StPu5gzWy6ZLZaotgYFifJdWJE2RmaTOVO0JcctQ4octp4/+p7nUWXIFaiYMcUx9B0hFPpeupjKVFAU4xCJA5RUcEZTtQbXZIqOlGREaxAVZUgC3LC3SWGWEjVxn9lvC5v1SM7izU0l4Rw0jZMYUw0xZIYUCSnirVTtWSuKFXfDWBJxjER1CwBLDLEQlSfmgjaR05Xl7NCwOjas7hmOTuBwYVAlo8rf4CvHe/8bA9dL+htFKpGry0hKiriwVHtNu1D4mdjbwl6Cd1KfYZzWJBqKR4rQ0ZLGKLyBVBg0YmHVPc4U6spzdmKoW8P55cjNTU8Mlgf3Wh7fTVxc1+hlYfh0z5bI2/9wyZf/aUNRnrgN7L/sePujJT//6QXOG7755ppXUfN//t/e5ZuXa/70x1c82jW87kf+yT98zLc/f8VnL3u+fD6yulP4IsAP7jh0zvzkcmR8vqUbe+6dWm6eKYbGUofIj78qPH1d+Olnz3n3oWeXM/uLxLpL+AaeXvcwDpRaYULk/ffPuOj3/Ief7JhZy9snLcd3l5zc0/ziL9+wi5FffdZx3Vk2m4HNPjPzlusvMq2D1aowWxT6i8jVxQAzj13A+nJPVXtmdww3IfLFJx3X13KxuhpSHEU7pCpSKexjQVlNs4TZYaFqM95bqtpQTKTrErt9oe8S2gjvolip1q1BoD1WOrwhJMrEIxhVxrYW74UAqDF4Famcph8CrvI0tRR+unLYGqzy5JywxrBZhym4RssqUKvvflUTbyBrcQBZJ1J9NbmiQCZxqQ9YqxlJ0+cI4lxbEea6UROVxS8gGnDJkaIm1Ykvg+XpzQGPTzbShMhcXI7HLGK5mAtWFYo2hCIgtRgyygj9MBdwylInxXoY6cfEn31haWaGo1VAe0sZJ/Q6cpYlZFReSpEi4LatyeBqg7GRFNWEas9EB6hIUZJboXVicLCfn3K1+pCtnjEKfAVbEjoplPQpZOWkMydiisH5Bt/OSc2OphZL8M2Y2ctmgliKYIQjZKWFBSPjZVKWjv3u41N+/49+k7c+eECX4WbzmsvLQNY9VQX7vTQ8Q47SqGlNzrdnLfjKUs89ox7IJTHmLU++es7/6//5p6xfF4wuHJ863v3Bgux29Gspjro+8/zrwPqNrGyLcaiS8AZak1j+3QCEf/diYBxHYlVRvCH0I7kLwo/PhqoOmAi7XUWKpyxMQ60z9x6/xX//T1a4+Ck//+ZLnrxasx4086NDzlYa6w3W1aSwY8gebRKlDJiSWRrL3Fiy0xNDXzq3XGDMhj7DGDMpGcpY4/R9xhKg7NGMkv6ULal4YgSvJJrYeYmztVaRop3Gb5GsZQQl46ck4r9iKDnjSHjtUEkQlUVrlJVRtdMapS3KijG0TJxoFQX0U7SeGPxyiecYiUlTdEHbIkRCRPRirBEhYpFAH5MSrsqkGtxoiH0ij5kcJfq4JMkqEEiHfE7OTLsI0QkoIxWDtTJi1c6hK4NtHLryKOsFo5sTOUVB5pIJRCIjiYGQ9pSciSXx3lnPRycNdYmUqElF4EB9X+hHJzqOCDEZQh5BgQdmM8t8pjBevMAlOSm2UiSNYje01qK8VOEqaMIetpeJ9Xrq9K0ojGsHvtH4WiKcxrGw7WEbFPXMc9Y4Qoxcx0RyljgU+iQRz0NO34XcWKdZtJnVITx6v+a9jxuOzkDbKGuLqFBDQgVF6hVxM0pxONd4rxh3maXW5KBQYyJdw/ZaY5tMdSDjxqq1jLvEsM6oXqGDhGVJKiIYb1C6kEIhBkXoxWI0KOh1kICpBu6eGVpfcfU68+nPLnn8/Zaryxvqq5qHjyo+/bTnozmMN4jnO0SO7lhe7RI//EHN81eG3/5hTRgTP/nZaz7+/RPs33R01lAz40///IKusfzizUCqHedXPadHLZ+8HvjhXcOnv7oma8M4jpz3hcWh4uurwLGO/Ozfv+QP7tcsjgbW0RMYWN9s6UfFB48aHj2YofuBn78IPOnh7Vbxn/9qy6O7K1Z3NP/+x2v+8NGc3Q5y0Tz9fMtca77/91u+vuk5Xc252nSkFx2//zuH1D7y6nnHxavI8sjja8vr8y3V3KFX8OTVyJubRCiRxk9RtmNCIe+vlETjUc00zYGlmit8m3F1Ea+/gjgWui4TRzBT2mfRYIp0iCVn+hIIudAPVpw/VvQ7MfXcO1pgEOGvk7GeuAqyNACmChTjsLVMCqw3rA4E2f36zV7ws65QiiarRCmZSjshH7pJ2GeyQLFUxlVO3us5o62I86z39KEXQarKlGLEImhHvDLMrGJbEoyaUENvMr5X7LqRL3ePOT78lgM7oFIC7yglUKJBm4KkejvyBChSWiYPKsskQiFZJnWr0PsdeSycvzH8q7+M/MFHnnffgugyOsvPfIyT1U6ZW4oxiQm2VKYgn0oAbBlDSKPgk42A0YotovbPG0J4xi48Zqxm4mACubizgIPEoi3PwlAS+ziSMtiqxTYNrrIoD4eNTPKGWNgmuB4V7KAr4piKFOLUMB4eVbz78UOOHx0yuIQi0ZSWareANGB0wZs947hl11tShEpbYU1kgaVtNxv2+1fQDuzjFZ999YR/9397wud/sUYnja/hwVst7VHisusZQiInxeY88OLbHTlqtBL7plOJmS8ceDhufs2aAe08yihSjlN+fMvgIq2vMFZTe8vRyQFVmeON4d7hig8eVgybL/nk22f86vkNuEPeevc9mqYCAs5aciwYnQl6g2huE7XNGJPQKknVWNQkoNGgjHTPaDSOnDJjjKS4onRnvLz8nF05R/vE2NesLxr2W4N1Fucd1hkZ96VMyXny2VusygTS9CDKNitPtg+lWqz2qBIY84DPmaKFKFhZh7tdzk9VudVOkJbFTIIeucRSSuSYCDEQU5SdniqkPFBKnEaBoknQSroO48RaZyolfzcKiyJHZBWR5Z9LURPedlpHTNoEY/WkrtX4tqFdLnFVS0FsjWMIqDFP6uNIzB1j2BHCjhA7YuwhbMlGY0zmeO6pkYs1dIohOLqQiKNjHDX7IbLuRK3vnebOXHFyLJeirqRaUamgsyXFwjBmNOK/1tM6QiVD6GC9Dmx3kZxEKGmswlqoGoPx8tr2Y2Q7ZPrR4Lzi9NiynFf0o8eHiFloduNARONbh2qgmtUsT+DwvuHonmJ+mHG1xA2XoVA6TRrF2mmy8AlEWiwj0FSmkah1GC+FXnRZfp5DIl5Df21wXmMPNb61VKtMqAJxDzZYVJRVQcyit5D8h4TzijTINEHFTJ8UaaOoisPME4crj9nDV7/oCaMizzWvQs/6OhM3FaYq3Jmv+PlFz5tvej5432DfXlFdbZkfNdS+4vVPI//6317z1juH/MlPL1AuoZTmJy8CJ8c1R7PMJ8/B5cT373vWu8TRoUGjOV1YfvmqcLISFf3BssJf7LmONYet5WrssbbiIgVyZTh+cMDq2HL5bUQTmfvC3/zlOT95GTk+zDz/2QW5qnn4luXmzQ6tHLODFbt+y3yh+ZtPeh794YrzrxWrpaW+U/H8VwNffLbn+OECO3OcX2xwraVXhW+fjOw68K7Qei9i2zBFdEd5jetZZr7QNEuFm2d0o1FeIqON0eQkav/UawjCEODW+24MRSuGITNkWRu6OooAcNT4rGi8wVSK8aanrhQGjTFeOAOVpmnAGCscDsM02s+42lGUpu+zOAeUALqUAj05lvXkLEBN7oRJR2CMmn5fzuucM9ZBGeQ9I0eQkhAjDN4oSRbNCh0iRIvXSkTRIfI6DbzYH7NcPsUqcTJhiuhXkIJKqfydZok0AXWU7ORVUUQLroZ5ZbkcCyUHXr2J/IcYaGYt988SJTmRDeqREA1jAKMNWStyshQ7krR8/9YYuhzAKgyir0rBSmhVSYISyBo7vGK2/4yoD8jGI2zVW53WFA+vCqlEhjAwDHu6safkhLGG6Cuc23NaJY4qETZuxsyzThFR9F1GBYM30s0ro3jnwzPe+f49tNcMocM6hzaaqqnIUfI0qkrha42r1Hc/o1QSih6Fp48brvafQVL86tvn/Lt/+YyvfzxQksZbzeO3Drn3zpwdN4zdSIo9cd/y+quOcZvE4QaoEjnwioPacFBlDuq/1WD9WoqBg9lMfK8aKl+jsiiw26VnNqswvuBGQ20cd1ZHvHem6a6/4WdffMrTiwG7fMSds0egDTdpoLaeGJ2ka+VALiNGWayyE+4WSomAxWiLwuGckf19zsQUUVrhtKL2DpsrqvoApwIX+QhTec73PU8vrohe0Xo97WBuBXVSIWotO381qUzTtF7Rk71QTW8wXTI6J3TJEIOIDWuN8RLy4ZxFKY3TGqM8Q/SEpLBWwlJKEdRsyokQAnEcSTGSQqTo8F3CWEGiJzWSeZ5VRnlQVcG0k1jGyT6dW5vhKJjKEEVhPIG4ME54BtpMX1szwzULlKtJyjCmCGOP9Q6lEzF39MOa/bBhHPaMYU/KAaMFYKJKoR8tb25gFTNFw2aXCcXwot9ztc1crg0vrwqLWeZ33nHcOXO080whSvFVDKlockjkqDEYTFMkEEUVGDXjHrZXgd0uE5L+bjTqHHgvXPRIYoiJfch0uYBNHB7WHB6AdSN4hclAlaiPHUePPMvTQrXM4D3KFSDIEnCAshuEFzFJl4yWFLWiEhiNxsiKqIhGIkchkSmtsAbCqImjIG6VKeiQGfeZ4SLRGYVbWupVQ90Usk+kXopBH2WyEJNCJYVSCecURXtylt2kCyNp51C5wiw1D77vuTO07DaFF88H7n58yMNHI3GbmN+teP7pBS+vEqWqMChuvh14dd3x5k3m8ds9btXy4z99zh/9oxMyhlmb+OK85917NYHEk73hB+8vGWJPGAuXunB2aInXhW+2ltWp43qz5d17nsvdwGrhuR4Lq7ZwWtd4HbHKcXKo+frFjpcvIr5ELndQ1ZFNdvzh+46cO6xu+Cc/WDGL8HKXOT6r+OHHFf/fl1f8//58x8LXdPvI4alHDYnnofDVdeTO3Rl3H1hevNrSWc2mT1xtRrQqzFqF007WNymQg8IqRdvC7MBSrRRurqjbgvcIx2IiwIVR3DvrdWbsxH1iXEE54euriUiop9F8UfK+1VHwM8ul4eioJg+Ryli8l4mdtUI+VKZgfUThKDaCMSgk1wRbGPc9+13CeDcJg2WFKc/elHhp5X1vPZNTQcvqoJTvsPEocBXktWgfYp6eayvq+NpqDp3itdagknwPFkYscejYDiO/2h5z17/iqBaWSs5KUkFVJbjsEjHWk1MmZcQyGQu1E/Ki0pHKVCy950r3ZO+oXObNzZ4//svMH/zGTLQgfsRmRVGGThti0djId19XUeK+sq6QlRFBtzEUb0gxyLcbLAbJi1Ax0nRv2DbXBHMiYB8MumhxPAElB0qO4gCZAtqGMEKOpKpicbCgaSUmvbaZfgjYbab0Dr+DXV84PF1y59Edqpnl4TsnnD08kAwarb6bLlsjTrFSApVTtI2lrgtxHKAYFGLHJiWG2LHbJ86fwZ/+q2u++psBUxSBzPJwzvsfH9MewmUXZbWaLZdvEhcvo7ixtIhN5z5zWBeOZoqDSjP/da8JjuY1SsEwhElxnwkqEAFTDH5YQhi4P295/96C64uf88Xzb/j8/Ibm8F0WBwdsYkeIAasNKSaMjTjrUKVglMVYjzOeKaMLcZgYlJa43RIjMYz0KbAZ9vRhpHUVx7MVJ25OMgkzX+LzHO9a5vNrqnpH0SPayKUiWzXJE1DWErVBK40xWpjW3IZSyH+fi5KOvkSIGVu0kPG0o9KORVNx72jGvK5kTB8S2z7LBME4Dmceqy1jhlgUQ8iMoya4TN+N5BxEYZ6LaCKQB0ORCSkQSRQd0R5Mi6iGa0UporovKZMGwVKqUaGHTM6azDSO9A5rHEo7qDzZWIpzZKMIBCCTQkcugRC37Icb+n4nQsISQeXJCjNCMXx5mfmTbzIvVhHnMtu94rofeLn1XO4yF9cwd4X/9kPFR48My1mQYKusUcmSAgwhklKm0pbKWagLxESJitAp1teB7U0ijnLgWqOwDqwXGNGYZaLQjYkxSkF3eKo5Wiq8kzG5Nom6aErWlBHckSS8lXWBPFLMdO3n9Lc20NvMhO88oWJFBXFyoOVX4xTKT5oShfiJA/TbPLkJpIPTCrJL5F7TnyeGVwXfgGvF4mmMAyXjVIsia8ES55LAKiyO2iRcVnSLhG1rZgtFrDO7px37jeIP/ldL/vp/es33fvMUpws3z3vscc19GznWFfWR4ctPLlkcLvn0yy3Hd5d88dUNxVr+41+8xPnEJ288ba2Zzw1Prws/eqthtprx2a9GmlPLaZqz2Q6YCs7uDrx5mjm80/B66ATR2ihUcdwoSH2mrQsuBdzW8/x8zar1/MPvNWy7Hl17Tk4qliry7ZvIWCyz+3N++vWaq5uR3/qNFZdXN3z+Ys9m0/AH//WK1zc7/uynW/7xbx5yfdWRtiN3Pzri1fmaF5eRPZp9hrr11C6gCnRDZhwyTmlmc83yQDFbKaqZwjYaU2mck5yVkAP9rrDbFDZXhaGT3aurLNoqUbIhl4bczQK1sYBVCWOV6GEs1HND7AYapXBzT5n2w0UnrAZlb1miSXJKyBQst4b+7W4kJovxmpgFWayU6H5UmqaGWmFUkZF/EbeKmmoAY/5WGKud6JR0BTHcumcKJWushscGfmll8lHEc0cEhjDgxsK3e8cv9Yx/eGekUPAZ6rphEwaslijyENO0BmVK30uEUtDekAYF9YhpHGar6UjEHLHO8eTGcvnnW37748D378PhTNaVRmXSkEkBsha9BVk6Y2s6+gIuGXKWHbpyEJUmRQUxyV1RHEZtseEV0R2I4wPQRTGCBD4BtmiSEtZM5Tw5JHZGk+oGde8Q7WuMi2Td0+Se+1mz8HM+yI4hJI7P7vHgrYf41qCcQdtMyP3fhspNWgutDFobvLOMreFms2OvRllhJCXZLBrCqLh6Y/jJj9d89Z+3qEGjjEyJ7j5ccfBQ0+U1aQjEZOm6ilcvduReROrGROaucFrDaQMHNSyrwsz/mtcEzsl/vu+CWEsm8UpKkKLDFcPhcsn7bz/k5cv/xFcvnvBsDXZ+H1W1rMcNqmQqU2OQC7/SwgFXWIquRJk/BVloZafkvkA/bOjzwDgK2Ww/jmz7nn4cWc0PwBqGVHC+ZmcrxrEnjD3FFY5PZlzvimCEJ7iFrAYMGocxTlShIvUXp34pUPQEjZDRtlOZ1jn0xPVeLTxnBzNODxoOW4N3YgS+2Q5c7K646TN3j044mzegPPsIoTicyfRKy9R5HKd0RkiTY+E2OZAiO/qsM8VmVCWhJbqaiJ55YgwkTeiLrA0GhRpkzJmLwhiH8x6rHcZWYB1BFfSkok1Jxt10AzH1hLQj5I5Ugqw3bvkF+Za+oHm1zvyrz3v+aq4xdqQUTxcc9J4xaBaq8E9/UPMH7yYOG9nzlwClaOIogsKC2KasB4xYBVWCtM10N4p+L5kQ6Cw+a1dkLWAkXaxPmTEXYlEYo1m0hpNjReMLJWhC0iQDTis8hrKPxItMRMsESEknpl3EeC2pbzlza+pgCoYqk+1ITR5jYakLElKiq+X3SgFjDfMDqHxhvIFRQTAi5ilawovKPjPsC/2V/P+0iXgDKYtTxlUWjEFbg3FCZ9RFUR3VhDTS3wTCkDDtgmFn+NP/OPDh37ccHmuuLnryeeCNV+QXgfUa0nHivjb8/Gvw5zu+fhOYnfY83Vsa17MbI3cPNH7IPHiv4flXW37zB3d5eR6p1Jp33jtkHS7p9pbVLHFy1nI9FhbLjof3Dnn2NHCwaoBRaHympTdbXDHMfSKNluO7QEhsiVQLQ+01H354wubpJfUl/O5vz/nZL9Z88+oGOwYenRX2QbHdFT58d+TsFP76z7fcmzvsoHny2SWz2vLF655vnmdCztRNZt7KeRF7TxgKhszpkeL4xDFbGKyTMBw9RfSmlNj0gX2f2O9gvy2EQdw5zmtcrQUiNeHI9aTeN2aCkOmMtRbnNN5p0bQYjeoDzVzjXEa7QIlW2poJPS2PnpM3MTJaV9/N9xXrTUJbK5CzKcnvFudLuZ1MlFu7kPw7Md9M0STSvBhrSDpx60AYUhK2yRR4Uwo89BqjhYBZigCvvLLE0qNiYsyBv7oyfG+lOG4iRhcaI1OWbYjMmpbtdj+BczRFZazThDzinaf0kaQS1hcaZxjGcTpHZMqx7ix/+ld7Xr6q+NEHnrPDTMVA0rArmr5TGOcwRZxRORuebgZaX7OoIzZEiWOvskw8soj6UoEyBPTuCuVv0UmBgiUrsfbaIpe00GgbmlpgQrkRq6NtD7CLQ3SlKDbgbaH2NaftHKMtuYw4X1M1LRi+i38na3SJaDLaOirfkJsFalB4q2grTWUHnHUURpy2sgZHM3aal08iz7/aYScdSYyZ5cJx/7En2g3b/VbuCzTX54GbNx21g1pD6w1HdeGoSpy1mllVmLnMzP2aswm6EGSP7W7H+AWvKrLSWBpUDFRlxi8+/0+cX3zB9R5ic1diPsc9VAavNd5IJQZMu3QDOJS2cvgCzsiSKKSRrt+x6zZ0KYlafgrmaWvPrK6pK08qI+so4vc+Km72e/b9SOwTo5hgBPWpzRTbqSaVvyTy3WKJVZSoQKFiFVJOJAWuMrRNQ6VkT1LPMnePW+4vlywqhdEyltuHyJvtltfbDc47VguYt4UUBf5wO1WolGVXDHujSUUQxqkkckmEHMllpCjBEhcjwCNlC9orARzlyWZUFCVCqRM5KugVeiikpCjJoLSTaYutsa4mawNakbQ8rKpE+bPUjlRGCgNaR4wV22ae9AdGKXIxWKRyP98XbnJNMYrWN7Qoqtowt5H/6t3M//JDx0mbGPRIlTU5GoYhMoaM0YbGe5yDoougl4dC6QrDJtPtlHz9Oss0xGq0l8dkzIWhZBEBKlFTzyrN6aGM3gqFPhf6KG9s7yVqNoTM1bPCyrYsTxOlj+Trkag02RuJpHYabbMEyLjy3UFrkpr2oUkmYgmIotXQgNK3eGVZ6jqbMUvZP+u9IUwWUoZE8kl2zt4wbjIEyZAPHQzbEZJQ3pTPqCZQzT1hGHGjYYum30TGS4O7H3j7rTnfexue/vmeew/n/OTzjvVLw3bTk8i8ebHnYN+yftWzPHakbHj4UPHmamTsCvPVkq+ubyBU3D802CHx0W/M2WxG5gtNLgNj1iznc3LYSvx0zNTJkZcjm6vA6tAybzWbS0OygdZlhhnUpiLsB5LaEUdHm+Cvf9nzW+/PeXbec/mXFzyq4Icfn/Li+ppf/PWO2YHmw0cVb15tuPvhitpo1r3m65eJ7ag5Xil++eUVj+434Ao3A7y6GTk5lenNdgO6JGYV3D8ynJ5qFiuxwYUI46DY7TP7MbDrM32QQBomdbnW4J3CV9N7JGjwSkSrVrDBxoBSeoqUVThjsFbG80trOaygqRTKQHEaXAIsOcmzLHljk3iNNHl25RnCCK58uwXlZAWnEMqeEjOByJKM6AS01t9pCPR3egHRV8UYcN4ypGGymSlJV50Ezmkqqhc1zHxml8X2ppTCxERIiiF1EBPPusifvgj87z+qGdOAinvmbcsYN9iSmVeO3TCAUrIqMBIjrFREOUOJAe9GfGuI0aJ0pougrCZqx6gW/HhzyOe/WvHDO4EPVm84avf4Rpxj3TCKA8omSqV5MjZswxlvH7TUdCziiCsdOo9onYQVnCAnR06NCBR1IZJRWi56xGSJNo5azZhlxVAUhMhsVuNbz6Jt8HWmaIPxDbqpRGztpkYCR9GFUBIqa3E1lExKMpWsnCZlTXIVzi+koXSZpkyrAm8YSyGZRCkabSwpGXbXsuoyRhF1wTl4/HjB4VnNbtgQhkRJlUS/x8LZyZJGJdR2YK4yh1XhoIFFJc1RbaH2v2bNwNgXnMtU3lE5R4gBUww5ipCouDmffvUZ3fAVpj4EuySGiLMGrMXpFmsnvjYdBUMyjURmFtA5iudcGdl/Dx1D6BnGyBigWP0dD6ByCqMMlfN4JWS2fV+47Hec7wOb0BHGQLfe0W3WGD2Rv6wSC47SaCWZBEYLAco6ix4TSiWsVXirGUvCN5b2uGU+a5lFUYAfHTbcO5wzNw5KFPuetcR+ZLMZUcly1DbMK03WEaV6KgsWR3GOjsBuuyblnpijaCZyIOaRlAayiiiNTAVKlgMpS2eeC+gyrTyyoqSCrQwlKJQTGlmOk/imTHoL6yVN0IrYxVqp1r2XFEblIrYUKUyyIiZDUhLmE6PELd/CPpzXWOdpqhbvoK7mzHMhMvLu3PDPfgOOF1fg5phSCUGxS+SisNZQeSPCpiLBJilB2CbCADGKSKNM6wntLMVqMIVQMkPOBKXIVmPItF5xuDDMWrm4931hG2RkWU8ToGwKA5mb88Cmzzz0NQd3MrotxL0nD5q8H4U57jV4Q3YyejUa0PLMaD0lR6YpwnRiheco+9vbfS6qoAxUrUwhdNEMJUNt0MZRRuhzJlrxSicrbHnlDf2N4nqb2F0F6iPD2bFnO2bOP9lymSy//0/nPL0OPP068WjZ8Q9+CH/yxzeYcghXHSerGQ/v1Pzpj69Y3q/5+N0lf/XJnr/49orf/9F96CIXIfFks+bvv9XS7jTMHKta05dAVjV9ChzVgdfXlg/ebXnycsdvfXTCJ8/3rIeOh0en/PjTG945cbiF5uKqI6LkfaAzdw8WLK3maSqgIwtr2e8DurH852/3bPeBf3Di+fmLTGDPs1c7Tk8UDx7OSLqglprdNtLvM+dvCpdPNmybmuefr/nhY4eqNdtek30S4t3oSMDMF+6fKU5PJSp6P8DTNz3XN5HLm5E+SOKqvLehslA50QplJQFhRkOagryUkwYBW0Rc6BW1szglBV9dKRqXSVn+/ckMvInTz3QSXOtCNoFSJgGiEjU8RXa8wt2FQkApQwyZsdcok0QorcQumMlkJZY+NRUDIqZO04hZT/ZnTaaQcsT5mvV2j1K3jAC5YIpSZF0YU2beapaucNmDz1lG8iqTImx2N1hbU4rmz15Efu8efHBk2I8ZkwszXRHjSGU9ZAvTFDWljHeWlEUwyyB45aqG82vFZWpxpqG4Gb5aoNwB1tU8C5lvv838ddXy4cE5bx2uuTeLtCWz6QJhLMwMPD6R4uHK3GExPyT4jI49ZexpEahS0MInwKzkfUuYVoAFkEu7TPbxWlcUr4ihI+gMvuC1wVQwqhFVHI32FKUZ4kAJO7QehddQzTAkIJBSJMaEMQrnxRk3hEQskjJYssIDRUWcSbhakXNFjFEAeKoI/n1MUMRiihtZtS1vf/8M5ol+k1AjaKU5mi+4+4MF9ndadptr9s/f4C5umI2BmYVZpak91KZQ/bqhQ7XxaFuhdABVcKoiBsUw9rSV4vX1cy4uv6L1FV0YuYov0RpODs+42cK8Hlk2NbSOWSNhkBZJ5EIXLAaFYC3HcWQM3aS0duL91UJtc9ZjTY3SFmMcaUzs9ntebraEUDAF2gQ3uy39dkM/JJw11E72fkp7aQOMjHrRamJM6+/SCo8PDzg+XBJij6oS7UlBmw1+tLT1nIOlZ2YLumSSVhRjQFtSHIhjYlHX3FkuaYwm5IjTU3COkosk7nfsh55+GIk5EPMgf6eRSJDxuFFYLXWsPNhTMNHUtMrGo6DSpIA2atprK3TW5OQgWwoWpS1aS+iPdRrvwNcKX2WMdhRjAUvMmjEYtBIuOWTZeRqLtRpbVVjvaJuahXO0lcO4lkZHFJ7fe+eahwc3YBtyjpSusN+PhGJoKkvlFVplAaEkRdclxr5gRoWKCm8c2WTBqGqN9oZsRLgUShRvtdJYpamdYjUzLBcWVSJDV9j2mc0g+QTtQhT6KSpi0Dhv2K8jX/9Nz1s/qji6B94kaMWqmZVkS6iksMlKKFQRmyM6f4dHLUXGxqaaJgPFiEe6JMiTlzlZSlI4F9EzYQqMeyhKU5xCVxkXJXDGFEUOAdNqlq3DbkfS68LuRsKYmgNN+8CTOojnAx98WFFfJ1SM7ILj+7+ruNruOTprOX824k4afvTRAQfzlm3qyVlRVZrLm8Tz9Y6T2nNyYPjy4oa2rbHJEPTIaeNIXnF2ZIjG8ejQs02eanHJ1eaQjz6ccXV9zn/+mzXvvD3n6ERx9cJgfM2Qe5pS0fWJ2TySc+JgVXP1umejhF0xq2BImlXx/OqrPXfuVnT7HTdJE/rE9psdhx6+f/eE89db3nq75ssnI9ViRX++52os2IVh3Y9sLwwnp3DQWpat5u2HDY1LNI3mah34+vM9375IXG3EflzXiZMTzdEcFo3Ba0MJmRSECqetaAeShmKKwKHqhGvEwaNtwVtonIx6K6tZzA2tV8RuwDkR5mUjQj7jCtjwnfZIOw0qTH45LeyOpIkh4RpLUcJC2HeRcWLPG2X/NiU3TXkD/8WYX1lASSeu9BRnS55wuVLEhBGMVuSpQMZLAmEpYrGd15aVFQ99URLJnpRMusYuMFssqXLFxWD4n3654+Pfn5NUZkyJ1lXsUw+5MGu8JKHmIqA0JVOVujJc5YS3mqM6MUTHV/tDHqyWFNeA9lhlgYLX4rJ6una82Nzh7tV93j+1HLV7tv0lut+ytJqDxQGP2zN8M6MyoExF1g2hkgmR1iIGUsjrbXJB5wlhDMIyoUwDFrFCyj1kwUIOYqmOWlOMxioxWvdDJOaM0wJ6mgRHpNRBCtOWWc5XpUQfVsh4VxFNS5cuGMaBkidKIXLha8sEkhPUuPYW6yqMH8i5cPfBkvmJYpPOGSfXR2ssy1nD3Ud3sasF51cNw6GHlzX+5oZ66JkpJW4HjWCzf53FwKwxRGcYiXKNZ8+uv8HqmjAGzl9/Rq08Oc94vTlnzQalNDfbG0owKGNoG8/ZyTE/eP89DuczbBFNvzMVrZkT0si+74gpoYzD2wqjjOyzFKRisLYihEgaE9o7Qs5sw4DRgfsnC87ahjev3/AXL69ZbzZkVWOMxISKqlRCPZQWDoCaaHcogzLQzisePnrIoweHdOGc6/ElwZwTUsa7JfN2SVNVFDS6WLST2NyURPG+OnA433K2OmRuFYERIdzliZYFUSmUryhqIIc9IewJOYigz9xytf/2wk/Tg6z5jgM06RqQzlVNKGWjwWpUMRhrIXtyspiSMdbhHFiXqL1oQLQp+MqgdU1BEYKFHDGqYJQwy7OxqOJx3lLVNU1TU9c1TeVoqgrvPb4UjurCB6dXtCpSsmXo94TBUrKn9kU80dNBlAbNrs9cbyIlaQ6MobIKYiTkgLIWazTKQ1RJqINKIlhTkNTIxdywbAzWZoYR1kPmeh/oBljVHlcblNWELjGkjGoVtdG8eRMYf1L4uLLM51IIqsbIGZsgx0zJt6mKk1isFAn+UiCaEyZ5q5JCDbFk3UosClK86UmEao3C6MSwz6Qgscy10QIwCQVlHENfyCqzPDLUquH1zcDyvkGnwLJPjK8D4ewOL84vefz9A44qw+tXMLMa82RkfmB58dXAzUXg7LBl7h1/9ZPXpOh4dFBxcLfh2etzmuMGnxV1WzHmIFOipGlmUuz1daYL8O73lvzxHz/jX/z3b/F//R++5n/9/e/x6aeGH/x2zeXLyLKtsGcbNn1NeBNwxrHvAnuVmFeGuo80c4vuC7tK0VQaHzTVLDJXDW/dq3jybMfVeuDRmaMPmQHF5nzPdr/l1bni8WnFbKV50EJoFLFovn5ZiN2etx7PCVtHs3RcbhPrm4GbXeLpi8Jmk/C+0C40Z3cMp0cNxwtNpSEOhaFLhIQo+bUmmQhVpjQaXRnqylI7cQJpq7HGMLNCf/SNYjazzCpF7DNYg5kCayS7BIpRk4CsTBtyuSBjypQgNr0YRaPipiJEK816nUlKo7RDkaVuIE/yAjVpaIygrVVGW0gKvJX+JmShrE60fmJn8FoRejmzsQpyJofCkDNzqzhwmooJYKQdUQ1oVUgRjHbo3lIry0/OC3/yjeG/e99yUzJBZZwypDJiTTXpIgq6sowhoXPiYF6RQwStqOaWe8vMl0XhTEBbj7MZa2X6GXMilUBTaVbzFVVzwFdxwc8u9+z7S2o9clh7rHfMdE3SipAHTJAVrbxeCqUcNtupccpQNEpJCFBhQr1TMAiN1mhFnS2LekZIMzapRyGTImc92iiyETaJUQprKrxpviMgCi0StCryvCj538Zi0WrEIELupAMxZSpX01SF1ia6PLIPmTQWvCqSgeM1ba3ZWc38qOHknqO4nr67BS4Z0mZgba5Y3VlhUqZeWjwH6MOactPQbrdUIWBjwoYRm8OvtxhQ1mBsT51b6Gr2+xuG4YZFO+f89VcwjsTqLhe7NYPKWF1DFpiKoicNhsuxY93vcXXD22f3OZuvWNYtOsGQ14QQKSmhtRHxXoGQIMRCVBo1GnLasBtfUtnMo9OHnCwPuVMZhuQ4Ws05aFrKcIPOkRgixUZyHinFkoqMdrVIxik4jPFoLeyB5aLh7p27HJ3O6NUlW/2Snb4mDpF+m7H0HLcZy7TEtlpSy4rgi+dNw1t3NbVZ8GD+AK8yV/unvBkv2Ic9Rs9ozSmrWYNGU6Klu8mEITKKkRanDFPRT6agsvjXb4Nu8kQJpEgHUIpCTYmJSmm0MWKDw1GyI2uDLpLLYuyEUZaaAaM11lpxd+RE0Vq6GyWJeZUTBbJSHu89dV1TNxVN7Wm8p/YVVeUhV7x18IbTNlBSYsiBsUuY0mJ8xFmpxkuxhGi43hfe7AJxSCyN0LwKhjFlghXccOUlgjWXQKUtznjhKrhMVcO8BaczYSysd4mrfeamy5ikcQcKWylSjFwNiV0u1JUVa6guvHmdePZM8f73NTrL+FbCSApK3boLHFlr1O0499boVSZf+ISqlkhrPQm8isDhb3fRadpLa3BVQhkYeulSEplSkuhkiijX1RhIXaE5ctw/cby56JkfVsL10InWJQ60ZvtSsT7fYq3j+Tpzv/FsX47c/7jFJCgucT5suc6Ou0cN5y/3fPLTp9SN4RfPronHlrNlojKWbV84uTPn2cUN796dUfSMJ69/yW/8gz/i3ccXONNw/5Hm2eUND989JAaFq3t+9O4d/vWPzylK8/b9mtrBL7+xdNGyzZG6clQLR58iZe9Y1IWL3HO91wwq82inudyPvHtvxVXf8+XLkT98vyWi+OzrRBo0d9+v+cWnV/z0l3u+96jlcF7z+dMdjxeO9S7zV19EfqQ95xcdb14NaJNpG8OjB4bDQ8vhgWbVOqyOjPvIeg0haeH1O4QEWYGvHK4Wy66xGqdlbakCqCzTnNEVjEpYn1CNpRsGiGCNFXCUUdOUEZSxZAQ8U0hC8MyKvk+kIL7foUusWkshYUxNyYX1TUFM8w7USFHCV7nF1irELaCdRquItZohjXhboRjRVpT3zkSGKCP7rITYaqaTXmtFiuJxB3CVrEzl0Q0oo4ip0IeR3djRjyPae7zO/MuvCj84czxeDeyixioROOdRoXUmaUtxSVRawaGzoq1lOuNay+EsUneSbKtLId76/pXkP8RbZ82spV42DLGn6TpcrjH2kGQtacpBUSWJeH0K9VIUwa5PPZKU6Ldyxf/5zlxrLemgiJMMY/B4qqqiHx0l5mmlZKfEWiWT5KmAEL2Hlwh7I2vTUuT7TlnJurvIHRhST69eMRRpYJ2taVymqkb6JPZL1JR7UwrkgM49sxS5d7ri+N6cIV0LKyMYfCzEmw19HOnu7ajnGlUZ1ALMak5eWPpuhY0jKmTc0BP73a+3GNBWDkaDZdvt6fZrmuqIzc1zdus3OD/n1brnYtzhW8mazllYAHr6fGcNfQj8/NPPePr0Od979Bbv3n3Asm5RJlKyJiXDGMXC2AcJnemHILuXXCjhNY17zcndOQ+OH3F/cYDNjptuTSIT48h+39GNkkeups9LBQJJDuGcSTFQtKIoAe742rA8OOHkzgmDuWa7f0Efbuj6wG5TWN9EVrOEumdln5NFhR6LIpcMaLzxE1dA1PLOVDA4gg30cUeFASJOOayKAjEKgSElChIYopDLRyFnSyZPi8Y0ydmzqGaLPHQ5TzGeWWOUCFGMtpjiKEmT5J2CMQWtJBBJSW8hSFxjBVdMIuuEcwaboVj5fWst2lqqqqKuGyrvaaqKykox4J0nG8fR/BqrAykExmyIFLQJZAdFN0Tr6dSC6+C4SHs6rmico9aeHAM5QRgz2mu819SVIipI5VYjAjkIrrRpBaQS+8Jmn7hYJzZbGKNm5g2+LpAjmz7zchcwGFbO0A2AzYQYWO8T4+jwRaFsAZMpQQs/QxfRBZQMNv8XCmhhAZRQyLFMehnzneOklIRmClKZSJIpZeIuobMwHyogOBmpFqNROtIryCRsMfQB9jpx+GBFk0e6y8TNdiCvLJdfX3I0b7i6uOH5TeHxMtAsPQcnhn//H3valRFdQPSUseByYr/dsR41d+8odsnSWIOuA7s9HJ7V0uHNK86/CJw9iFjV8cOPl3z+xVfMV54/+fNXHN1f4Cisw555nKHawrOLjlA5tjsDQ898UVH5CqUibzaBJhpOlo6zQ2CmicPIq03hfmU43wZevLlhMde8c78mfj3ijCaPEPeOo1XLbj3w4rLjxcvA4/stjx9VfPJkJOWexaLlz381cDMUfJPwLnPntOZ4ZVhUmbZJ1I1BJc36RWDXj4wlo7zF1gpfQ9OCawrOK5wV/ZBAfgqhT8RQhP/vg4QCbRQYcK2ibgYqXZhXUPtEM7P4UsT1YhQqBUpJeK0JSbFfjxgkZC0XQxjkeTCuoJRDm8x2ndnvDMpIAJuIU5XAhsykFVJKhM63wVa1JFpaDU1tiWj6XaCuoO+UBPAYoe8plyjFopUIam8/vBdPYowJrQeUrSbd0EiK0DQVoQBGcVMy/+Pnkf/TDzy+SYwxUZWWMCbsTELOdEo4JTyAGJQI5UYJBjusMpUO9CVhUyZGpkmkrNVUcXg/x9oZKRtKThTtwafJv58mC7ijJNDFTrksGTNd/mQhIaZpFaCm379NeNVqGk9mYUTknGEiPBql8dbLz9lKk3gbKHdrSy9EYimkcYCiqKuayreghYqYc5qgUJpEoAsvGeIlOTv6EHDjODmJ5JywxqJNkJUv+juEsGvh8NRTZpGhH2RSUwyzrDC1YXFYY+vJIZczYwpklcFromnIqaIpWmiZY//rLQaMLai4IO4V+/0rFBWqaNbrZ1jbcjVoXl6fQ2PwRaERpbrWBkVF0nKJOW1JIXBzcc7Pdluevn7BO++8y/3FXXIsxBDo+8C26wkpSihHiVCCJBWmraCJe0XuR/wy0HhDZRcMCTbbLZWtaHyFLgNxjKIAzQmjHc6CVhmtBMRTioxx2uUJd07vo13Hxe4lm2HNdhfZrwubHez2CmLmct2xaHY0vkbhpkSvONnMnIA3yshF/waja3bG0pTHuPpEdpRJREBjjLy+XnO+29HFjMNglFiUULL311oOJJ2TjCBzJpdInvypOSlSmrqGLGAcqwxWOYyZRD0KKOJ5V0VGZ2XCExatBeKhRIRknMEATEhlrS3OOYw1+MpT+5rKV1SuojIeZxzWWJQNxFBxM8wJOTCWhtEmlPFE1xLMgo0+YZ2O2WtDqtcYf4FKG65jR1XewDhglUwEqkZIaQaodYUz4svGgPeyZxvGzHZXuLjO3GwKoYeiNbqSJLf9CM828GabuDdXWBUwGrRLpH7k5say21U4rwndRHkzGaxGJ9nj3YKpShHs820suCpKhI6SX03JEEKZMhhkrauUvNaqOCHfJUhDQZERcrWmlCmy1mlGHSna4mxivx15/sk1VdWgY+LeWzPmjzXPv4g8+VZx9m7h3a3oXF483XOoMm1jaZJFGehGePyg4mmo+OKrDYNx7LuR7RA5vr+gXWzEStc27Edox0w9h6gjP3pX0acDPvviCnVyn6bu+PiDh1y86PjmxZe8/8Ml159veDFkTo6XvNlkFD1mGHh8PCeXzM+3mpQGjqLBkLGt4fW1JltN8YnKwot+ZFlqtucb3lz2rBpPruH+/ZoXVzdi073sOZzVHK40Xz3p+PmvEh+/VzF4y8VN4J37nphFCPbWI8u8tmxv9lycR9IUR218oZkbFkvNfKGYNzCvDbNWgdP0JRPSdMAXI351I510Koo+S0enigXl2Q2JeD3ijMNXCmMjVZ2pK0k4dG6KtFWZuanY7gNXb+DsyLFcGLQL9J3mcOFRtkcZTzGJV68GUdt7g0LE1BJopihKk3JEa3FxZRVpmxbfZLw2WCzei0hSFSEPhpuCVoZcJPTHTBTCnKcVgDXSJNjJXVVEm2StoYRCLInKVrhasd0Hsmhr+cmF4//zTeKfv59E1xMNgxVMsTNRmC1asycyREPdWnKJhKQ4qCyNDlyT5eukUKJcYrVrqOsZi8WC2jciyCuJoorkmKhMwVGSnGsoSWk00+qulEhGvm7BOkLRt78n3BCl5YwsJTL14sQiNu5SZAXTVo1kSbhqotFOkwct7/U8vU4R4QmYmAnR4p3HG7GRZiUapDFe0cdn5FFjadnGxPVuI0AzxDasJ7s7FEkYrAzVshLNxYliW3aTE0vRZMWKQnXSwJlhbzt89lTJk2MmpgFVCiOFMWuitiRj8bb99RYDKSp88uw2V8Qw4v2Sm81zGU/bJa8vz+lCT+sW6BTAjmjViH1GFUyR0aiy8gMxRmq586srovqa82oDRWO1x1pBVGpd8FOIT06BnAbGErmKjmpnOF2P3D2UDGprNUULn7td1JydHXKwGVjvI5U1VFWmmQVsMxDySCw1jiOsdxw2dzg4WDKfVdzsLtmvd1xc91zfFLpORr05wnm346c8RVt49+4ZlXbCBpiStXJJ0z5P08VORIFaUas7GLPlqn/CPnd451Fm5KBRHHnPEEZR0heFzRqnDdEoSQJMkmAoS4MkEJ2YRSMwfV0FJfx0bdDaYjFYZVF2CgCJiPYgS8Ucc56oiLfj8emSM0YqViPjs9tiwJoK5yzeWZy10wEiO66cEzpkXqUZrvqAud6TtCfZSYugZvTFEqI8C20VaZsVVtWofk3enrMZznElsXIa5xXKlympUp4HAQMFtJFJTLfL7HaFm5vC9U0mDGLPVCqRLGyjY90XvrhM5KAp2hNUoVcjo1KgLfs1nL9MtPcTsUAKRkb9NgrtUAt1zmopDmJShBHCmEgBmKZOymYZ3WqD9cK0DyGSo/hkdQaVBBmrDOSkCEH+TFMSLlmZNoyaPAAjGBxBQxoD80c1aaU4f62wpfDw/YrqNNHetahFw/mbEaznrbcKAU/cF25uBvZO8c1nkedXmQ/fW7J7c03vC/1+I04gk9mlzKqtaBc99+807NaRT14F3FhRNQbtE8vDyLwK5MWWv/e9uwSluP/4iPMx8fjokNfXz9Gh5usXG95bKg4OlIwu+8T1mFjZit2wp5p7ZhQ2MWJVxb3DJW/fa/jV5zfcXGVOThWfveh4vX6Gjprf+fCY8zeX2KowhEh3GfjDD2tO71W83GU+OjMcrizPno3cv+fZjiPPX3aUlKm8YT7PnCwcJyvLwVKzmhvqSmHt9HPQhSEpyqBIScAuZrpcUlaEqPDRQlHkiTGglFAClfWSb6+Q4j8kulToO6bAMrEAX4RIyIk0asaUSNkSe9n5t21EWYPyiavrwNVllOK8gHKFPGoUsm+nQMkZ7cQq622mmVm0GzBOYTHkIKtQO8UI73cj1jhyLDCNukuEOIkJtTHTWSAMFoo4KWTMLmeFUsK+F4BBQauKyhT+5LlhUSl+/0FP0JFAwsQ9xVoyBl8gmkgcM07JqH2ImVmtWLjA1RjJeSBFOWd8VXNwcMB8vsLYKcI5jeTSY3KSlY3xYBwhF1yRgr9MEfJGSax7yCMpBYySLlu6/yxskSzv0cp5yIasIBImy6ZMR7xxGFoRKdtKdAwZmXhoNZ2X4hqyOJjWzmPY4wzSJNmKlAu7tGUIV8R0g1YrjBuJBfa7QOOQ9S4ajQCDVJLnxmtYzCxuqejmiusUGQYRKi+14tAkzMyxqQp92mNoQFsRbE/rkjEGCIliDNGI1uvXWgzoOGezveF6/Zy6OSXFzG57g/YL+qTYjaO8yGOEmSalArHARNBDy4jKGbBVhcQN11hTk4Lizf4KpQzGVNRVQ11XNM5KB4vscDHyZktkXnaR/OwF1ll+8PAeS2chDRD2zJzm0fEh15vEq5sd7bJlfhQw/pxe7wgpQHFUesOiPqVtDnA2cX79Lc/Pn/L6YuTiKrHbT1xfBDUaUubiest6O8Jt/HEKFIRbYLgVnhkZ45MQzeCWXf+Si+0T9nFEF0OlHO3cc7xsuNjBLhZS1pSoKZOdTZmJwa3kws9KxLA5CBkxRyX2vKJBy6gz2+nzkaqz6DLpDDIxT3kMVjpVE6GMGm3KZLOcdqbOTlRGh3dW4BxWwpVykTfcWIxsLpDpT7QNIT7mwBuMyWInRcA5Ng+cqAsOzTkzH6gseDIx9Kzjmu0QqArUkyMiRAkHoRQRUCkRSqEK4xDp9om+K2z3hW7MKAxBCZhkExXnG3h1k9isA/dqSwaCUSRnyDYLxjUmLi80q7mn8p18b8GShxFsYpgoLs5m3ISgNV7CU2IRup2wHbRgalWc6HCaW4hMYdKoWBEbqaIxSIFWUppiqKXYMcWgvCamgDeW04c1oU2Ml56nv3yNnx3T2oI+LqS9YRdGxteJuVaMNrBda37x7AITDZdbcTLcXSWs89w7LvybzwMvvOXRMmKC5uThIb96esnbD+fcub8kjYZF3PHqOuNyz927M1bzgUrP6bbiffYmYN2CZ9fXeNXSNIq37ziubhTqdcfTqzXBOLyGaDIXXaJZyUVaTKY2ikhFdrDJGWUNwSr+0Y8OeH1VePmiZ1GDrRqsGsRpUslraIxoiF5fj7zeKu4ctbxad5isKFrx8llPaxxnB457x547R5rDA0tda+noAsReM1CIOpK1miZgmdoakhHAT1EQyQQLgxbMLtrK+J8k2GIjhaBRGa2kA5/uSzSaGLMEk01WwtVhwtdaJpujYbkYhW7oK/ow8uTbkZQN2hZClqlRLnKJMY21jVKYSlN0om0stpZuUlsDKUoRoizWSnEbguC9Y8wToljIluMgKwJtIKrMGPN0NgvUpxRB/4q9uyc3jYgPU2LImtZk+mD5199YDmvH9+9sIEryocoFZzLRiJg2EolRUznYxIz3cNQqXiVPRpSTvmmYL49YrI6w1hNTIJUAJeGzpuiOWAZU0SgqmfurSQQ+0UFvX3eFJhVxs+gi9nNu7cFKoyl467FaKItjjiK+Bih50hK4CUbkZPpKlInCxB0BEXtq5eR8zZmcI2PsccYyb5aQDevtK/rhSpIo9Qh2EF1UNOgiseZFaSHdKiUTB4SquJxZSqPYqEQcBDO/tJYza1kaQz+v8FahtMGVRA57dCooJ7o4k4XFo5TCOIt1f7dr/u9cDJSgWa/P6cfI6uCY6/1TUizYZsHVdk0cBtJYGOmIeYG1c2qfMaqgrRebhi60raOpJgFecZRSE5JCdYqCFmW+smQsqdgpHlIRVBJhhnaYEglj5NmmI3ffkFLmR3dPmVUOZUaULyyWNW/fP6Zd1pjW0cw7+njOph+E4qUCRl2SGNmGS276zOuLc56/XLNbW8ZefLNqYubLoQAnB3Puro5odEPOMjYyaHTRGEmbIRdNokwzrMCbzSuevHjOq+uBzTaSxgE/rU+2G82+K4xZLGvFKKIW/KjWhaQhTnnUBSUFQxL3QE5S6YdcyFbCQex0cWYt6t6cxeOfbhW7JIigQ5n08JBqhTZO7IdGT52uIFaN0Sg1SNxmKYyxEJJU02WCH83altY2FByZCq+iAI7QRAYqBcf2gkd8jSsRFTUqDez3PetthxtEQYstDBn6PlOy7OtiCaQMY06MQTHsIY2ZGDMhiJUyliyhRxFebeCbMHC9hvtKYW0kKUVWFmc11vRkC2p2h/XZ+3zTWI67z1imG4aS6YOMsb11UDIBeaNqLeNXbRTGGipdiCFLYRZl+pICUiBlCWwx2ojoTL410XYUiCGQ9oUyaBIJp8VtYGcFf1wTS2Gzj/RXmXE9Ml85Dt7S3HxeuHyyI5mGuwdQD/DJl3vs8y0PT47IrzPpqOHkAJq5psmaP/5iR6wyVWtZerBZcXR3htWaH74/47OXW340v8O3ccPRsWGhZ3zzZMty1WBNzfFRxS4lhtzy/Okr/tkfPuCnzw3rcMFj07BcLHj+5iW/+U6Nouf1Syh7zezIEjN8fdnz4emSq81AO1NsrgztPPN6O/Inf/UGWzveu9Pw6etLHh47jlRhl0f++hdrrBbOQj/2HLQVMSmunkdmK0NVRVI/cDivuXo9crjwLLTjpNWczg2t1XTbzPqiEKLArbTLYq/1Qr80Rnj8TttbxS6FTChaylgjEcdJKapK4bSlD5E+TIcBMq9DISLYIo2KdNsaxsSy9RweCJs+RPAU5hN6N+vI8yeB9Y3BGSGeKjPZV1WRLt2o7+K/tTegI21biVVxKkBKjChtGMeENpb9uiMlg1UCwbGVIidIOTIORax/Wor7WBTWOWyJjINAkMw0UczjSMo1MQUiCh16gpOCeFQH/GxToW3hw4NOQs5iodGO0WbqpOmVJg6K2krksDaa4+WcVTmDoqlsxWp1wHx5gLMVpUy6Jh1QRlxYA2uGfIHKGhcOMKxQegbaY6yl5DLRQcVyXIAhj5DBKiMrVONgWk2gNM5K0FLMIt6DTNGCqNd66tiVyLBN1jAFMqmJNaJRIiC+LQysBPj1Y8e8XWG0pU+XjOkGh6OiMCZQRZOUlpTEIBMyuS8gxohxHmM0rfe8aQZ2cZCmAzhwhjsLR6xgUxnGkighMOzX5KRwqmZet9Te4XxFoMcqJ1Po2/SqX1cxsNm8ouvWkI8gFcb9BqUU3WB4/eoNqQ/kMTKWwvVlz52zJadnHlMGlLagwVZCedMqo7ImhUyIwry3VSYXgyoOtAgG+5DJxU2jLxFlUSwlG0z2xFTx9PWeIb4gh8yH949orWZWGVoPByuHapckU1H5FabbMegdRkW8tbjiQA10ccOuD9x0gWGAMEIW6MD0dpeDYbWoef/hPe4dHqGKcKyKmaw+RZOL8BPybUymgpID19eBp08S3z7XXF8bUlCgElntRCNQRJFurCIZhTZZuARauiojm4dpXCiC9RIhRmGOhxLJ2Ux7/oJSGUska0nZK0jcpnxdEgwVQhLrm9Gynyy3QjlQKk3I01GkOKpHZ0sOmRTTNFIsFKWo6poDFtTWU1nZ4ylnpvjzjMoibFrnQ/bmkkW5oYSR3bbj4rznepuYuSl8SBW6qdgwRjb2YQiEoAhjYQxI4EqWiURlRQ65i4lh1FyGzLdb+HpQuAiP5/I1jEyWUgrMj1Bn76DuvcNudsB5jKwpvL/7GWW/Yz8qSi0/O1VkknL7Myq9nL9mGgVrDBHpALW51SxP3s8padFT8EXomjFExiHTrTPjTpFHeY01Fb6APg80i8JuH7jeGo7vauojhannPP/Vli9+Hvjh7znCGLm5LrhRYre1d9y9XzD1AX/88wFXBT66v+Tnn20ZI2RlGXVAD5Fm0TD2I6po7r/TcDfAT356zfHZis8/f8bJA8O9+y03mx0PzwzBVTx5smF1Nse1c7YBVk3mPAQuLxK4SIqJZqV5daOIOrM6VNja0bdwsw7MFpqX5wNkWM41KmdOnOfqqiPlwr/9s0sanVi0GuVrKlXQ3qOtuInaheK9R5ZXG82YHHfuWj75uuPmHLxLHDSaw5WBvrAbI188j2gjAVKNNbR1EbGdNvI+KxmTpOCVAmASjKpE0VIAeKOpnbhwUAVrjcR07wspFtBhCioyhCSXbkFPe/7C2GVOZpYHdyu6MmCQ2NpmUcT26hOX5yOvX2e08sQ8CK9kErUp4LYdlbhtgzGFqjb42kAZQZspU0D0AMMQKdZzdTMCXkBDRX3new9jEeuKUVN8OuySWGdjzsQccMmhtSWqSJ8DdTL0fSKbiCUwBo2i5c6sYUfDv/1mThk6Pr6XiBi6rHC3eQlK0Q+IjbYkcpk6c7/AKlg0DavVAqUjIV2i9Ehix5CuiXkHZIa4Ywg3kAu1uaHSC3JcosohphwJNwYlVEZlmE5jpp8GqEIkYop87yEavK1RRmzUpQS5V7SewssSJUlxYZTBaC1ZJ0ou/jL9md8xXrQIrXNhAg8NoBMlDRidaYqjGTNlG2hVy1gnxiFQYZnPF5yvN+x2gsFMJROVIVSOa9sz9IEUMzWGu96waBRf2sDzMJJDwloJTCNZDmcNla9YzRfMjKOPOymuJgH9r7UY2O0uCSEL5nHsSP2OYiyv3mzZXO1IWZSNKjm6m55h7nDVDFVGVI4kZ0lKAmok8CMBf2t9Ud/txaWDLXEgFek0DXbSGEz7nWxJKU0RlIpvX2zF/aU0H54dMHOGg8bQx0J2Dfvi6cKObpDP995RqQaVLH3u6EJkv0/0HaSg0Cp9p2S3pVBZw3xR8d5b93jv4X2qqiKrJCPKIlCfUgwKRyYRyp5u6FBKU1cN9w7PWB8Wnn3zDfubjlyk2sxaeOWg0KZgslzEasKN6qlStUpgJiJWK5MISBEjpFhIWoQ4OsprGUshJwGXZCV89dsDQ2kjr/JEXbNWuhClkYjSCa1cVCTBlKYYISnSKBHMuShCiLjGM6saKm9QTk978SIjwJhFhawdURWepwdELKt0Tre54OLNyNWrHbWDd+eaETmnhgwWGdd2MdENmRglu0CrwrwGV7S8VlrTxcx+a3iVDF/0kaeDZjdq7lSRYgoJT9ENZX5M9PdR/hHV7A69Az0OzM2Svvk9XrXH3K1/xmr7TC7sfcIaAbEoJbS3XBIpFQENRUmjE+HmxCK4nSPGIs6bKOREUxKpL4Quk/rC0CfiqCZ8aaQojc9gVWE/jORgcEeexVsVPYHhpcLHwJ2P5oRG8/VXV+yvIyYVDloDleHrb0bWxnDaNHxxseGbby1PXgy4xvCTr68ZjcUmy1Uq2IsbfvS7C774esNHH5zylz97zT/6xw/Ynp/zzYvA3/thy2rV0YXMzfWGWPYU1UBj+Zd/+hW/8a5nrY45XBg+++aCuVO8XA9cXBXuLg27jaa/Sexj5sGp58W6pyg7TRUN26uRPo+MwdJUkZJGHh7P2e0Gvvf9GV887dE6sRsiQWUOm5arneH1ZUfT1litefo6EEY4HQuPzoCi6HTmaig4XZg3itZZXFPIlYJKoWot6GkDeMgOlAFlxbPvJyy5tep2Ok9OmZAL/VC42PbsIvJ8ZyNpfSJMp2T9Xccd9gPHreF7b7es84Zu1MyLRenEfFFj/EiOhmffRsZo8KYQ4y1g7NarLnwRyRHRWKtxNjCfV4IvLtPeuSSY0i1TESHrelsm14DCUSg6E5Nj7G7PNo2ymTFl1nHSymgoSuiCKTsG1dPHzAyFopLsEjNC9hxXNYfzFX3o+frNnv/hZcf/oTT84EFgKGF6LQpYGPaBUCqc0UQi3dCx6bcczmqM02QSQ7hiSM9J6ppcBoaxJyMWwphGmbjFCu0HismkYUD1OzBrWn/GvF6gnScW2Ax7upCYO4fxYvXMMcl5pkcUUNsaYz2akZzKZC+eHFZlSmHRTtqH7yYDE5huaqiUUqgyxS2rMokTR3JKJC3p0AvtaIPDdJmSPHcPTln5gX4fmHmLrR3JRBpjefPymj4MxNqy9o4xF9IYyVEx146TxjNUiesS6VKgVhbvPVXV4s2Cw8URp6sD5tWckgzOtMQcZV3Fr7kY6DsJHlG6EHMnU4Exc3G9JmUHJXwHYzHa0I+B9X4QEUpMKJenyxymDTgKycQWYYYI6GR9YChKrCU5MeVme7GPTIrYnMLE/C/konh1MfKL5praFR6uGlZtxT4Z1p0lK+j7G2LcUZsKrTU5F0IWCuC2z9xsYbNNDEPCFIs3eoJ4JFLKrOZHvHXvDqtFCxaUsuiImJWnIBuUJpTAZrzkavcMZ2rOqvepZzWLw4a6URgzopKgRouadv8U8aTn/N0eU93uubQi3u6sAPL0/WdFSmVyFiCK3aKJfSJmR7YOqwrGGok2nVwF4pWVoYd2GuUmm4staJ0pSnZbKU97zyLK6JyEwqe9RqVCZSyLds6qmuO9x+qAKmtCnxhiISaNUhXGNVTOMWD5Nq8oe89m23J+E+i2W947DGQCG+WkA5KVLPuU2I0iYCq60DrNzBRqVfDZkLJlUwq7AM8GxSe7wpNRs8tgTZYciuoAf/c++qBhf3yPfvYB2R1hc8csBWy7Yuiu2fRXPFELbswpD/wFhyYx9okwMhVm+b8YH0qakTJ8hzjNKU2R0iJUIhYMom8JJGIpjIOi3yfSyCTKQuJ1yRQCYXQYlSfnhGExt4ylELuA0ZZyYKh14uWv9nzyWeF3v3fIwfGIszNev9hyufYoG/ij31ux+Q+WYbAsZhVP94WjQ8PFDuwC+hQ5mDU8u+jYbgeCVty7v+CrL59xfDrj8IFiHAeqmeXTX26oWgu+8OrZJe+8O+Plek/2C7abTDpKPFhFPvm6YCrHwSIzpsjT68QMQ+gi6sBysx8wVjOMirGXUbyKiqqGgubDexV3lo4nYyB0I2HfszytyWvNsB64utH88U8HlqvEP3ro2XSKu23h+EFFWzvx03tNCgPeaA5mmkWrqStN3UBdK2pXaKtMVRts5dCVWEqNKRPtU04mEQ/CGDJ9FGFa0dCVTIgQo9iUJT5ciw1V5AfElBm6wtHC8fGHNcFFbm7AFkscNatFkrwMlYkXAtC6/YsJJawAbhXmRSY/JWXBKHtxLZQcZF9egCTj6hjk83fbJOuzKRMh24GiIfTCa3HTDl1juYwDYxHLt1Kiig8lS1w5jjxqXDG0TcW+KyJuxnCwbCgmcrW5IA1bnlwq/t+fViin+PA0SRpsZEKldvTByFQmJMaScSrRWIV1gVguGPIr9uk5We3JsRBGTYpCic0lo1KNSopuCPRowjiwG69Z75/Smgsenj6mXR2y7gaevnpBPwzcXa04XlXMG9EZjKmX4DJfcbPfsO92aBVp2hpjwOgodnFuizrkfI4aiRiVy8tgSRkMiawhGSPkQCLoSC4RnTVKj7hs0OWA1DS41RJbz2hsR1rJnTTEnvv37tEvI69eXWFJUBW60pPHwBClSW5njnFl2TSBkArLommsZbmcM2+PqOySyrYYA2PaEUIiAWNKE0/i11wMpKAYO0M9Kwzjnozi4nrLro8Y574byYB0tTEldl3AWOHPa+FkfmfLktc2oU1CK7DaTkp8/nY8rgvWyFrBKHmQVRLhS1GANXJIa6FlvdkUfvW6w2nPyazGGMnh1nlE5x3eZIz2pJjIemCko4+Ffp/pd5kwIKuK6RJ2WkPRjDkTtcJ5hzEalEMpj9KKhCZmiQLWKhLCll13zr5/g6Jl7OZs1oZvnl7wZtMTtGRYixJVDgGlZEpSbl9CRD1cyJKNrZjgP6JoFfSlmn6VDjVrTQcQIlUpEBPRCCXxNrJZY6ZoXUFmqmkXZlxBGQGkpFxQyVAo8iAV6Zi8r6iqBmMqyAlnFLNqRm09yURC2JP6Hd24YdOPjElT2zmLakFdH+CciJByB13w7NWM7GuaCkrsCRO/3VDoUSKoLIXWahZW02oRaQ1kLlLmstc86+CbbeGLTeHFKKNQrxVVhlopvCpoG3H6mtn2nLz5lifhjPM8o64bztyONnzBIr9EExmvI2GfsQtNtczspi4+pIxKkkqH1lNkrBKCoE1yIaAB+93Y1miBNmkMJC3dA1qQsVoOFmVFk6HRIkwLhZQ0u/UOvzW4ZNl+M3C1hic3I+ebkYezwm//RsWdE4f2PTfnmjsP5hw/1HRD5PM3axbtjOt14Hc/WnLzn8+53CaW84b5KnO+S6jWc3U5cnJSc7XvOTqd8Tc/e84//gf3+dd/9pT33luw6hONTzSN5/qm5ujokINlxcfvD1y8uuHliz3vHK5wlearF1vu3FtBCIRdlkROH5kpuNkHjhvH+UXPrJqx248YY4hhZNVq1luN846zuw39kLh+lTiwlqXTbAuksRDzwLt3NcdHM1Yrx68+2bKYeU7uVPRdZBjhcKnotyJMrYrFZLBZxrSLxrA6NtRLjWkMCaHwqVjQ8baYFn5/iBLBW5RonDAQiqHbjex2hX6KnIUpDa9AGApDKuQA9w4sP/yoxswVzy72qGwhWpJKLGca4yDsHcMNaFWBisQs2R/iexcQ1W09kKcRrzXQ1hKKlpPgi0UTJM/iMEohc30zkvMkYi2K0ii6kBh2o5wZxpDJWK047zVDkrWWNgUzuQ+MLZgoyXyqiNYglh6P58CvOGznbMc1V9s33PQDy6ridTL8j19o/ptQ+O37GWUkgjybiu040roGFR317JDTpqayazr9ijhIk6a0wemWnj2hjAxRwSg2Qk1BEVBJk3Jm1+242XWsbyJht+f1+ZbZ4Qn7LnKzviSlkZvrS05WM04P5kIGJFBVDZc68fLiV7x58YKZqzg8OsRVlrOjJaeHR1jXYHQlgsw0Tf+0mdDjMIk6SFONoHJCkyYXyf+ftT/71exK0/yw35r28E1nipEMkskhyRyrO4eqHtylbgktCwJsGLYEoQ0bMAwD+rd8aV/YgC8sWDbcBXdLarW6KrsyszIrOQeDMceJM3zjHtbw+uLdJ7IM3+iCSfCCwWTEOefbe613eJ7fM73fJAxCLnNGdwtZBXwjyJQIGz2kUdc8i8WCi90zUhWpXSA0FUPc0cVEHC2+gJtbrheZNRGhULcF6xPJDRzKlmEcYTTYPmtjEccJ1qdgJg2r+i6LgWRIoyUcefb7DalYuiFja8Ea9djbyaNmjMIsUrqhQ3kmueQfxRlGqW0W8+afsVqVqXpTEZLOqv3EOxXV3cR6uuApXsU7bqLmFed43Vme7IQBGJLF2kKdM0EGshnJRi0oKQspq80mJkvfR+KYcDbQ1g2V84yxI2bBW8+YEn0uWNcSmCESFGEkiSwei9Owobgnx4HYNVxeCpcX3/Ly1cDFVUffy6SLKHoRTz8rM40F9Z9vfmEqCmSi3hkF30w/YaYfJiAUJxgqctLPIoqQYo93jpC1u/XOErxogYOGSxUJ085UoRsyHSAx6aTCWEPtPYvqmKatmbdLvAvk1FHMDmO2RDkw9oVOOtK4YdvvuN53xGiZ+RmreknT3qHylsq0BAJj7omy46iFucmUHCY8rCGZMimpPcdVzcprUdYneNkXvt4VHm/hVQeX0bCJhj4ZvPG0ksBo3oX3CSvn5OtXtDZw1kLqIs+eOn51pUTFf3Iv8c8/gHtnntz1vNhkrjrP1SjUAtVMQUFDp/vWNKJpck5HzdZp2BCiYTCiiiJc5QlOcxjIjpSEMeUpe95M74J+3rao/sA14FtPiZlq3nB6z5OM4frS8ruvErOZ8L2TQHPWcogH/uo3a+4uhGbVEiWx2e7ZXsBmFN69u+DV5pzffANtYzm4FRIicVc4XTjd8c8tXRQ+njuu6FlUcLnekXLN9Toxr3oyC3aHAw/u3aKp4OX5yKePBzbna7DCr75eg9PO++HXPYOMfHh3Tt5GhlwI1lJM5nKbiM6x6XrmzmN3Wuzve0sogXUp7IeR5RyqtuLrVx0vH+0Zu8KisdxeNZzdhkevPV9+2XHWwN47Pv+m52wG85OACVDNKioE4zyuslRHlnBc0dxe4OeWOBwYdpnYZcohI6OO5nVtpuu24vSgxxkiQpcLQ4FdzAyizp6Siq7qsuCNYZySD9+54/npJy31UeblttAPhtpkSs5Uy0jbBnKfOLwMbC46hiiqT7HyZkJopwyMkhUOllNRV09laVqrIB6xZLlZIeiFNWRD1xe2uxFjHHnUGFfjPd12JA6Jytkbdz3FwetBrdHOoAUGYRLKgeTCYdwCwjB0iE0Eabg1O2ZRN7y8fslhN5B6Q320xAXhvC/8vx/WRCP8yZ2kHfmk/Woqi62VIZN4wS5fMvYDkhLBVNS+IZE5jElH5KLfK2KQkrEmYkxgjHB96NntVUAseWS/viANhcNoGMuAyMhuHIndnsurC5wITW2gcexjYX21odsdCJXHb56TcuH9O3f5Rz/9KSfHFc6AN5qT462dYqOnKaAkxEQwSna1UijWUozXMDbn8LZQyYxoW7INWB+xjHg7I+ZAygVLwqNrwrbxfPDhu6T9SCnCdZfYx0JJFm/AzGDn9c7yRqAWskQO45593+mkcWqki2jw3RitNr1psll+l8VAf4gcdol7929z2BlydrTzJclq9GrMChoyqL1qPq9pKoezebJ9aAfvJ+qSswFnHBa16N14dq2ZrDsGDeHAY4yCNUDHN5qrMy0cbooBa7DeUNU1A57L0RNTJOU9krZAR7T9VFtN+BgxupeJQu1rquUSaypmzRIrhXHTk4xgMRx2Hc8vdtw99izrZhJ8ZIw4KqORzMPYsd0PrNeGi8uWVy8zr1/vub48MI4qmvST3FzXJJNIDVXq31zz3CTgcVMTTFuf/79pz7SnloIzGVc0vQwy0arwraQMruCdpdSiP1sPJRZ81OAnmxTMIWVieBetx2tfM2salvMly/mSVXuCkcy+37FL5wxlg+CIY1J7TRnpUmafo9oes/Ih6m6gChbvWoIHcQMpXbEMPY0U+uIY9MukDobaCSvnaIMnS+JVH/nDWvjDpeXpznKVhNGoyNIbS2WN6iWMYZSkyOWg4UTzBubzGQTNSxhjJHaJwyHRnziaukGF3xVXKXLwwiIZXp9nqpVltTD4Vu2XknS0m8YpgY5pp2GYxogFcUUPa+MZiuBEyE4oriCVPgM2G3JUxbbG6CYsnsopAe/k7ozDJnL5OnPy9pK33BZfV3zwoOF8X5HPR3CF+TuBkgpffL5n+7rwvXsNJ99f8OzJQEL49tue+z9dUd29xavPPuc01nzwbsXV+oAxLSdHjicv1/iF4/27DZ99ccHPvn9MqDKbne7YLzeZ5C9xeeT1uubry56hF4bouXiml/vsJLCotIvZ7SKHXaQ5cQy9etcPeyVKIgkXDa2zpKTitgczx/UQ+e9/s+V/+rMGQQg5sqhgvqi56HvCUcNXL/Y8fBFZngi3jmsePo4czw3bYjCbhGuEYR8JOIY603thvUmUfcY9E6qUqboDIQpWHMYVJBSoHOI1LjZOxQDOaDCY1WyM4gSxSvNMY1GC30QDzDYRnOHt2w0//EFN00S2h8huD0Y8ERX0No0ljY7Ds8j+leFinRkXDtNMEmWjGpKCcLN3MFhKUVulryzOKfUyy5RnMGURpCxE8ax3A0PU/y7nRGgMMTlir82FGBUgU8OBzHac4UPBjnAYhZITNivL3wCxKBzNWvDF0/oZ924fkThwGA4qhPMNVV0z5h1BHFcp8G8ez8hlxftnr/HsGQdlNWxnic+fvyIlGJ3oOs05shWG1JFzISaDmDAp9XUXnxHAIeIZBiXMjqPDiLCYB+ahxiSNJB6sBQk4E8g5st9vMQa6ItSMBANHrcUTcE4IbWZ/iLy+uODRsxeEUHP7aI61QiQibkqQxU8Oqoyz6kCwomLxLCrKDhacU9JgLieYMIIZCbZgpCZRMeYRn4u6Q8j0FFarGVXdcnl5zcvn5+zHTBwLTipcDbYWzWxBxcvJGJx4SjbqaCtZJ5OiGTmlOFIBisOh79l3Wgxs1gOH/QEo5OgoeObLmiR7TNIMeSn91Lsq837eNNRBCMZgjYY+OOfx1uOcxxiPI2B0A6O0wimn+/+3bbZI1nUDTAfvzcU5ebsrH1g0lqP5jFBVDCly6K4Zh+eUck1vruhFdeXeWUw5IidLTlcEa6kWZxSp2Q8jffEaVemERRNoQoUxmVevnvGwXnDv1gOapiUYBVvk0hPznm48sNnD9aZlt592NmiR4x2kokQxe+MrvREL2fJmLKjt5aReMpNCfRIA3nzT5ub/8+anUJMl0jSwrDzBOTqx7AZV8C6C4J3T3ZjR6Yxz9k0kqnNe95N4KtvgfEPtA20TCMFqipoLNM0SRyJKwZoNknYMKTCkRE6FUgySnYYsVZoKmRgxOWoFZwdS6ikl0vjE8czgi2FEu4jWWJbO0FYO7+A6FZ50mc+uC59eZ151geAqjuaF4hJ50OCXhNBnSzbqk56TqI3qPpoK5kGpbdtoWQ+QZPICWIMTwWS1juasYBZEeepX68zOwnIOdchUPlN5Q0ieHA05CWWU6WW8AZMoyMVgccUwlqQ2rlojcq3YN0hjyfoMlAEdOcSCJM/rZz3eZDbbyPgOfPxhw+shk9rAsO74+Ect27zl2/ORdjPw9jJwQUMVekKJXF13LFYzvmc93xjD027LeJl5cNcRD57bixpb6QqkHyLvnjZ8+oeB6tQzSM+zlwZTW96/LXz6KPP49ahAGlOoTMFVFXkcOZ4Z9rvE4dxxv1XL3MtnHfOF7vF9cIxd4mw+Jw6J0DiaXnfkMx9oimEzdJTiCM7x268iY0z8+MMZx0dQQsOvPxOePN3x8rzn+/cb7nzU8usvdhzXjg8/aPjVpwdyA7O54/mTAeesdqE+KQOgGAqWqhbmc5i3gVltCE70ObUy0SetRgJ4UTy11QI8Z0tKlhghj0m96sEQgkVq1Tq9db/mo49qmlbYbiObvZBjAIQhZRovhFyze57pnjqud4ntWgWMdrpMQHfUBg0yUv2JTOqqrPvrAhRLzIJUEwNFFMPdx8J6kxFTUZKGeVF5DutEiRZjM8mi00Xv2MaGITcYu6NgKKKT2pwSJukqpGRPjCOVc8zMklurOyyWgZebDV2vbIR5FRCTKKNOgI33HFLFHw5n7HzgjnuI7yOdFB6OlseXA7eqluzR6WMtDIyUrIJIZ8BO8xrn9LK1LhPMMZQjKK8x5UDtLW0NldeVDTayCg1te0TdzLk8dOz7FxzVE0zKQtP4SVdhyGNR3kyytK5ifz3w+99+zt3ZKT+8/yE5WIZSKDmp5koszgQwNzecJkBmM2IkUqEasJfrPVfrgYqaZuGoTIeTQMqBwQpJBubTdDGLI4kw5I7zzYbHry7Ybwf2/UgeCk4E33pFVGe1j1rr8OIn/ZfVwD2jTrEcpxUTQu0Mlbc0OKrvejLw6sWao1VgjAcuXq85uTVnsTyi6/TSzoyUoh5VZz1NHbh1csSs0Sbe23rCaQas0S7fGIthEtNJwVidArwR1XATG2uIMZNSntwIegBbIzgH3lnms5rTpqGpKnZD4mr3kmF4jpVLxIwk5zE4XGkYDjP23TG7HowJZOOIo2cYMqNkqmBpqoaj5W3FhRqn6W4u8frynM0h0c6OaX2glJE+7zEm4fH0PeRkdZSYkipZpZBzVEiFqA3JOo0lvNnLmzd/aUEgZlKuFj2Y4EbXMukq3ixZIBXDrBU++d6cd09nBDFcdyOvNwes8xwtAtiKUirG4iAEQutpWkdd6c625ESoKmZhQXANbd1Seci5ZzO+JpcOsT2zpuaQDm8KnVySdkxFFdZWDK2dDlNzUzlHBq/fgMhIKsLcBYKJFDE0qLV06fWz3BjDZV/4epP4agfnQ83gM6tTS+MhGMeQHZsUOYzKAkhBI0OdVdhIsFBbQ9MEamfoY+KyS1xHIZJpQ8YGh1BhxFJk1LZJYEjQdYU9gm0Mi7nHewiSoWgxaqxGEZeo9p6SrBYSGSrraJ3DiKbjqavJ6BEyiUGNN5jJQmarluL0EA7OQwXDujCWGVfnkatXhWITZrUnlcTTF7AylllxLN42zOrAcidQKioj3Jo7fv0i8Xro6aLh/NMrPp61bPeJyg+8/Zbj5bNIXDpunc7ZjD2lcfQm8dWTzLfPMz/8uObxy46coKo93ZBgVjhsCk7gZCY8WQu7JKzmlvUuIWWGay1FDNuN5d6dQuU9sp0yL5IwrwOpH6hMrQFCxnLvqOLqfOAQA0trWLmKbd9xcdUzdxYJkXFl2HSZzecD22vDzz6uoRqwKTJvGnZb2A6GZqYrrsZ6qgD1TJDgGa3jumScgZNWqD3Y4LFugm5ZQRwUaxgz9ENm7GHoNXLoRnjrKhX2lgIyWnyAamkYReg3mV1n2I9G7WWj7uKNdXAOhwvDdq3v5uU2US88rahWQIPT9MwrRbSDz8riF1NIeURQhXueBMGlWFJKxOjZ7g7s97p+ySlTB8NYoOuTNlsmI87jJJPE0XMLHxyh6ujJiFUBq8TJlueBEpEx0c5rvLPcOb7DMCYOQ8+hS4DD24oxW8bsqZwnF+3Ir4ZLrncJM1ju2cDLWPhs9Do98w7MiPGaH1gyb7IhjCjXRfkmlhIN1szx9hYiOl2a1Q6soaq0V8x50PwSyVRtomo9Ie9Yhp5Z45GsTqAxJg5ROPSGPmZitBx6R4mG4aCuBukyd+anmMrSWMt2qNn2HSmjX7stGmsuU6qkEepiMeJ5cb3m5cUVc7vgwVlNMFY/M7ET4yXinRq2SrZEI/gJHudiJh96dt2gFtBRcEaYtQGC8hFuZscmObLpkeLfwIpyEUo2WLF4D60NzI1liVMN2XdZDEiCW6cL9us16/Wa2+/NWNy2VMMpY9cxJEMIM+azhnlT0zaBs5M51QTDsaYCM4nnVGs9ief0AtSNtcPZoNZCI5NTQM/oGDJjMuRYKKmbKraEtYIVy5gjfVYl9OX2Ff3wCmc2RFfoxjljN2NMA9vrQt8bEnuK8YhUSBbGsUMKqqwVR8SyjaoCjmOhCRV1bbjcZHLeUFcjq5mlChkThMoH1RmMmZRGJCakL9o5xgRZ3hQD06Je/2cyb6KDjJ2GAHqZamKhevdTKVAsDk8yOkJzRJLUhKrj5z844s9/csb9laNkw6bPxLTASSC6GcXW9DGw7S2jqXGVx3nwTnQEjgMXqH1LsDVtNacJjq7fslm/4vX6Ka+3j2naCqQHO+KD0eS9v8NXcNO0A2MoVu03tui0ZyDqpEQMMRVed7pDX9rCzHpGgfUIzwo8ORRebDKH5GmDYx6c2sLsNIEolizKIjemUBkhoFGy3hiMc7iQaYJ2UOMEgrm/qlktHNYZFjOZ6I46GYhjUV1HccyaitMTw+l9x9HcYqNBRke2ChhKqeiYLoAJysAoSShjwcu0NRCDw0K2GARJWtAWbuyIVsWvzmKsEFrDauWwQXi5F956pwbf8vBxx+q45c5RYv9t4fpFZB4cBzK7IZPNnPZo5P7ZMdvrA9+7W/Hics/5aAgvI99fGlpvqAQGN/DwiSObhnfnhq9eRo5OE0cnc/qcuHiduX27obaeh692ZCnUYYavoIsDFRXBCbmMxMETUyaNjjENVCkr8GSXKamw7TMuBfK+EGZQFc2MN9HjZ5lun1k0lovXwknr+fC9QL7uOVpUPPpizateOJkF3vtgRX514LMvR1azyA/uz6gXjq8eRppgOD51jB20wdH4zKI1zIPTaF+v6dOMhpI9fqIjmiC4YDRcqBKiLSQKQ7b0o6EbJ7+/EXxlqGpDqCzGC+KEZAWPJ1aZb18fuNjUOD/ZDKPRqPAWTKm4/CbSXQips6x3kTElxiiYQWhwFFswGUy5cSqUSZxiFJBFQUrAY0kURmtpjCGXTD9YDofI66se41tdJYxCqS39IRNzwgaPK45kMtkWOlcz2LcI9SV1rDhUIw7BG3VyoVldFIE+D5yEYyoZCSGzH/dcrTv6bsQUT/aWrk/6My6BKgZyBV1ec0iv6aPwWjxfHTJ771i0jmgGMFldMiWoW0C0WUFatfF5wFZAwIjHSU2fHpHynroB68qbdUYwljokjMmMHBi6a5wVnM8c+kLslQsuRhijBjWlmEl9IEbVVyhZ1PD06oLXm0tun93hqDpl4VYch55DHtimPbu4nzgQmruhKZCGp9cHvnmxpc3w/oMz5nPLmDqd7hhVhltJOEYyFWW6A1wxNK7m7Vv3GTvLxcXXmAy1daxmDc2RoYSCMZ5cEqSiOQySsAVscRTrqI2h8Y6lQOMzc2upRqGOA1VK/I/53//oYuCDH7zP6lh48u05xSTuvXvE4tThZEEpgUhNU1W0TaD2Du80Gtc7hzF6QU1xTjojUX0mN0TJwrTPsn4SuWh1XIpW+tlbxgQxRGQ0pDQSU+TQbbi6PoemI64q2hkMZSCLwdmW3SHz/HWgJOiHis1mg7VC21aEIMQRMA7vLWSj3vxi6MZRLxqrWQD9vmgSn3cE71gZQ92esJo73JRtnUZVb47JMEShj5ExDpSSsDilGE6CyVLK9H0WVfejiE2ZViXBC6vGcryqCUHoxszmcmRzSEgIiImYImAz3/+g4p/94jZ/+t4JM5/J1lGKMgQOe8MmzhHXcsiB694ylppi/SQaLKrTMJ6MVRqZqQm2JVSQZaCp4Gozcrm7wHihqTyztqLNaq3Kmg2t+gejeNgbMIdOXD0kFWm5SY27k8TD4nhlYZ7B9YYshl0StgS2g+dAxNcWUyWKM5iJkEiBnAreCZXTn6NJFpctuWSSE8ZawGq2QDaW6A3LI8+PZzOwhlwMt2YJ4yJF9GL3QVieZk5PAqtTx/JYCDNR3Op2ulQmC1ia8NFF0VP6cnsDXpPmXM4KkVIGDbPaUTeOPBZK1hUBYlUUazLjCJu1KtJtZTm637A/ZLo93L9lOfu4Yvs4c+/2nKOTik//dsu3X+85PYIHP3+P8+dP6S4vaeuaGBIf3vEcKkt9FrjOkcsuUs0a0iDkYJktMucvE4tFIBjPbme4fddy3UK1sPzm8w3Hdw37rSHRMewd1TLTVpbNYeSQa6yMnDQBukLVaJJciSO1qaDVGOt+Z7lTC5sdzGuDc/rnd4dMzEqDu+oT755aHj3q+emHc2LQeOXiIvsE2ycj/Zh5cKtmdWI4WWQeng88fQnvPqhYHcEmZm6fCvPGE6ydbLVgjYKfZm7UqYwzHBJKiUwZl8EkyEYpoLFkRHRtEFqLrzXV0Hlw1cQhc9MzXjLeW71gRqHIxPIfhbNbjmArzr/o2XwTsb1DjGhgm7VTkaidcDEGN7mCcjFvIrK9U11KRMjF4UxhQIjJ0wbLpjuw3jg6t2S0NdW9U9abZ8xiIpuGfh+J3tGKOolgwPjMeTwi+1OsvSDGqO+VnWilN+Aea4lj5pBGjsTShhlJdqz7S7b7HikFZ2qicfRRpycuKyp5cVJRlgeiBFxdGGIhCVPjVlNMgGRJYnGywJQFaagZozaN3nmSc4jVdXSwibrdMIxr/VqDwRshTJRTZc4UFfUaXdsaNKAppkwXAacofIultmAqKFHx5oqNF8YsfP3sBX949CWdjFRVQ+M1mG02WzDPS9b7K3b9hkOOmJJALN9e7fny+TW587x/tmI5MwwUIgmvn6ymxorDFY81FWKSlhPW0KeehIBzmAwVhlBbFsuG0I6KkBf7ppFyToDAXAIzo7jnpTMcCRzlRGUSEoXYj5R9VMfMd1kMtKeF7dDz4nrD2++c8OEHDxRHbALer8iuZcJ9a9fzdyIauXEBGK12zfQhSvnj3lsopCTgdBQGuh6QIpQEJTucHXAlckhCPW8Z14lXF9cM+YKzZaayHpOFw27J+bXD+cJuJ2x3mkwVk6DDbY9Iq1MHm6eMeo3Hnc0Cfd8h5aB4Yc2bAQfeKrmrCoaqDZhQEa2nmExOiqeVWEhdoRsKgyRSTpMn+SaCWChk/TkZQ8Fjs+YFlJKn8VfDLECoRt69f8wnb3sO2z1PXu359cMD5wdRi5DJvP225z/8+Sk//d4RD06X+sB7sMYhFDaN5ajUJDvjUGpWoyWWiiyePsNY9AC6oRM657HUIEKfL9imb+nKa4o74CrdlpXJcWCtgFVltcgfbaVMnm0D6nc1gpWCFbU4FgydcRyyYS3QRkO0SfnmRpMXgxXaKoM3mLrgjaEyhpnTS78KkJyQxXDooe+FOGT6mJj5zFs+aDR8EaKAGM9iVhNMo5OQKDTGkbKFEJkfR949dlSNZdGqO4ViJqugXgjGFMaukDo0Hlayii+jstyLqFNGn91CLnlKlhQkKtmsJJnCjLIijIGSDN1+5OI1nN4P3H03ECTz7KuBTV/x058tuXyyoX9dMVsK377omHvLg6Xj7R/PiTLy8psd64Vl0Xhex4FbyyX3t1tYOb54WLh/z7M/FM7mqq84aQIbKzSV4foKtmXAXERunS759vUBBI4bx2IGvi5cxIpgB569HpgiQ/AV1BTGEKic5WoXOQpQyFSm0JgGU2dUhiHUvrBdG2Yzx3qfaHDk0dLMErMmKJ5Z4PPfXiIOhgGoRnbXWVdbK8OnFxF5nnh1mcnU5AxXryPdrlBS0TWLs7gq432h9hpiZv1kBXMT3wMNsSkUUlGap+Bx1igltdLmXBDNFJhWPSYB1kw5BoVQLE6mSECBTKJuA9XgODyO9I8ibawAwfgMwWK8Ata8zZiidDyjMoE3fxtr0JQEBXBlEcZi6EYhjpliCq93hks5o1+9xcE4MpG4fshJE7jqIGZDdbOuMhbjLfsYeNa3nKx6gqumohSlDmZdwxpjlH8hhWGIpJRZnM1J5cB23zOMOp2rQ0OmIZlI1w9KfTxOVKvE1l1SJYtUhuAMhjkpzui7FisLjakvI54GZ4OKf3PGyIizo6Yz4rDF0FYj1rykz2t8jX6mxuv5PL1rRabPZCrMFRWvv4YVikEDmZzDIrhQCI0h3QidKXjn2W53PH7+Fb7uqBZHVGHJ0i05mh0zrxvuzm4xCzMu+ys245rHr9Z88eQ1qUu8tTjj+HhOsoqJt8x0ty8Zax25VBjXIrZn123p+55UhM1hw/X6mudPL0ljpG1mVNZRVTrlzEXPseCgDZ4aCAgnOG4bw60qs3CFOoMjkSWzS4ldHOliIWf33RYD4/6a9bqnbjz/4B//jOXyiOGABmVYzdXWC0Cjb42zf7wYjMYE38zG3/jl/6gUnEb0hmIdgq4PciqqfgcwicY5duuOh48fce/efR49fsG3T19zcm9JkRW7sSN2lsfPIxe7LdaC5KDCm1xIJWFtSwgVgmGMupvHKOZWDPRjpI+jqnrRDwFUZFeMCvFCaIij4fmLLdY5QtCuvrKeqgQWszkWRzwMHMZOp35SlMUgqo0tf2fnX5yq0y1GFb02U1U1u8OB5+cH/sEnt/nBHc+DOxWjFf760459DJzcsfz5z4/55Yen3DuuaduGWfATPEhfriUNM2DEUxVHUxVKGSkUdtmzT45DjEQF8GOy4Eom5YHd4TEXuy/YdhsKI02rzg8rTH+Gip+M/BHTWYwSDO3N0yBQbAay7jut2jlLRuFLWHqM5pkXPbCLTbhgWFQqAqy90eARLDZB7IVxL+yvhKvLwnoDu0Om6zNDTpzMhW7lSEltTS6gyYkScD7jDYwYvAffJMwiMguZ1k07yyTkqIe7xWCiIAm8c5Rai77UCSU6JGsxIAmdfqRMyhMLPlsd62UhR000TElz3IekxW8qaitLUqgXDcen4Mn8/i87dhtDfV/oN4XNReb+HThcDDx8GHnnTsO7P7vFervHvrzknRPPtfdcbXqwmW/6PR9+eMpnX71gWRkevTB875aDkph5eHmd8H5EcsXZScOL55eYZkZrezZb4e2zwH4jnK0gmYrNIXKn9XTbxK0HAWcyG/Fc7iJ9gpyFmUBlPdfdyFFj2ewNi5lj2ESWlaHvgARjVziMhbqG83ViYSzrDYyd4dnrkdOjGYt5YvusY3Grxt63bEfLq23k5XlmOxQ+vNNwtAoEV+j2SvHsh5FcIk1w9DmrW8Z5FQvaSV8UwAWDdXpxZJn286JiWmN5Q/kszqridupnbgR7+lxMtOBp6SPZ4OqKQUb2VwV/Ca6PHNceakV9WzslggaNv7bBIVm7S8M0EcBMxbQhUXDeEqMw9sK2rxTkZgvnccZTVsTlB7zcj1xcv+ItGbjjDIJnvx/ACEFg8EJVIpts+Gw/50r2NHJOFWraZkbf9Zrc6CzR3LANlEw6DhnnPO2sYtNdsu8iYxIQh7c1Yw5TZ67BOxIMl9trWFqWlSGbgiQDZoHEE+JYY6WizyMxR2IuODdgfNKiOQecKxgXGVKhMRWNU36LM+BrQ+U9JEfJ+Q2HwTqLrxw5Z/2ZOj1PvIF6EmzbAlGKai7cNO3xOmHMxWBwHIaBJ6+es1gKq3TA+x2d37HrDyzmR7jgudxcswyBrjc8enkBEc5WS24tZ0hw9F3GEZBgeb49p8TEnVWLKYNOeLstL65fs133lGjIcWS/23J9eUATfQ3BNdQ1hFCYBZ2eLrxjhcXGQj1mFkU4c5YTSVRFU15HYxiLQ0okASOObL/jYiAONeOY+NkvPuLDj98lp4GqQil9VlX108fyxgYnEyHHGfcmeAYm37xkpdqhvHAjgnFOq2KEnG/Qxfr7xtLT7Tr+5ve/45tnT3nx6przlxv2Q88ex7aLtLNMykxpgx6STh9SGjFZH4DgaiyWGAcQgwtTnKUwZSUobrcYtT7iptAOuTHgOkrS8KBYBjAGHzxV5fHtgmq1JHAg7Na8fHFJMQYJ6hmlWO3YRe1nul0YEStIErBeDRQl4yvLTGY8f37Fv/mD8F/8k/f5+LhhO8DV5ZonVz2//PEZ/+CjFW8dz1jVjqbSZD0pMoUnqSq8FkFypmLEuhFrEhq2XGGoJtKiBielMhDjjsO4Yd+dk/IOrGCDdstq9ih6ENrp0o83ON4JZcyN20HV0RVMwlBDQi2ZWQpOFy901hEKNMUQssFjaUNgFtRrgnEMQ6EfYLfPXF6PXF5lXl9k1teZrhNi1Is1o8LBVAyjGNajsLOOURx7DA16gZzNI7dO9yznSZ/LUmAsmDypdJ06EKToOFesxVSGymmSXrWzdBsYe8FMiFvxCskxUQU/blpRMGUZeAPGZe08p+faSMC6gdB65u0MFwYSltmdmuUHNVdPN7xce773/oLLq55nTyMnDRxSZLftuPiq4/13Vnz/wxXfviw83V+zqFt+93DLclEheMYh8f6dluvXe47uVQSJrMIMW2oG8ZSUmTvH9lK46nvuvz2DccBIy3o/0hVhvR+pa8essVxfZ44az4vzyNmqYttHKoF3b1e8uoaFcxjx5DTiSkXOhbmdcd3vaKwnDJbaWWaVZT0m2hC065kF7jRw9taMXRkxITEeVC2dx5H9K8ODWaa67bl1v2a3jTy41zL0I4ddobYVIegBqjCyQDEquIulIEmQcaJ8WkV83xAIQ1BAlXWiwLnKYSs76Qr04rC+vCHRYdXmNQ5QrOAqw9VaePqoYC4jO1M4bQ0h6L46eEcwhsqpwAvrEOenicO0EtUHAmMU1y7B4otj22euDdztEyYmDuGYh/kW59WcyrS8Xl+xPn/Cj488rYfziz0pGbwzFCo8Qqbj29jyrMuESt/2UFUYCyF4hqHXdEjvSSlS1zU56TeaS2Lf7dkdBvZdZBgLtW2wtmE4ZAyWxaLBWcvFfsv++opmGThdwfHJHZyd0acWsiKIc1HoEeJJGXIuWDSl0Yhgi0XEYpLgKnA+UlURZxp9z7Ij5kQRbdlymXg2eRI/ThkPYsFGbT6jKKxJM1h0Pe3MBLWrtCAX40ix8OjJAeGCe7dG6nqD+IqqWSGu4mqz4cXT59w6OmOwYHLi3tEK4y1d6YjbHlsMc+8Qt+XZs69o9paP3jlmZnc87Jc83cDh0LPd9IxjJgQzNaOahAiOKgRuz4XbjeGsctzJhiMLIRW2h5F+NwXOBXXi5aroJMs6nbiLTiqjMWTzHRcDQ7E8eO8tfv5n30dkJI5RFZojOK+WB9UATLa4aeTknR6sWiEUkMkXOSnlpVhVZhsH3oNVz77kzNB1pKIFx+MnT3n58hVfPvyaKJnd9pyxB1ND3w9Ys6XrtQuEStnSWeN7jUnKNggtuWRijjgLGN3x56IXHPmG4qxxvt5OkA9nptGgdrJSlP1tvVpwXFXRVkdI9lyuN8S8RfoNm25LTAmRBMWRssFaRRy7KZ2sIFAMwTiKhSaoEDEDVV2TS+a3n284Wl7wz/7klAenSz5868DpLcPP3j/m7VXFqqqZh0DwGTGj7s8kqPCIQbfaRbvr2ow4G8lYRBpMqRi8J7iWUhIX/TmXw0s2cc0uXwMR61WFfeMAsXqWTfYY0ctyQh0LcCMEsUxJjDKJRiddgREFnVg3iQ2d0FhHayyVWDyGmoCXhlwshx6uryOXm8LlRji/FC6vI30n5MEgWacOxmq2uxUVAx2ycN1ZcoSD0Z3vUTtw/3Tk5PRAExIuOkyc8jG8jjTKNOWy2WMKb2w8IBhbCDPB1Qa3MKSdJ/XC2Ctq2DoL1ZQqOdktyTo50F/TusNMRbBYsN4TC1ALF9vC5XrANrB9McAWTm976lD48qvIflPxZ/9Ry+NHHY/+Zsuf/OkZ9cLy7bMN/Vjx+AWcnYy8d9Ty4Mzz+TeGqnH43HN03PLJPcv/9beF//n/7Mf8zb/+W+7cLmw34KXheyeJ3z61VLuO/uBxc+h6Q5RILYar15ajo8TTV57UKCY595mzeaHbeELjWF923DkVhsGysJD3nlBZQgjcqgP9oBqPInBSO3Zr4XQVWF/0/PzDmqNV4bDb8brPrGY1wSU224JL8MmHS/7914n13rCMhkOX2e564qBEzNVcN7TzuaNyYL3+fGU6GFXbofoAJnKpTo714vQ3WQUOjHeKoQ46TTATiAiVrJCLkLKuClIUnj7NPPkm0Z8njmvLYTXiRo/PqiatdENAl0Zq0dG1N1AXp9jvrHNSzW0pBGvIYtkfYNwVLnPhcNAL8Ru5z9PUYorBm8IYM8+uPa9Oz5jNMhxlTH5FGRSbO44ROblHMidsX3zJSuakQe2WIPjgp6GHQrQk6pRgcjyy221ompGIoR91KmBNrbhcJ8waw2HccMgj111PLoWry4GrynF865g7Jye4ag7icCYCEbGa8mrrEWPzVElbhasZARpWdY33B3y9w/mpyE4OrKVI1M6kMKn7FSKHUdtwEV0VYDRoKOVpVYDRxqUYTNGVZLYJB4S57lj7XebLb9c8v4jM2grrhdBaContZke/6/j24hGBhjvzY8Z8QCTROMN8PqfCsEwblsMlP9ptOCoVH2zXtHPP2jTaFIqmmuY8gDdkW8iu6BTbGY4q+NEMPq4ttxzMbMKZQl+UaXDoM/tOODhHwrFcGnylQCkkkY1lEGErljhxcL6zYsBVlh/86H1mVeCw3RNToY8DQsAnR13Veo0aBXRI0dhKcQ5xAFH3ODkjorhGVaCXycfpycZgXaVj/CKYEskp8+U3j/j006/p40gW3YdnEX3ZyRgyOY1YozGjSgUs059gqGwNGLKkqWsVStbdJtNYTImAOiaGiV1QVVCKomSd1/UHWhB4pwWQ8Z6qbilZ2Fy/ZLs/R2xkHgLHpycs21scdj2b9RV4tRUmkTeOCWe8TlCspakdISiLv+t7orX4toKt5f/zm3M2fc8v31ny/tsL3rXC3XlN23jaymvee86Qo+76poffFgWn5KLqU0cCEkYcwQiNyaxCSzBC13f08RXX/SM2+UDMSS/waaFTyoR+drpblZvizjJx+icNiJg3/m0NVzJTwId2GsErCz5MsaNOoPZehUvFUrJjGBoOh5Z+gOtuZLs2bHfCZm/ohpFcBO8LwRvlRWjKCxSHGEPM0EXhEC1DMcSq8OHRyMf3Bk6WA1Yy7INaIO00pkX1IxR1dZCs7oizTKMjnXBgC9YK9UyoKkceLNXBkHaF3BVK1ANKbtZhRQE3xiihzBBIoxYQ3VjYXQubbUGqyNk7NS++uqLfW2Z1w8//bEncR/7iL9bcOZnxo3+6gjoSSPy9Hx7TtIHnT7c0faI9nvHgwxX33zL4uWUVG5Zuz/17Lb/67Io/+1nL7kp4+/YZ/5ffXPG3v9vwz3424xd3G9aXPZ2zLBYVqwDHx57fPxuZzRztLDJbGLYHS+nh7olhvxbeOi2sN4FlFdkbw/PX0DaeoR+4fcuzfp6YtZqO2R06FtZjswoujyvDaQMPOxgHTRl8+6OGfugw55bd+ZbT2yts5Vg1sLm2XPeJJ1t4sVMny2IO5+cd3qpWQsZEEzKZQHLg64x4bTTs5NiR6RJmej6N0SLZFoudwrDs9Dy7YHAVGFeY2tZpOqgrzuIKXfE8vYKnTyJDDMxOa1weWdSGRagpjEDS90UsUTTwC1vwecTHClPpysIYMzUaarFOu5H9XrUl5lC42gf647t8KyuMjCAGGzLeV0hzi88PDYNxhLMV3p6Rr59TdRviUUV4+xPW37xi3xVyusZYz9v33qJpWg6HDgHFRGfwXgE7pWRyHDkcDvSdgypQssW7GlsacgnUS+GqO2ffX2FdmfJB1S7eDYn+6TMOu566XuBtSz1bEupa2SZk8oRGF5mmNdZgnaLpK7PH+AtEDnR9QsZMZTL1zOCs0UmgiDaiRmB633KSNzomKYUxC31OGKfCSFNUs5WdYJzggwWxWCOERgg+Mx5gZI/kHTMPlXE0HuYnAqc6oTCjY2QPHFhUiYWfUfsTGvHMrr/lbL2jMoF5EI5K5LiuOWGkXtdsGEgmUSTjs6fGM2DVgk5mYRPvVcKHJtEgFJsQo0WDdZa+RK4HzebBabjWzOm0S/s2R49wVTx7U33HxUDtaJqKuHek0TDkTCxOq2sBGaNewKIHZlGGLskW/CSOsxMf/2ZKYKxMSvpCKh39mMB4mqrFGe1CHz36hi+/+oLDYY/4miIWL47CgCIhA6ZoR2wspNHpgzBpFRwFTK1RyTbhvFa8cZwOfpt1soFCQqydHhhv9ZKwHo9m2FsXKEXV/yFUYDwuNNgM6/VL1vvXSBkIBoyrqU+OcKeBhTjed7dZ2cJv//3XvFwPuMZwtrLcamt8IxzGTDcI0Xq6pP7ebB0VM3zrGfvI3/xhTb+N/OlPzrh/bJhXRQ+CkFRkFAuZEbGZPEaMs2SylksxYkqk2EQqkYLTlDKTqRC6sfBqd2Dd7Ym9YixLcdoxJxWFYsy0HijTizatTUyeYil0pSITatlO5hFswpmgZdT00lXWU5sKhypocw50oyMmT0qOmBxj8vSxsE+WTMD4hK8KM7QQLFk78BQ9YxK17WXtABX/axgJ5Gx5d574k7eEk1qQzmsnYkFMVs2D6OFhJtur2KSCr0lZXeyEaP47GhdTDMZmfAuu8sjMkXpLHqaCIBu1jMG0j1bNQ78p7K8T+02ijzAMyjeTWEjryK15IC4cP/nTM+Yc+OzrA7I85oO/V5E48PBvE6czx6t1xD+OzFaeJzuLPc/MW7i3NPz62x67L7x1y9ObqBjgQySliv/ww5rf/T8/5zAW/uphYj7s+Oidmr/+auS9j2pKTpwcFVbWMg8jkgyRgHeZlFSE2B08w3akaSpS7ziygcOh53TlyF3AxYS3FdZ0+BKIfYQZgKUOmZXzhAI2Q4gZ1zgWR5bxVSCWkVEs0o9cbgqbMiCD5/l1z7tLwWbDNy8iy4XlyEPbFMYhqig0wM4rerdpCk2lvAHrdI11U7PKJGwxk8PAhUlt7tSFYFzWs23699bJH51PAgHDOhe+3NU8LXMOd3bY4GlSRXj5mmVlOVtZnFVUuRRHFgUI6Z47qMYmlqlT5E0D42zg0CfKzuGKrix8KVzJjKtwmzxkKmqNNU4OJ4XTOw2zYlgnuNr03G1PuXN0DMMle1NoV0c8Of8bhpRZVBVitOlCjCacOo1r1+mfI45Rc0pcoZRE10Vs8VPODFRuRtXOOY/PWI/XVLXHTetP1WFFHfenyPbwiv3uHONr2v2S0C5YzlZ4WxOzxfqKyqPrZArGduA2FK5oqsN0oWsBjvPTqN8iohQ6Y7UIKEUbj6w9iqLsndHJQLqxa1qdPHjB2UzlDF4CVmAsCUmGJliWt3QiZK1QV4Z57aidIQSHNYa28kjJjMbQmsDMZbK7h68+QVJH3D4ilQ0IhNbSB0d2YeJDTJOJfEOZ1NWGKQayg+RoDaxcoZVCLlC8wRmlUGZbGG3FQSCOhvogND7jvKduHdZ6ZjZjjLDHcSn1d1sMCHC96zEESkmaEW0cLqvIahynPZ0BQTPvjbGYIkhORImTIjJgjAcT8MEryMMUjElIOhATMBhc6/nsm4f85rd/yyCjVuI6DXrDJnA242xQoEnlQApjGnT3nuykJNWvSbUIRrvUMk0lisZkVj5otG/JugqwN7niTLtH++aSs9bhXMC7CmN037pdX7DdviDT4bxBsBy6A5gO7wx3zu7w3oNTrh9dsN1HRIR371T88HsND07mnB1XdAN89XTPp692DFvz5mEZxkRdeRZNYBwjX7/oWB0P3Fqt8N7iMkhKDAZsLhQZKbboyyAKG0kZZFRv6khkkEEnMLbGlMQw9LzcXvLtbs1Fv2M/Zt0XOjshIaYd3ySYElGVc5niVXVaV8iiuzlndFzvCXgD1mftdgqYpEFILteUUpOzI+UwwUCEmB1ivEI0bCKJmqtc0O+tpsf4iK8MJRtSFGIyMEIYHEM0FFM4RBiLwxvD7ZXlJ/cyxzYig0c8iFMapBGjz6jowa+niyqpxaj7gRvw001xc6OHYRofYzC5YGpDmBlCCuQBci+YXiAKMlriwdBtIrvrUVNARddR7Qzwit4eNweOWsdQHGx7Nr1h6Wv+/B/P+OrTCxqz4M4ZPH615//2rzL/u39xl6PqwP/xvzpwZzlwtnScnztOjxu+fLHl/R+c8uLRNTNnyNZycXXAO/j+PHF/Fqjmnuevt9y6p7wNkyNe4HKXWTaqjj8+WvDltwO28txeBc6vIovGMxwsK5MYk8GZBBhmAp0V8qEwb2BzKNyaW8iFpujz6IGTuuLiuuPB7ZY7bcTVGvVauszDb3c0reODuw1bAv/VXz7XPJK24U++F7jz4sA3144kkftvVcRDQsaEr/Tgtk7IRuiLo0SIJeGt4JyZLgiwXnUwOKMisgrdpQdLckI0GRMDJhmsz/hgccFORYFh6BKfXlueUJNmkHxgtAFf4CQ31Ek4qgvVzNN6Q6ueMfa9YzdYhgwlORg1BC6ZOLkwKnxfCAfd30cbwCc8FePsmC0ej2CxhCLEbBhjxtmB48UKqFnvYW8cZbXk+U4YZSDmLYfDHi+eXGrm85UWA1bBSX6wur92MMTMGAvZBMUcGc8h9pAi0guhagmLhm28YNtd4r3CvmwGm7MCtYLFFKeAIZTQWexALAkz7BnsFa6aU5saZxuccziXKXaHqbZY2+NymVYn05rOgFirE5GbsDV74xwoE8Plj9RWbyzeKF/BOXUxMa0AMVB7i6km9k1OuKJ3WVMF5i1UtcFYj/OGOngqo86UMDnlYsk0JlBZg5QVzv+A2exdhvSS1MwZmhkA23nAhJZIzZACapisJqyxTkvLm79Un1YVRyWFbARxOuHO3NxnluRg51p6M1C6nlw1lIPlSAqhcYgrBB8xNByGG9fed1QM5JhZb3pighAMTR0IIWgWdIbRZRofcPhJFcsbfKJBIUoxj8Q84hxUoYJs6MeIteBMS+08lfe0ruHh+SP+6vef0XUq8BrzlK4nqli3Tn9fHyZo0ZhU5GdVDCYilCh4X5FLmtYWWk0Y0avMOx1J3UA/jdWqeFLBvck80HPf4mzAhxpjLCKGkiNDv6fbX1DGA7isY0Bbg0RqB3fPTnhwq8Llnq8fPSfZzAfvz/jFJ54fv7fi/vGSxTywbDw/fW/Fv/zNc/77z/ZspyCbmFQVbFuHayEN8OnXa2Ag/8ltjlcjyz6QSsGWrNX5jXGjZIoRYtZAjJhGxpTpsyEZPdgwcLHb8nx9zqt+w74k0sQ8t5Owyfpp7M+N0lhFLsbc/LqO6HStbgneU1tH4zQ9zsZALkYnSqMwjo6cK0oJlOIp4ogpEXNGrIArFLKKikg4kym24PxAbbLu8DLkrKAgG1WTkIqq8ksR1kNgmxThuqyF5UrRp1gBr1hAUzwqaI3TIeImb5fBFq+XvgXxousSXTpiisMWkFxIvaGMIKOuEiQDscBoKYMQx6Le/j4zDoU+FSJgFp5gDTboQdcNkdhDFsN+GBE/ZxiE+tRTmUj/fM3nvx/4+d+/hc87vvh0x0/+/h1qn/iX/+4ahpF3P1rx6HXiYsjcGRyvryJ3rwZWJTB/4EnbyJ27Sx6dC++dtnx9MfCLdzyPX7R88+3IfGa4Pje8fz/w7auRusnscsv5RaQZCkenhu1eOKsNr7eZk0XFZl24dxtePh24f9pwcRk5PQlcvi7cWxSyC8QuclJVMBbePvJcXQv3FxVfPOs4OhZeXUfundaMoaXLe2wujMnwNHogMRShjAnJI3/zKHJ5lTmpDNklamNZzjzGG72UQsE4i3d2sgnKG+y2Qub1DLBOa13v9dB3tSBeKFZ5FynpmtNacKmQRzC+YBvwTWAvwuvBsTMeXzmcr3ClZ6wtF3fmPEuFe2Zk2QjHTcN8JfgmkUdH3wlXObI/QEdiHKY/xzgaZ7FFmDcV2UE0liIDeX7EMF8x9kmR4Q6gsBkOdEk75J211K7haOaZNUuu045uP7A8bnjx+iVXVwesLKniyHjQGOSmbrFuo0E8MgHISmZat2OzIcUCFfT9AU/L/OiIAx3r4bmec1NTJM7ou+UTOAc2a/dbHKWoJXAx9xwvPZVPVGGLM1u8qxAsIpHCqOeJWAaEPt1IysEUSzKFoHGBGrWcUZdHvmHXaOS4sZN1OymuvFimtaau77xRtoK9yRSZoGXOGsI0arfFUgWPD5ZQ63rTToFyadJl1VZX1rthTmMCrRxo6hXp+AHjqFC2rXd00WD6hDeJmRfWppqEyxozbyT+0ZpvLaNx9CQ6MYQyCbMnB0xyhm3JvIw1fTJcjYlrU+hEeCtl5qZQzPRN5MJh1323xYAkR38QxnFUZv88kBeCMxkzKU0ivLHqGLQrewPUkUApkVEiGGEoYLPFF0MIFQXB+YD3DcMo/M0XD7k8DDS+woyj3m6pYDGkknHOEaxCZoqAyQVxngqnXetkZzQWhhhVzS4aUlQFhzcVuaiCX0S94MYqClOjKu0bDYExnuBrrFW1qTGCSKIftqR0IJv+jcVSikNyAEnUbcPyeMU+C/H1jkUz8j/5swWfvHfMD95peHB0xOnMU7nMfBa4c9RyuTvw6ZM921FDnSzCKEnHqUFHVd0+8j/8bs8mZU5nd5ndavDR4UpW+p4308M8pQCWzCgD+6EjDoZ9rHjRDVyljhICsd9x2W/ZlB6xkypiWrMgRfHRqgzEGq+qi6KRnTqlyRO4RO0F1jjImnLZRxgOhu6QOewKMSrToFBwLqvFy2a1gRrBmkLJPTJ1Spg8gYwKJow4nzE+U7KQon59FEsOhhQN3gXGbHlxDefbwj9823DmE4erkXHarXk/CUNzVguSdX/UNyRBkTBmIkZmcp5shBEkZWwuOLGY7EgDpEGpb5IVhkQS1Rtky1Cyji2Npcw0Aa224MRCgj5HcnbkSevAzLGYO3IW9rHn1ZeZYg0vvxnZXBmOTwpPHxWud8KPfxoo1vLZl5H/+J++xcd3HeFRx5Ovd7TAP/yHS776rONH7y45OhG+eZ7ZxsyuO3D/JHDPJD66t8D3PZcvNWWwy5mlm+H6ga04nCvYVHjnnZYnTwfOzhyHQ2TlF9jUsxRDt7Y0ruawTbS1kHaG40rod4Ef3LZ8+ajn7ts1j1/uuVtXHEvmRx82nB0JK2v497+L3D7yvHqZ6A6QTSFLRdoObHcDbTF007To0aOBuvUsb1suNo6Li5HbR4HVQvCm6JTQWSqnkB51ARjlVQQwbsL/Wv6IQZ867Zynwn8SuIrTaSDWYIObdAYFkUw78/ziHvz++sCjsWZwjuAbZm1DL/ApBw4p8OeV8NbMMFs4XKs59stYmCfDMAjbXtjHQp91hG2tobgMXijGE6bp28vlgmtT4YvDen3PcoZD6olFn6GrbeZ0DjNnCSFz3o20tRBczTu3PuAf/iDz67/9lov9wCDPuXtnqdMBIIRAiuqHd97jrKNgyTHTjyNt7QFDu2goLrPeXZJDwmJwttKRfdC1780QUVzGWQfRAZ75zHD7judopXLcbBRY5krEYhhyxBYDxZJQKm3Wy0QDd9BLvIiuBLORN5kGTOW8nRgQzhpsMYxJCbiVs+DKFBHvJldUVpHoFNpy81fOiW5U4bNFWQ3IjZVaIDucdzRNII+Fi2tLNjUubBhSoPFnyOI9NnFk2F1xvh25PSbmUjjUc6JongOUSS8hb0TXMBUEPlD8CFYLGtW+QM6JXoRXvfDN9cCYHX6whC5yNarQ/7bJtAGGaOmHxHodv+NiQAo5ZcZUGHNi3/ccxoo6CFWwONdSOXA2YSZghS0Gb71+gNkQU8+YIrEXXOhZNEtms2PmoWFfOpyx7Ls9v/qb3/P4/Fy9/RkdoUYVIpXJwnaT/614V02vA41NNsZinNIPc06qUBWLTO5+56akxFwmbcPN6sHwR0Wvjo29D1jrwUCMowogSyHmnjHuETOCs6o5yBlvb8JG9EB//uqaIcI/fn/JP//BA85WltV8wfFxxUndMgsF7xWCElzhnVs183kibVR7YfH4nDGjfo3BWZzLjLHl0883fPH2nHvBMPOWqgiVs1ivo3vnLNGBxIFRBsYhkWJgN2Yebzserrdk47BT523DJOiDP4ZGSXrznBp09XNji3MYjHEEmQqron7rXBz7XeF6k9nuElf7qIz3UaNB69pS+UjwSnCToMJJY/Vnm8qAsQkjSReUfhJvScIZUaFSmsZmqagSl1FDn0QjQn/6VuA/+Uj4yVuRoxn4SlPjhovImKbugkI2GXsz5TAGYwpFphUCDoPTgrPorjenMqU6ChahDH9kEhhrKUEoDvCKkwarxbIUnCidENFMh0iieHUTzCuta2Jy9JuCbHX8OwyFd394zFcPL5ndX9Ic1fzmi4Fsa956Z8bjF5f8/R+esDskehd46yRz68MVD37Y8qs/bLg6RGYPjshly+Ntz8tnlh98YNhHR7vI3Js50v2KP1z0HPYjRyvD8+uRQzJIZVg4jcQ2Cd6aVxyyYb+Bjx4Inz3M3L4b2L00UIO1iXyoub+yXO8NJ0eGBcLP7wVyjCyN56PbFo4WmM3IRz9p2G4Cd9eGu2852qZDloUU4GI7cHZiOLlf86NVABe42A7MjzwRtVbWtXrLpWl41W2obeHMBeaNY9EWqupm5ZcBvdSxRg/WajIuOYfgKeXGWlwwWSdqWI84hwtga94gjIVCEbg9L/xZI7y1K3y2C1zlGf1BIESsF34vgZyFT1rPqo566fiIqSyzZJi1lkVq6VOhGxJjFpKBHstgMq5kiol435LrpQoUfSBNU44hRaQkTpo5OQWy9cxDRestzsGdao5vLFVbU5UV/8l/8Kfcv3PGX/zF7xhlZBgiob559kXxvsbq1NV6Yk5kA40PiAizWctiNWM7bDFuwPlKNVeSMbboz+tGdSOCEDBFKLnQ1obTlWc1txirwrksGqNe8s0lrsjIG9S8KQY3sR98gQCa7Gn0PczFAqoluHGGWPtH41pJaju0VtkSNw47iyZO2umdd9ZQyrQiFGVmyPQPMSWGvqiirNg390zjPOKFL592fPNw5M5pTdssyOWIYRyo7DFdc8rz19d883xLVwlHUtG1wi6GCaWvcL4sohN2XXCQUiGURCuiwm4pmspohCEXxkF4ss18+zohtsEnS6ZwNUay1HQFjurMZjRsDon1dvhui4E4HhiHA/iGmIRUNGTHGYP3BuMHvBW8yXifCQGNrfVOK7oCxRmKLVTiOa2XrOZH2KpiNIqEjLnw7PVz/vD4S7pccEY7fxtqTOxV3uIcYdq1dnHAOq8iLaMCjDId0mHKPUhFbWeCXtTOenISstGkPBF9CazVyUGMEWstdVXpKmOqFFNOSJlWFuOggkSjTHvJFm90hCNOJrW5xzrPdhvV5uFnNGenfHjXcKsJKgpyDnFZi6dcCD5w0jTMvUfygWiKbt69J8es9jMv+EponGXMFRevYXMawVtSEUaP5nQHcLVFvDIGUiqkaBmzYzNmrsaewzhgksO2UDlPZfX70AtXufkU7ZacVRqWB/VKo8JKsEjv2A2wPcD1NrHrRjbbkc0ucugjMdoJWawjuFgKTeVoRAFTwWS1NKFZ3LkMYAvGaddmsqp/MTcLHbUIZTFELINBue7F4EviX/zS87/5R4VP7gh1LYhN+rzmMoGoLMOIikyrafx3oy0RKFMxaKeRnfU6Qs2DAoNs8VinRVQKwkRlnvgR01BTwGTBRofJRl0XCEmgTOsoL4VKDLFLsPfE/cjVtuCMcPdtz6ePM6lqefu9im/+z4mf/mnF739/gU/C2x/UmDrz+Vd7zDbwk5/MebU9cH0p/JNfLvjv/uaSF08HSsl8+XzDYb1jfVn4+OMVP/5e5r/7dMeFC3z9aMf5kPjkrOEyCa93HakWYhZOrBD7wqwNPHmVeGtmuHpa+PDdOV9/vuP2Wyueveq5u0jEztMV+GjlaWxknz0fnVr6ZLh9a8EfPu1Y+sQn31+yDI6rJx0vvzhwvofb9+c8v+r5+IHj7r2WW13kD/92R3XhCNvIEBzGZS6vB7a9ZYiF3CWahdDOPHGIDL0h+wAmESWTxXIcLIs5VK0hNFoYpGQ0V0LKHwtArxHT2RaCGFIJGlVdbh6CjAsFV6uVuIglFeiS0NjCx6vE7VnhSTS8Zkk9u8V2v+bV4cDvxPIFnuMq03iwOIqx2CohxutoukCTnE67MPRZ6IpjHApRINU1hCPCaChWbdklG1LOzEPF3DkwDcVYXCWEymOy4XhWsx0Sthgse2zd8Cc/fZfKCP+v//b3VPUM6yIhBMZBmSmlyMTeVx0AdmqSxNLUDcUJoxkQP+JKQPXgcRLXWhChGAUmCUparJrC4sSyOBacF2LSoB5soSHgvcHaRG0NsQgjWV0nzpGMrvZq46iMBVNIRmmxJmVdod4UMzdkSIE8KKkRK1QzbZBubntrJ2u4mdJbjbrabmBTxppJ7G6gJMYM9E51UG1QmqUzvHw98Onne14975EuczKvWc6PFKHs5jThDAlb9mXPs33P6d5Sy0iwFQZ547Aqueh6dBIkiQgpJRyGJLBPYItgc2HAsxszL9eZ7T5hfSIURVYPqeCkMBY4awpDgusO+ijfbTFACcRRqK12jiKiCt5J1OdFRWRIxlfC0VHL6XFL5cxkJ0qMRcAG5rMjVvMVlfd0jBzSSBkN5+s1n379kCFFnKs10MYol9tXFUMcqa2ncoEhT5TAcrNDMkqf8ios8WixItP7XFVeSWEy0e+cVpUi06gGi/NOx7/oxZBLouRCLkXH1RKRHMk5TiKym044T7Yxj9hMEqidThUigrc1f/XlnutN4pP/xXuc3PG6HyqJnAb6sSfTkQkUa2hNhe0PZKOY4mQyVJZsCxIdLoQpAcsyHDK784x3wkhBAlS10FaWqskwMQJKVq/tOo+86AbW44gLjpl3+JnidkOYRJdmCtdA6Vw+VbROA4NaLLVzNLbSPO8k7GKmS4n1euTJecd+EGLSv8cCYbIppZQoOZGzJaeAlBpKrV/DFOiRyUiJGmltJu910oNbgSzKhYhJGEcYB0FioUsgyfK/+pnjv/xP4X6TOAxwvRZSNAo5EqEVy5gK/SHhbEVt3ESo0ws7Z6suCQzFJEyJWKMTAucts4UK0YwpxLGoVsBMc2eZpieo06CYor8HvAlVsSK4YjADlIOh38PmWohDwVjDfBVYzRTCdFQMx+94hnXPjz/2VHGkWc4Y7CWbncPFwke3lzzq16Q05+L5wHvfW7LZCX/92w3HjeMHH53y9aMNeRz59nWgqy559EyzAipveT1aVrfnDK96vt3CcWtoJDFvIEVHbTJxXaiTIS0yZyvL8xcjH9yq+c2rjvdPKi5ejdxaOo6rJfcaz68e7fjzD26xfdrzk58f44tgTkZab/jyceS47vnJRzNebSOvX+54ftkz2sDpnWPyqwPu4DhbeLqrzBebSOcbdqXjnWMh9p6zE8O977Vsu8x639P4wjt3KoLXD8B5GK1w3WViKbTJMDOW2bIwX3iMtWTRsJqcVB+EMRTnSUaIQKqUXidAKU73yhMr4qZzrKzDUhBTOG5gVu35IFjWswV/2XuKtYRQ868Pibd6z3tHESuWbD3ItBqjYFzGV7whGbYFVsnSVZo6eBkCvQm6C3eJKgs5aWFubSB6RyQzs0aLaO8ARyk7RnnJ+uKABEO4ynz/wS94vuu4e/o2x0cnHPpLfbcAFzw3wZxSMsF5EqrB0QsX9t2OVHQSUHlHxBFHBQdVVrQotirQboruwU+PZtw9rahDgmRwyTHmiLFW332n4KUwiabJNSSvuGjRM7txQb+enG8OcDTu5iaCWhkfRYSUldsQxGBbSzVNP4poIq5Fp3spCylFBBWKa0bLjX5DnVPF6lTZFAsRTC1YL1x3ic+/2PPq+YAMhrIZyOcvkOURclzYVT0zP2dxdpvVZs31q+d822Xu2IpcGRD7RsuojiUVBsqkd+uBwRS1oadCTAabCqMpPNsPvF4ntUAXjalnslOfbzVD56ISKPC6F+LN1OM7KwZMJsWBpp6plQ8oRT9QA2TnsCZgisJ+hiHQDxbv7STUC8zEcdSuODk6JZXEehjocmTX9Vyed1zstgxjJqVIZSu9lK2Ktooz2Kxe+rqqSDHjQ5gy4XnDAxARgnNYZxn6HmcNbdOCMaQxTiEk6ku9+SzMpBL13r+x2qSkfn3M5EIQtWowORlkQnYqNqyQkxYgiKEKFbM6qGWncuqz7ge+egH/p397yZ9+cpd3jxbMncH5A3WeY+0jbFqTJdJUHgtEUyDrvixKxBmhcjNN+mLAiLDeJ15dDhiryt/sC5UvpNrRLA2mKhgn9HgOCK/jwOUwkF2kbhyNM4TKUFmND1YBpSC5UBlL6zzH9Zx5XTGvHLURvLU4AjEJwxhJEYJLYBM2JCoMzqGdRVHXgYrxVcAlRS9wV1lqV+GKxeaiONOpW5Os2FBBsb4KswKQN8CXNEKKEFMhDZY/fy/wv/8nhiMiu7UwOlGaYe0wQTUKPsHeGZIRZsbQzgy2KqSkCGffGsxNTClexahFuwZdgxTG9MeqvmpUdJiTMiTKDallcqSEyiiUaTTYweL6QuyE/Tay245KG2xrjhYwqyqCU4dLHC0ff9DSPDBI8Xz4nme59PR54OIcfvIfzXnxrKfbFF5eJt7/UcO9+yM//w8a/uIvdvz6G8P/4X95xtnK8+s/bDkYSzIDb58t+cPne4Zeq/gX8xVP9muWo+Ht5Yy//WrN6mjOTgq3vGXVOs6HwNlp5vy88KP7FUMHF5uRt73j+mLgh3dn7HYjt1vLg2PPZ48qXr7Y8Yvvr5h3HfW85eiTGV9+1WF6YX3d83kSUmtgHnj5fMvtO4Fff3aJ7Ax336v4+MMFw2gpLwfq2tMlx627aut13pCycKsxSHaUlBgHx2JmOTky1PVEDkLJdoiluxbGrVAvMs0KwsKqWtxYUkJpqAIz63QEnzNDVqRrcZ58Yy8tFpxATqqbMRoMhvO0JhLslrob+B4VX9YzqCousrAfA/RX0KC4XZjOFNUlyU2eh9Fpqw3gjKX1nqGqGLImz1XO4k1NdoE+9owpkci64gtq/dNjK3GII7P2hFeX17x8usNLw8X6OV9/vWG1WLLb7cFmqsoTgicV0XRUPfQYsyY5ZqOTlBAUMy8pg2kIPmgw2wRsmtUaHe6chrlZpw3bYt4wr6xiubMGo1lrKUa1XViHybVO7kSorUOsn9Y7esHbZJBsNY445glcd0O9FUxwmNqoxsUI4h2lcSr4rAApJBQWJaojpBRt9MQagnMYp43Bze/KhJCvnMNnS06JOIINgfOLyMvzDoxmGtQ42G45vHpObQRzdMrQOJZtw3u3bpF3l1yNB6p2puvHiVnhrCa+OqeKTZmcSsXAaDQmuYvCZtCf274UHl5lLncy6eASEXuDWaBLwrBLXFhDBeyTNqffaTEgjKTUYc0pwSjbfyjqyy85k0UDEiiaOJgZsXbAmpZFG3DOcTQ/4qg9IufCy82Gq/2B/SGyvtqzPgyUlPBFwQnFJFy+EfM5hhRpvSYh7ocOsbp/NZOITUQtGNZpFVSAYD1VXRNL0Wq2yJsLPU8PffAVIbgpGARyzqQ4wRwmfLKdfn+MTIFEMn2AlhQHJBfqUOGMRbxyx7u+Z1ZVzKqGfhhpQ8XtkwVFFrx8OaffWXaxx5uan90Xln7ExoGM0DROLW2CTiJk2mlZTcKy1lJbSxbDN+uR+27EuYalKFjJC6RZRRMDfgmymJP9nF02bMo1JUSWQe1SwTh8MFTW4cThiiEUTTo7cRUnvuZsUdNUjuALRjSudBBhS2YsBesDmIh3jkVb0VZCGkeiz7ShEGNNjFCix9uaYNRpsGo18jrJADZyk1Vh8EjKRMmkrFgpIxM5bHJ6FNGdIAnc4FhZeHAvcNQmzLbQR4c0qjguGUrUEX2KQjeCDcKy1TFiNyr2OidB9trl6EOfdemUDTKJnNRBod2nDQYpBaaO0U58c0QnAyYbSFCGTDlkYpfZ7TOHIYGzhJMZt1vPbLLW1U5JgAWh2I60tlx9M1DOHOOl5XDL8fKy4yc/XuJHw9BF/vKzDT/4eM4PPwmk1zP+9ndXfPbVjj/5ZMXZceAvf/2aWHSl9l/+5+/y8LMtuS+0C8P5NRyGjqv1wDBf8Ocftrw8P+GnP1zy3/zumvdOhfm85fg48eSi5598eMx/+7srzk5XfO8s8KuvO350t8VLxLrAbQ//7nfX/PJ2xdhUfO/vLQhX17zc9Xx9NXKwDT9/b8b1K/jiqw25dfQ+c7CW6DyzCr59uSa+KJyPmcWqIswsXZXY9/Dqq0ztM6dHFbs+8vaZ4+07gXFQbdFmWygY7tx2nK00+S1bHV2r80Rz7LsrqDpHvbQ0c0NVGWqvSaEaquMYo+X6YNn0kzg2FEwo4KymVRbHmC1lyBOlMuK9WhePTeTnZ5GzLvN1v2BtZjyNiY+6xAyH1AYq1HZnQEp5s+tW6NC0oiNRLSyBghlGGlPjiiXULWlUDUKZdua1MVSmwrmGLvZsZavFTZxz/+RDTlvPv/o3T/ntHy7x4ni93nNnNTJfKrBMJ3cqVryByLrCzZgAZ53u1if+izWBpios2lotm/4mJVKovFHaYpnAbuyJ4glBs0EgUwvkEXXyZIeUwLrvtAGbkl9jtli5ib23ky7AgzQ4CnOv53mREXKmdHrOO+8JLqgLyIE12qh4o81aiROXQFCYnHMkKeQYqZyhCROfQGQSEFq8WPpdJEaP6zxlWzhuW5pKyL3gvWNjIv3VK059xbxZInbPys05PVpxebbk9dM1fYKZ17Akaz3OoQXJdLRZq5k+WTxp7NlheLpPvL7OFBO4TsKnLwxdDlTOkElkpzCvasKwd6lgCNRGz7yb4LjvrBhwxtINexW7Zce635BF4xFz0l26FAVIKJxGMNJibUXKwtAnsgxcHa7ZrNdcbnd0SfkEsRvoC9RFGHKn456sIi4NwNGq2FtLFRzb/YCvGjITIXSqosDircMZy7xpiWYkl6LCP6Odr6C7Nusd3nqCVQKgjpFV6GWm+U0uRcNKRJCkVhxTmOAPus8+Wq14/95tlrPAF9+85HKfcEW4dzpDJLOOmZgsJ/M5H719xt2jlj7vefhki4wj//yTltvNmqHrNWznAH2KRAq5eKxkjbcU7XPGkiipYOsa6xzblPh6UxMaeCsPuGEE6xlszdys8M0cP7/LISzZjB3ZDDTumsYHvASMKiyoe0MThSpBI5ZT77jjAkfe0fob4QyMVoVLQ9FUsZR0jOotnLRwuphRe6hsTUmZfsxsDoHrXaDrZgQzp61qqhDwk9Oky45cOpIpZNF1wph0v1uKFgN6OU/PA4ZSpn9XHBhNNvvqReS//hT+0x945lVkGwObUScHuWQFCWZDEUMwwususvIB0xv2+wTW6F7YojCkIhRRz7r+2WUSUxpKMvg0AUMMWNGxrStAtpikndAwJvqusN9rp2VnhqPTwKyqqH1FG4SqneTXSZAedpvEep84rC2HPtJ9NfLbZFjsHX9+f8lnr/ac3m+4eLJBMvziZ/cY12v+73+x5Z23LX0JbGPPV1/D/feW/ObhJT/6cI6Lnk+/6fiPf36L/8dvL5mtHNfrA99/b8mz18IwRH7yUcN+c+B6faC/e8KffG/Bt58/4+urzH+z3fHh/Tnri5G4Chw1wpm1PHptqErmJ2/NqPeJp6/3vP+jBddd4vitW3z1F495sR3oD45tV/PWezNSK3z2+0v6bNnFwuMXA8dzw9ZnHj7bswgVLzcdEcOrTSTYwDt3WqpqYN8JT59FagO3jhxt43RvnQwpBi5eQTyM3LrjmJ84jLckMiYXQla1eh6F/joyHjzzZaA5s8yWUAedmLje0idLFwvRFIVlJaPwKK8OF49BKj3foiIwKUb3yXNX+EGTONnveOhqLk1FPzTMpMcUDT0zopxiY0Xja4vGCSdvcTcjT2NwYjB7bViKCQyS2PeXhKqi9oE4Juowx7pAHEeG3GsT4x2ZkePFMZ9ut7y89lQJwjyR3IxD8axqjzFC5TzRGUbKG4uwbwJujJO7QkWW4jpm84Gm8swXibqKCiYyBWd13eYnR1ZbPOMIMRkSSvnre2G3T/Qd7PcQYyLHAykbxjQJbnGIeEBJhc4HQmWp64p5W3G2sLx96ri9EI5mhdpGREbGmNkcDlwdOq77ke0oOgGz4Cuhagy1V+z9aArFG6WLikzTIZnWDUa/FzKIRWwGlyFk4gDkQts0vHu/VrhZ0jtDimfMwnp3wGw3tMVxaCtsEzg+vc316wvW+w2hnZGdToVSztOaWRDJCNq8bHPh692c2vT89fMrNlshW+Flb/j2Apw305Q6KP7FWTQVvUxeKK3jmATh32kxEEumDpYxDozRYyY+/xgTlUO7OctkTQk66oyO63VSP/7BUC8tOW3pNwdSTCSjoxonlpJ7rIXgmeJuNQnPJFWPg+6Guu4wjWUz4j25FIJzerEYjxU4OT7CieG678hWRQMW9aAbY5jXrdoScyFPvnQDpJK00zNlyk3Qil0rRN0hmmAQU5FT4WQ558P37vHRvQXX15ccHzccnc2ZL4SmSTx7fiD1EYxnKJGvXrzmxfWM7Zjx45b/9T844uf3LxDZExPYaBh3PYf9gSwJUwIJ8CLYkhX5a5KKh6ylri3BOS7E8U0vjMawCjUnd27jbp0Rb5/CyZI8PyNZ9bk2Rb3seapwKQY7CH4s+Cg4gWAK9QxdCbhCtgnxQiRxkJEDmeTUYmgLtH7k7duBB3crlrOWxgm1U4xq3wvnm8CLdcVmP8fIDGcrcoEhDaTYk8fAEBOljIxRGIZCzoacrVqHmKY9chOVbNUuXmQCQxVm84aC49fPDLjIJyeReRUR4zRLfNIaGDO5IMSxT0LJkaM6UFvPdj1Qep0WeVtwqH/bWl313NAzXTFU1ij8xSsERbKqD0tSjcZQMl0X6QZ9UdtTw8o7KmupvKOqDKGd8ISDZdgWuoOw3x3YrEe6Tj+nlCEUx6ntOdltWXeFn/zylH7Y81//+z0vD4777wT+9V8cCL7w9ntzvnl8YFUZfvnLEz774pKAYbuG0xPHj77v2XWwCpnKWY5PPR++teTLr54Rm1t8/e1LLp8OHAWwo+XV49fUtuVHb1X8y9/3lPnAu8s5f/t1xz99N/DXX+948KDmx+/MufthRcgV36wHPrhfuPxmz5ffZo5X6sj59cWGv/r1FuN3vP/DI1annvfvHvHFl2sevxrYHsAtZty+7+i2kZwSs5nhrdbz+MnI1Wg49olZ45mFiqfnPSdHNR88qLCSyMGSJSNF6AfLs2eZ+VY4OrPMjgvNTH+mJQtuVijZkmJhfxiIYulGw3xuqfyUP98IfigaTJYK0WgUtbFGKYUhY5ylCpaA+6PbJBdK1gvl3qxjYbaUNMM1FeQeM2RdC1iDzUKu+COmwwlWEkZu8kB0Shhcy77fsVjcZ9tfM6aCDyoUDDZgbU3MiT51pNJT115nbN7ybHvFX/3qHCM1dZ05ao7ZOAE3J0nEeq+UxUkkXDeBbp/J4rBWyXeGiio0zI8j9aKjqRLG6eoNHDFBHIU4qEYnRWHYJw77xL4rjGMiJ+gOmaG3jINRt0JSmzBGLb7ega8cwaPhUUFwAapoGLMinVe1w7qKdm45OxJOm8wsRIKBmFs2w4GLzciry5Gnlx0vd5F9J6SDULcQKrAm0ugtqkwFV5DgVXSNTvqCDxSj4mdjdI05DEKoHacnM8CRciElbWBiSqSoroOSOuJQM3jPrJ5ztDjh5OQ2569eEfuMm2tjGtM4BTVNCaiTCG/bj/zhVWAYI79+qmswQ2adYUwe6zRPxTIJHSfxoa5jlZWjYma1p3+nxUCiUJXCGCMhzJAxMqaENwGDBk04a/FOH0JJhsM+st+PavcrKklPKZHGaSRldMGVC8Rhy+ANdqZBDMEFTJkqx6L8eOccXZffxP3eKC8xSs8XEVbzOUaEi4sLUilv/jvJSqRqqkoLgZQYc1KBn/e4ovsrpRUqeKOuAt45Yo6Y6dBPSRizcPvsFp+8f8K945qFbbCL27z/buBi/YQsax49T1xvjxDxGjAx9Dy/GCmy58688J/9oxm/fDCQ045un0ij4IvBjw6blX0gJYEpb5D/jmltkjJjr4KZqqoZZORxtLwsheO546P5jA/e+R7h1l3dneEwbg/DlrHr6HdCOVjcdaTsR6ohv8ENJ2uw3jIuG/rksYPBtVB8ZLCFwRVyZRGvlcCqrTitKuom0NSeefAEEtYoTbAbHMetZ75wXB0WlDQnJ8euH7g+6L4zx0yXB3Zx4NBrhX/j9S8UpLhJsCmT6EbnmAZV2Xqnu8q2Toj1PNnMoVTcmo/MfWFmlGWPzX9H76F6lCEXrseeZV1xclZxWEdiPzJMZE1nHI6MdYXgveJlnc5TCpocdgNVyaKfVVdgIOOODadVgGxgFIKzNI0mS+okrdDvhd11JnaQB9h1iUNWX3FtCo0HVp4fx4qvv7hm//HbvOU7fvU/nGNK4H/7X9xm3+14/AQ+/v4x/+6v15yFwOz+GQfp+epx4cc/XjC8yuzzSLcZ+O0XiV/8vROePt7zk49WPDkf6OrI88sDs6Hwzabwz/78Hv12x8PniT//2SmPZzgkfAABAABJREFU/+0l//iDmr982PEvfhm4c5p4/3bgdNHyb77aY77fYI9qrnzmrTszTr53gnDJN799hbtluH224JdmwW+/7YjjjPj7Ldiavnbcut1yRSEOI24Q7h7N+FePLtl0mXYGthLCSnj1omd2x3HrduKTjyzfPrN881QDkz540LKoM2NWOqYxeol3e2HoI+214+i2pz22uBkkBCmGqhhKKqQRri5hvU40M8N84RBnaJeB0Be6AbqUyAWcGA2cymbqGm/AXIqwJVi1LceiAkbZMJQdvTmiWQVCPyKm0hK305Q+6kmX5N0kWPv/svZfvbpl6ZUm9ky7zOe2P/6Ej8iMJJOebJoqUuxqVasgdDcgSEID+gX6HfoNAgRIECDoQgJ0I0iNAltdJItV9MxkkukiMjLs8dt/bplpdTHXiSxBNykoEzg4iIiMwNmfWfOd7xjjGRRPai7uOCXKQZkjdN0e29RIYVCyANCCC2ReFx4pchCMBO7Uhzx5eYHdDlTaIJqGps34VGFVRYpxihF6xjCijcUYTU+EOOKHAUUhxrZzqJaFLqhFRb9zXzeJbjaBzTrS7zKuK1TFYfQl+v26NAgxxXI1UAh+k127TEKpXMxeP8+TDAiVybFs4rwLOKXonGDfpyI9xoQkYlRgZkCLzMHccmepeeNA8/ax4el14PPLkWfrkX0/oqRD6TJ0qKpETI2hpMFSKa/Tdmq1LOaCIiNmwTg6jhcrzg5mSCWJKeNiwvmAD7HUTY8Rnz3Brxn7Yiw3WnJ0eMKwC2hdlwtnHki5bL2FMJBNMannxOgjL3ZwtQ982SlE0JgpaYEUJf02XZPk1zLAzzpg8lTMJXP+OUeB/x+GARkFoi4NccpIhFYYofFDz2V/w7xaYa0ta9TJoJGn1XImI1IAXyFiJsaxfDhUediP455+e0uqFG1TblsuRiqpS8xlAtqILIihGDYKi7I0JmqlCIOjnc/RSnFzc0OiNHDF6c2stGFW12Qf8OOIi5Eki5kwxVgm0ymsqpEIozDWTh9kRcoQXaLWgjfuH/HWo3scLQxGS5IySNVwWjcczAQ/+cmazXlA1JlGKVJ0DDmTRYtNI3/4vuBbZ2t2oyYPCekhjJ44JFwvialB0KFJJJUnpnnRkwRF0w7BwziZXqygE5ltylzc7LmSXxDv3eHDu/eZS0vO0O+2bM+vuX3Scft8xF+PqF1E+USlBLUxVJMBSEhJv1f0TpMWEllFvEr4CkItUK3GtpKqESzmFe2soW4sVkkqqbCyXMFd0OyNQsiKUFfIpmXsNWOXCDliHOwGT+c7rrsNu7EnxoCI5f1OiKnWdbL0fA0XKbyD8hdlkh4C5KxppKFVBZySdUM0mZADMTqyH1AyoGXEyIQWkojEBcnNNtJIiTIWUkFnx9f99UkjcilGySIU+BASZGHKa1kipXHS54zJzKaSq7wvUSZ1UPj3WgEpM/aBbhvwncSHkgIZXfFJtDOD1qXqOnnBfvT0O0/YSQ5WIHtYP4986xt3ODpyfO+vM7/zO3dg2PNrv77Ehszf/tMNf/Ingd/5g4d8+fQZcpM4vwh89mXg4ETw1tmMTz9Z8+zJjqv9yPtnhqrb8+itBTfrgXsPJN/96y1ddYJeCBod+c1fPmY7eF7u9vyP//MH/PjzK2YLuHus2ewjP/liw7MXgfferPn0R9ec3a85O53xxbMd+4eZ9x42fO9TTzCJ5aHms2cdP74aeHBsOD2sWS4zt7eRn/54zaqVzJeShVT0PtEsZ2zUQIyS9cZzdiB5607FPia268BTAffuwmpVDLAxZbKhAKuSwQ+RiycJey1YnghmR5qqEcQUcENCK0NtyrOg2ye6IVK1pXhp1lY0NrHdSboQChveQ9imcoBIUeJmWiJtuY0jE9F4jKjQYyKmwI+3N7yv5tyZaVSYYk1SI4Mg5Fj6AbItKSIZIGokBbmegqet5nTuFq0UlbbUqio/6xSJHp0DAVpZQggIaehi5JuPv8n5V9+jvxrRUjFQoES1ERhlClnVWmxIeB9wY8AaU4rNlC2mTanLpvd55GYd8bvAvsuMvaPvPX4sh2iKosCbkkDEUgldtguvtf8SA8xE8lTS9lp6K6CvIifkWAacJAJkgRZT4V3WxaQeMtJndCwHthKAjGQCQkaUhrpJHKZIIhGCwI+GV1vJfigBdGUScnBgJdKUbaMSMGslykpiDsUvpgRjzKU7JmQaY1nNG5RS5SKbMqP3+JBwPjNox350OD8Sxj09lrqtaOsZi8UBZPCpUAGVKo2Rr8FHOZd4vJAV+6xZh5ExCYqgW9BLSWZ0oTNPxVqTjfL184r/RNLMr+kFv8BhwFDegH7omC8ks2aO20X2+4EsRnzakkZF9BWVbtC6KlpMDMXgB0RfEJ/OD8Wg4TPdsGXod6T9gJANIkmMKiwAOXXCS6nQ020+hEhlLCGVWmBVGQSwbFps03B9c1NiKkoXJ6gQWGUw2tAPPT6UP08Ji03mw+lFk0JgTVVyqikzuFJ3HENGScXJ8Zz3Hp/y8PSA1lpy1iQh8EoUXreIXG0Cu41G60OytcQUCbEU0bgcqOcWXd/hZr2hGm7Q2aGyKiazIdFvBL4LqCRASNwEq/naXCcFImtSCIQxECgyhDWKUSRcgOcX1/zDP/4jZ8enVAcnDNs96xdPuPz0CTdfXrO/dITOUSK8iroW+CrjUqKSoEn0PrBPjnFfPiWjSuS5JLeSZiVojWY+V6wqyaxK2KpM2lpajJraKL1EC42NNRUzjDf0fSDkAZ/29KFj2++42lxzvVkzpISRGpMlIU8P8okcmZk0siniJCY/Sc4wJtiFzHpMLEJgniGq8m1R0iJNU2JGMZFGT/YDUQ0o61i0sKgKpGncR9xeoKuCYgYgBECShMblqcExvcZZFwLgSEASaKyhqi1Kly+r7zN5VZgYZX4tQ0C/FYRBIrKkqjQpDwwuEpWknlVoJOMQ6caE30c6N9JIy9uPGmZjj82Cex+sePCg4e++s+XJl55f//Ulz14JPvnIoUPkTpt5cNhwMg/8yUcDR8Zy+cqxz4nfuNewHQa0Lofvr/76Q37y0RPuLxpm7z1g80ng8koxyoarV7dcvlDkSrPvB775Ts1iYbi52nO9c4y54Ve/taSeK/76b17x7lnF2f2a7/zlJS9uAqpuON/vubPreZYP+LVvHvJ3n1/zycvEG48a/u6vtowBnt+MtHPBoxPL0QPD3CbGZEu3wK5HZMfxkcH7xHpbKKCHbeasEcgFhfmwN4jG0sxyKafJkEQpLrZp4lJEz805bG4E9TKzOLYsFlUh9ZEgCRYLxe0+0+0TYfDIA1i0GlspdJfZO4mqylkeA3ifCD6RxkzeFa1fKqA2KBHxWnPrAhL4yUVPdbTgcMmUVKF0nuSSVhEkpBZkqckyQQ4YVZ5nKRuyvGXerrBCI6Qlkxj7HYgEsuTVlS5IcBkjTilC2JKkRymLDzuubzSLucVXNyzmTTHlCY2Wliw9MntSCijVoHVp9pPasL6FT570rG878GP5+b+uiS81JqRUorNJgDJF4hUF0pQoUKM8IcxVNoRY4tulIro8h3MqOHJyeQ1EChh8KUCrKA2TKSOFQkuBlhEtA0YmZA4FEBYE2ZctrovTcBgzfhTseknKgsoUKmVIkZAjkQxa0CygqhWKYgyMObPrPL4T1M6gpC0YZ108QyYLtK7wIWN1QkuHUCO7fsT5jBt6lAZrGw4ODtjvdngfS8uuqvEpFO7F65ik0EhpGVOmjwkty2dTZibzJghR2CtKlktiqekuSYmyOI18fXP6RQ8DomrA7/CyI9uI0YpcS2aHTXlg5sg4OLp+YBxHmmqOVRYob65XisH1pScbSQ6Z0PX4/UAac8FQTvGxnBJC6el2D0lEtNSE0ZXncyzFF6SSt61mLdbWXNxe43IokJ7oqaShtTUxJbrdHk8CXRQVpRQ5ZcLgaJoKpdUUVUn4nKYPaWTe1JyeHXL3dMWdsxltZdFo3GRIs5UlKMf59Vc8+eoTvvryGd1oMLPjogFFgRMwMy13G8MbD07J9R2+ivepwzmNe0oz7Ii9K6VPneQgJOqc2ApJzgaZpra1NLnbBWWN5jMhe/raYGRkZjSYzN4LXlxe8c8//gH9/Udws2H96U+5/fQpfptJQeJjuclaNDJMHAVZKG0exZAS+xwQXdngjDKTO41cTMVSC2hkQ6NhoUuEVChDFgUHPcTEEKAPJd6y6XrW+5H9OLLzOy77G56vL7i4veb6dkPnHUmWrgCBmDwgUz74ddxp2gzk9DVivjR65ch273kpEtJopJW0laaVUMtSk2plQVBnVRFSS7d3XF6MjG4k5B5VlYdNDGCkQgtQohRrLWzmQA3MZKKymqxEMSWGQBKaqs7MKgqDY6o4TiIRVSHBSZUJQTBucxn0pMDUkm7IuO2Aj5KcFCIlYozsx0i3D+BKRfLxQYU2FrFPNHXF3373kk+eZh48jizbhJgZjh9U/NV/fILbQlQ1m17x5gcVymiuL+HRGy1vf/uA//jdC/7ot5f8X/70hkEr3rxnOF1afqxmDEvJl997Shog97vi6E6Z65RZnFgaVRgMtxcj//SDnuUStmnL4A/55q8aTivJ7EDyap0Jc8MPPvU4saPTmt9565S/+c6aKAIuSX7yvOfRPct//QeHfPVi4KMXPc+eZt69ozmcD4SUeXITOagljRU0o+dgZrEE5ielOKyPGZUlcymxOhByZL1OxCCp5hJdTQlDpYuBUOTiRcqZGGCzFmw3jtWB4+59xezYFsnQJYQW1DXse8fV7ci6jxysisfC7gLbIRC0QDUCnV4b0CAGSRjLuj4MEh9Hskj028TMKoY28eX1La2vqStNmmeEzKQeotAYkRDRk6Mm21T8KEIgRc0YHZWtkCh0MrgU6MaemAPI0tlS100pYENjpC2btqsfclJFvtAR30ty1OiUOTiwGFMuGVpqxuSQSJQyOOcYQ0/Mkug83dDjO7i5GUlJomUBsSVEMcxmWQxrmZL6ETAJrtNXt7jzXxfHCQEywGs2tJi+35lyUIs0GQl96UOJIqJUInsQwaGnbYCRilomKhGwuXieciqG3zGXOOLoEzuX2Y6BXQ+7rcRHgTYKY00xcApPJOBSYrMDoUpJmVKRKCLeR4QXHKhCBo25vG7lN1k2kZM8JaRG6Aah+tI/MUScG5HKMJs3pBzo1mliCyhi9oVsm6e/Jw05jowpM6ZQZABFAeUh0OVlnLDLxUD9syj2a+l8Ikv+/LPAzz8MuMowk5bgBpzvMLYhEEBDrS0uDMiciaF4CWLwVKIuL1hMjD4wEqlsBRjcGGAcsRFENox4eudpZYNQxQCRUkRrhReJSivWm65Q3lKc4DiU1re6Zn27xrsRU5f1bF01WFMxeM/ofZn64Gdkwlj0qrqqqKu6rNn6vhgSa8PBvOZ4teDhnVPODlc0VhOzZxgHnACUwRjLvt/y2ecf89Wzz+h6x2ZX4oHLJiFjWX2dLuc8Oj7hjbMDtI6E/RX7dMRaPmKxajncf4S/HsiDg1FwqhXHlWE3lkrnKOLUFSCm8pRiPooZxhgRISK1ntykxfCTQuIHP/iIZ89fcrrzyJc3cOvxaEJOjCIjhESlMqyQIjqWQS3JMt8NOcGQcCIStcEPI9pF5rOKWlraRhXKpJJlQJC5pAFiwXjuveamc7zaZc7XPTebTDeO3HYbnt5c8OzqkvXtFu9K5lZSGA9elDWeJKGEBvSU8ix0tBRzkW1CMWqlJEr5TyzVso3OnDaOfSWorKVWojiE1VRgNd1aokhsHby4yrzawDZl9jnhhMDLXCKYq5r7R8c8WsxYyR65u6bOPXcbwcOFZFULamVQlErVKB39GBl9wBiDUZmhj4y9LD3tM8XoAv0uMOwD3itkLIPd6CJjyEQPtpa0hxKjLePYMYw9x3caXn214fJpYDFTpJs9n37W85v/+Zt8729f8Qe/ccrFxR65Mnznrxx3Tix/9zfPOTmo+Bd/fMoXP7zhX/7OCS+cZ985braZB398wOXFmldrxfnfX3LaWN44rJgtWxarHV89S1xfDrx8ETn5jWOaNvDnH29xSXFHKn7zvQVPbjt+uq05HxK/9eCAH362J1WJN3/phH/7P3zGu/dn/NnfXoDN3NwkPvig4eVm4KfPIr//xw0P3pKsPs786KeR3a3HLCt09IS9x2nLW48qWgNKJjaXjkZJZoeWJAUuRDoJxlpam8kqso+SsBfUHqpZaR20tkTffMqkXCibOpeb4nqb2f00M3sVWB4mFiuwWiBMRgbQQrMfA7tXcLIUrOaKlYSb3hNCLgxcVeSorKeDIZXOHoQlhYL9rQBUYB0kf3+lWN19zPX+FR+KPSeyUFqFywQd0dISvSBXlpAPGGIGIo1eMsYeLyTDuCOTUVpMzzdJnmpwpdT46NBxwQeHAdXv+HKz4NU6MhoBWrA6mNGPA1AG6tfcGG0s+EiOoXxXKJXuIgSMiXgP2ZXEDCl/fZApSfE75AncVQRtRJ4In9N2L1MWGTn7Ym4T8rX6TS7NYFNyK5XBgGLii5NJMbqIyBodNSaJcjPPIMkokQkT/K4gjxM+JpwvB3rwihAlvRfEsdy0rRHYyTdgRERGCA4c5XWNaooB+ohPgcF7+jGgcym5S+K1Rl9W+MJojMg0wpJFIOSuSAhuRNQlIvmaJ4AongrF6ybdks6TukIogRQWIevC0LFlw6wjJd1A4bKEUEi4U6ClGAfVa1Mh/LymgZ97GLgYOhYHDXadwTl8U/R0mQV9TtOXLCHrEpFIIhJVQCmDyoK5M8ioph80kZMrN2dfinSEkOy2e45jzdzW5DFQCQpoITqUBHJEazvhIovOsljM6Yae226HsoYYI4u6pTKW3b4rFZBCIFRh7WdAS8VMW3LOjNFzvdlQVRUHqxUHsxlnpwccHbY0tcDqjMIxupFx6jJQSBqr6fsLfvDRD/ni2QtGH6mbhqbyiGi5s1jSNBXLleLO6Zy71QKVBaMUCDuHasnu4iV5/1Oa7Zq4zQxjRg+eOipmRqNiQIVAkKIcuNNyXFCwzmMs8I2+21HJBQiF0SU6lZ1k3HuexgvYQb0NaG/xQhJiIGQQOhOyo55oW1mKaRArvQoulFRFjyfIzCAiR23NwWLJwbKhrjXGKJLWZUolkEJm9IH9mFgPmvN14vNryXob2O0i+67n1c01Ty6uWG93hQKX5URfK9NtUuU2pBMFHVpGQ34GIy5GwhTBx1gaxKJk4xMhexZWcLMQHMwUi+gJIhGkwsCEKZhuQwqsFdStpAmWfR8gZkJO9LHAZFSquGTOPp8iBjDbhrfNJY8WjlXtmYuMCIKIJFewj4F176m1YaYz3b4UEJllibjuNxm/l2SnESkVvwuB0XswklltykNVlIOrW0cSijuHht0Az145ZosKLSOnJvEb37rDW28ofvps4BM0c2O4+nTPB++0rIdALWr6tOenn11iW8X8oOF/8398xe++K1jMNKuq5v/0H57z3p0Z/d7wy9884CfPLvjicsZ6gM/XkbMnO/74jz/g+z96ytUucrOveOe+517dcHbc8P1Pz8mfZj45N/zNZ46zec3Lpw67GvjD3z3iz/56y2wGv/vtA756csMPP4ff+5U57cGKP//zC44ay/Ke5Td+qRQwrW9Hzu7W9BmePHe8fJp57w3LwQzkocEHiB4OZoKq1fiUipfDGipTbq1KQ5KCEAQ6FAiWURltJFnKQnZLCVkVY7Fz5fOzexWobgSzhaSdTXl/kbBSM/rM+e3IflQcrywHM8N61xduhDSIKMihgI5yjMRYimaICaGgSxEzGNaHR3zpEmE45/nNmi0Nv6wDotoybzQ2ZWaieLNSCIx+RbYZ4ytkaEiMOBJSmdLiOuHTlBKkMJmBUyQZyS5rdtznweFPeesmsekarBA8PFngQ2QY3ZRsSEhdBu3iURJoaSZzWmEvxFiMtDKUw1jkkiYqAzmUIqf8dZ2wFECUkBUyl42AzPJnzacpfb1FEJP7Xb7unplCzzlPw35MZC/QfabrPS6UOmwfp+1gsRqTpSSnQi9EJIiC6GB0gj5pBgQBpsRH6QUYnceoxOHK0s4oUvLYQxJEWY7pGEqyLSPxXtKPmQooMHkxPaHk5HHKU7+JRoiAVJBDiar76ElTDXNKkZQ9EMlCIYSetp0jUixLukNCJUrhXJzaelXWCF2kFxcjY0yEkJF5oiiWPARQ5rHJlPGLGwZEv6ebNbS6pe872kWLk5KYAmPu8aMHymqlsrb0PktDZWqUsIheQ4A4epKPKCRdHBh8IGaIOAZg02XOTpb4eMk4ZmbVjMpB8hEfIrIWk4Msc7CaU0nFy9tLhNFomZg3LTEmrndrAhlrG6Q05YMVE9YYamOJwZOC52i14Oz0hOPVgkXbYIzCGFDSg8zEKHERspZEWTL+TVWjTeL8+QteXjyn6zwpaIiOcXS899YB3/rmKVpLamWLLjcKXNqjpcLUmhA2LLvPObw8x+0D4+1IDpEhSfbUJBkReiRqjYqlj1uEPPH11VSwUQ7K6DNj76mNxVQliRDIVAlUHBmCJnsFY0EWpwhZSlKIVBKMVySlQIGyJXs7pnKop5BxZHyOyHnkzhszjh/VtEe2dLyrQtDKEVwMDC6xD4Ft0Hx57fjipePpxrHdDvT7yHrfcX55y3qzJ8aJBy7LlFw6owrLgWm9GImUo1Yisywwn1S+zDmXKI1KgpwdOUG3Fzy98sznFYczwWGVcRai8mRR+AFaJayFbidwXUDETKMEtRbsE8QkSMJiTcNBPeOtJvJwcckD3XN2Z8Oh7lgpgcrQe4GUiWwSSVTs+khKgba1hF6hbCSTGfaZ0SlSyIWrXgtyoxhGSdgnZouGeZMYfGS3FeRR4roSgbp/MsMPI09+0rMXmVpGfvObFce/XfHmP/b8+D/sEQPcPNmynkuuL+EP//WM/+v/84b/+X/7EP7vn/LsacdgF/zR0nO39uxuMu+9fcb6+pZhE2jfy3x8IaAR/PTLEeIFezXyh9885Gbc8cNPb/jhT9ZoYzmsBD+5kgxiz7vfWmBOZzz90jEYx//2v7vif/2/uMfb9+b81Q+v+fC3TmlORr565vijueTsQcVPPx34RAtO7mz51jstf/3TjjtXijpHuqSIIbLtHA/vRaqoOb9NPPnKc/JLDcdt0VfHBHsvsTaxWmogEmIkqRphFcoEbPW6nS6jJCihy7CpM9qUOHQMES0T7RySgDRCdpltH9m6jK2grsG2ILxgGA3rXWQ/9pwe1CxnltxlXnWSnC1GS3YpMEbJqDXeC2S1ZKFqxNUV2+UxPzUDe3dD3CV8Hvk/fxn4n6wWvJP2GDNSLSsWNnK8gPmh5NInlDhA2kSKERUVIURUhGQ0Kd1ihEEIwy6OVEIjNCVmqXte8Ta1ybTtBYeHDevgWI9bxosL5vVRMSimIkm+PtCJIyFbaq3wKSGjKvAdY0hekNRkVpvAcAUjPLWdykINnBh0xQcw0RZTokSkmeS/yTz4OjoupsMsUS6CMoOIxRiYJxPhftDcbhV9ZxiXmTFG2uk/lSKkaZhJuYSDA8UAOHpBiKocwkz0QwREg5KJmak4mVUYBd2459VuRxoTKhYsMEnhgqJzpckxYwjRls3ABCkq4bYiNeMnE58qzx8hBS6UdIg2c8TgIAkUFiUEXiTEVJo1Zg9JU2lFpyIwwaiERNYJmRXOebxPpJAQgSI5qExSCpvjJIWXwewXOgyE6Fl3PfPVMaEfsaohIPE46BUQ0FqjtaSqLZWxVKaYCaWwCC2JYySIEmEJORFCj/ORlCRORKqmYnezZ7UyzJdz8r4nhsDR8oCb81ukMKAkGoHVilndcn1+RUqextasFnOGwbHvPBiDUeUHlNkXDUgY1BTNO1ouefzgLmfHh1TWTEUXfoKBgJQViIxPgZzLAIOQzKoZ5MCPP/mYjz/7lN0+IipFlhEXBQ7F6vSIo+MFIQzc9nsYEhaBNJZRZnS/RwrB8v5bmMHTXV1jHx7jdhv6Z1u2MhOERHmJz+MUu1GQSqZVKTBGT9GWREgJ5xwhVlgMSk4fTlkcvT5DiInowqSnFbSqz5FkFMYnpBHYAI1KjDGXSTakMmiESNaJt++vuPdGzfJEU881ptIoXXrLfQCXYAiC7ah4tcl89WLgkydrLveBfhjpuoFt17PZ9/gUi/YfSqqjGGOKhCMzBfgjCq1MijS1ihX0r0gZVYJU080kE1Ms6OMsud06Xlxm7swtx41kXglqpTFGokWBkLStRuPBBUKX2XeeahQspeVgHrmzSrx1Fnl8tOdufUuFp0qlYhkFY8ikpGgakFaShGdwjqEXtM2clBKmLrnj0QtGXzLqRmqiCYwhMnaSGDyLmUXqxE2X8PuMCoIwBuYzxdFRxfYq8NUXAzce5jrz7r0Zd373mKuvbrn+7sjxocJXGXtP8+XFhnc/fIPv/OCa3S7y3mnPP5rE7fPEO28KmoMZ59vEoW344391wv/uf/8F731jwYM3D/j333vOR886/uPTzH/7O3N+8IWn0yMz0fLJp2vqumykmgPD49kStdvz199fk2xFMpIjnRlIfOe7F/zatw453/Wc//05v/RGxauXAz99uuYb76w4aRL/7qOBg9uRP/r1JXIUfPls5Fe/VWOHxJWqeH4O9kJxuBI8buDyPHJzJXh0VxNzkYJcjBOdUzBbVGQiPoxkaYp/RSbUrLj7Uyy1wNaICUubsFUgVeV2x3SAVEtFYzTOF+Ty7dqjdgJbFTNZwQQLxlHz9HLkYGE5nNeslOI7+xVBLnmwOiNkRe97LvaXCNVwXNe088e88hec716RokeOAjdoXm1H/qQP/C9/5V3a55+TLwQ3duT5qz1n3/w1+rO3sV4z+FfE5PHOk8WAqBtyGJGyxgQBSgE1QpU2wMbMCbFnHNfsx5pgVsyWGT8ofKixNmNthVaqONIL6KCs7nPp0nCDR1pLWzVEGTjMDTtKDBjKACWlJMtJt56G9IIDkeXmn8tNXwiBnKiueTIAv+6BydMgIFUxjJfiMFEi6TmAmP6NHBmGwGaT6PYVwWtiSIQIKRefUakaTtPGYLqxT1q6EgKry7Mm5MJDyEgWreVo1XJ6UNNWAqFazsYFXzy/4OKmI6Oo6hLv64bAvisIcyYUdIwJnyYvV3YluupDKTryIynnyRchCbFsYF9LJwW1XqqxmTwIMcUSJ5VTpwlMsepSqhVEKbFKUREdEFNJbgiByMW3UDp6E/oXTSCMCVxwRAW1rBiHgEjlCymDRFeaumqoqoq6rmirmto21KZGZE02gmACQx7ACcbelTXvWJyco5HMhUTkyH4YOVzM6PodUkFlNd3Q4U2gkpIsFQerA/IYWe/3zFcLZvMl213xDdRNPYGlMtEPZCmZiSXzpuFwteTBvTOOj1YFQxwDiWLAQQpIpegiRL5us1JSUmlN1bYEP/DZ5z/ixcUzTJNYSE3dCCSefutZ7yQvL284Ojrm5uVzOud59923qLWiFmCUxtiGOPZgG9I7v4R+HAh+w/lf/wNhDAyVo0uB3gWULrWmUlCqiae1WpalSex1xDLmRAjha353zpMGJzTJZmKryWOGXdng+JBxJXVJNoHsBUYJTIqkJKhSyfAWrSvS1pmzBy2LQ03TarTVaKMQIjGEER8MQzT0SXLTZV5dRy7XmYubjl2nGcfIZt+z6XaMMZCzKu7prAsFTJRfQhSKVhIZJRM5FY0xxwDEkiRMU3QpTYNbytPGoMRDvY9c3UZe3ETuHkiOZ5OrWkSMkeRk2GwHvO85OEvcuQN5BOc8SEFbSWaVoNIDMTuiy/QpMwpNFgKGTKski4NAPTeElNntNdcbXx6KsjwAw5gYY4HRqCzIWjImTxcSwWeqKjKfa4ZhZH+biV1BWqcUmC80RwvF5UXHF88it0Ni2VrevTvjjV+bs750XP7Fhvd//ZQx97x4KdjtPK2w7Lo1snf8+q/UfPnTgWfryNHxjP/Rv5zz7/7Dc954KMky8WffuSbHkcVsgd9vuL9S/OCLDW8fGn79l+f8Dx/fkJ+OvH8f7p5J/u13E//1rx1T6ZHn13s+eCj507/tODoYeOfxku++2rAyls3W8eJacLis+c6nPUtr+NVvWn70eeDTyzW/+80l1eeSq+s9n3zpuXds+OyZ5/NPM48fJoYuMbOSUQm64DiqFO19g/OZkCW1DUgjaVJp6vQpM4REO5M089L2N/rSbhfHTKU01UJDTrhQwGIilaibrBJiAs2IFEmU4TckjakEMxR9n1lvIkrn0nuQS7NdiomLG89mEJyu4N0q8f2t4FnS3D94wOODU1JWJGkR1vIibHm1uyS7gZDhqh+4uciMe8GPxi1/dbPkX1c9aUgEL1i983tsTn6Li9jj3SUhRcbUI6TCmBYdM06WxA4WemGYuRElAkbNsO6GbgyMYYtDo5oVtdzQ0pBDTVMbhIolvi0i5AIKK7LDa26dRKkK7z2r4xoWCyoBIid220xOvmB9KRsBXjfXkUvnQjm/Cg5PTgVo07O5mOWYEN/yZ4OAkl+XJ2WRyz9XpcwnFxW/CBeyJM+01pRGU4mQ5ZoQhSBQEiRFlJAYmWitpK4MQmhSzgxjZOwjB0vF6XHD2VFNbWHWKpr6iHunDd/56BnnVz1KJ4SP7HY9t+vAYh5RKiNCIpJxEXxO+OzKoJAiKY6EXEiGloKLRujy2J5ej5zTtNSfBqUUS6+NnNg4pphFYy7HlMiUtIbPJEfhsJQ5A0SJcuakUDEiRJykg1/gMCC8gBC43V6zrI7puw1JO5zYYBvIpqapW5q6pa1b2qqhrVpqWyOzJpmMUw7pFaEPCKGJUeJ9MT8IJREiUy3rMt05j4wjpprjg8dHR72oUER0XdEuZjz94hn14ZzFakF3s6XrI+i6vBE5YtC0TcPp4ZLHd+9wenLKrG1RWhIJhFiKJgSptBMGWaiwKZKQNMbSzGpi3LLdr3ly2dF1V+z7C2hHVE7UValLzkKjc2KhMv3+OR//ZAseJAYpFVZLYhwgtUQ7Em1pNVNKEp8/Yf3PH+NfdgQ1Zx0SQwxkW0Q5pUxhVk8Rw9c1owXSAXpC87oQGFLJLJMMSmmMUIgGhNTopMhDIoXC+VdZ4H1mP5SpWwNCJQISmwM6Fa0zqcz8oKWeGQQZqUqhE1IwRk/vMkOCIUpue8HFJnO7F/QBxuiJzpG9g+CQqaw3Q0xfO4lzfi0JlMiYlOXzkKdhJ1J019cmZZHLMJDS/2eyoOhjiSgk+xEu1pHrnWVYKZwtFdkiSbyT7DpY30pcn2lM5PQ0cHoiaEi4PrAe1IRBlhhRuANKRZQQtJVhfgpVbdn3kdt1pBsgJo0SZSVdMBoWZcD1Ix5B5yMhZKzUtAtFELC+Atdn9CBorQIdaWeKWklePBl49Sqy3hoWRwPv3Rfce7dml/Zc/veeB48WzN8KuL1mc9uRhoZ3zwLnuw6zWLLe93z0SU/TKtrDmlcXATUITMgoG7n8aOR3v33An3684+ioIqmR6+uab79vudzsWcjMm2+2vNiP/MrRit12hDrxxcXA7PCQLBxbF8hJ8iBCTpI/+t2WVy8GfvTVljfvWf7FEr772ci//LVDPnrykr/71NHvIzJkDo5axiSoTeCNB5bbq0wTIzZ6Kis4rg3rPQgjaZNGtx2dU8zbYi6rqhovPCGVKLBzhU9RzzIhScZR4HqBTx4vS6mUT3GSajQmSYSD6FMxixmJlhlCohsDPgIITK1RFbghMHRhQpKXbZrWkbireDkk5rM979UtF7NjOgxx9NAu6GNkt16z3X8FYl8KjqJivPH0tx4wpLTnH378kt//vV+iaeY0s2Py8Qe8RNK5C7QyZGGppJ6wt3tS1qjsULLByRGVBpTec9kL3lqseaif8iKsuBYKkwwzadiLCh80SpSKeRiQKIw0GJWIQRDTFmRCixrbWpKU7Pc7Hr55wKytqBA0dcX1dcd6PdBtfelryQJyWU3HECFPpW4/O+em7pryv5wLIKnUpctSC/7aMCPF1218cormFvBXZtkojhaG1czQ1noyMZdTMubM9Mcosk/JM6CUpK4kQmmMsVhTOhlGH9l3ntWs4uy44nhlqY0ow/is4f7ZgtWq5W+/9zlfvdqSpGbf7Xh1fknKkuW8RqpISIFh9GzGkT54gvcICl8ii4xNkmpukbVG6AJ7SqkMYimVFFee0gApFnS+URphJbW3iKmQDZEROZHD5Dkodz4UYoqfZpKHqBMyjWgdScb8YocBO8W+xm5Hb2YcpIzLA9LEUtBhltSmpTIzGjujrWfMbENbt4VrbRIdHb73U7kRU+lPwTxqJZAWVFUysjoJyMWD0O9HBgGLqd3reL7i6uKaLCV35odcXV4xDoG2mWFsgYTMqgXzesGdeyc8uH9KJacXL0Wc63HJkVV5cdX0MEkilzILISfjSqIfd1xdP+Hlqwv2owdZSF8eRUoGmUp1pIthyrMWEM0Q1lSNQmbNxeU59YNTDucGco0TGeU6zNVz4pOXuK/OYTsSyGzTyHUKOBExtSFmW26VIeFjWfMnRMmkIiYHviLljAuZJpeJOaWSa88iE2UCq1FGwOtaTlHYEWMIJK/pQ0SERG4sUSVsjsykYGYNUmbaxpBcIo0QXPmziCQY+kAfM50f2UV4dQPXW8ngE8EPkB1CDCidWNQSq9qiuY2puHujJ6XpC69UYYOLgifJpK9jQgXVOUmM02o4hvLah3LZmAqlihnVx8ztPnCzz9x0IGUkGolFQEjst47tRWK7hU0X4KPE4YHk8WPJmw8EJzNQUTCGSCRihcIQaGaWagljDjy/iPRDWQ1GWYYVrYt/QcoKoaDfe/YOXM5oLVlYQW0rdvvA9Y3HOMGBbWkWqZS9eMjRc3GeeXVZMtJv3dPcf7Ti6EHFNkbO/2zH2ZlGfzhDLBWbr7ZY2/Ler9XcPhsZk+PL6y3v32uJS8GXlzOebSPy6Z4/+FdH/N3/oee3H1T8yq8YPvkysL2RfOennvU28d/8mzsca8cXLwaGKFjcqbj6PPDVq5H/1f90ydV6T9cH3n8v8o+fdvzqtxd0feBvPvWsVoqzNnIlA5dxzbMfVXzj3YYPPhBsfeTdB4Y3Hy3YXCU+urrl26ea0Sc+/nHgw3drTk4FS5m5c5DZ9YmzVk6tqIIgB6TPeA9jqDloJInSaGikKA17QTKGRFaRqrVoqxmGUB7UPfgwQX6kYh8D2XtmVlNbhfeCzabghI0sHPgkps+a8IUd0UrMTON9xo8KP3rqEaQakRi6dcboC2y+5lpXoEYu+oHBranTNW3aI5VGNQsgob0A37FOgrVLzGzNcO8PUdURYgbn/Q372KN8IIjIkAd0ghAVWQz0QOM9HQOIzKySrGXmns58qL9kpnqi0kS/4jbWhFyGoFn7+gC1xFGilEYIjRAepQRMpjdJkVf6oaOelwO0XTUIH6auAMOsrbhpBva7ETeU/uMcZTHRhULuE5P8J+B13cJ0wyzfe6nKzV8ahZyiciCmqF2ZJIQAIzPzSnC6spwdVixm088hFUqBkPH16FEqiRXTKJAxWjBrJFUq73ddK2ylSBicb1jOas6OahaNpq00y3nFoqmwWhcksgD9g+f85Mk1KQzcbC4LL0Ecs5jX5BTwfsfQD9zuHMPo8SGQckZLxVxZDvVUV63CFJd+/dqXbQeUbW8pKyqHu5SlhTd4Uf4fUiCFKiRHkVGkEiNEkYJA5kQlYBxLV4KuLfP25zvmf36ZIHpSKvEvrzIkWMwO0L0ueE5dYXSN1Q1NvaSdHP2VbTDK4PEFNzo5xdOkY4eYQZZVhtAFVHk4myOUI0lJbRsun6/JFBflndNTHp7d5VxfcfDOEco5VjODaefYeoZUFZIamQyazNFigZaR7bCG7Is5RhYTl4uZHErETk5RxTLJKbRW3K4vePL0J/TdiMwNLgRSylSq5t6sYVkZKjWyWigWtWBWK7a948nFhq/WA9d9wOfMJ198zKfPP+bdtz7gndP7LNYvmF88Rd84+qsO2Uf6XN4NnzwDkj4VnwZCk2P5UpbyJmCCTJTmxLKGFiGSfEZGsFoR5OtaVIFKiuwDaXSY15KDoHwAZSYlTU4ZLxK77BG1LkQ1I4tGqgUhOMb9yNhV7NZ7RG0IMjIMij4NrPvIzThweSvZ9pLdMDLsb1DRU9sSd8pSYkSCVFzSJS5UegO0khhbPAhSlqxyyqGkBnwk51IyBEydBGWgDKlovjkzpUVeE7cELmQ2feJiWyJSocosjED4xH7I7LrAbgdjLxmGxMtXme9+nlg+UHz4juCXHwXeWWoWsiWFhG4jQkfWG8+mkyghykGUR1IuxMQMaC2xJtFvwJfvKq3SKKMJyXN+syXv4V5dM1smpCnY2hwy9cxycxXZ7yKt1ty/rzg8lRy0it4JLv5iy+nZguW3BdWR5dWPLpCp5t6HmmGbsVVCZ8fDw4Zv/d6MH/xDx9WzDV5kbkRLY2f80kOBFQ3//OMdd795yltPOz5/LvjqWnDvEP4ff3rNIA1CB/7DP+34g/c02Tc82zj+7HuOB4eJe1eJDx/NeXE+8sH7LdvY8Q8fZ97/4ITcrPn2WeIv/2HD3/yTQzQCGxR/9JstByuJf2hZzQ+4/yDRjY5hnej2HUYa7GnDQZRIyr93oBRjyKAzsRMkV7PZBeraUOkCpKqsQaQCM3NBkIQmiEg1izQHitFLfIhEb4gpI7RD1iXP3fkC62kqwyJEbnbgkiGpSBBTcY2UxJAhRrSNGKUw84idKcImMmwTELC1IA+Q3UcMzRVmdkDe3XLcZirlwCS0TojpAnHvtGJ5IHn1pcN92dDmA3Ql2LlX3DqBY4fMspj0qpEaNdUVO+gtSe0YtSK4LbZe0WHBWd60z5mLgSEYGgZWLNgQGXNCy4aqlsVcKRqGVHgaMUdC9qAUStmyIUkQkyNJT13PmTUNbd2QD2ZoI6hspK41da3ZrA37/UjwGe8Sqg8MvSOWEpHpqPnZulpIWXB/qsTgpBFIXVDocqryTQmYUkOlml4ybxSrmWbRKGqrMUqilSrNiapcIESanpO5pI+kyFRGlEKrLKgrwaxR1I1GGgVZM28qVnPDrDK0lWXRGEylgcSiNnzzjVO0tPTR8+z5LWPsud5cknIgxmPaWhbJeexxO8dmO7LvAyEJlBSMVeBID8hkMejSEKsMSjqgDE2IIpMICmAv6TT5LAqpVwhBpVQJqHiBiRrGiBYKXUtmreSohcNGst9rnr7c0irJb71x/IsdBqqmrBoyELVgyIkzsyIGhcmZoDVGV1hTY02BABlVCkrEf1KnMBlJSYhSFBQj5AKPkEoxq2qMVpzf3qBEKQAhRz5844yzs3ssZof4JFgcGnxW9LJFz8/I0jJ0idpo2pmlmkeaRhDVnttdBDFAjFhjUChiKh8arYrJouhekpAL4cm7LTe353iX6TaOYdtNh07g4VHNO2eKB7NEpQXLI83xUcXhqioQIv+Qr15c8Oxqy26X2QX48bNbjm5f8SBdYW52yC4j+sxgJLpWNDLhcsms+sGXeuZUbvwxepSUCFX0MCTkNNWNKomKxVcQnMcNCl0bSAKhQGuDcoK870j7ALJMoq+NtEZO7VZSIoWYzDiZZBVoicuByiiGELm5GWif68IBqATKJWKo2Maeq33g+U3gel8OwHF0iNRxshAoKlQWRJ/Y7QNaREKAfixreK0FxhQPgtYKKcteIKeiy4ZJFkk5k8JUNJbK7zEWUAliMhAhS6RIQIiw6QQ3XQFNmQQiJEQQ9N7Qi8SQ3cQrKG2RDsnN2vDXTzV/u9U07chD6/lX92reyJLdLiENzOcemRP7qEjekENER5jXBoNkv8tEFTC1wKCIRHajo9vAUtacnEqMjNNaNqKzZXak8CIgd5aTA4W9r9BzyWyW2IbIy3+/4WS14PhXFfJUsv7klnZxQH0EadNjYuDJVWbZtBzd1fjrzPZq5LhSKDxJaHaDYnEy46MvJQ8PBcdWsFpofuebgvtHM/7xi47Li8jirOLxoyU35xv+8p8Nf/gHd7n48TN+652W9U3PP33h+OabNS4pvvfDkXlrkKrjr390yXEauCcW/OavNlzfZj59tqc5WvAX39/xm+9JwqB5+Kjl2csb3jjWnMw1PgQWLSgZaZtIzGUwXFgDfY/QgiFJRufJLnO5C9w/blAiEMZEPQNpBMOYCSGgC6UC246YyiCtJsWMloKYIASPtpGsNFsncMHT1BJWcHk70kWDFAoZApIBYyI5KWKyJJ0IRHCKcQQvNVpGGAQiRZZpSz6s6BaJx4tEdqm8r7VBMZLHFkyE1DPPgnu/csDdxw1/9t8/45Of/JR33nnAflgT6dEEvFS4XY+RFqEhpR6jW2y0DCFj9ApsBf6Wu0Jws70l9BZdK0LwDGFHCIEkKpp6hhUJU+ZzojZImbCVwTiD8wV/DJ4QRhICpSwpQVNX2EqxEDO0kVRVpG4MbVOzmDv2u4G+D3R7T1e50vY5eqLP/+kcAJSNAEoidUG/S6OQptyKXxvqREjkKImxmDYzhcmvjUTpSTKdfpXugFy2tum1f6o46YUoNc9GlyGkaRSzxjCbmdJBIBSNFSwbwawpNFetMlqXmKWOklZnHt9Z8JvfeIwbI1dXe9w4chVuyVFycjDDGEWlJJqSUCp/fnAps/ORi+s1MUraoKAuqa0YYkHLlx5sIpmUIjFEIkUinrjC5fIx9bD0OoAyCCVodOLRXcW33prx+ESxNInBJa5vFLPa8M23ql/sMHCyPGR9tcY34MaBvGrwLmFnc4TvqE2D0TVaV1/zpV9/Al7HSsrCpmgkMRUNN8Y4RdMKrGZ1eMT1+hznR9569JiD2RHz2RmmXuKS4mJbShycy6QUyMoiSMybyMmB4mguaWeCSMLHHpEydVVQZNqWW2dIvqxvdBlUQpkESCkgZDHO9N2I0oa33nwXeV/w0Q9+zNOXV2RlGL0mjpqkHYP36I2gaTR166hUpK0q3n7zgDfuz0h7R7/xfPu4wXUCtYvEIOmDY8AjsBgxUBmBUUtcEgxxzej3aKlRIYJSaK1JxOKvkP9JrEdApSAlwZgiQ4gIT9kShISwCjFa5DZBn0FRML/kqfACsoioXKqZxeuGx5gYQibrTIoB5Q3n5/synMgKNU/oYSSlkbUYuL7NPH1xS58SzcyireTk0NK2LUtdTDApJDY7z/PrgY+fbuidIyFLv4TVVJUtUoF8vQGQxBBLfDJ6Qi5Y0ZgKbSClPK0hVSHN5cJIgFiSIDFzu01cL0SRoSjAFOkyYe/JMZbSKiXIKmNRqNaQW42Xks7Dp68y+1rwrx6ANYnVaUJVgr7LrHdVOQzCCEJQGY2SRYM2RmN04nZIuCAIXcREeDirsWpyaifIWWAbqBeRsfOMV6p0RZwJtA0I3TL0W87/0WDnLbNfUZj7me2XPYtVi1gGhhceHQ1Dl1guMof35+Uh2DtWC8mhyrSnM+ZHie/+1XM+ep7o3J7FOws2N7e8cTLnk+sNF/uBH31v5N/8dss/fRq5s9C8SIKDZcN3//kJJzpxdLrks6c9fRr59KLnf/Z7S7ZXgeOF4f5Bw2dPtrz1Yc12N/L8meewybz/SOFC5B8/g+dHgtR1fPJZ5PhAkeeC0zpxOabSITEoThYGKkHUA0or2mgJIhIs9DEgsmTYJy5Nz9lS0bSCcXCoxtIuJaMLJC/wnSQHQ2wTZpFRVSrf7yDxo8I5ASYjq8CYJNFnjBXMjg23vuIi3adu73JkKwhrxv2XyHiFngh/4xBhgAUZrMBVCaxhY1puHYSbPUpYhnGDi4qIZK4Mye0YTaTVAicVQm44PqpZPaj5v/3Fd/gXmxveff+ISoAbA75JZCUQKRC7kUrDTo4QE9rMmMVbHm5/iB0GcjeyzS3X4pZQLfA5EmxGSI3JFcL0KOExqUFXgeAskYBSNUIGvL+FrClPUIhBlNixENhKMWsNUYM2lqr21H1DW3naZqCpLdtdXwZ6LXAB9kBMfnLLy5+lCqQErZBaoYxC2tfegQnGE0tyP07JhJjLpm/0cfIFlK4QHzI+gskU+uBUZlb8RAkxtc+SYlnBA0ZJKmuotMZaqIymNpKmylQmY3QBVElKZ00WmSFFlIQHqwXvPDih3ye2+6HgrdcbRIicnsxYzVcMQyDHQGssPglcTqXzxQhyVsTwOiUx2Qbza0NFnIYC8fVGQIiJrCJAa1UgeQrqpudZ2JOk585xzW+8P+O33m64e6ipKokQET0mtMrU81+wgRBtuf/mQ549+5LUjbDMRDkyNy1CWLJZopSmMuVwzqnAZEJOJSU+mSR8Kl7VEDPep1Ju4QMn9REP7x2z6beo6pD7x2+jTMvlzuOz4WYb6bpbvB/IueTF503NwQxODhsWM42uElk4fB4IzpWBShen5f52w9gPLI5nBGMIWaByRCZdHK7CoUQZwlwqLlGj4KZb01YL3v7GY3x0vLp2hKTYjYGdBuMScthjtMDqGZaEzcXxK3z5UKYsODSKXni6VAwe2gmkl+xTJgpJnrU4XbHrB8KYsKb04pWb7lReIRRaSzIRKYGpjTFbjRQB0SecS0hpEUISnEf6QJ0FZnL0ZkoUUZQuFXiNrJySe0IJAonOZUJWGC2Z1xkVE2mX6YceUXeYuUAvPMlInMhs1yN5cCwqzVxmmpnBGsliYTiqDdZqBIrdbmQ5jwhV4WPi2YUg61IsZHXR85Qoth+fFE4Wg2mJL0WiKN6OlAUZNQ2d0/cqJaQ2CK2RogRrbvvE08tMiIreRjaqRAS1F2ihEbpwAoSOmFajljXjLOOsZ5ciRPiNU81d7dl2mdFJQh8YBCTpMMBCK2ZNolGAgWQlInmGXUAOhloI6rqhFYIxOsYYqFTJdi9OA7qx9JeBsFbYRkxV2opoJMFnXn6vaLSn7xtmjzTxvGO2MDg845MMRn9tqDx4V9M+kuRLDXWP+UGJYL55F24uPF88TZwuM4TA3Urzo3++wssVz1/tmZvyeTDtirvHN9w9Ufz5jzJ//LtLPvn4gn//I89/9Zblw/uWk0ea/+4/dvzJ9zf89uOKGC2Lk8hp3xFc5K23DS8vM69uHAcPJbYW3DkKXL+CD08qPt0WpHhra/ShxF+DCwaXA8paVm1ghKInzxRb77BGMgbwKjHEXEq9yByJzNlhxXrwqCyo20LPHDYR4QqRU8WMqUHZSEwSNxa4lEzF56F1KZtKCqzI3M0dje4ZDu/gmhOut1/Qq55G17RhTdhFvB4Rc43D4ySMZLIx3DrJcDVgpAAxMAaHixBF2ZESCiukMhnvI5WUNM2Gx3cbbm57/t1f/TMvLu/z7W/cY76w6BxQSLTURCvo40g/OH5jMfDe4px68xJioptlnMzUu46r3UDsPEkoQu2QVaJV4FVDlNDngBwkPgiUmqH1QPB9abkTpZYYmZBKEEoJAcYYbGXK81wWA6LRisoalNFkFFEoQupw3lM3Eh8UxNfwoAwiobVC6dJXr7RAG1Fu+uI1cWCSCGMih9IMSpKMKXJrEpebzMlecbyAtU8YHwijxKaEJE/R5CnAqMBqidUZEzKREusNOSKULqVPBtpaUFuBkQGjSneNjEW+zkQkHpkytc58cPeIsYt8//OXjL0nMXK9K1C7e/dnnJ4u0HrEhZJMS0gERfufLywiGjwBIcrKv2zNi6k9ZY8QNXlKZqhcts9OCOZ1xUGjMcLRWsPWVdSV5Jt3a75x3/LgTsOdo4rWJHIOqBjQlK6KX+gwsO47EoFHDx8yDgPjGPAzgWkX5BxR/rUbVJOFIaDIPk6tbpK9iPSplL6EkL/GePbeM1vUvPHhfVYHB4wu0PWeq9uBEUPsR9x+zSg9o0/4nFnO5jw4PuSNewdY7dAq4HNPTKVswodi5jJSIIhs1jdcPr/g6P4JslkQuwRpRFtJln0J0khDAfr64pivanKCm2fPebW75ORoha1rRO7xPjCm8oaZuUHXFmM1RiaEMgQlEDEivSvaT4wEl3FjxnnPGEJ5OESFCoGoLHtteSki5+Me50aMFGRTwBgxZVL0xZ0rJsepKPCd6DNRaJSosEYRfaKbyGc5lqFBCFUs9ymX23TZxUwkMMpKfSKHqVyAITFnfPAoIUrcSEjqCNYHZucbZotE3WWYVQQZGPcOnSRza1lUhrpRzFvD0UHNUWOplAIkq8pgZMb1ifUGtt3ArYtUVYXSYIxAKwM5oydW+agTqAJDEcQyyIiyYhMCiIKcG4Sc+uxzoqJ0vfssudoKIBJnEKykFZkmZywZKSPBeoJRJGXICnojWQtP53t6n/lySFwNmnmI9DFipGCmKpSFyoLNEmU8WQuU0NTlWoNaWNpZRm8FO+e4diBRJONRZBb3K6TU7F8mcjRUJ5nQD4ioyVogQ+T8o6H4bh46lm80kB2qNngy+6cJKyALRZAjs2NL9VCShoisG4QNvHy+4Rtvt/i94vnWc7zULM8kro/cn8N3bjNO77g7txwdW57cdmQ8d+5o/uP3bnnnccXB0ZZX28DRCvrrW+Yn0JjA3Ab2Q6YbDfOVx6uB995ZEnYj3/n+nrfeavnic4XOkQPRcX+leXkbiALOVplNByTDYpYQsebVTSCKCiElphGIZLG1QkeF3xoiCWcsLka8SHz2wnF0WNHtPEOU3H8g6XaQnKCpQS4Eg0ukJEhdxHcCoRVRBvJUBhRdMTFLkTAmU9capSW1FKTuOfryL3nZnHIer9DB4Rx0lSJVgjEKhgSbIeNdxqLJvcOExCImZrJs2mJiircJvIBBCEYSg5P02dFruNhEjJKcrTxaRp5erLnqdrz9YM57j4+YtzW3/Q1HleaXVoKjdMGjoytMDJgTSZ42ZuNeMW8qRudw+4AUEZV8iapVkigcSRV5R2kxXQwyITpICmNmpNER4kgMmbqeIRNIqcsNOwuUEsSUMUKRdPHw2ApMk7A+o6wHXSRMbcprlVIut3NkWfObcngKXQa+ovS9jgcVuFKMYSrfkSXhlTK7LvLquqepFdZoslHEHFm6RGMKQEpPjAKpyoVk1mhWscSNhiDIIhHCiPeJ2lRIDFpKKgVGRRQFHAQl9kcOGFFK3Ba1otIapU+Bju9/fs7QCQwjN7vMcr/gYDUjrhQhpeJzEq9JjommNlhZs+lHxlCkjSimS1kWhSKYBSGGgsaXCi0rFP10lkJbK6yCula0NnPvuOHssGI2V9Q2YXQhK8hUSueEyP9f5/n/X8PAneUdNIm7B6elqlhJumHg5rxnebBkPp9htCHGAmLIMU/Yz6KJClcMKc4XE54GwjBSt4Z/+V/+EQ8eCV5cb3l2fs5ytWTeWrrdOfv9UA5BpTiaHfDmvbvcOa6x1ciYb+mnSJPMHhGLptJWCqMtKgpqCaqecbxcMeSKzVVPaw1V1SC1JsVATGNZL+nSjlilEaEUerbkaOXYrJ/y+edPcfsRIRwIRbs84s79GccLQVMLZjPNfCaxKhOIZFlIUD5ucC4xjok4JOgFuYOCNZQIYVlrwxdu5JO+43LfIWIxoUWZkZrJLp9JMRNCJKdc0JU+4/vypSkbmVKikeTE4p9iJw5fzvzXmvnk4J3wYBPgo3QSiJwhTXpcDJDLhiBrSZCSecwM+8juYsA5CXtJkCNDCKXWGkltBI2B5UxzNLfMrGFmNCJLRpsJznPbDhy0A8oOiFiTsismUaORumA5ZUglRiMzMUYUAl0eGeVWN1VUZ1nw1kZYJIYQDMlYnBJFWhiLSUplEDNQTaa1pQ2yDBYCOyTGHOlSKr0BeaRZVLz56D6hnvGn17f8xuKa9w9HVhKE8EgJUSWUVCVOaCQqJ/CC6ASDE+z3gZQ8KUy6aB2YLyT1aUXce3bnUBuNOVFs9j0yGrJ0WNPw6tOA8In2kcbeLw9eY2tS6BmuInWlCNmjSQSdsW9ERJ/AG+SJ4NM/27FatnzwXstf/801l93IneMWkRxvPVpgVWG+v3G3QQTHw0PLMIx8/mJN1Xh2N4H3l3Nefg4vrxO//+07WNYkrXBjhXKOD9+KvPvGjKc3A3GbOLg7kqzhyx9H7t0ZOD42uF5yZ+HQsmLsHLcDPDpU7LeefZc4PlOIxuFHwzAImtbiQgKX0CIzawP9qPEpM58X6UejeLlO/P3Tkf/iQ835F4lRVXz42LDrR4Y+k2tNO5N4B8GL4jOJBRZFzggdSpQtKSK6sFRiwlYZawWqkszcOcfbK15sJTdZoKKHAN6B20nSboRRYLJE9D0iJJo2Tf0iEq1LxE3I0oRZcNsKT2bbSLpYYpAb1XAbFT4HTpaa0fcIZbh94fl4t8Mezuj2I986bDi+e8SbR5raKaKsiLLE2KwzmGUmMLJoG7bdUAxBzuNiQvqMjXuyWTFkST96YpJoXaqEtVGkKEA6pMrYrFFKM+ZAFhBDxHtPShGfMjKUZ1Kcmgdz5mtzeMySJBRZadABkSnV8EIUaUAVt/+EEC2/Uyix8XXkLheHPQJSKOfQ4DKX65EoYKSiw9J7zdkSDupSTW61wprSkGi0YlaXrYMymf2YcAm0mqA+KqNFRGVRCuzIyOyKvJDLACNjQqeCH1Zag7Es5y2Hy4egE//40S3jfiQCNzd7DmaHzBfttOGQCFmX561IpUBKK8as2Y2ipFt0MUATNVJocpwgTEqjdY2SDikMzpfqeik0SUmMdrRSsJpJZjVo4QkhoXIhy4pcIM1k+HnChT/3MHB68ggBeCnYdgPGSoLXpCGx3d2yP7SslhVGwsxKrBLFJT6BZFJUxVlGhlzifVkFfvePfw85E4S05WieyWdLRmrOz9dkIrO6ZVZVPDo75O6dI9qZYD+sCwFLKiyaSlTU1RyjBIiIJ5bGvAy9hBQDV7eeZ+fXzOcNd6pchpY0omRNrS0xOPpRkaShUpZKa8YM9WLFsl3Tb7e4XHSoulbcfTDn7fdPObESFbvSJWkqrIFajOW/F+JUilTy/OOYGDtIfUZEGK3i0lp+kh2f7XZc7PviElUKlyDGhJqc8nJaG6VQSjFkNsQoSCEx9omoXDnQc0SaKWGgZIH26bJKjkoS+0IwfF38o2QhDQopS6wyyf9EwxITuKmQraQSLI0s6ZF9ZswRvx+IwpeV7lyRZ6lQAslomVEiIlWFMTUiC3x2KK1Q1pKFJaGIQZIMpJToxgGdMwJJ8tOgFnzpK88ZIw2jd4hJNjEUA5CwxacihcZKRUoOvEJLA1oweMG6z1Q2s2gCSeYiS7SS4ALjVqC7hNuNtG3F3cdvMFvN+fzJKy66p3x30/Hphw/4o9UdjvMNR/GCZR6ossDKjIiJJHuEgxANm2wYQ6YWxZm+WkmWRxl7oFFR0d9Ehm3BIFdn0HU7ktNQSVSyXH7R0feW+QMJdwMueUxV413PsHOoIAkqYirLsB5YPTpGyL4w4+vAbn/L5Q8s77xlOL+4pppr2tEymxtshPcetHz2haPVkmOd0EuNWcLmx44RC0vBnUN4cDTy5FXizlJye37F+x9owhD55Ks1q1ayCoZGenadw99E0txg55HTg4zvBAvd0yvNiObeCaTU8t3PBt47m3M497y67XjzbMas1jCXXLqR3tUcH9S8uu6QUqE0LOauDAEGTD3xGuaaj79MiE8jv/+u4kefO3SAt94vWNzdThGCw9Qe02icE2ULkGU5vMZAVhJhNMoqkigFW2Gf8GO5eQoJR8bx7mj4y+8MbF4GbCzx3ZqEFQGpMkaK4pVSCjPGUm0tElJKNBmtpux7jqXUK2VOx9JAGrMn6ABGsRdwJVueS7iJnlE4NrtM7Tpilnx33XF+3fF733zAf/FuoGpu0HlEjA0gcHi6GAkpo9VE3YsgXEQOHcLtyU1gEIrR2NIeG16Dh5jMa5BjhRCZYejZec9cVwyjR+4LfC6UBCIhRkaX6AbPfoj0Y2AYM6MvWn7KgCpSpJRiMgiWIUDpWDaYqgz0ZRgIlLBuLhcTJEKkr0mHQihczFzuPP1zz3aU3Gwsb50a7i4zq1oyaxVzUSBqVkmoQCowVtD4RO+K/t9WUKuElQJNQKWAEhkpihFRUroTNMU/IACrUykBspp5u+T3PnzMzTbwyZc70uhYb6+43dXcWS5QVekbkMqALJyElMphL+zEVcmgcxnKcyrV7SEF2lTig6+jmFIpQnQMPjOEEn3PCWaVotESQwZfhrZkircqyQJlSzFR/yKHgb0rD2pjFSGM0O/RQpYpTFdsb8/Zb69xPjBfzFks59R1xazRVLUhViNjFwpeUUkW94/49W/cJ4iB7e6SN9++x8PjI66urnlyvWN1sOSgaTloK45PDyELNt0Nl5sOoypW1RFtVRdUMJ6YEr337LuBmDNCG5I2rNee5y9uiYPn7Kzh6FgjkiAFjVYAHk+FVAYjHYjIOiTW64FNF0heoewxrb1l2A0kYclomloyn1sWdY3MC/aD52Yoa8S2qmmIRYc3FqHKumvwgX0YCTnjsuFFknxO4KvYsyOhlS4rolCwwzlOh6Is8RQhpk1BlMVVPxUJiSxIAYRMkxEykWNCZInJAlkbZKWJVuNF+BkuNCVUnHK5E1tb6VyMhCK/FvAmt2vR3oTSBe4TE2mAOAR2wTNkgQmCsYZxoTGtIKfSoqaFLNjYKOhHz955rrrIy1vPbq/xYygRokoSU3l4vYYIhZgQEciiPKhDyX9nQOaJ/S0yQlQkkRDWgUzkOBHUUiJjiULRJ+gCODQYibQJqzWzxhAqx9X5yOnigNk77zHWhvX6irvLhpduz4VLiPk9Lpv3OE8jOXSokMjRQ7dDJUeSO7xLjH7klZP8WrPh989GDk+gmY/ouiJ2it2VI42ZhaowZ8VT4reFHRFUoHte/nr2IGDuGjoPq9ZAcPSXkFVD1oJajVxdDRw+rtGznuwEHoFZVrz6f21ZLmukicRbS4djtTAczQVNTJw8bPjky46TuiX7xKM7cz693bCJgvt3DZvtyMxUHM8HfvyZ4d5SE8LIfm15PDd8ykhlIsu2ZRsCs0bQtZExW1Y5s1ooQoq0RpGA2yg4toq33zRcBcf1tePuYcWnFwNrByeVJdY9dUpsbgIP7lqqqjwJlRbM5oluKGx6GyN9LjFKYRQ/uYU71/Dt08Q/f+7YKcU331OsDgK3XcaH0qppVNHdXYhkqRBRE1MgjgFiQNlCGoVy2x16h6w0WVpOF4nffsPw9y9KX8esKXp0Iy1ZOYyStMkwEshSEXIiZ12Io7JEeLOeBu6ckSkR8OQ0krNCB4NOgTZFjoDHyvDSGJ6Mggs34vqITqUz5atrz8XfCT55tuRffese7y8vcf2OzWjY7jKbm8Cwn1C3ksIQiIk8RkiBMQ50emTvPbUCKTRGa6TK+NgV0JnJRJdIubxuAsGuG/A20A+70uzpInEilPZ9ZL8PdPvI+rZnvxuIsbx/xhZC4OtKHynl9DyLX/vMs4jkSeJLuZjo5GufwSSPKlVYKcIUA3Q3JJ5deXwsoBGRJPrQUjdmkn7Lpldp0AaqBE0Q9GVRSqUztQ5YmTFFSSxDIkVzF7n4jnIqzzuVFWiB0pIgSgfM2ycz/rNv3eVm+zm3VwkXBl7d3LA6nnO6bFG2JCQSpWQoJ0MOlms5TBJEhlykZAQTUro8+0MIkHyhCKqE89B5gR4EQpcGw9VU1a5SBldKq0IQiBwnQF0khMDyFzkM+DBSWUVjNUpHjJCs5g2mtsxmcxa6mAJ3Xc8YEuN+w/WV50ZKtK1QNtGohj5oFnfe5P7hAcFtWJqRRo3cv7vk4uoSnxyP7s+ok+XB4UOGnHmxv2C/32KE4c7yjGXdEkJg3W3Y+gJZUSqDnBOUQOuE94Evv1jz7LLHtpq3H7XcWbQkn/FEsEUHE0mThcSojOsz57cDF7ee/RiLvpsVrW6R9hAlbkii5HWlUphqjpOK3kde7Bw/+HLg82uPkJqTVnBWG5YY7GhxKREmS+BeCp7nyKeh50nMDNmXD4UqKN0xBEQUX28FsinOWQWoEqalcyPjEBEByK/z9aUNi8zXEZ0EpJyIMoMuelnhuU9nfcqFVTC5cHOArHKBfeSyGRAJXIj4XDoPgsh0U0tWTNB1kS4kdARTQT332JlCYEq6JCeCKzeIbec53zo+P9/z5OWe7jYRVSho6iBpksAGAaLcarzzpV0sCmzdQCygohzD5IGY0J0iE3Lk9nogR42xNUpJjK6mprqMMAqPYAwQhULYjDLQAj44Fg9PEI8/QLdLxHpkda+m278CFahnS47MDBkLZ93LJVEFYhrwKuNUBeoAKzxdChybG37z4SUP7np0ZcjAeBvoriNiELRGI088YmbxF6mAoZTGv8oMG4+9J1FHmbV3VCYTQ8t2HVGtwruehGSz1py9PUdVgew85Aq5GNi8ivhnhvlJonOJmAxddDy8p7EEHr25wKwkIkGjJSdnmqPjir/7zDObW5qDwPkmcTyvaGUNjMzngmGfsSkws4JVrRl3jtpqrl+O3NwEjmY1KQVmSnPcKNbRI23LkRWk5Er82Hp+//cO+ae/vaVSFY0JPLkduft4hkyRI13j94l9Fzg+slxcjWQExhqaVpKGRF3XDCFgq0BdeTon+cGLyMOZ5t4y8eXnjjBWvP2hYn6g6feC4OJEqE9oJZEpE2JEI8hBk4KAPUSVEZUovowEcQxYPyKU4fHjCq0EP/l+h3eO2mhqJRG1QsaETL6UUIViCRtDYowJZSSSgMoJLSVJQVaKnAPQoqRgCI4UFEpYdHaIMPKWtDxoLU9ty086uIwDJoAIkU5u+ctPO37wfMn7Jwt++Y6g7S5wWwW6IskeQ11MwXiIgsErhmTYpwW9bxhjQNiRHAdSKkVfQlSAI4QJeyyKtm+MYbsbufYd+/2GkAU+JEJMjH2g7wJ9FxmHwDCUIUEpiTYg9UQVFXKChmWkiCTihBhOkIp5jqlaPU/slywSSIlSAm0SyoDQhd0hBIwebveRq43npDGczUscmVRu00oJjJRYwKeICVDbYuhTIqElWFkkxBxKU2uMZWiSFPWigN5KEkuKCDEQ80BMEVvVvHd3xqsPjvmb71+yu1Hcbra8fHHNYj7joK3J088pBGQkWWl0FuhcQGtZRVLMSCVRQfK1yWTaCmgFyEQS5fPUjRlNwKrE3CosnugiQReMfpaCQDHKjlN18/1f5DBg6hL5Wi5bTlYnnK1mBL8ra/CUCMJgNdxZmrJSCcXIF1LJgGYf2HuHPtGYqsEKzwcfvM37D47Ab3l5fgG1pdVH+Lzl+PSQy+s1r9YXYCN3Z3c4WK3ox5Gn62u2Q18gFTlRa1OKKlKgbiSD9zx92vHiiaNtDY/vzLmzbCAGBIK5FRhVlRuZSoxp5OYmc3HVcbFxhKBQUmNtGSmFyGWCVXLi5XtG53FjIEbPxy9e8vGTLT/+YuTJeSCnGl1VzBeK49mCe7VimRQ6Cro8cJE9X+WRF8GzzwqR8tcGmRgTIibIalohllpRQdHVtDSAKO1cMaHQU4KzRDbl5BwVSk1bgoz3Ho1+bbyfojolSjgN1RNsoHzwY05TKLQws3XWBGDwsA+wdp4+RCqlETGw3xd3t86RfeWYLSWLowYZJSJmepcQKbDrA682jk9fOT575ri9TXgXGXIBPSkjcH2PxXP3bEnWAlNbYgCbM5t1x7Dvi6RhLSHCfj/iUqZCcLg85Pd//be5d3bEpl+z7QZubnY8v74u5VVes1QVWmmsAW0S0jiSEFSrdzg6fJ+trXHjyMmqppaCm6w5PH7Ii5sL5FyQVCZHicwjInlCHjB1wiaNpi2rQBX59uKGuw8k0pTI0HClGG4y1ZBRM4N8oFFYUl8+SxhNdz2Qd5r2TsO4GHAx084MfpvZp4HZYs5Nt8YONRA4eTsQqx3WN3gTMSmjrObi73vMQpYGPQdrHE0DR63HYzj55grcgOwsQl3zwQf3yMc1115zeiTIQ9FPT+9mmrqs3LUcsY3koGlY1DUre85zZ6ibQBNHjNEsbFkLV0rTmg6aAvs5WoEfDPthy9l8SaUD3/6DGec/Shx0cL0J3PSOk3ZGP3p8SlxfON7+RsvMOVKSGCVoZ6UM5nCmQHiqm4itE15ClzR/d5741yvBamV5fhPZ/1Pg3fcqDs8MgwHnFCkkFHFC26oCixLFaJuTKOwKnwlRIq0m28Q4GnLQ5BQ4fpD4hrV8/rEiDh45Sxy//wZxHEiv1iTvcXsHwKyao6YbsTFlwJVqYq2oiRBK8RTMZGlVS8njo0DkBcSByg28oeBgteCTXcMX64EueaSMGLfgxfaGq+sZzy9m/Gdv16yqy2JoqATRA1Ma3wvJWjXciDk7lngWoBu0aifiXWlCfL2J0Rb8mInel2eEbbi63XPtNgz7kRiLpBIDjGMsEUhXCI5CFIOgVAmlBcqULYpSumwxY9EP1PR8KzfxOD3jyoUjp7LaZ4pSKyVRpmT/sy5+IUF5vjnnCWNGJYUioUVJQdkptihU4ZDoWNb8+XV/QooFREfxJKQA3iWGMZBCnLpPyi+pBLKKyJQQwZFl2RjF6GkrxS+9ecLl7cBH/Yb9AC/Pr2hmNapS1I0t0pScYpLRg5i2rEqDGMtlLk9sAh+IVpWnbxZIpRETm0FMmOYQM43K1KokDlKA5DJpOtFDyow+sRvAjT/fGf9zDwND36ON5un5K3a9JaRD5kaQkqfSFWRH1weUVgglUHKqxNXl0KlmCxYiTQUUghgifXfFly/3zNuKO3fvc++RgRjo+w3//MUnvNhes1wsubs6IGN4dX3NutuCLlQ8LU2JYQiBNCM5a252kWcvPC9edlS15O13DrlzVjEMPQlDZUzJrWfJPiRutntubiO9Uyg5Y9bMy1peCERyQE8VbunjDcJqzFgT+8x2t2dwa06rmg+PW5o+s3mVeZoTnc/oPDJ4we2t4WVjOa1rmqi5igMXeWSbwWWJTp6YS8FGJQvcYgyecXL4y4n5raQo0ceCAUELiRdy2pdPOr0oDO7I1BwmCrqS/LrxfDriU4FpZAkyvfYJTCsxWTDRBdZRTD4uSXxM7GVk3TtkzlgpaTWoFNh3BWgjNMQxkcaMRpJjot91jFWFdZnrbc9PL/Z879NrPvp0y81VYAgjAU2UgrHf884bJ/ybP/4tFrPMv/+rv8BWFpci1azhjeM3OFye4JPgT//8PzAGybuP3+C//KPfYNt3VMcr6mXL+vaW/bOOrb9muZTcOXvIrpdsdz2V2rJsPMtGMK8ktr5LXr7DODthzI687ThUNQdHNZ0faNSK0W2oK0tlDW7sSdFTFvIarepSJV1JBJE+ah6aG37p8RULE8hK019E4haUB4419o5B9l0B6TjFGALqtsbcVoSjRL/sGF2krWbEccB1CrXwKDmyVBa70pgHYLxGugBNRI4aDgPP/sYTe8m81rjoGbxkR+D40GCUYH5UoU4FfBHxlxsePTxm9lDSLSt0ljSLwMVas6gixxgQILVmpgPRRkwTmFc9i1qRRaCdH7IZBqoKtI1EB6bKYBUzoXF9AJ1pW4HMFpUyOUhmdxIf/s6c8U9Grq4DX15GHr2hMXPFPkryGLm97VmtLH1XdrgHK4NtBZ3fY3TF2Q4aM7BHISrFl/vIv/0c/ui9wDsHlt2Q+PEPe+7cBh691bBYetyQiF4TvZjqdyNSZ1JVHOg2i6kAxhP6Eak0yRZIWR7LMDw/ULz3geb884heHVA9fo/RVlTvjBjvWH/8Y+Rux+wbj3Gba/zuloqMUhK0JMqSiKmyJaT0NWsjCo/PkjpKlOsYpSKkAsw6zhvmK81ZZfnO88TL3mDVNVWas/M7vp96NuKAf/HgPg+rS8TQg3IIJFZolLJ0ZsWlPGSUNSpWpKFHxYp5ZVGymr7vJfpMNsQp8x59IAfY7Aae39zi954wRIL3X0uBOb/Owye00RilEDJODIESZRYqTvyAXG66U6dKKeope1OlNBlBDJlELKkpOSFE5LQVmNgoqawyMBLmteZg1jCvDLWVVLWimlJeyFK+pCazeakzjJOhNBF9JPiEGzP7LrDZjsXbFUoHhhIJaxW20eg2oqzCmIysMtkKojbMas2vvXPKvg98+rlj9IFnr86ZrWbcv38HgBh82bjKRLIGdIWUHhlAxYyiyDEgSLJskZOQQJEk5MRKSXkoIDcKlE+k8md1LiHG8hzvQ2Q/ZDb7zDCkX+ww4PqRnBLej2wkfPHqlrtHM04XBqM9ZI1W9mcHlxIFKJQLyjalQM7ha7dpSpRb8qbovlXb0GjL4aLlzvEh3377A949vc/FbsuzzTW3wysSiqquS5ZUGVwCRMRawdVW8vT5huvbkcGPIBJ37q24c6cluwGFwWhQ2tP7zPpm4PLGsesKBU4lgVIb+u4lvdtxdFDz+GRJLRJ9t+WVCSQZicrjk+bq5v/N2p/9apal+XnYs+a99zedMSIyInKuIbO6qppd3WyyOalJiYMtiQZpGBR4ZRjwpf8MA4bv7EsZvjAg24AAyRIEk2xaEsfuJtlTdc2VQ+UQGeMZvvNNe1ijL9aX2byUgEogkagsVEXEOd/Ze633/f2eZ+R2s2d26dBNh5wFoon1ARwzjRTYI3VxmAJPkifELYOfiFkQSiZmKDRI5WmR5FAYSyYZgYwFnQTCuPq1k7LGarIipxo8kuo4vkr1cFRKIZfK9y54QBKpNjCtNEUXBlGRnCSB1JCJxFABTGMKOFGR0DmXo32vUheFgPWQGTaFs6A5k7A0HpEzBUlnBMomdFI0xtAZhQ+eYYApFK4O8Ol6w7/5yZ4f/OAVcd/x53/jN3lxfc0f/u7PmM00f+9v/w3+5v/yL/H05glGw9e/9nXyuOfxO3+Rk/MHDCmSReDpx5/yzffe5pvf/Bbf+bXvcHo653Z9zfr2iqurp9yu75BhQkvJF3drWr3h8fk5v/LNr3N6do7TGWcNWjdgZ/S5x/tbTHK8/uAdzk8XXG0/5eXdKz5/dkXvD5jOIMZ6+LK6hjyt7Vi4htZ22KIR2WLSwK89fM49u0EUQ/8yMd5FutjRXQjSfY/cBpLWaCkYdhNx70gHjzkxHDrJmGHVdkw+c70RtELS0NZb97nAngr0WKmJYkZNyt7ruP3Znv5Dj1kaigQ/FTZlRLWa2dKSwsDlm3PIPX5r2A+Jd77psK3hueoxeUDGFXE4cH5ia5XTSZwdKdqykIHOFFSjkG2tkZpmxLk5ejiQG42JdRS+LIJoJSJLRE40rt72Dj5wbhpSH2kezXjvr008+29u+Oxm5NFF4f3XHEyaMXpunnqa+QKzCPiSUUKxMhldGvat4FF03P8o8fQu0DiBQfPZJvJff7/wzdcSX78o3OsUr24C+z7x4FHD+X2NaRJTn/Ghej1ImZKOo2xTwGYotdESYqKkhEBRqGS4fhQ4o7h875xw+j55dUHyG+TpKQepOaSBNnrWF/dRpxfw9Ock5wjFYDZrZDlaUpWkhIzMAkWFihklKFhym1BZYpJkKgrhDY2OfOM+LM7m/NHHO372ypCdh2RQQ+GzZxuG8Yy/9M4jHnefYw9btLUkCVYXllHyfFIMBZoS0LbSMkVRzOYrXl29JEwejTkSFT1FWrSpOneVBPkucdiM5HikfOYKgTpG/ZBGgKpQN2UlQh+zTgJUyRTqhCTnik4XOdUqt6qrtFTjG+Qk0EmgSkGLWhNXsk4u68izNp5UScxnmnmnsFZgDXRG0hmHVua4aqiXE308RJTjxDTFgkgScmKaArt94OV14cVVYn0bGQ8JJQWdg0WnWC4K82VmcWbQc1mn5brg80gjNfdOl7z3juDV+iPubjK7O8WTz18xm884PzmvBycSOTmkmo6HmpFUqQOUMpFENbMKJIZU4U8Ciq4Kei1bpIBBFlYCXOE4bcnEKZFSoE/gD7DZZbYHxd3ul3wY6MtIFxOtVBzWO7a7PZvDju3FnHsnc7pmOPKjKwhGxLqzTulIHwwSaw1T6Empdud10qQU0UpyF+6QSfBiY/nwleaN83O++fAxn1y/4NXuDts1CD+Ss0aaGd57hEkkofj48y1fvNiz2w9IrVFS89qDljcft+SQENngbN2lbA4jr24GNttECEcSFpEp+/rCHUf8q2cMo2Fs77E4XVIsWAqNFASlmKbMs5c9H3yy5fblxKsh8vMXAz/9Rc++r6nRlC2DMPV0G/eU2BPSgZjGqt0tufbjjzmCqCQpR0JKX+3ztQahIqDI+SgIGgPB1+5/EQKhSl0VkIg5o6gJ5jruA1XqBxAZsa1AaGr/WErSMYlXRCGmgFGaaQxkJVHGMqUAUmFUQvRgFPhecTtl+jbRNYpOJc5UizQWJyKzRcOZaY8VrMSYZ0x2oBRPHALCR77+9e+RmkvWZFbvvsH7QfL2u/d5/Vfe4nf+5PeRMfDeo7f4tW/9LR6/9ib95Hl1uyYMG/Zhz+r8lL//D/4eXdviY+Cw3TNOnouzc/aHHYcwUIxATAWZA+cXF7z26JLVsmW5XJBFreggC4IdJM9cn7FYdfTqFT/40b/g5x9/yu4wEVMhlUjRGUTlplslKsWsmTHvOmbdjPncIJjx2xdrHizv0EKyezmS15pWCfRrCbFIiL0kqYDUkjxYRK9wWwhzw1U3IpLitF2wGffsbyJWN+gmILXDPLzEyS1iHIhSodvINCncecA/g6s/GpALSZlppv3IXdBMOtAYgRCZbiXpLjrIPZurA92ppZsp8plm+2SHVJKtn7Ba0DSOGANC1zFyERmlKqDHOI21ksUMnNNgagJca5C2ctQbLRgVNK0khoxpHSlPNcMSM7II7u6uefiO5lvfW/EHfzDwg5eRRet4655k4wXDVvHkZz2v/5qhMxJVIilrlkaiQyRfZP7Gt1qe/BtZkcBtRBmDj5k//GLkR18o7i0kb90PPFoqNn3k8lZyfk+wOG1wrSB6Xyu7ntpt93UyBrW+i6RW3bykJMNxlk8/jTBraL52wd46iJqSA2IcWZ6dcutHRn+Hsw2rR+9yI0CMgcfRM/M1XCxyJrlK4ovDDl0yGY0XPRqNEhOFjLGmsiSUIqeJN04Mp989Yf7xxI+e3KLQxOBxnefqeuJf+3P+wnuv8X4TCVPCZ4/QgoW9o8uJq+GcaSiIVYfrZggxMk57lDYYXVG+PhSUVoypyuKygilEgk/1EEplmpR03D3m+tIFwJTjSz6iqFTOujOveN6SqQG5IquBFepzqJT6so/1ZZ+P/pUvcwYlF3KqK8yKJStYpXFCoXOpAGpVa+h/RvGTx+pU+lK0WimsHAFzPhKCZxwjm7vM02eeDz4ZePIkcNiDNYqTueRyJbl3mXnwQGNbxcmiriGkBpklUhm0a3n9gePtRxt+tHvJNEbW6y3Pv3jJ3JzQdBahAkloNCOylKpfpzIGSkkVllSgpPyVj6aoChjQxqCERsqMoyBEIpRC8IIwAUoQe8GuL9ztAne7zPaQ2OzCL/cwsAl7lFsyjPWWXaKg30Wehj3PX2zobN2NzOYWJRU+hPqNING0is5pnIoYreg6RwkVqFDxrRW72hx1v0P23Gx2rC9G3njtIbfrW9bjiDUd0mgSgeICk0988mTN01e+0qG6BaUETuaGtx+fM7caciGJiVAUd3eRF696NjtPLvW2jBDE4On9RDBzZssT1Po528PAzeZA5xpkzihVkCWRQyJlzbMXA3+4f8qjhWWxsDSTZkZHaBvQAq2gUYlWD1gOzLPnRBhamtq3T4FXPvByClx7iS+JIApSC/ClhnlU5Q3EmCuQAlXTof6IrDxqLwW6shyOY/2a3OW4nKynQikzbdewtYpQEqqU+v8rQKj66xURKAomURDB14HVlAgSrNHEVJjGQNI1Ia6FYGVn4DJaBua2MOsa3CKybA2yLOnayNJlfrxZ8PPdQ25ngt2U2F/f4GRinHcs33mT6zyw//EP+M5bD3i0eIvXLt/mUDI/e/kJDxePUCjOlyv8zY6TsxW20aw3N/jJo4tBNYrNdlebJFIwHHry5Hn34WPeevt1ZqsO0yh82qGFqa0MkRFBsrT3CLzih0/+KT/75AueXwWCNygq3Ajq10gqjRQw5gRklD5wZaAzFlTHdy8lb74fWYpIeAX5INEmYy/m6FVE7BJZKpTKiAP4FxPcRMpcsTeCjKaZF3a7PbvbjHGK5TLTZwHNCGpHEQnhNFIawtRj7zccvsj84l9PdNow6hEXG0gGbyaKqJY6imdxoRFtgV6yeTFx9sYSgce8ecbm99YIpfCi0Dp1zKmUiqwWCmQFPAkN6IzVgtOVwKqCdKkePFWmmELMguVc41NiNpMcdjVA65yghIwPhbPZnL2v+9Vf/csd69vMT16M/Ph5z+PXLK7J5Ki4e9HzwZ8Ifu23HBcXqjZyhgkpCnIl+O63C3+51/zOjydmjcbIgralMt2BV0Pi5QcaJzNny5GH55b7J5nXzkfeedhwcSJQtiB0dd6WUNBBEnzClwAlYclEW8hZQAw1gHx6grn3LlEbdPBwTG6jKot+mAZMKbRoptYQw0jTzbmaLYjDNcuXH6FLT/fmb5KXr7O5esLpxYL9xrOkYIpk9Acckbv+BWW9IWdB6BPIA918wd/69QsWC8W/+tM1UreMfiDGRLw98G9/Bu6tC96eXSP6euFoxcBKaGzaktWcElPV+eqMD8PxAFT5/0JklFFoIQgxIPUSkKQMSlqG/aGm649ZN460P0TllkgNSWSSLhWl/NUzphJFS4acyp8FmY8v+5LrwYIEKWdkURWfnr/KRtewM7V1kIusl6r0Z+vPhCajvrr9VxGiOK6oAWpIMKfaXoixcBgy15vEk+eRjz+LfPKksN8JjBaczGF7US0IJytxVD5LrKpNAWNqxd1guC9nvPvGQ754ueP6yjNNI69e3HD/dM+8vUAeV71WKqyoX18tNIFaga+rC44zZb7iMkglKaWuRooCWwS5SA6xTgCDkQwpcdjC7VZyt4f1LrHrM/t9+uUeBnSI+GmCLBmDRwuJzIoBQZYwjAKxH4nPN+gvQ2yiwLEyslosOT2Zs2gcRiVWXQNWIpQgyUgSkb0fKVN1QI9p4sYP/MrrX+Obb32Ln37+IaNWFAICz64vfP75gevbiURlwUOkbSyPH5yybG3dRSjwQfH8qufViy27bcC6FmMlWWRCLoSS67g8eZSqnvq7fc/tdmTR9axmDbZpaWxEKwhJchjg1ZhYpERjDM1swYPHinMErQ2ctYkLG7hQkcu4ZO4FNhdK0kyhMPaBV7uRJ2Xip2Se5pHbAl4qiq5I4CIEOZT6oUXU4JxWRF2+kq5UkmNtEkhZagcWAHVEX1fYUymyBj0zVYUpKnRIIsmxoIQiThGpNT7Eo/NAQq5d2L4EEBYrFapA9lBcIGmHSRkRNZ2Zc2kD5ycdZqZ48zwi5pb/x/cv+P2PJDe7G8bNWLMQ1iKbZQ269Bs0I999713ee/Ntrp9/wcvd52SpKElws3+ONpIYRhatxUjBOPS01iJSQbctQhS2/R132zsaJA8fPMIZy+gHBBEtNFJqlNA411TT4DRxvnxAVDv+4Ef/nJ98+Bn9aKA4DBwbFRwTlxWAJL/Uscp6Q+Qg2JoeNe747V874a1VJNxm4k4zcxZ9GesEa+0r6awRHG4t+bYw3ni6h45NhjhGOqvZ94FxGzAI5qsZfcmkceLkZI4TnqzrA5Q0YC4lt0/3PPkfJaKBYAXl0OIZiSWTbYUtCZeQwrI4txQ5ISbFYQPtOw4zmxBnDS8/HYlEhLRYBzEEmrmrbCxy1YsLAaqCc6zWNO2IkAZl65pJqULQGX9MbJsp1XClN8SQWCw1WSamlMlTYtk07LeC2ZuJ3/xbDYf/qmc9wAcvRt57U3AYErNZw/pW8sXHhYePLPNlYoVjt0ncrQWXXeSvv595dg0/vxM0ziJEpPsSaYsie0UOnhf7zM2h8OFLjSgT5/ORt+8r3r0veO000TqDtBJUQAiI2RKTIkmgcRRpUU2DmJ9iV18jGs0wHcjZY5SiMZYxBUQ/MXMOJQpGWfowUogI4RmbGS/dOW2r2G7XfL5rWXYndGcN7rKDc4FUC5IMzDmhyZlw/QO2P/197HCgEYIpZYZxYq7XfPfdOZNa8Ud/8pQxK4ozmNRzu4n8yy+WuLdW3DMJlcEowUp4WgJ7WZ8R3u8xSiCFqVAcWUFoQoZKT82KtmtrsyfU4JufxuMhoM41CxVcIo47/WogLKii6hhe1K5TiYI4VUJkjpV4Kkp97Yk6uz/WCSUkKF9KyL7kneRMqTgR8pGZNobCViTW88TdIbMZM80UMI1C6Rq2rpckBaJ2lnNOlCDq31GSk2ScYL1JvLiJPL/J3B0yvZcIn4gI5jOJr8UGrCpoWbMEQtTaaFUJZhCF09MT7t27x93mObEP7PY7nj97zmox4+S0q2FGoRFCfUkzOBoav6TEZuJU1845/dlBRkhRhxwakJKEpo+wCRLVR8ZhYn0HV1vFZhPZHBKHAfrhl7wmaDqHSIUSClabY2q90gZtUSjna0CzCHLJSFn/bFo7SpL40PDsxYCUh6+0o6lEmlnDajHjXHtSI2mdo8mSWdYcdnd89NmH/MpbX+eN+5f8/OXnRGN5+crz5OmWw5RxraFRIIoi58DD+wsu7zlirE753SHx4ipxdeMZew9U6EeMiSQyQpe6X48jdvSUMEA4IAoM+wk/jJhFRysVTgiUKIToydnQC83eFxaHPbMm8q0Lw+Wp4myeaaxGZYuN4IJGHiIcau0Gn9ApMU+CZRA8DIKYFVkoDlLiRSDJcqSliX9PWZyRqqbgy3F8JhGga25Ali/fXhKBriFAMkSIY+FuM3HYhaPsqO6lQNXakxDVyhgzTmmyLEwx4ayhpOqBT8KT03FPdaz1pBDIU0IuFfNlw2KWMTPDN+6t+LwY/ovfO+OPftEzM1tK3DElhTKZbnGCNC1hfeCger5zdsZvPP46ykPXzrm6fsGs7bB2Rl8CWtXTuBLQ73usdbRNg+wUm2EDKbFbrzEIzto5J7M5s8WcJEBpQ9vNEGlCUtsHIiseXbyNMYrf/f6/5LNPrtFizswUcpKVGnis6SALEl1DpeJ4M8l1UGmtxu8z//F7Dd/+BnATSWtwTUKfRaJVpENGJkVoMtNVIa8l051n9qCjeQRXnw24RjNMPeO6ctC6e4KYPCVplt0RZyoLYiqUrFAngs3VxEf/VLAwB7Jz+F2VKd0h8bYQtOBEC4SDIjW6VYg0wdZxmBRtW2geO/Y3Pa+e9wgnaARVKY7GOsHtmKrIytTOttSKKAqu0SiZMcpS5MiXt0phBH0AIRWtTaALri0cW2PopkCRbLcjj2cNmch6I3j9keKv/fYl/+if3vLZ7cTFuaUzhl56mm7iyaeOy0eKb/4qkAKnpzNa5VmMjjMHu+8m/vM/yPTC4ghEoWoVt3q+KM6hfG0cRRUoQfHZteLJWvDDZ4nHF5HXzybO24KzktHD6Rtvc+9b30bPWhJA0nXlqSKxKCgeYahtBz0nE0mjZ6YNE44pemTskVLQyTnWKtah58HqHp/kOT/eFj754mfYn33Ao9WS3/rOe9x/7YJ1f4dIE0Ld4aRBLjS5kbgg0LkGxsRYGHUmNyO/+tYSEV7jT37ylNFPeGUpccezu8LvPlnyF98854JX5JCJAhqdORwkQQT6foc+hgfrrfu4fy6hVuyQ5JjwjBQUhQoakkVwpAfVQ6KorZN68z1eQFLNQZWSIQpyFKSxXiRyrK/549ytHizKly/GetOXuV5sUq4+EiEzRWTyEQpEzgwJrqeI0YXZ3NLMI7qRSD3VloYzNFLVFUZJUCrBtSTIoeCnxDBkDj1sdoX1NrHd105/PnIAchJoKWmtpHOy4n5lRsna8ArUC+Xee+4OiSQazs8vWTzf4KcNIQWeXz3n5HSBa1+vhsUsyEURy5fembpK0UahY62uK62rlI5aM7faIJIkpYDQGdNYvNU8L5nNkCl7weYu8PRm4PaOerEICh/KL/cwIPC0sxOGQ6JxDSV6REn1hiSgeMhC0DUroNQqDVCKohRJyvWwIJSmZNhNkVIEhzBwt/W8ELVDulg5LpYNs1bilnN6Mh9tnvPew9e5PIz8uw8+5aYHYzqWLpMElDEyjAfOL2ecXxr6aWS/S+x2B9Z3I+NYIEusmZN0IFBrK1pJQhyJsWo751YwV4bkGg5JEnxhd/D4HNFOoi0gQpVXFBjGyF4I3H3Lg3uSk/uFswXMrAAC0+gJvnCYImkIMGTKoMlDvRn1Y2QaA2WfED5hjK7GP60oaU+fIvnYQ5JC1nFXERSjiKUmc7WRNWBy5AJAveWX46EMGeisIfWFzdXINNYATkyVKphSpW5lUTHGSIks4tgGSBx2AaVA9p5swXYN0VdUqMyCYQfD2Zzpr/4F/ofB8OfVU/7XneK//+CM/9vPNK/WrzgpA5tc2I4JVGLerED03N3dMEsNM85oT8/YO5i2PQLN5ekZTWMJk8I1c9a7Ne1qWRP9IRKL5+BHyAlNIYyelW15sDpDOkOxVXxEigglKcpgmwZiQBbF5cV9umbGjz/8Y9abNQ/PH1e0qhAk5fHhQBwKY0iMPpF8osSKma7ZAYWSmYPPPLLwN39rxj084s5CA+q89oL9lSf2CWkdm+uEX8M8BeyFpH1YiIcJ3QQ2o8cPEiELbqEJMpA9zM1ERhFsgKgQMSIvDfsXhR/+455Wd5Suo/SJxMg2OUaZYTZDxi3d3JKO4b1c6vd8+2pk1AZjC/ZRy+c/vmOMiZMLi/SCMRdaKTFOcXvrkSqBrPU+pRQoRdfUW5JQFQ5VRCaWap3zOZMlWClJqjCbCw7bSIiZtpPHCnVdJbZWMOxH+vmMr/35hq9/0vCDjw58srB8+5HCdjB5iRwjP/mTLfPLJY/enJMGTzuXmDZzemH4uwvL82nHf/lHI2reYmRESYGPmTBVp72yIILAKoeWAlUCIcPNLrMJik83ipky3Gst33lnwfzeOd5U3DfS0tgGVTLRD0hxAKFrrDB4pnxAaIO1Hf3Uo4VCqjlT2tLaGY1dcAB+749/hlUv2K539LuezgXCJPm0H3n4+AHu9BRFzV6VPCB1YLu5gpzIJhNizVw46RkGCykSleebb99D6sf84Q8+ZwJUYyHBR+sNUp3w3XsPCdMrbkZIAkIcads5xraIo6VPyYYkIzGMaG0pEiyG/X7P7PSUbrGiafeEJtJPB9SXI355XC8ogcgCGYGoCCqSQ80HICqELPpSK48JbK5TgXw8FVS+kPhqhSByvVGWL/n6soLQMvW/LymRCwQJ+dpTxJ6QMlOcMcTAmfeczhwzJ2msqJPTEhA5UkIkToFxTByGxHZf2Gwz2y2MoyaFupoVCLSEVitmztA5g9EKY6p+uUhRgXcpsh8K231k9Jaz+QmPXntAPwzs9wP7Yc+zFy+YLxa0boYPEoT8Cq1OffTWw5USCKkqkjlltNBoURtfQkpU4zidt3QnHUkIPlqvmbYjbi3It5KX15H1qOjHumqJ/9MGA/8zDgPra8SpxjUr7vZbFo1FCphKwgtYqQ51jHukkhmnQCV51TtspgIgpskjRK6YSDIpjGityO2SNEXurjyHbb1RKF1qSKNs+OijLY/PzjF6Tog3DIdQGQaiUKLn7GzOW2/dZzj0vHgxsO0jPnkUx6qjqn/cnCQZj7E1ZDeMkpg7hCz4eMCahrlbsOtv2eXMzSGx6j3n8xblLNJoSorIWOt1acqsbz0n9wTzc0MsDUlalPYQByaRiKUiPKdDwt+N+H1mGAPbIXLTR24Pgu0hMOApXmBXFqk6hB7Zp1iBFLL+1ORUPzA1iVoBOCKmYwCF+lMla2ZAysRsrjhtFbfbzHjIdb5GOT6QZV3eyZaMZJgGhAF8rcAszha8/61vsPragrIf+dN//hP2V/tKTEMiF4HDSnH2v/oPuHGCz59v2c3PuPrFKb/7/I6FV8yaHsWM00VHexdY99cMQZBeTlAGpkaiyh3f/+jATz//lO+995D3Hj1EiRW+KHSnCX0VQPmYcNKghKxeiZJIoeCkxVjJcnFKIZMkRENVP0tZR3BkDt6zauc8PnlAKPDR80+4PlyxOj1BHcNGytXRYvQjcejZbUdubw9s+wNTiHUNIyU5JYRKNGPkP/s7Z3z7xBOvA6qTmFNN9prrl7X/rIPmdvDEDSyMws8Uy7cd0+BpTLXShSHiS6HpNFrDGDLaugojCmBKwzQMNGeK7asDP/zHW5y4QJxu8eMCQcHbhlEFtO5IasfZrAGXUUZgcwJrwTquXrwiL86xJsIFPP0nI01rax4glDphEhLhFFd3AdOK4xhYgMgUkWhaME6RYqpWQCvqFEZV8YovhZkpDKJgOsWUEynnquIuEaTi+hB52J7SRc/uEDFuy/f+SsOnTwdu+sTT/chlZ5Ap4RqYJsEf/t4GaWfcP7eMPQhhUDLw+r3MP/xrDbfbyP/vF57l0gAKbQIIRQoVflNEZJoytrHMtGScYhWricSUSw3PFot941s8/JX3GeTIdrtD+AThQBSZJASUBi1t3anLgSh7XO5QOLKQOFMvPckrtj3sxpEnn3/O1ZNbio/Y1oGDvRB0tkPrwPc/+TGuUXzzwWvVqqokU9zx8uUT7sWEtfVQ65Fk4VFxRCZNryHEKx5eXvL+N9/mo48+Y+/BJUMuE794cYdRX8cpy812TSiOyW5xoj3e7nO9HBiHjoJh7AlhBKEoudDaBq01zmm6ecdwGGmzBa/qdPVLJfpxQpAylFiBPuWrF9FxNTAWRDxWnI8tg3J8dn2J3q0HipobyEWQRZ0IkOqh4Og8qpO7YwV6GwvT0HPYT+wPE/u+440HLdOysGypl0srq4fgaGGMAXov2BwKN5vI1Tqw3UemIEmlAtwEdS0wc4LOHgFA9eEKHLMOpTD5QPAFkiVOiUY6Hl/e4+52wzC8IITM1e2akxfXvPZA4ydZZXZH4FIFx2VSquAgrakr3ZRwwqCOaxntLN2y5eLiPl4Xrje3fHY9srvz2LtIu4PhYNmNnvBlVOCrrMQv6TAg80RjFbd+ZD0ciKYgY8TikEIy6gFlYOwnhDoSvcSXbvpMyWOt0kkgC1J2aGExpiOniB0HkrSEcLTkUQhMyKPK8vOy4+5tSSs67p5+wNRUiYZMmWamWV0aXr3csrmJeCJBJ4quPHwlKqdfJKqcJUKJCV8kqBn3z8+wRPr9Z2SVaecz3LCmnzxxV5juGorrMNJhpEWKavoyTiGF4jBIbl4Wug5mNmPUSCckravJ7X4IBFm4K4mtT+z7xG4/se0T+31m4yFQzWPTtiemRLtsWTSabOqqIoTIOBZ8LCCPaxqZySUgpUIe07d18JZASNpOcnl/xlwm1p/7o+yjdnuFBK1rWnocR5Dw7psPee/dN9n1a976c9/gyg68lAde3r4kzCV/+R/8KnINrw6e9acv+OSHn/Nb//Bv0jea5588I5Q1Y7jk5Rj4jdPMh6+2iAl28kDatOQkQZyRy4ZcPLmYejaZBvRwx6/+yjf5zoPXIVUqohAKqzS99uScGfJY8a4IxrGnH/fkUDg/uYfUgkOJhBJxAkyuI+EpVxe8zfDw5BLXLbnuJ6Zp5MX1J1ytn+AaizSKrlnSuRmtEph2wag9erohFM80ARGKrI71xmheu3ef//RXW/7qd69xNwHlHGXZc9gUbm8T2o88Geesl+9zsn+G1c+QasblOwZNQJWCbCKbCFN0zG3VuW77gcZaLEfplfKMo2TxuqHsJZ/+956SznCnA4wzfJpQ80q0y05TmsAsWaxNlEZjgFZFiq6wqZubAXPW0TQ7ko1cPfW0nUboABKstpAEO9+z20dOziqjX0ooJIqQCJtxjSaWKqFRVqGcQoujswJF43QloemMObGMdwckEuMKU8kUYdnsDiznMzgkegcXJ5o/9+sd//Yne56PlotO0RrDKDKNgRwMP/gjz6/9hmK58NxuYK4curV8/XXJ//ZvSV7+157vbxIXZ4LoFVAd9aJEhBSEYoha1jCyqVhieTwwxhi4Hbb8v//R7/Kv/vTn/I3/4Fu8/mhJkoUo5bH2JfAlMWWPVg5tOtJ0V1coSmABQ4NaNGzKyL/6l3/AejOgRMIpg5wpRpGYpTopFeyhaPq7wie/+AXvXFwilaWRmZe3TxFW0i3mLMWO2HuEcngiwlpyNjQhME2ZfXzFw7NLhjce8vNfvABVyFMi6sTHT3/B/XsPuMoLxjhypg3DMJGWI8YusKZDqS1JSpSyCH0gxtoo0EApiTGMZJmxM0dj6qQt5mrXC77+jCIyWSayLFW8U2qjrJRC8RB9XRfVlH+VkHGUApVSmwuyiK9yTqIc+/ZFHOM7+dgqODIDSkKKWqn2RXO3FXzxImDdhDMWBZRQSDHSNAIrClaALIppqvv07b6w3kZ2h1gtl0WQj8FhowtdK1kuNPOZwVmFlJqcIMa6ri2irj0o4JRBJyg5sOxaHty/x+22Z78fORx6Xrx4QddahJijChgJShynK1RSapKSmGr74biXrOvzrqWdn+JOLSEpXmyvePbyOYe+BymZnKZMiWxB91+eWf794OQv6TAAktxmdBm4v2rqAUAWJhWYZ02hIEtESct+iGSZUWUkJsBqTKm3Wtc0KFmBHz5O0FWCYS6GkOueKnOcP8mCkKWa7JLkbrOhO71gefqAsRxDFSLSzDTbO0XwHqEVomh0kpRSZ5hJFsRxiqBFxhhHUprzpuH1ewtCGvjJT3/KOGy4vLxPJ1vmds5WTUxT/aDMV1PdE5eMkPkIhMgoLfFhYjhk4mRIU4JGInRCSYkrmkmFytxOhT5mtkNg7MH3MPhSR39KonRGp0LaRLaHHnPWsTp19C6zL4FUKtOeEsiNrGhOFFFLcgr161rUMehSSAnypNjlxN0mUoTDmHTUlVqGEUQp3L9/wf/u//D3ee8vPKYvno8+f8mf/uQDPv3iE3wKdNmg25Yv5B16LskLxa/9xb/Ee3/966zswO0o6Q97TmcnPDxf0YqBq9tbXt1tmM0eQn/Az0aigGI8OWRk2yGjxBVF0ya+9t47fPfXv0GeHCpO9eslNbthz+A3+FR75j4lihD4YUJjEFbhRSaLRMlVXxpzBgy+eGKcuNfc47X7D9j1O15eX9E1HVfrD/nxx79LJjIflrRthy4JLwNatpRiSRQQBlkMZTKk/R5lNcZYrvY9sX/B2V+bM99H1NIQZObuZUaMmXHM/OjVA7av/xbL3Q84G15ghGH2uKDmhbTLGBcRo4EhV2RyI9gdNljbopQmxYgRmjFH2ktISfDjf9XTbzzmRKKMYRxHpoXGaolMEukS1imaBFkLGqsYpsRipjHdAFvLzV3Dg8cT7oFA7RR3N3fIVUMbHXdiy2nr+PCmkGOh6ESSFpkysw5EK0jSIRkRso5SIwqjM62MYCWDi0y+I7uMyRKRFVYWghXsc2amC+iAKIqtT1gvaVTCjJrUSN7/bsuTVwPbHp60nq8tZsgD0HiUKPSHxPf/dM93vrug05ExBJap4A+FX3vL8b//jwr/x/9v4GqyLG1hYiLHTMFRJIg24IvgVC9RamIIB0ypArCiLbIIwjRysbKsTg0YA97V6ZuYEDnXkbOooKksM1kZ0vEmZ7sld+PEj370AR9/+gvG9cBMaKLUFSJTElYJYn00ooWiWEUYR4x2FFnopCcMTxgOPatuSTpdQv+UM3nLTozoICErEBknBeckGnHgZut58/IR+2HBp0+3NAJCDAQhsLs1Z6tzDjmwma5geoa99Sy6rxPjnhhvicGgSkPDjIMsjKOnZMGi6zAorFUUZ8Eauq6ha1u0EEzTxJQCMUyMfiCEiVAywQeItVKY4zH1XzKpJIR2kHIlFSrx1Yrgy1F5UEePSswcu4d1318g54TIGZAIY3CtpJ0JFnOB7ixjMmwO0NmIEQJrDSYZwpcOhByZpsTkE95HUi4gda1hZ5C5YBu4f+74+htLHt9XtDOPcgVn2gqAK5IQBGOKlGwQQEiRKAUlBtLgOXMtl4uWw2FLVrDeb7m6vuJkZepzSllM447EQ43GowkUauC7iITUgXbW4mb3yaLlblgzHvb4/sBKONq54TB5irZgC6M5VCBfPyKniBPyl3sYaI1Gx4iOdYemULTjiDu/IGXJbT9gvEGp2ouMUVBS7fFHnykmIqSklIlYJK21YBU+VuPfmOuYpB72ampbIyDWPmUshf1uQ1ws6WYdvh8xRuOcQ6i6X5JiOooZEkoHpCyUbCEarI4I6Sm0pNTQziWv3zMML3/Bv/v+D7mZRk7PFngSq6aja1pa1+Cnkc22Z3anaYyqN4NST3LKCc7OOs5tYjHPtMfwUeMMSmdyDITsiTmTEsRYmHyue+iQmWKuHwhhjpCTiFGGlMCPkf5qj4mC7kwy71pCUBy8J8VcJSASpBC4VHl4PmWMMbisiMNEiIm1OTCPkq4xHNREDgoZBBHPW+9f8J/8g9/mO997k2fPPuf3/+hj/ujHn/J8PWG7BnKiUwXRWkry+CEhlOZ0ueIf/3/+G9742ht87ZvfpGHON771NnF/jbYbnr3cMO4MC/eY5BPCGuJwoAhHUQ4tNVZm3FzV/9wsucuaf/5HH/Gtd17j0cmSfjcS/YgQhUafoFKPkIp+DDQm46xCNx2+KHzy5Dhh21OyGLHOklNEj5r7Z49pVzNevnrKMEZmy3PW2yt+9OEfs5t62lnDIPaU4ilhJE09QcyxYkZG0hdPzDXp7L0ipUzJE2Fd+N/8vY5ffVhAKbb7EXYZrVr+2Uc9/+1PJe0b7/IWH/J6c4UMAXdimd1viX1EZIcXhrTPOKcISA6HCWdnpByJacQpyZQis1Vhtpzz/f/uju3LgOo0864QUqA4ibH1pYAoGCcxRiJURmhBKBNOa1YLB03m5rPIRGR15ugeaV7eJsaomWlJSYVFt8IZ+GTouZ86pOGrqYCU1IqTyFX7bL/kW1exDaqGoKRV9DFUGZCshLjiJG3RlFTIGIyqVDgyTCnjTEZkCyXQNIpvvj/nD/5gz7YvrF1gtcyEojBdxGJZHxI///meb723ROtERKIUjGPgb3zHsL0T/J//VY+3DmcMnkzMqh6gQyFOmT4PWGvRWFJOaGX+DMutBG42x7oFgUSSI0XUSqlEkXJC4SlSYFXDIBIhRs5nHT9//or/8Y++z+5mjbK2StNK9YrIQrVqlnqRyLKutVJOdEry7muPWLiGMG64OYxo2dC2jptpz2IKvNVYeno2vaJMniIkQdZd8tI6isxspzXfvd/Rbwvr3b6Gj8PIdiNxdsnJaUsIgvzVHF/RNudc3fbk4hHW4/cjMaT6LFa2Po+Nop01kCVCGLrGMZ81NE0NIMYYGYYDu/0d+0PhMEZKzsRQ2ft1rF/zBUoLsvBYa2gbizGKlKkcg5AoReBqV5FAJoRE+FKTfOQJKKvRRtLMJN1KM19KXFvhRF5ldiGzDHCuWoQyNY0vJTEKYqqfmSQVyhqsThgx4VSmsTBbtpxfrHj39RPef8PxxtnIxaJn3mj0MStTpAQMsghUrlOjHBJxrJ6bkgskwWK2ZN7t8bGuGtfbHbNZQMja+BLyOCUh13wKX20hqqhJKKY4sb69oZ8U3h9QBKwUGNcgbcHLQh+m+rldrpjpQNorGKfaCvtlHga62YxXnz0lZcUAOCmxKTC/OOUmJu7Wdyzas3pT76odTBZbe+8lY3TlKeUiKLngfURpyRRivdEcxy3wZZXiz3ql9VCYGadEIjNbtBziRNMYJj/gh7EKZHRBKnuszSlyPvrqdU+kILMjBJh3E2/dsxBu+P7Pf8Krw0CzNHgChzRx365o27b+nkphioXdPiLnle4npaDkjHSF5bnm4YmjdSNdF9EyoIWqwSJRKn+8CEKCISTGUKsqUUqSyggj0Tnj41G4Y6o5rUjJdvTEXYJTy6zrCEEwbEdCToiiajiWQsi186K1pFWGNHmMtkwE9gnciUX2gdnGEXeZ7nTG+3/pG9z/1inPzEt+8js/xWnLxcUDnr26oztZkZWnhIyWLUJbSp5wuuPeg4f0ux1vv/MWb77zTSZmXH+84+rwGcPwnNPlCp00cRgRXR23S60p2eN9xGiH9xNu0dSxs1QM2w1dtrzz3kMWzjFuA0r3hJhxtqEVGq06UgpoXT8bqA5h5qhYHehCSoT3oCPbceLEzvnGG28wRfjg6WdIW1icrCBHrq+fcLe+prUGVSJa1103JRCmkSGNREbyqPB78PtE7msSug8Tm0PgW2+c8J33NfNmhF3EFsGPNpL/+z/p+Se/PzFoxX/2DyNf12u6qQfZcXoxx8iBMRSaxnDwI1IaVmeS26ceimOaPKBwrSBEQTurEqHv/9M1h1eCpDSy82AMPkpUC02jCDkSC1gj0LLUw4GW5JJZNIKmAbTm2csJYWF1D+Sl4vbfBNCK+Sxx5wP3Zyd8+uKaMVlOHVhbK4PaVLqcNLKuBiR0M0UM1WOgXSHKgiwa5SQ7Hwii6oelKiSdaVpF32diKVgUxUQKiiEGLhcnWBEZdwG3gkcPFU/vW56vAy+HicWJxpREUoqmgUXSXN8Efvrxhm9/Y1HBL8WjJMRJ8L/4847rXeQ//8FEXjToIiujPkVKslhjCExoCta0DCWAksj4Z13ujz98yXff/yavvTGjL/XmBgNNq9DCEKMnpEhK9XYunMN3ho+fP+Gw26PbGVCOavBMLIkkClJqGuco2VcWvxSQEuezBY9OzsjDAUFAmzmEke3B88X1Dd+aZxZt4UIaUtHsplBXhlRPAMKz6jT0t7RofvWtx/zeByO9z8hS99E3Nzcoc8LJ6pysJPcW76KVZQxrYjmQhcL7Qiz1QtOYjpgFicLkR2Lqma8s2misSEhdp7FWO7R1IDM+Dox+wJWK/p3KSIwZJWobAxHJoq5Z24VmudRYa8gJpiHhx0TxubYOyvEAmipbBaXQVmOdRDuB7QTtQjBbCdqZokjBNCU200TaJawzrEaNNTWDsZo7rMlEbZnkxOQPGBdYzCSXK8PwmuTkRCO6FQ8f3+fb757yzmVgJa6ZK8Wqs3StRbs6kVDCYLLECInyAk2hxECJgpAyu31g6CNCaIyp7I3DMLLvD3QnJ0gla1Aw56/aSpW5kCkpUWL1PYxEXm1uubnLhBwxJdE1hnmnsU6TqIcBPxRaM+PkfM7Fw3NWraF1v+TDQLGKi9UJ169uq6yhqSfFEj1hHzGhoJuBGDJZWQoDjV3QNB0xTaAUspQjaKLQe4/SX0oYQB5DbTU1WilSAllrXVLhpDpiKxWu00zPekKokwBjNMZa0FBK/cAp6RGI2mZQ1YIlhshKT7z9cMVp6/jhh3c8u94jnCInT8yFMXlCycxmc1bzBbs+sPeZ3RDqiCznqviVBd0o3FyyOBUsG0PnJF1bapKZUvWoqbrCUwlkEVEWbGdI1N2QiJlhGmpaWxp8LuTgKTkiS6ZVro7/KwyVQoRSMZqlCMTRQVCAZu5qwK1IfMxIqXFY/P6Ob3znu5jfuMc29Jil5MWrZ3z8x8+wyaJnGeMsH32xw3ZnpDFgLDi3JETJsO5ZNC1Gzdn3mSA7Hr/7GD8Wnn/8isGP+DFzefYm1mjGobA4X7KfDuTYo3SHcB1OR9K4xSlFyJLGNhgJ3/rGCd/+2kPyJPD7RD9sUY0lFsts7lAFlHGkpLCyA2kIQtAPa6Zxz6xZ4HNgLBP0mgenDzg9u+AXdy/YHTa0zpFFYZxGrq4+509/+m+ZhKdrNMpmrJE4BUZkZAykUZCmgj9IxptIf5fYjzt244j3lvEl/JW/veDrvyIJQ+HffVD4f/2LA7/zbz3PbjXLpWHaTTz54Yec/nZCG4Gwgva0wJTJ5BqyGhTNUnD7MpGmWA9dolaKQhDMliNNK/nT343ojcKrxFAkZ/OWqURSA7ITGJkpOZF1vXFRKuilUB0bXVeACUHD3U3ELRpmp4GiJbef3dDMJSVn2sUZ0+EV7fvv85Zeo8sB00qcFmgt0K1EKEjUvnYzV6yvEwaJNgWfoeSENpZxiPgEXVdXagDaCkyq6eks6tQwGUGgsBkmHiwDaTL024RtFe+8bXh1F9mlwtUe3n7YsutHxn7COsuyaPa3ng8+C7z3VqYFrBT0qYCM/O2/qPnZtvA7n0UWFnS0SGNIJLIICCGJZDrTQdHE7FFNre61pmXre57evOTs8RsIrTGy4Wbd84c//hGr5RnfePMRp01HKpFJFD59dcX3//THfPHZC5LITNGjRb1QWK3RptrksogIYfEp1hUoEpkSr51dVBfINOBMYdr0eBEYe4WNjseLkaYJzGVLyVUBvN6PxAOAIU4J1wpWs4bJe94xG3Zvdvzxp1t80KijHfV2fc391TvcP11xNnuEknUt1zQtUxbEYUcp9VAafEZqh0KwWLQYLclpROuIkIkYJqY8EIOr0h2dEK5yKawQaN0gVWFQE1DV6EJWmM98bpidGLpVrUv7CdhKjFRkCUEEYqw5Dy0zgoJx0M4VsxOLmxesA9uBbQtSF0IS+FI5/WsfcduAkD2jNyjTsZgb2s4iSfT9gZgzPhaWA5z3hSBhGRzt6Qlfe3PONx9LTmxA+8RCSeYLiesMwmqK0hRhMKJOI4wUGA1aJg5ZcugTL67vuNke8IDVjqQPTCGw2d1h5/Pq8CnH0HcBqQRKVMCQOKKGVa5UXUkgpoJPNTsWwlTtmU1FZ/sEPidIgYNrOHmwYHXvHot588s9DCQAIWiXc+KuRwnBJGGYPIvZgvVmpOCJOZH6jDCJrPaEIWCMQWR/5HvXE57UHKETdUwyklEUtKydBFlKTXgjUcqQiiTFiZvbLffv3WM+X3LoD3Tz+fFRk+sJu9QHVsl1fw4aU2ChCvcfr1guDSenK1xpYfycMkWsK0foReXIuKal04V522CtQsREiBkf85dwTFLK9d/lTBIJZQumFRhTEMJX9aQErar/WjUKO9PMokQYRVEBaQQlJhrvGCfNGAthjLUVIGr4Q8p6oxn9SEgjtjFQMtOUanFASfYy0bWONAYExxCOEkxkNv2e3/xz36LoOR+9eEoQgXQ1QpiwumBnA94LNKZ2eKPEigaCZ73foZuGk7N7tE1H6yp610foB8Xm+pboe0J6wWp2wrw7rx8WObHt75CiapWm0FfEa8osZ5Wxb5sGhKBrLKvLM168XNNpibYznh0+prMt9y4e8cOPf8AXuxe89do3gI4vXn1BLhv6wy0vnm+5d/E2f/ev/B3KsGPZdTx+/WsoLfji6mcEIZkvGpLP+DCy3+34+NOfMcYtzYlFq0xrFI2SOCkwxyRznDJhH5i2mXGXSL5Q+ogcC2EYuP9ax6P3Hf/lP7rmv/pHO374gWTyhbZrOT3P9GJEZPhLb25pjWAqcLGS6MZz2AjaThN9QMglz55uKaVlOVdcbXa4xZLCwGLR4pzh6Y8L403A2Z6NlywvNMK0JGFRbUA2lVleFAhb3QkSQZG1E+6QdDNAFUosjNuI6xymmWDoeL4dmJ02hCCgTOxl4d2//ffpD/8Fu90VxqqvDI/a1lZBkQUfoW0FORWsFlgpjsItkBr8qDlMhcW5IB4PrSgws0IZAwFLIwRZ16rmzWHHSdsxn0nWm1oxu39puTjJXPUDz7eZtjHcOxX0e80YArooctF8cX0AZXnvLcuilRhbGGNhtTD8/e8lfrGOfDG1dKYwlUCRlZ0glSGVTMgeJzSpZLzMaFloUIwi8sOff8DZRcPy1DLrWqbi+cknT3l1/X3un5zy3V/9Lg9fu8eHn3zCD37wU8Iu0LiGmbBVTKRgIoAtaFFY2BaRC6okSuOQpgY/G215dH4PUTJaau4OG6YQue337PrCmzPBQkWUVcwL2BIRR7nS6BNTKiQkfswYK2iEorFbfuXBklc38MmtQFvNNOzwuWX3asvj1x6ijWTyHteconfg0xYrE6ZAkLI+m8eJaCpwLMuMbRxSJqKMxFJZBSmKygIxGmEDZjHhokJQibN+WZHqHLHWTaOZtRIzF8gZxJIqIcVqdKzT19gJYq7flyYDRuFmgm6p6eYS6TxSVTOoUjV0LnKq2GwjGUPm6XrDZjfQjwtcZ+naQOMkjVZEqQjGMGlHrzOhSagzWCrDGw8UX3+cuVwcsPlA4ySn7ZxurqERoDVZaKBCqaRIX1katckMceLF7cTLmwPboUdph2sVUhV8TBz6He1+h3Qaqw3ZCvw4gQChBCV+eQuXKGFoTKGzmUaPlKyRVpNjZJwyMeRj1kHinENKWPd7uC0E4TkNs1/uYUDkwiQiqdV0ZcY0jkw+MBeV3S6drpUXk9FCopQjlQB5RCdNTpEYR5xr60tCSJAQY0KWXPvKWqMQdWwiBFImEpISA1koRBGs77bMZwteu/+Az58+IcaIlAIlC2R9BEsUGtcx7xraGZwuLCe2QRuNMBLTNByud2wPW7KswSFZJDkIht4TYsY2beXON5b9GBjHga2DTrv69UDUcVoSFCmJRKZUFZSpZEQolCyrDEUqhLXoTqJ9QYaMNJm5kZiimAbJXkpKHwlKMI6JgiIRj3hPSSkebSSzmUHkhJ/6KiZKmZnUlL4Cbnrvid4jCsznDb/27V9Fn53x059+hHOWHHriODFzCxCKHKv7wPcJrQWniwWFgk+K1y4vOD25qN8noZFGVxpigOtXtxyGHTFsIcx49OZjrvd3xwRxlRVJHaqeOFlIe7RS+GwwzhHHPSkmbl9MfPyzn/Gddx/yve+9z/V+zVX5EeN2x49fLBjWOwav+fzJJygFxkI7M5SscG1ByMg2Zi5fu8dJc8L19o5Pr74gp4k3Lx+QxognoYxjPz5lP1yznDmskMyMYOk0Tks0GZEK0Qv8KChTQfmCzODHjEDSaYfQtd3yf/q/POfFXUS2Le080S5GQimAJa0Vv/7WnHde1yQZCVPGdaCTYDsJ3EwyYdlsD5TiWC0t03piNncERuZzg5aeT38a8a8MSgc20lBcYNleMJkNwu7Q0tYwrB8puupNj2ftY7AF5q1FmUh2Cna1t2xmEmUzYQdjb1EdEAT9esO9v/n3GVjAq2eURcHYgjNgrECa2gYqCnwozF2thlkncEJSpCaUUKFFWrA+BC6KRRgJsQY/jVOVuU611qESStdQ2t5blrNI2wjuNiPzM8c7bxluftTji+AXX4xYpZkvHP7gEUkwP9EgCy9uM6pE3n+7pe0g4ZEp8+2Hin/wq4X/6+8d2GEwWpIwCN0gRfWipFgwRjNv5hzKgFags6BRjrtN4PrK8/DhQ168fMIf//inTP7AfGm52234p//sX9DYGcSENQozdwQhSAlaq5CiYNEUmbHWMF/OOex2aCGQxhBFYQqF8+WC105PcFqQSuJ2GHi529EPnuW85XIl8DRcj4JWeIqD2U4yaIe1hefriZ2vCnOXM1ZKcilYNfKNxyuudgcOY3Uf6CLYvnjO09MTLr/1JlPIxNSTSk/JGVEs1qojP6FSJoXItE2L6xyyBIQs9MmRwp7Dbss4TlgN3cwgncC1EkepDZeuysxEqV15ZQRNI3FGUHTGq4wfC1OSlKN4TmiJINMYAaq+L6QVNDOJbQrShH+PtlrridRYN0plrKvwoqlPbKaRJ3eF7kqhbSbIjFaF3diz2XvW28jNJOidRTeF1cLy+CJxuejpTMaJwKKxzGazGvR1CSEsWVRXADnXvFbIOJ1oTCCKgZvNlpu7Q204GYlSqqrX1UQ/jex2e1buBK0tQhz1zYhjxVJRSgJR0MahGof14FwhEyFlgqj1vJiOq1KtMa5mz7JU3O09/XDN81d3v9zDQGMtMXq2+w0mKMIYMAvHYbfm0Vsn3G333O72mNYhDeSYKCIzmztkDPWbm6t6t4jqOZdCV+JEAatlRRwflZVIWSE4pZAoqFzRjNY2DMOESrWSYjuHAkSWxDAx71pOVifMZ4nN4QPGYSLMv0ZZniGyYpoO7Kdbnjx9xodXL4hKYZIkF3FEirY419J0HbOuHih2Q2AzjAxTPvrJK1xjHAOb3cTtrSQMHquhdQZnJVJmShbEIBnHwm6AMWmmFJlS3Uk2jWUGJK0wZErwhKlgpSIJiVMaYRJGW7LQRFkI6cgA19W2lmMil4RCsbkZWS4XvPv+O9x7eJ/5csGTZ1/wwx/+hIXpyBOksTY6pjIh0TjVIsg8fPiI2WxJCnXN4joDFPwUIU9kmQDFbopsbzYEP0IZQGQefeN17tYDh+1Yf5ANjKkwzw4nM1OMrOYLEIXb/R6VA3nsaaTmZOb4i7/567z1+C32fuDDT/8FpenZBYXUgciMaYhYN2feCWYtDN6DUlw8cCjxin/94/8nuUT6kEg+o4KA6RL5V/5DHp2fsEKz3+3Z3rwi+APzmeOk6Zg3hbkqtChEyXU35yEfbXZZQLKR+blEzGeoMSF2E/12T/CJ09NC1gWnJV53FBGRBszB8nd/84QH3Q0xRYYc0G5J2mdaqwhB8+p6ooyBk/PM0HtOFh3TYUO7nDFuAy+eBOKhAZUZpcWLwKxtkG5DcnusnCFVQWUIAoSSSEqVweh6eG+1Zr6qoBe9Umw+D0QtkE2hGMt2O5F7z/LshM9f3PDOb/xlZt/7dX78O/8di+LJzLAu0GiBdQJkFWZkRbVhajAN9baXJFaI6vFJCWMlm6FwGGFhMtIUshS1HdTUB1gV1IhKK5WCIQWmLGnaBDvBy1cDs7njZKHYBUOxio++GHjnHUnbavr9hBSB08aRS+TJdWba3/L+12ecnGq2oTZ6fvNNxX90Df/DJw4RwGhJaTQl1vpYEYWxBFozowl1vRdEIqlEFpkPn34BreHnn3zEs6uXaCMga0SnWOWKZRfW4LMni0CjDVHDTnpmylQKboZhnPDc1eBgDKhU6XytgncfPmTVWpJM9L1niJHJB87mC1bLFjGNfHonScaxagaGXLDAszHw8TDy4RZeHQBZsFLQOIUSIMtIkY6L8yXD8x1StgRfQ7afffJzXr//NouTpiLZS0AqW8vJomcMqRImlURrQ1GCBDg3QymF3ymyj4yHnuAD2kmCLLU9oMC0tZpolEKIcvSnFEwjcbZ6VFLOEDJ4SKEGo4tIZFtf/soJlEnIY27FuAIqHCfMFo6jdEGu8icFUgtUqgZD0dVQ4nba89GLwCH1fL51WCdIORFCpveFoCV6qTix8NpCcDlPzGygsTXs3rQW6TTCWIyhcnOkqSjmnLAp0ZhMawtdC12T0SbUoHfRKCcgSyQNWfTVftkfWPhlDdmnaieEOjHX0lQgkSgkZRBKI03GNIWsInGaQP0ZqpmUELLUfE6p63KfMkOIiN7/cg8DIUZaJbicz1hf7RFGkcjYkvF31+hh5MRKYi4IP3ByvmKouwWaWcLZGdY6xn5k9InsR1Kp+zlrG9JRbWxypaQpUXG55YhqLLKOoUKuadS2a1G66pGl0HROc/pgwempQqTa4f7i02s+/Pgp4/QE0RgEmWkaaZq29t/3PcrpY1dUkotGaoe0Dm0s81lXVwXyQEmFwzChpKaTFeSbYmG/99ytBdMuIcholdBaoXUGJUhJkUJimCI+KEZPhUFohWsMJgaUSiw7iVYtKXt6n/BZsFq2zC800kpKCQzR13EQmSKo1TcliBL8FFi2C/7yb/9NrqPn0BiurtdsbgdO5JKUPH0aiSljJ5A6InRNo67cgtnyDJ/rPjcNAylJInB7mJgvHEtjYRD4vhBiRtmC05qFW7Jev2LY9sxPTzBZMhz2ZDGx3WaykuS0QeVLfBA0zZzJR8ZQePT6nP/kr/8WavQM4YZfPPsMoXfo6GijJJWR3XBA247GjjhXKNrQWo0QGedS5ad7jygCVzJ2BUIlxOARuYJUYhD0056XL5+hVMPCzjhRhaXUGDSKmgMJAryoEyrdarKiHuzmmjZEhl3PvjM1kb7vGZNAqAxHkUhbOibf8/ojw9fePNBlyW20SK1Q0bCLisVp4ublhrRRvPZWiw+J1argZoW76Hj2rKe/9ZipQ5TAJAOpaRBjqeHJJvNgdc56HZivBMOhQnCsVlCqNx4NKmVmM0PTFXZXAeM61nc9k1AUEcgYNvs1qTOM057m1PLO3/0H/MGP/pD8yZ+gTlv2GhqhcVpVoqg8kuQKpHLEI5tCzB6RDVoLsqswGJJkPxWGITGfFWgl8vi/E9ZixtpJN1KTSkE7wRRHYpnTGclqpXj16YF+8JydCjbPBLLbMwbJF68OvH5pcW5GiJFFG1k0midI1inx/U8GfkW0XJwK1sEz6zR/57sLxOWcz55Hnnx2zWairhgzTHHkkBOERCtaxjQQbH2pSgzDNPGnP/w+W39H0ykaFGlMkBRBZIIMCGVQ2WBybQZIVbBJYKTEq0zIsGhnUBJ+9LSuQRhBCZHL+Yqvv/6YkhPRFA5+4rDbs3Idjy9Pub3dMWwlH2rPobWYLEiy5ZXfsj4UPnkReHXQTFHX52QJaE29PeoGJydOheJkNeP6Zo1QsCmJNhV++OM/4Ht/4XsgLEPMFU3uNHEEIxoEEiFqGV5bg+kcxmikEMyConOS+cwiO8esM3XVNO2ZUmSgwnO0BmRGabDGIASMYSKXCt4hVcWwE4JBQsyFLCJG5OONuWAahTa6WgJlfVfkmI5I9VIZKwWE1Egt0RlESUhZJ56iOHwRPN30XIcRaxROS6w5/t4aiRWSlZM8aAwnVuOcwnWKzta2g3AS5TQym/peUrUqKmNGK4FVVRXdGThZOu5dznj1UrDbTSgRyUkiUotpNHlMjP3AOI5Ip4gpEuPR33y0/QYKQxg5TB6UYMqKfGy8GFOQqlJOSynkkAnHdbaUAkWkUptq++WXehgoMhJF3TuapiGMHp0rbWx9t+eb336HfvuK3d0dfenReoZzjhBHlFmhco+Rku7+HEIh71oOQ8IrQQzhKL2oy/KvTFRCkEoVIWkJKmlKjsQ0kURAK0F3MuNk6XiwkKxWLQyFq9sdi8WScLvkFx9+QjeLX+0thYCDGo+7JoXMkiwVFE2KEj8VUgxEILuAaSTaVtSm8oWh9ERlUcXUBCmQgma7H9huA5uDoI8S6TRtm5lZS+NyrXsdQ4BCJDqp0DlibB0fUmDWS25vR7RJ6CKxJtMuHb2b2IcKxOiHxN5Hcsm4maPEgAyFvRf8xt/+Kzy9fk4UGTc4+s2ekiP9YU/TWIrPSCpvXIkGYxouz8/oZivGcSQmSUiFKQX2dweslpzaBosm+sT67oaUAzlPjKNHa8n67nPcbM7pvRM2ux0UqkO9B5KnhMBsccb11cDFvXvYJpCnEWca2vmM//af/WsenS/4+re/xZ13FPUWe/8ZsSQ6FI+X9ykyo5ujyrVIChOp9NVz7iyLVhH6hNYN0cOunwgp8sXnP2fot+yHWw67LQFNpxuMXWC6GrZUMiMJCFmRZkWBbAwi1/6vRFJk1bW2i5Zm37OxknwnoA/kmPFkrAgIlbl5ZfnN12fc7zzoTDhAygN9aVho6MfE3bXljccOmTPzVoKbcX1z4ItPR3wGXao1LpAZVEZmKFqT54HHJy0nrcfMCuerlo9/ekCbSC4NSkVkUWSfmXWGe5ea/W2gmS+QJfPpdWSuxJHCORE2Dqs8Ie5Qv/63+GwvuProU2bpQFicsJh2YBqEFkhd2R9ZQkoKpSaS6MBlwk7jS8I6aLTCywSzhBWJ9U6yumxwpUcpV533IpJtRiQN1BtuIoN27CfB3AiU3XNyIXj+qcLOGhZ6pA+KRsP+UHguC28+TpjkCYPhjdctynk+eCHZDZEf/nzHt97puDiX7MfErPS8tepoFq9zPl/y5PMX2M5zce8M4c9Y3xRerfd4NdI0M7LfEUXEKYWfoGlals4zlYkxJrLUldKo6r5YUWuVsfh6iz2+pAo9hkhrLVYX9iGgbH1uGKUhJr7++IyLixPCkChck8PE3FpOl0ueXu+4/miNV47caKa9oo+SK7/li93AOCZ8qIdAYTyCTC6KUDRRVpuoD4GxkazOz9jstsRJoRN4IbnZXPHZ55+wWM6ZK4eX+YjiLUiZUDQUXyvQSkNr7VFElymuZXVSaJsOZQqljGwPIzkV/Ah5KuhYMAa0EUewUEBGUauB+Vg1LAJnJHIGUlVNcuEIzCl16iuiROqCrokyJJJ41ONI5JG1VkmtSiSSqZVNmyQ5Vb6/QKJkDTY6UTAqY3QN7BqpOFUNl63hxBU6l2kaiTGWpBVJa7RyiGIQstJ3C5BTruZFqTBaMWsqgOjhpNneU1y9NjEMEzl0zFpdlYTaMdgBkQKH/RYnFighjitbySQCIhmkCIiSCGNGaUsKA1omUgEtFVprUs6MQ0/w6djIq+TE+KX1QdYK+i/1MEARtRpSIl3TklIhJE859jRnQnGIB5QsqJgZrp5z8uAtvJxx6D2fHSJLvWV1OnHv/orz2YxvLlaEFOjHAzEHpNKMo8f7TONmHI4++SwEIUc0npAEq4XhtRPDibacrCSNKdim4WY3sdkGrLXMWsP9kzkKRREOSagnJqWOfdPKf4ajAqt2FYhZMPlEDBVP2zYdy3bG3T4whlARbtQbThRgrOFsfmQd7APbceLm1cB6O9LM4P6jwv1LyWJex2XGgHQZmSTaaExjECWSQmKKAW2h7TRh0IRDJE2JkiXJQ/EFBbhGo4Wh5IKm5eW4573vvseYBzKSZXfG3fqa2PeokhGh6p5t4yp1UWhM23FycoZwLeMYSFEwTT39uMM2c2I2XJ5eomUkpkqULGj2hy3GSrSxaGewJWBjByHTasl6vUGb+kOSUuLe/QvWdxvO752imkQ/gVY10PLDHz3j3oXhG3/uz/Hi01t2uxfgfoE0BastKUiWC1GhSjkRykQOIEvk4nKOlIq7W3h6dcBvLOMmMQ41vKn0yPVn30fbH2Bdx7xd0NglrenQogEUQhU4rj9iyXgZ8CYRtAbVoGSLERYlJSXDOPPoziKsqgeEsmXsIy4nYtY1m6A97z9eoHRmEwWuFfj1gmnMnCwaNi8nHr8lmZ16ptDw0WbLFx8f2D2XtDOwQlVz2Twy7i3CFBwZHwRWaU7PLLt9YHnmGO+Go/PdEHNEZMG8KeiSefigWjn7fsv88oTDWnLYZpaNYYqenASHfcLKgVer1/lnP13zm+1T5ndXGAFKC0LqaGX5inZZENVDUo6PhFIP8BU9L2rwSdapnpSCnARTiBz6iFLmGGyto2FrLb6PdaqhGnL0JAJDLPjs0MYwX0iS2VI83Hut44PnB4TUZBN4tZvgmeL+a5qbIJlvIo/PHUJkXqwt12vBH//c8947DfceQBcPPCpn5LTi/jce8s2vvYOYJPNOEfSBl7d7Tp6aenO2js0g2Ex7oskInUg4SA0xbJFa11tZNhVKJSIyeaQAaWx9XqVjVVFprDQooQih5/x8wX67O46xI48ftNxbdmg69Dyx23YoW3jtcs6Lz7Z88YcvKd7x7AxCGplU4GocuBk35FBvfFKAkorGNnSNBZEIYWIMpqrUuxnLbsnZ4h4hwJPPnyOR+ClSQuHV1RVN24AwFFF38eUYTvaDh5iR0uKcw5i6PlRScbKoLJYpDfi0px8iYsr1YC0SYcokJQlZYLKsU4JUK35FS4QtSGlqQ+X4oXJSoV2dcilZUErWdZiqf06o/f0ERyrhl1fI+pcS8qvPn1K5ypLyn31mlShoXWrQT0m0khitsFLjtMFqg9IFtKAcrYIFjZDuaHM0IOs+P+ZyBOZlpACjFY2VxBaWIXP/Ah6+1nH9aqDfZKzOGCkRRddWRkxMYUSFBqU0WTbVuVDqOk2IKqjLOSNyPuYivnSBHFkJKRBzQiEq6vtYU5TiSHlU+n/iXOB/xmFAKUtOHiVUvSHEuo8zpn5IfL/Bh55QJAKF7+9QYcI159zcrZlrh7WCFDLj5sC4ChyMoXGalXLgGnwILFctMVb4zumpQmoDUpClR2Xwh8TlfMkb98+5vpuIRKIXPH214W6/ZbU6R7tqVitCIjKUXOqLX9QXfl0/1NqhoI6cZBZkoUlYlHE0jYZs0cZiXP196ikRUx3FlBJwRWCcplsYunnBGVmT1QpuTjJyqVletizmpbrWdXVX51GSx4TQR9ZATviU6i5cJpSqhiofMv2hZzqR9HlAW4FFoXIFjaSQGQ4Huk7zxpsP2fce6yQvrz5jHLeoImtn17rKaSAibKHYjO1aXLdgGiCHnhQnQsykKFmv13SLliFOyFJrRev1llw85xeLo1HwnM5ecDp/nezXKBsY1nfk4JGqIWXPYjmvKWPTouyCMNbee6Dgh1u+9saCv/pb3+PZs4k//eTfcfr6x3RYTDDkaMlNJogdJEHTFNqmpbEOXQS3LyIf/mTg+kUme41WoJSsP1AkpJF0XYPVLYumZdbOmM1mOFc70tIoioEgMjnVvriXnkGP+ATOdjRugVYKrau3XbYe6RRFVcDTME2IUkghQvKkWIheQlO4zYnpUI69ao/IDTGNXD5aUsyBD79IfPZyx34n0UVgz3rm3nE7JdTKkLLElwGrW1yE2AwsrKO4wLSWnGrDi82OZAyNCVyeNghR6GYKIzLKjOxeZmazBhr4/EcTIipoC3g4rDNDHxAi84RTNmJF7vfou5d4bZk5UWmdx9sURYCoPzeppK8OA/WhXCrZr+72kEiEAKMFY4br2wOz2Rmjn5jNLbkcDXeu2uh8DCglETKRCPQe5tbgtEA5w81N4Y1Gcs9KPjtMLJYdBs/1IXL3LPDgpNoUzUzy5knHeZP5REY+v078/KOBKRlOTx3m9hm70nETT3FJ0ek5ggs6fcHD8x7LLdbMudkcOHMGUVKlRYoDRkr6bUcUK0ScsEngXMDnTBESZevoPMRMSqFeUJxBJgkxY+2BB6/PMJ3kuY8sZeadRy1npwuyAC8yRnTge87nLZv1wIvPX6BCxhu4QtaOenjJ5pApwqF0PXg1pmPWLpl3S5xR5DwR04BPAmMMi3ZG1y1ZzM9w7oQX17fs93u0chQENze3zOdznGuJ2dOPBzKSGCLTFFFUbbfWuoKZpKBtG1SoVb847Rl2G9bTmm0YGEulMiop6zozF4j10BijQAtIKiOzQDQSJeskTMq698fUl76sH7kqQfoSwlOOEwaODBpxhFcdX5KUutZQR4tiERVfzJFjc3SxobSowiEl0ELU0Ls0CNWAhCAikoKUmcrWqrLmSLXHcvQxxJLJlPq5P172uqKIUXDiR+6dzzhZBtavtpScmM0k2rQYVRhLzzR5tAtoY8hWkUPA5AJJIkSdOkzeI6axrlbicXouErFEYpXVVIX40W5bUkEqjl9XjTqilX9ph4Fx9Mw6S0mZcezRRhBSYXfY89rjRxS5p/eBKVZakJAw+gNudkbTtjRtQhfo3Bw7b2gWkhy3FGVo50t8qB+enOqOQwuFdbbeRIQAUessi1XhtLNIldnlxKth5O62R2TF62cnzBwkVW2KU6k3OI6nx3qykmQkFIlEIY96pUo6sPgkCLngrMXQsG4UQmeUSmiRmEqpGGIkshS0FayWM870nJnumdmJy3stfUlEp1Bdg7L1YWeUQIiKFB42kuJhjAETIBYB2mBahfaBHEdySqSkcUWybB1Df8TuaokRgpNlQy89swdvkZMkxdq974cbos8Y2ZALTPuBnCOmsZQp0q7mLJTFhYxRLXlluLm6Po7bRlaLOafnl1gzI6VEHEZOVjBMO3a7DfN2wcX5GaX0bA+/AF14+exjuuYSrVvCNCFlpjMtd7fXdO2Ku5snZBJa1+/v/bNzvv3e9/j4M8vHz3/B2cXAhThDJkhyRui2pOJZzRec6xl3a8H104l+D88/77m9npBSoK1FCMfgB1KK5KKQEuYLhRWOxsyxbkk3X7BczbGNrpMVUwOCPk1EEUg6Ecqe/bRjmAInZs5M11GcVBIpJdIqhKQ2Y6aONAXGnDiMnry3GGBHQqSOYdszTYmLZebeeUvrJXMFn7/c8MHzER8V0nga03IiPX02/ONnkm/ck9zXmru7gbaxODkRjMGgWOkFNntQkb7s2PtCJHKx0py1AeECfVBIq4iDISQ4nZ1y8+KOLz5NdK0kS4+wDdvb+nkuqmUnW5aNZfvB73N/2hPO54TsaeWAErNqpkN8BchJOaMQFaYVc83AyOq7yMfHtJRgtSAJxeATtxvP2YlmmCZaV8PIxlSrIAqSqFS5ZesQKTL4Eakalqea6yvPfoi8/dqcpz+9IQyBVlXiXTgYPt5nNltDSR7x+sS9heN1CpsSuLot/OTngouzjmYmOc0HkrhHEJIhjAz+M7pe8vD8LebzU+ZDVd8akViMZ3z+6hY3O+H1ywZzsuEwSpJypKKZtnV1V3SVsImSSa6gRBXbFC0Z9x43dzx4eEY7m5gYeOvtFW1yzBeWEAaWi3M2t8959/G3KcsOITLPP3uCyzvU5QmjyOzlhklMCCk5m3fkVGFj3WzOanbOvD1h0Z1gtSHlgTAdarW0mWFdR9vOUaZjvhA8fvSCn//852gDw1jhaOvNjouLDqSuimAhCWFESofRuh4EjEEqXVemst70x5Q4DDuu1rdstwf8WGmwRkiyylUkJ+CYcEKKKijKBdJ0bHGZTFGVP/Nl7kUp0Lr+3CG+fIJzJETWJ7b+6jN51CTn+s+cJVLUdo0UAtTxFxb1YKFUnXpIVScJWmqMatFqCbIjUBijR6SEQ1GEJNaIS/UmpFp9zEdpVz66FwRUgqyGzimWreBsETg9HfiFgsM+QI4s5nOMNXih8TER/ISx5lglF0hViCmSRSbGyORHitTHnEWqq0CRKx0SgTYWWTji8gsoWQP7+khL/GUfBpwzxxF7/cE3TjIeAkXCfOF48fxjEJYcJkqRxKKJObEwBREDalQ085beB4bbwmFSnM4kSgScG2mNZEqh0rysZTt5JBnXOLQWuFLwRbMvgr0/YLaeDz5bk6TmbH7CaxcGK0FLUz33WRL8iNAQhax/UFkPB1Kquu8TXw5QJFIYCpr+MHK73uDPNbNGMp9JurmkOUhMXw2BiJo/qCyEuhNzrva729WSM23JZSLk6lXPTpNJCCAFySiqzWzMiRglOQpCTngKo8/EmLAWzk87zGXLxkb8NFFi/Zr3ccQ1lj4kdDvj8vwhm37iMPTsdxO6zCBEvJ8Yhx2P7p/T+0TKCjeb8dbbb2I7jZSZfn/D8xcfMGs6dDHcW14wnz8kpMhm8zM2h1+Qc8BPHqiGsJPzR7y4+WOGYY+QnpAUbhaY+onF7A26LnJ5ecnzJ3dc3r+PnUP+dEs/9RTX8u67X+fhu+/z8dWGzc2/4eHFE5YzgfeOSUEUd5x1K1bSUPYd3//Rmp/9eIufNDHvcY1C2waEYvQJcn/UNRcKEecWLBfnrE5mrE46TpanrE7ndLOW5v/P2p/F6pbmZ57Q7/9Oa61v2tMZY8yIyEznZKfLQ1XXRFWrgQaB1GokUEsICZBAcImQkJC4Q+IGLgAJiRsQaiRQc0F1ixbVXdXYrrbLaZeddjqnmIcTJ860zx6+cQ3vyMX7nXDBlS8ypVBEZp7YZ5/9rfW+/+F5fk9b7WVa1y538JoYeyiJYdhxs3vKfjcAha51KLVAlPu6M6JpYJ5QoWAi3KZCeDHhi+cgif2Uub2OfPO3IfeZO/MFsQlsovDRFz07L7S2ZaEjJQjKGX4RO/7gkSIMmb+7LEyHHdFPLJcrojUsO8XzzyNP5gfejTXznGTZhMDdc1gtOsRGUlEYJYhYbraRXdnjnzs+f7ljmloWc0VShlgCYeo4mwnvF+HGg3Fwni9pHRjp0a5FfF1/VUISdaNGrumDWhEKjLFClNSxg9Mi1T6moViwpqoCNtv6zHal5sBro0ASTpsjLCZhxKIpnK1aDrsBnzNGj7SLwq1XPFg47r6+4OMnW95YWUxjySHgiuLpiwPX3OFzN+e921u+1XneWipKhNvB82IN80GzOAus3RKl7tLqQGtalFryJ19+xJ9++Efcny/53r1vQY6c2MA3ZM4QMiUI7WqOtIFYehollDNDsdXORhRyNJUuSkPyheATi5Wn6epV5keDW2myBLz3BK2YhgNvr77Br939uywkcXr6Az6//GPuPrxHuJl4ebOljQrlBxbtAqvu4KwjZ03KA4v5BaeLB8xnpzhXYTopd4RmiVUB1y4QHLrmr2O15b33vsXTp8/xhx1aa2KOXN/c0nar4zNfAW/WNGjTUkI8hgjp46pHkXMkkdj3G27WG7a3E+MAkmoOTcmJSapGq9J0qtIfVYvFIvV/CLEc80b4uog0po64czo2llLqJPeVY0CkXoLH7j+rOnXIlGPA0XGllervWS9/ajcvBS2A1BM5Y1A4rFnhmjuImTESkOzRYWKaqhg9m2MTKIIUXePklSBYChWfrESBsrgsBJNonWI1t5xdCLNF5nqT8IOQfaHtMqEpJDIp+2McsyKmqiGMyaNEUwMWE6RjvkM55vWUuj4XJZXfcNyDVBpkdZJorWvr+2qs8qssBnJKiKo2o+3Bk1Li/OIU8lTHE0FjigANWgKL2YoyjFx+/Ij5/B5v//Ccvt9jxozhhMuh2nCUgtFn0IZcCtN0QOtM5xTORbrOoMYDN9tIVi3LznExm9FpuLiw3Ds3KEnk0pC1Y5wK/XbH5fUNKWU6rep5dqS0Qh1piqgj+rjm09eZU6nhSmKwOjPvNKuF5XZnMJ3GeiAWcklEqdYZhaaIxs0KK1sFKdE4RilgM2NO9ONACJ4YBWUs7VxDivhs4VAvohwLyjSIqQMpM9P0OtQ8g6lmd2cNVhl2m309PKMifyOw2W0QHEKD9zvmK8G5htPT+/TpKSoV5oszQhy5Gn/JsN4wTT2lJKyesY23SAmksmGIV3z15EtEBRormHaFazqQhDaF5y9fkJNGO0FpjfPzCivqhG6+JpSeR5efELWnj8L+S1D2nHcefIdvffsH9G7Gi+unhP0f8+Y7z7H5jOnQ0diBB0vN3J2zu9J8/InwxdMbNruRaBNFJXRxjCGh+mMstipYqV515xyucZycLLhzd8n5nY7T0447K8dy2eKcxZk6zkVVPoZWIFNhvb9lv33BdnfLdjuSUoM2M+6sFPNOIVrXvZw26LZFzzJ58ATnODSKYAtlV1i1nv/wLx+RzIJ/92/NUXrL1W3kegxYm5ljMdkymZZ+9i5/uZv4+Eng6dXA/+S3FK8vPB9vIm/cWRF0wiXogvBPX3h+92zJ75rqtgm3IBjefNASfaJguFn3nN/rePZ84MXzzJThZtOzHQXTlToypSHrHUjmxGnWTcueFrN9RpfXMLMYXW28VnUg/uuOSqhTgUShGGGMCZ8y6Bos8+o/IoVyvAAMUtn7pX4vd87miBQWC4tCcEoxlQFnHK0obPHMWwd0jNtIFAddggiHPnDve+/w5GLBV7/4U+40CdVoFIq5Coyj8NT9Fp+aGX909TkPwiMW4RoNzJiQfWDmFDP9nL3WDLnjs9trvnz8JzTDZ/x79zK/fi6o2S95qt6kLw84nL7G8/4FrhuQvSc+3VH0SN9alFhE15Q+oyxFYCwDSQolCc42CBatNNurA7aA2WXmS8fZrEUHRdKGn37xZ4yj5Tff+Nu8fvIdVOPwj35C/40Zvf8x02HgQddR9BnL5UPsaoafLFoSzi2Zdyc0TUspgcnvyZKxTYM5MumVkUq0Q/A+8vDBPR7cu89nH1+h9KyugFD0/YG2OcHZGSncHLG4Ce9rgJzRrqYYGiHEA1fjNU9vXvDs6pptP5GjwlID6woZIzX4p0gNTqt0vXLc7xdGqucy57oXV0phralTg5SxpeoF5FgovFodKFXZKyKq5iRQO3Y5aljkeI4ryZjySsNCnSLIsVOWOu0qWSG5QbNA61OynjMVj847dPLgBzKFlhajBVM8VipLxRiFEoOWmt7J8S4pSoCIRmiVYTUzLGaWdYxVA+Grg8loi5JEiYkQAhFdKbM5VGYItRDQMaGl4LQlljrRkFwLoZgzOXpyOd5FokAJphTUUU3xK9cMjIce5xyNcwzjAas0mkSjDWEciL6qNZ0Trq8PuHnD+uUtjx99wsydMjzf0c8u2V1fEqzlje90vPHwhMNmw9PLKx5cvM5mfyCkjNWaEnVNlCKSZgr0DCRhlMIZx5v3z1ktLpjKhlQ8PjimMJByT0PL1e2Bxy9f0jSmVlVi6mV/RKRpbTCmKkBRghKNFE3rGrp2gWsWGLWnNYa5szRW4zRYkerjRMiiCQW8BJJSNK1lsaj4VrRiyhD0yNYbYhBCSFVQZDVKZ1TOGITQF+I2ses9PlnQBT9khkNPcMIhRqY+kPeZsAuEXHjjwev8/X/4j/j8syccsqexhn4cOb87w7QrbtaPyEbz1e1Lcqov83a7pnUGckKJMMOQsiEqT0kdMbbs1cCwvqLpWkqyGGUockOKLTEcX4BC3e/lmjKmzQEjDRjP1eZjlDQonWu3fRDOF9/jm+/+Bu78DlfxKdOT32cxv+beW4k2vM4w7XjrTHh49wHrveXP/tWWp88ntkNgCCP9thACZMmobJA8QwiIHsA6TNuwOm04XS04W8y5c9Zx97xjdTKn7SyzeaZpPTCRYiaWWtUnlfElMYZbdv1T9v2GFDUhWG5ve5xZY2SGKENRmsY4nLFIgeg8ulG4VrEwMw5pooTArHU8vQn8+39QeNI+5B986w1OZnuaTjEVIc46RntCsCuuh0i/ec44Ped37oz8w793wmd/ccVpu6ygIK9ZLQz/4pcT/+/Hwtt3TyBNuHni8umOu3NHSIFYhP1GQBy7g/DpFz06O8o4w8dA5xqGVMhS/f1iDKKFhRQm4yhqxevmmuU0kO1RS+I9Ood6tIquU5eSv/aKoyzjNFbWvtY1xa+kCkuRulhAFUwuRFMvxxAMm01Clg1CYHViERXptKXEQmszy6XF57qZ1aaQVMEZTVQTwwQyeNo3f4356j67v/zPacYbjHEYOtp4YHv5Ptz5Nmt1wXZ+h1neoYevmLmJuTmjMXu07NmIw9++4N3+Y/5br93wvfua1dJCGbnZbUifPOGnLzXm4e9wp71DEMiy5WzydFMC7ZmU4JUQlBDFMxVVu80jYGbSQwXtSE0pVEDc1WuiSGH0gUhD1Df86IP/gF88+nO++do7/Pa7f5/vvPlfIDc/RQrsP/wrTo3Fzc9ZnDxEuRlhViAJzrZH4VjdISulagDOMdEvF9D17vt6p9y0Mx6+9iZXLx6zPQxwBN1c37zAGrh7/oA4FZ4/eYZog9YaaxvarsO2Gh8OjH7H1fY515uXjH7E6GOQ1dEOWHJBQjzu0kEbhTJyFAHWDfwU43HgXwvcOupOSH61WuAo2D7+/9QRf8mKko8Xe66QN44Xvhzxz4qae4JwzPDSX2fc/PU3IQgapSyGDiULMi1jqqmGmRGfJg5TonHCLAmdU3QqYEtG5NVov05LOMYwg4ARnBFaDcvGcDp3DKuRBxct7sQwGc2YMiEPKDJxqomfIpW/gFVIqYTPkjNONKnqKinEykBJdTUAdSptlD4WPIKUBMfC6BU191dWDJwslhwOQ33xpQZLLGYNnWtrMZDquGkKA0k7bKPY3vYo5mhpsE3i80+e4mLLZjty/dVPSb/xJp/+5EPWN5HDP8689tabrK+vaZqGrjmO9AV8CHS6YT5XzE8alosO2xVigcubyBAKipGmtZyuOrqo+MvHX7DZ71HNgqPeA1EKYxy2cbVLtBqtpI5bimDQGK1JufIBcAajNW3rWLQtMxnZFU88MhDSlLjZ7Hh0cKi2Up+u80Mkz3hveeCuC2zifYYSaJpELDWpKkgmTyOUHgpsCDz83je56xueP35JfnrF/nqCnLn7xgUHbcCNLB/cp2vPMcsFqnV8MkzIaw/ZP33OlCO6DWzGx2yeX7FatpjGAj261ECNxjb4aQSp4RfKqKrOjhNZClo3+JAxtiXlCaQebqpf1eJBKh9cG0fJhpQKohK2sUxhIifDYn6Gn3b19yxzFktoz9eE/CPM9YbTRcS8Y4hRk4YWs9D88O1vYjnlT3/8iL/4YMtIJvuR/roQhoLkiCqlQqooKDxIFa/ee7Dk7p1TZrOW01XL6w8WdT/s9JGjX9W2YdhVolfJGGOJybAdJ9b9moO/YhhvmeJISjVtc5oGtptb5t0cZTQJjcwMVtcYvyyANSRryM5gtaEzYw3V6Vq++zsPeedb9zFvvcbQCAcClICoBqLH7wbS9pJ+PzBsMv/uv33CefZ8ZmeodmAIM+7M4U8/j/w/Pur4zTvnXNxVxJBoRXG9T7z9BvhdxjeKz57vee1Bx7OPRsw0J5UtSgRj6yhSFUeWDDkAFRPbucLJzDC8HDmxkZUyeFfIJaBboB8pagZInaiVRMrpqMIShlAtrkYrlK7x3logK1XxyKWgcxXVxhRAOQ5DROWRkoW208yWQmsbQojMnaZxNaK2FDAkSkhoadCiOIyR18uWRXjM7dlrmL/zb3D4+Y+ZTZc1oTEEXD/RYigGbNMR9JK8fIOpzBFbcP4XvDM94+/Zf8XDd15w5yQxJMfu4LkNglMte2mY9EC7H9l+8mPsO7+FT/eQsKFVpU7QFDiphMoGRRLAw3bSrIOhGMVsViN/Ex4vkcZqTNtys53Y3I5gFc61iG5IYrmJz/nx+7dcv9zz/fe+yw9f/x0+apas/YHDJhG7u/XfUXVFplxdZBdGimSqH1odJ54RsFCO6bFaUzIo49j3AdMuERSN1Xif8T4wWyg2mw2SKm1vsVigacgxo0RjlaW1Bh+2TNNA3O8x3nNiNU1nK/01ZbJoYsqoY0COaKmX5VFTUkqpvAmtKgukQuehFLIklBaUqxHPVUdQNQOvIo6P8m8CdUeeUkKJOo7HC5BJpX5VTM3DUTFRlKoXpQASyYBR7lhUaEpWTCHhx5Exj0QXUQT8WLHHZyeW86WCRkGTUSpUbQI1jCpRp2+17AtYG2msp3OZsxNN907Ld751RtCOT64m9qGQcyTH+mfRxtTOXoTE+HX2jEawxuCzkGIgxInM8c9NheU50VWsTM1+iErg1UzA/M2u+b+5tTAkDipy3rUs/IqbsmYUaGYLDtvHlCzkHKAs0ETEV6qb0hDTiEstMyAZxWxh6DcDP/nnH0DJuAxf/OjnnCwW3D9b1AM6a+6dnjNXltBPLOyS7sQiLrAbPONh4KKbsdILlsUzX3R0C0XjFB9/9CU//+Uv0c28EtNiFVo0TbXedJ3FOVULG+1QmComVApjDE7VXGqkjp4XXcOyNdgOlBNUrlVgTJ4vbxKryy3Fak7k2/SHt7nZZX7mFCdnls2uYb+3nLT3ePN8zaJbo3Ik6bcYtCGcwMmb57jlKSezGXf+duLFixc8fbHmy2cTN7LEtpq7KeL7kawNh9Hj9zsWixm7/TXTtENKz2ph8H3D+f1vsFlvGMfE3J1zyD3iFNuxJ2fBNS2lTUhOKH8gmhm28aA9BIvNLS7DbNYQEmixFO3pU0asJehMChElMHMKnYRYCtJUK9up7XCtoTsr3D8TFmZHoxumxlZRU69BC/MLy910hw9+tufTR09Y7wwERTlMDPue4AtFabLoStkVIda8V9o28/D1U37wvbd57d4KZQWjIycrR2MLGY9IjYw1ReFzJKpA0YqpOHZD5Gr3kjG8BAa0ylVLIHXvSYJpymzWBxQORRWAGgWqKFAtJdW0TGIN5VmPNcntze+f8L0f3uHhvQYlB8okVaaqEpF1tUnGiD80bHY7vnMB3/12ZlxPJAUlLbjTBH7x3PHPvzrnd9+dMTYH/uWnO/5r3zecy0RKFinC023POC3YjfDs2ZrxSvFHe8tvnZ2ztPu6U50cjQM/apoS0RmMHdikxDv3v8/iesZ5+jO0qfZAlEaVHlGnWDPhYyY6TxcykjtyWwi5JxwCUTeYVG2E2SSSrvAWXSJRZihzIGeL0ceVXCi83CcupeV7JFYzy2IG7SnEUPAHyDIRsiWmBmUK2JqsGEpGl4Hf3v4p60nzZFB8pi0v4oy7YaIpgRI9LBSiOiRMmNTjbORO/iXf50u+cZrwEWbnmUVr6LeG3UGRQg3PodTcAFMiuck0h4wbb7nWM0qsltSFMXXsLqnCqjgSV6Oi3xW2B0OSGWqRaU8m5vOCGMdQND5GRAoqKSRPlIPgFYgkWmfJi8BHl3/FV8NLnl1d8cP33uNvffu3+fFHHxNwdZIY9kizQErCYup0soCYAsGTYgCjcPl4gR739jFmUghs+wNj3jJfLpk2CQmelArTAKoL3O4uSfIAGsP19ZomeLoHZ7hmjk8DWoFSDmcMd1crSFVQilQeQbXCCSrWLlykCk+LCD5FYoo1cq1ktLGQI0I6Fiv17sqJOnlNQpZQR/CA0jXPJucKfcteEaWgXO2ahUKKArFqOZRROFVwR3iRmKoX0yiyTmD3OGmY+ZF4GBBl8cPE0N8S3UgUuBoyY9hxf1946wLyua1rZRUoYmm0QhWhqMyUKjCp0RqfBxZt5uwk8K134fytM85WC0rn0G7Bdr+lGMMQPSZ47JQo1iAUpFTabC4KGXvasSeUCtmrKsqMyVK5DxZS0SSJQMGIxeCqxkP+5lf83/hXKgWNgxB7QuopecK6DuOq6hFJNUf6qKxcLk/YH3aM04TrWrwURFV4TYr1h0Vj2B/2WNPhgvD+H/05P/wv/Sbvvn0Haw231zu0W3HnG6+z6kY2m1s+/egDSjG8+/ZvYOZzlF5ijKLPtww+Yibh8mqL0Y5ON0ypCr9aZ+ncsRBoKmu97lh0vTSOrgOlMtYmtAHrTE3+cwbrLI1zGB0Jx32TiBBD5vGmpzv7NVanv8a5NThzxTQ0PNsMPLp+hDVzNqsFT4YOR8eqm+N9hYHcuTijEcUXL18w9AOd0eAKuSsMacNX62tccQjQdB0ym1UamGimsSdNnjQJ89kJ8TCwvUmsb27o+wMgrE7mLO8Z9qMnScHohJ46ti8y4gr3vtHhhyqASRO02jBvDd4b9iGQC8BACVVY5JTC+4BVCYxijJqYdog4yibw2smC+2/NmK1GZkoxjYqoOka5xR4ajJlhly0Psia9TPzFoy+5PSRKtpAL2/WOXZ/JqVoIVSlEEmCOwkrFnftnvPbakocP57zx1il3L07IEgixJ5ZACglnHFI0QxhIJeB0Q0ma0Uf6MHEYe6ZpQFREVKngEBG0OSao+cI0TWwOW8QqtM4YqYWrEYOfJnyZmIpmSpYheKQJfOdbb/Gt33ybh29c0LUdhUpBg0QsiYxjd5g47AsejfjMv/d3HOd55MfPhCEXVvM973+l+Jcv3mF295QXuw0vnwkzWTE3mqEfKcXydA0vbx3TOHFhLJt14D/rv8e2fZNv+R9x7moUtrKFWBJmpohFY0tmLgCKrjF86913sJ8BTYNIFe4mFRHbVyCXqvrtqE09cHLhEDKTatG5pymQxWCKIxZFKoGEwnaJ7FuakGjI1XJmNaM1XJsZ8RApX/TY76942CaMlmqZS3CIwu20g1Kw2tHJxGaEYXJYY3k+KH75HL7anrORhzxn5IG6YbU/YD/+c9J4YL+5RsbAibOcv9XxxnsNy3nhxa2ivzHMLgIpKlKu2GHRIFqTfCLrAmKYtEfvX5CZsxkibQyUMpGNIxdT446PluWQhZSFDKQcuT0oYkm85SbeO4F7M8cYhEc9fD4OZLNgnwNzgeQjt31hNhhcp7jZP+FnhwPr7Ut+/Z3v8YO33uPpzYHLEZJYlA/4VphSwBRFW4Q+eYYSq20OIZRMaystb+8Htv3Adrej3+7JPrJannN5fQU5YV7FsktLzrC+vaY1Szq7IKWe7Tjx2bMnuK7QqIGShLPuPsodrXYUMgmfalJrKQVd8v/P6B7qDjyl+i5EYg2Uy1J1ABIRDIIilpGpBGwUStbVCVYqyClRUFkwWfAIqtTVpaQK6LIiTDoTUsJ6Q9SaUdXBCVPBHbVPpRTmaob1BZNvkblDa4f3O0bvuTwcmMLIYYDdNnE9F6ZJH4E+M851QqNIJtOYgoqeRin6khlzz9xaZNK82cx5860G7XTNV7GG2Dh+8agwLw26KIapx5kFKhcOfkSnTImQSyLgK53QuuoUUAYbC+3KQVGEENA60xr7NQCp4IGIELDqVzwZsHZF3Pcs5zOy8+SDsFrcpe0M3kemIeCUZpoGcjFMPrHbHTCurUAUGqzKGFuIU8GHTCowP59x//4pcUhcH6755M/+Csp3cTnx+NPPmN094+34Ojdqx+WzSw6HLSFl4uCZN79NqzqG3UQ0Gjdr2axv+OyrW5SxWFNQTjBKmLmWptV0jeCajLUVPIGKx11PoJR05A801aaWNCVV1GTXtVWBfnyojTGIwDROxGnBcv6A/dSjckvrWra7NTOd0YcDo92T4i0xFdp5g9IrlA8YaXjy7IBSdX0wlczgMzMszy9vGEri/HSF+II1Fk9iP24QEVZdx7A5sLm9QSvDYXvgxVefoySjlWG1qFTBMCUKLSntaxDOXnH52QZ1UPSSaM4azpca3czofSBJYj3tKhUyZKxt8FpwJqNMJKaEpkGKqz7b6HHS0d0rvHcyY35uyDLR+AWHLDjnKe2eRTnhbrtCK8P7j674cqOxKrErE8Mwpz94NpsDw6gQXEXe5oKkitstBC7OV7zz7n2+8fYZ9+/NWM4UTSsU6YnJk4lHIpwwpcj6ZsOz60sOaWCxOKFtHUIk5UDKI0UfsDagqJ2GkoJWGddASkKMMA0DWyqHQoqtRYoIZZqI48AUR4KK2GXDw/fu880fvM2Db9zFtBDTgFKppqlpTcia/RC5uR1IHoZ94rfvJ7777cDtHkoOLGbwex8LH95+l4s7DzhxDb7ViLH4zSUqD/RDYjMl1i8mxt5w2rXEw4E/8t/k9uR17i8N17sFr6VrtL7AKs/kE6pLpGirlVsUiOKrZ78gmzliAkk35JxxxRJKSzp6q4Uame31MW0zZ4YoRLEk1YLTRxJKolGVrZBR4CcyHciEMoLE+rVSq3h5UniWIsMaLv90w9//zQvOTyJ+2GNKRyoDU3KgVG1E5h2Unj7A5d7xL79UXMYVXgxzo+msQXQD8Qr54obYw2HKNI1GlEblQk4BVRz3Liw5ZnK0NTc+p/pXEfIru2TM5CI1Fn08sFp4NsMOSZmiNblUqXqqSjR8KgwehpgJOVEkMMaW7U6zy5qUAqdvZ773sOHvd4bsDT9+GfmjJz27scM0EPBsgkJH4URgVJHH9hFXf3HDv/HN3+GdOxdMz6+59SBOUXxd2SQtVYQXYtUHKEghYHQtBA5+5Nn1NU9evmS32RH2EybD6/cesFyeILImhMI0JcaDJwvMWkvJVR9VJ/WKr55fokxk5qhgN86xVlc9hMoUyfWSj3Wtl3OsEb0cAVUlk3MtBnI5rjhKTdbSuqAkHXf7iZgOjDmTQwWO5ePnU4jEHGphrSKu0YgkjA6UUohJ4yPgq/1bjrhhVQQtmlIq10Vipdv29NhU6KzjIIosggmBkgtXm8TlrjCNinEH7naPlszCZRY2sDCWdl7PR5tquuwh7XAl09gWWRhKk7jxLSsVmJ3MsAGwc/p9DaXT2pByIErBh56Z7XDaEdOIUC2SOVeLoXZtpVYqodOaLIkxJ1zbYvJEDpGmWeCLxzBHQXUV/KpxxMuTC9g8JeWAUoXWNSgl9MMGZWu85369QylDQRNDrrjflLFWMGPNX445gynYmUU74c5bp5ycWtYve9ygiKPw6GePaC00c4vLO55+/JdoazFiWbiGQKTvL3n6+S/5e9//bbbbgdMHSx69fMKHH/6M65s91rakCE43zJoZTaNoOmjbhLWV9GdMPRBj0qTEUQClEJ2O1pcWlEO0pmkMs66hdSMhpMpDUJoUCyfLC8awI6aR4s7w08C6X3Nxesqdh3f47MlTdoeJxfKE6BWbta9edx1wOuLHQlAZ13T4MfD4yRUlaRauZQge5Zpq6RonhArf2W7WTMOItgbIOK3RCnIIFJ1JKqN0IpeJ3c5yvjplWI9cPhrRsUW5gbmBw5dw9usdSkylq6njGN46iqsBGp3RGFcvyFLaKtAphXnXcn6/4XQRsCcGX9b4KWL9nL4JeBco2XDHd5wbxxePA//iJ8+YgvBgPq8wjyzcrPfsDpHRy1Ec5KusSCpytKSEc4U791q+9727vPHmkpl7ZWOrZEIjghVHIlJK5uBHnt9e8ejpcw6hZ3my5/TMspgVOldoG1C2imxKKGglGCMYU0EqIpbgLcErpjGxkV3lEpRUKWkxkYcRnw9kG7j7xinvvvc6D988w7kqnsslYHQmFWGMmut94PJqgx89Z/MlVhX+0XeEpS48Xvc8i4p/8seZZ+sFb7+uKItCqxpKXpB3GeNaerVnOwqj1+BnLGZ7vI/8sftd/MUZ37ANYjVr9T2ur55z0UZcqpOkogomCvOuMs5tiSi3Ynz5JbK4Qwk3latAJqaKbYVKPlNF40uuVt1J1YwLAkMxbI1iVSpxUkiU0mA6QwoKqAEsWmpGfc6JC9WCt7S58He+ueDOwfP+R1vuvdPw+rw+XzFqfPHsDgktM9ZlZJssFznyy2vH+5tzTjrFynrusudCbZiZDVpHpmRJypFFMXqhL5HJFtIx3MuZntgoJl+zTjJUpr3SR+W6gpIpTFhVO6+LTpNbIY6JyQqkWBXfpZByxkehD8IQq6NeNCS1x08jN7eZL3C0JmHsgR+0La+dzvhHy4nW3eUPPkrcbicsI1lGctsxpALrkf0YODvP/P5f/nN+8Ppv8N13vsWzoeeTqx6rTIXSpIksGWM07ZSIPpC1Qhc4TD0vt1seP33C869eQK7gs33wnOxHzlYnjP0adEF3Lf0QSNkjXSHbhISMkkzygdvLW6YSamBQ13Ex65jPWpazltZYWq1wpdDoUkXGUgvDkv96n56lVGhVKehjSmEhUKN6DY3LGJPIKdIPEe8NISdSjlXzkz3kgFGJzo1YFWnKiDPVW3/whtt9ZLcrDF7Y5RoZbIrgjrZDdCAVQSuHsy0vBs1Xuz1vrYSLucF2hjEpclnRCux2a8Y+oZpSqbB9/Z6nLKTo6awmlhZVFKtmjnXVtvjv/2TN//0PnnInLvmf/qMHnKwShxJ5edPwl58V0Gcod0kwCadm+FiLZqMaonNkCrpU1LKiusNUjKTiiaWKNI0ytUFVmrbryJlj2orUtEcyY5l+tcWA6CUzvUTE0rSK4APdyQWkgXGCEsJRwFCVi00zY7mAq+2mXh7K4zpN08xQpqDVCChco7h5sePx45dYCsuuoTXCYt5QuoRphEYcUQUsNUJXAY0obl9esrlZY4vlk59+yi8ffc52PxzHLZb5rMF2CmwNtWlawTXxGH9Z6h6LCsDIuUZmZiUE5kQCRXVI0TV6UimMqWPkattUHPYD3/3Gt/nuO98DPHn03Owv2Y4Db955yFl3zu3LDWkQppg4nSdmKmOloHP1uxoUuiuM/Z7pMLDeTLx4NqDViq6d1bzy3NO4lubo5vB+rA5ZpwlJ1x12ElQxlBhJuRBywBiFiKG/3KEPS14+6ZGoKTYRTSWj2XHgw/e3vPPNEyCg9YxSFJR0nDIoGtUgTATvK7UsKRarhuVJ4OTujiEdMHrJyXBOKg1qucOYPUu74Hv2gu88eMjPPu/505//nKuxMG8Um+lAox3aFGazFlEZOwn9vo4ZtdYEiVgDZ6sFd+/Nee31GYtlxtl64aek0LpmRBhztL4lzZhrsuEhTqhWWHYdq05YmMS8Kbiu0gpVUZRcCLpgDFgruKaKKz2FkiM+WmIopBQpcWDcJ8YiqCwkH0l+oFs6Xnv4gDsPT2k6QXThqFKpgCIUNwdfY2YPlk4aGnEsFo43lgd++vGaf/JBy48eQRoMZ42Qwp7msEDPLZOaSCmwnDf06y1PHt3S6o5ia8Hyc/09VPMeD5tAzEIZMuvmAdtyxiq8RMsMowyRCmY5n1cbXCoTo7rHm+eKGF+H8RKsIklBiznmCFRipAakZLLSHHxElcxMRy6d41nS/JZKGLNAaSgRlPd1j2pgEktUlceRiqKTFc1mzmnsee3U8+ChcP+u5YOnez7dBe5fnBFLYRqrdVHSwJA1T3eab73eUEqHtvCaS6zMxMptWOgRoibnSmc7hIHrrcZKw6pVMB0tbEdIVQ6RlDy+ZrlglUGLOnrGOe61I1IU45R5UAp60XJ5gBgDqlQbdMyRmIXeKw5BHbNUahE0REhREULkq91IMDBrHRet0LqE8R7tHJzN6MY3aPPEbXrBZrhhMpa5nmF6zTYcaE8UH918gls2PLz3Jndv9nyZegwGl6VOz6SQVBVwZoGgCn0/sLtds728ZtodsLYhW0WImW625HSpORxecjgc8KnG/lIMOUfisCeMmUYpGuvIWkhTwodIHhL9sGHWd6zmLauuY9G1tNbgTC2qdPa1sDoSDEVXMqWRCu0xKGJSlOIopdrDVTfRtA2aFcZNTL6GecUcj01IwqrCvNOcdp42X2PyFXPdYMWwj4ltN3JYaJ7vFU9vC9vdSCZSuo4xC2Xag22Ias56UFzvqQE/eEYU2+sDz682nLkF3zi7wJ0tWLuCUWB1RkxATYombxFpKapg8zW2W/JnL7b84ceZn3w58Zc/uUH3MC3WPNoumdbw5X7il18Ynj5tsJwg+RKjMt4PGKmptkpblK4Tj3SEKSkyjbF1EhVijRLH1s7famLWeGqwGpPgdI3Pjikg/IpTC928EOOeEgydrRnp82Vhd+tpO8WQPEXVBDNlZ8zmJ+xGge2uEv1amM0duclMfkvab8FbXn51y+6q+irtvSVKFLppwGhmXcugJlKu4p4YCikklFakmNmkgT95/6c0uuPF1TNCKjjnSKlgVMHahG0y0iSMSShbKCaRJJJKIodcE7JyhVVIrtKM/ajYDncYVwZHqAp6JWgKKQVSyYiqbPvdZsPzmxc8fO11LmbnXL98wjuvf4Mz13H9/Dm5BJqlpgSDnTv0sqve3f7A/OIO6+HA5CMzt+L5Zs8XL9bM9SlFbFWgT3XXryTSmAanLfvhtnYuUqNfnXYMNxvikJCsIVc6Vc4FYzWmFy5fbtHW4Zqq1B58xmpFkYCNlsZ0+NizH3qcndFojSKii+D9gU4WLJ2j62C+0OimR8zEFBqW9pzTvmFSI3k+MBXFvbji3XBGZMl/8OVT/vKLHTdFc6JAT4lBeVQzR6mO2ZnmVGuIcLueuL71HA4DjYM337zgnXfu8uDhBculom1r7oHSVdGeSx1mp5KwyqHNDH/I7HaRkAInJ7Ccabqmikutqx1LBZZXXp5SFWyiFTgDpal+aFUCpYxMEkEcU5gx+uprX7Yt3UzTLJacn97hwf3XWZ0uyGWs388RUzt4zW4sPL+F/aFBia77xVxw0vC/+1f3+fnThn0vmBhoVF/dJ2IZlLALE4cwInrgwJbDi0SKgI6ElPmp/m24eI8LlRmCMLkRLS2m7Plk9S4n/TPMvJBVJGWFmMSig8YkPrst/OLZjh98+xts1ppYFFZV4qdBk1NCFyHlWgAbClNWTD7RNga0cH1oeZYdQ+cxMTHmQmNrIJdqBB1AxFCq25aiqqhsTJ6DCrwssO8rq2OnFZ88nfjies1ry5ZtrygJVIqokNmWlj5a7jSJcw7cVT0rdUtHdS4EVShJIdmhUmGaCvvs6PaR3VgIyTAGhbPpCHOpY2Ok1GeCSMqewcPOB9KRstg2LV999j4jsQamFX1cXWVKUQxRuDkU1ofMlA1o8LmQs6+AGGUIh8DLZPlpsSxcxuo1d+0Mv/a8/GLNtc68+dYbvD075Ttx4JPNY3b7NQ09Ms4IeUYo1/zk859wdnKfb792n82Tr1jHA57qFkopVdW9U2QfmSQxjD3D7ZYyepyx5FIYhgEyPHv2jNbeZblcVpLeIVNUIfkaiT5rV2zGHsnVgptLYkpj1QKI0O8iwz6y30xsZiNnyzlnpwvaziBkyrTH2ganG6x2GKlTJ1X9caRSz15tWpROJP2SXtb0k8boDmVaKAZDJXCWUoXE1oKZNZg20kmgkQMzSTRJMS8TZ8s5j9YZTeIiJOKw4MU049l1IOE5tafIpLjxhTFHDAWTApc3A49eZvqDJ4dEWsG87ZjNGxadcOIKd2YTSkNQCs0JlkLue05OWz7oA//r/+dzfvahwurCwglqrth4xYdfDbxMEz96+pJh/RCx50wi6MUZSw1X5aY2N6kwpgEnDVZUFZuqRMgBX1JtYO1R4xYVVh3D77RhDCO7w5qHF29yAEqKxKxom8Wvthi42e2wyuKzQqkVF6dLxCqGvuCDwocMYsl4TlcrYoarly8wDlazlvmqQ9TA1l8R0kD2mbQR9jdTFa6dtLiiUM6hlaOY6tNvlWOSidjXPacUTQ4JKZYYPU+unlOKRekjdXDMFCM0c4N21ausLXUiYBJFIrmEqtTMr3yh1ccqUr2oMXtCmup+SSmUCI0xzNqO1jm2/fD1zjCkhCjP8+snXEaNa4TN4SXP1nCnmWECdDHirGWxWtRLLAt0whgCMiTaRtHHwO31ntBnhibTtIUUPMY5TEo409B287oeSIUQRwqa5CNqpvHjgRJjVTVLRqyQo2boJ2Q0LNqOfhoZQzlqKBzaajwjOcO4TyitMSVjqbumeLT6rDrLfKZYncB8BrO5oR9GinI4M8fkwqB35KahSwt+KJZ/8+Ihuxee/+zzF/zhVxt2+4gYTWkcPkxkrVgPiVZ5pqK5c9ox7wwP5kvO72iGMeIMvPb6GW9945TVqsE6oERKrslsoiratJEOERhiYTOMXN6uuT28xNqR2UzomozYyoLQRSPpaC86svQlF0Tlo3i0cs1LI8QOToJhGhz9CFMoFCyz+Yo7ZxfcWc5ZrTqss7RtSzxu2EMKFJVJKbOZMlfrxOFgkWIxTMBAFsuE5mYydHaBkTWTOpCKRgVg6Ok31wSbSBNYqRHIv3e1oGdOHzqku6AsXuNuCqxJzIqFpElR0JxwmQqPxlNWbayBpj5xb2mwUmic4l+9cPzogw/4/ntvoydo77zDYfMLTNPS6szESEmWohRJZciKGEFJwuiGZ7R8vAmUO3BrZtxRY82zrxg4JCmCCKbUNLZUBOUsRRJp8Owl84u1kG4Ujw7w4VXLi92KWUr8w7d63jk9kCZNpw2iK3BsXzTGBlax0HGg0wUlHgmFVhyTZDbi6b3msIlshwQC671QgkaniHGRUQw+HqE8Eo8+eSFEzeU68dWVYhgy85kjk4+5LIUsFlWETCJmxRjhuocX+8xuqowFIVdrVyrETHUpWccUCx8/22Ozpk0z5l3mi03hMDr6YctX5RH6jTd4896bfPPh61w9fcRXT78k5JHmkInRopYH/uTnf8gP3vsNvnE656OXA5fe45oZVpmjp77aqHfDnvV6zXa/OfIFHD7kOl2VQox1mmtNA1rQbUV51+guxTjWIKwiEGOmpMonKRlSitVlVCBHRYkaKYbW1Mnu5A+kVGibyihQqKPa/yg3rJsCtBKsnsjmlpCf0/s9Y8hoMSz0XRwtUr4GYlMQEhkfE71PNAthheAkYZuAo/DF3vH7n0S++FzxrXszehv4crOlnwItmZvY1GTcXHAJ1JFY6CfwIaEJtEpx2O/56dhjGsdJ5/juwyV37pxybymcuYGZGzhTM9pTx8cR/rf/yUsePdc8WBVK0uxiTzEnuOmAsplnt4mvnnbsDwdCuGKIlplumGlD0zimPuNDQAoYZysYr9RGFKhOAl1pqEd7FfnIERjGnil4nOtYhwPrQ50elzBxqn7FxYC2lsV8xVoHHtz/BjPn+eT5E+bdKdv9FaI7/JhwtmPwgbtnlvuv32HKPfO2Qbc903iAEjlZnkIWnl/tQY6HjVFkJXRH7nKyFpsKOXo0hRA8hxhApgpXiILVTc0LCIkcR0qw1YrhMtkMqFZjrKJpFKbJXxcCOcVjIVBtJpDqG1A0R1Il+Yh/lHLML1AVYau1PlrcXnl8C3EoxP0aawpn6i5X1y/QXcPy4UOMWdBng7WwMjW7/RATw+FAbiu4JosmbCPb6wklM1qj8GFDcZY09HRqRreacXO74WZzCwQW8xn77Z4SC1I0/X4gjhFdCsoYoj+G5CRDVIk8FDpd1y1iocRIyBmMwaTMzbM1i/sOt2qAxJQyzhpOZ3PeepAxXQ0J0QbW/RUXyzs00uKnA6+9ccGKt3j62SW/cVD89h3DW/eF/tyx1h0/GSJfaMOYR0Lfk8QgKqPFU3Jit1eE5Fl2irunKy4uVrT2BNGKdn4UbhLJxaNUoYhh8BPDNGCUZtktybnh5ebA8/UlQ7rFtiPLppLKfDaoKSKlWt/08S/LEUkpVfVs7JHXZetrUcox0fJEkROMU8Z7jzOBWSfMV0u0taAiB7+mBg5OTH5f6XxBsx4sh1HRdA1OKdJYI6Cj1N2yVoGU12TjOORIiiMrEiUGer/HF2EoI4Jgk+Wf7jX33SknTUto4c3yjJuwQqWaojl5Q6ssabxiN458PCz5bnNFaGo41oP7c5KfCHPDv3qquOElt9vn/Hf+6/8Of/wvb/A3hZmak8ZMso6cKqApq0gJiuAjViUy8Giy3HvvDLPYcPW48E0zkN2KXDTaVa1EVEKWHqQyFmamJaSGTYTbpBk/zXw2Wl4OPT4pZsqyN4Z/9rjww33m27Mdk4y8DA2SZkwp8f4XPcnv2RwSMSlWiw7tIqEkCJoSFUOEddY83Qo3OfLtS8fm4Ll/ATFIpX1SKilX1SS9XOBwEB595Xn/00BrNQ/vTbSLUsmfo6EHUAGS4KOwGzPrvrD1Bp9tHSOXUqdWCFYBoVSrHwUmxcfPE1/0kYsGfIoctgXjVvTXnmdmQ2863jHn3HvtNzi98wYvnn/A7eVjkm/YbQQfL+nTH/ODN3+TO11LZzXrkImpwnVFIAqUaWK727EdewIZkxUm1zWGDwHrDNrVDAtjNCpnChlrbU3Ecxm/j6gsOG0R3dAPA1PMKOtoJKGUMJ9bTlcNd0467p06tHjW0wG76JjPZhhj8FM8YofLsVopSKzUwSI3DNNT+mnCZ0sodcrsVINWLVFUBSpJOtbvkZCFXbKc+CVbvaXg6aTwUit+/4niRx/vCVPg8kV3bCAdbVKkFIhJVcy2ZExRqGIIZKLJkA1m6Oq+3WQMoL1hOxU+Lrd8+/UZDxcX3D85ZZCBD1LgLz7I/Md/PvDlZ5am1+zZ4hI00hKzx9rCHz6By9vCsFaE3KNDfe4G7Wnmirk0FOXZZ89SFscpV4EYUUZjlFTwEEKOYI0jSkC7jpwTfrwml0yyLVfbW1xWuJodTWvaX20xgDtjnx/z4OI1UjPx5ctbht0VV1c7KAYhoc1IOASCKjSrOT84f4+Nf85uSOwON3g70TQWvGI/ZqahEA8TXddidcFIZgiBJljMAYIDrSzeJ1I4Hs6lZjgrMkVVz6lEjY+6AlCcMHeGmVNYHbEuoS1EFUmpkJKilOrRppRjtaqqD5eCZI1TMzq3oLVzWpXoURiraF2msdUCJSgkGUQpnCssmhVmfsIbpw94e3yDz7Y3fHr1mLurGW2ra5pWMVAiKQWKUmQxNLohDhPX+x27ONI6g8+eIolxPzF3J8yWCy6vn9H3Y2XPY5FSVaZuZrn6/AuGm02NwZwCSuVKO5NIRXNBtpmQParoV6U/YqrQqbGG29uJ09fn3LnTEqctr9095c6JIYQdBxKxGJoE1jQ0zRkawZvILJ1w+/OGX/7sI95Twt3feAO5M6MfJkZ3wnV3l30LNHtU0Jj5jDRNiK7FX/CZRjIxZNYRQtnSp8DJcsaJm9FIYTtmJgzWaubdjOAjXz59xuPL5yhnOJ2tSFkYpwPa9MwXCWcLIWXGfcSHmu+tjLBoI8um/nyyMohWlJIwunqPi6lglEo8M7UbKbVIODmS2mI4IPKEXdnRJIP4BDFh6sPIfhy42gljOEXbU+Ztx7JrkBDZS6qukKmnyZkYIIsjMNLkSCiKKIocI9OUmUpiTENd+WBZpo5sr9kguN7wrGlRZUBSh+gZog299By2O9QYKO13eWL/hPuD0HaKE23JzcjPNgOf+8w3H97hJx9+zt9670vuvPt9Lj/7MTF6vDGUJEiqo8lOCdcBjI80s44/vlbchDMuFh2f/Fzx4qsnvP7DJW9rUEyQIaOxydOjkdywsIHrwfIkaS5T4PnYsA6BffAoZVlYDSrSaoVSS/5qt+KL/UNse+AwOlwuhE89j58ndJjDZse9c0VjE60Fi2U6BkddbxTPLws368JmE/n0cWa9cwTfgo7Ekii5omrFKJKdUN7ycgj80SPPj94v3FsmxqR4eFdYdIpgdE2pjELOkSk5Nt6ynjLkzMxlsrFMWteLJNczpqbwQU5V6CUpE8bMV9tKbbRaKHlCl4b9dk93s2HbtkQDs/mKt977Lc7P3+Dpo0/Y3V5TaLjJhT/f/SnvPHiT19/8BptnNyQ9J+IqP5+Kqy5F0R88KUFrKngoHyOkxVpIFRncSLVFGlEkr5ClQo2WWAZUu6CIMPQ9Y4h0psVohTMtVhku5i33zw337oNxE7f7A88OPRbHaxY6K9C26CmhUxXWRiLWtKh5YDOtWe/2iFb4PAIai6LEBmWg5IiIojUN6qiLiApapVnHwI8/HHBh4Afv3uGf/WLDX3x0STMqWjHkvq5+viZZKVAqoo8wI5TCl0KJBeULTRZoLT0V/26rrxZxmo1P/N6HL7gW4fCF4enllt02MfaGUhpUhp41IpmoZ9UNQGIqlsePr0mlui1SThjVoa1BMbHsHKo4xuSxXhAJGNURRl8Fk2VOju4oDExI05HoKyI+Vt1ZiYWgJ4bDQKdWx3wHQBm2h8OvuBgoiW7R4fSc9e1z9lvPbn9JzBbJAb/z+BAJzjLu91y++IrFd36NxdCx379kyqmSswbw1+CvIipolGsIur4v1dqTKwqyCJJqhx5CJIdqY6shVAqfYvWwlkooVMrSzWY0M0fbapwTXGPQth7Q5FghSPxr8ZfI138HQVSolLWSicVQpCFhiKlmVlvjsNZhdUQVw+hHiIFl22GN4jAMPJFrTtsFd5dnDL1jv98Sc2I1W9FZw24YSbmwHyNd0cwbx2FKvLzZgTLEarbgsO85PbngdHnBMIz4ccLaCkbKpbDf71ktVxiBy/1Qvf8otLP4ECpVy9SOtvCKB14jPaUcY5uPrISE0HSKPAW+9XahxFOK2jHSUaxi2TWQR9rckJjhVAM+s30sfPSjF1w/+4gHS4P9wQV+viG3Mz5eaz6eMn/wxcDBe5SzGJUoodCJraSynLHGkHxGKYMoCAG2+4mQjqPAGJiNFtM4nHUMMwg+8vLmwOXVhpgjL+0NXSPM58JJU9cAuw3sBtj3hZQ9yiq6RmMxLK3Bqpptrig1bwBV96U5fx2qIhVx/nWamgKMUqiZIHhENmSEoiIZTwwZUQ6tNc4ZshJmsxltM8MeO89CIIQRYh1NxqNVtZ5VUvPIj5skK7YGmOQKGcklEVKijAbtM0kMuR8Ai8iK4A/0/YQqcHF2zun5GU+efcxfHOAfn2152C444Dlbzvj/fDgxv7uiGUYup55nh2d85/Qhd7/1t3nx0R9xb7agT5mkW2bSs08N++GAz4lPr2d88bLljbfu8+LFFnPb8O077/F4/Yy3ljWozBkhUMjSQBloOTD6E/7FpeF6KqRpVmlquqFpakKcyqkmsSmwZKIkLuOE2mZGelJKPNlN6KIouz2DF1AJN1OoecLozBhgvc28uIqsNwU/1Hf8xWXh8ibXMfCskI9ZtpIKLmtMqnboZ88Ljz4rvHhS2LcFH2Hwitfuwumi/h4ZCNHRB1stlkpX45GtMC0n1Qv/KlTn1efLEbkrpkLLGit1DRMiJeTqjGGiX284tC2zrsWXSGkM3elr/NryAZfPH/H4i/fxYYOfWj6JX3G72fDavTcJUljHQD76zWtwT8FZyyF7DmGkFUdrHaFkptEjaLSyONcSU8TaCeOq6yKUirhOWUhZoZ2mcx0zZxHJmFngzsry7oMT7q1atMlspz1Pr57wwZdf8I2Lt7i4cEw7wTlHQZiCp9WWzjZEMzJN1+yGLTkLJWa0q8FtmgZtOlJRWITGNnTdjCyJYfLYFGg7zZfX8OOf79muR370s6f0vacNGiUZbCXIFoSUYkUya40yGkmFkjIpl4pALn8d+hNTxuSKPUZlSjOhlAerebHO/NM/fIa2oGwFs4lOpGlgGgMEaNwJyUYyCXJBikIrQYV0XDEptCnkMqAs2K6jlAZX5hQ3okpAdgOm1Jj7oiaC9AQdKFpjCDU3JGScEnZph5lBwdGgaIyunAetKKWme/5Ki4HHjy95+95D+sPIy9unOLs8Wj8cRizKaebWsNnucI1h198w3N5wu3/B7XhL52bQC9NVxL/IxI0iRU15tQPO+vjGVH9qSq+CJCJhDMc43/q9BKj65lTvcd00LOdzFsvKEtAmYdqCaaV+UCWQSwXLHHVjX+sFXnEDKBBKIcZI8huuNpdsz87RTUFpS9t2uKbHWoM2huAzJ8sl3//2r3E6W9GnkdV8zkwc9BNYx+LiDnbsOEwjjakya6MNwxSYYqJpYRhHrtYHRl9pUzlHSlGcnZxxfnaX3e1IzJ6u60gpEVJFXsznM5azOY8//ZT14UAIiUCiOdqNhBoQphIUd7zZKNT07VolS6nxzamMZK8JO8NJ+4Db65HmTFOkZTsErr+y9IPCmLqvdZvE5pMdL7/aE68mHlwsWInherfnuizgZeZJnPFnN5HHu0KjOjKhEh+P/HbMK46WoJuKE63fdd03p6lwKCMpZNKsofQBSo/We3IqbHZDFePZGpIjGpQWQoRxTAw+0weIWdC6q5dqrAeAOiJ0nQWja9KfFqGkQi4FjKCMgBRKApJ8HaYCNayrSCamiZh0JZ1JJGXQ2eH0guVsgSsrnOkwxpBS7VBi8IzjSCTjXIvSFq2P4Umv6u4CKUZ01KAzRkFWNVce8eTo6761WLxUpX8Z15x2p/z6N97Gtg3Xfsf7X/6cMF6TmsJPhpbffLPh7LTwkw96/g+/t+Fs6XFtQrLhrz78Cd88W/Kt7/xjbp88Zr/5FJZL9HbDvl3wkX+TF/2G9TDx1Xri9ZOITiOp03B2wAr89OYuF6dX/PB0y0BbOepZU4rCuAVfbIQPXhZmbakrQEnYo2VsShM5xto153IMLcsQBhYlc95odpNiHAyHQ2LaCkMPIhnb1L12ZxWbLVy+FC6vC8Mk9blPmuu15tmN4jBVUWNS1d8upopRS1asJ8+nLyIvbxQpCZsdTF9ENn3i9qB456Hm4qRgrSBGoYpGWY2V2oEnlUmlTuNUEUTbr4uBnKs4D2oRDpVaWHkwptL3lKHERL8f2G7W6Kbh5M4KZQQfwCrLa299h8VyxYcf/iVPrzesYuQwHMjR8813v082heu+JxQFRtN07dd2YaNNXXGWeNQKGFKIxJDww4T3da0WcgERrFN00iAhk6aebrZkdTLnztkSDRgtnJw57t8/p7MzvJ8wRbFqz7lodyznJ/gkVQWfMzu/5TAcOGtWnMwdygWu11d436NVW8/plJFcsLbDqJYsjtZY5s6hrGUzjYi1NM2cj66e8ac/+Yw4BKwYbvaZzli0NYSSkVw/4xIzJdWgpJrdQM02qG/O1/T+Iqqq9EslcWqrKQoSmeILThSmEXRbaMoMXQR8IZXabmlrQcBTIFfBZXUCVJx3Ou75G1Ot4ihFRrjaZCYf8UnQpsOIpVlUGu7CNTD0SMlIUlVPJx585mQeETNxNXiKmqPKhDMZVEJyBTSVUsXkv9JiQOkDRc4gR4iFg78il/brtKVMwU89KmaSKbRty7DZsJsOKGtpRsfh1jPcCNOoycpSTEYn0ElIqR62uqQa7pEglppe5X0kkTFiyTkfu7dqT2k6w2zeMp8ZXCs4l7FtxjQZsZlIJKVMjlTbUS5VSZwLMddQCUGgCLFkUiocYs/t7orDsOWim2FEE0P9NSnVhKnD6DldNLz3zXc4O2ng9ppEfQGxpl54fiIPntvbNYtFw3I+Y3OY6PtAmAb0akZOhpc3B3IWUqyc/7v37pF8Yn27xQ+QZKpJdSFQ6t1EN1ty+fQJl4+foKcK90ix7uRcZ4m5jrnEviqyahkgqgaHKKWqgA6FxTCVQncq2FPQ5ZovPil88NMbxkHYDSONfY2oN3zz7QWPfv8xzbViuUrcuz+jMQFdhGGn+Oxx5tOTzJfpwNOQocSKeqaG3mgFyhrQlRNe38KaqCblKIxJmZwCQx/wU2S3H2mahqa1lDIS4sQY9xiTj+EotS7ce8XBQ07HeI4SUZSKo9YRZxRdqzGuZqYbY3BOY1Tlf1fcoq5IXlPpX6Kpz2OBEFJNv1TVN51zIab6wusCJdVgEdMIzlgKtbMPYWScBsahZ7u/ZbNfM7OOhe1qLryoCghRFZBFKYQMEkP1HWuhGI2R+nlBZf7r4sjekDIkn7nNPQ+MIUrml598xJOXL1gthU++mDNYw//ib8+ITPyP/q9rtjewkz3JCZ0U/sX2hv/GPzhlSPBb/+C/yx/9x/8bTvYjw+nrDItfZ3r2BXu/x7Tv8MM3WkJZ89IcmEvH1MDLW6FPhf/8E/jO71iKH1GtoKaJmApZGX65bzG5wZRCTh6ZzzFpT5gKPgR8GuqhF2AOvGaEN8+EO9YzjfB453h07bl9mRhGwSeQBKjM6IW2gfU68/R5YXcQSql2YVRiM8CTm8y217SLWtSWkihSBWlTLDwbEo82gUPKiKqQm3EsPHsG/VjYHwpvPVQ8uKtYLWFlM14Z/GSJQZMFCp5ylHVxvFTrRVPjZ0FqrkUuFJVxSld0fD42PhHSNHJ9u6O0LdJB0S3WakLRHAbL/PQeP/ydf4sPP/wpl08+p4TA6AcOJfPu29/hrFEcfGAbIlMIiFJ0pq5GC/VdScBisaBtO4YgaOvQJTGMW0pUMNbOSUQjGRqtOVvOWJ63vPHaXVpnUb6gXA2R22dFoqFtG957cMKd2UNEa4pSTHGi4Otkb9jhpz2pzMhxS5r2FdXNMUgrB4xYRFu0GKxrsM6SRUgkJkl8+PhL1tuJl1dbHn/5jLldoFXBvXovS0UR51xIMdbQJOpqt5RamNXY39qAlCLHwrV+PkopohpIKqGUwYjDoCBkYlLY1tW1swgkRc7V3ZZTrkJkyahSJwvH1ouQMzEnQNUzWmeUsUyxsN0PIBAIlBF0FnrXQa4ibjcmZlajtQPVQh45WfYsVgu+uvIcRgENVnO0VzukVGtszq+m4L/CYoDmU3xuGYY9phh24xYfU2UOjAdyD0IgNAqrWnJM3OaehZsTQ2HaQJ4MKifEQDG+0rxy9fWWWAsCnXNla6t0VLEWYkwUU8McROk6bs0Z1zpm84auU5imoG1ENQXtCmLrRCCk6UjRUl8/BCkXUq7FAXDMVag4TY66gRQtpLqbL8f9klEGdWRsK2uYnyy42W0Qu6KzCwIJXzJjGrHKMLctu5I4jCNuYTnkxIvbA+PgWS4MRsGLFwcOvWcYPcYYzk9POPSHygwoQtGlhht5j3G2xmlaSxw9Tx9/RQ5VmKNFI0aTyVhdU8yqGDKRjmpU0UfQkn6FCK2q6ExNFLv/jhaf3N8AABgxSURBVOHTxxv+6k8SX74PbeOZuTkndxuK3zFzgXgIGAyLC+HuaWKmPUvXMbfgQuLmyRpxLTdpzz4I/Xasa1Orj0UItDXknlQyxhmkOHJMxFgLGJ/Ah6O0QQwzu6wQl5KYwoGQBooEjKnTjpgLJSWSJEQZtFiUsvVlzwmlNLNOODu3nJ855q6+OErq2kV0glQPjFIKORZyqFOColJ9mXItVLRUdwml5pUMRujHyHTMvCeMNDIirmpDfKrCrl2/Y7e7Zbtf0w8HmK3qJEEXGlEUZ5lKnVqk42hHiyGXhCqaQkErC1nX4JVkUcmhsiNNEIbE7TTxH370L9j1Q0W2asU2Cp9ev+B//O+8gXkN/qN/kvnpC8WDM8s0jkipPvA//dlT3v/iCT/81pKo3+Y3/s3/Pj/+f/3vOfnmO1wOt8T4nMY6fvbFz3kwtxh7yqJ7l9vNhnwwXIeIkZHnQ8cffmn5L759w82gMGQa17CZAh+t79HNXFWSm4QBhuNYuBSFxIhMkU403zwRfv0E3rywIJZPn3umwXPz0hN2BV0MKcLmSDNd7xXGZPa7wvVtwftSo5RLjV3e7TOfPe15emM4ObFoqYFIEVV5/fvEVy8CVzeRFF918AUEshduL2E4RDZ7xX4yvP5AMZ8fU+ZKQHLE2uoPj7nCYmp2wasI3vpnBFUnYjljtaPRmsYaNDCOgdEXYhKGYeTm5boKnIumnUUWc4OowjBkGrfiB9/5IV+ahi8++4CYCl8+fcH6esu333yX1WyG1y2XVlGOgWSuaMaSmXy1GU7TSD/0KOsQHSlxohyR8vOmxcmSzTiSpoC2DacXZ5iFYkp1mrDsFOglIRZyGpEcwTmaznKvO4eYyCSutj2HOJKnkVnRnJ4sSSZwtX5GTDWfwPuxWjnRaKNpjGbWdZi2qyFQR9ywE8t+iPzZTz9mXmYs7YopFbyv+QbZaFql0AhBpE6bqF02qq5Mc87HQoDjHrAglZv8tabCSlsLzVAwDlQ71XtFHb3Zx1jFQj6G4NWJM7z6Gnz9a2Kuz2E1p9fpUYgCJYDU5FwRKKk2TI1qUEoTSITsQSas0kxpj+QJndaYpXC177jcg2kEJYkYFD5BUaE6EF45qP+G1cDfuBhYndZRx3J+hzg9px88oi2zdsVuvGJxumLYvaTBINpSUJwvTrh9eUmYCqU0QESbjAsJL1UcFckUqUrWHCIlWKJONddZShX9xQSiaugKBZSibSzzhWU217SdxrmEaxTWFdD1hxiiJ9RW7phu9WoqIMf4x1eQJDmOzQ2CoMXizAxrFygaSl4fK8xEYzVWa8ZpopnPmJ3MMbbu2w7Z49CciGOInttxw2YcqjHWOnZToJ/qhXe6XDKMgSeXa0IsaFE8uP8A27i6t5oyPnqKVNGSfqUIyQWD4svPP8cfBlKMtFJ3kEqpyk0gIY0CdFUNaxCpWFc0NTyE+nctCu1mLJaeX/5kz/YPEkYMi0Udk6EjadA0s8gAaFnRtTcsmsSys5y5OacaLk6E865ldrEguMKCzGcePp4yt/2EdaamP85anJnV0WcOIIUUAlOciJMHLMa0WDtDacE4i3VzkJFSdvWANVWcBuXYqR/jqXNVKCNTfbJVteVYlVk1irNWszDQSEHHgoqRcSz4HCFBiYqUaqBLjPFYIB51FhS0CFqpCieyx2AbW90lr7QnKUXG8YjUTZr9GNkNPbthz2G3wceRXDJalSqIbDUL3TE6oR8Vm5wZxprGpo6wlqKrpVFlVy+3UCBqJDbY1OAiuCTMFKiZY2ZHdvsD3nsmM3Fuz/hv/44irQv/83++Znmi8WFCcULf33J6coqSwE8/ueW//Hd/gw8++Ih73/gB3/2v/A94/6s/ZtqNmOYhTu05nfe0d+6SfGBc77iaBnQswETGsOga/vzLkd+8t+BURrZJWLbwo2cNOQrORLK2OFFMeaJ4R0iZHDJ2m3C3iTMFD5zmjspMPvDkNvGLL4WPHyc2+/o517OgjnR3h8zB5zq98YUY/zq2NR8z7qYJHr2IfPp84o371QcuSvAFei+83MGz58LuVsi+VPGdKMgZfRQZjbvEVz5xGDLXW8vdu4I7MYhTLGeFttOg6xljcCiT0Kq+l7U/++vLIef6ezRa0zUOkcLoPcOYGfrCYe8pvae/PtC4BVrPGDWoWaE1CskjUQlvfvt7LM7O+egXf8Vht8fHHvviMb/x3rd5++yET7/8FEke0UI/1UlI6xpMLITkgWqFNa/2igUaY8m5sKe+R6o1eAKbaUtOgX6yR1BYS2NbjKhjZK6vtsMM2gi6FEIMtI2mGI0yhmYxw8uOZzc3DGGPKPmaQ0HOaLG0Yli0c3SjySnUUWapkb4qgEuORjkkVRCRlVwjtJUiGkNMsTZ/UHUaor4+5kU0CjnmSRy3p1LXSYXjmXF0mVitq35C5VekuSNMLKOKQspf689iTiSqK0WUIo6hTqRzhQaJHDt30UAmhYZMwtgKFApTBjSNc3TimAgYMuqYWlgSTNsN1lq0CuzH+3y1PdCXQGMiacrk3FR7uIv4qSCiKEeU8a+0GFDZ0s3vMewOTFONvB195LSzzJtTrg8bjECbFNviWWqYrqvdT3gVq5iOCVSpivmyQon5a6xnSjW3XteQ1aKE9KqsCYViajFgG0u3aOnmGtdk2q5gbcG4DFJhMynnGkOcLOm4bkhJHdcFdSeYE1Cq51xJoeRj5rWp496YMyHBFAOFTNe0NNpgRGisrtVY9CjmxBjriMiYr4lmShTFH4ip8PJqyziOHPotd+6cEYvh+fWaw+hRzvDu6++QU+IwBEoxhCnWyOEYKD4jWvAxMmtadtc3vPjyCZ11IIZRIjkl7LESlFShISlXDKcYaga9SHVSSgGt0EajraHQV01FaLk41YR4ADIi1WLT5ZZxGDm7exdzM3CWA/dbx2vzhjsd3F+13DkT7q3mNDNH7uAiKnTqeLHZ0OvKTUcXSlTEANposkQGPzAOa0KcSCnS6A7nHLOmqTtOBRN7cjyQwo5cRrKqO+Lj1peSM1obrAhKF4zKKJNRph4QFkWnBLzQr2Efc7U6TUNlyRdVR/xZkbIQU3WZ5HRcIolBJKJUQqnqSW9boWkVi5nC2EyS6sMPMZGGgJogcGC9j9wc9kzek1IEMs4ZnBVaJ5wtZzSlY5gabvdCGCdSjMSjfELRYDAUCoa2FrF+JKcKHZEpg3YUDbQGqyJq3zNfVpHhs8ue/+Hvzjj7O/C//F894cleeNjN6OPEpLe89cY9Njdbzs8Mf/Thz/jvTX+X7hw+ff4B+3GPHk84bZY8GR9z1W8oQbFe33C6vEPKM9K4JlFYlAVJDTB6iiv8yaOGf/vbnpUf+eyq8NH1Kae6EKNiyj1TFoJxtFNgCgG9CcxfFNoNWJu5bRUfaiEazecvE59+BZuNqWG3ppCTPjpGCiUJfioVZV3qRKDmTRSkVEZ9KcKTq8jPPo98682MswpnCjsf+WpbeP9L+PmX8Pxa6Pt6XihRR197qkE6QJo0Vy9qwTYmxT3nuHvWcrpyLBYN2lUljMES8NT51r+mEzgKUmPmiHmuzHyAJhq6JjPZxMJZpDRgGsJ2Ry8KU1rGnDELqj1YVZfM8t5DvpELH/7irzj0Wx7HW9ruCe+8bnjr4gSH4cuXW67jQMmFklO9mJ3BhwkfJgITOdZnM/SZHHMlVh6zBWI44KeOEEZK0Bh1TrO8A7EQ1Vijg0vBlOO+PAg+RMRYWqPB11TUKW1Y76/xZaoJgiRyLiixlfgogjWOmVuibbVIN9oSYmJIgethx5Onz7AZtMr4JJSk0BiMrtoIyXUXX1kwdSKKPpoHpLL6U0l1AncUkeeSyKVmZ1irKEVTbCGbCZGCFAfxGKFu6mAh5lL/odRVipPqvsmTx2iDshVzX0otLsTI11MiZ1UN/ZKCJMEYhXX1/AphIBmNxlCyB+NwdgHFEQsUGvo+MPlA5yxGOaIbMarGNfuxOqSQf00T9ystBlRkmiI5ZxazC9y0ZQgaGGiawj1Zosqc9b4g475WxIsFi2wY9xtiibROoFM1mrbUHY7KxxkQx11OEkhVECZK6kt0XApHnxGrvxbCiAFzBAtZB6hq26gXvxCDEAPEJKRsa/eXpfIFKreTo9QORNCS0Qq0CDlbfExENaBMPeiFcszMrnvc0+UKyTD4gVg8Nmdmxw1YEsV6HLi8XjPpOvZzSnHndAWS+eLJc756fotSMx48eA0fPGGKKO0Q7ZAo9Pt9zS/Xum7cRZACzx8/oclCmQJjDMRU2e+1+tQ4bQlEvI8sFnOKm+qLcHQPcDyiUk6kmLFlTvUURELIWOeIXqGNQ2lPbwcoGac8sltz0XZ0nWXWCW9cLHjzfMbDe5rVaYNeNiQL40sPo8ItZji1OY7QfI2hTZ7UF6bYE8qElBFn6yqoyMgke0pKkEz9TvUBokcRMVahlaps7lLTzlxJFFWJjEXqFMVgYSoUKUwSuZky6/VEioKPmilIHcmGXAtTMXV/LPrIqTfVdSGRIr52CkL1+2qh2VdGwEwpZjOFaktdg0RDCkCIhLJnGCb82JPJdVQJWCMIEU1k3jhaqWO+EBv61uFjtb3Vz0rXcWMBsORUjgFaAjnjlKEAzrWkybOUFnEnXA5birXMmsgPf13xf/6/bPk//lXk4eoEKRusbnnt3gm0mc4kbJhx9fQJcbvmoyeK/9N/9Hv8rYeJ9769IPSFcnDgFVGOPuh+wHcTKxr2amQqW1xaktpIcoqfhkLz6Izfvf+S33/5Gr1p0JMwGA1xJOqCioG9BMIh0L4YaV/W6NaBwrPLxPMboS+Jl6OwGdTXufWp1Bz3HBNZBFGV6HhkhuFTpqDr+SHpeOEortbw/meJR+9GTjrLooGbXeKnX0783k9HPv8Kbq80h8nWfy/7OjTUQpJSO1M0uQjTmInJonXHcr7g3vmC01WHcokkIzF6ptyQc0TKq+lABe/E2izis4csFdqTqiDPOGExr3vqGtZV0xAn7wl7UHTsVKRZKE6Uq7kIObG8uMsPfvO3+Ogv/gI/HPj8yxd8df2Mv/utv8W7d97GyheM4TkFhbMa11nun91jpucM15d1slit9/V8A5zp2PuAhESnO85mK5J0LNqO1+69xqxbMExrMhGtGnJU5BKZYqyrXImU1ODEMDMzduOGq92mivuSxelCIJKD1Hjl43SiaIMvBpMKjevq2R8Tj1484Rdffco69hQ0IUKgFm1O1byAXASlbG32C5UhoisKOR1DkRSqSm90vQ/q/VwplI1WdY0pnokA1qKVI48FlTOm0bWpHWsRWsNuU90RyNG5IAqVwvGe0HCMYK/ZDBlEMKqAWHIEssNaoZiIzxGrWzoCzpn6/CnFaJak6HGmYNyczbTjpJ1jRYgKel+bAwIUYk03PJ73Vv+KUwuj7HHlIwKvM2TPyjaMkgnFsmoL67SlRNj3deEbR3Bnc5KJ0BXsWDsuhVQkpRZSOcYfI5V9DnUlEHUleFGOQrPjnh+FURo5jmY1oNBosWjtaxEQFDlaUoDo68g3xEzMihwVgiUXTSqqxmcWhRaLUQatE1oLTixzfUpjT+jMjJBHRrnCqIBBarrcfMb9uxcYEfrDAecci/kKTcHnwK4fefT8kk2KhKwZc6azGiWW55e3XL/Y0dgFDx+8yTRNBKVAO1IopDRUAaCpIRWjyliEThvSODD1Q31xRDBKo1NCGtCdQKjCwELEWUWxsY5aqAfnsb+pfuc6fyHJUD8XrUDX5K6m1TX0RwwLNLLQLPpCt+05OZ+zMJmLWcvr9xRvvL7gtdeWuFZQSbgqQq8dX05rXvpbkt7TAF4DRdH7jBgBBZo6Lq/d9ytFfY+UocJbAK0UyoJRtlIHcwWovHq1yTWVrrIDShWdlnQcWR5z3GMmhDrCLaVGrOacj6IuVadVOr6aG9agD+HYZRaUqrhbXh2WWphZizfQbzKyKxibaHQlGOoUcWhmxRBtQ5S6GjCm6hScEbqFY9E4FJ69HzA5YNVR7R2EJK7qEGgoWvDKUFJGsiNPiSSKLBW9HX1NOYuiyMoxa1aUYc+DkzP+Z/+3KyblOW0d3377gpcbGE8CnChuN2vePrvg8uoKd3Kf/+SXW/7Tf/bPefO+5r/5X/1H/Nn7P2LaVny3mjtk25Nj5s5rrxPNikfjRDNW3UkyE045TDHEkvlgXfhg/y6IIZfAWCJhGOp7XRRJEiRNMZ5ppbkdMmaTsFHoh8r9OBRFHysjpFAT7NSRC5Kl6gEKqnbJKR/3o+r47NT/nqhiVZUUnz8u/OH7BdV4lvPIhy8K/+mPIj/9QDH4quZXsY5nMxx5JMBxVFuUpzuBi4cnvP7eKed3T9GmIYkmm4SxipAsXkEKHlUsSqljblwVj4lUBTriELHU3iQjwNzMMaKRVoCqixiniqAuUn9mOjsaXwtlayw+Z7b7nhQazt/8Ic+ffMb+cMksFf7qi095983IvQuFzktU29HNlyxmLcvlnJtD4tlN7axLHNGxEHWmaedMWaNLpliNmzlW8xnZZZauQynD5HusaikUSkmVE5AC6Wj7akxXRXu6cB0esz48R1LAFkUgVFvf8aKMgFIag0ZlSysnOLOgHDv2rDOexMsXgX7j0aKJZGwpaKlaMsRUeW39oOrZIFJTK/+1dXCosUlH4Xg6bgA0VimstgTqxW61hSLkcAxcMsepYZCvZQMpl2pxV4lcPMZYjAi5uKOAsF6yrhR80aA0DUfibakNb3bVidRoc2RECMo0FOUwZobkAOFA52YY29ZUSTvDl8zWe2Jf1wB1XVkXF6pAEqmwvPg3mw78zQWEAkYbZrMZMTZsNoEYS1Veq0pTysIRBlT3K+lrj/T/35eSVwetfP3PUMcpHAUcVV173MiIHHfelQCojcLaqgRvGoXRlhSqCDCF+uKkcBSCRcghk14VA8WS0XWXmDVSNEXXmazoV6ulWsVTFCK6Wr9ULUCM1eScOb9zzmKxQOvqpNBaVSZCyeyGkaeX11xvesasOAwHfJgYtWYTt7zcblnNTnhw/yE++uPPQtXVRAwV/flK/6rr3jhNE4vZkqcvnhP9VNGgxtSJv7VHRVwtnqy1aF1I5ZW/+fgBvhIyyatquBZc8vVeTb6+DF99HkVBtIl5ZzkrltPFjPNFy6xRnN2ZcfLwhNUDi8w1L0Ph8c2e929u+dmzLZ8NHm0NbTa0qorlHIpURft1Z1fK1/tdebXEO1pMj5NHoo9HUln5Ohu9lKOlp0BJtaOUnMlHF2U6fu1Xr0E5PndyvNBfaShe+b/L8Vflkqup4Phd5Vc/Ejg+5/VY+f+2d++6TQRRGID/M7OztpMYh4sIJGAlNIjXoKPklaFBoqGmQKKhSLiY2LHXc6U4s3Y6RBWJ/b8mUrSxrOzO7Jk5Z2YKSn0RiQ7csxatQbLOHrQCcQ2mIwdpDDyijjycg4jFdDTFvdEUrWtrWkfq9zM66q0/fUq1HELbWYm6zXEIASIZyepIWbKpR5Xq52hbcZDc4dGDGbYuYJIjZrMpvq9/YdKOsVguMBaLVQ6Yn55hPn+Fdx8/IMaIt6/foItLGGPqyMJrENbo1qkxRfxeLXS0ruc/wxjZtVHTz5BB4GNC5zv4bT3nvqm3GTrVLa6BGWXA6aFI2es599noMlTtR/pnErtntmS9sQX7+9xfU0qBZMDE2p8YQYHg24+A959WWCwbTNqEr1cJn78UrBcGkFCrwIzukikZfTMRyYCzODqe4Nn5fbx4+RxnFydwI4sQrtFtV1iugNZb+OzRhQ6jxkFs/yLaP9r9s6jpAc1nGzFw0uCgneixvy10mt1ZHNpGV6ag1gQVq+vWYeG3erBXiQWbGw8jFo8fPsG1TVitLnF1+ROpCzg/neP06Qmag0McHc+0hxODdQpwjdVd+IymWWIIulWxdRAptQhNZzdMo20g56iHfRmdXe2r5vv7pC8mncmJ0aPbrBGihzOay5bcp0x0hm+3sKhof2+N0ZUPMSGFjPV2g5ubNVLNf/fXafZzn/PX/uFWo68PRLm1dHf3u7J//5jaZjReKLX4rl9+vv+sAi1e16mFem1tc1a0UDHnDB/q30JqsHTr2YT2F32dkYjAWoFzbve/y0goWVPnTgTt2GE8miAmYNNtdOM6AEWL4Hb9Ut/P9/PtJmM3qPobKf+SVCAiIqL/jrnrL0BERER3i8EAERHRwDEYICIiGjgGA0RERAPHYICIiGjgGAwQERENHIMBIiKigWMwQERENHAMBoiIiAbuD+myrqAl+zEsAAAAAElFTkSuQmCC", + "text/plain": [ + "
    " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "IMG_WIDTH = 128\n", + "IMG_URL = \"https://storage.googleapis.com/apache-beam-ml/testing/inputs/vertex_images/sunflowers/1008566138_6927679c8a.jpg\"\n", + "\n", + "def download_image(image_url: str) -> bytes:\n", + " return requests.get(image_url).content\n", + "\n", + "def preprocess_image(data: bytes) -> List[float]:\n", + " \"\"\"Preprocess the image, resize it, and normalize it, and then\n", + " convert it to a list.\n", + " \"\"\"\n", + " image = tf.io.decode_jpeg(data, channels=3)\n", + " image = tf.image.resize_with_pad(image, IMG_WIDTH, IMG_WIDTH)\n", + " image = image / 255\n", + " return image.numpy().tolist()\n", + "\n", + "img = download_image(IMG_URL)\n", + "image = Image.open(BytesIO(img)).convert('RGB')\n", + "fig = plt.figure()\n", + "plt.axis('off')\n", + "plt.imshow(image)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0a1zerXycQ0z" + }, + "source": [ + "## Run the pipeline\n", + "Use the following code to run the pipeline. The pipeline gets an image classification and a probability from the Vertex AI endpoint." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 245 + }, + "id": "St07XoibcQSb", + "outputId": "c53da56d-3b09-4f8e-8fe6-2cc12c9008a9" + }, + "outputs": [ + { + "data": { + "application/javascript": "\n if (typeof window.interactive_beam_jquery == 'undefined') {\n var jqueryScript = document.createElement('script');\n jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n jqueryScript.type = 'text/javascript';\n jqueryScript.onload = function() {\n var datatableScript = document.createElement('script');\n datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n datatableScript.type = 'text/javascript';\n datatableScript.onload = function() {\n window.interactive_beam_jquery = jQuery.noConflict(true);\n window.interactive_beam_jquery(document).ready(function($){\n \n });\n }\n document.head.appendChild(datatableScript);\n };\n document.head.appendChild(jqueryScript);\n } else {\n window.interactive_beam_jquery(document).ready(function($){\n \n });\n }" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sunflowers (0.993382215)\n" + ] + } + ], + "source": [ + "# Column labels for the output probabilities.\n", + "COLUMNS = ['dandelion', 'daisy', 'tulips', 'sunflowers', 'roses']\n", + "\n", + "class PostProcessor(beam.DoFn):\n", + " def process(self, element: PredictionResult) -> Iterable[str]:\n", + " prediction_vals = element.inference\n", + " index = prediction_vals.index(max(prediction_vals))\n", + " yield str(COLUMNS[index]) + \" (\" + str(max(prediction_vals)) + \")\"\n", + "\n", + "model_handler = VertexAIModelHandlerJSON(endpoint_id=endpoint_id, project=project, location=location).with_preprocess_fn(preprocess_image).with_preprocess_fn(download_image)\n", + "\n", + "with beam.Pipeline() as p:\n", + " _ = (p | beam.Create([IMG_URL])\n", + " | RunInference(model_handler)\n", + " | beam.ParDo(PostProcessor())\n", + " | beam.Map(print)\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tRLArcjOcYuO" + }, + "source": [ + "## Use a keyed model handler\n", + "To use a keyed model handler, use `KeyedModelHandler` with Vertex AI by using `VertexAIModelHandlerJSON`.\n", + "\n", + "By default, the `ModelHandler` does not expect a key.\n", + "\n", + "* If you know that keys are associated with your examples, use `beam.KeyedModelHandler` to wrap the model handler.\n", + "* If you don't know whether keys are associated with your examples, use `beam.MaybeKeyedModelHandler`." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "P6l9RwL2cAW3", + "outputId": "a084b3b8-b78a-4f97-de8d-f7224c06dbb8" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "https://storage.googleapis.com/apache-beam-ml/testing/inputs/vertex_images/sunflowers/1008566138_6927679c8a.jpg: sunflowers (0.993382215)\n" + ] + } + ], + "source": [ + "class PostProcessorKeyed(beam.DoFn):\n", + " def process(self, element: Tuple[str, PredictionResult]) -> Iterable[str]:\n", + " img_name, prediction_result = element\n", + " prediction_vals = prediction_result.inference\n", + " index = prediction_vals.index(max(prediction_vals))\n", + " yield img_name + \": \" + str(COLUMNS[index]) + \" (\" + str(\n", + " max(prediction_vals)) + \")\"\n", + "\n", + "keyed_model_handler = KeyedModelHandler(VertexAIModelHandlerJSON(endpoint_id=endpoint_id, project=project, location=location))\n", + "with beam.Pipeline() as p:\n", + " _ = (p | 'CreateExamples' >> beam.Create([IMG_URL])\n", + " | beam.Map(lambda img_name: (img_name, download_image(img_name)))\n", + " | beam.MapTuple(lambda img_name, img: (img_name, preprocess_image(img)))\n", + " | RunInference(keyed_model_handler)\n", + " | beam.ParDo(PostProcessorKeyed())\n", + " | beam.Map(print)\n", + " )" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/examples/notebooks/beam-ml/run_inference_with_tensorflow_hub.ipynb b/examples/notebooks/beam-ml/run_inference_with_tensorflow_hub.ipynb index 63f1f7ccd02d3..b396851f9dcc2 100644 --- a/examples/notebooks/beam-ml/run_inference_with_tensorflow_hub.ipynb +++ b/examples/notebooks/beam-ml/run_inference_with_tensorflow_hub.ipynb @@ -40,7 +40,7 @@ "metadata": { "id": "fFjof1NgAJwu" }, - "execution_count": 1, + "execution_count": null, "outputs": [] }, { @@ -104,7 +104,10 @@ "cell_type": "markdown", "source": [ "## Use TensorFlow Hub's trained model URL\n", - "To use TensorFlow Hub's trained model URL, pass the model URL to the `model_uri` field of `TFModelHandler` class." + "To use TensorFlow Hub's trained model URL, pass the model URL to the `model_uri` field of `TFModelHandler` class.\n", + "\n", + "**Note:** Only models that you save in the [TF2 format](https://www.tensorflow.org/tutorials/keras/save_and_load#save_the_entire_model) are compatible with `TFModelHandler`.\n", + "To see TF2 models, go to the [TF2 section of the TensorFlow Hub](https://tfhub.dev/s?subtype=module,placeholder&tf-version=tf2)." ], "metadata": { "id": "dVLTtFxDuJT_" @@ -120,7 +123,7 @@ "metadata": { "id": "H4-ZvkcTv7MO" }, - "execution_count": 3, + "execution_count": null, "outputs": [] }, { @@ -132,7 +135,7 @@ "metadata": { "id": "n3M6FNaUwBbl" }, - "execution_count": 4, + "execution_count": null, "outputs": [] }, { @@ -154,7 +157,7 @@ "id": "q23ip_HkwL3G", "outputId": "051bfa77-ce1f-4ee2-abcd-26ae7f037180" }, - "execution_count": 5, + "execution_count": null, "outputs": [ { "output_type": "stream", @@ -187,7 +190,7 @@ "metadata": { "id": "QLFCEisBwPlz" }, - "execution_count": 6, + "execution_count": null, "outputs": [] }, { @@ -230,15 +233,8 @@ "id": "MSKMc_s6wSUH", "outputId": "73695bae-75ae-4f02-a0a5-750531b8f90b" }, - "execution_count": 8, + "execution_count": null, "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "WARNING:tensorflow:SavedModel saved prior to TF 2.5 detected when loading Keras model. Please ensure that you are saving the model with model.save() or tf.keras.models.save_model(), *NOT* tf.saved_model.save(). To confirm, there should be a file named \"keras_metadata.pb\" in the SavedModel directory.\n" - ] - }, { "output_type": "stream", "name": "stdout", @@ -249,4 +245,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/examples/notebooks/beam-ml/speech_emotion_tensorflow.ipynb b/examples/notebooks/beam-ml/speech_emotion_tensorflow.ipynb new file mode 100644 index 0000000000000..098cb150bfd21 --- /dev/null +++ b/examples/notebooks/beam-ml/speech_emotion_tensorflow.ipynb @@ -0,0 +1,2159 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kNv8XQ6-TM7W" + }, + "outputs": [], + "source": [ + "# @title ###### Licensed to the Apache Software Foundation (ASF), Version 2.0 (the \"License\")\n", + "\n", + "# Licensed to the Apache Software Foundation (ASF) under one\n", + "# or more contributor license agreements. See the NOTICE file\n", + "# distributed with this work for additional information\n", + "# regarding copyright ownership. The ASF licenses this file\n", + "# to you under the Apache License, Version 2.0 (the\n", + "# \"License\"); you may not use this file except in compliance\n", + "# with the License. You may obtain a copy of the License at\n", + "#\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing,\n", + "# software distributed under the License is distributed on an\n", + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n", + "# KIND, either express or implied. See the License for the\n", + "# specific language governing permissions and limitations\n", + "# under the License" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pjsWzeiITPd6" + }, + "source": [ + "# Speech Emotion Recognition using Apache Beam\n", + "\n", + "\n", + " \n", + " \n", + "
    \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "h3nLfqdsTZdZ" + }, + "source": [ + "Speech Emotion Classification is a machine learning technique that deciphers emotions from audio data. It involves data augmentation, feature extraction, preprocessing and training an appropriate model. For structured workflow design, Apache Beam is a suitable tool. This notebook showcases Apache Beam's use in speech emotion classification and achieves the following:\n", + "\n", + "* Imports and processes the CREMA-D dataset for speech emotion analysis.\n", + "* Perform various data augmentation and feature extraction techniques using the [Librosa](https://librosa.org/doc/latest/index.html) library.\n", + "* Develops a TensorFlow model to classify emotions.\n", + "* Stores the trained model.\n", + "* Constructs a Beam pipeline that:\n", + " * Creates a PCollection of audio samples.\n", + " * Applies preprocessing transforms.\n", + " * Utilizes the trained model to predict emotions.\n", + " * Stores the emotion predictions.\n", + "\n", + "For more insights into leveraging Apache Beam for machine learning pipelines, explore [AI/ML Pipelines using Beam](https://beam.apache.org/documentation/ml/overview/)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "s9xU1ws-DZwp" + }, + "source": [ + "## Installing Apache Beam" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "RbByEyZPMgbw", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "60db9d24-a8f5-4c40-dd6e-66878b7ce76b" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.6/14.6 MB\u001b[0m \u001b[31m89.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m89.7/89.7 kB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.9/139.9 kB\u001b[0m \u001b[31m17.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m152.0/152.0 kB\u001b[0m \u001b[31m19.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.7/2.7 MB\u001b[0m \u001b[31m92.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.4/43.4 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m671.3/671.3 kB\u001b[0m \u001b[31m57.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.7/2.7 MB\u001b[0m \u001b[31m105.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m300.4/300.4 kB\u001b[0m \u001b[31m32.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Building wheel for crcmod (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Building wheel for dill (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Building wheel for hdfs (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Building wheel for docopt (setup.py) ... \u001b[?25l\u001b[?25hdone\n" + ] + } + ], + "source": [ + "!pip install apache_beam --quiet" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dNg2hWBqrGwz" + }, + "source": [ + "\n", + "## Importing necessary libraries\n", + "\n", + "Here is a brief overview of the libraries imported:\n", + "* **[os](https://docs.python.org/3/library/os.html)**: Used for file and directory operations.\n", + "* **[NumPy](https://numpy.org/doc/stable/)**: Allows efficient numerical manipulation of arrays.\n", + "* **[Pandas](https://pandas.pydata.org/docs/)**: Facilitates data manipulation and analysis.\n", + "* **[Librosa](https://librosa.org/doc/latest/index.html)**: Provides tools for analyzing and working with audio data.\n", + "* **[IPython](https://ipython.readthedocs.io/en/stable/index.html)**: Creates visualizations for multimedia content. Here we have used it for playing audio files.\n", + "* **[Sklearn](https://scikit-learn.org/stable/index.html)**: Offers comprehensive tools for Machine Learning. Here we have used it for preprocessing and splitting the data.\n", + "* **[TensorFlow](https://www.tensorflow.org/api_docs)** and **[Keras](https://keras.io/api/)**: Enables building and training complex Machine Learning and Deep Learning model.\n", + "* **[TFModelHandlerNumpy](https://beam.apache.org/documentation/sdks/python-machine-learning/#tensorflow)**: Defines the configuration used to load/use the model that we train. We use TFModelHandlerNumpy because the model was trained with TensorFlow and takes numpy arrays as input.\n", + "* **[RunInference](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.inference.html#apache_beam.ml.inference.RunInference)**: Loads the model and obtains predictions as part of the Apache Beam pipeline. For more information, see docs on prediction and inference.\n", + "* **[Apache Beam](https://beam.apache.org/documentation/)**: Builds a pipeline for Image Processing." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CzQkOP4v-X0Z" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "import librosa\n", + "from IPython.display import Audio\n", + "\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n", + "\n", + "import tensorflow as tf\n", + "from tensorflow import keras\n", + "from tensorflow.python.keras.callbacks import EarlyStopping, ReduceLROnPlateau\n", + "\n", + "from keras import layers\n", + "from keras import models\n", + "from keras.utils import np_utils\n", + "from keras.models import Sequential\n", + "from keras.utils import np_utils, to_categorical\n", + "from keras.callbacks import ModelCheckpoint\n", + "\n", + "from apache_beam.ml.inference.tensorflow_inference import TFModelHandlerNumpy\n", + "from apache_beam.ml.inference.base import RunInference\n", + "import apache_beam as beam" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dxAtTL2VIztQ" + }, + "source": [ + "## Importing dataset from Google Drive\n", + "\n", + "[CREMA-D](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4313618/) is a dataset that contains a collection of 7442 audio recordings of actors portraying different emotions. The dataset can be downloaded from [Kaggle](https://www.kaggle.com/datasets/ejlok1/cremad). As it is large in size, it will be inconvenient to upload it on Colab every time we want to run the notebook. Instead, we have uploaded the dataset on Google Drive after downloading it from Kaggle. Then we can access it directly using Colab.\n", + "\n", + "Please ensure if you are following this method, then your Colab notebook must be created with the same Google account in which the folder is stored." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PmaLie0lOI0g", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "e86b0467-8ab3-4ec8-e5cd-40e015ee272e" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Mounted at /content/gdrive\n" + ] + } + ], + "source": [ + "from google.colab import drive\n", + "drive.mount('/content/gdrive', force_remount=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "W3ESVLNUvyqG" + }, + "source": [ + "Here we create a path for the folder in Google Drive containing the audios to access them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "0BWrk6bn91Uu" + }, + "outputs": [], + "source": [ + "root_dir = \"/content/gdrive/My Drive/\"\n", + "Crema = root_dir + 'CREMA/'" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "VoROV5Vyvt-g" + }, + "source": [ + "Using the os library, we can list all the audio files in the Google Drive folder" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "7EXG23Yl-TfG", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "d5b539c0-75a6-4ae3-be56-8a074539cd98" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "['1079_TIE_NEU_XX.wav',\n", + " '1079_TIE_SAD_XX.wav',\n", + " '1079_TSI_ANG_XX.wav',\n", + " '1079_TSI_DIS_XX.wav',\n", + " '1079_TSI_HAP_XX.wav',\n", + " '1079_TSI_FEA_XX.wav',\n", + " '1079_TSI_NEU_XX.wav',\n", + " '1079_TSI_SAD_XX.wav',\n", + " '1079_WSI_ANG_XX.wav',\n", + " '1079_WSI_DIS_XX.wav']" + ] + }, + "metadata": {}, + "execution_count": 6 + } + ], + "source": [ + "os.chdir(Crema)\n", + "os.listdir()[:10] # Listing the first 10 audio files" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3HFYUtJ5JYS_" + }, + "source": [ + "## Creating a DataFrame\n", + "We will create a DataFrame with two columns, path and emotion:\n", + "* Path: This will contain the path to a specific audio file in the directory.\n", + "* Emotion: This is the label which will state the emotion of an audio file.\n", + "\n", + "The emotion can be extracted from the audio file name." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "MVy9nx56-edb", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 206 + }, + "outputId": "bc5e89dd-78e6-458b-9728-5030902f837e" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " Emotion Path\n", + "0 neutral /content/gdrive/My Drive/CREMA//1079_TIE_NEU_X...\n", + "1 sad /content/gdrive/My Drive/CREMA//1079_TIE_SAD_X...\n", + "2 angry /content/gdrive/My Drive/CREMA//1079_TSI_ANG_X...\n", + "3 disgust /content/gdrive/My Drive/CREMA//1079_TSI_DIS_X...\n", + "4 happy /content/gdrive/My Drive/CREMA//1079_TSI_HAP_X..." + ], + "text/html": [ + "\n", + "
    \n", + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    EmotionPath
    0neutral/content/gdrive/My Drive/CREMA//1079_TIE_NEU_X...
    1sad/content/gdrive/My Drive/CREMA//1079_TIE_SAD_X...
    2angry/content/gdrive/My Drive/CREMA//1079_TSI_ANG_X...
    3disgust/content/gdrive/My Drive/CREMA//1079_TSI_DIS_X...
    4happy/content/gdrive/My Drive/CREMA//1079_TSI_HAP_X...
    \n", + "
    \n", + "
    \n", + "\n", + "
    \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
    \n", + "\n", + "\n", + "
    \n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
    \n", + "
    \n", + "
    \n" + ] + }, + "metadata": {}, + "execution_count": 7 + } + ], + "source": [ + "emotion_df = []\n", + "\n", + "for wav in os.listdir(Crema):\n", + " info = wav.partition(\".wav\")[0].split(\"_\")\n", + " if (len(info)<3):\n", + " continue;\n", + " if info[2] == 'SAD':\n", + " emotion_df.append((\"sad\", Crema + \"/\" + wav))\n", + " elif info[2] == 'ANG':\n", + " emotion_df.append((\"angry\", Crema + \"/\" + wav))\n", + " elif info[2] == 'DIS':\n", + " emotion_df.append((\"disgust\", Crema + \"/\" + wav))\n", + " elif info[2] == 'FEA':\n", + " emotion_df.append((\"fear\", Crema + \"/\" + wav))\n", + " elif info[2] == 'HAP':\n", + " emotion_df.append((\"happy\", Crema + \"/\" + wav))\n", + " elif info[2] == 'NEU':\n", + " emotion_df.append((\"neutral\", Crema + \"/\" + wav))\n", + "\n", + "\n", + "Crema_df = pd.DataFrame.from_dict(emotion_df)\n", + "Crema_df.rename(columns={1 : \"Path\", 0 : \"Emotion\"}, inplace=True)\n", + "\n", + "Crema_df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XhKkVexzA46N" + }, + "source": [ + "## Preprocessing\n", + "\n", + "The audio files we want to use are in .wav format. However, an ML model works on numerical data. So we need to perform some preprocessing operations to extract numerical features from the audios and transform these features to a more suitable form. This will improve the performance of our model." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-sGwDJTKSoax" + }, + "source": [ + "### Data Augmentation\n", + "\n", + "This is the process of transforming existing data in various ways to generate more samples and increase model robustness. We make multiple versions of the same data item but with some differences. This allows the model to recognize a wider variety of data and reduce overfitting. We have performed the following data augmentation techniques:\n", + "* **Noise injection**: Adds a random factor to all data items to provide some noise.\n", + "* **Stretching**: Alters the speed of an audio, simulating variations in speech rate or tempo.\n", + "* **Pitch Shifting**: Changes the pitch of an audio, depicting variations of speaker characteristics or musical notes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "KYBjIdsfRyf_" + }, + "outputs": [], + "source": [ + "def noise(data):\n", + " noise_amp = 0.035 * np.random.uniform() * np.amax(data)\n", + " data = data + noise_amp * np.random.normal(size = data.shape[0])\n", + " return data\n", + "\n", + "def stretch(data, rate = 0.8):\n", + " return librosa.effects.time_stretch(data, rate = rate)\n", + "\n", + "def pitch(data, sampling_rate, pitch_factor = 0.7):\n", + " return librosa.effects.pitch_shift(data, sr = sampling_rate, n_steps = pitch_factor)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IqOVow1ROH4p" + }, + "source": [ + "### Feature Extraction\n", + "\n", + "We need to extract some numerical features from the audios to feed our ML model. The [Librosa](https://librosa.org/doc/latest/index.html) library allows us to do this easily.\n", + "\n", + "First, we need to understand what a **mel scale** is. It is a scale of pitches that is based on the way humans perceive and discriminate between different frequencies of sound. Now, let us discuss the features we will extract from the audio:\n", + "\n", + "* **Zero Crossing Rate (ZCR)**: Measures how often the sound changes it's sign (positive or negative) over time.\n", + "* **Chroma Short-Time Fourier Transform (STFT)**: Breaks down the audio signal into small segments (frames) and calculates the Fourier Transform for each frame, resulting in a time-frequency representation of the signal.\n", + "* **Mel-Frequency Cepstral Coefficients (MFCC)**: A set of coefficients derived from the mel spectrogram\n", + "* **Melspectogram**: A visual representation of the frequency content of an audio signal mapped on the mel scale.\n", + "* **Root Mean Square**: Provides the Root Mean Square value for each frame, which is a measure of the amplitude or energy of a sound signal.\n", + "\n", + "You can read more about all the features we can extract using the Librosa library [here](https://librosa.org/doc/latest/feature.html)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vHuzjpIfR2an" + }, + "outputs": [], + "source": [ + "def extract_features(data, sample_rate):\n", + " # ZCR\n", + " result = np.array([])\n", + " zcr = np.mean(librosa.feature.zero_crossing_rate(y=data).T, axis=0)\n", + " result=np.hstack((result, zcr)) # stacking horizontally\n", + "\n", + " # Chroma STFT\n", + " stft = np.abs(librosa.stft(data))\n", + " chroma_stft = np.mean(librosa.feature.chroma_stft(S=stft, sr=sample_rate).T, axis=0)\n", + " result = np.hstack((result, chroma_stft)) # stacking horizontally\n", + "\n", + " # MFCC\n", + " mfcc = np.mean(librosa.feature.mfcc(y=data, sr=sample_rate).T, axis=0)\n", + " result = np.hstack((result, mfcc)) # stacking horizontally\n", + "\n", + " # Root Mean Square\n", + " rms = np.mean(librosa.feature.rms(y=data).T, axis=0)\n", + " result = np.hstack((result, rms)) # stacking horizontally\n", + "\n", + " # Melspectogram\n", + " mel = np.mean(librosa.feature.melspectrogram(y=data, sr=sample_rate).T, axis=0)\n", + " result = np.hstack((result, mel)) # stacking horizontally\n", + "\n", + " return result" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4eo-DgBiAFH4" + }, + "source": [ + "The function below is used to extract the features from the audio stored at a path. Then it applies the data augmentation techniques we defined previously, and extracts features for each augmented data too. This gives us three versions of a data item:\n", + "* Normal features\n", + "* Features from data with noise\n", + "* Features from time stretched and pitch shifted data\n", + "\n", + "These are added into our final dataset as individual samples." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rksrNPA4R8v9" + }, + "outputs": [], + "source": [ + "def get_features(path):\n", + " data, sample_rate = librosa.load(path, duration=2.5, offset=0.6)\n", + "\n", + " # without augmentation\n", + " normal_features = extract_features(data, sample_rate)\n", + " result = np.array(normal_features)\n", + "\n", + " # data with noise\n", + " noise_data = noise(data)\n", + " noise_features = extract_features(noise_data, sample_rate)\n", + " result = np.vstack((result, noise_features)) # stacking vertically\n", + "\n", + " # data with stretching and pitching\n", + " stretch_data = stretch(data)\n", + " stretch_pitch_data = pitch(stretch_data, sample_rate)\n", + " stretch_pitch_features = extract_features(stretch_pitch_data, sample_rate)\n", + " result = np.vstack((result, stretch_pitch_features)) # stacking vertically\n", + "\n", + " return result" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iIt_EokpBdHc" + }, + "source": [ + "Now we will iterate through the Crema_df DataFrame containing the path and emotion of each audio sample. We will extract features for each audio's three versions, add it to X, and add the corresponding emotion to Y." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "81rZsmb4SBCH", + "outputId": "a8b14400-7aee-434d-c9a9-236db687d112" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/usr/local/lib/python3.10/dist-packages/librosa/core/pitch.py:101: UserWarning: Trying to estimate tuning from empty frequency set.\n", + " return pitch_tuning(\n" + ] + } + ], + "source": [ + "X, Y = [], []\n", + "for path, emotion in zip(Crema_df.Path, Crema_df.Emotion):\n", + " feature = get_features(path)\n", + " for ele in feature:\n", + " X.append(ele)\n", + " Y.append(emotion)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Xa8RsEiYB-ru" + }, + "source": [ + "Here we have made a DataFrame using the lists X and Y." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 342 + }, + "id": "go4lU_2VSCxy", + "outputId": "05734eff-da43-4396-8c2e-c4c80ec2b0be" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " 0 1 2 3 4 5 6 \\\n", + "0 0.051835 0.552957 0.564289 0.512976 0.518041 0.528111 0.501150 \n", + "1 0.081790 0.611068 0.619012 0.578897 0.580346 0.604983 0.552418 \n", + "2 0.054339 0.525215 0.525026 0.478083 0.526773 0.554233 0.521426 \n", + "3 0.050157 0.514931 0.591693 0.464526 0.429137 0.480203 0.572344 \n", + "4 0.098122 0.606869 0.680955 0.572593 0.548943 0.581684 0.626757 \n", + "\n", + " 7 8 9 ... 153 154 \\\n", + "0 0.550490 0.673705 0.744412 ... 2.713831e-09 2.560777e-09 \n", + "1 0.557888 0.677792 0.749837 ... 8.333886e-05 7.936021e-05 \n", + "2 0.558976 0.671527 0.739728 ... 3.503047e-09 3.054322e-09 \n", + "3 0.722630 0.699706 0.676802 ... 3.512564e-09 3.153377e-09 \n", + "4 0.754920 0.735712 0.713573 ... 1.368801e-04 1.329551e-04 \n", + "\n", + " 155 156 157 158 159 \\\n", + "0 2.451516e-09 2.369350e-09 2.308000e-09 2.264365e-09 2.232698e-09 \n", + "1 7.905496e-05 8.138233e-05 7.764955e-05 7.412745e-05 7.555283e-05 \n", + "2 2.943538e-09 2.634693e-09 2.343703e-09 2.368675e-09 2.363831e-09 \n", + "3 2.901090e-09 2.715085e-09 2.576861e-09 2.476340e-09 2.403195e-09 \n", + "4 1.397343e-04 1.433890e-04 1.408767e-04 1.354171e-04 1.373235e-04 \n", + "\n", + " 160 161 labels \n", + "0 2.212761e-09 2.200083e-09 neutral \n", + "1 8.043366e-05 8.144332e-05 neutral \n", + "2 1.876258e-09 6.538760e-10 neutral \n", + "3 2.354688e-09 2.325111e-09 sad \n", + "4 1.433754e-04 1.442893e-04 sad \n", + "\n", + "[5 rows x 163 columns]" + ], + "text/html": [ + "\n", + "
    \n", + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    0123456789...153154155156157158159160161labels
    00.0518350.5529570.5642890.5129760.5180410.5281110.5011500.5504900.6737050.744412...2.713831e-092.560777e-092.451516e-092.369350e-092.308000e-092.264365e-092.232698e-092.212761e-092.200083e-09neutral
    10.0817900.6110680.6190120.5788970.5803460.6049830.5524180.5578880.6777920.749837...8.333886e-057.936021e-057.905496e-058.138233e-057.764955e-057.412745e-057.555283e-058.043366e-058.144332e-05neutral
    20.0543390.5252150.5250260.4780830.5267730.5542330.5214260.5589760.6715270.739728...3.503047e-093.054322e-092.943538e-092.634693e-092.343703e-092.368675e-092.363831e-091.876258e-096.538760e-10neutral
    30.0501570.5149310.5916930.4645260.4291370.4802030.5723440.7226300.6997060.676802...3.512564e-093.153377e-092.901090e-092.715085e-092.576861e-092.476340e-092.403195e-092.354688e-092.325111e-09sad
    40.0981220.6068690.6809550.5725930.5489430.5816840.6267570.7549200.7357120.713573...1.368801e-041.329551e-041.397343e-041.433890e-041.408767e-041.354171e-041.373235e-041.433754e-041.442893e-04sad
    \n", + "

    5 rows × 163 columns

    \n", + "
    \n", + "
    \n", + "\n", + "
    \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
    \n", + "\n", + "\n", + "
    \n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
    \n", + "
    \n", + "
    \n" + ] + }, + "metadata": {}, + "execution_count": 12 + } + ], + "source": [ + "Features = pd.DataFrame(X)\n", + "Features['labels'] = Y\n", + "Features.to_csv('features.csv', index=False)\n", + "Features.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uk4y87eECD4F" + }, + "source": [ + "The X and Y datasets are separated here. X stores the features of audio samples while Y stores the corresponding labels." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "EUObnydOSEXr" + }, + "outputs": [], + "source": [ + "X = Features.iloc[: ,:-1].values\n", + "Y = Features['labels'].values" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "B92B4YVoCPed" + }, + "source": [ + "The [pad sequences](https://www.tensorflow.org/api_docs/python/tf/keras/utils/pad_sequences) function is used to pad the input data to the same length, to ensure that all samples have the same shape." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "s_yiO3ZsrLjL" + }, + "outputs": [], + "source": [ + "X = tf.keras.utils.pad_sequences(X)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AvHBQsARUeNb" + }, + "source": [ + "Scikit Learn's [OneHotEncoder](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.OneHotEncoder.html) is used to convert categorical labels into numerical data. It creates a column in the labels dataset for each category, which contains only binary data. For example, if we have the following categories:\n", + "\n", + "`[Anger, Disgust, Fear, Happy, Neutral, Sad]`\n", + "\n", + "And a specific audio belongs to 'Anger' category, then the OneHotEncoder will transform it to:\n", + "\n", + "`[1, 0, 0, 0, 0, 0]`\n", + "\n", + "Please note that the order of which column represents which category may differ." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "OT2HLn0HSF_3" + }, + "outputs": [], + "source": [ + "encoder = OneHotEncoder()\n", + "Y = encoder.fit_transform(np.array(Y).reshape(-1,1)).toarray()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PZYKuz7FUfyj" + }, + "source": [ + "Splitting into train/test splits" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "snRl0fNdSJnv", + "outputId": "faf825f6-a7f6-4cd5-c3da-60bafaafddc2" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "((16744, 162), (16744, 6), (5582, 162), (5582, 6))" + ] + }, + "metadata": {}, + "execution_count": 16 + } + ], + "source": [ + "x_train, x_test, y_train, y_test = train_test_split(X, Y, random_state=0, shuffle=True)\n", + "x_train.shape, y_train.shape, x_test.shape, y_test.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IHoJ7egnUkrf" + }, + "source": [ + "Now we will scale the data and split it into training and testing sets.\n", + "* [Scaling](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.scale.html) is done to make all numerical data have similar magnitudes. This makes computations easier.\n", + "* The training sets are used to train the model.\n", + "* The testing sets are used to test the model's accuracy." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "l0Jg7MN4SLls", + "outputId": "071cc13e-c60a-4c9b-b81d-ae02a1f67ae3" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "((16744, 162), (16744, 6), (5582, 162), (5582, 6))" + ] + }, + "metadata": {}, + "execution_count": 17 + } + ], + "source": [ + "scaler = StandardScaler()\n", + "x_train = scaler.fit_transform(x_train)\n", + "x_test = scaler.transform(x_test)\n", + "x_train.shape, y_train.shape, x_test.shape, y_test.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "HVEcjoXSrF_i", + "outputId": "ef9101b8-7af0-4733-8c85-e58038b5263d" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "((22326, 162), (16744, 162), (5582, 162))" + ] + }, + "metadata": {}, + "execution_count": 18 + } + ], + "source": [ + "X.shape, x_train.shape, x_test.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lbJnHZExUmOY" + }, + "source": [ + "We will use a 1D Convolutional layer in our model, and for that, our input data needs to be a a 3D tensor with dimensions `(batch_size, time_steps, input_dim)`. So we will expand the dimensions of our X_train and X_test datasets. The extra 1 in the shape depicts that our data is 1 dimensional." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "DfHhUoUFSMFL", + "outputId": "2e1a9134-e1f7-4826-a9ad-3f74ebe12849" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "((16744, 162, 1), (16744, 6), (5582, 162, 1), (5582, 6))" + ] + }, + "metadata": {}, + "execution_count": 19 + } + ], + "source": [ + "x_train = np.expand_dims(x_train, axis=2)\n", + "x_test = np.expand_dims(x_test, axis=2)\n", + "x_train.shape, y_train.shape, x_test.shape, y_test.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TAnQsHL9Uphd" + }, + "source": [ + "### Training the model\n", + "We will build a sequential model to classify speech emotions using TensorFlow and Keras. Here is an overview of the layers used:\n", + "* **Conv1D**: Applies a set of filters to capture patterns in sequential data like time series or audio, enabling feature extraction through sliding convolutions.\n", + "* **Activation**: Introduces non-linearity by applying an element-wise activation function to the input, enhancing the network's learning capacity.\n", + "* **BatchNormalization**: Normalizes input activations within a mini-batch, accelerating training by stabilizing and improving gradient flow.\n", + "* **Dropout**: Randomly deactivates a fraction of neurons during training, reducing overfitting by promoting generalization.\n", + "* **MaxPooling1D**: Downsamples the input by retaining the maximum value in each local region, reducing computation.\n", + "* **Flatten**: Reshapes input data from a multidimensional format into a 1D vector, suitable for fully connected layers.\n", + "* **Dense**: Connects each neuron to every neuron in the previous layer, allowing complex relationships to be learned during training.\n", + "\n", + "In the end, we need probabilities for each of the 6 classes of emotions, so we need 6 outputs. This is why the last Dense layer returns an array of size 6.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "uFb71_ga5RoB", + "outputId": "cba3e5ba-f17e-4eb7-c5fd-0adc4ed060c9" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Model: \"sequential\"\n", + "_________________________________________________________________\n", + " Layer (type) Output Shape Param # \n", + "=================================================================\n", + " conv1d (Conv1D) (None, 162, 256) 1792 \n", + " \n", + " activation (Activation) (None, 162, 256) 0 \n", + " \n", + " conv1d_1 (Conv1D) (None, 162, 256) 393472 \n", + " \n", + " batch_normalization (BatchN (None, 162, 256) 1024 \n", + " ormalization) \n", + " \n", + " activation_1 (Activation) (None, 162, 256) 0 \n", + " \n", + " dropout (Dropout) (None, 162, 256) 0 \n", + " \n", + " max_pooling1d (MaxPooling1D (None, 20, 256) 0 \n", + " ) \n", + " \n", + " conv1d_2 (Conv1D) (None, 20, 128) 196736 \n", + " \n", + " activation_2 (Activation) (None, 20, 128) 0 \n", + " \n", + " conv1d_3 (Conv1D) (None, 20, 128) 98432 \n", + " \n", + " activation_3 (Activation) (None, 20, 128) 0 \n", + " \n", + " dropout_1 (Dropout) (None, 20, 128) 0 \n", + " \n", + " conv1d_4 (Conv1D) (None, 20, 128) 98432 \n", + " \n", + " activation_4 (Activation) (None, 20, 128) 0 \n", + " \n", + " conv1d_5 (Conv1D) (None, 20, 128) 98432 \n", + " \n", + " batch_normalization_1 (Batc (None, 20, 128) 512 \n", + " hNormalization) \n", + " \n", + " activation_5 (Activation) (None, 20, 128) 0 \n", + " \n", + " dropout_2 (Dropout) (None, 20, 128) 0 \n", + " \n", + " max_pooling1d_1 (MaxPooling (None, 2, 128) 0 \n", + " 1D) \n", + " \n", + " conv1d_6 (Conv1D) (None, 2, 64) 49216 \n", + " \n", + " activation_6 (Activation) (None, 2, 64) 0 \n", + " \n", + " conv1d_7 (Conv1D) (None, 2, 64) 24640 \n", + " \n", + " activation_7 (Activation) (None, 2, 64) 0 \n", + " \n", + " dropout_3 (Dropout) (None, 2, 64) 0 \n", + " \n", + " flatten (Flatten) (None, 128) 0 \n", + " \n", + " dense (Dense) (None, 6) 774 \n", + " \n", + " activation_8 (Activation) (None, 6) 0 \n", + " \n", + "=================================================================\n", + "Total params: 963,462\n", + "Trainable params: 962,694\n", + "Non-trainable params: 768\n", + "_________________________________________________________________\n" + ] + } + ], + "source": [ + "model = Sequential()\n", + "model.add(layers.Conv1D(256, 6, padding='same',input_shape=(x_train.shape[1],1)))\n", + "model.add(layers.Activation('relu'))\n", + "model.add(layers.Conv1D(256, 6, padding='same'))\n", + "model.add(layers.BatchNormalization())\n", + "model.add(layers.Activation('relu'))\n", + "model.add(layers.Dropout(0.2))\n", + "model.add(layers.MaxPooling1D(pool_size=(8)))\n", + "model.add(layers.Conv1D(128, 6, padding='same'))\n", + "model.add(layers.Activation('relu'))\n", + "model.add(layers.Conv1D(128, 6, padding='same'))\n", + "model.add(layers.Activation('relu'))\n", + "model.add(layers.Dropout(0.2))\n", + "model.add(layers.Conv1D(128, 6, padding='same'))\n", + "model.add(layers.Activation('relu'))\n", + "model.add(layers.Conv1D(128, 6, padding='same'))\n", + "model.add(layers.BatchNormalization())\n", + "model.add(layers.Activation('relu'))\n", + "model.add(layers.Dropout(0.2))\n", + "model.add(layers.MaxPooling1D(pool_size=(8)))\n", + "model.add(layers.Conv1D(64, 6, padding='same'))\n", + "model.add(layers.Activation('relu'))\n", + "model.add(layers.Conv1D(64, 6, padding='same'))\n", + "model.add(layers.Activation('relu'))\n", + "model.add(layers.Dropout(0.2))\n", + "model.add(layers.Flatten())\n", + "model.add(layers.Dense(6))\n", + "model.add(layers.Activation('softmax'))\n", + "opt = keras.optimizers.Adam(learning_rate=0.0001)\n", + "model.summary()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "g68nijvIF2Ca" + }, + "source": [ + "Now we will compile the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qWj3H05Q6pm_" + }, + "outputs": [], + "source": [ + "model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "p78y1ESDF5KQ" + }, + "source": [ + "Next, we will train our model. [ReduceLROnPlateau](https://keras.io/api/callbacks/reduce_lr_on_plateau/) is used to reduce the learning rate when the loss has stopped improving. [EarlyStopping](https://keras.io/api/callbacks/early_stopping/) monitors the val_loss and stops the training process when it doesn't improve." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "H1F2haOv6pvZ", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "f2f2bb9f-e445-4713-878c-82da952454fc" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Epoch 1/100\n", + "1047/1047 [==============================] - 29s 13ms/step - loss: 1.5803 - accuracy: 0.3272 - val_loss: 1.5216 - val_accuracy: 0.3739 - lr: 1.0000e-04\n", + "Epoch 2/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 1.4870 - accuracy: 0.3807 - val_loss: 1.5065 - val_accuracy: 0.3884 - lr: 1.0000e-04\n", + "Epoch 3/100\n", + "1047/1047 [==============================] - 12s 11ms/step - loss: 1.4541 - accuracy: 0.3952 - val_loss: 1.4635 - val_accuracy: 0.3954 - lr: 1.0000e-04\n", + "Epoch 4/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 1.4253 - accuracy: 0.4105 - val_loss: 1.4341 - val_accuracy: 0.4282 - lr: 1.0000e-04\n", + "Epoch 5/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 1.4092 - accuracy: 0.4199 - val_loss: 1.4595 - val_accuracy: 0.4077 - lr: 1.0000e-04\n", + "Epoch 6/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 1.3890 - accuracy: 0.4299 - val_loss: 1.4032 - val_accuracy: 0.4317 - lr: 1.0000e-04\n", + "Epoch 7/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 1.3709 - accuracy: 0.4471 - val_loss: 1.3958 - val_accuracy: 0.4294 - lr: 1.0000e-04\n", + "Epoch 8/100\n", + "1047/1047 [==============================] - 12s 11ms/step - loss: 1.3613 - accuracy: 0.4482 - val_loss: 1.4311 - val_accuracy: 0.4102 - lr: 1.0000e-04\n", + "Epoch 9/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 1.3420 - accuracy: 0.4563 - val_loss: 1.3901 - val_accuracy: 0.4409 - lr: 1.0000e-04\n", + "Epoch 10/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 1.3292 - accuracy: 0.4643 - val_loss: 1.3893 - val_accuracy: 0.4434 - lr: 1.0000e-04\n", + "Epoch 11/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 1.3162 - accuracy: 0.4689 - val_loss: 1.3742 - val_accuracy: 0.4482 - lr: 1.0000e-04\n", + "Epoch 12/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 1.3033 - accuracy: 0.4738 - val_loss: 1.3821 - val_accuracy: 0.4507 - lr: 1.0000e-04\n", + "Epoch 13/100\n", + "1047/1047 [==============================] - 12s 11ms/step - loss: 1.2889 - accuracy: 0.4833 - val_loss: 1.3452 - val_accuracy: 0.4609 - lr: 1.0000e-04\n", + "Epoch 14/100\n", + "1047/1047 [==============================] - 12s 11ms/step - loss: 1.2715 - accuracy: 0.4933 - val_loss: 1.3690 - val_accuracy: 0.4559 - lr: 1.0000e-04\n", + "Epoch 15/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 1.2642 - accuracy: 0.4916 - val_loss: 1.3460 - val_accuracy: 0.4618 - lr: 1.0000e-04\n", + "Epoch 16/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 1.2439 - accuracy: 0.5028 - val_loss: 1.3293 - val_accuracy: 0.4719 - lr: 1.0000e-04\n", + "Epoch 17/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 1.2287 - accuracy: 0.5073 - val_loss: 1.3309 - val_accuracy: 0.4663 - lr: 1.0000e-04\n", + "Epoch 18/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 1.2193 - accuracy: 0.5122 - val_loss: 1.3353 - val_accuracy: 0.4686 - lr: 1.0000e-04\n", + "Epoch 19/100\n", + "1047/1047 [==============================] - 13s 13ms/step - loss: 1.2044 - accuracy: 0.5237 - val_loss: 1.3370 - val_accuracy: 0.4636 - lr: 1.0000e-04\n", + "Epoch 20/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 1.1869 - accuracy: 0.5258 - val_loss: 1.3021 - val_accuracy: 0.4805 - lr: 1.0000e-04\n", + "Epoch 21/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 1.1744 - accuracy: 0.5288 - val_loss: 1.3028 - val_accuracy: 0.4807 - lr: 1.0000e-04\n", + "Epoch 22/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 1.1574 - accuracy: 0.5376 - val_loss: 1.3189 - val_accuracy: 0.4717 - lr: 1.0000e-04\n", + "Epoch 23/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 1.1437 - accuracy: 0.5492 - val_loss: 1.3197 - val_accuracy: 0.4694 - lr: 1.0000e-04\n", + "Epoch 24/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 1.1234 - accuracy: 0.5563 - val_loss: 1.3482 - val_accuracy: 0.4678 - lr: 1.0000e-04\n", + "Epoch 25/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 1.1152 - accuracy: 0.5568 - val_loss: 1.3050 - val_accuracy: 0.4821 - lr: 1.0000e-04\n", + "Epoch 26/100\n", + "1047/1047 [==============================] - 12s 11ms/step - loss: 1.1022 - accuracy: 0.5677 - val_loss: 1.2853 - val_accuracy: 0.4867 - lr: 1.0000e-04\n", + "Epoch 27/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 1.0844 - accuracy: 0.5678 - val_loss: 1.2719 - val_accuracy: 0.4925 - lr: 1.0000e-04\n", + "Epoch 28/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 1.0644 - accuracy: 0.5828 - val_loss: 1.2978 - val_accuracy: 0.4798 - lr: 1.0000e-04\n", + "Epoch 29/100\n", + "1047/1047 [==============================] - 14s 14ms/step - loss: 1.0524 - accuracy: 0.5882 - val_loss: 1.2986 - val_accuracy: 0.4844 - lr: 1.0000e-04\n", + "Epoch 30/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 1.0364 - accuracy: 0.5920 - val_loss: 1.2919 - val_accuracy: 0.4894 - lr: 1.0000e-04\n", + "Epoch 31/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 1.0160 - accuracy: 0.6043 - val_loss: 1.2651 - val_accuracy: 0.4937 - lr: 1.0000e-04\n", + "Epoch 32/100\n", + "1047/1047 [==============================] - 12s 11ms/step - loss: 1.0056 - accuracy: 0.6058 - val_loss: 1.2905 - val_accuracy: 0.4841 - lr: 1.0000e-04\n", + "Epoch 33/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 0.9838 - accuracy: 0.6154 - val_loss: 1.2708 - val_accuracy: 0.4955 - lr: 1.0000e-04\n", + "Epoch 34/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 0.9778 - accuracy: 0.6135 - val_loss: 1.2651 - val_accuracy: 0.5032 - lr: 1.0000e-04\n", + "Epoch 35/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 0.9610 - accuracy: 0.6234 - val_loss: 1.3275 - val_accuracy: 0.4751 - lr: 1.0000e-04\n", + "Epoch 36/100\n", + "1047/1047 [==============================] - 13s 13ms/step - loss: 0.9461 - accuracy: 0.6349 - val_loss: 1.2683 - val_accuracy: 0.4971 - lr: 1.0000e-04\n", + "Epoch 37/100\n", + "1047/1047 [==============================] - 12s 11ms/step - loss: 0.9443 - accuracy: 0.6332 - val_loss: 1.2852 - val_accuracy: 0.4923 - lr: 1.0000e-04\n", + "Epoch 38/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 0.9169 - accuracy: 0.6452 - val_loss: 1.2813 - val_accuracy: 0.4961 - lr: 1.0000e-04\n", + "Epoch 39/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 0.9133 - accuracy: 0.6436 - val_loss: 1.2613 - val_accuracy: 0.5050 - lr: 1.0000e-04\n", + "Epoch 40/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 0.8981 - accuracy: 0.6509 - val_loss: 1.2701 - val_accuracy: 0.5084 - lr: 1.0000e-04\n", + "Epoch 41/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 0.8874 - accuracy: 0.6515 - val_loss: 1.2848 - val_accuracy: 0.4928 - lr: 1.0000e-04\n", + "Epoch 42/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 0.8712 - accuracy: 0.6620 - val_loss: 1.2626 - val_accuracy: 0.5052 - lr: 1.0000e-04\n", + "Epoch 43/100\n", + "1047/1047 [==============================] - 15s 14ms/step - loss: 0.8702 - accuracy: 0.6597 - val_loss: 1.2687 - val_accuracy: 0.5109 - lr: 1.0000e-04\n", + "Epoch 44/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 0.8500 - accuracy: 0.6694 - val_loss: 1.2604 - val_accuracy: 0.5133 - lr: 1.0000e-04\n", + "Epoch 45/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 0.8305 - accuracy: 0.6759 - val_loss: 1.2698 - val_accuracy: 0.5122 - lr: 1.0000e-04\n", + "Epoch 46/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 0.8266 - accuracy: 0.6805 - val_loss: 1.2949 - val_accuracy: 0.5043 - lr: 1.0000e-04\n", + "Epoch 47/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 0.8132 - accuracy: 0.6860 - val_loss: 1.2778 - val_accuracy: 0.5021 - lr: 1.0000e-04\n", + "Epoch 48/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 0.7994 - accuracy: 0.6940 - val_loss: 1.2740 - val_accuracy: 0.5091 - lr: 1.0000e-04\n", + "Epoch 49/100\n", + "1047/1047 [==============================] - 13s 13ms/step - loss: 0.7836 - accuracy: 0.6936 - val_loss: 1.2925 - val_accuracy: 0.5070 - lr: 1.0000e-04\n", + "Epoch 50/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 0.7757 - accuracy: 0.7038 - val_loss: 1.3190 - val_accuracy: 0.5011 - lr: 1.0000e-04\n", + "Epoch 51/100\n", + "1047/1047 [==============================] - 12s 11ms/step - loss: 0.7679 - accuracy: 0.7001 - val_loss: 1.2861 - val_accuracy: 0.5027 - lr: 1.0000e-04\n", + "Epoch 52/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 0.7542 - accuracy: 0.7114 - val_loss: 1.3435 - val_accuracy: 0.4927 - lr: 1.0000e-04\n", + "Epoch 53/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 0.7459 - accuracy: 0.7093 - val_loss: 1.3164 - val_accuracy: 0.5072 - lr: 1.0000e-04\n", + "Epoch 54/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 0.7287 - accuracy: 0.7193 - val_loss: 1.2878 - val_accuracy: 0.5188 - lr: 1.0000e-04\n", + "Epoch 55/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 0.7178 - accuracy: 0.7262 - val_loss: 1.3178 - val_accuracy: 0.5054 - lr: 1.0000e-04\n", + "Epoch 56/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 0.7076 - accuracy: 0.7258 - val_loss: 1.3746 - val_accuracy: 0.4912 - lr: 1.0000e-04\n", + "Epoch 57/100\n", + "1047/1047 [==============================] - 12s 11ms/step - loss: 0.6955 - accuracy: 0.7306 - val_loss: 1.3457 - val_accuracy: 0.5097 - lr: 1.0000e-04\n", + "Epoch 58/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 0.6843 - accuracy: 0.7364 - val_loss: 1.3558 - val_accuracy: 0.5000 - lr: 1.0000e-04\n", + "Epoch 59/100\n", + "1047/1047 [==============================] - 12s 12ms/step - loss: 0.6790 - accuracy: 0.7370 - val_loss: 1.3310 - val_accuracy: 0.5150 - lr: 1.0000e-04\n", + "Epoch 60/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 0.6683 - accuracy: 0.7431 - val_loss: 1.3515 - val_accuracy: 0.5127 - lr: 1.0000e-04\n", + "Epoch 61/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 0.6628 - accuracy: 0.7419 - val_loss: 1.3877 - val_accuracy: 0.4955 - lr: 1.0000e-04\n", + "Epoch 62/100\n", + "1047/1047 [==============================] - 12s 11ms/step - loss: 0.6462 - accuracy: 0.7501 - val_loss: 1.3549 - val_accuracy: 0.5202 - lr: 1.0000e-04\n", + "Epoch 63/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 0.6305 - accuracy: 0.7597 - val_loss: 1.3709 - val_accuracy: 0.5109 - lr: 1.0000e-04\n", + "Epoch 64/100\n", + "1047/1047 [==============================] - 13s 12ms/step - loss: 0.6245 - accuracy: 0.7610 - val_loss: 1.3442 - val_accuracy: 0.5269 - lr: 1.0000e-04\n", + "Epoch 00064: early stopping\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 22 + } + ], + "source": [ + "rlrp = ReduceLROnPlateau(monitor='loss', factor=0.4, verbose=0, patience=2, min_lr=0.0000001)\n", + "es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=20)\n", + "\n", + "model.fit(x_train, y_train, batch_size=16, epochs=100, validation_data=(x_test, y_test), callbacks=[es, rlrp])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UiuEkw7DF_M5" + }, + "source": [ + "We can see that the accuracy of our model is not very high. This is because speech data is more complex than other forms of data and much more training data and/or preprocessing techniques are required to build a good speech emotion classifier. If you want to increase the accuracy, you can use multiple datasets instead of just one, and use more features from the Librosa library. You can also try experimenting with LSTM layers in the model. Here are some of the popular speech emotion datasets:\n", + "* [RAVDESS](https://www.kaggle.com/datasets/uwrfkaggler/ravdess-emotional-speech-audio)\n", + "* [LSSED](https://github.com/tobefans/LSSED)\n", + "* [TESS](https://www.kaggle.com/datasets/ejlok1/toronto-emotional-speech-set-tess)\n", + "* [IEMOCAP](https://www.kaggle.com/datasets/columbine/iemocap)\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "btAlJmJjqgGS" + }, + "source": [ + "### Saving model in Google Cloud Bucket\n", + "In our final Beam pipeline, we will use RunInference. For that, we need to have a pretrained model stored in a location that is accessible to a model handler. Storing the model in a Google Cloud Bucket is the easiest way to do this." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NoOhXDcUMYaA", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "bb7b2af8-86b3-48a5-a404-edf935147bac" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "WARNING:absl:Found untraced functions such as _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op while saving (showing 5 of 8). These functions will not be directly callable after loading.\n" + ] + } + ], + "source": [ + "save_model_dir = '' # Add the link to you GCS bucket here\n", + "model.save(save_model_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KRrMuKwmqo1I" + }, + "source": [ + "### Creating a model handler\n", + "A model handler is used to save, load and manage trained models. We have used TFModelHandlerNumpy since our model was built using TensorFlow and takes NumPy arrays as input." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "0dbI7-9KMbt6" + }, + "outputs": [], + "source": [ + "model_handler = TFModelHandlerNumpy(save_model_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZoStfR5HqVKV" + }, + "source": [ + "## Preprocessing functions for Beam pipeline\n", + "We need to define some functions to perform the same preprocessing tasks we did on our training data. We can't reuse the previously defined function directly since they processed multidimensional data, and in a pipeline we deal with a single data item, which requires different methods." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2MmGkjcpYJuf" + }, + "source": [ + "This function loads the audio data using Librosa and extracts features using the previously defined function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "MktIcCkzYJ8-" + }, + "outputs": [], + "source": [ + "def feature_extraction(element):\n", + " data, sample_rate = librosa.load(path, duration=2.5, offset=0.6)\n", + " return extract_features(data, sample_rate)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "s8S4N1W-MeAG" + }, + "source": [ + "Here we have scaled the data using standardization. The data is transformed such that it's mean is 0 and standard deviation is 1." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "XlYhtZf-p01y" + }, + "outputs": [], + "source": [ + "def scaling(element):\n", + " element = (element-np.mean(element))/np.std(element)\n", + " return element" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1MOL8vfzMtpX" + }, + "source": [ + "In the end we will save our predictions in a list. RunInference returns an array of probabilities for each class. We select the maximum probability, replace it by 1, and replace all other values with 0. Now our new list is in a standard one hot encoded format, and we can use the inverse transform function of the OneHotEncoder to return which class the resultant array represents." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ehwQB-PdqwWh" + }, + "outputs": [], + "source": [ + "predictions = []" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Dos4eVTIjfpq" + }, + "outputs": [], + "source": [ + "from tensorflow.python.ops.numpy_ops import np_config\n", + "np_config.enable_numpy_behavior()\n", + "def save_predictions(element):\n", + " list_of_predictions = element.inference.tolist()\n", + " highest_prediction = max(list_of_predictions)\n", + " l = []\n", + " for i in range(len(list_of_predictions)):\n", + " if list_of_predictions[i] == highest_prediction:\n", + " l.append(1)\n", + " else:\n", + " l.append(0);\n", + " ans = encoder.inverse_transform(np.array(l).reshape(1,-1))[0][0]\n", + " predictions.append(ans)\n", + " print(ans)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pZjR4A01q5iu" + }, + "source": [ + "## Building the Beam Pipeline\n", + "This pipeline performs the following tasks\n", + "* Creates a PCollection of input paths\n", + "* Extracts features using the previously defined functions\n", + "* Performs scaling\n", + "* Runs inference on new data using the previously trained model\n", + "* Saves predictions in a list" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ZAwVSGg_mWB_" + }, + "outputs": [], + "source": [ + "pipeline_input = Crema_df[:2].Path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wJqMVam6lIHX", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "f6e5bb00-ff15-4aca-f8cd-98ab867a2f07" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "sad\n", + "sad\n" + ] + } + ], + "source": [ + "with beam.Pipeline() as p:\n", + " _ = (p | beam.Create(pipeline_input)\n", + " | beam.Map(feature_extraction)\n", + " | beam.Map(scaling)\n", + " | RunInference(model_handler)\n", + " | beam.Map(save_predictions)\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Pt5zEoXxS6wh", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 112 + }, + "outputId": "8d2bf9c8-2a46-4ff5-d200-af370aaa24a7" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " Emotion Path\n", + "0 neutral /content/gdrive/My Drive/CREMA//1079_TIE_NEU_X...\n", + "1 sad /content/gdrive/My Drive/CREMA//1079_TIE_SAD_X..." + ], + "text/html": [ + "\n", + "
    \n", + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    EmotionPath
    0neutral/content/gdrive/My Drive/CREMA//1079_TIE_NEU_X...
    1sad/content/gdrive/My Drive/CREMA//1079_TIE_SAD_X...
    \n", + "
    \n", + "
    \n", + "\n", + "
    \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
    \n", + "\n", + "\n", + "
    \n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
    \n", + "
    \n", + "
    \n" + ] + }, + "metadata": {}, + "execution_count": 31 + } + ], + "source": [ + "Crema_df[:2]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "WNjJSGh8rpbP", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 75 + }, + "outputId": "74b9d9d4-daab-4b7d-c5db-c07078ea5d46" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "\n", + " \n", + " " + ] + }, + "metadata": {}, + "execution_count": 32 + } + ], + "source": [ + "from IPython.display import Audio\n", + "Audio(Crema_df.iloc[0].Path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "LN69bC_ksG0k", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 75 + }, + "outputId": "8b7d1eea-95fb-47d8-9643-ccd34dd3b87c" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "\n", + " \n", + " " + ] + }, + "metadata": {}, + "execution_count": 33 + } + ], + "source": [ + "Audio(Crema_df.iloc[1].Path)" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/examples/notebooks/get-started/learn_beam_basics_by_doing.ipynb b/examples/notebooks/get-started/learn_beam_basics_by_doing.ipynb index 0fc0ffcf4ae2c..0a47a38197ba8 100644 --- a/examples/notebooks/get-started/learn_beam_basics_by_doing.ipynb +++ b/examples/notebooks/get-started/learn_beam_basics_by_doing.ipynb @@ -75,7 +75,7 @@ "\n", "It may be tempting to just copy and paste solutions, but even if you do look at the Answer cells, try typing out the solutions manually. The muscle memory will be very helpful.\n", "\n", - "> Tip: For those who would like to learn concepts more from the ground up, check out these [notebooks](https://beam.apache.org/get-started/tour-of-beam/)!" + "> Tip: For those who would like to learn concepts more from the ground up, check out the [Tour of Beam](https://tour.beam.apache.org/)!" ] }, { diff --git a/examples/notebooks/healthcare/beam_nlp.ipynb b/examples/notebooks/healthcare/beam_nlp.ipynb new file mode 100644 index 0000000000000..c2061bc4d75f6 --- /dev/null +++ b/examples/notebooks/healthcare/beam_nlp.ipynb @@ -0,0 +1,879 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "include_colab_link": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "code", + "source": [ + "# @title ###### Licensed to the Apache Software Foundation (ASF), Version 2.0 (the \"License\")\n", + "\n", + "# Licensed to the Apache Software Foundation (ASF) under one\n", + "# or more contributor license agreements. See the NOTICE file\n", + "# distributed with this work for additional information\n", + "# regarding copyright ownership. The ASF licenses this file\n", + "# to you under the Apache License, Version 2.0 (the\n", + "# \"License\"); you may not use this file except in compliance\n", + "# with the License. You may obtain a copy of the License at\n", + "#\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing,\n", + "# software distributed under the License is distributed on an\n", + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n", + "# KIND, either express or implied. See the License for the\n", + "# specific language governing permissions and limitations\n", + "# under the License" + ], + "metadata": { + "id": "lBuUTzxD2mvJ", + "cellView": "form" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "# **Natural Language Processing Pipeline**\n", + "\n", + "**Note**: This example is used from [here](https://github.com/rasalt/healthcarenlp/blob/main/nlp_public.ipynb).\n", + "\n", + "\n", + "\n", + "This example demonstrates how to set up an Apache Beam pipeline that reads a file from [Google Cloud Storage](https://https://cloud.google.com/storage), and calls the [Google Cloud Healthcare NLP API](https://cloud.google.com/healthcare-api/docs/how-tos/nlp) to extract information from unstructured data. This application can be used in contexts such as reading scanned clinical documents and extracting structure from it.\n", + "\n", + "An Apache Beam pipeline is a pipeline that reads input data, transforms that data, and writes output data. It consists of PTransforms and PCollections. A PCollection represents a distributed data set that your Beam pipeline operates on. A PTransform represents a data processing operation, or a step, in your pipeline. It takes one or more PCollections as input, performs a processing function that you provide on the elements of that PCollection, and produces zero or more output PCollection objects.\n", + "\n", + "For details about Apache Beam pipelines, including PTransforms and PCollections, visit the [Beam Programming Guide](https://beam.apache.org/documentation/programming-guide/).\n", + "\n", + "You'll be able to use this notebook to explore the data in each PCollection." + ], + "metadata": { + "id": "nEUAYCTx4Ijj" + } + }, + { + "cell_type": "markdown", + "source": [ + "First, lets install the necessary packages." + ], + "metadata": { + "id": "ZLBB0PTG5CHw" + } + }, + { + "cell_type": "code", + "source": [ + "!pip install apache-beam[gcp]" + ], + "metadata": { + "id": "O7hq2sse8K4u" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + " **GCP Setup**" + ], + "metadata": { + "id": "5vQDhIv0E-LR" + } + }, + { + "cell_type": "markdown", + "source": [ + "1. Authenticate your notebook by `gcloud auth application-default login` in the Colab terminal.\n", + "\n", + "2. Run `gcloud config set project `" + ], + "metadata": { + "id": "DGYiBYfxsSCw" + } + }, + { + "cell_type": "markdown", + "source": [ + "Set the variables in the next cell based upon your project and preferences. The files referred to in this notebook nlpsample*.csv are in the format with one\n", + "blurb of clinical note.\n", + "\n", + "Note that below, **us-central1** is hardcoded as the location. This is because of the limited number of [locations](https://cloud.google.com/healthcare-api/docs/how-tos/nlp) the API currently supports." + ], + "metadata": { + "id": "D7lJqW2PRFcN" + } + }, + { + "cell_type": "code", + "source": [ + "DATASET=\"\"\n", + "TEMP_LOCATION=\"\"\n", + "PROJECT=''\n", + "LOCATION='us-central1'\n", + "URL=f'https://healthcare.googleapis.com/v1/projects/{PROJECT}/locations/{LOCATION}/services/nlp:analyzeEntities'\n", + "NLP_SERVICE=f'projects/{PROJECT}/locations/{LOCATION}/services/nlp'" + ], + "metadata": { + "id": "s9lhe5CZ5F3o" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "Then, download [this raw CSV file](https://github.com/socd06/medical-nlp/blob/master/data/test.csv), and then upload it into Colab. You should be able to view this file (*test.csv*) in the \"Files\" tab in Colab after uploading." + ], + "metadata": { + "id": "1IArtEm8QuCR" + } + }, + { + "cell_type": "markdown", + "source": [ + "**BigQuery Setup**\n", + "\n", + "We will be using BigQuery to warehouse the structured data revealed in the output of the Healthcare NLP API. For this purpose, we create 3 tables to organize the data. Specifically, these will be table entities, table relations, and table entity mentions, which are all outputs of interest from the Healthcare NLP API." + ], + "metadata": { + "id": "DI_Qkyn75LO-" + } + }, + { + "cell_type": "code", + "source": [ + "from google.cloud import bigquery\n", + "\n", + "# Construct a BigQuery client object.\n", + "\n", + "TABLE_ENTITY=\"entity\"\n", + "\n", + "\n", + "schemaEntity = [\n", + " bigquery.SchemaField(\"entityId\", \"STRING\", mode=\"NULLABLE\"),\n", + " bigquery.SchemaField(\"preferredTerm\", \"STRING\", mode=\"NULLABLE\"),\n", + " bigquery.SchemaField(\"vocabularyCodes\", \"STRING\", mode=\"REPEATED\"),\n", + "]\n", + "\n", + "\n", + "client = bigquery.Client()\n", + "\n", + "# Create Table IDs\n", + "table_ent = PROJECT+\".\"+DATASET+\".\"+TABLE_ENTITY\n", + "\n", + "\n", + "# If table exists, delete the tables.\n", + "client.delete_table(table_ent, not_found_ok=True)\n", + "\n", + "\n", + "# Create tables\n", + "\n", + "table = bigquery.Table(table_ent, schema=schemaEntity)\n", + "table = client.create_table(table) # Make an API request.\n", + "\n", + "print(\n", + " \"Created table {}.{}.{}\".format(table.project, table.dataset_id, table.table_id)\n", + ")" + ], + "metadata": { + "id": "bZDqtFVE5Wd_" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "from google.cloud import bigquery\n", + "\n", + "# Construct a BigQuery client object.\n", + "\n", + "TABLE_REL=\"relations\"\n", + "\n", + "schemaRelations = [\n", + " bigquery.SchemaField(\"subjectId\", \"STRING\", mode=\"NULLABLE\"),\n", + " bigquery.SchemaField(\"objectId\", \"STRING\", mode=\"NULLABLE\"),\n", + " bigquery.SchemaField(\"confidence\", \"FLOAT64\", mode=\"NULLABLE\"),\n", + " bigquery.SchemaField(\"id\", \"STRING\", mode=\"NULLABLE\"),\n", + "]\n", + "\n", + "client = bigquery.Client()\n", + "\n", + "# Create Table IDs\n", + "\n", + "table_rel = PROJECT+\".\"+DATASET+\".\"+TABLE_REL\n", + "\n", + "# If table exists, delete the tables.\n", + "\n", + "client.delete_table(table_rel, not_found_ok=True)\n", + "\n", + "# Create tables\n", + "\n", + "table = bigquery.Table(table_rel, schema=schemaRelations)\n", + "table = client.create_table(table) # Make an API request.\n", + "print(\n", + " \"Created table {}.{}.{}\".format(table.project, table.dataset_id, table.table_id)\n", + ")\n", + "\n", + "\n" + ], + "metadata": { + "id": "YK-G7uV5APuP" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "from google.cloud import bigquery\n", + "\n", + "# Construct a BigQuery client object.\n", + "\n", + "TABLE_ENTITYMENTIONS=\"entitymentions\"\n", + "\n", + "schemaEntityMentions = [\n", + " bigquery.SchemaField(\"mentionId\", \"STRING\", mode=\"NULLABLE\"),\n", + " bigquery.SchemaField(\"type\", \"STRING\", mode=\"NULLABLE\"),\n", + " bigquery.SchemaField(\n", + " \"text\",\n", + " \"RECORD\",\n", + " mode=\"NULLABLE\",\n", + " fields=[\n", + " bigquery.SchemaField(\"content\", \"STRING\", mode=\"NULLABLE\"),\n", + " bigquery.SchemaField(\"beginOffset\", \"INTEGER\", mode=\"NULLABLE\"),\n", + " ],\n", + " ),\n", + " bigquery.SchemaField(\n", + " \"linkedEntities\",\n", + " \"RECORD\",\n", + " mode=\"REPEATED\",\n", + " fields=[\n", + " bigquery.SchemaField(\"entityId\", \"STRING\", mode=\"NULLABLE\"),\n", + " ],\n", + " ),\n", + " bigquery.SchemaField(\n", + " \"temporalAssessment\",\n", + " \"RECORD\",\n", + " mode=\"NULLABLE\",\n", + " fields=[\n", + " bigquery.SchemaField(\"value\", \"STRING\", mode=\"NULLABLE\"),\n", + " bigquery.SchemaField(\"confidence\", \"FLOAT64\", mode=\"NULLABLE\"),\n", + " ],\n", + " ),\n", + " bigquery.SchemaField(\n", + " \"certaintyAssessment\",\n", + " \"RECORD\",\n", + " mode=\"NULLABLE\",\n", + " fields=[\n", + " bigquery.SchemaField(\"value\", \"STRING\", mode=\"NULLABLE\"),\n", + " bigquery.SchemaField(\"confidence\", \"FLOAT64\", mode=\"NULLABLE\"),\n", + " ],\n", + " ),\n", + " bigquery.SchemaField(\n", + " \"subject\",\n", + " \"RECORD\",\n", + " mode=\"NULLABLE\",\n", + " fields=[\n", + " bigquery.SchemaField(\"value\", \"STRING\", mode=\"NULLABLE\"),\n", + " bigquery.SchemaField(\"confidence\", \"FLOAT64\", mode=\"NULLABLE\"),\n", + " ],\n", + " ),\n", + " bigquery.SchemaField(\"confidence\", \"FLOAT64\", mode=\"NULLABLE\"),\n", + " bigquery.SchemaField(\"id\", \"STRING\", mode=\"NULLABLE\")\n", + "]\n", + "\n", + "client = bigquery.Client()\n", + "\n", + "# Create Table IDs\n", + "\n", + "table_mentions = PROJECT+\".\"+DATASET+\".\"+TABLE_ENTITYMENTIONS\n", + "\n", + "# If table exists, delete the tables.\n", + "\n", + "client.delete_table(table_mentions, not_found_ok=True)\n", + "\n", + "# Create tables\n", + "\n", + "table = bigquery.Table(table_mentions, schema=schemaEntityMentions)\n", + "table = client.create_table(table) # Make an API request.\n", + "print(\n", + " \"Created table {}.{}.{}\".format(table.project, table.dataset_id, table.table_id)\n", + ")" + ], + "metadata": { + "id": "R9IHgZKoAQWj" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "**Pipeline Setup**\n", + "\n", + "We will use InteractiveRunner in this notebook." + ], + "metadata": { + "id": "jc_iS_BP5aS4" + } + }, + { + "cell_type": "code", + "source": [ + "# Python's regular expression library\n", + "import re\n", + "from sys import argv\n", + "# Beam and interactive Beam imports\n", + "import apache_beam as beam\n", + "from apache_beam.runners.interactive.interactive_runner import InteractiveRunner\n", + "import apache_beam.runners.interactive.interactive_beam as ib\n", + "\n", + "#Reference https://cloud.google.com/dataflow/docs/guides/specifying-exec-params#python_1\n", + "from apache_beam.options.pipeline_options import PipelineOptions\n", + "\n", + "runnertype = \"InteractiveRunner\"\n", + "\n", + "options = PipelineOptions(\n", + " flags=argv,\n", + " runner=runnertype,\n", + " project=PROJECT,\n", + " job_name=\"my-healthcare-nlp-job\",\n", + " temp_location=TEMP_LOCATION,\n", + " region=LOCATION)" + ], + "metadata": { + "id": "07ct6kf55ihP" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "The following defines a `PTransform` named `ReadLinesFromText`, that extracts lines from a file." + ], + "metadata": { + "id": "dO1A9_WK5lb4" + } + }, + { + "cell_type": "code", + "source": [ + "class ReadLinesFromText(beam.PTransform):\n", + "\n", + " def __init__(self, file_pattern):\n", + " self._file_pattern = file_pattern\n", + "\n", + " def expand(self, pcoll):\n", + " return (pcoll.pipeline\n", + " | beam.io.ReadFromText(self._file_pattern))" + ], + "metadata": { + "id": "t5iDRKMK5n_B" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "The following sets up an Apache Beam pipeline with the *Interactive Runner*. The *Interactive Runner* is the runner suitable for running in notebooks. A runner is an execution engine for Apache Beam pipelines." + ], + "metadata": { + "id": "HI_HVB185sMQ" + } + }, + { + "cell_type": "code", + "source": [ + "p = beam.Pipeline(options = options)" + ], + "metadata": { + "id": "7osCZ1om5ql0" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "The following sets up a PTransform that extracts words from a Google Cloud Storage file that contains lines with each line containing a In our example, each line is a medical notes excerpt that will be passed through the Healthcare NLP API\n", + "\n", + "**\"|\"** is an overloaded operator that applies a PTransform to a PCollection to produce a new PCollection. Together with |, >> allows you to optionally name a PTransform.\n", + "\n", + "Usage:[PCollection] | [PTransform], **or** [PCollection] | [name] >> [PTransform]" + ], + "metadata": { + "id": "EaF8NfC_521y" + } + }, + { + "cell_type": "code", + "source": [ + "lines = p | 'read' >> ReadLinesFromText(\"test.csv\")" + ], + "metadata": { + "id": "2APAh6XQ6NYd", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 72 + }, + "outputId": "033c5110-fd5a-4da0-b59b-801a1ce9d3b1" + }, + "execution_count": null, + "outputs": [ + ] + }, + { + "cell_type": "markdown", + "source": [ + "We then write a **DoFn** that will invoke the [NLP API](https://cloud.google.com/healthcare-api/docs/how-tos/nlp)." + ], + "metadata": { + "id": "vM_FbhkbGI-E" + } + }, + { + "cell_type": "code", + "source": [ + "class InvokeNLP(beam.DoFn):\n", + "\n", + " def process(self, element):\n", + " # import requests\n", + " import uuid\n", + " from google.auth import compute_engine\n", + " credentials = compute_engine.Credentials()\n", + " from google.auth.transport.requests import AuthorizedSession\n", + " authed_session = AuthorizedSession(credentials)\n", + " url = URL\n", + " payload = {\n", + " 'nlp_service': NLP_SERVICE,\n", + " 'document_content': element\n", + " }\n", + " resp = authed_session.post(url, data=payload)\n", + " response = resp.json()\n", + " response['id'] = uuid.uuid4().hex[:8]\n", + " yield response\n", + "\n", + "class AnalyzeLines(beam.PTransform):\n", + " def expand(self, pcoll):\n", + " return (\n", + " pcoll\n", + " | \"Invoke NLP API\" >> beam.ParDo(InvokeNLP())\n", + " )" + ], + "metadata": { + "id": "3ZJ-0dex9WE5" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "From our elements, being processed, we will get the entity mentions, relationships, and entities respectively." + ], + "metadata": { + "id": "TeYxIlNgGdK0" + } + }, + { + "cell_type": "code", + "source": [ + "import json\n", + "from apache_beam import pvalue\n", + "\n", + "class breakUpEntities(beam.DoFn):\n", + " def process(self, element):\n", + " for e in element['entities']:\n", + " print(e)\n", + " yield e\n", + "\n", + "class getRelationships(beam.DoFn):\n", + " def process(self, element):\n", + " obj = {}\n", + " id = element['id']\n", + " for e in element['relationships']:\n", + " obj = e\n", + " obj['id'] = id\n", + " yield obj\n", + "\n", + "class getEntityMentions(beam.DoFn):\n", + " def process(self, element):\n", + " obj = {}\n", + " for e in element['entityMentions']:\n", + " e['id'] = element['id']\n", + " yield e\n" + ], + "metadata": { + "id": "3KZgUv3d6haf" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "from apache_beam.io.gcp.internal.clients import bigquery\n", + "\n", + "\n", + "table_spec = bigquery.TableReference(\n", + " projectId=PROJECT,\n", + " datasetId=DATASET,\n", + " tableId=TABLE_ENTITY)\n", + "\n", + "nlp_annotations = (lines\n", + " | \"Analyze\" >> AnalyzeLines()\n", + " )\n" + ], + "metadata": { + "id": "OkxgB2a-6iYN" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "We then write these results to [BigQuery](https://cloud.google.com/bigquery), a cloud data warehouse." + ], + "metadata": { + "id": "iTh65CXIGoQn" + } + }, + { + "cell_type": "code", + "source": [ + "resultsEntities = ( nlp_annotations\n", + " | \"Break\" >> beam.ParDo(breakUpEntities())\n", + " | \"WriteToBigQuery\" >> beam.io.WriteToBigQuery(\n", + " table_spec,\n", + " write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND,\n", + " create_disposition=beam.io.BigQueryDisposition.CREATE_NEVER)\n", + " )" + ], + "metadata": { + "id": "Q9GIyLeS6oAe" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "table_spec = bigquery.TableReference(\n", + " projectId=PROJECT,\n", + " datasetId=DATASET,\n", + " tableId=TABLE_REL)\n", + "\n", + "resultsRelationships = ( nlp_annotations\n", + " | \"GetRelationships\" >> beam.ParDo(getRelationships())\n", + " | \"WriteToBigQuery\" >> beam.io.WriteToBigQuery(\n", + " table_spec,\n", + " write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND,\n", + " create_disposition=beam.io.BigQueryDisposition.CREATE_NEVER)\n", + " )" + ], + "metadata": { + "id": "yOlHfkcT6s4y" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "table_spec = bigquery.TableReference(\n", + " projectId=PROJECT,\n", + " datasetId=DATASET,\n", + " tableId=TABLE_ENTITYMENTIONS)\n", + "\n", + "resultsEntityMentions = ( nlp_annotations\n", + " | \"GetEntityMentions\" >> beam.ParDo(getEntityMentions())\n", + " | \"WriteToBigQuery\" >> beam.io.WriteToBigQuery(\n", + " table_spec,\n", + " write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND,\n", + " create_disposition=beam.io.BigQueryDisposition.CREATE_NEVER)\n", + " )" + ], + "metadata": { + "id": "a6QxxnY890Za" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "You can see the job graph for the pipeline by doing:" + ], + "metadata": { + "id": "6rP2nO6Z60bt" + } + }, + { + "cell_type": "code", + "source": [ + "ib.show_graph(p)" + ], + "metadata": { + "id": "zQB5h1Zq6x8d", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 806 + }, + "outputId": "7885e493-fee8-402e-baf2-cbbf406a3eb9" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "\n", + " \n", + "
    \n", + "
    \n", + " Processing... show_graph\n", + "
    \n", + " " + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "G\n", + "\n", + "\n", + "\n", + "[10]: read\n", + "\n", + "[10]: read\n", + "\n", + "\n", + "\n", + "lines\n", + "\n", + "lines\n", + "\n", + "\n", + "\n", + "[10]: read->lines\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "[13]: Analyze\n", + "\n", + "[13]: Analyze\n", + "\n", + "\n", + "\n", + "lines->[13]: Analyze\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "nlp_annotations\n", + "\n", + "nlp_annotations\n", + "\n", + "\n", + "\n", + "[13]: Analyze->nlp_annotations\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "[14]: Break\n", + "\n", + "[14]: Break\n", + "\n", + "\n", + "\n", + "nlp_annotations->[14]: Break\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "[15]: GetRelationships\n", + "\n", + "[15]: GetRelationships\n", + "\n", + "\n", + "\n", + "nlp_annotations->[15]: GetRelationships\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "[16]: GetEntityMentions\n", + "\n", + "[16]: GetEntityMentions\n", + "\n", + "\n", + "\n", + "nlp_annotations->[16]: GetEntityMentions\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "pcoll3490\n", + "\n", + "\n", + "\n", + "\n", + "[14]: Break->pcoll3490\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "[14]: WriteToBigQuery\n", + "\n", + "[14]: WriteToBigQuery\n", + "\n", + "\n", + "\n", + "pcoll3490->[14]: WriteToBigQuery\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "pcoll628\n", + "\n", + "\n", + "\n", + "\n", + "[15]: GetRelationships->pcoll628\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "[15]: WriteToBigQuery\n", + "\n", + "[15]: WriteToBigQuery\n", + "\n", + "\n", + "\n", + "pcoll628->[15]: WriteToBigQuery\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "pcoll9933\n", + "\n", + "\n", + "\n", + "\n", + "[16]: GetEntityMentions->pcoll9933\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "[16]: WriteToBigQuery\n", + "\n", + "[16]: WriteToBigQuery\n", + "\n", + "\n", + "\n", + "pcoll9933->[16]: WriteToBigQuery\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "\n", + " if (typeof window.interactive_beam_jquery == 'undefined') {\n", + " var jqueryScript = document.createElement('script');\n", + " jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n", + " jqueryScript.type = 'text/javascript';\n", + " jqueryScript.onload = function() {\n", + " var datatableScript = document.createElement('script');\n", + " datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n", + " datatableScript.type = 'text/javascript';\n", + " datatableScript.onload = function() {\n", + " window.interactive_beam_jquery = jQuery.noConflict(true);\n", + " window.interactive_beam_jquery(document).ready(function($){\n", + " \n", + " $(\"#progress_indicator_fa6997b180fa86966dd888a7d59a34f7\").remove();\n", + " });\n", + " }\n", + " document.head.appendChild(datatableScript);\n", + " };\n", + " document.head.appendChild(jqueryScript);\n", + " } else {\n", + " window.interactive_beam_jquery(document).ready(function($){\n", + " \n", + " $(\"#progress_indicator_fa6997b180fa86966dd888a7d59a34f7\").remove();\n", + " });\n", + " }" + ] + }, + "metadata": {} + } + ] + } + ] +} diff --git a/examples/notebooks/tour-of-beam/dataframes.ipynb b/examples/notebooks/interactive-overview/dataframes.ipynb similarity index 100% rename from examples/notebooks/tour-of-beam/dataframes.ipynb rename to examples/notebooks/interactive-overview/dataframes.ipynb diff --git a/examples/notebooks/tour-of-beam/getting-started.ipynb b/examples/notebooks/interactive-overview/getting-started.ipynb similarity index 100% rename from examples/notebooks/tour-of-beam/getting-started.ipynb rename to examples/notebooks/interactive-overview/getting-started.ipynb diff --git a/examples/notebooks/tour-of-beam/reading-and-writing-data.ipynb b/examples/notebooks/interactive-overview/reading-and-writing-data.ipynb similarity index 100% rename from examples/notebooks/tour-of-beam/reading-and-writing-data.ipynb rename to examples/notebooks/interactive-overview/reading-and-writing-data.ipynb diff --git a/examples/notebooks/tour-of-beam/windowing.ipynb b/examples/notebooks/interactive-overview/windowing.ipynb similarity index 100% rename from examples/notebooks/tour-of-beam/windowing.ipynb rename to examples/notebooks/interactive-overview/windowing.ipynb diff --git a/gradle.properties b/gradle.properties index ccccee1420d4a..6bad220e641ba 100644 --- a/gradle.properties +++ b/gradle.properties @@ -30,8 +30,8 @@ signing.gnupg.useLegacyGpg=true # buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy. # To build a custom Beam version make sure you change it in both places, see # https://github.com/apache/beam/issues/21302. -version=2.49.0-SNAPSHOT -sdk_version=2.49.0.dev +version=2.52.0-SNAPSHOT +sdk_version=2.52.0.dev javaVersion=1.8 @@ -41,4 +41,4 @@ docker_image_default_repo_prefix=beam_ # supported flink versions flink_versions=1.12,1.13,1.14,1.15,1.16 # supported python versions -python_versions=3.7,3.8,3.9,3.10,3.11 +python_versions=3.8,3.9,3.10,3.11 diff --git a/gradle/OWNERS b/gradle/OWNERS deleted file mode 100644 index af32c311923a2..0000000000000 --- a/gradle/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lukecwik diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 7454180f2ae88..7f93135c49b76 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ae04661ee7334..ac72c34e8acc9 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 1b6c787337ffb..0adc8e1a53214 100755 --- a/gradlew +++ b/gradlew @@ -55,7 +55,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -80,13 +80,11 @@ do esac done -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" +# This is normally unused +# shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -133,22 +131,29 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac case $MAX_FD in #( '' | soft) :;; #( *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -193,6 +198,10 @@ if "$cygwin" || "$msys" ; then done fi + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + # Collect all arguments for the java command; # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of # shell script including quotes and variable substitutions, so put them in @@ -205,6 +214,12 @@ set -- \ org.gradle.wrapper.GradleWrapperMain \ "$@" +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + # Use "xargs" to parse quoted args. # # With -n1 it outputs one arg per line, with the quotes and backslashes removed. diff --git a/gradlew.bat b/gradlew.bat index 107acd32c4e68..93e3f59f135dd 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -14,7 +14,7 @@ @rem limitations under the License. @rem -@if "%DEBUG%" == "" @echo off +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -25,7 +25,8 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -40,7 +41,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute +if %ERRORLEVEL% equ 0 goto execute echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. @@ -75,13 +76,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal diff --git a/it/build.gradle b/it/build.gradle new file mode 100644 index 0000000000000..42a9ad9f4ee88 --- /dev/null +++ b/it/build.gradle @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.it', +) + +description = "Apache Beam :: IT" +ext.summary = "Integration test utilities suites." + +//These registrations exist to make our matrix Github Action simple to configure +tasks.register('GCSPerformanceTest') { + dependsOn(":it:google-cloud-platform:GCSPerformanceTest") +} + +tasks.register('BigTablePerformanceTest') { + dependsOn(":it:google-cloud-platform:BigTablePerformanceTest") +} + +tasks.register('BigQueryStorageApiStreamingPerformanceTest') { + dependsOn(":it:google-cloud-platform:BigQueryStorageApiStreamingPerformanceTest") +} \ No newline at end of file diff --git a/it/cassandra/build.gradle b/it/cassandra/build.gradle new file mode 100644 index 0000000000000..65ff622061ebb --- /dev/null +++ b/it/cassandra/build.gradle @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.it.cassandra', +) + +description = "Apache Beam :: IT :: Cassandra" +ext.summary = "Integration test utilities for Cassandra." + +dependencies { + implementation project(path: ":it:testcontainers", configuration: "shadow") + implementation project(path: ":it:truthmatchers", configuration: "shadow") + implementation library.java.testcontainers_cassandra + implementation 'com.datastax.oss:java-driver-core:4.15.0' + + // TODO: excluding Guava until Truth updates it to >32.1.x + testImplementation(library.java.truth) { + exclude group: 'com.google.guava', module: 'guava' + } + testImplementation library.java.mockito_core + testRuntimeOnly library.java.slf4j_simple +} diff --git a/it/cassandra/src/main/java/org/apache/beam/it/cassandra/CassandraResourceManager.java b/it/cassandra/src/main/java/org/apache/beam/it/cassandra/CassandraResourceManager.java new file mode 100644 index 0000000000000..f7bf15eb7646d --- /dev/null +++ b/it/cassandra/src/main/java/org/apache/beam/it/cassandra/CassandraResourceManager.java @@ -0,0 +1,321 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.cassandra; + +import static org.apache.beam.it.cassandra.CassandraResourceManagerUtils.generateKeyspaceName; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.DriverTimeoutException; +import com.datastax.oss.driver.api.core.cql.ResultSet; +import com.datastax.oss.driver.api.core.cql.Row; +import com.datastax.oss.driver.api.core.cql.SimpleStatement; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import dev.failsafe.Failsafe; +import dev.failsafe.RetryPolicy; +import java.net.InetSocketAddress; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; +import org.apache.beam.it.common.ResourceManager; +import org.apache.beam.it.common.utils.ExceptionUtils; +import org.apache.beam.it.testcontainers.TestContainerResourceManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.CassandraContainer; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + +/** + * Client for managing Cassandra resources. + * + *

    The class supports one database and multiple collections per database object. A database is + * created when the first collection is created if one has not been created already. + * + *

    The database name is formed using testId. The database name will be "{testId}-{ISO8601 time, + * microsecond precision}", with additional formatting. + * + *

    The class is thread-safe. + */ +public class CassandraResourceManager extends TestContainerResourceManager> + implements ResourceManager { + + private static final Logger LOG = LoggerFactory.getLogger(CassandraResourceManager.class); + + private static final String DEFAULT_CASSANDRA_CONTAINER_NAME = "cassandra"; + + // A list of available Cassandra Docker image tags can be found at + // https://hub.docker.com/_/cassandra/tags + private static final String DEFAULT_CASSANDRA_CONTAINER_TAG = "4.1.0"; + + // 9042 is the default port that Cassandra is configured to listen on + private static final int CASSANDRA_INTERNAL_PORT = 9042; + + private final CqlSession cassandraClient; + private final String keyspaceName; + private final boolean usingStaticDatabase; + + private CassandraResourceManager(Builder builder) { + this( + /* cassandraClient= */ null, + new CassandraContainer<>( + DockerImageName.parse(builder.containerImageName).withTag(builder.containerImageTag)), + builder); + } + + @VisibleForTesting + @SuppressWarnings("nullness") + CassandraResourceManager( + @Nullable CqlSession cassandraClient, CassandraContainer container, Builder builder) { + super(container, builder); + + this.usingStaticDatabase = builder.keyspaceName != null; + this.keyspaceName = + usingStaticDatabase ? builder.keyspaceName : generateKeyspaceName(builder.testId); + this.cassandraClient = + cassandraClient == null + ? CqlSession.builder() + .addContactPoint( + new InetSocketAddress(this.getHost(), this.getPort(CASSANDRA_INTERNAL_PORT))) + .withLocalDatacenter("datacenter1") + .build() + : cassandraClient; + + if (!usingStaticDatabase) { + // Keyspace request may timeout on a few environments, if Cassandra is warming up + Failsafe.with(buildRetryPolicy()) + .run( + () -> + this.cassandraClient.execute( + String.format( + "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class':'SimpleStrategy', 'replication_factor':1}", + this.keyspaceName))); + } + } + + public static Builder builder(String testId) { + return new Builder(testId); + } + + /** Returns the port to connect to the Cassandra Database. */ + public int getPort() { + return super.getPort(CASSANDRA_INTERNAL_PORT); + } + + /** + * Returns the name of the Database that this Cassandra manager will operate in. + * + * @return the name of the Cassandra Database. + */ + public synchronized String getKeyspaceName() { + return keyspaceName; + } + + /** + * Execute the given statement on the managed keyspace. + * + * @param statement The statement to execute. + * @return ResultSet from Cassandra. + */ + public synchronized ResultSet executeStatement(String statement) { + LOG.info("Executing statement: {}", statement); + + try { + return Failsafe.with(buildRetryPolicy()) + .get( + () -> + cassandraClient.execute( + SimpleStatement.newInstance(statement).setKeyspace(this.keyspaceName))); + } catch (Exception e) { + throw new CassandraResourceManagerException("Error reading collection.", e); + } + } + + /** + * Inserts the given Document into a table. + * + *

    A database will be created here, if one does not already exist. + * + * @param tableName The name of the table to insert the document into. + * @param document The document to insert into the table. + * @return A boolean indicating whether the Document was inserted successfully. + */ + public synchronized boolean insertDocument(String tableName, Map document) { + return insertDocuments(tableName, ImmutableList.of(document)); + } + + /** + * Inserts the given Documents into a collection. + * + *

    Note: Implementations may do collection creation here, if one does not already exist. + * + * @param tableName The name of the collection to insert the documents into. + * @param documents A list of documents to insert into the collection. + * @return A boolean indicating whether the Documents were inserted successfully. + * @throws CassandraResourceManagerException if there is an error inserting the documents. + */ + public synchronized boolean insertDocuments(String tableName, List> documents) + throws CassandraResourceManagerException { + LOG.info( + "Attempting to write {} documents to {}.{}.", documents.size(), keyspaceName, tableName); + + try { + for (Map document : documents) { + executeStatement(createInsertStatement(tableName, document)); + } + } catch (Exception e) { + throw new CassandraResourceManagerException("Error inserting documents.", e); + } + + LOG.info("Successfully wrote {} documents to {}.{}", documents.size(), keyspaceName, tableName); + + return true; + } + + /** + * Reads all the Documents in a collection. + * + * @param tableName The name of the collection to read from. + * @return An iterable of all the Documents in the collection. + * @throws CassandraResourceManagerException if there is an error reading the collection. + */ + public synchronized Iterable readTable(String tableName) + throws CassandraResourceManagerException { + LOG.info("Reading all documents from {}.{}", keyspaceName, tableName); + + Iterable documents; + try { + ResultSet resultSet = executeStatement(String.format("SELECT * FROM %s", tableName)); + documents = resultSet.all(); + } catch (Exception e) { + throw new CassandraResourceManagerException("Error reading table.", e); + } + + LOG.info("Successfully loaded documents from {}.{}", keyspaceName, tableName); + + return documents; + } + + @Override + public synchronized void cleanupAll() { + LOG.info("Attempting to cleanup Cassandra manager."); + + boolean producedError = false; + + // First, delete the database if it was not given as a static argument + if (!usingStaticDatabase) { + try { + executeStatement(String.format("DROP KEYSPACE IF EXISTS %s", this.keyspaceName)); + } catch (Exception e) { + LOG.error("Failed to drop Cassandra keyspace {}.", keyspaceName, e); + + // Only bubble exception if the cause is not timeout or does not exist + if (!ExceptionUtils.containsType(e, DriverTimeoutException.class) + && !ExceptionUtils.containsMessage(e, "does not exist")) { + producedError = true; + } + } + } + + // Next, try to close the Cassandra client connection + try { + cassandraClient.close(); + } catch (Exception e) { + LOG.error("Failed to delete Cassandra client.", e); + producedError = true; + } + + // Throw Exception at the end if there were any errors + if (producedError) { + throw new CassandraResourceManagerException( + "Failed to delete resources. Check above for errors."); + } + + super.cleanupAll(); + + LOG.info("Cassandra manager successfully cleaned up."); + } + + private String createInsertStatement(String tableName, Map map) { + StringBuilder columns = new StringBuilder(); + StringBuilder values = new StringBuilder(); + + for (Map.Entry entry : map.entrySet()) { + columns.append(entry.getKey()).append(", "); + + // add quotes around strings + if (entry.getValue() instanceof String) { + values.append("'").append(entry.getValue()).append("'"); + } else { + values.append(entry.getValue()); + } + values.append(", "); + } + + // Remove trailing comma and space + if (!map.isEmpty()) { + columns.delete(columns.length() - 2, columns.length()); + values.delete(values.length() - 2, values.length()); + } + + return String.format("INSERT INTO %s (%s) VALUES (%s)", tableName, columns, values); + } + + private static RetryPolicy buildRetryPolicy() { + return RetryPolicy.builder() + .withMaxRetries(5) + .withDelay(Duration.ofSeconds(1)) + .handle(DriverTimeoutException.class) + .build(); + } + + /** Builder for {@link CassandraResourceManager}. */ + public static final class Builder + extends TestContainerResourceManager.Builder { + + private @Nullable String keyspaceName; + + private Builder(String testId) { + super(testId, DEFAULT_CASSANDRA_CONTAINER_NAME, DEFAULT_CASSANDRA_CONTAINER_TAG); + this.keyspaceName = null; + } + + /** + * Sets the keyspace name to that of a static database instance. Use this method only when + * attempting to operate on a pre-existing Cassandra database. + * + *

    Note: if a database name is set, and a static Cassandra server is being used + * (useStaticContainer() is also called on the builder), then a database will be created on the + * static server if it does not exist, and it will not be removed when cleanupAll() is called on + * the CassandraResourceManager. + * + * @param keyspaceName The database name. + * @return this builder object with the database name set. + */ + public Builder setKeyspaceName(String keyspaceName) { + this.keyspaceName = keyspaceName; + return this; + } + + @Override + public CassandraResourceManager build() { + return new CassandraResourceManager(this); + } + } +} diff --git a/it/cassandra/src/main/java/org/apache/beam/it/cassandra/CassandraResourceManagerException.java b/it/cassandra/src/main/java/org/apache/beam/it/cassandra/CassandraResourceManagerException.java new file mode 100644 index 0000000000000..f4bee393e75a7 --- /dev/null +++ b/it/cassandra/src/main/java/org/apache/beam/it/cassandra/CassandraResourceManagerException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.cassandra; + +/** Custom exception for {@link CassandraResourceManager} implementations. */ +public class CassandraResourceManagerException extends RuntimeException { + + public CassandraResourceManagerException(String errorMessage, Throwable err) { + super(errorMessage, err); + } + + public CassandraResourceManagerException(String errorMessage) { + super(errorMessage); + } +} diff --git a/it/cassandra/src/main/java/org/apache/beam/it/cassandra/CassandraResourceManagerUtils.java b/it/cassandra/src/main/java/org/apache/beam/it/cassandra/CassandraResourceManagerUtils.java new file mode 100644 index 0000000000000..ef617de518b13 --- /dev/null +++ b/it/cassandra/src/main/java/org/apache/beam/it/cassandra/CassandraResourceManagerUtils.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.cassandra; + +import static org.apache.beam.it.common.utils.ResourceManagerUtils.generateResourceId; + +import java.time.format.DateTimeFormatter; +import java.util.regex.Pattern; + +/** Utilities for {@link CassandraResourceManager} implementations. */ +final class CassandraResourceManagerUtils { + + private static final int MAX_DATABASE_NAME_LENGTH = 63; + private static final Pattern ILLEGAL_DATABASE_NAME_CHARS = + Pattern.compile("[/\\\\. \"\0$]"); // i.e. [/\. "$] + private static final String REPLACE_DATABASE_NAME_CHAR = "-"; + private static final DateTimeFormatter TIME_FORMAT = + DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss"); + + private CassandraResourceManagerUtils() {} + + /** + * Generates a Cassandra keyspace name from a given string. + * + * @param baseString The string to generate the name from. + * @return The keyspace name string. + */ + static String generateKeyspaceName(String baseString) { + return generateResourceId( + baseString, + ILLEGAL_DATABASE_NAME_CHARS, + REPLACE_DATABASE_NAME_CHAR, + MAX_DATABASE_NAME_LENGTH, + TIME_FORMAT) + .replace('-', '_'); + } +} diff --git a/it/cassandra/src/main/java/org/apache/beam/it/cassandra/matchers/CassandraAsserts.java b/it/cassandra/src/main/java/org/apache/beam/it/cassandra/matchers/CassandraAsserts.java new file mode 100644 index 0000000000000..6aecc6609cfb4 --- /dev/null +++ b/it/cassandra/src/main/java/org/apache/beam/it/cassandra/matchers/CassandraAsserts.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.cassandra.matchers; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatRecords; + +import com.datastax.oss.driver.api.core.cql.ColumnDefinition; +import com.datastax.oss.driver.api.core.cql.Row; +import com.datastax.oss.driver.api.core.type.DataTypes; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.truthmatchers.RecordsSubject; + +public class CassandraAsserts { + + /** + * Convert Cassandra {@link com.datastax.oss.driver.api.core.cql.Row} list to a list of maps. + * + * @param rows Rows to parse. + * @return List of maps to use in {@link RecordsSubject}. + */ + @SuppressWarnings("nullness") + public static List> cassandraRowsToRecords(Iterable rows) { + try { + List> records = new ArrayList<>(); + + for (Row row : rows) { + Map converted = new HashMap<>(); + for (ColumnDefinition columnDefinition : row.getColumnDefinitions()) { + + Object value = null; + if (columnDefinition.getType().equals(DataTypes.TEXT)) { + value = row.getString(columnDefinition.getName()); + } else if (columnDefinition.getType().equals(DataTypes.INT)) { + value = row.getInt(columnDefinition.getName()); + } + converted.put(columnDefinition.getName().toString(), value); + } + records.add(converted); + } + + return records; + } catch (Exception e) { + throw new RuntimeException("Error converting Cassandra Rows to Records", e); + } + } + + /** + * Creates a {@link RecordsSubject} to assert information within a list of records. + * + * @param rows Records in Cassandra's {@link Row} format to use in the comparison. + * @return Truth subject to chain assertions on. + */ + public static RecordsSubject assertThatCassandraRecords(Iterable rows) { + return assertThatRecords(cassandraRowsToRecords(rows)); + } +} diff --git a/it/cassandra/src/main/java/org/apache/beam/it/cassandra/matchers/package-info.java b/it/cassandra/src/main/java/org/apache/beam/it/cassandra/matchers/package-info.java new file mode 100644 index 0000000000000..c5045c46c738e --- /dev/null +++ b/it/cassandra/src/main/java/org/apache/beam/it/cassandra/matchers/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for Cassandra Truth matchers / subjects to have reusable assertions. */ +package org.apache.beam.it.cassandra.matchers; diff --git a/it/cassandra/src/main/java/org/apache/beam/it/cassandra/package-info.java b/it/cassandra/src/main/java/org/apache/beam/it/cassandra/package-info.java new file mode 100644 index 0000000000000..5f32163465523 --- /dev/null +++ b/it/cassandra/src/main/java/org/apache/beam/it/cassandra/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing Cassandra resources within integration tests. */ +package org.apache.beam.it.cassandra; diff --git a/it/cassandra/src/test/java/org/apache/beam/it/cassandra/CassandraResourceManagerIT.java b/it/cassandra/src/test/java/org/apache/beam/it/cassandra/CassandraResourceManagerIT.java new file mode 100644 index 0000000000000..99d02a24b6ee1 --- /dev/null +++ b/it/cassandra/src/test/java/org/apache/beam/it/cassandra/CassandraResourceManagerIT.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.cassandra; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.cassandra.matchers.CassandraAsserts.assertThatCassandraRecords; + +import com.datastax.oss.driver.api.core.cql.Row; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.testcontainers.TestContainersIntegrationTest; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Integration tests for {@link CassandraResourceManager}. */ +@Category(TestContainersIntegrationTest.class) +@RunWith(JUnit4.class) +public class CassandraResourceManagerIT { + + private CassandraResourceManager cassandraResourceManager; + + @Before + public void setUp() throws IOException { + cassandraResourceManager = CassandraResourceManager.builder("dummy").build(); + } + + @Test + public void testResourceManagerE2E() { + + List> records = new ArrayList<>(); + records.add(ImmutableMap.of("id", 1, "company", "Google")); + records.add(ImmutableMap.of("id", 2, "company", "Alphabet")); + + cassandraResourceManager.executeStatement( + "CREATE TABLE dummy_insert ( id int PRIMARY KEY, company text )"); + + boolean insertDocuments = cassandraResourceManager.insertDocuments("dummy_insert", records); + assertThat(insertDocuments).isTrue(); + + Iterable fetchRecords = cassandraResourceManager.readTable("dummy_insert"); + assertThatCassandraRecords(fetchRecords).hasRows(2); + assertThatCassandraRecords(fetchRecords).hasRecordsUnordered(records); + } + + @After + public void tearDown() { + if (cassandraResourceManager != null) { + cassandraResourceManager.cleanupAll(); + } + } +} diff --git a/it/cassandra/src/test/java/org/apache/beam/it/cassandra/CassandraResourceManagerTest.java b/it/cassandra/src/test/java/org/apache/beam/it/cassandra/CassandraResourceManagerTest.java new file mode 100644 index 0000000000000..fe00457159fa5 --- /dev/null +++ b/it/cassandra/src/test/java/org/apache/beam/it/cassandra/CassandraResourceManagerTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.cassandra; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.SimpleStatement; +import java.io.IOException; +import java.util.HashMap; +import java.util.concurrent.RejectedExecutionException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.testcontainers.containers.CassandraContainer; + +/** Unit tests for {@link CassandraResourceManager}. */ +@RunWith(JUnit4.class) +public class CassandraResourceManagerTest { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock private CqlSession cassandraClient; + @Mock private CassandraContainer container; + + private static final String TEST_ID = "test-id"; + private static final String COLLECTION_NAME = "collection-name"; + private static final String STATIC_KEYSPACE_NAME = "keyspace"; + private static final String HOST = "localhost"; + + private CassandraResourceManager testManager; + + @Before + public void setUp() { + doReturn(container).when(container).withLogConsumer(any()); + + testManager = + new CassandraResourceManager( + cassandraClient, container, CassandraResourceManager.builder(TEST_ID)); + } + + @Test + public void testGetUriShouldReturnCorrectValue() { + assertThat(testManager.getHost()).matches(HOST); + } + + @Test + public void testGetKeyspaceNameShouldReturnCorrectValue() { + assertThat(testManager.getKeyspaceName()).matches(TEST_ID.replace('-', '_') + "_\\d{8}_\\d{6}"); + } + + @Test + public void testInsertDocumentsShouldThrowErrorWhenCassandraThrowsException() { + + doThrow(RejectedExecutionException.class) + .when(cassandraClient) + .execute(any(SimpleStatement.class)); + + assertThrows( + CassandraResourceManagerException.class, + () -> testManager.insertDocument(COLLECTION_NAME, new HashMap<>())); + } + + @Test + public void testCleanupAllShouldNotDropStaticDatabase() throws IOException { + CassandraResourceManager.Builder builder = + CassandraResourceManager.builder(TEST_ID).setKeyspaceName(STATIC_KEYSPACE_NAME); + CassandraResourceManager tm = new CassandraResourceManager(cassandraClient, container, builder); + + tm.cleanupAll(); + + verify(cassandraClient, never()).execute(any(SimpleStatement.class)); + verify(cassandraClient).close(); + } + + @Test + public void testCleanupShouldDropNonStaticDatabase() { + + testManager.cleanupAll(); + + verify(cassandraClient).execute(any(SimpleStatement.class)); + verify(cassandraClient).close(); + } + + @Test + public void testCleanupAllShouldThrowErrorWhenCassandraClientFailsToDropDatabase() { + doThrow(RuntimeException.class).when(cassandraClient).execute(any(SimpleStatement.class)); + + assertThrows(CassandraResourceManagerException.class, () -> testManager.cleanupAll()); + } + + @Test + public void testCleanupAllShouldThrowErrorWhenCassandraClientFailsToClose() { + doThrow(RuntimeException.class).when(cassandraClient).close(); + + assertThrows(CassandraResourceManagerException.class, () -> testManager.cleanupAll()); + } +} diff --git a/it/common/build.gradle b/it/common/build.gradle new file mode 100644 index 0000000000000..7d1159cd99b77 --- /dev/null +++ b/it/common/build.gradle @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.module' } +applyJavaNature( + exportJavadoc: false, + automaticModuleName: 'org.apache.beam.it.common', + validateShadowJar: false, + shadowClosure: {}, +) + +description = "Apache Beam :: IT :: Common" +ext.summary = "Code used by all integration test utilities." + +dependencies { + implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) + implementation project(path: ":sdks:java:core", configuration: "shadow") + implementation library.java.google_api_services_dataflow + implementation library.java.google_auth_library_credentials + implementation library.java.google_auth_library_oauth2_http + implementation library.java.vendored_guava_32_1_2_jre + implementation library.java.slf4j_api + implementation library.java.commons_lang3 + implementation library.java.failsafe + implementation library.java.google_code_gson + implementation library.java.google_http_client + implementation library.java.guava + + // TODO: excluding Guava until Truth updates it to >32.1.x + testImplementation(library.java.truth) { + exclude group: 'com.google.guava', module: 'guava' + } + testImplementation library.java.junit + testImplementation library.java.mockito_inline + testRuntimeOnly library.java.slf4j_simple +} diff --git a/it/common/src/main/java/org/apache/beam/it/common/PipelineLauncher.java b/it/common/src/main/java/org/apache/beam/it/common/PipelineLauncher.java new file mode 100644 index 0000000000000..6d1aeae21dd45 --- /dev/null +++ b/it/common/src/main/java/org/apache/beam/it/common/PipelineLauncher.java @@ -0,0 +1,453 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.common; + +import static org.apache.beam.it.common.utils.PipelineUtils.createJobName; + +import com.google.api.services.dataflow.model.Job; +import com.google.api.services.dataflow.model.JobMessage; +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; +import org.apache.beam.sdk.Pipeline; + +/** Client for working with Cloud Dataflow. */ +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/27438) +}) +public interface PipelineLauncher { + /** Enum representing Apache Beam SDKs. */ + enum Sdk { + JAVA("JAVA"), + PYTHON("PYTHON"), + GO("GO"); + + private final String text; + + Sdk(String text) { + this.text = text; + } + + @Override + public String toString() { + return text; + } + } + + /** Enum representing known Dataflow job states. */ + enum JobState { + UNKNOWN("JOB_STATE_UNKNOWN"), + STOPPED("JOB_STATE_STOPPED"), + RUNNING("JOB_STATE_RUNNING"), + DONE("JOB_STATE_DONE"), + FAILED("JOB_STATE_FAILED"), + CANCELLED("JOB_STATE_CANCELLED"), + UPDATED("JOB_STATE_UPDATED"), + DRAINING("JOB_STATE_DRAINING"), + DRAINED("JOB_STATE_DRAINED"), + PENDING("JOB_STATE_PENDING"), + CANCELLING("JOB_STATE_CANCELLING"), + QUEUED("JOB_STATE_QUEUED"), + RESOURCE_CLEANING_UP("JOB_STATE_RESOURCE_CLEANING_UP"); + + private static final String DATAFLOW_PREFIX = "JOB_STATE_"; + + /** States that indicate the job is getting ready to run. */ + public static final ImmutableSet PENDING_STATES = ImmutableSet.of(PENDING, QUEUED); + + /** States that indicate the job is running. */ + public static final ImmutableSet ACTIVE_STATES = ImmutableSet.of(RUNNING, UPDATED); + + /** States that indicate that the job is done. */ + public static final ImmutableSet DONE_STATES = + ImmutableSet.of(CANCELLED, DONE, DRAINED, STOPPED); + + /** States that indicate that the job has failed. */ + public static final ImmutableSet FAILED_STATES = ImmutableSet.of(FAILED); + + /** States that indicate that the job is in the process of finishing. */ + public static final ImmutableSet FINISHING_STATES = + ImmutableSet.of(DRAINING, CANCELLING, RESOURCE_CLEANING_UP); + + private final String text; + + JobState(String text) { + this.text = text; + } + + /** + * Parses the state from Dataflow. + * + *

    Always use this in place of valueOf. + */ + public static JobState parse(String fromDataflow) { + if (fromDataflow == null) { + return null; + } + return valueOf(fromDataflow.replace(DATAFLOW_PREFIX, "")); + } + + @Override + public String toString() { + return text; + } + } + + /** Config for starting a Dataflow job. */ + class LaunchConfig { + private final String jobName; + private final ImmutableMap parameters; + private final ImmutableMap environment; + private final @Nullable String specPath; + private final @Nullable Sdk sdk; + private final @Nullable String executable; + private final @Nullable String requirementsFile; + private final @Nullable Pipeline pipeline; + + private LaunchConfig(Builder builder) { + this.jobName = builder.jobName; + this.parameters = ImmutableMap.copyOf(builder.parameters); + this.environment = ImmutableMap.copyOf(builder.environment); + this.specPath = builder.specPath; + this.sdk = builder.sdk; + this.executable = builder.executable; + this.requirementsFile = builder.requirementsFile; + this.pipeline = builder.pipeline; + } + + public String jobName() { + return jobName; + } + + public ImmutableMap parameters() { + return parameters; + } + + public ImmutableMap environment() { + return environment; + } + + public @Nullable String getParameter(String key) { + return parameters.get(key); + } + + public @Nullable String specPath() { + return specPath; + } + + public @Nullable Sdk sdk() { + return sdk; + } + + public @Nullable String executable() { + return executable; + } + + public @Nullable String requirementsFile() { + return requirementsFile; + } + + public @Nullable Pipeline pipeline() { + return pipeline; + } + + public static Builder builderWithName(String jobName, String specPath) { + return new Builder(jobName, specPath); + } + + public static Builder builder(String testName, String specPath) { + return new Builder(createJobName(testName), specPath); + } + + public static Builder builder(String jobName) { + return builder(jobName, null); + } + + /** Builder for the {@link LaunchConfig}. */ + public static final class Builder { + private final String jobName; + private final String specPath; + private final Map environment; + private Map parameters; + private Sdk sdk; + private String executable; + private String requirementsFile; + private Pipeline pipeline; + + private Builder(String jobName, String specPath) { + this.jobName = jobName; + this.parameters = new HashMap<>(); + this.environment = new HashMap<>(); + this.specPath = specPath; + } + + public String getJobName() { + return jobName; + } + + public @Nullable String getParameter(String key) { + return parameters.get(key); + } + + public Builder setParameters(Map parameters) { + this.parameters = parameters; + return this; + } + + public Builder addParameter(String key, String value) { + parameters.put(key, value); + return this; + } + + public @Nullable Object getEnvironment(String key) { + return environment.get(key); + } + + public Builder addEnvironment(String key, Object value) { + environment.put(key, value); + return this; + } + + public @Nullable String getSpecPath() { + return specPath; + } + + public @Nullable Sdk getSdk() { + return sdk; + } + + public Builder setSdk(Sdk sdk) { + this.sdk = sdk; + return this; + } + + public @Nullable String getExecutable() { + return executable; + } + + public Builder setExecutable(String executable) { + this.executable = executable; + return this; + } + + public @Nullable String getRequirementsFile() { + return requirementsFile; + } + + public Builder setRequirementsFile(String requirementsFile) { + this.requirementsFile = requirementsFile; + return this; + } + + public @Nullable Pipeline getPipeline() { + return pipeline; + } + + public Builder setPipeline(Pipeline pipeline) { + this.pipeline = pipeline; + return this; + } + + public LaunchConfig build() { + return new LaunchConfig(this); + } + } + } + + /** Info about the job from what Dataflow returned. */ + @AutoValue + abstract class LaunchInfo { + public abstract String jobId(); + + public abstract String projectId(); + + public abstract String region(); + + public abstract JobState state(); + + public abstract String createTime(); + + public abstract String sdk(); + + public abstract String version(); + + public abstract String jobType(); + + public abstract String runner(); + + public abstract @Nullable String templateName(); + + public abstract @Nullable String templateType(); + + public abstract @Nullable String templateVersion(); + + public abstract @Nullable String pipelineName(); + + public abstract ImmutableMap parameters(); + + public static Builder builder() { + return new AutoValue_PipelineLauncher_LaunchInfo.Builder(); + } + + /** Builder for {@link LaunchInfo}. */ + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setProjectId(String value); + + public abstract Builder setJobId(String value); + + public abstract Builder setRegion(String value); + + public abstract Builder setState(JobState value); + + public abstract Builder setCreateTime(String value); + + public abstract Builder setSdk(String value); + + public abstract Builder setVersion(String value); + + public abstract Builder setJobType(String value); + + public abstract Builder setRunner(String value); + + public abstract Builder setTemplateName(@Nullable String value); + + public abstract Builder setTemplateType(@Nullable String value); + + public abstract Builder setTemplateVersion(@Nullable String value); + + public abstract Builder setPipelineName(@Nullable String value); + + public abstract Builder setParameters(ImmutableMap value); + + public abstract LaunchInfo build(); + } + } + + /** + * Launches a new Dataflow job. + * + * @param project the project to run the job in + * @param region the region to run the job in (e.g. us-east1) + * @param options options for configuring the job + * @return info about the request to launch a new job + * @throws IOException if there is an issue sending the request + */ + LaunchInfo launch(String project, String region, LaunchConfig options) throws IOException; + + /** + * Gets information of a job. + * + * @param project the project that the job is running under + * @param region the region that the job was launched in + * @param jobId the id of the job + * @return dataflow job information + * @throws IOException if there is an issue sending the request + */ + Job getJob(String project, String region, String jobId) throws IOException; + + /** + * Gets information of a job. + * + * @param project the project that the job is running under + * @param region the region that the job was launched in + * @param jobId the id of the job + * @param jobView the level of information requested in response. + * @return dataflow job information + * @throws IOException if there is an issue sending the request + */ + Job getJob(String project, String region, String jobId, String jobView) throws IOException; + + /** + * Gets the current status of a job. + * + * @param project the project that the job is running under + * @param region the region that the job was launched in + * @param jobId the id of the job + * @return the current state of the job + * @throws IOException if there is an issue sending the request + */ + JobState getJobStatus(String project, String region, String jobId) throws IOException; + + /** + * Gets the messages of a job. + * + * @param project the project that the job is running under + * @param region the region that the job was launched in + * @param jobId the id of the job + * @param minimumImportance indicates the importance of the message. + * @return the list of messages of the job. + */ + List listMessages( + String project, String region, String jobId, String minimumImportance); + + /** + * Cancels the given job. + * + * @param project the project that the job is running under + * @param region the region that the job was launched in + * @param jobId the id of the job to cancel + * @throws IOException if there is an issue sending the request + * @return Updated job instance + */ + Job cancelJob(String project, String region, String jobId) throws IOException; + + /** + * Drains the given job. + * + * @param project the project that the job is running under + * @param region the region that the job was launched in + * @param jobId the id of the job to drain + * @throws IOException if there is an issue sending the request + * @return Updated job instance + */ + Job drainJob(String project, String region, String jobId) throws IOException; + + /** + * Get the specified metric of the given job. + * + * @param project the project that the job is running under + * @param region the region that the job was launched in + * @param jobId the id of the job to query + * @param metricName metric name to query from dataflow + * @return value of the metric or null + * @throws IOException if there is an issue sending the request + */ + @Nullable + Double getMetric(String project, String region, String jobId, String metricName) + throws IOException; + + /** + * Get all metrics of the given job. + * + * @param project the project that the job is running under + * @param region the region that the job was launched in + * @param jobId the id of the job to query + * @return all metrics of the given job + * @throws IOException if there is an issue sending the request + */ + Map getMetrics(String project, String region, String jobId) throws IOException; + + /** Cancel all active jobs. */ + void cleanupAll() throws IOException; +} diff --git a/it/common/src/main/java/org/apache/beam/it/common/PipelineOperator.java b/it/common/src/main/java/org/apache/beam/it/common/PipelineOperator.java new file mode 100644 index 0000000000000..3f70d050062b7 --- /dev/null +++ b/it/common/src/main/java/org/apache/beam/it/common/PipelineOperator.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.common; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; + +import com.google.auto.value.AutoValue; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.function.Supplier; +import org.apache.beam.it.common.PipelineLauncher.JobState; +import org.apache.beam.sdk.function.ThrowingConsumer; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Utilities for managing Dataflow jobs. */ +public final class PipelineOperator { + + private static final Logger LOG = LoggerFactory.getLogger(PipelineOperator.class); + + /** The result of running an operation. */ + public enum Result { + CONDITION_MET, + LAUNCH_FINISHED, + LAUNCH_FAILED, + TIMEOUT + } + + private final PipelineLauncher client; + + public PipelineOperator(PipelineLauncher client) { + this.client = client; + } + + /** + * Waits until the given job is done, timing out it if runs for too long. + * + *

    If the job is a batch job, it should complete eventually. If it is a streaming job, this + * will time out unless the job is explicitly cancelled or drained. + * + * @param config the configuration for performing the operation + * @return the result, which will be {@link Result#LAUNCH_FINISHED}, {@link Result#LAUNCH_FAILED} + * or {@link Result#TIMEOUT} + */ + @SuppressWarnings("rawtypes") + public Result waitUntilDone(Config config) { + return finishOrTimeout( + config, + new Supplier[] {() -> false}, + () -> jobIsDone(config.project(), config.region(), config.jobId())); + } + + /** + * Waits until the given job is done, timing out it if runs for too long. In cases of timeout, the + * dataflow job is drained. + * + *

    If the job is a batch job, it should complete eventually. If it is a streaming job, this + * will time out unless the job is explicitly cancelled or drained. After timeout, the job will be + * drained. + * + *

    If the job is drained, this method will return once the drain call is finalized and the job + * is fully drained. + * + * @param config the configuration for performing the operation + * @return the result, which will be {@link Result#LAUNCH_FINISHED}, {@link Result#LAUNCH_FAILED} + * or {@link Result#TIMEOUT} + */ + public Result waitUntilDoneAndFinish(Config config) throws IOException { + Result result = waitUntilDone(config); + if (result == Result.TIMEOUT) { + drainJobAndFinish(config); + } + return result; + } + + /** + * Waits until a given condition is met OR when the job enters a state that indicates that it is + * done or ready to be done. + * + * @param config the configuration for performing operations + * @param conditionCheck a {@link Supplier} that will be called periodically to check if the + * condition is met + * @return the result, which could be any value in {@link Result} + */ + public Result waitForCondition(Config config, Supplier... conditionCheck) { + return finishOrTimeout( + config, + conditionCheck, + () -> jobIsDoneOrFinishing(config.project(), config.region(), config.jobId())); + } + + /** + * Waits until a given condition is met OR when a job enters a state that indicates that it is + * done or ready to be done. + * + * @see #waitForConditionsAndFinish(Config, Supplier[]) + */ + public Result waitForConditionAndFinish(Config config, Supplier conditionCheck) + throws IOException { + return waitForConditionsAndFinish(config, conditionCheck); + } + + /** + * Waits until a given condition is met OR when a job enters a state that indicates that it is + * done or ready to be done. + * + *

    If the condition was met before the job entered a done or finishing state, then this will + * drain the job and wait for the job to enter a done state. + * + * @param config the configuration for performing operations + * @param conditionChecks {@link Supplier} varargs that will be called periodically to check if + * the condition is met + * @return the result of waiting for the condition, not of waiting for the job to be done + * @throws IOException if there is an issue cancelling the job + */ + public Result waitForConditionsAndFinish(Config config, Supplier... conditionChecks) + throws IOException { + return waitForConditionAndExecute(config, conditionChecks, this::drainJobAndFinish); + } + + /** Similar to {@link #waitForConditionAndFinish} but cancels the job instead of draining. */ + public Result waitForConditionAndCancel(Config config, Supplier... conditionCheck) + throws IOException { + return waitForConditionAndExecute(config, conditionCheck, this::cancelJobAndFinish); + } + + private Result waitForConditionAndExecute( + Config config, + Supplier[] conditionCheck, + ThrowingConsumer executable) + throws IOException { + Result conditionStatus = waitForCondition(config, conditionCheck); + if (conditionStatus != Result.LAUNCH_FINISHED && conditionStatus != Result.LAUNCH_FAILED) { + executable.accept(config); + } + return conditionStatus; + } + + /** + * Drains the job and waits till it's drained. + * + * @param config the configuration for performing operations + * @return the result of waiting for the condition + * @throws IOException if DataflowClient fails while sending a request + */ + public Result drainJobAndFinish(Config config) throws IOException { + client.drainJob(config.project(), config.region(), config.jobId()); + return waitUntilDone(config); + } + + /** Similar to {@link #drainJobAndFinish} but cancels the job instead of draining. */ + public Result cancelJobAndFinish(Config config) throws IOException { + client.cancelJob(config.project(), config.region(), config.jobId()); + return waitUntilDone(config); + } + + private static Result finishOrTimeout( + Config config, Supplier[] conditionCheck, Supplier... stopChecking) { + Instant start = Instant.now(); + + boolean launchFinished = false; + + while (timeIsLeft(start, config.timeoutAfter())) { + LOG.debug("Checking if condition is met."); + try { + if (allMatch(conditionCheck)) { + LOG.info("Condition met!"); + return Result.CONDITION_MET; + } + } catch (Exception e) { + LOG.warn("Error happened when checking for condition", e); + } + + LOG.info("Condition was not met yet. Checking if job is finished."); + if (launchFinished) { + LOG.info("Launch was finished, stop checking."); + return Result.LAUNCH_FINISHED; + } + + if (allMatch(stopChecking)) { + LOG.info("Detected that launch was finished, checking conditions once more."); + launchFinished = true; + } else { + LOG.info( + "Job not finished and conditions not met. Will check again in {} seconds (total wait: {}s of max {}s)", + config.checkAfter().getSeconds(), + Duration.between(start, Instant.now()).getSeconds(), + config.timeoutAfter().getSeconds()); + } + try { + Thread.sleep(config.checkAfter().toMillis()); + } catch (InterruptedException e) { + LOG.warn("Wait interrupted. Checking now."); + } + } + + LOG.warn("Neither the condition or job completion were fulfilled on time."); + return Result.TIMEOUT; + } + + private boolean jobIsDone(String project, String region, String jobId) { + try { + JobState state = client.getJobStatus(project, region, jobId); + LOG.info("Job {} is in state {}", jobId, state); + if (JobState.FAILED_STATES.contains(state)) { + throw new RuntimeException( + String.format( + "Job ID %s under %s failed. Please check cloud console for more details.", + jobId, project)); + } + return JobState.DONE_STATES.contains(state); + } catch (IOException e) { + LOG.error("Failed to get current job state. Assuming not done.", e); + return false; + } + } + + private boolean jobIsDoneOrFinishing(String project, String region, String jobId) { + try { + JobState state = client.getJobStatus(project, region, jobId); + LOG.info("Job {} is in state {}", jobId, state); + if (JobState.FAILED_STATES.contains(state)) { + throw new RuntimeException( + String.format( + "Job ID %s under %s failed. Please check cloud console for more details.", + jobId, project)); + } + return JobState.DONE_STATES.contains(state) || JobState.FINISHING_STATES.contains(state); + } catch (IOException e) { + LOG.error("Failed to get current job state. Assuming not done.", e); + return false; + } + } + + private static boolean timeIsLeft(Instant start, Duration maxWaitTime) { + return Duration.between(start, Instant.now()).minus(maxWaitTime).isNegative(); + } + + /** + * Check if all checks return true, but makes sure that all of them are executed. This is + * important to have complete feedback of integration tests progress. + * + * @param checks Varargs with all checks to run. + * @return If all checks meet the criteria. + */ + private static boolean allMatch(Supplier... checks) { + boolean match = true; + for (Supplier check : checks) { + if (!check.get()) { + match = false; + } + } + return match; + } + + /** Configuration for running an operation. */ + @AutoValue + public abstract static class Config { + + public abstract String project(); + + public abstract String jobId(); + + public abstract String region(); + + public abstract Duration checkAfter(); + + public abstract Duration timeoutAfter(); + + // TODO(zhoufek): Also let users set the maximum number of exceptions. + + public static Builder builder() { + return new AutoValue_PipelineOperator_Config.Builder() + .setCheckAfter(Duration.ofSeconds(15)) + .setTimeoutAfter(Duration.ofMinutes(15)); + } + + /** Builder for a {@link Config}. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setProject(String value); + + public abstract Builder setRegion(String value); + + public abstract Builder setJobId(String value); + + public abstract Builder setCheckAfter(Duration value); + + public abstract Builder setTimeoutAfter(Duration value); + + abstract Config autoBuild(); + + public Config build() { + Config config = autoBuild(); + checkState(!Strings.isNullOrEmpty(config.project()), "Project must be set"); + checkState(!Strings.isNullOrEmpty(config.region()), "Region must be set"); + checkState(!Strings.isNullOrEmpty(config.jobId()), "Job id must be set"); + return config; + } + } + } +} diff --git a/it/common/src/main/java/org/apache/beam/it/common/ResourceManager.java b/it/common/src/main/java/org/apache/beam/it/common/ResourceManager.java new file mode 100644 index 0000000000000..7b27bf1fea2db --- /dev/null +++ b/it/common/src/main/java/org/apache/beam/it/common/ResourceManager.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.common; + +/** Common interface across resource managers. */ +public interface ResourceManager { + + /** Deletes all resources created by this instance of ResourceManager. */ + void cleanupAll(); +} diff --git a/it/common/src/main/java/org/apache/beam/it/common/TestProperties.java b/it/common/src/main/java/org/apache/beam/it/common/TestProperties.java new file mode 100644 index 0000000000000..664fa390866fd --- /dev/null +++ b/it/common/src/main/java/org/apache/beam/it/common/TestProperties.java @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.common; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; + +import com.google.auth.Credentials; +import com.google.auth.oauth2.AccessToken; +import com.google.auth.oauth2.ComputeEngineCredentials; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.auth.oauth2.ServiceAccountCredentials; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility for accessing system properties set for the test. + * + *

    There are two types of properties: those set on the command lines and those set as environment + * variables. Those set on the command line always follow a camelCase naming convention, and those + * set as environment variable always follow a CAPITALIZED_SNAKE_CASE naming convention. + */ +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/27438) +}) +public final class TestProperties { + private TestProperties() {} + + // For testability, it is normally best to expect each property from the command line. We should + // only expect an environment variable if we're trying to avoid an accidental log of the + // value. + + // From command line + public static final String ARTIFACT_BUCKET_KEY = "artifactBucket"; + public static final String PROJECT_KEY = "project"; + public static final String REGION_KEY = "region"; + public static final String STAGE_BUCKET = "stageBucket"; + public static final String EXPORT_DATASET_KEY = "exportDataset"; + public static final String EXPORT_PROJECT = "exportProject"; + public static final String EXPORT_TABLE_KEY = "exportTable"; + public static final String SPEC_PATH_KEY = "specPath"; + public static final String HOST_IP = "hostIp"; + // From environment variables + public static final String ACCESS_TOKEN_KEY = "DT_IT_ACCESS_TOKEN"; + + // Default values for optional properties + public static final String DEFAULT_REGION = "us-central1"; + + // Error messages + private static final String CLI_ERR_MSG = "-D%s is required on the command line"; + private static final String ENV_VAR_MSG = "%s is required as an environment variable"; + + private static final Logger LOG = LoggerFactory.getLogger(TestProperties.class); + + public static boolean hasAccessToken() { + return getProperty(ACCESS_TOKEN_KEY, null, Type.ENVIRONMENT_VARIABLE) != null; + } + + public static String accessToken() { + return getProperty(ACCESS_TOKEN_KEY, Type.ENVIRONMENT_VARIABLE, true); + } + + /** + * Create and return credentials based on whether access token was provided or not. + * + *

    If access token was provided, use the token for Bearer authentication. + * + *

    If not, use Application Default Credentials. Check + * https://cloud.google.com/docs/authentication/application-default-credentials for more + * information. + * + * @return Credentials. + */ + public static Credentials credentials() { + if (hasAccessToken()) { + return googleCredentials(); + } else { + return buildCredentialsFromEnv(); + } + } + + public static Credentials googleCredentials() { + Credentials credentials; + try { + if (hasAccessToken()) { + credentials = + new GoogleCredentials(new AccessToken(accessToken(), /* expirationTime= */ null)); + } else { + credentials = GoogleCredentials.getApplicationDefault(); + } + } catch (IOException e) { + throw new RuntimeException( + "Unable to get credentials! \n" + + "Please run the following command to set 60 minute access token, \n" + + "\t export DT_IT_ACCESS_TOKEN=$(gcloud auth application-default print-access-token) \n" + + "Please run the following command to set credentials using the gcloud command, " + + "\t gcloud auth application-default login"); + } + return credentials; + } + + public static boolean hasArtifactBucket() { + return getProperty(ARTIFACT_BUCKET_KEY, null, Type.PROPERTY) != null; + } + + public static String artifactBucket() { + return bucketNameOnly(getProperty(ARTIFACT_BUCKET_KEY, Type.PROPERTY, true)); + } + + public static String exportDataset() { + return getProperty(EXPORT_DATASET_KEY, Type.PROPERTY, false); + } + + public static String exportProject() { + return getProperty(EXPORT_PROJECT, Type.PROPERTY, false); + } + + public static String exportTable() { + return getProperty(EXPORT_TABLE_KEY, Type.PROPERTY, false); + } + + public static String project() { + return getProperty(PROJECT_KEY, Type.PROPERTY, true); + } + + public static String region() { + return getProperty(REGION_KEY, DEFAULT_REGION, Type.PROPERTY); + } + + public static String specPath() { + return getProperty(SPEC_PATH_KEY, Type.PROPERTY, false); + } + + public static boolean hasStageBucket() { + return getProperty(STAGE_BUCKET, null, Type.PROPERTY) != null; + } + + public static String stageBucket() { + return bucketNameOnly(getProperty(STAGE_BUCKET, Type.PROPERTY, false)); + } + + public static String hostIp() { + return getProperty(HOST_IP, "localhost", Type.PROPERTY); + } + + /** Gets a property or throws an exception if it is not found. */ + private static String getProperty(String name, Type type, boolean required) { + String value = getProperty(name, null, type); + + if (required) { + String errMsg = + type == Type.PROPERTY + ? String.format(CLI_ERR_MSG, name) + : String.format(ENV_VAR_MSG, name); + checkState(value != null, errMsg); + } + + return value; + } + + /** Gets a property or returns {@code defaultValue} if it is not found. */ + public static String getProperty(String name, @Nullable String defaultValue, Type type) { + String value = type == Type.PROPERTY ? System.getProperty(name) : System.getenv(name); + return value != null ? value : defaultValue; + } + + /** Defines the types of properties there may be. */ + public enum Type { + PROPERTY, + ENVIRONMENT_VARIABLE + } + + /** + * Infers the {@link Credentials} to use with Google services from the current environment + * settings. + * + *

    First, checks if {@link ServiceAccountCredentials#getApplicationDefault()} returns Compute + * Engine credentials, which means that it is running from a GCE instance and can use the Service + * Account configured for that VM. Will use that + * + *

    Secondly, it will try to get the environment variable + * GOOGLE_APPLICATION_CREDENTIALS, and use that Service Account if configured to + * doing so. The method {@link #getCredentialsStream()} will make sure to search for the specific + * file using both the file system and classpath. + * + *

    If GOOGLE_APPLICATION_CREDENTIALS is not configured, it will return the + * application default, which is often setup through gcloud auth application-default + * login. + */ + public static Credentials buildCredentialsFromEnv() { + try { + + // if on Compute Engine, return default credentials. + GoogleCredentials applicationDefault = ServiceAccountCredentials.getApplicationDefault(); + if (applicationDefault instanceof ComputeEngineCredentials) { + return applicationDefault; + } + + InputStream credentialsStream = getCredentialsStream(); + if (credentialsStream == null) { + return applicationDefault; + } + return ServiceAccountCredentials.fromStream(credentialsStream); + } catch (IOException e) { + throw new RuntimeException( + "Unable to get application default credentials! \n" + + "Check https://cloud.google.com/docs/authentication/application-default-credentials for more information."); + } + } + + private static InputStream getCredentialsStream() throws FileNotFoundException { + String credentialFile = System.getenv("GOOGLE_APPLICATION_CREDENTIALS"); + + if (credentialFile == null || credentialFile.isEmpty()) { + LOG.warn( + "Not found Google Cloud credentials: GOOGLE_APPLICATION_CREDENTIALS, assuming application" + + " default"); + return null; + } + + InputStream is = null; + + File credentialFileRead = new File(credentialFile); + if (credentialFileRead.exists()) { + is = new FileInputStream(credentialFile); + } + + if (is == null) { + is = TestProperties.class.getResourceAsStream(credentialFile); + } + + if (is == null) { + is = TestProperties.class.getResourceAsStream("/" + credentialFile); + } + + if (is == null) { + LOG.warn("Not found credentials with file name {}", credentialFile); + return null; + } + return is; + } + + /** + * There are cases in which users will pass a gs://{bucketName} or a gs://{bucketName}/path + * wrongly to a bucket name property. This will ensure that execution will run as expected + * considering some input variations. + * + * @param bucketName User input with the bucket name. + * @return Bucket name if parseable, or throw exception otherwise. + * @throws IllegalArgumentException If bucket name can not be handled as such. + */ + public static String bucketNameOnly(String bucketName) { + + String changedName = bucketName; + // replace leading gs:// + if (changedName.startsWith("gs://")) { + changedName = changedName.replaceFirst("gs://", ""); + } + // replace trailing slash + if (changedName.endsWith("/")) { + changedName = changedName.replaceAll("/$", ""); + } + + if (changedName.contains("/") || changedName.contains(":")) { + throw new IllegalArgumentException( + "Bucket name " + + bucketName + + " is invalid. It should only contain the name of the bucket (not a path or URL)."); + } + + return changedName; + } +} diff --git a/it/common/src/main/java/org/apache/beam/it/common/logging/LogStrings.java b/it/common/src/main/java/org/apache/beam/it/common/logging/LogStrings.java new file mode 100644 index 0000000000000..09cdd9a490645 --- /dev/null +++ b/it/common/src/main/java/org/apache/beam/it/common/logging/LogStrings.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.common.logging; + +import com.google.api.client.json.GenericJson; +import com.google.api.services.dataflow.model.Job; +import com.google.gson.GsonBuilder; +import java.util.Map; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; + +/** Utility for formatting different objects for easier readability in logs. */ +public final class LogStrings { + private LogStrings() {} + + /** Formats a Google API's {@link GenericJson} for pretty logging. */ + public static String formatForLogging(GenericJson genericJson) { + return formatForLogging(ImmutableMap.copyOf(genericJson)); + } + + /** + * Formats a Dataflow {@link Job} for pretty logging. + * + *

    Some information will be excluded from the logs in order to improve readability and avoid + * hitting log limits. + */ + public static String formatForLogging(Job job) { + // The environment and steps can really pollute the logging output, making it hard to read + // and potentially causing problems on systems with limits to how much logging is allowed. + Job simpleCopy = + new Job() + .setId(job.getId()) + .setName(job.getName()) + .setProjectId(job.getProjectId()) + .setLocation(job.getLocation()) + .setCreateTime(job.getCreateTime()) + .setCurrentStateTime(job.getCurrentStateTime()) + .setRequestedState(job.getRequestedState()) // For when we try to cancel it + .setCurrentState(job.getCurrentState()) + .setLabels(job.getLabels()) + .setJobMetadata(job.getJobMetadata()) + .setType(job.getType()); + return formatForLogging(ImmutableMap.copyOf(simpleCopy)); + } + + /** Formats a map for pretty logging. */ + public static String formatForLogging(Map map) { + return new GsonBuilder().setPrettyPrinting().create().toJson(map); + } +} diff --git a/it/common/src/main/java/org/apache/beam/it/common/logging/package-info.java b/it/common/src/main/java/org/apache/beam/it/common/logging/package-info.java new file mode 100644 index 0000000000000..914529774b63d --- /dev/null +++ b/it/common/src/main/java/org/apache/beam/it/common/logging/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for utilities to help make outputting readable logs easier. */ +package org.apache.beam.it.common.logging; diff --git a/it/common/src/main/java/org/apache/beam/it/common/package-info.java b/it/common/src/main/java/org/apache/beam/it/common/package-info.java new file mode 100644 index 0000000000000..4631955dc5ab0 --- /dev/null +++ b/it/common/src/main/java/org/apache/beam/it/common/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing common ResourceManager resources within integration tests. */ +package org.apache.beam.it.common; diff --git a/it/common/src/main/java/org/apache/beam/it/common/utils/ExceptionUtils.java b/it/common/src/main/java/org/apache/beam/it/common/utils/ExceptionUtils.java new file mode 100644 index 0000000000000..f0e34458545d8 --- /dev/null +++ b/it/common/src/main/java/org/apache/beam/it/common/utils/ExceptionUtils.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.common.utils; + +import javax.annotation.Nullable; + +/** Utility class for handling exceptions in tests or resource managers. */ +public class ExceptionUtils { + + /** + * Utility to check if the given exception or any of its causes contain a specific message. + * + * @param exception Exception to check. + * @param message Message to search for. + * @return true if the message is found in the exception or any of the causes, false otherwise. + */ + public static boolean containsMessage(@Nullable Throwable exception, String message) { + if (exception == null) { + return false; + } + + if (exception.getMessage() != null && exception.getMessage().contains(message)) { + return true; + } + + return containsMessage(exception.getCause(), message); + } + + /** + * Utility to check if the given exception or any of its causes have a specific type. + * + * @param exception Exception to check. + * @param type Type to search for. + * @return true if the type is found in the exception or any of the causes, false otherwise. + */ + public static boolean containsType( + @Nullable Throwable exception, Class type) { + if (exception == null) { + return false; + } + + if (type.isAssignableFrom(exception.getClass())) { + return true; + } + + return containsType(exception.getCause(), type); + } +} diff --git a/it/common/src/main/java/org/apache/beam/it/common/utils/IORedirectUtil.java b/it/common/src/main/java/org/apache/beam/it/common/utils/IORedirectUtil.java new file mode 100644 index 0000000000000..9ed6471525d74 --- /dev/null +++ b/it/common/src/main/java/org/apache/beam/it/common/utils/IORedirectUtil.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.common.utils; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import org.slf4j.Logger; + +/** IO Redirect Utility class. */ +public final class IORedirectUtil { + + /** + * Redirect an InputStream to a logger for testing and debugging purposes. + * + * @param inputStream The InputStream to redirect. + * @param log The logger to redirect the InputStream to. + */ + public static void redirectLinesLog(InputStream inputStream, Logger log) { + new Thread( + () -> { + try (InputStreamReader isr = + new InputStreamReader(inputStream, StandardCharsets.UTF_8); + BufferedReader bis = new BufferedReader(isr)) { + + String line; + while ((line = bis.readLine()) != null) { + log.info(line); + } + } catch (Exception e) { + log.error("Error redirecting stream", e); + } + }) + .start(); + } +} diff --git a/it/common/src/main/java/org/apache/beam/it/common/utils/PipelineUtils.java b/it/common/src/main/java/org/apache/beam/it/common/utils/PipelineUtils.java new file mode 100644 index 0000000000000..d249d43d3789d --- /dev/null +++ b/it/common/src/main/java/org/apache/beam/it/common/utils/PipelineUtils.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.common.utils; + +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.function.Supplier; +import org.apache.beam.sdk.PipelineResult; +import org.apache.beam.sdk.PipelineRunner; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CaseFormat; +import org.apache.commons.lang3.RandomStringUtils; + +/** Utilities to make working with Dataflow easier. */ +public class PipelineUtils { + + private PipelineUtils() {} + + /** + * Waits for a specified pipeline to reach a particular state, up to a specified timeout. + * + * @param pipeline the PipelineResult object to monitor + * @param expectedState the pipeline state to wait for + * @param timeoutMillis the maximum amount of time to wait for the expected state, in milliseconds + * @return true if the pipeline reaches the expected state within the timeout, false otherwise + * @throws InterruptedException if the thread is interrupted while waiting + */ + public static boolean waitUntilState( + PipelineResult pipeline, PipelineResult.State expectedState, Long timeoutMillis) + throws InterruptedException { + return waitUntil(pipeline, () -> pipeline.getState().equals(expectedState), timeoutMillis); + } + + /** + * Waits until a specified condition is true, up to a specified timeout. + * + * @param pipeline the PipelineResult object to monitor + * @param lambda a Supplier that returns a boolean indicating whether the condition is true + * @param timeoutMillis the maximum amount of time to wait for the condition to be true, in + * milliseconds + * @return true if the condition becomes true within the timeout, false otherwise + * @throws InterruptedException if the thread is interrupted while waiting + */ + public static boolean waitUntil( + PipelineResult pipeline, Supplier lambda, Long timeoutMillis) + throws InterruptedException { + Instant start = Instant.now(); + while (true) { + Thread.sleep(5_000); + if (lambda.get()) { + return true; + } + if (Duration.between(start, Instant.now()).compareTo(Duration.ofMillis(timeoutMillis)) > 0) { + return false; + } + } + } + + /** + * Creates a job name. Method uses {@link #createJobName(String, int)}} without a random suffix. + * + * @see #createJobName(String, int) + */ + public static String createJobName(String prefix) { + return createJobName(prefix, 0); + } + + /** + * Creates a job name. + * + *

    If there are uppercase characters in {@code prefix}, then this will convert them into a dash + * followed by the lowercase equivalent of that letter. + * + *

    The job name will normally be unique, but this is not guaranteed if multiple jobs with the + * same prefix are requested in a short period of time. + * + * @param prefix a prefix for the job + * @param randomChars if the string should contain random chars at the end, to increase the + * likelihood of being unique. + * @return the prefix plus some way of identifying it separate from other jobs with the same + * prefix + */ + public static String createJobName(String prefix, int randomChars) { + String convertedPrefix = + CaseFormat.UPPER_CAMEL.converterTo(CaseFormat.LOWER_HYPHEN).convert(prefix); + String formattedTimestamp = + DateTimeFormatter.ofPattern("yyyyMMddHHmmssSSS") + .withZone(ZoneId.of("UTC")) + .format(Instant.now()); + + String suffix = ""; + if (randomChars > 0) { + suffix = "-" + RandomStringUtils.randomAlphanumeric(randomChars).toLowerCase(); + } + return String.format("%s-%s%s", convertedPrefix, formattedTimestamp, suffix); + } + + /** Get raw job name (without prefix) from a jobName generated by createJobName. */ + public static String extractJobName(String nameWithPrefix) { + return nameWithPrefix.substring(0, nameWithPrefix.lastIndexOf("-")); + } + + /* + * Get runner class from name. + * + *

    This avoids having all runner dependencies (e.g. Dataflow, Flink, etc) explicitly in specific + * test. + * + * @param runner The runner name string + * @return runner class + */ + public static Class> getRunnerClass(String runner) { + PipelineOptions options = PipelineOptionsFactory.fromArgs("--runner=" + runner).create(); + return options.getRunner(); + } +} diff --git a/it/common/src/main/java/org/apache/beam/it/common/utils/ResourceManagerUtils.java b/it/common/src/main/java/org/apache/beam/it/common/utils/ResourceManagerUtils.java new file mode 100644 index 0000000000000..54f82dc7d9a0d --- /dev/null +++ b/it/common/src/main/java/org/apache/beam/it/common/utils/ResourceManagerUtils.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.common.utils; + +import static java.lang.Math.min; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing.goodFastHash; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Random; +import java.util.regex.Pattern; +import javax.annotation.Nullable; +import org.apache.beam.it.common.ResourceManager; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.HashFunction; +import org.apache.commons.lang3.RandomStringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Common utilities for ResourceManager implementations. */ +public class ResourceManagerUtils { + + private static final Logger LOG = LoggerFactory.getLogger(ResourceManagerUtils.class); + + private static final int MIN_PROJECT_ID_LENGTH = 4; + private static final int MAX_PROJECT_ID_LENGTH = 30; + private static final Pattern ILLEGAL_PROJECT_CHARS = Pattern.compile("[^a-zA-Z0-9-!:\\.']"); + private static final String TIME_ZONE = "UTC"; + + /** + * Generates a new id string from an existing one. + * + * @param id The id string to generate a new id from. + * @param targetLength The length of the new id to generate. Must be greater than 8. + */ + public static String generateNewId(String id, int targetLength) { + if (id.length() <= targetLength) { + return id; + } + + if (targetLength <= 8) { + throw new IllegalArgumentException("targetLength must be greater than 8"); + } + + HashFunction hashFunction = goodFastHash(32); + String hash = hashFunction.hashUnencodedChars(id).toString(); + return id.substring(0, targetLength - hash.length() - 1) + "-" + hash; + } + + /** + * Generates a generic resource id from a given string, avoiding characters specified in the + * illegalChars Pattern. The length of the generated string ID will not exceed the length + * specified by targetLength. + * + * @param baseString the base ID to generate the resource ID from. + * @param illegalChars a pattern of characters to remove from the generated ID. + * @param replaceChar the character to replace all illegal characters with. + * @param targetLength the max length of the generated ID. + * @return the generated resource ID. + */ + public static String generateResourceId( + String baseString, + Pattern illegalChars, + String replaceChar, + int targetLength, + DateTimeFormatter timeFormat) { + // first, make sure the baseString, typically the test ID, is not empty + checkArgument(baseString.length() != 0, "baseString cannot be empty."); + + // next, replace all illegal characters from given string with given replacement character + String illegalCharsRemoved = + illegalChars.matcher(baseString.toLowerCase()).replaceAll(replaceChar); + + // finally, append the date/time and return the substring that does not exceed the length limit + LocalDateTime localDateTime = LocalDateTime.now(ZoneId.of(TIME_ZONE)); + String timeAddOn = localDateTime.format(timeFormat); + return illegalCharsRemoved.subSequence( + 0, min(targetLength - timeAddOn.length() - 1, illegalCharsRemoved.length())) + + replaceChar + + localDateTime.format(timeFormat); + } + + /** Generates random letter for padding. */ + public static char generatePadding() { + Random random = new Random(); + return (char) ('a' + random.nextInt(26)); + } + + /** + * Checks whether the given project ID is valid according to GCP constraints. + * + * @param idToCheck the project ID to check. + * @throws IllegalArgumentException if the project ID is invalid. + */ + public static void checkValidProjectId(String idToCheck) { + if (idToCheck.length() < MIN_PROJECT_ID_LENGTH) { + throw new IllegalArgumentException("Project ID " + idToCheck + " cannot be empty."); + } + if (idToCheck.length() > MAX_PROJECT_ID_LENGTH) { + throw new IllegalArgumentException( + "Project ID " + + idToCheck + + " cannot be longer than " + + MAX_PROJECT_ID_LENGTH + + " characters."); + } + if (ILLEGAL_PROJECT_CHARS.matcher(idToCheck).find()) { + throw new IllegalArgumentException( + "Project ID " + + idToCheck + + " is not a valid ID. Only letters, numbers, hyphens, single quotes, colon, dot and" + + " exclamation points are allowed."); + } + } + + /** + * Cleanup Resources from the given ResourceManagers. It will guarantee that all the cleanups are + * invoked, but still throws / bubbles the first exception at the end if something went wrong. + * + * @param managers Varargs of the managers to clean + */ + public static void cleanResources(ResourceManager... managers) { + + if (managers == null || managers.length == 0) { + return; + } + + Exception bubbleException = null; + + for (ResourceManager manager : managers) { + if (manager == null) { + continue; + } + try { + LOG.info("Cleaning up resource manager {}", manager.getClass().getSimpleName()); + manager.cleanupAll(); + } catch (Exception e) { + LOG.error("Error cleaning the resource manager {}", manager.getClass().getSimpleName()); + if (bubbleException == null) { + bubbleException = e; + } + } + } + + if (bubbleException != null) { + throw new RuntimeException("Error cleaning up resources", bubbleException); + } + } + + /** + * Generates a password using random characters for tests. + * + *

    Note: The password generated is not cryptographically secure and should only be used in + * tests. + * + * @param minLength minimum length of password + * @param maxLength maximum length of password + * @param numLower number of lower case letters + * @param numUpper number of upper case letters + * @param numSpecial number of special characters + * @param specialChars special characters to use + * @return + */ + public static String generatePassword( + int minLength, + int maxLength, + int numLower, + int numUpper, + int numSpecial, + @Nullable List specialChars) { + StringBuilder password = new StringBuilder(); + password.append( + RandomStringUtils.randomAlphanumeric(minLength, maxLength - numSpecial).toUpperCase()); + for (int i = 0; i < numSpecial && specialChars != null; i++) { + password.insert( + new Random().nextInt(password.length()), + specialChars.get(new Random().nextInt(specialChars.size()))); + } + for (int i = 0; i < numLower; i++) { + password.insert( + new Random().nextInt(password.length()), + RandomStringUtils.randomAlphabetic(1).toLowerCase()); + } + for (int i = 0; i < numUpper; i++) { + password.insert( + new Random().nextInt(password.length()), + RandomStringUtils.randomAlphabetic(1).toUpperCase()); + } + return password.toString(); + } +} diff --git a/it/common/src/main/java/org/apache/beam/it/common/utils/RetryUtil.java b/it/common/src/main/java/org/apache/beam/it/common/utils/RetryUtil.java new file mode 100644 index 0000000000000..e51bd6d760f7d --- /dev/null +++ b/it/common/src/main/java/org/apache/beam/it/common/utils/RetryUtil.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.common.utils; + +import dev.failsafe.RetryPolicy; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.time.Duration; + +/** Utility Class related to retry operations. */ +public class RetryUtil { + + public static final int DEFAULT_BACKOFF_START_DELAY_MILLIS = 100; + public static final int DEFAULT_BACKOFF_MAX_DELAY_MILLIS = 5000; + public static final int DEFAULT_MAX_RETRIES = 3; + + private RetryUtil() {} + + /** + * Create a policy to retry when client returns a "Server is not responding" or a transient error. + * By default, it retries up to 3 times using a backoff strategy. + * + * @return Failsafe's RetryPolicy. + */ + public static RetryPolicy clientRetryPolicy() { + return RetryPolicy.builder() + .handle(IOException.class, SocketTimeoutException.class) + .handleIf( + throwable -> + throwable.getMessage() != null + && throwable.getMessage().contains("Server is not responding")) + .withBackoff( + Duration.ofMillis(DEFAULT_BACKOFF_START_DELAY_MILLIS), + Duration.ofMillis(DEFAULT_BACKOFF_MAX_DELAY_MILLIS)) + .withMaxRetries(DEFAULT_MAX_RETRIES) + .build(); + } +} diff --git a/it/common/src/main/java/org/apache/beam/it/common/utils/package-info.java b/it/common/src/main/java/org/apache/beam/it/common/utils/package-info.java new file mode 100644 index 0000000000000..de732c1b9736f --- /dev/null +++ b/it/common/src/main/java/org/apache/beam/it/common/utils/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for common utilities within integration tests. */ +package org.apache.beam.it.common.utils; diff --git a/it/common/src/test/java/org/apache/beam/it/common/PipelineOperatorTest.java b/it/common/src/test/java/org/apache/beam/it/common/PipelineOperatorTest.java new file mode 100644 index 0000000000000..480ca7660af5d --- /dev/null +++ b/it/common/src/test/java/org/apache/beam/it/common/PipelineOperatorTest.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.common; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.atMost; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.time.Duration; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import org.apache.beam.it.common.PipelineLauncher.JobState; +import org.apache.beam.it.common.PipelineOperator.Config; +import org.apache.beam.it.common.PipelineOperator.Result; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Unit tests for {@link PipelineOperator}. */ +@RunWith(JUnit4.class) +public final class PipelineOperatorTest { + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock private PipelineLauncher client; + + private static final String PROJECT = "test-project"; + private static final String REGION = "us-east1"; + private static final String JOB_ID = "test-job-id"; + private static final Duration CHECK_AFTER = Duration.ofMillis(100); + private static final Duration TIMEOUT_AFTER = Duration.ofSeconds(1); + + private static final Config DEFAULT_CONFIG = + Config.builder() + .setProject(PROJECT) + .setRegion(REGION) + .setJobId(JOB_ID) + .setCheckAfter(CHECK_AFTER) + .setTimeoutAfter(TIMEOUT_AFTER) + .build(); + + @Captor private ArgumentCaptor projectCaptor; + @Captor private ArgumentCaptor regionCaptor; + @Captor private ArgumentCaptor jobIdCaptor; + + @Test + public void testWaitUntilDone() throws IOException { + // Arrange + when(client.getJobStatus(any(), any(), any())) + .thenReturn(JobState.QUEUED) + .thenReturn(JobState.RUNNING) + .thenReturn(JobState.CANCELLING) + .thenReturn(JobState.CANCELLED); + + // Act + Result result = new PipelineOperator(client).waitUntilDone(DEFAULT_CONFIG); + + // Assert + verify(client, times(4)) + .getJobStatus(projectCaptor.capture(), regionCaptor.capture(), jobIdCaptor.capture()); + + Set allProjects = new HashSet<>(projectCaptor.getAllValues()); + Set allRegions = new HashSet<>(regionCaptor.getAllValues()); + Set allJobIds = new HashSet<>(jobIdCaptor.getAllValues()); + + assertThat(allProjects).containsExactly(PROJECT); + assertThat(allRegions).containsExactly(REGION); + assertThat(allJobIds).containsExactly(JOB_ID); + assertThat(result).isEqualTo(Result.LAUNCH_FINISHED); + } + + @Test + public void testWaitUntilDoneTimeout() throws IOException { + when(client.getJobStatus(any(), any(), any())).thenReturn(JobState.RUNNING); + Result result = new PipelineOperator(client).waitUntilDone(DEFAULT_CONFIG); + assertThat(result).isEqualTo(Result.TIMEOUT); + } + + @Test + public void testWaitForCondition() throws IOException { + AtomicInteger callCount = new AtomicInteger(); + int totalCalls = 3; + Supplier checker = () -> callCount.incrementAndGet() >= totalCalls; + when(client.getJobStatus(any(), any(), any())) + .thenReturn(JobState.RUNNING) + .thenThrow(new IOException()) + .thenReturn(JobState.RUNNING); + + Result result = new PipelineOperator(client).waitForCondition(DEFAULT_CONFIG, checker); + + verify(client, atMost(totalCalls)) + .getJobStatus(projectCaptor.capture(), regionCaptor.capture(), jobIdCaptor.capture()); + assertThat(projectCaptor.getValue()).isEqualTo(PROJECT); + assertThat(regionCaptor.getValue()).isEqualTo(REGION); + assertThat(jobIdCaptor.getValue()).isEqualTo(JOB_ID); + assertThat(result).isEqualTo(Result.CONDITION_MET); + } + + @Test + public void testWaitForConditionJobFinished() throws IOException { + when(client.getJobStatus(any(), any(), any())) + .thenReturn(JobState.RUNNING) + .thenReturn(JobState.CANCELLED); + + Result result = new PipelineOperator(client).waitForCondition(DEFAULT_CONFIG, () -> false); + + assertThat(result).isEqualTo(Result.LAUNCH_FINISHED); + } + + @Test + public void testWaitForConditionTimeout() throws IOException { + when(client.getJobStatus(any(), any(), any())).thenReturn(JobState.RUNNING); + + Result result = new PipelineOperator(client).waitForCondition(DEFAULT_CONFIG, () -> false); + + assertThat(result).isEqualTo(Result.TIMEOUT); + } + + @Test + public void testFinishAfterCondition() throws IOException { + // Arrange + AtomicInteger callCount = new AtomicInteger(); + int totalCalls = 3; + Supplier checker = () -> callCount.incrementAndGet() >= totalCalls; + + when(client.getJobStatus(any(), any(), any())) + .thenReturn(JobState.RUNNING) + .thenThrow(new IOException()) + .thenReturn(JobState.RUNNING) + .thenReturn(JobState.CANCELLING) + .thenReturn(JobState.CANCELLED); + + // Act + Result result = new PipelineOperator(client).waitForConditionAndFinish(DEFAULT_CONFIG, checker); + + // Assert + verify(client, atLeast(totalCalls)) + .getJobStatus(projectCaptor.capture(), regionCaptor.capture(), jobIdCaptor.capture()); + verify(client).drainJob(projectCaptor.capture(), regionCaptor.capture(), jobIdCaptor.capture()); + + Set allProjects = new HashSet<>(projectCaptor.getAllValues()); + Set allRegions = new HashSet<>(regionCaptor.getAllValues()); + Set allJobIds = new HashSet<>(jobIdCaptor.getAllValues()); + + assertThat(allProjects).containsExactly(PROJECT); + assertThat(allRegions).containsExactly(REGION); + assertThat(allJobIds).containsExactly(JOB_ID); + assertThat(result).isEqualTo(Result.CONDITION_MET); + } + + @Test + public void testFinishAfterConditionJobStopped() throws IOException { + when(client.getJobStatus(any(), any(), any())) + .thenReturn(JobState.RUNNING) + .thenReturn(JobState.CANCELLED); + + Result result = + new PipelineOperator(client).waitForConditionAndFinish(DEFAULT_CONFIG, () -> false); + + verify(client, never()).cancelJob(any(), any(), any()); + assertThat(result).isEqualTo(Result.LAUNCH_FINISHED); + } + + @Test + public void testFinishAfterConditionTimeout() throws IOException { + when(client.getJobStatus(any(), any(), any())).thenReturn(JobState.RUNNING); + + Result result = + new PipelineOperator(client).waitForConditionAndFinish(DEFAULT_CONFIG, () -> false); + + verify(client).drainJob(any(), any(), any()); + assertThat(result).isEqualTo(Result.TIMEOUT); + } +} diff --git a/it/common/src/test/java/org/apache/beam/it/common/TestPropertiesTest.java b/it/common/src/test/java/org/apache/beam/it/common/TestPropertiesTest.java new file mode 100644 index 0000000000000..9d36463697bb4 --- /dev/null +++ b/it/common/src/test/java/org/apache/beam/it/common/TestPropertiesTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.common; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link TestProperties}. */ +@RunWith(JUnit4.class) +public final class TestPropertiesTest { + private static final String ARTIFACT_BUCKET = "test-bucket"; + private static final String PROJECT = "test-project"; + private static final String REGION = "us-east1"; + private static final String SPEC_PATH = "gs://test-bucket/some/spec/path"; + + @After + public void tearDown() { + System.clearProperty(TestProperties.ARTIFACT_BUCKET_KEY); + System.clearProperty(TestProperties.PROJECT_KEY); + System.clearProperty(TestProperties.REGION_KEY); + System.clearProperty(TestProperties.SPEC_PATH_KEY); + } + + @Test + public void testAllPropertiesSet() { + System.setProperty(TestProperties.ARTIFACT_BUCKET_KEY, ARTIFACT_BUCKET); + System.setProperty(TestProperties.PROJECT_KEY, PROJECT); + System.setProperty(TestProperties.REGION_KEY, REGION); + System.setProperty(TestProperties.SPEC_PATH_KEY, SPEC_PATH); + + assertThat(TestProperties.artifactBucket()).isEqualTo(ARTIFACT_BUCKET); + assertThat(TestProperties.project()).isEqualTo(PROJECT); + assertThat(TestProperties.region()).isEqualTo(REGION); + assertThat(TestProperties.specPath()).isEqualTo(SPEC_PATH); + } + + @Test + public void testArtifactBucketNotSet() { + System.setProperty(TestProperties.PROJECT_KEY, PROJECT); + System.setProperty(TestProperties.REGION_KEY, REGION); + System.setProperty(TestProperties.SPEC_PATH_KEY, SPEC_PATH); + + assertThrows(IllegalStateException.class, TestProperties::artifactBucket); + } + + @Test + public void testProjectNotSet() { + System.setProperty(TestProperties.ARTIFACT_BUCKET_KEY, ARTIFACT_BUCKET); + System.setProperty(TestProperties.REGION_KEY, REGION); + System.setProperty(TestProperties.SPEC_PATH_KEY, SPEC_PATH); + + assertThrows(IllegalStateException.class, TestProperties::project); + } + + @Test + public void testRegionNotSet() { + System.setProperty(TestProperties.ARTIFACT_BUCKET_KEY, ARTIFACT_BUCKET); + System.setProperty(TestProperties.PROJECT_KEY, PROJECT); + System.setProperty(TestProperties.SPEC_PATH_KEY, SPEC_PATH); + + assertThat(TestProperties.region()).isEqualTo(TestProperties.DEFAULT_REGION); + } + + @Test + public void testSpecPathNotSet() { + System.setProperty(TestProperties.ARTIFACT_BUCKET_KEY, ARTIFACT_BUCKET); + System.setProperty(TestProperties.PROJECT_KEY, PROJECT); + System.setProperty(TestProperties.REGION_KEY, REGION); + + assertThat(TestProperties.specPath()).isNull(); + } +} diff --git a/it/common/src/test/java/org/apache/beam/it/common/utils/ExceptionUtilsTest.java b/it/common/src/test/java/org/apache/beam/it/common/utils/ExceptionUtilsTest.java new file mode 100644 index 0000000000000..d1431eb17181f --- /dev/null +++ b/it/common/src/test/java/org/apache/beam/it/common/utils/ExceptionUtilsTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.common.utils; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.common.utils.ExceptionUtils.containsMessage; +import static org.apache.beam.it.common.utils.ExceptionUtils.containsType; + +import org.junit.Test; + +public class ExceptionUtilsTest { + + @Test + public void testContainsPositive() { + assertThat( + containsMessage( + new IllegalStateException("RESOURCE_EXHAUSTED: Quota issues"), + "RESOURCE_EXHAUSTED")) + .isTrue(); + } + + @Test + public void testContainsNegative() { + assertThat( + containsMessage( + new IllegalStateException("RESOURCE_EXHAUSTED: Quota issues"), + "NullPointerException")) + .isFalse(); + } + + @Test + public void testContainsPositiveNested() { + assertThat( + containsMessage( + new IllegalStateException( + "There is a bad state in the client", + new IllegalArgumentException("RESOURCE_EXHAUSTED: Quota issues")), + "RESOURCE_EXHAUSTED")) + .isTrue(); + } + + @Test + public void testContainsNegativeWithNested() { + assertThat( + containsMessage( + new IllegalStateException( + "There is a bad state in the client", + new IllegalArgumentException("RESOURCE_EXHAUSTED: Quota issues")), + "401 Unauthorized")) + .isFalse(); + } + + @Test + public void testContainsType() { + assertThat( + containsType( + new IllegalStateException("RESOURCE_EXHAUSTED: Quota issues"), + IllegalStateException.class)) + .isTrue(); + } + + @Test + public void testContainsTypeNegative() { + assertThat( + containsType( + new IllegalStateException("RESOURCE_EXHAUSTED: Quota issues"), + IllegalArgumentException.class)) + .isFalse(); + } + + @Test + public void testContainsTypePositiveNested() { + assertThat( + containsType( + new IllegalStateException( + "There is a bad state in the client", + new IllegalArgumentException("RESOURCE_EXHAUSTED: Quota issues")), + IllegalArgumentException.class)) + .isTrue(); + } + + @Test + public void testContainsTypeNegativeWithNested() { + assertThat( + containsType( + new IllegalStateException( + "There is a bad state in the client", + new IllegalArgumentException("RESOURCE_EXHAUSTED: Quota issues")), + NoSuchFieldException.class)) + .isFalse(); + } +} diff --git a/it/common/src/test/java/org/apache/beam/it/common/utils/PipelineUtilsTest.java b/it/common/src/test/java/org/apache/beam/it/common/utils/PipelineUtilsTest.java new file mode 100644 index 0000000000000..316283cdf7d28 --- /dev/null +++ b/it/common/src/test/java/org/apache/beam/it/common/utils/PipelineUtilsTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.common.utils; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.common.utils.PipelineUtils.createJobName; +import static org.apache.beam.it.common.utils.PipelineUtils.extractJobName; +import static org.junit.Assert.assertEquals; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link PipelineUtils}. */ +@RunWith(JUnit4.class) +public class PipelineUtilsTest { + @Test + public void testCreateJobName() { + String name = "create-job-name"; + assertThat(createJobName(name)).matches(name + "-\\d{17}"); + } + + @Test + public void testCreateJobNameWithUppercase() { + assertThat(createJobName("testWithUpperCase")).matches("test-with-upper-case-\\d{17}"); + } + + @Test + public void testCreateJobNameWithUppercaseSuffix() { + assertThat(createJobName("testWithUpperCase", 8)) + .matches("test-with-upper-case-\\d{17}-[a-z0-9]{8}"); + } + + @Test + public void testCreateExtractJobName() { + String name = "create-job-name"; + assertEquals(name, extractJobName(createJobName(name))); + } +} diff --git a/it/common/src/test/java/org/apache/beam/it/common/utils/ResourceManagerUtilsTest.java b/it/common/src/test/java/org/apache/beam/it/common/utils/ResourceManagerUtilsTest.java new file mode 100644 index 0000000000000..76a6bbc0d3dcb --- /dev/null +++ b/it/common/src/test/java/org/apache/beam/it/common/utils/ResourceManagerUtilsTest.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.common.utils; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.common.utils.ResourceManagerUtils.checkValidProjectId; +import static org.apache.beam.it.common.utils.ResourceManagerUtils.generateNewId; +import static org.apache.beam.it.common.utils.ResourceManagerUtils.generateResourceId; +import static org.junit.Assert.assertThrows; + +import java.time.format.DateTimeFormatter; +import java.util.regex.Pattern; +import org.junit.Test; + +/** Unit tests for {@link ResourceManagerUtils}. */ +public class ResourceManagerUtilsTest { + + private static final Pattern ILLEGAL_INSTANCE_CHARS = Pattern.compile("[^a-z0-9-]"); + private static final String REPLACE_INSTANCE_CHAR = "-"; + public static final int MAX_INSTANCE_ID_LENGTH = 36; + private static final DateTimeFormatter TIME_FORMAT = + DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss-SSSSSS"); + + @Test + public void testGenerateResourceIdShouldReplaceDollarSignWithHyphen() { + String testBaseString = "test$instance"; + + String actual = + generateResourceId( + testBaseString, + ILLEGAL_INSTANCE_CHARS, + REPLACE_INSTANCE_CHAR, + MAX_INSTANCE_ID_LENGTH, + TIME_FORMAT); + + assertThat(actual).matches("test-instance-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testGenerateResourceIdShouldReplaceDotWithHyphen() { + String testBaseString = "test.instance"; + + String actual = + generateResourceId( + testBaseString, + ILLEGAL_INSTANCE_CHARS, + REPLACE_INSTANCE_CHAR, + MAX_INSTANCE_ID_LENGTH, + TIME_FORMAT); + + assertThat(actual).matches("test-instance-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testGenerateResourceIdShouldReplaceUnderscoreWithHyphen() { + String testBaseString = "test_inst"; + + String actual = + generateResourceId( + testBaseString, + ILLEGAL_INSTANCE_CHARS, + REPLACE_INSTANCE_CHAR, + MAX_INSTANCE_ID_LENGTH, + TIME_FORMAT); + + assertThat(actual).matches("test-inst-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testGenerateResourceIdShouldReplaceUpperCaseLettersWithLowerCase() { + String testBaseString = "Test-Instance"; + + String actual = + generateResourceId( + testBaseString, + ILLEGAL_INSTANCE_CHARS, + REPLACE_INSTANCE_CHAR, + MAX_INSTANCE_ID_LENGTH, + TIME_FORMAT); + + assertThat(actual).matches("test-instance-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testGenerateResourceIdShouldThrowErrorWithEmptyInput() { + String testBaseString = ""; + + assertThrows( + IllegalArgumentException.class, + () -> + generateResourceId( + testBaseString, + ILLEGAL_INSTANCE_CHARS, + REPLACE_INSTANCE_CHAR, + MAX_INSTANCE_ID_LENGTH, + TIME_FORMAT)); + } + + @Test + public void testGenerateResourceIdShouldThrowErrorWithSingleLetterInput() { + String testBaseString = ""; + + assertThrows( + IllegalArgumentException.class, + () -> + generateResourceId( + testBaseString, + ILLEGAL_INSTANCE_CHARS, + REPLACE_INSTANCE_CHAR, + MAX_INSTANCE_ID_LENGTH, + TIME_FORMAT)); + } + + @Test + public void testGenerateResourceIdWhenInputLengthIsLongerThanTargetLength() { + String longId = "test_instance_long"; + + String actual = + generateResourceId( + longId, + ILLEGAL_INSTANCE_CHARS, + REPLACE_INSTANCE_CHAR, + MAX_INSTANCE_ID_LENGTH, + TIME_FORMAT); + + assertThat(actual).matches("test-instance-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testGenerateNewIdShouldReturnNewIdWhenInputLengthIsLongerThanTargetLength() { + String longId = "long-test-id-string"; + + String actual = generateNewId(longId, 13); + + assertThat(actual).matches("long-([a-zA-Z0-9]){8}"); + } + + @Test + public void testGenerateNewIdShouldReturnOldIdWhenInputLengthIsNotLongerThanTargetLength() { + String shortId = "test-id"; + + String actual = generateNewId(shortId, shortId.length()); + + assertThat(actual).isEqualTo(shortId); + } + + @Test + public void testGenerateNewIdShouldThrowExceptionWhenTargetLengthIsNotGreaterThanEight() { + String id = "long-test-id"; + + assertThrows(IllegalArgumentException.class, () -> generateNewId(id, 8)); + } + + @Test + public void testCheckValidProjectIdWhenIdIsEmpty() { + assertThrows(IllegalArgumentException.class, () -> checkValidProjectId("")); + } + + @Test + public void testCheckValidProjectIdWhenIdIsTooShort() { + assertThrows(IllegalArgumentException.class, () -> checkValidProjectId("abc")); + } + + @Test + public void testCheckValidProjectIdWhenIdIsTooLong() { + assertThrows( + IllegalArgumentException.class, + () -> checkValidProjectId("really-really-really-really-long-project-id")); + } + + @Test + public void testCheckValidProjectIdWhenIdContainsIllegalCharacter() { + assertThrows(IllegalArgumentException.class, () -> checkValidProjectId("%pr$oject-id%")); + } + + @Test + public void testCheckValidProjectIdWhenIdIsValid() { + checkValidProjectId("'project-id-9'!"); + } +} diff --git a/it/conditions/build.gradle b/it/conditions/build.gradle new file mode 100644 index 0000000000000..3a0f29e05f9a2 --- /dev/null +++ b/it/conditions/build.gradle @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.it.conditions', + validateShadowJar: false, + shadowClosure: {}, +) + +description = "Apache Beam :: IT :: Conditions" +ext.summary = "Reusable conditions for tests." + +dependencies { + implementation library.java.slf4j_api +} \ No newline at end of file diff --git a/it/conditions/src/main/java/org/apache/beam/it/conditions/ConditionCheck.java b/it/conditions/src/main/java/org/apache/beam/it/conditions/ConditionCheck.java new file mode 100644 index 0000000000000..b562b4a068c8a --- /dev/null +++ b/it/conditions/src/main/java/org/apache/beam/it/conditions/ConditionCheck.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.conditions; + +import java.util.function.Supplier; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The {@link ConditionCheck} class provides a base interface for reusable/common conditions that + * can be helpful during integration testing. + */ +public abstract class ConditionCheck implements Supplier { + + private static final Logger LOG = LoggerFactory.getLogger(ConditionCheck.class); + + private @Nullable ConditionCheck prev = null; + + protected abstract String getDescription(); + + protected abstract CheckResult check(); + + @Override + public Boolean get() { + LOG.info("[?] Checking for condition '{}'...", getDescription()); + + CheckResult result = check(); + if (!result.success) { + LOG.info("[✗] Condition '{}' failed! {}", getDescription(), result.message); + return false; + } + + LOG.info( + "[✓] Condition '{}' succeeded! {}", + getDescription(), + result.message == null ? "" : result.message); + + if (prev != null) { + return result.success && prev.get(); + } + + return true; + } + + public ConditionCheck and(ConditionCheck next) { + next.prev = this; + return next; + } + + public static class CheckResult { + private final boolean success; + private final String message; + + public CheckResult(boolean success) { + this.success = success; + this.message = ""; + } + + public CheckResult(boolean success, String message) { + this.success = success; + this.message = message; + } + + @Override + public String toString() { + return "CheckResult{" + "success=" + success + ", message='" + message + '\'' + '}'; + } + } +} diff --git a/it/conditions/src/main/java/org/apache/beam/it/conditions/package-info.java b/it/conditions/src/main/java/org/apache/beam/it/conditions/package-info.java new file mode 100644 index 0000000000000..21da7f1091873 --- /dev/null +++ b/it/conditions/src/main/java/org/apache/beam/it/conditions/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package that contains reusable conditions. */ +package org.apache.beam.it.conditions; diff --git a/it/elasticsearch/build.gradle b/it/elasticsearch/build.gradle new file mode 100644 index 0000000000000..d1f3707bb4fca --- /dev/null +++ b/it/elasticsearch/build.gradle @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.it.elasticsearch', +) + +description = "Apache Beam :: IT :: Elasticsearch" +ext.summary = "Integration test utilities for Elasticsearch." + +def elastic_search_version = "7.9.2" + +dependencies { + implementation project(path: ":it:testcontainers", configuration: "shadow") + implementation library.java.testcontainers_elasticsearch + implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:$elastic_search_version" + implementation "org.elasticsearch.client:elasticsearch-rest-client:$elastic_search_version" + implementation "org.elasticsearch:elasticsearch:$elastic_search_version" + // TODO: excluding Guava until Truth updates it to >32.1.x + testImplementation(library.java.truth) { + exclude group: 'com.google.guava', module: 'guava' + } + testImplementation library.java.mockito_inline + testImplementation library.java.commons_lang3 + testRuntimeOnly library.java.slf4j_simple +} \ No newline at end of file diff --git a/it/elasticsearch/src/main/java/org/apache/beam/it/elasticsearch/ElasticsearchResourceManager.java b/it/elasticsearch/src/main/java/org/apache/beam/it/elasticsearch/ElasticsearchResourceManager.java new file mode 100644 index 0000000000000..c58875d7df86b --- /dev/null +++ b/it/elasticsearch/src/main/java/org/apache/beam/it/elasticsearch/ElasticsearchResourceManager.java @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.elasticsearch; + +import static org.apache.beam.it.elasticsearch.ElasticsearchUtils.checkValidIndexName; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.apache.beam.it.common.ResourceManager; +import org.apache.beam.it.testcontainers.TestContainerResourceManager; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.http.HttpHost; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.core.CountRequest; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; +import org.testcontainers.elasticsearch.ElasticsearchContainer; +import org.testcontainers.utility.DockerImageName; + +/** + * Client for managing Elasticsearch resources. + * + *

    The class supports many indices per resource manager object. + * + *

    The index name is formed using testId. The index name will be "{prefix}-{testId}-{ISO8601 + * time, microsecond precision}". + * + *

    The class is thread-safe. + */ +public class ElasticsearchResourceManager extends TestContainerResourceManager> + implements ResourceManager { + + private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchResourceManager.class); + + private static final String DEFAULT_ELASTICSEARCH_CONTAINER_NAME = + "docker.elastic.co/elasticsearch/elasticsearch-oss"; + + // A list of available Elasticsearch Docker image tags can be found at + // https://hub.docker.com/_/elasticsearch/tags + private static final String DEFAULT_ELASTICSEARCH_CONTAINER_TAG = "7.9.2"; + + // 9200 is the default port that Elasticsearch is configured to listen on + private static final int ELASTICSEARCH_INTERNAL_PORT = 9200; + + private final RestHighLevelClient elasticsearchClient; + private final String connectionString; + private final List managedIndexNames = new ArrayList<>(); + + private ElasticsearchResourceManager(Builder builder) { + this(/* elasticsearchClient= */ null, buildContainer(builder), builder); + } + + /** + * Create the {@link ElasticsearchContainer} instance for the given builder. + * + *

    The method override the wait strategy from the base container using the same regex, but + * increase the timeout to 2 minutes. + */ + private static ElasticsearchContainer buildContainer(Builder builder) { + ElasticsearchContainer container = + new ElasticsearchContainer( + DockerImageName.parse(builder.containerImageName).withTag(builder.containerImageTag)); + + // + // The regex is based on Elasticsearch container, but it's not exposed anywhere. + String regex = ".*(\"message\":\\s?\"started[\\s?|\"].*|] started\n$)"; + Duration startupTimeout = Duration.ofMinutes(2); + container.setWaitStrategy( + new LogMessageWaitStrategy().withRegEx(regex).withStartupTimeout(startupTimeout)); + + return container; + } + + @VisibleForTesting + @SuppressWarnings("nullness") + ElasticsearchResourceManager( + @Nullable RestHighLevelClient elasticsearchClient, + ElasticsearchContainer container, + Builder builder) { + super(container, builder); + + this.connectionString = + "http://" + this.getHost() + ":" + this.getPort(ELASTICSEARCH_INTERNAL_PORT); + + RestClientBuilder restClientBuilder = + RestClient.builder(HttpHost.create(container.getHttpHostAddress())); + this.elasticsearchClient = + elasticsearchClient != null + ? elasticsearchClient + : new RestHighLevelClient(restClientBuilder); + } + + public static Builder builder(String testId) { + return new Builder(testId); + } + + /** Returns the URI connection string to the Elasticsearch service. */ + public synchronized String getUri() { + return connectionString; + } + + private synchronized boolean indexExists(String indexName) throws IOException { + // Check index name + checkValidIndexName(indexName); + + return elasticsearchClient + .indices() + .exists(new GetIndexRequest(indexName), RequestOptions.DEFAULT); + } + + /** + * Creates an index on Elasticsearch. + * + * @param indexName Name of the index to create. + * @return A boolean indicating whether the resource was created. + * @throws ElasticsearchResourceManagerException if there is an error creating the collection in + * Elasticsearch. + */ + public synchronized boolean createIndex(String indexName) + throws ElasticsearchResourceManagerException { + LOG.info("Creating index using name '{}'.", indexName); + + try { + // Check to see if the index exists + if (indexExists(indexName)) { + return false; + } + + managedIndexNames.add(indexName); + + return elasticsearchClient + .indices() + .create(new CreateIndexRequest(indexName), RequestOptions.DEFAULT) + .isAcknowledged(); + } catch (Exception e) { + throw new ElasticsearchResourceManagerException("Error creating index.", e); + } + } + + /** + * Inserts the given Documents into a collection. + * + *

    Note: Implementations may do collection creation here, if one does not already exist. + * + * @param indexName The name of the index to insert the documents into. + * @param documents A map of (id, document) to insert into the collection. + * @return A boolean indicating whether the Documents were inserted successfully. + * @throws ElasticsearchResourceManagerException if there is an error inserting the documents. + */ + public synchronized boolean insertDocuments( + String indexName, Map> documents) + throws ElasticsearchResourceManagerException { + LOG.info("Attempting to write {} documents to {}.", documents.size(), indexName); + + try { + BulkRequest request = new BulkRequest(); + + for (Map.Entry> documentEntry : documents.entrySet()) { + request.add( + new IndexRequest(indexName) + .id(documentEntry.getKey()) + .source(documentEntry.getValue())); + } + + BulkResponse bulkResponse = elasticsearchClient.bulk(request, RequestOptions.DEFAULT); + if (!bulkResponse.hasFailures()) { + LOG.info("Successfully wrote {} documents to {}", documents.size(), indexName); + return true; + } + + return false; + } catch (Exception e) { + throw new ElasticsearchResourceManagerException("Error inserting documents.", e); + } + } + + /** + * Reads all the documents in an index. + * + * @param indexName The name of the index to read from. + * @return An iterable of all the Documents in the collection. + * @throws ElasticsearchResourceManagerException if there is an error reading the data. + */ + public synchronized List> fetchAll(String indexName) + throws ElasticsearchResourceManagerException { + LOG.info("Reading all documents from {}.", indexName); + + try { + // Elasticsearch is near realtime, so refresh will guarantee reads are consistent + elasticsearchClient.indices().refresh(new RefreshRequest(indexName), RequestOptions.DEFAULT); + + SearchRequest searchRequest = + new SearchRequest(indexName).source(new SearchSourceBuilder().size(10_000)); + return Arrays.stream( + elasticsearchClient.search(searchRequest, RequestOptions.DEFAULT).getHits().getHits()) + .map(SearchHit::getSourceAsMap) + .collect(Collectors.toList()); + } catch (Exception e) { + throw new ElasticsearchResourceManagerException("Error reading index " + indexName + ".", e); + } + } + + /** + * Gets the count of documents in an index. + * + * @param indexName The name of the index to read from. + * @return The number of documents for the given index + * @throws ElasticsearchResourceManagerException if there is an error reading the data. + */ + public long count(String indexName) throws ElasticsearchResourceManagerException { + LOG.info("Fetching count from {}.", indexName); + + try { + // Elasticsearch is near realtime, so refresh will guarantee reads are consistent + elasticsearchClient.indices().refresh(new RefreshRequest(indexName), RequestOptions.DEFAULT); + + return elasticsearchClient + .count(new CountRequest(indexName), RequestOptions.DEFAULT) + .getCount(); + } catch (Exception e) { + throw new ElasticsearchResourceManagerException( + "Error fetching count from " + indexName + ".", e); + } + } + + /** + * Deletes all created resources and cleans up the Elasticsearch client, making the manager object + * unusable. + * + * @throws ElasticsearchResourceManagerException if there is an error deleting the Elasticsearch + * resources. + */ + @Override + public synchronized void cleanupAll() throws ElasticsearchResourceManagerException { + LOG.info("Attempting to cleanup Elasticsearch manager."); + + // First, delete the database if it was not given as a static argument + if (!managedIndexNames.isEmpty()) { + try { + elasticsearchClient + .indices() + .delete( + new DeleteIndexRequest( + managedIndexNames.toArray(new String[managedIndexNames.size()])), + RequestOptions.DEFAULT); + } catch (Exception e) { + LOG.error("Failed to delete Elasticsearch indices {}.", managedIndexNames, e); + throw new RuntimeException("Failed deleting Elasticsearch indices", e); + } + } + + super.cleanupAll(); + + LOG.info("Elasticsearch manager successfully cleaned up."); + } + + /** Builder for {@link ElasticsearchResourceManager}. */ + public static final class Builder + extends TestContainerResourceManager.Builder { + + private Builder(String testId) { + super(testId, DEFAULT_ELASTICSEARCH_CONTAINER_NAME, DEFAULT_ELASTICSEARCH_CONTAINER_TAG); + } + + @Override + public ElasticsearchResourceManager build() { + return new ElasticsearchResourceManager(this); + } + } +} diff --git a/it/elasticsearch/src/main/java/org/apache/beam/it/elasticsearch/ElasticsearchResourceManagerException.java b/it/elasticsearch/src/main/java/org/apache/beam/it/elasticsearch/ElasticsearchResourceManagerException.java new file mode 100644 index 0000000000000..5579bacb14195 --- /dev/null +++ b/it/elasticsearch/src/main/java/org/apache/beam/it/elasticsearch/ElasticsearchResourceManagerException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.elasticsearch; + +/** Custom exception for {@link ElasticsearchResourceManager} implementations. */ +public class ElasticsearchResourceManagerException extends RuntimeException { + + public ElasticsearchResourceManagerException(String errorMessage, Throwable err) { + super(errorMessage, err); + } + + public ElasticsearchResourceManagerException(String errorMessage) { + super(errorMessage); + } +} diff --git a/it/elasticsearch/src/main/java/org/apache/beam/it/elasticsearch/ElasticsearchUtils.java b/it/elasticsearch/src/main/java/org/apache/beam/it/elasticsearch/ElasticsearchUtils.java new file mode 100644 index 0000000000000..51f0494fda4de --- /dev/null +++ b/it/elasticsearch/src/main/java/org/apache/beam/it/elasticsearch/ElasticsearchUtils.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.elasticsearch; + +import static org.apache.beam.it.common.utils.ResourceManagerUtils.generateResourceId; + +import java.time.format.DateTimeFormatter; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** Utilities for {@link ElasticsearchResourceManager} implementations. */ +final class ElasticsearchUtils { + + // Constraints from + // https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html + private static final int MAX_INDEX_NAME_LENGTH = 255; + + // Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, # + private static final Pattern ILLEGAL_INDEX_NAME_CHARS = Pattern.compile("[/\\\\ \"?*<>|,#\0]"); + private static final String REPLACE_INDEX_NAME_CHAR = "-"; + private static final DateTimeFormatter TIME_FORMAT = + DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss-SSSSSS"); + + private ElasticsearchUtils() {} + + /** + * Generates an Elasticsearch index name from a given string. + * + * @param baseString The string to generate the name from. + * @return The database name string. + */ + static String generateIndexName(String baseString) { + return generateResourceId( + baseString, + ILLEGAL_INDEX_NAME_CHARS, + REPLACE_INDEX_NAME_CHAR, + MAX_INDEX_NAME_LENGTH, + TIME_FORMAT); + } + + /** + * Checks whether the given index name is valid according to Elasticsearch constraints. + * + * @param indexName the index name to check. + * @throws IllegalArgumentException if the index name is invalid. + */ + static void checkValidIndexName(String indexName) { + if (indexName.length() > MAX_INDEX_NAME_LENGTH) { + throw new IllegalArgumentException( + "Index name " + + indexName + + " cannot be longer than " + + MAX_INDEX_NAME_LENGTH + + " characters."); + } + Matcher matcher = ILLEGAL_INDEX_NAME_CHARS.matcher(indexName); + if (matcher.find()) { + throw new IllegalArgumentException( + "Index name " + + indexName + + " is not a valid name. Character \"" + + matcher.group() + + "\" is not allowed."); + } + if (indexName.charAt(0) == '-' || indexName.charAt(0) == '_' || indexName.charAt(0) == '+') { + throw new IllegalArgumentException( + "Index name " + indexName + " can not start with -, _ or +."); + } + } +} diff --git a/it/elasticsearch/src/main/java/org/apache/beam/it/elasticsearch/package-info.java b/it/elasticsearch/src/main/java/org/apache/beam/it/elasticsearch/package-info.java new file mode 100644 index 0000000000000..7ffb9f92b556a --- /dev/null +++ b/it/elasticsearch/src/main/java/org/apache/beam/it/elasticsearch/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing Elasticsearch resources within integration tests. */ +package org.apache.beam.it.elasticsearch; diff --git a/it/elasticsearch/src/test/java/org/apache/beam/it/elasticsearch/ElasticsearchResourceManagerIT.java b/it/elasticsearch/src/test/java/org/apache/beam/it/elasticsearch/ElasticsearchResourceManagerIT.java new file mode 100644 index 0000000000000..83ff47a5efe19 --- /dev/null +++ b/it/elasticsearch/src/test/java/org/apache/beam/it/elasticsearch/ElasticsearchResourceManagerIT.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.elasticsearch; + +import static com.google.common.truth.Truth.assertThat; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.testcontainers.TestContainersIntegrationTest; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Integration tests for {@link ElasticsearchResourceManager}. */ +@Category(TestContainersIntegrationTest.class) +@RunWith(JUnit4.class) +public class ElasticsearchResourceManagerIT { + + private ElasticsearchResourceManager elasticsearchResourceManager; + + @Before + public void setUp() { + elasticsearchResourceManager = ElasticsearchResourceManager.builder("dummy").build(); + } + + @Test + public void testResourceManagerE2E() { + boolean createIndex = elasticsearchResourceManager.createIndex("dummy-insert"); + assertThat(createIndex).isTrue(); + + Map> records = new HashMap<>(); + records.put("1", ImmutableMap.of("company", "Google")); + records.put("2", ImmutableMap.of("company", "Alphabet")); + + boolean insertDocuments = elasticsearchResourceManager.insertDocuments("dummy-insert", records); + assertThat(insertDocuments).isTrue(); + + long count = elasticsearchResourceManager.count("dummy-insert"); + assertThat(count).isEqualTo(2L); + + List> fetchRecords = elasticsearchResourceManager.fetchAll("dummy-insert"); + assertThat(fetchRecords).hasSize(2); + assertThat(fetchRecords).containsExactlyElementsIn(new ArrayList<>(records.values())); + } + + @After + public void tearDown() { + ResourceManagerUtils.cleanResources(elasticsearchResourceManager); + } +} diff --git a/it/elasticsearch/src/test/java/org/apache/beam/it/elasticsearch/ElasticsearchResourceManagerTest.java b/it/elasticsearch/src/test/java/org/apache/beam/it/elasticsearch/ElasticsearchResourceManagerTest.java new file mode 100644 index 0000000000000..5778f24e4664c --- /dev/null +++ b/it/elasticsearch/src/test/java/org/apache/beam/it/elasticsearch/ElasticsearchResourceManagerTest.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.elasticsearch; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.core.CountRequest; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.GetIndexRequest; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +/** Unit tests for {@link ElasticsearchResourceManager}. */ +@RunWith(JUnit4.class) +public class ElasticsearchResourceManagerTest { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private RestHighLevelClient elasticsearchClient; + + @Mock private ElasticsearchContainer container; + + private static final String TEST_ID = "test-id"; + private static final String INDEX_NAME = "index-name"; + private static final String HOST = "localhost"; + private static final int ELASTICSEARCH_PORT = 9200; + private static final int MAPPED_PORT = 10000; + + private ElasticsearchResourceManager testManager; + + @Before + public void setUp() { + when(container.getHttpHostAddress()).thenReturn(HOST + ":" + MAPPED_PORT); + doReturn(container).when(container).withLogConsumer(any()); + + testManager = + new ElasticsearchResourceManager( + elasticsearchClient, container, ElasticsearchResourceManager.builder(TEST_ID)); + } + + @Test + public void testGetUriShouldReturnCorrectValue() { + when(container.getHost()).thenReturn(HOST); + when(container.getMappedPort(ELASTICSEARCH_PORT)).thenReturn(MAPPED_PORT); + + assertThat( + new ElasticsearchResourceManager( + elasticsearchClient, container, ElasticsearchResourceManager.builder(TEST_ID)) + .getUri()) + .matches("http://" + HOST + ":" + MAPPED_PORT); + } + + @Test + public void testCreateIndexShouldThrowErrorWhenCollectionNameIsInvalid() { + assertThrows( + ElasticsearchResourceManagerException.class, () -> testManager.createIndex("invalid#name")); + } + + @Test + public void testCreateCollectionShouldThrowErrorWhenElasticsearchFailsToGetDB() + throws IOException { + when(elasticsearchClient + .indices() + .exists(any(GetIndexRequest.class), eq(RequestOptions.DEFAULT))) + .thenThrow(IllegalArgumentException.class); + + assertThrows( + ElasticsearchResourceManagerException.class, () -> testManager.createIndex(INDEX_NAME)); + } + + @Test + public void testCreateCollectionShouldReturnTrueIfElasticsearchDoesNotThrowAnyError() + throws IOException { + when(elasticsearchClient + .indices() + .exists(any(GetIndexRequest.class), eq(RequestOptions.DEFAULT))) + .thenReturn(false); + when(elasticsearchClient + .indices() + .create(any(CreateIndexRequest.class), eq(RequestOptions.DEFAULT)) + .isAcknowledged()) + .thenReturn(true); + + assertThat(testManager.createIndex(INDEX_NAME)).isEqualTo(true); + verify(elasticsearchClient.indices()) + .exists(any(GetIndexRequest.class), eq(RequestOptions.DEFAULT)); + } + + @Test + public void testInsertDocumentsShouldInsert() throws IOException { + assertThat( + testManager.insertDocuments( + INDEX_NAME, ImmutableMap.of("1", ImmutableMap.of("company", "Google LLC")))) + .isEqualTo(true); + + verify(elasticsearchClient).bulk(any(BulkRequest.class), eq(RequestOptions.DEFAULT)); + } + + @Test + public void testCountDocumentsShouldReturnInt() throws IOException { + when(elasticsearchClient.count(any(CountRequest.class), eq(RequestOptions.DEFAULT)).getCount()) + .thenReturn(100L); + + assertThat(testManager.count(INDEX_NAME)).isEqualTo(100L); + } + + @Test + public void testCleanupShouldDropNonStaticIndex() throws IOException { + when(elasticsearchClient + .indices() + .exists(any(GetIndexRequest.class), eq(RequestOptions.DEFAULT))) + .thenReturn(false); + when(elasticsearchClient + .indices() + .create(any(CreateIndexRequest.class), eq(RequestOptions.DEFAULT)) + .isAcknowledged()) + .thenReturn(true); + + boolean index = testManager.createIndex("dummy-index"); + assertThat(index).isTrue(); + testManager.cleanupAll(); + + verify(elasticsearchClient.indices()) + .delete(any(DeleteIndexRequest.class), eq(RequestOptions.DEFAULT)); + } + + @Test + public void testCleanupAllShouldDropStaticIndex() throws IOException { + ElasticsearchResourceManager.Builder builder = ElasticsearchResourceManager.builder(TEST_ID); + builder.setHost("localhost").setPort(9200).useStaticContainer(); + + ElasticsearchResourceManager tm = + new ElasticsearchResourceManager(elasticsearchClient, container, builder); + + when(elasticsearchClient + .indices() + .exists(any(GetIndexRequest.class), eq(RequestOptions.DEFAULT))) + .thenReturn(false); + when(elasticsearchClient + .indices() + .create(any(CreateIndexRequest.class), eq(RequestOptions.DEFAULT)) + .isAcknowledged()) + .thenReturn(true); + + boolean index = tm.createIndex("dummy-index"); + assertThat(index).isTrue(); + tm.cleanupAll(); + + verify(elasticsearchClient.indices()) + .delete(any(DeleteIndexRequest.class), eq(RequestOptions.DEFAULT)); + } +} diff --git a/it/elasticsearch/src/test/java/org/apache/beam/it/elasticsearch/ElasticsearchUtilsTest.java b/it/elasticsearch/src/test/java/org/apache/beam/it/elasticsearch/ElasticsearchUtilsTest.java new file mode 100644 index 0000000000000..61d6b5d57c2c4 --- /dev/null +++ b/it/elasticsearch/src/test/java/org/apache/beam/it/elasticsearch/ElasticsearchUtilsTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.elasticsearch; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.elasticsearch.ElasticsearchUtils.checkValidIndexName; +import static org.apache.beam.it.elasticsearch.ElasticsearchUtils.generateIndexName; +import static org.junit.Assert.assertThrows; + +import org.apache.commons.lang3.StringUtils; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link ElasticsearchUtils}. */ +@RunWith(JUnit4.class) +public class ElasticsearchUtilsTest { + + @Test + public void testGenerateIndexNameShouldReplaceForwardSlash() { + String testBaseString = "Test/DB/Name"; + String actual = ElasticsearchUtils.generateIndexName(testBaseString); + assertThat(actual).matches("test-db-name-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testGenerateIndexNameShouldReplaceBackwardSlash() { + String testBaseString = "Test\\DB\\Name"; + String actual = generateIndexName(testBaseString); + assertThat(actual).matches("test-db-name-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testGenerateIndexNameShouldReplaceComma() { + String testBaseString = "Test,DB,Name"; + String actual = generateIndexName(testBaseString); + assertThat(actual).matches("test-db-name-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testGenerateIndexNameShouldReplaceSpace() { + String testBaseString = "Test DB Name"; + String actual = generateIndexName(testBaseString); + assertThat(actual).matches("test-db-name-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testGenerateIndexNameShouldReplaceDoubleQuotes() { + String testBaseString = "Test\"DB\"Name"; + String actual = generateIndexName(testBaseString); + assertThat(actual).matches("test-db-name-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testGenerateIndexNameShouldReplaceStar() { + String testBaseString = "Test*DB*Name"; + String actual = generateIndexName(testBaseString); + assertThat(actual).matches("test-db-name-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testGenerateIndexNameShouldReplaceNullCharacter() { + String testBaseString = "Test\0DB\0Name"; + String actual = generateIndexName(testBaseString); + assertThat(actual).matches("test-db-name-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testCheckValidIndexNameThrowsErrorWhenNameIsTooLong() { + assertThrows( + IllegalArgumentException.class, () -> checkValidIndexName(StringUtils.repeat("a", 300))); + } + + @Test + public void testCheckValidIndexNameThrowsErrorWhenNameContainsPoundSymbol() { + assertThrows(IllegalArgumentException.class, () -> checkValidIndexName("test#collection")); + } + + @Test + public void testCheckValidIndexNameThrowsErrorWhenNameContainsNull() { + assertThrows(IllegalArgumentException.class, () -> checkValidIndexName("test\0collection")); + } + + @Test + public void testCheckValidIndexNameThrowsErrorWhenNameBeginsWithUnderscore() { + assertThrows(IllegalArgumentException.class, () -> checkValidIndexName("_test-index")); + } + + @Test + public void testCheckValidIndexNameDoesNotThrowErrorWhenNameIsValid() { + checkValidIndexName("a_collection-name_valid.Test1"); + checkValidIndexName("123_a_collection-name_valid.Test1"); + } +} diff --git a/it/google-cloud-platform/build.gradle b/it/google-cloud-platform/build.gradle new file mode 100644 index 0000000000000..4c5327b44c9ac --- /dev/null +++ b/it/google-cloud-platform/build.gradle @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import groovy.json.JsonOutput +import org.apache.beam.gradle.IoPerformanceTestUtilities + +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.it.gcp', +) + +description = "Apache Beam :: IT :: Google Cloud Platform" +ext.summary = "Integration test utilities for Google Cloud Platform." + +dependencies { + implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) + implementation project(path: ":sdks:java:core", configuration: "shadow") + implementation project(path: ":runners:google-cloud-dataflow-java") + implementation project(path: ":it:conditions", configuration: "shadow") + implementation project(path: ":it:truthmatchers", configuration: "shadow") + implementation library.java.slf4j_api + implementation library.java.vendored_guava_32_1_2_jre + implementation library.java.jackson_core + implementation library.java.jackson_databind + implementation 'org.apache.hadoop:hadoop-common:3.3.5' + implementation 'org.apache.avro:avro:1.11.1' + implementation 'org.apache.parquet:parquet-avro:1.12.0' + implementation 'org.apache.parquet:parquet-common:1.12.0' + implementation 'org.apache.parquet:parquet-hadoop:1.12.0' + implementation library.java.gax + implementation library.java.google_api_common + implementation library.java.protobuf_java_util + implementation library.java.protobuf_java + implementation library.java.threetenbp + implementation 'org.awaitility:awaitility:4.2.0' + // Google Cloud Dependencies + implementation library.java.google_api_services_bigquery + implementation library.java.google_cloud_core + implementation 'com.google.cloud:google-cloud-storage' + implementation 'com.google.cloud:google-cloud-bigquery' + implementation 'com.google.cloud:google-cloud-monitoring' + provided 'com.google.api.grpc:proto-google-cloud-monitoring-v3' + implementation 'com.google.cloud:google-cloud-bigtable' + implementation 'com.google.cloud:google-cloud-spanner' + implementation 'com.google.cloud:google-cloud-pubsub' + provided 'com.google.api.grpc:proto-google-cloud-pubsub-v1' + implementation 'com.google.cloud:google-cloud-pubsublite' + provided 'com.google.api.grpc:proto-google-cloud-pubsublite-v1' + implementation 'com.google.cloud:google-cloud-datastore' + implementation 'com.google.cloud:google-cloud-datastream' + provided 'com.google.api.grpc:proto-google-cloud-datastream-v1' + implementation 'com.google.cloud:google-cloud-kms' + provided 'com.google.api.grpc:proto-google-cloud-kms-v1' + implementation 'com.google.cloud:google-cloud-dlp' + provided 'com.google.api.grpc:proto-google-cloud-dlp-v2' + implementation 'com.google.cloud:google-cloud-secretmanager' + provided 'com.google.api.grpc:proto-google-cloud-secretmanager-v1' + + testImplementation project(path: ":sdks:java:testing:test-utils") + testImplementation project(path: ":sdks:java:io:google-cloud-platform") + testImplementation project(path: ":sdks:java:io:synthetic") + testImplementation library.java.mockito_inline + testImplementation project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "testRuntimeMigration") + testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadowTest") + testRuntimeOnly library.java.slf4j_simple +} + +tasks.register("GCSPerformanceTest", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'FileBasedIOLT', ['configuration':'large','project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']) +tasks.register("BigTablePerformanceTest", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'BigTableIOLT', ['configuration':'large','project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']) +tasks.register("BigQueryStorageApiStreamingPerformanceTest", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'BigQueryStreamingLT', ['configuration':'large', 'project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']) diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/GCPBaseIT.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/GCPBaseIT.java new file mode 100644 index 0000000000000..2035f76f70af9 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/GCPBaseIT.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.auth.Credentials; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.apache.beam.it.common.TestProperties; +import org.junit.Before; + +@SuppressWarnings("nullness") +public abstract class GCPBaseIT { + protected static final String PROJECT = TestProperties.project(); + protected static final String REGION = TestProperties.region(); + + protected Credentials credentials; + + @SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD") + protected CredentialsProvider credentialsProvider; + + @Before + public void setUpBase() { + if (TestProperties.hasAccessToken()) { + credentials = TestProperties.googleCredentials(); + } else { + credentials = TestProperties.buildCredentialsFromEnv(); + } + credentialsProvider = FixedCredentialsProvider.create(credentials); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/GoogleCloudIntegrationTest.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/GoogleCloudIntegrationTest.java new file mode 100644 index 0000000000000..5fe4b5232183c --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/GoogleCloudIntegrationTest.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** Annotation that marks the test that requires GCP resources. */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface GoogleCloudIntegrationTest {} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/IOLoadTestBase.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/IOLoadTestBase.java new file mode 100644 index 0000000000000..6b728a6a60db8 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/IOLoadTestBase.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp; + +import java.io.IOException; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.TestProperties; +import org.apache.beam.it.gcp.dataflow.DefaultPipelineLauncher; +import org.apache.beam.sdk.metrics.Counter; +import org.apache.beam.sdk.metrics.Metrics; +import org.apache.beam.sdk.transforms.DoFn; +import org.junit.After; +import org.junit.Before; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Base class for IO Load tests. */ +@RunWith(JUnit4.class) +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/27438) +}) +public class IOLoadTestBase extends LoadTestBase { + + private static final Logger LOG = LoggerFactory.getLogger(IOLoadTestBase.class); + + protected String tempBucketName; + + @Before + public void setUpBase() { + // Prefer artifactBucket, but use the staging one if none given + if (TestProperties.hasArtifactBucket()) { + tempBucketName = TestProperties.artifactBucket(); + } else if (TestProperties.hasStageBucket()) { + tempBucketName = TestProperties.stageBucket(); + } else { + LOG.warn( + "Both -DartifactBucket and -DstageBucket were not given. Pipeline may fail if a temp gcs" + + " location is needed"); + } + } + + @After + public void tearDownBase() throws IOException { + pipelineLauncher.cleanupAll(); + } + + @Override + public PipelineLauncher launcher() { + return DefaultPipelineLauncher.builder(CREDENTIALS).build(); + } + + /** A utility DoFn that counts elements passed through. */ + public static final class CountingFn extends DoFn { + + private final Counter elementCounter; + + public CountingFn(String name) { + elementCounter = Metrics.counter(BEAM_METRICS_NAMESPACE, name); + } + + @ProcessElement + public void processElement(ProcessContext ctx) { + elementCounter.inc(1L); + ctx.output(ctx.element()); + } + } + + // To make PipelineLauncher.getMetric work in a unified way for both runner provided metrics and + // pipeline defined + // metrics, here we wrap Beam provided metrics as a pre-defined metrics name + // [name_space:metric_type:metric_name + // which will be recognized by getMetric method + public enum PipelineMetricsType { + COUNTER, + STARTTIME, + ENDTIME, + RUNTIME, + } + + /** Namespace for Beam provided pipeline metrics (set up by Metrics transform). */ + public static final String BEAM_METRICS_NAMESPACE = "BEAM_METRICS"; + + /** Given a metrics name, return Beam metrics name. */ + public static String getBeamMetricsName(PipelineMetricsType metricstype, String metricsName) { + return BEAM_METRICS_NAMESPACE + ":" + metricstype + ":" + metricsName; + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/LoadTestBase.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/LoadTestBase.java new file mode 100644 index 0000000000000..14bb05394de26 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/LoadTestBase.java @@ -0,0 +1,537 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp; + +import static org.apache.beam.it.common.logging.LogStrings.formatForLogging; +import static org.apache.beam.it.gcp.dataflow.AbstractPipelineLauncher.RUNNER_V2; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.api.services.bigquery.model.TableRow; +import com.google.api.services.dataflow.model.JobMessage; +import com.google.auth.Credentials; +import com.google.auto.value.AutoValue; +import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; +import com.google.monitoring.v3.TimeInterval; +import com.google.protobuf.util.Timestamps; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.io.IOException; +import java.text.ParseException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.regex.Pattern; +import javax.annotation.Nullable; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineLauncher.LaunchInfo; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.TestProperties; +import org.apache.beam.it.gcp.bigquery.BigQueryResourceManager; +import org.apache.beam.it.gcp.monitoring.MonitoringClient; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.rules.TestRule; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Base class for performance tests. It provides helper methods for common operations. */ +@RunWith(JUnit4.class) +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/27438) +}) +public abstract class LoadTestBase { + private static final Logger LOG = LoggerFactory.getLogger(LoadTestBase.class); + // Dataflow resources cost factors (showing us-central-1 pricing). + // See https://cloud.google.com/dataflow/pricing#pricing-details + private static final double VCPU_PER_HR_BATCH = 0.056; + private static final double VCPU_PER_HR_STREAMING = 0.069; + private static final double MEM_PER_GB_HR_BATCH = 0.003557; + private static final double MEM_PER_GB_HR_STREAMING = 0.0035557; + private static final double PD_PER_GB_HR = 0.000054; + private static final double PD_SSD_PER_GB_HR = 0.000298; + private static final double SHUFFLE_PER_GB_BATCH = 0.011; + private static final double SHUFFLE_PER_GB_STREAMING = 0.018; + private static final Pattern WORKER_START_PATTERN = + Pattern.compile( + "^All workers have finished the startup processes and began to receive work requests.*$"); + private static final Pattern WORKER_STOP_PATTERN = Pattern.compile("^Stopping worker pool.*$"); + + protected static final Credentials CREDENTIALS = TestProperties.googleCredentials(); + protected static final CredentialsProvider CREDENTIALS_PROVIDER = + FixedCredentialsProvider.create(CREDENTIALS); + + @SuppressFBWarnings("MS_PKGPROTECT") + protected static String project; + + @SuppressFBWarnings("MS_PKGPROTECT") + protected static String region; + + protected MonitoringClient monitoringClient; + protected PipelineLauncher pipelineLauncher; + protected PipelineOperator pipelineOperator; + + protected String testName; + + @Rule + public TestRule watcher = + new TestWatcher() { + @Override + protected void starting(Description description) { + LOG.info( + "Starting load test {}.{}", description.getClassName(), description.getMethodName()); + testName = description.getMethodName(); + } + }; + + @BeforeClass + public static void setUpClass() { + project = TestProperties.project(); + region = TestProperties.region(); + } + + @Before + public void setUp() throws IOException { + monitoringClient = MonitoringClient.builder(CREDENTIALS_PROVIDER).build(); + pipelineLauncher = launcher(); + pipelineOperator = new PipelineOperator(pipelineLauncher); + } + + @After + public void tearDownLoadTestBase() throws IOException { + pipelineLauncher.cleanupAll(); + monitoringClient.cleanupAll(); + } + + public abstract PipelineLauncher launcher(); + + /** + * Exports the metrics of given dataflow job to BigQuery. + * + * @param launchInfo Job info of the job + * @param metrics metrics to export + */ + protected void exportMetricsToBigQuery(LaunchInfo launchInfo, Map metrics) { + LOG.info("Exporting metrics:\n{}", formatForLogging(metrics)); + try { + // either use the user specified project for exporting, or the same project + String exportProject = MoreObjects.firstNonNull(TestProperties.exportProject(), project); + BigQueryResourceManager bigQueryResourceManager = + BigQueryResourceManager.builder(testName, exportProject, CREDENTIALS) + .setDatasetId(TestProperties.exportDataset()) + .build(); + // exporting metrics to bigQuery table + Map rowContent = new HashMap<>(); + rowContent.put("timestamp", launchInfo.createTime()); + rowContent.put("sdk", launchInfo.sdk()); + rowContent.put("version", launchInfo.version()); + rowContent.put("job_type", launchInfo.jobType()); + putOptional(rowContent, "template_name", launchInfo.templateName()); + putOptional(rowContent, "template_version", launchInfo.templateVersion()); + putOptional(rowContent, "template_type", launchInfo.templateType()); + putOptional(rowContent, "pipeline_name", launchInfo.pipelineName()); + rowContent.put("test_name", testName); + // Convert parameters map to list of table row since it's a repeated record + List parameterRows = new ArrayList<>(); + for (Entry entry : launchInfo.parameters().entrySet()) { + TableRow row = new TableRow().set("name", entry.getKey()).set("value", entry.getValue()); + parameterRows.add(row); + } + rowContent.put("parameters", parameterRows); + // Convert metrics map to list of table row since it's a repeated record + List metricRows = new ArrayList<>(); + for (Entry entry : metrics.entrySet()) { + TableRow row = new TableRow().set("name", entry.getKey()).set("value", entry.getValue()); + metricRows.add(row); + } + rowContent.put("metrics", metricRows); + bigQueryResourceManager.write( + TestProperties.exportTable(), RowToInsert.of("rowId", rowContent)); + } catch (IllegalStateException e) { + LOG.error("Unable to export results to datastore. ", e); + } + } + + /** + * Checks if the input PCollection has the expected number of messages. + * + * @param jobId JobId of the job + * @param pcollection the input pcollection name + * @param expectedElements expected number of messages + * @return whether the input pcollection has the expected number of messages. + */ + protected boolean waitForNumMessages(String jobId, String pcollection, Long expectedElements) { + try { + // the element count metric always follows the pattern -ElementCount + String metricName = pcollection.replace(".", "-") + "-ElementCount"; + Double metric = pipelineLauncher.getMetric(project, region, jobId, metricName); + if ((metric != null) && (metric >= (double) expectedElements)) { + return true; + } + LOG.info( + "Expected {} messages in input PCollection, but found {}.", expectedElements, metric); + return false; + } catch (Exception e) { + LOG.warn("Encountered error when trying to measure input elements. ", e); + return false; + } + } + + /** + * Compute metrics of a Dataflow runner job. + * + * @param metrics a map of raw metrics. The results are also appened in the map. + * @param launchInfo Job info of the job + * @param config a {@class MetricsConfiguration} + */ + private void computeDataflowMetrics( + Map metrics, LaunchInfo launchInfo, MetricsConfiguration config) + throws ParseException { + // cost info + LOG.info("Calculating approximate cost for {} under {}", launchInfo.jobId(), project); + TimeInterval workerTimeInterval = getWorkerTimeInterval(launchInfo); + metrics.put( + "RunTime", + (double) + Timestamps.between(workerTimeInterval.getStartTime(), workerTimeInterval.getEndTime()) + .getSeconds()); + double cost = 0; + if (launchInfo.jobType().equals("JOB_TYPE_STREAMING")) { + cost += metrics.get("TotalVcpuTime") / 3600 * VCPU_PER_HR_STREAMING; + cost += (metrics.get("TotalMemoryUsage") / 1000) / 3600 * MEM_PER_GB_HR_STREAMING; + cost += metrics.get("TotalStreamingDataProcessed") * SHUFFLE_PER_GB_STREAMING; + // Also, add other streaming metrics + metrics.putAll(getDataFreshnessMetrics(launchInfo.jobId(), workerTimeInterval)); + metrics.putAll(getSystemLatencyMetrics(launchInfo.jobId(), workerTimeInterval)); + } else { + cost += metrics.get("TotalVcpuTime") / 3600 * VCPU_PER_HR_BATCH; + cost += (metrics.get("TotalMemoryUsage") / 1000) / 3600 * MEM_PER_GB_HR_BATCH; + cost += metrics.get("TotalShuffleDataProcessed") * SHUFFLE_PER_GB_BATCH; + } + cost += metrics.get("TotalPdUsage") / 3600 * PD_PER_GB_HR; + cost += metrics.get("TotalSsdUsage") / 3600 * PD_SSD_PER_GB_HR; + metrics.put("EstimatedCost", cost); + metrics.put("ElapsedTime", monitoringClient.getElapsedTime(project, launchInfo)); + + Double dataProcessed = + monitoringClient.getDataProcessed(project, launchInfo, config.inputPCollection()); + if (dataProcessed != null) { + metrics.put("EstimatedDataProcessedGB", dataProcessed / 1e9d); + } + metrics.putAll(getCpuUtilizationMetrics(launchInfo.jobId(), workerTimeInterval)); + metrics.putAll(getThroughputMetrics(launchInfo, config, workerTimeInterval)); + } + + /** + * Compute metrics of a Direct runner job. + * + * @param metrics a map of raw metrics. The results are also appened in the map. + * @param launchInfo Job info of the job + */ + private void computeDirectMetrics(Map metrics, LaunchInfo launchInfo) + throws ParseException { + // TODO: determine elapsed time more accurately if Direct runner supports do so. + metrics.put( + "ElapsedTime", + 0.001 + * (System.currentTimeMillis() + - Timestamps.toMillis(Timestamps.parse(launchInfo.createTime())))); + } + + /** + * Computes the metrics of the given job using dataflow and monitoring clients. + * + * @param launchInfo Job info of the job + * @param inputPcollection input pcollection of the dataflow job to query additional metrics. If + * not provided, the metrics will not be calculated. + * @param outputPcollection output pcollection of the dataflow job to query additional metrics. If + * not provided, the metrics will not be calculated. + * @return metrics + * @throws IOException if there is an issue sending the request + * @throws ParseException if timestamp is inaccurate + * @throws InterruptedException thrown if thread is interrupted + */ + protected Map getMetrics( + LaunchInfo launchInfo, @Nullable String inputPcollection, @Nullable String outputPcollection) + throws InterruptedException, IOException, ParseException { + return getMetrics( + launchInfo, + MetricsConfiguration.builder() + .setInputPCollection(inputPcollection) + .setOutputPCollection(outputPcollection) + .build()); + } + + /** + * Computes the metrics of the given job using dataflow and monitoring clients. + * + * @param launchInfo Job info of the job + * @param inputPcollection input pcollection of the dataflow job to query additional metrics. If + * not provided, the metrics will not be calculated. + * @return metrics + * @throws IOException if there is an issue sending the request + * @throws ParseException if timestamp is inaccurate + * @throws InterruptedException thrown if thread is interrupted + */ + protected Map getMetrics(LaunchInfo launchInfo, String inputPcollection) + throws ParseException, InterruptedException, IOException { + return getMetrics(launchInfo, inputPcollection, null); + } + + /** + * Computes the metrics of the given job using dataflow and monitoring clients. + * + * @param launchInfo Job info of the job + * @return metrics + * @throws IOException if there is an issue sending the request + * @throws ParseException if timestamp is inaccurate + * @throws InterruptedException thrown if thread is interrupted + */ + protected Map getMetrics(LaunchInfo launchInfo) + throws ParseException, InterruptedException, IOException { + return getMetrics(launchInfo, null, null); + } + + protected Map getMetrics(LaunchInfo launchInfo, MetricsConfiguration config) + throws IOException, InterruptedException, ParseException { + Map metrics = pipelineLauncher.getMetrics(project, region, launchInfo.jobId()); + if (launchInfo.runner().contains("Dataflow")) { + // monitoring metrics take up to 3 minutes to show up + // TODO(pranavbhandari): We should use a library like http://awaitility.org/ to poll for + // metrics instead of hard coding X minutes. + LOG.info("Sleeping for 4 minutes to query Dataflow runner metrics."); + Thread.sleep(Duration.ofMinutes(4).toMillis()); + computeDataflowMetrics(metrics, launchInfo, config); + } else if ("DirectRunner".equalsIgnoreCase(launchInfo.runner())) { + computeDirectMetrics(metrics, launchInfo); + } + return metrics; + } + + /** + * Computes CPU Utilization metrics of the given job. + * + * @param jobId dataflow job id + * @param timeInterval interval for the monitoring query + * @return CPU Utilization metrics of the job + */ + protected Map getCpuUtilizationMetrics(String jobId, TimeInterval timeInterval) { + List cpuUtilization = monitoringClient.getCpuUtilization(project, jobId, timeInterval); + Map cpuUtilizationMetrics = new HashMap<>(); + if (cpuUtilization == null) { + return cpuUtilizationMetrics; + } + cpuUtilizationMetrics.put("AvgCpuUtilization", calculateAverage(cpuUtilization)); + cpuUtilizationMetrics.put("MaxCpuUtilization", Collections.max(cpuUtilization)); + return cpuUtilizationMetrics; + } + + /** + * Computes throughput metrics of the given pcollection in dataflow job. + * + * @param jobInfo dataflow job LaunchInfo + * @param config the {@class MetricsConfiguration} + * @param timeInterval interval for the monitoring query + * @return throughput metrics of the pcollection + */ + protected Map getThroughputMetrics( + LaunchInfo jobInfo, MetricsConfiguration config, TimeInterval timeInterval) { + String jobId = jobInfo.jobId(); + String iColl = + RUNNER_V2.equals(jobInfo.runner()) + ? config.inputPCollectionV2() + : config.inputPCollection(); + String oColl = + RUNNER_V2.equals(jobInfo.runner()) + ? config.outputPCollectionV2() + : config.outputPCollection(); + List inputThroughputBytesPerSec = + monitoringClient.getThroughputBytesPerSecond(project, jobId, iColl, timeInterval); + List inputThroughputElementsPerSec = + monitoringClient.getThroughputElementsPerSecond(project, jobId, iColl, timeInterval); + List outputThroughputBytesPerSec = + monitoringClient.getThroughputBytesPerSecond(project, jobId, oColl, timeInterval); + List outputThroughputElementsPerSec = + monitoringClient.getThroughputElementsPerSecond(project, jobId, oColl, timeInterval); + return getThroughputMetrics( + inputThroughputBytesPerSec, + inputThroughputElementsPerSec, + outputThroughputBytesPerSec, + outputThroughputElementsPerSec); + } + + /** + * Computes Data freshness metrics of the given job. + * + * @param jobId dataflow job id + * @param timeInterval interval for the monitoring query + * @return Data freshness metrics of the job + */ + protected Map getDataFreshnessMetrics(String jobId, TimeInterval timeInterval) { + List dataFreshness = monitoringClient.getDataFreshness(project, jobId, timeInterval); + Map dataFreshnessMetrics = new HashMap<>(); + if (dataFreshness == null) { + return dataFreshnessMetrics; + } + dataFreshnessMetrics.put("AvgDataFreshness", calculateAverage(dataFreshness)); + dataFreshnessMetrics.put("MaxDataFreshness", Collections.max(dataFreshness)); + return dataFreshnessMetrics; + } + + /** + * Computes System latency metrics of the given job. + * + * @param jobId dataflow job id + * @param timeInterval interval for the monitoring query + * @return System latency metrics of the job + */ + protected Map getSystemLatencyMetrics(String jobId, TimeInterval timeInterval) { + List systemLatency = monitoringClient.getSystemLatency(project, jobId, timeInterval); + Map systemLatencyMetrics = new HashMap<>(); + if (systemLatency == null) { + return systemLatencyMetrics; + } + systemLatencyMetrics.put("AvgSystemLatency", calculateAverage(systemLatency)); + systemLatencyMetrics.put("MaxSystemLatency", Collections.max(systemLatency)); + return systemLatencyMetrics; + } + + /** Gets the time interval when workers were active to be used by monitoring client. */ + protected TimeInterval getWorkerTimeInterval(LaunchInfo info) throws ParseException { + TimeInterval.Builder builder = TimeInterval.newBuilder(); + List messages = + pipelineLauncher.listMessages(project, region, info.jobId(), "JOB_MESSAGE_DETAILED"); + for (JobMessage jobMessage : messages) { + if (jobMessage.getMessageText() != null && !jobMessage.getMessageText().isEmpty()) { + if (WORKER_START_PATTERN.matcher(jobMessage.getMessageText()).find()) { + LOG.info("Found worker start message in job messages."); + builder.setStartTime(Timestamps.parse(jobMessage.getTime())); + } + if (WORKER_STOP_PATTERN.matcher(jobMessage.getMessageText()).find()) { + LOG.info("Found worker stop message in job messages."); + builder.setEndTime(Timestamps.parse(jobMessage.getTime())); + } + } + } + return builder.build(); + } + + private Map getThroughputMetrics( + List inputThroughput, + List inputThroughputElementsPerSec, + List outputThroughput, + List outputThroughputElementsPerSec) { + Map throughputMetrics = new HashMap<>(); + if (inputThroughput != null) { + throughputMetrics.put("AvgInputThroughputBytesPerSec", calculateAverage(inputThroughput)); + throughputMetrics.put("MaxInputThroughputBytesPerSec", Collections.max(inputThroughput)); + } + if (inputThroughputElementsPerSec != null) { + throughputMetrics.put( + "AvgInputThroughputElementsPerSec", calculateAverage(inputThroughputElementsPerSec)); + throughputMetrics.put( + "MaxInputThroughputElementsPerSec", Collections.max(inputThroughputElementsPerSec)); + } + if (outputThroughput != null) { + throughputMetrics.put("AvgOutputThroughputBytesPerSec", calculateAverage(outputThroughput)); + throughputMetrics.put("MaxOutputThroughputBytesPerSec", Collections.max(outputThroughput)); + } + if (outputThroughputElementsPerSec != null) { + throughputMetrics.put( + "AvgOutputThroughputElementsPerSec", calculateAverage(outputThroughputElementsPerSec)); + throughputMetrics.put( + "MaxOutputThroughputElementsPerSec", Collections.max(outputThroughputElementsPerSec)); + } + return throughputMetrics; + } + + /** + * Calculate the average from a series. + * + * @param values the input series. + * @return the averaged result. + */ + public static Double calculateAverage(List values) { + return values.stream().mapToDouble(d -> d).average().orElse(0.0); + } + + private static void putOptional(Map map, String key, @Nullable Object value) { + if (value != null) { + map.put(key, value); + } + } + + public static PipelineOperator.Config createConfig(LaunchInfo info, Duration timeout) { + return PipelineOperator.Config.builder() + .setJobId(info.jobId()) + .setProject(project) + .setRegion(region) + .setTimeoutAfter(timeout) + .build(); + } + + /** Utils for the metrics. */ + @AutoValue + public abstract static class MetricsConfiguration { + + /** + * Input PCollection of the Dataflow job to query additional metrics. If not provided, the + * metrics for inputPCollection will not be calculated. + */ + public abstract @Nullable String inputPCollection(); + + /** Input PCollection name under Dataflow runner v2. */ + public abstract @Nullable String inputPCollectionV2(); + + /** + * Input PCollection of the Dataflow job to query additional metrics. If not provided, the + * metrics for inputPCollection will not be calculated. + */ + public abstract @Nullable String outputPCollection(); + + public abstract @Nullable String outputPCollectionV2(); + + public static MetricsConfiguration.Builder builder() { + return new AutoValue_LoadTestBase_MetricsConfiguration.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + + public abstract MetricsConfiguration.Builder setInputPCollection(@Nullable String value); + + public abstract MetricsConfiguration.Builder setInputPCollectionV2(@Nullable String value); + + public abstract MetricsConfiguration.Builder setOutputPCollection(@Nullable String value); + + public abstract MetricsConfiguration.Builder setOutputPCollectionV2(@Nullable String value); + + public abstract MetricsConfiguration build(); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/Artifact.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/Artifact.java new file mode 100644 index 0000000000000..830f44598be76 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/Artifact.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.artifacts; + +/** + * Represents a single artifact. + * + *

    An "artifact" is an entity in object storage, file storage, or block storage. Artifacts should + * be able to be stored in-memory as a single byte array. Implementations with an underlying type + * that only supports streaming should stream in the full contents and make the full contents + * available. + * + *

    Implementations should remain read-only. Writing artifacts should be left to the + * responsibility of a {@link ArtifactClient} implementation. If an object of the artifact type + * returned allows writing of any type, then it should not be made available. + */ +public interface Artifact { + /** Returns the id of the artifact. */ + String id(); + + /** Returns the name/path of the artifact. */ + String name(); + + /** Returns the raw byte array of the artifact's contents. */ + byte[] contents(); +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/ArtifactClient.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/ArtifactClient.java new file mode 100644 index 0000000000000..e463baffeaf1b --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/ArtifactClient.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.artifacts; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.regex.Pattern; +import org.apache.beam.it.common.ResourceManager; +import org.junit.rules.TestName; + +/** + * Interface for working with test artifacts. + * + *

    It is the responsibility of implementations to make sure that artifacts are kept separate from + * each other. Using a GCS path, this isolation would create a path like the following: {@code + * gs://test-class-name/run-id/test-method-name}. Each directory means: + * + *

      + *
    • test-class-name: A name given to the directory for a test class. This does not need to be + * identical to the class name, but it should clearly identify the class from other test + * classes. This is intended for long-lived artifacts that have value beyond a specific run of + * a test, such as a result file. + *
    • run-id: An id assigned to a run of that test class. This will be handled by implementations + * of this client. It is intended for artifacts that may be referenced by multiple methods in + * a test class. + *
    • test-method-name: A name given to the directory for a method within the test class. This + * does not need to be identical to the method name, but it should clearly identify the method + * from other test methods within the same test class. This is intended for input and output + * artifacts specific to the test method. + *
    + * + *

    Separate input/output directories are optional and the responsibility of the test writer to + * maintain. + */ +public interface ArtifactClient extends ResourceManager { + + /** Returns the id associated with the particular run of the test class. */ + String runId(); + + /** + * Returns a path the artifact will be created at. + * + * @param artifactName Artifact name + * @return GCS path where the artifact will be created + */ + String getPathForArtifact(String artifactName); + + /** + * Creates a new artifact in whatever service is being used to store them. + * + * @param artifactName the name of the artifact. If this is supposed to go under an input/output + * directory, then it should include that (example: input/artifact.txt) + * @param contents the contents of the artifact in String format + * @return a representation of the created artifact + */ + Artifact createArtifact(String artifactName, String contents); + + /** + * Creates a new artifact in whatever service is being used to store them. + * + * @param artifactName the name of the artifact. If this is supposed to go under an input/output + * directory, then it should include that (example: input/artifact.txt) + * @param contents the contents of the artifact in byte array format + * @return a representation of the created artifact + */ + Artifact createArtifact(String artifactName, byte[] contents); + + /** + * Uploads a local file to the service being used for storing artifacts. + * + * @param artifactName the name of the artifact. If this is supposed to go under an input/output + * directory, then it should include that (example: input/artifact.txt) + * @param localPath the absolute local path to the file to upload + * @return a representation of the uploaded artifact + * @throws IOException if there is an issue reading the local file + */ + Artifact uploadArtifact(String artifactName, String localPath) throws IOException; + + /** + * Uploads a local file to the service being used for storing artifacts. + * + * @param artifactName the name of the artifact. If this is supposed to go under an input/output + * directory, then it should include that (example: input/artifact.txt) + * @param localPath the local path to the file to upload + * @return a representation of the uploaded artifact + * @throws IOException if there is an issue reading the local file + */ + Artifact uploadArtifact(String artifactName, Path localPath) throws IOException; + + // TODO(zhoufek): Add equivalents for the above for uploading artifacts of a test method + + /** + * Lists all artifacts under test-class-name/run-id/{@code prefix}. + * + * @param testName the test name to use as the prefix on the testing artifacts. + * @param regex a regex to use for filtering out unwanted artifacts + * @return all the artifacts whose name matches regex + */ + List listArtifacts(TestName testName, Pattern regex); + + /** + * Lists all artifacts under test-class-name/run-id/{@code prefix}. + * + * @param prefix the prefix to use along with the fixed values method above. This must include the + * test-method-name value, but it can include other directories or files under it. + * @param regex a regex to use for filtering out unwanted artifacts + * @return all the artifacts whose name matches regex + */ + List listArtifacts(String prefix, Pattern regex); + + /** Deletes all the files located under test-class-name/run-id. */ + @Override + void cleanupAll(); +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/GcsArtifact.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/GcsArtifact.java new file mode 100644 index 0000000000000..c60e3a0fe53a9 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/GcsArtifact.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.artifacts; + +import com.google.cloud.storage.Blob; + +/** Represents a single blob in Google Cloud Storage (GCS). */ +public final class GcsArtifact implements Artifact { + public final Blob blob; + + public GcsArtifact(Blob blob) { + this.blob = blob; + } + + @Override + public String id() { + return blob.getGeneratedId(); + } + + @Override + public String name() { + return blob.getName(); + } + + @Override + public byte[] contents() { + return blob.getContent(); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers/ArtifactAsserts.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers/ArtifactAsserts.java new file mode 100644 index 0000000000000..3b1251307d0d1 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers/ArtifactAsserts.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.artifacts.matchers; + +import static com.google.common.truth.Truth.assertAbout; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatRecords; + +import java.util.List; +import org.apache.avro.generic.GenericRecord; +import org.apache.beam.it.gcp.artifacts.Artifact; +import org.apache.beam.it.truthmatchers.RecordsSubject; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; + +public class ArtifactAsserts { + + /** + * Creates an {@link ArtifactsSubject} to assert information within a list of artifacts obtained + * from Cloud Storage. + * + * @param artifacts Artifacts in list format to use in the comparisons. + * @return Truth Subject to chain assertions. + */ + public static ArtifactsSubject assertThatArtifacts(List artifacts) { + return assertAbout(ArtifactsSubject.records()).that(artifacts); + } + + /** + * Creates an {@link ArtifactsSubject} to assert information for an artifact obtained from Cloud + * Storage. + * + * @param artifact Artifact to use in the comparisons. + * @return Truth Subject to chain assertions. + */ + public static ArtifactsSubject assertThatArtifact(Artifact artifact) { + return assertAbout(ArtifactsSubject.records()).that(ImmutableList.of(artifact)); + } + + /** + * Creates an {@link RecordsSubject} to assert information within a list of records. + * + * @param records Records in Avro/Parquet {@link GenericRecord} format to use in the comparison. + * @return Truth Subject to chain assertions. + */ + public static RecordsSubject assertThatGenericRecords(List records) { + return assertThatRecords(ArtifactsSubject.genericRecordToRecords(records)); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers/ArtifactsSubject.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers/ArtifactsSubject.java new file mode 100644 index 0000000000000..06ea14d0390b4 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers/ArtifactsSubject.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.artifacts.matchers; + +import static org.apache.beam.it.gcp.artifacts.matchers.ArtifactAsserts.assertThatGenericRecords; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatRecords; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing.sha256; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.truth.FailureMetadata; +import com.google.common.truth.Subject; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.beam.it.gcp.artifacts.Artifact; +import org.apache.beam.it.gcp.artifacts.utils.AvroTestUtil; +import org.apache.beam.it.gcp.artifacts.utils.JsonTestUtil; +import org.apache.beam.it.gcp.artifacts.utils.ParquetTestUtil; +import org.apache.beam.it.truthmatchers.RecordsSubject; + +/** + * Subject that has assertion operations for artifact lists (GCS files), usually coming from the + * result of a template. + */ +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) +public final class ArtifactsSubject extends Subject { + + private final List actual; + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private static final TypeReference> recordTypeReference = + new TypeReference>() {}; + + private ArtifactsSubject(FailureMetadata metadata, List actual) { + super(metadata, actual); + this.actual = actual; + } + + public static Factory> records() { + return ArtifactsSubject::new; + } + + /** Check if artifact list has files (is not empty). */ + public void hasFiles() { + check("there are files").that(actual).isNotEmpty(); + } + + /** + * Check if artifact list has a specific number of files. + * + * @param expectedFiles Expected files + */ + public void hasFiles(int expectedFiles) { + check("there are %d files", expectedFiles).that(actual.size()).isEqualTo(expectedFiles); + } + + /** + * Check if any of the artifacts has a specific content. + * + * @param content Content to search for + */ + public void hasContent(String content) { + if (actual.stream() + .noneMatch( + artifact -> + new String(artifact.contents(), StandardCharsets.UTF_8).contains(content))) { + failWithActual("expected to contain", content); + } + } + + /** + * Check if any of the artifacts have a specific content hash (using SHA-256). + * + * @param hash Content to search for + */ + public void hasHash(String hash) { + if (actual.stream() + .noneMatch(artifact -> sha256().hashBytes(artifact.contents()).toString().equals(hash))) { + failWithActual("expected to contain hash", hash); + } + } + + /** + * Parse artifacts to Avro records to be used for assertions. + * + * @param schema Avro Schema to use on the conversion. + */ + public RecordsSubject asAvroRecords(Schema schema) { + List allRecords = new ArrayList<>(); + + for (Artifact artifact : this.actual) { + try { + allRecords.addAll(AvroTestUtil.readRecords(schema, artifact.contents())); + } catch (Exception e) { + throw new RuntimeException("Error reading " + artifact.name() + " as Avro.", e); + } + } + return assertThatGenericRecords(allRecords); + } + + /** Parse artifacts to Parquet GenericRecord to be used for assertions. */ + public RecordsSubject asParquetRecords() { + List allRecords = new ArrayList<>(); + + for (Artifact artifact : this.actual) { + try { + allRecords.addAll(ParquetTestUtil.readRecords(artifact.contents())); + } catch (Exception e) { + throw new RuntimeException("Error reading " + artifact.name() + " as Parquet.", e); + } + } + return assertThatGenericRecords(allRecords); + } + + /** + * Convert Avro {@link GenericRecord} to a list of maps. + * + * @param avroRecords Avro Records to parse + * @return List of maps to use in {@link RecordsSubject} + */ + public static List> genericRecordToRecords(List avroRecords) { + try { + List> records = new ArrayList<>(); + + for (GenericRecord row : avroRecords) { + Map converted = objectMapper.readValue(row.toString(), recordTypeReference); + records.add(converted); + } + + return records; + } catch (Exception e) { + throw new RuntimeException("Error converting Avro Record to Map", e); + } + } + + /** Parse artifacts as Json records to be used for assertions. */ + public RecordsSubject asJsonRecords() { + List> allRecords = new ArrayList<>(); + + for (Artifact artifact : this.actual) { + try { + allRecords.addAll(JsonTestUtil.readRecords(artifact.contents())); + } catch (Exception e) { + throw new RuntimeException("Error reading " + artifact.name() + " as JSON.", e); + } + } + return assertThatRecords(allRecords); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers/package-info.java new file mode 100644 index 0000000000000..9759071b397e8 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for Artifact Truth matchers / subjects to have reusable assertions. */ +package org.apache.beam.it.gcp.artifacts.matchers; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/package-info.java new file mode 100644 index 0000000000000..de2a2541d5582 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for working with test artifacts. */ +package org.apache.beam.it.gcp.artifacts; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/utils/ArtifactUtils.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/utils/ArtifactUtils.java new file mode 100644 index 0000000000000..b74ec23dd7ec8 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/utils/ArtifactUtils.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.artifacts.utils; + +import static java.util.Arrays.stream; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import com.google.auth.Credentials; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.UUID; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; + +/** Utilities for working with test artifacts. */ +public final class ArtifactUtils { + private ArtifactUtils() {} + + /** Creates a unique id for the run. */ + public static String createRunId() { + return String.format( + "%s-%s", + DateTimeFormatter.ofPattern("yyyyMMdd").withZone(ZoneId.of("UTC")).format(Instant.now()), + // by default some templates replace "dd" in the output paths with a day of the month, since + // this id is used as a part of output paths replace "dd" with an arbitrary number to avoid + // confusion and potential flaky tests + UUID.randomUUID().toString().replace("dd", "99")); + } + + /** + * Returns the full GCS path given a list of path parts. + * + *

    "path parts" refers to the bucket, directories, and file. Only the bucket is mandatory and + * must be the first value provided. + * + * @param pathParts everything that makes up the path, minus the separators. There must be at + * least one value, and none of them can be empty + * @return the full path, such as 'gs://bucket/dir1/dir2/file' + */ + public static String getFullGcsPath(String... pathParts) { + checkArgument(pathParts.length != 0, "Must provide at least one path part"); + checkArgument( + stream(pathParts).noneMatch(Strings::isNullOrEmpty), "No path part can be null or empty"); + + return String.format("gs://%s", String.join("/", pathParts)); + } + + /** + * Creates a client for GCS with the given credentials. + * + * @param credentials credentials to use for connecting. If not chosen, then this will use the + * system credentials. Using system credentials is intended only for local testing. Otherwise, + * it is best to pass in a short-lived access token. + * @return a {@link Storage} client for running GCS operations + */ + public static Storage createStorageClient(Credentials credentials) { + StorageOptions.Builder builder = StorageOptions.newBuilder(); + if (credentials != null) { + builder.setCredentials(credentials); + } + return builder.build().getService(); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/utils/AvroTestUtil.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/utils/AvroTestUtil.java new file mode 100644 index 0000000000000..5cfe4c137e351 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/utils/AvroTestUtil.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.artifacts.utils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.avro.Schema; +import org.apache.avro.file.DataFileReader; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.file.SeekableByteArrayInput; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; + +/** + * The {@link AvroTestUtil} class provides common utilities used for executing tests that involve + * Avro. + */ +public class AvroTestUtil { + + /** + * Create Avro file for the given schema and records list. + * + * @param schema Schema to use. + * @param records Records to write on the sink. + * @return Byte array that represents the content. + */ + public static byte[] createAvroFile(Schema schema, List records) + throws IOException { + GenericDatumWriter writer = new GenericDatumWriter<>(schema); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataFileWriter fileWriter = new DataFileWriter<>(writer).create(schema, baos); + for (GenericRecord record : records) { + fileWriter.append(record); + } + fileWriter.close(); + + return baos.toByteArray(); + } + + /** + * Read Avro records to a list of {@link GenericRecord}. + * + * @param schema Schema to use for reading. + * @param contents Byte array with contents to read. + * @return A list with all records. + */ + public static List readRecords(Schema schema, byte[] contents) throws IOException { + List records = new ArrayList<>(); + + GenericDatumReader datum = new GenericDatumReader<>(schema); + DataFileReader reader = + new DataFileReader<>(new SeekableByteArrayInput(contents), datum); + + while (reader.hasNext()) { + records.add(reader.next()); + } + + return records; + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/utils/JsonTestUtil.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/utils/JsonTestUtil.java new file mode 100644 index 0000000000000..1ef12d33fa111 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/utils/JsonTestUtil.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.artifacts.utils; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.MappingIterator; +import com.fasterxml.jackson.databind.json.JsonMapper; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * The {@link JsonTestUtil} class provides common utilities used for executing tests that involve + * Json. + */ +public class JsonTestUtil { + + private static final TypeReference> mapTypeRef = + new TypeReference>() {}; + + /** + * Read JSON records to a list of Maps. + * + * @param contents Byte array with contents to read. + * @return A list with all records. + */ + public static List> readRecords(byte[] contents) throws IOException { + List> records = new ArrayList<>(); + + JsonMapper mapper = new JsonMapper(); + + try (MappingIterator> iterator = + mapper.readerFor(mapTypeRef).readValues(contents)) { + while (iterator.hasNextValue()) { + records.add(iterator.next()); + } + } + + return records; + } + + /** + * Read JSON records to a list of Maps. + * + * @param contents String with contents to read. + * @return A list with all records. + */ + public static List> readRecords(String contents) throws IOException { + return readRecords(contents.getBytes(StandardCharsets.UTF_8)); + } + + /** + * Read JSON record to a Map. + * + * @param contents Byte array with contents to read. + * @return A map with the records. + */ + public static Map readRecord(byte[] contents) throws IOException { + JsonMapper mapper = new JsonMapper(); + return mapper.readerFor(mapTypeRef).readValue(contents); + } + + /** + * Read JSON record to a Map. + * + * @param contents String with contents to read. + * @return A map with the records. + */ + public static Map readRecord(String contents) throws IOException { + return readRecord(contents.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/utils/ParquetTestUtil.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/utils/ParquetTestUtil.java new file mode 100644 index 0000000000000..d6ebf825e2c1e --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/utils/ParquetTestUtil.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.artifacts.utils; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.avro.AvroParquetReader; +import org.apache.parquet.avro.AvroParquetWriter; +import org.apache.parquet.hadoop.ParquetReader; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.io.DelegatingSeekableInputStream; +import org.apache.parquet.io.InputFile; +import org.apache.parquet.io.SeekableInputStream; + +/** + * The {@link ParquetTestUtil} class provides common utilities used for executing tests that involve + * Parquet. + */ +public class ParquetTestUtil { + + /** + * Create Parquet file for the given schema and records list. + * + * @param schema Schema to use. + * @param records Records to write to the sink. + * @return Byte array that represents the content. + */ + public static byte[] createParquetFile(Schema schema, List records) + throws IOException { + + File tempFile = createTempFile(); + Path file = new Path(tempFile.getPath()); + + AvroParquetWriter.Builder builder = AvroParquetWriter.builder(file); + ParquetWriter parquetWriter = builder.withSchema(schema).build(); + for (GenericRecord record : records) { + parquetWriter.write(record); + } + parquetWriter.close(); + + return Files.readAllBytes(tempFile.toPath()); + } + + /** + * Read Parquet records to a list of {@link GenericRecord}. + * + * @param contents Byte array with contents to read. + * @return A list with all records. + */ + public static List readRecords(byte[] contents) throws IOException { + List records = new ArrayList<>(); + + try (ParquetReader reader = + AvroParquetReader.builder(new ParquetByteArrayInput(contents)).build()) { + GenericRecord record; + while ((record = reader.read()) != null) { + records.add(record); + } + } + return records; + } + + private static File createTempFile() throws IOException { + File tempFile = File.createTempFile(ParquetTestUtil.class.getSimpleName(), ".tmp"); + tempFile.deleteOnExit(); + boolean unused = tempFile.delete(); + return tempFile; + } + + static class ParquetByteArrayInput implements InputFile { + private final byte[] data; + + private static class SeekByteArray extends ByteArrayInputStream { + public SeekByteArray(byte[] buf) { + super(buf); + } + + public void setPos(int pos) { + this.pos = pos; + } + + public int getPos() { + return this.pos; + } + } + + public ParquetByteArrayInput(byte[] data) { + this.data = data; + } + + @Override + public long getLength() { + return this.data.length; + } + + @Override + public SeekableInputStream newStream() throws IOException { + return new DelegatingSeekableInputStream(new SeekByteArray(this.data)) { + @Override + public void seek(long newPos) { + ((SeekByteArray) this.getStream()).setPos((int) newPos); + } + + @Override + public long getPos() { + return ((SeekByteArray) this.getStream()).getPos(); + } + }; + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/utils/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/utils/package-info.java new file mode 100644 index 0000000000000..53cb4ed1d3af5 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/utils/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for artifact utilities. */ +package org.apache.beam.it.gcp.artifacts.utils; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManager.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManager.java new file mode 100644 index 0000000000000..d6d348f524b29 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManager.java @@ -0,0 +1,501 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigquery; + +import com.google.api.gax.paging.Page; +import com.google.auth.Credentials; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryError; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.Dataset; +import com.google.cloud.bigquery.DatasetInfo; +import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; +import com.google.cloud.bigquery.InsertAllResponse; +import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableDefinition; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; +import com.google.cloud.bigquery.TableResult; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import org.apache.beam.it.common.ResourceManager; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Client for managing BigQuery resources. + * + *

    The class supports one dataset, and multiple tables per dataset object. + * + *

    The class is thread-safe. + */ +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/27438) +}) +public final class BigQueryResourceManager implements ResourceManager { + + private static final Logger LOG = LoggerFactory.getLogger(BigQueryResourceManager.class); + private static final String DEFAULT_DATASET_REGION = "us-central1"; + + private final String projectId; + private final String datasetId; + + private final BigQuery bigQuery; + private Dataset dataset; + + private BigQueryResourceManager(Builder builder) { + // create bigQuery client + BigQueryOptions.Builder bigQueryOptions = + BigQueryOptions.newBuilder().setProjectId(builder.projectId); + + // set credentials, if provided + if (builder.credentials != null) { + bigQueryOptions.setCredentials(builder.credentials); + } + this.bigQuery = bigQueryOptions.build().getService(); + this.projectId = builder.projectId; + + // If datasetId is provided, get the dataset. + if (builder.datasetId != null) { + this.datasetId = builder.datasetId; + dataset = getDatasetIfExists(this.datasetId); + } else { + this.datasetId = BigQueryResourceManagerUtils.generateDatasetId(builder.testId); + this.dataset = null; + } + } + + @VisibleForTesting + BigQueryResourceManager(String testId, String projectId, BigQuery bigQuery) { + this.datasetId = BigQueryResourceManagerUtils.generateDatasetId(testId); + this.projectId = projectId; + this.bigQuery = bigQuery; + } + + public static Builder builder(String testId, String projectId, Credentials credentials) { + return new Builder(testId, projectId, credentials); + } + + public String getProjectId() { + return projectId; + } + + /** + * Return the dataset ID this Resource Manager uses to create and manage tables in. + * + * @return the dataset ID. + */ + public String getDatasetId() { + return datasetId; + } + + /** + * Helper method for determining if a dataset exists in the resource manager. + * + * @throws IllegalStateException if a dataset does not exist. + */ + private void checkHasDataset() { + if (dataset == null) { + throw new IllegalStateException("There is no dataset for manager to perform operation on."); + } + } + + /** + * Helper method for fetching a dataset stored in the project given the datasetId. + * + * @param datasetId the name of the dataset to fetch. + * @return the dataset, if it exists. + * @throws IllegalStateException if the given dataset does not exist in the project. + */ + private synchronized Dataset getDatasetIfExists(String datasetId) throws IllegalStateException { + Dataset dataset = bigQuery.getDataset(datasetId); + if (dataset == null) { + throw new IllegalStateException( + "The dataset " + datasetId + " does not exist in project " + projectId + "."); + } + return dataset; + } + + /** + * Helper method for fetching a table stored in the dataset given a table name. + * + * @param tableId the name of the table to fetch. + * @return the table, if it exists. + * @throws IllegalStateException if the given table name does not exist in the dataset. + */ + private synchronized Table getTableIfExists(String tableId) throws IllegalStateException { + checkHasDataset(); + Table table = dataset.get(tableId); + if (table == null) { + throw new IllegalStateException( + "The table " + tableId + " does not exist in dataset " + datasetId + "."); + } + return table; + } + + /** + * Helper method for logging individual errors thrown by inserting rows to a table. This method is + * used to log errors thrown by inserting certain rows when other rows were successful. + * + * @param insertErrors the map of errors to log. + */ + private void logInsertErrors(Map> insertErrors) { + for (Map.Entry> entries : insertErrors.entrySet()) { + long index = entries.getKey(); + for (BigQueryError error : entries.getValue()) { + LOG.info("Error when inserting row with index {}: {}", index, error.getMessage()); + } + } + } + + /** + * Create a BigQuery dataset in which all tables will exist. + * + * @param region the region to store the dataset in. + * @return Dataset id that was created + * @throws BigQueryResourceManagerException if there is an error creating the dataset in BigQuery. + */ + public synchronized String createDataset(String region) throws BigQueryResourceManagerException { + + // Check to see if dataset already exists, and throw error if it does + if (dataset != null) { + throw new IllegalStateException( + "Dataset " + datasetId + " already exists for project " + projectId + "."); + } + + LOG.info("Creating dataset {} in project {}.", datasetId, projectId); + + // Send the dataset request to Google Cloud + try { + DatasetInfo datasetInfo = DatasetInfo.newBuilder(datasetId).setLocation(region).build(); + LOG.info("Dataset {} created successfully", datasetId); + dataset = bigQuery.create(datasetInfo); + return datasetId; + } catch (Exception e) { + throw new BigQueryResourceManagerException("Failed to create dataset.", e); + } + } + + /** + * Creates a table within the current dataset given a table name and schema. + * + *

    This table will automatically expire 1 hour after creation if not cleaned up manually or by + * calling the {@link BigQueryResourceManager#cleanupAll()} method. + * + *

    Note: Implementations may do dataset creation here, if one does not already exist. + * + * @param tableName The name of the table. + * @param schema A schema object that defines the table. + * @return The TableId (reference) to the table + * @throws BigQueryResourceManagerException if there is an error creating the table in BigQuery. + */ + public synchronized TableId createTable(String tableName, Schema schema) + throws BigQueryResourceManagerException { + return createTable(tableName, schema, System.currentTimeMillis() + 3600000); // 1h + } + + /** + * Creates a table within the current dataset given a table name and schema. + * + *

    This table will automatically expire at the time specified by {@code expirationTime} if not + * cleaned up manually or by calling the {@link BigQueryResourceManager#cleanupAll()} method. + * + *

    Note: Implementations may do dataset creation here, if one does not already exist. + * + * @param tableName The name of the table. + * @param schema A schema object that defines the table. + * @param expirationTimeMillis Sets the time when this table expires, in milliseconds since the + * epoch. + * @return The TableId (reference) to the table + * @throws BigQueryResourceManagerException if there is an error creating the table in BigQuery. + */ + public synchronized TableId createTable( + String tableName, Schema schema, Long expirationTimeMillis) + throws BigQueryResourceManagerException { + // Check table ID + BigQueryResourceManagerUtils.checkValidTableId(tableName); + + // Check schema + if (schema == null) { + throw new IllegalArgumentException("A valid schema must be provided to create a table."); + } + // Create a default dataset if this resource manager has not already created one + if (dataset == null) { + createDataset(DEFAULT_DATASET_REGION); + } + checkHasDataset(); + LOG.info("Creating table using tableName '{}'.", tableName); + + // Create the table if it does not already exist in the dataset + try { + TableId tableId = TableId.of(dataset.getDatasetId().getDataset(), tableName); + if (bigQuery.getTable(tableId) == null) { + TableDefinition tableDefinition = StandardTableDefinition.of(schema); + TableInfo tableInfo = + TableInfo.newBuilder(tableId, tableDefinition) + .setExpirationTime(expirationTimeMillis) + .build(); + bigQuery.create(tableInfo); + LOG.info( + "Successfully created table {}.{}", dataset.getDatasetId().getDataset(), tableName); + + return tableId; + } else { + throw new IllegalStateException( + "Table " + tableId + " already exists for dataset " + datasetId + "."); + } + } catch (Exception e) { + throw new BigQueryResourceManagerException("Failed to create table.", e); + } + } + + /** + * Writes a given row into a table. This method requires {@link + * BigQueryResourceManager#createTable(String, Schema)} to be called for the target table + * beforehand. + * + * @param tableName The name of the table to insert the given row into. + * @param row A row object representing the table row. + * @throws BigQueryResourceManagerException if method is called after resources have been cleaned + * up, if the manager object has no dataset, if the table does not exist or if there is an + * Exception when attempting to insert the rows. + */ + public synchronized void write(String tableName, RowToInsert row) + throws BigQueryResourceManagerException { + write(tableName, ImmutableList.of(row)); + } + + /** + * Writes a collection of table rows into a single table. This method requires {@link + * BigQueryResourceManager#createTable(String, Schema)} to be called for the target table + * beforehand. + * + * @param tableName The name of the table to insert the given rows into. + * @param rows A collection of table rows. + * @throws BigQueryResourceManagerException if method is called after resources have been cleaned + * up, if the manager object has no dataset, if the table does not exist or if there is an + * Exception when attempting to insert the rows. + */ + public synchronized void write(String tableName, List rows) + throws BigQueryResourceManagerException { + // Exit early if there are no mutations + if (!rows.iterator().hasNext()) { + return; + } + + Table table = getTableIfExists(tableName); + + LOG.info( + "Attempting to write {} records to {}.{}.", + rows.size(), + dataset.getDatasetId().getDataset(), + tableName); + + // Send row mutations to the table + int successfullyWrittenRecords = rows.size(); + try { + InsertAllResponse insertResponse = table.insert(rows); + successfullyWrittenRecords -= insertResponse.getInsertErrors().size(); + + if (insertResponse.hasErrors()) { + LOG.warn("Errors encountered when inserting rows: "); + logInsertErrors(insertResponse.getInsertErrors()); + } + + } catch (Exception e) { + throw new BigQueryResourceManagerException("Failed to write to table.", e); + } + + LOG.info( + "Successfully wrote {} records to {}.{}.", + successfullyWrittenRecords, + dataset.getDatasetId().getDataset(), + tableName); + } + + /** + * Runs the specified query. + * + * @param query the query to execute + */ + public TableResult runQuery(String query) { + try { + TableResult results = bigQuery.query(QueryJobConfiguration.newBuilder(query).build()); + LOG.info("Loaded {} rows from {}", results.getTotalRows(), query); + return results; + } catch (Exception e) { + throw new BigQueryResourceManagerException("Failed to read query " + query, e); + } + } + + /** + * Gets the number of rows in the table. + * + * @param table the name of the table + */ + public Long getRowCount(String table) { + TableResult r = + runQuery(String.format("SELECT COUNT(*) FROM `%s.%s.%s`", projectId, datasetId, table)); + return StreamSupport.stream(r.getValues().spliterator(), false) + .map(fieldValues -> fieldValues.get(0).getLongValue()) + .collect(Collectors.toList()) + .get(0); + } + + /** + * Reads all the rows in a table and returns a TableResult containing a JSON string + * representation. This method requires {@link BigQueryResourceManager#createTable(String, + * Schema)} to be called for the target table beforehand. + * + * @param table The table reference to read rows from. + * @return A TableResult containing all the rows in the table in JSON. + * @throws BigQueryResourceManagerException if method is called after resources have been cleaned + * up, if the manager object has no dataset, if the table does not exist or if there is an + * Exception when attempting to insert the rows. + */ + public synchronized TableResult readTable(TableId table) throws BigQueryResourceManagerException { + return readTable(table.getTable()); + } + + /** + * Reads all the rows in a table and returns a TableResult containing a JSON string + * representation. This method requires {@link BigQueryResourceManager#createTable(String, + * Schema)} to be called for the target table beforehand. + * + * @param tableName The name of the table to read rows from. + * @return A TableResult containing all the rows in the table in JSON. + * @throws BigQueryResourceManagerException if method is called after resources have been cleaned + * up, if the manager object has no dataset, if the table does not exist or if there is an + * Exception when attempting to insert the rows. + */ + public synchronized TableResult readTable(String tableName) + throws BigQueryResourceManagerException { + return readTable(tableName, -1); + } + + /** + * Reads number of rows in a table and returns a TableResult containing a JSON string + * representation. This method requires {@link BigQueryResourceManager#createTable(String, + * Schema)} to be called for the target table beforehand. + * + * @param table The table reference to read rows from. + * @return A TableResult containing all the rows in the table in JSON. + * @throws BigQueryResourceManagerException if method is called after resources have been cleaned + * up, if the manager object has no dataset, if the table does not exist or if there is an + * Exception when attempting to insert the rows. + */ + public synchronized TableResult readTable(TableId table, int numRows) + throws BigQueryResourceManagerException { + return readTable(table.getTable(), numRows); + } + + /** + * Reads number of rows in a table and returns a TableResult containing a JSON string + * representation. This method requires {@link BigQueryResourceManager#createTable(String, + * Schema)} to be called for the target table beforehand. + * + * @param tableName The name of the table to read rows from. + * @param numRows number of rows to read from the table + * @return A TableResult containing all the rows in the table in JSON. + * @throws BigQueryResourceManagerException if method is called after resources have been cleaned + * up, if the manager object has no dataset, if the table does not exist or if there is an + * Exception when attempting to insert the rows. + */ + public synchronized TableResult readTable(String tableName, int numRows) + throws BigQueryResourceManagerException { + getTableIfExists(tableName); + + LOG.info( + "Reading {} rows from {}.{}", + numRows == -1 ? "all" : Integer.toString(numRows), + dataset.getDatasetId().getDataset(), + tableName); + + // Read all the rows from the table given by tableId + String query = + "SELECT TO_JSON_STRING(t) FROM `" + + String.join(".", projectId, datasetId, tableName) + + "` AS t" + + (numRows != -1 ? " LIMIT " + numRows + ";" : ";"); + return runQuery(query); + } + + /** + * Deletes all created resources (dataset and tables) and cleans up the BigQuery client, making + * the manager object unusable. + * + * @throws BigQueryResourceManagerException if there is an error deleting the tables or dataset in + * BigQuery. + */ + @Override + public synchronized void cleanupAll() throws BigQueryResourceManagerException { + LOG.info("Attempting to cleanup manager."); + try { + if (dataset != null) { + Page tables = bigQuery.listTables(dataset.getDatasetId()); + for (Table table : tables.iterateAll()) { + bigQuery.delete( + TableId.of( + projectId, dataset.getDatasetId().getDataset(), table.getTableId().getTable())); + } + bigQuery.delete(dataset.getDatasetId()); + dataset = null; + } + } catch (Exception e) { + throw new BigQueryResourceManagerException("Failed to delete resources.", e); + } + LOG.info("Manager successfully cleaned up."); + } + + /** Builder for {@link BigQueryResourceManager}. */ + public static final class Builder { + + private final String testId; + private final String projectId; + private String datasetId; + private Credentials credentials; + + private Builder(String testId, String projectId, Credentials credentials) { + this.testId = testId; + this.projectId = projectId; + this.datasetId = null; + this.credentials = credentials; + } + + public Builder setDatasetId(String datasetId) { + this.datasetId = datasetId; + return this; + } + + public Builder setCredentials(Credentials credentials) { + this.credentials = credentials; + return this; + } + + public BigQueryResourceManager build() { + return new BigQueryResourceManager(this); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerException.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerException.java new file mode 100644 index 0000000000000..61ef27e452600 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerException.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigquery; + +/** Custom exception for {@link BigQueryResourceManager} implementations. */ +public class BigQueryResourceManagerException extends RuntimeException { + + public BigQueryResourceManagerException(String errorMessage, Throwable err) { + super(errorMessage, err); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerUtils.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerUtils.java new file mode 100644 index 0000000000000..f2b6849caa612 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerUtils.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigquery; + +import static org.apache.beam.it.common.utils.ResourceManagerUtils.generateResourceId; + +import com.google.cloud.bigquery.TableId; +import java.time.format.DateTimeFormatter; +import java.util.regex.Pattern; + +/** Utilities for {@link BigQueryResourceManager} implementations. */ +public final class BigQueryResourceManagerUtils { + + private static final int MAX_DATASET_ID_LENGTH = 1024; + private static final Pattern ILLEGAL_DATASET_ID_CHARS = Pattern.compile("[^a-zA-Z0-9_]"); + private static final int MIN_TABLE_ID_LENGTH = 1; + private static final int MAX_TABLE_ID_LENGTH = 1024; + private static final Pattern ILLEGAL_TABLE_CHARS = Pattern.compile("[^a-zA-Z0-9-_]"); + private static final DateTimeFormatter TIME_FORMAT = + DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss_SSSSSS"); + + private BigQueryResourceManagerUtils() {} + + /** + * Utility function to generate a formatted dataset name. + * + *

    A dataset name must contain only alphanumeric characters and underscores with a max length + * of 1024. + * + * @param datasetName the dataset name to be formatted into a valid ID. + * @return a BigQuery compatible dataset name. + */ + static String generateDatasetId(String datasetName) { + return generateResourceId( + datasetName, ILLEGAL_DATASET_ID_CHARS, "_", MAX_DATASET_ID_LENGTH, TIME_FORMAT); + } + + /** + * Checks whether the given table ID is valid according to GCP constraints. + * + * @param idToCheck the table ID to check. + * @throws IllegalArgumentException if the table ID is invalid. + */ + static void checkValidTableId(String idToCheck) { + if (idToCheck.length() < MIN_TABLE_ID_LENGTH) { + throw new IllegalArgumentException("Table ID cannot be empty. "); + } + if (idToCheck.length() > MAX_TABLE_ID_LENGTH) { + throw new IllegalArgumentException( + "Table ID " + + idToCheck + + " cannot be longer than " + + MAX_TABLE_ID_LENGTH + + " characters."); + } + if (ILLEGAL_TABLE_CHARS.matcher(idToCheck).find()) { + throw new IllegalArgumentException( + "Table ID " + + idToCheck + + " is not a valid ID. Only letters, numbers, hyphens and underscores are allowed."); + } + } + + /** + * Convert a BigQuery TableId to a table spec string. + * + * @param project project in which the table exists. + * @param table TableId to format. + * @return String in the format {project}:{dataset}.{table}. + */ + public static String toTableSpec(String project, TableId table) { + return String.format( + "%s:%s.%s", + table.getProject() != null ? table.getProject() : project, + table.getDataset(), + table.getTable()); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/conditions/BigQueryRowsCheck.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/conditions/BigQueryRowsCheck.java new file mode 100644 index 0000000000000..30ea9a2804844 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/conditions/BigQueryRowsCheck.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigquery.conditions; + +import com.google.auto.value.AutoValue; +import com.google.cloud.bigquery.TableId; +import javax.annotation.Nullable; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.bigquery.BigQueryResourceManager; + +/** ConditionCheck to validate if BigQuery has received a certain number of rows. */ +@AutoValue +public abstract class BigQueryRowsCheck extends ConditionCheck { + + abstract BigQueryResourceManager resourceManager(); + + abstract TableId tableId(); + + abstract Integer minRows(); + + @Nullable + abstract Integer maxRows(); + + @Override + public String getDescription() { + if (maxRows() != null) { + return String.format( + "BigQuery check if table %s has between %d and %d rows", + tableId().getTable(), minRows(), maxRows()); + } + return String.format("BigQuery check if table %s has %d rows", tableId().getTable(), minRows()); + } + + @Override + @SuppressWarnings("unboxing.of.nullable") + public CheckResult check() { + long totalRows = getRowCount(); + if (totalRows < minRows()) { + return new CheckResult( + false, String.format("Expected %d but has only %d", minRows(), totalRows)); + } + if (maxRows() != null && totalRows > maxRows()) { + return new CheckResult( + false, String.format("Expected up to %d but found %d rows", maxRows(), totalRows)); + } + + if (maxRows() != null) { + return new CheckResult( + true, + String.format( + "Expected between %d and %d rows and found %d", minRows(), maxRows(), totalRows)); + } + + return new CheckResult( + true, String.format("Expected at least %d rows and found %d", minRows(), totalRows)); + } + + public static Builder builder(BigQueryResourceManager resourceManager, TableId tableId) { + return new AutoValue_BigQueryRowsCheck.Builder() + .setResourceManager(resourceManager) + .setTableId(tableId); + } + + public Long getRowCount() { + return resourceManager().getRowCount(tableId().getTable()); + } + + /** Builder for {@link BigQueryRowsCheck}. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setResourceManager(BigQueryResourceManager resourceManager); + + public abstract Builder setTableId(TableId tableId); + + public abstract Builder setMinRows(Integer minRows); + + public abstract Builder setMaxRows(Integer maxRows); + + abstract BigQueryRowsCheck autoBuild(); + + public BigQueryRowsCheck build() { + return autoBuild(); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/conditions/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/conditions/package-info.java new file mode 100644 index 0000000000000..05342337d96ba --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/conditions/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** Package that contains reusable BigQuery conditions. */ +package org.apache.beam.it.gcp.bigquery.conditions; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/matchers/BigQueryAsserts.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/matchers/BigQueryAsserts.java new file mode 100644 index 0000000000000..d787c12be574d --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/matchers/BigQueryAsserts.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigquery.matchers; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatRecords; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.cloud.bigquery.FieldValueList; +import com.google.cloud.bigquery.InsertAllRequest; +import com.google.cloud.bigquery.TableResult; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.truthmatchers.RecordsSubject; + +/** Assert utilities for BigQuery tests. */ +public class BigQueryAsserts { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private static final TypeReference> recordTypeReference = + new TypeReference>() {}; + + /** + * Convert BigQuery {@link TableResult} to a list of maps. + * + * @param tableResult Table Result to parse + * @return List of maps to use in {@link RecordsSubject} + */ + public static List> tableResultToRecords(TableResult tableResult) { + try { + List> records = new ArrayList<>(); + + for (FieldValueList row : tableResult.iterateAll()) { + String jsonRow = row.get(0).getStringValue(); + Map converted = objectMapper.readValue(jsonRow, recordTypeReference); + records.add(converted); + } + + return records; + } catch (Exception e) { + throw new RuntimeException("Error converting TableResult to Records", e); + } + } + + /** + * Convert BigQuery {@link InsertAllRequest.RowToInsert} to a list of maps. + * + * @param rows BigQuery rows to parse. + * @return List of maps to use in {@link RecordsSubject} + */ + public static List> bigQueryRowsToRecords( + List rows) { + try { + List> records = new ArrayList<>(); + rows.forEach(row -> records.add(new HashMap<>(row.getContent()))); + + return records; + } catch (Exception e) { + throw new RuntimeException("Error converting BigQuery Row to Map", e); + } + } + + /** + * Convert BigQuery {@link InsertAllRequest.RowToInsert} to a list of maps. + * + * @param rows BigQuery rows to parse. + * @param excludeCols BigQuery columns to filter out of result. + * @return List of maps to use in {@link RecordsSubject} + */ + public static List> bigQueryRowsToRecords( + List rows, List excludeCols) { + List> records = bigQueryRowsToRecords(rows); + try { + excludeCols.forEach(col -> records.forEach(row -> row.remove(col))); + + return records; + } catch (Exception e) { + throw new RuntimeException("Error converting BigQuery Row to Map", e); + } + } + + /** + * Creates a {@link RecordsSubject} to assert information within a list of records. + * + * @param tableResult Records in BigQuery {@link TableResult} format to use in the comparison. + * @return Truth Subject to chain assertions. + */ + public static RecordsSubject assertThatBigQueryRecords(TableResult tableResult) { + return assertThatRecords(tableResultToRecords(tableResult)); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/matchers/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/matchers/package-info.java new file mode 100644 index 0000000000000..e7926c442a777 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/matchers/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for BigQuery Truth matchers / subjects to have reusable assertions. */ +package org.apache.beam.it.gcp.bigquery.matchers; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/package-info.java new file mode 100644 index 0000000000000..f5b250484ec86 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing BigQuery resources within integration tests. */ +package org.apache.beam.it.gcp.bigquery; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/utils/BigQueryTestUtil.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/utils/BigQueryTestUtil.java new file mode 100644 index 0000000000000..de68a6d84b2a9 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/utils/BigQueryTestUtil.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigquery.utils; + +import com.google.cloud.Tuple; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.RandomStringUtils; + +/** Utilities for BigQuery tests. */ +public final class BigQueryTestUtil { + + private BigQueryTestUtil() {} + + /** + * Generate data to be persisted to a BigQuery table for testing. + * + * @param idColumn Column name containing the ID. + * @param numRows Number of rows to generate. + * @param numFields Number of fields in the schema. + * @param maxEntryLength Maximum length for each field. Please note that maxEntryLength can not + * exceed 300 characters. + * @return Tuple containing the schema and the row values. + */ + public static Tuple> generateBigQueryTable( + String idColumn, int numRows, int numFields, int maxEntryLength) { + + Schema schema = createSchema(idColumn, numFields, maxEntryLength); + List bigQueryRows = generateRandomData(schema, idColumn, numRows, maxEntryLength); + + // Return tuple containing the randomly generated schema and table data + return Tuple.of(schema, bigQueryRows); + } + + /** + * Utility method to create a schema based on the given constraints. + * + * @param idColumn Name of the id column to use. + * @param numFields Number of fields. + * @param maxEntryLength Maximum entry. + * @return Generated Schema. + */ + public static Schema createSchema(String idColumn, int numFields, int maxEntryLength) { + // List to store BigQuery schema fields + List bqSchemaFields = new ArrayList<>(); + + // Add unique identifier field + bqSchemaFields.add(Field.of(idColumn, StandardSQLTypeName.INT64)); + + // Generate random fields + for (int i = 1; i < numFields; i++) { + StringBuilder randomField = new StringBuilder(); + + // Field must start with letter + String prependLetter = RandomStringUtils.randomAlphabetic(1); + // Field uses unique number at end to keep name unique + String appendNum = String.valueOf(i); + // Remaining field name is generated randomly within bounds of maxEntryLength + String randomString = + RandomStringUtils.randomAlphanumeric(0, maxEntryLength - appendNum.length()); + + randomField.append(prependLetter).append(randomString).append(appendNum); + bqSchemaFields.add(Field.of(randomField.toString(), StandardSQLTypeName.STRING)); + } + // Create schema and BigQuery table + return Schema.of(bqSchemaFields); + } + + /** + * Utility method to generate random data for a given schema. + * + * @param schema Schema to generate the data for. + * @param idColumn The name of the id column. + * @param numRows Number of rows to create. + * @param maxEntryLength Size of strings to use. + * @return Generated data, a {@link RowToInsert} list. + */ + public static List generateRandomData( + Schema schema, String idColumn, int numRows, int maxEntryLength) { + // Generate random data + List bigQueryRows = new ArrayList<>(); + for (int i = 0; i < numRows; i++) { + Map content = new HashMap<>(); + + // Iterate unique identifier column + content.put(idColumn, i); + + // Generate remaining cells in row + for (int j = 1; j < schema.getFields().size(); j++) { + content.put( + schema.getFields().get(j).getName(), + RandomStringUtils.randomAlphanumeric(1, maxEntryLength)); + } + bigQueryRows.add(RowToInsert.of(content)); + } + return bigQueryRows; + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/utils/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/utils/package-info.java new file mode 100644 index 0000000000000..fc13fc2f17f21 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/utils/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for BigQuery utilities. */ +package org.apache.beam.it.gcp.bigquery.utils; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManager.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManager.java new file mode 100644 index 0000000000000..311ce9575c2e0 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManager.java @@ -0,0 +1,705 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigtable; + +import static org.apache.beam.it.common.utils.ResourceManagerUtils.checkValidProjectId; +import static org.apache.beam.it.gcp.bigtable.BigtableResourceManagerUtils.checkValidTableId; +import static org.apache.beam.it.gcp.bigtable.BigtableResourceManagerUtils.generateDefaultClusters; +import static org.apache.beam.it.gcp.bigtable.BigtableResourceManagerUtils.generateInstanceId; +import static org.awaitility.Awaitility.await; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.api.gax.rpc.ServerStream; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminSettings; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.AppProfile; +import com.google.cloud.bigtable.admin.v2.models.AppProfile.MultiClusterRoutingPolicy; +import com.google.cloud.bigtable.admin.v2.models.AppProfile.RoutingPolicy; +import com.google.cloud.bigtable.admin.v2.models.AppProfile.SingleClusterRoutingPolicy; +import com.google.cloud.bigtable.admin.v2.models.Cluster; +import com.google.cloud.bigtable.admin.v2.models.CreateAppProfileRequest; +import com.google.cloud.bigtable.admin.v2.models.CreateInstanceRequest; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.admin.v2.models.GCRules; +import com.google.cloud.bigtable.admin.v2.models.StorageType; +import com.google.cloud.bigtable.admin.v2.models.Table; +import com.google.cloud.bigtable.admin.v2.models.UpdateTableRequest; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import javax.annotation.Nullable; +import org.apache.beam.it.common.ResourceManager; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.threeten.bp.Duration; + +/** + * Client for managing Bigtable resources. + * + *

    The class supports one instance, and multiple tables per manager object. An instance is + * created when the first table is created if one has not been created already. + * + *

    The instance id is formed using testId. The instance id will be "{testId}-{ISO8601 time, + * microsecond precision}", with additional formatting. Note: If testId is more than 30 characters, + * a new testId will be formed for naming: {first 21 chars of long testId} + “-” + {8 char hash of + * testId}. + * + *

    The class is thread-safe. + */ +public class BigtableResourceManager implements ResourceManager { + + private static final Logger LOG = LoggerFactory.getLogger(BigtableResourceManager.class); + private static final String DEFAULT_CLUSTER_ZONE = "us-central1-b"; + private static final int DEFAULT_CLUSTER_NUM_NODES = 1; + private static final StorageType DEFAULT_CLUSTER_STORAGE_TYPE = StorageType.SSD; + + private final String projectId; + private final String instanceId; + private final BigtableResourceManagerClientFactory bigtableResourceManagerClientFactory; + + // List to store created tables for static RM + private final List createdTables; + // List to store created app profiles for static RM + private final List createdAppProfiles; + // List of tables we enabled CDC for + private final Set cdcEnabledTables; + + private boolean hasInstance; + private List clusters; + + private final boolean usingStaticInstance; + + private BigtableResourceManager(Builder builder) throws IOException { + this(builder, null); + } + + @VisibleForTesting + BigtableResourceManager( + Builder builder, + @Nullable BigtableResourceManagerClientFactory bigtableResourceManagerClientFactory) + throws BigtableResourceManagerException, IOException { + + // Check that the project ID conforms to GCP standards + checkValidProjectId(builder.projectId); + this.projectId = builder.projectId; + this.createdTables = new ArrayList<>(); + this.createdAppProfiles = new ArrayList<>(); + this.cdcEnabledTables = new HashSet<>(); + this.clusters = new ArrayList<>(); + + // Check if RM was configured to use static Bigtable instance. + if (builder.useStaticInstance) { + if (builder.instanceId == null) { + throw new BigtableResourceManagerException( + "This manager was configured to use a static resource, but the instanceId was not properly set."); + } + this.instanceId = builder.instanceId; + this.hasInstance = true; + } else { + if (builder.instanceId != null) { + throw new BigtableResourceManagerException( + "The instanceId property was set in the builder, but the useStaticInstance() method was not called."); + } + // Generate instance id based on given test id. + this.instanceId = generateInstanceId(builder.testId); + this.hasInstance = false; + } + this.usingStaticInstance = builder.useStaticInstance; + + if (bigtableResourceManagerClientFactory != null) { + this.bigtableResourceManagerClientFactory = bigtableResourceManagerClientFactory; + + } else { + // Create the bigtable admin and data client settings builders, and set the necessary id's for + // each. + BigtableInstanceAdminSettings.Builder bigtableInstanceAdminSettings = + BigtableInstanceAdminSettings.newBuilder().setProjectId(builder.projectId); + BigtableTableAdminSettings.Builder bigtableTableAdminSettings = + BigtableTableAdminSettings.newBuilder() + .setProjectId(builder.projectId) + .setInstanceId(this.instanceId); + BigtableDataSettings.Builder bigtableDataSettings = + BigtableDataSettings.newBuilder() + .setProjectId(builder.projectId) + .setInstanceId(this.instanceId); + + // add the credentials to the builders, if set. + if (builder.credentialsProvider != null) { + bigtableInstanceAdminSettings.setCredentialsProvider(builder.credentialsProvider); + bigtableTableAdminSettings.setCredentialsProvider(builder.credentialsProvider); + bigtableDataSettings.setCredentialsProvider(builder.credentialsProvider); + } + + this.bigtableResourceManagerClientFactory = + new BigtableResourceManagerClientFactory( + bigtableInstanceAdminSettings.build(), + bigtableTableAdminSettings.build(), + bigtableDataSettings.build()); + } + } + + public static Builder builder( + String testId, String projectId, CredentialsProvider credentialsProvider) { + return new Builder(testId, projectId, credentialsProvider); + } + + /** + * Returns the project ID this Resource Manager is configured to operate on. + * + * @return the project ID. + */ + public String getProjectId() { + return projectId; + } + + /** + * Return the instance ID this Resource Manager uses to create and manage tables in. + * + * @return the instance ID. + */ + public String getInstanceId() { + return instanceId; + } + + /** + * Creates a Bigtable instance in which all clusters, nodes and tables will exist. + * + * @param clusters List of BigtableResourceManagerCluster objects to associate with the given + * Bigtable instance. + * @throws BigtableResourceManagerException if there is an error creating the instance in + * Bigtable. + */ + public synchronized void createInstance(List clusters) + throws BigtableResourceManagerException { + + // Check to see if instance already exists, and throw error if it does + if (hasInstance) { + LOG.warn( + "Skipping instance creation. Instance was already created or static instance was passed. Reusing : {}.", + instanceId); + return; + } + + LOG.info("Creating instance {} in project {}.", instanceId, projectId); + + // Create instance request object and add all the given clusters to the request + CreateInstanceRequest request = CreateInstanceRequest.of(instanceId); + for (BigtableResourceManagerCluster cluster : clusters) { + request.addCluster( + cluster.clusterId(), cluster.zone(), cluster.numNodes(), cluster.storageType()); + } + + // Send the instance request to Google Cloud + try (BigtableInstanceAdminClient instanceAdminClient = + bigtableResourceManagerClientFactory.bigtableInstanceAdminClient()) { + instanceAdminClient.createInstance(request); + } catch (Exception e) { + throw new BigtableResourceManagerException( + "Failed to create instance " + instanceId + ".", e); + } + hasInstance = true; + this.clusters = clusters; + + LOG.info("Successfully created instance {}.", instanceId); + } + + /** + * Helper method for determining if an instance has been created for the ResourceManager object. + * + * @throws IllegalStateException if an instance has not yet been created. + */ + private void checkHasInstance() { + if (!hasInstance) { + throw new IllegalStateException("There is no instance for manager to perform operation on."); + } + } + + /** + * Helper method for determining if the given tableId exists in the instance. + * + * @param tableId The id of the table to check. + * @throws IllegalStateException if the table does not exist in the instance. + */ + private void checkHasTable(String tableId) { + try (BigtableTableAdminClient tableAdminClient = + bigtableResourceManagerClientFactory.bigtableTableAdminClient()) { + if (!tableAdminClient.exists(tableId)) { + throw new IllegalStateException( + "The table " + tableId + " does not exist in instance " + instanceId + "."); + } + } + } + + /** + * Creates a table within the current instance given a table ID and a collection of column family + * names. + * + *

    Bigtable has the capability to store multiple versions of data in a single cell, which are + * indexed using timestamp values. The timestamp can be set by Bigtable, with the default + * timestamp value being 1970-01-01, or can be set explicitly. The columns in the created table + * will be automatically garbage collected once they reach an age of 1-hour after the set + * timestamp. + * + *

    Note: Implementations may do instance creation here, if one does not already exist. + * + * @param tableId The id of the table. + * @param columnFamilies A collection of column family names for the table. + * @throws BigtableResourceManagerException if there is an error creating the table in Bigtable. + */ + public synchronized void createTable(String tableId, Iterable columnFamilies) + throws BigtableResourceManagerException { + createTable(tableId, columnFamilies, Duration.ofHours(1)); + } + + /** + * Creates a table within the current instance given a table ID and a collection of column family + * names. + * + *

    Bigtable has the capability to store multiple versions of data in a single cell, which are + * indexed using timestamp values. The timestamp can be set by Bigtable, with the default + * timestamp value being 1970-01-01, or can be set explicitly. The columns in the created table + * will be automatically garbage collected once they reach an age of {@code maxAge} after the set + * timestamp. + * + *

    Note: Implementations may do instance creation here, if one does not already exist. + * + * @param tableId The id of the table. + * @param columnFamilies A collection of column family names for the table. + * @param maxAge Sets the maximum age the columns can persist before being garbage collected. + * @throws BigtableResourceManagerException if there is an error creating the table in Bigtable. + */ + public synchronized void createTable( + String tableId, Iterable columnFamilies, Duration maxAge) + throws BigtableResourceManagerException { + BigtableTableSpec spec = new BigtableTableSpec(); + spec.setColumnFamilies(columnFamilies); + spec.setMaxAge(maxAge); + createTable(tableId, spec); + } + + /** + * Creates a table within the current instance given a table ID and a collection of column family + * names. + * + *

    Bigtable has the capability to store multiple versions of data in a single cell, which are + * indexed using timestamp values. The timestamp can be set by Bigtable, with the default + * timestamp value being 1970-01-01, or can be set explicitly. The columns in the created table + * will be automatically garbage collected once they reach an age of {@code maxAge} after the set + * timestamp. + * + *

    Note: Implementations may do instance creation here, if one does not already exist. + * + * @param tableId The id of the table. + * @param bigtableTableSpec Other table configurations + * @throws BigtableResourceManagerException if there is an error creating the table in Bigtable. + */ + public synchronized void createTable(String tableId, BigtableTableSpec bigtableTableSpec) + throws BigtableResourceManagerException { + // Check table ID + checkValidTableId(tableId); + + // Check for at least one column family + if (!bigtableTableSpec.getColumnFamilies().iterator().hasNext()) { + throw new IllegalArgumentException( + "There must be at least one column family specified when creating a table."); + } + + // Create a default instance if this resource manager has not already created one + if (!hasInstance) { + createInstance( + generateDefaultClusters( + instanceId, + DEFAULT_CLUSTER_ZONE, + DEFAULT_CLUSTER_NUM_NODES, + DEFAULT_CLUSTER_STORAGE_TYPE)); + } + checkHasInstance(); + + LOG.info("Creating table using tableId '{}'.", tableId); + + // Fetch the Bigtable Table client and create the table if it does not already exist in the + // instance + try (BigtableTableAdminClient tableAdminClient = + bigtableResourceManagerClientFactory.bigtableTableAdminClient()) { + if (!tableAdminClient.exists(tableId)) { + CreateTableRequest createTableRequest = CreateTableRequest.of(tableId); + for (String columnFamily : bigtableTableSpec.getColumnFamilies()) { + createTableRequest.addFamily( + columnFamily, GCRules.GCRULES.maxAge(bigtableTableSpec.getMaxAge())); + } + if (bigtableTableSpec.getCdcEnabled()) { + createTableRequest.addChangeStreamRetention(Duration.ofDays(7)); + cdcEnabledTables.add(tableId); + } + tableAdminClient.createTable(createTableRequest); + + await("Waiting for all tables to be replicated.") + .atMost(java.time.Duration.ofMinutes(10)) + .pollInterval(java.time.Duration.ofSeconds(5)) + .until( + () -> { + Table t = tableAdminClient.getTable(tableId); + Map rs = t.getReplicationStatesByClusterId(); + return rs.values().stream().allMatch(Table.ReplicationState.READY::equals); + }); + + } else { + throw new IllegalStateException( + "Table " + tableId + " already exists for instance " + instanceId + "."); + } + } catch (Exception e) { + throw new BigtableResourceManagerException("Failed to create table.", e); + } + + if (usingStaticInstance) { + createdTables.add(tableId); + } + + LOG.info("Successfully created table {}.{}", instanceId, tableId); + } + + /** + * Creates an application profile within the current instance + * + *

    Note: Implementations may do instance creation here, if one does not already exist. + * + * @param appProfileId The id of the app profile. + * @param allowTransactionWrites Allows transactional writes when single cluster routing is + * enabled + * @param clusters Clusters where traffic is going to be routed. If more than one cluster is + * specified, a multi-cluster routing is used. A single-cluster routing is used when a single + * cluster is specified. + * @throws BigtableResourceManagerException if there is an error creating the application profile + * in Bigtable. + */ + public synchronized void createAppProfile( + String appProfileId, boolean allowTransactionWrites, List clusters) + throws BigtableResourceManagerException { + checkHasInstance(); + if (clusters == null || clusters.isEmpty()) { + throw new IllegalArgumentException("Cluster list cannot be empty"); + } + + RoutingPolicy routingPolicy; + + if (clusters.size() == 1) { + routingPolicy = SingleClusterRoutingPolicy.of(clusters.get(0), allowTransactionWrites); + } else { + routingPolicy = MultiClusterRoutingPolicy.of(new HashSet<>(clusters)); + } + + LOG.info("Creating appProfile {} for instance project {}.", appProfileId, instanceId); + + try (BigtableInstanceAdminClient instanceAdminClient = + bigtableResourceManagerClientFactory.bigtableInstanceAdminClient()) { + List existingAppProfiles = instanceAdminClient.listAppProfiles(instanceId); + if (!doesAppProfileExist(appProfileId, existingAppProfiles)) { + CreateAppProfileRequest request = + CreateAppProfileRequest.of(instanceId, appProfileId).setRoutingPolicy(routingPolicy); + instanceAdminClient.createAppProfile(request); + LOG.info("Successfully created appProfile {}.", appProfileId); + } else { + throw new IllegalStateException( + "App profile " + appProfileId + " already exists for instance " + instanceId + "."); + } + } catch (Exception e) { + throw new BigtableResourceManagerException("Failed to create app profile.", e); + } + + if (usingStaticInstance) { + createdAppProfiles.add(appProfileId); + } + } + + /** + * Writes a given row into a table. This method requires {@link + * BigtableResourceManager#createTable(String, Iterable)} to be called for the target table + * beforehand. + * + * @param tableRow A mutation object representing the table row. + * @throws BigtableResourceManagerException if method is called after resources have been cleaned + * up, if the manager object has no instance, if the table does not exist or if there is an + * IOException when attempting to retrieve the bigtable data client. + */ + public void write(RowMutation tableRow) throws BigtableResourceManagerException { + write(ImmutableList.of(tableRow)); + } + + /** + * Writes a collection of table rows into one or more tables. This method requires {@link + * BigtableResourceManager#createTable(String, Iterable)} to be called for the target table + * beforehand. + * + * @param tableRows A collection of mutation objects representing table rows. + * @throws BigtableResourceManagerException if method is called after resources have been cleaned + * up, if the manager object has no instance, if the table does not exist or if there is an + * IOException when attempting to retrieve the bigtable data client. + */ + public synchronized void write(Iterable tableRows) + throws BigtableResourceManagerException { + checkHasInstance(); + + // Exit early if there are no mutations + if (!tableRows.iterator().hasNext()) { + return; + } + + LOG.info("Sending {} mutations to instance {}.", Iterables.size(tableRows), instanceId); + + // Fetch the Bigtable data client and send row mutations to the table + try (BigtableDataClient dataClient = + bigtableResourceManagerClientFactory.bigtableDataClient()) { + for (RowMutation tableRow : tableRows) { + dataClient.mutateRow(tableRow); + } + } catch (Exception e) { + throw new BigtableResourceManagerException("Failed to write mutations.", e); + } + + LOG.info("Successfully sent mutations to instance {}.", instanceId); + } + + /** + * Reads all the rows in a table. This method requires {@link + * BigtableResourceManager#createTable(String, Iterable)} to be called for the target table + * beforehand. + * + * @param tableId The id of table to read rows from. + * @return A List object containing all the rows in the table. + * @throws BigtableResourceManagerException if method is called after resources have been cleaned + * up, if the manager object has no instance, if the table does not exist or if there is an + * IOException when attempting to retrieve the bigtable data client. + */ + public synchronized ImmutableList readTable(String tableId) + throws BigtableResourceManagerException { + return readTable(tableId, null); + } + + /** + * Reads all the rows in a table. This method requires {@link + * BigtableResourceManager#createTable(String, Iterable)} to be called for the target table + * beforehand. + * + * @param tableId The id of table to read rows from. + * @param limit Limits the number of rows that can be returned + * @return A List object containing all the rows in the table. + * @throws BigtableResourceManagerException if method is called after resources have been cleaned + * up, if the manager object has no instance, if the table does not exist or if there is an + * IOException when attempting to retrieve the bigtable data client. + */ + public synchronized ImmutableList readTable(String tableId, @Nullable Long limit) + throws BigtableResourceManagerException { + checkHasInstance(); + checkHasTable(tableId); + + // List to store fetched rows + ImmutableList.Builder tableRowsBuilder = ImmutableList.builder(); + + LOG.info("Reading all rows from {}.{}", instanceId, tableId); + + // Fetch the Bigtable data client and read all the rows from the table given by tableId + try (BigtableDataClient dataClient = + bigtableResourceManagerClientFactory.bigtableDataClient()) { + + Query query = Query.create(tableId); + if (limit != null) { + query.limit(limit); + } + ServerStream rowStream = dataClient.readRows(query); + for (Row row : rowStream) { + tableRowsBuilder.add(row); + } + + } catch (Exception e) { + throw new BigtableResourceManagerException("Error occurred while reading table rows.", e); + } + + ImmutableList tableRows = tableRowsBuilder.build(); + LOG.info("Loaded {} rows from {}.{}", tableRows.size(), instanceId, tableId); + + return tableRows; + } + + /** Get all the cluster names of the current instance. */ + public List getClusterNames() { + return StreamSupport.stream(getClusters().spliterator(), false) + .map(BigtableResourceManagerCluster::clusterId) + .collect(Collectors.toList()); + } + + private Iterable getClusters() { + if (usingStaticInstance && this.clusters.isEmpty()) { + try (BigtableInstanceAdminClient instanceAdminClient = + bigtableResourceManagerClientFactory.bigtableInstanceAdminClient()) { + List managedClusters = new ArrayList<>(); + for (Cluster cluster : instanceAdminClient.listClusters(instanceId)) { + managedClusters.add( + BigtableResourceManagerCluster.create( + cluster.getId(), + cluster.getZone(), + cluster.getServeNodes(), + cluster.getStorageType())); + } + this.clusters = managedClusters; + } + } + return this.clusters; + } + + /** + * Deletes all created resources (instance and tables) and cleans up all Bigtable clients, making + * the manager object unusable. + * + *

    If this Resource Manager was configured to use a static instance, the instance will not be + * cleaned up, but any created tables will be deleted. + * + * @throws BigtableResourceManagerException if there is an error deleting the instance or tables + * in Bigtable. + */ + @Override + public synchronized void cleanupAll() throws BigtableResourceManagerException { + LOG.info("Attempting to cleanup manager."); + + try (BigtableTableAdminClient tableAdminClient = + bigtableResourceManagerClientFactory.bigtableTableAdminClient()) { + // Change streams must be disabled before table or instance can be deleted + for (String tableId : cdcEnabledTables) { + tableAdminClient.updateTable(UpdateTableRequest.of(tableId).disableChangeStreamRetention()); + } + + if (usingStaticInstance) { + LOG.info( + "This manager was configured to use a static instance that will not be cleaned up."); + + // Remove managed tables + createdTables.forEach(tableAdminClient::deleteTable); + + // Remove managed app profiles + try (BigtableInstanceAdminClient instanceAdminClient = + bigtableResourceManagerClientFactory.bigtableInstanceAdminClient()) { + createdAppProfiles.forEach( + profile -> instanceAdminClient.deleteAppProfile(instanceId, profile, true)); + } + return; + } + } + + if (hasInstance) { + try (BigtableInstanceAdminClient instanceAdminClient = + bigtableResourceManagerClientFactory.bigtableInstanceAdminClient()) { + instanceAdminClient.deleteInstance(instanceId); + hasInstance = false; + } catch (Exception e) { + throw new BigtableResourceManagerException("Failed to delete resources.", e); + } + } + LOG.info("Manager successfully cleaned up."); + } + + private boolean doesAppProfileExist(String appProfileId, List existingAppProfiles) { + for (AppProfile existingProfile : existingAppProfiles) { + if (StringUtils.equals(appProfileId, existingProfile.getId())) { + return true; + } + } + return false; + } + + /** Builder for {@link BigtableResourceManager}. */ + public static final class Builder { + + private final String testId; + private final String projectId; + private @Nullable String instanceId; + private boolean useStaticInstance; + private CredentialsProvider credentialsProvider; + + private Builder(String testId, String projectId, CredentialsProvider credentialsProvider) { + this.testId = testId; + this.projectId = projectId; + this.credentialsProvider = credentialsProvider; + this.instanceId = null; + } + + /** + * Set the GCP credentials provider to connect to the project defined in the builder. + * + * @param credentialsProvider The GCP CredentialsProvider. + * @return this builder with the CredentialsProvider set. + */ + public Builder setCredentialsProvider(CredentialsProvider credentialsProvider) { + this.credentialsProvider = credentialsProvider; + return this; + } + + /** + * Set the instance ID of a static Bigtable instance for this Resource Manager to manage. + * + * @return this builder with the instance ID set. + */ + public Builder setInstanceId(String instanceId) { + this.instanceId = instanceId; + return this; + } + + /** + * Configures the resource manager to use a static GCP resource instead of creating a new + * instance of the resource. + * + * @return this builder object with the useStaticInstance option enabled. + */ + public Builder useStaticInstance() { + this.useStaticInstance = true; + return this; + } + + /** + * Looks at the system properties if there's an instance id, and reuses it if configured. + * + * @return this builder object with the useStaticInstance option enabled and instance set if + * configured, the same builder otherwise. + */ + public Builder maybeUseStaticInstance() { + if (System.getProperty("bigtableInstanceId") != null) { + this.useStaticInstance = true; + this.instanceId = System.getProperty("bigtableInstanceId"); + } + return this; + } + + public BigtableResourceManager build() throws IOException { + return new BigtableResourceManager(this); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerClientFactory.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerClientFactory.java new file mode 100644 index 0000000000000..d5e0b24c19887 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerClientFactory.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigtable; + +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminSettings; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import java.io.IOException; + +/** + * Class for storing the 3 Bigtable clients used to manage the Instance(s), Table(s) and Data + * mutations performed by the resource manager. + * + *

    This class is intended to simplify mock injections of Bigtable clients during unit testing + */ +public class BigtableResourceManagerClientFactory { + + private final BigtableInstanceAdminSettings bigtableInstanceAdminSettings; + private final BigtableTableAdminSettings bigtableTableAdminSettings; + private final BigtableDataSettings bigtableDataSettings; + + public BigtableResourceManagerClientFactory( + BigtableInstanceAdminSettings instanceSettings, + BigtableTableAdminSettings tableSettings, + BigtableDataSettings dataSettings) { + + this.bigtableInstanceAdminSettings = instanceSettings; + this.bigtableTableAdminSettings = tableSettings; + this.bigtableDataSettings = dataSettings; + } + + /** + * Returns the Instance admin client for managing Bigtable instances. + * + * @return the instance admin client + */ + public BigtableInstanceAdminClient bigtableInstanceAdminClient() { + try { + return BigtableInstanceAdminClient.create(bigtableInstanceAdminSettings); + } catch (IOException e) { + throw new BigtableResourceManagerException( + "Failed to create bigtable instance admin client.", e); + } + } + + /** + * Returns the Table admin client for managing Bigtable tables. + * + * @return the table admin client + */ + public BigtableTableAdminClient bigtableTableAdminClient() { + try { + return BigtableTableAdminClient.create(bigtableTableAdminSettings); + } catch (IOException e) { + throw new BigtableResourceManagerException( + "Failed to create bigtable table admin client.", e); + } + } + + /** + * Returns the Data client for reading and writing data to Bigtable tables. + * + * @return the data client + */ + public BigtableDataClient bigtableDataClient() { + try { + return BigtableDataClient.create(bigtableDataSettings); + } catch (IOException e) { + throw new BigtableResourceManagerException("Failed to create bigtable data client.", e); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerCluster.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerCluster.java new file mode 100644 index 0000000000000..e260285bbce13 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerCluster.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigtable; + +import com.google.auto.value.AutoValue; +import com.google.cloud.bigtable.admin.v2.models.StorageType; + +/** + * Class for storing the metadata of a Bigtable cluster object. + * + *

    A cluster belongs to a single Bigtable instance and represents the service in a given zone. A + * cluster can have multiple nodes operating on the data. The cluster also has a storage type of + * either SSD or HDD depending on the user's needs. + */ +@AutoValue +public abstract class BigtableResourceManagerCluster { + + public static BigtableResourceManagerCluster create( + String clusterId, String zone, int numNodes, StorageType storageType) { + return new AutoValue_BigtableResourceManagerCluster(clusterId, zone, numNodes, storageType); + } + + /** + * Returns the cluster ID of the Bigtable cluster object. + * + * @return the ID of the Bigtable cluster. + */ + public abstract String clusterId(); + + /** + * Returns the operating zone of the Bigtable cluster object. + * + * @return the zone of the Bigtable cluster. + */ + public abstract String zone(); + + /** + * Returns the number of nodes the Bigtable cluster object should be configured with. + * + * @return the number of nodes for the Bigtable cluster. + */ + public abstract int numNodes(); + + /** + * Returns the type of storage the Bigtable cluster object should be configured with (SSD or HDD). + * + * @return the storage type of the Bigtable cluster. + */ + public abstract StorageType storageType(); +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerException.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerException.java new file mode 100644 index 0000000000000..7a20591cac04b --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigtable; + +/** Custom exception for {@link BigtableResourceManager} implementations. */ +public class BigtableResourceManagerException extends RuntimeException { + + public BigtableResourceManagerException(String errorMessage) { + super(errorMessage); + } + + public BigtableResourceManagerException(String errorMessage, Throwable err) { + super(errorMessage, err); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerUtils.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerUtils.java new file mode 100644 index 0000000000000..28f1f5bf60c51 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerUtils.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigtable; + +import static org.apache.beam.it.common.utils.ResourceManagerUtils.generateResourceId; + +import com.google.cloud.bigtable.admin.v2.models.StorageType; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.regex.Pattern; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; + +/** Utilities for {@link BigtableResourceManager} implementations. */ +public final class BigtableResourceManagerUtils { + + private static final int MAX_CLUSTER_ID_LENGTH = 30; + private static final Pattern ILLEGAL_CLUSTER_CHARS = Pattern.compile("[^a-z0-9-]"); + private static final String REPLACE_CLUSTER_CHAR = "-"; + public static final int MAX_INSTANCE_ID_LENGTH = 30; + private static final Pattern ILLEGAL_INSTANCE_ID_CHARS = Pattern.compile("[^a-z0-9-]"); + private static final String REPLACE_INSTANCE_ID_CHAR = "-"; + private static final int MIN_TABLE_ID_LENGTH = 1; + private static final int MAX_TABLE_ID_LENGTH = 40; + private static final Pattern ILLEGAL_TABLE_CHARS = Pattern.compile("[^a-zA-Z0-9-_.]"); + private static final String REPLACE_TABLE_ID_CHAR = "-"; + + private static final DateTimeFormatter TIME_FORMAT = + DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss-SSSSSS"); + + private BigtableResourceManagerUtils() {} + + /** + * Generates a collection that contains a single BigtableResourceManagerCluster object using the + * given parameters. + * + * @param baseString the test id to associate the bigtable cluster to. + * @param zone the zone/region that the cluster will be deployed to. + * @param numNodes the number of nodes that the cluster will contain. + * @param storageType the type of storage to configure the cluster with (SSD or HDD). + * @return List containing a single BigtableResourceManagerCluster object. + */ + static List generateDefaultClusters( + String baseString, String zone, int numNodes, StorageType storageType) { + + String clusterId = + generateResourceId( + baseString.toLowerCase(), + ILLEGAL_CLUSTER_CHARS, + REPLACE_CLUSTER_CHAR, + MAX_CLUSTER_ID_LENGTH, + TIME_FORMAT); + BigtableResourceManagerCluster cluster = + BigtableResourceManagerCluster.create(clusterId, zone, numNodes, storageType); + + return ImmutableList.of(cluster); + } + + /** + * Generates an instance id from a given string. + * + * @param baseString The string to generate the id from. + * @return The instance id string. + */ + static String generateInstanceId(String baseString) { + return generateResourceId( + baseString.toLowerCase(), + ILLEGAL_INSTANCE_ID_CHARS, + REPLACE_INSTANCE_ID_CHAR, + MAX_INSTANCE_ID_LENGTH, + TIME_FORMAT); + } + + /** + * Generates a table id from a given string. + * + * @param baseString The string to generate the id from. + * @return The instance id string. + */ + public static String generateTableId(String baseString) { + return generateResourceId( + baseString.toLowerCase(), + ILLEGAL_TABLE_CHARS, + REPLACE_TABLE_ID_CHAR, + MAX_TABLE_ID_LENGTH, + TIME_FORMAT); + } + + /** + * Checks whether the given table ID is valid according to GCP constraints. + * + * @param idToCheck the table ID to check. + * @throws IllegalArgumentException if the table ID is invalid. + */ + static void checkValidTableId(String idToCheck) { + if (idToCheck.length() < MIN_TABLE_ID_LENGTH) { + throw new IllegalArgumentException("Table ID " + idToCheck + " cannot be empty."); + } + if (idToCheck.length() > MAX_TABLE_ID_LENGTH) { + throw new IllegalArgumentException( + "Table ID " + + idToCheck + + " cannot be longer than " + + MAX_TABLE_ID_LENGTH + + " characters."); + } + if (ILLEGAL_TABLE_CHARS.matcher(idToCheck).find()) { + throw new IllegalArgumentException( + "Table ID " + + idToCheck + + " is not a valid ID. Only letters, numbers, hyphens, underscores and exclamation points are allowed."); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/BigtableTableSpec.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/BigtableTableSpec.java new file mode 100644 index 0000000000000..1dd7cb0ef2fbc --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/BigtableTableSpec.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigtable; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import org.threeten.bp.Duration; + +public class BigtableTableSpec { + private Set columnFamilies; + private Duration maxAge; + private boolean cdcEnabled; + + public BigtableTableSpec() { + this.maxAge = Duration.ofHours(1); + this.columnFamilies = new HashSet<>(); + this.cdcEnabled = false; + } + + public Set getColumnFamilies() { + return Collections.unmodifiableSet(columnFamilies); + } + + public void setColumnFamilies(Set columnFamilies) { + this.columnFamilies = new HashSet<>(columnFamilies); + } + + public void setColumnFamilies(Iterable columnFamilies) { + this.columnFamilies = new HashSet<>(); + columnFamilies.forEach(this.columnFamilies::add); + } + + public Duration getMaxAge() { + return maxAge; + } + + public void setMaxAge(Duration maxAge) { + this.maxAge = maxAge; + } + + public boolean getCdcEnabled() { + return cdcEnabled; + } + + public void setCdcEnabled(boolean cdcEnabled) { + this.cdcEnabled = cdcEnabled; + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/matchers/BigtableAsserts.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/matchers/BigtableAsserts.java new file mode 100644 index 0000000000000..cb2d44ebeee7b --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/matchers/BigtableAsserts.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigtable.matchers; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatRecords; + +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.truthmatchers.RecordsSubject; + +public class BigtableAsserts { + + /** + * Convert Bigtable {@link Row} to a list of maps. + * + * @param rows Bigtable rows to parse. + * @param family Bigtable column family to parse from. + * @return List of maps to use in {@link RecordsSubject} + */ + public static List> bigtableRowsToRecords(Iterable rows, String family) { + try { + List> records = new ArrayList<>(); + + for (Row row : rows) { + Map converted = new HashMap<>(); + for (RowCell cell : row.getCells(family)) { + + String col = cell.getQualifier().toStringUtf8(); + String val = cell.getValue().toStringUtf8(); + converted.put(col, val); + } + records.add(converted); + } + + return records; + } catch (Exception e) { + throw new RuntimeException("Error converting Bigtable Row to Map", e); + } + } + + /** + * Creates a {@link RecordsSubject} to assert information within a list of records. + * + * @param rows Records in Bigtable's {@link Row} format to use in the comparison. + * @param family The column family to read records from. + * @return Truth subject to chain assertions. + */ + public static RecordsSubject assertThatBigtableRecords(Iterable rows, String family) { + return assertThatRecords(bigtableRowsToRecords(rows, family)); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/matchers/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/matchers/package-info.java new file mode 100644 index 0000000000000..cdecf1426ef6b --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/matchers/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for Bigtable Truth matchers / subjects to have reusable assertions. */ +package org.apache.beam.it.gcp.bigtable.matchers; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/package-info.java new file mode 100644 index 0000000000000..1b23d21755d3e --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigtable/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing Bigtable resources within integration tests. */ +package org.apache.beam.it.gcp.bigtable; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/AbstractPipelineLauncher.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/AbstractPipelineLauncher.java new file mode 100644 index 0000000000000..b5c9535953b78 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/AbstractPipelineLauncher.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.dataflow; + +import static org.apache.beam.it.common.PipelineLauncher.JobState.ACTIVE_STATES; +import static org.apache.beam.it.common.PipelineLauncher.JobState.FAILED; +import static org.apache.beam.it.common.PipelineLauncher.JobState.PENDING_STATES; +import static org.apache.beam.it.common.logging.LogStrings.formatForLogging; +import static org.apache.beam.it.common.utils.RetryUtil.clientRetryPolicy; + +import com.google.api.client.util.ArrayMap; +import com.google.api.services.dataflow.Dataflow; +import com.google.api.services.dataflow.model.Environment; +import com.google.api.services.dataflow.model.Job; +import com.google.api.services.dataflow.model.JobMessage; +import com.google.api.services.dataflow.model.ListJobMessagesResponse; +import com.google.api.services.dataflow.model.MetricUpdate; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import dev.failsafe.Failsafe; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.Nullable; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.TestProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Abstract class covering the common operations between Classic and Flex templates. + * + *

    Generally, the methods here are the ones that focus more on the Dataflow jobs rather than + * launching a specific type of template. + */ +public abstract class AbstractPipelineLauncher implements PipelineLauncher { + + private static final Logger LOG = LoggerFactory.getLogger(AbstractPipelineLauncher.class); + private static final Pattern CURRENT_METRICS = Pattern.compile(".*Current.*"); + public static final String LEGACY_RUNNER = "Dataflow Legacy Runner"; + public static final String RUNNER_V2 = "Dataflow Runner V2"; + public static final String PARAM_RUNNER = "runner"; + public static final String PARAM_JOB_TYPE = "jobType"; + public static final String PARAM_JOB_ID = "jobId"; + + protected final List launchedJobs = new ArrayList<>(); + + protected final Dataflow client; + + protected AbstractPipelineLauncher(Dataflow client) { + this.client = client; + } + + @Override + public Job getJob(String project, String region, String jobId) throws IOException { + return getJob(project, region, jobId, "JOB_VIEW_UNKNOWN"); + } + + @Override + public Job getJob(String project, String region, String jobId, String view) { + return Failsafe.with(clientRetryPolicy()) + .get( + () -> + client + .projects() + .locations() + .jobs() + .get(project, region, jobId) + .setView(view) + .execute()); + } + + @Override + public JobState getJobStatus(String project, String region, String jobId) throws IOException { + return handleJobState(getJob(project, region, jobId)); + } + + @Override + public List listMessages( + String project, String region, String jobId, String minimumImportance) { + LOG.info("Listing messages of {} under {}", jobId, project); + ListJobMessagesResponse response = + Failsafe.with(clientRetryPolicy()) + .get( + () -> + client + .projects() + .locations() + .jobs() + .messages() + .list(project, region, jobId) + .setMinimumImportance(minimumImportance) + .execute()); + List messages = response.getJobMessages(); + LOG.info("Received {} messages for {} under {}", messages.size(), jobId, project); + return messages; + } + + @Override + public Job cancelJob(String project, String region, String jobId) { + LOG.info("Cancelling {} under {}", jobId, project); + Job job = new Job().setRequestedState(JobState.CANCELLED.toString()); + LOG.info("Sending job to update {}:\n{}", jobId, formatForLogging(job)); + return Failsafe.with(clientRetryPolicy()) + .get( + () -> + client.projects().locations().jobs().update(project, region, jobId, job).execute()); + } + + @Override + public Job drainJob(String project, String region, String jobId) { + LOG.info("Draining {} under {}", jobId, project); + Job job = new Job().setRequestedState(JobState.DRAINED.toString()); + LOG.info("Sending job to update {}:\n{}", jobId, formatForLogging(job)); + return Failsafe.with(clientRetryPolicy()) + .get( + () -> + client.projects().locations().jobs().update(project, region, jobId, job).execute()); + } + + @Override + public @Nullable Double getMetric(String project, String region, String jobId, String metricName) + throws IOException { + LOG.info("Getting '{}' metric for {} under {}", metricName, jobId, project); + List metrics = + client + .projects() + .locations() + .jobs() + .getMetrics(project, region, jobId) + .execute() + .getMetrics(); + if (metrics == null) { + LOG.warn("No metrics received for the job {} under {}.", jobId, project); + return null; + } + for (MetricUpdate metricUpdate : metrics) { + String currentMetricName = metricUpdate.getName().getName(); + String currentMetricOriginalName = metricUpdate.getName().getContext().get("original_name"); + if (Objects.equals(metricName, currentMetricName) + || Objects.equals(metricName, currentMetricOriginalName)) { + // only return if the metric is a scalar + if (metricUpdate.getScalar() != null) { + return ((Number) metricUpdate.getScalar()).doubleValue(); + } else { + LOG.warn( + "The given metric '{}' is not a scalar metric. Please use getMetrics instead.", + metricName); + return null; + } + } + } + LOG.warn( + "Unable to find '{}' metric for {} under {}. Please check the metricName and try again!", + metricName, + jobId, + project); + return null; + } + + @Override + public Map getMetrics(String project, String region, String jobId) + throws IOException { + LOG.info("Getting metrics for {} under {}", jobId, project); + List metrics = + client + .projects() + .locations() + .jobs() + .getMetrics(project, region, jobId) + .execute() + .getMetrics(); + Map result = new HashMap<>(); + for (MetricUpdate metricUpdate : metrics) { + String metricName = metricUpdate.getName().getName(); + Matcher matcher = CURRENT_METRICS.matcher(metricName); + // Since we query metrics after the job finishes, we can ignore tentative and step metrics + if (metricUpdate.getName().getContext().containsKey("tentative") + || metricUpdate.getName().getContext().containsKey("execution_step") + || metricUpdate.getName().getContext().containsKey("step") + || metricName.equals("MeanByteCount") + || metricName.equals("ElementCount") + || matcher.find()) { + continue; + } + + if (result.containsKey(metricName)) { + LOG.warn("Key {} already exists in metrics. Something might be wrong.", metricName); + } + + if (metricUpdate.getScalar() != null) { + result.put(metricName, ((Number) metricUpdate.getScalar()).doubleValue()); + } else if (metricUpdate.getDistribution() != null) { + // currently, reporting distribution metrics as 4 separate scalar metrics + @SuppressWarnings("rawtypes") + ArrayMap distributionMap = (ArrayMap) metricUpdate.getDistribution(); + result.put(metricName + "_COUNT", ((Number) distributionMap.get("count")).doubleValue()); + result.put(metricName + "_MIN", ((Number) distributionMap.get("min")).doubleValue()); + result.put(metricName + "_MAX", ((Number) distributionMap.get("max")).doubleValue()); + result.put(metricName + "_SUM", ((Number) distributionMap.get("sum")).doubleValue()); + } else if (metricUpdate.getGauge() != null) { + LOG.warn("Gauge metric {} cannot be handled.", metricName); + // not sure how to handle gauge metrics + } + } + return result; + } + + protected void printJobResponse(Job job) { + LOG.info("Received job response: {}", formatForLogging(job)); + + LOG.info( + "Dataflow Console: https://console.cloud.google.com/dataflow/jobs/{}/{}?project={}", + job.getLocation(), + job.getId(), + job.getProjectId()); + } + + /** Parses the job state if available or returns {@link JobState#UNKNOWN} if not given. */ + protected JobState handleJobState(Job job) { + String currentState = job.getCurrentState(); + return Strings.isNullOrEmpty(currentState) ? JobState.UNKNOWN : JobState.parse(currentState); + } + + /** + * Creates a JobInfo builder object from the provided parameters, enable derived class to add info + * incrementally. + */ + protected LaunchInfo.Builder getJobInfoBuilder(LaunchConfig options, JobState state, Job job) { + Map labels = job.getLabels(); + String runner = LEGACY_RUNNER; + Environment environment = job.getEnvironment(); + if (environment != null + && environment.getExperiments() != null + && environment.getExperiments().contains("use_runner_v2")) { + runner = RUNNER_V2; + } + LaunchInfo.Builder builder = + LaunchInfo.builder() + .setProjectId(job.getProjectId()) + .setJobId(job.getId()) + .setRegion(job.getLocation()) + .setCreateTime(job.getCreateTime()) + .setSdk(job.getJobMetadata().getSdkVersion().getVersionDisplayName()) + .setVersion(job.getJobMetadata().getSdkVersion().getVersion()) + .setJobType(job.getType()) + .setRunner(runner) + .setState(state); + // add all environment params to parameters in LaunchInfo so that these are exported for load + // tests + Map parameters = new HashMap<>(options.parameters()); + options.environment().forEach((key, val) -> parameters.put(key, val.toString())); + // attach basic job info to parameters so that these are exported for load tests + parameters.put(PARAM_RUNNER, runner); + parameters.put(PARAM_JOB_TYPE, job.getType()); + parameters.put(PARAM_JOB_ID, job.getId()); + builder.setParameters(ImmutableMap.copyOf(parameters)); + if (labels != null && !labels.isEmpty()) { + // template job + builder + .setTemplateType(job.getLabels().get("goog-dataflow-provided-template-type")) + .setTemplateVersion(job.getLabels().get("goog-dataflow-provided-template-version")) + .setTemplateName(job.getLabels().get("goog-dataflow-provided-template-name")); + } + return builder; + } + + /** Creates a JobInfo object from the provided parameters. */ + protected final LaunchInfo getJobInfo(LaunchConfig options, JobState state, Job job) { + return getJobInfoBuilder(options, state, job).build(); + } + + /** Waits until the specified job is not in a pending state. */ + public JobState waitUntilActive(String project, String region, String jobId) throws IOException { + JobState state = getJobStatus(project, region, jobId); + while (PENDING_STATES.contains(state)) { + LOG.info("Job still pending. Will check again in 15 seconds"); + try { + TimeUnit.SECONDS.sleep(15); + } catch (InterruptedException e) { + LOG.warn("Wait interrupted. Checking now."); + } + state = getJobStatus(project, region, jobId); + } + if (state == FAILED) { + throw new RuntimeException( + String.format( + "The job failed before launch! For more " + + "information please check if the job log for Job ID: %s, under project %s.", + jobId, project)); + } + return state; + } + + @Override + public synchronized void cleanupAll() throws IOException { + LOG.info("Cleaning up Dataflow jobs..."); + for (String jobId : launchedJobs) { + try { + JobState state = getJobStatus(TestProperties.project(), TestProperties.region(), jobId); + if (ACTIVE_STATES.contains(state) || PENDING_STATES.contains(state)) { + cancelJob(TestProperties.project(), TestProperties.region(), jobId); + } + } catch (Exception e) { + LOG.warn("Unable to cancel {}. Encountered error.", jobId, e); + } + } + LOG.info("Dataflow jobs successfully cleaned up."); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/ClassicTemplateClient.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/ClassicTemplateClient.java new file mode 100644 index 0000000000000..00aa4003817f9 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/ClassicTemplateClient.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.dataflow; + +import static org.apache.beam.it.common.logging.LogStrings.formatForLogging; +import static org.apache.beam.it.common.utils.RetryUtil.clientRetryPolicy; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; + +import com.google.api.client.googleapis.util.Utils; +import com.google.api.services.dataflow.Dataflow; +import com.google.api.services.dataflow.model.CreateJobFromTemplateRequest; +import com.google.api.services.dataflow.model.Job; +import com.google.api.services.dataflow.model.RuntimeEnvironment; +import com.google.auth.Credentials; +import com.google.auth.http.HttpCredentialsAdapter; +import dev.failsafe.Failsafe; +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Client for interacting with Dataflow classic templates using the Dataflow SDK. */ +public final class ClassicTemplateClient extends AbstractPipelineLauncher { + + private static final Logger LOG = LoggerFactory.getLogger(ClassicTemplateClient.class); + + private ClassicTemplateClient(Builder builder) { + super( + new Dataflow( + Utils.getDefaultTransport(), + Utils.getDefaultJsonFactory(), + new HttpCredentialsAdapter(builder.getCredentials()))); + } + + private ClassicTemplateClient(Dataflow dataflow) { + super(dataflow); + } + + public static ClassicTemplateClient withDataflowClient(Dataflow dataflow) { + return new ClassicTemplateClient(dataflow); + } + + public static Builder builder(Credentials credentials) { + return new Builder(credentials); + } + + @Override + public LaunchInfo launch(String project, String region, LaunchConfig options) throws IOException { + checkState( + options.specPath() != null, + "Cannot launch a template job without specPath. Please specify specPath and try again!"); + LOG.info("Getting ready to launch {} in {} under {}", options.jobName(), region, project); + LOG.info("Using the spec at {}", options.specPath()); + LOG.info("Using parameters:\n{}", formatForLogging(options.parameters())); + + @SuppressWarnings("nullness") + CreateJobFromTemplateRequest parameter = + new CreateJobFromTemplateRequest() + .setJobName(options.jobName()) + .setParameters(options.parameters()) + .setLocation(region) + .setGcsPath(options.specPath()) + .setEnvironment(buildEnvironment(options)); + LOG.info("Sending request:\n{}", formatForLogging(parameter)); + + Job job = + Failsafe.with(clientRetryPolicy()) + .get( + () -> + client + .projects() + .locations() + .templates() + .create(project, region, parameter) + .execute()); + printJobResponse(job); + + // Wait until the job is active to get more information + JobState state = waitUntilActive(project, region, job.getId()); + job = getJob(project, region, job.getId(), "JOB_VIEW_DESCRIPTION"); + LOG.info("Received classic template job {}: {}", job.getId(), formatForLogging(job)); + + launchedJobs.add(job.getId()); + return getJobInfo(options, state, job); + } + + private RuntimeEnvironment buildEnvironment(LaunchConfig options) { + RuntimeEnvironment environment = new RuntimeEnvironment(); + environment.putAll(options.environment()); + return environment; + } + + /** Builder for {@link ClassicTemplateClient}. */ + public static final class Builder { + + private Credentials credentials; + + private Builder(Credentials credentials) { + this.credentials = credentials; + } + + public Credentials getCredentials() { + return credentials; + } + + public Builder setCredentials(Credentials value) { + credentials = value; + return this; + } + + public ClassicTemplateClient build() { + return new ClassicTemplateClient(this); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/DefaultPipelineLauncher.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/DefaultPipelineLauncher.java new file mode 100644 index 0000000000000..3d43618821aa5 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/DefaultPipelineLauncher.java @@ -0,0 +1,491 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.dataflow; + +import static org.apache.beam.it.common.logging.LogStrings.formatForLogging; +import static org.apache.beam.sdk.testing.TestPipeline.PROPERTY_BEAM_TEST_PIPELINE_OPTIONS; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.api.client.googleapis.util.Utils; +import com.google.api.services.dataflow.Dataflow; +import com.google.api.services.dataflow.model.Job; +import com.google.auth.Credentials; +import com.google.auth.http.HttpCredentialsAdapter; +import com.google.protobuf.util.Timestamps; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.StreamSupport; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.utils.PipelineUtils; +import org.apache.beam.it.gcp.IOLoadTestBase; +import org.apache.beam.runners.dataflow.DataflowPipelineJob; +import org.apache.beam.sdk.PipelineResult; +import org.apache.beam.sdk.metrics.DistributionResult; +import org.apache.beam.sdk.metrics.MetricNameFilter; +import org.apache.beam.sdk.metrics.MetricQueryResults; +import org.apache.beam.sdk.metrics.MetricResult; +import org.apache.beam.sdk.metrics.MetricsFilter; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.options.StreamingOptions; +import org.apache.beam.sdk.util.common.ReflectHelpers; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Default class for implementation of {@link PipelineLauncher} interface. */ +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/27438) +}) +public class DefaultPipelineLauncher extends AbstractPipelineLauncher { + private static final Logger LOG = LoggerFactory.getLogger(DefaultPipelineLauncher.class); + private static final String READ_PIPELINE_NAME_OVERWRITE = "readPipelineNameOverride"; + private static final String WRITE_PIPELINE_NAME_OVERWRITE = "writePipelineNameOverride"; + private static final Pattern JOB_ID_PATTERN = Pattern.compile("Submitted job: (\\S+)"); + + // For unsupported runners (other than dataflow), implement launcher methods by operating with + // PipelineResult. + private static final Map MANAGED_JOBS = new HashMap<>(); + + // For supported runners (e.g. DataflowRunner), still keep a PipelineResult for pipeline specific + // usages, e.g., + // polling custom metrics + private static final Map UNMANAGED_JOBS = new HashMap<>(); + + private static final long UNKNOWN_METRIC_VALUE = -1L; + + private static final ObjectMapper MAPPER = + new ObjectMapper() + .registerModules(ObjectMapper.findModules(ReflectHelpers.findClassLoader())); + + private static final Map PIPELINE_STATE_TRANSLATE = + ImmutableMap.builder() + .put(PipelineResult.State.CANCELLED, JobState.CANCELLED) + .put(PipelineResult.State.RUNNING, JobState.RUNNING) + .put(PipelineResult.State.DONE, JobState.DONE) + .put(PipelineResult.State.FAILED, JobState.FAILED) + .put(PipelineResult.State.STOPPED, JobState.STOPPED) + .put(PipelineResult.State.UNKNOWN, JobState.UNKNOWN) + .put(PipelineResult.State.UPDATED, JobState.UPDATED) + .put(PipelineResult.State.UNRECOGNIZED, JobState.UNKNOWN) + .build(); + + private DefaultPipelineLauncher(DefaultPipelineLauncher.Builder builder) { + super( + new Dataflow( + Utils.getDefaultTransport(), + Utils.getDefaultJsonFactory(), + builder.getCredentials() == null + ? null + : new HttpCredentialsAdapter(builder.getCredentials()))); + } + + public static DefaultPipelineLauncher.Builder builder(Credentials credentials) { + return new DefaultPipelineLauncher.Builder(credentials); + } + + @Override + public JobState getJobStatus(String project, String region, String jobId) throws IOException { + if (MANAGED_JOBS.containsKey(jobId)) { + return PIPELINE_STATE_TRANSLATE.get(MANAGED_JOBS.get(jobId).getState()); + } else { + return super.handleJobState(getJob(project, region, jobId)); + } + } + + @Override + public Job cancelJob(String project, String region, String jobId) { + if (MANAGED_JOBS.containsKey(jobId)) { + try { + MANAGED_JOBS.get(jobId).cancel(); + } catch (IOException e) { + throw new RuntimeException(e); + } + return new Job().setId(jobId).setRequestedState(JobState.CANCELLED.toString()); + } else { + return super.cancelJob(project, region, jobId); + } + } + + @Override + public Job getJob(String project, String region, String jobId) throws IOException { + if (MANAGED_JOBS.containsKey(jobId)) { + return new Job() + .setId(jobId) + .setRequestedState( + PIPELINE_STATE_TRANSLATE.get(MANAGED_JOBS.get(jobId).getState()).toString()); + } else { + return super.getJob(project, region, jobId); + } + } + + @Override + public Job drainJob(String project, String region, String jobId) { + if (MANAGED_JOBS.containsKey(jobId)) { + // drain unsupported. Just cancel. + Job job = new Job().setId(jobId).setRequestedState(JobState.DRAINED.toString()); + cancelJob(project, region, jobId); + return job; + } else { + return super.drainJob(project, region, jobId); + } + } + + private static void checkIfMetricResultIsUnique( + String name, Iterable> metricResult) throws IllegalStateException { + int resultCount = Iterables.size(metricResult); + Preconditions.checkState( + resultCount <= 1, + "More than one metric result matches name: %s in namespace %s. Metric results count: %s", + name, + IOLoadTestBase.BEAM_METRICS_NAMESPACE, + resultCount); + } + + private static Iterable> getDistributions( + PipelineResult result, String metricName) { + MetricQueryResults metrics = + result + .metrics() + .queryMetrics( + MetricsFilter.builder() + .addNameFilter( + MetricNameFilter.named(IOLoadTestBase.BEAM_METRICS_NAMESPACE, metricName)) + .build()); + return metrics.getDistributions(); + } + + /** Pull Beam pipeline defined metrics given the jobId. */ + public Long getBeamMetric( + String jobId, IOLoadTestBase.PipelineMetricsType metricType, String metricName) { + PipelineResult pipelineResult = + MANAGED_JOBS.getOrDefault(jobId, UNMANAGED_JOBS.getOrDefault(jobId, null)); + if (pipelineResult != null) { + MetricQueryResults metrics = + pipelineResult + .metrics() + .queryMetrics( + MetricsFilter.builder() + .addNameFilter( + MetricNameFilter.named(IOLoadTestBase.BEAM_METRICS_NAMESPACE, metricName)) + .build()); + + switch (metricType) { + case COUNTER: + Iterable> counters = metrics.getCounters(); + checkIfMetricResultIsUnique(metricName, counters); + try { + MetricResult metricResult = counters.iterator().next(); + return metricResult.getAttempted(); + } catch (NoSuchElementException e) { + LOG.error( + "Failed to get metric {}, from namespace {}", + metricName, + IOLoadTestBase.BEAM_METRICS_NAMESPACE); + } + return UNKNOWN_METRIC_VALUE; + case STARTTIME: + case ENDTIME: + case RUNTIME: + Iterable> distributions = + getDistributions(pipelineResult, metricName); + Long lowestMin = + StreamSupport.stream(distributions.spliterator(), true) + .map(element -> Objects.requireNonNull(element.getAttempted()).getMin()) + .min(Long::compareTo) + .orElse(UNKNOWN_METRIC_VALUE); + Long greatestMax = + StreamSupport.stream(distributions.spliterator(), true) + .map(element -> Objects.requireNonNull(element.getAttempted()).getMax()) + .max(Long::compareTo) + .orElse(UNKNOWN_METRIC_VALUE); + if (metricType == IOLoadTestBase.PipelineMetricsType.STARTTIME) { + return lowestMin; + } else if (metricType == IOLoadTestBase.PipelineMetricsType.ENDTIME) { + return greatestMax; + } else { + if (lowestMin != UNKNOWN_METRIC_VALUE && greatestMax != UNKNOWN_METRIC_VALUE) { + return greatestMax - lowestMin; + } else { + return UNKNOWN_METRIC_VALUE; + } + } + default: + throw new IllegalArgumentException( + String.format("Unexpected metric type %s.", metricType)); + } + } else { + LOG.warn("Query pipeline defined metrics this SDK or runner is currently unsupported."); + return UNKNOWN_METRIC_VALUE; + } + } + + @Override + public Double getMetric(String project, String region, String jobId, String metricName) + throws IOException { + if (metricName.startsWith(IOLoadTestBase.BEAM_METRICS_NAMESPACE)) { + String[] nameSpacedMetrics = metricName.split(":", 3); + Preconditions.checkState( + nameSpacedMetrics.length == 3, + String.format( + "Invalid Beam metrics name: %s, expected: '%s:metric_type:metric_name'", + metricName, IOLoadTestBase.BEAM_METRICS_NAMESPACE)); + IOLoadTestBase.PipelineMetricsType metricType = + IOLoadTestBase.PipelineMetricsType.valueOf(nameSpacedMetrics[1]); + + // Pipeline defined metrics are long values. Have to cast to double that is what the base + // class defined. + return getBeamMetric(jobId, metricType, nameSpacedMetrics[2]).doubleValue(); + } else { + return super.getMetric(project, region, jobId, metricName); + } + } + + @Override + public Map getMetrics(String project, String region, String jobId) + throws IOException { + if (MANAGED_JOBS.containsKey(jobId)) { + // unsupported. Just return an empty map + return new HashMap<>(); + } else { + return super.getMetrics(project, region, jobId); + } + } + + @Override + public LaunchInfo launch(String project, String region, LaunchConfig options) throws IOException { + checkState( + options.sdk() != null, + "Cannot launch a dataflow job " + + "without sdk specified. Please specify sdk and try again!"); + LOG.info("Getting ready to launch {} in {} under {}", options.jobName(), region, project); + LOG.info("Using parameters:\n{}", formatForLogging(options.parameters())); + // Create SDK specific command and execute to launch dataflow job + List cmd = new ArrayList<>(); + String jobId; + switch (options.sdk()) { + case JAVA: + checkState( + options.pipeline() != null, + "Cannot launch a dataflow job " + + "without pipeline specified. Please specify pipeline and try again!"); + PipelineOptions pipelineOptions = options.pipeline().getOptions(); + if ("DataflowRunner".equalsIgnoreCase(options.getParameter("runner"))) { + List optionFromConfig = extractOptions(project, region, options); + // a few options need to be set in pipeline expansion time, so they need to be preserved + // here. + // known options included: --streaming(expansion depends on) --tempLocation(validation + // depends on) + if (pipelineOptions.as(StreamingOptions.class).isStreaming()) { + optionFromConfig.add("--streaming"); + } + if (!Strings.isNullOrEmpty(pipelineOptions.getTempLocation())) { + optionFromConfig.add( + String.format("--tempLocation=%s", pipelineOptions.getTempLocation())); + } + + // dataflow runner specific options + PipelineOptions updatedOptions = + PipelineOptionsFactory.fromArgs(optionFromConfig.toArray(new String[] {})).create(); + updatedOptions.setJobName(options.jobName()); + PipelineResult pipelineResult = options.pipeline().run(updatedOptions); + // dataflow runner generated a jobId of certain format for each job + DataflowPipelineJob job = (DataflowPipelineJob) pipelineResult; + jobId = job.getJobId(); + UNMANAGED_JOBS.put(jobId, pipelineResult); + launchedJobs.add(jobId); + } else { + + pipelineOptions.setRunner(PipelineUtils.getRunnerClass(options.getParameter("runner"))); + pipelineOptions.setJobName(options.jobName()); + // for unsupported runners (e.g. direct runner) runner, manually record job properties + Map jobProperties = new HashMap<>(); + jobProperties.put( + "createTime", Timestamps.toString(Timestamps.fromMillis(System.currentTimeMillis()))); + if (pipelineOptions.as(StreamingOptions.class).isStreaming()) { + jobProperties.put("jobType", "JOB_TYPE_STREAMING"); + } else { + jobProperties.put("jobType", "JOB_TYPE_BATCH"); + } + PipelineResult pipelineResult = options.pipeline().run(); + // for unsupported runners (e.g. direct runner), set jobId the same as jobName + jobId = options.jobName(); + MANAGED_JOBS.put(jobId, pipelineResult); + // for unsupported runners (e.g. direct runner), return a wrapped LaunchInfo + return LaunchInfo.builder() + .setJobId(jobId) + .setProjectId(project) + .setRegion(region) + .setCreateTime(jobProperties.get("createTime")) + .setSdk("DirectBeam") + .setVersion("0.0.1") + .setJobType(jobProperties.get("jobType")) + .setRunner(options.getParameter("runner")) + .setParameters(options.parameters()) + .setState(JobState.RUNNING) + .build(); + } + break; + case PYTHON: + checkState( + options.executable() != null, + "Cannot launch a dataflow job " + + "without executable specified. Please specify executable and try again!"); + if (options.requirementsFile() != null) { + // install requirements + cmd.add( + "virtualenv . && source ./bin/activate && pip3 install -r " + + options.requirementsFile()); + cmd.add("&&"); + } + LOG.info("Using the executable at {}", options.executable()); + cmd.add("python3"); + cmd.add(options.executable()); + cmd.addAll(extractOptions(project, region, options)); + if (options.requirementsFile() != null) { + cmd.add("&&"); + cmd.add("deactivate"); + } + jobId = executeCommandAndParseResponse(String.join(" ", cmd)); + break; + case GO: + checkState( + options.executable() != null, + "Cannot launch a dataflow job " + + "without executable specified. Please specify executable and try again!"); + LOG.info("Using the executable at {}", options.executable()); + cmd.add("go"); + cmd.add("run"); + cmd.add(options.executable()); + cmd.addAll(extractOptions(project, region, options)); + jobId = executeCommandAndParseResponse(String.join(" ", cmd)); + break; + default: + throw new RuntimeException( + String.format( + "Invalid sdk %s specified. " + "sdk can be one of java, python, or go.", + options.sdk())); + } + // Wait until the job is active to get more information + JobState state = waitUntilActive(project, region, jobId); + Job job = getJob(project, region, jobId, "JOB_VIEW_DESCRIPTION"); + LOG.info("Received Dataflow job {}: {}", job.getId(), formatForLogging(job)); + + return getJobInfo(options, state, job); + } + + @Override + protected LaunchInfo.Builder getJobInfoBuilder(LaunchConfig options, JobState state, Job job) { + // get intermediate builder from base class method + LaunchInfo.Builder builder = super.getJobInfoBuilder(options, state, job); + // config pipelineName + String pipelineName = PipelineUtils.extractJobName(options.jobName()); + String overrideName = null; + if (pipelineName.endsWith("write")) { + overrideName = System.getProperty(WRITE_PIPELINE_NAME_OVERWRITE); + } else if (pipelineName.endsWith("read")) { + overrideName = System.getProperty(READ_PIPELINE_NAME_OVERWRITE); + } + if (!Strings.isNullOrEmpty(overrideName)) { + pipelineName = overrideName; + } + builder.setPipelineName(pipelineName); + return builder; + } + + private List extractOptions(String project, String region, LaunchConfig options) { + List additionalOptions = new ArrayList<>(); + + // add pipeline options from beamTestPipelineOptions system property to preserve the + // pipeline options already set in TestPipeline. + @Nullable + String beamTestPipelineOptions = System.getProperty(PROPERTY_BEAM_TEST_PIPELINE_OPTIONS); + if (!Strings.isNullOrEmpty(beamTestPipelineOptions)) { + try { + additionalOptions.addAll(MAPPER.readValue(beamTestPipelineOptions, List.class)); + } catch (IOException e) { + throw new RuntimeException( + "Unable to instantiate test options from system property " + + PROPERTY_BEAM_TEST_PIPELINE_OPTIONS + + ":" + + System.getProperty(PROPERTY_BEAM_TEST_PIPELINE_OPTIONS), + e); + } + } + + // add pipeline options from options.parameters + for (Map.Entry parameter : options.parameters().entrySet()) { + additionalOptions.add(String.format("--%s=%s", parameter.getKey(), parameter.getValue())); + } + additionalOptions.add(String.format("--project=%s", project)); + additionalOptions.add(String.format("--region=%s", region)); + return additionalOptions; + } + + /** Executes the specified command and parses the response to get the Job ID. */ + private String executeCommandAndParseResponse(String cmd) throws IOException { + LOG.info("Running command: {}", cmd); + Process process = + new ProcessBuilder().command("/bin/bash", "-c", cmd).redirectErrorStream(true).start(); + String output = + new String(ByteStreams.toByteArray(process.getInputStream()), StandardCharsets.UTF_8); + LOG.info(output); + Matcher m = JOB_ID_PATTERN.matcher(output); + if (!m.find()) { + throw new RuntimeException( + String.format( + "Dataflow output in unexpected format. Failed to parse Dataflow Job ID. " + + "Result from process: %s", + output)); + } + String jobId = m.group(1); + LOG.info("Submitted job: {}", jobId); + return jobId; + } + + /** Builder for {@link DefaultPipelineLauncher}. */ + public static final class Builder { + private Credentials credentials; + + private Builder(Credentials credentials) { + this.credentials = credentials; + } + + public Credentials getCredentials() { + return credentials; + } + + public DefaultPipelineLauncher build() { + return new DefaultPipelineLauncher(this); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/DirectRunnerClient.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/DirectRunnerClient.java new file mode 100644 index 0000000000000..57f8ad40c1b6e --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/DirectRunnerClient.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.dataflow; + +import static org.apache.beam.it.common.logging.LogStrings.formatForLogging; + +import com.google.api.services.dataflow.model.Job; +import com.google.api.services.dataflow.model.JobMessage; +import com.google.auth.Credentials; +import java.io.IOException; +import java.lang.reflect.Method; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Implementation of the {@link PipelineLauncher} interface which invokes the template class using + * DirectRunner, and manages the state in memory. + */ +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/27438) +}) +public class DirectRunnerClient implements PipelineLauncher { + + private static final Logger LOG = LoggerFactory.getLogger(DirectRunnerClient.class); + + private final Map managedJobs = new HashMap<>(); + + private final Class mainClass; + + DirectRunnerClient(Builder builder) { + this.mainClass = builder.getMainClass(); + } + + public static DirectRunnerClient.Builder builder(Class mainClass) { + return new DirectRunnerClient.Builder(mainClass); + } + + @Override + public LaunchInfo launch(String project, String region, LaunchConfig options) throws IOException { + + LOG.info("Getting ready to launch {} in {} under {}", options.jobName(), region, project); + LOG.info("Using parameters:\n{}", formatForLogging(options.parameters())); + + try { + List cmd = new ArrayList<>(); + + for (String parameter : options.parameters().keySet()) { + cmd.add(String.format("--%s=%s", parameter, options.getParameter(parameter))); + } + cmd.add(String.format("--project=%s", project)); + cmd.add(String.format("--region=%s", region)); + + String jobId = + "direct-" + + new SimpleDateFormat("yyyy-MM-dd_HH_mm_ss").format(new Date()) + + "-" + + System.currentTimeMillis(); + + DirectRunnerJobThread jobThread = + new DirectRunnerJobThread(project, region, jobId, mainClass, cmd); + managedJobs.put(jobId, jobThread); + jobThread.start(); + + return LaunchInfo.builder() + .setJobId(jobId) + .setProjectId(project) + .setRegion(region) + .setCreateTime("") + .setSdk("DirectBeam") + .setVersion("0.0.1") + .setJobType("JOB_TYPE_BATCH") + .setRunner("DirectRunner") + .setParameters(options.parameters()) + .setState(JobState.RUNNING) + .build(); + } catch (Exception e) { + throw new RuntimeException("Error launching DirectRunner test", e); + } + } + + @Override + public Job getJob(String project, String region, String jobId) { + return managedJobs.get(jobId).getJob(); + } + + @Override + public Job getJob(String project, String region, String jobId, String jobView) { + return managedJobs.get(jobId).getJob(); + } + + @Override + public JobState getJobStatus(String project, String region, String jobId) { + return managedJobs.get(jobId).getJobState(); + } + + @Override + public List listMessages( + String project, String region, String jobId, String minimumImportance) { + return new ArrayList<>(); + } + + @Override + public Job cancelJob(String project, String region, String jobId) { + LOG.warn("Cancelling direct runner job {}.", jobId); + + managedJobs.get(jobId).cancel(); + return new Job().setId(jobId).setRequestedState(JobState.CANCELLED.toString()); + } + + @Override + public Job drainJob(String project, String region, String jobId) { + LOG.warn("Cannot drain a direct runner job. Cancelling the job instead."); + return cancelJob(project, region, jobId); + } + + @Override + public Double getMetric(String project, String region, String jobId, String metricName) { + return null; + } + + @Override + public Map getMetrics(String project, String region, String jobId) + throws IOException { + return null; + } + + @Override + public synchronized void cleanupAll() throws IOException { + // Cancel / terminate all threads for DirectRunnerClient + for (DirectRunnerJobThread jobs : managedJobs.values()) { + jobs.cancel(); + } + } + + /** Builder for {@link DirectRunnerClient}. */ + public static final class Builder { + + private Credentials credentials; + private final Class mainClass; + + private Builder(Class mainClass) { + this.mainClass = mainClass; + } + + public Credentials getCredentials() { + return credentials; + } + + public Class getMainClass() { + return mainClass; + } + + public DirectRunnerClient.Builder setCredentials(Credentials value) { + credentials = value; + return this; + } + + public DirectRunnerClient build() { + return new DirectRunnerClient(this); + } + } + + static class DirectRunnerJobThread extends Thread { + + private final Job currentJob; + private final Class mainClass; + private final List commandLines; + private Throwable throwable; + private boolean cancelled; + + public DirectRunnerJobThread( + String projectId, + String location, + String jobId, + Class mainClass, + List commandLines) { + this.currentJob = + new Job() + .setProjectId(projectId) + .setLocation(location) + .setId(jobId) + .setCurrentState(JobState.QUEUED.toString()); + this.mainClass = mainClass; + this.commandLines = commandLines; + } + + @Override + public void run() { + + try { + String[] args = commandLines.toArray(new String[0]); + Method mainMethod = mainClass.getMethod("main", String[].class); + currentJob.setCurrentState(JobState.RUNNING.toString()); + + LOG.info("Starting job {}...", currentJob.getId()); + mainMethod.setAccessible(true); + mainMethod.invoke(null, (Object) args); + + currentJob.setCurrentState(JobState.DONE.toString()); + } catch (Throwable e) { + + // Errors are acceptable if thread was cancelled + if (!cancelled) { + LOG.warn("Error occurred with job {}", currentJob.getId(), e); + this.throwable = e; + currentJob.setCurrentState(JobState.FAILED.toString()); + } + } + } + + public Job getJob() { + return currentJob; + } + + public Throwable getThrowable() { + return throwable; + } + + public JobState getJobState() { + return JobState.parse(currentJob.getCurrentState()); + } + + public void cancel() { + if (this.cancelled || !isAlive()) { + return; + } + + LOG.info("Finishing job {}...", currentJob.getId()); + + this.cancelled = true; + currentJob.setCurrentState(JobState.CANCELLED.toString()); + + try { + this.stop(); + } catch (Exception e) { + LOG.warn("Error cancelling job", e); + } + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/FlexTemplateClient.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/FlexTemplateClient.java new file mode 100644 index 0000000000000..0a9731b13c32a --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/FlexTemplateClient.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.dataflow; + +import static org.apache.beam.it.common.logging.LogStrings.formatForLogging; +import static org.apache.beam.it.common.utils.RetryUtil.clientRetryPolicy; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; + +import com.google.api.client.googleapis.util.Utils; +import com.google.api.services.dataflow.Dataflow; +import com.google.api.services.dataflow.model.FlexTemplateRuntimeEnvironment; +import com.google.api.services.dataflow.model.Job; +import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter; +import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest; +import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse; +import com.google.auth.Credentials; +import com.google.auth.http.HttpCredentialsAdapter; +import dev.failsafe.Failsafe; +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Client for interacting with Dataflow Flex Templates using the Dataflow SDK. */ +public final class FlexTemplateClient extends AbstractPipelineLauncher { + private static final Logger LOG = LoggerFactory.getLogger(FlexTemplateClient.class); + + private FlexTemplateClient(Builder builder) { + super( + new Dataflow( + Utils.getDefaultTransport(), + Utils.getDefaultJsonFactory(), + new HttpCredentialsAdapter(builder.getCredentials()))); + } + + private FlexTemplateClient(Dataflow dataflow) { + super(dataflow); + } + + public static FlexTemplateClient withDataflowClient(Dataflow dataflow) { + return new FlexTemplateClient(dataflow); + } + + public static Builder builder(Credentials credentials) { + return new Builder(credentials); + } + + @Override + public LaunchInfo launch(String project, String region, LaunchConfig options) throws IOException { + checkState( + options.specPath() != null, + "Cannot launch a template job without specPath. Please specify specPath and try again!"); + LOG.info("Getting ready to launch {} in {} under {}", options.jobName(), region, project); + LOG.info("Using the spec at {}", options.specPath()); + LOG.info("Using parameters:\n{}", formatForLogging(options.parameters())); + + @SuppressWarnings("nullness") + LaunchFlexTemplateParameter parameter = + new LaunchFlexTemplateParameter() + .setJobName(options.jobName()) + .setParameters(options.parameters()) + .setContainerSpecGcsPath(options.specPath()) + .setEnvironment(buildEnvironment(options)); + LaunchFlexTemplateRequest request = + new LaunchFlexTemplateRequest().setLaunchParameter(parameter); + LOG.info("Sending request:\n{}", formatForLogging(request)); + + LaunchFlexTemplateResponse response = + Failsafe.with(clientRetryPolicy()) + .get( + () -> + client + .projects() + .locations() + .flexTemplates() + .launch(project, region, request) + .execute()); + Job job = response.getJob(); + printJobResponse(job); + + // Wait until the job is active to get more information + JobState state = waitUntilActive(project, region, job.getId()); + job = getJob(project, region, job.getId(), "JOB_VIEW_DESCRIPTION"); + LOG.info("Received flex template job {}: {}", job.getId(), formatForLogging(job)); + + launchedJobs.add(job.getId()); + return getJobInfo(options, state, job); + } + + private FlexTemplateRuntimeEnvironment buildEnvironment(LaunchConfig options) { + FlexTemplateRuntimeEnvironment environment = new FlexTemplateRuntimeEnvironment(); + environment.putAll(options.environment()); + + if (System.getProperty("launcherMachineType") != null) { + environment.setLauncherMachineType(System.getProperty("launcherMachineType")); + } + + return environment; + } + + /** Builder for {@link FlexTemplateClient}. */ + public static final class Builder { + private Credentials credentials; + + private Builder(Credentials credentials) { + this.credentials = credentials; + } + + public Credentials getCredentials() { + return credentials; + } + + public Builder setCredentials(Credentials value) { + credentials = value; + return this; + } + + public FlexTemplateClient build() { + return new FlexTemplateClient(this); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/package-info.java new file mode 100644 index 0000000000000..7fea880ac65cf --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing Dataflow jobs from integration tests. */ +package org.apache.beam.it.gcp.dataflow; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datagenerator/DataGenerator.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datagenerator/DataGenerator.java new file mode 100644 index 0000000000000..832a75defd95b --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datagenerator/DataGenerator.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.datagenerator; + +import static org.apache.beam.it.gcp.LoadTestBase.createConfig; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.auth.Credentials; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineLauncher.LaunchConfig; +import org.apache.beam.it.common.PipelineLauncher.LaunchInfo; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.PipelineOperator.Result; +import org.apache.beam.it.common.TestProperties; +import org.apache.beam.it.gcp.dataflow.FlexTemplateClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Helper class for starting a Streaming Data Generator Dataflow template job. */ +public class DataGenerator { + private static final Logger LOG = LoggerFactory.getLogger(DataGenerator.class); + private static final String SPEC_PATH = + "gs://dataflow-templates/latest/flex/Streaming_Data_Generator"; + private static final String PROJECT = TestProperties.project(); + private static final String REGION = TestProperties.region(); + private static final Credentials CREDENTIALS = TestProperties.googleCredentials(); + private static final String MESSAGES_GENERATED_METRIC_NAME = + "Generate Fake Messages-out0-ElementCount"; + private static final String MESSAGES_LIMIT = "messagesLimit"; + private final LaunchConfig dataGeneratorOptions; + private final PipelineLauncher pipelineLauncher; + private final PipelineOperator pipelineOperator; + + private DataGenerator(Builder builder) { + pipelineLauncher = FlexTemplateClient.builder(CREDENTIALS).build(); + pipelineOperator = new PipelineOperator(pipelineLauncher); + this.dataGeneratorOptions = + LaunchConfig.builder(builder.getJobName(), SPEC_PATH) + .setParameters(builder.getParameters()) + .addParameter("experiments", "disable_runner_v2") + .build(); + } + + public static DataGenerator.Builder builderWithSchemaLocation( + String testName, String schemaLocation) { + return new DataGenerator.Builder(testName + "-data-generator") + .setSchemaLocation(schemaLocation) + .setAutoscalingAlgorithm(AutoscalingAlgorithmType.THROUGHPUT_BASED); + } + + public static DataGenerator.Builder builderWithSchemaTemplate( + String testName, String schemaTemplate) { + return new DataGenerator.Builder(testName + "-data-generator") + .setSchemaTemplate(schemaTemplate) + .setAutoscalingAlgorithm(AutoscalingAlgorithmType.THROUGHPUT_BASED); + } + + /** + * Executes the data generator using the config provided. If messageLimit is provided, we wait + * until the data generator finishes, or we reach timeout. If a messageLimit is not provided we + * wait until timeout and cancel the data generator. + * + *

    Note: This is a blocking call. For backlog tests, start the Data generator before executing + * the template under test. For testing a pipeline against live incoming data, execute the data + * generator after starting the pipeline under test. + * + * @param timeout time to wait before cancelling the data generator. + * @return approximate number of messages generated + * @throws IOException if any errors are encountered. + */ + public Integer execute(Duration timeout) throws IOException { + LaunchInfo dataGeneratorLaunchInfo = + pipelineLauncher.launch(PROJECT, REGION, dataGeneratorOptions); + assertThatPipeline(dataGeneratorLaunchInfo).isRunning(); + PipelineOperator.Config config = createConfig(dataGeneratorLaunchInfo, timeout); + // check if the job will be BATCH or STREAMING + if (dataGeneratorOptions.parameters().containsKey(MESSAGES_LIMIT)) { + // Batch job, wait till data generator job finishes + Result dataGeneratorResult = pipelineOperator.waitUntilDone(config); + assertThatResult(dataGeneratorResult).isLaunchFinished(); + } else { + // Streaming job, wait till timeout and drain job + Result dataGeneratorResult = pipelineOperator.waitUntilDoneAndFinish(config); + assertThatResult(dataGeneratorResult).hasTimedOut(); + } + @SuppressWarnings("nullness") + int generatedMessages = + pipelineLauncher + .getMetric( + PROJECT, REGION, dataGeneratorLaunchInfo.jobId(), MESSAGES_GENERATED_METRIC_NAME) + .intValue(); + LOG.info("Data generator finished. Generated {} messages.", generatedMessages); + return generatedMessages; + } + + /** Builder for the {@link DataGenerator}. */ + public static final class Builder { + private final String jobName; + private final Map parameters; + + private Builder(String jobName) { + this.jobName = jobName; + this.parameters = new HashMap<>(); + } + + public String getJobName() { + return jobName; + } + + public Map getParameters() { + return parameters; + } + + public DataGenerator.Builder setSchemaTemplate(String value) { + parameters.put("schemaTemplate", value); + return this; + } + + public DataGenerator.Builder setSchemaLocation(String value) { + parameters.put("schemaLocation", value); + return this; + } + + public DataGenerator.Builder setMessagesLimit(String value) { + parameters.put(MESSAGES_LIMIT, value); + return this; + } + + public DataGenerator.Builder setQPS(String value) { + parameters.put("qps", value); + return this; + } + + public DataGenerator.Builder setSinkType(String value) { + parameters.put("sinkType", value); + return this; + } + + public Builder setWorkerMachineType(String value) { + parameters.put("workerMachineType", value); + return this; + } + + public Builder setNumWorkers(String value) { + parameters.put("numWorkers", value); + return this; + } + + public DataGenerator.Builder setMaxNumWorkers(String value) { + parameters.put("maxNumWorkers", value); + return this; + } + + public DataGenerator.Builder setAutoscalingAlgorithm(AutoscalingAlgorithmType value) { + parameters.put("autoscalingAlgorithm", value.toString()); + return this; + } + + public DataGenerator.Builder setOutputDirectory(String value) { + parameters.put("outputDirectory", value); + return this; + } + + public DataGenerator.Builder setOutputType(String value) { + parameters.put("outputType", value); + return this; + } + + public DataGenerator.Builder setNumShards(String value) { + parameters.put("numShards", value); + return this; + } + + public DataGenerator.Builder setAvroSchemaLocation(String value) { + parameters.put("avroSchemaLocation", value); + return this; + } + + public DataGenerator.Builder setTopic(String value) { + parameters.put("topic", value); + return this; + } + + public DataGenerator.Builder setProjectId(String value) { + parameters.put("projectId", value); + return this; + } + + public DataGenerator.Builder setSpannerInstanceName(String value) { + parameters.put("spannerInstanceName", value); + return this; + } + + public DataGenerator.Builder setSpannerDatabaseName(String value) { + parameters.put("spannerDatabaseName", value); + return this; + } + + public DataGenerator.Builder setSpannerTableName(String value) { + parameters.put("spannerTableName", value); + return this; + } + + public DataGenerator.Builder setDriverClassName(String value) { + parameters.put("driverClassName", value); + return this; + } + + public DataGenerator.Builder setConnectionUrl(String value) { + parameters.put("connectionUrl", value); + return this; + } + + public DataGenerator.Builder setUsername(String value) { + parameters.put("username", value); + return this; + } + + public DataGenerator.Builder setPassword(String value) { + parameters.put("password", value); + return this; + } + + public DataGenerator.Builder setConnectionProperties(String value) { + parameters.put("connectionProperties", value); + return this; + } + + public DataGenerator.Builder setStatement(String value) { + parameters.put("statement", value); + return this; + } + + public DataGenerator build() { + return new DataGenerator(this); + } + } + + /** Enum representing Autoscaling algorithm types. */ + public enum AutoscalingAlgorithmType { + NONE("NONE"), + THROUGHPUT_BASED("THROUGHPUT_BASED"); + + private final String text; + + AutoscalingAlgorithmType(String text) { + this.text = text; + } + + @Override + public String toString() { + return this.text; + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datagenerator/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datagenerator/package-info.java new file mode 100644 index 0000000000000..d563025ebeb48 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datagenerator/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Data generator for load tests. */ +package org.apache.beam.it.gcp.datagenerator; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastore/DatastoreResourceManager.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastore/DatastoreResourceManager.java new file mode 100644 index 0000000000000..4225d90e52b9f --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastore/DatastoreResourceManager.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.datastore; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import com.google.auth.Credentials; +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.DatastoreOptions; +import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.FullEntity; +import com.google.cloud.datastore.GqlQuery; +import com.google.cloud.datastore.Key; +import com.google.cloud.datastore.Query.ResultType; +import com.google.cloud.datastore.QueryResults; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.beam.it.common.ResourceManager; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; + +/** Client for managing Datastore resources. */ +public class DatastoreResourceManager implements ResourceManager { + + private final String namespace; + + private final Datastore datastore; + private final Set keys; + + public DatastoreResourceManager(Builder builder) { + this.namespace = builder.namespace; + + this.datastore = + DatastoreOptions.newBuilder() + .setProjectId(builder.project) + .setCredentials(builder.credentials) + .build() + .getService(); + this.keys = new HashSet<>(); + } + + @VisibleForTesting + DatastoreResourceManager(String namespace, Datastore datastore) { + this.namespace = namespace; + this.datastore = datastore; + this.keys = new HashSet<>(); + } + + /** + * Insert entities to Datastore. + * + * @param kind Kind of document to insert. + * @param entities Entities to insert to Datastore. + * @return Entities. + */ + public List insert(String kind, Map> entities) { + List created = new ArrayList<>(); + + try { + for (Map.Entry> entry : entities.entrySet()) { + Key entityKey = + datastore.newKeyFactory().setKind(kind).setNamespace(namespace).newKey(entry.getKey()); + Entity entity = Entity.newBuilder(entityKey, entry.getValue()).build(); + created.add(datastore.put(entity)); + keys.add(entityKey); + } + } catch (Exception e) { + throw new DatastoreResourceManagerException("Error inserting Datastore entity", e); + } + + return created; + } + + /** + * Run a Gql Query and return the results in entity format. + * + * @param gqlQuery Gql Query to run. + * @return Entities returned from the query. + */ + public List query(String gqlQuery) { + try { + QueryResults queryResults = + datastore.run( + GqlQuery.newGqlQueryBuilder(ResultType.ENTITY, gqlQuery) + .setNamespace(namespace) + .build()); + + List entities = new ArrayList<>(); + + while (queryResults.hasNext()) { + Entity entity = queryResults.next(); + entities.add(entity); + + // Mark for deletion if namespace matches the test + if (entity.getKey().getNamespace().equals(namespace)) { + keys.add(entity.getKey()); + } + } + return entities; + } catch (Exception e) { + throw new DatastoreResourceManagerException("Error running Datastore query", e); + } + } + + /** + * Deletes all created entities and cleans up the Datastore client. + * + * @throws DatastoreResourceManagerException if there is an error deleting the entities in + * Datastore. + */ + @Override + public void cleanupAll() throws DatastoreResourceManagerException { + try { + datastore.delete(keys.toArray(new Key[0])); + } catch (Exception e) { + throw new DatastoreResourceManagerException("Error cleaning up resources", e); + } + keys.clear(); + } + + public static Builder builder(String project, String namespace, Credentials credentials) { + checkArgument(!Strings.isNullOrEmpty(project), "project can not be empty"); + checkArgument(!Strings.isNullOrEmpty(namespace), "namespace can not be empty"); + return new Builder(project, namespace, credentials); + } + + public static final class Builder { + + private final String project; + private final String namespace; + private Credentials credentials; + + private Builder(String project, String namespace, Credentials credentials) { + this.project = project; + this.namespace = namespace; + this.credentials = credentials; + } + + public Builder credentials(Credentials credentials) { + this.credentials = credentials; + return this; + } + + public DatastoreResourceManager build() { + if (credentials == null) { + throw new IllegalArgumentException( + "Unable to find credentials. Please provide credentials to authenticate to GCP"); + } + return new DatastoreResourceManager(this); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastore/DatastoreResourceManagerException.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastore/DatastoreResourceManagerException.java new file mode 100644 index 0000000000000..197fdf888167c --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastore/DatastoreResourceManagerException.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.datastore; + +/** Custom exception for {@link DatastoreResourceManager} implementations. */ +public class DatastoreResourceManagerException extends RuntimeException { + + public DatastoreResourceManagerException(String errorMessage, Throwable err) { + super(errorMessage, err); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastore/DatastoreUtils.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastore/DatastoreUtils.java new file mode 100644 index 0000000000000..ebefa8d282728 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastore/DatastoreUtils.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.datastore; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CaseFormat; + +/** Utilities to make working with Datastore easier. */ +final class DatastoreUtils { + private DatastoreUtils() {} + + /** + * Creates a topic name. + * + *

    If there are uppercase characters in {@code prefix}, then this will convert them into a dash + * followed by the lowercase equivalent of that letter. + * + *

    The topic name will normally be unique, but this is not guaranteed if multiple topics with + * the same prefix are created in a short period of time. + * + * @param prefix a prefix for the topic + * @return the prefix plus some way of identifying it separate from other topics with the same + * prefix + */ + static String createTestId(String prefix) { + String convertedPrefix = + CaseFormat.UPPER_CAMEL.converterTo(CaseFormat.LOWER_HYPHEN).convert(prefix); + String formattedTimestamp = + DateTimeFormatter.ofPattern("yyyyMMddHHmmssSSS") + .withZone(ZoneId.of("UTC")) + .format(Instant.now()); + return String.format("%s-%s", convertedPrefix, formattedTimestamp); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastore/matchers/DatastoreAsserts.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastore/matchers/DatastoreAsserts.java new file mode 100644 index 0000000000000..78fa7543150fd --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastore/matchers/DatastoreAsserts.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.datastore.matchers; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatRecords; + +import com.google.cloud.datastore.Entity; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.truthmatchers.RecordsSubject; + +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/27438) +}) +public class DatastoreAsserts { + + /** + * Convert Datastore {@link com.google.cloud.datastore.QueryResults} to a list of maps. + * + * @param results Results to parse. + * @return List of maps to use in {@link RecordsSubject}. + */ + public static List> datastoreResultsToRecords(Collection results) { + try { + List> records = new ArrayList<>(); + + for (Entity entity : results) { + Map converted = new HashMap<>(); + + for (Map.Entry> entry : + entity.getProperties().entrySet()) { + converted.put(entry.getKey(), entry.getValue().get()); + } + records.add(converted); + } + + return records; + } catch (Exception e) { + throw new RuntimeException("Error converting Datastore Entities to Records", e); + } + } + + /** + * Creates a {@link RecordsSubject} to assert information within a list of records. + * + * @param results Records in Datastore {@link com.google.cloud.datastore.Entity} format to use in + * the comparison. + * @return Truth subject to chain assertions. + */ + public static RecordsSubject assertThatDatastoreRecords(Collection results) { + return assertThatRecords(datastoreResultsToRecords(results)); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastore/matchers/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastore/matchers/package-info.java new file mode 100644 index 0000000000000..e68f7b244d9a8 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastore/matchers/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for Datastore Truth matchers / subjects to have reusable assertions. */ +package org.apache.beam.it.gcp.datastore.matchers; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastore/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastore/package-info.java new file mode 100644 index 0000000000000..21c3bdea8babf --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastore/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing Datastore resources within integration tests. */ +package org.apache.beam.it.gcp.datastore; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/DatastreamResourceManager.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/DatastreamResourceManager.java new file mode 100644 index 0000000000000..bac25ac941353 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/DatastreamResourceManager.java @@ -0,0 +1,507 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.datastream; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.cloud.datastream.v1.AvroFileFormat; +import com.google.cloud.datastream.v1.BigQueryDestinationConfig; +import com.google.cloud.datastream.v1.BigQueryProfile; +import com.google.cloud.datastream.v1.ConnectionProfile; +import com.google.cloud.datastream.v1.ConnectionProfileName; +import com.google.cloud.datastream.v1.CreateConnectionProfileRequest; +import com.google.cloud.datastream.v1.CreateStreamRequest; +import com.google.cloud.datastream.v1.DatastreamClient; +import com.google.cloud.datastream.v1.DatastreamSettings; +import com.google.cloud.datastream.v1.DeleteConnectionProfileRequest; +import com.google.cloud.datastream.v1.DeleteStreamRequest; +import com.google.cloud.datastream.v1.DestinationConfig; +import com.google.cloud.datastream.v1.GcsDestinationConfig; +import com.google.cloud.datastream.v1.GcsProfile; +import com.google.cloud.datastream.v1.JsonFileFormat; +import com.google.cloud.datastream.v1.LocationName; +import com.google.cloud.datastream.v1.MysqlProfile; +import com.google.cloud.datastream.v1.MysqlSourceConfig; +import com.google.cloud.datastream.v1.OracleProfile; +import com.google.cloud.datastream.v1.OracleSourceConfig; +import com.google.cloud.datastream.v1.PostgresqlProfile; +import com.google.cloud.datastream.v1.PostgresqlSourceConfig; +import com.google.cloud.datastream.v1.SourceConfig; +import com.google.cloud.datastream.v1.StaticServiceIpConnectivity; +import com.google.cloud.datastream.v1.Stream; +import com.google.cloud.datastream.v1.StreamName; +import com.google.cloud.datastream.v1.UpdateStreamRequest; +import com.google.protobuf.Duration; +import com.google.protobuf.FieldMask; +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import org.apache.beam.it.common.ResourceManager; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Client for Datastream resources. + * + *

    This class is thread safe. + */ +public final class DatastreamResourceManager implements ResourceManager { + private static final Logger LOG = LoggerFactory.getLogger(DatastreamResourceManager.class); + private static final String FIELD_STATE = "state"; + private final DatastreamClient datastreamClient; + private final String location; + private final String projectId; + private final Set createdStreamIds; + private final Set createdConnectionProfileIds; + + enum DestinationOutputFormat { + AVRO_FILE_FORMAT, + JSON_FILE_FORMAT + } + + private DatastreamResourceManager(Builder builder) throws IOException { + this( + DatastreamClient.create( + DatastreamSettings.newBuilder() + .setCredentialsProvider(builder.credentialsProvider) + .build()), + builder); + } + + @VisibleForTesting + public DatastreamResourceManager(DatastreamClient datastreamClient, Builder builder) { + this.datastreamClient = datastreamClient; + this.location = builder.location; + this.projectId = builder.projectId; + this.createdStreamIds = Collections.synchronizedSet(new HashSet<>()); + this.createdConnectionProfileIds = Collections.synchronizedSet(new HashSet<>()); + } + + public static Builder builder( + String projectId, String location, CredentialsProvider credentialsProvider) { + checkArgument(!Strings.isNullOrEmpty(projectId), "projectID can not be null or empty"); + checkArgument(!Strings.isNullOrEmpty(location), "location can not be null or empty"); + return new Builder(projectId, location, credentialsProvider); + } + + /** + * @param connectionProfileId The ID of the connection profile. + * @param source An object representing the JDBC source. + * @return A Datastream JDBC source connection profile. + */ + private synchronized ConnectionProfile createJDBCSourceConnectionProfile( + String connectionProfileId, JDBCSource source) { + checkArgument( + !Strings.isNullOrEmpty(connectionProfileId), + "connectionProfileId can not be null or empty"); + LOG.info( + "Creating JDBC Source Connection Profile {} in project {}.", + connectionProfileId, + projectId); + ConnectionProfile.Builder connectionProfileBuilder = ConnectionProfile.newBuilder(); + JDBCSource.SourceType type = source.type(); + + try { + switch (type) { + case MYSQL: + MysqlProfile.Builder mysqlProfileBuilder = MysqlProfile.newBuilder(); + mysqlProfileBuilder + .setHostname(source.hostname()) + .setUsername(source.username()) + .setPassword(source.password()) + .setPort(source.port()); + connectionProfileBuilder.setMysqlProfile(mysqlProfileBuilder); + break; + case ORACLE: + OracleProfile.Builder oracleProfileBuilder = OracleProfile.newBuilder(); + oracleProfileBuilder + .setHostname(source.hostname()) + .setUsername(source.username()) + .setPassword(source.password()) + .setPort(source.port()); + connectionProfileBuilder.setOracleProfile(oracleProfileBuilder); + break; + case POSTGRESQL: + PostgresqlProfile.Builder postgresqlProfileBuilder = PostgresqlProfile.newBuilder(); + postgresqlProfileBuilder + .setHostname(source.hostname()) + .setUsername(source.username()) + .setPassword(source.password()) + .setPort(source.port()) + .setDatabase(((PostgresqlSource) source).database()); + connectionProfileBuilder.setPostgresqlProfile(postgresqlProfileBuilder); + break; + default: + throw new DatastreamResourceManagerException( + "Could not recognize JDBC source type " + type.name()); + } + + ConnectionProfile connectionProfile = + connectionProfileBuilder + .setDisplayName(connectionProfileId) + .setStaticServiceIpConnectivity(StaticServiceIpConnectivity.getDefaultInstance()) + .build(); + CreateConnectionProfileRequest request = + CreateConnectionProfileRequest.newBuilder() + .setParent(LocationName.of(projectId, location).toString()) + .setConnectionProfile(connectionProfile) + .setConnectionProfileId(connectionProfileId) + .build(); + ConnectionProfile reference = datastreamClient.createConnectionProfileAsync(request).get(); + createdConnectionProfileIds.add(connectionProfileId); + + LOG.info( + "Successfully created JDBC Source Connection Profile {} in project {}.", + connectionProfileId, + projectId); + return reference; + } catch (ExecutionException | InterruptedException e) { + throw new DatastreamResourceManagerException( + "Failed to create JDBC source connection profile. ", e); + } + } + + /** + * @param sourceConnectionProfileId The ID of the connection profile. + * @param source An object representing the JDBC source. + * @return a SourceConfig object which is required to configure a Stream. + */ + public synchronized SourceConfig buildSourceConfig( + String sourceConnectionProfileId, JDBCSource source) { + + createJDBCSourceConnectionProfile(sourceConnectionProfileId, source); + SourceConfig.Builder sourceConfigBuilder = + SourceConfig.newBuilder() + .setSourceConnectionProfile( + ConnectionProfileName.format(projectId, location, sourceConnectionProfileId)); + + switch (source.type()) { + case MYSQL: + sourceConfigBuilder.setMysqlSourceConfig((MysqlSourceConfig) source.config()); + break; + case POSTGRESQL: + sourceConfigBuilder.setPostgresqlSourceConfig((PostgresqlSourceConfig) source.config()); + break; + case ORACLE: + sourceConfigBuilder.setOracleSourceConfig((OracleSourceConfig) source.config()); + break; + default: + throw new DatastreamResourceManagerException( + "Could not recognize JDBC source type " + source.type().name()); + } + + return sourceConfigBuilder.build(); + } + + /** + * @param connectionProfileId The ID of the GCS connection profile. + * @param gcsBucketName The GCS Bucket to connect to. + * @param gcsRootPath The Path prefix to specific gcs location. Can either be empty or must start + * with '/'. + * @return A Datastream GCS destination connection profile. + */ + public synchronized ConnectionProfile createGCSDestinationConnectionProfile( + String connectionProfileId, String gcsBucketName, String gcsRootPath) { + checkArgument( + !Strings.isNullOrEmpty(connectionProfileId), + "connectionProfileId can not be null or empty"); + checkArgument(!Strings.isNullOrEmpty(gcsBucketName), "gcsBucketName can not be null or empty"); + checkArgument(gcsRootPath != null, "gcsRootPath can not be null"); + checkArgument( + gcsRootPath.isEmpty() || gcsRootPath.charAt(0) == '/', + "gcsRootPath must either be an empty string or start with a '/'"); + + LOG.info( + "Creating GCS Destination Connection Profile {} in project {}.", + connectionProfileId, + projectId); + + try { + ConnectionProfile.Builder connectionProfileBuilder = + ConnectionProfile.newBuilder() + .setDisplayName(connectionProfileId) + .setStaticServiceIpConnectivity(StaticServiceIpConnectivity.getDefaultInstance()) + .setGcsProfile( + GcsProfile.newBuilder().setBucket(gcsBucketName).setRootPath(gcsRootPath)); + + CreateConnectionProfileRequest request = + CreateConnectionProfileRequest.newBuilder() + .setParent(LocationName.of(projectId, location).toString()) + .setConnectionProfile(connectionProfileBuilder) + .setConnectionProfileId(connectionProfileId) + .build(); + + ConnectionProfile reference = datastreamClient.createConnectionProfileAsync(request).get(); + createdConnectionProfileIds.add(connectionProfileId); + LOG.info( + "Successfully created GCS Destination Connection Profile {} in project {}.", + connectionProfileId, + projectId); + + return reference; + } catch (ExecutionException | InterruptedException e) { + throw new DatastreamResourceManagerException( + "Failed to create GCS source connection profile. ", e); + } + } + + /** + * @param connectionProfileId The ID of the connection profile. + * @param path The Path prefix to specific GCS location. Can either be empty or must start with + * '/'. + * @param destinationOutputFormat The format of the files written to GCS. + * @return A DestinationConfig object representing a GCS destination configuration. + */ + public synchronized DestinationConfig buildGCSDestinationConfig( + String connectionProfileId, String path, DestinationOutputFormat destinationOutputFormat) { + + DestinationConfig.Builder destinationConfigBuilder = + DestinationConfig.newBuilder() + .setDestinationConnectionProfile( + ConnectionProfileName.format(projectId, location, connectionProfileId)); + + GcsDestinationConfig.Builder gcsDestinationConfigBuilder = + GcsDestinationConfig.newBuilder().setPath(path); + + if (destinationOutputFormat == DestinationOutputFormat.AVRO_FILE_FORMAT) { + gcsDestinationConfigBuilder.setAvroFileFormat(AvroFileFormat.getDefaultInstance()); + } else { + gcsDestinationConfigBuilder.setJsonFileFormat(JsonFileFormat.getDefaultInstance()); + } + + destinationConfigBuilder.setGcsDestinationConfig(gcsDestinationConfigBuilder); + return destinationConfigBuilder.build(); + } + + /** + * @param connectionProfileId The ID of the connection profile. + * @return A Datastream BigQuery destination connection profile. + */ + public synchronized ConnectionProfile createBQDestinationConnectionProfile( + String connectionProfileId) { + + LOG.info( + "Creating BQ Destination Connection Profile {} in project {}.", + connectionProfileId, + projectId); + + try { + ConnectionProfile.Builder connectionProfileBuilder = + ConnectionProfile.newBuilder() + .setDisplayName(connectionProfileId) + .setStaticServiceIpConnectivity(StaticServiceIpConnectivity.getDefaultInstance()) + .setBigqueryProfile(BigQueryProfile.newBuilder()); + + CreateConnectionProfileRequest request = + CreateConnectionProfileRequest.newBuilder() + .setParent(LocationName.of(projectId, location).toString()) + .setConnectionProfile(connectionProfileBuilder) + .setConnectionProfileId(connectionProfileId) + .build(); + + ConnectionProfile reference = datastreamClient.createConnectionProfileAsync(request).get(); + createdConnectionProfileIds.add(connectionProfileId); + + LOG.info( + "Successfully created BQ Destination Connection Profile {} in project {}.", + connectionProfileId, + projectId); + return reference; + } catch (ExecutionException | InterruptedException e) { + throw new DatastreamResourceManagerException( + "Failed to create BQ destination connection profile. ", e); + } + } + + /** + * @param connectionProfileId The ID of the connection profile. + * @param stalenessLimitSeconds The desired data freshness in seconds. + * @param datasetId The ID of the BigQuery dataset. + * @return A DestinationConfig object representing a BigQuery destination configuration. + */ + public synchronized DestinationConfig buildBQDestinationConfig( + String connectionProfileId, long stalenessLimitSeconds, String datasetId) { + + DestinationConfig.Builder destinationConfigBuilder = + DestinationConfig.newBuilder() + .setDestinationConnectionProfile( + ConnectionProfileName.format(projectId, location, connectionProfileId)); + + BigQueryDestinationConfig.Builder bigQueryDestinationConfig = + BigQueryDestinationConfig.newBuilder() + .setDataFreshness(Duration.newBuilder().setSeconds(stalenessLimitSeconds).build()) + .setSingleTargetDataset( + BigQueryDestinationConfig.SingleTargetDataset.newBuilder().setDatasetId(datasetId)); + + destinationConfigBuilder.setBigqueryDestinationConfig(bigQueryDestinationConfig); + return destinationConfigBuilder.build(); + } + + /** + * @param streamId The ID of the stream. + * @param sourceConfig A SourceConfig object representing the source configuration. + * @param destinationConfig A DestinationConfig object representing the destination configuration. + * @return A Datastream stream object. + */ + public synchronized Stream createStream( + String streamId, SourceConfig sourceConfig, DestinationConfig destinationConfig) { + + LOG.info("Creating Stream {} in project {}.", streamId, projectId); + + try { + CreateStreamRequest request = + CreateStreamRequest.newBuilder() + .setParent(LocationName.format(projectId, location)) + .setStreamId(streamId) + .setStream( + Stream.newBuilder() + .setSourceConfig(sourceConfig) + .setDisplayName(streamId) + .setDestinationConfig(destinationConfig) + .setBackfillAll(Stream.BackfillAllStrategy.getDefaultInstance()) + .build()) + .build(); + + Stream reference = datastreamClient.createStreamAsync(request).get(); + createdStreamIds.add(streamId); + + LOG.info("Successfully created Stream {} in project {}.", streamId, projectId); + return reference; + } catch (ExecutionException | InterruptedException e) { + throw new DatastreamResourceManagerException("Failed to create stream. ", e); + } + } + + public synchronized Stream updateStreamState(String streamId, Stream.State state) { + + LOG.info("Updating {}'s state to {} in project {}.", streamId, state.name(), projectId); + + try { + Stream.Builder streamBuilder = + Stream.newBuilder() + .setName(StreamName.format(projectId, location, streamId)) + .setState(state); + + FieldMask.Builder fieldMaskBuilder = FieldMask.newBuilder().addPaths(FIELD_STATE); + + UpdateStreamRequest request = + UpdateStreamRequest.newBuilder() + .setStream(streamBuilder) + .setUpdateMask(fieldMaskBuilder) + .build(); + + Stream reference = datastreamClient.updateStreamAsync(request).get(); + + LOG.info( + "Successfully updated {}'s state to {} in project {}.", + streamId, + state.name(), + projectId); + return reference; + } catch (InterruptedException | ExecutionException e) { + throw new DatastreamResourceManagerException("Failed to update stream. ", e); + } + } + + public synchronized Stream startStream(String streamId) { + LOG.info("Starting Stream {} in project {}.", streamId, projectId); + return updateStreamState(streamId, Stream.State.RUNNING); + } + + public synchronized Stream pauseStream(String streamId) { + LOG.info("Pausing Stream {} in project {}.", streamId, projectId); + return updateStreamState(streamId, Stream.State.PAUSED); + } + + @Override + public synchronized void cleanupAll() { + LOG.info("Cleaning up Datastream resource manager."); + boolean producedError = false; + + try { + for (String stream : createdStreamIds) { + datastreamClient + .deleteStreamAsync( + DeleteStreamRequest.newBuilder() + .setName(StreamName.format(projectId, location, stream)) + .build()) + .get(); + } + LOG.info("Successfully deleted stream(s). "); + } catch (InterruptedException | ExecutionException e) { + LOG.error("Failed to delete stream(s)."); + producedError = true; + } + + try { + for (String connectionProfile : createdConnectionProfileIds) { + datastreamClient + .deleteConnectionProfileAsync( + DeleteConnectionProfileRequest.newBuilder() + .setName(ConnectionProfileName.format(projectId, location, connectionProfile)) + .build()) + .get(); + } + LOG.info("Successfully deleted connection profile(s). "); + } catch (InterruptedException | ExecutionException e) { + LOG.error("Failed to delete connection profile(s)."); + producedError = true; + } + + try { + datastreamClient.close(); + } catch (Exception e) { + LOG.error("Failed to close datastream client. "); + producedError = true; + } + + if (producedError) { + throw new DatastreamResourceManagerException( + "Failed to delete resources. Check above for errors."); + } + + LOG.info("Successfully cleaned up Datastream resource manager."); + } + + /** Builder for {@link DatastreamResourceManager}. */ + public static final class Builder { + private final String projectId; + private final String location; + private CredentialsProvider credentialsProvider; + + private Builder(String projectId, String location, CredentialsProvider credentialsProvider) { + this.projectId = projectId; + this.location = location; + this.credentialsProvider = credentialsProvider; + } + + public Builder setCredentialsProvider(CredentialsProvider credentialsProvider) { + this.credentialsProvider = credentialsProvider; + return this; + } + + public DatastreamResourceManager build() throws IOException { + return new DatastreamResourceManager(this); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/DatastreamResourceManagerException.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/DatastreamResourceManagerException.java new file mode 100644 index 0000000000000..936f77e017150 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/DatastreamResourceManagerException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.datastream; + +/** Custom exception for {@link DatastreamResourceManager} implementations. */ +public class DatastreamResourceManagerException extends RuntimeException { + + public DatastreamResourceManagerException(String errorMessage, Throwable err) { + super(errorMessage, err); + } + + public DatastreamResourceManagerException(String errorMessage) { + super(errorMessage); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/JDBCSource.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/JDBCSource.java new file mode 100644 index 0000000000000..af8552c8a3119 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/JDBCSource.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.datastream; + +import com.google.protobuf.MessageOrBuilder; +import java.util.List; +import java.util.Map; + +/** Parent class for JDBC Resources required for configuring Datastream. */ +public abstract class JDBCSource { + private final String hostname; + private final String username; + private final String password; + private final int port; + private final Map> allowedTables; + + enum SourceType { + ORACLE, + MYSQL, + POSTGRESQL, + } + + JDBCSource(Builder builder) { + this.hostname = builder.hostname; + this.username = builder.username; + this.password = builder.password; + this.port = builder.port; + this.allowedTables = builder.allowedTables; + } + + public abstract SourceType type(); + + public abstract MessageOrBuilder config(); + + public String hostname() { + return this.hostname; + } + + public String username() { + return this.username; + } + + public String password() { + return this.password; + } + + public int port() { + return this.port; + } + + public Map> allowedTables() { + return this.allowedTables; + } + + public abstract static class Builder { + private String hostname; + private String username; + private String password; + private int port; + private Map> allowedTables; + + public Builder( + String hostname, + String username, + String password, + int port, + Map> allowedTables) { + this.hostname = hostname; + this.username = username; + this.password = password; + this.port = port; + this.allowedTables = allowedTables; + } + + public abstract T build(); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/MySQLSource.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/MySQLSource.java new file mode 100644 index 0000000000000..076d5dd43d63d --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/MySQLSource.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.datastream; + +import com.google.cloud.datastream.v1.MysqlDatabase; +import com.google.cloud.datastream.v1.MysqlRdbms; +import com.google.cloud.datastream.v1.MysqlSourceConfig; +import com.google.cloud.datastream.v1.MysqlTable; +import java.util.List; +import java.util.Map; + +/** + * Client for MySQL resource used by Datastream. + * + *

    Subclass of {@link JDBCSource}. + */ +public class MySQLSource extends JDBCSource { + + MySQLSource(Builder builder) { + super(builder); + } + + @Override + public SourceType type() { + return SourceType.MYSQL; + } + + @Override + public MysqlSourceConfig config() { + MysqlRdbms.Builder mysqlRdmsBuilder = MysqlRdbms.newBuilder(); + for (String db : this.allowedTables().keySet()) { + MysqlDatabase.Builder mysqlDbBuilder = MysqlDatabase.newBuilder().setDatabase(db); + for (String table : this.allowedTables().get(db)) { + mysqlDbBuilder.addMysqlTables(MysqlTable.newBuilder().setTable(table)); + } + mysqlRdmsBuilder.addMysqlDatabases(mysqlDbBuilder); + } + return MysqlSourceConfig.newBuilder().setIncludeObjects(mysqlRdmsBuilder).build(); + } + + public static Builder builder( + String hostname, + String username, + String password, + int port, + Map> allowedTables) { + return new Builder(hostname, username, password, port, allowedTables); + } + + public static class Builder extends JDBCSource.Builder { + public Builder( + String hostname, + String username, + String password, + int port, + Map> allowedTables) { + super(hostname, username, password, port, allowedTables); + } + + @Override + public MySQLSource build() { + return new MySQLSource(this); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/OracleSource.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/OracleSource.java new file mode 100644 index 0000000000000..bb7fd3ce5df89 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/OracleSource.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.datastream; + +import com.google.cloud.datastream.v1.OracleRdbms; +import com.google.cloud.datastream.v1.OracleSchema; +import com.google.cloud.datastream.v1.OracleSourceConfig; +import com.google.cloud.datastream.v1.OracleTable; +import com.google.protobuf.MessageOrBuilder; +import java.util.List; +import java.util.Map; + +/** + * Client for Oracle resource used by Datastream. + * + *

    Subclass of {@link JDBCSource}. + */ +public class OracleSource extends JDBCSource { + + OracleSource(Builder builder) { + super(builder); + } + + public static Builder builder( + String hostname, + String username, + String password, + int port, + Map> allowedTables) { + return new Builder(hostname, username, password, port, allowedTables); + } + + @Override + public SourceType type() { + return SourceType.ORACLE; + } + + @Override + public MessageOrBuilder config() { + OracleRdbms.Builder oracleRdmsBuilder = OracleRdbms.newBuilder(); + for (String schema : this.allowedTables().keySet()) { + OracleSchema.Builder oracleSchemaBuilder = OracleSchema.newBuilder().setSchema(schema); + for (String table : this.allowedTables().get(schema)) { + oracleSchemaBuilder.addOracleTables(OracleTable.newBuilder().setTable(table)); + } + oracleRdmsBuilder.addOracleSchemas(oracleSchemaBuilder); + } + return OracleSourceConfig.newBuilder().setIncludeObjects(oracleRdmsBuilder); + } + + public static class Builder extends JDBCSource.Builder { + public Builder( + String hostname, + String username, + String password, + int port, + Map> allowedTables) { + super(hostname, username, password, port, allowedTables); + } + + @Override + public OracleSource build() { + return new OracleSource(this); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/PostgresqlSource.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/PostgresqlSource.java new file mode 100644 index 0000000000000..3ee52f475a773 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/PostgresqlSource.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.datastream; + +import com.google.cloud.datastream.v1.PostgresqlRdbms; +import com.google.cloud.datastream.v1.PostgresqlSchema; +import com.google.cloud.datastream.v1.PostgresqlSourceConfig; +import com.google.cloud.datastream.v1.PostgresqlTable; +import com.google.protobuf.MessageOrBuilder; +import java.util.List; +import java.util.Map; + +/** + * Client for PostgreSQL resource used by Datastream. + * + *

    Subclass of {@link JDBCSource}. + */ +public class PostgresqlSource extends JDBCSource { + private final String database; + private final String publication; + private final String replicationSlot; + + PostgresqlSource(Builder builder) { + super(builder); + this.database = builder.database; + this.publication = builder.publication; + this.replicationSlot = builder.replicationSlot; + } + + public static Builder builder( + String hostname, + String username, + String password, + int port, + Map> allowedTables, + String database, + String replicationSlot, + String publication) { + return new Builder( + hostname, username, password, port, allowedTables, database, replicationSlot, publication); + } + + @Override + public SourceType type() { + return SourceType.POSTGRESQL; + } + + @Override + public MessageOrBuilder config() { + PostgresqlRdbms.Builder postgresqlRdbmsBuilder = PostgresqlRdbms.newBuilder(); + + for (String schema : this.allowedTables().keySet()) { + PostgresqlSchema.Builder postgresqlSchemaBuilder = + PostgresqlSchema.newBuilder().setSchema(schema); + for (String table : allowedTables().get(schema)) { + postgresqlSchemaBuilder.addPostgresqlTables(PostgresqlTable.newBuilder().setTable(table)); + } + postgresqlRdbmsBuilder.addPostgresqlSchemas(postgresqlSchemaBuilder); + } + return PostgresqlSourceConfig.newBuilder() + .setIncludeObjects(postgresqlRdbmsBuilder) + .setPublication(this.publication) + .setReplicationSlot(this.replicationSlot); + } + + public String database() { + return this.database; + } + + public static class Builder extends JDBCSource.Builder { + private String database; + private String publication; + private String replicationSlot; + + public Builder( + String hostname, + String username, + String password, + int port, + Map> allowedTables, + String database, + String replicationSlot, + String publication) { + super(hostname, username, password, port, allowedTables); + this.database = database; + this.replicationSlot = replicationSlot; + this.publication = publication; + } + + @Override + public PostgresqlSource build() { + return new PostgresqlSource(this); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/package-info.java new file mode 100644 index 0000000000000..d2259dd6161e2 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing Datastream resources within integration tests. */ +package org.apache.beam.it.gcp.datastream; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dlp/DlpResourceManager.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dlp/DlpResourceManager.java new file mode 100644 index 0000000000000..de818a1bbff18 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dlp/DlpResourceManager.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.dlp; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.cloud.dlp.v2.DlpServiceSettings; +import com.google.privacy.dlp.v2.DeidentifyTemplate; +import com.google.privacy.dlp.v2.ProjectName; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.beam.it.common.ResourceManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Client for managing Google Cloud DLP (Data Loss Prevention) resources. */ +public class DlpResourceManager implements ResourceManager { + + private static final Logger LOG = LoggerFactory.getLogger(DlpResourceManager.class); + + private final String project; + private final CredentialsProvider credentialsProvider; + + /** + * Constructs a new DlpResourceManager with the specified project and credentials provider. + * + * @param project the GCP project ID + * @param credentialsProvider the credentials provider for authentication + */ + public DlpResourceManager(String project, CredentialsProvider credentialsProvider) { + this.project = project; + this.credentialsProvider = credentialsProvider; + } + + private final List createdTemplates = new ArrayList<>(); + + /** + * Retrieves a DlpServiceClient with the configured settings. + * + * @return a DlpServiceClient instance + * @throws IOException if an error occurs during client creation + */ + public DlpServiceClient getDlpClient() throws IOException { + DlpServiceSettings.Builder dlpBuilder = DlpServiceSettings.newBuilder(); + if (credentialsProvider != null) { + dlpBuilder = dlpBuilder.setCredentialsProvider(credentialsProvider); + } + return DlpServiceClient.create(dlpBuilder.build()); + } + + /** + * Creates a deidentify template in the specified project. + * + * @param template the deidentify template to create + * @return the created DeidentifyTemplate + * @throws IOException if an error occurs during template creation + */ + public DeidentifyTemplate createDeidentifyTemplate(DeidentifyTemplate template) + throws IOException { + try (DlpServiceClient client = getDlpClient()) { + DeidentifyTemplate deidentifyTemplate = + client.createDeidentifyTemplate(ProjectName.of(this.project), template); + + createdTemplates.add(deidentifyTemplate.getName()); + + return deidentifyTemplate; + } + } + + /** + * Removes a deidentify template by its name. + * + * @param templateName the name of the template to remove + * @throws IOException if an error occurs during template deletion + */ + public void removeDeidentifyTemplate(String templateName) throws IOException { + try (DlpServiceClient client = getDlpClient()) { + client.deleteDeidentifyTemplate(templateName); + } + } + + @Override + public void cleanupAll() { + for (String templateName : createdTemplates) { + try { + removeDeidentifyTemplate(templateName); + } catch (Exception e) { + LOG.error("Error deleting managed template: {}", templateName, e); + } + } + createdTemplates.clear(); + } + + /** + * Creates a new Builder for constructing a DlpResourceManager instance. + * + * @param project the GCP project ID + * @return a new instance of Builder + */ + public static DlpResourceManager.Builder builder( + String project, CredentialsProvider credentialsProvider) { + return new DlpResourceManager.Builder(project, credentialsProvider); + } + + /** A builder class for creating instances of {@link DlpResourceManager}. */ + public static class Builder { + + private final String project; + private CredentialsProvider credentialsProvider; + + /** + * Constructs a new Builder with the specified project. + * + * @param project the GCP project ID + */ + public Builder(String project, CredentialsProvider credentialsProvider) { + this.project = project; + this.credentialsProvider = credentialsProvider; + } + + /** + * Set the credentials provider to use with this client. + * + * @param credentialsProvider the CredentialsProvider instance + * @return the Builder instance + */ + public Builder setCredentialsProvider(CredentialsProvider credentialsProvider) { + this.credentialsProvider = credentialsProvider; + return this; + } + + /** + * Builds a new instance of {@link DlpResourceManager} with the specified project. + * + * @return a new instance of {@link DlpResourceManager} + * @throws IllegalArgumentException if the project is not set + */ + public DlpResourceManager build() { + if (project == null) { + throw new IllegalArgumentException( + "A GCP project must be provided to build a DLP resource manager."); + } + return new DlpResourceManager(project, credentialsProvider); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dlp/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dlp/package-info.java new file mode 100644 index 0000000000000..aa932a7a89a84 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dlp/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package for managing Google Cloud DLP (Data Loss Prevention) resources within integration tests. + */ +package org.apache.beam.it.gcp.dlp; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/kms/KMSClientFactory.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/kms/KMSClientFactory.java new file mode 100644 index 0000000000000..844755292c043 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/kms/KMSClientFactory.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.kms; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.cloud.kms.v1.KeyManagementServiceClient; +import com.google.cloud.kms.v1.KeyManagementServiceSettings; +import java.io.IOException; + +/** KMS Client Factory class. */ +class KMSClientFactory { + + private final CredentialsProvider credentialsProvider; + + KMSClientFactory(CredentialsProvider credentialsProvider) { + this.credentialsProvider = credentialsProvider; + } + + /** + * Returns a KMS client for connection to Google Cloud KMS. + * + * @return KMS client. + */ + KeyManagementServiceClient getKMSClient() { + KeyManagementServiceSettings.Builder settings = KeyManagementServiceSettings.newBuilder(); + + if (credentialsProvider != null) { + settings.setCredentialsProvider(credentialsProvider); + } + + try { + return KeyManagementServiceClient.create(settings.build()); + } catch (IOException e) { + throw new KMSResourceManagerException("Failed to create KMS client.", e); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/kms/KMSResourceManager.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/kms/KMSResourceManager.java new file mode 100644 index 0000000000000..2cad6d0b9faba --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/kms/KMSResourceManager.java @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.kms; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.cloud.kms.v1.CryptoKey; +import com.google.cloud.kms.v1.CryptoKeyName; +import com.google.cloud.kms.v1.CryptoKeyVersion; +import com.google.cloud.kms.v1.CryptoKeyVersionTemplate; +import com.google.cloud.kms.v1.DecryptResponse; +import com.google.cloud.kms.v1.EncryptResponse; +import com.google.cloud.kms.v1.KeyManagementServiceClient; +import com.google.cloud.kms.v1.KeyRing; +import com.google.cloud.kms.v1.KeyRingName; +import com.google.cloud.kms.v1.LocationName; +import com.google.protobuf.ByteString; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Optional; +import java.util.stream.StreamSupport; +import org.apache.beam.it.common.ResourceManager; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Client for managing Cloud KMS resources. + * + *

    The class supports one keyring and multiple crypto keys per keyring object. + * + *

    The class is thread-safe. + */ +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/27438) +}) +public class KMSResourceManager implements ResourceManager { + + private static final Logger LOG = LoggerFactory.getLogger(KMSResourceManager.class); + + private static final String DEFAULT_KMS_REGION = "us-central1"; + + private final String projectId; + private final String region; + private final KMSClientFactory clientFactory; + + private KeyRing keyRing; + + private KMSResourceManager(Builder builder) { + this(new KMSClientFactory(builder.credentialsProvider), builder); + } + + @VisibleForTesting + KMSResourceManager(KMSClientFactory clientFactory, Builder builder) { + this.clientFactory = clientFactory; + this.projectId = builder.projectId; + this.region = builder.region; + this.keyRing = null; + } + + public static KMSResourceManager.Builder builder( + String projectId, CredentialsProvider credentialsProvider) { + return new KMSResourceManager.Builder(projectId, credentialsProvider); + } + + /** + * Creates a keyring in KMS if it does not exist. + * + * @param keyRingId The name of the keyring to create. + */ + private void maybeCreateKeyRing(String keyRingId) { + + try (KeyManagementServiceClient client = clientFactory.getKMSClient()) { + + LocationName locationName = LocationName.of(projectId, region); + KeyRing keyRingToCreate = KeyRing.newBuilder().build(); + + LOG.info("Checking if keyring {} already exists in KMS.", keyRingId); + + String newKeyName = KeyRingName.of(projectId, region, keyRingId).toString(); + Optional existingKeyRing = + StreamSupport.stream(client.listKeyRings(locationName).iterateAll().spliterator(), false) + .filter(kRing -> kRing.getName().equals(newKeyName)) + .findFirst(); + + // Create the keyring if it does not exist, otherwise, return the found keyring. + if (!existingKeyRing.isPresent()) { + LOG.info("Keyring {} does not exist. Creating the keyring in KMS.", keyRingId); + this.keyRing = client.createKeyRing(locationName, keyRingId, keyRingToCreate); + LOG.info("Created keyring {}.", keyRing.getName()); + } else { + LOG.info("Keyring {} already exists. Retrieving the keyring from KMS.", keyRingId); + this.keyRing = existingKeyRing.get(); + LOG.info("Retrieved keyring {}.", keyRing.getName()); + } + } + } + + /** + * Retrieves a KMS crypto key, creating it if it does not exist. If the given keyring also does + * not already exist, it will be created. + * + * @param keyRingId The name of the keyring to insert the key to. + * @param keyName The name of the KMS crypto key to retrieve. + * @return the created CryptoKey. + */ + public synchronized CryptoKey getOrCreateCryptoKey(String keyRingId, String keyName) { + + // Get the keyring, creating it if it does not already exist + if (keyRing == null) { + maybeCreateKeyRing(keyRingId); + } + + try (KeyManagementServiceClient client = clientFactory.getKMSClient()) { + + // Build the symmetric key to create. + CryptoKey keyToCreate = + CryptoKey.newBuilder() + .setPurpose(CryptoKey.CryptoKeyPurpose.ENCRYPT_DECRYPT) + .setVersionTemplate( + CryptoKeyVersionTemplate.newBuilder() + .setAlgorithm( + CryptoKeyVersion.CryptoKeyVersionAlgorithm.GOOGLE_SYMMETRIC_ENCRYPTION)) + .build(); + + LOG.info("Checking if symmetric key {} already exists in KMS.", keyName); + + // Loop through the existing keys in the given keyring to see if the + // key already exists. + String newKeyName = CryptoKeyName.of(projectId, region, keyRingId, keyName).toString(); + Optional existingKey = + StreamSupport.stream( + client.listCryptoKeys(keyRing.getName()).iterateAll().spliterator(), false) + .filter(kRing -> kRing.getName().equals(newKeyName)) + .findFirst(); + + // Create the symmetric key if it does not exist, otherwise, return the found key. + CryptoKey cryptoKey; + if (!existingKey.isPresent()) { + LOG.info("Symmetric key {} does not exist. Creating the key in KMS.", keyName); + cryptoKey = client.createCryptoKey(keyRing.getName(), keyName, keyToCreate); + LOG.info("Created symmetric key {}.", cryptoKey.getName()); + } else { + LOG.info("Symmetric key {} already exists. Retrieving the key from KMS.", keyName); + cryptoKey = existingKey.get(); + LOG.info("Retrieved symmetric key {}.", cryptoKey.getName()); + } + + return cryptoKey; + } + } + + /** + * Encrypt the given message using the crypto key specified. The given crypto key should exist + * within the given keyring. + * + *

    The given message should be in UTF-8 String format. + * + *

    The resulting ciphertext is encoded in Base64 String format. + * + * @param keyRingId The name of the keyring that contains the given key. + * @param keyId The name of the key to use for encryption. + * @param message The message to encrypt. + * @return The ciphertext of the encrypted message. + */ + public synchronized String encrypt(String keyRingId, String keyId, String message) { + + CryptoKeyName keyName = CryptoKeyName.of(projectId, region, keyRingId, keyId); + LOG.info("Encrypting given message using key {}.", keyName.toString()); + + try (KeyManagementServiceClient client = clientFactory.getKMSClient()) { + + EncryptResponse response = client.encrypt(keyName, ByteString.copyFromUtf8(message)); + + LOG.info("Successfully encrypted message."); + return new String( + Base64.getEncoder().encode(response.getCiphertext().toByteArray()), + StandardCharsets.UTF_8); + } + } + + /** + * Decrypt the given ciphertext using the crypto key specified. The given crypto key should exist + * within the given keyring. + * + *

    The given ciphertext should be in Base64 String format. + * + *

    The resulting decrypted message is encoded in UTF-8 String format. + * + * @param keyRingId The name of the keyring that contains the given key. + * @param keyId The name of the key to use for decryption. + * @param ciphertext The ciphertext to decrypt. + * @return The decrypted message. + */ + public synchronized String decrypt(String keyRingId, String keyId, String ciphertext) { + + CryptoKeyName keyName = CryptoKeyName.of(projectId, region, keyRingId, keyId); + LOG.info("Decrypting given ciphertext using key {}.", keyName.toString()); + + try (KeyManagementServiceClient client = clientFactory.getKMSClient()) { + + DecryptResponse response = + client.decrypt( + keyName, + ByteString.copyFrom( + Base64.getDecoder().decode(ciphertext.getBytes(StandardCharsets.UTF_8)))); + + LOG.info("Successfully decrypted ciphertext."); + return response.getPlaintext().toStringUtf8(); + } + } + + @Override + public void cleanupAll() { + LOG.info("Not cleaning up KMS keys."); + } + + /** Builder for {@link KMSResourceManager}. */ + public static final class Builder { + private final String projectId; + private CredentialsProvider credentialsProvider; + private String region; + + private Builder(String projectId, CredentialsProvider credentialsProvider) { + this.projectId = projectId; + this.region = DEFAULT_KMS_REGION; + this.credentialsProvider = credentialsProvider; + } + + /** + * Set the GCP credentials provider to connect to the project defined in the builder. + * + * @param credentialsProvider The GCP CredentialsProvider. + * @return this builder with the CredentialsProvider set. + */ + public Builder setCredentialsProvider(CredentialsProvider credentialsProvider) { + this.credentialsProvider = credentialsProvider; + return this; + } + + /** + * Set the GCP region that the keys should be configured to write to. Defaults to {@value + * #DEFAULT_KMS_REGION}. + * + * @param region The region to use. + * @return this builder with the GCS region set. + */ + public Builder setRegion(String region) { + this.region = region; + return this; + } + + public KMSResourceManager build() { + return new KMSResourceManager(this); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/kms/KMSResourceManagerException.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/kms/KMSResourceManagerException.java new file mode 100644 index 0000000000000..436240ffd31f0 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/kms/KMSResourceManagerException.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.kms; + +/** Custom exception for {@link KMSResourceManager} implementations. */ +public class KMSResourceManagerException extends RuntimeException { + + public KMSResourceManagerException(String errorMessage, Throwable err) { + super(errorMessage, err); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/kms/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/kms/package-info.java new file mode 100644 index 0000000000000..8065def21eea5 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/kms/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing KMS resources within integration tests. */ +package org.apache.beam.it.gcp.kms; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/monitoring/MonitoringClient.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/monitoring/MonitoringClient.java new file mode 100644 index 0000000000000..06591ea4fe0ae --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/monitoring/MonitoringClient.java @@ -0,0 +1,467 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.monitoring; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.cloud.monitoring.v3.MetricServiceClient; +import com.google.cloud.monitoring.v3.MetricServiceClient.ListTimeSeriesPagedResponse; +import com.google.cloud.monitoring.v3.MetricServiceSettings; +import com.google.monitoring.v3.Aggregation; +import com.google.monitoring.v3.Aggregation.Aligner; +import com.google.monitoring.v3.Aggregation.Reducer; +import com.google.monitoring.v3.ListTimeSeriesRequest; +import com.google.monitoring.v3.Point; +import com.google.monitoring.v3.ProjectName; +import com.google.monitoring.v3.TimeInterval; +import com.google.monitoring.v3.TimeSeries; +import com.google.protobuf.Duration; +import com.google.protobuf.util.Timestamps; +import java.io.IOException; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.beam.it.common.PipelineLauncher.LaunchInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Client for working with Google Cloud Monitoring. */ +public final class MonitoringClient { + private static final Logger LOG = LoggerFactory.getLogger(MonitoringClient.class); + private final MetricServiceClient metricServiceClient; + + private MonitoringClient(Builder builder) throws IOException { + MetricServiceSettings metricServiceSettings = + MetricServiceSettings.newBuilder() + .setCredentialsProvider(builder.getCredentialsProvider()) + .build(); + this.metricServiceClient = MetricServiceClient.create(metricServiceSettings); + } + + private MonitoringClient(MetricServiceClient metricServiceClient) { + this.metricServiceClient = metricServiceClient; + } + + public static MonitoringClient withMonitoringClient(MetricServiceClient metricServiceClient) { + return new MonitoringClient(metricServiceClient); + } + + public static Builder builder(CredentialsProvider credentialsProvider) { + return new Builder(credentialsProvider); + } + + /** + * Lists time series that match a filter. + * + * @param request time series request to execute + * @return list of Double values of time series + */ + public List listTimeSeriesAsDouble(ListTimeSeriesRequest request) { + return extractValuesFromTimeSeriesAsDouble(metricServiceClient.listTimeSeries(request)); + } + + /** + * Lists time series that match a filter. + * + * @param request time series request to execute + * @return list of Long values of time series + */ + public List listTimeSeriesAsLong(ListTimeSeriesRequest request) { + return extractValuesFromTimeSeriesAsLong(metricServiceClient.listTimeSeries(request)); + } + + /** + * Gets the number of undelivered messages in a given Pub/Sub subscription. + * + * @param project the project that the job is running under + * @param subscriptionName name of the Pub/Sub subscription + * @return number of undelivered messages in the subscription + */ + public @Nullable Long getNumMessagesInSubscription(String project, String subscriptionName) { + LOG.info( + "Getting number of messages in subscription for {} under {}", subscriptionName, project); + String filter = + String.format( + "metric.type = \"pubsub.googleapis.com/subscription/num_undelivered_messages\" " + + "AND resource.labels.subscription_id = \"%s\"", + subscriptionName); + // only consider the time interval of last 60 seconds to get the most recent value + TimeInterval timeInterval = + TimeInterval.newBuilder() + .setStartTime(Timestamps.fromMillis(System.currentTimeMillis() - 60000)) + .setEndTime(Timestamps.fromMillis(System.currentTimeMillis())) + .build(); + Aggregation aggregation = + Aggregation.newBuilder() + .setAlignmentPeriod(Duration.newBuilder().setSeconds(60).build()) + .setPerSeriesAligner(Aligner.ALIGN_MAX) + .build(); + ListTimeSeriesRequest request = + ListTimeSeriesRequest.newBuilder() + .setName(ProjectName.of(project).toString()) + .setFilter(filter) + .setInterval(timeInterval) + .setAggregation(aggregation) + .build(); + List timeSeries = listTimeSeriesAsLong(request); + if (timeSeries.isEmpty()) { + LOG.warn( + "No monitoring data found. Unable to get number of messages in {}.", subscriptionName); + return null; + } + // get the last recorded value of number of messages in the subscription (there should be only 1 + // value) + return timeSeries.get(0); + } + + /** + * Gets the CPU Utilization time series data for a given Job. + * + * @param project the project that the job is running under + * @param jobId dataflow job id + * @param timeInterval interval for the monitoring query + * @return CPU Utilization time series data for the given job. + */ + public @Nullable List getCpuUtilization( + String project, String jobId, TimeInterval timeInterval) { + LOG.info("Getting CPU utilization for {} under {}", jobId, project); + String filter = + String.format( + "metric.type = \"compute.googleapis.com/instance/cpu/utilization\" " + + "AND resource.labels.project_id = \"%s\" " + + "AND metadata.user_labels.dataflow_job_id = \"%s\"", + project, jobId); + Aggregation aggregation = + Aggregation.newBuilder() + .setAlignmentPeriod(Duration.newBuilder().setSeconds(60).build()) + .setPerSeriesAligner(Aggregation.Aligner.ALIGN_MEAN) + .setCrossSeriesReducer(Aggregation.Reducer.REDUCE_MEAN) + .addGroupByFields("resource.instance_id") + .build(); + ListTimeSeriesRequest request = + ListTimeSeriesRequest.newBuilder() + .setName(ProjectName.of(project).toString()) + .setFilter(filter) + .setInterval(timeInterval) + .setAggregation(aggregation) + .build(); + List timeSeries = listTimeSeriesAsDouble(request); + if (timeSeries.isEmpty()) { + LOG.warn("No monitoring data found. Unable to get CPU utilization information."); + return null; + } + return timeSeries; + } + + /** + * Gets the System Latency time series data for a given Job. + * + * @param project the project that the job is running under + * @param jobId dataflow job id + * @param timeInterval interval for the monitoring query + * @return System Latency time series data for the given job. + */ + public @Nullable List getSystemLatency( + String project, String jobId, TimeInterval timeInterval) { + LOG.info("Getting system latency for {} under {}", jobId, project); + String filter = + String.format( + "metric.type = \"dataflow.googleapis.com/job/per_stage_system_lag\" " + + "AND metric.labels.job_id = \"%s\"", + jobId); + Aggregation aggregation = + Aggregation.newBuilder() + .setAlignmentPeriod(Duration.newBuilder().setSeconds(60).build()) + .setPerSeriesAligner(Aggregation.Aligner.ALIGN_MEAN) + .setCrossSeriesReducer(Reducer.REDUCE_MAX) + .build(); + ListTimeSeriesRequest request = + ListTimeSeriesRequest.newBuilder() + .setName(ProjectName.of(project).toString()) + .setFilter(filter) + .setInterval(timeInterval) + .setAggregation(aggregation) + .build(); + List timeSeries = listTimeSeriesAsDouble(request); + if (timeSeries.isEmpty()) { + LOG.warn("No monitoring data found. Unable to get System Latency information."); + return null; + } + return timeSeries; + } + + /** + * Gets the Data freshness time series data for a given Job. + * + * @param project the project that the job is running under + * @param jobId dataflow job id + * @param timeInterval interval for the monitoring query + * @return Data freshness time series data for the given job. + */ + public @Nullable List getDataFreshness( + String project, String jobId, TimeInterval timeInterval) { + LOG.info("Getting data freshness for {} under {}", jobId, project); + String filter = + String.format( + "metric.type = \"dataflow.googleapis.com/job/per_stage_data_watermark_age\" " + + "AND metric.labels.job_id = \"%s\"", + jobId); + Aggregation aggregation = + Aggregation.newBuilder() + .setAlignmentPeriod(Duration.newBuilder().setSeconds(60).build()) + .setPerSeriesAligner(Aggregation.Aligner.ALIGN_MEAN) + .setCrossSeriesReducer(Reducer.REDUCE_MAX) + .build(); + ListTimeSeriesRequest request = + ListTimeSeriesRequest.newBuilder() + .setName(ProjectName.of(project).toString()) + .setFilter(filter) + .setInterval(timeInterval) + .setAggregation(aggregation) + .build(); + List timeSeries = listTimeSeriesAsDouble(request); + if (timeSeries.isEmpty()) { + LOG.warn("No monitoring data found. Unable to get Data freshness information."); + return null; + } + return timeSeries; + } + + /** + * Gets the output throughput in bytes per second from a particular PCollection during job run + * interval. + * + * @param project the project that the job is running under + * @param jobId dataflow job id + * @param pcollection name of the pcollection + * @param timeInterval interval for the monitoring query + * @return output throughput from a particular PCollection during job run interval + */ + public @Nullable List getThroughputBytesPerSecond( + String project, String jobId, String pcollection, TimeInterval timeInterval) { + if (pcollection == null) { + LOG.warn("Output PCollection name not provided. Unable to calculate throughput."); + return null; + } + LOG.info("Getting throughput (bytes/sec) for {} under {}", jobId, project); + String filter = + String.format( + "metric.type = \"dataflow.googleapis.com/job/estimated_bytes_produced_count\" " + + "AND metric.labels.job_id=\"%s\" " + + "AND metric.labels.pcollection=\"%s\" ", + jobId, pcollection); + Aggregation aggregation = + Aggregation.newBuilder() + .setAlignmentPeriod(Duration.newBuilder().setSeconds(60).build()) + .setPerSeriesAligner(Aggregation.Aligner.ALIGN_RATE) + .build(); + ListTimeSeriesRequest request = + ListTimeSeriesRequest.newBuilder() + .setName(ProjectName.of(project).toString()) + .setFilter(filter) + .setInterval(timeInterval) + .setAggregation(aggregation) + .build(); + List timeSeries = listTimeSeriesAsDouble(request); + if (timeSeries.isEmpty()) { + LOG.warn("No monitoring data found. Unable to get throughput information."); + return null; + } + return timeSeries; + } + + /** + * Gets the output throughput in elements per second from a particular PCollection during job run + * interval. + * + * @param project the project that the job is running under + * @param jobId dataflow job id + * @param pcollection name of the pcollection + * @param timeInterval interval for the monitoring query + * @return output throughput from a particular PCollection during job run interval + */ + public @Nullable List getThroughputElementsPerSecond( + String project, String jobId, String pcollection, TimeInterval timeInterval) { + if (pcollection == null) { + LOG.warn("Output PCollection name not provided. Unable to calculate throughput."); + return null; + } + LOG.info("Getting throughput (elements/sec) for {} under {}", jobId, project); + String filter = + String.format( + "metric.type = \"dataflow.googleapis.com/job/elements_produced_count\" " + + "AND metric.labels.job_id=\"%s\" " + + "AND metric.labels.pcollection=\"%s\" ", + jobId, pcollection); + Aggregation aggregation = + Aggregation.newBuilder() + .setAlignmentPeriod(Duration.newBuilder().setSeconds(60).build()) + .setPerSeriesAligner(Aggregation.Aligner.ALIGN_RATE) + .build(); + ListTimeSeriesRequest request = + ListTimeSeriesRequest.newBuilder() + .setName(ProjectName.of(project).toString()) + .setFilter(filter) + .setInterval(timeInterval) + .setAggregation(aggregation) + .build(); + List timeSeries = listTimeSeriesAsDouble(request); + if (timeSeries.isEmpty()) { + LOG.warn("No monitoring data found. Unable to get throughput information."); + return null; + } + return timeSeries; + } + + /** + * Get elapsed time for a job. + * + * @param project the project that the job is running under + * @param launchInfo information about the job + * @return elapsed time + * @throws ParseException if timestamp is inaccurate + */ + public @Nullable Double getElapsedTime(String project, LaunchInfo launchInfo) + throws ParseException { + LOG.info("Getting elapsed time for {} under {}", launchInfo.jobId(), project); + String filter = + String.format( + "metric.type = \"dataflow.googleapis.com/job/elapsed_time\" " + + "AND metric.labels.job_id=\"%s\" ", + launchInfo.jobId()); + TimeInterval timeInterval = getTimeInterval(launchInfo.createTime()); + Aggregation aggregation = + Aggregation.newBuilder() + .setAlignmentPeriod(Duration.newBuilder().setSeconds(60).build()) + .setPerSeriesAligner(Aligner.ALIGN_MEAN) + .build(); + ListTimeSeriesRequest request = + ListTimeSeriesRequest.newBuilder() + .setName(ProjectName.of(project).toString()) + .setFilter(filter) + .setInterval(timeInterval) + .setAggregation(aggregation) + .build(); + List timeSeries = listTimeSeriesAsDouble(request); + if (timeSeries.isEmpty()) { + LOG.warn("No monitoring data found. Unable to get elapsed time information."); + return null; + } + // getting max since this is a gauge metric + return Collections.max(timeSeries); + } + + /** + * Get data processed for a job. + * + * @param project the project that the job is running under + * @param launchInfo information about the job + * @return data processed + * @throws ParseException if timestamp is inaccurate + */ + public @Nullable Double getDataProcessed( + String project, LaunchInfo launchInfo, String pCollection) throws ParseException { + if (pCollection == null) { + LOG.warn("PCollection name not provided. Unable to calculate data processed."); + return null; + } + LOG.info("Getting data processed for {} under {}", launchInfo.jobId(), project); + String filter = + String.format( + "metric.type = \"dataflow.googleapis.com/job/estimated_byte_count\" " + + "AND metric.labels.job_id=\"%s\" " + + "AND metric.labels.pcollection=\"%s\" ", + launchInfo.jobId(), pCollection); + TimeInterval timeInterval = getTimeInterval(launchInfo.createTime()); + Aggregation aggregation = + Aggregation.newBuilder() + .setAlignmentPeriod(Duration.newBuilder().setSeconds(60).build()) + .setPerSeriesAligner(Aligner.ALIGN_MEAN) + .build(); + ListTimeSeriesRequest request = + ListTimeSeriesRequest.newBuilder() + .setName(ProjectName.of(project).toString()) + .setFilter(filter) + .setInterval(timeInterval) + .setAggregation(aggregation) + .build(); + List timeSeries = listTimeSeriesAsDouble(request); + if (timeSeries.isEmpty()) { + LOG.warn("No monitoring data found. Unable to get data processed information."); + return null; + } + // getting max since this is a gauge metric + return Collections.max(timeSeries); + } + + public synchronized void cleanupAll() { + LOG.info("Attempting to cleanup monitoring client."); + metricServiceClient.close(); + LOG.info("Monitoring client successfully cleaned up."); + } + + private TimeInterval getTimeInterval(String startTime) throws ParseException { + return TimeInterval.newBuilder() + .setStartTime(Timestamps.parse(startTime)) + .setEndTime(Timestamps.fromMillis(System.currentTimeMillis())) + .build(); + } + + private List extractValuesFromTimeSeriesAsDouble(ListTimeSeriesPagedResponse response) { + List values = new ArrayList<>(); + for (TimeSeries ts : response.iterateAll()) { + for (Point point : ts.getPointsList()) { + values.add(point.getValue().getDoubleValue()); + } + } + return values; + } + + private List extractValuesFromTimeSeriesAsLong(ListTimeSeriesPagedResponse response) { + List values = new ArrayList<>(); + for (TimeSeries ts : response.iterateAll()) { + for (Point point : ts.getPointsList()) { + values.add(point.getValue().getInt64Value()); + } + } + return values; + } + + /** Builder for {@link MonitoringClient}. */ + public static final class Builder { + private CredentialsProvider credentialsProvider; + + private Builder(CredentialsProvider credentialsProvider) { + this.credentialsProvider = credentialsProvider; + } + + public CredentialsProvider getCredentialsProvider() { + return credentialsProvider; + } + + public Builder setCredentialsProvider(CredentialsProvider value) { + credentialsProvider = value; + return this; + } + + public MonitoringClient build() throws IOException { + return new MonitoringClient(this); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/monitoring/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/monitoring/package-info.java new file mode 100644 index 0000000000000..9cd79ea9cf12d --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/monitoring/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for querying metrics from cloud monitoring. */ +package org.apache.beam.it.gcp.monitoring; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/package-info.java new file mode 100644 index 0000000000000..eff1a6be7d1c4 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing Google Cloud resources within integration tests. */ +package org.apache.beam.it.gcp; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/PubsubPublisherFactory.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/PubsubPublisherFactory.java new file mode 100644 index 0000000000000..111ac64313bcc --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/PubsubPublisherFactory.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.pubsub; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.cloud.pubsub.v1.Publisher; +import com.google.pubsub.v1.TopicName; +import java.io.IOException; + +/** + * Client for building Pub/Sub publishers. + * + *

    The class provides an interaction with the real Pub/Sub client, with operations related to + * creating a Pub/Sub Publisher. + */ +public class PubsubPublisherFactory { + + private final CredentialsProvider credentialsProvider; + + PubsubPublisherFactory(CredentialsProvider credentialsProvider) { + this.credentialsProvider = credentialsProvider; + } + + /** Create a {@link Publisher} instance for the given topic reference. */ + public Publisher createPublisher(TopicName topic) { + try { + return Publisher.newBuilder(topic).setCredentialsProvider(credentialsProvider).build(); + } catch (IOException e) { + throw new PubsubResourceManagerException("Error creating publisher for topic", e); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/PubsubResourceManager.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/PubsubResourceManager.java new file mode 100644 index 0000000000000..738620c15b7ea --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/PubsubResourceManager.java @@ -0,0 +1,411 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.pubsub; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.api.gax.rpc.DeadlineExceededException; +import com.google.cloud.pubsub.v1.Publisher; +import com.google.cloud.pubsub.v1.SchemaServiceClient; +import com.google.cloud.pubsub.v1.SchemaServiceSettings; +import com.google.cloud.pubsub.v1.SubscriptionAdminClient; +import com.google.cloud.pubsub.v1.SubscriptionAdminSettings; +import com.google.cloud.pubsub.v1.TopicAdminClient; +import com.google.cloud.pubsub.v1.TopicAdminSettings; +import com.google.protobuf.ByteString; +import com.google.protobuf.FieldMask; +import com.google.pubsub.v1.Encoding; +import com.google.pubsub.v1.ProjectName; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.v1.PullResponse; +import com.google.pubsub.v1.PushConfig; +import com.google.pubsub.v1.Schema; +import com.google.pubsub.v1.SchemaName; +import com.google.pubsub.v1.SchemaSettings; +import com.google.pubsub.v1.Subscription; +import com.google.pubsub.v1.SubscriptionName; +import com.google.pubsub.v1.Topic; +import com.google.pubsub.v1.TopicName; +import com.google.pubsub.v1.UpdateTopicRequest; +import dev.failsafe.Failsafe; +import dev.failsafe.RetryPolicy; +import java.io.IOException; +import java.time.Duration; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.apache.beam.it.common.ResourceManager; +import org.apache.beam.it.common.utils.ExceptionUtils; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Client for managing Pub/Sub resources. + * + *

    The class provides an interaction with the real Pub/Sub client, with operations related to + * management of topics and subscriptions. + */ +public final class PubsubResourceManager implements ResourceManager { + + private static final Logger LOG = LoggerFactory.getLogger(PubsubResourceManager.class); + + private static final int DEFAULT_ACK_DEADLINE_SECONDS = 600; + private static final String RESOURCE_NAME_SEPARATOR = "-"; + + // Retry settings for client operations + private static final int FAILSAFE_MAX_RETRIES = 5; + private static final Duration FAILSAFE_RETRY_DELAY = Duration.ofSeconds(10); + private static final Duration FAILSAFE_RETRY_MAX_DELAY = Duration.ofSeconds(60); + private static final double FAILSAFE_RETRY_JITTER = 0.1; + + private final String testId; + private final String projectId; + private final PubsubPublisherFactory publisherFactory; + private final TopicAdminClient topicAdminClient; + private final SubscriptionAdminClient subscriptionAdminClient; + + private final SchemaServiceClient schemaServiceClient; + + private final Set createdTopics; + private final Set createdSubscriptions; + + private final Set createdSchemas; + + private PubsubResourceManager(Builder builder) throws IOException { + this( + builder.testName, + builder.projectId, + new PubsubPublisherFactory(builder.credentialsProvider), + TopicAdminClient.create( + TopicAdminSettings.newBuilder() + .setCredentialsProvider(builder.credentialsProvider) + .build()), + SubscriptionAdminClient.create( + SubscriptionAdminSettings.newBuilder() + .setCredentialsProvider(builder.credentialsProvider) + .build()), + SchemaServiceClient.create( + SchemaServiceSettings.newBuilder() + .setCredentialsProvider(builder.credentialsProvider) + .build())); + } + + @VisibleForTesting + PubsubResourceManager( + String testName, + String projectId, + PubsubPublisherFactory publisherFactory, + TopicAdminClient topicAdminClient, + SubscriptionAdminClient subscriptionAdminClient, + SchemaServiceClient schemaServiceClient) { + this.projectId = projectId; + this.testId = PubsubUtils.createTestId(testName); + this.publisherFactory = publisherFactory; + this.topicAdminClient = topicAdminClient; + this.subscriptionAdminClient = subscriptionAdminClient; + this.createdTopics = Collections.synchronizedSet(new HashSet<>()); + this.createdSubscriptions = Collections.synchronizedSet(new HashSet<>()); + this.createdSchemas = Collections.synchronizedSet(new HashSet<>()); + this.schemaServiceClient = schemaServiceClient; + } + + public static Builder builder( + String testName, String projectId, CredentialsProvider credentialsProvider) { + checkArgument(!Strings.isNullOrEmpty(testName), "testName can not be null or empty"); + checkArgument(!projectId.isEmpty(), "projectId can not be empty"); + return new Builder(testName, projectId, credentialsProvider); + } + + /** + * Return the test ID this Resource Manager uses to manage Pub/Sub instances. + * + * @return the test ID. + */ + public String getTestId() { + return testId; + } + + /** + * Creates a topic with the given name on Pub/Sub. + * + * @param topicName Topic name to create. The underlying implementation may not use the topic name + * directly, and can add a prefix or a suffix to identify specific executions. + * @return The instance of the TopicName that was just created. + */ + public TopicName createTopic(String topicName) { + checkArgument(!topicName.isEmpty(), "topicName can not be empty"); + checkIsUsable(); + + TopicName name = getTopicName(topicName); + return createTopicInternal(name); + } + + /** + * Creates a topic with the given name on Pub/Sub. + * + * @param topicName Topic name to create. The underlying implementation will use the topic name + * directly. + * @return The instance of the TopicName that was just created. + */ + public TopicName createTopicWithoutPrefix(String topicName) { + checkArgument(!topicName.isEmpty(), "topicName can not be empty"); + checkIsUsable(); + + TopicName name = TopicName.of(projectId, topicName); + return createTopicInternal(name); + } + + /** + * Creates a subscription at the specific topic, with a given name. + * + * @param topicName Topic Name reference to add the subscription. + * @param subscriptionName Name of the subscription to use. Note that the underlying + * implementation may not use the subscription name literally, and can use a prefix or a + * suffix to identify specific executions. + * @return The instance of the SubscriptionName that was just created. + */ + public SubscriptionName createSubscription(TopicName topicName, String subscriptionName) { + checkArgument(!subscriptionName.isEmpty(), "subscriptionName can not be empty"); + checkIsUsable(); + + if (!createdTopics.contains(topicName)) { + throw new IllegalArgumentException( + "Can not create a subscription for a topic not managed by this instance."); + } + + LOG.info("Creating subscription '{}' for topic '{}'", subscriptionName, topicName); + + Subscription subscription = + Failsafe.with(retryOnDeadlineExceeded()) + .get( + () -> + subscriptionAdminClient.createSubscription( + getSubscriptionName(subscriptionName), + topicName, + PushConfig.getDefaultInstance(), + DEFAULT_ACK_DEADLINE_SECONDS)); + SubscriptionName reference = PubsubUtils.toSubscriptionName(subscription); + createdSubscriptions.add(getSubscriptionName(subscriptionName)); + + LOG.info( + "Subscription '{}' for topic '{}' was created successfully!", + subscription.getName(), + topicName); + + return reference; + } + + /** + * Publishes a message with the given data to the publisher context's topic. + * + * @param topic Reference to the topic to send the message. + * @param attributes Attributes to send with the message. + * @param data Byte data to send. + * @return The message id that was generated. + */ + public String publish(TopicName topic, Map attributes, ByteString data) + throws PubsubResourceManagerException { + checkIsUsable(); + + if (!createdTopics.contains(topic)) { + throw new IllegalArgumentException( + "Can not publish to a topic not managed by this instance."); + } + + LOG.info("Publishing message with {} bytes to topic '{}'", data.size(), topic); + + PubsubMessage pubsubMessage = + PubsubMessage.newBuilder().putAllAttributes(attributes).setData(data).build(); + + try { + Publisher publisher = publisherFactory.createPublisher(topic); + String messageId = publisher.publish(pubsubMessage).get(); + LOG.info("Message published with id '{}'", messageId); + publisher.shutdown(); + return messageId; + } catch (Exception e) { + throw new PubsubResourceManagerException("Error publishing message to Pubsub", e); + } + } + + /** + * Pulls messages from the given subscription. + * + * @param subscriptionName Name of the subscription to use. + * @return The message id that was generated. + */ + public PullResponse pull(SubscriptionName subscriptionName, int maxMessages) { + LOG.info("Pulling messages from subscription '{}'", subscriptionName); + PullResponse response = subscriptionAdminClient.pull(subscriptionName, maxMessages); + LOG.info( + "Received {} messages from subscription '{}'", + response.getReceivedMessagesCount(), + subscriptionName); + return response; + } + + /** + * Registers a new schema of the given type and definition, then assigns it to the specified + * topic. + * + * @param schemaType the type of schema to create (e.g. AVRO, PROTOBUF) + * @param schemaDefinition the definition of the schema to create in AVRO or Protobuf syntax. + * @param dataEncoding the encoding of the data in pubsub (e.g. BINARY_ENCODING, JSON) + * @param schemaTopic the name of the topic to which assign the schema. + * @return the name of the newly created schema + */ + public String createSchema( + Schema.Type schemaType, + String schemaDefinition, + Encoding dataEncoding, + TopicName schemaTopic) { + Schema schema = + schemaServiceClient.createSchema( + ProjectName.newBuilder().setProject(projectId).build(), + Schema.newBuilder().setType(schemaType).setDefinition(schemaDefinition).build(), + "schema-" + testId + "-" + schemaTopic.getTopic()); + createdSchemas.add(SchemaName.parse(schema.getName())); + topicAdminClient.updateTopic( + UpdateTopicRequest.newBuilder() + .setUpdateMask(FieldMask.newBuilder().addPaths("schema_settings")) + .setTopic( + Topic.newBuilder() + .setName(schemaTopic.toString()) + .setSchemaSettings( + SchemaSettings.newBuilder() + .setSchema(schema.getName()) + .setEncoding(dataEncoding) + .build()) + .build()) + .build()); + return schema.getName(); + } + + /** Delete any topics or subscriptions created by this manager. */ + @Override + public synchronized void cleanupAll() { + // Ignore call if it was cleaned up before + if (isNotUsable()) { + return; + } + + LOG.info("Attempting to cleanup Pub/Sub resource manager."); + + try { + for (SubscriptionName subscription : createdSubscriptions) { + LOG.info("Deleting subscription '{}'", subscription); + Failsafe.with(retryOnDeadlineExceeded()) + .run(() -> subscriptionAdminClient.deleteSubscription(subscription)); + } + + for (TopicName topic : createdTopics) { + LOG.info("Deleting topic '{}'", topic); + Failsafe.with(retryOnDeadlineExceeded()).run(() -> topicAdminClient.deleteTopic(topic)); + } + + for (SchemaName schemaName : createdSchemas) { + LOG.info("Deleting schema '{}'", schemaName); + Failsafe.with(retryOnDeadlineExceeded()) + .run(() -> schemaServiceClient.deleteSchema(schemaName)); + } + } finally { + subscriptionAdminClient.close(); + topicAdminClient.close(); + schemaServiceClient.close(); + } + + LOG.info("Manager successfully cleaned up."); + } + + SubscriptionName getSubscriptionName(String subscriptionName) { + return SubscriptionName.of(projectId, testId + RESOURCE_NAME_SEPARATOR + subscriptionName); + } + + TopicName getTopicName(String topicName) { + return TopicName.of(projectId, testId + RESOURCE_NAME_SEPARATOR + topicName); + } + + /** + * Check if the clients started by this instance are still usable, and throwing {@link + * IllegalStateException} otherwise. + */ + private void checkIsUsable() throws IllegalStateException { + if (isNotUsable()) { + throw new IllegalStateException("Manager has cleaned up resources and is unusable."); + } + } + + /** Internal method to create a Pub/Sub topic used by this resource manager. */ + private TopicName createTopicInternal(TopicName topicName) { + LOG.info("Creating topic '{}'...", topicName.toString()); + + Topic topic = + Failsafe.with(retryOnDeadlineExceeded()).get(() -> topicAdminClient.createTopic(topicName)); + TopicName reference = PubsubUtils.toTopicName(topic); + createdTopics.add(reference); + + LOG.info("Topic '{}' was created successfully!", reference); + + return reference; + } + + private boolean isNotUsable() { + return topicAdminClient.isShutdown() || subscriptionAdminClient.isShutdown(); + } + + private static RetryPolicy retryOnDeadlineExceeded() { + return RetryPolicy.builder() + .handleIf( + exception -> ExceptionUtils.containsType(exception, DeadlineExceededException.class)) + .withMaxRetries(FAILSAFE_MAX_RETRIES) + .withBackoff(FAILSAFE_RETRY_DELAY, FAILSAFE_RETRY_MAX_DELAY) + .withJitter(FAILSAFE_RETRY_JITTER) + .build(); + } + + /** Builder for {@link PubsubResourceManager}. */ + public static final class Builder { + + private final String projectId; + private final String testName; + private CredentialsProvider credentialsProvider; + + private Builder(String testName, String projectId, CredentialsProvider credentialsProvider) { + this.testName = testName; + this.projectId = projectId; + this.credentialsProvider = credentialsProvider; + } + + public Builder credentialsProvider(CredentialsProvider credentialsProvider) { + this.credentialsProvider = credentialsProvider; + return this; + } + + public PubsubResourceManager build() throws IOException { + if (credentialsProvider == null) { + throw new IllegalArgumentException( + "Unable to find credentials. Please provide credentials to authenticate to GCP"); + } + return new PubsubResourceManager(this); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/PubsubResourceManagerException.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/PubsubResourceManagerException.java new file mode 100644 index 0000000000000..42133429fdc22 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/PubsubResourceManagerException.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.pubsub; + +/** Custom exception for {@link PubsubResourceManager} implementations. */ +class PubsubResourceManagerException extends RuntimeException { + + PubsubResourceManagerException(String errorMessage, Throwable err) { + super(errorMessage, err); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/PubsubUtils.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/PubsubUtils.java new file mode 100644 index 0000000000000..f195232b23502 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/PubsubUtils.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.pubsub; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import com.google.pubsub.v1.Subscription; +import com.google.pubsub.v1.SubscriptionName; +import com.google.pubsub.v1.Topic; +import com.google.pubsub.v1.TopicName; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CaseFormat; + +/** Utilities to make working with Pubsub easier. */ +final class PubsubUtils { + private PubsubUtils() {} + + /** + * Regular expression to validate/extract a topic name string, following + * projects/{project_id}/subscription/{topic_name}. + */ + private static final Pattern TOPICS_PATTERN = + Pattern.compile("^projects/(?[^/]+)/topics/(?[^/]+)$"); + + /** + * Regular expression to validate/extract a subscription name string, following + * projects/{project_id}/subscription/{subscription_name}. + */ + private static final Pattern SUBSCRIPTIONS_PATTERN = + Pattern.compile("^projects/(?[^/]+)/subscriptions/(?[^/]+)$"); + + /** + * Convert the instance of {@link Topic} to a {@link TopicName}. + * + *

    The idea of this project is to abstract the conversion/parse of the format + * projects/{project_id}/topics/{topic_name} + * + * @param topic Base topic. + * @return The reference to the topic. + */ + static TopicName toTopicName(Topic topic) { + Matcher matcher = TOPICS_PATTERN.matcher(topic.getName()); + checkArgument( + matcher.find(), + "Topic name must be in the format 'projects/{project_id}/topics/{topic_name}."); + + @SuppressWarnings("nullness") + TopicName topicName = TopicName.of(matcher.group("projectId"), matcher.group("topicName")); + return topicName; + } + + /** + * Convert the instance of {@link Subscription} to a {@link SubscriptionName}. + * + *

    The idea of this project is to abstract the conversion/parse of the format + * projects/{project_id}/subscription/{subscription_name} + * + * @param subscription Base subscription. + * @return The reference to the subscription. + */ + static SubscriptionName toSubscriptionName(Subscription subscription) { + Matcher matcher = SUBSCRIPTIONS_PATTERN.matcher(subscription.getName()); + checkArgument( + matcher.find(), + "Subscription name must be in the format" + + " 'projects/{project_id}/subscriptions/{subscription_name}."); + + @SuppressWarnings("nullness") + SubscriptionName subscriptionName = + SubscriptionName.of(matcher.group("projectId"), matcher.group("subscriptionName")); + return subscriptionName; + } + + /** + * Creates a topic name. + * + *

    If there are uppercase characters in {@code prefix}, then this will convert them into a dash + * followed by the lowercase equivalent of that letter. + * + *

    The topic name will normally be unique, but this is not guaranteed if multiple topics with + * the same prefix are created in a short period of time. + * + * @param prefix a prefix for the topic + * @return the prefix plus some way of identifying it separate from other topics with the same + * prefix + */ + static String createTestId(String prefix) { + String convertedPrefix = + CaseFormat.UPPER_CAMEL.converterTo(CaseFormat.LOWER_HYPHEN).convert(prefix); + String formattedTimestamp = + DateTimeFormatter.ofPattern("yyyyMMddHHmmssSSS") + .withZone(ZoneId.of("UTC")) + .format(Instant.now()); + return String.format("%s-%s", convertedPrefix, formattedTimestamp); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/conditions/PubsubMessagesCheck.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/conditions/PubsubMessagesCheck.java new file mode 100644 index 0000000000000..6803eec291b07 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/conditions/PubsubMessagesCheck.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.pubsub.conditions; + +import com.google.auto.value.AutoValue; +import com.google.pubsub.v1.PullResponse; +import com.google.pubsub.v1.ReceivedMessage; +import com.google.pubsub.v1.SubscriptionName; +import java.util.ArrayList; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; + +/** ConditionCheck to validate if Pub/Sub has received a certain amount of messages. */ +@AutoValue +public abstract class PubsubMessagesCheck extends ConditionCheck { + + abstract PubsubResourceManager resourceManager(); + + abstract SubscriptionName subscription(); + + abstract Integer minMessages(); + + @Nullable + abstract Integer maxMessages(); + + private final List receivedMessageList = new ArrayList<>(); + + @Override + public String getDescription() { + if (maxMessages() != null) { + return String.format( + "Pub/Sub check if subscription %s received between %d and %d messages", + subscription(), minMessages(), maxMessages()); + } + return String.format( + "Pub/Sub check if subscription %s received %d messages", subscription(), minMessages()); + } + + @Override + @SuppressWarnings("unboxing.of.nullable") + public CheckResult check() { + PullResponse pullResponse = + resourceManager() + .pull(subscription(), MoreObjects.firstNonNull(maxMessages(), minMessages()) + 1); + receivedMessageList.addAll(pullResponse.getReceivedMessagesList()); + + long totalRows = receivedMessageList.size(); + if (totalRows < minMessages()) { + return new CheckResult( + false, String.format("Expected %d messages but has only %d", minMessages(), totalRows)); + } + if (maxMessages() != null && totalRows > maxMessages()) { + return new CheckResult( + false, + String.format("Expected up to %d but found %d messages", maxMessages(), totalRows)); + } + + if (maxMessages() != null) { + return new CheckResult( + true, + String.format( + "Expected between %d and %d messages and found %d", + minMessages(), maxMessages(), totalRows)); + } + + return new CheckResult( + true, + String.format("Expected at least %d messages and found %d", minMessages(), totalRows)); + } + + public static Builder builder( + PubsubResourceManager resourceManager, SubscriptionName subscription) { + return new AutoValue_PubsubMessagesCheck.Builder() + .setResourceManager(resourceManager) + .setSubscription(subscription); + } + + public List getReceivedMessageList() { + return receivedMessageList; + } + + /** Builder for {@link PubsubMessagesCheck}. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setResourceManager(PubsubResourceManager resourceManager); + + public abstract Builder setSubscription(SubscriptionName subscription); + + public abstract Builder setMinMessages(Integer minMessages); + + public abstract Builder setMaxMessages(Integer maxMessages); + + abstract PubsubMessagesCheck autoBuild(); + + public PubsubMessagesCheck build() { + return autoBuild(); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/conditions/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/conditions/package-info.java new file mode 100644 index 0000000000000..7b216244203a3 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/conditions/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package that contains reusable Pub/Sub conditions. */ +package org.apache.beam.it.gcp.pubsub.conditions; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/package-info.java new file mode 100644 index 0000000000000..7709f41448b79 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsub/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing Pub/Sub resources within integration tests. */ +package org.apache.beam.it.gcp.pubsub; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite/PubsubliteResourceManager.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite/PubsubliteResourceManager.java new file mode 100644 index 0000000000000..6f6e21bfd327a --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite/PubsubliteResourceManager.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.pubsublite; + +import com.google.cloud.pubsublite.AdminClient; +import com.google.cloud.pubsublite.AdminClientSettings; +import com.google.cloud.pubsublite.CloudRegion; +import com.google.cloud.pubsublite.ProjectId; +import com.google.cloud.pubsublite.ReservationName; +import com.google.cloud.pubsublite.ReservationPath; +import com.google.cloud.pubsublite.SubscriptionName; +import com.google.cloud.pubsublite.SubscriptionPath; +import com.google.cloud.pubsublite.TopicName; +import com.google.cloud.pubsublite.TopicPath; +import com.google.cloud.pubsublite.proto.Reservation; +import com.google.cloud.pubsublite.proto.Subscription; +import com.google.cloud.pubsublite.proto.Topic; +import com.google.protobuf.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import org.apache.beam.it.common.ResourceManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Client for managing Pub/Sub Lite resources. */ +public class PubsubliteResourceManager implements ResourceManager { + private static final Logger LOG = LoggerFactory.getLogger(PubsubliteResourceManager.class); + + private final List cleanupReservations = new ArrayList<>(); + private final List cleanupTopics = new ArrayList<>(); + + public static final Integer DEFAULT_NUM_PARTITIONS = 100; + + // 5 days is the default retention period for the PSLite topic + public static final Duration DEFAULT_RETENTION_PERIOD = + Duration.newBuilder().setSeconds(3600 * 24 * 5).build(); + + // 30 GB per partition + public static final Long DEFAULT_PARTITION_SIZE = 30 * 1024 * 1024 * 1024L; + + /** + * Creates a new PubsubLite reservation with the specified number of capacity units. Capacity + * units represent 0.25 MiBps on a regional reservation, and 1 MiBps on a zonal reservation. + * + * @param reservationName the name of the reservation to create. + * @param cloudRegion the region in which the reservation will be created. + * @param projectId the project id associated with the reservation. + * @param capacity the number of capacity units for the reservation. + * @return the path of the created reservation. + */ + public ReservationPath createReservation( + String reservationName, String cloudRegion, String projectId, Long capacity) { + try (AdminClient client = + AdminClient.create( + AdminClientSettings.newBuilder().setRegion(CloudRegion.of(cloudRegion)).build())) { + ReservationPath reservationPath = + ReservationPath.newBuilder() + .setProject(ProjectId.of(projectId)) + .setLocation(CloudRegion.of(cloudRegion)) + .setName(ReservationName.of(reservationName)) + .build(); + client + .createReservation( + Reservation.newBuilder() + .setName(reservationPath.toString()) + .setThroughputCapacity(capacity) + .build()) + .get(); + cleanupReservations.add(reservationPath); + LOG.info("Created reservation {}", reservationPath); + return reservationPath; + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException( + String.format( + "Unable to create reservation %s in region %s with capacity %d", + reservationName, cloudRegion, capacity), + e); + } + } + + /** + * Creates a topic with the given name on Pub/Sub. + * + *

    https://cloud.google.com/pubsub/lite/docs/reservations + * + * @param topicName Topic name to create. The underlying implementation may not use the topic name + * directly, and can add a prefix or a suffix to identify specific executions. + * @param reservationPath the path of the reservation under which to create the topic. + * @return The instance of the TopicName that was just created. + */ + public TopicName createTopic(String topicName, ReservationPath reservationPath) { + try (AdminClient client = + AdminClient.create( + AdminClientSettings.newBuilder().setRegion(reservationPath.location()).build())) { + TopicPath topicPath = + TopicPath.newBuilder() + .setName(TopicName.of(topicName)) + .setLocation(reservationPath.location()) + .setProject(reservationPath.project()) + .build(); + Topic topic = + client + .createTopic( + Topic.newBuilder() + .setName(topicPath.toString()) + .setPartitionConfig( + Topic.PartitionConfig.newBuilder() + .setCount(DEFAULT_NUM_PARTITIONS) + .build()) + .setRetentionConfig( + Topic.RetentionConfig.newBuilder() + .setPeriod(DEFAULT_RETENTION_PERIOD) + .setPerPartitionBytes(DEFAULT_PARTITION_SIZE) + .build()) + .setReservationConfig( + Topic.ReservationConfig.newBuilder() + .setThroughputReservation(reservationPath.toString()) + .build()) + .build()) + .get(); + cleanupTopics.add(topicPath); + LOG.info("Created topic {}", topicPath); + return TopicName.of(topic.getName()); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException( + String.format("Unable to create topic %s in reservation %s", topicName, reservationPath), + e); + } + } + + /** + * Creates a new Pub/Sub Lite subscription for a specified topic. + * + * @param reservationPath the path of the reservation to add the subscription. + * @param topicName the name of the topic to add the subscription to. + * @param subscriptionName the name to use for the subscription. + * @return the created {@link SubscriptionName} instance. + */ + public SubscriptionName createSubscription( + ReservationPath reservationPath, TopicName topicName, String subscriptionName) { + try (AdminClient client = + AdminClient.create( + AdminClientSettings.newBuilder().setRegion(reservationPath.location()).build())) { + Subscription subscription = + client + .createSubscription( + Subscription.newBuilder() + .setTopic(topicName.toString()) + .setName( + SubscriptionPath.newBuilder() + .setLocation(reservationPath.location()) + .setName(SubscriptionName.of(subscriptionName)) + .setProject(reservationPath.project()) + .build() + .toString()) + .setDeliveryConfig( + Subscription.DeliveryConfig.newBuilder() + .setDeliveryRequirement( + Subscription.DeliveryConfig.DeliveryRequirement + .DELIVER_IMMEDIATELY) + .build()) + .build()) + .get(); + LOG.info("Created subscription {}", subscription.getName()); + return SubscriptionName.of(subscription.getName()); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException( + String.format( + "Unable to create subscription %s for topic %s", subscriptionName, topicName), + e); + } + } + + /** Delete any topics or subscriptions created by this manager. */ + @Override + public void cleanupAll() { + for (TopicPath t : cleanupTopics) { + try (AdminClient client = + AdminClient.create( + AdminClientSettings.newBuilder().setRegion(t.location().region()).build())) { + client.deleteTopic(t).get(); + LOG.info("Deleted topic {}", t); + } catch (InterruptedException | ExecutionException e) { + System.out.println("Unable to delete topic " + t); + e.printStackTrace(); + } + } + for (ReservationPath r : cleanupReservations) { + try (AdminClient client = + AdminClient.create(AdminClientSettings.newBuilder().setRegion(r.location()).build())) { + client.deleteReservation(r).get(); + LOG.info("Deleted reservation {}", r); + } catch (InterruptedException | ExecutionException e) { + System.out.println("Unable to delete reservation " + r); + e.printStackTrace(); + } + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite/package-info.java new file mode 100644 index 0000000000000..c135952d6cdfa --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing Pub/Sub lite resources within integration tests. */ +package org.apache.beam.it.gcp.pubsublite; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/secretmanager/SecretManagerResourceManager.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/secretmanager/SecretManagerResourceManager.java new file mode 100644 index 0000000000000..5d09f17cb5823 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/secretmanager/SecretManagerResourceManager.java @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.secretmanager; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.cloud.secretmanager.v1.AccessSecretVersionResponse; +import com.google.cloud.secretmanager.v1.ProjectName; +import com.google.cloud.secretmanager.v1.Replication; +import com.google.cloud.secretmanager.v1.Secret; +import com.google.cloud.secretmanager.v1.SecretManagerServiceClient; +import com.google.cloud.secretmanager.v1.SecretName; +import com.google.cloud.secretmanager.v1.SecretPayload; +import com.google.cloud.secretmanager.v1.SecretVersion; +import com.google.cloud.secretmanager.v1.SecretVersionName; +import com.google.common.annotations.VisibleForTesting; +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import org.apache.beam.it.common.ResourceManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Default class for implementation of {@link ResourceManager} interface. + * + *

    The class provides an interaction with the real GCP Secret Manager client, with operations + * related to management of Secrets in the SecretManager. + */ +public class SecretManagerResourceManager implements ResourceManager { + + private static final Logger LOG = LoggerFactory.getLogger(SecretManagerResourceManager.class); + + private final String projectId; + + private final SecretManagerServiceClient secretManagerServiceClient; + + private final Set createdSecretIds; + + private SecretManagerResourceManager(Builder builder) throws IOException { + this(builder.projectId, SecretManagerServiceClient.create()); + } + + @VisibleForTesting + public SecretManagerResourceManager( + String projectId, SecretManagerServiceClient secretManagerServiceClient) { + this.projectId = projectId; + this.secretManagerServiceClient = secretManagerServiceClient; + this.createdSecretIds = Collections.synchronizedSet(new HashSet<>()); + } + + public static Builder builder(String projectId, CredentialsProvider credentialsProvider) { + checkArgument(!projectId.isEmpty(), "projectId can not be empty"); + return new Builder(projectId, credentialsProvider); + } + + /** + * Creates a Secret with given name on GCP Secret Manager. + * + * @param secretId Secret ID of the secret to be created + * @param secretData Value of the secret to be added + */ + public void createSecret(String secretId, String secretData) { + checkArgument(!secretId.isEmpty(), "secretId can not be empty"); + checkArgument(!secretData.isEmpty(), "secretData can not be empty"); + try { + checkIsUsable(); + ProjectName projectName = ProjectName.of(projectId); + + // Create the parent secret. + Secret secret = + Secret.newBuilder() + .setReplication( + Replication.newBuilder() + .setAutomatic(Replication.Automatic.newBuilder().build()) + .build()) + .build(); + + Secret createdSecret = secretManagerServiceClient.createSecret(projectName, secretId, secret); + + // Add a secret version. + SecretPayload payload = + SecretPayload.newBuilder().setData(ByteString.copyFromUtf8(secretData)).build(); + secretManagerServiceClient.addSecretVersion(createdSecret.getName(), payload); + + createdSecretIds.add(secretId); + + LOG.info("Created secret successfully."); + + } catch (Exception e) { + throw new SecretManagerResourceManagerException("Error while creating secret", e); + } + } + + /** + * Creates a new SecretVersion containing secret data and attaches it to an existing Secret. + * + * @param secretId Parent secret to which version will be added + * @param secretData Value of the secret to be added + */ + public void addSecretVersion(String secretId, String secretData) { + checkArgument(!secretId.isEmpty(), "secretId can not be empty"); + checkArgument(!secretData.isEmpty(), "secretData can not be empty"); + checkIsUsable(); + try { + SecretName parent = SecretName.of(projectId, secretId); + SecretPayload secretPayload = + SecretPayload.newBuilder().setData(ByteString.copyFromUtf8(secretData)).build(); + secretManagerServiceClient.addSecretVersion(parent, secretPayload); + } catch (Exception e) { + throw new SecretManagerResourceManagerException("Error while adding version to a secret", e); + } + } + + /** + * Calls Secret Manager with a Secret Version and returns the secret value. + * + * @param secretVersion Secret Version of the form + * projects/{project}/secrets/{secret}/versions/{secret_version} + * @return the secret value in Secret Manager + */ + public String accessSecret(String secretVersion) { + + checkArgument(!secretVersion.isEmpty(), "secretVersion can not be empty"); + checkIsUsable(); + try { + SecretVersionName secretVersionName; + + if (SecretVersionName.isParsableFrom(secretVersion)) { + secretVersionName = SecretVersionName.parse(secretVersion); + + } else { + throw new IllegalArgumentException( + "Provided Secret must be in the form" + + " projects/{project}/secrets/{secret}/versions/{secret_version}"); + } + AccessSecretVersionResponse response = + secretManagerServiceClient.accessSecretVersion(secretVersionName); + return response.getPayload().getData().toStringUtf8(); + + } catch (Exception e) { + throw new SecretManagerResourceManagerException("Error while accessing a secret version", e); + } + } + + /** + * Enables a SecretVersion. Sets the state of the SecretVersion to ENABLED. + * + * @param secretVersion The resource name of the SecretVersion to destroy in the format of + * projects/{project}/secrets/{secret}/versions/{secret_version} + */ + public void enableSecretVersion(String secretVersion) { + checkIsUsable(); + if (!SecretVersionName.isParsableFrom(secretVersion)) { + throw new IllegalArgumentException( + "Provided Secret must be in the form" + + " projects/{project}/secrets/{secret}/versions/{secret_version}"); + } + SecretVersionName secretVersionName = SecretVersionName.parse(secretVersion); + SecretVersion response = secretManagerServiceClient.enableSecretVersion(secretVersionName); + LOG.info("The current state of secret version is '{}'", response.getState().toString()); + } + + /** + * Disables a SecretVersion. Sets the state of the SecretVersion to DISABLED. + * + * @param secretVersion The resource name of the SecretVersion to destroy in the format of + * projects/{project}/secrets/{secret}/versions/{secret_version} + */ + public void disableSecretVersion(String secretVersion) { + checkIsUsable(); + if (!SecretVersionName.isParsableFrom(secretVersion)) { + throw new IllegalArgumentException( + "Provided Secret must be in the form" + + " projects/{project}/secrets/{secret}/versions/{secret_version}"); + } + SecretVersionName secretVersionName = SecretVersionName.parse(secretVersion); + SecretVersion response = secretManagerServiceClient.disableSecretVersion(secretVersionName); + LOG.info("The current state of secret version is '{}'", response.getState().toString()); + } + + /** + * Sets the state of the SecretVersion to DESTROYED and irrevocably destroys the secret data. + * + * @param secretVersion The resource name of the SecretVersion to destroy in the format of + * projects/{project}/secrets/{secret}/versions/{secret_version} + */ + public void destroySecretVersion(String secretVersion) { + checkIsUsable(); + if (!SecretVersionName.isParsableFrom(secretVersion)) { + throw new IllegalArgumentException( + "Provided Secret must be in the form" + + " projects/{project}/secrets/{secret}/versions/{secret_version}"); + } + SecretVersionName secretVersionName = SecretVersionName.parse(secretVersion); + SecretVersion response = secretManagerServiceClient.destroySecretVersion(secretVersionName); + LOG.info("The current state of secret version is '{}'", response.getState().toString()); + } + + /** + * Deletes a Secret. + * + * @param secretId The resource name of the SecretVersion to delete in the format of + * projects/{project}/secrets/{secret}/versions/{secret_version} + */ + public void deleteSecret(String secretId) { + checkIsUsable(); + try { + SecretName secret = SecretName.of(projectId, secretId); + secretManagerServiceClient.deleteSecret(secret); + createdSecretIds.remove(secretId); + + LOG.info("Successfully deleted secret"); + } catch (Exception e) { + throw new SecretManagerResourceManagerException("Error while deleting a secret", e); + } + } + + @Override + public synchronized void cleanupAll() { + + LOG.info("Attempting to cleanup manager."); + + try { + for (String secretId : createdSecretIds) { + LOG.info("Deleting secretId '{}'", secretId); + deleteSecret(secretId); + } + + } finally { + secretManagerServiceClient.close(); + } + + LOG.info("Manager successfully cleaned up."); + } + + /** + * Check if the clients started by this instance are still usable, and throwing {@link + * IllegalStateException} otherwise. + */ + private void checkIsUsable() throws IllegalStateException { + if (isNotUsable()) { + throw new IllegalStateException("Manager has cleaned up resources and is unusable."); + } + } + + private boolean isNotUsable() { + return secretManagerServiceClient.isShutdown() || secretManagerServiceClient.isTerminated(); + } + + /** Builder for {@link SecretManagerResourceManager}. */ + public static final class Builder { + private final String projectId; + + private CredentialsProvider credentialsProvider; + + private Builder(String projectId, CredentialsProvider credentialsProvider) { + this.projectId = projectId; + this.credentialsProvider = credentialsProvider; + } + + public Builder credentialsProvider(CredentialsProvider credentialsProvider) { + this.credentialsProvider = credentialsProvider; + return this; + } + + public SecretManagerResourceManager build() throws IOException { + if (credentialsProvider == null) { + throw new IllegalArgumentException( + "Unable to find credentials. Please provide credentials to authenticate to GCP"); + } + return new SecretManagerResourceManager(this); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/secretmanager/SecretManagerResourceManagerException.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/secretmanager/SecretManagerResourceManagerException.java new file mode 100644 index 0000000000000..c701affee82e8 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/secretmanager/SecretManagerResourceManagerException.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.secretmanager; + +/** + * Custom exception for {@link + * com.google.cloud.teleport.it.secretmanager.SecretManagerResourceManager} implementations. + */ +public class SecretManagerResourceManagerException extends RuntimeException { + + public SecretManagerResourceManagerException(String errorMessage, Throwable err) { + super(errorMessage, err); + } + + public SecretManagerResourceManagerException(String errorMessage) { + super(errorMessage); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/secretmanager/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/secretmanager/package-info.java new file mode 100644 index 0000000000000..da6a107a90957 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/secretmanager/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing Secret Manager resources within integration tests. */ +package org.apache.beam.it.gcp.secretmanager; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/SpannerResourceManager.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/SpannerResourceManager.java new file mode 100644 index 0000000000000..c2db76fb14b1d --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/SpannerResourceManager.java @@ -0,0 +1,458 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.spanner; + +import static org.apache.beam.it.common.utils.ResourceManagerUtils.checkValidProjectId; +import static org.apache.beam.it.common.utils.ResourceManagerUtils.generateNewId; + +import com.google.cloud.spanner.Database; +import com.google.cloud.spanner.DatabaseAdminClient; +import com.google.cloud.spanner.DatabaseClient; +import com.google.cloud.spanner.DatabaseId; +import com.google.cloud.spanner.Dialect; +import com.google.cloud.spanner.Instance; +import com.google.cloud.spanner.InstanceAdminClient; +import com.google.cloud.spanner.InstanceConfigId; +import com.google.cloud.spanner.InstanceId; +import com.google.cloud.spanner.InstanceInfo; +import com.google.cloud.spanner.KeySet; +import com.google.cloud.spanner.Mutation; +import com.google.cloud.spanner.ReadContext; +import com.google.cloud.spanner.ResultSet; +import com.google.cloud.spanner.Spanner; +import com.google.cloud.spanner.SpannerException; +import com.google.cloud.spanner.SpannerOptions; +import com.google.cloud.spanner.Struct; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import dev.failsafe.Failsafe; +import dev.failsafe.RetryPolicy; +import java.time.Duration; +import java.util.concurrent.ExecutionException; +import javax.annotation.Nullable; +import org.apache.beam.it.common.ResourceManager; +import org.apache.beam.it.common.utils.ExceptionUtils; +import org.apache.beam.it.gcp.spanner.utils.SpannerResourceManagerUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Client for managing Spanner resources. + * + *

    The class supports one instance, one database, and multiple tables per manager object. The + * instance and database are created when the first table is created. + * + *

    The instance and database ids are formed using testId. The database id will be {testId}, with + * some extra formatting. The instance id will be "{testId}-{ISO8601 time, microsecond precision}", + * with additional formatting. Note: If testId is more than 30 characters, a new testId will be + * formed for naming: {first 21 chars of long testId} + “-” + {8 char hash of testId}. + * + *

    The class is thread-safe. + */ +public final class SpannerResourceManager implements ResourceManager { + + private static final Logger LOG = LoggerFactory.getLogger(SpannerResourceManager.class); + private static final int MAX_BASE_ID_LENGTH = 30; + + // Retry settings for instance creation + private static final int CREATE_MAX_RETRIES = 5; + private static final Duration CREATE_BACKOFF_DELAY = Duration.ofSeconds(10); + private static final Duration CREATE_BACKOFF_MAX_DELAY = Duration.ofSeconds(60); + private static final double CREATE_BACKOFF_JITTER = 0.1; + + private boolean hasInstance = false; + private boolean hasDatabase = false; + + private final String projectId; + private final String instanceId; + private final boolean usingStaticInstance; + private final String databaseId; + private final String region; + + private final Dialect dialect; + + private final Spanner spanner; + private final InstanceAdminClient instanceAdminClient; + private final DatabaseAdminClient databaseAdminClient; + + private SpannerResourceManager(Builder builder) { + this( + SpannerOptions.newBuilder().setProjectId(builder.projectId).build().getService(), + builder.testId, + builder.projectId, + builder.region, + builder.dialect, + builder.useStaticInstance, + builder.instanceId); + } + + @VisibleForTesting + SpannerResourceManager( + Spanner spanner, + String testId, + String projectId, + String region, + Dialect dialect, + boolean useStaticInstance, + @Nullable String instanceId) { + // Check that the project ID conforms to GCP standards + checkValidProjectId(projectId); + + if (testId.length() > MAX_BASE_ID_LENGTH) { + testId = generateNewId(testId, MAX_BASE_ID_LENGTH); + } + this.projectId = projectId; + + if (useStaticInstance) { + if (instanceId == null) { + throw new SpannerResourceManagerException( + "This manager was configured to use a static resource, but the instanceId was not properly set."); + } + this.instanceId = instanceId; + } else { + this.instanceId = SpannerResourceManagerUtils.generateInstanceId(testId); + } + this.usingStaticInstance = useStaticInstance; + this.databaseId = SpannerResourceManagerUtils.generateDatabaseId(testId); + this.region = region; + this.dialect = dialect; + this.spanner = spanner; + this.instanceAdminClient = spanner.getInstanceAdminClient(); + this.databaseAdminClient = spanner.getDatabaseAdminClient(); + } + + public static Builder builder(String testId, String projectId, String region) { + return new Builder(testId, projectId, region, Dialect.GOOGLE_STANDARD_SQL); + } + + public static Builder builder(String testId, String projectId, String region, Dialect dialect) { + return new Builder(testId, projectId, region, dialect); + } + + private synchronized void maybeCreateInstance() { + checkIsUsable(); + + if (usingStaticInstance) { + LOG.info("Not creating Spanner instance - reusing static {}", instanceId); + hasInstance = true; + return; + } + + if (hasInstance) { + return; + } + + LOG.info("Creating instance {} in project {}.", instanceId, projectId); + try { + InstanceInfo instanceInfo = + InstanceInfo.newBuilder(InstanceId.of(projectId, instanceId)) + .setInstanceConfigId(InstanceConfigId.of(projectId, "regional-" + region)) + .setDisplayName(instanceId) + .setNodeCount(1) + .build(); + + // Retry creation if there's a quota error + Instance instance = + Failsafe.with(retryOnQuotaException()) + .get(() -> instanceAdminClient.createInstance(instanceInfo).get()); + + hasInstance = true; + LOG.info("Successfully created instance {}: {}.", instanceId, instance.getState()); + } catch (Exception e) { + cleanupAll(); + throw new SpannerResourceManagerException("Failed to create instance.", e); + } + } + + private synchronized void maybeCreateDatabase() { + checkIsUsable(); + if (hasDatabase) { + return; + } + LOG.info("Creating database {} in instance {}.", databaseId, instanceId); + + try { + Database database = + Failsafe.with(retryOnQuotaException()) + .get( + () -> + databaseAdminClient + .createDatabase( + databaseAdminClient + .newDatabaseBuilder( + DatabaseId.of(projectId, instanceId, databaseId)) + .setDialect(dialect) + .build(), + ImmutableList.of()) + .get()); + + hasDatabase = true; + LOG.info("Successfully created database {}: {}.", databaseId, database.getState()); + } catch (Exception e) { + cleanupAll(); + throw new SpannerResourceManagerException("Failed to create database.", e); + } + } + + private static RetryPolicy retryOnQuotaException() { + return RetryPolicy.builder() + .handleIf(exception -> ExceptionUtils.containsMessage(exception, "RESOURCE_EXHAUSTED")) + .withMaxRetries(CREATE_MAX_RETRIES) + .withBackoff(CREATE_BACKOFF_DELAY, CREATE_BACKOFF_MAX_DELAY) + .withJitter(CREATE_BACKOFF_JITTER) + .build(); + } + + private void checkIsUsable() throws IllegalStateException { + if (spanner.isClosed()) { + throw new IllegalStateException("Manager has cleaned up all resources and is unusable."); + } + } + + private void checkHasInstanceAndDatabase() throws IllegalStateException { + if (!hasInstance) { + throw new IllegalStateException("There is no instance for manager to perform operation on."); + } + if (!hasDatabase) { + throw new IllegalStateException("There is no database for manager to perform operation on"); + } + } + + /** + * Return the instance ID this Resource Manager uses to create and manage tables in. + * + * @return the instance ID. + */ + public String getInstanceId() { + return this.instanceId; + } + + /** + * Return the dataset ID this Resource Manager uses to create and manage tables in. + * + * @return the dataset ID. + */ + public String getDatabaseId() { + return this.databaseId; + } + + /** + * Executes a DDL statement. + * + *

    Note: Implementations may do instance creation and database creation here. + * + * @param statement The DDL statement. + * @throws IllegalStateException if method is called after resources have been cleaned up. + */ + public synchronized void executeDdlStatement(String statement) throws IllegalStateException { + checkIsUsable(); + maybeCreateInstance(); + maybeCreateDatabase(); + + LOG.info("Executing DDL statement '{}' on database {}.", statement, databaseId); + try { + databaseAdminClient + .updateDatabaseDdl( + instanceId, databaseId, ImmutableList.of(statement), /* operationId= */ null) + .get(); + LOG.info("Successfully executed DDL statement '{}' on database {}.", statement, databaseId); + } catch (ExecutionException | InterruptedException | SpannerException e) { + throw new SpannerResourceManagerException("Failed to execute statement.", e); + } + } + + /** + * Writes a given record into a table. This method requires {@link + * SpannerResourceManager#executeDdlStatement(String)} to be called for the target table + * beforehand. + * + * @param tableRecord A mutation object representing the table record. + * @throws IllegalStateException if method is called after resources have been cleaned up or if + * the manager object has no instance or database. + */ + public synchronized void write(Mutation tableRecord) throws IllegalStateException { + write(ImmutableList.of(tableRecord)); + } + + /** + * Writes a collection of table records into one or more tables. This method requires {@link + * SpannerResourceManager#executeDdlStatement(String)} to be called for the target table + * beforehand. + * + * @param tableRecords A collection of mutation objects representing table records. + * @throws IllegalStateException if method is called after resources have been cleaned up or if + * the manager object has no instance or database. + */ + public synchronized void write(Iterable tableRecords) throws IllegalStateException { + checkIsUsable(); + checkHasInstanceAndDatabase(); + + LOG.info("Sending {} mutations to {}.{}", Iterables.size(tableRecords), instanceId, databaseId); + try { + DatabaseClient databaseClient = + spanner.getDatabaseClient(DatabaseId.of(projectId, instanceId, databaseId)); + databaseClient.write(tableRecords); + LOG.info("Successfully sent mutations to {}.{}", instanceId, databaseId); + } catch (SpannerException e) { + throw new SpannerResourceManagerException("Failed to write mutations.", e); + } + } + + /** + * Reads all the rows in a table. This method requires {@link + * SpannerResourceManager#executeDdlStatement(String)} to be called for the target table + * beforehand. + * + * @param tableId The id of the table to read rows from. + * @param columnNames The table's column names. + * @return A ResultSet object containing all the rows in the table. + * @throws IllegalStateException if method is called after resources have been cleaned up or if + * the manager object has no instance or database. + */ + public synchronized ImmutableList readTableRecords(String tableId, String... columnNames) + throws IllegalStateException { + return readTableRecords(tableId, ImmutableList.copyOf(columnNames)); + } + + /** + * Reads all the rows in a table.This method requires {@link + * SpannerResourceManager#executeDdlStatement(String)} to be called for the target table + * beforehand. + * + * @param tableId The id of table to read rows from. + * @param columnNames A collection of the table's column names. + * @return A ResultSet object containing all the rows in the table. + * @throws IllegalStateException if method is called after resources have been cleaned up or if + * the manager object has no instance or database. + */ + public synchronized ImmutableList readTableRecords( + String tableId, Iterable columnNames) throws IllegalStateException { + checkIsUsable(); + checkHasInstanceAndDatabase(); + + LOG.info( + "Loading columns {} from {}.{}.{}", + Iterables.toString(columnNames), + instanceId, + databaseId, + tableId); + DatabaseClient databaseClient = + spanner.getDatabaseClient(DatabaseId.of(projectId, instanceId, databaseId)); + + try (ReadContext readContext = databaseClient.singleUse(); + ResultSet resultSet = readContext.read(tableId, KeySet.all(), columnNames)) { + ImmutableList.Builder tableRecordsBuilder = ImmutableList.builder(); + + while (resultSet.next()) { + tableRecordsBuilder.add(resultSet.getCurrentRowAsStruct()); + } + ImmutableList tableRecords = tableRecordsBuilder.build(); + LOG.info( + "Loaded {} records from {}.{}.{}", tableRecords.size(), instanceId, databaseId, tableId); + return tableRecords; + } catch (SpannerException e) { + throw new SpannerResourceManagerException("Error occurred while reading table records.", e); + } + } + + /** + * Deletes all created resources (instance, database, and tables) and cleans up all Spanner + * sessions, making the manager object unusable. + */ + @Override + public synchronized void cleanupAll() { + try { + + if (usingStaticInstance) { + if (databaseAdminClient != null) { + Failsafe.with(retryOnQuotaException()) + .run(() -> databaseAdminClient.dropDatabase(instanceId, databaseId)); + } + } else { + LOG.info("Deleting instance {}...", instanceId); + + if (instanceAdminClient != null) { + Failsafe.with(retryOnQuotaException()) + .run(() -> instanceAdminClient.deleteInstance(instanceId)); + } + + hasInstance = false; + } + + hasDatabase = false; + } catch (SpannerException e) { + throw new SpannerResourceManagerException("Failed to delete instance.", e); + } finally { + if (!spanner.isClosed()) { + spanner.close(); + } + } + LOG.info("Manager successfully cleaned up."); + } + + /** Builder for {@link SpannerResourceManager}. */ + public static final class Builder { + + private final String testId; + private final String projectId; + private final String region; + private boolean useStaticInstance; + private @Nullable String instanceId; + + private final Dialect dialect; + + private Builder(String testId, String projectId, String region, Dialect dialect) { + this.testId = testId; + this.projectId = projectId; + this.region = region; + this.dialect = dialect; + this.instanceId = null; + } + + /** + * Configures the resource manager to use a static GCP resource instead of creating a new + * instance of the resource. + * + * @return this builder object with the useStaticInstance option enabled. + */ + public Builder useStaticInstance() { + this.useStaticInstance = true; + return this; + } + + /** + * Looks at the system properties if there's an instance id, and reuses it if configured. + * + * @return this builder object with the useStaticInstance option enabled and instance set if + * configured, the same builder otherwise. + */ + @SuppressWarnings("nullness") + public Builder maybeUseStaticInstance() { + if (System.getProperty("spannerInstanceId") != null) { + this.useStaticInstance = true; + this.instanceId = System.getProperty("spannerInstanceId"); + } + return this; + } + + public SpannerResourceManager build() { + return new SpannerResourceManager(this); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/SpannerResourceManagerException.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/SpannerResourceManagerException.java new file mode 100644 index 0000000000000..ec2ef64bab2df --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/SpannerResourceManagerException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.spanner; + +/** Custom exception for {@link SpannerResourceManager} implementations. */ +public class SpannerResourceManagerException extends RuntimeException { + + public SpannerResourceManagerException(String errorMessage) { + super(errorMessage); + } + + public SpannerResourceManagerException(String errorMessage, Throwable err) { + super(errorMessage, err); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/matchers/SpannerAsserts.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/matchers/SpannerAsserts.java new file mode 100644 index 0000000000000..c9964d16f3b1e --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/matchers/SpannerAsserts.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.spanner.matchers; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatRecords; + +import com.google.cloud.spanner.Mutation; +import com.google.cloud.spanner.Struct; +import com.google.cloud.spanner.Type; +import com.google.cloud.spanner.Value; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.beam.it.truthmatchers.RecordsSubject; + +public class SpannerAsserts { + + /** + * Convert Spanner {@link Struct} list to a list of maps. + * + * @param structs Structs to parse + * @return List of maps to use in {@link RecordsSubject} + */ + public static List> structsToRecords(List structs) { + try { + List> records = new ArrayList<>(); + + for (Struct struct : structs) { + Map record = new HashMap<>(); + + for (Type.StructField field : struct.getType().getStructFields()) { + Value fieldValue = struct.getValue(field.getName()); + // May need to explore using typed methods instead of .toString() + record.put(field.getName(), fieldValue.toString()); + } + + records.add(record); + } + + return records; + } catch (Exception e) { + throw new RuntimeException("Error converting TableResult to Records", e); + } + } + + /** + * Convert Spanner {@link Mutation} list to a list of maps. + * + * @param mutations Mutations to parse + * @return List of maps to use in {@link RecordsSubject} + */ + public static List> mutationsToRecords(List mutations) { + try { + List> records = new ArrayList<>(); + mutations.forEach( + entry -> + records.add( + entry.asMap().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)))); + + return records; + } catch (Exception e) { + throw new RuntimeException("Error converting TableResult to Records", e); + } + } + + /** + * Creates a {@link RecordsSubject} to assert information within a list of records. + * + * @param structs Records in Spanner {@link Struct} format to use in the comparison. + * @return Truth Subject to chain assertions. + */ + public static RecordsSubject assertThatStructs(List structs) { + return assertThatRecords(structsToRecords(structs)); + } + + /** + * Creates a {@link RecordsSubject} to assert information within a list of records. + * + * @param mutations Mutations in Spanner {@link Mutation} format to use in the comparison. + * @return Truth Subject to chain assertions. + */ + public static RecordsSubject assertThatMutations(List mutations) { + return assertThatRecords(mutationsToRecords(mutations)); + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/matchers/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/matchers/package-info.java new file mode 100644 index 0000000000000..86f087c7345ae --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/matchers/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for Spanner Truth matchers / subjects to have reusable assertions. */ +package org.apache.beam.it.gcp.spanner.matchers; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/package-info.java new file mode 100644 index 0000000000000..5c9dad74fed2d --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing Spanner resources within integration tests. */ +package org.apache.beam.it.gcp.spanner; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/utils/SpannerResourceManagerUtils.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/utils/SpannerResourceManagerUtils.java new file mode 100644 index 0000000000000..3dce40511adf4 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/utils/SpannerResourceManagerUtils.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.spanner.utils; + +import static org.apache.beam.it.common.utils.ResourceManagerUtils.generatePadding; +import static org.apache.beam.it.common.utils.ResourceManagerUtils.generateResourceId; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import java.time.format.DateTimeFormatter; +import java.util.regex.Pattern; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CharMatcher; + +/** Utilities for {@link SpannerResourceManager} implementations. */ +public final class SpannerResourceManagerUtils { + private static final Pattern ILLEGAL_INSTANCE_CHARS = Pattern.compile("[^a-z0-9-]"); + private static final Pattern ILLEGAL_DATABASE_CHARS = Pattern.compile("[\\W-]"); + private static final String REPLACE_INSTANCE_CHAR = "-"; + private static final String REPLACE_DATABASE_CHAR = "_"; + public static final int MAX_INSTANCE_ID_LENGTH = 30; + public static final int MAX_DATABASE_ID_LENGTH = 30; + private static final DateTimeFormatter INSTANCE_TIME_FORMAT = + DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss-SSSSSS"); + private static final DateTimeFormatter DATABASE_TIME_FORMAT = + DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss_SSSSSS"); + + private SpannerResourceManagerUtils() {} + + /** + * Generates a database id from a given string. + * + * @param baseString The string to generate the id from. + * @return The database id string + */ + public static String generateDatabaseId(String baseString) { + checkArgument(baseString.length() != 0, "baseString cannot be empty!"); + + String databaseId = + generateResourceId( + baseString, + ILLEGAL_DATABASE_CHARS, + REPLACE_DATABASE_CHAR, + MAX_DATABASE_ID_LENGTH, + DATABASE_TIME_FORMAT); + + // replace hyphen with underscore, so there's no need for backticks + String trimmed = CharMatcher.is('_').trimTrailingFrom(databaseId); + + checkArgument( + trimmed.length() > 0, + "Database id is empty after removing illegal characters and trailing underscores"); + + // if first char is not a letter, replace with a padding letter, so it doesn't + // violate spanner's database naming rules + char padding = generatePadding(); + if (!Character.isLetter(trimmed.charAt(0))) { + trimmed = padding + trimmed.substring(1); + } + return trimmed; + } + + /** + * Generates an instance id from a given string. + * + * @param baseString The string to generate the id from. + * @return The instance id string. + */ + public static String generateInstanceId(String baseString) { + String instanceId = + generateResourceId( + baseString, + ILLEGAL_INSTANCE_CHARS, + REPLACE_INSTANCE_CHAR, + MAX_INSTANCE_ID_LENGTH, + INSTANCE_TIME_FORMAT); + + // if first char is not a letter, replace with letter, so it doesn't + // violate spanner's instance naming rules + if (!Character.isLetter(instanceId.charAt(0))) { + char padding = generatePadding(); + instanceId = padding + instanceId.substring(1); + } + + return instanceId; + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/utils/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/utils/package-info.java new file mode 100644 index 0000000000000..1dea4c04a402a --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/utils/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for Spanner utilities. */ +package org.apache.beam.it.gcp.spanner.utils; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/storage/GcsResourceManager.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/storage/GcsResourceManager.java new file mode 100644 index 0000000000000..730ca23ee54b5 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/storage/GcsResourceManager.java @@ -0,0 +1,310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.storage; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import com.google.api.gax.paging.Page; +import com.google.auth.Credentials; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Notification; +import com.google.cloud.storage.NotificationInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobListOption; +import com.google.cloud.storage.StorageException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; +import java.util.regex.Pattern; +import org.apache.beam.it.common.ResourceManager; +import org.apache.beam.it.gcp.artifacts.Artifact; +import org.apache.beam.it.gcp.artifacts.ArtifactClient; +import org.apache.beam.it.gcp.artifacts.GcsArtifact; +import org.apache.beam.it.gcp.artifacts.utils.ArtifactUtils; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.junit.rules.TestName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Client for working with test artifacts stored in Google Cloud Storage (GCS). + * + *

    Tests should store this as a static value of the class and call {@link + * ArtifactClient#cleanupAll()} in the {@code @AfterClass} method. + */ +public final class GcsResourceManager implements ArtifactClient, ResourceManager { + private static final Logger LOG = LoggerFactory.getLogger(GcsResourceManager.class); + private final List managedTempDirs = new ArrayList<>(); + private final List notificationList = new ArrayList<>(); + + private final Storage client; + private final String bucket; + private final String testClassName; + private final String runId; + + public GcsResourceManager(Builder builder) { + this.client = ArtifactUtils.createStorageClient(builder.credentials); + this.bucket = builder.bucket; + this.testClassName = builder.testClassName; + this.runId = ArtifactUtils.createRunId(); + + managedTempDirs.add(joinPathParts(testClassName, runId)); + } + + @VisibleForTesting + GcsResourceManager(Storage client, String bucket, String testClassName) { + this.client = client; + this.bucket = bucket; + this.testClassName = testClassName; + this.runId = ArtifactUtils.createRunId(); + + managedTempDirs.add(joinPathParts(testClassName, runId)); + } + + /** Returns a new {@link Builder} for configuring a client. */ + public static Builder builder(String bucket, String testClassName, Credentials credentials) { + checkArgument(!bucket.equals("")); + checkArgument(!testClassName.equals("")); + + return new Builder(bucket, testClassName, credentials); + } + + @Override + public String runId() { + return runId; + } + + @Override + public String getPathForArtifact(String artifactName) { + return joinPathParts(testClassName, runId, artifactName); + } + + @Override + public Artifact createArtifact(String artifactName, String contents) { + return this.createArtifact(artifactName, contents.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public Artifact createArtifact(String artifactName, byte[] contents) { + String path = joinPathParts(testClassName, runId, artifactName); + return handleCreate(path, contents); + } + + @Override + public Artifact uploadArtifact(String artifactName, String localPath) throws IOException { + return uploadArtifact(artifactName, Paths.get(localPath)); + } + + @Override + public Artifact uploadArtifact(String artifactName, Path localPath) throws IOException { + LOG.info( + "Uploading '{}' to file '{}' under '{}'", + localPath, + artifactName, + joinPathParts(testClassName, runId)); + return createArtifact(artifactName, Files.readAllBytes(localPath)); + } + + /** + * Copies a file from a local path to a specified object name in Google Cloud Storage. + * + * @param localPath the path of the file to be copied. + * @param objectName the name of the object to be created in Google Cloud Storage. + * @return the URI of the copied object in Google Cloud Storage. + * @throws IOException if there is an error reading the file at the specified local path. + */ + public Artifact copyFileToGcs(Path localPath, String objectName) throws IOException { + return createArtifact(objectName, Files.readAllBytes(localPath)); + } + + /** + * Helper for creating an artifact. + * + * @param path the full path under the bucket + * @param contents the contents of the artifact + * @return a representation of the artifact + */ + private Artifact handleCreate(String path, byte[] contents) { + LOG.info("Uploading {} bytes to '{}' under bucket '{}'", contents.length, path, bucket); + + BlobId id = BlobId.of(bucket, path); + BlobInfo info = BlobInfo.newBuilder(id).build(); + Blob blob = client.create(info, contents); + LOG.info( + "Successfully uploaded {} bytes to '{}' under bucket '{}'", contents.length, path, bucket); + + return new GcsArtifact(blob); + } + + @Override + public List listArtifacts(TestName testName, Pattern regex) { + return listArtifacts(testName.getMethodName(), regex); + } + + @Override + public List listArtifacts(String prefix, Pattern regex) { + String listFrom = joinPathParts(testClassName, runId, prefix); + LOG.info( + "Listing everything under 'gs://{}/{}' that matches '{}'", + bucket, + listFrom, + regex.pattern()); + + List matched = new ArrayList<>(); + Page firstPage = getFirstPage(listFrom); + consumePages( + firstPage, + blobs -> { + for (Blob blob : blobs) { + if (regex.matcher(blob.getName()).matches()) { + matched.add(new GcsArtifact(blob)); + } + } + }); + + return matched; + } + + /** + * Creates a new notification for the given topic and GCS prefix. + * + * @param topicName the name of the Pub/Sub topic to which the notification should be sent. + * @param gcsPrefix the prefix of the object names to which the notification applies. + * @return the created notification. + */ + public Notification createNotification(String topicName, String gcsPrefix) { + NotificationInfo notificationInfo = + NotificationInfo.newBuilder(topicName) + .setEventTypes(NotificationInfo.EventType.OBJECT_FINALIZE) + .setObjectNamePrefix(gcsPrefix) + .setPayloadFormat(NotificationInfo.PayloadFormat.JSON_API_V1) + .build(); + try { + Notification notification = client.createNotification(bucket, notificationInfo); + LOG.info("Successfully created notification {}", notification); + notificationList.add(notification); + return notification; + } catch (StorageException e) { + throw new RuntimeException( + String.format( + "Unable to create notification for bucket %s. Notification: %s", + bucket, notificationInfo), + e); + } + } + + /** + * Register a temporary directory that will be cleaned up after test. + * + * @param dirName name of the temporary directory + */ + public void registerTempDir(String dirName) { + managedTempDirs.add(dirName); + } + + @Override + public synchronized void cleanupAll() { + if (notificationList.size() > 0) { + for (Notification notification : notificationList) { + client.deleteNotification(bucket, notification.getNotificationId()); + } + } + + if (managedTempDirs.size() > 0) { + LOG.info("managed temp dir size : {}", managedTempDirs.size()); + for (String tempDir : managedTempDirs) { + LOG.info("Cleaning up everything under '{}' under bucket '{}'", tempDir, bucket); + Page firstPage = getFirstPage(tempDir); + consumePages( + firstPage, + blobs -> { + // For testability, use the Iterable overload + List blobIds = new ArrayList<>(); + for (Blob blob : blobs) { + blobIds.add(blob.getBlobId()); + } + if (blobIds.isEmpty()) { + return; + } + + List deleted = client.delete(blobIds); + for (int i = 0; i < deleted.size(); ++i) { + if (deleted.get(i)) { + LOG.debug("Blob '{}' was deleted", blobIds.get(i).getName()); + } else { + LOG.warn("Blob '{}' not deleted", blobIds.get(i).getName()); + } + } + }); + } + } + managedTempDirs.clear(); + notificationList.clear(); + } + + private void consumePages(Page firstPage, Consumer> consumeBlobs) { + Page currentPage = firstPage; + while (true) { + consumeBlobs.accept(currentPage.getValues()); + if (currentPage.hasNextPage()) { + currentPage = currentPage.getNextPage(); + } else { + break; + } + } + } + + private Page getFirstPage(String prefix) { + return client.list(bucket, BlobListOption.prefix(prefix)); + } + + private static String joinPathParts(String... parts) { + return String.join("/", parts); + } + + /** Builder for {@link GcsResourceManager}. */ + public static final class Builder { + private final String bucket; + private final String testClassName; + private Credentials credentials; + + private Builder(String bucket, String testClassName, Credentials credentials) { + this.bucket = bucket; + this.testClassName = testClassName; + this.credentials = credentials; + } + + public Builder setCredentials(Credentials credentials) { + this.credentials = credentials; + return this; + } + + // TODO(zhoufek): Let users control the page size and other configurations + + public GcsResourceManager build() { + return new GcsResourceManager(this); + } + } +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/storage/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/storage/package-info.java new file mode 100644 index 0000000000000..efcea3aba2670 --- /dev/null +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/storage/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing Google Cloud Storage resources within integration tests. */ +package org.apache.beam.it.gcp.storage; diff --git a/it/google-cloud-platform/src/main/resources/test-artifact.json b/it/google-cloud-platform/src/main/resources/test-artifact.json new file mode 100644 index 0000000000000..551c80d14a660 --- /dev/null +++ b/it/google-cloud-platform/src/main/resources/test-artifact.json @@ -0,0 +1 @@ +["This is a test artifact."] \ No newline at end of file diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/WordCountIT.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/WordCountIT.java new file mode 100644 index 0000000000000..511c322b8c4fc --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/WordCountIT.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import java.io.IOException; +import java.time.Duration; +import java.util.Arrays; +import org.apache.beam.it.common.PipelineLauncher.LaunchConfig; +import org.apache.beam.it.common.PipelineLauncher.LaunchInfo; +import org.apache.beam.it.common.PipelineLauncher.Sdk; +import org.apache.beam.it.common.PipelineOperator.Result; +import org.apache.beam.sdk.io.TextIO; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Count; +import org.apache.beam.sdk.transforms.Filter; +import org.apache.beam.sdk.transforms.FlatMapElements; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.TypeDescriptors; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class WordCountIT extends IOLoadTestBase { + + @Rule public TestPipeline wcPipeline = TestPipeline.create(); + + @Before + public void setup() { + buildPipeline(); + } + + @Test + public void testWordCountDataflow() throws IOException { + LaunchConfig options = + LaunchConfig.builder("test-wordcount") + .setSdk(Sdk.JAVA) + .setPipeline(wcPipeline) + .addParameter("runner", "DataflowRunner") + .build(); + + LaunchInfo launchInfo = pipelineLauncher.launch(project, region, options); + assertThatPipeline(launchInfo).isRunning(); + Result result = + pipelineOperator.waitUntilDone(createConfig(launchInfo, Duration.ofMinutes(20))); + assertThatResult(result).isLaunchFinished(); + } + + /** Build WordCount pipeline. */ + private void buildPipeline() { + wcPipeline + .apply(TextIO.read().from("gs://apache-beam-samples/shakespeare/kinglear.txt")) + .apply( + FlatMapElements.into(TypeDescriptors.strings()) + .via((String line) -> Arrays.asList(line.split("[^\\p{L}]+")))) + .apply(Filter.by((String word) -> !word.isEmpty())) + .apply(Count.perElement()) + .apply( + MapElements.into(TypeDescriptors.strings()) + .via( + (KV wordCount) -> + wordCount.getKey() + ": " + wordCount.getValue())) + .apply(TextIO.write().to("wordcounts")); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/artifacts/GcsArtifactTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/artifacts/GcsArtifactTest.java new file mode 100644 index 0000000000000..59b7931b603a0 --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/artifacts/GcsArtifactTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.artifacts; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.when; + +import com.google.cloud.storage.Blob; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Unit tests for {@link GcsArtifact}. */ +@RunWith(JUnit4.class) +public class GcsArtifactTest { + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock private Blob blob; + private GcsArtifact artifact; + + @Before + public void setUp() { + artifact = new GcsArtifact(blob); + } + + @Test + public void testId() { + String id = "test-id"; + when(blob.getGeneratedId()).thenReturn(id); + assertThat(artifact.id()).isEqualTo(id); + } + + @Test + public void testName() { + String name = "test-name"; + when(blob.getName()).thenReturn(name); + assertThat(artifact.name()).isEqualTo(name); + } + + @Test + public void testContents() { + byte[] contents = new byte[] {0, 1, 2}; + when(blob.getContent()).thenReturn(contents); + assertThat(artifact.contents()).isEqualTo(contents); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/artifacts/utils/ArtifactUtilsTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/artifacts/utils/ArtifactUtilsTest.java new file mode 100644 index 0000000000000..491d32d0ce6ae --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/artifacts/utils/ArtifactUtilsTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.artifacts.utils; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Artifacts for {@link ArtifactUtils}. */ +@RunWith(JUnit4.class) +public final class ArtifactUtilsTest { + + // Not matching exact date, since it may fail if the test runs close enough to the change of + // date. + private static final String TEST_DIR_REGEX = + "\\d{8}-[a-fA-F0-9]{8}-([a-fA-F0-9]{4}-){3}[a-fA-F0-9]{12}"; + + @Test + public void testCreateTestDirName() { + assertThat(ArtifactUtils.createRunId()).matches(TEST_DIR_REGEX); + } + + @Test + public void testGetFullGcsPath() { + assertThat(ArtifactUtils.getFullGcsPath("bucket", "dir1", "dir2", "file")) + .isEqualTo("gs://bucket/dir1/dir2/file"); + } + + @Test + public void testGetFullGcsPathOnlyBucket() { + assertThat(ArtifactUtils.getFullGcsPath("bucket")).isEqualTo("gs://bucket"); + } + + @Test + public void testGetFullGcsPathEmpty() { + assertThrows(IllegalArgumentException.class, ArtifactUtils::getFullGcsPath); + } + + @Test + public void testGetFullGcsPathOneNullValue() { + assertThrows( + IllegalArgumentException.class, + () -> ArtifactUtils.getFullGcsPath("bucket", null, "dir2", "file")); + } + + @Test + public void testGetFullGcsPathOneEmptyValue() { + assertThrows( + IllegalArgumentException.class, + () -> ArtifactUtils.getFullGcsPath("bucket", "", "dir2", "file")); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOLT.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOLT.java new file mode 100644 index 0000000000000..a9ae68142778e --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOLT.java @@ -0,0 +1,457 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigquery; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.api.services.bigquery.model.TableFieldSchema; +import com.google.api.services.bigquery.model.TableRow; +import com.google.api.services.bigquery.model.TableSchema; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.text.ParseException; +import java.time.Duration; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.TestProperties; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.IOLoadTestBase; +import org.apache.beam.sdk.io.Read; +import org.apache.beam.sdk.io.gcp.bigquery.AvroWriteRequest; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; +import org.apache.beam.sdk.io.synthetic.SyntheticBoundedSource; +import org.apache.beam.sdk.io.synthetic.SyntheticSourceOptions; +import org.apache.beam.sdk.options.StreamingOptions; +import org.apache.beam.sdk.options.ValueProvider; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.testing.TestPipelineOptions; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +/** + * BigQueryIO performance tests. + * + *

    Example trigger command for all tests: + * + *

    + * mvn test -pl it/google-cloud-platform -am -Dtest="BigQueryIOLT" -Dproject=[gcpProject] \
    + * -DartifactBucket=[temp bucket] -DfailIfNoTests=false
    + * 
    + * + *

    Example trigger command for specific test running on Dataflow runner: + * + *

    + * mvn test -pl it/google-cloud-platform -am -Dtest="BigQueryIOLT#testAvroFileLoadsWriteThenRead" \
    + * -Dconfiguration=medium -Dproject=[gcpProject] -DartifactBucket=[temp bucket] -DfailIfNoTests=false
    + * 
    + * + *

    Example trigger command for specific test and custom data configuration: + * + *

    mvn test -pl it/google-cloud-platform -am \
    + * -Dconfiguration="{\"numRecords\":1000,\"valueSizeBytes\":1000,\"numColumns\":100,\"pipelineTimeout\":2,\"runner\":\"DataflowRunner\"}" \
    + * -Dtest="BigQueryIOLT#testAvroFileLoadsWriteThenRead" -Dconfiguration=local -Dproject=[gcpProject] \
    + * -DartifactBucket=[temp bucket] -DfailIfNoTests=false
    + * 
    + */ +public final class BigQueryIOLT extends IOLoadTestBase { + + private static BigQueryResourceManager resourceManager; + private static String tableQualifier; + private static final String READ_ELEMENT_METRIC_NAME = "read_count"; + private Configuration configuration; + private String tempLocation; + private TableSchema schema; + + @Rule public TestPipeline writePipeline = TestPipeline.create(); + @Rule public TestPipeline readPipeline = TestPipeline.create(); + + @BeforeClass + public static void beforeClass() { + resourceManager = + BigQueryResourceManager.builder("io-bigquery-lt", project, CREDENTIALS).build(); + resourceManager.createDataset(region); + } + + @Before + public void setup() { + // generate a random table name + String tableName = + "io-bq-table-" + + DateTimeFormatter.ofPattern("MMddHHmmssSSS") + .withZone(ZoneId.of("UTC")) + .format(java.time.Instant.now()) + + UUID.randomUUID().toString().substring(0, 10); + tableQualifier = String.format("%s:%s.%s", project, resourceManager.getDatasetId(), tableName); + + // parse configuration + String testConfig = + TestProperties.getProperty("configuration", "local", TestProperties.Type.PROPERTY); + configuration = TEST_CONFIGS_PRESET.get(testConfig); + if (configuration == null) { + try { + configuration = Configuration.fromJsonString(testConfig, Configuration.class); + } catch (IOException e) { + throw new IllegalArgumentException( + String.format( + "Unknown test configuration: [%s]. Pass to a valid configuration json, or use" + + " config presets: %s", + testConfig, TEST_CONFIGS_PRESET.keySet())); + } + } + + // prepare schema + List fields = new ArrayList<>(configuration.numColumns); + for (int idx = 0; idx < configuration.numColumns; ++idx) { + fields.add(new TableFieldSchema().setName("data_" + idx).setType("BYTES")); + } + schema = new TableSchema().setFields(fields); + + // tempLocation needs to be set for bigquery IO writes + if (!Strings.isNullOrEmpty(tempBucketName)) { + tempLocation = String.format("gs://%s/temp/", tempBucketName); + writePipeline.getOptions().as(TestPipelineOptions.class).setTempRoot(tempLocation); + writePipeline.getOptions().setTempLocation(tempLocation); + readPipeline.getOptions().as(TestPipelineOptions.class).setTempRoot(tempLocation); + readPipeline.getOptions().setTempLocation(tempLocation); + } + } + + @AfterClass + public static void tearDownClass() { + ResourceManagerUtils.cleanResources(resourceManager); + } + + private static final Map TEST_CONFIGS_PRESET; + + static { + try { + TEST_CONFIGS_PRESET = + ImmutableMap.of( + "local", + Configuration.fromJsonString( + "{\"numRecords\":1000,\"valueSizeBytes\":1000,\"pipelineTimeout\":2,\"runner\":\"DirectRunner\"}", + Configuration.class), // 1 MB + "medium", + Configuration.fromJsonString( + "{\"numRecords\":10000000,\"valueSizeBytes\":1000,\"pipelineTimeout\":20,\"runner\":\"DataflowRunner\"}", + Configuration.class), // 10 GB + "large", + Configuration.fromJsonString( + "{\"numRecords\":100000000,\"valueSizeBytes\":1000,\"pipelineTimeout\":80,\"runner\":\"DataflowRunner\"}", + Configuration.class) // 100 GB + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Test + public void testAvroFileLoadsWriteThenRead() throws IOException { + configuration.writeFormat = "AVRO"; + configuration.writeMethod = "FILE_LOADS"; + testWriteAndRead(); + } + + @Test + public void testJsonFileLoadsWriteThenRead() throws IOException { + configuration.writeFormat = "JSON"; + configuration.writeMethod = "FILE_LOADS"; + testWriteAndRead(); + } + + @Test + @Ignore("Avro streaming write is not supported as of Beam v2.45.0") + public void testAvroStreamingWriteThenRead() throws IOException { + configuration.writeFormat = "AVRO"; + configuration.writeMethod = "STREAMING_INSERTS"; + testWriteAndRead(); + } + + @Test + public void testJsonStreamingWriteThenRead() throws IOException { + configuration.writeFormat = "JSON"; + configuration.writeMethod = "STREAMING_INSERTS"; + testWriteAndRead(); + } + + @Test + public void testStorageAPIWriteThenRead() throws IOException { + configuration.readMethod = "DIRECT_READ"; + configuration.writeFormat = "AVRO"; + configuration.writeMethod = "STORAGE_WRITE_API"; + testWriteAndRead(); + } + + /** Run integration test with configurations specified by TestProperties. */ + public void testWriteAndRead() throws IOException { + WriteFormat writeFormat = WriteFormat.valueOf(configuration.writeFormat); + BigQueryIO.Write writeIO = null; + switch (writeFormat) { + case AVRO: + writeIO = + BigQueryIO.write() + .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE) + .withAvroFormatFunction( + new AvroFormatFn( + configuration.numColumns, + !("STORAGE_WRITE_API".equalsIgnoreCase(configuration.writeMethod)))); + + break; + case JSON: + writeIO = + BigQueryIO.write() + .withSuccessfulInsertsPropagation(false) + .withFormatFunction(new JsonFormatFn(configuration.numColumns)); + break; + } + testWrite(writeIO); + testRead(); + } + + private void testWrite(BigQueryIO.Write writeIO) throws IOException { + BigQueryIO.Write.Method method = BigQueryIO.Write.Method.valueOf(configuration.writeMethod); + if (method == BigQueryIO.Write.Method.STREAMING_INSERTS) { + writePipeline.getOptions().as(StreamingOptions.class).setStreaming(true); + } + writePipeline + .apply("Read from source", Read.from(new SyntheticBoundedSource(configuration))) + .apply("Map records", ParDo.of(new MapKVToV())) + .apply( + "Write to BQ", + writeIO + .to(tableQualifier) + .withMethod(method) + .withSchema(schema) + .withCustomGcsTempLocation(ValueProvider.StaticValueProvider.of(tempLocation))); + + PipelineLauncher.LaunchConfig options = + PipelineLauncher.LaunchConfig.builder("write-bigquery") + .setSdk(PipelineLauncher.Sdk.JAVA) + .setPipeline(writePipeline) + .addParameter("runner", configuration.runner) + .build(); + + PipelineLauncher.LaunchInfo launchInfo = pipelineLauncher.launch(project, region, options); + PipelineOperator.Result result = + pipelineOperator.waitUntilDone( + createConfig(launchInfo, Duration.ofMinutes(configuration.pipelineTimeout))); + + // Fail the test if pipeline failed. + assertNotEquals(PipelineOperator.Result.LAUNCH_FAILED, result); + + // export metrics + MetricsConfiguration metricsConfig = + MetricsConfiguration.builder() + .setInputPCollection("Map records.out0") + .setInputPCollectionV2("Map records/ParMultiDo(MapKVToV).out0") + .build(); + try { + exportMetricsToBigQuery(launchInfo, getMetrics(launchInfo, metricsConfig)); + } catch (ParseException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + private void testRead() throws IOException { + BigQueryIO.TypedRead.Method method = + BigQueryIO.TypedRead.Method.valueOf(configuration.readMethod); + + readPipeline + .apply("Read from BQ", BigQueryIO.readTableRows().from(tableQualifier).withMethod(method)) + .apply("Counting element", ParDo.of(new CountingFn<>(READ_ELEMENT_METRIC_NAME))); + + PipelineLauncher.LaunchConfig options = + PipelineLauncher.LaunchConfig.builder("read-bigquery") + .setSdk(PipelineLauncher.Sdk.JAVA) + .setPipeline(readPipeline) + .addParameter("runner", configuration.runner) + .build(); + + PipelineLauncher.LaunchInfo launchInfo = pipelineLauncher.launch(project, region, options); + PipelineOperator.Result result = + pipelineOperator.waitUntilDone( + createConfig(launchInfo, Duration.ofMinutes(configuration.pipelineTimeout))); + + // Fail the test if pipeline failed or timeout. + assertThatResult(result).isLaunchFinished(); + + // check metrics + double numRecords = + pipelineLauncher.getMetric( + project, + region, + launchInfo.jobId(), + getBeamMetricsName(PipelineMetricsType.COUNTER, READ_ELEMENT_METRIC_NAME)); + assertEquals(configuration.numRecords, numRecords, 0.5); + + // export metrics + MetricsConfiguration metricsConfig = + MetricsConfiguration.builder() + .setOutputPCollection("Counting element.out0") + .setOutputPCollectionV2("Counting element/ParMultiDo(Counting).out0") + .build(); + try { + exportMetricsToBigQuery(launchInfo, getMetrics(launchInfo, metricsConfig)); + } catch (ParseException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + private static class MapKVToV extends DoFn, byte[]> { + @ProcessElement + public void process(ProcessContext context) { + context.output(Objects.requireNonNull(context.element()).getValue()); + } + } + + abstract static class FormatFn implements SerializableFunction { + protected final int numColumns; + + public FormatFn(int numColumns) { + this.numColumns = numColumns; + } + } + + private static class AvroFormatFn extends FormatFn, GenericRecord> { + + protected final boolean isWrapBytes; + + public AvroFormatFn(int numColumns, boolean isWrapBytes) { + super(numColumns); + this.isWrapBytes = isWrapBytes; + } + + // TODO(https://github.com/apache/beam/issues/26408) eliminate this method once the Beam issue + // resolved + private Object maybeWrapBytes(byte[] input) { + if (isWrapBytes) { + return ByteBuffer.wrap(input); + } else { + return input; + } + } + + @Override + public GenericRecord apply(AvroWriteRequest writeRequest) { + byte[] data = Objects.requireNonNull(writeRequest.getElement()); + GenericRecord record = new GenericData.Record(writeRequest.getSchema()); + if (numColumns == 1) { + // only one column, just wrap incoming bytes + record.put("data_0", maybeWrapBytes(data)); + } else { + // otherwise, distribute bytes + int bytePerCol = data.length / numColumns; + int curIdx = 0; + for (int idx = 0; idx < numColumns - 1; ++idx) { + record.put( + "data_" + idx, maybeWrapBytes(Arrays.copyOfRange(data, curIdx, curIdx + bytePerCol))); + curIdx += bytePerCol; + } + record.put( + "data_" + (numColumns - 1), + maybeWrapBytes(Arrays.copyOfRange(data, curIdx, data.length))); + } + return record; + } + } + + private static class JsonFormatFn extends FormatFn { + + public JsonFormatFn(int numColumns) { + super(numColumns); + } + + @Override + public TableRow apply(byte[] input) { + TableRow tableRow = new TableRow(); + Base64.Encoder encoder = Base64.getEncoder(); + if (numColumns == 1) { + // only one column, just wrap incoming bytes + tableRow.set("data_0", encoder.encodeToString(input)); + } else { + // otherwise, distribute bytes + int bytePerCol = input.length / numColumns; + int curIdx = 0; + for (int idx = 0; idx < numColumns - 1; ++idx) { + tableRow.set( + "data_" + idx, + encoder.encodeToString(Arrays.copyOfRange(input, curIdx, curIdx + bytePerCol))); + curIdx += bytePerCol; + } + tableRow.set( + "data_" + (numColumns - 1), + encoder.encodeToString(Arrays.copyOfRange(input, curIdx, input.length))); + } + return tableRow; + } + } + + private enum WriteFormat { + AVRO, + JSON + } + + /** Options for Bigquery IO load test. */ + static class Configuration extends SyntheticSourceOptions { + + /** + * Number of columns of each record. The column size is equally distributed as + * valueSizeBytes/numColumns. + */ + @JsonProperty public int numColumns = 1; + + /** Pipeline timeout in minutes. Must be a positive value. */ + @JsonProperty public int pipelineTimeout = 20; + + /** Runner specified to run the pipeline. */ + @JsonProperty public String runner = "DirectRunner"; + + /** BigQuery read method: DEFAULT/DIRECT_READ/EXPORT. */ + @JsonProperty public String readMethod = "DEFAULT"; + + /** BigQuery write method: DEFAULT/FILE_LOADS/STREAMING_INSERTS/STORAGE_WRITE_API. */ + @JsonProperty public String writeMethod = "DEFAULT"; + + /** BigQuery write format: AVRO/JSON. */ + @JsonProperty public String writeFormat = "AVRO"; + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerTest.java new file mode 100644 index 0000000000000..e9cd252387559 --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerTest.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigquery; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.Dataset; +import com.google.cloud.bigquery.DatasetId; +import com.google.cloud.bigquery.DatasetInfo; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; +import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Unit tests for {@link BigQueryResourceManager}. */ +@RunWith(JUnit4.class) +public class BigQueryResourceManagerTest { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private BigQuery bigQuery; + + private Schema schema; + private RowToInsert rowToInsert; + private static final String TABLE_NAME = "table-name"; + private static final String DATASET_ID = "dataset-id"; + private static final String TEST_ID = "test-id"; + private static final String PROJECT_ID = "test-project"; + private static final String REGION = "us-central1"; + + private BigQueryResourceManager testManager; + + @Before + public void setUp() { + schema = Schema.of(Field.of("name", StandardSQLTypeName.STRING)); + rowToInsert = RowToInsert.of("1", ImmutableMap.of("name", "Jake")); + testManager = new BigQueryResourceManager(TEST_ID, PROJECT_ID, bigQuery); + } + + @Test + public void testGetProjectIdReturnsCorrectValue() { + assertThat(new BigQueryResourceManager(TEST_ID, PROJECT_ID, bigQuery).getProjectId()) + .isEqualTo(PROJECT_ID); + } + + @Test + public void testGetDatasetIdReturnsCorrectValue() { + BigQueryResourceManager tm = BigQueryResourceManager.builder(TEST_ID, PROJECT_ID, null).build(); + + assertThat(tm.getDatasetId()).matches(TEST_ID.replace('-', '_') + "_\\d{8}_\\d{6}_\\d{6}"); + } + + @Test + public void testCreateDatasetShouldThrowErrorWhenDatasetCreateFails() { + when(bigQuery.create(any(DatasetInfo.class))).thenThrow(RuntimeException.class); + + assertThrows( + BigQueryResourceManagerException.class, () -> testManager.createDataset(DATASET_ID)); + } + + @Test + public void testCreateDatasetShouldNotCreateDatasetWhenDatasetAlreadyExists() { + testManager.createDataset(DATASET_ID); + + assertThrows(IllegalStateException.class, () -> testManager.createDataset(DATASET_ID)); + } + + @Test + public void testCreateDatasetShouldWorkWhenBigQueryDoesNotThrowAnyError() { + testManager.createDataset(DATASET_ID); + verify(bigQuery).create(any(DatasetInfo.class)); + } + + @Test + public void testCreateTableShouldThrowErrorWhenTableNameIsNotValid() { + assertThrows(IllegalArgumentException.class, () -> testManager.createTable("", schema)); + } + + @Test + public void testCreateTableShouldThrowErrorWhenSchemaIsNull() { + assertThrows(IllegalArgumentException.class, () -> testManager.createTable(TABLE_NAME, null)); + } + + @Test + public void testCreateTableShouldCreateDatasetWhenDatasetDoesNotExist() { + when(bigQuery.create(any(DatasetInfo.class)).getDatasetId().getDataset()) + .thenReturn(DATASET_ID); + when(bigQuery.getTable(any())).thenReturn(null); + + testManager.createTable(TABLE_NAME, schema); + + verify(bigQuery).create(any(DatasetInfo.class)); + verify(bigQuery).create(any(TableInfo.class)); + } + + @Test + public void testCreateTableShouldThrowErrorWhenCreateFails() { + testManager.createDataset(DATASET_ID); + when(bigQuery.create(any(TableInfo.class))).thenThrow(BigQueryException.class); + + assertThrows( + BigQueryResourceManagerException.class, () -> testManager.createTable(TABLE_NAME, schema)); + } + + @Test + public void testCreateTableShouldThrowErrorWhenTableExists() { + testManager.createDataset(DATASET_ID); + + when(bigQuery.getTable(any())).thenReturn(any()); + + assertThrows( + BigQueryResourceManagerException.class, () -> testManager.createTable(TABLE_NAME, schema)); + } + + @Test + public void testCreateTableShouldWorkWhenBigQueryDoesNotThrowAnyError() { + when(bigQuery.create(any(DatasetInfo.class)).getDatasetId().getDataset()) + .thenReturn(DATASET_ID); + when(bigQuery.getTable(any())).thenReturn(null); + + testManager.createTable(TABLE_NAME, schema); + + verify(bigQuery).create(any(DatasetInfo.class)); + } + + @Test + public void testWriteShouldThrowErrorWhenDatasetDoesNotExist() { + assertThrows(IllegalStateException.class, () -> testManager.write(TABLE_NAME, rowToInsert)); + assertThrows( + IllegalStateException.class, + () -> testManager.write(TABLE_NAME, ImmutableList.of(rowToInsert))); + } + + @Test + public void testWriteShouldThrowErrorWhenTableDoesNotExist() { + testManager.createDataset(DATASET_ID); + + when(bigQuery.create(any(DatasetInfo.class)).get(anyString())).thenReturn(null); + + assertThrows(IllegalStateException.class, () -> testManager.write(TABLE_NAME, rowToInsert)); + assertThrows( + IllegalStateException.class, + () -> testManager.write(TABLE_NAME, ImmutableList.of(rowToInsert))); + } + + @Test + public void testWriteShouldThrowErrorWhenInsertFails() { + Dataset mockDataset = bigQuery.create(any(DatasetInfo.class)); + when(mockDataset.getDatasetId().getDataset()).thenReturn(DATASET_ID); + + testManager.createDataset(DATASET_ID); + + when(mockDataset.get(anyString()).insert(any())).thenThrow(BigQueryException.class); + + assertThrows( + BigQueryResourceManagerException.class, () -> testManager.write(TABLE_NAME, rowToInsert)); + assertThrows( + BigQueryResourceManagerException.class, + () -> testManager.write(TABLE_NAME, ImmutableList.of(rowToInsert))); + } + + @Test + public void testWriteShouldWorkWhenBigQueryDoesNotThrowAnyError() { + when(bigQuery.create(any(DatasetInfo.class)).getDatasetId().getDataset()) + .thenReturn(DATASET_ID); + when(bigQuery.getTable(any())).thenReturn(null); + + testManager.createTable(TABLE_NAME, schema); + + testManager.write(TABLE_NAME, rowToInsert); + testManager.write(TABLE_NAME, ImmutableList.of(rowToInsert)); + } + + @Test + public void testReadTableShouldThrowErrorWhenDatasetDoesNotExist() { + assertThrows(IllegalStateException.class, () -> testManager.readTable(TABLE_NAME)); + } + + @Test + public void testReadTableShouldThrowErrorWhenTableDoesNotExist() { + testManager.createDataset(REGION); + when(bigQuery.create(any(DatasetInfo.class)).get(anyString())).thenReturn(null); + assertThrows(IllegalStateException.class, () -> testManager.readTable(TABLE_NAME)); + } + + @Test + public void testReadTableShouldThrowErrorWhenBigQueryFailsToReadRows() + throws InterruptedException { + when(bigQuery.create(any(DatasetInfo.class)).getDatasetId().getDataset()) + .thenReturn(DATASET_ID); + when(bigQuery.getTable(any())).thenReturn(null); + + testManager.createTable(TABLE_NAME, schema); + + doThrow(BigQueryException.class).when(bigQuery).query(any(QueryJobConfiguration.class)); + + assertThrows(BigQueryResourceManagerException.class, () -> testManager.readTable(TABLE_NAME)); + } + + @Test + public void testReadTableShouldWorkWhenBigQueryDoesNotThrowAnyError() + throws InterruptedException { + when(bigQuery.create(any(DatasetInfo.class)).getDatasetId().getDataset()) + .thenReturn(DATASET_ID); + when(bigQuery.getTable(any())).thenReturn(null); + + testManager.createTable(TABLE_NAME, schema); + + testManager.readTable(TABLE_NAME); + + verify(bigQuery).query(any(QueryJobConfiguration.class)); + } + + @Test + public void testCleanupShouldThrowErrorWhenTableDeleteFails() { + testManager.createDataset(DATASET_ID); + + Table mockTable = mock(Table.class); + when(bigQuery.listTables(any(DatasetId.class)).iterateAll()) + .thenReturn(ImmutableList.of(mockTable)); + when(bigQuery.delete(any(TableId.class))).thenThrow(BigQueryException.class); + + assertThrows(BigQueryResourceManagerException.class, () -> testManager.cleanupAll()); + } + + @Test + public void testCleanupShouldThrowErrorWhenDatasetDeleteFails() { + testManager.createDataset(DATASET_ID); + when(bigQuery.delete(any(DatasetId.class))).thenThrow(BigQueryException.class); + + assertThrows(BigQueryResourceManagerException.class, () -> testManager.cleanupAll()); + } + + @Test + public void testCleanupShouldWorkWhenBigQueryDoesNotThrowAnyError() { + when(bigQuery.create(any(DatasetInfo.class)).getDatasetId().getDataset()) + .thenReturn(DATASET_ID); + + testManager.createDataset(DATASET_ID); + Table mockTable = mock(Table.class, Answers.RETURNS_DEEP_STUBS); + when(bigQuery.listTables(any(DatasetId.class)).iterateAll()) + .thenReturn(ImmutableList.of(mockTable)); + when(mockTable.getTableId().getTable()).thenReturn(TABLE_NAME); + + testManager.cleanupAll(); + + verify(bigQuery).delete(any(TableId.class)); + verify(bigQuery).delete(any(DatasetId.class)); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerUtilsTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerUtilsTest.java new file mode 100644 index 0000000000000..432b7ef7362fd --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerUtilsTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigquery; + +import static org.apache.beam.it.gcp.bigquery.BigQueryResourceManagerUtils.checkValidTableId; +import static org.junit.Assert.assertThrows; + +import java.util.Arrays; +import org.junit.Test; + +/** Unit tests for {@link BigQueryResourceManagerUtils}. */ +public class BigQueryResourceManagerUtilsTest { + + @Test + public void testCheckValidTableIdWhenIdIsTooShort() { + assertThrows(IllegalArgumentException.class, () -> checkValidTableId("")); + } + + @Test + public void testCheckValidTableIdWhenIdIsTooLong() { + char[] chars = new char[1025]; + Arrays.fill(chars, 'a'); + String s = new String(chars); + assertThrows(IllegalArgumentException.class, () -> checkValidTableId(s)); + } + + @Test + public void testCheckValidTableIdWhenIdContainsIllegalCharacter() { + assertThrows(IllegalArgumentException.class, () -> checkValidTableId("table-id%")); + assertThrows(IllegalArgumentException.class, () -> checkValidTableId("ta#ble-id")); + } + + @Test + public void testCheckValidTableIdShouldWorkWhenGivenCorrectId() { + char[] chars = new char[1024]; + Arrays.fill(chars, 'a'); + String s = new String(chars); + + checkValidTableId(s); + checkValidTableId("a"); + checkValidTableId("this-is_a_valid-id-1"); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryStreamingLT.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryStreamingLT.java new file mode 100644 index 0000000000000..4589f79f1aaa6 --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryStreamingLT.java @@ -0,0 +1,643 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigquery; + +import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils.toTableReference; +import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils.toTableSpec; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import com.google.api.core.ApiFuture; +import com.google.api.services.bigquery.model.TableFieldSchema; +import com.google.api.services.bigquery.model.TableRow; +import com.google.api.services.bigquery.model.TableSchema; +import com.google.auto.value.AutoValue; +import com.google.cloud.bigquery.storage.v1.FlushRowsResponse; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.TestProperties; +import org.apache.beam.it.gcp.IOLoadTestBase; +import org.apache.beam.runners.dataflow.DataflowRunner; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; +import org.apache.beam.sdk.io.GenerateSequence; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryOptions; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServicesImpl; +import org.apache.beam.sdk.io.gcp.testing.BigqueryClient; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.testing.TestPipelineOptions; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.PeriodicImpulse; +import org.apache.beam.sdk.transforms.Reshuffle; +import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.TypeDescriptors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Duration; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Load test for the Storage Write API sink + * + *

    This test is set up to first write rows using batch FILE_LOADS mode to a "source of truth" + * table. Afterwards, it will write the same rows in streaming mode with Storage API to a second + * table. Then it will query between these two tables to check that they are identical. There is + * also the option of providing an existing table with the expected data, in which case the test + * will skip the first step. + * + *

    The throughput, length of test (in minutes), and data shape can be changed via pipeline + * options. See the cases in `getOptions()` for examples. + * + *

    This also includes the option of testing the sink's retry resilience by setting the + * `crashIntervalSeconds` System property. This intentionally fails the worker or work item + * periodically and expects the sink to recover appropriately. Note: Metrics are not published when + * this is used. + */ +public class BigQueryStreamingLT extends IOLoadTestBase { + private static final Logger LOG = LoggerFactory.getLogger(BigQueryStreamingLT.class); + + private static final BigqueryClient BQ_CLIENT = new BigqueryClient("BigQueryStreamingLT"); + private static final String BIG_QUERY_DATASET_ID = + "storage_api_sink_load_test_" + System.nanoTime(); + + private TestConfiguration config; + private Integer crashIntervalSeconds; + + @Rule public final transient TestPipeline fileLoadsPipeline = TestPipeline.create(); + @Rule public final transient TestPipeline storageApiPipeline = TestPipeline.create(); + + @BeforeClass + public static void setUpTestClass() throws IOException, InterruptedException { + PipelineOptionsFactory.register(TestPipelineOptions.class); + BQ_CLIENT.createNewDataset(project, BIG_QUERY_DATASET_ID); + } + + @Before + public void setUpTest() { + String testConfig = + TestProperties.getProperty("configuration", "small", TestProperties.Type.PROPERTY); + config = TEST_CONFIGS.get(testConfig); + if (config == null) { + throw new IllegalArgumentException( + String.format( + "Unknown test configuration: [%s]. Known configs: %s", + testConfig, TEST_CONFIGS.keySet())); + } + // tempLocation needs to be set for file loads + if (!Strings.isNullOrEmpty(tempBucketName)) { + String tempLocation = String.format("gs://%s/temp/", tempBucketName); + fileLoadsPipeline.getOptions().as(TestPipelineOptions.class).setTempRoot(tempLocation); + fileLoadsPipeline.getOptions().setTempLocation(tempLocation); + } + + // Set expected table if the property is provided, + @Nullable + String expectedTable = + TestProperties.getProperty("expectedTable", "", TestProperties.Type.PROPERTY); + if (!Strings.isNullOrEmpty(expectedTable)) { + config.toBuilder().setExpectedTable(expectedTable).build(); + } + + crashIntervalSeconds = + Integer.parseInt( + TestProperties.getProperty("crashIntervalSeconds", "-1", TestProperties.Type.PROPERTY)); + } + + @AfterClass + public static void cleanup() { + BQ_CLIENT.deleteDataset(project, BIG_QUERY_DATASET_ID); + } + + private static final Map TEST_CONFIGS = + ImmutableMap.of( + "local", // 300K rows, >3 MB, 1K rows/s, >10KB/s + TestConfiguration.of(5, 5, 2, 1_000, "DirectRunner", null), + "small", // 600K rows, >30 MB, 1K rows/s, >50KB/s + TestConfiguration.of(10, 10, 5, 1_000, "DataflowRunner", null), + "medium", // 6M rows, >1.2 GB, 5K rows/s, >1MB/s + TestConfiguration.of(20, 20, 10, 5_000, "DataflowRunner", null), + "large", // 18M rows, >18 GB, 10K rows/s, >10MB/s + TestConfiguration.of(30, 50, 20, 10_000, "DataflowRunner", null)); + + /** Options for Bigquery IO Streaming load test. */ + @AutoValue + abstract static class TestConfiguration { + /** Rows will be generated for this many minutes. */ + abstract Integer getMinutes(); + + /** Data shape: The byte-size for each field. */ + abstract Integer getByteSizePerField(); + + /** Data shape: The number of fields per row. */ + abstract Integer getNumFields(); + + /** + * Rate of generated elements sent to the sink. Will run with a minimum of 1k rows per second. + */ + abstract Integer getRowsPerSecond(); + + abstract String getRunner(); + + /** + * The expected table to check against for correctness. If unset, the test will run a batch + * FILE_LOADS job and use the resulting table as a source of truth. + */ + @Nullable + abstract String getExpectedTable(); + + static TestConfiguration of( + int numMin, + int byteSizePerField, + int numFields, + int rowsPerSecond, + String runner, + @Nullable String expectedTable) { + return new AutoValue_BigQueryStreamingLT_TestConfiguration.Builder() + .setMinutes(numMin) + .setByteSizePerField(byteSizePerField) + .setNumFields(numFields) + .setRowsPerSecond(rowsPerSecond) + .setRunner(runner) + .setExpectedTable(expectedTable) + .build(); + } + + @AutoValue.Builder + abstract static class Builder { + abstract Builder setMinutes(int numMin); + + abstract Builder setByteSizePerField(int byteSizePerField); + + abstract Builder setNumFields(int numFields); + + abstract Builder setRowsPerSecond(int rowsPerSecond); + + abstract Builder setRunner(String runner); + + abstract Builder setExpectedTable(@Nullable String expectedTable); + + abstract TestConfiguration build(); + } + + abstract Builder toBuilder(); + } + + @Test + public void testExactlyOnceStreaming() throws IOException, InterruptedException { + runTest(BigQueryIO.Write.Method.STORAGE_WRITE_API); + } + + @Test + @Ignore + public void testAtLeastOnceStreaming() throws IOException, InterruptedException { + runTest(BigQueryIO.Write.Method.STORAGE_API_AT_LEAST_ONCE); + } + + public void runTest(BigQueryIO.Write.Method writeMethod) + throws IOException, InterruptedException { + long millis = Duration.standardMinutes(config.getMinutes()).getMillis(); + int rowsPerSecond = Math.max(config.getRowsPerSecond(), 1000); + + // The PeriodicImpulse source will generate an element every this many millis: + int fireInterval = 1; + // Each element from PeriodicImpulse will fan out to this many elements + // (applicable when a high row-per-second rate is set) + long multiplier = rowsPerSecond / 1000; + long totalRows = multiplier * millis / fireInterval; + // If we run with DataflowRunner and have not specified a positive crash duration for the sink, + // this signifies a performance test, and so we publish metrics to a BigQuery dataset + boolean publishMetrics = + config.getRunner().equalsIgnoreCase(DataflowRunner.class.getSimpleName()) + && crashIntervalSeconds <= 0; + + String expectedTable = config.getExpectedTable(); + GenerateTableRow genRow = + new GenerateTableRow(config.getNumFields(), config.getByteSizePerField()); + TableSchema schema = generateTableSchema(config.getNumFields()); + if (Strings.isNullOrEmpty(expectedTable)) { + String fileLoadsDescription = + String.format("fileloads-%s-records", withScaleSymbol(totalRows)); + expectedTable = + String.format("%s.%s.%s", project, BIG_QUERY_DATASET_ID, fileLoadsDescription); + LOG.info( + "No expected table was set. Will run a batch job to load {} rows to {}." + + " This will be used as the source of truth.", + totalRows, + expectedTable); + + fileLoadsPipeline + .apply(GenerateSequence.from(0).to(totalRows)) + .apply( + "Write to source of truth", + BigQueryIO.write() + .to(expectedTable) + .withFormatFunction(genRow) + .withMethod(BigQueryIO.Write.Method.FILE_LOADS) + .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE) + .withSchema(schema)); + + // If running on Dataflow, launch pipeline via launcher utils + if (publishMetrics) { + PipelineLauncher.LaunchConfig options = + PipelineLauncher.LaunchConfig.builder("test-" + fileLoadsDescription) + .setSdk(PipelineLauncher.Sdk.JAVA) + .setPipeline(fileLoadsPipeline) + .addParameter("runner", config.getRunner()) + .build(); + + // Don't use PipelineOperator because we don't want to wait on this batch job + // The streaming job will run in parallel and it will take longer anyways; this job will + // finish by then. + pipelineLauncher.launch(project, region, options); + } else { + fileLoadsPipeline.run(); + } + } + + String atLeastOnce = + writeMethod == BigQueryIO.Write.Method.STORAGE_API_AT_LEAST_ONCE ? "-atleastonce" : ""; + String storageApiDescription = + String.format( + "storageapi%s-load-%sqps-%smin-%stotal", + atLeastOnce, + withScaleSymbol(rowsPerSecond), + config.getMinutes(), + withScaleSymbol(totalRows)); + String destTable = + String.format("%s.%s.%s", project, BIG_QUERY_DATASET_ID, storageApiDescription); + LOG.info( + "Preparing a source generating at a rate of {} rows per second for a period of {} minutes." + + " This results in a total of {} rows written to {}.", + rowsPerSecond, + config.getMinutes(), + totalRows, + destTable); + + PCollection source = + storageApiPipeline + .apply( + PeriodicImpulse.create() + .stopAfter(Duration.millis(millis - 1)) + .withInterval(Duration.millis(fireInterval))) + .apply( + "Extract row IDs", + MapElements.into(TypeDescriptors.longs()) + .via(instant -> instant.getMillis() % totalRows)); + if (multiplier > 1) { + source = + source + .apply( + String.format("One input to %s outputs", multiplier), + ParDo.of(new MultiplierDoFn(multiplier))) + .apply("Reshuffle fanout", Reshuffle.viaRandomKey()); + } + + BigQueryIO.Write storageWriteTransform = + BigQueryIO.write() + .to(destTable) + .withFormatFunction(genRow) + .withMethod(writeMethod) + .withTriggeringFrequency(Duration.standardSeconds(1)) + .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND) + .withSchema(schema); + + // If a crash interval is specified, use our crashing service implementation + if (crashIntervalSeconds > 0) { + LOG.info( + "A crash interval of {} seconds has been set. The Storage API sink will periodically crash.", + crashIntervalSeconds); + storageWriteTransform = + storageWriteTransform.withTestServices( + new CrashingBigQueryServices(crashIntervalSeconds)); + } + source.apply(storageWriteTransform); + + // If we're publishing metrics, launch pipeline via Dataflow launcher utils and export metrics + if (publishMetrics) { + // Set up dataflow job + PipelineLauncher.LaunchConfig storageApiOptions = + PipelineLauncher.LaunchConfig.builder("test-" + storageApiDescription) + .setSdk(PipelineLauncher.Sdk.JAVA) + .setPipeline(storageApiPipeline) + .addParameter("runner", config.getRunner()) + .addParameter("streaming", "true") + .addParameter("experiments", GcpOptions.STREAMING_ENGINE_EXPERIMENT) + .build(); + // Launch job + PipelineLauncher.LaunchInfo storageApiInfo = + pipelineLauncher.launch(project, region, storageApiOptions); + // Wait until the streaming pipeline is finished and drained, get the result. + PipelineOperator.Result storageApiResult = + pipelineOperator.waitUntilDoneAndFinish( + PipelineOperator.Config.builder() + .setJobId(storageApiInfo.jobId()) + .setProject(project) + .setRegion(region) + .setTimeoutAfter(java.time.Duration.ofMinutes(config.getMinutes() * 2L)) + .setCheckAfter(java.time.Duration.ofSeconds(config.getMinutes() * 60 / 20)) + .build()); + // Check the initial launch didn't fail + assertNotEquals(PipelineOperator.Result.LAUNCH_FAILED, storageApiResult); + // Check that the pipeline succeeded + assertEquals( + PipelineLauncher.JobState.DONE, + pipelineLauncher.getJobStatus(project, region, storageApiInfo.jobId())); + + // Export metrics + MetricsConfiguration metricsConfig = + MetricsConfiguration.builder() + .setInputPCollection( + (multiplier > 1) ? "Extract row IDs.out0" : "Reshuffle fanout.out0") + .build(); + try { + exportMetricsToBigQuery(storageApiInfo, getMetrics(storageApiInfo, metricsConfig)); + } catch (Exception e) { + // Just log the error. Don't re-throw because we have accuracy checks that are more + // important below + LOG.error("Encountered an error while exporting metrics to BigQuery:\n{}", e); + } + } + // If we're not publishing metrics, just run the pipeline normally + else { + storageApiPipeline.run().waitUntilFinish(); + } + + LOG.info( + "Write pipeline finished writing to {}. Will now perform accuracy checks against the rows in {}.", + destTable, + expectedTable); + // Filter our structs and arrays because they are not supported when querying with `EXCEPT + // DISTINCT` + String columnNames = + schema.getFields().stream() + .map(TableFieldSchema::getName) + .filter(fieldName -> fieldName.startsWith(FIELD_PREFIX)) + .collect(Collectors.joining(", ")); + checkCorrectness(columnNames, destTable, expectedTable); + // check non-duplication for STORAGE_WRITE_API + if (writeMethod == BigQueryIO.Write.Method.STORAGE_WRITE_API) { + checkNonDuplication(destTable, expectedTable, totalRows); + } + } + + // A BigQueryServices class that is almost identical to BigQueryServicesImpl, except that + // it returns a dataset service implementation that periodically crashes on flush() + private static class CrashingBigQueryServices extends BigQueryServicesImpl { + public final Integer crashIntervalSeconds; + + public CrashingBigQueryServices(Integer crashIntervalSeconds) { + this.crashIntervalSeconds = crashIntervalSeconds; + } + + @Override + public DatasetService getDatasetService(BigQueryOptions options) { + return new CrashingDatasetService(options); + } + + private class CrashingDatasetService extends BigQueryServicesImpl.DatasetServiceImpl { + private Instant lastCrash; + + public CrashingDatasetService(BigQueryOptions bqOptions) { + super(bqOptions); + } + + // We choose flush() to host the crash logic because it's called frequently during + // the span of a Storage Write API pipeline + @Override + public ApiFuture flush(String streamName, long flushOffset) + throws IOException, InterruptedException { + maybeCrash(); + return super.flush(streamName, flushOffset); + } + + // When specified, crash when the interval is met by: + // throwing an exception (failed work item) or + // performing a System exit (worker failure) + private void maybeCrash() { + if (crashIntervalSeconds != -1) { + Instant last = lastCrash; + if (last == null) { + lastCrash = Instant.now(); + } else if (Instant.now().isAfter(last.plusSeconds(crashIntervalSeconds))) { + lastCrash = Instant.now(); + + // Only crash 30% of the time (this is arbitrary) + if (ThreadLocalRandom.current().nextInt(100) < 30) { + // Half the time throw an exception (which fails this specific work item) + // Other half crash the entire worker, which fails all work items on this worker + if (ThreadLocalRandom.current().nextBoolean()) { + throw new RuntimeException( + "Throwing a random exception! This is for testing retry resilience."); + } else { + LOG.error("Crashing this worker! This is for testing retry resilience."); + System.exit(0); + } + } + } + } + } + } + } + + public void checkCorrectness(String columnNames, String destTable, String expectedTable) + throws IOException, InterruptedException { + // Need table spec to be in the format `myproject.mydataset.mytable` to include in BQ queries. + destTable = toTableSpec(toTableReference(destTable)); + expectedTable = toTableSpec(toTableReference(expectedTable)); + + String checkCorrectnessQuery = + String.format( + "WITH \n" + + "storage_api_table AS (SELECT %s FROM `%s`), \n" + + "expected_table AS (SELECT %s FROM `%s`), \n" + + "rows_mismatched AS (SELECT * FROM expected_table EXCEPT DISTINCT SELECT * FROM storage_api_table) \n" + + "SELECT COUNT(*) FROM rows_mismatched", + columnNames, destTable, columnNames, expectedTable); + + LOG.info("Executing query to check correctness:\n{}", checkCorrectnessQuery); + + TableRow queryResponse = + Iterables.getOnlyElement( + BQ_CLIENT.queryUnflattened(checkCorrectnessQuery, "google.com:clouddfe", true, true)); + long result = Long.parseLong((String) queryResponse.get("f0_")); + + LOG.info("Number of mismatched rows: {}", result); + assertEquals( + String.format("Saw %s rows that are missing from %s.", result, destTable), 0, result); + } + + public void checkNonDuplication(String destTable, String expectedTable, long totalRows) + throws IOException, InterruptedException { + String checkDuplicationQuery = + String.format( + "SELECT \n" + + "(SELECT COUNT(*) FROM `%s`) AS actualCount,\n" + + "(SELECT COUNT(*) FROM `%s`) AS expectedCount", + destTable, expectedTable); + + LOG.info("Executing query to check non-duplication:\n{}", checkDuplicationQuery); + + TableRow queryResponse = + Iterables.getOnlyElement( + BQ_CLIENT.queryUnflattened(checkDuplicationQuery, "google.com:clouddfe", true, true)); + long actualCount = Long.parseLong((String) queryResponse.get("actualCount")); + long expectedCount = Long.parseLong((String) queryResponse.get("expectedCount")); + assertEquals( + "Comparing actual table count and expected table count.", expectedCount, actualCount); + assertEquals( + "Comparing actual table count and calculated expected count.", totalRows, actualCount); + } + + // From a value, get the appropriate shortened name that includes the scale + // For example, from 12,345,678 return 12M + public String withScaleSymbol(long value) { + List scales = Arrays.asList("", "K", "M", "B", "T", "Q"); + int scaleIndex = 0; + while (value / 1000 > 0) { + scaleIndex++; + value /= 1000; + } + + return String.format("%s%s", value, scales.get(scaleIndex)); + } + + public static class MultiplierDoFn extends DoFn { + private long multiplier; + + MultiplierDoFn(long multiplier) { + this.multiplier = multiplier; + } + + @ProcessElement + public void processElement(@Element Long element, OutputReceiver outputReceiver) { + for (int i = 0; i < multiplier; i++) { + outputReceiver.output(element); + } + } + } + + static final String FIELD_PREFIX = "byte_field_"; + static final String RECORD_FIELD_PREFIX = "record_" + FIELD_PREFIX; + static final String NESTED_FIELD_PREFIX = "nested_" + FIELD_PREFIX; + static final String REPEATED_FIELD_PREFIX = "repeated_" + FIELD_PREFIX; + + public static TableSchema generateTableSchema(int numFields) { + List fields = new ArrayList<>(numFields); + fields.add(new TableFieldSchema().setType("INTEGER").setName("id")); + int j = 1; + for (int i = 1; i <= numFields; i++) { + TableFieldSchema fieldSchema = new TableFieldSchema(); + // Every 4th field will be a struct, every 5th field will be an array + if (j == 4) { + fieldSchema + .setType("RECORD") + .setName(RECORD_FIELD_PREFIX + i) + .setFields( + Arrays.asList( + new TableFieldSchema().setType("BYTES").setName(NESTED_FIELD_PREFIX + 1), + new TableFieldSchema().setType("BYTES").setName(NESTED_FIELD_PREFIX + 2))); + } else if (j == 5) { + fieldSchema.setType("BYTES").setMode("REPEATED").setName(REPEATED_FIELD_PREFIX + i); + j = 0; + } else { + fieldSchema.setType("BYTES").setName(FIELD_PREFIX + i); + } + j++; + fields.add(fieldSchema); + } + return new TableSchema().setFields(fields); + } + + static class GenerateTableRow implements SerializableFunction { + private final int numFields; + private final int sizePerField; + + public GenerateTableRow(int numFields, int sizePerField) { + assert numFields >= 0; + this.numFields = numFields; + this.sizePerField = sizePerField; + } + + @Override + public TableRow apply(Long rowId) { + TableRow row = new TableRow(); + row.set("id", rowId); + byte[] payload = getPayload(sizePerField, rowId).array(); + int j = 1; + for (int i = 1; i <= numFields; i++) { + // TODO: we can also make the struct and array sizes variable + if (j == 4) { + row.set( + RECORD_FIELD_PREFIX + i, + new TableRow() + .set(NESTED_FIELD_PREFIX + 1, Arrays.copyOfRange(payload, 0, sizePerField / 2)) + .set( + NESTED_FIELD_PREFIX + 2, + Arrays.copyOfRange(payload, sizePerField / 2, sizePerField))); + } else if (j == 5) { + row.set( + REPEATED_FIELD_PREFIX + i, + Arrays.asList( + Arrays.copyOfRange(payload, 0, sizePerField / 3), + Arrays.copyOfRange(payload, sizePerField / 3, sizePerField * 2 / 3), + Arrays.copyOfRange(payload, sizePerField * 2 / 3, sizePerField))); + j = 0; + } else { + row.set(FIELD_PREFIX + i, payload); + } + j++; + } + return row; + } + + private @Nullable ByteBuffer getPayload(int payloadSize, long rowId) { + if (payloadSize <= 0) { + return null; + } + byte[] payload = new byte[payloadSize]; + Random localRandom = ThreadLocal.withInitial(() -> new Random(rowId)).get(); + localRandom.setSeed(rowId); + localRandom.nextBytes(payload); + + return ByteBuffer.wrap(payload); + } + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigTableIOLT.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigTableIOLT.java new file mode 100644 index 0000000000000..e232ed31cb5a3 --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigTableIOLT.java @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigtable; + +import static org.apache.beam.it.gcp.bigtable.BigtableResourceManagerUtils.generateTableId; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import com.google.auto.value.AutoValue; +import com.google.bigtable.v2.Mutation; +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.io.Serializable; +import java.text.ParseException; +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import java.util.Random; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.TestProperties; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.IOLoadTestBase; +import org.apache.beam.sdk.io.GenerateSequence; +import org.apache.beam.sdk.io.gcp.bigtable.BigtableIO; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.testing.TestPipelineOptions; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +/** + * BigTableIO performance tests. + * + *

    Example trigger command for all tests: "mvn test -pl it/google-cloud-platform -am + * -Dtest=BigTableIOLT \ -Dproject=[gcpProject] -DartifactBucket=[temp bucket] + * -DfailIfNoTests=false". + * + *

    Example trigger command for specific test: "mvn test -pl it/google-cloud-platform -am \ + * -Dtest="BigTableIOLT#testWriteAndRead" -Dconfiguration=local -Dproject=[gcpProject] \ + * -DartifactBucket=[temp bucket] -DfailIfNoTests=false". + */ +public class BigTableIOLT extends IOLoadTestBase { + + private static final String COLUMN_FAMILY_NAME = "cf"; + private static final long TABLE_MAX_AGE_MINUTES = 100L; + + private static BigtableResourceManager resourceManager; + private static final String READ_ELEMENT_METRIC_NAME = "read_count"; + private Configuration configuration; + private String tableId; + + @Rule public TestPipeline writePipeline = TestPipeline.create(); + @Rule public TestPipeline readPipeline = TestPipeline.create(); + + @Before + public void setup() throws IOException { + resourceManager = + BigtableResourceManager.builder(testName, project, CREDENTIALS_PROVIDER).build(); + + String testConfig = + TestProperties.getProperty("configuration", "small", TestProperties.Type.PROPERTY); + configuration = TEST_CONFIGS.get(testConfig); + if (configuration == null) { + throw new IllegalArgumentException( + String.format( + "Unknown test configuration: [%s]. Known configs: %s", + testConfig, TEST_CONFIGS.keySet())); + } + // tempLocation needs to be set for bigtable IO writes + if (!Strings.isNullOrEmpty(tempBucketName)) { + String tempLocation = String.format("gs://%s/temp/", tempBucketName); + writePipeline.getOptions().as(TestPipelineOptions.class).setTempRoot(tempLocation); + writePipeline.getOptions().setTempLocation(tempLocation); + readPipeline.getOptions().as(TestPipelineOptions.class).setTempRoot(tempLocation); + readPipeline.getOptions().setTempLocation(tempLocation); + } + } + + @After + public void teardown() { + ResourceManagerUtils.cleanResources(resourceManager); + } + + private static final Map TEST_CONFIGS = + ImmutableMap.of( + "local", Configuration.of(1_000L, 5, "DirectRunner", 1000), // 1MB + "small", Configuration.of(1_000_000L, 20, "DataflowRunner", 1000), // 1 GB + "medium", Configuration.of(10_000_000L, 40, "DataflowRunner", 1000), // 10 GB + "large", Configuration.of(100_000_000L, 100, "DataflowRunner", 1000) // 100 GB + ); + + /** Run integration test with configurations specified by TestProperties. */ + @Test + public void testWriteAndRead() throws IOException { + + tableId = generateTableId(testName); + resourceManager.createTable( + tableId, + ImmutableList.of(COLUMN_FAMILY_NAME), + org.threeten.bp.Duration.ofMinutes(TABLE_MAX_AGE_MINUTES)); + + PipelineLauncher.LaunchInfo writeInfo = testWrite(); + PipelineOperator.Result writeResult = + pipelineOperator.waitUntilDone( + createConfig(writeInfo, Duration.ofMinutes(configuration.getPipelineTimeout()))); + assertNotEquals(PipelineOperator.Result.LAUNCH_FAILED, writeResult); + + PipelineLauncher.LaunchInfo readInfo = testRead(); + PipelineOperator.Result result = + pipelineOperator.waitUntilDone( + createConfig(readInfo, Duration.ofMinutes(configuration.getPipelineTimeout()))); + assertNotEquals(PipelineOperator.Result.LAUNCH_FAILED, result); + assertEquals( + PipelineLauncher.JobState.DONE, + pipelineLauncher.getJobStatus(project, region, readInfo.jobId())); + double numRecords = + pipelineLauncher.getMetric( + project, + region, + readInfo.jobId(), + getBeamMetricsName(PipelineMetricsType.COUNTER, READ_ELEMENT_METRIC_NAME)); + assertEquals(configuration.getNumRows(), numRecords, 0.5); + + // export metrics + MetricsConfiguration metricsConfig = + MetricsConfiguration.builder() + .setInputPCollection("Map records.out0") + .setInputPCollectionV2("Map records/ParMultiDo(MapToBigTableFormat).out0") + .setOutputPCollection("Counting element.out0") + .setOutputPCollectionV2("Counting element/ParMultiDo(Counting).out0") + .build(); + try { + exportMetricsToBigQuery(writeInfo, getMetrics(writeInfo, metricsConfig)); + exportMetricsToBigQuery(readInfo, getMetrics(readInfo, metricsConfig)); + } catch (ParseException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + private PipelineLauncher.LaunchInfo testWrite() throws IOException { + + BigtableIO.Write writeIO = + BigtableIO.write() + .withProjectId(project) + .withInstanceId(resourceManager.getInstanceId()) + .withTableId(tableId); + + writePipeline + .apply(GenerateSequence.from(0).to(configuration.getNumRows())) + .apply("Map records", ParDo.of(new MapToBigTableFormat(configuration.getValueSizeBytes()))) + .apply("Write to BigTable", writeIO); + + PipelineLauncher.LaunchConfig options = + PipelineLauncher.LaunchConfig.builder("write-bigtable") + .setSdk(PipelineLauncher.Sdk.JAVA) + .setPipeline(writePipeline) + .addParameter("runner", configuration.getRunner()) + .build(); + + return pipelineLauncher.launch(project, region, options); + } + + private PipelineLauncher.LaunchInfo testRead() throws IOException { + BigtableIO.Read readIO = + BigtableIO.read() + .withoutValidation() + .withProjectId(project) + .withInstanceId(resourceManager.getInstanceId()) + .withTableId(tableId); + + readPipeline + .apply("Read from BigTable", readIO) + .apply("Counting element", ParDo.of(new CountingFn<>(READ_ELEMENT_METRIC_NAME))); + + PipelineLauncher.LaunchConfig options = + PipelineLauncher.LaunchConfig.builder("read-bigtable") + .setSdk(PipelineLauncher.Sdk.JAVA) + .setPipeline(readPipeline) + .addParameter("runner", configuration.getRunner()) + .build(); + + return pipelineLauncher.launch(project, region, options); + } + + /** Options for Bigquery IO load test. */ + @AutoValue + abstract static class Configuration { + abstract Long getNumRows(); + + abstract Integer getPipelineTimeout(); + + abstract String getRunner(); + + abstract Integer getValueSizeBytes(); + + static Configuration of(long numRows, int pipelineTimeout, String runner, int valueSizeBytes) { + return new AutoValue_BigTableIOLT_Configuration.Builder() + .setNumRows(numRows) + .setPipelineTimeout(pipelineTimeout) + .setRunner(runner) + .setValueSizeBytes(valueSizeBytes) + .build(); + } + + @AutoValue.Builder + abstract static class Builder { + abstract Configuration.Builder setNumRows(long numRows); + + abstract Configuration.Builder setPipelineTimeout(int timeOutMinutes); + + abstract Configuration.Builder setRunner(String runner); + + abstract Configuration.Builder setValueSizeBytes(int valueSizeBytes); + + abstract Configuration build(); + } + + abstract Configuration.Builder toBuilder(); + } + + /** Maps long number to the BigTable format record. */ + private static class MapToBigTableFormat extends DoFn>> + implements Serializable { + + private final int valueSizeBytes; + + public MapToBigTableFormat(int valueSizeBytes) { + this.valueSizeBytes = valueSizeBytes; + } + + @ProcessElement + public void processElement(ProcessContext c) { + Long index = c.element(); + + ByteString key = ByteString.copyFromUtf8(String.format("key%09d", index)); + Random random = new Random(index); + byte[] valBytes = new byte[this.valueSizeBytes]; + random.nextBytes(valBytes); + ByteString value = ByteString.copyFrom(valBytes); + + Iterable mutations = + ImmutableList.of( + Mutation.newBuilder() + .setSetCell( + Mutation.SetCell.newBuilder() + .setValue(value) + .setTimestampMicros(Instant.now().toEpochMilli() * 1000L) + .setFamilyName(COLUMN_FAMILY_NAME)) + .build()); + c.output(KV.of(key, mutations)); + } + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerClientFactoryTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerClientFactoryTest.java new file mode 100644 index 0000000000000..2b02326c1fe5e --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerClientFactoryTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigtable; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminSettings; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import java.io.IOException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Unit tests for {@link BigtableResourceManagerClientFactory}. */ +@RunWith(JUnit4.class) +public class BigtableResourceManagerClientFactoryTest { + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock private CredentialsProvider credentialsProvider; + + private static final String INSTANCE_ID = "table-id"; + private static final String PROJECT_ID = "test-project"; + private BigtableResourceManagerClientFactory client; + + @Before + public void setup() throws IOException { + BigtableInstanceAdminSettings bigtableInstanceAdminSettings = + BigtableInstanceAdminSettings.newBuilder() + .setProjectId(PROJECT_ID) + .setCredentialsProvider(credentialsProvider) + .build(); + BigtableTableAdminSettings bigtableTableAdminSettings = + BigtableTableAdminSettings.newBuilder() + .setProjectId(PROJECT_ID) + .setInstanceId(INSTANCE_ID) + .setCredentialsProvider(credentialsProvider) + .build(); + BigtableDataSettings bigtableDataSettings = + BigtableDataSettings.newBuilder() + .setProjectId(PROJECT_ID) + .setInstanceId(INSTANCE_ID) + .setCredentialsProvider(credentialsProvider) + .build(); + + client = + new BigtableResourceManagerClientFactory( + bigtableInstanceAdminSettings, bigtableTableAdminSettings, bigtableDataSettings); + } + + @Test + public void testClientFactoryGettersReturnCorrectClasses() { + try (BigtableInstanceAdminClient instanceAdminClient = client.bigtableInstanceAdminClient()) { + assertThat(instanceAdminClient).isInstanceOf(BigtableInstanceAdminClient.class); + } + try (BigtableTableAdminClient tableAdminClient = client.bigtableTableAdminClient()) { + assertThat(tableAdminClient).isInstanceOf(BigtableTableAdminClient.class); + } + try (BigtableDataClient dataClient = client.bigtableDataClient()) { + assertThat(dataClient).isInstanceOf(BigtableDataClient.class); + } + } + + @Test + public void testClientFactoryGettersReturnClientsWithCorrectIds() { + try (BigtableInstanceAdminClient instanceAdminClient = client.bigtableInstanceAdminClient()) { + assertThat(instanceAdminClient.getProjectId()).isEqualTo(PROJECT_ID); + } + try (BigtableTableAdminClient tableAdminClient = client.bigtableTableAdminClient()) { + assertThat(tableAdminClient.getProjectId()).isEqualTo(PROJECT_ID); + } + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerClusterTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerClusterTest.java new file mode 100644 index 0000000000000..ccd69d8af0d5e --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerClusterTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigtable; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.bigtable.admin.v2.models.StorageType; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link BigtableResourceManagerCluster}. */ +@RunWith(JUnit4.class) +public class BigtableResourceManagerClusterTest { + + private static final String CLUSTER_ID = "cluster-id"; + private static final String CLUSTER_ZONE = "us-central1-a"; + private static final int CLUSTER_NUM_NODES = 1; + private static final StorageType CLUSTER_STORAGE_TYPE = StorageType.SSD; + + @Test + public void testBigtableResourceManagerClusterBuilderSetsCorrectValues() { + BigtableResourceManagerCluster cluster = + BigtableResourceManagerCluster.create( + CLUSTER_ID, CLUSTER_ZONE, CLUSTER_NUM_NODES, CLUSTER_STORAGE_TYPE); + + assertThat(cluster.clusterId()).isEqualTo(CLUSTER_ID); + assertThat(cluster.zone()).isEqualTo(CLUSTER_ZONE); + assertThat(cluster.numNodes()).isEqualTo(CLUSTER_NUM_NODES); + assertThat(cluster.storageType()).isEqualTo(CLUSTER_STORAGE_TYPE); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerIT.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerIT.java new file mode 100644 index 0000000000000..e39a2727762f8 --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerIT.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigtable; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.gcp.bigtable.matchers.BigtableAsserts.assertThatBigtableRecords; + +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.GCPBaseIT; +import org.apache.beam.it.gcp.GoogleCloudIntegrationTest; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@Category(GoogleCloudIntegrationTest.class) +@RunWith(JUnit4.class) +public class BigtableResourceManagerIT extends GCPBaseIT { + + private static final String TABLE_ID = "dummy-table"; + private static final String COLUMN_FAMILY = "dummy-cf"; + private BigtableResourceManager bigtableResourceManager; + + @Before + public void setUp() throws IOException { + bigtableResourceManager = + BigtableResourceManager.builder("dummy", PROJECT, null) + .setCredentialsProvider(credentialsProvider) + .build(); + } + + @Test + public void testResourceManagerE2E() { + bigtableResourceManager.createTable(TABLE_ID, ImmutableList.of(COLUMN_FAMILY)); + + List mutations = new ArrayList<>(); + mutations.add( + RowMutation.create(TABLE_ID, "row_0").setCell(COLUMN_FAMILY, "company", "Google")); + mutations.add( + RowMutation.create(TABLE_ID, "row_1").setCell(COLUMN_FAMILY, "company", "Alphabet")); + bigtableResourceManager.write(mutations); + + List fetchRecords = bigtableResourceManager.readTable(TABLE_ID); + + assertThat(fetchRecords).hasSize(2); + assertThatBigtableRecords(fetchRecords, COLUMN_FAMILY) + .hasRecordsUnordered( + ImmutableList.of( + Collections.singletonMap("company", "Google"), + Collections.singletonMap("company", "Alphabet"))); + } + + @After + public void tearDown() { + ResourceManagerUtils.cleanResources(bigtableResourceManager); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerTest.java new file mode 100644 index 0000000000000..74b25e84c691b --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerTest.java @@ -0,0 +1,459 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigtable; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.gax.rpc.NotFoundException; +import com.google.api.gax.rpc.ServerStream; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.models.StorageType; +import com.google.cloud.bigtable.admin.v2.models.Table.ReplicationState; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.internal.verification.Times; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Unit tests for {@link BigtableResourceManager}. */ +@RunWith(JUnit4.class) +public class BigtableResourceManagerTest { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private BigtableResourceManagerClientFactory bigtableResourceManagerClientFactory; + + private static final String TEST_ID = "test-id"; + private static final String TABLE_ID = "table-id"; + private static final String PROJECT_ID = "test-project"; + + private static final String CLUSTER_ID = "cluster-id"; + private static final String CLUSTER_ZONE = "us-central1-a"; + private static final int CLUSTER_NUM_NODES = 1; + private static final StorageType CLUSTER_STORAGE_TYPE = StorageType.SSD; + + private BigtableResourceManager testManager; + private List cluster; + + @Before + public void setUp() throws IOException { + testManager = + new BigtableResourceManager( + BigtableResourceManager.builder(TEST_ID, PROJECT_ID, null), + bigtableResourceManagerClientFactory); + cluster = + ImmutableList.of( + BigtableResourceManagerCluster.create( + CLUSTER_ID, CLUSTER_ZONE, CLUSTER_NUM_NODES, CLUSTER_STORAGE_TYPE)); + } + + @Test + public void testCreateResourceManagerCreatesCorrectIdValues() throws IOException { + BigtableResourceManager rm = + new BigtableResourceManager( + BigtableResourceManager.builder(TEST_ID, PROJECT_ID, null), + bigtableResourceManagerClientFactory); + + assertThat(rm.getInstanceId()).matches(TEST_ID + "-\\d{8}-\\d{6}-\\d{6}"); + assertThat(rm.getProjectId()).matches(PROJECT_ID); + } + + @Test + public void testCreateInstanceShouldThrowExceptionWhenClientFailsToCreateInstance() { + when(bigtableResourceManagerClientFactory.bigtableInstanceAdminClient().createInstance(any())) + .thenThrow(IllegalStateException.class); + + assertThrows(BigtableResourceManagerException.class, () -> testManager.createInstance(cluster)); + } + + @Test + public void testCreateInstanceShouldThrowErrorWhenInstanceAdminClientFailsToClose() { + BigtableInstanceAdminClient mockClient = + bigtableResourceManagerClientFactory.bigtableInstanceAdminClient(); + doThrow(RuntimeException.class).when(mockClient).close(); + + assertThrows(BigtableResourceManagerException.class, () -> testManager.createInstance(cluster)); + } + + @Test + public void testCreateInstanceShouldWorkWhenBigtableDoesNotThrowAnyError() { + testManager.createInstance(cluster); + + verify(bigtableResourceManagerClientFactory.bigtableInstanceAdminClient()) + .createInstance(any()); + } + + @Test + public void testCreateTableShouldNotCreateInstanceWhenInstanceAlreadyExists() { + setupReadyTable(); + + testManager.createInstance(cluster); + Mockito.lenient() + .when( + bigtableResourceManagerClientFactory + .bigtableInstanceAdminClient() + .createInstance(any())) + .thenThrow(IllegalStateException.class); + + when(bigtableResourceManagerClientFactory.bigtableTableAdminClient().exists(anyString())) + .thenReturn(false); + testManager.createTable(TABLE_ID, ImmutableList.of("cf1")); + } + + @Test + public void testCreateTableShouldCreateInstanceWhenInstanceDoesNotExist() { + when(bigtableResourceManagerClientFactory.bigtableInstanceAdminClient().createInstance(any())) + .thenThrow(IllegalStateException.class); + + assertThrows( + BigtableResourceManagerException.class, + () -> testManager.createTable(TABLE_ID, ImmutableList.of("cf1"))); + } + + @Test + public void testCreateTableShouldThrowErrorWhenNoColumnFamilyGiven() { + assertThrows( + IllegalArgumentException.class, () -> testManager.createTable(TABLE_ID, new ArrayList<>())); + } + + @Test + public void testCreateTableShouldNotCreateTableWhenTableAlreadyExists() { + when(bigtableResourceManagerClientFactory.bigtableTableAdminClient().exists(anyString())) + .thenReturn(true); + assertThrows( + BigtableResourceManagerException.class, + () -> testManager.createTable(TABLE_ID, ImmutableList.of("cf1"))); + } + + @Test + public void testCreateTableShouldThrowErrorWhenTableAdminClientFailsToCreateTable() { + when(bigtableResourceManagerClientFactory.bigtableTableAdminClient().exists(anyString())) + .thenReturn(false); + when(bigtableResourceManagerClientFactory.bigtableTableAdminClient().createTable(any())) + .thenThrow(RuntimeException.class); + + assertThrows( + BigtableResourceManagerException.class, + () -> testManager.createTable(TABLE_ID, ImmutableList.of("cf1"))); + } + + @Test + public void testCreateTableShouldThrowErrorWhenTableAdminClientFailsToClose() { + setupReadyTable(); + BigtableTableAdminClient mockClient = + bigtableResourceManagerClientFactory.bigtableTableAdminClient(); + doThrow(RuntimeException.class).when(mockClient).close(); + + assertThrows( + BigtableResourceManagerException.class, + () -> testManager.createTable(TABLE_ID, ImmutableList.of("cf1"))); + } + + @Test + public void testCreateTableShouldWorkWhenBigtableDoesNotThrowAnyError() { + setupReadyTable(); + + when(bigtableResourceManagerClientFactory.bigtableTableAdminClient().exists(anyString())) + .thenReturn(false); + + testManager.createTable(TABLE_ID, ImmutableList.of("cf1")); + + verify(bigtableResourceManagerClientFactory.bigtableTableAdminClient()).createTable(any()); + } + + @Test + public void testWriteShouldThrowErrorWhenInstanceDoesNotExist() { + assertThrows( + IllegalStateException.class, + () -> testManager.write(RowMutation.create(TABLE_ID, "sample-key"))); + assertThrows( + IllegalStateException.class, + () -> testManager.write(ImmutableList.of(RowMutation.create(TABLE_ID, "sample-key")))); + } + + @Test + public void testWriteShouldExitEarlyWhenNoRowMutationsGiven() { + testManager.createInstance(cluster); + + Mockito.lenient() + .when(bigtableResourceManagerClientFactory.bigtableDataClient()) + .thenThrow(RuntimeException.class); + + testManager.write(ImmutableList.of()); + } + + @Test + public void testWriteShouldThrowErrorWhenDataClientFailsToInstantiate() { + testManager.createInstance(cluster); + + when(bigtableResourceManagerClientFactory.bigtableDataClient()) + .thenThrow(BigtableResourceManagerException.class); + + assertThrows( + BigtableResourceManagerException.class, + () -> testManager.write(RowMutation.create(TABLE_ID, "sample-key"))); + assertThrows( + BigtableResourceManagerException.class, + () -> testManager.write(ImmutableList.of(RowMutation.create(TABLE_ID, "sample-key")))); + } + + @Test + public void testWriteShouldThrowErrorWhenDataClientFailsToSendMutations() { + testManager.createInstance(cluster); + BigtableDataClient mockClient = bigtableResourceManagerClientFactory.bigtableDataClient(); + doThrow(RuntimeException.class).when(mockClient).mutateRow(any()); + + assertThrows( + BigtableResourceManagerException.class, + () -> testManager.write(RowMutation.create(TABLE_ID, "sample-key"))); + assertThrows( + BigtableResourceManagerException.class, + () -> testManager.write(ImmutableList.of(RowMutation.create(TABLE_ID, "sample-key")))); + } + + @Test + public void testWriteShouldThrowErrorWhenDataClientFailsToClose() { + testManager.createInstance(cluster); + BigtableDataClient mockClient = bigtableResourceManagerClientFactory.bigtableDataClient(); + doThrow(RuntimeException.class).when(mockClient).close(); + + assertThrows( + BigtableResourceManagerException.class, + () -> testManager.write(RowMutation.create(TABLE_ID, "sample-key"))); + } + + @Test + public void testWriteShouldWorkWhenBigtableDoesNotThrowAnyError() { + testManager.createInstance(cluster); + + testManager.write(RowMutation.create(TABLE_ID, "sample-key")); + testManager.write(ImmutableList.of(RowMutation.create(TABLE_ID, "sample-key"))); + + verify(bigtableResourceManagerClientFactory.bigtableDataClient(), times(2)).mutateRow(any()); + } + + @Test + public void testReadTableShouldThrowErrorWhenInstanceDoesNotExist() { + assertThrows(IllegalStateException.class, () -> testManager.readTable(TABLE_ID)); + } + + @Test + public void testReadTableShouldThrowErrorWhenTableDoesNotExist() { + testManager.createInstance(cluster); + + assertThrows(IllegalStateException.class, () -> testManager.readTable(TABLE_ID)); + } + + @Test + public void testReadTableShouldThrowErrorWhenDataClientFailsToInstantiate() { + testManager.createInstance(cluster); + + when(bigtableResourceManagerClientFactory.bigtableTableAdminClient().exists(anyString())) + .thenReturn(true); + when(bigtableResourceManagerClientFactory.bigtableDataClient()) + .thenThrow(BigtableResourceManagerException.class); + + assertThrows(BigtableResourceManagerException.class, () -> testManager.readTable(TABLE_ID)); + } + + @Test + public void testReadTableShouldThrowErrorWhenDataClientFailsToReadRows() { + testManager.createInstance(cluster); + + when(bigtableResourceManagerClientFactory.bigtableTableAdminClient().exists(anyString())) + .thenReturn(true); + when(bigtableResourceManagerClientFactory.bigtableDataClient().readRows(any())) + .thenThrow(NotFoundException.class); + + assertThrows(BigtableResourceManagerException.class, () -> testManager.readTable(TABLE_ID)); + } + + @Test + public void testReadTableShouldThrowErrorWhenReadRowsReturnsNull() { + testManager.createInstance(cluster); + + when(bigtableResourceManagerClientFactory.bigtableTableAdminClient().exists(anyString())) + .thenReturn(true); + when(bigtableResourceManagerClientFactory.bigtableDataClient().readRows(any())) + .thenReturn(null); + + assertThrows(BigtableResourceManagerException.class, () -> testManager.readTable(TABLE_ID)); + } + + @Test + public void testReadTableShouldThrowErrorWhenDataClientFailsToClose() { + testManager.createInstance(cluster); + + when(bigtableResourceManagerClientFactory.bigtableTableAdminClient().exists(anyString())) + .thenReturn(true); + BigtableDataClient mockClient = bigtableResourceManagerClientFactory.bigtableDataClient(); + doThrow(RuntimeException.class).when(mockClient).close(); + + assertThrows(BigtableResourceManagerException.class, () -> testManager.readTable(TABLE_ID)); + } + + @Test + public void testReadTableShouldWorkWhenBigtableDoesNotThrowAnyError() { + testManager.createInstance(cluster); + + ServerStream mockReadRows = mock(ServerStream.class, Answers.RETURNS_DEEP_STUBS); + Row mockRow = mock(Row.class); + when(bigtableResourceManagerClientFactory.bigtableDataClient().readRows(any())) + .thenReturn(mockReadRows); + when(mockReadRows.iterator().hasNext()).thenReturn(true, false); + when(mockReadRows.iterator().next()).thenReturn(mockRow); + + when(bigtableResourceManagerClientFactory.bigtableTableAdminClient().exists(anyString())) + .thenReturn(true); + + testManager.readTable(TABLE_ID); + + verify(bigtableResourceManagerClientFactory.bigtableDataClient()).readRows(any()); + } + + @Test + public void testCleanupAllCallsDeleteInstance() { + testManager.createInstance(cluster); + BigtableInstanceAdminClient mockClient = + bigtableResourceManagerClientFactory.bigtableInstanceAdminClient(); + doThrow(RuntimeException.class).when(mockClient).deleteInstance(anyString()); + + assertThrows(RuntimeException.class, () -> testManager.cleanupAll()); + } + + @Test + public void testCleanupAllClosesInstanceAdminClient() { + testManager.createInstance(cluster); + BigtableInstanceAdminClient mockClient = + bigtableResourceManagerClientFactory.bigtableInstanceAdminClient(); + doThrow(RuntimeException.class).when(mockClient).close(); + + assertThrows(RuntimeException.class, () -> testManager.cleanupAll()); + } + + @Test + public void testCleanupAllClosesTableAdminClient() { + testManager.createInstance(cluster); + BigtableInstanceAdminClient mockClient = + bigtableResourceManagerClientFactory.bigtableInstanceAdminClient(); + doThrow(RuntimeException.class).when(mockClient).deleteInstance(any()); + + assertThrows(RuntimeException.class, () -> testManager.cleanupAll()); + } + + @Test + public void testCleanupAllShouldThrowErrorWhenInstanceFailsToDelete() { + testManager.createInstance(cluster); + BigtableInstanceAdminClient mockClient = + bigtableResourceManagerClientFactory.bigtableInstanceAdminClient(); + doThrow(RuntimeException.class).when(mockClient).deleteInstance(anyString()); + + assertThrows(BigtableResourceManagerException.class, () -> testManager.cleanupAll()); + } + + @Test + public void testCleanupAllShouldThrowErrorWhenInstanceAdminClientFailsToClose() { + testManager.createInstance(cluster); + BigtableInstanceAdminClient mockClient = + bigtableResourceManagerClientFactory.bigtableInstanceAdminClient(); + doThrow(RuntimeException.class).when(mockClient).close(); + + assertThrows(BigtableResourceManagerException.class, () -> testManager.cleanupAll()); + } + + @Test + public void testCleanupAllShouldNotCleanupStaticInstance() throws IOException { + String instanceId = "static-instance"; + testManager = + new BigtableResourceManager( + BigtableResourceManager.builder(TEST_ID, PROJECT_ID, null) + .setInstanceId(instanceId) + .useStaticInstance(), + bigtableResourceManagerClientFactory); + + when(bigtableResourceManagerClientFactory.bigtableTableAdminClient().exists(anyString())) + .thenReturn(false); + + setupReadyTable(); + + testManager.createTable(TABLE_ID, ImmutableList.of("cf1")); + + testManager.cleanupAll(); + verify(bigtableResourceManagerClientFactory.bigtableTableAdminClient()).deleteTable(TABLE_ID); + verify(bigtableResourceManagerClientFactory.bigtableTableAdminClient(), new Times(1)) + .deleteTable(anyString()); + verify(bigtableResourceManagerClientFactory.bigtableInstanceAdminClient(), never()) + .deleteInstance(any()); + } + + private void setupReadyTable() { + Map allReplicated = new HashMap<>(); + allReplicated.put(CLUSTER_ID, ReplicationState.READY); + + when(bigtableResourceManagerClientFactory + .bigtableTableAdminClient() + .getTable(TABLE_ID) + .getReplicationStatesByClusterId()) + .thenReturn(allReplicated); + } + + @Test + public void testCleanupAllShouldWorkWhenBigtableDoesNotThrowAnyError() { + setupReadyTable(); + + testManager.createTable(TABLE_ID, ImmutableList.of("cf1")); + + when(bigtableResourceManagerClientFactory.bigtableTableAdminClient().exists(anyString())) + .thenReturn(true); + testManager.readTable(TABLE_ID); + + testManager.cleanupAll(); + + verify(bigtableResourceManagerClientFactory.bigtableDataClient()).close(); + verify(bigtableResourceManagerClientFactory.bigtableTableAdminClient(), never()) + .deleteTable(TABLE_ID); + verify(bigtableResourceManagerClientFactory.bigtableInstanceAdminClient()) + .deleteInstance(anyString()); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerUtilsTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerUtilsTest.java new file mode 100644 index 0000000000000..9634dc2d04006 --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigtableResourceManagerUtilsTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.bigtable; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.gcp.bigtable.BigtableResourceManagerUtils.checkValidTableId; +import static org.apache.beam.it.gcp.bigtable.BigtableResourceManagerUtils.generateDefaultClusters; +import static org.junit.Assert.assertThrows; + +import com.google.cloud.bigtable.admin.v2.models.StorageType; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link BigtableResourceManagerUtils}. */ +@RunWith(JUnit4.class) +public class BigtableResourceManagerUtilsTest { + private static final String TEST_ID = "test-id"; + private static final String ZONE = "us-central1-a"; + private static final int NUM_NODES = 1; + private static final StorageType STORAGE_TYPE = StorageType.SSD; + + @Test + public void testGenerateDefaultClustersShouldWorkWhenAllParametersValid() { + Iterable cluster = + generateDefaultClusters(TEST_ID, ZONE, NUM_NODES, STORAGE_TYPE); + BigtableResourceManagerCluster thisCluster = cluster.iterator().next(); + + assertThat(thisCluster.clusterId()).matches(TEST_ID + "-\\d{8}-\\d{6}-\\d{6}"); + assertThat(thisCluster.zone()).isEqualTo(ZONE); + assertThat(thisCluster.numNodes()).isEqualTo(NUM_NODES); + assertThat(thisCluster.storageType()).isEqualTo(STORAGE_TYPE); + } + + @Test + public void testGenerateDefaultClustersShouldThrowErrorWhenTestIdIsEmpty() { + assertThrows( + IllegalArgumentException.class, + () -> generateDefaultClusters("", ZONE, NUM_NODES, STORAGE_TYPE)); + } + + @Test + public void testGenerateDefaultClustersShouldShortenTestIdWhenTooLong() { + Iterable cluster = + generateDefaultClusters("longer-id", ZONE, NUM_NODES, STORAGE_TYPE); + assertThat(cluster.iterator().next().clusterId()).matches("longer--\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testCheckValidTableIdWhenIdIsTooShort() { + assertThrows(IllegalArgumentException.class, () -> checkValidTableId("")); + } + + @Test + public void testCheckValidTableIdWhenIdIsTooLong() { + assertThrows( + IllegalArgumentException.class, + () -> checkValidTableId("really-really-really-really-long-table-id")); + } + + @Test + public void testCheckValidTableIdWhenIdContainsIllegalCharacter() { + assertThrows(IllegalArgumentException.class, () -> checkValidTableId("table-id%")); + } + + @Test + public void testCheckValidTableIdWhenIdIsValid() { + checkValidTableId("table-id_valid.Test1"); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/AbstractPipelineLauncherTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/AbstractPipelineLauncherTest.java new file mode 100644 index 0000000000000..f2c33d70116b4 --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/AbstractPipelineLauncherTest.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.dataflow; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.services.dataflow.Dataflow; +import com.google.api.services.dataflow.Dataflow.Projects.Locations; +import com.google.api.services.dataflow.Dataflow.Projects.Locations.Jobs.Get; +import com.google.api.services.dataflow.Dataflow.Projects.Locations.Jobs.Update; +import com.google.api.services.dataflow.model.Job; +import dev.failsafe.FailsafeException; +import java.io.IOException; +import java.net.SocketTimeoutException; +import org.apache.beam.it.common.PipelineLauncher.JobState; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Unit tests for {@link AbstractPipelineLauncher}. */ +@RunWith(JUnit4.class) +public final class AbstractPipelineLauncherTest { + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private Dataflow client; + + private static final String PROJECT = "test-project"; + private static final String REGION = "us-east1"; + private static final String JOB_ID = "test-job-id"; + + @Captor private ArgumentCaptor projectCaptor; + @Captor private ArgumentCaptor regionCaptor; + @Captor private ArgumentCaptor jobIdCaptor; + @Captor private ArgumentCaptor jobCaptor; + + @Test + public void testGetJobStatus() throws IOException { + Get get = mock(Get.class); + Job job = new Job().setCurrentState(JobState.RUNNING.toString()); + when(getLocationJobs(client).get(any(), any(), any()).setView(any())) + .thenThrow(new RuntimeException("Server is not responding")) + .thenReturn(get); + when(get.execute()).thenThrow(new IOException("Connection reset")).thenReturn(job); + + JobState actual = new FakePipelineLauncher(client).getJobStatus(PROJECT, REGION, JOB_ID); + + verify(getLocationJobs(client), times(4)) + .get(projectCaptor.capture(), regionCaptor.capture(), jobIdCaptor.capture()); + assertThat(projectCaptor.getValue()).isEqualTo(PROJECT); + assertThat(regionCaptor.getValue()).isEqualTo(REGION); + assertThat(jobIdCaptor.getValue()).isEqualTo(JOB_ID); + assertThat(actual).isEqualTo(JobState.RUNNING); + } + + @Test + public void testGetJobThrowsException() throws IOException { + when(getLocationJobs(client).get(any(), any(), any())).thenThrow(new IOException()); + assertThrows( + FailsafeException.class, + () -> new FakePipelineLauncher(client).getJobStatus(PROJECT, REGION, JOB_ID)); + } + + @Test + public void testCancelJob() throws IOException { + Update update = mock(Update.class); + when(getLocationJobs(client).update(any(), any(), any(), any())) + .thenThrow(new IOException("Connection reset")) + .thenThrow(new SocketTimeoutException("Read timed out")) + .thenReturn(update); + when(update.execute()).thenThrow(new IOException("Connection reset")).thenReturn(new Job()); + + new FakePipelineLauncher(client).cancelJob(PROJECT, REGION, JOB_ID); + + verify(getLocationJobs(client), times(4)) + .update( + projectCaptor.capture(), + regionCaptor.capture(), + jobIdCaptor.capture(), + jobCaptor.capture()); + assertThat(projectCaptor.getValue()).isEqualTo(PROJECT); + assertThat(regionCaptor.getValue()).isEqualTo(REGION); + assertThat(jobIdCaptor.getValue()).isEqualTo(JOB_ID); + assertThat(jobCaptor.getValue().getRequestedState()).isEqualTo(JobState.CANCELLED.toString()); + } + + @Test + public void testCancelJobThrowsException() throws IOException { + when(getLocationJobs(client).update(any(), any(), any(), any())).thenThrow(new IOException()); + assertThrows( + FailsafeException.class, + () -> new FakePipelineLauncher(client).cancelJob(PROJECT, REGION, JOB_ID)); + } + + @Test + public void testDrainJob() throws IOException { + Update update = mock(Update.class); + when(getLocationJobs(client).update(any(), any(), any(), any())) + .thenThrow(new IOException("Connection reset")) + .thenReturn(update); + when(update.execute()).thenThrow(new IOException("Connection reset")).thenReturn(new Job()); + + new FakePipelineLauncher(client).drainJob(PROJECT, REGION, JOB_ID); + + verify(getLocationJobs(client), times(3)) + .update( + projectCaptor.capture(), + regionCaptor.capture(), + jobIdCaptor.capture(), + jobCaptor.capture()); + assertThat(projectCaptor.getValue()).isEqualTo(PROJECT); + assertThat(regionCaptor.getValue()).isEqualTo(REGION); + assertThat(jobIdCaptor.getValue()).isEqualTo(JOB_ID); + assertThat(jobCaptor.getValue().getRequestedState()).isEqualTo(JobState.DRAINED.toString()); + } + + @Test + public void testDrainJobThrowsException() throws IOException { + when(getLocationJobs(client).update(any(), any(), any(), any())).thenThrow(new IOException()); + assertThrows( + FailsafeException.class, + () -> new FakePipelineLauncher(client).drainJob(PROJECT, REGION, JOB_ID)); + } + + private static Locations.Jobs getLocationJobs(Dataflow client) { + return client.projects().locations().jobs(); + } + + /** + * Fake implementation that simply throws {@link UnsupportedOperationException} for some methods. + */ + private static final class FakePipelineLauncher extends AbstractPipelineLauncher { + FakePipelineLauncher(Dataflow client) { + super(client); + } + + @Override + public LaunchInfo launch(String project, String region, LaunchConfig options) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/ClassicTemplateClientTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/ClassicTemplateClientTest.java new file mode 100644 index 0000000000000..88c35589f2be8 --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/ClassicTemplateClientTest.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.dataflow; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.gcp.dataflow.AbstractPipelineLauncher.LEGACY_RUNNER; +import static org.apache.beam.it.gcp.dataflow.AbstractPipelineLauncher.PARAM_JOB_ID; +import static org.apache.beam.it.gcp.dataflow.AbstractPipelineLauncher.PARAM_JOB_TYPE; +import static org.apache.beam.it.gcp.dataflow.AbstractPipelineLauncher.PARAM_RUNNER; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.services.dataflow.Dataflow; +import com.google.api.services.dataflow.Dataflow.Projects.Locations; +import com.google.api.services.dataflow.Dataflow.Projects.Locations.Jobs.Get; +import com.google.api.services.dataflow.Dataflow.Projects.Locations.Templates; +import com.google.api.services.dataflow.Dataflow.Projects.Locations.Templates.Create; +import com.google.api.services.dataflow.model.CreateJobFromTemplateRequest; +import com.google.api.services.dataflow.model.Environment; +import com.google.api.services.dataflow.model.Job; +import com.google.api.services.dataflow.model.JobMetadata; +import com.google.api.services.dataflow.model.RuntimeEnvironment; +import com.google.api.services.dataflow.model.SdkVersion; +import com.google.auth.Credentials; +import com.google.common.collect.ImmutableMap; +import dev.failsafe.FailsafeException; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.util.Collections; +import org.apache.beam.it.common.PipelineLauncher.JobState; +import org.apache.beam.it.common.PipelineLauncher.LaunchConfig; +import org.apache.beam.it.common.PipelineLauncher.LaunchInfo; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Unit test for {@link ClassicTemplateClient}. */ +@RunWith(JUnit4.class) +public final class ClassicTemplateClientTest { + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private Dataflow client; + + private static final String PROJECT = "test-project"; + private static final String REGION = "us-east1"; + private static final String JOB_ID = "test-job-id"; + private static final String JOB_NAME = "test-job"; + private static final String SPEC_PATH = "gs://test-bucket/test-dir/test-spec.json"; + + private static final String PARAM_KEY = "key"; + private static final String PARAM_VALUE = "value"; + + @Captor private ArgumentCaptor projectCaptor; + @Captor private ArgumentCaptor regionCaptor; + @Captor private ArgumentCaptor jobIdCaptor; + @Captor private ArgumentCaptor requestCaptor; + + @Test + public void testCreateWithCredentials() { + Credentials credentials = mock(Credentials.class); + ClassicTemplateClient.builder(credentials).build(); + // Lack of exception is all we really can test + } + + @Test + public void testLaunchNewJob() throws IOException { + // Arrange + Create launch = mock(Create.class); + Get get = mock(Get.class); + Job launchJob = new Job().setId(JOB_ID); + Job getJob = + new Job() + .setId(JOB_ID) + .setProjectId(PROJECT) + .setLocation(REGION) + .setCurrentState(JobState.RUNNING.toString()) + .setCreateTime("") + .setJobMetadata( + new JobMetadata() + .setSdkVersion( + new SdkVersion() + .setVersionDisplayName("Apache Beam Java") + .setVersion("2.42.0"))) + .setType("JOB_TYPE_BATCH") + .setEnvironment(new Environment().setExperiments(Collections.singletonList(""))); + + LaunchConfig options = + LaunchConfig.builderWithName(JOB_NAME, SPEC_PATH) + .addParameter(PARAM_KEY, PARAM_VALUE) + .build(); + + when(getTemplates(client).create(any(), any(), any())).thenReturn(launch); + when(getLocationJobs(client).get(any(), any(), any()).setView(any())).thenReturn(get); + when(launch.execute()) + .thenThrow(new SocketTimeoutException("Read timed out")) + .thenReturn(launchJob); + when(get.execute()).thenReturn(getJob); + + // Act + LaunchInfo actual = + ClassicTemplateClient.withDataflowClient(client).launch(PROJECT, REGION, options); + + // Assert + CreateJobFromTemplateRequest expectedRequest = + new CreateJobFromTemplateRequest() + .setJobName(JOB_NAME) + .setGcsPath(SPEC_PATH) + .setParameters(ImmutableMap.of(PARAM_KEY, PARAM_VALUE)) + .setLocation(REGION) + .setEnvironment(new RuntimeEnvironment()); + verify(getTemplates(client), times(2)) + .create(projectCaptor.capture(), regionCaptor.capture(), requestCaptor.capture()); + assertThat(projectCaptor.getValue()).isEqualTo(PROJECT); + assertThat(regionCaptor.getValue()).isEqualTo(REGION); + assertThat(requestCaptor.getValue()).isEqualTo(expectedRequest); + + verify(getLocationJobs(client), times(3)) + .get(projectCaptor.capture(), regionCaptor.capture(), jobIdCaptor.capture()); + assertThat(projectCaptor.getValue()).isEqualTo(PROJECT); + assertThat(regionCaptor.getValue()).isEqualTo(REGION); + assertThat(jobIdCaptor.getValue()).isEqualTo(JOB_ID); + + LaunchInfo expected = + LaunchInfo.builder() + .setJobId(JOB_ID) + .setProjectId(PROJECT) + .setRegion(REGION) + .setState(JobState.RUNNING) + .setCreateTime("") + .setSdk("Apache Beam Java") + .setVersion("2.42.0") + .setJobType("JOB_TYPE_BATCH") + .setRunner(AbstractPipelineLauncher.LEGACY_RUNNER) + .setParameters( + ImmutableMap.builder() + .put(PARAM_KEY, PARAM_VALUE) + .put(PARAM_JOB_ID, JOB_ID) + .put(PARAM_RUNNER, LEGACY_RUNNER) + .put(PARAM_JOB_TYPE, "JOB_TYPE_BATCH") + .build()) + .build(); + assertThat(actual).isEqualTo(expected); + } + + @Test + public void testLaunchNewJobThrowsException() throws IOException { + when(getTemplates(client).create(any(), any(), any())).thenThrow(new IOException()); + assertThrows( + FailsafeException.class, + () -> + ClassicTemplateClient.withDataflowClient(client) + .launch(PROJECT, REGION, LaunchConfig.builder(JOB_NAME, SPEC_PATH).build())); + } + + private static Locations.Jobs getLocationJobs(Dataflow client) { + return client.projects().locations().jobs(); + } + + private static Templates getTemplates(Dataflow client) { + return client.projects().locations().templates(); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/DefaultPipelineLauncherTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/DefaultPipelineLauncherTest.java new file mode 100644 index 0000000000000..b6c0f8cdc58fd --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/DefaultPipelineLauncherTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.dataflow; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.time.Instant; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.gcp.IOLoadTestBase; +import org.apache.beam.it.gcp.IOLoadTestBase.PipelineMetricsType; +import org.apache.beam.sdk.io.GenerateSequence; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.testutils.metrics.TimeMonitor; +import org.apache.beam.sdk.transforms.ParDo; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link DefaultPipelineLauncher}. */ +@RunWith(JUnit4.class) +public class DefaultPipelineLauncherTest { + @Rule public TestPipeline pipeline = TestPipeline.create(); + + @Test + public void testPipelineMetrics() throws IOException { + DefaultPipelineLauncher launcher = DefaultPipelineLauncher.builder(null).build(); + final String timeMetrics = "run_time"; + final String counterMetrics = "counter"; + final long numElements = 1000L; + + pipeline + .apply(GenerateSequence.from(0).to(numElements)) + .apply(ParDo.of(new TimeMonitor<>(IOLoadTestBase.BEAM_METRICS_NAMESPACE, timeMetrics))) + .apply(ParDo.of(new IOLoadTestBase.CountingFn<>(counterMetrics))); + + PipelineLauncher.LaunchConfig options = + PipelineLauncher.LaunchConfig.builder("test-bigquery-write") + .setSdk(PipelineLauncher.Sdk.JAVA) + .setPipeline(pipeline) + .addParameter("runner", "DirectRunner") + .build(); + + PipelineLauncher.LaunchInfo launchInfo = launcher.launch("", "", options); + long currentTime = System.currentTimeMillis(); + long startTime = + launcher.getBeamMetric(launchInfo.jobId(), PipelineMetricsType.STARTTIME, timeMetrics); + long endTime = + launcher.getBeamMetric(launchInfo.jobId(), PipelineMetricsType.ENDTIME, timeMetrics); + long runTime = + launcher.getBeamMetric(launchInfo.jobId(), PipelineMetricsType.RUNTIME, timeMetrics); + long count = + launcher.getBeamMetric(launchInfo.jobId(), PipelineMetricsType.COUNTER, counterMetrics); + + // check the validity of time metrics: start time and end time metrics is within 1 minute of + // current + assertTrue( + String.format( + "start time metrics (%s) is not around current time", Instant.ofEpochMilli(startTime)), + Math.abs(startTime - currentTime) < 10_000); + assertTrue( + String.format( + "start time metrics (%s) is not around current time", Instant.ofEpochMilli(endTime)), + Math.abs(endTime - currentTime) < 10_000); + assertTrue("run time should be greater than 0", runTime > 0); + + assertEquals(numElements, count); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/FlexTemplateClientTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/FlexTemplateClientTest.java new file mode 100644 index 0000000000000..06f44437414a3 --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/FlexTemplateClientTest.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.dataflow; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.gcp.dataflow.AbstractPipelineLauncher.LEGACY_RUNNER; +import static org.apache.beam.it.gcp.dataflow.AbstractPipelineLauncher.PARAM_JOB_ID; +import static org.apache.beam.it.gcp.dataflow.AbstractPipelineLauncher.PARAM_JOB_TYPE; +import static org.apache.beam.it.gcp.dataflow.AbstractPipelineLauncher.PARAM_RUNNER; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.services.dataflow.Dataflow; +import com.google.api.services.dataflow.Dataflow.Projects.Locations; +import com.google.api.services.dataflow.Dataflow.Projects.Locations.FlexTemplates; +import com.google.api.services.dataflow.Dataflow.Projects.Locations.FlexTemplates.Launch; +import com.google.api.services.dataflow.Dataflow.Projects.Locations.Jobs.Get; +import com.google.api.services.dataflow.model.Environment; +import com.google.api.services.dataflow.model.FlexTemplateRuntimeEnvironment; +import com.google.api.services.dataflow.model.Job; +import com.google.api.services.dataflow.model.JobMetadata; +import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter; +import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest; +import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse; +import com.google.api.services.dataflow.model.SdkVersion; +import com.google.auth.Credentials; +import com.google.common.collect.ImmutableMap; +import dev.failsafe.FailsafeException; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.util.Collections; +import org.apache.beam.it.common.PipelineLauncher.JobState; +import org.apache.beam.it.common.PipelineLauncher.LaunchConfig; +import org.apache.beam.it.common.PipelineLauncher.LaunchInfo; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Unit test for {@link FlexTemplateClient}. */ +@RunWith(JUnit4.class) +public final class FlexTemplateClientTest { + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private Dataflow client; + + private static final String PROJECT = "test-project"; + private static final String REGION = "us-east1"; + private static final String JOB_ID = "test-job-id"; + private static final String JOB_NAME = "test-job"; + private static final String SPEC_PATH = "gs://test-bucket/test-dir/test-spec.json"; + + private static final String PARAM_KEY = "key"; + private static final String PARAM_VALUE = "value"; + + @Captor private ArgumentCaptor projectCaptor; + @Captor private ArgumentCaptor regionCaptor; + @Captor private ArgumentCaptor jobIdCaptor; + @Captor private ArgumentCaptor requestCaptor; + + @Test + public void testCreateWithCredentials() { + Credentials credentials = mock(Credentials.class); + FlexTemplateClient.builder(credentials).build(); + // Lack of exception is all we really can test + } + + @Test + public void testLaunchNewJob() throws IOException { + // Arrange + Launch launch = mock(Launch.class); + Get get = mock(Get.class); + Job launchJob = new Job().setId(JOB_ID); + Job getJob = + new Job() + .setId(JOB_ID) + .setProjectId(PROJECT) + .setLocation(REGION) + .setCurrentState(JobState.RUNNING.toString()) + .setCreateTime("") + .setJobMetadata( + new JobMetadata() + .setSdkVersion( + new SdkVersion() + .setVersionDisplayName("Apache Beam Java") + .setVersion("2.42.0"))) + .setType("JOB_TYPE_BATCH") + .setEnvironment(new Environment().setExperiments(Collections.singletonList(""))); + + LaunchFlexTemplateResponse response = new LaunchFlexTemplateResponse().setJob(launchJob); + + LaunchConfig options = + LaunchConfig.builderWithName(JOB_NAME, SPEC_PATH) + .addParameter(PARAM_KEY, PARAM_VALUE) + .build(); + + when(getFlexTemplates(client).launch(any(), any(), any())).thenReturn(launch); + when(getLocationJobs(client).get(any(), any(), any()).setView(any())).thenReturn(get); + when(launch.execute()) + .thenThrow(new SocketTimeoutException("Read timed out")) + .thenReturn(response); + when(get.execute()).thenReturn(getJob); + + // Act + LaunchInfo actual = + FlexTemplateClient.withDataflowClient(client).launch(PROJECT, REGION, options); + + // Assert + LaunchFlexTemplateRequest expectedRequest = + new LaunchFlexTemplateRequest() + .setLaunchParameter( + new LaunchFlexTemplateParameter() + .setJobName(JOB_NAME) + .setContainerSpecGcsPath(SPEC_PATH) + .setParameters(ImmutableMap.of(PARAM_KEY, PARAM_VALUE)) + .setEnvironment(new FlexTemplateRuntimeEnvironment())); + verify(getFlexTemplates(client), times(2)) + .launch(projectCaptor.capture(), regionCaptor.capture(), requestCaptor.capture()); + assertThat(projectCaptor.getValue()).isEqualTo(PROJECT); + assertThat(regionCaptor.getValue()).isEqualTo(REGION); + assertThat(requestCaptor.getValue()).isEqualTo(expectedRequest); + + verify(getLocationJobs(client), times(3)) + .get(projectCaptor.capture(), regionCaptor.capture(), jobIdCaptor.capture()); + assertThat(projectCaptor.getValue()).isEqualTo(PROJECT); + assertThat(regionCaptor.getValue()).isEqualTo(REGION); + assertThat(jobIdCaptor.getValue()).isEqualTo(JOB_ID); + + LaunchInfo expected = + LaunchInfo.builder() + .setJobId(JOB_ID) + .setProjectId(PROJECT) + .setRegion(REGION) + .setState(JobState.RUNNING) + .setCreateTime("") + .setSdk("Apache Beam Java") + .setVersion("2.42.0") + .setJobType("JOB_TYPE_BATCH") + .setRunner(LEGACY_RUNNER) + .setParameters( + ImmutableMap.builder() + .put(PARAM_KEY, PARAM_VALUE) + .put(PARAM_JOB_ID, JOB_ID) + .put(PARAM_RUNNER, LEGACY_RUNNER) + .put(PARAM_JOB_TYPE, "JOB_TYPE_BATCH") + .build()) + .build(); + assertThat(actual).isEqualTo(expected); + } + + @Test + public void testLaunchNewJobThrowsException() throws IOException { + when(getFlexTemplates(client).launch(any(), any(), any())).thenThrow(new IOException()); + assertThrows( + FailsafeException.class, + () -> + FlexTemplateClient.withDataflowClient(client) + .launch(PROJECT, REGION, LaunchConfig.builder(JOB_NAME, SPEC_PATH).build())); + } + + private static Locations.Jobs getLocationJobs(Dataflow client) { + return client.projects().locations().jobs(); + } + + private static FlexTemplates getFlexTemplates(Dataflow client) { + return client.projects().locations().flexTemplates(); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/datastore/DatastoreResourceManagerIT.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/datastore/DatastoreResourceManagerIT.java new file mode 100644 index 0000000000000..02bf43f32757c --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/datastore/DatastoreResourceManagerIT.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.datastore; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.datastore.Entity; +import java.util.List; +import org.apache.beam.it.common.TestProperties; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.junit.Test; + +/** Integration tests for {@link DatastoreResourceManager}. */ +public class DatastoreResourceManagerIT { + + @Test + public void testInsert() { + DatastoreResourceManager resourceManager = + DatastoreResourceManager.builder( + TestProperties.project(), + DatastoreUtils.createTestId("testInsert"), + TestProperties.credentials()) + .build(); + List entities = + resourceManager.insert( + "person", + ImmutableMap.of( + 1L, + Entity.newBuilder().set("name", "John Doe").build(), + 2L, + Entity.newBuilder().set("name", "Joan of Arc").build())); + assertThat(entities).hasSize(2); + + resourceManager.cleanupAll(); + } + + @Test + public void testInsertQuery() { + DatastoreResourceManager resourceManager = + DatastoreResourceManager.builder( + TestProperties.project(), + DatastoreUtils.createTestId("testInsertQuery"), + TestProperties.buildCredentialsFromEnv()) + .build(); + + List entities = + resourceManager.insert( + "person", ImmutableMap.of(1L, Entity.newBuilder().set("name", "John Doe").build())); + + assertThat(entities).hasSize(1); + List queryResults = resourceManager.query("SELECT * from person"); + assertThat(queryResults).isNotEmpty(); + Entity person = queryResults.get(0); + assertThat(person).isNotNull(); + assertThat(person.getKey().getId()).isEqualTo(1L); + assertThat(person.getString("name")).isEqualTo("John Doe"); + + resourceManager.cleanupAll(); + } + + @Test + public void testInsertCleanUp() { + DatastoreResourceManager resourceManager = + DatastoreResourceManager.builder( + TestProperties.project(), + DatastoreUtils.createTestId("testInsertCleanUp"), + TestProperties.buildCredentialsFromEnv()) + .build(); + resourceManager.insert( + "person", ImmutableMap.of(1L, Entity.newBuilder().set("name", "John Doe").build())); + + resourceManager.cleanupAll(); + + List queryResults = resourceManager.query("SELECT * from person"); + assertThat(queryResults).isEmpty(); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/datastore/DatastoreResourceManagerTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/datastore/DatastoreResourceManagerTest.java new file mode 100644 index 0000000000000..6aacccb15d836 --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/datastore/DatastoreResourceManagerTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.datastore; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.FullEntity; +import com.google.cloud.datastore.GqlQuery; +import com.google.cloud.datastore.Key; +import com.google.cloud.datastore.KeyFactory; +import com.google.cloud.datastore.QueryResults; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +/** Unit tests for {@link DatastoreResourceManager}. */ +public class DatastoreResourceManagerTest { + + @Mock private Datastore datastoreMock; + private DatastoreResourceManager resourceManager; + @Mock private KeyFactory keyFactory; + @Mock private Key key; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + + when(datastoreMock.newKeyFactory()).thenReturn(keyFactory); + when(keyFactory.newKey(anyLong())).thenReturn(key); + when(keyFactory.setKind(anyString())).thenReturn(keyFactory); + when(keyFactory.setNamespace(anyString())).thenReturn(keyFactory); + + resourceManager = new DatastoreResourceManager("test-namespace", datastoreMock); + } + + @Test + public void testInsert() { + // Prepare test data + Map> entities = new HashMap<>(); + Entity entity = Entity.newBuilder(datastoreMock.newKeyFactory().newKey(1L)).build(); + entities.put(1L, entity); + + // Mock the Datastore put method + when(datastoreMock.put(any(FullEntity.class))).thenReturn(entity); + + // Execute the method under test + List result = resourceManager.insert("test_kind", entities); + + // Verify the result + assertThat(result).hasSize(1); + assertThat(result).contains(entity); + } + + @Test + public void testQuery() { + // Prepare test data + String gqlQuery = "SELECT * FROM test_kind"; + + // Mock the Datastore run method + QueryResults mockResult = mock(QueryResults.class); + Entity mockEntity = mock(Entity.class); + when(datastoreMock.run(any(GqlQuery.class))).thenReturn(mockResult); + when(mockResult.hasNext()).thenReturn(true).thenReturn(false); + when(mockResult.next()).thenReturn(mockEntity); + + Key mockKey = mock(Key.class); + when(mockEntity.getKey()).thenReturn(mockKey); + when(mockKey.getNamespace()).thenReturn("test-namespace"); + + // Execute the method under test + List result = resourceManager.query(gqlQuery); + resourceManager.cleanupAll(); + + // Verify the result + assertThat(result).isNotEmpty(); + assertThat(result.get(0)).isEqualTo(mockEntity); + + verify(datastoreMock).delete(mockKey); + } + + @Test + public void testCleanupAll() { + // Prepare test data + Key key = datastoreMock.newKeyFactory().newKey(1L); + resourceManager.insert( + "test_kind", Collections.singletonMap(1L, Entity.newBuilder(key).build())); + + // Execute the method under test + // Calling twice to assert that the key is just deleted once + resourceManager.cleanupAll(); + resourceManager.cleanupAll(); + + // Verify that the Datastore delete method was called with the correct key + verify(datastoreMock, times(1)).delete(key); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/datastream/DatastreamResourceManagerTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/datastream/DatastreamResourceManagerTest.java new file mode 100644 index 0000000000000..d0afe8ae90f4c --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/datastream/DatastreamResourceManagerTest.java @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.datastream; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.cloud.datastream.v1.ConnectionProfile; +import com.google.cloud.datastream.v1.CreateConnectionProfileRequest; +import com.google.cloud.datastream.v1.CreateStreamRequest; +import com.google.cloud.datastream.v1.DatastreamClient; +import com.google.cloud.datastream.v1.DeleteConnectionProfileRequest; +import com.google.cloud.datastream.v1.DeleteStreamRequest; +import com.google.cloud.datastream.v1.DestinationConfig; +import com.google.cloud.datastream.v1.SourceConfig; +import com.google.cloud.datastream.v1.Stream; +import com.google.cloud.datastream.v1.Stream.State; +import com.google.cloud.datastream.v1.UpdateStreamRequest; +import java.util.concurrent.ExecutionException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Unit test for {@link DatastreamResourceManager}. */ +@RunWith(JUnit4.class) +public class DatastreamResourceManagerTest { + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + private static final String CONNECTION_PROFILE_ID = "test-connection-profile-id"; + private static final String STREAM_ID = "test-stream-id"; + private static final String PROJECT_ID = "test-project"; + private static final String LOCATION = "test-location"; + private static final String BUCKET = "test-bucket"; + private static final String ROOT_PATH = "/test-root-path"; + private static final int RESOURCE_COUNT = 5; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private DatastreamClient datastreamClient; + + private DatastreamResourceManager testManager; + + @Before + public void setup() { + testManager = + new DatastreamResourceManager( + datastreamClient, DatastreamResourceManager.builder(PROJECT_ID, LOCATION, null)); + } + + @Test + public void testBuilderWithInvalidProjectShouldFail() { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> DatastreamResourceManager.builder("", LOCATION, null)); + assertThat(exception).hasMessageThat().contains("projectID can not be null or empty"); + } + + @Test + public void testBuilderWithInvalidLocationShouldFail() { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> DatastreamResourceManager.builder(PROJECT_ID, "", null)); + assertThat(exception).hasMessageThat().contains("location can not be null or empty"); + } + + @Test + public void testCreateBQDestinationConnectionProfileExecutionExceptionShouldFail() + throws ExecutionException, InterruptedException { + when(datastreamClient + .createConnectionProfileAsync(any(CreateConnectionProfileRequest.class)) + .get()) + .thenThrow(ExecutionException.class); + DatastreamResourceManagerException exception = + assertThrows( + DatastreamResourceManagerException.class, + () -> testManager.createBQDestinationConnectionProfile(CONNECTION_PROFILE_ID)); + assertThat(exception) + .hasMessageThat() + .contains("Failed to create BQ destination connection profile."); + } + + @Test + public void testCreateBQDestinationConnectionInterruptedExceptionShouldFail() + throws ExecutionException, InterruptedException { + when(datastreamClient + .createConnectionProfileAsync(any(CreateConnectionProfileRequest.class)) + .get()) + .thenThrow(InterruptedException.class); + DatastreamResourceManagerException exception = + assertThrows( + DatastreamResourceManagerException.class, + () -> testManager.createBQDestinationConnectionProfile(CONNECTION_PROFILE_ID)); + assertThat(exception) + .hasMessageThat() + .contains("Failed to create BQ destination connection profile."); + } + + @Test + public void testCreateBQDestinationConnectionShouldCreateSuccessfully() + throws ExecutionException, InterruptedException { + ConnectionProfile connectionProfile = ConnectionProfile.getDefaultInstance(); + when(datastreamClient + .createConnectionProfileAsync(any(CreateConnectionProfileRequest.class)) + .get()) + .thenReturn(connectionProfile); + assertThat(testManager.createBQDestinationConnectionProfile(CONNECTION_PROFILE_ID)) + .isEqualTo(connectionProfile); + } + + @Test + public void testCreateGCSDestinationConnectionProfileWithInvalidGCSRootPathShouldFail() { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> + testManager.createGCSDestinationConnectionProfile( + CONNECTION_PROFILE_ID, BUCKET, "invalid")); + assertThat(exception) + .hasMessageThat() + .contains("gcsRootPath must either be an empty string or start with a '/'"); + } + + @Test + public void testCreateGCSDestinationConnectionProfileExecutionExceptionShouldFail() + throws ExecutionException, InterruptedException { + when(datastreamClient + .createConnectionProfileAsync(any(CreateConnectionProfileRequest.class)) + .get()) + .thenThrow(ExecutionException.class); + DatastreamResourceManagerException exception = + assertThrows( + DatastreamResourceManagerException.class, + () -> + testManager.createGCSDestinationConnectionProfile( + CONNECTION_PROFILE_ID, BUCKET, ROOT_PATH)); + assertThat(exception) + .hasMessageThat() + .contains("Failed to create GCS source connection profile."); + } + + @Test + public void testCreateGCSDestinationConnectionProfileInterruptedExceptionShouldFail() + throws ExecutionException, InterruptedException { + when(datastreamClient + .createConnectionProfileAsync(any(CreateConnectionProfileRequest.class)) + .get()) + .thenThrow(InterruptedException.class); + DatastreamResourceManagerException exception = + assertThrows( + DatastreamResourceManagerException.class, + () -> + testManager.createGCSDestinationConnectionProfile( + CONNECTION_PROFILE_ID, BUCKET, ROOT_PATH)); + assertThat(exception) + .hasMessageThat() + .contains("Failed to create GCS source connection profile."); + } + + @Test + public void testCreateGCSDestinationConnectionShouldCreateSuccessfully() + throws ExecutionException, InterruptedException { + ConnectionProfile connectionProfile = ConnectionProfile.getDefaultInstance(); + when(datastreamClient + .createConnectionProfileAsync(any(CreateConnectionProfileRequest.class)) + .get()) + .thenReturn(connectionProfile); + assertThat( + testManager.createGCSDestinationConnectionProfile( + CONNECTION_PROFILE_ID, BUCKET, ROOT_PATH)) + .isEqualTo(connectionProfile); + } + + @Test + public void testCreateStreamExecutionExceptionShouldFail() + throws ExecutionException, InterruptedException { + when(datastreamClient.createStreamAsync(any(CreateStreamRequest.class)).get()) + .thenThrow(ExecutionException.class); + DatastreamResourceManagerException exception = + assertThrows( + DatastreamResourceManagerException.class, + () -> + testManager.createStream( + STREAM_ID, + SourceConfig.getDefaultInstance(), + DestinationConfig.getDefaultInstance())); + assertThat(exception).hasMessageThat().contains("Failed to create stream."); + } + + @Test + public void testCreateStreamInterruptedExceptionShouldFail() + throws ExecutionException, InterruptedException { + when(datastreamClient.createStreamAsync(any(CreateStreamRequest.class)).get()) + .thenThrow(InterruptedException.class); + DatastreamResourceManagerException exception = + assertThrows( + DatastreamResourceManagerException.class, + () -> + testManager.createStream( + STREAM_ID, + SourceConfig.getDefaultInstance(), + DestinationConfig.getDefaultInstance())); + assertThat(exception).hasMessageThat().contains("Failed to create stream."); + } + + @Test + public void testCreateStreamShouldCreateSuccessfully() + throws ExecutionException, InterruptedException { + Stream stream = Stream.getDefaultInstance(); + when(datastreamClient.createStreamAsync(any(CreateStreamRequest.class)).get()) + .thenReturn(stream); + assertThat( + testManager.createStream( + STREAM_ID, + SourceConfig.getDefaultInstance(), + DestinationConfig.getDefaultInstance())) + .isEqualTo(stream); + } + + @Test + public void testUpdateStreamStateInterruptedExceptionShouldFail() + throws ExecutionException, InterruptedException { + when(datastreamClient.updateStreamAsync(any(UpdateStreamRequest.class)).get()) + .thenThrow(InterruptedException.class); + DatastreamResourceManagerException exception = + assertThrows( + DatastreamResourceManagerException.class, + () -> testManager.updateStreamState(STREAM_ID, State.RUNNING)); + assertThat(exception).hasMessageThat().contains("Failed to update stream."); + } + + @Test + public void testUpdateStreamStateExecutionExceptionShouldFail() + throws ExecutionException, InterruptedException { + when(datastreamClient.updateStreamAsync(any(UpdateStreamRequest.class)).get()) + .thenThrow(ExecutionException.class); + DatastreamResourceManagerException exception = + assertThrows( + DatastreamResourceManagerException.class, + () -> testManager.updateStreamState(STREAM_ID, State.RUNNING)); + assertThat(exception).hasMessageThat().contains("Failed to update stream."); + } + + @Test + public void testUpdateStreamStateShouldCreateSuccessfully() + throws ExecutionException, InterruptedException { + Stream stream = Stream.getDefaultInstance(); + when(datastreamClient.updateStreamAsync(any(UpdateStreamRequest.class)).get()) + .thenReturn(stream); + assertThat(testManager.updateStreamState(STREAM_ID, State.RUNNING)).isEqualTo(stream); + } + + @Test + public void testCleanupAllShouldDeleteSuccessfullyWhenNoErrorIsThrown() { + doNothing().when(datastreamClient).close(); + + for (int i = 0; i < RESOURCE_COUNT; i++) { + testManager.createGCSDestinationConnectionProfile( + "gcs-" + CONNECTION_PROFILE_ID + i, BUCKET, ROOT_PATH); + testManager.createBQDestinationConnectionProfile("bq-" + CONNECTION_PROFILE_ID + i); + testManager.createStream( + STREAM_ID + i, SourceConfig.getDefaultInstance(), DestinationConfig.getDefaultInstance()); + } + + testManager.cleanupAll(); + verify(datastreamClient, times(RESOURCE_COUNT)) + .deleteStreamAsync(any(DeleteStreamRequest.class)); + verify(datastreamClient, times(RESOURCE_COUNT * 2)) + .deleteConnectionProfileAsync(any(DeleteConnectionProfileRequest.class)); + verify(datastreamClient).close(); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/kms/KMSResourceManagerIT.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/kms/KMSResourceManagerIT.java new file mode 100644 index 0000000000000..1dd0a5eda3073 --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/kms/KMSResourceManagerIT.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.kms; + +import static com.google.common.truth.Truth.assertThat; + +import org.apache.beam.it.gcp.GCPBaseIT; +import org.apache.beam.it.gcp.GoogleCloudIntegrationTest; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Integration test for KMS Resource Manager. */ +@Category(GoogleCloudIntegrationTest.class) +@RunWith(JUnit4.class) +public class KMSResourceManagerIT extends GCPBaseIT { + + private static final String KEYRING_ID = "KMSResourceManagerIT"; + private static final String KEY_ID = "testKey"; + private static final String KMS_REGION = "global"; + + private KMSResourceManager kmsResourceManager; + + @Before + public void setUp() { + kmsResourceManager = + KMSResourceManager.builder(PROJECT, credentialsProvider).setRegion(KMS_REGION).build(); + } + + @Test + public void testKMSResourceManagerE2E() { + + String message = RandomStringUtils.randomAlphanumeric(5, 20); + + kmsResourceManager.getOrCreateCryptoKey(KEYRING_ID, KEY_ID); + String encryptedMessage = kmsResourceManager.encrypt(KEYRING_ID, KEY_ID, message); + String decryptedMessage = kmsResourceManager.decrypt(KEYRING_ID, KEY_ID, encryptedMessage); + + assertThat(encryptedMessage).isNotEqualTo(message); + assertThat(decryptedMessage).isEqualTo(message); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/kms/KMSResourceManagerTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/kms/KMSResourceManagerTest.java new file mode 100644 index 0000000000000..a386eb63fc6a0 --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/kms/KMSResourceManagerTest.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.kms; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.cloud.kms.v1.CryptoKey; +import com.google.cloud.kms.v1.CryptoKeyName; +import com.google.cloud.kms.v1.DecryptResponse; +import com.google.cloud.kms.v1.EncryptResponse; +import com.google.cloud.kms.v1.KeyManagementServiceClient; +import com.google.cloud.kms.v1.KeyRing; +import com.google.cloud.kms.v1.KeyRingName; +import com.google.cloud.kms.v1.LocationName; +import com.google.protobuf.ByteString; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Unit tests for {@link KMSResourceManager}. */ +@RunWith(JUnit4.class) +public class KMSResourceManagerTest { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + private static final String PROJECT_ID = "test-project"; + private static final String REGION = "us-central1"; + private static final String KEYRING_ID = "test-keyring"; + private static final String KEY_ID = "test-key"; + + @Mock private KMSClientFactory kmsClientFactory; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private KeyManagementServiceClient serviceClient; + + private KMSResourceManager testManager; + + @Before + public void setUp() { + testManager = + new KMSResourceManager( + kmsClientFactory, KMSResourceManager.builder(PROJECT_ID, null).setRegion(REGION)); + } + + @Test + public void testGetOrCreateCryptoKeyShouldThrowErrorWhenClientFailsToConnect() { + when(kmsClientFactory.getKMSClient()).thenThrow(KMSResourceManagerException.class); + + assertThrows( + KMSResourceManagerException.class, + () -> testManager.getOrCreateCryptoKey(KEYRING_ID, KEY_ID)); + } + + @Test + public void testGetOrCreateCryptoKeyShouldCreateKeyRingWhenItDoesNotExist() { + when(kmsClientFactory.getKMSClient()).thenReturn(serviceClient); + when(serviceClient.listKeyRings(any(LocationName.class)).iterateAll()) + .thenReturn(ImmutableList.of()); + + testManager.getOrCreateCryptoKey(KEYRING_ID, KEY_ID); + verify(serviceClient).createKeyRing(any(LocationName.class), anyString(), any(KeyRing.class)); + } + + @Test + public void testGetOrCreateCryptoKeyShouldNotCreateKeyRingWhenItAlreadyExists() { + KeyRing keyRing = + KeyRing.newBuilder() + .setName(KeyRingName.of(PROJECT_ID, REGION, KEYRING_ID).toString()) + .build(); + when(kmsClientFactory.getKMSClient()).thenReturn(serviceClient); + when(serviceClient.listKeyRings(any(LocationName.class)).iterateAll()) + .thenReturn(ImmutableList.of(keyRing)); + + testManager.getOrCreateCryptoKey(KEYRING_ID, KEY_ID); + verify(serviceClient, never()) + .createKeyRing(any(LocationName.class), anyString(), any(KeyRing.class)); + } + + @Test + public void testGetOrCreateCryptoKeyShouldCreateCryptoKeyWhenItDoesNotExist() { + KeyRing keyRing = + KeyRing.newBuilder() + .setName(KeyRingName.of(PROJECT_ID, REGION, KEYRING_ID).toString()) + .build(); + when(kmsClientFactory.getKMSClient()).thenReturn(serviceClient); + when(serviceClient.createKeyRing(any(LocationName.class), anyString(), any(KeyRing.class))) + .thenReturn(keyRing); + when(serviceClient.listCryptoKeys(KEYRING_ID).iterateAll()).thenReturn(ImmutableList.of()); + + testManager.getOrCreateCryptoKey(KEYRING_ID, KEY_ID); + verify(serviceClient).createCryptoKey(anyString(), anyString(), any(CryptoKey.class)); + } + + @Test + public void testGetOrCreateCryptoKeyShouldNotCreateCryptoKeyWhenItAlreadyExists() { + String keyRingName = KeyRingName.of(PROJECT_ID, REGION, KEYRING_ID).toString(); + KeyRing keyRing = KeyRing.newBuilder().setName(keyRingName).build(); + CryptoKey cryptoKey = + CryptoKey.newBuilder() + .setName(CryptoKeyName.of(PROJECT_ID, REGION, KEYRING_ID, KEY_ID).toString()) + .build(); + + when(kmsClientFactory.getKMSClient()).thenReturn(serviceClient); + when(serviceClient.createKeyRing(any(LocationName.class), anyString(), any(KeyRing.class))) + .thenReturn(keyRing); + when(serviceClient.listCryptoKeys(keyRingName).iterateAll()) + .thenReturn(ImmutableList.of(cryptoKey)); + + testManager.getOrCreateCryptoKey(KEYRING_ID, KEY_ID); + verify(serviceClient, never()).createCryptoKey(anyString(), anyString(), any(CryptoKey.class)); + } + + @Test + public void testEncryptShouldThrowErrorWhenClientFailsToConnect() { + when(kmsClientFactory.getKMSClient()).thenThrow(KMSResourceManagerException.class); + + assertThrows( + KMSResourceManagerException.class, + () -> testManager.encrypt(KEYRING_ID, KEY_ID, "test message")); + } + + @Test + public void testEncryptShouldEncodeEncryptedMessageWithBase64() { + String ciphertext = "ciphertext"; + EncryptResponse encryptedResponse = + EncryptResponse.newBuilder().setCiphertext(ByteString.copyFromUtf8(ciphertext)).build(); + + when(kmsClientFactory.getKMSClient()).thenReturn(serviceClient); + when(serviceClient.encrypt(any(CryptoKeyName.class), any(ByteString.class))) + .thenReturn(encryptedResponse); + + String encryptedMessage = testManager.encrypt(KEYRING_ID, KEY_ID, "test message"); + String actual = + new String( + Base64.getDecoder().decode(encryptedMessage.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8); + + assertThat(actual).isEqualTo(ciphertext); + } + + @Test + public void testDecryptShouldThrowErrorWhenClientFailsToConnect() { + when(kmsClientFactory.getKMSClient()).thenThrow(KMSResourceManagerException.class); + + assertThrows( + KMSResourceManagerException.class, + () -> testManager.decrypt(KEYRING_ID, KEY_ID, "ciphertext")); + } + + @Test + public void testDecryptShouldEncodeEncryptedMessageWithUTF8() { + String ciphertext = "ciphertext"; + DecryptResponse decryptedResponse = + DecryptResponse.newBuilder().setPlaintext(ByteString.copyFromUtf8(ciphertext)).build(); + String base64EncodedCiphertext = + new String( + Base64.getEncoder().encode(ciphertext.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8); + + when(kmsClientFactory.getKMSClient()).thenReturn(serviceClient); + when(serviceClient.decrypt(any(CryptoKeyName.class), any(ByteString.class))) + .thenReturn(decryptedResponse); + + String actual = testManager.decrypt(KEYRING_ID, KEY_ID, base64EncodedCiphertext); + + verify(serviceClient) + .decrypt(any(CryptoKeyName.class), eq(ByteString.copyFromUtf8(ciphertext))); + assertThat(actual).isEqualTo(ciphertext); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/pubsub/PubsubResourceManagerTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/pubsub/PubsubResourceManagerTest.java new file mode 100644 index 0000000000000..d531862d96026 --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/pubsub/PubsubResourceManagerTest.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.pubsub; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.core.ApiFutures; +import com.google.cloud.pubsub.v1.Publisher; +import com.google.cloud.pubsub.v1.SchemaServiceClient; +import com.google.cloud.pubsub.v1.SubscriptionAdminClient; +import com.google.cloud.pubsub.v1.TopicAdminClient; +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.v1.Subscription; +import com.google.pubsub.v1.SubscriptionName; +import com.google.pubsub.v1.Topic; +import com.google.pubsub.v1.TopicName; +import java.io.IOException; +import java.util.Map; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Unit tests for {@link PubsubResourceManager}. */ +@RunWith(JUnit4.class) +public final class PubsubResourceManagerTest { + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + private static final String TEST_ID = "test-id"; + private static final String PROJECT_ID = "test-project"; + private static final String TOPIC_NAME = "test-topic-name"; + private static final String SUBSCRIPTION_NAME = "test-topic-name-sub0"; + private static final TopicName TOPIC_REFERENCE = TopicName.of("test-project", "test-topic"); + private static final String VALID_MESSAGE_ID = "abcdef"; + + @Mock private TopicAdminClient topicAdminClient; + @Mock private SubscriptionAdminClient subscriptionAdminClient; + + @Mock private SchemaServiceClient schemaServiceClient; + private Topic topic; + private Subscription subscription; + @Mock private Publisher publisher; + @Mock private PubsubPublisherFactory publisherFactory; + + private PubsubResourceManager testManager; + + @Captor private ArgumentCaptor topicNameCaptor; + @Captor private ArgumentCaptor subscriptionNameCaptor; + @Captor private ArgumentCaptor pubsubMessageCaptor; + + @Before + public void setUp() throws IOException { + // Using spy to inject our mocked publisher at getPublisher(topic) + testManager = + new PubsubResourceManager( + TEST_ID, + PROJECT_ID, + publisherFactory, + topicAdminClient, + subscriptionAdminClient, + schemaServiceClient); + + topic = Topic.newBuilder().setName(TopicName.of(PROJECT_ID, TOPIC_NAME).toString()).build(); + subscription = + Subscription.newBuilder() + .setName(SubscriptionName.of(PROJECT_ID, SUBSCRIPTION_NAME).toString()) + .build(); + when(publisherFactory.createPublisher(any())).thenReturn(publisher); + } + + @Test + public void testBuilderWithInvalidProjectShouldFail() { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> PubsubResourceManager.builder("test-a", "", null)); + assertThat(exception).hasMessageThat().contains("projectId can not be empty"); + } + + @Test + public void testCreateTopicWithInvalidNameShouldFail() { + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> testManager.createTopic("")); + assertThat(exception).hasMessageThat().contains("topicName can not be empty"); + } + + @Test + public void testCreateTopicWithoutPrefixWithInvalidNameShouldFail() { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, () -> testManager.createTopicWithoutPrefix("")); + assertThat(exception).hasMessageThat().contains("topicName can not be empty"); + } + + @Test + public void testCreateSubscriptionWithInvalidNameShouldFail() { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> testManager.createSubscription(TopicName.of(PROJECT_ID, "topic-a"), "")); + assertThat(exception).hasMessageThat().contains("subscriptionName can not be empty"); + } + + @Test + public void testCreateTopicShouldCreate() { + when(topicAdminClient.createTopic(any(TopicName.class))).thenReturn(topic); + + TopicName createTopic = testManager.createTopic("topic-name"); + + assertThat(createTopic).isNotNull(); + verify(topicAdminClient).createTopic(topicNameCaptor.capture()); + TopicName actualTopicName = topicNameCaptor.getValue(); + assertThat(actualTopicName.getProject()).isEqualTo(PROJECT_ID); + assertThat(actualTopicName.getTopic()).matches(TEST_ID + "-\\d{17}-topic-name"); + } + + @Test + public void testCreateTopicWithoutPrefixShouldCreate() { + when(topicAdminClient.createTopic(any(TopicName.class))).thenReturn(topic); + + TopicName createTopic = testManager.createTopicWithoutPrefix("topic-name"); + + assertThat(createTopic).isNotNull(); + verify(topicAdminClient).createTopic(topicNameCaptor.capture()); + TopicName actualTopicName = topicNameCaptor.getValue(); + assertThat(actualTopicName.getProject()).isEqualTo(PROJECT_ID); + assertThat(actualTopicName.getTopic()).matches("topic-name"); + } + + @Test + public void testCreateSubscriptionShouldCreate() { + when(topicAdminClient.createTopic(any(TopicName.class))).thenReturn(topic); + when(subscriptionAdminClient.createSubscription( + any(SubscriptionName.class), any(TopicName.class), any(), anyInt())) + .thenReturn(subscription); + + TopicName createTopic = testManager.createTopic("topic-name"); + SubscriptionName createSub = testManager.createSubscription(createTopic, "subscription-name"); + + assertThat(createSub).isNotNull(); + verify(subscriptionAdminClient) + .createSubscription( + subscriptionNameCaptor.capture(), topicNameCaptor.capture(), any(), anyInt()); + SubscriptionName subscriptionName = subscriptionNameCaptor.getValue(); + TopicName actualTopicName = topicNameCaptor.getValue(); + assertThat(subscriptionName.getProject()).isEqualTo(PROJECT_ID); + assertThat(subscriptionName.getSubscription()).matches(TEST_ID + "-\\d{17}-subscription-name"); + assertThat(actualTopicName.getProject()).isEqualTo(PROJECT_ID); + assertThat(actualTopicName.getTopic()).matches(createTopic.getTopic()); + } + + @Test + public void testCreateSubscriptionUnmanagedTopicShouldFail() { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> + testManager.createSubscription( + TopicName.of(PROJECT_ID, "topic-name"), "subscription-name")); + assertThat(exception).hasMessageThat().contains("topic not managed"); + } + + @Test + public void testPublishMessageShouldPublish() { + when(topicAdminClient.createTopic(any(TopicName.class))).thenReturn(topic); + when(publisher.publish(any())).thenReturn(ApiFutures.immediateFuture(VALID_MESSAGE_ID)); + Map attributes = ImmutableMap.of("key1", "value1"); + ByteString data = ByteString.copyFromUtf8("valid message"); + + TopicName topic = testManager.createTopic(TOPIC_NAME); + String publishMessage = testManager.publish(topic, attributes, data); + + assertThat(publishMessage).isEqualTo(VALID_MESSAGE_ID); + verify(publisher).publish(pubsubMessageCaptor.capture()); + PubsubMessage actualMessage = pubsubMessageCaptor.getValue(); + assertThat(actualMessage.getAttributesMap()).isEqualTo(attributes); + assertThat(actualMessage.getData()).isEqualTo(data); + } + + @Test + public void testPublishMessageUnmanagedTopicShouldFail() { + Map attributes = ImmutableMap.of("key1", "value1"); + ByteString data = ByteString.copyFromUtf8("valid message"); + + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> testManager.publish(TOPIC_REFERENCE, attributes, data)); + assertThat(exception).hasMessageThat().contains("topic not managed"); + } + + @Test + public void testCleanupTopicsShouldDeleteTopics() { + TopicName topicName1 = testManager.getTopicName("topic1"); + TopicName topicName2 = testManager.getTopicName("topic2"); + Topic topic1 = Topic.newBuilder().setName(topicName1.toString()).build(); + Topic topic2 = Topic.newBuilder().setName(topicName2.toString()).build(); + when(topicAdminClient.createTopic(any(TopicName.class))).thenReturn(topic1, topic2); + + testManager.createTopic("topic1"); + testManager.createTopic("topic2"); + testManager.cleanupAll(); + + verify(topicAdminClient, times(2)).deleteTopic(topicNameCaptor.capture()); + assertThat(topicNameCaptor.getAllValues()).hasSize(2); + assertThat(topicNameCaptor.getAllValues()).containsExactly(topicName1, topicName2); + } + + @Test + public void testCleanupSubscriptionsShouldDeleteResources() { + SubscriptionName subscriptionName1 = testManager.getSubscriptionName("topic1-sub0"); + SubscriptionName subscriptionName2 = testManager.getSubscriptionName("topic1-sub1"); + SubscriptionName subscriptionName3 = testManager.getSubscriptionName("topic2-sub0"); + Subscription subscription1 = + Subscription.newBuilder().setName(subscriptionName1.toString()).build(); + Subscription subscription2 = + Subscription.newBuilder().setName(subscriptionName2.toString()).build(); + Subscription subscription3 = + Subscription.newBuilder().setName(subscriptionName3.toString()).build(); + when(subscriptionAdminClient.createSubscription( + any(SubscriptionName.class), any(TopicName.class), any(), anyInt())) + .thenReturn(subscription1, subscription2, subscription3); + Topic topic1 = + Topic.newBuilder().setName(testManager.getTopicName("topic1").toString()).build(); + Topic topic2 = + Topic.newBuilder().setName(testManager.getTopicName("topic2").toString()).build(); + when(topicAdminClient.createTopic(any(TopicName.class))).thenReturn(topic1, topic2); + + TopicName createdTopic1 = testManager.createTopic("topic1"); + TopicName createdTopic2 = testManager.createTopic("topic2"); + testManager.createSubscription(createdTopic1, "topic1-sub0"); + testManager.createSubscription(createdTopic1, "topic1-sub1"); + testManager.createSubscription(createdTopic2, "topic2-sub0"); + testManager.cleanupAll(); + + verify(subscriptionAdminClient, times(3)).deleteSubscription(subscriptionNameCaptor.capture()); + assertThat(subscriptionNameCaptor.getAllValues()).hasSize(3); + assertThat(subscriptionNameCaptor.getAllValues()) + .containsExactly(subscriptionName1, subscriptionName2, subscriptionName3); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/pubsub/PubsubUtilsTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/pubsub/PubsubUtilsTest.java new file mode 100644 index 0000000000000..8e6703adf457f --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/pubsub/PubsubUtilsTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.pubsub; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.gcp.pubsub.PubsubUtils.createTestId; +import static org.apache.beam.it.gcp.pubsub.PubsubUtils.toSubscriptionName; +import static org.apache.beam.it.gcp.pubsub.PubsubUtils.toTopicName; +import static org.junit.Assert.assertThrows; + +import com.google.pubsub.v1.Subscription; +import com.google.pubsub.v1.SubscriptionName; +import com.google.pubsub.v1.Topic; +import com.google.pubsub.v1.TopicName; +import org.junit.Test; + +/** Unit tests for {@link PubsubUtils}. */ +public class PubsubUtilsTest { + + @Test + public void testToTopicNameValid() { + TopicName topicName = + toTopicName(Topic.newBuilder().setName("projects/project-a/topics/topic-x").build()); + assertThat(topicName.getProject()).isEqualTo("project-a"); + assertThat(topicName.getTopic()).isEqualTo("topic-x"); + } + + @Test + public void testToTopicNameInvalidShouldFail() { + assertThrows( + IllegalArgumentException.class, + () -> toTopicName(Topic.newBuilder().setName("project-a.topic-x").build())); + } + + @Test + public void testToSubscriptionNameValid() { + SubscriptionName subscriptionName = + toSubscriptionName( + Subscription.newBuilder() + .setName("projects/project-a/subscriptions/topic-x-sub0") + .build()); + assertThat(subscriptionName.getProject()).isEqualTo("project-a"); + assertThat(subscriptionName.getSubscription()).isEqualTo("topic-x-sub0"); + } + + @Test + public void testToSubscriptionNameInvalidShouldFail() { + assertThrows( + IllegalArgumentException.class, + () -> + toSubscriptionName( + Subscription.newBuilder().setName("project-a.topic-x-sub0").build())); + } + + @Test + public void testCreateTopicName() { + String name = "create-topic-name"; + assertThat(createTestId(name)).matches(name + "-\\d{17}"); + } + + @Test + public void testCreateTopicNameWithUppercase() { + assertThat(createTestId("testWithUpperCase")).matches("test-with-upper-case-\\d{17}"); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/secretmanager/SecretManagerResourceManagerTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/secretmanager/SecretManagerResourceManagerTest.java new file mode 100644 index 0000000000000..507d53ba327d2 --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/secretmanager/SecretManagerResourceManagerTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.secretmanager; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.cloud.secretmanager.v1.ProjectName; +import com.google.cloud.secretmanager.v1.Secret; +import com.google.cloud.secretmanager.v1.SecretManagerServiceClient; +import com.google.cloud.secretmanager.v1.SecretName; +import java.io.IOException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Unit tests for {@link SecretManagerResourceManager}. */ +@RunWith(JUnit4.class) +public final class SecretManagerResourceManagerTest { + private static final String PROJECT_ID = "testProject"; + private static final String SECRET_ID = "testSecretId"; + private static final String SECRET_DATA = "testSecretData"; + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + @Mock private SecretManagerServiceClient secretManagerServiceClient; + private SecretManagerResourceManager testManager; + + @Captor private ArgumentCaptor secretCaptor; + @Captor private ArgumentCaptor secretNameStringCaptor; + + @Captor private ArgumentCaptor secretNameClassCaptor; + @Captor private ArgumentCaptor projectNameCaptor; + + @Before + public void setUp() throws IOException { + testManager = new SecretManagerResourceManager(PROJECT_ID, secretManagerServiceClient); + } + + @Test + public void testBuilderWithInvalidProjectShouldFail() { + + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, () -> SecretManagerResourceManager.builder("", null)); + assertThat(exception).hasMessageThat().contains("projectId can not be empty"); + } + + @Test + public void testCreateSecretWithInvalidNameShouldFail() { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, () -> testManager.createSecret("", SECRET_DATA)); + assertThat(exception).hasMessageThat().contains("secretId can not be empty"); + } + + @Test + public void testAddSecretVersionWithInvalidNameShouldFail() { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, () -> testManager.addSecretVersion(SECRET_ID, "")); + assertThat(exception).hasMessageThat().contains("secretData can not be empty"); + } + + @Test + public void testAccessSecretWithInvalidNameShouldFail() { + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> testManager.accessSecret("")); + assertThat(exception).hasMessageThat().contains("secretVersion can not be empty"); + } + + @Test + public void testCreateSecretShouldCreate() { + Secret secret = Secret.getDefaultInstance(); + when(secretManagerServiceClient.createSecret( + any(ProjectName.class), any(String.class), any(Secret.class))) + .thenReturn(secret); + + testManager.createSecret(SECRET_ID, SECRET_DATA); + + verify(secretManagerServiceClient) + .createSecret( + projectNameCaptor.capture(), secretNameStringCaptor.capture(), secretCaptor.capture()); + ProjectName actualProjectName = projectNameCaptor.getValue(); + assertThat(actualProjectName.getProject()).isEqualTo(PROJECT_ID); + assertThat(secretNameStringCaptor.getValue()).matches(SECRET_ID); + } + + @Test + public void testCleanupTopicsShouldDeleteTopics() { + Secret secret = Secret.getDefaultInstance(); + when(secretManagerServiceClient.createSecret( + any(ProjectName.class), any(String.class), any(Secret.class))) + .thenReturn(secret); + + testManager.createSecret("secret_id_test_1", "secret_data_test_1"); + testManager.cleanupAll(); + + verify(secretManagerServiceClient, times(1)).deleteSecret(secretNameClassCaptor.capture()); + assertThat(secretNameClassCaptor.getAllValues()).hasSize(1); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/spanner/SpannerResourceManagerTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/spanner/SpannerResourceManagerTest.java new file mode 100644 index 0000000000000..0d3aed34f3a0b --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/spanner/SpannerResourceManagerTest.java @@ -0,0 +1,540 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.spanner; + +import static com.google.cloud.spanner.Value.int64; +import static com.google.cloud.spanner.Value.string; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.cloud.Timestamp; +import com.google.cloud.spanner.Database; +import com.google.cloud.spanner.DatabaseAdminClient; +import com.google.cloud.spanner.Dialect; +import com.google.cloud.spanner.Instance; +import com.google.cloud.spanner.InstanceAdminClient; +import com.google.cloud.spanner.Mutation; +import com.google.cloud.spanner.ResultSet; +import com.google.cloud.spanner.Spanner; +import com.google.cloud.spanner.SpannerException; +import com.google.cloud.spanner.Struct; +import com.google.common.collect.ImmutableList; +import java.util.concurrent.ExecutionException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Unit tests for {@link SpannerResourceManager}. */ +@RunWith(JUnit4.class) +public final class SpannerResourceManagerTest { + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private Spanner spanner; + + @Mock private Database database; + @Mock private Instance instance; + @Mock private InstanceAdminClient instanceAdminClient; + @Mock private DatabaseAdminClient databaseAdminClient; + @Mock private ResultSet resultSet; + + private static final String TEST_ID = "test"; + private static final String PROJECT_ID = "test-project"; + private static final String REGION = "us-east1"; + private static final Dialect DIALECT = Dialect.GOOGLE_STANDARD_SQL; + private SpannerResourceManager testManager; + + @Captor private ArgumentCaptor> writeMutationCaptor; + @Captor private ArgumentCaptor> statementCaptor; + @Captor private ArgumentCaptor instanceIdCaptor; + @Captor private ArgumentCaptor databaseIdCaptor; + + @Before + public void setUp() { + testManager = + new SpannerResourceManager(spanner, TEST_ID, PROJECT_ID, REGION, DIALECT, false, null); + } + + private void prepareCreateInstanceMock() throws ExecutionException, InterruptedException { + when(spanner.getInstanceAdminClient().createInstance(any()).get()).thenReturn(instance); + } + + @Test + public void testExecuteDdlStatementShouldThrowExceptionWhenSpannerCreateInstanceFails() + throws ExecutionException, InterruptedException { + // arrange + when(spanner.getInstanceAdminClient().createInstance(any()).get()) + .thenThrow(InterruptedException.class); + prepareCreateDatabaseMock(); + prepareUpdateDatabaseMock(); + String statement = + "CREATE TABLE Singers (\n" + + " SingerId INT64 NOT NULL,\n" + + " FirstName STRING(1024),\n" + + " LastName STRING(1024),\n" + + ") PRIMARY KEY (SingerId)"; + + // act & assert + assertThrows( + SpannerResourceManagerException.class, () -> testManager.executeDdlStatement(statement)); + } + + @Test + public void testExecuteDdlStatementShouldThrowExceptionWhenSpannerCreateDatabaseFails() + throws ExecutionException, InterruptedException { + // arrange + prepareCreateInstanceMock(); + when(spanner.getDatabaseAdminClient().createDatabase(any(), any()).get()) + .thenThrow(InterruptedException.class); + prepareUpdateDatabaseMock(); + String statement = + "CREATE TABLE Singers (\n" + + " SingerId INT64 NOT NULL,\n" + + " FirstName STRING(1024),\n" + + " LastName STRING(1024),\n" + + ") PRIMARY KEY (SingerId)"; + + // act & assert + assertThrows( + SpannerResourceManagerException.class, () -> testManager.executeDdlStatement(statement)); + } + + @Test + public void testExecuteDdlStatementShouldThrowExceptionWhenSpannerUpdateDatabaseFails() + throws ExecutionException, InterruptedException { + // arrange + prepareCreateInstanceMock(); + prepareCreateDatabaseMock(); + when(spanner.getDatabaseAdminClient().updateDatabaseDdl(any(), any(), any(), any()).get()) + .thenThrow(InterruptedException.class); + String statement = + "CREATE TABLE Singers (\n" + + " SingerId INT64 NOT NULL,\n" + + " FirstName STRING(1024),\n" + + " LastName STRING(1024),\n" + + ") PRIMARY KEY (SingerId)"; + + // act & assert + assertThrows( + SpannerResourceManagerException.class, () -> testManager.executeDdlStatement(statement)); + } + + @Test + public void testExecuteDdlStatementShouldWorkWhenSpannerDoesntThrowAnyError() + throws ExecutionException, InterruptedException { + // arrange + prepareCreateInstanceMock(); + prepareCreateDatabaseMock(); + prepareUpdateDatabaseMock(); + String statement = + "CREATE TABLE Singers (\n" + + " SingerId INT64 NOT NULL,\n" + + " FirstName STRING(1024),\n" + + " LastName STRING(1024),\n" + + ") PRIMARY KEY (SingerId)"; + + // act + testManager.executeDdlStatement(statement); + + // assert + // verify createInstance, createDatabase, and updateDatabaseDdl were called twice - once in + // create table, once in their respective prepareMock helper methods. + verify(spanner.getInstanceAdminClient(), times(2)).createInstance(any()); + verify(spanner.getDatabaseAdminClient(), times(2)).createDatabase(any(), any()); + verify(spanner.getDatabaseAdminClient(), times(2)) + .updateDatabaseDdl( + instanceIdCaptor.capture(), + databaseIdCaptor.capture(), + statementCaptor.capture(), + any()); + + String actualInstanceId = instanceIdCaptor.getValue(); + String actualDatabaseId = databaseIdCaptor.getValue(); + Iterable actualStatement = statementCaptor.getValue(); + + assertThat(actualInstanceId).matches(TEST_ID + "-\\d{8}-\\d{6}-\\d{6}"); + + assertThat(actualDatabaseId).matches(TEST_ID + "_\\d{8}_\\d{6}_\\d{6}"); + assertThat(actualStatement).containsExactlyElementsIn(ImmutableList.of(statement)); + } + + @Test + public void testWriteSingleRecordShouldWorkWhenSpannerWriteSucceeds() + throws ExecutionException, InterruptedException { + // arrange + prepareTable(); + when(spanner.getDatabaseClient(any()).write(any())).thenReturn(Timestamp.now()); + Mutation testMutation = + Mutation.newInsertOrUpdateBuilder("SingerId") + .set("SingerId") + .to(1) + .set("FirstName") + .to("Marc") + .set("LastName") + .to("Richards") + .build(); + + // act + testManager.write(testMutation); + + // assert + verify(spanner.getDatabaseClient(any())).write(writeMutationCaptor.capture()); + Iterable actualWriteMutation = writeMutationCaptor.getValue(); + assertThat(actualWriteMutation).containsExactlyElementsIn(ImmutableList.of(testMutation)); + } + + @Test + public void testWriteSingleRecordShouldThrowExceptionWhenCalledBeforeExecuteDdlStatement() { + // arrange + Mutation testMutation = + Mutation.newInsertOrUpdateBuilder("SingerId") + .set("SingerId") + .to(1) + .set("FirstName") + .to("Marc") + .set("LastName") + .to("Richards") + .build(); + + // act & assert + assertThrows(IllegalStateException.class, () -> testManager.write(testMutation)); + } + + @Test + public void testWriteSingleRecordShouldThrowExceptionWhenSpannerWriteFails() + throws ExecutionException, InterruptedException { + // arrange + prepareTable(); + when(spanner.getDatabaseClient(any()).write(any())).thenThrow(SpannerException.class); + Mutation testMutation = + Mutation.newInsertOrUpdateBuilder("SingerId") + .set("SingerId") + .to(1) + .set("FirstName") + .to("Marc") + .set("LastName") + .to("Richards") + .build(); + + // act & assert + assertThrows(SpannerResourceManagerException.class, () -> testManager.write(testMutation)); + } + + @Test + public void testWriteMultipleRecordsShouldWorkWhenSpannerWriteSucceeds() + throws ExecutionException, InterruptedException { + // arrange + prepareTable(); + when(spanner.getDatabaseClient(any()).write(any())).thenReturn(Timestamp.now()); + ImmutableList testMutations = + ImmutableList.of( + Mutation.newInsertOrUpdateBuilder("SingerId") + .set("SingerId") + .to(1) + .set("FirstName") + .to("Marc") + .set("LastName") + .to("Richards") + .build(), + Mutation.newInsertOrUpdateBuilder("SingerId") + .set("SingerId") + .to(2) + .set("FirstName") + .to("Catalina") + .set("LastName") + .to("Smith") + .build()); + + // act + testManager.write(testMutations); + + // assert + verify(spanner.getDatabaseClient(any())).write(writeMutationCaptor.capture()); + Iterable actualWriteMutation = writeMutationCaptor.getValue(); + + assertThat(actualWriteMutation).containsExactlyElementsIn(testMutations); + } + + @Test + public void testWriteMultipleRecordsShouldThrowExceptionWhenCalledBeforeExecuteDdlStatement() { + // arrange + ImmutableList testMutations = + ImmutableList.of( + Mutation.newInsertOrUpdateBuilder("SingerId") + .set("SingerId") + .to(1) + .set("FirstName") + .to("Marc") + .set("LastName") + .to("Richards") + .build(), + Mutation.newInsertOrUpdateBuilder("SingerId") + .set("SingerId") + .to(2) + .set("FirstName") + .to("Catalina") + .set("LastName") + .to("Smith") + .build()); + + // act & assert + assertThrows(IllegalStateException.class, () -> testManager.write(testMutations)); + } + + @Test + public void testWriteMultipleRecordsShouldThrowExceptionWhenSpannerWriteFails() + throws ExecutionException, InterruptedException { + // arrange + prepareTable(); + when(spanner.getDatabaseClient(any()).write(any())).thenThrow(SpannerException.class); + ImmutableList testMutations = + ImmutableList.of( + Mutation.newInsertOrUpdateBuilder("SingerId") + .set("SingerId") + .to(1) + .set("FirstName") + .to("Marc") + .set("LastName") + .to("Richards") + .build(), + Mutation.newInsertOrUpdateBuilder("SingerId") + .set("SingerId") + .to(2) + .set("FirstName") + .to("Catalina") + .set("LastName") + .to("Smith") + .build()); + + // act & assert + assertThrows(SpannerResourceManagerException.class, () -> testManager.write(testMutations)); + } + + @Test + public void testReadRecordsShouldWorkWhenSpannerReadSucceeds() + throws ExecutionException, InterruptedException { + // arrange + prepareTable(); + when(resultSet.next()).thenReturn(true).thenReturn(true).thenReturn(false); + Struct struct1 = + Struct.newBuilder() + .set("SingerId") + .to(int64(1)) + .set("FirstName") + .to(string("Marc")) + .set("LastName") + .to(string("Richards")) + .build(); + Struct struct2 = + Struct.newBuilder() + .set("SingerId") + .to(int64(2)) + .set("FirstName") + .to(string("Catalina")) + .set("LastName") + .to(string("Smith")) + .build(); + when(resultSet.getCurrentRowAsStruct()).thenReturn(struct1).thenReturn(struct2); + when(spanner.getDatabaseClient(any()).singleUse().read(any(), any(), any())) + .thenReturn(resultSet); + + // act + ImmutableList actual = + testManager.readTableRecords("Singers", "SingerId", "FirstName", "LastName"); + + // assert + ImmutableList expected = ImmutableList.of(struct1, struct2); + assertThat(actual).containsExactlyElementsIn(expected); + } + + @Test + public void testReadRecordsWithListOfColumnNamesShouldWorkWhenSpannerReadSucceeds() + throws ExecutionException, InterruptedException { + // arrange + prepareTable(); + when(resultSet.next()).thenReturn(true).thenReturn(false); + Struct struct = + Struct.newBuilder() + .set("SingerId") + .to(int64(1)) + .set("FirstName") + .to(string("Marc")) + .set("LastName") + .to(string("Richards")) + .build(); + when(resultSet.getCurrentRowAsStruct()).thenReturn(struct); + when(spanner.getDatabaseClient(any()).singleUse().read(any(), any(), any())) + .thenReturn(resultSet); + ImmutableList columnNames = ImmutableList.of("SingerId", "FirstName", "LastName"); + + // act + ImmutableList actual = testManager.readTableRecords("Singers", columnNames); + + // assert + ImmutableList expected = ImmutableList.of(struct); + assertThat(actual).containsExactlyElementsIn(expected); + } + + @Test + public void testReadRecordsShouldThrowExceptionWhenCalledBeforeExecuteDdlStatement() { + ImmutableList columnNames = ImmutableList.of("SingerId"); + + assertThrows( + IllegalStateException.class, () -> testManager.readTableRecords("Singers", columnNames)); + assertThrows( + IllegalStateException.class, () -> testManager.readTableRecords("Singers", "SingerId")); + } + + @Test + public void testReadRecordsShouldThrowExceptionWhenSpannerReadFails() + throws ExecutionException, InterruptedException { + // arrange + prepareTable(); + when(spanner.getDatabaseClient(any()).singleUse().read(any(), any(), any())) + .thenThrow(SpannerException.class); + ImmutableList columnNames = ImmutableList.of("SingerId"); + + // act & assert + assertThrows( + SpannerResourceManagerException.class, + () -> testManager.readTableRecords("Singers", "SingerId")); + assertThrows( + SpannerResourceManagerException.class, + () -> testManager.readTableRecords("Singers", columnNames)); + } + + @Test + public void testCleanupAllShouldThrowExceptionWhenSpannerDeleteInstanceFails() { + // arrange + doThrow(SpannerException.class).when(instanceAdminClient).deleteInstance(any()); + when(spanner.getInstanceAdminClient()).thenReturn(instanceAdminClient); + testManager = + new SpannerResourceManager(spanner, TEST_ID, PROJECT_ID, REGION, DIALECT, false, null); + + // act & assert + assertThrows(SpannerResourceManagerException.class, () -> testManager.cleanupAll()); + } + + @Test + public void testCleanupAllShouldWorkWhenSpannerDeleteInstanceSucceeds() { + // arrange + doNothing().when(instanceAdminClient).deleteInstance(any()); + when(spanner.getInstanceAdminClient()).thenReturn(instanceAdminClient); + testManager = + new SpannerResourceManager(spanner, TEST_ID, PROJECT_ID, REGION, DIALECT, false, null); + + // act + testManager.cleanupAll(); + + // assert + verify(spanner.getInstanceAdminClient()).deleteInstance(any()); + verify(spanner).close(); + } + + @Test + public void testManagerShouldBeUnusableAfterCleanup() { + // arrange + doNothing().when(instanceAdminClient).deleteInstance(any()); + when(spanner.getInstanceAdminClient()).thenReturn(instanceAdminClient); + when(spanner.isClosed()).thenReturn(true); + testManager = + new SpannerResourceManager(spanner, TEST_ID, PROJECT_ID, REGION, DIALECT, false, null); + testManager.cleanupAll(); + String statement = + "CREATE TABLE Singers (\n" + + " SingerId INT64 NOT NULL,\n" + + " FirstName STRING(1024),\n" + + " LastName STRING(1024),\n" + + ") PRIMARY KEY (SingerId)"; + Mutation testMutation = + Mutation.newInsertOrUpdateBuilder("SingerId") + .set("SingerId") + .to(1) + .set("FirstName") + .to("Marc") + .set("LastName") + .to("Richards") + .build(); + ImmutableList columnNames = ImmutableList.of("SingerId"); + + // act & assert + assertThrows(IllegalStateException.class, () -> testManager.executeDdlStatement(statement)); + assertThrows( + IllegalStateException.class, () -> testManager.readTableRecords("Singers", "SingerId")); + assertThrows( + IllegalStateException.class, () -> testManager.readTableRecords("Singers", columnNames)); + assertThrows(IllegalStateException.class, () -> testManager.write(testMutation)); + assertThrows( + IllegalStateException.class, () -> testManager.write(ImmutableList.of(testMutation))); + } + + @Test + public void testCleanupAllShouldNotDeleteInstanceWhenStatic() { + // arrange + doNothing().when(databaseAdminClient).dropDatabase(any(), any()); + when(spanner.getInstanceAdminClient()).thenReturn(instanceAdminClient); + when(spanner.getDatabaseAdminClient()).thenReturn(databaseAdminClient); + testManager = + new SpannerResourceManager( + spanner, TEST_ID, PROJECT_ID, REGION, DIALECT, true, "existing-instance"); + + // act + testManager.cleanupAll(); + + // assert + verify(spanner.getDatabaseAdminClient()).dropDatabase(eq("existing-instance"), any()); + verify(spanner.getInstanceAdminClient(), never()).deleteInstance(any()); + verify(spanner).close(); + } + + private void prepareCreateDatabaseMock() throws ExecutionException, InterruptedException { + Mockito.lenient() + .when(spanner.getDatabaseAdminClient().createDatabase(any(), any()).get()) + .thenReturn(database); + } + + private void prepareUpdateDatabaseMock() throws ExecutionException, InterruptedException { + Mockito.lenient() + .when(spanner.getDatabaseAdminClient().updateDatabaseDdl(any(), any(), any(), any()).get()) + .thenReturn(null); + } + + private void prepareTable() throws ExecutionException, InterruptedException { + prepareCreateInstanceMock(); + prepareCreateDatabaseMock(); + prepareUpdateDatabaseMock(); + testManager.executeDdlStatement(""); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/spanner/utils/SpannerResourceManagerUtilsTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/spanner/utils/SpannerResourceManagerUtilsTest.java new file mode 100644 index 0000000000000..cd1a418a186d3 --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/spanner/utils/SpannerResourceManagerUtilsTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.spanner.utils; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.gcp.spanner.utils.SpannerResourceManagerUtils.generateDatabaseId; +import static org.apache.beam.it.gcp.spanner.utils.SpannerResourceManagerUtils.generateInstanceId; +import static org.junit.Assert.assertThrows; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link SpannerResourceManagerUtils}. */ +@RunWith(JUnit4.class) +public final class SpannerResourceManagerUtilsTest { + + @Test + public void testGenerateInstanceIdShouldReplaceNonLetterFirstCharWithLetter() { + String testBaseString = "0-test"; + + String actual = generateInstanceId(testBaseString); + + assertThat(actual).matches("[a-z]-test-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testGenerateDatabaseIdShouldNotReplaceDigitLastCharWithLetter() { + String testBaseString = "db_0"; + + String actual = generateDatabaseId(testBaseString); + + assertThat(actual).matches("db_0_\\d{8}_\\d{6}_\\d{6}"); + } + + @Test + public void testGenerateDatabaseIdShouldReplaceDollarSignWithUnderscore() { + String testBaseString = "t$db"; + + String actual = generateDatabaseId(testBaseString); + + assertThat(actual).matches("t_db_\\d{8}_\\d{6}_\\d{6}"); + } + + @Test + public void testGenerateDatabaseIdShouldReplaceDotWithUnderscore() { + String testBaseString = "test.database"; + + String actual = generateDatabaseId(testBaseString); + + assertThat(actual).matches("test_da_\\d{8}_\\d{6}_\\d{6}"); + } + + @Test + public void testGenerateDatabaseIdShouldReplaceHyphenWithUnderscore() { + String testBaseString = "test-database"; + + String actual = generateDatabaseId(testBaseString); + + assertThat(actual).matches("test_da_\\d{8}_\\d{6}_\\d{6}"); + } + + @Test + public void testGenerateDatabaseIdShouldReplaceNonLetterFirstCharWithLetter() { + String testBaseString = "0_database"; + + String actual = generateDatabaseId(testBaseString); + + assertThat(actual).matches("[a-z]_datab_\\d{8}_\\d{6}_\\d{6}"); + } + + @Test + public void testGenerateDatabaseIdShouldReplaceUpperCaseLettersWithLowerCase() { + String testBaseString = "TDa"; + + String actual = generateDatabaseId(testBaseString); + + assertThat(actual).matches("tda_\\d{8}_\\d{6}_\\d{6}"); + } + + @Test + public void testGenerateDatabaseIdShouldTrimTrailingUnderscore() { + String testBaseString = "test_database___"; + + String actual = generateDatabaseId(testBaseString); + + assertThat(actual).matches("test_da_\\d{8}_\\d{6}_\\d{6}"); + } + + @Test + public void testGenerateDatabaseIdShouldThrowErrorWithEmptyInput() { + String testBaseString = ""; + + assertThrows(IllegalArgumentException.class, () -> generateDatabaseId(testBaseString)); + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/storage/FileBasedIOLT.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/storage/FileBasedIOLT.java new file mode 100644 index 0000000000000..704f8337c66ff --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/storage/FileBasedIOLT.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.storage; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; +import static org.junit.Assert.assertEquals; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.IOException; +import java.text.ParseException; +import java.time.Duration; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Base64; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.TestProperties; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.IOLoadTestBase; +import org.apache.beam.sdk.io.Compression; +import org.apache.beam.sdk.io.Read; +import org.apache.beam.sdk.io.TextIO; +import org.apache.beam.sdk.io.synthetic.SyntheticBoundedSource; +import org.apache.beam.sdk.io.synthetic.SyntheticSourceOptions; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; + +/** + * FileBasedIO performance tests. + * + *

    Example trigger command for all tests: + * + *

    + * mvn test -pl it/google-cloud-platform -am -Dtest="FileBasedIOLT" -Dproject=[gcpProject] \
    + * -DartifactBucket=[temp bucket] -DfailIfNoTests=false
    + * 
    + * + *

    Example trigger command for specific test running on direct runner: + * + *

    + * mvn test -pl it/google-cloud-platform -am -Dtest="FileBasedIOLT#testTextIOWriteThenRead" \
    + * -Dconfiguration=medium -Dproject=[gcpProject] -DartifactBucket=[temp bucket] -DfailIfNoTests=false
    + * 
    + * + *

    Example trigger command for specific test and custom data configuration: + * + *

    mvn test -pl it/google-cloud-platform -am \
    + * -Dconfiguration="{\"numRecords\":10000000,\"valueSizeBytes\":750,\"pipelineTimeout\":20,\"runner\":\"DataflowRunner\"}" \
    + * -Dtest="FileBasedIOLT#testTextIOWriteThenRead" -Dconfiguration=local -Dproject=[gcpProject] \
    + * -DartifactBucket=[temp bucket] -DfailIfNoTests=false
    + * 
    + */ +public class FileBasedIOLT extends IOLoadTestBase { + + private static final String READ_ELEMENT_METRIC_NAME = "read_count"; + + private static GcsResourceManager resourceManager; + + private String filePrefix; + + private Configuration configuration; + + @Rule public TestPipeline writePipeline = TestPipeline.create(); + + @Rule public TestPipeline readPipeline = TestPipeline.create(); + + private static final Map TEST_CONFIGS_PRESET; + + static { + try { + TEST_CONFIGS_PRESET = + ImmutableMap.of( + "local", + Configuration.fromJsonString( + "{\"numRecords\":1000,\"valueSizeBytes\":750,\"pipelineTimeout\":2,\"runner\":\"DirectRunner\"}", + Configuration + .class), // 1 MB - bytes get encoded in Base64 so 740 bytes gives 1 kB line + // size + "medium", + Configuration.fromJsonString( + "{\"numRecords\":10000000,\"valueSizeBytes\":750,\"pipelineTimeout\":20,\"runner\":\"DataflowRunner\"}", + Configuration.class), // 10 GB + "large", + Configuration.fromJsonString( + "{\"numRecords\":100000000,\"valueSizeBytes\":750,\"pipelineTimeout\":80,\"runner\":\"DataflowRunner\"}", + Configuration.class) // 100 GB + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @BeforeClass + public static void beforeClass() { + resourceManager = + GcsResourceManager.builder(TestProperties.artifactBucket(), "textiolt", CREDENTIALS) + .build(); + } + + @AfterClass + public static void tearDownClass() { + ResourceManagerUtils.cleanResources(resourceManager); + } + + @Before + public void setup() { + + // parse configuration + String testConfig = + TestProperties.getProperty("configuration", "local", TestProperties.Type.PROPERTY); + configuration = TEST_CONFIGS_PRESET.get(testConfig); + if (configuration == null) { + try { + configuration = Configuration.fromJsonString(testConfig, Configuration.class); + } catch (IOException e) { + throw new IllegalArgumentException( + String.format( + "Unknown test configuration: [%s]. Pass to a valid configuration json, or use" + + " config presets: %s", + testConfig, TEST_CONFIGS_PRESET.keySet())); + } + } + + // filePath needs to be set for TextIO writes + String tempDirName = + "textiolt-" + + DateTimeFormatter.ofPattern("MMddHHmmssSSS") + .withZone(ZoneId.of("UTC")) + .format(java.time.Instant.now()) + + UUID.randomUUID().toString().substring(0, 10); + resourceManager.registerTempDir(tempDirName); + filePrefix = String.format("gs://%s/%s/test", TestProperties.artifactBucket(), tempDirName); + } + + @Test + public void testTextIOWriteThenRead() throws IOException { + + TextIO.TypedWrite write = + TextIO.write() + .to(filePrefix) + .withOutputFilenames() + .withCompression(Compression.valueOf(configuration.compressionType)); + if (configuration.numShards > 0) { + write = write.withNumShards(configuration.numShards); + } + + writePipeline + .apply("Read from source", Read.from(new SyntheticBoundedSource(configuration))) + .apply("Map records", ParDo.of(new MapKVToString())) + .apply("Write content to files", write); + + readPipeline + .apply(TextIO.read().from(filePrefix + "*")) + .apply("Counting element", ParDo.of(new CountingFn<>(READ_ELEMENT_METRIC_NAME))); + + PipelineLauncher.LaunchConfig writeOptions = + PipelineLauncher.LaunchConfig.builder("write-textio") + .setSdk(PipelineLauncher.Sdk.JAVA) + .setPipeline(writePipeline) + .addParameter("runner", configuration.runner) + .build(); + PipelineLauncher.LaunchInfo writeInfo = pipelineLauncher.launch(project, region, writeOptions); + PipelineOperator.Result writeResult = + pipelineOperator.waitUntilDone( + createConfig(writeInfo, Duration.ofMinutes(configuration.pipelineTimeout))); + + // Fail the test if pipeline failed or timeout. + assertThatResult(writeResult).isLaunchFinished(); + + PipelineLauncher.LaunchConfig readOptions = + PipelineLauncher.LaunchConfig.builder("read-textio") + .setSdk(PipelineLauncher.Sdk.JAVA) + .setPipeline(readPipeline) + .addParameter("runner", configuration.runner) + .build(); + PipelineLauncher.LaunchInfo readInfo = pipelineLauncher.launch(project, region, readOptions); + PipelineOperator.Result readResult = + pipelineOperator.waitUntilDone( + createConfig(readInfo, Duration.ofMinutes(configuration.pipelineTimeout))); + + // Fail the test if pipeline failed or timeout. + assertThatResult(readResult).isLaunchFinished(); + + // check metrics + double numRecords = + pipelineLauncher.getMetric( + project, + region, + readInfo.jobId(), + getBeamMetricsName(PipelineMetricsType.COUNTER, READ_ELEMENT_METRIC_NAME)); + + assertEquals(configuration.numRecords, numRecords, 0.5); + + // export metrics + MetricsConfiguration metricsConfig = + MetricsConfiguration.builder() + .setInputPCollection("Map records.out0") + .setInputPCollectionV2("Map records/ParMultiDo(MapKVToString).out0") + .setOutputPCollection("Counting element.out0") + .setOutputPCollectionV2("Counting element/ParMultiDo(Counting).out0") + .build(); + try { + exportMetricsToBigQuery(writeInfo, getMetrics(writeInfo, metricsConfig)); + exportMetricsToBigQuery(readInfo, getMetrics(readInfo, metricsConfig)); + } catch (ParseException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + private static class MapKVToString extends DoFn, String> { + @ProcessElement + public void process(ProcessContext context) { + context.output( + Base64.getEncoder().encodeToString(Objects.requireNonNull(context.element()).getValue())); + } + } + + static class Configuration extends SyntheticSourceOptions { + /** Number of dynamic destinations to write. */ + @JsonProperty public int numShards = 0; + + /** See {@class org.apache.beam.sdk.io.Compression}. */ + @JsonProperty public String compressionType = "UNCOMPRESSED"; + + /** Runner specified to run the pipeline. */ + @JsonProperty public String runner = "DirectRunner"; + + /** Pipeline timeout in minutes. Must be a positive value. */ + @JsonProperty public int pipelineTimeout = 20; + } +} diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/storage/GcsResourceManagerTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/storage/GcsResourceManagerTest.java new file mode 100644 index 0000000000000..0153573feaed3 --- /dev/null +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/storage/GcsResourceManagerTest.java @@ -0,0 +1,377 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.gcp.storage; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyIterable; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.gax.paging.Page; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobListOption; +import com.google.cloud.storage.Storage.BucketListOption; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.UUID; +import java.util.regex.Pattern; +import org.apache.beam.it.gcp.artifacts.Artifact; +import org.apache.beam.it.gcp.artifacts.GcsArtifact; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Resources; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Unit tests for {@link GcsResourceManager}. */ +@RunWith(JUnit4.class) +public final class GcsResourceManagerTest { + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private Storage client; + + @Mock private Blob blob; + private GcsResourceManager gcsClient; + + private static final String ARTIFACT_NAME = "test-artifact.json"; + private static final Path LOCAL_PATH; + private static final byte[] TEST_ARTIFACT_CONTENTS; + + static { + try { + LOCAL_PATH = Paths.get(Resources.getResource(ARTIFACT_NAME).toURI()); + TEST_ARTIFACT_CONTENTS = Files.readAllBytes(LOCAL_PATH); + } catch (IOException | URISyntaxException e) { + throw new RuntimeException(e); + } + } + + private static final String BUCKET = "test-bucket"; + private static final String TEST_CLASS = "test-class-name"; + private static final String TEST_METHOD = "test-method-name"; + + @Captor private ArgumentCaptor bucketCaptor; + @Captor private ArgumentCaptor blobInfoCaptor; + @Captor private ArgumentCaptor contentsCaptor; + @Captor private ArgumentCaptor listOptionsCaptor; + @Captor private ArgumentCaptor> blobIdCaptor; + + @Before + public void setUp() { + gcsClient = new GcsResourceManager(client, BUCKET, TEST_CLASS); + } + + @After + public void tearDown() { + gcsClient.cleanupAll(); + } + + @Test + public void testBuilderWithEmptyBucket() { + assertThrows( + IllegalArgumentException.class, + () -> GcsResourceManager.builder("", TEST_CLASS, null).build()); + } + + @Test + public void testBuilderWithEmptyTestClassName() { + assertThrows( + IllegalArgumentException.class, () -> GcsResourceManager.builder(BUCKET, "", null).build()); + } + + @Test + public void testCreateArtifactInRunDir() { + String artifactName = "artifact.txt"; + byte[] contents = new byte[] {0, 1, 2}; + when(client.create(any(BlobInfo.class), any(byte[].class))).thenReturn(blob); + + GcsArtifact actual = (GcsArtifact) gcsClient.createArtifact(artifactName, contents); + + verify(client).create(blobInfoCaptor.capture(), contentsCaptor.capture()); + BlobInfo actualInfo = blobInfoCaptor.getValue(); + + assertThat(actual.blob).isSameInstanceAs(blob); + assertThat(actualInfo.getBucket()).isEqualTo(BUCKET); + assertThat(actualInfo.getName()) + .isEqualTo(String.format("%s/%s/%s", TEST_CLASS, gcsClient.runId(), artifactName)); + assertThat(contentsCaptor.getValue()).isEqualTo(contents); + } + + @Test + public void testUploadArtifact() throws IOException { + when(client.create(any(BlobInfo.class), any(byte[].class))).thenReturn(blob); + + GcsArtifact actual = (GcsArtifact) gcsClient.uploadArtifact(ARTIFACT_NAME, LOCAL_PATH); + + verify(client).create(blobInfoCaptor.capture(), contentsCaptor.capture()); + BlobInfo actualInfo = blobInfoCaptor.getValue(); + + assertThat(actual.blob).isSameInstanceAs(blob); + assertThat(actualInfo.getBucket()).isEqualTo(BUCKET); + assertThat(actualInfo.getName()) + .isEqualTo(String.format("%s/%s/%s", TEST_CLASS, gcsClient.runId(), ARTIFACT_NAME)); + assertThat(contentsCaptor.getValue()).isEqualTo(TEST_ARTIFACT_CONTENTS); + } + + @Test + public void testUploadArtifactInvalidLocalPath() { + when(client.create(any(BlobInfo.class), any())).thenReturn(blob); + assertThrows( + IOException.class, () -> gcsClient.uploadArtifact(ARTIFACT_NAME, "/" + UUID.randomUUID())); + } + + @Test + public void testListArtifactsInMethodDirSinglePage() { + // Arrange + String name1 = "blob1"; + String name2 = "blob2"; + String name3 = "blob3"; + ImmutableList page1 = + ImmutableList.of(mock(Blob.class), mock(Blob.class), mock(Blob.class)); + when(page1.get(0).getName()).thenReturn(name1); + when(page1.get(1).getName()).thenReturn(name2); + when(page1.get(2).getName()).thenReturn(name3); + + TestBlobPage allPages = createPages(page1); + when(client.list(anyString(), any(BlobListOption.class))).thenReturn(allPages); + + Pattern pattern = Pattern.compile(".*blob[13].*"); + + // Act + List actual = gcsClient.listArtifacts(TEST_METHOD, pattern); + + // Assert + verify(client).list(bucketCaptor.capture(), listOptionsCaptor.capture()); + + String actualBucket = bucketCaptor.getValue(); + BlobListOption actualOptions = listOptionsCaptor.getValue(); + + assertThat(actual).hasSize(2); + assertThat(actual.get(0).name()).isEqualTo(name1); + assertThat(actual.get(1).name()).isEqualTo(name3); + assertThat(actualBucket).isEqualTo(BUCKET); + assertThat(actualOptions) + .isEqualTo( + BucketListOption.prefix( + String.format("%s/%s/%s", TEST_CLASS, gcsClient.runId(), TEST_METHOD))); + } + + @Test + public void testListArtifactsInMethodDirMultiplePages() { + // Arrange + String name1 = "blob1"; + String name2 = "blob2"; + String name3 = "blob3"; + ImmutableList page1 = ImmutableList.of(mock(Blob.class), mock(Blob.class)); + ImmutableList page2 = ImmutableList.of(mock(Blob.class)); + when(page1.get(0).getName()).thenReturn(name1); + when(page1.get(1).getName()).thenReturn(name2); + when(page2.get(0).getName()).thenReturn(name3); + + TestBlobPage allPages = createPages(page1, page2); + when(client.list(anyString(), any(BlobListOption.class))).thenReturn(allPages); + + Pattern pattern = Pattern.compile(".*blob[13].*"); + + // Act + List actual = gcsClient.listArtifacts(TEST_METHOD, pattern); + + // Assert + verify(client).list(bucketCaptor.capture(), listOptionsCaptor.capture()); + + String actualBucket = bucketCaptor.getValue(); + BlobListOption actualOptions = listOptionsCaptor.getValue(); + + assertThat(actual).hasSize(2); + assertThat(actual.get(0).name()).isEqualTo(name1); + assertThat(actual.get(1).name()).isEqualTo(name3); + assertThat(actualBucket).isEqualTo(BUCKET); + assertThat(actualOptions) + .isEqualTo( + BucketListOption.prefix( + String.format("%s/%s/%s", TEST_CLASS, gcsClient.runId(), TEST_METHOD))); + } + + @Test + public void testListArtifactsInMethodDirNoArtifacts() { + TestBlobPage allPages = createPages(ImmutableList.of()); + when(client.list(anyString(), any(BlobListOption.class))).thenReturn(allPages); + Pattern pattern = Pattern.compile(".*blob[13].*"); + + List actual = gcsClient.listArtifacts(TEST_METHOD, pattern); + + verify(client).list(anyString(), any(BlobListOption.class)); + assertThat(actual).isEmpty(); + } + + @Test + public void testCleanupRunSinglePage() { + // Arrange + BlobId id1 = BlobId.of(BUCKET, "blob1"); + BlobId id2 = BlobId.of(BUCKET, "blob2"); + BlobId id3 = BlobId.of(BUCKET, "blob3"); + ImmutableList page1 = + ImmutableList.of(mock(Blob.class), mock(Blob.class), mock(Blob.class)); + when(page1.get(0).getBlobId()).thenReturn(id1); + when(page1.get(1).getBlobId()).thenReturn(id2); + when(page1.get(2).getBlobId()).thenReturn(id3); + + TestBlobPage allPages = createPages(page1); + when(client.list(anyString(), any(BlobListOption.class))).thenReturn(allPages); + + when(client.delete(anyIterable())).thenReturn(ImmutableList.of(true, false, true)); + + // Act + gcsClient.cleanupAll(); + + // Assert + verify(client).list(bucketCaptor.capture(), listOptionsCaptor.capture()); + verify(client).delete(blobIdCaptor.capture()); + + String actualBucket = bucketCaptor.getValue(); + BlobListOption actualOption = listOptionsCaptor.getValue(); + Iterable actualIds = blobIdCaptor.getValue(); + + assertThat(actualBucket).isEqualTo(BUCKET); + assertThat(actualOption) + .isEqualTo(BucketListOption.prefix(String.format("%s/%s", TEST_CLASS, gcsClient.runId()))); + assertThat(actualIds).containsExactly(id1, id2, id3); + } + + @Test + public void testCleanupRunMultiplePages() { + // Arrange + BlobId id1 = BlobId.of(BUCKET, "blob1"); + BlobId id2 = BlobId.of(BUCKET, "blob2"); + BlobId id3 = BlobId.of(BUCKET, "blob3"); + ImmutableList page1 = ImmutableList.of(mock(Blob.class), mock(Blob.class)); + ImmutableList page2 = ImmutableList.of(mock(Blob.class)); + when(page1.get(0).getBlobId()).thenReturn(id1); + when(page1.get(1).getBlobId()).thenReturn(id2); + when(page2.get(0).getBlobId()).thenReturn(id3); + + TestBlobPage allPages = createPages(page1, page2); + when(client.list(anyString(), any(BlobListOption.class))).thenReturn(allPages); + + when(client.delete(anyIterable())) + .thenReturn(ImmutableList.of(true, false)) + .thenReturn(ImmutableList.of(true)); + + // Act + gcsClient.cleanupAll(); + + // Assert + verify(client).list(bucketCaptor.capture(), listOptionsCaptor.capture()); + verify(client, times(2)).delete(blobIdCaptor.capture()); + + String actualBucket = bucketCaptor.getValue(); + BlobListOption actualOption = listOptionsCaptor.getValue(); + List> actualBlobIds = blobIdCaptor.getAllValues(); + + assertThat(actualBucket).isEqualTo(BUCKET); + assertThat(actualOption) + .isEqualTo(BucketListOption.prefix(String.format("%s/%s", TEST_CLASS, gcsClient.runId()))); + assertThat(actualBlobIds.get(0)).containsExactly(id1, id2); + assertThat(actualBlobIds.get(1)).containsExactly(id3); + } + + @Test + public void testDeleteArtifactsNoArtifacts() { + TestBlobPage allPages = createPages(ImmutableList.of()); + when(client.list(anyString(), any(BlobListOption.class))).thenReturn(allPages); + + gcsClient.cleanupAll(); + + verify(client, never()).delete(anyIterable()); + } + + private static TestBlobPage createPages(ImmutableList... pageContents) { + if (pageContents.length == 0) { + return new TestBlobPage(ImmutableList.of()); + } + TestBlobPage first = new TestBlobPage(pageContents[0]); + TestBlobPage current = first; + for (int i = 1; i < pageContents.length; ++i) { + current.setNext(pageContents[i]); + current = current.next; + } + return first; + } + + private static final class TestBlobPage implements Page { + private TestBlobPage next; + private final ImmutableList contents; + + public TestBlobPage(ImmutableList contents) { + this.contents = contents; + this.next = null; + } + + public void setNext(ImmutableList contents) { + next = new TestBlobPage(contents); + } + + @Override + public boolean hasNextPage() { + return next != null; + } + + @Override + public String getNextPageToken() { + return "token"; + } + + @Override + public Page getNextPage() { + return next; + } + + @Override + public Iterable iterateAll() { + return contents; + } + + @Override + public Iterable getValues() { + return contents; + } + } +} diff --git a/it/jdbc/build.gradle b/it/jdbc/build.gradle new file mode 100644 index 0000000000000..72cc8794159f6 --- /dev/null +++ b/it/jdbc/build.gradle @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.it.jdbc', + enableSpotbugs: false, +) + +description = "Apache Beam :: IT :: Jdbc" +ext.summary = "Integration test utilities for Jdbc." + +dependencies { + implementation project(path: ":it:testcontainers", configuration: "shadow") + implementation library.java.testcontainers_jdbc + implementation library.java.testcontainers_postgresql + implementation library.java.testcontainers_mysql + implementation library.java.testcontainers_oracle + implementation library.java.testcontainers_mssqlserver + + testImplementation project(":it:truthmatchers") + // TODO: excluding Guava until Truth updates it to >32.1.x + testImplementation(library.java.truth) { + exclude group: 'com.google.guava', module: 'guava' + } + testImplementation library.java.mockito_core + testImplementation library.java.mockito_inline + testRuntimeOnly library.java.slf4j_simple +} diff --git a/it/jdbc/src/main/java/org/apache/beam/it/jdbc/AbstractJDBCResourceManager.java b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/AbstractJDBCResourceManager.java new file mode 100644 index 0000000000000..b57185b70ebce --- /dev/null +++ b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/AbstractJDBCResourceManager.java @@ -0,0 +1,367 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.jdbc; + +import static org.apache.beam.it.jdbc.JDBCResourceManagerUtils.checkValidTableName; +import static org.apache.beam.it.jdbc.JDBCResourceManagerUtils.generateDatabaseName; +import static org.apache.beam.it.jdbc.JDBCResourceManagerUtils.generateJdbcPassword; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.testcontainers.TestContainerResourceManager; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang3.math.NumberUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.JdbcDatabaseContainer; + +/** + * Abstract class for implementation of {@link JDBCResourceManager} interface. + * + *

    The class supports one database, and multiple tables per database object. A database is + * created when the container first spins up, if one is not given. + * + *

    The database name is formed using testId. The database name will be "{testId}-{ISO8601 time, + * microsecond precision}", with additional formatting. + * + *

    The class is thread-safe. + */ +public abstract class AbstractJDBCResourceManager> + extends TestContainerResourceManager> implements JDBCResourceManager { + + private static final Logger LOG = LoggerFactory.getLogger(AbstractJDBCResourceManager.class); + + protected static final String DEFAULT_JDBC_USERNAME = "root"; + + protected final JDBCDriverFactory driver; + protected final String databaseName; + protected final String username; + protected final String password; + + private final Map tableIds; + + @VisibleForTesting + AbstractJDBCResourceManager(T container, Builder builder, JDBCDriverFactory driver) { + super( + container + .withUsername(builder.username) + .withPassword(builder.password) + .withDatabaseName(builder.databaseName), + builder); + + this.databaseName = container.getDatabaseName(); + this.username = container.getUsername(); + this.password = container.getPassword(); + this.tableIds = new HashMap<>(); + this.driver = driver; + } + + protected AbstractJDBCResourceManager(T container, Builder builder) { + this(container, builder, new JDBCDriverFactory()); + } + + /** + * Return the default port that this JDBC implementation listens on. + * + * @return the JDBC port. + */ + protected abstract int getJDBCPort(); + + @Override + public String getUsername() { + return username; + } + + @Override + public String getPassword() { + return password; + } + + @Override + public synchronized String getUri() { + return String.format( + "jdbc:%s://%s:%d/%s", + getJDBCPrefix(), this.getHost(), this.getPort(getJDBCPort()), this.getDatabaseName()); + } + + public abstract String getJDBCPrefix(); + + @Override + public synchronized String getDatabaseName() { + return databaseName; + } + + @Override + public boolean createTable(String tableName, JDBCSchema schema) { + // Check table ID + checkValidTableName(tableName); + + // Check if table already exists + if (tableIds.containsKey(tableName)) { + throw new IllegalStateException( + "Table " + tableName + " already exists for database " + databaseName + "."); + } + + LOG.info("Creating table using tableName '{}'.", tableName); + + StringBuilder sql = new StringBuilder(); + try (Connection con = driver.getConnection(getUri(), username, password)) { + Statement stmt = con.createStatement(); + sql.append("CREATE TABLE ") + .append(tableName) + .append(" (") + .append(schema.toSqlStatement()) + .append(")"); + + stmt.executeUpdate(sql.toString()); + stmt.close(); + } catch (Exception e) { + throw new JDBCResourceManagerException( + "Error creating table with SQL statement: " + + sql + + " (for connection with URL " + + getUri() + + ")", + e); + } + + tableIds.put(tableName, schema.getIdColumn()); + LOG.info("Successfully created table {}.{}", databaseName, tableName); + + return true; + } + + /** + * Writes the given mapped rows into the specified columns. This method requires {@link + * JDBCResourceManager#createTable(String, JDBCSchema)} to be called for the target table + * beforehand. + * + *

    The rows map must use the row id as the key, and the values will be inserted into the + * columns at the row with that id. i.e. {0: [val1, val2, ...], 1: [val1, val2, ...], ...} + * + * @param tableName The name of the table to insert the given rows into. + * @param rows A map representing the rows to be inserted into the table. + * @throws JDBCResourceManagerException if method is called after resources have been cleaned up, + * if the manager object has no dataset, if the table does not exist or if there is an + * Exception when attempting to insert the rows. + */ + @Override + @SuppressWarnings("nullness") + public boolean write(String tableName, List> rows) + throws JDBCResourceManagerException { + if (rows.size() == 0) { + return false; + } + + LOG.info("Attempting to write {} rows to {}.{}.", rows.size(), databaseName, tableName); + + try (Connection con = driver.getConnection(getUri(), username, password)) { + Statement stmt = con.createStatement(); + + for (Map row : rows) { + List columns = new ArrayList<>(row.keySet()); + StringBuilder sql = + new StringBuilder("INSERT INTO ") + .append(tableName) + .append("(") + .append(String.join(",", columns)) + .append(") VALUES ("); + + List valueList = new ArrayList<>(); + for (String colName : columns) { + Object value = row.get(colName); + if (value == null) { + valueList.add(null); + } else if (NumberUtils.isCreatable(value.toString()) + || "true".equalsIgnoreCase(value.toString()) + || "false".equalsIgnoreCase(value.toString())) { + valueList.add(String.valueOf(value)); + } else { + valueList.add("'" + value + "'"); + } + } + sql.append(String.join(",", valueList)).append(")"); + + try { + LOG.info("Running SQL statement: " + sql); + stmt.executeUpdate(sql.toString()); + } catch (SQLException e) { + throw new JDBCResourceManagerException( + "Failed to insert values into table with SQL statement: " + sql, e); + } + } + stmt.close(); + } catch (SQLException e) { + throw new JDBCResourceManagerException( + String.format("Exception occurred when trying to write records to %s.", tableName), e); + } + + LOG.info("Successfully wrote {} rows to {}.{}.", rows.size(), databaseName, tableName); + return true; + } + + @Override + @SuppressWarnings("nullness") + public List> readTable(String tableName) { + LOG.info("Reading all rows from {}.{}", databaseName, tableName); + + List> resultSet = new ArrayList<>(); + + StringBuilder sql = new StringBuilder(); + try (Connection con = driver.getConnection(getUri(), username, password)) { + Statement stmt = con.createStatement(); + + sql.append("SELECT * FROM ").append(tableName); + ResultSet result = stmt.executeQuery(sql.toString()); + + while (result.next()) { + Map row = new HashMap<>(); + ResultSetMetaData metadata = result.getMetaData(); + // Columns list in table metadata is 1-indexed + for (int i = 1; i <= metadata.getColumnCount(); i++) { + row.put(metadata.getColumnName(i), result.getObject(i)); + } + resultSet.add(row); + } + result.close(); + stmt.close(); + } catch (Exception e) { + throw new JDBCResourceManagerException( + "Failed to fetch rows from table. SQL statement: " + sql, e); + } + + LOG.info("Successfully loaded rows from {}.{}", databaseName, tableName); + return resultSet; + } + + @Override + public synchronized List getTableSchema(String tableName) { + String sql = ""; + try (Connection con = driver.getConnection(getUri(), username, password)) { + Statement stmt = con.createStatement(); + sql = getFirstRow(tableName); + ResultSet result = stmt.executeQuery(sql); + + ResultSetMetaData metadata = result.getMetaData(); + List columnNames = new ArrayList<>(); + // Columns list in table metadata is 1-indexed + for (int i = 1; i <= metadata.getColumnCount(); i++) { + columnNames.add(metadata.getColumnName(i)); + } + result.close(); + stmt.close(); + return columnNames; + } catch (Exception e) { + throw new JDBCResourceManagerException( + "Failed to fetch table schema. SQL statement: " + sql, e); + } + } + + /** + * Retrieves the first row from the table. + * + * @param tableName the name of the table to query. + * @return the first row from the table. + */ + protected String getFirstRow(String tableName) { + return "SELECT * FROM " + tableName + " LIMIT 1"; + } + + @Override + public synchronized ResultSet runSQLQuery(String sql) { + try (Statement stmt = driver.getConnection(getUri(), username, password).createStatement()) { + return stmt.executeQuery(sql); + } catch (Exception e) { + throw new JDBCResourceManagerException("Failed to execute SQL statement: " + sql, e); + } + } + + @Override + public synchronized void runSQLUpdate(String sql) { + try (Statement stmt = driver.getConnection(getUri(), username, password).createStatement()) { + stmt.executeUpdate(sql); + } catch (Exception e) { + throw new JDBCResourceManagerException("Failed to execute SQL statement: " + sql, e); + } + } + + /** + * Builder for {@link AbstractJDBCResourceManager}. + * + * @param A class that extends {@link JdbcDatabaseContainer} for specific JDBC + * implementations. + */ + public abstract static class Builder> + extends TestContainerResourceManager.Builder> { + + protected String databaseName; + protected String username; + protected String password; + + public Builder(String testId, String containerImageName, String containerImageTag) { + super(testId, containerImageName, containerImageTag); + + this.username = DEFAULT_JDBC_USERNAME; + this.password = generateJdbcPassword(); + this.databaseName = generateDatabaseName(testId); + } + + /** + * Sets the database name to that of a static database instance. Use this method only when + * attempting to operate on a pre-existing JDBC database. + * + * @param databaseName The database name. + * @return this builder object with the database name set. + */ + public Builder setDatabaseName(String databaseName) { + this.databaseName = databaseName; + return this; + } + + /** + * Manually set the JDBC database username to the given username. This username will be used by + * the resource manager to authenticate with the JDBC database. + * + * @param username the username for the JDBC database. + * @return this builder with the username manually set. + */ + public Builder setUsername(String username) { + this.username = username; + return this; + } + + /** + * Manually set the JDBC database password to the given password. This password will be used by + * the resource manager to authenticate with the JDBC database. + * + * @param password the password for the JDBC database. + * @return this builder with the password manually set. + */ + public Builder setPassword(String password) { + this.password = password; + return this; + } + } +} diff --git a/it/jdbc/src/main/java/org/apache/beam/it/jdbc/JDBCDriverFactory.java b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/JDBCDriverFactory.java new file mode 100644 index 0000000000000..f546d29bf12f5 --- /dev/null +++ b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/JDBCDriverFactory.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.jdbc; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +/** JDBC Driver Factory class. */ +class JDBCDriverFactory { + + JDBCDriverFactory() {} + + /** + * Returns a Connection session to the given database URI. + * + * @param uri the JDBC connection string to connect to. + * @param username the username used to log in to the database. + * @param password the password used to log in to the database. + * @return the Connection session. + * @throws SQLException if there is an error creating a connection. + */ + Connection getConnection(String uri, String username, String password) throws SQLException { + return DriverManager.getConnection(uri, username, password); + } +} diff --git a/it/jdbc/src/main/java/org/apache/beam/it/jdbc/JDBCResourceManager.java b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/JDBCResourceManager.java new file mode 100644 index 0000000000000..9292d4cb42ece --- /dev/null +++ b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/JDBCResourceManager.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.jdbc; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import java.sql.ResultSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.ResourceManager; + +/** Interface for managing JDBC resources in integration tests. */ +public interface JDBCResourceManager extends ResourceManager { + + /** Returns the URI connection string to the JDBC Database. */ + String getUri(); + + /** + * Returns the username used to log in to the JDBC database. + * + * @return the database username. + */ + String getUsername(); + + /** + * Returns the password used to log in to the JDBC database. + * + * @return the database password. + */ + String getPassword(); + + /** + * Returns the name of the Database that this JDBC manager will operate in. + * + * @return the name of the JDBC Database. + */ + String getDatabaseName(); + + /** + * Creates a table within the current database given a table name and JDBC schema. + * + * @param tableName The name of the table. + * @param schema A {@link JDBCSchema} object that defines the table. + * @return A boolean indicating whether the resource was created. + * @throws JDBCResourceManagerException if there is an error creating the table or if the table + * already exists. + */ + boolean createTable(String tableName, JDBCSchema schema) throws JDBCResourceManagerException; + + /** + * Writes the given mapped rows into the specified columns. This method requires {@link + * JDBCResourceManager#createTable(String, JDBCSchema)} to be called for the target table + * beforehand. + * + *

    The maps in the rows list must use the column name as the key. i.e. [{col1: val1, col2: + * val2, ...}, {col1: val3, col2: val4, ...}, ...] + * + * @param tableName The name of the table to insert the given rows into. + * @param rows A list of maps representing the rows to be inserted into the table. + * @throws JDBCResourceManagerException if method is called after resources have been cleaned up, + * if the manager object has no dataset, if the table does not exist or if there is an + * Exception when attempting to insert the rows. + */ + boolean write(String tableName, List> rows) + throws JDBCResourceManagerException; + + /** + * Reads all the rows in a table and returns in the format of a list of Maps, which contain all + * the columns (including ID). + * + * @param tableName The name of the table to read rows from. + * @return a list containing the table rows. + */ + List> readTable(String tableName); + + /** + * Returns the schema of the given table as a list of strings. + * + * @param tableName the name of the table to fetch the schema of. + * @return the list of column names. + */ + List getTableSchema(String tableName); + + /** + * Run the given SQL query. + * + * @param sql The SQL query to run. + * @return A ResultSet containing the result of the execution. + */ + ResultSet runSQLQuery(String sql); + + /** + * Run the given SQL DML statement (INSERT, UPDATE and DELETE). + * + * @param sql The SQL DML statement to run. + */ + void runSQLUpdate(String sql); + + /** Object for managing JDBC table schemas in {@link JDBCResourceManager} instances. */ + class JDBCSchema { + + private final Map columns; + private final String idColumn; + + /** + * Creates a {@link JDBCSchema} object using the map given and assigns the unique id column to + * the given idColumn. + * + *

    The columns map should map column name to SQL type. For example, {{"example": + * "VARCHAR(200)}, {"example2": "INTEGER"}, {"example3": "BOOLEAN"}} + * + * @param columns a map containing the schema columns. + * @param idColumn the unique id column. + */ + public JDBCSchema(Map columns, String idColumn) { + checkArgument( + columns.get(idColumn) != null, + String.format("%s must be one of the columns passed in the columns map.", idColumn)); + this.columns = columns; + this.idColumn = idColumn; + } + + /** + * Returns the name of the column used as the unique ID column. + * + * @return the id column. + */ + public String getIdColumn() { + return idColumn; + } + + /** + * Return this schema object as a SQL statement. + * + * @return this schema object as a SQL statement. + */ + @SuppressWarnings("nullness") + public String toSqlStatement() { + StringBuilder sql = new StringBuilder(idColumn + " " + columns.get(idColumn)); + if (!columns.get(idColumn).toUpperCase().contains(" NOT NULL")) { + sql.append(" NOT NULL"); + } + for (Map.Entry entry : columns.entrySet()) { + if (entry.getKey().contains(idColumn)) { + continue; + } + sql.append(", "); + sql.append(entry.getKey()).append(" ").append(entry.getValue().toUpperCase()); + } + sql.append(", PRIMARY KEY ( ").append(idColumn).append(" )"); + return sql.toString(); + } + } +} diff --git a/it/jdbc/src/main/java/org/apache/beam/it/jdbc/JDBCResourceManagerException.java b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/JDBCResourceManagerException.java new file mode 100644 index 0000000000000..5e3ae28b59eaf --- /dev/null +++ b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/JDBCResourceManagerException.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.jdbc; + +/** Custom exception for {@link JDBCResourceManager} implementations. */ +public class JDBCResourceManagerException extends RuntimeException { + public JDBCResourceManagerException(String errorMessage, Throwable err) { + super(errorMessage, err); + } + + public JDBCResourceManagerException(String errorMessage) { + super(errorMessage); + } +} diff --git a/it/jdbc/src/main/java/org/apache/beam/it/jdbc/JDBCResourceManagerUtils.java b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/JDBCResourceManagerUtils.java new file mode 100644 index 0000000000000..3d09bafeba639 --- /dev/null +++ b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/JDBCResourceManagerUtils.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.jdbc; + +import static org.apache.beam.it.common.utils.ResourceManagerUtils.generatePassword; +import static org.apache.beam.it.common.utils.ResourceManagerUtils.generateResourceId; + +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.regex.Pattern; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Chars; + +/** Utilities for {@link JDBCResourceManager} implementations. */ +final class JDBCResourceManagerUtils { + + // Naming restrictions can be found at: + // mySQL - + // https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/acreldb/n0rfg6x1shw0ppn1cwhco6yn09f7.htm + // postgreSQL - + // https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/acreldb/p1iw263fz6wvnbn1d6nyw71a9sf2.htm + // oracle - + // https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/acreldb/p0t80fm3l1okawn1x3mvo098svir.htm + // + // The tightest restrictions were used across all flavors of JDBC for simplicity. + private static final int MAX_DATABASE_NAME_LENGTH = 30; + private static final Pattern ILLEGAL_DATABASE_NAME_CHARS = Pattern.compile("[^a-zA-Z0-9_$#]"); + private static final String REPLACE_DATABASE_NAME_CHAR = "_"; + private static final int MIN_PASSWORD_LENGTH = 9; + private static final int MAX_PASSWORD_LENGTH = 25; + private static final int MIN_TABLE_ID_LENGTH = 1; + private static final int MAX_TABLE_ID_LENGTH = 30; + private static final Pattern ILLEGAL_TABLE_CHARS = Pattern.compile("[/.]"); // i.e. [/.] + private static final DateTimeFormatter TIME_FORMAT = + DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss_SSSSSS"); + + public static final List ALLOWED_SPECIAL_CHARS = + Chars.asList("~!@#$%^&*()_-+={}[]/<>,.?:|".toCharArray()); + + private JDBCResourceManagerUtils() {} + + /** + * Generates a JDBC database name from a given string. + * + * @param baseString The string to generate the name from. + * @return The database name string. + */ + static String generateDatabaseName(String baseString) { + baseString = Character.isLetter(baseString.charAt(0)) ? baseString : "d_" + baseString; + return generateResourceId( + baseString, + ILLEGAL_DATABASE_NAME_CHARS, + REPLACE_DATABASE_NAME_CHAR, + MAX_DATABASE_NAME_LENGTH, + TIME_FORMAT); + } + + /** + * Generates a secure, valid JDBC password. + * + * @return The generated password. + */ + static String generateJdbcPassword() { + int numLower = 2; + int numUpper = 2; + int numSpecial = 2; + return generatePassword( + MIN_PASSWORD_LENGTH, + MAX_PASSWORD_LENGTH, + numLower, + numUpper, + numSpecial, + ALLOWED_SPECIAL_CHARS); + } + + /** + * Checks whether the given table name is valid according to JDBC constraints. + * + * @param nameToCheck the table name to check. + * @throws IllegalArgumentException if the table name is invalid. + */ + static void checkValidTableName(String nameToCheck) { + if (nameToCheck.length() < MIN_TABLE_ID_LENGTH) { + throw new IllegalArgumentException("Table name cannot be empty. "); + } + if (nameToCheck.length() > MAX_TABLE_ID_LENGTH) { + throw new IllegalArgumentException( + "Table name " + + nameToCheck + + " cannot be longer than " + + MAX_TABLE_ID_LENGTH + + " characters."); + } + if (ILLEGAL_TABLE_CHARS.matcher(nameToCheck).find()) { + throw new IllegalArgumentException( + "Table name " + + nameToCheck + + " is not a valid name. Periods and forward slashes are not allowed."); + } + } +} diff --git a/it/jdbc/src/main/java/org/apache/beam/it/jdbc/MSSQLResourceManager.java b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/MSSQLResourceManager.java new file mode 100644 index 0000000000000..c515b2c4844f7 --- /dev/null +++ b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/MSSQLResourceManager.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.jdbc; + +import java.sql.Connection; +import java.sql.Statement; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.MSSQLServerContainer; +import org.testcontainers.utility.DockerImageName; + +/** + * Default class for the MS SQL implementation of {@link AbstractJDBCResourceManager} abstract + * class. + * + *

    The class supports one database, and multiple tables per database object. A database is * + * created when the container first spins up, if one is not given. + * + *

    The class is thread-safe. + */ +public class MSSQLResourceManager + extends AbstractJDBCResourceManager> { + + private static final Logger LOG = LoggerFactory.getLogger(MSSQLResourceManager.class); + + private static final String DEFAULT_MSSQL_CONTAINER_NAME = "mcr.microsoft.com/azure-sql-edge"; + + // A list of available MSSQL Docker image tags can be found at + // https://hub.docker.com/_/microsoft-azure-sql-edge + private static final String DEFAULT_MSSQL_CONTAINER_TAG = "1.0.6"; + + private boolean initialized; + + @SuppressWarnings("nullness") + private MSSQLResourceManager(Builder builder) { + super( + new DefaultMSSQLServerContainer<>( + DockerImageName.parse(builder.containerImageName) + .withTag(builder.containerImageTag) + .asCompatibleSubstituteFor("mcr.microsoft.com/mssql/server"), + builder.databaseName, + builder.useStaticContainer), + builder); + createDatabase(builder.databaseName); + } + + @VisibleForTesting + > MSSQLResourceManager( + T container, Builder builder) { + super(container, builder); + initialized = true; + } + + public static MSSQLResourceManager.Builder builder(String testId) { + return new MSSQLResourceManager.Builder(testId); + } + + private synchronized void createDatabase(String databaseName) { + LOG.info("Creating database using databaseName '{}'.", databaseName); + + StringBuilder sql = new StringBuilder(); + try (Connection con = driver.getConnection(getUri(), username, password)) { + Statement stmt = con.createStatement(); + sql.append("CREATE DATABASE ").append(databaseName); + stmt.executeUpdate(sql.toString()); + stmt.close(); + } catch (Exception e) { + throw new JDBCResourceManagerException( + "Error creating database with SQL statement: " + sql, e); + } + + initialized = true; + LOG.info("Successfully created database {}.{}", databaseName, databaseName); + } + + @Override + public synchronized String getUri() { + return String.format( + "jdbc:%s://%s:%d%s%s", + getJDBCPrefix(), + this.getHost(), + this.getPort(getJDBCPort()), + initialized ? ";DatabaseName=" + databaseName : "", + ";encrypt=true;trustServerCertificate=true;"); + } + + @Override + public String getJDBCPrefix() { + return "sqlserver"; + } + + @Override + protected int getJDBCPort() { + return DefaultMSSQLServerContainer.MS_SQL_SERVER_PORT; + } + + @Override + protected String getFirstRow(String tableName) { + return "SELECT TOP 1 * FROM " + tableName; + } + + /** Builder for {@link MSSQLResourceManager}. */ + public static final class Builder + extends AbstractJDBCResourceManager.Builder> { + + public Builder(String testId) { + super(testId, DEFAULT_MSSQL_CONTAINER_NAME, DEFAULT_MSSQL_CONTAINER_TAG); + } + + @Override + public MSSQLResourceManager build() { + return new MSSQLResourceManager(this); + } + } + + static final class DefaultMSSQLServerContainer> + extends MSSQLServerContainer { + + private final String databaseName; + private final boolean usingStaticDatabase; + + @SuppressWarnings("nullness") + DefaultMSSQLServerContainer( + DockerImageName dockerImageName, String databaseName, boolean usingStaticDatabase) { + super(dockerImageName); + this.addEnv("ACCEPT_EULA", "Y"); + this.databaseName = databaseName; + this.usingStaticDatabase = usingStaticDatabase; + } + + @Override + public T withUsername(String username) { + if (!username.equals(AbstractJDBCResourceManager.DEFAULT_JDBC_USERNAME) + && !usingStaticDatabase) { + throw new UnsupportedOperationException("The username cannot be overridden to " + username); + } + return this.self(); + } + + @Override + public T withDatabaseName(String dbName) { + // AbstractJDBCResourceManager constructor calls this method, but MSSQLResourceManager + // does + // database creation after class initialization, so just pass container instance back as-is. + return this.self(); + } + + @Override + public String getDatabaseName() { + return databaseName; + } + + @Override + @SuppressWarnings("nullness") + public boolean equals(Object o) { + return super.equals(o); + } + + @Override + public int hashCode() { + return super.hashCode(); + } + } +} diff --git a/it/jdbc/src/main/java/org/apache/beam/it/jdbc/MySQLResourceManager.java b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/MySQLResourceManager.java new file mode 100644 index 0000000000000..688c26dfb56da --- /dev/null +++ b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/MySQLResourceManager.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.jdbc; + +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.utility.DockerImageName; + +/** + * Default class for the MySQL implementation of {@link AbstractJDBCResourceManager} abstract class. + * + *

    The class supports one database, and multiple tables per database object. A database is * + * created when the container first spins up, if one is not given. + * + *

    The class is thread-safe. + */ +public class MySQLResourceManager extends AbstractJDBCResourceManager> { + + private static final String DEFAULT_MYSQL_CONTAINER_NAME = "mysql"; + + // A list of available mySQL Docker image tags can be found at + // https://hub.docker.com/_/mysql/tags?tab=tags + private static final String DEFAULT_MYSQL_CONTAINER_TAG = "8.0.30"; + + private MySQLResourceManager(Builder builder) { + this( + new MySQLContainer<>( + DockerImageName.parse(builder.containerImageName).withTag(builder.containerImageTag)), + builder); + } + + @VisibleForTesting + MySQLResourceManager(MySQLContainer container, Builder builder) { + super(container, builder); + } + + public static MySQLResourceManager.Builder builder(String testId) { + return new MySQLResourceManager.Builder(testId); + } + + @Override + protected int getJDBCPort() { + return MySQLContainer.MYSQL_PORT; + } + + @Override + public String getJDBCPrefix() { + return "mysql"; + } + + /** Builder for {@link MySQLResourceManager}. */ + public static final class Builder extends AbstractJDBCResourceManager.Builder> { + + public Builder(String testId) { + super(testId, DEFAULT_MYSQL_CONTAINER_NAME, DEFAULT_MYSQL_CONTAINER_TAG); + } + + @Override + public MySQLResourceManager build() { + return new MySQLResourceManager(this); + } + } +} diff --git a/it/jdbc/src/main/java/org/apache/beam/it/jdbc/OracleResourceManager.java b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/OracleResourceManager.java new file mode 100644 index 0000000000000..8054d26c33f70 --- /dev/null +++ b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/OracleResourceManager.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.jdbc; + +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.testcontainers.containers.OracleContainer; +import org.testcontainers.utility.DockerImageName; + +/** + * Default class for the Oracle implementation of {@link AbstractJDBCResourceManager} abstract + * class. + * + *

    The class supports one database, and multiple tables per database object. A database is * + * created when the container first spins up, if one is not given. + * + *

    The class is thread-safe. + */ +public class OracleResourceManager extends AbstractJDBCResourceManager { + + private static final String DEFAULT_ORACLE_CONTAINER_NAME = "gvenzl/oracle-xe"; + + // A list of available oracle-xe Docker image tags can be found at + // https://hub.docker.com/r/gvenzl/oracle-xe/tags?tab=tags + private static final String DEFAULT_ORACLE_CONTAINER_TAG = "21-slim-faststart"; + private static final int DEFAULT_ORACLE_INTERNAL_PORT = 1521; + + // TODO - oracle-xe seems to require these credentials to spin up the container. + // Adding a more secure user after startup, or finding an image without these + // restrictions may be required. + private static final String DEFAULT_ORACLE_USERNAME = "testUser"; + private static final String DEFAULT_ORACLE_PASSWORD = "testPassword"; + + private OracleResourceManager(OracleResourceManager.Builder builder) { + this( + new OracleContainer( + DockerImageName.parse(builder.containerImageName).withTag(builder.containerImageTag)), + builder); + } + + @VisibleForTesting + OracleResourceManager(OracleContainer container, OracleResourceManager.Builder builder) { + super(container, builder); + } + + public static OracleResourceManager.Builder builder(String testId) { + return new OracleResourceManager.Builder(testId); + } + + @Override + protected int getJDBCPort() { + return DEFAULT_ORACLE_INTERNAL_PORT; + } + + @Override + public String getJDBCPrefix() { + return "oracle"; + } + + @Override + public synchronized String getUri() { + return String.format( + "jdbc:%s:thin:@%s:%d/%s", + getJDBCPrefix(), this.getHost(), this.getPort(getJDBCPort()), this.getDatabaseName()); + } + + @Override + protected String getFirstRow(String tableName) { + return "SELECT * FROM " + tableName + " WHERE ROWNUM <= 1"; + } + + /** Builder for {@link OracleResourceManager}. */ + public static final class Builder extends AbstractJDBCResourceManager.Builder { + + public Builder(String testId) { + super(testId, DEFAULT_ORACLE_CONTAINER_NAME, DEFAULT_ORACLE_CONTAINER_TAG); + this.username = DEFAULT_ORACLE_USERNAME; + this.password = DEFAULT_ORACLE_PASSWORD; + } + + @Override + public OracleResourceManager build() { + return new OracleResourceManager(this); + } + } +} diff --git a/it/jdbc/src/main/java/org/apache/beam/it/jdbc/PostgresResourceManager.java b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/PostgresResourceManager.java new file mode 100644 index 0000000000000..7f054dfbc5dbe --- /dev/null +++ b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/PostgresResourceManager.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.jdbc; + +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.utility.DockerImageName; + +/** + * Default class for the Postgres implementation of {@link AbstractJDBCResourceManager} abstract + * class. + * + *

    The class supports one database, and multiple tables per database object. A database is * + * created when the container first spins up, if one is not given. + * + *

    The class is thread-safe. + */ +public class PostgresResourceManager extends AbstractJDBCResourceManager> { + + private static final String DEFAULT_POSTGRES_CONTAINER_NAME = "postgres"; + + // A list of available PostgreSQL Docker image tags can be found at + // https://hub.docker.com/_/postgres/tags?tab=tags + private static final String DEFAULT_POSTGRES_CONTAINER_TAG = "15.1"; + + private PostgresResourceManager(PostgresResourceManager.Builder builder) { + this( + new PostgreSQLContainer<>( + DockerImageName.parse(builder.containerImageName).withTag(builder.containerImageTag)), + builder); + } + + @VisibleForTesting + PostgresResourceManager( + PostgreSQLContainer container, PostgresResourceManager.Builder builder) { + super(container, builder); + } + + public static PostgresResourceManager.Builder builder(String testId) { + return new PostgresResourceManager.Builder(testId); + } + + @Override + protected int getJDBCPort() { + return PostgreSQLContainer.POSTGRESQL_PORT; + } + + @Override + public String getJDBCPrefix() { + return "postgresql"; + } + + @Override + public synchronized String getDockerImageName() { + return "postgresql"; + } + + /** Builder for {@link PostgresResourceManager}. */ + public static final class Builder + extends AbstractJDBCResourceManager.Builder> { + + public Builder(String testId) { + super(testId, DEFAULT_POSTGRES_CONTAINER_NAME, DEFAULT_POSTGRES_CONTAINER_TAG); + } + + @Override + public PostgresResourceManager build() { + return new PostgresResourceManager(this); + } + } +} diff --git a/it/jdbc/src/main/java/org/apache/beam/it/jdbc/package-info.java b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/package-info.java new file mode 100644 index 0000000000000..8ea41af5309d7 --- /dev/null +++ b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing JDBC resources within integration tests. */ +package org.apache.beam.it.jdbc; diff --git a/it/jdbc/src/test/java/org/apache/beam/it/jdbc/AbstractJDBCResourceManagerIT.java b/it/jdbc/src/test/java/org/apache/beam/it/jdbc/AbstractJDBCResourceManagerIT.java new file mode 100644 index 0000000000000..f61c82ba138fc --- /dev/null +++ b/it/jdbc/src/test/java/org/apache/beam/it/jdbc/AbstractJDBCResourceManagerIT.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.jdbc; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatRecords; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.beam.it.testcontainers.TestContainersIntegrationTest; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Integration tests for JDBC Resource Managers. */ +@Category(TestContainersIntegrationTest.class) +@RunWith(JUnit4.class) +public class AbstractJDBCResourceManagerIT { + + private static final Logger LOG = LoggerFactory.getLogger(AbstractJDBCResourceManagerIT.class); + + private static final String TEST_ID = "dummy-test"; + private static final String TABLE_NAME = "dummy_table"; + + @Test + public void testDefaultMySQLResourceManagerE2E() { + MySQLResourceManager mySQL = MySQLResourceManager.builder(TEST_ID).build(); + + simpleTest(mySQL); + } + + @Test + public void testDefaultPostgresResourceManagerE2E() { + PostgresResourceManager postgres = PostgresResourceManager.builder(TEST_ID).build(); + + simpleTest(postgres); + } + + @Test + public void testDefaultOracleResourceManagerE2E() { + // Oracle image does not work on M1 + if (System.getProperty("testOnM1") != null) { + LOG.info("M1 is being used, Oracle tests are not being executed."); + return; + } + + OracleResourceManager oracle = OracleResourceManager.builder(TEST_ID).build(); + simpleTest(oracle); + } + + @Test + public void testDefaultMSSQLResourceManagerE2E() { + MSSQLResourceManager mssqlBuilder = MSSQLResourceManager.builder(TEST_ID).build(); + simpleTest(mssqlBuilder); + } + + private > void simpleTest(T rm) { + try { + Map columns = new LinkedHashMap<>(); + columns.put("id", "INTEGER"); + columns.put("first", "VARCHAR(32)"); + columns.put("last", "VARCHAR(32)"); + columns.put("age", "VARCHAR(32)"); + JDBCResourceManager.JDBCSchema schema = new JDBCResourceManager.JDBCSchema(columns, "id"); + rm.createTable(TABLE_NAME, schema); + + List> rows = new ArrayList<>(); + rows.add(ImmutableMap.of("id", 0, "first", "John", "last", "Doe", "age", 23)); + rows.add(ImmutableMap.of("id", 1, "first", "Jane", "last", "Doe", "age", 42)); + rows.add(ImmutableMap.of("id", 2, "first", "A", "last", "B", "age", 1)); + rm.write(TABLE_NAME, rows); + + List validateSchema = new ArrayList<>(columns.keySet()); + List> fetchRows = rm.readTable(TABLE_NAME); + + // toUpperCase expected because some databases (Postgres, Oracle) transform column names + assertThat(toUpperCase(rm.getTableSchema(TABLE_NAME))) + .containsExactlyElementsIn(toUpperCase(validateSchema)); + assertThat(fetchRows).hasSize(3); + assertThatRecords(fetchRows).hasRecordsUnorderedCaseInsensitiveColumns(rows); + } finally { + rm.cleanupAll(); + } + } + + private List toUpperCase(List list) { + return list.stream().map(String::toUpperCase).collect(Collectors.toList()); + } +} diff --git a/it/jdbc/src/test/java/org/apache/beam/it/jdbc/AbstractJDBCResourceManagerTest.java b/it/jdbc/src/test/java/org/apache/beam/it/jdbc/AbstractJDBCResourceManagerTest.java new file mode 100644 index 0000000000000..f1f8c5dd65aec --- /dev/null +++ b/it/jdbc/src/test/java/org/apache/beam/it/jdbc/AbstractJDBCResourceManagerTest.java @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.jdbc; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.SQLException; +import java.sql.Statement; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.testcontainers.containers.JdbcDatabaseContainer; + +/** Unit tests for {@link AbstractJDBCResourceManager}. */ +@RunWith(JUnit4.class) +public class AbstractJDBCResourceManagerTest> { + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private JDBCDriverFactory driver; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private T container; + + private static final String TEST_ID = "test_id"; + private static final String DATABASE_NAME = "database"; + private static final String TABLE_NAME = "test-table"; + private static final String HOST = "localhost"; + private static final int JDBC_PORT = 1234; + private static final int MAPPED_PORT = 4321; + private static final String JDBC_PREFIX = "mysql"; + + private AbstractJDBCResourceManager testManager; + + @Before + public void setUp() { + when(container.withUsername(anyString())).thenReturn(container); + when(container.withPassword(anyString())).thenReturn(container); + when(container.withDatabaseName(anyString())).thenReturn(container); + when(container.getDatabaseName()).thenReturn(DATABASE_NAME); + + testManager = + new AbstractJDBCResourceManager( + container, + new AbstractJDBCResourceManager.Builder(TEST_ID, "", "") { + @Override + public AbstractJDBCResourceManager build() { + return new AbstractJDBCResourceManager(container, this) { + @Override + protected int getJDBCPort() { + return JDBC_PORT; + } + + @Override + public String getJDBCPrefix() { + return JDBC_PREFIX; + } + }; + } + }, + driver) { + @Override + protected int getJDBCPort() { + return JDBC_PORT; + } + + @Override + public String getJDBCPrefix() { + return JDBC_PREFIX; + } + }; + } + + @Test + public void testGetUriShouldReturnCorrectValue() { + when(container.getHost()).thenReturn(HOST); + when(container.getMappedPort(JDBC_PORT)).thenReturn(MAPPED_PORT); + assertThat(testManager.getUri()) + .matches("jdbc:" + JDBC_PREFIX + "://" + HOST + ":" + MAPPED_PORT + "/" + DATABASE_NAME); + } + + @Test + public void testGetDatabaseNameShouldReturnCorrectValue() { + assertThat(testManager.getDatabaseName()).matches(DATABASE_NAME); + } + + @Test + public void testCreateTableShouldThrowErrorWhenTableNameIsInvalid() { + assertThrows( + IllegalArgumentException.class, + () -> + testManager.createTable( + "invalid/name", + new JDBCResourceManager.JDBCSchema(ImmutableMap.of("id", "INTEGER"), "id"))); + } + + @Test + public void testCreateTableShouldThrowErrorWhenTableAlreadyExists() throws SQLException { + when(container.getHost()).thenReturn(HOST); + when(container.getMappedPort(JDBC_PORT)).thenReturn(MAPPED_PORT); + + testManager.createTable( + TABLE_NAME, new JDBCResourceManager.JDBCSchema(ImmutableMap.of("id", "INTEGER"), "id")); + + assertThrows( + IllegalStateException.class, + () -> + testManager.createTable( + TABLE_NAME, + new JDBCResourceManager.JDBCSchema(ImmutableMap.of("id", "INTEGER"), "id"))); + } + + @Test + public void testCreateTableShouldThrowErrorWhenDriverFailsToEstablishConnection() + throws SQLException { + when(container.getHost()).thenReturn(HOST); + when(container.getMappedPort(JDBC_PORT)).thenReturn(MAPPED_PORT); + doThrow(SQLException.class).when(driver).getConnection(any(), any(), any()); + + assertThrows( + JDBCResourceManagerException.class, + () -> + testManager.createTable( + TABLE_NAME, + new JDBCResourceManager.JDBCSchema(ImmutableMap.of("id", "INTEGER"), "id"))); + } + + @Test + public void testCreateTableShouldThrowErrorWhenJDBCFailsToExecuteSQL() throws SQLException { + when(container.getHost()).thenReturn(HOST); + when(container.getMappedPort(JDBC_PORT)).thenReturn(MAPPED_PORT); + Statement statement = driver.getConnection(any(), any(), any()).createStatement(); + doThrow(SQLException.class).when(statement).executeUpdate(anyString()); + + assertThrows( + JDBCResourceManagerException.class, + () -> + testManager.createTable( + TABLE_NAME, + new JDBCResourceManager.JDBCSchema(ImmutableMap.of("id", "INTEGER"), "id"))); + } + + @Test + public void testCreateTableShouldReturnTrueIfJDBCDoesNotThrowAnyError() throws SQLException { + when(container.getHost()).thenReturn(HOST); + when(container.getMappedPort(JDBC_PORT)).thenReturn(MAPPED_PORT); + + assertTrue( + testManager.createTable( + TABLE_NAME, + new JDBCResourceManager.JDBCSchema(ImmutableMap.of("id", "INTEGER"), "id"))); + + verify(driver.getConnection(any(), any(), any()).createStatement()).executeUpdate(anyString()); + } + + @Test + public void testWriteShouldThrowErrorWhenDriverFailsToEstablishConnection() throws SQLException { + when(container.getHost()).thenReturn(HOST); + when(container.getMappedPort(JDBC_PORT)).thenReturn(MAPPED_PORT); + doThrow(SQLException.class).when(driver).getConnection(any(), any(), any()); + + assertThrows( + JDBCResourceManagerException.class, + () -> testManager.write(TABLE_NAME, ImmutableList.of(ImmutableMap.of("key", "test")))); + } + + @Test + public void testWriteShouldThrowErrorWhenJDBCFailsToExecuteSQL() throws SQLException { + when(container.getHost()).thenReturn(HOST); + when(container.getMappedPort(JDBC_PORT)).thenReturn(MAPPED_PORT); + Statement statement = driver.getConnection(any(), any(), any()).createStatement(); + doThrow(SQLException.class).when(statement).executeUpdate(anyString()); + + assertThrows( + JDBCResourceManagerException.class, + () -> testManager.write(TABLE_NAME, ImmutableList.of(ImmutableMap.of("key", "test")))); + } + + @Test + public void testWriteShouldReturnTrueIfJDBCDoesNotThrowAnyError() throws SQLException { + when(container.getHost()).thenReturn(HOST); + when(container.getMappedPort(JDBC_PORT)).thenReturn(MAPPED_PORT); + + assertTrue(testManager.write(TABLE_NAME, ImmutableList.of(ImmutableMap.of("key", "test")))); + + verify(driver.getConnection(any(), any(), any()).createStatement()).executeUpdate(anyString()); + } + + @Test + public void testReadTableShouldThrowErrorWhenDriverFailsToEstablishConnection() + throws SQLException { + when(container.getHost()).thenReturn(HOST); + when(container.getMappedPort(JDBC_PORT)).thenReturn(MAPPED_PORT); + doThrow(SQLException.class).when(driver).getConnection(any(), any(), any()); + + assertThrows(JDBCResourceManagerException.class, () -> testManager.readTable(TABLE_NAME)); + } + + @Test + public void testReadTableShouldThrowErrorWhenJDBCFailsToExecuteSQL() throws SQLException { + when(container.getHost()).thenReturn(HOST); + when(container.getMappedPort(JDBC_PORT)).thenReturn(MAPPED_PORT); + Statement statement = driver.getConnection(any(), any(), any()).createStatement(); + doThrow(SQLException.class).when(statement).executeQuery(anyString()); + + assertThrows(JDBCResourceManagerException.class, () -> testManager.readTable(TABLE_NAME)); + } + + @Test + public void testReadTableShouldNotThrowErrorIfJDBCDoesNotThrowAnyError() throws SQLException { + when(container.getHost()).thenReturn(HOST); + when(container.getMappedPort(JDBC_PORT)).thenReturn(MAPPED_PORT); + + testManager.readTable(TABLE_NAME); + + verify(driver.getConnection(any(), any(), any()).createStatement()).executeQuery(anyString()); + } + + @Test + public void testRunSQLStatementShouldThrowErrorWhenDriverFailsToEstablishConnection() + throws SQLException { + when(container.getHost()).thenReturn(HOST); + when(container.getMappedPort(JDBC_PORT)).thenReturn(MAPPED_PORT); + doThrow(SQLException.class).when(driver).getConnection(any(), any(), any()); + + assertThrows( + JDBCResourceManagerException.class, () -> testManager.runSQLQuery("SQL statement")); + } + + @Test + public void testRunSQLStatementShouldThrowErrorWhenJDBCFailsToExecuteSQL() throws SQLException { + when(container.getHost()).thenReturn(HOST); + when(container.getMappedPort(JDBC_PORT)).thenReturn(MAPPED_PORT); + Statement statement = driver.getConnection(any(), any(), any()).createStatement(); + doThrow(SQLException.class).when(statement).executeQuery(anyString()); + + assertThrows( + JDBCResourceManagerException.class, () -> testManager.runSQLQuery("SQL statement")); + } + + @Test + public void testRunSQLStatementShouldNotThrowErrorIfJDBCDoesNotThrowAnyError() + throws SQLException { + when(container.getHost()).thenReturn(HOST); + when(container.getMappedPort(JDBC_PORT)).thenReturn(MAPPED_PORT); + + testManager.runSQLQuery("SQL statement"); + + verify(driver.getConnection(any(), any(), any()).createStatement()).executeQuery(anyString()); + } + + @Test + public void testCleanupAllShouldReturnTrueIfJDBCDoesNotThrowAnyError() { + testManager.cleanupAll(); + verify(container).close(); + } +} diff --git a/it/jdbc/src/test/java/org/apache/beam/it/jdbc/JDBCResourceManagerUtilsTest.java b/it/jdbc/src/test/java/org/apache/beam/it/jdbc/JDBCResourceManagerUtilsTest.java new file mode 100644 index 0000000000000..b0e686acb5c68 --- /dev/null +++ b/it/jdbc/src/test/java/org/apache/beam/it/jdbc/JDBCResourceManagerUtilsTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.jdbc; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.jdbc.JDBCResourceManagerUtils.ALLOWED_SPECIAL_CHARS; +import static org.apache.beam.it.jdbc.JDBCResourceManagerUtils.checkValidTableName; +import static org.apache.beam.it.jdbc.JDBCResourceManagerUtils.generateDatabaseName; +import static org.apache.beam.it.jdbc.JDBCResourceManagerUtils.generateJdbcPassword; +import static org.junit.Assert.assertThrows; + +import org.apache.commons.lang3.StringUtils; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link JDBCResourceManagerUtils}. */ +@RunWith(JUnit4.class) +public class JDBCResourceManagerUtilsTest { + + @Test + public void testGenerateDatabaseNameShouldReplaceHyphen() { + String testBaseString = "test-id"; + String actual = generateDatabaseName(testBaseString); + assertThat(actual).matches("test_id_\\d{8}_\\d{6}_\\d{6}"); + } + + @Test + public void testGenerateDatabaseNameShouldReplaceIllegalCharacters() { + String testBaseString = "!@#_()"; + String actual = generateDatabaseName(testBaseString); + assertThat(actual).matches("d___#___\\d{8}_\\d{6}_\\d{6}"); + } + + @Test + public void testGeneratePasswordMeetsRequirements() { + for (int i = 0; i < 10000; i++) { + String password = generateJdbcPassword(); + int lower = 0; + int upper = 0; + int special = 0; + + for (int j = 0; j < password.length(); j++) { + char c = password.charAt(j); + String s = String.valueOf(c); + lower += s.toLowerCase().equals(s) ? 1 : 0; + upper += s.toUpperCase().equals(s) ? 1 : 0; + special += ALLOWED_SPECIAL_CHARS.contains(c) ? 1 : 0; + } + + assertThat(lower).isAtLeast(2); + assertThat(upper).isAtLeast(2); + assertThat(special).isAtLeast(2); + } + } + + @Test + public void testCheckValidTableNameThrowsErrorWhenNameIsTooShort() { + assertThrows(IllegalArgumentException.class, () -> checkValidTableName("")); + } + + @Test + public void testCheckValidTableNameThrowsErrorWhenNameIsTooLong() { + assertThrows( + IllegalArgumentException.class, () -> checkValidTableName(StringUtils.repeat("a", 31))); + } + + @Test + public void testCheckValidTableNameThrowsErrorWhenContainsBackslash() { + assertThrows(IllegalArgumentException.class, () -> checkValidTableName("table/name")); + } + + @Test + public void testCheckValidTableNameThrowsErrorWhenContainsPeriod() { + assertThrows(IllegalArgumentException.class, () -> checkValidTableName("table.name")); + } + + @Test + public void testCheckValidTableNameDoesNotThrowErrorWhenNameIsValid() { + checkValidTableName("A-l3gal_t4ble NAME!"); + } +} diff --git a/it/jdbc/src/test/java/org/apache/beam/it/jdbc/MSSQLResourceManagerTest.java b/it/jdbc/src/test/java/org/apache/beam/it/jdbc/MSSQLResourceManagerTest.java new file mode 100644 index 0000000000000..0ed192bf3c4a4 --- /dev/null +++ b/it/jdbc/src/test/java/org/apache/beam/it/jdbc/MSSQLResourceManagerTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.jdbc; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.when; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Integration tests for {@link MSSQLResourceManagerTest}. */ +@RunWith(JUnit4.class) +public class MSSQLResourceManagerTest< + T extends MSSQLResourceManager.DefaultMSSQLServerContainer> { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock private T container; + + private MSSQLResourceManager testManager; + + private static final String TEST_ID = "test_id"; + private static final String DATABASE_NAME = "database"; + private static final String HOST = "localhost"; + private static final int MAPPED_PORT = 1234; + + @Before + public void setUp() { + when(container.withUsername(any())).thenReturn(container); + when(container.withPassword(any())).thenReturn(container); + when(container.withDatabaseName(anyString())).thenReturn(container); + when(container.getDatabaseName()).thenReturn(DATABASE_NAME); + doReturn(container).when(container).withLogConsumer(any()); + testManager = new MSSQLResourceManager(container, new MSSQLResourceManager.Builder(TEST_ID)); + } + + @Test + public void testGetJDBCPortReturnsCorrectValue() { + assertThat(testManager.getJDBCPort()) + .isEqualTo(MSSQLResourceManager.DefaultMSSQLServerContainer.MS_SQL_SERVER_PORT); + } + + @Test + public void testGetUriShouldReturnCorrectValue() { + when(container.getHost()).thenReturn(HOST); + when(container.getMappedPort( + MSSQLResourceManager.DefaultMSSQLServerContainer.MS_SQL_SERVER_PORT)) + .thenReturn(MAPPED_PORT); + assertThat(testManager.getUri()) + .matches( + "jdbc:sqlserver://" + + HOST + + ":" + + MAPPED_PORT + + ";DatabaseName=" + + DATABASE_NAME + + ";encrypt=true;trustServerCertificate=true;"); + } +} diff --git a/it/jdbc/src/test/java/org/apache/beam/it/jdbc/MySQLResourceManagerTest.java b/it/jdbc/src/test/java/org/apache/beam/it/jdbc/MySQLResourceManagerTest.java new file mode 100644 index 0000000000000..402c45ec8cea4 --- /dev/null +++ b/it/jdbc/src/test/java/org/apache/beam/it/jdbc/MySQLResourceManagerTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.jdbc; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.when; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.testcontainers.containers.MySQLContainer; + +/** Integration tests for {@link MySQLResourceManager}. */ +@RunWith(JUnit4.class) +public class MySQLResourceManagerTest> { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock private T container; + + private MySQLResourceManager testManager; + + private static final String TEST_ID = "test_id"; + + @Before + public void setUp() { + when(container.withUsername(any())).thenReturn(container); + when(container.withPassword(any())).thenReturn(container); + when(container.withDatabaseName(anyString())).thenReturn(container); + doReturn(container).when(container).withLogConsumer(any()); + testManager = new MySQLResourceManager(container, new MySQLResourceManager.Builder(TEST_ID)); + } + + @Test + public void testGetJDBCPortReturnsCorrectValue() { + assertThat(testManager.getJDBCPort()).isEqualTo(MySQLContainer.MYSQL_PORT); + } +} diff --git a/it/jdbc/src/test/java/org/apache/beam/it/jdbc/OracleResourceManagerTest.java b/it/jdbc/src/test/java/org/apache/beam/it/jdbc/OracleResourceManagerTest.java new file mode 100644 index 0000000000000..a2da44d78493d --- /dev/null +++ b/it/jdbc/src/test/java/org/apache/beam/it/jdbc/OracleResourceManagerTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.jdbc; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.when; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.testcontainers.containers.OracleContainer; + +/** Integration tests for {@link OracleResourceManagerTest}. */ +@RunWith(JUnit4.class) +public class OracleResourceManagerTest { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock private T container; + + private OracleResourceManager testManager; + + private static final String TEST_ID = "test_id"; + private static final int DEFAULT_ORACLE_INTERNAL_PORT = 1521; + + @Before + public void setUp() { + when(container.withUsername(any())).thenReturn(container); + when(container.withPassword(any())).thenReturn(container); + when(container.withDatabaseName(anyString())).thenReturn(container); + doReturn(container).when(container).withLogConsumer(any()); + testManager = new OracleResourceManager(container, new OracleResourceManager.Builder(TEST_ID)); + } + + @Test + public void testGetJDBCPortReturnsCorrectValue() { + assertThat(testManager.getJDBCPort()).isEqualTo(DEFAULT_ORACLE_INTERNAL_PORT); + } +} diff --git a/it/jdbc/src/test/java/org/apache/beam/it/jdbc/PostgresResourceManagerTest.java b/it/jdbc/src/test/java/org/apache/beam/it/jdbc/PostgresResourceManagerTest.java new file mode 100644 index 0000000000000..a8a29816a6ef0 --- /dev/null +++ b/it/jdbc/src/test/java/org/apache/beam/it/jdbc/PostgresResourceManagerTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.jdbc; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.when; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.testcontainers.containers.PostgreSQLContainer; + +/** Integration tests for {@link PostgresResourceManagerTest}. */ +@RunWith(JUnit4.class) +public class PostgresResourceManagerTest> { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock private T container; + + private PostgresResourceManager testManager; + + private static final String TEST_ID = "test_id"; + + @Before + public void setUp() { + when(container.withUsername(any())).thenReturn(container); + when(container.withPassword(any())).thenReturn(container); + when(container.withDatabaseName(anyString())).thenReturn(container); + doReturn(container).when(container).withLogConsumer(any()); + testManager = + new PostgresResourceManager(container, new PostgresResourceManager.Builder(TEST_ID)); + } + + @Test + public void testGetJDBCPortReturnsCorrectValue() { + assertThat(testManager.getJDBCPort()).isEqualTo(PostgreSQLContainer.POSTGRESQL_PORT); + } +} diff --git a/it/kafka/build.gradle b/it/kafka/build.gradle new file mode 100644 index 0000000000000..faae88bb9f5b3 --- /dev/null +++ b/it/kafka/build.gradle @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.it.kafka', +) + +description = "Apache Beam :: IT :: Kafka" +ext.summary = "Integration test utilities for Kafka." + +dependencies { + implementation project(path: ":it:testcontainers", configuration: "shadow") + implementation library.java.testcontainers_kafka + implementation library.java.kafka_clients + + // TODO: excluding Guava until Truth updates it to >32.1.x + testImplementation(library.java.truth) { + exclude group: 'com.google.guava', module: 'guava' + } + testImplementation library.java.mockito_core + testRuntimeOnly library.java.slf4j_simple + // TODO: Remove the below dependencies after KafkaIOLT has been moved to the right directory. + testImplementation project(path: ":it:google-cloud-platform") + testImplementation project(path: ":sdks:java:io:kafka") + testImplementation project(path: ":sdks:java:io:synthetic") + testImplementation project(path: ":runners:direct-java") + testImplementation project(path: ":sdks:java:testing:test-utils") +} diff --git a/it/kafka/src/main/java/org/apache/beam/it/kafka/KafkaResourceManager.java b/it/kafka/src/main/java/org/apache/beam/it/kafka/KafkaResourceManager.java new file mode 100644 index 0000000000000..7f7fb5b695698 --- /dev/null +++ b/it/kafka/src/main/java/org/apache/beam/it/kafka/KafkaResourceManager.java @@ -0,0 +1,303 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.kafka; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; +import javax.annotation.Nullable; +import org.apache.beam.it.common.ResourceManager; +import org.apache.beam.it.testcontainers.TestContainerResourceManager; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.KafkaContainer; +import org.testcontainers.utility.DockerImageName; + +/** + * Client for managing Kafka resources. + * + *

    The class supports multiple topic names per server object. A topic is created if one has not + * been created already. + * + *

    The topic name is formed using testId. The topic name will be "{testId}-{ISO8601 time, + * microsecond precision}", with additional formatting. + * + *

    The class is thread-safe. + */ +public class KafkaResourceManager extends TestContainerResourceManager> + implements ResourceManager { + + private static final Logger LOG = LoggerFactory.getLogger(KafkaResourceManager.class); + + private static final String DEFAULT_KAFKA_CONTAINER_NAME = "confluentinc/cp-kafka"; + + // A list of available Kafka Docker image tags can be found at + // https://hub.docker.com/r/confluentinc/cp-kafka/tags + private static final String DEFAULT_KAFKA_CONTAINER_TAG = "7.3.1"; + + // 9093 is the default port that kafka broker is configured to listen on. + private static final int KAFKA_BROKER_PORT = 9093; + + private final AdminClient kafkaClient; + private final Set topicNames; + private final String connectionString; + private final boolean usingStaticTopic; + + private KafkaResourceManager(KafkaResourceManager.Builder builder) { + this(null, new DefaultKafkaContainer(builder), builder); + } + + @VisibleForTesting + @SuppressWarnings("nullness") + KafkaResourceManager( + @Nullable AdminClient client, + KafkaContainer container, + KafkaResourceManager.Builder builder) { + super(container, builder); + + this.usingStaticTopic = builder.topicNames.size() > 0; + if (this.usingStaticTopic) { + this.topicNames = new HashSet<>(builder.topicNames); + } else { + Set setOfTopicNames = new HashSet<>(); + for (int index = 0; index < builder.numTopics; ++index) { + setOfTopicNames.add( + KafkaResourceManagerUtils.generateTopicName( + String.format("%s-%d", builder.testId, index))); + } + this.topicNames = new HashSet<>(setOfTopicNames); + } + + this.connectionString = + String.format("PLAINTEXT://%s:%d", this.getHost(), this.getPort(KAFKA_BROKER_PORT)); + + this.kafkaClient = + client != null + ? client + : AdminClient.create(ImmutableMap.of("bootstrap.servers", this.connectionString)); + } + + public static KafkaResourceManager.Builder builder(String testId) { + return new KafkaResourceManager.Builder(testId); + } + + /** Returns the kafka bootstrap server connection string. */ + public synchronized String getBootstrapServers() { + return connectionString; + } + + /** + * Returns a list of names of the topics that this kafka manager will operate in. + * + * @return names of the kafka topics. + */ + public synchronized Set getTopicNames() { + return topicNames; + } + + /** + * Creates a kafka topic. + * + *

    Note: Implementations may do topic creation here, if one does not already exist. + * + * @param topicName Topic name to associate with the given kafka instance. + * @param partitions Number of partitions on the topic. + * @return The name of the topic that was created. + * @throws KafkaResourceManagerException if there is an error creating the kafka topic. + */ + public synchronized String createTopic(String topicName, int partitions) + throws KafkaResourceManagerException { + checkArgument(partitions > 0, "partitions must be positive."); + + String uniqueName = KafkaResourceManagerUtils.generateTopicName(topicName); + try { + Set currentTopics = kafkaClient.listTopics().names().get(); + if (!currentTopics.contains(uniqueName)) { + kafkaClient + .createTopics( + Collections.singletonList(new NewTopic(uniqueName, partitions, (short) 1))) + .all() + .get(); + topicNames.add(uniqueName); + } + } catch (Exception e) { + throw new KafkaResourceManagerException("Error creating topics.", e); + } + + LOG.info("Successfully created topic {}.", uniqueName); + + return uniqueName; + } + + /** Build a {@link KafkaProducer} for the given serializer and deserializers. */ + public synchronized KafkaProducer buildProducer( + Serializer keySerializer, Serializer valueSerializer) { + return new KafkaProducer<>( + ImmutableMap.of("bootstrap.servers", getBootstrapServers()), + keySerializer, + valueSerializer); + } + + /** Build a {@link KafkaConsumer} for the given serializer and deserializers. */ + public synchronized KafkaConsumer buildConsumer( + Deserializer keyDeserializer, Deserializer valueDeserializer) { + + return new KafkaConsumer<>( + ImmutableMap.of( + ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, + getBootstrapServers().replace("PLAINTEXT://", ""), + ConsumerConfig.GROUP_ID_CONFIG, + "cg-" + UUID.randomUUID(), + ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, + "earliest"), + keyDeserializer, + valueDeserializer); + } + + /** + * Deletes all created resources and cleans up the Kafka client, making the manager object + * unusable. + * + * @throws KafkaResourceManagerException if there is an error deleting the Kafka resources. + */ + @Override + public synchronized void cleanupAll() throws KafkaResourceManagerException { + LOG.info("Attempting to cleanup Kafka manager."); + + boolean producedError = false; + + // First, delete kafka topics if it was not given as a static argument + try { + if (!usingStaticTopic) { + kafkaClient.deleteTopics(topicNames).all().get(); + } + } catch (Exception e) { + LOG.error("Failed to delete kafka topic.", e); + producedError = true; + } + + // Throw Exception at the end if there were any errors + if (producedError) { + throw new KafkaResourceManagerException( + "Failed to delete resources. Check above for errors."); + } + + super.cleanupAll(); + + LOG.info("Kafka manager successfully cleaned up."); + } + + /** Builder for {@link KafkaResourceManager}. */ + public static final class Builder + extends TestContainerResourceManager.Builder { + + private final Set topicNames; + int numTopics; + + private Builder(String testId) { + super(testId, DEFAULT_KAFKA_CONTAINER_NAME, DEFAULT_KAFKA_CONTAINER_TAG); + this.topicNames = new HashSet<>(); + this.numTopics = 0; + } + + /** + * Sets names of topics to be used to that of a static kafka instance. Use this method only when + * attempting to operate on pre-existing kafka topics. + * + *

    Note: if a topic name is set, and a static kafka server is being used + * (useStaticContainer() is also called on the builder), then a topic will be created on the + * static server if it does not exist, and it will NOT be removed when cleanupAll() is called on + * the KafkaResourceManager. + * + * @param topicNames A set of topic names. + * @return this builder object with the topic name set. + */ + public Builder setTopicNames(Set topicNames) { + this.topicNames.clear(); + this.topicNames.addAll(topicNames); + return this; + } + + /** + * Sets the number of topics to be used instead of explicitly assign the topic names. + * + *

    Note: if number of topics is set, and a static kafka server is being used + * (useStaticContainer() is also called on the builder), then the assigned number of topics will + * be created on the static server, and it will be removed when cleanupAll() is called on the + * KafkaResourceManager. + * + * @param numTopics number of topics to be created. Default is 1. + * @return this builder object with numTopics set. + */ + public Builder setNumTopics(int numTopics) { + checkArgument(numTopics >= 0, "numTopics must be non-negative."); + checkArgument( + this.topicNames.size() == 0, + "setTopicNames and setNumTopics cannot be set at the same time."); + this.numTopics = numTopics; + return this; + } + + @Override + public KafkaResourceManager build() { + return new KafkaResourceManager(this); + } + } + + static class DefaultKafkaContainer extends KafkaContainer { + + private final @Nullable String host; + + public DefaultKafkaContainer(Builder builder) { + super(DockerImageName.parse(builder.containerImageName).withTag(builder.containerImageTag)); + this.host = builder.host; + } + + @Override + public String getHost() { + if (this.host == null) { + return super.getHost(); + } + return this.host; + } + + @Override + @SuppressWarnings("nullness") + public boolean equals(Object o) { + return super.equals(o); + } + + @Override + public int hashCode() { + return super.hashCode(); + } + } +} diff --git a/it/kafka/src/main/java/org/apache/beam/it/kafka/KafkaResourceManagerException.java b/it/kafka/src/main/java/org/apache/beam/it/kafka/KafkaResourceManagerException.java new file mode 100644 index 0000000000000..6cc390c9f5bc1 --- /dev/null +++ b/it/kafka/src/main/java/org/apache/beam/it/kafka/KafkaResourceManagerException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.kafka; + +/** Custom exception for {@link KafkaResourceManager} implementations. */ +public class KafkaResourceManagerException extends RuntimeException { + + public KafkaResourceManagerException(String errorMessage, Throwable err) { + super(errorMessage, err); + } + + public KafkaResourceManagerException(String errorMessage) { + super(errorMessage); + } +} diff --git a/it/kafka/src/main/java/org/apache/beam/it/kafka/KafkaResourceManagerUtils.java b/it/kafka/src/main/java/org/apache/beam/it/kafka/KafkaResourceManagerUtils.java new file mode 100644 index 0000000000000..d9829184a9fde --- /dev/null +++ b/it/kafka/src/main/java/org/apache/beam/it/kafka/KafkaResourceManagerUtils.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.kafka; + +import static org.apache.beam.it.common.utils.ResourceManagerUtils.generateResourceId; + +import java.time.format.DateTimeFormatter; +import java.util.regex.Pattern; + +/** Utilities for {@link KafkaResourceManager} implementations. */ +final class KafkaResourceManagerUtils { + + // from + // https://github.com/apache/kafka/blob/0.10.2/core/src/main/scala/kafka/common/Topic.scala#L24 + // legalChars = "[a-zA-Z0-9\\._\\-]" + // maxNameLength = 249 + private static final int MAX_TOPIC_NAME_LENGTH = 249; + private static final Pattern ILLEGAL_TOPIC_NAME_CHARS = Pattern.compile("[^a-zA-Z0-9._\\-]"); + private static final String REPLACE_TOPIC_NAME_CHAR = "-"; + private static final DateTimeFormatter TIME_FORMAT = + DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss-SSSSSS"); + + private KafkaResourceManagerUtils() {} + + /** + * Generates a kafka topic name from a given string. + * + * @param baseString The string to generate the name from. + * @return The topic name string. + */ + static String generateTopicName(String baseString) { + + return generateResourceId( + baseString, + ILLEGAL_TOPIC_NAME_CHARS, + REPLACE_TOPIC_NAME_CHAR, + MAX_TOPIC_NAME_LENGTH, + TIME_FORMAT); + } +} diff --git a/it/kafka/src/main/java/org/apache/beam/it/kafka/package-info.java b/it/kafka/src/main/java/org/apache/beam/it/kafka/package-info.java new file mode 100644 index 0000000000000..ebde89d0a085a --- /dev/null +++ b/it/kafka/src/main/java/org/apache/beam/it/kafka/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing Kafka resources within integration tests. */ +package org.apache.beam.it.kafka; diff --git a/it/kafka/src/test/java/org/apache/beam/it/kafka/KafkaIOLT.java b/it/kafka/src/test/java/org/apache/beam/it/kafka/KafkaIOLT.java new file mode 100644 index 0000000000000..ce6ad877c375f --- /dev/null +++ b/it/kafka/src/test/java/org/apache/beam/it/kafka/KafkaIOLT.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.kafka; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import com.google.auto.value.AutoValue; +import java.io.IOException; +import java.time.Duration; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Map; +import java.util.UUID; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.TestProperties; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.IOLoadTestBase; +import org.apache.beam.runners.direct.DirectOptions; +import org.apache.beam.sdk.io.kafka.KafkaIO; +import org.apache.beam.sdk.io.synthetic.SyntheticOptions; +import org.apache.beam.sdk.io.synthetic.SyntheticSourceOptions; +import org.apache.beam.sdk.io.synthetic.SyntheticUnboundedSource; +import org.apache.beam.sdk.options.StreamingOptions; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.testing.TestPipelineOptions; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; + +/** + * KafkaIO performance tests. + * + *

    Example trigger command: "mvn test -pl it -am -Dtest="KafkaIOLT" -Dproject=[gcpProject] \ + * -DartifactBucket=[temp bucket] -DfailIfNoTests=false". + */ +public final class KafkaIOLT extends IOLoadTestBase { + private static KafkaResourceManager resourceManager; + private static final String READ_ELEMENT_METRIC_NAME = "read_count"; + private static final int ROW_SIZE = 1024; + private Configuration configuration; + private String kafkaTopic; + + private SyntheticSourceOptions sourceOptions; + + @Rule public TestPipeline writePipeline = TestPipeline.create(); + @Rule public TestPipeline readPipeline = TestPipeline.create(); + + @BeforeClass + public static void beforeClass() { + resourceManager = KafkaResourceManager.builder("io-kafka-lt").build(); + } + + @Before + public void setup() throws IOException { + kafkaTopic = + "io-kafka-" + + DateTimeFormatter.ofPattern("MMddHHmmssSSS") + .withZone(ZoneId.of("UTC")) + .format(java.time.Instant.now()) + + UUID.randomUUID().toString().substring(0, 10); + + String testConfig = + TestProperties.getProperty("configuration", "local", TestProperties.Type.PROPERTY); + configuration = TEST_CONFIGS.get(testConfig); + if (configuration == null) { + throw new IllegalArgumentException( + String.format( + "Unknown test configuration: [%s]. Known configs: %s", + testConfig, TEST_CONFIGS.keySet())); + } + sourceOptions = + SyntheticOptions.fromJsonString( + configuration.getSourceOptions(), SyntheticSourceOptions.class); + + // tempLocation needs to be set for DataflowRunner + if (!Strings.isNullOrEmpty(tempBucketName)) { + String tempLocation = String.format("gs://%s/temp/", tempBucketName); + writePipeline.getOptions().as(TestPipelineOptions.class).setTempRoot(tempLocation); + writePipeline.getOptions().setTempLocation(tempLocation); + readPipeline.getOptions().as(TestPipelineOptions.class).setTempRoot(tempLocation); + readPipeline.getOptions().setTempLocation(tempLocation); + } + // Use streaming pipeline to write and read records + writePipeline.getOptions().as(StreamingOptions.class).setStreaming(true); + writePipeline.getOptions().as(DirectOptions.class).setBlockOnRun(false); + readPipeline.getOptions().as(StreamingOptions.class).setStreaming(true); + readPipeline.getOptions().as(DirectOptions.class).setBlockOnRun(false); + } + + @AfterClass + public static void tearDownClass() { + ResourceManagerUtils.cleanResources(resourceManager); + } + + private static final Map TEST_CONFIGS = + ImmutableMap.of( + "local", Configuration.of(1000L, 2, "DirectRunner"), // 1MB + "medium", Configuration.of(10_000_000L, 20, "DataflowRunner"), // 10 GB + "large", Configuration.of(100_000_000L, 80, "DataflowRunner") // 100 GB + ); + + /** Run integration test with configurations specified by TestProperties. */ + @Test + public void testWriteAndRead() throws IOException { + PipelineLauncher.LaunchInfo writeInfo = testWrite(); + PipelineLauncher.LaunchInfo readInfo = testRead(); + try { + PipelineOperator.Result result = + pipelineOperator.waitUntilDone( + createConfig(readInfo, Duration.ofMinutes(configuration.getPipelineTimeout()))); + assertNotEquals(PipelineOperator.Result.LAUNCH_FAILED, result); + // streaming read pipeline does not end itself + assertEquals( + PipelineLauncher.JobState.RUNNING, + pipelineLauncher.getJobStatus(project, region, readInfo.jobId())); + // Fail the test if write pipeline (streaming) not in running state. + // TODO: there is a limitation (or bug) that the cache in KafkaWriter can stay indefinitely if + // there is no upcoming records. Currently set expected records = (records generated - 10). + double numRecords = + pipelineLauncher.getMetric( + project, + region, + readInfo.jobId(), + getBeamMetricsName(PipelineMetricsType.COUNTER, READ_ELEMENT_METRIC_NAME)); + assertEquals(configuration.getNumRows(), numRecords, 10.0); + } finally { + // clean up pipelines + if (pipelineLauncher.getJobStatus(project, region, writeInfo.jobId()) + == PipelineLauncher.JobState.RUNNING) { + pipelineLauncher.cancelJob(project, region, writeInfo.jobId()); + } + if (pipelineLauncher.getJobStatus(project, region, readInfo.jobId()) + == PipelineLauncher.JobState.RUNNING) { + pipelineLauncher.cancelJob(project, region, readInfo.jobId()); + } + } + } + + private PipelineLauncher.LaunchInfo testWrite() throws IOException { + + KafkaIO.Write writeIO = + KafkaIO.write() + .withBootstrapServers(resourceManager.getBootstrapServers()) + .withTopic(kafkaTopic) + .withKeySerializer(ByteArraySerializer.class) + .withValueSerializer(ByteArraySerializer.class); + + writePipeline + .apply( + "Generate records", + org.apache.beam.sdk.io.Read.from(new SyntheticUnboundedSource(sourceOptions))) + .apply("Write to Kafka", writeIO.withTopic(kafkaTopic)); + + PipelineLauncher.LaunchConfig options = + PipelineLauncher.LaunchConfig.builder("write-kafka") + .setSdk(PipelineLauncher.Sdk.JAVA) + .setPipeline(writePipeline) + .addParameter("runner", configuration.getRunner()) + .build(); + + return pipelineLauncher.launch(project, region, options); + } + + private PipelineLauncher.LaunchInfo testRead() throws IOException { + KafkaIO.Read readIO = + KafkaIO.readBytes() + .withBootstrapServers(resourceManager.getBootstrapServers()) + .withTopic(kafkaTopic) + .withConsumerConfigUpdates(ImmutableMap.of("auto.offset.reset", "earliest")); + readPipeline + .apply("Read from unbounded Kafka", readIO) + .apply("Counting element", ParDo.of(new CountingFn<>(READ_ELEMENT_METRIC_NAME))); + + PipelineLauncher.LaunchConfig options = + PipelineLauncher.LaunchConfig.builder("read-kafka") + .setSdk(PipelineLauncher.Sdk.JAVA) + .setPipeline(readPipeline) + .addParameter("runner", configuration.getRunner()) + .build(); + + return pipelineLauncher.launch(project, region, options); + } + + /** Options for Kafka IO load test. */ + @AutoValue + abstract static class Configuration { + abstract Long getNumRows(); + + abstract Integer getPipelineTimeout(); + + abstract String getRunner(); + + abstract Integer getRowSize(); + + static Configuration of(long numRows, int pipelineTimeout, String runner) { + return new AutoValue_KafkaIOLT_Configuration.Builder() + .setNumRows(numRows) + .setPipelineTimeout(pipelineTimeout) + .setRunner(runner) + .setRowSize(ROW_SIZE) + .build(); + } + + @AutoValue.Builder + abstract static class Builder { + abstract Builder setNumRows(long numRows); + + abstract Builder setPipelineTimeout(int timeOutMinutes); + + abstract Builder setRunner(String runner); + + abstract Builder setRowSize(int rowSize); + + abstract Configuration build(); + } + + abstract Builder toBuilder(); + + /** Synthetic source options. */ + String getSourceOptions() { + return String.format( + "{\"numRecords\":%d,\"keySizeBytes\":4,\"valueSizeBytes\":%d}", + getNumRows(), getRowSize()); + } + } +} diff --git a/it/kafka/src/test/java/org/apache/beam/it/kafka/KafkaResourceManagerTest.java b/it/kafka/src/test/java/org/apache/beam/it/kafka/KafkaResourceManagerTest.java new file mode 100644 index 0000000000000..8c870815efc4c --- /dev/null +++ b/it/kafka/src/test/java/org/apache/beam/it/kafka/KafkaResourceManagerTest.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.kafka; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.ExecutionException; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.kafka.clients.admin.AdminClient; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.testcontainers.containers.KafkaContainer; + +/** Unit tests for {@link KafkaResourceManager}. */ +@RunWith(JUnit4.class) +public final class KafkaResourceManagerTest { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock private KafkaContainer container; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private AdminClient kafkaClient; + + private static final String TEST_ID = "test-id"; + private static final String HOST = "localhost"; + private static final String TOPIC_NAME = "topic-name"; + private static final int KAFKA_PORT = 9093; + private static final int MAPPED_PORT = 10000; + + private KafkaResourceManager testManager; + + @Before + public void setUp() throws IOException { + doReturn(container).when(container).withLogConsumer(any()); + testManager = + new KafkaResourceManager(kafkaClient, container, KafkaResourceManager.builder(TEST_ID)); + } + + @Test + public void testCreateResourceManagerBuilderReturnsKafkaResourceManager() throws IOException { + assertThat( + KafkaResourceManager.builder(TEST_ID) + .useStaticContainer() + .setHost(HOST) + .setPort(KAFKA_PORT) + .build()) + .isInstanceOf(KafkaResourceManager.class); + } + + @Test + public void testGetBootstrapServersShouldReturnCorrectValue() throws IOException { + when(container.getHost()).thenReturn(HOST); + when(container.getMappedPort(KAFKA_PORT)).thenReturn(MAPPED_PORT); + assertThat( + new KafkaResourceManager(kafkaClient, container, KafkaResourceManager.builder(TEST_ID)) + .getBootstrapServers()) + .matches("PLAINTEXT://" + HOST + ":" + MAPPED_PORT); + } + + @Test + public void testGetTopicNameShouldReturnCorrectValue() { + for (String topicName : testManager.getTopicNames()) { + assertThat(topicName).matches(TEST_ID + "-\\d" + "-\\d{8}-\\d{6}-\\d{6}"); + } + } + + @Test + public void testSetTopicNamesAndSetNumTopicsExclusive() { + assertThrows( + IllegalArgumentException.class, + () -> + KafkaResourceManager.builder(TEST_ID) + .setTopicNames(ImmutableSet.of(TOPIC_NAME)) + .setNumTopics(1)); + } + + @Test + public void testCreateTopicZeroPartitionsThrowErrors() { + assertThrows(IllegalArgumentException.class, () -> testManager.createTopic(TOPIC_NAME, 0)); + } + + @Test + public void testCreateTopicShouldThrowErrorWhenKafkaFailsToListTopics() + throws ExecutionException, InterruptedException { + when(kafkaClient.listTopics().names().get()) + .thenThrow(new ExecutionException(new RuntimeException("list topic future fails"))); + + assertThrows(KafkaResourceManagerException.class, () -> testManager.createTopic(TOPIC_NAME, 1)); + } + + @Test + public void testCreateTopicShouldThrowErrorWhenKafkaFailsToCreateTopic() + throws ExecutionException, InterruptedException { + when(kafkaClient.createTopics(any(Collection.class)).all().get()) + .thenThrow(new ExecutionException(new RuntimeException("create topic future fails"))); + + assertThrows(KafkaResourceManagerException.class, () -> testManager.createTopic(TOPIC_NAME, 1)); + } + + @Test + public void testCreateTopicShouldReturnTopicIfTopicExists() + throws ExecutionException, InterruptedException { + when(kafkaClient.listTopics().names().get()).thenReturn(Collections.singleton(TOPIC_NAME)); + + assertNotNull(testManager.createTopic(TOPIC_NAME, 1)); + } + + @Test + public void testCreateTopicShouldWork() throws ExecutionException, InterruptedException { + when(kafkaClient.createTopics(anyCollection()).all().get()).thenReturn(null); + + assertNotNull(testManager.createTopic(TOPIC_NAME, 1)); + } + + @Test + public void testCleanupAllShouldNotDropStaticTopic() throws IOException { + KafkaResourceManager.Builder builder = + KafkaResourceManager.builder(TEST_ID).setTopicNames(Collections.singleton(TOPIC_NAME)); + KafkaResourceManager tm = new KafkaResourceManager(kafkaClient, container, builder); + + tm.cleanupAll(); + + verify(kafkaClient, never()).deleteTopics(anyCollection()); + } + + @Test + public void testCleanupShouldDropNonStaticTopic() throws IOException { + final int numTopics = 3; + KafkaResourceManager.Builder builder = + KafkaResourceManager.builder(TEST_ID).setNumTopics(numTopics); + KafkaResourceManager tm = new KafkaResourceManager(kafkaClient, container, builder); + + tm.cleanupAll(); + verify(kafkaClient).deleteTopics(argThat(list -> list.size() == numTopics)); + } + + @Test + public void testCleanupAllShouldThrowErrorWhenKafkaClientFailsToDeleteTopic() + throws ExecutionException, InterruptedException { + when(kafkaClient.deleteTopics(anyCollection()).all().get()) + .thenThrow(new ExecutionException(new RuntimeException("delete topic future fails"))); + + assertThrows(KafkaResourceManagerException.class, () -> testManager.cleanupAll()); + } +} diff --git a/it/kafka/src/test/java/org/apache/beam/it/kafka/KafkaResourceManagerUtilsTest.java b/it/kafka/src/test/java/org/apache/beam/it/kafka/KafkaResourceManagerUtilsTest.java new file mode 100644 index 0000000000000..e2a1dc57d42bf --- /dev/null +++ b/it/kafka/src/test/java/org/apache/beam/it/kafka/KafkaResourceManagerUtilsTest.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.kafka; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link KafkaResourceManagerUtils}. */ +@RunWith(JUnit4.class) +public class KafkaResourceManagerUtilsTest { + @Test + public void testGenerateTopicNameShouldReplaceIllegalChars() { + String testBaseString = "^apache_beam/io\\kafka\0"; + String actual = KafkaResourceManagerUtils.generateTopicName(testBaseString); + assertThat(actual).matches("-apache_beam-io-kafka--\\d{8}-\\d{6}-\\d{6}"); + } +} diff --git a/it/mongodb/build.gradle b/it/mongodb/build.gradle new file mode 100644 index 0000000000000..6b4fa400abd37 --- /dev/null +++ b/it/mongodb/build.gradle @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.it.mongodb', +) + +description = "Apache Beam :: IT :: MongoDB" +ext.summary = "Integration test utilities for MongoDB." + +dependencies { + implementation project(path: ":it:testcontainers", configuration: "shadow") + implementation project(path: ":it:conditions", configuration: "shadow") + implementation project(path: ":it:truthmatchers", configuration: "shadow") + implementation library.java.testcontainers_mongodb + implementation library.java.mongo_java_driver + + testImplementation library.java.mockito_core + // TODO: excluding Guava until Truth updates it to >32.1.x + testImplementation(library.java.truth) { + exclude group: 'com.google.guava', module: 'guava' + } + testImplementation library.java.commons_lang3 + testRuntimeOnly library.java.slf4j_simple +} diff --git a/it/mongodb/src/main/java/org/apache/beam/it/mongodb/MongoDBResourceManager.java b/it/mongodb/src/main/java/org/apache/beam/it/mongodb/MongoDBResourceManager.java new file mode 100644 index 0000000000000..80216b14ac0e6 --- /dev/null +++ b/it/mongodb/src/main/java/org/apache/beam/it/mongodb/MongoDBResourceManager.java @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.mongodb; + +import static org.apache.beam.it.mongodb.MongoDBResourceManagerUtils.checkValidCollectionName; +import static org.apache.beam.it.mongodb.MongoDBResourceManagerUtils.generateDatabaseName; + +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import javax.annotation.Nullable; +import org.apache.beam.it.common.ResourceManager; +import org.apache.beam.it.testcontainers.TestContainerResourceManager; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.bson.Document; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.MongoDBContainer; +import org.testcontainers.utility.DockerImageName; + +/** + * Client for managing MongoDB resources. + * + *

    The class supports one database and multiple collections per database object. A database is + * created when the first collection is created if one has not been created already. + * + *

    The database name is formed using testId. The database name will be "{testId}-{ISO8601 time, + * microsecond precision}", with additional formatting. + * + *

    The class is thread-safe. + */ +public class MongoDBResourceManager extends TestContainerResourceManager + implements ResourceManager { + + private static final Logger LOG = LoggerFactory.getLogger(MongoDBResourceManager.class); + + private static final String DEFAULT_MONGODB_CONTAINER_NAME = "mongo"; + + // A list of available MongoDB Docker image tags can be found at + // https://hub.docker.com/_/mongo/tags + private static final String DEFAULT_MONGODB_CONTAINER_TAG = "4.0.18"; + + // 27017 is the default port that MongoDB is configured to listen on + private static final int MONGODB_INTERNAL_PORT = 27017; + + private final MongoClient mongoClient; + private final String databaseName; + private final String connectionString; + private final boolean usingStaticDatabase; + + private MongoDBResourceManager(MongoDBResourceManager.Builder builder) { + this( + /* mongoClient= */ null, + new MongoDBContainer( + DockerImageName.parse(builder.containerImageName).withTag(builder.containerImageTag)), + builder); + } + + @VisibleForTesting + @SuppressWarnings("nullness") + MongoDBResourceManager( + @Nullable MongoClient mongoClient, + MongoDBContainer container, + MongoDBResourceManager.Builder builder) { + super(container, builder); + + this.usingStaticDatabase = builder.databaseName != null; + this.databaseName = + usingStaticDatabase ? builder.databaseName : generateDatabaseName(builder.testId); + this.connectionString = + String.format("mongodb://%s:%d", this.getHost(), this.getPort(MONGODB_INTERNAL_PORT)); + this.mongoClient = mongoClient == null ? MongoClients.create(connectionString) : mongoClient; + } + + public static MongoDBResourceManager.Builder builder(String testId) { + return new MongoDBResourceManager.Builder(testId); + } + + /** Returns the URI connection string to the MongoDB Database. */ + public synchronized String getUri() { + return connectionString; + } + + /** + * Returns the name of the Database that this MongoDB manager will operate in. + * + * @return the name of the MongoDB Database. + */ + public synchronized String getDatabaseName() { + return databaseName; + } + + private synchronized MongoDatabase getDatabase() { + try { + return mongoClient.getDatabase(databaseName); + } catch (Exception e) { + throw new MongoDBResourceManagerException( + "Error retrieving database " + databaseName + " from MongoDB.", e); + } + } + + private synchronized boolean collectionExists(String collectionName) { + // Check collection name + checkValidCollectionName(databaseName, collectionName); + + Iterable collectionNames = getDatabase().listCollectionNames(); + for (String name : collectionNames) { + // The Collection already exists in the database, return false. + if (collectionName.equals(name)) { + return true; + } + } + + return false; + } + + /** + * Creates a collection in MongoDB for storing Documents. + * + *

    Note: Implementations may do database creation here, if one does not already exist. + * + * @param collectionName Collection name to associate with the given MongoDB instance. + * @return A boolean indicating whether the resource was created. + * @throws MongoDBResourceManagerException if there is an error creating the collection in + * MongoDB. + */ + public synchronized boolean createCollection(String collectionName) + throws MongoDBResourceManagerException { + LOG.info("Creating collection using collectionName '{}'.", collectionName); + + try { + // Check to see if the Collection exists + if (collectionExists(collectionName)) { + return false; + } + // The Collection does not exist in the database, create it and return true. + getDatabase().createCollection(collectionName); + } catch (Exception e) { + throw new MongoDBResourceManagerException("Error creating collection.", e); + } + + LOG.info("Successfully created collection {}.{}", databaseName, collectionName); + + return true; + } + + /** + * Helper method to retrieve a MongoCollection with the given name from the database and return + * it. + * + * @param collectionName The name of the MongoCollection. + * @param createCollection A boolean that specifies to create the Collection if it does not exist. + * @return A MongoCollection with the given name. + */ + private MongoCollection getMongoDBCollection( + String collectionName, boolean createCollection) { + if (!collectionExists(collectionName) && !createCollection) { + throw new MongoDBResourceManagerException( + "Collection " + collectionName + " does not exists in database " + databaseName); + } + + return getDatabase().getCollection(collectionName); + } + + /** + * Inserts the given Document into a collection. + * + *

    A database will be created here, if one does not already exist. + * + * @param collectionName The name of the collection to insert the document into. + * @param document The document to insert into the collection. + * @return A boolean indicating whether the Document was inserted successfully. + */ + public synchronized boolean insertDocument(String collectionName, Document document) { + return insertDocuments(collectionName, ImmutableList.of(document)); + } + + /** + * Inserts the given Documents into a collection. + * + *

    Note: Implementations may do collection creation here, if one does not already exist. + * + * @param collectionName The name of the collection to insert the documents into. + * @param documents A list of documents to insert into the collection. + * @return A boolean indicating whether the Documents were inserted successfully. + * @throws MongoDBResourceManagerException if there is an error inserting the documents. + */ + public synchronized boolean insertDocuments(String collectionName, List documents) + throws MongoDBResourceManagerException { + LOG.info( + "Attempting to write {} documents to {}.{}.", + documents.size(), + databaseName, + collectionName); + + try { + getMongoDBCollection(collectionName, /* createCollection= */ true).insertMany(documents); + } catch (Exception e) { + throw new MongoDBResourceManagerException("Error inserting documents.", e); + } + + LOG.info( + "Successfully wrote {} documents to {}.{}", documents.size(), databaseName, collectionName); + + return true; + } + + /** + * Reads all the Documents in a collection. + * + * @param collectionName The name of the collection to read from. + * @return A List of all the Documents in the collection. + * @throws MongoDBResourceManagerException if there is an error reading the collection. + */ + public synchronized List readCollection(String collectionName) + throws MongoDBResourceManagerException { + LOG.info("Reading all documents from {}.{}", databaseName, collectionName); + + FindIterable fetchRecords; + try { + fetchRecords = getMongoDBCollection(collectionName, /* createCollection= */ false).find(); + } catch (Exception e) { + throw new MongoDBResourceManagerException("Error reading collection.", e); + } + List documents = + StreamSupport.stream(fetchRecords.spliterator(), false).collect(Collectors.toList()); + + LOG.info("Successfully loaded documents from {}.{}", databaseName, collectionName); + + return documents; + } + + /** + * Counts the number of Documents in a collection. + * + * @param collectionName The name of the collection to read from. + * @return The number of Documents in the collection. + * @throws MongoDBResourceManagerException if there is an error reading the collection. + */ + public synchronized long countCollection(String collectionName) + throws MongoDBResourceManagerException { + LOG.info("Counting all documents from {}.{}", databaseName, collectionName); + + long numDocuments = + getMongoDBCollection(collectionName, /* createCollection= */ false).countDocuments(); + + LOG.info( + "Successfully counted {} documents from {}.{}", numDocuments, databaseName, collectionName); + + return numDocuments; + } + + @Override + public synchronized void cleanupAll() { + LOG.info("Attempting to cleanup MongoDB manager."); + + boolean producedError = false; + + // First, delete the database if it was not given as a static argument + try { + if (!usingStaticDatabase) { + mongoClient.getDatabase(databaseName).drop(); + } + } catch (Exception e) { + LOG.error("Failed to delete MongoDB database {}.", databaseName, e); + producedError = true; + } + + // Next, try to close the MongoDB client connection + try { + mongoClient.close(); + } catch (Exception e) { + LOG.error("Failed to delete MongoDB client.", e); + producedError = true; + } + + // Throw Exception at the end if there were any errors + if (producedError) { + throw new MongoDBResourceManagerException( + "Failed to delete resources. Check above for errors."); + } + + super.cleanupAll(); + + LOG.info("MongoDB manager successfully cleaned up."); + } + + /** Builder for {@link MongoDBResourceManager}. */ + public static final class Builder + extends TestContainerResourceManager.Builder { + + private @Nullable String databaseName; + + private Builder(String testId) { + super(testId, DEFAULT_MONGODB_CONTAINER_NAME, DEFAULT_MONGODB_CONTAINER_TAG); + this.databaseName = null; + } + + /** + * Sets the database name to that of a static database instance. Use this method only when + * attempting to operate on a pre-existing Mongo database. + * + *

    Note: if a database name is set, and a static MongoDB server is being used + * (useStaticContainer() is also called on the builder), then a database will be created on the + * static server if it does not exist, and it will not be removed when cleanupAll() is called on + * the MongoDBResourceManager. + * + * @param databaseName The database name. + * @return this builder object with the database name set. + */ + public Builder setDatabaseName(String databaseName) { + this.databaseName = databaseName; + return this; + } + + @Override + public MongoDBResourceManager build() { + return new MongoDBResourceManager(this); + } + } +} diff --git a/it/mongodb/src/main/java/org/apache/beam/it/mongodb/MongoDBResourceManagerException.java b/it/mongodb/src/main/java/org/apache/beam/it/mongodb/MongoDBResourceManagerException.java new file mode 100644 index 0000000000000..0b6027eee1d12 --- /dev/null +++ b/it/mongodb/src/main/java/org/apache/beam/it/mongodb/MongoDBResourceManagerException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.mongodb; + +/** Custom exception for {@link MongoDBResourceManager} implementations. */ +public class MongoDBResourceManagerException extends RuntimeException { + + public MongoDBResourceManagerException(String errorMessage, Throwable err) { + super(errorMessage, err); + } + + public MongoDBResourceManagerException(String errorMessage) { + super(errorMessage); + } +} diff --git a/it/mongodb/src/main/java/org/apache/beam/it/mongodb/MongoDBResourceManagerUtils.java b/it/mongodb/src/main/java/org/apache/beam/it/mongodb/MongoDBResourceManagerUtils.java new file mode 100644 index 0000000000000..3e75a5e1ee51c --- /dev/null +++ b/it/mongodb/src/main/java/org/apache/beam/it/mongodb/MongoDBResourceManagerUtils.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.mongodb; + +import static org.apache.beam.it.common.utils.ResourceManagerUtils.generateResourceId; + +import java.time.format.DateTimeFormatter; +import java.util.regex.Pattern; + +/** Utilities for {@link MongoDBResourceManager} implementations. */ +final class MongoDBResourceManagerUtils { + + // MongoDB Database and Collection naming restrictions can be found at + // https://www.mongodb.com/docs/manual/reference/limits/#naming-restrictions + private static final int MAX_DATABASE_NAME_LENGTH = 63; + private static final Pattern ILLEGAL_DATABASE_NAME_CHARS = + Pattern.compile("[/\\\\. \"\0$]"); // i.e. [/\. "$] + private static final String REPLACE_DATABASE_NAME_CHAR = "-"; + private static final int MIN_COLLECTION_NAME_LENGTH = 1; + private static final int MAX_COLLECTION_NAME_LENGTH = 99; + private static final Pattern ILLEGAL_COLLECTION_CHARS = Pattern.compile("[$\0]"); + private static final DateTimeFormatter TIME_FORMAT = + DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss-SSSSSS"); + + private MongoDBResourceManagerUtils() {} + + /** + * Generates a MongoDB database name from a given string. + * + * @param baseString The string to generate the name from. + * @return The database name string. + */ + static String generateDatabaseName(String baseString) { + return generateResourceId( + baseString, + ILLEGAL_DATABASE_NAME_CHARS, + REPLACE_DATABASE_NAME_CHAR, + MAX_DATABASE_NAME_LENGTH, + TIME_FORMAT); + } + + /** + * Checks whether the given collection name is valid according to MongoDB constraints. + * + * @param databaseName the database name associated with the collection + * @param collectionName the collection name to check. + * @throws IllegalArgumentException if the collection name is invalid. + */ + static void checkValidCollectionName(String databaseName, String collectionName) { + String fullCollectionName = databaseName + "." + collectionName; + if (collectionName.length() < MIN_COLLECTION_NAME_LENGTH) { + throw new IllegalArgumentException("Collection name cannot be empty."); + } + if (fullCollectionName.length() > MAX_COLLECTION_NAME_LENGTH) { + throw new IllegalArgumentException( + "Collection name " + + fullCollectionName + + " cannot be longer than " + + MAX_COLLECTION_NAME_LENGTH + + " characters, including the database name and dot."); + } + if (ILLEGAL_COLLECTION_CHARS.matcher(collectionName).find()) { + throw new IllegalArgumentException( + "Collection name " + + collectionName + + " is not a valid name. Only letters, numbers, hyphens, underscores and exclamation points are allowed."); + } + if (collectionName.charAt(0) != '_' && !Character.isLetter(collectionName.charAt(0))) { + throw new IllegalArgumentException( + "Collection name " + collectionName + " must start with a letter or an underscore."); + } + String illegalKeyword = "system."; + if (collectionName.startsWith(illegalKeyword)) { + throw new IllegalArgumentException( + "Collection name " + + collectionName + + " cannot start with the prefix \"" + + illegalKeyword + + "\"."); + } + } +} diff --git a/it/mongodb/src/main/java/org/apache/beam/it/mongodb/conditions/MongoDBDocumentsCheck.java b/it/mongodb/src/main/java/org/apache/beam/it/mongodb/conditions/MongoDBDocumentsCheck.java new file mode 100644 index 0000000000000..926215d103a19 --- /dev/null +++ b/it/mongodb/src/main/java/org/apache/beam/it/mongodb/conditions/MongoDBDocumentsCheck.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.mongodb.conditions; + +import com.google.auto.value.AutoValue; +import javax.annotation.Nullable; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.mongodb.MongoDBResourceManager; + +/** ConditionCheck to validate if MongoDB has received a certain amount of documents. */ +@AutoValue +public abstract class MongoDBDocumentsCheck extends ConditionCheck { + + abstract MongoDBResourceManager resourceManager(); + + abstract String collectionName(); + + abstract Integer minDocuments(); + + @Nullable + abstract Integer maxDocuments(); + + @Override + public String getDescription() { + if (maxDocuments() != null) { + return String.format( + "MongoDB check if collection %s has between %d and %d documents", + collectionName(), minDocuments(), maxDocuments()); + } + return String.format( + "MongoDB check if collection %s has %d documents", collectionName(), minDocuments()); + } + + @Override + @SuppressWarnings("unboxing.of.nullable") + public CheckResult check() { + long totalDocuments = resourceManager().countCollection(collectionName()); + if (totalDocuments < minDocuments()) { + return new CheckResult( + false, String.format("Expected %d but has only %d", minDocuments(), totalDocuments)); + } + if (maxDocuments() != null && totalDocuments > maxDocuments()) { + return new CheckResult( + false, + String.format( + "Expected up to %d but found %d documents", maxDocuments(), totalDocuments)); + } + + if (maxDocuments() != null) { + return new CheckResult( + true, + String.format( + "Expected between %d and %d documents and found %d", + minDocuments(), maxDocuments(), totalDocuments)); + } + + return new CheckResult( + true, + String.format( + "Expected at least %d documents and found %d", minDocuments(), totalDocuments)); + } + + public static Builder builder(MongoDBResourceManager resourceManager, String collectionName) { + return new AutoValue_MongoDBDocumentsCheck.Builder() + .setResourceManager(resourceManager) + .setCollectionName(collectionName); + } + + /** Builder for {@link MongoDBDocumentsCheck}. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setResourceManager(MongoDBResourceManager resourceManager); + + public abstract Builder setCollectionName(String collectionName); + + public abstract Builder setMinDocuments(Integer minDocuments); + + public abstract Builder setMaxDocuments(Integer maxDocuments); + + abstract MongoDBDocumentsCheck autoBuild(); + + public MongoDBDocumentsCheck build() { + return autoBuild(); + } + } +} diff --git a/it/mongodb/src/main/java/org/apache/beam/it/mongodb/conditions/package-info.java b/it/mongodb/src/main/java/org/apache/beam/it/mongodb/conditions/package-info.java new file mode 100644 index 0000000000000..635e48e122729 --- /dev/null +++ b/it/mongodb/src/main/java/org/apache/beam/it/mongodb/conditions/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package that contains reusable MongoDB conditions. */ +package org.apache.beam.it.mongodb.conditions; diff --git a/it/mongodb/src/main/java/org/apache/beam/it/mongodb/matchers/MongoDBAsserts.java b/it/mongodb/src/main/java/org/apache/beam/it/mongodb/matchers/MongoDBAsserts.java new file mode 100644 index 0000000000000..1a1b86acf5623 --- /dev/null +++ b/it/mongodb/src/main/java/org/apache/beam/it/mongodb/matchers/MongoDBAsserts.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.mongodb.matchers; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatRecords; + +import com.google.gson.Gson; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.truthmatchers.RecordsSubject; +import org.bson.Document; + +/** Assert utilities for MongoDB tests. */ +public class MongoDBAsserts { + + /** + * Convert MongoDB {@link org.bson.Document} to a list of maps. + * + * @param documents List of Documents to parse + * @return List of maps to use in {@link RecordsSubject} + */ + public static List> mongoDBDocumentsToRecords(Iterable documents) { + try { + List> records = new ArrayList<>(); + + for (Document document : documents) { + Map converted = + new Gson().>fromJson(document.toJson(), Map.class); + records.add(converted); + } + + return records; + } catch (Exception e) { + throw new RuntimeException("Error converting MongoDB Documents to Records", e); + } + } + + /** + * Creates a {@link RecordsSubject} to assert information within a list of records. + * + * @param documents List of Documents in MongoDB {@link Document} format to use in the comparison. + * @return Truth Subject to chain assertions. + */ + public static RecordsSubject assertThatMongoDBDocuments(Collection documents) { + return assertThatRecords(mongoDBDocumentsToRecords(documents)); + } +} diff --git a/it/mongodb/src/main/java/org/apache/beam/it/mongodb/matchers/package-info.java b/it/mongodb/src/main/java/org/apache/beam/it/mongodb/matchers/package-info.java new file mode 100644 index 0000000000000..69caf9a68c6e7 --- /dev/null +++ b/it/mongodb/src/main/java/org/apache/beam/it/mongodb/matchers/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for MongoDB Truth matchers / subjects to have reusable assertions. */ +package org.apache.beam.it.mongodb.matchers; diff --git a/it/mongodb/src/main/java/org/apache/beam/it/mongodb/package-info.java b/it/mongodb/src/main/java/org/apache/beam/it/mongodb/package-info.java new file mode 100644 index 0000000000000..5f0c0c5c813ce --- /dev/null +++ b/it/mongodb/src/main/java/org/apache/beam/it/mongodb/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing MongoDB resources within integration tests. */ +package org.apache.beam.it.mongodb; diff --git a/it/mongodb/src/test/java/org/apache/beam/it/mongodb/MongoDBResourceManagerIT.java b/it/mongodb/src/test/java/org/apache/beam/it/mongodb/MongoDBResourceManagerIT.java new file mode 100644 index 0000000000000..fb4974d41b272 --- /dev/null +++ b/it/mongodb/src/test/java/org/apache/beam/it/mongodb/MongoDBResourceManagerIT.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.mongodb; + +import static com.google.common.truth.Truth.assertThat; + +import java.util.ArrayList; +import java.util.List; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.testcontainers.TestContainersIntegrationTest; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.bson.Document; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Integration tests for {@link MongoDBResourceManager}. */ +@Category(TestContainersIntegrationTest.class) +@RunWith(JUnit4.class) +public class MongoDBResourceManagerIT { + + public static final String COLLECTION_NAME = "dummy-collection"; + private MongoDBResourceManager mongoResourceManager; + + @Before + public void setUp() { + mongoResourceManager = MongoDBResourceManager.builder("dummy").build(); + } + + @Test + public void testResourceManagerE2E() { + boolean createIndex = mongoResourceManager.createCollection(COLLECTION_NAME); + assertThat(createIndex).isTrue(); + + List documents = new ArrayList<>(); + documents.add(new Document(ImmutableMap.of("id", 1, "company", "Google"))); + documents.add(new Document(ImmutableMap.of("id", 2, "company", "Alphabet"))); + + boolean insertDocuments = mongoResourceManager.insertDocuments(COLLECTION_NAME, documents); + assertThat(insertDocuments).isTrue(); + + List fetchRecords = mongoResourceManager.readCollection(COLLECTION_NAME); + + assertThat(fetchRecords).hasSize(2); + assertThat(fetchRecords).containsExactlyElementsIn(documents); + } + + @After + public void tearDown() { + ResourceManagerUtils.cleanResources(mongoResourceManager); + } +} diff --git a/it/mongodb/src/test/java/org/apache/beam/it/mongodb/MongoDBResourceManagerTest.java b/it/mongodb/src/test/java/org/apache/beam/it/mongodb/MongoDBResourceManagerTest.java new file mode 100644 index 0000000000000..b3ad34b70ff6e --- /dev/null +++ b/it/mongodb/src/test/java/org/apache/beam/it/mongodb/MongoDBResourceManagerTest.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.mongodb; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.mongodb.MongoBulkWriteException; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.bson.Document; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.testcontainers.containers.MongoDBContainer; + +/** Unit tests for {@link MongoDBResourceManager}. */ +@RunWith(JUnit4.class) +public class MongoDBResourceManagerTest { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private MongoClient mongoClient; + + @Mock private MongoDBContainer container; + + private static final String TEST_ID = "test-id"; + private static final String COLLECTION_NAME = "collection-name"; + private static final String STATIC_DATABASE_NAME = "database"; + private static final String HOST = "localhost"; + private static final int MONGO_DB_PORT = 27017; + private static final int MAPPED_PORT = 10000; + + private MongoDBResourceManager testManager; + + @Before + public void setUp() { + doReturn(container).when(container).withLogConsumer(any()); + testManager = + new MongoDBResourceManager(mongoClient, container, MongoDBResourceManager.builder(TEST_ID)); + } + + @Test + public void testCreateResourceManagerBuilderReturnsMongoDBResourceManager() { + assertThat( + MongoDBResourceManager.builder(TEST_ID) + .useStaticContainer() + .setHost(HOST) + .setPort(MONGO_DB_PORT) + .build()) + .isInstanceOf(MongoDBResourceManager.class); + } + + @Test + public void testGetUriShouldReturnCorrectValue() { + when(container.getMappedPort(MONGO_DB_PORT)).thenReturn(MAPPED_PORT); + assertThat( + new MongoDBResourceManager( + mongoClient, container, MongoDBResourceManager.builder(TEST_ID)) + .getUri()) + .matches("mongodb://" + HOST + ":" + MAPPED_PORT); + } + + @Test + public void testGetDatabaseNameShouldReturnCorrectValue() { + assertThat( + new MongoDBResourceManager( + mongoClient, container, MongoDBResourceManager.builder(TEST_ID)) + .getDatabaseName()) + .matches(TEST_ID + "-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testCreateCollectionShouldThrowErrorWhenCollectionNameIsInvalid() { + assertThrows( + MongoDBResourceManagerException.class, () -> testManager.createCollection("invalid$name")); + } + + @Test + public void testCreateCollectionShouldThrowErrorWhenMongoDBFailsToGetDB() { + doThrow(IllegalArgumentException.class).when(mongoClient).getDatabase(anyString()); + + assertThrows( + MongoDBResourceManagerException.class, () -> testManager.createCollection(COLLECTION_NAME)); + } + + @Test + public void testCreateCollectionShouldThrowErrorIfDatabaseFailsToListCollectionNames() { + when(mongoClient.getDatabase(anyString()).listCollectionNames()).thenReturn(null); + + assertThrows( + MongoDBResourceManagerException.class, () -> testManager.createCollection(COLLECTION_NAME)); + } + + @Test + public void testCreateCollectionShouldThrowErrorWhenMongoDBFailsToCreateCollection() { + MongoDatabase mockDatabase = mongoClient.getDatabase(anyString()); + doThrow(IllegalArgumentException.class).when(mockDatabase).createCollection(anyString()); + + assertThrows( + MongoDBResourceManagerException.class, () -> testManager.createCollection(COLLECTION_NAME)); + } + + @Test + public void testCreateCollectionShouldReturnTrueIfMongoDBDoesNotThrowAnyError() { + assertThat(testManager.createCollection(COLLECTION_NAME)).isEqualTo(true); + verify(mongoClient.getDatabase(anyString())).createCollection(anyString()); + } + + @Test + public void testCreateCollectionShouldReturnFalseIfCollectionAlreadyExists() { + when(mongoClient.getDatabase(anyString()).listCollectionNames().iterator().hasNext()) + .thenReturn(true, false); + when(mongoClient.getDatabase(anyString()).listCollectionNames().iterator().next()) + .thenReturn(COLLECTION_NAME); + + assertThat(testManager.createCollection(COLLECTION_NAME)).isEqualTo(false); + verify(mongoClient.getDatabase(anyString()), never()).getCollection(anyString()); + } + + @Test + public void testInsertDocumentsShouldCreateCollectionIfOneDoesNotExist() { + assertThat(testManager.insertDocument(COLLECTION_NAME, new Document())).isEqualTo(true); + + verify(mongoClient.getDatabase(anyString())).getCollection(anyString()); + verify(mongoClient.getDatabase(anyString())).listCollectionNames(); + verify(mongoClient.getDatabase(anyString()).getCollection(anyString())).insertMany(any()); + } + + @Test + public void + testInsertDocumentsShouldCreateCollectionIfUsingStaticDatabaseAndCollectionDoesNotExist() { + MongoDBResourceManager.Builder builder = + MongoDBResourceManager.builder(TEST_ID).setDatabaseName(STATIC_DATABASE_NAME); + MongoDBResourceManager tm = new MongoDBResourceManager(mongoClient, container, builder); + + assertThat(tm.insertDocument(COLLECTION_NAME, new Document())).isEqualTo(true); + + verify(mongoClient.getDatabase(anyString())).getCollection(anyString()); + verify(mongoClient.getDatabase(anyString())).listCollectionNames(); + verify(mongoClient.getDatabase(anyString()).getCollection(anyString())).insertMany(any()); + } + + @Test + public void testInsertDocumentsShouldReturnTrueIfUsingStaticDatabaseAndCollectionDoesExist() { + when(mongoClient.getDatabase(anyString()).listCollectionNames().iterator().hasNext()) + .thenReturn(true, false); + when(mongoClient.getDatabase(anyString()).listCollectionNames().iterator().next()) + .thenReturn(COLLECTION_NAME); + + MongoDBResourceManager.Builder builder = + MongoDBResourceManager.builder(TEST_ID).setDatabaseName(STATIC_DATABASE_NAME); + MongoDBResourceManager tm = new MongoDBResourceManager(mongoClient, container, builder); + + assertThat(tm.insertDocument(COLLECTION_NAME, new Document())).isEqualTo(true); + + verify(mongoClient.getDatabase(anyString())).getCollection(anyString()); + verify(mongoClient.getDatabase(anyString()).getCollection(anyString())).insertMany(any()); + } + + @Test + public void testInsertDocumentsShouldThrowErrorWhenMongoDBThrowsException() { + when(mongoClient.getDatabase(anyString()).listCollectionNames().iterator().hasNext()) + .thenReturn(true, false); + when(mongoClient.getDatabase(anyString()).listCollectionNames().iterator().next()) + .thenReturn(COLLECTION_NAME); + MongoCollection mockCollection = + mongoClient.getDatabase(anyString()).getCollection(anyString()); + + doThrow(MongoBulkWriteException.class) + .when(mockCollection) + .insertMany(ImmutableList.of(new Document())); + + assertThrows( + MongoDBResourceManagerException.class, + () -> testManager.insertDocument(COLLECTION_NAME, new Document())); + } + + @Test + public void testReadCollectionShouldThrowErrorWhenCollectionDoesNotExist() { + when(mongoClient.getDatabase(anyString()).listCollectionNames().iterator().hasNext()) + .thenReturn(true, false); + when(mongoClient.getDatabase(anyString()).listCollectionNames().iterator().next()) + .thenReturn("fake-collection-name"); + + assertThrows( + MongoDBResourceManagerException.class, () -> testManager.readCollection(COLLECTION_NAME)); + } + + @Test + public void testReadCollectionShouldThrowErrorWhenMongoDBFailsToFindDocuments() { + when(mongoClient.getDatabase(anyString()).listCollectionNames().iterator().hasNext()) + .thenReturn(true, false); + when(mongoClient.getDatabase(anyString()).listCollectionNames().iterator().next()) + .thenReturn(COLLECTION_NAME); + MongoCollection mockCollection = + mongoClient.getDatabase(anyString()).getCollection(anyString()); + doThrow(RuntimeException.class).when(mockCollection).find(); + + assertThrows( + MongoDBResourceManagerException.class, () -> testManager.readCollection(COLLECTION_NAME)); + } + + @Test + public void testReadCollectionShouldWorkWhenMongoDBDoesNotThrowAnyError() { + when(mongoClient.getDatabase(anyString()).listCollectionNames().iterator().hasNext()) + .thenReturn(true, false); + when(mongoClient.getDatabase(anyString()).listCollectionNames().iterator().next()) + .thenReturn(COLLECTION_NAME); + + testManager.readCollection(COLLECTION_NAME); + + verify(mongoClient.getDatabase(anyString()).getCollection(anyString())).find(); + } + + @Test + public void testCleanupAllShouldNotDropStaticDatabase() { + MongoDBResourceManager.Builder builder = + MongoDBResourceManager.builder(TEST_ID).setDatabaseName(STATIC_DATABASE_NAME); + MongoDBResourceManager tm = new MongoDBResourceManager(mongoClient, container, builder); + + tm.cleanupAll(); + + verify(mongoClient, never()).getDatabase(anyString()); + verify(mongoClient.getDatabase(anyString()), never()).drop(); + verify(mongoClient).close(); + } + + @Test + public void testCleanupShouldDropNonStaticDatabase() { + testManager.cleanupAll(); + + verify(mongoClient.getDatabase(anyString())).drop(); + verify(mongoClient).close(); + } + + @Test + public void testCleanupAllShouldThrowErrorWhenMongoClientFailsToDropDatabase() { + MongoDatabase mockDatabase = mongoClient.getDatabase(anyString()); + doThrow(RuntimeException.class).when(mockDatabase).drop(); + + assertThrows(MongoDBResourceManagerException.class, () -> testManager.cleanupAll()); + } + + @Test + public void testCleanupAllShouldThrowErrorWhenMongoClientFailsToClose() { + doThrow(RuntimeException.class).when(mongoClient).close(); + + assertThrows(MongoDBResourceManagerException.class, () -> testManager.cleanupAll()); + } +} diff --git a/it/mongodb/src/test/java/org/apache/beam/it/mongodb/MongoDBResourceManagerUtilsTest.java b/it/mongodb/src/test/java/org/apache/beam/it/mongodb/MongoDBResourceManagerUtilsTest.java new file mode 100644 index 0000000000000..e0edcf2cafa6f --- /dev/null +++ b/it/mongodb/src/test/java/org/apache/beam/it/mongodb/MongoDBResourceManagerUtilsTest.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.mongodb; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.mongodb.MongoDBResourceManagerUtils.checkValidCollectionName; +import static org.apache.beam.it.mongodb.MongoDBResourceManagerUtils.generateDatabaseName; +import static org.junit.Assert.assertThrows; + +import org.apache.commons.lang3.StringUtils; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link MongoDBResourceManagerUtils}. */ +@RunWith(JUnit4.class) +public class MongoDBResourceManagerUtilsTest { + + @Test + public void testGenerateDatabaseNameShouldReplaceForwardSlash() { + String testBaseString = "Test/DB/Name"; + String actual = generateDatabaseName(testBaseString); + assertThat(actual).matches("test-db-name-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testGenerateDatabaseNameShouldReplaceBackwardSlash() { + String testBaseString = "Test\\DB\\Name"; + String actual = generateDatabaseName(testBaseString); + assertThat(actual).matches("test-db-name-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testGenerateDatabaseNameShouldReplacePeriod() { + String testBaseString = "Test.DB.Name"; + String actual = generateDatabaseName(testBaseString); + assertThat(actual).matches("test-db-name-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testGenerateDatabaseNameShouldReplaceSpace() { + String testBaseString = "Test DB Name"; + String actual = generateDatabaseName(testBaseString); + assertThat(actual).matches("test-db-name-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testGenerateDatabaseNameShouldReplaceDoubleQuotes() { + String testBaseString = "Test\"DB\"Name"; + String actual = generateDatabaseName(testBaseString); + assertThat(actual).matches("test-db-name-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testGenerateDatabaseNameShouldReplaceDollarSign() { + String testBaseString = "Test$DB$Name"; + String actual = generateDatabaseName(testBaseString); + assertThat(actual).matches("test-db-name-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testGenerateDatabaseNameShouldReplaceNullCharacter() { + String testBaseString = "Test\0DB\0Name"; + String actual = generateDatabaseName(testBaseString); + assertThat(actual).matches("test-db-name-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testCheckValidCollectionNameThrowsErrorWhenNameIsTooShort() { + assertThrows( + IllegalArgumentException.class, () -> checkValidCollectionName("test-database", "")); + } + + @Test + public void testCheckValidCollectionNameThrowsErrorWhenNameIsTooLong() { + assertThrows( + IllegalArgumentException.class, + () -> checkValidCollectionName(StringUtils.repeat("a", 1), StringUtils.repeat("b", 100))); + assertThrows( + IllegalArgumentException.class, + () -> checkValidCollectionName(StringUtils.repeat("a", 50), StringUtils.repeat("b", 50))); + assertThrows( + IllegalArgumentException.class, + () -> checkValidCollectionName(StringUtils.repeat("a", 100), StringUtils.repeat("b", 1))); + } + + @Test + public void testCheckValidCollectionNameThrowsErrorWhenNameContainsDollarSign() { + assertThrows( + IllegalArgumentException.class, + () -> checkValidCollectionName("test-database", "test$collection")); + } + + @Test + public void testCheckValidCollectionNameThrowsErrorWhenNameContainsNull() { + assertThrows( + IllegalArgumentException.class, + () -> checkValidCollectionName("test-database", "test\0collection")); + } + + @Test + public void testCheckValidCollectionNameThrowsErrorWhenNameBeginsWithSystemKeyword() { + assertThrows( + IllegalArgumentException.class, + () -> checkValidCollectionName("test-database", "system.test-collection")); + } + + @Test + public void testCheckValidCollectionNameThrowsErrorWhenNameDoesNotBeginWithLetterOrUnderscore() { + assertThrows( + IllegalArgumentException.class, + () -> checkValidCollectionName("test-database", "1test-collection")); + } + + @Test + public void testCheckValidCollectionNameDoesNotThrowErrorWhenNameIsValid() { + checkValidCollectionName("test-database", "a collection-name_valid.Test1"); + checkValidCollectionName("test-database", "_a collection-name_valid.Test1"); + } +} diff --git a/it/neo4j/build.gradle b/it/neo4j/build.gradle new file mode 100644 index 0000000000000..25650739646c3 --- /dev/null +++ b/it/neo4j/build.gradle @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.it.neo4j', +) + +description = "Apache Beam :: IT :: Neo4j" +ext.summary = "Integration test utilities for Neo4j." + +dependencies { + implementation project(path: ":it:testcontainers", configuration: "shadow") + implementation project(path: ":it:conditions", configuration: "shadow") + implementation library.java.testcontainers_neo4j + implementation "org.neo4j.driver:neo4j-java-driver:4.4.3" + + testImplementation library.java.mockito_core + // TODO: excluding Guava until Truth updates it to >32.1.x + testImplementation(library.java.truth) { + exclude group: 'com.google.guava', module: 'guava' + } + testRuntimeOnly library.java.slf4j_simple +} diff --git a/it/neo4j/src/main/java/org/apache/beam/it/neo4j/Neo4jResourceManager.java b/it/neo4j/src/main/java/org/apache/beam/it/neo4j/Neo4jResourceManager.java new file mode 100644 index 0000000000000..97bcca9e84b99 --- /dev/null +++ b/it/neo4j/src/main/java/org/apache/beam/it/neo4j/Neo4jResourceManager.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.neo4j; + +import static org.apache.beam.it.common.utils.ResourceManagerUtils.generatePassword; +import static org.apache.beam.it.neo4j.Neo4jResourceManagerUtils.generateDatabaseName; + +import com.google.common.annotations.VisibleForTesting; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; +import org.apache.beam.it.common.ResourceManager; +import org.apache.beam.it.testcontainers.TestContainerResourceManager; +import org.neo4j.driver.AuthTokens; +import org.neo4j.driver.Driver; +import org.neo4j.driver.GraphDatabase; +import org.neo4j.driver.Session; +import org.neo4j.driver.SessionConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Neo4jContainer; +import org.testcontainers.utility.DockerImageName; + +/** + * Client for managing Neo4j resources. + * + *

    The database name is formed using testId. The database name will be "{testId}-{ISO8601 time, + * microsecond precision}", with additional formatting. + * + *

    The class is thread-safe. + */ +public class Neo4jResourceManager extends TestContainerResourceManager> + implements ResourceManager { + + private static final Logger LOG = LoggerFactory.getLogger(Neo4jResourceManager.class); + + private static final String DEFAULT_NEO4J_CONTAINER_NAME = "neo4j"; + + // A list of available Neo4j Docker image tags can be found at + // https://hub.docker.com/_/neo4j/tags + private static final String DEFAULT_NEO4J_CONTAINER_TAG = "5-enterprise"; + + // 7687 is the default Bolt port that Neo4j is configured to listen on + private static final int NEO4J_BOLT_PORT = 7687; + + private final Driver neo4jDriver; + private final String databaseName; + private final String connectionString; + private final boolean usingStaticDatabase; + + private final String adminPassword; + + private Neo4jResourceManager(Neo4jResourceManager.Builder builder) { + this( + builder.driver, + new Neo4jContainer<>( + DockerImageName.parse(builder.containerImageName) + .withTag(builder.containerImageTag)) + .withEnv("NEO4J_ACCEPT_LICENSE_AGREEMENT", "yes") + .withAdminPassword(builder.adminPassword), + builder); + } + + @VisibleForTesting + @SuppressWarnings("nullness") + Neo4jResourceManager( + @Nullable Driver neo4jDriver, + Neo4jContainer container, + Neo4jResourceManager.Builder builder) { + super(container, builder); + + this.adminPassword = builder.adminPassword; + this.connectionString = + String.format("neo4j://%s:%d", this.getHost(), this.getPort(NEO4J_BOLT_PORT)); + this.neo4jDriver = + neo4jDriver == null + ? GraphDatabase.driver(connectionString, AuthTokens.basic("neo4j", this.adminPassword)) + : neo4jDriver; + this.usingStaticDatabase = builder.databaseName != null; + if (usingStaticDatabase) { + this.databaseName = builder.databaseName; + } else { + this.databaseName = generateDatabaseName(builder.testId); + createDatabase(databaseName); + } + } + + public static Neo4jResourceManager.Builder builder(String testId) { + return new Neo4jResourceManager.Builder(testId); + } + + /** Returns the URI connection string to the Neo4j Database. */ + public synchronized String getUri() { + return connectionString; + } + + public List> run(String query) { + return this.run(query, Collections.emptyMap()); + } + + public List> run(String query, Map parameters) { + try (Session session = + neo4jDriver.session(SessionConfig.builder().withDatabase(databaseName).build())) { + return session.run(query, parameters).list(record -> record.asMap()); + } catch (Exception e) { + throw new Neo4jResourceManagerException(String.format("Error running query %s.", query), e); + } + } + + /** + * Returns the name of the Database that this Neo4j manager will operate in. + * + * @return the name of the Neo4j Database. + */ + public synchronized String getDatabaseName() { + return databaseName; + } + + @Override + public synchronized void cleanupAll() { + LOG.info("Attempting to clean up Neo4j manager."); + + boolean producedError = false; + + // First, delete the database if it was not given as a static argument + try { + if (!usingStaticDatabase) { + dropDatabase(databaseName); + } + } catch (Exception e) { + LOG.error("Failed to delete Neo4j database {}.", databaseName, e); + producedError = true; + } + + // Next, try to close the Neo4j client connection + try { + neo4jDriver.close(); + } catch (Exception e) { + LOG.error("Failed to delete Neo4j client.", e); + producedError = true; + } + + // Throw Exception at the end if there were any errors + if (producedError) { + throw new Neo4jResourceManagerException( + "Failed to delete resources. Check above for errors."); + } + + super.cleanupAll(); + + LOG.info("Neo4j manager successfully cleaned up."); + } + + private void createDatabase(String databaseName) { + try (Session session = + neo4jDriver.session(SessionConfig.builder().withDatabase("system").build())) { + session.run("CREATE DATABASE $db", Collections.singletonMap("db", databaseName)).consume(); + } catch (Exception e) { + throw new Neo4jResourceManagerException( + String.format("Error dropping database %s.", databaseName), e); + } + } + + @VisibleForTesting + void dropDatabase(String databaseName) { + try (Session session = + neo4jDriver.session(SessionConfig.builder().withDatabase("system").build())) { + session.run("DROP DATABASE $db", Collections.singletonMap("db", databaseName)).consume(); + } catch (Exception e) { + throw new Neo4jResourceManagerException( + String.format("Error dropping database %s.", databaseName), e); + } + } + + public String getAdminPassword() { + return adminPassword; + } + + /** Builder for {@link Neo4jResourceManager}. */ + public static final class Builder + extends TestContainerResourceManager.Builder { + + private @Nullable String databaseName; + + private String adminPassword; + + private @Nullable Driver driver; + + private Builder(String testId) { + super(testId, DEFAULT_NEO4J_CONTAINER_NAME, DEFAULT_NEO4J_CONTAINER_TAG); + this.adminPassword = generatePassword(4, 10, 2, 2, 0, Collections.emptyList()); + this.databaseName = null; + this.driver = null; + } + + /** + * Sets the database name to that of a static database instance. Use this method only when + * attempting to operate on a pre-existing Neo4j database. + * + *

    Note: if a database name is set, and a static Neo4j server is being used + * (useStaticContainer() is also called on the builder), then a database will be created on the + * static server if it does not exist, and it will not be removed when cleanupAll() is called on + * the Neo4jResourceManager. + * + * @param databaseName The database name. + * @return this builder object with the database name set. + */ + public Builder setDatabaseName(String databaseName) { + this.databaseName = databaseName; + return this; + } + + public Builder setAdminPassword(String password) { + this.adminPassword = password; + return this; + } + + @VisibleForTesting + Builder setDriver(Driver driver) { + this.driver = driver; + return this; + } + + @Override + public Neo4jResourceManager build() { + return new Neo4jResourceManager(this); + } + } +} diff --git a/it/neo4j/src/main/java/org/apache/beam/it/neo4j/Neo4jResourceManagerException.java b/it/neo4j/src/main/java/org/apache/beam/it/neo4j/Neo4jResourceManagerException.java new file mode 100644 index 0000000000000..5c0e94d2b8be0 --- /dev/null +++ b/it/neo4j/src/main/java/org/apache/beam/it/neo4j/Neo4jResourceManagerException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.neo4j; + +/** Custom exception for {@link Neo4jResourceManager} implementations. */ +public class Neo4jResourceManagerException extends RuntimeException { + + public Neo4jResourceManagerException(String errorMessage, Throwable err) { + super(errorMessage, err); + } + + public Neo4jResourceManagerException(String errorMessage) { + super(errorMessage); + } +} diff --git a/it/neo4j/src/main/java/org/apache/beam/it/neo4j/Neo4jResourceManagerUtils.java b/it/neo4j/src/main/java/org/apache/beam/it/neo4j/Neo4jResourceManagerUtils.java new file mode 100644 index 0000000000000..7288265a1707d --- /dev/null +++ b/it/neo4j/src/main/java/org/apache/beam/it/neo4j/Neo4jResourceManagerUtils.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.neo4j; + +import static org.apache.beam.it.common.utils.ResourceManagerUtils.generateResourceId; + +import java.time.format.DateTimeFormatter; +import java.util.regex.Pattern; + +/** Utilities for {@link Neo4jResourceManager} implementations. */ +final class Neo4jResourceManagerUtils { + + // Neo4j Database naming restrictions can be found at + // https://neo4j.com/docs/cypher-manual/current/administration/databases/#administration-databases-create-database + private static final int MAX_DATABASE_NAME_LENGTH = 63; + static final Pattern ILLEGAL_DATABASE_NAME_CHARS = + Pattern.compile("^[^a-zA-Z]|^system|[^a-zA-Z0-9.-]+|[.-]$"); + private static final String REPLACE_DATABASE_NAME_CHAR = "-"; + private static final DateTimeFormatter TIME_FORMAT = + DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss-SSSSSS"); + + private Neo4jResourceManagerUtils() {} + + /** + * Generates a Neo4j database name from a given string. + * + * @param baseString The string to generate the name from. + * @return The database name string. + */ + static String generateDatabaseName(String baseString) { + return generateResourceId( + baseString, + ILLEGAL_DATABASE_NAME_CHARS, + REPLACE_DATABASE_NAME_CHAR, + MAX_DATABASE_NAME_LENGTH, + TIME_FORMAT); + } +} diff --git a/it/neo4j/src/main/java/org/apache/beam/it/neo4j/conditions/Neo4jQueryCheck.java b/it/neo4j/src/main/java/org/apache/beam/it/neo4j/conditions/Neo4jQueryCheck.java new file mode 100644 index 0000000000000..32e283edb72c1 --- /dev/null +++ b/it/neo4j/src/main/java/org/apache/beam/it/neo4j/conditions/Neo4jQueryCheck.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.neo4j.conditions; + +import com.google.auto.value.AutoValue; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.neo4j.Neo4jResourceManager; + +@AutoValue +public abstract class Neo4jQueryCheck extends ConditionCheck { + + abstract Neo4jResourceManager resourceManager(); + + abstract List> expectedResult(); + + abstract String query(); + + @Nullable + abstract Map parameters(); + + @Override + public String getDescription() { + return String.format( + "Neo4j check if query %s matches expected result %s", query(), expectedResult()); + } + + @Override + @SuppressWarnings("nullness") + protected CheckResult check() { + List> actualResult; + if (parameters() != null) { + actualResult = resourceManager().run(query(), parameters()); + } else { + actualResult = resourceManager().run(query()); + } + List> expectedResult = expectedResult(); + if (actualResult == null) { + return new CheckResult(expectedResult == null); + } + return new CheckResult( + actualResult.equals(expectedResult), + String.format("Expected %s to equal %s", actualResult, expectedResult)); + } + + public static Builder builder(Neo4jResourceManager resourceManager) { + return new AutoValue_Neo4jQueryCheck.Builder().setResourceManager(resourceManager); + } + + /** Builder for {@link Neo4jQueryCheck}. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setResourceManager(Neo4jResourceManager resourceManager); + + public abstract Builder setQuery(String query); + + public abstract Builder setParameters(Map parameters); + + public abstract Builder setExpectedResult(List> result); + + abstract Neo4jQueryCheck autoBuild(); + + public Neo4jQueryCheck build() { + return autoBuild(); + } + } +} diff --git a/it/neo4j/src/main/java/org/apache/beam/it/neo4j/conditions/package-info.java b/it/neo4j/src/main/java/org/apache/beam/it/neo4j/conditions/package-info.java new file mode 100644 index 0000000000000..213a41c3d7934 --- /dev/null +++ b/it/neo4j/src/main/java/org/apache/beam/it/neo4j/conditions/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing Neo4j runtime checks within integration tests. */ +package org.apache.beam.it.neo4j.conditions; diff --git a/it/neo4j/src/main/java/org/apache/beam/it/neo4j/package-info.java b/it/neo4j/src/main/java/org/apache/beam/it/neo4j/package-info.java new file mode 100644 index 0000000000000..6d913fbed1925 --- /dev/null +++ b/it/neo4j/src/main/java/org/apache/beam/it/neo4j/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing Neo4j resources within integration tests. */ +package org.apache.beam.it.neo4j; diff --git a/it/neo4j/src/test/java/org/apache/beam/it/neo4j/Neo4jResourceManagerIT.java b/it/neo4j/src/test/java/org/apache/beam/it/neo4j/Neo4jResourceManagerIT.java new file mode 100644 index 0000000000000..c1038c2e12c95 --- /dev/null +++ b/it/neo4j/src/test/java/org/apache/beam/it/neo4j/Neo4jResourceManagerIT.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.neo4j; + +import static com.google.common.truth.Truth.assertThat; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.testcontainers.TestContainersIntegrationTest; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Integration tests for {@link Neo4jResourceManager}. */ +@Category(TestContainersIntegrationTest.class) +@RunWith(JUnit4.class) +public class Neo4jResourceManagerIT { + + private Neo4jResourceManager neo4jResourceManager; + + @Before + public void setUp() { + neo4jResourceManager = + Neo4jResourceManager.builder("placeholder") + .setDatabaseName("neo4j") + .setAdminPassword("password") + .build(); + } + + @Test + public void testResourceManagerE2E() { + neo4jResourceManager.run( + "CREATE (:Hello {whom: $whom})", Collections.singletonMap("whom", "world")); + + List> results = + neo4jResourceManager.run("MATCH (h:Hello) RETURN h.whom AS whom"); + + assertThat(results).hasSize(1); + assertThat(results) + .containsExactlyElementsIn( + Collections.singletonList(Collections.singletonMap("whom", "world"))); + } + + @After + public void tearDown() { + ResourceManagerUtils.cleanResources(neo4jResourceManager); + } +} diff --git a/it/neo4j/src/test/java/org/apache/beam/it/neo4j/Neo4jResourceManagerTest.java b/it/neo4j/src/test/java/org/apache/beam/it/neo4j/Neo4jResourceManagerTest.java new file mode 100644 index 0000000000000..5014505ece9c1 --- /dev/null +++ b/it/neo4j/src/test/java/org/apache/beam/it/neo4j/Neo4jResourceManagerTest.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.neo4j; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.startsWith; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Collections; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.neo4j.driver.Driver; +import org.neo4j.driver.Result; +import org.neo4j.driver.Session; +import org.neo4j.driver.exceptions.ClientException; +import org.testcontainers.containers.Neo4jContainer; + +/** Unit tests for {@link Neo4jResourceManager}. */ +@RunWith(JUnit4.class) +public class Neo4jResourceManagerTest { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock private Driver neo4jDriver; + @Mock private Session session; + @Mock private Result result; + @Mock private Neo4jContainer container; + + private static final String TEST_ID = "test-id"; + private static final String STATIC_DATABASE_NAME = "database"; + private static final String HOST = "localhost"; + private static final int NEO4J_BOLT_PORT = 7687; + private static final int MAPPED_PORT = 10000; + + private Neo4jResourceManager testManager; + + @Before + public void setUp() { + when(container.getMappedPort(NEO4J_BOLT_PORT)).thenReturn(MAPPED_PORT); + when(session.run(anyString(), anyMap())).thenReturn(result); + when(neo4jDriver.session(any())).thenReturn(session); + doReturn(container).when(container).withLogConsumer(any()); + + testManager = + new Neo4jResourceManager(neo4jDriver, container, Neo4jResourceManager.builder(TEST_ID)); + } + + @Test + public void testCreateResourceManagerBuilderReturnsDefaultNeo4jResourceManager() { + assertThat( + Neo4jResourceManager.builder(TEST_ID) + .setAdminPassword("letmein!") + .setDriver(neo4jDriver) + .useStaticContainer() + .setHost(HOST) + .setPort(NEO4J_BOLT_PORT) + .build()) + .isInstanceOf(Neo4jResourceManager.class); + } + + @Test + public void testGetUriShouldReturnCorrectValue() { + assertThat(testManager.getUri()).matches("neo4j://" + HOST + ":" + MAPPED_PORT); + } + + @Test + public void testGetDatabaseNameShouldReturnCorrectValue() { + assertThat(testManager.getDatabaseName()).matches(TEST_ID + "-\\d{8}-\\d{6}-\\d{6}"); + } + + @Test + public void testDropDatabaseShouldThrowErrorIfDriverFailsToRunQuery() { + doThrow(ClientException.class).when(session).run(anyString(), anyMap()); + + assertThrows( + Neo4jResourceManagerException.class, () -> testManager.dropDatabase(STATIC_DATABASE_NAME)); + } + + @Test + public void testRunShouldThrowErrorIfDriverFailsToRunParameterlessQuery() { + doThrow(ClientException.class).when(session).run(anyString(), anyMap()); + + assertThrows( + Neo4jResourceManagerException.class, () -> testManager.run("MATCH (n) RETURN n LIMIT 1")); + } + + @Test + public void testRunShouldThrowErrorIfDriverFailsToRunQuery() { + doThrow(ClientException.class).when(session).run(anyString(), anyMap()); + + assertThrows( + Neo4jResourceManagerException.class, + () -> + testManager.run( + "MATCH (n) WHERE n < $val RETURN n LIMIT 1", Collections.singletonMap("val", 2))); + } + + @Test + public void testCleanupAllShouldNotDropStaticDatabase() { + Neo4jResourceManager.Builder builder = + Neo4jResourceManager.builder(TEST_ID).setDatabaseName(STATIC_DATABASE_NAME); + Neo4jResourceManager tm = new Neo4jResourceManager(neo4jDriver, container, builder); + + tm.cleanupAll(); + + verify(session, never()).run(startsWith("DROP DATABASE"), anyMap()); + verify(neo4jDriver).close(); + } + + @Test + public void testCleanupShouldDropNonStaticDatabase() { + when(session.run(anyString(), anyMap())).thenReturn(mock(Result.class)); + + testManager.cleanupAll(); + + verify(session).run(startsWith("DROP DATABASE"), anyMap()); + verify(neo4jDriver).close(); + } + + @Test + public void testCleanupAllShouldThrowErrorWhenNeo4jDriverFailsToDropDatabase() { + doThrow(ClientException.class).when(session).run(anyString(), anyMap()); + + assertThrows(Neo4jResourceManagerException.class, () -> testManager.cleanupAll()); + } + + @Test + public void testCleanupAllShouldThrowErrorWhenNeo4jDriverFailsToClose() { + doThrow(RuntimeException.class).when(neo4jDriver).close(); + + assertThrows(Neo4jResourceManagerException.class, () -> testManager.cleanupAll()); + } +} diff --git a/it/neo4j/src/test/java/org/apache/beam/it/neo4j/Neo4jResourceManagerUtilsTest.java b/it/neo4j/src/test/java/org/apache/beam/it/neo4j/Neo4jResourceManagerUtilsTest.java new file mode 100644 index 0000000000000..b3d37408f88b9 --- /dev/null +++ b/it/neo4j/src/test/java/org/apache/beam/it/neo4j/Neo4jResourceManagerUtilsTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.neo4j; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.neo4j.Neo4jResourceManagerUtils.ILLEGAL_DATABASE_NAME_CHARS; + +import org.junit.Test; + +public class Neo4jResourceManagerUtilsTest { + + @Test + public void excludesInvalidDatabaseNames() { + assertThat(isValid("1db")).isFalse(); + assertThat(isValid(".db")).isFalse(); + assertThat(isValid("_db")).isFalse(); + assertThat(isValid(" db")).isFalse(); + assertThat(isValid("myDb.")).isFalse(); + assertThat(isValid("myDb-")).isFalse(); + assertThat(isValid("_internal")).isFalse(); + assertThat(isValid("system-of-a-db")).isFalse(); + assertThat(isValid("my_db")).isFalse(); + } + + @Test + public void doesNotExcludeValidDatabaseNames() { + assertThat(isValid("db1")).isTrue(); + assertThat(isValid("my-db")).isTrue(); + assertThat(isValid("a.database")).isTrue(); + assertThat(isValid("my-db2")).isTrue(); + assertThat(isValid("a.gr8.database")).isTrue(); + } + + private static boolean isValid(String name) { + return !ILLEGAL_DATABASE_NAME_CHARS.matcher(name).find(); + } +} diff --git a/it/splunk/build.gradle b/it/splunk/build.gradle new file mode 100644 index 0000000000000..deacc1051b440 --- /dev/null +++ b/it/splunk/build.gradle @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.it.splunk', +) + +description = "Apache Beam :: IT :: Splunk" +ext.summary = "Integration test utilities for Splunk." + +repositories { + mavenCentral() + maven { + url "https://splunk.jfrog.io/splunk/ext-releases-local" + } +} + +dependencies { + implementation project(path: ":it:testcontainers", configuration: "shadow") + implementation project(path: ":it:conditions", configuration: "shadow") + implementation project(path: ":it:truthmatchers", configuration: "shadow") + implementation project(path: ":sdks:java:io:splunk") + implementation 'com.splunk:splunk:1.9.4' + implementation 'org.awaitility:awaitility:4.2.0' + provided library.java.json_org + + testImplementation library.java.mockito_core + // TODO: excluding Guava until Truth updates it to >32.1.x + testImplementation(library.java.truth) { + exclude group: 'com.google.guava', module: 'guava' + } + testImplementation library.java.commons_lang3 + testRuntimeOnly library.java.slf4j_simple +} \ No newline at end of file diff --git a/it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkClientFactory.java b/it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkClientFactory.java new file mode 100644 index 0000000000000..086e9c9f96566 --- /dev/null +++ b/it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkClientFactory.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.splunk; + +import com.splunk.Service; +import com.splunk.ServiceArgs; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; + +/** Splunk Driver Factory class. */ +class SplunkClientFactory { + SplunkClientFactory() {} + + /** + * Returns an HTTP client that is used to send HTTP messages to a Splunk server instance with HEC. + * + * @return An HTTP client for sending HTTP messages to Splunk HEC. + */ + CloseableHttpClient getHttpClient() { + return HttpClientBuilder.create().build(); + } + + /** + * Returns a Splunk Service client for sending requests to a Splunk server instance. + * + * @param serviceArgs the service arguments to connect to the server with. + * @return A Splunk service client to retrieve messages from a Splunk server instance. + */ + Service getServiceClient(ServiceArgs serviceArgs) { + return Service.connect(serviceArgs); + } +} diff --git a/it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkContainer.java b/it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkContainer.java new file mode 100644 index 0000000000000..68950a2628a29 --- /dev/null +++ b/it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkContainer.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.splunk; + +import java.time.Duration; +import java.util.Collections; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.builder.Transferable; +import org.testcontainers.utility.DockerImageName; + +/** + * Constructs a Splunk container. + * + *

    Tested on a Splunk version 8.2. + * + *

    More information about docker-splunk can be found here: + * + *

    https://splunk.github.io/docker-splunk/ + */ +public class SplunkContainer extends GenericContainer { + + /** Splunk Default HTTP port. */ + private static final int SPLUNK_INTERNAL_PORT = 8000; + + /** Splunk Default HTTP Event Collector (HEC) port. */ + private static final int SPLUNK_HEC_INTERNAL_PORT = 8088; + + private static final int SPLUNKD_INTERNAL_PORT = 8089; + + /** Splunk Docker base image. */ + private static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("splunk/splunk"); + + private static final String DEFAULTS_FILE_PATH = "/tmp/defaults/default.yml"; + + public SplunkContainer(@NonNull String dockerImageName) { + this(DockerImageName.parse(dockerImageName)); + } + + @SuppressWarnings("nullness") + public SplunkContainer(DockerImageName dockerImageName) { + super(dockerImageName); + dockerImageName.assertCompatibleWith(DEFAULT_IMAGE_NAME); + + this.withExposedPorts(SPLUNK_INTERNAL_PORT, SPLUNK_HEC_INTERNAL_PORT, SPLUNKD_INTERNAL_PORT); + this.withEnv(Collections.singletonMap("SPLUNK_START_ARGS", "--accept-license")); + this.waitingFor( + Wait.forLogMessage("(?i).*Ansible playbook complete.*", 1) + .withStartupTimeout(Duration.ofMinutes(3))); + } + + /** + * Define the Splunk password to set. + * + * @param password Password to set + * @return this + */ + public SplunkContainer withPassword(String password) { + this.withEnv("SPLUNK_PASSWORD", password); + return this; + } + + /** + * Define the Splunk HTTP Event Collector (HEC) token to set. + * + * @param hecToken Token to set + * @return this + */ + public SplunkContainer withHecToken(String hecToken) { + this.withEnv("SPLUNK_HEC_TOKEN", hecToken); + return this; + } + + /** + * Define whether SSL will be used for connecting to the Splunk server. + * + * @return this + */ + public SplunkContainer withSplunkdSslDisabled() { + this.withEnv("SPLUNKD_SSL_ENABLE", "false"); + return this; + } + + /** + * Define a defaults file to use for configuring the Splunk server. + * + *

    More information about the defaults file can be found here: + * + *

    https://splunk.github.io/docker-splunk/ADVANCED.html#runtime-configuration + * + * @param defaults A Splunk defaults file to copy to container. + * @return this + */ + public SplunkContainer withDefaultsFile(Transferable defaults) { + this.withCopyToContainer(defaults, DEFAULTS_FILE_PATH); + return this; + } + + // TODO - Future config environment variables that may be useful to add + // SPLUNK_S2S_PORT + // SPLUNK_SVC_PORT + // SPLUNK_SECRET + // SPLUNKD_SSL_CERT + // SPLUNKD_SSL_CA + // SPLUNKD_SSL_PASSWORD +} diff --git a/it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkResourceManager.java b/it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkResourceManager.java new file mode 100644 index 0000000000000..1ef4726df43aa --- /dev/null +++ b/it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkResourceManager.java @@ -0,0 +1,440 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.splunk; + +import static org.apache.beam.it.splunk.SplunkResourceManagerUtils.generateHecToken; +import static org.apache.beam.it.splunk.SplunkResourceManagerUtils.generateSplunkPassword; +import static org.apache.beam.it.splunk.SplunkResourceManagerUtils.splunkEventToMap; + +import com.splunk.Job; +import com.splunk.ResultsReader; +import com.splunk.ResultsReaderXml; +import com.splunk.ServiceArgs; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.time.Duration; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.beam.it.common.ResourceManager; +import org.apache.beam.it.testcontainers.TestContainerResourceManager; +import org.apache.beam.sdk.io.splunk.SplunkEvent; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.awaitility.Awaitility; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.images.builder.Transferable; +import org.testcontainers.utility.DockerImageName; + +/** + * Client for managing Kafka resources. + * + *

    The class supports one Splunk server instance. + * + *

    The class is thread-safe. + * + *

    Note: The Splunk TestContainer will only run on M1 Mac's if the Docker version is >= 4.16.0 + * and the "Use Rosetta for x86/amd64 emulation on Apple Silicon" setting is enabled. + */ +public class SplunkResourceManager extends TestContainerResourceManager + implements ResourceManager { + + private static final Logger LOG = LoggerFactory.getLogger(SplunkResourceManager.class); + private static final String DEFAULT_SPLUNK_CONTAINER_NAME = "splunk/splunk"; + + // A list of available Splunk Docker image tags can be found at + // https://hub.docker.com/r/splunk/splunk/tags + private static final String DEFAULT_SPLUNK_CONTAINER_TAG = "8.2"; + + // 8088 is the default port that Splunk HEC is configured to listen on + private static final int DEFAULT_SPLUNK_HEC_INTERNAL_PORT = 8088; + // 8089 is the default port that Splunkd is configured to listen on + private static final int DEFAULT_SPLUNKD_INTERNAL_PORT = 8089; + + private static final String DEFAULT_SPLUNK_USERNAME = "admin"; + + private final ServiceArgs loginArgs; + private final int hecPort; + private final String hecScheme; + private final String hecToken; + + private final SplunkClientFactory clientFactory; + + @SuppressWarnings("resource") + private SplunkResourceManager(SplunkResourceManager.Builder builder) { + this( + new SplunkClientFactory(), + new SplunkContainer( + DockerImageName.parse(builder.containerImageName) + .withTag(builder.containerImageTag)) + .withSplunkdSslDisabled(), + builder); + } + + @VisibleForTesting + @SuppressWarnings("nullness") + SplunkResourceManager( + SplunkClientFactory clientFactory, + SplunkContainer container, + SplunkResourceManager.Builder builder) { + super(setup(container, builder), builder); + + String username = DEFAULT_SPLUNK_USERNAME; + if (builder.useStaticContainer && builder.username != null) { + username = builder.username; + } + hecPort = + builder.useStaticContainer ? builder.hecPort : container.getMappedPort(builder.hecPort); + int splunkdPort = + builder.useStaticContainer + ? builder.splunkdPort + : container.getMappedPort(builder.splunkdPort); + + // TODO - add support for https scheme + String splunkdScheme = "http"; + hecScheme = "http"; + + hecToken = builder.hecToken; + + // Create Splunk service login args + this.loginArgs = new ServiceArgs(); + this.loginArgs.setPort(splunkdPort); + this.loginArgs.setHost(this.getHost()); + this.loginArgs.setScheme(splunkdScheme); + this.loginArgs.setUsername(username); + this.loginArgs.setPassword(builder.password); + + // Initialize the clients + this.clientFactory = clientFactory; + } + + /** + * Helper method for injecting config information from the builder into the given SplunkContainer. + * + * @param container The SplunkContainer before config info is injected. + * @param builder The SplunkResourceManager.Builder to extract config info from. + * @return The SplunkContainer with all config info injected. + */ + @SuppressWarnings("nullness") + private static SplunkContainer setup(SplunkContainer container, Builder builder) { + // Validate builder args + if (builder.useStaticContainer) { + if (builder.hecPort < 0 || builder.splunkdPort < 0) { + throw new SplunkResourceManagerException( + "This manager was configured to use a static resource, but the hecPort and splunkdPort were not properly set."); + } + } + + builder.hecPort = builder.hecPort < 0 ? DEFAULT_SPLUNK_HEC_INTERNAL_PORT : builder.hecPort; + builder.splunkdPort = + builder.splunkdPort < 0 ? DEFAULT_SPLUNKD_INTERNAL_PORT : builder.splunkdPort; + builder.hecToken = builder.hecToken == null ? generateHecToken() : builder.hecToken; + builder.password = builder.password == null ? generateSplunkPassword() : builder.password; + // TODO - add support for ssl + return container + .withDefaultsFile( + Transferable.of( + String.format( + "splunk:%n" + + " hec:%n" + + " enable: true%n" + + " ssl: %b%n" + + " port: %s%n" + + " token: %s", + false, builder.hecPort, builder.hecToken))) + .withPassword(builder.password); + } + + public static SplunkResourceManager.Builder builder(String testId) { + return new SplunkResourceManager.Builder(testId); + } + + /** + * Returns the HTTP endpoint that this Splunk server is configured to listen on. + * + * @return the HTTP endpoint. + */ + public String getHttpEndpoint() { + return String.format("%s://%s:%d", hecScheme, getHost(), hecPort); + } + + /** + * Returns the HTTP Event Collector (HEC) endpoint that this Splunk server is configured to + * receive events at. + * + *

    This will be the HTTP endpoint concatenated with '/services/collector/event'. + * + * @return the HEC service endpoint. + */ + public String getHecEndpoint() { + return getHttpEndpoint() + "/services/collector/event"; + } + + /** + * Returns the Splunk Http Event Collector (HEC) authentication token used to connect to this + * Splunk instance's HEC service. + * + * @return the HEC authentication token. + */ + public String getHecToken() { + return hecToken; + } + + /** + * Helper method for converting the given SplunkEvent into JSON format. + * + * @param event The SplunkEvent to parse. + * @return JSON String. + */ + private static String splunkEventToJson(SplunkEvent event) { + return new JSONObject(splunkEventToMap(event)).toString(); + } + + /** + * Sends the given HTTP event to the Splunk Http Event Collector (HEC) service. + * + *

    Note: Setting the index field in the Splunk event requires the index already + * being configured in the Splunk instance. Unless using a static Splunk instance, omit this field + * from the event. + * + * @param event The SpunkEvent to send to the HEC service. + * @return True, if the request was successful. + */ + public synchronized boolean sendHttpEvent(SplunkEvent event) { + return sendHttpEvents(Collections.singletonList(event)); + } + + /** + * Sends the given HTTP events to the Splunk Http Event Collector (HEC) service. + * + *

    Note: Setting the index field in the Splunk event requires the index already + * being configured in the Splunk instance. Unless using a static Splunk instance, omit this field + * from the events. + * + * @param events The SpunkEvents to send to the HEC service. + * @return True, if the request was successful. + */ + public synchronized boolean sendHttpEvents(Collection events) { + + LOG.info("Attempting to send {} events to {}.", events.size(), getHecEndpoint()); + + // Construct base HEC request + HttpPost httppost = new HttpPost(getHecEndpoint()); + httppost.addHeader("Authorization", "Splunk " + hecToken); + + // Loop over events and send one-by-one + StringBuilder eventsData = new StringBuilder(); + events.forEach( + event -> { + String eventStr = splunkEventToJson(event); + eventsData.append(eventStr); + LOG.info("Sending HTTP event: {}", eventStr); + }); + + try (CloseableHttpClient httpClient = clientFactory.getHttpClient()) { + // Set request data + try { + httppost.setEntity(new StringEntity(eventsData.toString())); + } catch (UnsupportedEncodingException e) { + throw new SplunkResourceManagerException( + "Error setting HTTP message data to " + eventsData, e); + } + + // Send request + try (CloseableHttpResponse response = httpClient.execute(httppost)) { + // Check error code + int code = response.getStatusLine().getStatusCode(); + if (code != 200) { + throw new SplunkResourceManagerException( + "Received http error code " + code + " sending event."); + } + } catch (Exception e) { + throw new SplunkResourceManagerException("Error sending event.", e); + } + } catch (IOException e) { + throw new SplunkResourceManagerException("Error with HTTP client.", e); + } + + LOG.info("Successfully sent {} events.", events.size()); + + return true; + } + + /** + * Return a list of all Splunk events retrieved from the Splunk server. + * + * @return All Splunk events on the server. + */ + public synchronized List getEvents() { + return getEvents("search"); + } + + /** + * Return a list of Splunk events retrieved from the Splunk server based on the given query. + * + *

    e.g. query: 'search source=mySource sourcetype=mySourceType host=myHost' + * + * @param query The query to filter events by. + * @return All Splunk events on the server that match the given query. + */ + public synchronized List getEvents(String query) { + LOG.info("Reading events from Splunk using query: {}.", query); + + // Run a simple search by first creating the search job + Job job = clientFactory.getServiceClient(loginArgs).getJobs().create(query); + + // Wait up to 1 minute for search results to be ready + Awaitility.await("Retrieving events from Splunk") + .atMost(Duration.ofMinutes(1)) + .pollInterval(Duration.ofMillis(500)) + .until(job::isDone); + + // Read results + List results = new ArrayList<>(); + try { + ResultsReader reader = new ResultsReaderXml(job.getEvents()); + reader.forEach( + event -> + results.add( + SplunkEvent.newBuilder() + .withEvent(event.get("_raw")) + .withSource(event.get("source")) + .withSourceType(event.get("_sourcetype")) + .withHost(event.get("host")) + .withTime( + OffsetDateTime.parse( + event.get("_time"), DateTimeFormatter.ISO_OFFSET_DATE_TIME) + .toInstant() + .toEpochMilli()) + .withIndex(event.get("index")) + .create())); + + } catch (Exception e) { + throw new SplunkResourceManagerException("Error parsing XML results from Splunk.", e); + } + + LOG.info("Successfully retrieved {} events.", results.size()); + return results; + } + + /** Builder for {@link SplunkResourceManager}. */ + public static final class Builder + extends TestContainerResourceManager.Builder { + + private @Nullable String username; + private @Nullable String password; + private @Nullable String hecToken; + private int hecPort; + private int splunkdPort; + + private Builder(String testId) { + super(testId, DEFAULT_SPLUNK_CONTAINER_NAME, DEFAULT_SPLUNK_CONTAINER_TAG); + this.username = null; + this.password = null; + this.hecToken = null; + this.hecPort = -1; + this.splunkdPort = -1; + } + + /** + * Set the username used to connect to a static Splunk instance. + * + *

    Note: This method should only be used if {@code useStaticContainer()} is also called. + * + * @param username the username for the Splunk instance. + * @return this builder with the username manually set. + */ + public Builder setUsername(String username) { + this.username = username; + return this; + } + + /** + * Manually set the Splunk password to the given password. This password will be used by the + * resource manager to authenticate with Splunk. + * + * @param password the password for the Splunk instance. + * @return this builder with the password manually set. + */ + public Builder setPassword(String password) { + this.password = password; + return this; + } + + /** + * Manually set the Splunk HTTP Event Collector (HEC) token to the given token. This token will + * be used by the resource manager to authenticate with Splunk. + * + * @param hecToken the HEC token for the Splunk instance. + * @return this builder with the HEC token manually set. + */ + public Builder setHecToken(String hecToken) { + this.hecToken = hecToken; + return this; + } + + @Override + public Builder setHost(String containerHost) { + super.setHost(containerHost); + this.port = 0; + return this; + } + + @Override + public Builder setPort(int port) { + throw new UnsupportedOperationException( + "Please use setHecPort() and setSplunkdPort() instead."); + } + + /** + * Sets the port that the Splunk Http Event Collector (HEC) service is hosted on. + * + * @param port the port hosting the HEC service. + * @return this builder object with the HEC port set. + */ + public Builder setHecPort(int port) { + this.hecPort = port; + return this; + } + + /** + * Sets the port that the Splunkd service is hosted on. + * + * @param port the port hosting the Splunkd service. + * @return this builder object with the Splunkd port set. + */ + public Builder setSplunkdPort(int port) { + this.splunkdPort = port; + return this; + } + + @Override + public SplunkResourceManager build() { + return new SplunkResourceManager(this); + } + } +} diff --git a/it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkResourceManagerException.java b/it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkResourceManagerException.java new file mode 100644 index 0000000000000..3e0b57ffd1d85 --- /dev/null +++ b/it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkResourceManagerException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.splunk; + +/** Custom exception for {@link SplunkResourceManager} implementations. */ +public class SplunkResourceManagerException extends RuntimeException { + + public SplunkResourceManagerException(String errorMessage, Throwable err) { + super(errorMessage, err); + } + + public SplunkResourceManagerException(String errorMessage) { + super(errorMessage); + } +} diff --git a/it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkResourceManagerUtils.java b/it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkResourceManagerUtils.java new file mode 100644 index 0000000000000..dbf70d5481c03 --- /dev/null +++ b/it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkResourceManagerUtils.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.splunk; + +import static org.apache.beam.it.common.utils.ResourceManagerUtils.generatePassword; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import org.apache.beam.sdk.io.splunk.SplunkEvent; + +/** Utilities for {@link SplunkResourceManager} implementations. */ +public final class SplunkResourceManagerUtils { + + static final String DEFAULT_SPLUNK_INDEX = "main"; + + // Splunk event metadata keys + private static final String SPLUNK_EVENT_KEY = "event"; + private static final String SPLUNK_HOST_KEY = "host"; + private static final String SPLUNK_INDEX_KEY = "index"; + private static final String SPLUNK_TIME_KEY = "time"; + private static final String SPLUNK_SOURCE_KEY = "source"; + private static final String SPLUNK_SOURCE_TYPE_KEY = "sourcetype"; + + private static final int MIN_PASSWORD_LENGTH = 8; + private static final int MAX_PASSWORD_LENGTH = 20; + + private SplunkResourceManagerUtils() {} + + @SuppressWarnings("nullness") + public static Map splunkEventToMap(SplunkEvent event) { + Map eventMap = new HashMap<>(); + eventMap.put(SPLUNK_EVENT_KEY, event.event()); + eventMap.put(SPLUNK_HOST_KEY, event.host()); + eventMap.put(SPLUNK_INDEX_KEY, event.index() != null ? event.index() : DEFAULT_SPLUNK_INDEX); + eventMap.put(SPLUNK_SOURCE_KEY, event.source()); + eventMap.put(SPLUNK_SOURCE_TYPE_KEY, event.sourceType()); + eventMap.put(SPLUNK_TIME_KEY, event.time()); + + return eventMap; + } + + /** + * Generates a secure, valid Splunk password. + * + * @return The generated password. + */ + static String generateSplunkPassword() { + int numLower = 2; + int numUpper = 2; + int numSpecial = 0; + return generatePassword( + MIN_PASSWORD_LENGTH, + MAX_PASSWORD_LENGTH, + numLower, + numUpper, + numSpecial, + /* specialChars= */ null); + } + + /** + * Generates a secure, valid Splunk HEC authentication token. + * + * @return The generated token. + */ + static String generateHecToken() { + return UUID.randomUUID().toString(); + } +} diff --git a/it/splunk/src/main/java/org/apache/beam/it/splunk/conditions/SplunkEventsCheck.java b/it/splunk/src/main/java/org/apache/beam/it/splunk/conditions/SplunkEventsCheck.java new file mode 100644 index 0000000000000..e9ae4f68c90f7 --- /dev/null +++ b/it/splunk/src/main/java/org/apache/beam/it/splunk/conditions/SplunkEventsCheck.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.splunk.conditions; + +import com.google.auto.value.AutoValue; +import javax.annotation.Nullable; +import org.apache.beam.it.conditions.ConditionCheck; +import org.apache.beam.it.splunk.SplunkResourceManager; + +/** ConditionCheck to validate if Splunk has received a certain amount of events. */ +@AutoValue +public abstract class SplunkEventsCheck extends ConditionCheck { + + abstract SplunkResourceManager resourceManager(); + + @Nullable + abstract String query(); + + abstract Integer minEvents(); + + @Nullable + abstract Integer maxEvents(); + + @Override + public String getDescription() { + if (maxEvents() != null) { + return String.format( + "Splunk check if logs have between %d and %d events", minEvents(), maxEvents()); + } + return String.format("Splunk check if logs have %d events", minEvents()); + } + + @Override + @SuppressWarnings("nullness") + public CheckResult check() { + long totalEvents; + if (query() != null) { + totalEvents = resourceManager().getEvents(query()).size(); + } else { + totalEvents = resourceManager().getEvents().size(); + } + if (totalEvents < minEvents()) { + return new CheckResult( + false, String.format("Expected %d but has only %d", minEvents(), totalEvents)); + } + if (maxEvents() != null && totalEvents > maxEvents()) { + return new CheckResult( + false, String.format("Expected up to %d but found %d events", maxEvents(), totalEvents)); + } + + if (maxEvents() != null) { + return new CheckResult( + true, + String.format( + "Expected between %d and %d events and found %d", + minEvents(), maxEvents(), totalEvents)); + } + + return new CheckResult( + true, String.format("Expected at least %d events and found %d", minEvents(), totalEvents)); + } + + public static Builder builder(SplunkResourceManager resourceManager) { + return new AutoValue_SplunkEventsCheck.Builder().setResourceManager(resourceManager); + } + + /** Builder for {@link SplunkEventsCheck}. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setResourceManager(SplunkResourceManager resourceManager); + + public abstract Builder setQuery(String query); + + public abstract Builder setMinEvents(Integer minEvents); + + public abstract Builder setMaxEvents(Integer maxEvents); + + abstract SplunkEventsCheck autoBuild(); + + public SplunkEventsCheck build() { + return autoBuild(); + } + } +} diff --git a/it/splunk/src/main/java/org/apache/beam/it/splunk/conditions/package-info.java b/it/splunk/src/main/java/org/apache/beam/it/splunk/conditions/package-info.java new file mode 100644 index 0000000000000..d4ae6db311a00 --- /dev/null +++ b/it/splunk/src/main/java/org/apache/beam/it/splunk/conditions/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package that contains reusable Splunk conditions. */ +package org.apache.beam.it.splunk.conditions; diff --git a/it/splunk/src/main/java/org/apache/beam/it/splunk/matchers/SplunkAsserts.java b/it/splunk/src/main/java/org/apache/beam/it/splunk/matchers/SplunkAsserts.java new file mode 100644 index 0000000000000..62a11f5ab7c62 --- /dev/null +++ b/it/splunk/src/main/java/org/apache/beam/it/splunk/matchers/SplunkAsserts.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.splunk.matchers; + +import static org.apache.beam.it.splunk.SplunkResourceManagerUtils.splunkEventToMap; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatRecords; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.apache.beam.it.truthmatchers.RecordsSubject; +import org.apache.beam.sdk.io.splunk.SplunkEvent; + +/** Assert utilities for Splunk tests. */ +public class SplunkAsserts { + + /** + * Convert Splunk {@link SplunkEvent} to a list of maps. + * + * @param events List of SplunkEvents to parse + * @return List of maps to use in {@link RecordsSubject} + */ + public static List> splunkEventsToRecords(Collection events) { + try { + List> records = new ArrayList<>(); + + for (SplunkEvent event : events) { + Map converted = splunkEventToMap(event); + records.add(converted); + } + + return records; + } catch (Exception e) { + throw new RuntimeException("Error converting SplunkEvents to Records", e); + } + } + + /** + * Creates a {@link RecordsSubject} to assert information within a list of records. + * + * @param events List of SplunkEvents in Splunk {@link SplunkEvent} format to use in the + * comparison. + * @return Truth subject to chain assertions. + */ + public static RecordsSubject assertThatSplunkEvents(Collection events) { + return assertThatRecords(splunkEventsToRecords(events)); + } +} diff --git a/it/splunk/src/main/java/org/apache/beam/it/splunk/matchers/package-info.java b/it/splunk/src/main/java/org/apache/beam/it/splunk/matchers/package-info.java new file mode 100644 index 0000000000000..f56eff15599bb --- /dev/null +++ b/it/splunk/src/main/java/org/apache/beam/it/splunk/matchers/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for Splunk Truth matchers / subjects to have reusable assertions. */ +package org.apache.beam.it.splunk.matchers; diff --git a/it/splunk/src/main/java/org/apache/beam/it/splunk/package-info.java b/it/splunk/src/main/java/org/apache/beam/it/splunk/package-info.java new file mode 100644 index 0000000000000..6097113cd7050 --- /dev/null +++ b/it/splunk/src/main/java/org/apache/beam/it/splunk/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing Splunk resources within integration tests. */ +package org.apache.beam.it.splunk; diff --git a/it/splunk/src/test/java/org/apache/beam/it/splunk/SplunkResourceManagerIT.java b/it/splunk/src/test/java/org/apache/beam/it/splunk/SplunkResourceManagerIT.java new file mode 100644 index 0000000000000..d11f01a70b0ab --- /dev/null +++ b/it/splunk/src/test/java/org/apache/beam/it/splunk/SplunkResourceManagerIT.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.splunk; + +import static org.apache.beam.it.splunk.matchers.SplunkAsserts.assertThatSplunkEvents; +import static org.apache.beam.it.splunk.matchers.SplunkAsserts.splunkEventsToRecords; +import static org.awaitility.Awaitility.await; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import org.apache.beam.it.splunk.conditions.SplunkEventsCheck; +import org.apache.beam.it.testcontainers.TestContainersIntegrationTest; +import org.apache.beam.sdk.io.splunk.SplunkEvent; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Integration tests for Splunk Resource Managers. */ +@Category(TestContainersIntegrationTest.class) +@RunWith(JUnit4.class) +public class SplunkResourceManagerIT { + private static final String TEST_ID = "dummy-test"; + private static final int NUM_EVENTS = 100; + + private SplunkResourceManager splunkResourceManager; + + @Before + public void setUp() { + splunkResourceManager = SplunkResourceManager.builder(TEST_ID).build(); + } + + @Test + public void testDefaultSplunkResourceManagerE2E() { + // Arrange + String source = RandomStringUtils.randomAlphabetic(1, 20); + String host = RandomStringUtils.randomAlphabetic(1, 20); + String sourceType = RandomStringUtils.randomAlphabetic(1, 20); + List httpEventsSent = generateHttpEvents(source, sourceType, host); + + splunkResourceManager.sendHttpEvents(httpEventsSent); + + // Act + String query = "search source=" + source + " sourcetype=" + sourceType + " host=" + host; + await("Retrieving events from Splunk") + .atMost(Duration.ofMinutes(1)) + .pollInterval(Duration.ofMillis(500)) + .until( + () -> + SplunkEventsCheck.builder(splunkResourceManager) + .setQuery(query) + .setMinEvents(httpEventsSent.size()) + .build() + .get()); + + List httpEventsReceived = splunkResourceManager.getEvents(query); + + // Assert + assertThatSplunkEvents(httpEventsReceived) + .hasRecordsUnordered(splunkEventsToRecords(httpEventsSent)); + } + + private static List generateHttpEvents( + String source, String sourceType, String host) { + List events = new ArrayList<>(); + long currentTime = System.currentTimeMillis(); + for (int i = 0; i < NUM_EVENTS; i++) { + String event = RandomStringUtils.randomAlphabetic(1, 20); + events.add( + SplunkEvent.newBuilder() + .withEvent(event) + .withSource(source) + .withSourceType(sourceType) + .withHost(host) + .withTime(currentTime + i) + .create()); + } + + return events; + } +} diff --git a/it/splunk/src/test/java/org/apache/beam/it/splunk/SplunkResourceManagerTest.java b/it/splunk/src/test/java/org/apache/beam/it/splunk/SplunkResourceManagerTest.java new file mode 100644 index 0000000000000..ed97bd9f36746 --- /dev/null +++ b/it/splunk/src/test/java/org/apache/beam/it/splunk/SplunkResourceManagerTest.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.splunk; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.splunk.Job; +import com.splunk.ServiceArgs; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import org.apache.beam.sdk.io.splunk.SplunkEvent; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; +import org.awaitility.core.ConditionTimeoutException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.testcontainers.images.builder.Transferable; + +/** Unit tests for {@link SplunkResourceManager}. */ +@RunWith(JUnit4.class) +public class SplunkResourceManagerTest { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private SplunkClientFactory clientFactory; + + @Mock private SplunkContainer container; + + private static final String TEST_ID = "test-id"; + private static final String HOST = "localhost"; + private static final String HEC_SCHEMA = "http"; + private static final String HEC_TOKEN = "token"; + private static final String QUERY = "query"; + private static final String EVENT = "myEvent"; + private static final int DEFAULT_SPLUNK_HEC_INTERNAL_PORT = 8088; + private static final int MAPPED_SPLUNK_HEC_INTERNAL_PORT = 50000; + private static final int DEFAULT_SPLUNKD_INTERNAL_PORT = 8089; + private static final int MAPPED_SPLUNKD_INTERNAL_PORT = 50001; + + private SplunkResourceManager testManager; + + @Before + public void setUp() { + when(container.withDefaultsFile(any(Transferable.class))).thenReturn(container); + when(container.withPassword(anyString())).thenReturn(container); + when(container.getMappedPort(DEFAULT_SPLUNKD_INTERNAL_PORT)) + .thenReturn(MAPPED_SPLUNKD_INTERNAL_PORT); + doReturn(container).when(container).withLogConsumer(any()); + + testManager = + new SplunkResourceManager(clientFactory, container, SplunkResourceManager.builder(TEST_ID)); + } + + @Test + public void testCreateResourceManagerBuilderReturnsSplunkResourceManager() { + assertThat( + SplunkResourceManager.builder(TEST_ID) + .setHecPort(DEFAULT_SPLUNK_HEC_INTERNAL_PORT) + .setSplunkdPort(DEFAULT_SPLUNKD_INTERNAL_PORT) + .setHost(HOST) + .useStaticContainer() + .build()) + .isInstanceOf(SplunkResourceManager.class); + } + + @Test + public void testCreateResourceManagerThrowsCustomPortErrorWhenUsingStaticContainer() { + assertThat( + assertThrows( + SplunkResourceManagerException.class, + () -> + SplunkResourceManager.builder(TEST_ID) + .setHost(HOST) + .useStaticContainer() + .build()) + .getMessage()) + .containsMatch("the hecPort and splunkdPort were not properly set"); + } + + @Test + public void testGetHttpEndpointReturnsCorrectValue() { + when(container.getMappedPort(DEFAULT_SPLUNK_HEC_INTERNAL_PORT)) + .thenReturn(MAPPED_SPLUNK_HEC_INTERNAL_PORT); + assertThat( + new SplunkResourceManager( + clientFactory, container, SplunkResourceManager.builder(TEST_ID)) + .getHttpEndpoint()) + .isEqualTo(String.format("%s://%s:%d", HEC_SCHEMA, HOST, MAPPED_SPLUNK_HEC_INTERNAL_PORT)); + } + + @Test + public void testGetHecEndpointReturnsCorrectValue() { + when(container.getMappedPort(DEFAULT_SPLUNK_HEC_INTERNAL_PORT)) + .thenReturn(MAPPED_SPLUNK_HEC_INTERNAL_PORT); + assertThat( + new SplunkResourceManager( + clientFactory, container, SplunkResourceManager.builder(TEST_ID)) + .getHecEndpoint()) + .isEqualTo( + String.format( + "%s://%s:%d/services/collector/event", + HEC_SCHEMA, HOST, MAPPED_SPLUNK_HEC_INTERNAL_PORT)); + } + + @Test + public void testGetHecTokenReturnsCorrectValueWhenSet() { + assertThat( + new SplunkResourceManager( + clientFactory, + container, + SplunkResourceManager.builder(TEST_ID).setHecToken(HEC_TOKEN)) + .getHecToken()) + .isEqualTo(HEC_TOKEN); + } + + @Test + public void testSendHttpEventsShouldThrowErrorWhenHttpClientFailsToExecuteRequest() + throws IOException { + SplunkEvent event = SplunkEvent.newBuilder().withEvent(EVENT).create(); + + CloseableHttpClient mockHttpClient = clientFactory.getHttpClient(); + doThrow(IOException.class).when(mockHttpClient).execute(any(HttpPost.class)); + + assertThrows(SplunkResourceManagerException.class, () -> testManager.sendHttpEvent(event)); + } + + @Test + public void testSendHttpEventsShouldThrowErrorWhenHttpClientReturnsErrorCode() + throws IOException { + SplunkEvent event = SplunkEvent.newBuilder().withEvent(EVENT).create(); + + try (CloseableHttpResponse mockResponse = + clientFactory.getHttpClient().execute(any(HttpPost.class))) { + when(mockResponse.getStatusLine().getStatusCode()).thenReturn(404); + } + + assertThrows(SplunkResourceManagerException.class, () -> testManager.sendHttpEvent(event)); + } + + @Test + public void testSendHttpEventsShouldReturnTrueIfSplunkDoesNotThrowAnyError() throws IOException { + SplunkEvent event = SplunkEvent.newBuilder().withEvent(EVENT).create(); + + try (CloseableHttpResponse mockResponse = + clientFactory.getHttpClient().execute(any(HttpPost.class))) { + when(mockResponse.getStatusLine().getStatusCode()).thenReturn(200); + } + + assertThat(testManager.sendHttpEvents(ImmutableList.of(event, event))).isTrue(); + verify(clientFactory.getHttpClient()).execute(any(HttpPost.class)); + } + + @Test + public void testGetEventsShouldThrowErrorWhenServiceClientFailsToExecuteRequest() { + Job mockJob = + clientFactory.getServiceClient(any(ServiceArgs.class)).getJobs().create(anyString()); + doThrow(ConditionTimeoutException.class).when(mockJob).isDone(); + + assertThrows(ConditionTimeoutException.class, () -> testManager.getEvents(QUERY)); + } + + @Test + public void testGetEventsShouldThrowErrorWhenXmlReaderFailsToParseResponse() { + Job mockJob = + clientFactory.getServiceClient(any(ServiceArgs.class)).getJobs().create(anyString()); + + when(mockJob.isDone()).thenReturn(true); + when(mockJob.getEvents()) + .thenReturn( + new InputStream() { + @Override + public int read() throws IOException { + throw new IOException(); + } + }); + + assertThrows(SplunkResourceManagerException.class, () -> testManager.getEvents(QUERY)); + } + + @Test + public void testGetEventsShouldReturnTrueIfSplunkDoesNotThrowAnyError() { + Job mockJob = + clientFactory.getServiceClient(any(ServiceArgs.class)).getJobs().create(anyString()); + String rawEvent = + "" + + "" + + "_raw_sourcetype_time" + + "hostindexsource" + + "" + + "" + + "myEvent" + + "mySourceType" + + "1970-01-01T00:00:00.123+00:00" + + "myHost" + + "myIndex" + + "mySource" + + ""; + InputStream inputStream = new ByteArrayInputStream(rawEvent.getBytes(StandardCharsets.UTF_8)); + SplunkEvent splunkEvent = + SplunkEvent.newBuilder() + .withEvent("myEvent") + .withHost("myHost") + .withSource("mySource") + .withSourceType("mySourceType") + .withIndex("myIndex") + .withTime(123L) + .create(); + + when(mockJob.isDone()).thenReturn(true); + when(mockJob.getEvents()).thenReturn(inputStream); + + assertThat(testManager.getEvents()) + .containsExactlyElementsIn(Collections.singletonList(splunkEvent)); + } +} diff --git a/it/splunk/src/test/java/org/apache/beam/it/splunk/SplunkResourceManagerUtilsTest.java b/it/splunk/src/test/java/org/apache/beam/it/splunk/SplunkResourceManagerUtilsTest.java new file mode 100644 index 0000000000000..b4aaec3b45c49 --- /dev/null +++ b/it/splunk/src/test/java/org/apache/beam/it/splunk/SplunkResourceManagerUtilsTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.splunk; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.splunk.SplunkResourceManagerUtils.DEFAULT_SPLUNK_INDEX; +import static org.apache.beam.it.splunk.SplunkResourceManagerUtils.generateHecToken; +import static org.apache.beam.it.splunk.SplunkResourceManagerUtils.generateSplunkPassword; +import static org.apache.beam.it.splunk.SplunkResourceManagerUtils.splunkEventToMap; + +import java.util.HashMap; +import java.util.Map; +import org.apache.beam.sdk.io.splunk.SplunkEvent; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link SplunkResourceManagerUtils}. */ +@RunWith(JUnit4.class) +public class SplunkResourceManagerUtilsTest { + + @Test + public void testSplunkEventToMapWithValuesSet() { + SplunkEvent event = + SplunkEvent.newBuilder() + .withEvent("myEvent") + .withSource("mySource") + .withSourceType("mySourceType") + .withIndex("myIndex") + .withTime(123L) + .create(); + + Map expected = new HashMap<>(); + expected.put("event", "myEvent"); + expected.put("source", "mySource"); + expected.put("sourcetype", "mySourceType"); + expected.put("index", "myIndex"); + expected.put("time", 123L); + expected.put("host", null); + + Map actual = splunkEventToMap(event); + assertThat(actual).containsExactlyEntriesIn(expected); + } + + @Test + public void testSplunkEventToMapWithDefaultValueForIndex() { + SplunkEvent event = SplunkEvent.newBuilder().withEvent("myEvent").create(); + + Map expected = new HashMap<>(); + expected.put("event", "myEvent"); + expected.put("index", DEFAULT_SPLUNK_INDEX); + expected.put("source", null); + expected.put("sourcetype", null); + expected.put("host", null); + expected.put("time", null); + + assertThat(splunkEventToMap(event)).containsExactlyEntriesIn(expected); + } + + @Test + public void testGeneratePasswordMeetsRequirements() { + for (int i = 0; i < 10000; i++) { + String password = generateSplunkPassword(); + int lower = 0; + int upper = 0; + + for (int j = 0; j < password.length(); j++) { + char c = password.charAt(j); + String s = String.valueOf(c); + lower += s.toLowerCase().equals(s) ? 1 : 0; + upper += s.toUpperCase().equals(s) ? 1 : 0; + } + + assertThat(lower).isAtLeast(2); + assertThat(upper).isAtLeast(2); + } + } + + @Test + public void testGenerateHecTokenMeetsRequirements() { + for (int i = 0; i < 10000; i++) { + String password = generateHecToken(); + int lower = 0; + int upper = 0; + + for (int j = 0; j < password.length(); j++) { + char c = password.charAt(j); + String s = String.valueOf(c); + lower += s.toLowerCase().equals(s) ? 1 : 0; + upper += s.toUpperCase().equals(s) ? 1 : 0; + } + + assertThat(lower).isAtLeast(1); + assertThat(upper).isAtLeast(1); + } + } +} diff --git a/it/testcontainers/build.gradle b/it/testcontainers/build.gradle new file mode 100644 index 0000000000000..f29977fd24a59 --- /dev/null +++ b/it/testcontainers/build.gradle @@ -0,0 +1,39 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* License); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an AS IS BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.it.testcontainers', + enableSpotbugs: false, + validateShadowJar: false, + shadowClosure: {}, +) + +description = "Apache Beam :: IT :: Testcontainers" +ext.summary = "Integration test utilities for Testcontainers." + +dependencies { + implementation project(path: ":it:common", configuration: "shadow") + implementation library.java.testcontainers_base + // TODO: excluding Guava until Truth updates it to >32.1.x + testImplementation(library.java.truth) { + exclude group: 'com.google.guava', module: 'guava' + } + testImplementation library.java.mockito_core + testRuntimeOnly library.java.slf4j_simple +} \ No newline at end of file diff --git a/it/testcontainers/src/main/java/org/apache/beam/it/testcontainers/TestContainerResourceManager.java b/it/testcontainers/src/main/java/org/apache/beam/it/testcontainers/TestContainerResourceManager.java new file mode 100644 index 0000000000000..77dd6da58053f --- /dev/null +++ b/it/testcontainers/src/main/java/org/apache/beam/it/testcontainers/TestContainerResourceManager.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.testcontainers; + +import java.util.concurrent.Callable; +import javax.annotation.Nullable; +import org.apache.beam.it.common.ResourceManager; +import org.apache.beam.it.common.TestProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; + +/** + * Abstract class for managing TestContainers resources in integration tests. + * + *

    Resource managers that extend this class will use TestContainers as a backend to spin up + * resources. + * + *

    Optionally, a static resource can be specified by calling the useStaticContainer() method in + * the {@link TestContainerResourceManager.Builder} class. A static resource is a pre-configured + * database or other resource that is ready to be connected to by the resource manager. This could + * be a pre-existing TestContainer that has not been closed, a local database instance, a remote VM, + * or any other source that can be connected to. If a static container is used, the host and port + * must also be configured using the Builder's setHost() and setPort() methods, respectively. + */ +public abstract class TestContainerResourceManager> + implements ResourceManager { + private static final Logger LOG = LoggerFactory.getLogger(TestContainerResourceManager.class); + + protected static final String HOST_IP = TestProperties.hostIp(); + private final T container; + private final boolean usingStaticContainer; + private final String host; + protected int port; + + protected > TestContainerResourceManager( + T container, B builder) { + this(container, builder, null); + } + + protected > TestContainerResourceManager( + T container, B builder, @Nullable Callable setup) { + this.container = container; + this.usingStaticContainer = builder.useStaticContainer; + this.host = builder.host == null ? HOST_IP : builder.host; + this.port = builder.port; + + if (setup != null) { + try { + setup.call(); + } catch (Exception e) { + throw new TestContainerResourceManagerException("Error running setup function.", e); + } + } + + if (!usingStaticContainer) { + // TODO(pranavbhandari): Change this to use log.getUtf8StringWithoutLineEnding() when + // testcontainers dependency is updated. + container + .withLogConsumer( + log -> LOG.info("{}: {}", container.getDockerImageName(), log.getUtf8String())) + .start(); + } else if (builder.host == null || builder.port < 0) { + throw new TestContainerResourceManagerException( + "This manager was configured to use a static resource, but the host and port were not properly set."); + } + } + + protected String getDockerImageName() { + return container.getDockerImageName(); + } + + /** + * Returns the host that the resource is running on. If a host was specified in the builder, then + * this method will return that value. Otherwise, the host will default to localhost. + * + * @return the host that the resource is running on + */ + public String getHost() { + return host; + } + + /** + * Returns the port that the resource is actually mapped to on the host. When using a non-static + * resource, the TestContainers framework will map the resource port to a random port on the host. + * This method will return the mapped port that the resource traffic is routed through. + * + *

    Note: When using a static container, the port that was specified in the builder will be + * returned regardless of what is passed into this method as a parameter. + * + * @param mappedPort the port the resource was configured to listen on + * @return the mapped port on the host + */ + protected int getPort(int mappedPort) { + if (port < 0) { + return container.getMappedPort(mappedPort); + } + return port; + } + + /** + * Deletes all created resources (VM's, etc.) and stops the container, making the manager object + * unusable. + * + * @throws TestContainerResourceManagerException if there is an error deleting the TestContainers + * resources. + */ + @Override + public void cleanupAll() throws TestContainerResourceManagerException { + if (usingStaticContainer) { + LOG.info("This manager was configured to use a static resource that will not be cleaned up."); + return; + } + + LOG.info("Attempting to cleanup TestContainers manager."); + try { + container.close(); + LOG.info("TestContainers manager successfully cleaned up."); + } catch (Exception e) { + throw new TestContainerResourceManagerException("Failed to close TestContainer resources", e); + } + } + + /** Builder for {@link TestContainerResourceManager}. */ + public abstract static class Builder> { + + public String testId; + public String containerImageName; + public String containerImageTag; + public @Nullable String host; + public int port; + public boolean useStaticContainer; + + public Builder(String testId, String containerImageName, String containerImageTag) { + this.testId = testId; + this.containerImageName = containerImageName; + this.containerImageTag = containerImageTag; + this.host = null; + this.port = -1; + } + + /** + * Sets the name of the test container image. The tag is typically the version of the image. + * + * @param containerName The name of the container image. + * @return this builder object with the image name set. + */ + public Builder setContainerImageName(String containerName) { + this.containerImageName = containerName; + return this; + } + + /** + * Sets the tag for the test container. The tag is typically the version of the image. + * + * @param containerTag The tag to use for the container. + * @return this builder object with the tag set. + */ + public Builder setContainerImageTag(String containerTag) { + this.containerImageTag = containerTag; + return this; + } + + /** + * Sets the host of the resource that the resource manager will connect to. This will default to + * localhost if not set. + * + * @param containerHost the resource host address. + * @return this builder object with the host set. + */ + public Builder setHost(String containerHost) { + this.host = containerHost; + return this; + } + + /** + * Sets the port that the resource is hosted on. + * + * @param port the port the resource is hosted on. + * @return this builder object with the port set. + */ + public Builder setPort(int port) { + this.port = port; + return this; + } + + /** + * Configures the resource manager to use a static resource instead of creating a new + * TestContainer instance of the resource. This can be another TestContainer or any other VM or + * server hosting the resource. + * + *

    Note: When this option is enabled, the setPort() and setHost() methods must also be called + * to configure the static resource address. + * + * @return this builder object with the useStaticContainer option enabled. + */ + public Builder useStaticContainer() { + this.useStaticContainer = true; + return this; + } + + /** + * Builds and returns a Resource Manager that extends TestContainerResourceManager. + * + * @return an instance of TestContainerResourceManager + */ + public abstract T build(); + } +} diff --git a/it/testcontainers/src/main/java/org/apache/beam/it/testcontainers/TestContainerResourceManagerException.java b/it/testcontainers/src/main/java/org/apache/beam/it/testcontainers/TestContainerResourceManagerException.java new file mode 100644 index 0000000000000..a617e008ce944 --- /dev/null +++ b/it/testcontainers/src/main/java/org/apache/beam/it/testcontainers/TestContainerResourceManagerException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.testcontainers; + +/** Custom exception for {@link TestContainerResourceManager} implementations. */ +public class TestContainerResourceManagerException extends RuntimeException { + + public TestContainerResourceManagerException(String errorMessage) { + super(errorMessage); + } + + public TestContainerResourceManagerException(String errorMessage, Exception cause) { + super(errorMessage, cause); + } +} diff --git a/it/testcontainers/src/main/java/org/apache/beam/it/testcontainers/TestContainersIntegrationTest.java b/it/testcontainers/src/main/java/org/apache/beam/it/testcontainers/TestContainersIntegrationTest.java new file mode 100644 index 0000000000000..1fb8f707bce21 --- /dev/null +++ b/it/testcontainers/src/main/java/org/apache/beam/it/testcontainers/TestContainersIntegrationTest.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.testcontainers; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** Annotation that marks the test that requires TestContainers. */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface TestContainersIntegrationTest {} diff --git a/it/testcontainers/src/main/java/org/apache/beam/it/testcontainers/package-info.java b/it/testcontainers/src/main/java/org/apache/beam/it/testcontainers/package-info.java new file mode 100644 index 0000000000000..f6817da5a5fd8 --- /dev/null +++ b/it/testcontainers/src/main/java/org/apache/beam/it/testcontainers/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for managing TestContainers resources within integration tests. */ +package org.apache.beam.it.testcontainers; diff --git a/it/testcontainers/src/test/java/org/apache/beam/it/testcontainers/TestContainerResourceManagerTest.java b/it/testcontainers/src/test/java/org/apache/beam/it/testcontainers/TestContainerResourceManagerTest.java new file mode 100644 index 0000000000000..7e2d686d28a80 --- /dev/null +++ b/it/testcontainers/src/test/java/org/apache/beam/it/testcontainers/TestContainerResourceManagerTest.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.testcontainers; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.apache.beam.it.common.TestProperties; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.testcontainers.containers.GenericContainer; + +/** Unit tests for {@link TestContainerResourceManager}. */ +@RunWith(JUnit4.class) +public class TestContainerResourceManagerTest { + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock private GenericContainer container; + + private static final String TEST_ID = "test-id"; + private static final String HOST = "1.2.3.4"; + private static final int PORT = 10000; + + private TestContainerResourceManager.Builder testManagerBuilder; + + @Before + public void setUp() { + testManagerBuilder = + new TestContainerResourceManager.Builder( + TEST_ID, null, null) { + + @Override + public TestContainerResourceManagerImpl build() { + return new TestContainerResourceManagerImpl(container, this); + } + }; + } + + @Test + public void testCreateResourceManagerSetsCorrectDockerImageName() { + when(container.getDockerImageName()).thenReturn("container-test:test"); + doReturn(container).when(container).withLogConsumer(any()); + + testManagerBuilder.setContainerImageName("container-test").setContainerImageTag("test").build(); + + assertThat(container.getDockerImageName()) + .isEqualTo( + testManagerBuilder.containerImageName + ":" + testManagerBuilder.containerImageTag); + } + + @Test + public void testCreateResourceManagerShouldStartContainerWhenNotUsingStaticResource() { + doReturn(container).when(container).withLogConsumer(any()); + testManagerBuilder.build(); + + verify(container).start(); + } + + @Test + public void testCreateResourceManagerShouldNotStartContainerWhenUsingStaticResource() { + testManagerBuilder.useStaticContainer().setHost(HOST).setPort(PORT).build(); + + verify(container, never()).start(); + } + + @Test + public void + testCreateResourceManagerShouldThrowErrorWhenUsingStaticResourceWithoutHostOrPortSet() { + assertThrows( + TestContainerResourceManagerException.class, + () -> testManagerBuilder.useStaticContainer().build()); + } + + @Test + public void testCreateResourceManagerShouldThrowErrorWhenUsingStaticResourceWithoutHostSet() { + assertThrows( + TestContainerResourceManagerException.class, + () -> testManagerBuilder.useStaticContainer().setPort(PORT).build()); + } + + @Test + public void testCreateResourceManagerShouldThrowErrorWhenUsingStaticResourceWithoutPortSet() { + assertThrows( + TestContainerResourceManagerException.class, + () -> testManagerBuilder.useStaticContainer().setHost(HOST).build()); + } + + @Test + public void testGetHostShouldReturnCorrectHostWhenManuallySet() { + doReturn(container).when(container).withLogConsumer(any()); + TestContainerResourceManager testManager = testManagerBuilder.setHost(HOST).build(); + + assertThat(testManager.getHost()).matches(HOST); + } + + @Test + public void testGetHostShouldReturnCorrectHostWhenHostNotSet() { + doReturn(container).when(container).withLogConsumer(any()); + String host = TestProperties.hostIp(); + TestContainerResourceManager testManager = testManagerBuilder.build(); + + assertThat(testManager.getHost()).matches(host); + } + + @Test + public void testGetPortShouldReturnCorrectPortWhenManuallySet() { + doReturn(container).when(container).withLogConsumer(any()); + TestContainerResourceManager testManager = + testManagerBuilder.setHost(HOST).setPort(PORT).build(); + + assertThat(testManager.getPort(-1)).isEqualTo(PORT); + } + + @Test + public void testGetPortShouldReturnContainerHostWhenPortNotSet() { + int mappedPort = 5000; + when(container.getMappedPort(anyInt())).thenReturn(mappedPort); + doReturn(container).when(container).withLogConsumer(any()); + + TestContainerResourceManager testManager = testManagerBuilder.build(); + + assertThat(testManager.getPort(PORT)).isEqualTo(mappedPort); + } + + @Test + public void testCleanupAllShouldCloseContainerWhenNotUsingStaticResource() { + doReturn(container).when(container).withLogConsumer(any()); + TestContainerResourceManager testManager = testManagerBuilder.build(); + + testManager.cleanupAll(); + verify(container).close(); + } + + @Test + public void testCleanupAllShouldReturnFalseWhenContainerFailsToClose() { + doThrow(RuntimeException.class).when(container).close(); + doReturn(container).when(container).withLogConsumer(any()); + + TestContainerResourceManager testManager = testManagerBuilder.build(); + + assertThrows(TestContainerResourceManagerException.class, testManager::cleanupAll); + } + + @Test + public void testCleanupAllShouldNotCloseContainerWhenUsingStaticResource() { + TestContainerResourceManager testManager = + testManagerBuilder.useStaticContainer().setHost(HOST).setPort(PORT).build(); + + testManager.cleanupAll(); + verify(container, never()).close(); + } + + private static class TestContainerResourceManagerImpl + extends TestContainerResourceManager> { + protected TestContainerResourceManagerImpl( + GenericContainer container, Builder builder) { + super(container, builder); + } + } +} diff --git a/it/truthmatchers/build.gradle b/it/truthmatchers/build.gradle new file mode 100644 index 0000000000000..8363523dced3f --- /dev/null +++ b/it/truthmatchers/build.gradle @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.module' } +applyJavaNature( + exportJavadoc: false, + automaticModuleName: 'org.apache.beam.it.truthmatchers', + validateShadowJar: false, + shadowClosure: {}, +) + +description = "Apache Beam :: IT :: Truth matchers" +ext.summary = "Truth matchers for integration tests." + +dependencies { + implementation project(path: ":it:common", configuration: "shadow") + // TODO: excluding Guava until Truth updates it to >32.1.x + implementation(library.java.truth) { + exclude group: 'com.google.guava', module: 'guava' + } + implementation library.java.vendored_guava_32_1_2_jre + permitUnusedDeclared library.java.vendored_guava_32_1_2_jre +} \ No newline at end of file diff --git a/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/LaunchInfoSubject.java b/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/LaunchInfoSubject.java new file mode 100644 index 0000000000000..30a27c9ad2592 --- /dev/null +++ b/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/LaunchInfoSubject.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.truthmatchers; + +import com.google.common.truth.FailureMetadata; +import com.google.common.truth.Subject; +import org.apache.beam.it.common.PipelineLauncher.JobState; +import org.apache.beam.it.common.PipelineLauncher.LaunchInfo; + +/** + * Subject that has assertion operations for {@link LaunchInfo}, which has the information for a + * recently launched pipeline. + */ +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) +public final class LaunchInfoSubject extends Subject { + + private final LaunchInfo actual; + + private LaunchInfoSubject(FailureMetadata metadata, LaunchInfo actual) { + super(metadata, actual); + this.actual = actual; + } + + public static Factory launchInfo() { + return LaunchInfoSubject::new; + } + + /** + * Check if the subject reflects succeeded states. A successful {@link LaunchInfo} does not mean + * that the pipeline finished and no errors happened, it just means that the job was able to get + * itself into an active state (RUNNING, UPDATED). + */ + public void isRunning() { + check("check if succeeded").that(actual.state()).isIn(JobState.ACTIVE_STATES); + } + + /** + * Check if the subject reflects failure states. A failed {@link LaunchInfo} often means that the + * request for launching a pipeline didn't make through validations, so the job couldn't even + * start or do any processing. + */ + public void failed() { + check("check if succeeded").that(actual.state()).isIn(JobState.FAILED_STATES); + } +} diff --git a/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/ListAccumulator.java b/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/ListAccumulator.java new file mode 100644 index 0000000000000..5e8f8c4aff8f9 --- /dev/null +++ b/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/ListAccumulator.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.truthmatchers; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Provides a wrapper around a List which can be used to accumulate results, useful for assertions + * that consume inputs and need to keep state across calls. + */ +public class ListAccumulator { + + private List backingList; + + /** Initialize the accumulator. */ + public ListAccumulator() { + this.backingList = new ArrayList<>(); + } + + /** + * Add the partial results to the backingList, and return the accumulated count. + * + * @param partial Partial results to accumulate. + * @return Accumulated count. + */ + public int accumulate(Collection partial) { + this.backingList.addAll(partial); + return this.backingList.size(); + } + + /** + * Return the count of objects accumulated so far. + * + * @return Count, which is the size of the backingList. + */ + public int count() { + return this.backingList.size(); + } + + /** @return the internal backing list being accumulated. */ + public List getBackingList() { + return this.backingList; + } +} diff --git a/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/PipelineAsserts.java b/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/PipelineAsserts.java new file mode 100644 index 0000000000000..4acbb0bb06b26 --- /dev/null +++ b/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/PipelineAsserts.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.truthmatchers; + +import static com.google.common.truth.Truth.assertAbout; + +import com.google.gson.Gson; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; +import org.apache.beam.it.common.PipelineLauncher.LaunchInfo; +import org.apache.beam.it.common.PipelineOperator.Result; + +/** Assert utilities for Template DSL-like tests. */ +public class PipelineAsserts { + + /** + * Creates a {@link LaunchInfoSubject} to assert information returned from pipeline launches. + * + * @param launchInfo Launch information returned from the launcher. + * @return Truth Subject to chain assertions. + */ + public static LaunchInfoSubject assertThatPipeline(LaunchInfo launchInfo) { + return assertAbout(LaunchInfoSubject.launchInfo()).that(launchInfo); + } + + /** + * Creates a {@link ResultSubject} to add assertions based on a pipeline result. + * + * @param result Pipeline result returned from the launcher. + * @return Truth Subject to chain assertions. + */ + public static ResultSubject assertThatResult(Result result) { + return assertAbout(ResultSubject.result()).that(result); + } + + /** + * Creates a {@link RecordsSubject} to assert information within a list of records. + * + * @param records Records in a map list format to use in the comparison. + * @return Truth Subject to chain assertions. + */ + public static RecordsSubject assertThatRecords(@Nullable List> records) { + return assertAbout(RecordsSubject.records()).that(records); + } + + /** + * Convert JSON Strings to a list of maps. + * + * @param jsonRecords List of JSON Strings to parse + * @return List of maps to use in {@link RecordsSubject} + */ + public static List> jsonRecordsToRecords( + @Nullable Iterable jsonRecords) { + try { + List> records = new ArrayList<>(); + if (jsonRecords != null) { + jsonRecords.forEach( + json -> records.add(new Gson().>fromJson(json, Map.class))); + } + + return records; + } catch (Exception e) { + throw new RuntimeException("Error converting JSON Records to Records", e); + } + } + + /** + * Creates a {@link RecordsSubject} to assert information within a list of JSON Strings. + * + * @param jsonRecords Strings in JSON format to use in the comparison. + * @return Truth Subject to chain assertions. + */ + public static RecordsSubject assertThatJsonRecords(@Nullable List jsonRecords) { + return assertThatRecords(jsonRecordsToRecords(jsonRecords)); + } +} diff --git a/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/RecordsSubject.java b/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/RecordsSubject.java new file mode 100644 index 0000000000000..39a0c0cebedcd --- /dev/null +++ b/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/RecordsSubject.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.truthmatchers; + +import com.google.common.truth.Fact; +import com.google.common.truth.FailureMetadata; +import com.google.common.truth.Subject; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; +import java.util.stream.Collectors; +import javax.annotation.Nullable; + +/** + * Subject that has assertion operations for record lists, usually coming from the result of a + * template. + */ +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/27438) +}) +public class RecordsSubject extends Subject { + + @Nullable private final List> actual; + + protected RecordsSubject(FailureMetadata metadata, @Nullable List> actual) { + super(metadata, actual); + this.actual = actual; + } + + public static Factory>> records() { + return RecordsSubject::new; + } + + /** Check if records list has rows (i.e., is not empty). */ + public void hasRows() { + check("there are rows").that(actual.size()).isGreaterThan(0); + } + + /** + * Check if records list has a specific number of rows. + * + * @param expectedRows Expected Rows + */ + public void hasRows(int expectedRows) { + check("there are %s rows", expectedRows).that(actual.size()).isEqualTo(expectedRows); + } + + /** + * Check if the records list has a specific row. + * + * @param record Expected row to search + */ + public void hasRecord(Map record) { + check("expected that contains record %s", record.toString()).that(actual).contains(record); + } + + /** + * Check if the records list matches a specific row (using partial / subset comparison). + * + * @param subset Expected subset to search in a record. + */ + public void hasRecordSubset(Map subset) { + + Map expected = convertMapToTreeMap(subset); + for (Map candidate : actual) { + boolean match = true; + for (Map.Entry entry : subset.entrySet()) { + if (!candidate.containsKey(entry.getKey()) + || !candidate.get(entry.getKey()).equals(entry.getValue())) { + match = false; + break; + } + } + + if (match) { + return; + } + } + + failWithoutActual( + Fact.simpleFact( + "expected that contains partial record " + expected + ", but only had " + actual)); + } + + /** + * Check if the records list has specific rows, without guarantees of ordering. The way that + * ordering is taken out of the equation is by converting all records to TreeMap, which guarantee + * natural key ordering. + * + * @param records Expected rows to search + */ + public void hasRecordsUnordered(List> records) { + + for (Map record : records) { + String expected = convertMapToTreeMap(record).toString(); + if (actual.stream() + .noneMatch(candidate -> convertMapToTreeMap(candidate).toString().equals(expected))) { + failWithoutActual( + Fact.simpleFact( + "expected that contains unordered record " + + expected + + ", but only had " + + actual)); + } + } + } + + /** + * Helper method that does a deep recursive mapping of a Map to TreeMap such that nested Map's are + * also converted to TreeMap's. + * + * @param record The Map to convert to TreeMap. + * @return The converted TreeMap. + */ + private TreeMap convertMapToTreeMap(Map record) { + TreeMap result = new TreeMap<>(record); + result.forEach( + (key, value) -> { + if (value instanceof Map) { + result.put( + key, + convertMapToTreeMap( + ((Map) value) + .entrySet().stream() + .collect( + Collectors.toMap(e -> e.getKey().toString(), Entry::getValue)))); + } + }); + return result; + } + + /** + * Check if the records list has specific strings. It may be useful to evaluate deeply nested + * structures, when a simple part must be matched. + * + * @param strings Expected strings to search. + */ + public void hasRecordsWithStrings(List strings) { + + for (String expected : strings) { + if (actual.stream() + .noneMatch(candidate -> convertMapToTreeMap(candidate).toString().contains(expected))) { + failWithoutActual( + Fact.simpleFact( + "expected that contains the string '" + expected + "', but only had " + actual)); + } + } + } + + /** + * Check if the records list has specific rows, without guarantees of ordering. The way that + * ordering is taken out of the equation is by converting all records to TreeMap, which guarantee + * natural key ordering. + * + *

    In this particular method, force the columns to be case-insensitive to maximize + * compatibility. + * + * @param records Expected rows to search + */ + public void hasRecordsUnorderedCaseInsensitiveColumns(List> records) { + + for (Map record : records) { + String expected = convertKeysToUpperCase(convertMapToTreeMap(record)).toString(); + if (actual.stream() + .noneMatch( + candidate -> + convertKeysToUpperCase(convertMapToTreeMap(candidate)) + .toString() + .equals(expected))) { + failWithoutActual( + Fact.simpleFact( + "expected that contains unordered record (and case insensitive) " + + expected + + ", but only had " + + actual)); + } + } + } + + private Map convertKeysToUpperCase(Map map) { + return convertMapToTreeMap( + map.entrySet().stream() + .collect(Collectors.toMap(entry -> entry.getKey().toUpperCase(), Entry::getValue))); + } + + /** + * Check if the records list has a specific row, without guarantees of ordering. The way that + * ordering is taken out of the equation is by converting all records to TreeMap, which guarantee + * natural key ordering. + * + * @param record Expected row to search + */ + public void hasRecordUnordered(Map record) { + this.hasRecordsUnordered(Collections.singletonList(record)); + } + + /** + * Check if the records list match exactly another list. + * + * @param records Expected records + */ + public void hasRecords(List> records) { + check("records %s are there", records.toString()) + .that(actual) + .containsExactlyElementsIn(records); + } + + /** + * Check if all the records match given record. + * + * @param record Expected record + */ + public void allMatch(Map record) { + List> records = Collections.nCopies(actual.size(), record); + hasRecords(records); + } +} diff --git a/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/ResultSubject.java b/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/ResultSubject.java new file mode 100644 index 0000000000000..d8cb0be9610c0 --- /dev/null +++ b/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/ResultSubject.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.it.truthmatchers; + +import com.google.common.truth.FailureMetadata; +import com.google.common.truth.Subject; +import java.util.function.Supplier; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.PipelineOperator.Config; +import org.apache.beam.it.common.PipelineOperator.Result; + +/** + * Subject that has assertion operations for {@link Result}, which is the end result of a pipeline + * run. + */ +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) +public final class ResultSubject extends Subject { + + private final Result actual; + + private ResultSubject(FailureMetadata metadata, Result actual) { + super(metadata, actual); + this.actual = actual; + } + + public static Factory result() { + return ResultSubject::new; + } + + /** + * Check if the launch meets expected conditions given to a `waitForCondition...` method such as + * {@link PipelineOperator#waitForCondition(Config, Supplier...)} or {@link + * PipelineOperator#waitForConditionAndFinish(Config, Supplier)}. + */ + public void meetsConditions() { + check("check if meets all conditions").that(actual).isEqualTo(Result.CONDITION_MET); + } + + /** + * Check if the launch finished (is in a successful final state). It doesn't mean that the jobs + * meet any conditions. For any `waitForCondition...` methods, checking if the launch {@link + * #meetsConditions()} is more appropriate for testing purposes. + */ + public void isLaunchFinished() { + check("check if result is finished").that(actual).isEqualTo(Result.LAUNCH_FINISHED); + } + + /** Check if the launch finished with a timeout. */ + public void hasTimedOut() { + check("check if result timed out").that(actual).isEqualTo(Result.TIMEOUT); + } + + /** Check if the subject finished with a failed state. */ + public void hasFailed() { + check("check if result timed out").that(actual).isEqualTo(Result.LAUNCH_FAILED); + } +} diff --git a/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/package-info.java b/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/package-info.java new file mode 100644 index 0000000000000..81cda113ee5e6 --- /dev/null +++ b/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package for Truth matchers / subjects to have reusable assertions. */ +package org.apache.beam.it.truthmatchers; diff --git a/learning/tour-of-beam/frontend/lib/components/profile/user_menu.dart b/learning/tour-of-beam/frontend/lib/components/profile/user_menu.dart index f3f633b0028f9..c50e20acc2946 100644 --- a/learning/tour-of-beam/frontend/lib/components/profile/user_menu.dart +++ b/learning/tour-of-beam/frontend/lib/components/profile/user_menu.dart @@ -16,12 +16,15 @@ * limitations under the License. */ +import 'dart:async'; + import 'package:easy_localization/easy_localization.dart'; import 'package:firebase_auth/firebase_auth.dart'; import 'package:flutter/material.dart'; import 'package:flutter_svg/svg.dart'; import 'package:get_it/get_it.dart'; import 'package:playground_components/playground_components.dart'; +import 'package:url_launcher/url_launcher.dart'; import '../../assets/assets.gen.dart'; import '../../auth/notifier.dart'; @@ -104,17 +107,13 @@ class _Buttons extends StatelessWidget { children: [ _IconLabel( isSvg: false, - onTap: () {}, + onTap: () { + unawaited(launchUrl(Uri.parse(BeamLinks.website))); + }, iconPath: Assets.png.profileWebsite.path, label: 'ui.toWebsite'.tr(), ), const BeamDivider(), - _IconLabel( - onTap: () {}, - iconPath: Assets.svg.profileAbout, - label: 'ui.about'.tr(), - ), - const BeamDivider(), _IconLabel( onTap: () async { await authNotifier.logOut(); diff --git a/learning/tour-of-beam/frontend/lib/pages/tour/state.dart b/learning/tour-of-beam/frontend/lib/pages/tour/state.dart index 8b2a91f1d0231..3094a5e199172 100644 --- a/learning/tour-of-beam/frontend/lib/pages/tour/state.dart +++ b/learning/tour-of-beam/frontend/lib/pages/tour/state.dart @@ -169,10 +169,10 @@ class TourNotifier extends ChangeNotifier with PageStateMixin { _trackUnitClosed(); } + _currentUnitContent = unitContent; if (_currentUnitContent != null) { _isPlaygroundShown = _currentUnitContent!.taskSnippetId != null; } - _currentUnitContent = unitContent; if (_currentUnitContent != null) { _trackUnitOpened(_currentUnitContent!.id); diff --git a/learning/tour-of-beam/frontend/pubspec.lock b/learning/tour-of-beam/frontend/pubspec.lock index 5cde8a54211f8..49bdc9ef95a84 100644 --- a/learning/tour-of-beam/frontend/pubspec.lock +++ b/learning/tour-of-beam/frontend/pubspec.lock @@ -45,10 +45,10 @@ packages: dependency: transitive description: name: archive - sha256: "0c8368c9b3f0abbc193b9d6133649a614204b528982bebc7026372d61677ce3a" + sha256: "49b1fad315e57ab0bbc15bcbb874e83116a1d78f77ebd500a4af6c9407d6b28e" url: "https://pub.dev" source: hosted - version: "3.3.7" + version: "3.3.8" args: dependency: transitive description: @@ -189,10 +189,10 @@ packages: dependency: "direct main" description: name: collection - sha256: "4a07be6cb69c84d677a6c3096fcf960cc3285a8330b4603e0d463d15d9bd934c" + sha256: f092b211a4319e98e5ff58223576de6c2803db36221657b46c82574721240687 url: "https://pub.dev" source: hosted - version: "1.17.1" + version: "1.17.2" color: dependency: transitive description: @@ -656,10 +656,10 @@ packages: dependency: transitive description: name: intl - sha256: a3715e3bc90294e971cb7dc063fbf3cd9ee0ebf8604ffeafabd9e6f16abbdbe6 + sha256: "3bc132a9dbce73a7e4a21a17d06e1878839ffbf975568bc875c60537824b0c4d" url: "https://pub.dev" source: hosted - version: "0.18.0" + version: "0.18.1" io: dependency: transitive description: @@ -728,18 +728,18 @@ packages: dependency: transitive description: name: matcher - sha256: "6501fbd55da300384b768785b83e5ce66991266cec21af89ab9ae7f5ce1c4cbb" + sha256: "1803e76e6653768d64ed8ff2e1e67bea3ad4b923eb5c56a295c3e634bad5960e" url: "https://pub.dev" source: hosted - version: "0.12.15" + version: "0.12.16" material_color_utilities: dependency: transitive description: name: material_color_utilities - sha256: d92141dc6fe1dad30722f9aa826c7fbc896d021d792f80678280601aff8cf724 + sha256: "9528f2f296073ff54cb9fee677df673ace1218163c3bc7628093e7eed5203d41" url: "https://pub.dev" source: hosted - version: "0.2.0" + version: "0.5.0" meta: dependency: transitive description: @@ -1115,10 +1115,10 @@ packages: dependency: transitive description: name: source_span - sha256: dd904f795d4b4f3b870833847c461801f6750a9fa8e61ea5ac53f9422b31f250 + sha256: "53e943d4206a5e30df338fd4c6e7a077e02254531b138a15aec3bd143c1a8b3c" url: "https://pub.dev" source: hosted - version: "1.9.1" + version: "1.10.0" stack_trace: dependency: transitive description: @@ -1171,26 +1171,26 @@ packages: dependency: transitive description: name: test - sha256: "3dac9aecf2c3991d09b9cdde4f98ded7b30804a88a0d7e4e7e1678e78d6b97f4" + sha256: "13b41f318e2a5751c3169137103b60c584297353d4b1761b66029bae6411fe46" url: "https://pub.dev" source: hosted - version: "1.24.1" + version: "1.24.3" test_api: dependency: transitive description: name: test_api - sha256: eb6ac1540b26de412b3403a163d919ba86f6a973fe6cc50ae3541b80092fdcfb + sha256: "75760ffd7786fffdfb9597c35c5b27eaeec82be8edfb6d71d32651128ed7aab8" url: "https://pub.dev" source: hosted - version: "0.5.1" + version: "0.6.0" test_core: dependency: transitive description: name: test_core - sha256: "5138dbffb77b2289ecb12b81c11ba46036590b72a64a7a90d6ffb880f1a29e93" + sha256: "99806e9e6d95c7b059b7a0fc08f07fc53fabe54a829497f0d9676299f1e8637e" url: "https://pub.dev" source: hosted - version: "0.5.1" + version: "0.5.3" time: dependency: transitive description: @@ -1347,10 +1347,10 @@ packages: dependency: transitive description: name: vm_service - sha256: f6deed8ed625c52864792459709183da231ebf66ff0cf09e69b573227c377efe + sha256: c620a6f783fa22436da68e42db7ebbf18b8c44b9a46ab911f666ff09ffd9153f url: "https://pub.dev" source: hosted - version: "11.3.0" + version: "11.7.1" watcher: dependency: transitive description: @@ -1359,6 +1359,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.1.0" + web: + dependency: transitive + description: + name: web + sha256: dc8ccd225a2005c1be616fe02951e2e342092edf968cf0844220383757ef8f10 + url: "https://pub.dev" + source: hosted + version: "0.1.4-beta" web_socket_channel: dependency: transitive description: diff --git a/learning/tour-of-beam/learning-content/content-info.yaml b/learning/tour-of-beam/learning-content/content-info.yaml index c1bb720e370b6..05ab3b69d3c73 100644 --- a/learning/tour-of-beam/learning-content/content-info.yaml +++ b/learning/tour-of-beam/learning-content/content-info.yaml @@ -31,4 +31,5 @@ content: - io - splittable-dofn - cross-language - + - final-challenge + \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/description.md b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/description.md new file mode 100644 index 0000000000000..aa3978bb27eaa --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/description.md @@ -0,0 +1,27 @@ + +### Final challenge 1 + +You’re given a csv file with purchase transactions. Write a Beam pipeline to prepare a report every 30 seconds. The report needs to be created only for transactions where quantity is more than 20. + +Report should consist of two files named "**price_more_than_10.txt**" and "**price_less_than_10.txt**": + +* Total transactions amount grouped by **ProductNo** for products with **price** greater than 10 +* Total transactions amount grouped by **ProductNo** for products with **price** less than 10 + +Example rows from input file: + +| TransactionNo | Date | ProductNo | ProductName | Price | Quantity | CustomerNo | Country | +|---------------|-----------|-----------|-------------------------------------|-------|----------|------------|----------------| +| 581482 | 12/9/2019 | 22485 | Set Of 2 Wooden Market Crates | 21 | 47 | 17490 | United Kingdom | +| 581475 | 12/9/2019 | 22596 | Christmas Star Wish List Chalkboard | 10.65 | 36 | 13069 | United Kingdom | +| 581475 | 12/9/2019 | 23235 | Storage Tin Vintage Leaf | 11.53 | 12 | 13069 | United Kingdom | \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/go-challenge/input.csv b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/go-challenge/input.csv new file mode 100644 index 0000000000000..85854a2d43584 --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/go-challenge/input.csv @@ -0,0 +1,1500 @@ +TransactionNo,Date,ProductNo,ProductName,Price,Quantity,CustomerNo,Country +581482,12/9/2019,22485,Set Of 2 Wooden Market Crates,21.47,12,17490,United Kingdom +581475,12/9/2019,22596,Christmas Star Wish List Chalkboard,10.65,36,13069,United Ki +581475,12/9/2019,23235,Storage Tin Vintage Leaf,11.53,12,13069,United Kingdom +581475,12/9/2019,23272,Tree T-Light Holder Willie Winkie,10.65,12,13069,United King +581475,12/9/2019,23239,Set Of 4 Knick Knack Tins Poppies,11.94,6,13069,United Kingd +581475,12/9/2019,21705,Bag 500g Swirly Marbles,10.65,24,13069,United Kingdom +581475,12/9/2019,22118,Joy Wooden Block Letters,11.53,18,13069,United Kingdom +581475,12/9/2019,22119,Peace Wooden Block Letters,12.25,12,13069,United Kingdom +581475,12/9/2019,22217,T-Light Holder Hanging Lace,10.65,12,13069,United Kingdom +581475,12/9/2019,22216,T-Light Holder White Lace,10.55,24,13069,United Kingdom +581475,12/9/2019,22380,Toy Tidy Spaceboy,11.06,20,13069,United Kingdom +581475,12/9/2019,22442,Grow Your Own Flowers Set Of 3,12.25,12,13069,United Kingdom +581475,12/9/2019,22664,Toy Tidy Dolly Girl Design,11.06,20,13069,United Kingdom +581475,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,12.25,12,13069,United Kingdom +581475,12/9/2019,22723,Set Of 6 Herb Tins Sketchbook,11.53,12,13069,United Kingdom +581475,12/9/2019,22785,Squarecushion Cover Pink Union Jack,11.53,12,13069,United Ki +581475,12/9/2019,22955,36 Foil Star Cake Cases,11.06,24,13069,United Kingdom +581475,12/9/2019,23141,Triple Wire Hook Pink Heart,11.06,12,13069,United Kingdom +581475,12/9/2019,22956,36 Foil Heart Cake Cases,11.06,24,13069,United Kingdom +581475,12/9/2019,22581,Wood Stocking Christmas Scandispot,10.55,48,13069,United Kin +581476,12/9/2019,23198,Pantry Magnetic Shopping List,11.53,48,12433,Norway +581476,12/9/2019,23197,Sketchbook Magnetic Shopping List,11.74,24,12433,Norway +581476,12/9/2019,23184,Bull Dog Bottle Opener,15.32,8,12433,Norway +581476,12/9/2019,23168,Classic Cafe Sugar Dispenser,11.53,12,12433,Norway +581476,12/9/2019,23167,Small Ceramic Top Storage Jar,10.96,96,12433,Norway +581476,12/9/2019,23166,Medium Ceramic Top Storage Jar,11.32,48,12433,Norway +581476,12/9/2019,23165,Large Ceramic Top Storage Jar,11.74,48,12433,Norway +581476,12/9/2019,23004,Travel Card Wallet Pantry,10.68,48,12433,Norway +581476,12/9/2019,23002,Travel Card Wallet Skulls,10.68,24,12433,Norway +581476,12/9/2019,23000,Travel Card Wallet Transport,10.68,24,12433,Norway +581476,12/9/2019,22998,Travel Card Wallet Keep Calm,10.68,72,12433,Norway +581476,12/9/2019,22994,Travel Card Wallet Retrospot,10.68,48,12433,Norway +581476,12/9/2019,22835,Hot Water Bottle I Am So Poorly,15.32,8,12433,Norway +581476,12/9/2019,22730,Alarm Clock Bakelike Ivory,14.09,4,12433,Norway +581476,12/9/2019,22728,Alarm Clock Bakelike Pink,14.09,8,12433,Norway +581476,12/9/2019,22727,Alarm Clock Bakelike Red,14.09,8,12433,Norway +581476,12/9/2019,22726,Alarm Clock Bakelike Green,14.09,4,12433,Norway +581476,12/9/2019,22720,Set Of 3 Cake Tins Pantry Design,14.61,16,12433,Norway +581476,12/9/2019,22693,Grow A Flytrap Or Sunflower In Tin,11.34,192,12433,Norway +581476,12/9/2019,22670,French Wc Sign Blue Metal,11.53,24,12433,Norway +581476,12/9/2019,22667,Recipe Box Retrospot,12.86,24,12433,Norway +581476,12/9/2019,22666,Recipe Box Pantry Yellow Design,12.86,24,12433,Norway +581476,12/9/2019,22631,Circus Parade Lunch Box,12.25,12,12433,Norway +581476,12/9/2019,22628,Picnic Boxes Set Of 3 Retrospot,15.32,16,12433,Norway +581476,12/9/2019,22467,Gumball Coat Rack,12.86,6,12433,Norway +581476,12/9/2019,22197,Popcorn Holder,10.99,100,12433,Norway +581476,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,14.61,8,12433,Norway +581476,12/9/2019,22112,Chocolate Hot Water Bottle,15.32,9,12433,Norway +581476,12/9/2019,21908,Chocolate This Way Metal Sign,12.40,36,12433,Norway +581476,12/9/2019,21874,Gin And Tonic Mug,11.74,36,12433,Norway +581476,12/9/2019,21872,Glamorous Mug,11.53,12,12433,Norway +581476,12/9/2019,21871,Save The Planet Mug,11.74,36,12433,Norway +581476,12/9/2019,21533,Retrospot Large Milk Jug,15.32,6,12433,Norway +581476,12/9/2019,21481,Fawn Blue Hot Water Bottle,14.09,12,12433,Norway +581476,12/9/2019,21479,White Skull Hot Water Bottle,14.61,4,12433,Norway +581476,12/9/2019,21248,Door Hanger Mum + Dads Room,11.74,12,12433,Norway +581476,12/9/2019,21216,Set 3 Retrospot Tea/Coffee/Sugar,14.61,24,12433,Norway +581476,12/9/2019,21181,Please One Person Metal Sign,12.15,48,12433,Norway +581476,12/9/2019,21175,Gin And Tonic Diet Metal Sign,12.38,48,12433,Norway +581476,12/9/2019,21169,You're Confusing Me Metal Sign,11.74,48,12433,Norway +581476,12/9/2019,21162,Toxic Area Door Hanger,10.65,24,12433,Norway +581476,12/9/2019,21159,Moody Boy Door Hanger,10.65,24,12433,Norway +581476,12/9/2019,21158,Moody Girl Door Hanger,10.65,24,12433,Norway +581476,12/9/2019,21154,Red Retrospot Oven Glove,11.53,10,12433,Norway +581476,12/9/2019,16016,Large Chinese Style Scissor,11.12,40,12433,Norway +581476,12/9/2019,16014,Small Chinese Style Scissor,10.68,60,12433,Norway +581476,12/9/2019,16008,Small Folding Scissor(Pointed Edge),10.37,240,12433,Norway +581476,12/9/2019,85152,Hand Over The Chocolate Sign,12.15,48,12433,Norway +581476,12/9/2019,84596F,Small Marshmallows Pink Bowl,10.68,32,12433,Norway +581476,12/9/2019,84596B,Small Dolly Mix Design Orange Bowl,10.68,16,12433,Norway +581476,12/9/2019,84510A,Set Of 4 English Rose Coasters,11.53,20,12433,Norway +581476,12/9/2019,82600,N0 Singing Metal Sign,12.15,48,12433,Norway +581476,12/9/2019,82581,Toilet Metal Sign,10.81,48,12433,Norway +581476,12/9/2019,72232,Feng Shui Pillar Candle,10.44,144,12433,Norway +581476,12/9/2019,47559B,Tea Time Oven Glove,11.53,10,12433,Norway +581476,12/9/2019,47504H,English Rose Spirit Level,11.06,36,12433,Norway +581476,12/9/2019,23493,Vintage Doily Travel Sewing Kit,12.25,30,12433,Norway +581476,12/9/2019,23430,Blue Retro Kitchen Wall Clock,18.60,2,12433,Norway +581476,12/9/2019,23429,Red Retro Kitchen Wall Clock,18.60,2,12433,Norway +581476,12/9/2019,23428,Ivory Retro Kitchen Wall Clock,18.60,2,12433,Norway +581476,12/9/2019,23358,Hot Stuff Hot Water Bottle,11.53,18,12433,Norway +581476,12/9/2019,23293,Set Of 12 Fairy Cake Baking Cases,11.10,32,12433,Norway +581476,12/9/2019,23243,Set Of Tea Coffee Sugar Tins Pantry,15.32,12,12433,Norway +581476,12/9/2019,23240,Set Of 4 Knick Knack Tins Doily,14.50,6,12433,Norway +581477,12/9/2019,48111,Doormat 3 Smiley Cats,17.51,10,13426,United Kingdom +581477,12/9/2019,22464,Hanging Metal Heart Lantern,11.06,12,13426,United Kingdom +581477,12/9/2019,20982,12 Pencils Tall Tube Skulls,11.12,12,13426,United Kingdom +581477,12/9/2019,20981,12 Pencils Tall Tube Woodland,11.12,12,13426,United Kingdom +581477,12/9/2019,23424,Gingham Recipe Book Box,15.32,8,13426,United Kingdom +581477,12/9/2019,23338,Egg Frying Pan Red,12.15,48,13426,United Kingdom +581477,12/9/2019,84970L,Single Heart Zinc T-Light Holder,11.53,12,13426,United King +581477,12/9/2019,22457,Natural Slate Heart Chalkboard,13.27,6,13426,United Kingdom +581477,12/9/2019,22469,Heart Of Wicker Small,11.94,12,13426,United Kingdom +581477,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,11.12,12,13426,United Ki +581478,12/9/2019,84947,Antique Silver Tea Glass Engraved,11.53,24,17364,United King +581478,12/9/2019,23503,Playing Cards Keep Calm & Carry On,11.53,12,17364,United Kin +581478,12/9/2019,23445,Ice Cream Bubbles,11.10,20,17364,United Kingdom +581478,12/9/2019,23530,Wall Art Only One Person,15.32,12,17364,United Kingdom +581478,12/9/2019,84997C,Childrens Cutlery Polkadot Blue,14.50,4,17364,United Kingdo +581478,12/9/2019,84997D,Childrens Cutlery Polkadot Pink,14.50,4,17364,United Kingdo +581478,12/9/2019,84077,World War 2 Gliders Asstd Designs,10.55,48,17364,United King +581478,12/9/2019,22749,Feltcraft Princess Charlotte Doll,14.09,4,17364,United Kingd +581478,12/9/2019,23127,Feltcraft Girl Nicole Kit,15.32,4,17364,United Kingdom +581478,12/9/2019,23126,Feltcraft Girl Amelie Kit,15.32,4,17364,United Kingdom +581478,12/9/2019,22747,Poppy's Playhouse Bathroom,12.40,6,17364,United Kingdom +581478,12/9/2019,22078,Ribbon Reel Lace Design,12.40,10,17364,United Kingdom +581478,12/9/2019,84946,Antique Silver T-Light Glass,11.53,12,17364,United Kingdom +581478,12/9/2019,22791,T-Light Glass Fluted Antique,11.53,12,17364,United Kingdom +581478,12/9/2019,21326,Aged Glass Silver T-Light Holder,10.92,12,17364,United Kingd +581478,12/9/2019,22170,Picture Frame Wood Triple Portrait,17.17,8,17364,United King +581478,12/9/2019,23082,Set 6 Paper Table Lantern Hearts,14.09,6,17364,United Kingdo +581478,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,11.12,24,17364,United Ki +581479,12/9/2019,22087,Paper Bunting White Lace,13.27,10,17364,United Kingdom +581480,12/9/2019,23464,Vintage Zinc Watering Can Small,15.32,4,14441,United Kingdom +581480,12/9/2019,22411,Jumbo Shopper Vintage Red Paisley,12.38,10,14441,United King +581480,12/9/2019,84029E,Red Woolly Hottie White Heart,14.61,8,14441,United Kingdom +581480,12/9/2019,22633,Hand Warmer Union Jack,12.40,12,14441,United Kingdom +581480,12/9/2019,23355,Hot Water Bottle Keep Calm,15.32,12,14441,United Kingdom +581480,12/9/2019,84029G,Knitted Union Flag Hot Water Bottle,14.61,12,14441,United K +581480,12/9/2019,22112,Chocolate Hot Water Bottle,15.32,6,14441,United Kingdom +581480,12/9/2019,84347,Rotating Silver Angels T-Light Hldr,12.86,18,14441,United Ki +581481,12/9/2019,21115,Rose Caravan Doorstop,12.25,8,17490,United Kingdom +581481,12/9/2019,22059,Ceramic Strawberry Design Mug,10.65,24,17490,United Kingdom +581481,12/9/2019,22072,Red Retrospot Tea Cup And Saucer,11.53,24,17490,United Kingd +581481,12/9/2019,22123,Ping Microwave Apron,11.06,24,17490,United Kingdom +581481,12/9/2019,22476,Empire Union Jack Tv Dinner Tray,12.25,8,17490,United Kingdo +581481,12/9/2019,22495,Set Of 2 Round Tins Camembert,11.06,12,17490,United Kingdom +581481,12/9/2019,22496,Set Of 2 Round Tins Dutch Cheese,11.06,12,17490,United Kingd +581481,12/9/2019,22513,Doorstop Football Design,11.06,8,17490,United Kingdom +581481,12/9/2019,22500,Set Of 2 Tins Jardin De Provence,11.53,12,17490,United Kingd +581481,12/9/2019,22664,Toy Tidy Dolly Girl Design,11.06,20,17490,United Kingdom +581481,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,12.25,12,17490,United Kingdom +581481,12/9/2019,22785,Squarecushion Cover Pink Union Jack,11.53,12,17490,United Ki +581481,12/9/2019,23178,Jam Clock Magnet,11.53,12,17490,United Kingdom +581481,12/9/2019,23302,Kneeling Mat Housework Design,11.06,24,17490,United Kingdom +581481,12/9/2019,23533,Wall Art Garden Haven,12.25,6,17490,United Kingdom +581481,12/9/2019,84819,Danish Rose Round Sewing Box,11.06,16,17490,United Kingdom +581482,12/9/2019,22371,Airline Bag Vintage Tokyo 78,14.30,12,17490,United Kingdom +581482,12/9/2019,21875,Kings Choice Mug,11.34,36,17490,United Kingdom +581482,12/9/2019,23251,Vintage Red Enamel Trim Mug,11.32,96,17490,United Kingdom +581482,12/9/2019,23300,Gardeners Kneeling Pad Cup Of Tea,11.74,48,17490,United King +581482,12/9/2019,22138,Baking Set 9 Piece Retrospot,14.61,24,17490,United Kingdom +581483,12/9/2019,23843,Paper Craft Little Birdie,12.38,80995,16446,United Kingdom +581485,12/9/2019,22617,Baking Set Spaceboy Design,15.32,18,17389,United Kingdom +581485,12/9/2019,20749,Assorted Colour Mini Cases,16.76,84,17389,United Kingdom +581486,12/9/2019,22910,Paper Chain Kit Vintage Christmas,13.27,12,17001,United King +581486,12/9/2019,22086,Paper Chain Kit 50'S Christmas,13.27,12,17001,United Kingdom +581486,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,12,17001,United Kingdom +581486,12/9/2019,23352,Roll Wrap 50'S Red Christmas,6.19,12,17001,United Kingdom +581486,12/9/2019,22727,Alarm Clock Bakelike Red,6.19,8,17001,United Kingdom +581486,12/9/2019,22730,Alarm Clock Bakelike Ivory,6.19,4,17001,United Kingdom +581486,12/9/2019,22491,Pack Of 12 Coloured Pencils,6.04,12,17001,United Kingdom +581486,12/9/2019,22561,Wooden School Colouring Set,6.04,12,17001,United Kingdom +581486,12/9/2019,22489,Pack Of 12 Traditional Crayons,6.04,24,17001,United Kingdom +581486,12/9/2019,22560,Traditional Modelling Clay,6.04,24,17001,United Kingdom +581486,12/9/2019,22139,Retrospot Tea Set Ceramic 11 Pc,6.04,6,17001,United Kingdom +581486,12/9/2019,23344,Jumbo Bag 50'S Christmas,6.19,20,17001,United Kingdom +581486,12/9/2019,23203,Jumbo Bag Vintage Doily,6.19,20,17001,United Kingdom +581486,12/9/2019,85099B,Jumbo Bag Red Retrospot,6.19,10,17001,United Kingdom +581486,12/9/2019,23201,Jumbo Bag Alphabet,6.19,20,17001,United Kingdom +581486,12/9/2019,23207,Lunch Bag Alphabet Design,6.19,10,17001,United Kingdom +581487,12/9/2019,21137,Black Record Cover Frame,6.04,120,15694,United Kingdom +581488,12/9/2019,23118,Parisienne Jewellery Drawer,6.04,16,17428,United Kingdom +581488,12/9/2019,23111,Parisienne Sewing Box,6.04,16,17428,United Kingdom +581488,12/9/2019,22179,Set 10 Night Owl Lights,6.04,24,17428,United Kingdom +581489,12/9/2019,22061,Large Cake Stand Hanging Strawbery,7.24,48,16954,United King +581489,12/9/2019,22182,Cake Stand Victorian Filigree Small,7.24,24,16954,United Kin +581491,12/9/2019,23571,Traditional Naughts & Crosses,7.24,12,12433,Norway +581492,12/9/2019,23369,Set 36 Colour Pencils Love London,6.19,2,15492,United Kingdo +581492,12/9/2019,23370,Set 36 Colouring Pencils Doily,6.19,2,15492,United Kingdom +581492,12/9/2019,23371,Set 36 Colour Pencils Spaceboy,6.19,3,15492,United Kingdom +581492,12/9/2019,23372,Set 36 Colour Pencils Dolly Girl,6.19,1,15492,United Kingdom +581492,12/9/2019,23376,Pack Of 12 Vintage Christmas Tissue,6.19,3,15492,United King +581492,12/9/2019,23377,Pack Of 12 Dolly Girl Tissues,6.19,6,15492,United Kingdom +581492,12/9/2019,23378,Pack Of 12 50'S Christmas Tissues,6.19,9,15492,United Kingdo +581492,12/9/2019,23379,Pack Of 12 Red Apple Tissues,6.19,4,15492,United Kingdom +581492,12/9/2019,23380,Pack Of 12 Vintage Doily Tissues,6.19,3,15492,United Kingdom +581492,12/9/2019,23381,Pack Of 12 Vintage Leaf Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,23382,Box Of 6 Christmas Cake Decorations,6.19,4,15492,United King +581492,12/9/2019,23391,I Love London Mini Backpack,6.19,2,15492,United Kingdom +581492,12/9/2019,23392,Spaceboy Rocket Lolly Makers,6.19,2,15492,United Kingdom +581492,12/9/2019,23399,Home Sweet Home Hanging Heart,6.19,4,15492,United Kingdom +581492,12/9/2019,23405,Home Sweet Home 2 Drawer Cabinet,6.19,3,15492,United Kingdom +581492,12/9/2019,23418,Lavender Toilette Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,23426,Metal Sign Drop Your Pants,6.19,1,15492,United Kingdom +581492,12/9/2019,23434,3 Raffia Ribbons 50'S Christmas,6.19,9,15492,United Kingdom +581492,12/9/2019,23435,3 Raffia Ribbons Vintage Christmas,6.04,3,15492,United Kingd +581492,12/9/2019,23439,Hand Warmer Red Love Heart,6.19,1,15492,United Kingdom +581492,12/9/2019,23451,Square Mini Portrait Frame,6.19,1,15492,United Kingdom +581492,12/9/2019,23467,Vintage Zinc Planter,6.19,1,15492,United Kingdom +581492,12/9/2019,23469,Card Holder Love Bird Small,6.19,3,15492,United Kingdom +581492,12/9/2019,23480,Mini Lights Woodland Mushrooms,6.19,2,15492,United Kingdom +581492,12/9/2019,23493,Vintage Doily Travel Sewing Kit,6.19,3,15492,United Kingdom +581492,12/9/2019,23494,Vintage Doily Deluxe Sewing Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,23495,Set Of 3 Pantry Wooden Spoons,6.19,1,15492,United Kingdom +581492,12/9/2019,23497,Classic Chrome Bicycle Bell,6.19,4,15492,United Kingdom +581492,12/9/2019,23498,Classic Bicycle Clips,6.19,2,15492,United Kingdom +581492,12/9/2019,23501,Key Ring Baseball Boot Union Jack,6.19,3,15492,United Kingdo +581492,12/9/2019,23506,Mini Playing Cards Spaceboy,6.04,1,15492,United Kingdom +581492,12/9/2019,23508,Mini Playing Cards Dolly Girl,6.04,2,15492,United Kingdom +581492,12/9/2019,23510,Mini Playing Cards Gymkhana,6.04,1,15492,United Kingdom +581492,12/9/2019,23521,Wall Art Cat And Bird,6.04,1,15492,United Kingdom +581492,12/9/2019,23526,Wall Art Dog Licence,6.04,4,15492,United Kingdom +581492,12/9/2019,23530,Wall Art Only One Person,6.04,2,15492,United Kingdom +581492,12/9/2019,23534,Wall Art Stop For Tea,6.19,6,15492,United Kingdom +581492,12/9/2019,23535,Wall Art Bicycle Safety,6.19,4,15492,United Kingdom +581492,12/9/2019,23536,Wall Art Village Show,6.19,1,15492,United Kingdom +581492,12/9/2019,23538,Wall Art Vintage Heart,6.19,1,15492,United Kingdom +581492,12/9/2019,23551,Pack Of 12 Paisley Park Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,23552,Bicycle Puncture Repair Kit,6.19,12,15492,United Kingdom +581492,12/9/2019,23555,Landmark Frame Notting Hill,6.19,1,15492,United Kingdom +581492,12/9/2019,23559,Woodland Bunnies Lolly Makers,6.19,5,15492,United Kingdom +581492,12/9/2019,23561,Set Of 6 Ribbons Party,6.19,1,15492,United Kingdom +581492,12/9/2019,23564,Egg Cup Milkmaid Ingrid,6.19,2,15492,United Kingdom +581492,12/9/2019,23565,Egg Cup Milkmaid Helga,6.19,2,15492,United Kingdom +581492,12/9/2019,23567,Egg Cup Henrietta Hen Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23569,Tradtional Alphabet Stamp Set,6.19,2,15492,United Kingdom +581492,12/9/2019,23570,Traditional Pick Up Sticks Game,6.19,7,15492,United Kingdom +581492,12/9/2019,23571,Traditional Naughts & Crosses,6.19,2,15492,United Kingdom +581492,12/9/2019,23575,Snack Tray Paisley Park,6.19,3,15492,United Kingdom +581492,12/9/2019,23579,Snack Tray I Love London,6.19,1,15492,United Kingdom +581492,12/9/2019,23611,Set 10 Cards Red Riding Hood 17214,6.19,3,15492,United Kingd +581492,12/9/2019,23616,Set 10 Cards Jingle Bells 17217,6.19,1,15492,United Kingdom +581492,12/9/2019,23621,Set 10 Cards David's Madonna 17074,6.19,3,15492,United Kingd +581492,12/9/2019,23635,Set 10 Cards Christmas Holly 17259,6.19,1,15492,United Kingd +581492,12/9/2019,23644,Set 10 Cards Christmas Tree 16955,6.19,1,15492,United Kingdo +581492,12/9/2019,23660,Henrietta Hen Mug,6.19,1,15492,United Kingdom +581492,12/9/2019,23661,Milk Maids Mug,6.19,1,15492,United Kingdom +581492,12/9/2019,35095A,Blue Victorian Fabric Oval Box,6.19,1,15492,United Kingdom +581492,12/9/2019,35095B,Red Victorian Fabric Oval Box,6.19,1,15492,United Kingdom +581492,12/9/2019,35646,Vintage Bead Pink Evening Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,35648,Vintage Bead Pink Purse,6.19,2,15492,United Kingdom +581492,12/9/2019,35953,Folkart Star Christmas Decorations,6.19,12,15492,United King +581492,12/9/2019,35964,Folkart Clip On Stars,6.19,4,15492,United Kingdom +581492,12/9/2019,35970,Zinc Folkart Sleigh Bells,6.04,1,15492,United Kingdom +581492,12/9/2019,46118,Funky Monkey Cushion Cover,6.19,1,15492,United Kingdom +581492,12/9/2019,46776A,Woven Bubble Gum Cushion Cover,7.24,2,15492,United Kingdom +581492,12/9/2019,46776B,Woven Berries Cushion Cover,7.24,3,15492,United Kingdom +581492,12/9/2019,46776C,Woven Frost Cushion Cover,7.24,1,15492,United Kingdom +581492,12/9/2019,46776D,Woven Sunset Cushion Cover,7.24,2,15492,United Kingdom +581492,12/9/2019,46776E,Woven Candy Cushion Cover,7.24,5,15492,United Kingdom +581492,12/9/2019,46776F,Woven Rose Garden Cushion Cover,7.24,6,15492,United Kingdom +581492,12/9/2019,47310M,Small Pop Box Funky Monkey,6.19,1,15492,United Kingdom +581492,12/9/2019,47480,Hanging Photo Clip Rope Ladder,6.19,3,15492,United Kingdom +581492,12/9/2019,47504H,English Rose Spirit Level,7.24,1,15492,United Kingdom +581492,12/9/2019,47559B,Tea Time Oven Glove,7.24,3,15492,United Kingdom +581492,12/9/2019,47563A,Retro Longboard Ironing Board Cover,7.24,2,15492,United Kin +581492,12/9/2019,47566,Party Bunting,7.24,2,15492,United Kingdom +581492,12/9/2019,47590B,Pink Happy Birthday Bunting,7.24,1,15492,United Kingdom +581492,12/9/2019,51014A,Feather Pen Hot Pink,7.24,2,15492,United Kingdom +581492,12/9/2019,51014L,Feather Pen Light Pink,7.24,1,15492,United Kingdom +581492,12/9/2019,71270,Photo Clip Line,7.24,1,15492,United Kingdom +581492,12/9/2019,72349B,Set/6 Purple Butterfly T-Lights,6.39,1,15492,United Kingdom +581492,12/9/2019,72741,Grand Chocolatecandle,7.24,3,15492,United Kingdom +581492,12/9/2019,72800E,4 Ivory Dinner Candles Silver Flock,7.24,3,15492,United Kin +581492,12/9/2019,79321,Chilli Lights,6.19,1,15492,United Kingdom +581492,12/9/2019,82484,Wood Black Board Ant White Finish,6.19,2,15492,United Kingdo +581492,12/9/2019,82567,Airline Lounge Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,82582,Area Patrolled Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,82600,N0 Singing Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,84031A,Charlie+Lola Red Hot Water Bottle,6.19,4,15492,United Kingd +581492,12/9/2019,84077,World War 2 Gliders Asstd Designs,6.19,1,15492,United Kingdo +581492,12/9/2019,84249A,Greeting Card Square Doughnuts,6.19,1,15492,United Kingdom +581492,12/9/2019,84347,Rotating Silver Angels T-Light Hldr,6.19,2,15492,United King +581492,12/9/2019,84375,Set Of 20 Kids Cookie Cutters,6.19,1,15492,United Kingdom +581492,12/9/2019,84378,Set Of 3 Heart Cookie Cutters,6.19,4,15492,United Kingdom +581492,12/9/2019,84519B,Carrot Charlie+Lola Coaster Set,6.19,1,15492,United Kingdom +581492,12/9/2019,84559A,3d Sheet Of Dog Stickers,6.19,2,15492,United Kingdom +581492,12/9/2019,84568,Girls Alphabet Iron On Patches,6.19,31,15492,United Kingdom +581492,12/9/2019,84569D,Pack 6 Heart/Ice-Cream Patches,6.19,1,15492,United Kingdom +581492,12/9/2019,84596B,Small Dolly Mix Design Orange Bowl,6.19,4,15492,United King +581492,12/9/2019,84596F,Small Marshmallows Pink Bowl,6.19,4,15492,United Kingdom +581492,12/9/2019,84598,Boys Alphabet Iron On Patches,6.19,5,15492,United Kingdom +581492,12/9/2019,84692,Box Of 24 Cocktail Parasols,6.19,1,15492,United Kingdom +581492,12/9/2019,84755,Colour Glass T-Light Holder Hanging,6.19,4,15492,United King +581492,12/9/2019,84828,Jungle Popsicles Ice Lolly Moulds,6.19,1,15492,United Kingdo +581492,12/9/2019,84879,Assorted Colour Bird Ornament,6.19,16,15492,United Kingdom +581492,12/9/2019,84923,Pink Butterfly Handbag W Bobbles,6.04,3,15492,United Kingdom +581492,12/9/2019,84946,Antique Silver T-Light Glass,6.04,18,15492,United Kingdom +581492,12/9/2019,84970S,Hanging Heart Zinc T-Light Holder,6.04,3,15492,United Kingd +581492,12/9/2019,84978,Hanging Heart Jar T-Light Holder,6.04,4,15492,United Kingdom +581492,12/9/2019,84991,60 Teatime Fairy Cake Cases,6.04,1,15492,United Kingdom +581492,12/9/2019,84997A,Childrens Cutlery Polkadot Green,6.04,1,15492,United Kingdo +581492,12/9/2019,84997B,Childrens Cutlery Retrospot Red,6.04,1,15492,United Kingdom +581492,12/9/2019,20733,Gold Mini Tape Measure,6.04,3,15492,United Kingdom +581492,12/9/2019,20777,Chrysanthemum Notebook,6.19,2,15492,United Kingdom +581492,12/9/2019,20782,Camouflage Ear Muff Headphones,6.19,1,15492,United Kingdom +581492,12/9/2019,20914,Set/5 Red Retrospot Lid Glass Bowls,6.19,2,15492,United King +581492,12/9/2019,20931,Blue Pot Plant Candle,6.19,1,15492,United Kingdom +581492,12/9/2019,20970,Pink Floral Feltcraft Shoulder Bag,6.19,1,15492,United Kingd +581492,12/9/2019,20971,Pink Blue Felt Craft Trinket Box,6.19,2,15492,United Kingdom +581492,12/9/2019,20972,Pink Cream Felt Craft Trinket Box,6.19,3,15492,United Kingdo +581492,12/9/2019,20973,12 Pencil Small Tube Woodland,6.19,4,15492,United Kingdom +581492,12/9/2019,20978,36 Pencils Tube Skulls,6.19,1,15492,United Kingdom +581492,12/9/2019,20979,36 Pencils Tube Red Retrospot,6.19,3,15492,United Kingdom +581492,12/9/2019,20982,12 Pencils Tall Tube Skulls,6.19,2,15492,United Kingdom +581492,12/9/2019,20983,12 Pencils Tall Tube Red Retrospot,6.19,4,15492,United Kingd +581492,12/9/2019,20985,Heart Calculator,6.19,1,15492,United Kingdom +581492,12/9/2019,20986,Blue Calculator Ruler,6.19,1,15492,United Kingdom +581492,12/9/2019,21000,Rose Du Sud Cosmetics Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,21012,Antique All Glass Candlestick,6.19,3,15492,United Kingdom +581492,12/9/2019,21015,Dark Bird House Tree Decoration,6.19,8,15492,United Kingdom +581492,12/9/2019,21026,Space Owl,6.19,1,15492,United Kingdom +581492,12/9/2019,21035,Set/2 Red Retrospot Tea Towels,6.19,1,15492,United Kingdom +581492,12/9/2019,21064,Boom Box Speaker Boys,6.19,4,15492,United Kingdom +581492,12/9/2019,21065,Boom Box Speaker Girls,6.19,2,15492,United Kingdom +581492,12/9/2019,21098,Christmas Toilet Roll,6.19,1,15492,United Kingdom +581492,12/9/2019,21123,Set/10 Ivory Polkadot Party Candles,6.19,1,15492,United King +581492,12/9/2019,21125,Set 6 Football Celebration Candles,6.19,1,15492,United Kingd +581492,12/9/2019,21126,Set Of 6 Girls Celebration Candles,6.19,1,15492,United Kingd +581492,12/9/2019,21137,Black Record Cover Frame,6.19,17,15492,United Kingdom +581492,12/9/2019,21154,Red Retrospot Oven Glove,6.19,2,15492,United Kingdom +581492,12/9/2019,21158,Moody Girl Door Hanger,6.19,1,15492,United Kingdom +581492,12/9/2019,21162,Toxic Area Door Hanger,6.19,1,15492,United Kingdom +581492,12/9/2019,21166,Cook With Wine Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,21172,Party Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,21174,Pottering In The Shed Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,21175,Gin And Tonic Diet Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,21200,Multicolour Honeycomb Paper Garland,6.04,6,15492,United King +581492,12/9/2019,21201,Tropical Honeycomb Paper Garland,6.04,4,15492,United Kingdom +581492,12/9/2019,21212,Pack Of 72 Retrospot Cake Cases,6.04,2,15492,United Kingdom +581492,12/9/2019,21213,Pack Of 72 Skull Cake Cases,6.04,2,15492,United Kingdom +581492,12/9/2019,21220,Set/4 Badges Dogs,6.19,2,15492,United Kingdom +581492,12/9/2019,21224,Set/4 Skull Badges,6.19,1,15492,United Kingdom +581492,12/9/2019,21232,Strawberry Ceramic Trinket Pot,6.19,1,15492,United Kingdom +581492,12/9/2019,21257,Victorian Sewing Box Medium,6.19,2,15492,United Kingdom +581492,12/9/2019,21258,Victorian Sewing Box Large,6.19,1,15492,United Kingdom +581492,12/9/2019,21259,Victorian Sewing Box Small,6.19,1,15492,United Kingdom +581492,12/9/2019,21313,Glass Heart T-Light Holder,6.19,1,15492,United Kingdom +581492,12/9/2019,21314,Small Glass Heart Trinket Pot,6.19,2,15492,United Kingdom +581492,12/9/2019,21328,Balloons Writing Set,6.19,2,15492,United Kingdom +581492,12/9/2019,21329,Dinosaurs Writing Set,6.19,2,15492,United Kingdom +581492,12/9/2019,21356,Toast Its - Fairy Flower,6.19,16,15492,United Kingdom +581492,12/9/2019,21378,Small Tall Camphor Wood Toadstool,6.19,2,15492,United Kingdo +581492,12/9/2019,21379,Camphor Wood Portobello Mushroom,6.19,1,15492,United Kingdom +581492,12/9/2019,21383,Pack Of 12 Sticky Bunnies,6.19,1,15492,United Kingdom +581492,12/9/2019,21402,Red Egg Spoon,6.19,1,15492,United Kingdom +581492,12/9/2019,21408,Spotty Pink Duck Doorstop,6.19,2,15492,United Kingdom +581492,12/9/2019,21411,Gingham Heart Doorstop Red,6.19,1,15492,United Kingdom +581492,12/9/2019,21467,Cherry Crochet Food Cover,6.19,1,15492,United Kingdom +581492,12/9/2019,21471,Strawberry Raffia Food Cover,6.19,2,15492,United Kingdom +581492,12/9/2019,21479,White Skull Hot Water Bottle,6.19,2,15492,United Kingdom +581492,12/9/2019,21481,Fawn Blue Hot Water Bottle,6.19,3,15492,United Kingdom +581492,12/9/2019,21484,Chick Grey Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,21485,Retrospot Heart Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,21494,Rotating Leaves T-Light Holder,6.19,11,15492,United Kingdom +581492,12/9/2019,21506,Fancy Font Birthday Card,6.19,3,15492,United Kingdom +581492,12/9/2019,21507,Elephant Birthday Card,7.24,1,15492,United Kingdom +581492,12/9/2019,21508,Vintage Kid Dolly Card,7.24,1,15492,United Kingdom +581492,12/9/2019,21509,Cowboys And Indians Birthday Card,7.24,2,15492,United Kingdo +581492,12/9/2019,21528,Dairy Maid Traditional Teapot,7.24,1,15492,United Kingdom +581492,12/9/2019,21544,Skulls Water Transfer Tattoos,7.24,2,15492,United Kingdom +581492,12/9/2019,21558,Skull Lunch Box With Cutlery,6.39,1,15492,United Kingdom +581492,12/9/2019,21559,Strawberry Lunch Box With Cutlery,7.24,1,15492,United Kingdo +581492,12/9/2019,21615,4 Lavender Botanical Dinner Candles,7.24,2,15492,United King +581492,12/9/2019,21616,4 Pear Botanical Dinner Candles,7.24,6,15492,United Kingdom +581492,12/9/2019,21620,Set Of 4 Rose Botanical Candles,7.24,5,15492,United Kingdom +581492,12/9/2019,21642,Assorted Tutti Frutti Pen,7.24,4,15492,United Kingdom +581492,12/9/2019,21648,Assorted Tutti Frutti Small Purse,7.24,2,15492,United Kingdo +581492,12/9/2019,21650,Assorted Tutti Frutti Bracelet,7.24,14,15492,United Kingdom +581492,12/9/2019,21675,Butterflies Stickers,6.19,2,15492,United Kingdom +581492,12/9/2019,21677,Hearts Stickers,6.19,1,15492,United Kingdom +581492,12/9/2019,21678,Paisley Pattern Stickers,6.19,2,15492,United Kingdom +581492,12/9/2019,21680,Woodland Stickers,6.19,1,15492,United Kingdom +581492,12/9/2019,21703,Bag 125g Swirly Marbles,6.19,1,15492,United Kingdom +581492,12/9/2019,21704,Bag 250g Swirly Marbles,6.19,1,15492,United Kingdom +581492,12/9/2019,21716,Boys Vintage Tin Seaside Bucket,6.19,1,15492,United Kingdom +581492,12/9/2019,21718,Red Metal Beach Spade,6.19,1,15492,United Kingdom +581492,12/9/2019,21724,Panda And Bunnies Sticker Sheet,6.19,1,15492,United Kingdom +581492,12/9/2019,21731,Red Toadstool Led Night Light,6.19,6,15492,United Kingdom +581492,12/9/2019,21739,Cosy Slipper Shoes Small Green,6.19,2,15492,United Kingdom +581492,12/9/2019,21774,Decorative Cats Bathroom Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,21786,Polkadot Rain Hat,6.19,3,15492,United Kingdom +581492,12/9/2019,21787,Rain Poncho Retrospot,6.19,2,15492,United Kingdom +581492,12/9/2019,21790,Vintage Snap Cards,6.19,5,15492,United Kingdom +581492,12/9/2019,21791,Vintage Heads And Tails Card Game,6.19,1,15492,United Kingdo +581492,12/9/2019,21808,Christmas Garland Stars Trees,6.19,3,15492,United Kingdom +581492,12/9/2019,21810,Christmas Hanging Star With Bell,6.19,25,15492,United Kingdo +581492,12/9/2019,21812,Garland With Hearts And Bells,6.19,7,15492,United Kingdom +581492,12/9/2019,21813,Garland With Stars And Bells,6.19,3,15492,United Kingdom +581492,12/9/2019,21822,Glitter Christmas Tree With Bells,6.19,12,15492,United Kingd +581492,12/9/2019,21828,Eight Piece Snake Set,6.19,1,15492,United Kingdom +581492,12/9/2019,21832,Chocolate Calculator,6.19,1,15492,United Kingdom +581492,12/9/2019,21833,Camouflage Led Torch,6.04,2,15492,United Kingdom +581492,12/9/2019,21871,Save The Planet Mug,6.19,1,15492,United Kingdom +581492,12/9/2019,21874,Gin And Tonic Mug,6.19,2,15492,United Kingdom +581492,12/9/2019,21876,Pottering Mug,6.19,4,15492,United Kingdom +581492,12/9/2019,21879,Hearts Gift Tape,6.19,1,15492,United Kingdom +581492,12/9/2019,21889,Wooden Box Of Dominoes,6.19,3,15492,United Kingdom +581492,12/9/2019,21890,S/6 Wooden Skittles In Cotton Bag,6.19,1,15492,United Kingdo +581492,12/9/2019,21892,Traditional Wooden Catch Cup Game,6.19,1,15492,United Kingdo +581492,12/9/2019,21900,Key Fob Shed,6.19,4,15492,United Kingdom +581492,12/9/2019,21901,Key Fob Back Door,6.19,2,15492,United Kingdom +581492,12/9/2019,21902,Key Fob Front Door,6.19,1,15492,United Kingdom +581492,12/9/2019,21905,More Butter Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,21906,Pharmacie First Aid Tin,6.19,1,15492,United Kingdom +581492,12/9/2019,21914,Blue Harmonica In Box,6.19,2,15492,United Kingdom +581492,12/9/2019,21916,Set 12 Retro White Chalk Sticks,6.19,2,15492,United Kingdom +581492,12/9/2019,21930,Jumbo Storage Bag Skulls,6.19,1,15492,United Kingdom +581492,12/9/2019,21931,Jumbo Storage Bag Suki,6.19,1,15492,United Kingdom +581492,12/9/2019,21932,Scandinavian Paisley Picnic Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,21934,Skull Shoulder Bag,6.19,3,15492,United Kingdom +581492,12/9/2019,21935,Suki Shoulder Bag,6.19,9,15492,United Kingdom +581492,12/9/2019,21936,Red Retrospot Picnic Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,21942,Skulls Design Flannel,6.19,2,15492,United Kingdom +581492,12/9/2019,21945,Strawberries Design Flannel,6.19,2,15492,United Kingdom +581492,12/9/2019,21947,Set Of 6 Heart Chopsticks,6.19,1,15492,United Kingdom +581492,12/9/2019,21949,Set Of 6 Strawberry Chopsticks,6.19,2,15492,United Kingdom +581492,12/9/2019,21967,Pack Of 12 Skull Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,21976,Pack Of 60 Mushroom Cake Cases,6.19,3,15492,United Kingdom +581492,12/9/2019,21977,Pack Of 60 Pink Paisley Cake Cases,6.19,2,15492,United Kingd +581492,12/9/2019,21980,Pack Of 12 Red Retrospot Tissues,6.19,2,15492,United Kingdom +581492,12/9/2019,21981,Pack Of 12 Woodland Tissues,6.19,4,15492,United Kingdom +581492,12/9/2019,21982,Pack Of 12 Suki Tissues,6.19,2,15492,United Kingdom +581492,12/9/2019,21983,Pack Of 12 Blue Paisley Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,21984,Pack Of 12 Pink Paisley Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,21986,Pack Of 12 Pink Polkadot Tissues,6.19,3,15492,United Kingdom +581492,12/9/2019,21989,Pack Of 20 Skull Paper Napkins,6.19,2,15492,United Kingdom +581492,12/9/2019,21990,Modern Floral Stationery Set,6.19,8,15492,United Kingdom +581492,12/9/2019,21993,Floral Folk Stationery Set,6.19,8,15492,United Kingdom +581492,12/9/2019,22024,Rainy Ladies Birthday Card,6.19,3,15492,United Kingdom +581492,12/9/2019,22025,Ring Of Roses Birthday Card,6.19,3,15492,United Kingdom +581492,12/9/2019,22026,Banquet Birthday Card,6.19,4,15492,United Kingdom +581492,12/9/2019,22027,Tea Party Birthday Card,6.19,2,15492,United Kingdom +581492,12/9/2019,22028,Penny Farthing Birthday Card,6.19,1,15492,United Kingdom +581492,12/9/2019,22029,Spaceboy Birthday Card,6.19,4,15492,United Kingdom +581492,12/9/2019,22031,Botanical Lavender Birthday Card,6.19,1,15492,United Kingdom +581492,12/9/2019,22037,Robot Birthday Card,6.19,4,15492,United Kingdom +581492,12/9/2019,22064,Pink Doughnut Trinket Pot,6.19,1,15492,United Kingdom +581492,12/9/2019,22065,Christmas Pudding Trinket Pot,6.19,2,15492,United Kingdom +581492,12/9/2019,22068,Black Pirate Treasure Chest,6.19,1,15492,United Kingdom +581492,12/9/2019,22070,Small Red Retrospot Mug In Box,6.19,3,15492,United Kingdom +581492,12/9/2019,22071,Small White Retrospot Mug In Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22074,6 Ribbons Shimmering Pinks,6.19,1,15492,United Kingdom +581492,12/9/2019,22075,6 Ribbons Elegant Christmas,6.19,8,15492,United Kingdom +581492,12/9/2019,22076,6 Ribbons Empire,6.19,4,15492,United Kingdom +581492,12/9/2019,22083,Paper Chain Kit Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22085,Paper Chain Kit Skulls,6.19,1,15492,United Kingdom +581492,12/9/2019,22086,Paper Chain Kit 50'S Christmas,6.19,38,15492,United Kingdom +581492,12/9/2019,22091,Empire Tissue Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22099,Caravan Square Tissue Box,6.19,3,15492,United Kingdom +581492,12/9/2019,22104,Mirror Mosaic Candle Plate,6.19,4,15492,United Kingdom +581492,12/9/2019,22106,Mirror Mosaic Hurricane Lamp,6.19,1,15492,United Kingdom +581492,12/9/2019,22107,Pizza Plate In Box,6.19,10,15492,United Kingdom +581492,12/9/2019,22108,Ping! Microwave Plate,6.04,2,15492,United Kingdom +581492,12/9/2019,22109,Full English Breakfast Plate,6.04,2,15492,United Kingdom +581492,12/9/2019,22110,Bird House Hot Water Bottle,6.04,3,15492,United Kingdom +581492,12/9/2019,22111,Scottie Dog Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,22112,Chocolate Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,6.19,4,15492,United Kingdo +581492,12/9/2019,22116,Metal Sign His Dinner Is Served,6.19,2,15492,United Kingdom +581492,12/9/2019,22121,Noel Wooden Block Letters,6.19,1,15492,United Kingdom +581492,12/9/2019,22123,Ping Microwave Apron,6.19,1,15492,United Kingdom +581492,12/9/2019,22124,Set Of 2 Tea Towels Ping Microwave,6.19,1,15492,United Kingd +581492,12/9/2019,22129,Party Cones Candy Decoration,6.19,3,15492,United Kingdom +581492,12/9/2019,22134,Mini Ladle Love Heart Red,6.19,1,15492,United Kingdom +581492,12/9/2019,15036,Assorted Colours Silk Fan,6.19,1,15492,United Kingdom +581492,12/9/2019,15039,Sandalwood Fan,6.19,1,15492,United Kingdom +581492,12/9/2019,15058C,Ice Cream Design Garden Parasol,6.19,1,15492,United Kingdom +581492,12/9/2019,16218,Cartoon Pencil Sharpeners,6.19,5,15492,United Kingdom +581492,12/9/2019,16225,Rattle Snake Eggs,6.19,1,15492,United Kingdom +581492,12/9/2019,16235,Recycled Pencil With Rabbit Eraser,6.19,3,15492,United Kingd +581492,12/9/2019,16237,Sleeping Cat Erasers,6.19,1,15492,United Kingdom +581492,12/9/2019,17038,Porcelain Budah Incense Holder,6.19,1,15492,United Kingdom +581492,12/9/2019,17084N,Fairy Dreams Incense,6.19,1,15492,United Kingdom +581492,12/9/2019,20659,Economy Luggage Tag,6.19,7,15492,United Kingdom +581492,12/9/2019,20665,Red Retrospot Purse,6.19,2,15492,United Kingdom +581492,12/9/2019,20668,Disco Ball Christmas Decoration,6.19,1,15492,United Kingdom +581492,12/9/2019,20718,Red Retrospot Shopper Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,20719,Woodland Charlotte Bag,6.19,7,15492,United Kingdom +581492,12/9/2019,20723,Strawberry Charlotte Bag,6.19,2,15492,United Kingdom +581492,12/9/2019,20724,Red Retrospot Charlotte Bag,6.19,4,15492,United Kingdom +581492,12/9/2019,22135,Mini Ladle Love Heart Pink,6.19,6,15492,United Kingdom +581492,12/9/2019,22138,Baking Set 9 Piece Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22141,Christmas Craft Tree Top Angel,6.19,2,15492,United Kingdom +581492,12/9/2019,22150,3 Stripey Mice Feltcraft,7.24,2,15492,United Kingdom +581492,12/9/2019,22153,Angel Decoration Stars On Dress,7.24,2,15492,United Kingdom +581492,12/9/2019,22154,Angel Decoration 3 Buttons,7.24,3,15492,United Kingdom +581492,12/9/2019,22155,Star Decoration Rustic,7.24,7,15492,United Kingdom +581492,12/9/2019,22156,Heart Decoration With Pearls,7.24,4,15492,United Kingdom +581492,12/9/2019,22163,Heart String Memo Holder Hanging,7.24,9,15492,United Kingdom +581492,12/9/2019,22165,Diamante Heart Shaped Wall Mirror,7.24,1,15492,United Kingdo +581492,12/9/2019,22167,Oval Wall Mirror Diamante,7.24,1,15492,United Kingdom +581492,12/9/2019,22170,Picture Frame Wood Triple Portrait,7.24,1,15492,United Kingd +581492,12/9/2019,22173,Metal 4 Hook Hanger French Chateau,7.24,2,15492,United Kingd +581492,12/9/2019,22174,Photo Cube,6.19,7,15492,United Kingdom +581492,12/9/2019,22178,Victorian Glass Hanging T-Light,6.19,5,15492,United Kingdom +581492,12/9/2019,22186,Red Star Card Holder,7.24,2,15492,United Kingdom +581492,12/9/2019,22190,Local Cafe Mug,7.24,3,15492,United Kingdom +581492,12/9/2019,22193,Red Diner Wall Clock,7.24,1,15492,United Kingdom +581492,12/9/2019,22195,Large Heart Measuring Spoons,7.24,1,15492,United Kingdom +581492,12/9/2019,22196,Small Heart Measuring Spoons,7.24,11,15492,United Kingdom +581492,12/9/2019,22197,Popcorn Holder,7.24,34,15492,United Kingdom +581492,12/9/2019,22199,Frying Pan Red Retrospot,7.24,1,15492,United Kingdom +581492,12/9/2019,22209,Wood Stamp Set Happy Birthday,7.24,1,15492,United Kingdom +581492,12/9/2019,22212,Four Hook White Lovebirds,7.24,1,15492,United Kingdom +581492,12/9/2019,22219,Lovebird Hanging Decoration White,7.24,4,15492,United Kingdo +581492,12/9/2019,22224,White Lovebird Lantern,7.24,4,15492,United Kingdom +581492,12/9/2019,22227,Hanging Heart Mirror Decoration,7.24,1,15492,United Kingdom +581492,12/9/2019,22260,Felt Egg Cosy Blue Rabbit,6.39,2,15492,United Kingdom +581492,12/9/2019,22261,Felt Egg Cosy White Rabbit,7.24,1,15492,United Kingdom +581492,12/9/2019,22264,Felt Farm Animal White Bunny,7.24,1,15492,United Kingdom +581492,12/9/2019,22273,Feltcraft Doll Molly,7.24,2,15492,United Kingdom +581492,12/9/2019,22277,Cosmetic Bag Vintage Rose Paisley,7.24,1,15492,United Kingdo +581492,12/9/2019,22279,Pocket Bag Blue Paisley Red Spot,7.24,5,15492,United Kingdom +581492,12/9/2019,22280,Pocket Bag Pink Paisely Brown Spot,7.24,4,15492,United Kingd +581492,12/9/2019,22300,Coffee Mug Dog + Ball Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22301,Coffee Mug Cat + Bird Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22307,Gold Mug Bone China Tree Of Life,7.24,1,15492,United Kingdom +581492,12/9/2019,22308,Tea Cosy Blue Stripe,7.24,2,15492,United Kingdom +581492,12/9/2019,22309,Tea Cosy Red Stripe,7.24,2,15492,United Kingdom +581492,12/9/2019,22324,Blue Polkadot Kids Bag,7.24,2,15492,United Kingdom +581492,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,7.24,3,15492,United Kingd +581492,12/9/2019,22327,Round Snack Boxes Set Of 4 Skulls,7.24,2,15492,United Kingdo +581492,12/9/2019,22329,Round Container Set Of 5 Retrospot,6.19,2,15492,United Kingd +581492,12/9/2019,22338,Star Decoration Painted Zinc,6.19,4,15492,United Kingdom +581492,12/9/2019,22340,Noel Garland Painted Zinc,6.19,17,15492,United Kingdom +581492,12/9/2019,22342,Home Garland Painted Zinc,6.19,7,15492,United Kingdom +581492,12/9/2019,22348,Tea Bag Plate Red Retrospot,6.19,3,15492,United Kingdom +581492,12/9/2019,22355,Charlotte Bag Suki Design,6.19,3,15492,United Kingdom +581492,12/9/2019,22356,Charlotte Bag Pink Polkadot,6.19,1,15492,United Kingdom +581492,12/9/2019,22357,Kings Choice Biscuit Tin,6.19,2,15492,United Kingdom +581492,12/9/2019,22358,Kings Choice Tea Caddy,6.19,1,15492,United Kingdom +581492,12/9/2019,22361,Glass Jar Daisy Fresh Cotton Wool,6.19,2,15492,United Kingdo +581492,12/9/2019,22362,Glass Jar Peacock Bath Salts,6.19,2,15492,United Kingdom +581492,12/9/2019,22367,Childrens Apron Spaceboy Design,6.19,1,15492,United Kingdom +581492,12/9/2019,22372,Airline Bag Vintage World Champion,6.19,1,15492,United Kingd +581492,12/9/2019,22374,Airline Bag Vintage Jet Set Red,6.19,1,15492,United Kingdom +581492,12/9/2019,85014B,Red Retrospot Umbrella,6.19,1,15492,United Kingdom +581492,12/9/2019,85032C,Curious Images Gift Wrap Set,6.19,1,15492,United Kingdom +581492,12/9/2019,85038,6 Chocolate Love Heart T-Lights,6.19,1,15492,United Kingdom +581492,12/9/2019,85039A,Set/4 Red Mini Rose Candle In Bowl,6.19,5,15492,United King +581492,12/9/2019,85039B,S/4 Ivory Mini Rose Candle In Bowl,6.19,4,15492,United King +581492,12/9/2019,85040A,S/4 Pink Flower Candles In Bowl,6.19,1,15492,United Kingdom +581492,12/9/2019,85048,15cm Christmas Glass Ball 20 Lights,6.19,2,15492,United King +581492,12/9/2019,85049C,Romantic Pinks Ribbons,6.19,1,15492,United Kingdom +581492,12/9/2019,85049G,Chocolate Box Ribbons,6.19,1,15492,United Kingdom +581492,12/9/2019,85049H,Urban Black Ribbons,6.19,2,15492,United Kingdom +581492,12/9/2019,85053,French Enamel Candleholder,6.19,1,15492,United Kingdom +581492,12/9/2019,85059,French Enamel Water Basin,6.19,1,15492,United Kingdom +581492,12/9/2019,85066,Cream Sweetheart Mini Chest,6.19,1,15492,United Kingdom +581492,12/9/2019,85094,Candy Spot Egg Warmer Rabbit,6.19,2,15492,United Kingdom +581492,12/9/2019,85114C,Red Enchanted Forest Placemat,6.19,1,15492,United Kingdom +581492,12/9/2019,85123A,Cream Hanging Heart T-Light Holder,6.19,3,15492,United King +581492,12/9/2019,85131B,Beaded Crystal Heart Green On Stick,6.19,1,15492,United Kin +581492,12/9/2019,85131D,Beaded Crystal Heart Pink On Stick,6.19,2,15492,United King +581492,12/9/2019,85152,Hand Over The Chocolate Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,85168B,Black Baroque Carriage Clock,6.19,3,15492,United Kingdom +581492,12/9/2019,85169C,Eau De Nil Love Bird Candle,6.19,1,15492,United Kingdom +581492,12/9/2019,85170C,Set/6 Eau De Nil Bird T-Lights,6.19,1,15492,United Kingdom +581492,12/9/2019,85173,Set/6 Frog Prince T-Light Candles,6.19,1,15492,United Kingdo +581492,12/9/2019,85177,Basket Of Flowers Sewing Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,85178,Victorian Sewing Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,85179A,Green Bitty Light Chain,6.19,4,15492,United Kingdom +581492,12/9/2019,85199S,Small Hanging Ivory/Red Wood Bird,6.19,9,15492,United Kingd +581492,12/9/2019,85227,Set Of 6 3d Kit Cards For Kids,6.19,3,15492,United Kingdom +581492,12/9/2019,90003C,Midnight Blue Pair Heart Hair Slide,6.19,1,15492,United Kin +581492,12/9/2019,90003E,Green Pair Heart Hair Slides,6.19,2,15492,United Kingdom +581492,12/9/2019,90010A,Midnight Blue Glass/Silver Bracelet,6.04,1,15492,United Kin +581492,12/9/2019,90013A,Midnight Blue Vintage Earrings,6.19,3,15492,United Kingdom +581492,12/9/2019,90013C,Green Vintage Earrings,6.19,2,15492,United Kingdom +581492,12/9/2019,90014A,Silver Mop Orbit Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90016B,Gold/Mop Pendant Orbit Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90018B,Gold Mop Orbit Drop Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90019B,Gold Mop Orbit Bracelet,6.19,1,15492,United Kingdom +581492,12/9/2019,90027D,Glass Bead Hoop Earrings Amethyst,6.19,1,15492,United Kingd +581492,12/9/2019,90030B,Red Kukui Coconut Seed Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90055,Cracked Glaze Earrings Brown,6.19,1,15492,United Kingdom +581492,12/9/2019,90059A,Diamante Hair Grip Pack/2 Crystal,6.19,1,15492,United Kingd +581492,12/9/2019,90059D,Diamante Hair Grip Pack/2 Peridot,6.19,2,15492,United Kingd +581492,12/9/2019,90059E,Diamante Hair Grip Pack/2 Ruby,6.04,1,15492,United Kingdom +581492,12/9/2019,90059F,Diamante Hair Grip Pack/2 Lt Rose,6.19,1,15492,United Kingd +581492,12/9/2019,90072,Ruby Drop Chandelier Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90086,Crystal Frog Phone Charm,6.19,1,15492,United Kingdom +581492,12/9/2019,90120B,Blue Murano Twist Bracelet,6.19,2,15492,United Kingdom +581492,12/9/2019,90120C,Green Murano Twist Bracelet,6.19,1,15492,United Kingdom +581492,12/9/2019,90130B,Turq Stone/Crystal Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90130C,Green Stone/Crystal Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90134,Old Rose Combo Bead Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90138,White/Pink Mini Crystals Necklace,6.19,1,15492,United Kingdo +581492,12/9/2019,90141B,Ivory Pendant Triple Shell Necklace,6.19,1,15492,United Kin +581492,12/9/2019,90145,Silver Hoop Earrings With Flower,6.19,1,15492,United Kingdom +581492,12/9/2019,90155,Resin Necklace W Pastel Beads,6.19,1,15492,United Kingdom +581492,12/9/2019,90161D,Ant Copper Pink Boudicca Bracelet,6.19,1,15492,United Kingd +581492,12/9/2019,90163A,Pink Rosebud & Pearl Necklace,6.19,2,15492,United Kingdom +581492,12/9/2019,90165B,White Rosebud Pearl Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90168,2 Daisies Hair Comb,6.19,1,15492,United Kingdom +581492,12/9/2019,90169,Daisy Hair Comb,6.19,1,15492,United Kingdom +581492,12/9/2019,90170,Daisy Hair Band,6.19,1,15492,United Kingdom +581492,12/9/2019,90174,Butterfly Hair Band,6.19,2,15492,United Kingdom +581492,12/9/2019,90175A,White Glass Chunky Charm Bracelet,6.19,2,15492,United Kingd +581492,12/9/2019,90175D,Tigris Eye Chunky Charm Bracelet,6.19,1,15492,United Kingdo +581492,12/9/2019,90177A,Classic Diamante Earrings Jet,6.19,1,15492,United Kingdom +581492,12/9/2019,90177C,Drop Diamante Earrings Crystal,6.19,1,15492,United Kingdom +581492,12/9/2019,90181B,Amethyst Glass/Shell/Pearl Necklace,6.04,1,15492,United Kin +581492,12/9/2019,90182C,Black 3 Bead Drop Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90183A,Amber Drop Earrings W Long Beads,6.19,2,15492,United Kingdo +581492,12/9/2019,90183C,Black Drop Earrings W Long Beads,6.19,1,15492,United Kingdo +581492,12/9/2019,90184C,Black Chunky Bead Bracelet W Strap,6.19,1,15492,United King +581492,12/9/2019,90185B,Amethyst Diamante Expandable Ring,6.19,1,15492,United Kingd +581492,12/9/2019,90188,Drop Earrings W Flower & Leaf,6.19,2,15492,United Kingdom +581492,12/9/2019,90191,Silver Lariat 40cm,6.19,2,15492,United Kingdom +581492,12/9/2019,90192,Jade Drop Earrings W Filigree,6.19,1,15492,United Kingdom +581492,12/9/2019,90198A,Vintage Rose Bead Bracelet Raspberr,6.19,2,15492,United Kin +581492,12/9/2019,90200A,Purple Sweetheart Bracelet,6.19,1,15492,United Kingdom +581492,12/9/2019,90201A,Purple Enamel Flower Ring,6.19,2,15492,United Kingdom +581492,12/9/2019,90201B,Black Enamel Flower Ring,6.19,1,15492,United Kingdom +581492,12/9/2019,90201C,Red Enamel Flower Ring,6.19,1,15492,United Kingdom +581492,12/9/2019,90201D,Green Enamel Flower Ring,6.19,1,15492,United Kingdom +581492,12/9/2019,90202C,Green Enamel Flower Hair Tie,6.19,1,15492,United Kingdom +581492,12/9/2019,90202D,Pink Enamel Flower Hair Tie,6.19,1,15492,United Kingdom +581492,12/9/2019,90206C,Crystal Diamante Star Brooch,6.19,1,15492,United Kingdom +581492,12/9/2019,90208,Pair Of Pink Flower Cluster Slide,6.19,1,15492,United Kingdo +581492,12/9/2019,90210A,Grey Acrylic Faceted Bangle,6.19,1,15492,United Kingdom +581492,12/9/2019,22378,Wall Tidy Retrospot,6.19,2,15492,United Kingdom +581492,12/9/2019,22379,Recycling Bag Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22380,Toy Tidy Spaceboy,6.19,2,15492,United Kingdom +581492,12/9/2019,22396,Magnets Pack Of 4 Retro Photo,6.19,1,15492,United Kingdom +581492,12/9/2019,22411,Jumbo Shopper Vintage Red Paisley,6.19,1,15492,United Kingdo +581492,12/9/2019,22418,10 Colour Spaceboy Pen,6.19,3,15492,United Kingdom +581492,12/9/2019,22419,Lipstick Pen Red,6.19,3,15492,United Kingdom +581492,12/9/2019,22420,Lipstick Pen Baby Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,22421,Lipstick Pen Fuschia,6.19,2,15492,United Kingdom +581492,12/9/2019,22422,Toothpaste Tube Pen,6.19,4,15492,United Kingdom +581492,12/9/2019,22437,Set Of 9 Black Skull Balloons,7.24,1,15492,United Kingdom +581492,12/9/2019,22441,Grow Your Own Basil In Enamel Mug,7.24,1,15492,United Kingdo +581492,12/9/2019,22446,Pin Cushion Babushka Pink,7.24,7,15492,United Kingdom +581492,12/9/2019,22457,Natural Slate Heart Chalkboard,7.24,1,15492,United Kingdom +581492,12/9/2019,22464,Hanging Metal Heart Lantern,7.24,1,15492,United Kingdom +581492,12/9/2019,22466,Fairy Tale Cottage Night Light,7.24,1,15492,United Kingdom +581492,12/9/2019,22467,Gumball Coat Rack,7.24,1,15492,United Kingdom +581492,12/9/2019,22478,Birdhouse Garden Marker,7.24,1,15492,United Kingdom +581492,12/9/2019,22486,Plasmatronic Lamp,7.24,1,15492,United Kingdom +581492,12/9/2019,22488,Natural Slate Rectangle Chalkboard,7.24,1,15492,United Kingd +581492,12/9/2019,22489,Pack Of 12 Traditional Crayons,7.24,5,15492,United Kingdom +581492,12/9/2019,22495,Set Of 2 Round Tins Camembert,7.24,1,15492,United Kingdom +581492,12/9/2019,22540,Mini Jigsaw Circus Parade,7.24,1,15492,United Kingdom +581492,12/9/2019,22544,Mini Jigsaw Spaceboy,7.24,1,15492,United Kingdom +581492,12/9/2019,22549,Picture Dominoes,7.24,3,15492,United Kingdom +581492,12/9/2019,22551,Plasters In Tin Spaceboy,7.24,2,15492,United Kingdom +581492,12/9/2019,22553,Plasters In Tin Skulls,7.24,2,15492,United Kingdom +581492,12/9/2019,22554,Plasters In Tin Woodland Animals,7.24,3,15492,United Kingdom +581492,12/9/2019,22555,Plasters In Tin Strongman,7.24,3,15492,United Kingdom +581492,12/9/2019,22557,Plasters In Tin Vintage Paisley,7.24,2,15492,United Kingdom +581492,12/9/2019,22558,Clothes Pegs Retrospot Pack 24,7.24,3,15492,United Kingdom +581492,12/9/2019,22560,Traditional Modelling Clay,7.24,5,15492,United Kingdom +581492,12/9/2019,22565,Feltcraft Hairbands Pink And White,7.24,4,15492,United Kingd +581492,12/9/2019,22566,Feltcraft Hairband Pink And Purple,7.24,3,15492,United Kingd +581492,12/9/2019,22571,Rocking Horse Red Christmas,7.24,4,15492,United Kingdom +581492,12/9/2019,22573,Star Wooden Christmas Decoration,6.39,2,15492,United Kingdom +581492,12/9/2019,22574,Heart Wooden Christmas Decoration,7.24,5,15492,United Kingdo +581492,12/9/2019,22576,Swallow Wooden Christmas Decoration,7.24,3,15492,United King +581492,12/9/2019,22577,Wooden Heart Christmas Scandinavian,7.24,4,15492,United King +581492,12/9/2019,22578,Wooden Star Christmas Scandinavian,7.24,19,15492,United King +581492,12/9/2019,22580,Advent Calendar Gingham Sack,7.24,9,15492,United Kingdom +581492,12/9/2019,22581,Wood Stocking Christmas Scandispot,7.24,11,15492,United King +581492,12/9/2019,22585,Pack Of 6 Birdy Gift Tags,7.24,2,15492,United Kingdom +581492,12/9/2019,22586,Feltcraft Hairband Pink And Blue,7.24,2,15492,United Kingdom +581492,12/9/2019,22589,Cardholder Gingham Star,7.24,6,15492,United Kingdom +581492,12/9/2019,22591,Cardholder Gingham Christmas Tree,7.24,1,15492,United Kingdo +581492,12/9/2019,22592,Cardholder Holly Wreath Metal,7.24,3,15492,United Kingdom +581492,12/9/2019,22593,Christmas Gingham Star,6.39,2,15492,United Kingdom +581492,12/9/2019,22594,Christmas Gingham Tree,7.24,3,15492,United Kingdom +581492,12/9/2019,22597,Musical Zinc Heart Decoration,7.24,9,15492,United Kingdom +581492,12/9/2019,22598,Christmas Musical Zinc Tree,7.24,4,15492,United Kingdom +581492,12/9/2019,22599,Christmas Musical Zinc Star,6.19,5,15492,United Kingdom +581492,12/9/2019,22600,Christmas Retrospot Star Wood,6.19,2,15492,United Kingdom +581492,12/9/2019,22601,Christmas Retrospot Angel Wood,6.19,3,15492,United Kingdom +581492,12/9/2019,22602,Retrospot Wooden Heart Decoration,6.19,3,15492,United Kingdo +581492,12/9/2019,22608,Pens Assorted Funky Jeweled,6.19,3,15492,United Kingdom +581492,12/9/2019,22614,Pack Of 12 Spaceboy Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,22615,Pack Of 12 Circus Parade Tissues,6.19,4,15492,United Kingdom +581492,12/9/2019,22616,Pack Of 12 London Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,22619,Set Of 6 Soldier Skittles,6.19,4,15492,United Kingdom +581492,12/9/2019,22620,4 Traditional Spinning Tops,6.19,3,15492,United Kingdom +581492,12/9/2019,22621,Traditional Knitting Nancy,6.19,2,15492,United Kingdom +581492,12/9/2019,22629,Spaceboy Lunch Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22630,Dolly Girl Lunch Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22632,Hand Warmer Red Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22633,Hand Warmer Union Jack,6.19,1,15492,United Kingdom +581492,12/9/2019,22634,Childs Breakfast Set Spaceboy,6.19,1,15492,United Kingdom +581492,12/9/2019,22650,Ceramic Pirate Chest Money Bank,6.19,2,15492,United Kingdom +581492,12/9/2019,22651,Gentleman Shirt Repair Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,22653,Button Box,6.19,16,15492,United Kingdom +581492,12/9/2019,22654,Deluxe Sewing Kit,6.19,2,15492,United Kingdom +581492,12/9/2019,22659,Lunch Box I Love London,6.19,1,15492,United Kingdom +581492,12/9/2019,22666,Recipe Box Pantry Yellow Design,6.19,5,15492,United Kingdom +581492,12/9/2019,22693,Grow A Flytrap Or Sunflower In Tin,6.19,3,15492,United Kingd +581492,12/9/2019,22694,Wicker Star,6.19,1,15492,United Kingdom +581492,12/9/2019,22697,Green Regency Teacup And Saucer,6.19,1,15492,United Kingdom +581492,12/9/2019,22699,Roses Regency Teacup And Saucer,6.19,2,15492,United Kingdom +581492,12/9/2019,22701,Pink Dog Bowl,6.19,5,15492,United Kingdom +581492,12/9/2019,22703,Pink Cat Bowl,6.19,6,15492,United Kingdom +581492,12/9/2019,22712,Card Dolly Girl,6.19,1,15492,United Kingdom +581492,12/9/2019,22713,Card I Love London,6.19,1,15492,United Kingdom +581492,12/9/2019,22714,Card Birthday Cowboy,6.19,2,15492,United Kingdom +581492,12/9/2019,22716,Card Circus Parade,6.19,3,15492,United Kingdom +581492,12/9/2019,22717,Card Dog And Ball,6.19,1,15492,United Kingdom +581492,12/9/2019,22720,Set Of 3 Cake Tins Pantry Design,6.19,2,15492,United Kingdom +581492,12/9/2019,22725,Alarm Clock Bakelike Chocolate,6.19,1,15492,United Kingdom +581492,12/9/2019,22726,Alarm Clock Bakelike Green,6.19,4,15492,United Kingdom +581492,12/9/2019,22727,Alarm Clock Bakelike Red,6.19,4,15492,United Kingdom +581492,12/9/2019,22730,Alarm Clock Bakelike Ivory,6.19,1,15492,United Kingdom +581492,12/9/2019,22741,Funky Diva Pen,6.19,4,15492,United Kingdom +581492,12/9/2019,22745,Poppy's Playhouse Bedroom,6.19,2,15492,United Kingdom +581492,12/9/2019,22748,Poppy's Playhouse Kitchen,6.19,1,15492,United Kingdom +581492,12/9/2019,22749,Feltcraft Princess Charlotte Doll,6.19,1,15492,United Kingdo +581492,12/9/2019,22751,Feltcraft Princess Olivia Doll,6.19,2,15492,United Kingdom +581492,12/9/2019,22753,Small Yellow Babushka Notebook,6.19,1,15492,United Kingdom +581492,12/9/2019,22754,Small Red Babushka Notebook,6.19,1,15492,United Kingdom +581492,12/9/2019,22757,Large Red Babushka Notebook,6.19,3,15492,United Kingdom +581492,12/9/2019,22758,Large Purple Babushka Notebook,6.19,2,15492,United Kingdom +581492,12/9/2019,22763,Key Cabinet Ma Campagne,6.19,1,15492,United Kingdom +581492,12/9/2019,22768,Family Photo Frame Cornice,6.19,2,15492,United Kingdom +581492,12/9/2019,22776,Sweetheart 3 Tier Cake Stand,6.19,1,15492,United Kingdom +581492,12/9/2019,22795,Sweetheart Recipe Book Stand,6.19,1,15492,United Kingdom +581492,12/9/2019,22798,Antique Glass Dressing Table Pot,6.19,3,15492,United Kingdom +581492,12/9/2019,22800,Antique Tall Swirlglass Trinket Pot,6.19,3,15492,United King +581492,12/9/2019,22809,Set Of 6 T-Lights Santa,6.19,1,15492,United Kingdom +581492,12/9/2019,22811,Set Of 6 T-Lights Cacti,6.19,1,15492,United Kingdom +581492,12/9/2019,22814,Card Party Games,6.19,9,15492,United Kingdom +581492,12/9/2019,22815,Card Psychedelic Apples,6.19,18,15492,United Kingdom +581492,12/9/2019,22816,Card Motorbike Santa,6.19,2,15492,United Kingdom +581492,12/9/2019,22817,Card Suki Birthday,6.19,5,15492,United Kingdom +581492,12/9/2019,22818,Card Christmas Village,7.24,6,15492,United Kingdom +581492,12/9/2019,22819,Birthday Card Retro Spot,7.24,3,15492,United Kingdom +581492,12/9/2019,22835,Hot Water Bottle I Am So Poorly,7.24,3,15492,United Kingdom +581492,12/9/2019,22865,Hand Warmer Owl Design,7.24,2,15492,United Kingdom +581492,12/9/2019,22866,Hand Warmer Scotty Dog Design,7.24,3,15492,United Kingdom +581492,12/9/2019,22867,Hand Warmer Bird Design,7.24,4,15492,United Kingdom +581492,12/9/2019,22881,Number Tile Vintage Font 2,7.24,1,15492,United Kingdom +581492,12/9/2019,22885,Number Tile Vintage Font 6,7.24,1,15492,United Kingdom +581492,12/9/2019,22888,Number Tile Vintage Font 9,7.24,1,15492,United Kingdom +581492,12/9/2019,22890,Novelty Biscuits Cake Stand 3 Tier,7.24,1,15492,United Kingd +581492,12/9/2019,22898,Childrens Apron Apples Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22899,Children's Apron Dolly Girl,7.24,2,15492,United Kingdom +581492,12/9/2019,22900,Set 2 Tea Towels I Love London,7.24,3,15492,United Kingdom +581492,12/9/2019,22905,Calendar In Season Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22907,Pack Of 20 Napkins Pantry Design,6.39,1,15492,United Kingdom +581492,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,7.24,5,15492,United King +581492,12/9/2019,22910,Paper Chain Kit Vintage Christmas,7.24,7,15492,United Kingdo +581492,12/9/2019,22914,Blue Coat Rack Paris Fashion,7.24,1,15492,United Kingdom +581492,12/9/2019,22922,Fridge Magnets Us Diner Assorted,7.24,6,15492,United Kingdom +581492,12/9/2019,22924,Fridge Magnets La Vie En Rose,7.24,6,15492,United Kingdom +581492,12/9/2019,22928,Yellow Giant Garden Thermometer,7.24,1,15492,United Kingdom +581492,12/9/2019,22940,Feltcraft Christmas Fairy,6.19,1,15492,United Kingdom +581492,12/9/2019,22941,Christmas Lights 10 Reindeer,6.19,2,15492,United Kingdom +581492,12/9/2019,22943,Christmas Lights 10 Vintage Baubles,6.19,1,15492,United King +581492,12/9/2019,22948,Metal Decoration Naughty Children,6.19,4,15492,United Kingdo +581492,12/9/2019,22951,60 Cake Cases Dolly Girl Design,6.19,2,15492,United Kingdom +581492,12/9/2019,22952,60 Cake Cases Vintage Christmas,6.19,2,15492,United Kingdom +581492,12/9/2019,22955,36 Foil Star Cake Cases,6.19,1,15492,United Kingdom +581492,12/9/2019,22960,Jam Making Set With Jars,6.19,2,15492,United Kingdom +581492,12/9/2019,22961,Jam Making Set Printed,6.19,9,15492,United Kingdom +581492,12/9/2019,22962,Jam Jar With Pink Lid,6.19,3,15492,United Kingdom +581492,12/9/2019,22963,Jam Jar With Green Lid,6.19,3,15492,United Kingdom +581492,12/9/2019,22964,3 Piece Spaceboy Cookie Cutter Set,6.19,1,15492,United Kingd +581492,12/9/2019,22965,3 Traditional Biscuit Cutters Set,6.19,1,15492,United Kingdo +581492,12/9/2019,22966,Gingerbread Man Cookie Cutter,6.19,4,15492,United Kingdom +581492,12/9/2019,22969,Homemade Jam Scented Candles,6.19,6,15492,United Kingdom +581492,12/9/2019,22975,Spaceboy Childrens Egg Cup,6.19,2,15492,United Kingdom +581492,12/9/2019,22976,Circus Parade Childrens Egg Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,22977,Dolly Girl Childrens Egg Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,22979,Pantry Washing Up Brush,6.19,2,15492,United Kingdom +581492,12/9/2019,22980,Pantry Scrubbing Brush,6.19,1,15492,United Kingdom +581492,12/9/2019,22982,Pantry Pastry Brush,6.19,3,15492,United Kingdom +581492,12/9/2019,22983,Card Billboard Font,6.19,1,15492,United Kingdom +581492,12/9/2019,22984,Card Gingham Rose,6.19,1,15492,United Kingdom +581492,12/9/2019,22988,Soldiers Egg Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,22989,Set 2 Pantry Design Tea Towels,6.19,2,15492,United Kingdom +581492,12/9/2019,22992,Revolver Wooden Ruler,6.19,3,15492,United Kingdom +581492,12/9/2019,22993,Set Of 4 Pantry Jelly Moulds,6.19,3,15492,United Kingdom +581492,12/9/2019,22995,Travel Card Wallet Suki,6.19,4,15492,United Kingdom +581492,12/9/2019,22996,Travel Card Wallet Vintage Ticket,6.19,5,15492,United Kingdo +581492,12/9/2019,22997,Travel Card Wallet Union Jack,6.19,1,15492,United Kingdom +581492,12/9/2019,22998,Travel Card Wallet Keep Calm,6.19,4,15492,United Kingdom +581492,12/9/2019,22999,Travel Card Wallet Vintage Leaf,6.19,1,15492,United Kingdom +581492,12/9/2019,23005,Travel Card Wallet I Love London,6.19,4,15492,United Kingdom +581492,12/9/2019,23007,Spaceboy Baby Gift Set,6.19,1,15492,United Kingdom +581492,12/9/2019,23009,I Love London Baby Gift Set,6.19,2,15492,United Kingdom +581492,12/9/2019,23012,Glass Apothecary Bottle Perfume,6.19,1,15492,United Kingdom +581492,12/9/2019,23013,Glass Apothecary Bottle Tonic,6.19,2,15492,United Kingdom +581492,12/9/2019,23014,Glass Apothecary Bottle Elixir,6.19,1,15492,United Kingdom +581492,12/9/2019,23050,Recycled Acapulco Mat Green,6.19,1,15492,United Kingdom +581492,12/9/2019,23051,Recycled Acapulco Mat Blue,6.19,1,15492,United Kingdom +581492,12/9/2019,23052,Recycled Acapulco Mat Turquoise,6.19,5,15492,United Kingdom +581492,12/9/2019,23053,Recycled Acapulco Mat Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23054,Recycled Acapulco Mat Lavender,6.19,1,15492,United Kingdom +581492,12/9/2019,23074,Embossed Heart Trinket Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23076,Ice Cream Sundae Lip Gloss,6.19,3,15492,United Kingdom +581492,12/9/2019,23077,Doughnut Lip Gloss,6.19,3,15492,United Kingdom +581492,12/9/2019,23082,Set 6 Paper Table Lantern Hearts,6.19,1,15492,United Kingdom +581492,12/9/2019,23083,Set 6 Paper Table Lantern Stars,6.19,1,15492,United Kingdom +581492,12/9/2019,23084,Rabbit Night Light,6.19,57,15492,United Kingdom +581492,12/9/2019,23088,Zinc Heart Flower T-Light Holder,6.19,1,15492,United Kingdom +581492,12/9/2019,23089,Glass Bon Bon Jar,6.19,4,15492,United Kingdom +581492,12/9/2019,23093,Small Parisienne Heart Photo Frame,6.19,3,15492,United Kingd +581492,12/9/2019,23103,Jingle Bell Heart Decoration,6.19,2,15492,United Kingdom +581492,12/9/2019,23110,Parisienne Key Cabinet,6.19,1,15492,United Kingdom +581492,12/9/2019,23111,Parisienne Sewing Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23112,Parisienne Curio Cabinet,6.19,1,15492,United Kingdom +581492,12/9/2019,23118,Parisienne Jewellery Drawer,6.19,1,15492,United Kingdom +581492,12/9/2019,23158,Set Of 5 Lucky Cat Magnets,6.19,1,15492,United Kingdom +581492,12/9/2019,23165,Large Ceramic Top Storage Jar,6.19,2,15492,United Kingdom +581492,12/9/2019,23166,Medium Ceramic Top Storage Jar,6.19,2,15492,United Kingdom +581492,12/9/2019,23168,Classic Cafe Sugar Dispenser,6.19,2,15492,United Kingdom +581492,12/9/2019,23171,Regency Tea Plate Green,6.19,1,15492,United Kingdom +581492,12/9/2019,23172,Regency Tea Plate Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23173,Regency Teapot Roses,6.19,1,15492,United Kingdom +581492,12/9/2019,23175,Regency Milk Jug Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23177,Treasure Island Book Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23183,Mother's Kitchen Spoon Rest,6.19,3,15492,United Kingdom +581492,12/9/2019,23184,Bull Dog Bottle Opener,6.19,2,15492,United Kingdom +581492,12/9/2019,23188,Vintage 2 Metre Folding Ruler,6.19,1,15492,United Kingdom +581492,12/9/2019,23191,Bundle Of 3 Retro Note Books,6.19,1,15492,United Kingdom +581492,12/9/2019,23194,Gymkhana Treasure Book Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23196,Vintage Leaf Magnetic Notepad,6.19,1,15492,United Kingdom +581492,12/9/2019,23197,Sketchbook Magnetic Shopping List,6.19,2,15492,United Kingdo +581492,12/9/2019,23198,Pantry Magnetic Shopping List,6.19,2,15492,United Kingdom +581492,12/9/2019,23204,Charlotte Bag Apples Design,6.19,6,15492,United Kingdom +581492,12/9/2019,23205,Charlotte Bag Vintage Alphabet,6.19,3,15492,United Kingdom +581492,12/9/2019,23212,Heart Wreath Decoration With Bell,6.19,7,15492,United Kingdo +581492,12/9/2019,23213,Star Wreath Decoration With Bell,6.19,7,15492,United Kingdom +581492,12/9/2019,23220,Reindeer Heart Decoration Gold,6.19,2,15492,United Kingdom +581492,12/9/2019,23224,Cherub Heart Decoration Gold,6.19,1,15492,United Kingdom +581492,12/9/2019,23229,Vintage Donkey Tail Game,6.19,2,15492,United Kingdom +581492,12/9/2019,23234,Biscuit Tin Vintage Christmas,6.19,4,15492,United Kingdom +581492,12/9/2019,23241,Treasure Tin Gymkhana Design,6.19,3,15492,United Kingdom +581492,12/9/2019,23243,Set Of Tea Coffee Sugar Tins Pantry,6.19,1,15492,United King +581492,12/9/2019,23245,Set Of 3 Regency Cake Tins,6.19,7,15492,United Kingdom +581492,12/9/2019,23247,Biscuit Tin 50'S Christmas,6.19,3,15492,United Kingdom +581492,12/9/2019,23264,Set Of 3 Wooden Sleigh Decorations,6.19,3,15492,United Kingd +581492,12/9/2019,23265,Set Of 3 Wooden Tree Decorations,6.19,3,15492,United Kingdom +581492,12/9/2019,23266,Set Of 3 Wooden Stocking Decoration,6.19,2,15492,United King +581492,12/9/2019,23274,Star T-Light Holder Willie Winkie,6.19,3,15492,United Kingdo +581492,12/9/2019,23275,Set Of 3 Hanging Owls Ollie Beak,6.19,2,15492,United Kingdom +581492,12/9/2019,23280,Folding Butterfly Mirror Hot Pink,6.19,2,15492,United Kingdo +581492,12/9/2019,23284,Doormat Keep Calm And Come In,6.19,1,15492,United Kingdom +581492,12/9/2019,23285,Pink Vintage Spot Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23287,Red Vintage Spot Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23290,Spaceboy Childrens Bowl,6.19,1,15492,United Kingdom +581492,12/9/2019,23292,Spaceboy Childrens Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,23293,Set Of 12 Fairy Cake Baking Cases,6.19,3,15492,United Kingdo +581492,12/9/2019,23294,Set Of 6 Snack Loaf Baking Cases,6.19,1,15492,United Kingdom +581492,12/9/2019,23295,Set Of 12 Mini Loaf Baking Cases,6.19,4,15492,United Kingdom +581492,12/9/2019,23298,Spotty Bunting,6.19,2,15492,United Kingdom +581492,12/9/2019,23299,Food Cover With Beads Set 2,6.19,1,15492,United Kingdom +581492,12/9/2019,23300,Gardeners Kneeling Pad Cup Of Tea,6.19,3,15492,United Kingdo +581492,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,6,15492,United Kingdom +581492,12/9/2019,23307,Set Of 60 Pantry Design Cake Cases,6.19,7,15492,United Kingd +581492,12/9/2019,23308,Set Of 60 Vintage Leaf Cake Cases,6.19,1,15492,United Kingdo +581492,12/9/2019,23309,Set Of 60 I Love London Cake Cases,6.19,2,15492,United Kingd +581492,12/9/2019,23310,Bubblegum Ring Assorted,6.19,4,15492,United Kingdom +581492,12/9/2019,23311,Vintage Christmas Stocking,6.19,1,15492,United Kingdom +581492,12/9/2019,23312,Vintage Christmas Gift Sack,6.19,6,15492,United Kingdom +581492,12/9/2019,23313,Vintage Christmas Bunting,6.19,4,15492,United Kingdom +581492,12/9/2019,23314,Vintage Christmas Tablecloth,6.19,1,15492,United Kingdom +581492,12/9/2019,23319,Box Of 6 Mini 50'S Crackers,6.19,3,15492,United Kingdom +581492,12/9/2019,23322,Large White Heart Of Wicker,6.19,1,15492,United Kingdom +581492,12/9/2019,23323,White Wicker Star,6.19,6,15492,United Kingdom +581492,12/9/2019,23328,Set 6 School Milk Bottles In Crate,6.19,5,15492,United Kingd +581492,12/9/2019,23332,Ivory Wicker Heart Large,6.19,1,15492,United Kingdom +581492,12/9/2019,23334,Ivory Wicker Heart Small,6.19,1,15492,United Kingdom +581492,12/9/2019,23336,Egg Frying Pan Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23340,Vintage Christmas Cake Frill,6.19,3,15492,United Kingdom +581492,12/9/2019,23345,Dolly Girl Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23347,I Love London Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,2,15492,United Kingdom +581492,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,5,15492,United Kingdom +581492,12/9/2019,23352,Roll Wrap 50'S Red Christmas,6.19,1,15492,United Kingdom +581492,12/9/2019,23353,6 Gift Tags Vintage Christmas,6.19,3,15492,United Kingdom +581492,12/9/2019,23354,6 Gift Tags 50'S Christmas,6.19,4,15492,United Kingdom +581492,12/9/2019,23357,Hot Water Bottle Sex Bomb,6.19,1,15492,United Kingdom +581492,12/9/2019,23358,Hot Stuff Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,23365,Set 12 Colour Pencils Love London,6.19,1,15492,United Kingdo +581492,12/9/2019,23366,Set 12 Colouring Pencils Doily,6.04,5,15492,United Kingdom +581492,12/9/2019,23367,Set 12 Colour Pencils Spaceboy,6.04,10,15492,United Kingdom +581492,12/9/2019,23368,Set 12 Colour Pencils Dolly Girl,6.19,2,15492,United Kingdom +581492,12/9/2019,23581,Jumbo Bag Paisley Park,6.19,3,15492,United Kingdom +581492,12/9/2019,21928,Jumbo Bag Scandinavian Blue Paisley,6.19,2,15492,United King +581492,12/9/2019,21929,Jumbo Bag Pink Vintage Paisley,6.19,1,15492,United Kingdom +581492,12/9/2019,85099C,Jumbo Bag Baroque Black White,6.19,7,15492,United Kingdom +581492,12/9/2019,23199,Jumbo Bag Apples,6.19,2,15492,United Kingdom +581492,12/9/2019,23200,Jumbo Bag Pears,6.19,1,15492,United Kingdom +581492,12/9/2019,23202,Jumbo Bag Vintage Leaf,6.19,2,15492,United Kingdom +581492,12/9/2019,23343,Jumbo Bag Vintage Christmas,6.19,4,15492,United Kingdom +581492,12/9/2019,23344,Jumbo Bag 50'S Christmas,6.19,5,15492,United Kingdom +581492,12/9/2019,23583,Lunch Bag Paisley Park,6.19,2,15492,United Kingdom +581492,12/9/2019,20727,Lunch Bag Black Skull,6.04,2,15492,United Kingdom +581492,12/9/2019,20728,Lunch Bag Cars Blue,6.04,1,15492,United Kingdom +581492,12/9/2019,20725,Lunch Bag Red Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,20726,Lunch Bag Woodland,6.19,1,15492,United Kingdom +581492,12/9/2019,22662,Lunch Bag Dolly Girl Design,6.19,1,15492,United Kingdom +581492,12/9/2019,23206,Lunch Bag Apple Design,6.19,3,15492,United Kingdom +581492,12/9/2019,23208,Lunch Bag Vintage Leaf Design,6.19,3,15492,United Kingdom +581492,12/9/2019,23375,50'S Christmas Paper Gift Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,23437,50'S Christmas Gift Bag Large,6.19,1,15492,United Kingdom +581492,12/9/2019,22821,Gift Bag Psychedelic Apples,7.24,3,15492,United Kingdom +581493,12/9/2019,79190B,Retro Plastic Polka Tray,7.24,15,12423,Belgium +581493,12/9/2019,79190A,Retro Plastic 70'S Tray,7.24,15,12423,Belgium +581493,12/9/2019,22915,Assorted Bottle Top Magnets,7.24,12,12423,Belgium +581493,12/9/2019,22151,Place Setting White Heart,7.24,24,12423,Belgium +581493,12/9/2019,22632,Hand Warmer Red Retrospot,7.24,12,12423,Belgium +581493,12/9/2019,22865,Hand Warmer Owl Design,7.24,12,12423,Belgium +581493,12/9/2019,20718,Red Retrospot Shopper Bag,7.24,10,12423,Belgium +581493,12/9/2019,79190B,Retro Plastic Polka Tray,7.24,12,12423,Belgium +581493,12/9/2019,71459,Hanging Jam Jar T-Light Holders,7.24,12,12423,Belgium +581493,12/9/2019,84945,Multi Colour Silver T-Light Holder,7.24,12,12423,Belgium +581493,12/9/2019,22356,Charlotte Bag Pink Polkadot,7.24,10,12423,Belgium +581493,12/9/2019,20724,Red Retrospot Charlotte Bag,7.24,10,12423,Belgium +581493,12/9/2019,23204,Charlotte Bag Apples Design,7.24,10,12423,Belgium +581493,12/9/2019,21108,Fairy Cake Flannel Assorted Colour,7.24,18,12423,Belgium +581493,12/9/2019,22252,Birdcage Decoration Tealight Holder,7.24,12,12423,Belgium +581493,12/9/2019,22807,Set Of 6 T-Lights Toadstools,6.19,6,12423,Belgium +581494,12/9/2019,23084,Rabbit Night Light,6.19,24,12518,Germany +581494,12/9/2019,21559,Strawberry Lunch Box With Cutlery,6.19,6,12518,Germany +581494,12/9/2019,21506,Fancy Font Birthday Card,6.19,12,12518,Germany +581494,12/9/2019,22716,Card Circus Parade,6.19,12,12518,Germany +581494,12/9/2019,22556,Plasters In Tin Circus Parade,6.19,12,12518,Germany +581494,12/9/2019,22328,Round Snack Boxes Set Of 4 Fruits,6.19,6,12518,Germany +581494,12/9/2019,22730,Alarm Clock Bakelike Ivory,6.19,4,12518,Germany +581494,12/9/2019,22633,Hand Warmer Union Jack,6.19,12,12518,Germany +581494,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,6.19,6,12518,Germany +581494,12/9/2019,10125,Mini Funky Design Tapes,6.19,20,12518,Germany +581494,12/9/2019,23367,Set 12 Colour Pencils Spaceboy,6.19,24,12518,Germany +581494,12/9/2019,22551,Plasters In Tin Spaceboy,6.04,12,12518,Germany +581494,12/9/2019,22554,Plasters In Tin Woodland Animals,6.04,12,12518,Germany +581494,12/9/2019,22549,Picture Dominoes,6.19,12,12518,Germany +581494,12/9/2019,23388,Woodland Mini Backpack,6.19,4,12518,Germany +581494,12/9/2019,23390,Dolly Girl Mini Backpack,6.19,4,12518,Germany +581494,12/9/2019,23389,Spaceboy Mini Backpack,6.19,4,12518,Germany +581495,12/9/2019,23535,Wall Art Bicycle Safety,6.19,12,14051,United Kingdom +581495,12/9/2019,22699,Roses Regency Teacup And Saucer,6.19,12,14051,United Kingdom +581495,12/9/2019,22698,Pink Regency Teacup And Saucer,6.19,12,14051,United Kingdom +581495,12/9/2019,22697,Green Regency Teacup And Saucer,6.19,12,14051,United Kingdom +581495,12/9/2019,22633,Hand Warmer Union Jack,6.04,18,14051,United Kingdom +581495,12/9/2019,15056N,Edwardian Parasol Natural,6.19,36,14051,United Kingdom +581495,12/9/2019,23439,Hand Warmer Red Love Heart,6.19,36,14051,United Kingdom +581495,12/9/2019,48138,Doormat Union Flag,6.19,10,14051,United Kingdom +581495,12/9/2019,21523,Doormat Fancy Font Home Sweet Home,6.19,10,14051,United King +581495,12/9/2019,21877,Home Sweet Home Mug,6.19,12,14051,United Kingdom +581495,12/9/2019,21871,Save The Planet Mug,6.19,18,14051,United Kingdom +581495,12/9/2019,22798,Antique Glass Dressing Table Pot,6.19,36,14051,United Kingdo +581495,12/9/2019,23173,Regency Teapot Roses,6.19,6,14051,United Kingdom +581495,12/9/2019,22423,Regency Cakestand 3 Tier,6.19,10,14051,United Kingdom +581496,12/9/2019,22664,Toy Tidy Dolly Girl Design,6.19,20,16558,United Kingdom +581496,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,6.19,12,16558,United Kingdom +581496,12/9/2019,72800E,4 Ivory Dinner Candles Silver Flock,6.19,12,16558,United Ki +581496,12/9/2019,23313,Vintage Christmas Bunting,6.19,10,16558,United Kingdom +581496,12/9/2019,22952,60 Cake Cases Vintage Christmas,6.19,24,16558,United Kingdom +581496,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,6.04,12,16558,United Kin +581496,12/9/2019,23298,Spotty Bunting,6.19,3,16558,United Kingdom +581496,12/9/2019,23598,Paper Bunting Vintage Party,6.19,6,16558,United Kingdom +581496,12/9/2019,16169E,Wrap 50'S Christmas,6.19,25,16558,United Kingdom +581496,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,12,16558,United Kingdom +581496,12/9/2019,23350,Roll Wrap Vintage Spot,6.19,24,16558,United Kingdom +581496,12/9/2019,22865,Hand Warmer Owl Design,6.19,24,16558,United Kingdom +581496,12/9/2019,22632,Hand Warmer Red Retrospot,6.19,12,16558,United Kingdom +581496,12/9/2019,22835,Hot Water Bottle I Am So Poorly,6.19,4,16558,United Kingdom +581496,12/9/2019,22112,Chocolate Hot Water Bottle,6.19,6,16558,United Kingdom +581496,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,4,16558,United Kingdom +581496,12/9/2019,22314,Office Mug Warmer Choc+Blue,6.19,24,16558,United Kingdom +581496,12/9/2019,22313,Office Mug Warmer Pink,6.19,12,16558,United Kingdom +581496,12/9/2019,23084,Rabbit Night Light,6.19,18,16558,United Kingdom +581496,12/9/2019,20831,Gold Photo Frame,6.19,12,16558,United Kingdom +581496,12/9/2019,21115,Rose Caravan Doorstop,6.19,8,16558,United Kingdom +581496,12/9/2019,21462,Nursery A B C Painted Letters,7.24,8,16558,United Kingdom +581496,12/9/2019,22076,6 Ribbons Empire,7.24,24,16558,United Kingdom +581496,12/9/2019,22190,Local Cafe Mug,7.24,24,16558,United Kingdom +581496,12/9/2019,22215,Cake Stand White Two Tier Lace,7.24,6,16558,United Kingdom +581496,12/9/2019,22220,Cake Stand Lovebird 2 Tier White,7.24,6,16558,United Kingdom +581496,12/9/2019,22464,Hanging Metal Heart Lantern,7.24,12,16558,United Kingdom +581496,12/9/2019,22465,Hanging Metal Star Lantern,7.24,12,16558,United Kingdom +581496,12/9/2019,22539,Mini Jigsaw Dolly Girl,7.24,48,16558,United Kingdom +581496,12/9/2019,22544,Mini Jigsaw Spaceboy,7.24,48,16558,United Kingdom +581496,12/9/2019,23438,Red Spot Gift Bag Large,6.19,12,16558,United Kingdom +581496,12/9/2019,23437,50'S Christmas Gift Bag Large,6.19,12,16558,United Kingdom +581497,12/9/2019,20719,Woodland Charlotte Bag,6.19,33,17497,United Kingdom +581497,12/9/2019,20723,Strawberry Charlotte Bag,6.19,42,17497,United Kingdom +581497,12/9/2019,20724,Red Retrospot Charlotte Bag,6.19,55,17497,United Kingdom +581497,12/9/2019,21212,Pack Of 72 Retrospot Cake Cases,7.24,7,17497,United Kingdom +581497,12/9/2019,21238,Red Retrospot Cup,7.24,8,17497,United Kingdom +581497,12/9/2019,21242,Red Retrospot Plate,7.24,2,17497,United Kingdom +581497,12/9/2019,21479,White Skull Hot Water Bottle,7.24,25,17497,United Kingdom +581497,12/9/2019,21481,Fawn Blue Hot Water Bottle,7.24,1,17497,United Kingdom +581497,12/9/2019,21481,Fawn Blue Hot Water Bottle,7.24,1,17497,United Kingdom +581497,12/9/2019,21484,Chick Grey Hot Water Bottle,7.24,1,17497,United Kingdom +581497,12/9/2019,21668,Red Stripe Ceramic Drawer Knob,7.24,1,17497,United Kingdom +581497,12/9/2019,21670,Blue Spot Ceramic Drawer Knob,7.24,1,17497,United Kingdom +581497,12/9/2019,21671,Red Spot Ceramic Drawer Knob,7.24,1,17497,United Kingdom +581497,12/9/2019,21672,White Spot Red Ceramic Drawer Knob,7.24,6,17497,United Kingd +581497,12/9/2019,21673,White Spot Blue Ceramic Drawer Knob,7.24,1,17497,United King +581497,12/9/2019,21714,Citronella Candle Garden Pot,7.24,2,17497,United Kingdom +581497,12/9/2019,22110,Bird House Hot Water Bottle,7.24,7,17497,United Kingdom +581497,12/9/2019,22111,Scottie Dog Hot Water Bottle,7.24,5,17497,United Kingdom +581497,12/9/2019,22112,Chocolate Hot Water Bottle,7.24,13,17497,United Kingdom +581497,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,7.24,48,17497,United Kingd +581497,12/9/2019,22197,Popcorn Holder,7.24,68,17497,United Kingdom +581497,12/9/2019,22355,Charlotte Bag Suki Design,7.24,110,17497,United Kingdom +581497,12/9/2019,22356,Charlotte Bag Pink Polkadot,7.24,25,17497,United Kingdom +581497,12/9/2019,22356,Charlotte Bag Pink Polkadot,7.24,1,17497,United Kingdom +581497,12/9/2019,22423,Regency Cakestand 3 Tier,7.24,8,17497,United Kingdom +581497,12/9/2019,22725,Alarm Clock Bakelike Chocolate,7.24,3,17497,United Kingdom +581497,12/9/2019,22726,Alarm Clock Bakelike Green,7.24,1,17497,United Kingdom +581497,12/9/2019,22727,Alarm Clock Bakelike Red,7.24,10,17497,United Kingdom +581497,12/9/2019,22730,Alarm Clock Bakelike Ivory,7.24,5,17497,United Kingdom +581497,12/9/2019,22735,Ribbon Reel Socks And Mittens,7.24,2,17497,United Kingdom +581497,12/9/2019,22736,Ribbon Reel Making Snowmen,7.24,3,17497,United Kingdom +581497,12/9/2019,22738,Ribbon Reel Snowy Village,7.24,1,17497,United Kingdom +581497,12/9/2019,22805,Blue Drawer Knob Acrylic Edwardian,7.24,1,17497,United Kingd +581497,12/9/2019,22835,Hot Water Bottle I Am So Poorly,7.24,4,17497,United Kingdom +581497,12/9/2019,22895,Set Of 2 Tea Towels Apple And Pears,7.24,1,17497,United King +581497,12/9/2019,22896,Peg Bag Apples Design,7.24,2,17497,United Kingdom +581497,12/9/2019,22898,Childrens Apron Apples Design,7.24,2,17497,United Kingdom +581497,12/9/2019,22943,Christmas Lights 10 Vintage Baubles,7.24,1,17497,United King +581497,12/9/2019,23084,Rabbit Night Light,6.19,37,17497,United Kingdom +581497,12/9/2019,23168,Classic Cafe Sugar Dispenser,6.19,1,17497,United Kingdom +581497,12/9/2019,23204,Charlotte Bag Apples Design,6.04,13,17497,United Kingdom +581497,12/9/2019,23205,Charlotte Bag Vintage Alphabet,6.04,8,17497,United Kingdom +581497,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,1,17497,United Kingdom +581497,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,36,17497,United Kingdom +581497,12/9/2019,23356,Love Hot Water Bottle,6.19,1,17497,United Kingdom +581497,12/9/2019,23357,Hot Water Bottle Sex Bomb,6.19,2,17497,United Kingdom +581497,12/9/2019,23358,Hot Stuff Hot Water Bottle,6.19,2,17497,United Kingdom +581497,12/9/2019,35970,Zinc Folkart Sleigh Bells,6.19,2,17497,United Kingdom +581497,12/9/2019,47566,Party Bunting,6.19,5,17497,United Kingdom +581497,12/9/2019,82583,Hot Baths Metal Sign,6.19,4,17497,United Kingdom +581497,12/9/2019,84029G,Knitted Union Flag Hot Water Bottle,6.19,11,17497,United Ki +581497,12/9/2019,84997C,Childrens Cutlery Polkadot Blue,6.19,3,17497,United Kingdom +581497,12/9/2019,85049G,Chocolate Box Ribbons,6.19,2,17497,United Kingdom +581497,12/9/2019,20727,Lunch Bag Black Skull,6.19,8,17497,United Kingdom +581497,12/9/2019,22383,Lunch Bag Suki Design,7.24,2,17497,United Kingdom +581497,12/9/2019,23206,Lunch Bag Apple Design,6.04,3,17497,United Kingdom +581497,12/9/2019,23206,Lunch Bag Apple Design,6.04,1,17497,United Kingdom +581497,12/9/2019,23207,Lunch Bag Alphabet Design,6.19,1,17497,United Kingdom +581497,12/9/2019,23208,Lunch Bag Vintage Leaf Design,6.19,3,17497,United Kingdom +581498,12/9/2019,20669,Red Heart Luggage Tag,6.19,3,14498,United Kingdom +581498,12/9/2019,20679,Edwardian Parasol Red,6.19,5,14498,United Kingdom +581498,12/9/2019,20717,Strawberry Shopper Bag,6.19,1,14498,United Kingdom +581498,12/9/2019,20718,Red Retrospot Shopper Bag,6.19,1,14498,United Kingdom +581498,12/9/2019,20750,Red Retrospot Mini Cases,6.19,1,14498,United Kingdom +581498,12/9/2019,20961,Strawberry Bath Sponge,6.19,1,14498,United Kingdom +581498,12/9/2019,20963,Apple Bath Sponge,6.19,1,14498,United Kingdom +581498,12/9/2019,21115,Rose Caravan Doorstop,6.19,1,14498,United Kingdom +581498,12/9/2019,21125,Set 6 Football Celebration Candles,6.19,3,14498,United Kingd +581498,12/9/2019,21137,Black Record Cover Frame,6.19,4,14498,United Kingdom +581498,12/9/2019,21155,Red Retrospot Peg Bag,6.19,4,14498,United Kingdom +581498,12/9/2019,21166,Cook With Wine Metal Sign,6.19,3,14498,United Kingdom +581498,12/9/2019,21169,You're Confusing Me Metal Sign,6.19,2,14498,United Kingdom +581498,12/9/2019,21174,Pottering In The Shed Metal Sign,6.19,2,14498,United Kingdom +581498,12/9/2019,21181,Please One Person Metal Sign,6.19,6,14498,United Kingdom +581498,12/9/2019,21216,Set 3 Retrospot Tea/Coffee/Sugar,6.19,2,14498,United Kingdom +581498,12/9/2019,21217,Red Retrospot Round Cake Tins,6.19,3,14498,United Kingdom +581498,12/9/2019,21218,Red Spotty Biscuit Tin,6.19,4,14498,United Kingdom +581498,12/9/2019,21232,Strawberry Ceramic Trinket Pot,6.19,13,14498,United Kingdom +581498,12/9/2019,21239,Pink Polkadot Cup,6.19,1,14498,United Kingdom +581498,12/9/2019,21257,Victorian Sewing Box Medium,6.19,3,14498,United Kingdom +581498,12/9/2019,21327,Skulls Writing Set,6.19,1,14498,United Kingdom +581498,12/9/2019,21328,Balloons Writing Set,6.19,1,14498,United Kingdom +581498,12/9/2019,21329,Dinosaurs Writing Set,6.19,1,14498,United Kingdom +581498,12/9/2019,21411,Gingham Heart Doorstop Red,6.19,1,14498,United Kingdom +581498,12/9/2019,21430,Set/3 Red Gingham Rose Storage Box,6.19,3,14498,United Kingd +581498,12/9/2019,21494,Rotating Leaves T-Light Holder,6.19,7,14498,United Kingdom +581498,12/9/2019,21523,Doormat Fancy Font Home Sweet Home,6.19,1,14498,United Kingd +581498,12/9/2019,21557,Set Of 6 Funky Beakers,6.19,1,14498,United Kingdom +581498,12/9/2019,21558,Skull Lunch Box With Cutlery,6.19,1,14498,United Kingdom +581498,12/9/2019,21731,Red Toadstool Led Night Light,6.19,9,14498,United Kingdom +581498,12/9/2019,21754,Home Building Block Word,6.19,2,14498,United Kingdom +581498,12/9/2019,21790,Vintage Snap Cards,6.19,1,14498,United Kingdom +581498,12/9/2019,21843,Red Retrospot Cake Stand,6.19,5,14498,United Kingdom +581498,12/9/2019,21864,Union Jack Flag Passport Cover,6.19,2,14498,United Kingdom +581498,12/9/2019,21874,Gin And Tonic Mug,6.19,2,14498,United Kingdom +581498,12/9/2019,21876,Pottering Mug,6.19,4,14498,United Kingdom +581498,12/9/2019,21890,S/6 Wooden Skittles In Cotton Bag,6.19,1,14498,United Kingdo +581498,12/9/2019,21912,Vintage Snakes & Ladders,6.19,1,14498,United Kingdom +581498,12/9/2019,21916,Set 12 Retro White Chalk Sticks,6.19,7,14498,United Kingdom +581498,12/9/2019,21930,Jumbo Storage Bag Skulls,6.19,2,14498,United Kingdom +581498,12/9/2019,21931,Jumbo Storage Bag Suki,6.19,6,14498,United Kingdom +581498,12/9/2019,21934,Skull Shoulder Bag,6.19,1,14498,United Kingdom +581498,12/9/2019,21935,Suki Shoulder Bag,6.19,7,14498,United Kingdom +581498,12/9/2019,21942,Skulls Design Flannel,6.19,1,14498,United Kingdom +581498,12/9/2019,21955,Doormat Union Jack Guns And Roses,6.19,1,14498,United Kingdo +581498,12/9/2019,21977,Pack Of 60 Pink Paisley Cake Cases,6.19,1,14498,United Kingd +581498,12/9/2019,21983,Pack Of 12 Blue Paisley Tissues,6.19,1,14498,United Kingdom +581498,12/9/2019,21987,Pack Of 6 Skull Paper Cups,6.19,2,14498,United Kingdom +581498,12/9/2019,21989,Pack Of 20 Skull Paper Napkins,6.19,1,14498,United Kingdom +581498,12/9/2019,22041,"Record Frame 7"" Single Size",6.19,2,14498,United Kingdom +581498,12/9/2019,22059,Ceramic Strawberry Design Mug,6.19,1,14498,United Kingdom +581498,12/9/2019,22069,Brown Pirate Treasure Chest,6.19,3,14498,United Kingdom +581498,12/9/2019,22077,6 Ribbons Rustic Charm,6.19,2,14498,United Kingdom +581498,12/9/2019,22083,Paper Chain Kit Retrospot,6.19,2,14498,United Kingdom +581498,12/9/2019,22086,Paper Chain Kit 50'S Christmas,6.19,52,14498,United Kingdom +581498,12/9/2019,22099,Caravan Square Tissue Box,6.19,2,14498,United Kingdom +581498,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,6.19,4,14498,United Kingdo +581498,12/9/2019,22139,Retrospot Tea Set Ceramic 11 Pc,6.19,2,14498,United Kingdom +581498,12/9/2019,22141,Christmas Craft Tree Top Angel,6.19,7,14498,United Kingdom +581498,12/9/2019,22142,Christmas Craft White Fairy,6.19,2,14498,United Kingdom +581498,12/9/2019,22144,Christmas Craft Little Friends,6.19,8,14498,United Kingdom +581498,12/9/2019,22161,Heart Decoration Rustic Hanging,6.19,2,14498,United Kingdom +581498,12/9/2019,22173,Metal 4 Hook Hanger French Chateau,6.19,3,14498,United Kingd +581498,12/9/2019,22174,Photo Cube,6.19,3,14498,United Kingdom +581498,12/9/2019,22175,Pink Owl Soft Toy,6.19,1,14498,United Kingdom +581498,12/9/2019,22178,Victorian Glass Hanging T-Light,6.19,1,14498,United Kingdom +581498,12/9/2019,22179,Set 10 Night Owl Lights,6.19,1,14498,United Kingdom +581498,12/9/2019,22186,Red Star Card Holder,6.19,1,14498,United Kingdom +581498,12/9/2019,22187,Green Christmas Tree Card Holder,6.19,6,14498,United Kingdom +581498,12/9/2019,22193,Red Diner Wall Clock,6.19,1,14498,United Kingdom +581498,12/9/2019,22195,Large Heart Measuring Spoons,6.19,3,14498,United Kingdom +581498,12/9/2019,22196,Small Heart Measuring Spoons,6.19,2,14498,United Kingdom +581498,12/9/2019,22207,Frying Pan Union Flag,6.19,1,14498,United Kingdom +581498,12/9/2019,22278,Overnight Bag Vintage Rose Paisley,6.19,2,14498,United Kingd +581498,12/9/2019,22301,Coffee Mug Cat + Bird Design,6.19,2,14498,United Kingdom +581498,12/9/2019,22302,Coffee Mug Pears Design,6.19,4,14498,United Kingdom +581498,12/9/2019,22303,Coffee Mug Apples Design,6.19,4,14498,United Kingdom +581498,12/9/2019,22304,Coffee Mug Blue Paisley Design,6.19,1,14498,United Kingdom +581498,12/9/2019,22308,Tea Cosy Blue Stripe,6.19,2,14498,United Kingdom +581498,12/9/2019,22327,Round Snack Boxes Set Of 4 Skulls,6.19,4,14498,United Kingdo +581498,12/9/2019,22328,Round Snack Boxes Set Of 4 Fruits,6.19,4,14498,United Kingdo +581498,12/9/2019,22352,Lunch Box With Cutlery Retrospot,6.19,6,14498,United Kingdom +581498,12/9/2019,22357,Kings Choice Biscuit Tin,6.19,1,14498,United Kingdom +581498,12/9/2019,22358,Kings Choice Tea Caddy,6.19,1,14498,United Kingdom +581498,12/9/2019,22367,Childrens Apron Spaceboy Design,6.19,7,14498,United Kingdom +581498,12/9/2019,22371,Airline Bag Vintage Tokyo 78,6.19,1,14498,United Kingdom +581498,12/9/2019,22374,Airline Bag Vintage Jet Set Red,6.19,1,14498,United Kingdom +581498,12/9/2019,22378,Wall Tidy Retrospot,7.24,2,14498,United Kingdom +581498,12/9/2019,22379,Recycling Bag Retrospot,7.24,4,14498,United Kingdom +581498,12/9/2019,22381,Toy Tidy Pink Polkadot,7.24,1,14498,United Kingdom +581498,12/9/2019,22411,Jumbo Shopper Vintage Red Paisley,7.24,2,14498,United Kingdo +581498,12/9/2019,22422,Toothpaste Tube Pen,7.24,1,14498,United Kingdom +581498,12/9/2019,22424,Enamel Bread Bin Cream,7.24,1,14498,United Kingdom +581498,12/9/2019,22429,Enamel Measuring Jug Cream,7.24,1,14498,United Kingdom +581498,12/9/2019,22456,Natural Slate Chalkboard Large,7.24,4,14498,United Kingdom +581498,12/9/2019,22457,Natural Slate Heart Chalkboard,7.24,2,14498,United Kingdom +581498,12/9/2019,22471,Tv Dinner Tray Air Hostess,7.24,1,14498,United Kingdom +581498,12/9/2019,22474,Spaceboy Tv Dinner Tray,7.24,1,14498,United Kingdom +581498,12/9/2019,22488,Natural Slate Rectangle Chalkboard,7.24,2,14498,United Kingd +581498,12/9/2019,22498,Wooden Regatta Bunting,7.24,1,14498,United Kingdom +581498,12/9/2019,22507,Memo Board Retrospot Design,7.24,1,14498,United Kingdom +581498,12/9/2019,22526,Wheelbarrow For Children,7.24,1,14498,United Kingdom +581498,12/9/2019,22553,Plasters In Tin Skulls,7.24,1,14498,United Kingdom +581498,12/9/2019,22557,Plasters In Tin Vintage Paisley,7.24,1,14498,United Kingdom +581498,12/9/2019,22619,Set Of 6 Soldier Skittles,7.24,1,14498,United Kingdom +581498,12/9/2019,22622,Box Of Vintage Alphabet Blocks,7.24,1,14498,United Kingdom +581498,12/9/2019,22624,Ivory Kitchen Scales,7.24,1,14498,United Kingdom +581498,12/9/2019,22629,Spaceboy Lunch Box,7.24,2,14498,United Kingdom +581498,12/9/2019,22632,Hand Warmer Red Retrospot,7.24,1,14498,United Kingdom +581498,12/9/2019,22645,Ceramic Heart Fairy Cake Money Bank,7.24,4,14498,United King +581498,12/9/2019,22654,Deluxe Sewing Kit,6.19,2,14498,United Kingdom +581498,12/9/2019,22659,Lunch Box I Love London,6.19,1,14498,United Kingdom +581498,12/9/2019,22666,Recipe Box Pantry Yellow Design,6.19,11,14498,United Kingdom +581498,12/9/2019,22676,French Blue Metal Door Sign 1,6.04,1,14498,United Kingdom +581498,12/9/2019,22684,French Blue Metal Door Sign 9,6.04,1,14498,United Kingdom +581498,12/9/2019,22694,Wicker Star,6.04,1,14498,United Kingdom +581498,12/9/2019,22697,Green Regency Teacup And Saucer,6.04,6,14498,United Kingdom +581498,12/9/2019,22698,Pink Regency Teacup And Saucer,6.04,9,14498,United Kingdom +581498,12/9/2019,22699,Roses Regency Teacup And Saucer,6.19,6,14498,United Kingdom +581498,12/9/2019,22722,Set Of 6 Spice Tins Pantry Design,6.19,3,14498,United Kingdo +581498,12/9/2019,22733,3d Traditional Christmas Stickers,6.19,1,14498,United Kingdo +581498,12/9/2019,22734,Set Of 6 Ribbons Vintage Christmas,6.19,5,14498,United Kingd +581498,12/9/2019,22776,Sweetheart 3 Tier Cake Stand,6.19,1,14498,United Kingdom +581498,12/9/2019,22795,Sweetheart Recipe Book Stand,6.19,1,14498,United Kingdom +581498,12/9/2019,22807,Set Of 6 T-Lights Toadstools,6.19,2,14498,United Kingdom +581498,12/9/2019,22813,Pack 3 Boxes Bird Panettone,6.19,4,14498,United Kingdom +581498,12/9/2019,22838,3 Tier Cake Tin Red And Cream,6.19,1,14498,United Kingdom +581498,12/9/2019,22844,Vintage Cream Dog Food Container,6.19,2,14498,United Kingdom +581498,12/9/2019,22845,Vintage Cream Cat Food Container,6.19,1,14498,United Kingdom +581498,12/9/2019,22865,Hand Warmer Owl Design,6.19,1,14498,United Kingdom +581498,12/9/2019,22866,Hand Warmer Scotty Dog Design,6.19,2,14498,United Kingdom +581498,12/9/2019,22891,Tea For One Polkadot,6.19,1,14498,United Kingdom +581498,12/9/2019,22900,Set 2 Tea Towels I Love London,6.19,2,14498,United Kingdom +581498,12/9/2019,22910,Paper Chain Kit Vintage Christmas,6.19,5,14498,United Kingdo +581498,12/9/2019,22952,60 Cake Cases Vintage Christmas,6.19,7,14498,United Kingdom +581498,12/9/2019,22960,Jam Making Set With Jars,6.19,5,14498,United Kingdom +581498,12/9/2019,22961,Jam Making Set Printed,6.19,4,14498,United Kingdom +581498,12/9/2019,22966,Gingerbread Man Cookie Cutter,6.19,2,14498,United Kingdom +581498,12/9/2019,22993,Set Of 4 Pantry Jelly Moulds,6.19,2,14498,United Kingdom +581498,12/9/2019,23012,Glass Apothecary Bottle Perfume,6.19,1,14498,United Kingdom +581498,12/9/2019,23013,Glass Apothecary Bottle Tonic,6.19,1,14498,United Kingdom +581498,12/9/2019,23014,Glass Apothecary Bottle Elixir,7.24,2,14498,United Kingdom +581498,12/9/2019,23080,Red Metal Box Top Secret,7.24,5,14498,United Kingdom +581498,12/9/2019,23170,Regency Tea Plate Roses,7.24,4,14498,United Kingdom +581498,12/9/2019,23171,Regency Tea Plate Green,7.24,5,14498,United Kingdom +581498,12/9/2019,23172,Regency Tea Plate Pink,6.19,4,14498,United Kingdom +581498,12/9/2019,23181,Bull Dog Bottle Top Wall Clock,6.19,1,14498,United Kingdom +581498,12/9/2019,23196,Vintage Leaf Magnetic Notepad,6.19,1,14498,United Kingdom +581498,12/9/2019,23198,Pantry Magnetic Shopping List,6.19,2,14498,United Kingdom +581498,12/9/2019,23245,Set Of 3 Regency Cake Tins,6.19,5,14498,United Kingdom +581498,12/9/2019,23295,Set Of 12 Mini Loaf Baking Cases,6.19,1,14498,United Kingdom +581498,12/9/2019,23298,Spotty Bunting,6.19,1,14498,United Kingdom +581498,12/9/2019,23300,Gardeners Kneeling Pad Cup Of Tea,6.19,9,14498,United Kingdo +581498,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,32,14498,United Kingdo +581498,12/9/2019,23311,Vintage Christmas Stocking,6.19,6,14498,United Kingdom +581498,12/9/2019,23312,Vintage Christmas Gift Sack,6.19,3,14498,United Kingdom +581498,12/9/2019,23328,Set 6 School Milk Bottles In Crate,6.19,6,14498,United Kingd +581498,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,8,14498,United Kingdom +581498,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,22,14498,United Kingdom +581498,12/9/2019,23352,Roll Wrap 50'S Red Christmas,6.19,8,14498,United Kingdom +581498,12/9/2019,23353,6 Gift Tags Vintage Christmas,6.19,10,14498,United Kingdom +581498,12/9/2019,23354,6 Gift Tags 50'S Christmas,6.19,9,14498,United Kingdom +581498,12/9/2019,23368,Set 12 Colour Pencils Dolly Girl,6.19,3,14498,United Kingdom +581498,12/9/2019,23369,Set 36 Colour Pencils Love London,6.19,1,14498,United Kingdo +581498,12/9/2019,23370,Set 36 Colouring Pencils Doily,6.19,3,14498,United Kingdom +581498,12/9/2019,23382,Box Of 6 Christmas Cake Decorations,6.19,10,14498,United Kin +581498,12/9/2019,23388,Woodland Mini Backpack,6.19,1,14498,United Kingdom +581498,12/9/2019,23480,Mini Lights Woodland Mushrooms,7.24,1,14498,United Kingdom +581498,12/9/2019,23493,Vintage Doily Travel Sewing Kit,7.24,1,14498,United Kingdom +581498,12/9/2019,23494,Vintage Doily Deluxe Sewing Kit,7.24,1,14498,United Kingdom +581498,12/9/2019,23497,Classic Chrome Bicycle Bell,7.24,1,14498,United Kingdom +581498,12/9/2019,23501,Key Ring Baseball Boot Union Jack,7.24,1,14498,United Kingdo +581498,12/9/2019,23564,Egg Cup Milkmaid Ingrid,7.24,1,14498,United Kingdom +581498,12/9/2019,35970,Zinc Folkart Sleigh Bells,7.24,6,14498,United Kingdom +581498,12/9/2019,48138,Doormat Union Flag,7.24,1,14498,United Kingdom +581498,12/9/2019,71053,White Moroccan Metal Lantern,7.24,1,14498,United Kingdom +581498,12/9/2019,72349B,Set/6 Purple Butterfly T-Lights,7.24,2,14498,United Kingdom +581498,12/9/2019,79321,Chilli Lights,7.24,10,14498,United Kingdom +581498,12/9/2019,82001S,Silver Record Cover Frame,6.19,2,14498,United Kingdom +581498,12/9/2019,82482,Wooden Picture Frame White Finish,6.04,4,14498,United Kingdo +581498,12/9/2019,82552,Washroom Metal Sign,6.04,1,14498,United Kingdom +581498,12/9/2019,21171,Bathroom Metal Sign,6.19,1,14498,United Kingdom +581498,12/9/2019,82581,Toilet Metal Sign,6.19,1,14498,United Kingdom +581498,12/9/2019,82600,N0 Singing Metal Sign,6.19,4,14498,United Kingdom +581498,12/9/2019,84029E,Red Woolly Hottie White Heart,6.19,4,14498,United Kingdom +581498,12/9/2019,84032A,Charlie+Lola Pink Hot Water Bottle,6.19,4,14498,United King +581498,12/9/2019,84032B,Charlie + Lola Red Hot Water Bottle,6.19,3,14498,United Kin +581498,12/9/2019,84375,Set Of 20 Kids Cookie Cutters,6.19,3,14498,United Kingdom +581498,12/9/2019,84509A,Set Of 4 English Rose Placemats,6.19,1,14498,United Kingdom +581498,12/9/2019,84558A,3d Dog Picture Playing Cards,6.19,1,14498,United Kingdom +581498,12/9/2019,84832,Zinc Willie Winkie Candle Stick,6.19,26,14498,United Kingdom +581498,12/9/2019,84968E,Set Of 16 Vintage Black Cutlery,6.19,1,14498,United Kingdom +581498,12/9/2019,84970S,Hanging Heart Zinc T-Light Holder,6.19,1,14498,United Kingd +581498,12/9/2019,84997A,Childrens Cutlery Polkadot Green,6.19,2,14498,United Kingdo +581498,12/9/2019,84997B,Childrens Cutlery Retrospot Red,6.19,3,14498,United Kingdom +581498,12/9/2019,84997D,Childrens Cutlery Polkadot Pink,6.19,1,14498,United Kingdom +581498,12/9/2019,85038,6 Chocolate Love Heart T-Lights,6.19,1,14498,United Kingdom +581498,12/9/2019,85048,15cm Christmas Glass Ball 20 Lights,6.19,1,14498,United King +581498,12/9/2019,85049A,Traditional Christmas Ribbons,6.17,5,14498,United Kingdom +581498,12/9/2019,85049E,Scandinavian Reds Ribbons,6.19,4,14498,United Kingdom +581498,12/9/2019,85150,Ladies & Gentlemen Metal Sign,6.19,1,14498,United Kingdom +581498,12/9/2019,85174,S/4 Cacti Candles,6.19,1,14498,United Kingdom +581498,12/9/2019,20712,Jumbo Bag Woodland Animals,6.19,3,14498,United Kingdom +581498,12/9/2019,20713,Jumbo Bag Owls,6.19,8,14498,United Kingdom +581498,12/9/2019,21928,Jumbo Bag Scandinavian Blue Paisley,6.19,2,14498,United King +581498,12/9/2019,22386,Jumbo Bag Pink Polkadot,7.24,1,14498,United Kingdom +581498,12/9/2019,22663,Jumbo Bag Dolly Girl Design,6.19,2,14498,United Kingdom +581498,12/9/2019,23199,Jumbo Bag Apples,6.19,6,14498,United Kingdom +581498,12/9/2019,23201,Jumbo Bag Alphabet,6.19,6,14498,United Kingdom +581498,12/9/2019,85099B,Jumbo Bag Red Retrospot,6.19,5,14498,United Kingdom +581498,12/9/2019,85099C,Jumbo Bag Baroque Black White,6.19,4,14498,United Kingdom +581498,12/9/2019,20726,Lunch Bag Woodland,6.19,3,14498,United Kingdom +581498,12/9/2019,20728,Lunch Bag Cars Blue,6.19,4,14498,United Kingdom +581498,12/9/2019,22384,Lunch Bag Pink Polkadot,7.24,1,14498,United Kingdom +581498,12/9/2019,23437,50'S Christmas Gift Bag Large,7.24,1,14498,United Kingdom +581500,12/9/2019,82486,3 Drawer Antique White Wood Cabinet,7.24,4,15344,United King +581500,12/9/2019,85066,Cream Sweetheart Mini Chest,7.24,4,15344,United Kingdom +581500,12/9/2019,48187,Doormat New England,7.24,2,15344,United Kingdom +581501,12/9/2019,22319,Hairclips Forties Fabric Assorted,7.24,180,12985,United King +581501,12/9/2019,20704,Mr Robot Soft Toy,7.24,8,12985,United Kingdom +581501,12/9/2019,21564,Pink Heart Shape Love Bucket,6.19,24,12985,United Kingdom +581501,12/9/2019,21563,Red Heart Shape Love Bucket,7.24,24,12985,United Kingdom +581501,12/9/2019,22165,Diamante Heart Shaped Wall Mirror,7.24,12,12985,United Kingd +581501,12/9/2019,22299,Pig Keyring With Light & Sound,7.24,48,12985,United Kingdom +581501,12/9/2019,22447,Pin Cushion Babushka Blue,7.24,12,12985,United Kingdom +581501,12/9/2019,22442,Grow Your Own Flowers Set Of 3,7.24,12,12985,United Kingdom +581501,12/9/2019,22495,Set Of 2 Round Tins Camembert,7.24,12,12985,United Kingdom +581501,12/9/2019,22544,Mini Jigsaw Spaceboy,7.24,96,12985,United Kingdom +581501,12/9/2019,22695,Wicker Wreath Small,7.24,24,12985,United Kingdom +581501,12/9/2019,22785,Squarecushion Cover Pink Union Jack,7.24,12,12985,United Kin +581501,12/9/2019,22811,Set Of 6 T-Lights Cacti,7.24,12,12985,United Kingdom +581501,12/9/2019,22807,Set Of 6 T-Lights Toadstools,7.24,12,12985,United Kingdom +581501,12/9/2019,22942,Christmas Lights 10 Santas,7.24,12,12985,United Kingdom +581501,12/9/2019,22808,Set Of 6 T-Lights Easter Chicks,6.19,12,12985,United Kingdom +581501,12/9/2019,23143,Zinc Wire Kitchen Organiser,7.24,4,12985,United Kingdom +581501,12/9/2019,23151,Zinc Sweetheart Soap Dish,7.24,12,12985,United Kingdom +581501,12/9/2019,23425,Storage Tin Home Sweet Home,7.24,12,12985,United Kingdom +581501,12/9/2019,84356,Pompom Curtain,7.24,12,12985,United Kingdom +581501,12/9/2019,85173,Set/6 Frog Prince T-Light Candles,7.24,12,12985,United Kingd +581501,12/9/2019,21731,Red Toadstool Led Night Light,7.24,24,12985,United Kingdom +581501,12/9/2019,23480,Mini Lights Woodland Mushrooms,7.24,8,12985,United Kingdom +581501,12/9/2019,22545,Mini Jigsaw Bunnies,7.24,96,12985,United Kingdom +581502,12/9/2019,22087,Paper Bunting White Lace,7.24,6,15910,United Kingdom +581502,12/9/2019,21209,Multicolour Honeycomb Fan,7.24,5,15910,United Kingdom +581502,12/9/2019,20668,Disco Ball Christmas Decoration,7.24,24,15910,United Kingdom +581502,12/9/2019,21790,Vintage Snap Cards,7.24,6,15910,United Kingdom +581502,12/9/2019,23270,Set Of 2 Ceramic Painted Hearts,7.24,4,15910,United Kingdom +581502,12/9/2019,23103,Jingle Bell Heart Decoration,7.24,8,15910,United Kingdom +581502,12/9/2019,22576,Swallow Wooden Christmas Decoration,7.24,2,15910,United King +581502,12/9/2019,22153,Angel Decoration Stars On Dress,7.24,13,15910,United Kingdom +581502,12/9/2019,21810,Christmas Hanging Star With Bell,7.24,6,15910,United Kingdom +581502,12/9/2019,22155,Star Decoration Rustic,7.24,6,15910,United Kingdom +581502,12/9/2019,23210,White Rocking Horse Hand Painted,7.24,12,15910,United Kingdo +581502,12/9/2019,22573,Star Wooden Christmas Decoration,7.24,8,15910,United Kingdom +581502,12/9/2019,22075,6 Ribbons Elegant Christmas,7.24,4,15910,United Kingdom +581502,12/9/2019,85049A,Traditional Christmas Ribbons,7.24,4,15910,United Kingdom +581502,12/9/2019,22734,Set Of 6 Ribbons Vintage Christmas,7.24,4,15910,United Kingd +581502,12/9/2019,23274,Star T-Light Holder Willie Winkie,7.24,8,15910,United Kingdo +581502,12/9/2019,22153,Angel Decoration Stars On Dress,7.24,1,15910,United Kingdom +581502,12/9/2019,22596,Christmas Star Wish List Chalkboard,7.24,24,15910,United Kin +581502,12/9/2019,22952,60 Cake Cases Vintage Christmas,7.24,10,15910,United Kingdom +581502,12/9/2019,22141,Christmas Craft Tree Top Angel,7.24,3,15910,United Kingdom +581514,12/9/2019,22753,Small Yellow Babushka Notebook,7.24,12,17754,United Kingdom +581514,12/9/2019,22755,Small Purple Babushka Notebook,7.24,12,17754,United Kingdom +581514,12/9/2019,22754,Small Red Babushka Notebook,6.19,12,17754,United Kingdom +581514,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,6.19,4,17754,United Kingdom +581514,12/9/2019,22059,Ceramic Strawberry Design Mug,6.19,12,17754,United Kingdom +581514,12/9/2019,22199,Frying Pan Red Retrospot,6.19,13,17754,United Kingdom +581514,12/9/2019,22200,Frying Pan Pink Polkadot,6.19,2,17754,United Kingdom +581514,12/9/2019,84032A,Charlie+Lola Pink Hot Water Bottle,6.19,9,17754,United King +581514,12/9/2019,84032B,Charlie + Lola Red Hot Water Bottle,6.19,9,17754,United Kin +581514,12/9/2019,84031A,Charlie+Lola Red Hot Water Bottle,6.19,10,17754,United King +581514,12/9/2019,84031B,Charlie Lola Blue Hot Water Bottle,6.19,14,17754,United Kin +581514,12/9/2019,22646,Ceramic Strawberry Cake Money Bank,6.19,4,17754,United Kingd +581514,12/9/2019,22644,Ceramic Cherry Cake Money Bank,6.19,4,17754,United Kingdom +581514,12/9/2019,22645,Ceramic Heart Fairy Cake Money Bank,6.19,4,17754,United King +581514,12/9/2019,22394,Paperweight Kings Choice,6.04,12,17754,United Kingdom +581514,12/9/2019,17091J,Vanilla Incense In Tin,6.04,66,17754,United Kingdom +581514,12/9/2019,35471D,Set Of 3 Bird Light Pink Feather,6.04,12,17754,United Kingd +581514,12/9/2019,22075,6 Ribbons Elegant Christmas,6.04,24,17754,United Kingdom +581514,12/9/2019,17091J,Vanilla Incense In Tin,6.19,24,17754,United Kingdom +581514,12/9/2019,22069,Brown Pirate Treasure Chest,6.19,20,17754,United Kingdom +581514,12/9/2019,22068,Black Pirate Treasure Chest,6.19,14,17754,United Kingdom +581514,12/9/2019,22500,Set Of 2 Tins Jardin De Provence,6.19,4,17754,United Kingdom +581514,12/9/2019,22075,6 Ribbons Elegant Christmas,6.19,24,17754,United Kingdom +581514,12/9/2019,21705,Bag 500g Swirly Marbles,6.19,84,17754,United Kingdom +581516,12/9/2019,21109,Large Cake Towel Chocolate Spots,6.19,12,14422,United Kingdo +581516,12/9/2019,21111,Swiss Roll Towel Chocolate Spots,6.19,24,14422,United Kingdo +581516,12/9/2019,21705,Bag 500g Swirly Marbles,6.19,24,14422,United Kingdom +581516,12/9/2019,22185,Slate Tile Natural Hanging,6.19,12,14422,United Kingdom +581516,12/9/2019,22442,Grow Your Own Flowers Set Of 3,6.19,12,14422,United Kingdom +581516,12/9/2019,21620,Set Of 4 Rose Botanical Candles,6.19,12,14422,United Kingdom +581516,12/9/2019,84029G,Knitted Union Flag Hot Water Bottle,6.19,8,14422,United Kin +581516,12/9/2019,21485,Retrospot Heart Hot Water Bottle,6.19,3,14422,United Kingdom +581516,12/9/2019,22111,Scottie Dog Hot Water Bottle,6.19,6,14422,United Kingdom +581516,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,8,14422,United Kingdom +581516,12/9/2019,22112,Chocolate Hot Water Bottle,6.19,6,14422,United Kingdom +581516,12/9/2019,23356,Love Hot Water Bottle,6.19,6,14422,United Kingdom +581516,12/9/2019,21108,Fairy Cake Flannel Assorted Colour,6.19,18,14422,United King +581516,12/9/2019,22171,3 Hook Photo Shelf Antique White,6.19,4,14422,United Kingdom +581516,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,24,14422,United Kingdo +581538,12/9/2019,23193,Buffalo Bill Treasure Book Box,6.19,6,14446,United Kingdom +581538,12/9/2019,23194,Gymkhana Treasure Book Box,6.19,1,14446,United Kingdom +581538,12/9/2019,23084,Rabbit Night Light,6.19,2,14446,United Kingdom +581538,12/9/2019,22068,Black Pirate Treasure Chest,6.19,1,14446,United Kingdom +581538,12/9/2019,22956,36 Foil Heart Cake Cases,6.19,1,14446,United Kingdom +581538,12/9/2019,20936,Forked Cactus Candle,6.19,1,14446,United Kingdom +581538,12/9/2019,23040,Paper Lantern 9 Point Snow Star,6.19,1,14446,United Kingdom +581538,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,6.19,1,14446,United King +581538,12/9/2019,21222,Set/4 Badges Beetles,6.19,1,14446,United Kingdom +581538,12/9/2019,21220,Set/4 Badges Dogs,6.19,1,14446,United Kingdom +581538,12/9/2019,21224,Set/4 Skull Badges,6.19,1,14446,United Kingdom +581538,12/9/2019,85123A,Cream Hanging Heart T-Light Holder,6.19,1,14446,United King +581538,12/9/2019,22992,Revolver Wooden Ruler,6.19,1,14446,United Kingdom +581538,12/9/2019,79066K,Retro Mod Tray,6.19,1,14446,United Kingdom +581538,12/9/2019,84380,Set Of 3 Butterfly Cookie Cutters,6.19,1,14446,United Kingdo +581538,12/9/2019,23371,Set 36 Colour Pencils Spaceboy,6.19,1,14446,United Kingdom +581538,12/9/2019,22694,Wicker Star,6.19,1,14446,United Kingdom +581538,12/9/2019,22095,Lads Only Tissue Box,6.19,1,14446,United Kingdom +581538,12/9/2019,23275,Set Of 3 Hanging Owls Ollie Beak,6.19,1,14446,United Kingdom +581538,12/9/2019,21673,White Spot Blue Ceramic Drawer Knob,6.19,1,14446,United King +581538,12/9/2019,21670,Blue Spot Ceramic Drawer Knob,6.19,1,14446,United Kingdom +581538,12/9/2019,85071C,"Charlie+Lola""Extremely Busy"" Sign",6.19,1,14446,United K +581538,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,3,14446,United Kingdom +581538,12/9/2019,23034,Drawer Knob Ceramic Black,6.19,1,14446,United Kingdom +581538,12/9/2019,21669,Blue Stripe Ceramic Drawer Knob,6.19,1,14446,United Kingdom +581538,12/9/2019,23033,Drawer Knob Ceramic Red,6.19,1,14446,United Kingdom +581538,12/9/2019,21668,Red Stripe Ceramic Drawer Knob,6.19,1,14446,United Kingdom +581538,12/9/2019,23275,Set Of 3 Hanging Owls Ollie Beak,6.19,1,14446,United Kingdom +581538,12/9/2019,23318,Box Of 6 Mini Vintage Crackers,6.19,1,14446,United Kingdom +581538,12/9/2019,23382,Box Of 6 Christmas Cake Decorations,6.19,1,14446,United King +581538,12/9/2019,22469,Heart Of Wicker Small,6.19,1,14446,United Kingdom +581538,12/9/2019,22899,Children's Apron Dolly Girl,6.19,2,14446,United Kingdom +581538,12/9/2019,22367,Childrens Apron Spaceboy Design,6.19,1,14446,United Kingdom +581538,12/9/2019,22899,Children's Apron Dolly Girl,6.19,3,14446,United Kingdom +581538,12/9/2019,23527,Wall Art Animals And Nature,6.19,1,14446,United Kingdom +581538,12/9/2019,23524,Wall Art Horse & Pony,6.19,1,14446,United Kingdom +581538,12/9/2019,23525,Wall Art Buffalo Bill,6.19,1,14446,United Kingdom +581538,12/9/2019,84380,Set Of 3 Butterfly Cookie Cutters,6.19,4,14446,United Kingdo +581538,12/9/2019,23122,Party Charms 50 Pieces,7.24,1,14446,United Kingdom +581538,12/9/2019,21990,Modern Floral Stationery Set,7.24,1,14446,United Kingdom +581538,12/9/2019,21329,Dinosaurs Writing Set,7.24,1,14446,United Kingdom +581538,12/9/2019,21328,Balloons Writing Set,7.24,1,14446,United Kingdom +581538,12/9/2019,22561,Wooden School Colouring Set,7.24,1,14446,United Kingdom +581538,12/9/2019,84519A,Tomato Charlie+Lola Coaster Set,7.24,1,14446,United Kingdom +581538,12/9/2019,84519B,Carrot Charlie+Lola Coaster Set,7.24,1,14446,United Kingdom +581538,12/9/2019,35004B,Set Of 3 Black Flying Ducks,7.24,2,14446,United Kingdom +581538,12/9/2019,22068,Black Pirate Treasure Chest,7.24,1,14446,United Kingdom +581538,12/9/2019,23353,6 Gift Tags Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,21591,Cosy Hour Cigar Box Matches,6.19,1,14446,United Kingdom +581538,12/9/2019,22197,Popcorn Holder,6.19,4,14446,United Kingdom +581538,12/9/2019,23320,Giant 50'S Christmas Cracker,6.19,1,14446,United Kingdom +581538,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,22985,Wrap Billboard Fonts Design,6.19,25,14446,United Kingdom +581538,12/9/2019,23040,Paper Lantern 9 Point Snow Star,6.19,2,14446,United Kingdom +581538,12/9/2019,23040,Paper Lantern 9 Point Snow Star,6.19,3,14446,United Kingdom +581538,12/9/2019,21194,Pink Honeycomb Paper Fan,6.19,2,14446,United Kingdom +581538,12/9/2019,21208,Pastel Colour Honeycomb Fan,6.19,1,14446,United Kingdom +581538,12/9/2019,79190D,Retro Plastic Daisy Tray,6.19,1,14446,United Kingdom +581538,12/9/2019,79190B,Retro Plastic Polka Tray,6.19,1,14446,United Kingdom +581538,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,6.02,2,14446,United King +581538,12/9/2019,23318,Box Of 6 Mini Vintage Crackers,6.02,1,14446,United Kingdom +581538,12/9/2019,23319,Box Of 6 Mini 50'S Crackers,6.02,1,14446,United Kingdom +581538,12/9/2019,23537,Wall Art I Love London,6.19,1,14446,United Kingdom +581538,12/9/2019,22992,Revolver Wooden Ruler,6.19,1,14446,United Kingdom +581538,12/9/2019,22991,Giraffe Wooden Ruler,6.19,1,14446,United Kingdom +581538,12/9/2019,23190,Bundle Of 3 School Exercise Books,6.19,1,14446,United Kingdo +581538,12/9/2019,21194,Pink Honeycomb Paper Fan,6.19,1,14446,United Kingdom +581538,12/9/2019,35004B,Set Of 3 Black Flying Ducks,6.19,1,14446,United Kingdom +581538,12/9/2019,22694,Wicker Star,6.19,1,14446,United Kingdom +581538,12/9/2019,21355,Toast Its - I Love You,6.19,1,14446,United Kingdom +581538,12/9/2019,23343,Jumbo Bag Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,23343,Jumbo Bag Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,20727,Lunch Bag Black Skull,6.19,1,14446,United Kingdom +581538,12/9/2019,20725,Lunch Bag Red Retrospot,6.19,1,14446,United Kingdom +581566,12/9/2019,23404,Home Sweet Home Blackboard,6.19,144,18102,United Kingdom +581567,12/9/2019,21417,Cockle Shell Dish,6.19,84,16626,United Kingdom +581567,12/9/2019,22464,Hanging Metal Heart Lantern,6.19,24,16626,United Kingdom +581567,12/9/2019,22465,Hanging Metal Star Lantern,6.19,24,16626,United Kingdom +581567,12/9/2019,84971S,Small Heart Flowers Hook,6.19,48,16626,United Kingdom +581567,12/9/2019,22624,Ivory Kitchen Scales,6.19,2,16626,United Kingdom +581567,12/9/2019,22627,Mint Kitchen Scales,6.19,2,16626,United Kingdom +581567,12/9/2019,22625,Red Kitchen Scales,6.19,2,16626,United Kingdom +581567,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,4,16626,United Kingdom +581567,12/9/2019,21326,Aged Glass Silver T-Light Holder,6.19,144,16626,United Kingd +581567,12/9/2019,21479,White Skull Hot Water Bottle,6.19,4,16626,United Kingdom +581567,12/9/2019,23356,Love Hot Water Bottle,6.19,3,16626,United Kingdom +581567,12/9/2019,21137,Black Record Cover Frame,6.19,24,16626,United Kingdom +581570,12/9/2019,22141,Christmas Craft Tree Top Angel,6.19,6,12662,Germany +581570,12/9/2019,22175,Pink Owl Soft Toy,6.19,6,12662,Germany +581570,12/9/2019,21481,Fawn Blue Hot Water Bottle,6.19,4,12662,Germany +581570,12/9/2019,23570,Traditional Pick Up Sticks Game,6.19,12,12662,Germany +581570,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,6.19,6,12662,Germany +581570,12/9/2019,22331,Woodland Party Bag + Sticker Set,6.19,8,12662,Germany +581570,12/9/2019,22834,Hand Warmer Babushka Design,6.19,12,12662,Germany +581570,12/9/2019,21914,Blue Harmonica In Box,6.19,12,12662,Germany +581570,12/9/2019,22139,Retrospot Tea Set Ceramic 11 Pc,6.19,3,12662,Germany +581570,12/9/2019,23077,Doughnut Lip Gloss,6.19,20,12662,Germany +581570,12/9/2019,20750,Red Retrospot Mini Cases,6.19,2,12662,Germany +581570,12/9/2019,22505,Memo Board Cottage Design,6.19,4,12662,Germany +581571,12/9/2019,23326,Hanging Mini Coloured Bottles,6.19,6,15311,United Kingdom +581571,12/9/2019,21313,Glass Heart T-Light Holder,6.19,1,15311,United Kingdom +581571,12/9/2019,48187,Doormat New England,6.19,1,15311,United Kingdom +581571,12/9/2019,23317,Blue Refectory Clock,6.19,1,15311,United Kingdom +581571,12/9/2019,23197,Sketchbook Magnetic Shopping List,6.19,4,15311,United Kingdo +581571,12/9/2019,21012,Antique All Glass Candlestick,6.19,2,15311,United Kingdom +581571,12/9/2019,22227,Hanging Heart Mirror Decoration,6.19,10,15311,United Kingdom +581571,12/9/2019,22794,Sweetheart Wire Magazine Rack,6.19,1,15311,United Kingdom +581571,12/9/2019,23182,Toilet Sign Occupied Or Vacant,6.19,1,15311,United Kingdom +581571,12/9/2019,21755,Love Building Block Word,6.19,1,15311,United Kingdom +581571,12/9/2019,85053,French Enamel Candleholder,6.19,2,15311,United Kingdom +581571,12/9/2019,23110,Parisienne Key Cabinet,6.19,2,15311,United Kingdom +581571,12/9/2019,21169,You're Confusing Me Metal Sign,6.19,1,15311,United Kingdom +581571,12/9/2019,21258,Victorian Sewing Box Large,6.19,8,15311,United Kingdom +581571,12/9/2019,23168,Classic Cafe Sugar Dispenser,6.19,36,15311,United Kingdom +581571,12/9/2019,23167,Small Ceramic Top Storage Jar,6.19,96,15311,United Kingdom +581571,12/9/2019,21314,Small Glass Heart Trinket Pot,6.19,48,15311,United Kingdom +581571,12/9/2019,21137,Black Record Cover Frame,6.19,24,15311,United Kingdom +581571,12/9/2019,44234,Assorted Circular Mobile,6.19,1,15311,United Kingdom +581571,12/9/2019,84347,Rotating Silver Angels T-Light Hldr,6.19,24,15311,United Kin +581572,12/9/2019,23328,Set 6 School Milk Bottles In Crate,6.19,48,16705,United King +581572,12/9/2019,22627,Mint Kitchen Scales,6.19,4,16705,United Kingdom +581572,12/9/2019,22624,Ivory Kitchen Scales,6.19,4,16705,United Kingdom +581572,12/9/2019,23245,Set Of 3 Regency Cake Tins,6.19,4,16705,United Kingdom +581574,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,6.19,6,12526,Germany +581574,12/9/2019,22328,Round Snack Boxes Set Of 4 Fruits,6.19,6,12526,Germany +581574,12/9/2019,23238,Set Of 4 Knick Knack Tins London,6.19,6,12526,Germany +581574,12/9/2019,23237,Set Of 4 Knick Knack Tins Leaf,6.19,6,12526,Germany +581574,12/9/2019,21257,Victorian Sewing Box Medium,6.19,2,12526,Germany +581574,12/9/2019,21258,Victorian Sewing Box Large,6.19,2,12526,Germany +581574,12/9/2019,23111,Parisienne Sewing Box,6.19,2,12526,Germany +581574,12/9/2019,22077,6 Ribbons Rustic Charm,6.19,12,12526,Germany +581574,12/9/2019,22074,6 Ribbons Shimmering Pinks,6.19,12,12526,Germany +581574,12/9/2019,22621,Traditional Knitting Nancy,6.19,12,12526,Germany +581574,12/9/2019,23199,Jumbo Bag Apples,6.19,10,12526,Germany +581574,12/9/2019,23581,Jumbo Bag Paisley Park,6.19,10,12526,Germany +581578,12/9/2019,21124,Set/10 Blue Polkadot Party Candles,6.19,24,12713,Germany +581578,12/9/2019,21122,Set/10 Pink Polkadot Party Candles,6.19,24,12713,Germany +581578,12/9/2019,21121,Set/10 Red Polkadot Party Candles,6.19,24,12713,Germany +581578,12/9/2019,23389,Spaceboy Mini Backpack,6.04,4,12713,Germany +581578,12/9/2019,22556,Plasters In Tin Circus Parade,6.19,12,12713,Germany +581578,12/9/2019,22976,Circus Parade Childrens Egg Cup,6.19,12,12713,Germany +581578,12/9/2019,23255,Childrens Cutlery Circus Parade,7.24,12,12713,Germany +581578,12/9/2019,84997D,Childrens Cutlery Polkadot Pink,7.24,8,12713,Germany +581578,12/9/2019,84997B,Childrens Cutlery Retrospot Red,7.24,8,12713,Germany +581578,12/9/2019,84997C,Childrens Cutlery Polkadot Blue,7.24,8,12713,Germany +581578,12/9/2019,22555,Plasters In Tin Strongman,7.24,12,12713,Germany +581578,12/9/2019,21914,Blue Harmonica In Box,7.24,12,12713,Germany +581578,12/9/2019,22549,Picture Dominoes,7.24,24,12713,Germany +581578,12/9/2019,21918,Set 12 Kids Colour Chalk Sticks,7.24,24,12713,Germany +581578,12/9/2019,22992,Revolver Wooden Ruler,7.24,12,12713,Germany +581578,12/9/2019,22991,Giraffe Wooden Ruler,7.24,12,12713,Germany +581578,12/9/2019,23229,Vintage Donkey Tail Game,7.24,6,12713,Germany +581578,12/9/2019,22622,Box Of Vintage Alphabet Blocks,6.19,6,12713,Germany +581578,12/9/2019,21506,Fancy Font Birthday Card,6.19,12,12713,Germany +581578,12/9/2019,21507,Elephant Birthday Card,6.19,12,12713,Germany +581578,12/9/2019,23037,Candle Holder Silver Madeline,6.19,12,12713,Germany +581578,12/9/2019,23550,Wrap Alphabet Poster,6.19,25,12713,Germany +581578,12/9/2019,22711,Wrap Circus Parade,6.19,25,12713,Germany +581578,12/9/2019,21497,Fancy Fonts Birthday Wrap,6.19,25,12713,Germany +581578,12/9/2019,22704,Wrap Red Apples,6.19,25,12713,Germany +581578,12/9/2019,22585,Pack Of 6 Birdy Gift Tags,6.19,12,12713,Germany diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/go-challenge/main.go b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/go-challenge/main.go new file mode 100644 index 0000000000000..7161dcc40fc33 --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/go-challenge/main.go @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// beam-playground: +// name: FinalChallenge1 +// description: Final challenge 1. +// multifile: true +// files: +// - name: input.csv +// context_line: 54 +// categories: +// - Quickstart +// complexity: ADVANCED +// tags: +// - hellobeam + +package main + +import ( + "context" + "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/io/textio" + "github.com/apache/beam/sdks/v2/go/pkg/beam/x/beamx" + "log" +) + +type Transaction struct { + ID int64 + Date string + ProductID string + ProductName string + Price float64 + Quantity int64 + CustomerID int64 + Country string +} + +func main() { + beam.Init() + p := beam.NewPipeline() + s := p.Root() + + file := textio.Read(s, "input.csv") + + textio.Write(s, "price_less_than_10.txt", file) + + if err := beamx.Run(context.Background(), p); err != nil { + log.Fatalf("Failed to execute job: %v", err) + } +} diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/go-solution/input.csv b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/go-solution/input.csv new file mode 100644 index 0000000000000..85854a2d43584 --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/go-solution/input.csv @@ -0,0 +1,1500 @@ +TransactionNo,Date,ProductNo,ProductName,Price,Quantity,CustomerNo,Country +581482,12/9/2019,22485,Set Of 2 Wooden Market Crates,21.47,12,17490,United Kingdom +581475,12/9/2019,22596,Christmas Star Wish List Chalkboard,10.65,36,13069,United Ki +581475,12/9/2019,23235,Storage Tin Vintage Leaf,11.53,12,13069,United Kingdom +581475,12/9/2019,23272,Tree T-Light Holder Willie Winkie,10.65,12,13069,United King +581475,12/9/2019,23239,Set Of 4 Knick Knack Tins Poppies,11.94,6,13069,United Kingd +581475,12/9/2019,21705,Bag 500g Swirly Marbles,10.65,24,13069,United Kingdom +581475,12/9/2019,22118,Joy Wooden Block Letters,11.53,18,13069,United Kingdom +581475,12/9/2019,22119,Peace Wooden Block Letters,12.25,12,13069,United Kingdom +581475,12/9/2019,22217,T-Light Holder Hanging Lace,10.65,12,13069,United Kingdom +581475,12/9/2019,22216,T-Light Holder White Lace,10.55,24,13069,United Kingdom +581475,12/9/2019,22380,Toy Tidy Spaceboy,11.06,20,13069,United Kingdom +581475,12/9/2019,22442,Grow Your Own Flowers Set Of 3,12.25,12,13069,United Kingdom +581475,12/9/2019,22664,Toy Tidy Dolly Girl Design,11.06,20,13069,United Kingdom +581475,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,12.25,12,13069,United Kingdom +581475,12/9/2019,22723,Set Of 6 Herb Tins Sketchbook,11.53,12,13069,United Kingdom +581475,12/9/2019,22785,Squarecushion Cover Pink Union Jack,11.53,12,13069,United Ki +581475,12/9/2019,22955,36 Foil Star Cake Cases,11.06,24,13069,United Kingdom +581475,12/9/2019,23141,Triple Wire Hook Pink Heart,11.06,12,13069,United Kingdom +581475,12/9/2019,22956,36 Foil Heart Cake Cases,11.06,24,13069,United Kingdom +581475,12/9/2019,22581,Wood Stocking Christmas Scandispot,10.55,48,13069,United Kin +581476,12/9/2019,23198,Pantry Magnetic Shopping List,11.53,48,12433,Norway +581476,12/9/2019,23197,Sketchbook Magnetic Shopping List,11.74,24,12433,Norway +581476,12/9/2019,23184,Bull Dog Bottle Opener,15.32,8,12433,Norway +581476,12/9/2019,23168,Classic Cafe Sugar Dispenser,11.53,12,12433,Norway +581476,12/9/2019,23167,Small Ceramic Top Storage Jar,10.96,96,12433,Norway +581476,12/9/2019,23166,Medium Ceramic Top Storage Jar,11.32,48,12433,Norway +581476,12/9/2019,23165,Large Ceramic Top Storage Jar,11.74,48,12433,Norway +581476,12/9/2019,23004,Travel Card Wallet Pantry,10.68,48,12433,Norway +581476,12/9/2019,23002,Travel Card Wallet Skulls,10.68,24,12433,Norway +581476,12/9/2019,23000,Travel Card Wallet Transport,10.68,24,12433,Norway +581476,12/9/2019,22998,Travel Card Wallet Keep Calm,10.68,72,12433,Norway +581476,12/9/2019,22994,Travel Card Wallet Retrospot,10.68,48,12433,Norway +581476,12/9/2019,22835,Hot Water Bottle I Am So Poorly,15.32,8,12433,Norway +581476,12/9/2019,22730,Alarm Clock Bakelike Ivory,14.09,4,12433,Norway +581476,12/9/2019,22728,Alarm Clock Bakelike Pink,14.09,8,12433,Norway +581476,12/9/2019,22727,Alarm Clock Bakelike Red,14.09,8,12433,Norway +581476,12/9/2019,22726,Alarm Clock Bakelike Green,14.09,4,12433,Norway +581476,12/9/2019,22720,Set Of 3 Cake Tins Pantry Design,14.61,16,12433,Norway +581476,12/9/2019,22693,Grow A Flytrap Or Sunflower In Tin,11.34,192,12433,Norway +581476,12/9/2019,22670,French Wc Sign Blue Metal,11.53,24,12433,Norway +581476,12/9/2019,22667,Recipe Box Retrospot,12.86,24,12433,Norway +581476,12/9/2019,22666,Recipe Box Pantry Yellow Design,12.86,24,12433,Norway +581476,12/9/2019,22631,Circus Parade Lunch Box,12.25,12,12433,Norway +581476,12/9/2019,22628,Picnic Boxes Set Of 3 Retrospot,15.32,16,12433,Norway +581476,12/9/2019,22467,Gumball Coat Rack,12.86,6,12433,Norway +581476,12/9/2019,22197,Popcorn Holder,10.99,100,12433,Norway +581476,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,14.61,8,12433,Norway +581476,12/9/2019,22112,Chocolate Hot Water Bottle,15.32,9,12433,Norway +581476,12/9/2019,21908,Chocolate This Way Metal Sign,12.40,36,12433,Norway +581476,12/9/2019,21874,Gin And Tonic Mug,11.74,36,12433,Norway +581476,12/9/2019,21872,Glamorous Mug,11.53,12,12433,Norway +581476,12/9/2019,21871,Save The Planet Mug,11.74,36,12433,Norway +581476,12/9/2019,21533,Retrospot Large Milk Jug,15.32,6,12433,Norway +581476,12/9/2019,21481,Fawn Blue Hot Water Bottle,14.09,12,12433,Norway +581476,12/9/2019,21479,White Skull Hot Water Bottle,14.61,4,12433,Norway +581476,12/9/2019,21248,Door Hanger Mum + Dads Room,11.74,12,12433,Norway +581476,12/9/2019,21216,Set 3 Retrospot Tea/Coffee/Sugar,14.61,24,12433,Norway +581476,12/9/2019,21181,Please One Person Metal Sign,12.15,48,12433,Norway +581476,12/9/2019,21175,Gin And Tonic Diet Metal Sign,12.38,48,12433,Norway +581476,12/9/2019,21169,You're Confusing Me Metal Sign,11.74,48,12433,Norway +581476,12/9/2019,21162,Toxic Area Door Hanger,10.65,24,12433,Norway +581476,12/9/2019,21159,Moody Boy Door Hanger,10.65,24,12433,Norway +581476,12/9/2019,21158,Moody Girl Door Hanger,10.65,24,12433,Norway +581476,12/9/2019,21154,Red Retrospot Oven Glove,11.53,10,12433,Norway +581476,12/9/2019,16016,Large Chinese Style Scissor,11.12,40,12433,Norway +581476,12/9/2019,16014,Small Chinese Style Scissor,10.68,60,12433,Norway +581476,12/9/2019,16008,Small Folding Scissor(Pointed Edge),10.37,240,12433,Norway +581476,12/9/2019,85152,Hand Over The Chocolate Sign,12.15,48,12433,Norway +581476,12/9/2019,84596F,Small Marshmallows Pink Bowl,10.68,32,12433,Norway +581476,12/9/2019,84596B,Small Dolly Mix Design Orange Bowl,10.68,16,12433,Norway +581476,12/9/2019,84510A,Set Of 4 English Rose Coasters,11.53,20,12433,Norway +581476,12/9/2019,82600,N0 Singing Metal Sign,12.15,48,12433,Norway +581476,12/9/2019,82581,Toilet Metal Sign,10.81,48,12433,Norway +581476,12/9/2019,72232,Feng Shui Pillar Candle,10.44,144,12433,Norway +581476,12/9/2019,47559B,Tea Time Oven Glove,11.53,10,12433,Norway +581476,12/9/2019,47504H,English Rose Spirit Level,11.06,36,12433,Norway +581476,12/9/2019,23493,Vintage Doily Travel Sewing Kit,12.25,30,12433,Norway +581476,12/9/2019,23430,Blue Retro Kitchen Wall Clock,18.60,2,12433,Norway +581476,12/9/2019,23429,Red Retro Kitchen Wall Clock,18.60,2,12433,Norway +581476,12/9/2019,23428,Ivory Retro Kitchen Wall Clock,18.60,2,12433,Norway +581476,12/9/2019,23358,Hot Stuff Hot Water Bottle,11.53,18,12433,Norway +581476,12/9/2019,23293,Set Of 12 Fairy Cake Baking Cases,11.10,32,12433,Norway +581476,12/9/2019,23243,Set Of Tea Coffee Sugar Tins Pantry,15.32,12,12433,Norway +581476,12/9/2019,23240,Set Of 4 Knick Knack Tins Doily,14.50,6,12433,Norway +581477,12/9/2019,48111,Doormat 3 Smiley Cats,17.51,10,13426,United Kingdom +581477,12/9/2019,22464,Hanging Metal Heart Lantern,11.06,12,13426,United Kingdom +581477,12/9/2019,20982,12 Pencils Tall Tube Skulls,11.12,12,13426,United Kingdom +581477,12/9/2019,20981,12 Pencils Tall Tube Woodland,11.12,12,13426,United Kingdom +581477,12/9/2019,23424,Gingham Recipe Book Box,15.32,8,13426,United Kingdom +581477,12/9/2019,23338,Egg Frying Pan Red,12.15,48,13426,United Kingdom +581477,12/9/2019,84970L,Single Heart Zinc T-Light Holder,11.53,12,13426,United King +581477,12/9/2019,22457,Natural Slate Heart Chalkboard,13.27,6,13426,United Kingdom +581477,12/9/2019,22469,Heart Of Wicker Small,11.94,12,13426,United Kingdom +581477,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,11.12,12,13426,United Ki +581478,12/9/2019,84947,Antique Silver Tea Glass Engraved,11.53,24,17364,United King +581478,12/9/2019,23503,Playing Cards Keep Calm & Carry On,11.53,12,17364,United Kin +581478,12/9/2019,23445,Ice Cream Bubbles,11.10,20,17364,United Kingdom +581478,12/9/2019,23530,Wall Art Only One Person,15.32,12,17364,United Kingdom +581478,12/9/2019,84997C,Childrens Cutlery Polkadot Blue,14.50,4,17364,United Kingdo +581478,12/9/2019,84997D,Childrens Cutlery Polkadot Pink,14.50,4,17364,United Kingdo +581478,12/9/2019,84077,World War 2 Gliders Asstd Designs,10.55,48,17364,United King +581478,12/9/2019,22749,Feltcraft Princess Charlotte Doll,14.09,4,17364,United Kingd +581478,12/9/2019,23127,Feltcraft Girl Nicole Kit,15.32,4,17364,United Kingdom +581478,12/9/2019,23126,Feltcraft Girl Amelie Kit,15.32,4,17364,United Kingdom +581478,12/9/2019,22747,Poppy's Playhouse Bathroom,12.40,6,17364,United Kingdom +581478,12/9/2019,22078,Ribbon Reel Lace Design,12.40,10,17364,United Kingdom +581478,12/9/2019,84946,Antique Silver T-Light Glass,11.53,12,17364,United Kingdom +581478,12/9/2019,22791,T-Light Glass Fluted Antique,11.53,12,17364,United Kingdom +581478,12/9/2019,21326,Aged Glass Silver T-Light Holder,10.92,12,17364,United Kingd +581478,12/9/2019,22170,Picture Frame Wood Triple Portrait,17.17,8,17364,United King +581478,12/9/2019,23082,Set 6 Paper Table Lantern Hearts,14.09,6,17364,United Kingdo +581478,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,11.12,24,17364,United Ki +581479,12/9/2019,22087,Paper Bunting White Lace,13.27,10,17364,United Kingdom +581480,12/9/2019,23464,Vintage Zinc Watering Can Small,15.32,4,14441,United Kingdom +581480,12/9/2019,22411,Jumbo Shopper Vintage Red Paisley,12.38,10,14441,United King +581480,12/9/2019,84029E,Red Woolly Hottie White Heart,14.61,8,14441,United Kingdom +581480,12/9/2019,22633,Hand Warmer Union Jack,12.40,12,14441,United Kingdom +581480,12/9/2019,23355,Hot Water Bottle Keep Calm,15.32,12,14441,United Kingdom +581480,12/9/2019,84029G,Knitted Union Flag Hot Water Bottle,14.61,12,14441,United K +581480,12/9/2019,22112,Chocolate Hot Water Bottle,15.32,6,14441,United Kingdom +581480,12/9/2019,84347,Rotating Silver Angels T-Light Hldr,12.86,18,14441,United Ki +581481,12/9/2019,21115,Rose Caravan Doorstop,12.25,8,17490,United Kingdom +581481,12/9/2019,22059,Ceramic Strawberry Design Mug,10.65,24,17490,United Kingdom +581481,12/9/2019,22072,Red Retrospot Tea Cup And Saucer,11.53,24,17490,United Kingd +581481,12/9/2019,22123,Ping Microwave Apron,11.06,24,17490,United Kingdom +581481,12/9/2019,22476,Empire Union Jack Tv Dinner Tray,12.25,8,17490,United Kingdo +581481,12/9/2019,22495,Set Of 2 Round Tins Camembert,11.06,12,17490,United Kingdom +581481,12/9/2019,22496,Set Of 2 Round Tins Dutch Cheese,11.06,12,17490,United Kingd +581481,12/9/2019,22513,Doorstop Football Design,11.06,8,17490,United Kingdom +581481,12/9/2019,22500,Set Of 2 Tins Jardin De Provence,11.53,12,17490,United Kingd +581481,12/9/2019,22664,Toy Tidy Dolly Girl Design,11.06,20,17490,United Kingdom +581481,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,12.25,12,17490,United Kingdom +581481,12/9/2019,22785,Squarecushion Cover Pink Union Jack,11.53,12,17490,United Ki +581481,12/9/2019,23178,Jam Clock Magnet,11.53,12,17490,United Kingdom +581481,12/9/2019,23302,Kneeling Mat Housework Design,11.06,24,17490,United Kingdom +581481,12/9/2019,23533,Wall Art Garden Haven,12.25,6,17490,United Kingdom +581481,12/9/2019,84819,Danish Rose Round Sewing Box,11.06,16,17490,United Kingdom +581482,12/9/2019,22371,Airline Bag Vintage Tokyo 78,14.30,12,17490,United Kingdom +581482,12/9/2019,21875,Kings Choice Mug,11.34,36,17490,United Kingdom +581482,12/9/2019,23251,Vintage Red Enamel Trim Mug,11.32,96,17490,United Kingdom +581482,12/9/2019,23300,Gardeners Kneeling Pad Cup Of Tea,11.74,48,17490,United King +581482,12/9/2019,22138,Baking Set 9 Piece Retrospot,14.61,24,17490,United Kingdom +581483,12/9/2019,23843,Paper Craft Little Birdie,12.38,80995,16446,United Kingdom +581485,12/9/2019,22617,Baking Set Spaceboy Design,15.32,18,17389,United Kingdom +581485,12/9/2019,20749,Assorted Colour Mini Cases,16.76,84,17389,United Kingdom +581486,12/9/2019,22910,Paper Chain Kit Vintage Christmas,13.27,12,17001,United King +581486,12/9/2019,22086,Paper Chain Kit 50'S Christmas,13.27,12,17001,United Kingdom +581486,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,12,17001,United Kingdom +581486,12/9/2019,23352,Roll Wrap 50'S Red Christmas,6.19,12,17001,United Kingdom +581486,12/9/2019,22727,Alarm Clock Bakelike Red,6.19,8,17001,United Kingdom +581486,12/9/2019,22730,Alarm Clock Bakelike Ivory,6.19,4,17001,United Kingdom +581486,12/9/2019,22491,Pack Of 12 Coloured Pencils,6.04,12,17001,United Kingdom +581486,12/9/2019,22561,Wooden School Colouring Set,6.04,12,17001,United Kingdom +581486,12/9/2019,22489,Pack Of 12 Traditional Crayons,6.04,24,17001,United Kingdom +581486,12/9/2019,22560,Traditional Modelling Clay,6.04,24,17001,United Kingdom +581486,12/9/2019,22139,Retrospot Tea Set Ceramic 11 Pc,6.04,6,17001,United Kingdom +581486,12/9/2019,23344,Jumbo Bag 50'S Christmas,6.19,20,17001,United Kingdom +581486,12/9/2019,23203,Jumbo Bag Vintage Doily,6.19,20,17001,United Kingdom +581486,12/9/2019,85099B,Jumbo Bag Red Retrospot,6.19,10,17001,United Kingdom +581486,12/9/2019,23201,Jumbo Bag Alphabet,6.19,20,17001,United Kingdom +581486,12/9/2019,23207,Lunch Bag Alphabet Design,6.19,10,17001,United Kingdom +581487,12/9/2019,21137,Black Record Cover Frame,6.04,120,15694,United Kingdom +581488,12/9/2019,23118,Parisienne Jewellery Drawer,6.04,16,17428,United Kingdom +581488,12/9/2019,23111,Parisienne Sewing Box,6.04,16,17428,United Kingdom +581488,12/9/2019,22179,Set 10 Night Owl Lights,6.04,24,17428,United Kingdom +581489,12/9/2019,22061,Large Cake Stand Hanging Strawbery,7.24,48,16954,United King +581489,12/9/2019,22182,Cake Stand Victorian Filigree Small,7.24,24,16954,United Kin +581491,12/9/2019,23571,Traditional Naughts & Crosses,7.24,12,12433,Norway +581492,12/9/2019,23369,Set 36 Colour Pencils Love London,6.19,2,15492,United Kingdo +581492,12/9/2019,23370,Set 36 Colouring Pencils Doily,6.19,2,15492,United Kingdom +581492,12/9/2019,23371,Set 36 Colour Pencils Spaceboy,6.19,3,15492,United Kingdom +581492,12/9/2019,23372,Set 36 Colour Pencils Dolly Girl,6.19,1,15492,United Kingdom +581492,12/9/2019,23376,Pack Of 12 Vintage Christmas Tissue,6.19,3,15492,United King +581492,12/9/2019,23377,Pack Of 12 Dolly Girl Tissues,6.19,6,15492,United Kingdom +581492,12/9/2019,23378,Pack Of 12 50'S Christmas Tissues,6.19,9,15492,United Kingdo +581492,12/9/2019,23379,Pack Of 12 Red Apple Tissues,6.19,4,15492,United Kingdom +581492,12/9/2019,23380,Pack Of 12 Vintage Doily Tissues,6.19,3,15492,United Kingdom +581492,12/9/2019,23381,Pack Of 12 Vintage Leaf Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,23382,Box Of 6 Christmas Cake Decorations,6.19,4,15492,United King +581492,12/9/2019,23391,I Love London Mini Backpack,6.19,2,15492,United Kingdom +581492,12/9/2019,23392,Spaceboy Rocket Lolly Makers,6.19,2,15492,United Kingdom +581492,12/9/2019,23399,Home Sweet Home Hanging Heart,6.19,4,15492,United Kingdom +581492,12/9/2019,23405,Home Sweet Home 2 Drawer Cabinet,6.19,3,15492,United Kingdom +581492,12/9/2019,23418,Lavender Toilette Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,23426,Metal Sign Drop Your Pants,6.19,1,15492,United Kingdom +581492,12/9/2019,23434,3 Raffia Ribbons 50'S Christmas,6.19,9,15492,United Kingdom +581492,12/9/2019,23435,3 Raffia Ribbons Vintage Christmas,6.04,3,15492,United Kingd +581492,12/9/2019,23439,Hand Warmer Red Love Heart,6.19,1,15492,United Kingdom +581492,12/9/2019,23451,Square Mini Portrait Frame,6.19,1,15492,United Kingdom +581492,12/9/2019,23467,Vintage Zinc Planter,6.19,1,15492,United Kingdom +581492,12/9/2019,23469,Card Holder Love Bird Small,6.19,3,15492,United Kingdom +581492,12/9/2019,23480,Mini Lights Woodland Mushrooms,6.19,2,15492,United Kingdom +581492,12/9/2019,23493,Vintage Doily Travel Sewing Kit,6.19,3,15492,United Kingdom +581492,12/9/2019,23494,Vintage Doily Deluxe Sewing Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,23495,Set Of 3 Pantry Wooden Spoons,6.19,1,15492,United Kingdom +581492,12/9/2019,23497,Classic Chrome Bicycle Bell,6.19,4,15492,United Kingdom +581492,12/9/2019,23498,Classic Bicycle Clips,6.19,2,15492,United Kingdom +581492,12/9/2019,23501,Key Ring Baseball Boot Union Jack,6.19,3,15492,United Kingdo +581492,12/9/2019,23506,Mini Playing Cards Spaceboy,6.04,1,15492,United Kingdom +581492,12/9/2019,23508,Mini Playing Cards Dolly Girl,6.04,2,15492,United Kingdom +581492,12/9/2019,23510,Mini Playing Cards Gymkhana,6.04,1,15492,United Kingdom +581492,12/9/2019,23521,Wall Art Cat And Bird,6.04,1,15492,United Kingdom +581492,12/9/2019,23526,Wall Art Dog Licence,6.04,4,15492,United Kingdom +581492,12/9/2019,23530,Wall Art Only One Person,6.04,2,15492,United Kingdom +581492,12/9/2019,23534,Wall Art Stop For Tea,6.19,6,15492,United Kingdom +581492,12/9/2019,23535,Wall Art Bicycle Safety,6.19,4,15492,United Kingdom +581492,12/9/2019,23536,Wall Art Village Show,6.19,1,15492,United Kingdom +581492,12/9/2019,23538,Wall Art Vintage Heart,6.19,1,15492,United Kingdom +581492,12/9/2019,23551,Pack Of 12 Paisley Park Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,23552,Bicycle Puncture Repair Kit,6.19,12,15492,United Kingdom +581492,12/9/2019,23555,Landmark Frame Notting Hill,6.19,1,15492,United Kingdom +581492,12/9/2019,23559,Woodland Bunnies Lolly Makers,6.19,5,15492,United Kingdom +581492,12/9/2019,23561,Set Of 6 Ribbons Party,6.19,1,15492,United Kingdom +581492,12/9/2019,23564,Egg Cup Milkmaid Ingrid,6.19,2,15492,United Kingdom +581492,12/9/2019,23565,Egg Cup Milkmaid Helga,6.19,2,15492,United Kingdom +581492,12/9/2019,23567,Egg Cup Henrietta Hen Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23569,Tradtional Alphabet Stamp Set,6.19,2,15492,United Kingdom +581492,12/9/2019,23570,Traditional Pick Up Sticks Game,6.19,7,15492,United Kingdom +581492,12/9/2019,23571,Traditional Naughts & Crosses,6.19,2,15492,United Kingdom +581492,12/9/2019,23575,Snack Tray Paisley Park,6.19,3,15492,United Kingdom +581492,12/9/2019,23579,Snack Tray I Love London,6.19,1,15492,United Kingdom +581492,12/9/2019,23611,Set 10 Cards Red Riding Hood 17214,6.19,3,15492,United Kingd +581492,12/9/2019,23616,Set 10 Cards Jingle Bells 17217,6.19,1,15492,United Kingdom +581492,12/9/2019,23621,Set 10 Cards David's Madonna 17074,6.19,3,15492,United Kingd +581492,12/9/2019,23635,Set 10 Cards Christmas Holly 17259,6.19,1,15492,United Kingd +581492,12/9/2019,23644,Set 10 Cards Christmas Tree 16955,6.19,1,15492,United Kingdo +581492,12/9/2019,23660,Henrietta Hen Mug,6.19,1,15492,United Kingdom +581492,12/9/2019,23661,Milk Maids Mug,6.19,1,15492,United Kingdom +581492,12/9/2019,35095A,Blue Victorian Fabric Oval Box,6.19,1,15492,United Kingdom +581492,12/9/2019,35095B,Red Victorian Fabric Oval Box,6.19,1,15492,United Kingdom +581492,12/9/2019,35646,Vintage Bead Pink Evening Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,35648,Vintage Bead Pink Purse,6.19,2,15492,United Kingdom +581492,12/9/2019,35953,Folkart Star Christmas Decorations,6.19,12,15492,United King +581492,12/9/2019,35964,Folkart Clip On Stars,6.19,4,15492,United Kingdom +581492,12/9/2019,35970,Zinc Folkart Sleigh Bells,6.04,1,15492,United Kingdom +581492,12/9/2019,46118,Funky Monkey Cushion Cover,6.19,1,15492,United Kingdom +581492,12/9/2019,46776A,Woven Bubble Gum Cushion Cover,7.24,2,15492,United Kingdom +581492,12/9/2019,46776B,Woven Berries Cushion Cover,7.24,3,15492,United Kingdom +581492,12/9/2019,46776C,Woven Frost Cushion Cover,7.24,1,15492,United Kingdom +581492,12/9/2019,46776D,Woven Sunset Cushion Cover,7.24,2,15492,United Kingdom +581492,12/9/2019,46776E,Woven Candy Cushion Cover,7.24,5,15492,United Kingdom +581492,12/9/2019,46776F,Woven Rose Garden Cushion Cover,7.24,6,15492,United Kingdom +581492,12/9/2019,47310M,Small Pop Box Funky Monkey,6.19,1,15492,United Kingdom +581492,12/9/2019,47480,Hanging Photo Clip Rope Ladder,6.19,3,15492,United Kingdom +581492,12/9/2019,47504H,English Rose Spirit Level,7.24,1,15492,United Kingdom +581492,12/9/2019,47559B,Tea Time Oven Glove,7.24,3,15492,United Kingdom +581492,12/9/2019,47563A,Retro Longboard Ironing Board Cover,7.24,2,15492,United Kin +581492,12/9/2019,47566,Party Bunting,7.24,2,15492,United Kingdom +581492,12/9/2019,47590B,Pink Happy Birthday Bunting,7.24,1,15492,United Kingdom +581492,12/9/2019,51014A,Feather Pen Hot Pink,7.24,2,15492,United Kingdom +581492,12/9/2019,51014L,Feather Pen Light Pink,7.24,1,15492,United Kingdom +581492,12/9/2019,71270,Photo Clip Line,7.24,1,15492,United Kingdom +581492,12/9/2019,72349B,Set/6 Purple Butterfly T-Lights,6.39,1,15492,United Kingdom +581492,12/9/2019,72741,Grand Chocolatecandle,7.24,3,15492,United Kingdom +581492,12/9/2019,72800E,4 Ivory Dinner Candles Silver Flock,7.24,3,15492,United Kin +581492,12/9/2019,79321,Chilli Lights,6.19,1,15492,United Kingdom +581492,12/9/2019,82484,Wood Black Board Ant White Finish,6.19,2,15492,United Kingdo +581492,12/9/2019,82567,Airline Lounge Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,82582,Area Patrolled Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,82600,N0 Singing Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,84031A,Charlie+Lola Red Hot Water Bottle,6.19,4,15492,United Kingd +581492,12/9/2019,84077,World War 2 Gliders Asstd Designs,6.19,1,15492,United Kingdo +581492,12/9/2019,84249A,Greeting Card Square Doughnuts,6.19,1,15492,United Kingdom +581492,12/9/2019,84347,Rotating Silver Angels T-Light Hldr,6.19,2,15492,United King +581492,12/9/2019,84375,Set Of 20 Kids Cookie Cutters,6.19,1,15492,United Kingdom +581492,12/9/2019,84378,Set Of 3 Heart Cookie Cutters,6.19,4,15492,United Kingdom +581492,12/9/2019,84519B,Carrot Charlie+Lola Coaster Set,6.19,1,15492,United Kingdom +581492,12/9/2019,84559A,3d Sheet Of Dog Stickers,6.19,2,15492,United Kingdom +581492,12/9/2019,84568,Girls Alphabet Iron On Patches,6.19,31,15492,United Kingdom +581492,12/9/2019,84569D,Pack 6 Heart/Ice-Cream Patches,6.19,1,15492,United Kingdom +581492,12/9/2019,84596B,Small Dolly Mix Design Orange Bowl,6.19,4,15492,United King +581492,12/9/2019,84596F,Small Marshmallows Pink Bowl,6.19,4,15492,United Kingdom +581492,12/9/2019,84598,Boys Alphabet Iron On Patches,6.19,5,15492,United Kingdom +581492,12/9/2019,84692,Box Of 24 Cocktail Parasols,6.19,1,15492,United Kingdom +581492,12/9/2019,84755,Colour Glass T-Light Holder Hanging,6.19,4,15492,United King +581492,12/9/2019,84828,Jungle Popsicles Ice Lolly Moulds,6.19,1,15492,United Kingdo +581492,12/9/2019,84879,Assorted Colour Bird Ornament,6.19,16,15492,United Kingdom +581492,12/9/2019,84923,Pink Butterfly Handbag W Bobbles,6.04,3,15492,United Kingdom +581492,12/9/2019,84946,Antique Silver T-Light Glass,6.04,18,15492,United Kingdom +581492,12/9/2019,84970S,Hanging Heart Zinc T-Light Holder,6.04,3,15492,United Kingd +581492,12/9/2019,84978,Hanging Heart Jar T-Light Holder,6.04,4,15492,United Kingdom +581492,12/9/2019,84991,60 Teatime Fairy Cake Cases,6.04,1,15492,United Kingdom +581492,12/9/2019,84997A,Childrens Cutlery Polkadot Green,6.04,1,15492,United Kingdo +581492,12/9/2019,84997B,Childrens Cutlery Retrospot Red,6.04,1,15492,United Kingdom +581492,12/9/2019,20733,Gold Mini Tape Measure,6.04,3,15492,United Kingdom +581492,12/9/2019,20777,Chrysanthemum Notebook,6.19,2,15492,United Kingdom +581492,12/9/2019,20782,Camouflage Ear Muff Headphones,6.19,1,15492,United Kingdom +581492,12/9/2019,20914,Set/5 Red Retrospot Lid Glass Bowls,6.19,2,15492,United King +581492,12/9/2019,20931,Blue Pot Plant Candle,6.19,1,15492,United Kingdom +581492,12/9/2019,20970,Pink Floral Feltcraft Shoulder Bag,6.19,1,15492,United Kingd +581492,12/9/2019,20971,Pink Blue Felt Craft Trinket Box,6.19,2,15492,United Kingdom +581492,12/9/2019,20972,Pink Cream Felt Craft Trinket Box,6.19,3,15492,United Kingdo +581492,12/9/2019,20973,12 Pencil Small Tube Woodland,6.19,4,15492,United Kingdom +581492,12/9/2019,20978,36 Pencils Tube Skulls,6.19,1,15492,United Kingdom +581492,12/9/2019,20979,36 Pencils Tube Red Retrospot,6.19,3,15492,United Kingdom +581492,12/9/2019,20982,12 Pencils Tall Tube Skulls,6.19,2,15492,United Kingdom +581492,12/9/2019,20983,12 Pencils Tall Tube Red Retrospot,6.19,4,15492,United Kingd +581492,12/9/2019,20985,Heart Calculator,6.19,1,15492,United Kingdom +581492,12/9/2019,20986,Blue Calculator Ruler,6.19,1,15492,United Kingdom +581492,12/9/2019,21000,Rose Du Sud Cosmetics Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,21012,Antique All Glass Candlestick,6.19,3,15492,United Kingdom +581492,12/9/2019,21015,Dark Bird House Tree Decoration,6.19,8,15492,United Kingdom +581492,12/9/2019,21026,Space Owl,6.19,1,15492,United Kingdom +581492,12/9/2019,21035,Set/2 Red Retrospot Tea Towels,6.19,1,15492,United Kingdom +581492,12/9/2019,21064,Boom Box Speaker Boys,6.19,4,15492,United Kingdom +581492,12/9/2019,21065,Boom Box Speaker Girls,6.19,2,15492,United Kingdom +581492,12/9/2019,21098,Christmas Toilet Roll,6.19,1,15492,United Kingdom +581492,12/9/2019,21123,Set/10 Ivory Polkadot Party Candles,6.19,1,15492,United King +581492,12/9/2019,21125,Set 6 Football Celebration Candles,6.19,1,15492,United Kingd +581492,12/9/2019,21126,Set Of 6 Girls Celebration Candles,6.19,1,15492,United Kingd +581492,12/9/2019,21137,Black Record Cover Frame,6.19,17,15492,United Kingdom +581492,12/9/2019,21154,Red Retrospot Oven Glove,6.19,2,15492,United Kingdom +581492,12/9/2019,21158,Moody Girl Door Hanger,6.19,1,15492,United Kingdom +581492,12/9/2019,21162,Toxic Area Door Hanger,6.19,1,15492,United Kingdom +581492,12/9/2019,21166,Cook With Wine Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,21172,Party Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,21174,Pottering In The Shed Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,21175,Gin And Tonic Diet Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,21200,Multicolour Honeycomb Paper Garland,6.04,6,15492,United King +581492,12/9/2019,21201,Tropical Honeycomb Paper Garland,6.04,4,15492,United Kingdom +581492,12/9/2019,21212,Pack Of 72 Retrospot Cake Cases,6.04,2,15492,United Kingdom +581492,12/9/2019,21213,Pack Of 72 Skull Cake Cases,6.04,2,15492,United Kingdom +581492,12/9/2019,21220,Set/4 Badges Dogs,6.19,2,15492,United Kingdom +581492,12/9/2019,21224,Set/4 Skull Badges,6.19,1,15492,United Kingdom +581492,12/9/2019,21232,Strawberry Ceramic Trinket Pot,6.19,1,15492,United Kingdom +581492,12/9/2019,21257,Victorian Sewing Box Medium,6.19,2,15492,United Kingdom +581492,12/9/2019,21258,Victorian Sewing Box Large,6.19,1,15492,United Kingdom +581492,12/9/2019,21259,Victorian Sewing Box Small,6.19,1,15492,United Kingdom +581492,12/9/2019,21313,Glass Heart T-Light Holder,6.19,1,15492,United Kingdom +581492,12/9/2019,21314,Small Glass Heart Trinket Pot,6.19,2,15492,United Kingdom +581492,12/9/2019,21328,Balloons Writing Set,6.19,2,15492,United Kingdom +581492,12/9/2019,21329,Dinosaurs Writing Set,6.19,2,15492,United Kingdom +581492,12/9/2019,21356,Toast Its - Fairy Flower,6.19,16,15492,United Kingdom +581492,12/9/2019,21378,Small Tall Camphor Wood Toadstool,6.19,2,15492,United Kingdo +581492,12/9/2019,21379,Camphor Wood Portobello Mushroom,6.19,1,15492,United Kingdom +581492,12/9/2019,21383,Pack Of 12 Sticky Bunnies,6.19,1,15492,United Kingdom +581492,12/9/2019,21402,Red Egg Spoon,6.19,1,15492,United Kingdom +581492,12/9/2019,21408,Spotty Pink Duck Doorstop,6.19,2,15492,United Kingdom +581492,12/9/2019,21411,Gingham Heart Doorstop Red,6.19,1,15492,United Kingdom +581492,12/9/2019,21467,Cherry Crochet Food Cover,6.19,1,15492,United Kingdom +581492,12/9/2019,21471,Strawberry Raffia Food Cover,6.19,2,15492,United Kingdom +581492,12/9/2019,21479,White Skull Hot Water Bottle,6.19,2,15492,United Kingdom +581492,12/9/2019,21481,Fawn Blue Hot Water Bottle,6.19,3,15492,United Kingdom +581492,12/9/2019,21484,Chick Grey Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,21485,Retrospot Heart Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,21494,Rotating Leaves T-Light Holder,6.19,11,15492,United Kingdom +581492,12/9/2019,21506,Fancy Font Birthday Card,6.19,3,15492,United Kingdom +581492,12/9/2019,21507,Elephant Birthday Card,7.24,1,15492,United Kingdom +581492,12/9/2019,21508,Vintage Kid Dolly Card,7.24,1,15492,United Kingdom +581492,12/9/2019,21509,Cowboys And Indians Birthday Card,7.24,2,15492,United Kingdo +581492,12/9/2019,21528,Dairy Maid Traditional Teapot,7.24,1,15492,United Kingdom +581492,12/9/2019,21544,Skulls Water Transfer Tattoos,7.24,2,15492,United Kingdom +581492,12/9/2019,21558,Skull Lunch Box With Cutlery,6.39,1,15492,United Kingdom +581492,12/9/2019,21559,Strawberry Lunch Box With Cutlery,7.24,1,15492,United Kingdo +581492,12/9/2019,21615,4 Lavender Botanical Dinner Candles,7.24,2,15492,United King +581492,12/9/2019,21616,4 Pear Botanical Dinner Candles,7.24,6,15492,United Kingdom +581492,12/9/2019,21620,Set Of 4 Rose Botanical Candles,7.24,5,15492,United Kingdom +581492,12/9/2019,21642,Assorted Tutti Frutti Pen,7.24,4,15492,United Kingdom +581492,12/9/2019,21648,Assorted Tutti Frutti Small Purse,7.24,2,15492,United Kingdo +581492,12/9/2019,21650,Assorted Tutti Frutti Bracelet,7.24,14,15492,United Kingdom +581492,12/9/2019,21675,Butterflies Stickers,6.19,2,15492,United Kingdom +581492,12/9/2019,21677,Hearts Stickers,6.19,1,15492,United Kingdom +581492,12/9/2019,21678,Paisley Pattern Stickers,6.19,2,15492,United Kingdom +581492,12/9/2019,21680,Woodland Stickers,6.19,1,15492,United Kingdom +581492,12/9/2019,21703,Bag 125g Swirly Marbles,6.19,1,15492,United Kingdom +581492,12/9/2019,21704,Bag 250g Swirly Marbles,6.19,1,15492,United Kingdom +581492,12/9/2019,21716,Boys Vintage Tin Seaside Bucket,6.19,1,15492,United Kingdom +581492,12/9/2019,21718,Red Metal Beach Spade,6.19,1,15492,United Kingdom +581492,12/9/2019,21724,Panda And Bunnies Sticker Sheet,6.19,1,15492,United Kingdom +581492,12/9/2019,21731,Red Toadstool Led Night Light,6.19,6,15492,United Kingdom +581492,12/9/2019,21739,Cosy Slipper Shoes Small Green,6.19,2,15492,United Kingdom +581492,12/9/2019,21774,Decorative Cats Bathroom Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,21786,Polkadot Rain Hat,6.19,3,15492,United Kingdom +581492,12/9/2019,21787,Rain Poncho Retrospot,6.19,2,15492,United Kingdom +581492,12/9/2019,21790,Vintage Snap Cards,6.19,5,15492,United Kingdom +581492,12/9/2019,21791,Vintage Heads And Tails Card Game,6.19,1,15492,United Kingdo +581492,12/9/2019,21808,Christmas Garland Stars Trees,6.19,3,15492,United Kingdom +581492,12/9/2019,21810,Christmas Hanging Star With Bell,6.19,25,15492,United Kingdo +581492,12/9/2019,21812,Garland With Hearts And Bells,6.19,7,15492,United Kingdom +581492,12/9/2019,21813,Garland With Stars And Bells,6.19,3,15492,United Kingdom +581492,12/9/2019,21822,Glitter Christmas Tree With Bells,6.19,12,15492,United Kingd +581492,12/9/2019,21828,Eight Piece Snake Set,6.19,1,15492,United Kingdom +581492,12/9/2019,21832,Chocolate Calculator,6.19,1,15492,United Kingdom +581492,12/9/2019,21833,Camouflage Led Torch,6.04,2,15492,United Kingdom +581492,12/9/2019,21871,Save The Planet Mug,6.19,1,15492,United Kingdom +581492,12/9/2019,21874,Gin And Tonic Mug,6.19,2,15492,United Kingdom +581492,12/9/2019,21876,Pottering Mug,6.19,4,15492,United Kingdom +581492,12/9/2019,21879,Hearts Gift Tape,6.19,1,15492,United Kingdom +581492,12/9/2019,21889,Wooden Box Of Dominoes,6.19,3,15492,United Kingdom +581492,12/9/2019,21890,S/6 Wooden Skittles In Cotton Bag,6.19,1,15492,United Kingdo +581492,12/9/2019,21892,Traditional Wooden Catch Cup Game,6.19,1,15492,United Kingdo +581492,12/9/2019,21900,Key Fob Shed,6.19,4,15492,United Kingdom +581492,12/9/2019,21901,Key Fob Back Door,6.19,2,15492,United Kingdom +581492,12/9/2019,21902,Key Fob Front Door,6.19,1,15492,United Kingdom +581492,12/9/2019,21905,More Butter Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,21906,Pharmacie First Aid Tin,6.19,1,15492,United Kingdom +581492,12/9/2019,21914,Blue Harmonica In Box,6.19,2,15492,United Kingdom +581492,12/9/2019,21916,Set 12 Retro White Chalk Sticks,6.19,2,15492,United Kingdom +581492,12/9/2019,21930,Jumbo Storage Bag Skulls,6.19,1,15492,United Kingdom +581492,12/9/2019,21931,Jumbo Storage Bag Suki,6.19,1,15492,United Kingdom +581492,12/9/2019,21932,Scandinavian Paisley Picnic Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,21934,Skull Shoulder Bag,6.19,3,15492,United Kingdom +581492,12/9/2019,21935,Suki Shoulder Bag,6.19,9,15492,United Kingdom +581492,12/9/2019,21936,Red Retrospot Picnic Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,21942,Skulls Design Flannel,6.19,2,15492,United Kingdom +581492,12/9/2019,21945,Strawberries Design Flannel,6.19,2,15492,United Kingdom +581492,12/9/2019,21947,Set Of 6 Heart Chopsticks,6.19,1,15492,United Kingdom +581492,12/9/2019,21949,Set Of 6 Strawberry Chopsticks,6.19,2,15492,United Kingdom +581492,12/9/2019,21967,Pack Of 12 Skull Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,21976,Pack Of 60 Mushroom Cake Cases,6.19,3,15492,United Kingdom +581492,12/9/2019,21977,Pack Of 60 Pink Paisley Cake Cases,6.19,2,15492,United Kingd +581492,12/9/2019,21980,Pack Of 12 Red Retrospot Tissues,6.19,2,15492,United Kingdom +581492,12/9/2019,21981,Pack Of 12 Woodland Tissues,6.19,4,15492,United Kingdom +581492,12/9/2019,21982,Pack Of 12 Suki Tissues,6.19,2,15492,United Kingdom +581492,12/9/2019,21983,Pack Of 12 Blue Paisley Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,21984,Pack Of 12 Pink Paisley Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,21986,Pack Of 12 Pink Polkadot Tissues,6.19,3,15492,United Kingdom +581492,12/9/2019,21989,Pack Of 20 Skull Paper Napkins,6.19,2,15492,United Kingdom +581492,12/9/2019,21990,Modern Floral Stationery Set,6.19,8,15492,United Kingdom +581492,12/9/2019,21993,Floral Folk Stationery Set,6.19,8,15492,United Kingdom +581492,12/9/2019,22024,Rainy Ladies Birthday Card,6.19,3,15492,United Kingdom +581492,12/9/2019,22025,Ring Of Roses Birthday Card,6.19,3,15492,United Kingdom +581492,12/9/2019,22026,Banquet Birthday Card,6.19,4,15492,United Kingdom +581492,12/9/2019,22027,Tea Party Birthday Card,6.19,2,15492,United Kingdom +581492,12/9/2019,22028,Penny Farthing Birthday Card,6.19,1,15492,United Kingdom +581492,12/9/2019,22029,Spaceboy Birthday Card,6.19,4,15492,United Kingdom +581492,12/9/2019,22031,Botanical Lavender Birthday Card,6.19,1,15492,United Kingdom +581492,12/9/2019,22037,Robot Birthday Card,6.19,4,15492,United Kingdom +581492,12/9/2019,22064,Pink Doughnut Trinket Pot,6.19,1,15492,United Kingdom +581492,12/9/2019,22065,Christmas Pudding Trinket Pot,6.19,2,15492,United Kingdom +581492,12/9/2019,22068,Black Pirate Treasure Chest,6.19,1,15492,United Kingdom +581492,12/9/2019,22070,Small Red Retrospot Mug In Box,6.19,3,15492,United Kingdom +581492,12/9/2019,22071,Small White Retrospot Mug In Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22074,6 Ribbons Shimmering Pinks,6.19,1,15492,United Kingdom +581492,12/9/2019,22075,6 Ribbons Elegant Christmas,6.19,8,15492,United Kingdom +581492,12/9/2019,22076,6 Ribbons Empire,6.19,4,15492,United Kingdom +581492,12/9/2019,22083,Paper Chain Kit Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22085,Paper Chain Kit Skulls,6.19,1,15492,United Kingdom +581492,12/9/2019,22086,Paper Chain Kit 50'S Christmas,6.19,38,15492,United Kingdom +581492,12/9/2019,22091,Empire Tissue Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22099,Caravan Square Tissue Box,6.19,3,15492,United Kingdom +581492,12/9/2019,22104,Mirror Mosaic Candle Plate,6.19,4,15492,United Kingdom +581492,12/9/2019,22106,Mirror Mosaic Hurricane Lamp,6.19,1,15492,United Kingdom +581492,12/9/2019,22107,Pizza Plate In Box,6.19,10,15492,United Kingdom +581492,12/9/2019,22108,Ping! Microwave Plate,6.04,2,15492,United Kingdom +581492,12/9/2019,22109,Full English Breakfast Plate,6.04,2,15492,United Kingdom +581492,12/9/2019,22110,Bird House Hot Water Bottle,6.04,3,15492,United Kingdom +581492,12/9/2019,22111,Scottie Dog Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,22112,Chocolate Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,6.19,4,15492,United Kingdo +581492,12/9/2019,22116,Metal Sign His Dinner Is Served,6.19,2,15492,United Kingdom +581492,12/9/2019,22121,Noel Wooden Block Letters,6.19,1,15492,United Kingdom +581492,12/9/2019,22123,Ping Microwave Apron,6.19,1,15492,United Kingdom +581492,12/9/2019,22124,Set Of 2 Tea Towels Ping Microwave,6.19,1,15492,United Kingd +581492,12/9/2019,22129,Party Cones Candy Decoration,6.19,3,15492,United Kingdom +581492,12/9/2019,22134,Mini Ladle Love Heart Red,6.19,1,15492,United Kingdom +581492,12/9/2019,15036,Assorted Colours Silk Fan,6.19,1,15492,United Kingdom +581492,12/9/2019,15039,Sandalwood Fan,6.19,1,15492,United Kingdom +581492,12/9/2019,15058C,Ice Cream Design Garden Parasol,6.19,1,15492,United Kingdom +581492,12/9/2019,16218,Cartoon Pencil Sharpeners,6.19,5,15492,United Kingdom +581492,12/9/2019,16225,Rattle Snake Eggs,6.19,1,15492,United Kingdom +581492,12/9/2019,16235,Recycled Pencil With Rabbit Eraser,6.19,3,15492,United Kingd +581492,12/9/2019,16237,Sleeping Cat Erasers,6.19,1,15492,United Kingdom +581492,12/9/2019,17038,Porcelain Budah Incense Holder,6.19,1,15492,United Kingdom +581492,12/9/2019,17084N,Fairy Dreams Incense,6.19,1,15492,United Kingdom +581492,12/9/2019,20659,Economy Luggage Tag,6.19,7,15492,United Kingdom +581492,12/9/2019,20665,Red Retrospot Purse,6.19,2,15492,United Kingdom +581492,12/9/2019,20668,Disco Ball Christmas Decoration,6.19,1,15492,United Kingdom +581492,12/9/2019,20718,Red Retrospot Shopper Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,20719,Woodland Charlotte Bag,6.19,7,15492,United Kingdom +581492,12/9/2019,20723,Strawberry Charlotte Bag,6.19,2,15492,United Kingdom +581492,12/9/2019,20724,Red Retrospot Charlotte Bag,6.19,4,15492,United Kingdom +581492,12/9/2019,22135,Mini Ladle Love Heart Pink,6.19,6,15492,United Kingdom +581492,12/9/2019,22138,Baking Set 9 Piece Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22141,Christmas Craft Tree Top Angel,6.19,2,15492,United Kingdom +581492,12/9/2019,22150,3 Stripey Mice Feltcraft,7.24,2,15492,United Kingdom +581492,12/9/2019,22153,Angel Decoration Stars On Dress,7.24,2,15492,United Kingdom +581492,12/9/2019,22154,Angel Decoration 3 Buttons,7.24,3,15492,United Kingdom +581492,12/9/2019,22155,Star Decoration Rustic,7.24,7,15492,United Kingdom +581492,12/9/2019,22156,Heart Decoration With Pearls,7.24,4,15492,United Kingdom +581492,12/9/2019,22163,Heart String Memo Holder Hanging,7.24,9,15492,United Kingdom +581492,12/9/2019,22165,Diamante Heart Shaped Wall Mirror,7.24,1,15492,United Kingdo +581492,12/9/2019,22167,Oval Wall Mirror Diamante,7.24,1,15492,United Kingdom +581492,12/9/2019,22170,Picture Frame Wood Triple Portrait,7.24,1,15492,United Kingd +581492,12/9/2019,22173,Metal 4 Hook Hanger French Chateau,7.24,2,15492,United Kingd +581492,12/9/2019,22174,Photo Cube,6.19,7,15492,United Kingdom +581492,12/9/2019,22178,Victorian Glass Hanging T-Light,6.19,5,15492,United Kingdom +581492,12/9/2019,22186,Red Star Card Holder,7.24,2,15492,United Kingdom +581492,12/9/2019,22190,Local Cafe Mug,7.24,3,15492,United Kingdom +581492,12/9/2019,22193,Red Diner Wall Clock,7.24,1,15492,United Kingdom +581492,12/9/2019,22195,Large Heart Measuring Spoons,7.24,1,15492,United Kingdom +581492,12/9/2019,22196,Small Heart Measuring Spoons,7.24,11,15492,United Kingdom +581492,12/9/2019,22197,Popcorn Holder,7.24,34,15492,United Kingdom +581492,12/9/2019,22199,Frying Pan Red Retrospot,7.24,1,15492,United Kingdom +581492,12/9/2019,22209,Wood Stamp Set Happy Birthday,7.24,1,15492,United Kingdom +581492,12/9/2019,22212,Four Hook White Lovebirds,7.24,1,15492,United Kingdom +581492,12/9/2019,22219,Lovebird Hanging Decoration White,7.24,4,15492,United Kingdo +581492,12/9/2019,22224,White Lovebird Lantern,7.24,4,15492,United Kingdom +581492,12/9/2019,22227,Hanging Heart Mirror Decoration,7.24,1,15492,United Kingdom +581492,12/9/2019,22260,Felt Egg Cosy Blue Rabbit,6.39,2,15492,United Kingdom +581492,12/9/2019,22261,Felt Egg Cosy White Rabbit,7.24,1,15492,United Kingdom +581492,12/9/2019,22264,Felt Farm Animal White Bunny,7.24,1,15492,United Kingdom +581492,12/9/2019,22273,Feltcraft Doll Molly,7.24,2,15492,United Kingdom +581492,12/9/2019,22277,Cosmetic Bag Vintage Rose Paisley,7.24,1,15492,United Kingdo +581492,12/9/2019,22279,Pocket Bag Blue Paisley Red Spot,7.24,5,15492,United Kingdom +581492,12/9/2019,22280,Pocket Bag Pink Paisely Brown Spot,7.24,4,15492,United Kingd +581492,12/9/2019,22300,Coffee Mug Dog + Ball Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22301,Coffee Mug Cat + Bird Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22307,Gold Mug Bone China Tree Of Life,7.24,1,15492,United Kingdom +581492,12/9/2019,22308,Tea Cosy Blue Stripe,7.24,2,15492,United Kingdom +581492,12/9/2019,22309,Tea Cosy Red Stripe,7.24,2,15492,United Kingdom +581492,12/9/2019,22324,Blue Polkadot Kids Bag,7.24,2,15492,United Kingdom +581492,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,7.24,3,15492,United Kingd +581492,12/9/2019,22327,Round Snack Boxes Set Of 4 Skulls,7.24,2,15492,United Kingdo +581492,12/9/2019,22329,Round Container Set Of 5 Retrospot,6.19,2,15492,United Kingd +581492,12/9/2019,22338,Star Decoration Painted Zinc,6.19,4,15492,United Kingdom +581492,12/9/2019,22340,Noel Garland Painted Zinc,6.19,17,15492,United Kingdom +581492,12/9/2019,22342,Home Garland Painted Zinc,6.19,7,15492,United Kingdom +581492,12/9/2019,22348,Tea Bag Plate Red Retrospot,6.19,3,15492,United Kingdom +581492,12/9/2019,22355,Charlotte Bag Suki Design,6.19,3,15492,United Kingdom +581492,12/9/2019,22356,Charlotte Bag Pink Polkadot,6.19,1,15492,United Kingdom +581492,12/9/2019,22357,Kings Choice Biscuit Tin,6.19,2,15492,United Kingdom +581492,12/9/2019,22358,Kings Choice Tea Caddy,6.19,1,15492,United Kingdom +581492,12/9/2019,22361,Glass Jar Daisy Fresh Cotton Wool,6.19,2,15492,United Kingdo +581492,12/9/2019,22362,Glass Jar Peacock Bath Salts,6.19,2,15492,United Kingdom +581492,12/9/2019,22367,Childrens Apron Spaceboy Design,6.19,1,15492,United Kingdom +581492,12/9/2019,22372,Airline Bag Vintage World Champion,6.19,1,15492,United Kingd +581492,12/9/2019,22374,Airline Bag Vintage Jet Set Red,6.19,1,15492,United Kingdom +581492,12/9/2019,85014B,Red Retrospot Umbrella,6.19,1,15492,United Kingdom +581492,12/9/2019,85032C,Curious Images Gift Wrap Set,6.19,1,15492,United Kingdom +581492,12/9/2019,85038,6 Chocolate Love Heart T-Lights,6.19,1,15492,United Kingdom +581492,12/9/2019,85039A,Set/4 Red Mini Rose Candle In Bowl,6.19,5,15492,United King +581492,12/9/2019,85039B,S/4 Ivory Mini Rose Candle In Bowl,6.19,4,15492,United King +581492,12/9/2019,85040A,S/4 Pink Flower Candles In Bowl,6.19,1,15492,United Kingdom +581492,12/9/2019,85048,15cm Christmas Glass Ball 20 Lights,6.19,2,15492,United King +581492,12/9/2019,85049C,Romantic Pinks Ribbons,6.19,1,15492,United Kingdom +581492,12/9/2019,85049G,Chocolate Box Ribbons,6.19,1,15492,United Kingdom +581492,12/9/2019,85049H,Urban Black Ribbons,6.19,2,15492,United Kingdom +581492,12/9/2019,85053,French Enamel Candleholder,6.19,1,15492,United Kingdom +581492,12/9/2019,85059,French Enamel Water Basin,6.19,1,15492,United Kingdom +581492,12/9/2019,85066,Cream Sweetheart Mini Chest,6.19,1,15492,United Kingdom +581492,12/9/2019,85094,Candy Spot Egg Warmer Rabbit,6.19,2,15492,United Kingdom +581492,12/9/2019,85114C,Red Enchanted Forest Placemat,6.19,1,15492,United Kingdom +581492,12/9/2019,85123A,Cream Hanging Heart T-Light Holder,6.19,3,15492,United King +581492,12/9/2019,85131B,Beaded Crystal Heart Green On Stick,6.19,1,15492,United Kin +581492,12/9/2019,85131D,Beaded Crystal Heart Pink On Stick,6.19,2,15492,United King +581492,12/9/2019,85152,Hand Over The Chocolate Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,85168B,Black Baroque Carriage Clock,6.19,3,15492,United Kingdom +581492,12/9/2019,85169C,Eau De Nil Love Bird Candle,6.19,1,15492,United Kingdom +581492,12/9/2019,85170C,Set/6 Eau De Nil Bird T-Lights,6.19,1,15492,United Kingdom +581492,12/9/2019,85173,Set/6 Frog Prince T-Light Candles,6.19,1,15492,United Kingdo +581492,12/9/2019,85177,Basket Of Flowers Sewing Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,85178,Victorian Sewing Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,85179A,Green Bitty Light Chain,6.19,4,15492,United Kingdom +581492,12/9/2019,85199S,Small Hanging Ivory/Red Wood Bird,6.19,9,15492,United Kingd +581492,12/9/2019,85227,Set Of 6 3d Kit Cards For Kids,6.19,3,15492,United Kingdom +581492,12/9/2019,90003C,Midnight Blue Pair Heart Hair Slide,6.19,1,15492,United Kin +581492,12/9/2019,90003E,Green Pair Heart Hair Slides,6.19,2,15492,United Kingdom +581492,12/9/2019,90010A,Midnight Blue Glass/Silver Bracelet,6.04,1,15492,United Kin +581492,12/9/2019,90013A,Midnight Blue Vintage Earrings,6.19,3,15492,United Kingdom +581492,12/9/2019,90013C,Green Vintage Earrings,6.19,2,15492,United Kingdom +581492,12/9/2019,90014A,Silver Mop Orbit Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90016B,Gold/Mop Pendant Orbit Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90018B,Gold Mop Orbit Drop Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90019B,Gold Mop Orbit Bracelet,6.19,1,15492,United Kingdom +581492,12/9/2019,90027D,Glass Bead Hoop Earrings Amethyst,6.19,1,15492,United Kingd +581492,12/9/2019,90030B,Red Kukui Coconut Seed Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90055,Cracked Glaze Earrings Brown,6.19,1,15492,United Kingdom +581492,12/9/2019,90059A,Diamante Hair Grip Pack/2 Crystal,6.19,1,15492,United Kingd +581492,12/9/2019,90059D,Diamante Hair Grip Pack/2 Peridot,6.19,2,15492,United Kingd +581492,12/9/2019,90059E,Diamante Hair Grip Pack/2 Ruby,6.04,1,15492,United Kingdom +581492,12/9/2019,90059F,Diamante Hair Grip Pack/2 Lt Rose,6.19,1,15492,United Kingd +581492,12/9/2019,90072,Ruby Drop Chandelier Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90086,Crystal Frog Phone Charm,6.19,1,15492,United Kingdom +581492,12/9/2019,90120B,Blue Murano Twist Bracelet,6.19,2,15492,United Kingdom +581492,12/9/2019,90120C,Green Murano Twist Bracelet,6.19,1,15492,United Kingdom +581492,12/9/2019,90130B,Turq Stone/Crystal Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90130C,Green Stone/Crystal Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90134,Old Rose Combo Bead Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90138,White/Pink Mini Crystals Necklace,6.19,1,15492,United Kingdo +581492,12/9/2019,90141B,Ivory Pendant Triple Shell Necklace,6.19,1,15492,United Kin +581492,12/9/2019,90145,Silver Hoop Earrings With Flower,6.19,1,15492,United Kingdom +581492,12/9/2019,90155,Resin Necklace W Pastel Beads,6.19,1,15492,United Kingdom +581492,12/9/2019,90161D,Ant Copper Pink Boudicca Bracelet,6.19,1,15492,United Kingd +581492,12/9/2019,90163A,Pink Rosebud & Pearl Necklace,6.19,2,15492,United Kingdom +581492,12/9/2019,90165B,White Rosebud Pearl Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90168,2 Daisies Hair Comb,6.19,1,15492,United Kingdom +581492,12/9/2019,90169,Daisy Hair Comb,6.19,1,15492,United Kingdom +581492,12/9/2019,90170,Daisy Hair Band,6.19,1,15492,United Kingdom +581492,12/9/2019,90174,Butterfly Hair Band,6.19,2,15492,United Kingdom +581492,12/9/2019,90175A,White Glass Chunky Charm Bracelet,6.19,2,15492,United Kingd +581492,12/9/2019,90175D,Tigris Eye Chunky Charm Bracelet,6.19,1,15492,United Kingdo +581492,12/9/2019,90177A,Classic Diamante Earrings Jet,6.19,1,15492,United Kingdom +581492,12/9/2019,90177C,Drop Diamante Earrings Crystal,6.19,1,15492,United Kingdom +581492,12/9/2019,90181B,Amethyst Glass/Shell/Pearl Necklace,6.04,1,15492,United Kin +581492,12/9/2019,90182C,Black 3 Bead Drop Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90183A,Amber Drop Earrings W Long Beads,6.19,2,15492,United Kingdo +581492,12/9/2019,90183C,Black Drop Earrings W Long Beads,6.19,1,15492,United Kingdo +581492,12/9/2019,90184C,Black Chunky Bead Bracelet W Strap,6.19,1,15492,United King +581492,12/9/2019,90185B,Amethyst Diamante Expandable Ring,6.19,1,15492,United Kingd +581492,12/9/2019,90188,Drop Earrings W Flower & Leaf,6.19,2,15492,United Kingdom +581492,12/9/2019,90191,Silver Lariat 40cm,6.19,2,15492,United Kingdom +581492,12/9/2019,90192,Jade Drop Earrings W Filigree,6.19,1,15492,United Kingdom +581492,12/9/2019,90198A,Vintage Rose Bead Bracelet Raspberr,6.19,2,15492,United Kin +581492,12/9/2019,90200A,Purple Sweetheart Bracelet,6.19,1,15492,United Kingdom +581492,12/9/2019,90201A,Purple Enamel Flower Ring,6.19,2,15492,United Kingdom +581492,12/9/2019,90201B,Black Enamel Flower Ring,6.19,1,15492,United Kingdom +581492,12/9/2019,90201C,Red Enamel Flower Ring,6.19,1,15492,United Kingdom +581492,12/9/2019,90201D,Green Enamel Flower Ring,6.19,1,15492,United Kingdom +581492,12/9/2019,90202C,Green Enamel Flower Hair Tie,6.19,1,15492,United Kingdom +581492,12/9/2019,90202D,Pink Enamel Flower Hair Tie,6.19,1,15492,United Kingdom +581492,12/9/2019,90206C,Crystal Diamante Star Brooch,6.19,1,15492,United Kingdom +581492,12/9/2019,90208,Pair Of Pink Flower Cluster Slide,6.19,1,15492,United Kingdo +581492,12/9/2019,90210A,Grey Acrylic Faceted Bangle,6.19,1,15492,United Kingdom +581492,12/9/2019,22378,Wall Tidy Retrospot,6.19,2,15492,United Kingdom +581492,12/9/2019,22379,Recycling Bag Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22380,Toy Tidy Spaceboy,6.19,2,15492,United Kingdom +581492,12/9/2019,22396,Magnets Pack Of 4 Retro Photo,6.19,1,15492,United Kingdom +581492,12/9/2019,22411,Jumbo Shopper Vintage Red Paisley,6.19,1,15492,United Kingdo +581492,12/9/2019,22418,10 Colour Spaceboy Pen,6.19,3,15492,United Kingdom +581492,12/9/2019,22419,Lipstick Pen Red,6.19,3,15492,United Kingdom +581492,12/9/2019,22420,Lipstick Pen Baby Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,22421,Lipstick Pen Fuschia,6.19,2,15492,United Kingdom +581492,12/9/2019,22422,Toothpaste Tube Pen,6.19,4,15492,United Kingdom +581492,12/9/2019,22437,Set Of 9 Black Skull Balloons,7.24,1,15492,United Kingdom +581492,12/9/2019,22441,Grow Your Own Basil In Enamel Mug,7.24,1,15492,United Kingdo +581492,12/9/2019,22446,Pin Cushion Babushka Pink,7.24,7,15492,United Kingdom +581492,12/9/2019,22457,Natural Slate Heart Chalkboard,7.24,1,15492,United Kingdom +581492,12/9/2019,22464,Hanging Metal Heart Lantern,7.24,1,15492,United Kingdom +581492,12/9/2019,22466,Fairy Tale Cottage Night Light,7.24,1,15492,United Kingdom +581492,12/9/2019,22467,Gumball Coat Rack,7.24,1,15492,United Kingdom +581492,12/9/2019,22478,Birdhouse Garden Marker,7.24,1,15492,United Kingdom +581492,12/9/2019,22486,Plasmatronic Lamp,7.24,1,15492,United Kingdom +581492,12/9/2019,22488,Natural Slate Rectangle Chalkboard,7.24,1,15492,United Kingd +581492,12/9/2019,22489,Pack Of 12 Traditional Crayons,7.24,5,15492,United Kingdom +581492,12/9/2019,22495,Set Of 2 Round Tins Camembert,7.24,1,15492,United Kingdom +581492,12/9/2019,22540,Mini Jigsaw Circus Parade,7.24,1,15492,United Kingdom +581492,12/9/2019,22544,Mini Jigsaw Spaceboy,7.24,1,15492,United Kingdom +581492,12/9/2019,22549,Picture Dominoes,7.24,3,15492,United Kingdom +581492,12/9/2019,22551,Plasters In Tin Spaceboy,7.24,2,15492,United Kingdom +581492,12/9/2019,22553,Plasters In Tin Skulls,7.24,2,15492,United Kingdom +581492,12/9/2019,22554,Plasters In Tin Woodland Animals,7.24,3,15492,United Kingdom +581492,12/9/2019,22555,Plasters In Tin Strongman,7.24,3,15492,United Kingdom +581492,12/9/2019,22557,Plasters In Tin Vintage Paisley,7.24,2,15492,United Kingdom +581492,12/9/2019,22558,Clothes Pegs Retrospot Pack 24,7.24,3,15492,United Kingdom +581492,12/9/2019,22560,Traditional Modelling Clay,7.24,5,15492,United Kingdom +581492,12/9/2019,22565,Feltcraft Hairbands Pink And White,7.24,4,15492,United Kingd +581492,12/9/2019,22566,Feltcraft Hairband Pink And Purple,7.24,3,15492,United Kingd +581492,12/9/2019,22571,Rocking Horse Red Christmas,7.24,4,15492,United Kingdom +581492,12/9/2019,22573,Star Wooden Christmas Decoration,6.39,2,15492,United Kingdom +581492,12/9/2019,22574,Heart Wooden Christmas Decoration,7.24,5,15492,United Kingdo +581492,12/9/2019,22576,Swallow Wooden Christmas Decoration,7.24,3,15492,United King +581492,12/9/2019,22577,Wooden Heart Christmas Scandinavian,7.24,4,15492,United King +581492,12/9/2019,22578,Wooden Star Christmas Scandinavian,7.24,19,15492,United King +581492,12/9/2019,22580,Advent Calendar Gingham Sack,7.24,9,15492,United Kingdom +581492,12/9/2019,22581,Wood Stocking Christmas Scandispot,7.24,11,15492,United King +581492,12/9/2019,22585,Pack Of 6 Birdy Gift Tags,7.24,2,15492,United Kingdom +581492,12/9/2019,22586,Feltcraft Hairband Pink And Blue,7.24,2,15492,United Kingdom +581492,12/9/2019,22589,Cardholder Gingham Star,7.24,6,15492,United Kingdom +581492,12/9/2019,22591,Cardholder Gingham Christmas Tree,7.24,1,15492,United Kingdo +581492,12/9/2019,22592,Cardholder Holly Wreath Metal,7.24,3,15492,United Kingdom +581492,12/9/2019,22593,Christmas Gingham Star,6.39,2,15492,United Kingdom +581492,12/9/2019,22594,Christmas Gingham Tree,7.24,3,15492,United Kingdom +581492,12/9/2019,22597,Musical Zinc Heart Decoration,7.24,9,15492,United Kingdom +581492,12/9/2019,22598,Christmas Musical Zinc Tree,7.24,4,15492,United Kingdom +581492,12/9/2019,22599,Christmas Musical Zinc Star,6.19,5,15492,United Kingdom +581492,12/9/2019,22600,Christmas Retrospot Star Wood,6.19,2,15492,United Kingdom +581492,12/9/2019,22601,Christmas Retrospot Angel Wood,6.19,3,15492,United Kingdom +581492,12/9/2019,22602,Retrospot Wooden Heart Decoration,6.19,3,15492,United Kingdo +581492,12/9/2019,22608,Pens Assorted Funky Jeweled,6.19,3,15492,United Kingdom +581492,12/9/2019,22614,Pack Of 12 Spaceboy Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,22615,Pack Of 12 Circus Parade Tissues,6.19,4,15492,United Kingdom +581492,12/9/2019,22616,Pack Of 12 London Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,22619,Set Of 6 Soldier Skittles,6.19,4,15492,United Kingdom +581492,12/9/2019,22620,4 Traditional Spinning Tops,6.19,3,15492,United Kingdom +581492,12/9/2019,22621,Traditional Knitting Nancy,6.19,2,15492,United Kingdom +581492,12/9/2019,22629,Spaceboy Lunch Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22630,Dolly Girl Lunch Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22632,Hand Warmer Red Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22633,Hand Warmer Union Jack,6.19,1,15492,United Kingdom +581492,12/9/2019,22634,Childs Breakfast Set Spaceboy,6.19,1,15492,United Kingdom +581492,12/9/2019,22650,Ceramic Pirate Chest Money Bank,6.19,2,15492,United Kingdom +581492,12/9/2019,22651,Gentleman Shirt Repair Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,22653,Button Box,6.19,16,15492,United Kingdom +581492,12/9/2019,22654,Deluxe Sewing Kit,6.19,2,15492,United Kingdom +581492,12/9/2019,22659,Lunch Box I Love London,6.19,1,15492,United Kingdom +581492,12/9/2019,22666,Recipe Box Pantry Yellow Design,6.19,5,15492,United Kingdom +581492,12/9/2019,22693,Grow A Flytrap Or Sunflower In Tin,6.19,3,15492,United Kingd +581492,12/9/2019,22694,Wicker Star,6.19,1,15492,United Kingdom +581492,12/9/2019,22697,Green Regency Teacup And Saucer,6.19,1,15492,United Kingdom +581492,12/9/2019,22699,Roses Regency Teacup And Saucer,6.19,2,15492,United Kingdom +581492,12/9/2019,22701,Pink Dog Bowl,6.19,5,15492,United Kingdom +581492,12/9/2019,22703,Pink Cat Bowl,6.19,6,15492,United Kingdom +581492,12/9/2019,22712,Card Dolly Girl,6.19,1,15492,United Kingdom +581492,12/9/2019,22713,Card I Love London,6.19,1,15492,United Kingdom +581492,12/9/2019,22714,Card Birthday Cowboy,6.19,2,15492,United Kingdom +581492,12/9/2019,22716,Card Circus Parade,6.19,3,15492,United Kingdom +581492,12/9/2019,22717,Card Dog And Ball,6.19,1,15492,United Kingdom +581492,12/9/2019,22720,Set Of 3 Cake Tins Pantry Design,6.19,2,15492,United Kingdom +581492,12/9/2019,22725,Alarm Clock Bakelike Chocolate,6.19,1,15492,United Kingdom +581492,12/9/2019,22726,Alarm Clock Bakelike Green,6.19,4,15492,United Kingdom +581492,12/9/2019,22727,Alarm Clock Bakelike Red,6.19,4,15492,United Kingdom +581492,12/9/2019,22730,Alarm Clock Bakelike Ivory,6.19,1,15492,United Kingdom +581492,12/9/2019,22741,Funky Diva Pen,6.19,4,15492,United Kingdom +581492,12/9/2019,22745,Poppy's Playhouse Bedroom,6.19,2,15492,United Kingdom +581492,12/9/2019,22748,Poppy's Playhouse Kitchen,6.19,1,15492,United Kingdom +581492,12/9/2019,22749,Feltcraft Princess Charlotte Doll,6.19,1,15492,United Kingdo +581492,12/9/2019,22751,Feltcraft Princess Olivia Doll,6.19,2,15492,United Kingdom +581492,12/9/2019,22753,Small Yellow Babushka Notebook,6.19,1,15492,United Kingdom +581492,12/9/2019,22754,Small Red Babushka Notebook,6.19,1,15492,United Kingdom +581492,12/9/2019,22757,Large Red Babushka Notebook,6.19,3,15492,United Kingdom +581492,12/9/2019,22758,Large Purple Babushka Notebook,6.19,2,15492,United Kingdom +581492,12/9/2019,22763,Key Cabinet Ma Campagne,6.19,1,15492,United Kingdom +581492,12/9/2019,22768,Family Photo Frame Cornice,6.19,2,15492,United Kingdom +581492,12/9/2019,22776,Sweetheart 3 Tier Cake Stand,6.19,1,15492,United Kingdom +581492,12/9/2019,22795,Sweetheart Recipe Book Stand,6.19,1,15492,United Kingdom +581492,12/9/2019,22798,Antique Glass Dressing Table Pot,6.19,3,15492,United Kingdom +581492,12/9/2019,22800,Antique Tall Swirlglass Trinket Pot,6.19,3,15492,United King +581492,12/9/2019,22809,Set Of 6 T-Lights Santa,6.19,1,15492,United Kingdom +581492,12/9/2019,22811,Set Of 6 T-Lights Cacti,6.19,1,15492,United Kingdom +581492,12/9/2019,22814,Card Party Games,6.19,9,15492,United Kingdom +581492,12/9/2019,22815,Card Psychedelic Apples,6.19,18,15492,United Kingdom +581492,12/9/2019,22816,Card Motorbike Santa,6.19,2,15492,United Kingdom +581492,12/9/2019,22817,Card Suki Birthday,6.19,5,15492,United Kingdom +581492,12/9/2019,22818,Card Christmas Village,7.24,6,15492,United Kingdom +581492,12/9/2019,22819,Birthday Card Retro Spot,7.24,3,15492,United Kingdom +581492,12/9/2019,22835,Hot Water Bottle I Am So Poorly,7.24,3,15492,United Kingdom +581492,12/9/2019,22865,Hand Warmer Owl Design,7.24,2,15492,United Kingdom +581492,12/9/2019,22866,Hand Warmer Scotty Dog Design,7.24,3,15492,United Kingdom +581492,12/9/2019,22867,Hand Warmer Bird Design,7.24,4,15492,United Kingdom +581492,12/9/2019,22881,Number Tile Vintage Font 2,7.24,1,15492,United Kingdom +581492,12/9/2019,22885,Number Tile Vintage Font 6,7.24,1,15492,United Kingdom +581492,12/9/2019,22888,Number Tile Vintage Font 9,7.24,1,15492,United Kingdom +581492,12/9/2019,22890,Novelty Biscuits Cake Stand 3 Tier,7.24,1,15492,United Kingd +581492,12/9/2019,22898,Childrens Apron Apples Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22899,Children's Apron Dolly Girl,7.24,2,15492,United Kingdom +581492,12/9/2019,22900,Set 2 Tea Towels I Love London,7.24,3,15492,United Kingdom +581492,12/9/2019,22905,Calendar In Season Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22907,Pack Of 20 Napkins Pantry Design,6.39,1,15492,United Kingdom +581492,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,7.24,5,15492,United King +581492,12/9/2019,22910,Paper Chain Kit Vintage Christmas,7.24,7,15492,United Kingdo +581492,12/9/2019,22914,Blue Coat Rack Paris Fashion,7.24,1,15492,United Kingdom +581492,12/9/2019,22922,Fridge Magnets Us Diner Assorted,7.24,6,15492,United Kingdom +581492,12/9/2019,22924,Fridge Magnets La Vie En Rose,7.24,6,15492,United Kingdom +581492,12/9/2019,22928,Yellow Giant Garden Thermometer,7.24,1,15492,United Kingdom +581492,12/9/2019,22940,Feltcraft Christmas Fairy,6.19,1,15492,United Kingdom +581492,12/9/2019,22941,Christmas Lights 10 Reindeer,6.19,2,15492,United Kingdom +581492,12/9/2019,22943,Christmas Lights 10 Vintage Baubles,6.19,1,15492,United King +581492,12/9/2019,22948,Metal Decoration Naughty Children,6.19,4,15492,United Kingdo +581492,12/9/2019,22951,60 Cake Cases Dolly Girl Design,6.19,2,15492,United Kingdom +581492,12/9/2019,22952,60 Cake Cases Vintage Christmas,6.19,2,15492,United Kingdom +581492,12/9/2019,22955,36 Foil Star Cake Cases,6.19,1,15492,United Kingdom +581492,12/9/2019,22960,Jam Making Set With Jars,6.19,2,15492,United Kingdom +581492,12/9/2019,22961,Jam Making Set Printed,6.19,9,15492,United Kingdom +581492,12/9/2019,22962,Jam Jar With Pink Lid,6.19,3,15492,United Kingdom +581492,12/9/2019,22963,Jam Jar With Green Lid,6.19,3,15492,United Kingdom +581492,12/9/2019,22964,3 Piece Spaceboy Cookie Cutter Set,6.19,1,15492,United Kingd +581492,12/9/2019,22965,3 Traditional Biscuit Cutters Set,6.19,1,15492,United Kingdo +581492,12/9/2019,22966,Gingerbread Man Cookie Cutter,6.19,4,15492,United Kingdom +581492,12/9/2019,22969,Homemade Jam Scented Candles,6.19,6,15492,United Kingdom +581492,12/9/2019,22975,Spaceboy Childrens Egg Cup,6.19,2,15492,United Kingdom +581492,12/9/2019,22976,Circus Parade Childrens Egg Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,22977,Dolly Girl Childrens Egg Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,22979,Pantry Washing Up Brush,6.19,2,15492,United Kingdom +581492,12/9/2019,22980,Pantry Scrubbing Brush,6.19,1,15492,United Kingdom +581492,12/9/2019,22982,Pantry Pastry Brush,6.19,3,15492,United Kingdom +581492,12/9/2019,22983,Card Billboard Font,6.19,1,15492,United Kingdom +581492,12/9/2019,22984,Card Gingham Rose,6.19,1,15492,United Kingdom +581492,12/9/2019,22988,Soldiers Egg Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,22989,Set 2 Pantry Design Tea Towels,6.19,2,15492,United Kingdom +581492,12/9/2019,22992,Revolver Wooden Ruler,6.19,3,15492,United Kingdom +581492,12/9/2019,22993,Set Of 4 Pantry Jelly Moulds,6.19,3,15492,United Kingdom +581492,12/9/2019,22995,Travel Card Wallet Suki,6.19,4,15492,United Kingdom +581492,12/9/2019,22996,Travel Card Wallet Vintage Ticket,6.19,5,15492,United Kingdo +581492,12/9/2019,22997,Travel Card Wallet Union Jack,6.19,1,15492,United Kingdom +581492,12/9/2019,22998,Travel Card Wallet Keep Calm,6.19,4,15492,United Kingdom +581492,12/9/2019,22999,Travel Card Wallet Vintage Leaf,6.19,1,15492,United Kingdom +581492,12/9/2019,23005,Travel Card Wallet I Love London,6.19,4,15492,United Kingdom +581492,12/9/2019,23007,Spaceboy Baby Gift Set,6.19,1,15492,United Kingdom +581492,12/9/2019,23009,I Love London Baby Gift Set,6.19,2,15492,United Kingdom +581492,12/9/2019,23012,Glass Apothecary Bottle Perfume,6.19,1,15492,United Kingdom +581492,12/9/2019,23013,Glass Apothecary Bottle Tonic,6.19,2,15492,United Kingdom +581492,12/9/2019,23014,Glass Apothecary Bottle Elixir,6.19,1,15492,United Kingdom +581492,12/9/2019,23050,Recycled Acapulco Mat Green,6.19,1,15492,United Kingdom +581492,12/9/2019,23051,Recycled Acapulco Mat Blue,6.19,1,15492,United Kingdom +581492,12/9/2019,23052,Recycled Acapulco Mat Turquoise,6.19,5,15492,United Kingdom +581492,12/9/2019,23053,Recycled Acapulco Mat Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23054,Recycled Acapulco Mat Lavender,6.19,1,15492,United Kingdom +581492,12/9/2019,23074,Embossed Heart Trinket Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23076,Ice Cream Sundae Lip Gloss,6.19,3,15492,United Kingdom +581492,12/9/2019,23077,Doughnut Lip Gloss,6.19,3,15492,United Kingdom +581492,12/9/2019,23082,Set 6 Paper Table Lantern Hearts,6.19,1,15492,United Kingdom +581492,12/9/2019,23083,Set 6 Paper Table Lantern Stars,6.19,1,15492,United Kingdom +581492,12/9/2019,23084,Rabbit Night Light,6.19,57,15492,United Kingdom +581492,12/9/2019,23088,Zinc Heart Flower T-Light Holder,6.19,1,15492,United Kingdom +581492,12/9/2019,23089,Glass Bon Bon Jar,6.19,4,15492,United Kingdom +581492,12/9/2019,23093,Small Parisienne Heart Photo Frame,6.19,3,15492,United Kingd +581492,12/9/2019,23103,Jingle Bell Heart Decoration,6.19,2,15492,United Kingdom +581492,12/9/2019,23110,Parisienne Key Cabinet,6.19,1,15492,United Kingdom +581492,12/9/2019,23111,Parisienne Sewing Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23112,Parisienne Curio Cabinet,6.19,1,15492,United Kingdom +581492,12/9/2019,23118,Parisienne Jewellery Drawer,6.19,1,15492,United Kingdom +581492,12/9/2019,23158,Set Of 5 Lucky Cat Magnets,6.19,1,15492,United Kingdom +581492,12/9/2019,23165,Large Ceramic Top Storage Jar,6.19,2,15492,United Kingdom +581492,12/9/2019,23166,Medium Ceramic Top Storage Jar,6.19,2,15492,United Kingdom +581492,12/9/2019,23168,Classic Cafe Sugar Dispenser,6.19,2,15492,United Kingdom +581492,12/9/2019,23171,Regency Tea Plate Green,6.19,1,15492,United Kingdom +581492,12/9/2019,23172,Regency Tea Plate Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23173,Regency Teapot Roses,6.19,1,15492,United Kingdom +581492,12/9/2019,23175,Regency Milk Jug Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23177,Treasure Island Book Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23183,Mother's Kitchen Spoon Rest,6.19,3,15492,United Kingdom +581492,12/9/2019,23184,Bull Dog Bottle Opener,6.19,2,15492,United Kingdom +581492,12/9/2019,23188,Vintage 2 Metre Folding Ruler,6.19,1,15492,United Kingdom +581492,12/9/2019,23191,Bundle Of 3 Retro Note Books,6.19,1,15492,United Kingdom +581492,12/9/2019,23194,Gymkhana Treasure Book Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23196,Vintage Leaf Magnetic Notepad,6.19,1,15492,United Kingdom +581492,12/9/2019,23197,Sketchbook Magnetic Shopping List,6.19,2,15492,United Kingdo +581492,12/9/2019,23198,Pantry Magnetic Shopping List,6.19,2,15492,United Kingdom +581492,12/9/2019,23204,Charlotte Bag Apples Design,6.19,6,15492,United Kingdom +581492,12/9/2019,23205,Charlotte Bag Vintage Alphabet,6.19,3,15492,United Kingdom +581492,12/9/2019,23212,Heart Wreath Decoration With Bell,6.19,7,15492,United Kingdo +581492,12/9/2019,23213,Star Wreath Decoration With Bell,6.19,7,15492,United Kingdom +581492,12/9/2019,23220,Reindeer Heart Decoration Gold,6.19,2,15492,United Kingdom +581492,12/9/2019,23224,Cherub Heart Decoration Gold,6.19,1,15492,United Kingdom +581492,12/9/2019,23229,Vintage Donkey Tail Game,6.19,2,15492,United Kingdom +581492,12/9/2019,23234,Biscuit Tin Vintage Christmas,6.19,4,15492,United Kingdom +581492,12/9/2019,23241,Treasure Tin Gymkhana Design,6.19,3,15492,United Kingdom +581492,12/9/2019,23243,Set Of Tea Coffee Sugar Tins Pantry,6.19,1,15492,United King +581492,12/9/2019,23245,Set Of 3 Regency Cake Tins,6.19,7,15492,United Kingdom +581492,12/9/2019,23247,Biscuit Tin 50'S Christmas,6.19,3,15492,United Kingdom +581492,12/9/2019,23264,Set Of 3 Wooden Sleigh Decorations,6.19,3,15492,United Kingd +581492,12/9/2019,23265,Set Of 3 Wooden Tree Decorations,6.19,3,15492,United Kingdom +581492,12/9/2019,23266,Set Of 3 Wooden Stocking Decoration,6.19,2,15492,United King +581492,12/9/2019,23274,Star T-Light Holder Willie Winkie,6.19,3,15492,United Kingdo +581492,12/9/2019,23275,Set Of 3 Hanging Owls Ollie Beak,6.19,2,15492,United Kingdom +581492,12/9/2019,23280,Folding Butterfly Mirror Hot Pink,6.19,2,15492,United Kingdo +581492,12/9/2019,23284,Doormat Keep Calm And Come In,6.19,1,15492,United Kingdom +581492,12/9/2019,23285,Pink Vintage Spot Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23287,Red Vintage Spot Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23290,Spaceboy Childrens Bowl,6.19,1,15492,United Kingdom +581492,12/9/2019,23292,Spaceboy Childrens Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,23293,Set Of 12 Fairy Cake Baking Cases,6.19,3,15492,United Kingdo +581492,12/9/2019,23294,Set Of 6 Snack Loaf Baking Cases,6.19,1,15492,United Kingdom +581492,12/9/2019,23295,Set Of 12 Mini Loaf Baking Cases,6.19,4,15492,United Kingdom +581492,12/9/2019,23298,Spotty Bunting,6.19,2,15492,United Kingdom +581492,12/9/2019,23299,Food Cover With Beads Set 2,6.19,1,15492,United Kingdom +581492,12/9/2019,23300,Gardeners Kneeling Pad Cup Of Tea,6.19,3,15492,United Kingdo +581492,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,6,15492,United Kingdom +581492,12/9/2019,23307,Set Of 60 Pantry Design Cake Cases,6.19,7,15492,United Kingd +581492,12/9/2019,23308,Set Of 60 Vintage Leaf Cake Cases,6.19,1,15492,United Kingdo +581492,12/9/2019,23309,Set Of 60 I Love London Cake Cases,6.19,2,15492,United Kingd +581492,12/9/2019,23310,Bubblegum Ring Assorted,6.19,4,15492,United Kingdom +581492,12/9/2019,23311,Vintage Christmas Stocking,6.19,1,15492,United Kingdom +581492,12/9/2019,23312,Vintage Christmas Gift Sack,6.19,6,15492,United Kingdom +581492,12/9/2019,23313,Vintage Christmas Bunting,6.19,4,15492,United Kingdom +581492,12/9/2019,23314,Vintage Christmas Tablecloth,6.19,1,15492,United Kingdom +581492,12/9/2019,23319,Box Of 6 Mini 50'S Crackers,6.19,3,15492,United Kingdom +581492,12/9/2019,23322,Large White Heart Of Wicker,6.19,1,15492,United Kingdom +581492,12/9/2019,23323,White Wicker Star,6.19,6,15492,United Kingdom +581492,12/9/2019,23328,Set 6 School Milk Bottles In Crate,6.19,5,15492,United Kingd +581492,12/9/2019,23332,Ivory Wicker Heart Large,6.19,1,15492,United Kingdom +581492,12/9/2019,23334,Ivory Wicker Heart Small,6.19,1,15492,United Kingdom +581492,12/9/2019,23336,Egg Frying Pan Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23340,Vintage Christmas Cake Frill,6.19,3,15492,United Kingdom +581492,12/9/2019,23345,Dolly Girl Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23347,I Love London Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,2,15492,United Kingdom +581492,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,5,15492,United Kingdom +581492,12/9/2019,23352,Roll Wrap 50'S Red Christmas,6.19,1,15492,United Kingdom +581492,12/9/2019,23353,6 Gift Tags Vintage Christmas,6.19,3,15492,United Kingdom +581492,12/9/2019,23354,6 Gift Tags 50'S Christmas,6.19,4,15492,United Kingdom +581492,12/9/2019,23357,Hot Water Bottle Sex Bomb,6.19,1,15492,United Kingdom +581492,12/9/2019,23358,Hot Stuff Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,23365,Set 12 Colour Pencils Love London,6.19,1,15492,United Kingdo +581492,12/9/2019,23366,Set 12 Colouring Pencils Doily,6.04,5,15492,United Kingdom +581492,12/9/2019,23367,Set 12 Colour Pencils Spaceboy,6.04,10,15492,United Kingdom +581492,12/9/2019,23368,Set 12 Colour Pencils Dolly Girl,6.19,2,15492,United Kingdom +581492,12/9/2019,23581,Jumbo Bag Paisley Park,6.19,3,15492,United Kingdom +581492,12/9/2019,21928,Jumbo Bag Scandinavian Blue Paisley,6.19,2,15492,United King +581492,12/9/2019,21929,Jumbo Bag Pink Vintage Paisley,6.19,1,15492,United Kingdom +581492,12/9/2019,85099C,Jumbo Bag Baroque Black White,6.19,7,15492,United Kingdom +581492,12/9/2019,23199,Jumbo Bag Apples,6.19,2,15492,United Kingdom +581492,12/9/2019,23200,Jumbo Bag Pears,6.19,1,15492,United Kingdom +581492,12/9/2019,23202,Jumbo Bag Vintage Leaf,6.19,2,15492,United Kingdom +581492,12/9/2019,23343,Jumbo Bag Vintage Christmas,6.19,4,15492,United Kingdom +581492,12/9/2019,23344,Jumbo Bag 50'S Christmas,6.19,5,15492,United Kingdom +581492,12/9/2019,23583,Lunch Bag Paisley Park,6.19,2,15492,United Kingdom +581492,12/9/2019,20727,Lunch Bag Black Skull,6.04,2,15492,United Kingdom +581492,12/9/2019,20728,Lunch Bag Cars Blue,6.04,1,15492,United Kingdom +581492,12/9/2019,20725,Lunch Bag Red Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,20726,Lunch Bag Woodland,6.19,1,15492,United Kingdom +581492,12/9/2019,22662,Lunch Bag Dolly Girl Design,6.19,1,15492,United Kingdom +581492,12/9/2019,23206,Lunch Bag Apple Design,6.19,3,15492,United Kingdom +581492,12/9/2019,23208,Lunch Bag Vintage Leaf Design,6.19,3,15492,United Kingdom +581492,12/9/2019,23375,50'S Christmas Paper Gift Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,23437,50'S Christmas Gift Bag Large,6.19,1,15492,United Kingdom +581492,12/9/2019,22821,Gift Bag Psychedelic Apples,7.24,3,15492,United Kingdom +581493,12/9/2019,79190B,Retro Plastic Polka Tray,7.24,15,12423,Belgium +581493,12/9/2019,79190A,Retro Plastic 70'S Tray,7.24,15,12423,Belgium +581493,12/9/2019,22915,Assorted Bottle Top Magnets,7.24,12,12423,Belgium +581493,12/9/2019,22151,Place Setting White Heart,7.24,24,12423,Belgium +581493,12/9/2019,22632,Hand Warmer Red Retrospot,7.24,12,12423,Belgium +581493,12/9/2019,22865,Hand Warmer Owl Design,7.24,12,12423,Belgium +581493,12/9/2019,20718,Red Retrospot Shopper Bag,7.24,10,12423,Belgium +581493,12/9/2019,79190B,Retro Plastic Polka Tray,7.24,12,12423,Belgium +581493,12/9/2019,71459,Hanging Jam Jar T-Light Holders,7.24,12,12423,Belgium +581493,12/9/2019,84945,Multi Colour Silver T-Light Holder,7.24,12,12423,Belgium +581493,12/9/2019,22356,Charlotte Bag Pink Polkadot,7.24,10,12423,Belgium +581493,12/9/2019,20724,Red Retrospot Charlotte Bag,7.24,10,12423,Belgium +581493,12/9/2019,23204,Charlotte Bag Apples Design,7.24,10,12423,Belgium +581493,12/9/2019,21108,Fairy Cake Flannel Assorted Colour,7.24,18,12423,Belgium +581493,12/9/2019,22252,Birdcage Decoration Tealight Holder,7.24,12,12423,Belgium +581493,12/9/2019,22807,Set Of 6 T-Lights Toadstools,6.19,6,12423,Belgium +581494,12/9/2019,23084,Rabbit Night Light,6.19,24,12518,Germany +581494,12/9/2019,21559,Strawberry Lunch Box With Cutlery,6.19,6,12518,Germany +581494,12/9/2019,21506,Fancy Font Birthday Card,6.19,12,12518,Germany +581494,12/9/2019,22716,Card Circus Parade,6.19,12,12518,Germany +581494,12/9/2019,22556,Plasters In Tin Circus Parade,6.19,12,12518,Germany +581494,12/9/2019,22328,Round Snack Boxes Set Of 4 Fruits,6.19,6,12518,Germany +581494,12/9/2019,22730,Alarm Clock Bakelike Ivory,6.19,4,12518,Germany +581494,12/9/2019,22633,Hand Warmer Union Jack,6.19,12,12518,Germany +581494,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,6.19,6,12518,Germany +581494,12/9/2019,10125,Mini Funky Design Tapes,6.19,20,12518,Germany +581494,12/9/2019,23367,Set 12 Colour Pencils Spaceboy,6.19,24,12518,Germany +581494,12/9/2019,22551,Plasters In Tin Spaceboy,6.04,12,12518,Germany +581494,12/9/2019,22554,Plasters In Tin Woodland Animals,6.04,12,12518,Germany +581494,12/9/2019,22549,Picture Dominoes,6.19,12,12518,Germany +581494,12/9/2019,23388,Woodland Mini Backpack,6.19,4,12518,Germany +581494,12/9/2019,23390,Dolly Girl Mini Backpack,6.19,4,12518,Germany +581494,12/9/2019,23389,Spaceboy Mini Backpack,6.19,4,12518,Germany +581495,12/9/2019,23535,Wall Art Bicycle Safety,6.19,12,14051,United Kingdom +581495,12/9/2019,22699,Roses Regency Teacup And Saucer,6.19,12,14051,United Kingdom +581495,12/9/2019,22698,Pink Regency Teacup And Saucer,6.19,12,14051,United Kingdom +581495,12/9/2019,22697,Green Regency Teacup And Saucer,6.19,12,14051,United Kingdom +581495,12/9/2019,22633,Hand Warmer Union Jack,6.04,18,14051,United Kingdom +581495,12/9/2019,15056N,Edwardian Parasol Natural,6.19,36,14051,United Kingdom +581495,12/9/2019,23439,Hand Warmer Red Love Heart,6.19,36,14051,United Kingdom +581495,12/9/2019,48138,Doormat Union Flag,6.19,10,14051,United Kingdom +581495,12/9/2019,21523,Doormat Fancy Font Home Sweet Home,6.19,10,14051,United King +581495,12/9/2019,21877,Home Sweet Home Mug,6.19,12,14051,United Kingdom +581495,12/9/2019,21871,Save The Planet Mug,6.19,18,14051,United Kingdom +581495,12/9/2019,22798,Antique Glass Dressing Table Pot,6.19,36,14051,United Kingdo +581495,12/9/2019,23173,Regency Teapot Roses,6.19,6,14051,United Kingdom +581495,12/9/2019,22423,Regency Cakestand 3 Tier,6.19,10,14051,United Kingdom +581496,12/9/2019,22664,Toy Tidy Dolly Girl Design,6.19,20,16558,United Kingdom +581496,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,6.19,12,16558,United Kingdom +581496,12/9/2019,72800E,4 Ivory Dinner Candles Silver Flock,6.19,12,16558,United Ki +581496,12/9/2019,23313,Vintage Christmas Bunting,6.19,10,16558,United Kingdom +581496,12/9/2019,22952,60 Cake Cases Vintage Christmas,6.19,24,16558,United Kingdom +581496,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,6.04,12,16558,United Kin +581496,12/9/2019,23298,Spotty Bunting,6.19,3,16558,United Kingdom +581496,12/9/2019,23598,Paper Bunting Vintage Party,6.19,6,16558,United Kingdom +581496,12/9/2019,16169E,Wrap 50'S Christmas,6.19,25,16558,United Kingdom +581496,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,12,16558,United Kingdom +581496,12/9/2019,23350,Roll Wrap Vintage Spot,6.19,24,16558,United Kingdom +581496,12/9/2019,22865,Hand Warmer Owl Design,6.19,24,16558,United Kingdom +581496,12/9/2019,22632,Hand Warmer Red Retrospot,6.19,12,16558,United Kingdom +581496,12/9/2019,22835,Hot Water Bottle I Am So Poorly,6.19,4,16558,United Kingdom +581496,12/9/2019,22112,Chocolate Hot Water Bottle,6.19,6,16558,United Kingdom +581496,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,4,16558,United Kingdom +581496,12/9/2019,22314,Office Mug Warmer Choc+Blue,6.19,24,16558,United Kingdom +581496,12/9/2019,22313,Office Mug Warmer Pink,6.19,12,16558,United Kingdom +581496,12/9/2019,23084,Rabbit Night Light,6.19,18,16558,United Kingdom +581496,12/9/2019,20831,Gold Photo Frame,6.19,12,16558,United Kingdom +581496,12/9/2019,21115,Rose Caravan Doorstop,6.19,8,16558,United Kingdom +581496,12/9/2019,21462,Nursery A B C Painted Letters,7.24,8,16558,United Kingdom +581496,12/9/2019,22076,6 Ribbons Empire,7.24,24,16558,United Kingdom +581496,12/9/2019,22190,Local Cafe Mug,7.24,24,16558,United Kingdom +581496,12/9/2019,22215,Cake Stand White Two Tier Lace,7.24,6,16558,United Kingdom +581496,12/9/2019,22220,Cake Stand Lovebird 2 Tier White,7.24,6,16558,United Kingdom +581496,12/9/2019,22464,Hanging Metal Heart Lantern,7.24,12,16558,United Kingdom +581496,12/9/2019,22465,Hanging Metal Star Lantern,7.24,12,16558,United Kingdom +581496,12/9/2019,22539,Mini Jigsaw Dolly Girl,7.24,48,16558,United Kingdom +581496,12/9/2019,22544,Mini Jigsaw Spaceboy,7.24,48,16558,United Kingdom +581496,12/9/2019,23438,Red Spot Gift Bag Large,6.19,12,16558,United Kingdom +581496,12/9/2019,23437,50'S Christmas Gift Bag Large,6.19,12,16558,United Kingdom +581497,12/9/2019,20719,Woodland Charlotte Bag,6.19,33,17497,United Kingdom +581497,12/9/2019,20723,Strawberry Charlotte Bag,6.19,42,17497,United Kingdom +581497,12/9/2019,20724,Red Retrospot Charlotte Bag,6.19,55,17497,United Kingdom +581497,12/9/2019,21212,Pack Of 72 Retrospot Cake Cases,7.24,7,17497,United Kingdom +581497,12/9/2019,21238,Red Retrospot Cup,7.24,8,17497,United Kingdom +581497,12/9/2019,21242,Red Retrospot Plate,7.24,2,17497,United Kingdom +581497,12/9/2019,21479,White Skull Hot Water Bottle,7.24,25,17497,United Kingdom +581497,12/9/2019,21481,Fawn Blue Hot Water Bottle,7.24,1,17497,United Kingdom +581497,12/9/2019,21481,Fawn Blue Hot Water Bottle,7.24,1,17497,United Kingdom +581497,12/9/2019,21484,Chick Grey Hot Water Bottle,7.24,1,17497,United Kingdom +581497,12/9/2019,21668,Red Stripe Ceramic Drawer Knob,7.24,1,17497,United Kingdom +581497,12/9/2019,21670,Blue Spot Ceramic Drawer Knob,7.24,1,17497,United Kingdom +581497,12/9/2019,21671,Red Spot Ceramic Drawer Knob,7.24,1,17497,United Kingdom +581497,12/9/2019,21672,White Spot Red Ceramic Drawer Knob,7.24,6,17497,United Kingd +581497,12/9/2019,21673,White Spot Blue Ceramic Drawer Knob,7.24,1,17497,United King +581497,12/9/2019,21714,Citronella Candle Garden Pot,7.24,2,17497,United Kingdom +581497,12/9/2019,22110,Bird House Hot Water Bottle,7.24,7,17497,United Kingdom +581497,12/9/2019,22111,Scottie Dog Hot Water Bottle,7.24,5,17497,United Kingdom +581497,12/9/2019,22112,Chocolate Hot Water Bottle,7.24,13,17497,United Kingdom +581497,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,7.24,48,17497,United Kingd +581497,12/9/2019,22197,Popcorn Holder,7.24,68,17497,United Kingdom +581497,12/9/2019,22355,Charlotte Bag Suki Design,7.24,110,17497,United Kingdom +581497,12/9/2019,22356,Charlotte Bag Pink Polkadot,7.24,25,17497,United Kingdom +581497,12/9/2019,22356,Charlotte Bag Pink Polkadot,7.24,1,17497,United Kingdom +581497,12/9/2019,22423,Regency Cakestand 3 Tier,7.24,8,17497,United Kingdom +581497,12/9/2019,22725,Alarm Clock Bakelike Chocolate,7.24,3,17497,United Kingdom +581497,12/9/2019,22726,Alarm Clock Bakelike Green,7.24,1,17497,United Kingdom +581497,12/9/2019,22727,Alarm Clock Bakelike Red,7.24,10,17497,United Kingdom +581497,12/9/2019,22730,Alarm Clock Bakelike Ivory,7.24,5,17497,United Kingdom +581497,12/9/2019,22735,Ribbon Reel Socks And Mittens,7.24,2,17497,United Kingdom +581497,12/9/2019,22736,Ribbon Reel Making Snowmen,7.24,3,17497,United Kingdom +581497,12/9/2019,22738,Ribbon Reel Snowy Village,7.24,1,17497,United Kingdom +581497,12/9/2019,22805,Blue Drawer Knob Acrylic Edwardian,7.24,1,17497,United Kingd +581497,12/9/2019,22835,Hot Water Bottle I Am So Poorly,7.24,4,17497,United Kingdom +581497,12/9/2019,22895,Set Of 2 Tea Towels Apple And Pears,7.24,1,17497,United King +581497,12/9/2019,22896,Peg Bag Apples Design,7.24,2,17497,United Kingdom +581497,12/9/2019,22898,Childrens Apron Apples Design,7.24,2,17497,United Kingdom +581497,12/9/2019,22943,Christmas Lights 10 Vintage Baubles,7.24,1,17497,United King +581497,12/9/2019,23084,Rabbit Night Light,6.19,37,17497,United Kingdom +581497,12/9/2019,23168,Classic Cafe Sugar Dispenser,6.19,1,17497,United Kingdom +581497,12/9/2019,23204,Charlotte Bag Apples Design,6.04,13,17497,United Kingdom +581497,12/9/2019,23205,Charlotte Bag Vintage Alphabet,6.04,8,17497,United Kingdom +581497,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,1,17497,United Kingdom +581497,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,36,17497,United Kingdom +581497,12/9/2019,23356,Love Hot Water Bottle,6.19,1,17497,United Kingdom +581497,12/9/2019,23357,Hot Water Bottle Sex Bomb,6.19,2,17497,United Kingdom +581497,12/9/2019,23358,Hot Stuff Hot Water Bottle,6.19,2,17497,United Kingdom +581497,12/9/2019,35970,Zinc Folkart Sleigh Bells,6.19,2,17497,United Kingdom +581497,12/9/2019,47566,Party Bunting,6.19,5,17497,United Kingdom +581497,12/9/2019,82583,Hot Baths Metal Sign,6.19,4,17497,United Kingdom +581497,12/9/2019,84029G,Knitted Union Flag Hot Water Bottle,6.19,11,17497,United Ki +581497,12/9/2019,84997C,Childrens Cutlery Polkadot Blue,6.19,3,17497,United Kingdom +581497,12/9/2019,85049G,Chocolate Box Ribbons,6.19,2,17497,United Kingdom +581497,12/9/2019,20727,Lunch Bag Black Skull,6.19,8,17497,United Kingdom +581497,12/9/2019,22383,Lunch Bag Suki Design,7.24,2,17497,United Kingdom +581497,12/9/2019,23206,Lunch Bag Apple Design,6.04,3,17497,United Kingdom +581497,12/9/2019,23206,Lunch Bag Apple Design,6.04,1,17497,United Kingdom +581497,12/9/2019,23207,Lunch Bag Alphabet Design,6.19,1,17497,United Kingdom +581497,12/9/2019,23208,Lunch Bag Vintage Leaf Design,6.19,3,17497,United Kingdom +581498,12/9/2019,20669,Red Heart Luggage Tag,6.19,3,14498,United Kingdom +581498,12/9/2019,20679,Edwardian Parasol Red,6.19,5,14498,United Kingdom +581498,12/9/2019,20717,Strawberry Shopper Bag,6.19,1,14498,United Kingdom +581498,12/9/2019,20718,Red Retrospot Shopper Bag,6.19,1,14498,United Kingdom +581498,12/9/2019,20750,Red Retrospot Mini Cases,6.19,1,14498,United Kingdom +581498,12/9/2019,20961,Strawberry Bath Sponge,6.19,1,14498,United Kingdom +581498,12/9/2019,20963,Apple Bath Sponge,6.19,1,14498,United Kingdom +581498,12/9/2019,21115,Rose Caravan Doorstop,6.19,1,14498,United Kingdom +581498,12/9/2019,21125,Set 6 Football Celebration Candles,6.19,3,14498,United Kingd +581498,12/9/2019,21137,Black Record Cover Frame,6.19,4,14498,United Kingdom +581498,12/9/2019,21155,Red Retrospot Peg Bag,6.19,4,14498,United Kingdom +581498,12/9/2019,21166,Cook With Wine Metal Sign,6.19,3,14498,United Kingdom +581498,12/9/2019,21169,You're Confusing Me Metal Sign,6.19,2,14498,United Kingdom +581498,12/9/2019,21174,Pottering In The Shed Metal Sign,6.19,2,14498,United Kingdom +581498,12/9/2019,21181,Please One Person Metal Sign,6.19,6,14498,United Kingdom +581498,12/9/2019,21216,Set 3 Retrospot Tea/Coffee/Sugar,6.19,2,14498,United Kingdom +581498,12/9/2019,21217,Red Retrospot Round Cake Tins,6.19,3,14498,United Kingdom +581498,12/9/2019,21218,Red Spotty Biscuit Tin,6.19,4,14498,United Kingdom +581498,12/9/2019,21232,Strawberry Ceramic Trinket Pot,6.19,13,14498,United Kingdom +581498,12/9/2019,21239,Pink Polkadot Cup,6.19,1,14498,United Kingdom +581498,12/9/2019,21257,Victorian Sewing Box Medium,6.19,3,14498,United Kingdom +581498,12/9/2019,21327,Skulls Writing Set,6.19,1,14498,United Kingdom +581498,12/9/2019,21328,Balloons Writing Set,6.19,1,14498,United Kingdom +581498,12/9/2019,21329,Dinosaurs Writing Set,6.19,1,14498,United Kingdom +581498,12/9/2019,21411,Gingham Heart Doorstop Red,6.19,1,14498,United Kingdom +581498,12/9/2019,21430,Set/3 Red Gingham Rose Storage Box,6.19,3,14498,United Kingd +581498,12/9/2019,21494,Rotating Leaves T-Light Holder,6.19,7,14498,United Kingdom +581498,12/9/2019,21523,Doormat Fancy Font Home Sweet Home,6.19,1,14498,United Kingd +581498,12/9/2019,21557,Set Of 6 Funky Beakers,6.19,1,14498,United Kingdom +581498,12/9/2019,21558,Skull Lunch Box With Cutlery,6.19,1,14498,United Kingdom +581498,12/9/2019,21731,Red Toadstool Led Night Light,6.19,9,14498,United Kingdom +581498,12/9/2019,21754,Home Building Block Word,6.19,2,14498,United Kingdom +581498,12/9/2019,21790,Vintage Snap Cards,6.19,1,14498,United Kingdom +581498,12/9/2019,21843,Red Retrospot Cake Stand,6.19,5,14498,United Kingdom +581498,12/9/2019,21864,Union Jack Flag Passport Cover,6.19,2,14498,United Kingdom +581498,12/9/2019,21874,Gin And Tonic Mug,6.19,2,14498,United Kingdom +581498,12/9/2019,21876,Pottering Mug,6.19,4,14498,United Kingdom +581498,12/9/2019,21890,S/6 Wooden Skittles In Cotton Bag,6.19,1,14498,United Kingdo +581498,12/9/2019,21912,Vintage Snakes & Ladders,6.19,1,14498,United Kingdom +581498,12/9/2019,21916,Set 12 Retro White Chalk Sticks,6.19,7,14498,United Kingdom +581498,12/9/2019,21930,Jumbo Storage Bag Skulls,6.19,2,14498,United Kingdom +581498,12/9/2019,21931,Jumbo Storage Bag Suki,6.19,6,14498,United Kingdom +581498,12/9/2019,21934,Skull Shoulder Bag,6.19,1,14498,United Kingdom +581498,12/9/2019,21935,Suki Shoulder Bag,6.19,7,14498,United Kingdom +581498,12/9/2019,21942,Skulls Design Flannel,6.19,1,14498,United Kingdom +581498,12/9/2019,21955,Doormat Union Jack Guns And Roses,6.19,1,14498,United Kingdo +581498,12/9/2019,21977,Pack Of 60 Pink Paisley Cake Cases,6.19,1,14498,United Kingd +581498,12/9/2019,21983,Pack Of 12 Blue Paisley Tissues,6.19,1,14498,United Kingdom +581498,12/9/2019,21987,Pack Of 6 Skull Paper Cups,6.19,2,14498,United Kingdom +581498,12/9/2019,21989,Pack Of 20 Skull Paper Napkins,6.19,1,14498,United Kingdom +581498,12/9/2019,22041,"Record Frame 7"" Single Size",6.19,2,14498,United Kingdom +581498,12/9/2019,22059,Ceramic Strawberry Design Mug,6.19,1,14498,United Kingdom +581498,12/9/2019,22069,Brown Pirate Treasure Chest,6.19,3,14498,United Kingdom +581498,12/9/2019,22077,6 Ribbons Rustic Charm,6.19,2,14498,United Kingdom +581498,12/9/2019,22083,Paper Chain Kit Retrospot,6.19,2,14498,United Kingdom +581498,12/9/2019,22086,Paper Chain Kit 50'S Christmas,6.19,52,14498,United Kingdom +581498,12/9/2019,22099,Caravan Square Tissue Box,6.19,2,14498,United Kingdom +581498,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,6.19,4,14498,United Kingdo +581498,12/9/2019,22139,Retrospot Tea Set Ceramic 11 Pc,6.19,2,14498,United Kingdom +581498,12/9/2019,22141,Christmas Craft Tree Top Angel,6.19,7,14498,United Kingdom +581498,12/9/2019,22142,Christmas Craft White Fairy,6.19,2,14498,United Kingdom +581498,12/9/2019,22144,Christmas Craft Little Friends,6.19,8,14498,United Kingdom +581498,12/9/2019,22161,Heart Decoration Rustic Hanging,6.19,2,14498,United Kingdom +581498,12/9/2019,22173,Metal 4 Hook Hanger French Chateau,6.19,3,14498,United Kingd +581498,12/9/2019,22174,Photo Cube,6.19,3,14498,United Kingdom +581498,12/9/2019,22175,Pink Owl Soft Toy,6.19,1,14498,United Kingdom +581498,12/9/2019,22178,Victorian Glass Hanging T-Light,6.19,1,14498,United Kingdom +581498,12/9/2019,22179,Set 10 Night Owl Lights,6.19,1,14498,United Kingdom +581498,12/9/2019,22186,Red Star Card Holder,6.19,1,14498,United Kingdom +581498,12/9/2019,22187,Green Christmas Tree Card Holder,6.19,6,14498,United Kingdom +581498,12/9/2019,22193,Red Diner Wall Clock,6.19,1,14498,United Kingdom +581498,12/9/2019,22195,Large Heart Measuring Spoons,6.19,3,14498,United Kingdom +581498,12/9/2019,22196,Small Heart Measuring Spoons,6.19,2,14498,United Kingdom +581498,12/9/2019,22207,Frying Pan Union Flag,6.19,1,14498,United Kingdom +581498,12/9/2019,22278,Overnight Bag Vintage Rose Paisley,6.19,2,14498,United Kingd +581498,12/9/2019,22301,Coffee Mug Cat + Bird Design,6.19,2,14498,United Kingdom +581498,12/9/2019,22302,Coffee Mug Pears Design,6.19,4,14498,United Kingdom +581498,12/9/2019,22303,Coffee Mug Apples Design,6.19,4,14498,United Kingdom +581498,12/9/2019,22304,Coffee Mug Blue Paisley Design,6.19,1,14498,United Kingdom +581498,12/9/2019,22308,Tea Cosy Blue Stripe,6.19,2,14498,United Kingdom +581498,12/9/2019,22327,Round Snack Boxes Set Of 4 Skulls,6.19,4,14498,United Kingdo +581498,12/9/2019,22328,Round Snack Boxes Set Of 4 Fruits,6.19,4,14498,United Kingdo +581498,12/9/2019,22352,Lunch Box With Cutlery Retrospot,6.19,6,14498,United Kingdom +581498,12/9/2019,22357,Kings Choice Biscuit Tin,6.19,1,14498,United Kingdom +581498,12/9/2019,22358,Kings Choice Tea Caddy,6.19,1,14498,United Kingdom +581498,12/9/2019,22367,Childrens Apron Spaceboy Design,6.19,7,14498,United Kingdom +581498,12/9/2019,22371,Airline Bag Vintage Tokyo 78,6.19,1,14498,United Kingdom +581498,12/9/2019,22374,Airline Bag Vintage Jet Set Red,6.19,1,14498,United Kingdom +581498,12/9/2019,22378,Wall Tidy Retrospot,7.24,2,14498,United Kingdom +581498,12/9/2019,22379,Recycling Bag Retrospot,7.24,4,14498,United Kingdom +581498,12/9/2019,22381,Toy Tidy Pink Polkadot,7.24,1,14498,United Kingdom +581498,12/9/2019,22411,Jumbo Shopper Vintage Red Paisley,7.24,2,14498,United Kingdo +581498,12/9/2019,22422,Toothpaste Tube Pen,7.24,1,14498,United Kingdom +581498,12/9/2019,22424,Enamel Bread Bin Cream,7.24,1,14498,United Kingdom +581498,12/9/2019,22429,Enamel Measuring Jug Cream,7.24,1,14498,United Kingdom +581498,12/9/2019,22456,Natural Slate Chalkboard Large,7.24,4,14498,United Kingdom +581498,12/9/2019,22457,Natural Slate Heart Chalkboard,7.24,2,14498,United Kingdom +581498,12/9/2019,22471,Tv Dinner Tray Air Hostess,7.24,1,14498,United Kingdom +581498,12/9/2019,22474,Spaceboy Tv Dinner Tray,7.24,1,14498,United Kingdom +581498,12/9/2019,22488,Natural Slate Rectangle Chalkboard,7.24,2,14498,United Kingd +581498,12/9/2019,22498,Wooden Regatta Bunting,7.24,1,14498,United Kingdom +581498,12/9/2019,22507,Memo Board Retrospot Design,7.24,1,14498,United Kingdom +581498,12/9/2019,22526,Wheelbarrow For Children,7.24,1,14498,United Kingdom +581498,12/9/2019,22553,Plasters In Tin Skulls,7.24,1,14498,United Kingdom +581498,12/9/2019,22557,Plasters In Tin Vintage Paisley,7.24,1,14498,United Kingdom +581498,12/9/2019,22619,Set Of 6 Soldier Skittles,7.24,1,14498,United Kingdom +581498,12/9/2019,22622,Box Of Vintage Alphabet Blocks,7.24,1,14498,United Kingdom +581498,12/9/2019,22624,Ivory Kitchen Scales,7.24,1,14498,United Kingdom +581498,12/9/2019,22629,Spaceboy Lunch Box,7.24,2,14498,United Kingdom +581498,12/9/2019,22632,Hand Warmer Red Retrospot,7.24,1,14498,United Kingdom +581498,12/9/2019,22645,Ceramic Heart Fairy Cake Money Bank,7.24,4,14498,United King +581498,12/9/2019,22654,Deluxe Sewing Kit,6.19,2,14498,United Kingdom +581498,12/9/2019,22659,Lunch Box I Love London,6.19,1,14498,United Kingdom +581498,12/9/2019,22666,Recipe Box Pantry Yellow Design,6.19,11,14498,United Kingdom +581498,12/9/2019,22676,French Blue Metal Door Sign 1,6.04,1,14498,United Kingdom +581498,12/9/2019,22684,French Blue Metal Door Sign 9,6.04,1,14498,United Kingdom +581498,12/9/2019,22694,Wicker Star,6.04,1,14498,United Kingdom +581498,12/9/2019,22697,Green Regency Teacup And Saucer,6.04,6,14498,United Kingdom +581498,12/9/2019,22698,Pink Regency Teacup And Saucer,6.04,9,14498,United Kingdom +581498,12/9/2019,22699,Roses Regency Teacup And Saucer,6.19,6,14498,United Kingdom +581498,12/9/2019,22722,Set Of 6 Spice Tins Pantry Design,6.19,3,14498,United Kingdo +581498,12/9/2019,22733,3d Traditional Christmas Stickers,6.19,1,14498,United Kingdo +581498,12/9/2019,22734,Set Of 6 Ribbons Vintage Christmas,6.19,5,14498,United Kingd +581498,12/9/2019,22776,Sweetheart 3 Tier Cake Stand,6.19,1,14498,United Kingdom +581498,12/9/2019,22795,Sweetheart Recipe Book Stand,6.19,1,14498,United Kingdom +581498,12/9/2019,22807,Set Of 6 T-Lights Toadstools,6.19,2,14498,United Kingdom +581498,12/9/2019,22813,Pack 3 Boxes Bird Panettone,6.19,4,14498,United Kingdom +581498,12/9/2019,22838,3 Tier Cake Tin Red And Cream,6.19,1,14498,United Kingdom +581498,12/9/2019,22844,Vintage Cream Dog Food Container,6.19,2,14498,United Kingdom +581498,12/9/2019,22845,Vintage Cream Cat Food Container,6.19,1,14498,United Kingdom +581498,12/9/2019,22865,Hand Warmer Owl Design,6.19,1,14498,United Kingdom +581498,12/9/2019,22866,Hand Warmer Scotty Dog Design,6.19,2,14498,United Kingdom +581498,12/9/2019,22891,Tea For One Polkadot,6.19,1,14498,United Kingdom +581498,12/9/2019,22900,Set 2 Tea Towels I Love London,6.19,2,14498,United Kingdom +581498,12/9/2019,22910,Paper Chain Kit Vintage Christmas,6.19,5,14498,United Kingdo +581498,12/9/2019,22952,60 Cake Cases Vintage Christmas,6.19,7,14498,United Kingdom +581498,12/9/2019,22960,Jam Making Set With Jars,6.19,5,14498,United Kingdom +581498,12/9/2019,22961,Jam Making Set Printed,6.19,4,14498,United Kingdom +581498,12/9/2019,22966,Gingerbread Man Cookie Cutter,6.19,2,14498,United Kingdom +581498,12/9/2019,22993,Set Of 4 Pantry Jelly Moulds,6.19,2,14498,United Kingdom +581498,12/9/2019,23012,Glass Apothecary Bottle Perfume,6.19,1,14498,United Kingdom +581498,12/9/2019,23013,Glass Apothecary Bottle Tonic,6.19,1,14498,United Kingdom +581498,12/9/2019,23014,Glass Apothecary Bottle Elixir,7.24,2,14498,United Kingdom +581498,12/9/2019,23080,Red Metal Box Top Secret,7.24,5,14498,United Kingdom +581498,12/9/2019,23170,Regency Tea Plate Roses,7.24,4,14498,United Kingdom +581498,12/9/2019,23171,Regency Tea Plate Green,7.24,5,14498,United Kingdom +581498,12/9/2019,23172,Regency Tea Plate Pink,6.19,4,14498,United Kingdom +581498,12/9/2019,23181,Bull Dog Bottle Top Wall Clock,6.19,1,14498,United Kingdom +581498,12/9/2019,23196,Vintage Leaf Magnetic Notepad,6.19,1,14498,United Kingdom +581498,12/9/2019,23198,Pantry Magnetic Shopping List,6.19,2,14498,United Kingdom +581498,12/9/2019,23245,Set Of 3 Regency Cake Tins,6.19,5,14498,United Kingdom +581498,12/9/2019,23295,Set Of 12 Mini Loaf Baking Cases,6.19,1,14498,United Kingdom +581498,12/9/2019,23298,Spotty Bunting,6.19,1,14498,United Kingdom +581498,12/9/2019,23300,Gardeners Kneeling Pad Cup Of Tea,6.19,9,14498,United Kingdo +581498,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,32,14498,United Kingdo +581498,12/9/2019,23311,Vintage Christmas Stocking,6.19,6,14498,United Kingdom +581498,12/9/2019,23312,Vintage Christmas Gift Sack,6.19,3,14498,United Kingdom +581498,12/9/2019,23328,Set 6 School Milk Bottles In Crate,6.19,6,14498,United Kingd +581498,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,8,14498,United Kingdom +581498,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,22,14498,United Kingdom +581498,12/9/2019,23352,Roll Wrap 50'S Red Christmas,6.19,8,14498,United Kingdom +581498,12/9/2019,23353,6 Gift Tags Vintage Christmas,6.19,10,14498,United Kingdom +581498,12/9/2019,23354,6 Gift Tags 50'S Christmas,6.19,9,14498,United Kingdom +581498,12/9/2019,23368,Set 12 Colour Pencils Dolly Girl,6.19,3,14498,United Kingdom +581498,12/9/2019,23369,Set 36 Colour Pencils Love London,6.19,1,14498,United Kingdo +581498,12/9/2019,23370,Set 36 Colouring Pencils Doily,6.19,3,14498,United Kingdom +581498,12/9/2019,23382,Box Of 6 Christmas Cake Decorations,6.19,10,14498,United Kin +581498,12/9/2019,23388,Woodland Mini Backpack,6.19,1,14498,United Kingdom +581498,12/9/2019,23480,Mini Lights Woodland Mushrooms,7.24,1,14498,United Kingdom +581498,12/9/2019,23493,Vintage Doily Travel Sewing Kit,7.24,1,14498,United Kingdom +581498,12/9/2019,23494,Vintage Doily Deluxe Sewing Kit,7.24,1,14498,United Kingdom +581498,12/9/2019,23497,Classic Chrome Bicycle Bell,7.24,1,14498,United Kingdom +581498,12/9/2019,23501,Key Ring Baseball Boot Union Jack,7.24,1,14498,United Kingdo +581498,12/9/2019,23564,Egg Cup Milkmaid Ingrid,7.24,1,14498,United Kingdom +581498,12/9/2019,35970,Zinc Folkart Sleigh Bells,7.24,6,14498,United Kingdom +581498,12/9/2019,48138,Doormat Union Flag,7.24,1,14498,United Kingdom +581498,12/9/2019,71053,White Moroccan Metal Lantern,7.24,1,14498,United Kingdom +581498,12/9/2019,72349B,Set/6 Purple Butterfly T-Lights,7.24,2,14498,United Kingdom +581498,12/9/2019,79321,Chilli Lights,7.24,10,14498,United Kingdom +581498,12/9/2019,82001S,Silver Record Cover Frame,6.19,2,14498,United Kingdom +581498,12/9/2019,82482,Wooden Picture Frame White Finish,6.04,4,14498,United Kingdo +581498,12/9/2019,82552,Washroom Metal Sign,6.04,1,14498,United Kingdom +581498,12/9/2019,21171,Bathroom Metal Sign,6.19,1,14498,United Kingdom +581498,12/9/2019,82581,Toilet Metal Sign,6.19,1,14498,United Kingdom +581498,12/9/2019,82600,N0 Singing Metal Sign,6.19,4,14498,United Kingdom +581498,12/9/2019,84029E,Red Woolly Hottie White Heart,6.19,4,14498,United Kingdom +581498,12/9/2019,84032A,Charlie+Lola Pink Hot Water Bottle,6.19,4,14498,United King +581498,12/9/2019,84032B,Charlie + Lola Red Hot Water Bottle,6.19,3,14498,United Kin +581498,12/9/2019,84375,Set Of 20 Kids Cookie Cutters,6.19,3,14498,United Kingdom +581498,12/9/2019,84509A,Set Of 4 English Rose Placemats,6.19,1,14498,United Kingdom +581498,12/9/2019,84558A,3d Dog Picture Playing Cards,6.19,1,14498,United Kingdom +581498,12/9/2019,84832,Zinc Willie Winkie Candle Stick,6.19,26,14498,United Kingdom +581498,12/9/2019,84968E,Set Of 16 Vintage Black Cutlery,6.19,1,14498,United Kingdom +581498,12/9/2019,84970S,Hanging Heart Zinc T-Light Holder,6.19,1,14498,United Kingd +581498,12/9/2019,84997A,Childrens Cutlery Polkadot Green,6.19,2,14498,United Kingdo +581498,12/9/2019,84997B,Childrens Cutlery Retrospot Red,6.19,3,14498,United Kingdom +581498,12/9/2019,84997D,Childrens Cutlery Polkadot Pink,6.19,1,14498,United Kingdom +581498,12/9/2019,85038,6 Chocolate Love Heart T-Lights,6.19,1,14498,United Kingdom +581498,12/9/2019,85048,15cm Christmas Glass Ball 20 Lights,6.19,1,14498,United King +581498,12/9/2019,85049A,Traditional Christmas Ribbons,6.17,5,14498,United Kingdom +581498,12/9/2019,85049E,Scandinavian Reds Ribbons,6.19,4,14498,United Kingdom +581498,12/9/2019,85150,Ladies & Gentlemen Metal Sign,6.19,1,14498,United Kingdom +581498,12/9/2019,85174,S/4 Cacti Candles,6.19,1,14498,United Kingdom +581498,12/9/2019,20712,Jumbo Bag Woodland Animals,6.19,3,14498,United Kingdom +581498,12/9/2019,20713,Jumbo Bag Owls,6.19,8,14498,United Kingdom +581498,12/9/2019,21928,Jumbo Bag Scandinavian Blue Paisley,6.19,2,14498,United King +581498,12/9/2019,22386,Jumbo Bag Pink Polkadot,7.24,1,14498,United Kingdom +581498,12/9/2019,22663,Jumbo Bag Dolly Girl Design,6.19,2,14498,United Kingdom +581498,12/9/2019,23199,Jumbo Bag Apples,6.19,6,14498,United Kingdom +581498,12/9/2019,23201,Jumbo Bag Alphabet,6.19,6,14498,United Kingdom +581498,12/9/2019,85099B,Jumbo Bag Red Retrospot,6.19,5,14498,United Kingdom +581498,12/9/2019,85099C,Jumbo Bag Baroque Black White,6.19,4,14498,United Kingdom +581498,12/9/2019,20726,Lunch Bag Woodland,6.19,3,14498,United Kingdom +581498,12/9/2019,20728,Lunch Bag Cars Blue,6.19,4,14498,United Kingdom +581498,12/9/2019,22384,Lunch Bag Pink Polkadot,7.24,1,14498,United Kingdom +581498,12/9/2019,23437,50'S Christmas Gift Bag Large,7.24,1,14498,United Kingdom +581500,12/9/2019,82486,3 Drawer Antique White Wood Cabinet,7.24,4,15344,United King +581500,12/9/2019,85066,Cream Sweetheart Mini Chest,7.24,4,15344,United Kingdom +581500,12/9/2019,48187,Doormat New England,7.24,2,15344,United Kingdom +581501,12/9/2019,22319,Hairclips Forties Fabric Assorted,7.24,180,12985,United King +581501,12/9/2019,20704,Mr Robot Soft Toy,7.24,8,12985,United Kingdom +581501,12/9/2019,21564,Pink Heart Shape Love Bucket,6.19,24,12985,United Kingdom +581501,12/9/2019,21563,Red Heart Shape Love Bucket,7.24,24,12985,United Kingdom +581501,12/9/2019,22165,Diamante Heart Shaped Wall Mirror,7.24,12,12985,United Kingd +581501,12/9/2019,22299,Pig Keyring With Light & Sound,7.24,48,12985,United Kingdom +581501,12/9/2019,22447,Pin Cushion Babushka Blue,7.24,12,12985,United Kingdom +581501,12/9/2019,22442,Grow Your Own Flowers Set Of 3,7.24,12,12985,United Kingdom +581501,12/9/2019,22495,Set Of 2 Round Tins Camembert,7.24,12,12985,United Kingdom +581501,12/9/2019,22544,Mini Jigsaw Spaceboy,7.24,96,12985,United Kingdom +581501,12/9/2019,22695,Wicker Wreath Small,7.24,24,12985,United Kingdom +581501,12/9/2019,22785,Squarecushion Cover Pink Union Jack,7.24,12,12985,United Kin +581501,12/9/2019,22811,Set Of 6 T-Lights Cacti,7.24,12,12985,United Kingdom +581501,12/9/2019,22807,Set Of 6 T-Lights Toadstools,7.24,12,12985,United Kingdom +581501,12/9/2019,22942,Christmas Lights 10 Santas,7.24,12,12985,United Kingdom +581501,12/9/2019,22808,Set Of 6 T-Lights Easter Chicks,6.19,12,12985,United Kingdom +581501,12/9/2019,23143,Zinc Wire Kitchen Organiser,7.24,4,12985,United Kingdom +581501,12/9/2019,23151,Zinc Sweetheart Soap Dish,7.24,12,12985,United Kingdom +581501,12/9/2019,23425,Storage Tin Home Sweet Home,7.24,12,12985,United Kingdom +581501,12/9/2019,84356,Pompom Curtain,7.24,12,12985,United Kingdom +581501,12/9/2019,85173,Set/6 Frog Prince T-Light Candles,7.24,12,12985,United Kingd +581501,12/9/2019,21731,Red Toadstool Led Night Light,7.24,24,12985,United Kingdom +581501,12/9/2019,23480,Mini Lights Woodland Mushrooms,7.24,8,12985,United Kingdom +581501,12/9/2019,22545,Mini Jigsaw Bunnies,7.24,96,12985,United Kingdom +581502,12/9/2019,22087,Paper Bunting White Lace,7.24,6,15910,United Kingdom +581502,12/9/2019,21209,Multicolour Honeycomb Fan,7.24,5,15910,United Kingdom +581502,12/9/2019,20668,Disco Ball Christmas Decoration,7.24,24,15910,United Kingdom +581502,12/9/2019,21790,Vintage Snap Cards,7.24,6,15910,United Kingdom +581502,12/9/2019,23270,Set Of 2 Ceramic Painted Hearts,7.24,4,15910,United Kingdom +581502,12/9/2019,23103,Jingle Bell Heart Decoration,7.24,8,15910,United Kingdom +581502,12/9/2019,22576,Swallow Wooden Christmas Decoration,7.24,2,15910,United King +581502,12/9/2019,22153,Angel Decoration Stars On Dress,7.24,13,15910,United Kingdom +581502,12/9/2019,21810,Christmas Hanging Star With Bell,7.24,6,15910,United Kingdom +581502,12/9/2019,22155,Star Decoration Rustic,7.24,6,15910,United Kingdom +581502,12/9/2019,23210,White Rocking Horse Hand Painted,7.24,12,15910,United Kingdo +581502,12/9/2019,22573,Star Wooden Christmas Decoration,7.24,8,15910,United Kingdom +581502,12/9/2019,22075,6 Ribbons Elegant Christmas,7.24,4,15910,United Kingdom +581502,12/9/2019,85049A,Traditional Christmas Ribbons,7.24,4,15910,United Kingdom +581502,12/9/2019,22734,Set Of 6 Ribbons Vintage Christmas,7.24,4,15910,United Kingd +581502,12/9/2019,23274,Star T-Light Holder Willie Winkie,7.24,8,15910,United Kingdo +581502,12/9/2019,22153,Angel Decoration Stars On Dress,7.24,1,15910,United Kingdom +581502,12/9/2019,22596,Christmas Star Wish List Chalkboard,7.24,24,15910,United Kin +581502,12/9/2019,22952,60 Cake Cases Vintage Christmas,7.24,10,15910,United Kingdom +581502,12/9/2019,22141,Christmas Craft Tree Top Angel,7.24,3,15910,United Kingdom +581514,12/9/2019,22753,Small Yellow Babushka Notebook,7.24,12,17754,United Kingdom +581514,12/9/2019,22755,Small Purple Babushka Notebook,7.24,12,17754,United Kingdom +581514,12/9/2019,22754,Small Red Babushka Notebook,6.19,12,17754,United Kingdom +581514,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,6.19,4,17754,United Kingdom +581514,12/9/2019,22059,Ceramic Strawberry Design Mug,6.19,12,17754,United Kingdom +581514,12/9/2019,22199,Frying Pan Red Retrospot,6.19,13,17754,United Kingdom +581514,12/9/2019,22200,Frying Pan Pink Polkadot,6.19,2,17754,United Kingdom +581514,12/9/2019,84032A,Charlie+Lola Pink Hot Water Bottle,6.19,9,17754,United King +581514,12/9/2019,84032B,Charlie + Lola Red Hot Water Bottle,6.19,9,17754,United Kin +581514,12/9/2019,84031A,Charlie+Lola Red Hot Water Bottle,6.19,10,17754,United King +581514,12/9/2019,84031B,Charlie Lola Blue Hot Water Bottle,6.19,14,17754,United Kin +581514,12/9/2019,22646,Ceramic Strawberry Cake Money Bank,6.19,4,17754,United Kingd +581514,12/9/2019,22644,Ceramic Cherry Cake Money Bank,6.19,4,17754,United Kingdom +581514,12/9/2019,22645,Ceramic Heart Fairy Cake Money Bank,6.19,4,17754,United King +581514,12/9/2019,22394,Paperweight Kings Choice,6.04,12,17754,United Kingdom +581514,12/9/2019,17091J,Vanilla Incense In Tin,6.04,66,17754,United Kingdom +581514,12/9/2019,35471D,Set Of 3 Bird Light Pink Feather,6.04,12,17754,United Kingd +581514,12/9/2019,22075,6 Ribbons Elegant Christmas,6.04,24,17754,United Kingdom +581514,12/9/2019,17091J,Vanilla Incense In Tin,6.19,24,17754,United Kingdom +581514,12/9/2019,22069,Brown Pirate Treasure Chest,6.19,20,17754,United Kingdom +581514,12/9/2019,22068,Black Pirate Treasure Chest,6.19,14,17754,United Kingdom +581514,12/9/2019,22500,Set Of 2 Tins Jardin De Provence,6.19,4,17754,United Kingdom +581514,12/9/2019,22075,6 Ribbons Elegant Christmas,6.19,24,17754,United Kingdom +581514,12/9/2019,21705,Bag 500g Swirly Marbles,6.19,84,17754,United Kingdom +581516,12/9/2019,21109,Large Cake Towel Chocolate Spots,6.19,12,14422,United Kingdo +581516,12/9/2019,21111,Swiss Roll Towel Chocolate Spots,6.19,24,14422,United Kingdo +581516,12/9/2019,21705,Bag 500g Swirly Marbles,6.19,24,14422,United Kingdom +581516,12/9/2019,22185,Slate Tile Natural Hanging,6.19,12,14422,United Kingdom +581516,12/9/2019,22442,Grow Your Own Flowers Set Of 3,6.19,12,14422,United Kingdom +581516,12/9/2019,21620,Set Of 4 Rose Botanical Candles,6.19,12,14422,United Kingdom +581516,12/9/2019,84029G,Knitted Union Flag Hot Water Bottle,6.19,8,14422,United Kin +581516,12/9/2019,21485,Retrospot Heart Hot Water Bottle,6.19,3,14422,United Kingdom +581516,12/9/2019,22111,Scottie Dog Hot Water Bottle,6.19,6,14422,United Kingdom +581516,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,8,14422,United Kingdom +581516,12/9/2019,22112,Chocolate Hot Water Bottle,6.19,6,14422,United Kingdom +581516,12/9/2019,23356,Love Hot Water Bottle,6.19,6,14422,United Kingdom +581516,12/9/2019,21108,Fairy Cake Flannel Assorted Colour,6.19,18,14422,United King +581516,12/9/2019,22171,3 Hook Photo Shelf Antique White,6.19,4,14422,United Kingdom +581516,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,24,14422,United Kingdo +581538,12/9/2019,23193,Buffalo Bill Treasure Book Box,6.19,6,14446,United Kingdom +581538,12/9/2019,23194,Gymkhana Treasure Book Box,6.19,1,14446,United Kingdom +581538,12/9/2019,23084,Rabbit Night Light,6.19,2,14446,United Kingdom +581538,12/9/2019,22068,Black Pirate Treasure Chest,6.19,1,14446,United Kingdom +581538,12/9/2019,22956,36 Foil Heart Cake Cases,6.19,1,14446,United Kingdom +581538,12/9/2019,20936,Forked Cactus Candle,6.19,1,14446,United Kingdom +581538,12/9/2019,23040,Paper Lantern 9 Point Snow Star,6.19,1,14446,United Kingdom +581538,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,6.19,1,14446,United King +581538,12/9/2019,21222,Set/4 Badges Beetles,6.19,1,14446,United Kingdom +581538,12/9/2019,21220,Set/4 Badges Dogs,6.19,1,14446,United Kingdom +581538,12/9/2019,21224,Set/4 Skull Badges,6.19,1,14446,United Kingdom +581538,12/9/2019,85123A,Cream Hanging Heart T-Light Holder,6.19,1,14446,United King +581538,12/9/2019,22992,Revolver Wooden Ruler,6.19,1,14446,United Kingdom +581538,12/9/2019,79066K,Retro Mod Tray,6.19,1,14446,United Kingdom +581538,12/9/2019,84380,Set Of 3 Butterfly Cookie Cutters,6.19,1,14446,United Kingdo +581538,12/9/2019,23371,Set 36 Colour Pencils Spaceboy,6.19,1,14446,United Kingdom +581538,12/9/2019,22694,Wicker Star,6.19,1,14446,United Kingdom +581538,12/9/2019,22095,Lads Only Tissue Box,6.19,1,14446,United Kingdom +581538,12/9/2019,23275,Set Of 3 Hanging Owls Ollie Beak,6.19,1,14446,United Kingdom +581538,12/9/2019,21673,White Spot Blue Ceramic Drawer Knob,6.19,1,14446,United King +581538,12/9/2019,21670,Blue Spot Ceramic Drawer Knob,6.19,1,14446,United Kingdom +581538,12/9/2019,85071C,"Charlie+Lola""Extremely Busy"" Sign",6.19,1,14446,United K +581538,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,3,14446,United Kingdom +581538,12/9/2019,23034,Drawer Knob Ceramic Black,6.19,1,14446,United Kingdom +581538,12/9/2019,21669,Blue Stripe Ceramic Drawer Knob,6.19,1,14446,United Kingdom +581538,12/9/2019,23033,Drawer Knob Ceramic Red,6.19,1,14446,United Kingdom +581538,12/9/2019,21668,Red Stripe Ceramic Drawer Knob,6.19,1,14446,United Kingdom +581538,12/9/2019,23275,Set Of 3 Hanging Owls Ollie Beak,6.19,1,14446,United Kingdom +581538,12/9/2019,23318,Box Of 6 Mini Vintage Crackers,6.19,1,14446,United Kingdom +581538,12/9/2019,23382,Box Of 6 Christmas Cake Decorations,6.19,1,14446,United King +581538,12/9/2019,22469,Heart Of Wicker Small,6.19,1,14446,United Kingdom +581538,12/9/2019,22899,Children's Apron Dolly Girl,6.19,2,14446,United Kingdom +581538,12/9/2019,22367,Childrens Apron Spaceboy Design,6.19,1,14446,United Kingdom +581538,12/9/2019,22899,Children's Apron Dolly Girl,6.19,3,14446,United Kingdom +581538,12/9/2019,23527,Wall Art Animals And Nature,6.19,1,14446,United Kingdom +581538,12/9/2019,23524,Wall Art Horse & Pony,6.19,1,14446,United Kingdom +581538,12/9/2019,23525,Wall Art Buffalo Bill,6.19,1,14446,United Kingdom +581538,12/9/2019,84380,Set Of 3 Butterfly Cookie Cutters,6.19,4,14446,United Kingdo +581538,12/9/2019,23122,Party Charms 50 Pieces,7.24,1,14446,United Kingdom +581538,12/9/2019,21990,Modern Floral Stationery Set,7.24,1,14446,United Kingdom +581538,12/9/2019,21329,Dinosaurs Writing Set,7.24,1,14446,United Kingdom +581538,12/9/2019,21328,Balloons Writing Set,7.24,1,14446,United Kingdom +581538,12/9/2019,22561,Wooden School Colouring Set,7.24,1,14446,United Kingdom +581538,12/9/2019,84519A,Tomato Charlie+Lola Coaster Set,7.24,1,14446,United Kingdom +581538,12/9/2019,84519B,Carrot Charlie+Lola Coaster Set,7.24,1,14446,United Kingdom +581538,12/9/2019,35004B,Set Of 3 Black Flying Ducks,7.24,2,14446,United Kingdom +581538,12/9/2019,22068,Black Pirate Treasure Chest,7.24,1,14446,United Kingdom +581538,12/9/2019,23353,6 Gift Tags Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,21591,Cosy Hour Cigar Box Matches,6.19,1,14446,United Kingdom +581538,12/9/2019,22197,Popcorn Holder,6.19,4,14446,United Kingdom +581538,12/9/2019,23320,Giant 50'S Christmas Cracker,6.19,1,14446,United Kingdom +581538,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,22985,Wrap Billboard Fonts Design,6.19,25,14446,United Kingdom +581538,12/9/2019,23040,Paper Lantern 9 Point Snow Star,6.19,2,14446,United Kingdom +581538,12/9/2019,23040,Paper Lantern 9 Point Snow Star,6.19,3,14446,United Kingdom +581538,12/9/2019,21194,Pink Honeycomb Paper Fan,6.19,2,14446,United Kingdom +581538,12/9/2019,21208,Pastel Colour Honeycomb Fan,6.19,1,14446,United Kingdom +581538,12/9/2019,79190D,Retro Plastic Daisy Tray,6.19,1,14446,United Kingdom +581538,12/9/2019,79190B,Retro Plastic Polka Tray,6.19,1,14446,United Kingdom +581538,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,6.02,2,14446,United King +581538,12/9/2019,23318,Box Of 6 Mini Vintage Crackers,6.02,1,14446,United Kingdom +581538,12/9/2019,23319,Box Of 6 Mini 50'S Crackers,6.02,1,14446,United Kingdom +581538,12/9/2019,23537,Wall Art I Love London,6.19,1,14446,United Kingdom +581538,12/9/2019,22992,Revolver Wooden Ruler,6.19,1,14446,United Kingdom +581538,12/9/2019,22991,Giraffe Wooden Ruler,6.19,1,14446,United Kingdom +581538,12/9/2019,23190,Bundle Of 3 School Exercise Books,6.19,1,14446,United Kingdo +581538,12/9/2019,21194,Pink Honeycomb Paper Fan,6.19,1,14446,United Kingdom +581538,12/9/2019,35004B,Set Of 3 Black Flying Ducks,6.19,1,14446,United Kingdom +581538,12/9/2019,22694,Wicker Star,6.19,1,14446,United Kingdom +581538,12/9/2019,21355,Toast Its - I Love You,6.19,1,14446,United Kingdom +581538,12/9/2019,23343,Jumbo Bag Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,23343,Jumbo Bag Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,20727,Lunch Bag Black Skull,6.19,1,14446,United Kingdom +581538,12/9/2019,20725,Lunch Bag Red Retrospot,6.19,1,14446,United Kingdom +581566,12/9/2019,23404,Home Sweet Home Blackboard,6.19,144,18102,United Kingdom +581567,12/9/2019,21417,Cockle Shell Dish,6.19,84,16626,United Kingdom +581567,12/9/2019,22464,Hanging Metal Heart Lantern,6.19,24,16626,United Kingdom +581567,12/9/2019,22465,Hanging Metal Star Lantern,6.19,24,16626,United Kingdom +581567,12/9/2019,84971S,Small Heart Flowers Hook,6.19,48,16626,United Kingdom +581567,12/9/2019,22624,Ivory Kitchen Scales,6.19,2,16626,United Kingdom +581567,12/9/2019,22627,Mint Kitchen Scales,6.19,2,16626,United Kingdom +581567,12/9/2019,22625,Red Kitchen Scales,6.19,2,16626,United Kingdom +581567,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,4,16626,United Kingdom +581567,12/9/2019,21326,Aged Glass Silver T-Light Holder,6.19,144,16626,United Kingd +581567,12/9/2019,21479,White Skull Hot Water Bottle,6.19,4,16626,United Kingdom +581567,12/9/2019,23356,Love Hot Water Bottle,6.19,3,16626,United Kingdom +581567,12/9/2019,21137,Black Record Cover Frame,6.19,24,16626,United Kingdom +581570,12/9/2019,22141,Christmas Craft Tree Top Angel,6.19,6,12662,Germany +581570,12/9/2019,22175,Pink Owl Soft Toy,6.19,6,12662,Germany +581570,12/9/2019,21481,Fawn Blue Hot Water Bottle,6.19,4,12662,Germany +581570,12/9/2019,23570,Traditional Pick Up Sticks Game,6.19,12,12662,Germany +581570,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,6.19,6,12662,Germany +581570,12/9/2019,22331,Woodland Party Bag + Sticker Set,6.19,8,12662,Germany +581570,12/9/2019,22834,Hand Warmer Babushka Design,6.19,12,12662,Germany +581570,12/9/2019,21914,Blue Harmonica In Box,6.19,12,12662,Germany +581570,12/9/2019,22139,Retrospot Tea Set Ceramic 11 Pc,6.19,3,12662,Germany +581570,12/9/2019,23077,Doughnut Lip Gloss,6.19,20,12662,Germany +581570,12/9/2019,20750,Red Retrospot Mini Cases,6.19,2,12662,Germany +581570,12/9/2019,22505,Memo Board Cottage Design,6.19,4,12662,Germany +581571,12/9/2019,23326,Hanging Mini Coloured Bottles,6.19,6,15311,United Kingdom +581571,12/9/2019,21313,Glass Heart T-Light Holder,6.19,1,15311,United Kingdom +581571,12/9/2019,48187,Doormat New England,6.19,1,15311,United Kingdom +581571,12/9/2019,23317,Blue Refectory Clock,6.19,1,15311,United Kingdom +581571,12/9/2019,23197,Sketchbook Magnetic Shopping List,6.19,4,15311,United Kingdo +581571,12/9/2019,21012,Antique All Glass Candlestick,6.19,2,15311,United Kingdom +581571,12/9/2019,22227,Hanging Heart Mirror Decoration,6.19,10,15311,United Kingdom +581571,12/9/2019,22794,Sweetheart Wire Magazine Rack,6.19,1,15311,United Kingdom +581571,12/9/2019,23182,Toilet Sign Occupied Or Vacant,6.19,1,15311,United Kingdom +581571,12/9/2019,21755,Love Building Block Word,6.19,1,15311,United Kingdom +581571,12/9/2019,85053,French Enamel Candleholder,6.19,2,15311,United Kingdom +581571,12/9/2019,23110,Parisienne Key Cabinet,6.19,2,15311,United Kingdom +581571,12/9/2019,21169,You're Confusing Me Metal Sign,6.19,1,15311,United Kingdom +581571,12/9/2019,21258,Victorian Sewing Box Large,6.19,8,15311,United Kingdom +581571,12/9/2019,23168,Classic Cafe Sugar Dispenser,6.19,36,15311,United Kingdom +581571,12/9/2019,23167,Small Ceramic Top Storage Jar,6.19,96,15311,United Kingdom +581571,12/9/2019,21314,Small Glass Heart Trinket Pot,6.19,48,15311,United Kingdom +581571,12/9/2019,21137,Black Record Cover Frame,6.19,24,15311,United Kingdom +581571,12/9/2019,44234,Assorted Circular Mobile,6.19,1,15311,United Kingdom +581571,12/9/2019,84347,Rotating Silver Angels T-Light Hldr,6.19,24,15311,United Kin +581572,12/9/2019,23328,Set 6 School Milk Bottles In Crate,6.19,48,16705,United King +581572,12/9/2019,22627,Mint Kitchen Scales,6.19,4,16705,United Kingdom +581572,12/9/2019,22624,Ivory Kitchen Scales,6.19,4,16705,United Kingdom +581572,12/9/2019,23245,Set Of 3 Regency Cake Tins,6.19,4,16705,United Kingdom +581574,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,6.19,6,12526,Germany +581574,12/9/2019,22328,Round Snack Boxes Set Of 4 Fruits,6.19,6,12526,Germany +581574,12/9/2019,23238,Set Of 4 Knick Knack Tins London,6.19,6,12526,Germany +581574,12/9/2019,23237,Set Of 4 Knick Knack Tins Leaf,6.19,6,12526,Germany +581574,12/9/2019,21257,Victorian Sewing Box Medium,6.19,2,12526,Germany +581574,12/9/2019,21258,Victorian Sewing Box Large,6.19,2,12526,Germany +581574,12/9/2019,23111,Parisienne Sewing Box,6.19,2,12526,Germany +581574,12/9/2019,22077,6 Ribbons Rustic Charm,6.19,12,12526,Germany +581574,12/9/2019,22074,6 Ribbons Shimmering Pinks,6.19,12,12526,Germany +581574,12/9/2019,22621,Traditional Knitting Nancy,6.19,12,12526,Germany +581574,12/9/2019,23199,Jumbo Bag Apples,6.19,10,12526,Germany +581574,12/9/2019,23581,Jumbo Bag Paisley Park,6.19,10,12526,Germany +581578,12/9/2019,21124,Set/10 Blue Polkadot Party Candles,6.19,24,12713,Germany +581578,12/9/2019,21122,Set/10 Pink Polkadot Party Candles,6.19,24,12713,Germany +581578,12/9/2019,21121,Set/10 Red Polkadot Party Candles,6.19,24,12713,Germany +581578,12/9/2019,23389,Spaceboy Mini Backpack,6.04,4,12713,Germany +581578,12/9/2019,22556,Plasters In Tin Circus Parade,6.19,12,12713,Germany +581578,12/9/2019,22976,Circus Parade Childrens Egg Cup,6.19,12,12713,Germany +581578,12/9/2019,23255,Childrens Cutlery Circus Parade,7.24,12,12713,Germany +581578,12/9/2019,84997D,Childrens Cutlery Polkadot Pink,7.24,8,12713,Germany +581578,12/9/2019,84997B,Childrens Cutlery Retrospot Red,7.24,8,12713,Germany +581578,12/9/2019,84997C,Childrens Cutlery Polkadot Blue,7.24,8,12713,Germany +581578,12/9/2019,22555,Plasters In Tin Strongman,7.24,12,12713,Germany +581578,12/9/2019,21914,Blue Harmonica In Box,7.24,12,12713,Germany +581578,12/9/2019,22549,Picture Dominoes,7.24,24,12713,Germany +581578,12/9/2019,21918,Set 12 Kids Colour Chalk Sticks,7.24,24,12713,Germany +581578,12/9/2019,22992,Revolver Wooden Ruler,7.24,12,12713,Germany +581578,12/9/2019,22991,Giraffe Wooden Ruler,7.24,12,12713,Germany +581578,12/9/2019,23229,Vintage Donkey Tail Game,7.24,6,12713,Germany +581578,12/9/2019,22622,Box Of Vintage Alphabet Blocks,6.19,6,12713,Germany +581578,12/9/2019,21506,Fancy Font Birthday Card,6.19,12,12713,Germany +581578,12/9/2019,21507,Elephant Birthday Card,6.19,12,12713,Germany +581578,12/9/2019,23037,Candle Holder Silver Madeline,6.19,12,12713,Germany +581578,12/9/2019,23550,Wrap Alphabet Poster,6.19,25,12713,Germany +581578,12/9/2019,22711,Wrap Circus Parade,6.19,25,12713,Germany +581578,12/9/2019,21497,Fancy Fonts Birthday Wrap,6.19,25,12713,Germany +581578,12/9/2019,22704,Wrap Red Apples,6.19,25,12713,Germany +581578,12/9/2019,22585,Pack Of 6 Birdy Gift Tags,6.19,12,12713,Germany diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/go-solution/main.go b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/go-solution/main.go new file mode 100644 index 0000000000000..ce5980b8e5b8a --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/go-solution/main.go @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// beam-playground: +// name: FinalSolution1 +// description: Final challenge solution 1. +// multifile: true +// files: +// - name: input.csv +// context_line: 54 +// categories: +// - Quickstart +// complexity: ADVANCED +// tags: +// - hellobeam + +package main + +import ( + "context" + "fmt" + "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/window" + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/window/trigger" + "github.com/apache/beam/sdks/v2/go/pkg/beam/io/textio" + "github.com/apache/beam/sdks/v2/go/pkg/beam/transforms/filter" + "github.com/apache/beam/sdks/v2/go/pkg/beam/transforms/stats" + "github.com/apache/beam/sdks/v2/go/pkg/beam/x/beamx" + "log" + "strconv" + "strings" + "time" +) + +type Transaction struct { + ID int64 + Date string + ProductID string + ProductName string + Price float64 + Quantity int64 + CustomerID int64 + Country string +} + +func main() { + ctx := context.Background() + + beam.Init() + p := beam.NewPipeline() + s := p.Root() + + file := textio.Read(s, "input.csv") + + transactions := getTransactions(s, file) + + trigger := trigger.AfterEndOfWindow(). + EarlyFiring(trigger.AfterProcessingTime(). + PlusDelay(5 * time.Second)). + LateFiring(trigger.Repeat(trigger.AfterCount(1))) + + fixedWindowedItems := beam.WindowInto(s, window.NewFixedWindows(30*time.Second), transactions, + beam.Trigger(trigger), + beam.AllowedLateness(30*time.Minute), + beam.PanesDiscard(), + ) + + filtered := filtering(s, fixedWindowedItems) + + result := getPartition(s, filtered) + + biggerThan10 := sumCombine(s, mapIdWithPrice(s, result[0])) + textio.Write(s, "price_more_than_10.txt", convertToString(s, biggerThan10)) + + smallerThan10 := sumCombine(s, mapIdWithPrice(s, result[1])) + textio.Write(s, "price_less_than_10.txt", convertToString(s, smallerThan10)) + + if err := beamx.Run(ctx, p); err != nil { + log.Fatalf("Failed to execute job: %v", err) + } +} + +func getTransactions(s beam.Scope, input beam.PCollection) beam.PCollection { + return beam.ParDo(s, func(line string, emit func(transaction Transaction)) { + csv := strings.Split(line, ",") + + if csv[0] != "TransactionNo" { + id, _ := strconv.ParseInt(csv[0], 10, 64) + price, _ := strconv.ParseFloat(csv[4], 64) + quantity, _ := strconv.ParseInt(csv[5], 10, 64) + customerID, _ := strconv.ParseInt(csv[6], 10, 64) + emit(Transaction{ + ID: id, + Date: csv[1], + ProductID: csv[2], + ProductName: csv[3], + Price: price, + Quantity: quantity, + CustomerID: customerID, + Country: csv[7], + }) + } + }, input) +} + +func getPartition(s beam.Scope, input beam.PCollection) []beam.PCollection { + return beam.Partition(s, 2, func(element Transaction) int { + if element.Price >= 10 { + return 0 + } + return 1 + }, input) +} + +func convertToString(s beam.Scope, input beam.PCollection) beam.PCollection { + return beam.ParDo(s, func(product string, sum float64, emit func(string)) { + emit(fmt.Sprint("product: ", product, " , sum: ", sum)) + }, input) +} + +func filtering(s beam.Scope, input beam.PCollection) beam.PCollection { + return filter.Include(s, input, func(element Transaction) bool { + return element.Quantity >= 20 + }) +} + +func mapIdWithPrice(s beam.Scope, input beam.PCollection) beam.PCollection { + return beam.ParDo(s, func(element Transaction, emit func(string, float64)) { + emit(element.ProductID, element.Price) + }, input) +} + +func sumCombine(s beam.Scope, input beam.PCollection) beam.PCollection { + return stats.SumPerKey(s, input) +} diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/hint1.md b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/hint1.md new file mode 100644 index 0000000000000..692accb622c2e --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/hint1.md @@ -0,0 +1,160 @@ + +{{if (eq .Sdk "go")}} +1. Parse the csv file into a `Transaction` object. + + Extract: `func getTransactions(s beam.Scope, input beam.PCollection) beam.PCollection { + return beam.ParDo(s, func(line string, emit func(transaction Transaction)) { + csv := strings.Split(line, ",") + if csv[0] != "TransactionNo" { + id, _ := strconv.ParseInt(csv[0], 10, 64) + price, _ := strconv.ParseFloat(csv[4], 64) + quantity, _ := strconv.ParseInt(csv[5], 10, 64) + customerID, _ := strconv.ParseInt(csv[6], 10, 64) + emit(Transaction{ + ID: id, + Date: csv[1], + ProductID: csv[2], + ProductName: csv[3], + Price: price, + Quantity: quantity, + CustomerID: customerID, + Country: csv[7], + }) + } + }, input) + }` +2. Add a fixed-window that runs for 30 seconds + + Window: `fixedWindowedItems := beam.WindowInto(s, window.NewFixedWindows(30*time.Second), transactions, + beam.Trigger(trigger), + beam.AllowedLateness(30*time.Minute), + beam.PanesDiscard(), + )`. + + And add a trigger that works after the first element with a delay of 5 seconds. + + Trigger: `trigger := trigger.AfterEndOfWindow(). + EarlyFiring(trigger.AfterProcessingTime(). + PlusDelay(5 * time.Second)). + LateFiring(trigger.Repeat(trigger.AfterCount(1)))` + +3. Filter so that the quantity is higher or equal to 20. + + Filter: `func filtering(s beam.Scope, input beam.PCollection) beam.PCollection { + return filter.Include(s, input, func(element Transaction) bool { + return element.Quantity >= 20 + }) + }` + +4. Divide transactions into parts, the first contains transactions whose **price** is more than 10. And the rest are in the second. + + Partition: `func getPartition(s beam.Scope, input beam.PCollection) []beam.PCollection { + return beam.Partition(s, 2, func(element Transaction) int { + if element.Price >= 10 { + return 0 + } + return 1 + }, input) + }` + +5. Create a map function `func mapIdWithPrice(s beam.Scope, input beam.PCollection) beam.PCollection { + return beam.ParDo(s, func(element Transaction, emit func(int64, float64)) { + emit(element.ID, element.Price) + }, input) + }` for group + +6. Combine by key, the function that summarizes the prices `func sumCombine(s beam.Scope, input beam.PCollection) beam.PCollection { + return stats.SumPerKey(s, input) + }` + +7. To write to a file, first you need to convert to a string `func convertToString(s beam.Scope, input beam.PCollection) beam.PCollection { + return beam.ParDo(s, func(id int64, sum float64, emit func(string)) { + emit(fmt.Sprint("id: ", id, " , sum: ", sum)) + }, input) + }` + +8. Write to a txt file: `textio.Write(s, "smallerThan10.txt", smallerThan10)` + +{{end}} + +{{if (eq .Sdk "java")}} +1. Parse the csv file into a `Transaction` object. + + Extract: `static class ExtractDataFn extends DoFn { + @ProcessElement + public void processElement(ProcessContext c) { + String[] items = c.element().split(REGEX_FOR_CSV); + try { + c.output(new Transaction(Long.valueOf(items[0]), items[1], items[2], items[3], Double.valueOf(items[4]), Integer.parseInt(items[5]), Long.valueOf(items[6]), items[7])); + } catch (Exception e) { + System.out.println("Skip header"); + } + } + }` +2. Add a fixed-window that runs for 30 seconds `Window.into(Fixed Windows.of(Duration.standard Seconds(30)))`. And add a trigger that works after the first element with a delay of 5 seconds `AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.standardSeconds(5))` +3. Filter so that the number is higher or equal to 20 `Filter.by(it -> it.quantity >= 20)` +4. Divide transactions into parts, the first contains transactions whose **price** is more than 10. And the rest are in the second. + + Partition: `static class TransactionPartitionFn extends PTransform, PCollectionList> { + @Override + public PCollectionList expand(PCollection input) { + return input.apply(Partition.of(2, + (Partition.PartitionFn) (transaction, numPartitions) -> { + if (transaction.price > 10) { + return 0; + } else { + return 1; + } + })); + } + }` +5. Create a map function `MapElements.into(TypeDescriptors.kvs(TypeDescriptors.longs(), TypeDescriptors.doubles())).via(it->KV.of(it.id,it.price))` for group +6. Combine by key, the function that summarizes the prices `Combine.perKey(new SumDoubleBinaryCombineFn())`. + + Sum function: `static class SumDoubleBinaryCombineFn extends Combine.BinaryCombineFn { + @Override + public Double apply(Double left, Double right) { + return left + right; + } + } + ` +7. To write to a file, first you need to convert to a string `.apply(MapElements.into(TypeDescriptor.of(String.class)).via(it -> it.toString()))` + +8. Write to a txt file: `TextIO.write().to("biggerThan10").withSuffix(".txt")` +{{end}} + +{{if (eq .Sdk "python")}} +1. Parse the csv file into a `Transaction` object. + + Extract: `class ExtractDataFn(beam.DoFn): + def process(self, element): + items = re.split(r',(?=(?:[^"]*"[^"]*")*[^"]*$)', element) + if items[0] != 'TransactionNo': + yield Transaction(items[0], items[1], items[2], items[3], items[4], items[5], items[6], items[7])` + +2. Add a fixed-window that runs for 30 seconds. And add a trigger that works after the first element with a delay of 5 seconds. + + Window and trigger: `windowed_transactions = (transactions + | 'Window' >> beam.WindowInto(window.FixedWindows(30), trigger=trigger.AfterWatermark( + early=trigger.AfterProcessingTime(5).has_ontime_pane(), late=trigger.AfterAll()), + allowed_lateness=30, + accumulation_mode=trigger.AccumulationMode.DISCARDING))`. + +3. Filter so that the number is higher or equal to 20 `'Filtering' >> beam.Filter(lambda t: int(t.quantity) >= 20)` +4. Divide transactions into parts, the first contains transactions whose **price** is more than 10. And the rest are in the second. `'Partition transactions' >> beam.Partition(partitionTransactions, 2))` +5. Create a map function `'Map id and price for bigger' >> beam.Map(lambda transaction: (transaction.transaction_no, float(transaction.price)))` for group +6. Combine by key, the function that summarizes the prices. + Sum function: `'Calculate sum for biggerThan10' >> beam.CombinePerKey(sum)` + +7. Write to a txt file: `'Write biggerThan10 results to text file' >> beam.io.WriteToText('biggerThan10', '.txt', shard_name_template=''))` +{{end}} \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/java-challenge/Task.java b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/java-challenge/Task.java new file mode 100644 index 0000000000000..9286cd89f6cf8 --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/java-challenge/Task.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// beam-playground: +// name: FinalChallenge1 +// description: Final challenge 1. +// multifile: true +// files: +// - name: input.csv +// context_line: 50 +// categories: +// - Quickstart +// complexity: ADVANCED +// tags: +// - hellobeam + + +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.io.TextIO; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.schemas.JavaFieldSchema; +import org.apache.beam.sdk.schemas.annotations.DefaultSchema; +import org.apache.beam.sdk.schemas.annotations.SchemaCreate; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.values.PCollection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Task { + private static final Logger LOG = LoggerFactory.getLogger(Task.class); + private static final Integer WINDOW_TIME = 30; + private static final Integer TIME_OUTPUT_AFTER_FIRST_ELEMENT = 5; + private static final Integer ALLOWED_LATENESS_TIME = 1; + + private static final String REGEX_FOR_CSV = ",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)"; + + public static void main(String[] args) { + LOG.info("Running Task"); + PipelineOptions options = PipelineOptionsFactory.fromArgs(args).create(); + Pipeline pipeline = Pipeline.create(options); + + PCollection input = pipeline.apply(TextIO.read().from("input.csv")); + + pipeline.run().waitUntilFinish(); + } + + static class LogOutput extends DoFn { + private final String prefix; + + LogOutput() { + this.prefix = "Processing element"; + } + + LogOutput(String prefix) { + this.prefix = prefix; + } + + @DoFn.ProcessElement + public void processElement(ProcessContext c) throws Exception { + LOG.info(prefix + ": {}", c.element()); + } + } + + @DefaultSchema(JavaFieldSchema.class) + public static class Transaction { + public Long id; + public String date; + public String productId; + public String productName; + public Double price; + public Integer quantity; + public Long customerId; + public String country; + + public Transaction() { + } + + @SchemaCreate + public Transaction(Long id, String date, String productId, String productName, Double price, Integer quantity, Long customerId, String country) { + this.id = id; + this.date = date; + this.productId = productId; + this.productName = productName; + this.price = price; + this.quantity = quantity; + this.customerId = customerId; + this.country = country; + } + + @Override + public String toString() { + return "Transaction{" + + "id=" + id + + ", date='" + date + '\'' + + ", productId='" + productId + '\'' + + ", productName='" + productName + '\'' + + ", price='" + price + '\'' + + ", quantity=" + quantity + + ", customerId=" + customerId + + ", country='" + country + '\'' + + '}'; + } + } +} \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/java-challenge/input.csv b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/java-challenge/input.csv new file mode 100644 index 0000000000000..85854a2d43584 --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/java-challenge/input.csv @@ -0,0 +1,1500 @@ +TransactionNo,Date,ProductNo,ProductName,Price,Quantity,CustomerNo,Country +581482,12/9/2019,22485,Set Of 2 Wooden Market Crates,21.47,12,17490,United Kingdom +581475,12/9/2019,22596,Christmas Star Wish List Chalkboard,10.65,36,13069,United Ki +581475,12/9/2019,23235,Storage Tin Vintage Leaf,11.53,12,13069,United Kingdom +581475,12/9/2019,23272,Tree T-Light Holder Willie Winkie,10.65,12,13069,United King +581475,12/9/2019,23239,Set Of 4 Knick Knack Tins Poppies,11.94,6,13069,United Kingd +581475,12/9/2019,21705,Bag 500g Swirly Marbles,10.65,24,13069,United Kingdom +581475,12/9/2019,22118,Joy Wooden Block Letters,11.53,18,13069,United Kingdom +581475,12/9/2019,22119,Peace Wooden Block Letters,12.25,12,13069,United Kingdom +581475,12/9/2019,22217,T-Light Holder Hanging Lace,10.65,12,13069,United Kingdom +581475,12/9/2019,22216,T-Light Holder White Lace,10.55,24,13069,United Kingdom +581475,12/9/2019,22380,Toy Tidy Spaceboy,11.06,20,13069,United Kingdom +581475,12/9/2019,22442,Grow Your Own Flowers Set Of 3,12.25,12,13069,United Kingdom +581475,12/9/2019,22664,Toy Tidy Dolly Girl Design,11.06,20,13069,United Kingdom +581475,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,12.25,12,13069,United Kingdom +581475,12/9/2019,22723,Set Of 6 Herb Tins Sketchbook,11.53,12,13069,United Kingdom +581475,12/9/2019,22785,Squarecushion Cover Pink Union Jack,11.53,12,13069,United Ki +581475,12/9/2019,22955,36 Foil Star Cake Cases,11.06,24,13069,United Kingdom +581475,12/9/2019,23141,Triple Wire Hook Pink Heart,11.06,12,13069,United Kingdom +581475,12/9/2019,22956,36 Foil Heart Cake Cases,11.06,24,13069,United Kingdom +581475,12/9/2019,22581,Wood Stocking Christmas Scandispot,10.55,48,13069,United Kin +581476,12/9/2019,23198,Pantry Magnetic Shopping List,11.53,48,12433,Norway +581476,12/9/2019,23197,Sketchbook Magnetic Shopping List,11.74,24,12433,Norway +581476,12/9/2019,23184,Bull Dog Bottle Opener,15.32,8,12433,Norway +581476,12/9/2019,23168,Classic Cafe Sugar Dispenser,11.53,12,12433,Norway +581476,12/9/2019,23167,Small Ceramic Top Storage Jar,10.96,96,12433,Norway +581476,12/9/2019,23166,Medium Ceramic Top Storage Jar,11.32,48,12433,Norway +581476,12/9/2019,23165,Large Ceramic Top Storage Jar,11.74,48,12433,Norway +581476,12/9/2019,23004,Travel Card Wallet Pantry,10.68,48,12433,Norway +581476,12/9/2019,23002,Travel Card Wallet Skulls,10.68,24,12433,Norway +581476,12/9/2019,23000,Travel Card Wallet Transport,10.68,24,12433,Norway +581476,12/9/2019,22998,Travel Card Wallet Keep Calm,10.68,72,12433,Norway +581476,12/9/2019,22994,Travel Card Wallet Retrospot,10.68,48,12433,Norway +581476,12/9/2019,22835,Hot Water Bottle I Am So Poorly,15.32,8,12433,Norway +581476,12/9/2019,22730,Alarm Clock Bakelike Ivory,14.09,4,12433,Norway +581476,12/9/2019,22728,Alarm Clock Bakelike Pink,14.09,8,12433,Norway +581476,12/9/2019,22727,Alarm Clock Bakelike Red,14.09,8,12433,Norway +581476,12/9/2019,22726,Alarm Clock Bakelike Green,14.09,4,12433,Norway +581476,12/9/2019,22720,Set Of 3 Cake Tins Pantry Design,14.61,16,12433,Norway +581476,12/9/2019,22693,Grow A Flytrap Or Sunflower In Tin,11.34,192,12433,Norway +581476,12/9/2019,22670,French Wc Sign Blue Metal,11.53,24,12433,Norway +581476,12/9/2019,22667,Recipe Box Retrospot,12.86,24,12433,Norway +581476,12/9/2019,22666,Recipe Box Pantry Yellow Design,12.86,24,12433,Norway +581476,12/9/2019,22631,Circus Parade Lunch Box,12.25,12,12433,Norway +581476,12/9/2019,22628,Picnic Boxes Set Of 3 Retrospot,15.32,16,12433,Norway +581476,12/9/2019,22467,Gumball Coat Rack,12.86,6,12433,Norway +581476,12/9/2019,22197,Popcorn Holder,10.99,100,12433,Norway +581476,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,14.61,8,12433,Norway +581476,12/9/2019,22112,Chocolate Hot Water Bottle,15.32,9,12433,Norway +581476,12/9/2019,21908,Chocolate This Way Metal Sign,12.40,36,12433,Norway +581476,12/9/2019,21874,Gin And Tonic Mug,11.74,36,12433,Norway +581476,12/9/2019,21872,Glamorous Mug,11.53,12,12433,Norway +581476,12/9/2019,21871,Save The Planet Mug,11.74,36,12433,Norway +581476,12/9/2019,21533,Retrospot Large Milk Jug,15.32,6,12433,Norway +581476,12/9/2019,21481,Fawn Blue Hot Water Bottle,14.09,12,12433,Norway +581476,12/9/2019,21479,White Skull Hot Water Bottle,14.61,4,12433,Norway +581476,12/9/2019,21248,Door Hanger Mum + Dads Room,11.74,12,12433,Norway +581476,12/9/2019,21216,Set 3 Retrospot Tea/Coffee/Sugar,14.61,24,12433,Norway +581476,12/9/2019,21181,Please One Person Metal Sign,12.15,48,12433,Norway +581476,12/9/2019,21175,Gin And Tonic Diet Metal Sign,12.38,48,12433,Norway +581476,12/9/2019,21169,You're Confusing Me Metal Sign,11.74,48,12433,Norway +581476,12/9/2019,21162,Toxic Area Door Hanger,10.65,24,12433,Norway +581476,12/9/2019,21159,Moody Boy Door Hanger,10.65,24,12433,Norway +581476,12/9/2019,21158,Moody Girl Door Hanger,10.65,24,12433,Norway +581476,12/9/2019,21154,Red Retrospot Oven Glove,11.53,10,12433,Norway +581476,12/9/2019,16016,Large Chinese Style Scissor,11.12,40,12433,Norway +581476,12/9/2019,16014,Small Chinese Style Scissor,10.68,60,12433,Norway +581476,12/9/2019,16008,Small Folding Scissor(Pointed Edge),10.37,240,12433,Norway +581476,12/9/2019,85152,Hand Over The Chocolate Sign,12.15,48,12433,Norway +581476,12/9/2019,84596F,Small Marshmallows Pink Bowl,10.68,32,12433,Norway +581476,12/9/2019,84596B,Small Dolly Mix Design Orange Bowl,10.68,16,12433,Norway +581476,12/9/2019,84510A,Set Of 4 English Rose Coasters,11.53,20,12433,Norway +581476,12/9/2019,82600,N0 Singing Metal Sign,12.15,48,12433,Norway +581476,12/9/2019,82581,Toilet Metal Sign,10.81,48,12433,Norway +581476,12/9/2019,72232,Feng Shui Pillar Candle,10.44,144,12433,Norway +581476,12/9/2019,47559B,Tea Time Oven Glove,11.53,10,12433,Norway +581476,12/9/2019,47504H,English Rose Spirit Level,11.06,36,12433,Norway +581476,12/9/2019,23493,Vintage Doily Travel Sewing Kit,12.25,30,12433,Norway +581476,12/9/2019,23430,Blue Retro Kitchen Wall Clock,18.60,2,12433,Norway +581476,12/9/2019,23429,Red Retro Kitchen Wall Clock,18.60,2,12433,Norway +581476,12/9/2019,23428,Ivory Retro Kitchen Wall Clock,18.60,2,12433,Norway +581476,12/9/2019,23358,Hot Stuff Hot Water Bottle,11.53,18,12433,Norway +581476,12/9/2019,23293,Set Of 12 Fairy Cake Baking Cases,11.10,32,12433,Norway +581476,12/9/2019,23243,Set Of Tea Coffee Sugar Tins Pantry,15.32,12,12433,Norway +581476,12/9/2019,23240,Set Of 4 Knick Knack Tins Doily,14.50,6,12433,Norway +581477,12/9/2019,48111,Doormat 3 Smiley Cats,17.51,10,13426,United Kingdom +581477,12/9/2019,22464,Hanging Metal Heart Lantern,11.06,12,13426,United Kingdom +581477,12/9/2019,20982,12 Pencils Tall Tube Skulls,11.12,12,13426,United Kingdom +581477,12/9/2019,20981,12 Pencils Tall Tube Woodland,11.12,12,13426,United Kingdom +581477,12/9/2019,23424,Gingham Recipe Book Box,15.32,8,13426,United Kingdom +581477,12/9/2019,23338,Egg Frying Pan Red,12.15,48,13426,United Kingdom +581477,12/9/2019,84970L,Single Heart Zinc T-Light Holder,11.53,12,13426,United King +581477,12/9/2019,22457,Natural Slate Heart Chalkboard,13.27,6,13426,United Kingdom +581477,12/9/2019,22469,Heart Of Wicker Small,11.94,12,13426,United Kingdom +581477,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,11.12,12,13426,United Ki +581478,12/9/2019,84947,Antique Silver Tea Glass Engraved,11.53,24,17364,United King +581478,12/9/2019,23503,Playing Cards Keep Calm & Carry On,11.53,12,17364,United Kin +581478,12/9/2019,23445,Ice Cream Bubbles,11.10,20,17364,United Kingdom +581478,12/9/2019,23530,Wall Art Only One Person,15.32,12,17364,United Kingdom +581478,12/9/2019,84997C,Childrens Cutlery Polkadot Blue,14.50,4,17364,United Kingdo +581478,12/9/2019,84997D,Childrens Cutlery Polkadot Pink,14.50,4,17364,United Kingdo +581478,12/9/2019,84077,World War 2 Gliders Asstd Designs,10.55,48,17364,United King +581478,12/9/2019,22749,Feltcraft Princess Charlotte Doll,14.09,4,17364,United Kingd +581478,12/9/2019,23127,Feltcraft Girl Nicole Kit,15.32,4,17364,United Kingdom +581478,12/9/2019,23126,Feltcraft Girl Amelie Kit,15.32,4,17364,United Kingdom +581478,12/9/2019,22747,Poppy's Playhouse Bathroom,12.40,6,17364,United Kingdom +581478,12/9/2019,22078,Ribbon Reel Lace Design,12.40,10,17364,United Kingdom +581478,12/9/2019,84946,Antique Silver T-Light Glass,11.53,12,17364,United Kingdom +581478,12/9/2019,22791,T-Light Glass Fluted Antique,11.53,12,17364,United Kingdom +581478,12/9/2019,21326,Aged Glass Silver T-Light Holder,10.92,12,17364,United Kingd +581478,12/9/2019,22170,Picture Frame Wood Triple Portrait,17.17,8,17364,United King +581478,12/9/2019,23082,Set 6 Paper Table Lantern Hearts,14.09,6,17364,United Kingdo +581478,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,11.12,24,17364,United Ki +581479,12/9/2019,22087,Paper Bunting White Lace,13.27,10,17364,United Kingdom +581480,12/9/2019,23464,Vintage Zinc Watering Can Small,15.32,4,14441,United Kingdom +581480,12/9/2019,22411,Jumbo Shopper Vintage Red Paisley,12.38,10,14441,United King +581480,12/9/2019,84029E,Red Woolly Hottie White Heart,14.61,8,14441,United Kingdom +581480,12/9/2019,22633,Hand Warmer Union Jack,12.40,12,14441,United Kingdom +581480,12/9/2019,23355,Hot Water Bottle Keep Calm,15.32,12,14441,United Kingdom +581480,12/9/2019,84029G,Knitted Union Flag Hot Water Bottle,14.61,12,14441,United K +581480,12/9/2019,22112,Chocolate Hot Water Bottle,15.32,6,14441,United Kingdom +581480,12/9/2019,84347,Rotating Silver Angels T-Light Hldr,12.86,18,14441,United Ki +581481,12/9/2019,21115,Rose Caravan Doorstop,12.25,8,17490,United Kingdom +581481,12/9/2019,22059,Ceramic Strawberry Design Mug,10.65,24,17490,United Kingdom +581481,12/9/2019,22072,Red Retrospot Tea Cup And Saucer,11.53,24,17490,United Kingd +581481,12/9/2019,22123,Ping Microwave Apron,11.06,24,17490,United Kingdom +581481,12/9/2019,22476,Empire Union Jack Tv Dinner Tray,12.25,8,17490,United Kingdo +581481,12/9/2019,22495,Set Of 2 Round Tins Camembert,11.06,12,17490,United Kingdom +581481,12/9/2019,22496,Set Of 2 Round Tins Dutch Cheese,11.06,12,17490,United Kingd +581481,12/9/2019,22513,Doorstop Football Design,11.06,8,17490,United Kingdom +581481,12/9/2019,22500,Set Of 2 Tins Jardin De Provence,11.53,12,17490,United Kingd +581481,12/9/2019,22664,Toy Tidy Dolly Girl Design,11.06,20,17490,United Kingdom +581481,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,12.25,12,17490,United Kingdom +581481,12/9/2019,22785,Squarecushion Cover Pink Union Jack,11.53,12,17490,United Ki +581481,12/9/2019,23178,Jam Clock Magnet,11.53,12,17490,United Kingdom +581481,12/9/2019,23302,Kneeling Mat Housework Design,11.06,24,17490,United Kingdom +581481,12/9/2019,23533,Wall Art Garden Haven,12.25,6,17490,United Kingdom +581481,12/9/2019,84819,Danish Rose Round Sewing Box,11.06,16,17490,United Kingdom +581482,12/9/2019,22371,Airline Bag Vintage Tokyo 78,14.30,12,17490,United Kingdom +581482,12/9/2019,21875,Kings Choice Mug,11.34,36,17490,United Kingdom +581482,12/9/2019,23251,Vintage Red Enamel Trim Mug,11.32,96,17490,United Kingdom +581482,12/9/2019,23300,Gardeners Kneeling Pad Cup Of Tea,11.74,48,17490,United King +581482,12/9/2019,22138,Baking Set 9 Piece Retrospot,14.61,24,17490,United Kingdom +581483,12/9/2019,23843,Paper Craft Little Birdie,12.38,80995,16446,United Kingdom +581485,12/9/2019,22617,Baking Set Spaceboy Design,15.32,18,17389,United Kingdom +581485,12/9/2019,20749,Assorted Colour Mini Cases,16.76,84,17389,United Kingdom +581486,12/9/2019,22910,Paper Chain Kit Vintage Christmas,13.27,12,17001,United King +581486,12/9/2019,22086,Paper Chain Kit 50'S Christmas,13.27,12,17001,United Kingdom +581486,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,12,17001,United Kingdom +581486,12/9/2019,23352,Roll Wrap 50'S Red Christmas,6.19,12,17001,United Kingdom +581486,12/9/2019,22727,Alarm Clock Bakelike Red,6.19,8,17001,United Kingdom +581486,12/9/2019,22730,Alarm Clock Bakelike Ivory,6.19,4,17001,United Kingdom +581486,12/9/2019,22491,Pack Of 12 Coloured Pencils,6.04,12,17001,United Kingdom +581486,12/9/2019,22561,Wooden School Colouring Set,6.04,12,17001,United Kingdom +581486,12/9/2019,22489,Pack Of 12 Traditional Crayons,6.04,24,17001,United Kingdom +581486,12/9/2019,22560,Traditional Modelling Clay,6.04,24,17001,United Kingdom +581486,12/9/2019,22139,Retrospot Tea Set Ceramic 11 Pc,6.04,6,17001,United Kingdom +581486,12/9/2019,23344,Jumbo Bag 50'S Christmas,6.19,20,17001,United Kingdom +581486,12/9/2019,23203,Jumbo Bag Vintage Doily,6.19,20,17001,United Kingdom +581486,12/9/2019,85099B,Jumbo Bag Red Retrospot,6.19,10,17001,United Kingdom +581486,12/9/2019,23201,Jumbo Bag Alphabet,6.19,20,17001,United Kingdom +581486,12/9/2019,23207,Lunch Bag Alphabet Design,6.19,10,17001,United Kingdom +581487,12/9/2019,21137,Black Record Cover Frame,6.04,120,15694,United Kingdom +581488,12/9/2019,23118,Parisienne Jewellery Drawer,6.04,16,17428,United Kingdom +581488,12/9/2019,23111,Parisienne Sewing Box,6.04,16,17428,United Kingdom +581488,12/9/2019,22179,Set 10 Night Owl Lights,6.04,24,17428,United Kingdom +581489,12/9/2019,22061,Large Cake Stand Hanging Strawbery,7.24,48,16954,United King +581489,12/9/2019,22182,Cake Stand Victorian Filigree Small,7.24,24,16954,United Kin +581491,12/9/2019,23571,Traditional Naughts & Crosses,7.24,12,12433,Norway +581492,12/9/2019,23369,Set 36 Colour Pencils Love London,6.19,2,15492,United Kingdo +581492,12/9/2019,23370,Set 36 Colouring Pencils Doily,6.19,2,15492,United Kingdom +581492,12/9/2019,23371,Set 36 Colour Pencils Spaceboy,6.19,3,15492,United Kingdom +581492,12/9/2019,23372,Set 36 Colour Pencils Dolly Girl,6.19,1,15492,United Kingdom +581492,12/9/2019,23376,Pack Of 12 Vintage Christmas Tissue,6.19,3,15492,United King +581492,12/9/2019,23377,Pack Of 12 Dolly Girl Tissues,6.19,6,15492,United Kingdom +581492,12/9/2019,23378,Pack Of 12 50'S Christmas Tissues,6.19,9,15492,United Kingdo +581492,12/9/2019,23379,Pack Of 12 Red Apple Tissues,6.19,4,15492,United Kingdom +581492,12/9/2019,23380,Pack Of 12 Vintage Doily Tissues,6.19,3,15492,United Kingdom +581492,12/9/2019,23381,Pack Of 12 Vintage Leaf Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,23382,Box Of 6 Christmas Cake Decorations,6.19,4,15492,United King +581492,12/9/2019,23391,I Love London Mini Backpack,6.19,2,15492,United Kingdom +581492,12/9/2019,23392,Spaceboy Rocket Lolly Makers,6.19,2,15492,United Kingdom +581492,12/9/2019,23399,Home Sweet Home Hanging Heart,6.19,4,15492,United Kingdom +581492,12/9/2019,23405,Home Sweet Home 2 Drawer Cabinet,6.19,3,15492,United Kingdom +581492,12/9/2019,23418,Lavender Toilette Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,23426,Metal Sign Drop Your Pants,6.19,1,15492,United Kingdom +581492,12/9/2019,23434,3 Raffia Ribbons 50'S Christmas,6.19,9,15492,United Kingdom +581492,12/9/2019,23435,3 Raffia Ribbons Vintage Christmas,6.04,3,15492,United Kingd +581492,12/9/2019,23439,Hand Warmer Red Love Heart,6.19,1,15492,United Kingdom +581492,12/9/2019,23451,Square Mini Portrait Frame,6.19,1,15492,United Kingdom +581492,12/9/2019,23467,Vintage Zinc Planter,6.19,1,15492,United Kingdom +581492,12/9/2019,23469,Card Holder Love Bird Small,6.19,3,15492,United Kingdom +581492,12/9/2019,23480,Mini Lights Woodland Mushrooms,6.19,2,15492,United Kingdom +581492,12/9/2019,23493,Vintage Doily Travel Sewing Kit,6.19,3,15492,United Kingdom +581492,12/9/2019,23494,Vintage Doily Deluxe Sewing Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,23495,Set Of 3 Pantry Wooden Spoons,6.19,1,15492,United Kingdom +581492,12/9/2019,23497,Classic Chrome Bicycle Bell,6.19,4,15492,United Kingdom +581492,12/9/2019,23498,Classic Bicycle Clips,6.19,2,15492,United Kingdom +581492,12/9/2019,23501,Key Ring Baseball Boot Union Jack,6.19,3,15492,United Kingdo +581492,12/9/2019,23506,Mini Playing Cards Spaceboy,6.04,1,15492,United Kingdom +581492,12/9/2019,23508,Mini Playing Cards Dolly Girl,6.04,2,15492,United Kingdom +581492,12/9/2019,23510,Mini Playing Cards Gymkhana,6.04,1,15492,United Kingdom +581492,12/9/2019,23521,Wall Art Cat And Bird,6.04,1,15492,United Kingdom +581492,12/9/2019,23526,Wall Art Dog Licence,6.04,4,15492,United Kingdom +581492,12/9/2019,23530,Wall Art Only One Person,6.04,2,15492,United Kingdom +581492,12/9/2019,23534,Wall Art Stop For Tea,6.19,6,15492,United Kingdom +581492,12/9/2019,23535,Wall Art Bicycle Safety,6.19,4,15492,United Kingdom +581492,12/9/2019,23536,Wall Art Village Show,6.19,1,15492,United Kingdom +581492,12/9/2019,23538,Wall Art Vintage Heart,6.19,1,15492,United Kingdom +581492,12/9/2019,23551,Pack Of 12 Paisley Park Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,23552,Bicycle Puncture Repair Kit,6.19,12,15492,United Kingdom +581492,12/9/2019,23555,Landmark Frame Notting Hill,6.19,1,15492,United Kingdom +581492,12/9/2019,23559,Woodland Bunnies Lolly Makers,6.19,5,15492,United Kingdom +581492,12/9/2019,23561,Set Of 6 Ribbons Party,6.19,1,15492,United Kingdom +581492,12/9/2019,23564,Egg Cup Milkmaid Ingrid,6.19,2,15492,United Kingdom +581492,12/9/2019,23565,Egg Cup Milkmaid Helga,6.19,2,15492,United Kingdom +581492,12/9/2019,23567,Egg Cup Henrietta Hen Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23569,Tradtional Alphabet Stamp Set,6.19,2,15492,United Kingdom +581492,12/9/2019,23570,Traditional Pick Up Sticks Game,6.19,7,15492,United Kingdom +581492,12/9/2019,23571,Traditional Naughts & Crosses,6.19,2,15492,United Kingdom +581492,12/9/2019,23575,Snack Tray Paisley Park,6.19,3,15492,United Kingdom +581492,12/9/2019,23579,Snack Tray I Love London,6.19,1,15492,United Kingdom +581492,12/9/2019,23611,Set 10 Cards Red Riding Hood 17214,6.19,3,15492,United Kingd +581492,12/9/2019,23616,Set 10 Cards Jingle Bells 17217,6.19,1,15492,United Kingdom +581492,12/9/2019,23621,Set 10 Cards David's Madonna 17074,6.19,3,15492,United Kingd +581492,12/9/2019,23635,Set 10 Cards Christmas Holly 17259,6.19,1,15492,United Kingd +581492,12/9/2019,23644,Set 10 Cards Christmas Tree 16955,6.19,1,15492,United Kingdo +581492,12/9/2019,23660,Henrietta Hen Mug,6.19,1,15492,United Kingdom +581492,12/9/2019,23661,Milk Maids Mug,6.19,1,15492,United Kingdom +581492,12/9/2019,35095A,Blue Victorian Fabric Oval Box,6.19,1,15492,United Kingdom +581492,12/9/2019,35095B,Red Victorian Fabric Oval Box,6.19,1,15492,United Kingdom +581492,12/9/2019,35646,Vintage Bead Pink Evening Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,35648,Vintage Bead Pink Purse,6.19,2,15492,United Kingdom +581492,12/9/2019,35953,Folkart Star Christmas Decorations,6.19,12,15492,United King +581492,12/9/2019,35964,Folkart Clip On Stars,6.19,4,15492,United Kingdom +581492,12/9/2019,35970,Zinc Folkart Sleigh Bells,6.04,1,15492,United Kingdom +581492,12/9/2019,46118,Funky Monkey Cushion Cover,6.19,1,15492,United Kingdom +581492,12/9/2019,46776A,Woven Bubble Gum Cushion Cover,7.24,2,15492,United Kingdom +581492,12/9/2019,46776B,Woven Berries Cushion Cover,7.24,3,15492,United Kingdom +581492,12/9/2019,46776C,Woven Frost Cushion Cover,7.24,1,15492,United Kingdom +581492,12/9/2019,46776D,Woven Sunset Cushion Cover,7.24,2,15492,United Kingdom +581492,12/9/2019,46776E,Woven Candy Cushion Cover,7.24,5,15492,United Kingdom +581492,12/9/2019,46776F,Woven Rose Garden Cushion Cover,7.24,6,15492,United Kingdom +581492,12/9/2019,47310M,Small Pop Box Funky Monkey,6.19,1,15492,United Kingdom +581492,12/9/2019,47480,Hanging Photo Clip Rope Ladder,6.19,3,15492,United Kingdom +581492,12/9/2019,47504H,English Rose Spirit Level,7.24,1,15492,United Kingdom +581492,12/9/2019,47559B,Tea Time Oven Glove,7.24,3,15492,United Kingdom +581492,12/9/2019,47563A,Retro Longboard Ironing Board Cover,7.24,2,15492,United Kin +581492,12/9/2019,47566,Party Bunting,7.24,2,15492,United Kingdom +581492,12/9/2019,47590B,Pink Happy Birthday Bunting,7.24,1,15492,United Kingdom +581492,12/9/2019,51014A,Feather Pen Hot Pink,7.24,2,15492,United Kingdom +581492,12/9/2019,51014L,Feather Pen Light Pink,7.24,1,15492,United Kingdom +581492,12/9/2019,71270,Photo Clip Line,7.24,1,15492,United Kingdom +581492,12/9/2019,72349B,Set/6 Purple Butterfly T-Lights,6.39,1,15492,United Kingdom +581492,12/9/2019,72741,Grand Chocolatecandle,7.24,3,15492,United Kingdom +581492,12/9/2019,72800E,4 Ivory Dinner Candles Silver Flock,7.24,3,15492,United Kin +581492,12/9/2019,79321,Chilli Lights,6.19,1,15492,United Kingdom +581492,12/9/2019,82484,Wood Black Board Ant White Finish,6.19,2,15492,United Kingdo +581492,12/9/2019,82567,Airline Lounge Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,82582,Area Patrolled Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,82600,N0 Singing Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,84031A,Charlie+Lola Red Hot Water Bottle,6.19,4,15492,United Kingd +581492,12/9/2019,84077,World War 2 Gliders Asstd Designs,6.19,1,15492,United Kingdo +581492,12/9/2019,84249A,Greeting Card Square Doughnuts,6.19,1,15492,United Kingdom +581492,12/9/2019,84347,Rotating Silver Angels T-Light Hldr,6.19,2,15492,United King +581492,12/9/2019,84375,Set Of 20 Kids Cookie Cutters,6.19,1,15492,United Kingdom +581492,12/9/2019,84378,Set Of 3 Heart Cookie Cutters,6.19,4,15492,United Kingdom +581492,12/9/2019,84519B,Carrot Charlie+Lola Coaster Set,6.19,1,15492,United Kingdom +581492,12/9/2019,84559A,3d Sheet Of Dog Stickers,6.19,2,15492,United Kingdom +581492,12/9/2019,84568,Girls Alphabet Iron On Patches,6.19,31,15492,United Kingdom +581492,12/9/2019,84569D,Pack 6 Heart/Ice-Cream Patches,6.19,1,15492,United Kingdom +581492,12/9/2019,84596B,Small Dolly Mix Design Orange Bowl,6.19,4,15492,United King +581492,12/9/2019,84596F,Small Marshmallows Pink Bowl,6.19,4,15492,United Kingdom +581492,12/9/2019,84598,Boys Alphabet Iron On Patches,6.19,5,15492,United Kingdom +581492,12/9/2019,84692,Box Of 24 Cocktail Parasols,6.19,1,15492,United Kingdom +581492,12/9/2019,84755,Colour Glass T-Light Holder Hanging,6.19,4,15492,United King +581492,12/9/2019,84828,Jungle Popsicles Ice Lolly Moulds,6.19,1,15492,United Kingdo +581492,12/9/2019,84879,Assorted Colour Bird Ornament,6.19,16,15492,United Kingdom +581492,12/9/2019,84923,Pink Butterfly Handbag W Bobbles,6.04,3,15492,United Kingdom +581492,12/9/2019,84946,Antique Silver T-Light Glass,6.04,18,15492,United Kingdom +581492,12/9/2019,84970S,Hanging Heart Zinc T-Light Holder,6.04,3,15492,United Kingd +581492,12/9/2019,84978,Hanging Heart Jar T-Light Holder,6.04,4,15492,United Kingdom +581492,12/9/2019,84991,60 Teatime Fairy Cake Cases,6.04,1,15492,United Kingdom +581492,12/9/2019,84997A,Childrens Cutlery Polkadot Green,6.04,1,15492,United Kingdo +581492,12/9/2019,84997B,Childrens Cutlery Retrospot Red,6.04,1,15492,United Kingdom +581492,12/9/2019,20733,Gold Mini Tape Measure,6.04,3,15492,United Kingdom +581492,12/9/2019,20777,Chrysanthemum Notebook,6.19,2,15492,United Kingdom +581492,12/9/2019,20782,Camouflage Ear Muff Headphones,6.19,1,15492,United Kingdom +581492,12/9/2019,20914,Set/5 Red Retrospot Lid Glass Bowls,6.19,2,15492,United King +581492,12/9/2019,20931,Blue Pot Plant Candle,6.19,1,15492,United Kingdom +581492,12/9/2019,20970,Pink Floral Feltcraft Shoulder Bag,6.19,1,15492,United Kingd +581492,12/9/2019,20971,Pink Blue Felt Craft Trinket Box,6.19,2,15492,United Kingdom +581492,12/9/2019,20972,Pink Cream Felt Craft Trinket Box,6.19,3,15492,United Kingdo +581492,12/9/2019,20973,12 Pencil Small Tube Woodland,6.19,4,15492,United Kingdom +581492,12/9/2019,20978,36 Pencils Tube Skulls,6.19,1,15492,United Kingdom +581492,12/9/2019,20979,36 Pencils Tube Red Retrospot,6.19,3,15492,United Kingdom +581492,12/9/2019,20982,12 Pencils Tall Tube Skulls,6.19,2,15492,United Kingdom +581492,12/9/2019,20983,12 Pencils Tall Tube Red Retrospot,6.19,4,15492,United Kingd +581492,12/9/2019,20985,Heart Calculator,6.19,1,15492,United Kingdom +581492,12/9/2019,20986,Blue Calculator Ruler,6.19,1,15492,United Kingdom +581492,12/9/2019,21000,Rose Du Sud Cosmetics Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,21012,Antique All Glass Candlestick,6.19,3,15492,United Kingdom +581492,12/9/2019,21015,Dark Bird House Tree Decoration,6.19,8,15492,United Kingdom +581492,12/9/2019,21026,Space Owl,6.19,1,15492,United Kingdom +581492,12/9/2019,21035,Set/2 Red Retrospot Tea Towels,6.19,1,15492,United Kingdom +581492,12/9/2019,21064,Boom Box Speaker Boys,6.19,4,15492,United Kingdom +581492,12/9/2019,21065,Boom Box Speaker Girls,6.19,2,15492,United Kingdom +581492,12/9/2019,21098,Christmas Toilet Roll,6.19,1,15492,United Kingdom +581492,12/9/2019,21123,Set/10 Ivory Polkadot Party Candles,6.19,1,15492,United King +581492,12/9/2019,21125,Set 6 Football Celebration Candles,6.19,1,15492,United Kingd +581492,12/9/2019,21126,Set Of 6 Girls Celebration Candles,6.19,1,15492,United Kingd +581492,12/9/2019,21137,Black Record Cover Frame,6.19,17,15492,United Kingdom +581492,12/9/2019,21154,Red Retrospot Oven Glove,6.19,2,15492,United Kingdom +581492,12/9/2019,21158,Moody Girl Door Hanger,6.19,1,15492,United Kingdom +581492,12/9/2019,21162,Toxic Area Door Hanger,6.19,1,15492,United Kingdom +581492,12/9/2019,21166,Cook With Wine Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,21172,Party Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,21174,Pottering In The Shed Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,21175,Gin And Tonic Diet Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,21200,Multicolour Honeycomb Paper Garland,6.04,6,15492,United King +581492,12/9/2019,21201,Tropical Honeycomb Paper Garland,6.04,4,15492,United Kingdom +581492,12/9/2019,21212,Pack Of 72 Retrospot Cake Cases,6.04,2,15492,United Kingdom +581492,12/9/2019,21213,Pack Of 72 Skull Cake Cases,6.04,2,15492,United Kingdom +581492,12/9/2019,21220,Set/4 Badges Dogs,6.19,2,15492,United Kingdom +581492,12/9/2019,21224,Set/4 Skull Badges,6.19,1,15492,United Kingdom +581492,12/9/2019,21232,Strawberry Ceramic Trinket Pot,6.19,1,15492,United Kingdom +581492,12/9/2019,21257,Victorian Sewing Box Medium,6.19,2,15492,United Kingdom +581492,12/9/2019,21258,Victorian Sewing Box Large,6.19,1,15492,United Kingdom +581492,12/9/2019,21259,Victorian Sewing Box Small,6.19,1,15492,United Kingdom +581492,12/9/2019,21313,Glass Heart T-Light Holder,6.19,1,15492,United Kingdom +581492,12/9/2019,21314,Small Glass Heart Trinket Pot,6.19,2,15492,United Kingdom +581492,12/9/2019,21328,Balloons Writing Set,6.19,2,15492,United Kingdom +581492,12/9/2019,21329,Dinosaurs Writing Set,6.19,2,15492,United Kingdom +581492,12/9/2019,21356,Toast Its - Fairy Flower,6.19,16,15492,United Kingdom +581492,12/9/2019,21378,Small Tall Camphor Wood Toadstool,6.19,2,15492,United Kingdo +581492,12/9/2019,21379,Camphor Wood Portobello Mushroom,6.19,1,15492,United Kingdom +581492,12/9/2019,21383,Pack Of 12 Sticky Bunnies,6.19,1,15492,United Kingdom +581492,12/9/2019,21402,Red Egg Spoon,6.19,1,15492,United Kingdom +581492,12/9/2019,21408,Spotty Pink Duck Doorstop,6.19,2,15492,United Kingdom +581492,12/9/2019,21411,Gingham Heart Doorstop Red,6.19,1,15492,United Kingdom +581492,12/9/2019,21467,Cherry Crochet Food Cover,6.19,1,15492,United Kingdom +581492,12/9/2019,21471,Strawberry Raffia Food Cover,6.19,2,15492,United Kingdom +581492,12/9/2019,21479,White Skull Hot Water Bottle,6.19,2,15492,United Kingdom +581492,12/9/2019,21481,Fawn Blue Hot Water Bottle,6.19,3,15492,United Kingdom +581492,12/9/2019,21484,Chick Grey Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,21485,Retrospot Heart Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,21494,Rotating Leaves T-Light Holder,6.19,11,15492,United Kingdom +581492,12/9/2019,21506,Fancy Font Birthday Card,6.19,3,15492,United Kingdom +581492,12/9/2019,21507,Elephant Birthday Card,7.24,1,15492,United Kingdom +581492,12/9/2019,21508,Vintage Kid Dolly Card,7.24,1,15492,United Kingdom +581492,12/9/2019,21509,Cowboys And Indians Birthday Card,7.24,2,15492,United Kingdo +581492,12/9/2019,21528,Dairy Maid Traditional Teapot,7.24,1,15492,United Kingdom +581492,12/9/2019,21544,Skulls Water Transfer Tattoos,7.24,2,15492,United Kingdom +581492,12/9/2019,21558,Skull Lunch Box With Cutlery,6.39,1,15492,United Kingdom +581492,12/9/2019,21559,Strawberry Lunch Box With Cutlery,7.24,1,15492,United Kingdo +581492,12/9/2019,21615,4 Lavender Botanical Dinner Candles,7.24,2,15492,United King +581492,12/9/2019,21616,4 Pear Botanical Dinner Candles,7.24,6,15492,United Kingdom +581492,12/9/2019,21620,Set Of 4 Rose Botanical Candles,7.24,5,15492,United Kingdom +581492,12/9/2019,21642,Assorted Tutti Frutti Pen,7.24,4,15492,United Kingdom +581492,12/9/2019,21648,Assorted Tutti Frutti Small Purse,7.24,2,15492,United Kingdo +581492,12/9/2019,21650,Assorted Tutti Frutti Bracelet,7.24,14,15492,United Kingdom +581492,12/9/2019,21675,Butterflies Stickers,6.19,2,15492,United Kingdom +581492,12/9/2019,21677,Hearts Stickers,6.19,1,15492,United Kingdom +581492,12/9/2019,21678,Paisley Pattern Stickers,6.19,2,15492,United Kingdom +581492,12/9/2019,21680,Woodland Stickers,6.19,1,15492,United Kingdom +581492,12/9/2019,21703,Bag 125g Swirly Marbles,6.19,1,15492,United Kingdom +581492,12/9/2019,21704,Bag 250g Swirly Marbles,6.19,1,15492,United Kingdom +581492,12/9/2019,21716,Boys Vintage Tin Seaside Bucket,6.19,1,15492,United Kingdom +581492,12/9/2019,21718,Red Metal Beach Spade,6.19,1,15492,United Kingdom +581492,12/9/2019,21724,Panda And Bunnies Sticker Sheet,6.19,1,15492,United Kingdom +581492,12/9/2019,21731,Red Toadstool Led Night Light,6.19,6,15492,United Kingdom +581492,12/9/2019,21739,Cosy Slipper Shoes Small Green,6.19,2,15492,United Kingdom +581492,12/9/2019,21774,Decorative Cats Bathroom Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,21786,Polkadot Rain Hat,6.19,3,15492,United Kingdom +581492,12/9/2019,21787,Rain Poncho Retrospot,6.19,2,15492,United Kingdom +581492,12/9/2019,21790,Vintage Snap Cards,6.19,5,15492,United Kingdom +581492,12/9/2019,21791,Vintage Heads And Tails Card Game,6.19,1,15492,United Kingdo +581492,12/9/2019,21808,Christmas Garland Stars Trees,6.19,3,15492,United Kingdom +581492,12/9/2019,21810,Christmas Hanging Star With Bell,6.19,25,15492,United Kingdo +581492,12/9/2019,21812,Garland With Hearts And Bells,6.19,7,15492,United Kingdom +581492,12/9/2019,21813,Garland With Stars And Bells,6.19,3,15492,United Kingdom +581492,12/9/2019,21822,Glitter Christmas Tree With Bells,6.19,12,15492,United Kingd +581492,12/9/2019,21828,Eight Piece Snake Set,6.19,1,15492,United Kingdom +581492,12/9/2019,21832,Chocolate Calculator,6.19,1,15492,United Kingdom +581492,12/9/2019,21833,Camouflage Led Torch,6.04,2,15492,United Kingdom +581492,12/9/2019,21871,Save The Planet Mug,6.19,1,15492,United Kingdom +581492,12/9/2019,21874,Gin And Tonic Mug,6.19,2,15492,United Kingdom +581492,12/9/2019,21876,Pottering Mug,6.19,4,15492,United Kingdom +581492,12/9/2019,21879,Hearts Gift Tape,6.19,1,15492,United Kingdom +581492,12/9/2019,21889,Wooden Box Of Dominoes,6.19,3,15492,United Kingdom +581492,12/9/2019,21890,S/6 Wooden Skittles In Cotton Bag,6.19,1,15492,United Kingdo +581492,12/9/2019,21892,Traditional Wooden Catch Cup Game,6.19,1,15492,United Kingdo +581492,12/9/2019,21900,Key Fob Shed,6.19,4,15492,United Kingdom +581492,12/9/2019,21901,Key Fob Back Door,6.19,2,15492,United Kingdom +581492,12/9/2019,21902,Key Fob Front Door,6.19,1,15492,United Kingdom +581492,12/9/2019,21905,More Butter Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,21906,Pharmacie First Aid Tin,6.19,1,15492,United Kingdom +581492,12/9/2019,21914,Blue Harmonica In Box,6.19,2,15492,United Kingdom +581492,12/9/2019,21916,Set 12 Retro White Chalk Sticks,6.19,2,15492,United Kingdom +581492,12/9/2019,21930,Jumbo Storage Bag Skulls,6.19,1,15492,United Kingdom +581492,12/9/2019,21931,Jumbo Storage Bag Suki,6.19,1,15492,United Kingdom +581492,12/9/2019,21932,Scandinavian Paisley Picnic Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,21934,Skull Shoulder Bag,6.19,3,15492,United Kingdom +581492,12/9/2019,21935,Suki Shoulder Bag,6.19,9,15492,United Kingdom +581492,12/9/2019,21936,Red Retrospot Picnic Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,21942,Skulls Design Flannel,6.19,2,15492,United Kingdom +581492,12/9/2019,21945,Strawberries Design Flannel,6.19,2,15492,United Kingdom +581492,12/9/2019,21947,Set Of 6 Heart Chopsticks,6.19,1,15492,United Kingdom +581492,12/9/2019,21949,Set Of 6 Strawberry Chopsticks,6.19,2,15492,United Kingdom +581492,12/9/2019,21967,Pack Of 12 Skull Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,21976,Pack Of 60 Mushroom Cake Cases,6.19,3,15492,United Kingdom +581492,12/9/2019,21977,Pack Of 60 Pink Paisley Cake Cases,6.19,2,15492,United Kingd +581492,12/9/2019,21980,Pack Of 12 Red Retrospot Tissues,6.19,2,15492,United Kingdom +581492,12/9/2019,21981,Pack Of 12 Woodland Tissues,6.19,4,15492,United Kingdom +581492,12/9/2019,21982,Pack Of 12 Suki Tissues,6.19,2,15492,United Kingdom +581492,12/9/2019,21983,Pack Of 12 Blue Paisley Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,21984,Pack Of 12 Pink Paisley Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,21986,Pack Of 12 Pink Polkadot Tissues,6.19,3,15492,United Kingdom +581492,12/9/2019,21989,Pack Of 20 Skull Paper Napkins,6.19,2,15492,United Kingdom +581492,12/9/2019,21990,Modern Floral Stationery Set,6.19,8,15492,United Kingdom +581492,12/9/2019,21993,Floral Folk Stationery Set,6.19,8,15492,United Kingdom +581492,12/9/2019,22024,Rainy Ladies Birthday Card,6.19,3,15492,United Kingdom +581492,12/9/2019,22025,Ring Of Roses Birthday Card,6.19,3,15492,United Kingdom +581492,12/9/2019,22026,Banquet Birthday Card,6.19,4,15492,United Kingdom +581492,12/9/2019,22027,Tea Party Birthday Card,6.19,2,15492,United Kingdom +581492,12/9/2019,22028,Penny Farthing Birthday Card,6.19,1,15492,United Kingdom +581492,12/9/2019,22029,Spaceboy Birthday Card,6.19,4,15492,United Kingdom +581492,12/9/2019,22031,Botanical Lavender Birthday Card,6.19,1,15492,United Kingdom +581492,12/9/2019,22037,Robot Birthday Card,6.19,4,15492,United Kingdom +581492,12/9/2019,22064,Pink Doughnut Trinket Pot,6.19,1,15492,United Kingdom +581492,12/9/2019,22065,Christmas Pudding Trinket Pot,6.19,2,15492,United Kingdom +581492,12/9/2019,22068,Black Pirate Treasure Chest,6.19,1,15492,United Kingdom +581492,12/9/2019,22070,Small Red Retrospot Mug In Box,6.19,3,15492,United Kingdom +581492,12/9/2019,22071,Small White Retrospot Mug In Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22074,6 Ribbons Shimmering Pinks,6.19,1,15492,United Kingdom +581492,12/9/2019,22075,6 Ribbons Elegant Christmas,6.19,8,15492,United Kingdom +581492,12/9/2019,22076,6 Ribbons Empire,6.19,4,15492,United Kingdom +581492,12/9/2019,22083,Paper Chain Kit Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22085,Paper Chain Kit Skulls,6.19,1,15492,United Kingdom +581492,12/9/2019,22086,Paper Chain Kit 50'S Christmas,6.19,38,15492,United Kingdom +581492,12/9/2019,22091,Empire Tissue Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22099,Caravan Square Tissue Box,6.19,3,15492,United Kingdom +581492,12/9/2019,22104,Mirror Mosaic Candle Plate,6.19,4,15492,United Kingdom +581492,12/9/2019,22106,Mirror Mosaic Hurricane Lamp,6.19,1,15492,United Kingdom +581492,12/9/2019,22107,Pizza Plate In Box,6.19,10,15492,United Kingdom +581492,12/9/2019,22108,Ping! Microwave Plate,6.04,2,15492,United Kingdom +581492,12/9/2019,22109,Full English Breakfast Plate,6.04,2,15492,United Kingdom +581492,12/9/2019,22110,Bird House Hot Water Bottle,6.04,3,15492,United Kingdom +581492,12/9/2019,22111,Scottie Dog Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,22112,Chocolate Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,6.19,4,15492,United Kingdo +581492,12/9/2019,22116,Metal Sign His Dinner Is Served,6.19,2,15492,United Kingdom +581492,12/9/2019,22121,Noel Wooden Block Letters,6.19,1,15492,United Kingdom +581492,12/9/2019,22123,Ping Microwave Apron,6.19,1,15492,United Kingdom +581492,12/9/2019,22124,Set Of 2 Tea Towels Ping Microwave,6.19,1,15492,United Kingd +581492,12/9/2019,22129,Party Cones Candy Decoration,6.19,3,15492,United Kingdom +581492,12/9/2019,22134,Mini Ladle Love Heart Red,6.19,1,15492,United Kingdom +581492,12/9/2019,15036,Assorted Colours Silk Fan,6.19,1,15492,United Kingdom +581492,12/9/2019,15039,Sandalwood Fan,6.19,1,15492,United Kingdom +581492,12/9/2019,15058C,Ice Cream Design Garden Parasol,6.19,1,15492,United Kingdom +581492,12/9/2019,16218,Cartoon Pencil Sharpeners,6.19,5,15492,United Kingdom +581492,12/9/2019,16225,Rattle Snake Eggs,6.19,1,15492,United Kingdom +581492,12/9/2019,16235,Recycled Pencil With Rabbit Eraser,6.19,3,15492,United Kingd +581492,12/9/2019,16237,Sleeping Cat Erasers,6.19,1,15492,United Kingdom +581492,12/9/2019,17038,Porcelain Budah Incense Holder,6.19,1,15492,United Kingdom +581492,12/9/2019,17084N,Fairy Dreams Incense,6.19,1,15492,United Kingdom +581492,12/9/2019,20659,Economy Luggage Tag,6.19,7,15492,United Kingdom +581492,12/9/2019,20665,Red Retrospot Purse,6.19,2,15492,United Kingdom +581492,12/9/2019,20668,Disco Ball Christmas Decoration,6.19,1,15492,United Kingdom +581492,12/9/2019,20718,Red Retrospot Shopper Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,20719,Woodland Charlotte Bag,6.19,7,15492,United Kingdom +581492,12/9/2019,20723,Strawberry Charlotte Bag,6.19,2,15492,United Kingdom +581492,12/9/2019,20724,Red Retrospot Charlotte Bag,6.19,4,15492,United Kingdom +581492,12/9/2019,22135,Mini Ladle Love Heart Pink,6.19,6,15492,United Kingdom +581492,12/9/2019,22138,Baking Set 9 Piece Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22141,Christmas Craft Tree Top Angel,6.19,2,15492,United Kingdom +581492,12/9/2019,22150,3 Stripey Mice Feltcraft,7.24,2,15492,United Kingdom +581492,12/9/2019,22153,Angel Decoration Stars On Dress,7.24,2,15492,United Kingdom +581492,12/9/2019,22154,Angel Decoration 3 Buttons,7.24,3,15492,United Kingdom +581492,12/9/2019,22155,Star Decoration Rustic,7.24,7,15492,United Kingdom +581492,12/9/2019,22156,Heart Decoration With Pearls,7.24,4,15492,United Kingdom +581492,12/9/2019,22163,Heart String Memo Holder Hanging,7.24,9,15492,United Kingdom +581492,12/9/2019,22165,Diamante Heart Shaped Wall Mirror,7.24,1,15492,United Kingdo +581492,12/9/2019,22167,Oval Wall Mirror Diamante,7.24,1,15492,United Kingdom +581492,12/9/2019,22170,Picture Frame Wood Triple Portrait,7.24,1,15492,United Kingd +581492,12/9/2019,22173,Metal 4 Hook Hanger French Chateau,7.24,2,15492,United Kingd +581492,12/9/2019,22174,Photo Cube,6.19,7,15492,United Kingdom +581492,12/9/2019,22178,Victorian Glass Hanging T-Light,6.19,5,15492,United Kingdom +581492,12/9/2019,22186,Red Star Card Holder,7.24,2,15492,United Kingdom +581492,12/9/2019,22190,Local Cafe Mug,7.24,3,15492,United Kingdom +581492,12/9/2019,22193,Red Diner Wall Clock,7.24,1,15492,United Kingdom +581492,12/9/2019,22195,Large Heart Measuring Spoons,7.24,1,15492,United Kingdom +581492,12/9/2019,22196,Small Heart Measuring Spoons,7.24,11,15492,United Kingdom +581492,12/9/2019,22197,Popcorn Holder,7.24,34,15492,United Kingdom +581492,12/9/2019,22199,Frying Pan Red Retrospot,7.24,1,15492,United Kingdom +581492,12/9/2019,22209,Wood Stamp Set Happy Birthday,7.24,1,15492,United Kingdom +581492,12/9/2019,22212,Four Hook White Lovebirds,7.24,1,15492,United Kingdom +581492,12/9/2019,22219,Lovebird Hanging Decoration White,7.24,4,15492,United Kingdo +581492,12/9/2019,22224,White Lovebird Lantern,7.24,4,15492,United Kingdom +581492,12/9/2019,22227,Hanging Heart Mirror Decoration,7.24,1,15492,United Kingdom +581492,12/9/2019,22260,Felt Egg Cosy Blue Rabbit,6.39,2,15492,United Kingdom +581492,12/9/2019,22261,Felt Egg Cosy White Rabbit,7.24,1,15492,United Kingdom +581492,12/9/2019,22264,Felt Farm Animal White Bunny,7.24,1,15492,United Kingdom +581492,12/9/2019,22273,Feltcraft Doll Molly,7.24,2,15492,United Kingdom +581492,12/9/2019,22277,Cosmetic Bag Vintage Rose Paisley,7.24,1,15492,United Kingdo +581492,12/9/2019,22279,Pocket Bag Blue Paisley Red Spot,7.24,5,15492,United Kingdom +581492,12/9/2019,22280,Pocket Bag Pink Paisely Brown Spot,7.24,4,15492,United Kingd +581492,12/9/2019,22300,Coffee Mug Dog + Ball Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22301,Coffee Mug Cat + Bird Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22307,Gold Mug Bone China Tree Of Life,7.24,1,15492,United Kingdom +581492,12/9/2019,22308,Tea Cosy Blue Stripe,7.24,2,15492,United Kingdom +581492,12/9/2019,22309,Tea Cosy Red Stripe,7.24,2,15492,United Kingdom +581492,12/9/2019,22324,Blue Polkadot Kids Bag,7.24,2,15492,United Kingdom +581492,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,7.24,3,15492,United Kingd +581492,12/9/2019,22327,Round Snack Boxes Set Of 4 Skulls,7.24,2,15492,United Kingdo +581492,12/9/2019,22329,Round Container Set Of 5 Retrospot,6.19,2,15492,United Kingd +581492,12/9/2019,22338,Star Decoration Painted Zinc,6.19,4,15492,United Kingdom +581492,12/9/2019,22340,Noel Garland Painted Zinc,6.19,17,15492,United Kingdom +581492,12/9/2019,22342,Home Garland Painted Zinc,6.19,7,15492,United Kingdom +581492,12/9/2019,22348,Tea Bag Plate Red Retrospot,6.19,3,15492,United Kingdom +581492,12/9/2019,22355,Charlotte Bag Suki Design,6.19,3,15492,United Kingdom +581492,12/9/2019,22356,Charlotte Bag Pink Polkadot,6.19,1,15492,United Kingdom +581492,12/9/2019,22357,Kings Choice Biscuit Tin,6.19,2,15492,United Kingdom +581492,12/9/2019,22358,Kings Choice Tea Caddy,6.19,1,15492,United Kingdom +581492,12/9/2019,22361,Glass Jar Daisy Fresh Cotton Wool,6.19,2,15492,United Kingdo +581492,12/9/2019,22362,Glass Jar Peacock Bath Salts,6.19,2,15492,United Kingdom +581492,12/9/2019,22367,Childrens Apron Spaceboy Design,6.19,1,15492,United Kingdom +581492,12/9/2019,22372,Airline Bag Vintage World Champion,6.19,1,15492,United Kingd +581492,12/9/2019,22374,Airline Bag Vintage Jet Set Red,6.19,1,15492,United Kingdom +581492,12/9/2019,85014B,Red Retrospot Umbrella,6.19,1,15492,United Kingdom +581492,12/9/2019,85032C,Curious Images Gift Wrap Set,6.19,1,15492,United Kingdom +581492,12/9/2019,85038,6 Chocolate Love Heart T-Lights,6.19,1,15492,United Kingdom +581492,12/9/2019,85039A,Set/4 Red Mini Rose Candle In Bowl,6.19,5,15492,United King +581492,12/9/2019,85039B,S/4 Ivory Mini Rose Candle In Bowl,6.19,4,15492,United King +581492,12/9/2019,85040A,S/4 Pink Flower Candles In Bowl,6.19,1,15492,United Kingdom +581492,12/9/2019,85048,15cm Christmas Glass Ball 20 Lights,6.19,2,15492,United King +581492,12/9/2019,85049C,Romantic Pinks Ribbons,6.19,1,15492,United Kingdom +581492,12/9/2019,85049G,Chocolate Box Ribbons,6.19,1,15492,United Kingdom +581492,12/9/2019,85049H,Urban Black Ribbons,6.19,2,15492,United Kingdom +581492,12/9/2019,85053,French Enamel Candleholder,6.19,1,15492,United Kingdom +581492,12/9/2019,85059,French Enamel Water Basin,6.19,1,15492,United Kingdom +581492,12/9/2019,85066,Cream Sweetheart Mini Chest,6.19,1,15492,United Kingdom +581492,12/9/2019,85094,Candy Spot Egg Warmer Rabbit,6.19,2,15492,United Kingdom +581492,12/9/2019,85114C,Red Enchanted Forest Placemat,6.19,1,15492,United Kingdom +581492,12/9/2019,85123A,Cream Hanging Heart T-Light Holder,6.19,3,15492,United King +581492,12/9/2019,85131B,Beaded Crystal Heart Green On Stick,6.19,1,15492,United Kin +581492,12/9/2019,85131D,Beaded Crystal Heart Pink On Stick,6.19,2,15492,United King +581492,12/9/2019,85152,Hand Over The Chocolate Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,85168B,Black Baroque Carriage Clock,6.19,3,15492,United Kingdom +581492,12/9/2019,85169C,Eau De Nil Love Bird Candle,6.19,1,15492,United Kingdom +581492,12/9/2019,85170C,Set/6 Eau De Nil Bird T-Lights,6.19,1,15492,United Kingdom +581492,12/9/2019,85173,Set/6 Frog Prince T-Light Candles,6.19,1,15492,United Kingdo +581492,12/9/2019,85177,Basket Of Flowers Sewing Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,85178,Victorian Sewing Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,85179A,Green Bitty Light Chain,6.19,4,15492,United Kingdom +581492,12/9/2019,85199S,Small Hanging Ivory/Red Wood Bird,6.19,9,15492,United Kingd +581492,12/9/2019,85227,Set Of 6 3d Kit Cards For Kids,6.19,3,15492,United Kingdom +581492,12/9/2019,90003C,Midnight Blue Pair Heart Hair Slide,6.19,1,15492,United Kin +581492,12/9/2019,90003E,Green Pair Heart Hair Slides,6.19,2,15492,United Kingdom +581492,12/9/2019,90010A,Midnight Blue Glass/Silver Bracelet,6.04,1,15492,United Kin +581492,12/9/2019,90013A,Midnight Blue Vintage Earrings,6.19,3,15492,United Kingdom +581492,12/9/2019,90013C,Green Vintage Earrings,6.19,2,15492,United Kingdom +581492,12/9/2019,90014A,Silver Mop Orbit Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90016B,Gold/Mop Pendant Orbit Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90018B,Gold Mop Orbit Drop Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90019B,Gold Mop Orbit Bracelet,6.19,1,15492,United Kingdom +581492,12/9/2019,90027D,Glass Bead Hoop Earrings Amethyst,6.19,1,15492,United Kingd +581492,12/9/2019,90030B,Red Kukui Coconut Seed Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90055,Cracked Glaze Earrings Brown,6.19,1,15492,United Kingdom +581492,12/9/2019,90059A,Diamante Hair Grip Pack/2 Crystal,6.19,1,15492,United Kingd +581492,12/9/2019,90059D,Diamante Hair Grip Pack/2 Peridot,6.19,2,15492,United Kingd +581492,12/9/2019,90059E,Diamante Hair Grip Pack/2 Ruby,6.04,1,15492,United Kingdom +581492,12/9/2019,90059F,Diamante Hair Grip Pack/2 Lt Rose,6.19,1,15492,United Kingd +581492,12/9/2019,90072,Ruby Drop Chandelier Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90086,Crystal Frog Phone Charm,6.19,1,15492,United Kingdom +581492,12/9/2019,90120B,Blue Murano Twist Bracelet,6.19,2,15492,United Kingdom +581492,12/9/2019,90120C,Green Murano Twist Bracelet,6.19,1,15492,United Kingdom +581492,12/9/2019,90130B,Turq Stone/Crystal Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90130C,Green Stone/Crystal Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90134,Old Rose Combo Bead Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90138,White/Pink Mini Crystals Necklace,6.19,1,15492,United Kingdo +581492,12/9/2019,90141B,Ivory Pendant Triple Shell Necklace,6.19,1,15492,United Kin +581492,12/9/2019,90145,Silver Hoop Earrings With Flower,6.19,1,15492,United Kingdom +581492,12/9/2019,90155,Resin Necklace W Pastel Beads,6.19,1,15492,United Kingdom +581492,12/9/2019,90161D,Ant Copper Pink Boudicca Bracelet,6.19,1,15492,United Kingd +581492,12/9/2019,90163A,Pink Rosebud & Pearl Necklace,6.19,2,15492,United Kingdom +581492,12/9/2019,90165B,White Rosebud Pearl Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90168,2 Daisies Hair Comb,6.19,1,15492,United Kingdom +581492,12/9/2019,90169,Daisy Hair Comb,6.19,1,15492,United Kingdom +581492,12/9/2019,90170,Daisy Hair Band,6.19,1,15492,United Kingdom +581492,12/9/2019,90174,Butterfly Hair Band,6.19,2,15492,United Kingdom +581492,12/9/2019,90175A,White Glass Chunky Charm Bracelet,6.19,2,15492,United Kingd +581492,12/9/2019,90175D,Tigris Eye Chunky Charm Bracelet,6.19,1,15492,United Kingdo +581492,12/9/2019,90177A,Classic Diamante Earrings Jet,6.19,1,15492,United Kingdom +581492,12/9/2019,90177C,Drop Diamante Earrings Crystal,6.19,1,15492,United Kingdom +581492,12/9/2019,90181B,Amethyst Glass/Shell/Pearl Necklace,6.04,1,15492,United Kin +581492,12/9/2019,90182C,Black 3 Bead Drop Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90183A,Amber Drop Earrings W Long Beads,6.19,2,15492,United Kingdo +581492,12/9/2019,90183C,Black Drop Earrings W Long Beads,6.19,1,15492,United Kingdo +581492,12/9/2019,90184C,Black Chunky Bead Bracelet W Strap,6.19,1,15492,United King +581492,12/9/2019,90185B,Amethyst Diamante Expandable Ring,6.19,1,15492,United Kingd +581492,12/9/2019,90188,Drop Earrings W Flower & Leaf,6.19,2,15492,United Kingdom +581492,12/9/2019,90191,Silver Lariat 40cm,6.19,2,15492,United Kingdom +581492,12/9/2019,90192,Jade Drop Earrings W Filigree,6.19,1,15492,United Kingdom +581492,12/9/2019,90198A,Vintage Rose Bead Bracelet Raspberr,6.19,2,15492,United Kin +581492,12/9/2019,90200A,Purple Sweetheart Bracelet,6.19,1,15492,United Kingdom +581492,12/9/2019,90201A,Purple Enamel Flower Ring,6.19,2,15492,United Kingdom +581492,12/9/2019,90201B,Black Enamel Flower Ring,6.19,1,15492,United Kingdom +581492,12/9/2019,90201C,Red Enamel Flower Ring,6.19,1,15492,United Kingdom +581492,12/9/2019,90201D,Green Enamel Flower Ring,6.19,1,15492,United Kingdom +581492,12/9/2019,90202C,Green Enamel Flower Hair Tie,6.19,1,15492,United Kingdom +581492,12/9/2019,90202D,Pink Enamel Flower Hair Tie,6.19,1,15492,United Kingdom +581492,12/9/2019,90206C,Crystal Diamante Star Brooch,6.19,1,15492,United Kingdom +581492,12/9/2019,90208,Pair Of Pink Flower Cluster Slide,6.19,1,15492,United Kingdo +581492,12/9/2019,90210A,Grey Acrylic Faceted Bangle,6.19,1,15492,United Kingdom +581492,12/9/2019,22378,Wall Tidy Retrospot,6.19,2,15492,United Kingdom +581492,12/9/2019,22379,Recycling Bag Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22380,Toy Tidy Spaceboy,6.19,2,15492,United Kingdom +581492,12/9/2019,22396,Magnets Pack Of 4 Retro Photo,6.19,1,15492,United Kingdom +581492,12/9/2019,22411,Jumbo Shopper Vintage Red Paisley,6.19,1,15492,United Kingdo +581492,12/9/2019,22418,10 Colour Spaceboy Pen,6.19,3,15492,United Kingdom +581492,12/9/2019,22419,Lipstick Pen Red,6.19,3,15492,United Kingdom +581492,12/9/2019,22420,Lipstick Pen Baby Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,22421,Lipstick Pen Fuschia,6.19,2,15492,United Kingdom +581492,12/9/2019,22422,Toothpaste Tube Pen,6.19,4,15492,United Kingdom +581492,12/9/2019,22437,Set Of 9 Black Skull Balloons,7.24,1,15492,United Kingdom +581492,12/9/2019,22441,Grow Your Own Basil In Enamel Mug,7.24,1,15492,United Kingdo +581492,12/9/2019,22446,Pin Cushion Babushka Pink,7.24,7,15492,United Kingdom +581492,12/9/2019,22457,Natural Slate Heart Chalkboard,7.24,1,15492,United Kingdom +581492,12/9/2019,22464,Hanging Metal Heart Lantern,7.24,1,15492,United Kingdom +581492,12/9/2019,22466,Fairy Tale Cottage Night Light,7.24,1,15492,United Kingdom +581492,12/9/2019,22467,Gumball Coat Rack,7.24,1,15492,United Kingdom +581492,12/9/2019,22478,Birdhouse Garden Marker,7.24,1,15492,United Kingdom +581492,12/9/2019,22486,Plasmatronic Lamp,7.24,1,15492,United Kingdom +581492,12/9/2019,22488,Natural Slate Rectangle Chalkboard,7.24,1,15492,United Kingd +581492,12/9/2019,22489,Pack Of 12 Traditional Crayons,7.24,5,15492,United Kingdom +581492,12/9/2019,22495,Set Of 2 Round Tins Camembert,7.24,1,15492,United Kingdom +581492,12/9/2019,22540,Mini Jigsaw Circus Parade,7.24,1,15492,United Kingdom +581492,12/9/2019,22544,Mini Jigsaw Spaceboy,7.24,1,15492,United Kingdom +581492,12/9/2019,22549,Picture Dominoes,7.24,3,15492,United Kingdom +581492,12/9/2019,22551,Plasters In Tin Spaceboy,7.24,2,15492,United Kingdom +581492,12/9/2019,22553,Plasters In Tin Skulls,7.24,2,15492,United Kingdom +581492,12/9/2019,22554,Plasters In Tin Woodland Animals,7.24,3,15492,United Kingdom +581492,12/9/2019,22555,Plasters In Tin Strongman,7.24,3,15492,United Kingdom +581492,12/9/2019,22557,Plasters In Tin Vintage Paisley,7.24,2,15492,United Kingdom +581492,12/9/2019,22558,Clothes Pegs Retrospot Pack 24,7.24,3,15492,United Kingdom +581492,12/9/2019,22560,Traditional Modelling Clay,7.24,5,15492,United Kingdom +581492,12/9/2019,22565,Feltcraft Hairbands Pink And White,7.24,4,15492,United Kingd +581492,12/9/2019,22566,Feltcraft Hairband Pink And Purple,7.24,3,15492,United Kingd +581492,12/9/2019,22571,Rocking Horse Red Christmas,7.24,4,15492,United Kingdom +581492,12/9/2019,22573,Star Wooden Christmas Decoration,6.39,2,15492,United Kingdom +581492,12/9/2019,22574,Heart Wooden Christmas Decoration,7.24,5,15492,United Kingdo +581492,12/9/2019,22576,Swallow Wooden Christmas Decoration,7.24,3,15492,United King +581492,12/9/2019,22577,Wooden Heart Christmas Scandinavian,7.24,4,15492,United King +581492,12/9/2019,22578,Wooden Star Christmas Scandinavian,7.24,19,15492,United King +581492,12/9/2019,22580,Advent Calendar Gingham Sack,7.24,9,15492,United Kingdom +581492,12/9/2019,22581,Wood Stocking Christmas Scandispot,7.24,11,15492,United King +581492,12/9/2019,22585,Pack Of 6 Birdy Gift Tags,7.24,2,15492,United Kingdom +581492,12/9/2019,22586,Feltcraft Hairband Pink And Blue,7.24,2,15492,United Kingdom +581492,12/9/2019,22589,Cardholder Gingham Star,7.24,6,15492,United Kingdom +581492,12/9/2019,22591,Cardholder Gingham Christmas Tree,7.24,1,15492,United Kingdo +581492,12/9/2019,22592,Cardholder Holly Wreath Metal,7.24,3,15492,United Kingdom +581492,12/9/2019,22593,Christmas Gingham Star,6.39,2,15492,United Kingdom +581492,12/9/2019,22594,Christmas Gingham Tree,7.24,3,15492,United Kingdom +581492,12/9/2019,22597,Musical Zinc Heart Decoration,7.24,9,15492,United Kingdom +581492,12/9/2019,22598,Christmas Musical Zinc Tree,7.24,4,15492,United Kingdom +581492,12/9/2019,22599,Christmas Musical Zinc Star,6.19,5,15492,United Kingdom +581492,12/9/2019,22600,Christmas Retrospot Star Wood,6.19,2,15492,United Kingdom +581492,12/9/2019,22601,Christmas Retrospot Angel Wood,6.19,3,15492,United Kingdom +581492,12/9/2019,22602,Retrospot Wooden Heart Decoration,6.19,3,15492,United Kingdo +581492,12/9/2019,22608,Pens Assorted Funky Jeweled,6.19,3,15492,United Kingdom +581492,12/9/2019,22614,Pack Of 12 Spaceboy Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,22615,Pack Of 12 Circus Parade Tissues,6.19,4,15492,United Kingdom +581492,12/9/2019,22616,Pack Of 12 London Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,22619,Set Of 6 Soldier Skittles,6.19,4,15492,United Kingdom +581492,12/9/2019,22620,4 Traditional Spinning Tops,6.19,3,15492,United Kingdom +581492,12/9/2019,22621,Traditional Knitting Nancy,6.19,2,15492,United Kingdom +581492,12/9/2019,22629,Spaceboy Lunch Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22630,Dolly Girl Lunch Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22632,Hand Warmer Red Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22633,Hand Warmer Union Jack,6.19,1,15492,United Kingdom +581492,12/9/2019,22634,Childs Breakfast Set Spaceboy,6.19,1,15492,United Kingdom +581492,12/9/2019,22650,Ceramic Pirate Chest Money Bank,6.19,2,15492,United Kingdom +581492,12/9/2019,22651,Gentleman Shirt Repair Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,22653,Button Box,6.19,16,15492,United Kingdom +581492,12/9/2019,22654,Deluxe Sewing Kit,6.19,2,15492,United Kingdom +581492,12/9/2019,22659,Lunch Box I Love London,6.19,1,15492,United Kingdom +581492,12/9/2019,22666,Recipe Box Pantry Yellow Design,6.19,5,15492,United Kingdom +581492,12/9/2019,22693,Grow A Flytrap Or Sunflower In Tin,6.19,3,15492,United Kingd +581492,12/9/2019,22694,Wicker Star,6.19,1,15492,United Kingdom +581492,12/9/2019,22697,Green Regency Teacup And Saucer,6.19,1,15492,United Kingdom +581492,12/9/2019,22699,Roses Regency Teacup And Saucer,6.19,2,15492,United Kingdom +581492,12/9/2019,22701,Pink Dog Bowl,6.19,5,15492,United Kingdom +581492,12/9/2019,22703,Pink Cat Bowl,6.19,6,15492,United Kingdom +581492,12/9/2019,22712,Card Dolly Girl,6.19,1,15492,United Kingdom +581492,12/9/2019,22713,Card I Love London,6.19,1,15492,United Kingdom +581492,12/9/2019,22714,Card Birthday Cowboy,6.19,2,15492,United Kingdom +581492,12/9/2019,22716,Card Circus Parade,6.19,3,15492,United Kingdom +581492,12/9/2019,22717,Card Dog And Ball,6.19,1,15492,United Kingdom +581492,12/9/2019,22720,Set Of 3 Cake Tins Pantry Design,6.19,2,15492,United Kingdom +581492,12/9/2019,22725,Alarm Clock Bakelike Chocolate,6.19,1,15492,United Kingdom +581492,12/9/2019,22726,Alarm Clock Bakelike Green,6.19,4,15492,United Kingdom +581492,12/9/2019,22727,Alarm Clock Bakelike Red,6.19,4,15492,United Kingdom +581492,12/9/2019,22730,Alarm Clock Bakelike Ivory,6.19,1,15492,United Kingdom +581492,12/9/2019,22741,Funky Diva Pen,6.19,4,15492,United Kingdom +581492,12/9/2019,22745,Poppy's Playhouse Bedroom,6.19,2,15492,United Kingdom +581492,12/9/2019,22748,Poppy's Playhouse Kitchen,6.19,1,15492,United Kingdom +581492,12/9/2019,22749,Feltcraft Princess Charlotte Doll,6.19,1,15492,United Kingdo +581492,12/9/2019,22751,Feltcraft Princess Olivia Doll,6.19,2,15492,United Kingdom +581492,12/9/2019,22753,Small Yellow Babushka Notebook,6.19,1,15492,United Kingdom +581492,12/9/2019,22754,Small Red Babushka Notebook,6.19,1,15492,United Kingdom +581492,12/9/2019,22757,Large Red Babushka Notebook,6.19,3,15492,United Kingdom +581492,12/9/2019,22758,Large Purple Babushka Notebook,6.19,2,15492,United Kingdom +581492,12/9/2019,22763,Key Cabinet Ma Campagne,6.19,1,15492,United Kingdom +581492,12/9/2019,22768,Family Photo Frame Cornice,6.19,2,15492,United Kingdom +581492,12/9/2019,22776,Sweetheart 3 Tier Cake Stand,6.19,1,15492,United Kingdom +581492,12/9/2019,22795,Sweetheart Recipe Book Stand,6.19,1,15492,United Kingdom +581492,12/9/2019,22798,Antique Glass Dressing Table Pot,6.19,3,15492,United Kingdom +581492,12/9/2019,22800,Antique Tall Swirlglass Trinket Pot,6.19,3,15492,United King +581492,12/9/2019,22809,Set Of 6 T-Lights Santa,6.19,1,15492,United Kingdom +581492,12/9/2019,22811,Set Of 6 T-Lights Cacti,6.19,1,15492,United Kingdom +581492,12/9/2019,22814,Card Party Games,6.19,9,15492,United Kingdom +581492,12/9/2019,22815,Card Psychedelic Apples,6.19,18,15492,United Kingdom +581492,12/9/2019,22816,Card Motorbike Santa,6.19,2,15492,United Kingdom +581492,12/9/2019,22817,Card Suki Birthday,6.19,5,15492,United Kingdom +581492,12/9/2019,22818,Card Christmas Village,7.24,6,15492,United Kingdom +581492,12/9/2019,22819,Birthday Card Retro Spot,7.24,3,15492,United Kingdom +581492,12/9/2019,22835,Hot Water Bottle I Am So Poorly,7.24,3,15492,United Kingdom +581492,12/9/2019,22865,Hand Warmer Owl Design,7.24,2,15492,United Kingdom +581492,12/9/2019,22866,Hand Warmer Scotty Dog Design,7.24,3,15492,United Kingdom +581492,12/9/2019,22867,Hand Warmer Bird Design,7.24,4,15492,United Kingdom +581492,12/9/2019,22881,Number Tile Vintage Font 2,7.24,1,15492,United Kingdom +581492,12/9/2019,22885,Number Tile Vintage Font 6,7.24,1,15492,United Kingdom +581492,12/9/2019,22888,Number Tile Vintage Font 9,7.24,1,15492,United Kingdom +581492,12/9/2019,22890,Novelty Biscuits Cake Stand 3 Tier,7.24,1,15492,United Kingd +581492,12/9/2019,22898,Childrens Apron Apples Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22899,Children's Apron Dolly Girl,7.24,2,15492,United Kingdom +581492,12/9/2019,22900,Set 2 Tea Towels I Love London,7.24,3,15492,United Kingdom +581492,12/9/2019,22905,Calendar In Season Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22907,Pack Of 20 Napkins Pantry Design,6.39,1,15492,United Kingdom +581492,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,7.24,5,15492,United King +581492,12/9/2019,22910,Paper Chain Kit Vintage Christmas,7.24,7,15492,United Kingdo +581492,12/9/2019,22914,Blue Coat Rack Paris Fashion,7.24,1,15492,United Kingdom +581492,12/9/2019,22922,Fridge Magnets Us Diner Assorted,7.24,6,15492,United Kingdom +581492,12/9/2019,22924,Fridge Magnets La Vie En Rose,7.24,6,15492,United Kingdom +581492,12/9/2019,22928,Yellow Giant Garden Thermometer,7.24,1,15492,United Kingdom +581492,12/9/2019,22940,Feltcraft Christmas Fairy,6.19,1,15492,United Kingdom +581492,12/9/2019,22941,Christmas Lights 10 Reindeer,6.19,2,15492,United Kingdom +581492,12/9/2019,22943,Christmas Lights 10 Vintage Baubles,6.19,1,15492,United King +581492,12/9/2019,22948,Metal Decoration Naughty Children,6.19,4,15492,United Kingdo +581492,12/9/2019,22951,60 Cake Cases Dolly Girl Design,6.19,2,15492,United Kingdom +581492,12/9/2019,22952,60 Cake Cases Vintage Christmas,6.19,2,15492,United Kingdom +581492,12/9/2019,22955,36 Foil Star Cake Cases,6.19,1,15492,United Kingdom +581492,12/9/2019,22960,Jam Making Set With Jars,6.19,2,15492,United Kingdom +581492,12/9/2019,22961,Jam Making Set Printed,6.19,9,15492,United Kingdom +581492,12/9/2019,22962,Jam Jar With Pink Lid,6.19,3,15492,United Kingdom +581492,12/9/2019,22963,Jam Jar With Green Lid,6.19,3,15492,United Kingdom +581492,12/9/2019,22964,3 Piece Spaceboy Cookie Cutter Set,6.19,1,15492,United Kingd +581492,12/9/2019,22965,3 Traditional Biscuit Cutters Set,6.19,1,15492,United Kingdo +581492,12/9/2019,22966,Gingerbread Man Cookie Cutter,6.19,4,15492,United Kingdom +581492,12/9/2019,22969,Homemade Jam Scented Candles,6.19,6,15492,United Kingdom +581492,12/9/2019,22975,Spaceboy Childrens Egg Cup,6.19,2,15492,United Kingdom +581492,12/9/2019,22976,Circus Parade Childrens Egg Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,22977,Dolly Girl Childrens Egg Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,22979,Pantry Washing Up Brush,6.19,2,15492,United Kingdom +581492,12/9/2019,22980,Pantry Scrubbing Brush,6.19,1,15492,United Kingdom +581492,12/9/2019,22982,Pantry Pastry Brush,6.19,3,15492,United Kingdom +581492,12/9/2019,22983,Card Billboard Font,6.19,1,15492,United Kingdom +581492,12/9/2019,22984,Card Gingham Rose,6.19,1,15492,United Kingdom +581492,12/9/2019,22988,Soldiers Egg Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,22989,Set 2 Pantry Design Tea Towels,6.19,2,15492,United Kingdom +581492,12/9/2019,22992,Revolver Wooden Ruler,6.19,3,15492,United Kingdom +581492,12/9/2019,22993,Set Of 4 Pantry Jelly Moulds,6.19,3,15492,United Kingdom +581492,12/9/2019,22995,Travel Card Wallet Suki,6.19,4,15492,United Kingdom +581492,12/9/2019,22996,Travel Card Wallet Vintage Ticket,6.19,5,15492,United Kingdo +581492,12/9/2019,22997,Travel Card Wallet Union Jack,6.19,1,15492,United Kingdom +581492,12/9/2019,22998,Travel Card Wallet Keep Calm,6.19,4,15492,United Kingdom +581492,12/9/2019,22999,Travel Card Wallet Vintage Leaf,6.19,1,15492,United Kingdom +581492,12/9/2019,23005,Travel Card Wallet I Love London,6.19,4,15492,United Kingdom +581492,12/9/2019,23007,Spaceboy Baby Gift Set,6.19,1,15492,United Kingdom +581492,12/9/2019,23009,I Love London Baby Gift Set,6.19,2,15492,United Kingdom +581492,12/9/2019,23012,Glass Apothecary Bottle Perfume,6.19,1,15492,United Kingdom +581492,12/9/2019,23013,Glass Apothecary Bottle Tonic,6.19,2,15492,United Kingdom +581492,12/9/2019,23014,Glass Apothecary Bottle Elixir,6.19,1,15492,United Kingdom +581492,12/9/2019,23050,Recycled Acapulco Mat Green,6.19,1,15492,United Kingdom +581492,12/9/2019,23051,Recycled Acapulco Mat Blue,6.19,1,15492,United Kingdom +581492,12/9/2019,23052,Recycled Acapulco Mat Turquoise,6.19,5,15492,United Kingdom +581492,12/9/2019,23053,Recycled Acapulco Mat Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23054,Recycled Acapulco Mat Lavender,6.19,1,15492,United Kingdom +581492,12/9/2019,23074,Embossed Heart Trinket Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23076,Ice Cream Sundae Lip Gloss,6.19,3,15492,United Kingdom +581492,12/9/2019,23077,Doughnut Lip Gloss,6.19,3,15492,United Kingdom +581492,12/9/2019,23082,Set 6 Paper Table Lantern Hearts,6.19,1,15492,United Kingdom +581492,12/9/2019,23083,Set 6 Paper Table Lantern Stars,6.19,1,15492,United Kingdom +581492,12/9/2019,23084,Rabbit Night Light,6.19,57,15492,United Kingdom +581492,12/9/2019,23088,Zinc Heart Flower T-Light Holder,6.19,1,15492,United Kingdom +581492,12/9/2019,23089,Glass Bon Bon Jar,6.19,4,15492,United Kingdom +581492,12/9/2019,23093,Small Parisienne Heart Photo Frame,6.19,3,15492,United Kingd +581492,12/9/2019,23103,Jingle Bell Heart Decoration,6.19,2,15492,United Kingdom +581492,12/9/2019,23110,Parisienne Key Cabinet,6.19,1,15492,United Kingdom +581492,12/9/2019,23111,Parisienne Sewing Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23112,Parisienne Curio Cabinet,6.19,1,15492,United Kingdom +581492,12/9/2019,23118,Parisienne Jewellery Drawer,6.19,1,15492,United Kingdom +581492,12/9/2019,23158,Set Of 5 Lucky Cat Magnets,6.19,1,15492,United Kingdom +581492,12/9/2019,23165,Large Ceramic Top Storage Jar,6.19,2,15492,United Kingdom +581492,12/9/2019,23166,Medium Ceramic Top Storage Jar,6.19,2,15492,United Kingdom +581492,12/9/2019,23168,Classic Cafe Sugar Dispenser,6.19,2,15492,United Kingdom +581492,12/9/2019,23171,Regency Tea Plate Green,6.19,1,15492,United Kingdom +581492,12/9/2019,23172,Regency Tea Plate Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23173,Regency Teapot Roses,6.19,1,15492,United Kingdom +581492,12/9/2019,23175,Regency Milk Jug Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23177,Treasure Island Book Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23183,Mother's Kitchen Spoon Rest,6.19,3,15492,United Kingdom +581492,12/9/2019,23184,Bull Dog Bottle Opener,6.19,2,15492,United Kingdom +581492,12/9/2019,23188,Vintage 2 Metre Folding Ruler,6.19,1,15492,United Kingdom +581492,12/9/2019,23191,Bundle Of 3 Retro Note Books,6.19,1,15492,United Kingdom +581492,12/9/2019,23194,Gymkhana Treasure Book Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23196,Vintage Leaf Magnetic Notepad,6.19,1,15492,United Kingdom +581492,12/9/2019,23197,Sketchbook Magnetic Shopping List,6.19,2,15492,United Kingdo +581492,12/9/2019,23198,Pantry Magnetic Shopping List,6.19,2,15492,United Kingdom +581492,12/9/2019,23204,Charlotte Bag Apples Design,6.19,6,15492,United Kingdom +581492,12/9/2019,23205,Charlotte Bag Vintage Alphabet,6.19,3,15492,United Kingdom +581492,12/9/2019,23212,Heart Wreath Decoration With Bell,6.19,7,15492,United Kingdo +581492,12/9/2019,23213,Star Wreath Decoration With Bell,6.19,7,15492,United Kingdom +581492,12/9/2019,23220,Reindeer Heart Decoration Gold,6.19,2,15492,United Kingdom +581492,12/9/2019,23224,Cherub Heart Decoration Gold,6.19,1,15492,United Kingdom +581492,12/9/2019,23229,Vintage Donkey Tail Game,6.19,2,15492,United Kingdom +581492,12/9/2019,23234,Biscuit Tin Vintage Christmas,6.19,4,15492,United Kingdom +581492,12/9/2019,23241,Treasure Tin Gymkhana Design,6.19,3,15492,United Kingdom +581492,12/9/2019,23243,Set Of Tea Coffee Sugar Tins Pantry,6.19,1,15492,United King +581492,12/9/2019,23245,Set Of 3 Regency Cake Tins,6.19,7,15492,United Kingdom +581492,12/9/2019,23247,Biscuit Tin 50'S Christmas,6.19,3,15492,United Kingdom +581492,12/9/2019,23264,Set Of 3 Wooden Sleigh Decorations,6.19,3,15492,United Kingd +581492,12/9/2019,23265,Set Of 3 Wooden Tree Decorations,6.19,3,15492,United Kingdom +581492,12/9/2019,23266,Set Of 3 Wooden Stocking Decoration,6.19,2,15492,United King +581492,12/9/2019,23274,Star T-Light Holder Willie Winkie,6.19,3,15492,United Kingdo +581492,12/9/2019,23275,Set Of 3 Hanging Owls Ollie Beak,6.19,2,15492,United Kingdom +581492,12/9/2019,23280,Folding Butterfly Mirror Hot Pink,6.19,2,15492,United Kingdo +581492,12/9/2019,23284,Doormat Keep Calm And Come In,6.19,1,15492,United Kingdom +581492,12/9/2019,23285,Pink Vintage Spot Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23287,Red Vintage Spot Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23290,Spaceboy Childrens Bowl,6.19,1,15492,United Kingdom +581492,12/9/2019,23292,Spaceboy Childrens Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,23293,Set Of 12 Fairy Cake Baking Cases,6.19,3,15492,United Kingdo +581492,12/9/2019,23294,Set Of 6 Snack Loaf Baking Cases,6.19,1,15492,United Kingdom +581492,12/9/2019,23295,Set Of 12 Mini Loaf Baking Cases,6.19,4,15492,United Kingdom +581492,12/9/2019,23298,Spotty Bunting,6.19,2,15492,United Kingdom +581492,12/9/2019,23299,Food Cover With Beads Set 2,6.19,1,15492,United Kingdom +581492,12/9/2019,23300,Gardeners Kneeling Pad Cup Of Tea,6.19,3,15492,United Kingdo +581492,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,6,15492,United Kingdom +581492,12/9/2019,23307,Set Of 60 Pantry Design Cake Cases,6.19,7,15492,United Kingd +581492,12/9/2019,23308,Set Of 60 Vintage Leaf Cake Cases,6.19,1,15492,United Kingdo +581492,12/9/2019,23309,Set Of 60 I Love London Cake Cases,6.19,2,15492,United Kingd +581492,12/9/2019,23310,Bubblegum Ring Assorted,6.19,4,15492,United Kingdom +581492,12/9/2019,23311,Vintage Christmas Stocking,6.19,1,15492,United Kingdom +581492,12/9/2019,23312,Vintage Christmas Gift Sack,6.19,6,15492,United Kingdom +581492,12/9/2019,23313,Vintage Christmas Bunting,6.19,4,15492,United Kingdom +581492,12/9/2019,23314,Vintage Christmas Tablecloth,6.19,1,15492,United Kingdom +581492,12/9/2019,23319,Box Of 6 Mini 50'S Crackers,6.19,3,15492,United Kingdom +581492,12/9/2019,23322,Large White Heart Of Wicker,6.19,1,15492,United Kingdom +581492,12/9/2019,23323,White Wicker Star,6.19,6,15492,United Kingdom +581492,12/9/2019,23328,Set 6 School Milk Bottles In Crate,6.19,5,15492,United Kingd +581492,12/9/2019,23332,Ivory Wicker Heart Large,6.19,1,15492,United Kingdom +581492,12/9/2019,23334,Ivory Wicker Heart Small,6.19,1,15492,United Kingdom +581492,12/9/2019,23336,Egg Frying Pan Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23340,Vintage Christmas Cake Frill,6.19,3,15492,United Kingdom +581492,12/9/2019,23345,Dolly Girl Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23347,I Love London Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,2,15492,United Kingdom +581492,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,5,15492,United Kingdom +581492,12/9/2019,23352,Roll Wrap 50'S Red Christmas,6.19,1,15492,United Kingdom +581492,12/9/2019,23353,6 Gift Tags Vintage Christmas,6.19,3,15492,United Kingdom +581492,12/9/2019,23354,6 Gift Tags 50'S Christmas,6.19,4,15492,United Kingdom +581492,12/9/2019,23357,Hot Water Bottle Sex Bomb,6.19,1,15492,United Kingdom +581492,12/9/2019,23358,Hot Stuff Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,23365,Set 12 Colour Pencils Love London,6.19,1,15492,United Kingdo +581492,12/9/2019,23366,Set 12 Colouring Pencils Doily,6.04,5,15492,United Kingdom +581492,12/9/2019,23367,Set 12 Colour Pencils Spaceboy,6.04,10,15492,United Kingdom +581492,12/9/2019,23368,Set 12 Colour Pencils Dolly Girl,6.19,2,15492,United Kingdom +581492,12/9/2019,23581,Jumbo Bag Paisley Park,6.19,3,15492,United Kingdom +581492,12/9/2019,21928,Jumbo Bag Scandinavian Blue Paisley,6.19,2,15492,United King +581492,12/9/2019,21929,Jumbo Bag Pink Vintage Paisley,6.19,1,15492,United Kingdom +581492,12/9/2019,85099C,Jumbo Bag Baroque Black White,6.19,7,15492,United Kingdom +581492,12/9/2019,23199,Jumbo Bag Apples,6.19,2,15492,United Kingdom +581492,12/9/2019,23200,Jumbo Bag Pears,6.19,1,15492,United Kingdom +581492,12/9/2019,23202,Jumbo Bag Vintage Leaf,6.19,2,15492,United Kingdom +581492,12/9/2019,23343,Jumbo Bag Vintage Christmas,6.19,4,15492,United Kingdom +581492,12/9/2019,23344,Jumbo Bag 50'S Christmas,6.19,5,15492,United Kingdom +581492,12/9/2019,23583,Lunch Bag Paisley Park,6.19,2,15492,United Kingdom +581492,12/9/2019,20727,Lunch Bag Black Skull,6.04,2,15492,United Kingdom +581492,12/9/2019,20728,Lunch Bag Cars Blue,6.04,1,15492,United Kingdom +581492,12/9/2019,20725,Lunch Bag Red Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,20726,Lunch Bag Woodland,6.19,1,15492,United Kingdom +581492,12/9/2019,22662,Lunch Bag Dolly Girl Design,6.19,1,15492,United Kingdom +581492,12/9/2019,23206,Lunch Bag Apple Design,6.19,3,15492,United Kingdom +581492,12/9/2019,23208,Lunch Bag Vintage Leaf Design,6.19,3,15492,United Kingdom +581492,12/9/2019,23375,50'S Christmas Paper Gift Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,23437,50'S Christmas Gift Bag Large,6.19,1,15492,United Kingdom +581492,12/9/2019,22821,Gift Bag Psychedelic Apples,7.24,3,15492,United Kingdom +581493,12/9/2019,79190B,Retro Plastic Polka Tray,7.24,15,12423,Belgium +581493,12/9/2019,79190A,Retro Plastic 70'S Tray,7.24,15,12423,Belgium +581493,12/9/2019,22915,Assorted Bottle Top Magnets,7.24,12,12423,Belgium +581493,12/9/2019,22151,Place Setting White Heart,7.24,24,12423,Belgium +581493,12/9/2019,22632,Hand Warmer Red Retrospot,7.24,12,12423,Belgium +581493,12/9/2019,22865,Hand Warmer Owl Design,7.24,12,12423,Belgium +581493,12/9/2019,20718,Red Retrospot Shopper Bag,7.24,10,12423,Belgium +581493,12/9/2019,79190B,Retro Plastic Polka Tray,7.24,12,12423,Belgium +581493,12/9/2019,71459,Hanging Jam Jar T-Light Holders,7.24,12,12423,Belgium +581493,12/9/2019,84945,Multi Colour Silver T-Light Holder,7.24,12,12423,Belgium +581493,12/9/2019,22356,Charlotte Bag Pink Polkadot,7.24,10,12423,Belgium +581493,12/9/2019,20724,Red Retrospot Charlotte Bag,7.24,10,12423,Belgium +581493,12/9/2019,23204,Charlotte Bag Apples Design,7.24,10,12423,Belgium +581493,12/9/2019,21108,Fairy Cake Flannel Assorted Colour,7.24,18,12423,Belgium +581493,12/9/2019,22252,Birdcage Decoration Tealight Holder,7.24,12,12423,Belgium +581493,12/9/2019,22807,Set Of 6 T-Lights Toadstools,6.19,6,12423,Belgium +581494,12/9/2019,23084,Rabbit Night Light,6.19,24,12518,Germany +581494,12/9/2019,21559,Strawberry Lunch Box With Cutlery,6.19,6,12518,Germany +581494,12/9/2019,21506,Fancy Font Birthday Card,6.19,12,12518,Germany +581494,12/9/2019,22716,Card Circus Parade,6.19,12,12518,Germany +581494,12/9/2019,22556,Plasters In Tin Circus Parade,6.19,12,12518,Germany +581494,12/9/2019,22328,Round Snack Boxes Set Of 4 Fruits,6.19,6,12518,Germany +581494,12/9/2019,22730,Alarm Clock Bakelike Ivory,6.19,4,12518,Germany +581494,12/9/2019,22633,Hand Warmer Union Jack,6.19,12,12518,Germany +581494,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,6.19,6,12518,Germany +581494,12/9/2019,10125,Mini Funky Design Tapes,6.19,20,12518,Germany +581494,12/9/2019,23367,Set 12 Colour Pencils Spaceboy,6.19,24,12518,Germany +581494,12/9/2019,22551,Plasters In Tin Spaceboy,6.04,12,12518,Germany +581494,12/9/2019,22554,Plasters In Tin Woodland Animals,6.04,12,12518,Germany +581494,12/9/2019,22549,Picture Dominoes,6.19,12,12518,Germany +581494,12/9/2019,23388,Woodland Mini Backpack,6.19,4,12518,Germany +581494,12/9/2019,23390,Dolly Girl Mini Backpack,6.19,4,12518,Germany +581494,12/9/2019,23389,Spaceboy Mini Backpack,6.19,4,12518,Germany +581495,12/9/2019,23535,Wall Art Bicycle Safety,6.19,12,14051,United Kingdom +581495,12/9/2019,22699,Roses Regency Teacup And Saucer,6.19,12,14051,United Kingdom +581495,12/9/2019,22698,Pink Regency Teacup And Saucer,6.19,12,14051,United Kingdom +581495,12/9/2019,22697,Green Regency Teacup And Saucer,6.19,12,14051,United Kingdom +581495,12/9/2019,22633,Hand Warmer Union Jack,6.04,18,14051,United Kingdom +581495,12/9/2019,15056N,Edwardian Parasol Natural,6.19,36,14051,United Kingdom +581495,12/9/2019,23439,Hand Warmer Red Love Heart,6.19,36,14051,United Kingdom +581495,12/9/2019,48138,Doormat Union Flag,6.19,10,14051,United Kingdom +581495,12/9/2019,21523,Doormat Fancy Font Home Sweet Home,6.19,10,14051,United King +581495,12/9/2019,21877,Home Sweet Home Mug,6.19,12,14051,United Kingdom +581495,12/9/2019,21871,Save The Planet Mug,6.19,18,14051,United Kingdom +581495,12/9/2019,22798,Antique Glass Dressing Table Pot,6.19,36,14051,United Kingdo +581495,12/9/2019,23173,Regency Teapot Roses,6.19,6,14051,United Kingdom +581495,12/9/2019,22423,Regency Cakestand 3 Tier,6.19,10,14051,United Kingdom +581496,12/9/2019,22664,Toy Tidy Dolly Girl Design,6.19,20,16558,United Kingdom +581496,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,6.19,12,16558,United Kingdom +581496,12/9/2019,72800E,4 Ivory Dinner Candles Silver Flock,6.19,12,16558,United Ki +581496,12/9/2019,23313,Vintage Christmas Bunting,6.19,10,16558,United Kingdom +581496,12/9/2019,22952,60 Cake Cases Vintage Christmas,6.19,24,16558,United Kingdom +581496,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,6.04,12,16558,United Kin +581496,12/9/2019,23298,Spotty Bunting,6.19,3,16558,United Kingdom +581496,12/9/2019,23598,Paper Bunting Vintage Party,6.19,6,16558,United Kingdom +581496,12/9/2019,16169E,Wrap 50'S Christmas,6.19,25,16558,United Kingdom +581496,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,12,16558,United Kingdom +581496,12/9/2019,23350,Roll Wrap Vintage Spot,6.19,24,16558,United Kingdom +581496,12/9/2019,22865,Hand Warmer Owl Design,6.19,24,16558,United Kingdom +581496,12/9/2019,22632,Hand Warmer Red Retrospot,6.19,12,16558,United Kingdom +581496,12/9/2019,22835,Hot Water Bottle I Am So Poorly,6.19,4,16558,United Kingdom +581496,12/9/2019,22112,Chocolate Hot Water Bottle,6.19,6,16558,United Kingdom +581496,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,4,16558,United Kingdom +581496,12/9/2019,22314,Office Mug Warmer Choc+Blue,6.19,24,16558,United Kingdom +581496,12/9/2019,22313,Office Mug Warmer Pink,6.19,12,16558,United Kingdom +581496,12/9/2019,23084,Rabbit Night Light,6.19,18,16558,United Kingdom +581496,12/9/2019,20831,Gold Photo Frame,6.19,12,16558,United Kingdom +581496,12/9/2019,21115,Rose Caravan Doorstop,6.19,8,16558,United Kingdom +581496,12/9/2019,21462,Nursery A B C Painted Letters,7.24,8,16558,United Kingdom +581496,12/9/2019,22076,6 Ribbons Empire,7.24,24,16558,United Kingdom +581496,12/9/2019,22190,Local Cafe Mug,7.24,24,16558,United Kingdom +581496,12/9/2019,22215,Cake Stand White Two Tier Lace,7.24,6,16558,United Kingdom +581496,12/9/2019,22220,Cake Stand Lovebird 2 Tier White,7.24,6,16558,United Kingdom +581496,12/9/2019,22464,Hanging Metal Heart Lantern,7.24,12,16558,United Kingdom +581496,12/9/2019,22465,Hanging Metal Star Lantern,7.24,12,16558,United Kingdom +581496,12/9/2019,22539,Mini Jigsaw Dolly Girl,7.24,48,16558,United Kingdom +581496,12/9/2019,22544,Mini Jigsaw Spaceboy,7.24,48,16558,United Kingdom +581496,12/9/2019,23438,Red Spot Gift Bag Large,6.19,12,16558,United Kingdom +581496,12/9/2019,23437,50'S Christmas Gift Bag Large,6.19,12,16558,United Kingdom +581497,12/9/2019,20719,Woodland Charlotte Bag,6.19,33,17497,United Kingdom +581497,12/9/2019,20723,Strawberry Charlotte Bag,6.19,42,17497,United Kingdom +581497,12/9/2019,20724,Red Retrospot Charlotte Bag,6.19,55,17497,United Kingdom +581497,12/9/2019,21212,Pack Of 72 Retrospot Cake Cases,7.24,7,17497,United Kingdom +581497,12/9/2019,21238,Red Retrospot Cup,7.24,8,17497,United Kingdom +581497,12/9/2019,21242,Red Retrospot Plate,7.24,2,17497,United Kingdom +581497,12/9/2019,21479,White Skull Hot Water Bottle,7.24,25,17497,United Kingdom +581497,12/9/2019,21481,Fawn Blue Hot Water Bottle,7.24,1,17497,United Kingdom +581497,12/9/2019,21481,Fawn Blue Hot Water Bottle,7.24,1,17497,United Kingdom +581497,12/9/2019,21484,Chick Grey Hot Water Bottle,7.24,1,17497,United Kingdom +581497,12/9/2019,21668,Red Stripe Ceramic Drawer Knob,7.24,1,17497,United Kingdom +581497,12/9/2019,21670,Blue Spot Ceramic Drawer Knob,7.24,1,17497,United Kingdom +581497,12/9/2019,21671,Red Spot Ceramic Drawer Knob,7.24,1,17497,United Kingdom +581497,12/9/2019,21672,White Spot Red Ceramic Drawer Knob,7.24,6,17497,United Kingd +581497,12/9/2019,21673,White Spot Blue Ceramic Drawer Knob,7.24,1,17497,United King +581497,12/9/2019,21714,Citronella Candle Garden Pot,7.24,2,17497,United Kingdom +581497,12/9/2019,22110,Bird House Hot Water Bottle,7.24,7,17497,United Kingdom +581497,12/9/2019,22111,Scottie Dog Hot Water Bottle,7.24,5,17497,United Kingdom +581497,12/9/2019,22112,Chocolate Hot Water Bottle,7.24,13,17497,United Kingdom +581497,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,7.24,48,17497,United Kingd +581497,12/9/2019,22197,Popcorn Holder,7.24,68,17497,United Kingdom +581497,12/9/2019,22355,Charlotte Bag Suki Design,7.24,110,17497,United Kingdom +581497,12/9/2019,22356,Charlotte Bag Pink Polkadot,7.24,25,17497,United Kingdom +581497,12/9/2019,22356,Charlotte Bag Pink Polkadot,7.24,1,17497,United Kingdom +581497,12/9/2019,22423,Regency Cakestand 3 Tier,7.24,8,17497,United Kingdom +581497,12/9/2019,22725,Alarm Clock Bakelike Chocolate,7.24,3,17497,United Kingdom +581497,12/9/2019,22726,Alarm Clock Bakelike Green,7.24,1,17497,United Kingdom +581497,12/9/2019,22727,Alarm Clock Bakelike Red,7.24,10,17497,United Kingdom +581497,12/9/2019,22730,Alarm Clock Bakelike Ivory,7.24,5,17497,United Kingdom +581497,12/9/2019,22735,Ribbon Reel Socks And Mittens,7.24,2,17497,United Kingdom +581497,12/9/2019,22736,Ribbon Reel Making Snowmen,7.24,3,17497,United Kingdom +581497,12/9/2019,22738,Ribbon Reel Snowy Village,7.24,1,17497,United Kingdom +581497,12/9/2019,22805,Blue Drawer Knob Acrylic Edwardian,7.24,1,17497,United Kingd +581497,12/9/2019,22835,Hot Water Bottle I Am So Poorly,7.24,4,17497,United Kingdom +581497,12/9/2019,22895,Set Of 2 Tea Towels Apple And Pears,7.24,1,17497,United King +581497,12/9/2019,22896,Peg Bag Apples Design,7.24,2,17497,United Kingdom +581497,12/9/2019,22898,Childrens Apron Apples Design,7.24,2,17497,United Kingdom +581497,12/9/2019,22943,Christmas Lights 10 Vintage Baubles,7.24,1,17497,United King +581497,12/9/2019,23084,Rabbit Night Light,6.19,37,17497,United Kingdom +581497,12/9/2019,23168,Classic Cafe Sugar Dispenser,6.19,1,17497,United Kingdom +581497,12/9/2019,23204,Charlotte Bag Apples Design,6.04,13,17497,United Kingdom +581497,12/9/2019,23205,Charlotte Bag Vintage Alphabet,6.04,8,17497,United Kingdom +581497,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,1,17497,United Kingdom +581497,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,36,17497,United Kingdom +581497,12/9/2019,23356,Love Hot Water Bottle,6.19,1,17497,United Kingdom +581497,12/9/2019,23357,Hot Water Bottle Sex Bomb,6.19,2,17497,United Kingdom +581497,12/9/2019,23358,Hot Stuff Hot Water Bottle,6.19,2,17497,United Kingdom +581497,12/9/2019,35970,Zinc Folkart Sleigh Bells,6.19,2,17497,United Kingdom +581497,12/9/2019,47566,Party Bunting,6.19,5,17497,United Kingdom +581497,12/9/2019,82583,Hot Baths Metal Sign,6.19,4,17497,United Kingdom +581497,12/9/2019,84029G,Knitted Union Flag Hot Water Bottle,6.19,11,17497,United Ki +581497,12/9/2019,84997C,Childrens Cutlery Polkadot Blue,6.19,3,17497,United Kingdom +581497,12/9/2019,85049G,Chocolate Box Ribbons,6.19,2,17497,United Kingdom +581497,12/9/2019,20727,Lunch Bag Black Skull,6.19,8,17497,United Kingdom +581497,12/9/2019,22383,Lunch Bag Suki Design,7.24,2,17497,United Kingdom +581497,12/9/2019,23206,Lunch Bag Apple Design,6.04,3,17497,United Kingdom +581497,12/9/2019,23206,Lunch Bag Apple Design,6.04,1,17497,United Kingdom +581497,12/9/2019,23207,Lunch Bag Alphabet Design,6.19,1,17497,United Kingdom +581497,12/9/2019,23208,Lunch Bag Vintage Leaf Design,6.19,3,17497,United Kingdom +581498,12/9/2019,20669,Red Heart Luggage Tag,6.19,3,14498,United Kingdom +581498,12/9/2019,20679,Edwardian Parasol Red,6.19,5,14498,United Kingdom +581498,12/9/2019,20717,Strawberry Shopper Bag,6.19,1,14498,United Kingdom +581498,12/9/2019,20718,Red Retrospot Shopper Bag,6.19,1,14498,United Kingdom +581498,12/9/2019,20750,Red Retrospot Mini Cases,6.19,1,14498,United Kingdom +581498,12/9/2019,20961,Strawberry Bath Sponge,6.19,1,14498,United Kingdom +581498,12/9/2019,20963,Apple Bath Sponge,6.19,1,14498,United Kingdom +581498,12/9/2019,21115,Rose Caravan Doorstop,6.19,1,14498,United Kingdom +581498,12/9/2019,21125,Set 6 Football Celebration Candles,6.19,3,14498,United Kingd +581498,12/9/2019,21137,Black Record Cover Frame,6.19,4,14498,United Kingdom +581498,12/9/2019,21155,Red Retrospot Peg Bag,6.19,4,14498,United Kingdom +581498,12/9/2019,21166,Cook With Wine Metal Sign,6.19,3,14498,United Kingdom +581498,12/9/2019,21169,You're Confusing Me Metal Sign,6.19,2,14498,United Kingdom +581498,12/9/2019,21174,Pottering In The Shed Metal Sign,6.19,2,14498,United Kingdom +581498,12/9/2019,21181,Please One Person Metal Sign,6.19,6,14498,United Kingdom +581498,12/9/2019,21216,Set 3 Retrospot Tea/Coffee/Sugar,6.19,2,14498,United Kingdom +581498,12/9/2019,21217,Red Retrospot Round Cake Tins,6.19,3,14498,United Kingdom +581498,12/9/2019,21218,Red Spotty Biscuit Tin,6.19,4,14498,United Kingdom +581498,12/9/2019,21232,Strawberry Ceramic Trinket Pot,6.19,13,14498,United Kingdom +581498,12/9/2019,21239,Pink Polkadot Cup,6.19,1,14498,United Kingdom +581498,12/9/2019,21257,Victorian Sewing Box Medium,6.19,3,14498,United Kingdom +581498,12/9/2019,21327,Skulls Writing Set,6.19,1,14498,United Kingdom +581498,12/9/2019,21328,Balloons Writing Set,6.19,1,14498,United Kingdom +581498,12/9/2019,21329,Dinosaurs Writing Set,6.19,1,14498,United Kingdom +581498,12/9/2019,21411,Gingham Heart Doorstop Red,6.19,1,14498,United Kingdom +581498,12/9/2019,21430,Set/3 Red Gingham Rose Storage Box,6.19,3,14498,United Kingd +581498,12/9/2019,21494,Rotating Leaves T-Light Holder,6.19,7,14498,United Kingdom +581498,12/9/2019,21523,Doormat Fancy Font Home Sweet Home,6.19,1,14498,United Kingd +581498,12/9/2019,21557,Set Of 6 Funky Beakers,6.19,1,14498,United Kingdom +581498,12/9/2019,21558,Skull Lunch Box With Cutlery,6.19,1,14498,United Kingdom +581498,12/9/2019,21731,Red Toadstool Led Night Light,6.19,9,14498,United Kingdom +581498,12/9/2019,21754,Home Building Block Word,6.19,2,14498,United Kingdom +581498,12/9/2019,21790,Vintage Snap Cards,6.19,1,14498,United Kingdom +581498,12/9/2019,21843,Red Retrospot Cake Stand,6.19,5,14498,United Kingdom +581498,12/9/2019,21864,Union Jack Flag Passport Cover,6.19,2,14498,United Kingdom +581498,12/9/2019,21874,Gin And Tonic Mug,6.19,2,14498,United Kingdom +581498,12/9/2019,21876,Pottering Mug,6.19,4,14498,United Kingdom +581498,12/9/2019,21890,S/6 Wooden Skittles In Cotton Bag,6.19,1,14498,United Kingdo +581498,12/9/2019,21912,Vintage Snakes & Ladders,6.19,1,14498,United Kingdom +581498,12/9/2019,21916,Set 12 Retro White Chalk Sticks,6.19,7,14498,United Kingdom +581498,12/9/2019,21930,Jumbo Storage Bag Skulls,6.19,2,14498,United Kingdom +581498,12/9/2019,21931,Jumbo Storage Bag Suki,6.19,6,14498,United Kingdom +581498,12/9/2019,21934,Skull Shoulder Bag,6.19,1,14498,United Kingdom +581498,12/9/2019,21935,Suki Shoulder Bag,6.19,7,14498,United Kingdom +581498,12/9/2019,21942,Skulls Design Flannel,6.19,1,14498,United Kingdom +581498,12/9/2019,21955,Doormat Union Jack Guns And Roses,6.19,1,14498,United Kingdo +581498,12/9/2019,21977,Pack Of 60 Pink Paisley Cake Cases,6.19,1,14498,United Kingd +581498,12/9/2019,21983,Pack Of 12 Blue Paisley Tissues,6.19,1,14498,United Kingdom +581498,12/9/2019,21987,Pack Of 6 Skull Paper Cups,6.19,2,14498,United Kingdom +581498,12/9/2019,21989,Pack Of 20 Skull Paper Napkins,6.19,1,14498,United Kingdom +581498,12/9/2019,22041,"Record Frame 7"" Single Size",6.19,2,14498,United Kingdom +581498,12/9/2019,22059,Ceramic Strawberry Design Mug,6.19,1,14498,United Kingdom +581498,12/9/2019,22069,Brown Pirate Treasure Chest,6.19,3,14498,United Kingdom +581498,12/9/2019,22077,6 Ribbons Rustic Charm,6.19,2,14498,United Kingdom +581498,12/9/2019,22083,Paper Chain Kit Retrospot,6.19,2,14498,United Kingdom +581498,12/9/2019,22086,Paper Chain Kit 50'S Christmas,6.19,52,14498,United Kingdom +581498,12/9/2019,22099,Caravan Square Tissue Box,6.19,2,14498,United Kingdom +581498,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,6.19,4,14498,United Kingdo +581498,12/9/2019,22139,Retrospot Tea Set Ceramic 11 Pc,6.19,2,14498,United Kingdom +581498,12/9/2019,22141,Christmas Craft Tree Top Angel,6.19,7,14498,United Kingdom +581498,12/9/2019,22142,Christmas Craft White Fairy,6.19,2,14498,United Kingdom +581498,12/9/2019,22144,Christmas Craft Little Friends,6.19,8,14498,United Kingdom +581498,12/9/2019,22161,Heart Decoration Rustic Hanging,6.19,2,14498,United Kingdom +581498,12/9/2019,22173,Metal 4 Hook Hanger French Chateau,6.19,3,14498,United Kingd +581498,12/9/2019,22174,Photo Cube,6.19,3,14498,United Kingdom +581498,12/9/2019,22175,Pink Owl Soft Toy,6.19,1,14498,United Kingdom +581498,12/9/2019,22178,Victorian Glass Hanging T-Light,6.19,1,14498,United Kingdom +581498,12/9/2019,22179,Set 10 Night Owl Lights,6.19,1,14498,United Kingdom +581498,12/9/2019,22186,Red Star Card Holder,6.19,1,14498,United Kingdom +581498,12/9/2019,22187,Green Christmas Tree Card Holder,6.19,6,14498,United Kingdom +581498,12/9/2019,22193,Red Diner Wall Clock,6.19,1,14498,United Kingdom +581498,12/9/2019,22195,Large Heart Measuring Spoons,6.19,3,14498,United Kingdom +581498,12/9/2019,22196,Small Heart Measuring Spoons,6.19,2,14498,United Kingdom +581498,12/9/2019,22207,Frying Pan Union Flag,6.19,1,14498,United Kingdom +581498,12/9/2019,22278,Overnight Bag Vintage Rose Paisley,6.19,2,14498,United Kingd +581498,12/9/2019,22301,Coffee Mug Cat + Bird Design,6.19,2,14498,United Kingdom +581498,12/9/2019,22302,Coffee Mug Pears Design,6.19,4,14498,United Kingdom +581498,12/9/2019,22303,Coffee Mug Apples Design,6.19,4,14498,United Kingdom +581498,12/9/2019,22304,Coffee Mug Blue Paisley Design,6.19,1,14498,United Kingdom +581498,12/9/2019,22308,Tea Cosy Blue Stripe,6.19,2,14498,United Kingdom +581498,12/9/2019,22327,Round Snack Boxes Set Of 4 Skulls,6.19,4,14498,United Kingdo +581498,12/9/2019,22328,Round Snack Boxes Set Of 4 Fruits,6.19,4,14498,United Kingdo +581498,12/9/2019,22352,Lunch Box With Cutlery Retrospot,6.19,6,14498,United Kingdom +581498,12/9/2019,22357,Kings Choice Biscuit Tin,6.19,1,14498,United Kingdom +581498,12/9/2019,22358,Kings Choice Tea Caddy,6.19,1,14498,United Kingdom +581498,12/9/2019,22367,Childrens Apron Spaceboy Design,6.19,7,14498,United Kingdom +581498,12/9/2019,22371,Airline Bag Vintage Tokyo 78,6.19,1,14498,United Kingdom +581498,12/9/2019,22374,Airline Bag Vintage Jet Set Red,6.19,1,14498,United Kingdom +581498,12/9/2019,22378,Wall Tidy Retrospot,7.24,2,14498,United Kingdom +581498,12/9/2019,22379,Recycling Bag Retrospot,7.24,4,14498,United Kingdom +581498,12/9/2019,22381,Toy Tidy Pink Polkadot,7.24,1,14498,United Kingdom +581498,12/9/2019,22411,Jumbo Shopper Vintage Red Paisley,7.24,2,14498,United Kingdo +581498,12/9/2019,22422,Toothpaste Tube Pen,7.24,1,14498,United Kingdom +581498,12/9/2019,22424,Enamel Bread Bin Cream,7.24,1,14498,United Kingdom +581498,12/9/2019,22429,Enamel Measuring Jug Cream,7.24,1,14498,United Kingdom +581498,12/9/2019,22456,Natural Slate Chalkboard Large,7.24,4,14498,United Kingdom +581498,12/9/2019,22457,Natural Slate Heart Chalkboard,7.24,2,14498,United Kingdom +581498,12/9/2019,22471,Tv Dinner Tray Air Hostess,7.24,1,14498,United Kingdom +581498,12/9/2019,22474,Spaceboy Tv Dinner Tray,7.24,1,14498,United Kingdom +581498,12/9/2019,22488,Natural Slate Rectangle Chalkboard,7.24,2,14498,United Kingd +581498,12/9/2019,22498,Wooden Regatta Bunting,7.24,1,14498,United Kingdom +581498,12/9/2019,22507,Memo Board Retrospot Design,7.24,1,14498,United Kingdom +581498,12/9/2019,22526,Wheelbarrow For Children,7.24,1,14498,United Kingdom +581498,12/9/2019,22553,Plasters In Tin Skulls,7.24,1,14498,United Kingdom +581498,12/9/2019,22557,Plasters In Tin Vintage Paisley,7.24,1,14498,United Kingdom +581498,12/9/2019,22619,Set Of 6 Soldier Skittles,7.24,1,14498,United Kingdom +581498,12/9/2019,22622,Box Of Vintage Alphabet Blocks,7.24,1,14498,United Kingdom +581498,12/9/2019,22624,Ivory Kitchen Scales,7.24,1,14498,United Kingdom +581498,12/9/2019,22629,Spaceboy Lunch Box,7.24,2,14498,United Kingdom +581498,12/9/2019,22632,Hand Warmer Red Retrospot,7.24,1,14498,United Kingdom +581498,12/9/2019,22645,Ceramic Heart Fairy Cake Money Bank,7.24,4,14498,United King +581498,12/9/2019,22654,Deluxe Sewing Kit,6.19,2,14498,United Kingdom +581498,12/9/2019,22659,Lunch Box I Love London,6.19,1,14498,United Kingdom +581498,12/9/2019,22666,Recipe Box Pantry Yellow Design,6.19,11,14498,United Kingdom +581498,12/9/2019,22676,French Blue Metal Door Sign 1,6.04,1,14498,United Kingdom +581498,12/9/2019,22684,French Blue Metal Door Sign 9,6.04,1,14498,United Kingdom +581498,12/9/2019,22694,Wicker Star,6.04,1,14498,United Kingdom +581498,12/9/2019,22697,Green Regency Teacup And Saucer,6.04,6,14498,United Kingdom +581498,12/9/2019,22698,Pink Regency Teacup And Saucer,6.04,9,14498,United Kingdom +581498,12/9/2019,22699,Roses Regency Teacup And Saucer,6.19,6,14498,United Kingdom +581498,12/9/2019,22722,Set Of 6 Spice Tins Pantry Design,6.19,3,14498,United Kingdo +581498,12/9/2019,22733,3d Traditional Christmas Stickers,6.19,1,14498,United Kingdo +581498,12/9/2019,22734,Set Of 6 Ribbons Vintage Christmas,6.19,5,14498,United Kingd +581498,12/9/2019,22776,Sweetheart 3 Tier Cake Stand,6.19,1,14498,United Kingdom +581498,12/9/2019,22795,Sweetheart Recipe Book Stand,6.19,1,14498,United Kingdom +581498,12/9/2019,22807,Set Of 6 T-Lights Toadstools,6.19,2,14498,United Kingdom +581498,12/9/2019,22813,Pack 3 Boxes Bird Panettone,6.19,4,14498,United Kingdom +581498,12/9/2019,22838,3 Tier Cake Tin Red And Cream,6.19,1,14498,United Kingdom +581498,12/9/2019,22844,Vintage Cream Dog Food Container,6.19,2,14498,United Kingdom +581498,12/9/2019,22845,Vintage Cream Cat Food Container,6.19,1,14498,United Kingdom +581498,12/9/2019,22865,Hand Warmer Owl Design,6.19,1,14498,United Kingdom +581498,12/9/2019,22866,Hand Warmer Scotty Dog Design,6.19,2,14498,United Kingdom +581498,12/9/2019,22891,Tea For One Polkadot,6.19,1,14498,United Kingdom +581498,12/9/2019,22900,Set 2 Tea Towels I Love London,6.19,2,14498,United Kingdom +581498,12/9/2019,22910,Paper Chain Kit Vintage Christmas,6.19,5,14498,United Kingdo +581498,12/9/2019,22952,60 Cake Cases Vintage Christmas,6.19,7,14498,United Kingdom +581498,12/9/2019,22960,Jam Making Set With Jars,6.19,5,14498,United Kingdom +581498,12/9/2019,22961,Jam Making Set Printed,6.19,4,14498,United Kingdom +581498,12/9/2019,22966,Gingerbread Man Cookie Cutter,6.19,2,14498,United Kingdom +581498,12/9/2019,22993,Set Of 4 Pantry Jelly Moulds,6.19,2,14498,United Kingdom +581498,12/9/2019,23012,Glass Apothecary Bottle Perfume,6.19,1,14498,United Kingdom +581498,12/9/2019,23013,Glass Apothecary Bottle Tonic,6.19,1,14498,United Kingdom +581498,12/9/2019,23014,Glass Apothecary Bottle Elixir,7.24,2,14498,United Kingdom +581498,12/9/2019,23080,Red Metal Box Top Secret,7.24,5,14498,United Kingdom +581498,12/9/2019,23170,Regency Tea Plate Roses,7.24,4,14498,United Kingdom +581498,12/9/2019,23171,Regency Tea Plate Green,7.24,5,14498,United Kingdom +581498,12/9/2019,23172,Regency Tea Plate Pink,6.19,4,14498,United Kingdom +581498,12/9/2019,23181,Bull Dog Bottle Top Wall Clock,6.19,1,14498,United Kingdom +581498,12/9/2019,23196,Vintage Leaf Magnetic Notepad,6.19,1,14498,United Kingdom +581498,12/9/2019,23198,Pantry Magnetic Shopping List,6.19,2,14498,United Kingdom +581498,12/9/2019,23245,Set Of 3 Regency Cake Tins,6.19,5,14498,United Kingdom +581498,12/9/2019,23295,Set Of 12 Mini Loaf Baking Cases,6.19,1,14498,United Kingdom +581498,12/9/2019,23298,Spotty Bunting,6.19,1,14498,United Kingdom +581498,12/9/2019,23300,Gardeners Kneeling Pad Cup Of Tea,6.19,9,14498,United Kingdo +581498,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,32,14498,United Kingdo +581498,12/9/2019,23311,Vintage Christmas Stocking,6.19,6,14498,United Kingdom +581498,12/9/2019,23312,Vintage Christmas Gift Sack,6.19,3,14498,United Kingdom +581498,12/9/2019,23328,Set 6 School Milk Bottles In Crate,6.19,6,14498,United Kingd +581498,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,8,14498,United Kingdom +581498,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,22,14498,United Kingdom +581498,12/9/2019,23352,Roll Wrap 50'S Red Christmas,6.19,8,14498,United Kingdom +581498,12/9/2019,23353,6 Gift Tags Vintage Christmas,6.19,10,14498,United Kingdom +581498,12/9/2019,23354,6 Gift Tags 50'S Christmas,6.19,9,14498,United Kingdom +581498,12/9/2019,23368,Set 12 Colour Pencils Dolly Girl,6.19,3,14498,United Kingdom +581498,12/9/2019,23369,Set 36 Colour Pencils Love London,6.19,1,14498,United Kingdo +581498,12/9/2019,23370,Set 36 Colouring Pencils Doily,6.19,3,14498,United Kingdom +581498,12/9/2019,23382,Box Of 6 Christmas Cake Decorations,6.19,10,14498,United Kin +581498,12/9/2019,23388,Woodland Mini Backpack,6.19,1,14498,United Kingdom +581498,12/9/2019,23480,Mini Lights Woodland Mushrooms,7.24,1,14498,United Kingdom +581498,12/9/2019,23493,Vintage Doily Travel Sewing Kit,7.24,1,14498,United Kingdom +581498,12/9/2019,23494,Vintage Doily Deluxe Sewing Kit,7.24,1,14498,United Kingdom +581498,12/9/2019,23497,Classic Chrome Bicycle Bell,7.24,1,14498,United Kingdom +581498,12/9/2019,23501,Key Ring Baseball Boot Union Jack,7.24,1,14498,United Kingdo +581498,12/9/2019,23564,Egg Cup Milkmaid Ingrid,7.24,1,14498,United Kingdom +581498,12/9/2019,35970,Zinc Folkart Sleigh Bells,7.24,6,14498,United Kingdom +581498,12/9/2019,48138,Doormat Union Flag,7.24,1,14498,United Kingdom +581498,12/9/2019,71053,White Moroccan Metal Lantern,7.24,1,14498,United Kingdom +581498,12/9/2019,72349B,Set/6 Purple Butterfly T-Lights,7.24,2,14498,United Kingdom +581498,12/9/2019,79321,Chilli Lights,7.24,10,14498,United Kingdom +581498,12/9/2019,82001S,Silver Record Cover Frame,6.19,2,14498,United Kingdom +581498,12/9/2019,82482,Wooden Picture Frame White Finish,6.04,4,14498,United Kingdo +581498,12/9/2019,82552,Washroom Metal Sign,6.04,1,14498,United Kingdom +581498,12/9/2019,21171,Bathroom Metal Sign,6.19,1,14498,United Kingdom +581498,12/9/2019,82581,Toilet Metal Sign,6.19,1,14498,United Kingdom +581498,12/9/2019,82600,N0 Singing Metal Sign,6.19,4,14498,United Kingdom +581498,12/9/2019,84029E,Red Woolly Hottie White Heart,6.19,4,14498,United Kingdom +581498,12/9/2019,84032A,Charlie+Lola Pink Hot Water Bottle,6.19,4,14498,United King +581498,12/9/2019,84032B,Charlie + Lola Red Hot Water Bottle,6.19,3,14498,United Kin +581498,12/9/2019,84375,Set Of 20 Kids Cookie Cutters,6.19,3,14498,United Kingdom +581498,12/9/2019,84509A,Set Of 4 English Rose Placemats,6.19,1,14498,United Kingdom +581498,12/9/2019,84558A,3d Dog Picture Playing Cards,6.19,1,14498,United Kingdom +581498,12/9/2019,84832,Zinc Willie Winkie Candle Stick,6.19,26,14498,United Kingdom +581498,12/9/2019,84968E,Set Of 16 Vintage Black Cutlery,6.19,1,14498,United Kingdom +581498,12/9/2019,84970S,Hanging Heart Zinc T-Light Holder,6.19,1,14498,United Kingd +581498,12/9/2019,84997A,Childrens Cutlery Polkadot Green,6.19,2,14498,United Kingdo +581498,12/9/2019,84997B,Childrens Cutlery Retrospot Red,6.19,3,14498,United Kingdom +581498,12/9/2019,84997D,Childrens Cutlery Polkadot Pink,6.19,1,14498,United Kingdom +581498,12/9/2019,85038,6 Chocolate Love Heart T-Lights,6.19,1,14498,United Kingdom +581498,12/9/2019,85048,15cm Christmas Glass Ball 20 Lights,6.19,1,14498,United King +581498,12/9/2019,85049A,Traditional Christmas Ribbons,6.17,5,14498,United Kingdom +581498,12/9/2019,85049E,Scandinavian Reds Ribbons,6.19,4,14498,United Kingdom +581498,12/9/2019,85150,Ladies & Gentlemen Metal Sign,6.19,1,14498,United Kingdom +581498,12/9/2019,85174,S/4 Cacti Candles,6.19,1,14498,United Kingdom +581498,12/9/2019,20712,Jumbo Bag Woodland Animals,6.19,3,14498,United Kingdom +581498,12/9/2019,20713,Jumbo Bag Owls,6.19,8,14498,United Kingdom +581498,12/9/2019,21928,Jumbo Bag Scandinavian Blue Paisley,6.19,2,14498,United King +581498,12/9/2019,22386,Jumbo Bag Pink Polkadot,7.24,1,14498,United Kingdom +581498,12/9/2019,22663,Jumbo Bag Dolly Girl Design,6.19,2,14498,United Kingdom +581498,12/9/2019,23199,Jumbo Bag Apples,6.19,6,14498,United Kingdom +581498,12/9/2019,23201,Jumbo Bag Alphabet,6.19,6,14498,United Kingdom +581498,12/9/2019,85099B,Jumbo Bag Red Retrospot,6.19,5,14498,United Kingdom +581498,12/9/2019,85099C,Jumbo Bag Baroque Black White,6.19,4,14498,United Kingdom +581498,12/9/2019,20726,Lunch Bag Woodland,6.19,3,14498,United Kingdom +581498,12/9/2019,20728,Lunch Bag Cars Blue,6.19,4,14498,United Kingdom +581498,12/9/2019,22384,Lunch Bag Pink Polkadot,7.24,1,14498,United Kingdom +581498,12/9/2019,23437,50'S Christmas Gift Bag Large,7.24,1,14498,United Kingdom +581500,12/9/2019,82486,3 Drawer Antique White Wood Cabinet,7.24,4,15344,United King +581500,12/9/2019,85066,Cream Sweetheart Mini Chest,7.24,4,15344,United Kingdom +581500,12/9/2019,48187,Doormat New England,7.24,2,15344,United Kingdom +581501,12/9/2019,22319,Hairclips Forties Fabric Assorted,7.24,180,12985,United King +581501,12/9/2019,20704,Mr Robot Soft Toy,7.24,8,12985,United Kingdom +581501,12/9/2019,21564,Pink Heart Shape Love Bucket,6.19,24,12985,United Kingdom +581501,12/9/2019,21563,Red Heart Shape Love Bucket,7.24,24,12985,United Kingdom +581501,12/9/2019,22165,Diamante Heart Shaped Wall Mirror,7.24,12,12985,United Kingd +581501,12/9/2019,22299,Pig Keyring With Light & Sound,7.24,48,12985,United Kingdom +581501,12/9/2019,22447,Pin Cushion Babushka Blue,7.24,12,12985,United Kingdom +581501,12/9/2019,22442,Grow Your Own Flowers Set Of 3,7.24,12,12985,United Kingdom +581501,12/9/2019,22495,Set Of 2 Round Tins Camembert,7.24,12,12985,United Kingdom +581501,12/9/2019,22544,Mini Jigsaw Spaceboy,7.24,96,12985,United Kingdom +581501,12/9/2019,22695,Wicker Wreath Small,7.24,24,12985,United Kingdom +581501,12/9/2019,22785,Squarecushion Cover Pink Union Jack,7.24,12,12985,United Kin +581501,12/9/2019,22811,Set Of 6 T-Lights Cacti,7.24,12,12985,United Kingdom +581501,12/9/2019,22807,Set Of 6 T-Lights Toadstools,7.24,12,12985,United Kingdom +581501,12/9/2019,22942,Christmas Lights 10 Santas,7.24,12,12985,United Kingdom +581501,12/9/2019,22808,Set Of 6 T-Lights Easter Chicks,6.19,12,12985,United Kingdom +581501,12/9/2019,23143,Zinc Wire Kitchen Organiser,7.24,4,12985,United Kingdom +581501,12/9/2019,23151,Zinc Sweetheart Soap Dish,7.24,12,12985,United Kingdom +581501,12/9/2019,23425,Storage Tin Home Sweet Home,7.24,12,12985,United Kingdom +581501,12/9/2019,84356,Pompom Curtain,7.24,12,12985,United Kingdom +581501,12/9/2019,85173,Set/6 Frog Prince T-Light Candles,7.24,12,12985,United Kingd +581501,12/9/2019,21731,Red Toadstool Led Night Light,7.24,24,12985,United Kingdom +581501,12/9/2019,23480,Mini Lights Woodland Mushrooms,7.24,8,12985,United Kingdom +581501,12/9/2019,22545,Mini Jigsaw Bunnies,7.24,96,12985,United Kingdom +581502,12/9/2019,22087,Paper Bunting White Lace,7.24,6,15910,United Kingdom +581502,12/9/2019,21209,Multicolour Honeycomb Fan,7.24,5,15910,United Kingdom +581502,12/9/2019,20668,Disco Ball Christmas Decoration,7.24,24,15910,United Kingdom +581502,12/9/2019,21790,Vintage Snap Cards,7.24,6,15910,United Kingdom +581502,12/9/2019,23270,Set Of 2 Ceramic Painted Hearts,7.24,4,15910,United Kingdom +581502,12/9/2019,23103,Jingle Bell Heart Decoration,7.24,8,15910,United Kingdom +581502,12/9/2019,22576,Swallow Wooden Christmas Decoration,7.24,2,15910,United King +581502,12/9/2019,22153,Angel Decoration Stars On Dress,7.24,13,15910,United Kingdom +581502,12/9/2019,21810,Christmas Hanging Star With Bell,7.24,6,15910,United Kingdom +581502,12/9/2019,22155,Star Decoration Rustic,7.24,6,15910,United Kingdom +581502,12/9/2019,23210,White Rocking Horse Hand Painted,7.24,12,15910,United Kingdo +581502,12/9/2019,22573,Star Wooden Christmas Decoration,7.24,8,15910,United Kingdom +581502,12/9/2019,22075,6 Ribbons Elegant Christmas,7.24,4,15910,United Kingdom +581502,12/9/2019,85049A,Traditional Christmas Ribbons,7.24,4,15910,United Kingdom +581502,12/9/2019,22734,Set Of 6 Ribbons Vintage Christmas,7.24,4,15910,United Kingd +581502,12/9/2019,23274,Star T-Light Holder Willie Winkie,7.24,8,15910,United Kingdo +581502,12/9/2019,22153,Angel Decoration Stars On Dress,7.24,1,15910,United Kingdom +581502,12/9/2019,22596,Christmas Star Wish List Chalkboard,7.24,24,15910,United Kin +581502,12/9/2019,22952,60 Cake Cases Vintage Christmas,7.24,10,15910,United Kingdom +581502,12/9/2019,22141,Christmas Craft Tree Top Angel,7.24,3,15910,United Kingdom +581514,12/9/2019,22753,Small Yellow Babushka Notebook,7.24,12,17754,United Kingdom +581514,12/9/2019,22755,Small Purple Babushka Notebook,7.24,12,17754,United Kingdom +581514,12/9/2019,22754,Small Red Babushka Notebook,6.19,12,17754,United Kingdom +581514,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,6.19,4,17754,United Kingdom +581514,12/9/2019,22059,Ceramic Strawberry Design Mug,6.19,12,17754,United Kingdom +581514,12/9/2019,22199,Frying Pan Red Retrospot,6.19,13,17754,United Kingdom +581514,12/9/2019,22200,Frying Pan Pink Polkadot,6.19,2,17754,United Kingdom +581514,12/9/2019,84032A,Charlie+Lola Pink Hot Water Bottle,6.19,9,17754,United King +581514,12/9/2019,84032B,Charlie + Lola Red Hot Water Bottle,6.19,9,17754,United Kin +581514,12/9/2019,84031A,Charlie+Lola Red Hot Water Bottle,6.19,10,17754,United King +581514,12/9/2019,84031B,Charlie Lola Blue Hot Water Bottle,6.19,14,17754,United Kin +581514,12/9/2019,22646,Ceramic Strawberry Cake Money Bank,6.19,4,17754,United Kingd +581514,12/9/2019,22644,Ceramic Cherry Cake Money Bank,6.19,4,17754,United Kingdom +581514,12/9/2019,22645,Ceramic Heart Fairy Cake Money Bank,6.19,4,17754,United King +581514,12/9/2019,22394,Paperweight Kings Choice,6.04,12,17754,United Kingdom +581514,12/9/2019,17091J,Vanilla Incense In Tin,6.04,66,17754,United Kingdom +581514,12/9/2019,35471D,Set Of 3 Bird Light Pink Feather,6.04,12,17754,United Kingd +581514,12/9/2019,22075,6 Ribbons Elegant Christmas,6.04,24,17754,United Kingdom +581514,12/9/2019,17091J,Vanilla Incense In Tin,6.19,24,17754,United Kingdom +581514,12/9/2019,22069,Brown Pirate Treasure Chest,6.19,20,17754,United Kingdom +581514,12/9/2019,22068,Black Pirate Treasure Chest,6.19,14,17754,United Kingdom +581514,12/9/2019,22500,Set Of 2 Tins Jardin De Provence,6.19,4,17754,United Kingdom +581514,12/9/2019,22075,6 Ribbons Elegant Christmas,6.19,24,17754,United Kingdom +581514,12/9/2019,21705,Bag 500g Swirly Marbles,6.19,84,17754,United Kingdom +581516,12/9/2019,21109,Large Cake Towel Chocolate Spots,6.19,12,14422,United Kingdo +581516,12/9/2019,21111,Swiss Roll Towel Chocolate Spots,6.19,24,14422,United Kingdo +581516,12/9/2019,21705,Bag 500g Swirly Marbles,6.19,24,14422,United Kingdom +581516,12/9/2019,22185,Slate Tile Natural Hanging,6.19,12,14422,United Kingdom +581516,12/9/2019,22442,Grow Your Own Flowers Set Of 3,6.19,12,14422,United Kingdom +581516,12/9/2019,21620,Set Of 4 Rose Botanical Candles,6.19,12,14422,United Kingdom +581516,12/9/2019,84029G,Knitted Union Flag Hot Water Bottle,6.19,8,14422,United Kin +581516,12/9/2019,21485,Retrospot Heart Hot Water Bottle,6.19,3,14422,United Kingdom +581516,12/9/2019,22111,Scottie Dog Hot Water Bottle,6.19,6,14422,United Kingdom +581516,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,8,14422,United Kingdom +581516,12/9/2019,22112,Chocolate Hot Water Bottle,6.19,6,14422,United Kingdom +581516,12/9/2019,23356,Love Hot Water Bottle,6.19,6,14422,United Kingdom +581516,12/9/2019,21108,Fairy Cake Flannel Assorted Colour,6.19,18,14422,United King +581516,12/9/2019,22171,3 Hook Photo Shelf Antique White,6.19,4,14422,United Kingdom +581516,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,24,14422,United Kingdo +581538,12/9/2019,23193,Buffalo Bill Treasure Book Box,6.19,6,14446,United Kingdom +581538,12/9/2019,23194,Gymkhana Treasure Book Box,6.19,1,14446,United Kingdom +581538,12/9/2019,23084,Rabbit Night Light,6.19,2,14446,United Kingdom +581538,12/9/2019,22068,Black Pirate Treasure Chest,6.19,1,14446,United Kingdom +581538,12/9/2019,22956,36 Foil Heart Cake Cases,6.19,1,14446,United Kingdom +581538,12/9/2019,20936,Forked Cactus Candle,6.19,1,14446,United Kingdom +581538,12/9/2019,23040,Paper Lantern 9 Point Snow Star,6.19,1,14446,United Kingdom +581538,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,6.19,1,14446,United King +581538,12/9/2019,21222,Set/4 Badges Beetles,6.19,1,14446,United Kingdom +581538,12/9/2019,21220,Set/4 Badges Dogs,6.19,1,14446,United Kingdom +581538,12/9/2019,21224,Set/4 Skull Badges,6.19,1,14446,United Kingdom +581538,12/9/2019,85123A,Cream Hanging Heart T-Light Holder,6.19,1,14446,United King +581538,12/9/2019,22992,Revolver Wooden Ruler,6.19,1,14446,United Kingdom +581538,12/9/2019,79066K,Retro Mod Tray,6.19,1,14446,United Kingdom +581538,12/9/2019,84380,Set Of 3 Butterfly Cookie Cutters,6.19,1,14446,United Kingdo +581538,12/9/2019,23371,Set 36 Colour Pencils Spaceboy,6.19,1,14446,United Kingdom +581538,12/9/2019,22694,Wicker Star,6.19,1,14446,United Kingdom +581538,12/9/2019,22095,Lads Only Tissue Box,6.19,1,14446,United Kingdom +581538,12/9/2019,23275,Set Of 3 Hanging Owls Ollie Beak,6.19,1,14446,United Kingdom +581538,12/9/2019,21673,White Spot Blue Ceramic Drawer Knob,6.19,1,14446,United King +581538,12/9/2019,21670,Blue Spot Ceramic Drawer Knob,6.19,1,14446,United Kingdom +581538,12/9/2019,85071C,"Charlie+Lola""Extremely Busy"" Sign",6.19,1,14446,United K +581538,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,3,14446,United Kingdom +581538,12/9/2019,23034,Drawer Knob Ceramic Black,6.19,1,14446,United Kingdom +581538,12/9/2019,21669,Blue Stripe Ceramic Drawer Knob,6.19,1,14446,United Kingdom +581538,12/9/2019,23033,Drawer Knob Ceramic Red,6.19,1,14446,United Kingdom +581538,12/9/2019,21668,Red Stripe Ceramic Drawer Knob,6.19,1,14446,United Kingdom +581538,12/9/2019,23275,Set Of 3 Hanging Owls Ollie Beak,6.19,1,14446,United Kingdom +581538,12/9/2019,23318,Box Of 6 Mini Vintage Crackers,6.19,1,14446,United Kingdom +581538,12/9/2019,23382,Box Of 6 Christmas Cake Decorations,6.19,1,14446,United King +581538,12/9/2019,22469,Heart Of Wicker Small,6.19,1,14446,United Kingdom +581538,12/9/2019,22899,Children's Apron Dolly Girl,6.19,2,14446,United Kingdom +581538,12/9/2019,22367,Childrens Apron Spaceboy Design,6.19,1,14446,United Kingdom +581538,12/9/2019,22899,Children's Apron Dolly Girl,6.19,3,14446,United Kingdom +581538,12/9/2019,23527,Wall Art Animals And Nature,6.19,1,14446,United Kingdom +581538,12/9/2019,23524,Wall Art Horse & Pony,6.19,1,14446,United Kingdom +581538,12/9/2019,23525,Wall Art Buffalo Bill,6.19,1,14446,United Kingdom +581538,12/9/2019,84380,Set Of 3 Butterfly Cookie Cutters,6.19,4,14446,United Kingdo +581538,12/9/2019,23122,Party Charms 50 Pieces,7.24,1,14446,United Kingdom +581538,12/9/2019,21990,Modern Floral Stationery Set,7.24,1,14446,United Kingdom +581538,12/9/2019,21329,Dinosaurs Writing Set,7.24,1,14446,United Kingdom +581538,12/9/2019,21328,Balloons Writing Set,7.24,1,14446,United Kingdom +581538,12/9/2019,22561,Wooden School Colouring Set,7.24,1,14446,United Kingdom +581538,12/9/2019,84519A,Tomato Charlie+Lola Coaster Set,7.24,1,14446,United Kingdom +581538,12/9/2019,84519B,Carrot Charlie+Lola Coaster Set,7.24,1,14446,United Kingdom +581538,12/9/2019,35004B,Set Of 3 Black Flying Ducks,7.24,2,14446,United Kingdom +581538,12/9/2019,22068,Black Pirate Treasure Chest,7.24,1,14446,United Kingdom +581538,12/9/2019,23353,6 Gift Tags Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,21591,Cosy Hour Cigar Box Matches,6.19,1,14446,United Kingdom +581538,12/9/2019,22197,Popcorn Holder,6.19,4,14446,United Kingdom +581538,12/9/2019,23320,Giant 50'S Christmas Cracker,6.19,1,14446,United Kingdom +581538,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,22985,Wrap Billboard Fonts Design,6.19,25,14446,United Kingdom +581538,12/9/2019,23040,Paper Lantern 9 Point Snow Star,6.19,2,14446,United Kingdom +581538,12/9/2019,23040,Paper Lantern 9 Point Snow Star,6.19,3,14446,United Kingdom +581538,12/9/2019,21194,Pink Honeycomb Paper Fan,6.19,2,14446,United Kingdom +581538,12/9/2019,21208,Pastel Colour Honeycomb Fan,6.19,1,14446,United Kingdom +581538,12/9/2019,79190D,Retro Plastic Daisy Tray,6.19,1,14446,United Kingdom +581538,12/9/2019,79190B,Retro Plastic Polka Tray,6.19,1,14446,United Kingdom +581538,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,6.02,2,14446,United King +581538,12/9/2019,23318,Box Of 6 Mini Vintage Crackers,6.02,1,14446,United Kingdom +581538,12/9/2019,23319,Box Of 6 Mini 50'S Crackers,6.02,1,14446,United Kingdom +581538,12/9/2019,23537,Wall Art I Love London,6.19,1,14446,United Kingdom +581538,12/9/2019,22992,Revolver Wooden Ruler,6.19,1,14446,United Kingdom +581538,12/9/2019,22991,Giraffe Wooden Ruler,6.19,1,14446,United Kingdom +581538,12/9/2019,23190,Bundle Of 3 School Exercise Books,6.19,1,14446,United Kingdo +581538,12/9/2019,21194,Pink Honeycomb Paper Fan,6.19,1,14446,United Kingdom +581538,12/9/2019,35004B,Set Of 3 Black Flying Ducks,6.19,1,14446,United Kingdom +581538,12/9/2019,22694,Wicker Star,6.19,1,14446,United Kingdom +581538,12/9/2019,21355,Toast Its - I Love You,6.19,1,14446,United Kingdom +581538,12/9/2019,23343,Jumbo Bag Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,23343,Jumbo Bag Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,20727,Lunch Bag Black Skull,6.19,1,14446,United Kingdom +581538,12/9/2019,20725,Lunch Bag Red Retrospot,6.19,1,14446,United Kingdom +581566,12/9/2019,23404,Home Sweet Home Blackboard,6.19,144,18102,United Kingdom +581567,12/9/2019,21417,Cockle Shell Dish,6.19,84,16626,United Kingdom +581567,12/9/2019,22464,Hanging Metal Heart Lantern,6.19,24,16626,United Kingdom +581567,12/9/2019,22465,Hanging Metal Star Lantern,6.19,24,16626,United Kingdom +581567,12/9/2019,84971S,Small Heart Flowers Hook,6.19,48,16626,United Kingdom +581567,12/9/2019,22624,Ivory Kitchen Scales,6.19,2,16626,United Kingdom +581567,12/9/2019,22627,Mint Kitchen Scales,6.19,2,16626,United Kingdom +581567,12/9/2019,22625,Red Kitchen Scales,6.19,2,16626,United Kingdom +581567,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,4,16626,United Kingdom +581567,12/9/2019,21326,Aged Glass Silver T-Light Holder,6.19,144,16626,United Kingd +581567,12/9/2019,21479,White Skull Hot Water Bottle,6.19,4,16626,United Kingdom +581567,12/9/2019,23356,Love Hot Water Bottle,6.19,3,16626,United Kingdom +581567,12/9/2019,21137,Black Record Cover Frame,6.19,24,16626,United Kingdom +581570,12/9/2019,22141,Christmas Craft Tree Top Angel,6.19,6,12662,Germany +581570,12/9/2019,22175,Pink Owl Soft Toy,6.19,6,12662,Germany +581570,12/9/2019,21481,Fawn Blue Hot Water Bottle,6.19,4,12662,Germany +581570,12/9/2019,23570,Traditional Pick Up Sticks Game,6.19,12,12662,Germany +581570,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,6.19,6,12662,Germany +581570,12/9/2019,22331,Woodland Party Bag + Sticker Set,6.19,8,12662,Germany +581570,12/9/2019,22834,Hand Warmer Babushka Design,6.19,12,12662,Germany +581570,12/9/2019,21914,Blue Harmonica In Box,6.19,12,12662,Germany +581570,12/9/2019,22139,Retrospot Tea Set Ceramic 11 Pc,6.19,3,12662,Germany +581570,12/9/2019,23077,Doughnut Lip Gloss,6.19,20,12662,Germany +581570,12/9/2019,20750,Red Retrospot Mini Cases,6.19,2,12662,Germany +581570,12/9/2019,22505,Memo Board Cottage Design,6.19,4,12662,Germany +581571,12/9/2019,23326,Hanging Mini Coloured Bottles,6.19,6,15311,United Kingdom +581571,12/9/2019,21313,Glass Heart T-Light Holder,6.19,1,15311,United Kingdom +581571,12/9/2019,48187,Doormat New England,6.19,1,15311,United Kingdom +581571,12/9/2019,23317,Blue Refectory Clock,6.19,1,15311,United Kingdom +581571,12/9/2019,23197,Sketchbook Magnetic Shopping List,6.19,4,15311,United Kingdo +581571,12/9/2019,21012,Antique All Glass Candlestick,6.19,2,15311,United Kingdom +581571,12/9/2019,22227,Hanging Heart Mirror Decoration,6.19,10,15311,United Kingdom +581571,12/9/2019,22794,Sweetheart Wire Magazine Rack,6.19,1,15311,United Kingdom +581571,12/9/2019,23182,Toilet Sign Occupied Or Vacant,6.19,1,15311,United Kingdom +581571,12/9/2019,21755,Love Building Block Word,6.19,1,15311,United Kingdom +581571,12/9/2019,85053,French Enamel Candleholder,6.19,2,15311,United Kingdom +581571,12/9/2019,23110,Parisienne Key Cabinet,6.19,2,15311,United Kingdom +581571,12/9/2019,21169,You're Confusing Me Metal Sign,6.19,1,15311,United Kingdom +581571,12/9/2019,21258,Victorian Sewing Box Large,6.19,8,15311,United Kingdom +581571,12/9/2019,23168,Classic Cafe Sugar Dispenser,6.19,36,15311,United Kingdom +581571,12/9/2019,23167,Small Ceramic Top Storage Jar,6.19,96,15311,United Kingdom +581571,12/9/2019,21314,Small Glass Heart Trinket Pot,6.19,48,15311,United Kingdom +581571,12/9/2019,21137,Black Record Cover Frame,6.19,24,15311,United Kingdom +581571,12/9/2019,44234,Assorted Circular Mobile,6.19,1,15311,United Kingdom +581571,12/9/2019,84347,Rotating Silver Angels T-Light Hldr,6.19,24,15311,United Kin +581572,12/9/2019,23328,Set 6 School Milk Bottles In Crate,6.19,48,16705,United King +581572,12/9/2019,22627,Mint Kitchen Scales,6.19,4,16705,United Kingdom +581572,12/9/2019,22624,Ivory Kitchen Scales,6.19,4,16705,United Kingdom +581572,12/9/2019,23245,Set Of 3 Regency Cake Tins,6.19,4,16705,United Kingdom +581574,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,6.19,6,12526,Germany +581574,12/9/2019,22328,Round Snack Boxes Set Of 4 Fruits,6.19,6,12526,Germany +581574,12/9/2019,23238,Set Of 4 Knick Knack Tins London,6.19,6,12526,Germany +581574,12/9/2019,23237,Set Of 4 Knick Knack Tins Leaf,6.19,6,12526,Germany +581574,12/9/2019,21257,Victorian Sewing Box Medium,6.19,2,12526,Germany +581574,12/9/2019,21258,Victorian Sewing Box Large,6.19,2,12526,Germany +581574,12/9/2019,23111,Parisienne Sewing Box,6.19,2,12526,Germany +581574,12/9/2019,22077,6 Ribbons Rustic Charm,6.19,12,12526,Germany +581574,12/9/2019,22074,6 Ribbons Shimmering Pinks,6.19,12,12526,Germany +581574,12/9/2019,22621,Traditional Knitting Nancy,6.19,12,12526,Germany +581574,12/9/2019,23199,Jumbo Bag Apples,6.19,10,12526,Germany +581574,12/9/2019,23581,Jumbo Bag Paisley Park,6.19,10,12526,Germany +581578,12/9/2019,21124,Set/10 Blue Polkadot Party Candles,6.19,24,12713,Germany +581578,12/9/2019,21122,Set/10 Pink Polkadot Party Candles,6.19,24,12713,Germany +581578,12/9/2019,21121,Set/10 Red Polkadot Party Candles,6.19,24,12713,Germany +581578,12/9/2019,23389,Spaceboy Mini Backpack,6.04,4,12713,Germany +581578,12/9/2019,22556,Plasters In Tin Circus Parade,6.19,12,12713,Germany +581578,12/9/2019,22976,Circus Parade Childrens Egg Cup,6.19,12,12713,Germany +581578,12/9/2019,23255,Childrens Cutlery Circus Parade,7.24,12,12713,Germany +581578,12/9/2019,84997D,Childrens Cutlery Polkadot Pink,7.24,8,12713,Germany +581578,12/9/2019,84997B,Childrens Cutlery Retrospot Red,7.24,8,12713,Germany +581578,12/9/2019,84997C,Childrens Cutlery Polkadot Blue,7.24,8,12713,Germany +581578,12/9/2019,22555,Plasters In Tin Strongman,7.24,12,12713,Germany +581578,12/9/2019,21914,Blue Harmonica In Box,7.24,12,12713,Germany +581578,12/9/2019,22549,Picture Dominoes,7.24,24,12713,Germany +581578,12/9/2019,21918,Set 12 Kids Colour Chalk Sticks,7.24,24,12713,Germany +581578,12/9/2019,22992,Revolver Wooden Ruler,7.24,12,12713,Germany +581578,12/9/2019,22991,Giraffe Wooden Ruler,7.24,12,12713,Germany +581578,12/9/2019,23229,Vintage Donkey Tail Game,7.24,6,12713,Germany +581578,12/9/2019,22622,Box Of Vintage Alphabet Blocks,6.19,6,12713,Germany +581578,12/9/2019,21506,Fancy Font Birthday Card,6.19,12,12713,Germany +581578,12/9/2019,21507,Elephant Birthday Card,6.19,12,12713,Germany +581578,12/9/2019,23037,Candle Holder Silver Madeline,6.19,12,12713,Germany +581578,12/9/2019,23550,Wrap Alphabet Poster,6.19,25,12713,Germany +581578,12/9/2019,22711,Wrap Circus Parade,6.19,25,12713,Germany +581578,12/9/2019,21497,Fancy Fonts Birthday Wrap,6.19,25,12713,Germany +581578,12/9/2019,22704,Wrap Red Apples,6.19,25,12713,Germany +581578,12/9/2019,22585,Pack Of 6 Birdy Gift Tags,6.19,12,12713,Germany diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/java-solution/Task.java b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/java-solution/Task.java new file mode 100644 index 0000000000000..e15c9167342dc --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/java-solution/Task.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// beam-playground: +// name: FinalSolution1 +// description: Final challenge solution 1. +// multifile: true +// files: +// - name: input.csv +// context_line: 50 +// categories: +// - Quickstart +// complexity: ADVANCED +// tags: +// - hellobeam + +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.io.TextIO; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.schemas.JavaFieldSchema; +import org.apache.beam.sdk.schemas.annotations.DefaultSchema; +import org.apache.beam.sdk.schemas.annotations.SchemaCreate; +import org.apache.beam.sdk.transforms.Combine; +import org.apache.beam.sdk.transforms.Filter; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.Partition; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime; +import org.apache.beam.sdk.transforms.windowing.FixedWindows; +import org.apache.beam.sdk.transforms.windowing.Trigger; +import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.transforms.windowing.Repeatedly; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionList; +import org.apache.beam.sdk.values.TypeDescriptor; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.TypeDescriptors; +import org.joda.time.Duration; + +import java.io.*; +import java.util.List; +import java.util.stream.Collectors; + +public class Task { + private static final Integer WINDOW_TIME = 30; + private static final Integer TIME_OUTPUT_AFTER_FIRST_ELEMENT = 5; + private static final Integer ALLOWED_LATENESS_TIME = 1; + + private static final String REGEX_FOR_CSV = ",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)"; + + public static void main(String[] args) { + PipelineOptions options = PipelineOptionsFactory.fromArgs(args).create(); + Pipeline pipeline = Pipeline.create(options); + + Window window = Window.into(FixedWindows.of(Duration.standardSeconds(WINDOW_TIME))); + + Trigger trigger = AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.standardSeconds(TIME_OUTPUT_AFTER_FIRST_ELEMENT)); + + PCollection input = pipeline.apply(TextIO.read().from("input.csv")) + .apply("Data", ParDo.of(new ExtractDataFn())) + .setCoder(TransactionSerializableCoder.of()); + + PCollectionList parts = input + .apply(window.triggering(Repeatedly.forever(trigger)).withAllowedLateness(Duration.standardMinutes(ALLOWED_LATENESS_TIME)).discardingFiredPanes()) + .apply(Filter.by(it -> it.quantity >= 20)) + .apply(new TransactionPartitionFn()); + + parts.get(0) + .apply(MapElements.into(TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.doubles())).via(it -> KV.of(it.productId, it.price))) + .apply(Combine.perKey(new SumDoubleBinaryCombineFn())) + .apply(MapElements.into(TypeDescriptor.of(String.class)).via(it -> it.toString())) + .apply("WriteToFile", TextIO.write().withoutSharding().to("price_more_than_10").withSuffix(".txt")); + + parts.get(1) + .apply(MapElements.into(TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.doubles())).via(it -> KV.of(it.productId, it.price))) + .apply(Combine.perKey(new SumDoubleBinaryCombineFn())) + .apply(MapElements.into(TypeDescriptor.of(String.class)).via(it -> it.toString())) + .apply("WriteToFile", TextIO.write().withoutSharding().to("price_less_than_10").withSuffix(".txt")); + + pipeline.run().waitUntilFinish(); + } + + static class SumDoubleBinaryCombineFn extends Combine.BinaryCombineFn { + @Override + public Double apply(Double left, Double right) { + return left + right; + } + + } + + static class TransactionPartitionFn extends PTransform, PCollectionList> { + @Override + public PCollectionList expand(PCollection input) { + return input.apply(Partition.of(2, + (Partition.PartitionFn) (transaction, numPartitions) -> { + if (transaction.price > 10) { + return 0; + } else { + return 1; + } + })); + } + } + + static class TransactionSerializableCoder extends Coder { + private static final TransactionSerializableCoder INSTANCE = new TransactionSerializableCoder(); + + public static TransactionSerializableCoder of() { + return INSTANCE; + } + + private static final String NAMES_SEPARATOR = "_"; + + @Override + public void encode(Transaction transaction, OutputStream outStream) throws IOException { + String serializableRecord = transaction.id + NAMES_SEPARATOR + transaction.date + NAMES_SEPARATOR + transaction.productId + + NAMES_SEPARATOR + transaction.productName + NAMES_SEPARATOR + transaction.price + NAMES_SEPARATOR + transaction.quantity + + NAMES_SEPARATOR + transaction.customerId + NAMES_SEPARATOR + transaction.country; + outStream.write(serializableRecord.getBytes()); + } + + @Override + public Transaction decode(InputStream inStream) { + String serializedRecord = new BufferedReader(new InputStreamReader(inStream)).lines() + .parallel().collect(Collectors.joining("\n")); + String[] items = serializedRecord.split(NAMES_SEPARATOR); + return new Transaction(Long.valueOf(items[0]), items[1], items[2], items[3], Double.valueOf(items[4]), Integer.parseInt(items[5]), Long.valueOf(items[6]), items[7]); + } + + @Override + public List> getCoderArguments() { + return null; + } + + @Override + public void verifyDeterministic() { + } + } + + + static class ExtractDataFn extends DoFn { + @ProcessElement + public void processElement(ProcessContext c) { + String[] items = c.element().split(REGEX_FOR_CSV); + try { + c.output(new Transaction(Long.valueOf(items[0]), items[1], items[2], items[3], Double.valueOf(items[4]), Integer.parseInt(items[5]), Long.valueOf(items[6]), items[7])); + } catch (Exception e) { + System.out.println("Skip header"); + } + } + } + + @DefaultSchema(JavaFieldSchema.class) + public static class Transaction { + public Long id; + public String date; + public String productId; + public String productName; + public Double price; + public Integer quantity; + public Long customerId; + public String country; + + public Transaction() { + } + + @SchemaCreate + public Transaction(Long id, String date, String productId, String productName, Double price, Integer quantity, Long customerId, String country) { + this.id = id; + this.date = date; + this.productId = productId; + this.productName = productName; + this.price = price; + this.quantity = quantity; + this.customerId = customerId; + this.country = country; + } + + @Override + public String toString() { + return "Transaction{" + + "id=" + id + + ", date='" + date + '\'' + + ", productId='" + productId + '\'' + + ", productName='" + productName + '\'' + + ", price='" + price + '\'' + + ", quantity=" + quantity + + ", customerId=" + customerId + + ", country='" + country + '\'' + + '}'; + } + } +} \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/java-solution/input.csv b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/java-solution/input.csv new file mode 100644 index 0000000000000..85854a2d43584 --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/java-solution/input.csv @@ -0,0 +1,1500 @@ +TransactionNo,Date,ProductNo,ProductName,Price,Quantity,CustomerNo,Country +581482,12/9/2019,22485,Set Of 2 Wooden Market Crates,21.47,12,17490,United Kingdom +581475,12/9/2019,22596,Christmas Star Wish List Chalkboard,10.65,36,13069,United Ki +581475,12/9/2019,23235,Storage Tin Vintage Leaf,11.53,12,13069,United Kingdom +581475,12/9/2019,23272,Tree T-Light Holder Willie Winkie,10.65,12,13069,United King +581475,12/9/2019,23239,Set Of 4 Knick Knack Tins Poppies,11.94,6,13069,United Kingd +581475,12/9/2019,21705,Bag 500g Swirly Marbles,10.65,24,13069,United Kingdom +581475,12/9/2019,22118,Joy Wooden Block Letters,11.53,18,13069,United Kingdom +581475,12/9/2019,22119,Peace Wooden Block Letters,12.25,12,13069,United Kingdom +581475,12/9/2019,22217,T-Light Holder Hanging Lace,10.65,12,13069,United Kingdom +581475,12/9/2019,22216,T-Light Holder White Lace,10.55,24,13069,United Kingdom +581475,12/9/2019,22380,Toy Tidy Spaceboy,11.06,20,13069,United Kingdom +581475,12/9/2019,22442,Grow Your Own Flowers Set Of 3,12.25,12,13069,United Kingdom +581475,12/9/2019,22664,Toy Tidy Dolly Girl Design,11.06,20,13069,United Kingdom +581475,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,12.25,12,13069,United Kingdom +581475,12/9/2019,22723,Set Of 6 Herb Tins Sketchbook,11.53,12,13069,United Kingdom +581475,12/9/2019,22785,Squarecushion Cover Pink Union Jack,11.53,12,13069,United Ki +581475,12/9/2019,22955,36 Foil Star Cake Cases,11.06,24,13069,United Kingdom +581475,12/9/2019,23141,Triple Wire Hook Pink Heart,11.06,12,13069,United Kingdom +581475,12/9/2019,22956,36 Foil Heart Cake Cases,11.06,24,13069,United Kingdom +581475,12/9/2019,22581,Wood Stocking Christmas Scandispot,10.55,48,13069,United Kin +581476,12/9/2019,23198,Pantry Magnetic Shopping List,11.53,48,12433,Norway +581476,12/9/2019,23197,Sketchbook Magnetic Shopping List,11.74,24,12433,Norway +581476,12/9/2019,23184,Bull Dog Bottle Opener,15.32,8,12433,Norway +581476,12/9/2019,23168,Classic Cafe Sugar Dispenser,11.53,12,12433,Norway +581476,12/9/2019,23167,Small Ceramic Top Storage Jar,10.96,96,12433,Norway +581476,12/9/2019,23166,Medium Ceramic Top Storage Jar,11.32,48,12433,Norway +581476,12/9/2019,23165,Large Ceramic Top Storage Jar,11.74,48,12433,Norway +581476,12/9/2019,23004,Travel Card Wallet Pantry,10.68,48,12433,Norway +581476,12/9/2019,23002,Travel Card Wallet Skulls,10.68,24,12433,Norway +581476,12/9/2019,23000,Travel Card Wallet Transport,10.68,24,12433,Norway +581476,12/9/2019,22998,Travel Card Wallet Keep Calm,10.68,72,12433,Norway +581476,12/9/2019,22994,Travel Card Wallet Retrospot,10.68,48,12433,Norway +581476,12/9/2019,22835,Hot Water Bottle I Am So Poorly,15.32,8,12433,Norway +581476,12/9/2019,22730,Alarm Clock Bakelike Ivory,14.09,4,12433,Norway +581476,12/9/2019,22728,Alarm Clock Bakelike Pink,14.09,8,12433,Norway +581476,12/9/2019,22727,Alarm Clock Bakelike Red,14.09,8,12433,Norway +581476,12/9/2019,22726,Alarm Clock Bakelike Green,14.09,4,12433,Norway +581476,12/9/2019,22720,Set Of 3 Cake Tins Pantry Design,14.61,16,12433,Norway +581476,12/9/2019,22693,Grow A Flytrap Or Sunflower In Tin,11.34,192,12433,Norway +581476,12/9/2019,22670,French Wc Sign Blue Metal,11.53,24,12433,Norway +581476,12/9/2019,22667,Recipe Box Retrospot,12.86,24,12433,Norway +581476,12/9/2019,22666,Recipe Box Pantry Yellow Design,12.86,24,12433,Norway +581476,12/9/2019,22631,Circus Parade Lunch Box,12.25,12,12433,Norway +581476,12/9/2019,22628,Picnic Boxes Set Of 3 Retrospot,15.32,16,12433,Norway +581476,12/9/2019,22467,Gumball Coat Rack,12.86,6,12433,Norway +581476,12/9/2019,22197,Popcorn Holder,10.99,100,12433,Norway +581476,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,14.61,8,12433,Norway +581476,12/9/2019,22112,Chocolate Hot Water Bottle,15.32,9,12433,Norway +581476,12/9/2019,21908,Chocolate This Way Metal Sign,12.40,36,12433,Norway +581476,12/9/2019,21874,Gin And Tonic Mug,11.74,36,12433,Norway +581476,12/9/2019,21872,Glamorous Mug,11.53,12,12433,Norway +581476,12/9/2019,21871,Save The Planet Mug,11.74,36,12433,Norway +581476,12/9/2019,21533,Retrospot Large Milk Jug,15.32,6,12433,Norway +581476,12/9/2019,21481,Fawn Blue Hot Water Bottle,14.09,12,12433,Norway +581476,12/9/2019,21479,White Skull Hot Water Bottle,14.61,4,12433,Norway +581476,12/9/2019,21248,Door Hanger Mum + Dads Room,11.74,12,12433,Norway +581476,12/9/2019,21216,Set 3 Retrospot Tea/Coffee/Sugar,14.61,24,12433,Norway +581476,12/9/2019,21181,Please One Person Metal Sign,12.15,48,12433,Norway +581476,12/9/2019,21175,Gin And Tonic Diet Metal Sign,12.38,48,12433,Norway +581476,12/9/2019,21169,You're Confusing Me Metal Sign,11.74,48,12433,Norway +581476,12/9/2019,21162,Toxic Area Door Hanger,10.65,24,12433,Norway +581476,12/9/2019,21159,Moody Boy Door Hanger,10.65,24,12433,Norway +581476,12/9/2019,21158,Moody Girl Door Hanger,10.65,24,12433,Norway +581476,12/9/2019,21154,Red Retrospot Oven Glove,11.53,10,12433,Norway +581476,12/9/2019,16016,Large Chinese Style Scissor,11.12,40,12433,Norway +581476,12/9/2019,16014,Small Chinese Style Scissor,10.68,60,12433,Norway +581476,12/9/2019,16008,Small Folding Scissor(Pointed Edge),10.37,240,12433,Norway +581476,12/9/2019,85152,Hand Over The Chocolate Sign,12.15,48,12433,Norway +581476,12/9/2019,84596F,Small Marshmallows Pink Bowl,10.68,32,12433,Norway +581476,12/9/2019,84596B,Small Dolly Mix Design Orange Bowl,10.68,16,12433,Norway +581476,12/9/2019,84510A,Set Of 4 English Rose Coasters,11.53,20,12433,Norway +581476,12/9/2019,82600,N0 Singing Metal Sign,12.15,48,12433,Norway +581476,12/9/2019,82581,Toilet Metal Sign,10.81,48,12433,Norway +581476,12/9/2019,72232,Feng Shui Pillar Candle,10.44,144,12433,Norway +581476,12/9/2019,47559B,Tea Time Oven Glove,11.53,10,12433,Norway +581476,12/9/2019,47504H,English Rose Spirit Level,11.06,36,12433,Norway +581476,12/9/2019,23493,Vintage Doily Travel Sewing Kit,12.25,30,12433,Norway +581476,12/9/2019,23430,Blue Retro Kitchen Wall Clock,18.60,2,12433,Norway +581476,12/9/2019,23429,Red Retro Kitchen Wall Clock,18.60,2,12433,Norway +581476,12/9/2019,23428,Ivory Retro Kitchen Wall Clock,18.60,2,12433,Norway +581476,12/9/2019,23358,Hot Stuff Hot Water Bottle,11.53,18,12433,Norway +581476,12/9/2019,23293,Set Of 12 Fairy Cake Baking Cases,11.10,32,12433,Norway +581476,12/9/2019,23243,Set Of Tea Coffee Sugar Tins Pantry,15.32,12,12433,Norway +581476,12/9/2019,23240,Set Of 4 Knick Knack Tins Doily,14.50,6,12433,Norway +581477,12/9/2019,48111,Doormat 3 Smiley Cats,17.51,10,13426,United Kingdom +581477,12/9/2019,22464,Hanging Metal Heart Lantern,11.06,12,13426,United Kingdom +581477,12/9/2019,20982,12 Pencils Tall Tube Skulls,11.12,12,13426,United Kingdom +581477,12/9/2019,20981,12 Pencils Tall Tube Woodland,11.12,12,13426,United Kingdom +581477,12/9/2019,23424,Gingham Recipe Book Box,15.32,8,13426,United Kingdom +581477,12/9/2019,23338,Egg Frying Pan Red,12.15,48,13426,United Kingdom +581477,12/9/2019,84970L,Single Heart Zinc T-Light Holder,11.53,12,13426,United King +581477,12/9/2019,22457,Natural Slate Heart Chalkboard,13.27,6,13426,United Kingdom +581477,12/9/2019,22469,Heart Of Wicker Small,11.94,12,13426,United Kingdom +581477,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,11.12,12,13426,United Ki +581478,12/9/2019,84947,Antique Silver Tea Glass Engraved,11.53,24,17364,United King +581478,12/9/2019,23503,Playing Cards Keep Calm & Carry On,11.53,12,17364,United Kin +581478,12/9/2019,23445,Ice Cream Bubbles,11.10,20,17364,United Kingdom +581478,12/9/2019,23530,Wall Art Only One Person,15.32,12,17364,United Kingdom +581478,12/9/2019,84997C,Childrens Cutlery Polkadot Blue,14.50,4,17364,United Kingdo +581478,12/9/2019,84997D,Childrens Cutlery Polkadot Pink,14.50,4,17364,United Kingdo +581478,12/9/2019,84077,World War 2 Gliders Asstd Designs,10.55,48,17364,United King +581478,12/9/2019,22749,Feltcraft Princess Charlotte Doll,14.09,4,17364,United Kingd +581478,12/9/2019,23127,Feltcraft Girl Nicole Kit,15.32,4,17364,United Kingdom +581478,12/9/2019,23126,Feltcraft Girl Amelie Kit,15.32,4,17364,United Kingdom +581478,12/9/2019,22747,Poppy's Playhouse Bathroom,12.40,6,17364,United Kingdom +581478,12/9/2019,22078,Ribbon Reel Lace Design,12.40,10,17364,United Kingdom +581478,12/9/2019,84946,Antique Silver T-Light Glass,11.53,12,17364,United Kingdom +581478,12/9/2019,22791,T-Light Glass Fluted Antique,11.53,12,17364,United Kingdom +581478,12/9/2019,21326,Aged Glass Silver T-Light Holder,10.92,12,17364,United Kingd +581478,12/9/2019,22170,Picture Frame Wood Triple Portrait,17.17,8,17364,United King +581478,12/9/2019,23082,Set 6 Paper Table Lantern Hearts,14.09,6,17364,United Kingdo +581478,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,11.12,24,17364,United Ki +581479,12/9/2019,22087,Paper Bunting White Lace,13.27,10,17364,United Kingdom +581480,12/9/2019,23464,Vintage Zinc Watering Can Small,15.32,4,14441,United Kingdom +581480,12/9/2019,22411,Jumbo Shopper Vintage Red Paisley,12.38,10,14441,United King +581480,12/9/2019,84029E,Red Woolly Hottie White Heart,14.61,8,14441,United Kingdom +581480,12/9/2019,22633,Hand Warmer Union Jack,12.40,12,14441,United Kingdom +581480,12/9/2019,23355,Hot Water Bottle Keep Calm,15.32,12,14441,United Kingdom +581480,12/9/2019,84029G,Knitted Union Flag Hot Water Bottle,14.61,12,14441,United K +581480,12/9/2019,22112,Chocolate Hot Water Bottle,15.32,6,14441,United Kingdom +581480,12/9/2019,84347,Rotating Silver Angels T-Light Hldr,12.86,18,14441,United Ki +581481,12/9/2019,21115,Rose Caravan Doorstop,12.25,8,17490,United Kingdom +581481,12/9/2019,22059,Ceramic Strawberry Design Mug,10.65,24,17490,United Kingdom +581481,12/9/2019,22072,Red Retrospot Tea Cup And Saucer,11.53,24,17490,United Kingd +581481,12/9/2019,22123,Ping Microwave Apron,11.06,24,17490,United Kingdom +581481,12/9/2019,22476,Empire Union Jack Tv Dinner Tray,12.25,8,17490,United Kingdo +581481,12/9/2019,22495,Set Of 2 Round Tins Camembert,11.06,12,17490,United Kingdom +581481,12/9/2019,22496,Set Of 2 Round Tins Dutch Cheese,11.06,12,17490,United Kingd +581481,12/9/2019,22513,Doorstop Football Design,11.06,8,17490,United Kingdom +581481,12/9/2019,22500,Set Of 2 Tins Jardin De Provence,11.53,12,17490,United Kingd +581481,12/9/2019,22664,Toy Tidy Dolly Girl Design,11.06,20,17490,United Kingdom +581481,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,12.25,12,17490,United Kingdom +581481,12/9/2019,22785,Squarecushion Cover Pink Union Jack,11.53,12,17490,United Ki +581481,12/9/2019,23178,Jam Clock Magnet,11.53,12,17490,United Kingdom +581481,12/9/2019,23302,Kneeling Mat Housework Design,11.06,24,17490,United Kingdom +581481,12/9/2019,23533,Wall Art Garden Haven,12.25,6,17490,United Kingdom +581481,12/9/2019,84819,Danish Rose Round Sewing Box,11.06,16,17490,United Kingdom +581482,12/9/2019,22371,Airline Bag Vintage Tokyo 78,14.30,12,17490,United Kingdom +581482,12/9/2019,21875,Kings Choice Mug,11.34,36,17490,United Kingdom +581482,12/9/2019,23251,Vintage Red Enamel Trim Mug,11.32,96,17490,United Kingdom +581482,12/9/2019,23300,Gardeners Kneeling Pad Cup Of Tea,11.74,48,17490,United King +581482,12/9/2019,22138,Baking Set 9 Piece Retrospot,14.61,24,17490,United Kingdom +581483,12/9/2019,23843,Paper Craft Little Birdie,12.38,80995,16446,United Kingdom +581485,12/9/2019,22617,Baking Set Spaceboy Design,15.32,18,17389,United Kingdom +581485,12/9/2019,20749,Assorted Colour Mini Cases,16.76,84,17389,United Kingdom +581486,12/9/2019,22910,Paper Chain Kit Vintage Christmas,13.27,12,17001,United King +581486,12/9/2019,22086,Paper Chain Kit 50'S Christmas,13.27,12,17001,United Kingdom +581486,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,12,17001,United Kingdom +581486,12/9/2019,23352,Roll Wrap 50'S Red Christmas,6.19,12,17001,United Kingdom +581486,12/9/2019,22727,Alarm Clock Bakelike Red,6.19,8,17001,United Kingdom +581486,12/9/2019,22730,Alarm Clock Bakelike Ivory,6.19,4,17001,United Kingdom +581486,12/9/2019,22491,Pack Of 12 Coloured Pencils,6.04,12,17001,United Kingdom +581486,12/9/2019,22561,Wooden School Colouring Set,6.04,12,17001,United Kingdom +581486,12/9/2019,22489,Pack Of 12 Traditional Crayons,6.04,24,17001,United Kingdom +581486,12/9/2019,22560,Traditional Modelling Clay,6.04,24,17001,United Kingdom +581486,12/9/2019,22139,Retrospot Tea Set Ceramic 11 Pc,6.04,6,17001,United Kingdom +581486,12/9/2019,23344,Jumbo Bag 50'S Christmas,6.19,20,17001,United Kingdom +581486,12/9/2019,23203,Jumbo Bag Vintage Doily,6.19,20,17001,United Kingdom +581486,12/9/2019,85099B,Jumbo Bag Red Retrospot,6.19,10,17001,United Kingdom +581486,12/9/2019,23201,Jumbo Bag Alphabet,6.19,20,17001,United Kingdom +581486,12/9/2019,23207,Lunch Bag Alphabet Design,6.19,10,17001,United Kingdom +581487,12/9/2019,21137,Black Record Cover Frame,6.04,120,15694,United Kingdom +581488,12/9/2019,23118,Parisienne Jewellery Drawer,6.04,16,17428,United Kingdom +581488,12/9/2019,23111,Parisienne Sewing Box,6.04,16,17428,United Kingdom +581488,12/9/2019,22179,Set 10 Night Owl Lights,6.04,24,17428,United Kingdom +581489,12/9/2019,22061,Large Cake Stand Hanging Strawbery,7.24,48,16954,United King +581489,12/9/2019,22182,Cake Stand Victorian Filigree Small,7.24,24,16954,United Kin +581491,12/9/2019,23571,Traditional Naughts & Crosses,7.24,12,12433,Norway +581492,12/9/2019,23369,Set 36 Colour Pencils Love London,6.19,2,15492,United Kingdo +581492,12/9/2019,23370,Set 36 Colouring Pencils Doily,6.19,2,15492,United Kingdom +581492,12/9/2019,23371,Set 36 Colour Pencils Spaceboy,6.19,3,15492,United Kingdom +581492,12/9/2019,23372,Set 36 Colour Pencils Dolly Girl,6.19,1,15492,United Kingdom +581492,12/9/2019,23376,Pack Of 12 Vintage Christmas Tissue,6.19,3,15492,United King +581492,12/9/2019,23377,Pack Of 12 Dolly Girl Tissues,6.19,6,15492,United Kingdom +581492,12/9/2019,23378,Pack Of 12 50'S Christmas Tissues,6.19,9,15492,United Kingdo +581492,12/9/2019,23379,Pack Of 12 Red Apple Tissues,6.19,4,15492,United Kingdom +581492,12/9/2019,23380,Pack Of 12 Vintage Doily Tissues,6.19,3,15492,United Kingdom +581492,12/9/2019,23381,Pack Of 12 Vintage Leaf Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,23382,Box Of 6 Christmas Cake Decorations,6.19,4,15492,United King +581492,12/9/2019,23391,I Love London Mini Backpack,6.19,2,15492,United Kingdom +581492,12/9/2019,23392,Spaceboy Rocket Lolly Makers,6.19,2,15492,United Kingdom +581492,12/9/2019,23399,Home Sweet Home Hanging Heart,6.19,4,15492,United Kingdom +581492,12/9/2019,23405,Home Sweet Home 2 Drawer Cabinet,6.19,3,15492,United Kingdom +581492,12/9/2019,23418,Lavender Toilette Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,23426,Metal Sign Drop Your Pants,6.19,1,15492,United Kingdom +581492,12/9/2019,23434,3 Raffia Ribbons 50'S Christmas,6.19,9,15492,United Kingdom +581492,12/9/2019,23435,3 Raffia Ribbons Vintage Christmas,6.04,3,15492,United Kingd +581492,12/9/2019,23439,Hand Warmer Red Love Heart,6.19,1,15492,United Kingdom +581492,12/9/2019,23451,Square Mini Portrait Frame,6.19,1,15492,United Kingdom +581492,12/9/2019,23467,Vintage Zinc Planter,6.19,1,15492,United Kingdom +581492,12/9/2019,23469,Card Holder Love Bird Small,6.19,3,15492,United Kingdom +581492,12/9/2019,23480,Mini Lights Woodland Mushrooms,6.19,2,15492,United Kingdom +581492,12/9/2019,23493,Vintage Doily Travel Sewing Kit,6.19,3,15492,United Kingdom +581492,12/9/2019,23494,Vintage Doily Deluxe Sewing Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,23495,Set Of 3 Pantry Wooden Spoons,6.19,1,15492,United Kingdom +581492,12/9/2019,23497,Classic Chrome Bicycle Bell,6.19,4,15492,United Kingdom +581492,12/9/2019,23498,Classic Bicycle Clips,6.19,2,15492,United Kingdom +581492,12/9/2019,23501,Key Ring Baseball Boot Union Jack,6.19,3,15492,United Kingdo +581492,12/9/2019,23506,Mini Playing Cards Spaceboy,6.04,1,15492,United Kingdom +581492,12/9/2019,23508,Mini Playing Cards Dolly Girl,6.04,2,15492,United Kingdom +581492,12/9/2019,23510,Mini Playing Cards Gymkhana,6.04,1,15492,United Kingdom +581492,12/9/2019,23521,Wall Art Cat And Bird,6.04,1,15492,United Kingdom +581492,12/9/2019,23526,Wall Art Dog Licence,6.04,4,15492,United Kingdom +581492,12/9/2019,23530,Wall Art Only One Person,6.04,2,15492,United Kingdom +581492,12/9/2019,23534,Wall Art Stop For Tea,6.19,6,15492,United Kingdom +581492,12/9/2019,23535,Wall Art Bicycle Safety,6.19,4,15492,United Kingdom +581492,12/9/2019,23536,Wall Art Village Show,6.19,1,15492,United Kingdom +581492,12/9/2019,23538,Wall Art Vintage Heart,6.19,1,15492,United Kingdom +581492,12/9/2019,23551,Pack Of 12 Paisley Park Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,23552,Bicycle Puncture Repair Kit,6.19,12,15492,United Kingdom +581492,12/9/2019,23555,Landmark Frame Notting Hill,6.19,1,15492,United Kingdom +581492,12/9/2019,23559,Woodland Bunnies Lolly Makers,6.19,5,15492,United Kingdom +581492,12/9/2019,23561,Set Of 6 Ribbons Party,6.19,1,15492,United Kingdom +581492,12/9/2019,23564,Egg Cup Milkmaid Ingrid,6.19,2,15492,United Kingdom +581492,12/9/2019,23565,Egg Cup Milkmaid Helga,6.19,2,15492,United Kingdom +581492,12/9/2019,23567,Egg Cup Henrietta Hen Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23569,Tradtional Alphabet Stamp Set,6.19,2,15492,United Kingdom +581492,12/9/2019,23570,Traditional Pick Up Sticks Game,6.19,7,15492,United Kingdom +581492,12/9/2019,23571,Traditional Naughts & Crosses,6.19,2,15492,United Kingdom +581492,12/9/2019,23575,Snack Tray Paisley Park,6.19,3,15492,United Kingdom +581492,12/9/2019,23579,Snack Tray I Love London,6.19,1,15492,United Kingdom +581492,12/9/2019,23611,Set 10 Cards Red Riding Hood 17214,6.19,3,15492,United Kingd +581492,12/9/2019,23616,Set 10 Cards Jingle Bells 17217,6.19,1,15492,United Kingdom +581492,12/9/2019,23621,Set 10 Cards David's Madonna 17074,6.19,3,15492,United Kingd +581492,12/9/2019,23635,Set 10 Cards Christmas Holly 17259,6.19,1,15492,United Kingd +581492,12/9/2019,23644,Set 10 Cards Christmas Tree 16955,6.19,1,15492,United Kingdo +581492,12/9/2019,23660,Henrietta Hen Mug,6.19,1,15492,United Kingdom +581492,12/9/2019,23661,Milk Maids Mug,6.19,1,15492,United Kingdom +581492,12/9/2019,35095A,Blue Victorian Fabric Oval Box,6.19,1,15492,United Kingdom +581492,12/9/2019,35095B,Red Victorian Fabric Oval Box,6.19,1,15492,United Kingdom +581492,12/9/2019,35646,Vintage Bead Pink Evening Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,35648,Vintage Bead Pink Purse,6.19,2,15492,United Kingdom +581492,12/9/2019,35953,Folkart Star Christmas Decorations,6.19,12,15492,United King +581492,12/9/2019,35964,Folkart Clip On Stars,6.19,4,15492,United Kingdom +581492,12/9/2019,35970,Zinc Folkart Sleigh Bells,6.04,1,15492,United Kingdom +581492,12/9/2019,46118,Funky Monkey Cushion Cover,6.19,1,15492,United Kingdom +581492,12/9/2019,46776A,Woven Bubble Gum Cushion Cover,7.24,2,15492,United Kingdom +581492,12/9/2019,46776B,Woven Berries Cushion Cover,7.24,3,15492,United Kingdom +581492,12/9/2019,46776C,Woven Frost Cushion Cover,7.24,1,15492,United Kingdom +581492,12/9/2019,46776D,Woven Sunset Cushion Cover,7.24,2,15492,United Kingdom +581492,12/9/2019,46776E,Woven Candy Cushion Cover,7.24,5,15492,United Kingdom +581492,12/9/2019,46776F,Woven Rose Garden Cushion Cover,7.24,6,15492,United Kingdom +581492,12/9/2019,47310M,Small Pop Box Funky Monkey,6.19,1,15492,United Kingdom +581492,12/9/2019,47480,Hanging Photo Clip Rope Ladder,6.19,3,15492,United Kingdom +581492,12/9/2019,47504H,English Rose Spirit Level,7.24,1,15492,United Kingdom +581492,12/9/2019,47559B,Tea Time Oven Glove,7.24,3,15492,United Kingdom +581492,12/9/2019,47563A,Retro Longboard Ironing Board Cover,7.24,2,15492,United Kin +581492,12/9/2019,47566,Party Bunting,7.24,2,15492,United Kingdom +581492,12/9/2019,47590B,Pink Happy Birthday Bunting,7.24,1,15492,United Kingdom +581492,12/9/2019,51014A,Feather Pen Hot Pink,7.24,2,15492,United Kingdom +581492,12/9/2019,51014L,Feather Pen Light Pink,7.24,1,15492,United Kingdom +581492,12/9/2019,71270,Photo Clip Line,7.24,1,15492,United Kingdom +581492,12/9/2019,72349B,Set/6 Purple Butterfly T-Lights,6.39,1,15492,United Kingdom +581492,12/9/2019,72741,Grand Chocolatecandle,7.24,3,15492,United Kingdom +581492,12/9/2019,72800E,4 Ivory Dinner Candles Silver Flock,7.24,3,15492,United Kin +581492,12/9/2019,79321,Chilli Lights,6.19,1,15492,United Kingdom +581492,12/9/2019,82484,Wood Black Board Ant White Finish,6.19,2,15492,United Kingdo +581492,12/9/2019,82567,Airline Lounge Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,82582,Area Patrolled Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,82600,N0 Singing Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,84031A,Charlie+Lola Red Hot Water Bottle,6.19,4,15492,United Kingd +581492,12/9/2019,84077,World War 2 Gliders Asstd Designs,6.19,1,15492,United Kingdo +581492,12/9/2019,84249A,Greeting Card Square Doughnuts,6.19,1,15492,United Kingdom +581492,12/9/2019,84347,Rotating Silver Angels T-Light Hldr,6.19,2,15492,United King +581492,12/9/2019,84375,Set Of 20 Kids Cookie Cutters,6.19,1,15492,United Kingdom +581492,12/9/2019,84378,Set Of 3 Heart Cookie Cutters,6.19,4,15492,United Kingdom +581492,12/9/2019,84519B,Carrot Charlie+Lola Coaster Set,6.19,1,15492,United Kingdom +581492,12/9/2019,84559A,3d Sheet Of Dog Stickers,6.19,2,15492,United Kingdom +581492,12/9/2019,84568,Girls Alphabet Iron On Patches,6.19,31,15492,United Kingdom +581492,12/9/2019,84569D,Pack 6 Heart/Ice-Cream Patches,6.19,1,15492,United Kingdom +581492,12/9/2019,84596B,Small Dolly Mix Design Orange Bowl,6.19,4,15492,United King +581492,12/9/2019,84596F,Small Marshmallows Pink Bowl,6.19,4,15492,United Kingdom +581492,12/9/2019,84598,Boys Alphabet Iron On Patches,6.19,5,15492,United Kingdom +581492,12/9/2019,84692,Box Of 24 Cocktail Parasols,6.19,1,15492,United Kingdom +581492,12/9/2019,84755,Colour Glass T-Light Holder Hanging,6.19,4,15492,United King +581492,12/9/2019,84828,Jungle Popsicles Ice Lolly Moulds,6.19,1,15492,United Kingdo +581492,12/9/2019,84879,Assorted Colour Bird Ornament,6.19,16,15492,United Kingdom +581492,12/9/2019,84923,Pink Butterfly Handbag W Bobbles,6.04,3,15492,United Kingdom +581492,12/9/2019,84946,Antique Silver T-Light Glass,6.04,18,15492,United Kingdom +581492,12/9/2019,84970S,Hanging Heart Zinc T-Light Holder,6.04,3,15492,United Kingd +581492,12/9/2019,84978,Hanging Heart Jar T-Light Holder,6.04,4,15492,United Kingdom +581492,12/9/2019,84991,60 Teatime Fairy Cake Cases,6.04,1,15492,United Kingdom +581492,12/9/2019,84997A,Childrens Cutlery Polkadot Green,6.04,1,15492,United Kingdo +581492,12/9/2019,84997B,Childrens Cutlery Retrospot Red,6.04,1,15492,United Kingdom +581492,12/9/2019,20733,Gold Mini Tape Measure,6.04,3,15492,United Kingdom +581492,12/9/2019,20777,Chrysanthemum Notebook,6.19,2,15492,United Kingdom +581492,12/9/2019,20782,Camouflage Ear Muff Headphones,6.19,1,15492,United Kingdom +581492,12/9/2019,20914,Set/5 Red Retrospot Lid Glass Bowls,6.19,2,15492,United King +581492,12/9/2019,20931,Blue Pot Plant Candle,6.19,1,15492,United Kingdom +581492,12/9/2019,20970,Pink Floral Feltcraft Shoulder Bag,6.19,1,15492,United Kingd +581492,12/9/2019,20971,Pink Blue Felt Craft Trinket Box,6.19,2,15492,United Kingdom +581492,12/9/2019,20972,Pink Cream Felt Craft Trinket Box,6.19,3,15492,United Kingdo +581492,12/9/2019,20973,12 Pencil Small Tube Woodland,6.19,4,15492,United Kingdom +581492,12/9/2019,20978,36 Pencils Tube Skulls,6.19,1,15492,United Kingdom +581492,12/9/2019,20979,36 Pencils Tube Red Retrospot,6.19,3,15492,United Kingdom +581492,12/9/2019,20982,12 Pencils Tall Tube Skulls,6.19,2,15492,United Kingdom +581492,12/9/2019,20983,12 Pencils Tall Tube Red Retrospot,6.19,4,15492,United Kingd +581492,12/9/2019,20985,Heart Calculator,6.19,1,15492,United Kingdom +581492,12/9/2019,20986,Blue Calculator Ruler,6.19,1,15492,United Kingdom +581492,12/9/2019,21000,Rose Du Sud Cosmetics Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,21012,Antique All Glass Candlestick,6.19,3,15492,United Kingdom +581492,12/9/2019,21015,Dark Bird House Tree Decoration,6.19,8,15492,United Kingdom +581492,12/9/2019,21026,Space Owl,6.19,1,15492,United Kingdom +581492,12/9/2019,21035,Set/2 Red Retrospot Tea Towels,6.19,1,15492,United Kingdom +581492,12/9/2019,21064,Boom Box Speaker Boys,6.19,4,15492,United Kingdom +581492,12/9/2019,21065,Boom Box Speaker Girls,6.19,2,15492,United Kingdom +581492,12/9/2019,21098,Christmas Toilet Roll,6.19,1,15492,United Kingdom +581492,12/9/2019,21123,Set/10 Ivory Polkadot Party Candles,6.19,1,15492,United King +581492,12/9/2019,21125,Set 6 Football Celebration Candles,6.19,1,15492,United Kingd +581492,12/9/2019,21126,Set Of 6 Girls Celebration Candles,6.19,1,15492,United Kingd +581492,12/9/2019,21137,Black Record Cover Frame,6.19,17,15492,United Kingdom +581492,12/9/2019,21154,Red Retrospot Oven Glove,6.19,2,15492,United Kingdom +581492,12/9/2019,21158,Moody Girl Door Hanger,6.19,1,15492,United Kingdom +581492,12/9/2019,21162,Toxic Area Door Hanger,6.19,1,15492,United Kingdom +581492,12/9/2019,21166,Cook With Wine Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,21172,Party Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,21174,Pottering In The Shed Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,21175,Gin And Tonic Diet Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,21200,Multicolour Honeycomb Paper Garland,6.04,6,15492,United King +581492,12/9/2019,21201,Tropical Honeycomb Paper Garland,6.04,4,15492,United Kingdom +581492,12/9/2019,21212,Pack Of 72 Retrospot Cake Cases,6.04,2,15492,United Kingdom +581492,12/9/2019,21213,Pack Of 72 Skull Cake Cases,6.04,2,15492,United Kingdom +581492,12/9/2019,21220,Set/4 Badges Dogs,6.19,2,15492,United Kingdom +581492,12/9/2019,21224,Set/4 Skull Badges,6.19,1,15492,United Kingdom +581492,12/9/2019,21232,Strawberry Ceramic Trinket Pot,6.19,1,15492,United Kingdom +581492,12/9/2019,21257,Victorian Sewing Box Medium,6.19,2,15492,United Kingdom +581492,12/9/2019,21258,Victorian Sewing Box Large,6.19,1,15492,United Kingdom +581492,12/9/2019,21259,Victorian Sewing Box Small,6.19,1,15492,United Kingdom +581492,12/9/2019,21313,Glass Heart T-Light Holder,6.19,1,15492,United Kingdom +581492,12/9/2019,21314,Small Glass Heart Trinket Pot,6.19,2,15492,United Kingdom +581492,12/9/2019,21328,Balloons Writing Set,6.19,2,15492,United Kingdom +581492,12/9/2019,21329,Dinosaurs Writing Set,6.19,2,15492,United Kingdom +581492,12/9/2019,21356,Toast Its - Fairy Flower,6.19,16,15492,United Kingdom +581492,12/9/2019,21378,Small Tall Camphor Wood Toadstool,6.19,2,15492,United Kingdo +581492,12/9/2019,21379,Camphor Wood Portobello Mushroom,6.19,1,15492,United Kingdom +581492,12/9/2019,21383,Pack Of 12 Sticky Bunnies,6.19,1,15492,United Kingdom +581492,12/9/2019,21402,Red Egg Spoon,6.19,1,15492,United Kingdom +581492,12/9/2019,21408,Spotty Pink Duck Doorstop,6.19,2,15492,United Kingdom +581492,12/9/2019,21411,Gingham Heart Doorstop Red,6.19,1,15492,United Kingdom +581492,12/9/2019,21467,Cherry Crochet Food Cover,6.19,1,15492,United Kingdom +581492,12/9/2019,21471,Strawberry Raffia Food Cover,6.19,2,15492,United Kingdom +581492,12/9/2019,21479,White Skull Hot Water Bottle,6.19,2,15492,United Kingdom +581492,12/9/2019,21481,Fawn Blue Hot Water Bottle,6.19,3,15492,United Kingdom +581492,12/9/2019,21484,Chick Grey Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,21485,Retrospot Heart Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,21494,Rotating Leaves T-Light Holder,6.19,11,15492,United Kingdom +581492,12/9/2019,21506,Fancy Font Birthday Card,6.19,3,15492,United Kingdom +581492,12/9/2019,21507,Elephant Birthday Card,7.24,1,15492,United Kingdom +581492,12/9/2019,21508,Vintage Kid Dolly Card,7.24,1,15492,United Kingdom +581492,12/9/2019,21509,Cowboys And Indians Birthday Card,7.24,2,15492,United Kingdo +581492,12/9/2019,21528,Dairy Maid Traditional Teapot,7.24,1,15492,United Kingdom +581492,12/9/2019,21544,Skulls Water Transfer Tattoos,7.24,2,15492,United Kingdom +581492,12/9/2019,21558,Skull Lunch Box With Cutlery,6.39,1,15492,United Kingdom +581492,12/9/2019,21559,Strawberry Lunch Box With Cutlery,7.24,1,15492,United Kingdo +581492,12/9/2019,21615,4 Lavender Botanical Dinner Candles,7.24,2,15492,United King +581492,12/9/2019,21616,4 Pear Botanical Dinner Candles,7.24,6,15492,United Kingdom +581492,12/9/2019,21620,Set Of 4 Rose Botanical Candles,7.24,5,15492,United Kingdom +581492,12/9/2019,21642,Assorted Tutti Frutti Pen,7.24,4,15492,United Kingdom +581492,12/9/2019,21648,Assorted Tutti Frutti Small Purse,7.24,2,15492,United Kingdo +581492,12/9/2019,21650,Assorted Tutti Frutti Bracelet,7.24,14,15492,United Kingdom +581492,12/9/2019,21675,Butterflies Stickers,6.19,2,15492,United Kingdom +581492,12/9/2019,21677,Hearts Stickers,6.19,1,15492,United Kingdom +581492,12/9/2019,21678,Paisley Pattern Stickers,6.19,2,15492,United Kingdom +581492,12/9/2019,21680,Woodland Stickers,6.19,1,15492,United Kingdom +581492,12/9/2019,21703,Bag 125g Swirly Marbles,6.19,1,15492,United Kingdom +581492,12/9/2019,21704,Bag 250g Swirly Marbles,6.19,1,15492,United Kingdom +581492,12/9/2019,21716,Boys Vintage Tin Seaside Bucket,6.19,1,15492,United Kingdom +581492,12/9/2019,21718,Red Metal Beach Spade,6.19,1,15492,United Kingdom +581492,12/9/2019,21724,Panda And Bunnies Sticker Sheet,6.19,1,15492,United Kingdom +581492,12/9/2019,21731,Red Toadstool Led Night Light,6.19,6,15492,United Kingdom +581492,12/9/2019,21739,Cosy Slipper Shoes Small Green,6.19,2,15492,United Kingdom +581492,12/9/2019,21774,Decorative Cats Bathroom Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,21786,Polkadot Rain Hat,6.19,3,15492,United Kingdom +581492,12/9/2019,21787,Rain Poncho Retrospot,6.19,2,15492,United Kingdom +581492,12/9/2019,21790,Vintage Snap Cards,6.19,5,15492,United Kingdom +581492,12/9/2019,21791,Vintage Heads And Tails Card Game,6.19,1,15492,United Kingdo +581492,12/9/2019,21808,Christmas Garland Stars Trees,6.19,3,15492,United Kingdom +581492,12/9/2019,21810,Christmas Hanging Star With Bell,6.19,25,15492,United Kingdo +581492,12/9/2019,21812,Garland With Hearts And Bells,6.19,7,15492,United Kingdom +581492,12/9/2019,21813,Garland With Stars And Bells,6.19,3,15492,United Kingdom +581492,12/9/2019,21822,Glitter Christmas Tree With Bells,6.19,12,15492,United Kingd +581492,12/9/2019,21828,Eight Piece Snake Set,6.19,1,15492,United Kingdom +581492,12/9/2019,21832,Chocolate Calculator,6.19,1,15492,United Kingdom +581492,12/9/2019,21833,Camouflage Led Torch,6.04,2,15492,United Kingdom +581492,12/9/2019,21871,Save The Planet Mug,6.19,1,15492,United Kingdom +581492,12/9/2019,21874,Gin And Tonic Mug,6.19,2,15492,United Kingdom +581492,12/9/2019,21876,Pottering Mug,6.19,4,15492,United Kingdom +581492,12/9/2019,21879,Hearts Gift Tape,6.19,1,15492,United Kingdom +581492,12/9/2019,21889,Wooden Box Of Dominoes,6.19,3,15492,United Kingdom +581492,12/9/2019,21890,S/6 Wooden Skittles In Cotton Bag,6.19,1,15492,United Kingdo +581492,12/9/2019,21892,Traditional Wooden Catch Cup Game,6.19,1,15492,United Kingdo +581492,12/9/2019,21900,Key Fob Shed,6.19,4,15492,United Kingdom +581492,12/9/2019,21901,Key Fob Back Door,6.19,2,15492,United Kingdom +581492,12/9/2019,21902,Key Fob Front Door,6.19,1,15492,United Kingdom +581492,12/9/2019,21905,More Butter Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,21906,Pharmacie First Aid Tin,6.19,1,15492,United Kingdom +581492,12/9/2019,21914,Blue Harmonica In Box,6.19,2,15492,United Kingdom +581492,12/9/2019,21916,Set 12 Retro White Chalk Sticks,6.19,2,15492,United Kingdom +581492,12/9/2019,21930,Jumbo Storage Bag Skulls,6.19,1,15492,United Kingdom +581492,12/9/2019,21931,Jumbo Storage Bag Suki,6.19,1,15492,United Kingdom +581492,12/9/2019,21932,Scandinavian Paisley Picnic Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,21934,Skull Shoulder Bag,6.19,3,15492,United Kingdom +581492,12/9/2019,21935,Suki Shoulder Bag,6.19,9,15492,United Kingdom +581492,12/9/2019,21936,Red Retrospot Picnic Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,21942,Skulls Design Flannel,6.19,2,15492,United Kingdom +581492,12/9/2019,21945,Strawberries Design Flannel,6.19,2,15492,United Kingdom +581492,12/9/2019,21947,Set Of 6 Heart Chopsticks,6.19,1,15492,United Kingdom +581492,12/9/2019,21949,Set Of 6 Strawberry Chopsticks,6.19,2,15492,United Kingdom +581492,12/9/2019,21967,Pack Of 12 Skull Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,21976,Pack Of 60 Mushroom Cake Cases,6.19,3,15492,United Kingdom +581492,12/9/2019,21977,Pack Of 60 Pink Paisley Cake Cases,6.19,2,15492,United Kingd +581492,12/9/2019,21980,Pack Of 12 Red Retrospot Tissues,6.19,2,15492,United Kingdom +581492,12/9/2019,21981,Pack Of 12 Woodland Tissues,6.19,4,15492,United Kingdom +581492,12/9/2019,21982,Pack Of 12 Suki Tissues,6.19,2,15492,United Kingdom +581492,12/9/2019,21983,Pack Of 12 Blue Paisley Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,21984,Pack Of 12 Pink Paisley Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,21986,Pack Of 12 Pink Polkadot Tissues,6.19,3,15492,United Kingdom +581492,12/9/2019,21989,Pack Of 20 Skull Paper Napkins,6.19,2,15492,United Kingdom +581492,12/9/2019,21990,Modern Floral Stationery Set,6.19,8,15492,United Kingdom +581492,12/9/2019,21993,Floral Folk Stationery Set,6.19,8,15492,United Kingdom +581492,12/9/2019,22024,Rainy Ladies Birthday Card,6.19,3,15492,United Kingdom +581492,12/9/2019,22025,Ring Of Roses Birthday Card,6.19,3,15492,United Kingdom +581492,12/9/2019,22026,Banquet Birthday Card,6.19,4,15492,United Kingdom +581492,12/9/2019,22027,Tea Party Birthday Card,6.19,2,15492,United Kingdom +581492,12/9/2019,22028,Penny Farthing Birthday Card,6.19,1,15492,United Kingdom +581492,12/9/2019,22029,Spaceboy Birthday Card,6.19,4,15492,United Kingdom +581492,12/9/2019,22031,Botanical Lavender Birthday Card,6.19,1,15492,United Kingdom +581492,12/9/2019,22037,Robot Birthday Card,6.19,4,15492,United Kingdom +581492,12/9/2019,22064,Pink Doughnut Trinket Pot,6.19,1,15492,United Kingdom +581492,12/9/2019,22065,Christmas Pudding Trinket Pot,6.19,2,15492,United Kingdom +581492,12/9/2019,22068,Black Pirate Treasure Chest,6.19,1,15492,United Kingdom +581492,12/9/2019,22070,Small Red Retrospot Mug In Box,6.19,3,15492,United Kingdom +581492,12/9/2019,22071,Small White Retrospot Mug In Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22074,6 Ribbons Shimmering Pinks,6.19,1,15492,United Kingdom +581492,12/9/2019,22075,6 Ribbons Elegant Christmas,6.19,8,15492,United Kingdom +581492,12/9/2019,22076,6 Ribbons Empire,6.19,4,15492,United Kingdom +581492,12/9/2019,22083,Paper Chain Kit Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22085,Paper Chain Kit Skulls,6.19,1,15492,United Kingdom +581492,12/9/2019,22086,Paper Chain Kit 50'S Christmas,6.19,38,15492,United Kingdom +581492,12/9/2019,22091,Empire Tissue Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22099,Caravan Square Tissue Box,6.19,3,15492,United Kingdom +581492,12/9/2019,22104,Mirror Mosaic Candle Plate,6.19,4,15492,United Kingdom +581492,12/9/2019,22106,Mirror Mosaic Hurricane Lamp,6.19,1,15492,United Kingdom +581492,12/9/2019,22107,Pizza Plate In Box,6.19,10,15492,United Kingdom +581492,12/9/2019,22108,Ping! Microwave Plate,6.04,2,15492,United Kingdom +581492,12/9/2019,22109,Full English Breakfast Plate,6.04,2,15492,United Kingdom +581492,12/9/2019,22110,Bird House Hot Water Bottle,6.04,3,15492,United Kingdom +581492,12/9/2019,22111,Scottie Dog Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,22112,Chocolate Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,6.19,4,15492,United Kingdo +581492,12/9/2019,22116,Metal Sign His Dinner Is Served,6.19,2,15492,United Kingdom +581492,12/9/2019,22121,Noel Wooden Block Letters,6.19,1,15492,United Kingdom +581492,12/9/2019,22123,Ping Microwave Apron,6.19,1,15492,United Kingdom +581492,12/9/2019,22124,Set Of 2 Tea Towels Ping Microwave,6.19,1,15492,United Kingd +581492,12/9/2019,22129,Party Cones Candy Decoration,6.19,3,15492,United Kingdom +581492,12/9/2019,22134,Mini Ladle Love Heart Red,6.19,1,15492,United Kingdom +581492,12/9/2019,15036,Assorted Colours Silk Fan,6.19,1,15492,United Kingdom +581492,12/9/2019,15039,Sandalwood Fan,6.19,1,15492,United Kingdom +581492,12/9/2019,15058C,Ice Cream Design Garden Parasol,6.19,1,15492,United Kingdom +581492,12/9/2019,16218,Cartoon Pencil Sharpeners,6.19,5,15492,United Kingdom +581492,12/9/2019,16225,Rattle Snake Eggs,6.19,1,15492,United Kingdom +581492,12/9/2019,16235,Recycled Pencil With Rabbit Eraser,6.19,3,15492,United Kingd +581492,12/9/2019,16237,Sleeping Cat Erasers,6.19,1,15492,United Kingdom +581492,12/9/2019,17038,Porcelain Budah Incense Holder,6.19,1,15492,United Kingdom +581492,12/9/2019,17084N,Fairy Dreams Incense,6.19,1,15492,United Kingdom +581492,12/9/2019,20659,Economy Luggage Tag,6.19,7,15492,United Kingdom +581492,12/9/2019,20665,Red Retrospot Purse,6.19,2,15492,United Kingdom +581492,12/9/2019,20668,Disco Ball Christmas Decoration,6.19,1,15492,United Kingdom +581492,12/9/2019,20718,Red Retrospot Shopper Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,20719,Woodland Charlotte Bag,6.19,7,15492,United Kingdom +581492,12/9/2019,20723,Strawberry Charlotte Bag,6.19,2,15492,United Kingdom +581492,12/9/2019,20724,Red Retrospot Charlotte Bag,6.19,4,15492,United Kingdom +581492,12/9/2019,22135,Mini Ladle Love Heart Pink,6.19,6,15492,United Kingdom +581492,12/9/2019,22138,Baking Set 9 Piece Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22141,Christmas Craft Tree Top Angel,6.19,2,15492,United Kingdom +581492,12/9/2019,22150,3 Stripey Mice Feltcraft,7.24,2,15492,United Kingdom +581492,12/9/2019,22153,Angel Decoration Stars On Dress,7.24,2,15492,United Kingdom +581492,12/9/2019,22154,Angel Decoration 3 Buttons,7.24,3,15492,United Kingdom +581492,12/9/2019,22155,Star Decoration Rustic,7.24,7,15492,United Kingdom +581492,12/9/2019,22156,Heart Decoration With Pearls,7.24,4,15492,United Kingdom +581492,12/9/2019,22163,Heart String Memo Holder Hanging,7.24,9,15492,United Kingdom +581492,12/9/2019,22165,Diamante Heart Shaped Wall Mirror,7.24,1,15492,United Kingdo +581492,12/9/2019,22167,Oval Wall Mirror Diamante,7.24,1,15492,United Kingdom +581492,12/9/2019,22170,Picture Frame Wood Triple Portrait,7.24,1,15492,United Kingd +581492,12/9/2019,22173,Metal 4 Hook Hanger French Chateau,7.24,2,15492,United Kingd +581492,12/9/2019,22174,Photo Cube,6.19,7,15492,United Kingdom +581492,12/9/2019,22178,Victorian Glass Hanging T-Light,6.19,5,15492,United Kingdom +581492,12/9/2019,22186,Red Star Card Holder,7.24,2,15492,United Kingdom +581492,12/9/2019,22190,Local Cafe Mug,7.24,3,15492,United Kingdom +581492,12/9/2019,22193,Red Diner Wall Clock,7.24,1,15492,United Kingdom +581492,12/9/2019,22195,Large Heart Measuring Spoons,7.24,1,15492,United Kingdom +581492,12/9/2019,22196,Small Heart Measuring Spoons,7.24,11,15492,United Kingdom +581492,12/9/2019,22197,Popcorn Holder,7.24,34,15492,United Kingdom +581492,12/9/2019,22199,Frying Pan Red Retrospot,7.24,1,15492,United Kingdom +581492,12/9/2019,22209,Wood Stamp Set Happy Birthday,7.24,1,15492,United Kingdom +581492,12/9/2019,22212,Four Hook White Lovebirds,7.24,1,15492,United Kingdom +581492,12/9/2019,22219,Lovebird Hanging Decoration White,7.24,4,15492,United Kingdo +581492,12/9/2019,22224,White Lovebird Lantern,7.24,4,15492,United Kingdom +581492,12/9/2019,22227,Hanging Heart Mirror Decoration,7.24,1,15492,United Kingdom +581492,12/9/2019,22260,Felt Egg Cosy Blue Rabbit,6.39,2,15492,United Kingdom +581492,12/9/2019,22261,Felt Egg Cosy White Rabbit,7.24,1,15492,United Kingdom +581492,12/9/2019,22264,Felt Farm Animal White Bunny,7.24,1,15492,United Kingdom +581492,12/9/2019,22273,Feltcraft Doll Molly,7.24,2,15492,United Kingdom +581492,12/9/2019,22277,Cosmetic Bag Vintage Rose Paisley,7.24,1,15492,United Kingdo +581492,12/9/2019,22279,Pocket Bag Blue Paisley Red Spot,7.24,5,15492,United Kingdom +581492,12/9/2019,22280,Pocket Bag Pink Paisely Brown Spot,7.24,4,15492,United Kingd +581492,12/9/2019,22300,Coffee Mug Dog + Ball Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22301,Coffee Mug Cat + Bird Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22307,Gold Mug Bone China Tree Of Life,7.24,1,15492,United Kingdom +581492,12/9/2019,22308,Tea Cosy Blue Stripe,7.24,2,15492,United Kingdom +581492,12/9/2019,22309,Tea Cosy Red Stripe,7.24,2,15492,United Kingdom +581492,12/9/2019,22324,Blue Polkadot Kids Bag,7.24,2,15492,United Kingdom +581492,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,7.24,3,15492,United Kingd +581492,12/9/2019,22327,Round Snack Boxes Set Of 4 Skulls,7.24,2,15492,United Kingdo +581492,12/9/2019,22329,Round Container Set Of 5 Retrospot,6.19,2,15492,United Kingd +581492,12/9/2019,22338,Star Decoration Painted Zinc,6.19,4,15492,United Kingdom +581492,12/9/2019,22340,Noel Garland Painted Zinc,6.19,17,15492,United Kingdom +581492,12/9/2019,22342,Home Garland Painted Zinc,6.19,7,15492,United Kingdom +581492,12/9/2019,22348,Tea Bag Plate Red Retrospot,6.19,3,15492,United Kingdom +581492,12/9/2019,22355,Charlotte Bag Suki Design,6.19,3,15492,United Kingdom +581492,12/9/2019,22356,Charlotte Bag Pink Polkadot,6.19,1,15492,United Kingdom +581492,12/9/2019,22357,Kings Choice Biscuit Tin,6.19,2,15492,United Kingdom +581492,12/9/2019,22358,Kings Choice Tea Caddy,6.19,1,15492,United Kingdom +581492,12/9/2019,22361,Glass Jar Daisy Fresh Cotton Wool,6.19,2,15492,United Kingdo +581492,12/9/2019,22362,Glass Jar Peacock Bath Salts,6.19,2,15492,United Kingdom +581492,12/9/2019,22367,Childrens Apron Spaceboy Design,6.19,1,15492,United Kingdom +581492,12/9/2019,22372,Airline Bag Vintage World Champion,6.19,1,15492,United Kingd +581492,12/9/2019,22374,Airline Bag Vintage Jet Set Red,6.19,1,15492,United Kingdom +581492,12/9/2019,85014B,Red Retrospot Umbrella,6.19,1,15492,United Kingdom +581492,12/9/2019,85032C,Curious Images Gift Wrap Set,6.19,1,15492,United Kingdom +581492,12/9/2019,85038,6 Chocolate Love Heart T-Lights,6.19,1,15492,United Kingdom +581492,12/9/2019,85039A,Set/4 Red Mini Rose Candle In Bowl,6.19,5,15492,United King +581492,12/9/2019,85039B,S/4 Ivory Mini Rose Candle In Bowl,6.19,4,15492,United King +581492,12/9/2019,85040A,S/4 Pink Flower Candles In Bowl,6.19,1,15492,United Kingdom +581492,12/9/2019,85048,15cm Christmas Glass Ball 20 Lights,6.19,2,15492,United King +581492,12/9/2019,85049C,Romantic Pinks Ribbons,6.19,1,15492,United Kingdom +581492,12/9/2019,85049G,Chocolate Box Ribbons,6.19,1,15492,United Kingdom +581492,12/9/2019,85049H,Urban Black Ribbons,6.19,2,15492,United Kingdom +581492,12/9/2019,85053,French Enamel Candleholder,6.19,1,15492,United Kingdom +581492,12/9/2019,85059,French Enamel Water Basin,6.19,1,15492,United Kingdom +581492,12/9/2019,85066,Cream Sweetheart Mini Chest,6.19,1,15492,United Kingdom +581492,12/9/2019,85094,Candy Spot Egg Warmer Rabbit,6.19,2,15492,United Kingdom +581492,12/9/2019,85114C,Red Enchanted Forest Placemat,6.19,1,15492,United Kingdom +581492,12/9/2019,85123A,Cream Hanging Heart T-Light Holder,6.19,3,15492,United King +581492,12/9/2019,85131B,Beaded Crystal Heart Green On Stick,6.19,1,15492,United Kin +581492,12/9/2019,85131D,Beaded Crystal Heart Pink On Stick,6.19,2,15492,United King +581492,12/9/2019,85152,Hand Over The Chocolate Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,85168B,Black Baroque Carriage Clock,6.19,3,15492,United Kingdom +581492,12/9/2019,85169C,Eau De Nil Love Bird Candle,6.19,1,15492,United Kingdom +581492,12/9/2019,85170C,Set/6 Eau De Nil Bird T-Lights,6.19,1,15492,United Kingdom +581492,12/9/2019,85173,Set/6 Frog Prince T-Light Candles,6.19,1,15492,United Kingdo +581492,12/9/2019,85177,Basket Of Flowers Sewing Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,85178,Victorian Sewing Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,85179A,Green Bitty Light Chain,6.19,4,15492,United Kingdom +581492,12/9/2019,85199S,Small Hanging Ivory/Red Wood Bird,6.19,9,15492,United Kingd +581492,12/9/2019,85227,Set Of 6 3d Kit Cards For Kids,6.19,3,15492,United Kingdom +581492,12/9/2019,90003C,Midnight Blue Pair Heart Hair Slide,6.19,1,15492,United Kin +581492,12/9/2019,90003E,Green Pair Heart Hair Slides,6.19,2,15492,United Kingdom +581492,12/9/2019,90010A,Midnight Blue Glass/Silver Bracelet,6.04,1,15492,United Kin +581492,12/9/2019,90013A,Midnight Blue Vintage Earrings,6.19,3,15492,United Kingdom +581492,12/9/2019,90013C,Green Vintage Earrings,6.19,2,15492,United Kingdom +581492,12/9/2019,90014A,Silver Mop Orbit Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90016B,Gold/Mop Pendant Orbit Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90018B,Gold Mop Orbit Drop Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90019B,Gold Mop Orbit Bracelet,6.19,1,15492,United Kingdom +581492,12/9/2019,90027D,Glass Bead Hoop Earrings Amethyst,6.19,1,15492,United Kingd +581492,12/9/2019,90030B,Red Kukui Coconut Seed Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90055,Cracked Glaze Earrings Brown,6.19,1,15492,United Kingdom +581492,12/9/2019,90059A,Diamante Hair Grip Pack/2 Crystal,6.19,1,15492,United Kingd +581492,12/9/2019,90059D,Diamante Hair Grip Pack/2 Peridot,6.19,2,15492,United Kingd +581492,12/9/2019,90059E,Diamante Hair Grip Pack/2 Ruby,6.04,1,15492,United Kingdom +581492,12/9/2019,90059F,Diamante Hair Grip Pack/2 Lt Rose,6.19,1,15492,United Kingd +581492,12/9/2019,90072,Ruby Drop Chandelier Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90086,Crystal Frog Phone Charm,6.19,1,15492,United Kingdom +581492,12/9/2019,90120B,Blue Murano Twist Bracelet,6.19,2,15492,United Kingdom +581492,12/9/2019,90120C,Green Murano Twist Bracelet,6.19,1,15492,United Kingdom +581492,12/9/2019,90130B,Turq Stone/Crystal Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90130C,Green Stone/Crystal Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90134,Old Rose Combo Bead Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90138,White/Pink Mini Crystals Necklace,6.19,1,15492,United Kingdo +581492,12/9/2019,90141B,Ivory Pendant Triple Shell Necklace,6.19,1,15492,United Kin +581492,12/9/2019,90145,Silver Hoop Earrings With Flower,6.19,1,15492,United Kingdom +581492,12/9/2019,90155,Resin Necklace W Pastel Beads,6.19,1,15492,United Kingdom +581492,12/9/2019,90161D,Ant Copper Pink Boudicca Bracelet,6.19,1,15492,United Kingd +581492,12/9/2019,90163A,Pink Rosebud & Pearl Necklace,6.19,2,15492,United Kingdom +581492,12/9/2019,90165B,White Rosebud Pearl Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90168,2 Daisies Hair Comb,6.19,1,15492,United Kingdom +581492,12/9/2019,90169,Daisy Hair Comb,6.19,1,15492,United Kingdom +581492,12/9/2019,90170,Daisy Hair Band,6.19,1,15492,United Kingdom +581492,12/9/2019,90174,Butterfly Hair Band,6.19,2,15492,United Kingdom +581492,12/9/2019,90175A,White Glass Chunky Charm Bracelet,6.19,2,15492,United Kingd +581492,12/9/2019,90175D,Tigris Eye Chunky Charm Bracelet,6.19,1,15492,United Kingdo +581492,12/9/2019,90177A,Classic Diamante Earrings Jet,6.19,1,15492,United Kingdom +581492,12/9/2019,90177C,Drop Diamante Earrings Crystal,6.19,1,15492,United Kingdom +581492,12/9/2019,90181B,Amethyst Glass/Shell/Pearl Necklace,6.04,1,15492,United Kin +581492,12/9/2019,90182C,Black 3 Bead Drop Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90183A,Amber Drop Earrings W Long Beads,6.19,2,15492,United Kingdo +581492,12/9/2019,90183C,Black Drop Earrings W Long Beads,6.19,1,15492,United Kingdo +581492,12/9/2019,90184C,Black Chunky Bead Bracelet W Strap,6.19,1,15492,United King +581492,12/9/2019,90185B,Amethyst Diamante Expandable Ring,6.19,1,15492,United Kingd +581492,12/9/2019,90188,Drop Earrings W Flower & Leaf,6.19,2,15492,United Kingdom +581492,12/9/2019,90191,Silver Lariat 40cm,6.19,2,15492,United Kingdom +581492,12/9/2019,90192,Jade Drop Earrings W Filigree,6.19,1,15492,United Kingdom +581492,12/9/2019,90198A,Vintage Rose Bead Bracelet Raspberr,6.19,2,15492,United Kin +581492,12/9/2019,90200A,Purple Sweetheart Bracelet,6.19,1,15492,United Kingdom +581492,12/9/2019,90201A,Purple Enamel Flower Ring,6.19,2,15492,United Kingdom +581492,12/9/2019,90201B,Black Enamel Flower Ring,6.19,1,15492,United Kingdom +581492,12/9/2019,90201C,Red Enamel Flower Ring,6.19,1,15492,United Kingdom +581492,12/9/2019,90201D,Green Enamel Flower Ring,6.19,1,15492,United Kingdom +581492,12/9/2019,90202C,Green Enamel Flower Hair Tie,6.19,1,15492,United Kingdom +581492,12/9/2019,90202D,Pink Enamel Flower Hair Tie,6.19,1,15492,United Kingdom +581492,12/9/2019,90206C,Crystal Diamante Star Brooch,6.19,1,15492,United Kingdom +581492,12/9/2019,90208,Pair Of Pink Flower Cluster Slide,6.19,1,15492,United Kingdo +581492,12/9/2019,90210A,Grey Acrylic Faceted Bangle,6.19,1,15492,United Kingdom +581492,12/9/2019,22378,Wall Tidy Retrospot,6.19,2,15492,United Kingdom +581492,12/9/2019,22379,Recycling Bag Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22380,Toy Tidy Spaceboy,6.19,2,15492,United Kingdom +581492,12/9/2019,22396,Magnets Pack Of 4 Retro Photo,6.19,1,15492,United Kingdom +581492,12/9/2019,22411,Jumbo Shopper Vintage Red Paisley,6.19,1,15492,United Kingdo +581492,12/9/2019,22418,10 Colour Spaceboy Pen,6.19,3,15492,United Kingdom +581492,12/9/2019,22419,Lipstick Pen Red,6.19,3,15492,United Kingdom +581492,12/9/2019,22420,Lipstick Pen Baby Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,22421,Lipstick Pen Fuschia,6.19,2,15492,United Kingdom +581492,12/9/2019,22422,Toothpaste Tube Pen,6.19,4,15492,United Kingdom +581492,12/9/2019,22437,Set Of 9 Black Skull Balloons,7.24,1,15492,United Kingdom +581492,12/9/2019,22441,Grow Your Own Basil In Enamel Mug,7.24,1,15492,United Kingdo +581492,12/9/2019,22446,Pin Cushion Babushka Pink,7.24,7,15492,United Kingdom +581492,12/9/2019,22457,Natural Slate Heart Chalkboard,7.24,1,15492,United Kingdom +581492,12/9/2019,22464,Hanging Metal Heart Lantern,7.24,1,15492,United Kingdom +581492,12/9/2019,22466,Fairy Tale Cottage Night Light,7.24,1,15492,United Kingdom +581492,12/9/2019,22467,Gumball Coat Rack,7.24,1,15492,United Kingdom +581492,12/9/2019,22478,Birdhouse Garden Marker,7.24,1,15492,United Kingdom +581492,12/9/2019,22486,Plasmatronic Lamp,7.24,1,15492,United Kingdom +581492,12/9/2019,22488,Natural Slate Rectangle Chalkboard,7.24,1,15492,United Kingd +581492,12/9/2019,22489,Pack Of 12 Traditional Crayons,7.24,5,15492,United Kingdom +581492,12/9/2019,22495,Set Of 2 Round Tins Camembert,7.24,1,15492,United Kingdom +581492,12/9/2019,22540,Mini Jigsaw Circus Parade,7.24,1,15492,United Kingdom +581492,12/9/2019,22544,Mini Jigsaw Spaceboy,7.24,1,15492,United Kingdom +581492,12/9/2019,22549,Picture Dominoes,7.24,3,15492,United Kingdom +581492,12/9/2019,22551,Plasters In Tin Spaceboy,7.24,2,15492,United Kingdom +581492,12/9/2019,22553,Plasters In Tin Skulls,7.24,2,15492,United Kingdom +581492,12/9/2019,22554,Plasters In Tin Woodland Animals,7.24,3,15492,United Kingdom +581492,12/9/2019,22555,Plasters In Tin Strongman,7.24,3,15492,United Kingdom +581492,12/9/2019,22557,Plasters In Tin Vintage Paisley,7.24,2,15492,United Kingdom +581492,12/9/2019,22558,Clothes Pegs Retrospot Pack 24,7.24,3,15492,United Kingdom +581492,12/9/2019,22560,Traditional Modelling Clay,7.24,5,15492,United Kingdom +581492,12/9/2019,22565,Feltcraft Hairbands Pink And White,7.24,4,15492,United Kingd +581492,12/9/2019,22566,Feltcraft Hairband Pink And Purple,7.24,3,15492,United Kingd +581492,12/9/2019,22571,Rocking Horse Red Christmas,7.24,4,15492,United Kingdom +581492,12/9/2019,22573,Star Wooden Christmas Decoration,6.39,2,15492,United Kingdom +581492,12/9/2019,22574,Heart Wooden Christmas Decoration,7.24,5,15492,United Kingdo +581492,12/9/2019,22576,Swallow Wooden Christmas Decoration,7.24,3,15492,United King +581492,12/9/2019,22577,Wooden Heart Christmas Scandinavian,7.24,4,15492,United King +581492,12/9/2019,22578,Wooden Star Christmas Scandinavian,7.24,19,15492,United King +581492,12/9/2019,22580,Advent Calendar Gingham Sack,7.24,9,15492,United Kingdom +581492,12/9/2019,22581,Wood Stocking Christmas Scandispot,7.24,11,15492,United King +581492,12/9/2019,22585,Pack Of 6 Birdy Gift Tags,7.24,2,15492,United Kingdom +581492,12/9/2019,22586,Feltcraft Hairband Pink And Blue,7.24,2,15492,United Kingdom +581492,12/9/2019,22589,Cardholder Gingham Star,7.24,6,15492,United Kingdom +581492,12/9/2019,22591,Cardholder Gingham Christmas Tree,7.24,1,15492,United Kingdo +581492,12/9/2019,22592,Cardholder Holly Wreath Metal,7.24,3,15492,United Kingdom +581492,12/9/2019,22593,Christmas Gingham Star,6.39,2,15492,United Kingdom +581492,12/9/2019,22594,Christmas Gingham Tree,7.24,3,15492,United Kingdom +581492,12/9/2019,22597,Musical Zinc Heart Decoration,7.24,9,15492,United Kingdom +581492,12/9/2019,22598,Christmas Musical Zinc Tree,7.24,4,15492,United Kingdom +581492,12/9/2019,22599,Christmas Musical Zinc Star,6.19,5,15492,United Kingdom +581492,12/9/2019,22600,Christmas Retrospot Star Wood,6.19,2,15492,United Kingdom +581492,12/9/2019,22601,Christmas Retrospot Angel Wood,6.19,3,15492,United Kingdom +581492,12/9/2019,22602,Retrospot Wooden Heart Decoration,6.19,3,15492,United Kingdo +581492,12/9/2019,22608,Pens Assorted Funky Jeweled,6.19,3,15492,United Kingdom +581492,12/9/2019,22614,Pack Of 12 Spaceboy Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,22615,Pack Of 12 Circus Parade Tissues,6.19,4,15492,United Kingdom +581492,12/9/2019,22616,Pack Of 12 London Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,22619,Set Of 6 Soldier Skittles,6.19,4,15492,United Kingdom +581492,12/9/2019,22620,4 Traditional Spinning Tops,6.19,3,15492,United Kingdom +581492,12/9/2019,22621,Traditional Knitting Nancy,6.19,2,15492,United Kingdom +581492,12/9/2019,22629,Spaceboy Lunch Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22630,Dolly Girl Lunch Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22632,Hand Warmer Red Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22633,Hand Warmer Union Jack,6.19,1,15492,United Kingdom +581492,12/9/2019,22634,Childs Breakfast Set Spaceboy,6.19,1,15492,United Kingdom +581492,12/9/2019,22650,Ceramic Pirate Chest Money Bank,6.19,2,15492,United Kingdom +581492,12/9/2019,22651,Gentleman Shirt Repair Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,22653,Button Box,6.19,16,15492,United Kingdom +581492,12/9/2019,22654,Deluxe Sewing Kit,6.19,2,15492,United Kingdom +581492,12/9/2019,22659,Lunch Box I Love London,6.19,1,15492,United Kingdom +581492,12/9/2019,22666,Recipe Box Pantry Yellow Design,6.19,5,15492,United Kingdom +581492,12/9/2019,22693,Grow A Flytrap Or Sunflower In Tin,6.19,3,15492,United Kingd +581492,12/9/2019,22694,Wicker Star,6.19,1,15492,United Kingdom +581492,12/9/2019,22697,Green Regency Teacup And Saucer,6.19,1,15492,United Kingdom +581492,12/9/2019,22699,Roses Regency Teacup And Saucer,6.19,2,15492,United Kingdom +581492,12/9/2019,22701,Pink Dog Bowl,6.19,5,15492,United Kingdom +581492,12/9/2019,22703,Pink Cat Bowl,6.19,6,15492,United Kingdom +581492,12/9/2019,22712,Card Dolly Girl,6.19,1,15492,United Kingdom +581492,12/9/2019,22713,Card I Love London,6.19,1,15492,United Kingdom +581492,12/9/2019,22714,Card Birthday Cowboy,6.19,2,15492,United Kingdom +581492,12/9/2019,22716,Card Circus Parade,6.19,3,15492,United Kingdom +581492,12/9/2019,22717,Card Dog And Ball,6.19,1,15492,United Kingdom +581492,12/9/2019,22720,Set Of 3 Cake Tins Pantry Design,6.19,2,15492,United Kingdom +581492,12/9/2019,22725,Alarm Clock Bakelike Chocolate,6.19,1,15492,United Kingdom +581492,12/9/2019,22726,Alarm Clock Bakelike Green,6.19,4,15492,United Kingdom +581492,12/9/2019,22727,Alarm Clock Bakelike Red,6.19,4,15492,United Kingdom +581492,12/9/2019,22730,Alarm Clock Bakelike Ivory,6.19,1,15492,United Kingdom +581492,12/9/2019,22741,Funky Diva Pen,6.19,4,15492,United Kingdom +581492,12/9/2019,22745,Poppy's Playhouse Bedroom,6.19,2,15492,United Kingdom +581492,12/9/2019,22748,Poppy's Playhouse Kitchen,6.19,1,15492,United Kingdom +581492,12/9/2019,22749,Feltcraft Princess Charlotte Doll,6.19,1,15492,United Kingdo +581492,12/9/2019,22751,Feltcraft Princess Olivia Doll,6.19,2,15492,United Kingdom +581492,12/9/2019,22753,Small Yellow Babushka Notebook,6.19,1,15492,United Kingdom +581492,12/9/2019,22754,Small Red Babushka Notebook,6.19,1,15492,United Kingdom +581492,12/9/2019,22757,Large Red Babushka Notebook,6.19,3,15492,United Kingdom +581492,12/9/2019,22758,Large Purple Babushka Notebook,6.19,2,15492,United Kingdom +581492,12/9/2019,22763,Key Cabinet Ma Campagne,6.19,1,15492,United Kingdom +581492,12/9/2019,22768,Family Photo Frame Cornice,6.19,2,15492,United Kingdom +581492,12/9/2019,22776,Sweetheart 3 Tier Cake Stand,6.19,1,15492,United Kingdom +581492,12/9/2019,22795,Sweetheart Recipe Book Stand,6.19,1,15492,United Kingdom +581492,12/9/2019,22798,Antique Glass Dressing Table Pot,6.19,3,15492,United Kingdom +581492,12/9/2019,22800,Antique Tall Swirlglass Trinket Pot,6.19,3,15492,United King +581492,12/9/2019,22809,Set Of 6 T-Lights Santa,6.19,1,15492,United Kingdom +581492,12/9/2019,22811,Set Of 6 T-Lights Cacti,6.19,1,15492,United Kingdom +581492,12/9/2019,22814,Card Party Games,6.19,9,15492,United Kingdom +581492,12/9/2019,22815,Card Psychedelic Apples,6.19,18,15492,United Kingdom +581492,12/9/2019,22816,Card Motorbike Santa,6.19,2,15492,United Kingdom +581492,12/9/2019,22817,Card Suki Birthday,6.19,5,15492,United Kingdom +581492,12/9/2019,22818,Card Christmas Village,7.24,6,15492,United Kingdom +581492,12/9/2019,22819,Birthday Card Retro Spot,7.24,3,15492,United Kingdom +581492,12/9/2019,22835,Hot Water Bottle I Am So Poorly,7.24,3,15492,United Kingdom +581492,12/9/2019,22865,Hand Warmer Owl Design,7.24,2,15492,United Kingdom +581492,12/9/2019,22866,Hand Warmer Scotty Dog Design,7.24,3,15492,United Kingdom +581492,12/9/2019,22867,Hand Warmer Bird Design,7.24,4,15492,United Kingdom +581492,12/9/2019,22881,Number Tile Vintage Font 2,7.24,1,15492,United Kingdom +581492,12/9/2019,22885,Number Tile Vintage Font 6,7.24,1,15492,United Kingdom +581492,12/9/2019,22888,Number Tile Vintage Font 9,7.24,1,15492,United Kingdom +581492,12/9/2019,22890,Novelty Biscuits Cake Stand 3 Tier,7.24,1,15492,United Kingd +581492,12/9/2019,22898,Childrens Apron Apples Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22899,Children's Apron Dolly Girl,7.24,2,15492,United Kingdom +581492,12/9/2019,22900,Set 2 Tea Towels I Love London,7.24,3,15492,United Kingdom +581492,12/9/2019,22905,Calendar In Season Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22907,Pack Of 20 Napkins Pantry Design,6.39,1,15492,United Kingdom +581492,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,7.24,5,15492,United King +581492,12/9/2019,22910,Paper Chain Kit Vintage Christmas,7.24,7,15492,United Kingdo +581492,12/9/2019,22914,Blue Coat Rack Paris Fashion,7.24,1,15492,United Kingdom +581492,12/9/2019,22922,Fridge Magnets Us Diner Assorted,7.24,6,15492,United Kingdom +581492,12/9/2019,22924,Fridge Magnets La Vie En Rose,7.24,6,15492,United Kingdom +581492,12/9/2019,22928,Yellow Giant Garden Thermometer,7.24,1,15492,United Kingdom +581492,12/9/2019,22940,Feltcraft Christmas Fairy,6.19,1,15492,United Kingdom +581492,12/9/2019,22941,Christmas Lights 10 Reindeer,6.19,2,15492,United Kingdom +581492,12/9/2019,22943,Christmas Lights 10 Vintage Baubles,6.19,1,15492,United King +581492,12/9/2019,22948,Metal Decoration Naughty Children,6.19,4,15492,United Kingdo +581492,12/9/2019,22951,60 Cake Cases Dolly Girl Design,6.19,2,15492,United Kingdom +581492,12/9/2019,22952,60 Cake Cases Vintage Christmas,6.19,2,15492,United Kingdom +581492,12/9/2019,22955,36 Foil Star Cake Cases,6.19,1,15492,United Kingdom +581492,12/9/2019,22960,Jam Making Set With Jars,6.19,2,15492,United Kingdom +581492,12/9/2019,22961,Jam Making Set Printed,6.19,9,15492,United Kingdom +581492,12/9/2019,22962,Jam Jar With Pink Lid,6.19,3,15492,United Kingdom +581492,12/9/2019,22963,Jam Jar With Green Lid,6.19,3,15492,United Kingdom +581492,12/9/2019,22964,3 Piece Spaceboy Cookie Cutter Set,6.19,1,15492,United Kingd +581492,12/9/2019,22965,3 Traditional Biscuit Cutters Set,6.19,1,15492,United Kingdo +581492,12/9/2019,22966,Gingerbread Man Cookie Cutter,6.19,4,15492,United Kingdom +581492,12/9/2019,22969,Homemade Jam Scented Candles,6.19,6,15492,United Kingdom +581492,12/9/2019,22975,Spaceboy Childrens Egg Cup,6.19,2,15492,United Kingdom +581492,12/9/2019,22976,Circus Parade Childrens Egg Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,22977,Dolly Girl Childrens Egg Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,22979,Pantry Washing Up Brush,6.19,2,15492,United Kingdom +581492,12/9/2019,22980,Pantry Scrubbing Brush,6.19,1,15492,United Kingdom +581492,12/9/2019,22982,Pantry Pastry Brush,6.19,3,15492,United Kingdom +581492,12/9/2019,22983,Card Billboard Font,6.19,1,15492,United Kingdom +581492,12/9/2019,22984,Card Gingham Rose,6.19,1,15492,United Kingdom +581492,12/9/2019,22988,Soldiers Egg Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,22989,Set 2 Pantry Design Tea Towels,6.19,2,15492,United Kingdom +581492,12/9/2019,22992,Revolver Wooden Ruler,6.19,3,15492,United Kingdom +581492,12/9/2019,22993,Set Of 4 Pantry Jelly Moulds,6.19,3,15492,United Kingdom +581492,12/9/2019,22995,Travel Card Wallet Suki,6.19,4,15492,United Kingdom +581492,12/9/2019,22996,Travel Card Wallet Vintage Ticket,6.19,5,15492,United Kingdo +581492,12/9/2019,22997,Travel Card Wallet Union Jack,6.19,1,15492,United Kingdom +581492,12/9/2019,22998,Travel Card Wallet Keep Calm,6.19,4,15492,United Kingdom +581492,12/9/2019,22999,Travel Card Wallet Vintage Leaf,6.19,1,15492,United Kingdom +581492,12/9/2019,23005,Travel Card Wallet I Love London,6.19,4,15492,United Kingdom +581492,12/9/2019,23007,Spaceboy Baby Gift Set,6.19,1,15492,United Kingdom +581492,12/9/2019,23009,I Love London Baby Gift Set,6.19,2,15492,United Kingdom +581492,12/9/2019,23012,Glass Apothecary Bottle Perfume,6.19,1,15492,United Kingdom +581492,12/9/2019,23013,Glass Apothecary Bottle Tonic,6.19,2,15492,United Kingdom +581492,12/9/2019,23014,Glass Apothecary Bottle Elixir,6.19,1,15492,United Kingdom +581492,12/9/2019,23050,Recycled Acapulco Mat Green,6.19,1,15492,United Kingdom +581492,12/9/2019,23051,Recycled Acapulco Mat Blue,6.19,1,15492,United Kingdom +581492,12/9/2019,23052,Recycled Acapulco Mat Turquoise,6.19,5,15492,United Kingdom +581492,12/9/2019,23053,Recycled Acapulco Mat Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23054,Recycled Acapulco Mat Lavender,6.19,1,15492,United Kingdom +581492,12/9/2019,23074,Embossed Heart Trinket Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23076,Ice Cream Sundae Lip Gloss,6.19,3,15492,United Kingdom +581492,12/9/2019,23077,Doughnut Lip Gloss,6.19,3,15492,United Kingdom +581492,12/9/2019,23082,Set 6 Paper Table Lantern Hearts,6.19,1,15492,United Kingdom +581492,12/9/2019,23083,Set 6 Paper Table Lantern Stars,6.19,1,15492,United Kingdom +581492,12/9/2019,23084,Rabbit Night Light,6.19,57,15492,United Kingdom +581492,12/9/2019,23088,Zinc Heart Flower T-Light Holder,6.19,1,15492,United Kingdom +581492,12/9/2019,23089,Glass Bon Bon Jar,6.19,4,15492,United Kingdom +581492,12/9/2019,23093,Small Parisienne Heart Photo Frame,6.19,3,15492,United Kingd +581492,12/9/2019,23103,Jingle Bell Heart Decoration,6.19,2,15492,United Kingdom +581492,12/9/2019,23110,Parisienne Key Cabinet,6.19,1,15492,United Kingdom +581492,12/9/2019,23111,Parisienne Sewing Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23112,Parisienne Curio Cabinet,6.19,1,15492,United Kingdom +581492,12/9/2019,23118,Parisienne Jewellery Drawer,6.19,1,15492,United Kingdom +581492,12/9/2019,23158,Set Of 5 Lucky Cat Magnets,6.19,1,15492,United Kingdom +581492,12/9/2019,23165,Large Ceramic Top Storage Jar,6.19,2,15492,United Kingdom +581492,12/9/2019,23166,Medium Ceramic Top Storage Jar,6.19,2,15492,United Kingdom +581492,12/9/2019,23168,Classic Cafe Sugar Dispenser,6.19,2,15492,United Kingdom +581492,12/9/2019,23171,Regency Tea Plate Green,6.19,1,15492,United Kingdom +581492,12/9/2019,23172,Regency Tea Plate Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23173,Regency Teapot Roses,6.19,1,15492,United Kingdom +581492,12/9/2019,23175,Regency Milk Jug Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23177,Treasure Island Book Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23183,Mother's Kitchen Spoon Rest,6.19,3,15492,United Kingdom +581492,12/9/2019,23184,Bull Dog Bottle Opener,6.19,2,15492,United Kingdom +581492,12/9/2019,23188,Vintage 2 Metre Folding Ruler,6.19,1,15492,United Kingdom +581492,12/9/2019,23191,Bundle Of 3 Retro Note Books,6.19,1,15492,United Kingdom +581492,12/9/2019,23194,Gymkhana Treasure Book Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23196,Vintage Leaf Magnetic Notepad,6.19,1,15492,United Kingdom +581492,12/9/2019,23197,Sketchbook Magnetic Shopping List,6.19,2,15492,United Kingdo +581492,12/9/2019,23198,Pantry Magnetic Shopping List,6.19,2,15492,United Kingdom +581492,12/9/2019,23204,Charlotte Bag Apples Design,6.19,6,15492,United Kingdom +581492,12/9/2019,23205,Charlotte Bag Vintage Alphabet,6.19,3,15492,United Kingdom +581492,12/9/2019,23212,Heart Wreath Decoration With Bell,6.19,7,15492,United Kingdo +581492,12/9/2019,23213,Star Wreath Decoration With Bell,6.19,7,15492,United Kingdom +581492,12/9/2019,23220,Reindeer Heart Decoration Gold,6.19,2,15492,United Kingdom +581492,12/9/2019,23224,Cherub Heart Decoration Gold,6.19,1,15492,United Kingdom +581492,12/9/2019,23229,Vintage Donkey Tail Game,6.19,2,15492,United Kingdom +581492,12/9/2019,23234,Biscuit Tin Vintage Christmas,6.19,4,15492,United Kingdom +581492,12/9/2019,23241,Treasure Tin Gymkhana Design,6.19,3,15492,United Kingdom +581492,12/9/2019,23243,Set Of Tea Coffee Sugar Tins Pantry,6.19,1,15492,United King +581492,12/9/2019,23245,Set Of 3 Regency Cake Tins,6.19,7,15492,United Kingdom +581492,12/9/2019,23247,Biscuit Tin 50'S Christmas,6.19,3,15492,United Kingdom +581492,12/9/2019,23264,Set Of 3 Wooden Sleigh Decorations,6.19,3,15492,United Kingd +581492,12/9/2019,23265,Set Of 3 Wooden Tree Decorations,6.19,3,15492,United Kingdom +581492,12/9/2019,23266,Set Of 3 Wooden Stocking Decoration,6.19,2,15492,United King +581492,12/9/2019,23274,Star T-Light Holder Willie Winkie,6.19,3,15492,United Kingdo +581492,12/9/2019,23275,Set Of 3 Hanging Owls Ollie Beak,6.19,2,15492,United Kingdom +581492,12/9/2019,23280,Folding Butterfly Mirror Hot Pink,6.19,2,15492,United Kingdo +581492,12/9/2019,23284,Doormat Keep Calm And Come In,6.19,1,15492,United Kingdom +581492,12/9/2019,23285,Pink Vintage Spot Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23287,Red Vintage Spot Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23290,Spaceboy Childrens Bowl,6.19,1,15492,United Kingdom +581492,12/9/2019,23292,Spaceboy Childrens Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,23293,Set Of 12 Fairy Cake Baking Cases,6.19,3,15492,United Kingdo +581492,12/9/2019,23294,Set Of 6 Snack Loaf Baking Cases,6.19,1,15492,United Kingdom +581492,12/9/2019,23295,Set Of 12 Mini Loaf Baking Cases,6.19,4,15492,United Kingdom +581492,12/9/2019,23298,Spotty Bunting,6.19,2,15492,United Kingdom +581492,12/9/2019,23299,Food Cover With Beads Set 2,6.19,1,15492,United Kingdom +581492,12/9/2019,23300,Gardeners Kneeling Pad Cup Of Tea,6.19,3,15492,United Kingdo +581492,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,6,15492,United Kingdom +581492,12/9/2019,23307,Set Of 60 Pantry Design Cake Cases,6.19,7,15492,United Kingd +581492,12/9/2019,23308,Set Of 60 Vintage Leaf Cake Cases,6.19,1,15492,United Kingdo +581492,12/9/2019,23309,Set Of 60 I Love London Cake Cases,6.19,2,15492,United Kingd +581492,12/9/2019,23310,Bubblegum Ring Assorted,6.19,4,15492,United Kingdom +581492,12/9/2019,23311,Vintage Christmas Stocking,6.19,1,15492,United Kingdom +581492,12/9/2019,23312,Vintage Christmas Gift Sack,6.19,6,15492,United Kingdom +581492,12/9/2019,23313,Vintage Christmas Bunting,6.19,4,15492,United Kingdom +581492,12/9/2019,23314,Vintage Christmas Tablecloth,6.19,1,15492,United Kingdom +581492,12/9/2019,23319,Box Of 6 Mini 50'S Crackers,6.19,3,15492,United Kingdom +581492,12/9/2019,23322,Large White Heart Of Wicker,6.19,1,15492,United Kingdom +581492,12/9/2019,23323,White Wicker Star,6.19,6,15492,United Kingdom +581492,12/9/2019,23328,Set 6 School Milk Bottles In Crate,6.19,5,15492,United Kingd +581492,12/9/2019,23332,Ivory Wicker Heart Large,6.19,1,15492,United Kingdom +581492,12/9/2019,23334,Ivory Wicker Heart Small,6.19,1,15492,United Kingdom +581492,12/9/2019,23336,Egg Frying Pan Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23340,Vintage Christmas Cake Frill,6.19,3,15492,United Kingdom +581492,12/9/2019,23345,Dolly Girl Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23347,I Love London Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,2,15492,United Kingdom +581492,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,5,15492,United Kingdom +581492,12/9/2019,23352,Roll Wrap 50'S Red Christmas,6.19,1,15492,United Kingdom +581492,12/9/2019,23353,6 Gift Tags Vintage Christmas,6.19,3,15492,United Kingdom +581492,12/9/2019,23354,6 Gift Tags 50'S Christmas,6.19,4,15492,United Kingdom +581492,12/9/2019,23357,Hot Water Bottle Sex Bomb,6.19,1,15492,United Kingdom +581492,12/9/2019,23358,Hot Stuff Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,23365,Set 12 Colour Pencils Love London,6.19,1,15492,United Kingdo +581492,12/9/2019,23366,Set 12 Colouring Pencils Doily,6.04,5,15492,United Kingdom +581492,12/9/2019,23367,Set 12 Colour Pencils Spaceboy,6.04,10,15492,United Kingdom +581492,12/9/2019,23368,Set 12 Colour Pencils Dolly Girl,6.19,2,15492,United Kingdom +581492,12/9/2019,23581,Jumbo Bag Paisley Park,6.19,3,15492,United Kingdom +581492,12/9/2019,21928,Jumbo Bag Scandinavian Blue Paisley,6.19,2,15492,United King +581492,12/9/2019,21929,Jumbo Bag Pink Vintage Paisley,6.19,1,15492,United Kingdom +581492,12/9/2019,85099C,Jumbo Bag Baroque Black White,6.19,7,15492,United Kingdom +581492,12/9/2019,23199,Jumbo Bag Apples,6.19,2,15492,United Kingdom +581492,12/9/2019,23200,Jumbo Bag Pears,6.19,1,15492,United Kingdom +581492,12/9/2019,23202,Jumbo Bag Vintage Leaf,6.19,2,15492,United Kingdom +581492,12/9/2019,23343,Jumbo Bag Vintage Christmas,6.19,4,15492,United Kingdom +581492,12/9/2019,23344,Jumbo Bag 50'S Christmas,6.19,5,15492,United Kingdom +581492,12/9/2019,23583,Lunch Bag Paisley Park,6.19,2,15492,United Kingdom +581492,12/9/2019,20727,Lunch Bag Black Skull,6.04,2,15492,United Kingdom +581492,12/9/2019,20728,Lunch Bag Cars Blue,6.04,1,15492,United Kingdom +581492,12/9/2019,20725,Lunch Bag Red Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,20726,Lunch Bag Woodland,6.19,1,15492,United Kingdom +581492,12/9/2019,22662,Lunch Bag Dolly Girl Design,6.19,1,15492,United Kingdom +581492,12/9/2019,23206,Lunch Bag Apple Design,6.19,3,15492,United Kingdom +581492,12/9/2019,23208,Lunch Bag Vintage Leaf Design,6.19,3,15492,United Kingdom +581492,12/9/2019,23375,50'S Christmas Paper Gift Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,23437,50'S Christmas Gift Bag Large,6.19,1,15492,United Kingdom +581492,12/9/2019,22821,Gift Bag Psychedelic Apples,7.24,3,15492,United Kingdom +581493,12/9/2019,79190B,Retro Plastic Polka Tray,7.24,15,12423,Belgium +581493,12/9/2019,79190A,Retro Plastic 70'S Tray,7.24,15,12423,Belgium +581493,12/9/2019,22915,Assorted Bottle Top Magnets,7.24,12,12423,Belgium +581493,12/9/2019,22151,Place Setting White Heart,7.24,24,12423,Belgium +581493,12/9/2019,22632,Hand Warmer Red Retrospot,7.24,12,12423,Belgium +581493,12/9/2019,22865,Hand Warmer Owl Design,7.24,12,12423,Belgium +581493,12/9/2019,20718,Red Retrospot Shopper Bag,7.24,10,12423,Belgium +581493,12/9/2019,79190B,Retro Plastic Polka Tray,7.24,12,12423,Belgium +581493,12/9/2019,71459,Hanging Jam Jar T-Light Holders,7.24,12,12423,Belgium +581493,12/9/2019,84945,Multi Colour Silver T-Light Holder,7.24,12,12423,Belgium +581493,12/9/2019,22356,Charlotte Bag Pink Polkadot,7.24,10,12423,Belgium +581493,12/9/2019,20724,Red Retrospot Charlotte Bag,7.24,10,12423,Belgium +581493,12/9/2019,23204,Charlotte Bag Apples Design,7.24,10,12423,Belgium +581493,12/9/2019,21108,Fairy Cake Flannel Assorted Colour,7.24,18,12423,Belgium +581493,12/9/2019,22252,Birdcage Decoration Tealight Holder,7.24,12,12423,Belgium +581493,12/9/2019,22807,Set Of 6 T-Lights Toadstools,6.19,6,12423,Belgium +581494,12/9/2019,23084,Rabbit Night Light,6.19,24,12518,Germany +581494,12/9/2019,21559,Strawberry Lunch Box With Cutlery,6.19,6,12518,Germany +581494,12/9/2019,21506,Fancy Font Birthday Card,6.19,12,12518,Germany +581494,12/9/2019,22716,Card Circus Parade,6.19,12,12518,Germany +581494,12/9/2019,22556,Plasters In Tin Circus Parade,6.19,12,12518,Germany +581494,12/9/2019,22328,Round Snack Boxes Set Of 4 Fruits,6.19,6,12518,Germany +581494,12/9/2019,22730,Alarm Clock Bakelike Ivory,6.19,4,12518,Germany +581494,12/9/2019,22633,Hand Warmer Union Jack,6.19,12,12518,Germany +581494,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,6.19,6,12518,Germany +581494,12/9/2019,10125,Mini Funky Design Tapes,6.19,20,12518,Germany +581494,12/9/2019,23367,Set 12 Colour Pencils Spaceboy,6.19,24,12518,Germany +581494,12/9/2019,22551,Plasters In Tin Spaceboy,6.04,12,12518,Germany +581494,12/9/2019,22554,Plasters In Tin Woodland Animals,6.04,12,12518,Germany +581494,12/9/2019,22549,Picture Dominoes,6.19,12,12518,Germany +581494,12/9/2019,23388,Woodland Mini Backpack,6.19,4,12518,Germany +581494,12/9/2019,23390,Dolly Girl Mini Backpack,6.19,4,12518,Germany +581494,12/9/2019,23389,Spaceboy Mini Backpack,6.19,4,12518,Germany +581495,12/9/2019,23535,Wall Art Bicycle Safety,6.19,12,14051,United Kingdom +581495,12/9/2019,22699,Roses Regency Teacup And Saucer,6.19,12,14051,United Kingdom +581495,12/9/2019,22698,Pink Regency Teacup And Saucer,6.19,12,14051,United Kingdom +581495,12/9/2019,22697,Green Regency Teacup And Saucer,6.19,12,14051,United Kingdom +581495,12/9/2019,22633,Hand Warmer Union Jack,6.04,18,14051,United Kingdom +581495,12/9/2019,15056N,Edwardian Parasol Natural,6.19,36,14051,United Kingdom +581495,12/9/2019,23439,Hand Warmer Red Love Heart,6.19,36,14051,United Kingdom +581495,12/9/2019,48138,Doormat Union Flag,6.19,10,14051,United Kingdom +581495,12/9/2019,21523,Doormat Fancy Font Home Sweet Home,6.19,10,14051,United King +581495,12/9/2019,21877,Home Sweet Home Mug,6.19,12,14051,United Kingdom +581495,12/9/2019,21871,Save The Planet Mug,6.19,18,14051,United Kingdom +581495,12/9/2019,22798,Antique Glass Dressing Table Pot,6.19,36,14051,United Kingdo +581495,12/9/2019,23173,Regency Teapot Roses,6.19,6,14051,United Kingdom +581495,12/9/2019,22423,Regency Cakestand 3 Tier,6.19,10,14051,United Kingdom +581496,12/9/2019,22664,Toy Tidy Dolly Girl Design,6.19,20,16558,United Kingdom +581496,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,6.19,12,16558,United Kingdom +581496,12/9/2019,72800E,4 Ivory Dinner Candles Silver Flock,6.19,12,16558,United Ki +581496,12/9/2019,23313,Vintage Christmas Bunting,6.19,10,16558,United Kingdom +581496,12/9/2019,22952,60 Cake Cases Vintage Christmas,6.19,24,16558,United Kingdom +581496,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,6.04,12,16558,United Kin +581496,12/9/2019,23298,Spotty Bunting,6.19,3,16558,United Kingdom +581496,12/9/2019,23598,Paper Bunting Vintage Party,6.19,6,16558,United Kingdom +581496,12/9/2019,16169E,Wrap 50'S Christmas,6.19,25,16558,United Kingdom +581496,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,12,16558,United Kingdom +581496,12/9/2019,23350,Roll Wrap Vintage Spot,6.19,24,16558,United Kingdom +581496,12/9/2019,22865,Hand Warmer Owl Design,6.19,24,16558,United Kingdom +581496,12/9/2019,22632,Hand Warmer Red Retrospot,6.19,12,16558,United Kingdom +581496,12/9/2019,22835,Hot Water Bottle I Am So Poorly,6.19,4,16558,United Kingdom +581496,12/9/2019,22112,Chocolate Hot Water Bottle,6.19,6,16558,United Kingdom +581496,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,4,16558,United Kingdom +581496,12/9/2019,22314,Office Mug Warmer Choc+Blue,6.19,24,16558,United Kingdom +581496,12/9/2019,22313,Office Mug Warmer Pink,6.19,12,16558,United Kingdom +581496,12/9/2019,23084,Rabbit Night Light,6.19,18,16558,United Kingdom +581496,12/9/2019,20831,Gold Photo Frame,6.19,12,16558,United Kingdom +581496,12/9/2019,21115,Rose Caravan Doorstop,6.19,8,16558,United Kingdom +581496,12/9/2019,21462,Nursery A B C Painted Letters,7.24,8,16558,United Kingdom +581496,12/9/2019,22076,6 Ribbons Empire,7.24,24,16558,United Kingdom +581496,12/9/2019,22190,Local Cafe Mug,7.24,24,16558,United Kingdom +581496,12/9/2019,22215,Cake Stand White Two Tier Lace,7.24,6,16558,United Kingdom +581496,12/9/2019,22220,Cake Stand Lovebird 2 Tier White,7.24,6,16558,United Kingdom +581496,12/9/2019,22464,Hanging Metal Heart Lantern,7.24,12,16558,United Kingdom +581496,12/9/2019,22465,Hanging Metal Star Lantern,7.24,12,16558,United Kingdom +581496,12/9/2019,22539,Mini Jigsaw Dolly Girl,7.24,48,16558,United Kingdom +581496,12/9/2019,22544,Mini Jigsaw Spaceboy,7.24,48,16558,United Kingdom +581496,12/9/2019,23438,Red Spot Gift Bag Large,6.19,12,16558,United Kingdom +581496,12/9/2019,23437,50'S Christmas Gift Bag Large,6.19,12,16558,United Kingdom +581497,12/9/2019,20719,Woodland Charlotte Bag,6.19,33,17497,United Kingdom +581497,12/9/2019,20723,Strawberry Charlotte Bag,6.19,42,17497,United Kingdom +581497,12/9/2019,20724,Red Retrospot Charlotte Bag,6.19,55,17497,United Kingdom +581497,12/9/2019,21212,Pack Of 72 Retrospot Cake Cases,7.24,7,17497,United Kingdom +581497,12/9/2019,21238,Red Retrospot Cup,7.24,8,17497,United Kingdom +581497,12/9/2019,21242,Red Retrospot Plate,7.24,2,17497,United Kingdom +581497,12/9/2019,21479,White Skull Hot Water Bottle,7.24,25,17497,United Kingdom +581497,12/9/2019,21481,Fawn Blue Hot Water Bottle,7.24,1,17497,United Kingdom +581497,12/9/2019,21481,Fawn Blue Hot Water Bottle,7.24,1,17497,United Kingdom +581497,12/9/2019,21484,Chick Grey Hot Water Bottle,7.24,1,17497,United Kingdom +581497,12/9/2019,21668,Red Stripe Ceramic Drawer Knob,7.24,1,17497,United Kingdom +581497,12/9/2019,21670,Blue Spot Ceramic Drawer Knob,7.24,1,17497,United Kingdom +581497,12/9/2019,21671,Red Spot Ceramic Drawer Knob,7.24,1,17497,United Kingdom +581497,12/9/2019,21672,White Spot Red Ceramic Drawer Knob,7.24,6,17497,United Kingd +581497,12/9/2019,21673,White Spot Blue Ceramic Drawer Knob,7.24,1,17497,United King +581497,12/9/2019,21714,Citronella Candle Garden Pot,7.24,2,17497,United Kingdom +581497,12/9/2019,22110,Bird House Hot Water Bottle,7.24,7,17497,United Kingdom +581497,12/9/2019,22111,Scottie Dog Hot Water Bottle,7.24,5,17497,United Kingdom +581497,12/9/2019,22112,Chocolate Hot Water Bottle,7.24,13,17497,United Kingdom +581497,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,7.24,48,17497,United Kingd +581497,12/9/2019,22197,Popcorn Holder,7.24,68,17497,United Kingdom +581497,12/9/2019,22355,Charlotte Bag Suki Design,7.24,110,17497,United Kingdom +581497,12/9/2019,22356,Charlotte Bag Pink Polkadot,7.24,25,17497,United Kingdom +581497,12/9/2019,22356,Charlotte Bag Pink Polkadot,7.24,1,17497,United Kingdom +581497,12/9/2019,22423,Regency Cakestand 3 Tier,7.24,8,17497,United Kingdom +581497,12/9/2019,22725,Alarm Clock Bakelike Chocolate,7.24,3,17497,United Kingdom +581497,12/9/2019,22726,Alarm Clock Bakelike Green,7.24,1,17497,United Kingdom +581497,12/9/2019,22727,Alarm Clock Bakelike Red,7.24,10,17497,United Kingdom +581497,12/9/2019,22730,Alarm Clock Bakelike Ivory,7.24,5,17497,United Kingdom +581497,12/9/2019,22735,Ribbon Reel Socks And Mittens,7.24,2,17497,United Kingdom +581497,12/9/2019,22736,Ribbon Reel Making Snowmen,7.24,3,17497,United Kingdom +581497,12/9/2019,22738,Ribbon Reel Snowy Village,7.24,1,17497,United Kingdom +581497,12/9/2019,22805,Blue Drawer Knob Acrylic Edwardian,7.24,1,17497,United Kingd +581497,12/9/2019,22835,Hot Water Bottle I Am So Poorly,7.24,4,17497,United Kingdom +581497,12/9/2019,22895,Set Of 2 Tea Towels Apple And Pears,7.24,1,17497,United King +581497,12/9/2019,22896,Peg Bag Apples Design,7.24,2,17497,United Kingdom +581497,12/9/2019,22898,Childrens Apron Apples Design,7.24,2,17497,United Kingdom +581497,12/9/2019,22943,Christmas Lights 10 Vintage Baubles,7.24,1,17497,United King +581497,12/9/2019,23084,Rabbit Night Light,6.19,37,17497,United Kingdom +581497,12/9/2019,23168,Classic Cafe Sugar Dispenser,6.19,1,17497,United Kingdom +581497,12/9/2019,23204,Charlotte Bag Apples Design,6.04,13,17497,United Kingdom +581497,12/9/2019,23205,Charlotte Bag Vintage Alphabet,6.04,8,17497,United Kingdom +581497,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,1,17497,United Kingdom +581497,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,36,17497,United Kingdom +581497,12/9/2019,23356,Love Hot Water Bottle,6.19,1,17497,United Kingdom +581497,12/9/2019,23357,Hot Water Bottle Sex Bomb,6.19,2,17497,United Kingdom +581497,12/9/2019,23358,Hot Stuff Hot Water Bottle,6.19,2,17497,United Kingdom +581497,12/9/2019,35970,Zinc Folkart Sleigh Bells,6.19,2,17497,United Kingdom +581497,12/9/2019,47566,Party Bunting,6.19,5,17497,United Kingdom +581497,12/9/2019,82583,Hot Baths Metal Sign,6.19,4,17497,United Kingdom +581497,12/9/2019,84029G,Knitted Union Flag Hot Water Bottle,6.19,11,17497,United Ki +581497,12/9/2019,84997C,Childrens Cutlery Polkadot Blue,6.19,3,17497,United Kingdom +581497,12/9/2019,85049G,Chocolate Box Ribbons,6.19,2,17497,United Kingdom +581497,12/9/2019,20727,Lunch Bag Black Skull,6.19,8,17497,United Kingdom +581497,12/9/2019,22383,Lunch Bag Suki Design,7.24,2,17497,United Kingdom +581497,12/9/2019,23206,Lunch Bag Apple Design,6.04,3,17497,United Kingdom +581497,12/9/2019,23206,Lunch Bag Apple Design,6.04,1,17497,United Kingdom +581497,12/9/2019,23207,Lunch Bag Alphabet Design,6.19,1,17497,United Kingdom +581497,12/9/2019,23208,Lunch Bag Vintage Leaf Design,6.19,3,17497,United Kingdom +581498,12/9/2019,20669,Red Heart Luggage Tag,6.19,3,14498,United Kingdom +581498,12/9/2019,20679,Edwardian Parasol Red,6.19,5,14498,United Kingdom +581498,12/9/2019,20717,Strawberry Shopper Bag,6.19,1,14498,United Kingdom +581498,12/9/2019,20718,Red Retrospot Shopper Bag,6.19,1,14498,United Kingdom +581498,12/9/2019,20750,Red Retrospot Mini Cases,6.19,1,14498,United Kingdom +581498,12/9/2019,20961,Strawberry Bath Sponge,6.19,1,14498,United Kingdom +581498,12/9/2019,20963,Apple Bath Sponge,6.19,1,14498,United Kingdom +581498,12/9/2019,21115,Rose Caravan Doorstop,6.19,1,14498,United Kingdom +581498,12/9/2019,21125,Set 6 Football Celebration Candles,6.19,3,14498,United Kingd +581498,12/9/2019,21137,Black Record Cover Frame,6.19,4,14498,United Kingdom +581498,12/9/2019,21155,Red Retrospot Peg Bag,6.19,4,14498,United Kingdom +581498,12/9/2019,21166,Cook With Wine Metal Sign,6.19,3,14498,United Kingdom +581498,12/9/2019,21169,You're Confusing Me Metal Sign,6.19,2,14498,United Kingdom +581498,12/9/2019,21174,Pottering In The Shed Metal Sign,6.19,2,14498,United Kingdom +581498,12/9/2019,21181,Please One Person Metal Sign,6.19,6,14498,United Kingdom +581498,12/9/2019,21216,Set 3 Retrospot Tea/Coffee/Sugar,6.19,2,14498,United Kingdom +581498,12/9/2019,21217,Red Retrospot Round Cake Tins,6.19,3,14498,United Kingdom +581498,12/9/2019,21218,Red Spotty Biscuit Tin,6.19,4,14498,United Kingdom +581498,12/9/2019,21232,Strawberry Ceramic Trinket Pot,6.19,13,14498,United Kingdom +581498,12/9/2019,21239,Pink Polkadot Cup,6.19,1,14498,United Kingdom +581498,12/9/2019,21257,Victorian Sewing Box Medium,6.19,3,14498,United Kingdom +581498,12/9/2019,21327,Skulls Writing Set,6.19,1,14498,United Kingdom +581498,12/9/2019,21328,Balloons Writing Set,6.19,1,14498,United Kingdom +581498,12/9/2019,21329,Dinosaurs Writing Set,6.19,1,14498,United Kingdom +581498,12/9/2019,21411,Gingham Heart Doorstop Red,6.19,1,14498,United Kingdom +581498,12/9/2019,21430,Set/3 Red Gingham Rose Storage Box,6.19,3,14498,United Kingd +581498,12/9/2019,21494,Rotating Leaves T-Light Holder,6.19,7,14498,United Kingdom +581498,12/9/2019,21523,Doormat Fancy Font Home Sweet Home,6.19,1,14498,United Kingd +581498,12/9/2019,21557,Set Of 6 Funky Beakers,6.19,1,14498,United Kingdom +581498,12/9/2019,21558,Skull Lunch Box With Cutlery,6.19,1,14498,United Kingdom +581498,12/9/2019,21731,Red Toadstool Led Night Light,6.19,9,14498,United Kingdom +581498,12/9/2019,21754,Home Building Block Word,6.19,2,14498,United Kingdom +581498,12/9/2019,21790,Vintage Snap Cards,6.19,1,14498,United Kingdom +581498,12/9/2019,21843,Red Retrospot Cake Stand,6.19,5,14498,United Kingdom +581498,12/9/2019,21864,Union Jack Flag Passport Cover,6.19,2,14498,United Kingdom +581498,12/9/2019,21874,Gin And Tonic Mug,6.19,2,14498,United Kingdom +581498,12/9/2019,21876,Pottering Mug,6.19,4,14498,United Kingdom +581498,12/9/2019,21890,S/6 Wooden Skittles In Cotton Bag,6.19,1,14498,United Kingdo +581498,12/9/2019,21912,Vintage Snakes & Ladders,6.19,1,14498,United Kingdom +581498,12/9/2019,21916,Set 12 Retro White Chalk Sticks,6.19,7,14498,United Kingdom +581498,12/9/2019,21930,Jumbo Storage Bag Skulls,6.19,2,14498,United Kingdom +581498,12/9/2019,21931,Jumbo Storage Bag Suki,6.19,6,14498,United Kingdom +581498,12/9/2019,21934,Skull Shoulder Bag,6.19,1,14498,United Kingdom +581498,12/9/2019,21935,Suki Shoulder Bag,6.19,7,14498,United Kingdom +581498,12/9/2019,21942,Skulls Design Flannel,6.19,1,14498,United Kingdom +581498,12/9/2019,21955,Doormat Union Jack Guns And Roses,6.19,1,14498,United Kingdo +581498,12/9/2019,21977,Pack Of 60 Pink Paisley Cake Cases,6.19,1,14498,United Kingd +581498,12/9/2019,21983,Pack Of 12 Blue Paisley Tissues,6.19,1,14498,United Kingdom +581498,12/9/2019,21987,Pack Of 6 Skull Paper Cups,6.19,2,14498,United Kingdom +581498,12/9/2019,21989,Pack Of 20 Skull Paper Napkins,6.19,1,14498,United Kingdom +581498,12/9/2019,22041,"Record Frame 7"" Single Size",6.19,2,14498,United Kingdom +581498,12/9/2019,22059,Ceramic Strawberry Design Mug,6.19,1,14498,United Kingdom +581498,12/9/2019,22069,Brown Pirate Treasure Chest,6.19,3,14498,United Kingdom +581498,12/9/2019,22077,6 Ribbons Rustic Charm,6.19,2,14498,United Kingdom +581498,12/9/2019,22083,Paper Chain Kit Retrospot,6.19,2,14498,United Kingdom +581498,12/9/2019,22086,Paper Chain Kit 50'S Christmas,6.19,52,14498,United Kingdom +581498,12/9/2019,22099,Caravan Square Tissue Box,6.19,2,14498,United Kingdom +581498,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,6.19,4,14498,United Kingdo +581498,12/9/2019,22139,Retrospot Tea Set Ceramic 11 Pc,6.19,2,14498,United Kingdom +581498,12/9/2019,22141,Christmas Craft Tree Top Angel,6.19,7,14498,United Kingdom +581498,12/9/2019,22142,Christmas Craft White Fairy,6.19,2,14498,United Kingdom +581498,12/9/2019,22144,Christmas Craft Little Friends,6.19,8,14498,United Kingdom +581498,12/9/2019,22161,Heart Decoration Rustic Hanging,6.19,2,14498,United Kingdom +581498,12/9/2019,22173,Metal 4 Hook Hanger French Chateau,6.19,3,14498,United Kingd +581498,12/9/2019,22174,Photo Cube,6.19,3,14498,United Kingdom +581498,12/9/2019,22175,Pink Owl Soft Toy,6.19,1,14498,United Kingdom +581498,12/9/2019,22178,Victorian Glass Hanging T-Light,6.19,1,14498,United Kingdom +581498,12/9/2019,22179,Set 10 Night Owl Lights,6.19,1,14498,United Kingdom +581498,12/9/2019,22186,Red Star Card Holder,6.19,1,14498,United Kingdom +581498,12/9/2019,22187,Green Christmas Tree Card Holder,6.19,6,14498,United Kingdom +581498,12/9/2019,22193,Red Diner Wall Clock,6.19,1,14498,United Kingdom +581498,12/9/2019,22195,Large Heart Measuring Spoons,6.19,3,14498,United Kingdom +581498,12/9/2019,22196,Small Heart Measuring Spoons,6.19,2,14498,United Kingdom +581498,12/9/2019,22207,Frying Pan Union Flag,6.19,1,14498,United Kingdom +581498,12/9/2019,22278,Overnight Bag Vintage Rose Paisley,6.19,2,14498,United Kingd +581498,12/9/2019,22301,Coffee Mug Cat + Bird Design,6.19,2,14498,United Kingdom +581498,12/9/2019,22302,Coffee Mug Pears Design,6.19,4,14498,United Kingdom +581498,12/9/2019,22303,Coffee Mug Apples Design,6.19,4,14498,United Kingdom +581498,12/9/2019,22304,Coffee Mug Blue Paisley Design,6.19,1,14498,United Kingdom +581498,12/9/2019,22308,Tea Cosy Blue Stripe,6.19,2,14498,United Kingdom +581498,12/9/2019,22327,Round Snack Boxes Set Of 4 Skulls,6.19,4,14498,United Kingdo +581498,12/9/2019,22328,Round Snack Boxes Set Of 4 Fruits,6.19,4,14498,United Kingdo +581498,12/9/2019,22352,Lunch Box With Cutlery Retrospot,6.19,6,14498,United Kingdom +581498,12/9/2019,22357,Kings Choice Biscuit Tin,6.19,1,14498,United Kingdom +581498,12/9/2019,22358,Kings Choice Tea Caddy,6.19,1,14498,United Kingdom +581498,12/9/2019,22367,Childrens Apron Spaceboy Design,6.19,7,14498,United Kingdom +581498,12/9/2019,22371,Airline Bag Vintage Tokyo 78,6.19,1,14498,United Kingdom +581498,12/9/2019,22374,Airline Bag Vintage Jet Set Red,6.19,1,14498,United Kingdom +581498,12/9/2019,22378,Wall Tidy Retrospot,7.24,2,14498,United Kingdom +581498,12/9/2019,22379,Recycling Bag Retrospot,7.24,4,14498,United Kingdom +581498,12/9/2019,22381,Toy Tidy Pink Polkadot,7.24,1,14498,United Kingdom +581498,12/9/2019,22411,Jumbo Shopper Vintage Red Paisley,7.24,2,14498,United Kingdo +581498,12/9/2019,22422,Toothpaste Tube Pen,7.24,1,14498,United Kingdom +581498,12/9/2019,22424,Enamel Bread Bin Cream,7.24,1,14498,United Kingdom +581498,12/9/2019,22429,Enamel Measuring Jug Cream,7.24,1,14498,United Kingdom +581498,12/9/2019,22456,Natural Slate Chalkboard Large,7.24,4,14498,United Kingdom +581498,12/9/2019,22457,Natural Slate Heart Chalkboard,7.24,2,14498,United Kingdom +581498,12/9/2019,22471,Tv Dinner Tray Air Hostess,7.24,1,14498,United Kingdom +581498,12/9/2019,22474,Spaceboy Tv Dinner Tray,7.24,1,14498,United Kingdom +581498,12/9/2019,22488,Natural Slate Rectangle Chalkboard,7.24,2,14498,United Kingd +581498,12/9/2019,22498,Wooden Regatta Bunting,7.24,1,14498,United Kingdom +581498,12/9/2019,22507,Memo Board Retrospot Design,7.24,1,14498,United Kingdom +581498,12/9/2019,22526,Wheelbarrow For Children,7.24,1,14498,United Kingdom +581498,12/9/2019,22553,Plasters In Tin Skulls,7.24,1,14498,United Kingdom +581498,12/9/2019,22557,Plasters In Tin Vintage Paisley,7.24,1,14498,United Kingdom +581498,12/9/2019,22619,Set Of 6 Soldier Skittles,7.24,1,14498,United Kingdom +581498,12/9/2019,22622,Box Of Vintage Alphabet Blocks,7.24,1,14498,United Kingdom +581498,12/9/2019,22624,Ivory Kitchen Scales,7.24,1,14498,United Kingdom +581498,12/9/2019,22629,Spaceboy Lunch Box,7.24,2,14498,United Kingdom +581498,12/9/2019,22632,Hand Warmer Red Retrospot,7.24,1,14498,United Kingdom +581498,12/9/2019,22645,Ceramic Heart Fairy Cake Money Bank,7.24,4,14498,United King +581498,12/9/2019,22654,Deluxe Sewing Kit,6.19,2,14498,United Kingdom +581498,12/9/2019,22659,Lunch Box I Love London,6.19,1,14498,United Kingdom +581498,12/9/2019,22666,Recipe Box Pantry Yellow Design,6.19,11,14498,United Kingdom +581498,12/9/2019,22676,French Blue Metal Door Sign 1,6.04,1,14498,United Kingdom +581498,12/9/2019,22684,French Blue Metal Door Sign 9,6.04,1,14498,United Kingdom +581498,12/9/2019,22694,Wicker Star,6.04,1,14498,United Kingdom +581498,12/9/2019,22697,Green Regency Teacup And Saucer,6.04,6,14498,United Kingdom +581498,12/9/2019,22698,Pink Regency Teacup And Saucer,6.04,9,14498,United Kingdom +581498,12/9/2019,22699,Roses Regency Teacup And Saucer,6.19,6,14498,United Kingdom +581498,12/9/2019,22722,Set Of 6 Spice Tins Pantry Design,6.19,3,14498,United Kingdo +581498,12/9/2019,22733,3d Traditional Christmas Stickers,6.19,1,14498,United Kingdo +581498,12/9/2019,22734,Set Of 6 Ribbons Vintage Christmas,6.19,5,14498,United Kingd +581498,12/9/2019,22776,Sweetheart 3 Tier Cake Stand,6.19,1,14498,United Kingdom +581498,12/9/2019,22795,Sweetheart Recipe Book Stand,6.19,1,14498,United Kingdom +581498,12/9/2019,22807,Set Of 6 T-Lights Toadstools,6.19,2,14498,United Kingdom +581498,12/9/2019,22813,Pack 3 Boxes Bird Panettone,6.19,4,14498,United Kingdom +581498,12/9/2019,22838,3 Tier Cake Tin Red And Cream,6.19,1,14498,United Kingdom +581498,12/9/2019,22844,Vintage Cream Dog Food Container,6.19,2,14498,United Kingdom +581498,12/9/2019,22845,Vintage Cream Cat Food Container,6.19,1,14498,United Kingdom +581498,12/9/2019,22865,Hand Warmer Owl Design,6.19,1,14498,United Kingdom +581498,12/9/2019,22866,Hand Warmer Scotty Dog Design,6.19,2,14498,United Kingdom +581498,12/9/2019,22891,Tea For One Polkadot,6.19,1,14498,United Kingdom +581498,12/9/2019,22900,Set 2 Tea Towels I Love London,6.19,2,14498,United Kingdom +581498,12/9/2019,22910,Paper Chain Kit Vintage Christmas,6.19,5,14498,United Kingdo +581498,12/9/2019,22952,60 Cake Cases Vintage Christmas,6.19,7,14498,United Kingdom +581498,12/9/2019,22960,Jam Making Set With Jars,6.19,5,14498,United Kingdom +581498,12/9/2019,22961,Jam Making Set Printed,6.19,4,14498,United Kingdom +581498,12/9/2019,22966,Gingerbread Man Cookie Cutter,6.19,2,14498,United Kingdom +581498,12/9/2019,22993,Set Of 4 Pantry Jelly Moulds,6.19,2,14498,United Kingdom +581498,12/9/2019,23012,Glass Apothecary Bottle Perfume,6.19,1,14498,United Kingdom +581498,12/9/2019,23013,Glass Apothecary Bottle Tonic,6.19,1,14498,United Kingdom +581498,12/9/2019,23014,Glass Apothecary Bottle Elixir,7.24,2,14498,United Kingdom +581498,12/9/2019,23080,Red Metal Box Top Secret,7.24,5,14498,United Kingdom +581498,12/9/2019,23170,Regency Tea Plate Roses,7.24,4,14498,United Kingdom +581498,12/9/2019,23171,Regency Tea Plate Green,7.24,5,14498,United Kingdom +581498,12/9/2019,23172,Regency Tea Plate Pink,6.19,4,14498,United Kingdom +581498,12/9/2019,23181,Bull Dog Bottle Top Wall Clock,6.19,1,14498,United Kingdom +581498,12/9/2019,23196,Vintage Leaf Magnetic Notepad,6.19,1,14498,United Kingdom +581498,12/9/2019,23198,Pantry Magnetic Shopping List,6.19,2,14498,United Kingdom +581498,12/9/2019,23245,Set Of 3 Regency Cake Tins,6.19,5,14498,United Kingdom +581498,12/9/2019,23295,Set Of 12 Mini Loaf Baking Cases,6.19,1,14498,United Kingdom +581498,12/9/2019,23298,Spotty Bunting,6.19,1,14498,United Kingdom +581498,12/9/2019,23300,Gardeners Kneeling Pad Cup Of Tea,6.19,9,14498,United Kingdo +581498,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,32,14498,United Kingdo +581498,12/9/2019,23311,Vintage Christmas Stocking,6.19,6,14498,United Kingdom +581498,12/9/2019,23312,Vintage Christmas Gift Sack,6.19,3,14498,United Kingdom +581498,12/9/2019,23328,Set 6 School Milk Bottles In Crate,6.19,6,14498,United Kingd +581498,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,8,14498,United Kingdom +581498,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,22,14498,United Kingdom +581498,12/9/2019,23352,Roll Wrap 50'S Red Christmas,6.19,8,14498,United Kingdom +581498,12/9/2019,23353,6 Gift Tags Vintage Christmas,6.19,10,14498,United Kingdom +581498,12/9/2019,23354,6 Gift Tags 50'S Christmas,6.19,9,14498,United Kingdom +581498,12/9/2019,23368,Set 12 Colour Pencils Dolly Girl,6.19,3,14498,United Kingdom +581498,12/9/2019,23369,Set 36 Colour Pencils Love London,6.19,1,14498,United Kingdo +581498,12/9/2019,23370,Set 36 Colouring Pencils Doily,6.19,3,14498,United Kingdom +581498,12/9/2019,23382,Box Of 6 Christmas Cake Decorations,6.19,10,14498,United Kin +581498,12/9/2019,23388,Woodland Mini Backpack,6.19,1,14498,United Kingdom +581498,12/9/2019,23480,Mini Lights Woodland Mushrooms,7.24,1,14498,United Kingdom +581498,12/9/2019,23493,Vintage Doily Travel Sewing Kit,7.24,1,14498,United Kingdom +581498,12/9/2019,23494,Vintage Doily Deluxe Sewing Kit,7.24,1,14498,United Kingdom +581498,12/9/2019,23497,Classic Chrome Bicycle Bell,7.24,1,14498,United Kingdom +581498,12/9/2019,23501,Key Ring Baseball Boot Union Jack,7.24,1,14498,United Kingdo +581498,12/9/2019,23564,Egg Cup Milkmaid Ingrid,7.24,1,14498,United Kingdom +581498,12/9/2019,35970,Zinc Folkart Sleigh Bells,7.24,6,14498,United Kingdom +581498,12/9/2019,48138,Doormat Union Flag,7.24,1,14498,United Kingdom +581498,12/9/2019,71053,White Moroccan Metal Lantern,7.24,1,14498,United Kingdom +581498,12/9/2019,72349B,Set/6 Purple Butterfly T-Lights,7.24,2,14498,United Kingdom +581498,12/9/2019,79321,Chilli Lights,7.24,10,14498,United Kingdom +581498,12/9/2019,82001S,Silver Record Cover Frame,6.19,2,14498,United Kingdom +581498,12/9/2019,82482,Wooden Picture Frame White Finish,6.04,4,14498,United Kingdo +581498,12/9/2019,82552,Washroom Metal Sign,6.04,1,14498,United Kingdom +581498,12/9/2019,21171,Bathroom Metal Sign,6.19,1,14498,United Kingdom +581498,12/9/2019,82581,Toilet Metal Sign,6.19,1,14498,United Kingdom +581498,12/9/2019,82600,N0 Singing Metal Sign,6.19,4,14498,United Kingdom +581498,12/9/2019,84029E,Red Woolly Hottie White Heart,6.19,4,14498,United Kingdom +581498,12/9/2019,84032A,Charlie+Lola Pink Hot Water Bottle,6.19,4,14498,United King +581498,12/9/2019,84032B,Charlie + Lola Red Hot Water Bottle,6.19,3,14498,United Kin +581498,12/9/2019,84375,Set Of 20 Kids Cookie Cutters,6.19,3,14498,United Kingdom +581498,12/9/2019,84509A,Set Of 4 English Rose Placemats,6.19,1,14498,United Kingdom +581498,12/9/2019,84558A,3d Dog Picture Playing Cards,6.19,1,14498,United Kingdom +581498,12/9/2019,84832,Zinc Willie Winkie Candle Stick,6.19,26,14498,United Kingdom +581498,12/9/2019,84968E,Set Of 16 Vintage Black Cutlery,6.19,1,14498,United Kingdom +581498,12/9/2019,84970S,Hanging Heart Zinc T-Light Holder,6.19,1,14498,United Kingd +581498,12/9/2019,84997A,Childrens Cutlery Polkadot Green,6.19,2,14498,United Kingdo +581498,12/9/2019,84997B,Childrens Cutlery Retrospot Red,6.19,3,14498,United Kingdom +581498,12/9/2019,84997D,Childrens Cutlery Polkadot Pink,6.19,1,14498,United Kingdom +581498,12/9/2019,85038,6 Chocolate Love Heart T-Lights,6.19,1,14498,United Kingdom +581498,12/9/2019,85048,15cm Christmas Glass Ball 20 Lights,6.19,1,14498,United King +581498,12/9/2019,85049A,Traditional Christmas Ribbons,6.17,5,14498,United Kingdom +581498,12/9/2019,85049E,Scandinavian Reds Ribbons,6.19,4,14498,United Kingdom +581498,12/9/2019,85150,Ladies & Gentlemen Metal Sign,6.19,1,14498,United Kingdom +581498,12/9/2019,85174,S/4 Cacti Candles,6.19,1,14498,United Kingdom +581498,12/9/2019,20712,Jumbo Bag Woodland Animals,6.19,3,14498,United Kingdom +581498,12/9/2019,20713,Jumbo Bag Owls,6.19,8,14498,United Kingdom +581498,12/9/2019,21928,Jumbo Bag Scandinavian Blue Paisley,6.19,2,14498,United King +581498,12/9/2019,22386,Jumbo Bag Pink Polkadot,7.24,1,14498,United Kingdom +581498,12/9/2019,22663,Jumbo Bag Dolly Girl Design,6.19,2,14498,United Kingdom +581498,12/9/2019,23199,Jumbo Bag Apples,6.19,6,14498,United Kingdom +581498,12/9/2019,23201,Jumbo Bag Alphabet,6.19,6,14498,United Kingdom +581498,12/9/2019,85099B,Jumbo Bag Red Retrospot,6.19,5,14498,United Kingdom +581498,12/9/2019,85099C,Jumbo Bag Baroque Black White,6.19,4,14498,United Kingdom +581498,12/9/2019,20726,Lunch Bag Woodland,6.19,3,14498,United Kingdom +581498,12/9/2019,20728,Lunch Bag Cars Blue,6.19,4,14498,United Kingdom +581498,12/9/2019,22384,Lunch Bag Pink Polkadot,7.24,1,14498,United Kingdom +581498,12/9/2019,23437,50'S Christmas Gift Bag Large,7.24,1,14498,United Kingdom +581500,12/9/2019,82486,3 Drawer Antique White Wood Cabinet,7.24,4,15344,United King +581500,12/9/2019,85066,Cream Sweetheart Mini Chest,7.24,4,15344,United Kingdom +581500,12/9/2019,48187,Doormat New England,7.24,2,15344,United Kingdom +581501,12/9/2019,22319,Hairclips Forties Fabric Assorted,7.24,180,12985,United King +581501,12/9/2019,20704,Mr Robot Soft Toy,7.24,8,12985,United Kingdom +581501,12/9/2019,21564,Pink Heart Shape Love Bucket,6.19,24,12985,United Kingdom +581501,12/9/2019,21563,Red Heart Shape Love Bucket,7.24,24,12985,United Kingdom +581501,12/9/2019,22165,Diamante Heart Shaped Wall Mirror,7.24,12,12985,United Kingd +581501,12/9/2019,22299,Pig Keyring With Light & Sound,7.24,48,12985,United Kingdom +581501,12/9/2019,22447,Pin Cushion Babushka Blue,7.24,12,12985,United Kingdom +581501,12/9/2019,22442,Grow Your Own Flowers Set Of 3,7.24,12,12985,United Kingdom +581501,12/9/2019,22495,Set Of 2 Round Tins Camembert,7.24,12,12985,United Kingdom +581501,12/9/2019,22544,Mini Jigsaw Spaceboy,7.24,96,12985,United Kingdom +581501,12/9/2019,22695,Wicker Wreath Small,7.24,24,12985,United Kingdom +581501,12/9/2019,22785,Squarecushion Cover Pink Union Jack,7.24,12,12985,United Kin +581501,12/9/2019,22811,Set Of 6 T-Lights Cacti,7.24,12,12985,United Kingdom +581501,12/9/2019,22807,Set Of 6 T-Lights Toadstools,7.24,12,12985,United Kingdom +581501,12/9/2019,22942,Christmas Lights 10 Santas,7.24,12,12985,United Kingdom +581501,12/9/2019,22808,Set Of 6 T-Lights Easter Chicks,6.19,12,12985,United Kingdom +581501,12/9/2019,23143,Zinc Wire Kitchen Organiser,7.24,4,12985,United Kingdom +581501,12/9/2019,23151,Zinc Sweetheart Soap Dish,7.24,12,12985,United Kingdom +581501,12/9/2019,23425,Storage Tin Home Sweet Home,7.24,12,12985,United Kingdom +581501,12/9/2019,84356,Pompom Curtain,7.24,12,12985,United Kingdom +581501,12/9/2019,85173,Set/6 Frog Prince T-Light Candles,7.24,12,12985,United Kingd +581501,12/9/2019,21731,Red Toadstool Led Night Light,7.24,24,12985,United Kingdom +581501,12/9/2019,23480,Mini Lights Woodland Mushrooms,7.24,8,12985,United Kingdom +581501,12/9/2019,22545,Mini Jigsaw Bunnies,7.24,96,12985,United Kingdom +581502,12/9/2019,22087,Paper Bunting White Lace,7.24,6,15910,United Kingdom +581502,12/9/2019,21209,Multicolour Honeycomb Fan,7.24,5,15910,United Kingdom +581502,12/9/2019,20668,Disco Ball Christmas Decoration,7.24,24,15910,United Kingdom +581502,12/9/2019,21790,Vintage Snap Cards,7.24,6,15910,United Kingdom +581502,12/9/2019,23270,Set Of 2 Ceramic Painted Hearts,7.24,4,15910,United Kingdom +581502,12/9/2019,23103,Jingle Bell Heart Decoration,7.24,8,15910,United Kingdom +581502,12/9/2019,22576,Swallow Wooden Christmas Decoration,7.24,2,15910,United King +581502,12/9/2019,22153,Angel Decoration Stars On Dress,7.24,13,15910,United Kingdom +581502,12/9/2019,21810,Christmas Hanging Star With Bell,7.24,6,15910,United Kingdom +581502,12/9/2019,22155,Star Decoration Rustic,7.24,6,15910,United Kingdom +581502,12/9/2019,23210,White Rocking Horse Hand Painted,7.24,12,15910,United Kingdo +581502,12/9/2019,22573,Star Wooden Christmas Decoration,7.24,8,15910,United Kingdom +581502,12/9/2019,22075,6 Ribbons Elegant Christmas,7.24,4,15910,United Kingdom +581502,12/9/2019,85049A,Traditional Christmas Ribbons,7.24,4,15910,United Kingdom +581502,12/9/2019,22734,Set Of 6 Ribbons Vintage Christmas,7.24,4,15910,United Kingd +581502,12/9/2019,23274,Star T-Light Holder Willie Winkie,7.24,8,15910,United Kingdo +581502,12/9/2019,22153,Angel Decoration Stars On Dress,7.24,1,15910,United Kingdom +581502,12/9/2019,22596,Christmas Star Wish List Chalkboard,7.24,24,15910,United Kin +581502,12/9/2019,22952,60 Cake Cases Vintage Christmas,7.24,10,15910,United Kingdom +581502,12/9/2019,22141,Christmas Craft Tree Top Angel,7.24,3,15910,United Kingdom +581514,12/9/2019,22753,Small Yellow Babushka Notebook,7.24,12,17754,United Kingdom +581514,12/9/2019,22755,Small Purple Babushka Notebook,7.24,12,17754,United Kingdom +581514,12/9/2019,22754,Small Red Babushka Notebook,6.19,12,17754,United Kingdom +581514,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,6.19,4,17754,United Kingdom +581514,12/9/2019,22059,Ceramic Strawberry Design Mug,6.19,12,17754,United Kingdom +581514,12/9/2019,22199,Frying Pan Red Retrospot,6.19,13,17754,United Kingdom +581514,12/9/2019,22200,Frying Pan Pink Polkadot,6.19,2,17754,United Kingdom +581514,12/9/2019,84032A,Charlie+Lola Pink Hot Water Bottle,6.19,9,17754,United King +581514,12/9/2019,84032B,Charlie + Lola Red Hot Water Bottle,6.19,9,17754,United Kin +581514,12/9/2019,84031A,Charlie+Lola Red Hot Water Bottle,6.19,10,17754,United King +581514,12/9/2019,84031B,Charlie Lola Blue Hot Water Bottle,6.19,14,17754,United Kin +581514,12/9/2019,22646,Ceramic Strawberry Cake Money Bank,6.19,4,17754,United Kingd +581514,12/9/2019,22644,Ceramic Cherry Cake Money Bank,6.19,4,17754,United Kingdom +581514,12/9/2019,22645,Ceramic Heart Fairy Cake Money Bank,6.19,4,17754,United King +581514,12/9/2019,22394,Paperweight Kings Choice,6.04,12,17754,United Kingdom +581514,12/9/2019,17091J,Vanilla Incense In Tin,6.04,66,17754,United Kingdom +581514,12/9/2019,35471D,Set Of 3 Bird Light Pink Feather,6.04,12,17754,United Kingd +581514,12/9/2019,22075,6 Ribbons Elegant Christmas,6.04,24,17754,United Kingdom +581514,12/9/2019,17091J,Vanilla Incense In Tin,6.19,24,17754,United Kingdom +581514,12/9/2019,22069,Brown Pirate Treasure Chest,6.19,20,17754,United Kingdom +581514,12/9/2019,22068,Black Pirate Treasure Chest,6.19,14,17754,United Kingdom +581514,12/9/2019,22500,Set Of 2 Tins Jardin De Provence,6.19,4,17754,United Kingdom +581514,12/9/2019,22075,6 Ribbons Elegant Christmas,6.19,24,17754,United Kingdom +581514,12/9/2019,21705,Bag 500g Swirly Marbles,6.19,84,17754,United Kingdom +581516,12/9/2019,21109,Large Cake Towel Chocolate Spots,6.19,12,14422,United Kingdo +581516,12/9/2019,21111,Swiss Roll Towel Chocolate Spots,6.19,24,14422,United Kingdo +581516,12/9/2019,21705,Bag 500g Swirly Marbles,6.19,24,14422,United Kingdom +581516,12/9/2019,22185,Slate Tile Natural Hanging,6.19,12,14422,United Kingdom +581516,12/9/2019,22442,Grow Your Own Flowers Set Of 3,6.19,12,14422,United Kingdom +581516,12/9/2019,21620,Set Of 4 Rose Botanical Candles,6.19,12,14422,United Kingdom +581516,12/9/2019,84029G,Knitted Union Flag Hot Water Bottle,6.19,8,14422,United Kin +581516,12/9/2019,21485,Retrospot Heart Hot Water Bottle,6.19,3,14422,United Kingdom +581516,12/9/2019,22111,Scottie Dog Hot Water Bottle,6.19,6,14422,United Kingdom +581516,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,8,14422,United Kingdom +581516,12/9/2019,22112,Chocolate Hot Water Bottle,6.19,6,14422,United Kingdom +581516,12/9/2019,23356,Love Hot Water Bottle,6.19,6,14422,United Kingdom +581516,12/9/2019,21108,Fairy Cake Flannel Assorted Colour,6.19,18,14422,United King +581516,12/9/2019,22171,3 Hook Photo Shelf Antique White,6.19,4,14422,United Kingdom +581516,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,24,14422,United Kingdo +581538,12/9/2019,23193,Buffalo Bill Treasure Book Box,6.19,6,14446,United Kingdom +581538,12/9/2019,23194,Gymkhana Treasure Book Box,6.19,1,14446,United Kingdom +581538,12/9/2019,23084,Rabbit Night Light,6.19,2,14446,United Kingdom +581538,12/9/2019,22068,Black Pirate Treasure Chest,6.19,1,14446,United Kingdom +581538,12/9/2019,22956,36 Foil Heart Cake Cases,6.19,1,14446,United Kingdom +581538,12/9/2019,20936,Forked Cactus Candle,6.19,1,14446,United Kingdom +581538,12/9/2019,23040,Paper Lantern 9 Point Snow Star,6.19,1,14446,United Kingdom +581538,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,6.19,1,14446,United King +581538,12/9/2019,21222,Set/4 Badges Beetles,6.19,1,14446,United Kingdom +581538,12/9/2019,21220,Set/4 Badges Dogs,6.19,1,14446,United Kingdom +581538,12/9/2019,21224,Set/4 Skull Badges,6.19,1,14446,United Kingdom +581538,12/9/2019,85123A,Cream Hanging Heart T-Light Holder,6.19,1,14446,United King +581538,12/9/2019,22992,Revolver Wooden Ruler,6.19,1,14446,United Kingdom +581538,12/9/2019,79066K,Retro Mod Tray,6.19,1,14446,United Kingdom +581538,12/9/2019,84380,Set Of 3 Butterfly Cookie Cutters,6.19,1,14446,United Kingdo +581538,12/9/2019,23371,Set 36 Colour Pencils Spaceboy,6.19,1,14446,United Kingdom +581538,12/9/2019,22694,Wicker Star,6.19,1,14446,United Kingdom +581538,12/9/2019,22095,Lads Only Tissue Box,6.19,1,14446,United Kingdom +581538,12/9/2019,23275,Set Of 3 Hanging Owls Ollie Beak,6.19,1,14446,United Kingdom +581538,12/9/2019,21673,White Spot Blue Ceramic Drawer Knob,6.19,1,14446,United King +581538,12/9/2019,21670,Blue Spot Ceramic Drawer Knob,6.19,1,14446,United Kingdom +581538,12/9/2019,85071C,"Charlie+Lola""Extremely Busy"" Sign",6.19,1,14446,United K +581538,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,3,14446,United Kingdom +581538,12/9/2019,23034,Drawer Knob Ceramic Black,6.19,1,14446,United Kingdom +581538,12/9/2019,21669,Blue Stripe Ceramic Drawer Knob,6.19,1,14446,United Kingdom +581538,12/9/2019,23033,Drawer Knob Ceramic Red,6.19,1,14446,United Kingdom +581538,12/9/2019,21668,Red Stripe Ceramic Drawer Knob,6.19,1,14446,United Kingdom +581538,12/9/2019,23275,Set Of 3 Hanging Owls Ollie Beak,6.19,1,14446,United Kingdom +581538,12/9/2019,23318,Box Of 6 Mini Vintage Crackers,6.19,1,14446,United Kingdom +581538,12/9/2019,23382,Box Of 6 Christmas Cake Decorations,6.19,1,14446,United King +581538,12/9/2019,22469,Heart Of Wicker Small,6.19,1,14446,United Kingdom +581538,12/9/2019,22899,Children's Apron Dolly Girl,6.19,2,14446,United Kingdom +581538,12/9/2019,22367,Childrens Apron Spaceboy Design,6.19,1,14446,United Kingdom +581538,12/9/2019,22899,Children's Apron Dolly Girl,6.19,3,14446,United Kingdom +581538,12/9/2019,23527,Wall Art Animals And Nature,6.19,1,14446,United Kingdom +581538,12/9/2019,23524,Wall Art Horse & Pony,6.19,1,14446,United Kingdom +581538,12/9/2019,23525,Wall Art Buffalo Bill,6.19,1,14446,United Kingdom +581538,12/9/2019,84380,Set Of 3 Butterfly Cookie Cutters,6.19,4,14446,United Kingdo +581538,12/9/2019,23122,Party Charms 50 Pieces,7.24,1,14446,United Kingdom +581538,12/9/2019,21990,Modern Floral Stationery Set,7.24,1,14446,United Kingdom +581538,12/9/2019,21329,Dinosaurs Writing Set,7.24,1,14446,United Kingdom +581538,12/9/2019,21328,Balloons Writing Set,7.24,1,14446,United Kingdom +581538,12/9/2019,22561,Wooden School Colouring Set,7.24,1,14446,United Kingdom +581538,12/9/2019,84519A,Tomato Charlie+Lola Coaster Set,7.24,1,14446,United Kingdom +581538,12/9/2019,84519B,Carrot Charlie+Lola Coaster Set,7.24,1,14446,United Kingdom +581538,12/9/2019,35004B,Set Of 3 Black Flying Ducks,7.24,2,14446,United Kingdom +581538,12/9/2019,22068,Black Pirate Treasure Chest,7.24,1,14446,United Kingdom +581538,12/9/2019,23353,6 Gift Tags Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,21591,Cosy Hour Cigar Box Matches,6.19,1,14446,United Kingdom +581538,12/9/2019,22197,Popcorn Holder,6.19,4,14446,United Kingdom +581538,12/9/2019,23320,Giant 50'S Christmas Cracker,6.19,1,14446,United Kingdom +581538,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,22985,Wrap Billboard Fonts Design,6.19,25,14446,United Kingdom +581538,12/9/2019,23040,Paper Lantern 9 Point Snow Star,6.19,2,14446,United Kingdom +581538,12/9/2019,23040,Paper Lantern 9 Point Snow Star,6.19,3,14446,United Kingdom +581538,12/9/2019,21194,Pink Honeycomb Paper Fan,6.19,2,14446,United Kingdom +581538,12/9/2019,21208,Pastel Colour Honeycomb Fan,6.19,1,14446,United Kingdom +581538,12/9/2019,79190D,Retro Plastic Daisy Tray,6.19,1,14446,United Kingdom +581538,12/9/2019,79190B,Retro Plastic Polka Tray,6.19,1,14446,United Kingdom +581538,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,6.02,2,14446,United King +581538,12/9/2019,23318,Box Of 6 Mini Vintage Crackers,6.02,1,14446,United Kingdom +581538,12/9/2019,23319,Box Of 6 Mini 50'S Crackers,6.02,1,14446,United Kingdom +581538,12/9/2019,23537,Wall Art I Love London,6.19,1,14446,United Kingdom +581538,12/9/2019,22992,Revolver Wooden Ruler,6.19,1,14446,United Kingdom +581538,12/9/2019,22991,Giraffe Wooden Ruler,6.19,1,14446,United Kingdom +581538,12/9/2019,23190,Bundle Of 3 School Exercise Books,6.19,1,14446,United Kingdo +581538,12/9/2019,21194,Pink Honeycomb Paper Fan,6.19,1,14446,United Kingdom +581538,12/9/2019,35004B,Set Of 3 Black Flying Ducks,6.19,1,14446,United Kingdom +581538,12/9/2019,22694,Wicker Star,6.19,1,14446,United Kingdom +581538,12/9/2019,21355,Toast Its - I Love You,6.19,1,14446,United Kingdom +581538,12/9/2019,23343,Jumbo Bag Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,23343,Jumbo Bag Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,20727,Lunch Bag Black Skull,6.19,1,14446,United Kingdom +581538,12/9/2019,20725,Lunch Bag Red Retrospot,6.19,1,14446,United Kingdom +581566,12/9/2019,23404,Home Sweet Home Blackboard,6.19,144,18102,United Kingdom +581567,12/9/2019,21417,Cockle Shell Dish,6.19,84,16626,United Kingdom +581567,12/9/2019,22464,Hanging Metal Heart Lantern,6.19,24,16626,United Kingdom +581567,12/9/2019,22465,Hanging Metal Star Lantern,6.19,24,16626,United Kingdom +581567,12/9/2019,84971S,Small Heart Flowers Hook,6.19,48,16626,United Kingdom +581567,12/9/2019,22624,Ivory Kitchen Scales,6.19,2,16626,United Kingdom +581567,12/9/2019,22627,Mint Kitchen Scales,6.19,2,16626,United Kingdom +581567,12/9/2019,22625,Red Kitchen Scales,6.19,2,16626,United Kingdom +581567,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,4,16626,United Kingdom +581567,12/9/2019,21326,Aged Glass Silver T-Light Holder,6.19,144,16626,United Kingd +581567,12/9/2019,21479,White Skull Hot Water Bottle,6.19,4,16626,United Kingdom +581567,12/9/2019,23356,Love Hot Water Bottle,6.19,3,16626,United Kingdom +581567,12/9/2019,21137,Black Record Cover Frame,6.19,24,16626,United Kingdom +581570,12/9/2019,22141,Christmas Craft Tree Top Angel,6.19,6,12662,Germany +581570,12/9/2019,22175,Pink Owl Soft Toy,6.19,6,12662,Germany +581570,12/9/2019,21481,Fawn Blue Hot Water Bottle,6.19,4,12662,Germany +581570,12/9/2019,23570,Traditional Pick Up Sticks Game,6.19,12,12662,Germany +581570,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,6.19,6,12662,Germany +581570,12/9/2019,22331,Woodland Party Bag + Sticker Set,6.19,8,12662,Germany +581570,12/9/2019,22834,Hand Warmer Babushka Design,6.19,12,12662,Germany +581570,12/9/2019,21914,Blue Harmonica In Box,6.19,12,12662,Germany +581570,12/9/2019,22139,Retrospot Tea Set Ceramic 11 Pc,6.19,3,12662,Germany +581570,12/9/2019,23077,Doughnut Lip Gloss,6.19,20,12662,Germany +581570,12/9/2019,20750,Red Retrospot Mini Cases,6.19,2,12662,Germany +581570,12/9/2019,22505,Memo Board Cottage Design,6.19,4,12662,Germany +581571,12/9/2019,23326,Hanging Mini Coloured Bottles,6.19,6,15311,United Kingdom +581571,12/9/2019,21313,Glass Heart T-Light Holder,6.19,1,15311,United Kingdom +581571,12/9/2019,48187,Doormat New England,6.19,1,15311,United Kingdom +581571,12/9/2019,23317,Blue Refectory Clock,6.19,1,15311,United Kingdom +581571,12/9/2019,23197,Sketchbook Magnetic Shopping List,6.19,4,15311,United Kingdo +581571,12/9/2019,21012,Antique All Glass Candlestick,6.19,2,15311,United Kingdom +581571,12/9/2019,22227,Hanging Heart Mirror Decoration,6.19,10,15311,United Kingdom +581571,12/9/2019,22794,Sweetheart Wire Magazine Rack,6.19,1,15311,United Kingdom +581571,12/9/2019,23182,Toilet Sign Occupied Or Vacant,6.19,1,15311,United Kingdom +581571,12/9/2019,21755,Love Building Block Word,6.19,1,15311,United Kingdom +581571,12/9/2019,85053,French Enamel Candleholder,6.19,2,15311,United Kingdom +581571,12/9/2019,23110,Parisienne Key Cabinet,6.19,2,15311,United Kingdom +581571,12/9/2019,21169,You're Confusing Me Metal Sign,6.19,1,15311,United Kingdom +581571,12/9/2019,21258,Victorian Sewing Box Large,6.19,8,15311,United Kingdom +581571,12/9/2019,23168,Classic Cafe Sugar Dispenser,6.19,36,15311,United Kingdom +581571,12/9/2019,23167,Small Ceramic Top Storage Jar,6.19,96,15311,United Kingdom +581571,12/9/2019,21314,Small Glass Heart Trinket Pot,6.19,48,15311,United Kingdom +581571,12/9/2019,21137,Black Record Cover Frame,6.19,24,15311,United Kingdom +581571,12/9/2019,44234,Assorted Circular Mobile,6.19,1,15311,United Kingdom +581571,12/9/2019,84347,Rotating Silver Angels T-Light Hldr,6.19,24,15311,United Kin +581572,12/9/2019,23328,Set 6 School Milk Bottles In Crate,6.19,48,16705,United King +581572,12/9/2019,22627,Mint Kitchen Scales,6.19,4,16705,United Kingdom +581572,12/9/2019,22624,Ivory Kitchen Scales,6.19,4,16705,United Kingdom +581572,12/9/2019,23245,Set Of 3 Regency Cake Tins,6.19,4,16705,United Kingdom +581574,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,6.19,6,12526,Germany +581574,12/9/2019,22328,Round Snack Boxes Set Of 4 Fruits,6.19,6,12526,Germany +581574,12/9/2019,23238,Set Of 4 Knick Knack Tins London,6.19,6,12526,Germany +581574,12/9/2019,23237,Set Of 4 Knick Knack Tins Leaf,6.19,6,12526,Germany +581574,12/9/2019,21257,Victorian Sewing Box Medium,6.19,2,12526,Germany +581574,12/9/2019,21258,Victorian Sewing Box Large,6.19,2,12526,Germany +581574,12/9/2019,23111,Parisienne Sewing Box,6.19,2,12526,Germany +581574,12/9/2019,22077,6 Ribbons Rustic Charm,6.19,12,12526,Germany +581574,12/9/2019,22074,6 Ribbons Shimmering Pinks,6.19,12,12526,Germany +581574,12/9/2019,22621,Traditional Knitting Nancy,6.19,12,12526,Germany +581574,12/9/2019,23199,Jumbo Bag Apples,6.19,10,12526,Germany +581574,12/9/2019,23581,Jumbo Bag Paisley Park,6.19,10,12526,Germany +581578,12/9/2019,21124,Set/10 Blue Polkadot Party Candles,6.19,24,12713,Germany +581578,12/9/2019,21122,Set/10 Pink Polkadot Party Candles,6.19,24,12713,Germany +581578,12/9/2019,21121,Set/10 Red Polkadot Party Candles,6.19,24,12713,Germany +581578,12/9/2019,23389,Spaceboy Mini Backpack,6.04,4,12713,Germany +581578,12/9/2019,22556,Plasters In Tin Circus Parade,6.19,12,12713,Germany +581578,12/9/2019,22976,Circus Parade Childrens Egg Cup,6.19,12,12713,Germany +581578,12/9/2019,23255,Childrens Cutlery Circus Parade,7.24,12,12713,Germany +581578,12/9/2019,84997D,Childrens Cutlery Polkadot Pink,7.24,8,12713,Germany +581578,12/9/2019,84997B,Childrens Cutlery Retrospot Red,7.24,8,12713,Germany +581578,12/9/2019,84997C,Childrens Cutlery Polkadot Blue,7.24,8,12713,Germany +581578,12/9/2019,22555,Plasters In Tin Strongman,7.24,12,12713,Germany +581578,12/9/2019,21914,Blue Harmonica In Box,7.24,12,12713,Germany +581578,12/9/2019,22549,Picture Dominoes,7.24,24,12713,Germany +581578,12/9/2019,21918,Set 12 Kids Colour Chalk Sticks,7.24,24,12713,Germany +581578,12/9/2019,22992,Revolver Wooden Ruler,7.24,12,12713,Germany +581578,12/9/2019,22991,Giraffe Wooden Ruler,7.24,12,12713,Germany +581578,12/9/2019,23229,Vintage Donkey Tail Game,7.24,6,12713,Germany +581578,12/9/2019,22622,Box Of Vintage Alphabet Blocks,6.19,6,12713,Germany +581578,12/9/2019,21506,Fancy Font Birthday Card,6.19,12,12713,Germany +581578,12/9/2019,21507,Elephant Birthday Card,6.19,12,12713,Germany +581578,12/9/2019,23037,Candle Holder Silver Madeline,6.19,12,12713,Germany +581578,12/9/2019,23550,Wrap Alphabet Poster,6.19,25,12713,Germany +581578,12/9/2019,22711,Wrap Circus Parade,6.19,25,12713,Germany +581578,12/9/2019,21497,Fancy Fonts Birthday Wrap,6.19,25,12713,Germany +581578,12/9/2019,22704,Wrap Red Apples,6.19,25,12713,Germany +581578,12/9/2019,22585,Pack Of 6 Birdy Gift Tags,6.19,12,12713,Germany diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/python-challenge/input.csv b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/python-challenge/input.csv new file mode 100644 index 0000000000000..85854a2d43584 --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/python-challenge/input.csv @@ -0,0 +1,1500 @@ +TransactionNo,Date,ProductNo,ProductName,Price,Quantity,CustomerNo,Country +581482,12/9/2019,22485,Set Of 2 Wooden Market Crates,21.47,12,17490,United Kingdom +581475,12/9/2019,22596,Christmas Star Wish List Chalkboard,10.65,36,13069,United Ki +581475,12/9/2019,23235,Storage Tin Vintage Leaf,11.53,12,13069,United Kingdom +581475,12/9/2019,23272,Tree T-Light Holder Willie Winkie,10.65,12,13069,United King +581475,12/9/2019,23239,Set Of 4 Knick Knack Tins Poppies,11.94,6,13069,United Kingd +581475,12/9/2019,21705,Bag 500g Swirly Marbles,10.65,24,13069,United Kingdom +581475,12/9/2019,22118,Joy Wooden Block Letters,11.53,18,13069,United Kingdom +581475,12/9/2019,22119,Peace Wooden Block Letters,12.25,12,13069,United Kingdom +581475,12/9/2019,22217,T-Light Holder Hanging Lace,10.65,12,13069,United Kingdom +581475,12/9/2019,22216,T-Light Holder White Lace,10.55,24,13069,United Kingdom +581475,12/9/2019,22380,Toy Tidy Spaceboy,11.06,20,13069,United Kingdom +581475,12/9/2019,22442,Grow Your Own Flowers Set Of 3,12.25,12,13069,United Kingdom +581475,12/9/2019,22664,Toy Tidy Dolly Girl Design,11.06,20,13069,United Kingdom +581475,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,12.25,12,13069,United Kingdom +581475,12/9/2019,22723,Set Of 6 Herb Tins Sketchbook,11.53,12,13069,United Kingdom +581475,12/9/2019,22785,Squarecushion Cover Pink Union Jack,11.53,12,13069,United Ki +581475,12/9/2019,22955,36 Foil Star Cake Cases,11.06,24,13069,United Kingdom +581475,12/9/2019,23141,Triple Wire Hook Pink Heart,11.06,12,13069,United Kingdom +581475,12/9/2019,22956,36 Foil Heart Cake Cases,11.06,24,13069,United Kingdom +581475,12/9/2019,22581,Wood Stocking Christmas Scandispot,10.55,48,13069,United Kin +581476,12/9/2019,23198,Pantry Magnetic Shopping List,11.53,48,12433,Norway +581476,12/9/2019,23197,Sketchbook Magnetic Shopping List,11.74,24,12433,Norway +581476,12/9/2019,23184,Bull Dog Bottle Opener,15.32,8,12433,Norway +581476,12/9/2019,23168,Classic Cafe Sugar Dispenser,11.53,12,12433,Norway +581476,12/9/2019,23167,Small Ceramic Top Storage Jar,10.96,96,12433,Norway +581476,12/9/2019,23166,Medium Ceramic Top Storage Jar,11.32,48,12433,Norway +581476,12/9/2019,23165,Large Ceramic Top Storage Jar,11.74,48,12433,Norway +581476,12/9/2019,23004,Travel Card Wallet Pantry,10.68,48,12433,Norway +581476,12/9/2019,23002,Travel Card Wallet Skulls,10.68,24,12433,Norway +581476,12/9/2019,23000,Travel Card Wallet Transport,10.68,24,12433,Norway +581476,12/9/2019,22998,Travel Card Wallet Keep Calm,10.68,72,12433,Norway +581476,12/9/2019,22994,Travel Card Wallet Retrospot,10.68,48,12433,Norway +581476,12/9/2019,22835,Hot Water Bottle I Am So Poorly,15.32,8,12433,Norway +581476,12/9/2019,22730,Alarm Clock Bakelike Ivory,14.09,4,12433,Norway +581476,12/9/2019,22728,Alarm Clock Bakelike Pink,14.09,8,12433,Norway +581476,12/9/2019,22727,Alarm Clock Bakelike Red,14.09,8,12433,Norway +581476,12/9/2019,22726,Alarm Clock Bakelike Green,14.09,4,12433,Norway +581476,12/9/2019,22720,Set Of 3 Cake Tins Pantry Design,14.61,16,12433,Norway +581476,12/9/2019,22693,Grow A Flytrap Or Sunflower In Tin,11.34,192,12433,Norway +581476,12/9/2019,22670,French Wc Sign Blue Metal,11.53,24,12433,Norway +581476,12/9/2019,22667,Recipe Box Retrospot,12.86,24,12433,Norway +581476,12/9/2019,22666,Recipe Box Pantry Yellow Design,12.86,24,12433,Norway +581476,12/9/2019,22631,Circus Parade Lunch Box,12.25,12,12433,Norway +581476,12/9/2019,22628,Picnic Boxes Set Of 3 Retrospot,15.32,16,12433,Norway +581476,12/9/2019,22467,Gumball Coat Rack,12.86,6,12433,Norway +581476,12/9/2019,22197,Popcorn Holder,10.99,100,12433,Norway +581476,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,14.61,8,12433,Norway +581476,12/9/2019,22112,Chocolate Hot Water Bottle,15.32,9,12433,Norway +581476,12/9/2019,21908,Chocolate This Way Metal Sign,12.40,36,12433,Norway +581476,12/9/2019,21874,Gin And Tonic Mug,11.74,36,12433,Norway +581476,12/9/2019,21872,Glamorous Mug,11.53,12,12433,Norway +581476,12/9/2019,21871,Save The Planet Mug,11.74,36,12433,Norway +581476,12/9/2019,21533,Retrospot Large Milk Jug,15.32,6,12433,Norway +581476,12/9/2019,21481,Fawn Blue Hot Water Bottle,14.09,12,12433,Norway +581476,12/9/2019,21479,White Skull Hot Water Bottle,14.61,4,12433,Norway +581476,12/9/2019,21248,Door Hanger Mum + Dads Room,11.74,12,12433,Norway +581476,12/9/2019,21216,Set 3 Retrospot Tea/Coffee/Sugar,14.61,24,12433,Norway +581476,12/9/2019,21181,Please One Person Metal Sign,12.15,48,12433,Norway +581476,12/9/2019,21175,Gin And Tonic Diet Metal Sign,12.38,48,12433,Norway +581476,12/9/2019,21169,You're Confusing Me Metal Sign,11.74,48,12433,Norway +581476,12/9/2019,21162,Toxic Area Door Hanger,10.65,24,12433,Norway +581476,12/9/2019,21159,Moody Boy Door Hanger,10.65,24,12433,Norway +581476,12/9/2019,21158,Moody Girl Door Hanger,10.65,24,12433,Norway +581476,12/9/2019,21154,Red Retrospot Oven Glove,11.53,10,12433,Norway +581476,12/9/2019,16016,Large Chinese Style Scissor,11.12,40,12433,Norway +581476,12/9/2019,16014,Small Chinese Style Scissor,10.68,60,12433,Norway +581476,12/9/2019,16008,Small Folding Scissor(Pointed Edge),10.37,240,12433,Norway +581476,12/9/2019,85152,Hand Over The Chocolate Sign,12.15,48,12433,Norway +581476,12/9/2019,84596F,Small Marshmallows Pink Bowl,10.68,32,12433,Norway +581476,12/9/2019,84596B,Small Dolly Mix Design Orange Bowl,10.68,16,12433,Norway +581476,12/9/2019,84510A,Set Of 4 English Rose Coasters,11.53,20,12433,Norway +581476,12/9/2019,82600,N0 Singing Metal Sign,12.15,48,12433,Norway +581476,12/9/2019,82581,Toilet Metal Sign,10.81,48,12433,Norway +581476,12/9/2019,72232,Feng Shui Pillar Candle,10.44,144,12433,Norway +581476,12/9/2019,47559B,Tea Time Oven Glove,11.53,10,12433,Norway +581476,12/9/2019,47504H,English Rose Spirit Level,11.06,36,12433,Norway +581476,12/9/2019,23493,Vintage Doily Travel Sewing Kit,12.25,30,12433,Norway +581476,12/9/2019,23430,Blue Retro Kitchen Wall Clock,18.60,2,12433,Norway +581476,12/9/2019,23429,Red Retro Kitchen Wall Clock,18.60,2,12433,Norway +581476,12/9/2019,23428,Ivory Retro Kitchen Wall Clock,18.60,2,12433,Norway +581476,12/9/2019,23358,Hot Stuff Hot Water Bottle,11.53,18,12433,Norway +581476,12/9/2019,23293,Set Of 12 Fairy Cake Baking Cases,11.10,32,12433,Norway +581476,12/9/2019,23243,Set Of Tea Coffee Sugar Tins Pantry,15.32,12,12433,Norway +581476,12/9/2019,23240,Set Of 4 Knick Knack Tins Doily,14.50,6,12433,Norway +581477,12/9/2019,48111,Doormat 3 Smiley Cats,17.51,10,13426,United Kingdom +581477,12/9/2019,22464,Hanging Metal Heart Lantern,11.06,12,13426,United Kingdom +581477,12/9/2019,20982,12 Pencils Tall Tube Skulls,11.12,12,13426,United Kingdom +581477,12/9/2019,20981,12 Pencils Tall Tube Woodland,11.12,12,13426,United Kingdom +581477,12/9/2019,23424,Gingham Recipe Book Box,15.32,8,13426,United Kingdom +581477,12/9/2019,23338,Egg Frying Pan Red,12.15,48,13426,United Kingdom +581477,12/9/2019,84970L,Single Heart Zinc T-Light Holder,11.53,12,13426,United King +581477,12/9/2019,22457,Natural Slate Heart Chalkboard,13.27,6,13426,United Kingdom +581477,12/9/2019,22469,Heart Of Wicker Small,11.94,12,13426,United Kingdom +581477,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,11.12,12,13426,United Ki +581478,12/9/2019,84947,Antique Silver Tea Glass Engraved,11.53,24,17364,United King +581478,12/9/2019,23503,Playing Cards Keep Calm & Carry On,11.53,12,17364,United Kin +581478,12/9/2019,23445,Ice Cream Bubbles,11.10,20,17364,United Kingdom +581478,12/9/2019,23530,Wall Art Only One Person,15.32,12,17364,United Kingdom +581478,12/9/2019,84997C,Childrens Cutlery Polkadot Blue,14.50,4,17364,United Kingdo +581478,12/9/2019,84997D,Childrens Cutlery Polkadot Pink,14.50,4,17364,United Kingdo +581478,12/9/2019,84077,World War 2 Gliders Asstd Designs,10.55,48,17364,United King +581478,12/9/2019,22749,Feltcraft Princess Charlotte Doll,14.09,4,17364,United Kingd +581478,12/9/2019,23127,Feltcraft Girl Nicole Kit,15.32,4,17364,United Kingdom +581478,12/9/2019,23126,Feltcraft Girl Amelie Kit,15.32,4,17364,United Kingdom +581478,12/9/2019,22747,Poppy's Playhouse Bathroom,12.40,6,17364,United Kingdom +581478,12/9/2019,22078,Ribbon Reel Lace Design,12.40,10,17364,United Kingdom +581478,12/9/2019,84946,Antique Silver T-Light Glass,11.53,12,17364,United Kingdom +581478,12/9/2019,22791,T-Light Glass Fluted Antique,11.53,12,17364,United Kingdom +581478,12/9/2019,21326,Aged Glass Silver T-Light Holder,10.92,12,17364,United Kingd +581478,12/9/2019,22170,Picture Frame Wood Triple Portrait,17.17,8,17364,United King +581478,12/9/2019,23082,Set 6 Paper Table Lantern Hearts,14.09,6,17364,United Kingdo +581478,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,11.12,24,17364,United Ki +581479,12/9/2019,22087,Paper Bunting White Lace,13.27,10,17364,United Kingdom +581480,12/9/2019,23464,Vintage Zinc Watering Can Small,15.32,4,14441,United Kingdom +581480,12/9/2019,22411,Jumbo Shopper Vintage Red Paisley,12.38,10,14441,United King +581480,12/9/2019,84029E,Red Woolly Hottie White Heart,14.61,8,14441,United Kingdom +581480,12/9/2019,22633,Hand Warmer Union Jack,12.40,12,14441,United Kingdom +581480,12/9/2019,23355,Hot Water Bottle Keep Calm,15.32,12,14441,United Kingdom +581480,12/9/2019,84029G,Knitted Union Flag Hot Water Bottle,14.61,12,14441,United K +581480,12/9/2019,22112,Chocolate Hot Water Bottle,15.32,6,14441,United Kingdom +581480,12/9/2019,84347,Rotating Silver Angels T-Light Hldr,12.86,18,14441,United Ki +581481,12/9/2019,21115,Rose Caravan Doorstop,12.25,8,17490,United Kingdom +581481,12/9/2019,22059,Ceramic Strawberry Design Mug,10.65,24,17490,United Kingdom +581481,12/9/2019,22072,Red Retrospot Tea Cup And Saucer,11.53,24,17490,United Kingd +581481,12/9/2019,22123,Ping Microwave Apron,11.06,24,17490,United Kingdom +581481,12/9/2019,22476,Empire Union Jack Tv Dinner Tray,12.25,8,17490,United Kingdo +581481,12/9/2019,22495,Set Of 2 Round Tins Camembert,11.06,12,17490,United Kingdom +581481,12/9/2019,22496,Set Of 2 Round Tins Dutch Cheese,11.06,12,17490,United Kingd +581481,12/9/2019,22513,Doorstop Football Design,11.06,8,17490,United Kingdom +581481,12/9/2019,22500,Set Of 2 Tins Jardin De Provence,11.53,12,17490,United Kingd +581481,12/9/2019,22664,Toy Tidy Dolly Girl Design,11.06,20,17490,United Kingdom +581481,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,12.25,12,17490,United Kingdom +581481,12/9/2019,22785,Squarecushion Cover Pink Union Jack,11.53,12,17490,United Ki +581481,12/9/2019,23178,Jam Clock Magnet,11.53,12,17490,United Kingdom +581481,12/9/2019,23302,Kneeling Mat Housework Design,11.06,24,17490,United Kingdom +581481,12/9/2019,23533,Wall Art Garden Haven,12.25,6,17490,United Kingdom +581481,12/9/2019,84819,Danish Rose Round Sewing Box,11.06,16,17490,United Kingdom +581482,12/9/2019,22371,Airline Bag Vintage Tokyo 78,14.30,12,17490,United Kingdom +581482,12/9/2019,21875,Kings Choice Mug,11.34,36,17490,United Kingdom +581482,12/9/2019,23251,Vintage Red Enamel Trim Mug,11.32,96,17490,United Kingdom +581482,12/9/2019,23300,Gardeners Kneeling Pad Cup Of Tea,11.74,48,17490,United King +581482,12/9/2019,22138,Baking Set 9 Piece Retrospot,14.61,24,17490,United Kingdom +581483,12/9/2019,23843,Paper Craft Little Birdie,12.38,80995,16446,United Kingdom +581485,12/9/2019,22617,Baking Set Spaceboy Design,15.32,18,17389,United Kingdom +581485,12/9/2019,20749,Assorted Colour Mini Cases,16.76,84,17389,United Kingdom +581486,12/9/2019,22910,Paper Chain Kit Vintage Christmas,13.27,12,17001,United King +581486,12/9/2019,22086,Paper Chain Kit 50'S Christmas,13.27,12,17001,United Kingdom +581486,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,12,17001,United Kingdom +581486,12/9/2019,23352,Roll Wrap 50'S Red Christmas,6.19,12,17001,United Kingdom +581486,12/9/2019,22727,Alarm Clock Bakelike Red,6.19,8,17001,United Kingdom +581486,12/9/2019,22730,Alarm Clock Bakelike Ivory,6.19,4,17001,United Kingdom +581486,12/9/2019,22491,Pack Of 12 Coloured Pencils,6.04,12,17001,United Kingdom +581486,12/9/2019,22561,Wooden School Colouring Set,6.04,12,17001,United Kingdom +581486,12/9/2019,22489,Pack Of 12 Traditional Crayons,6.04,24,17001,United Kingdom +581486,12/9/2019,22560,Traditional Modelling Clay,6.04,24,17001,United Kingdom +581486,12/9/2019,22139,Retrospot Tea Set Ceramic 11 Pc,6.04,6,17001,United Kingdom +581486,12/9/2019,23344,Jumbo Bag 50'S Christmas,6.19,20,17001,United Kingdom +581486,12/9/2019,23203,Jumbo Bag Vintage Doily,6.19,20,17001,United Kingdom +581486,12/9/2019,85099B,Jumbo Bag Red Retrospot,6.19,10,17001,United Kingdom +581486,12/9/2019,23201,Jumbo Bag Alphabet,6.19,20,17001,United Kingdom +581486,12/9/2019,23207,Lunch Bag Alphabet Design,6.19,10,17001,United Kingdom +581487,12/9/2019,21137,Black Record Cover Frame,6.04,120,15694,United Kingdom +581488,12/9/2019,23118,Parisienne Jewellery Drawer,6.04,16,17428,United Kingdom +581488,12/9/2019,23111,Parisienne Sewing Box,6.04,16,17428,United Kingdom +581488,12/9/2019,22179,Set 10 Night Owl Lights,6.04,24,17428,United Kingdom +581489,12/9/2019,22061,Large Cake Stand Hanging Strawbery,7.24,48,16954,United King +581489,12/9/2019,22182,Cake Stand Victorian Filigree Small,7.24,24,16954,United Kin +581491,12/9/2019,23571,Traditional Naughts & Crosses,7.24,12,12433,Norway +581492,12/9/2019,23369,Set 36 Colour Pencils Love London,6.19,2,15492,United Kingdo +581492,12/9/2019,23370,Set 36 Colouring Pencils Doily,6.19,2,15492,United Kingdom +581492,12/9/2019,23371,Set 36 Colour Pencils Spaceboy,6.19,3,15492,United Kingdom +581492,12/9/2019,23372,Set 36 Colour Pencils Dolly Girl,6.19,1,15492,United Kingdom +581492,12/9/2019,23376,Pack Of 12 Vintage Christmas Tissue,6.19,3,15492,United King +581492,12/9/2019,23377,Pack Of 12 Dolly Girl Tissues,6.19,6,15492,United Kingdom +581492,12/9/2019,23378,Pack Of 12 50'S Christmas Tissues,6.19,9,15492,United Kingdo +581492,12/9/2019,23379,Pack Of 12 Red Apple Tissues,6.19,4,15492,United Kingdom +581492,12/9/2019,23380,Pack Of 12 Vintage Doily Tissues,6.19,3,15492,United Kingdom +581492,12/9/2019,23381,Pack Of 12 Vintage Leaf Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,23382,Box Of 6 Christmas Cake Decorations,6.19,4,15492,United King +581492,12/9/2019,23391,I Love London Mini Backpack,6.19,2,15492,United Kingdom +581492,12/9/2019,23392,Spaceboy Rocket Lolly Makers,6.19,2,15492,United Kingdom +581492,12/9/2019,23399,Home Sweet Home Hanging Heart,6.19,4,15492,United Kingdom +581492,12/9/2019,23405,Home Sweet Home 2 Drawer Cabinet,6.19,3,15492,United Kingdom +581492,12/9/2019,23418,Lavender Toilette Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,23426,Metal Sign Drop Your Pants,6.19,1,15492,United Kingdom +581492,12/9/2019,23434,3 Raffia Ribbons 50'S Christmas,6.19,9,15492,United Kingdom +581492,12/9/2019,23435,3 Raffia Ribbons Vintage Christmas,6.04,3,15492,United Kingd +581492,12/9/2019,23439,Hand Warmer Red Love Heart,6.19,1,15492,United Kingdom +581492,12/9/2019,23451,Square Mini Portrait Frame,6.19,1,15492,United Kingdom +581492,12/9/2019,23467,Vintage Zinc Planter,6.19,1,15492,United Kingdom +581492,12/9/2019,23469,Card Holder Love Bird Small,6.19,3,15492,United Kingdom +581492,12/9/2019,23480,Mini Lights Woodland Mushrooms,6.19,2,15492,United Kingdom +581492,12/9/2019,23493,Vintage Doily Travel Sewing Kit,6.19,3,15492,United Kingdom +581492,12/9/2019,23494,Vintage Doily Deluxe Sewing Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,23495,Set Of 3 Pantry Wooden Spoons,6.19,1,15492,United Kingdom +581492,12/9/2019,23497,Classic Chrome Bicycle Bell,6.19,4,15492,United Kingdom +581492,12/9/2019,23498,Classic Bicycle Clips,6.19,2,15492,United Kingdom +581492,12/9/2019,23501,Key Ring Baseball Boot Union Jack,6.19,3,15492,United Kingdo +581492,12/9/2019,23506,Mini Playing Cards Spaceboy,6.04,1,15492,United Kingdom +581492,12/9/2019,23508,Mini Playing Cards Dolly Girl,6.04,2,15492,United Kingdom +581492,12/9/2019,23510,Mini Playing Cards Gymkhana,6.04,1,15492,United Kingdom +581492,12/9/2019,23521,Wall Art Cat And Bird,6.04,1,15492,United Kingdom +581492,12/9/2019,23526,Wall Art Dog Licence,6.04,4,15492,United Kingdom +581492,12/9/2019,23530,Wall Art Only One Person,6.04,2,15492,United Kingdom +581492,12/9/2019,23534,Wall Art Stop For Tea,6.19,6,15492,United Kingdom +581492,12/9/2019,23535,Wall Art Bicycle Safety,6.19,4,15492,United Kingdom +581492,12/9/2019,23536,Wall Art Village Show,6.19,1,15492,United Kingdom +581492,12/9/2019,23538,Wall Art Vintage Heart,6.19,1,15492,United Kingdom +581492,12/9/2019,23551,Pack Of 12 Paisley Park Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,23552,Bicycle Puncture Repair Kit,6.19,12,15492,United Kingdom +581492,12/9/2019,23555,Landmark Frame Notting Hill,6.19,1,15492,United Kingdom +581492,12/9/2019,23559,Woodland Bunnies Lolly Makers,6.19,5,15492,United Kingdom +581492,12/9/2019,23561,Set Of 6 Ribbons Party,6.19,1,15492,United Kingdom +581492,12/9/2019,23564,Egg Cup Milkmaid Ingrid,6.19,2,15492,United Kingdom +581492,12/9/2019,23565,Egg Cup Milkmaid Helga,6.19,2,15492,United Kingdom +581492,12/9/2019,23567,Egg Cup Henrietta Hen Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23569,Tradtional Alphabet Stamp Set,6.19,2,15492,United Kingdom +581492,12/9/2019,23570,Traditional Pick Up Sticks Game,6.19,7,15492,United Kingdom +581492,12/9/2019,23571,Traditional Naughts & Crosses,6.19,2,15492,United Kingdom +581492,12/9/2019,23575,Snack Tray Paisley Park,6.19,3,15492,United Kingdom +581492,12/9/2019,23579,Snack Tray I Love London,6.19,1,15492,United Kingdom +581492,12/9/2019,23611,Set 10 Cards Red Riding Hood 17214,6.19,3,15492,United Kingd +581492,12/9/2019,23616,Set 10 Cards Jingle Bells 17217,6.19,1,15492,United Kingdom +581492,12/9/2019,23621,Set 10 Cards David's Madonna 17074,6.19,3,15492,United Kingd +581492,12/9/2019,23635,Set 10 Cards Christmas Holly 17259,6.19,1,15492,United Kingd +581492,12/9/2019,23644,Set 10 Cards Christmas Tree 16955,6.19,1,15492,United Kingdo +581492,12/9/2019,23660,Henrietta Hen Mug,6.19,1,15492,United Kingdom +581492,12/9/2019,23661,Milk Maids Mug,6.19,1,15492,United Kingdom +581492,12/9/2019,35095A,Blue Victorian Fabric Oval Box,6.19,1,15492,United Kingdom +581492,12/9/2019,35095B,Red Victorian Fabric Oval Box,6.19,1,15492,United Kingdom +581492,12/9/2019,35646,Vintage Bead Pink Evening Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,35648,Vintage Bead Pink Purse,6.19,2,15492,United Kingdom +581492,12/9/2019,35953,Folkart Star Christmas Decorations,6.19,12,15492,United King +581492,12/9/2019,35964,Folkart Clip On Stars,6.19,4,15492,United Kingdom +581492,12/9/2019,35970,Zinc Folkart Sleigh Bells,6.04,1,15492,United Kingdom +581492,12/9/2019,46118,Funky Monkey Cushion Cover,6.19,1,15492,United Kingdom +581492,12/9/2019,46776A,Woven Bubble Gum Cushion Cover,7.24,2,15492,United Kingdom +581492,12/9/2019,46776B,Woven Berries Cushion Cover,7.24,3,15492,United Kingdom +581492,12/9/2019,46776C,Woven Frost Cushion Cover,7.24,1,15492,United Kingdom +581492,12/9/2019,46776D,Woven Sunset Cushion Cover,7.24,2,15492,United Kingdom +581492,12/9/2019,46776E,Woven Candy Cushion Cover,7.24,5,15492,United Kingdom +581492,12/9/2019,46776F,Woven Rose Garden Cushion Cover,7.24,6,15492,United Kingdom +581492,12/9/2019,47310M,Small Pop Box Funky Monkey,6.19,1,15492,United Kingdom +581492,12/9/2019,47480,Hanging Photo Clip Rope Ladder,6.19,3,15492,United Kingdom +581492,12/9/2019,47504H,English Rose Spirit Level,7.24,1,15492,United Kingdom +581492,12/9/2019,47559B,Tea Time Oven Glove,7.24,3,15492,United Kingdom +581492,12/9/2019,47563A,Retro Longboard Ironing Board Cover,7.24,2,15492,United Kin +581492,12/9/2019,47566,Party Bunting,7.24,2,15492,United Kingdom +581492,12/9/2019,47590B,Pink Happy Birthday Bunting,7.24,1,15492,United Kingdom +581492,12/9/2019,51014A,Feather Pen Hot Pink,7.24,2,15492,United Kingdom +581492,12/9/2019,51014L,Feather Pen Light Pink,7.24,1,15492,United Kingdom +581492,12/9/2019,71270,Photo Clip Line,7.24,1,15492,United Kingdom +581492,12/9/2019,72349B,Set/6 Purple Butterfly T-Lights,6.39,1,15492,United Kingdom +581492,12/9/2019,72741,Grand Chocolatecandle,7.24,3,15492,United Kingdom +581492,12/9/2019,72800E,4 Ivory Dinner Candles Silver Flock,7.24,3,15492,United Kin +581492,12/9/2019,79321,Chilli Lights,6.19,1,15492,United Kingdom +581492,12/9/2019,82484,Wood Black Board Ant White Finish,6.19,2,15492,United Kingdo +581492,12/9/2019,82567,Airline Lounge Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,82582,Area Patrolled Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,82600,N0 Singing Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,84031A,Charlie+Lola Red Hot Water Bottle,6.19,4,15492,United Kingd +581492,12/9/2019,84077,World War 2 Gliders Asstd Designs,6.19,1,15492,United Kingdo +581492,12/9/2019,84249A,Greeting Card Square Doughnuts,6.19,1,15492,United Kingdom +581492,12/9/2019,84347,Rotating Silver Angels T-Light Hldr,6.19,2,15492,United King +581492,12/9/2019,84375,Set Of 20 Kids Cookie Cutters,6.19,1,15492,United Kingdom +581492,12/9/2019,84378,Set Of 3 Heart Cookie Cutters,6.19,4,15492,United Kingdom +581492,12/9/2019,84519B,Carrot Charlie+Lola Coaster Set,6.19,1,15492,United Kingdom +581492,12/9/2019,84559A,3d Sheet Of Dog Stickers,6.19,2,15492,United Kingdom +581492,12/9/2019,84568,Girls Alphabet Iron On Patches,6.19,31,15492,United Kingdom +581492,12/9/2019,84569D,Pack 6 Heart/Ice-Cream Patches,6.19,1,15492,United Kingdom +581492,12/9/2019,84596B,Small Dolly Mix Design Orange Bowl,6.19,4,15492,United King +581492,12/9/2019,84596F,Small Marshmallows Pink Bowl,6.19,4,15492,United Kingdom +581492,12/9/2019,84598,Boys Alphabet Iron On Patches,6.19,5,15492,United Kingdom +581492,12/9/2019,84692,Box Of 24 Cocktail Parasols,6.19,1,15492,United Kingdom +581492,12/9/2019,84755,Colour Glass T-Light Holder Hanging,6.19,4,15492,United King +581492,12/9/2019,84828,Jungle Popsicles Ice Lolly Moulds,6.19,1,15492,United Kingdo +581492,12/9/2019,84879,Assorted Colour Bird Ornament,6.19,16,15492,United Kingdom +581492,12/9/2019,84923,Pink Butterfly Handbag W Bobbles,6.04,3,15492,United Kingdom +581492,12/9/2019,84946,Antique Silver T-Light Glass,6.04,18,15492,United Kingdom +581492,12/9/2019,84970S,Hanging Heart Zinc T-Light Holder,6.04,3,15492,United Kingd +581492,12/9/2019,84978,Hanging Heart Jar T-Light Holder,6.04,4,15492,United Kingdom +581492,12/9/2019,84991,60 Teatime Fairy Cake Cases,6.04,1,15492,United Kingdom +581492,12/9/2019,84997A,Childrens Cutlery Polkadot Green,6.04,1,15492,United Kingdo +581492,12/9/2019,84997B,Childrens Cutlery Retrospot Red,6.04,1,15492,United Kingdom +581492,12/9/2019,20733,Gold Mini Tape Measure,6.04,3,15492,United Kingdom +581492,12/9/2019,20777,Chrysanthemum Notebook,6.19,2,15492,United Kingdom +581492,12/9/2019,20782,Camouflage Ear Muff Headphones,6.19,1,15492,United Kingdom +581492,12/9/2019,20914,Set/5 Red Retrospot Lid Glass Bowls,6.19,2,15492,United King +581492,12/9/2019,20931,Blue Pot Plant Candle,6.19,1,15492,United Kingdom +581492,12/9/2019,20970,Pink Floral Feltcraft Shoulder Bag,6.19,1,15492,United Kingd +581492,12/9/2019,20971,Pink Blue Felt Craft Trinket Box,6.19,2,15492,United Kingdom +581492,12/9/2019,20972,Pink Cream Felt Craft Trinket Box,6.19,3,15492,United Kingdo +581492,12/9/2019,20973,12 Pencil Small Tube Woodland,6.19,4,15492,United Kingdom +581492,12/9/2019,20978,36 Pencils Tube Skulls,6.19,1,15492,United Kingdom +581492,12/9/2019,20979,36 Pencils Tube Red Retrospot,6.19,3,15492,United Kingdom +581492,12/9/2019,20982,12 Pencils Tall Tube Skulls,6.19,2,15492,United Kingdom +581492,12/9/2019,20983,12 Pencils Tall Tube Red Retrospot,6.19,4,15492,United Kingd +581492,12/9/2019,20985,Heart Calculator,6.19,1,15492,United Kingdom +581492,12/9/2019,20986,Blue Calculator Ruler,6.19,1,15492,United Kingdom +581492,12/9/2019,21000,Rose Du Sud Cosmetics Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,21012,Antique All Glass Candlestick,6.19,3,15492,United Kingdom +581492,12/9/2019,21015,Dark Bird House Tree Decoration,6.19,8,15492,United Kingdom +581492,12/9/2019,21026,Space Owl,6.19,1,15492,United Kingdom +581492,12/9/2019,21035,Set/2 Red Retrospot Tea Towels,6.19,1,15492,United Kingdom +581492,12/9/2019,21064,Boom Box Speaker Boys,6.19,4,15492,United Kingdom +581492,12/9/2019,21065,Boom Box Speaker Girls,6.19,2,15492,United Kingdom +581492,12/9/2019,21098,Christmas Toilet Roll,6.19,1,15492,United Kingdom +581492,12/9/2019,21123,Set/10 Ivory Polkadot Party Candles,6.19,1,15492,United King +581492,12/9/2019,21125,Set 6 Football Celebration Candles,6.19,1,15492,United Kingd +581492,12/9/2019,21126,Set Of 6 Girls Celebration Candles,6.19,1,15492,United Kingd +581492,12/9/2019,21137,Black Record Cover Frame,6.19,17,15492,United Kingdom +581492,12/9/2019,21154,Red Retrospot Oven Glove,6.19,2,15492,United Kingdom +581492,12/9/2019,21158,Moody Girl Door Hanger,6.19,1,15492,United Kingdom +581492,12/9/2019,21162,Toxic Area Door Hanger,6.19,1,15492,United Kingdom +581492,12/9/2019,21166,Cook With Wine Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,21172,Party Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,21174,Pottering In The Shed Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,21175,Gin And Tonic Diet Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,21200,Multicolour Honeycomb Paper Garland,6.04,6,15492,United King +581492,12/9/2019,21201,Tropical Honeycomb Paper Garland,6.04,4,15492,United Kingdom +581492,12/9/2019,21212,Pack Of 72 Retrospot Cake Cases,6.04,2,15492,United Kingdom +581492,12/9/2019,21213,Pack Of 72 Skull Cake Cases,6.04,2,15492,United Kingdom +581492,12/9/2019,21220,Set/4 Badges Dogs,6.19,2,15492,United Kingdom +581492,12/9/2019,21224,Set/4 Skull Badges,6.19,1,15492,United Kingdom +581492,12/9/2019,21232,Strawberry Ceramic Trinket Pot,6.19,1,15492,United Kingdom +581492,12/9/2019,21257,Victorian Sewing Box Medium,6.19,2,15492,United Kingdom +581492,12/9/2019,21258,Victorian Sewing Box Large,6.19,1,15492,United Kingdom +581492,12/9/2019,21259,Victorian Sewing Box Small,6.19,1,15492,United Kingdom +581492,12/9/2019,21313,Glass Heart T-Light Holder,6.19,1,15492,United Kingdom +581492,12/9/2019,21314,Small Glass Heart Trinket Pot,6.19,2,15492,United Kingdom +581492,12/9/2019,21328,Balloons Writing Set,6.19,2,15492,United Kingdom +581492,12/9/2019,21329,Dinosaurs Writing Set,6.19,2,15492,United Kingdom +581492,12/9/2019,21356,Toast Its - Fairy Flower,6.19,16,15492,United Kingdom +581492,12/9/2019,21378,Small Tall Camphor Wood Toadstool,6.19,2,15492,United Kingdo +581492,12/9/2019,21379,Camphor Wood Portobello Mushroom,6.19,1,15492,United Kingdom +581492,12/9/2019,21383,Pack Of 12 Sticky Bunnies,6.19,1,15492,United Kingdom +581492,12/9/2019,21402,Red Egg Spoon,6.19,1,15492,United Kingdom +581492,12/9/2019,21408,Spotty Pink Duck Doorstop,6.19,2,15492,United Kingdom +581492,12/9/2019,21411,Gingham Heart Doorstop Red,6.19,1,15492,United Kingdom +581492,12/9/2019,21467,Cherry Crochet Food Cover,6.19,1,15492,United Kingdom +581492,12/9/2019,21471,Strawberry Raffia Food Cover,6.19,2,15492,United Kingdom +581492,12/9/2019,21479,White Skull Hot Water Bottle,6.19,2,15492,United Kingdom +581492,12/9/2019,21481,Fawn Blue Hot Water Bottle,6.19,3,15492,United Kingdom +581492,12/9/2019,21484,Chick Grey Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,21485,Retrospot Heart Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,21494,Rotating Leaves T-Light Holder,6.19,11,15492,United Kingdom +581492,12/9/2019,21506,Fancy Font Birthday Card,6.19,3,15492,United Kingdom +581492,12/9/2019,21507,Elephant Birthday Card,7.24,1,15492,United Kingdom +581492,12/9/2019,21508,Vintage Kid Dolly Card,7.24,1,15492,United Kingdom +581492,12/9/2019,21509,Cowboys And Indians Birthday Card,7.24,2,15492,United Kingdo +581492,12/9/2019,21528,Dairy Maid Traditional Teapot,7.24,1,15492,United Kingdom +581492,12/9/2019,21544,Skulls Water Transfer Tattoos,7.24,2,15492,United Kingdom +581492,12/9/2019,21558,Skull Lunch Box With Cutlery,6.39,1,15492,United Kingdom +581492,12/9/2019,21559,Strawberry Lunch Box With Cutlery,7.24,1,15492,United Kingdo +581492,12/9/2019,21615,4 Lavender Botanical Dinner Candles,7.24,2,15492,United King +581492,12/9/2019,21616,4 Pear Botanical Dinner Candles,7.24,6,15492,United Kingdom +581492,12/9/2019,21620,Set Of 4 Rose Botanical Candles,7.24,5,15492,United Kingdom +581492,12/9/2019,21642,Assorted Tutti Frutti Pen,7.24,4,15492,United Kingdom +581492,12/9/2019,21648,Assorted Tutti Frutti Small Purse,7.24,2,15492,United Kingdo +581492,12/9/2019,21650,Assorted Tutti Frutti Bracelet,7.24,14,15492,United Kingdom +581492,12/9/2019,21675,Butterflies Stickers,6.19,2,15492,United Kingdom +581492,12/9/2019,21677,Hearts Stickers,6.19,1,15492,United Kingdom +581492,12/9/2019,21678,Paisley Pattern Stickers,6.19,2,15492,United Kingdom +581492,12/9/2019,21680,Woodland Stickers,6.19,1,15492,United Kingdom +581492,12/9/2019,21703,Bag 125g Swirly Marbles,6.19,1,15492,United Kingdom +581492,12/9/2019,21704,Bag 250g Swirly Marbles,6.19,1,15492,United Kingdom +581492,12/9/2019,21716,Boys Vintage Tin Seaside Bucket,6.19,1,15492,United Kingdom +581492,12/9/2019,21718,Red Metal Beach Spade,6.19,1,15492,United Kingdom +581492,12/9/2019,21724,Panda And Bunnies Sticker Sheet,6.19,1,15492,United Kingdom +581492,12/9/2019,21731,Red Toadstool Led Night Light,6.19,6,15492,United Kingdom +581492,12/9/2019,21739,Cosy Slipper Shoes Small Green,6.19,2,15492,United Kingdom +581492,12/9/2019,21774,Decorative Cats Bathroom Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,21786,Polkadot Rain Hat,6.19,3,15492,United Kingdom +581492,12/9/2019,21787,Rain Poncho Retrospot,6.19,2,15492,United Kingdom +581492,12/9/2019,21790,Vintage Snap Cards,6.19,5,15492,United Kingdom +581492,12/9/2019,21791,Vintage Heads And Tails Card Game,6.19,1,15492,United Kingdo +581492,12/9/2019,21808,Christmas Garland Stars Trees,6.19,3,15492,United Kingdom +581492,12/9/2019,21810,Christmas Hanging Star With Bell,6.19,25,15492,United Kingdo +581492,12/9/2019,21812,Garland With Hearts And Bells,6.19,7,15492,United Kingdom +581492,12/9/2019,21813,Garland With Stars And Bells,6.19,3,15492,United Kingdom +581492,12/9/2019,21822,Glitter Christmas Tree With Bells,6.19,12,15492,United Kingd +581492,12/9/2019,21828,Eight Piece Snake Set,6.19,1,15492,United Kingdom +581492,12/9/2019,21832,Chocolate Calculator,6.19,1,15492,United Kingdom +581492,12/9/2019,21833,Camouflage Led Torch,6.04,2,15492,United Kingdom +581492,12/9/2019,21871,Save The Planet Mug,6.19,1,15492,United Kingdom +581492,12/9/2019,21874,Gin And Tonic Mug,6.19,2,15492,United Kingdom +581492,12/9/2019,21876,Pottering Mug,6.19,4,15492,United Kingdom +581492,12/9/2019,21879,Hearts Gift Tape,6.19,1,15492,United Kingdom +581492,12/9/2019,21889,Wooden Box Of Dominoes,6.19,3,15492,United Kingdom +581492,12/9/2019,21890,S/6 Wooden Skittles In Cotton Bag,6.19,1,15492,United Kingdo +581492,12/9/2019,21892,Traditional Wooden Catch Cup Game,6.19,1,15492,United Kingdo +581492,12/9/2019,21900,Key Fob Shed,6.19,4,15492,United Kingdom +581492,12/9/2019,21901,Key Fob Back Door,6.19,2,15492,United Kingdom +581492,12/9/2019,21902,Key Fob Front Door,6.19,1,15492,United Kingdom +581492,12/9/2019,21905,More Butter Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,21906,Pharmacie First Aid Tin,6.19,1,15492,United Kingdom +581492,12/9/2019,21914,Blue Harmonica In Box,6.19,2,15492,United Kingdom +581492,12/9/2019,21916,Set 12 Retro White Chalk Sticks,6.19,2,15492,United Kingdom +581492,12/9/2019,21930,Jumbo Storage Bag Skulls,6.19,1,15492,United Kingdom +581492,12/9/2019,21931,Jumbo Storage Bag Suki,6.19,1,15492,United Kingdom +581492,12/9/2019,21932,Scandinavian Paisley Picnic Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,21934,Skull Shoulder Bag,6.19,3,15492,United Kingdom +581492,12/9/2019,21935,Suki Shoulder Bag,6.19,9,15492,United Kingdom +581492,12/9/2019,21936,Red Retrospot Picnic Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,21942,Skulls Design Flannel,6.19,2,15492,United Kingdom +581492,12/9/2019,21945,Strawberries Design Flannel,6.19,2,15492,United Kingdom +581492,12/9/2019,21947,Set Of 6 Heart Chopsticks,6.19,1,15492,United Kingdom +581492,12/9/2019,21949,Set Of 6 Strawberry Chopsticks,6.19,2,15492,United Kingdom +581492,12/9/2019,21967,Pack Of 12 Skull Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,21976,Pack Of 60 Mushroom Cake Cases,6.19,3,15492,United Kingdom +581492,12/9/2019,21977,Pack Of 60 Pink Paisley Cake Cases,6.19,2,15492,United Kingd +581492,12/9/2019,21980,Pack Of 12 Red Retrospot Tissues,6.19,2,15492,United Kingdom +581492,12/9/2019,21981,Pack Of 12 Woodland Tissues,6.19,4,15492,United Kingdom +581492,12/9/2019,21982,Pack Of 12 Suki Tissues,6.19,2,15492,United Kingdom +581492,12/9/2019,21983,Pack Of 12 Blue Paisley Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,21984,Pack Of 12 Pink Paisley Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,21986,Pack Of 12 Pink Polkadot Tissues,6.19,3,15492,United Kingdom +581492,12/9/2019,21989,Pack Of 20 Skull Paper Napkins,6.19,2,15492,United Kingdom +581492,12/9/2019,21990,Modern Floral Stationery Set,6.19,8,15492,United Kingdom +581492,12/9/2019,21993,Floral Folk Stationery Set,6.19,8,15492,United Kingdom +581492,12/9/2019,22024,Rainy Ladies Birthday Card,6.19,3,15492,United Kingdom +581492,12/9/2019,22025,Ring Of Roses Birthday Card,6.19,3,15492,United Kingdom +581492,12/9/2019,22026,Banquet Birthday Card,6.19,4,15492,United Kingdom +581492,12/9/2019,22027,Tea Party Birthday Card,6.19,2,15492,United Kingdom +581492,12/9/2019,22028,Penny Farthing Birthday Card,6.19,1,15492,United Kingdom +581492,12/9/2019,22029,Spaceboy Birthday Card,6.19,4,15492,United Kingdom +581492,12/9/2019,22031,Botanical Lavender Birthday Card,6.19,1,15492,United Kingdom +581492,12/9/2019,22037,Robot Birthday Card,6.19,4,15492,United Kingdom +581492,12/9/2019,22064,Pink Doughnut Trinket Pot,6.19,1,15492,United Kingdom +581492,12/9/2019,22065,Christmas Pudding Trinket Pot,6.19,2,15492,United Kingdom +581492,12/9/2019,22068,Black Pirate Treasure Chest,6.19,1,15492,United Kingdom +581492,12/9/2019,22070,Small Red Retrospot Mug In Box,6.19,3,15492,United Kingdom +581492,12/9/2019,22071,Small White Retrospot Mug In Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22074,6 Ribbons Shimmering Pinks,6.19,1,15492,United Kingdom +581492,12/9/2019,22075,6 Ribbons Elegant Christmas,6.19,8,15492,United Kingdom +581492,12/9/2019,22076,6 Ribbons Empire,6.19,4,15492,United Kingdom +581492,12/9/2019,22083,Paper Chain Kit Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22085,Paper Chain Kit Skulls,6.19,1,15492,United Kingdom +581492,12/9/2019,22086,Paper Chain Kit 50'S Christmas,6.19,38,15492,United Kingdom +581492,12/9/2019,22091,Empire Tissue Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22099,Caravan Square Tissue Box,6.19,3,15492,United Kingdom +581492,12/9/2019,22104,Mirror Mosaic Candle Plate,6.19,4,15492,United Kingdom +581492,12/9/2019,22106,Mirror Mosaic Hurricane Lamp,6.19,1,15492,United Kingdom +581492,12/9/2019,22107,Pizza Plate In Box,6.19,10,15492,United Kingdom +581492,12/9/2019,22108,Ping! Microwave Plate,6.04,2,15492,United Kingdom +581492,12/9/2019,22109,Full English Breakfast Plate,6.04,2,15492,United Kingdom +581492,12/9/2019,22110,Bird House Hot Water Bottle,6.04,3,15492,United Kingdom +581492,12/9/2019,22111,Scottie Dog Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,22112,Chocolate Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,6.19,4,15492,United Kingdo +581492,12/9/2019,22116,Metal Sign His Dinner Is Served,6.19,2,15492,United Kingdom +581492,12/9/2019,22121,Noel Wooden Block Letters,6.19,1,15492,United Kingdom +581492,12/9/2019,22123,Ping Microwave Apron,6.19,1,15492,United Kingdom +581492,12/9/2019,22124,Set Of 2 Tea Towels Ping Microwave,6.19,1,15492,United Kingd +581492,12/9/2019,22129,Party Cones Candy Decoration,6.19,3,15492,United Kingdom +581492,12/9/2019,22134,Mini Ladle Love Heart Red,6.19,1,15492,United Kingdom +581492,12/9/2019,15036,Assorted Colours Silk Fan,6.19,1,15492,United Kingdom +581492,12/9/2019,15039,Sandalwood Fan,6.19,1,15492,United Kingdom +581492,12/9/2019,15058C,Ice Cream Design Garden Parasol,6.19,1,15492,United Kingdom +581492,12/9/2019,16218,Cartoon Pencil Sharpeners,6.19,5,15492,United Kingdom +581492,12/9/2019,16225,Rattle Snake Eggs,6.19,1,15492,United Kingdom +581492,12/9/2019,16235,Recycled Pencil With Rabbit Eraser,6.19,3,15492,United Kingd +581492,12/9/2019,16237,Sleeping Cat Erasers,6.19,1,15492,United Kingdom +581492,12/9/2019,17038,Porcelain Budah Incense Holder,6.19,1,15492,United Kingdom +581492,12/9/2019,17084N,Fairy Dreams Incense,6.19,1,15492,United Kingdom +581492,12/9/2019,20659,Economy Luggage Tag,6.19,7,15492,United Kingdom +581492,12/9/2019,20665,Red Retrospot Purse,6.19,2,15492,United Kingdom +581492,12/9/2019,20668,Disco Ball Christmas Decoration,6.19,1,15492,United Kingdom +581492,12/9/2019,20718,Red Retrospot Shopper Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,20719,Woodland Charlotte Bag,6.19,7,15492,United Kingdom +581492,12/9/2019,20723,Strawberry Charlotte Bag,6.19,2,15492,United Kingdom +581492,12/9/2019,20724,Red Retrospot Charlotte Bag,6.19,4,15492,United Kingdom +581492,12/9/2019,22135,Mini Ladle Love Heart Pink,6.19,6,15492,United Kingdom +581492,12/9/2019,22138,Baking Set 9 Piece Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22141,Christmas Craft Tree Top Angel,6.19,2,15492,United Kingdom +581492,12/9/2019,22150,3 Stripey Mice Feltcraft,7.24,2,15492,United Kingdom +581492,12/9/2019,22153,Angel Decoration Stars On Dress,7.24,2,15492,United Kingdom +581492,12/9/2019,22154,Angel Decoration 3 Buttons,7.24,3,15492,United Kingdom +581492,12/9/2019,22155,Star Decoration Rustic,7.24,7,15492,United Kingdom +581492,12/9/2019,22156,Heart Decoration With Pearls,7.24,4,15492,United Kingdom +581492,12/9/2019,22163,Heart String Memo Holder Hanging,7.24,9,15492,United Kingdom +581492,12/9/2019,22165,Diamante Heart Shaped Wall Mirror,7.24,1,15492,United Kingdo +581492,12/9/2019,22167,Oval Wall Mirror Diamante,7.24,1,15492,United Kingdom +581492,12/9/2019,22170,Picture Frame Wood Triple Portrait,7.24,1,15492,United Kingd +581492,12/9/2019,22173,Metal 4 Hook Hanger French Chateau,7.24,2,15492,United Kingd +581492,12/9/2019,22174,Photo Cube,6.19,7,15492,United Kingdom +581492,12/9/2019,22178,Victorian Glass Hanging T-Light,6.19,5,15492,United Kingdom +581492,12/9/2019,22186,Red Star Card Holder,7.24,2,15492,United Kingdom +581492,12/9/2019,22190,Local Cafe Mug,7.24,3,15492,United Kingdom +581492,12/9/2019,22193,Red Diner Wall Clock,7.24,1,15492,United Kingdom +581492,12/9/2019,22195,Large Heart Measuring Spoons,7.24,1,15492,United Kingdom +581492,12/9/2019,22196,Small Heart Measuring Spoons,7.24,11,15492,United Kingdom +581492,12/9/2019,22197,Popcorn Holder,7.24,34,15492,United Kingdom +581492,12/9/2019,22199,Frying Pan Red Retrospot,7.24,1,15492,United Kingdom +581492,12/9/2019,22209,Wood Stamp Set Happy Birthday,7.24,1,15492,United Kingdom +581492,12/9/2019,22212,Four Hook White Lovebirds,7.24,1,15492,United Kingdom +581492,12/9/2019,22219,Lovebird Hanging Decoration White,7.24,4,15492,United Kingdo +581492,12/9/2019,22224,White Lovebird Lantern,7.24,4,15492,United Kingdom +581492,12/9/2019,22227,Hanging Heart Mirror Decoration,7.24,1,15492,United Kingdom +581492,12/9/2019,22260,Felt Egg Cosy Blue Rabbit,6.39,2,15492,United Kingdom +581492,12/9/2019,22261,Felt Egg Cosy White Rabbit,7.24,1,15492,United Kingdom +581492,12/9/2019,22264,Felt Farm Animal White Bunny,7.24,1,15492,United Kingdom +581492,12/9/2019,22273,Feltcraft Doll Molly,7.24,2,15492,United Kingdom +581492,12/9/2019,22277,Cosmetic Bag Vintage Rose Paisley,7.24,1,15492,United Kingdo +581492,12/9/2019,22279,Pocket Bag Blue Paisley Red Spot,7.24,5,15492,United Kingdom +581492,12/9/2019,22280,Pocket Bag Pink Paisely Brown Spot,7.24,4,15492,United Kingd +581492,12/9/2019,22300,Coffee Mug Dog + Ball Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22301,Coffee Mug Cat + Bird Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22307,Gold Mug Bone China Tree Of Life,7.24,1,15492,United Kingdom +581492,12/9/2019,22308,Tea Cosy Blue Stripe,7.24,2,15492,United Kingdom +581492,12/9/2019,22309,Tea Cosy Red Stripe,7.24,2,15492,United Kingdom +581492,12/9/2019,22324,Blue Polkadot Kids Bag,7.24,2,15492,United Kingdom +581492,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,7.24,3,15492,United Kingd +581492,12/9/2019,22327,Round Snack Boxes Set Of 4 Skulls,7.24,2,15492,United Kingdo +581492,12/9/2019,22329,Round Container Set Of 5 Retrospot,6.19,2,15492,United Kingd +581492,12/9/2019,22338,Star Decoration Painted Zinc,6.19,4,15492,United Kingdom +581492,12/9/2019,22340,Noel Garland Painted Zinc,6.19,17,15492,United Kingdom +581492,12/9/2019,22342,Home Garland Painted Zinc,6.19,7,15492,United Kingdom +581492,12/9/2019,22348,Tea Bag Plate Red Retrospot,6.19,3,15492,United Kingdom +581492,12/9/2019,22355,Charlotte Bag Suki Design,6.19,3,15492,United Kingdom +581492,12/9/2019,22356,Charlotte Bag Pink Polkadot,6.19,1,15492,United Kingdom +581492,12/9/2019,22357,Kings Choice Biscuit Tin,6.19,2,15492,United Kingdom +581492,12/9/2019,22358,Kings Choice Tea Caddy,6.19,1,15492,United Kingdom +581492,12/9/2019,22361,Glass Jar Daisy Fresh Cotton Wool,6.19,2,15492,United Kingdo +581492,12/9/2019,22362,Glass Jar Peacock Bath Salts,6.19,2,15492,United Kingdom +581492,12/9/2019,22367,Childrens Apron Spaceboy Design,6.19,1,15492,United Kingdom +581492,12/9/2019,22372,Airline Bag Vintage World Champion,6.19,1,15492,United Kingd +581492,12/9/2019,22374,Airline Bag Vintage Jet Set Red,6.19,1,15492,United Kingdom +581492,12/9/2019,85014B,Red Retrospot Umbrella,6.19,1,15492,United Kingdom +581492,12/9/2019,85032C,Curious Images Gift Wrap Set,6.19,1,15492,United Kingdom +581492,12/9/2019,85038,6 Chocolate Love Heart T-Lights,6.19,1,15492,United Kingdom +581492,12/9/2019,85039A,Set/4 Red Mini Rose Candle In Bowl,6.19,5,15492,United King +581492,12/9/2019,85039B,S/4 Ivory Mini Rose Candle In Bowl,6.19,4,15492,United King +581492,12/9/2019,85040A,S/4 Pink Flower Candles In Bowl,6.19,1,15492,United Kingdom +581492,12/9/2019,85048,15cm Christmas Glass Ball 20 Lights,6.19,2,15492,United King +581492,12/9/2019,85049C,Romantic Pinks Ribbons,6.19,1,15492,United Kingdom +581492,12/9/2019,85049G,Chocolate Box Ribbons,6.19,1,15492,United Kingdom +581492,12/9/2019,85049H,Urban Black Ribbons,6.19,2,15492,United Kingdom +581492,12/9/2019,85053,French Enamel Candleholder,6.19,1,15492,United Kingdom +581492,12/9/2019,85059,French Enamel Water Basin,6.19,1,15492,United Kingdom +581492,12/9/2019,85066,Cream Sweetheart Mini Chest,6.19,1,15492,United Kingdom +581492,12/9/2019,85094,Candy Spot Egg Warmer Rabbit,6.19,2,15492,United Kingdom +581492,12/9/2019,85114C,Red Enchanted Forest Placemat,6.19,1,15492,United Kingdom +581492,12/9/2019,85123A,Cream Hanging Heart T-Light Holder,6.19,3,15492,United King +581492,12/9/2019,85131B,Beaded Crystal Heart Green On Stick,6.19,1,15492,United Kin +581492,12/9/2019,85131D,Beaded Crystal Heart Pink On Stick,6.19,2,15492,United King +581492,12/9/2019,85152,Hand Over The Chocolate Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,85168B,Black Baroque Carriage Clock,6.19,3,15492,United Kingdom +581492,12/9/2019,85169C,Eau De Nil Love Bird Candle,6.19,1,15492,United Kingdom +581492,12/9/2019,85170C,Set/6 Eau De Nil Bird T-Lights,6.19,1,15492,United Kingdom +581492,12/9/2019,85173,Set/6 Frog Prince T-Light Candles,6.19,1,15492,United Kingdo +581492,12/9/2019,85177,Basket Of Flowers Sewing Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,85178,Victorian Sewing Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,85179A,Green Bitty Light Chain,6.19,4,15492,United Kingdom +581492,12/9/2019,85199S,Small Hanging Ivory/Red Wood Bird,6.19,9,15492,United Kingd +581492,12/9/2019,85227,Set Of 6 3d Kit Cards For Kids,6.19,3,15492,United Kingdom +581492,12/9/2019,90003C,Midnight Blue Pair Heart Hair Slide,6.19,1,15492,United Kin +581492,12/9/2019,90003E,Green Pair Heart Hair Slides,6.19,2,15492,United Kingdom +581492,12/9/2019,90010A,Midnight Blue Glass/Silver Bracelet,6.04,1,15492,United Kin +581492,12/9/2019,90013A,Midnight Blue Vintage Earrings,6.19,3,15492,United Kingdom +581492,12/9/2019,90013C,Green Vintage Earrings,6.19,2,15492,United Kingdom +581492,12/9/2019,90014A,Silver Mop Orbit Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90016B,Gold/Mop Pendant Orbit Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90018B,Gold Mop Orbit Drop Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90019B,Gold Mop Orbit Bracelet,6.19,1,15492,United Kingdom +581492,12/9/2019,90027D,Glass Bead Hoop Earrings Amethyst,6.19,1,15492,United Kingd +581492,12/9/2019,90030B,Red Kukui Coconut Seed Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90055,Cracked Glaze Earrings Brown,6.19,1,15492,United Kingdom +581492,12/9/2019,90059A,Diamante Hair Grip Pack/2 Crystal,6.19,1,15492,United Kingd +581492,12/9/2019,90059D,Diamante Hair Grip Pack/2 Peridot,6.19,2,15492,United Kingd +581492,12/9/2019,90059E,Diamante Hair Grip Pack/2 Ruby,6.04,1,15492,United Kingdom +581492,12/9/2019,90059F,Diamante Hair Grip Pack/2 Lt Rose,6.19,1,15492,United Kingd +581492,12/9/2019,90072,Ruby Drop Chandelier Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90086,Crystal Frog Phone Charm,6.19,1,15492,United Kingdom +581492,12/9/2019,90120B,Blue Murano Twist Bracelet,6.19,2,15492,United Kingdom +581492,12/9/2019,90120C,Green Murano Twist Bracelet,6.19,1,15492,United Kingdom +581492,12/9/2019,90130B,Turq Stone/Crystal Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90130C,Green Stone/Crystal Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90134,Old Rose Combo Bead Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90138,White/Pink Mini Crystals Necklace,6.19,1,15492,United Kingdo +581492,12/9/2019,90141B,Ivory Pendant Triple Shell Necklace,6.19,1,15492,United Kin +581492,12/9/2019,90145,Silver Hoop Earrings With Flower,6.19,1,15492,United Kingdom +581492,12/9/2019,90155,Resin Necklace W Pastel Beads,6.19,1,15492,United Kingdom +581492,12/9/2019,90161D,Ant Copper Pink Boudicca Bracelet,6.19,1,15492,United Kingd +581492,12/9/2019,90163A,Pink Rosebud & Pearl Necklace,6.19,2,15492,United Kingdom +581492,12/9/2019,90165B,White Rosebud Pearl Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90168,2 Daisies Hair Comb,6.19,1,15492,United Kingdom +581492,12/9/2019,90169,Daisy Hair Comb,6.19,1,15492,United Kingdom +581492,12/9/2019,90170,Daisy Hair Band,6.19,1,15492,United Kingdom +581492,12/9/2019,90174,Butterfly Hair Band,6.19,2,15492,United Kingdom +581492,12/9/2019,90175A,White Glass Chunky Charm Bracelet,6.19,2,15492,United Kingd +581492,12/9/2019,90175D,Tigris Eye Chunky Charm Bracelet,6.19,1,15492,United Kingdo +581492,12/9/2019,90177A,Classic Diamante Earrings Jet,6.19,1,15492,United Kingdom +581492,12/9/2019,90177C,Drop Diamante Earrings Crystal,6.19,1,15492,United Kingdom +581492,12/9/2019,90181B,Amethyst Glass/Shell/Pearl Necklace,6.04,1,15492,United Kin +581492,12/9/2019,90182C,Black 3 Bead Drop Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90183A,Amber Drop Earrings W Long Beads,6.19,2,15492,United Kingdo +581492,12/9/2019,90183C,Black Drop Earrings W Long Beads,6.19,1,15492,United Kingdo +581492,12/9/2019,90184C,Black Chunky Bead Bracelet W Strap,6.19,1,15492,United King +581492,12/9/2019,90185B,Amethyst Diamante Expandable Ring,6.19,1,15492,United Kingd +581492,12/9/2019,90188,Drop Earrings W Flower & Leaf,6.19,2,15492,United Kingdom +581492,12/9/2019,90191,Silver Lariat 40cm,6.19,2,15492,United Kingdom +581492,12/9/2019,90192,Jade Drop Earrings W Filigree,6.19,1,15492,United Kingdom +581492,12/9/2019,90198A,Vintage Rose Bead Bracelet Raspberr,6.19,2,15492,United Kin +581492,12/9/2019,90200A,Purple Sweetheart Bracelet,6.19,1,15492,United Kingdom +581492,12/9/2019,90201A,Purple Enamel Flower Ring,6.19,2,15492,United Kingdom +581492,12/9/2019,90201B,Black Enamel Flower Ring,6.19,1,15492,United Kingdom +581492,12/9/2019,90201C,Red Enamel Flower Ring,6.19,1,15492,United Kingdom +581492,12/9/2019,90201D,Green Enamel Flower Ring,6.19,1,15492,United Kingdom +581492,12/9/2019,90202C,Green Enamel Flower Hair Tie,6.19,1,15492,United Kingdom +581492,12/9/2019,90202D,Pink Enamel Flower Hair Tie,6.19,1,15492,United Kingdom +581492,12/9/2019,90206C,Crystal Diamante Star Brooch,6.19,1,15492,United Kingdom +581492,12/9/2019,90208,Pair Of Pink Flower Cluster Slide,6.19,1,15492,United Kingdo +581492,12/9/2019,90210A,Grey Acrylic Faceted Bangle,6.19,1,15492,United Kingdom +581492,12/9/2019,22378,Wall Tidy Retrospot,6.19,2,15492,United Kingdom +581492,12/9/2019,22379,Recycling Bag Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22380,Toy Tidy Spaceboy,6.19,2,15492,United Kingdom +581492,12/9/2019,22396,Magnets Pack Of 4 Retro Photo,6.19,1,15492,United Kingdom +581492,12/9/2019,22411,Jumbo Shopper Vintage Red Paisley,6.19,1,15492,United Kingdo +581492,12/9/2019,22418,10 Colour Spaceboy Pen,6.19,3,15492,United Kingdom +581492,12/9/2019,22419,Lipstick Pen Red,6.19,3,15492,United Kingdom +581492,12/9/2019,22420,Lipstick Pen Baby Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,22421,Lipstick Pen Fuschia,6.19,2,15492,United Kingdom +581492,12/9/2019,22422,Toothpaste Tube Pen,6.19,4,15492,United Kingdom +581492,12/9/2019,22437,Set Of 9 Black Skull Balloons,7.24,1,15492,United Kingdom +581492,12/9/2019,22441,Grow Your Own Basil In Enamel Mug,7.24,1,15492,United Kingdo +581492,12/9/2019,22446,Pin Cushion Babushka Pink,7.24,7,15492,United Kingdom +581492,12/9/2019,22457,Natural Slate Heart Chalkboard,7.24,1,15492,United Kingdom +581492,12/9/2019,22464,Hanging Metal Heart Lantern,7.24,1,15492,United Kingdom +581492,12/9/2019,22466,Fairy Tale Cottage Night Light,7.24,1,15492,United Kingdom +581492,12/9/2019,22467,Gumball Coat Rack,7.24,1,15492,United Kingdom +581492,12/9/2019,22478,Birdhouse Garden Marker,7.24,1,15492,United Kingdom +581492,12/9/2019,22486,Plasmatronic Lamp,7.24,1,15492,United Kingdom +581492,12/9/2019,22488,Natural Slate Rectangle Chalkboard,7.24,1,15492,United Kingd +581492,12/9/2019,22489,Pack Of 12 Traditional Crayons,7.24,5,15492,United Kingdom +581492,12/9/2019,22495,Set Of 2 Round Tins Camembert,7.24,1,15492,United Kingdom +581492,12/9/2019,22540,Mini Jigsaw Circus Parade,7.24,1,15492,United Kingdom +581492,12/9/2019,22544,Mini Jigsaw Spaceboy,7.24,1,15492,United Kingdom +581492,12/9/2019,22549,Picture Dominoes,7.24,3,15492,United Kingdom +581492,12/9/2019,22551,Plasters In Tin Spaceboy,7.24,2,15492,United Kingdom +581492,12/9/2019,22553,Plasters In Tin Skulls,7.24,2,15492,United Kingdom +581492,12/9/2019,22554,Plasters In Tin Woodland Animals,7.24,3,15492,United Kingdom +581492,12/9/2019,22555,Plasters In Tin Strongman,7.24,3,15492,United Kingdom +581492,12/9/2019,22557,Plasters In Tin Vintage Paisley,7.24,2,15492,United Kingdom +581492,12/9/2019,22558,Clothes Pegs Retrospot Pack 24,7.24,3,15492,United Kingdom +581492,12/9/2019,22560,Traditional Modelling Clay,7.24,5,15492,United Kingdom +581492,12/9/2019,22565,Feltcraft Hairbands Pink And White,7.24,4,15492,United Kingd +581492,12/9/2019,22566,Feltcraft Hairband Pink And Purple,7.24,3,15492,United Kingd +581492,12/9/2019,22571,Rocking Horse Red Christmas,7.24,4,15492,United Kingdom +581492,12/9/2019,22573,Star Wooden Christmas Decoration,6.39,2,15492,United Kingdom +581492,12/9/2019,22574,Heart Wooden Christmas Decoration,7.24,5,15492,United Kingdo +581492,12/9/2019,22576,Swallow Wooden Christmas Decoration,7.24,3,15492,United King +581492,12/9/2019,22577,Wooden Heart Christmas Scandinavian,7.24,4,15492,United King +581492,12/9/2019,22578,Wooden Star Christmas Scandinavian,7.24,19,15492,United King +581492,12/9/2019,22580,Advent Calendar Gingham Sack,7.24,9,15492,United Kingdom +581492,12/9/2019,22581,Wood Stocking Christmas Scandispot,7.24,11,15492,United King +581492,12/9/2019,22585,Pack Of 6 Birdy Gift Tags,7.24,2,15492,United Kingdom +581492,12/9/2019,22586,Feltcraft Hairband Pink And Blue,7.24,2,15492,United Kingdom +581492,12/9/2019,22589,Cardholder Gingham Star,7.24,6,15492,United Kingdom +581492,12/9/2019,22591,Cardholder Gingham Christmas Tree,7.24,1,15492,United Kingdo +581492,12/9/2019,22592,Cardholder Holly Wreath Metal,7.24,3,15492,United Kingdom +581492,12/9/2019,22593,Christmas Gingham Star,6.39,2,15492,United Kingdom +581492,12/9/2019,22594,Christmas Gingham Tree,7.24,3,15492,United Kingdom +581492,12/9/2019,22597,Musical Zinc Heart Decoration,7.24,9,15492,United Kingdom +581492,12/9/2019,22598,Christmas Musical Zinc Tree,7.24,4,15492,United Kingdom +581492,12/9/2019,22599,Christmas Musical Zinc Star,6.19,5,15492,United Kingdom +581492,12/9/2019,22600,Christmas Retrospot Star Wood,6.19,2,15492,United Kingdom +581492,12/9/2019,22601,Christmas Retrospot Angel Wood,6.19,3,15492,United Kingdom +581492,12/9/2019,22602,Retrospot Wooden Heart Decoration,6.19,3,15492,United Kingdo +581492,12/9/2019,22608,Pens Assorted Funky Jeweled,6.19,3,15492,United Kingdom +581492,12/9/2019,22614,Pack Of 12 Spaceboy Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,22615,Pack Of 12 Circus Parade Tissues,6.19,4,15492,United Kingdom +581492,12/9/2019,22616,Pack Of 12 London Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,22619,Set Of 6 Soldier Skittles,6.19,4,15492,United Kingdom +581492,12/9/2019,22620,4 Traditional Spinning Tops,6.19,3,15492,United Kingdom +581492,12/9/2019,22621,Traditional Knitting Nancy,6.19,2,15492,United Kingdom +581492,12/9/2019,22629,Spaceboy Lunch Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22630,Dolly Girl Lunch Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22632,Hand Warmer Red Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22633,Hand Warmer Union Jack,6.19,1,15492,United Kingdom +581492,12/9/2019,22634,Childs Breakfast Set Spaceboy,6.19,1,15492,United Kingdom +581492,12/9/2019,22650,Ceramic Pirate Chest Money Bank,6.19,2,15492,United Kingdom +581492,12/9/2019,22651,Gentleman Shirt Repair Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,22653,Button Box,6.19,16,15492,United Kingdom +581492,12/9/2019,22654,Deluxe Sewing Kit,6.19,2,15492,United Kingdom +581492,12/9/2019,22659,Lunch Box I Love London,6.19,1,15492,United Kingdom +581492,12/9/2019,22666,Recipe Box Pantry Yellow Design,6.19,5,15492,United Kingdom +581492,12/9/2019,22693,Grow A Flytrap Or Sunflower In Tin,6.19,3,15492,United Kingd +581492,12/9/2019,22694,Wicker Star,6.19,1,15492,United Kingdom +581492,12/9/2019,22697,Green Regency Teacup And Saucer,6.19,1,15492,United Kingdom +581492,12/9/2019,22699,Roses Regency Teacup And Saucer,6.19,2,15492,United Kingdom +581492,12/9/2019,22701,Pink Dog Bowl,6.19,5,15492,United Kingdom +581492,12/9/2019,22703,Pink Cat Bowl,6.19,6,15492,United Kingdom +581492,12/9/2019,22712,Card Dolly Girl,6.19,1,15492,United Kingdom +581492,12/9/2019,22713,Card I Love London,6.19,1,15492,United Kingdom +581492,12/9/2019,22714,Card Birthday Cowboy,6.19,2,15492,United Kingdom +581492,12/9/2019,22716,Card Circus Parade,6.19,3,15492,United Kingdom +581492,12/9/2019,22717,Card Dog And Ball,6.19,1,15492,United Kingdom +581492,12/9/2019,22720,Set Of 3 Cake Tins Pantry Design,6.19,2,15492,United Kingdom +581492,12/9/2019,22725,Alarm Clock Bakelike Chocolate,6.19,1,15492,United Kingdom +581492,12/9/2019,22726,Alarm Clock Bakelike Green,6.19,4,15492,United Kingdom +581492,12/9/2019,22727,Alarm Clock Bakelike Red,6.19,4,15492,United Kingdom +581492,12/9/2019,22730,Alarm Clock Bakelike Ivory,6.19,1,15492,United Kingdom +581492,12/9/2019,22741,Funky Diva Pen,6.19,4,15492,United Kingdom +581492,12/9/2019,22745,Poppy's Playhouse Bedroom,6.19,2,15492,United Kingdom +581492,12/9/2019,22748,Poppy's Playhouse Kitchen,6.19,1,15492,United Kingdom +581492,12/9/2019,22749,Feltcraft Princess Charlotte Doll,6.19,1,15492,United Kingdo +581492,12/9/2019,22751,Feltcraft Princess Olivia Doll,6.19,2,15492,United Kingdom +581492,12/9/2019,22753,Small Yellow Babushka Notebook,6.19,1,15492,United Kingdom +581492,12/9/2019,22754,Small Red Babushka Notebook,6.19,1,15492,United Kingdom +581492,12/9/2019,22757,Large Red Babushka Notebook,6.19,3,15492,United Kingdom +581492,12/9/2019,22758,Large Purple Babushka Notebook,6.19,2,15492,United Kingdom +581492,12/9/2019,22763,Key Cabinet Ma Campagne,6.19,1,15492,United Kingdom +581492,12/9/2019,22768,Family Photo Frame Cornice,6.19,2,15492,United Kingdom +581492,12/9/2019,22776,Sweetheart 3 Tier Cake Stand,6.19,1,15492,United Kingdom +581492,12/9/2019,22795,Sweetheart Recipe Book Stand,6.19,1,15492,United Kingdom +581492,12/9/2019,22798,Antique Glass Dressing Table Pot,6.19,3,15492,United Kingdom +581492,12/9/2019,22800,Antique Tall Swirlglass Trinket Pot,6.19,3,15492,United King +581492,12/9/2019,22809,Set Of 6 T-Lights Santa,6.19,1,15492,United Kingdom +581492,12/9/2019,22811,Set Of 6 T-Lights Cacti,6.19,1,15492,United Kingdom +581492,12/9/2019,22814,Card Party Games,6.19,9,15492,United Kingdom +581492,12/9/2019,22815,Card Psychedelic Apples,6.19,18,15492,United Kingdom +581492,12/9/2019,22816,Card Motorbike Santa,6.19,2,15492,United Kingdom +581492,12/9/2019,22817,Card Suki Birthday,6.19,5,15492,United Kingdom +581492,12/9/2019,22818,Card Christmas Village,7.24,6,15492,United Kingdom +581492,12/9/2019,22819,Birthday Card Retro Spot,7.24,3,15492,United Kingdom +581492,12/9/2019,22835,Hot Water Bottle I Am So Poorly,7.24,3,15492,United Kingdom +581492,12/9/2019,22865,Hand Warmer Owl Design,7.24,2,15492,United Kingdom +581492,12/9/2019,22866,Hand Warmer Scotty Dog Design,7.24,3,15492,United Kingdom +581492,12/9/2019,22867,Hand Warmer Bird Design,7.24,4,15492,United Kingdom +581492,12/9/2019,22881,Number Tile Vintage Font 2,7.24,1,15492,United Kingdom +581492,12/9/2019,22885,Number Tile Vintage Font 6,7.24,1,15492,United Kingdom +581492,12/9/2019,22888,Number Tile Vintage Font 9,7.24,1,15492,United Kingdom +581492,12/9/2019,22890,Novelty Biscuits Cake Stand 3 Tier,7.24,1,15492,United Kingd +581492,12/9/2019,22898,Childrens Apron Apples Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22899,Children's Apron Dolly Girl,7.24,2,15492,United Kingdom +581492,12/9/2019,22900,Set 2 Tea Towels I Love London,7.24,3,15492,United Kingdom +581492,12/9/2019,22905,Calendar In Season Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22907,Pack Of 20 Napkins Pantry Design,6.39,1,15492,United Kingdom +581492,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,7.24,5,15492,United King +581492,12/9/2019,22910,Paper Chain Kit Vintage Christmas,7.24,7,15492,United Kingdo +581492,12/9/2019,22914,Blue Coat Rack Paris Fashion,7.24,1,15492,United Kingdom +581492,12/9/2019,22922,Fridge Magnets Us Diner Assorted,7.24,6,15492,United Kingdom +581492,12/9/2019,22924,Fridge Magnets La Vie En Rose,7.24,6,15492,United Kingdom +581492,12/9/2019,22928,Yellow Giant Garden Thermometer,7.24,1,15492,United Kingdom +581492,12/9/2019,22940,Feltcraft Christmas Fairy,6.19,1,15492,United Kingdom +581492,12/9/2019,22941,Christmas Lights 10 Reindeer,6.19,2,15492,United Kingdom +581492,12/9/2019,22943,Christmas Lights 10 Vintage Baubles,6.19,1,15492,United King +581492,12/9/2019,22948,Metal Decoration Naughty Children,6.19,4,15492,United Kingdo +581492,12/9/2019,22951,60 Cake Cases Dolly Girl Design,6.19,2,15492,United Kingdom +581492,12/9/2019,22952,60 Cake Cases Vintage Christmas,6.19,2,15492,United Kingdom +581492,12/9/2019,22955,36 Foil Star Cake Cases,6.19,1,15492,United Kingdom +581492,12/9/2019,22960,Jam Making Set With Jars,6.19,2,15492,United Kingdom +581492,12/9/2019,22961,Jam Making Set Printed,6.19,9,15492,United Kingdom +581492,12/9/2019,22962,Jam Jar With Pink Lid,6.19,3,15492,United Kingdom +581492,12/9/2019,22963,Jam Jar With Green Lid,6.19,3,15492,United Kingdom +581492,12/9/2019,22964,3 Piece Spaceboy Cookie Cutter Set,6.19,1,15492,United Kingd +581492,12/9/2019,22965,3 Traditional Biscuit Cutters Set,6.19,1,15492,United Kingdo +581492,12/9/2019,22966,Gingerbread Man Cookie Cutter,6.19,4,15492,United Kingdom +581492,12/9/2019,22969,Homemade Jam Scented Candles,6.19,6,15492,United Kingdom +581492,12/9/2019,22975,Spaceboy Childrens Egg Cup,6.19,2,15492,United Kingdom +581492,12/9/2019,22976,Circus Parade Childrens Egg Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,22977,Dolly Girl Childrens Egg Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,22979,Pantry Washing Up Brush,6.19,2,15492,United Kingdom +581492,12/9/2019,22980,Pantry Scrubbing Brush,6.19,1,15492,United Kingdom +581492,12/9/2019,22982,Pantry Pastry Brush,6.19,3,15492,United Kingdom +581492,12/9/2019,22983,Card Billboard Font,6.19,1,15492,United Kingdom +581492,12/9/2019,22984,Card Gingham Rose,6.19,1,15492,United Kingdom +581492,12/9/2019,22988,Soldiers Egg Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,22989,Set 2 Pantry Design Tea Towels,6.19,2,15492,United Kingdom +581492,12/9/2019,22992,Revolver Wooden Ruler,6.19,3,15492,United Kingdom +581492,12/9/2019,22993,Set Of 4 Pantry Jelly Moulds,6.19,3,15492,United Kingdom +581492,12/9/2019,22995,Travel Card Wallet Suki,6.19,4,15492,United Kingdom +581492,12/9/2019,22996,Travel Card Wallet Vintage Ticket,6.19,5,15492,United Kingdo +581492,12/9/2019,22997,Travel Card Wallet Union Jack,6.19,1,15492,United Kingdom +581492,12/9/2019,22998,Travel Card Wallet Keep Calm,6.19,4,15492,United Kingdom +581492,12/9/2019,22999,Travel Card Wallet Vintage Leaf,6.19,1,15492,United Kingdom +581492,12/9/2019,23005,Travel Card Wallet I Love London,6.19,4,15492,United Kingdom +581492,12/9/2019,23007,Spaceboy Baby Gift Set,6.19,1,15492,United Kingdom +581492,12/9/2019,23009,I Love London Baby Gift Set,6.19,2,15492,United Kingdom +581492,12/9/2019,23012,Glass Apothecary Bottle Perfume,6.19,1,15492,United Kingdom +581492,12/9/2019,23013,Glass Apothecary Bottle Tonic,6.19,2,15492,United Kingdom +581492,12/9/2019,23014,Glass Apothecary Bottle Elixir,6.19,1,15492,United Kingdom +581492,12/9/2019,23050,Recycled Acapulco Mat Green,6.19,1,15492,United Kingdom +581492,12/9/2019,23051,Recycled Acapulco Mat Blue,6.19,1,15492,United Kingdom +581492,12/9/2019,23052,Recycled Acapulco Mat Turquoise,6.19,5,15492,United Kingdom +581492,12/9/2019,23053,Recycled Acapulco Mat Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23054,Recycled Acapulco Mat Lavender,6.19,1,15492,United Kingdom +581492,12/9/2019,23074,Embossed Heart Trinket Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23076,Ice Cream Sundae Lip Gloss,6.19,3,15492,United Kingdom +581492,12/9/2019,23077,Doughnut Lip Gloss,6.19,3,15492,United Kingdom +581492,12/9/2019,23082,Set 6 Paper Table Lantern Hearts,6.19,1,15492,United Kingdom +581492,12/9/2019,23083,Set 6 Paper Table Lantern Stars,6.19,1,15492,United Kingdom +581492,12/9/2019,23084,Rabbit Night Light,6.19,57,15492,United Kingdom +581492,12/9/2019,23088,Zinc Heart Flower T-Light Holder,6.19,1,15492,United Kingdom +581492,12/9/2019,23089,Glass Bon Bon Jar,6.19,4,15492,United Kingdom +581492,12/9/2019,23093,Small Parisienne Heart Photo Frame,6.19,3,15492,United Kingd +581492,12/9/2019,23103,Jingle Bell Heart Decoration,6.19,2,15492,United Kingdom +581492,12/9/2019,23110,Parisienne Key Cabinet,6.19,1,15492,United Kingdom +581492,12/9/2019,23111,Parisienne Sewing Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23112,Parisienne Curio Cabinet,6.19,1,15492,United Kingdom +581492,12/9/2019,23118,Parisienne Jewellery Drawer,6.19,1,15492,United Kingdom +581492,12/9/2019,23158,Set Of 5 Lucky Cat Magnets,6.19,1,15492,United Kingdom +581492,12/9/2019,23165,Large Ceramic Top Storage Jar,6.19,2,15492,United Kingdom +581492,12/9/2019,23166,Medium Ceramic Top Storage Jar,6.19,2,15492,United Kingdom +581492,12/9/2019,23168,Classic Cafe Sugar Dispenser,6.19,2,15492,United Kingdom +581492,12/9/2019,23171,Regency Tea Plate Green,6.19,1,15492,United Kingdom +581492,12/9/2019,23172,Regency Tea Plate Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23173,Regency Teapot Roses,6.19,1,15492,United Kingdom +581492,12/9/2019,23175,Regency Milk Jug Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23177,Treasure Island Book Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23183,Mother's Kitchen Spoon Rest,6.19,3,15492,United Kingdom +581492,12/9/2019,23184,Bull Dog Bottle Opener,6.19,2,15492,United Kingdom +581492,12/9/2019,23188,Vintage 2 Metre Folding Ruler,6.19,1,15492,United Kingdom +581492,12/9/2019,23191,Bundle Of 3 Retro Note Books,6.19,1,15492,United Kingdom +581492,12/9/2019,23194,Gymkhana Treasure Book Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23196,Vintage Leaf Magnetic Notepad,6.19,1,15492,United Kingdom +581492,12/9/2019,23197,Sketchbook Magnetic Shopping List,6.19,2,15492,United Kingdo +581492,12/9/2019,23198,Pantry Magnetic Shopping List,6.19,2,15492,United Kingdom +581492,12/9/2019,23204,Charlotte Bag Apples Design,6.19,6,15492,United Kingdom +581492,12/9/2019,23205,Charlotte Bag Vintage Alphabet,6.19,3,15492,United Kingdom +581492,12/9/2019,23212,Heart Wreath Decoration With Bell,6.19,7,15492,United Kingdo +581492,12/9/2019,23213,Star Wreath Decoration With Bell,6.19,7,15492,United Kingdom +581492,12/9/2019,23220,Reindeer Heart Decoration Gold,6.19,2,15492,United Kingdom +581492,12/9/2019,23224,Cherub Heart Decoration Gold,6.19,1,15492,United Kingdom +581492,12/9/2019,23229,Vintage Donkey Tail Game,6.19,2,15492,United Kingdom +581492,12/9/2019,23234,Biscuit Tin Vintage Christmas,6.19,4,15492,United Kingdom +581492,12/9/2019,23241,Treasure Tin Gymkhana Design,6.19,3,15492,United Kingdom +581492,12/9/2019,23243,Set Of Tea Coffee Sugar Tins Pantry,6.19,1,15492,United King +581492,12/9/2019,23245,Set Of 3 Regency Cake Tins,6.19,7,15492,United Kingdom +581492,12/9/2019,23247,Biscuit Tin 50'S Christmas,6.19,3,15492,United Kingdom +581492,12/9/2019,23264,Set Of 3 Wooden Sleigh Decorations,6.19,3,15492,United Kingd +581492,12/9/2019,23265,Set Of 3 Wooden Tree Decorations,6.19,3,15492,United Kingdom +581492,12/9/2019,23266,Set Of 3 Wooden Stocking Decoration,6.19,2,15492,United King +581492,12/9/2019,23274,Star T-Light Holder Willie Winkie,6.19,3,15492,United Kingdo +581492,12/9/2019,23275,Set Of 3 Hanging Owls Ollie Beak,6.19,2,15492,United Kingdom +581492,12/9/2019,23280,Folding Butterfly Mirror Hot Pink,6.19,2,15492,United Kingdo +581492,12/9/2019,23284,Doormat Keep Calm And Come In,6.19,1,15492,United Kingdom +581492,12/9/2019,23285,Pink Vintage Spot Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23287,Red Vintage Spot Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23290,Spaceboy Childrens Bowl,6.19,1,15492,United Kingdom +581492,12/9/2019,23292,Spaceboy Childrens Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,23293,Set Of 12 Fairy Cake Baking Cases,6.19,3,15492,United Kingdo +581492,12/9/2019,23294,Set Of 6 Snack Loaf Baking Cases,6.19,1,15492,United Kingdom +581492,12/9/2019,23295,Set Of 12 Mini Loaf Baking Cases,6.19,4,15492,United Kingdom +581492,12/9/2019,23298,Spotty Bunting,6.19,2,15492,United Kingdom +581492,12/9/2019,23299,Food Cover With Beads Set 2,6.19,1,15492,United Kingdom +581492,12/9/2019,23300,Gardeners Kneeling Pad Cup Of Tea,6.19,3,15492,United Kingdo +581492,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,6,15492,United Kingdom +581492,12/9/2019,23307,Set Of 60 Pantry Design Cake Cases,6.19,7,15492,United Kingd +581492,12/9/2019,23308,Set Of 60 Vintage Leaf Cake Cases,6.19,1,15492,United Kingdo +581492,12/9/2019,23309,Set Of 60 I Love London Cake Cases,6.19,2,15492,United Kingd +581492,12/9/2019,23310,Bubblegum Ring Assorted,6.19,4,15492,United Kingdom +581492,12/9/2019,23311,Vintage Christmas Stocking,6.19,1,15492,United Kingdom +581492,12/9/2019,23312,Vintage Christmas Gift Sack,6.19,6,15492,United Kingdom +581492,12/9/2019,23313,Vintage Christmas Bunting,6.19,4,15492,United Kingdom +581492,12/9/2019,23314,Vintage Christmas Tablecloth,6.19,1,15492,United Kingdom +581492,12/9/2019,23319,Box Of 6 Mini 50'S Crackers,6.19,3,15492,United Kingdom +581492,12/9/2019,23322,Large White Heart Of Wicker,6.19,1,15492,United Kingdom +581492,12/9/2019,23323,White Wicker Star,6.19,6,15492,United Kingdom +581492,12/9/2019,23328,Set 6 School Milk Bottles In Crate,6.19,5,15492,United Kingd +581492,12/9/2019,23332,Ivory Wicker Heart Large,6.19,1,15492,United Kingdom +581492,12/9/2019,23334,Ivory Wicker Heart Small,6.19,1,15492,United Kingdom +581492,12/9/2019,23336,Egg Frying Pan Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23340,Vintage Christmas Cake Frill,6.19,3,15492,United Kingdom +581492,12/9/2019,23345,Dolly Girl Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23347,I Love London Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,2,15492,United Kingdom +581492,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,5,15492,United Kingdom +581492,12/9/2019,23352,Roll Wrap 50'S Red Christmas,6.19,1,15492,United Kingdom +581492,12/9/2019,23353,6 Gift Tags Vintage Christmas,6.19,3,15492,United Kingdom +581492,12/9/2019,23354,6 Gift Tags 50'S Christmas,6.19,4,15492,United Kingdom +581492,12/9/2019,23357,Hot Water Bottle Sex Bomb,6.19,1,15492,United Kingdom +581492,12/9/2019,23358,Hot Stuff Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,23365,Set 12 Colour Pencils Love London,6.19,1,15492,United Kingdo +581492,12/9/2019,23366,Set 12 Colouring Pencils Doily,6.04,5,15492,United Kingdom +581492,12/9/2019,23367,Set 12 Colour Pencils Spaceboy,6.04,10,15492,United Kingdom +581492,12/9/2019,23368,Set 12 Colour Pencils Dolly Girl,6.19,2,15492,United Kingdom +581492,12/9/2019,23581,Jumbo Bag Paisley Park,6.19,3,15492,United Kingdom +581492,12/9/2019,21928,Jumbo Bag Scandinavian Blue Paisley,6.19,2,15492,United King +581492,12/9/2019,21929,Jumbo Bag Pink Vintage Paisley,6.19,1,15492,United Kingdom +581492,12/9/2019,85099C,Jumbo Bag Baroque Black White,6.19,7,15492,United Kingdom +581492,12/9/2019,23199,Jumbo Bag Apples,6.19,2,15492,United Kingdom +581492,12/9/2019,23200,Jumbo Bag Pears,6.19,1,15492,United Kingdom +581492,12/9/2019,23202,Jumbo Bag Vintage Leaf,6.19,2,15492,United Kingdom +581492,12/9/2019,23343,Jumbo Bag Vintage Christmas,6.19,4,15492,United Kingdom +581492,12/9/2019,23344,Jumbo Bag 50'S Christmas,6.19,5,15492,United Kingdom +581492,12/9/2019,23583,Lunch Bag Paisley Park,6.19,2,15492,United Kingdom +581492,12/9/2019,20727,Lunch Bag Black Skull,6.04,2,15492,United Kingdom +581492,12/9/2019,20728,Lunch Bag Cars Blue,6.04,1,15492,United Kingdom +581492,12/9/2019,20725,Lunch Bag Red Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,20726,Lunch Bag Woodland,6.19,1,15492,United Kingdom +581492,12/9/2019,22662,Lunch Bag Dolly Girl Design,6.19,1,15492,United Kingdom +581492,12/9/2019,23206,Lunch Bag Apple Design,6.19,3,15492,United Kingdom +581492,12/9/2019,23208,Lunch Bag Vintage Leaf Design,6.19,3,15492,United Kingdom +581492,12/9/2019,23375,50'S Christmas Paper Gift Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,23437,50'S Christmas Gift Bag Large,6.19,1,15492,United Kingdom +581492,12/9/2019,22821,Gift Bag Psychedelic Apples,7.24,3,15492,United Kingdom +581493,12/9/2019,79190B,Retro Plastic Polka Tray,7.24,15,12423,Belgium +581493,12/9/2019,79190A,Retro Plastic 70'S Tray,7.24,15,12423,Belgium +581493,12/9/2019,22915,Assorted Bottle Top Magnets,7.24,12,12423,Belgium +581493,12/9/2019,22151,Place Setting White Heart,7.24,24,12423,Belgium +581493,12/9/2019,22632,Hand Warmer Red Retrospot,7.24,12,12423,Belgium +581493,12/9/2019,22865,Hand Warmer Owl Design,7.24,12,12423,Belgium +581493,12/9/2019,20718,Red Retrospot Shopper Bag,7.24,10,12423,Belgium +581493,12/9/2019,79190B,Retro Plastic Polka Tray,7.24,12,12423,Belgium +581493,12/9/2019,71459,Hanging Jam Jar T-Light Holders,7.24,12,12423,Belgium +581493,12/9/2019,84945,Multi Colour Silver T-Light Holder,7.24,12,12423,Belgium +581493,12/9/2019,22356,Charlotte Bag Pink Polkadot,7.24,10,12423,Belgium +581493,12/9/2019,20724,Red Retrospot Charlotte Bag,7.24,10,12423,Belgium +581493,12/9/2019,23204,Charlotte Bag Apples Design,7.24,10,12423,Belgium +581493,12/9/2019,21108,Fairy Cake Flannel Assorted Colour,7.24,18,12423,Belgium +581493,12/9/2019,22252,Birdcage Decoration Tealight Holder,7.24,12,12423,Belgium +581493,12/9/2019,22807,Set Of 6 T-Lights Toadstools,6.19,6,12423,Belgium +581494,12/9/2019,23084,Rabbit Night Light,6.19,24,12518,Germany +581494,12/9/2019,21559,Strawberry Lunch Box With Cutlery,6.19,6,12518,Germany +581494,12/9/2019,21506,Fancy Font Birthday Card,6.19,12,12518,Germany +581494,12/9/2019,22716,Card Circus Parade,6.19,12,12518,Germany +581494,12/9/2019,22556,Plasters In Tin Circus Parade,6.19,12,12518,Germany +581494,12/9/2019,22328,Round Snack Boxes Set Of 4 Fruits,6.19,6,12518,Germany +581494,12/9/2019,22730,Alarm Clock Bakelike Ivory,6.19,4,12518,Germany +581494,12/9/2019,22633,Hand Warmer Union Jack,6.19,12,12518,Germany +581494,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,6.19,6,12518,Germany +581494,12/9/2019,10125,Mini Funky Design Tapes,6.19,20,12518,Germany +581494,12/9/2019,23367,Set 12 Colour Pencils Spaceboy,6.19,24,12518,Germany +581494,12/9/2019,22551,Plasters In Tin Spaceboy,6.04,12,12518,Germany +581494,12/9/2019,22554,Plasters In Tin Woodland Animals,6.04,12,12518,Germany +581494,12/9/2019,22549,Picture Dominoes,6.19,12,12518,Germany +581494,12/9/2019,23388,Woodland Mini Backpack,6.19,4,12518,Germany +581494,12/9/2019,23390,Dolly Girl Mini Backpack,6.19,4,12518,Germany +581494,12/9/2019,23389,Spaceboy Mini Backpack,6.19,4,12518,Germany +581495,12/9/2019,23535,Wall Art Bicycle Safety,6.19,12,14051,United Kingdom +581495,12/9/2019,22699,Roses Regency Teacup And Saucer,6.19,12,14051,United Kingdom +581495,12/9/2019,22698,Pink Regency Teacup And Saucer,6.19,12,14051,United Kingdom +581495,12/9/2019,22697,Green Regency Teacup And Saucer,6.19,12,14051,United Kingdom +581495,12/9/2019,22633,Hand Warmer Union Jack,6.04,18,14051,United Kingdom +581495,12/9/2019,15056N,Edwardian Parasol Natural,6.19,36,14051,United Kingdom +581495,12/9/2019,23439,Hand Warmer Red Love Heart,6.19,36,14051,United Kingdom +581495,12/9/2019,48138,Doormat Union Flag,6.19,10,14051,United Kingdom +581495,12/9/2019,21523,Doormat Fancy Font Home Sweet Home,6.19,10,14051,United King +581495,12/9/2019,21877,Home Sweet Home Mug,6.19,12,14051,United Kingdom +581495,12/9/2019,21871,Save The Planet Mug,6.19,18,14051,United Kingdom +581495,12/9/2019,22798,Antique Glass Dressing Table Pot,6.19,36,14051,United Kingdo +581495,12/9/2019,23173,Regency Teapot Roses,6.19,6,14051,United Kingdom +581495,12/9/2019,22423,Regency Cakestand 3 Tier,6.19,10,14051,United Kingdom +581496,12/9/2019,22664,Toy Tidy Dolly Girl Design,6.19,20,16558,United Kingdom +581496,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,6.19,12,16558,United Kingdom +581496,12/9/2019,72800E,4 Ivory Dinner Candles Silver Flock,6.19,12,16558,United Ki +581496,12/9/2019,23313,Vintage Christmas Bunting,6.19,10,16558,United Kingdom +581496,12/9/2019,22952,60 Cake Cases Vintage Christmas,6.19,24,16558,United Kingdom +581496,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,6.04,12,16558,United Kin +581496,12/9/2019,23298,Spotty Bunting,6.19,3,16558,United Kingdom +581496,12/9/2019,23598,Paper Bunting Vintage Party,6.19,6,16558,United Kingdom +581496,12/9/2019,16169E,Wrap 50'S Christmas,6.19,25,16558,United Kingdom +581496,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,12,16558,United Kingdom +581496,12/9/2019,23350,Roll Wrap Vintage Spot,6.19,24,16558,United Kingdom +581496,12/9/2019,22865,Hand Warmer Owl Design,6.19,24,16558,United Kingdom +581496,12/9/2019,22632,Hand Warmer Red Retrospot,6.19,12,16558,United Kingdom +581496,12/9/2019,22835,Hot Water Bottle I Am So Poorly,6.19,4,16558,United Kingdom +581496,12/9/2019,22112,Chocolate Hot Water Bottle,6.19,6,16558,United Kingdom +581496,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,4,16558,United Kingdom +581496,12/9/2019,22314,Office Mug Warmer Choc+Blue,6.19,24,16558,United Kingdom +581496,12/9/2019,22313,Office Mug Warmer Pink,6.19,12,16558,United Kingdom +581496,12/9/2019,23084,Rabbit Night Light,6.19,18,16558,United Kingdom +581496,12/9/2019,20831,Gold Photo Frame,6.19,12,16558,United Kingdom +581496,12/9/2019,21115,Rose Caravan Doorstop,6.19,8,16558,United Kingdom +581496,12/9/2019,21462,Nursery A B C Painted Letters,7.24,8,16558,United Kingdom +581496,12/9/2019,22076,6 Ribbons Empire,7.24,24,16558,United Kingdom +581496,12/9/2019,22190,Local Cafe Mug,7.24,24,16558,United Kingdom +581496,12/9/2019,22215,Cake Stand White Two Tier Lace,7.24,6,16558,United Kingdom +581496,12/9/2019,22220,Cake Stand Lovebird 2 Tier White,7.24,6,16558,United Kingdom +581496,12/9/2019,22464,Hanging Metal Heart Lantern,7.24,12,16558,United Kingdom +581496,12/9/2019,22465,Hanging Metal Star Lantern,7.24,12,16558,United Kingdom +581496,12/9/2019,22539,Mini Jigsaw Dolly Girl,7.24,48,16558,United Kingdom +581496,12/9/2019,22544,Mini Jigsaw Spaceboy,7.24,48,16558,United Kingdom +581496,12/9/2019,23438,Red Spot Gift Bag Large,6.19,12,16558,United Kingdom +581496,12/9/2019,23437,50'S Christmas Gift Bag Large,6.19,12,16558,United Kingdom +581497,12/9/2019,20719,Woodland Charlotte Bag,6.19,33,17497,United Kingdom +581497,12/9/2019,20723,Strawberry Charlotte Bag,6.19,42,17497,United Kingdom +581497,12/9/2019,20724,Red Retrospot Charlotte Bag,6.19,55,17497,United Kingdom +581497,12/9/2019,21212,Pack Of 72 Retrospot Cake Cases,7.24,7,17497,United Kingdom +581497,12/9/2019,21238,Red Retrospot Cup,7.24,8,17497,United Kingdom +581497,12/9/2019,21242,Red Retrospot Plate,7.24,2,17497,United Kingdom +581497,12/9/2019,21479,White Skull Hot Water Bottle,7.24,25,17497,United Kingdom +581497,12/9/2019,21481,Fawn Blue Hot Water Bottle,7.24,1,17497,United Kingdom +581497,12/9/2019,21481,Fawn Blue Hot Water Bottle,7.24,1,17497,United Kingdom +581497,12/9/2019,21484,Chick Grey Hot Water Bottle,7.24,1,17497,United Kingdom +581497,12/9/2019,21668,Red Stripe Ceramic Drawer Knob,7.24,1,17497,United Kingdom +581497,12/9/2019,21670,Blue Spot Ceramic Drawer Knob,7.24,1,17497,United Kingdom +581497,12/9/2019,21671,Red Spot Ceramic Drawer Knob,7.24,1,17497,United Kingdom +581497,12/9/2019,21672,White Spot Red Ceramic Drawer Knob,7.24,6,17497,United Kingd +581497,12/9/2019,21673,White Spot Blue Ceramic Drawer Knob,7.24,1,17497,United King +581497,12/9/2019,21714,Citronella Candle Garden Pot,7.24,2,17497,United Kingdom +581497,12/9/2019,22110,Bird House Hot Water Bottle,7.24,7,17497,United Kingdom +581497,12/9/2019,22111,Scottie Dog Hot Water Bottle,7.24,5,17497,United Kingdom +581497,12/9/2019,22112,Chocolate Hot Water Bottle,7.24,13,17497,United Kingdom +581497,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,7.24,48,17497,United Kingd +581497,12/9/2019,22197,Popcorn Holder,7.24,68,17497,United Kingdom +581497,12/9/2019,22355,Charlotte Bag Suki Design,7.24,110,17497,United Kingdom +581497,12/9/2019,22356,Charlotte Bag Pink Polkadot,7.24,25,17497,United Kingdom +581497,12/9/2019,22356,Charlotte Bag Pink Polkadot,7.24,1,17497,United Kingdom +581497,12/9/2019,22423,Regency Cakestand 3 Tier,7.24,8,17497,United Kingdom +581497,12/9/2019,22725,Alarm Clock Bakelike Chocolate,7.24,3,17497,United Kingdom +581497,12/9/2019,22726,Alarm Clock Bakelike Green,7.24,1,17497,United Kingdom +581497,12/9/2019,22727,Alarm Clock Bakelike Red,7.24,10,17497,United Kingdom +581497,12/9/2019,22730,Alarm Clock Bakelike Ivory,7.24,5,17497,United Kingdom +581497,12/9/2019,22735,Ribbon Reel Socks And Mittens,7.24,2,17497,United Kingdom +581497,12/9/2019,22736,Ribbon Reel Making Snowmen,7.24,3,17497,United Kingdom +581497,12/9/2019,22738,Ribbon Reel Snowy Village,7.24,1,17497,United Kingdom +581497,12/9/2019,22805,Blue Drawer Knob Acrylic Edwardian,7.24,1,17497,United Kingd +581497,12/9/2019,22835,Hot Water Bottle I Am So Poorly,7.24,4,17497,United Kingdom +581497,12/9/2019,22895,Set Of 2 Tea Towels Apple And Pears,7.24,1,17497,United King +581497,12/9/2019,22896,Peg Bag Apples Design,7.24,2,17497,United Kingdom +581497,12/9/2019,22898,Childrens Apron Apples Design,7.24,2,17497,United Kingdom +581497,12/9/2019,22943,Christmas Lights 10 Vintage Baubles,7.24,1,17497,United King +581497,12/9/2019,23084,Rabbit Night Light,6.19,37,17497,United Kingdom +581497,12/9/2019,23168,Classic Cafe Sugar Dispenser,6.19,1,17497,United Kingdom +581497,12/9/2019,23204,Charlotte Bag Apples Design,6.04,13,17497,United Kingdom +581497,12/9/2019,23205,Charlotte Bag Vintage Alphabet,6.04,8,17497,United Kingdom +581497,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,1,17497,United Kingdom +581497,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,36,17497,United Kingdom +581497,12/9/2019,23356,Love Hot Water Bottle,6.19,1,17497,United Kingdom +581497,12/9/2019,23357,Hot Water Bottle Sex Bomb,6.19,2,17497,United Kingdom +581497,12/9/2019,23358,Hot Stuff Hot Water Bottle,6.19,2,17497,United Kingdom +581497,12/9/2019,35970,Zinc Folkart Sleigh Bells,6.19,2,17497,United Kingdom +581497,12/9/2019,47566,Party Bunting,6.19,5,17497,United Kingdom +581497,12/9/2019,82583,Hot Baths Metal Sign,6.19,4,17497,United Kingdom +581497,12/9/2019,84029G,Knitted Union Flag Hot Water Bottle,6.19,11,17497,United Ki +581497,12/9/2019,84997C,Childrens Cutlery Polkadot Blue,6.19,3,17497,United Kingdom +581497,12/9/2019,85049G,Chocolate Box Ribbons,6.19,2,17497,United Kingdom +581497,12/9/2019,20727,Lunch Bag Black Skull,6.19,8,17497,United Kingdom +581497,12/9/2019,22383,Lunch Bag Suki Design,7.24,2,17497,United Kingdom +581497,12/9/2019,23206,Lunch Bag Apple Design,6.04,3,17497,United Kingdom +581497,12/9/2019,23206,Lunch Bag Apple Design,6.04,1,17497,United Kingdom +581497,12/9/2019,23207,Lunch Bag Alphabet Design,6.19,1,17497,United Kingdom +581497,12/9/2019,23208,Lunch Bag Vintage Leaf Design,6.19,3,17497,United Kingdom +581498,12/9/2019,20669,Red Heart Luggage Tag,6.19,3,14498,United Kingdom +581498,12/9/2019,20679,Edwardian Parasol Red,6.19,5,14498,United Kingdom +581498,12/9/2019,20717,Strawberry Shopper Bag,6.19,1,14498,United Kingdom +581498,12/9/2019,20718,Red Retrospot Shopper Bag,6.19,1,14498,United Kingdom +581498,12/9/2019,20750,Red Retrospot Mini Cases,6.19,1,14498,United Kingdom +581498,12/9/2019,20961,Strawberry Bath Sponge,6.19,1,14498,United Kingdom +581498,12/9/2019,20963,Apple Bath Sponge,6.19,1,14498,United Kingdom +581498,12/9/2019,21115,Rose Caravan Doorstop,6.19,1,14498,United Kingdom +581498,12/9/2019,21125,Set 6 Football Celebration Candles,6.19,3,14498,United Kingd +581498,12/9/2019,21137,Black Record Cover Frame,6.19,4,14498,United Kingdom +581498,12/9/2019,21155,Red Retrospot Peg Bag,6.19,4,14498,United Kingdom +581498,12/9/2019,21166,Cook With Wine Metal Sign,6.19,3,14498,United Kingdom +581498,12/9/2019,21169,You're Confusing Me Metal Sign,6.19,2,14498,United Kingdom +581498,12/9/2019,21174,Pottering In The Shed Metal Sign,6.19,2,14498,United Kingdom +581498,12/9/2019,21181,Please One Person Metal Sign,6.19,6,14498,United Kingdom +581498,12/9/2019,21216,Set 3 Retrospot Tea/Coffee/Sugar,6.19,2,14498,United Kingdom +581498,12/9/2019,21217,Red Retrospot Round Cake Tins,6.19,3,14498,United Kingdom +581498,12/9/2019,21218,Red Spotty Biscuit Tin,6.19,4,14498,United Kingdom +581498,12/9/2019,21232,Strawberry Ceramic Trinket Pot,6.19,13,14498,United Kingdom +581498,12/9/2019,21239,Pink Polkadot Cup,6.19,1,14498,United Kingdom +581498,12/9/2019,21257,Victorian Sewing Box Medium,6.19,3,14498,United Kingdom +581498,12/9/2019,21327,Skulls Writing Set,6.19,1,14498,United Kingdom +581498,12/9/2019,21328,Balloons Writing Set,6.19,1,14498,United Kingdom +581498,12/9/2019,21329,Dinosaurs Writing Set,6.19,1,14498,United Kingdom +581498,12/9/2019,21411,Gingham Heart Doorstop Red,6.19,1,14498,United Kingdom +581498,12/9/2019,21430,Set/3 Red Gingham Rose Storage Box,6.19,3,14498,United Kingd +581498,12/9/2019,21494,Rotating Leaves T-Light Holder,6.19,7,14498,United Kingdom +581498,12/9/2019,21523,Doormat Fancy Font Home Sweet Home,6.19,1,14498,United Kingd +581498,12/9/2019,21557,Set Of 6 Funky Beakers,6.19,1,14498,United Kingdom +581498,12/9/2019,21558,Skull Lunch Box With Cutlery,6.19,1,14498,United Kingdom +581498,12/9/2019,21731,Red Toadstool Led Night Light,6.19,9,14498,United Kingdom +581498,12/9/2019,21754,Home Building Block Word,6.19,2,14498,United Kingdom +581498,12/9/2019,21790,Vintage Snap Cards,6.19,1,14498,United Kingdom +581498,12/9/2019,21843,Red Retrospot Cake Stand,6.19,5,14498,United Kingdom +581498,12/9/2019,21864,Union Jack Flag Passport Cover,6.19,2,14498,United Kingdom +581498,12/9/2019,21874,Gin And Tonic Mug,6.19,2,14498,United Kingdom +581498,12/9/2019,21876,Pottering Mug,6.19,4,14498,United Kingdom +581498,12/9/2019,21890,S/6 Wooden Skittles In Cotton Bag,6.19,1,14498,United Kingdo +581498,12/9/2019,21912,Vintage Snakes & Ladders,6.19,1,14498,United Kingdom +581498,12/9/2019,21916,Set 12 Retro White Chalk Sticks,6.19,7,14498,United Kingdom +581498,12/9/2019,21930,Jumbo Storage Bag Skulls,6.19,2,14498,United Kingdom +581498,12/9/2019,21931,Jumbo Storage Bag Suki,6.19,6,14498,United Kingdom +581498,12/9/2019,21934,Skull Shoulder Bag,6.19,1,14498,United Kingdom +581498,12/9/2019,21935,Suki Shoulder Bag,6.19,7,14498,United Kingdom +581498,12/9/2019,21942,Skulls Design Flannel,6.19,1,14498,United Kingdom +581498,12/9/2019,21955,Doormat Union Jack Guns And Roses,6.19,1,14498,United Kingdo +581498,12/9/2019,21977,Pack Of 60 Pink Paisley Cake Cases,6.19,1,14498,United Kingd +581498,12/9/2019,21983,Pack Of 12 Blue Paisley Tissues,6.19,1,14498,United Kingdom +581498,12/9/2019,21987,Pack Of 6 Skull Paper Cups,6.19,2,14498,United Kingdom +581498,12/9/2019,21989,Pack Of 20 Skull Paper Napkins,6.19,1,14498,United Kingdom +581498,12/9/2019,22041,"Record Frame 7"" Single Size",6.19,2,14498,United Kingdom +581498,12/9/2019,22059,Ceramic Strawberry Design Mug,6.19,1,14498,United Kingdom +581498,12/9/2019,22069,Brown Pirate Treasure Chest,6.19,3,14498,United Kingdom +581498,12/9/2019,22077,6 Ribbons Rustic Charm,6.19,2,14498,United Kingdom +581498,12/9/2019,22083,Paper Chain Kit Retrospot,6.19,2,14498,United Kingdom +581498,12/9/2019,22086,Paper Chain Kit 50'S Christmas,6.19,52,14498,United Kingdom +581498,12/9/2019,22099,Caravan Square Tissue Box,6.19,2,14498,United Kingdom +581498,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,6.19,4,14498,United Kingdo +581498,12/9/2019,22139,Retrospot Tea Set Ceramic 11 Pc,6.19,2,14498,United Kingdom +581498,12/9/2019,22141,Christmas Craft Tree Top Angel,6.19,7,14498,United Kingdom +581498,12/9/2019,22142,Christmas Craft White Fairy,6.19,2,14498,United Kingdom +581498,12/9/2019,22144,Christmas Craft Little Friends,6.19,8,14498,United Kingdom +581498,12/9/2019,22161,Heart Decoration Rustic Hanging,6.19,2,14498,United Kingdom +581498,12/9/2019,22173,Metal 4 Hook Hanger French Chateau,6.19,3,14498,United Kingd +581498,12/9/2019,22174,Photo Cube,6.19,3,14498,United Kingdom +581498,12/9/2019,22175,Pink Owl Soft Toy,6.19,1,14498,United Kingdom +581498,12/9/2019,22178,Victorian Glass Hanging T-Light,6.19,1,14498,United Kingdom +581498,12/9/2019,22179,Set 10 Night Owl Lights,6.19,1,14498,United Kingdom +581498,12/9/2019,22186,Red Star Card Holder,6.19,1,14498,United Kingdom +581498,12/9/2019,22187,Green Christmas Tree Card Holder,6.19,6,14498,United Kingdom +581498,12/9/2019,22193,Red Diner Wall Clock,6.19,1,14498,United Kingdom +581498,12/9/2019,22195,Large Heart Measuring Spoons,6.19,3,14498,United Kingdom +581498,12/9/2019,22196,Small Heart Measuring Spoons,6.19,2,14498,United Kingdom +581498,12/9/2019,22207,Frying Pan Union Flag,6.19,1,14498,United Kingdom +581498,12/9/2019,22278,Overnight Bag Vintage Rose Paisley,6.19,2,14498,United Kingd +581498,12/9/2019,22301,Coffee Mug Cat + Bird Design,6.19,2,14498,United Kingdom +581498,12/9/2019,22302,Coffee Mug Pears Design,6.19,4,14498,United Kingdom +581498,12/9/2019,22303,Coffee Mug Apples Design,6.19,4,14498,United Kingdom +581498,12/9/2019,22304,Coffee Mug Blue Paisley Design,6.19,1,14498,United Kingdom +581498,12/9/2019,22308,Tea Cosy Blue Stripe,6.19,2,14498,United Kingdom +581498,12/9/2019,22327,Round Snack Boxes Set Of 4 Skulls,6.19,4,14498,United Kingdo +581498,12/9/2019,22328,Round Snack Boxes Set Of 4 Fruits,6.19,4,14498,United Kingdo +581498,12/9/2019,22352,Lunch Box With Cutlery Retrospot,6.19,6,14498,United Kingdom +581498,12/9/2019,22357,Kings Choice Biscuit Tin,6.19,1,14498,United Kingdom +581498,12/9/2019,22358,Kings Choice Tea Caddy,6.19,1,14498,United Kingdom +581498,12/9/2019,22367,Childrens Apron Spaceboy Design,6.19,7,14498,United Kingdom +581498,12/9/2019,22371,Airline Bag Vintage Tokyo 78,6.19,1,14498,United Kingdom +581498,12/9/2019,22374,Airline Bag Vintage Jet Set Red,6.19,1,14498,United Kingdom +581498,12/9/2019,22378,Wall Tidy Retrospot,7.24,2,14498,United Kingdom +581498,12/9/2019,22379,Recycling Bag Retrospot,7.24,4,14498,United Kingdom +581498,12/9/2019,22381,Toy Tidy Pink Polkadot,7.24,1,14498,United Kingdom +581498,12/9/2019,22411,Jumbo Shopper Vintage Red Paisley,7.24,2,14498,United Kingdo +581498,12/9/2019,22422,Toothpaste Tube Pen,7.24,1,14498,United Kingdom +581498,12/9/2019,22424,Enamel Bread Bin Cream,7.24,1,14498,United Kingdom +581498,12/9/2019,22429,Enamel Measuring Jug Cream,7.24,1,14498,United Kingdom +581498,12/9/2019,22456,Natural Slate Chalkboard Large,7.24,4,14498,United Kingdom +581498,12/9/2019,22457,Natural Slate Heart Chalkboard,7.24,2,14498,United Kingdom +581498,12/9/2019,22471,Tv Dinner Tray Air Hostess,7.24,1,14498,United Kingdom +581498,12/9/2019,22474,Spaceboy Tv Dinner Tray,7.24,1,14498,United Kingdom +581498,12/9/2019,22488,Natural Slate Rectangle Chalkboard,7.24,2,14498,United Kingd +581498,12/9/2019,22498,Wooden Regatta Bunting,7.24,1,14498,United Kingdom +581498,12/9/2019,22507,Memo Board Retrospot Design,7.24,1,14498,United Kingdom +581498,12/9/2019,22526,Wheelbarrow For Children,7.24,1,14498,United Kingdom +581498,12/9/2019,22553,Plasters In Tin Skulls,7.24,1,14498,United Kingdom +581498,12/9/2019,22557,Plasters In Tin Vintage Paisley,7.24,1,14498,United Kingdom +581498,12/9/2019,22619,Set Of 6 Soldier Skittles,7.24,1,14498,United Kingdom +581498,12/9/2019,22622,Box Of Vintage Alphabet Blocks,7.24,1,14498,United Kingdom +581498,12/9/2019,22624,Ivory Kitchen Scales,7.24,1,14498,United Kingdom +581498,12/9/2019,22629,Spaceboy Lunch Box,7.24,2,14498,United Kingdom +581498,12/9/2019,22632,Hand Warmer Red Retrospot,7.24,1,14498,United Kingdom +581498,12/9/2019,22645,Ceramic Heart Fairy Cake Money Bank,7.24,4,14498,United King +581498,12/9/2019,22654,Deluxe Sewing Kit,6.19,2,14498,United Kingdom +581498,12/9/2019,22659,Lunch Box I Love London,6.19,1,14498,United Kingdom +581498,12/9/2019,22666,Recipe Box Pantry Yellow Design,6.19,11,14498,United Kingdom +581498,12/9/2019,22676,French Blue Metal Door Sign 1,6.04,1,14498,United Kingdom +581498,12/9/2019,22684,French Blue Metal Door Sign 9,6.04,1,14498,United Kingdom +581498,12/9/2019,22694,Wicker Star,6.04,1,14498,United Kingdom +581498,12/9/2019,22697,Green Regency Teacup And Saucer,6.04,6,14498,United Kingdom +581498,12/9/2019,22698,Pink Regency Teacup And Saucer,6.04,9,14498,United Kingdom +581498,12/9/2019,22699,Roses Regency Teacup And Saucer,6.19,6,14498,United Kingdom +581498,12/9/2019,22722,Set Of 6 Spice Tins Pantry Design,6.19,3,14498,United Kingdo +581498,12/9/2019,22733,3d Traditional Christmas Stickers,6.19,1,14498,United Kingdo +581498,12/9/2019,22734,Set Of 6 Ribbons Vintage Christmas,6.19,5,14498,United Kingd +581498,12/9/2019,22776,Sweetheart 3 Tier Cake Stand,6.19,1,14498,United Kingdom +581498,12/9/2019,22795,Sweetheart Recipe Book Stand,6.19,1,14498,United Kingdom +581498,12/9/2019,22807,Set Of 6 T-Lights Toadstools,6.19,2,14498,United Kingdom +581498,12/9/2019,22813,Pack 3 Boxes Bird Panettone,6.19,4,14498,United Kingdom +581498,12/9/2019,22838,3 Tier Cake Tin Red And Cream,6.19,1,14498,United Kingdom +581498,12/9/2019,22844,Vintage Cream Dog Food Container,6.19,2,14498,United Kingdom +581498,12/9/2019,22845,Vintage Cream Cat Food Container,6.19,1,14498,United Kingdom +581498,12/9/2019,22865,Hand Warmer Owl Design,6.19,1,14498,United Kingdom +581498,12/9/2019,22866,Hand Warmer Scotty Dog Design,6.19,2,14498,United Kingdom +581498,12/9/2019,22891,Tea For One Polkadot,6.19,1,14498,United Kingdom +581498,12/9/2019,22900,Set 2 Tea Towels I Love London,6.19,2,14498,United Kingdom +581498,12/9/2019,22910,Paper Chain Kit Vintage Christmas,6.19,5,14498,United Kingdo +581498,12/9/2019,22952,60 Cake Cases Vintage Christmas,6.19,7,14498,United Kingdom +581498,12/9/2019,22960,Jam Making Set With Jars,6.19,5,14498,United Kingdom +581498,12/9/2019,22961,Jam Making Set Printed,6.19,4,14498,United Kingdom +581498,12/9/2019,22966,Gingerbread Man Cookie Cutter,6.19,2,14498,United Kingdom +581498,12/9/2019,22993,Set Of 4 Pantry Jelly Moulds,6.19,2,14498,United Kingdom +581498,12/9/2019,23012,Glass Apothecary Bottle Perfume,6.19,1,14498,United Kingdom +581498,12/9/2019,23013,Glass Apothecary Bottle Tonic,6.19,1,14498,United Kingdom +581498,12/9/2019,23014,Glass Apothecary Bottle Elixir,7.24,2,14498,United Kingdom +581498,12/9/2019,23080,Red Metal Box Top Secret,7.24,5,14498,United Kingdom +581498,12/9/2019,23170,Regency Tea Plate Roses,7.24,4,14498,United Kingdom +581498,12/9/2019,23171,Regency Tea Plate Green,7.24,5,14498,United Kingdom +581498,12/9/2019,23172,Regency Tea Plate Pink,6.19,4,14498,United Kingdom +581498,12/9/2019,23181,Bull Dog Bottle Top Wall Clock,6.19,1,14498,United Kingdom +581498,12/9/2019,23196,Vintage Leaf Magnetic Notepad,6.19,1,14498,United Kingdom +581498,12/9/2019,23198,Pantry Magnetic Shopping List,6.19,2,14498,United Kingdom +581498,12/9/2019,23245,Set Of 3 Regency Cake Tins,6.19,5,14498,United Kingdom +581498,12/9/2019,23295,Set Of 12 Mini Loaf Baking Cases,6.19,1,14498,United Kingdom +581498,12/9/2019,23298,Spotty Bunting,6.19,1,14498,United Kingdom +581498,12/9/2019,23300,Gardeners Kneeling Pad Cup Of Tea,6.19,9,14498,United Kingdo +581498,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,32,14498,United Kingdo +581498,12/9/2019,23311,Vintage Christmas Stocking,6.19,6,14498,United Kingdom +581498,12/9/2019,23312,Vintage Christmas Gift Sack,6.19,3,14498,United Kingdom +581498,12/9/2019,23328,Set 6 School Milk Bottles In Crate,6.19,6,14498,United Kingd +581498,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,8,14498,United Kingdom +581498,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,22,14498,United Kingdom +581498,12/9/2019,23352,Roll Wrap 50'S Red Christmas,6.19,8,14498,United Kingdom +581498,12/9/2019,23353,6 Gift Tags Vintage Christmas,6.19,10,14498,United Kingdom +581498,12/9/2019,23354,6 Gift Tags 50'S Christmas,6.19,9,14498,United Kingdom +581498,12/9/2019,23368,Set 12 Colour Pencils Dolly Girl,6.19,3,14498,United Kingdom +581498,12/9/2019,23369,Set 36 Colour Pencils Love London,6.19,1,14498,United Kingdo +581498,12/9/2019,23370,Set 36 Colouring Pencils Doily,6.19,3,14498,United Kingdom +581498,12/9/2019,23382,Box Of 6 Christmas Cake Decorations,6.19,10,14498,United Kin +581498,12/9/2019,23388,Woodland Mini Backpack,6.19,1,14498,United Kingdom +581498,12/9/2019,23480,Mini Lights Woodland Mushrooms,7.24,1,14498,United Kingdom +581498,12/9/2019,23493,Vintage Doily Travel Sewing Kit,7.24,1,14498,United Kingdom +581498,12/9/2019,23494,Vintage Doily Deluxe Sewing Kit,7.24,1,14498,United Kingdom +581498,12/9/2019,23497,Classic Chrome Bicycle Bell,7.24,1,14498,United Kingdom +581498,12/9/2019,23501,Key Ring Baseball Boot Union Jack,7.24,1,14498,United Kingdo +581498,12/9/2019,23564,Egg Cup Milkmaid Ingrid,7.24,1,14498,United Kingdom +581498,12/9/2019,35970,Zinc Folkart Sleigh Bells,7.24,6,14498,United Kingdom +581498,12/9/2019,48138,Doormat Union Flag,7.24,1,14498,United Kingdom +581498,12/9/2019,71053,White Moroccan Metal Lantern,7.24,1,14498,United Kingdom +581498,12/9/2019,72349B,Set/6 Purple Butterfly T-Lights,7.24,2,14498,United Kingdom +581498,12/9/2019,79321,Chilli Lights,7.24,10,14498,United Kingdom +581498,12/9/2019,82001S,Silver Record Cover Frame,6.19,2,14498,United Kingdom +581498,12/9/2019,82482,Wooden Picture Frame White Finish,6.04,4,14498,United Kingdo +581498,12/9/2019,82552,Washroom Metal Sign,6.04,1,14498,United Kingdom +581498,12/9/2019,21171,Bathroom Metal Sign,6.19,1,14498,United Kingdom +581498,12/9/2019,82581,Toilet Metal Sign,6.19,1,14498,United Kingdom +581498,12/9/2019,82600,N0 Singing Metal Sign,6.19,4,14498,United Kingdom +581498,12/9/2019,84029E,Red Woolly Hottie White Heart,6.19,4,14498,United Kingdom +581498,12/9/2019,84032A,Charlie+Lola Pink Hot Water Bottle,6.19,4,14498,United King +581498,12/9/2019,84032B,Charlie + Lola Red Hot Water Bottle,6.19,3,14498,United Kin +581498,12/9/2019,84375,Set Of 20 Kids Cookie Cutters,6.19,3,14498,United Kingdom +581498,12/9/2019,84509A,Set Of 4 English Rose Placemats,6.19,1,14498,United Kingdom +581498,12/9/2019,84558A,3d Dog Picture Playing Cards,6.19,1,14498,United Kingdom +581498,12/9/2019,84832,Zinc Willie Winkie Candle Stick,6.19,26,14498,United Kingdom +581498,12/9/2019,84968E,Set Of 16 Vintage Black Cutlery,6.19,1,14498,United Kingdom +581498,12/9/2019,84970S,Hanging Heart Zinc T-Light Holder,6.19,1,14498,United Kingd +581498,12/9/2019,84997A,Childrens Cutlery Polkadot Green,6.19,2,14498,United Kingdo +581498,12/9/2019,84997B,Childrens Cutlery Retrospot Red,6.19,3,14498,United Kingdom +581498,12/9/2019,84997D,Childrens Cutlery Polkadot Pink,6.19,1,14498,United Kingdom +581498,12/9/2019,85038,6 Chocolate Love Heart T-Lights,6.19,1,14498,United Kingdom +581498,12/9/2019,85048,15cm Christmas Glass Ball 20 Lights,6.19,1,14498,United King +581498,12/9/2019,85049A,Traditional Christmas Ribbons,6.17,5,14498,United Kingdom +581498,12/9/2019,85049E,Scandinavian Reds Ribbons,6.19,4,14498,United Kingdom +581498,12/9/2019,85150,Ladies & Gentlemen Metal Sign,6.19,1,14498,United Kingdom +581498,12/9/2019,85174,S/4 Cacti Candles,6.19,1,14498,United Kingdom +581498,12/9/2019,20712,Jumbo Bag Woodland Animals,6.19,3,14498,United Kingdom +581498,12/9/2019,20713,Jumbo Bag Owls,6.19,8,14498,United Kingdom +581498,12/9/2019,21928,Jumbo Bag Scandinavian Blue Paisley,6.19,2,14498,United King +581498,12/9/2019,22386,Jumbo Bag Pink Polkadot,7.24,1,14498,United Kingdom +581498,12/9/2019,22663,Jumbo Bag Dolly Girl Design,6.19,2,14498,United Kingdom +581498,12/9/2019,23199,Jumbo Bag Apples,6.19,6,14498,United Kingdom +581498,12/9/2019,23201,Jumbo Bag Alphabet,6.19,6,14498,United Kingdom +581498,12/9/2019,85099B,Jumbo Bag Red Retrospot,6.19,5,14498,United Kingdom +581498,12/9/2019,85099C,Jumbo Bag Baroque Black White,6.19,4,14498,United Kingdom +581498,12/9/2019,20726,Lunch Bag Woodland,6.19,3,14498,United Kingdom +581498,12/9/2019,20728,Lunch Bag Cars Blue,6.19,4,14498,United Kingdom +581498,12/9/2019,22384,Lunch Bag Pink Polkadot,7.24,1,14498,United Kingdom +581498,12/9/2019,23437,50'S Christmas Gift Bag Large,7.24,1,14498,United Kingdom +581500,12/9/2019,82486,3 Drawer Antique White Wood Cabinet,7.24,4,15344,United King +581500,12/9/2019,85066,Cream Sweetheart Mini Chest,7.24,4,15344,United Kingdom +581500,12/9/2019,48187,Doormat New England,7.24,2,15344,United Kingdom +581501,12/9/2019,22319,Hairclips Forties Fabric Assorted,7.24,180,12985,United King +581501,12/9/2019,20704,Mr Robot Soft Toy,7.24,8,12985,United Kingdom +581501,12/9/2019,21564,Pink Heart Shape Love Bucket,6.19,24,12985,United Kingdom +581501,12/9/2019,21563,Red Heart Shape Love Bucket,7.24,24,12985,United Kingdom +581501,12/9/2019,22165,Diamante Heart Shaped Wall Mirror,7.24,12,12985,United Kingd +581501,12/9/2019,22299,Pig Keyring With Light & Sound,7.24,48,12985,United Kingdom +581501,12/9/2019,22447,Pin Cushion Babushka Blue,7.24,12,12985,United Kingdom +581501,12/9/2019,22442,Grow Your Own Flowers Set Of 3,7.24,12,12985,United Kingdom +581501,12/9/2019,22495,Set Of 2 Round Tins Camembert,7.24,12,12985,United Kingdom +581501,12/9/2019,22544,Mini Jigsaw Spaceboy,7.24,96,12985,United Kingdom +581501,12/9/2019,22695,Wicker Wreath Small,7.24,24,12985,United Kingdom +581501,12/9/2019,22785,Squarecushion Cover Pink Union Jack,7.24,12,12985,United Kin +581501,12/9/2019,22811,Set Of 6 T-Lights Cacti,7.24,12,12985,United Kingdom +581501,12/9/2019,22807,Set Of 6 T-Lights Toadstools,7.24,12,12985,United Kingdom +581501,12/9/2019,22942,Christmas Lights 10 Santas,7.24,12,12985,United Kingdom +581501,12/9/2019,22808,Set Of 6 T-Lights Easter Chicks,6.19,12,12985,United Kingdom +581501,12/9/2019,23143,Zinc Wire Kitchen Organiser,7.24,4,12985,United Kingdom +581501,12/9/2019,23151,Zinc Sweetheart Soap Dish,7.24,12,12985,United Kingdom +581501,12/9/2019,23425,Storage Tin Home Sweet Home,7.24,12,12985,United Kingdom +581501,12/9/2019,84356,Pompom Curtain,7.24,12,12985,United Kingdom +581501,12/9/2019,85173,Set/6 Frog Prince T-Light Candles,7.24,12,12985,United Kingd +581501,12/9/2019,21731,Red Toadstool Led Night Light,7.24,24,12985,United Kingdom +581501,12/9/2019,23480,Mini Lights Woodland Mushrooms,7.24,8,12985,United Kingdom +581501,12/9/2019,22545,Mini Jigsaw Bunnies,7.24,96,12985,United Kingdom +581502,12/9/2019,22087,Paper Bunting White Lace,7.24,6,15910,United Kingdom +581502,12/9/2019,21209,Multicolour Honeycomb Fan,7.24,5,15910,United Kingdom +581502,12/9/2019,20668,Disco Ball Christmas Decoration,7.24,24,15910,United Kingdom +581502,12/9/2019,21790,Vintage Snap Cards,7.24,6,15910,United Kingdom +581502,12/9/2019,23270,Set Of 2 Ceramic Painted Hearts,7.24,4,15910,United Kingdom +581502,12/9/2019,23103,Jingle Bell Heart Decoration,7.24,8,15910,United Kingdom +581502,12/9/2019,22576,Swallow Wooden Christmas Decoration,7.24,2,15910,United King +581502,12/9/2019,22153,Angel Decoration Stars On Dress,7.24,13,15910,United Kingdom +581502,12/9/2019,21810,Christmas Hanging Star With Bell,7.24,6,15910,United Kingdom +581502,12/9/2019,22155,Star Decoration Rustic,7.24,6,15910,United Kingdom +581502,12/9/2019,23210,White Rocking Horse Hand Painted,7.24,12,15910,United Kingdo +581502,12/9/2019,22573,Star Wooden Christmas Decoration,7.24,8,15910,United Kingdom +581502,12/9/2019,22075,6 Ribbons Elegant Christmas,7.24,4,15910,United Kingdom +581502,12/9/2019,85049A,Traditional Christmas Ribbons,7.24,4,15910,United Kingdom +581502,12/9/2019,22734,Set Of 6 Ribbons Vintage Christmas,7.24,4,15910,United Kingd +581502,12/9/2019,23274,Star T-Light Holder Willie Winkie,7.24,8,15910,United Kingdo +581502,12/9/2019,22153,Angel Decoration Stars On Dress,7.24,1,15910,United Kingdom +581502,12/9/2019,22596,Christmas Star Wish List Chalkboard,7.24,24,15910,United Kin +581502,12/9/2019,22952,60 Cake Cases Vintage Christmas,7.24,10,15910,United Kingdom +581502,12/9/2019,22141,Christmas Craft Tree Top Angel,7.24,3,15910,United Kingdom +581514,12/9/2019,22753,Small Yellow Babushka Notebook,7.24,12,17754,United Kingdom +581514,12/9/2019,22755,Small Purple Babushka Notebook,7.24,12,17754,United Kingdom +581514,12/9/2019,22754,Small Red Babushka Notebook,6.19,12,17754,United Kingdom +581514,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,6.19,4,17754,United Kingdom +581514,12/9/2019,22059,Ceramic Strawberry Design Mug,6.19,12,17754,United Kingdom +581514,12/9/2019,22199,Frying Pan Red Retrospot,6.19,13,17754,United Kingdom +581514,12/9/2019,22200,Frying Pan Pink Polkadot,6.19,2,17754,United Kingdom +581514,12/9/2019,84032A,Charlie+Lola Pink Hot Water Bottle,6.19,9,17754,United King +581514,12/9/2019,84032B,Charlie + Lola Red Hot Water Bottle,6.19,9,17754,United Kin +581514,12/9/2019,84031A,Charlie+Lola Red Hot Water Bottle,6.19,10,17754,United King +581514,12/9/2019,84031B,Charlie Lola Blue Hot Water Bottle,6.19,14,17754,United Kin +581514,12/9/2019,22646,Ceramic Strawberry Cake Money Bank,6.19,4,17754,United Kingd +581514,12/9/2019,22644,Ceramic Cherry Cake Money Bank,6.19,4,17754,United Kingdom +581514,12/9/2019,22645,Ceramic Heart Fairy Cake Money Bank,6.19,4,17754,United King +581514,12/9/2019,22394,Paperweight Kings Choice,6.04,12,17754,United Kingdom +581514,12/9/2019,17091J,Vanilla Incense In Tin,6.04,66,17754,United Kingdom +581514,12/9/2019,35471D,Set Of 3 Bird Light Pink Feather,6.04,12,17754,United Kingd +581514,12/9/2019,22075,6 Ribbons Elegant Christmas,6.04,24,17754,United Kingdom +581514,12/9/2019,17091J,Vanilla Incense In Tin,6.19,24,17754,United Kingdom +581514,12/9/2019,22069,Brown Pirate Treasure Chest,6.19,20,17754,United Kingdom +581514,12/9/2019,22068,Black Pirate Treasure Chest,6.19,14,17754,United Kingdom +581514,12/9/2019,22500,Set Of 2 Tins Jardin De Provence,6.19,4,17754,United Kingdom +581514,12/9/2019,22075,6 Ribbons Elegant Christmas,6.19,24,17754,United Kingdom +581514,12/9/2019,21705,Bag 500g Swirly Marbles,6.19,84,17754,United Kingdom +581516,12/9/2019,21109,Large Cake Towel Chocolate Spots,6.19,12,14422,United Kingdo +581516,12/9/2019,21111,Swiss Roll Towel Chocolate Spots,6.19,24,14422,United Kingdo +581516,12/9/2019,21705,Bag 500g Swirly Marbles,6.19,24,14422,United Kingdom +581516,12/9/2019,22185,Slate Tile Natural Hanging,6.19,12,14422,United Kingdom +581516,12/9/2019,22442,Grow Your Own Flowers Set Of 3,6.19,12,14422,United Kingdom +581516,12/9/2019,21620,Set Of 4 Rose Botanical Candles,6.19,12,14422,United Kingdom +581516,12/9/2019,84029G,Knitted Union Flag Hot Water Bottle,6.19,8,14422,United Kin +581516,12/9/2019,21485,Retrospot Heart Hot Water Bottle,6.19,3,14422,United Kingdom +581516,12/9/2019,22111,Scottie Dog Hot Water Bottle,6.19,6,14422,United Kingdom +581516,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,8,14422,United Kingdom +581516,12/9/2019,22112,Chocolate Hot Water Bottle,6.19,6,14422,United Kingdom +581516,12/9/2019,23356,Love Hot Water Bottle,6.19,6,14422,United Kingdom +581516,12/9/2019,21108,Fairy Cake Flannel Assorted Colour,6.19,18,14422,United King +581516,12/9/2019,22171,3 Hook Photo Shelf Antique White,6.19,4,14422,United Kingdom +581516,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,24,14422,United Kingdo +581538,12/9/2019,23193,Buffalo Bill Treasure Book Box,6.19,6,14446,United Kingdom +581538,12/9/2019,23194,Gymkhana Treasure Book Box,6.19,1,14446,United Kingdom +581538,12/9/2019,23084,Rabbit Night Light,6.19,2,14446,United Kingdom +581538,12/9/2019,22068,Black Pirate Treasure Chest,6.19,1,14446,United Kingdom +581538,12/9/2019,22956,36 Foil Heart Cake Cases,6.19,1,14446,United Kingdom +581538,12/9/2019,20936,Forked Cactus Candle,6.19,1,14446,United Kingdom +581538,12/9/2019,23040,Paper Lantern 9 Point Snow Star,6.19,1,14446,United Kingdom +581538,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,6.19,1,14446,United King +581538,12/9/2019,21222,Set/4 Badges Beetles,6.19,1,14446,United Kingdom +581538,12/9/2019,21220,Set/4 Badges Dogs,6.19,1,14446,United Kingdom +581538,12/9/2019,21224,Set/4 Skull Badges,6.19,1,14446,United Kingdom +581538,12/9/2019,85123A,Cream Hanging Heart T-Light Holder,6.19,1,14446,United King +581538,12/9/2019,22992,Revolver Wooden Ruler,6.19,1,14446,United Kingdom +581538,12/9/2019,79066K,Retro Mod Tray,6.19,1,14446,United Kingdom +581538,12/9/2019,84380,Set Of 3 Butterfly Cookie Cutters,6.19,1,14446,United Kingdo +581538,12/9/2019,23371,Set 36 Colour Pencils Spaceboy,6.19,1,14446,United Kingdom +581538,12/9/2019,22694,Wicker Star,6.19,1,14446,United Kingdom +581538,12/9/2019,22095,Lads Only Tissue Box,6.19,1,14446,United Kingdom +581538,12/9/2019,23275,Set Of 3 Hanging Owls Ollie Beak,6.19,1,14446,United Kingdom +581538,12/9/2019,21673,White Spot Blue Ceramic Drawer Knob,6.19,1,14446,United King +581538,12/9/2019,21670,Blue Spot Ceramic Drawer Knob,6.19,1,14446,United Kingdom +581538,12/9/2019,85071C,"Charlie+Lola""Extremely Busy"" Sign",6.19,1,14446,United K +581538,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,3,14446,United Kingdom +581538,12/9/2019,23034,Drawer Knob Ceramic Black,6.19,1,14446,United Kingdom +581538,12/9/2019,21669,Blue Stripe Ceramic Drawer Knob,6.19,1,14446,United Kingdom +581538,12/9/2019,23033,Drawer Knob Ceramic Red,6.19,1,14446,United Kingdom +581538,12/9/2019,21668,Red Stripe Ceramic Drawer Knob,6.19,1,14446,United Kingdom +581538,12/9/2019,23275,Set Of 3 Hanging Owls Ollie Beak,6.19,1,14446,United Kingdom +581538,12/9/2019,23318,Box Of 6 Mini Vintage Crackers,6.19,1,14446,United Kingdom +581538,12/9/2019,23382,Box Of 6 Christmas Cake Decorations,6.19,1,14446,United King +581538,12/9/2019,22469,Heart Of Wicker Small,6.19,1,14446,United Kingdom +581538,12/9/2019,22899,Children's Apron Dolly Girl,6.19,2,14446,United Kingdom +581538,12/9/2019,22367,Childrens Apron Spaceboy Design,6.19,1,14446,United Kingdom +581538,12/9/2019,22899,Children's Apron Dolly Girl,6.19,3,14446,United Kingdom +581538,12/9/2019,23527,Wall Art Animals And Nature,6.19,1,14446,United Kingdom +581538,12/9/2019,23524,Wall Art Horse & Pony,6.19,1,14446,United Kingdom +581538,12/9/2019,23525,Wall Art Buffalo Bill,6.19,1,14446,United Kingdom +581538,12/9/2019,84380,Set Of 3 Butterfly Cookie Cutters,6.19,4,14446,United Kingdo +581538,12/9/2019,23122,Party Charms 50 Pieces,7.24,1,14446,United Kingdom +581538,12/9/2019,21990,Modern Floral Stationery Set,7.24,1,14446,United Kingdom +581538,12/9/2019,21329,Dinosaurs Writing Set,7.24,1,14446,United Kingdom +581538,12/9/2019,21328,Balloons Writing Set,7.24,1,14446,United Kingdom +581538,12/9/2019,22561,Wooden School Colouring Set,7.24,1,14446,United Kingdom +581538,12/9/2019,84519A,Tomato Charlie+Lola Coaster Set,7.24,1,14446,United Kingdom +581538,12/9/2019,84519B,Carrot Charlie+Lola Coaster Set,7.24,1,14446,United Kingdom +581538,12/9/2019,35004B,Set Of 3 Black Flying Ducks,7.24,2,14446,United Kingdom +581538,12/9/2019,22068,Black Pirate Treasure Chest,7.24,1,14446,United Kingdom +581538,12/9/2019,23353,6 Gift Tags Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,21591,Cosy Hour Cigar Box Matches,6.19,1,14446,United Kingdom +581538,12/9/2019,22197,Popcorn Holder,6.19,4,14446,United Kingdom +581538,12/9/2019,23320,Giant 50'S Christmas Cracker,6.19,1,14446,United Kingdom +581538,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,22985,Wrap Billboard Fonts Design,6.19,25,14446,United Kingdom +581538,12/9/2019,23040,Paper Lantern 9 Point Snow Star,6.19,2,14446,United Kingdom +581538,12/9/2019,23040,Paper Lantern 9 Point Snow Star,6.19,3,14446,United Kingdom +581538,12/9/2019,21194,Pink Honeycomb Paper Fan,6.19,2,14446,United Kingdom +581538,12/9/2019,21208,Pastel Colour Honeycomb Fan,6.19,1,14446,United Kingdom +581538,12/9/2019,79190D,Retro Plastic Daisy Tray,6.19,1,14446,United Kingdom +581538,12/9/2019,79190B,Retro Plastic Polka Tray,6.19,1,14446,United Kingdom +581538,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,6.02,2,14446,United King +581538,12/9/2019,23318,Box Of 6 Mini Vintage Crackers,6.02,1,14446,United Kingdom +581538,12/9/2019,23319,Box Of 6 Mini 50'S Crackers,6.02,1,14446,United Kingdom +581538,12/9/2019,23537,Wall Art I Love London,6.19,1,14446,United Kingdom +581538,12/9/2019,22992,Revolver Wooden Ruler,6.19,1,14446,United Kingdom +581538,12/9/2019,22991,Giraffe Wooden Ruler,6.19,1,14446,United Kingdom +581538,12/9/2019,23190,Bundle Of 3 School Exercise Books,6.19,1,14446,United Kingdo +581538,12/9/2019,21194,Pink Honeycomb Paper Fan,6.19,1,14446,United Kingdom +581538,12/9/2019,35004B,Set Of 3 Black Flying Ducks,6.19,1,14446,United Kingdom +581538,12/9/2019,22694,Wicker Star,6.19,1,14446,United Kingdom +581538,12/9/2019,21355,Toast Its - I Love You,6.19,1,14446,United Kingdom +581538,12/9/2019,23343,Jumbo Bag Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,23343,Jumbo Bag Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,20727,Lunch Bag Black Skull,6.19,1,14446,United Kingdom +581538,12/9/2019,20725,Lunch Bag Red Retrospot,6.19,1,14446,United Kingdom +581566,12/9/2019,23404,Home Sweet Home Blackboard,6.19,144,18102,United Kingdom +581567,12/9/2019,21417,Cockle Shell Dish,6.19,84,16626,United Kingdom +581567,12/9/2019,22464,Hanging Metal Heart Lantern,6.19,24,16626,United Kingdom +581567,12/9/2019,22465,Hanging Metal Star Lantern,6.19,24,16626,United Kingdom +581567,12/9/2019,84971S,Small Heart Flowers Hook,6.19,48,16626,United Kingdom +581567,12/9/2019,22624,Ivory Kitchen Scales,6.19,2,16626,United Kingdom +581567,12/9/2019,22627,Mint Kitchen Scales,6.19,2,16626,United Kingdom +581567,12/9/2019,22625,Red Kitchen Scales,6.19,2,16626,United Kingdom +581567,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,4,16626,United Kingdom +581567,12/9/2019,21326,Aged Glass Silver T-Light Holder,6.19,144,16626,United Kingd +581567,12/9/2019,21479,White Skull Hot Water Bottle,6.19,4,16626,United Kingdom +581567,12/9/2019,23356,Love Hot Water Bottle,6.19,3,16626,United Kingdom +581567,12/9/2019,21137,Black Record Cover Frame,6.19,24,16626,United Kingdom +581570,12/9/2019,22141,Christmas Craft Tree Top Angel,6.19,6,12662,Germany +581570,12/9/2019,22175,Pink Owl Soft Toy,6.19,6,12662,Germany +581570,12/9/2019,21481,Fawn Blue Hot Water Bottle,6.19,4,12662,Germany +581570,12/9/2019,23570,Traditional Pick Up Sticks Game,6.19,12,12662,Germany +581570,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,6.19,6,12662,Germany +581570,12/9/2019,22331,Woodland Party Bag + Sticker Set,6.19,8,12662,Germany +581570,12/9/2019,22834,Hand Warmer Babushka Design,6.19,12,12662,Germany +581570,12/9/2019,21914,Blue Harmonica In Box,6.19,12,12662,Germany +581570,12/9/2019,22139,Retrospot Tea Set Ceramic 11 Pc,6.19,3,12662,Germany +581570,12/9/2019,23077,Doughnut Lip Gloss,6.19,20,12662,Germany +581570,12/9/2019,20750,Red Retrospot Mini Cases,6.19,2,12662,Germany +581570,12/9/2019,22505,Memo Board Cottage Design,6.19,4,12662,Germany +581571,12/9/2019,23326,Hanging Mini Coloured Bottles,6.19,6,15311,United Kingdom +581571,12/9/2019,21313,Glass Heart T-Light Holder,6.19,1,15311,United Kingdom +581571,12/9/2019,48187,Doormat New England,6.19,1,15311,United Kingdom +581571,12/9/2019,23317,Blue Refectory Clock,6.19,1,15311,United Kingdom +581571,12/9/2019,23197,Sketchbook Magnetic Shopping List,6.19,4,15311,United Kingdo +581571,12/9/2019,21012,Antique All Glass Candlestick,6.19,2,15311,United Kingdom +581571,12/9/2019,22227,Hanging Heart Mirror Decoration,6.19,10,15311,United Kingdom +581571,12/9/2019,22794,Sweetheart Wire Magazine Rack,6.19,1,15311,United Kingdom +581571,12/9/2019,23182,Toilet Sign Occupied Or Vacant,6.19,1,15311,United Kingdom +581571,12/9/2019,21755,Love Building Block Word,6.19,1,15311,United Kingdom +581571,12/9/2019,85053,French Enamel Candleholder,6.19,2,15311,United Kingdom +581571,12/9/2019,23110,Parisienne Key Cabinet,6.19,2,15311,United Kingdom +581571,12/9/2019,21169,You're Confusing Me Metal Sign,6.19,1,15311,United Kingdom +581571,12/9/2019,21258,Victorian Sewing Box Large,6.19,8,15311,United Kingdom +581571,12/9/2019,23168,Classic Cafe Sugar Dispenser,6.19,36,15311,United Kingdom +581571,12/9/2019,23167,Small Ceramic Top Storage Jar,6.19,96,15311,United Kingdom +581571,12/9/2019,21314,Small Glass Heart Trinket Pot,6.19,48,15311,United Kingdom +581571,12/9/2019,21137,Black Record Cover Frame,6.19,24,15311,United Kingdom +581571,12/9/2019,44234,Assorted Circular Mobile,6.19,1,15311,United Kingdom +581571,12/9/2019,84347,Rotating Silver Angels T-Light Hldr,6.19,24,15311,United Kin +581572,12/9/2019,23328,Set 6 School Milk Bottles In Crate,6.19,48,16705,United King +581572,12/9/2019,22627,Mint Kitchen Scales,6.19,4,16705,United Kingdom +581572,12/9/2019,22624,Ivory Kitchen Scales,6.19,4,16705,United Kingdom +581572,12/9/2019,23245,Set Of 3 Regency Cake Tins,6.19,4,16705,United Kingdom +581574,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,6.19,6,12526,Germany +581574,12/9/2019,22328,Round Snack Boxes Set Of 4 Fruits,6.19,6,12526,Germany +581574,12/9/2019,23238,Set Of 4 Knick Knack Tins London,6.19,6,12526,Germany +581574,12/9/2019,23237,Set Of 4 Knick Knack Tins Leaf,6.19,6,12526,Germany +581574,12/9/2019,21257,Victorian Sewing Box Medium,6.19,2,12526,Germany +581574,12/9/2019,21258,Victorian Sewing Box Large,6.19,2,12526,Germany +581574,12/9/2019,23111,Parisienne Sewing Box,6.19,2,12526,Germany +581574,12/9/2019,22077,6 Ribbons Rustic Charm,6.19,12,12526,Germany +581574,12/9/2019,22074,6 Ribbons Shimmering Pinks,6.19,12,12526,Germany +581574,12/9/2019,22621,Traditional Knitting Nancy,6.19,12,12526,Germany +581574,12/9/2019,23199,Jumbo Bag Apples,6.19,10,12526,Germany +581574,12/9/2019,23581,Jumbo Bag Paisley Park,6.19,10,12526,Germany +581578,12/9/2019,21124,Set/10 Blue Polkadot Party Candles,6.19,24,12713,Germany +581578,12/9/2019,21122,Set/10 Pink Polkadot Party Candles,6.19,24,12713,Germany +581578,12/9/2019,21121,Set/10 Red Polkadot Party Candles,6.19,24,12713,Germany +581578,12/9/2019,23389,Spaceboy Mini Backpack,6.04,4,12713,Germany +581578,12/9/2019,22556,Plasters In Tin Circus Parade,6.19,12,12713,Germany +581578,12/9/2019,22976,Circus Parade Childrens Egg Cup,6.19,12,12713,Germany +581578,12/9/2019,23255,Childrens Cutlery Circus Parade,7.24,12,12713,Germany +581578,12/9/2019,84997D,Childrens Cutlery Polkadot Pink,7.24,8,12713,Germany +581578,12/9/2019,84997B,Childrens Cutlery Retrospot Red,7.24,8,12713,Germany +581578,12/9/2019,84997C,Childrens Cutlery Polkadot Blue,7.24,8,12713,Germany +581578,12/9/2019,22555,Plasters In Tin Strongman,7.24,12,12713,Germany +581578,12/9/2019,21914,Blue Harmonica In Box,7.24,12,12713,Germany +581578,12/9/2019,22549,Picture Dominoes,7.24,24,12713,Germany +581578,12/9/2019,21918,Set 12 Kids Colour Chalk Sticks,7.24,24,12713,Germany +581578,12/9/2019,22992,Revolver Wooden Ruler,7.24,12,12713,Germany +581578,12/9/2019,22991,Giraffe Wooden Ruler,7.24,12,12713,Germany +581578,12/9/2019,23229,Vintage Donkey Tail Game,7.24,6,12713,Germany +581578,12/9/2019,22622,Box Of Vintage Alphabet Blocks,6.19,6,12713,Germany +581578,12/9/2019,21506,Fancy Font Birthday Card,6.19,12,12713,Germany +581578,12/9/2019,21507,Elephant Birthday Card,6.19,12,12713,Germany +581578,12/9/2019,23037,Candle Holder Silver Madeline,6.19,12,12713,Germany +581578,12/9/2019,23550,Wrap Alphabet Poster,6.19,25,12713,Germany +581578,12/9/2019,22711,Wrap Circus Parade,6.19,25,12713,Germany +581578,12/9/2019,21497,Fancy Fonts Birthday Wrap,6.19,25,12713,Germany +581578,12/9/2019,22704,Wrap Red Apples,6.19,25,12713,Germany +581578,12/9/2019,22585,Pack Of 6 Birdy Gift Tags,6.19,12,12713,Germany diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/python-challenge/task.py b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/python-challenge/task.py new file mode 100644 index 0000000000000..5b2a00a98fbcb --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/python-challenge/task.py @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# beam-playground: +# name: FinalChallenge1 +# description: Final challenge 1. +# multifile: true +# files: +# - name: input.csv +# context_line: 50 +# categories: +# - Quickstart +# complexity: ADVANCED +# tags: +# - hellobeam + +import apache_beam as beam +import logging +import re +from apache_beam.transforms import window, trigger +from apache_beam.transforms.combiners import CountCombineFn + + +class Transaction: + def __init__(self, transaction_no, date, product_no, product_name, price, quantity, customer_no, country): + self.transaction_no = transaction_no + self.date = date + self.product_no = product_no + self.product_name = product_name + self.price = price + self.quantity = quantity + self.customer_no = customer_no + self.country = country + + def __str__(self): + return f"Transaction(transaction_no={self.transaction_no}, date='{self.date}', product_no='{self.product_no}', product_name='{self.product_name}', price={self.price}, quantity={self.quantity}, customer_no={self.customer_no}, country='{self.country}')" + + +def run(): + with beam.Pipeline() as pipeline: + transactions = (pipeline + | 'Read from text file' >> beam.io.ReadFromText('input.csv')) + + +if __name__ == '__main__': + run() diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/python-solution/input.csv b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/python-solution/input.csv new file mode 100644 index 0000000000000..85854a2d43584 --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/python-solution/input.csv @@ -0,0 +1,1500 @@ +TransactionNo,Date,ProductNo,ProductName,Price,Quantity,CustomerNo,Country +581482,12/9/2019,22485,Set Of 2 Wooden Market Crates,21.47,12,17490,United Kingdom +581475,12/9/2019,22596,Christmas Star Wish List Chalkboard,10.65,36,13069,United Ki +581475,12/9/2019,23235,Storage Tin Vintage Leaf,11.53,12,13069,United Kingdom +581475,12/9/2019,23272,Tree T-Light Holder Willie Winkie,10.65,12,13069,United King +581475,12/9/2019,23239,Set Of 4 Knick Knack Tins Poppies,11.94,6,13069,United Kingd +581475,12/9/2019,21705,Bag 500g Swirly Marbles,10.65,24,13069,United Kingdom +581475,12/9/2019,22118,Joy Wooden Block Letters,11.53,18,13069,United Kingdom +581475,12/9/2019,22119,Peace Wooden Block Letters,12.25,12,13069,United Kingdom +581475,12/9/2019,22217,T-Light Holder Hanging Lace,10.65,12,13069,United Kingdom +581475,12/9/2019,22216,T-Light Holder White Lace,10.55,24,13069,United Kingdom +581475,12/9/2019,22380,Toy Tidy Spaceboy,11.06,20,13069,United Kingdom +581475,12/9/2019,22442,Grow Your Own Flowers Set Of 3,12.25,12,13069,United Kingdom +581475,12/9/2019,22664,Toy Tidy Dolly Girl Design,11.06,20,13069,United Kingdom +581475,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,12.25,12,13069,United Kingdom +581475,12/9/2019,22723,Set Of 6 Herb Tins Sketchbook,11.53,12,13069,United Kingdom +581475,12/9/2019,22785,Squarecushion Cover Pink Union Jack,11.53,12,13069,United Ki +581475,12/9/2019,22955,36 Foil Star Cake Cases,11.06,24,13069,United Kingdom +581475,12/9/2019,23141,Triple Wire Hook Pink Heart,11.06,12,13069,United Kingdom +581475,12/9/2019,22956,36 Foil Heart Cake Cases,11.06,24,13069,United Kingdom +581475,12/9/2019,22581,Wood Stocking Christmas Scandispot,10.55,48,13069,United Kin +581476,12/9/2019,23198,Pantry Magnetic Shopping List,11.53,48,12433,Norway +581476,12/9/2019,23197,Sketchbook Magnetic Shopping List,11.74,24,12433,Norway +581476,12/9/2019,23184,Bull Dog Bottle Opener,15.32,8,12433,Norway +581476,12/9/2019,23168,Classic Cafe Sugar Dispenser,11.53,12,12433,Norway +581476,12/9/2019,23167,Small Ceramic Top Storage Jar,10.96,96,12433,Norway +581476,12/9/2019,23166,Medium Ceramic Top Storage Jar,11.32,48,12433,Norway +581476,12/9/2019,23165,Large Ceramic Top Storage Jar,11.74,48,12433,Norway +581476,12/9/2019,23004,Travel Card Wallet Pantry,10.68,48,12433,Norway +581476,12/9/2019,23002,Travel Card Wallet Skulls,10.68,24,12433,Norway +581476,12/9/2019,23000,Travel Card Wallet Transport,10.68,24,12433,Norway +581476,12/9/2019,22998,Travel Card Wallet Keep Calm,10.68,72,12433,Norway +581476,12/9/2019,22994,Travel Card Wallet Retrospot,10.68,48,12433,Norway +581476,12/9/2019,22835,Hot Water Bottle I Am So Poorly,15.32,8,12433,Norway +581476,12/9/2019,22730,Alarm Clock Bakelike Ivory,14.09,4,12433,Norway +581476,12/9/2019,22728,Alarm Clock Bakelike Pink,14.09,8,12433,Norway +581476,12/9/2019,22727,Alarm Clock Bakelike Red,14.09,8,12433,Norway +581476,12/9/2019,22726,Alarm Clock Bakelike Green,14.09,4,12433,Norway +581476,12/9/2019,22720,Set Of 3 Cake Tins Pantry Design,14.61,16,12433,Norway +581476,12/9/2019,22693,Grow A Flytrap Or Sunflower In Tin,11.34,192,12433,Norway +581476,12/9/2019,22670,French Wc Sign Blue Metal,11.53,24,12433,Norway +581476,12/9/2019,22667,Recipe Box Retrospot,12.86,24,12433,Norway +581476,12/9/2019,22666,Recipe Box Pantry Yellow Design,12.86,24,12433,Norway +581476,12/9/2019,22631,Circus Parade Lunch Box,12.25,12,12433,Norway +581476,12/9/2019,22628,Picnic Boxes Set Of 3 Retrospot,15.32,16,12433,Norway +581476,12/9/2019,22467,Gumball Coat Rack,12.86,6,12433,Norway +581476,12/9/2019,22197,Popcorn Holder,10.99,100,12433,Norway +581476,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,14.61,8,12433,Norway +581476,12/9/2019,22112,Chocolate Hot Water Bottle,15.32,9,12433,Norway +581476,12/9/2019,21908,Chocolate This Way Metal Sign,12.40,36,12433,Norway +581476,12/9/2019,21874,Gin And Tonic Mug,11.74,36,12433,Norway +581476,12/9/2019,21872,Glamorous Mug,11.53,12,12433,Norway +581476,12/9/2019,21871,Save The Planet Mug,11.74,36,12433,Norway +581476,12/9/2019,21533,Retrospot Large Milk Jug,15.32,6,12433,Norway +581476,12/9/2019,21481,Fawn Blue Hot Water Bottle,14.09,12,12433,Norway +581476,12/9/2019,21479,White Skull Hot Water Bottle,14.61,4,12433,Norway +581476,12/9/2019,21248,Door Hanger Mum + Dads Room,11.74,12,12433,Norway +581476,12/9/2019,21216,Set 3 Retrospot Tea/Coffee/Sugar,14.61,24,12433,Norway +581476,12/9/2019,21181,Please One Person Metal Sign,12.15,48,12433,Norway +581476,12/9/2019,21175,Gin And Tonic Diet Metal Sign,12.38,48,12433,Norway +581476,12/9/2019,21169,You're Confusing Me Metal Sign,11.74,48,12433,Norway +581476,12/9/2019,21162,Toxic Area Door Hanger,10.65,24,12433,Norway +581476,12/9/2019,21159,Moody Boy Door Hanger,10.65,24,12433,Norway +581476,12/9/2019,21158,Moody Girl Door Hanger,10.65,24,12433,Norway +581476,12/9/2019,21154,Red Retrospot Oven Glove,11.53,10,12433,Norway +581476,12/9/2019,16016,Large Chinese Style Scissor,11.12,40,12433,Norway +581476,12/9/2019,16014,Small Chinese Style Scissor,10.68,60,12433,Norway +581476,12/9/2019,16008,Small Folding Scissor(Pointed Edge),10.37,240,12433,Norway +581476,12/9/2019,85152,Hand Over The Chocolate Sign,12.15,48,12433,Norway +581476,12/9/2019,84596F,Small Marshmallows Pink Bowl,10.68,32,12433,Norway +581476,12/9/2019,84596B,Small Dolly Mix Design Orange Bowl,10.68,16,12433,Norway +581476,12/9/2019,84510A,Set Of 4 English Rose Coasters,11.53,20,12433,Norway +581476,12/9/2019,82600,N0 Singing Metal Sign,12.15,48,12433,Norway +581476,12/9/2019,82581,Toilet Metal Sign,10.81,48,12433,Norway +581476,12/9/2019,72232,Feng Shui Pillar Candle,10.44,144,12433,Norway +581476,12/9/2019,47559B,Tea Time Oven Glove,11.53,10,12433,Norway +581476,12/9/2019,47504H,English Rose Spirit Level,11.06,36,12433,Norway +581476,12/9/2019,23493,Vintage Doily Travel Sewing Kit,12.25,30,12433,Norway +581476,12/9/2019,23430,Blue Retro Kitchen Wall Clock,18.60,2,12433,Norway +581476,12/9/2019,23429,Red Retro Kitchen Wall Clock,18.60,2,12433,Norway +581476,12/9/2019,23428,Ivory Retro Kitchen Wall Clock,18.60,2,12433,Norway +581476,12/9/2019,23358,Hot Stuff Hot Water Bottle,11.53,18,12433,Norway +581476,12/9/2019,23293,Set Of 12 Fairy Cake Baking Cases,11.10,32,12433,Norway +581476,12/9/2019,23243,Set Of Tea Coffee Sugar Tins Pantry,15.32,12,12433,Norway +581476,12/9/2019,23240,Set Of 4 Knick Knack Tins Doily,14.50,6,12433,Norway +581477,12/9/2019,48111,Doormat 3 Smiley Cats,17.51,10,13426,United Kingdom +581477,12/9/2019,22464,Hanging Metal Heart Lantern,11.06,12,13426,United Kingdom +581477,12/9/2019,20982,12 Pencils Tall Tube Skulls,11.12,12,13426,United Kingdom +581477,12/9/2019,20981,12 Pencils Tall Tube Woodland,11.12,12,13426,United Kingdom +581477,12/9/2019,23424,Gingham Recipe Book Box,15.32,8,13426,United Kingdom +581477,12/9/2019,23338,Egg Frying Pan Red,12.15,48,13426,United Kingdom +581477,12/9/2019,84970L,Single Heart Zinc T-Light Holder,11.53,12,13426,United King +581477,12/9/2019,22457,Natural Slate Heart Chalkboard,13.27,6,13426,United Kingdom +581477,12/9/2019,22469,Heart Of Wicker Small,11.94,12,13426,United Kingdom +581477,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,11.12,12,13426,United Ki +581478,12/9/2019,84947,Antique Silver Tea Glass Engraved,11.53,24,17364,United King +581478,12/9/2019,23503,Playing Cards Keep Calm & Carry On,11.53,12,17364,United Kin +581478,12/9/2019,23445,Ice Cream Bubbles,11.10,20,17364,United Kingdom +581478,12/9/2019,23530,Wall Art Only One Person,15.32,12,17364,United Kingdom +581478,12/9/2019,84997C,Childrens Cutlery Polkadot Blue,14.50,4,17364,United Kingdo +581478,12/9/2019,84997D,Childrens Cutlery Polkadot Pink,14.50,4,17364,United Kingdo +581478,12/9/2019,84077,World War 2 Gliders Asstd Designs,10.55,48,17364,United King +581478,12/9/2019,22749,Feltcraft Princess Charlotte Doll,14.09,4,17364,United Kingd +581478,12/9/2019,23127,Feltcraft Girl Nicole Kit,15.32,4,17364,United Kingdom +581478,12/9/2019,23126,Feltcraft Girl Amelie Kit,15.32,4,17364,United Kingdom +581478,12/9/2019,22747,Poppy's Playhouse Bathroom,12.40,6,17364,United Kingdom +581478,12/9/2019,22078,Ribbon Reel Lace Design,12.40,10,17364,United Kingdom +581478,12/9/2019,84946,Antique Silver T-Light Glass,11.53,12,17364,United Kingdom +581478,12/9/2019,22791,T-Light Glass Fluted Antique,11.53,12,17364,United Kingdom +581478,12/9/2019,21326,Aged Glass Silver T-Light Holder,10.92,12,17364,United Kingd +581478,12/9/2019,22170,Picture Frame Wood Triple Portrait,17.17,8,17364,United King +581478,12/9/2019,23082,Set 6 Paper Table Lantern Hearts,14.09,6,17364,United Kingdo +581478,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,11.12,24,17364,United Ki +581479,12/9/2019,22087,Paper Bunting White Lace,13.27,10,17364,United Kingdom +581480,12/9/2019,23464,Vintage Zinc Watering Can Small,15.32,4,14441,United Kingdom +581480,12/9/2019,22411,Jumbo Shopper Vintage Red Paisley,12.38,10,14441,United King +581480,12/9/2019,84029E,Red Woolly Hottie White Heart,14.61,8,14441,United Kingdom +581480,12/9/2019,22633,Hand Warmer Union Jack,12.40,12,14441,United Kingdom +581480,12/9/2019,23355,Hot Water Bottle Keep Calm,15.32,12,14441,United Kingdom +581480,12/9/2019,84029G,Knitted Union Flag Hot Water Bottle,14.61,12,14441,United K +581480,12/9/2019,22112,Chocolate Hot Water Bottle,15.32,6,14441,United Kingdom +581480,12/9/2019,84347,Rotating Silver Angels T-Light Hldr,12.86,18,14441,United Ki +581481,12/9/2019,21115,Rose Caravan Doorstop,12.25,8,17490,United Kingdom +581481,12/9/2019,22059,Ceramic Strawberry Design Mug,10.65,24,17490,United Kingdom +581481,12/9/2019,22072,Red Retrospot Tea Cup And Saucer,11.53,24,17490,United Kingd +581481,12/9/2019,22123,Ping Microwave Apron,11.06,24,17490,United Kingdom +581481,12/9/2019,22476,Empire Union Jack Tv Dinner Tray,12.25,8,17490,United Kingdo +581481,12/9/2019,22495,Set Of 2 Round Tins Camembert,11.06,12,17490,United Kingdom +581481,12/9/2019,22496,Set Of 2 Round Tins Dutch Cheese,11.06,12,17490,United Kingd +581481,12/9/2019,22513,Doorstop Football Design,11.06,8,17490,United Kingdom +581481,12/9/2019,22500,Set Of 2 Tins Jardin De Provence,11.53,12,17490,United Kingd +581481,12/9/2019,22664,Toy Tidy Dolly Girl Design,11.06,20,17490,United Kingdom +581481,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,12.25,12,17490,United Kingdom +581481,12/9/2019,22785,Squarecushion Cover Pink Union Jack,11.53,12,17490,United Ki +581481,12/9/2019,23178,Jam Clock Magnet,11.53,12,17490,United Kingdom +581481,12/9/2019,23302,Kneeling Mat Housework Design,11.06,24,17490,United Kingdom +581481,12/9/2019,23533,Wall Art Garden Haven,12.25,6,17490,United Kingdom +581481,12/9/2019,84819,Danish Rose Round Sewing Box,11.06,16,17490,United Kingdom +581482,12/9/2019,22371,Airline Bag Vintage Tokyo 78,14.30,12,17490,United Kingdom +581482,12/9/2019,21875,Kings Choice Mug,11.34,36,17490,United Kingdom +581482,12/9/2019,23251,Vintage Red Enamel Trim Mug,11.32,96,17490,United Kingdom +581482,12/9/2019,23300,Gardeners Kneeling Pad Cup Of Tea,11.74,48,17490,United King +581482,12/9/2019,22138,Baking Set 9 Piece Retrospot,14.61,24,17490,United Kingdom +581483,12/9/2019,23843,Paper Craft Little Birdie,12.38,80995,16446,United Kingdom +581485,12/9/2019,22617,Baking Set Spaceboy Design,15.32,18,17389,United Kingdom +581485,12/9/2019,20749,Assorted Colour Mini Cases,16.76,84,17389,United Kingdom +581486,12/9/2019,22910,Paper Chain Kit Vintage Christmas,13.27,12,17001,United King +581486,12/9/2019,22086,Paper Chain Kit 50'S Christmas,13.27,12,17001,United Kingdom +581486,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,12,17001,United Kingdom +581486,12/9/2019,23352,Roll Wrap 50'S Red Christmas,6.19,12,17001,United Kingdom +581486,12/9/2019,22727,Alarm Clock Bakelike Red,6.19,8,17001,United Kingdom +581486,12/9/2019,22730,Alarm Clock Bakelike Ivory,6.19,4,17001,United Kingdom +581486,12/9/2019,22491,Pack Of 12 Coloured Pencils,6.04,12,17001,United Kingdom +581486,12/9/2019,22561,Wooden School Colouring Set,6.04,12,17001,United Kingdom +581486,12/9/2019,22489,Pack Of 12 Traditional Crayons,6.04,24,17001,United Kingdom +581486,12/9/2019,22560,Traditional Modelling Clay,6.04,24,17001,United Kingdom +581486,12/9/2019,22139,Retrospot Tea Set Ceramic 11 Pc,6.04,6,17001,United Kingdom +581486,12/9/2019,23344,Jumbo Bag 50'S Christmas,6.19,20,17001,United Kingdom +581486,12/9/2019,23203,Jumbo Bag Vintage Doily,6.19,20,17001,United Kingdom +581486,12/9/2019,85099B,Jumbo Bag Red Retrospot,6.19,10,17001,United Kingdom +581486,12/9/2019,23201,Jumbo Bag Alphabet,6.19,20,17001,United Kingdom +581486,12/9/2019,23207,Lunch Bag Alphabet Design,6.19,10,17001,United Kingdom +581487,12/9/2019,21137,Black Record Cover Frame,6.04,120,15694,United Kingdom +581488,12/9/2019,23118,Parisienne Jewellery Drawer,6.04,16,17428,United Kingdom +581488,12/9/2019,23111,Parisienne Sewing Box,6.04,16,17428,United Kingdom +581488,12/9/2019,22179,Set 10 Night Owl Lights,6.04,24,17428,United Kingdom +581489,12/9/2019,22061,Large Cake Stand Hanging Strawbery,7.24,48,16954,United King +581489,12/9/2019,22182,Cake Stand Victorian Filigree Small,7.24,24,16954,United Kin +581491,12/9/2019,23571,Traditional Naughts & Crosses,7.24,12,12433,Norway +581492,12/9/2019,23369,Set 36 Colour Pencils Love London,6.19,2,15492,United Kingdo +581492,12/9/2019,23370,Set 36 Colouring Pencils Doily,6.19,2,15492,United Kingdom +581492,12/9/2019,23371,Set 36 Colour Pencils Spaceboy,6.19,3,15492,United Kingdom +581492,12/9/2019,23372,Set 36 Colour Pencils Dolly Girl,6.19,1,15492,United Kingdom +581492,12/9/2019,23376,Pack Of 12 Vintage Christmas Tissue,6.19,3,15492,United King +581492,12/9/2019,23377,Pack Of 12 Dolly Girl Tissues,6.19,6,15492,United Kingdom +581492,12/9/2019,23378,Pack Of 12 50'S Christmas Tissues,6.19,9,15492,United Kingdo +581492,12/9/2019,23379,Pack Of 12 Red Apple Tissues,6.19,4,15492,United Kingdom +581492,12/9/2019,23380,Pack Of 12 Vintage Doily Tissues,6.19,3,15492,United Kingdom +581492,12/9/2019,23381,Pack Of 12 Vintage Leaf Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,23382,Box Of 6 Christmas Cake Decorations,6.19,4,15492,United King +581492,12/9/2019,23391,I Love London Mini Backpack,6.19,2,15492,United Kingdom +581492,12/9/2019,23392,Spaceboy Rocket Lolly Makers,6.19,2,15492,United Kingdom +581492,12/9/2019,23399,Home Sweet Home Hanging Heart,6.19,4,15492,United Kingdom +581492,12/9/2019,23405,Home Sweet Home 2 Drawer Cabinet,6.19,3,15492,United Kingdom +581492,12/9/2019,23418,Lavender Toilette Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,23426,Metal Sign Drop Your Pants,6.19,1,15492,United Kingdom +581492,12/9/2019,23434,3 Raffia Ribbons 50'S Christmas,6.19,9,15492,United Kingdom +581492,12/9/2019,23435,3 Raffia Ribbons Vintage Christmas,6.04,3,15492,United Kingd +581492,12/9/2019,23439,Hand Warmer Red Love Heart,6.19,1,15492,United Kingdom +581492,12/9/2019,23451,Square Mini Portrait Frame,6.19,1,15492,United Kingdom +581492,12/9/2019,23467,Vintage Zinc Planter,6.19,1,15492,United Kingdom +581492,12/9/2019,23469,Card Holder Love Bird Small,6.19,3,15492,United Kingdom +581492,12/9/2019,23480,Mini Lights Woodland Mushrooms,6.19,2,15492,United Kingdom +581492,12/9/2019,23493,Vintage Doily Travel Sewing Kit,6.19,3,15492,United Kingdom +581492,12/9/2019,23494,Vintage Doily Deluxe Sewing Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,23495,Set Of 3 Pantry Wooden Spoons,6.19,1,15492,United Kingdom +581492,12/9/2019,23497,Classic Chrome Bicycle Bell,6.19,4,15492,United Kingdom +581492,12/9/2019,23498,Classic Bicycle Clips,6.19,2,15492,United Kingdom +581492,12/9/2019,23501,Key Ring Baseball Boot Union Jack,6.19,3,15492,United Kingdo +581492,12/9/2019,23506,Mini Playing Cards Spaceboy,6.04,1,15492,United Kingdom +581492,12/9/2019,23508,Mini Playing Cards Dolly Girl,6.04,2,15492,United Kingdom +581492,12/9/2019,23510,Mini Playing Cards Gymkhana,6.04,1,15492,United Kingdom +581492,12/9/2019,23521,Wall Art Cat And Bird,6.04,1,15492,United Kingdom +581492,12/9/2019,23526,Wall Art Dog Licence,6.04,4,15492,United Kingdom +581492,12/9/2019,23530,Wall Art Only One Person,6.04,2,15492,United Kingdom +581492,12/9/2019,23534,Wall Art Stop For Tea,6.19,6,15492,United Kingdom +581492,12/9/2019,23535,Wall Art Bicycle Safety,6.19,4,15492,United Kingdom +581492,12/9/2019,23536,Wall Art Village Show,6.19,1,15492,United Kingdom +581492,12/9/2019,23538,Wall Art Vintage Heart,6.19,1,15492,United Kingdom +581492,12/9/2019,23551,Pack Of 12 Paisley Park Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,23552,Bicycle Puncture Repair Kit,6.19,12,15492,United Kingdom +581492,12/9/2019,23555,Landmark Frame Notting Hill,6.19,1,15492,United Kingdom +581492,12/9/2019,23559,Woodland Bunnies Lolly Makers,6.19,5,15492,United Kingdom +581492,12/9/2019,23561,Set Of 6 Ribbons Party,6.19,1,15492,United Kingdom +581492,12/9/2019,23564,Egg Cup Milkmaid Ingrid,6.19,2,15492,United Kingdom +581492,12/9/2019,23565,Egg Cup Milkmaid Helga,6.19,2,15492,United Kingdom +581492,12/9/2019,23567,Egg Cup Henrietta Hen Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23569,Tradtional Alphabet Stamp Set,6.19,2,15492,United Kingdom +581492,12/9/2019,23570,Traditional Pick Up Sticks Game,6.19,7,15492,United Kingdom +581492,12/9/2019,23571,Traditional Naughts & Crosses,6.19,2,15492,United Kingdom +581492,12/9/2019,23575,Snack Tray Paisley Park,6.19,3,15492,United Kingdom +581492,12/9/2019,23579,Snack Tray I Love London,6.19,1,15492,United Kingdom +581492,12/9/2019,23611,Set 10 Cards Red Riding Hood 17214,6.19,3,15492,United Kingd +581492,12/9/2019,23616,Set 10 Cards Jingle Bells 17217,6.19,1,15492,United Kingdom +581492,12/9/2019,23621,Set 10 Cards David's Madonna 17074,6.19,3,15492,United Kingd +581492,12/9/2019,23635,Set 10 Cards Christmas Holly 17259,6.19,1,15492,United Kingd +581492,12/9/2019,23644,Set 10 Cards Christmas Tree 16955,6.19,1,15492,United Kingdo +581492,12/9/2019,23660,Henrietta Hen Mug,6.19,1,15492,United Kingdom +581492,12/9/2019,23661,Milk Maids Mug,6.19,1,15492,United Kingdom +581492,12/9/2019,35095A,Blue Victorian Fabric Oval Box,6.19,1,15492,United Kingdom +581492,12/9/2019,35095B,Red Victorian Fabric Oval Box,6.19,1,15492,United Kingdom +581492,12/9/2019,35646,Vintage Bead Pink Evening Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,35648,Vintage Bead Pink Purse,6.19,2,15492,United Kingdom +581492,12/9/2019,35953,Folkart Star Christmas Decorations,6.19,12,15492,United King +581492,12/9/2019,35964,Folkart Clip On Stars,6.19,4,15492,United Kingdom +581492,12/9/2019,35970,Zinc Folkart Sleigh Bells,6.04,1,15492,United Kingdom +581492,12/9/2019,46118,Funky Monkey Cushion Cover,6.19,1,15492,United Kingdom +581492,12/9/2019,46776A,Woven Bubble Gum Cushion Cover,7.24,2,15492,United Kingdom +581492,12/9/2019,46776B,Woven Berries Cushion Cover,7.24,3,15492,United Kingdom +581492,12/9/2019,46776C,Woven Frost Cushion Cover,7.24,1,15492,United Kingdom +581492,12/9/2019,46776D,Woven Sunset Cushion Cover,7.24,2,15492,United Kingdom +581492,12/9/2019,46776E,Woven Candy Cushion Cover,7.24,5,15492,United Kingdom +581492,12/9/2019,46776F,Woven Rose Garden Cushion Cover,7.24,6,15492,United Kingdom +581492,12/9/2019,47310M,Small Pop Box Funky Monkey,6.19,1,15492,United Kingdom +581492,12/9/2019,47480,Hanging Photo Clip Rope Ladder,6.19,3,15492,United Kingdom +581492,12/9/2019,47504H,English Rose Spirit Level,7.24,1,15492,United Kingdom +581492,12/9/2019,47559B,Tea Time Oven Glove,7.24,3,15492,United Kingdom +581492,12/9/2019,47563A,Retro Longboard Ironing Board Cover,7.24,2,15492,United Kin +581492,12/9/2019,47566,Party Bunting,7.24,2,15492,United Kingdom +581492,12/9/2019,47590B,Pink Happy Birthday Bunting,7.24,1,15492,United Kingdom +581492,12/9/2019,51014A,Feather Pen Hot Pink,7.24,2,15492,United Kingdom +581492,12/9/2019,51014L,Feather Pen Light Pink,7.24,1,15492,United Kingdom +581492,12/9/2019,71270,Photo Clip Line,7.24,1,15492,United Kingdom +581492,12/9/2019,72349B,Set/6 Purple Butterfly T-Lights,6.39,1,15492,United Kingdom +581492,12/9/2019,72741,Grand Chocolatecandle,7.24,3,15492,United Kingdom +581492,12/9/2019,72800E,4 Ivory Dinner Candles Silver Flock,7.24,3,15492,United Kin +581492,12/9/2019,79321,Chilli Lights,6.19,1,15492,United Kingdom +581492,12/9/2019,82484,Wood Black Board Ant White Finish,6.19,2,15492,United Kingdo +581492,12/9/2019,82567,Airline Lounge Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,82582,Area Patrolled Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,82600,N0 Singing Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,84031A,Charlie+Lola Red Hot Water Bottle,6.19,4,15492,United Kingd +581492,12/9/2019,84077,World War 2 Gliders Asstd Designs,6.19,1,15492,United Kingdo +581492,12/9/2019,84249A,Greeting Card Square Doughnuts,6.19,1,15492,United Kingdom +581492,12/9/2019,84347,Rotating Silver Angels T-Light Hldr,6.19,2,15492,United King +581492,12/9/2019,84375,Set Of 20 Kids Cookie Cutters,6.19,1,15492,United Kingdom +581492,12/9/2019,84378,Set Of 3 Heart Cookie Cutters,6.19,4,15492,United Kingdom +581492,12/9/2019,84519B,Carrot Charlie+Lola Coaster Set,6.19,1,15492,United Kingdom +581492,12/9/2019,84559A,3d Sheet Of Dog Stickers,6.19,2,15492,United Kingdom +581492,12/9/2019,84568,Girls Alphabet Iron On Patches,6.19,31,15492,United Kingdom +581492,12/9/2019,84569D,Pack 6 Heart/Ice-Cream Patches,6.19,1,15492,United Kingdom +581492,12/9/2019,84596B,Small Dolly Mix Design Orange Bowl,6.19,4,15492,United King +581492,12/9/2019,84596F,Small Marshmallows Pink Bowl,6.19,4,15492,United Kingdom +581492,12/9/2019,84598,Boys Alphabet Iron On Patches,6.19,5,15492,United Kingdom +581492,12/9/2019,84692,Box Of 24 Cocktail Parasols,6.19,1,15492,United Kingdom +581492,12/9/2019,84755,Colour Glass T-Light Holder Hanging,6.19,4,15492,United King +581492,12/9/2019,84828,Jungle Popsicles Ice Lolly Moulds,6.19,1,15492,United Kingdo +581492,12/9/2019,84879,Assorted Colour Bird Ornament,6.19,16,15492,United Kingdom +581492,12/9/2019,84923,Pink Butterfly Handbag W Bobbles,6.04,3,15492,United Kingdom +581492,12/9/2019,84946,Antique Silver T-Light Glass,6.04,18,15492,United Kingdom +581492,12/9/2019,84970S,Hanging Heart Zinc T-Light Holder,6.04,3,15492,United Kingd +581492,12/9/2019,84978,Hanging Heart Jar T-Light Holder,6.04,4,15492,United Kingdom +581492,12/9/2019,84991,60 Teatime Fairy Cake Cases,6.04,1,15492,United Kingdom +581492,12/9/2019,84997A,Childrens Cutlery Polkadot Green,6.04,1,15492,United Kingdo +581492,12/9/2019,84997B,Childrens Cutlery Retrospot Red,6.04,1,15492,United Kingdom +581492,12/9/2019,20733,Gold Mini Tape Measure,6.04,3,15492,United Kingdom +581492,12/9/2019,20777,Chrysanthemum Notebook,6.19,2,15492,United Kingdom +581492,12/9/2019,20782,Camouflage Ear Muff Headphones,6.19,1,15492,United Kingdom +581492,12/9/2019,20914,Set/5 Red Retrospot Lid Glass Bowls,6.19,2,15492,United King +581492,12/9/2019,20931,Blue Pot Plant Candle,6.19,1,15492,United Kingdom +581492,12/9/2019,20970,Pink Floral Feltcraft Shoulder Bag,6.19,1,15492,United Kingd +581492,12/9/2019,20971,Pink Blue Felt Craft Trinket Box,6.19,2,15492,United Kingdom +581492,12/9/2019,20972,Pink Cream Felt Craft Trinket Box,6.19,3,15492,United Kingdo +581492,12/9/2019,20973,12 Pencil Small Tube Woodland,6.19,4,15492,United Kingdom +581492,12/9/2019,20978,36 Pencils Tube Skulls,6.19,1,15492,United Kingdom +581492,12/9/2019,20979,36 Pencils Tube Red Retrospot,6.19,3,15492,United Kingdom +581492,12/9/2019,20982,12 Pencils Tall Tube Skulls,6.19,2,15492,United Kingdom +581492,12/9/2019,20983,12 Pencils Tall Tube Red Retrospot,6.19,4,15492,United Kingd +581492,12/9/2019,20985,Heart Calculator,6.19,1,15492,United Kingdom +581492,12/9/2019,20986,Blue Calculator Ruler,6.19,1,15492,United Kingdom +581492,12/9/2019,21000,Rose Du Sud Cosmetics Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,21012,Antique All Glass Candlestick,6.19,3,15492,United Kingdom +581492,12/9/2019,21015,Dark Bird House Tree Decoration,6.19,8,15492,United Kingdom +581492,12/9/2019,21026,Space Owl,6.19,1,15492,United Kingdom +581492,12/9/2019,21035,Set/2 Red Retrospot Tea Towels,6.19,1,15492,United Kingdom +581492,12/9/2019,21064,Boom Box Speaker Boys,6.19,4,15492,United Kingdom +581492,12/9/2019,21065,Boom Box Speaker Girls,6.19,2,15492,United Kingdom +581492,12/9/2019,21098,Christmas Toilet Roll,6.19,1,15492,United Kingdom +581492,12/9/2019,21123,Set/10 Ivory Polkadot Party Candles,6.19,1,15492,United King +581492,12/9/2019,21125,Set 6 Football Celebration Candles,6.19,1,15492,United Kingd +581492,12/9/2019,21126,Set Of 6 Girls Celebration Candles,6.19,1,15492,United Kingd +581492,12/9/2019,21137,Black Record Cover Frame,6.19,17,15492,United Kingdom +581492,12/9/2019,21154,Red Retrospot Oven Glove,6.19,2,15492,United Kingdom +581492,12/9/2019,21158,Moody Girl Door Hanger,6.19,1,15492,United Kingdom +581492,12/9/2019,21162,Toxic Area Door Hanger,6.19,1,15492,United Kingdom +581492,12/9/2019,21166,Cook With Wine Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,21172,Party Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,21174,Pottering In The Shed Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,21175,Gin And Tonic Diet Metal Sign,6.19,2,15492,United Kingdom +581492,12/9/2019,21200,Multicolour Honeycomb Paper Garland,6.04,6,15492,United King +581492,12/9/2019,21201,Tropical Honeycomb Paper Garland,6.04,4,15492,United Kingdom +581492,12/9/2019,21212,Pack Of 72 Retrospot Cake Cases,6.04,2,15492,United Kingdom +581492,12/9/2019,21213,Pack Of 72 Skull Cake Cases,6.04,2,15492,United Kingdom +581492,12/9/2019,21220,Set/4 Badges Dogs,6.19,2,15492,United Kingdom +581492,12/9/2019,21224,Set/4 Skull Badges,6.19,1,15492,United Kingdom +581492,12/9/2019,21232,Strawberry Ceramic Trinket Pot,6.19,1,15492,United Kingdom +581492,12/9/2019,21257,Victorian Sewing Box Medium,6.19,2,15492,United Kingdom +581492,12/9/2019,21258,Victorian Sewing Box Large,6.19,1,15492,United Kingdom +581492,12/9/2019,21259,Victorian Sewing Box Small,6.19,1,15492,United Kingdom +581492,12/9/2019,21313,Glass Heart T-Light Holder,6.19,1,15492,United Kingdom +581492,12/9/2019,21314,Small Glass Heart Trinket Pot,6.19,2,15492,United Kingdom +581492,12/9/2019,21328,Balloons Writing Set,6.19,2,15492,United Kingdom +581492,12/9/2019,21329,Dinosaurs Writing Set,6.19,2,15492,United Kingdom +581492,12/9/2019,21356,Toast Its - Fairy Flower,6.19,16,15492,United Kingdom +581492,12/9/2019,21378,Small Tall Camphor Wood Toadstool,6.19,2,15492,United Kingdo +581492,12/9/2019,21379,Camphor Wood Portobello Mushroom,6.19,1,15492,United Kingdom +581492,12/9/2019,21383,Pack Of 12 Sticky Bunnies,6.19,1,15492,United Kingdom +581492,12/9/2019,21402,Red Egg Spoon,6.19,1,15492,United Kingdom +581492,12/9/2019,21408,Spotty Pink Duck Doorstop,6.19,2,15492,United Kingdom +581492,12/9/2019,21411,Gingham Heart Doorstop Red,6.19,1,15492,United Kingdom +581492,12/9/2019,21467,Cherry Crochet Food Cover,6.19,1,15492,United Kingdom +581492,12/9/2019,21471,Strawberry Raffia Food Cover,6.19,2,15492,United Kingdom +581492,12/9/2019,21479,White Skull Hot Water Bottle,6.19,2,15492,United Kingdom +581492,12/9/2019,21481,Fawn Blue Hot Water Bottle,6.19,3,15492,United Kingdom +581492,12/9/2019,21484,Chick Grey Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,21485,Retrospot Heart Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,21494,Rotating Leaves T-Light Holder,6.19,11,15492,United Kingdom +581492,12/9/2019,21506,Fancy Font Birthday Card,6.19,3,15492,United Kingdom +581492,12/9/2019,21507,Elephant Birthday Card,7.24,1,15492,United Kingdom +581492,12/9/2019,21508,Vintage Kid Dolly Card,7.24,1,15492,United Kingdom +581492,12/9/2019,21509,Cowboys And Indians Birthday Card,7.24,2,15492,United Kingdo +581492,12/9/2019,21528,Dairy Maid Traditional Teapot,7.24,1,15492,United Kingdom +581492,12/9/2019,21544,Skulls Water Transfer Tattoos,7.24,2,15492,United Kingdom +581492,12/9/2019,21558,Skull Lunch Box With Cutlery,6.39,1,15492,United Kingdom +581492,12/9/2019,21559,Strawberry Lunch Box With Cutlery,7.24,1,15492,United Kingdo +581492,12/9/2019,21615,4 Lavender Botanical Dinner Candles,7.24,2,15492,United King +581492,12/9/2019,21616,4 Pear Botanical Dinner Candles,7.24,6,15492,United Kingdom +581492,12/9/2019,21620,Set Of 4 Rose Botanical Candles,7.24,5,15492,United Kingdom +581492,12/9/2019,21642,Assorted Tutti Frutti Pen,7.24,4,15492,United Kingdom +581492,12/9/2019,21648,Assorted Tutti Frutti Small Purse,7.24,2,15492,United Kingdo +581492,12/9/2019,21650,Assorted Tutti Frutti Bracelet,7.24,14,15492,United Kingdom +581492,12/9/2019,21675,Butterflies Stickers,6.19,2,15492,United Kingdom +581492,12/9/2019,21677,Hearts Stickers,6.19,1,15492,United Kingdom +581492,12/9/2019,21678,Paisley Pattern Stickers,6.19,2,15492,United Kingdom +581492,12/9/2019,21680,Woodland Stickers,6.19,1,15492,United Kingdom +581492,12/9/2019,21703,Bag 125g Swirly Marbles,6.19,1,15492,United Kingdom +581492,12/9/2019,21704,Bag 250g Swirly Marbles,6.19,1,15492,United Kingdom +581492,12/9/2019,21716,Boys Vintage Tin Seaside Bucket,6.19,1,15492,United Kingdom +581492,12/9/2019,21718,Red Metal Beach Spade,6.19,1,15492,United Kingdom +581492,12/9/2019,21724,Panda And Bunnies Sticker Sheet,6.19,1,15492,United Kingdom +581492,12/9/2019,21731,Red Toadstool Led Night Light,6.19,6,15492,United Kingdom +581492,12/9/2019,21739,Cosy Slipper Shoes Small Green,6.19,2,15492,United Kingdom +581492,12/9/2019,21774,Decorative Cats Bathroom Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,21786,Polkadot Rain Hat,6.19,3,15492,United Kingdom +581492,12/9/2019,21787,Rain Poncho Retrospot,6.19,2,15492,United Kingdom +581492,12/9/2019,21790,Vintage Snap Cards,6.19,5,15492,United Kingdom +581492,12/9/2019,21791,Vintage Heads And Tails Card Game,6.19,1,15492,United Kingdo +581492,12/9/2019,21808,Christmas Garland Stars Trees,6.19,3,15492,United Kingdom +581492,12/9/2019,21810,Christmas Hanging Star With Bell,6.19,25,15492,United Kingdo +581492,12/9/2019,21812,Garland With Hearts And Bells,6.19,7,15492,United Kingdom +581492,12/9/2019,21813,Garland With Stars And Bells,6.19,3,15492,United Kingdom +581492,12/9/2019,21822,Glitter Christmas Tree With Bells,6.19,12,15492,United Kingd +581492,12/9/2019,21828,Eight Piece Snake Set,6.19,1,15492,United Kingdom +581492,12/9/2019,21832,Chocolate Calculator,6.19,1,15492,United Kingdom +581492,12/9/2019,21833,Camouflage Led Torch,6.04,2,15492,United Kingdom +581492,12/9/2019,21871,Save The Planet Mug,6.19,1,15492,United Kingdom +581492,12/9/2019,21874,Gin And Tonic Mug,6.19,2,15492,United Kingdom +581492,12/9/2019,21876,Pottering Mug,6.19,4,15492,United Kingdom +581492,12/9/2019,21879,Hearts Gift Tape,6.19,1,15492,United Kingdom +581492,12/9/2019,21889,Wooden Box Of Dominoes,6.19,3,15492,United Kingdom +581492,12/9/2019,21890,S/6 Wooden Skittles In Cotton Bag,6.19,1,15492,United Kingdo +581492,12/9/2019,21892,Traditional Wooden Catch Cup Game,6.19,1,15492,United Kingdo +581492,12/9/2019,21900,Key Fob Shed,6.19,4,15492,United Kingdom +581492,12/9/2019,21901,Key Fob Back Door,6.19,2,15492,United Kingdom +581492,12/9/2019,21902,Key Fob Front Door,6.19,1,15492,United Kingdom +581492,12/9/2019,21905,More Butter Metal Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,21906,Pharmacie First Aid Tin,6.19,1,15492,United Kingdom +581492,12/9/2019,21914,Blue Harmonica In Box,6.19,2,15492,United Kingdom +581492,12/9/2019,21916,Set 12 Retro White Chalk Sticks,6.19,2,15492,United Kingdom +581492,12/9/2019,21930,Jumbo Storage Bag Skulls,6.19,1,15492,United Kingdom +581492,12/9/2019,21931,Jumbo Storage Bag Suki,6.19,1,15492,United Kingdom +581492,12/9/2019,21932,Scandinavian Paisley Picnic Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,21934,Skull Shoulder Bag,6.19,3,15492,United Kingdom +581492,12/9/2019,21935,Suki Shoulder Bag,6.19,9,15492,United Kingdom +581492,12/9/2019,21936,Red Retrospot Picnic Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,21942,Skulls Design Flannel,6.19,2,15492,United Kingdom +581492,12/9/2019,21945,Strawberries Design Flannel,6.19,2,15492,United Kingdom +581492,12/9/2019,21947,Set Of 6 Heart Chopsticks,6.19,1,15492,United Kingdom +581492,12/9/2019,21949,Set Of 6 Strawberry Chopsticks,6.19,2,15492,United Kingdom +581492,12/9/2019,21967,Pack Of 12 Skull Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,21976,Pack Of 60 Mushroom Cake Cases,6.19,3,15492,United Kingdom +581492,12/9/2019,21977,Pack Of 60 Pink Paisley Cake Cases,6.19,2,15492,United Kingd +581492,12/9/2019,21980,Pack Of 12 Red Retrospot Tissues,6.19,2,15492,United Kingdom +581492,12/9/2019,21981,Pack Of 12 Woodland Tissues,6.19,4,15492,United Kingdom +581492,12/9/2019,21982,Pack Of 12 Suki Tissues,6.19,2,15492,United Kingdom +581492,12/9/2019,21983,Pack Of 12 Blue Paisley Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,21984,Pack Of 12 Pink Paisley Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,21986,Pack Of 12 Pink Polkadot Tissues,6.19,3,15492,United Kingdom +581492,12/9/2019,21989,Pack Of 20 Skull Paper Napkins,6.19,2,15492,United Kingdom +581492,12/9/2019,21990,Modern Floral Stationery Set,6.19,8,15492,United Kingdom +581492,12/9/2019,21993,Floral Folk Stationery Set,6.19,8,15492,United Kingdom +581492,12/9/2019,22024,Rainy Ladies Birthday Card,6.19,3,15492,United Kingdom +581492,12/9/2019,22025,Ring Of Roses Birthday Card,6.19,3,15492,United Kingdom +581492,12/9/2019,22026,Banquet Birthday Card,6.19,4,15492,United Kingdom +581492,12/9/2019,22027,Tea Party Birthday Card,6.19,2,15492,United Kingdom +581492,12/9/2019,22028,Penny Farthing Birthday Card,6.19,1,15492,United Kingdom +581492,12/9/2019,22029,Spaceboy Birthday Card,6.19,4,15492,United Kingdom +581492,12/9/2019,22031,Botanical Lavender Birthday Card,6.19,1,15492,United Kingdom +581492,12/9/2019,22037,Robot Birthday Card,6.19,4,15492,United Kingdom +581492,12/9/2019,22064,Pink Doughnut Trinket Pot,6.19,1,15492,United Kingdom +581492,12/9/2019,22065,Christmas Pudding Trinket Pot,6.19,2,15492,United Kingdom +581492,12/9/2019,22068,Black Pirate Treasure Chest,6.19,1,15492,United Kingdom +581492,12/9/2019,22070,Small Red Retrospot Mug In Box,6.19,3,15492,United Kingdom +581492,12/9/2019,22071,Small White Retrospot Mug In Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22074,6 Ribbons Shimmering Pinks,6.19,1,15492,United Kingdom +581492,12/9/2019,22075,6 Ribbons Elegant Christmas,6.19,8,15492,United Kingdom +581492,12/9/2019,22076,6 Ribbons Empire,6.19,4,15492,United Kingdom +581492,12/9/2019,22083,Paper Chain Kit Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22085,Paper Chain Kit Skulls,6.19,1,15492,United Kingdom +581492,12/9/2019,22086,Paper Chain Kit 50'S Christmas,6.19,38,15492,United Kingdom +581492,12/9/2019,22091,Empire Tissue Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22099,Caravan Square Tissue Box,6.19,3,15492,United Kingdom +581492,12/9/2019,22104,Mirror Mosaic Candle Plate,6.19,4,15492,United Kingdom +581492,12/9/2019,22106,Mirror Mosaic Hurricane Lamp,6.19,1,15492,United Kingdom +581492,12/9/2019,22107,Pizza Plate In Box,6.19,10,15492,United Kingdom +581492,12/9/2019,22108,Ping! Microwave Plate,6.04,2,15492,United Kingdom +581492,12/9/2019,22109,Full English Breakfast Plate,6.04,2,15492,United Kingdom +581492,12/9/2019,22110,Bird House Hot Water Bottle,6.04,3,15492,United Kingdom +581492,12/9/2019,22111,Scottie Dog Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,22112,Chocolate Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,6.19,4,15492,United Kingdo +581492,12/9/2019,22116,Metal Sign His Dinner Is Served,6.19,2,15492,United Kingdom +581492,12/9/2019,22121,Noel Wooden Block Letters,6.19,1,15492,United Kingdom +581492,12/9/2019,22123,Ping Microwave Apron,6.19,1,15492,United Kingdom +581492,12/9/2019,22124,Set Of 2 Tea Towels Ping Microwave,6.19,1,15492,United Kingd +581492,12/9/2019,22129,Party Cones Candy Decoration,6.19,3,15492,United Kingdom +581492,12/9/2019,22134,Mini Ladle Love Heart Red,6.19,1,15492,United Kingdom +581492,12/9/2019,15036,Assorted Colours Silk Fan,6.19,1,15492,United Kingdom +581492,12/9/2019,15039,Sandalwood Fan,6.19,1,15492,United Kingdom +581492,12/9/2019,15058C,Ice Cream Design Garden Parasol,6.19,1,15492,United Kingdom +581492,12/9/2019,16218,Cartoon Pencil Sharpeners,6.19,5,15492,United Kingdom +581492,12/9/2019,16225,Rattle Snake Eggs,6.19,1,15492,United Kingdom +581492,12/9/2019,16235,Recycled Pencil With Rabbit Eraser,6.19,3,15492,United Kingd +581492,12/9/2019,16237,Sleeping Cat Erasers,6.19,1,15492,United Kingdom +581492,12/9/2019,17038,Porcelain Budah Incense Holder,6.19,1,15492,United Kingdom +581492,12/9/2019,17084N,Fairy Dreams Incense,6.19,1,15492,United Kingdom +581492,12/9/2019,20659,Economy Luggage Tag,6.19,7,15492,United Kingdom +581492,12/9/2019,20665,Red Retrospot Purse,6.19,2,15492,United Kingdom +581492,12/9/2019,20668,Disco Ball Christmas Decoration,6.19,1,15492,United Kingdom +581492,12/9/2019,20718,Red Retrospot Shopper Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,20719,Woodland Charlotte Bag,6.19,7,15492,United Kingdom +581492,12/9/2019,20723,Strawberry Charlotte Bag,6.19,2,15492,United Kingdom +581492,12/9/2019,20724,Red Retrospot Charlotte Bag,6.19,4,15492,United Kingdom +581492,12/9/2019,22135,Mini Ladle Love Heart Pink,6.19,6,15492,United Kingdom +581492,12/9/2019,22138,Baking Set 9 Piece Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22141,Christmas Craft Tree Top Angel,6.19,2,15492,United Kingdom +581492,12/9/2019,22150,3 Stripey Mice Feltcraft,7.24,2,15492,United Kingdom +581492,12/9/2019,22153,Angel Decoration Stars On Dress,7.24,2,15492,United Kingdom +581492,12/9/2019,22154,Angel Decoration 3 Buttons,7.24,3,15492,United Kingdom +581492,12/9/2019,22155,Star Decoration Rustic,7.24,7,15492,United Kingdom +581492,12/9/2019,22156,Heart Decoration With Pearls,7.24,4,15492,United Kingdom +581492,12/9/2019,22163,Heart String Memo Holder Hanging,7.24,9,15492,United Kingdom +581492,12/9/2019,22165,Diamante Heart Shaped Wall Mirror,7.24,1,15492,United Kingdo +581492,12/9/2019,22167,Oval Wall Mirror Diamante,7.24,1,15492,United Kingdom +581492,12/9/2019,22170,Picture Frame Wood Triple Portrait,7.24,1,15492,United Kingd +581492,12/9/2019,22173,Metal 4 Hook Hanger French Chateau,7.24,2,15492,United Kingd +581492,12/9/2019,22174,Photo Cube,6.19,7,15492,United Kingdom +581492,12/9/2019,22178,Victorian Glass Hanging T-Light,6.19,5,15492,United Kingdom +581492,12/9/2019,22186,Red Star Card Holder,7.24,2,15492,United Kingdom +581492,12/9/2019,22190,Local Cafe Mug,7.24,3,15492,United Kingdom +581492,12/9/2019,22193,Red Diner Wall Clock,7.24,1,15492,United Kingdom +581492,12/9/2019,22195,Large Heart Measuring Spoons,7.24,1,15492,United Kingdom +581492,12/9/2019,22196,Small Heart Measuring Spoons,7.24,11,15492,United Kingdom +581492,12/9/2019,22197,Popcorn Holder,7.24,34,15492,United Kingdom +581492,12/9/2019,22199,Frying Pan Red Retrospot,7.24,1,15492,United Kingdom +581492,12/9/2019,22209,Wood Stamp Set Happy Birthday,7.24,1,15492,United Kingdom +581492,12/9/2019,22212,Four Hook White Lovebirds,7.24,1,15492,United Kingdom +581492,12/9/2019,22219,Lovebird Hanging Decoration White,7.24,4,15492,United Kingdo +581492,12/9/2019,22224,White Lovebird Lantern,7.24,4,15492,United Kingdom +581492,12/9/2019,22227,Hanging Heart Mirror Decoration,7.24,1,15492,United Kingdom +581492,12/9/2019,22260,Felt Egg Cosy Blue Rabbit,6.39,2,15492,United Kingdom +581492,12/9/2019,22261,Felt Egg Cosy White Rabbit,7.24,1,15492,United Kingdom +581492,12/9/2019,22264,Felt Farm Animal White Bunny,7.24,1,15492,United Kingdom +581492,12/9/2019,22273,Feltcraft Doll Molly,7.24,2,15492,United Kingdom +581492,12/9/2019,22277,Cosmetic Bag Vintage Rose Paisley,7.24,1,15492,United Kingdo +581492,12/9/2019,22279,Pocket Bag Blue Paisley Red Spot,7.24,5,15492,United Kingdom +581492,12/9/2019,22280,Pocket Bag Pink Paisely Brown Spot,7.24,4,15492,United Kingd +581492,12/9/2019,22300,Coffee Mug Dog + Ball Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22301,Coffee Mug Cat + Bird Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22307,Gold Mug Bone China Tree Of Life,7.24,1,15492,United Kingdom +581492,12/9/2019,22308,Tea Cosy Blue Stripe,7.24,2,15492,United Kingdom +581492,12/9/2019,22309,Tea Cosy Red Stripe,7.24,2,15492,United Kingdom +581492,12/9/2019,22324,Blue Polkadot Kids Bag,7.24,2,15492,United Kingdom +581492,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,7.24,3,15492,United Kingd +581492,12/9/2019,22327,Round Snack Boxes Set Of 4 Skulls,7.24,2,15492,United Kingdo +581492,12/9/2019,22329,Round Container Set Of 5 Retrospot,6.19,2,15492,United Kingd +581492,12/9/2019,22338,Star Decoration Painted Zinc,6.19,4,15492,United Kingdom +581492,12/9/2019,22340,Noel Garland Painted Zinc,6.19,17,15492,United Kingdom +581492,12/9/2019,22342,Home Garland Painted Zinc,6.19,7,15492,United Kingdom +581492,12/9/2019,22348,Tea Bag Plate Red Retrospot,6.19,3,15492,United Kingdom +581492,12/9/2019,22355,Charlotte Bag Suki Design,6.19,3,15492,United Kingdom +581492,12/9/2019,22356,Charlotte Bag Pink Polkadot,6.19,1,15492,United Kingdom +581492,12/9/2019,22357,Kings Choice Biscuit Tin,6.19,2,15492,United Kingdom +581492,12/9/2019,22358,Kings Choice Tea Caddy,6.19,1,15492,United Kingdom +581492,12/9/2019,22361,Glass Jar Daisy Fresh Cotton Wool,6.19,2,15492,United Kingdo +581492,12/9/2019,22362,Glass Jar Peacock Bath Salts,6.19,2,15492,United Kingdom +581492,12/9/2019,22367,Childrens Apron Spaceboy Design,6.19,1,15492,United Kingdom +581492,12/9/2019,22372,Airline Bag Vintage World Champion,6.19,1,15492,United Kingd +581492,12/9/2019,22374,Airline Bag Vintage Jet Set Red,6.19,1,15492,United Kingdom +581492,12/9/2019,85014B,Red Retrospot Umbrella,6.19,1,15492,United Kingdom +581492,12/9/2019,85032C,Curious Images Gift Wrap Set,6.19,1,15492,United Kingdom +581492,12/9/2019,85038,6 Chocolate Love Heart T-Lights,6.19,1,15492,United Kingdom +581492,12/9/2019,85039A,Set/4 Red Mini Rose Candle In Bowl,6.19,5,15492,United King +581492,12/9/2019,85039B,S/4 Ivory Mini Rose Candle In Bowl,6.19,4,15492,United King +581492,12/9/2019,85040A,S/4 Pink Flower Candles In Bowl,6.19,1,15492,United Kingdom +581492,12/9/2019,85048,15cm Christmas Glass Ball 20 Lights,6.19,2,15492,United King +581492,12/9/2019,85049C,Romantic Pinks Ribbons,6.19,1,15492,United Kingdom +581492,12/9/2019,85049G,Chocolate Box Ribbons,6.19,1,15492,United Kingdom +581492,12/9/2019,85049H,Urban Black Ribbons,6.19,2,15492,United Kingdom +581492,12/9/2019,85053,French Enamel Candleholder,6.19,1,15492,United Kingdom +581492,12/9/2019,85059,French Enamel Water Basin,6.19,1,15492,United Kingdom +581492,12/9/2019,85066,Cream Sweetheart Mini Chest,6.19,1,15492,United Kingdom +581492,12/9/2019,85094,Candy Spot Egg Warmer Rabbit,6.19,2,15492,United Kingdom +581492,12/9/2019,85114C,Red Enchanted Forest Placemat,6.19,1,15492,United Kingdom +581492,12/9/2019,85123A,Cream Hanging Heart T-Light Holder,6.19,3,15492,United King +581492,12/9/2019,85131B,Beaded Crystal Heart Green On Stick,6.19,1,15492,United Kin +581492,12/9/2019,85131D,Beaded Crystal Heart Pink On Stick,6.19,2,15492,United King +581492,12/9/2019,85152,Hand Over The Chocolate Sign,6.19,1,15492,United Kingdom +581492,12/9/2019,85168B,Black Baroque Carriage Clock,6.19,3,15492,United Kingdom +581492,12/9/2019,85169C,Eau De Nil Love Bird Candle,6.19,1,15492,United Kingdom +581492,12/9/2019,85170C,Set/6 Eau De Nil Bird T-Lights,6.19,1,15492,United Kingdom +581492,12/9/2019,85173,Set/6 Frog Prince T-Light Candles,6.19,1,15492,United Kingdo +581492,12/9/2019,85177,Basket Of Flowers Sewing Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,85178,Victorian Sewing Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,85179A,Green Bitty Light Chain,6.19,4,15492,United Kingdom +581492,12/9/2019,85199S,Small Hanging Ivory/Red Wood Bird,6.19,9,15492,United Kingd +581492,12/9/2019,85227,Set Of 6 3d Kit Cards For Kids,6.19,3,15492,United Kingdom +581492,12/9/2019,90003C,Midnight Blue Pair Heart Hair Slide,6.19,1,15492,United Kin +581492,12/9/2019,90003E,Green Pair Heart Hair Slides,6.19,2,15492,United Kingdom +581492,12/9/2019,90010A,Midnight Blue Glass/Silver Bracelet,6.04,1,15492,United Kin +581492,12/9/2019,90013A,Midnight Blue Vintage Earrings,6.19,3,15492,United Kingdom +581492,12/9/2019,90013C,Green Vintage Earrings,6.19,2,15492,United Kingdom +581492,12/9/2019,90014A,Silver Mop Orbit Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90016B,Gold/Mop Pendant Orbit Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90018B,Gold Mop Orbit Drop Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90019B,Gold Mop Orbit Bracelet,6.19,1,15492,United Kingdom +581492,12/9/2019,90027D,Glass Bead Hoop Earrings Amethyst,6.19,1,15492,United Kingd +581492,12/9/2019,90030B,Red Kukui Coconut Seed Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90055,Cracked Glaze Earrings Brown,6.19,1,15492,United Kingdom +581492,12/9/2019,90059A,Diamante Hair Grip Pack/2 Crystal,6.19,1,15492,United Kingd +581492,12/9/2019,90059D,Diamante Hair Grip Pack/2 Peridot,6.19,2,15492,United Kingd +581492,12/9/2019,90059E,Diamante Hair Grip Pack/2 Ruby,6.04,1,15492,United Kingdom +581492,12/9/2019,90059F,Diamante Hair Grip Pack/2 Lt Rose,6.19,1,15492,United Kingd +581492,12/9/2019,90072,Ruby Drop Chandelier Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90086,Crystal Frog Phone Charm,6.19,1,15492,United Kingdom +581492,12/9/2019,90120B,Blue Murano Twist Bracelet,6.19,2,15492,United Kingdom +581492,12/9/2019,90120C,Green Murano Twist Bracelet,6.19,1,15492,United Kingdom +581492,12/9/2019,90130B,Turq Stone/Crystal Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90130C,Green Stone/Crystal Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90134,Old Rose Combo Bead Necklace,6.19,1,15492,United Kingdom +581492,12/9/2019,90138,White/Pink Mini Crystals Necklace,6.19,1,15492,United Kingdo +581492,12/9/2019,90141B,Ivory Pendant Triple Shell Necklace,6.19,1,15492,United Kin +581492,12/9/2019,90145,Silver Hoop Earrings With Flower,6.19,1,15492,United Kingdom +581492,12/9/2019,90155,Resin Necklace W Pastel Beads,6.19,1,15492,United Kingdom +581492,12/9/2019,90161D,Ant Copper Pink Boudicca Bracelet,6.19,1,15492,United Kingd +581492,12/9/2019,90163A,Pink Rosebud & Pearl Necklace,6.19,2,15492,United Kingdom +581492,12/9/2019,90165B,White Rosebud Pearl Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90168,2 Daisies Hair Comb,6.19,1,15492,United Kingdom +581492,12/9/2019,90169,Daisy Hair Comb,6.19,1,15492,United Kingdom +581492,12/9/2019,90170,Daisy Hair Band,6.19,1,15492,United Kingdom +581492,12/9/2019,90174,Butterfly Hair Band,6.19,2,15492,United Kingdom +581492,12/9/2019,90175A,White Glass Chunky Charm Bracelet,6.19,2,15492,United Kingd +581492,12/9/2019,90175D,Tigris Eye Chunky Charm Bracelet,6.19,1,15492,United Kingdo +581492,12/9/2019,90177A,Classic Diamante Earrings Jet,6.19,1,15492,United Kingdom +581492,12/9/2019,90177C,Drop Diamante Earrings Crystal,6.19,1,15492,United Kingdom +581492,12/9/2019,90181B,Amethyst Glass/Shell/Pearl Necklace,6.04,1,15492,United Kin +581492,12/9/2019,90182C,Black 3 Bead Drop Earrings,6.19,1,15492,United Kingdom +581492,12/9/2019,90183A,Amber Drop Earrings W Long Beads,6.19,2,15492,United Kingdo +581492,12/9/2019,90183C,Black Drop Earrings W Long Beads,6.19,1,15492,United Kingdo +581492,12/9/2019,90184C,Black Chunky Bead Bracelet W Strap,6.19,1,15492,United King +581492,12/9/2019,90185B,Amethyst Diamante Expandable Ring,6.19,1,15492,United Kingd +581492,12/9/2019,90188,Drop Earrings W Flower & Leaf,6.19,2,15492,United Kingdom +581492,12/9/2019,90191,Silver Lariat 40cm,6.19,2,15492,United Kingdom +581492,12/9/2019,90192,Jade Drop Earrings W Filigree,6.19,1,15492,United Kingdom +581492,12/9/2019,90198A,Vintage Rose Bead Bracelet Raspberr,6.19,2,15492,United Kin +581492,12/9/2019,90200A,Purple Sweetheart Bracelet,6.19,1,15492,United Kingdom +581492,12/9/2019,90201A,Purple Enamel Flower Ring,6.19,2,15492,United Kingdom +581492,12/9/2019,90201B,Black Enamel Flower Ring,6.19,1,15492,United Kingdom +581492,12/9/2019,90201C,Red Enamel Flower Ring,6.19,1,15492,United Kingdom +581492,12/9/2019,90201D,Green Enamel Flower Ring,6.19,1,15492,United Kingdom +581492,12/9/2019,90202C,Green Enamel Flower Hair Tie,6.19,1,15492,United Kingdom +581492,12/9/2019,90202D,Pink Enamel Flower Hair Tie,6.19,1,15492,United Kingdom +581492,12/9/2019,90206C,Crystal Diamante Star Brooch,6.19,1,15492,United Kingdom +581492,12/9/2019,90208,Pair Of Pink Flower Cluster Slide,6.19,1,15492,United Kingdo +581492,12/9/2019,90210A,Grey Acrylic Faceted Bangle,6.19,1,15492,United Kingdom +581492,12/9/2019,22378,Wall Tidy Retrospot,6.19,2,15492,United Kingdom +581492,12/9/2019,22379,Recycling Bag Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22380,Toy Tidy Spaceboy,6.19,2,15492,United Kingdom +581492,12/9/2019,22396,Magnets Pack Of 4 Retro Photo,6.19,1,15492,United Kingdom +581492,12/9/2019,22411,Jumbo Shopper Vintage Red Paisley,6.19,1,15492,United Kingdo +581492,12/9/2019,22418,10 Colour Spaceboy Pen,6.19,3,15492,United Kingdom +581492,12/9/2019,22419,Lipstick Pen Red,6.19,3,15492,United Kingdom +581492,12/9/2019,22420,Lipstick Pen Baby Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,22421,Lipstick Pen Fuschia,6.19,2,15492,United Kingdom +581492,12/9/2019,22422,Toothpaste Tube Pen,6.19,4,15492,United Kingdom +581492,12/9/2019,22437,Set Of 9 Black Skull Balloons,7.24,1,15492,United Kingdom +581492,12/9/2019,22441,Grow Your Own Basil In Enamel Mug,7.24,1,15492,United Kingdo +581492,12/9/2019,22446,Pin Cushion Babushka Pink,7.24,7,15492,United Kingdom +581492,12/9/2019,22457,Natural Slate Heart Chalkboard,7.24,1,15492,United Kingdom +581492,12/9/2019,22464,Hanging Metal Heart Lantern,7.24,1,15492,United Kingdom +581492,12/9/2019,22466,Fairy Tale Cottage Night Light,7.24,1,15492,United Kingdom +581492,12/9/2019,22467,Gumball Coat Rack,7.24,1,15492,United Kingdom +581492,12/9/2019,22478,Birdhouse Garden Marker,7.24,1,15492,United Kingdom +581492,12/9/2019,22486,Plasmatronic Lamp,7.24,1,15492,United Kingdom +581492,12/9/2019,22488,Natural Slate Rectangle Chalkboard,7.24,1,15492,United Kingd +581492,12/9/2019,22489,Pack Of 12 Traditional Crayons,7.24,5,15492,United Kingdom +581492,12/9/2019,22495,Set Of 2 Round Tins Camembert,7.24,1,15492,United Kingdom +581492,12/9/2019,22540,Mini Jigsaw Circus Parade,7.24,1,15492,United Kingdom +581492,12/9/2019,22544,Mini Jigsaw Spaceboy,7.24,1,15492,United Kingdom +581492,12/9/2019,22549,Picture Dominoes,7.24,3,15492,United Kingdom +581492,12/9/2019,22551,Plasters In Tin Spaceboy,7.24,2,15492,United Kingdom +581492,12/9/2019,22553,Plasters In Tin Skulls,7.24,2,15492,United Kingdom +581492,12/9/2019,22554,Plasters In Tin Woodland Animals,7.24,3,15492,United Kingdom +581492,12/9/2019,22555,Plasters In Tin Strongman,7.24,3,15492,United Kingdom +581492,12/9/2019,22557,Plasters In Tin Vintage Paisley,7.24,2,15492,United Kingdom +581492,12/9/2019,22558,Clothes Pegs Retrospot Pack 24,7.24,3,15492,United Kingdom +581492,12/9/2019,22560,Traditional Modelling Clay,7.24,5,15492,United Kingdom +581492,12/9/2019,22565,Feltcraft Hairbands Pink And White,7.24,4,15492,United Kingd +581492,12/9/2019,22566,Feltcraft Hairband Pink And Purple,7.24,3,15492,United Kingd +581492,12/9/2019,22571,Rocking Horse Red Christmas,7.24,4,15492,United Kingdom +581492,12/9/2019,22573,Star Wooden Christmas Decoration,6.39,2,15492,United Kingdom +581492,12/9/2019,22574,Heart Wooden Christmas Decoration,7.24,5,15492,United Kingdo +581492,12/9/2019,22576,Swallow Wooden Christmas Decoration,7.24,3,15492,United King +581492,12/9/2019,22577,Wooden Heart Christmas Scandinavian,7.24,4,15492,United King +581492,12/9/2019,22578,Wooden Star Christmas Scandinavian,7.24,19,15492,United King +581492,12/9/2019,22580,Advent Calendar Gingham Sack,7.24,9,15492,United Kingdom +581492,12/9/2019,22581,Wood Stocking Christmas Scandispot,7.24,11,15492,United King +581492,12/9/2019,22585,Pack Of 6 Birdy Gift Tags,7.24,2,15492,United Kingdom +581492,12/9/2019,22586,Feltcraft Hairband Pink And Blue,7.24,2,15492,United Kingdom +581492,12/9/2019,22589,Cardholder Gingham Star,7.24,6,15492,United Kingdom +581492,12/9/2019,22591,Cardholder Gingham Christmas Tree,7.24,1,15492,United Kingdo +581492,12/9/2019,22592,Cardholder Holly Wreath Metal,7.24,3,15492,United Kingdom +581492,12/9/2019,22593,Christmas Gingham Star,6.39,2,15492,United Kingdom +581492,12/9/2019,22594,Christmas Gingham Tree,7.24,3,15492,United Kingdom +581492,12/9/2019,22597,Musical Zinc Heart Decoration,7.24,9,15492,United Kingdom +581492,12/9/2019,22598,Christmas Musical Zinc Tree,7.24,4,15492,United Kingdom +581492,12/9/2019,22599,Christmas Musical Zinc Star,6.19,5,15492,United Kingdom +581492,12/9/2019,22600,Christmas Retrospot Star Wood,6.19,2,15492,United Kingdom +581492,12/9/2019,22601,Christmas Retrospot Angel Wood,6.19,3,15492,United Kingdom +581492,12/9/2019,22602,Retrospot Wooden Heart Decoration,6.19,3,15492,United Kingdo +581492,12/9/2019,22608,Pens Assorted Funky Jeweled,6.19,3,15492,United Kingdom +581492,12/9/2019,22614,Pack Of 12 Spaceboy Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,22615,Pack Of 12 Circus Parade Tissues,6.19,4,15492,United Kingdom +581492,12/9/2019,22616,Pack Of 12 London Tissues,6.19,1,15492,United Kingdom +581492,12/9/2019,22619,Set Of 6 Soldier Skittles,6.19,4,15492,United Kingdom +581492,12/9/2019,22620,4 Traditional Spinning Tops,6.19,3,15492,United Kingdom +581492,12/9/2019,22621,Traditional Knitting Nancy,6.19,2,15492,United Kingdom +581492,12/9/2019,22629,Spaceboy Lunch Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22630,Dolly Girl Lunch Box,6.19,1,15492,United Kingdom +581492,12/9/2019,22632,Hand Warmer Red Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,22633,Hand Warmer Union Jack,6.19,1,15492,United Kingdom +581492,12/9/2019,22634,Childs Breakfast Set Spaceboy,6.19,1,15492,United Kingdom +581492,12/9/2019,22650,Ceramic Pirate Chest Money Bank,6.19,2,15492,United Kingdom +581492,12/9/2019,22651,Gentleman Shirt Repair Kit,6.19,1,15492,United Kingdom +581492,12/9/2019,22653,Button Box,6.19,16,15492,United Kingdom +581492,12/9/2019,22654,Deluxe Sewing Kit,6.19,2,15492,United Kingdom +581492,12/9/2019,22659,Lunch Box I Love London,6.19,1,15492,United Kingdom +581492,12/9/2019,22666,Recipe Box Pantry Yellow Design,6.19,5,15492,United Kingdom +581492,12/9/2019,22693,Grow A Flytrap Or Sunflower In Tin,6.19,3,15492,United Kingd +581492,12/9/2019,22694,Wicker Star,6.19,1,15492,United Kingdom +581492,12/9/2019,22697,Green Regency Teacup And Saucer,6.19,1,15492,United Kingdom +581492,12/9/2019,22699,Roses Regency Teacup And Saucer,6.19,2,15492,United Kingdom +581492,12/9/2019,22701,Pink Dog Bowl,6.19,5,15492,United Kingdom +581492,12/9/2019,22703,Pink Cat Bowl,6.19,6,15492,United Kingdom +581492,12/9/2019,22712,Card Dolly Girl,6.19,1,15492,United Kingdom +581492,12/9/2019,22713,Card I Love London,6.19,1,15492,United Kingdom +581492,12/9/2019,22714,Card Birthday Cowboy,6.19,2,15492,United Kingdom +581492,12/9/2019,22716,Card Circus Parade,6.19,3,15492,United Kingdom +581492,12/9/2019,22717,Card Dog And Ball,6.19,1,15492,United Kingdom +581492,12/9/2019,22720,Set Of 3 Cake Tins Pantry Design,6.19,2,15492,United Kingdom +581492,12/9/2019,22725,Alarm Clock Bakelike Chocolate,6.19,1,15492,United Kingdom +581492,12/9/2019,22726,Alarm Clock Bakelike Green,6.19,4,15492,United Kingdom +581492,12/9/2019,22727,Alarm Clock Bakelike Red,6.19,4,15492,United Kingdom +581492,12/9/2019,22730,Alarm Clock Bakelike Ivory,6.19,1,15492,United Kingdom +581492,12/9/2019,22741,Funky Diva Pen,6.19,4,15492,United Kingdom +581492,12/9/2019,22745,Poppy's Playhouse Bedroom,6.19,2,15492,United Kingdom +581492,12/9/2019,22748,Poppy's Playhouse Kitchen,6.19,1,15492,United Kingdom +581492,12/9/2019,22749,Feltcraft Princess Charlotte Doll,6.19,1,15492,United Kingdo +581492,12/9/2019,22751,Feltcraft Princess Olivia Doll,6.19,2,15492,United Kingdom +581492,12/9/2019,22753,Small Yellow Babushka Notebook,6.19,1,15492,United Kingdom +581492,12/9/2019,22754,Small Red Babushka Notebook,6.19,1,15492,United Kingdom +581492,12/9/2019,22757,Large Red Babushka Notebook,6.19,3,15492,United Kingdom +581492,12/9/2019,22758,Large Purple Babushka Notebook,6.19,2,15492,United Kingdom +581492,12/9/2019,22763,Key Cabinet Ma Campagne,6.19,1,15492,United Kingdom +581492,12/9/2019,22768,Family Photo Frame Cornice,6.19,2,15492,United Kingdom +581492,12/9/2019,22776,Sweetheart 3 Tier Cake Stand,6.19,1,15492,United Kingdom +581492,12/9/2019,22795,Sweetheart Recipe Book Stand,6.19,1,15492,United Kingdom +581492,12/9/2019,22798,Antique Glass Dressing Table Pot,6.19,3,15492,United Kingdom +581492,12/9/2019,22800,Antique Tall Swirlglass Trinket Pot,6.19,3,15492,United King +581492,12/9/2019,22809,Set Of 6 T-Lights Santa,6.19,1,15492,United Kingdom +581492,12/9/2019,22811,Set Of 6 T-Lights Cacti,6.19,1,15492,United Kingdom +581492,12/9/2019,22814,Card Party Games,6.19,9,15492,United Kingdom +581492,12/9/2019,22815,Card Psychedelic Apples,6.19,18,15492,United Kingdom +581492,12/9/2019,22816,Card Motorbike Santa,6.19,2,15492,United Kingdom +581492,12/9/2019,22817,Card Suki Birthday,6.19,5,15492,United Kingdom +581492,12/9/2019,22818,Card Christmas Village,7.24,6,15492,United Kingdom +581492,12/9/2019,22819,Birthday Card Retro Spot,7.24,3,15492,United Kingdom +581492,12/9/2019,22835,Hot Water Bottle I Am So Poorly,7.24,3,15492,United Kingdom +581492,12/9/2019,22865,Hand Warmer Owl Design,7.24,2,15492,United Kingdom +581492,12/9/2019,22866,Hand Warmer Scotty Dog Design,7.24,3,15492,United Kingdom +581492,12/9/2019,22867,Hand Warmer Bird Design,7.24,4,15492,United Kingdom +581492,12/9/2019,22881,Number Tile Vintage Font 2,7.24,1,15492,United Kingdom +581492,12/9/2019,22885,Number Tile Vintage Font 6,7.24,1,15492,United Kingdom +581492,12/9/2019,22888,Number Tile Vintage Font 9,7.24,1,15492,United Kingdom +581492,12/9/2019,22890,Novelty Biscuits Cake Stand 3 Tier,7.24,1,15492,United Kingd +581492,12/9/2019,22898,Childrens Apron Apples Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22899,Children's Apron Dolly Girl,7.24,2,15492,United Kingdom +581492,12/9/2019,22900,Set 2 Tea Towels I Love London,7.24,3,15492,United Kingdom +581492,12/9/2019,22905,Calendar In Season Design,7.24,1,15492,United Kingdom +581492,12/9/2019,22907,Pack Of 20 Napkins Pantry Design,6.39,1,15492,United Kingdom +581492,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,7.24,5,15492,United King +581492,12/9/2019,22910,Paper Chain Kit Vintage Christmas,7.24,7,15492,United Kingdo +581492,12/9/2019,22914,Blue Coat Rack Paris Fashion,7.24,1,15492,United Kingdom +581492,12/9/2019,22922,Fridge Magnets Us Diner Assorted,7.24,6,15492,United Kingdom +581492,12/9/2019,22924,Fridge Magnets La Vie En Rose,7.24,6,15492,United Kingdom +581492,12/9/2019,22928,Yellow Giant Garden Thermometer,7.24,1,15492,United Kingdom +581492,12/9/2019,22940,Feltcraft Christmas Fairy,6.19,1,15492,United Kingdom +581492,12/9/2019,22941,Christmas Lights 10 Reindeer,6.19,2,15492,United Kingdom +581492,12/9/2019,22943,Christmas Lights 10 Vintage Baubles,6.19,1,15492,United King +581492,12/9/2019,22948,Metal Decoration Naughty Children,6.19,4,15492,United Kingdo +581492,12/9/2019,22951,60 Cake Cases Dolly Girl Design,6.19,2,15492,United Kingdom +581492,12/9/2019,22952,60 Cake Cases Vintage Christmas,6.19,2,15492,United Kingdom +581492,12/9/2019,22955,36 Foil Star Cake Cases,6.19,1,15492,United Kingdom +581492,12/9/2019,22960,Jam Making Set With Jars,6.19,2,15492,United Kingdom +581492,12/9/2019,22961,Jam Making Set Printed,6.19,9,15492,United Kingdom +581492,12/9/2019,22962,Jam Jar With Pink Lid,6.19,3,15492,United Kingdom +581492,12/9/2019,22963,Jam Jar With Green Lid,6.19,3,15492,United Kingdom +581492,12/9/2019,22964,3 Piece Spaceboy Cookie Cutter Set,6.19,1,15492,United Kingd +581492,12/9/2019,22965,3 Traditional Biscuit Cutters Set,6.19,1,15492,United Kingdo +581492,12/9/2019,22966,Gingerbread Man Cookie Cutter,6.19,4,15492,United Kingdom +581492,12/9/2019,22969,Homemade Jam Scented Candles,6.19,6,15492,United Kingdom +581492,12/9/2019,22975,Spaceboy Childrens Egg Cup,6.19,2,15492,United Kingdom +581492,12/9/2019,22976,Circus Parade Childrens Egg Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,22977,Dolly Girl Childrens Egg Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,22979,Pantry Washing Up Brush,6.19,2,15492,United Kingdom +581492,12/9/2019,22980,Pantry Scrubbing Brush,6.19,1,15492,United Kingdom +581492,12/9/2019,22982,Pantry Pastry Brush,6.19,3,15492,United Kingdom +581492,12/9/2019,22983,Card Billboard Font,6.19,1,15492,United Kingdom +581492,12/9/2019,22984,Card Gingham Rose,6.19,1,15492,United Kingdom +581492,12/9/2019,22988,Soldiers Egg Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,22989,Set 2 Pantry Design Tea Towels,6.19,2,15492,United Kingdom +581492,12/9/2019,22992,Revolver Wooden Ruler,6.19,3,15492,United Kingdom +581492,12/9/2019,22993,Set Of 4 Pantry Jelly Moulds,6.19,3,15492,United Kingdom +581492,12/9/2019,22995,Travel Card Wallet Suki,6.19,4,15492,United Kingdom +581492,12/9/2019,22996,Travel Card Wallet Vintage Ticket,6.19,5,15492,United Kingdo +581492,12/9/2019,22997,Travel Card Wallet Union Jack,6.19,1,15492,United Kingdom +581492,12/9/2019,22998,Travel Card Wallet Keep Calm,6.19,4,15492,United Kingdom +581492,12/9/2019,22999,Travel Card Wallet Vintage Leaf,6.19,1,15492,United Kingdom +581492,12/9/2019,23005,Travel Card Wallet I Love London,6.19,4,15492,United Kingdom +581492,12/9/2019,23007,Spaceboy Baby Gift Set,6.19,1,15492,United Kingdom +581492,12/9/2019,23009,I Love London Baby Gift Set,6.19,2,15492,United Kingdom +581492,12/9/2019,23012,Glass Apothecary Bottle Perfume,6.19,1,15492,United Kingdom +581492,12/9/2019,23013,Glass Apothecary Bottle Tonic,6.19,2,15492,United Kingdom +581492,12/9/2019,23014,Glass Apothecary Bottle Elixir,6.19,1,15492,United Kingdom +581492,12/9/2019,23050,Recycled Acapulco Mat Green,6.19,1,15492,United Kingdom +581492,12/9/2019,23051,Recycled Acapulco Mat Blue,6.19,1,15492,United Kingdom +581492,12/9/2019,23052,Recycled Acapulco Mat Turquoise,6.19,5,15492,United Kingdom +581492,12/9/2019,23053,Recycled Acapulco Mat Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23054,Recycled Acapulco Mat Lavender,6.19,1,15492,United Kingdom +581492,12/9/2019,23074,Embossed Heart Trinket Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23076,Ice Cream Sundae Lip Gloss,6.19,3,15492,United Kingdom +581492,12/9/2019,23077,Doughnut Lip Gloss,6.19,3,15492,United Kingdom +581492,12/9/2019,23082,Set 6 Paper Table Lantern Hearts,6.19,1,15492,United Kingdom +581492,12/9/2019,23083,Set 6 Paper Table Lantern Stars,6.19,1,15492,United Kingdom +581492,12/9/2019,23084,Rabbit Night Light,6.19,57,15492,United Kingdom +581492,12/9/2019,23088,Zinc Heart Flower T-Light Holder,6.19,1,15492,United Kingdom +581492,12/9/2019,23089,Glass Bon Bon Jar,6.19,4,15492,United Kingdom +581492,12/9/2019,23093,Small Parisienne Heart Photo Frame,6.19,3,15492,United Kingd +581492,12/9/2019,23103,Jingle Bell Heart Decoration,6.19,2,15492,United Kingdom +581492,12/9/2019,23110,Parisienne Key Cabinet,6.19,1,15492,United Kingdom +581492,12/9/2019,23111,Parisienne Sewing Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23112,Parisienne Curio Cabinet,6.19,1,15492,United Kingdom +581492,12/9/2019,23118,Parisienne Jewellery Drawer,6.19,1,15492,United Kingdom +581492,12/9/2019,23158,Set Of 5 Lucky Cat Magnets,6.19,1,15492,United Kingdom +581492,12/9/2019,23165,Large Ceramic Top Storage Jar,6.19,2,15492,United Kingdom +581492,12/9/2019,23166,Medium Ceramic Top Storage Jar,6.19,2,15492,United Kingdom +581492,12/9/2019,23168,Classic Cafe Sugar Dispenser,6.19,2,15492,United Kingdom +581492,12/9/2019,23171,Regency Tea Plate Green,6.19,1,15492,United Kingdom +581492,12/9/2019,23172,Regency Tea Plate Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23173,Regency Teapot Roses,6.19,1,15492,United Kingdom +581492,12/9/2019,23175,Regency Milk Jug Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23177,Treasure Island Book Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23183,Mother's Kitchen Spoon Rest,6.19,3,15492,United Kingdom +581492,12/9/2019,23184,Bull Dog Bottle Opener,6.19,2,15492,United Kingdom +581492,12/9/2019,23188,Vintage 2 Metre Folding Ruler,6.19,1,15492,United Kingdom +581492,12/9/2019,23191,Bundle Of 3 Retro Note Books,6.19,1,15492,United Kingdom +581492,12/9/2019,23194,Gymkhana Treasure Book Box,6.19,1,15492,United Kingdom +581492,12/9/2019,23196,Vintage Leaf Magnetic Notepad,6.19,1,15492,United Kingdom +581492,12/9/2019,23197,Sketchbook Magnetic Shopping List,6.19,2,15492,United Kingdo +581492,12/9/2019,23198,Pantry Magnetic Shopping List,6.19,2,15492,United Kingdom +581492,12/9/2019,23204,Charlotte Bag Apples Design,6.19,6,15492,United Kingdom +581492,12/9/2019,23205,Charlotte Bag Vintage Alphabet,6.19,3,15492,United Kingdom +581492,12/9/2019,23212,Heart Wreath Decoration With Bell,6.19,7,15492,United Kingdo +581492,12/9/2019,23213,Star Wreath Decoration With Bell,6.19,7,15492,United Kingdom +581492,12/9/2019,23220,Reindeer Heart Decoration Gold,6.19,2,15492,United Kingdom +581492,12/9/2019,23224,Cherub Heart Decoration Gold,6.19,1,15492,United Kingdom +581492,12/9/2019,23229,Vintage Donkey Tail Game,6.19,2,15492,United Kingdom +581492,12/9/2019,23234,Biscuit Tin Vintage Christmas,6.19,4,15492,United Kingdom +581492,12/9/2019,23241,Treasure Tin Gymkhana Design,6.19,3,15492,United Kingdom +581492,12/9/2019,23243,Set Of Tea Coffee Sugar Tins Pantry,6.19,1,15492,United King +581492,12/9/2019,23245,Set Of 3 Regency Cake Tins,6.19,7,15492,United Kingdom +581492,12/9/2019,23247,Biscuit Tin 50'S Christmas,6.19,3,15492,United Kingdom +581492,12/9/2019,23264,Set Of 3 Wooden Sleigh Decorations,6.19,3,15492,United Kingd +581492,12/9/2019,23265,Set Of 3 Wooden Tree Decorations,6.19,3,15492,United Kingdom +581492,12/9/2019,23266,Set Of 3 Wooden Stocking Decoration,6.19,2,15492,United King +581492,12/9/2019,23274,Star T-Light Holder Willie Winkie,6.19,3,15492,United Kingdo +581492,12/9/2019,23275,Set Of 3 Hanging Owls Ollie Beak,6.19,2,15492,United Kingdom +581492,12/9/2019,23280,Folding Butterfly Mirror Hot Pink,6.19,2,15492,United Kingdo +581492,12/9/2019,23284,Doormat Keep Calm And Come In,6.19,1,15492,United Kingdom +581492,12/9/2019,23285,Pink Vintage Spot Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23287,Red Vintage Spot Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23290,Spaceboy Childrens Bowl,6.19,1,15492,United Kingdom +581492,12/9/2019,23292,Spaceboy Childrens Cup,6.19,1,15492,United Kingdom +581492,12/9/2019,23293,Set Of 12 Fairy Cake Baking Cases,6.19,3,15492,United Kingdo +581492,12/9/2019,23294,Set Of 6 Snack Loaf Baking Cases,6.19,1,15492,United Kingdom +581492,12/9/2019,23295,Set Of 12 Mini Loaf Baking Cases,6.19,4,15492,United Kingdom +581492,12/9/2019,23298,Spotty Bunting,6.19,2,15492,United Kingdom +581492,12/9/2019,23299,Food Cover With Beads Set 2,6.19,1,15492,United Kingdom +581492,12/9/2019,23300,Gardeners Kneeling Pad Cup Of Tea,6.19,3,15492,United Kingdo +581492,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,6,15492,United Kingdom +581492,12/9/2019,23307,Set Of 60 Pantry Design Cake Cases,6.19,7,15492,United Kingd +581492,12/9/2019,23308,Set Of 60 Vintage Leaf Cake Cases,6.19,1,15492,United Kingdo +581492,12/9/2019,23309,Set Of 60 I Love London Cake Cases,6.19,2,15492,United Kingd +581492,12/9/2019,23310,Bubblegum Ring Assorted,6.19,4,15492,United Kingdom +581492,12/9/2019,23311,Vintage Christmas Stocking,6.19,1,15492,United Kingdom +581492,12/9/2019,23312,Vintage Christmas Gift Sack,6.19,6,15492,United Kingdom +581492,12/9/2019,23313,Vintage Christmas Bunting,6.19,4,15492,United Kingdom +581492,12/9/2019,23314,Vintage Christmas Tablecloth,6.19,1,15492,United Kingdom +581492,12/9/2019,23319,Box Of 6 Mini 50'S Crackers,6.19,3,15492,United Kingdom +581492,12/9/2019,23322,Large White Heart Of Wicker,6.19,1,15492,United Kingdom +581492,12/9/2019,23323,White Wicker Star,6.19,6,15492,United Kingdom +581492,12/9/2019,23328,Set 6 School Milk Bottles In Crate,6.19,5,15492,United Kingd +581492,12/9/2019,23332,Ivory Wicker Heart Large,6.19,1,15492,United Kingdom +581492,12/9/2019,23334,Ivory Wicker Heart Small,6.19,1,15492,United Kingdom +581492,12/9/2019,23336,Egg Frying Pan Pink,6.19,1,15492,United Kingdom +581492,12/9/2019,23340,Vintage Christmas Cake Frill,6.19,3,15492,United Kingdom +581492,12/9/2019,23345,Dolly Girl Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23347,I Love London Beaker,6.19,1,15492,United Kingdom +581492,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,2,15492,United Kingdom +581492,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,5,15492,United Kingdom +581492,12/9/2019,23352,Roll Wrap 50'S Red Christmas,6.19,1,15492,United Kingdom +581492,12/9/2019,23353,6 Gift Tags Vintage Christmas,6.19,3,15492,United Kingdom +581492,12/9/2019,23354,6 Gift Tags 50'S Christmas,6.19,4,15492,United Kingdom +581492,12/9/2019,23357,Hot Water Bottle Sex Bomb,6.19,1,15492,United Kingdom +581492,12/9/2019,23358,Hot Stuff Hot Water Bottle,6.19,1,15492,United Kingdom +581492,12/9/2019,23365,Set 12 Colour Pencils Love London,6.19,1,15492,United Kingdo +581492,12/9/2019,23366,Set 12 Colouring Pencils Doily,6.04,5,15492,United Kingdom +581492,12/9/2019,23367,Set 12 Colour Pencils Spaceboy,6.04,10,15492,United Kingdom +581492,12/9/2019,23368,Set 12 Colour Pencils Dolly Girl,6.19,2,15492,United Kingdom +581492,12/9/2019,23581,Jumbo Bag Paisley Park,6.19,3,15492,United Kingdom +581492,12/9/2019,21928,Jumbo Bag Scandinavian Blue Paisley,6.19,2,15492,United King +581492,12/9/2019,21929,Jumbo Bag Pink Vintage Paisley,6.19,1,15492,United Kingdom +581492,12/9/2019,85099C,Jumbo Bag Baroque Black White,6.19,7,15492,United Kingdom +581492,12/9/2019,23199,Jumbo Bag Apples,6.19,2,15492,United Kingdom +581492,12/9/2019,23200,Jumbo Bag Pears,6.19,1,15492,United Kingdom +581492,12/9/2019,23202,Jumbo Bag Vintage Leaf,6.19,2,15492,United Kingdom +581492,12/9/2019,23343,Jumbo Bag Vintage Christmas,6.19,4,15492,United Kingdom +581492,12/9/2019,23344,Jumbo Bag 50'S Christmas,6.19,5,15492,United Kingdom +581492,12/9/2019,23583,Lunch Bag Paisley Park,6.19,2,15492,United Kingdom +581492,12/9/2019,20727,Lunch Bag Black Skull,6.04,2,15492,United Kingdom +581492,12/9/2019,20728,Lunch Bag Cars Blue,6.04,1,15492,United Kingdom +581492,12/9/2019,20725,Lunch Bag Red Retrospot,6.19,1,15492,United Kingdom +581492,12/9/2019,20726,Lunch Bag Woodland,6.19,1,15492,United Kingdom +581492,12/9/2019,22662,Lunch Bag Dolly Girl Design,6.19,1,15492,United Kingdom +581492,12/9/2019,23206,Lunch Bag Apple Design,6.19,3,15492,United Kingdom +581492,12/9/2019,23208,Lunch Bag Vintage Leaf Design,6.19,3,15492,United Kingdom +581492,12/9/2019,23375,50'S Christmas Paper Gift Bag,6.19,1,15492,United Kingdom +581492,12/9/2019,23437,50'S Christmas Gift Bag Large,6.19,1,15492,United Kingdom +581492,12/9/2019,22821,Gift Bag Psychedelic Apples,7.24,3,15492,United Kingdom +581493,12/9/2019,79190B,Retro Plastic Polka Tray,7.24,15,12423,Belgium +581493,12/9/2019,79190A,Retro Plastic 70'S Tray,7.24,15,12423,Belgium +581493,12/9/2019,22915,Assorted Bottle Top Magnets,7.24,12,12423,Belgium +581493,12/9/2019,22151,Place Setting White Heart,7.24,24,12423,Belgium +581493,12/9/2019,22632,Hand Warmer Red Retrospot,7.24,12,12423,Belgium +581493,12/9/2019,22865,Hand Warmer Owl Design,7.24,12,12423,Belgium +581493,12/9/2019,20718,Red Retrospot Shopper Bag,7.24,10,12423,Belgium +581493,12/9/2019,79190B,Retro Plastic Polka Tray,7.24,12,12423,Belgium +581493,12/9/2019,71459,Hanging Jam Jar T-Light Holders,7.24,12,12423,Belgium +581493,12/9/2019,84945,Multi Colour Silver T-Light Holder,7.24,12,12423,Belgium +581493,12/9/2019,22356,Charlotte Bag Pink Polkadot,7.24,10,12423,Belgium +581493,12/9/2019,20724,Red Retrospot Charlotte Bag,7.24,10,12423,Belgium +581493,12/9/2019,23204,Charlotte Bag Apples Design,7.24,10,12423,Belgium +581493,12/9/2019,21108,Fairy Cake Flannel Assorted Colour,7.24,18,12423,Belgium +581493,12/9/2019,22252,Birdcage Decoration Tealight Holder,7.24,12,12423,Belgium +581493,12/9/2019,22807,Set Of 6 T-Lights Toadstools,6.19,6,12423,Belgium +581494,12/9/2019,23084,Rabbit Night Light,6.19,24,12518,Germany +581494,12/9/2019,21559,Strawberry Lunch Box With Cutlery,6.19,6,12518,Germany +581494,12/9/2019,21506,Fancy Font Birthday Card,6.19,12,12518,Germany +581494,12/9/2019,22716,Card Circus Parade,6.19,12,12518,Germany +581494,12/9/2019,22556,Plasters In Tin Circus Parade,6.19,12,12518,Germany +581494,12/9/2019,22328,Round Snack Boxes Set Of 4 Fruits,6.19,6,12518,Germany +581494,12/9/2019,22730,Alarm Clock Bakelike Ivory,6.19,4,12518,Germany +581494,12/9/2019,22633,Hand Warmer Union Jack,6.19,12,12518,Germany +581494,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,6.19,6,12518,Germany +581494,12/9/2019,10125,Mini Funky Design Tapes,6.19,20,12518,Germany +581494,12/9/2019,23367,Set 12 Colour Pencils Spaceboy,6.19,24,12518,Germany +581494,12/9/2019,22551,Plasters In Tin Spaceboy,6.04,12,12518,Germany +581494,12/9/2019,22554,Plasters In Tin Woodland Animals,6.04,12,12518,Germany +581494,12/9/2019,22549,Picture Dominoes,6.19,12,12518,Germany +581494,12/9/2019,23388,Woodland Mini Backpack,6.19,4,12518,Germany +581494,12/9/2019,23390,Dolly Girl Mini Backpack,6.19,4,12518,Germany +581494,12/9/2019,23389,Spaceboy Mini Backpack,6.19,4,12518,Germany +581495,12/9/2019,23535,Wall Art Bicycle Safety,6.19,12,14051,United Kingdom +581495,12/9/2019,22699,Roses Regency Teacup And Saucer,6.19,12,14051,United Kingdom +581495,12/9/2019,22698,Pink Regency Teacup And Saucer,6.19,12,14051,United Kingdom +581495,12/9/2019,22697,Green Regency Teacup And Saucer,6.19,12,14051,United Kingdom +581495,12/9/2019,22633,Hand Warmer Union Jack,6.04,18,14051,United Kingdom +581495,12/9/2019,15056N,Edwardian Parasol Natural,6.19,36,14051,United Kingdom +581495,12/9/2019,23439,Hand Warmer Red Love Heart,6.19,36,14051,United Kingdom +581495,12/9/2019,48138,Doormat Union Flag,6.19,10,14051,United Kingdom +581495,12/9/2019,21523,Doormat Fancy Font Home Sweet Home,6.19,10,14051,United King +581495,12/9/2019,21877,Home Sweet Home Mug,6.19,12,14051,United Kingdom +581495,12/9/2019,21871,Save The Planet Mug,6.19,18,14051,United Kingdom +581495,12/9/2019,22798,Antique Glass Dressing Table Pot,6.19,36,14051,United Kingdo +581495,12/9/2019,23173,Regency Teapot Roses,6.19,6,14051,United Kingdom +581495,12/9/2019,22423,Regency Cakestand 3 Tier,6.19,10,14051,United Kingdom +581496,12/9/2019,22664,Toy Tidy Dolly Girl Design,6.19,20,16558,United Kingdom +581496,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,6.19,12,16558,United Kingdom +581496,12/9/2019,72800E,4 Ivory Dinner Candles Silver Flock,6.19,12,16558,United Ki +581496,12/9/2019,23313,Vintage Christmas Bunting,6.19,10,16558,United Kingdom +581496,12/9/2019,22952,60 Cake Cases Vintage Christmas,6.19,24,16558,United Kingdom +581496,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,6.04,12,16558,United Kin +581496,12/9/2019,23298,Spotty Bunting,6.19,3,16558,United Kingdom +581496,12/9/2019,23598,Paper Bunting Vintage Party,6.19,6,16558,United Kingdom +581496,12/9/2019,16169E,Wrap 50'S Christmas,6.19,25,16558,United Kingdom +581496,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,12,16558,United Kingdom +581496,12/9/2019,23350,Roll Wrap Vintage Spot,6.19,24,16558,United Kingdom +581496,12/9/2019,22865,Hand Warmer Owl Design,6.19,24,16558,United Kingdom +581496,12/9/2019,22632,Hand Warmer Red Retrospot,6.19,12,16558,United Kingdom +581496,12/9/2019,22835,Hot Water Bottle I Am So Poorly,6.19,4,16558,United Kingdom +581496,12/9/2019,22112,Chocolate Hot Water Bottle,6.19,6,16558,United Kingdom +581496,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,4,16558,United Kingdom +581496,12/9/2019,22314,Office Mug Warmer Choc+Blue,6.19,24,16558,United Kingdom +581496,12/9/2019,22313,Office Mug Warmer Pink,6.19,12,16558,United Kingdom +581496,12/9/2019,23084,Rabbit Night Light,6.19,18,16558,United Kingdom +581496,12/9/2019,20831,Gold Photo Frame,6.19,12,16558,United Kingdom +581496,12/9/2019,21115,Rose Caravan Doorstop,6.19,8,16558,United Kingdom +581496,12/9/2019,21462,Nursery A B C Painted Letters,7.24,8,16558,United Kingdom +581496,12/9/2019,22076,6 Ribbons Empire,7.24,24,16558,United Kingdom +581496,12/9/2019,22190,Local Cafe Mug,7.24,24,16558,United Kingdom +581496,12/9/2019,22215,Cake Stand White Two Tier Lace,7.24,6,16558,United Kingdom +581496,12/9/2019,22220,Cake Stand Lovebird 2 Tier White,7.24,6,16558,United Kingdom +581496,12/9/2019,22464,Hanging Metal Heart Lantern,7.24,12,16558,United Kingdom +581496,12/9/2019,22465,Hanging Metal Star Lantern,7.24,12,16558,United Kingdom +581496,12/9/2019,22539,Mini Jigsaw Dolly Girl,7.24,48,16558,United Kingdom +581496,12/9/2019,22544,Mini Jigsaw Spaceboy,7.24,48,16558,United Kingdom +581496,12/9/2019,23438,Red Spot Gift Bag Large,6.19,12,16558,United Kingdom +581496,12/9/2019,23437,50'S Christmas Gift Bag Large,6.19,12,16558,United Kingdom +581497,12/9/2019,20719,Woodland Charlotte Bag,6.19,33,17497,United Kingdom +581497,12/9/2019,20723,Strawberry Charlotte Bag,6.19,42,17497,United Kingdom +581497,12/9/2019,20724,Red Retrospot Charlotte Bag,6.19,55,17497,United Kingdom +581497,12/9/2019,21212,Pack Of 72 Retrospot Cake Cases,7.24,7,17497,United Kingdom +581497,12/9/2019,21238,Red Retrospot Cup,7.24,8,17497,United Kingdom +581497,12/9/2019,21242,Red Retrospot Plate,7.24,2,17497,United Kingdom +581497,12/9/2019,21479,White Skull Hot Water Bottle,7.24,25,17497,United Kingdom +581497,12/9/2019,21481,Fawn Blue Hot Water Bottle,7.24,1,17497,United Kingdom +581497,12/9/2019,21481,Fawn Blue Hot Water Bottle,7.24,1,17497,United Kingdom +581497,12/9/2019,21484,Chick Grey Hot Water Bottle,7.24,1,17497,United Kingdom +581497,12/9/2019,21668,Red Stripe Ceramic Drawer Knob,7.24,1,17497,United Kingdom +581497,12/9/2019,21670,Blue Spot Ceramic Drawer Knob,7.24,1,17497,United Kingdom +581497,12/9/2019,21671,Red Spot Ceramic Drawer Knob,7.24,1,17497,United Kingdom +581497,12/9/2019,21672,White Spot Red Ceramic Drawer Knob,7.24,6,17497,United Kingd +581497,12/9/2019,21673,White Spot Blue Ceramic Drawer Knob,7.24,1,17497,United King +581497,12/9/2019,21714,Citronella Candle Garden Pot,7.24,2,17497,United Kingdom +581497,12/9/2019,22110,Bird House Hot Water Bottle,7.24,7,17497,United Kingdom +581497,12/9/2019,22111,Scottie Dog Hot Water Bottle,7.24,5,17497,United Kingdom +581497,12/9/2019,22112,Chocolate Hot Water Bottle,7.24,13,17497,United Kingdom +581497,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,7.24,48,17497,United Kingd +581497,12/9/2019,22197,Popcorn Holder,7.24,68,17497,United Kingdom +581497,12/9/2019,22355,Charlotte Bag Suki Design,7.24,110,17497,United Kingdom +581497,12/9/2019,22356,Charlotte Bag Pink Polkadot,7.24,25,17497,United Kingdom +581497,12/9/2019,22356,Charlotte Bag Pink Polkadot,7.24,1,17497,United Kingdom +581497,12/9/2019,22423,Regency Cakestand 3 Tier,7.24,8,17497,United Kingdom +581497,12/9/2019,22725,Alarm Clock Bakelike Chocolate,7.24,3,17497,United Kingdom +581497,12/9/2019,22726,Alarm Clock Bakelike Green,7.24,1,17497,United Kingdom +581497,12/9/2019,22727,Alarm Clock Bakelike Red,7.24,10,17497,United Kingdom +581497,12/9/2019,22730,Alarm Clock Bakelike Ivory,7.24,5,17497,United Kingdom +581497,12/9/2019,22735,Ribbon Reel Socks And Mittens,7.24,2,17497,United Kingdom +581497,12/9/2019,22736,Ribbon Reel Making Snowmen,7.24,3,17497,United Kingdom +581497,12/9/2019,22738,Ribbon Reel Snowy Village,7.24,1,17497,United Kingdom +581497,12/9/2019,22805,Blue Drawer Knob Acrylic Edwardian,7.24,1,17497,United Kingd +581497,12/9/2019,22835,Hot Water Bottle I Am So Poorly,7.24,4,17497,United Kingdom +581497,12/9/2019,22895,Set Of 2 Tea Towels Apple And Pears,7.24,1,17497,United King +581497,12/9/2019,22896,Peg Bag Apples Design,7.24,2,17497,United Kingdom +581497,12/9/2019,22898,Childrens Apron Apples Design,7.24,2,17497,United Kingdom +581497,12/9/2019,22943,Christmas Lights 10 Vintage Baubles,7.24,1,17497,United King +581497,12/9/2019,23084,Rabbit Night Light,6.19,37,17497,United Kingdom +581497,12/9/2019,23168,Classic Cafe Sugar Dispenser,6.19,1,17497,United Kingdom +581497,12/9/2019,23204,Charlotte Bag Apples Design,6.04,13,17497,United Kingdom +581497,12/9/2019,23205,Charlotte Bag Vintage Alphabet,6.04,8,17497,United Kingdom +581497,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,1,17497,United Kingdom +581497,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,36,17497,United Kingdom +581497,12/9/2019,23356,Love Hot Water Bottle,6.19,1,17497,United Kingdom +581497,12/9/2019,23357,Hot Water Bottle Sex Bomb,6.19,2,17497,United Kingdom +581497,12/9/2019,23358,Hot Stuff Hot Water Bottle,6.19,2,17497,United Kingdom +581497,12/9/2019,35970,Zinc Folkart Sleigh Bells,6.19,2,17497,United Kingdom +581497,12/9/2019,47566,Party Bunting,6.19,5,17497,United Kingdom +581497,12/9/2019,82583,Hot Baths Metal Sign,6.19,4,17497,United Kingdom +581497,12/9/2019,84029G,Knitted Union Flag Hot Water Bottle,6.19,11,17497,United Ki +581497,12/9/2019,84997C,Childrens Cutlery Polkadot Blue,6.19,3,17497,United Kingdom +581497,12/9/2019,85049G,Chocolate Box Ribbons,6.19,2,17497,United Kingdom +581497,12/9/2019,20727,Lunch Bag Black Skull,6.19,8,17497,United Kingdom +581497,12/9/2019,22383,Lunch Bag Suki Design,7.24,2,17497,United Kingdom +581497,12/9/2019,23206,Lunch Bag Apple Design,6.04,3,17497,United Kingdom +581497,12/9/2019,23206,Lunch Bag Apple Design,6.04,1,17497,United Kingdom +581497,12/9/2019,23207,Lunch Bag Alphabet Design,6.19,1,17497,United Kingdom +581497,12/9/2019,23208,Lunch Bag Vintage Leaf Design,6.19,3,17497,United Kingdom +581498,12/9/2019,20669,Red Heart Luggage Tag,6.19,3,14498,United Kingdom +581498,12/9/2019,20679,Edwardian Parasol Red,6.19,5,14498,United Kingdom +581498,12/9/2019,20717,Strawberry Shopper Bag,6.19,1,14498,United Kingdom +581498,12/9/2019,20718,Red Retrospot Shopper Bag,6.19,1,14498,United Kingdom +581498,12/9/2019,20750,Red Retrospot Mini Cases,6.19,1,14498,United Kingdom +581498,12/9/2019,20961,Strawberry Bath Sponge,6.19,1,14498,United Kingdom +581498,12/9/2019,20963,Apple Bath Sponge,6.19,1,14498,United Kingdom +581498,12/9/2019,21115,Rose Caravan Doorstop,6.19,1,14498,United Kingdom +581498,12/9/2019,21125,Set 6 Football Celebration Candles,6.19,3,14498,United Kingd +581498,12/9/2019,21137,Black Record Cover Frame,6.19,4,14498,United Kingdom +581498,12/9/2019,21155,Red Retrospot Peg Bag,6.19,4,14498,United Kingdom +581498,12/9/2019,21166,Cook With Wine Metal Sign,6.19,3,14498,United Kingdom +581498,12/9/2019,21169,You're Confusing Me Metal Sign,6.19,2,14498,United Kingdom +581498,12/9/2019,21174,Pottering In The Shed Metal Sign,6.19,2,14498,United Kingdom +581498,12/9/2019,21181,Please One Person Metal Sign,6.19,6,14498,United Kingdom +581498,12/9/2019,21216,Set 3 Retrospot Tea/Coffee/Sugar,6.19,2,14498,United Kingdom +581498,12/9/2019,21217,Red Retrospot Round Cake Tins,6.19,3,14498,United Kingdom +581498,12/9/2019,21218,Red Spotty Biscuit Tin,6.19,4,14498,United Kingdom +581498,12/9/2019,21232,Strawberry Ceramic Trinket Pot,6.19,13,14498,United Kingdom +581498,12/9/2019,21239,Pink Polkadot Cup,6.19,1,14498,United Kingdom +581498,12/9/2019,21257,Victorian Sewing Box Medium,6.19,3,14498,United Kingdom +581498,12/9/2019,21327,Skulls Writing Set,6.19,1,14498,United Kingdom +581498,12/9/2019,21328,Balloons Writing Set,6.19,1,14498,United Kingdom +581498,12/9/2019,21329,Dinosaurs Writing Set,6.19,1,14498,United Kingdom +581498,12/9/2019,21411,Gingham Heart Doorstop Red,6.19,1,14498,United Kingdom +581498,12/9/2019,21430,Set/3 Red Gingham Rose Storage Box,6.19,3,14498,United Kingd +581498,12/9/2019,21494,Rotating Leaves T-Light Holder,6.19,7,14498,United Kingdom +581498,12/9/2019,21523,Doormat Fancy Font Home Sweet Home,6.19,1,14498,United Kingd +581498,12/9/2019,21557,Set Of 6 Funky Beakers,6.19,1,14498,United Kingdom +581498,12/9/2019,21558,Skull Lunch Box With Cutlery,6.19,1,14498,United Kingdom +581498,12/9/2019,21731,Red Toadstool Led Night Light,6.19,9,14498,United Kingdom +581498,12/9/2019,21754,Home Building Block Word,6.19,2,14498,United Kingdom +581498,12/9/2019,21790,Vintage Snap Cards,6.19,1,14498,United Kingdom +581498,12/9/2019,21843,Red Retrospot Cake Stand,6.19,5,14498,United Kingdom +581498,12/9/2019,21864,Union Jack Flag Passport Cover,6.19,2,14498,United Kingdom +581498,12/9/2019,21874,Gin And Tonic Mug,6.19,2,14498,United Kingdom +581498,12/9/2019,21876,Pottering Mug,6.19,4,14498,United Kingdom +581498,12/9/2019,21890,S/6 Wooden Skittles In Cotton Bag,6.19,1,14498,United Kingdo +581498,12/9/2019,21912,Vintage Snakes & Ladders,6.19,1,14498,United Kingdom +581498,12/9/2019,21916,Set 12 Retro White Chalk Sticks,6.19,7,14498,United Kingdom +581498,12/9/2019,21930,Jumbo Storage Bag Skulls,6.19,2,14498,United Kingdom +581498,12/9/2019,21931,Jumbo Storage Bag Suki,6.19,6,14498,United Kingdom +581498,12/9/2019,21934,Skull Shoulder Bag,6.19,1,14498,United Kingdom +581498,12/9/2019,21935,Suki Shoulder Bag,6.19,7,14498,United Kingdom +581498,12/9/2019,21942,Skulls Design Flannel,6.19,1,14498,United Kingdom +581498,12/9/2019,21955,Doormat Union Jack Guns And Roses,6.19,1,14498,United Kingdo +581498,12/9/2019,21977,Pack Of 60 Pink Paisley Cake Cases,6.19,1,14498,United Kingd +581498,12/9/2019,21983,Pack Of 12 Blue Paisley Tissues,6.19,1,14498,United Kingdom +581498,12/9/2019,21987,Pack Of 6 Skull Paper Cups,6.19,2,14498,United Kingdom +581498,12/9/2019,21989,Pack Of 20 Skull Paper Napkins,6.19,1,14498,United Kingdom +581498,12/9/2019,22041,"Record Frame 7"" Single Size",6.19,2,14498,United Kingdom +581498,12/9/2019,22059,Ceramic Strawberry Design Mug,6.19,1,14498,United Kingdom +581498,12/9/2019,22069,Brown Pirate Treasure Chest,6.19,3,14498,United Kingdom +581498,12/9/2019,22077,6 Ribbons Rustic Charm,6.19,2,14498,United Kingdom +581498,12/9/2019,22083,Paper Chain Kit Retrospot,6.19,2,14498,United Kingdom +581498,12/9/2019,22086,Paper Chain Kit 50'S Christmas,6.19,52,14498,United Kingdom +581498,12/9/2019,22099,Caravan Square Tissue Box,6.19,2,14498,United Kingdom +581498,12/9/2019,22114,Hot Water Bottle Tea And Sympathy,6.19,4,14498,United Kingdo +581498,12/9/2019,22139,Retrospot Tea Set Ceramic 11 Pc,6.19,2,14498,United Kingdom +581498,12/9/2019,22141,Christmas Craft Tree Top Angel,6.19,7,14498,United Kingdom +581498,12/9/2019,22142,Christmas Craft White Fairy,6.19,2,14498,United Kingdom +581498,12/9/2019,22144,Christmas Craft Little Friends,6.19,8,14498,United Kingdom +581498,12/9/2019,22161,Heart Decoration Rustic Hanging,6.19,2,14498,United Kingdom +581498,12/9/2019,22173,Metal 4 Hook Hanger French Chateau,6.19,3,14498,United Kingd +581498,12/9/2019,22174,Photo Cube,6.19,3,14498,United Kingdom +581498,12/9/2019,22175,Pink Owl Soft Toy,6.19,1,14498,United Kingdom +581498,12/9/2019,22178,Victorian Glass Hanging T-Light,6.19,1,14498,United Kingdom +581498,12/9/2019,22179,Set 10 Night Owl Lights,6.19,1,14498,United Kingdom +581498,12/9/2019,22186,Red Star Card Holder,6.19,1,14498,United Kingdom +581498,12/9/2019,22187,Green Christmas Tree Card Holder,6.19,6,14498,United Kingdom +581498,12/9/2019,22193,Red Diner Wall Clock,6.19,1,14498,United Kingdom +581498,12/9/2019,22195,Large Heart Measuring Spoons,6.19,3,14498,United Kingdom +581498,12/9/2019,22196,Small Heart Measuring Spoons,6.19,2,14498,United Kingdom +581498,12/9/2019,22207,Frying Pan Union Flag,6.19,1,14498,United Kingdom +581498,12/9/2019,22278,Overnight Bag Vintage Rose Paisley,6.19,2,14498,United Kingd +581498,12/9/2019,22301,Coffee Mug Cat + Bird Design,6.19,2,14498,United Kingdom +581498,12/9/2019,22302,Coffee Mug Pears Design,6.19,4,14498,United Kingdom +581498,12/9/2019,22303,Coffee Mug Apples Design,6.19,4,14498,United Kingdom +581498,12/9/2019,22304,Coffee Mug Blue Paisley Design,6.19,1,14498,United Kingdom +581498,12/9/2019,22308,Tea Cosy Blue Stripe,6.19,2,14498,United Kingdom +581498,12/9/2019,22327,Round Snack Boxes Set Of 4 Skulls,6.19,4,14498,United Kingdo +581498,12/9/2019,22328,Round Snack Boxes Set Of 4 Fruits,6.19,4,14498,United Kingdo +581498,12/9/2019,22352,Lunch Box With Cutlery Retrospot,6.19,6,14498,United Kingdom +581498,12/9/2019,22357,Kings Choice Biscuit Tin,6.19,1,14498,United Kingdom +581498,12/9/2019,22358,Kings Choice Tea Caddy,6.19,1,14498,United Kingdom +581498,12/9/2019,22367,Childrens Apron Spaceboy Design,6.19,7,14498,United Kingdom +581498,12/9/2019,22371,Airline Bag Vintage Tokyo 78,6.19,1,14498,United Kingdom +581498,12/9/2019,22374,Airline Bag Vintage Jet Set Red,6.19,1,14498,United Kingdom +581498,12/9/2019,22378,Wall Tidy Retrospot,7.24,2,14498,United Kingdom +581498,12/9/2019,22379,Recycling Bag Retrospot,7.24,4,14498,United Kingdom +581498,12/9/2019,22381,Toy Tidy Pink Polkadot,7.24,1,14498,United Kingdom +581498,12/9/2019,22411,Jumbo Shopper Vintage Red Paisley,7.24,2,14498,United Kingdo +581498,12/9/2019,22422,Toothpaste Tube Pen,7.24,1,14498,United Kingdom +581498,12/9/2019,22424,Enamel Bread Bin Cream,7.24,1,14498,United Kingdom +581498,12/9/2019,22429,Enamel Measuring Jug Cream,7.24,1,14498,United Kingdom +581498,12/9/2019,22456,Natural Slate Chalkboard Large,7.24,4,14498,United Kingdom +581498,12/9/2019,22457,Natural Slate Heart Chalkboard,7.24,2,14498,United Kingdom +581498,12/9/2019,22471,Tv Dinner Tray Air Hostess,7.24,1,14498,United Kingdom +581498,12/9/2019,22474,Spaceboy Tv Dinner Tray,7.24,1,14498,United Kingdom +581498,12/9/2019,22488,Natural Slate Rectangle Chalkboard,7.24,2,14498,United Kingd +581498,12/9/2019,22498,Wooden Regatta Bunting,7.24,1,14498,United Kingdom +581498,12/9/2019,22507,Memo Board Retrospot Design,7.24,1,14498,United Kingdom +581498,12/9/2019,22526,Wheelbarrow For Children,7.24,1,14498,United Kingdom +581498,12/9/2019,22553,Plasters In Tin Skulls,7.24,1,14498,United Kingdom +581498,12/9/2019,22557,Plasters In Tin Vintage Paisley,7.24,1,14498,United Kingdom +581498,12/9/2019,22619,Set Of 6 Soldier Skittles,7.24,1,14498,United Kingdom +581498,12/9/2019,22622,Box Of Vintage Alphabet Blocks,7.24,1,14498,United Kingdom +581498,12/9/2019,22624,Ivory Kitchen Scales,7.24,1,14498,United Kingdom +581498,12/9/2019,22629,Spaceboy Lunch Box,7.24,2,14498,United Kingdom +581498,12/9/2019,22632,Hand Warmer Red Retrospot,7.24,1,14498,United Kingdom +581498,12/9/2019,22645,Ceramic Heart Fairy Cake Money Bank,7.24,4,14498,United King +581498,12/9/2019,22654,Deluxe Sewing Kit,6.19,2,14498,United Kingdom +581498,12/9/2019,22659,Lunch Box I Love London,6.19,1,14498,United Kingdom +581498,12/9/2019,22666,Recipe Box Pantry Yellow Design,6.19,11,14498,United Kingdom +581498,12/9/2019,22676,French Blue Metal Door Sign 1,6.04,1,14498,United Kingdom +581498,12/9/2019,22684,French Blue Metal Door Sign 9,6.04,1,14498,United Kingdom +581498,12/9/2019,22694,Wicker Star,6.04,1,14498,United Kingdom +581498,12/9/2019,22697,Green Regency Teacup And Saucer,6.04,6,14498,United Kingdom +581498,12/9/2019,22698,Pink Regency Teacup And Saucer,6.04,9,14498,United Kingdom +581498,12/9/2019,22699,Roses Regency Teacup And Saucer,6.19,6,14498,United Kingdom +581498,12/9/2019,22722,Set Of 6 Spice Tins Pantry Design,6.19,3,14498,United Kingdo +581498,12/9/2019,22733,3d Traditional Christmas Stickers,6.19,1,14498,United Kingdo +581498,12/9/2019,22734,Set Of 6 Ribbons Vintage Christmas,6.19,5,14498,United Kingd +581498,12/9/2019,22776,Sweetheart 3 Tier Cake Stand,6.19,1,14498,United Kingdom +581498,12/9/2019,22795,Sweetheart Recipe Book Stand,6.19,1,14498,United Kingdom +581498,12/9/2019,22807,Set Of 6 T-Lights Toadstools,6.19,2,14498,United Kingdom +581498,12/9/2019,22813,Pack 3 Boxes Bird Panettone,6.19,4,14498,United Kingdom +581498,12/9/2019,22838,3 Tier Cake Tin Red And Cream,6.19,1,14498,United Kingdom +581498,12/9/2019,22844,Vintage Cream Dog Food Container,6.19,2,14498,United Kingdom +581498,12/9/2019,22845,Vintage Cream Cat Food Container,6.19,1,14498,United Kingdom +581498,12/9/2019,22865,Hand Warmer Owl Design,6.19,1,14498,United Kingdom +581498,12/9/2019,22866,Hand Warmer Scotty Dog Design,6.19,2,14498,United Kingdom +581498,12/9/2019,22891,Tea For One Polkadot,6.19,1,14498,United Kingdom +581498,12/9/2019,22900,Set 2 Tea Towels I Love London,6.19,2,14498,United Kingdom +581498,12/9/2019,22910,Paper Chain Kit Vintage Christmas,6.19,5,14498,United Kingdo +581498,12/9/2019,22952,60 Cake Cases Vintage Christmas,6.19,7,14498,United Kingdom +581498,12/9/2019,22960,Jam Making Set With Jars,6.19,5,14498,United Kingdom +581498,12/9/2019,22961,Jam Making Set Printed,6.19,4,14498,United Kingdom +581498,12/9/2019,22966,Gingerbread Man Cookie Cutter,6.19,2,14498,United Kingdom +581498,12/9/2019,22993,Set Of 4 Pantry Jelly Moulds,6.19,2,14498,United Kingdom +581498,12/9/2019,23012,Glass Apothecary Bottle Perfume,6.19,1,14498,United Kingdom +581498,12/9/2019,23013,Glass Apothecary Bottle Tonic,6.19,1,14498,United Kingdom +581498,12/9/2019,23014,Glass Apothecary Bottle Elixir,7.24,2,14498,United Kingdom +581498,12/9/2019,23080,Red Metal Box Top Secret,7.24,5,14498,United Kingdom +581498,12/9/2019,23170,Regency Tea Plate Roses,7.24,4,14498,United Kingdom +581498,12/9/2019,23171,Regency Tea Plate Green,7.24,5,14498,United Kingdom +581498,12/9/2019,23172,Regency Tea Plate Pink,6.19,4,14498,United Kingdom +581498,12/9/2019,23181,Bull Dog Bottle Top Wall Clock,6.19,1,14498,United Kingdom +581498,12/9/2019,23196,Vintage Leaf Magnetic Notepad,6.19,1,14498,United Kingdom +581498,12/9/2019,23198,Pantry Magnetic Shopping List,6.19,2,14498,United Kingdom +581498,12/9/2019,23245,Set Of 3 Regency Cake Tins,6.19,5,14498,United Kingdom +581498,12/9/2019,23295,Set Of 12 Mini Loaf Baking Cases,6.19,1,14498,United Kingdom +581498,12/9/2019,23298,Spotty Bunting,6.19,1,14498,United Kingdom +581498,12/9/2019,23300,Gardeners Kneeling Pad Cup Of Tea,6.19,9,14498,United Kingdo +581498,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,32,14498,United Kingdo +581498,12/9/2019,23311,Vintage Christmas Stocking,6.19,6,14498,United Kingdom +581498,12/9/2019,23312,Vintage Christmas Gift Sack,6.19,3,14498,United Kingdom +581498,12/9/2019,23328,Set 6 School Milk Bottles In Crate,6.19,6,14498,United Kingd +581498,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,8,14498,United Kingdom +581498,12/9/2019,23351,Roll Wrap 50'S Christmas,6.19,22,14498,United Kingdom +581498,12/9/2019,23352,Roll Wrap 50'S Red Christmas,6.19,8,14498,United Kingdom +581498,12/9/2019,23353,6 Gift Tags Vintage Christmas,6.19,10,14498,United Kingdom +581498,12/9/2019,23354,6 Gift Tags 50'S Christmas,6.19,9,14498,United Kingdom +581498,12/9/2019,23368,Set 12 Colour Pencils Dolly Girl,6.19,3,14498,United Kingdom +581498,12/9/2019,23369,Set 36 Colour Pencils Love London,6.19,1,14498,United Kingdo +581498,12/9/2019,23370,Set 36 Colouring Pencils Doily,6.19,3,14498,United Kingdom +581498,12/9/2019,23382,Box Of 6 Christmas Cake Decorations,6.19,10,14498,United Kin +581498,12/9/2019,23388,Woodland Mini Backpack,6.19,1,14498,United Kingdom +581498,12/9/2019,23480,Mini Lights Woodland Mushrooms,7.24,1,14498,United Kingdom +581498,12/9/2019,23493,Vintage Doily Travel Sewing Kit,7.24,1,14498,United Kingdom +581498,12/9/2019,23494,Vintage Doily Deluxe Sewing Kit,7.24,1,14498,United Kingdom +581498,12/9/2019,23497,Classic Chrome Bicycle Bell,7.24,1,14498,United Kingdom +581498,12/9/2019,23501,Key Ring Baseball Boot Union Jack,7.24,1,14498,United Kingdo +581498,12/9/2019,23564,Egg Cup Milkmaid Ingrid,7.24,1,14498,United Kingdom +581498,12/9/2019,35970,Zinc Folkart Sleigh Bells,7.24,6,14498,United Kingdom +581498,12/9/2019,48138,Doormat Union Flag,7.24,1,14498,United Kingdom +581498,12/9/2019,71053,White Moroccan Metal Lantern,7.24,1,14498,United Kingdom +581498,12/9/2019,72349B,Set/6 Purple Butterfly T-Lights,7.24,2,14498,United Kingdom +581498,12/9/2019,79321,Chilli Lights,7.24,10,14498,United Kingdom +581498,12/9/2019,82001S,Silver Record Cover Frame,6.19,2,14498,United Kingdom +581498,12/9/2019,82482,Wooden Picture Frame White Finish,6.04,4,14498,United Kingdo +581498,12/9/2019,82552,Washroom Metal Sign,6.04,1,14498,United Kingdom +581498,12/9/2019,21171,Bathroom Metal Sign,6.19,1,14498,United Kingdom +581498,12/9/2019,82581,Toilet Metal Sign,6.19,1,14498,United Kingdom +581498,12/9/2019,82600,N0 Singing Metal Sign,6.19,4,14498,United Kingdom +581498,12/9/2019,84029E,Red Woolly Hottie White Heart,6.19,4,14498,United Kingdom +581498,12/9/2019,84032A,Charlie+Lola Pink Hot Water Bottle,6.19,4,14498,United King +581498,12/9/2019,84032B,Charlie + Lola Red Hot Water Bottle,6.19,3,14498,United Kin +581498,12/9/2019,84375,Set Of 20 Kids Cookie Cutters,6.19,3,14498,United Kingdom +581498,12/9/2019,84509A,Set Of 4 English Rose Placemats,6.19,1,14498,United Kingdom +581498,12/9/2019,84558A,3d Dog Picture Playing Cards,6.19,1,14498,United Kingdom +581498,12/9/2019,84832,Zinc Willie Winkie Candle Stick,6.19,26,14498,United Kingdom +581498,12/9/2019,84968E,Set Of 16 Vintage Black Cutlery,6.19,1,14498,United Kingdom +581498,12/9/2019,84970S,Hanging Heart Zinc T-Light Holder,6.19,1,14498,United Kingd +581498,12/9/2019,84997A,Childrens Cutlery Polkadot Green,6.19,2,14498,United Kingdo +581498,12/9/2019,84997B,Childrens Cutlery Retrospot Red,6.19,3,14498,United Kingdom +581498,12/9/2019,84997D,Childrens Cutlery Polkadot Pink,6.19,1,14498,United Kingdom +581498,12/9/2019,85038,6 Chocolate Love Heart T-Lights,6.19,1,14498,United Kingdom +581498,12/9/2019,85048,15cm Christmas Glass Ball 20 Lights,6.19,1,14498,United King +581498,12/9/2019,85049A,Traditional Christmas Ribbons,6.17,5,14498,United Kingdom +581498,12/9/2019,85049E,Scandinavian Reds Ribbons,6.19,4,14498,United Kingdom +581498,12/9/2019,85150,Ladies & Gentlemen Metal Sign,6.19,1,14498,United Kingdom +581498,12/9/2019,85174,S/4 Cacti Candles,6.19,1,14498,United Kingdom +581498,12/9/2019,20712,Jumbo Bag Woodland Animals,6.19,3,14498,United Kingdom +581498,12/9/2019,20713,Jumbo Bag Owls,6.19,8,14498,United Kingdom +581498,12/9/2019,21928,Jumbo Bag Scandinavian Blue Paisley,6.19,2,14498,United King +581498,12/9/2019,22386,Jumbo Bag Pink Polkadot,7.24,1,14498,United Kingdom +581498,12/9/2019,22663,Jumbo Bag Dolly Girl Design,6.19,2,14498,United Kingdom +581498,12/9/2019,23199,Jumbo Bag Apples,6.19,6,14498,United Kingdom +581498,12/9/2019,23201,Jumbo Bag Alphabet,6.19,6,14498,United Kingdom +581498,12/9/2019,85099B,Jumbo Bag Red Retrospot,6.19,5,14498,United Kingdom +581498,12/9/2019,85099C,Jumbo Bag Baroque Black White,6.19,4,14498,United Kingdom +581498,12/9/2019,20726,Lunch Bag Woodland,6.19,3,14498,United Kingdom +581498,12/9/2019,20728,Lunch Bag Cars Blue,6.19,4,14498,United Kingdom +581498,12/9/2019,22384,Lunch Bag Pink Polkadot,7.24,1,14498,United Kingdom +581498,12/9/2019,23437,50'S Christmas Gift Bag Large,7.24,1,14498,United Kingdom +581500,12/9/2019,82486,3 Drawer Antique White Wood Cabinet,7.24,4,15344,United King +581500,12/9/2019,85066,Cream Sweetheart Mini Chest,7.24,4,15344,United Kingdom +581500,12/9/2019,48187,Doormat New England,7.24,2,15344,United Kingdom +581501,12/9/2019,22319,Hairclips Forties Fabric Assorted,7.24,180,12985,United King +581501,12/9/2019,20704,Mr Robot Soft Toy,7.24,8,12985,United Kingdom +581501,12/9/2019,21564,Pink Heart Shape Love Bucket,6.19,24,12985,United Kingdom +581501,12/9/2019,21563,Red Heart Shape Love Bucket,7.24,24,12985,United Kingdom +581501,12/9/2019,22165,Diamante Heart Shaped Wall Mirror,7.24,12,12985,United Kingd +581501,12/9/2019,22299,Pig Keyring With Light & Sound,7.24,48,12985,United Kingdom +581501,12/9/2019,22447,Pin Cushion Babushka Blue,7.24,12,12985,United Kingdom +581501,12/9/2019,22442,Grow Your Own Flowers Set Of 3,7.24,12,12985,United Kingdom +581501,12/9/2019,22495,Set Of 2 Round Tins Camembert,7.24,12,12985,United Kingdom +581501,12/9/2019,22544,Mini Jigsaw Spaceboy,7.24,96,12985,United Kingdom +581501,12/9/2019,22695,Wicker Wreath Small,7.24,24,12985,United Kingdom +581501,12/9/2019,22785,Squarecushion Cover Pink Union Jack,7.24,12,12985,United Kin +581501,12/9/2019,22811,Set Of 6 T-Lights Cacti,7.24,12,12985,United Kingdom +581501,12/9/2019,22807,Set Of 6 T-Lights Toadstools,7.24,12,12985,United Kingdom +581501,12/9/2019,22942,Christmas Lights 10 Santas,7.24,12,12985,United Kingdom +581501,12/9/2019,22808,Set Of 6 T-Lights Easter Chicks,6.19,12,12985,United Kingdom +581501,12/9/2019,23143,Zinc Wire Kitchen Organiser,7.24,4,12985,United Kingdom +581501,12/9/2019,23151,Zinc Sweetheart Soap Dish,7.24,12,12985,United Kingdom +581501,12/9/2019,23425,Storage Tin Home Sweet Home,7.24,12,12985,United Kingdom +581501,12/9/2019,84356,Pompom Curtain,7.24,12,12985,United Kingdom +581501,12/9/2019,85173,Set/6 Frog Prince T-Light Candles,7.24,12,12985,United Kingd +581501,12/9/2019,21731,Red Toadstool Led Night Light,7.24,24,12985,United Kingdom +581501,12/9/2019,23480,Mini Lights Woodland Mushrooms,7.24,8,12985,United Kingdom +581501,12/9/2019,22545,Mini Jigsaw Bunnies,7.24,96,12985,United Kingdom +581502,12/9/2019,22087,Paper Bunting White Lace,7.24,6,15910,United Kingdom +581502,12/9/2019,21209,Multicolour Honeycomb Fan,7.24,5,15910,United Kingdom +581502,12/9/2019,20668,Disco Ball Christmas Decoration,7.24,24,15910,United Kingdom +581502,12/9/2019,21790,Vintage Snap Cards,7.24,6,15910,United Kingdom +581502,12/9/2019,23270,Set Of 2 Ceramic Painted Hearts,7.24,4,15910,United Kingdom +581502,12/9/2019,23103,Jingle Bell Heart Decoration,7.24,8,15910,United Kingdom +581502,12/9/2019,22576,Swallow Wooden Christmas Decoration,7.24,2,15910,United King +581502,12/9/2019,22153,Angel Decoration Stars On Dress,7.24,13,15910,United Kingdom +581502,12/9/2019,21810,Christmas Hanging Star With Bell,7.24,6,15910,United Kingdom +581502,12/9/2019,22155,Star Decoration Rustic,7.24,6,15910,United Kingdom +581502,12/9/2019,23210,White Rocking Horse Hand Painted,7.24,12,15910,United Kingdo +581502,12/9/2019,22573,Star Wooden Christmas Decoration,7.24,8,15910,United Kingdom +581502,12/9/2019,22075,6 Ribbons Elegant Christmas,7.24,4,15910,United Kingdom +581502,12/9/2019,85049A,Traditional Christmas Ribbons,7.24,4,15910,United Kingdom +581502,12/9/2019,22734,Set Of 6 Ribbons Vintage Christmas,7.24,4,15910,United Kingd +581502,12/9/2019,23274,Star T-Light Holder Willie Winkie,7.24,8,15910,United Kingdo +581502,12/9/2019,22153,Angel Decoration Stars On Dress,7.24,1,15910,United Kingdom +581502,12/9/2019,22596,Christmas Star Wish List Chalkboard,7.24,24,15910,United Kin +581502,12/9/2019,22952,60 Cake Cases Vintage Christmas,7.24,10,15910,United Kingdom +581502,12/9/2019,22141,Christmas Craft Tree Top Angel,7.24,3,15910,United Kingdom +581514,12/9/2019,22753,Small Yellow Babushka Notebook,7.24,12,17754,United Kingdom +581514,12/9/2019,22755,Small Purple Babushka Notebook,7.24,12,17754,United Kingdom +581514,12/9/2019,22754,Small Red Babushka Notebook,6.19,12,17754,United Kingdom +581514,12/9/2019,22721,Set Of 3 Cake Tins Sketchbook,6.19,4,17754,United Kingdom +581514,12/9/2019,22059,Ceramic Strawberry Design Mug,6.19,12,17754,United Kingdom +581514,12/9/2019,22199,Frying Pan Red Retrospot,6.19,13,17754,United Kingdom +581514,12/9/2019,22200,Frying Pan Pink Polkadot,6.19,2,17754,United Kingdom +581514,12/9/2019,84032A,Charlie+Lola Pink Hot Water Bottle,6.19,9,17754,United King +581514,12/9/2019,84032B,Charlie + Lola Red Hot Water Bottle,6.19,9,17754,United Kin +581514,12/9/2019,84031A,Charlie+Lola Red Hot Water Bottle,6.19,10,17754,United King +581514,12/9/2019,84031B,Charlie Lola Blue Hot Water Bottle,6.19,14,17754,United Kin +581514,12/9/2019,22646,Ceramic Strawberry Cake Money Bank,6.19,4,17754,United Kingd +581514,12/9/2019,22644,Ceramic Cherry Cake Money Bank,6.19,4,17754,United Kingdom +581514,12/9/2019,22645,Ceramic Heart Fairy Cake Money Bank,6.19,4,17754,United King +581514,12/9/2019,22394,Paperweight Kings Choice,6.04,12,17754,United Kingdom +581514,12/9/2019,17091J,Vanilla Incense In Tin,6.04,66,17754,United Kingdom +581514,12/9/2019,35471D,Set Of 3 Bird Light Pink Feather,6.04,12,17754,United Kingd +581514,12/9/2019,22075,6 Ribbons Elegant Christmas,6.04,24,17754,United Kingdom +581514,12/9/2019,17091J,Vanilla Incense In Tin,6.19,24,17754,United Kingdom +581514,12/9/2019,22069,Brown Pirate Treasure Chest,6.19,20,17754,United Kingdom +581514,12/9/2019,22068,Black Pirate Treasure Chest,6.19,14,17754,United Kingdom +581514,12/9/2019,22500,Set Of 2 Tins Jardin De Provence,6.19,4,17754,United Kingdom +581514,12/9/2019,22075,6 Ribbons Elegant Christmas,6.19,24,17754,United Kingdom +581514,12/9/2019,21705,Bag 500g Swirly Marbles,6.19,84,17754,United Kingdom +581516,12/9/2019,21109,Large Cake Towel Chocolate Spots,6.19,12,14422,United Kingdo +581516,12/9/2019,21111,Swiss Roll Towel Chocolate Spots,6.19,24,14422,United Kingdo +581516,12/9/2019,21705,Bag 500g Swirly Marbles,6.19,24,14422,United Kingdom +581516,12/9/2019,22185,Slate Tile Natural Hanging,6.19,12,14422,United Kingdom +581516,12/9/2019,22442,Grow Your Own Flowers Set Of 3,6.19,12,14422,United Kingdom +581516,12/9/2019,21620,Set Of 4 Rose Botanical Candles,6.19,12,14422,United Kingdom +581516,12/9/2019,84029G,Knitted Union Flag Hot Water Bottle,6.19,8,14422,United Kin +581516,12/9/2019,21485,Retrospot Heart Hot Water Bottle,6.19,3,14422,United Kingdom +581516,12/9/2019,22111,Scottie Dog Hot Water Bottle,6.19,6,14422,United Kingdom +581516,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,8,14422,United Kingdom +581516,12/9/2019,22112,Chocolate Hot Water Bottle,6.19,6,14422,United Kingdom +581516,12/9/2019,23356,Love Hot Water Bottle,6.19,6,14422,United Kingdom +581516,12/9/2019,21108,Fairy Cake Flannel Assorted Colour,6.19,18,14422,United King +581516,12/9/2019,22171,3 Hook Photo Shelf Antique White,6.19,4,14422,United Kingdom +581516,12/9/2019,23301,Gardeners Kneeling Pad Keep Calm,6.19,24,14422,United Kingdo +581538,12/9/2019,23193,Buffalo Bill Treasure Book Box,6.19,6,14446,United Kingdom +581538,12/9/2019,23194,Gymkhana Treasure Book Box,6.19,1,14446,United Kingdom +581538,12/9/2019,23084,Rabbit Night Light,6.19,2,14446,United Kingdom +581538,12/9/2019,22068,Black Pirate Treasure Chest,6.19,1,14446,United Kingdom +581538,12/9/2019,22956,36 Foil Heart Cake Cases,6.19,1,14446,United Kingdom +581538,12/9/2019,20936,Forked Cactus Candle,6.19,1,14446,United Kingdom +581538,12/9/2019,23040,Paper Lantern 9 Point Snow Star,6.19,1,14446,United Kingdom +581538,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,6.19,1,14446,United King +581538,12/9/2019,21222,Set/4 Badges Beetles,6.19,1,14446,United Kingdom +581538,12/9/2019,21220,Set/4 Badges Dogs,6.19,1,14446,United Kingdom +581538,12/9/2019,21224,Set/4 Skull Badges,6.19,1,14446,United Kingdom +581538,12/9/2019,85123A,Cream Hanging Heart T-Light Holder,6.19,1,14446,United King +581538,12/9/2019,22992,Revolver Wooden Ruler,6.19,1,14446,United Kingdom +581538,12/9/2019,79066K,Retro Mod Tray,6.19,1,14446,United Kingdom +581538,12/9/2019,84380,Set Of 3 Butterfly Cookie Cutters,6.19,1,14446,United Kingdo +581538,12/9/2019,23371,Set 36 Colour Pencils Spaceboy,6.19,1,14446,United Kingdom +581538,12/9/2019,22694,Wicker Star,6.19,1,14446,United Kingdom +581538,12/9/2019,22095,Lads Only Tissue Box,6.19,1,14446,United Kingdom +581538,12/9/2019,23275,Set Of 3 Hanging Owls Ollie Beak,6.19,1,14446,United Kingdom +581538,12/9/2019,21673,White Spot Blue Ceramic Drawer Knob,6.19,1,14446,United King +581538,12/9/2019,21670,Blue Spot Ceramic Drawer Knob,6.19,1,14446,United Kingdom +581538,12/9/2019,85071C,"Charlie+Lola""Extremely Busy"" Sign",6.19,1,14446,United K +581538,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,3,14446,United Kingdom +581538,12/9/2019,23034,Drawer Knob Ceramic Black,6.19,1,14446,United Kingdom +581538,12/9/2019,21669,Blue Stripe Ceramic Drawer Knob,6.19,1,14446,United Kingdom +581538,12/9/2019,23033,Drawer Knob Ceramic Red,6.19,1,14446,United Kingdom +581538,12/9/2019,21668,Red Stripe Ceramic Drawer Knob,6.19,1,14446,United Kingdom +581538,12/9/2019,23275,Set Of 3 Hanging Owls Ollie Beak,6.19,1,14446,United Kingdom +581538,12/9/2019,23318,Box Of 6 Mini Vintage Crackers,6.19,1,14446,United Kingdom +581538,12/9/2019,23382,Box Of 6 Christmas Cake Decorations,6.19,1,14446,United King +581538,12/9/2019,22469,Heart Of Wicker Small,6.19,1,14446,United Kingdom +581538,12/9/2019,22899,Children's Apron Dolly Girl,6.19,2,14446,United Kingdom +581538,12/9/2019,22367,Childrens Apron Spaceboy Design,6.19,1,14446,United Kingdom +581538,12/9/2019,22899,Children's Apron Dolly Girl,6.19,3,14446,United Kingdom +581538,12/9/2019,23527,Wall Art Animals And Nature,6.19,1,14446,United Kingdom +581538,12/9/2019,23524,Wall Art Horse & Pony,6.19,1,14446,United Kingdom +581538,12/9/2019,23525,Wall Art Buffalo Bill,6.19,1,14446,United Kingdom +581538,12/9/2019,84380,Set Of 3 Butterfly Cookie Cutters,6.19,4,14446,United Kingdo +581538,12/9/2019,23122,Party Charms 50 Pieces,7.24,1,14446,United Kingdom +581538,12/9/2019,21990,Modern Floral Stationery Set,7.24,1,14446,United Kingdom +581538,12/9/2019,21329,Dinosaurs Writing Set,7.24,1,14446,United Kingdom +581538,12/9/2019,21328,Balloons Writing Set,7.24,1,14446,United Kingdom +581538,12/9/2019,22561,Wooden School Colouring Set,7.24,1,14446,United Kingdom +581538,12/9/2019,84519A,Tomato Charlie+Lola Coaster Set,7.24,1,14446,United Kingdom +581538,12/9/2019,84519B,Carrot Charlie+Lola Coaster Set,7.24,1,14446,United Kingdom +581538,12/9/2019,35004B,Set Of 3 Black Flying Ducks,7.24,2,14446,United Kingdom +581538,12/9/2019,22068,Black Pirate Treasure Chest,7.24,1,14446,United Kingdom +581538,12/9/2019,23353,6 Gift Tags Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,21591,Cosy Hour Cigar Box Matches,6.19,1,14446,United Kingdom +581538,12/9/2019,22197,Popcorn Holder,6.19,4,14446,United Kingdom +581538,12/9/2019,23320,Giant 50'S Christmas Cracker,6.19,1,14446,United Kingdom +581538,12/9/2019,23349,Roll Wrap Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,22985,Wrap Billboard Fonts Design,6.19,25,14446,United Kingdom +581538,12/9/2019,23040,Paper Lantern 9 Point Snow Star,6.19,2,14446,United Kingdom +581538,12/9/2019,23040,Paper Lantern 9 Point Snow Star,6.19,3,14446,United Kingdom +581538,12/9/2019,21194,Pink Honeycomb Paper Fan,6.19,2,14446,United Kingdom +581538,12/9/2019,21208,Pastel Colour Honeycomb Fan,6.19,1,14446,United Kingdom +581538,12/9/2019,79190D,Retro Plastic Daisy Tray,6.19,1,14446,United Kingdom +581538,12/9/2019,79190B,Retro Plastic Polka Tray,6.19,1,14446,United Kingdom +581538,12/9/2019,22909,Set Of 20 Vintage Christmas Napkins,6.02,2,14446,United King +581538,12/9/2019,23318,Box Of 6 Mini Vintage Crackers,6.02,1,14446,United Kingdom +581538,12/9/2019,23319,Box Of 6 Mini 50'S Crackers,6.02,1,14446,United Kingdom +581538,12/9/2019,23537,Wall Art I Love London,6.19,1,14446,United Kingdom +581538,12/9/2019,22992,Revolver Wooden Ruler,6.19,1,14446,United Kingdom +581538,12/9/2019,22991,Giraffe Wooden Ruler,6.19,1,14446,United Kingdom +581538,12/9/2019,23190,Bundle Of 3 School Exercise Books,6.19,1,14446,United Kingdo +581538,12/9/2019,21194,Pink Honeycomb Paper Fan,6.19,1,14446,United Kingdom +581538,12/9/2019,35004B,Set Of 3 Black Flying Ducks,6.19,1,14446,United Kingdom +581538,12/9/2019,22694,Wicker Star,6.19,1,14446,United Kingdom +581538,12/9/2019,21355,Toast Its - I Love You,6.19,1,14446,United Kingdom +581538,12/9/2019,23343,Jumbo Bag Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,23343,Jumbo Bag Vintage Christmas,6.19,1,14446,United Kingdom +581538,12/9/2019,20727,Lunch Bag Black Skull,6.19,1,14446,United Kingdom +581538,12/9/2019,20725,Lunch Bag Red Retrospot,6.19,1,14446,United Kingdom +581566,12/9/2019,23404,Home Sweet Home Blackboard,6.19,144,18102,United Kingdom +581567,12/9/2019,21417,Cockle Shell Dish,6.19,84,16626,United Kingdom +581567,12/9/2019,22464,Hanging Metal Heart Lantern,6.19,24,16626,United Kingdom +581567,12/9/2019,22465,Hanging Metal Star Lantern,6.19,24,16626,United Kingdom +581567,12/9/2019,84971S,Small Heart Flowers Hook,6.19,48,16626,United Kingdom +581567,12/9/2019,22624,Ivory Kitchen Scales,6.19,2,16626,United Kingdom +581567,12/9/2019,22627,Mint Kitchen Scales,6.19,2,16626,United Kingdom +581567,12/9/2019,22625,Red Kitchen Scales,6.19,2,16626,United Kingdom +581567,12/9/2019,23355,Hot Water Bottle Keep Calm,6.19,4,16626,United Kingdom +581567,12/9/2019,21326,Aged Glass Silver T-Light Holder,6.19,144,16626,United Kingd +581567,12/9/2019,21479,White Skull Hot Water Bottle,6.19,4,16626,United Kingdom +581567,12/9/2019,23356,Love Hot Water Bottle,6.19,3,16626,United Kingdom +581567,12/9/2019,21137,Black Record Cover Frame,6.19,24,16626,United Kingdom +581570,12/9/2019,22141,Christmas Craft Tree Top Angel,6.19,6,12662,Germany +581570,12/9/2019,22175,Pink Owl Soft Toy,6.19,6,12662,Germany +581570,12/9/2019,21481,Fawn Blue Hot Water Bottle,6.19,4,12662,Germany +581570,12/9/2019,23570,Traditional Pick Up Sticks Game,6.19,12,12662,Germany +581570,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,6.19,6,12662,Germany +581570,12/9/2019,22331,Woodland Party Bag + Sticker Set,6.19,8,12662,Germany +581570,12/9/2019,22834,Hand Warmer Babushka Design,6.19,12,12662,Germany +581570,12/9/2019,21914,Blue Harmonica In Box,6.19,12,12662,Germany +581570,12/9/2019,22139,Retrospot Tea Set Ceramic 11 Pc,6.19,3,12662,Germany +581570,12/9/2019,23077,Doughnut Lip Gloss,6.19,20,12662,Germany +581570,12/9/2019,20750,Red Retrospot Mini Cases,6.19,2,12662,Germany +581570,12/9/2019,22505,Memo Board Cottage Design,6.19,4,12662,Germany +581571,12/9/2019,23326,Hanging Mini Coloured Bottles,6.19,6,15311,United Kingdom +581571,12/9/2019,21313,Glass Heart T-Light Holder,6.19,1,15311,United Kingdom +581571,12/9/2019,48187,Doormat New England,6.19,1,15311,United Kingdom +581571,12/9/2019,23317,Blue Refectory Clock,6.19,1,15311,United Kingdom +581571,12/9/2019,23197,Sketchbook Magnetic Shopping List,6.19,4,15311,United Kingdo +581571,12/9/2019,21012,Antique All Glass Candlestick,6.19,2,15311,United Kingdom +581571,12/9/2019,22227,Hanging Heart Mirror Decoration,6.19,10,15311,United Kingdom +581571,12/9/2019,22794,Sweetheart Wire Magazine Rack,6.19,1,15311,United Kingdom +581571,12/9/2019,23182,Toilet Sign Occupied Or Vacant,6.19,1,15311,United Kingdom +581571,12/9/2019,21755,Love Building Block Word,6.19,1,15311,United Kingdom +581571,12/9/2019,85053,French Enamel Candleholder,6.19,2,15311,United Kingdom +581571,12/9/2019,23110,Parisienne Key Cabinet,6.19,2,15311,United Kingdom +581571,12/9/2019,21169,You're Confusing Me Metal Sign,6.19,1,15311,United Kingdom +581571,12/9/2019,21258,Victorian Sewing Box Large,6.19,8,15311,United Kingdom +581571,12/9/2019,23168,Classic Cafe Sugar Dispenser,6.19,36,15311,United Kingdom +581571,12/9/2019,23167,Small Ceramic Top Storage Jar,6.19,96,15311,United Kingdom +581571,12/9/2019,21314,Small Glass Heart Trinket Pot,6.19,48,15311,United Kingdom +581571,12/9/2019,21137,Black Record Cover Frame,6.19,24,15311,United Kingdom +581571,12/9/2019,44234,Assorted Circular Mobile,6.19,1,15311,United Kingdom +581571,12/9/2019,84347,Rotating Silver Angels T-Light Hldr,6.19,24,15311,United Kin +581572,12/9/2019,23328,Set 6 School Milk Bottles In Crate,6.19,48,16705,United King +581572,12/9/2019,22627,Mint Kitchen Scales,6.19,4,16705,United Kingdom +581572,12/9/2019,22624,Ivory Kitchen Scales,6.19,4,16705,United Kingdom +581572,12/9/2019,23245,Set Of 3 Regency Cake Tins,6.19,4,16705,United Kingdom +581574,12/9/2019,22326,Round Snack Boxes Set Of4 Woodland,6.19,6,12526,Germany +581574,12/9/2019,22328,Round Snack Boxes Set Of 4 Fruits,6.19,6,12526,Germany +581574,12/9/2019,23238,Set Of 4 Knick Knack Tins London,6.19,6,12526,Germany +581574,12/9/2019,23237,Set Of 4 Knick Knack Tins Leaf,6.19,6,12526,Germany +581574,12/9/2019,21257,Victorian Sewing Box Medium,6.19,2,12526,Germany +581574,12/9/2019,21258,Victorian Sewing Box Large,6.19,2,12526,Germany +581574,12/9/2019,23111,Parisienne Sewing Box,6.19,2,12526,Germany +581574,12/9/2019,22077,6 Ribbons Rustic Charm,6.19,12,12526,Germany +581574,12/9/2019,22074,6 Ribbons Shimmering Pinks,6.19,12,12526,Germany +581574,12/9/2019,22621,Traditional Knitting Nancy,6.19,12,12526,Germany +581574,12/9/2019,23199,Jumbo Bag Apples,6.19,10,12526,Germany +581574,12/9/2019,23581,Jumbo Bag Paisley Park,6.19,10,12526,Germany +581578,12/9/2019,21124,Set/10 Blue Polkadot Party Candles,6.19,24,12713,Germany +581578,12/9/2019,21122,Set/10 Pink Polkadot Party Candles,6.19,24,12713,Germany +581578,12/9/2019,21121,Set/10 Red Polkadot Party Candles,6.19,24,12713,Germany +581578,12/9/2019,23389,Spaceboy Mini Backpack,6.04,4,12713,Germany +581578,12/9/2019,22556,Plasters In Tin Circus Parade,6.19,12,12713,Germany +581578,12/9/2019,22976,Circus Parade Childrens Egg Cup,6.19,12,12713,Germany +581578,12/9/2019,23255,Childrens Cutlery Circus Parade,7.24,12,12713,Germany +581578,12/9/2019,84997D,Childrens Cutlery Polkadot Pink,7.24,8,12713,Germany +581578,12/9/2019,84997B,Childrens Cutlery Retrospot Red,7.24,8,12713,Germany +581578,12/9/2019,84997C,Childrens Cutlery Polkadot Blue,7.24,8,12713,Germany +581578,12/9/2019,22555,Plasters In Tin Strongman,7.24,12,12713,Germany +581578,12/9/2019,21914,Blue Harmonica In Box,7.24,12,12713,Germany +581578,12/9/2019,22549,Picture Dominoes,7.24,24,12713,Germany +581578,12/9/2019,21918,Set 12 Kids Colour Chalk Sticks,7.24,24,12713,Germany +581578,12/9/2019,22992,Revolver Wooden Ruler,7.24,12,12713,Germany +581578,12/9/2019,22991,Giraffe Wooden Ruler,7.24,12,12713,Germany +581578,12/9/2019,23229,Vintage Donkey Tail Game,7.24,6,12713,Germany +581578,12/9/2019,22622,Box Of Vintage Alphabet Blocks,6.19,6,12713,Germany +581578,12/9/2019,21506,Fancy Font Birthday Card,6.19,12,12713,Germany +581578,12/9/2019,21507,Elephant Birthday Card,6.19,12,12713,Germany +581578,12/9/2019,23037,Candle Holder Silver Madeline,6.19,12,12713,Germany +581578,12/9/2019,23550,Wrap Alphabet Poster,6.19,25,12713,Germany +581578,12/9/2019,22711,Wrap Circus Parade,6.19,25,12713,Germany +581578,12/9/2019,21497,Fancy Fonts Birthday Wrap,6.19,25,12713,Germany +581578,12/9/2019,22704,Wrap Red Apples,6.19,25,12713,Germany +581578,12/9/2019,22585,Pack Of 6 Birdy Gift Tags,6.19,12,12713,Germany diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/python-solution/task.py b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/python-solution/task.py new file mode 100644 index 0000000000000..59da926dc6a18 --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/python-solution/task.py @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# beam-playground: +# name: FinalSolution1 +# description: Final challenge solution 1. +# multifile: true +# files: +# - name: input.csv +# context_line: 57 +# categories: +# - Quickstart +# complexity: ADVANCED +# tags: +# - hellobeam + + +import apache_beam as beam +import logging +import re +from apache_beam.transforms import window, trigger +from apache_beam.transforms.combiners import CountCombineFn + + +class Transaction: + def __init__(self, transaction_no, date, product_no, product_name, price, quantity, customer_no, country): + self.transaction_no = transaction_no + self.date = date + self.product_no = product_no + self.product_name = product_name + self.price = price + self.quantity = quantity + self.customer_no = customer_no + self.country = country + + def __str__(self): + return f"Transaction(transaction_no={self.transaction_no}, date='{self.date}', product_no='{self.product_no}', product_name='{self.product_name}', price={self.price}, quantity={self.quantity}, customer_no={self.customer_no}, country='{self.country}')" + + +class ExtractDataFn(beam.DoFn): + def process(self, element): + items = re.split(r',(?=(?:[^"]*"[^"]*")*[^"]*$)', element) + if items[0] != 'TransactionNo': + yield Transaction(items[0], items[1], items[2], items[3], items[4], items[5], items[6], items[7]) + + +def partitionTransactions(element, num_partitions): + if float(element.price) >= 10: + return 0 + else: + return 1 + + +def run(): + with beam.Pipeline() as pipeline: + transactions = (pipeline + | 'Read from text file' >> beam.io.ReadFromText('input.csv') + | 'Extract Data' >> beam.ParDo(ExtractDataFn()) + ) + + windowed_transactions = (transactions + | 'Window' >> beam.WindowInto(window.FixedWindows(30), trigger=trigger.AfterWatermark( + early=trigger.AfterProcessingTime(5).has_ontime_pane(), late=trigger.AfterAll()), + allowed_lateness=30, + accumulation_mode=trigger.AccumulationMode.DISCARDING)) + + partition = (windowed_transactions + | 'Filtering' >> beam.Filter(lambda t: int(t.quantity) >= 20) + | 'Partition transactions' >> beam.Partition(partitionTransactions, 2)) + + biggerThan10 = partition[0] + smallerThan10 = partition[1] + + (biggerThan10 + | 'Map product_no and price for bigger' >> beam.Map(lambda transaction: (transaction.product_no, float(transaction.price))) + | 'Calculate sum for price more than 10' >> beam.CombinePerKey(sum) + | 'Write price more than 10 results to text file' >> beam.io.WriteToText('price_more_than_10', '.txt', shard_name_template='')) + + (smallerThan10 + | 'Map product_no and price for smaller' >> beam.Map(lambda transaction: (transaction.product_no, float(transaction.price))) + | 'Calculate sum for price less than 10' >> beam.CombinePerKey(sum) + | 'Write price less than 10 results to text file' >> beam.io.WriteToText('price_less_than_10', '.txt', shard_name_template='')) + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + run() diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/unit-info.yaml b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/unit-info.yaml new file mode 100644 index 0000000000000..d6483f02d2db7 --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/unit-info.yaml @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +sdk: + - Java + - Python + - Go +id: final-challenge-1 +name: Final Challenge 1 +taskName: FinalChallenge1 +solutionName: FinalSolution1 diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/description.md b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/description.md new file mode 100644 index 0000000000000..cfef46df9cdc2 --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/description.md @@ -0,0 +1,22 @@ + +### Final challenge 2 + +You are given a file analyzed.csv which maps words to sentiments. Using this, analyze kinglear.txt. Output PCollections counting the number of **negative words** and **positive words** as well as PCollections counting the number of **positive words with strong modal** and **positive words with weak modal**. + +Example rows from input file: + +| Word | Negative | Positive | Uncertainty | Litigious | Strong_Modal | Weak_Modal | Constraining | +|--------------|----------|----------|-------------|-----------|--------------|------------|--------------| +| NONSEVERABLE | 0 | 0 | 0 | 2011 | 0 | 0 | 0 | +| DISFAVOR | 2009 | 0 | 0 | 0 | 0 | 0 | 0 | +| COMPLIMENT | 0 | 2009 | 0 | 0 | 0 | 0 | 0 | \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/go-challenge/analysis.csv b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/go-challenge/analysis.csv new file mode 100644 index 0000000000000..5c4a1246021eb --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/go-challenge/analysis.csv @@ -0,0 +1,3877 @@ +Word,Negative,Positive,Uncertainty,Litigious,Strong_Modal,Weak_Modal,Constraining +NONSEVERABLE,0,0,0,2011,0,0,0 +DISFAVOR,2009,0,0,0,0,0,0 +DISGORGE,2009,0,0,0,0,0,0 +COMPLICATES,2009,0,0,0,0,0,0 +COMPLIMENT,0,2009,0,0,0,0,0 +COMPLIMENTS,0,2009,0,0,0,0,0 +MISSTATE,2009,0,0,0,0,0,0 +MISSTATES,2009,0,0,0,0,0,0 +MISTAKE,2009,0,0,0,0,0,0 +MISTAKING,2009,0,0,0,0,0,0 +MISUNDERSTANDING,2009,0,0,0,0,0,0 +MISUSED,2009,0,0,0,0,0,0 +MONOPOLISTS,2009,0,0,0,0,0,0 +MONOPOLIZES,2009,0,0,0,0,0,0 +MORATORIUM,2009,0,0,0,0,0,0 +MOTHBALLING,2009,0,0,0,0,0,0 +SLOW,2009,0,0,0,0,0,0 +SLOWER,2009,0,0,0,0,0,0 +SLOWNESS,2009,0,0,0,0,0,0 +SMOOTH,0,2009,0,0,0,0,0 +DEPRECATION,2009,0,0,0,0,0,0 +DEPRESSING,2009,0,0,0,0,0,0 +DEPRIVES,2009,0,0,0,0,0,0 +DEROGATE,0,0,0,2009,0,0,0 +DEROGATION,0,0,0,2009,0,0,0 +DESIRABLE,0,2009,0,0,0,0,0 +DESTABILIZATION,2009,0,0,0,0,0,0 +DESTINED,0,2009,0,0,0,0,0 +DESTROYS,2009,0,0,0,0,0,0 +DETAINED,2009,0,0,0,0,0,0 +DETER,2009,0,0,0,0,0,0 +DETERIORATING,2009,0,0,0,0,0,0 +DETERRENCE,2009,0,0,0,0,0,0 +DETERRING,2009,0,0,0,0,0,0 +DETRACTING,2009,0,0,0,0,0,0 +DETRIMENTS,2009,0,0,0,0,0,0 +DEVALUING,2009,0,0,0,0,0,0 +DEVASTATION,2009,0,0,0,0,0,0 +DEVIATING,2009,0,2009,0,0,0,0 +DEVOLVE,2009,0,0,0,0,0,0 +DICTATE,0,0,0,0,0,0,2009 +DIFFER,0,0,2009,0,0,0,0 +DIFFICULT,2009,0,0,0,0,0,0 +ASSAULT,2009,0,0,0,0,0,0 +ASSERTABLE,0,0,0,2011,0,0,0 +ASSUMABLE,0,0,0,2009,0,0,0 +ASSUMING,0,0,2009,0,0,0,0 +ASSURED,0,2009,0,0,0,0,0 +ATTAINED,0,2009,0,0,0,0,0 +ATTAINS,0,2009,0,0,0,0,0 +ATTESTED,0,0,0,2009,0,0,0 +ATTORNEYS,0,0,0,2009,0,0,0 +ATTRACTIVENESS,0,2009,0,0,0,0,0 +BAD,2009,0,0,0,0,0,0 +BAILEES,0,0,0,2011,0,0,0 +BAILOUT,2009,0,0,0,0,0,0 +BANKRUPTCIES,2009,0,0,0,0,0,0 +BANKRUPTS,2009,0,0,0,0,0,0 +CLAIM,0,0,0,2009,0,0,0 +CLAIMHOLDER,0,0,0,2014,0,0,0 +CLARIFICATIONS,0,0,2009,0,0,0,0 +CLOSED,-2020,0,0,0,0,0,0 +CLOSINGS,2009,0,0,0,0,0,0 +CODEFENDANTS,0,0,0,2011,0,0,0 +CODIFICATIONS,0,0,0,2009,0,0,0 +CODIFYING,0,0,0,2009,0,0,0 +COERCING,2009,0,0,0,0,0,0 +COLLABORATES,0,2009,0,0,0,0,0 +COLLABORATIVE,0,2009,0,0,0,0,0 +COLLAPSED,2009,0,0,0,0,0,0 +COLLISIONS,2009,0,0,0,0,0,0 +COLLUDING,2009,0,0,0,0,0,0 +POSES,2009,0,0,0,0,0,0 +POSSESSORY,0,0,0,2009,0,0,0 +POSSIBLY,0,0,2009,0,0,2009,0 +POSTJUDGMENT,0,0,0,2011,0,0,0 +POSTPONEMENTS,2009,0,0,0,0,0,0 +WRITEDOWNS,2009,0,0,0,0,0,0 +WRONG,2009,0,0,0,0,0,0 +WRONGFULLY,2009,0,0,0,0,0,0 +HONORABLE,0,-2020,0,0,0,0,0 +HOSTILE,2009,0,0,0,0,0,0 +REASSESS,0,0,2009,0,0,0,0 +REASSESSMENT,2009,0,2009,0,0,0,0 +REASSIGNING,2009,0,0,0,0,0,0 +REBOUND,0,2009,0,0,0,0,0 +REBUTS,0,0,0,2009,0,0,0 +REBUTTALS,0,0,0,2009,0,0,0 +RECALCULATED,0,0,2009,0,0,0,0 +RECALCULATIONS,0,0,2009,0,0,0,0 +RECALLS,2009,0,0,0,0,0,0 +RECESSIONS,2009,0,0,0,0,0,0 +RECONSIDER,0,0,2009,0,0,0,0 +RECORDATION,0,0,0,2009,0,0,0 +RECOURSE,0,0,0,2009,0,0,0 +RECUSAL,0,0,0,2011,0,0,0 +RECUSING,0,0,0,2011,0,0,0 +REDACTION,2009,0,0,2009,0,0,0 +REDEFAULTS,2014,0,0,0,0,0,0 +REDRESSING,2009,0,0,0,0,0,0 +REFERENDA,0,0,0,2009,0,0,0 +REFILED,0,0,0,2009,0,0,0 +REFRAINING,0,0,0,0,0,0,2009 +REFUSE,2009,0,0,0,0,0,0 +REGAIN,0,2009,0,0,0,0,0 +REGULATED,0,0,0,2009,0,0,0 +REGULATIONS,0,0,0,2009,0,0,0 +REGULATORY,0,0,0,2009,0,0,0 +REHEARINGS,0,0,0,2009,0,0,0 +VARIABILITY,0,0,2009,0,0,0,0 +VARIANCE,0,0,2009,0,0,0,0 +VARIATION,0,0,2009,0,0,0,0 +VARY,0,0,2009,0,0,0,0 +VERDICT,2009,0,0,2009,0,0,0 +VETOED,2009,0,0,0,0,0,0 +VICTIMS,2009,0,0,0,0,0,0 +VIOLATING,2009,0,0,0,0,0,0 +VIOLATOR,2009,0,0,0,0,0,0 +VIOLENTLY,2009,0,0,0,0,0,0 +VITIATING,2009,0,0,0,0,0,0 +VOIDING,2009,0,0,2009,0,0,0 +VULNERABILITIES,2009,0,0,0,0,0,0 +WARN,2009,0,0,0,0,0,0 +WARNS,2009,0,0,0,0,0,0 +WASTEFUL,2009,0,0,0,0,0,0 +WEAKENED,2009,0,0,0,0,0,0 +WEAKEST,2009,0,0,0,0,0,0 +DISHONESTLY,2009,0,0,0,0,0,0 +DISHONORABLY,2009,0,0,0,0,0,0 +DISINTERESTEDNESS,2009,0,0,0,0,0,0 +DISMAL,2009,0,0,0,0,0,0 +DISMISSALS,2009,0,0,0,0,0,0 +DISORDERLY,2009,0,0,0,0,0,0 +DISPARAGEMENTS,2009,0,0,0,0,0,0 +DISPARITIES,2009,0,0,0,0,0,0 +DISPLACEMENT,2009,0,0,0,0,0,0 +DISPOSE,2009,0,0,0,0,0,0 +DISPOSSESSES,2009,0,0,0,0,0,0 +DISPROPORTION,2009,0,0,0,0,0,0 +DISPUTE,2009,0,0,0,0,0,0 +DISQUALIFICATION,2009,0,0,0,0,0,0 +DISQUALIFY,2009,0,0,0,0,0,0 +DISREGARDING,2009,0,0,0,0,0,0 +DISRUPT,2009,0,0,0,0,0,0 +DISRUPTIONS,2009,0,0,0,0,0,0 +DISSATISFIED,2009,0,0,0,0,0,0 +DISSENTERS,2009,0,0,0,0,0,0 +DISSIDENTS,2009,0,0,0,0,0,0 +DISTINCTIONS,0,2009,0,0,0,0,0 +DISTORT,2009,0,0,0,0,0,0 +DISTORTIONS,2009,0,0,0,0,0,0 +DISTRACTING,2009,0,0,0,0,0,0 +DISTRAINT,0,0,0,2009,0,0,0 +DISTRIBUTEES,0,0,0,2009,0,0,0 +DISTURBED,2009,0,0,0,0,0,0 +DIVERT,2009,0,0,0,0,0,0 +DIVEST,2009,0,0,0,0,0,0 +DIVESTITURES,2009,0,0,0,0,0,0 +DIVORCE,2009,0,0,0,0,0,0 +DIVULGES,2009,0,0,0,0,0,0 +DOCKETING,0,0,0,2009,0,0,0 +DOUBTED,2009,0,2009,0,0,0,0 +DOWNGRADED,2009,0,0,0,0,0,0 +DOWNSIZED,2009,0,0,0,0,0,0 +DOWNTIME,2009,0,0,0,0,0,0 +DOWNWARD,2009,0,0,0,0,0,0 +DRASTICALLY,2009,0,0,0,0,0,0 +DROPPED,2009,0,0,0,0,0,0 +DURESS,2009,0,0,0,0,0,0 +EARMARK,0,0,0,0,0,0,2009 +EASIER,0,2009,0,0,0,0,0 +EFFECTIVE,0,-2020,0,0,0,0,0 +EFFICIENTLY,0,2009,0,0,0,0,0 +EMBARGO,2009,0,0,0,0,0,0 +EMBARRASS,2009,0,0,0,0,0,0 +EMBARRASSMENT,2009,0,0,0,0,0,0 +EMBEZZLEMENT,2009,0,0,0,0,0,0 +EMBEZZLING,2009,0,0,0,0,0,0 +EMPOWERS,0,2009,0,0,0,0,0 +ENABLING,0,2009,0,0,0,0,0 +ENCOURAGING,0,2009,0,0,0,0,0 +ENCROACHING,2009,0,0,0,0,0,0 +ENCUMBERED,2009,0,0,2009,0,0,2009 +ENCUMBRANCER,0,0,0,2009,0,0,0 +ENDANGERED,2009,0,0,0,0,0,0 +ENDORSEE,0,0,0,2011,0,0,0 +ENHANCE,0,2009,0,0,0,0,0 +ENHANCES,0,2009,0,0,0,0,0 +ENJOINING,2009,0,0,0,0,0,0 +ENJOYABLY,0,2009,0,0,0,0,0 +ENJOYS,0,2009,0,0,0,0,0 +ENTAILS,0,0,0,0,0,0,2009 +PATENTEE,0,0,0,2014,0,0,0 +PENALIZES,2009,0,0,0,0,0,0 +PENDING,0,0,2009,0,0,0,0 +PERFECTS,0,2009,0,0,0,0,0 +PERJURY,2009,0,0,2009,0,0,0 +PERMITTED,0,0,0,0,0,0,2009 +PERPETRATE,2009,0,0,2009,0,0,0 +PERPETRATION,2009,0,0,2009,0,0,0 +PERSISTENT,2009,0,0,0,0,0,0 +PERSONAM,0,0,0,2011,0,0,0 +PETITION,0,0,0,2009,0,0,0 +PETITIONING,0,0,0,2009,0,0,0 +PICKETED,2009,0,0,0,0,0,0 +HENCEFORTH,0,0,0,2009,0,0,0 +HEREDITAMENTS,0,0,0,2009,0,0,0 +HEREIN,0,0,0,2009,0,0,0 +HEREINBELOW,0,0,0,2009,0,0,0 +HERETOFORE,0,0,0,2009,0,0,0 +HEREWITH,0,0,0,2009,0,0,0 +HINDER,2009,0,0,0,0,0,0 +HINDRANCE,2009,0,0,0,0,0,0 +WHATSOEVER,0,0,0,2009,0,0,0 +WHEREAT,0,0,0,2009,0,0,0 +WHEREOF,0,0,0,2009,0,0,0 +WHEREUPON,0,0,0,2009,0,0,0 +WHOMSOEVER,0,0,0,2009,0,0,0 +WILLFUL,0,0,0,2009,0,0,0 +WINNER,0,2009,0,0,0,0,0 +WITNESSES,0,0,0,2009,0,0,0 +FLUCTUATES,0,0,2009,0,0,0,0 +FORBADE,0,0,0,2009,0,0,2011 +FORBEARING,0,0,0,2009,0,0,0 +FORBIDDING,2009,0,0,0,0,0,2009 +FORCING,2009,0,0,0,0,0,0 +FORECLOSE,2009,0,0,0,0,0,0 +FORECLOSURE,2009,0,0,0,0,0,0 +FOREGONE,2009,0,0,0,0,0,0 +FORESTALLS,2009,0,0,0,0,0,0 +FORFEITED,2009,0,0,0,0,0,0 +FORFEITURES,2009,0,0,0,0,0,0 +FORWHICH,0,0,0,2012,0,0,0 +FRAUDULENT,2009,0,0,0,0,0,0 +FRIVOLOUSLY,2009,0,0,0,0,0,0 +FRUSTRATING,2009,0,0,0,0,0,0 +FUGITIVE,-2020,0,0,2009,0,0,0 +GAINED,0,2009,0,0,0,0,0 +GRANTOR,0,0,0,2009,0,0,0 +DISGORGING,2009,0,0,0,0,0,0 +DISHONEST,2009,0,0,0,0,0,0 +LITIGANT,2009,0,0,2009,0,0,0 +LITIGATES,2009,0,0,2009,0,0,0 +LITIGATOR,0,0,0,2009,0,0,0 +LACKLUSTER,2009,0,0,0,0,0,0 +LAGGING,2009,0,0,0,0,0,0 +LAPSES,2009,0,0,0,0,0,0 +LAW,0,0,0,2009,0,0,0 +LAWMAKERS,0,0,0,2009,0,0,0 +LAWSUITS,0,0,0,2009,0,0,0 +LAYOFFS,2009,0,0,0,0,0,0 +LEGALESE,0,0,0,2009,0,0,0 +LEGALIZE,0,0,0,2009,0,0,0 +LEGALLY,0,0,0,2009,0,0,0 +NONPRODUCING,2009,0,0,0,0,0,0 +LIQUIDATE,2009,0,0,0,0,0,0 +LIQUIDATION,2009,0,0,0,0,0,0 +BENEFICIALLY,0,2009,0,0,0,0,0 +BENEFITED,0,2009,0,0,0,0,0 +BEST,0,2012,0,0,2009,0,0 +POPULAR,0,2009,0,0,0,0,0 +REINTERPRETED,0,0,2009,0,0,0,0 +REJECTED,2009,0,0,0,0,0,0 +REJECTS,2009,0,0,0,0,0,0 +RELINQUISHES,2009,0,0,0,0,0,0 +RELUCTANCE,2009,0,0,0,0,0,0 +REMANDING,0,0,0,2009,0,0,0 +REMEDIATING,0,0,0,2009,0,0,0 +REMISED,0,0,0,2011,0,0,0 +RENEGOTIATING,2009,0,0,0,0,0,0 +RENOUNCED,2009,0,0,0,0,0,0 +RENOUNCING,2009,0,0,0,0,0,0 +REPLEVIN,0,0,0,2009,0,0,0 +REPOSSESSION,2009,0,0,0,0,0,0 +TURBULENCE,2009,0,2009,0,0,0,0 +UNACCEPTABLY,2009,0,0,0,0,0,0 +UNANTICIPATED,2009,0,0,0,0,0,0 +UNATTRACTIVE,2009,0,0,0,0,0,0 +UNAVOIDABLE,2009,0,0,0,0,0,0 +UNCERTAINLY,0,0,2009,0,0,2009,0 +UNCOLLECTABLE,2009,0,0,0,0,0,0 +UNCOLLECTIBLES,2009,0,0,0,0,0,0 +UNCONFIRMED,0,0,2009,0,0,0,0 +UNCONSTITUTIONALITY,0,0,0,2009,0,0,0 +UNCONTROLLABLY,2009,0,0,0,0,0,0 +UNCOVERED,2009,0,0,0,0,0,0 +UNDEFEASED,0,0,0,2012,0,0,0 +UNDERCAPITALIZED,2009,0,0,0,0,0,0 +UNDERESTIMATE,2009,0,0,0,0,0,0 +UNDERESTIMATION,2009,0,0,0,0,0,0 +UNDERMINED,2009,0,0,0,0,0,0 +UNDERPAYMENT,2009,0,0,0,0,0,0 +UNDERPERFORMANCE,2009,0,0,0,0,0,0 +UNDERPRODUCED,2009,0,0,0,0,0,0 +UNDERSTATED,2009,0,0,0,0,0,0 +UNDERSTATING,2009,0,0,0,0,0,0 +UNDESIRABLE,2009,0,0,0,0,0,0 +UNDETERMINABLE,0,0,2009,0,0,0,0 +UNDISPUTED,0,0,0,0,2009,0,0 +UNDULY,2009,0,0,0,0,0,0 +UNEMPLOYED,2009,0,0,0,0,0,0 +UNENFORCEABILITY,0,0,0,2009,0,0,0 +UNETHICAL,2009,0,0,0,0,0,0 +UNEXPECTEDLY,2009,0,2009,0,0,0,0 +UNFAMILIARITY,0,0,2009,0,0,0,0 +UNFAVOURABLE,2011,0,0,0,0,0,0 +UNFORECASTED,0,0,2011,0,0,0,0 +UNFORTUNATE,2009,0,0,0,0,0,0 +UNFULFILLED,2009,0,0,0,0,0,0 +UNIDENTIFIABLE,0,0,2009,0,0,0,0 +UNINTENTIONAL,2009,0,0,0,0,0,0 +UNJUSTIFIABLY,2009,0,0,0,0,0,0 +UNKNOWINGLY,2009,0,0,0,0,0,0 +UNLAWFULLY,2009,0,0,2009,0,0,0 +UNMARKETABLE,2009,0,0,0,0,0,0 +UNNECESSARILY,2009,0,0,0,0,0,0 +UNOBTAINABLE,2009,0,0,0,0,0,0 +UNPERFORMED,2009,0,0,0,0,0,0 +UNPREDICTABLE,2009,0,2009,0,0,0,0 +UNPROFITABILITY,2011,0,0,0,0,0,0 +UNQUALIFIED,2009,0,0,0,0,0,0 +UNREASONABLE,2009,0,0,0,0,0,0 +UNRECONCILED,0,0,2011,0,0,0,0 +UNRELIABLE,2009,0,0,0,0,0,0 +UNRESOLVED,2009,0,0,0,0,0,0 +UNSALEABLE,2009,0,0,0,0,0,0 +UNSCHEDULED,2009,0,0,0,0,0,0 +UNSETTLED,0,0,2009,0,0,0,0 +UNSPECIFIED,0,0,2009,0,0,0,0 +UNSUBSTANTIATED,2009,0,0,0,0,0,0 +UNSUITABLE,2009,0,0,0,0,0,0 +UNSURPASSED,0,2009,0,0,2009,0,0 +UNTENABLE,2009,0,0,0,0,0,0 +UNTRUSTED,2014,0,0,0,0,0,0 +UNTRUTHFULNESS,2009,0,0,0,0,0,0 +UNUSUALLY,0,0,2009,0,0,0,0 +UNWILLING,2009,0,0,0,0,0,0 +UPTURN,0,2009,0,0,0,0,0 +USURIOUS,2009,0,0,2009,0,0,0 +USURPING,2009,0,0,2009,0,0,0 +VAGUE,0,0,2012,0,0,0,0 +VAGUER,0,0,2012,0,0,0,0 +PLAINTIFFS,2009,0,0,2009,0,0,0 +PLEADING,2009,0,0,2009,0,0,0 +PLEASANT,0,2009,0,0,0,0,0 +PLED,2009,0,0,0,0,0,0 +PLEDGEES,0,0,0,2011,0,0,0 +PLEDGORS,0,0,0,2012,0,0,0 +COMPELLING,0,0,0,0,0,0,2011 +COMPLAINANT,0,0,0,2009,0,0,0 +COMPLAINS,2009,0,0,0,0,0,0 +COMMITS,0,0,0,0,0,0,2009 +LIKELIHOOD,0,0,2009,0,0,0,0 +LIMITING,0,0,0,0,0,0,2009 +RESTRAIN,0,0,0,0,0,0,2009 +RESTRAINT,0,0,0,0,0,0,2009 +RESTRICTING,0,0,0,0,0,0,2009 +RESTRICTIVELY,0,0,0,0,0,0,2009 +RESTRUCTURED,2009,0,0,0,0,0,0 +RETALIATE,2009,0,0,0,0,0,0 +RETALIATION,2009,0,0,0,0,0,0 +PRECAUTION,0,0,2009,0,0,0,0 +PRECIPITOUS,2009,0,0,0,0,0,0 +PRECLUDES,2009,0,0,0,0,0,2009 +PREDATORY,2009,0,0,0,0,0,0 +PREDECEASING,0,0,0,2009,0,0,0 +PREDICTING,0,0,2009,0,0,0,0 +PREDICTOR,0,0,2009,0,0,0,0 +PREEMINENT,0,2009,0,0,0,0,0 +PREJUDICES,2009,0,0,2009,0,0,0 +PRELIMINARY,0,0,2009,0,0,0,0 +PREMIERE,0,2009,0,0,0,0,0 +PRESTIGE,0,2009,0,0,0,0,0 +PRESUMED,0,0,2009,0,0,0,0 +PRESUMPTIONS,0,0,2009,0,0,0,0 +PREVENTED,0,0,0,0,0,0,2009 +PRIMA,0,0,0,2009,0,0,0 +PROBABILISTIC,0,0,2009,0,0,0,0 +PROBABLY,0,0,2009,0,0,0,0 +PROBATING,0,0,0,2009,0,0,0 +PROBATIONER,0,0,0,2009,0,0,0 +PROBLEMATIC,2009,0,0,0,0,0,0 +PROFICIENT,0,2009,0,0,0,0,0 +PROFITABLY,0,2009,0,0,0,0,0 +PROGRESSING,0,2009,0,0,0,0,0 +PROHIBITION,0,0,0,0,0,0,2009 +PROHIBITORY,0,0,0,0,0,0,2009 +PROLONGATIONS,2009,0,0,0,0,0,0 +PROMULGATE,0,0,0,2009,0,0,0 +PROMULGATION,0,0,0,2009,0,0,0 +PRONE,2009,0,0,0,0,0,0 +PROSECUTED,2009,0,0,2009,0,0,0 +PROSECUTIONS,2009,0,0,2009,0,0,0 +PROSPERED,0,2009,0,0,0,0,0 +PROSPERS,0,2009,0,0,0,0,0 +PROTESTERS,2009,0,0,0,0,0,0 +PROTESTS,2009,0,0,0,0,0,0 +PROVISOES,0,0,0,2009,0,0,0 +PROVOKES,2009,0,0,0,0,0,0 +PUNISHES,2009,0,0,0,0,0,0 +PUNITIVE,2009,0,0,0,0,0,0 +PURPORTING,2009,0,0,0,0,0,0 +QUESTIONABLY,2009,0,0,0,0,0,0 +QUIT,2009,0,0,0,0,0,0 +RACKETEER,2009,0,0,0,0,0,0 +RANDOMIZED,0,0,2009,0,0,0,0 +RANDOMNESS,0,0,2009,0,0,0,0 +RATIONALIZATION,2009,0,0,0,0,0,0 +RATIONALIZES,2009,0,0,0,0,0,0 +ABANDONMENT,2009,0,0,0,0,0,0 +ABDICATES,2009,0,0,0,0,0,0 +ABERRANT,2009,0,0,0,0,0,0 +ABETTING,2009,0,0,0,0,0,0 +ABIDING,0,0,0,0,0,0,2009 +ABNORMALITY,2009,0,0,0,0,0,0 +ABOLISHES,2009,0,0,0,0,0,0 +ABROGATED,2009,0,0,2009,0,0,0 +ABROGATIONS,2009,0,0,2009,0,0,0 +ABSENCE,2009,0,0,0,0,0,0 +ABSOLVED,0,0,0,2009,0,0,0 +ABUNDANT,0,2009,0,0,0,0,0 +ABUSING,2009,0,0,0,0,0,0 +ACCESSION,0,0,0,2009,0,0,0 +ACCIDENTALLY,2009,0,0,0,0,0,0 +ACCOMPLISHED,0,2009,0,0,0,0,0 +ACCOMPLISHMENTS,0,2009,0,0,0,0,0 +ACCUSED,2009,0,0,0,0,0,0 +ACHIEVED,0,2009,0,0,0,0,0 +ACHIEVING,0,2009,0,0,0,0,0 +ACQUIESCING,2009,0,0,0,0,0,0 +ACQUITS,2009,0,0,2009,0,0,0 +ACQUITTANCES,0,0,0,2011,0,0,0 +ADEQUATELY,0,2009,0,0,0,0,0 +ADJOURNMENT,0,0,0,2009,0,0,0 +ADJUDGED,0,0,0,2009,0,0,0 +ADJUDICATED,0,0,0,2009,0,0,0 +ADJUDICATIONS,0,0,0,2009,0,0,0 +ADJUDICATORY,0,0,0,2009,0,0,0 +ADMISSION,0,0,0,2009,0,0,0 +ADULTERATING,2009,0,0,0,0,0,0 +ADVANCEMENTS,0,2009,0,0,0,0,0 +ADVANTAGED,0,2009,0,0,0,0,0 +ADVERSARIAL,2009,0,0,0,0,0,0 +ADVERSELY,2009,0,0,0,0,0,0 +AFFIDAVITS,0,0,0,2009,0,0,0 +AFOREMENTIONED,0,0,0,2009,0,0,0 +AFTERMATHS,2009,0,0,0,0,0,0 +AGGRAVATES,2009,0,0,0,0,0,0 +AGGRIEVED,0,0,0,2009,0,0,0 +ALIENATED,2009,0,0,0,0,0,0 +ALIENATIONS,2009,0,0,0,0,0,0 +ALLEGED,2009,0,0,2009,0,0,0 +ALLIANCE,0,2009,0,0,0,0,0 +ALTERATIONS,0,0,2009,0,0,0,0 +AMBIGUOUS,0,0,2009,0,0,0,0 +AMENDED,0,0,0,2009,0,0,0 +AMENDS,0,0,0,2009,0,0,0 +ANNOYED,2009,0,0,0,0,0,0 +ANNULLED,2009,0,0,0,0,0,0 +ANNULS,2009,0,0,0,0,0,0 +ANOMALY,2009,0,2009,0,0,0,0 +ANTICIPATED,0,0,2009,0,0,0,0 +ANTICIPATIONS,0,0,2009,0,0,0,0 +ANYWISE,0,0,0,2009,0,0,0 +APPEALABLE,0,0,0,2009,0,0,0 +APPEAR,0,0,2009,0,0,0,0 +APPELLANT,0,0,0,2009,0,0,0 +APPOINTOR,0,0,0,2011,0,0,0 +APPROXIMATES,0,0,2009,0,0,0,0 +APPURTENANCE,0,0,0,2009,0,0,0 +ARBITRAL,0,0,0,2009,0,0,0 +ARBITRATE,0,0,0,2009,0,0,0 +ARBITRATION,0,0,0,2009,0,0,0 +ARBITRATOR,0,0,0,2009,0,0,0 +ARGUING,2009,0,0,0,0,0,0 +ARREARAGE,2009,0,0,2009,0,0,0 +ARRESTED,2009,0,0,0,0,0,0 +RETROCEDE,0,0,0,2011,0,0,0 +BOLSTERED,0,2009,0,0,0,0,0 +BONAFIDE,0,0,0,2009,0,0,0 +BOOSTED,0,2009,0,0,0,0,0 +BOUNDED,0,0,0,0,0,0,2009 +BOYCOTTS,2009,0,0,0,0,0,0 +BREACHING,2009,0,0,2009,0,0,0 +BREAKDOWN,2009,0,0,0,0,0,0 +BREAKTHROUGH,0,2009,0,0,0,0,0 +BRIBERIES,2009,0,0,0,0,0,0 +BRIDGE,-2020,0,0,0,0,0,0 +BURDENED,2009,0,0,0,0,0,0 +BURNED,2009,0,0,0,0,0,0 +CANCEL,2009,0,0,0,0,0,0 +CANCELLATIONS,2009,0,0,0,0,0,0 +CARELESS,2009,0,0,0,0,0,0 +CATASTROPHE,2009,0,0,0,0,0,0 +CAUTION,2009,0,0,0,0,0,0 +CAUTIONS,2009,0,0,0,0,0,0 +CEASE,2009,0,0,0,0,0,0 +CEDANT,0,0,0,2012,0,0,0 +CENSURES,2009,0,0,0,0,0,0 +CHALLENGE,2009,0,0,0,0,0,0 +CHARGEOFFS,2009,0,0,0,0,0,0 +CHOATE,0,0,0,2011,0,0,0 +CIRCUMVENTION,2009,0,0,0,0,0,0 +SHOCKED,2009,0,0,0,0,0,0 +SHORTFALLS,2009,0,0,0,0,0,0 +SHUTDOWN,2009,0,0,0,0,0,0 +SLANDER,2009,0,0,0,0,0,0 +REPUDIATED,2009,0,0,0,0,0,0 +REPUDIATIONS,2009,0,0,0,0,0,0 +REQUIRED,0,0,0,0,0,0,2009 +REQUIRING,0,0,0,0,0,0,2009 +RESCINDING,0,0,0,2009,0,0,0 +RESIGN,2009,0,0,0,0,0,0 +RESIGNING,2009,0,0,0,0,0,0 +RESTATED,2009,0,0,0,0,0,0 +RESTATING,2009,0,0,0,0,0,0 +NONUSURIOUS,0,0,0,2011,0,0,0 +NOTARIZATIONS,0,0,0,2009,0,0,0 +NOTARY,0,0,0,2009,0,0,0 +NUISANCES,2009,0,0,0,0,0,0 +NULLIFIES,2009,0,0,2009,0,0,0 +NULLITY,0,0,0,2009,0,0,0 +OBJECTIONABLE,2009,0,0,0,0,0,0 +OBLIGATED,0,0,0,0,0,0,2009 +OBLIGATIONS,0,0,0,0,0,0,2009 +OBLIGEE,0,0,0,2009,0,0,0 +OBLIGORS,0,0,0,2009,0,0,0 +OBSOLETE,2009,0,0,0,0,0,0 +OBSTRUCTED,2009,0,0,0,0,0,0 +OCCASIONALLY,0,0,2009,0,0,2009,0 +OFFENDED,2009,0,0,0,0,0,0 +OFFENDS,2009,0,0,0,0,0,0 +OFFEROR,0,0,0,2009,0,0,0 +OMIT,2009,0,0,0,0,0,0 +ONEROUS,2009,0,0,0,0,0,0 +OPPORTUNITY,0,2009,0,0,0,0,0 +OPPOSING,2009,0,0,0,0,0,0 +OPTIONEE,0,0,0,2009,0,0,0 +OUTAGES,2009,0,0,0,0,0,0 +OUTPERFORMED,0,2009,0,0,0,0,0 +OVERAGES,2009,0,0,0,0,0,0 +OVERBUILT,2009,0,0,0,0,0,0 +OVERCAPACITIES,2009,0,0,0,0,0,0 +OVERCHARGES,2009,0,0,0,0,0,0 +OVERCOMING,2009,0,0,0,0,0,0 +OVERESTIMATES,2009,0,0,0,0,0,0 +OVERLOAD,2009,0,0,0,0,0,0 +OVERLOOK,2009,0,0,0,0,0,0 +OVERPAID,2009,0,0,0,0,0,0 +OVERPRODUCES,2009,0,0,0,0,0,0 +OVERRULED,0,0,0,2009,0,0,0 +OVERRUNNING,2009,0,0,0,0,0,0 +OVERSHADOWING,2009,0,0,0,0,0,0 +OVERSTATEMENT,2009,0,0,0,0,0,0 +OVERSUPPLIED,2009,0,0,0,0,0,0 +OVERTLY,2009,0,0,0,0,0,0 +OVERTURNS,2009,0,0,0,0,0,0 +PANIC,2009,0,0,0,0,0,0 +NEARLY,0,0,2009,0,0,2009,0 +NECESSITATING,0,0,0,0,0,0,2009 +NEGLECT,2009,0,0,0,0,0,0 +NEGLECTS,2009,0,0,0,0,0,0 +NEGLIGENTLY,2009,0,0,0,0,0,0 +BARRIER,2009,0,0,0,0,0,0 +BELIEVE,0,0,2009,0,0,0,0 +IDLED,2009,0,0,0,0,0,0 +IGNORES,2009,0,0,0,0,0,0 +ILLEGALITIES,2009,0,0,0,0,0,0 +ILLICIT,2009,0,0,0,0,0,0 +IMBALANCE,2009,0,0,0,0,0,0 +IMMORAL,2009,0,0,0,0,0,0 +IMPAIRMENT,2009,0,0,0,0,0,2011 +IMPASSES,2009,0,0,0,0,0,0 +IMPEDIMENT,2009,0,0,0,0,0,0 +IMPERATIVE,2009,0,0,0,0,0,0 +IMPERMISSIBLE,2009,0,0,0,0,0,0 +IMPLICATES,2009,0,0,0,0,0,0 +IMPOSES,0,0,0,0,0,0,2009 +IMPOSSIBILITY,2009,0,0,0,0,0,0 +IMPOUNDING,2009,0,0,0,0,0,0 +IMPRACTICALITIES,2009,0,0,0,0,0,0 +IMPRECISIONS,0,0,2009,0,0,0,0 +IMPRESSING,0,2009,0,0,0,0,0 +IMPROBABILITY,0,0,2009,0,0,0,0 +IMPROPRIETIES,2009,0,0,0,0,0,0 +IMPROVEMENT,0,2009,0,0,0,0,0 +IMPRUDENT,2009,0,0,0,0,0,0 +INACCURACIES,2009,0,0,0,0,0,0 +INACTION,2009,0,0,0,0,0,0 +INACTIVATES,2009,0,0,0,0,0,0 +INACTIVITY,2009,0,0,0,0,0,0 +INADEQUATELY,2009,0,0,0,0,0,0 +INADVISABLE,2009,0,0,0,0,0,0 +INATTENTION,2009,0,0,0,0,0,0 +INCARCERATE,2009,0,0,2009,0,0,0 +INCARCERATION,2009,0,0,2009,0,0,0 +INCIDENCES,2009,0,0,0,0,0,0 +INCOMPATIBILITY,2009,0,0,0,0,0,0 +INCOMPETENT,2009,0,0,0,0,0,0 +INCOMPLETELY,2009,0,0,0,0,0,0 +INCONSISTENCY,2009,0,0,0,0,0,0 +INCONTESTABLE,0,0,0,2009,0,0,0 +INCORRECT,2009,0,0,0,0,0,0 +INCREDIBLY,0,2009,0,0,0,0,0 +INDEFEASIBLE,2009,0,0,0,0,0,0 +INDEFINITENESS,0,0,2009,0,0,0,0 +INDEMNIFIED,0,0,0,2009,0,0,0 +INDEMNITEE,0,0,0,2009,0,0,0 +INDEMNITORS,0,0,0,2009,0,0,0 +INDICT,2009,0,0,2009,0,0,0 +INDICTMENT,2009,0,0,2009,0,0,0 +INEFFECTIVELY,2009,0,0,0,0,0,0 +INEFFICIENT,2009,0,0,0,0,0,0 +INEQUITABLE,2009,0,0,0,0,0,0 +INEVITABLE,2009,0,0,0,0,0,0 +INEXPERIENCED,2009,0,0,0,0,0,0 +INFORCE,0,0,0,2009,0,0,0 +INFRINGE,2009,0,0,0,0,0,0 +INFRINGER,0,0,0,2009,0,0,0 +INHIBIT,0,0,0,0,0,0,2011 +INIMICAL,2009,0,0,0,0,0,0 +INJURE,2009,0,0,0,0,0,0 +INJURING,2009,0,0,0,0,0,0 +INNOVATED,0,2009,0,0,0,0,0 +INNOVATIONS,0,2009,0,0,0,0,0 +INNOVATORS,0,2009,0,0,0,0,0 +INSECURE,2009,0,0,0,0,0,0 +INSISTED,0,0,0,0,0,0,2009 +INSOFAR,0,0,0,2009,0,0,0 +INSPIRATION,0,2009,0,0,0,0,0 +INSUBORDINATION,2009,0,0,0,0,0,0 +INSURRECTION,2009,0,0,0,0,0,0 +INTEGRITY,0,2009,0,0,0,0,0 +INTERFERENCE,2009,0,0,0,0,0,0 +INTERLOCUTORY,0,0,0,2009,0,0,0 +INTERPOSE,0,0,0,2009,0,0,0 +INTERPOSITION,0,0,0,2009,0,0,0 +INTERROGATES,0,0,0,2009,0,0,0 +INTERROGATOR,0,0,0,2009,0,0,0 +INTERRUPT,2009,0,0,0,0,0,0 +INTERRUPTIONS,2009,0,0,0,0,0,0 +INTIMIDATION,2009,0,0,0,0,0,0 +SPORADICALLY,0,0,2009,0,0,0,0 +STABILIZE,0,2009,0,0,0,0,0 +STABLE,0,2009,0,0,0,0,0 +STAGNATED,2009,0,0,0,0,0,0 +STANDSTILL,2009,0,0,0,0,0,0 +STATUTORILY,0,0,0,2009,0,0,0 +STIPULATES,0,0,0,0,0,0,2009 +STOLEN,2009,0,0,0,0,0,0 +STOPPING,2009,0,0,0,0,0,0 +STRAINING,2009,0,0,0,0,0,0 +STRENGTHENED,0,2009,0,0,0,0,0 +STRESS,2009,0,0,0,0,0,0 +STRESSING,2009,0,0,0,0,0,0 +STRICTLY,0,0,0,0,0,0,2009 +STRONGEST,0,2009,0,0,0,0,0 +SUBDOCKET,0,0,0,2012,0,0,0 +SUBLEASEE,0,0,0,2011,0,0,0 +SUBLICENSOR,0,0,0,2011,0,0,0 +SUBPOENAED,2009,0,0,2009,0,0,0 +SUBSTANDARD,2009,0,0,0,0,0,0 +SUCCEEDED,0,2009,0,0,0,0,0 +SUCCESSES,0,2009,0,0,0,0,0 +SUDDENLY,0,0,2009,0,0,0,0 +SUFFER,2009,0,0,0,0,0,0 +SUGGEST,0,0,2009,0,0,2009,0 +SUING,2009,0,0,2009,0,0,0 +SUMMONSES,2009,0,0,2009,0,0,0 +SUPERSEDED,0,0,0,2009,0,0,0 +SURETY,0,0,0,2009,0,0,0 +SURPASSING,0,2009,0,0,0,0,0 +SUSPECTED,2009,0,0,0,0,0,0 +SUSPENDING,2009,0,0,0,0,0,0 +SUSPICION,2009,0,0,0,0,0,0 +REVISED,0,0,2009,0,0,0,0 +REVOKE,2009,0,0,0,0,0,0 +REVOLUTIONIZE,0,2009,0,0,0,0,0 +REWARD,0,2009,0,0,0,0,0 +RIDICULE,2009,0,0,0,0,0,0 +RISK,0,0,2009,0,0,0,0 +RISKINESS,0,0,2009,0,0,0,0 +ROUGHLY,0,0,2009,0,0,0,0 +SABOTAGE,2009,0,0,0,0,0,0 +SACRIFICIAL,2009,0,0,0,0,0,0 +SATISFACTORY,0,2009,0,0,0,0,0 +SATISFYING,0,2009,0,0,0,0,0 +SCRUTINIZED,2009,0,0,0,0,0,0 +SECRECY,-2020,0,0,0,0,0,0 +SEIZES,2009,0,0,0,0,0,0 +SENTENCED,2009,0,0,2009,0,0,0 +SERIOUSLY,2009,0,0,0,0,0,0 +SETTLEMENT,0,0,0,2009,0,0,0 +SEVERABLE,0,0,0,2009,0,0,0 +SEVERE,2009,0,0,0,0,0,0 +SEVERITY,2009,0,0,0,0,0,0 +ENTHUSIASTIC,0,2009,0,0,0,0,0 +ERODE,2009,0,0,0,0,0,0 +EROSION,2009,0,0,0,0,0,0 +ERRING,2009,0,0,0,0,0,0 +ERRORS,2009,0,0,0,0,0,0 +ESCALATES,2009,0,0,0,0,0,0 +ESCHEATMENT,0,0,0,2011,0,0,0 +ESCROWS,0,0,0,0,0,0,2009 +EVADES,2009,0,0,0,0,0,0 +EVASIVE,2009,0,0,0,0,0,0 +EVICTION,2009,0,0,0,0,0,0 +EVIDENTIARY,0,0,0,2009,0,0,0 +EXACERBATING,2009,0,0,0,0,0,0 +EXAGGERATED,2009,0,0,0,0,0,0 +EXCEEDANCE,0,0,0,2011,0,0,0 +EXCELLENT,0,2009,0,0,0,0,0 +EXCEPTIONALLY,0,2009,0,0,0,0,0 +EXCITED,0,2009,0,0,0,0,0 +EXCLUSIVELY,0,2009,0,0,0,0,0 +EXCULPATE,2009,0,0,2009,0,0,0 +EXCULPATION,2009,0,0,2009,0,0,0 +EXECUTORS,0,0,0,2009,0,0,0 +EXECUTRIXES,0,0,0,2009,0,0,0 +EXONERATES,2009,0,0,0,0,0,0 +EXPLOIT,2009,0,0,0,0,0,0 +EXPLOITED,2009,0,0,0,0,0,0 +EXPOSED,2009,0,0,0,0,0,0 +EXPOSURES,0,0,2009,0,0,0,0 +EXPROPRIATING,2009,0,0,0,0,0,0 +EXPULSIONS,2009,0,0,0,0,0,0 +EXTRAJUDICIAL,0,0,0,2014,0,0,0 +SOLVES,0,2009,0,0,0,0,0 +SOMEWHAT,0,0,2009,0,0,2009,0 +SPAMMING,2014,0,0,0,0,0,0 +TREMENDOUSLY,0,2009,0,0,0,0,0 +LOCKOUTS,2009,0,0,0,0,0,0 +LOSS,2009,0,0,0,0,0,0 +LOYAL,0,2009,0,0,0,0,0 +MALFEASANCE,2009,0,0,0,0,0,0 +MALFUNCTIONS,2009,0,0,0,0,0,0 +MALPRACTICE,2009,0,0,0,0,0,0 +MANDATES,0,0,0,0,0,0,2009 +MANIPULATE,2009,0,0,0,0,0,0 +MANIPULATION,2009,0,0,0,0,0,0 +MARKDOWNS,2009,0,0,0,0,0,0 +MEDIATED,0,0,0,2009,0,0,0 +MEDIATIONS,0,0,0,2009,0,0,0 +MIGHT,0,0,2009,0,0,2009,0 +MISAPPLIES,2009,0,0,0,0,0,0 +MISAPPROPRIATED,2009,0,0,0,0,0,0 +MISAPPROPRIATIONS,2009,0,0,0,0,0,0 +MISCALCULATES,2009,0,0,0,0,0,0 +MISCHARACTERIZATION,2014,0,0,0,0,0,0 +MISCLASSIFIED,2011,0,0,0,0,0,0 +MISDATED,2011,0,0,0,0,0,0 +MISFEASANCE,0,0,0,2009,0,0,0 +MISHANDLING,2009,0,0,0,0,0,0 +MISINFORMING,2009,0,0,0,0,0,0 +MISINTERPRETATIONS,2009,0,0,0,0,0,0 +MISJUDGE,2009,0,0,0,0,0,0 +MISJUDGMENT,2009,0,0,0,0,0,0 +MISLABELING,2009,0,0,0,0,0,0 +MISLEADING,2009,0,0,0,0,0,0 +MISMANAGE,2009,0,0,0,0,0,0 +MISMANAGING,2009,0,0,0,0,0,0 +MISMATCHING,2009,0,0,0,0,0,0 +MISPRICINGS,2014,0,0,0,0,0,0 +MISREPRESENTED,2009,0,0,0,0,0,0 +MISSED,2009,0,0,0,0,0,0 +WORRIES,2009,0,0,0,0,0,0 +WORSEN,2009,0,0,0,0,0,0 +WORST,2009,0,0,0,0,0,0 +TIGHTENING,2009,0,0,0,0,0,0 +TOLERATING,2009,0,0,0,0,0,0 +TORTIOUSLY,0,0,0,2009,0,0,0 +FINES,2009,0,0,0,0,0,0 +NONASSESSABLE,0,0,2009,0,0,0,0 +NONCANCELLABLE,0,0,0,0,0,0,2009 +NONCOMPLIANT,2009,0,0,0,0,0,0 +NONCONFORMITY,2009,0,0,0,0,0,0 +NONCONTRIBUTORY,0,0,0,2009,0,0,0 +NONFORFEITABILITY,0,0,0,2011,0,0,0 +NONGUARANTOR,0,0,0,2011,0,0,0 +DISCIPLINARY,2009,0,0,0,0,0,0 +DISCLAIMERS,2009,0,0,0,0,0,0 +DISCLOSED,2009,0,0,0,0,0,0 +DISCONTINUANCES,2009,0,0,0,0,0,0 +DISCONTINUED,2009,0,0,0,0,0,0 +DISCOURAGED,2009,0,0,0,0,0,0 +DISCREDITED,2009,0,0,0,0,0,0 +DISCREPANCY,2009,0,0,0,0,0,0 +SPECULATED,0,0,2009,0,0,0,0 +SPECULATIONS,0,0,2009,0,0,0,0 +TRAGICALLY,2009,0,0,0,0,0,0 +LEGISLATED,0,0,0,2009,0,0,0 +LEGISLATIONS,0,0,0,2009,0,0,0 +LEGISLATORS,0,0,0,2009,0,0,0 +LIBELED,0,0,0,2009,0,0,0 +LIE,2009,0,0,0,0,0,0 +COMPULSION,2009,0,0,0,0,0,2009 +CONCEDE,2009,0,0,0,0,0,0 +CONCEIVABLE,0,0,2009,0,0,2009,0 +CONCERNS,2009,0,0,0,0,0,0 +CONCLUSIVE,0,2009,0,0,0,0,0 +CONDEMNATIONS,2009,0,0,0,0,0,0 +CONDEMNS,2009,0,0,0,0,0,0 +CONDONED,2009,0,0,0,0,0,0 +CONFESSES,2009,0,0,0,0,0,0 +CONFINE,2009,0,0,0,0,0,2009 +CONFINES,2009,0,0,0,0,0,2009 +CONFISCATES,2009,0,0,0,0,0,0 +CONFISCATORY,0,0,0,2009,0,0,0 +CONFLICTS,2009,0,0,0,0,0,0 +CONFRONTATIONS,2009,0,0,0,0,0,0 +CONFUSE,2009,0,0,0,0,0,0 +CONFUSINGLY,2009,0,2009,0,0,0,0 +CONSENTING,0,0,0,2009,0,0,0 +CONSPIRACY,2009,0,0,0,0,0,0 +CONSPIRE,2009,0,0,0,0,0,0 +CONSTITUTION,0,0,0,2009,0,0,0 +CONSTITUTIONS,0,0,0,2009,0,0,0 +CONSTRAINING,0,0,0,0,0,0,2009 +CONSTRUCTIVE,0,2009,0,0,0,0,0 +CONSTRUES,0,0,0,2012,0,0,0 +CONTENDED,2009,0,0,0,0,0,0 +CONTENTIONS,2009,0,0,0,0,0,0 +CONTESTATION,0,0,0,2011,0,0,0 +CONTINGENCY,0,0,2009,0,0,0,0 +CONTRACT,0,0,0,2009,0,0,0 +CONTRACTIBLE,0,0,0,2009,0,0,0 +CONTRACTIONS,2009,0,0,0,0,0,0 +CONTRADICT,2009,0,0,0,0,0,0 +CONTRADICTIONS,2009,0,0,0,0,0,0 +CONTRAVENE,0,0,0,2009,0,0,0 +CONTRAVENTION,0,0,0,2009,0,0,0 +CONTROVERSY,2009,0,0,0,0,0,0 +CONVENIENS,0,0,0,2012,0,0,0 +CONVICTED,2009,0,0,2009,0,0,0 +CORRECTED,2009,0,0,0,0,0,0 +CORRECTS,2009,0,0,0,0,0,0 +CORRUPTION,2009,0,0,0,0,0,0 +COSTLY,2009,0,0,0,0,0,0 +COUNSELED,0,0,0,2009,0,0,0 +COUNTERCLAIMED,2009,0,0,0,0,0,0 +COUNTERFEITED,2009,0,0,0,0,0,0 +COUNTERFEITS,2009,0,0,0,0,0,0 +COUNTERSUED,0,0,0,2011,0,0,0 +COURTEOUS,0,2009,0,0,0,0,0 +COVENANTED,0,0,0,0,0,0,2011 +CREATIVELY,0,2009,0,0,0,0,0 +CRIMES,2009,0,0,2009,0,0,0 +CRIMINALIZING,0,0,0,2014,0,0,0 +CRISIS,2009,0,0,0,0,0,0 +CRITICISMS,2009,0,0,0,0,0,0 +CRITICIZING,2009,0,0,0,0,0,0 +CROSSROADS,0,0,2009,0,0,0,0 +CULPABLE,2009,0,0,0,0,0,0 +CURTAILED,2009,0,0,0,0,0,0 +CURTAILS,2009,0,0,0,0,0,0 +CYBERATTACK,2014,0,0,0,0,0,0 +CYBERCRIMES,2014,0,0,0,0,0,0 +TAINTS,2009,0,0,0,0,0,0 +TENSE,2009,0,0,0,0,0,0 +TERMINATE,2009,0,0,0,0,0,0 +TERMINATION,2009,0,0,0,0,0,0 +TESTIFY,2009,0,0,2009,0,0,0 +THENCEFORTH,0,0,0,2009,0,0,0 +THEREFROM,0,0,0,2009,0,0,0 +THEREON,0,0,0,2009,0,0,0 +THERETOFORE,0,0,0,2009,0,0,0 +THEREWITH,0,0,0,2009,0,0,0 +THREATENING,2009,0,0,0,0,0,0 +FACIE,0,0,0,2009,0,0,0 +FAILING,2009,0,0,0,0,0,0 +FAILURES,2009,0,0,0,0,0,0 +FALSIFICATION,2009,0,0,0,0,0,0 +FALSIFY,2009,0,0,0,0,0,0 +FATALITIES,2009,0,0,0,0,0,0 +FAULTED,2009,0,0,0,0,0,0 +FAVORABLY,0,2009,0,0,0,0,0 +FAVORITES,0,2009,0,0,0,0,0 +FELONIOUS,2009,0,0,2009,0,0,0 +DAMAGES,2009,0,0,0,0,0,0 +DANGER,2009,0,0,0,0,0,0 +DEADLOCK,2009,0,0,0,0,0,0 +DEADWEIGHT,2009,0,0,0,0,0,0 +DEBARRED,2009,0,0,0,0,0,0 +DECEIT,2009,0,0,0,0,0,0 +DECEIVED,2009,0,0,0,0,0,0 +DECEPTIONS,2009,0,0,0,0,0,0 +DECLINE,2009,0,0,0,0,0,0 +DECREE,0,0,0,2009,0,0,0 +DEFACE,2009,0,0,0,0,0,0 +DEFALCATIONS,0,0,0,2009,0,0,0 +DEFAME,2009,0,0,0,0,0,0 +DEFAULT,2009,0,0,0,0,0,0 +DEFEASANCE,0,0,0,2009,0,0,0 +DEFEASEMENT,0,0,0,2014,0,0,0 +DEFEATED,2009,0,0,0,0,0,0 +DEFECTIVE,2009,0,0,0,0,0,0 +DEFENDABLE,0,0,0,2014,0,0,0 +DEFENDING,2009,0,0,0,0,0,0 +DEFERENCE,0,0,0,2009,0,0,0 +DEFICIT,2009,0,0,0,0,0,0 +DEFRAUD,2009,0,0,0,0,0,0 +DEFUNCT,2009,0,0,0,0,0,0 +DEGRADED,2009,0,0,0,0,0,0 +DELAYED,2009,0,0,0,0,0,0 +DELEGATABLE,0,0,0,2011,0,0,0 +DELIBERATE,2009,0,0,0,0,0,0 +DELIGHTED,0,2009,0,0,0,0,0 +DELIGHTS,0,2009,0,0,0,0,0 +DELINQUENTLY,2009,0,0,0,0,0,0 +DELISTING,2009,0,0,0,0,0,0 +DEMISES,2009,0,0,0,0,0,0 +DEMOLISHES,2009,0,0,0,0,0,0 +DEMOTE,2009,0,0,0,0,0,0 +DEMOTION,2009,0,0,0,0,0,0 +DEMURRERS,0,0,0,2009,0,0,0 +DENIALS,2009,0,0,0,0,0,0 +DENIGRATED,2009,0,0,0,0,0,0 +DENY,2009,0,0,0,0,0,0 +DEPENDABLE,0,2009,0,0,0,0,0 +DEPENDED,0,0,2009,0,0,2009,0 +DEPENDENT,0,0,2009,0,0,0,2011 +DEPLETED,2009,0,0,0,0,0,0 +DEPLETIONS,2009,0,0,0,0,0,0 +DEPOSING,0,0,0,2009,0,0,0 +INVALID,2009,0,0,0,0,0,0 +INVALIDATING,2009,0,0,0,0,0,0 +INVENTED,0,2009,0,0,0,0,0 +INVENTIVE,0,2009,0,0,0,0,0 +INVESTIGATE,2009,0,0,0,0,0,0 +INVESTIGATION,2009,0,0,0,0,0,0 +IRRECONCILABLE,2009,0,0,0,0,0,0 +IRREGULAR,2009,0,0,0,0,0,0 +IRREPARABLE,2009,0,0,0,0,0,0 +IRREVOCABLE,0,0,0,2009,0,0,2009 +JOINDER,0,0,0,2009,0,0,0 +JUDICIARY,0,0,0,2009,0,0,0 +JURISDICTIONAL,0,0,0,2009,0,0,0 +JURIST,0,0,0,2009,0,0,0 +JURY,0,0,0,2009,0,0,0 +JUSTIFIABLE,2009,0,0,0,0,0,0 +DILIGENTLY,0,2009,0,0,0,0,0 +DIMINISHING,2009,0,0,0,0,0,0 +DISADVANTAGE,2009,0,0,0,0,0,0 +DISAFFILIATION,2009,0,0,2009,0,0,0 +DISAFFIRMS,0,0,0,2011,0,0,0 +DISAGREEING,2009,0,0,0,0,0,0 +DISALLOW,2009,0,0,0,0,0,0 +DISALLOWING,2009,0,0,0,0,0,0 +DISAPPEARANCES,2009,0,0,0,0,0,0 +DISAPPOINT,2009,0,0,0,0,0,0 +DISAPPOINTMENT,2009,0,0,0,0,0,0 +DISAPPROVALS,2009,0,0,0,0,0,0 +DISAPPROVING,2009,0,0,0,0,0,0 +DISASSOCIATIONS,2009,0,0,0,0,0,0 +DISASTROUSLY,2009,0,0,0,0,0,0 +DISAVOWING,2009,0,0,0,0,0,0 +GREATER,0,-2020,0,0,0,0,0 +GRIEVANCE,2009,0,0,0,0,0,0 +GUILTY,2009,0,0,0,0,0,0 +HAMPERED,2009,0,0,0,0,0,0 +HAPPILY,0,2009,0,0,0,0,0 +HARASSED,2009,0,0,0,0,0,0 +HARDSHIPS,2009,0,0,0,0,0,0 +HARMFULLY,2009,0,0,0,0,0,0 +HARSHER,2009,0,0,0,0,0,0 +HAZARD,2009,0,0,0,0,0,0 +NONJUDICIALLY,0,0,0,2011,0,0,0 +NONPERFORMANCE,2009,0,0,0,0,0,0 +HURT,2009,0,0,0,0,0,0 +DISFAVORED,2009,0,0,0,0,0,0 +DISGORGED,2009,0,0,0,0,0,0 +COMPLICATING,2009,0,0,0,0,0,0 +COMPLIMENTARY,0,2009,0,0,0,0,0 +COMPLY,0,0,0,0,0,0,2009 +MISSTATED,2009,0,0,0,0,0,0 +MISSTATING,2009,0,0,0,0,0,0 +MISTAKEN,2009,0,0,0,0,0,0 +MISTRIAL,2009,0,0,2009,0,0,0 +MISUNDERSTANDINGS,2009,0,0,0,0,0,0 +MISUSES,2009,0,0,0,0,0,0 +MONOPOLIZATION,2009,0,0,0,0,0,0 +MONOPOLIZING,2009,0,0,0,0,0,0 +MORATORIUMS,2009,0,0,0,0,0,0 +MOTIONS,0,0,0,2009,0,0,0 +SLOWDOWN,2009,0,0,0,0,0,0 +SLOWEST,2009,0,0,0,0,0,0 +SLUGGISH,2009,0,0,0,0,0,0 +SMOOTHING,0,2009,0,0,0,0,0 +DEPRESS,2009,0,0,0,0,0,0 +DEPRIVATION,2009,0,0,0,0,0,0 +DEPRIVING,2009,0,0,0,0,0,0 +DEROGATED,0,0,0,2009,0,0,0 +DEROGATIONS,0,0,0,2009,0,0,0 +DESIRED,0,2009,0,0,0,0,0 +DESTABILIZE,2009,0,0,0,0,0,0 +DESTROY,2009,0,0,0,0,0,0 +DESTRUCTION,2009,0,0,0,0,0,0 +DETAINER,0,0,0,2009,0,0,0 +DETERIORATE,2009,0,0,0,0,0,0 +DETERIORATION,2009,0,0,0,0,0,0 +DETERRENCES,2009,0,0,0,0,0,0 +DETERS,2009,0,0,0,0,0,0 +DETRIMENT,2009,0,0,0,0,0,0 +DEVALUE,2009,0,0,0,0,0,0 +DEVASTATE,2009,0,0,0,0,0,0 +DEVIATE,2009,0,2009,0,0,0,0 +DEVIATION,2009,0,2009,0,0,0,0 +DEVOLVED,2009,0,0,0,0,0,0 +DICTATED,0,0,0,0,0,0,2009 +DIFFERED,0,0,2009,0,0,0,0 +DIFFICULTIES,2009,0,0,0,0,0,0 +ASCENDANCY,0,0,0,2009,0,0,0 +ASSAULTED,2009,0,0,0,0,0,0 +ASSERTIONS,2009,0,0,0,0,0,0 +ASSUME,0,0,2009,0,0,0,0 +ASSUMPTION,0,0,2009,0,0,0,0 +ASSURES,0,2009,0,0,0,0,0 +ATTAINING,0,2009,0,0,0,0,0 +ATTEST,0,0,0,2009,0,0,0 +ATTESTING,0,0,0,2009,0,0,0 +ATTORNMENT,0,0,0,2009,0,0,0 +ATTRITION,2009,0,0,0,0,0,0 +BAIL,2009,0,0,2009,0,0,0 +BAILIFF,0,0,0,2009,0,0,0 +BALK,2009,0,0,0,0,0,0 +BANKRUPTCY,2009,0,0,0,0,0,0 +BANS,2009,0,0,0,0,0,0 +CLAIMABLE,0,0,0,2009,0,0,0 +CLAIMING,2009,0,0,0,0,0,0 +CLAWBACK,2009,0,0,0,0,0,0 +CLOSEOUT,2009,0,0,0,0,0,0 +CLOSURE,2009,0,0,0,0,0,0 +CODICIL,0,0,0,2009,0,0,0 +CODIFIED,0,0,0,2009,0,0,0 +COERCE,2009,0,0,0,0,0,0 +COERCION,2009,0,0,0,0,0,0 +COLLABORATING,0,2009,0,0,0,0,0 +COLLABORATOR,0,2009,0,0,0,0,0 +COLLAPSES,2009,0,0,0,0,0,0 +COLLUDE,2009,0,0,0,0,0,0 +COLLUSION,2009,0,0,2009,0,0,0 +POSING,2009,0,0,0,0,0,0 +POSSIBILITIES,0,0,2009,0,0,0,0 +POSTCLOSING,0,0,0,2011,0,0,0 +POSTPONE,2009,0,0,0,0,0,0 +POSTPONES,2009,0,0,0,0,0,0 +WRITEOFF,2009,0,0,0,0,0,0 +WRONGDOING,2009,0,0,0,0,0,0 +WRONGLY,2009,0,0,0,0,0,0 +HONORED,0,2009,0,0,0,0,0 +HOSTILITY,2009,0,0,0,0,0,0 +REASSESSED,0,0,2009,0,0,0,0 +REASSESSMENTS,2009,0,2009,0,0,0,0 +REASSIGNMENT,2009,0,0,0,0,0,0 +REBOUNDED,0,2009,0,0,0,0,0 +REBUTTABLE,0,0,0,2009,0,0,0 +REBUTTED,0,0,0,2009,0,0,0 +RECALCULATES,0,0,2009,0,0,0,0 +RECALL,2009,0,0,0,0,0,0 +RECEPTIVE,0,2009,0,0,0,0,0 +RECKLESS,2009,0,0,0,0,0,0 +RECONSIDERED,0,0,2009,0,0,0,0 +RECOUPABLE,0,0,0,2009,0,0,0 +RECOURSES,0,0,0,2009,0,0,0 +RECUSE,0,0,0,2011,0,0,0 +REDACT,2009,0,0,2009,0,0,0 +REDACTIONS,2009,0,0,2009,0,0,0 +REDRESS,2009,0,0,0,0,0,0 +REEXAMINATION,0,0,2009,0,0,0,0 +REFERENDUM,0,0,0,2009,0,0,0 +REFILES,0,0,0,2009,0,0,0 +REFRAINS,0,0,0,0,0,0,2009 +REFUSED,2009,0,0,0,0,0,0 +REGAINED,0,2009,0,0,0,0,0 +REGULATES,0,0,0,2009,0,0,0 +REGULATIVE,0,0,0,2009,0,0,0 +REHEAR,0,0,0,2009,0,0,0 +VARIABLE,0,0,2009,0,0,0,0 +VARIANCES,0,0,2009,0,0,0,0 +VARIATIONS,0,0,2009,0,0,0,0 +VARYING,0,0,2009,0,0,0,0 +VERDICTS,2009,0,0,2009,0,0,0 +VIATICAL,0,0,0,2011,0,0,0 +VIOLATE,2009,0,0,0,0,0,0 +VIOLATION,2009,0,0,0,0,0,0 +VIOLATORS,2009,0,0,0,0,0,0 +VITIATE,2009,0,0,0,0,0,0 +VITIATION,2009,0,0,0,0,0,0 +VOLATILE,2009,0,2009,0,0,0,0 +VULNERABILITY,2009,0,0,0,0,0,0 +WARNED,2009,0,0,0,0,0,0 +WARRANTEES,0,0,0,2011,0,0,0 +WASTING,2009,0,0,0,0,0,0 +WEAKENING,2009,0,0,0,0,0,0 +WEAKLY,2009,0,0,0,0,0,0 +DISHONESTY,2009,0,0,0,0,0,0 +DISHONORED,2009,0,0,0,0,0,0 +DISLOYAL,2009,0,0,0,0,0,0 +DISMALLY,2009,0,0,0,0,0,0 +DISMISSED,2009,0,0,0,0,0,0 +DISPARAGE,2009,0,0,0,0,0,0 +DISPARAGES,2009,0,0,0,0,0,0 +DISPARITY,2009,0,0,0,0,0,0 +DISPLACEMENTS,2009,0,0,0,0,0,0 +DISPOSITIVE,0,0,0,2009,0,0,0 +DISPOSSESSING,2009,0,0,0,0,0,0 +DISPROPORTIONAL,2009,0,0,0,0,0,0 +DISPUTED,2009,0,0,0,0,0,0 +DISQUALIFICATIONS,2009,0,0,0,0,0,0 +DISQUALIFYING,2009,0,0,0,0,0,0 +DISREGARDS,2009,0,0,0,0,0,0 +DISRUPTED,2009,0,0,0,0,0,0 +DISRUPTIVE,2009,0,0,0,0,0,0 +DISSENT,2009,0,0,0,0,0,0 +DISSENTING,2009,0,0,0,0,0,0 +DISSOLUTION,2009,0,0,0,0,0,0 +DISTINCTIVE,0,2009,0,0,0,0,0 +DISTORTED,2009,0,0,0,0,0,0 +DISTORTS,2009,0,0,0,0,0,0 +DISTRACTION,2009,0,0,0,0,0,0 +DISTRESS,2009,0,0,0,0,0,0 +DISTURB,2009,0,0,0,0,0,0 +DISTURBING,2009,0,0,0,0,0,0 +DIVERTED,2009,0,0,0,0,0,0 +DIVESTED,2009,0,0,0,0,0,0 +DIVESTMENT,2009,0,0,0,0,0,0 +DIVORCED,2009,0,0,0,0,0,0 +DIVULGING,2009,0,0,0,0,0,0 +DOCKETS,0,0,0,2009,0,0,0 +DOUBTFUL,2009,0,2009,0,0,0,0 +DOWNGRADES,2009,0,0,0,0,0,0 +DOWNSIZES,2009,0,0,0,0,0,0 +DOWNTIMES,2009,0,0,0,0,0,0 +DOWNWARDS,2009,0,0,0,0,0,0 +DRAWBACK,2009,0,0,0,0,0,0 +DROUGHT,2009,0,0,0,0,0,0 +DYSFUNCTION,2009,0,0,0,0,0,0 +EARMARKED,0,0,0,0,0,0,2009 +EASILY,0,2009,0,0,0,0,0 +EFFICIENCIES,0,2009,0,0,0,0,0 +EGREGIOUS,2009,0,0,0,0,0,0 +EMBARGOED,2009,0,0,0,0,0,0 +EMBARRASSED,2009,0,0,0,0,0,0 +EMBARRASSMENTS,2009,0,0,0,0,0,0 +EMBEZZLEMENTS,2009,0,0,0,0,0,0 +EMPOWER,0,2009,0,0,0,0,0 +ENABLE,0,2009,0,0,0,0,0 +ENCOURAGED,0,2009,0,0,0,0,0 +ENCROACH,2009,0,0,0,0,0,0 +ENCROACHMENT,2009,0,0,0,0,0,0 +ENCUMBERING,2009,0,0,2009,0,0,2009 +ENCUMBRANCERS,0,0,0,2011,0,0,0 +ENDANGERING,2009,0,0,0,0,0,0 +ENFORCEABILITY,0,0,0,2009,0,0,0 +ENHANCED,0,2009,0,0,0,0,0 +ENHANCING,0,2009,0,0,0,0,0 +ENJOINS,2009,0,0,0,0,0,0 +ENJOYED,0,2009,0,0,0,0,0 +ENTAIL,0,0,0,0,0,0,2009 +PECUNIARILY,0,0,0,2009,0,0,0 +PENALIZING,2009,0,0,0,0,0,0 +PERFECT,0,2009,0,0,0,0,0 +PERHAPS,0,0,2009,0,0,2009,0 +PERMISSIBLE,0,0,0,0,0,0,2009 +PERMITTEE,0,0,0,2011,0,0,0 +PERPETRATED,2009,0,0,2009,0,0,0 +PERSIST,2009,0,0,0,0,0,0 +PERSISTENTLY,2009,0,0,0,0,0,0 +PERVASIVE,2009,0,0,0,0,0,0 +PETITIONED,0,0,0,2009,0,0,0 +PETITIONS,0,0,0,2009,0,0,0 +PICKETING,2009,0,0,0,0,0,0 +HENCEFORWARD,0,0,0,2009,0,0,0 +HEREFOR,0,0,0,2009,0,0,0 +HEREINABOVE,0,0,0,2009,0,0,0 +HEREOF,0,0,0,2009,0,0,0 +HEREUNDER,0,0,0,2009,0,0,0 +HEREWITHIN,0,0,0,2014,0,0,0 +HINDERED,2009,0,0,0,0,0,0 +HINDRANCES,2009,0,0,0,0,0,0 +WHENSOEVER,0,0,0,2009,0,0,0 +WHEREBY,0,0,0,2009,0,0,0 +WHEREON,0,0,0,2009,0,0,0 +WHEREWITH,0,0,0,2009,0,0,0 +WHOSOEVER,0,0,0,2009,0,0,0 +WILLFULLY,2009,0,0,2009,0,0,0 +WINNERS,0,2009,0,0,0,0,0 +FLUCTUATING,0,0,2009,0,0,0,0 +FORBEAR,0,0,0,2009,0,0,0 +FORBEARS,0,0,0,2009,0,0,0 +FORBIDS,2009,0,0,0,0,0,2009 +FOREBEAR,0,0,0,2009,0,0,0 +FORECLOSED,2009,0,0,0,0,0,0 +FORECLOSURES,2009,0,0,0,0,0,0 +FORESTALL,2009,0,0,0,0,0,0 +FORFEIT,2009,0,0,0,0,0,0 +FORFEITING,2009,0,0,0,0,0,0 +FORGERS,2009,0,0,0,0,0,0 +FRAUD,2009,0,0,0,0,0,0 +FRAUDULENTLY,2009,0,0,0,0,0,0 +FRUSTRATE,2009,0,0,0,0,0,0 +FRUSTRATINGLY,2009,0,0,0,0,0,0 +FUGITIVES,2009,0,0,2009,0,0,0 +GAINING,0,2009,0,0,0,0,0 +GRANTORS,0,0,0,2009,0,0,0 +DISGRACE,2009,0,0,0,0,0,0 +LITIGANTS,2009,0,0,2009,0,0,0 +LITIGATING,2009,0,0,2009,0,0,0 +LITIGATORS,0,0,0,2009,0,0,0 +LACK,2009,0,0,0,0,0,0 +LACKS,2009,0,0,0,0,0,0 +LAGS,2009,0,0,0,0,0,0 +LAPSING,2009,0,0,0,0,0,0 +LAWFUL,0,0,0,2009,0,0,0 +LAWMAKING,0,0,0,2009,0,0,0 +LAWYER,0,0,0,2009,0,0,0 +LEADERSHIP,0,2009,0,0,0,0,0 +LEGALITY,0,0,0,2009,0,0,0 +LEGALIZED,0,0,0,2009,0,0,0 +LEGALS,0,0,0,2009,0,0,0 +FLAW,2009,0,0,0,0,0,0 +NONPRODUCTIVE,2009,0,0,0,0,0,0 +LIQUIDATED,2009,0,0,0,0,0,0 +LIQUIDATIONS,2009,0,0,0,0,0,0 +BENEFICIATED,0,0,0,2014,0,0,0 +BENEFITING,0,2009,0,0,0,0,0 +BETTER,0,2009,0,0,0,0,0 +POPULARITY,0,2009,0,0,0,0,0 +REINTERPRET,0,0,2009,0,0,0,0 +REINTERPRETING,0,0,2009,0,0,0,0 +REJECTING,2009,0,0,0,0,0,0 +RELEASEES,0,0,0,2011,0,0,0 +RELINQUISHING,2009,0,0,0,0,0,0 +RELUCTANT,2009,0,0,0,0,0,0 +REMANDS,0,0,0,2009,0,0,0 +REMEDIATION,0,0,0,2009,0,0,0 +RENEGOTIATE,2009,0,0,0,0,0,0 +RENEGOTIATION,2009,0,0,0,0,0,0 +RENOUNCEMENT,2009,0,0,0,0,0,0 +REPARATION,2009,0,0,0,0,0,0 +REPOSSESSED,2009,0,0,0,0,0,0 +REPOSSESSIONS,2009,0,0,0,0,0,0 +TROUBLE,2009,0,0,0,0,0,0 +TURMOIL,2009,0,0,0,0,0,0 +UNACCOUNTED,2009,0,0,0,0,0,0 +UNAPPEALABLE,0,0,0,2009,0,0,0 +UNAUTHORIZED,2009,0,0,0,0,0,0 +UNAVOIDABLY,2009,0,0,0,0,0,0 +UNCERTAINTIES,0,0,2009,0,0,0,0 +UNCOLLECTED,2009,0,0,0,0,0,0 +UNCOMPETITIVE,2009,0,0,0,0,0,0 +UNCONSCIONABLE,2009,0,0,0,0,0,0 +UNCONSTITUTIONALLY,0,0,0,2009,0,0,0 +UNCONTROLLED,2009,0,0,0,0,0,0 +UNCOVERING,2009,0,0,0,0,0,0 +UNDEFINED,0,0,2009,0,0,0,0 +UNDERCUT,2009,0,0,0,0,0,0 +UNDERESTIMATED,2009,0,0,0,0,0,0 +UNDERFUNDED,2009,0,0,0,0,0,0 +UNDERMINES,2009,0,0,0,0,0,0 +UNDERPAYMENTS,2009,0,0,0,0,0,0 +UNDERPERFORMED,2011,0,0,0,0,0,0 +UNDERPRODUCTION,2009,0,0,0,0,0,0 +UNDERSTATEMENT,2009,0,0,0,0,0,0 +UNDERUTILIZATION,2009,0,0,0,0,0,0 +UNDESIRED,2009,0,0,0,0,0,0 +UNDETERMINED,2009,0,2009,0,0,0,0 +UNDOCUMENTED,2009,0,2009,0,0,0,0 +UNECONOMIC,2009,0,0,0,0,0,0 +UNEMPLOYMENT,2009,0,0,0,0,0,0 +UNENFORCEABLE,0,0,0,2009,0,0,0 +UNETHICALLY,2009,0,0,0,0,0,0 +UNFAIR,2009,0,0,0,0,0,0 +UNFAVORABILITY,2014,0,0,0,0,0,0 +UNFEASIBLE,2009,0,0,0,0,0,0 +UNFORESEEABLE,2009,0,0,0,0,0,0 +UNFORTUNATELY,2009,0,0,0,0,0,0 +UNFUNDED,2009,0,0,0,0,0,0 +UNIDENTIFIED,0,0,2009,0,0,0,0 +UNINTENTIONALLY,2009,0,0,0,0,0,0 +UNJUSTIFIED,2009,0,0,0,0,0,0 +UNKNOWN,0,0,2009,0,0,0,0 +UNLAWFULNESS,0,0,0,2009,0,0,0 +UNMATCHED,0,2009,0,0,0,0,0 +UNNECESSARY,2009,0,0,0,0,0,0 +UNOCCUPIED,2009,0,0,0,0,0,0 +UNPLANNED,2009,0,2009,0,0,0,0 +UNPREDICTABLY,2009,0,2009,0,0,0,0 +UNPROFITABLE,2009,0,0,0,0,0,0 +UNQUANTIFIABLE,0,0,2011,0,0,0,0 +UNREASONABLENESS,2009,0,0,0,0,0,0 +UNRECOVERABLE,2009,0,0,0,0,0,0 +UNREMEDIATED,0,0,0,2011,0,0,0 +UNREST,2009,0,0,0,0,0,0 +UNSATISFACTORY,2009,0,0,0,0,0,0 +UNSEASONABLE,0,0,2009,0,0,0,0 +UNSOLD,2009,0,0,0,0,0,0 +UNSTABILIZED,2014,0,0,0,0,0,0 +UNSUCCESSFUL,2009,0,0,0,0,0,0 +UNSUITABLY,2009,0,0,0,0,0,0 +UNSUSPECTED,2009,0,0,0,0,0,0 +UNTESTED,0,0,2009,0,0,0,0 +UNTRUTH,2009,0,0,0,0,0,0 +UNTRUTHS,2009,0,0,0,0,0,0 +UNWANTED,2009,0,0,0,0,0,0 +UNWILLINGNESS,2009,0,0,0,0,0,0 +UPTURNS,0,2009,0,0,0,0,0 +USURP,2009,0,0,2009,0,0,0 +USURPS,2009,0,0,2009,0,0,0 +VAGUELY,0,0,2012,0,0,0,0 +VAGUEST,0,0,2012,0,0,0,0 +PLEA,2009,0,0,0,0,0,0 +PLEADINGS,2009,0,0,2009,0,0,0 +PLEASANTLY,0,2009,0,0,0,0,0 +PLEDGE,0,0,0,0,0,0,2009 +PLEDGES,0,0,0,0,0,0,2009 +PLENTIFUL,0,2009,0,0,0,0,0 +COMPELS,0,0,0,0,0,0,2009 +COMPLAINANTS,0,0,0,2009,0,0,0 +COMPLAINT,2009,0,0,0,0,0,0 +COMMIT,0,0,0,0,0,0,2009 +COMMITTED,0,0,0,0,0,0,2009 +LIMIT,0,0,0,0,0,0,2009 +LIMITS,0,0,0,0,0,0,2009 +RESTRAINED,0,0,0,0,0,0,2009 +RESTRAINTS,0,0,0,0,0,0,2009 +RESTRICTION,0,0,0,0,0,0,2009 +RESTRICTIVENESS,0,0,0,0,0,0,2009 +RESTRUCTURES,2009,0,0,0,0,0,0 +RETALIATED,2009,0,0,0,0,0,0 +RETALIATIONS,2009,0,0,0,0,0,0 +PRECAUTIONARY,0,0,2009,0,0,0,0 +PRECIPITOUSLY,2009,0,0,0,0,0,0 +PRECLUDING,2009,0,0,0,0,0,2009 +PREDECEASE,0,0,0,2009,0,0,0 +PREDICT,0,0,2009,0,0,0,0 +PREDICTION,0,0,2009,0,0,0,0 +PREDICTORS,0,0,2009,0,0,0,0 +PREHEARING,0,0,0,2011,0,0,0 +PREJUDICIAL,2009,0,0,2009,0,0,0 +PREMATURE,2009,0,0,0,0,0,0 +PREPETITION,0,0,0,2009,0,0,0 +PRESTIGIOUS,0,2009,0,0,0,0,0 +PRESUMES,0,0,2009,0,0,0,0 +PRESUMPTIVELY,0,0,0,2009,0,0,0 +PREVENTING,2009,0,0,0,0,0,2009 +PRIVITY,0,0,0,2011,0,0,0 +PROBABILITIES,0,0,2009,0,0,0,0 +PROBATE,0,0,0,2009,0,0,0 +PROBATION,0,0,0,2009,0,0,0 +PROBATIONERS,0,0,0,2009,0,0,0 +PROBLEMATICAL,2009,0,0,0,0,0,0 +PROFICIENTLY,0,2009,0,0,0,0,0 +PROGRESS,0,2009,0,0,0,0,0 +PROHIBIT,0,0,0,0,0,0,2009 +PROHIBITIONS,0,0,0,0,0,0,2009 +PROHIBITS,0,0,0,0,0,0,2009 +PROLONGED,2009,0,0,0,0,0,0 +PROMULGATED,0,0,0,2009,0,0,0 +PROMULGATIONS,0,0,0,2009,0,0,0 +PRORATA,0,0,0,2009,0,0,0 +PROSECUTES,2009,0,0,2009,0,0,0 +PROSECUTOR,0,0,0,2009,0,0,0 +PROSPERING,0,2009,0,0,0,0,0 +PROTEST,2009,0,0,0,0,0,0 +PROTESTING,2009,0,0,0,0,0,0 +PROTRACTED,2009,0,0,0,0,0,0 +PROVISOS,0,0,0,2009,0,0,0 +PROVOKING,2009,0,0,0,0,0,0 +PUNISHING,2009,0,0,0,0,0,0 +PURPORT,2009,0,0,0,0,0,0 +PURPORTS,2009,0,0,0,0,0,0 +QUESTIONED,2009,0,0,0,0,0,0 +QUITCLAIM,0,0,0,2009,0,0,0 +RACKETEERING,2009,0,0,0,0,0,0 +RANDOMIZES,0,0,2009,0,0,0,0 +RATA,0,0,0,2009,0,0,0 +RATIONALIZATIONS,2009,0,0,0,0,0,0 +RATIONALIZING,2009,0,0,0,0,0,0 +ABANDON,2009,0,0,0,0,0,0 +ABANDONMENTS,2009,0,0,0,0,0,0 +ABDICATING,2009,0,0,0,0,0,0 +ABERRATION,2009,0,0,0,0,0,0 +ABEYANCE,0,0,2009,0,0,0,0 +ABLE,0,2009,0,0,0,0,0 +ABNORMALLY,2009,0,0,0,0,0,0 +ABOLISHING,2009,0,0,0,0,0,0 +ABROGATES,2009,0,0,2009,0,0,0 +ABRUPT,2009,0,0,0,0,0,0 +ABSENCES,2009,0,0,0,0,0,0 +ABSOLVES,0,0,0,2009,0,0,0 +ABUSE,2009,0,0,0,0,0,0 +ABUSIVE,2009,0,0,0,0,0,0 +ACCESSIONS,0,0,0,2009,0,0,0 +ACCIDENTS,2009,0,0,0,0,0,0 +ACCOMPLISHES,0,2009,0,0,0,0,0 +ACCUSATION,2009,0,0,0,0,0,0 +ACCUSES,2009,0,0,0,0,0,0 +ACHIEVEMENT,0,2009,0,0,0,0,0 +ACQUIESCE,2009,0,0,0,0,0,0 +ACQUIREES,0,0,0,2011,0,0,0 +ACQUITTAL,2009,0,0,2009,0,0,0 +ACQUITTED,2009,0,0,2009,0,0,0 +ADJOURN,0,0,0,2009,0,0,0 +ADJOURNMENTS,0,0,0,2009,0,0,0 +ADJUDGES,0,0,0,2009,0,0,0 +ADJUDICATES,0,0,0,2009,0,0,0 +ADJUDICATIVE,0,0,0,2009,0,0,0 +ADMISSIBILITY,0,0,0,2009,0,0,0 +ADMISSIONS,0,0,0,2009,0,0,0 +ADULTERATION,2009,0,0,0,0,0,0 +ADVANCES,0,2009,0,0,0,0,0 +ADVANTAGEOUS,0,2009,0,0,0,0,0 +ADVERSARIES,2009,0,0,0,0,0,0 +ADVERSITIES,2009,0,0,0,0,0,0 +AFFIRMANCE,0,0,0,2011,0,0,0 +AFORESAID,0,0,0,2009,0,0,0 +AGAINST,2009,0,0,0,0,0,0 +AGGRAVATING,2009,0,0,0,0,0,0 +ALERTED,2009,0,0,0,0,0,0 +ALIENATES,2009,0,0,0,0,0,0 +ALLEGATION,2009,0,0,2009,0,0,0 +ALLEGEDLY,2009,0,0,2009,0,0,0 +ALLIANCES,0,2009,0,0,0,0,0 +ALWAYS,0,0,0,0,2009,0,0 +AMEND,0,0,0,2009,0,0,0 +AMENDING,0,0,0,2009,0,0,0 +ANNOY,2009,0,0,0,0,0,0 +ANNOYING,2009,0,0,0,0,0,0 +ANNULLING,2009,0,0,0,0,0,0 +ANOMALIES,2009,0,2009,0,0,0,0 +ANTECEDENT,0,0,0,2009,0,0,0 +ANTICIPATES,0,0,2009,0,0,0,0 +ANTICOMPETITIVE,2009,0,0,0,0,0,0 +APPARENT,0,0,2009,0,0,0,0 +APPEALED,0,0,0,2009,0,0,0 +APPEARED,0,0,2009,0,0,2009,0 +APPELLANTS,0,0,0,2009,0,0,0 +APPROXIMATE,0,0,2009,0,0,0,0 +APPROXIMATING,0,0,2009,0,0,0,0 +APPURTENANCES,0,0,0,2009,0,0,0 +ARBITRARILY,0,0,2009,0,0,0,0 +ARBITRATED,0,0,0,2009,0,0,0 +ARBITRATIONAL,0,0,0,2011,0,0,0 +ARBITRATORS,0,0,0,2009,0,0,0 +ARGUMENT,2009,0,0,0,0,0,0 +ARREARAGES,2009,0,0,2009,0,0,0 +ARRESTS,2009,0,0,0,0,0,0 +RETROCEDED,0,0,0,2011,0,0,0 +BOLSTERING,0,2009,0,0,0,0,0 +BOOM,0,2009,0,0,0,0,0 +BOTTLENECK,2009,0,0,0,0,0,0 +BOYCOTT,2009,0,0,0,0,0,0 +BREACH,2009,0,0,2009,0,0,0 +BREAK,2009,0,0,0,0,0,0 +BREAKDOWNS,2009,0,0,0,0,0,0 +BREAKTHROUGHS,0,2009,0,0,0,0,0 +BRIBERY,2009,0,0,0,0,0,0 +BRILLIANT,0,2009,0,0,0,0,0 +BURDENING,2009,0,0,0,0,0,0 +CALAMITIES,2009,0,0,0,0,0,0 +CANCELED,2009,0,0,0,0,0,0 +CANCELLED,2009,0,0,0,0,0,0 +CARELESSLY,2009,0,0,0,0,0,0 +CATASTROPHES,2009,0,0,0,0,0,0 +CAUTIONARY,2009,0,0,0,0,0,0 +CAUTIOUS,0,0,2009,0,0,0,0 +CEASED,2009,0,0,0,0,0,0 +CEDANTS,0,0,0,2012,0,0,0 +CENSURING,2009,0,0,0,0,0,0 +CHALLENGED,2009,0,0,0,0,0,0 +CHARITABLE,0,2009,0,0,0,0,0 +CIRCUMVENT,2009,0,0,0,0,0,0 +CIRCUMVENTIONS,2009,0,0,0,0,0,0 +SHORTAGE,2009,0,0,0,0,0,0 +SHRINKAGE,2009,0,0,0,0,0,0 +SHUTDOWNS,2009,0,0,0,0,0,0 +SLANDERED,2009,0,0,0,0,0,0 +REPUDIATES,2009,0,0,0,0,0,0 +REQUESTER,0,0,0,2011,0,0,0 +REQUIREMENT,0,0,0,0,0,0,2009 +REREGULATION,0,0,0,2011,0,0,0 +RESCINDS,0,0,0,2009,0,0,0 +RESIGNATION,2009,0,0,0,0,0,0 +RESIGNS,2009,0,0,0,0,0,0 +RESTATEMENT,2009,0,0,0,0,0,0 +RESTITUTIONARY,0,0,0,2011,0,0,0 +NOTARIAL,0,0,0,2009,0,0,0 +NOTARIZE,0,0,0,2009,0,0,0 +NOTWITHSTANDING,0,0,0,2009,0,0,0 +NULLIFICATION,2009,0,0,2009,0,0,0 +NULLIFY,2009,0,0,2009,0,0,0 +OBJECTED,2009,0,0,0,0,0,0 +OBJECTIONABLY,2009,0,0,0,0,0,0 +OBLIGATES,0,0,0,0,0,0,2009 +OBLIGATORY,0,0,0,0,0,0,2009 +OBLIGEES,0,0,0,2011,0,0,0 +OBSCENE,2009,0,0,0,0,0,0 +OBSTACLE,2009,0,0,0,0,0,0 +OBSTRUCTING,2009,0,0,0,0,0,0 +OFFENCE,2009,0,0,0,0,0,0 +OFFENDER,2009,0,0,0,0,0,0 +OFFENSE,0,0,0,2009,0,0,0 +OFFERORS,0,0,0,2011,0,0,0 +OMITS,2009,0,0,0,0,0,0 +OPPORTUNISTIC,2009,0,0,0,0,0,0 +OPPOSE,2009,0,0,0,0,0,0 +OPPOSITION,2009,0,0,0,0,0,0 +OPTIONEES,0,0,0,2009,0,0,0 +OUTDATED,2009,0,0,0,0,0,0 +OUTPERFORMING,0,2009,0,0,0,0,0 +OVERBUILD,2009,0,0,0,0,0,0 +OVERBURDEN,2009,0,0,0,0,0,0 +OVERCAPACITY,2009,0,0,0,0,0,0 +OVERCHARGING,2009,0,0,0,0,0,0 +OVERDUE,2009,0,0,0,0,0,0 +OVERESTIMATING,2009,0,0,0,0,0,0 +OVERLOADED,2009,0,0,0,0,0,0 +OVERLOOKED,2009,0,0,0,0,0,0 +OVERPAYMENT,2009,0,0,0,0,0,0 +OVERPRODUCING,2009,0,0,0,0,0,0 +OVERRULES,0,0,0,2009,0,0,0 +OVERRUNS,2009,0,0,0,0,0,0 +OVERSHADOWS,2009,0,0,0,0,0,0 +OVERSTATEMENTS,2009,0,0,0,0,0,0 +OVERSUPPLIES,2009,0,0,0,0,0,0 +OVERTURN,2009,0,0,0,0,0,0 +OVERVALUE,2009,0,0,0,0,0,0 +PANICS,2009,0,0,0,0,0,0 +NECESSITATE,0,0,0,0,0,0,2009 +NEGATIVE,2009,0,0,0,0,0,0 +NEGLECTED,2009,0,0,0,0,0,0 +NEGLIGENCE,2009,0,0,0,0,0,0 +NEVER,0,0,0,0,2009,0,0 +BARRIERS,2009,0,0,0,0,0,0 +BELIEVED,0,0,2009,0,0,0,0 +IDLING,2009,0,0,0,0,0,0 +IGNORING,2009,0,0,0,0,0,0 +ILLEGALITY,2009,0,0,0,0,0,0 +ILLICITLY,2009,0,0,0,0,0,0 +IMBALANCES,2009,0,0,0,0,0,0 +IMPAIR,2009,0,0,0,0,0,2011 +IMPAIRMENTS,2009,0,0,0,0,0,2011 +IMPEDE,2009,0,0,0,0,0,0 +IMPEDIMENTS,2009,0,0,0,0,0,0 +IMPERFECTION,2009,0,0,0,0,0,0 +IMPLEADED,0,0,0,2009,0,0,0 +IMPLICATING,2009,0,0,0,0,0,0 +IMPOSING,0,0,0,0,0,0,2009 +IMPOSSIBLE,2009,0,0,0,0,0,0 +IMPOUNDS,2009,0,0,0,0,0,0 +IMPRACTICALITY,2009,0,0,0,0,0,0 +IMPRESS,0,2009,0,0,0,0,0 +IMPRESSIVE,0,2009,0,0,0,0,0 +IMPROBABLE,0,0,2009,0,0,0,0 +IMPROPRIETY,2009,0,0,0,0,0,0 +IMPROVEMENTS,0,2009,0,0,0,0,0 +IMPRUDENTLY,2009,0,0,0,0,0,0 +INACCURACY,2009,0,0,0,0,0,0 +INACTIONS,2009,0,0,0,0,0,0 +INACTIVATING,2009,0,0,0,0,0,0 +INADEQUACIES,2009,0,0,0,0,0,0 +INADVERTENT,2009,0,0,0,0,0,0 +INAPPROPRIATE,2009,0,0,0,0,0,0 +INCAPABLE,2009,0,0,0,0,0,0 +INCARCERATED,2009,0,0,2009,0,0,0 +INCARCERATIONS,2009,0,0,2009,0,0,0 +INCIDENT,2009,0,0,0,0,0,0 +INCOMPATIBLE,2009,0,0,0,0,0,0 +INCOMPETENTLY,2009,0,0,0,0,0,0 +INCOMPLETENESS,2009,0,2009,0,0,0,0 +INCONSISTENT,2009,0,0,0,0,0,0 +INCONVENIENCE,2009,0,0,0,0,0,0 +INCORRECTLY,2009,0,0,0,0,0,0 +INDEBTED,0,0,0,0,0,0,2009 +INDEFEASIBLY,2009,0,0,0,0,0,0 +INDEMNIFIABLE,0,0,0,2009,0,0,0 +INDEMNIFIES,0,0,0,2009,0,0,0 +INDEMNITEES,0,0,0,2009,0,0,0 +INDEMNITY,0,0,0,2009,0,0,0 +INDICTABLE,2009,0,0,2009,0,0,0 +INDICTMENTS,2009,0,0,2009,0,0,0 +INEFFECTIVENESS,2009,0,0,0,0,0,0 +INEFFICIENTLY,2009,0,0,0,0,0,0 +INEQUITABLY,2009,0,0,0,0,0,0 +INEXACT,0,0,2009,0,0,0,0 +INFERIOR,2009,0,0,0,0,0,0 +INFORMATIVE,0,2009,0,0,0,0,0 +INFRINGED,2009,0,0,0,0,0,0 +INFRINGES,2009,0,0,0,0,0,0 +INHIBITED,2009,0,0,0,0,0,2009 +INJUNCTION,2009,0,0,2009,0,0,0 +INJURED,2009,0,0,0,0,0,0 +INJURIOUS,2009,0,0,0,0,0,0 +INNOVATES,0,2009,0,0,0,0,0 +INNOVATIVE,0,2009,0,0,0,0,0 +INORDINATE,2009,0,0,0,0,0,0 +INSENSITIVE,2009,0,0,0,0,0,0 +INSISTENCE,0,0,0,0,0,0,2009 +INSOLVENCIES,2009,0,0,0,0,0,0 +INSPIRATIONAL,0,2009,0,0,0,0,0 +INSUFFICIENCY,2009,0,0,0,0,0,0 +INSURRECTIONS,2009,0,0,0,0,0,0 +INTENTIONAL,2009,0,0,0,0,0,0 +INTERFERENCES,2009,0,0,0,0,0,0 +INTERMITTENT,2009,0,0,0,0,0,0 +INTERPOSED,0,0,0,2009,0,0,0 +INTERPOSITIONS,0,0,0,2009,0,0,0 +INTERROGATING,0,0,0,2009,0,0,0 +INTERROGATORIES,0,0,0,2009,0,0,0 +INTERRUPTED,2009,0,0,0,0,0,0 +INTERRUPTS,2009,0,0,0,0,0,0 +STABILITY,0,2009,0,0,0,0,0 +STABILIZED,0,2009,0,0,0,0,0 +STAGGERING,2009,0,0,0,0,0,0 +STAGNATES,2009,0,0,0,0,0,0 +STANDSTILLS,2009,0,0,0,0,0,0 +STATUTORY,0,0,0,2009,0,0,0 +STIPULATING,0,0,0,0,0,0,2009 +STOPPAGE,2009,0,0,0,0,0,0 +STOPS,2009,0,0,0,0,0,0 +STRAINS,2009,0,0,0,0,0,0 +STRENGTHENING,0,2009,0,0,0,0,0 +STRESSED,2009,0,0,0,0,0,0 +STRICT,0,0,0,0,0,0,2009 +STRINGENT,2009,0,0,0,0,0,0 +STRONGLY,0,0,0,0,2009,0,0 +SUBJECTED,2009,0,0,0,0,0,0 +SUBLEASEHOLD,0,0,0,2011,0,0,0 +SUBPARAGRAPH,0,0,0,2009,0,0,0 +SUBPOENAS,2009,0,0,2009,0,0,0 +SUBTRUST,0,0,0,2011,0,0,0 +SUCCEEDING,0,2009,0,0,0,0,0 +SUCCESSFUL,0,2009,0,0,0,0,0 +SUE,2009,0,0,2009,0,0,0 +SUFFERED,2009,0,0,0,0,0,0 +SUGGESTED,0,0,2009,0,0,0,0 +SUMMONED,2009,0,0,2009,0,0,0 +SUPERIOR,0,2009,0,0,0,0,0 +SUPERSEDES,0,0,0,2009,0,0,0 +SURPASS,0,2009,0,0,0,0,0 +SUSCEPTIBILITY,2009,0,2009,0,0,0,0 +SUSPECTS,2009,0,0,0,0,0,0 +SUSPENDS,2009,0,0,0,0,0,0 +SUSPICIONS,2009,0,0,0,0,0,0 +REVOCABILITY,0,0,0,2011,0,0,0 +REVOKED,2009,0,0,0,0,0,0 +REVOLUTIONIZED,0,2009,0,0,0,0,0 +REWARDED,0,2009,0,0,0,0,0 +RIDICULED,2009,0,0,0,0,0,0 +RISKED,0,0,2009,0,0,0,0 +RISKING,0,0,2009,0,0,0,0 +RULING,0,0,0,2009,0,0,0 +SACRIFICE,2009,0,0,0,0,0,0 +SACRIFICING,2009,0,0,0,0,0,0 +SATISFIED,0,2009,0,0,0,0,0 +SCANDALOUS,2009,0,0,0,0,0,0 +SCRUTINIZES,2009,0,0,0,0,0,0 +SEEMS,0,0,2009,0,0,0,0 +SEIZING,2009,0,0,0,0,0,0 +SENTENCING,2009,0,0,2009,0,0,0 +SERIOUSNESS,2009,0,0,0,0,0,0 +SETTLEMENTS,0,0,0,2009,0,0,0 +SEVERALLY,0,0,0,2009,0,0,0 +SEVERED,2009,0,0,0,0,0,0 +ENTHUSIASTICALLY,0,2009,0,0,0,0,0 +ERODED,2009,0,0,0,0,0,0 +ERRATIC,2009,0,0,0,0,0,0 +ERRONEOUS,2009,0,0,0,0,0,0 +ERRS,2009,0,0,0,0,0,0 +ESCALATING,2009,0,0,0,0,0,0 +ESCROW,0,0,0,0,0,0,2009 +ESTOPPEL,0,0,0,2011,0,0,0 +EVADING,2009,0,0,0,0,0,0 +EVICT,2009,0,0,0,0,0,0 +EVICTIONS,2009,0,0,0,0,0,0 +EXACERBATE,2009,0,0,0,0,0,0 +EXACERBATION,2009,0,0,0,0,0,0 +EXAGGERATES,2009,0,0,0,0,0,0 +EXCEEDANCES,0,0,0,2011,0,0,0 +EXCELLING,0,2009,0,0,0,0,0 +EXCESSIVE,2009,0,0,0,0,0,0 +EXCITEMENT,0,2009,0,0,0,0,0 +EXCLUSIVENESS,0,2009,0,0,0,0,0 +EXCULPATED,2009,0,0,2009,0,0,0 +EXCULPATIONS,2009,0,0,2009,0,0,0 +EXECUTORY,0,0,0,2009,0,0,0 +EXEMPLARY,0,2009,0,0,0,0,0 +EXONERATING,2009,0,0,0,0,0,0 +EXPLOITATION,2009,0,0,0,0,0,0 +EXPLOITING,2009,0,0,0,0,0,0 +EXPOSES,2009,0,0,0,0,0,0 +EXPROPRIATE,2009,0,0,0,0,0,0 +EXPROPRIATION,2009,0,0,0,0,0,0 +EXTENUATING,2009,0,0,0,0,0,0 +SOLVING,0,2009,0,0,0,0,0 +SOMEWHERE,0,0,2009,0,0,0,0 +LOSE,2009,0,0,0,0,0,0 +LOSSES,2009,0,0,0,0,0,0 +LUCRATIVE,0,2009,0,0,0,0,0 +MALFUNCTION,2009,0,0,0,0,0,0 +MALICE,2009,0,0,0,0,0,0 +MANDAMUS,0,0,0,2009,0,0,0 +MANDATING,0,0,0,0,0,0,2009 +MANIPULATED,2009,0,0,0,0,0,0 +MANIPULATIONS,2009,0,0,0,0,0,0 +MAY,0,0,2009,0,0,2009,0 +MEDIATES,0,0,0,2009,0,0,0 +MEDIATOR,0,0,0,2009,0,0,0 +MISAPPLICATION,2009,0,0,0,0,0,0 +MISAPPLY,2009,0,0,0,0,0,0 +MISAPPROPRIATES,2009,0,0,0,0,0,0 +MISBRANDED,2009,0,0,0,0,0,0 +MISCALCULATING,2009,0,0,0,0,0,0 +MISCHIEF,2009,0,0,0,0,0,0 +MISCLASSIFY,2014,0,0,0,0,0,0 +MISDEMEANOR,2009,0,0,2009,0,0,0 +MISHANDLE,2009,0,0,0,0,0,0 +MISINFORM,2009,0,0,0,0,0,0 +MISINFORMS,2009,0,0,0,0,0,0 +MISINTERPRETED,2009,0,0,0,0,0,0 +MISJUDGED,2009,0,0,0,0,0,0 +MISJUDGMENTS,2009,0,0,0,0,0,0 +MISLABELLED,2009,0,0,0,0,0,0 +MISLEADINGLY,2009,0,0,0,0,0,0 +MISMANAGED,2009,0,0,0,0,0,0 +MISMATCH,2009,0,0,0,0,0,0 +MISPLACED,2009,0,0,0,0,0,0 +MISREPRESENT,2009,0,0,0,0,0,0 +MISREPRESENTING,2009,0,0,0,0,0,0 +MISSES,2009,0,0,0,0,0,0 +WORRY,2009,0,0,0,0,0,0 +WORSENED,2009,0,0,0,0,0,0 +WORTHLESS,2009,0,0,0,0,0,0 +TOLERATE,2009,0,0,0,0,0,0 +TOLERATION,2009,0,0,0,0,0,0 +TORTS,0,0,0,2009,0,0,0 +FICTITIOUS,2009,0,0,0,0,0,0 +FIRED,2009,0,0,0,0,0,0 +NONATTAINMENT,2009,0,0,0,0,0,0 +NONCOMPETITIVE,2009,0,0,0,0,0,0 +NONCOMPLYING,2009,0,0,0,0,0,0 +NONCONTINGENT,0,0,0,2011,0,0,0 +NONDISCLOSURE,2009,0,0,0,0,0,0 +NONFORFEITABLE,0,0,0,2009,0,0,0 +DISCLAIM,2009,0,0,0,0,0,0 +DISCLAIMING,2009,0,0,0,0,0,0 +DISCLOSES,2009,0,0,0,0,0,0 +DISCONTINUATION,2009,0,0,0,0,0,0 +DISCONTINUES,2009,0,0,0,0,0,0 +DISCOURAGES,2009,0,0,0,0,0,0 +DISCREDITING,2009,0,0,0,0,0,0 +SPECTACULAR,0,2009,0,0,0,0,0 +SPECULATES,0,0,2009,0,0,0,0 +SPECULATIVE,0,0,2009,0,0,0,0 +TRAGEDIES,2009,0,0,0,0,0,0 +TRANSFEROR,0,0,0,2009,0,0,0 +LEGISLATES,0,0,0,2009,0,0,0 +LEGISLATIVE,0,0,0,2009,0,0,0 +LEGISLATURE,0,0,0,2009,0,0,0 +LIBELOUS,0,0,0,2009,0,0,0 +LIENHOLDERS,0,0,0,2011,0,0,0 +COMPULSORY,0,0,0,0,0,0,2009 +CONCEDED,2009,0,0,0,0,0,0 +CONCEIVABLY,0,0,2009,0,0,0,0 +CONCILIATING,2009,0,0,0,0,0,0 +CONCLUSIVELY,0,2009,0,0,0,0,0 +CONDEMNED,2009,0,0,0,0,0,0 +CONDITIONAL,0,0,2009,0,0,0,0 +CONDUCIVE,0,2009,0,0,0,0,0 +CONFESSING,2009,0,0,0,0,0,0 +CONFINED,2009,0,0,0,0,0,2009 +CONFINING,2009,0,0,0,0,0,2009 +CONFISCATING,2009,0,0,0,0,0,0 +CONFLICT,2009,0,0,0,0,0,0 +CONFRONT,2009,0,0,0,0,0,0 +CONFRONTED,2009,0,0,0,0,0,0 +CONFUSED,2009,0,0,0,0,0,0 +CONFUSION,2009,0,2009,0,0,0,0 +CONSENTS,0,0,0,2009,0,0,0 +CONSPIRATOR,2009,0,0,0,0,0,0 +CONSPIRED,2009,0,0,0,0,0,0 +CONSTITUTIONAL,0,0,0,2009,0,0,0 +CONSTITUTIVE,0,0,0,2009,0,0,0 +CONSTRAINS,0,0,0,0,0,0,2009 +CONSTRUCTIVELY,0,2009,0,0,0,0,0 +CONSTRUING,0,0,0,2012,0,0,0 +CONTENDING,2009,0,0,0,0,0,0 +CONTENTIOUS,2009,0,0,0,0,0,0 +CONTESTED,2009,0,0,0,0,0,0 +CONTINGENT,0,0,2009,0,0,0,0 +CONTRACTED,0,0,0,2009,0,0,0 +CONTRACTILE,0,0,0,2009,0,0,0 +CONTRACTS,0,0,0,2009,0,0,0 +CONTRADICTED,2009,0,0,0,0,0,0 +CONTRADICTORY,2009,0,0,0,0,0,0 +CONTRAVENED,0,0,0,2009,0,0,0 +CONTRAVENTIONS,0,0,0,2009,0,0,0 +CONTROVERT,0,0,0,2009,0,0,0 +CONVEYANCE,0,0,0,2009,0,0,0 +CONVICTING,2009,0,0,2009,0,0,0 +CORRECTING,2009,0,0,0,0,0,0 +CORRUPT,2009,0,0,0,0,0,0 +CORRUPTIONS,2009,0,0,0,0,0,0 +COTERMINOUS,0,0,0,2009,0,0,0 +COUNSELLED,0,0,0,2009,0,0,0 +COUNTERCLAIMING,2009,0,0,0,0,0,0 +COUNTERFEITER,2009,0,0,0,0,0,0 +COUNTERMEASURE,2009,0,0,0,0,0,0 +COUNTERSUIT,0,0,0,2011,0,0,0 +COURTROOM,0,0,0,2009,0,0,0 +COVENANTING,0,0,0,0,0,0,2011 +CREATIVENESS,0,2009,0,0,0,0,0 +CRIMINAL,2009,0,0,2009,0,0,0 +CRIMINALLY,2009,0,0,2009,0,0,0 +CRITICAL,-2020,0,0,0,0,0,0 +CRITICIZE,2009,0,0,0,0,0,0 +CROSSCLAIM,0,0,0,2011,0,0,0 +CRUCIAL,2009,0,0,0,0,0,0 +CULPABLY,2009,0,0,0,0,0,0 +CURTAILING,2009,0,0,0,0,0,0 +CUT,2009,0,0,0,0,0,0 +CYBERATTACKS,2014,0,0,0,0,0,0 +CYBERCRIMINAL,2014,0,0,0,0,0,0 +TAINT,2009,0,0,0,0,0,0 +TAMPERED,2009,0,0,0,0,0,0 +TENTATIVE,0,0,2009,0,0,0,0 +TERMINATED,2009,0,0,0,0,0,0 +TERMINATIONS,2009,0,0,0,0,0,0 +TESTIFYING,2009,0,0,2009,0,0,0 +THENCEFORWARD,0,0,0,2009,0,0,0 +THEREIN,0,0,0,2009,0,0,0 +THEREOVER,0,0,0,2011,0,0,0 +THEREUNDER,0,0,0,2009,0,0,0 +THREAT,2009,0,0,0,0,0,0 +THREATENS,2009,0,0,0,0,0,0 +FACTO,0,0,0,2011,0,0,0 +FAILINGS,2009,0,0,0,0,0,0 +FALLOUT,2009,0,0,0,0,0,0 +FALSIFICATIONS,2009,0,0,0,0,0,0 +FALSIFYING,2009,0,0,0,0,0,0 +FATALITY,2009,0,0,0,0,0,0 +FAULTS,2009,0,0,0,0,0,0 +FAVORED,0,2009,0,0,0,0,0 +FEAR,2009,0,0,0,0,0,0 +FELONY,2009,0,0,2009,0,0,0 +DAMAGING,2009,0,0,0,0,0,0 +DANGEROUS,2009,0,0,0,0,0,0 +DEADLOCKED,2009,0,0,0,0,0,0 +DEADWEIGHTS,2009,0,0,0,0,0,0 +DECEASED,2009,0,0,0,0,0,0 +DECEITFUL,2009,0,0,0,0,0,0 +DECEIVES,2009,0,0,0,0,0,0 +DECEPTIVE,2009,0,0,0,0,0,0 +DECLINED,2009,0,0,0,0,0,0 +DECREED,0,0,0,2009,0,0,0 +DEFACED,2009,0,0,0,0,0,0 +DEFAMATION,2009,0,0,0,0,0,0 +DEFAMED,2009,0,0,0,0,0,0 +DEFAULTED,2009,0,0,0,0,0,0 +DEFEASANCES,0,0,0,2011,0,0,0 +DEFEASES,0,0,0,2014,0,0,0 +DEFEATING,2009,0,0,0,0,0,0 +DEFECTIVELY,0,0,0,2009,0,0,0 +DEFENDANT,2009,0,0,2009,0,0,0 +DEFENDS,2009,0,0,0,0,0,0 +DEFICIENCIES,2009,0,0,0,0,0,0 +DEFICITS,2009,0,0,0,0,0,0 +DEFRAUDED,2009,0,0,0,0,0,0 +DEGRADATION,2009,0,0,0,0,0,0 +DEGRADES,2009,0,0,0,0,0,0 +DELAYING,2009,0,0,0,0,0,0 +DELEGATEE,0,0,0,2011,0,0,0 +DELIBERATED,2009,0,0,0,0,0,0 +DELIGHTFUL,0,2009,0,0,0,0,0 +DELINQUENCIES,2009,0,0,0,0,0,0 +DELINQUENTS,2009,0,0,0,0,0,0 +DELISTS,2011,0,0,0,0,0,0 +DEMISING,2009,0,0,0,0,0,0 +DEMOLISHING,2009,0,0,0,0,0,0 +DEMOTED,2009,0,0,0,0,0,0 +DEMOTIONS,2009,0,0,0,0,0,0 +DEMURRING,0,0,0,2009,0,0,0 +DENIED,2009,0,0,0,0,0,0 +DENIGRATES,2009,0,0,0,0,0,0 +DENYING,2009,0,0,0,0,0,0 +DEPENDANCE,0,0,0,0,0,0,2011 +DEPENDENCE,0,0,2009,0,0,0,0 +DEPENDING,0,0,2009,0,0,2009,2011 +DEPLETES,2009,0,0,0,0,0,0 +DEPOSE,0,0,0,2009,0,0,0 +DEPOSITION,0,0,0,2009,0,0,0 +INVALIDATE,2009,0,0,0,0,0,0 +INVALIDATION,2009,0,0,0,0,0,0 +INVENTING,0,2009,0,0,0,0,0 +INVENTIVENESS,0,2009,0,0,0,0,0 +INVESTIGATED,2009,0,0,0,0,0,0 +INVESTIGATIONS,2009,0,0,0,0,0,0 +IRRECONCILABLY,2009,0,0,0,0,0,0 +IRREGULARITIES,2009,0,0,0,0,0,0 +IRREPARABLY,2009,0,0,0,0,0,0 +IRREVOCABLY,0,0,0,2009,0,0,2009 +JUDICIAL,0,0,0,2009,0,0,0 +JURIES,0,0,0,2009,0,0,0 +JURISDICTIONALLY,0,0,0,2011,0,0,0 +JURISTS,0,0,0,2009,0,0,0 +JURYMAN,0,0,0,2009,0,0,0 +KICKBACK,2009,0,0,0,0,0,0 +DIMINISH,2009,0,0,0,0,0,0 +DIMINUTION,2009,0,0,0,0,0,0 +DISADVANTAGED,2009,0,0,0,0,0,0 +DISAFFIRM,0,0,0,2009,0,0,0 +DISAGREE,2009,0,0,0,0,0,0 +DISAGREEMENT,2009,0,0,0,0,0,0 +DISALLOWANCE,2009,0,0,0,0,0,0 +DISALLOWS,2009,0,0,0,0,0,0 +DISAPPEARED,2009,0,0,0,0,0,0 +DISAPPOINTED,2009,0,0,0,0,0,0 +DISAPPOINTMENTS,2009,0,0,0,0,0,0 +DISAPPROVE,2009,0,0,0,0,0,0 +DISASSOCIATES,2009,0,0,0,0,0,0 +DISASTER,2009,0,0,0,0,0,0 +DISAVOW,2009,0,0,0,0,0,0 +DISAVOWS,2009,0,0,0,0,0,0 +GRATUITOUS,2009,0,0,0,0,0,0 +GREATEST,0,2009,0,0,0,0,0 +GRIEVANCES,2009,0,0,0,0,0,0 +HALT,2009,0,0,0,0,0,0 +HAMPERING,2009,0,0,0,0,0,0 +HAPPINESS,0,2009,0,0,0,0,0 +HARASSING,2009,0,0,0,0,0,0 +HARM,2009,0,0,0,0,0,0 +HARMING,2009,0,0,0,0,0,0 +HARSHEST,2009,0,0,0,0,0,0 +HAZARDOUS,2009,0,0,0,0,0,0 +NONINFRINGEMENT,0,0,0,2011,0,0,0 +NONJURISDICTIONAL,0,0,0,2011,0,0,0 +NONPERFORMANCES,2009,0,0,0,0,0,0 +HURTING,2009,0,0,0,0,0,0 +DISFAVORING,2009,0,0,0,0,0,0 +DISGORGEMENT,2009,0,0,0,0,0,0 +COMPLICATE,2009,0,0,0,0,0,0 +COMPLICATION,2009,0,0,0,0,0,0 +COMPLIMENTED,0,2009,0,0,0,0,0 +MISSTATEMENT,2009,0,0,0,0,0,0 +MISSTEP,2009,0,0,0,0,0,0 +MISTAKENLY,2009,0,0,0,0,0,0 +MISTRIALS,2009,0,0,2009,0,0,0 +MISUNDERSTOOD,2009,0,0,0,0,0,0 +MISUSING,2009,0,0,0,0,0,0 +MONOPOLIZE,2009,0,0,0,0,0,0 +MONOPOLY,2009,0,0,0,0,0,0 +MOREOVER,0,0,0,2009,0,0,0 +MUST,0,0,0,0,2009,0,0 +SLIPPAGE,2009,0,0,0,0,0,0 +SLOWDOWNS,2009,0,0,0,0,0,0 +SLOWING,2009,0,0,0,0,0,0 +SLUGGISHLY,2009,0,0,0,0,0,0 +SMOOTHLY,0,2009,0,0,0,0,0 +DEPRESSED,2009,0,0,0,0,0,0 +DEPRIVE,2009,0,0,0,0,0,0 +DERELICT,2009,0,0,0,0,0,0 +DEROGATES,0,0,0,2009,0,0,0 +DEROGATORY,2009,0,0,0,0,0,0 +DESIST,0,0,0,2009,0,0,0 +DESTABILIZED,2009,0,0,0,0,0,0 +DESTROYED,2009,0,0,0,0,0,0 +DESTRUCTIVE,2009,0,0,0,0,0,0 +DETENTION,2009,0,0,0,0,0,0 +DETERIORATED,2009,0,0,0,0,0,0 +DETERIORATIONS,2009,0,0,0,0,0,0 +DETERRENT,2009,0,0,0,0,0,0 +DETRACT,2009,0,0,0,0,0,0 +DETRIMENTAL,2009,0,0,0,0,0,0 +DEVALUED,2009,0,0,0,0,0,0 +DEVASTATED,2009,0,0,0,0,0,0 +DEVIATED,2009,0,2009,0,0,0,0 +DEVIATIONS,2009,0,2009,0,0,0,0 +DEVOLVES,2009,0,0,0,0,0,0 +DICTATES,0,0,0,0,0,0,2009 +DIFFERING,0,0,2009,0,0,0,0 +DIFFICULTLY,2009,0,0,0,0,0,0 +ASCENDANT,0,0,0,2009,0,0,0 +ASSAULTING,2009,0,0,0,0,0,0 +ASSIGNATION,0,0,0,2009,0,0,0 +ASSUMED,0,0,2009,0,0,0,0 +ASSUMPTIONS,0,0,2009,0,0,0,0 +ASSURING,0,2009,0,0,0,0,0 +ATTAINMENT,0,2009,0,0,0,0,0 +ATTESTATION,0,0,0,2009,0,0,0 +ATTORN,0,0,0,2011,0,0,0 +ATTORNS,0,0,0,2011,0,0,0 +AVERSELY,2011,0,0,0,0,0,0 +BAILED,0,0,0,2009,0,0,0 +BAILIFFS,0,0,0,2009,0,0,0 +BALKED,2009,0,0,0,0,0,0 +BANKRUPTED,2009,0,0,0,0,0,0 +CLAIMANT,0,0,0,2009,0,0,0 +CLAIMS,2009,0,0,2009,0,0,0 +CLAWBACKS,0,0,0,2014,0,0,0 +CLOSEOUTS,2009,0,0,0,0,0,0 +CLOSURES,2009,0,0,0,0,0,0 +CODICILS,0,0,0,2009,0,0,0 +CODIFIES,0,0,0,2009,0,0,0 +COERCED,2009,0,0,0,0,0,0 +COERCIVE,2009,0,0,0,0,0,0 +DISINCENTIVES,2009,0,0,0,0,0,0 +COLLABORATE,0,2009,0,0,0,0,0 +COLLABORATION,0,2009,0,0,0,0,0 +COLLABORATORS,0,2009,0,0,0,0,0 +COLLAPSING,2009,0,0,0,0,0,0 +COLLUDED,2009,0,0,0,0,0,0 +COLLUSIONS,2009,0,0,0,0,0,0 +POSITIVE,0,2009,0,0,0,0,0 +POSSIBILITY,0,0,2009,0,0,0,0 +POSTCLOSURE,0,0,0,2011,0,0,0 +POSTPONED,2009,0,0,0,0,0,0 +POSTPONING,2009,0,0,0,0,0,0 +WRIT,0,0,0,2009,0,0,0 +WRITEOFFS,2009,0,0,0,0,0,0 +WRONGDOINGS,2009,0,0,0,0,0,0 +HONORING,0,2009,0,0,0,0,0 +REASSESSES,0,0,2009,0,0,0,0 +REASSIGN,2009,0,0,0,0,0,0 +REASSIGNMENTS,2009,0,0,0,0,0,0 +REBOUNDING,0,2009,0,0,0,0,0 +REBUTTABLY,0,0,0,2011,0,0,0 +REBUTTING,0,0,0,2009,0,0,0 +RECALCULATING,0,0,2009,0,0,0,0 +RECALLED,2009,0,0,0,0,0,0 +RECESSION,2009,0,0,0,0,0,0 +RECKLESSLY,2009,0,0,0,0,0,0 +RECONSIDERING,0,0,2009,0,0,0,0 +RECOUPMENT,0,0,0,2009,0,0,0 +RECTIFICATION,0,0,0,2009,0,0,0 +RECUSED,0,0,0,2011,0,0,0 +REDACTED,2009,0,0,2009,0,0,0 +REDEFAULT,2014,0,0,0,0,0,0 +REDRESSED,2009,0,0,0,0,0,0 +REEXAMINE,0,0,2009,0,0,0,0 +REFERENDUMS,0,0,0,2009,0,0,0 +REFILING,0,0,0,2009,0,0,0 +REFUSAL,2009,0,0,0,0,0,0 +REFUSES,2009,0,0,0,0,0,0 +REGAINING,0,2009,0,0,0,0,0 +REGULATING,0,0,0,2009,0,0,0 +REGULATOR,0,0,0,2009,0,0,0 +REHEARD,0,0,0,2009,0,0,0 +VARIABLES,0,0,2009,0,0,0,0 +VARIANT,0,0,2009,0,0,0,0 +VARIED,0,0,2009,0,0,0,0 +VENDEE,0,0,0,2011,0,0,0 +VERSATILE,0,2009,0,0,0,0,0 +VIBRANCY,0,2009,0,0,0,0,0 +VIOLATED,2009,0,0,0,0,0,0 +VIOLATIONS,2009,0,0,0,0,0,0 +VIOLENCE,2009,0,0,0,0,0,0 +VITIATED,2009,0,0,0,0,0,0 +VOIDABLE,0,0,0,2009,0,0,0 +VOLATILITIES,0,0,2009,0,0,0,0 +VULNERABLE,2009,0,0,0,0,0,0 +WARNING,2009,0,0,0,0,0,0 +WARRANTOR,0,0,0,2011,0,0,0 +WEAK,2009,0,0,0,0,0,0 +WEAKENS,2009,0,0,0,0,0,0 +WEAKNESS,2009,0,0,0,0,0,0 +DISHONOR,2009,0,0,0,0,0,0 +DISHONORING,2009,0,0,0,0,0,0 +DISINTERESTED,2009,0,0,0,0,0,0 +DISLOYALLY,2009,0,0,0,0,0,0 +DISMISS,2009,0,0,0,0,0,0 +DISMISSES,2009,0,0,0,0,0,0 +DISPARAGED,2009,0,0,0,0,0,0 +DISPARAGING,2009,0,0,0,0,0,0 +DISPLACE,2009,0,0,0,0,0,0 +DISPLACES,2009,0,0,0,0,0,0 +DISPOSSESS,2009,0,0,0,0,0,0 +DISPOSSESSION,0,0,0,2009,0,0,0 +DISPROPORTIONATE,2009,0,0,0,0,0,0 +DISPUTES,2009,0,0,0,0,0,0 +DISQUALIFIED,2009,0,0,0,0,0,0 +DISREGARD,2009,0,0,0,0,0,0 +DISREPUTABLE,2009,0,0,0,0,0,0 +DISRUPTING,2009,0,0,0,0,0,0 +DISRUPTS,2009,0,0,0,0,0,0 +DISSENTED,2009,0,0,0,0,0,0 +DISSENTS,2009,0,0,0,0,0,0 +DISSOLUTIONS,2009,0,0,0,0,0,0 +DISTINCTIVELY,0,2009,0,0,0,0,0 +DISTORTING,2009,0,0,0,0,0,0 +DISTRACT,2009,0,0,0,0,0,0 +DISTRACTIONS,2009,0,0,0,0,0,0 +DISTRESSED,2009,0,0,0,0,0,0 +DISTURBANCE,2009,0,0,0,0,0,0 +DISTURBS,2009,0,0,0,0,0,0 +DIVERTING,2009,0,0,0,0,0,0 +DIVESTING,2009,0,0,0,0,0,0 +DIVESTMENTS,2009,0,0,0,0,0,0 +DIVULGE,2009,0,0,0,0,0,0 +DOCKET,0,0,0,2009,0,0,0 +DONEES,0,0,0,2011,0,0,0 +DOUBTS,2009,0,2009,0,0,0,0 +DOWNGRADING,2009,0,0,0,0,0,0 +DOWNSIZING,2009,0,0,0,0,0,0 +DOWNTURN,2009,0,0,0,0,0,0 +DRAG,2009,0,0,0,0,0,0 +DRAWBACKS,2009,0,0,0,0,0,0 +DROUGHTS,2009,0,0,0,0,0,0 +DYSFUNCTIONAL,2009,0,0,0,0,0,0 +EARMARKING,0,0,0,0,0,0,2009 +EASING,2009,0,0,0,0,0,0 +EFFICIENCY,0,2009,0,0,0,0,0 +EGREGIOUSLY,2009,0,0,0,0,0,0 +EMBARGOES,2009,0,0,0,0,0,0 +EMBARRASSES,2009,0,0,0,0,0,0 +EMBEZZLE,2009,0,0,0,0,0,0 +EMBEZZLER,2009,0,0,0,0,0,0 +EMPOWERED,0,2009,0,0,0,0,0 +ENABLED,0,2009,0,0,0,0,0 +ENCOURAGEMENT,0,2009,0,0,0,0,0 +ENCROACHED,2009,0,0,0,0,0,0 +ENCROACHMENTS,2009,0,0,0,0,0,0 +ENCUMBERS,2009,0,0,2009,0,0,2009 +ENCUMBRANCES,2009,0,0,2009,0,0,2009 +ENDANGERMENT,2009,0,0,0,0,0,0 +ENFORCEABLE,0,0,0,2009,0,0,0 +ENHANCEMENT,0,2009,0,0,0,0,0 +ENJOIN,2009,0,0,0,0,0,0 +ENJOY,0,2009,0,0,0,0,0 +ENJOYING,0,2009,0,0,0,0,0 +ENTAILED,0,0,0,0,0,0,2009 +PENALIZE,2009,0,0,0,0,0,0 +PENALTIES,2009,0,0,0,0,0,0 +PERFECTED,0,2009,0,0,0,0,0 +PERIL,2009,0,0,0,0,0,0 +PERMISSION,0,0,0,0,0,0,2009 +PERMITTEES,0,0,0,2011,0,0,0 +PERPETRATES,2009,0,0,2009,0,0,0 +PERSISTED,2009,0,0,0,0,0,0 +PERSISTING,2009,0,0,0,0,0,0 +PERVASIVELY,2009,0,0,0,0,0,0 +PETITIONER,0,0,0,2009,0,0,0 +PETTY,2009,0,0,0,0,0,0 +HEREAFTER,0,0,0,2009,0,0,0 +HEREFORE,0,0,0,2014,0,0,0 +HEREINAFTER,0,0,0,2009,0,0,0 +HEREON,0,0,0,2009,0,0,0 +HEREUNTO,0,0,0,2009,0,0,0 +HIDDEN,0,0,2009,0,0,0,0 +HINDERING,2009,0,0,0,0,0,0 +HINGES,0,0,2009,0,0,0,0 +WHEREABOUTS,0,0,0,2009,0,0,0 +WHEREFORE,0,0,0,2009,0,0,0 +WHERETO,0,0,0,2009,0,0,0 +WHISTLEBLOWERS,0,0,0,2011,0,0,0 +WILFUL,0,0,0,2009,0,0,0 +WILLFULNESS,0,0,0,2009,0,0,0 +WINNING,0,2009,0,0,0,0,0 +FLUCTUATE,0,0,2009,0,0,0,0 +FLUCTUATION,0,0,2009,0,0,0,0 +FORBEARANCE,0,0,0,2009,0,0,0 +FORBID,2009,0,0,0,0,0,2009 +FORCE,-2020,0,0,0,0,0,0 +FOREBEARANCE,0,0,0,2011,0,0,0 +FORECLOSES,2009,0,0,0,0,0,0 +FOREGO,2009,0,0,0,0,0,0 +FORESTALLED,2009,0,0,0,0,0,0 +FORFEITABILITY,0,0,0,2011,0,0,0 +FORFEITS,2009,0,0,0,0,0,0 +FORGERY,2009,0,0,0,0,0,0 +FRAUDS,2009,0,0,0,0,0,0 +FRIENDLY,0,2009,0,0,0,0,0 +FRUSTRATED,2009,0,0,0,0,0,0 +FRUSTRATION,2009,0,0,0,0,0,0 +FURTHERANCE,0,0,0,2009,0,0,0 +GAINS,0,2009,0,0,0,0,0 +DISGORGEMENTS,2009,0,0,0,0,0,0 +DISGRACEFUL,2009,0,0,0,0,0,0 +LITIGATE,2009,0,0,2009,0,0,0 +LITIGATION,2009,0,0,2009,0,0,0 +LITIGIOUS,0,0,0,2009,0,0,0 +LACKED,2009,0,0,0,0,0,0 +LAG,2009,0,0,0,0,0,0 +LAPSE,2009,0,0,0,0,0,0 +LATE,-2020,0,0,0,0,0,0 +LAWFULLY,0,0,0,2009,0,0,0 +LAWS,0,0,0,2009,0,0,0 +LAWYERS,0,0,0,2009,0,0,0 +LEADING,0,2009,0,0,0,0,0 +LEGALIZATION,0,0,0,2009,0,0,0 +LEGALIZES,0,0,0,2009,0,0,0 +LEGATEE,0,0,0,2009,0,0,0 +FLAWED,2009,0,0,0,0,0,0 +NONRECOVERABLE,2009,0,0,0,0,0,0 +LIQUIDATES,2009,0,0,0,0,0,0 +LIQUIDATOR,2009,0,0,0,0,0,0 +BENEFICIATION,0,0,0,2011,0,0,0 +BENEFITTED,0,2009,0,0,0,0,0 +POOR,2009,0,0,0,0,0,0 +REINTERPRETATION,0,0,2009,0,0,0,0 +REINTERPRETS,0,0,2009,0,0,0,0 +REJECTION,2009,0,0,0,0,0,0 +RELINQUISH,2009,0,0,0,0,0,0 +RELINQUISHMENT,2009,0,0,0,0,0,0 +REMAND,0,0,0,2009,0,0,0 +REMEDIATE,0,0,0,2009,0,0,0 +REMEDIATIONS,0,0,0,2009,0,0,0 +RENEGOTIATED,2009,0,0,0,0,0,0 +RENEGOTIATIONS,2009,0,0,0,0,0,0 +RENOUNCEMENTS,2009,0,0,0,0,0,0 +REPARATIONS,2009,0,0,0,0,0,0 +REPOSSESSES,2009,0,0,0,0,0,0 +REPRORATED,0,0,0,2018,0,0,0 +TROUBLED,2009,0,0,0,0,0,0 +UNABLE,2009,0,0,0,0,0,0 +UNAMBIGUOUSLY,0,0,0,0,2009,0,0 +UNAPPEALED,0,0,0,2011,0,0,0 +UNAVAILABILITY,2009,0,0,0,0,0,2011 +UNAWARE,2009,0,0,0,0,0,0 +UNCERTAINTY,0,0,2009,0,0,0,0 +UNCOLLECTIBILITY,2009,0,0,0,0,0,0 +UNCOMPLETED,2009,0,0,0,0,0,0 +UNCONSCIONABLY,2009,0,0,0,0,0,0 +UNCONTRACTED,0,0,0,2011,0,0,0 +UNCORRECTED,2009,0,0,0,0,0,0 +UNCOVERS,2009,0,0,0,0,0,0 +UNDELIVERABLE,2009,0,0,0,0,0,0 +UNDERCUTS,2009,0,0,0,0,0,0 +UNDERESTIMATES,2009,0,0,0,0,0,0 +UNDERINSURED,2009,0,0,0,0,0,0 +UNDERMINING,2009,0,0,0,0,0,0 +UNDERPAYS,2009,0,0,0,0,0,0 +UNDERPERFORMING,2009,0,0,0,0,0,0 +UNDERREPORTING,2011,0,0,0,0,0,0 +UNDERSTATEMENTS,2009,0,0,0,0,0,0 +UNDERUTILIZED,2009,0,0,0,0,0,0 +UNDETECTABLE,0,0,2009,0,0,0,0 +UNDISCHARGED,0,0,0,2009,0,0,0 +UNDOUBTEDLY,0,0,0,0,2009,0,0 +UNECONOMICAL,2009,0,0,0,0,0,0 +UNENCUMBER,0,0,0,2014,0,0,0 +UNEQUIVOCAL,0,0,0,0,2009,0,0 +UNEXCUSED,2009,0,0,0,0,0,0 +UNFAIRLY,2009,0,0,0,0,0,0 +UNFAVORABLE,2009,0,0,0,0,0,0 +UNFIT,2009,0,0,0,0,0,0 +UNFORESEEN,2009,0,0,0,0,0,0 +UNFOUNDED,2009,0,0,0,0,0,0 +UNGUARANTEED,0,0,2009,0,0,0,0 +UNINSURED,2009,0,0,0,0,0,0 +UNJUST,2009,0,0,0,0,0,0 +UNJUSTLY,2009,0,0,0,0,0,0 +UNKNOWNS,0,0,2009,0,0,0,0 +UNLICENSED,2009,0,0,0,0,0,0 +UNMERCHANTABLE,2011,0,0,0,0,0,0 +UNNEEDED,2009,0,0,0,0,0,0 +UNPAID,2009,0,0,0,0,0,0 +UNPOPULAR,2009,0,0,0,0,0,0 +UNPREDICTED,2011,0,2011,0,0,0,0 +UNPROVED,0,0,2009,0,0,0,0 +UNQUANTIFIED,0,0,2011,0,0,0,0 +UNREASONABLY,2009,0,0,0,0,0,0 +UNRECOVERED,2009,0,0,0,0,0,0 +UNREMEDIED,2009,0,0,0,0,0,0 +UNSAFE,2009,0,0,0,0,0,0 +UNSATISFIED,2009,0,0,0,0,0,0 +UNSEASONABLY,0,0,2009,0,0,0,0 +UNSOUND,2009,0,0,0,0,0,0 +UNSTABLE,2009,0,0,0,0,0,0 +UNSUCCESSFULLY,2009,0,0,0,0,0,0 +UNSUITED,2009,0,0,0,0,0,0 +UNSUSPECTING,2009,0,0,0,0,0,0 +UNTIMELY,2009,0,0,0,0,0,0 +UNTRUTHFUL,2009,0,0,0,0,0,0 +UNUSABLE,2009,0,0,0,0,0,0 +UNWARRANTED,2009,0,0,0,0,0,0 +UNWRITTEN,0,0,2009,0,0,0,0 +URGENCY,2009,0,0,0,0,0,0 +USURPATION,0,0,0,2009,0,0,0 +USURY,2009,0,0,2009,0,0,0 +VAGUENESS,0,0,2012,0,0,0,0 +VALUABLE,0,2009,0,0,0,0,0 +PLEAD,2009,0,0,0,0,0,0 +PLEADS,2009,0,0,2009,0,0,0 +PLEASED,0,2009,0,0,0,0,0 +PLEDGED,0,0,0,0,0,0,2009 +PLEDGING,0,0,0,0,0,0,2009 +COMPEL,0,0,0,0,0,0,2011 +COMPENSATORY,0,0,0,2009,0,0,0 +COMPLAINED,2009,0,0,0,0,0,0 +COMPLAINTS,2009,0,0,0,0,0,0 +COMMITMENT,0,0,0,0,0,0,2009 +COMMITTING,0,0,0,0,0,0,2009 +LIMITATION,2009,0,0,0,0,0,0 +LINGERING,2009,0,0,0,0,0,0 +RESTRAINING,0,0,0,0,0,0,2009 +RESTRICT,0,0,0,0,0,0,2009 +RESTRICTIONS,0,0,0,0,0,0,2009 +RESTRICTS,0,0,0,0,0,0,2009 +RESTRUCTURING,2009,0,0,0,0,0,0 +RETALIATES,2009,0,0,0,0,0,0 +RETALIATORY,2009,0,0,0,0,0,0 +PRECAUTIONS,0,0,2009,0,0,0,0 +PRECLUDE,2009,0,0,0,0,0,2009 +PRECONDITION,0,0,0,0,0,0,2009 +PREDECEASED,0,0,0,2009,0,0,0 +PREDICTABILITY,0,0,2009,0,0,0,0 +PREDICTIONS,0,0,2009,0,0,0,0 +PREDICTS,0,0,2009,0,0,0,0 +PREJUDICE,2009,0,0,2009,0,0,0 +PREJUDICING,2009,0,0,2009,0,0,0 +PREMATURELY,2009,0,0,0,0,0,0 +PRESET,0,0,0,0,0,0,2009 +PRESUMABLY,0,0,2009,0,0,0,0 +PRESUMING,0,0,2009,0,0,0,0 +PRETRIAL,2009,0,0,2009,0,0,0 +PREVENTION,2009,0,0,0,0,0,0 +PROACTIVE,0,2009,0,0,0,0,0 +PROBABILITY,0,0,2009,0,0,0,0 +PROBATED,0,0,0,2009,0,0,0 +PROBATIONAL,0,0,0,2009,0,0,0 +PROBATIONS,0,0,0,2009,0,0,0 +PROBLEMS,2009,0,0,0,0,0,0 +PROFITABILITY,0,2009,0,0,0,0,0 +PROGRESSED,0,2009,0,0,0,0,0 +PROHIBITED,0,0,0,0,0,0,2009 +PROHIBITIVE,0,0,0,0,0,0,2009 +PROLONG,2009,0,0,0,0,0,0 +PROLONGING,2009,0,0,0,0,0,0 +PROMULGATES,0,0,0,2009,0,0,0 +PROMULGATOR,0,0,0,2009,0,0,0 +PRORATION,0,0,0,2009,0,0,0 +PROSECUTING,2009,0,0,2009,0,0,0 +PROSECUTORIAL,0,0,0,2011,0,0,0 +PROSPERITY,0,2009,0,0,0,0,0 +PROTESTED,2009,0,0,0,0,0,0 +PROTESTOR,2009,0,0,0,0,0,0 +PROTRACTION,2009,0,0,0,0,0,0 +PROVOKE,2009,0,0,0,0,0,0 +PUNISHABLE,0,0,0,2009,0,0,0 +PUNISHMENT,2009,0,0,0,0,0,0 +PURPORTED,2009,0,0,0,0,0,0 +QUESTION,2009,0,0,0,0,0,0 +QUESTIONING,2009,0,0,0,0,0,0 +QUITCLAIMS,0,0,0,2009,0,0,0 +RANDOM,0,0,2009,0,0,0,0 +RANDOMIZING,0,0,2009,0,0,0,0 +RATABLE,0,0,0,2009,0,0,0 +RATIONALIZE,2009,0,0,0,0,0,0 +ABANDONED,2009,0,0,0,0,0,0 +ABANDONS,2009,0,0,0,0,0,0 +ABDICATION,2009,0,0,0,0,0,0 +ABERRATIONAL,2009,0,0,0,0,0,0 +ABEYANCES,0,0,2009,0,0,0,0 +ABNORMAL,2009,0,0,0,0,0,0 +ABOLISH,2009,0,0,0,0,0,0 +ABOVEMENTIONED,0,0,0,2011,0,0,0 +ABROGATING,2009,0,0,2009,0,0,0 +ABRUPTLY,2009,0,0,0,0,0,0 +ABSENTEEISM,2009,0,0,0,0,0,0 +ABSOLVING,0,0,0,2009,0,0,0 +ABUSED,2009,0,0,0,0,0,0 +ABUSIVELY,2009,0,0,0,0,0,0 +ACCIDENT,2009,0,0,0,0,0,0 +ACCLAIMED,0,2009,0,0,0,0,0 +ACCOMPLISHING,0,2009,0,0,0,0,0 +ACCUSATIONS,2009,0,0,0,0,0,0 +ACCUSING,2009,0,0,0,0,0,0 +ACHIEVEMENTS,0,2009,0,0,0,0,0 +ACQUIESCED,2009,0,0,0,0,0,0 +ACQUIRORS,0,0,0,2011,0,0,0 +ACQUITTALS,2009,0,0,2009,0,0,0 +ACQUITTING,2009,0,0,2009,0,0,0 +ADJOURNED,0,0,0,2009,0,0,0 +ADJOURNS,0,0,0,2009,0,0,0 +ADJUDGING,0,0,0,2009,0,0,0 +ADJUDICATING,0,0,0,2009,0,0,0 +ADJUDICATOR,0,0,0,2009,0,0,0 +ADMISSIBLE,0,0,0,2009,0,0,0 +ADULTERATE,2009,0,0,0,0,0,0 +ADULTERATIONS,2009,0,0,0,0,0,0 +ADVANCING,0,2009,0,0,0,0,0 +ADVANTAGEOUSLY,0,2009,0,0,0,0,0 +ADVERSARY,2009,0,0,0,0,0,0 +ADVERSITY,2009,0,0,0,0,0,0 +AFFREIGHTMENT,0,0,0,2012,0,0,0 +AFORESTATED,0,0,0,2011,0,0,0 +AGGRAVATE,2009,0,0,0,0,0,0 +AGGRAVATION,2009,0,0,0,0,0,0 +ALERTING,2009,0,0,0,0,0,0 +ALIENATING,2009,0,0,0,0,0,0 +ALLEGATIONS,2009,0,0,2009,0,0,0 +ALLEGES,2009,0,0,2009,0,0,0 +ALMOST,0,0,2009,0,0,2009,0 +AMBIGUITIES,0,0,2009,0,0,0,0 +AMENDABLE,0,0,0,2009,0,0,0 +AMENDMENT,0,0,0,2009,0,0,0 +ANNOYANCE,2009,0,0,0,0,0,0 +ANNOYS,2009,0,0,0,0,0,0 +ANNULMENT,2009,0,0,0,0,0,0 +ANOMALOUS,2009,0,2009,0,0,0,0 +ANTECEDENTS,0,0,0,2009,0,0,0 +ANTICIPATING,0,0,2009,0,0,0,0 +ANTICORRUPTION,0,0,0,2014,0,0,0 +APPARENTLY,0,0,2009,0,0,2009,0 +APPEALING,0,0,0,2009,0,0,0 +APPEARING,0,0,2009,0,0,2009,0 +APPELLATE,0,0,0,2009,0,0,0 +APPROXIMATED,0,0,2009,0,0,0,0 +APPROXIMATION,0,0,2009,0,0,0,0 +APPURTENANT,0,0,0,2009,0,0,0 +ARBITRARINESS,0,0,2009,0,0,0,0 +ARBITRATES,0,0,0,2009,0,0,0 +ARBITRATIONS,0,0,0,2009,0,0,0 +ARGUE,2009,0,0,0,0,0,0 +ARGUMENTATIVE,2009,0,0,0,0,0,0 +ARREARS,2009,0,0,0,0,0,0 +ARTIFICIALLY,2009,0,0,0,0,0,0 +RETRIBUTION,2009,0,0,0,0,0,0 +RETROCESSIONAIRES,0,0,0,2011,0,0,0 +BOLSTERS,0,2009,0,0,0,0,0 +BOOMING,0,2009,0,0,0,0,0 +BOTTLENECKS,2009,0,0,0,0,0,0 +BOYCOTTED,2009,0,0,0,0,0,0 +BREACHED,2009,0,0,2009,0,0,0 +BREAKAGE,2009,0,0,0,0,0,0 +BREAKING,-2020,0,0,0,0,0,0 +BRIBE,2009,0,0,0,0,0,0 +BRIBES,2009,0,0,0,0,0,0 +BROKEN,-2020,0,0,0,0,0,0 +BURDENS,2009,0,0,0,0,0,0 +CALAMITOUS,2009,0,0,0,0,0,0 +CANCELING,2009,0,0,0,0,0,0 +CANCELLING,2009,0,0,0,0,0,0 +CARELESSNESS,2009,0,0,0,0,0,0 +CATASTROPHIC,2009,0,0,0,0,0,0 +CAUTIONED,2009,0,0,0,0,0,0 +CAUTIOUSLY,0,0,2009,0,0,0,0 +CEASES,2009,0,0,0,0,0,0 +CENSURE,2009,0,0,0,0,0,0 +CERTIORARI,0,0,0,2011,0,0,0 +CHALLENGES,2009,0,0,0,0,0,0 +CHATTEL,0,0,0,2009,0,0,0 +CIRCUMVENTED,2009,0,0,0,0,0,0 +CIRCUMVENTS,2009,0,0,0,0,0,0 +SHALL,0,0,0,2009,0,0,0 +SHORTAGES,2009,0,0,0,0,0,0 +SHRINKAGES,2009,0,0,0,0,0,0 +SHUTS,2009,0,0,0,0,0,0 +SLANDEROUS,2009,0,0,0,0,0,0 +REPUDIATING,2009,0,0,0,0,0,0 +REQUESTOR,0,0,0,2011,0,0,0 +REQUIREMENTS,0,0,0,0,0,0,2009 +RESCIND,0,0,0,2009,0,0,0 +RESCISSION,0,0,0,2009,0,0,0 +RESIGNATIONS,2009,0,0,0,0,0,0 +RESOLVE,0,2009,0,0,0,0,0 +RESTATEMENTS,2009,0,0,0,0,0,0 +NOTARIES,0,0,0,2009,0,0,0 +NOTARIZED,0,0,0,2009,0,0,0 +NOVO,0,0,0,2011,0,0,0 +NULLIFICATIONS,2009,0,0,2009,0,0,0 +NULLIFYING,2009,0,0,2009,0,0,0 +OBJECTING,2009,0,0,0,0,0,0 +OBJECTIONS,2009,0,0,0,0,0,0 +OBLIGATING,0,0,0,0,0,0,2009 +OBLIGE,0,0,0,0,0,0,2009 +OBLIGES,0,0,0,0,0,0,2009 +OBSCENITY,2009,0,0,0,0,0,0 +OBSTACLES,2009,0,0,0,0,0,0 +OBSTRUCTION,2009,0,0,0,0,0,0 +OFFENCES,2009,0,0,0,0,0,0 +OFFENDERS,2009,0,0,0,0,0,0 +OFFEREE,0,0,0,2009,0,0,0 +OMISSION,2009,0,0,0,0,0,0 +OMITTED,2009,0,0,0,0,0,0 +OPPORTUNISTICALLY,2009,0,0,0,0,0,0 +OPPOSED,2009,0,0,0,0,0,0 +OPPOSITIONS,2009,0,0,0,0,0,0 +ORDINARILY,0,0,2009,0,0,0,0 +OUTMODED,2009,0,0,0,0,0,0 +OUTPERFORMS,0,2009,0,0,0,0,0 +OVERBUILDING,2009,0,0,0,0,0,0 +OVERBURDENED,2009,0,0,0,0,0,0 +OVERCHARGE,2009,0,0,0,0,0,0 +OVERCOME,2009,0,0,0,0,0,0 +OVERESTIMATE,2009,0,0,0,0,0,0 +OVERESTIMATION,2009,0,0,0,0,0,0 +OVERLOADING,2009,0,0,0,0,0,0 +OVERLOOKING,2009,0,0,0,0,0,0 +OVERPAYMENTS,2009,0,0,0,0,0,0 +OVERPRODUCTION,2009,0,0,0,0,0,0 +OVERRULING,0,0,0,2009,0,0,0 +OVERSHADOW,2009,0,0,0,0,0,0 +OVERSTATE,2009,0,0,0,0,0,0 +OVERSTATES,2009,0,0,0,0,0,0 +OVERSUPPLY,2009,0,0,0,0,0,0 +OVERTURNED,2009,0,0,0,0,0,0 +OVERVALUED,2009,0,0,0,0,0,0 +PARA,0,0,0,-2020,0,0,0 +NECESSITATED,0,0,0,0,0,0,2009 +NEGATIVELY,2009,0,0,0,0,0,0 +NEGLECTFUL,2009,0,0,0,0,0,0 +NEGLIGENCES,2009,0,0,0,0,0,0 +NOLO,0,0,0,2011,0,0,0 +BEAUTIFUL,0,2009,0,0,0,0,0 +BELIEVES,0,0,2009,0,0,0,0 +IDEAL,0,2009,0,0,0,0,0 +IGNORE,2009,0,0,0,0,0,0 +ILL,2009,0,0,0,0,0,0 +ILLEGALLY,2009,0,0,0,0,0,0 +ILLIQUID,2009,0,0,0,0,0,0 +IMMATERIALITY,0,0,0,2009,0,0,0 +IMPAIRED,2009,0,0,0,0,0,2011 +IMPAIRS,2009,0,0,0,0,0,2011 +IMPEDED,2009,0,0,0,0,0,0 +IMPEDING,2009,0,0,0,0,0,0 +IMPERFECTIONS,2009,0,0,0,0,0,0 +IMPLICATE,2009,0,0,0,0,0,0 +IMPOSE,0,0,0,0,0,0,2009 +IMPOSITION,0,0,0,0,0,0,2009 +IMPOUND,2009,0,0,0,0,0,0 +IMPRACTICABLE,2009,0,0,0,0,0,0 +IMPRECISE,0,0,2009,0,0,0,0 +IMPRESSED,0,2009,0,0,0,0,0 +IMPRESSIVELY,0,2009,0,0,0,0,0 +IMPROPER,2009,0,0,0,0,0,0 +IMPROVE,0,2009,0,0,0,0,0 +IMPROVES,0,2009,0,0,0,0,0 +INABILITY,2009,0,0,0,0,0,0 +INACCURATE,2009,0,0,0,0,0,0 +INACTIVATE,2009,0,0,0,0,0,0 +INACTIVATION,2009,0,0,0,0,0,0 +INADEQUACY,2009,0,0,0,0,0,0 +INADVERTENTLY,2009,0,0,0,0,0,0 +INAPPROPRIATELY,2009,0,0,0,0,0,0 +INCAPACITATED,2009,0,0,0,0,0,0 +INCARCERATES,2009,0,0,2009,0,0,0 +INCHOATE,0,0,0,2009,0,0,0 +INCIDENTS,2009,0,0,0,0,0,0 +INCOMPETENCE,2009,0,0,0,0,0,0 +INCOMPETENTS,2009,0,0,0,0,0,0 +INCONCLUSIVE,2009,0,0,0,0,0,0 +INCONSISTENTLY,2009,0,0,0,0,0,0 +INCONVENIENCES,2009,0,0,0,0,0,0 +INCORRECTNESS,2009,0,0,0,0,0,0 +INDECENCY,2009,0,0,0,0,0,0 +INDEFINITE,0,0,2009,0,0,0,0 +INDEMNIFICATION,0,0,0,2009,0,0,0 +INDEMNIFY,0,0,0,2009,0,0,0 +INDEMNITIES,0,0,0,2009,0,0,0 +INDETERMINABLE,0,0,2009,0,0,0,0 +INDICTED,2009,0,0,2009,0,0,0 +INDORSEES,0,0,0,2011,0,0,0 +INEFFICIENCIES,2009,0,0,0,0,0,0 +INELIGIBILITY,2009,0,0,0,0,0,0 +INEQUITIES,2009,0,0,0,0,0,0 +INEXACTNESS,0,0,2009,0,0,0,0 +INFLICTED,2009,0,0,0,0,0,0 +INFRACTION,2009,0,0,2009,0,0,0 +INFRINGEMENT,2009,0,0,0,0,0,0 +INFRINGING,2009,0,0,0,0,0,0 +INHIBITING,0,0,0,0,0,0,2011 +INJUNCTIONS,2009,0,0,2009,0,0,0 +INJURES,2009,0,0,0,0,0,0 +INJURY,2009,0,0,0,0,0,0 +INNOVATING,0,2009,0,0,0,0,0 +INNOVATIVENESS,0,2011,0,0,0,0,0 +INORDINATELY,2009,0,0,0,0,0,0 +INSIGHTFUL,0,2009,0,0,0,0,0 +INSISTING,0,0,0,0,0,0,2009 +INSOLVENCY,2009,0,0,0,0,0,0 +INSTABILITIES,0,0,2009,0,0,0,0 +INSUFFICIENT,2009,0,0,0,0,0,0 +INTANGIBLE,0,0,2009,0,0,0,0 +INTERFERE,2009,0,0,0,0,0,0 +INTERFERES,2009,0,0,0,0,0,0 +INTERMITTENTLY,2009,0,0,0,0,0,0 +INTERPOSES,0,0,0,2009,0,0,0 +INTERROGATE,0,0,0,2009,0,0,0 +INTERROGATION,0,0,0,2009,0,0,0 +INTERROGATORS,0,0,0,2009,0,0,0 +INTERRUPTING,2009,0,0,0,0,0,0 +INTESTACY,0,0,0,2009,0,0,0 +STABILIZATION,0,2009,0,0,0,0,0 +STABILIZES,0,2009,0,0,0,0,0 +STAGNANT,2009,0,0,0,0,0,0 +STAGNATING,2009,0,0,0,0,0,0 +STATUTE,0,0,0,2009,0,0,0 +STIPULATE,0,0,0,0,0,0,2009 +STIPULATION,0,0,0,0,0,0,2009 +STOPPAGES,2009,0,0,0,0,0,0 +STRAIN,2009,0,0,0,0,0,0 +STRENGTH,0,2009,0,0,0,0,0 +STRENGTHENS,0,2009,0,0,0,0,0 +STRESSES,2009,0,0,0,0,0,0 +STRICTER,0,0,0,0,0,0,2009 +STRONG,0,2009,0,0,0,0,0 +SUBCLAUSE,0,0,0,2009,0,0,0 +SUBJECTING,2009,0,0,0,0,0,0 +SUBLESSORS,0,0,0,2011,0,0,0 +SUBPARAGRAPHS,0,0,0,2009,0,0,0 +SUBROGATED,0,0,0,2009,0,0,0 +SUBTRUSTS,0,0,0,2011,0,0,0 +SUCCEEDS,0,2009,0,0,0,0,0 +SUCCESSFULLY,0,2009,0,0,0,0,0 +SUED,2009,0,0,2009,0,0,0 +SUFFERING,2009,0,0,0,0,0,0 +SUGGESTING,0,0,2009,0,0,0,0 +SUMMONING,2009,0,0,2009,0,0,0 +SUPERSEDE,0,0,0,2009,0,0,0 +SUPERSEDING,0,0,0,2009,0,0,0 +SURPASSED,0,2009,0,0,0,0,0 +SUSCEPTIBLE,2009,0,0,0,0,0,0 +SUSPEND,2009,0,0,0,0,0,0 +SUSPENSION,2009,0,0,0,0,0,0 +SUSPICIOUS,2009,0,0,0,0,0,0 +REVOCATION,2009,0,0,2009,0,0,0 +REVOKES,2009,0,0,0,0,0,0 +REVOLUTIONIZES,0,2009,0,0,0,0,0 +REWARDING,0,2009,0,0,0,0,0 +RIDICULES,2009,0,0,0,0,0,0 +RISKIER,2009,0,2009,0,0,0,0 +RISKS,0,0,2009,0,0,0,0 +RULINGS,0,0,0,2009,0,0,0 +SACRIFICED,2009,0,0,0,0,0,0 +SATISFACTION,0,2009,0,0,0,0,0 +SATISFIES,0,2009,0,0,0,0,0 +SCANDALS,2009,0,0,0,0,0,0 +SCRUTINIZING,2009,0,0,0,0,0,0 +SEIZE,2009,0,0,0,0,0,0 +SELDOM,0,0,2009,0,0,2009,0 +SEQUESTRATOR,0,0,0,2009,0,0,0 +SETBACK,2009,0,0,0,0,0,0 +SEVER,2009,0,0,0,0,0,0 +SEVERANCE,0,0,0,2009,0,0,0 +SEVERELY,2009,0,0,0,0,0,0 +ENTRENCH,0,0,0,0,0,0,2009 +ERODES,2009,0,0,0,0,0,0 +ERRATICALLY,2009,0,0,0,0,0,0 +ERRONEOUSLY,2009,0,0,0,0,0,0 +ESCALATE,2009,0,0,0,0,0,0 +ESCHEAT,0,0,0,2011,0,0,0 +ESCROWED,0,0,0,0,0,0,2009 +EVADE,2009,0,0,0,0,0,0 +EVASION,2009,0,0,0,0,0,0 +EVICTED,2009,0,0,0,0,0,0 +EVICTS,2009,0,0,0,0,0,0 +EXACERBATED,2009,0,0,0,0,0,0 +EXACERBATIONS,2009,0,0,0,0,0,0 +EXAGGERATING,2009,0,0,0,0,0,0 +EXCEEDENCES,0,0,0,2011,0,0,0 +EXCELS,0,2009,0,0,0,0,0 +EXCESSIVELY,2009,0,0,0,0,0,0 +EXCITING,0,2009,0,0,0,0,0 +EXCLUSIVES,0,2009,0,0,0,0,0 +EXCULPATES,2009,0,0,2009,0,0,0 +EXCULPATORY,2009,0,0,2009,0,0,0 +EXECUTRICES,0,0,0,2009,0,0,0 +EXONERATE,2009,0,0,0,0,0,0 +EXONERATION,2009,0,0,0,0,0,0 +EXPLOITATIONS,2009,0,0,0,0,0,0 +EXPLOITS,2009,0,0,0,0,0,0 +EXPOSING,2009,0,0,0,0,0,0 +EXPROPRIATED,2009,0,0,0,0,0,0 +EXPROPRIATIONS,2009,0,0,0,0,0,0 +EXTRACONTRACTUAL,0,0,0,2011,0,0,0 +SOLVENCIES,2009,0,0,0,0,0,0 +SOMETIME,0,0,2009,0,0,0,0 +SPAM,2014,0,0,0,0,0,0 +TRAUMATIC,2009,0,0,0,0,0,0 +LOSES,2009,0,0,0,0,0,0 +LOST,2009,0,0,0,0,0,0 +LYING,2009,0,0,0,0,0,0 +MALFUNCTIONED,2009,0,0,0,0,0,0 +MALICIOUS,2009,0,0,0,0,0,0 +MANDATE,0,0,0,0,0,0,2009 +MANDATORY,0,0,0,0,0,0,2009 +MANIPULATES,2009,0,0,0,0,0,0 +MANIPULATIVE,2009,0,0,0,0,0,0 +MAYBE,0,0,2009,0,0,2009,0 +MEDIATING,0,0,0,2009,0,0,0 +MEDIATORS,0,0,0,2009,0,0,0 +MISAPPLICATIONS,2009,0,0,0,0,0,0 +MISAPPLYING,2009,0,0,0,0,0,0 +MISAPPROPRIATING,2009,0,0,0,0,0,0 +MISCALCULATE,2009,0,0,0,0,0,0 +MISCALCULATION,2009,0,0,0,0,0,0 +MISCLASSIFICATION,2011,0,0,0,0,0,0 +MISCOMMUNICATION,2014,0,0,0,0,0,0 +MISDEMEANORS,2009,0,0,0,0,0,0 +MISHANDLED,2009,0,0,0,0,0,0 +MISINFORMATION,2009,0,0,0,0,0,0 +MISINTERPRET,2009,0,0,0,0,0,0 +MISINTERPRETING,2009,0,0,0,0,0,0 +MISJUDGES,2009,0,0,0,0,0,0 +MISLABEL,2009,0,0,0,0,0,0 +MISLABELS,2009,0,0,0,0,0,0 +MISLEADS,2009,0,0,0,0,0,0 +MISMANAGEMENT,2009,0,0,0,0,0,0 +MISMATCHED,2009,0,0,0,0,0,0 +MISPRICE,2014,0,0,0,0,0,0 +MISREPRESENTATION,2009,0,0,0,0,0,0 +MISREPRESENTS,2009,0,0,0,0,0,0 +WORRYING,2009,0,0,0,0,0,0 +WORSENING,2009,0,0,0,0,0,0 +WORTHY,0,2009,0,0,0,0,0 +TOLERATED,2009,0,0,0,0,0,0 +TORT,0,0,0,2009,0,0,0 +TORTUOUS,2009,0,0,0,0,0,0 +FIDE,0,0,0,2009,0,0,0 +FIRING,2009,0,0,0,0,0,0 +NONBREACHING,0,0,0,2011,0,0,0 +NONCOMPLIANCE,2009,0,0,0,0,0,0 +NONCONFORMING,2009,0,0,0,0,0,0 +NONCONTRACT,0,0,0,2012,0,0,0 +NONFEASANCE,0,0,0,2011,0,0,0 +NONFORFEITURE,0,0,0,2011,0,0,0 +DISCLAIMED,2009,0,0,0,0,0,0 +DISCLAIMS,2009,0,0,0,0,0,0 +DISCLOSING,2009,0,0,0,0,0,0 +DISCONTINUATIONS,2009,0,0,0,0,0,0 +DISCONTINUING,2009,0,0,0,0,0,0 +DISCOURAGING,2009,0,0,0,0,0,0 +DISCREDITS,2009,0,0,0,0,0,0 +SPECTACULARLY,0,2009,0,0,0,0,0 +SPECULATING,0,0,2009,0,0,0,0 +SPECULATIVELY,0,0,2009,0,0,0,0 +TRAGEDY,2009,0,0,0,0,0,0 +TRANSFERORS,0,0,0,2012,0,0,0 +LEGISLATING,0,0,0,2009,0,0,0 +LEGISLATIVELY,0,0,0,2009,0,0,0 +LEGISLATURES,0,0,0,2009,0,0,0 +LIBELS,0,0,0,2009,0,0,0 +CONCEALED,2009,0,0,0,0,0,0 +CONCEDES,2009,0,0,0,0,0,0 +CONCERN,2009,0,0,0,0,0,0 +CONCILIATION,2009,0,0,0,0,0,0 +CONDEMN,2009,0,0,0,0,0,0 +CONDEMNING,2009,0,0,0,0,0,0 +CONDITIONALLY,0,0,2009,0,0,0,0 +CONFESS,2009,0,0,0,0,0,0 +CONFESSION,2009,0,0,0,0,0,0 +CONFINEMENT,2009,0,0,0,0,0,2009 +CONFISCATE,2009,0,0,0,0,0,0 +CONFISCATION,2009,0,0,0,0,0,0 +CONFLICTED,2009,0,0,0,0,0,0 +CONFRONTATION,2009,0,0,0,0,0,0 +CONFRONTING,2009,0,0,0,0,0,0 +CONFUSES,2009,0,2009,0,0,0,0 +CONSENT,0,0,0,2009,0,0,0 +CONSERVATORSHIPS,0,0,0,2011,0,0,0 +CONSPIRATORIAL,2009,0,0,0,0,0,0 +CONSPIRES,2009,0,0,0,0,0,0 +CONSTITUTIONALITY,0,0,0,2009,0,0,0 +CONSTRAIN,0,0,0,0,0,0,2009 +CONSTRAINT,0,0,0,0,0,0,2009 +CONSTRUE,0,0,0,2012,0,0,0 +CONTEMPT,2009,0,0,0,0,0,0 +CONTENDS,2009,0,0,0,0,0,0 +CONTENTIOUSLY,2009,0,0,0,0,0,0 +CONTESTING,2009,0,0,0,0,0,0 +CONTINGENTLY,0,0,2009,0,0,0,0 +CONTRACTHOLDER,0,0,0,2009,0,0,0 +CONTRACTING,0,0,0,2009,0,0,0 +CONTRACTUAL,0,0,0,2009,0,0,0 +CONTRADICTING,2009,0,0,0,0,0,0 +CONTRADICTS,2009,0,0,0,0,0,0 +CONTRAVENES,0,0,0,2009,0,0,0 +CONTROVERSIAL,2009,0,0,0,0,0,0 +CONTROVERTED,0,0,0,2009,0,0,0 +CONVEYANCES,0,0,0,2009,0,0,0 +CONVICTION,2009,0,0,2009,0,0,0 +CORRECTION,2009,0,0,0,0,0,0 +CORRUPTED,2009,0,0,0,0,0,0 +CORRUPTLY,2009,0,0,0,0,0,0 +COULD,0,0,2009,0,0,2009,0 +COUNSELS,0,0,0,2009,0,0,0 +COUNTERCLAIMS,2009,0,0,0,0,0,0 +COUNTERFEITERS,2009,0,0,0,0,0,0 +COUNTERMEASURES,2009,0,0,0,0,0,0 +COUNTERSUITS,0,0,0,2014,0,0,0 +COURTS,0,0,0,2009,0,0,0 +COVENANTS,0,0,0,0,0,0,2011 +CREATIVITY,0,2009,0,0,0,0,0 +CRIMINALITY,0,0,0,2009,0,0,0 +CRIMINALS,2009,0,0,2009,0,0,0 +CRITICALLY,2009,0,0,0,0,0,0 +CRITICIZED,2009,0,0,0,0,0,0 +CROSSCLAIMS,0,0,0,2011,0,0,0 +CRUCIALLY,2009,0,0,0,0,0,0 +CUMBERSOME,2009,0,0,0,0,0,0 +CURTAILMENT,2009,0,0,0,0,0,0 +CUTBACK,2009,0,0,0,0,0,0 +CYBERBULLYING,2014,0,0,0,0,0,0 +CYBERCRIMINALS,2014,0,0,0,0,0,0 +TAINTED,2009,0,0,0,0,0,0 +TENANTABILITY,0,0,0,2011,0,0,0 +TENTATIVELY,0,0,2009,0,0,0,0 +TERMINATES,2009,0,0,0,0,0,0 +TERMINUS,0,0,0,-2020,0,0,0 +TESTIMONY,0,0,0,2009,0,0,0 +THEREAFTER,0,0,0,2009,0,0,0 +THEREINAFTER,0,0,0,2011,0,0,0 +THERETO,0,0,0,2009,0,0,0 +THEREUNTO,0,0,0,2009,0,0,0 +THREATEN,2009,0,0,0,0,0,0 +THREATS,2009,0,0,0,0,0,0 +FAIL,2009,0,0,0,0,0,0 +FAILS,2009,0,0,0,0,0,0 +FALSE,2009,0,0,0,0,0,0 +FALSIFIED,2009,0,0,0,0,0,0 +FALSITY,2009,0,0,0,0,0,0 +FATALLY,2009,0,0,0,0,0,0 +FAULTY,2009,0,0,0,0,0,0 +FAVORING,0,2009,0,0,0,0,0 +FEARS,2009,0,0,0,0,0,0 +DAMAGE,2009,0,0,0,0,0,0 +DAMPEN,2009,0,0,0,0,0,0 +DANGEROUSLY,2009,0,0,0,0,0,0 +DEADLOCKING,2009,0,0,0,0,0,0 +DEBARMENT,2009,0,0,0,0,0,0 +DECEDENT,0,0,0,2009,0,0,0 +DECEITFULNESS,2009,0,0,0,0,0,0 +DECEIVING,2009,0,0,0,0,0,0 +DECEPTIVELY,2009,0,0,0,0,0,0 +DECLINES,2009,0,0,0,0,0,0 +DECREEING,0,0,0,2009,0,0,0 +DEFACEMENT,2009,0,0,0,0,0,0 +DEFAMATIONS,2009,0,0,0,0,0,0 +DEFAMES,2009,0,0,0,0,0,0 +DEFAULTING,2009,0,0,0,0,0,0 +DEFEASE,0,0,0,2009,0,0,0 +DEFEASING,0,0,0,2011,0,0,0 +DEFEATS,2009,0,0,0,0,0,0 +DEFECTS,2009,0,0,0,0,0,0 +DEFENDANTS,2009,0,0,2009,0,0,0 +DEFENSIVE,2009,0,0,0,0,0,0 +DEFICIENCY,2009,0,0,0,0,0,0 +DEFINITELY,0,0,0,0,2009,0,0 +DEFRAUDING,2009,0,0,0,0,0,0 +DEGRADATIONS,2009,0,0,0,0,0,0 +DEGRADING,2009,0,0,0,0,0,0 +DELAYS,2009,0,0,0,0,0,0 +DELEGEES,0,0,0,2011,0,0,0 +DELIBERATELY,2009,0,0,0,0,0,0 +DELIGHTFULLY,0,2009,0,0,0,0,0 +DELINQUENCY,2009,0,0,0,0,0,0 +DELIST,2009,0,0,0,0,0,0 +DEMISE,2009,0,0,0,0,0,0 +DEMOLISH,2009,0,0,0,0,0,0 +DEMOLITION,2009,0,0,0,0,0,0 +DEMOTES,2009,0,0,0,0,0,0 +DEMURRED,0,0,0,2009,0,0,0 +DEMURS,0,0,0,2009,0,0,0 +DENIES,2009,0,0,0,0,0,0 +DENIGRATING,2009,0,0,0,0,0,0 +DEPEND,0,0,2009,0,0,2009,2011 +DEPENDANCES,0,0,0,0,0,0,2011 +DEPENDENCIES,0,0,2009,0,0,0,2011 +DEPENDS,0,0,2009,0,0,2009,2011 +DEPLETING,2009,0,0,0,0,0,0 +DEPOSED,0,0,0,2009,0,0,0 +DEPOSITIONAL,0,0,0,2011,0,0,0 +INVALIDATED,2009,0,0,0,0,0,0 +INVALIDITY,2009,0,0,0,0,0,0 +INVENTION,0,2009,0,0,0,0,0 +INVENTOR,0,2009,0,0,0,0,0 +INVESTIGATES,2009,0,0,0,0,0,0 +INVOLUNTARILY,2009,0,0,0,0,0,0 +IRRECOVERABLE,2009,0,0,0,0,0,0 +IRREGULARITY,2009,0,0,0,0,0,0 +IRREVERSIBLE,2009,0,0,0,0,0,0 +JEOPARDIZE,2009,0,0,0,0,0,0 +JUDICIALLY,0,0,0,2009,0,0,0 +JURIS,0,0,0,2011,0,0,0 +JURISDICTIONS,0,0,0,2009,0,0,0 +JUROR,0,0,0,2009,0,0,0 +JUSTICE,0,0,0,2009,0,0,0 +KICKBACKS,2009,0,0,0,0,0,0 +DIMINISHED,2009,0,0,0,0,0,0 +DIRECTIVE,0,0,0,0,0,0,2009 +DISADVANTAGEOUS,2009,0,0,0,0,0,0 +DISAFFIRMANCE,0,0,0,2009,0,0,0 +DISAGREEABLE,2009,0,0,0,0,0,0 +DISAGREEMENTS,2009,0,0,0,0,0,0 +DISALLOWANCES,2009,0,0,0,0,0,0 +DISAPPEAR,2009,0,0,0,0,0,0 +DISAPPEARING,2009,0,0,0,0,0,0 +DISAPPOINTING,2009,0,0,0,0,0,0 +DISAPPOINTS,2009,0,0,0,0,0,0 +DISAPPROVED,2009,0,0,0,0,0,0 +DISASSOCIATING,2009,0,0,0,0,0,0 +DISASTERS,2009,0,0,0,0,0,0 +DISAVOWAL,2009,0,0,0,0,0,0 +GRATUITOUSLY,2009,0,0,0,0,0,0 +GREATLY,0,2009,0,0,0,0,0 +GROSSLY,2009,0,0,0,0,0,0 +HALTED,2009,0,0,0,0,0,0 +HAMPERS,2009,0,0,0,0,0,0 +HAPPY,0,2009,0,0,0,0,0 +HARASSMENT,2009,0,0,0,0,0,0 +HARMED,2009,0,0,0,0,0,0 +HARMS,2009,0,0,0,0,0,0 +HARSHLY,2009,0,0,0,0,0,0 +HAZARDS,2009,0,0,0,0,0,0 +NONINFRINGING,0,0,0,2011,0,0,0 +NONPAYMENT,2009,0,0,0,0,0,0 +NONPERFORMING,2009,0,0,0,0,0,0 +DISFAVORS,2009,0,0,0,0,0,0 +PREAMENDMENT,0,0,0,2011,0,0,0 +COMPLICATED,2009,0,0,0,0,0,0 +COMPLICATIONS,2009,0,0,0,0,0,0 +COMPLIMENTING,0,2009,0,0,0,0,0 +MISSTATEMENTS,2009,0,0,0,0,0,0 +MISSTEPS,2009,0,0,0,0,0,0 +MISTAKES,2009,0,0,0,0,0,0 +MISUNDERSTAND,2009,0,0,0,0,0,0 +MISUSE,2009,0,0,0,0,0,0 +MONOPOLISTIC,2009,0,0,0,0,0,0 +MONOPOLIZED,2009,0,0,0,0,0,0 +MORATORIA,2009,0,0,0,0,0,0 +MOTHBALLED,2009,0,0,0,0,0,0 +MUTANDIS,0,0,0,2011,0,0,0 +SLIPPAGES,2009,0,0,0,0,0,0 +SLOWED,2009,0,0,0,0,0,0 +SLOWLY,2009,0,0,0,0,0,0 +SLUGGISHNESS,2009,0,0,0,0,0,0 +SMOOTHS,0,2009,0,0,0,0,0 +DEPRESSES,2009,0,0,0,0,0,0 +DEPRIVED,2009,0,0,0,0,0,0 +DERELICTION,2009,0,0,0,0,0,0 +DEROGATING,0,0,0,2009,0,0,0 +DESIGNATOR,0,0,0,2011,0,0,0 +DESPITE,0,2009,0,0,0,0,0 +DESTABILIZING,2009,0,2009,0,0,0,0 +DESTROYING,2009,0,0,0,0,0,0 +DETAIN,2009,0,0,0,0,0,0 +DETENTIONS,2009,0,0,0,0,0,0 +DETERIORATES,2009,0,0,0,0,0,0 +DETERRED,2009,0,0,0,0,0,0 +DETERRENTS,2009,0,0,0,0,0,0 +DETRACTED,2009,0,0,0,0,0,0 +DETRIMENTALLY,2009,0,0,0,0,0,0 +DEVALUES,2009,0,0,0,0,0,0 +DEVASTATING,2009,0,0,0,0,0,0 +DEVIATES,2009,0,2009,0,0,0,0 +DEVISEES,0,0,0,2011,0,0,0 +DEVOLVING,2009,0,0,0,0,0,0 +DICTATING,0,0,0,0,0,0,2009 +DIFFERS,0,0,2009,0,0,0,0 +DIFFICULTY,2009,0,0,0,0,0,0 +ASCENDANTS,0,0,0,2009,0,0,0 +ASSAULTS,2009,0,0,0,0,0,0 +ASSIGNATIONS,0,0,0,2009,0,0,0 +ASSUMES,0,0,2009,0,0,0,0 +ASSURE,0,2009,0,0,0,0,0 +ATTAIN,0,2009,0,0,0,0,0 +ATTAINMENTS,0,2009,0,0,0,0,0 +ATTESTATIONS,0,0,0,2009,0,0,0 +ATTORNEY,0,0,0,2009,0,0,0 +ATTRACTIVE,0,2009,0,0,0,0,0 +BACKDATING,2009,0,0,0,0,0,0 +BAILEE,0,0,0,2009,0,0,0 +BAILMENT,0,0,0,2011,0,0,0 +BANKRUPT,2009,0,0,0,0,0,0 +BANKRUPTING,2009,0,0,0,0,0,0 +CLAIMANTS,0,0,0,2009,0,0,0 +CLARIFICATION,0,0,2009,0,0,0,0 +CLEARLY,0,0,0,0,2009,0,0 +CLOSING,-2020,0,0,0,0,0,0 +CODEFENDANT,0,0,0,2011,0,0,0 +CODIFICATION,0,0,0,2009,0,0,0 +CODIFY,0,0,0,2009,0,0,0 +COERCES,2009,0,0,0,0,0,0 +COLLABORATED,0,2009,0,0,0,0,0 +COLLABORATIONS,0,2009,0,0,0,0,0 +COLLAPSE,2009,0,0,0,0,0,0 +COLLISION,2009,0,0,0,0,0,0 +COLLUDES,2009,0,0,0,0,0,0 +COLLUSIVE,2009,0,0,0,0,0,0 +POSITIVELY,0,2009,0,0,0,0,0 +POSSIBLE,0,0,2009,0,0,2009,0 +POSTCONTRACT,0,0,0,2011,0,0,0 +POSTPONEMENT,2009,0,0,0,0,0,0 +WRITEDOWN,2009,0,0,0,0,0,0 +WRITS,0,0,0,2009,0,0,0 +WRONGFUL,2009,0,0,0,0,0,0 +HONOR,0,2009,0,0,0,0,0 +HONORS,0,2009,0,0,0,0,0 +REARGUMENT,0,0,0,2011,0,0,0 +REASSESSING,0,0,2009,0,0,0,0 +REASSIGNED,2009,0,0,0,0,0,0 +REASSIGNS,2009,0,0,0,0,0,0 +REBUT,0,0,0,2009,0,0,0 +REBUTTAL,0,0,0,2009,0,0,0 +RECALCULATE,0,0,2009,0,0,0,0 +RECALCULATION,0,0,2009,0,0,0,0 +RECALLING,2009,0,0,0,0,0,0 +RECESSIONARY,2009,0,0,0,0,0,0 +RECKLESSNESS,2009,0,0,0,0,0,0 +RECONSIDERS,0,0,2009,0,0,0,0 +RECOUPMENTS,0,0,0,2009,0,0,0 +RECTIFICATIONS,0,0,0,2009,0,0,0 +RECUSES,0,0,0,2014,0,0,0 +REDACTING,2009,0,0,2009,0,0,0 +REDEFAULTED,2012,0,0,0,0,0,0 +REDRESSES,2009,0,0,0,0,0,0 +REEXAMINING,0,0,2009,0,0,0,0 +REFILE,0,0,0,2009,0,0,0 +REFRAIN,0,0,0,0,0,0,2009 +REFUSALS,2009,0,0,0,0,0,0 +REFUSING,2009,0,0,0,0,0,0 +REGULATE,0,0,0,2009,0,0,0 +REGULATION,0,0,0,2009,0,0,0 +REGULATORS,0,0,0,2009,0,0,0 +REHEARING,0,0,0,2009,0,0,0 +VARIABLY,0,0,2009,0,0,0,0 +VARIANTS,0,0,2009,0,0,0,0 +VARIES,0,0,2009,0,0,0,0 +VENDEES,0,0,0,2011,0,0,0 +VERSATILITY,0,2009,0,0,0,0,0 +VIBRANT,0,2009,0,0,0,0,0 +VIOLATES,2009,0,0,0,0,0,0 +VIOLATIVE,2009,0,0,2009,0,0,0 +VIOLENT,2009,0,0,0,0,0,0 +VITIATES,2009,0,0,0,0,0,0 +VOIDED,2009,0,0,2009,0,0,0 +VOLATILITY,2009,0,2009,0,0,0,0 +VULNERABLY,2009,0,0,0,0,0,0 +WARNINGS,2009,0,0,0,0,0,0 +WASTED,2009,0,0,0,0,0,0 +WEAKEN,2009,0,0,0,0,0,0 +WEAKER,2009,0,0,0,0,0,0 +WEAKNESSES,2009,0,0,0,0,0,0 +DISHONORABLE,2009,0,0,0,0,0,0 +DISHONORS,2009,0,0,0,0,0,0 +DISINTERESTEDLY,2009,0,0,0,0,0,0 +DISLOYALTY,2009,0,0,0,0,0,0 +DISMISSAL,2009,0,0,0,0,0,0 +DISMISSING,2009,0,0,0,0,0,0 +DISPARAGEMENT,2009,0,0,0,0,0,0 +DISPARAGINGLY,2009,0,0,0,0,0,0 +DISPLACED,2009,0,0,0,0,0,0 +DISPLACING,2009,0,0,0,0,0,0 +DISPOSSESSED,2009,0,0,0,0,0,0 +DISPOSSESSORY,0,0,0,2011,0,0,0 +DISPROPORTIONATELY,2009,0,0,0,0,0,0 +DISPUTING,2009,0,0,0,0,0,0 +DISQUALIFIES,2009,0,0,0,0,0,0 +DISREGARDED,2009,0,0,0,0,0,0 +DISREPUTE,2009,0,0,0,0,0,0 +DISRUPTION,2009,0,0,0,0,0,0 +DISSATISFACTION,2009,0,0,0,0,0,0 +DISSENTER,2009,0,0,0,0,0,0 +DISSIDENT,2009,0,0,0,0,0,0 +DISTINCTION,0,2009,0,0,0,0,0 +DISTINCTIVENESS,0,2009,0,0,0,0,0 +DISTORTION,2009,0,0,0,0,0,0 +DISTRACTED,2009,0,0,0,0,0,0 +DISTRACTS,2009,0,0,0,0,0,0 +DISTRIBUTEE,0,0,0,2009,0,0,0 +DISTURBANCES,2009,0,0,0,0,0,0 +DIVERSION,2009,0,0,0,0,0,0 +DIVERTS,2009,0,0,0,0,0,0 +DIVESTITURE,2009,0,0,0,0,0,0 +DIVESTS,2009,0,0,0,0,0,0 +DIVULGED,2009,0,0,0,0,0,0 +DOCKETED,0,0,0,2009,0,0,0 +DOUBT,2009,0,2009,0,0,0,0 +DOWNGRADE,2009,0,0,0,0,0,0 +DOWNSIZE,2009,0,0,0,0,0,0 +DOWNSIZINGS,2009,0,0,0,0,0,0 +DOWNTURNS,2009,0,0,0,0,0,0 +DRASTIC,2009,0,0,0,0,0,0 +DREAM,0,2009,0,0,0,0,0 +DULY,0,0,0,2009,0,0,0 +DYSFUNCTIONS,2009,0,0,0,0,0,0 +EARMARKS,0,0,0,0,0,0,2009 +EASY,0,2009,0,0,0,0,0 +EFFICIENT,0,2009,0,0,0,0,0 +EJECTMENT,0,0,0,2011,0,0,0 +EMBARGOING,2009,0,0,0,0,0,0 +EMBARRASSING,2009,0,0,0,0,0,0 +EMBEZZLED,2009,0,0,0,0,0,0 +EMBEZZLES,2009,0,0,0,0,0,0 +EMPOWERING,0,2009,0,0,0,0,0 +ENABLES,0,2009,0,0,0,0,0 +ENCOURAGES,0,2009,0,0,0,0,0 +ENCROACHES,2009,0,0,0,0,0,0 +ENCUMBER,2009,0,0,2009,0,0,2009 +ENCUMBRANCE,2009,0,0,2009,0,0,2009 +ENDANGER,2009,0,0,0,0,0,0 +ENDANGERS,2009,0,0,0,0,0,0 +ENFORCEABLY,0,0,0,2011,0,0,0 +ENHANCEMENTS,0,2009,0,0,0,0,0 +ENJOINED,2009,0,0,0,0,0,0 +ENJOYABLE,0,2009,0,0,0,0,0 +ENJOYMENT,0,2009,0,0,0,0,0 +ENTAILING,0,0,0,0,0,0,2009 +PASSU,0,0,0,2009,0,0,0 +PENALIZED,2009,0,0,0,0,0,0 +PENALTY,2009,0,0,0,0,0,0 +PERFECTLY,0,2009,0,0,0,0,0 +PERILS,2009,0,0,0,0,0,0 +PERMISSIONS,0,0,0,0,0,0,2009 +PERMITTING,0,0,0,0,0,0,2009 +PERPETRATING,2009,0,0,2009,0,0,0 +PERSISTENCE,2009,0,0,0,0,0,0 +PERSISTS,2009,0,0,0,0,0,0 +PERVASIVENESS,2009,0,0,0,0,0,0 +PETITIONERS,0,0,0,2009,0,0,0 +PICKET,2009,0,0,0,0,0,0 +HEREBY,0,0,0,2009,0,0,0 +HEREFROM,0,0,0,2009,0,0,0 +HEREINBEFORE,0,0,0,2009,0,0,0 +HERETO,0,0,0,2009,0,0,0 +HEREUPON,0,0,0,2009,0,0,0 +HIGHEST,0,2009,0,0,2009,0,0 +HINDERS,2009,0,0,0,0,0,0 +WHATEVER,0,0,0,2009,0,0,0 +WHEREAS,0,0,0,2009,0,0,0 +WHEREIN,0,0,0,2009,0,0,0 +WHEREUNDER,0,0,0,2011,0,0,0 +WHOMEVER,0,0,0,2009,0,0,0 +WILL,0,0,0,0,2009,0,0 +WIN,0,2009,0,0,0,0,0 +WITNESS,0,0,0,2009,0,0,0 +FLUCTUATED,0,0,2009,0,0,0,0 +FLUCTUATIONS,0,0,2009,0,0,0,0 +FORBEARANCES,0,0,0,2009,0,0,0 +FORBIDDEN,2009,0,0,0,0,0,2009 +FORCED,2009,0,0,0,0,0,0 +FOREBEARS,0,0,0,2009,0,0,0 +FORECLOSING,2009,0,0,0,0,0,0 +FOREGOES,2009,0,0,0,0,0,0 +FORESTALLING,2009,0,0,0,0,0,0 +FORFEITABLE,0,0,0,2009,0,0,0 +FORFEITURE,2009,0,0,0,0,0,0 +FORTHWITH,0,0,0,2009,0,0,0 +FRAUDULENCE,2009,0,0,0,0,0,0 +FRIVOLOUS,2009,0,0,0,0,0,0 +FRUSTRATES,2009,0,0,0,0,0,0 +FRUSTRATIONS,2009,0,0,0,0,0,0 +GAIN,0,2009,0,0,0,0,0 +GOOD,0,2009,0,0,0,0,0 +DISGORGES,2009,0,0,0,0,0,0 +DISGRACEFULLY,2009,0,0,0,0,0,0 +LITIGATED,2009,0,0,2009,0,0,0 +LITIGATIONS,2009,0,0,2009,0,0,0 +LITIGIOUSNESS,0,0,0,2009,0,0,0 +LACKING,2009,0,0,0,0,0,0 +LAGGED,2009,0,0,0,0,0,0 +LAPSED,2009,0,0,0,0,0,0 +LAUNDERING,2009,0,0,0,0,0,0 +LAWFULNESS,0,0,0,2009,0,0,0 +LAWSUIT,0,0,0,2009,0,0,0 +LAYOFF,2009,0,0,0,0,0,0 +LEGAL,0,0,0,2009,0,0,0 +LEGALIZATIONS,0,0,0,2009,0,0,0 +LEGALIZING,0,0,0,2009,0,0,0 +LEGATEES,0,0,0,2009,0,0,0 +FLAWS,2009,0,0,0,0,0,0 +NONRENEWAL,2009,0,0,0,0,0,0 +LIQUIDATING,2009,0,0,0,0,0,0 +LIQUIDATORS,2009,0,0,0,0,0,0 +BENEFICIAL,0,-2020,0,2020,0,0,0 +BENEFIT,0,-2020,0,0,0,0,0 +BENEFITTING,0,2009,0,0,0,0,0 +POORLY,2009,0,0,0,0,0,0 +REINTERPRETATIONS,0,0,2009,0,0,0,0 +REJECT,2009,0,0,0,0,0,0 +REJECTIONS,2009,0,0,0,0,0,0 +RELINQUISHED,2009,0,0,0,0,0,0 +RELINQUISHMENTS,2009,0,0,0,0,0,0 +REMANDED,0,0,0,2009,0,0,0 +REMEDIATED,0,0,0,2009,0,0,0 +REMEDIED,0,0,0,2009,0,0,0 +RENEGOTIATES,2009,0,0,0,0,0,0 +RENOUNCE,2009,0,0,0,0,0,0 +RENOUNCES,2009,0,0,0,0,0,0 +REPLEDGED,0,0,0,2012,0,0,0 +REPOSSESSING,2009,0,0,0,0,0,0 +TROUBLES,2009,0,0,0,0,0,0 +UNACCEPTABLE,2009,0,0,0,0,0,0 +UNANNOUNCED,2009,0,0,0,0,0,0 +UNAPPROVED,2009,0,0,0,0,0,0 +UNAVAILABLE,2009,0,0,0,0,0,2011 +UNCERTAIN,0,0,2009,0,0,2009,0 +UNCLEAR,0,0,2009,0,0,0,0 +UNCOLLECTIBLE,2009,0,0,0,0,0,0 +UNCOMPROMISING,0,0,0,0,2009,0,0 +UNCONSTITUTIONAL,0,0,0,2009,0,0,0 +UNCONTROLLABLE,2009,0,0,0,0,0,0 +UNCOVER,2009,0,0,0,0,0,0 +UNDECIDED,0,0,2009,0,0,0,0 +UNDELIVERED,2009,0,0,0,0,0,0 +UNDERCUTTING,2009,0,0,0,0,0,0 +UNDERESTIMATING,2009,0,0,0,0,0,0 +UNDERMINE,2009,0,0,0,0,0,0 +UNDERPAID,2009,0,0,0,0,0,0 +UNDERPERFORM,2011,0,0,0,0,0,0 +UNDERPERFORMS,2014,0,0,0,0,0,0 +UNDERSTATE,2009,0,0,0,0,0,0 +UNDERSTATES,2009,0,0,0,0,0,0 +UNDESIGNATED,0,0,2009,0,0,0,0 +UNDETECTED,2009,0,0,0,0,0,0 +UNDISCLOSED,2009,0,0,0,0,0,0 +UNDUE,2009,0,0,0,0,0,0 +UNECONOMICALLY,2009,0,0,0,0,0,0 +UNENCUMBERED,0,0,0,2009,0,0,0 +UNEQUIVOCALLY,0,0,0,0,2009,0,0 +UNEXPECTED,2009,0,2009,0,0,0,0 +UNFAMILIAR,0,0,2009,0,0,0,0 +UNFAVORABLY,2009,0,0,0,0,0,0 +UNFITNESS,2009,0,0,0,0,0,0 +UNFORSEEN,2011,0,2011,0,0,0,0 +UNFRIENDLY,2009,0,0,0,0,0,0 +UNHEDGED,0,0,2009,0,0,0,0 +UNINTENDED,2009,0,0,0,0,0,0 +UNJUSTIFIABLE,2009,0,0,0,0,0,0 +UNKNOWING,2009,0,0,0,0,0,0 +UNLAWFUL,2009,0,0,2009,0,0,0 +UNLIQUIDATED,2009,0,0,0,0,0,0 +UNMERITORIOUS,2014,0,0,0,0,0,0 +UNOBSERVABLE,0,0,2009,0,0,0,0 +UNPARALLELED,0,2009,0,0,2009,0,0 +UNPREDICTABILITY,2009,0,2009,0,0,0,0 +UNPRODUCTIVE,2009,0,0,0,0,0,0 +UNPROVEN,0,0,2009,0,0,0,0 +UNREALISTIC,2009,0,0,0,0,0,0 +UNRECEPTIVE,2014,0,0,0,0,0,0 +UNREIMBURSED,2009,0,0,0,0,0,0 +UNREPORTED,2009,0,0,0,0,0,0 +UNSALABLE,2009,0,0,0,0,0,0 +UNSAVORY,2009,0,0,0,0,0,0 +UNSELLABLE,2014,0,0,0,0,0,0 +UNSPECIFIC,0,0,2009,0,0,0,0 +UNSTAYED,0,0,0,2009,0,0,0 +UNSUITABILITY,2009,0,0,0,0,0,0 +UNSURE,2009,0,0,0,0,0,0 +UNSUSTAINABLE,2009,0,0,0,0,0,0 +UNTO,0,0,0,2009,0,0,0 +UNTRUTHFULLY,2009,0,0,0,0,0,0 +UNUSUAL,0,0,2009,0,0,0,0 +UNWELCOME,2009,0,0,0,0,0,0 +UPSET,2009,0,0,0,0,0,0 +URGENT,2009,0,0,0,0,0,0 +USURPED,2009,0,0,2009,0,0,0 +VAGARIES,0,0,2009,0,0,0,0 +VAGUENESSES,0,0,2012,0,0,0,0 +VANDALISM,2009,0,0,0,0,0,0 +PLAINTIFF,2009,0,0,2009,0,0,0 +PLEADED,2009,0,0,0,0,0,0 +PLEAS,2009,0,0,2009,0,0,0 +PLEASURE,0,2009,0,0,0,0,0 +PLEDGEE,0,0,0,2009,0,0,0 +PLEDGOR,0,0,0,2009,0,0,0 +COMPELLED,0,0,0,0,0,0,2009 +COMPLAIN,2009,0,0,0,0,0,0 +COMPLAINING,2009,0,0,0,0,0,0 +COMMITMENTS,0,0,0,0,0,0,2009 +LIMITATIONS,2009,0,0,0,0,0,0 +RESTRAINS,0,0,0,0,0,0,2009 +RESTRICTED,0,0,0,0,0,0,2009 +RESTRICTIVE,0,0,0,0,0,0,2009 +RESTRUCTURE,2009,0,0,0,0,0,0 +RESTRUCTURINGS,2009,0,0,0,0,0,0 +RETALIATING,2009,0,0,0,0,0,0 +RETENDERING,0,0,0,2011,0,0,0 +PRECIPITATED,2009,0,0,0,0,0,0 +PRECLUDED,2009,0,0,0,0,0,2009 +PRECONDITIONS,0,0,0,0,0,0,2009 +PREDECEASES,0,0,0,2009,0,0,0 +PREDICTED,0,0,2009,0,0,0,0 +PREDICTIVE,0,0,2009,0,0,0,0 +PREEMINENCE,0,2009,0,0,0,0,0 +PREJUDICED,2009,0,0,2009,0,0,0 +PRELIMINARILY,0,0,2009,0,0,0,0 +PREMIER,0,2009,0,0,0,0,0 +PRESSING,2009,0,0,0,0,0,0 +PRESUME,0,0,2009,0,0,0,0 +PRESUMPTION,0,0,2009,0,0,0,0 +PREVENT,0,0,0,0,0,0,2009 +PREVENTS,2009,0,0,0,0,0,2009 +PROACTIVELY,0,2009,0,0,0,0,0 +PROBABLE,0,0,2009,0,0,0,0 +PROBATES,0,0,0,2009,0,0,0 +PROBATIONARY,0,0,0,2009,0,0,0 +PROBLEM,2009,0,0,0,0,0,0 +PROFICIENCY,0,2009,0,0,0,0,0 +PROFITABLE,0,2009,0,0,0,0,0 +PROGRESSES,0,2009,0,0,0,0,0 +PROHIBITING,0,0,0,0,0,0,2009 +PROHIBITIVELY,0,0,0,0,0,0,2009 +PROLONGATION,2009,0,0,0,0,0,0 +PROLONGS,2009,0,0,0,0,0,0 +PROMULGATING,0,0,0,2009,0,0,0 +PROMULGATORS,0,0,0,2009,0,0,0 +PROSECUTE,2009,0,0,2009,0,0,0 +PROSECUTION,2009,0,0,2009,0,0,0 +PROSECUTORS,0,0,0,2009,0,0,0 +PROSPEROUS,0,2009,0,0,0,0,0 +PROTESTER,2009,0,0,0,0,0,0 +PROTESTORS,2009,0,0,0,0,0,0 +PROVISO,0,0,0,2009,0,0,0 +PROVOKED,2009,0,0,0,0,0,0 +PUNISHED,2009,0,0,0,0,0,0 +PUNISHMENTS,2009,0,0,0,0,0,0 +PURPORTEDLY,2009,0,0,0,0,0,0 +QUESTIONABLE,2009,0,0,0,0,0,0 +QUESTIONS,2009,0,0,0,0,0,0 +QUITTING,2009,0,0,0,0,0,0 +RANDOMIZE,0,0,2009,0,0,0,0 +RANDOMLY,0,0,2009,0,0,0,0 +RATABLY,0,0,0,2009,0,0,0 +RATIONALIZED,2009,0,0,0,0,0,0 +ABANDONING,2009,0,0,0,0,0,0 +ABDICATED,2009,0,0,0,0,0,0 +ABDICATIONS,2009,0,0,0,0,0,0 +ABERRATIONS,2009,0,0,0,0,0,0 +ABIDE,0,0,0,0,0,0,2009 +ABNORMALITIES,2009,0,0,0,0,0,0 +ABOLISHED,2009,0,0,0,0,0,0 +ABROGATE,2009,0,0,2009,0,0,0 +ABROGATION,2009,0,0,2009,0,0,0 +ABRUPTNESS,2009,0,0,0,0,0,0 +ABSOLVE,0,0,0,2009,0,0,0 +ABUNDANCE,0,2009,0,0,0,0,0 +ABUSES,2009,0,0,0,0,0,0 +ABUSIVENESS,2009,0,0,0,0,0,0 +ACCIDENTAL,2009,0,0,0,0,0,0 +ACCOMPLISH,0,2009,0,0,0,0,0 +ACCOMPLISHMENT,0,2009,0,0,0,0,0 +ACCUSE,2009,0,0,0,0,0,0 +ACHIEVE,0,2009,0,0,0,0,0 +ACHIEVES,0,2009,0,0,0,0,0 +ACQUIESCES,2009,0,0,0,0,0,0 +ACQUIT,2009,0,0,2009,0,0,0 +ACQUITTANCE,0,0,0,2009,0,0,0 +ADDENDUMS,0,0,0,2011,0,0,0 +ADJOURNING,0,0,0,2009,0,0,0 +ADJUDGE,0,0,0,2009,0,0,0 +ADJUDICATE,0,0,0,2009,0,0,0 +ADJUDICATION,0,0,0,2009,0,0,0 +ADJUDICATORS,0,0,0,2009,0,0,0 +ADMISSIBLY,0,0,0,2009,0,0,0 +ADULTERATED,2009,0,0,0,0,0,0 +ADVANCEMENT,0,2009,0,0,0,0,0 +ADVANTAGE,0,2009,0,0,0,0,0 +ADVANTAGES,0,2009,0,0,0,0,0 +ADVERSE,2009,0,0,0,0,0,0 +AFFIDAVIT,0,0,0,2009,0,0,0 +AFOREDESCRIBED,0,0,0,2011,0,0,0 +AFTERMATH,2009,0,0,0,0,0,0 +AGGRAVATED,2009,0,0,0,0,0,0 +AGGRAVATIONS,2009,0,0,0,0,0,0 +ALIENATE,2009,0,0,0,0,0,0 +ALIENATION,2009,0,0,0,0,0,0 +ALLEGE,2009,0,0,2009,0,0,0 +ALLEGING,2009,0,0,2009,0,0,0 +ALTERATION,0,0,2009,0,0,0,0 +AMBIGUITY,0,0,2009,0,0,0,0 +AMENDATORY,0,0,0,2009,0,0,0 +AMENDMENTS,0,0,0,2009,0,0,0 +ANNOYANCES,2009,0,0,0,0,0,0 +ANNUL,2009,0,0,0,0,0,0 +ANNULMENTS,2009,0,0,0,0,0,0 +ANOMALOUSLY,2009,0,2009,0,0,0,0 +ANTICIPATE,0,0,2009,0,0,0,0 +ANTICIPATION,0,0,2009,0,0,0,0 +ANTITRUST,2009,0,0,2009,0,0,0 +APPEAL,0,0,0,2009,0,0,0 +APPEALS,0,0,0,2009,0,0,0 +APPEARS,0,0,2009,0,0,2009,0 +APPELLEES,0,0,0,2011,0,0,0 +APPROXIMATELY,0,0,2009,0,0,0,0 +APPROXIMATIONS,0,0,2009,0,0,0,0 +ARBITRABILITY,0,0,0,2011,0,0,0 +ARBITRARY,0,0,2009,0,0,0,0 +ARBITRATING,0,0,0,2009,0,0,0 +ARBITRATIVE,0,0,0,2011,0,0,0 +ARGUED,2009,0,0,0,0,0,0 +ARGUMENTS,2009,0,0,0,0,0,0 +ARREST,2009,0,0,0,0,0,0 +RETRIBUTIONS,2009,0,0,0,0,0,0 +BONA,0,0,0,2009,0,0,0 +BOOST,0,2009,0,0,0,0,0 +BOUND,0,0,0,0,0,0,2011 +BOYCOTTING,2009,0,0,0,0,0,0 +BREACHES,2009,0,0,2009,0,0,0 +BREAKAGES,2009,0,0,0,0,0,0 +BREAKS,2009,0,0,0,0,0,0 +BRIBED,2009,0,0,0,0,0,0 +BRIBING,2009,0,0,0,0,0,0 +BURDEN,2009,0,0,0,0,0,0 +BURDENSOME,2009,0,0,0,0,0,0 +CALAMITY,2009,0,0,0,0,0,0 +CANCELLATION,2009,0,0,0,0,0,0 +CANCELS,2009,0,0,0,0,0,0 +CATASTROPHICALLY,2009,0,0,0,0,0,0 +CAUTIONING,2009,0,0,0,0,0,0 +CAUTIOUSNESS,0,0,2009,0,0,0,0 +CEASING,2009,0,0,0,0,0,0 +CENSURED,2009,0,0,0,0,0,0 +CESSION,0,0,0,2009,0,0,0 +CHALLENGING,2009,0,0,0,0,0,0 +CHATTELS,0,0,0,2009,0,0,0 +CIRCUMVENTING,2009,0,0,0,0,0,0 +SHARPLY,2009,0,0,0,0,0,0 +SHORTFALL,2009,0,0,0,0,0,0 +SHUT,2009,0,0,0,0,0,0 +SHUTTING,2009,0,0,0,0,0,0 +SLANDERS,2009,0,0,0,0,0,0 +REPUDIATE,2009,0,0,0,0,0,0 +REPUDIATION,2009,0,0,0,0,0,0 +REQUIRE,0,0,0,0,0,0,2009 +REQUIRES,0,0,0,0,0,0,2009 +RESCINDED,0,0,0,2009,0,0,0 +RESCISSIONS,0,0,0,2009,0,0,0 +RESIGNED,2009,0,0,0,0,0,0 +RESTATE,2009,0,0,0,0,0,0 +RESTATES,2009,0,0,0,0,0,0 +NONTERMINABLE,0,0,0,2011,0,0,0 +NOTARIZATION,0,0,0,2009,0,0,0 +NOTARIZING,0,0,0,2009,0,0,0 +NUISANCE,2009,0,0,0,0,0,0 +NULLIFIED,2009,0,0,2009,0,0,0 +NULLITIES,0,0,0,2009,0,0,0 +OBJECTION,2009,0,0,0,0,0,0 +OBLIGATE,0,0,0,0,0,0,2009 +OBLIGATION,0,0,0,0,0,0,2009 +OBLIGED,0,0,0,0,0,0,2009 +OBLIGOR,0,0,0,2009,0,0,0 +OBSOLESCENCE,2009,0,0,0,0,0,0 +OBSTRUCT,2009,0,0,0,0,0,0 +OBSTRUCTIONS,2009,0,0,0,0,0,0 +OFFEND,2009,0,0,0,0,0,0 +OFFENDING,2009,0,0,0,0,0,0 +OFFEREES,0,0,0,2011,0,0,0 +OMISSIONS,2009,0,0,0,0,0,0 +OMITTING,2009,0,0,0,0,0,0 +OPPORTUNITIES,0,2009,0,0,0,0,0 +OPPOSES,2009,0,0,0,0,0,0 +OPTIMISTIC,0,2009,0,0,0,0,0 +OUTAGE,2009,0,0,0,0,0,0 +OUTPERFORM,0,2009,0,0,0,0,0 +OVERAGE,2009,0,0,0,0,0,0 +OVERBUILDS,2009,0,0,0,0,0,0 +OVERBURDENING,2009,0,0,0,0,0,0 +OVERCHARGED,2009,0,0,0,0,0,0 +OVERCOMES,2009,0,0,0,0,0,0 +OVERESTIMATED,2009,0,0,0,0,0,0 +OVERESTIMATIONS,2009,0,0,0,0,0,0 +OVERLOADS,2009,0,0,0,0,0,0 +OVERLOOKS,2009,0,0,0,0,0,0 +OVERPRODUCED,2009,0,0,0,0,0,0 +OVERRULE,0,0,0,2009,0,0,0 +OVERRUN,2009,0,0,0,0,0,0 +OVERSHADOWED,2009,0,0,0,0,0,0 +OVERSTATED,2009,0,0,0,0,0,0 +OVERSTATING,2009,0,0,0,0,0,0 +OVERSUPPLYING,2009,0,0,0,0,0,0 +OVERTURNING,2009,0,0,0,0,0,0 +OVERVALUING,2009,0,0,0,0,0,0 +PARI,0,0,0,2009,0,0,0 +NECESSITATES,0,0,0,0,0,0,2009 +NEGATIVES,2009,0,0,0,0,0,0 +NEGLECTING,2009,0,0,0,0,0,0 +NEGLIGENT,2009,0,0,0,0,0,0 +BARRED,2009,0,0,0,0,0,0 +BEAUTIFULLY,0,2009,0,0,0,0,0 +BELIEVING,0,0,2009,0,0,0,0 +IDLE,2009,0,0,0,0,0,0 +IGNORED,2009,0,0,0,0,0,0 +ILLEGAL,2009,0,0,0,0,0,0 +ILLEGIBLE,2009,0,0,0,0,0,0 +ILLIQUIDITY,2009,0,0,0,0,0,0 +IMMATURE,2009,0,0,0,0,0,0 +IMPAIRING,2009,0,0,0,0,0,2011 +IMPASSE,2009,0,0,0,0,0,0 +IMPEDES,2009,0,0,0,0,0,0 +IMPENDING,2009,0,0,0,0,0,0 +IMPERIL,2009,0,0,0,0,0,0 +IMPLICATED,2009,0,0,0,0,0,0 +IMPOSED,0,0,0,0,0,0,2009 +IMPOSITIONS,0,0,0,0,0,0,2009 +IMPOUNDED,2009,0,0,0,0,0,0 +IMPRACTICAL,2009,0,0,0,0,0,0 +IMPRECISION,0,0,2009,0,0,0,0 +IMPRESSES,0,2009,0,0,0,0,0 +IMPRISONMENT,2009,0,0,0,0,0,0 +IMPROPERLY,2009,0,0,0,0,0,0 +IMPROVED,0,2009,0,0,0,0,0 +IMPROVING,0,2009,0,0,0,0,0 +INACCESSIBLE,2009,0,0,0,0,0,0 +INACCURATELY,2009,0,0,0,0,0,0 +INACTIVATED,2009,0,0,0,0,0,0 +INACTIVATIONS,2009,0,0,0,0,0,0 +INADEQUATE,2009,0,0,0,0,0,0 +INADVISABILITY,2009,0,0,0,0,0,0 +INASMUCH,0,0,0,2009,0,0,0 +INCAPACITY,2009,0,0,2009,0,0,0 +INCARCERATING,2009,0,0,2009,0,0,0 +INCIDENCE,2009,0,0,0,0,0,0 +INCOMPATIBILITIES,2009,0,0,0,0,0,0 +INCOMPETENCY,2009,0,0,0,0,0,0 +INCOMPLETE,2009,0,0,0,0,0,0 +INCONSISTENCIES,2009,0,0,0,0,0,0 +INCONTESTABILITY,0,0,0,2009,0,0,0 +INCONVENIENT,2009,0,0,0,0,0,0 +INCREDIBLE,0,2009,0,0,0,0,0 +INDECENT,2009,0,0,0,0,0,0 +INDEFINITELY,0,0,2009,0,0,0,0 +INDEMNIFICATIONS,0,0,0,2009,0,0,0 +INDEMNIFYING,0,0,0,2009,0,0,0 +INDEMNITOR,0,0,0,2009,0,0,0 +INDETERMINATE,0,0,2009,0,0,0,0 +INDICTING,2009,0,0,2009,0,0,0 +INEFFECTIVE,2009,0,0,0,0,0,0 +INEFFICIENCY,2009,0,0,0,0,0,0 +INELIGIBLE,2009,0,0,0,0,0,0 +INEQUITY,2009,0,0,0,0,0,0 +INEXPERIENCE,2009,0,0,0,0,0,0 +INFLUENTIAL,0,2009,0,0,0,0,0 +INFRACTIONS,2009,0,0,2009,0,0,0 +INFRINGEMENTS,2009,0,0,0,0,0,0 +INGENUITY,0,2009,0,0,0,0,0 +INHIBITS,0,0,0,0,0,0,2011 +INJUNCTIVE,0,0,0,2009,0,0,0 +INJURIES,2009,0,0,0,0,0,0 +INNOVATE,0,2009,0,0,0,0,0 +INNOVATION,0,2009,0,0,0,0,0 +INNOVATOR,0,2009,0,0,0,0,0 +INQUIRY,2009,0,0,0,0,0,0 +INSIST,0,0,0,0,0,0,2009 +INSISTS,0,0,0,0,0,0,2009 +INSOLVENT,2009,0,0,0,0,0,0 +INSTABILITY,2009,0,2009,0,0,0,0 +INSUFFICIENTLY,2009,0,0,0,0,0,0 +INTANGIBLES,0,0,2009,0,0,0,0 +INTERFERED,2009,0,0,0,0,0,0 +INTERFERING,2009,0,0,0,0,0,0 +INTERPLEADER,0,0,0,2009,0,0,0 +INTERPOSING,0,0,0,2009,0,0,0 +INTERROGATED,0,0,0,2009,0,0,0 +INTERROGATIONS,0,0,0,2009,0,0,0 +INTERROGATORY,0,0,0,2009,0,0,0 +INTERRUPTION,2009,0,0,0,0,0,0 +INTESTATE,0,0,0,2009,0,0,0 +SPORADIC,0,0,2009,0,0,0,0 +STABILIZATIONS,0,2009,0,0,0,0,0 +STABILIZING,0,2009,0,0,0,0,0 +STAGNATE,2009,0,0,0,0,0,0 +STAGNATION,2009,0,0,0,0,0,0 +STATUTES,0,0,0,2009,0,0,0 +STIPULATED,0,0,0,0,0,0,2009 +STIPULATIONS,0,0,0,0,0,0,2009 +STOPPED,2009,0,0,0,0,0,0 +STRAINED,2009,0,0,0,0,0,0 +STRENGTHEN,0,2009,0,0,0,0,0 +STRENGTHS,0,2009,0,0,0,0,0 +STRESSFUL,2009,0,0,0,0,0,0 +STRICTEST,0,0,0,0,0,0,2009 +STRONGER,0,2009,0,0,0,0,0 +SUBCLAUSES,0,0,0,2009,0,0,0 +SUBJECTION,2009,0,0,0,0,0,0 +SUBLICENSEE,0,0,0,2009,0,0,0 +SUBPOENA,2009,0,0,2009,0,0,0 +SUBROGATION,0,0,0,2009,0,0,0 +SUCCEED,0,2009,0,0,0,0,0 +SUCCESS,0,2009,0,0,0,0,0 +SUDDEN,0,0,2009,0,0,0,0 +SUES,2009,0,0,2009,0,0,0 +SUFFERS,2009,0,0,0,0,0,0 +SUGGESTS,0,0,2009,0,0,2009,0 +SUMMONS,2009,0,0,2009,0,0,0 +SUPERSEDEAS,0,0,0,2011,0,0,0 +SURETIES,0,0,0,2009,0,0,0 +SURPASSES,0,2009,0,0,0,0,0 +SUSPECT,2009,0,0,0,0,0,0 +SUSPENDED,2009,0,0,0,0,0,0 +SUSPENSIONS,2009,0,0,0,0,0,0 +SUSPICIOUSLY,2009,0,0,0,0,0,0 +REVISE,0,0,2009,0,0,0,0 +REVOCATIONS,2009,0,0,2009,0,0,0 +REVOKING,2009,0,0,0,0,0,0 +REVOLUTIONIZING,0,2009,0,0,0,0,0 +REWARDS,0,-2020,0,0,0,0,0 +RIDICULING,2009,0,0,0,0,0,0 +RISKIEST,2009,0,2009,0,0,0,0 +RISKY,2009,0,2009,0,0,0,0 +RUMORS,0,0,2009,0,0,0,0 +SACRIFICES,2009,0,0,0,0,0,0 +SATISFACTORILY,0,2009,0,0,0,0,0 +SATISFY,0,2009,0,0,0,0,0 +SCRUTINIZE,2009,0,0,0,0,0,0 +SCRUTINY,2009,0,0,0,0,0,0 +SEIZED,2009,0,0,0,0,0,0 +SELDOMLY,0,0,2009,0,0,2009,0 +SERIOUS,2009,0,0,0,0,0,0 +SETBACKS,2009,0,0,0,0,0,0 +SEVERABILITY,0,0,0,2009,0,0,0 +SEVERANCES,0,0,0,2009,0,0,0 +SEVERITIES,2009,0,0,0,0,0,0 +ENTHUSIASM,0,2009,0,0,0,0,0 +ENTRENCHED,0,0,0,0,0,0,2009 +ERODING,2009,0,0,0,0,0,0 +ERRED,2009,0,0,0,0,0,0 +ERROR,2009,0,0,0,0,0,0 +ESCALATED,2009,0,0,0,0,0,0 +ESCHEATED,0,0,0,2011,0,0,0 +ESCROWING,0,0,0,2011,0,0,0 +EVADED,2009,0,0,0,0,0,0 +EVASIONS,2009,0,0,0,0,0,0 +EVICTING,2009,0,0,0,0,0,0 +EVIDENTIAL,0,0,0,2011,0,0,0 +EXACERBATES,2009,0,0,0,0,0,0 +EXAGGERATE,2009,0,0,0,0,0,0 +EXAGGERATION,2009,0,0,0,0,0,0 +EXCELLENCE,0,2009,0,0,0,0,0 +EXCEPTIONAL,0,2009,0,0,0,0,0 +EXCISED,0,0,0,2009,0,0,0 +EXCLUSIVE,0,2009,0,0,0,0,0 +EXCLUSIVITY,0,2009,0,0,0,0,0 +EXCULPATING,2009,0,0,2009,0,0,0 +EXECUTOR,0,0,0,2009,0,0,0 +EXECUTRIX,0,0,0,2009,0,0,0 +EXONERATED,2009,0,0,0,0,0,0 +EXONERATIONS,2009,0,0,0,0,0,0 +EXPLOITATIVE,2009,0,0,0,0,0,0 +EXPOSE,2009,0,0,0,0,0,0 +EXPOSURE,0,0,2009,0,0,0,0 +EXPROPRIATES,2009,0,0,0,0,0,0 +EXPULSION,2009,0,0,0,0,0,0 +EXTRACORPOREAL,0,0,0,2011,0,0,0 +SOLVENCY,2009,0,0,0,0,0,0 +SOMETIMES,0,0,2009,0,0,2009,0 +SPAMMERS,2014,0,0,0,0,0,0 +TREMENDOUS,0,2009,0,0,0,0,0 +LOCKOUT,2009,0,0,0,0,0,0 +LOSING,2009,0,0,0,0,0,0 +LOWEST,0,0,0,0,2009,0,0 +MAJEURE,0,0,0,2011,0,0,0 +MALFUNCTIONING,2009,0,0,0,0,0,0 +MALICIOUSLY,2009,0,0,0,0,0,0 +MANDATED,0,0,0,0,0,0,2009 +MANDITORILY,0,0,0,0,0,0,2011 +MANIPULATING,2009,0,0,0,0,0,0 +MARKDOWN,2009,0,0,0,0,0,0 +MEDIATE,0,0,0,2009,0,0,0 +MEDIATION,0,0,0,2009,0,0,0 +MERITORIOUS,0,2009,0,0,0,0,0 +MISAPPLIED,2009,0,0,0,0,0,0 +MISAPPROPRIATE,2009,0,0,0,0,0,0 +MISAPPROPRIATION,2009,0,0,0,0,0,0 +MISCALCULATED,2009,0,0,0,0,0,0 +MISCALCULATIONS,2009,0,0,0,0,0,0 +MISCLASSIFICATIONS,2014,0,0,0,0,0,0 +MISCONDUCT,2009,0,0,0,0,0,0 +MISDIRECTED,2009,0,0,0,0,0,0 +MISHANDLES,2009,0,0,0,0,0,0 +MISINFORMED,2009,0,0,0,0,0,0 +MISINTERPRETATION,2009,0,0,0,0,0,0 +MISINTERPRETS,2009,0,0,0,0,0,0 +MISJUDGING,2009,0,0,0,0,0,0 +MISLABELED,2009,0,0,0,0,0,0 +MISLEAD,2009,0,0,0,0,0,0 +MISLED,2009,0,0,0,0,0,0 +MISMANAGES,2009,0,0,0,0,0,0 +MISMATCHES,2009,0,0,0,0,0,0 +MISPRICING,2014,0,0,0,0,0,0 +MISREPRESENTATIONS,2009,0,0,0,0,0,0 +MISS,2009,0,0,0,0,0,0 +WORSE,2009,0,0,0,0,0,0 +WORSENS,2009,0,0,0,0,0,0 +TOLERATES,2009,0,0,0,0,0,0 +TORTIOUS,0,0,0,2009,0,0,0 +TORTUOUSLY,2009,0,0,0,0,0,0 +FINED,2009,0,0,0,0,0,0 +NONAPPEALABLE,0,0,0,2009,0,0,0 +NONCANCELABLE,0,0,0,0,0,0,2009 +NONCOMPLIANCES,2009,0,0,0,0,0,0 +NONCONFORMITIES,2009,0,0,0,0,0,0 +NONCONTRACTUAL,0,0,0,2011,0,0,0 +NONFIDUCIARY,0,0,0,2011,0,0,0 +NONFUNCTIONAL,2009,0,0,0,0,0,0 +DISCLAIMER,2009,0,0,0,0,0,0 +DISCLOSE,2009,0,0,0,0,0,0 +DISCONTINUANCE,2009,0,0,0,0,0,0 +DISCONTINUE,2009,0,0,0,0,0,0 +DISCOURAGE,2009,0,0,0,0,0,0 +DISCREDIT,2009,0,0,0,0,0,0 +DISCREPANCIES,2009,0,0,0,0,0,0 +SPECULATE,0,0,2009,0,0,0,0 +SPECULATION,0,0,2009,0,0,0,0 +TRAGIC,2009,0,0,0,0,0,0 +TRANSPARENCY,0,2009,0,0,0,0,0 +LEGISLATE,0,0,0,2009,0,0,0 +LEGISLATION,0,0,0,2009,0,0,0 +LEGISLATOR,0,0,0,2009,0,0,0 +LIBEL,0,0,0,2009,0,0,0 +LICENSABLE,0,0,0,2011,0,0,0 +CONCEALING,2009,0,0,0,0,0,0 +CONCEDING,2009,0,0,0,0,0,0 +CONCERNED,2009,0,0,0,0,0,0 +CONCILIATIONS,2009,0,0,0,0,0,0 +CONDEMNATION,2009,0,0,0,0,0,0 +CONDEMNOR,0,0,0,2011,0,0,0 +CONDONE,2009,0,0,0,0,0,0 +CONFESSED,2009,0,0,0,0,0,0 +CONFIDENT,0,2009,0,0,0,0,0 +CONFINEMENTS,2009,0,0,0,0,0,0 +CONFISCATED,2009,0,0,0,0,0,0 +CONFISCATIONS,2009,0,0,0,0,0,0 +CONFLICTING,2009,0,0,0,0,0,0 +CONFRONTATIONAL,2009,0,0,0,0,0,0 +CONFRONTS,2009,0,0,0,0,0,0 +CONFUSING,2009,0,2009,0,0,0,0 +CONSENTED,0,0,0,2009,0,0,0 +CONSPIRACIES,2009,0,0,0,0,0,0 +CONSPIRATORS,2009,0,0,0,0,0,0 +CONSPIRING,2009,0,0,0,0,0,0 +CONSTITUTIONALLY,0,0,0,2009,0,0,0 +CONSTRAINED,0,0,0,0,0,0,2009 +CONSTRAINTS,0,0,0,0,0,0,2009 +CONSTRUED,0,0,0,2012,0,0,0 +CONTEND,2009,0,0,0,0,0,0 +CONTENTION,2009,0,0,0,0,0,0 +CONTESTABILITY,0,0,0,2014,0,0,0 +CONTINGENCIES,0,0,2009,0,0,0,0 +CONTINGENTS,0,0,2009,0,0,0,0 +CONTRACTHOLDERS,0,0,0,2009,0,0,0 +CONTRACTION,2009,0,0,0,0,0,0 +CONTRACTUALLY,0,0,0,2009,0,0,0 +CONTRADICTION,2009,0,0,0,0,0,0 +CONTRARY,2009,0,0,0,0,0,0 +CONTRAVENING,0,0,0,2009,0,0,0 +CONTROVERSIES,2009,0,0,0,0,0,0 +CONTROVERTING,0,0,0,2009,0,0,0 +CONVICT,2009,0,0,2009,0,0,0 +CONVICTIONS,2009,0,0,2009,0,0,0 +CORRECTIONS,2009,0,0,0,0,0,0 +CORRUPTING,2009,0,0,0,0,0,0 +CORRUPTNESS,2009,0,0,0,0,0,0 +COUNSEL,0,0,0,2009,0,0,0 +COUNTERCLAIM,2009,0,0,0,0,0,0 +COUNTERFEIT,2009,0,0,0,0,0,0 +COUNTERFEITING,2009,0,0,0,0,0,0 +COUNTERSIGNOR,0,0,0,2011,0,0,0 +COURT,0,0,0,2009,0,0,0 +COVENANT,0,0,0,0,0,0,2011 +CREATIVE,0,2009,0,0,0,0,0 +CRIME,2009,0,0,2009,0,0,0 +CRIMINALIZE,0,0,0,2014,0,0,0 +CRISES,2009,0,0,0,0,0,0 +CRITICISM,2009,0,0,0,0,0,0 +CRITICIZES,2009,0,0,0,0,0,0 +CROSSROAD,0,0,2009,0,0,0,0 +CULPABILITY,2009,0,0,0,0,0,0 +CURTAIL,2009,0,0,0,0,0,0 +CURTAILMENTS,2009,0,0,0,0,0,0 +CUTBACKS,2009,0,0,0,0,0,0 +CYBERCRIME,2014,0,0,0,0,0,0 +TAINTING,2009,0,0,0,0,0,0 +TENDING,0,0,2009,0,0,0,0 +TERMINABLE,0,0,0,2009,0,0,0 +TERMINATING,2009,0,0,0,0,0,0 +TESTAMENTARY,0,0,0,2009,0,0,0 +THENCE,0,0,0,2009,0,0,0 +THEREAT,0,0,0,2009,0,0,0 +THEREOF,0,0,0,2009,0,0,0 +THERETOFOR,0,0,0,2011,0,0,0 +THEREUPON,0,0,0,2009,0,0,0 +THREATENED,2009,0,0,0,0,0,0 +FAILED,2009,0,0,0,0,0,0 +FAILURE,2009,0,0,0,0,0,0 +FALSELY,2009,0,0,0,0,0,0 +FALSIFIES,2009,0,0,0,0,0,0 +FANTASTIC,0,2009,0,0,0,0,0 +FAULT,2009,0,0,0,0,0,0 +FAVORABLE,0,2009,0,0,0,0,0 +FAVORITE,0,2009,0,0,0,0,0 +FELONIES,2009,0,0,2009,0,0,0 +DAMAGED,2009,0,0,0,0,0,0 +DAMPENED,2009,0,0,0,0,0,0 +DANGERS,2009,0,0,0,0,0,0 +DEADLOCKS,2009,0,0,0,0,0,0 +DEBARMENTS,2009,0,0,0,0,0,0 +DECEDENTS,0,0,0,2009,0,0,0 +DECEIVE,2009,0,0,0,0,0,0 +DECEPTION,2009,0,0,0,0,0,0 +DECLARANT,0,0,0,2011,0,0,0 +DECLINING,2009,0,0,0,0,0,0 +DECREES,0,0,0,2009,0,0,0 +DEFALCATION,0,0,0,2009,0,0,0 +DEFAMATORY,2009,0,0,0,0,0,0 +DEFAMING,2009,0,0,0,0,0,0 +DEFAULTS,2009,0,0,0,0,0,0 +DEFEASED,0,0,0,2009,0,0,0 +DEFEAT,2009,0,0,0,0,0,0 +DEFECT,2009,0,0,0,0,0,0 +DEFEND,2009,0,0,0,0,0,0 +DEFENDED,2009,0,0,0,0,0,0 +DEFER,2009,0,0,0,0,0,0 +DEFICIENT,2009,0,0,0,0,0,0 +DEFINITIVELY,0,0,0,0,2009,0,0 +DEFRAUDS,2009,0,0,0,0,0,0 +DEGRADE,2009,0,0,0,0,0,0 +DELAY,2009,0,0,0,0,0,0 +DELEGABLE,0,0,0,2009,0,0,0 +DELETERIOUS,2009,0,0,0,0,0,0 +DELIGHT,0,2009,0,0,0,0,0 +DELIGHTING,0,2009,0,0,0,0,0 +DELINQUENT,2009,0,0,0,0,0,0 +DELISTED,2009,0,0,0,0,0,0 +DEMISED,2009,0,0,0,0,0,0 +DEMOLISHED,2009,0,0,0,0,0,0 +DEMOLITIONS,2009,0,0,0,0,0,0 +DEMOTING,2009,0,0,0,0,0,0 +DEMURRER,0,0,0,2009,0,0,0 +DENIAL,2009,0,0,0,0,0,0 +DENIGRATE,2009,0,0,0,0,0,0 +DENIGRATION,2009,0,0,0,0,0,0 +DEPENDABILITY,0,2009,0,0,0,0,0 +DEPENDANT,0,0,0,0,0,0,2011 +DEPENDENCY,0,0,2009,0,0,0,0 +DEPLETE,2009,0,0,0,0,0,0 +DEPLETION,2009,0,0,0,0,0,0 +DEPOSES,0,0,0,2009,0,0,0 +DEPOSITIONS,0,0,0,2009,0,0,0 +INTRUSION,2009,0,0,0,0,0,0 +INVALIDATES,2009,0,0,0,0,0,0 +INVENT,0,2009,0,0,0,0,0 +INVENTIONS,0,2009,0,0,0,0,0 +INVENTORS,0,2009,0,0,0,0,0 +INVESTIGATING,2009,0,0,0,0,0,0 +INVOLUNTARY,2009,0,0,0,0,0,0 +IRRECOVERABLY,2009,0,0,0,0,0,0 +IRREGULARLY,2009,0,0,0,0,0,0 +IRREVOCABILITY,0,0,0,2011,0,0,0 +JEOPARDIZED,2009,0,0,0,0,0,0 +JUDICIARIES,0,0,0,2009,0,0,0 +JURISDICTION,0,0,0,2009,0,0,0 +JURISPRUDENCE,0,0,0,2009,0,0,0 +JURORS,0,0,0,2009,0,0,0 +JUSTICES,0,0,0,2009,0,0,0 +KNOWINGLY,2009,0,0,0,0,0,0 +DILIGENT,0,2009,0,0,0,0,0 +DIMINISHES,2009,0,0,0,0,0,0 +DIRECTIVES,0,0,0,0,0,0,2009 +DISADVANTAGES,2009,0,0,0,0,0,0 +DISAFFIRMED,0,0,0,2011,0,0,0 +DISAGREED,2009,0,0,0,0,0,0 +DISAGREES,2009,0,0,0,0,0,0 +DISALLOWED,2009,0,0,0,0,0,0 +DISAPPEARANCE,2009,0,0,0,0,0,0 +DISAPPEARS,2009,0,0,0,0,0,0 +DISAPPOINTINGLY,2009,0,0,0,0,0,0 +DISAPPROVAL,2009,0,0,0,0,0,0 +DISAPPROVES,2009,0,0,0,0,0,0 +DISASSOCIATION,2009,0,0,0,0,0,0 +DISASTROUS,2009,0,0,0,0,0,0 +DISAVOWED,2009,0,0,0,0,0,0 +GREAT,0,-2020,0,0,0,0,0 +GREATNESS,0,2009,0,0,0,0,0 +GROUNDLESS,2009,0,0,0,0,0,0 +HAMPER,2009,0,0,0,0,0,0 +HAPPIEST,0,2009,0,0,0,0,0 +HARASS,2009,0,0,0,0,0,0 +HARDSHIP,2009,0,0,0,0,0,0 +HARMFUL,2009,0,0,0,0,0,0 +HARSH,2009,0,0,0,0,0,0 +HARSHNESS,2009,0,0,0,0,0,0 +NONJUDICIAL,0,0,0,2009,0,0,0 +NONPAYMENTS,2009,0,0,0,0,0,0 \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/go-challenge/main.go b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/go-challenge/main.go new file mode 100644 index 0000000000000..29e6db175a93a --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/go-challenge/main.go @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// beam-playground: +// name: FinalChallenge2 +// description: Final challenge 2. +// multifile: true +// files: +// - name: analysis.csv +// context_line: 54 +// categories: +// - Quickstart +// complexity: ADVANCED +// tags: +// - hellobeam + +package main + +import ( + "context" + "log" + "strings" + + "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/io/textio" + "github.com/apache/beam/sdks/v2/go/pkg/beam/x/beamx" + "github.com/apache/beam/sdks/v2/go/pkg/beam/x/debug" +) + +func main() { + + beam.Init() + + p := beam.NewPipeline() + s := p.Root() + + shakespeare := textio.Read(s, "gs://apache-beam-samples/shakespeare/kinglear.txt") + + debug.Print(s, getWords(s, shakespeare)) + + err := beamx.Run(context.Background(), p) + if err != nil { + log.Fatalf("Failed to execute job: %v", err) + } +} + +func getWords(s beam.Scope, input beam.PCollection) beam.PCollection { + return beam.ParDo(s, func(line string, emit func(string)) { + c := strings.Split(strings.ToLower(line), " ") + for _, word := range c { + emit(word) + } + }, input) +} diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/go-solution/analysis.csv b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/go-solution/analysis.csv new file mode 100644 index 0000000000000..5c4a1246021eb --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/go-solution/analysis.csv @@ -0,0 +1,3877 @@ +Word,Negative,Positive,Uncertainty,Litigious,Strong_Modal,Weak_Modal,Constraining +NONSEVERABLE,0,0,0,2011,0,0,0 +DISFAVOR,2009,0,0,0,0,0,0 +DISGORGE,2009,0,0,0,0,0,0 +COMPLICATES,2009,0,0,0,0,0,0 +COMPLIMENT,0,2009,0,0,0,0,0 +COMPLIMENTS,0,2009,0,0,0,0,0 +MISSTATE,2009,0,0,0,0,0,0 +MISSTATES,2009,0,0,0,0,0,0 +MISTAKE,2009,0,0,0,0,0,0 +MISTAKING,2009,0,0,0,0,0,0 +MISUNDERSTANDING,2009,0,0,0,0,0,0 +MISUSED,2009,0,0,0,0,0,0 +MONOPOLISTS,2009,0,0,0,0,0,0 +MONOPOLIZES,2009,0,0,0,0,0,0 +MORATORIUM,2009,0,0,0,0,0,0 +MOTHBALLING,2009,0,0,0,0,0,0 +SLOW,2009,0,0,0,0,0,0 +SLOWER,2009,0,0,0,0,0,0 +SLOWNESS,2009,0,0,0,0,0,0 +SMOOTH,0,2009,0,0,0,0,0 +DEPRECATION,2009,0,0,0,0,0,0 +DEPRESSING,2009,0,0,0,0,0,0 +DEPRIVES,2009,0,0,0,0,0,0 +DEROGATE,0,0,0,2009,0,0,0 +DEROGATION,0,0,0,2009,0,0,0 +DESIRABLE,0,2009,0,0,0,0,0 +DESTABILIZATION,2009,0,0,0,0,0,0 +DESTINED,0,2009,0,0,0,0,0 +DESTROYS,2009,0,0,0,0,0,0 +DETAINED,2009,0,0,0,0,0,0 +DETER,2009,0,0,0,0,0,0 +DETERIORATING,2009,0,0,0,0,0,0 +DETERRENCE,2009,0,0,0,0,0,0 +DETERRING,2009,0,0,0,0,0,0 +DETRACTING,2009,0,0,0,0,0,0 +DETRIMENTS,2009,0,0,0,0,0,0 +DEVALUING,2009,0,0,0,0,0,0 +DEVASTATION,2009,0,0,0,0,0,0 +DEVIATING,2009,0,2009,0,0,0,0 +DEVOLVE,2009,0,0,0,0,0,0 +DICTATE,0,0,0,0,0,0,2009 +DIFFER,0,0,2009,0,0,0,0 +DIFFICULT,2009,0,0,0,0,0,0 +ASSAULT,2009,0,0,0,0,0,0 +ASSERTABLE,0,0,0,2011,0,0,0 +ASSUMABLE,0,0,0,2009,0,0,0 +ASSUMING,0,0,2009,0,0,0,0 +ASSURED,0,2009,0,0,0,0,0 +ATTAINED,0,2009,0,0,0,0,0 +ATTAINS,0,2009,0,0,0,0,0 +ATTESTED,0,0,0,2009,0,0,0 +ATTORNEYS,0,0,0,2009,0,0,0 +ATTRACTIVENESS,0,2009,0,0,0,0,0 +BAD,2009,0,0,0,0,0,0 +BAILEES,0,0,0,2011,0,0,0 +BAILOUT,2009,0,0,0,0,0,0 +BANKRUPTCIES,2009,0,0,0,0,0,0 +BANKRUPTS,2009,0,0,0,0,0,0 +CLAIM,0,0,0,2009,0,0,0 +CLAIMHOLDER,0,0,0,2014,0,0,0 +CLARIFICATIONS,0,0,2009,0,0,0,0 +CLOSED,-2020,0,0,0,0,0,0 +CLOSINGS,2009,0,0,0,0,0,0 +CODEFENDANTS,0,0,0,2011,0,0,0 +CODIFICATIONS,0,0,0,2009,0,0,0 +CODIFYING,0,0,0,2009,0,0,0 +COERCING,2009,0,0,0,0,0,0 +COLLABORATES,0,2009,0,0,0,0,0 +COLLABORATIVE,0,2009,0,0,0,0,0 +COLLAPSED,2009,0,0,0,0,0,0 +COLLISIONS,2009,0,0,0,0,0,0 +COLLUDING,2009,0,0,0,0,0,0 +POSES,2009,0,0,0,0,0,0 +POSSESSORY,0,0,0,2009,0,0,0 +POSSIBLY,0,0,2009,0,0,2009,0 +POSTJUDGMENT,0,0,0,2011,0,0,0 +POSTPONEMENTS,2009,0,0,0,0,0,0 +WRITEDOWNS,2009,0,0,0,0,0,0 +WRONG,2009,0,0,0,0,0,0 +WRONGFULLY,2009,0,0,0,0,0,0 +HONORABLE,0,-2020,0,0,0,0,0 +HOSTILE,2009,0,0,0,0,0,0 +REASSESS,0,0,2009,0,0,0,0 +REASSESSMENT,2009,0,2009,0,0,0,0 +REASSIGNING,2009,0,0,0,0,0,0 +REBOUND,0,2009,0,0,0,0,0 +REBUTS,0,0,0,2009,0,0,0 +REBUTTALS,0,0,0,2009,0,0,0 +RECALCULATED,0,0,2009,0,0,0,0 +RECALCULATIONS,0,0,2009,0,0,0,0 +RECALLS,2009,0,0,0,0,0,0 +RECESSIONS,2009,0,0,0,0,0,0 +RECONSIDER,0,0,2009,0,0,0,0 +RECORDATION,0,0,0,2009,0,0,0 +RECOURSE,0,0,0,2009,0,0,0 +RECUSAL,0,0,0,2011,0,0,0 +RECUSING,0,0,0,2011,0,0,0 +REDACTION,2009,0,0,2009,0,0,0 +REDEFAULTS,2014,0,0,0,0,0,0 +REDRESSING,2009,0,0,0,0,0,0 +REFERENDA,0,0,0,2009,0,0,0 +REFILED,0,0,0,2009,0,0,0 +REFRAINING,0,0,0,0,0,0,2009 +REFUSE,2009,0,0,0,0,0,0 +REGAIN,0,2009,0,0,0,0,0 +REGULATED,0,0,0,2009,0,0,0 +REGULATIONS,0,0,0,2009,0,0,0 +REGULATORY,0,0,0,2009,0,0,0 +REHEARINGS,0,0,0,2009,0,0,0 +VARIABILITY,0,0,2009,0,0,0,0 +VARIANCE,0,0,2009,0,0,0,0 +VARIATION,0,0,2009,0,0,0,0 +VARY,0,0,2009,0,0,0,0 +VERDICT,2009,0,0,2009,0,0,0 +VETOED,2009,0,0,0,0,0,0 +VICTIMS,2009,0,0,0,0,0,0 +VIOLATING,2009,0,0,0,0,0,0 +VIOLATOR,2009,0,0,0,0,0,0 +VIOLENTLY,2009,0,0,0,0,0,0 +VITIATING,2009,0,0,0,0,0,0 +VOIDING,2009,0,0,2009,0,0,0 +VULNERABILITIES,2009,0,0,0,0,0,0 +WARN,2009,0,0,0,0,0,0 +WARNS,2009,0,0,0,0,0,0 +WASTEFUL,2009,0,0,0,0,0,0 +WEAKENED,2009,0,0,0,0,0,0 +WEAKEST,2009,0,0,0,0,0,0 +DISHONESTLY,2009,0,0,0,0,0,0 +DISHONORABLY,2009,0,0,0,0,0,0 +DISINTERESTEDNESS,2009,0,0,0,0,0,0 +DISMAL,2009,0,0,0,0,0,0 +DISMISSALS,2009,0,0,0,0,0,0 +DISORDERLY,2009,0,0,0,0,0,0 +DISPARAGEMENTS,2009,0,0,0,0,0,0 +DISPARITIES,2009,0,0,0,0,0,0 +DISPLACEMENT,2009,0,0,0,0,0,0 +DISPOSE,2009,0,0,0,0,0,0 +DISPOSSESSES,2009,0,0,0,0,0,0 +DISPROPORTION,2009,0,0,0,0,0,0 +DISPUTE,2009,0,0,0,0,0,0 +DISQUALIFICATION,2009,0,0,0,0,0,0 +DISQUALIFY,2009,0,0,0,0,0,0 +DISREGARDING,2009,0,0,0,0,0,0 +DISRUPT,2009,0,0,0,0,0,0 +DISRUPTIONS,2009,0,0,0,0,0,0 +DISSATISFIED,2009,0,0,0,0,0,0 +DISSENTERS,2009,0,0,0,0,0,0 +DISSIDENTS,2009,0,0,0,0,0,0 +DISTINCTIONS,0,2009,0,0,0,0,0 +DISTORT,2009,0,0,0,0,0,0 +DISTORTIONS,2009,0,0,0,0,0,0 +DISTRACTING,2009,0,0,0,0,0,0 +DISTRAINT,0,0,0,2009,0,0,0 +DISTRIBUTEES,0,0,0,2009,0,0,0 +DISTURBED,2009,0,0,0,0,0,0 +DIVERT,2009,0,0,0,0,0,0 +DIVEST,2009,0,0,0,0,0,0 +DIVESTITURES,2009,0,0,0,0,0,0 +DIVORCE,2009,0,0,0,0,0,0 +DIVULGES,2009,0,0,0,0,0,0 +DOCKETING,0,0,0,2009,0,0,0 +DOUBTED,2009,0,2009,0,0,0,0 +DOWNGRADED,2009,0,0,0,0,0,0 +DOWNSIZED,2009,0,0,0,0,0,0 +DOWNTIME,2009,0,0,0,0,0,0 +DOWNWARD,2009,0,0,0,0,0,0 +DRASTICALLY,2009,0,0,0,0,0,0 +DROPPED,2009,0,0,0,0,0,0 +DURESS,2009,0,0,0,0,0,0 +EARMARK,0,0,0,0,0,0,2009 +EASIER,0,2009,0,0,0,0,0 +EFFECTIVE,0,-2020,0,0,0,0,0 +EFFICIENTLY,0,2009,0,0,0,0,0 +EMBARGO,2009,0,0,0,0,0,0 +EMBARRASS,2009,0,0,0,0,0,0 +EMBARRASSMENT,2009,0,0,0,0,0,0 +EMBEZZLEMENT,2009,0,0,0,0,0,0 +EMBEZZLING,2009,0,0,0,0,0,0 +EMPOWERS,0,2009,0,0,0,0,0 +ENABLING,0,2009,0,0,0,0,0 +ENCOURAGING,0,2009,0,0,0,0,0 +ENCROACHING,2009,0,0,0,0,0,0 +ENCUMBERED,2009,0,0,2009,0,0,2009 +ENCUMBRANCER,0,0,0,2009,0,0,0 +ENDANGERED,2009,0,0,0,0,0,0 +ENDORSEE,0,0,0,2011,0,0,0 +ENHANCE,0,2009,0,0,0,0,0 +ENHANCES,0,2009,0,0,0,0,0 +ENJOINING,2009,0,0,0,0,0,0 +ENJOYABLY,0,2009,0,0,0,0,0 +ENJOYS,0,2009,0,0,0,0,0 +ENTAILS,0,0,0,0,0,0,2009 +PATENTEE,0,0,0,2014,0,0,0 +PENALIZES,2009,0,0,0,0,0,0 +PENDING,0,0,2009,0,0,0,0 +PERFECTS,0,2009,0,0,0,0,0 +PERJURY,2009,0,0,2009,0,0,0 +PERMITTED,0,0,0,0,0,0,2009 +PERPETRATE,2009,0,0,2009,0,0,0 +PERPETRATION,2009,0,0,2009,0,0,0 +PERSISTENT,2009,0,0,0,0,0,0 +PERSONAM,0,0,0,2011,0,0,0 +PETITION,0,0,0,2009,0,0,0 +PETITIONING,0,0,0,2009,0,0,0 +PICKETED,2009,0,0,0,0,0,0 +HENCEFORTH,0,0,0,2009,0,0,0 +HEREDITAMENTS,0,0,0,2009,0,0,0 +HEREIN,0,0,0,2009,0,0,0 +HEREINBELOW,0,0,0,2009,0,0,0 +HERETOFORE,0,0,0,2009,0,0,0 +HEREWITH,0,0,0,2009,0,0,0 +HINDER,2009,0,0,0,0,0,0 +HINDRANCE,2009,0,0,0,0,0,0 +WHATSOEVER,0,0,0,2009,0,0,0 +WHEREAT,0,0,0,2009,0,0,0 +WHEREOF,0,0,0,2009,0,0,0 +WHEREUPON,0,0,0,2009,0,0,0 +WHOMSOEVER,0,0,0,2009,0,0,0 +WILLFUL,0,0,0,2009,0,0,0 +WINNER,0,2009,0,0,0,0,0 +WITNESSES,0,0,0,2009,0,0,0 +FLUCTUATES,0,0,2009,0,0,0,0 +FORBADE,0,0,0,2009,0,0,2011 +FORBEARING,0,0,0,2009,0,0,0 +FORBIDDING,2009,0,0,0,0,0,2009 +FORCING,2009,0,0,0,0,0,0 +FORECLOSE,2009,0,0,0,0,0,0 +FORECLOSURE,2009,0,0,0,0,0,0 +FOREGONE,2009,0,0,0,0,0,0 +FORESTALLS,2009,0,0,0,0,0,0 +FORFEITED,2009,0,0,0,0,0,0 +FORFEITURES,2009,0,0,0,0,0,0 +FORWHICH,0,0,0,2012,0,0,0 +FRAUDULENT,2009,0,0,0,0,0,0 +FRIVOLOUSLY,2009,0,0,0,0,0,0 +FRUSTRATING,2009,0,0,0,0,0,0 +FUGITIVE,-2020,0,0,2009,0,0,0 +GAINED,0,2009,0,0,0,0,0 +GRANTOR,0,0,0,2009,0,0,0 +DISGORGING,2009,0,0,0,0,0,0 +DISHONEST,2009,0,0,0,0,0,0 +LITIGANT,2009,0,0,2009,0,0,0 +LITIGATES,2009,0,0,2009,0,0,0 +LITIGATOR,0,0,0,2009,0,0,0 +LACKLUSTER,2009,0,0,0,0,0,0 +LAGGING,2009,0,0,0,0,0,0 +LAPSES,2009,0,0,0,0,0,0 +LAW,0,0,0,2009,0,0,0 +LAWMAKERS,0,0,0,2009,0,0,0 +LAWSUITS,0,0,0,2009,0,0,0 +LAYOFFS,2009,0,0,0,0,0,0 +LEGALESE,0,0,0,2009,0,0,0 +LEGALIZE,0,0,0,2009,0,0,0 +LEGALLY,0,0,0,2009,0,0,0 +NONPRODUCING,2009,0,0,0,0,0,0 +LIQUIDATE,2009,0,0,0,0,0,0 +LIQUIDATION,2009,0,0,0,0,0,0 +BENEFICIALLY,0,2009,0,0,0,0,0 +BENEFITED,0,2009,0,0,0,0,0 +BEST,0,2012,0,0,2009,0,0 +POPULAR,0,2009,0,0,0,0,0 +REINTERPRETED,0,0,2009,0,0,0,0 +REJECTED,2009,0,0,0,0,0,0 +REJECTS,2009,0,0,0,0,0,0 +RELINQUISHES,2009,0,0,0,0,0,0 +RELUCTANCE,2009,0,0,0,0,0,0 +REMANDING,0,0,0,2009,0,0,0 +REMEDIATING,0,0,0,2009,0,0,0 +REMISED,0,0,0,2011,0,0,0 +RENEGOTIATING,2009,0,0,0,0,0,0 +RENOUNCED,2009,0,0,0,0,0,0 +RENOUNCING,2009,0,0,0,0,0,0 +REPLEVIN,0,0,0,2009,0,0,0 +REPOSSESSION,2009,0,0,0,0,0,0 +TURBULENCE,2009,0,2009,0,0,0,0 +UNACCEPTABLY,2009,0,0,0,0,0,0 +UNANTICIPATED,2009,0,0,0,0,0,0 +UNATTRACTIVE,2009,0,0,0,0,0,0 +UNAVOIDABLE,2009,0,0,0,0,0,0 +UNCERTAINLY,0,0,2009,0,0,2009,0 +UNCOLLECTABLE,2009,0,0,0,0,0,0 +UNCOLLECTIBLES,2009,0,0,0,0,0,0 +UNCONFIRMED,0,0,2009,0,0,0,0 +UNCONSTITUTIONALITY,0,0,0,2009,0,0,0 +UNCONTROLLABLY,2009,0,0,0,0,0,0 +UNCOVERED,2009,0,0,0,0,0,0 +UNDEFEASED,0,0,0,2012,0,0,0 +UNDERCAPITALIZED,2009,0,0,0,0,0,0 +UNDERESTIMATE,2009,0,0,0,0,0,0 +UNDERESTIMATION,2009,0,0,0,0,0,0 +UNDERMINED,2009,0,0,0,0,0,0 +UNDERPAYMENT,2009,0,0,0,0,0,0 +UNDERPERFORMANCE,2009,0,0,0,0,0,0 +UNDERPRODUCED,2009,0,0,0,0,0,0 +UNDERSTATED,2009,0,0,0,0,0,0 +UNDERSTATING,2009,0,0,0,0,0,0 +UNDESIRABLE,2009,0,0,0,0,0,0 +UNDETERMINABLE,0,0,2009,0,0,0,0 +UNDISPUTED,0,0,0,0,2009,0,0 +UNDULY,2009,0,0,0,0,0,0 +UNEMPLOYED,2009,0,0,0,0,0,0 +UNENFORCEABILITY,0,0,0,2009,0,0,0 +UNETHICAL,2009,0,0,0,0,0,0 +UNEXPECTEDLY,2009,0,2009,0,0,0,0 +UNFAMILIARITY,0,0,2009,0,0,0,0 +UNFAVOURABLE,2011,0,0,0,0,0,0 +UNFORECASTED,0,0,2011,0,0,0,0 +UNFORTUNATE,2009,0,0,0,0,0,0 +UNFULFILLED,2009,0,0,0,0,0,0 +UNIDENTIFIABLE,0,0,2009,0,0,0,0 +UNINTENTIONAL,2009,0,0,0,0,0,0 +UNJUSTIFIABLY,2009,0,0,0,0,0,0 +UNKNOWINGLY,2009,0,0,0,0,0,0 +UNLAWFULLY,2009,0,0,2009,0,0,0 +UNMARKETABLE,2009,0,0,0,0,0,0 +UNNECESSARILY,2009,0,0,0,0,0,0 +UNOBTAINABLE,2009,0,0,0,0,0,0 +UNPERFORMED,2009,0,0,0,0,0,0 +UNPREDICTABLE,2009,0,2009,0,0,0,0 +UNPROFITABILITY,2011,0,0,0,0,0,0 +UNQUALIFIED,2009,0,0,0,0,0,0 +UNREASONABLE,2009,0,0,0,0,0,0 +UNRECONCILED,0,0,2011,0,0,0,0 +UNRELIABLE,2009,0,0,0,0,0,0 +UNRESOLVED,2009,0,0,0,0,0,0 +UNSALEABLE,2009,0,0,0,0,0,0 +UNSCHEDULED,2009,0,0,0,0,0,0 +UNSETTLED,0,0,2009,0,0,0,0 +UNSPECIFIED,0,0,2009,0,0,0,0 +UNSUBSTANTIATED,2009,0,0,0,0,0,0 +UNSUITABLE,2009,0,0,0,0,0,0 +UNSURPASSED,0,2009,0,0,2009,0,0 +UNTENABLE,2009,0,0,0,0,0,0 +UNTRUSTED,2014,0,0,0,0,0,0 +UNTRUTHFULNESS,2009,0,0,0,0,0,0 +UNUSUALLY,0,0,2009,0,0,0,0 +UNWILLING,2009,0,0,0,0,0,0 +UPTURN,0,2009,0,0,0,0,0 +USURIOUS,2009,0,0,2009,0,0,0 +USURPING,2009,0,0,2009,0,0,0 +VAGUE,0,0,2012,0,0,0,0 +VAGUER,0,0,2012,0,0,0,0 +PLAINTIFFS,2009,0,0,2009,0,0,0 +PLEADING,2009,0,0,2009,0,0,0 +PLEASANT,0,2009,0,0,0,0,0 +PLED,2009,0,0,0,0,0,0 +PLEDGEES,0,0,0,2011,0,0,0 +PLEDGORS,0,0,0,2012,0,0,0 +COMPELLING,0,0,0,0,0,0,2011 +COMPLAINANT,0,0,0,2009,0,0,0 +COMPLAINS,2009,0,0,0,0,0,0 +COMMITS,0,0,0,0,0,0,2009 +LIKELIHOOD,0,0,2009,0,0,0,0 +LIMITING,0,0,0,0,0,0,2009 +RESTRAIN,0,0,0,0,0,0,2009 +RESTRAINT,0,0,0,0,0,0,2009 +RESTRICTING,0,0,0,0,0,0,2009 +RESTRICTIVELY,0,0,0,0,0,0,2009 +RESTRUCTURED,2009,0,0,0,0,0,0 +RETALIATE,2009,0,0,0,0,0,0 +RETALIATION,2009,0,0,0,0,0,0 +PRECAUTION,0,0,2009,0,0,0,0 +PRECIPITOUS,2009,0,0,0,0,0,0 +PRECLUDES,2009,0,0,0,0,0,2009 +PREDATORY,2009,0,0,0,0,0,0 +PREDECEASING,0,0,0,2009,0,0,0 +PREDICTING,0,0,2009,0,0,0,0 +PREDICTOR,0,0,2009,0,0,0,0 +PREEMINENT,0,2009,0,0,0,0,0 +PREJUDICES,2009,0,0,2009,0,0,0 +PRELIMINARY,0,0,2009,0,0,0,0 +PREMIERE,0,2009,0,0,0,0,0 +PRESTIGE,0,2009,0,0,0,0,0 +PRESUMED,0,0,2009,0,0,0,0 +PRESUMPTIONS,0,0,2009,0,0,0,0 +PREVENTED,0,0,0,0,0,0,2009 +PRIMA,0,0,0,2009,0,0,0 +PROBABILISTIC,0,0,2009,0,0,0,0 +PROBABLY,0,0,2009,0,0,0,0 +PROBATING,0,0,0,2009,0,0,0 +PROBATIONER,0,0,0,2009,0,0,0 +PROBLEMATIC,2009,0,0,0,0,0,0 +PROFICIENT,0,2009,0,0,0,0,0 +PROFITABLY,0,2009,0,0,0,0,0 +PROGRESSING,0,2009,0,0,0,0,0 +PROHIBITION,0,0,0,0,0,0,2009 +PROHIBITORY,0,0,0,0,0,0,2009 +PROLONGATIONS,2009,0,0,0,0,0,0 +PROMULGATE,0,0,0,2009,0,0,0 +PROMULGATION,0,0,0,2009,0,0,0 +PRONE,2009,0,0,0,0,0,0 +PROSECUTED,2009,0,0,2009,0,0,0 +PROSECUTIONS,2009,0,0,2009,0,0,0 +PROSPERED,0,2009,0,0,0,0,0 +PROSPERS,0,2009,0,0,0,0,0 +PROTESTERS,2009,0,0,0,0,0,0 +PROTESTS,2009,0,0,0,0,0,0 +PROVISOES,0,0,0,2009,0,0,0 +PROVOKES,2009,0,0,0,0,0,0 +PUNISHES,2009,0,0,0,0,0,0 +PUNITIVE,2009,0,0,0,0,0,0 +PURPORTING,2009,0,0,0,0,0,0 +QUESTIONABLY,2009,0,0,0,0,0,0 +QUIT,2009,0,0,0,0,0,0 +RACKETEER,2009,0,0,0,0,0,0 +RANDOMIZED,0,0,2009,0,0,0,0 +RANDOMNESS,0,0,2009,0,0,0,0 +RATIONALIZATION,2009,0,0,0,0,0,0 +RATIONALIZES,2009,0,0,0,0,0,0 +ABANDONMENT,2009,0,0,0,0,0,0 +ABDICATES,2009,0,0,0,0,0,0 +ABERRANT,2009,0,0,0,0,0,0 +ABETTING,2009,0,0,0,0,0,0 +ABIDING,0,0,0,0,0,0,2009 +ABNORMALITY,2009,0,0,0,0,0,0 +ABOLISHES,2009,0,0,0,0,0,0 +ABROGATED,2009,0,0,2009,0,0,0 +ABROGATIONS,2009,0,0,2009,0,0,0 +ABSENCE,2009,0,0,0,0,0,0 +ABSOLVED,0,0,0,2009,0,0,0 +ABUNDANT,0,2009,0,0,0,0,0 +ABUSING,2009,0,0,0,0,0,0 +ACCESSION,0,0,0,2009,0,0,0 +ACCIDENTALLY,2009,0,0,0,0,0,0 +ACCOMPLISHED,0,2009,0,0,0,0,0 +ACCOMPLISHMENTS,0,2009,0,0,0,0,0 +ACCUSED,2009,0,0,0,0,0,0 +ACHIEVED,0,2009,0,0,0,0,0 +ACHIEVING,0,2009,0,0,0,0,0 +ACQUIESCING,2009,0,0,0,0,0,0 +ACQUITS,2009,0,0,2009,0,0,0 +ACQUITTANCES,0,0,0,2011,0,0,0 +ADEQUATELY,0,2009,0,0,0,0,0 +ADJOURNMENT,0,0,0,2009,0,0,0 +ADJUDGED,0,0,0,2009,0,0,0 +ADJUDICATED,0,0,0,2009,0,0,0 +ADJUDICATIONS,0,0,0,2009,0,0,0 +ADJUDICATORY,0,0,0,2009,0,0,0 +ADMISSION,0,0,0,2009,0,0,0 +ADULTERATING,2009,0,0,0,0,0,0 +ADVANCEMENTS,0,2009,0,0,0,0,0 +ADVANTAGED,0,2009,0,0,0,0,0 +ADVERSARIAL,2009,0,0,0,0,0,0 +ADVERSELY,2009,0,0,0,0,0,0 +AFFIDAVITS,0,0,0,2009,0,0,0 +AFOREMENTIONED,0,0,0,2009,0,0,0 +AFTERMATHS,2009,0,0,0,0,0,0 +AGGRAVATES,2009,0,0,0,0,0,0 +AGGRIEVED,0,0,0,2009,0,0,0 +ALIENATED,2009,0,0,0,0,0,0 +ALIENATIONS,2009,0,0,0,0,0,0 +ALLEGED,2009,0,0,2009,0,0,0 +ALLIANCE,0,2009,0,0,0,0,0 +ALTERATIONS,0,0,2009,0,0,0,0 +AMBIGUOUS,0,0,2009,0,0,0,0 +AMENDED,0,0,0,2009,0,0,0 +AMENDS,0,0,0,2009,0,0,0 +ANNOYED,2009,0,0,0,0,0,0 +ANNULLED,2009,0,0,0,0,0,0 +ANNULS,2009,0,0,0,0,0,0 +ANOMALY,2009,0,2009,0,0,0,0 +ANTICIPATED,0,0,2009,0,0,0,0 +ANTICIPATIONS,0,0,2009,0,0,0,0 +ANYWISE,0,0,0,2009,0,0,0 +APPEALABLE,0,0,0,2009,0,0,0 +APPEAR,0,0,2009,0,0,0,0 +APPELLANT,0,0,0,2009,0,0,0 +APPOINTOR,0,0,0,2011,0,0,0 +APPROXIMATES,0,0,2009,0,0,0,0 +APPURTENANCE,0,0,0,2009,0,0,0 +ARBITRAL,0,0,0,2009,0,0,0 +ARBITRATE,0,0,0,2009,0,0,0 +ARBITRATION,0,0,0,2009,0,0,0 +ARBITRATOR,0,0,0,2009,0,0,0 +ARGUING,2009,0,0,0,0,0,0 +ARREARAGE,2009,0,0,2009,0,0,0 +ARRESTED,2009,0,0,0,0,0,0 +RETROCEDE,0,0,0,2011,0,0,0 +BOLSTERED,0,2009,0,0,0,0,0 +BONAFIDE,0,0,0,2009,0,0,0 +BOOSTED,0,2009,0,0,0,0,0 +BOUNDED,0,0,0,0,0,0,2009 +BOYCOTTS,2009,0,0,0,0,0,0 +BREACHING,2009,0,0,2009,0,0,0 +BREAKDOWN,2009,0,0,0,0,0,0 +BREAKTHROUGH,0,2009,0,0,0,0,0 +BRIBERIES,2009,0,0,0,0,0,0 +BRIDGE,-2020,0,0,0,0,0,0 +BURDENED,2009,0,0,0,0,0,0 +BURNED,2009,0,0,0,0,0,0 +CANCEL,2009,0,0,0,0,0,0 +CANCELLATIONS,2009,0,0,0,0,0,0 +CARELESS,2009,0,0,0,0,0,0 +CATASTROPHE,2009,0,0,0,0,0,0 +CAUTION,2009,0,0,0,0,0,0 +CAUTIONS,2009,0,0,0,0,0,0 +CEASE,2009,0,0,0,0,0,0 +CEDANT,0,0,0,2012,0,0,0 +CENSURES,2009,0,0,0,0,0,0 +CHALLENGE,2009,0,0,0,0,0,0 +CHARGEOFFS,2009,0,0,0,0,0,0 +CHOATE,0,0,0,2011,0,0,0 +CIRCUMVENTION,2009,0,0,0,0,0,0 +SHOCKED,2009,0,0,0,0,0,0 +SHORTFALLS,2009,0,0,0,0,0,0 +SHUTDOWN,2009,0,0,0,0,0,0 +SLANDER,2009,0,0,0,0,0,0 +REPUDIATED,2009,0,0,0,0,0,0 +REPUDIATIONS,2009,0,0,0,0,0,0 +REQUIRED,0,0,0,0,0,0,2009 +REQUIRING,0,0,0,0,0,0,2009 +RESCINDING,0,0,0,2009,0,0,0 +RESIGN,2009,0,0,0,0,0,0 +RESIGNING,2009,0,0,0,0,0,0 +RESTATED,2009,0,0,0,0,0,0 +RESTATING,2009,0,0,0,0,0,0 +NONUSURIOUS,0,0,0,2011,0,0,0 +NOTARIZATIONS,0,0,0,2009,0,0,0 +NOTARY,0,0,0,2009,0,0,0 +NUISANCES,2009,0,0,0,0,0,0 +NULLIFIES,2009,0,0,2009,0,0,0 +NULLITY,0,0,0,2009,0,0,0 +OBJECTIONABLE,2009,0,0,0,0,0,0 +OBLIGATED,0,0,0,0,0,0,2009 +OBLIGATIONS,0,0,0,0,0,0,2009 +OBLIGEE,0,0,0,2009,0,0,0 +OBLIGORS,0,0,0,2009,0,0,0 +OBSOLETE,2009,0,0,0,0,0,0 +OBSTRUCTED,2009,0,0,0,0,0,0 +OCCASIONALLY,0,0,2009,0,0,2009,0 +OFFENDED,2009,0,0,0,0,0,0 +OFFENDS,2009,0,0,0,0,0,0 +OFFEROR,0,0,0,2009,0,0,0 +OMIT,2009,0,0,0,0,0,0 +ONEROUS,2009,0,0,0,0,0,0 +OPPORTUNITY,0,2009,0,0,0,0,0 +OPPOSING,2009,0,0,0,0,0,0 +OPTIONEE,0,0,0,2009,0,0,0 +OUTAGES,2009,0,0,0,0,0,0 +OUTPERFORMED,0,2009,0,0,0,0,0 +OVERAGES,2009,0,0,0,0,0,0 +OVERBUILT,2009,0,0,0,0,0,0 +OVERCAPACITIES,2009,0,0,0,0,0,0 +OVERCHARGES,2009,0,0,0,0,0,0 +OVERCOMING,2009,0,0,0,0,0,0 +OVERESTIMATES,2009,0,0,0,0,0,0 +OVERLOAD,2009,0,0,0,0,0,0 +OVERLOOK,2009,0,0,0,0,0,0 +OVERPAID,2009,0,0,0,0,0,0 +OVERPRODUCES,2009,0,0,0,0,0,0 +OVERRULED,0,0,0,2009,0,0,0 +OVERRUNNING,2009,0,0,0,0,0,0 +OVERSHADOWING,2009,0,0,0,0,0,0 +OVERSTATEMENT,2009,0,0,0,0,0,0 +OVERSUPPLIED,2009,0,0,0,0,0,0 +OVERTLY,2009,0,0,0,0,0,0 +OVERTURNS,2009,0,0,0,0,0,0 +PANIC,2009,0,0,0,0,0,0 +NEARLY,0,0,2009,0,0,2009,0 +NECESSITATING,0,0,0,0,0,0,2009 +NEGLECT,2009,0,0,0,0,0,0 +NEGLECTS,2009,0,0,0,0,0,0 +NEGLIGENTLY,2009,0,0,0,0,0,0 +BARRIER,2009,0,0,0,0,0,0 +BELIEVE,0,0,2009,0,0,0,0 +IDLED,2009,0,0,0,0,0,0 +IGNORES,2009,0,0,0,0,0,0 +ILLEGALITIES,2009,0,0,0,0,0,0 +ILLICIT,2009,0,0,0,0,0,0 +IMBALANCE,2009,0,0,0,0,0,0 +IMMORAL,2009,0,0,0,0,0,0 +IMPAIRMENT,2009,0,0,0,0,0,2011 +IMPASSES,2009,0,0,0,0,0,0 +IMPEDIMENT,2009,0,0,0,0,0,0 +IMPERATIVE,2009,0,0,0,0,0,0 +IMPERMISSIBLE,2009,0,0,0,0,0,0 +IMPLICATES,2009,0,0,0,0,0,0 +IMPOSES,0,0,0,0,0,0,2009 +IMPOSSIBILITY,2009,0,0,0,0,0,0 +IMPOUNDING,2009,0,0,0,0,0,0 +IMPRACTICALITIES,2009,0,0,0,0,0,0 +IMPRECISIONS,0,0,2009,0,0,0,0 +IMPRESSING,0,2009,0,0,0,0,0 +IMPROBABILITY,0,0,2009,0,0,0,0 +IMPROPRIETIES,2009,0,0,0,0,0,0 +IMPROVEMENT,0,2009,0,0,0,0,0 +IMPRUDENT,2009,0,0,0,0,0,0 +INACCURACIES,2009,0,0,0,0,0,0 +INACTION,2009,0,0,0,0,0,0 +INACTIVATES,2009,0,0,0,0,0,0 +INACTIVITY,2009,0,0,0,0,0,0 +INADEQUATELY,2009,0,0,0,0,0,0 +INADVISABLE,2009,0,0,0,0,0,0 +INATTENTION,2009,0,0,0,0,0,0 +INCARCERATE,2009,0,0,2009,0,0,0 +INCARCERATION,2009,0,0,2009,0,0,0 +INCIDENCES,2009,0,0,0,0,0,0 +INCOMPATIBILITY,2009,0,0,0,0,0,0 +INCOMPETENT,2009,0,0,0,0,0,0 +INCOMPLETELY,2009,0,0,0,0,0,0 +INCONSISTENCY,2009,0,0,0,0,0,0 +INCONTESTABLE,0,0,0,2009,0,0,0 +INCORRECT,2009,0,0,0,0,0,0 +INCREDIBLY,0,2009,0,0,0,0,0 +INDEFEASIBLE,2009,0,0,0,0,0,0 +INDEFINITENESS,0,0,2009,0,0,0,0 +INDEMNIFIED,0,0,0,2009,0,0,0 +INDEMNITEE,0,0,0,2009,0,0,0 +INDEMNITORS,0,0,0,2009,0,0,0 +INDICT,2009,0,0,2009,0,0,0 +INDICTMENT,2009,0,0,2009,0,0,0 +INEFFECTIVELY,2009,0,0,0,0,0,0 +INEFFICIENT,2009,0,0,0,0,0,0 +INEQUITABLE,2009,0,0,0,0,0,0 +INEVITABLE,2009,0,0,0,0,0,0 +INEXPERIENCED,2009,0,0,0,0,0,0 +INFORCE,0,0,0,2009,0,0,0 +INFRINGE,2009,0,0,0,0,0,0 +INFRINGER,0,0,0,2009,0,0,0 +INHIBIT,0,0,0,0,0,0,2011 +INIMICAL,2009,0,0,0,0,0,0 +INJURE,2009,0,0,0,0,0,0 +INJURING,2009,0,0,0,0,0,0 +INNOVATED,0,2009,0,0,0,0,0 +INNOVATIONS,0,2009,0,0,0,0,0 +INNOVATORS,0,2009,0,0,0,0,0 +INSECURE,2009,0,0,0,0,0,0 +INSISTED,0,0,0,0,0,0,2009 +INSOFAR,0,0,0,2009,0,0,0 +INSPIRATION,0,2009,0,0,0,0,0 +INSUBORDINATION,2009,0,0,0,0,0,0 +INSURRECTION,2009,0,0,0,0,0,0 +INTEGRITY,0,2009,0,0,0,0,0 +INTERFERENCE,2009,0,0,0,0,0,0 +INTERLOCUTORY,0,0,0,2009,0,0,0 +INTERPOSE,0,0,0,2009,0,0,0 +INTERPOSITION,0,0,0,2009,0,0,0 +INTERROGATES,0,0,0,2009,0,0,0 +INTERROGATOR,0,0,0,2009,0,0,0 +INTERRUPT,2009,0,0,0,0,0,0 +INTERRUPTIONS,2009,0,0,0,0,0,0 +INTIMIDATION,2009,0,0,0,0,0,0 +SPORADICALLY,0,0,2009,0,0,0,0 +STABILIZE,0,2009,0,0,0,0,0 +STABLE,0,2009,0,0,0,0,0 +STAGNATED,2009,0,0,0,0,0,0 +STANDSTILL,2009,0,0,0,0,0,0 +STATUTORILY,0,0,0,2009,0,0,0 +STIPULATES,0,0,0,0,0,0,2009 +STOLEN,2009,0,0,0,0,0,0 +STOPPING,2009,0,0,0,0,0,0 +STRAINING,2009,0,0,0,0,0,0 +STRENGTHENED,0,2009,0,0,0,0,0 +STRESS,2009,0,0,0,0,0,0 +STRESSING,2009,0,0,0,0,0,0 +STRICTLY,0,0,0,0,0,0,2009 +STRONGEST,0,2009,0,0,0,0,0 +SUBDOCKET,0,0,0,2012,0,0,0 +SUBLEASEE,0,0,0,2011,0,0,0 +SUBLICENSOR,0,0,0,2011,0,0,0 +SUBPOENAED,2009,0,0,2009,0,0,0 +SUBSTANDARD,2009,0,0,0,0,0,0 +SUCCEEDED,0,2009,0,0,0,0,0 +SUCCESSES,0,2009,0,0,0,0,0 +SUDDENLY,0,0,2009,0,0,0,0 +SUFFER,2009,0,0,0,0,0,0 +SUGGEST,0,0,2009,0,0,2009,0 +SUING,2009,0,0,2009,0,0,0 +SUMMONSES,2009,0,0,2009,0,0,0 +SUPERSEDED,0,0,0,2009,0,0,0 +SURETY,0,0,0,2009,0,0,0 +SURPASSING,0,2009,0,0,0,0,0 +SUSPECTED,2009,0,0,0,0,0,0 +SUSPENDING,2009,0,0,0,0,0,0 +SUSPICION,2009,0,0,0,0,0,0 +REVISED,0,0,2009,0,0,0,0 +REVOKE,2009,0,0,0,0,0,0 +REVOLUTIONIZE,0,2009,0,0,0,0,0 +REWARD,0,2009,0,0,0,0,0 +RIDICULE,2009,0,0,0,0,0,0 +RISK,0,0,2009,0,0,0,0 +RISKINESS,0,0,2009,0,0,0,0 +ROUGHLY,0,0,2009,0,0,0,0 +SABOTAGE,2009,0,0,0,0,0,0 +SACRIFICIAL,2009,0,0,0,0,0,0 +SATISFACTORY,0,2009,0,0,0,0,0 +SATISFYING,0,2009,0,0,0,0,0 +SCRUTINIZED,2009,0,0,0,0,0,0 +SECRECY,-2020,0,0,0,0,0,0 +SEIZES,2009,0,0,0,0,0,0 +SENTENCED,2009,0,0,2009,0,0,0 +SERIOUSLY,2009,0,0,0,0,0,0 +SETTLEMENT,0,0,0,2009,0,0,0 +SEVERABLE,0,0,0,2009,0,0,0 +SEVERE,2009,0,0,0,0,0,0 +SEVERITY,2009,0,0,0,0,0,0 +ENTHUSIASTIC,0,2009,0,0,0,0,0 +ERODE,2009,0,0,0,0,0,0 +EROSION,2009,0,0,0,0,0,0 +ERRING,2009,0,0,0,0,0,0 +ERRORS,2009,0,0,0,0,0,0 +ESCALATES,2009,0,0,0,0,0,0 +ESCHEATMENT,0,0,0,2011,0,0,0 +ESCROWS,0,0,0,0,0,0,2009 +EVADES,2009,0,0,0,0,0,0 +EVASIVE,2009,0,0,0,0,0,0 +EVICTION,2009,0,0,0,0,0,0 +EVIDENTIARY,0,0,0,2009,0,0,0 +EXACERBATING,2009,0,0,0,0,0,0 +EXAGGERATED,2009,0,0,0,0,0,0 +EXCEEDANCE,0,0,0,2011,0,0,0 +EXCELLENT,0,2009,0,0,0,0,0 +EXCEPTIONALLY,0,2009,0,0,0,0,0 +EXCITED,0,2009,0,0,0,0,0 +EXCLUSIVELY,0,2009,0,0,0,0,0 +EXCULPATE,2009,0,0,2009,0,0,0 +EXCULPATION,2009,0,0,2009,0,0,0 +EXECUTORS,0,0,0,2009,0,0,0 +EXECUTRIXES,0,0,0,2009,0,0,0 +EXONERATES,2009,0,0,0,0,0,0 +EXPLOIT,2009,0,0,0,0,0,0 +EXPLOITED,2009,0,0,0,0,0,0 +EXPOSED,2009,0,0,0,0,0,0 +EXPOSURES,0,0,2009,0,0,0,0 +EXPROPRIATING,2009,0,0,0,0,0,0 +EXPULSIONS,2009,0,0,0,0,0,0 +EXTRAJUDICIAL,0,0,0,2014,0,0,0 +SOLVES,0,2009,0,0,0,0,0 +SOMEWHAT,0,0,2009,0,0,2009,0 +SPAMMING,2014,0,0,0,0,0,0 +TREMENDOUSLY,0,2009,0,0,0,0,0 +LOCKOUTS,2009,0,0,0,0,0,0 +LOSS,2009,0,0,0,0,0,0 +LOYAL,0,2009,0,0,0,0,0 +MALFEASANCE,2009,0,0,0,0,0,0 +MALFUNCTIONS,2009,0,0,0,0,0,0 +MALPRACTICE,2009,0,0,0,0,0,0 +MANDATES,0,0,0,0,0,0,2009 +MANIPULATE,2009,0,0,0,0,0,0 +MANIPULATION,2009,0,0,0,0,0,0 +MARKDOWNS,2009,0,0,0,0,0,0 +MEDIATED,0,0,0,2009,0,0,0 +MEDIATIONS,0,0,0,2009,0,0,0 +MIGHT,0,0,2009,0,0,2009,0 +MISAPPLIES,2009,0,0,0,0,0,0 +MISAPPROPRIATED,2009,0,0,0,0,0,0 +MISAPPROPRIATIONS,2009,0,0,0,0,0,0 +MISCALCULATES,2009,0,0,0,0,0,0 +MISCHARACTERIZATION,2014,0,0,0,0,0,0 +MISCLASSIFIED,2011,0,0,0,0,0,0 +MISDATED,2011,0,0,0,0,0,0 +MISFEASANCE,0,0,0,2009,0,0,0 +MISHANDLING,2009,0,0,0,0,0,0 +MISINFORMING,2009,0,0,0,0,0,0 +MISINTERPRETATIONS,2009,0,0,0,0,0,0 +MISJUDGE,2009,0,0,0,0,0,0 +MISJUDGMENT,2009,0,0,0,0,0,0 +MISLABELING,2009,0,0,0,0,0,0 +MISLEADING,2009,0,0,0,0,0,0 +MISMANAGE,2009,0,0,0,0,0,0 +MISMANAGING,2009,0,0,0,0,0,0 +MISMATCHING,2009,0,0,0,0,0,0 +MISPRICINGS,2014,0,0,0,0,0,0 +MISREPRESENTED,2009,0,0,0,0,0,0 +MISSED,2009,0,0,0,0,0,0 +WORRIES,2009,0,0,0,0,0,0 +WORSEN,2009,0,0,0,0,0,0 +WORST,2009,0,0,0,0,0,0 +TIGHTENING,2009,0,0,0,0,0,0 +TOLERATING,2009,0,0,0,0,0,0 +TORTIOUSLY,0,0,0,2009,0,0,0 +FINES,2009,0,0,0,0,0,0 +NONASSESSABLE,0,0,2009,0,0,0,0 +NONCANCELLABLE,0,0,0,0,0,0,2009 +NONCOMPLIANT,2009,0,0,0,0,0,0 +NONCONFORMITY,2009,0,0,0,0,0,0 +NONCONTRIBUTORY,0,0,0,2009,0,0,0 +NONFORFEITABILITY,0,0,0,2011,0,0,0 +NONGUARANTOR,0,0,0,2011,0,0,0 +DISCIPLINARY,2009,0,0,0,0,0,0 +DISCLAIMERS,2009,0,0,0,0,0,0 +DISCLOSED,2009,0,0,0,0,0,0 +DISCONTINUANCES,2009,0,0,0,0,0,0 +DISCONTINUED,2009,0,0,0,0,0,0 +DISCOURAGED,2009,0,0,0,0,0,0 +DISCREDITED,2009,0,0,0,0,0,0 +DISCREPANCY,2009,0,0,0,0,0,0 +SPECULATED,0,0,2009,0,0,0,0 +SPECULATIONS,0,0,2009,0,0,0,0 +TRAGICALLY,2009,0,0,0,0,0,0 +LEGISLATED,0,0,0,2009,0,0,0 +LEGISLATIONS,0,0,0,2009,0,0,0 +LEGISLATORS,0,0,0,2009,0,0,0 +LIBELED,0,0,0,2009,0,0,0 +LIE,2009,0,0,0,0,0,0 +COMPULSION,2009,0,0,0,0,0,2009 +CONCEDE,2009,0,0,0,0,0,0 +CONCEIVABLE,0,0,2009,0,0,2009,0 +CONCERNS,2009,0,0,0,0,0,0 +CONCLUSIVE,0,2009,0,0,0,0,0 +CONDEMNATIONS,2009,0,0,0,0,0,0 +CONDEMNS,2009,0,0,0,0,0,0 +CONDONED,2009,0,0,0,0,0,0 +CONFESSES,2009,0,0,0,0,0,0 +CONFINE,2009,0,0,0,0,0,2009 +CONFINES,2009,0,0,0,0,0,2009 +CONFISCATES,2009,0,0,0,0,0,0 +CONFISCATORY,0,0,0,2009,0,0,0 +CONFLICTS,2009,0,0,0,0,0,0 +CONFRONTATIONS,2009,0,0,0,0,0,0 +CONFUSE,2009,0,0,0,0,0,0 +CONFUSINGLY,2009,0,2009,0,0,0,0 +CONSENTING,0,0,0,2009,0,0,0 +CONSPIRACY,2009,0,0,0,0,0,0 +CONSPIRE,2009,0,0,0,0,0,0 +CONSTITUTION,0,0,0,2009,0,0,0 +CONSTITUTIONS,0,0,0,2009,0,0,0 +CONSTRAINING,0,0,0,0,0,0,2009 +CONSTRUCTIVE,0,2009,0,0,0,0,0 +CONSTRUES,0,0,0,2012,0,0,0 +CONTENDED,2009,0,0,0,0,0,0 +CONTENTIONS,2009,0,0,0,0,0,0 +CONTESTATION,0,0,0,2011,0,0,0 +CONTINGENCY,0,0,2009,0,0,0,0 +CONTRACT,0,0,0,2009,0,0,0 +CONTRACTIBLE,0,0,0,2009,0,0,0 +CONTRACTIONS,2009,0,0,0,0,0,0 +CONTRADICT,2009,0,0,0,0,0,0 +CONTRADICTIONS,2009,0,0,0,0,0,0 +CONTRAVENE,0,0,0,2009,0,0,0 +CONTRAVENTION,0,0,0,2009,0,0,0 +CONTROVERSY,2009,0,0,0,0,0,0 +CONVENIENS,0,0,0,2012,0,0,0 +CONVICTED,2009,0,0,2009,0,0,0 +CORRECTED,2009,0,0,0,0,0,0 +CORRECTS,2009,0,0,0,0,0,0 +CORRUPTION,2009,0,0,0,0,0,0 +COSTLY,2009,0,0,0,0,0,0 +COUNSELED,0,0,0,2009,0,0,0 +COUNTERCLAIMED,2009,0,0,0,0,0,0 +COUNTERFEITED,2009,0,0,0,0,0,0 +COUNTERFEITS,2009,0,0,0,0,0,0 +COUNTERSUED,0,0,0,2011,0,0,0 +COURTEOUS,0,2009,0,0,0,0,0 +COVENANTED,0,0,0,0,0,0,2011 +CREATIVELY,0,2009,0,0,0,0,0 +CRIMES,2009,0,0,2009,0,0,0 +CRIMINALIZING,0,0,0,2014,0,0,0 +CRISIS,2009,0,0,0,0,0,0 +CRITICISMS,2009,0,0,0,0,0,0 +CRITICIZING,2009,0,0,0,0,0,0 +CROSSROADS,0,0,2009,0,0,0,0 +CULPABLE,2009,0,0,0,0,0,0 +CURTAILED,2009,0,0,0,0,0,0 +CURTAILS,2009,0,0,0,0,0,0 +CYBERATTACK,2014,0,0,0,0,0,0 +CYBERCRIMES,2014,0,0,0,0,0,0 +TAINTS,2009,0,0,0,0,0,0 +TENSE,2009,0,0,0,0,0,0 +TERMINATE,2009,0,0,0,0,0,0 +TERMINATION,2009,0,0,0,0,0,0 +TESTIFY,2009,0,0,2009,0,0,0 +THENCEFORTH,0,0,0,2009,0,0,0 +THEREFROM,0,0,0,2009,0,0,0 +THEREON,0,0,0,2009,0,0,0 +THERETOFORE,0,0,0,2009,0,0,0 +THEREWITH,0,0,0,2009,0,0,0 +THREATENING,2009,0,0,0,0,0,0 +FACIE,0,0,0,2009,0,0,0 +FAILING,2009,0,0,0,0,0,0 +FAILURES,2009,0,0,0,0,0,0 +FALSIFICATION,2009,0,0,0,0,0,0 +FALSIFY,2009,0,0,0,0,0,0 +FATALITIES,2009,0,0,0,0,0,0 +FAULTED,2009,0,0,0,0,0,0 +FAVORABLY,0,2009,0,0,0,0,0 +FAVORITES,0,2009,0,0,0,0,0 +FELONIOUS,2009,0,0,2009,0,0,0 +DAMAGES,2009,0,0,0,0,0,0 +DANGER,2009,0,0,0,0,0,0 +DEADLOCK,2009,0,0,0,0,0,0 +DEADWEIGHT,2009,0,0,0,0,0,0 +DEBARRED,2009,0,0,0,0,0,0 +DECEIT,2009,0,0,0,0,0,0 +DECEIVED,2009,0,0,0,0,0,0 +DECEPTIONS,2009,0,0,0,0,0,0 +DECLINE,2009,0,0,0,0,0,0 +DECREE,0,0,0,2009,0,0,0 +DEFACE,2009,0,0,0,0,0,0 +DEFALCATIONS,0,0,0,2009,0,0,0 +DEFAME,2009,0,0,0,0,0,0 +DEFAULT,2009,0,0,0,0,0,0 +DEFEASANCE,0,0,0,2009,0,0,0 +DEFEASEMENT,0,0,0,2014,0,0,0 +DEFEATED,2009,0,0,0,0,0,0 +DEFECTIVE,2009,0,0,0,0,0,0 +DEFENDABLE,0,0,0,2014,0,0,0 +DEFENDING,2009,0,0,0,0,0,0 +DEFERENCE,0,0,0,2009,0,0,0 +DEFICIT,2009,0,0,0,0,0,0 +DEFRAUD,2009,0,0,0,0,0,0 +DEFUNCT,2009,0,0,0,0,0,0 +DEGRADED,2009,0,0,0,0,0,0 +DELAYED,2009,0,0,0,0,0,0 +DELEGATABLE,0,0,0,2011,0,0,0 +DELIBERATE,2009,0,0,0,0,0,0 +DELIGHTED,0,2009,0,0,0,0,0 +DELIGHTS,0,2009,0,0,0,0,0 +DELINQUENTLY,2009,0,0,0,0,0,0 +DELISTING,2009,0,0,0,0,0,0 +DEMISES,2009,0,0,0,0,0,0 +DEMOLISHES,2009,0,0,0,0,0,0 +DEMOTE,2009,0,0,0,0,0,0 +DEMOTION,2009,0,0,0,0,0,0 +DEMURRERS,0,0,0,2009,0,0,0 +DENIALS,2009,0,0,0,0,0,0 +DENIGRATED,2009,0,0,0,0,0,0 +DENY,2009,0,0,0,0,0,0 +DEPENDABLE,0,2009,0,0,0,0,0 +DEPENDED,0,0,2009,0,0,2009,0 +DEPENDENT,0,0,2009,0,0,0,2011 +DEPLETED,2009,0,0,0,0,0,0 +DEPLETIONS,2009,0,0,0,0,0,0 +DEPOSING,0,0,0,2009,0,0,0 +INVALID,2009,0,0,0,0,0,0 +INVALIDATING,2009,0,0,0,0,0,0 +INVENTED,0,2009,0,0,0,0,0 +INVENTIVE,0,2009,0,0,0,0,0 +INVESTIGATE,2009,0,0,0,0,0,0 +INVESTIGATION,2009,0,0,0,0,0,0 +IRRECONCILABLE,2009,0,0,0,0,0,0 +IRREGULAR,2009,0,0,0,0,0,0 +IRREPARABLE,2009,0,0,0,0,0,0 +IRREVOCABLE,0,0,0,2009,0,0,2009 +JOINDER,0,0,0,2009,0,0,0 +JUDICIARY,0,0,0,2009,0,0,0 +JURISDICTIONAL,0,0,0,2009,0,0,0 +JURIST,0,0,0,2009,0,0,0 +JURY,0,0,0,2009,0,0,0 +JUSTIFIABLE,2009,0,0,0,0,0,0 +DILIGENTLY,0,2009,0,0,0,0,0 +DIMINISHING,2009,0,0,0,0,0,0 +DISADVANTAGE,2009,0,0,0,0,0,0 +DISAFFILIATION,2009,0,0,2009,0,0,0 +DISAFFIRMS,0,0,0,2011,0,0,0 +DISAGREEING,2009,0,0,0,0,0,0 +DISALLOW,2009,0,0,0,0,0,0 +DISALLOWING,2009,0,0,0,0,0,0 +DISAPPEARANCES,2009,0,0,0,0,0,0 +DISAPPOINT,2009,0,0,0,0,0,0 +DISAPPOINTMENT,2009,0,0,0,0,0,0 +DISAPPROVALS,2009,0,0,0,0,0,0 +DISAPPROVING,2009,0,0,0,0,0,0 +DISASSOCIATIONS,2009,0,0,0,0,0,0 +DISASTROUSLY,2009,0,0,0,0,0,0 +DISAVOWING,2009,0,0,0,0,0,0 +GREATER,0,-2020,0,0,0,0,0 +GRIEVANCE,2009,0,0,0,0,0,0 +GUILTY,2009,0,0,0,0,0,0 +HAMPERED,2009,0,0,0,0,0,0 +HAPPILY,0,2009,0,0,0,0,0 +HARASSED,2009,0,0,0,0,0,0 +HARDSHIPS,2009,0,0,0,0,0,0 +HARMFULLY,2009,0,0,0,0,0,0 +HARSHER,2009,0,0,0,0,0,0 +HAZARD,2009,0,0,0,0,0,0 +NONJUDICIALLY,0,0,0,2011,0,0,0 +NONPERFORMANCE,2009,0,0,0,0,0,0 +HURT,2009,0,0,0,0,0,0 +DISFAVORED,2009,0,0,0,0,0,0 +DISGORGED,2009,0,0,0,0,0,0 +COMPLICATING,2009,0,0,0,0,0,0 +COMPLIMENTARY,0,2009,0,0,0,0,0 +COMPLY,0,0,0,0,0,0,2009 +MISSTATED,2009,0,0,0,0,0,0 +MISSTATING,2009,0,0,0,0,0,0 +MISTAKEN,2009,0,0,0,0,0,0 +MISTRIAL,2009,0,0,2009,0,0,0 +MISUNDERSTANDINGS,2009,0,0,0,0,0,0 +MISUSES,2009,0,0,0,0,0,0 +MONOPOLIZATION,2009,0,0,0,0,0,0 +MONOPOLIZING,2009,0,0,0,0,0,0 +MORATORIUMS,2009,0,0,0,0,0,0 +MOTIONS,0,0,0,2009,0,0,0 +SLOWDOWN,2009,0,0,0,0,0,0 +SLOWEST,2009,0,0,0,0,0,0 +SLUGGISH,2009,0,0,0,0,0,0 +SMOOTHING,0,2009,0,0,0,0,0 +DEPRESS,2009,0,0,0,0,0,0 +DEPRIVATION,2009,0,0,0,0,0,0 +DEPRIVING,2009,0,0,0,0,0,0 +DEROGATED,0,0,0,2009,0,0,0 +DEROGATIONS,0,0,0,2009,0,0,0 +DESIRED,0,2009,0,0,0,0,0 +DESTABILIZE,2009,0,0,0,0,0,0 +DESTROY,2009,0,0,0,0,0,0 +DESTRUCTION,2009,0,0,0,0,0,0 +DETAINER,0,0,0,2009,0,0,0 +DETERIORATE,2009,0,0,0,0,0,0 +DETERIORATION,2009,0,0,0,0,0,0 +DETERRENCES,2009,0,0,0,0,0,0 +DETERS,2009,0,0,0,0,0,0 +DETRIMENT,2009,0,0,0,0,0,0 +DEVALUE,2009,0,0,0,0,0,0 +DEVASTATE,2009,0,0,0,0,0,0 +DEVIATE,2009,0,2009,0,0,0,0 +DEVIATION,2009,0,2009,0,0,0,0 +DEVOLVED,2009,0,0,0,0,0,0 +DICTATED,0,0,0,0,0,0,2009 +DIFFERED,0,0,2009,0,0,0,0 +DIFFICULTIES,2009,0,0,0,0,0,0 +ASCENDANCY,0,0,0,2009,0,0,0 +ASSAULTED,2009,0,0,0,0,0,0 +ASSERTIONS,2009,0,0,0,0,0,0 +ASSUME,0,0,2009,0,0,0,0 +ASSUMPTION,0,0,2009,0,0,0,0 +ASSURES,0,2009,0,0,0,0,0 +ATTAINING,0,2009,0,0,0,0,0 +ATTEST,0,0,0,2009,0,0,0 +ATTESTING,0,0,0,2009,0,0,0 +ATTORNMENT,0,0,0,2009,0,0,0 +ATTRITION,2009,0,0,0,0,0,0 +BAIL,2009,0,0,2009,0,0,0 +BAILIFF,0,0,0,2009,0,0,0 +BALK,2009,0,0,0,0,0,0 +BANKRUPTCY,2009,0,0,0,0,0,0 +BANS,2009,0,0,0,0,0,0 +CLAIMABLE,0,0,0,2009,0,0,0 +CLAIMING,2009,0,0,0,0,0,0 +CLAWBACK,2009,0,0,0,0,0,0 +CLOSEOUT,2009,0,0,0,0,0,0 +CLOSURE,2009,0,0,0,0,0,0 +CODICIL,0,0,0,2009,0,0,0 +CODIFIED,0,0,0,2009,0,0,0 +COERCE,2009,0,0,0,0,0,0 +COERCION,2009,0,0,0,0,0,0 +COLLABORATING,0,2009,0,0,0,0,0 +COLLABORATOR,0,2009,0,0,0,0,0 +COLLAPSES,2009,0,0,0,0,0,0 +COLLUDE,2009,0,0,0,0,0,0 +COLLUSION,2009,0,0,2009,0,0,0 +POSING,2009,0,0,0,0,0,0 +POSSIBILITIES,0,0,2009,0,0,0,0 +POSTCLOSING,0,0,0,2011,0,0,0 +POSTPONE,2009,0,0,0,0,0,0 +POSTPONES,2009,0,0,0,0,0,0 +WRITEOFF,2009,0,0,0,0,0,0 +WRONGDOING,2009,0,0,0,0,0,0 +WRONGLY,2009,0,0,0,0,0,0 +HONORED,0,2009,0,0,0,0,0 +HOSTILITY,2009,0,0,0,0,0,0 +REASSESSED,0,0,2009,0,0,0,0 +REASSESSMENTS,2009,0,2009,0,0,0,0 +REASSIGNMENT,2009,0,0,0,0,0,0 +REBOUNDED,0,2009,0,0,0,0,0 +REBUTTABLE,0,0,0,2009,0,0,0 +REBUTTED,0,0,0,2009,0,0,0 +RECALCULATES,0,0,2009,0,0,0,0 +RECALL,2009,0,0,0,0,0,0 +RECEPTIVE,0,2009,0,0,0,0,0 +RECKLESS,2009,0,0,0,0,0,0 +RECONSIDERED,0,0,2009,0,0,0,0 +RECOUPABLE,0,0,0,2009,0,0,0 +RECOURSES,0,0,0,2009,0,0,0 +RECUSE,0,0,0,2011,0,0,0 +REDACT,2009,0,0,2009,0,0,0 +REDACTIONS,2009,0,0,2009,0,0,0 +REDRESS,2009,0,0,0,0,0,0 +REEXAMINATION,0,0,2009,0,0,0,0 +REFERENDUM,0,0,0,2009,0,0,0 +REFILES,0,0,0,2009,0,0,0 +REFRAINS,0,0,0,0,0,0,2009 +REFUSED,2009,0,0,0,0,0,0 +REGAINED,0,2009,0,0,0,0,0 +REGULATES,0,0,0,2009,0,0,0 +REGULATIVE,0,0,0,2009,0,0,0 +REHEAR,0,0,0,2009,0,0,0 +VARIABLE,0,0,2009,0,0,0,0 +VARIANCES,0,0,2009,0,0,0,0 +VARIATIONS,0,0,2009,0,0,0,0 +VARYING,0,0,2009,0,0,0,0 +VERDICTS,2009,0,0,2009,0,0,0 +VIATICAL,0,0,0,2011,0,0,0 +VIOLATE,2009,0,0,0,0,0,0 +VIOLATION,2009,0,0,0,0,0,0 +VIOLATORS,2009,0,0,0,0,0,0 +VITIATE,2009,0,0,0,0,0,0 +VITIATION,2009,0,0,0,0,0,0 +VOLATILE,2009,0,2009,0,0,0,0 +VULNERABILITY,2009,0,0,0,0,0,0 +WARNED,2009,0,0,0,0,0,0 +WARRANTEES,0,0,0,2011,0,0,0 +WASTING,2009,0,0,0,0,0,0 +WEAKENING,2009,0,0,0,0,0,0 +WEAKLY,2009,0,0,0,0,0,0 +DISHONESTY,2009,0,0,0,0,0,0 +DISHONORED,2009,0,0,0,0,0,0 +DISLOYAL,2009,0,0,0,0,0,0 +DISMALLY,2009,0,0,0,0,0,0 +DISMISSED,2009,0,0,0,0,0,0 +DISPARAGE,2009,0,0,0,0,0,0 +DISPARAGES,2009,0,0,0,0,0,0 +DISPARITY,2009,0,0,0,0,0,0 +DISPLACEMENTS,2009,0,0,0,0,0,0 +DISPOSITIVE,0,0,0,2009,0,0,0 +DISPOSSESSING,2009,0,0,0,0,0,0 +DISPROPORTIONAL,2009,0,0,0,0,0,0 +DISPUTED,2009,0,0,0,0,0,0 +DISQUALIFICATIONS,2009,0,0,0,0,0,0 +DISQUALIFYING,2009,0,0,0,0,0,0 +DISREGARDS,2009,0,0,0,0,0,0 +DISRUPTED,2009,0,0,0,0,0,0 +DISRUPTIVE,2009,0,0,0,0,0,0 +DISSENT,2009,0,0,0,0,0,0 +DISSENTING,2009,0,0,0,0,0,0 +DISSOLUTION,2009,0,0,0,0,0,0 +DISTINCTIVE,0,2009,0,0,0,0,0 +DISTORTED,2009,0,0,0,0,0,0 +DISTORTS,2009,0,0,0,0,0,0 +DISTRACTION,2009,0,0,0,0,0,0 +DISTRESS,2009,0,0,0,0,0,0 +DISTURB,2009,0,0,0,0,0,0 +DISTURBING,2009,0,0,0,0,0,0 +DIVERTED,2009,0,0,0,0,0,0 +DIVESTED,2009,0,0,0,0,0,0 +DIVESTMENT,2009,0,0,0,0,0,0 +DIVORCED,2009,0,0,0,0,0,0 +DIVULGING,2009,0,0,0,0,0,0 +DOCKETS,0,0,0,2009,0,0,0 +DOUBTFUL,2009,0,2009,0,0,0,0 +DOWNGRADES,2009,0,0,0,0,0,0 +DOWNSIZES,2009,0,0,0,0,0,0 +DOWNTIMES,2009,0,0,0,0,0,0 +DOWNWARDS,2009,0,0,0,0,0,0 +DRAWBACK,2009,0,0,0,0,0,0 +DROUGHT,2009,0,0,0,0,0,0 +DYSFUNCTION,2009,0,0,0,0,0,0 +EARMARKED,0,0,0,0,0,0,2009 +EASILY,0,2009,0,0,0,0,0 +EFFICIENCIES,0,2009,0,0,0,0,0 +EGREGIOUS,2009,0,0,0,0,0,0 +EMBARGOED,2009,0,0,0,0,0,0 +EMBARRASSED,2009,0,0,0,0,0,0 +EMBARRASSMENTS,2009,0,0,0,0,0,0 +EMBEZZLEMENTS,2009,0,0,0,0,0,0 +EMPOWER,0,2009,0,0,0,0,0 +ENABLE,0,2009,0,0,0,0,0 +ENCOURAGED,0,2009,0,0,0,0,0 +ENCROACH,2009,0,0,0,0,0,0 +ENCROACHMENT,2009,0,0,0,0,0,0 +ENCUMBERING,2009,0,0,2009,0,0,2009 +ENCUMBRANCERS,0,0,0,2011,0,0,0 +ENDANGERING,2009,0,0,0,0,0,0 +ENFORCEABILITY,0,0,0,2009,0,0,0 +ENHANCED,0,2009,0,0,0,0,0 +ENHANCING,0,2009,0,0,0,0,0 +ENJOINS,2009,0,0,0,0,0,0 +ENJOYED,0,2009,0,0,0,0,0 +ENTAIL,0,0,0,0,0,0,2009 +PECUNIARILY,0,0,0,2009,0,0,0 +PENALIZING,2009,0,0,0,0,0,0 +PERFECT,0,2009,0,0,0,0,0 +PERHAPS,0,0,2009,0,0,2009,0 +PERMISSIBLE,0,0,0,0,0,0,2009 +PERMITTEE,0,0,0,2011,0,0,0 +PERPETRATED,2009,0,0,2009,0,0,0 +PERSIST,2009,0,0,0,0,0,0 +PERSISTENTLY,2009,0,0,0,0,0,0 +PERVASIVE,2009,0,0,0,0,0,0 +PETITIONED,0,0,0,2009,0,0,0 +PETITIONS,0,0,0,2009,0,0,0 +PICKETING,2009,0,0,0,0,0,0 +HENCEFORWARD,0,0,0,2009,0,0,0 +HEREFOR,0,0,0,2009,0,0,0 +HEREINABOVE,0,0,0,2009,0,0,0 +HEREOF,0,0,0,2009,0,0,0 +HEREUNDER,0,0,0,2009,0,0,0 +HEREWITHIN,0,0,0,2014,0,0,0 +HINDERED,2009,0,0,0,0,0,0 +HINDRANCES,2009,0,0,0,0,0,0 +WHENSOEVER,0,0,0,2009,0,0,0 +WHEREBY,0,0,0,2009,0,0,0 +WHEREON,0,0,0,2009,0,0,0 +WHEREWITH,0,0,0,2009,0,0,0 +WHOSOEVER,0,0,0,2009,0,0,0 +WILLFULLY,2009,0,0,2009,0,0,0 +WINNERS,0,2009,0,0,0,0,0 +FLUCTUATING,0,0,2009,0,0,0,0 +FORBEAR,0,0,0,2009,0,0,0 +FORBEARS,0,0,0,2009,0,0,0 +FORBIDS,2009,0,0,0,0,0,2009 +FOREBEAR,0,0,0,2009,0,0,0 +FORECLOSED,2009,0,0,0,0,0,0 +FORECLOSURES,2009,0,0,0,0,0,0 +FORESTALL,2009,0,0,0,0,0,0 +FORFEIT,2009,0,0,0,0,0,0 +FORFEITING,2009,0,0,0,0,0,0 +FORGERS,2009,0,0,0,0,0,0 +FRAUD,2009,0,0,0,0,0,0 +FRAUDULENTLY,2009,0,0,0,0,0,0 +FRUSTRATE,2009,0,0,0,0,0,0 +FRUSTRATINGLY,2009,0,0,0,0,0,0 +FUGITIVES,2009,0,0,2009,0,0,0 +GAINING,0,2009,0,0,0,0,0 +GRANTORS,0,0,0,2009,0,0,0 +DISGRACE,2009,0,0,0,0,0,0 +LITIGANTS,2009,0,0,2009,0,0,0 +LITIGATING,2009,0,0,2009,0,0,0 +LITIGATORS,0,0,0,2009,0,0,0 +LACK,2009,0,0,0,0,0,0 +LACKS,2009,0,0,0,0,0,0 +LAGS,2009,0,0,0,0,0,0 +LAPSING,2009,0,0,0,0,0,0 +LAWFUL,0,0,0,2009,0,0,0 +LAWMAKING,0,0,0,2009,0,0,0 +LAWYER,0,0,0,2009,0,0,0 +LEADERSHIP,0,2009,0,0,0,0,0 +LEGALITY,0,0,0,2009,0,0,0 +LEGALIZED,0,0,0,2009,0,0,0 +LEGALS,0,0,0,2009,0,0,0 +FLAW,2009,0,0,0,0,0,0 +NONPRODUCTIVE,2009,0,0,0,0,0,0 +LIQUIDATED,2009,0,0,0,0,0,0 +LIQUIDATIONS,2009,0,0,0,0,0,0 +BENEFICIATED,0,0,0,2014,0,0,0 +BENEFITING,0,2009,0,0,0,0,0 +BETTER,0,2009,0,0,0,0,0 +POPULARITY,0,2009,0,0,0,0,0 +REINTERPRET,0,0,2009,0,0,0,0 +REINTERPRETING,0,0,2009,0,0,0,0 +REJECTING,2009,0,0,0,0,0,0 +RELEASEES,0,0,0,2011,0,0,0 +RELINQUISHING,2009,0,0,0,0,0,0 +RELUCTANT,2009,0,0,0,0,0,0 +REMANDS,0,0,0,2009,0,0,0 +REMEDIATION,0,0,0,2009,0,0,0 +RENEGOTIATE,2009,0,0,0,0,0,0 +RENEGOTIATION,2009,0,0,0,0,0,0 +RENOUNCEMENT,2009,0,0,0,0,0,0 +REPARATION,2009,0,0,0,0,0,0 +REPOSSESSED,2009,0,0,0,0,0,0 +REPOSSESSIONS,2009,0,0,0,0,0,0 +TROUBLE,2009,0,0,0,0,0,0 +TURMOIL,2009,0,0,0,0,0,0 +UNACCOUNTED,2009,0,0,0,0,0,0 +UNAPPEALABLE,0,0,0,2009,0,0,0 +UNAUTHORIZED,2009,0,0,0,0,0,0 +UNAVOIDABLY,2009,0,0,0,0,0,0 +UNCERTAINTIES,0,0,2009,0,0,0,0 +UNCOLLECTED,2009,0,0,0,0,0,0 +UNCOMPETITIVE,2009,0,0,0,0,0,0 +UNCONSCIONABLE,2009,0,0,0,0,0,0 +UNCONSTITUTIONALLY,0,0,0,2009,0,0,0 +UNCONTROLLED,2009,0,0,0,0,0,0 +UNCOVERING,2009,0,0,0,0,0,0 +UNDEFINED,0,0,2009,0,0,0,0 +UNDERCUT,2009,0,0,0,0,0,0 +UNDERESTIMATED,2009,0,0,0,0,0,0 +UNDERFUNDED,2009,0,0,0,0,0,0 +UNDERMINES,2009,0,0,0,0,0,0 +UNDERPAYMENTS,2009,0,0,0,0,0,0 +UNDERPERFORMED,2011,0,0,0,0,0,0 +UNDERPRODUCTION,2009,0,0,0,0,0,0 +UNDERSTATEMENT,2009,0,0,0,0,0,0 +UNDERUTILIZATION,2009,0,0,0,0,0,0 +UNDESIRED,2009,0,0,0,0,0,0 +UNDETERMINED,2009,0,2009,0,0,0,0 +UNDOCUMENTED,2009,0,2009,0,0,0,0 +UNECONOMIC,2009,0,0,0,0,0,0 +UNEMPLOYMENT,2009,0,0,0,0,0,0 +UNENFORCEABLE,0,0,0,2009,0,0,0 +UNETHICALLY,2009,0,0,0,0,0,0 +UNFAIR,2009,0,0,0,0,0,0 +UNFAVORABILITY,2014,0,0,0,0,0,0 +UNFEASIBLE,2009,0,0,0,0,0,0 +UNFORESEEABLE,2009,0,0,0,0,0,0 +UNFORTUNATELY,2009,0,0,0,0,0,0 +UNFUNDED,2009,0,0,0,0,0,0 +UNIDENTIFIED,0,0,2009,0,0,0,0 +UNINTENTIONALLY,2009,0,0,0,0,0,0 +UNJUSTIFIED,2009,0,0,0,0,0,0 +UNKNOWN,0,0,2009,0,0,0,0 +UNLAWFULNESS,0,0,0,2009,0,0,0 +UNMATCHED,0,2009,0,0,0,0,0 +UNNECESSARY,2009,0,0,0,0,0,0 +UNOCCUPIED,2009,0,0,0,0,0,0 +UNPLANNED,2009,0,2009,0,0,0,0 +UNPREDICTABLY,2009,0,2009,0,0,0,0 +UNPROFITABLE,2009,0,0,0,0,0,0 +UNQUANTIFIABLE,0,0,2011,0,0,0,0 +UNREASONABLENESS,2009,0,0,0,0,0,0 +UNRECOVERABLE,2009,0,0,0,0,0,0 +UNREMEDIATED,0,0,0,2011,0,0,0 +UNREST,2009,0,0,0,0,0,0 +UNSATISFACTORY,2009,0,0,0,0,0,0 +UNSEASONABLE,0,0,2009,0,0,0,0 +UNSOLD,2009,0,0,0,0,0,0 +UNSTABILIZED,2014,0,0,0,0,0,0 +UNSUCCESSFUL,2009,0,0,0,0,0,0 +UNSUITABLY,2009,0,0,0,0,0,0 +UNSUSPECTED,2009,0,0,0,0,0,0 +UNTESTED,0,0,2009,0,0,0,0 +UNTRUTH,2009,0,0,0,0,0,0 +UNTRUTHS,2009,0,0,0,0,0,0 +UNWANTED,2009,0,0,0,0,0,0 +UNWILLINGNESS,2009,0,0,0,0,0,0 +UPTURNS,0,2009,0,0,0,0,0 +USURP,2009,0,0,2009,0,0,0 +USURPS,2009,0,0,2009,0,0,0 +VAGUELY,0,0,2012,0,0,0,0 +VAGUEST,0,0,2012,0,0,0,0 +PLEA,2009,0,0,0,0,0,0 +PLEADINGS,2009,0,0,2009,0,0,0 +PLEASANTLY,0,2009,0,0,0,0,0 +PLEDGE,0,0,0,0,0,0,2009 +PLEDGES,0,0,0,0,0,0,2009 +PLENTIFUL,0,2009,0,0,0,0,0 +COMPELS,0,0,0,0,0,0,2009 +COMPLAINANTS,0,0,0,2009,0,0,0 +COMPLAINT,2009,0,0,0,0,0,0 +COMMIT,0,0,0,0,0,0,2009 +COMMITTED,0,0,0,0,0,0,2009 +LIMIT,0,0,0,0,0,0,2009 +LIMITS,0,0,0,0,0,0,2009 +RESTRAINED,0,0,0,0,0,0,2009 +RESTRAINTS,0,0,0,0,0,0,2009 +RESTRICTION,0,0,0,0,0,0,2009 +RESTRICTIVENESS,0,0,0,0,0,0,2009 +RESTRUCTURES,2009,0,0,0,0,0,0 +RETALIATED,2009,0,0,0,0,0,0 +RETALIATIONS,2009,0,0,0,0,0,0 +PRECAUTIONARY,0,0,2009,0,0,0,0 +PRECIPITOUSLY,2009,0,0,0,0,0,0 +PRECLUDING,2009,0,0,0,0,0,2009 +PREDECEASE,0,0,0,2009,0,0,0 +PREDICT,0,0,2009,0,0,0,0 +PREDICTION,0,0,2009,0,0,0,0 +PREDICTORS,0,0,2009,0,0,0,0 +PREHEARING,0,0,0,2011,0,0,0 +PREJUDICIAL,2009,0,0,2009,0,0,0 +PREMATURE,2009,0,0,0,0,0,0 +PREPETITION,0,0,0,2009,0,0,0 +PRESTIGIOUS,0,2009,0,0,0,0,0 +PRESUMES,0,0,2009,0,0,0,0 +PRESUMPTIVELY,0,0,0,2009,0,0,0 +PREVENTING,2009,0,0,0,0,0,2009 +PRIVITY,0,0,0,2011,0,0,0 +PROBABILITIES,0,0,2009,0,0,0,0 +PROBATE,0,0,0,2009,0,0,0 +PROBATION,0,0,0,2009,0,0,0 +PROBATIONERS,0,0,0,2009,0,0,0 +PROBLEMATICAL,2009,0,0,0,0,0,0 +PROFICIENTLY,0,2009,0,0,0,0,0 +PROGRESS,0,2009,0,0,0,0,0 +PROHIBIT,0,0,0,0,0,0,2009 +PROHIBITIONS,0,0,0,0,0,0,2009 +PROHIBITS,0,0,0,0,0,0,2009 +PROLONGED,2009,0,0,0,0,0,0 +PROMULGATED,0,0,0,2009,0,0,0 +PROMULGATIONS,0,0,0,2009,0,0,0 +PRORATA,0,0,0,2009,0,0,0 +PROSECUTES,2009,0,0,2009,0,0,0 +PROSECUTOR,0,0,0,2009,0,0,0 +PROSPERING,0,2009,0,0,0,0,0 +PROTEST,2009,0,0,0,0,0,0 +PROTESTING,2009,0,0,0,0,0,0 +PROTRACTED,2009,0,0,0,0,0,0 +PROVISOS,0,0,0,2009,0,0,0 +PROVOKING,2009,0,0,0,0,0,0 +PUNISHING,2009,0,0,0,0,0,0 +PURPORT,2009,0,0,0,0,0,0 +PURPORTS,2009,0,0,0,0,0,0 +QUESTIONED,2009,0,0,0,0,0,0 +QUITCLAIM,0,0,0,2009,0,0,0 +RACKETEERING,2009,0,0,0,0,0,0 +RANDOMIZES,0,0,2009,0,0,0,0 +RATA,0,0,0,2009,0,0,0 +RATIONALIZATIONS,2009,0,0,0,0,0,0 +RATIONALIZING,2009,0,0,0,0,0,0 +ABANDON,2009,0,0,0,0,0,0 +ABANDONMENTS,2009,0,0,0,0,0,0 +ABDICATING,2009,0,0,0,0,0,0 +ABERRATION,2009,0,0,0,0,0,0 +ABEYANCE,0,0,2009,0,0,0,0 +ABLE,0,2009,0,0,0,0,0 +ABNORMALLY,2009,0,0,0,0,0,0 +ABOLISHING,2009,0,0,0,0,0,0 +ABROGATES,2009,0,0,2009,0,0,0 +ABRUPT,2009,0,0,0,0,0,0 +ABSENCES,2009,0,0,0,0,0,0 +ABSOLVES,0,0,0,2009,0,0,0 +ABUSE,2009,0,0,0,0,0,0 +ABUSIVE,2009,0,0,0,0,0,0 +ACCESSIONS,0,0,0,2009,0,0,0 +ACCIDENTS,2009,0,0,0,0,0,0 +ACCOMPLISHES,0,2009,0,0,0,0,0 +ACCUSATION,2009,0,0,0,0,0,0 +ACCUSES,2009,0,0,0,0,0,0 +ACHIEVEMENT,0,2009,0,0,0,0,0 +ACQUIESCE,2009,0,0,0,0,0,0 +ACQUIREES,0,0,0,2011,0,0,0 +ACQUITTAL,2009,0,0,2009,0,0,0 +ACQUITTED,2009,0,0,2009,0,0,0 +ADJOURN,0,0,0,2009,0,0,0 +ADJOURNMENTS,0,0,0,2009,0,0,0 +ADJUDGES,0,0,0,2009,0,0,0 +ADJUDICATES,0,0,0,2009,0,0,0 +ADJUDICATIVE,0,0,0,2009,0,0,0 +ADMISSIBILITY,0,0,0,2009,0,0,0 +ADMISSIONS,0,0,0,2009,0,0,0 +ADULTERATION,2009,0,0,0,0,0,0 +ADVANCES,0,2009,0,0,0,0,0 +ADVANTAGEOUS,0,2009,0,0,0,0,0 +ADVERSARIES,2009,0,0,0,0,0,0 +ADVERSITIES,2009,0,0,0,0,0,0 +AFFIRMANCE,0,0,0,2011,0,0,0 +AFORESAID,0,0,0,2009,0,0,0 +AGAINST,2009,0,0,0,0,0,0 +AGGRAVATING,2009,0,0,0,0,0,0 +ALERTED,2009,0,0,0,0,0,0 +ALIENATES,2009,0,0,0,0,0,0 +ALLEGATION,2009,0,0,2009,0,0,0 +ALLEGEDLY,2009,0,0,2009,0,0,0 +ALLIANCES,0,2009,0,0,0,0,0 +ALWAYS,0,0,0,0,2009,0,0 +AMEND,0,0,0,2009,0,0,0 +AMENDING,0,0,0,2009,0,0,0 +ANNOY,2009,0,0,0,0,0,0 +ANNOYING,2009,0,0,0,0,0,0 +ANNULLING,2009,0,0,0,0,0,0 +ANOMALIES,2009,0,2009,0,0,0,0 +ANTECEDENT,0,0,0,2009,0,0,0 +ANTICIPATES,0,0,2009,0,0,0,0 +ANTICOMPETITIVE,2009,0,0,0,0,0,0 +APPARENT,0,0,2009,0,0,0,0 +APPEALED,0,0,0,2009,0,0,0 +APPEARED,0,0,2009,0,0,2009,0 +APPELLANTS,0,0,0,2009,0,0,0 +APPROXIMATE,0,0,2009,0,0,0,0 +APPROXIMATING,0,0,2009,0,0,0,0 +APPURTENANCES,0,0,0,2009,0,0,0 +ARBITRARILY,0,0,2009,0,0,0,0 +ARBITRATED,0,0,0,2009,0,0,0 +ARBITRATIONAL,0,0,0,2011,0,0,0 +ARBITRATORS,0,0,0,2009,0,0,0 +ARGUMENT,2009,0,0,0,0,0,0 +ARREARAGES,2009,0,0,2009,0,0,0 +ARRESTS,2009,0,0,0,0,0,0 +RETROCEDED,0,0,0,2011,0,0,0 +BOLSTERING,0,2009,0,0,0,0,0 +BOOM,0,2009,0,0,0,0,0 +BOTTLENECK,2009,0,0,0,0,0,0 +BOYCOTT,2009,0,0,0,0,0,0 +BREACH,2009,0,0,2009,0,0,0 +BREAK,2009,0,0,0,0,0,0 +BREAKDOWNS,2009,0,0,0,0,0,0 +BREAKTHROUGHS,0,2009,0,0,0,0,0 +BRIBERY,2009,0,0,0,0,0,0 +BRILLIANT,0,2009,0,0,0,0,0 +BURDENING,2009,0,0,0,0,0,0 +CALAMITIES,2009,0,0,0,0,0,0 +CANCELED,2009,0,0,0,0,0,0 +CANCELLED,2009,0,0,0,0,0,0 +CARELESSLY,2009,0,0,0,0,0,0 +CATASTROPHES,2009,0,0,0,0,0,0 +CAUTIONARY,2009,0,0,0,0,0,0 +CAUTIOUS,0,0,2009,0,0,0,0 +CEASED,2009,0,0,0,0,0,0 +CEDANTS,0,0,0,2012,0,0,0 +CENSURING,2009,0,0,0,0,0,0 +CHALLENGED,2009,0,0,0,0,0,0 +CHARITABLE,0,2009,0,0,0,0,0 +CIRCUMVENT,2009,0,0,0,0,0,0 +CIRCUMVENTIONS,2009,0,0,0,0,0,0 +SHORTAGE,2009,0,0,0,0,0,0 +SHRINKAGE,2009,0,0,0,0,0,0 +SHUTDOWNS,2009,0,0,0,0,0,0 +SLANDERED,2009,0,0,0,0,0,0 +REPUDIATES,2009,0,0,0,0,0,0 +REQUESTER,0,0,0,2011,0,0,0 +REQUIREMENT,0,0,0,0,0,0,2009 +REREGULATION,0,0,0,2011,0,0,0 +RESCINDS,0,0,0,2009,0,0,0 +RESIGNATION,2009,0,0,0,0,0,0 +RESIGNS,2009,0,0,0,0,0,0 +RESTATEMENT,2009,0,0,0,0,0,0 +RESTITUTIONARY,0,0,0,2011,0,0,0 +NOTARIAL,0,0,0,2009,0,0,0 +NOTARIZE,0,0,0,2009,0,0,0 +NOTWITHSTANDING,0,0,0,2009,0,0,0 +NULLIFICATION,2009,0,0,2009,0,0,0 +NULLIFY,2009,0,0,2009,0,0,0 +OBJECTED,2009,0,0,0,0,0,0 +OBJECTIONABLY,2009,0,0,0,0,0,0 +OBLIGATES,0,0,0,0,0,0,2009 +OBLIGATORY,0,0,0,0,0,0,2009 +OBLIGEES,0,0,0,2011,0,0,0 +OBSCENE,2009,0,0,0,0,0,0 +OBSTACLE,2009,0,0,0,0,0,0 +OBSTRUCTING,2009,0,0,0,0,0,0 +OFFENCE,2009,0,0,0,0,0,0 +OFFENDER,2009,0,0,0,0,0,0 +OFFENSE,0,0,0,2009,0,0,0 +OFFERORS,0,0,0,2011,0,0,0 +OMITS,2009,0,0,0,0,0,0 +OPPORTUNISTIC,2009,0,0,0,0,0,0 +OPPOSE,2009,0,0,0,0,0,0 +OPPOSITION,2009,0,0,0,0,0,0 +OPTIONEES,0,0,0,2009,0,0,0 +OUTDATED,2009,0,0,0,0,0,0 +OUTPERFORMING,0,2009,0,0,0,0,0 +OVERBUILD,2009,0,0,0,0,0,0 +OVERBURDEN,2009,0,0,0,0,0,0 +OVERCAPACITY,2009,0,0,0,0,0,0 +OVERCHARGING,2009,0,0,0,0,0,0 +OVERDUE,2009,0,0,0,0,0,0 +OVERESTIMATING,2009,0,0,0,0,0,0 +OVERLOADED,2009,0,0,0,0,0,0 +OVERLOOKED,2009,0,0,0,0,0,0 +OVERPAYMENT,2009,0,0,0,0,0,0 +OVERPRODUCING,2009,0,0,0,0,0,0 +OVERRULES,0,0,0,2009,0,0,0 +OVERRUNS,2009,0,0,0,0,0,0 +OVERSHADOWS,2009,0,0,0,0,0,0 +OVERSTATEMENTS,2009,0,0,0,0,0,0 +OVERSUPPLIES,2009,0,0,0,0,0,0 +OVERTURN,2009,0,0,0,0,0,0 +OVERVALUE,2009,0,0,0,0,0,0 +PANICS,2009,0,0,0,0,0,0 +NECESSITATE,0,0,0,0,0,0,2009 +NEGATIVE,2009,0,0,0,0,0,0 +NEGLECTED,2009,0,0,0,0,0,0 +NEGLIGENCE,2009,0,0,0,0,0,0 +NEVER,0,0,0,0,2009,0,0 +BARRIERS,2009,0,0,0,0,0,0 +BELIEVED,0,0,2009,0,0,0,0 +IDLING,2009,0,0,0,0,0,0 +IGNORING,2009,0,0,0,0,0,0 +ILLEGALITY,2009,0,0,0,0,0,0 +ILLICITLY,2009,0,0,0,0,0,0 +IMBALANCES,2009,0,0,0,0,0,0 +IMPAIR,2009,0,0,0,0,0,2011 +IMPAIRMENTS,2009,0,0,0,0,0,2011 +IMPEDE,2009,0,0,0,0,0,0 +IMPEDIMENTS,2009,0,0,0,0,0,0 +IMPERFECTION,2009,0,0,0,0,0,0 +IMPLEADED,0,0,0,2009,0,0,0 +IMPLICATING,2009,0,0,0,0,0,0 +IMPOSING,0,0,0,0,0,0,2009 +IMPOSSIBLE,2009,0,0,0,0,0,0 +IMPOUNDS,2009,0,0,0,0,0,0 +IMPRACTICALITY,2009,0,0,0,0,0,0 +IMPRESS,0,2009,0,0,0,0,0 +IMPRESSIVE,0,2009,0,0,0,0,0 +IMPROBABLE,0,0,2009,0,0,0,0 +IMPROPRIETY,2009,0,0,0,0,0,0 +IMPROVEMENTS,0,2009,0,0,0,0,0 +IMPRUDENTLY,2009,0,0,0,0,0,0 +INACCURACY,2009,0,0,0,0,0,0 +INACTIONS,2009,0,0,0,0,0,0 +INACTIVATING,2009,0,0,0,0,0,0 +INADEQUACIES,2009,0,0,0,0,0,0 +INADVERTENT,2009,0,0,0,0,0,0 +INAPPROPRIATE,2009,0,0,0,0,0,0 +INCAPABLE,2009,0,0,0,0,0,0 +INCARCERATED,2009,0,0,2009,0,0,0 +INCARCERATIONS,2009,0,0,2009,0,0,0 +INCIDENT,2009,0,0,0,0,0,0 +INCOMPATIBLE,2009,0,0,0,0,0,0 +INCOMPETENTLY,2009,0,0,0,0,0,0 +INCOMPLETENESS,2009,0,2009,0,0,0,0 +INCONSISTENT,2009,0,0,0,0,0,0 +INCONVENIENCE,2009,0,0,0,0,0,0 +INCORRECTLY,2009,0,0,0,0,0,0 +INDEBTED,0,0,0,0,0,0,2009 +INDEFEASIBLY,2009,0,0,0,0,0,0 +INDEMNIFIABLE,0,0,0,2009,0,0,0 +INDEMNIFIES,0,0,0,2009,0,0,0 +INDEMNITEES,0,0,0,2009,0,0,0 +INDEMNITY,0,0,0,2009,0,0,0 +INDICTABLE,2009,0,0,2009,0,0,0 +INDICTMENTS,2009,0,0,2009,0,0,0 +INEFFECTIVENESS,2009,0,0,0,0,0,0 +INEFFICIENTLY,2009,0,0,0,0,0,0 +INEQUITABLY,2009,0,0,0,0,0,0 +INEXACT,0,0,2009,0,0,0,0 +INFERIOR,2009,0,0,0,0,0,0 +INFORMATIVE,0,2009,0,0,0,0,0 +INFRINGED,2009,0,0,0,0,0,0 +INFRINGES,2009,0,0,0,0,0,0 +INHIBITED,2009,0,0,0,0,0,2009 +INJUNCTION,2009,0,0,2009,0,0,0 +INJURED,2009,0,0,0,0,0,0 +INJURIOUS,2009,0,0,0,0,0,0 +INNOVATES,0,2009,0,0,0,0,0 +INNOVATIVE,0,2009,0,0,0,0,0 +INORDINATE,2009,0,0,0,0,0,0 +INSENSITIVE,2009,0,0,0,0,0,0 +INSISTENCE,0,0,0,0,0,0,2009 +INSOLVENCIES,2009,0,0,0,0,0,0 +INSPIRATIONAL,0,2009,0,0,0,0,0 +INSUFFICIENCY,2009,0,0,0,0,0,0 +INSURRECTIONS,2009,0,0,0,0,0,0 +INTENTIONAL,2009,0,0,0,0,0,0 +INTERFERENCES,2009,0,0,0,0,0,0 +INTERMITTENT,2009,0,0,0,0,0,0 +INTERPOSED,0,0,0,2009,0,0,0 +INTERPOSITIONS,0,0,0,2009,0,0,0 +INTERROGATING,0,0,0,2009,0,0,0 +INTERROGATORIES,0,0,0,2009,0,0,0 +INTERRUPTED,2009,0,0,0,0,0,0 +INTERRUPTS,2009,0,0,0,0,0,0 +STABILITY,0,2009,0,0,0,0,0 +STABILIZED,0,2009,0,0,0,0,0 +STAGGERING,2009,0,0,0,0,0,0 +STAGNATES,2009,0,0,0,0,0,0 +STANDSTILLS,2009,0,0,0,0,0,0 +STATUTORY,0,0,0,2009,0,0,0 +STIPULATING,0,0,0,0,0,0,2009 +STOPPAGE,2009,0,0,0,0,0,0 +STOPS,2009,0,0,0,0,0,0 +STRAINS,2009,0,0,0,0,0,0 +STRENGTHENING,0,2009,0,0,0,0,0 +STRESSED,2009,0,0,0,0,0,0 +STRICT,0,0,0,0,0,0,2009 +STRINGENT,2009,0,0,0,0,0,0 +STRONGLY,0,0,0,0,2009,0,0 +SUBJECTED,2009,0,0,0,0,0,0 +SUBLEASEHOLD,0,0,0,2011,0,0,0 +SUBPARAGRAPH,0,0,0,2009,0,0,0 +SUBPOENAS,2009,0,0,2009,0,0,0 +SUBTRUST,0,0,0,2011,0,0,0 +SUCCEEDING,0,2009,0,0,0,0,0 +SUCCESSFUL,0,2009,0,0,0,0,0 +SUE,2009,0,0,2009,0,0,0 +SUFFERED,2009,0,0,0,0,0,0 +SUGGESTED,0,0,2009,0,0,0,0 +SUMMONED,2009,0,0,2009,0,0,0 +SUPERIOR,0,2009,0,0,0,0,0 +SUPERSEDES,0,0,0,2009,0,0,0 +SURPASS,0,2009,0,0,0,0,0 +SUSCEPTIBILITY,2009,0,2009,0,0,0,0 +SUSPECTS,2009,0,0,0,0,0,0 +SUSPENDS,2009,0,0,0,0,0,0 +SUSPICIONS,2009,0,0,0,0,0,0 +REVOCABILITY,0,0,0,2011,0,0,0 +REVOKED,2009,0,0,0,0,0,0 +REVOLUTIONIZED,0,2009,0,0,0,0,0 +REWARDED,0,2009,0,0,0,0,0 +RIDICULED,2009,0,0,0,0,0,0 +RISKED,0,0,2009,0,0,0,0 +RISKING,0,0,2009,0,0,0,0 +RULING,0,0,0,2009,0,0,0 +SACRIFICE,2009,0,0,0,0,0,0 +SACRIFICING,2009,0,0,0,0,0,0 +SATISFIED,0,2009,0,0,0,0,0 +SCANDALOUS,2009,0,0,0,0,0,0 +SCRUTINIZES,2009,0,0,0,0,0,0 +SEEMS,0,0,2009,0,0,0,0 +SEIZING,2009,0,0,0,0,0,0 +SENTENCING,2009,0,0,2009,0,0,0 +SERIOUSNESS,2009,0,0,0,0,0,0 +SETTLEMENTS,0,0,0,2009,0,0,0 +SEVERALLY,0,0,0,2009,0,0,0 +SEVERED,2009,0,0,0,0,0,0 +ENTHUSIASTICALLY,0,2009,0,0,0,0,0 +ERODED,2009,0,0,0,0,0,0 +ERRATIC,2009,0,0,0,0,0,0 +ERRONEOUS,2009,0,0,0,0,0,0 +ERRS,2009,0,0,0,0,0,0 +ESCALATING,2009,0,0,0,0,0,0 +ESCROW,0,0,0,0,0,0,2009 +ESTOPPEL,0,0,0,2011,0,0,0 +EVADING,2009,0,0,0,0,0,0 +EVICT,2009,0,0,0,0,0,0 +EVICTIONS,2009,0,0,0,0,0,0 +EXACERBATE,2009,0,0,0,0,0,0 +EXACERBATION,2009,0,0,0,0,0,0 +EXAGGERATES,2009,0,0,0,0,0,0 +EXCEEDANCES,0,0,0,2011,0,0,0 +EXCELLING,0,2009,0,0,0,0,0 +EXCESSIVE,2009,0,0,0,0,0,0 +EXCITEMENT,0,2009,0,0,0,0,0 +EXCLUSIVENESS,0,2009,0,0,0,0,0 +EXCULPATED,2009,0,0,2009,0,0,0 +EXCULPATIONS,2009,0,0,2009,0,0,0 +EXECUTORY,0,0,0,2009,0,0,0 +EXEMPLARY,0,2009,0,0,0,0,0 +EXONERATING,2009,0,0,0,0,0,0 +EXPLOITATION,2009,0,0,0,0,0,0 +EXPLOITING,2009,0,0,0,0,0,0 +EXPOSES,2009,0,0,0,0,0,0 +EXPROPRIATE,2009,0,0,0,0,0,0 +EXPROPRIATION,2009,0,0,0,0,0,0 +EXTENUATING,2009,0,0,0,0,0,0 +SOLVING,0,2009,0,0,0,0,0 +SOMEWHERE,0,0,2009,0,0,0,0 +LOSE,2009,0,0,0,0,0,0 +LOSSES,2009,0,0,0,0,0,0 +LUCRATIVE,0,2009,0,0,0,0,0 +MALFUNCTION,2009,0,0,0,0,0,0 +MALICE,2009,0,0,0,0,0,0 +MANDAMUS,0,0,0,2009,0,0,0 +MANDATING,0,0,0,0,0,0,2009 +MANIPULATED,2009,0,0,0,0,0,0 +MANIPULATIONS,2009,0,0,0,0,0,0 +MAY,0,0,2009,0,0,2009,0 +MEDIATES,0,0,0,2009,0,0,0 +MEDIATOR,0,0,0,2009,0,0,0 +MISAPPLICATION,2009,0,0,0,0,0,0 +MISAPPLY,2009,0,0,0,0,0,0 +MISAPPROPRIATES,2009,0,0,0,0,0,0 +MISBRANDED,2009,0,0,0,0,0,0 +MISCALCULATING,2009,0,0,0,0,0,0 +MISCHIEF,2009,0,0,0,0,0,0 +MISCLASSIFY,2014,0,0,0,0,0,0 +MISDEMEANOR,2009,0,0,2009,0,0,0 +MISHANDLE,2009,0,0,0,0,0,0 +MISINFORM,2009,0,0,0,0,0,0 +MISINFORMS,2009,0,0,0,0,0,0 +MISINTERPRETED,2009,0,0,0,0,0,0 +MISJUDGED,2009,0,0,0,0,0,0 +MISJUDGMENTS,2009,0,0,0,0,0,0 +MISLABELLED,2009,0,0,0,0,0,0 +MISLEADINGLY,2009,0,0,0,0,0,0 +MISMANAGED,2009,0,0,0,0,0,0 +MISMATCH,2009,0,0,0,0,0,0 +MISPLACED,2009,0,0,0,0,0,0 +MISREPRESENT,2009,0,0,0,0,0,0 +MISREPRESENTING,2009,0,0,0,0,0,0 +MISSES,2009,0,0,0,0,0,0 +WORRY,2009,0,0,0,0,0,0 +WORSENED,2009,0,0,0,0,0,0 +WORTHLESS,2009,0,0,0,0,0,0 +TOLERATE,2009,0,0,0,0,0,0 +TOLERATION,2009,0,0,0,0,0,0 +TORTS,0,0,0,2009,0,0,0 +FICTITIOUS,2009,0,0,0,0,0,0 +FIRED,2009,0,0,0,0,0,0 +NONATTAINMENT,2009,0,0,0,0,0,0 +NONCOMPETITIVE,2009,0,0,0,0,0,0 +NONCOMPLYING,2009,0,0,0,0,0,0 +NONCONTINGENT,0,0,0,2011,0,0,0 +NONDISCLOSURE,2009,0,0,0,0,0,0 +NONFORFEITABLE,0,0,0,2009,0,0,0 +DISCLAIM,2009,0,0,0,0,0,0 +DISCLAIMING,2009,0,0,0,0,0,0 +DISCLOSES,2009,0,0,0,0,0,0 +DISCONTINUATION,2009,0,0,0,0,0,0 +DISCONTINUES,2009,0,0,0,0,0,0 +DISCOURAGES,2009,0,0,0,0,0,0 +DISCREDITING,2009,0,0,0,0,0,0 +SPECTACULAR,0,2009,0,0,0,0,0 +SPECULATES,0,0,2009,0,0,0,0 +SPECULATIVE,0,0,2009,0,0,0,0 +TRAGEDIES,2009,0,0,0,0,0,0 +TRANSFEROR,0,0,0,2009,0,0,0 +LEGISLATES,0,0,0,2009,0,0,0 +LEGISLATIVE,0,0,0,2009,0,0,0 +LEGISLATURE,0,0,0,2009,0,0,0 +LIBELOUS,0,0,0,2009,0,0,0 +LIENHOLDERS,0,0,0,2011,0,0,0 +COMPULSORY,0,0,0,0,0,0,2009 +CONCEDED,2009,0,0,0,0,0,0 +CONCEIVABLY,0,0,2009,0,0,0,0 +CONCILIATING,2009,0,0,0,0,0,0 +CONCLUSIVELY,0,2009,0,0,0,0,0 +CONDEMNED,2009,0,0,0,0,0,0 +CONDITIONAL,0,0,2009,0,0,0,0 +CONDUCIVE,0,2009,0,0,0,0,0 +CONFESSING,2009,0,0,0,0,0,0 +CONFINED,2009,0,0,0,0,0,2009 +CONFINING,2009,0,0,0,0,0,2009 +CONFISCATING,2009,0,0,0,0,0,0 +CONFLICT,2009,0,0,0,0,0,0 +CONFRONT,2009,0,0,0,0,0,0 +CONFRONTED,2009,0,0,0,0,0,0 +CONFUSED,2009,0,0,0,0,0,0 +CONFUSION,2009,0,2009,0,0,0,0 +CONSENTS,0,0,0,2009,0,0,0 +CONSPIRATOR,2009,0,0,0,0,0,0 +CONSPIRED,2009,0,0,0,0,0,0 +CONSTITUTIONAL,0,0,0,2009,0,0,0 +CONSTITUTIVE,0,0,0,2009,0,0,0 +CONSTRAINS,0,0,0,0,0,0,2009 +CONSTRUCTIVELY,0,2009,0,0,0,0,0 +CONSTRUING,0,0,0,2012,0,0,0 +CONTENDING,2009,0,0,0,0,0,0 +CONTENTIOUS,2009,0,0,0,0,0,0 +CONTESTED,2009,0,0,0,0,0,0 +CONTINGENT,0,0,2009,0,0,0,0 +CONTRACTED,0,0,0,2009,0,0,0 +CONTRACTILE,0,0,0,2009,0,0,0 +CONTRACTS,0,0,0,2009,0,0,0 +CONTRADICTED,2009,0,0,0,0,0,0 +CONTRADICTORY,2009,0,0,0,0,0,0 +CONTRAVENED,0,0,0,2009,0,0,0 +CONTRAVENTIONS,0,0,0,2009,0,0,0 +CONTROVERT,0,0,0,2009,0,0,0 +CONVEYANCE,0,0,0,2009,0,0,0 +CONVICTING,2009,0,0,2009,0,0,0 +CORRECTING,2009,0,0,0,0,0,0 +CORRUPT,2009,0,0,0,0,0,0 +CORRUPTIONS,2009,0,0,0,0,0,0 +COTERMINOUS,0,0,0,2009,0,0,0 +COUNSELLED,0,0,0,2009,0,0,0 +COUNTERCLAIMING,2009,0,0,0,0,0,0 +COUNTERFEITER,2009,0,0,0,0,0,0 +COUNTERMEASURE,2009,0,0,0,0,0,0 +COUNTERSUIT,0,0,0,2011,0,0,0 +COURTROOM,0,0,0,2009,0,0,0 +COVENANTING,0,0,0,0,0,0,2011 +CREATIVENESS,0,2009,0,0,0,0,0 +CRIMINAL,2009,0,0,2009,0,0,0 +CRIMINALLY,2009,0,0,2009,0,0,0 +CRITICAL,-2020,0,0,0,0,0,0 +CRITICIZE,2009,0,0,0,0,0,0 +CROSSCLAIM,0,0,0,2011,0,0,0 +CRUCIAL,2009,0,0,0,0,0,0 +CULPABLY,2009,0,0,0,0,0,0 +CURTAILING,2009,0,0,0,0,0,0 +CUT,2009,0,0,0,0,0,0 +CYBERATTACKS,2014,0,0,0,0,0,0 +CYBERCRIMINAL,2014,0,0,0,0,0,0 +TAINT,2009,0,0,0,0,0,0 +TAMPERED,2009,0,0,0,0,0,0 +TENTATIVE,0,0,2009,0,0,0,0 +TERMINATED,2009,0,0,0,0,0,0 +TERMINATIONS,2009,0,0,0,0,0,0 +TESTIFYING,2009,0,0,2009,0,0,0 +THENCEFORWARD,0,0,0,2009,0,0,0 +THEREIN,0,0,0,2009,0,0,0 +THEREOVER,0,0,0,2011,0,0,0 +THEREUNDER,0,0,0,2009,0,0,0 +THREAT,2009,0,0,0,0,0,0 +THREATENS,2009,0,0,0,0,0,0 +FACTO,0,0,0,2011,0,0,0 +FAILINGS,2009,0,0,0,0,0,0 +FALLOUT,2009,0,0,0,0,0,0 +FALSIFICATIONS,2009,0,0,0,0,0,0 +FALSIFYING,2009,0,0,0,0,0,0 +FATALITY,2009,0,0,0,0,0,0 +FAULTS,2009,0,0,0,0,0,0 +FAVORED,0,2009,0,0,0,0,0 +FEAR,2009,0,0,0,0,0,0 +FELONY,2009,0,0,2009,0,0,0 +DAMAGING,2009,0,0,0,0,0,0 +DANGEROUS,2009,0,0,0,0,0,0 +DEADLOCKED,2009,0,0,0,0,0,0 +DEADWEIGHTS,2009,0,0,0,0,0,0 +DECEASED,2009,0,0,0,0,0,0 +DECEITFUL,2009,0,0,0,0,0,0 +DECEIVES,2009,0,0,0,0,0,0 +DECEPTIVE,2009,0,0,0,0,0,0 +DECLINED,2009,0,0,0,0,0,0 +DECREED,0,0,0,2009,0,0,0 +DEFACED,2009,0,0,0,0,0,0 +DEFAMATION,2009,0,0,0,0,0,0 +DEFAMED,2009,0,0,0,0,0,0 +DEFAULTED,2009,0,0,0,0,0,0 +DEFEASANCES,0,0,0,2011,0,0,0 +DEFEASES,0,0,0,2014,0,0,0 +DEFEATING,2009,0,0,0,0,0,0 +DEFECTIVELY,0,0,0,2009,0,0,0 +DEFENDANT,2009,0,0,2009,0,0,0 +DEFENDS,2009,0,0,0,0,0,0 +DEFICIENCIES,2009,0,0,0,0,0,0 +DEFICITS,2009,0,0,0,0,0,0 +DEFRAUDED,2009,0,0,0,0,0,0 +DEGRADATION,2009,0,0,0,0,0,0 +DEGRADES,2009,0,0,0,0,0,0 +DELAYING,2009,0,0,0,0,0,0 +DELEGATEE,0,0,0,2011,0,0,0 +DELIBERATED,2009,0,0,0,0,0,0 +DELIGHTFUL,0,2009,0,0,0,0,0 +DELINQUENCIES,2009,0,0,0,0,0,0 +DELINQUENTS,2009,0,0,0,0,0,0 +DELISTS,2011,0,0,0,0,0,0 +DEMISING,2009,0,0,0,0,0,0 +DEMOLISHING,2009,0,0,0,0,0,0 +DEMOTED,2009,0,0,0,0,0,0 +DEMOTIONS,2009,0,0,0,0,0,0 +DEMURRING,0,0,0,2009,0,0,0 +DENIED,2009,0,0,0,0,0,0 +DENIGRATES,2009,0,0,0,0,0,0 +DENYING,2009,0,0,0,0,0,0 +DEPENDANCE,0,0,0,0,0,0,2011 +DEPENDENCE,0,0,2009,0,0,0,0 +DEPENDING,0,0,2009,0,0,2009,2011 +DEPLETES,2009,0,0,0,0,0,0 +DEPOSE,0,0,0,2009,0,0,0 +DEPOSITION,0,0,0,2009,0,0,0 +INVALIDATE,2009,0,0,0,0,0,0 +INVALIDATION,2009,0,0,0,0,0,0 +INVENTING,0,2009,0,0,0,0,0 +INVENTIVENESS,0,2009,0,0,0,0,0 +INVESTIGATED,2009,0,0,0,0,0,0 +INVESTIGATIONS,2009,0,0,0,0,0,0 +IRRECONCILABLY,2009,0,0,0,0,0,0 +IRREGULARITIES,2009,0,0,0,0,0,0 +IRREPARABLY,2009,0,0,0,0,0,0 +IRREVOCABLY,0,0,0,2009,0,0,2009 +JUDICIAL,0,0,0,2009,0,0,0 +JURIES,0,0,0,2009,0,0,0 +JURISDICTIONALLY,0,0,0,2011,0,0,0 +JURISTS,0,0,0,2009,0,0,0 +JURYMAN,0,0,0,2009,0,0,0 +KICKBACK,2009,0,0,0,0,0,0 +DIMINISH,2009,0,0,0,0,0,0 +DIMINUTION,2009,0,0,0,0,0,0 +DISADVANTAGED,2009,0,0,0,0,0,0 +DISAFFIRM,0,0,0,2009,0,0,0 +DISAGREE,2009,0,0,0,0,0,0 +DISAGREEMENT,2009,0,0,0,0,0,0 +DISALLOWANCE,2009,0,0,0,0,0,0 +DISALLOWS,2009,0,0,0,0,0,0 +DISAPPEARED,2009,0,0,0,0,0,0 +DISAPPOINTED,2009,0,0,0,0,0,0 +DISAPPOINTMENTS,2009,0,0,0,0,0,0 +DISAPPROVE,2009,0,0,0,0,0,0 +DISASSOCIATES,2009,0,0,0,0,0,0 +DISASTER,2009,0,0,0,0,0,0 +DISAVOW,2009,0,0,0,0,0,0 +DISAVOWS,2009,0,0,0,0,0,0 +GRATUITOUS,2009,0,0,0,0,0,0 +GREATEST,0,2009,0,0,0,0,0 +GRIEVANCES,2009,0,0,0,0,0,0 +HALT,2009,0,0,0,0,0,0 +HAMPERING,2009,0,0,0,0,0,0 +HAPPINESS,0,2009,0,0,0,0,0 +HARASSING,2009,0,0,0,0,0,0 +HARM,2009,0,0,0,0,0,0 +HARMING,2009,0,0,0,0,0,0 +HARSHEST,2009,0,0,0,0,0,0 +HAZARDOUS,2009,0,0,0,0,0,0 +NONINFRINGEMENT,0,0,0,2011,0,0,0 +NONJURISDICTIONAL,0,0,0,2011,0,0,0 +NONPERFORMANCES,2009,0,0,0,0,0,0 +HURTING,2009,0,0,0,0,0,0 +DISFAVORING,2009,0,0,0,0,0,0 +DISGORGEMENT,2009,0,0,0,0,0,0 +COMPLICATE,2009,0,0,0,0,0,0 +COMPLICATION,2009,0,0,0,0,0,0 +COMPLIMENTED,0,2009,0,0,0,0,0 +MISSTATEMENT,2009,0,0,0,0,0,0 +MISSTEP,2009,0,0,0,0,0,0 +MISTAKENLY,2009,0,0,0,0,0,0 +MISTRIALS,2009,0,0,2009,0,0,0 +MISUNDERSTOOD,2009,0,0,0,0,0,0 +MISUSING,2009,0,0,0,0,0,0 +MONOPOLIZE,2009,0,0,0,0,0,0 +MONOPOLY,2009,0,0,0,0,0,0 +MOREOVER,0,0,0,2009,0,0,0 +MUST,0,0,0,0,2009,0,0 +SLIPPAGE,2009,0,0,0,0,0,0 +SLOWDOWNS,2009,0,0,0,0,0,0 +SLOWING,2009,0,0,0,0,0,0 +SLUGGISHLY,2009,0,0,0,0,0,0 +SMOOTHLY,0,2009,0,0,0,0,0 +DEPRESSED,2009,0,0,0,0,0,0 +DEPRIVE,2009,0,0,0,0,0,0 +DERELICT,2009,0,0,0,0,0,0 +DEROGATES,0,0,0,2009,0,0,0 +DEROGATORY,2009,0,0,0,0,0,0 +DESIST,0,0,0,2009,0,0,0 +DESTABILIZED,2009,0,0,0,0,0,0 +DESTROYED,2009,0,0,0,0,0,0 +DESTRUCTIVE,2009,0,0,0,0,0,0 +DETENTION,2009,0,0,0,0,0,0 +DETERIORATED,2009,0,0,0,0,0,0 +DETERIORATIONS,2009,0,0,0,0,0,0 +DETERRENT,2009,0,0,0,0,0,0 +DETRACT,2009,0,0,0,0,0,0 +DETRIMENTAL,2009,0,0,0,0,0,0 +DEVALUED,2009,0,0,0,0,0,0 +DEVASTATED,2009,0,0,0,0,0,0 +DEVIATED,2009,0,2009,0,0,0,0 +DEVIATIONS,2009,0,2009,0,0,0,0 +DEVOLVES,2009,0,0,0,0,0,0 +DICTATES,0,0,0,0,0,0,2009 +DIFFERING,0,0,2009,0,0,0,0 +DIFFICULTLY,2009,0,0,0,0,0,0 +ASCENDANT,0,0,0,2009,0,0,0 +ASSAULTING,2009,0,0,0,0,0,0 +ASSIGNATION,0,0,0,2009,0,0,0 +ASSUMED,0,0,2009,0,0,0,0 +ASSUMPTIONS,0,0,2009,0,0,0,0 +ASSURING,0,2009,0,0,0,0,0 +ATTAINMENT,0,2009,0,0,0,0,0 +ATTESTATION,0,0,0,2009,0,0,0 +ATTORN,0,0,0,2011,0,0,0 +ATTORNS,0,0,0,2011,0,0,0 +AVERSELY,2011,0,0,0,0,0,0 +BAILED,0,0,0,2009,0,0,0 +BAILIFFS,0,0,0,2009,0,0,0 +BALKED,2009,0,0,0,0,0,0 +BANKRUPTED,2009,0,0,0,0,0,0 +CLAIMANT,0,0,0,2009,0,0,0 +CLAIMS,2009,0,0,2009,0,0,0 +CLAWBACKS,0,0,0,2014,0,0,0 +CLOSEOUTS,2009,0,0,0,0,0,0 +CLOSURES,2009,0,0,0,0,0,0 +CODICILS,0,0,0,2009,0,0,0 +CODIFIES,0,0,0,2009,0,0,0 +COERCED,2009,0,0,0,0,0,0 +COERCIVE,2009,0,0,0,0,0,0 +DISINCENTIVES,2009,0,0,0,0,0,0 +COLLABORATE,0,2009,0,0,0,0,0 +COLLABORATION,0,2009,0,0,0,0,0 +COLLABORATORS,0,2009,0,0,0,0,0 +COLLAPSING,2009,0,0,0,0,0,0 +COLLUDED,2009,0,0,0,0,0,0 +COLLUSIONS,2009,0,0,0,0,0,0 +POSITIVE,0,2009,0,0,0,0,0 +POSSIBILITY,0,0,2009,0,0,0,0 +POSTCLOSURE,0,0,0,2011,0,0,0 +POSTPONED,2009,0,0,0,0,0,0 +POSTPONING,2009,0,0,0,0,0,0 +WRIT,0,0,0,2009,0,0,0 +WRITEOFFS,2009,0,0,0,0,0,0 +WRONGDOINGS,2009,0,0,0,0,0,0 +HONORING,0,2009,0,0,0,0,0 +REASSESSES,0,0,2009,0,0,0,0 +REASSIGN,2009,0,0,0,0,0,0 +REASSIGNMENTS,2009,0,0,0,0,0,0 +REBOUNDING,0,2009,0,0,0,0,0 +REBUTTABLY,0,0,0,2011,0,0,0 +REBUTTING,0,0,0,2009,0,0,0 +RECALCULATING,0,0,2009,0,0,0,0 +RECALLED,2009,0,0,0,0,0,0 +RECESSION,2009,0,0,0,0,0,0 +RECKLESSLY,2009,0,0,0,0,0,0 +RECONSIDERING,0,0,2009,0,0,0,0 +RECOUPMENT,0,0,0,2009,0,0,0 +RECTIFICATION,0,0,0,2009,0,0,0 +RECUSED,0,0,0,2011,0,0,0 +REDACTED,2009,0,0,2009,0,0,0 +REDEFAULT,2014,0,0,0,0,0,0 +REDRESSED,2009,0,0,0,0,0,0 +REEXAMINE,0,0,2009,0,0,0,0 +REFERENDUMS,0,0,0,2009,0,0,0 +REFILING,0,0,0,2009,0,0,0 +REFUSAL,2009,0,0,0,0,0,0 +REFUSES,2009,0,0,0,0,0,0 +REGAINING,0,2009,0,0,0,0,0 +REGULATING,0,0,0,2009,0,0,0 +REGULATOR,0,0,0,2009,0,0,0 +REHEARD,0,0,0,2009,0,0,0 +VARIABLES,0,0,2009,0,0,0,0 +VARIANT,0,0,2009,0,0,0,0 +VARIED,0,0,2009,0,0,0,0 +VENDEE,0,0,0,2011,0,0,0 +VERSATILE,0,2009,0,0,0,0,0 +VIBRANCY,0,2009,0,0,0,0,0 +VIOLATED,2009,0,0,0,0,0,0 +VIOLATIONS,2009,0,0,0,0,0,0 +VIOLENCE,2009,0,0,0,0,0,0 +VITIATED,2009,0,0,0,0,0,0 +VOIDABLE,0,0,0,2009,0,0,0 +VOLATILITIES,0,0,2009,0,0,0,0 +VULNERABLE,2009,0,0,0,0,0,0 +WARNING,2009,0,0,0,0,0,0 +WARRANTOR,0,0,0,2011,0,0,0 +WEAK,2009,0,0,0,0,0,0 +WEAKENS,2009,0,0,0,0,0,0 +WEAKNESS,2009,0,0,0,0,0,0 +DISHONOR,2009,0,0,0,0,0,0 +DISHONORING,2009,0,0,0,0,0,0 +DISINTERESTED,2009,0,0,0,0,0,0 +DISLOYALLY,2009,0,0,0,0,0,0 +DISMISS,2009,0,0,0,0,0,0 +DISMISSES,2009,0,0,0,0,0,0 +DISPARAGED,2009,0,0,0,0,0,0 +DISPARAGING,2009,0,0,0,0,0,0 +DISPLACE,2009,0,0,0,0,0,0 +DISPLACES,2009,0,0,0,0,0,0 +DISPOSSESS,2009,0,0,0,0,0,0 +DISPOSSESSION,0,0,0,2009,0,0,0 +DISPROPORTIONATE,2009,0,0,0,0,0,0 +DISPUTES,2009,0,0,0,0,0,0 +DISQUALIFIED,2009,0,0,0,0,0,0 +DISREGARD,2009,0,0,0,0,0,0 +DISREPUTABLE,2009,0,0,0,0,0,0 +DISRUPTING,2009,0,0,0,0,0,0 +DISRUPTS,2009,0,0,0,0,0,0 +DISSENTED,2009,0,0,0,0,0,0 +DISSENTS,2009,0,0,0,0,0,0 +DISSOLUTIONS,2009,0,0,0,0,0,0 +DISTINCTIVELY,0,2009,0,0,0,0,0 +DISTORTING,2009,0,0,0,0,0,0 +DISTRACT,2009,0,0,0,0,0,0 +DISTRACTIONS,2009,0,0,0,0,0,0 +DISTRESSED,2009,0,0,0,0,0,0 +DISTURBANCE,2009,0,0,0,0,0,0 +DISTURBS,2009,0,0,0,0,0,0 +DIVERTING,2009,0,0,0,0,0,0 +DIVESTING,2009,0,0,0,0,0,0 +DIVESTMENTS,2009,0,0,0,0,0,0 +DIVULGE,2009,0,0,0,0,0,0 +DOCKET,0,0,0,2009,0,0,0 +DONEES,0,0,0,2011,0,0,0 +DOUBTS,2009,0,2009,0,0,0,0 +DOWNGRADING,2009,0,0,0,0,0,0 +DOWNSIZING,2009,0,0,0,0,0,0 +DOWNTURN,2009,0,0,0,0,0,0 +DRAG,2009,0,0,0,0,0,0 +DRAWBACKS,2009,0,0,0,0,0,0 +DROUGHTS,2009,0,0,0,0,0,0 +DYSFUNCTIONAL,2009,0,0,0,0,0,0 +EARMARKING,0,0,0,0,0,0,2009 +EASING,2009,0,0,0,0,0,0 +EFFICIENCY,0,2009,0,0,0,0,0 +EGREGIOUSLY,2009,0,0,0,0,0,0 +EMBARGOES,2009,0,0,0,0,0,0 +EMBARRASSES,2009,0,0,0,0,0,0 +EMBEZZLE,2009,0,0,0,0,0,0 +EMBEZZLER,2009,0,0,0,0,0,0 +EMPOWERED,0,2009,0,0,0,0,0 +ENABLED,0,2009,0,0,0,0,0 +ENCOURAGEMENT,0,2009,0,0,0,0,0 +ENCROACHED,2009,0,0,0,0,0,0 +ENCROACHMENTS,2009,0,0,0,0,0,0 +ENCUMBERS,2009,0,0,2009,0,0,2009 +ENCUMBRANCES,2009,0,0,2009,0,0,2009 +ENDANGERMENT,2009,0,0,0,0,0,0 +ENFORCEABLE,0,0,0,2009,0,0,0 +ENHANCEMENT,0,2009,0,0,0,0,0 +ENJOIN,2009,0,0,0,0,0,0 +ENJOY,0,2009,0,0,0,0,0 +ENJOYING,0,2009,0,0,0,0,0 +ENTAILED,0,0,0,0,0,0,2009 +PENALIZE,2009,0,0,0,0,0,0 +PENALTIES,2009,0,0,0,0,0,0 +PERFECTED,0,2009,0,0,0,0,0 +PERIL,2009,0,0,0,0,0,0 +PERMISSION,0,0,0,0,0,0,2009 +PERMITTEES,0,0,0,2011,0,0,0 +PERPETRATES,2009,0,0,2009,0,0,0 +PERSISTED,2009,0,0,0,0,0,0 +PERSISTING,2009,0,0,0,0,0,0 +PERVASIVELY,2009,0,0,0,0,0,0 +PETITIONER,0,0,0,2009,0,0,0 +PETTY,2009,0,0,0,0,0,0 +HEREAFTER,0,0,0,2009,0,0,0 +HEREFORE,0,0,0,2014,0,0,0 +HEREINAFTER,0,0,0,2009,0,0,0 +HEREON,0,0,0,2009,0,0,0 +HEREUNTO,0,0,0,2009,0,0,0 +HIDDEN,0,0,2009,0,0,0,0 +HINDERING,2009,0,0,0,0,0,0 +HINGES,0,0,2009,0,0,0,0 +WHEREABOUTS,0,0,0,2009,0,0,0 +WHEREFORE,0,0,0,2009,0,0,0 +WHERETO,0,0,0,2009,0,0,0 +WHISTLEBLOWERS,0,0,0,2011,0,0,0 +WILFUL,0,0,0,2009,0,0,0 +WILLFULNESS,0,0,0,2009,0,0,0 +WINNING,0,2009,0,0,0,0,0 +FLUCTUATE,0,0,2009,0,0,0,0 +FLUCTUATION,0,0,2009,0,0,0,0 +FORBEARANCE,0,0,0,2009,0,0,0 +FORBID,2009,0,0,0,0,0,2009 +FORCE,-2020,0,0,0,0,0,0 +FOREBEARANCE,0,0,0,2011,0,0,0 +FORECLOSES,2009,0,0,0,0,0,0 +FOREGO,2009,0,0,0,0,0,0 +FORESTALLED,2009,0,0,0,0,0,0 +FORFEITABILITY,0,0,0,2011,0,0,0 +FORFEITS,2009,0,0,0,0,0,0 +FORGERY,2009,0,0,0,0,0,0 +FRAUDS,2009,0,0,0,0,0,0 +FRIENDLY,0,2009,0,0,0,0,0 +FRUSTRATED,2009,0,0,0,0,0,0 +FRUSTRATION,2009,0,0,0,0,0,0 +FURTHERANCE,0,0,0,2009,0,0,0 +GAINS,0,2009,0,0,0,0,0 +DISGORGEMENTS,2009,0,0,0,0,0,0 +DISGRACEFUL,2009,0,0,0,0,0,0 +LITIGATE,2009,0,0,2009,0,0,0 +LITIGATION,2009,0,0,2009,0,0,0 +LITIGIOUS,0,0,0,2009,0,0,0 +LACKED,2009,0,0,0,0,0,0 +LAG,2009,0,0,0,0,0,0 +LAPSE,2009,0,0,0,0,0,0 +LATE,-2020,0,0,0,0,0,0 +LAWFULLY,0,0,0,2009,0,0,0 +LAWS,0,0,0,2009,0,0,0 +LAWYERS,0,0,0,2009,0,0,0 +LEADING,0,2009,0,0,0,0,0 +LEGALIZATION,0,0,0,2009,0,0,0 +LEGALIZES,0,0,0,2009,0,0,0 +LEGATEE,0,0,0,2009,0,0,0 +FLAWED,2009,0,0,0,0,0,0 +NONRECOVERABLE,2009,0,0,0,0,0,0 +LIQUIDATES,2009,0,0,0,0,0,0 +LIQUIDATOR,2009,0,0,0,0,0,0 +BENEFICIATION,0,0,0,2011,0,0,0 +BENEFITTED,0,2009,0,0,0,0,0 +POOR,2009,0,0,0,0,0,0 +REINTERPRETATION,0,0,2009,0,0,0,0 +REINTERPRETS,0,0,2009,0,0,0,0 +REJECTION,2009,0,0,0,0,0,0 +RELINQUISH,2009,0,0,0,0,0,0 +RELINQUISHMENT,2009,0,0,0,0,0,0 +REMAND,0,0,0,2009,0,0,0 +REMEDIATE,0,0,0,2009,0,0,0 +REMEDIATIONS,0,0,0,2009,0,0,0 +RENEGOTIATED,2009,0,0,0,0,0,0 +RENEGOTIATIONS,2009,0,0,0,0,0,0 +RENOUNCEMENTS,2009,0,0,0,0,0,0 +REPARATIONS,2009,0,0,0,0,0,0 +REPOSSESSES,2009,0,0,0,0,0,0 +REPRORATED,0,0,0,2018,0,0,0 +TROUBLED,2009,0,0,0,0,0,0 +UNABLE,2009,0,0,0,0,0,0 +UNAMBIGUOUSLY,0,0,0,0,2009,0,0 +UNAPPEALED,0,0,0,2011,0,0,0 +UNAVAILABILITY,2009,0,0,0,0,0,2011 +UNAWARE,2009,0,0,0,0,0,0 +UNCERTAINTY,0,0,2009,0,0,0,0 +UNCOLLECTIBILITY,2009,0,0,0,0,0,0 +UNCOMPLETED,2009,0,0,0,0,0,0 +UNCONSCIONABLY,2009,0,0,0,0,0,0 +UNCONTRACTED,0,0,0,2011,0,0,0 +UNCORRECTED,2009,0,0,0,0,0,0 +UNCOVERS,2009,0,0,0,0,0,0 +UNDELIVERABLE,2009,0,0,0,0,0,0 +UNDERCUTS,2009,0,0,0,0,0,0 +UNDERESTIMATES,2009,0,0,0,0,0,0 +UNDERINSURED,2009,0,0,0,0,0,0 +UNDERMINING,2009,0,0,0,0,0,0 +UNDERPAYS,2009,0,0,0,0,0,0 +UNDERPERFORMING,2009,0,0,0,0,0,0 +UNDERREPORTING,2011,0,0,0,0,0,0 +UNDERSTATEMENTS,2009,0,0,0,0,0,0 +UNDERUTILIZED,2009,0,0,0,0,0,0 +UNDETECTABLE,0,0,2009,0,0,0,0 +UNDISCHARGED,0,0,0,2009,0,0,0 +UNDOUBTEDLY,0,0,0,0,2009,0,0 +UNECONOMICAL,2009,0,0,0,0,0,0 +UNENCUMBER,0,0,0,2014,0,0,0 +UNEQUIVOCAL,0,0,0,0,2009,0,0 +UNEXCUSED,2009,0,0,0,0,0,0 +UNFAIRLY,2009,0,0,0,0,0,0 +UNFAVORABLE,2009,0,0,0,0,0,0 +UNFIT,2009,0,0,0,0,0,0 +UNFORESEEN,2009,0,0,0,0,0,0 +UNFOUNDED,2009,0,0,0,0,0,0 +UNGUARANTEED,0,0,2009,0,0,0,0 +UNINSURED,2009,0,0,0,0,0,0 +UNJUST,2009,0,0,0,0,0,0 +UNJUSTLY,2009,0,0,0,0,0,0 +UNKNOWNS,0,0,2009,0,0,0,0 +UNLICENSED,2009,0,0,0,0,0,0 +UNMERCHANTABLE,2011,0,0,0,0,0,0 +UNNEEDED,2009,0,0,0,0,0,0 +UNPAID,2009,0,0,0,0,0,0 +UNPOPULAR,2009,0,0,0,0,0,0 +UNPREDICTED,2011,0,2011,0,0,0,0 +UNPROVED,0,0,2009,0,0,0,0 +UNQUANTIFIED,0,0,2011,0,0,0,0 +UNREASONABLY,2009,0,0,0,0,0,0 +UNRECOVERED,2009,0,0,0,0,0,0 +UNREMEDIED,2009,0,0,0,0,0,0 +UNSAFE,2009,0,0,0,0,0,0 +UNSATISFIED,2009,0,0,0,0,0,0 +UNSEASONABLY,0,0,2009,0,0,0,0 +UNSOUND,2009,0,0,0,0,0,0 +UNSTABLE,2009,0,0,0,0,0,0 +UNSUCCESSFULLY,2009,0,0,0,0,0,0 +UNSUITED,2009,0,0,0,0,0,0 +UNSUSPECTING,2009,0,0,0,0,0,0 +UNTIMELY,2009,0,0,0,0,0,0 +UNTRUTHFUL,2009,0,0,0,0,0,0 +UNUSABLE,2009,0,0,0,0,0,0 +UNWARRANTED,2009,0,0,0,0,0,0 +UNWRITTEN,0,0,2009,0,0,0,0 +URGENCY,2009,0,0,0,0,0,0 +USURPATION,0,0,0,2009,0,0,0 +USURY,2009,0,0,2009,0,0,0 +VAGUENESS,0,0,2012,0,0,0,0 +VALUABLE,0,2009,0,0,0,0,0 +PLEAD,2009,0,0,0,0,0,0 +PLEADS,2009,0,0,2009,0,0,0 +PLEASED,0,2009,0,0,0,0,0 +PLEDGED,0,0,0,0,0,0,2009 +PLEDGING,0,0,0,0,0,0,2009 +COMPEL,0,0,0,0,0,0,2011 +COMPENSATORY,0,0,0,2009,0,0,0 +COMPLAINED,2009,0,0,0,0,0,0 +COMPLAINTS,2009,0,0,0,0,0,0 +COMMITMENT,0,0,0,0,0,0,2009 +COMMITTING,0,0,0,0,0,0,2009 +LIMITATION,2009,0,0,0,0,0,0 +LINGERING,2009,0,0,0,0,0,0 +RESTRAINING,0,0,0,0,0,0,2009 +RESTRICT,0,0,0,0,0,0,2009 +RESTRICTIONS,0,0,0,0,0,0,2009 +RESTRICTS,0,0,0,0,0,0,2009 +RESTRUCTURING,2009,0,0,0,0,0,0 +RETALIATES,2009,0,0,0,0,0,0 +RETALIATORY,2009,0,0,0,0,0,0 +PRECAUTIONS,0,0,2009,0,0,0,0 +PRECLUDE,2009,0,0,0,0,0,2009 +PRECONDITION,0,0,0,0,0,0,2009 +PREDECEASED,0,0,0,2009,0,0,0 +PREDICTABILITY,0,0,2009,0,0,0,0 +PREDICTIONS,0,0,2009,0,0,0,0 +PREDICTS,0,0,2009,0,0,0,0 +PREJUDICE,2009,0,0,2009,0,0,0 +PREJUDICING,2009,0,0,2009,0,0,0 +PREMATURELY,2009,0,0,0,0,0,0 +PRESET,0,0,0,0,0,0,2009 +PRESUMABLY,0,0,2009,0,0,0,0 +PRESUMING,0,0,2009,0,0,0,0 +PRETRIAL,2009,0,0,2009,0,0,0 +PREVENTION,2009,0,0,0,0,0,0 +PROACTIVE,0,2009,0,0,0,0,0 +PROBABILITY,0,0,2009,0,0,0,0 +PROBATED,0,0,0,2009,0,0,0 +PROBATIONAL,0,0,0,2009,0,0,0 +PROBATIONS,0,0,0,2009,0,0,0 +PROBLEMS,2009,0,0,0,0,0,0 +PROFITABILITY,0,2009,0,0,0,0,0 +PROGRESSED,0,2009,0,0,0,0,0 +PROHIBITED,0,0,0,0,0,0,2009 +PROHIBITIVE,0,0,0,0,0,0,2009 +PROLONG,2009,0,0,0,0,0,0 +PROLONGING,2009,0,0,0,0,0,0 +PROMULGATES,0,0,0,2009,0,0,0 +PROMULGATOR,0,0,0,2009,0,0,0 +PRORATION,0,0,0,2009,0,0,0 +PROSECUTING,2009,0,0,2009,0,0,0 +PROSECUTORIAL,0,0,0,2011,0,0,0 +PROSPERITY,0,2009,0,0,0,0,0 +PROTESTED,2009,0,0,0,0,0,0 +PROTESTOR,2009,0,0,0,0,0,0 +PROTRACTION,2009,0,0,0,0,0,0 +PROVOKE,2009,0,0,0,0,0,0 +PUNISHABLE,0,0,0,2009,0,0,0 +PUNISHMENT,2009,0,0,0,0,0,0 +PURPORTED,2009,0,0,0,0,0,0 +QUESTION,2009,0,0,0,0,0,0 +QUESTIONING,2009,0,0,0,0,0,0 +QUITCLAIMS,0,0,0,2009,0,0,0 +RANDOM,0,0,2009,0,0,0,0 +RANDOMIZING,0,0,2009,0,0,0,0 +RATABLE,0,0,0,2009,0,0,0 +RATIONALIZE,2009,0,0,0,0,0,0 +ABANDONED,2009,0,0,0,0,0,0 +ABANDONS,2009,0,0,0,0,0,0 +ABDICATION,2009,0,0,0,0,0,0 +ABERRATIONAL,2009,0,0,0,0,0,0 +ABEYANCES,0,0,2009,0,0,0,0 +ABNORMAL,2009,0,0,0,0,0,0 +ABOLISH,2009,0,0,0,0,0,0 +ABOVEMENTIONED,0,0,0,2011,0,0,0 +ABROGATING,2009,0,0,2009,0,0,0 +ABRUPTLY,2009,0,0,0,0,0,0 +ABSENTEEISM,2009,0,0,0,0,0,0 +ABSOLVING,0,0,0,2009,0,0,0 +ABUSED,2009,0,0,0,0,0,0 +ABUSIVELY,2009,0,0,0,0,0,0 +ACCIDENT,2009,0,0,0,0,0,0 +ACCLAIMED,0,2009,0,0,0,0,0 +ACCOMPLISHING,0,2009,0,0,0,0,0 +ACCUSATIONS,2009,0,0,0,0,0,0 +ACCUSING,2009,0,0,0,0,0,0 +ACHIEVEMENTS,0,2009,0,0,0,0,0 +ACQUIESCED,2009,0,0,0,0,0,0 +ACQUIRORS,0,0,0,2011,0,0,0 +ACQUITTALS,2009,0,0,2009,0,0,0 +ACQUITTING,2009,0,0,2009,0,0,0 +ADJOURNED,0,0,0,2009,0,0,0 +ADJOURNS,0,0,0,2009,0,0,0 +ADJUDGING,0,0,0,2009,0,0,0 +ADJUDICATING,0,0,0,2009,0,0,0 +ADJUDICATOR,0,0,0,2009,0,0,0 +ADMISSIBLE,0,0,0,2009,0,0,0 +ADULTERATE,2009,0,0,0,0,0,0 +ADULTERATIONS,2009,0,0,0,0,0,0 +ADVANCING,0,2009,0,0,0,0,0 +ADVANTAGEOUSLY,0,2009,0,0,0,0,0 +ADVERSARY,2009,0,0,0,0,0,0 +ADVERSITY,2009,0,0,0,0,0,0 +AFFREIGHTMENT,0,0,0,2012,0,0,0 +AFORESTATED,0,0,0,2011,0,0,0 +AGGRAVATE,2009,0,0,0,0,0,0 +AGGRAVATION,2009,0,0,0,0,0,0 +ALERTING,2009,0,0,0,0,0,0 +ALIENATING,2009,0,0,0,0,0,0 +ALLEGATIONS,2009,0,0,2009,0,0,0 +ALLEGES,2009,0,0,2009,0,0,0 +ALMOST,0,0,2009,0,0,2009,0 +AMBIGUITIES,0,0,2009,0,0,0,0 +AMENDABLE,0,0,0,2009,0,0,0 +AMENDMENT,0,0,0,2009,0,0,0 +ANNOYANCE,2009,0,0,0,0,0,0 +ANNOYS,2009,0,0,0,0,0,0 +ANNULMENT,2009,0,0,0,0,0,0 +ANOMALOUS,2009,0,2009,0,0,0,0 +ANTECEDENTS,0,0,0,2009,0,0,0 +ANTICIPATING,0,0,2009,0,0,0,0 +ANTICORRUPTION,0,0,0,2014,0,0,0 +APPARENTLY,0,0,2009,0,0,2009,0 +APPEALING,0,0,0,2009,0,0,0 +APPEARING,0,0,2009,0,0,2009,0 +APPELLATE,0,0,0,2009,0,0,0 +APPROXIMATED,0,0,2009,0,0,0,0 +APPROXIMATION,0,0,2009,0,0,0,0 +APPURTENANT,0,0,0,2009,0,0,0 +ARBITRARINESS,0,0,2009,0,0,0,0 +ARBITRATES,0,0,0,2009,0,0,0 +ARBITRATIONS,0,0,0,2009,0,0,0 +ARGUE,2009,0,0,0,0,0,0 +ARGUMENTATIVE,2009,0,0,0,0,0,0 +ARREARS,2009,0,0,0,0,0,0 +ARTIFICIALLY,2009,0,0,0,0,0,0 +RETRIBUTION,2009,0,0,0,0,0,0 +RETROCESSIONAIRES,0,0,0,2011,0,0,0 +BOLSTERS,0,2009,0,0,0,0,0 +BOOMING,0,2009,0,0,0,0,0 +BOTTLENECKS,2009,0,0,0,0,0,0 +BOYCOTTED,2009,0,0,0,0,0,0 +BREACHED,2009,0,0,2009,0,0,0 +BREAKAGE,2009,0,0,0,0,0,0 +BREAKING,-2020,0,0,0,0,0,0 +BRIBE,2009,0,0,0,0,0,0 +BRIBES,2009,0,0,0,0,0,0 +BROKEN,-2020,0,0,0,0,0,0 +BURDENS,2009,0,0,0,0,0,0 +CALAMITOUS,2009,0,0,0,0,0,0 +CANCELING,2009,0,0,0,0,0,0 +CANCELLING,2009,0,0,0,0,0,0 +CARELESSNESS,2009,0,0,0,0,0,0 +CATASTROPHIC,2009,0,0,0,0,0,0 +CAUTIONED,2009,0,0,0,0,0,0 +CAUTIOUSLY,0,0,2009,0,0,0,0 +CEASES,2009,0,0,0,0,0,0 +CENSURE,2009,0,0,0,0,0,0 +CERTIORARI,0,0,0,2011,0,0,0 +CHALLENGES,2009,0,0,0,0,0,0 +CHATTEL,0,0,0,2009,0,0,0 +CIRCUMVENTED,2009,0,0,0,0,0,0 +CIRCUMVENTS,2009,0,0,0,0,0,0 +SHALL,0,0,0,2009,0,0,0 +SHORTAGES,2009,0,0,0,0,0,0 +SHRINKAGES,2009,0,0,0,0,0,0 +SHUTS,2009,0,0,0,0,0,0 +SLANDEROUS,2009,0,0,0,0,0,0 +REPUDIATING,2009,0,0,0,0,0,0 +REQUESTOR,0,0,0,2011,0,0,0 +REQUIREMENTS,0,0,0,0,0,0,2009 +RESCIND,0,0,0,2009,0,0,0 +RESCISSION,0,0,0,2009,0,0,0 +RESIGNATIONS,2009,0,0,0,0,0,0 +RESOLVE,0,2009,0,0,0,0,0 +RESTATEMENTS,2009,0,0,0,0,0,0 +NOTARIES,0,0,0,2009,0,0,0 +NOTARIZED,0,0,0,2009,0,0,0 +NOVO,0,0,0,2011,0,0,0 +NULLIFICATIONS,2009,0,0,2009,0,0,0 +NULLIFYING,2009,0,0,2009,0,0,0 +OBJECTING,2009,0,0,0,0,0,0 +OBJECTIONS,2009,0,0,0,0,0,0 +OBLIGATING,0,0,0,0,0,0,2009 +OBLIGE,0,0,0,0,0,0,2009 +OBLIGES,0,0,0,0,0,0,2009 +OBSCENITY,2009,0,0,0,0,0,0 +OBSTACLES,2009,0,0,0,0,0,0 +OBSTRUCTION,2009,0,0,0,0,0,0 +OFFENCES,2009,0,0,0,0,0,0 +OFFENDERS,2009,0,0,0,0,0,0 +OFFEREE,0,0,0,2009,0,0,0 +OMISSION,2009,0,0,0,0,0,0 +OMITTED,2009,0,0,0,0,0,0 +OPPORTUNISTICALLY,2009,0,0,0,0,0,0 +OPPOSED,2009,0,0,0,0,0,0 +OPPOSITIONS,2009,0,0,0,0,0,0 +ORDINARILY,0,0,2009,0,0,0,0 +OUTMODED,2009,0,0,0,0,0,0 +OUTPERFORMS,0,2009,0,0,0,0,0 +OVERBUILDING,2009,0,0,0,0,0,0 +OVERBURDENED,2009,0,0,0,0,0,0 +OVERCHARGE,2009,0,0,0,0,0,0 +OVERCOME,2009,0,0,0,0,0,0 +OVERESTIMATE,2009,0,0,0,0,0,0 +OVERESTIMATION,2009,0,0,0,0,0,0 +OVERLOADING,2009,0,0,0,0,0,0 +OVERLOOKING,2009,0,0,0,0,0,0 +OVERPAYMENTS,2009,0,0,0,0,0,0 +OVERPRODUCTION,2009,0,0,0,0,0,0 +OVERRULING,0,0,0,2009,0,0,0 +OVERSHADOW,2009,0,0,0,0,0,0 +OVERSTATE,2009,0,0,0,0,0,0 +OVERSTATES,2009,0,0,0,0,0,0 +OVERSUPPLY,2009,0,0,0,0,0,0 +OVERTURNED,2009,0,0,0,0,0,0 +OVERVALUED,2009,0,0,0,0,0,0 +PARA,0,0,0,-2020,0,0,0 +NECESSITATED,0,0,0,0,0,0,2009 +NEGATIVELY,2009,0,0,0,0,0,0 +NEGLECTFUL,2009,0,0,0,0,0,0 +NEGLIGENCES,2009,0,0,0,0,0,0 +NOLO,0,0,0,2011,0,0,0 +BEAUTIFUL,0,2009,0,0,0,0,0 +BELIEVES,0,0,2009,0,0,0,0 +IDEAL,0,2009,0,0,0,0,0 +IGNORE,2009,0,0,0,0,0,0 +ILL,2009,0,0,0,0,0,0 +ILLEGALLY,2009,0,0,0,0,0,0 +ILLIQUID,2009,0,0,0,0,0,0 +IMMATERIALITY,0,0,0,2009,0,0,0 +IMPAIRED,2009,0,0,0,0,0,2011 +IMPAIRS,2009,0,0,0,0,0,2011 +IMPEDED,2009,0,0,0,0,0,0 +IMPEDING,2009,0,0,0,0,0,0 +IMPERFECTIONS,2009,0,0,0,0,0,0 +IMPLICATE,2009,0,0,0,0,0,0 +IMPOSE,0,0,0,0,0,0,2009 +IMPOSITION,0,0,0,0,0,0,2009 +IMPOUND,2009,0,0,0,0,0,0 +IMPRACTICABLE,2009,0,0,0,0,0,0 +IMPRECISE,0,0,2009,0,0,0,0 +IMPRESSED,0,2009,0,0,0,0,0 +IMPRESSIVELY,0,2009,0,0,0,0,0 +IMPROPER,2009,0,0,0,0,0,0 +IMPROVE,0,2009,0,0,0,0,0 +IMPROVES,0,2009,0,0,0,0,0 +INABILITY,2009,0,0,0,0,0,0 +INACCURATE,2009,0,0,0,0,0,0 +INACTIVATE,2009,0,0,0,0,0,0 +INACTIVATION,2009,0,0,0,0,0,0 +INADEQUACY,2009,0,0,0,0,0,0 +INADVERTENTLY,2009,0,0,0,0,0,0 +INAPPROPRIATELY,2009,0,0,0,0,0,0 +INCAPACITATED,2009,0,0,0,0,0,0 +INCARCERATES,2009,0,0,2009,0,0,0 +INCHOATE,0,0,0,2009,0,0,0 +INCIDENTS,2009,0,0,0,0,0,0 +INCOMPETENCE,2009,0,0,0,0,0,0 +INCOMPETENTS,2009,0,0,0,0,0,0 +INCONCLUSIVE,2009,0,0,0,0,0,0 +INCONSISTENTLY,2009,0,0,0,0,0,0 +INCONVENIENCES,2009,0,0,0,0,0,0 +INCORRECTNESS,2009,0,0,0,0,0,0 +INDECENCY,2009,0,0,0,0,0,0 +INDEFINITE,0,0,2009,0,0,0,0 +INDEMNIFICATION,0,0,0,2009,0,0,0 +INDEMNIFY,0,0,0,2009,0,0,0 +INDEMNITIES,0,0,0,2009,0,0,0 +INDETERMINABLE,0,0,2009,0,0,0,0 +INDICTED,2009,0,0,2009,0,0,0 +INDORSEES,0,0,0,2011,0,0,0 +INEFFICIENCIES,2009,0,0,0,0,0,0 +INELIGIBILITY,2009,0,0,0,0,0,0 +INEQUITIES,2009,0,0,0,0,0,0 +INEXACTNESS,0,0,2009,0,0,0,0 +INFLICTED,2009,0,0,0,0,0,0 +INFRACTION,2009,0,0,2009,0,0,0 +INFRINGEMENT,2009,0,0,0,0,0,0 +INFRINGING,2009,0,0,0,0,0,0 +INHIBITING,0,0,0,0,0,0,2011 +INJUNCTIONS,2009,0,0,2009,0,0,0 +INJURES,2009,0,0,0,0,0,0 +INJURY,2009,0,0,0,0,0,0 +INNOVATING,0,2009,0,0,0,0,0 +INNOVATIVENESS,0,2011,0,0,0,0,0 +INORDINATELY,2009,0,0,0,0,0,0 +INSIGHTFUL,0,2009,0,0,0,0,0 +INSISTING,0,0,0,0,0,0,2009 +INSOLVENCY,2009,0,0,0,0,0,0 +INSTABILITIES,0,0,2009,0,0,0,0 +INSUFFICIENT,2009,0,0,0,0,0,0 +INTANGIBLE,0,0,2009,0,0,0,0 +INTERFERE,2009,0,0,0,0,0,0 +INTERFERES,2009,0,0,0,0,0,0 +INTERMITTENTLY,2009,0,0,0,0,0,0 +INTERPOSES,0,0,0,2009,0,0,0 +INTERROGATE,0,0,0,2009,0,0,0 +INTERROGATION,0,0,0,2009,0,0,0 +INTERROGATORS,0,0,0,2009,0,0,0 +INTERRUPTING,2009,0,0,0,0,0,0 +INTESTACY,0,0,0,2009,0,0,0 +STABILIZATION,0,2009,0,0,0,0,0 +STABILIZES,0,2009,0,0,0,0,0 +STAGNANT,2009,0,0,0,0,0,0 +STAGNATING,2009,0,0,0,0,0,0 +STATUTE,0,0,0,2009,0,0,0 +STIPULATE,0,0,0,0,0,0,2009 +STIPULATION,0,0,0,0,0,0,2009 +STOPPAGES,2009,0,0,0,0,0,0 +STRAIN,2009,0,0,0,0,0,0 +STRENGTH,0,2009,0,0,0,0,0 +STRENGTHENS,0,2009,0,0,0,0,0 +STRESSES,2009,0,0,0,0,0,0 +STRICTER,0,0,0,0,0,0,2009 +STRONG,0,2009,0,0,0,0,0 +SUBCLAUSE,0,0,0,2009,0,0,0 +SUBJECTING,2009,0,0,0,0,0,0 +SUBLESSORS,0,0,0,2011,0,0,0 +SUBPARAGRAPHS,0,0,0,2009,0,0,0 +SUBROGATED,0,0,0,2009,0,0,0 +SUBTRUSTS,0,0,0,2011,0,0,0 +SUCCEEDS,0,2009,0,0,0,0,0 +SUCCESSFULLY,0,2009,0,0,0,0,0 +SUED,2009,0,0,2009,0,0,0 +SUFFERING,2009,0,0,0,0,0,0 +SUGGESTING,0,0,2009,0,0,0,0 +SUMMONING,2009,0,0,2009,0,0,0 +SUPERSEDE,0,0,0,2009,0,0,0 +SUPERSEDING,0,0,0,2009,0,0,0 +SURPASSED,0,2009,0,0,0,0,0 +SUSCEPTIBLE,2009,0,0,0,0,0,0 +SUSPEND,2009,0,0,0,0,0,0 +SUSPENSION,2009,0,0,0,0,0,0 +SUSPICIOUS,2009,0,0,0,0,0,0 +REVOCATION,2009,0,0,2009,0,0,0 +REVOKES,2009,0,0,0,0,0,0 +REVOLUTIONIZES,0,2009,0,0,0,0,0 +REWARDING,0,2009,0,0,0,0,0 +RIDICULES,2009,0,0,0,0,0,0 +RISKIER,2009,0,2009,0,0,0,0 +RISKS,0,0,2009,0,0,0,0 +RULINGS,0,0,0,2009,0,0,0 +SACRIFICED,2009,0,0,0,0,0,0 +SATISFACTION,0,2009,0,0,0,0,0 +SATISFIES,0,2009,0,0,0,0,0 +SCANDALS,2009,0,0,0,0,0,0 +SCRUTINIZING,2009,0,0,0,0,0,0 +SEIZE,2009,0,0,0,0,0,0 +SELDOM,0,0,2009,0,0,2009,0 +SEQUESTRATOR,0,0,0,2009,0,0,0 +SETBACK,2009,0,0,0,0,0,0 +SEVER,2009,0,0,0,0,0,0 +SEVERANCE,0,0,0,2009,0,0,0 +SEVERELY,2009,0,0,0,0,0,0 +ENTRENCH,0,0,0,0,0,0,2009 +ERODES,2009,0,0,0,0,0,0 +ERRATICALLY,2009,0,0,0,0,0,0 +ERRONEOUSLY,2009,0,0,0,0,0,0 +ESCALATE,2009,0,0,0,0,0,0 +ESCHEAT,0,0,0,2011,0,0,0 +ESCROWED,0,0,0,0,0,0,2009 +EVADE,2009,0,0,0,0,0,0 +EVASION,2009,0,0,0,0,0,0 +EVICTED,2009,0,0,0,0,0,0 +EVICTS,2009,0,0,0,0,0,0 +EXACERBATED,2009,0,0,0,0,0,0 +EXACERBATIONS,2009,0,0,0,0,0,0 +EXAGGERATING,2009,0,0,0,0,0,0 +EXCEEDENCES,0,0,0,2011,0,0,0 +EXCELS,0,2009,0,0,0,0,0 +EXCESSIVELY,2009,0,0,0,0,0,0 +EXCITING,0,2009,0,0,0,0,0 +EXCLUSIVES,0,2009,0,0,0,0,0 +EXCULPATES,2009,0,0,2009,0,0,0 +EXCULPATORY,2009,0,0,2009,0,0,0 +EXECUTRICES,0,0,0,2009,0,0,0 +EXONERATE,2009,0,0,0,0,0,0 +EXONERATION,2009,0,0,0,0,0,0 +EXPLOITATIONS,2009,0,0,0,0,0,0 +EXPLOITS,2009,0,0,0,0,0,0 +EXPOSING,2009,0,0,0,0,0,0 +EXPROPRIATED,2009,0,0,0,0,0,0 +EXPROPRIATIONS,2009,0,0,0,0,0,0 +EXTRACONTRACTUAL,0,0,0,2011,0,0,0 +SOLVENCIES,2009,0,0,0,0,0,0 +SOMETIME,0,0,2009,0,0,0,0 +SPAM,2014,0,0,0,0,0,0 +TRAUMATIC,2009,0,0,0,0,0,0 +LOSES,2009,0,0,0,0,0,0 +LOST,2009,0,0,0,0,0,0 +LYING,2009,0,0,0,0,0,0 +MALFUNCTIONED,2009,0,0,0,0,0,0 +MALICIOUS,2009,0,0,0,0,0,0 +MANDATE,0,0,0,0,0,0,2009 +MANDATORY,0,0,0,0,0,0,2009 +MANIPULATES,2009,0,0,0,0,0,0 +MANIPULATIVE,2009,0,0,0,0,0,0 +MAYBE,0,0,2009,0,0,2009,0 +MEDIATING,0,0,0,2009,0,0,0 +MEDIATORS,0,0,0,2009,0,0,0 +MISAPPLICATIONS,2009,0,0,0,0,0,0 +MISAPPLYING,2009,0,0,0,0,0,0 +MISAPPROPRIATING,2009,0,0,0,0,0,0 +MISCALCULATE,2009,0,0,0,0,0,0 +MISCALCULATION,2009,0,0,0,0,0,0 +MISCLASSIFICATION,2011,0,0,0,0,0,0 +MISCOMMUNICATION,2014,0,0,0,0,0,0 +MISDEMEANORS,2009,0,0,0,0,0,0 +MISHANDLED,2009,0,0,0,0,0,0 +MISINFORMATION,2009,0,0,0,0,0,0 +MISINTERPRET,2009,0,0,0,0,0,0 +MISINTERPRETING,2009,0,0,0,0,0,0 +MISJUDGES,2009,0,0,0,0,0,0 +MISLABEL,2009,0,0,0,0,0,0 +MISLABELS,2009,0,0,0,0,0,0 +MISLEADS,2009,0,0,0,0,0,0 +MISMANAGEMENT,2009,0,0,0,0,0,0 +MISMATCHED,2009,0,0,0,0,0,0 +MISPRICE,2014,0,0,0,0,0,0 +MISREPRESENTATION,2009,0,0,0,0,0,0 +MISREPRESENTS,2009,0,0,0,0,0,0 +WORRYING,2009,0,0,0,0,0,0 +WORSENING,2009,0,0,0,0,0,0 +WORTHY,0,2009,0,0,0,0,0 +TOLERATED,2009,0,0,0,0,0,0 +TORT,0,0,0,2009,0,0,0 +TORTUOUS,2009,0,0,0,0,0,0 +FIDE,0,0,0,2009,0,0,0 +FIRING,2009,0,0,0,0,0,0 +NONBREACHING,0,0,0,2011,0,0,0 +NONCOMPLIANCE,2009,0,0,0,0,0,0 +NONCONFORMING,2009,0,0,0,0,0,0 +NONCONTRACT,0,0,0,2012,0,0,0 +NONFEASANCE,0,0,0,2011,0,0,0 +NONFORFEITURE,0,0,0,2011,0,0,0 +DISCLAIMED,2009,0,0,0,0,0,0 +DISCLAIMS,2009,0,0,0,0,0,0 +DISCLOSING,2009,0,0,0,0,0,0 +DISCONTINUATIONS,2009,0,0,0,0,0,0 +DISCONTINUING,2009,0,0,0,0,0,0 +DISCOURAGING,2009,0,0,0,0,0,0 +DISCREDITS,2009,0,0,0,0,0,0 +SPECTACULARLY,0,2009,0,0,0,0,0 +SPECULATING,0,0,2009,0,0,0,0 +SPECULATIVELY,0,0,2009,0,0,0,0 +TRAGEDY,2009,0,0,0,0,0,0 +TRANSFERORS,0,0,0,2012,0,0,0 +LEGISLATING,0,0,0,2009,0,0,0 +LEGISLATIVELY,0,0,0,2009,0,0,0 +LEGISLATURES,0,0,0,2009,0,0,0 +LIBELS,0,0,0,2009,0,0,0 +CONCEALED,2009,0,0,0,0,0,0 +CONCEDES,2009,0,0,0,0,0,0 +CONCERN,2009,0,0,0,0,0,0 +CONCILIATION,2009,0,0,0,0,0,0 +CONDEMN,2009,0,0,0,0,0,0 +CONDEMNING,2009,0,0,0,0,0,0 +CONDITIONALLY,0,0,2009,0,0,0,0 +CONFESS,2009,0,0,0,0,0,0 +CONFESSION,2009,0,0,0,0,0,0 +CONFINEMENT,2009,0,0,0,0,0,2009 +CONFISCATE,2009,0,0,0,0,0,0 +CONFISCATION,2009,0,0,0,0,0,0 +CONFLICTED,2009,0,0,0,0,0,0 +CONFRONTATION,2009,0,0,0,0,0,0 +CONFRONTING,2009,0,0,0,0,0,0 +CONFUSES,2009,0,2009,0,0,0,0 +CONSENT,0,0,0,2009,0,0,0 +CONSERVATORSHIPS,0,0,0,2011,0,0,0 +CONSPIRATORIAL,2009,0,0,0,0,0,0 +CONSPIRES,2009,0,0,0,0,0,0 +CONSTITUTIONALITY,0,0,0,2009,0,0,0 +CONSTRAIN,0,0,0,0,0,0,2009 +CONSTRAINT,0,0,0,0,0,0,2009 +CONSTRUE,0,0,0,2012,0,0,0 +CONTEMPT,2009,0,0,0,0,0,0 +CONTENDS,2009,0,0,0,0,0,0 +CONTENTIOUSLY,2009,0,0,0,0,0,0 +CONTESTING,2009,0,0,0,0,0,0 +CONTINGENTLY,0,0,2009,0,0,0,0 +CONTRACTHOLDER,0,0,0,2009,0,0,0 +CONTRACTING,0,0,0,2009,0,0,0 +CONTRACTUAL,0,0,0,2009,0,0,0 +CONTRADICTING,2009,0,0,0,0,0,0 +CONTRADICTS,2009,0,0,0,0,0,0 +CONTRAVENES,0,0,0,2009,0,0,0 +CONTROVERSIAL,2009,0,0,0,0,0,0 +CONTROVERTED,0,0,0,2009,0,0,0 +CONVEYANCES,0,0,0,2009,0,0,0 +CONVICTION,2009,0,0,2009,0,0,0 +CORRECTION,2009,0,0,0,0,0,0 +CORRUPTED,2009,0,0,0,0,0,0 +CORRUPTLY,2009,0,0,0,0,0,0 +COULD,0,0,2009,0,0,2009,0 +COUNSELS,0,0,0,2009,0,0,0 +COUNTERCLAIMS,2009,0,0,0,0,0,0 +COUNTERFEITERS,2009,0,0,0,0,0,0 +COUNTERMEASURES,2009,0,0,0,0,0,0 +COUNTERSUITS,0,0,0,2014,0,0,0 +COURTS,0,0,0,2009,0,0,0 +COVENANTS,0,0,0,0,0,0,2011 +CREATIVITY,0,2009,0,0,0,0,0 +CRIMINALITY,0,0,0,2009,0,0,0 +CRIMINALS,2009,0,0,2009,0,0,0 +CRITICALLY,2009,0,0,0,0,0,0 +CRITICIZED,2009,0,0,0,0,0,0 +CROSSCLAIMS,0,0,0,2011,0,0,0 +CRUCIALLY,2009,0,0,0,0,0,0 +CUMBERSOME,2009,0,0,0,0,0,0 +CURTAILMENT,2009,0,0,0,0,0,0 +CUTBACK,2009,0,0,0,0,0,0 +CYBERBULLYING,2014,0,0,0,0,0,0 +CYBERCRIMINALS,2014,0,0,0,0,0,0 +TAINTED,2009,0,0,0,0,0,0 +TENANTABILITY,0,0,0,2011,0,0,0 +TENTATIVELY,0,0,2009,0,0,0,0 +TERMINATES,2009,0,0,0,0,0,0 +TERMINUS,0,0,0,-2020,0,0,0 +TESTIMONY,0,0,0,2009,0,0,0 +THEREAFTER,0,0,0,2009,0,0,0 +THEREINAFTER,0,0,0,2011,0,0,0 +THERETO,0,0,0,2009,0,0,0 +THEREUNTO,0,0,0,2009,0,0,0 +THREATEN,2009,0,0,0,0,0,0 +THREATS,2009,0,0,0,0,0,0 +FAIL,2009,0,0,0,0,0,0 +FAILS,2009,0,0,0,0,0,0 +FALSE,2009,0,0,0,0,0,0 +FALSIFIED,2009,0,0,0,0,0,0 +FALSITY,2009,0,0,0,0,0,0 +FATALLY,2009,0,0,0,0,0,0 +FAULTY,2009,0,0,0,0,0,0 +FAVORING,0,2009,0,0,0,0,0 +FEARS,2009,0,0,0,0,0,0 +DAMAGE,2009,0,0,0,0,0,0 +DAMPEN,2009,0,0,0,0,0,0 +DANGEROUSLY,2009,0,0,0,0,0,0 +DEADLOCKING,2009,0,0,0,0,0,0 +DEBARMENT,2009,0,0,0,0,0,0 +DECEDENT,0,0,0,2009,0,0,0 +DECEITFULNESS,2009,0,0,0,0,0,0 +DECEIVING,2009,0,0,0,0,0,0 +DECEPTIVELY,2009,0,0,0,0,0,0 +DECLINES,2009,0,0,0,0,0,0 +DECREEING,0,0,0,2009,0,0,0 +DEFACEMENT,2009,0,0,0,0,0,0 +DEFAMATIONS,2009,0,0,0,0,0,0 +DEFAMES,2009,0,0,0,0,0,0 +DEFAULTING,2009,0,0,0,0,0,0 +DEFEASE,0,0,0,2009,0,0,0 +DEFEASING,0,0,0,2011,0,0,0 +DEFEATS,2009,0,0,0,0,0,0 +DEFECTS,2009,0,0,0,0,0,0 +DEFENDANTS,2009,0,0,2009,0,0,0 +DEFENSIVE,2009,0,0,0,0,0,0 +DEFICIENCY,2009,0,0,0,0,0,0 +DEFINITELY,0,0,0,0,2009,0,0 +DEFRAUDING,2009,0,0,0,0,0,0 +DEGRADATIONS,2009,0,0,0,0,0,0 +DEGRADING,2009,0,0,0,0,0,0 +DELAYS,2009,0,0,0,0,0,0 +DELEGEES,0,0,0,2011,0,0,0 +DELIBERATELY,2009,0,0,0,0,0,0 +DELIGHTFULLY,0,2009,0,0,0,0,0 +DELINQUENCY,2009,0,0,0,0,0,0 +DELIST,2009,0,0,0,0,0,0 +DEMISE,2009,0,0,0,0,0,0 +DEMOLISH,2009,0,0,0,0,0,0 +DEMOLITION,2009,0,0,0,0,0,0 +DEMOTES,2009,0,0,0,0,0,0 +DEMURRED,0,0,0,2009,0,0,0 +DEMURS,0,0,0,2009,0,0,0 +DENIES,2009,0,0,0,0,0,0 +DENIGRATING,2009,0,0,0,0,0,0 +DEPEND,0,0,2009,0,0,2009,2011 +DEPENDANCES,0,0,0,0,0,0,2011 +DEPENDENCIES,0,0,2009,0,0,0,2011 +DEPENDS,0,0,2009,0,0,2009,2011 +DEPLETING,2009,0,0,0,0,0,0 +DEPOSED,0,0,0,2009,0,0,0 +DEPOSITIONAL,0,0,0,2011,0,0,0 +INVALIDATED,2009,0,0,0,0,0,0 +INVALIDITY,2009,0,0,0,0,0,0 +INVENTION,0,2009,0,0,0,0,0 +INVENTOR,0,2009,0,0,0,0,0 +INVESTIGATES,2009,0,0,0,0,0,0 +INVOLUNTARILY,2009,0,0,0,0,0,0 +IRRECOVERABLE,2009,0,0,0,0,0,0 +IRREGULARITY,2009,0,0,0,0,0,0 +IRREVERSIBLE,2009,0,0,0,0,0,0 +JEOPARDIZE,2009,0,0,0,0,0,0 +JUDICIALLY,0,0,0,2009,0,0,0 +JURIS,0,0,0,2011,0,0,0 +JURISDICTIONS,0,0,0,2009,0,0,0 +JUROR,0,0,0,2009,0,0,0 +JUSTICE,0,0,0,2009,0,0,0 +KICKBACKS,2009,0,0,0,0,0,0 +DIMINISHED,2009,0,0,0,0,0,0 +DIRECTIVE,0,0,0,0,0,0,2009 +DISADVANTAGEOUS,2009,0,0,0,0,0,0 +DISAFFIRMANCE,0,0,0,2009,0,0,0 +DISAGREEABLE,2009,0,0,0,0,0,0 +DISAGREEMENTS,2009,0,0,0,0,0,0 +DISALLOWANCES,2009,0,0,0,0,0,0 +DISAPPEAR,2009,0,0,0,0,0,0 +DISAPPEARING,2009,0,0,0,0,0,0 +DISAPPOINTING,2009,0,0,0,0,0,0 +DISAPPOINTS,2009,0,0,0,0,0,0 +DISAPPROVED,2009,0,0,0,0,0,0 +DISASSOCIATING,2009,0,0,0,0,0,0 +DISASTERS,2009,0,0,0,0,0,0 +DISAVOWAL,2009,0,0,0,0,0,0 +GRATUITOUSLY,2009,0,0,0,0,0,0 +GREATLY,0,2009,0,0,0,0,0 +GROSSLY,2009,0,0,0,0,0,0 +HALTED,2009,0,0,0,0,0,0 +HAMPERS,2009,0,0,0,0,0,0 +HAPPY,0,2009,0,0,0,0,0 +HARASSMENT,2009,0,0,0,0,0,0 +HARMED,2009,0,0,0,0,0,0 +HARMS,2009,0,0,0,0,0,0 +HARSHLY,2009,0,0,0,0,0,0 +HAZARDS,2009,0,0,0,0,0,0 +NONINFRINGING,0,0,0,2011,0,0,0 +NONPAYMENT,2009,0,0,0,0,0,0 +NONPERFORMING,2009,0,0,0,0,0,0 +DISFAVORS,2009,0,0,0,0,0,0 +PREAMENDMENT,0,0,0,2011,0,0,0 +COMPLICATED,2009,0,0,0,0,0,0 +COMPLICATIONS,2009,0,0,0,0,0,0 +COMPLIMENTING,0,2009,0,0,0,0,0 +MISSTATEMENTS,2009,0,0,0,0,0,0 +MISSTEPS,2009,0,0,0,0,0,0 +MISTAKES,2009,0,0,0,0,0,0 +MISUNDERSTAND,2009,0,0,0,0,0,0 +MISUSE,2009,0,0,0,0,0,0 +MONOPOLISTIC,2009,0,0,0,0,0,0 +MONOPOLIZED,2009,0,0,0,0,0,0 +MORATORIA,2009,0,0,0,0,0,0 +MOTHBALLED,2009,0,0,0,0,0,0 +MUTANDIS,0,0,0,2011,0,0,0 +SLIPPAGES,2009,0,0,0,0,0,0 +SLOWED,2009,0,0,0,0,0,0 +SLOWLY,2009,0,0,0,0,0,0 +SLUGGISHNESS,2009,0,0,0,0,0,0 +SMOOTHS,0,2009,0,0,0,0,0 +DEPRESSES,2009,0,0,0,0,0,0 +DEPRIVED,2009,0,0,0,0,0,0 +DERELICTION,2009,0,0,0,0,0,0 +DEROGATING,0,0,0,2009,0,0,0 +DESIGNATOR,0,0,0,2011,0,0,0 +DESPITE,0,2009,0,0,0,0,0 +DESTABILIZING,2009,0,2009,0,0,0,0 +DESTROYING,2009,0,0,0,0,0,0 +DETAIN,2009,0,0,0,0,0,0 +DETENTIONS,2009,0,0,0,0,0,0 +DETERIORATES,2009,0,0,0,0,0,0 +DETERRED,2009,0,0,0,0,0,0 +DETERRENTS,2009,0,0,0,0,0,0 +DETRACTED,2009,0,0,0,0,0,0 +DETRIMENTALLY,2009,0,0,0,0,0,0 +DEVALUES,2009,0,0,0,0,0,0 +DEVASTATING,2009,0,0,0,0,0,0 +DEVIATES,2009,0,2009,0,0,0,0 +DEVISEES,0,0,0,2011,0,0,0 +DEVOLVING,2009,0,0,0,0,0,0 +DICTATING,0,0,0,0,0,0,2009 +DIFFERS,0,0,2009,0,0,0,0 +DIFFICULTY,2009,0,0,0,0,0,0 +ASCENDANTS,0,0,0,2009,0,0,0 +ASSAULTS,2009,0,0,0,0,0,0 +ASSIGNATIONS,0,0,0,2009,0,0,0 +ASSUMES,0,0,2009,0,0,0,0 +ASSURE,0,2009,0,0,0,0,0 +ATTAIN,0,2009,0,0,0,0,0 +ATTAINMENTS,0,2009,0,0,0,0,0 +ATTESTATIONS,0,0,0,2009,0,0,0 +ATTORNEY,0,0,0,2009,0,0,0 +ATTRACTIVE,0,2009,0,0,0,0,0 +BACKDATING,2009,0,0,0,0,0,0 +BAILEE,0,0,0,2009,0,0,0 +BAILMENT,0,0,0,2011,0,0,0 +BANKRUPT,2009,0,0,0,0,0,0 +BANKRUPTING,2009,0,0,0,0,0,0 +CLAIMANTS,0,0,0,2009,0,0,0 +CLARIFICATION,0,0,2009,0,0,0,0 +CLEARLY,0,0,0,0,2009,0,0 +CLOSING,-2020,0,0,0,0,0,0 +CODEFENDANT,0,0,0,2011,0,0,0 +CODIFICATION,0,0,0,2009,0,0,0 +CODIFY,0,0,0,2009,0,0,0 +COERCES,2009,0,0,0,0,0,0 +COLLABORATED,0,2009,0,0,0,0,0 +COLLABORATIONS,0,2009,0,0,0,0,0 +COLLAPSE,2009,0,0,0,0,0,0 +COLLISION,2009,0,0,0,0,0,0 +COLLUDES,2009,0,0,0,0,0,0 +COLLUSIVE,2009,0,0,0,0,0,0 +POSITIVELY,0,2009,0,0,0,0,0 +POSSIBLE,0,0,2009,0,0,2009,0 +POSTCONTRACT,0,0,0,2011,0,0,0 +POSTPONEMENT,2009,0,0,0,0,0,0 +WRITEDOWN,2009,0,0,0,0,0,0 +WRITS,0,0,0,2009,0,0,0 +WRONGFUL,2009,0,0,0,0,0,0 +HONOR,0,2009,0,0,0,0,0 +HONORS,0,2009,0,0,0,0,0 +REARGUMENT,0,0,0,2011,0,0,0 +REASSESSING,0,0,2009,0,0,0,0 +REASSIGNED,2009,0,0,0,0,0,0 +REASSIGNS,2009,0,0,0,0,0,0 +REBUT,0,0,0,2009,0,0,0 +REBUTTAL,0,0,0,2009,0,0,0 +RECALCULATE,0,0,2009,0,0,0,0 +RECALCULATION,0,0,2009,0,0,0,0 +RECALLING,2009,0,0,0,0,0,0 +RECESSIONARY,2009,0,0,0,0,0,0 +RECKLESSNESS,2009,0,0,0,0,0,0 +RECONSIDERS,0,0,2009,0,0,0,0 +RECOUPMENTS,0,0,0,2009,0,0,0 +RECTIFICATIONS,0,0,0,2009,0,0,0 +RECUSES,0,0,0,2014,0,0,0 +REDACTING,2009,0,0,2009,0,0,0 +REDEFAULTED,2012,0,0,0,0,0,0 +REDRESSES,2009,0,0,0,0,0,0 +REEXAMINING,0,0,2009,0,0,0,0 +REFILE,0,0,0,2009,0,0,0 +REFRAIN,0,0,0,0,0,0,2009 +REFUSALS,2009,0,0,0,0,0,0 +REFUSING,2009,0,0,0,0,0,0 +REGULATE,0,0,0,2009,0,0,0 +REGULATION,0,0,0,2009,0,0,0 +REGULATORS,0,0,0,2009,0,0,0 +REHEARING,0,0,0,2009,0,0,0 +VARIABLY,0,0,2009,0,0,0,0 +VARIANTS,0,0,2009,0,0,0,0 +VARIES,0,0,2009,0,0,0,0 +VENDEES,0,0,0,2011,0,0,0 +VERSATILITY,0,2009,0,0,0,0,0 +VIBRANT,0,2009,0,0,0,0,0 +VIOLATES,2009,0,0,0,0,0,0 +VIOLATIVE,2009,0,0,2009,0,0,0 +VIOLENT,2009,0,0,0,0,0,0 +VITIATES,2009,0,0,0,0,0,0 +VOIDED,2009,0,0,2009,0,0,0 +VOLATILITY,2009,0,2009,0,0,0,0 +VULNERABLY,2009,0,0,0,0,0,0 +WARNINGS,2009,0,0,0,0,0,0 +WASTED,2009,0,0,0,0,0,0 +WEAKEN,2009,0,0,0,0,0,0 +WEAKER,2009,0,0,0,0,0,0 +WEAKNESSES,2009,0,0,0,0,0,0 +DISHONORABLE,2009,0,0,0,0,0,0 +DISHONORS,2009,0,0,0,0,0,0 +DISINTERESTEDLY,2009,0,0,0,0,0,0 +DISLOYALTY,2009,0,0,0,0,0,0 +DISMISSAL,2009,0,0,0,0,0,0 +DISMISSING,2009,0,0,0,0,0,0 +DISPARAGEMENT,2009,0,0,0,0,0,0 +DISPARAGINGLY,2009,0,0,0,0,0,0 +DISPLACED,2009,0,0,0,0,0,0 +DISPLACING,2009,0,0,0,0,0,0 +DISPOSSESSED,2009,0,0,0,0,0,0 +DISPOSSESSORY,0,0,0,2011,0,0,0 +DISPROPORTIONATELY,2009,0,0,0,0,0,0 +DISPUTING,2009,0,0,0,0,0,0 +DISQUALIFIES,2009,0,0,0,0,0,0 +DISREGARDED,2009,0,0,0,0,0,0 +DISREPUTE,2009,0,0,0,0,0,0 +DISRUPTION,2009,0,0,0,0,0,0 +DISSATISFACTION,2009,0,0,0,0,0,0 +DISSENTER,2009,0,0,0,0,0,0 +DISSIDENT,2009,0,0,0,0,0,0 +DISTINCTION,0,2009,0,0,0,0,0 +DISTINCTIVENESS,0,2009,0,0,0,0,0 +DISTORTION,2009,0,0,0,0,0,0 +DISTRACTED,2009,0,0,0,0,0,0 +DISTRACTS,2009,0,0,0,0,0,0 +DISTRIBUTEE,0,0,0,2009,0,0,0 +DISTURBANCES,2009,0,0,0,0,0,0 +DIVERSION,2009,0,0,0,0,0,0 +DIVERTS,2009,0,0,0,0,0,0 +DIVESTITURE,2009,0,0,0,0,0,0 +DIVESTS,2009,0,0,0,0,0,0 +DIVULGED,2009,0,0,0,0,0,0 +DOCKETED,0,0,0,2009,0,0,0 +DOUBT,2009,0,2009,0,0,0,0 +DOWNGRADE,2009,0,0,0,0,0,0 +DOWNSIZE,2009,0,0,0,0,0,0 +DOWNSIZINGS,2009,0,0,0,0,0,0 +DOWNTURNS,2009,0,0,0,0,0,0 +DRASTIC,2009,0,0,0,0,0,0 +DREAM,0,2009,0,0,0,0,0 +DULY,0,0,0,2009,0,0,0 +DYSFUNCTIONS,2009,0,0,0,0,0,0 +EARMARKS,0,0,0,0,0,0,2009 +EASY,0,2009,0,0,0,0,0 +EFFICIENT,0,2009,0,0,0,0,0 +EJECTMENT,0,0,0,2011,0,0,0 +EMBARGOING,2009,0,0,0,0,0,0 +EMBARRASSING,2009,0,0,0,0,0,0 +EMBEZZLED,2009,0,0,0,0,0,0 +EMBEZZLES,2009,0,0,0,0,0,0 +EMPOWERING,0,2009,0,0,0,0,0 +ENABLES,0,2009,0,0,0,0,0 +ENCOURAGES,0,2009,0,0,0,0,0 +ENCROACHES,2009,0,0,0,0,0,0 +ENCUMBER,2009,0,0,2009,0,0,2009 +ENCUMBRANCE,2009,0,0,2009,0,0,2009 +ENDANGER,2009,0,0,0,0,0,0 +ENDANGERS,2009,0,0,0,0,0,0 +ENFORCEABLY,0,0,0,2011,0,0,0 +ENHANCEMENTS,0,2009,0,0,0,0,0 +ENJOINED,2009,0,0,0,0,0,0 +ENJOYABLE,0,2009,0,0,0,0,0 +ENJOYMENT,0,2009,0,0,0,0,0 +ENTAILING,0,0,0,0,0,0,2009 +PASSU,0,0,0,2009,0,0,0 +PENALIZED,2009,0,0,0,0,0,0 +PENALTY,2009,0,0,0,0,0,0 +PERFECTLY,0,2009,0,0,0,0,0 +PERILS,2009,0,0,0,0,0,0 +PERMISSIONS,0,0,0,0,0,0,2009 +PERMITTING,0,0,0,0,0,0,2009 +PERPETRATING,2009,0,0,2009,0,0,0 +PERSISTENCE,2009,0,0,0,0,0,0 +PERSISTS,2009,0,0,0,0,0,0 +PERVASIVENESS,2009,0,0,0,0,0,0 +PETITIONERS,0,0,0,2009,0,0,0 +PICKET,2009,0,0,0,0,0,0 +HEREBY,0,0,0,2009,0,0,0 +HEREFROM,0,0,0,2009,0,0,0 +HEREINBEFORE,0,0,0,2009,0,0,0 +HERETO,0,0,0,2009,0,0,0 +HEREUPON,0,0,0,2009,0,0,0 +HIGHEST,0,2009,0,0,2009,0,0 +HINDERS,2009,0,0,0,0,0,0 +WHATEVER,0,0,0,2009,0,0,0 +WHEREAS,0,0,0,2009,0,0,0 +WHEREIN,0,0,0,2009,0,0,0 +WHEREUNDER,0,0,0,2011,0,0,0 +WHOMEVER,0,0,0,2009,0,0,0 +WILL,0,0,0,0,2009,0,0 +WIN,0,2009,0,0,0,0,0 +WITNESS,0,0,0,2009,0,0,0 +FLUCTUATED,0,0,2009,0,0,0,0 +FLUCTUATIONS,0,0,2009,0,0,0,0 +FORBEARANCES,0,0,0,2009,0,0,0 +FORBIDDEN,2009,0,0,0,0,0,2009 +FORCED,2009,0,0,0,0,0,0 +FOREBEARS,0,0,0,2009,0,0,0 +FORECLOSING,2009,0,0,0,0,0,0 +FOREGOES,2009,0,0,0,0,0,0 +FORESTALLING,2009,0,0,0,0,0,0 +FORFEITABLE,0,0,0,2009,0,0,0 +FORFEITURE,2009,0,0,0,0,0,0 +FORTHWITH,0,0,0,2009,0,0,0 +FRAUDULENCE,2009,0,0,0,0,0,0 +FRIVOLOUS,2009,0,0,0,0,0,0 +FRUSTRATES,2009,0,0,0,0,0,0 +FRUSTRATIONS,2009,0,0,0,0,0,0 +GAIN,0,2009,0,0,0,0,0 +GOOD,0,2009,0,0,0,0,0 +DISGORGES,2009,0,0,0,0,0,0 +DISGRACEFULLY,2009,0,0,0,0,0,0 +LITIGATED,2009,0,0,2009,0,0,0 +LITIGATIONS,2009,0,0,2009,0,0,0 +LITIGIOUSNESS,0,0,0,2009,0,0,0 +LACKING,2009,0,0,0,0,0,0 +LAGGED,2009,0,0,0,0,0,0 +LAPSED,2009,0,0,0,0,0,0 +LAUNDERING,2009,0,0,0,0,0,0 +LAWFULNESS,0,0,0,2009,0,0,0 +LAWSUIT,0,0,0,2009,0,0,0 +LAYOFF,2009,0,0,0,0,0,0 +LEGAL,0,0,0,2009,0,0,0 +LEGALIZATIONS,0,0,0,2009,0,0,0 +LEGALIZING,0,0,0,2009,0,0,0 +LEGATEES,0,0,0,2009,0,0,0 +FLAWS,2009,0,0,0,0,0,0 +NONRENEWAL,2009,0,0,0,0,0,0 +LIQUIDATING,2009,0,0,0,0,0,0 +LIQUIDATORS,2009,0,0,0,0,0,0 +BENEFICIAL,0,-2020,0,2020,0,0,0 +BENEFIT,0,-2020,0,0,0,0,0 +BENEFITTING,0,2009,0,0,0,0,0 +POORLY,2009,0,0,0,0,0,0 +REINTERPRETATIONS,0,0,2009,0,0,0,0 +REJECT,2009,0,0,0,0,0,0 +REJECTIONS,2009,0,0,0,0,0,0 +RELINQUISHED,2009,0,0,0,0,0,0 +RELINQUISHMENTS,2009,0,0,0,0,0,0 +REMANDED,0,0,0,2009,0,0,0 +REMEDIATED,0,0,0,2009,0,0,0 +REMEDIED,0,0,0,2009,0,0,0 +RENEGOTIATES,2009,0,0,0,0,0,0 +RENOUNCE,2009,0,0,0,0,0,0 +RENOUNCES,2009,0,0,0,0,0,0 +REPLEDGED,0,0,0,2012,0,0,0 +REPOSSESSING,2009,0,0,0,0,0,0 +TROUBLES,2009,0,0,0,0,0,0 +UNACCEPTABLE,2009,0,0,0,0,0,0 +UNANNOUNCED,2009,0,0,0,0,0,0 +UNAPPROVED,2009,0,0,0,0,0,0 +UNAVAILABLE,2009,0,0,0,0,0,2011 +UNCERTAIN,0,0,2009,0,0,2009,0 +UNCLEAR,0,0,2009,0,0,0,0 +UNCOLLECTIBLE,2009,0,0,0,0,0,0 +UNCOMPROMISING,0,0,0,0,2009,0,0 +UNCONSTITUTIONAL,0,0,0,2009,0,0,0 +UNCONTROLLABLE,2009,0,0,0,0,0,0 +UNCOVER,2009,0,0,0,0,0,0 +UNDECIDED,0,0,2009,0,0,0,0 +UNDELIVERED,2009,0,0,0,0,0,0 +UNDERCUTTING,2009,0,0,0,0,0,0 +UNDERESTIMATING,2009,0,0,0,0,0,0 +UNDERMINE,2009,0,0,0,0,0,0 +UNDERPAID,2009,0,0,0,0,0,0 +UNDERPERFORM,2011,0,0,0,0,0,0 +UNDERPERFORMS,2014,0,0,0,0,0,0 +UNDERSTATE,2009,0,0,0,0,0,0 +UNDERSTATES,2009,0,0,0,0,0,0 +UNDESIGNATED,0,0,2009,0,0,0,0 +UNDETECTED,2009,0,0,0,0,0,0 +UNDISCLOSED,2009,0,0,0,0,0,0 +UNDUE,2009,0,0,0,0,0,0 +UNECONOMICALLY,2009,0,0,0,0,0,0 +UNENCUMBERED,0,0,0,2009,0,0,0 +UNEQUIVOCALLY,0,0,0,0,2009,0,0 +UNEXPECTED,2009,0,2009,0,0,0,0 +UNFAMILIAR,0,0,2009,0,0,0,0 +UNFAVORABLY,2009,0,0,0,0,0,0 +UNFITNESS,2009,0,0,0,0,0,0 +UNFORSEEN,2011,0,2011,0,0,0,0 +UNFRIENDLY,2009,0,0,0,0,0,0 +UNHEDGED,0,0,2009,0,0,0,0 +UNINTENDED,2009,0,0,0,0,0,0 +UNJUSTIFIABLE,2009,0,0,0,0,0,0 +UNKNOWING,2009,0,0,0,0,0,0 +UNLAWFUL,2009,0,0,2009,0,0,0 +UNLIQUIDATED,2009,0,0,0,0,0,0 +UNMERITORIOUS,2014,0,0,0,0,0,0 +UNOBSERVABLE,0,0,2009,0,0,0,0 +UNPARALLELED,0,2009,0,0,2009,0,0 +UNPREDICTABILITY,2009,0,2009,0,0,0,0 +UNPRODUCTIVE,2009,0,0,0,0,0,0 +UNPROVEN,0,0,2009,0,0,0,0 +UNREALISTIC,2009,0,0,0,0,0,0 +UNRECEPTIVE,2014,0,0,0,0,0,0 +UNREIMBURSED,2009,0,0,0,0,0,0 +UNREPORTED,2009,0,0,0,0,0,0 +UNSALABLE,2009,0,0,0,0,0,0 +UNSAVORY,2009,0,0,0,0,0,0 +UNSELLABLE,2014,0,0,0,0,0,0 +UNSPECIFIC,0,0,2009,0,0,0,0 +UNSTAYED,0,0,0,2009,0,0,0 +UNSUITABILITY,2009,0,0,0,0,0,0 +UNSURE,2009,0,0,0,0,0,0 +UNSUSTAINABLE,2009,0,0,0,0,0,0 +UNTO,0,0,0,2009,0,0,0 +UNTRUTHFULLY,2009,0,0,0,0,0,0 +UNUSUAL,0,0,2009,0,0,0,0 +UNWELCOME,2009,0,0,0,0,0,0 +UPSET,2009,0,0,0,0,0,0 +URGENT,2009,0,0,0,0,0,0 +USURPED,2009,0,0,2009,0,0,0 +VAGARIES,0,0,2009,0,0,0,0 +VAGUENESSES,0,0,2012,0,0,0,0 +VANDALISM,2009,0,0,0,0,0,0 +PLAINTIFF,2009,0,0,2009,0,0,0 +PLEADED,2009,0,0,0,0,0,0 +PLEAS,2009,0,0,2009,0,0,0 +PLEASURE,0,2009,0,0,0,0,0 +PLEDGEE,0,0,0,2009,0,0,0 +PLEDGOR,0,0,0,2009,0,0,0 +COMPELLED,0,0,0,0,0,0,2009 +COMPLAIN,2009,0,0,0,0,0,0 +COMPLAINING,2009,0,0,0,0,0,0 +COMMITMENTS,0,0,0,0,0,0,2009 +LIMITATIONS,2009,0,0,0,0,0,0 +RESTRAINS,0,0,0,0,0,0,2009 +RESTRICTED,0,0,0,0,0,0,2009 +RESTRICTIVE,0,0,0,0,0,0,2009 +RESTRUCTURE,2009,0,0,0,0,0,0 +RESTRUCTURINGS,2009,0,0,0,0,0,0 +RETALIATING,2009,0,0,0,0,0,0 +RETENDERING,0,0,0,2011,0,0,0 +PRECIPITATED,2009,0,0,0,0,0,0 +PRECLUDED,2009,0,0,0,0,0,2009 +PRECONDITIONS,0,0,0,0,0,0,2009 +PREDECEASES,0,0,0,2009,0,0,0 +PREDICTED,0,0,2009,0,0,0,0 +PREDICTIVE,0,0,2009,0,0,0,0 +PREEMINENCE,0,2009,0,0,0,0,0 +PREJUDICED,2009,0,0,2009,0,0,0 +PRELIMINARILY,0,0,2009,0,0,0,0 +PREMIER,0,2009,0,0,0,0,0 +PRESSING,2009,0,0,0,0,0,0 +PRESUME,0,0,2009,0,0,0,0 +PRESUMPTION,0,0,2009,0,0,0,0 +PREVENT,0,0,0,0,0,0,2009 +PREVENTS,2009,0,0,0,0,0,2009 +PROACTIVELY,0,2009,0,0,0,0,0 +PROBABLE,0,0,2009,0,0,0,0 +PROBATES,0,0,0,2009,0,0,0 +PROBATIONARY,0,0,0,2009,0,0,0 +PROBLEM,2009,0,0,0,0,0,0 +PROFICIENCY,0,2009,0,0,0,0,0 +PROFITABLE,0,2009,0,0,0,0,0 +PROGRESSES,0,2009,0,0,0,0,0 +PROHIBITING,0,0,0,0,0,0,2009 +PROHIBITIVELY,0,0,0,0,0,0,2009 +PROLONGATION,2009,0,0,0,0,0,0 +PROLONGS,2009,0,0,0,0,0,0 +PROMULGATING,0,0,0,2009,0,0,0 +PROMULGATORS,0,0,0,2009,0,0,0 +PROSECUTE,2009,0,0,2009,0,0,0 +PROSECUTION,2009,0,0,2009,0,0,0 +PROSECUTORS,0,0,0,2009,0,0,0 +PROSPEROUS,0,2009,0,0,0,0,0 +PROTESTER,2009,0,0,0,0,0,0 +PROTESTORS,2009,0,0,0,0,0,0 +PROVISO,0,0,0,2009,0,0,0 +PROVOKED,2009,0,0,0,0,0,0 +PUNISHED,2009,0,0,0,0,0,0 +PUNISHMENTS,2009,0,0,0,0,0,0 +PURPORTEDLY,2009,0,0,0,0,0,0 +QUESTIONABLE,2009,0,0,0,0,0,0 +QUESTIONS,2009,0,0,0,0,0,0 +QUITTING,2009,0,0,0,0,0,0 +RANDOMIZE,0,0,2009,0,0,0,0 +RANDOMLY,0,0,2009,0,0,0,0 +RATABLY,0,0,0,2009,0,0,0 +RATIONALIZED,2009,0,0,0,0,0,0 +ABANDONING,2009,0,0,0,0,0,0 +ABDICATED,2009,0,0,0,0,0,0 +ABDICATIONS,2009,0,0,0,0,0,0 +ABERRATIONS,2009,0,0,0,0,0,0 +ABIDE,0,0,0,0,0,0,2009 +ABNORMALITIES,2009,0,0,0,0,0,0 +ABOLISHED,2009,0,0,0,0,0,0 +ABROGATE,2009,0,0,2009,0,0,0 +ABROGATION,2009,0,0,2009,0,0,0 +ABRUPTNESS,2009,0,0,0,0,0,0 +ABSOLVE,0,0,0,2009,0,0,0 +ABUNDANCE,0,2009,0,0,0,0,0 +ABUSES,2009,0,0,0,0,0,0 +ABUSIVENESS,2009,0,0,0,0,0,0 +ACCIDENTAL,2009,0,0,0,0,0,0 +ACCOMPLISH,0,2009,0,0,0,0,0 +ACCOMPLISHMENT,0,2009,0,0,0,0,0 +ACCUSE,2009,0,0,0,0,0,0 +ACHIEVE,0,2009,0,0,0,0,0 +ACHIEVES,0,2009,0,0,0,0,0 +ACQUIESCES,2009,0,0,0,0,0,0 +ACQUIT,2009,0,0,2009,0,0,0 +ACQUITTANCE,0,0,0,2009,0,0,0 +ADDENDUMS,0,0,0,2011,0,0,0 +ADJOURNING,0,0,0,2009,0,0,0 +ADJUDGE,0,0,0,2009,0,0,0 +ADJUDICATE,0,0,0,2009,0,0,0 +ADJUDICATION,0,0,0,2009,0,0,0 +ADJUDICATORS,0,0,0,2009,0,0,0 +ADMISSIBLY,0,0,0,2009,0,0,0 +ADULTERATED,2009,0,0,0,0,0,0 +ADVANCEMENT,0,2009,0,0,0,0,0 +ADVANTAGE,0,2009,0,0,0,0,0 +ADVANTAGES,0,2009,0,0,0,0,0 +ADVERSE,2009,0,0,0,0,0,0 +AFFIDAVIT,0,0,0,2009,0,0,0 +AFOREDESCRIBED,0,0,0,2011,0,0,0 +AFTERMATH,2009,0,0,0,0,0,0 +AGGRAVATED,2009,0,0,0,0,0,0 +AGGRAVATIONS,2009,0,0,0,0,0,0 +ALIENATE,2009,0,0,0,0,0,0 +ALIENATION,2009,0,0,0,0,0,0 +ALLEGE,2009,0,0,2009,0,0,0 +ALLEGING,2009,0,0,2009,0,0,0 +ALTERATION,0,0,2009,0,0,0,0 +AMBIGUITY,0,0,2009,0,0,0,0 +AMENDATORY,0,0,0,2009,0,0,0 +AMENDMENTS,0,0,0,2009,0,0,0 +ANNOYANCES,2009,0,0,0,0,0,0 +ANNUL,2009,0,0,0,0,0,0 +ANNULMENTS,2009,0,0,0,0,0,0 +ANOMALOUSLY,2009,0,2009,0,0,0,0 +ANTICIPATE,0,0,2009,0,0,0,0 +ANTICIPATION,0,0,2009,0,0,0,0 +ANTITRUST,2009,0,0,2009,0,0,0 +APPEAL,0,0,0,2009,0,0,0 +APPEALS,0,0,0,2009,0,0,0 +APPEARS,0,0,2009,0,0,2009,0 +APPELLEES,0,0,0,2011,0,0,0 +APPROXIMATELY,0,0,2009,0,0,0,0 +APPROXIMATIONS,0,0,2009,0,0,0,0 +ARBITRABILITY,0,0,0,2011,0,0,0 +ARBITRARY,0,0,2009,0,0,0,0 +ARBITRATING,0,0,0,2009,0,0,0 +ARBITRATIVE,0,0,0,2011,0,0,0 +ARGUED,2009,0,0,0,0,0,0 +ARGUMENTS,2009,0,0,0,0,0,0 +ARREST,2009,0,0,0,0,0,0 +RETRIBUTIONS,2009,0,0,0,0,0,0 +BONA,0,0,0,2009,0,0,0 +BOOST,0,2009,0,0,0,0,0 +BOUND,0,0,0,0,0,0,2011 +BOYCOTTING,2009,0,0,0,0,0,0 +BREACHES,2009,0,0,2009,0,0,0 +BREAKAGES,2009,0,0,0,0,0,0 +BREAKS,2009,0,0,0,0,0,0 +BRIBED,2009,0,0,0,0,0,0 +BRIBING,2009,0,0,0,0,0,0 +BURDEN,2009,0,0,0,0,0,0 +BURDENSOME,2009,0,0,0,0,0,0 +CALAMITY,2009,0,0,0,0,0,0 +CANCELLATION,2009,0,0,0,0,0,0 +CANCELS,2009,0,0,0,0,0,0 +CATASTROPHICALLY,2009,0,0,0,0,0,0 +CAUTIONING,2009,0,0,0,0,0,0 +CAUTIOUSNESS,0,0,2009,0,0,0,0 +CEASING,2009,0,0,0,0,0,0 +CENSURED,2009,0,0,0,0,0,0 +CESSION,0,0,0,2009,0,0,0 +CHALLENGING,2009,0,0,0,0,0,0 +CHATTELS,0,0,0,2009,0,0,0 +CIRCUMVENTING,2009,0,0,0,0,0,0 +SHARPLY,2009,0,0,0,0,0,0 +SHORTFALL,2009,0,0,0,0,0,0 +SHUT,2009,0,0,0,0,0,0 +SHUTTING,2009,0,0,0,0,0,0 +SLANDERS,2009,0,0,0,0,0,0 +REPUDIATE,2009,0,0,0,0,0,0 +REPUDIATION,2009,0,0,0,0,0,0 +REQUIRE,0,0,0,0,0,0,2009 +REQUIRES,0,0,0,0,0,0,2009 +RESCINDED,0,0,0,2009,0,0,0 +RESCISSIONS,0,0,0,2009,0,0,0 +RESIGNED,2009,0,0,0,0,0,0 +RESTATE,2009,0,0,0,0,0,0 +RESTATES,2009,0,0,0,0,0,0 +NONTERMINABLE,0,0,0,2011,0,0,0 +NOTARIZATION,0,0,0,2009,0,0,0 +NOTARIZING,0,0,0,2009,0,0,0 +NUISANCE,2009,0,0,0,0,0,0 +NULLIFIED,2009,0,0,2009,0,0,0 +NULLITIES,0,0,0,2009,0,0,0 +OBJECTION,2009,0,0,0,0,0,0 +OBLIGATE,0,0,0,0,0,0,2009 +OBLIGATION,0,0,0,0,0,0,2009 +OBLIGED,0,0,0,0,0,0,2009 +OBLIGOR,0,0,0,2009,0,0,0 +OBSOLESCENCE,2009,0,0,0,0,0,0 +OBSTRUCT,2009,0,0,0,0,0,0 +OBSTRUCTIONS,2009,0,0,0,0,0,0 +OFFEND,2009,0,0,0,0,0,0 +OFFENDING,2009,0,0,0,0,0,0 +OFFEREES,0,0,0,2011,0,0,0 +OMISSIONS,2009,0,0,0,0,0,0 +OMITTING,2009,0,0,0,0,0,0 +OPPORTUNITIES,0,2009,0,0,0,0,0 +OPPOSES,2009,0,0,0,0,0,0 +OPTIMISTIC,0,2009,0,0,0,0,0 +OUTAGE,2009,0,0,0,0,0,0 +OUTPERFORM,0,2009,0,0,0,0,0 +OVERAGE,2009,0,0,0,0,0,0 +OVERBUILDS,2009,0,0,0,0,0,0 +OVERBURDENING,2009,0,0,0,0,0,0 +OVERCHARGED,2009,0,0,0,0,0,0 +OVERCOMES,2009,0,0,0,0,0,0 +OVERESTIMATED,2009,0,0,0,0,0,0 +OVERESTIMATIONS,2009,0,0,0,0,0,0 +OVERLOADS,2009,0,0,0,0,0,0 +OVERLOOKS,2009,0,0,0,0,0,0 +OVERPRODUCED,2009,0,0,0,0,0,0 +OVERRULE,0,0,0,2009,0,0,0 +OVERRUN,2009,0,0,0,0,0,0 +OVERSHADOWED,2009,0,0,0,0,0,0 +OVERSTATED,2009,0,0,0,0,0,0 +OVERSTATING,2009,0,0,0,0,0,0 +OVERSUPPLYING,2009,0,0,0,0,0,0 +OVERTURNING,2009,0,0,0,0,0,0 +OVERVALUING,2009,0,0,0,0,0,0 +PARI,0,0,0,2009,0,0,0 +NECESSITATES,0,0,0,0,0,0,2009 +NEGATIVES,2009,0,0,0,0,0,0 +NEGLECTING,2009,0,0,0,0,0,0 +NEGLIGENT,2009,0,0,0,0,0,0 +BARRED,2009,0,0,0,0,0,0 +BEAUTIFULLY,0,2009,0,0,0,0,0 +BELIEVING,0,0,2009,0,0,0,0 +IDLE,2009,0,0,0,0,0,0 +IGNORED,2009,0,0,0,0,0,0 +ILLEGAL,2009,0,0,0,0,0,0 +ILLEGIBLE,2009,0,0,0,0,0,0 +ILLIQUIDITY,2009,0,0,0,0,0,0 +IMMATURE,2009,0,0,0,0,0,0 +IMPAIRING,2009,0,0,0,0,0,2011 +IMPASSE,2009,0,0,0,0,0,0 +IMPEDES,2009,0,0,0,0,0,0 +IMPENDING,2009,0,0,0,0,0,0 +IMPERIL,2009,0,0,0,0,0,0 +IMPLICATED,2009,0,0,0,0,0,0 +IMPOSED,0,0,0,0,0,0,2009 +IMPOSITIONS,0,0,0,0,0,0,2009 +IMPOUNDED,2009,0,0,0,0,0,0 +IMPRACTICAL,2009,0,0,0,0,0,0 +IMPRECISION,0,0,2009,0,0,0,0 +IMPRESSES,0,2009,0,0,0,0,0 +IMPRISONMENT,2009,0,0,0,0,0,0 +IMPROPERLY,2009,0,0,0,0,0,0 +IMPROVED,0,2009,0,0,0,0,0 +IMPROVING,0,2009,0,0,0,0,0 +INACCESSIBLE,2009,0,0,0,0,0,0 +INACCURATELY,2009,0,0,0,0,0,0 +INACTIVATED,2009,0,0,0,0,0,0 +INACTIVATIONS,2009,0,0,0,0,0,0 +INADEQUATE,2009,0,0,0,0,0,0 +INADVISABILITY,2009,0,0,0,0,0,0 +INASMUCH,0,0,0,2009,0,0,0 +INCAPACITY,2009,0,0,2009,0,0,0 +INCARCERATING,2009,0,0,2009,0,0,0 +INCIDENCE,2009,0,0,0,0,0,0 +INCOMPATIBILITIES,2009,0,0,0,0,0,0 +INCOMPETENCY,2009,0,0,0,0,0,0 +INCOMPLETE,2009,0,0,0,0,0,0 +INCONSISTENCIES,2009,0,0,0,0,0,0 +INCONTESTABILITY,0,0,0,2009,0,0,0 +INCONVENIENT,2009,0,0,0,0,0,0 +INCREDIBLE,0,2009,0,0,0,0,0 +INDECENT,2009,0,0,0,0,0,0 +INDEFINITELY,0,0,2009,0,0,0,0 +INDEMNIFICATIONS,0,0,0,2009,0,0,0 +INDEMNIFYING,0,0,0,2009,0,0,0 +INDEMNITOR,0,0,0,2009,0,0,0 +INDETERMINATE,0,0,2009,0,0,0,0 +INDICTING,2009,0,0,2009,0,0,0 +INEFFECTIVE,2009,0,0,0,0,0,0 +INEFFICIENCY,2009,0,0,0,0,0,0 +INELIGIBLE,2009,0,0,0,0,0,0 +INEQUITY,2009,0,0,0,0,0,0 +INEXPERIENCE,2009,0,0,0,0,0,0 +INFLUENTIAL,0,2009,0,0,0,0,0 +INFRACTIONS,2009,0,0,2009,0,0,0 +INFRINGEMENTS,2009,0,0,0,0,0,0 +INGENUITY,0,2009,0,0,0,0,0 +INHIBITS,0,0,0,0,0,0,2011 +INJUNCTIVE,0,0,0,2009,0,0,0 +INJURIES,2009,0,0,0,0,0,0 +INNOVATE,0,2009,0,0,0,0,0 +INNOVATION,0,2009,0,0,0,0,0 +INNOVATOR,0,2009,0,0,0,0,0 +INQUIRY,2009,0,0,0,0,0,0 +INSIST,0,0,0,0,0,0,2009 +INSISTS,0,0,0,0,0,0,2009 +INSOLVENT,2009,0,0,0,0,0,0 +INSTABILITY,2009,0,2009,0,0,0,0 +INSUFFICIENTLY,2009,0,0,0,0,0,0 +INTANGIBLES,0,0,2009,0,0,0,0 +INTERFERED,2009,0,0,0,0,0,0 +INTERFERING,2009,0,0,0,0,0,0 +INTERPLEADER,0,0,0,2009,0,0,0 +INTERPOSING,0,0,0,2009,0,0,0 +INTERROGATED,0,0,0,2009,0,0,0 +INTERROGATIONS,0,0,0,2009,0,0,0 +INTERROGATORY,0,0,0,2009,0,0,0 +INTERRUPTION,2009,0,0,0,0,0,0 +INTESTATE,0,0,0,2009,0,0,0 +SPORADIC,0,0,2009,0,0,0,0 +STABILIZATIONS,0,2009,0,0,0,0,0 +STABILIZING,0,2009,0,0,0,0,0 +STAGNATE,2009,0,0,0,0,0,0 +STAGNATION,2009,0,0,0,0,0,0 +STATUTES,0,0,0,2009,0,0,0 +STIPULATED,0,0,0,0,0,0,2009 +STIPULATIONS,0,0,0,0,0,0,2009 +STOPPED,2009,0,0,0,0,0,0 +STRAINED,2009,0,0,0,0,0,0 +STRENGTHEN,0,2009,0,0,0,0,0 +STRENGTHS,0,2009,0,0,0,0,0 +STRESSFUL,2009,0,0,0,0,0,0 +STRICTEST,0,0,0,0,0,0,2009 +STRONGER,0,2009,0,0,0,0,0 +SUBCLAUSES,0,0,0,2009,0,0,0 +SUBJECTION,2009,0,0,0,0,0,0 +SUBLICENSEE,0,0,0,2009,0,0,0 +SUBPOENA,2009,0,0,2009,0,0,0 +SUBROGATION,0,0,0,2009,0,0,0 +SUCCEED,0,2009,0,0,0,0,0 +SUCCESS,0,2009,0,0,0,0,0 +SUDDEN,0,0,2009,0,0,0,0 +SUES,2009,0,0,2009,0,0,0 +SUFFERS,2009,0,0,0,0,0,0 +SUGGESTS,0,0,2009,0,0,2009,0 +SUMMONS,2009,0,0,2009,0,0,0 +SUPERSEDEAS,0,0,0,2011,0,0,0 +SURETIES,0,0,0,2009,0,0,0 +SURPASSES,0,2009,0,0,0,0,0 +SUSPECT,2009,0,0,0,0,0,0 +SUSPENDED,2009,0,0,0,0,0,0 +SUSPENSIONS,2009,0,0,0,0,0,0 +SUSPICIOUSLY,2009,0,0,0,0,0,0 +REVISE,0,0,2009,0,0,0,0 +REVOCATIONS,2009,0,0,2009,0,0,0 +REVOKING,2009,0,0,0,0,0,0 +REVOLUTIONIZING,0,2009,0,0,0,0,0 +REWARDS,0,-2020,0,0,0,0,0 +RIDICULING,2009,0,0,0,0,0,0 +RISKIEST,2009,0,2009,0,0,0,0 +RISKY,2009,0,2009,0,0,0,0 +RUMORS,0,0,2009,0,0,0,0 +SACRIFICES,2009,0,0,0,0,0,0 +SATISFACTORILY,0,2009,0,0,0,0,0 +SATISFY,0,2009,0,0,0,0,0 +SCRUTINIZE,2009,0,0,0,0,0,0 +SCRUTINY,2009,0,0,0,0,0,0 +SEIZED,2009,0,0,0,0,0,0 +SELDOMLY,0,0,2009,0,0,2009,0 +SERIOUS,2009,0,0,0,0,0,0 +SETBACKS,2009,0,0,0,0,0,0 +SEVERABILITY,0,0,0,2009,0,0,0 +SEVERANCES,0,0,0,2009,0,0,0 +SEVERITIES,2009,0,0,0,0,0,0 +ENTHUSIASM,0,2009,0,0,0,0,0 +ENTRENCHED,0,0,0,0,0,0,2009 +ERODING,2009,0,0,0,0,0,0 +ERRED,2009,0,0,0,0,0,0 +ERROR,2009,0,0,0,0,0,0 +ESCALATED,2009,0,0,0,0,0,0 +ESCHEATED,0,0,0,2011,0,0,0 +ESCROWING,0,0,0,2011,0,0,0 +EVADED,2009,0,0,0,0,0,0 +EVASIONS,2009,0,0,0,0,0,0 +EVICTING,2009,0,0,0,0,0,0 +EVIDENTIAL,0,0,0,2011,0,0,0 +EXACERBATES,2009,0,0,0,0,0,0 +EXAGGERATE,2009,0,0,0,0,0,0 +EXAGGERATION,2009,0,0,0,0,0,0 +EXCELLENCE,0,2009,0,0,0,0,0 +EXCEPTIONAL,0,2009,0,0,0,0,0 +EXCISED,0,0,0,2009,0,0,0 +EXCLUSIVE,0,2009,0,0,0,0,0 +EXCLUSIVITY,0,2009,0,0,0,0,0 +EXCULPATING,2009,0,0,2009,0,0,0 +EXECUTOR,0,0,0,2009,0,0,0 +EXECUTRIX,0,0,0,2009,0,0,0 +EXONERATED,2009,0,0,0,0,0,0 +EXONERATIONS,2009,0,0,0,0,0,0 +EXPLOITATIVE,2009,0,0,0,0,0,0 +EXPOSE,2009,0,0,0,0,0,0 +EXPOSURE,0,0,2009,0,0,0,0 +EXPROPRIATES,2009,0,0,0,0,0,0 +EXPULSION,2009,0,0,0,0,0,0 +EXTRACORPOREAL,0,0,0,2011,0,0,0 +SOLVENCY,2009,0,0,0,0,0,0 +SOMETIMES,0,0,2009,0,0,2009,0 +SPAMMERS,2014,0,0,0,0,0,0 +TREMENDOUS,0,2009,0,0,0,0,0 +LOCKOUT,2009,0,0,0,0,0,0 +LOSING,2009,0,0,0,0,0,0 +LOWEST,0,0,0,0,2009,0,0 +MAJEURE,0,0,0,2011,0,0,0 +MALFUNCTIONING,2009,0,0,0,0,0,0 +MALICIOUSLY,2009,0,0,0,0,0,0 +MANDATED,0,0,0,0,0,0,2009 +MANDITORILY,0,0,0,0,0,0,2011 +MANIPULATING,2009,0,0,0,0,0,0 +MARKDOWN,2009,0,0,0,0,0,0 +MEDIATE,0,0,0,2009,0,0,0 +MEDIATION,0,0,0,2009,0,0,0 +MERITORIOUS,0,2009,0,0,0,0,0 +MISAPPLIED,2009,0,0,0,0,0,0 +MISAPPROPRIATE,2009,0,0,0,0,0,0 +MISAPPROPRIATION,2009,0,0,0,0,0,0 +MISCALCULATED,2009,0,0,0,0,0,0 +MISCALCULATIONS,2009,0,0,0,0,0,0 +MISCLASSIFICATIONS,2014,0,0,0,0,0,0 +MISCONDUCT,2009,0,0,0,0,0,0 +MISDIRECTED,2009,0,0,0,0,0,0 +MISHANDLES,2009,0,0,0,0,0,0 +MISINFORMED,2009,0,0,0,0,0,0 +MISINTERPRETATION,2009,0,0,0,0,0,0 +MISINTERPRETS,2009,0,0,0,0,0,0 +MISJUDGING,2009,0,0,0,0,0,0 +MISLABELED,2009,0,0,0,0,0,0 +MISLEAD,2009,0,0,0,0,0,0 +MISLED,2009,0,0,0,0,0,0 +MISMANAGES,2009,0,0,0,0,0,0 +MISMATCHES,2009,0,0,0,0,0,0 +MISPRICING,2014,0,0,0,0,0,0 +MISREPRESENTATIONS,2009,0,0,0,0,0,0 +MISS,2009,0,0,0,0,0,0 +WORSE,2009,0,0,0,0,0,0 +WORSENS,2009,0,0,0,0,0,0 +TOLERATES,2009,0,0,0,0,0,0 +TORTIOUS,0,0,0,2009,0,0,0 +TORTUOUSLY,2009,0,0,0,0,0,0 +FINED,2009,0,0,0,0,0,0 +NONAPPEALABLE,0,0,0,2009,0,0,0 +NONCANCELABLE,0,0,0,0,0,0,2009 +NONCOMPLIANCES,2009,0,0,0,0,0,0 +NONCONFORMITIES,2009,0,0,0,0,0,0 +NONCONTRACTUAL,0,0,0,2011,0,0,0 +NONFIDUCIARY,0,0,0,2011,0,0,0 +NONFUNCTIONAL,2009,0,0,0,0,0,0 +DISCLAIMER,2009,0,0,0,0,0,0 +DISCLOSE,2009,0,0,0,0,0,0 +DISCONTINUANCE,2009,0,0,0,0,0,0 +DISCONTINUE,2009,0,0,0,0,0,0 +DISCOURAGE,2009,0,0,0,0,0,0 +DISCREDIT,2009,0,0,0,0,0,0 +DISCREPANCIES,2009,0,0,0,0,0,0 +SPECULATE,0,0,2009,0,0,0,0 +SPECULATION,0,0,2009,0,0,0,0 +TRAGIC,2009,0,0,0,0,0,0 +TRANSPARENCY,0,2009,0,0,0,0,0 +LEGISLATE,0,0,0,2009,0,0,0 +LEGISLATION,0,0,0,2009,0,0,0 +LEGISLATOR,0,0,0,2009,0,0,0 +LIBEL,0,0,0,2009,0,0,0 +LICENSABLE,0,0,0,2011,0,0,0 +CONCEALING,2009,0,0,0,0,0,0 +CONCEDING,2009,0,0,0,0,0,0 +CONCERNED,2009,0,0,0,0,0,0 +CONCILIATIONS,2009,0,0,0,0,0,0 +CONDEMNATION,2009,0,0,0,0,0,0 +CONDEMNOR,0,0,0,2011,0,0,0 +CONDONE,2009,0,0,0,0,0,0 +CONFESSED,2009,0,0,0,0,0,0 +CONFIDENT,0,2009,0,0,0,0,0 +CONFINEMENTS,2009,0,0,0,0,0,0 +CONFISCATED,2009,0,0,0,0,0,0 +CONFISCATIONS,2009,0,0,0,0,0,0 +CONFLICTING,2009,0,0,0,0,0,0 +CONFRONTATIONAL,2009,0,0,0,0,0,0 +CONFRONTS,2009,0,0,0,0,0,0 +CONFUSING,2009,0,2009,0,0,0,0 +CONSENTED,0,0,0,2009,0,0,0 +CONSPIRACIES,2009,0,0,0,0,0,0 +CONSPIRATORS,2009,0,0,0,0,0,0 +CONSPIRING,2009,0,0,0,0,0,0 +CONSTITUTIONALLY,0,0,0,2009,0,0,0 +CONSTRAINED,0,0,0,0,0,0,2009 +CONSTRAINTS,0,0,0,0,0,0,2009 +CONSTRUED,0,0,0,2012,0,0,0 +CONTEND,2009,0,0,0,0,0,0 +CONTENTION,2009,0,0,0,0,0,0 +CONTESTABILITY,0,0,0,2014,0,0,0 +CONTINGENCIES,0,0,2009,0,0,0,0 +CONTINGENTS,0,0,2009,0,0,0,0 +CONTRACTHOLDERS,0,0,0,2009,0,0,0 +CONTRACTION,2009,0,0,0,0,0,0 +CONTRACTUALLY,0,0,0,2009,0,0,0 +CONTRADICTION,2009,0,0,0,0,0,0 +CONTRARY,2009,0,0,0,0,0,0 +CONTRAVENING,0,0,0,2009,0,0,0 +CONTROVERSIES,2009,0,0,0,0,0,0 +CONTROVERTING,0,0,0,2009,0,0,0 +CONVICT,2009,0,0,2009,0,0,0 +CONVICTIONS,2009,0,0,2009,0,0,0 +CORRECTIONS,2009,0,0,0,0,0,0 +CORRUPTING,2009,0,0,0,0,0,0 +CORRUPTNESS,2009,0,0,0,0,0,0 +COUNSEL,0,0,0,2009,0,0,0 +COUNTERCLAIM,2009,0,0,0,0,0,0 +COUNTERFEIT,2009,0,0,0,0,0,0 +COUNTERFEITING,2009,0,0,0,0,0,0 +COUNTERSIGNOR,0,0,0,2011,0,0,0 +COURT,0,0,0,2009,0,0,0 +COVENANT,0,0,0,0,0,0,2011 +CREATIVE,0,2009,0,0,0,0,0 +CRIME,2009,0,0,2009,0,0,0 +CRIMINALIZE,0,0,0,2014,0,0,0 +CRISES,2009,0,0,0,0,0,0 +CRITICISM,2009,0,0,0,0,0,0 +CRITICIZES,2009,0,0,0,0,0,0 +CROSSROAD,0,0,2009,0,0,0,0 +CULPABILITY,2009,0,0,0,0,0,0 +CURTAIL,2009,0,0,0,0,0,0 +CURTAILMENTS,2009,0,0,0,0,0,0 +CUTBACKS,2009,0,0,0,0,0,0 +CYBERCRIME,2014,0,0,0,0,0,0 +TAINTING,2009,0,0,0,0,0,0 +TENDING,0,0,2009,0,0,0,0 +TERMINABLE,0,0,0,2009,0,0,0 +TERMINATING,2009,0,0,0,0,0,0 +TESTAMENTARY,0,0,0,2009,0,0,0 +THENCE,0,0,0,2009,0,0,0 +THEREAT,0,0,0,2009,0,0,0 +THEREOF,0,0,0,2009,0,0,0 +THERETOFOR,0,0,0,2011,0,0,0 +THEREUPON,0,0,0,2009,0,0,0 +THREATENED,2009,0,0,0,0,0,0 +FAILED,2009,0,0,0,0,0,0 +FAILURE,2009,0,0,0,0,0,0 +FALSELY,2009,0,0,0,0,0,0 +FALSIFIES,2009,0,0,0,0,0,0 +FANTASTIC,0,2009,0,0,0,0,0 +FAULT,2009,0,0,0,0,0,0 +FAVORABLE,0,2009,0,0,0,0,0 +FAVORITE,0,2009,0,0,0,0,0 +FELONIES,2009,0,0,2009,0,0,0 +DAMAGED,2009,0,0,0,0,0,0 +DAMPENED,2009,0,0,0,0,0,0 +DANGERS,2009,0,0,0,0,0,0 +DEADLOCKS,2009,0,0,0,0,0,0 +DEBARMENTS,2009,0,0,0,0,0,0 +DECEDENTS,0,0,0,2009,0,0,0 +DECEIVE,2009,0,0,0,0,0,0 +DECEPTION,2009,0,0,0,0,0,0 +DECLARANT,0,0,0,2011,0,0,0 +DECLINING,2009,0,0,0,0,0,0 +DECREES,0,0,0,2009,0,0,0 +DEFALCATION,0,0,0,2009,0,0,0 +DEFAMATORY,2009,0,0,0,0,0,0 +DEFAMING,2009,0,0,0,0,0,0 +DEFAULTS,2009,0,0,0,0,0,0 +DEFEASED,0,0,0,2009,0,0,0 +DEFEAT,2009,0,0,0,0,0,0 +DEFECT,2009,0,0,0,0,0,0 +DEFEND,2009,0,0,0,0,0,0 +DEFENDED,2009,0,0,0,0,0,0 +DEFER,2009,0,0,0,0,0,0 +DEFICIENT,2009,0,0,0,0,0,0 +DEFINITIVELY,0,0,0,0,2009,0,0 +DEFRAUDS,2009,0,0,0,0,0,0 +DEGRADE,2009,0,0,0,0,0,0 +DELAY,2009,0,0,0,0,0,0 +DELEGABLE,0,0,0,2009,0,0,0 +DELETERIOUS,2009,0,0,0,0,0,0 +DELIGHT,0,2009,0,0,0,0,0 +DELIGHTING,0,2009,0,0,0,0,0 +DELINQUENT,2009,0,0,0,0,0,0 +DELISTED,2009,0,0,0,0,0,0 +DEMISED,2009,0,0,0,0,0,0 +DEMOLISHED,2009,0,0,0,0,0,0 +DEMOLITIONS,2009,0,0,0,0,0,0 +DEMOTING,2009,0,0,0,0,0,0 +DEMURRER,0,0,0,2009,0,0,0 +DENIAL,2009,0,0,0,0,0,0 +DENIGRATE,2009,0,0,0,0,0,0 +DENIGRATION,2009,0,0,0,0,0,0 +DEPENDABILITY,0,2009,0,0,0,0,0 +DEPENDANT,0,0,0,0,0,0,2011 +DEPENDENCY,0,0,2009,0,0,0,0 +DEPLETE,2009,0,0,0,0,0,0 +DEPLETION,2009,0,0,0,0,0,0 +DEPOSES,0,0,0,2009,0,0,0 +DEPOSITIONS,0,0,0,2009,0,0,0 +INTRUSION,2009,0,0,0,0,0,0 +INVALIDATES,2009,0,0,0,0,0,0 +INVENT,0,2009,0,0,0,0,0 +INVENTIONS,0,2009,0,0,0,0,0 +INVENTORS,0,2009,0,0,0,0,0 +INVESTIGATING,2009,0,0,0,0,0,0 +INVOLUNTARY,2009,0,0,0,0,0,0 +IRRECOVERABLY,2009,0,0,0,0,0,0 +IRREGULARLY,2009,0,0,0,0,0,0 +IRREVOCABILITY,0,0,0,2011,0,0,0 +JEOPARDIZED,2009,0,0,0,0,0,0 +JUDICIARIES,0,0,0,2009,0,0,0 +JURISDICTION,0,0,0,2009,0,0,0 +JURISPRUDENCE,0,0,0,2009,0,0,0 +JURORS,0,0,0,2009,0,0,0 +JUSTICES,0,0,0,2009,0,0,0 +KNOWINGLY,2009,0,0,0,0,0,0 +DILIGENT,0,2009,0,0,0,0,0 +DIMINISHES,2009,0,0,0,0,0,0 +DIRECTIVES,0,0,0,0,0,0,2009 +DISADVANTAGES,2009,0,0,0,0,0,0 +DISAFFIRMED,0,0,0,2011,0,0,0 +DISAGREED,2009,0,0,0,0,0,0 +DISAGREES,2009,0,0,0,0,0,0 +DISALLOWED,2009,0,0,0,0,0,0 +DISAPPEARANCE,2009,0,0,0,0,0,0 +DISAPPEARS,2009,0,0,0,0,0,0 +DISAPPOINTINGLY,2009,0,0,0,0,0,0 +DISAPPROVAL,2009,0,0,0,0,0,0 +DISAPPROVES,2009,0,0,0,0,0,0 +DISASSOCIATION,2009,0,0,0,0,0,0 +DISASTROUS,2009,0,0,0,0,0,0 +DISAVOWED,2009,0,0,0,0,0,0 +GREAT,0,-2020,0,0,0,0,0 +GREATNESS,0,2009,0,0,0,0,0 +GROUNDLESS,2009,0,0,0,0,0,0 +HAMPER,2009,0,0,0,0,0,0 +HAPPIEST,0,2009,0,0,0,0,0 +HARASS,2009,0,0,0,0,0,0 +HARDSHIP,2009,0,0,0,0,0,0 +HARMFUL,2009,0,0,0,0,0,0 +HARSH,2009,0,0,0,0,0,0 +HARSHNESS,2009,0,0,0,0,0,0 +NONJUDICIAL,0,0,0,2009,0,0,0 +NONPAYMENTS,2009,0,0,0,0,0,0 \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/go-solution/main.go b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/go-solution/main.go new file mode 100644 index 0000000000000..0765b87574d27 --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/go-solution/main.go @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// beam-playground: +// name: FinalSolution2 +// description: Final challenge solution 2. +// multifile: true +// files: +// - name: analysis.csv +// context_line: 54 +// categories: +// - Quickstart +// complexity: ADVANCED +// tags: +// - hellobeam + +package main + +import ( + "context" + "fmt" + "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/io/textio" + "github.com/apache/beam/sdks/v2/go/pkg/beam/log" + "github.com/apache/beam/sdks/v2/go/pkg/beam/transforms/filter" + "github.com/apache/beam/sdks/v2/go/pkg/beam/transforms/stats" + "github.com/apache/beam/sdks/v2/go/pkg/beam/x/beamx" + "github.com/apache/beam/sdks/v2/go/pkg/beam/x/debug" + "strings" +) + +type Analysis struct { + Word string + Negative string + Positive string + Uncertainty string + Litigious string + Strong string + Weak string + Constraining string +} + +func (a Analysis) toString() string { + return fmt.Sprintf("Word: %s, Negative: %s, Positive: %s, Uncertainty: %s, Litigious: %s, Strong: %s, Weak: %s, Constraining: %s", + a.Word, a.Negative, a.Positive, a.Uncertainty, a.Litigious, a.Strong, a.Weak, a.Constraining) +} + +func main() { + + ctx := context.Background() + + beam.Init() + + p := beam.NewPipeline() + s := p.Root() + + shakespeare := textio.Read(s, "gs://apache-beam-samples/shakespeare/kinglear.txt") + shakespeareWords := getWords(s, shakespeare) + analysis := textio.Read(s, "analysis.csv") + analysisRecords := parseAnalysis(s, analysis) + + result := matchWords(s, shakespeareWords, analysisRecords) + + parts := partition(s, result) + + negativeWords := parts[0] + positiveWords := parts[1] + + negativeWordsCount := extractCountFn("negative", s, negativeWords) + positiveWordsCount := extractCountFn("positive", s, positiveWords) + debug.Print(s, negativeWordsCount) + debug.Print(s, positiveWordsCount) + + negativeWordsCountWithModel := extractModelCountFn("negative-with-model", s, negativeWords) + positiveWordsCountWithModel := extractModelCountFn("positive-with-model", s, positiveWords) + debug.Print(s, negativeWordsCountWithModel) + debug.Print(s, positiveWordsCountWithModel) + + err := beamx.Run(ctx, p) + + if err != nil { + log.Exitf(context.Background(), "Failed to execute job: %v", err) + } +} + +func parseAnalysis(s beam.Scope, input beam.PCollection) beam.PCollection { + return beam.ParDo(s, func(line string, emit func(analysis Analysis)) { + parts := strings.Split(line, ",") + if parts[1] != "Negative" { + emit(Analysis{ + Word: strings.ToLower(parts[0]), + Negative: parts[1], + Positive: parts[2], + Uncertainty: parts[3], + Litigious: parts[4], + Strong: parts[5], + Weak: parts[6], + Constraining: parts[7], + }) + } + }, input) +} + +func getWords(s beam.Scope, input beam.PCollection) beam.PCollection { + return beam.ParDo(s, func(line string, emit func(string)) { + c := strings.Split(strings.ToLower(line), " ") + for _, word := range c { + emit(word) + } + }, input) +} + +func matchWords(s beam.Scope, input beam.PCollection, viewPCollection beam.PCollection) beam.PCollection { + view := beam.SideInput{ + Input: viewPCollection, + } + return beam.ParDo(s, matchFn, input, view) +} + +func matchFn(word string, view func(analysis *Analysis) bool, emit func(Analysis)) { + var newAnalysis Analysis + for view(&newAnalysis) { + if word == newAnalysis.Word { + emit(newAnalysis) + } + } +} + +func partition(s beam.Scope, input beam.PCollection) []beam.PCollection { + return beam.Partition(s, 3, func(analysis Analysis) int { + if analysis.Negative != "0" { + return 0 + } + if analysis.Positive != "0" { + return 1 + } + return 2 + }, input) +} + +func extractCountFn(prefix string, s beam.Scope, input beam.PCollection) beam.PCollection { + col := beam.ParDo(s, func(analysis Analysis, emit func(string2 string)) { + emit(prefix) + }, input) + return stats.Count(s, col) +} + +func extractModelCountFn(prefix string, s beam.Scope, input beam.PCollection) beam.PCollection { + col := filter.Include(s, input, func(analysis Analysis) bool { + return analysis.Strong != "0" || analysis.Weak != "0" + }) + result := beam.ParDo(s, func(analysis Analysis, emit func(string2 string)) { + emit(prefix) + }, col) + return stats.Count(s, result) +} diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/hint1.md b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/hint1.md new file mode 100644 index 0000000000000..e95938f45123c --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/hint1.md @@ -0,0 +1,182 @@ + +{{if (eq .Sdk "go")}} +1. Process the file with the analyzed words to return a `PCollection` with `Analysis` objects. + + Write own DoFn `func parseAnalysis(s beam.Scope, input beam.PCollection) beam.PCollection { + return beam.ParDo(s, func(line string, emit func(analysis Analysis)) { + parts := strings.Split(line, ",") + if parts[0] != "Word" { + emit(Analysis{ + Word: strings.ToLower(parts[0]), + Negative: parts[1], + Positive: parts[2], + Uncertainty: parts[3], + Litigious: parts[4], + Strong: parts[5], + Weak: parts[6], + Constraining: parts[7], + }) + } + }, input) + }` +2. Add a fixed-window that runs for 30 seconds. And add a trigger that works after the first element with a delay of 5 seconds. + + Window and trigger: `trigger := trigger.AfterEndOfWindow().EarlyFiring(trigger.AfterProcessingTime(). + PlusDelay(5 * time.Second)). + LateFiring(trigger.Repeat(trigger.AfterCount(1))) + fixedWindowedItems := beam.WindowInto(s, window.NewFixedWindows(30*time.Second), shakespeareWords, + beam.Trigger(trigger), + beam.AllowedLateness(30*time.Second), + beam.PanesDiscard(), + ) + ` + +3. Write your own function that works with `side-input`. Its logic should check whether the words from shakespeare are contained in the list of analyzed words + + Function: `func matchWords(s beam.Scope, input beam.PCollection, viewPCollection beam.PCollection) beam.PCollection { + view := beam.SideInput{ + Input: viewPCollection, + } + return beam.ParDo(s, matchFn, input, view) + } + func matchFn(word string, view func(analysis *Analysis) bool, emit func(analysis Analysis)) { + var newAnalysis Analysis + for view(&newAnalysis) { + if word == newAnalysis.Word { + emit(newAnalysis) + } + } + }` +4. Divide the words into portions in the first **positive** words. In the **second** negative. And all the others in the third. + + Partition:`func partition(s beam.Scope, input beam.PCollection) []beam.PCollection { + return beam.Partition(s, 3, func(analysis Analysis) int { + if analysis.Negative != "0" { + return 0 + } + if analysis.Positive != "0" { + return 1 + } + return 2 + }, input) + } + ` +5. To calculate the count with windows, use `ParDo` with `stats.Count(s, col)`. + + Apply the transformation: `func extractCountFn(prefix string, s beam.Scope, input beam.PCollection) beam.PCollection { + col := beam.ParDo(s, func(analysis Analysis, emit func(string2 string)) { + emit(prefix) + }, input) + return stats.Count(s, col) + }` + +6. To identify words with amplifying effects, you need to add a filter. + + Function: `func extractModelCountFn(prefix string, s beam.Scope, input beam.PCollection) beam.PCollection { + col := filter.Include(s, input, func(analysis Analysis) bool { + return analysis.Strong != "0" || analysis.Weak != "0" + }) + result := beam.ParDo(s, func(analysis Analysis, emit func(string2 string)) { + emit(prefix) + }, col) + return stats.Count(s, result) + }` +{{end}} + +{{if (eq .Sdk "java")}} +1. Change `getAnalysisPCollection` so that it returns a `PCollection` with `Row` objects. + + Write own DoFn `static class SentimentAnalysisExtractFn extends DoFn { + @ProcessElement + public void processElement(ProcessContext c) { + String[] items = c.element().split(","); + if (!items[1].equals("Negative")) { + c.output(Row.withSchema(schema) + .addValues(items[0].toLowerCase(), items[1], items[2], items[3], items[4], items[5], items[6], items[7]) + .build()); + } + } + }` +2. To use the analyzed words in the `side-input`, turn to `.apply(View.asList())` +3. Add a fixed-window that runs for 30 seconds `Window.into(Fixed Windows.of(Duration.standard Seconds(30)))`. And add a trigger that works after the first element with a delay of 5 seconds `AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.standardSeconds(5))` +4. Write your own function that works with `side-input`. Its logic should check whether the words from shakespeare are contained in the list of analyzed words + + Function: `static PCollection getAnalysis(PCollection pCollection, PCollectionView> viewAnalysisPCollection) { + return pCollection.apply(ParDo.of(new DoFn() { + @ProcessElement + public void processElement(@Element String word, OutputReceiver out, ProcessContext context) { + List analysisPCollection = context.sideInput(viewAnalysisPCollection); + analysisPCollection.forEach(it -> { + if (it.getString("word").equals(word)) { + out.output(it); + } + }); + } + }).withSideInputs(viewAnalysisPCollection)).setCoder(RowCoder.of(schema)); + }` +5. Divide the words into portions in the first **positive** words. In the **second** negative. And all the others in the third. + + Partition:`static PCollectionList getPartitions(PCollection input) { + return input + .apply(Partition.of(3, + (Partition.PartitionFn) (analysis, numPartitions) -> { + if (!analysis.getString("positive").equals("0")) { + return 0; + } + if (!analysis.getString("negative").equals("0")) { + return 1; + } + return 2; + })); + }` +6. To calculate the count with windows, use `Combine.globally` with `withoutDefaults()`. Apply the transformation `.apply(Combine.globally(Count.combineFn()).withoutDefaults())` + +7. To identify words with amplifying effects, you need to add a filter `.apply(Filter.by(it -> !it.getString("strong").equals("0") || !it.getString("weak").equals("0")))` +{{end}} +{{if (eq .Sdk "python")}} +1. Process the file with the analyzed words to return a `PCollection` with `Analysis` objects. + + Write own DoFn `class ExtractAnalysis(beam.DoFn): + def process(self, element): + items = re.split(r',(?=(?:[^"]*"[^"]*")*[^"]*$)', element) + if items[1] != 'Negative': + yield Analysis(items[0].lower(), items[1], items[2], items[3], items[4], items[5], items[6], items[7]) + ` +2. Add a fixed-window that runs for 30 seconds. And add a trigger that works after the first element with a delay of 5 seconds. + + Window and trigger: `windowed_words = (shakespeare + | 'Window' >> beam.WindowInto(window.FixedWindows(30), trigger=trigger.AfterWatermark( + early=trigger.AfterProcessingTime(5).has_ontime_pane(), late=trigger.AfterAll()), allowed_lateness=30, + accumulation_mode=trigger.AccumulationMode.DISCARDING))` +3. To use the analyzed words in the `side-input`, turn to `windowed_words | beam.ParDo(MatchWordDoFn(), beam.pvalue.AsList(analysis))` +4. Write your own function that works with `side-input`. Its logic should check whether the words from shakespeare are contained in the list of analyzed words + + Function: `class MatchWordDoFn(beam.DoFn): + def process(self, element, analysis): + for a in analysis: + if a.word == element: + yield a` +5. Divide the words into portions in the first **positive** words. In the **second** negative. And all the others in the third. + Partition: `class Partition(beam.PTransform):def expand(self, pcoll):return pcoll | beam.Partition(self._analysis_partition_fn, 3) + @staticmethod + def _analysis_partition_fn(analysis, num_partitions): + if analysis.positive != "0": + return 0 + elif analysis.negative != "0": + return 1 + else:return 2 + ` +6. To calculate the count with windows, use `beam.CombineGlobally` with `withoutDefaults()`. Apply the transformation `beam.CombineGlobally(CountCombineFn()).without_defaults()` + +7. To identify words with amplifying effects, you need to add a filter `beam.Filter(lambda analysis: analysis.strong != '0' or analysis.weak != '0')` +{{end}} \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/java-challenge/Task.java b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/java-challenge/Task.java new file mode 100644 index 0000000000000..99eab35d359f2 --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/java-challenge/Task.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// beam-playground: +// name: FinalChallenge2 +// description: Final challenge 2. +// multifile: true +// files: +// - name: analysis.csv +// context_line: 50 +// categories: +// - Quickstart +// complexity: ADVANCED +// tags: +// - hellobeam + +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.io.TextIO; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.Filter; +import org.apache.beam.sdk.transforms.FlatMapElements; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.TypeDescriptors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; + +public class Task { + private static final Logger LOG = LoggerFactory.getLogger(Task.class); + private static final Integer WINDOW_TIME = 30; + private static final Integer TIME_OUTPUT_AFTER_FIRST_ELEMENT = 5; + private static final Integer ALLOWED_LATENESS_TIME = 1; + + private static final Schema schema = Schema.builder() + .addField("word", Schema.FieldType.STRING) + .addField("negative", Schema.FieldType.STRING) + .addField("positive", Schema.FieldType.STRING) + .addField("uncertainty", Schema.FieldType.STRING) + .addField("litigious", Schema.FieldType.STRING) + .addField("strong", Schema.FieldType.STRING) + .addField("weak", Schema.FieldType.STRING) + .addField("constraining", Schema.FieldType.STRING) + .build(); + + + public static void main(String[] args) { + PipelineOptions options = PipelineOptionsFactory.fromArgs(args).withValidation().create(); + runChallenge(options); + } + + static void runChallenge(PipelineOptions options) { + Pipeline pipeline = Pipeline.create(options); + + PCollection shakespeare = getPCollection(pipeline); + + pipeline.run(); + } + + public static PCollection getPCollection(Pipeline pipeline) { + PCollection rides = pipeline.apply(TextIO.read().from("gs://apache-beam-samples/shakespeare/kinglear.txt")); + return rides.apply(FlatMapElements.into(TypeDescriptors.strings()).via(line -> Arrays.asList(line.toLowerCase().split(" ")))) + .apply(Filter.by((String word) -> !word.isEmpty())); + } + + public static PCollection getAnalysisPCollection(Pipeline pipeline) { + PCollection words = pipeline.apply(TextIO.read().from("analysis.csv")); + return words; + } + + static class LogOutput extends DoFn { + + private final String prefix; + + LogOutput() { + this.prefix = "Processing element"; + } + + LogOutput(String prefix) { + this.prefix = prefix; + } + + @ProcessElement + public void processElement(ProcessContext c) { + LOG.info(prefix + ": " + c.element()); + c.output(c.element()); + } + } +} diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/java-challenge/analysis.csv b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/java-challenge/analysis.csv new file mode 100644 index 0000000000000..5c4a1246021eb --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/java-challenge/analysis.csv @@ -0,0 +1,3877 @@ +Word,Negative,Positive,Uncertainty,Litigious,Strong_Modal,Weak_Modal,Constraining +NONSEVERABLE,0,0,0,2011,0,0,0 +DISFAVOR,2009,0,0,0,0,0,0 +DISGORGE,2009,0,0,0,0,0,0 +COMPLICATES,2009,0,0,0,0,0,0 +COMPLIMENT,0,2009,0,0,0,0,0 +COMPLIMENTS,0,2009,0,0,0,0,0 +MISSTATE,2009,0,0,0,0,0,0 +MISSTATES,2009,0,0,0,0,0,0 +MISTAKE,2009,0,0,0,0,0,0 +MISTAKING,2009,0,0,0,0,0,0 +MISUNDERSTANDING,2009,0,0,0,0,0,0 +MISUSED,2009,0,0,0,0,0,0 +MONOPOLISTS,2009,0,0,0,0,0,0 +MONOPOLIZES,2009,0,0,0,0,0,0 +MORATORIUM,2009,0,0,0,0,0,0 +MOTHBALLING,2009,0,0,0,0,0,0 +SLOW,2009,0,0,0,0,0,0 +SLOWER,2009,0,0,0,0,0,0 +SLOWNESS,2009,0,0,0,0,0,0 +SMOOTH,0,2009,0,0,0,0,0 +DEPRECATION,2009,0,0,0,0,0,0 +DEPRESSING,2009,0,0,0,0,0,0 +DEPRIVES,2009,0,0,0,0,0,0 +DEROGATE,0,0,0,2009,0,0,0 +DEROGATION,0,0,0,2009,0,0,0 +DESIRABLE,0,2009,0,0,0,0,0 +DESTABILIZATION,2009,0,0,0,0,0,0 +DESTINED,0,2009,0,0,0,0,0 +DESTROYS,2009,0,0,0,0,0,0 +DETAINED,2009,0,0,0,0,0,0 +DETER,2009,0,0,0,0,0,0 +DETERIORATING,2009,0,0,0,0,0,0 +DETERRENCE,2009,0,0,0,0,0,0 +DETERRING,2009,0,0,0,0,0,0 +DETRACTING,2009,0,0,0,0,0,0 +DETRIMENTS,2009,0,0,0,0,0,0 +DEVALUING,2009,0,0,0,0,0,0 +DEVASTATION,2009,0,0,0,0,0,0 +DEVIATING,2009,0,2009,0,0,0,0 +DEVOLVE,2009,0,0,0,0,0,0 +DICTATE,0,0,0,0,0,0,2009 +DIFFER,0,0,2009,0,0,0,0 +DIFFICULT,2009,0,0,0,0,0,0 +ASSAULT,2009,0,0,0,0,0,0 +ASSERTABLE,0,0,0,2011,0,0,0 +ASSUMABLE,0,0,0,2009,0,0,0 +ASSUMING,0,0,2009,0,0,0,0 +ASSURED,0,2009,0,0,0,0,0 +ATTAINED,0,2009,0,0,0,0,0 +ATTAINS,0,2009,0,0,0,0,0 +ATTESTED,0,0,0,2009,0,0,0 +ATTORNEYS,0,0,0,2009,0,0,0 +ATTRACTIVENESS,0,2009,0,0,0,0,0 +BAD,2009,0,0,0,0,0,0 +BAILEES,0,0,0,2011,0,0,0 +BAILOUT,2009,0,0,0,0,0,0 +BANKRUPTCIES,2009,0,0,0,0,0,0 +BANKRUPTS,2009,0,0,0,0,0,0 +CLAIM,0,0,0,2009,0,0,0 +CLAIMHOLDER,0,0,0,2014,0,0,0 +CLARIFICATIONS,0,0,2009,0,0,0,0 +CLOSED,-2020,0,0,0,0,0,0 +CLOSINGS,2009,0,0,0,0,0,0 +CODEFENDANTS,0,0,0,2011,0,0,0 +CODIFICATIONS,0,0,0,2009,0,0,0 +CODIFYING,0,0,0,2009,0,0,0 +COERCING,2009,0,0,0,0,0,0 +COLLABORATES,0,2009,0,0,0,0,0 +COLLABORATIVE,0,2009,0,0,0,0,0 +COLLAPSED,2009,0,0,0,0,0,0 +COLLISIONS,2009,0,0,0,0,0,0 +COLLUDING,2009,0,0,0,0,0,0 +POSES,2009,0,0,0,0,0,0 +POSSESSORY,0,0,0,2009,0,0,0 +POSSIBLY,0,0,2009,0,0,2009,0 +POSTJUDGMENT,0,0,0,2011,0,0,0 +POSTPONEMENTS,2009,0,0,0,0,0,0 +WRITEDOWNS,2009,0,0,0,0,0,0 +WRONG,2009,0,0,0,0,0,0 +WRONGFULLY,2009,0,0,0,0,0,0 +HONORABLE,0,-2020,0,0,0,0,0 +HOSTILE,2009,0,0,0,0,0,0 +REASSESS,0,0,2009,0,0,0,0 +REASSESSMENT,2009,0,2009,0,0,0,0 +REASSIGNING,2009,0,0,0,0,0,0 +REBOUND,0,2009,0,0,0,0,0 +REBUTS,0,0,0,2009,0,0,0 +REBUTTALS,0,0,0,2009,0,0,0 +RECALCULATED,0,0,2009,0,0,0,0 +RECALCULATIONS,0,0,2009,0,0,0,0 +RECALLS,2009,0,0,0,0,0,0 +RECESSIONS,2009,0,0,0,0,0,0 +RECONSIDER,0,0,2009,0,0,0,0 +RECORDATION,0,0,0,2009,0,0,0 +RECOURSE,0,0,0,2009,0,0,0 +RECUSAL,0,0,0,2011,0,0,0 +RECUSING,0,0,0,2011,0,0,0 +REDACTION,2009,0,0,2009,0,0,0 +REDEFAULTS,2014,0,0,0,0,0,0 +REDRESSING,2009,0,0,0,0,0,0 +REFERENDA,0,0,0,2009,0,0,0 +REFILED,0,0,0,2009,0,0,0 +REFRAINING,0,0,0,0,0,0,2009 +REFUSE,2009,0,0,0,0,0,0 +REGAIN,0,2009,0,0,0,0,0 +REGULATED,0,0,0,2009,0,0,0 +REGULATIONS,0,0,0,2009,0,0,0 +REGULATORY,0,0,0,2009,0,0,0 +REHEARINGS,0,0,0,2009,0,0,0 +VARIABILITY,0,0,2009,0,0,0,0 +VARIANCE,0,0,2009,0,0,0,0 +VARIATION,0,0,2009,0,0,0,0 +VARY,0,0,2009,0,0,0,0 +VERDICT,2009,0,0,2009,0,0,0 +VETOED,2009,0,0,0,0,0,0 +VICTIMS,2009,0,0,0,0,0,0 +VIOLATING,2009,0,0,0,0,0,0 +VIOLATOR,2009,0,0,0,0,0,0 +VIOLENTLY,2009,0,0,0,0,0,0 +VITIATING,2009,0,0,0,0,0,0 +VOIDING,2009,0,0,2009,0,0,0 +VULNERABILITIES,2009,0,0,0,0,0,0 +WARN,2009,0,0,0,0,0,0 +WARNS,2009,0,0,0,0,0,0 +WASTEFUL,2009,0,0,0,0,0,0 +WEAKENED,2009,0,0,0,0,0,0 +WEAKEST,2009,0,0,0,0,0,0 +DISHONESTLY,2009,0,0,0,0,0,0 +DISHONORABLY,2009,0,0,0,0,0,0 +DISINTERESTEDNESS,2009,0,0,0,0,0,0 +DISMAL,2009,0,0,0,0,0,0 +DISMISSALS,2009,0,0,0,0,0,0 +DISORDERLY,2009,0,0,0,0,0,0 +DISPARAGEMENTS,2009,0,0,0,0,0,0 +DISPARITIES,2009,0,0,0,0,0,0 +DISPLACEMENT,2009,0,0,0,0,0,0 +DISPOSE,2009,0,0,0,0,0,0 +DISPOSSESSES,2009,0,0,0,0,0,0 +DISPROPORTION,2009,0,0,0,0,0,0 +DISPUTE,2009,0,0,0,0,0,0 +DISQUALIFICATION,2009,0,0,0,0,0,0 +DISQUALIFY,2009,0,0,0,0,0,0 +DISREGARDING,2009,0,0,0,0,0,0 +DISRUPT,2009,0,0,0,0,0,0 +DISRUPTIONS,2009,0,0,0,0,0,0 +DISSATISFIED,2009,0,0,0,0,0,0 +DISSENTERS,2009,0,0,0,0,0,0 +DISSIDENTS,2009,0,0,0,0,0,0 +DISTINCTIONS,0,2009,0,0,0,0,0 +DISTORT,2009,0,0,0,0,0,0 +DISTORTIONS,2009,0,0,0,0,0,0 +DISTRACTING,2009,0,0,0,0,0,0 +DISTRAINT,0,0,0,2009,0,0,0 +DISTRIBUTEES,0,0,0,2009,0,0,0 +DISTURBED,2009,0,0,0,0,0,0 +DIVERT,2009,0,0,0,0,0,0 +DIVEST,2009,0,0,0,0,0,0 +DIVESTITURES,2009,0,0,0,0,0,0 +DIVORCE,2009,0,0,0,0,0,0 +DIVULGES,2009,0,0,0,0,0,0 +DOCKETING,0,0,0,2009,0,0,0 +DOUBTED,2009,0,2009,0,0,0,0 +DOWNGRADED,2009,0,0,0,0,0,0 +DOWNSIZED,2009,0,0,0,0,0,0 +DOWNTIME,2009,0,0,0,0,0,0 +DOWNWARD,2009,0,0,0,0,0,0 +DRASTICALLY,2009,0,0,0,0,0,0 +DROPPED,2009,0,0,0,0,0,0 +DURESS,2009,0,0,0,0,0,0 +EARMARK,0,0,0,0,0,0,2009 +EASIER,0,2009,0,0,0,0,0 +EFFECTIVE,0,-2020,0,0,0,0,0 +EFFICIENTLY,0,2009,0,0,0,0,0 +EMBARGO,2009,0,0,0,0,0,0 +EMBARRASS,2009,0,0,0,0,0,0 +EMBARRASSMENT,2009,0,0,0,0,0,0 +EMBEZZLEMENT,2009,0,0,0,0,0,0 +EMBEZZLING,2009,0,0,0,0,0,0 +EMPOWERS,0,2009,0,0,0,0,0 +ENABLING,0,2009,0,0,0,0,0 +ENCOURAGING,0,2009,0,0,0,0,0 +ENCROACHING,2009,0,0,0,0,0,0 +ENCUMBERED,2009,0,0,2009,0,0,2009 +ENCUMBRANCER,0,0,0,2009,0,0,0 +ENDANGERED,2009,0,0,0,0,0,0 +ENDORSEE,0,0,0,2011,0,0,0 +ENHANCE,0,2009,0,0,0,0,0 +ENHANCES,0,2009,0,0,0,0,0 +ENJOINING,2009,0,0,0,0,0,0 +ENJOYABLY,0,2009,0,0,0,0,0 +ENJOYS,0,2009,0,0,0,0,0 +ENTAILS,0,0,0,0,0,0,2009 +PATENTEE,0,0,0,2014,0,0,0 +PENALIZES,2009,0,0,0,0,0,0 +PENDING,0,0,2009,0,0,0,0 +PERFECTS,0,2009,0,0,0,0,0 +PERJURY,2009,0,0,2009,0,0,0 +PERMITTED,0,0,0,0,0,0,2009 +PERPETRATE,2009,0,0,2009,0,0,0 +PERPETRATION,2009,0,0,2009,0,0,0 +PERSISTENT,2009,0,0,0,0,0,0 +PERSONAM,0,0,0,2011,0,0,0 +PETITION,0,0,0,2009,0,0,0 +PETITIONING,0,0,0,2009,0,0,0 +PICKETED,2009,0,0,0,0,0,0 +HENCEFORTH,0,0,0,2009,0,0,0 +HEREDITAMENTS,0,0,0,2009,0,0,0 +HEREIN,0,0,0,2009,0,0,0 +HEREINBELOW,0,0,0,2009,0,0,0 +HERETOFORE,0,0,0,2009,0,0,0 +HEREWITH,0,0,0,2009,0,0,0 +HINDER,2009,0,0,0,0,0,0 +HINDRANCE,2009,0,0,0,0,0,0 +WHATSOEVER,0,0,0,2009,0,0,0 +WHEREAT,0,0,0,2009,0,0,0 +WHEREOF,0,0,0,2009,0,0,0 +WHEREUPON,0,0,0,2009,0,0,0 +WHOMSOEVER,0,0,0,2009,0,0,0 +WILLFUL,0,0,0,2009,0,0,0 +WINNER,0,2009,0,0,0,0,0 +WITNESSES,0,0,0,2009,0,0,0 +FLUCTUATES,0,0,2009,0,0,0,0 +FORBADE,0,0,0,2009,0,0,2011 +FORBEARING,0,0,0,2009,0,0,0 +FORBIDDING,2009,0,0,0,0,0,2009 +FORCING,2009,0,0,0,0,0,0 +FORECLOSE,2009,0,0,0,0,0,0 +FORECLOSURE,2009,0,0,0,0,0,0 +FOREGONE,2009,0,0,0,0,0,0 +FORESTALLS,2009,0,0,0,0,0,0 +FORFEITED,2009,0,0,0,0,0,0 +FORFEITURES,2009,0,0,0,0,0,0 +FORWHICH,0,0,0,2012,0,0,0 +FRAUDULENT,2009,0,0,0,0,0,0 +FRIVOLOUSLY,2009,0,0,0,0,0,0 +FRUSTRATING,2009,0,0,0,0,0,0 +FUGITIVE,-2020,0,0,2009,0,0,0 +GAINED,0,2009,0,0,0,0,0 +GRANTOR,0,0,0,2009,0,0,0 +DISGORGING,2009,0,0,0,0,0,0 +DISHONEST,2009,0,0,0,0,0,0 +LITIGANT,2009,0,0,2009,0,0,0 +LITIGATES,2009,0,0,2009,0,0,0 +LITIGATOR,0,0,0,2009,0,0,0 +LACKLUSTER,2009,0,0,0,0,0,0 +LAGGING,2009,0,0,0,0,0,0 +LAPSES,2009,0,0,0,0,0,0 +LAW,0,0,0,2009,0,0,0 +LAWMAKERS,0,0,0,2009,0,0,0 +LAWSUITS,0,0,0,2009,0,0,0 +LAYOFFS,2009,0,0,0,0,0,0 +LEGALESE,0,0,0,2009,0,0,0 +LEGALIZE,0,0,0,2009,0,0,0 +LEGALLY,0,0,0,2009,0,0,0 +NONPRODUCING,2009,0,0,0,0,0,0 +LIQUIDATE,2009,0,0,0,0,0,0 +LIQUIDATION,2009,0,0,0,0,0,0 +BENEFICIALLY,0,2009,0,0,0,0,0 +BENEFITED,0,2009,0,0,0,0,0 +BEST,0,2012,0,0,2009,0,0 +POPULAR,0,2009,0,0,0,0,0 +REINTERPRETED,0,0,2009,0,0,0,0 +REJECTED,2009,0,0,0,0,0,0 +REJECTS,2009,0,0,0,0,0,0 +RELINQUISHES,2009,0,0,0,0,0,0 +RELUCTANCE,2009,0,0,0,0,0,0 +REMANDING,0,0,0,2009,0,0,0 +REMEDIATING,0,0,0,2009,0,0,0 +REMISED,0,0,0,2011,0,0,0 +RENEGOTIATING,2009,0,0,0,0,0,0 +RENOUNCED,2009,0,0,0,0,0,0 +RENOUNCING,2009,0,0,0,0,0,0 +REPLEVIN,0,0,0,2009,0,0,0 +REPOSSESSION,2009,0,0,0,0,0,0 +TURBULENCE,2009,0,2009,0,0,0,0 +UNACCEPTABLY,2009,0,0,0,0,0,0 +UNANTICIPATED,2009,0,0,0,0,0,0 +UNATTRACTIVE,2009,0,0,0,0,0,0 +UNAVOIDABLE,2009,0,0,0,0,0,0 +UNCERTAINLY,0,0,2009,0,0,2009,0 +UNCOLLECTABLE,2009,0,0,0,0,0,0 +UNCOLLECTIBLES,2009,0,0,0,0,0,0 +UNCONFIRMED,0,0,2009,0,0,0,0 +UNCONSTITUTIONALITY,0,0,0,2009,0,0,0 +UNCONTROLLABLY,2009,0,0,0,0,0,0 +UNCOVERED,2009,0,0,0,0,0,0 +UNDEFEASED,0,0,0,2012,0,0,0 +UNDERCAPITALIZED,2009,0,0,0,0,0,0 +UNDERESTIMATE,2009,0,0,0,0,0,0 +UNDERESTIMATION,2009,0,0,0,0,0,0 +UNDERMINED,2009,0,0,0,0,0,0 +UNDERPAYMENT,2009,0,0,0,0,0,0 +UNDERPERFORMANCE,2009,0,0,0,0,0,0 +UNDERPRODUCED,2009,0,0,0,0,0,0 +UNDERSTATED,2009,0,0,0,0,0,0 +UNDERSTATING,2009,0,0,0,0,0,0 +UNDESIRABLE,2009,0,0,0,0,0,0 +UNDETERMINABLE,0,0,2009,0,0,0,0 +UNDISPUTED,0,0,0,0,2009,0,0 +UNDULY,2009,0,0,0,0,0,0 +UNEMPLOYED,2009,0,0,0,0,0,0 +UNENFORCEABILITY,0,0,0,2009,0,0,0 +UNETHICAL,2009,0,0,0,0,0,0 +UNEXPECTEDLY,2009,0,2009,0,0,0,0 +UNFAMILIARITY,0,0,2009,0,0,0,0 +UNFAVOURABLE,2011,0,0,0,0,0,0 +UNFORECASTED,0,0,2011,0,0,0,0 +UNFORTUNATE,2009,0,0,0,0,0,0 +UNFULFILLED,2009,0,0,0,0,0,0 +UNIDENTIFIABLE,0,0,2009,0,0,0,0 +UNINTENTIONAL,2009,0,0,0,0,0,0 +UNJUSTIFIABLY,2009,0,0,0,0,0,0 +UNKNOWINGLY,2009,0,0,0,0,0,0 +UNLAWFULLY,2009,0,0,2009,0,0,0 +UNMARKETABLE,2009,0,0,0,0,0,0 +UNNECESSARILY,2009,0,0,0,0,0,0 +UNOBTAINABLE,2009,0,0,0,0,0,0 +UNPERFORMED,2009,0,0,0,0,0,0 +UNPREDICTABLE,2009,0,2009,0,0,0,0 +UNPROFITABILITY,2011,0,0,0,0,0,0 +UNQUALIFIED,2009,0,0,0,0,0,0 +UNREASONABLE,2009,0,0,0,0,0,0 +UNRECONCILED,0,0,2011,0,0,0,0 +UNRELIABLE,2009,0,0,0,0,0,0 +UNRESOLVED,2009,0,0,0,0,0,0 +UNSALEABLE,2009,0,0,0,0,0,0 +UNSCHEDULED,2009,0,0,0,0,0,0 +UNSETTLED,0,0,2009,0,0,0,0 +UNSPECIFIED,0,0,2009,0,0,0,0 +UNSUBSTANTIATED,2009,0,0,0,0,0,0 +UNSUITABLE,2009,0,0,0,0,0,0 +UNSURPASSED,0,2009,0,0,2009,0,0 +UNTENABLE,2009,0,0,0,0,0,0 +UNTRUSTED,2014,0,0,0,0,0,0 +UNTRUTHFULNESS,2009,0,0,0,0,0,0 +UNUSUALLY,0,0,2009,0,0,0,0 +UNWILLING,2009,0,0,0,0,0,0 +UPTURN,0,2009,0,0,0,0,0 +USURIOUS,2009,0,0,2009,0,0,0 +USURPING,2009,0,0,2009,0,0,0 +VAGUE,0,0,2012,0,0,0,0 +VAGUER,0,0,2012,0,0,0,0 +PLAINTIFFS,2009,0,0,2009,0,0,0 +PLEADING,2009,0,0,2009,0,0,0 +PLEASANT,0,2009,0,0,0,0,0 +PLED,2009,0,0,0,0,0,0 +PLEDGEES,0,0,0,2011,0,0,0 +PLEDGORS,0,0,0,2012,0,0,0 +COMPELLING,0,0,0,0,0,0,2011 +COMPLAINANT,0,0,0,2009,0,0,0 +COMPLAINS,2009,0,0,0,0,0,0 +COMMITS,0,0,0,0,0,0,2009 +LIKELIHOOD,0,0,2009,0,0,0,0 +LIMITING,0,0,0,0,0,0,2009 +RESTRAIN,0,0,0,0,0,0,2009 +RESTRAINT,0,0,0,0,0,0,2009 +RESTRICTING,0,0,0,0,0,0,2009 +RESTRICTIVELY,0,0,0,0,0,0,2009 +RESTRUCTURED,2009,0,0,0,0,0,0 +RETALIATE,2009,0,0,0,0,0,0 +RETALIATION,2009,0,0,0,0,0,0 +PRECAUTION,0,0,2009,0,0,0,0 +PRECIPITOUS,2009,0,0,0,0,0,0 +PRECLUDES,2009,0,0,0,0,0,2009 +PREDATORY,2009,0,0,0,0,0,0 +PREDECEASING,0,0,0,2009,0,0,0 +PREDICTING,0,0,2009,0,0,0,0 +PREDICTOR,0,0,2009,0,0,0,0 +PREEMINENT,0,2009,0,0,0,0,0 +PREJUDICES,2009,0,0,2009,0,0,0 +PRELIMINARY,0,0,2009,0,0,0,0 +PREMIERE,0,2009,0,0,0,0,0 +PRESTIGE,0,2009,0,0,0,0,0 +PRESUMED,0,0,2009,0,0,0,0 +PRESUMPTIONS,0,0,2009,0,0,0,0 +PREVENTED,0,0,0,0,0,0,2009 +PRIMA,0,0,0,2009,0,0,0 +PROBABILISTIC,0,0,2009,0,0,0,0 +PROBABLY,0,0,2009,0,0,0,0 +PROBATING,0,0,0,2009,0,0,0 +PROBATIONER,0,0,0,2009,0,0,0 +PROBLEMATIC,2009,0,0,0,0,0,0 +PROFICIENT,0,2009,0,0,0,0,0 +PROFITABLY,0,2009,0,0,0,0,0 +PROGRESSING,0,2009,0,0,0,0,0 +PROHIBITION,0,0,0,0,0,0,2009 +PROHIBITORY,0,0,0,0,0,0,2009 +PROLONGATIONS,2009,0,0,0,0,0,0 +PROMULGATE,0,0,0,2009,0,0,0 +PROMULGATION,0,0,0,2009,0,0,0 +PRONE,2009,0,0,0,0,0,0 +PROSECUTED,2009,0,0,2009,0,0,0 +PROSECUTIONS,2009,0,0,2009,0,0,0 +PROSPERED,0,2009,0,0,0,0,0 +PROSPERS,0,2009,0,0,0,0,0 +PROTESTERS,2009,0,0,0,0,0,0 +PROTESTS,2009,0,0,0,0,0,0 +PROVISOES,0,0,0,2009,0,0,0 +PROVOKES,2009,0,0,0,0,0,0 +PUNISHES,2009,0,0,0,0,0,0 +PUNITIVE,2009,0,0,0,0,0,0 +PURPORTING,2009,0,0,0,0,0,0 +QUESTIONABLY,2009,0,0,0,0,0,0 +QUIT,2009,0,0,0,0,0,0 +RACKETEER,2009,0,0,0,0,0,0 +RANDOMIZED,0,0,2009,0,0,0,0 +RANDOMNESS,0,0,2009,0,0,0,0 +RATIONALIZATION,2009,0,0,0,0,0,0 +RATIONALIZES,2009,0,0,0,0,0,0 +ABANDONMENT,2009,0,0,0,0,0,0 +ABDICATES,2009,0,0,0,0,0,0 +ABERRANT,2009,0,0,0,0,0,0 +ABETTING,2009,0,0,0,0,0,0 +ABIDING,0,0,0,0,0,0,2009 +ABNORMALITY,2009,0,0,0,0,0,0 +ABOLISHES,2009,0,0,0,0,0,0 +ABROGATED,2009,0,0,2009,0,0,0 +ABROGATIONS,2009,0,0,2009,0,0,0 +ABSENCE,2009,0,0,0,0,0,0 +ABSOLVED,0,0,0,2009,0,0,0 +ABUNDANT,0,2009,0,0,0,0,0 +ABUSING,2009,0,0,0,0,0,0 +ACCESSION,0,0,0,2009,0,0,0 +ACCIDENTALLY,2009,0,0,0,0,0,0 +ACCOMPLISHED,0,2009,0,0,0,0,0 +ACCOMPLISHMENTS,0,2009,0,0,0,0,0 +ACCUSED,2009,0,0,0,0,0,0 +ACHIEVED,0,2009,0,0,0,0,0 +ACHIEVING,0,2009,0,0,0,0,0 +ACQUIESCING,2009,0,0,0,0,0,0 +ACQUITS,2009,0,0,2009,0,0,0 +ACQUITTANCES,0,0,0,2011,0,0,0 +ADEQUATELY,0,2009,0,0,0,0,0 +ADJOURNMENT,0,0,0,2009,0,0,0 +ADJUDGED,0,0,0,2009,0,0,0 +ADJUDICATED,0,0,0,2009,0,0,0 +ADJUDICATIONS,0,0,0,2009,0,0,0 +ADJUDICATORY,0,0,0,2009,0,0,0 +ADMISSION,0,0,0,2009,0,0,0 +ADULTERATING,2009,0,0,0,0,0,0 +ADVANCEMENTS,0,2009,0,0,0,0,0 +ADVANTAGED,0,2009,0,0,0,0,0 +ADVERSARIAL,2009,0,0,0,0,0,0 +ADVERSELY,2009,0,0,0,0,0,0 +AFFIDAVITS,0,0,0,2009,0,0,0 +AFOREMENTIONED,0,0,0,2009,0,0,0 +AFTERMATHS,2009,0,0,0,0,0,0 +AGGRAVATES,2009,0,0,0,0,0,0 +AGGRIEVED,0,0,0,2009,0,0,0 +ALIENATED,2009,0,0,0,0,0,0 +ALIENATIONS,2009,0,0,0,0,0,0 +ALLEGED,2009,0,0,2009,0,0,0 +ALLIANCE,0,2009,0,0,0,0,0 +ALTERATIONS,0,0,2009,0,0,0,0 +AMBIGUOUS,0,0,2009,0,0,0,0 +AMENDED,0,0,0,2009,0,0,0 +AMENDS,0,0,0,2009,0,0,0 +ANNOYED,2009,0,0,0,0,0,0 +ANNULLED,2009,0,0,0,0,0,0 +ANNULS,2009,0,0,0,0,0,0 +ANOMALY,2009,0,2009,0,0,0,0 +ANTICIPATED,0,0,2009,0,0,0,0 +ANTICIPATIONS,0,0,2009,0,0,0,0 +ANYWISE,0,0,0,2009,0,0,0 +APPEALABLE,0,0,0,2009,0,0,0 +APPEAR,0,0,2009,0,0,0,0 +APPELLANT,0,0,0,2009,0,0,0 +APPOINTOR,0,0,0,2011,0,0,0 +APPROXIMATES,0,0,2009,0,0,0,0 +APPURTENANCE,0,0,0,2009,0,0,0 +ARBITRAL,0,0,0,2009,0,0,0 +ARBITRATE,0,0,0,2009,0,0,0 +ARBITRATION,0,0,0,2009,0,0,0 +ARBITRATOR,0,0,0,2009,0,0,0 +ARGUING,2009,0,0,0,0,0,0 +ARREARAGE,2009,0,0,2009,0,0,0 +ARRESTED,2009,0,0,0,0,0,0 +RETROCEDE,0,0,0,2011,0,0,0 +BOLSTERED,0,2009,0,0,0,0,0 +BONAFIDE,0,0,0,2009,0,0,0 +BOOSTED,0,2009,0,0,0,0,0 +BOUNDED,0,0,0,0,0,0,2009 +BOYCOTTS,2009,0,0,0,0,0,0 +BREACHING,2009,0,0,2009,0,0,0 +BREAKDOWN,2009,0,0,0,0,0,0 +BREAKTHROUGH,0,2009,0,0,0,0,0 +BRIBERIES,2009,0,0,0,0,0,0 +BRIDGE,-2020,0,0,0,0,0,0 +BURDENED,2009,0,0,0,0,0,0 +BURNED,2009,0,0,0,0,0,0 +CANCEL,2009,0,0,0,0,0,0 +CANCELLATIONS,2009,0,0,0,0,0,0 +CARELESS,2009,0,0,0,0,0,0 +CATASTROPHE,2009,0,0,0,0,0,0 +CAUTION,2009,0,0,0,0,0,0 +CAUTIONS,2009,0,0,0,0,0,0 +CEASE,2009,0,0,0,0,0,0 +CEDANT,0,0,0,2012,0,0,0 +CENSURES,2009,0,0,0,0,0,0 +CHALLENGE,2009,0,0,0,0,0,0 +CHARGEOFFS,2009,0,0,0,0,0,0 +CHOATE,0,0,0,2011,0,0,0 +CIRCUMVENTION,2009,0,0,0,0,0,0 +SHOCKED,2009,0,0,0,0,0,0 +SHORTFALLS,2009,0,0,0,0,0,0 +SHUTDOWN,2009,0,0,0,0,0,0 +SLANDER,2009,0,0,0,0,0,0 +REPUDIATED,2009,0,0,0,0,0,0 +REPUDIATIONS,2009,0,0,0,0,0,0 +REQUIRED,0,0,0,0,0,0,2009 +REQUIRING,0,0,0,0,0,0,2009 +RESCINDING,0,0,0,2009,0,0,0 +RESIGN,2009,0,0,0,0,0,0 +RESIGNING,2009,0,0,0,0,0,0 +RESTATED,2009,0,0,0,0,0,0 +RESTATING,2009,0,0,0,0,0,0 +NONUSURIOUS,0,0,0,2011,0,0,0 +NOTARIZATIONS,0,0,0,2009,0,0,0 +NOTARY,0,0,0,2009,0,0,0 +NUISANCES,2009,0,0,0,0,0,0 +NULLIFIES,2009,0,0,2009,0,0,0 +NULLITY,0,0,0,2009,0,0,0 +OBJECTIONABLE,2009,0,0,0,0,0,0 +OBLIGATED,0,0,0,0,0,0,2009 +OBLIGATIONS,0,0,0,0,0,0,2009 +OBLIGEE,0,0,0,2009,0,0,0 +OBLIGORS,0,0,0,2009,0,0,0 +OBSOLETE,2009,0,0,0,0,0,0 +OBSTRUCTED,2009,0,0,0,0,0,0 +OCCASIONALLY,0,0,2009,0,0,2009,0 +OFFENDED,2009,0,0,0,0,0,0 +OFFENDS,2009,0,0,0,0,0,0 +OFFEROR,0,0,0,2009,0,0,0 +OMIT,2009,0,0,0,0,0,0 +ONEROUS,2009,0,0,0,0,0,0 +OPPORTUNITY,0,2009,0,0,0,0,0 +OPPOSING,2009,0,0,0,0,0,0 +OPTIONEE,0,0,0,2009,0,0,0 +OUTAGES,2009,0,0,0,0,0,0 +OUTPERFORMED,0,2009,0,0,0,0,0 +OVERAGES,2009,0,0,0,0,0,0 +OVERBUILT,2009,0,0,0,0,0,0 +OVERCAPACITIES,2009,0,0,0,0,0,0 +OVERCHARGES,2009,0,0,0,0,0,0 +OVERCOMING,2009,0,0,0,0,0,0 +OVERESTIMATES,2009,0,0,0,0,0,0 +OVERLOAD,2009,0,0,0,0,0,0 +OVERLOOK,2009,0,0,0,0,0,0 +OVERPAID,2009,0,0,0,0,0,0 +OVERPRODUCES,2009,0,0,0,0,0,0 +OVERRULED,0,0,0,2009,0,0,0 +OVERRUNNING,2009,0,0,0,0,0,0 +OVERSHADOWING,2009,0,0,0,0,0,0 +OVERSTATEMENT,2009,0,0,0,0,0,0 +OVERSUPPLIED,2009,0,0,0,0,0,0 +OVERTLY,2009,0,0,0,0,0,0 +OVERTURNS,2009,0,0,0,0,0,0 +PANIC,2009,0,0,0,0,0,0 +NEARLY,0,0,2009,0,0,2009,0 +NECESSITATING,0,0,0,0,0,0,2009 +NEGLECT,2009,0,0,0,0,0,0 +NEGLECTS,2009,0,0,0,0,0,0 +NEGLIGENTLY,2009,0,0,0,0,0,0 +BARRIER,2009,0,0,0,0,0,0 +BELIEVE,0,0,2009,0,0,0,0 +IDLED,2009,0,0,0,0,0,0 +IGNORES,2009,0,0,0,0,0,0 +ILLEGALITIES,2009,0,0,0,0,0,0 +ILLICIT,2009,0,0,0,0,0,0 +IMBALANCE,2009,0,0,0,0,0,0 +IMMORAL,2009,0,0,0,0,0,0 +IMPAIRMENT,2009,0,0,0,0,0,2011 +IMPASSES,2009,0,0,0,0,0,0 +IMPEDIMENT,2009,0,0,0,0,0,0 +IMPERATIVE,2009,0,0,0,0,0,0 +IMPERMISSIBLE,2009,0,0,0,0,0,0 +IMPLICATES,2009,0,0,0,0,0,0 +IMPOSES,0,0,0,0,0,0,2009 +IMPOSSIBILITY,2009,0,0,0,0,0,0 +IMPOUNDING,2009,0,0,0,0,0,0 +IMPRACTICALITIES,2009,0,0,0,0,0,0 +IMPRECISIONS,0,0,2009,0,0,0,0 +IMPRESSING,0,2009,0,0,0,0,0 +IMPROBABILITY,0,0,2009,0,0,0,0 +IMPROPRIETIES,2009,0,0,0,0,0,0 +IMPROVEMENT,0,2009,0,0,0,0,0 +IMPRUDENT,2009,0,0,0,0,0,0 +INACCURACIES,2009,0,0,0,0,0,0 +INACTION,2009,0,0,0,0,0,0 +INACTIVATES,2009,0,0,0,0,0,0 +INACTIVITY,2009,0,0,0,0,0,0 +INADEQUATELY,2009,0,0,0,0,0,0 +INADVISABLE,2009,0,0,0,0,0,0 +INATTENTION,2009,0,0,0,0,0,0 +INCARCERATE,2009,0,0,2009,0,0,0 +INCARCERATION,2009,0,0,2009,0,0,0 +INCIDENCES,2009,0,0,0,0,0,0 +INCOMPATIBILITY,2009,0,0,0,0,0,0 +INCOMPETENT,2009,0,0,0,0,0,0 +INCOMPLETELY,2009,0,0,0,0,0,0 +INCONSISTENCY,2009,0,0,0,0,0,0 +INCONTESTABLE,0,0,0,2009,0,0,0 +INCORRECT,2009,0,0,0,0,0,0 +INCREDIBLY,0,2009,0,0,0,0,0 +INDEFEASIBLE,2009,0,0,0,0,0,0 +INDEFINITENESS,0,0,2009,0,0,0,0 +INDEMNIFIED,0,0,0,2009,0,0,0 +INDEMNITEE,0,0,0,2009,0,0,0 +INDEMNITORS,0,0,0,2009,0,0,0 +INDICT,2009,0,0,2009,0,0,0 +INDICTMENT,2009,0,0,2009,0,0,0 +INEFFECTIVELY,2009,0,0,0,0,0,0 +INEFFICIENT,2009,0,0,0,0,0,0 +INEQUITABLE,2009,0,0,0,0,0,0 +INEVITABLE,2009,0,0,0,0,0,0 +INEXPERIENCED,2009,0,0,0,0,0,0 +INFORCE,0,0,0,2009,0,0,0 +INFRINGE,2009,0,0,0,0,0,0 +INFRINGER,0,0,0,2009,0,0,0 +INHIBIT,0,0,0,0,0,0,2011 +INIMICAL,2009,0,0,0,0,0,0 +INJURE,2009,0,0,0,0,0,0 +INJURING,2009,0,0,0,0,0,0 +INNOVATED,0,2009,0,0,0,0,0 +INNOVATIONS,0,2009,0,0,0,0,0 +INNOVATORS,0,2009,0,0,0,0,0 +INSECURE,2009,0,0,0,0,0,0 +INSISTED,0,0,0,0,0,0,2009 +INSOFAR,0,0,0,2009,0,0,0 +INSPIRATION,0,2009,0,0,0,0,0 +INSUBORDINATION,2009,0,0,0,0,0,0 +INSURRECTION,2009,0,0,0,0,0,0 +INTEGRITY,0,2009,0,0,0,0,0 +INTERFERENCE,2009,0,0,0,0,0,0 +INTERLOCUTORY,0,0,0,2009,0,0,0 +INTERPOSE,0,0,0,2009,0,0,0 +INTERPOSITION,0,0,0,2009,0,0,0 +INTERROGATES,0,0,0,2009,0,0,0 +INTERROGATOR,0,0,0,2009,0,0,0 +INTERRUPT,2009,0,0,0,0,0,0 +INTERRUPTIONS,2009,0,0,0,0,0,0 +INTIMIDATION,2009,0,0,0,0,0,0 +SPORADICALLY,0,0,2009,0,0,0,0 +STABILIZE,0,2009,0,0,0,0,0 +STABLE,0,2009,0,0,0,0,0 +STAGNATED,2009,0,0,0,0,0,0 +STANDSTILL,2009,0,0,0,0,0,0 +STATUTORILY,0,0,0,2009,0,0,0 +STIPULATES,0,0,0,0,0,0,2009 +STOLEN,2009,0,0,0,0,0,0 +STOPPING,2009,0,0,0,0,0,0 +STRAINING,2009,0,0,0,0,0,0 +STRENGTHENED,0,2009,0,0,0,0,0 +STRESS,2009,0,0,0,0,0,0 +STRESSING,2009,0,0,0,0,0,0 +STRICTLY,0,0,0,0,0,0,2009 +STRONGEST,0,2009,0,0,0,0,0 +SUBDOCKET,0,0,0,2012,0,0,0 +SUBLEASEE,0,0,0,2011,0,0,0 +SUBLICENSOR,0,0,0,2011,0,0,0 +SUBPOENAED,2009,0,0,2009,0,0,0 +SUBSTANDARD,2009,0,0,0,0,0,0 +SUCCEEDED,0,2009,0,0,0,0,0 +SUCCESSES,0,2009,0,0,0,0,0 +SUDDENLY,0,0,2009,0,0,0,0 +SUFFER,2009,0,0,0,0,0,0 +SUGGEST,0,0,2009,0,0,2009,0 +SUING,2009,0,0,2009,0,0,0 +SUMMONSES,2009,0,0,2009,0,0,0 +SUPERSEDED,0,0,0,2009,0,0,0 +SURETY,0,0,0,2009,0,0,0 +SURPASSING,0,2009,0,0,0,0,0 +SUSPECTED,2009,0,0,0,0,0,0 +SUSPENDING,2009,0,0,0,0,0,0 +SUSPICION,2009,0,0,0,0,0,0 +REVISED,0,0,2009,0,0,0,0 +REVOKE,2009,0,0,0,0,0,0 +REVOLUTIONIZE,0,2009,0,0,0,0,0 +REWARD,0,2009,0,0,0,0,0 +RIDICULE,2009,0,0,0,0,0,0 +RISK,0,0,2009,0,0,0,0 +RISKINESS,0,0,2009,0,0,0,0 +ROUGHLY,0,0,2009,0,0,0,0 +SABOTAGE,2009,0,0,0,0,0,0 +SACRIFICIAL,2009,0,0,0,0,0,0 +SATISFACTORY,0,2009,0,0,0,0,0 +SATISFYING,0,2009,0,0,0,0,0 +SCRUTINIZED,2009,0,0,0,0,0,0 +SECRECY,-2020,0,0,0,0,0,0 +SEIZES,2009,0,0,0,0,0,0 +SENTENCED,2009,0,0,2009,0,0,0 +SERIOUSLY,2009,0,0,0,0,0,0 +SETTLEMENT,0,0,0,2009,0,0,0 +SEVERABLE,0,0,0,2009,0,0,0 +SEVERE,2009,0,0,0,0,0,0 +SEVERITY,2009,0,0,0,0,0,0 +ENTHUSIASTIC,0,2009,0,0,0,0,0 +ERODE,2009,0,0,0,0,0,0 +EROSION,2009,0,0,0,0,0,0 +ERRING,2009,0,0,0,0,0,0 +ERRORS,2009,0,0,0,0,0,0 +ESCALATES,2009,0,0,0,0,0,0 +ESCHEATMENT,0,0,0,2011,0,0,0 +ESCROWS,0,0,0,0,0,0,2009 +EVADES,2009,0,0,0,0,0,0 +EVASIVE,2009,0,0,0,0,0,0 +EVICTION,2009,0,0,0,0,0,0 +EVIDENTIARY,0,0,0,2009,0,0,0 +EXACERBATING,2009,0,0,0,0,0,0 +EXAGGERATED,2009,0,0,0,0,0,0 +EXCEEDANCE,0,0,0,2011,0,0,0 +EXCELLENT,0,2009,0,0,0,0,0 +EXCEPTIONALLY,0,2009,0,0,0,0,0 +EXCITED,0,2009,0,0,0,0,0 +EXCLUSIVELY,0,2009,0,0,0,0,0 +EXCULPATE,2009,0,0,2009,0,0,0 +EXCULPATION,2009,0,0,2009,0,0,0 +EXECUTORS,0,0,0,2009,0,0,0 +EXECUTRIXES,0,0,0,2009,0,0,0 +EXONERATES,2009,0,0,0,0,0,0 +EXPLOIT,2009,0,0,0,0,0,0 +EXPLOITED,2009,0,0,0,0,0,0 +EXPOSED,2009,0,0,0,0,0,0 +EXPOSURES,0,0,2009,0,0,0,0 +EXPROPRIATING,2009,0,0,0,0,0,0 +EXPULSIONS,2009,0,0,0,0,0,0 +EXTRAJUDICIAL,0,0,0,2014,0,0,0 +SOLVES,0,2009,0,0,0,0,0 +SOMEWHAT,0,0,2009,0,0,2009,0 +SPAMMING,2014,0,0,0,0,0,0 +TREMENDOUSLY,0,2009,0,0,0,0,0 +LOCKOUTS,2009,0,0,0,0,0,0 +LOSS,2009,0,0,0,0,0,0 +LOYAL,0,2009,0,0,0,0,0 +MALFEASANCE,2009,0,0,0,0,0,0 +MALFUNCTIONS,2009,0,0,0,0,0,0 +MALPRACTICE,2009,0,0,0,0,0,0 +MANDATES,0,0,0,0,0,0,2009 +MANIPULATE,2009,0,0,0,0,0,0 +MANIPULATION,2009,0,0,0,0,0,0 +MARKDOWNS,2009,0,0,0,0,0,0 +MEDIATED,0,0,0,2009,0,0,0 +MEDIATIONS,0,0,0,2009,0,0,0 +MIGHT,0,0,2009,0,0,2009,0 +MISAPPLIES,2009,0,0,0,0,0,0 +MISAPPROPRIATED,2009,0,0,0,0,0,0 +MISAPPROPRIATIONS,2009,0,0,0,0,0,0 +MISCALCULATES,2009,0,0,0,0,0,0 +MISCHARACTERIZATION,2014,0,0,0,0,0,0 +MISCLASSIFIED,2011,0,0,0,0,0,0 +MISDATED,2011,0,0,0,0,0,0 +MISFEASANCE,0,0,0,2009,0,0,0 +MISHANDLING,2009,0,0,0,0,0,0 +MISINFORMING,2009,0,0,0,0,0,0 +MISINTERPRETATIONS,2009,0,0,0,0,0,0 +MISJUDGE,2009,0,0,0,0,0,0 +MISJUDGMENT,2009,0,0,0,0,0,0 +MISLABELING,2009,0,0,0,0,0,0 +MISLEADING,2009,0,0,0,0,0,0 +MISMANAGE,2009,0,0,0,0,0,0 +MISMANAGING,2009,0,0,0,0,0,0 +MISMATCHING,2009,0,0,0,0,0,0 +MISPRICINGS,2014,0,0,0,0,0,0 +MISREPRESENTED,2009,0,0,0,0,0,0 +MISSED,2009,0,0,0,0,0,0 +WORRIES,2009,0,0,0,0,0,0 +WORSEN,2009,0,0,0,0,0,0 +WORST,2009,0,0,0,0,0,0 +TIGHTENING,2009,0,0,0,0,0,0 +TOLERATING,2009,0,0,0,0,0,0 +TORTIOUSLY,0,0,0,2009,0,0,0 +FINES,2009,0,0,0,0,0,0 +NONASSESSABLE,0,0,2009,0,0,0,0 +NONCANCELLABLE,0,0,0,0,0,0,2009 +NONCOMPLIANT,2009,0,0,0,0,0,0 +NONCONFORMITY,2009,0,0,0,0,0,0 +NONCONTRIBUTORY,0,0,0,2009,0,0,0 +NONFORFEITABILITY,0,0,0,2011,0,0,0 +NONGUARANTOR,0,0,0,2011,0,0,0 +DISCIPLINARY,2009,0,0,0,0,0,0 +DISCLAIMERS,2009,0,0,0,0,0,0 +DISCLOSED,2009,0,0,0,0,0,0 +DISCONTINUANCES,2009,0,0,0,0,0,0 +DISCONTINUED,2009,0,0,0,0,0,0 +DISCOURAGED,2009,0,0,0,0,0,0 +DISCREDITED,2009,0,0,0,0,0,0 +DISCREPANCY,2009,0,0,0,0,0,0 +SPECULATED,0,0,2009,0,0,0,0 +SPECULATIONS,0,0,2009,0,0,0,0 +TRAGICALLY,2009,0,0,0,0,0,0 +LEGISLATED,0,0,0,2009,0,0,0 +LEGISLATIONS,0,0,0,2009,0,0,0 +LEGISLATORS,0,0,0,2009,0,0,0 +LIBELED,0,0,0,2009,0,0,0 +LIE,2009,0,0,0,0,0,0 +COMPULSION,2009,0,0,0,0,0,2009 +CONCEDE,2009,0,0,0,0,0,0 +CONCEIVABLE,0,0,2009,0,0,2009,0 +CONCERNS,2009,0,0,0,0,0,0 +CONCLUSIVE,0,2009,0,0,0,0,0 +CONDEMNATIONS,2009,0,0,0,0,0,0 +CONDEMNS,2009,0,0,0,0,0,0 +CONDONED,2009,0,0,0,0,0,0 +CONFESSES,2009,0,0,0,0,0,0 +CONFINE,2009,0,0,0,0,0,2009 +CONFINES,2009,0,0,0,0,0,2009 +CONFISCATES,2009,0,0,0,0,0,0 +CONFISCATORY,0,0,0,2009,0,0,0 +CONFLICTS,2009,0,0,0,0,0,0 +CONFRONTATIONS,2009,0,0,0,0,0,0 +CONFUSE,2009,0,0,0,0,0,0 +CONFUSINGLY,2009,0,2009,0,0,0,0 +CONSENTING,0,0,0,2009,0,0,0 +CONSPIRACY,2009,0,0,0,0,0,0 +CONSPIRE,2009,0,0,0,0,0,0 +CONSTITUTION,0,0,0,2009,0,0,0 +CONSTITUTIONS,0,0,0,2009,0,0,0 +CONSTRAINING,0,0,0,0,0,0,2009 +CONSTRUCTIVE,0,2009,0,0,0,0,0 +CONSTRUES,0,0,0,2012,0,0,0 +CONTENDED,2009,0,0,0,0,0,0 +CONTENTIONS,2009,0,0,0,0,0,0 +CONTESTATION,0,0,0,2011,0,0,0 +CONTINGENCY,0,0,2009,0,0,0,0 +CONTRACT,0,0,0,2009,0,0,0 +CONTRACTIBLE,0,0,0,2009,0,0,0 +CONTRACTIONS,2009,0,0,0,0,0,0 +CONTRADICT,2009,0,0,0,0,0,0 +CONTRADICTIONS,2009,0,0,0,0,0,0 +CONTRAVENE,0,0,0,2009,0,0,0 +CONTRAVENTION,0,0,0,2009,0,0,0 +CONTROVERSY,2009,0,0,0,0,0,0 +CONVENIENS,0,0,0,2012,0,0,0 +CONVICTED,2009,0,0,2009,0,0,0 +CORRECTED,2009,0,0,0,0,0,0 +CORRECTS,2009,0,0,0,0,0,0 +CORRUPTION,2009,0,0,0,0,0,0 +COSTLY,2009,0,0,0,0,0,0 +COUNSELED,0,0,0,2009,0,0,0 +COUNTERCLAIMED,2009,0,0,0,0,0,0 +COUNTERFEITED,2009,0,0,0,0,0,0 +COUNTERFEITS,2009,0,0,0,0,0,0 +COUNTERSUED,0,0,0,2011,0,0,0 +COURTEOUS,0,2009,0,0,0,0,0 +COVENANTED,0,0,0,0,0,0,2011 +CREATIVELY,0,2009,0,0,0,0,0 +CRIMES,2009,0,0,2009,0,0,0 +CRIMINALIZING,0,0,0,2014,0,0,0 +CRISIS,2009,0,0,0,0,0,0 +CRITICISMS,2009,0,0,0,0,0,0 +CRITICIZING,2009,0,0,0,0,0,0 +CROSSROADS,0,0,2009,0,0,0,0 +CULPABLE,2009,0,0,0,0,0,0 +CURTAILED,2009,0,0,0,0,0,0 +CURTAILS,2009,0,0,0,0,0,0 +CYBERATTACK,2014,0,0,0,0,0,0 +CYBERCRIMES,2014,0,0,0,0,0,0 +TAINTS,2009,0,0,0,0,0,0 +TENSE,2009,0,0,0,0,0,0 +TERMINATE,2009,0,0,0,0,0,0 +TERMINATION,2009,0,0,0,0,0,0 +TESTIFY,2009,0,0,2009,0,0,0 +THENCEFORTH,0,0,0,2009,0,0,0 +THEREFROM,0,0,0,2009,0,0,0 +THEREON,0,0,0,2009,0,0,0 +THERETOFORE,0,0,0,2009,0,0,0 +THEREWITH,0,0,0,2009,0,0,0 +THREATENING,2009,0,0,0,0,0,0 +FACIE,0,0,0,2009,0,0,0 +FAILING,2009,0,0,0,0,0,0 +FAILURES,2009,0,0,0,0,0,0 +FALSIFICATION,2009,0,0,0,0,0,0 +FALSIFY,2009,0,0,0,0,0,0 +FATALITIES,2009,0,0,0,0,0,0 +FAULTED,2009,0,0,0,0,0,0 +FAVORABLY,0,2009,0,0,0,0,0 +FAVORITES,0,2009,0,0,0,0,0 +FELONIOUS,2009,0,0,2009,0,0,0 +DAMAGES,2009,0,0,0,0,0,0 +DANGER,2009,0,0,0,0,0,0 +DEADLOCK,2009,0,0,0,0,0,0 +DEADWEIGHT,2009,0,0,0,0,0,0 +DEBARRED,2009,0,0,0,0,0,0 +DECEIT,2009,0,0,0,0,0,0 +DECEIVED,2009,0,0,0,0,0,0 +DECEPTIONS,2009,0,0,0,0,0,0 +DECLINE,2009,0,0,0,0,0,0 +DECREE,0,0,0,2009,0,0,0 +DEFACE,2009,0,0,0,0,0,0 +DEFALCATIONS,0,0,0,2009,0,0,0 +DEFAME,2009,0,0,0,0,0,0 +DEFAULT,2009,0,0,0,0,0,0 +DEFEASANCE,0,0,0,2009,0,0,0 +DEFEASEMENT,0,0,0,2014,0,0,0 +DEFEATED,2009,0,0,0,0,0,0 +DEFECTIVE,2009,0,0,0,0,0,0 +DEFENDABLE,0,0,0,2014,0,0,0 +DEFENDING,2009,0,0,0,0,0,0 +DEFERENCE,0,0,0,2009,0,0,0 +DEFICIT,2009,0,0,0,0,0,0 +DEFRAUD,2009,0,0,0,0,0,0 +DEFUNCT,2009,0,0,0,0,0,0 +DEGRADED,2009,0,0,0,0,0,0 +DELAYED,2009,0,0,0,0,0,0 +DELEGATABLE,0,0,0,2011,0,0,0 +DELIBERATE,2009,0,0,0,0,0,0 +DELIGHTED,0,2009,0,0,0,0,0 +DELIGHTS,0,2009,0,0,0,0,0 +DELINQUENTLY,2009,0,0,0,0,0,0 +DELISTING,2009,0,0,0,0,0,0 +DEMISES,2009,0,0,0,0,0,0 +DEMOLISHES,2009,0,0,0,0,0,0 +DEMOTE,2009,0,0,0,0,0,0 +DEMOTION,2009,0,0,0,0,0,0 +DEMURRERS,0,0,0,2009,0,0,0 +DENIALS,2009,0,0,0,0,0,0 +DENIGRATED,2009,0,0,0,0,0,0 +DENY,2009,0,0,0,0,0,0 +DEPENDABLE,0,2009,0,0,0,0,0 +DEPENDED,0,0,2009,0,0,2009,0 +DEPENDENT,0,0,2009,0,0,0,2011 +DEPLETED,2009,0,0,0,0,0,0 +DEPLETIONS,2009,0,0,0,0,0,0 +DEPOSING,0,0,0,2009,0,0,0 +INVALID,2009,0,0,0,0,0,0 +INVALIDATING,2009,0,0,0,0,0,0 +INVENTED,0,2009,0,0,0,0,0 +INVENTIVE,0,2009,0,0,0,0,0 +INVESTIGATE,2009,0,0,0,0,0,0 +INVESTIGATION,2009,0,0,0,0,0,0 +IRRECONCILABLE,2009,0,0,0,0,0,0 +IRREGULAR,2009,0,0,0,0,0,0 +IRREPARABLE,2009,0,0,0,0,0,0 +IRREVOCABLE,0,0,0,2009,0,0,2009 +JOINDER,0,0,0,2009,0,0,0 +JUDICIARY,0,0,0,2009,0,0,0 +JURISDICTIONAL,0,0,0,2009,0,0,0 +JURIST,0,0,0,2009,0,0,0 +JURY,0,0,0,2009,0,0,0 +JUSTIFIABLE,2009,0,0,0,0,0,0 +DILIGENTLY,0,2009,0,0,0,0,0 +DIMINISHING,2009,0,0,0,0,0,0 +DISADVANTAGE,2009,0,0,0,0,0,0 +DISAFFILIATION,2009,0,0,2009,0,0,0 +DISAFFIRMS,0,0,0,2011,0,0,0 +DISAGREEING,2009,0,0,0,0,0,0 +DISALLOW,2009,0,0,0,0,0,0 +DISALLOWING,2009,0,0,0,0,0,0 +DISAPPEARANCES,2009,0,0,0,0,0,0 +DISAPPOINT,2009,0,0,0,0,0,0 +DISAPPOINTMENT,2009,0,0,0,0,0,0 +DISAPPROVALS,2009,0,0,0,0,0,0 +DISAPPROVING,2009,0,0,0,0,0,0 +DISASSOCIATIONS,2009,0,0,0,0,0,0 +DISASTROUSLY,2009,0,0,0,0,0,0 +DISAVOWING,2009,0,0,0,0,0,0 +GREATER,0,-2020,0,0,0,0,0 +GRIEVANCE,2009,0,0,0,0,0,0 +GUILTY,2009,0,0,0,0,0,0 +HAMPERED,2009,0,0,0,0,0,0 +HAPPILY,0,2009,0,0,0,0,0 +HARASSED,2009,0,0,0,0,0,0 +HARDSHIPS,2009,0,0,0,0,0,0 +HARMFULLY,2009,0,0,0,0,0,0 +HARSHER,2009,0,0,0,0,0,0 +HAZARD,2009,0,0,0,0,0,0 +NONJUDICIALLY,0,0,0,2011,0,0,0 +NONPERFORMANCE,2009,0,0,0,0,0,0 +HURT,2009,0,0,0,0,0,0 +DISFAVORED,2009,0,0,0,0,0,0 +DISGORGED,2009,0,0,0,0,0,0 +COMPLICATING,2009,0,0,0,0,0,0 +COMPLIMENTARY,0,2009,0,0,0,0,0 +COMPLY,0,0,0,0,0,0,2009 +MISSTATED,2009,0,0,0,0,0,0 +MISSTATING,2009,0,0,0,0,0,0 +MISTAKEN,2009,0,0,0,0,0,0 +MISTRIAL,2009,0,0,2009,0,0,0 +MISUNDERSTANDINGS,2009,0,0,0,0,0,0 +MISUSES,2009,0,0,0,0,0,0 +MONOPOLIZATION,2009,0,0,0,0,0,0 +MONOPOLIZING,2009,0,0,0,0,0,0 +MORATORIUMS,2009,0,0,0,0,0,0 +MOTIONS,0,0,0,2009,0,0,0 +SLOWDOWN,2009,0,0,0,0,0,0 +SLOWEST,2009,0,0,0,0,0,0 +SLUGGISH,2009,0,0,0,0,0,0 +SMOOTHING,0,2009,0,0,0,0,0 +DEPRESS,2009,0,0,0,0,0,0 +DEPRIVATION,2009,0,0,0,0,0,0 +DEPRIVING,2009,0,0,0,0,0,0 +DEROGATED,0,0,0,2009,0,0,0 +DEROGATIONS,0,0,0,2009,0,0,0 +DESIRED,0,2009,0,0,0,0,0 +DESTABILIZE,2009,0,0,0,0,0,0 +DESTROY,2009,0,0,0,0,0,0 +DESTRUCTION,2009,0,0,0,0,0,0 +DETAINER,0,0,0,2009,0,0,0 +DETERIORATE,2009,0,0,0,0,0,0 +DETERIORATION,2009,0,0,0,0,0,0 +DETERRENCES,2009,0,0,0,0,0,0 +DETERS,2009,0,0,0,0,0,0 +DETRIMENT,2009,0,0,0,0,0,0 +DEVALUE,2009,0,0,0,0,0,0 +DEVASTATE,2009,0,0,0,0,0,0 +DEVIATE,2009,0,2009,0,0,0,0 +DEVIATION,2009,0,2009,0,0,0,0 +DEVOLVED,2009,0,0,0,0,0,0 +DICTATED,0,0,0,0,0,0,2009 +DIFFERED,0,0,2009,0,0,0,0 +DIFFICULTIES,2009,0,0,0,0,0,0 +ASCENDANCY,0,0,0,2009,0,0,0 +ASSAULTED,2009,0,0,0,0,0,0 +ASSERTIONS,2009,0,0,0,0,0,0 +ASSUME,0,0,2009,0,0,0,0 +ASSUMPTION,0,0,2009,0,0,0,0 +ASSURES,0,2009,0,0,0,0,0 +ATTAINING,0,2009,0,0,0,0,0 +ATTEST,0,0,0,2009,0,0,0 +ATTESTING,0,0,0,2009,0,0,0 +ATTORNMENT,0,0,0,2009,0,0,0 +ATTRITION,2009,0,0,0,0,0,0 +BAIL,2009,0,0,2009,0,0,0 +BAILIFF,0,0,0,2009,0,0,0 +BALK,2009,0,0,0,0,0,0 +BANKRUPTCY,2009,0,0,0,0,0,0 +BANS,2009,0,0,0,0,0,0 +CLAIMABLE,0,0,0,2009,0,0,0 +CLAIMING,2009,0,0,0,0,0,0 +CLAWBACK,2009,0,0,0,0,0,0 +CLOSEOUT,2009,0,0,0,0,0,0 +CLOSURE,2009,0,0,0,0,0,0 +CODICIL,0,0,0,2009,0,0,0 +CODIFIED,0,0,0,2009,0,0,0 +COERCE,2009,0,0,0,0,0,0 +COERCION,2009,0,0,0,0,0,0 +COLLABORATING,0,2009,0,0,0,0,0 +COLLABORATOR,0,2009,0,0,0,0,0 +COLLAPSES,2009,0,0,0,0,0,0 +COLLUDE,2009,0,0,0,0,0,0 +COLLUSION,2009,0,0,2009,0,0,0 +POSING,2009,0,0,0,0,0,0 +POSSIBILITIES,0,0,2009,0,0,0,0 +POSTCLOSING,0,0,0,2011,0,0,0 +POSTPONE,2009,0,0,0,0,0,0 +POSTPONES,2009,0,0,0,0,0,0 +WRITEOFF,2009,0,0,0,0,0,0 +WRONGDOING,2009,0,0,0,0,0,0 +WRONGLY,2009,0,0,0,0,0,0 +HONORED,0,2009,0,0,0,0,0 +HOSTILITY,2009,0,0,0,0,0,0 +REASSESSED,0,0,2009,0,0,0,0 +REASSESSMENTS,2009,0,2009,0,0,0,0 +REASSIGNMENT,2009,0,0,0,0,0,0 +REBOUNDED,0,2009,0,0,0,0,0 +REBUTTABLE,0,0,0,2009,0,0,0 +REBUTTED,0,0,0,2009,0,0,0 +RECALCULATES,0,0,2009,0,0,0,0 +RECALL,2009,0,0,0,0,0,0 +RECEPTIVE,0,2009,0,0,0,0,0 +RECKLESS,2009,0,0,0,0,0,0 +RECONSIDERED,0,0,2009,0,0,0,0 +RECOUPABLE,0,0,0,2009,0,0,0 +RECOURSES,0,0,0,2009,0,0,0 +RECUSE,0,0,0,2011,0,0,0 +REDACT,2009,0,0,2009,0,0,0 +REDACTIONS,2009,0,0,2009,0,0,0 +REDRESS,2009,0,0,0,0,0,0 +REEXAMINATION,0,0,2009,0,0,0,0 +REFERENDUM,0,0,0,2009,0,0,0 +REFILES,0,0,0,2009,0,0,0 +REFRAINS,0,0,0,0,0,0,2009 +REFUSED,2009,0,0,0,0,0,0 +REGAINED,0,2009,0,0,0,0,0 +REGULATES,0,0,0,2009,0,0,0 +REGULATIVE,0,0,0,2009,0,0,0 +REHEAR,0,0,0,2009,0,0,0 +VARIABLE,0,0,2009,0,0,0,0 +VARIANCES,0,0,2009,0,0,0,0 +VARIATIONS,0,0,2009,0,0,0,0 +VARYING,0,0,2009,0,0,0,0 +VERDICTS,2009,0,0,2009,0,0,0 +VIATICAL,0,0,0,2011,0,0,0 +VIOLATE,2009,0,0,0,0,0,0 +VIOLATION,2009,0,0,0,0,0,0 +VIOLATORS,2009,0,0,0,0,0,0 +VITIATE,2009,0,0,0,0,0,0 +VITIATION,2009,0,0,0,0,0,0 +VOLATILE,2009,0,2009,0,0,0,0 +VULNERABILITY,2009,0,0,0,0,0,0 +WARNED,2009,0,0,0,0,0,0 +WARRANTEES,0,0,0,2011,0,0,0 +WASTING,2009,0,0,0,0,0,0 +WEAKENING,2009,0,0,0,0,0,0 +WEAKLY,2009,0,0,0,0,0,0 +DISHONESTY,2009,0,0,0,0,0,0 +DISHONORED,2009,0,0,0,0,0,0 +DISLOYAL,2009,0,0,0,0,0,0 +DISMALLY,2009,0,0,0,0,0,0 +DISMISSED,2009,0,0,0,0,0,0 +DISPARAGE,2009,0,0,0,0,0,0 +DISPARAGES,2009,0,0,0,0,0,0 +DISPARITY,2009,0,0,0,0,0,0 +DISPLACEMENTS,2009,0,0,0,0,0,0 +DISPOSITIVE,0,0,0,2009,0,0,0 +DISPOSSESSING,2009,0,0,0,0,0,0 +DISPROPORTIONAL,2009,0,0,0,0,0,0 +DISPUTED,2009,0,0,0,0,0,0 +DISQUALIFICATIONS,2009,0,0,0,0,0,0 +DISQUALIFYING,2009,0,0,0,0,0,0 +DISREGARDS,2009,0,0,0,0,0,0 +DISRUPTED,2009,0,0,0,0,0,0 +DISRUPTIVE,2009,0,0,0,0,0,0 +DISSENT,2009,0,0,0,0,0,0 +DISSENTING,2009,0,0,0,0,0,0 +DISSOLUTION,2009,0,0,0,0,0,0 +DISTINCTIVE,0,2009,0,0,0,0,0 +DISTORTED,2009,0,0,0,0,0,0 +DISTORTS,2009,0,0,0,0,0,0 +DISTRACTION,2009,0,0,0,0,0,0 +DISTRESS,2009,0,0,0,0,0,0 +DISTURB,2009,0,0,0,0,0,0 +DISTURBING,2009,0,0,0,0,0,0 +DIVERTED,2009,0,0,0,0,0,0 +DIVESTED,2009,0,0,0,0,0,0 +DIVESTMENT,2009,0,0,0,0,0,0 +DIVORCED,2009,0,0,0,0,0,0 +DIVULGING,2009,0,0,0,0,0,0 +DOCKETS,0,0,0,2009,0,0,0 +DOUBTFUL,2009,0,2009,0,0,0,0 +DOWNGRADES,2009,0,0,0,0,0,0 +DOWNSIZES,2009,0,0,0,0,0,0 +DOWNTIMES,2009,0,0,0,0,0,0 +DOWNWARDS,2009,0,0,0,0,0,0 +DRAWBACK,2009,0,0,0,0,0,0 +DROUGHT,2009,0,0,0,0,0,0 +DYSFUNCTION,2009,0,0,0,0,0,0 +EARMARKED,0,0,0,0,0,0,2009 +EASILY,0,2009,0,0,0,0,0 +EFFICIENCIES,0,2009,0,0,0,0,0 +EGREGIOUS,2009,0,0,0,0,0,0 +EMBARGOED,2009,0,0,0,0,0,0 +EMBARRASSED,2009,0,0,0,0,0,0 +EMBARRASSMENTS,2009,0,0,0,0,0,0 +EMBEZZLEMENTS,2009,0,0,0,0,0,0 +EMPOWER,0,2009,0,0,0,0,0 +ENABLE,0,2009,0,0,0,0,0 +ENCOURAGED,0,2009,0,0,0,0,0 +ENCROACH,2009,0,0,0,0,0,0 +ENCROACHMENT,2009,0,0,0,0,0,0 +ENCUMBERING,2009,0,0,2009,0,0,2009 +ENCUMBRANCERS,0,0,0,2011,0,0,0 +ENDANGERING,2009,0,0,0,0,0,0 +ENFORCEABILITY,0,0,0,2009,0,0,0 +ENHANCED,0,2009,0,0,0,0,0 +ENHANCING,0,2009,0,0,0,0,0 +ENJOINS,2009,0,0,0,0,0,0 +ENJOYED,0,2009,0,0,0,0,0 +ENTAIL,0,0,0,0,0,0,2009 +PECUNIARILY,0,0,0,2009,0,0,0 +PENALIZING,2009,0,0,0,0,0,0 +PERFECT,0,2009,0,0,0,0,0 +PERHAPS,0,0,2009,0,0,2009,0 +PERMISSIBLE,0,0,0,0,0,0,2009 +PERMITTEE,0,0,0,2011,0,0,0 +PERPETRATED,2009,0,0,2009,0,0,0 +PERSIST,2009,0,0,0,0,0,0 +PERSISTENTLY,2009,0,0,0,0,0,0 +PERVASIVE,2009,0,0,0,0,0,0 +PETITIONED,0,0,0,2009,0,0,0 +PETITIONS,0,0,0,2009,0,0,0 +PICKETING,2009,0,0,0,0,0,0 +HENCEFORWARD,0,0,0,2009,0,0,0 +HEREFOR,0,0,0,2009,0,0,0 +HEREINABOVE,0,0,0,2009,0,0,0 +HEREOF,0,0,0,2009,0,0,0 +HEREUNDER,0,0,0,2009,0,0,0 +HEREWITHIN,0,0,0,2014,0,0,0 +HINDERED,2009,0,0,0,0,0,0 +HINDRANCES,2009,0,0,0,0,0,0 +WHENSOEVER,0,0,0,2009,0,0,0 +WHEREBY,0,0,0,2009,0,0,0 +WHEREON,0,0,0,2009,0,0,0 +WHEREWITH,0,0,0,2009,0,0,0 +WHOSOEVER,0,0,0,2009,0,0,0 +WILLFULLY,2009,0,0,2009,0,0,0 +WINNERS,0,2009,0,0,0,0,0 +FLUCTUATING,0,0,2009,0,0,0,0 +FORBEAR,0,0,0,2009,0,0,0 +FORBEARS,0,0,0,2009,0,0,0 +FORBIDS,2009,0,0,0,0,0,2009 +FOREBEAR,0,0,0,2009,0,0,0 +FORECLOSED,2009,0,0,0,0,0,0 +FORECLOSURES,2009,0,0,0,0,0,0 +FORESTALL,2009,0,0,0,0,0,0 +FORFEIT,2009,0,0,0,0,0,0 +FORFEITING,2009,0,0,0,0,0,0 +FORGERS,2009,0,0,0,0,0,0 +FRAUD,2009,0,0,0,0,0,0 +FRAUDULENTLY,2009,0,0,0,0,0,0 +FRUSTRATE,2009,0,0,0,0,0,0 +FRUSTRATINGLY,2009,0,0,0,0,0,0 +FUGITIVES,2009,0,0,2009,0,0,0 +GAINING,0,2009,0,0,0,0,0 +GRANTORS,0,0,0,2009,0,0,0 +DISGRACE,2009,0,0,0,0,0,0 +LITIGANTS,2009,0,0,2009,0,0,0 +LITIGATING,2009,0,0,2009,0,0,0 +LITIGATORS,0,0,0,2009,0,0,0 +LACK,2009,0,0,0,0,0,0 +LACKS,2009,0,0,0,0,0,0 +LAGS,2009,0,0,0,0,0,0 +LAPSING,2009,0,0,0,0,0,0 +LAWFUL,0,0,0,2009,0,0,0 +LAWMAKING,0,0,0,2009,0,0,0 +LAWYER,0,0,0,2009,0,0,0 +LEADERSHIP,0,2009,0,0,0,0,0 +LEGALITY,0,0,0,2009,0,0,0 +LEGALIZED,0,0,0,2009,0,0,0 +LEGALS,0,0,0,2009,0,0,0 +FLAW,2009,0,0,0,0,0,0 +NONPRODUCTIVE,2009,0,0,0,0,0,0 +LIQUIDATED,2009,0,0,0,0,0,0 +LIQUIDATIONS,2009,0,0,0,0,0,0 +BENEFICIATED,0,0,0,2014,0,0,0 +BENEFITING,0,2009,0,0,0,0,0 +BETTER,0,2009,0,0,0,0,0 +POPULARITY,0,2009,0,0,0,0,0 +REINTERPRET,0,0,2009,0,0,0,0 +REINTERPRETING,0,0,2009,0,0,0,0 +REJECTING,2009,0,0,0,0,0,0 +RELEASEES,0,0,0,2011,0,0,0 +RELINQUISHING,2009,0,0,0,0,0,0 +RELUCTANT,2009,0,0,0,0,0,0 +REMANDS,0,0,0,2009,0,0,0 +REMEDIATION,0,0,0,2009,0,0,0 +RENEGOTIATE,2009,0,0,0,0,0,0 +RENEGOTIATION,2009,0,0,0,0,0,0 +RENOUNCEMENT,2009,0,0,0,0,0,0 +REPARATION,2009,0,0,0,0,0,0 +REPOSSESSED,2009,0,0,0,0,0,0 +REPOSSESSIONS,2009,0,0,0,0,0,0 +TROUBLE,2009,0,0,0,0,0,0 +TURMOIL,2009,0,0,0,0,0,0 +UNACCOUNTED,2009,0,0,0,0,0,0 +UNAPPEALABLE,0,0,0,2009,0,0,0 +UNAUTHORIZED,2009,0,0,0,0,0,0 +UNAVOIDABLY,2009,0,0,0,0,0,0 +UNCERTAINTIES,0,0,2009,0,0,0,0 +UNCOLLECTED,2009,0,0,0,0,0,0 +UNCOMPETITIVE,2009,0,0,0,0,0,0 +UNCONSCIONABLE,2009,0,0,0,0,0,0 +UNCONSTITUTIONALLY,0,0,0,2009,0,0,0 +UNCONTROLLED,2009,0,0,0,0,0,0 +UNCOVERING,2009,0,0,0,0,0,0 +UNDEFINED,0,0,2009,0,0,0,0 +UNDERCUT,2009,0,0,0,0,0,0 +UNDERESTIMATED,2009,0,0,0,0,0,0 +UNDERFUNDED,2009,0,0,0,0,0,0 +UNDERMINES,2009,0,0,0,0,0,0 +UNDERPAYMENTS,2009,0,0,0,0,0,0 +UNDERPERFORMED,2011,0,0,0,0,0,0 +UNDERPRODUCTION,2009,0,0,0,0,0,0 +UNDERSTATEMENT,2009,0,0,0,0,0,0 +UNDERUTILIZATION,2009,0,0,0,0,0,0 +UNDESIRED,2009,0,0,0,0,0,0 +UNDETERMINED,2009,0,2009,0,0,0,0 +UNDOCUMENTED,2009,0,2009,0,0,0,0 +UNECONOMIC,2009,0,0,0,0,0,0 +UNEMPLOYMENT,2009,0,0,0,0,0,0 +UNENFORCEABLE,0,0,0,2009,0,0,0 +UNETHICALLY,2009,0,0,0,0,0,0 +UNFAIR,2009,0,0,0,0,0,0 +UNFAVORABILITY,2014,0,0,0,0,0,0 +UNFEASIBLE,2009,0,0,0,0,0,0 +UNFORESEEABLE,2009,0,0,0,0,0,0 +UNFORTUNATELY,2009,0,0,0,0,0,0 +UNFUNDED,2009,0,0,0,0,0,0 +UNIDENTIFIED,0,0,2009,0,0,0,0 +UNINTENTIONALLY,2009,0,0,0,0,0,0 +UNJUSTIFIED,2009,0,0,0,0,0,0 +UNKNOWN,0,0,2009,0,0,0,0 +UNLAWFULNESS,0,0,0,2009,0,0,0 +UNMATCHED,0,2009,0,0,0,0,0 +UNNECESSARY,2009,0,0,0,0,0,0 +UNOCCUPIED,2009,0,0,0,0,0,0 +UNPLANNED,2009,0,2009,0,0,0,0 +UNPREDICTABLY,2009,0,2009,0,0,0,0 +UNPROFITABLE,2009,0,0,0,0,0,0 +UNQUANTIFIABLE,0,0,2011,0,0,0,0 +UNREASONABLENESS,2009,0,0,0,0,0,0 +UNRECOVERABLE,2009,0,0,0,0,0,0 +UNREMEDIATED,0,0,0,2011,0,0,0 +UNREST,2009,0,0,0,0,0,0 +UNSATISFACTORY,2009,0,0,0,0,0,0 +UNSEASONABLE,0,0,2009,0,0,0,0 +UNSOLD,2009,0,0,0,0,0,0 +UNSTABILIZED,2014,0,0,0,0,0,0 +UNSUCCESSFUL,2009,0,0,0,0,0,0 +UNSUITABLY,2009,0,0,0,0,0,0 +UNSUSPECTED,2009,0,0,0,0,0,0 +UNTESTED,0,0,2009,0,0,0,0 +UNTRUTH,2009,0,0,0,0,0,0 +UNTRUTHS,2009,0,0,0,0,0,0 +UNWANTED,2009,0,0,0,0,0,0 +UNWILLINGNESS,2009,0,0,0,0,0,0 +UPTURNS,0,2009,0,0,0,0,0 +USURP,2009,0,0,2009,0,0,0 +USURPS,2009,0,0,2009,0,0,0 +VAGUELY,0,0,2012,0,0,0,0 +VAGUEST,0,0,2012,0,0,0,0 +PLEA,2009,0,0,0,0,0,0 +PLEADINGS,2009,0,0,2009,0,0,0 +PLEASANTLY,0,2009,0,0,0,0,0 +PLEDGE,0,0,0,0,0,0,2009 +PLEDGES,0,0,0,0,0,0,2009 +PLENTIFUL,0,2009,0,0,0,0,0 +COMPELS,0,0,0,0,0,0,2009 +COMPLAINANTS,0,0,0,2009,0,0,0 +COMPLAINT,2009,0,0,0,0,0,0 +COMMIT,0,0,0,0,0,0,2009 +COMMITTED,0,0,0,0,0,0,2009 +LIMIT,0,0,0,0,0,0,2009 +LIMITS,0,0,0,0,0,0,2009 +RESTRAINED,0,0,0,0,0,0,2009 +RESTRAINTS,0,0,0,0,0,0,2009 +RESTRICTION,0,0,0,0,0,0,2009 +RESTRICTIVENESS,0,0,0,0,0,0,2009 +RESTRUCTURES,2009,0,0,0,0,0,0 +RETALIATED,2009,0,0,0,0,0,0 +RETALIATIONS,2009,0,0,0,0,0,0 +PRECAUTIONARY,0,0,2009,0,0,0,0 +PRECIPITOUSLY,2009,0,0,0,0,0,0 +PRECLUDING,2009,0,0,0,0,0,2009 +PREDECEASE,0,0,0,2009,0,0,0 +PREDICT,0,0,2009,0,0,0,0 +PREDICTION,0,0,2009,0,0,0,0 +PREDICTORS,0,0,2009,0,0,0,0 +PREHEARING,0,0,0,2011,0,0,0 +PREJUDICIAL,2009,0,0,2009,0,0,0 +PREMATURE,2009,0,0,0,0,0,0 +PREPETITION,0,0,0,2009,0,0,0 +PRESTIGIOUS,0,2009,0,0,0,0,0 +PRESUMES,0,0,2009,0,0,0,0 +PRESUMPTIVELY,0,0,0,2009,0,0,0 +PREVENTING,2009,0,0,0,0,0,2009 +PRIVITY,0,0,0,2011,0,0,0 +PROBABILITIES,0,0,2009,0,0,0,0 +PROBATE,0,0,0,2009,0,0,0 +PROBATION,0,0,0,2009,0,0,0 +PROBATIONERS,0,0,0,2009,0,0,0 +PROBLEMATICAL,2009,0,0,0,0,0,0 +PROFICIENTLY,0,2009,0,0,0,0,0 +PROGRESS,0,2009,0,0,0,0,0 +PROHIBIT,0,0,0,0,0,0,2009 +PROHIBITIONS,0,0,0,0,0,0,2009 +PROHIBITS,0,0,0,0,0,0,2009 +PROLONGED,2009,0,0,0,0,0,0 +PROMULGATED,0,0,0,2009,0,0,0 +PROMULGATIONS,0,0,0,2009,0,0,0 +PRORATA,0,0,0,2009,0,0,0 +PROSECUTES,2009,0,0,2009,0,0,0 +PROSECUTOR,0,0,0,2009,0,0,0 +PROSPERING,0,2009,0,0,0,0,0 +PROTEST,2009,0,0,0,0,0,0 +PROTESTING,2009,0,0,0,0,0,0 +PROTRACTED,2009,0,0,0,0,0,0 +PROVISOS,0,0,0,2009,0,0,0 +PROVOKING,2009,0,0,0,0,0,0 +PUNISHING,2009,0,0,0,0,0,0 +PURPORT,2009,0,0,0,0,0,0 +PURPORTS,2009,0,0,0,0,0,0 +QUESTIONED,2009,0,0,0,0,0,0 +QUITCLAIM,0,0,0,2009,0,0,0 +RACKETEERING,2009,0,0,0,0,0,0 +RANDOMIZES,0,0,2009,0,0,0,0 +RATA,0,0,0,2009,0,0,0 +RATIONALIZATIONS,2009,0,0,0,0,0,0 +RATIONALIZING,2009,0,0,0,0,0,0 +ABANDON,2009,0,0,0,0,0,0 +ABANDONMENTS,2009,0,0,0,0,0,0 +ABDICATING,2009,0,0,0,0,0,0 +ABERRATION,2009,0,0,0,0,0,0 +ABEYANCE,0,0,2009,0,0,0,0 +ABLE,0,2009,0,0,0,0,0 +ABNORMALLY,2009,0,0,0,0,0,0 +ABOLISHING,2009,0,0,0,0,0,0 +ABROGATES,2009,0,0,2009,0,0,0 +ABRUPT,2009,0,0,0,0,0,0 +ABSENCES,2009,0,0,0,0,0,0 +ABSOLVES,0,0,0,2009,0,0,0 +ABUSE,2009,0,0,0,0,0,0 +ABUSIVE,2009,0,0,0,0,0,0 +ACCESSIONS,0,0,0,2009,0,0,0 +ACCIDENTS,2009,0,0,0,0,0,0 +ACCOMPLISHES,0,2009,0,0,0,0,0 +ACCUSATION,2009,0,0,0,0,0,0 +ACCUSES,2009,0,0,0,0,0,0 +ACHIEVEMENT,0,2009,0,0,0,0,0 +ACQUIESCE,2009,0,0,0,0,0,0 +ACQUIREES,0,0,0,2011,0,0,0 +ACQUITTAL,2009,0,0,2009,0,0,0 +ACQUITTED,2009,0,0,2009,0,0,0 +ADJOURN,0,0,0,2009,0,0,0 +ADJOURNMENTS,0,0,0,2009,0,0,0 +ADJUDGES,0,0,0,2009,0,0,0 +ADJUDICATES,0,0,0,2009,0,0,0 +ADJUDICATIVE,0,0,0,2009,0,0,0 +ADMISSIBILITY,0,0,0,2009,0,0,0 +ADMISSIONS,0,0,0,2009,0,0,0 +ADULTERATION,2009,0,0,0,0,0,0 +ADVANCES,0,2009,0,0,0,0,0 +ADVANTAGEOUS,0,2009,0,0,0,0,0 +ADVERSARIES,2009,0,0,0,0,0,0 +ADVERSITIES,2009,0,0,0,0,0,0 +AFFIRMANCE,0,0,0,2011,0,0,0 +AFORESAID,0,0,0,2009,0,0,0 +AGAINST,2009,0,0,0,0,0,0 +AGGRAVATING,2009,0,0,0,0,0,0 +ALERTED,2009,0,0,0,0,0,0 +ALIENATES,2009,0,0,0,0,0,0 +ALLEGATION,2009,0,0,2009,0,0,0 +ALLEGEDLY,2009,0,0,2009,0,0,0 +ALLIANCES,0,2009,0,0,0,0,0 +ALWAYS,0,0,0,0,2009,0,0 +AMEND,0,0,0,2009,0,0,0 +AMENDING,0,0,0,2009,0,0,0 +ANNOY,2009,0,0,0,0,0,0 +ANNOYING,2009,0,0,0,0,0,0 +ANNULLING,2009,0,0,0,0,0,0 +ANOMALIES,2009,0,2009,0,0,0,0 +ANTECEDENT,0,0,0,2009,0,0,0 +ANTICIPATES,0,0,2009,0,0,0,0 +ANTICOMPETITIVE,2009,0,0,0,0,0,0 +APPARENT,0,0,2009,0,0,0,0 +APPEALED,0,0,0,2009,0,0,0 +APPEARED,0,0,2009,0,0,2009,0 +APPELLANTS,0,0,0,2009,0,0,0 +APPROXIMATE,0,0,2009,0,0,0,0 +APPROXIMATING,0,0,2009,0,0,0,0 +APPURTENANCES,0,0,0,2009,0,0,0 +ARBITRARILY,0,0,2009,0,0,0,0 +ARBITRATED,0,0,0,2009,0,0,0 +ARBITRATIONAL,0,0,0,2011,0,0,0 +ARBITRATORS,0,0,0,2009,0,0,0 +ARGUMENT,2009,0,0,0,0,0,0 +ARREARAGES,2009,0,0,2009,0,0,0 +ARRESTS,2009,0,0,0,0,0,0 +RETROCEDED,0,0,0,2011,0,0,0 +BOLSTERING,0,2009,0,0,0,0,0 +BOOM,0,2009,0,0,0,0,0 +BOTTLENECK,2009,0,0,0,0,0,0 +BOYCOTT,2009,0,0,0,0,0,0 +BREACH,2009,0,0,2009,0,0,0 +BREAK,2009,0,0,0,0,0,0 +BREAKDOWNS,2009,0,0,0,0,0,0 +BREAKTHROUGHS,0,2009,0,0,0,0,0 +BRIBERY,2009,0,0,0,0,0,0 +BRILLIANT,0,2009,0,0,0,0,0 +BURDENING,2009,0,0,0,0,0,0 +CALAMITIES,2009,0,0,0,0,0,0 +CANCELED,2009,0,0,0,0,0,0 +CANCELLED,2009,0,0,0,0,0,0 +CARELESSLY,2009,0,0,0,0,0,0 +CATASTROPHES,2009,0,0,0,0,0,0 +CAUTIONARY,2009,0,0,0,0,0,0 +CAUTIOUS,0,0,2009,0,0,0,0 +CEASED,2009,0,0,0,0,0,0 +CEDANTS,0,0,0,2012,0,0,0 +CENSURING,2009,0,0,0,0,0,0 +CHALLENGED,2009,0,0,0,0,0,0 +CHARITABLE,0,2009,0,0,0,0,0 +CIRCUMVENT,2009,0,0,0,0,0,0 +CIRCUMVENTIONS,2009,0,0,0,0,0,0 +SHORTAGE,2009,0,0,0,0,0,0 +SHRINKAGE,2009,0,0,0,0,0,0 +SHUTDOWNS,2009,0,0,0,0,0,0 +SLANDERED,2009,0,0,0,0,0,0 +REPUDIATES,2009,0,0,0,0,0,0 +REQUESTER,0,0,0,2011,0,0,0 +REQUIREMENT,0,0,0,0,0,0,2009 +REREGULATION,0,0,0,2011,0,0,0 +RESCINDS,0,0,0,2009,0,0,0 +RESIGNATION,2009,0,0,0,0,0,0 +RESIGNS,2009,0,0,0,0,0,0 +RESTATEMENT,2009,0,0,0,0,0,0 +RESTITUTIONARY,0,0,0,2011,0,0,0 +NOTARIAL,0,0,0,2009,0,0,0 +NOTARIZE,0,0,0,2009,0,0,0 +NOTWITHSTANDING,0,0,0,2009,0,0,0 +NULLIFICATION,2009,0,0,2009,0,0,0 +NULLIFY,2009,0,0,2009,0,0,0 +OBJECTED,2009,0,0,0,0,0,0 +OBJECTIONABLY,2009,0,0,0,0,0,0 +OBLIGATES,0,0,0,0,0,0,2009 +OBLIGATORY,0,0,0,0,0,0,2009 +OBLIGEES,0,0,0,2011,0,0,0 +OBSCENE,2009,0,0,0,0,0,0 +OBSTACLE,2009,0,0,0,0,0,0 +OBSTRUCTING,2009,0,0,0,0,0,0 +OFFENCE,2009,0,0,0,0,0,0 +OFFENDER,2009,0,0,0,0,0,0 +OFFENSE,0,0,0,2009,0,0,0 +OFFERORS,0,0,0,2011,0,0,0 +OMITS,2009,0,0,0,0,0,0 +OPPORTUNISTIC,2009,0,0,0,0,0,0 +OPPOSE,2009,0,0,0,0,0,0 +OPPOSITION,2009,0,0,0,0,0,0 +OPTIONEES,0,0,0,2009,0,0,0 +OUTDATED,2009,0,0,0,0,0,0 +OUTPERFORMING,0,2009,0,0,0,0,0 +OVERBUILD,2009,0,0,0,0,0,0 +OVERBURDEN,2009,0,0,0,0,0,0 +OVERCAPACITY,2009,0,0,0,0,0,0 +OVERCHARGING,2009,0,0,0,0,0,0 +OVERDUE,2009,0,0,0,0,0,0 +OVERESTIMATING,2009,0,0,0,0,0,0 +OVERLOADED,2009,0,0,0,0,0,0 +OVERLOOKED,2009,0,0,0,0,0,0 +OVERPAYMENT,2009,0,0,0,0,0,0 +OVERPRODUCING,2009,0,0,0,0,0,0 +OVERRULES,0,0,0,2009,0,0,0 +OVERRUNS,2009,0,0,0,0,0,0 +OVERSHADOWS,2009,0,0,0,0,0,0 +OVERSTATEMENTS,2009,0,0,0,0,0,0 +OVERSUPPLIES,2009,0,0,0,0,0,0 +OVERTURN,2009,0,0,0,0,0,0 +OVERVALUE,2009,0,0,0,0,0,0 +PANICS,2009,0,0,0,0,0,0 +NECESSITATE,0,0,0,0,0,0,2009 +NEGATIVE,2009,0,0,0,0,0,0 +NEGLECTED,2009,0,0,0,0,0,0 +NEGLIGENCE,2009,0,0,0,0,0,0 +NEVER,0,0,0,0,2009,0,0 +BARRIERS,2009,0,0,0,0,0,0 +BELIEVED,0,0,2009,0,0,0,0 +IDLING,2009,0,0,0,0,0,0 +IGNORING,2009,0,0,0,0,0,0 +ILLEGALITY,2009,0,0,0,0,0,0 +ILLICITLY,2009,0,0,0,0,0,0 +IMBALANCES,2009,0,0,0,0,0,0 +IMPAIR,2009,0,0,0,0,0,2011 +IMPAIRMENTS,2009,0,0,0,0,0,2011 +IMPEDE,2009,0,0,0,0,0,0 +IMPEDIMENTS,2009,0,0,0,0,0,0 +IMPERFECTION,2009,0,0,0,0,0,0 +IMPLEADED,0,0,0,2009,0,0,0 +IMPLICATING,2009,0,0,0,0,0,0 +IMPOSING,0,0,0,0,0,0,2009 +IMPOSSIBLE,2009,0,0,0,0,0,0 +IMPOUNDS,2009,0,0,0,0,0,0 +IMPRACTICALITY,2009,0,0,0,0,0,0 +IMPRESS,0,2009,0,0,0,0,0 +IMPRESSIVE,0,2009,0,0,0,0,0 +IMPROBABLE,0,0,2009,0,0,0,0 +IMPROPRIETY,2009,0,0,0,0,0,0 +IMPROVEMENTS,0,2009,0,0,0,0,0 +IMPRUDENTLY,2009,0,0,0,0,0,0 +INACCURACY,2009,0,0,0,0,0,0 +INACTIONS,2009,0,0,0,0,0,0 +INACTIVATING,2009,0,0,0,0,0,0 +INADEQUACIES,2009,0,0,0,0,0,0 +INADVERTENT,2009,0,0,0,0,0,0 +INAPPROPRIATE,2009,0,0,0,0,0,0 +INCAPABLE,2009,0,0,0,0,0,0 +INCARCERATED,2009,0,0,2009,0,0,0 +INCARCERATIONS,2009,0,0,2009,0,0,0 +INCIDENT,2009,0,0,0,0,0,0 +INCOMPATIBLE,2009,0,0,0,0,0,0 +INCOMPETENTLY,2009,0,0,0,0,0,0 +INCOMPLETENESS,2009,0,2009,0,0,0,0 +INCONSISTENT,2009,0,0,0,0,0,0 +INCONVENIENCE,2009,0,0,0,0,0,0 +INCORRECTLY,2009,0,0,0,0,0,0 +INDEBTED,0,0,0,0,0,0,2009 +INDEFEASIBLY,2009,0,0,0,0,0,0 +INDEMNIFIABLE,0,0,0,2009,0,0,0 +INDEMNIFIES,0,0,0,2009,0,0,0 +INDEMNITEES,0,0,0,2009,0,0,0 +INDEMNITY,0,0,0,2009,0,0,0 +INDICTABLE,2009,0,0,2009,0,0,0 +INDICTMENTS,2009,0,0,2009,0,0,0 +INEFFECTIVENESS,2009,0,0,0,0,0,0 +INEFFICIENTLY,2009,0,0,0,0,0,0 +INEQUITABLY,2009,0,0,0,0,0,0 +INEXACT,0,0,2009,0,0,0,0 +INFERIOR,2009,0,0,0,0,0,0 +INFORMATIVE,0,2009,0,0,0,0,0 +INFRINGED,2009,0,0,0,0,0,0 +INFRINGES,2009,0,0,0,0,0,0 +INHIBITED,2009,0,0,0,0,0,2009 +INJUNCTION,2009,0,0,2009,0,0,0 +INJURED,2009,0,0,0,0,0,0 +INJURIOUS,2009,0,0,0,0,0,0 +INNOVATES,0,2009,0,0,0,0,0 +INNOVATIVE,0,2009,0,0,0,0,0 +INORDINATE,2009,0,0,0,0,0,0 +INSENSITIVE,2009,0,0,0,0,0,0 +INSISTENCE,0,0,0,0,0,0,2009 +INSOLVENCIES,2009,0,0,0,0,0,0 +INSPIRATIONAL,0,2009,0,0,0,0,0 +INSUFFICIENCY,2009,0,0,0,0,0,0 +INSURRECTIONS,2009,0,0,0,0,0,0 +INTENTIONAL,2009,0,0,0,0,0,0 +INTERFERENCES,2009,0,0,0,0,0,0 +INTERMITTENT,2009,0,0,0,0,0,0 +INTERPOSED,0,0,0,2009,0,0,0 +INTERPOSITIONS,0,0,0,2009,0,0,0 +INTERROGATING,0,0,0,2009,0,0,0 +INTERROGATORIES,0,0,0,2009,0,0,0 +INTERRUPTED,2009,0,0,0,0,0,0 +INTERRUPTS,2009,0,0,0,0,0,0 +STABILITY,0,2009,0,0,0,0,0 +STABILIZED,0,2009,0,0,0,0,0 +STAGGERING,2009,0,0,0,0,0,0 +STAGNATES,2009,0,0,0,0,0,0 +STANDSTILLS,2009,0,0,0,0,0,0 +STATUTORY,0,0,0,2009,0,0,0 +STIPULATING,0,0,0,0,0,0,2009 +STOPPAGE,2009,0,0,0,0,0,0 +STOPS,2009,0,0,0,0,0,0 +STRAINS,2009,0,0,0,0,0,0 +STRENGTHENING,0,2009,0,0,0,0,0 +STRESSED,2009,0,0,0,0,0,0 +STRICT,0,0,0,0,0,0,2009 +STRINGENT,2009,0,0,0,0,0,0 +STRONGLY,0,0,0,0,2009,0,0 +SUBJECTED,2009,0,0,0,0,0,0 +SUBLEASEHOLD,0,0,0,2011,0,0,0 +SUBPARAGRAPH,0,0,0,2009,0,0,0 +SUBPOENAS,2009,0,0,2009,0,0,0 +SUBTRUST,0,0,0,2011,0,0,0 +SUCCEEDING,0,2009,0,0,0,0,0 +SUCCESSFUL,0,2009,0,0,0,0,0 +SUE,2009,0,0,2009,0,0,0 +SUFFERED,2009,0,0,0,0,0,0 +SUGGESTED,0,0,2009,0,0,0,0 +SUMMONED,2009,0,0,2009,0,0,0 +SUPERIOR,0,2009,0,0,0,0,0 +SUPERSEDES,0,0,0,2009,0,0,0 +SURPASS,0,2009,0,0,0,0,0 +SUSCEPTIBILITY,2009,0,2009,0,0,0,0 +SUSPECTS,2009,0,0,0,0,0,0 +SUSPENDS,2009,0,0,0,0,0,0 +SUSPICIONS,2009,0,0,0,0,0,0 +REVOCABILITY,0,0,0,2011,0,0,0 +REVOKED,2009,0,0,0,0,0,0 +REVOLUTIONIZED,0,2009,0,0,0,0,0 +REWARDED,0,2009,0,0,0,0,0 +RIDICULED,2009,0,0,0,0,0,0 +RISKED,0,0,2009,0,0,0,0 +RISKING,0,0,2009,0,0,0,0 +RULING,0,0,0,2009,0,0,0 +SACRIFICE,2009,0,0,0,0,0,0 +SACRIFICING,2009,0,0,0,0,0,0 +SATISFIED,0,2009,0,0,0,0,0 +SCANDALOUS,2009,0,0,0,0,0,0 +SCRUTINIZES,2009,0,0,0,0,0,0 +SEEMS,0,0,2009,0,0,0,0 +SEIZING,2009,0,0,0,0,0,0 +SENTENCING,2009,0,0,2009,0,0,0 +SERIOUSNESS,2009,0,0,0,0,0,0 +SETTLEMENTS,0,0,0,2009,0,0,0 +SEVERALLY,0,0,0,2009,0,0,0 +SEVERED,2009,0,0,0,0,0,0 +ENTHUSIASTICALLY,0,2009,0,0,0,0,0 +ERODED,2009,0,0,0,0,0,0 +ERRATIC,2009,0,0,0,0,0,0 +ERRONEOUS,2009,0,0,0,0,0,0 +ERRS,2009,0,0,0,0,0,0 +ESCALATING,2009,0,0,0,0,0,0 +ESCROW,0,0,0,0,0,0,2009 +ESTOPPEL,0,0,0,2011,0,0,0 +EVADING,2009,0,0,0,0,0,0 +EVICT,2009,0,0,0,0,0,0 +EVICTIONS,2009,0,0,0,0,0,0 +EXACERBATE,2009,0,0,0,0,0,0 +EXACERBATION,2009,0,0,0,0,0,0 +EXAGGERATES,2009,0,0,0,0,0,0 +EXCEEDANCES,0,0,0,2011,0,0,0 +EXCELLING,0,2009,0,0,0,0,0 +EXCESSIVE,2009,0,0,0,0,0,0 +EXCITEMENT,0,2009,0,0,0,0,0 +EXCLUSIVENESS,0,2009,0,0,0,0,0 +EXCULPATED,2009,0,0,2009,0,0,0 +EXCULPATIONS,2009,0,0,2009,0,0,0 +EXECUTORY,0,0,0,2009,0,0,0 +EXEMPLARY,0,2009,0,0,0,0,0 +EXONERATING,2009,0,0,0,0,0,0 +EXPLOITATION,2009,0,0,0,0,0,0 +EXPLOITING,2009,0,0,0,0,0,0 +EXPOSES,2009,0,0,0,0,0,0 +EXPROPRIATE,2009,0,0,0,0,0,0 +EXPROPRIATION,2009,0,0,0,0,0,0 +EXTENUATING,2009,0,0,0,0,0,0 +SOLVING,0,2009,0,0,0,0,0 +SOMEWHERE,0,0,2009,0,0,0,0 +LOSE,2009,0,0,0,0,0,0 +LOSSES,2009,0,0,0,0,0,0 +LUCRATIVE,0,2009,0,0,0,0,0 +MALFUNCTION,2009,0,0,0,0,0,0 +MALICE,2009,0,0,0,0,0,0 +MANDAMUS,0,0,0,2009,0,0,0 +MANDATING,0,0,0,0,0,0,2009 +MANIPULATED,2009,0,0,0,0,0,0 +MANIPULATIONS,2009,0,0,0,0,0,0 +MAY,0,0,2009,0,0,2009,0 +MEDIATES,0,0,0,2009,0,0,0 +MEDIATOR,0,0,0,2009,0,0,0 +MISAPPLICATION,2009,0,0,0,0,0,0 +MISAPPLY,2009,0,0,0,0,0,0 +MISAPPROPRIATES,2009,0,0,0,0,0,0 +MISBRANDED,2009,0,0,0,0,0,0 +MISCALCULATING,2009,0,0,0,0,0,0 +MISCHIEF,2009,0,0,0,0,0,0 +MISCLASSIFY,2014,0,0,0,0,0,0 +MISDEMEANOR,2009,0,0,2009,0,0,0 +MISHANDLE,2009,0,0,0,0,0,0 +MISINFORM,2009,0,0,0,0,0,0 +MISINFORMS,2009,0,0,0,0,0,0 +MISINTERPRETED,2009,0,0,0,0,0,0 +MISJUDGED,2009,0,0,0,0,0,0 +MISJUDGMENTS,2009,0,0,0,0,0,0 +MISLABELLED,2009,0,0,0,0,0,0 +MISLEADINGLY,2009,0,0,0,0,0,0 +MISMANAGED,2009,0,0,0,0,0,0 +MISMATCH,2009,0,0,0,0,0,0 +MISPLACED,2009,0,0,0,0,0,0 +MISREPRESENT,2009,0,0,0,0,0,0 +MISREPRESENTING,2009,0,0,0,0,0,0 +MISSES,2009,0,0,0,0,0,0 +WORRY,2009,0,0,0,0,0,0 +WORSENED,2009,0,0,0,0,0,0 +WORTHLESS,2009,0,0,0,0,0,0 +TOLERATE,2009,0,0,0,0,0,0 +TOLERATION,2009,0,0,0,0,0,0 +TORTS,0,0,0,2009,0,0,0 +FICTITIOUS,2009,0,0,0,0,0,0 +FIRED,2009,0,0,0,0,0,0 +NONATTAINMENT,2009,0,0,0,0,0,0 +NONCOMPETITIVE,2009,0,0,0,0,0,0 +NONCOMPLYING,2009,0,0,0,0,0,0 +NONCONTINGENT,0,0,0,2011,0,0,0 +NONDISCLOSURE,2009,0,0,0,0,0,0 +NONFORFEITABLE,0,0,0,2009,0,0,0 +DISCLAIM,2009,0,0,0,0,0,0 +DISCLAIMING,2009,0,0,0,0,0,0 +DISCLOSES,2009,0,0,0,0,0,0 +DISCONTINUATION,2009,0,0,0,0,0,0 +DISCONTINUES,2009,0,0,0,0,0,0 +DISCOURAGES,2009,0,0,0,0,0,0 +DISCREDITING,2009,0,0,0,0,0,0 +SPECTACULAR,0,2009,0,0,0,0,0 +SPECULATES,0,0,2009,0,0,0,0 +SPECULATIVE,0,0,2009,0,0,0,0 +TRAGEDIES,2009,0,0,0,0,0,0 +TRANSFEROR,0,0,0,2009,0,0,0 +LEGISLATES,0,0,0,2009,0,0,0 +LEGISLATIVE,0,0,0,2009,0,0,0 +LEGISLATURE,0,0,0,2009,0,0,0 +LIBELOUS,0,0,0,2009,0,0,0 +LIENHOLDERS,0,0,0,2011,0,0,0 +COMPULSORY,0,0,0,0,0,0,2009 +CONCEDED,2009,0,0,0,0,0,0 +CONCEIVABLY,0,0,2009,0,0,0,0 +CONCILIATING,2009,0,0,0,0,0,0 +CONCLUSIVELY,0,2009,0,0,0,0,0 +CONDEMNED,2009,0,0,0,0,0,0 +CONDITIONAL,0,0,2009,0,0,0,0 +CONDUCIVE,0,2009,0,0,0,0,0 +CONFESSING,2009,0,0,0,0,0,0 +CONFINED,2009,0,0,0,0,0,2009 +CONFINING,2009,0,0,0,0,0,2009 +CONFISCATING,2009,0,0,0,0,0,0 +CONFLICT,2009,0,0,0,0,0,0 +CONFRONT,2009,0,0,0,0,0,0 +CONFRONTED,2009,0,0,0,0,0,0 +CONFUSED,2009,0,0,0,0,0,0 +CONFUSION,2009,0,2009,0,0,0,0 +CONSENTS,0,0,0,2009,0,0,0 +CONSPIRATOR,2009,0,0,0,0,0,0 +CONSPIRED,2009,0,0,0,0,0,0 +CONSTITUTIONAL,0,0,0,2009,0,0,0 +CONSTITUTIVE,0,0,0,2009,0,0,0 +CONSTRAINS,0,0,0,0,0,0,2009 +CONSTRUCTIVELY,0,2009,0,0,0,0,0 +CONSTRUING,0,0,0,2012,0,0,0 +CONTENDING,2009,0,0,0,0,0,0 +CONTENTIOUS,2009,0,0,0,0,0,0 +CONTESTED,2009,0,0,0,0,0,0 +CONTINGENT,0,0,2009,0,0,0,0 +CONTRACTED,0,0,0,2009,0,0,0 +CONTRACTILE,0,0,0,2009,0,0,0 +CONTRACTS,0,0,0,2009,0,0,0 +CONTRADICTED,2009,0,0,0,0,0,0 +CONTRADICTORY,2009,0,0,0,0,0,0 +CONTRAVENED,0,0,0,2009,0,0,0 +CONTRAVENTIONS,0,0,0,2009,0,0,0 +CONTROVERT,0,0,0,2009,0,0,0 +CONVEYANCE,0,0,0,2009,0,0,0 +CONVICTING,2009,0,0,2009,0,0,0 +CORRECTING,2009,0,0,0,0,0,0 +CORRUPT,2009,0,0,0,0,0,0 +CORRUPTIONS,2009,0,0,0,0,0,0 +COTERMINOUS,0,0,0,2009,0,0,0 +COUNSELLED,0,0,0,2009,0,0,0 +COUNTERCLAIMING,2009,0,0,0,0,0,0 +COUNTERFEITER,2009,0,0,0,0,0,0 +COUNTERMEASURE,2009,0,0,0,0,0,0 +COUNTERSUIT,0,0,0,2011,0,0,0 +COURTROOM,0,0,0,2009,0,0,0 +COVENANTING,0,0,0,0,0,0,2011 +CREATIVENESS,0,2009,0,0,0,0,0 +CRIMINAL,2009,0,0,2009,0,0,0 +CRIMINALLY,2009,0,0,2009,0,0,0 +CRITICAL,-2020,0,0,0,0,0,0 +CRITICIZE,2009,0,0,0,0,0,0 +CROSSCLAIM,0,0,0,2011,0,0,0 +CRUCIAL,2009,0,0,0,0,0,0 +CULPABLY,2009,0,0,0,0,0,0 +CURTAILING,2009,0,0,0,0,0,0 +CUT,2009,0,0,0,0,0,0 +CYBERATTACKS,2014,0,0,0,0,0,0 +CYBERCRIMINAL,2014,0,0,0,0,0,0 +TAINT,2009,0,0,0,0,0,0 +TAMPERED,2009,0,0,0,0,0,0 +TENTATIVE,0,0,2009,0,0,0,0 +TERMINATED,2009,0,0,0,0,0,0 +TERMINATIONS,2009,0,0,0,0,0,0 +TESTIFYING,2009,0,0,2009,0,0,0 +THENCEFORWARD,0,0,0,2009,0,0,0 +THEREIN,0,0,0,2009,0,0,0 +THEREOVER,0,0,0,2011,0,0,0 +THEREUNDER,0,0,0,2009,0,0,0 +THREAT,2009,0,0,0,0,0,0 +THREATENS,2009,0,0,0,0,0,0 +FACTO,0,0,0,2011,0,0,0 +FAILINGS,2009,0,0,0,0,0,0 +FALLOUT,2009,0,0,0,0,0,0 +FALSIFICATIONS,2009,0,0,0,0,0,0 +FALSIFYING,2009,0,0,0,0,0,0 +FATALITY,2009,0,0,0,0,0,0 +FAULTS,2009,0,0,0,0,0,0 +FAVORED,0,2009,0,0,0,0,0 +FEAR,2009,0,0,0,0,0,0 +FELONY,2009,0,0,2009,0,0,0 +DAMAGING,2009,0,0,0,0,0,0 +DANGEROUS,2009,0,0,0,0,0,0 +DEADLOCKED,2009,0,0,0,0,0,0 +DEADWEIGHTS,2009,0,0,0,0,0,0 +DECEASED,2009,0,0,0,0,0,0 +DECEITFUL,2009,0,0,0,0,0,0 +DECEIVES,2009,0,0,0,0,0,0 +DECEPTIVE,2009,0,0,0,0,0,0 +DECLINED,2009,0,0,0,0,0,0 +DECREED,0,0,0,2009,0,0,0 +DEFACED,2009,0,0,0,0,0,0 +DEFAMATION,2009,0,0,0,0,0,0 +DEFAMED,2009,0,0,0,0,0,0 +DEFAULTED,2009,0,0,0,0,0,0 +DEFEASANCES,0,0,0,2011,0,0,0 +DEFEASES,0,0,0,2014,0,0,0 +DEFEATING,2009,0,0,0,0,0,0 +DEFECTIVELY,0,0,0,2009,0,0,0 +DEFENDANT,2009,0,0,2009,0,0,0 +DEFENDS,2009,0,0,0,0,0,0 +DEFICIENCIES,2009,0,0,0,0,0,0 +DEFICITS,2009,0,0,0,0,0,0 +DEFRAUDED,2009,0,0,0,0,0,0 +DEGRADATION,2009,0,0,0,0,0,0 +DEGRADES,2009,0,0,0,0,0,0 +DELAYING,2009,0,0,0,0,0,0 +DELEGATEE,0,0,0,2011,0,0,0 +DELIBERATED,2009,0,0,0,0,0,0 +DELIGHTFUL,0,2009,0,0,0,0,0 +DELINQUENCIES,2009,0,0,0,0,0,0 +DELINQUENTS,2009,0,0,0,0,0,0 +DELISTS,2011,0,0,0,0,0,0 +DEMISING,2009,0,0,0,0,0,0 +DEMOLISHING,2009,0,0,0,0,0,0 +DEMOTED,2009,0,0,0,0,0,0 +DEMOTIONS,2009,0,0,0,0,0,0 +DEMURRING,0,0,0,2009,0,0,0 +DENIED,2009,0,0,0,0,0,0 +DENIGRATES,2009,0,0,0,0,0,0 +DENYING,2009,0,0,0,0,0,0 +DEPENDANCE,0,0,0,0,0,0,2011 +DEPENDENCE,0,0,2009,0,0,0,0 +DEPENDING,0,0,2009,0,0,2009,2011 +DEPLETES,2009,0,0,0,0,0,0 +DEPOSE,0,0,0,2009,0,0,0 +DEPOSITION,0,0,0,2009,0,0,0 +INVALIDATE,2009,0,0,0,0,0,0 +INVALIDATION,2009,0,0,0,0,0,0 +INVENTING,0,2009,0,0,0,0,0 +INVENTIVENESS,0,2009,0,0,0,0,0 +INVESTIGATED,2009,0,0,0,0,0,0 +INVESTIGATIONS,2009,0,0,0,0,0,0 +IRRECONCILABLY,2009,0,0,0,0,0,0 +IRREGULARITIES,2009,0,0,0,0,0,0 +IRREPARABLY,2009,0,0,0,0,0,0 +IRREVOCABLY,0,0,0,2009,0,0,2009 +JUDICIAL,0,0,0,2009,0,0,0 +JURIES,0,0,0,2009,0,0,0 +JURISDICTIONALLY,0,0,0,2011,0,0,0 +JURISTS,0,0,0,2009,0,0,0 +JURYMAN,0,0,0,2009,0,0,0 +KICKBACK,2009,0,0,0,0,0,0 +DIMINISH,2009,0,0,0,0,0,0 +DIMINUTION,2009,0,0,0,0,0,0 +DISADVANTAGED,2009,0,0,0,0,0,0 +DISAFFIRM,0,0,0,2009,0,0,0 +DISAGREE,2009,0,0,0,0,0,0 +DISAGREEMENT,2009,0,0,0,0,0,0 +DISALLOWANCE,2009,0,0,0,0,0,0 +DISALLOWS,2009,0,0,0,0,0,0 +DISAPPEARED,2009,0,0,0,0,0,0 +DISAPPOINTED,2009,0,0,0,0,0,0 +DISAPPOINTMENTS,2009,0,0,0,0,0,0 +DISAPPROVE,2009,0,0,0,0,0,0 +DISASSOCIATES,2009,0,0,0,0,0,0 +DISASTER,2009,0,0,0,0,0,0 +DISAVOW,2009,0,0,0,0,0,0 +DISAVOWS,2009,0,0,0,0,0,0 +GRATUITOUS,2009,0,0,0,0,0,0 +GREATEST,0,2009,0,0,0,0,0 +GRIEVANCES,2009,0,0,0,0,0,0 +HALT,2009,0,0,0,0,0,0 +HAMPERING,2009,0,0,0,0,0,0 +HAPPINESS,0,2009,0,0,0,0,0 +HARASSING,2009,0,0,0,0,0,0 +HARM,2009,0,0,0,0,0,0 +HARMING,2009,0,0,0,0,0,0 +HARSHEST,2009,0,0,0,0,0,0 +HAZARDOUS,2009,0,0,0,0,0,0 +NONINFRINGEMENT,0,0,0,2011,0,0,0 +NONJURISDICTIONAL,0,0,0,2011,0,0,0 +NONPERFORMANCES,2009,0,0,0,0,0,0 +HURTING,2009,0,0,0,0,0,0 +DISFAVORING,2009,0,0,0,0,0,0 +DISGORGEMENT,2009,0,0,0,0,0,0 +COMPLICATE,2009,0,0,0,0,0,0 +COMPLICATION,2009,0,0,0,0,0,0 +COMPLIMENTED,0,2009,0,0,0,0,0 +MISSTATEMENT,2009,0,0,0,0,0,0 +MISSTEP,2009,0,0,0,0,0,0 +MISTAKENLY,2009,0,0,0,0,0,0 +MISTRIALS,2009,0,0,2009,0,0,0 +MISUNDERSTOOD,2009,0,0,0,0,0,0 +MISUSING,2009,0,0,0,0,0,0 +MONOPOLIZE,2009,0,0,0,0,0,0 +MONOPOLY,2009,0,0,0,0,0,0 +MOREOVER,0,0,0,2009,0,0,0 +MUST,0,0,0,0,2009,0,0 +SLIPPAGE,2009,0,0,0,0,0,0 +SLOWDOWNS,2009,0,0,0,0,0,0 +SLOWING,2009,0,0,0,0,0,0 +SLUGGISHLY,2009,0,0,0,0,0,0 +SMOOTHLY,0,2009,0,0,0,0,0 +DEPRESSED,2009,0,0,0,0,0,0 +DEPRIVE,2009,0,0,0,0,0,0 +DERELICT,2009,0,0,0,0,0,0 +DEROGATES,0,0,0,2009,0,0,0 +DEROGATORY,2009,0,0,0,0,0,0 +DESIST,0,0,0,2009,0,0,0 +DESTABILIZED,2009,0,0,0,0,0,0 +DESTROYED,2009,0,0,0,0,0,0 +DESTRUCTIVE,2009,0,0,0,0,0,0 +DETENTION,2009,0,0,0,0,0,0 +DETERIORATED,2009,0,0,0,0,0,0 +DETERIORATIONS,2009,0,0,0,0,0,0 +DETERRENT,2009,0,0,0,0,0,0 +DETRACT,2009,0,0,0,0,0,0 +DETRIMENTAL,2009,0,0,0,0,0,0 +DEVALUED,2009,0,0,0,0,0,0 +DEVASTATED,2009,0,0,0,0,0,0 +DEVIATED,2009,0,2009,0,0,0,0 +DEVIATIONS,2009,0,2009,0,0,0,0 +DEVOLVES,2009,0,0,0,0,0,0 +DICTATES,0,0,0,0,0,0,2009 +DIFFERING,0,0,2009,0,0,0,0 +DIFFICULTLY,2009,0,0,0,0,0,0 +ASCENDANT,0,0,0,2009,0,0,0 +ASSAULTING,2009,0,0,0,0,0,0 +ASSIGNATION,0,0,0,2009,0,0,0 +ASSUMED,0,0,2009,0,0,0,0 +ASSUMPTIONS,0,0,2009,0,0,0,0 +ASSURING,0,2009,0,0,0,0,0 +ATTAINMENT,0,2009,0,0,0,0,0 +ATTESTATION,0,0,0,2009,0,0,0 +ATTORN,0,0,0,2011,0,0,0 +ATTORNS,0,0,0,2011,0,0,0 +AVERSELY,2011,0,0,0,0,0,0 +BAILED,0,0,0,2009,0,0,0 +BAILIFFS,0,0,0,2009,0,0,0 +BALKED,2009,0,0,0,0,0,0 +BANKRUPTED,2009,0,0,0,0,0,0 +CLAIMANT,0,0,0,2009,0,0,0 +CLAIMS,2009,0,0,2009,0,0,0 +CLAWBACKS,0,0,0,2014,0,0,0 +CLOSEOUTS,2009,0,0,0,0,0,0 +CLOSURES,2009,0,0,0,0,0,0 +CODICILS,0,0,0,2009,0,0,0 +CODIFIES,0,0,0,2009,0,0,0 +COERCED,2009,0,0,0,0,0,0 +COERCIVE,2009,0,0,0,0,0,0 +DISINCENTIVES,2009,0,0,0,0,0,0 +COLLABORATE,0,2009,0,0,0,0,0 +COLLABORATION,0,2009,0,0,0,0,0 +COLLABORATORS,0,2009,0,0,0,0,0 +COLLAPSING,2009,0,0,0,0,0,0 +COLLUDED,2009,0,0,0,0,0,0 +COLLUSIONS,2009,0,0,0,0,0,0 +POSITIVE,0,2009,0,0,0,0,0 +POSSIBILITY,0,0,2009,0,0,0,0 +POSTCLOSURE,0,0,0,2011,0,0,0 +POSTPONED,2009,0,0,0,0,0,0 +POSTPONING,2009,0,0,0,0,0,0 +WRIT,0,0,0,2009,0,0,0 +WRITEOFFS,2009,0,0,0,0,0,0 +WRONGDOINGS,2009,0,0,0,0,0,0 +HONORING,0,2009,0,0,0,0,0 +REASSESSES,0,0,2009,0,0,0,0 +REASSIGN,2009,0,0,0,0,0,0 +REASSIGNMENTS,2009,0,0,0,0,0,0 +REBOUNDING,0,2009,0,0,0,0,0 +REBUTTABLY,0,0,0,2011,0,0,0 +REBUTTING,0,0,0,2009,0,0,0 +RECALCULATING,0,0,2009,0,0,0,0 +RECALLED,2009,0,0,0,0,0,0 +RECESSION,2009,0,0,0,0,0,0 +RECKLESSLY,2009,0,0,0,0,0,0 +RECONSIDERING,0,0,2009,0,0,0,0 +RECOUPMENT,0,0,0,2009,0,0,0 +RECTIFICATION,0,0,0,2009,0,0,0 +RECUSED,0,0,0,2011,0,0,0 +REDACTED,2009,0,0,2009,0,0,0 +REDEFAULT,2014,0,0,0,0,0,0 +REDRESSED,2009,0,0,0,0,0,0 +REEXAMINE,0,0,2009,0,0,0,0 +REFERENDUMS,0,0,0,2009,0,0,0 +REFILING,0,0,0,2009,0,0,0 +REFUSAL,2009,0,0,0,0,0,0 +REFUSES,2009,0,0,0,0,0,0 +REGAINING,0,2009,0,0,0,0,0 +REGULATING,0,0,0,2009,0,0,0 +REGULATOR,0,0,0,2009,0,0,0 +REHEARD,0,0,0,2009,0,0,0 +VARIABLES,0,0,2009,0,0,0,0 +VARIANT,0,0,2009,0,0,0,0 +VARIED,0,0,2009,0,0,0,0 +VENDEE,0,0,0,2011,0,0,0 +VERSATILE,0,2009,0,0,0,0,0 +VIBRANCY,0,2009,0,0,0,0,0 +VIOLATED,2009,0,0,0,0,0,0 +VIOLATIONS,2009,0,0,0,0,0,0 +VIOLENCE,2009,0,0,0,0,0,0 +VITIATED,2009,0,0,0,0,0,0 +VOIDABLE,0,0,0,2009,0,0,0 +VOLATILITIES,0,0,2009,0,0,0,0 +VULNERABLE,2009,0,0,0,0,0,0 +WARNING,2009,0,0,0,0,0,0 +WARRANTOR,0,0,0,2011,0,0,0 +WEAK,2009,0,0,0,0,0,0 +WEAKENS,2009,0,0,0,0,0,0 +WEAKNESS,2009,0,0,0,0,0,0 +DISHONOR,2009,0,0,0,0,0,0 +DISHONORING,2009,0,0,0,0,0,0 +DISINTERESTED,2009,0,0,0,0,0,0 +DISLOYALLY,2009,0,0,0,0,0,0 +DISMISS,2009,0,0,0,0,0,0 +DISMISSES,2009,0,0,0,0,0,0 +DISPARAGED,2009,0,0,0,0,0,0 +DISPARAGING,2009,0,0,0,0,0,0 +DISPLACE,2009,0,0,0,0,0,0 +DISPLACES,2009,0,0,0,0,0,0 +DISPOSSESS,2009,0,0,0,0,0,0 +DISPOSSESSION,0,0,0,2009,0,0,0 +DISPROPORTIONATE,2009,0,0,0,0,0,0 +DISPUTES,2009,0,0,0,0,0,0 +DISQUALIFIED,2009,0,0,0,0,0,0 +DISREGARD,2009,0,0,0,0,0,0 +DISREPUTABLE,2009,0,0,0,0,0,0 +DISRUPTING,2009,0,0,0,0,0,0 +DISRUPTS,2009,0,0,0,0,0,0 +DISSENTED,2009,0,0,0,0,0,0 +DISSENTS,2009,0,0,0,0,0,0 +DISSOLUTIONS,2009,0,0,0,0,0,0 +DISTINCTIVELY,0,2009,0,0,0,0,0 +DISTORTING,2009,0,0,0,0,0,0 +DISTRACT,2009,0,0,0,0,0,0 +DISTRACTIONS,2009,0,0,0,0,0,0 +DISTRESSED,2009,0,0,0,0,0,0 +DISTURBANCE,2009,0,0,0,0,0,0 +DISTURBS,2009,0,0,0,0,0,0 +DIVERTING,2009,0,0,0,0,0,0 +DIVESTING,2009,0,0,0,0,0,0 +DIVESTMENTS,2009,0,0,0,0,0,0 +DIVULGE,2009,0,0,0,0,0,0 +DOCKET,0,0,0,2009,0,0,0 +DONEES,0,0,0,2011,0,0,0 +DOUBTS,2009,0,2009,0,0,0,0 +DOWNGRADING,2009,0,0,0,0,0,0 +DOWNSIZING,2009,0,0,0,0,0,0 +DOWNTURN,2009,0,0,0,0,0,0 +DRAG,2009,0,0,0,0,0,0 +DRAWBACKS,2009,0,0,0,0,0,0 +DROUGHTS,2009,0,0,0,0,0,0 +DYSFUNCTIONAL,2009,0,0,0,0,0,0 +EARMARKING,0,0,0,0,0,0,2009 +EASING,2009,0,0,0,0,0,0 +EFFICIENCY,0,2009,0,0,0,0,0 +EGREGIOUSLY,2009,0,0,0,0,0,0 +EMBARGOES,2009,0,0,0,0,0,0 +EMBARRASSES,2009,0,0,0,0,0,0 +EMBEZZLE,2009,0,0,0,0,0,0 +EMBEZZLER,2009,0,0,0,0,0,0 +EMPOWERED,0,2009,0,0,0,0,0 +ENABLED,0,2009,0,0,0,0,0 +ENCOURAGEMENT,0,2009,0,0,0,0,0 +ENCROACHED,2009,0,0,0,0,0,0 +ENCROACHMENTS,2009,0,0,0,0,0,0 +ENCUMBERS,2009,0,0,2009,0,0,2009 +ENCUMBRANCES,2009,0,0,2009,0,0,2009 +ENDANGERMENT,2009,0,0,0,0,0,0 +ENFORCEABLE,0,0,0,2009,0,0,0 +ENHANCEMENT,0,2009,0,0,0,0,0 +ENJOIN,2009,0,0,0,0,0,0 +ENJOY,0,2009,0,0,0,0,0 +ENJOYING,0,2009,0,0,0,0,0 +ENTAILED,0,0,0,0,0,0,2009 +PENALIZE,2009,0,0,0,0,0,0 +PENALTIES,2009,0,0,0,0,0,0 +PERFECTED,0,2009,0,0,0,0,0 +PERIL,2009,0,0,0,0,0,0 +PERMISSION,0,0,0,0,0,0,2009 +PERMITTEES,0,0,0,2011,0,0,0 +PERPETRATES,2009,0,0,2009,0,0,0 +PERSISTED,2009,0,0,0,0,0,0 +PERSISTING,2009,0,0,0,0,0,0 +PERVASIVELY,2009,0,0,0,0,0,0 +PETITIONER,0,0,0,2009,0,0,0 +PETTY,2009,0,0,0,0,0,0 +HEREAFTER,0,0,0,2009,0,0,0 +HEREFORE,0,0,0,2014,0,0,0 +HEREINAFTER,0,0,0,2009,0,0,0 +HEREON,0,0,0,2009,0,0,0 +HEREUNTO,0,0,0,2009,0,0,0 +HIDDEN,0,0,2009,0,0,0,0 +HINDERING,2009,0,0,0,0,0,0 +HINGES,0,0,2009,0,0,0,0 +WHEREABOUTS,0,0,0,2009,0,0,0 +WHEREFORE,0,0,0,2009,0,0,0 +WHERETO,0,0,0,2009,0,0,0 +WHISTLEBLOWERS,0,0,0,2011,0,0,0 +WILFUL,0,0,0,2009,0,0,0 +WILLFULNESS,0,0,0,2009,0,0,0 +WINNING,0,2009,0,0,0,0,0 +FLUCTUATE,0,0,2009,0,0,0,0 +FLUCTUATION,0,0,2009,0,0,0,0 +FORBEARANCE,0,0,0,2009,0,0,0 +FORBID,2009,0,0,0,0,0,2009 +FORCE,-2020,0,0,0,0,0,0 +FOREBEARANCE,0,0,0,2011,0,0,0 +FORECLOSES,2009,0,0,0,0,0,0 +FOREGO,2009,0,0,0,0,0,0 +FORESTALLED,2009,0,0,0,0,0,0 +FORFEITABILITY,0,0,0,2011,0,0,0 +FORFEITS,2009,0,0,0,0,0,0 +FORGERY,2009,0,0,0,0,0,0 +FRAUDS,2009,0,0,0,0,0,0 +FRIENDLY,0,2009,0,0,0,0,0 +FRUSTRATED,2009,0,0,0,0,0,0 +FRUSTRATION,2009,0,0,0,0,0,0 +FURTHERANCE,0,0,0,2009,0,0,0 +GAINS,0,2009,0,0,0,0,0 +DISGORGEMENTS,2009,0,0,0,0,0,0 +DISGRACEFUL,2009,0,0,0,0,0,0 +LITIGATE,2009,0,0,2009,0,0,0 +LITIGATION,2009,0,0,2009,0,0,0 +LITIGIOUS,0,0,0,2009,0,0,0 +LACKED,2009,0,0,0,0,0,0 +LAG,2009,0,0,0,0,0,0 +LAPSE,2009,0,0,0,0,0,0 +LATE,-2020,0,0,0,0,0,0 +LAWFULLY,0,0,0,2009,0,0,0 +LAWS,0,0,0,2009,0,0,0 +LAWYERS,0,0,0,2009,0,0,0 +LEADING,0,2009,0,0,0,0,0 +LEGALIZATION,0,0,0,2009,0,0,0 +LEGALIZES,0,0,0,2009,0,0,0 +LEGATEE,0,0,0,2009,0,0,0 +FLAWED,2009,0,0,0,0,0,0 +NONRECOVERABLE,2009,0,0,0,0,0,0 +LIQUIDATES,2009,0,0,0,0,0,0 +LIQUIDATOR,2009,0,0,0,0,0,0 +BENEFICIATION,0,0,0,2011,0,0,0 +BENEFITTED,0,2009,0,0,0,0,0 +POOR,2009,0,0,0,0,0,0 +REINTERPRETATION,0,0,2009,0,0,0,0 +REINTERPRETS,0,0,2009,0,0,0,0 +REJECTION,2009,0,0,0,0,0,0 +RELINQUISH,2009,0,0,0,0,0,0 +RELINQUISHMENT,2009,0,0,0,0,0,0 +REMAND,0,0,0,2009,0,0,0 +REMEDIATE,0,0,0,2009,0,0,0 +REMEDIATIONS,0,0,0,2009,0,0,0 +RENEGOTIATED,2009,0,0,0,0,0,0 +RENEGOTIATIONS,2009,0,0,0,0,0,0 +RENOUNCEMENTS,2009,0,0,0,0,0,0 +REPARATIONS,2009,0,0,0,0,0,0 +REPOSSESSES,2009,0,0,0,0,0,0 +REPRORATED,0,0,0,2018,0,0,0 +TROUBLED,2009,0,0,0,0,0,0 +UNABLE,2009,0,0,0,0,0,0 +UNAMBIGUOUSLY,0,0,0,0,2009,0,0 +UNAPPEALED,0,0,0,2011,0,0,0 +UNAVAILABILITY,2009,0,0,0,0,0,2011 +UNAWARE,2009,0,0,0,0,0,0 +UNCERTAINTY,0,0,2009,0,0,0,0 +UNCOLLECTIBILITY,2009,0,0,0,0,0,0 +UNCOMPLETED,2009,0,0,0,0,0,0 +UNCONSCIONABLY,2009,0,0,0,0,0,0 +UNCONTRACTED,0,0,0,2011,0,0,0 +UNCORRECTED,2009,0,0,0,0,0,0 +UNCOVERS,2009,0,0,0,0,0,0 +UNDELIVERABLE,2009,0,0,0,0,0,0 +UNDERCUTS,2009,0,0,0,0,0,0 +UNDERESTIMATES,2009,0,0,0,0,0,0 +UNDERINSURED,2009,0,0,0,0,0,0 +UNDERMINING,2009,0,0,0,0,0,0 +UNDERPAYS,2009,0,0,0,0,0,0 +UNDERPERFORMING,2009,0,0,0,0,0,0 +UNDERREPORTING,2011,0,0,0,0,0,0 +UNDERSTATEMENTS,2009,0,0,0,0,0,0 +UNDERUTILIZED,2009,0,0,0,0,0,0 +UNDETECTABLE,0,0,2009,0,0,0,0 +UNDISCHARGED,0,0,0,2009,0,0,0 +UNDOUBTEDLY,0,0,0,0,2009,0,0 +UNECONOMICAL,2009,0,0,0,0,0,0 +UNENCUMBER,0,0,0,2014,0,0,0 +UNEQUIVOCAL,0,0,0,0,2009,0,0 +UNEXCUSED,2009,0,0,0,0,0,0 +UNFAIRLY,2009,0,0,0,0,0,0 +UNFAVORABLE,2009,0,0,0,0,0,0 +UNFIT,2009,0,0,0,0,0,0 +UNFORESEEN,2009,0,0,0,0,0,0 +UNFOUNDED,2009,0,0,0,0,0,0 +UNGUARANTEED,0,0,2009,0,0,0,0 +UNINSURED,2009,0,0,0,0,0,0 +UNJUST,2009,0,0,0,0,0,0 +UNJUSTLY,2009,0,0,0,0,0,0 +UNKNOWNS,0,0,2009,0,0,0,0 +UNLICENSED,2009,0,0,0,0,0,0 +UNMERCHANTABLE,2011,0,0,0,0,0,0 +UNNEEDED,2009,0,0,0,0,0,0 +UNPAID,2009,0,0,0,0,0,0 +UNPOPULAR,2009,0,0,0,0,0,0 +UNPREDICTED,2011,0,2011,0,0,0,0 +UNPROVED,0,0,2009,0,0,0,0 +UNQUANTIFIED,0,0,2011,0,0,0,0 +UNREASONABLY,2009,0,0,0,0,0,0 +UNRECOVERED,2009,0,0,0,0,0,0 +UNREMEDIED,2009,0,0,0,0,0,0 +UNSAFE,2009,0,0,0,0,0,0 +UNSATISFIED,2009,0,0,0,0,0,0 +UNSEASONABLY,0,0,2009,0,0,0,0 +UNSOUND,2009,0,0,0,0,0,0 +UNSTABLE,2009,0,0,0,0,0,0 +UNSUCCESSFULLY,2009,0,0,0,0,0,0 +UNSUITED,2009,0,0,0,0,0,0 +UNSUSPECTING,2009,0,0,0,0,0,0 +UNTIMELY,2009,0,0,0,0,0,0 +UNTRUTHFUL,2009,0,0,0,0,0,0 +UNUSABLE,2009,0,0,0,0,0,0 +UNWARRANTED,2009,0,0,0,0,0,0 +UNWRITTEN,0,0,2009,0,0,0,0 +URGENCY,2009,0,0,0,0,0,0 +USURPATION,0,0,0,2009,0,0,0 +USURY,2009,0,0,2009,0,0,0 +VAGUENESS,0,0,2012,0,0,0,0 +VALUABLE,0,2009,0,0,0,0,0 +PLEAD,2009,0,0,0,0,0,0 +PLEADS,2009,0,0,2009,0,0,0 +PLEASED,0,2009,0,0,0,0,0 +PLEDGED,0,0,0,0,0,0,2009 +PLEDGING,0,0,0,0,0,0,2009 +COMPEL,0,0,0,0,0,0,2011 +COMPENSATORY,0,0,0,2009,0,0,0 +COMPLAINED,2009,0,0,0,0,0,0 +COMPLAINTS,2009,0,0,0,0,0,0 +COMMITMENT,0,0,0,0,0,0,2009 +COMMITTING,0,0,0,0,0,0,2009 +LIMITATION,2009,0,0,0,0,0,0 +LINGERING,2009,0,0,0,0,0,0 +RESTRAINING,0,0,0,0,0,0,2009 +RESTRICT,0,0,0,0,0,0,2009 +RESTRICTIONS,0,0,0,0,0,0,2009 +RESTRICTS,0,0,0,0,0,0,2009 +RESTRUCTURING,2009,0,0,0,0,0,0 +RETALIATES,2009,0,0,0,0,0,0 +RETALIATORY,2009,0,0,0,0,0,0 +PRECAUTIONS,0,0,2009,0,0,0,0 +PRECLUDE,2009,0,0,0,0,0,2009 +PRECONDITION,0,0,0,0,0,0,2009 +PREDECEASED,0,0,0,2009,0,0,0 +PREDICTABILITY,0,0,2009,0,0,0,0 +PREDICTIONS,0,0,2009,0,0,0,0 +PREDICTS,0,0,2009,0,0,0,0 +PREJUDICE,2009,0,0,2009,0,0,0 +PREJUDICING,2009,0,0,2009,0,0,0 +PREMATURELY,2009,0,0,0,0,0,0 +PRESET,0,0,0,0,0,0,2009 +PRESUMABLY,0,0,2009,0,0,0,0 +PRESUMING,0,0,2009,0,0,0,0 +PRETRIAL,2009,0,0,2009,0,0,0 +PREVENTION,2009,0,0,0,0,0,0 +PROACTIVE,0,2009,0,0,0,0,0 +PROBABILITY,0,0,2009,0,0,0,0 +PROBATED,0,0,0,2009,0,0,0 +PROBATIONAL,0,0,0,2009,0,0,0 +PROBATIONS,0,0,0,2009,0,0,0 +PROBLEMS,2009,0,0,0,0,0,0 +PROFITABILITY,0,2009,0,0,0,0,0 +PROGRESSED,0,2009,0,0,0,0,0 +PROHIBITED,0,0,0,0,0,0,2009 +PROHIBITIVE,0,0,0,0,0,0,2009 +PROLONG,2009,0,0,0,0,0,0 +PROLONGING,2009,0,0,0,0,0,0 +PROMULGATES,0,0,0,2009,0,0,0 +PROMULGATOR,0,0,0,2009,0,0,0 +PRORATION,0,0,0,2009,0,0,0 +PROSECUTING,2009,0,0,2009,0,0,0 +PROSECUTORIAL,0,0,0,2011,0,0,0 +PROSPERITY,0,2009,0,0,0,0,0 +PROTESTED,2009,0,0,0,0,0,0 +PROTESTOR,2009,0,0,0,0,0,0 +PROTRACTION,2009,0,0,0,0,0,0 +PROVOKE,2009,0,0,0,0,0,0 +PUNISHABLE,0,0,0,2009,0,0,0 +PUNISHMENT,2009,0,0,0,0,0,0 +PURPORTED,2009,0,0,0,0,0,0 +QUESTION,2009,0,0,0,0,0,0 +QUESTIONING,2009,0,0,0,0,0,0 +QUITCLAIMS,0,0,0,2009,0,0,0 +RANDOM,0,0,2009,0,0,0,0 +RANDOMIZING,0,0,2009,0,0,0,0 +RATABLE,0,0,0,2009,0,0,0 +RATIONALIZE,2009,0,0,0,0,0,0 +ABANDONED,2009,0,0,0,0,0,0 +ABANDONS,2009,0,0,0,0,0,0 +ABDICATION,2009,0,0,0,0,0,0 +ABERRATIONAL,2009,0,0,0,0,0,0 +ABEYANCES,0,0,2009,0,0,0,0 +ABNORMAL,2009,0,0,0,0,0,0 +ABOLISH,2009,0,0,0,0,0,0 +ABOVEMENTIONED,0,0,0,2011,0,0,0 +ABROGATING,2009,0,0,2009,0,0,0 +ABRUPTLY,2009,0,0,0,0,0,0 +ABSENTEEISM,2009,0,0,0,0,0,0 +ABSOLVING,0,0,0,2009,0,0,0 +ABUSED,2009,0,0,0,0,0,0 +ABUSIVELY,2009,0,0,0,0,0,0 +ACCIDENT,2009,0,0,0,0,0,0 +ACCLAIMED,0,2009,0,0,0,0,0 +ACCOMPLISHING,0,2009,0,0,0,0,0 +ACCUSATIONS,2009,0,0,0,0,0,0 +ACCUSING,2009,0,0,0,0,0,0 +ACHIEVEMENTS,0,2009,0,0,0,0,0 +ACQUIESCED,2009,0,0,0,0,0,0 +ACQUIRORS,0,0,0,2011,0,0,0 +ACQUITTALS,2009,0,0,2009,0,0,0 +ACQUITTING,2009,0,0,2009,0,0,0 +ADJOURNED,0,0,0,2009,0,0,0 +ADJOURNS,0,0,0,2009,0,0,0 +ADJUDGING,0,0,0,2009,0,0,0 +ADJUDICATING,0,0,0,2009,0,0,0 +ADJUDICATOR,0,0,0,2009,0,0,0 +ADMISSIBLE,0,0,0,2009,0,0,0 +ADULTERATE,2009,0,0,0,0,0,0 +ADULTERATIONS,2009,0,0,0,0,0,0 +ADVANCING,0,2009,0,0,0,0,0 +ADVANTAGEOUSLY,0,2009,0,0,0,0,0 +ADVERSARY,2009,0,0,0,0,0,0 +ADVERSITY,2009,0,0,0,0,0,0 +AFFREIGHTMENT,0,0,0,2012,0,0,0 +AFORESTATED,0,0,0,2011,0,0,0 +AGGRAVATE,2009,0,0,0,0,0,0 +AGGRAVATION,2009,0,0,0,0,0,0 +ALERTING,2009,0,0,0,0,0,0 +ALIENATING,2009,0,0,0,0,0,0 +ALLEGATIONS,2009,0,0,2009,0,0,0 +ALLEGES,2009,0,0,2009,0,0,0 +ALMOST,0,0,2009,0,0,2009,0 +AMBIGUITIES,0,0,2009,0,0,0,0 +AMENDABLE,0,0,0,2009,0,0,0 +AMENDMENT,0,0,0,2009,0,0,0 +ANNOYANCE,2009,0,0,0,0,0,0 +ANNOYS,2009,0,0,0,0,0,0 +ANNULMENT,2009,0,0,0,0,0,0 +ANOMALOUS,2009,0,2009,0,0,0,0 +ANTECEDENTS,0,0,0,2009,0,0,0 +ANTICIPATING,0,0,2009,0,0,0,0 +ANTICORRUPTION,0,0,0,2014,0,0,0 +APPARENTLY,0,0,2009,0,0,2009,0 +APPEALING,0,0,0,2009,0,0,0 +APPEARING,0,0,2009,0,0,2009,0 +APPELLATE,0,0,0,2009,0,0,0 +APPROXIMATED,0,0,2009,0,0,0,0 +APPROXIMATION,0,0,2009,0,0,0,0 +APPURTENANT,0,0,0,2009,0,0,0 +ARBITRARINESS,0,0,2009,0,0,0,0 +ARBITRATES,0,0,0,2009,0,0,0 +ARBITRATIONS,0,0,0,2009,0,0,0 +ARGUE,2009,0,0,0,0,0,0 +ARGUMENTATIVE,2009,0,0,0,0,0,0 +ARREARS,2009,0,0,0,0,0,0 +ARTIFICIALLY,2009,0,0,0,0,0,0 +RETRIBUTION,2009,0,0,0,0,0,0 +RETROCESSIONAIRES,0,0,0,2011,0,0,0 +BOLSTERS,0,2009,0,0,0,0,0 +BOOMING,0,2009,0,0,0,0,0 +BOTTLENECKS,2009,0,0,0,0,0,0 +BOYCOTTED,2009,0,0,0,0,0,0 +BREACHED,2009,0,0,2009,0,0,0 +BREAKAGE,2009,0,0,0,0,0,0 +BREAKING,-2020,0,0,0,0,0,0 +BRIBE,2009,0,0,0,0,0,0 +BRIBES,2009,0,0,0,0,0,0 +BROKEN,-2020,0,0,0,0,0,0 +BURDENS,2009,0,0,0,0,0,0 +CALAMITOUS,2009,0,0,0,0,0,0 +CANCELING,2009,0,0,0,0,0,0 +CANCELLING,2009,0,0,0,0,0,0 +CARELESSNESS,2009,0,0,0,0,0,0 +CATASTROPHIC,2009,0,0,0,0,0,0 +CAUTIONED,2009,0,0,0,0,0,0 +CAUTIOUSLY,0,0,2009,0,0,0,0 +CEASES,2009,0,0,0,0,0,0 +CENSURE,2009,0,0,0,0,0,0 +CERTIORARI,0,0,0,2011,0,0,0 +CHALLENGES,2009,0,0,0,0,0,0 +CHATTEL,0,0,0,2009,0,0,0 +CIRCUMVENTED,2009,0,0,0,0,0,0 +CIRCUMVENTS,2009,0,0,0,0,0,0 +SHALL,0,0,0,2009,0,0,0 +SHORTAGES,2009,0,0,0,0,0,0 +SHRINKAGES,2009,0,0,0,0,0,0 +SHUTS,2009,0,0,0,0,0,0 +SLANDEROUS,2009,0,0,0,0,0,0 +REPUDIATING,2009,0,0,0,0,0,0 +REQUESTOR,0,0,0,2011,0,0,0 +REQUIREMENTS,0,0,0,0,0,0,2009 +RESCIND,0,0,0,2009,0,0,0 +RESCISSION,0,0,0,2009,0,0,0 +RESIGNATIONS,2009,0,0,0,0,0,0 +RESOLVE,0,2009,0,0,0,0,0 +RESTATEMENTS,2009,0,0,0,0,0,0 +NOTARIES,0,0,0,2009,0,0,0 +NOTARIZED,0,0,0,2009,0,0,0 +NOVO,0,0,0,2011,0,0,0 +NULLIFICATIONS,2009,0,0,2009,0,0,0 +NULLIFYING,2009,0,0,2009,0,0,0 +OBJECTING,2009,0,0,0,0,0,0 +OBJECTIONS,2009,0,0,0,0,0,0 +OBLIGATING,0,0,0,0,0,0,2009 +OBLIGE,0,0,0,0,0,0,2009 +OBLIGES,0,0,0,0,0,0,2009 +OBSCENITY,2009,0,0,0,0,0,0 +OBSTACLES,2009,0,0,0,0,0,0 +OBSTRUCTION,2009,0,0,0,0,0,0 +OFFENCES,2009,0,0,0,0,0,0 +OFFENDERS,2009,0,0,0,0,0,0 +OFFEREE,0,0,0,2009,0,0,0 +OMISSION,2009,0,0,0,0,0,0 +OMITTED,2009,0,0,0,0,0,0 +OPPORTUNISTICALLY,2009,0,0,0,0,0,0 +OPPOSED,2009,0,0,0,0,0,0 +OPPOSITIONS,2009,0,0,0,0,0,0 +ORDINARILY,0,0,2009,0,0,0,0 +OUTMODED,2009,0,0,0,0,0,0 +OUTPERFORMS,0,2009,0,0,0,0,0 +OVERBUILDING,2009,0,0,0,0,0,0 +OVERBURDENED,2009,0,0,0,0,0,0 +OVERCHARGE,2009,0,0,0,0,0,0 +OVERCOME,2009,0,0,0,0,0,0 +OVERESTIMATE,2009,0,0,0,0,0,0 +OVERESTIMATION,2009,0,0,0,0,0,0 +OVERLOADING,2009,0,0,0,0,0,0 +OVERLOOKING,2009,0,0,0,0,0,0 +OVERPAYMENTS,2009,0,0,0,0,0,0 +OVERPRODUCTION,2009,0,0,0,0,0,0 +OVERRULING,0,0,0,2009,0,0,0 +OVERSHADOW,2009,0,0,0,0,0,0 +OVERSTATE,2009,0,0,0,0,0,0 +OVERSTATES,2009,0,0,0,0,0,0 +OVERSUPPLY,2009,0,0,0,0,0,0 +OVERTURNED,2009,0,0,0,0,0,0 +OVERVALUED,2009,0,0,0,0,0,0 +PARA,0,0,0,-2020,0,0,0 +NECESSITATED,0,0,0,0,0,0,2009 +NEGATIVELY,2009,0,0,0,0,0,0 +NEGLECTFUL,2009,0,0,0,0,0,0 +NEGLIGENCES,2009,0,0,0,0,0,0 +NOLO,0,0,0,2011,0,0,0 +BEAUTIFUL,0,2009,0,0,0,0,0 +BELIEVES,0,0,2009,0,0,0,0 +IDEAL,0,2009,0,0,0,0,0 +IGNORE,2009,0,0,0,0,0,0 +ILL,2009,0,0,0,0,0,0 +ILLEGALLY,2009,0,0,0,0,0,0 +ILLIQUID,2009,0,0,0,0,0,0 +IMMATERIALITY,0,0,0,2009,0,0,0 +IMPAIRED,2009,0,0,0,0,0,2011 +IMPAIRS,2009,0,0,0,0,0,2011 +IMPEDED,2009,0,0,0,0,0,0 +IMPEDING,2009,0,0,0,0,0,0 +IMPERFECTIONS,2009,0,0,0,0,0,0 +IMPLICATE,2009,0,0,0,0,0,0 +IMPOSE,0,0,0,0,0,0,2009 +IMPOSITION,0,0,0,0,0,0,2009 +IMPOUND,2009,0,0,0,0,0,0 +IMPRACTICABLE,2009,0,0,0,0,0,0 +IMPRECISE,0,0,2009,0,0,0,0 +IMPRESSED,0,2009,0,0,0,0,0 +IMPRESSIVELY,0,2009,0,0,0,0,0 +IMPROPER,2009,0,0,0,0,0,0 +IMPROVE,0,2009,0,0,0,0,0 +IMPROVES,0,2009,0,0,0,0,0 +INABILITY,2009,0,0,0,0,0,0 +INACCURATE,2009,0,0,0,0,0,0 +INACTIVATE,2009,0,0,0,0,0,0 +INACTIVATION,2009,0,0,0,0,0,0 +INADEQUACY,2009,0,0,0,0,0,0 +INADVERTENTLY,2009,0,0,0,0,0,0 +INAPPROPRIATELY,2009,0,0,0,0,0,0 +INCAPACITATED,2009,0,0,0,0,0,0 +INCARCERATES,2009,0,0,2009,0,0,0 +INCHOATE,0,0,0,2009,0,0,0 +INCIDENTS,2009,0,0,0,0,0,0 +INCOMPETENCE,2009,0,0,0,0,0,0 +INCOMPETENTS,2009,0,0,0,0,0,0 +INCONCLUSIVE,2009,0,0,0,0,0,0 +INCONSISTENTLY,2009,0,0,0,0,0,0 +INCONVENIENCES,2009,0,0,0,0,0,0 +INCORRECTNESS,2009,0,0,0,0,0,0 +INDECENCY,2009,0,0,0,0,0,0 +INDEFINITE,0,0,2009,0,0,0,0 +INDEMNIFICATION,0,0,0,2009,0,0,0 +INDEMNIFY,0,0,0,2009,0,0,0 +INDEMNITIES,0,0,0,2009,0,0,0 +INDETERMINABLE,0,0,2009,0,0,0,0 +INDICTED,2009,0,0,2009,0,0,0 +INDORSEES,0,0,0,2011,0,0,0 +INEFFICIENCIES,2009,0,0,0,0,0,0 +INELIGIBILITY,2009,0,0,0,0,0,0 +INEQUITIES,2009,0,0,0,0,0,0 +INEXACTNESS,0,0,2009,0,0,0,0 +INFLICTED,2009,0,0,0,0,0,0 +INFRACTION,2009,0,0,2009,0,0,0 +INFRINGEMENT,2009,0,0,0,0,0,0 +INFRINGING,2009,0,0,0,0,0,0 +INHIBITING,0,0,0,0,0,0,2011 +INJUNCTIONS,2009,0,0,2009,0,0,0 +INJURES,2009,0,0,0,0,0,0 +INJURY,2009,0,0,0,0,0,0 +INNOVATING,0,2009,0,0,0,0,0 +INNOVATIVENESS,0,2011,0,0,0,0,0 +INORDINATELY,2009,0,0,0,0,0,0 +INSIGHTFUL,0,2009,0,0,0,0,0 +INSISTING,0,0,0,0,0,0,2009 +INSOLVENCY,2009,0,0,0,0,0,0 +INSTABILITIES,0,0,2009,0,0,0,0 +INSUFFICIENT,2009,0,0,0,0,0,0 +INTANGIBLE,0,0,2009,0,0,0,0 +INTERFERE,2009,0,0,0,0,0,0 +INTERFERES,2009,0,0,0,0,0,0 +INTERMITTENTLY,2009,0,0,0,0,0,0 +INTERPOSES,0,0,0,2009,0,0,0 +INTERROGATE,0,0,0,2009,0,0,0 +INTERROGATION,0,0,0,2009,0,0,0 +INTERROGATORS,0,0,0,2009,0,0,0 +INTERRUPTING,2009,0,0,0,0,0,0 +INTESTACY,0,0,0,2009,0,0,0 +STABILIZATION,0,2009,0,0,0,0,0 +STABILIZES,0,2009,0,0,0,0,0 +STAGNANT,2009,0,0,0,0,0,0 +STAGNATING,2009,0,0,0,0,0,0 +STATUTE,0,0,0,2009,0,0,0 +STIPULATE,0,0,0,0,0,0,2009 +STIPULATION,0,0,0,0,0,0,2009 +STOPPAGES,2009,0,0,0,0,0,0 +STRAIN,2009,0,0,0,0,0,0 +STRENGTH,0,2009,0,0,0,0,0 +STRENGTHENS,0,2009,0,0,0,0,0 +STRESSES,2009,0,0,0,0,0,0 +STRICTER,0,0,0,0,0,0,2009 +STRONG,0,2009,0,0,0,0,0 +SUBCLAUSE,0,0,0,2009,0,0,0 +SUBJECTING,2009,0,0,0,0,0,0 +SUBLESSORS,0,0,0,2011,0,0,0 +SUBPARAGRAPHS,0,0,0,2009,0,0,0 +SUBROGATED,0,0,0,2009,0,0,0 +SUBTRUSTS,0,0,0,2011,0,0,0 +SUCCEEDS,0,2009,0,0,0,0,0 +SUCCESSFULLY,0,2009,0,0,0,0,0 +SUED,2009,0,0,2009,0,0,0 +SUFFERING,2009,0,0,0,0,0,0 +SUGGESTING,0,0,2009,0,0,0,0 +SUMMONING,2009,0,0,2009,0,0,0 +SUPERSEDE,0,0,0,2009,0,0,0 +SUPERSEDING,0,0,0,2009,0,0,0 +SURPASSED,0,2009,0,0,0,0,0 +SUSCEPTIBLE,2009,0,0,0,0,0,0 +SUSPEND,2009,0,0,0,0,0,0 +SUSPENSION,2009,0,0,0,0,0,0 +SUSPICIOUS,2009,0,0,0,0,0,0 +REVOCATION,2009,0,0,2009,0,0,0 +REVOKES,2009,0,0,0,0,0,0 +REVOLUTIONIZES,0,2009,0,0,0,0,0 +REWARDING,0,2009,0,0,0,0,0 +RIDICULES,2009,0,0,0,0,0,0 +RISKIER,2009,0,2009,0,0,0,0 +RISKS,0,0,2009,0,0,0,0 +RULINGS,0,0,0,2009,0,0,0 +SACRIFICED,2009,0,0,0,0,0,0 +SATISFACTION,0,2009,0,0,0,0,0 +SATISFIES,0,2009,0,0,0,0,0 +SCANDALS,2009,0,0,0,0,0,0 +SCRUTINIZING,2009,0,0,0,0,0,0 +SEIZE,2009,0,0,0,0,0,0 +SELDOM,0,0,2009,0,0,2009,0 +SEQUESTRATOR,0,0,0,2009,0,0,0 +SETBACK,2009,0,0,0,0,0,0 +SEVER,2009,0,0,0,0,0,0 +SEVERANCE,0,0,0,2009,0,0,0 +SEVERELY,2009,0,0,0,0,0,0 +ENTRENCH,0,0,0,0,0,0,2009 +ERODES,2009,0,0,0,0,0,0 +ERRATICALLY,2009,0,0,0,0,0,0 +ERRONEOUSLY,2009,0,0,0,0,0,0 +ESCALATE,2009,0,0,0,0,0,0 +ESCHEAT,0,0,0,2011,0,0,0 +ESCROWED,0,0,0,0,0,0,2009 +EVADE,2009,0,0,0,0,0,0 +EVASION,2009,0,0,0,0,0,0 +EVICTED,2009,0,0,0,0,0,0 +EVICTS,2009,0,0,0,0,0,0 +EXACERBATED,2009,0,0,0,0,0,0 +EXACERBATIONS,2009,0,0,0,0,0,0 +EXAGGERATING,2009,0,0,0,0,0,0 +EXCEEDENCES,0,0,0,2011,0,0,0 +EXCELS,0,2009,0,0,0,0,0 +EXCESSIVELY,2009,0,0,0,0,0,0 +EXCITING,0,2009,0,0,0,0,0 +EXCLUSIVES,0,2009,0,0,0,0,0 +EXCULPATES,2009,0,0,2009,0,0,0 +EXCULPATORY,2009,0,0,2009,0,0,0 +EXECUTRICES,0,0,0,2009,0,0,0 +EXONERATE,2009,0,0,0,0,0,0 +EXONERATION,2009,0,0,0,0,0,0 +EXPLOITATIONS,2009,0,0,0,0,0,0 +EXPLOITS,2009,0,0,0,0,0,0 +EXPOSING,2009,0,0,0,0,0,0 +EXPROPRIATED,2009,0,0,0,0,0,0 +EXPROPRIATIONS,2009,0,0,0,0,0,0 +EXTRACONTRACTUAL,0,0,0,2011,0,0,0 +SOLVENCIES,2009,0,0,0,0,0,0 +SOMETIME,0,0,2009,0,0,0,0 +SPAM,2014,0,0,0,0,0,0 +TRAUMATIC,2009,0,0,0,0,0,0 +LOSES,2009,0,0,0,0,0,0 +LOST,2009,0,0,0,0,0,0 +LYING,2009,0,0,0,0,0,0 +MALFUNCTIONED,2009,0,0,0,0,0,0 +MALICIOUS,2009,0,0,0,0,0,0 +MANDATE,0,0,0,0,0,0,2009 +MANDATORY,0,0,0,0,0,0,2009 +MANIPULATES,2009,0,0,0,0,0,0 +MANIPULATIVE,2009,0,0,0,0,0,0 +MAYBE,0,0,2009,0,0,2009,0 +MEDIATING,0,0,0,2009,0,0,0 +MEDIATORS,0,0,0,2009,0,0,0 +MISAPPLICATIONS,2009,0,0,0,0,0,0 +MISAPPLYING,2009,0,0,0,0,0,0 +MISAPPROPRIATING,2009,0,0,0,0,0,0 +MISCALCULATE,2009,0,0,0,0,0,0 +MISCALCULATION,2009,0,0,0,0,0,0 +MISCLASSIFICATION,2011,0,0,0,0,0,0 +MISCOMMUNICATION,2014,0,0,0,0,0,0 +MISDEMEANORS,2009,0,0,0,0,0,0 +MISHANDLED,2009,0,0,0,0,0,0 +MISINFORMATION,2009,0,0,0,0,0,0 +MISINTERPRET,2009,0,0,0,0,0,0 +MISINTERPRETING,2009,0,0,0,0,0,0 +MISJUDGES,2009,0,0,0,0,0,0 +MISLABEL,2009,0,0,0,0,0,0 +MISLABELS,2009,0,0,0,0,0,0 +MISLEADS,2009,0,0,0,0,0,0 +MISMANAGEMENT,2009,0,0,0,0,0,0 +MISMATCHED,2009,0,0,0,0,0,0 +MISPRICE,2014,0,0,0,0,0,0 +MISREPRESENTATION,2009,0,0,0,0,0,0 +MISREPRESENTS,2009,0,0,0,0,0,0 +WORRYING,2009,0,0,0,0,0,0 +WORSENING,2009,0,0,0,0,0,0 +WORTHY,0,2009,0,0,0,0,0 +TOLERATED,2009,0,0,0,0,0,0 +TORT,0,0,0,2009,0,0,0 +TORTUOUS,2009,0,0,0,0,0,0 +FIDE,0,0,0,2009,0,0,0 +FIRING,2009,0,0,0,0,0,0 +NONBREACHING,0,0,0,2011,0,0,0 +NONCOMPLIANCE,2009,0,0,0,0,0,0 +NONCONFORMING,2009,0,0,0,0,0,0 +NONCONTRACT,0,0,0,2012,0,0,0 +NONFEASANCE,0,0,0,2011,0,0,0 +NONFORFEITURE,0,0,0,2011,0,0,0 +DISCLAIMED,2009,0,0,0,0,0,0 +DISCLAIMS,2009,0,0,0,0,0,0 +DISCLOSING,2009,0,0,0,0,0,0 +DISCONTINUATIONS,2009,0,0,0,0,0,0 +DISCONTINUING,2009,0,0,0,0,0,0 +DISCOURAGING,2009,0,0,0,0,0,0 +DISCREDITS,2009,0,0,0,0,0,0 +SPECTACULARLY,0,2009,0,0,0,0,0 +SPECULATING,0,0,2009,0,0,0,0 +SPECULATIVELY,0,0,2009,0,0,0,0 +TRAGEDY,2009,0,0,0,0,0,0 +TRANSFERORS,0,0,0,2012,0,0,0 +LEGISLATING,0,0,0,2009,0,0,0 +LEGISLATIVELY,0,0,0,2009,0,0,0 +LEGISLATURES,0,0,0,2009,0,0,0 +LIBELS,0,0,0,2009,0,0,0 +CONCEALED,2009,0,0,0,0,0,0 +CONCEDES,2009,0,0,0,0,0,0 +CONCERN,2009,0,0,0,0,0,0 +CONCILIATION,2009,0,0,0,0,0,0 +CONDEMN,2009,0,0,0,0,0,0 +CONDEMNING,2009,0,0,0,0,0,0 +CONDITIONALLY,0,0,2009,0,0,0,0 +CONFESS,2009,0,0,0,0,0,0 +CONFESSION,2009,0,0,0,0,0,0 +CONFINEMENT,2009,0,0,0,0,0,2009 +CONFISCATE,2009,0,0,0,0,0,0 +CONFISCATION,2009,0,0,0,0,0,0 +CONFLICTED,2009,0,0,0,0,0,0 +CONFRONTATION,2009,0,0,0,0,0,0 +CONFRONTING,2009,0,0,0,0,0,0 +CONFUSES,2009,0,2009,0,0,0,0 +CONSENT,0,0,0,2009,0,0,0 +CONSERVATORSHIPS,0,0,0,2011,0,0,0 +CONSPIRATORIAL,2009,0,0,0,0,0,0 +CONSPIRES,2009,0,0,0,0,0,0 +CONSTITUTIONALITY,0,0,0,2009,0,0,0 +CONSTRAIN,0,0,0,0,0,0,2009 +CONSTRAINT,0,0,0,0,0,0,2009 +CONSTRUE,0,0,0,2012,0,0,0 +CONTEMPT,2009,0,0,0,0,0,0 +CONTENDS,2009,0,0,0,0,0,0 +CONTENTIOUSLY,2009,0,0,0,0,0,0 +CONTESTING,2009,0,0,0,0,0,0 +CONTINGENTLY,0,0,2009,0,0,0,0 +CONTRACTHOLDER,0,0,0,2009,0,0,0 +CONTRACTING,0,0,0,2009,0,0,0 +CONTRACTUAL,0,0,0,2009,0,0,0 +CONTRADICTING,2009,0,0,0,0,0,0 +CONTRADICTS,2009,0,0,0,0,0,0 +CONTRAVENES,0,0,0,2009,0,0,0 +CONTROVERSIAL,2009,0,0,0,0,0,0 +CONTROVERTED,0,0,0,2009,0,0,0 +CONVEYANCES,0,0,0,2009,0,0,0 +CONVICTION,2009,0,0,2009,0,0,0 +CORRECTION,2009,0,0,0,0,0,0 +CORRUPTED,2009,0,0,0,0,0,0 +CORRUPTLY,2009,0,0,0,0,0,0 +COULD,0,0,2009,0,0,2009,0 +COUNSELS,0,0,0,2009,0,0,0 +COUNTERCLAIMS,2009,0,0,0,0,0,0 +COUNTERFEITERS,2009,0,0,0,0,0,0 +COUNTERMEASURES,2009,0,0,0,0,0,0 +COUNTERSUITS,0,0,0,2014,0,0,0 +COURTS,0,0,0,2009,0,0,0 +COVENANTS,0,0,0,0,0,0,2011 +CREATIVITY,0,2009,0,0,0,0,0 +CRIMINALITY,0,0,0,2009,0,0,0 +CRIMINALS,2009,0,0,2009,0,0,0 +CRITICALLY,2009,0,0,0,0,0,0 +CRITICIZED,2009,0,0,0,0,0,0 +CROSSCLAIMS,0,0,0,2011,0,0,0 +CRUCIALLY,2009,0,0,0,0,0,0 +CUMBERSOME,2009,0,0,0,0,0,0 +CURTAILMENT,2009,0,0,0,0,0,0 +CUTBACK,2009,0,0,0,0,0,0 +CYBERBULLYING,2014,0,0,0,0,0,0 +CYBERCRIMINALS,2014,0,0,0,0,0,0 +TAINTED,2009,0,0,0,0,0,0 +TENANTABILITY,0,0,0,2011,0,0,0 +TENTATIVELY,0,0,2009,0,0,0,0 +TERMINATES,2009,0,0,0,0,0,0 +TERMINUS,0,0,0,-2020,0,0,0 +TESTIMONY,0,0,0,2009,0,0,0 +THEREAFTER,0,0,0,2009,0,0,0 +THEREINAFTER,0,0,0,2011,0,0,0 +THERETO,0,0,0,2009,0,0,0 +THEREUNTO,0,0,0,2009,0,0,0 +THREATEN,2009,0,0,0,0,0,0 +THREATS,2009,0,0,0,0,0,0 +FAIL,2009,0,0,0,0,0,0 +FAILS,2009,0,0,0,0,0,0 +FALSE,2009,0,0,0,0,0,0 +FALSIFIED,2009,0,0,0,0,0,0 +FALSITY,2009,0,0,0,0,0,0 +FATALLY,2009,0,0,0,0,0,0 +FAULTY,2009,0,0,0,0,0,0 +FAVORING,0,2009,0,0,0,0,0 +FEARS,2009,0,0,0,0,0,0 +DAMAGE,2009,0,0,0,0,0,0 +DAMPEN,2009,0,0,0,0,0,0 +DANGEROUSLY,2009,0,0,0,0,0,0 +DEADLOCKING,2009,0,0,0,0,0,0 +DEBARMENT,2009,0,0,0,0,0,0 +DECEDENT,0,0,0,2009,0,0,0 +DECEITFULNESS,2009,0,0,0,0,0,0 +DECEIVING,2009,0,0,0,0,0,0 +DECEPTIVELY,2009,0,0,0,0,0,0 +DECLINES,2009,0,0,0,0,0,0 +DECREEING,0,0,0,2009,0,0,0 +DEFACEMENT,2009,0,0,0,0,0,0 +DEFAMATIONS,2009,0,0,0,0,0,0 +DEFAMES,2009,0,0,0,0,0,0 +DEFAULTING,2009,0,0,0,0,0,0 +DEFEASE,0,0,0,2009,0,0,0 +DEFEASING,0,0,0,2011,0,0,0 +DEFEATS,2009,0,0,0,0,0,0 +DEFECTS,2009,0,0,0,0,0,0 +DEFENDANTS,2009,0,0,2009,0,0,0 +DEFENSIVE,2009,0,0,0,0,0,0 +DEFICIENCY,2009,0,0,0,0,0,0 +DEFINITELY,0,0,0,0,2009,0,0 +DEFRAUDING,2009,0,0,0,0,0,0 +DEGRADATIONS,2009,0,0,0,0,0,0 +DEGRADING,2009,0,0,0,0,0,0 +DELAYS,2009,0,0,0,0,0,0 +DELEGEES,0,0,0,2011,0,0,0 +DELIBERATELY,2009,0,0,0,0,0,0 +DELIGHTFULLY,0,2009,0,0,0,0,0 +DELINQUENCY,2009,0,0,0,0,0,0 +DELIST,2009,0,0,0,0,0,0 +DEMISE,2009,0,0,0,0,0,0 +DEMOLISH,2009,0,0,0,0,0,0 +DEMOLITION,2009,0,0,0,0,0,0 +DEMOTES,2009,0,0,0,0,0,0 +DEMURRED,0,0,0,2009,0,0,0 +DEMURS,0,0,0,2009,0,0,0 +DENIES,2009,0,0,0,0,0,0 +DENIGRATING,2009,0,0,0,0,0,0 +DEPEND,0,0,2009,0,0,2009,2011 +DEPENDANCES,0,0,0,0,0,0,2011 +DEPENDENCIES,0,0,2009,0,0,0,2011 +DEPENDS,0,0,2009,0,0,2009,2011 +DEPLETING,2009,0,0,0,0,0,0 +DEPOSED,0,0,0,2009,0,0,0 +DEPOSITIONAL,0,0,0,2011,0,0,0 +INVALIDATED,2009,0,0,0,0,0,0 +INVALIDITY,2009,0,0,0,0,0,0 +INVENTION,0,2009,0,0,0,0,0 +INVENTOR,0,2009,0,0,0,0,0 +INVESTIGATES,2009,0,0,0,0,0,0 +INVOLUNTARILY,2009,0,0,0,0,0,0 +IRRECOVERABLE,2009,0,0,0,0,0,0 +IRREGULARITY,2009,0,0,0,0,0,0 +IRREVERSIBLE,2009,0,0,0,0,0,0 +JEOPARDIZE,2009,0,0,0,0,0,0 +JUDICIALLY,0,0,0,2009,0,0,0 +JURIS,0,0,0,2011,0,0,0 +JURISDICTIONS,0,0,0,2009,0,0,0 +JUROR,0,0,0,2009,0,0,0 +JUSTICE,0,0,0,2009,0,0,0 +KICKBACKS,2009,0,0,0,0,0,0 +DIMINISHED,2009,0,0,0,0,0,0 +DIRECTIVE,0,0,0,0,0,0,2009 +DISADVANTAGEOUS,2009,0,0,0,0,0,0 +DISAFFIRMANCE,0,0,0,2009,0,0,0 +DISAGREEABLE,2009,0,0,0,0,0,0 +DISAGREEMENTS,2009,0,0,0,0,0,0 +DISALLOWANCES,2009,0,0,0,0,0,0 +DISAPPEAR,2009,0,0,0,0,0,0 +DISAPPEARING,2009,0,0,0,0,0,0 +DISAPPOINTING,2009,0,0,0,0,0,0 +DISAPPOINTS,2009,0,0,0,0,0,0 +DISAPPROVED,2009,0,0,0,0,0,0 +DISASSOCIATING,2009,0,0,0,0,0,0 +DISASTERS,2009,0,0,0,0,0,0 +DISAVOWAL,2009,0,0,0,0,0,0 +GRATUITOUSLY,2009,0,0,0,0,0,0 +GREATLY,0,2009,0,0,0,0,0 +GROSSLY,2009,0,0,0,0,0,0 +HALTED,2009,0,0,0,0,0,0 +HAMPERS,2009,0,0,0,0,0,0 +HAPPY,0,2009,0,0,0,0,0 +HARASSMENT,2009,0,0,0,0,0,0 +HARMED,2009,0,0,0,0,0,0 +HARMS,2009,0,0,0,0,0,0 +HARSHLY,2009,0,0,0,0,0,0 +HAZARDS,2009,0,0,0,0,0,0 +NONINFRINGING,0,0,0,2011,0,0,0 +NONPAYMENT,2009,0,0,0,0,0,0 +NONPERFORMING,2009,0,0,0,0,0,0 +DISFAVORS,2009,0,0,0,0,0,0 +PREAMENDMENT,0,0,0,2011,0,0,0 +COMPLICATED,2009,0,0,0,0,0,0 +COMPLICATIONS,2009,0,0,0,0,0,0 +COMPLIMENTING,0,2009,0,0,0,0,0 +MISSTATEMENTS,2009,0,0,0,0,0,0 +MISSTEPS,2009,0,0,0,0,0,0 +MISTAKES,2009,0,0,0,0,0,0 +MISUNDERSTAND,2009,0,0,0,0,0,0 +MISUSE,2009,0,0,0,0,0,0 +MONOPOLISTIC,2009,0,0,0,0,0,0 +MONOPOLIZED,2009,0,0,0,0,0,0 +MORATORIA,2009,0,0,0,0,0,0 +MOTHBALLED,2009,0,0,0,0,0,0 +MUTANDIS,0,0,0,2011,0,0,0 +SLIPPAGES,2009,0,0,0,0,0,0 +SLOWED,2009,0,0,0,0,0,0 +SLOWLY,2009,0,0,0,0,0,0 +SLUGGISHNESS,2009,0,0,0,0,0,0 +SMOOTHS,0,2009,0,0,0,0,0 +DEPRESSES,2009,0,0,0,0,0,0 +DEPRIVED,2009,0,0,0,0,0,0 +DERELICTION,2009,0,0,0,0,0,0 +DEROGATING,0,0,0,2009,0,0,0 +DESIGNATOR,0,0,0,2011,0,0,0 +DESPITE,0,2009,0,0,0,0,0 +DESTABILIZING,2009,0,2009,0,0,0,0 +DESTROYING,2009,0,0,0,0,0,0 +DETAIN,2009,0,0,0,0,0,0 +DETENTIONS,2009,0,0,0,0,0,0 +DETERIORATES,2009,0,0,0,0,0,0 +DETERRED,2009,0,0,0,0,0,0 +DETERRENTS,2009,0,0,0,0,0,0 +DETRACTED,2009,0,0,0,0,0,0 +DETRIMENTALLY,2009,0,0,0,0,0,0 +DEVALUES,2009,0,0,0,0,0,0 +DEVASTATING,2009,0,0,0,0,0,0 +DEVIATES,2009,0,2009,0,0,0,0 +DEVISEES,0,0,0,2011,0,0,0 +DEVOLVING,2009,0,0,0,0,0,0 +DICTATING,0,0,0,0,0,0,2009 +DIFFERS,0,0,2009,0,0,0,0 +DIFFICULTY,2009,0,0,0,0,0,0 +ASCENDANTS,0,0,0,2009,0,0,0 +ASSAULTS,2009,0,0,0,0,0,0 +ASSIGNATIONS,0,0,0,2009,0,0,0 +ASSUMES,0,0,2009,0,0,0,0 +ASSURE,0,2009,0,0,0,0,0 +ATTAIN,0,2009,0,0,0,0,0 +ATTAINMENTS,0,2009,0,0,0,0,0 +ATTESTATIONS,0,0,0,2009,0,0,0 +ATTORNEY,0,0,0,2009,0,0,0 +ATTRACTIVE,0,2009,0,0,0,0,0 +BACKDATING,2009,0,0,0,0,0,0 +BAILEE,0,0,0,2009,0,0,0 +BAILMENT,0,0,0,2011,0,0,0 +BANKRUPT,2009,0,0,0,0,0,0 +BANKRUPTING,2009,0,0,0,0,0,0 +CLAIMANTS,0,0,0,2009,0,0,0 +CLARIFICATION,0,0,2009,0,0,0,0 +CLEARLY,0,0,0,0,2009,0,0 +CLOSING,-2020,0,0,0,0,0,0 +CODEFENDANT,0,0,0,2011,0,0,0 +CODIFICATION,0,0,0,2009,0,0,0 +CODIFY,0,0,0,2009,0,0,0 +COERCES,2009,0,0,0,0,0,0 +COLLABORATED,0,2009,0,0,0,0,0 +COLLABORATIONS,0,2009,0,0,0,0,0 +COLLAPSE,2009,0,0,0,0,0,0 +COLLISION,2009,0,0,0,0,0,0 +COLLUDES,2009,0,0,0,0,0,0 +COLLUSIVE,2009,0,0,0,0,0,0 +POSITIVELY,0,2009,0,0,0,0,0 +POSSIBLE,0,0,2009,0,0,2009,0 +POSTCONTRACT,0,0,0,2011,0,0,0 +POSTPONEMENT,2009,0,0,0,0,0,0 +WRITEDOWN,2009,0,0,0,0,0,0 +WRITS,0,0,0,2009,0,0,0 +WRONGFUL,2009,0,0,0,0,0,0 +HONOR,0,2009,0,0,0,0,0 +HONORS,0,2009,0,0,0,0,0 +REARGUMENT,0,0,0,2011,0,0,0 +REASSESSING,0,0,2009,0,0,0,0 +REASSIGNED,2009,0,0,0,0,0,0 +REASSIGNS,2009,0,0,0,0,0,0 +REBUT,0,0,0,2009,0,0,0 +REBUTTAL,0,0,0,2009,0,0,0 +RECALCULATE,0,0,2009,0,0,0,0 +RECALCULATION,0,0,2009,0,0,0,0 +RECALLING,2009,0,0,0,0,0,0 +RECESSIONARY,2009,0,0,0,0,0,0 +RECKLESSNESS,2009,0,0,0,0,0,0 +RECONSIDERS,0,0,2009,0,0,0,0 +RECOUPMENTS,0,0,0,2009,0,0,0 +RECTIFICATIONS,0,0,0,2009,0,0,0 +RECUSES,0,0,0,2014,0,0,0 +REDACTING,2009,0,0,2009,0,0,0 +REDEFAULTED,2012,0,0,0,0,0,0 +REDRESSES,2009,0,0,0,0,0,0 +REEXAMINING,0,0,2009,0,0,0,0 +REFILE,0,0,0,2009,0,0,0 +REFRAIN,0,0,0,0,0,0,2009 +REFUSALS,2009,0,0,0,0,0,0 +REFUSING,2009,0,0,0,0,0,0 +REGULATE,0,0,0,2009,0,0,0 +REGULATION,0,0,0,2009,0,0,0 +REGULATORS,0,0,0,2009,0,0,0 +REHEARING,0,0,0,2009,0,0,0 +VARIABLY,0,0,2009,0,0,0,0 +VARIANTS,0,0,2009,0,0,0,0 +VARIES,0,0,2009,0,0,0,0 +VENDEES,0,0,0,2011,0,0,0 +VERSATILITY,0,2009,0,0,0,0,0 +VIBRANT,0,2009,0,0,0,0,0 +VIOLATES,2009,0,0,0,0,0,0 +VIOLATIVE,2009,0,0,2009,0,0,0 +VIOLENT,2009,0,0,0,0,0,0 +VITIATES,2009,0,0,0,0,0,0 +VOIDED,2009,0,0,2009,0,0,0 +VOLATILITY,2009,0,2009,0,0,0,0 +VULNERABLY,2009,0,0,0,0,0,0 +WARNINGS,2009,0,0,0,0,0,0 +WASTED,2009,0,0,0,0,0,0 +WEAKEN,2009,0,0,0,0,0,0 +WEAKER,2009,0,0,0,0,0,0 +WEAKNESSES,2009,0,0,0,0,0,0 +DISHONORABLE,2009,0,0,0,0,0,0 +DISHONORS,2009,0,0,0,0,0,0 +DISINTERESTEDLY,2009,0,0,0,0,0,0 +DISLOYALTY,2009,0,0,0,0,0,0 +DISMISSAL,2009,0,0,0,0,0,0 +DISMISSING,2009,0,0,0,0,0,0 +DISPARAGEMENT,2009,0,0,0,0,0,0 +DISPARAGINGLY,2009,0,0,0,0,0,0 +DISPLACED,2009,0,0,0,0,0,0 +DISPLACING,2009,0,0,0,0,0,0 +DISPOSSESSED,2009,0,0,0,0,0,0 +DISPOSSESSORY,0,0,0,2011,0,0,0 +DISPROPORTIONATELY,2009,0,0,0,0,0,0 +DISPUTING,2009,0,0,0,0,0,0 +DISQUALIFIES,2009,0,0,0,0,0,0 +DISREGARDED,2009,0,0,0,0,0,0 +DISREPUTE,2009,0,0,0,0,0,0 +DISRUPTION,2009,0,0,0,0,0,0 +DISSATISFACTION,2009,0,0,0,0,0,0 +DISSENTER,2009,0,0,0,0,0,0 +DISSIDENT,2009,0,0,0,0,0,0 +DISTINCTION,0,2009,0,0,0,0,0 +DISTINCTIVENESS,0,2009,0,0,0,0,0 +DISTORTION,2009,0,0,0,0,0,0 +DISTRACTED,2009,0,0,0,0,0,0 +DISTRACTS,2009,0,0,0,0,0,0 +DISTRIBUTEE,0,0,0,2009,0,0,0 +DISTURBANCES,2009,0,0,0,0,0,0 +DIVERSION,2009,0,0,0,0,0,0 +DIVERTS,2009,0,0,0,0,0,0 +DIVESTITURE,2009,0,0,0,0,0,0 +DIVESTS,2009,0,0,0,0,0,0 +DIVULGED,2009,0,0,0,0,0,0 +DOCKETED,0,0,0,2009,0,0,0 +DOUBT,2009,0,2009,0,0,0,0 +DOWNGRADE,2009,0,0,0,0,0,0 +DOWNSIZE,2009,0,0,0,0,0,0 +DOWNSIZINGS,2009,0,0,0,0,0,0 +DOWNTURNS,2009,0,0,0,0,0,0 +DRASTIC,2009,0,0,0,0,0,0 +DREAM,0,2009,0,0,0,0,0 +DULY,0,0,0,2009,0,0,0 +DYSFUNCTIONS,2009,0,0,0,0,0,0 +EARMARKS,0,0,0,0,0,0,2009 +EASY,0,2009,0,0,0,0,0 +EFFICIENT,0,2009,0,0,0,0,0 +EJECTMENT,0,0,0,2011,0,0,0 +EMBARGOING,2009,0,0,0,0,0,0 +EMBARRASSING,2009,0,0,0,0,0,0 +EMBEZZLED,2009,0,0,0,0,0,0 +EMBEZZLES,2009,0,0,0,0,0,0 +EMPOWERING,0,2009,0,0,0,0,0 +ENABLES,0,2009,0,0,0,0,0 +ENCOURAGES,0,2009,0,0,0,0,0 +ENCROACHES,2009,0,0,0,0,0,0 +ENCUMBER,2009,0,0,2009,0,0,2009 +ENCUMBRANCE,2009,0,0,2009,0,0,2009 +ENDANGER,2009,0,0,0,0,0,0 +ENDANGERS,2009,0,0,0,0,0,0 +ENFORCEABLY,0,0,0,2011,0,0,0 +ENHANCEMENTS,0,2009,0,0,0,0,0 +ENJOINED,2009,0,0,0,0,0,0 +ENJOYABLE,0,2009,0,0,0,0,0 +ENJOYMENT,0,2009,0,0,0,0,0 +ENTAILING,0,0,0,0,0,0,2009 +PASSU,0,0,0,2009,0,0,0 +PENALIZED,2009,0,0,0,0,0,0 +PENALTY,2009,0,0,0,0,0,0 +PERFECTLY,0,2009,0,0,0,0,0 +PERILS,2009,0,0,0,0,0,0 +PERMISSIONS,0,0,0,0,0,0,2009 +PERMITTING,0,0,0,0,0,0,2009 +PERPETRATING,2009,0,0,2009,0,0,0 +PERSISTENCE,2009,0,0,0,0,0,0 +PERSISTS,2009,0,0,0,0,0,0 +PERVASIVENESS,2009,0,0,0,0,0,0 +PETITIONERS,0,0,0,2009,0,0,0 +PICKET,2009,0,0,0,0,0,0 +HEREBY,0,0,0,2009,0,0,0 +HEREFROM,0,0,0,2009,0,0,0 +HEREINBEFORE,0,0,0,2009,0,0,0 +HERETO,0,0,0,2009,0,0,0 +HEREUPON,0,0,0,2009,0,0,0 +HIGHEST,0,2009,0,0,2009,0,0 +HINDERS,2009,0,0,0,0,0,0 +WHATEVER,0,0,0,2009,0,0,0 +WHEREAS,0,0,0,2009,0,0,0 +WHEREIN,0,0,0,2009,0,0,0 +WHEREUNDER,0,0,0,2011,0,0,0 +WHOMEVER,0,0,0,2009,0,0,0 +WILL,0,0,0,0,2009,0,0 +WIN,0,2009,0,0,0,0,0 +WITNESS,0,0,0,2009,0,0,0 +FLUCTUATED,0,0,2009,0,0,0,0 +FLUCTUATIONS,0,0,2009,0,0,0,0 +FORBEARANCES,0,0,0,2009,0,0,0 +FORBIDDEN,2009,0,0,0,0,0,2009 +FORCED,2009,0,0,0,0,0,0 +FOREBEARS,0,0,0,2009,0,0,0 +FORECLOSING,2009,0,0,0,0,0,0 +FOREGOES,2009,0,0,0,0,0,0 +FORESTALLING,2009,0,0,0,0,0,0 +FORFEITABLE,0,0,0,2009,0,0,0 +FORFEITURE,2009,0,0,0,0,0,0 +FORTHWITH,0,0,0,2009,0,0,0 +FRAUDULENCE,2009,0,0,0,0,0,0 +FRIVOLOUS,2009,0,0,0,0,0,0 +FRUSTRATES,2009,0,0,0,0,0,0 +FRUSTRATIONS,2009,0,0,0,0,0,0 +GAIN,0,2009,0,0,0,0,0 +GOOD,0,2009,0,0,0,0,0 +DISGORGES,2009,0,0,0,0,0,0 +DISGRACEFULLY,2009,0,0,0,0,0,0 +LITIGATED,2009,0,0,2009,0,0,0 +LITIGATIONS,2009,0,0,2009,0,0,0 +LITIGIOUSNESS,0,0,0,2009,0,0,0 +LACKING,2009,0,0,0,0,0,0 +LAGGED,2009,0,0,0,0,0,0 +LAPSED,2009,0,0,0,0,0,0 +LAUNDERING,2009,0,0,0,0,0,0 +LAWFULNESS,0,0,0,2009,0,0,0 +LAWSUIT,0,0,0,2009,0,0,0 +LAYOFF,2009,0,0,0,0,0,0 +LEGAL,0,0,0,2009,0,0,0 +LEGALIZATIONS,0,0,0,2009,0,0,0 +LEGALIZING,0,0,0,2009,0,0,0 +LEGATEES,0,0,0,2009,0,0,0 +FLAWS,2009,0,0,0,0,0,0 +NONRENEWAL,2009,0,0,0,0,0,0 +LIQUIDATING,2009,0,0,0,0,0,0 +LIQUIDATORS,2009,0,0,0,0,0,0 +BENEFICIAL,0,-2020,0,2020,0,0,0 +BENEFIT,0,-2020,0,0,0,0,0 +BENEFITTING,0,2009,0,0,0,0,0 +POORLY,2009,0,0,0,0,0,0 +REINTERPRETATIONS,0,0,2009,0,0,0,0 +REJECT,2009,0,0,0,0,0,0 +REJECTIONS,2009,0,0,0,0,0,0 +RELINQUISHED,2009,0,0,0,0,0,0 +RELINQUISHMENTS,2009,0,0,0,0,0,0 +REMANDED,0,0,0,2009,0,0,0 +REMEDIATED,0,0,0,2009,0,0,0 +REMEDIED,0,0,0,2009,0,0,0 +RENEGOTIATES,2009,0,0,0,0,0,0 +RENOUNCE,2009,0,0,0,0,0,0 +RENOUNCES,2009,0,0,0,0,0,0 +REPLEDGED,0,0,0,2012,0,0,0 +REPOSSESSING,2009,0,0,0,0,0,0 +TROUBLES,2009,0,0,0,0,0,0 +UNACCEPTABLE,2009,0,0,0,0,0,0 +UNANNOUNCED,2009,0,0,0,0,0,0 +UNAPPROVED,2009,0,0,0,0,0,0 +UNAVAILABLE,2009,0,0,0,0,0,2011 +UNCERTAIN,0,0,2009,0,0,2009,0 +UNCLEAR,0,0,2009,0,0,0,0 +UNCOLLECTIBLE,2009,0,0,0,0,0,0 +UNCOMPROMISING,0,0,0,0,2009,0,0 +UNCONSTITUTIONAL,0,0,0,2009,0,0,0 +UNCONTROLLABLE,2009,0,0,0,0,0,0 +UNCOVER,2009,0,0,0,0,0,0 +UNDECIDED,0,0,2009,0,0,0,0 +UNDELIVERED,2009,0,0,0,0,0,0 +UNDERCUTTING,2009,0,0,0,0,0,0 +UNDERESTIMATING,2009,0,0,0,0,0,0 +UNDERMINE,2009,0,0,0,0,0,0 +UNDERPAID,2009,0,0,0,0,0,0 +UNDERPERFORM,2011,0,0,0,0,0,0 +UNDERPERFORMS,2014,0,0,0,0,0,0 +UNDERSTATE,2009,0,0,0,0,0,0 +UNDERSTATES,2009,0,0,0,0,0,0 +UNDESIGNATED,0,0,2009,0,0,0,0 +UNDETECTED,2009,0,0,0,0,0,0 +UNDISCLOSED,2009,0,0,0,0,0,0 +UNDUE,2009,0,0,0,0,0,0 +UNECONOMICALLY,2009,0,0,0,0,0,0 +UNENCUMBERED,0,0,0,2009,0,0,0 +UNEQUIVOCALLY,0,0,0,0,2009,0,0 +UNEXPECTED,2009,0,2009,0,0,0,0 +UNFAMILIAR,0,0,2009,0,0,0,0 +UNFAVORABLY,2009,0,0,0,0,0,0 +UNFITNESS,2009,0,0,0,0,0,0 +UNFORSEEN,2011,0,2011,0,0,0,0 +UNFRIENDLY,2009,0,0,0,0,0,0 +UNHEDGED,0,0,2009,0,0,0,0 +UNINTENDED,2009,0,0,0,0,0,0 +UNJUSTIFIABLE,2009,0,0,0,0,0,0 +UNKNOWING,2009,0,0,0,0,0,0 +UNLAWFUL,2009,0,0,2009,0,0,0 +UNLIQUIDATED,2009,0,0,0,0,0,0 +UNMERITORIOUS,2014,0,0,0,0,0,0 +UNOBSERVABLE,0,0,2009,0,0,0,0 +UNPARALLELED,0,2009,0,0,2009,0,0 +UNPREDICTABILITY,2009,0,2009,0,0,0,0 +UNPRODUCTIVE,2009,0,0,0,0,0,0 +UNPROVEN,0,0,2009,0,0,0,0 +UNREALISTIC,2009,0,0,0,0,0,0 +UNRECEPTIVE,2014,0,0,0,0,0,0 +UNREIMBURSED,2009,0,0,0,0,0,0 +UNREPORTED,2009,0,0,0,0,0,0 +UNSALABLE,2009,0,0,0,0,0,0 +UNSAVORY,2009,0,0,0,0,0,0 +UNSELLABLE,2014,0,0,0,0,0,0 +UNSPECIFIC,0,0,2009,0,0,0,0 +UNSTAYED,0,0,0,2009,0,0,0 +UNSUITABILITY,2009,0,0,0,0,0,0 +UNSURE,2009,0,0,0,0,0,0 +UNSUSTAINABLE,2009,0,0,0,0,0,0 +UNTO,0,0,0,2009,0,0,0 +UNTRUTHFULLY,2009,0,0,0,0,0,0 +UNUSUAL,0,0,2009,0,0,0,0 +UNWELCOME,2009,0,0,0,0,0,0 +UPSET,2009,0,0,0,0,0,0 +URGENT,2009,0,0,0,0,0,0 +USURPED,2009,0,0,2009,0,0,0 +VAGARIES,0,0,2009,0,0,0,0 +VAGUENESSES,0,0,2012,0,0,0,0 +VANDALISM,2009,0,0,0,0,0,0 +PLAINTIFF,2009,0,0,2009,0,0,0 +PLEADED,2009,0,0,0,0,0,0 +PLEAS,2009,0,0,2009,0,0,0 +PLEASURE,0,2009,0,0,0,0,0 +PLEDGEE,0,0,0,2009,0,0,0 +PLEDGOR,0,0,0,2009,0,0,0 +COMPELLED,0,0,0,0,0,0,2009 +COMPLAIN,2009,0,0,0,0,0,0 +COMPLAINING,2009,0,0,0,0,0,0 +COMMITMENTS,0,0,0,0,0,0,2009 +LIMITATIONS,2009,0,0,0,0,0,0 +RESTRAINS,0,0,0,0,0,0,2009 +RESTRICTED,0,0,0,0,0,0,2009 +RESTRICTIVE,0,0,0,0,0,0,2009 +RESTRUCTURE,2009,0,0,0,0,0,0 +RESTRUCTURINGS,2009,0,0,0,0,0,0 +RETALIATING,2009,0,0,0,0,0,0 +RETENDERING,0,0,0,2011,0,0,0 +PRECIPITATED,2009,0,0,0,0,0,0 +PRECLUDED,2009,0,0,0,0,0,2009 +PRECONDITIONS,0,0,0,0,0,0,2009 +PREDECEASES,0,0,0,2009,0,0,0 +PREDICTED,0,0,2009,0,0,0,0 +PREDICTIVE,0,0,2009,0,0,0,0 +PREEMINENCE,0,2009,0,0,0,0,0 +PREJUDICED,2009,0,0,2009,0,0,0 +PRELIMINARILY,0,0,2009,0,0,0,0 +PREMIER,0,2009,0,0,0,0,0 +PRESSING,2009,0,0,0,0,0,0 +PRESUME,0,0,2009,0,0,0,0 +PRESUMPTION,0,0,2009,0,0,0,0 +PREVENT,0,0,0,0,0,0,2009 +PREVENTS,2009,0,0,0,0,0,2009 +PROACTIVELY,0,2009,0,0,0,0,0 +PROBABLE,0,0,2009,0,0,0,0 +PROBATES,0,0,0,2009,0,0,0 +PROBATIONARY,0,0,0,2009,0,0,0 +PROBLEM,2009,0,0,0,0,0,0 +PROFICIENCY,0,2009,0,0,0,0,0 +PROFITABLE,0,2009,0,0,0,0,0 +PROGRESSES,0,2009,0,0,0,0,0 +PROHIBITING,0,0,0,0,0,0,2009 +PROHIBITIVELY,0,0,0,0,0,0,2009 +PROLONGATION,2009,0,0,0,0,0,0 +PROLONGS,2009,0,0,0,0,0,0 +PROMULGATING,0,0,0,2009,0,0,0 +PROMULGATORS,0,0,0,2009,0,0,0 +PROSECUTE,2009,0,0,2009,0,0,0 +PROSECUTION,2009,0,0,2009,0,0,0 +PROSECUTORS,0,0,0,2009,0,0,0 +PROSPEROUS,0,2009,0,0,0,0,0 +PROTESTER,2009,0,0,0,0,0,0 +PROTESTORS,2009,0,0,0,0,0,0 +PROVISO,0,0,0,2009,0,0,0 +PROVOKED,2009,0,0,0,0,0,0 +PUNISHED,2009,0,0,0,0,0,0 +PUNISHMENTS,2009,0,0,0,0,0,0 +PURPORTEDLY,2009,0,0,0,0,0,0 +QUESTIONABLE,2009,0,0,0,0,0,0 +QUESTIONS,2009,0,0,0,0,0,0 +QUITTING,2009,0,0,0,0,0,0 +RANDOMIZE,0,0,2009,0,0,0,0 +RANDOMLY,0,0,2009,0,0,0,0 +RATABLY,0,0,0,2009,0,0,0 +RATIONALIZED,2009,0,0,0,0,0,0 +ABANDONING,2009,0,0,0,0,0,0 +ABDICATED,2009,0,0,0,0,0,0 +ABDICATIONS,2009,0,0,0,0,0,0 +ABERRATIONS,2009,0,0,0,0,0,0 +ABIDE,0,0,0,0,0,0,2009 +ABNORMALITIES,2009,0,0,0,0,0,0 +ABOLISHED,2009,0,0,0,0,0,0 +ABROGATE,2009,0,0,2009,0,0,0 +ABROGATION,2009,0,0,2009,0,0,0 +ABRUPTNESS,2009,0,0,0,0,0,0 +ABSOLVE,0,0,0,2009,0,0,0 +ABUNDANCE,0,2009,0,0,0,0,0 +ABUSES,2009,0,0,0,0,0,0 +ABUSIVENESS,2009,0,0,0,0,0,0 +ACCIDENTAL,2009,0,0,0,0,0,0 +ACCOMPLISH,0,2009,0,0,0,0,0 +ACCOMPLISHMENT,0,2009,0,0,0,0,0 +ACCUSE,2009,0,0,0,0,0,0 +ACHIEVE,0,2009,0,0,0,0,0 +ACHIEVES,0,2009,0,0,0,0,0 +ACQUIESCES,2009,0,0,0,0,0,0 +ACQUIT,2009,0,0,2009,0,0,0 +ACQUITTANCE,0,0,0,2009,0,0,0 +ADDENDUMS,0,0,0,2011,0,0,0 +ADJOURNING,0,0,0,2009,0,0,0 +ADJUDGE,0,0,0,2009,0,0,0 +ADJUDICATE,0,0,0,2009,0,0,0 +ADJUDICATION,0,0,0,2009,0,0,0 +ADJUDICATORS,0,0,0,2009,0,0,0 +ADMISSIBLY,0,0,0,2009,0,0,0 +ADULTERATED,2009,0,0,0,0,0,0 +ADVANCEMENT,0,2009,0,0,0,0,0 +ADVANTAGE,0,2009,0,0,0,0,0 +ADVANTAGES,0,2009,0,0,0,0,0 +ADVERSE,2009,0,0,0,0,0,0 +AFFIDAVIT,0,0,0,2009,0,0,0 +AFOREDESCRIBED,0,0,0,2011,0,0,0 +AFTERMATH,2009,0,0,0,0,0,0 +AGGRAVATED,2009,0,0,0,0,0,0 +AGGRAVATIONS,2009,0,0,0,0,0,0 +ALIENATE,2009,0,0,0,0,0,0 +ALIENATION,2009,0,0,0,0,0,0 +ALLEGE,2009,0,0,2009,0,0,0 +ALLEGING,2009,0,0,2009,0,0,0 +ALTERATION,0,0,2009,0,0,0,0 +AMBIGUITY,0,0,2009,0,0,0,0 +AMENDATORY,0,0,0,2009,0,0,0 +AMENDMENTS,0,0,0,2009,0,0,0 +ANNOYANCES,2009,0,0,0,0,0,0 +ANNUL,2009,0,0,0,0,0,0 +ANNULMENTS,2009,0,0,0,0,0,0 +ANOMALOUSLY,2009,0,2009,0,0,0,0 +ANTICIPATE,0,0,2009,0,0,0,0 +ANTICIPATION,0,0,2009,0,0,0,0 +ANTITRUST,2009,0,0,2009,0,0,0 +APPEAL,0,0,0,2009,0,0,0 +APPEALS,0,0,0,2009,0,0,0 +APPEARS,0,0,2009,0,0,2009,0 +APPELLEES,0,0,0,2011,0,0,0 +APPROXIMATELY,0,0,2009,0,0,0,0 +APPROXIMATIONS,0,0,2009,0,0,0,0 +ARBITRABILITY,0,0,0,2011,0,0,0 +ARBITRARY,0,0,2009,0,0,0,0 +ARBITRATING,0,0,0,2009,0,0,0 +ARBITRATIVE,0,0,0,2011,0,0,0 +ARGUED,2009,0,0,0,0,0,0 +ARGUMENTS,2009,0,0,0,0,0,0 +ARREST,2009,0,0,0,0,0,0 +RETRIBUTIONS,2009,0,0,0,0,0,0 +BONA,0,0,0,2009,0,0,0 +BOOST,0,2009,0,0,0,0,0 +BOUND,0,0,0,0,0,0,2011 +BOYCOTTING,2009,0,0,0,0,0,0 +BREACHES,2009,0,0,2009,0,0,0 +BREAKAGES,2009,0,0,0,0,0,0 +BREAKS,2009,0,0,0,0,0,0 +BRIBED,2009,0,0,0,0,0,0 +BRIBING,2009,0,0,0,0,0,0 +BURDEN,2009,0,0,0,0,0,0 +BURDENSOME,2009,0,0,0,0,0,0 +CALAMITY,2009,0,0,0,0,0,0 +CANCELLATION,2009,0,0,0,0,0,0 +CANCELS,2009,0,0,0,0,0,0 +CATASTROPHICALLY,2009,0,0,0,0,0,0 +CAUTIONING,2009,0,0,0,0,0,0 +CAUTIOUSNESS,0,0,2009,0,0,0,0 +CEASING,2009,0,0,0,0,0,0 +CENSURED,2009,0,0,0,0,0,0 +CESSION,0,0,0,2009,0,0,0 +CHALLENGING,2009,0,0,0,0,0,0 +CHATTELS,0,0,0,2009,0,0,0 +CIRCUMVENTING,2009,0,0,0,0,0,0 +SHARPLY,2009,0,0,0,0,0,0 +SHORTFALL,2009,0,0,0,0,0,0 +SHUT,2009,0,0,0,0,0,0 +SHUTTING,2009,0,0,0,0,0,0 +SLANDERS,2009,0,0,0,0,0,0 +REPUDIATE,2009,0,0,0,0,0,0 +REPUDIATION,2009,0,0,0,0,0,0 +REQUIRE,0,0,0,0,0,0,2009 +REQUIRES,0,0,0,0,0,0,2009 +RESCINDED,0,0,0,2009,0,0,0 +RESCISSIONS,0,0,0,2009,0,0,0 +RESIGNED,2009,0,0,0,0,0,0 +RESTATE,2009,0,0,0,0,0,0 +RESTATES,2009,0,0,0,0,0,0 +NONTERMINABLE,0,0,0,2011,0,0,0 +NOTARIZATION,0,0,0,2009,0,0,0 +NOTARIZING,0,0,0,2009,0,0,0 +NUISANCE,2009,0,0,0,0,0,0 +NULLIFIED,2009,0,0,2009,0,0,0 +NULLITIES,0,0,0,2009,0,0,0 +OBJECTION,2009,0,0,0,0,0,0 +OBLIGATE,0,0,0,0,0,0,2009 +OBLIGATION,0,0,0,0,0,0,2009 +OBLIGED,0,0,0,0,0,0,2009 +OBLIGOR,0,0,0,2009,0,0,0 +OBSOLESCENCE,2009,0,0,0,0,0,0 +OBSTRUCT,2009,0,0,0,0,0,0 +OBSTRUCTIONS,2009,0,0,0,0,0,0 +OFFEND,2009,0,0,0,0,0,0 +OFFENDING,2009,0,0,0,0,0,0 +OFFEREES,0,0,0,2011,0,0,0 +OMISSIONS,2009,0,0,0,0,0,0 +OMITTING,2009,0,0,0,0,0,0 +OPPORTUNITIES,0,2009,0,0,0,0,0 +OPPOSES,2009,0,0,0,0,0,0 +OPTIMISTIC,0,2009,0,0,0,0,0 +OUTAGE,2009,0,0,0,0,0,0 +OUTPERFORM,0,2009,0,0,0,0,0 +OVERAGE,2009,0,0,0,0,0,0 +OVERBUILDS,2009,0,0,0,0,0,0 +OVERBURDENING,2009,0,0,0,0,0,0 +OVERCHARGED,2009,0,0,0,0,0,0 +OVERCOMES,2009,0,0,0,0,0,0 +OVERESTIMATED,2009,0,0,0,0,0,0 +OVERESTIMATIONS,2009,0,0,0,0,0,0 +OVERLOADS,2009,0,0,0,0,0,0 +OVERLOOKS,2009,0,0,0,0,0,0 +OVERPRODUCED,2009,0,0,0,0,0,0 +OVERRULE,0,0,0,2009,0,0,0 +OVERRUN,2009,0,0,0,0,0,0 +OVERSHADOWED,2009,0,0,0,0,0,0 +OVERSTATED,2009,0,0,0,0,0,0 +OVERSTATING,2009,0,0,0,0,0,0 +OVERSUPPLYING,2009,0,0,0,0,0,0 +OVERTURNING,2009,0,0,0,0,0,0 +OVERVALUING,2009,0,0,0,0,0,0 +PARI,0,0,0,2009,0,0,0 +NECESSITATES,0,0,0,0,0,0,2009 +NEGATIVES,2009,0,0,0,0,0,0 +NEGLECTING,2009,0,0,0,0,0,0 +NEGLIGENT,2009,0,0,0,0,0,0 +BARRED,2009,0,0,0,0,0,0 +BEAUTIFULLY,0,2009,0,0,0,0,0 +BELIEVING,0,0,2009,0,0,0,0 +IDLE,2009,0,0,0,0,0,0 +IGNORED,2009,0,0,0,0,0,0 +ILLEGAL,2009,0,0,0,0,0,0 +ILLEGIBLE,2009,0,0,0,0,0,0 +ILLIQUIDITY,2009,0,0,0,0,0,0 +IMMATURE,2009,0,0,0,0,0,0 +IMPAIRING,2009,0,0,0,0,0,2011 +IMPASSE,2009,0,0,0,0,0,0 +IMPEDES,2009,0,0,0,0,0,0 +IMPENDING,2009,0,0,0,0,0,0 +IMPERIL,2009,0,0,0,0,0,0 +IMPLICATED,2009,0,0,0,0,0,0 +IMPOSED,0,0,0,0,0,0,2009 +IMPOSITIONS,0,0,0,0,0,0,2009 +IMPOUNDED,2009,0,0,0,0,0,0 +IMPRACTICAL,2009,0,0,0,0,0,0 +IMPRECISION,0,0,2009,0,0,0,0 +IMPRESSES,0,2009,0,0,0,0,0 +IMPRISONMENT,2009,0,0,0,0,0,0 +IMPROPERLY,2009,0,0,0,0,0,0 +IMPROVED,0,2009,0,0,0,0,0 +IMPROVING,0,2009,0,0,0,0,0 +INACCESSIBLE,2009,0,0,0,0,0,0 +INACCURATELY,2009,0,0,0,0,0,0 +INACTIVATED,2009,0,0,0,0,0,0 +INACTIVATIONS,2009,0,0,0,0,0,0 +INADEQUATE,2009,0,0,0,0,0,0 +INADVISABILITY,2009,0,0,0,0,0,0 +INASMUCH,0,0,0,2009,0,0,0 +INCAPACITY,2009,0,0,2009,0,0,0 +INCARCERATING,2009,0,0,2009,0,0,0 +INCIDENCE,2009,0,0,0,0,0,0 +INCOMPATIBILITIES,2009,0,0,0,0,0,0 +INCOMPETENCY,2009,0,0,0,0,0,0 +INCOMPLETE,2009,0,0,0,0,0,0 +INCONSISTENCIES,2009,0,0,0,0,0,0 +INCONTESTABILITY,0,0,0,2009,0,0,0 +INCONVENIENT,2009,0,0,0,0,0,0 +INCREDIBLE,0,2009,0,0,0,0,0 +INDECENT,2009,0,0,0,0,0,0 +INDEFINITELY,0,0,2009,0,0,0,0 +INDEMNIFICATIONS,0,0,0,2009,0,0,0 +INDEMNIFYING,0,0,0,2009,0,0,0 +INDEMNITOR,0,0,0,2009,0,0,0 +INDETERMINATE,0,0,2009,0,0,0,0 +INDICTING,2009,0,0,2009,0,0,0 +INEFFECTIVE,2009,0,0,0,0,0,0 +INEFFICIENCY,2009,0,0,0,0,0,0 +INELIGIBLE,2009,0,0,0,0,0,0 +INEQUITY,2009,0,0,0,0,0,0 +INEXPERIENCE,2009,0,0,0,0,0,0 +INFLUENTIAL,0,2009,0,0,0,0,0 +INFRACTIONS,2009,0,0,2009,0,0,0 +INFRINGEMENTS,2009,0,0,0,0,0,0 +INGENUITY,0,2009,0,0,0,0,0 +INHIBITS,0,0,0,0,0,0,2011 +INJUNCTIVE,0,0,0,2009,0,0,0 +INJURIES,2009,0,0,0,0,0,0 +INNOVATE,0,2009,0,0,0,0,0 +INNOVATION,0,2009,0,0,0,0,0 +INNOVATOR,0,2009,0,0,0,0,0 +INQUIRY,2009,0,0,0,0,0,0 +INSIST,0,0,0,0,0,0,2009 +INSISTS,0,0,0,0,0,0,2009 +INSOLVENT,2009,0,0,0,0,0,0 +INSTABILITY,2009,0,2009,0,0,0,0 +INSUFFICIENTLY,2009,0,0,0,0,0,0 +INTANGIBLES,0,0,2009,0,0,0,0 +INTERFERED,2009,0,0,0,0,0,0 +INTERFERING,2009,0,0,0,0,0,0 +INTERPLEADER,0,0,0,2009,0,0,0 +INTERPOSING,0,0,0,2009,0,0,0 +INTERROGATED,0,0,0,2009,0,0,0 +INTERROGATIONS,0,0,0,2009,0,0,0 +INTERROGATORY,0,0,0,2009,0,0,0 +INTERRUPTION,2009,0,0,0,0,0,0 +INTESTATE,0,0,0,2009,0,0,0 +SPORADIC,0,0,2009,0,0,0,0 +STABILIZATIONS,0,2009,0,0,0,0,0 +STABILIZING,0,2009,0,0,0,0,0 +STAGNATE,2009,0,0,0,0,0,0 +STAGNATION,2009,0,0,0,0,0,0 +STATUTES,0,0,0,2009,0,0,0 +STIPULATED,0,0,0,0,0,0,2009 +STIPULATIONS,0,0,0,0,0,0,2009 +STOPPED,2009,0,0,0,0,0,0 +STRAINED,2009,0,0,0,0,0,0 +STRENGTHEN,0,2009,0,0,0,0,0 +STRENGTHS,0,2009,0,0,0,0,0 +STRESSFUL,2009,0,0,0,0,0,0 +STRICTEST,0,0,0,0,0,0,2009 +STRONGER,0,2009,0,0,0,0,0 +SUBCLAUSES,0,0,0,2009,0,0,0 +SUBJECTION,2009,0,0,0,0,0,0 +SUBLICENSEE,0,0,0,2009,0,0,0 +SUBPOENA,2009,0,0,2009,0,0,0 +SUBROGATION,0,0,0,2009,0,0,0 +SUCCEED,0,2009,0,0,0,0,0 +SUCCESS,0,2009,0,0,0,0,0 +SUDDEN,0,0,2009,0,0,0,0 +SUES,2009,0,0,2009,0,0,0 +SUFFERS,2009,0,0,0,0,0,0 +SUGGESTS,0,0,2009,0,0,2009,0 +SUMMONS,2009,0,0,2009,0,0,0 +SUPERSEDEAS,0,0,0,2011,0,0,0 +SURETIES,0,0,0,2009,0,0,0 +SURPASSES,0,2009,0,0,0,0,0 +SUSPECT,2009,0,0,0,0,0,0 +SUSPENDED,2009,0,0,0,0,0,0 +SUSPENSIONS,2009,0,0,0,0,0,0 +SUSPICIOUSLY,2009,0,0,0,0,0,0 +REVISE,0,0,2009,0,0,0,0 +REVOCATIONS,2009,0,0,2009,0,0,0 +REVOKING,2009,0,0,0,0,0,0 +REVOLUTIONIZING,0,2009,0,0,0,0,0 +REWARDS,0,-2020,0,0,0,0,0 +RIDICULING,2009,0,0,0,0,0,0 +RISKIEST,2009,0,2009,0,0,0,0 +RISKY,2009,0,2009,0,0,0,0 +RUMORS,0,0,2009,0,0,0,0 +SACRIFICES,2009,0,0,0,0,0,0 +SATISFACTORILY,0,2009,0,0,0,0,0 +SATISFY,0,2009,0,0,0,0,0 +SCRUTINIZE,2009,0,0,0,0,0,0 +SCRUTINY,2009,0,0,0,0,0,0 +SEIZED,2009,0,0,0,0,0,0 +SELDOMLY,0,0,2009,0,0,2009,0 +SERIOUS,2009,0,0,0,0,0,0 +SETBACKS,2009,0,0,0,0,0,0 +SEVERABILITY,0,0,0,2009,0,0,0 +SEVERANCES,0,0,0,2009,0,0,0 +SEVERITIES,2009,0,0,0,0,0,0 +ENTHUSIASM,0,2009,0,0,0,0,0 +ENTRENCHED,0,0,0,0,0,0,2009 +ERODING,2009,0,0,0,0,0,0 +ERRED,2009,0,0,0,0,0,0 +ERROR,2009,0,0,0,0,0,0 +ESCALATED,2009,0,0,0,0,0,0 +ESCHEATED,0,0,0,2011,0,0,0 +ESCROWING,0,0,0,2011,0,0,0 +EVADED,2009,0,0,0,0,0,0 +EVASIONS,2009,0,0,0,0,0,0 +EVICTING,2009,0,0,0,0,0,0 +EVIDENTIAL,0,0,0,2011,0,0,0 +EXACERBATES,2009,0,0,0,0,0,0 +EXAGGERATE,2009,0,0,0,0,0,0 +EXAGGERATION,2009,0,0,0,0,0,0 +EXCELLENCE,0,2009,0,0,0,0,0 +EXCEPTIONAL,0,2009,0,0,0,0,0 +EXCISED,0,0,0,2009,0,0,0 +EXCLUSIVE,0,2009,0,0,0,0,0 +EXCLUSIVITY,0,2009,0,0,0,0,0 +EXCULPATING,2009,0,0,2009,0,0,0 +EXECUTOR,0,0,0,2009,0,0,0 +EXECUTRIX,0,0,0,2009,0,0,0 +EXONERATED,2009,0,0,0,0,0,0 +EXONERATIONS,2009,0,0,0,0,0,0 +EXPLOITATIVE,2009,0,0,0,0,0,0 +EXPOSE,2009,0,0,0,0,0,0 +EXPOSURE,0,0,2009,0,0,0,0 +EXPROPRIATES,2009,0,0,0,0,0,0 +EXPULSION,2009,0,0,0,0,0,0 +EXTRACORPOREAL,0,0,0,2011,0,0,0 +SOLVENCY,2009,0,0,0,0,0,0 +SOMETIMES,0,0,2009,0,0,2009,0 +SPAMMERS,2014,0,0,0,0,0,0 +TREMENDOUS,0,2009,0,0,0,0,0 +LOCKOUT,2009,0,0,0,0,0,0 +LOSING,2009,0,0,0,0,0,0 +LOWEST,0,0,0,0,2009,0,0 +MAJEURE,0,0,0,2011,0,0,0 +MALFUNCTIONING,2009,0,0,0,0,0,0 +MALICIOUSLY,2009,0,0,0,0,0,0 +MANDATED,0,0,0,0,0,0,2009 +MANDITORILY,0,0,0,0,0,0,2011 +MANIPULATING,2009,0,0,0,0,0,0 +MARKDOWN,2009,0,0,0,0,0,0 +MEDIATE,0,0,0,2009,0,0,0 +MEDIATION,0,0,0,2009,0,0,0 +MERITORIOUS,0,2009,0,0,0,0,0 +MISAPPLIED,2009,0,0,0,0,0,0 +MISAPPROPRIATE,2009,0,0,0,0,0,0 +MISAPPROPRIATION,2009,0,0,0,0,0,0 +MISCALCULATED,2009,0,0,0,0,0,0 +MISCALCULATIONS,2009,0,0,0,0,0,0 +MISCLASSIFICATIONS,2014,0,0,0,0,0,0 +MISCONDUCT,2009,0,0,0,0,0,0 +MISDIRECTED,2009,0,0,0,0,0,0 +MISHANDLES,2009,0,0,0,0,0,0 +MISINFORMED,2009,0,0,0,0,0,0 +MISINTERPRETATION,2009,0,0,0,0,0,0 +MISINTERPRETS,2009,0,0,0,0,0,0 +MISJUDGING,2009,0,0,0,0,0,0 +MISLABELED,2009,0,0,0,0,0,0 +MISLEAD,2009,0,0,0,0,0,0 +MISLED,2009,0,0,0,0,0,0 +MISMANAGES,2009,0,0,0,0,0,0 +MISMATCHES,2009,0,0,0,0,0,0 +MISPRICING,2014,0,0,0,0,0,0 +MISREPRESENTATIONS,2009,0,0,0,0,0,0 +MISS,2009,0,0,0,0,0,0 +WORSE,2009,0,0,0,0,0,0 +WORSENS,2009,0,0,0,0,0,0 +TOLERATES,2009,0,0,0,0,0,0 +TORTIOUS,0,0,0,2009,0,0,0 +TORTUOUSLY,2009,0,0,0,0,0,0 +FINED,2009,0,0,0,0,0,0 +NONAPPEALABLE,0,0,0,2009,0,0,0 +NONCANCELABLE,0,0,0,0,0,0,2009 +NONCOMPLIANCES,2009,0,0,0,0,0,0 +NONCONFORMITIES,2009,0,0,0,0,0,0 +NONCONTRACTUAL,0,0,0,2011,0,0,0 +NONFIDUCIARY,0,0,0,2011,0,0,0 +NONFUNCTIONAL,2009,0,0,0,0,0,0 +DISCLAIMER,2009,0,0,0,0,0,0 +DISCLOSE,2009,0,0,0,0,0,0 +DISCONTINUANCE,2009,0,0,0,0,0,0 +DISCONTINUE,2009,0,0,0,0,0,0 +DISCOURAGE,2009,0,0,0,0,0,0 +DISCREDIT,2009,0,0,0,0,0,0 +DISCREPANCIES,2009,0,0,0,0,0,0 +SPECULATE,0,0,2009,0,0,0,0 +SPECULATION,0,0,2009,0,0,0,0 +TRAGIC,2009,0,0,0,0,0,0 +TRANSPARENCY,0,2009,0,0,0,0,0 +LEGISLATE,0,0,0,2009,0,0,0 +LEGISLATION,0,0,0,2009,0,0,0 +LEGISLATOR,0,0,0,2009,0,0,0 +LIBEL,0,0,0,2009,0,0,0 +LICENSABLE,0,0,0,2011,0,0,0 +CONCEALING,2009,0,0,0,0,0,0 +CONCEDING,2009,0,0,0,0,0,0 +CONCERNED,2009,0,0,0,0,0,0 +CONCILIATIONS,2009,0,0,0,0,0,0 +CONDEMNATION,2009,0,0,0,0,0,0 +CONDEMNOR,0,0,0,2011,0,0,0 +CONDONE,2009,0,0,0,0,0,0 +CONFESSED,2009,0,0,0,0,0,0 +CONFIDENT,0,2009,0,0,0,0,0 +CONFINEMENTS,2009,0,0,0,0,0,0 +CONFISCATED,2009,0,0,0,0,0,0 +CONFISCATIONS,2009,0,0,0,0,0,0 +CONFLICTING,2009,0,0,0,0,0,0 +CONFRONTATIONAL,2009,0,0,0,0,0,0 +CONFRONTS,2009,0,0,0,0,0,0 +CONFUSING,2009,0,2009,0,0,0,0 +CONSENTED,0,0,0,2009,0,0,0 +CONSPIRACIES,2009,0,0,0,0,0,0 +CONSPIRATORS,2009,0,0,0,0,0,0 +CONSPIRING,2009,0,0,0,0,0,0 +CONSTITUTIONALLY,0,0,0,2009,0,0,0 +CONSTRAINED,0,0,0,0,0,0,2009 +CONSTRAINTS,0,0,0,0,0,0,2009 +CONSTRUED,0,0,0,2012,0,0,0 +CONTEND,2009,0,0,0,0,0,0 +CONTENTION,2009,0,0,0,0,0,0 +CONTESTABILITY,0,0,0,2014,0,0,0 +CONTINGENCIES,0,0,2009,0,0,0,0 +CONTINGENTS,0,0,2009,0,0,0,0 +CONTRACTHOLDERS,0,0,0,2009,0,0,0 +CONTRACTION,2009,0,0,0,0,0,0 +CONTRACTUALLY,0,0,0,2009,0,0,0 +CONTRADICTION,2009,0,0,0,0,0,0 +CONTRARY,2009,0,0,0,0,0,0 +CONTRAVENING,0,0,0,2009,0,0,0 +CONTROVERSIES,2009,0,0,0,0,0,0 +CONTROVERTING,0,0,0,2009,0,0,0 +CONVICT,2009,0,0,2009,0,0,0 +CONVICTIONS,2009,0,0,2009,0,0,0 +CORRECTIONS,2009,0,0,0,0,0,0 +CORRUPTING,2009,0,0,0,0,0,0 +CORRUPTNESS,2009,0,0,0,0,0,0 +COUNSEL,0,0,0,2009,0,0,0 +COUNTERCLAIM,2009,0,0,0,0,0,0 +COUNTERFEIT,2009,0,0,0,0,0,0 +COUNTERFEITING,2009,0,0,0,0,0,0 +COUNTERSIGNOR,0,0,0,2011,0,0,0 +COURT,0,0,0,2009,0,0,0 +COVENANT,0,0,0,0,0,0,2011 +CREATIVE,0,2009,0,0,0,0,0 +CRIME,2009,0,0,2009,0,0,0 +CRIMINALIZE,0,0,0,2014,0,0,0 +CRISES,2009,0,0,0,0,0,0 +CRITICISM,2009,0,0,0,0,0,0 +CRITICIZES,2009,0,0,0,0,0,0 +CROSSROAD,0,0,2009,0,0,0,0 +CULPABILITY,2009,0,0,0,0,0,0 +CURTAIL,2009,0,0,0,0,0,0 +CURTAILMENTS,2009,0,0,0,0,0,0 +CUTBACKS,2009,0,0,0,0,0,0 +CYBERCRIME,2014,0,0,0,0,0,0 +TAINTING,2009,0,0,0,0,0,0 +TENDING,0,0,2009,0,0,0,0 +TERMINABLE,0,0,0,2009,0,0,0 +TERMINATING,2009,0,0,0,0,0,0 +TESTAMENTARY,0,0,0,2009,0,0,0 +THENCE,0,0,0,2009,0,0,0 +THEREAT,0,0,0,2009,0,0,0 +THEREOF,0,0,0,2009,0,0,0 +THERETOFOR,0,0,0,2011,0,0,0 +THEREUPON,0,0,0,2009,0,0,0 +THREATENED,2009,0,0,0,0,0,0 +FAILED,2009,0,0,0,0,0,0 +FAILURE,2009,0,0,0,0,0,0 +FALSELY,2009,0,0,0,0,0,0 +FALSIFIES,2009,0,0,0,0,0,0 +FANTASTIC,0,2009,0,0,0,0,0 +FAULT,2009,0,0,0,0,0,0 +FAVORABLE,0,2009,0,0,0,0,0 +FAVORITE,0,2009,0,0,0,0,0 +FELONIES,2009,0,0,2009,0,0,0 +DAMAGED,2009,0,0,0,0,0,0 +DAMPENED,2009,0,0,0,0,0,0 +DANGERS,2009,0,0,0,0,0,0 +DEADLOCKS,2009,0,0,0,0,0,0 +DEBARMENTS,2009,0,0,0,0,0,0 +DECEDENTS,0,0,0,2009,0,0,0 +DECEIVE,2009,0,0,0,0,0,0 +DECEPTION,2009,0,0,0,0,0,0 +DECLARANT,0,0,0,2011,0,0,0 +DECLINING,2009,0,0,0,0,0,0 +DECREES,0,0,0,2009,0,0,0 +DEFALCATION,0,0,0,2009,0,0,0 +DEFAMATORY,2009,0,0,0,0,0,0 +DEFAMING,2009,0,0,0,0,0,0 +DEFAULTS,2009,0,0,0,0,0,0 +DEFEASED,0,0,0,2009,0,0,0 +DEFEAT,2009,0,0,0,0,0,0 +DEFECT,2009,0,0,0,0,0,0 +DEFEND,2009,0,0,0,0,0,0 +DEFENDED,2009,0,0,0,0,0,0 +DEFER,2009,0,0,0,0,0,0 +DEFICIENT,2009,0,0,0,0,0,0 +DEFINITIVELY,0,0,0,0,2009,0,0 +DEFRAUDS,2009,0,0,0,0,0,0 +DEGRADE,2009,0,0,0,0,0,0 +DELAY,2009,0,0,0,0,0,0 +DELEGABLE,0,0,0,2009,0,0,0 +DELETERIOUS,2009,0,0,0,0,0,0 +DELIGHT,0,2009,0,0,0,0,0 +DELIGHTING,0,2009,0,0,0,0,0 +DELINQUENT,2009,0,0,0,0,0,0 +DELISTED,2009,0,0,0,0,0,0 +DEMISED,2009,0,0,0,0,0,0 +DEMOLISHED,2009,0,0,0,0,0,0 +DEMOLITIONS,2009,0,0,0,0,0,0 +DEMOTING,2009,0,0,0,0,0,0 +DEMURRER,0,0,0,2009,0,0,0 +DENIAL,2009,0,0,0,0,0,0 +DENIGRATE,2009,0,0,0,0,0,0 +DENIGRATION,2009,0,0,0,0,0,0 +DEPENDABILITY,0,2009,0,0,0,0,0 +DEPENDANT,0,0,0,0,0,0,2011 +DEPENDENCY,0,0,2009,0,0,0,0 +DEPLETE,2009,0,0,0,0,0,0 +DEPLETION,2009,0,0,0,0,0,0 +DEPOSES,0,0,0,2009,0,0,0 +DEPOSITIONS,0,0,0,2009,0,0,0 +INTRUSION,2009,0,0,0,0,0,0 +INVALIDATES,2009,0,0,0,0,0,0 +INVENT,0,2009,0,0,0,0,0 +INVENTIONS,0,2009,0,0,0,0,0 +INVENTORS,0,2009,0,0,0,0,0 +INVESTIGATING,2009,0,0,0,0,0,0 +INVOLUNTARY,2009,0,0,0,0,0,0 +IRRECOVERABLY,2009,0,0,0,0,0,0 +IRREGULARLY,2009,0,0,0,0,0,0 +IRREVOCABILITY,0,0,0,2011,0,0,0 +JEOPARDIZED,2009,0,0,0,0,0,0 +JUDICIARIES,0,0,0,2009,0,0,0 +JURISDICTION,0,0,0,2009,0,0,0 +JURISPRUDENCE,0,0,0,2009,0,0,0 +JURORS,0,0,0,2009,0,0,0 +JUSTICES,0,0,0,2009,0,0,0 +KNOWINGLY,2009,0,0,0,0,0,0 +DILIGENT,0,2009,0,0,0,0,0 +DIMINISHES,2009,0,0,0,0,0,0 +DIRECTIVES,0,0,0,0,0,0,2009 +DISADVANTAGES,2009,0,0,0,0,0,0 +DISAFFIRMED,0,0,0,2011,0,0,0 +DISAGREED,2009,0,0,0,0,0,0 +DISAGREES,2009,0,0,0,0,0,0 +DISALLOWED,2009,0,0,0,0,0,0 +DISAPPEARANCE,2009,0,0,0,0,0,0 +DISAPPEARS,2009,0,0,0,0,0,0 +DISAPPOINTINGLY,2009,0,0,0,0,0,0 +DISAPPROVAL,2009,0,0,0,0,0,0 +DISAPPROVES,2009,0,0,0,0,0,0 +DISASSOCIATION,2009,0,0,0,0,0,0 +DISASTROUS,2009,0,0,0,0,0,0 +DISAVOWED,2009,0,0,0,0,0,0 +GREAT,0,-2020,0,0,0,0,0 +GREATNESS,0,2009,0,0,0,0,0 +GROUNDLESS,2009,0,0,0,0,0,0 +HAMPER,2009,0,0,0,0,0,0 +HAPPIEST,0,2009,0,0,0,0,0 +HARASS,2009,0,0,0,0,0,0 +HARDSHIP,2009,0,0,0,0,0,0 +HARMFUL,2009,0,0,0,0,0,0 +HARSH,2009,0,0,0,0,0,0 +HARSHNESS,2009,0,0,0,0,0,0 +NONJUDICIAL,0,0,0,2009,0,0,0 +NONPAYMENTS,2009,0,0,0,0,0,0 \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/java-solution/Task.java b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/java-solution/Task.java new file mode 100644 index 0000000000000..2c94779c33e8d --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/java-solution/Task.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// beam-playground: +// name: FinalSolution2 +// description: Final challenge solution 2. +// multifile: true +// files: +// - name: analysis.csv +// context_line: 98 +// categories: +// - Quickstart +// complexity: ADVANCED +// tags: +// - hellobeam + +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.coders.RowCoder; +import org.apache.beam.sdk.io.TextIO; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.transforms.*; +import org.apache.beam.sdk.values.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.List; + +public class Task { + private static final Logger LOG = LoggerFactory.getLogger(Task.class); + private static final Schema schema = Schema.builder() + .addField("word", Schema.FieldType.STRING) + .addField("negative", Schema.FieldType.STRING) + .addField("positive", Schema.FieldType.STRING) + .addField("uncertainty", Schema.FieldType.STRING) + .addField("litigious", Schema.FieldType.STRING) + .addField("strong", Schema.FieldType.STRING) + .addField("weak", Schema.FieldType.STRING) + .addField("constraining", Schema.FieldType.STRING) + .build(); + + public static void main(String[] args) { + PipelineOptions options = PipelineOptionsFactory.fromArgs(args).create(); + Pipeline pipeline = Pipeline.create(options); + + PCollection shakespeare = getPCollection(pipeline); + PCollection analysisPCollection = getAnalysisPCollection(pipeline); + + PCollectionView> viewAnalysisPCollection = analysisPCollection + .setCoder(RowCoder.of(schema)) + .apply(View.asList()); + + PCollection result = getAnalysis(shakespeare, viewAnalysisPCollection); + + PCollectionList pCollectionList = getPartitions(result); + + PCollection positivePCollection = pCollectionList.get(0).setCoder(RowCoder.of(schema)); + PCollection negativePCollection = pCollectionList.get(1).setCoder(RowCoder.of(schema)); + + positivePCollection + .apply(Combine.globally(Count.combineFn()).withoutDefaults()) + .apply(ParDo.of(new LogOutput<>("Positive word count"))); + + positivePCollection + .apply(Filter.by(it -> !it.getString("strong").equals("0") || !it.getString("weak").equals("0"))) + .apply(Combine.globally(Count.combineFn()).withoutDefaults()) + .apply(ParDo.of(new LogOutput<>("Positive words with enhanced effect count"))); + + negativePCollection + .apply(Combine.globally(Count.combineFn()).withoutDefaults()) + .apply(ParDo.of(new LogOutput<>("Negative word count"))); + + negativePCollection + .apply(Filter.by(it -> !it.getString("strong").equals("0") || !it.getString("weak").equals("0"))) + .apply(Combine.globally(Count.combineFn()).withoutDefaults()) + .apply(ParDo.of(new LogOutput<>("Negative words with enhanced effect count"))); + + pipeline.run().waitUntilFinish(); + } + + static PCollectionList getPartitions(PCollection input) { + return input + .apply(Partition.of(3, + (Partition.PartitionFn) (analysis, numPartitions) -> { + if (!analysis.getString("positive").equals("0")) { + return 0; + } + if (!analysis.getString("negative").equals("0")) { + return 1; + } + return 2; + })); + } + + static PCollection getAnalysis(PCollection pCollection, PCollectionView> viewAnalysisPCollection) { + return pCollection.apply(ParDo.of(new DoFn() { + @ProcessElement + public void processElement(@Element String word, OutputReceiver out, ProcessContext context) { + List analysisPCollection = context.sideInput(viewAnalysisPCollection); + analysisPCollection.forEach(it -> { + if (it.getString("word").equals(word)) { + out.output(it); + } + }); + } + }).withSideInputs(viewAnalysisPCollection)).setCoder(RowCoder.of(schema)); + } + + public static PCollection getPCollection(Pipeline pipeline) { + PCollection rides = pipeline.apply(TextIO.read().from("gs://apache-beam-samples/shakespeare/kinglear.txt")); + return rides.apply(FlatMapElements.into(TypeDescriptors.strings()).via(line -> Arrays.asList(line.toLowerCase().split(" ")))) + .apply(Filter.by((String word) -> !word.isEmpty())); + } + + public static PCollection getAnalysisPCollection(Pipeline pipeline) { + PCollection words = pipeline.apply(TextIO.read().from("analysis.csv")); + return words.apply(ParDo.of(new SentimentAnalysisExtractFn())); + } + + static class SentimentAnalysisExtractFn extends DoFn { + @ProcessElement + public void processElement(ProcessContext c) { + String[] items = c.element().split(","); + if (!items[1].equals("Negative")) { + c.output(Row.withSchema(schema) + .addValues(items[0].toLowerCase(), items[1], items[2], items[3], items[4], items[5], items[6], items[7]) + .build()); + } + } + } + + static class LogOutput extends DoFn { + + private final String prefix; + + LogOutput() { + this.prefix = "Processing element"; + } + + LogOutput(String prefix) { + this.prefix = prefix; + } + + @ProcessElement + public void processElement(ProcessContext c) { + LOG.info(prefix + ": " + c.element()); + c.output(c.element()); + } + } +} \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/java-solution/analysis.csv b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/java-solution/analysis.csv new file mode 100644 index 0000000000000..5c4a1246021eb --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/java-solution/analysis.csv @@ -0,0 +1,3877 @@ +Word,Negative,Positive,Uncertainty,Litigious,Strong_Modal,Weak_Modal,Constraining +NONSEVERABLE,0,0,0,2011,0,0,0 +DISFAVOR,2009,0,0,0,0,0,0 +DISGORGE,2009,0,0,0,0,0,0 +COMPLICATES,2009,0,0,0,0,0,0 +COMPLIMENT,0,2009,0,0,0,0,0 +COMPLIMENTS,0,2009,0,0,0,0,0 +MISSTATE,2009,0,0,0,0,0,0 +MISSTATES,2009,0,0,0,0,0,0 +MISTAKE,2009,0,0,0,0,0,0 +MISTAKING,2009,0,0,0,0,0,0 +MISUNDERSTANDING,2009,0,0,0,0,0,0 +MISUSED,2009,0,0,0,0,0,0 +MONOPOLISTS,2009,0,0,0,0,0,0 +MONOPOLIZES,2009,0,0,0,0,0,0 +MORATORIUM,2009,0,0,0,0,0,0 +MOTHBALLING,2009,0,0,0,0,0,0 +SLOW,2009,0,0,0,0,0,0 +SLOWER,2009,0,0,0,0,0,0 +SLOWNESS,2009,0,0,0,0,0,0 +SMOOTH,0,2009,0,0,0,0,0 +DEPRECATION,2009,0,0,0,0,0,0 +DEPRESSING,2009,0,0,0,0,0,0 +DEPRIVES,2009,0,0,0,0,0,0 +DEROGATE,0,0,0,2009,0,0,0 +DEROGATION,0,0,0,2009,0,0,0 +DESIRABLE,0,2009,0,0,0,0,0 +DESTABILIZATION,2009,0,0,0,0,0,0 +DESTINED,0,2009,0,0,0,0,0 +DESTROYS,2009,0,0,0,0,0,0 +DETAINED,2009,0,0,0,0,0,0 +DETER,2009,0,0,0,0,0,0 +DETERIORATING,2009,0,0,0,0,0,0 +DETERRENCE,2009,0,0,0,0,0,0 +DETERRING,2009,0,0,0,0,0,0 +DETRACTING,2009,0,0,0,0,0,0 +DETRIMENTS,2009,0,0,0,0,0,0 +DEVALUING,2009,0,0,0,0,0,0 +DEVASTATION,2009,0,0,0,0,0,0 +DEVIATING,2009,0,2009,0,0,0,0 +DEVOLVE,2009,0,0,0,0,0,0 +DICTATE,0,0,0,0,0,0,2009 +DIFFER,0,0,2009,0,0,0,0 +DIFFICULT,2009,0,0,0,0,0,0 +ASSAULT,2009,0,0,0,0,0,0 +ASSERTABLE,0,0,0,2011,0,0,0 +ASSUMABLE,0,0,0,2009,0,0,0 +ASSUMING,0,0,2009,0,0,0,0 +ASSURED,0,2009,0,0,0,0,0 +ATTAINED,0,2009,0,0,0,0,0 +ATTAINS,0,2009,0,0,0,0,0 +ATTESTED,0,0,0,2009,0,0,0 +ATTORNEYS,0,0,0,2009,0,0,0 +ATTRACTIVENESS,0,2009,0,0,0,0,0 +BAD,2009,0,0,0,0,0,0 +BAILEES,0,0,0,2011,0,0,0 +BAILOUT,2009,0,0,0,0,0,0 +BANKRUPTCIES,2009,0,0,0,0,0,0 +BANKRUPTS,2009,0,0,0,0,0,0 +CLAIM,0,0,0,2009,0,0,0 +CLAIMHOLDER,0,0,0,2014,0,0,0 +CLARIFICATIONS,0,0,2009,0,0,0,0 +CLOSED,-2020,0,0,0,0,0,0 +CLOSINGS,2009,0,0,0,0,0,0 +CODEFENDANTS,0,0,0,2011,0,0,0 +CODIFICATIONS,0,0,0,2009,0,0,0 +CODIFYING,0,0,0,2009,0,0,0 +COERCING,2009,0,0,0,0,0,0 +COLLABORATES,0,2009,0,0,0,0,0 +COLLABORATIVE,0,2009,0,0,0,0,0 +COLLAPSED,2009,0,0,0,0,0,0 +COLLISIONS,2009,0,0,0,0,0,0 +COLLUDING,2009,0,0,0,0,0,0 +POSES,2009,0,0,0,0,0,0 +POSSESSORY,0,0,0,2009,0,0,0 +POSSIBLY,0,0,2009,0,0,2009,0 +POSTJUDGMENT,0,0,0,2011,0,0,0 +POSTPONEMENTS,2009,0,0,0,0,0,0 +WRITEDOWNS,2009,0,0,0,0,0,0 +WRONG,2009,0,0,0,0,0,0 +WRONGFULLY,2009,0,0,0,0,0,0 +HONORABLE,0,-2020,0,0,0,0,0 +HOSTILE,2009,0,0,0,0,0,0 +REASSESS,0,0,2009,0,0,0,0 +REASSESSMENT,2009,0,2009,0,0,0,0 +REASSIGNING,2009,0,0,0,0,0,0 +REBOUND,0,2009,0,0,0,0,0 +REBUTS,0,0,0,2009,0,0,0 +REBUTTALS,0,0,0,2009,0,0,0 +RECALCULATED,0,0,2009,0,0,0,0 +RECALCULATIONS,0,0,2009,0,0,0,0 +RECALLS,2009,0,0,0,0,0,0 +RECESSIONS,2009,0,0,0,0,0,0 +RECONSIDER,0,0,2009,0,0,0,0 +RECORDATION,0,0,0,2009,0,0,0 +RECOURSE,0,0,0,2009,0,0,0 +RECUSAL,0,0,0,2011,0,0,0 +RECUSING,0,0,0,2011,0,0,0 +REDACTION,2009,0,0,2009,0,0,0 +REDEFAULTS,2014,0,0,0,0,0,0 +REDRESSING,2009,0,0,0,0,0,0 +REFERENDA,0,0,0,2009,0,0,0 +REFILED,0,0,0,2009,0,0,0 +REFRAINING,0,0,0,0,0,0,2009 +REFUSE,2009,0,0,0,0,0,0 +REGAIN,0,2009,0,0,0,0,0 +REGULATED,0,0,0,2009,0,0,0 +REGULATIONS,0,0,0,2009,0,0,0 +REGULATORY,0,0,0,2009,0,0,0 +REHEARINGS,0,0,0,2009,0,0,0 +VARIABILITY,0,0,2009,0,0,0,0 +VARIANCE,0,0,2009,0,0,0,0 +VARIATION,0,0,2009,0,0,0,0 +VARY,0,0,2009,0,0,0,0 +VERDICT,2009,0,0,2009,0,0,0 +VETOED,2009,0,0,0,0,0,0 +VICTIMS,2009,0,0,0,0,0,0 +VIOLATING,2009,0,0,0,0,0,0 +VIOLATOR,2009,0,0,0,0,0,0 +VIOLENTLY,2009,0,0,0,0,0,0 +VITIATING,2009,0,0,0,0,0,0 +VOIDING,2009,0,0,2009,0,0,0 +VULNERABILITIES,2009,0,0,0,0,0,0 +WARN,2009,0,0,0,0,0,0 +WARNS,2009,0,0,0,0,0,0 +WASTEFUL,2009,0,0,0,0,0,0 +WEAKENED,2009,0,0,0,0,0,0 +WEAKEST,2009,0,0,0,0,0,0 +DISHONESTLY,2009,0,0,0,0,0,0 +DISHONORABLY,2009,0,0,0,0,0,0 +DISINTERESTEDNESS,2009,0,0,0,0,0,0 +DISMAL,2009,0,0,0,0,0,0 +DISMISSALS,2009,0,0,0,0,0,0 +DISORDERLY,2009,0,0,0,0,0,0 +DISPARAGEMENTS,2009,0,0,0,0,0,0 +DISPARITIES,2009,0,0,0,0,0,0 +DISPLACEMENT,2009,0,0,0,0,0,0 +DISPOSE,2009,0,0,0,0,0,0 +DISPOSSESSES,2009,0,0,0,0,0,0 +DISPROPORTION,2009,0,0,0,0,0,0 +DISPUTE,2009,0,0,0,0,0,0 +DISQUALIFICATION,2009,0,0,0,0,0,0 +DISQUALIFY,2009,0,0,0,0,0,0 +DISREGARDING,2009,0,0,0,0,0,0 +DISRUPT,2009,0,0,0,0,0,0 +DISRUPTIONS,2009,0,0,0,0,0,0 +DISSATISFIED,2009,0,0,0,0,0,0 +DISSENTERS,2009,0,0,0,0,0,0 +DISSIDENTS,2009,0,0,0,0,0,0 +DISTINCTIONS,0,2009,0,0,0,0,0 +DISTORT,2009,0,0,0,0,0,0 +DISTORTIONS,2009,0,0,0,0,0,0 +DISTRACTING,2009,0,0,0,0,0,0 +DISTRAINT,0,0,0,2009,0,0,0 +DISTRIBUTEES,0,0,0,2009,0,0,0 +DISTURBED,2009,0,0,0,0,0,0 +DIVERT,2009,0,0,0,0,0,0 +DIVEST,2009,0,0,0,0,0,0 +DIVESTITURES,2009,0,0,0,0,0,0 +DIVORCE,2009,0,0,0,0,0,0 +DIVULGES,2009,0,0,0,0,0,0 +DOCKETING,0,0,0,2009,0,0,0 +DOUBTED,2009,0,2009,0,0,0,0 +DOWNGRADED,2009,0,0,0,0,0,0 +DOWNSIZED,2009,0,0,0,0,0,0 +DOWNTIME,2009,0,0,0,0,0,0 +DOWNWARD,2009,0,0,0,0,0,0 +DRASTICALLY,2009,0,0,0,0,0,0 +DROPPED,2009,0,0,0,0,0,0 +DURESS,2009,0,0,0,0,0,0 +EARMARK,0,0,0,0,0,0,2009 +EASIER,0,2009,0,0,0,0,0 +EFFECTIVE,0,-2020,0,0,0,0,0 +EFFICIENTLY,0,2009,0,0,0,0,0 +EMBARGO,2009,0,0,0,0,0,0 +EMBARRASS,2009,0,0,0,0,0,0 +EMBARRASSMENT,2009,0,0,0,0,0,0 +EMBEZZLEMENT,2009,0,0,0,0,0,0 +EMBEZZLING,2009,0,0,0,0,0,0 +EMPOWERS,0,2009,0,0,0,0,0 +ENABLING,0,2009,0,0,0,0,0 +ENCOURAGING,0,2009,0,0,0,0,0 +ENCROACHING,2009,0,0,0,0,0,0 +ENCUMBERED,2009,0,0,2009,0,0,2009 +ENCUMBRANCER,0,0,0,2009,0,0,0 +ENDANGERED,2009,0,0,0,0,0,0 +ENDORSEE,0,0,0,2011,0,0,0 +ENHANCE,0,2009,0,0,0,0,0 +ENHANCES,0,2009,0,0,0,0,0 +ENJOINING,2009,0,0,0,0,0,0 +ENJOYABLY,0,2009,0,0,0,0,0 +ENJOYS,0,2009,0,0,0,0,0 +ENTAILS,0,0,0,0,0,0,2009 +PATENTEE,0,0,0,2014,0,0,0 +PENALIZES,2009,0,0,0,0,0,0 +PENDING,0,0,2009,0,0,0,0 +PERFECTS,0,2009,0,0,0,0,0 +PERJURY,2009,0,0,2009,0,0,0 +PERMITTED,0,0,0,0,0,0,2009 +PERPETRATE,2009,0,0,2009,0,0,0 +PERPETRATION,2009,0,0,2009,0,0,0 +PERSISTENT,2009,0,0,0,0,0,0 +PERSONAM,0,0,0,2011,0,0,0 +PETITION,0,0,0,2009,0,0,0 +PETITIONING,0,0,0,2009,0,0,0 +PICKETED,2009,0,0,0,0,0,0 +HENCEFORTH,0,0,0,2009,0,0,0 +HEREDITAMENTS,0,0,0,2009,0,0,0 +HEREIN,0,0,0,2009,0,0,0 +HEREINBELOW,0,0,0,2009,0,0,0 +HERETOFORE,0,0,0,2009,0,0,0 +HEREWITH,0,0,0,2009,0,0,0 +HINDER,2009,0,0,0,0,0,0 +HINDRANCE,2009,0,0,0,0,0,0 +WHATSOEVER,0,0,0,2009,0,0,0 +WHEREAT,0,0,0,2009,0,0,0 +WHEREOF,0,0,0,2009,0,0,0 +WHEREUPON,0,0,0,2009,0,0,0 +WHOMSOEVER,0,0,0,2009,0,0,0 +WILLFUL,0,0,0,2009,0,0,0 +WINNER,0,2009,0,0,0,0,0 +WITNESSES,0,0,0,2009,0,0,0 +FLUCTUATES,0,0,2009,0,0,0,0 +FORBADE,0,0,0,2009,0,0,2011 +FORBEARING,0,0,0,2009,0,0,0 +FORBIDDING,2009,0,0,0,0,0,2009 +FORCING,2009,0,0,0,0,0,0 +FORECLOSE,2009,0,0,0,0,0,0 +FORECLOSURE,2009,0,0,0,0,0,0 +FOREGONE,2009,0,0,0,0,0,0 +FORESTALLS,2009,0,0,0,0,0,0 +FORFEITED,2009,0,0,0,0,0,0 +FORFEITURES,2009,0,0,0,0,0,0 +FORWHICH,0,0,0,2012,0,0,0 +FRAUDULENT,2009,0,0,0,0,0,0 +FRIVOLOUSLY,2009,0,0,0,0,0,0 +FRUSTRATING,2009,0,0,0,0,0,0 +FUGITIVE,-2020,0,0,2009,0,0,0 +GAINED,0,2009,0,0,0,0,0 +GRANTOR,0,0,0,2009,0,0,0 +DISGORGING,2009,0,0,0,0,0,0 +DISHONEST,2009,0,0,0,0,0,0 +LITIGANT,2009,0,0,2009,0,0,0 +LITIGATES,2009,0,0,2009,0,0,0 +LITIGATOR,0,0,0,2009,0,0,0 +LACKLUSTER,2009,0,0,0,0,0,0 +LAGGING,2009,0,0,0,0,0,0 +LAPSES,2009,0,0,0,0,0,0 +LAW,0,0,0,2009,0,0,0 +LAWMAKERS,0,0,0,2009,0,0,0 +LAWSUITS,0,0,0,2009,0,0,0 +LAYOFFS,2009,0,0,0,0,0,0 +LEGALESE,0,0,0,2009,0,0,0 +LEGALIZE,0,0,0,2009,0,0,0 +LEGALLY,0,0,0,2009,0,0,0 +NONPRODUCING,2009,0,0,0,0,0,0 +LIQUIDATE,2009,0,0,0,0,0,0 +LIQUIDATION,2009,0,0,0,0,0,0 +BENEFICIALLY,0,2009,0,0,0,0,0 +BENEFITED,0,2009,0,0,0,0,0 +BEST,0,2012,0,0,2009,0,0 +POPULAR,0,2009,0,0,0,0,0 +REINTERPRETED,0,0,2009,0,0,0,0 +REJECTED,2009,0,0,0,0,0,0 +REJECTS,2009,0,0,0,0,0,0 +RELINQUISHES,2009,0,0,0,0,0,0 +RELUCTANCE,2009,0,0,0,0,0,0 +REMANDING,0,0,0,2009,0,0,0 +REMEDIATING,0,0,0,2009,0,0,0 +REMISED,0,0,0,2011,0,0,0 +RENEGOTIATING,2009,0,0,0,0,0,0 +RENOUNCED,2009,0,0,0,0,0,0 +RENOUNCING,2009,0,0,0,0,0,0 +REPLEVIN,0,0,0,2009,0,0,0 +REPOSSESSION,2009,0,0,0,0,0,0 +TURBULENCE,2009,0,2009,0,0,0,0 +UNACCEPTABLY,2009,0,0,0,0,0,0 +UNANTICIPATED,2009,0,0,0,0,0,0 +UNATTRACTIVE,2009,0,0,0,0,0,0 +UNAVOIDABLE,2009,0,0,0,0,0,0 +UNCERTAINLY,0,0,2009,0,0,2009,0 +UNCOLLECTABLE,2009,0,0,0,0,0,0 +UNCOLLECTIBLES,2009,0,0,0,0,0,0 +UNCONFIRMED,0,0,2009,0,0,0,0 +UNCONSTITUTIONALITY,0,0,0,2009,0,0,0 +UNCONTROLLABLY,2009,0,0,0,0,0,0 +UNCOVERED,2009,0,0,0,0,0,0 +UNDEFEASED,0,0,0,2012,0,0,0 +UNDERCAPITALIZED,2009,0,0,0,0,0,0 +UNDERESTIMATE,2009,0,0,0,0,0,0 +UNDERESTIMATION,2009,0,0,0,0,0,0 +UNDERMINED,2009,0,0,0,0,0,0 +UNDERPAYMENT,2009,0,0,0,0,0,0 +UNDERPERFORMANCE,2009,0,0,0,0,0,0 +UNDERPRODUCED,2009,0,0,0,0,0,0 +UNDERSTATED,2009,0,0,0,0,0,0 +UNDERSTATING,2009,0,0,0,0,0,0 +UNDESIRABLE,2009,0,0,0,0,0,0 +UNDETERMINABLE,0,0,2009,0,0,0,0 +UNDISPUTED,0,0,0,0,2009,0,0 +UNDULY,2009,0,0,0,0,0,0 +UNEMPLOYED,2009,0,0,0,0,0,0 +UNENFORCEABILITY,0,0,0,2009,0,0,0 +UNETHICAL,2009,0,0,0,0,0,0 +UNEXPECTEDLY,2009,0,2009,0,0,0,0 +UNFAMILIARITY,0,0,2009,0,0,0,0 +UNFAVOURABLE,2011,0,0,0,0,0,0 +UNFORECASTED,0,0,2011,0,0,0,0 +UNFORTUNATE,2009,0,0,0,0,0,0 +UNFULFILLED,2009,0,0,0,0,0,0 +UNIDENTIFIABLE,0,0,2009,0,0,0,0 +UNINTENTIONAL,2009,0,0,0,0,0,0 +UNJUSTIFIABLY,2009,0,0,0,0,0,0 +UNKNOWINGLY,2009,0,0,0,0,0,0 +UNLAWFULLY,2009,0,0,2009,0,0,0 +UNMARKETABLE,2009,0,0,0,0,0,0 +UNNECESSARILY,2009,0,0,0,0,0,0 +UNOBTAINABLE,2009,0,0,0,0,0,0 +UNPERFORMED,2009,0,0,0,0,0,0 +UNPREDICTABLE,2009,0,2009,0,0,0,0 +UNPROFITABILITY,2011,0,0,0,0,0,0 +UNQUALIFIED,2009,0,0,0,0,0,0 +UNREASONABLE,2009,0,0,0,0,0,0 +UNRECONCILED,0,0,2011,0,0,0,0 +UNRELIABLE,2009,0,0,0,0,0,0 +UNRESOLVED,2009,0,0,0,0,0,0 +UNSALEABLE,2009,0,0,0,0,0,0 +UNSCHEDULED,2009,0,0,0,0,0,0 +UNSETTLED,0,0,2009,0,0,0,0 +UNSPECIFIED,0,0,2009,0,0,0,0 +UNSUBSTANTIATED,2009,0,0,0,0,0,0 +UNSUITABLE,2009,0,0,0,0,0,0 +UNSURPASSED,0,2009,0,0,2009,0,0 +UNTENABLE,2009,0,0,0,0,0,0 +UNTRUSTED,2014,0,0,0,0,0,0 +UNTRUTHFULNESS,2009,0,0,0,0,0,0 +UNUSUALLY,0,0,2009,0,0,0,0 +UNWILLING,2009,0,0,0,0,0,0 +UPTURN,0,2009,0,0,0,0,0 +USURIOUS,2009,0,0,2009,0,0,0 +USURPING,2009,0,0,2009,0,0,0 +VAGUE,0,0,2012,0,0,0,0 +VAGUER,0,0,2012,0,0,0,0 +PLAINTIFFS,2009,0,0,2009,0,0,0 +PLEADING,2009,0,0,2009,0,0,0 +PLEASANT,0,2009,0,0,0,0,0 +PLED,2009,0,0,0,0,0,0 +PLEDGEES,0,0,0,2011,0,0,0 +PLEDGORS,0,0,0,2012,0,0,0 +COMPELLING,0,0,0,0,0,0,2011 +COMPLAINANT,0,0,0,2009,0,0,0 +COMPLAINS,2009,0,0,0,0,0,0 +COMMITS,0,0,0,0,0,0,2009 +LIKELIHOOD,0,0,2009,0,0,0,0 +LIMITING,0,0,0,0,0,0,2009 +RESTRAIN,0,0,0,0,0,0,2009 +RESTRAINT,0,0,0,0,0,0,2009 +RESTRICTING,0,0,0,0,0,0,2009 +RESTRICTIVELY,0,0,0,0,0,0,2009 +RESTRUCTURED,2009,0,0,0,0,0,0 +RETALIATE,2009,0,0,0,0,0,0 +RETALIATION,2009,0,0,0,0,0,0 +PRECAUTION,0,0,2009,0,0,0,0 +PRECIPITOUS,2009,0,0,0,0,0,0 +PRECLUDES,2009,0,0,0,0,0,2009 +PREDATORY,2009,0,0,0,0,0,0 +PREDECEASING,0,0,0,2009,0,0,0 +PREDICTING,0,0,2009,0,0,0,0 +PREDICTOR,0,0,2009,0,0,0,0 +PREEMINENT,0,2009,0,0,0,0,0 +PREJUDICES,2009,0,0,2009,0,0,0 +PRELIMINARY,0,0,2009,0,0,0,0 +PREMIERE,0,2009,0,0,0,0,0 +PRESTIGE,0,2009,0,0,0,0,0 +PRESUMED,0,0,2009,0,0,0,0 +PRESUMPTIONS,0,0,2009,0,0,0,0 +PREVENTED,0,0,0,0,0,0,2009 +PRIMA,0,0,0,2009,0,0,0 +PROBABILISTIC,0,0,2009,0,0,0,0 +PROBABLY,0,0,2009,0,0,0,0 +PROBATING,0,0,0,2009,0,0,0 +PROBATIONER,0,0,0,2009,0,0,0 +PROBLEMATIC,2009,0,0,0,0,0,0 +PROFICIENT,0,2009,0,0,0,0,0 +PROFITABLY,0,2009,0,0,0,0,0 +PROGRESSING,0,2009,0,0,0,0,0 +PROHIBITION,0,0,0,0,0,0,2009 +PROHIBITORY,0,0,0,0,0,0,2009 +PROLONGATIONS,2009,0,0,0,0,0,0 +PROMULGATE,0,0,0,2009,0,0,0 +PROMULGATION,0,0,0,2009,0,0,0 +PRONE,2009,0,0,0,0,0,0 +PROSECUTED,2009,0,0,2009,0,0,0 +PROSECUTIONS,2009,0,0,2009,0,0,0 +PROSPERED,0,2009,0,0,0,0,0 +PROSPERS,0,2009,0,0,0,0,0 +PROTESTERS,2009,0,0,0,0,0,0 +PROTESTS,2009,0,0,0,0,0,0 +PROVISOES,0,0,0,2009,0,0,0 +PROVOKES,2009,0,0,0,0,0,0 +PUNISHES,2009,0,0,0,0,0,0 +PUNITIVE,2009,0,0,0,0,0,0 +PURPORTING,2009,0,0,0,0,0,0 +QUESTIONABLY,2009,0,0,0,0,0,0 +QUIT,2009,0,0,0,0,0,0 +RACKETEER,2009,0,0,0,0,0,0 +RANDOMIZED,0,0,2009,0,0,0,0 +RANDOMNESS,0,0,2009,0,0,0,0 +RATIONALIZATION,2009,0,0,0,0,0,0 +RATIONALIZES,2009,0,0,0,0,0,0 +ABANDONMENT,2009,0,0,0,0,0,0 +ABDICATES,2009,0,0,0,0,0,0 +ABERRANT,2009,0,0,0,0,0,0 +ABETTING,2009,0,0,0,0,0,0 +ABIDING,0,0,0,0,0,0,2009 +ABNORMALITY,2009,0,0,0,0,0,0 +ABOLISHES,2009,0,0,0,0,0,0 +ABROGATED,2009,0,0,2009,0,0,0 +ABROGATIONS,2009,0,0,2009,0,0,0 +ABSENCE,2009,0,0,0,0,0,0 +ABSOLVED,0,0,0,2009,0,0,0 +ABUNDANT,0,2009,0,0,0,0,0 +ABUSING,2009,0,0,0,0,0,0 +ACCESSION,0,0,0,2009,0,0,0 +ACCIDENTALLY,2009,0,0,0,0,0,0 +ACCOMPLISHED,0,2009,0,0,0,0,0 +ACCOMPLISHMENTS,0,2009,0,0,0,0,0 +ACCUSED,2009,0,0,0,0,0,0 +ACHIEVED,0,2009,0,0,0,0,0 +ACHIEVING,0,2009,0,0,0,0,0 +ACQUIESCING,2009,0,0,0,0,0,0 +ACQUITS,2009,0,0,2009,0,0,0 +ACQUITTANCES,0,0,0,2011,0,0,0 +ADEQUATELY,0,2009,0,0,0,0,0 +ADJOURNMENT,0,0,0,2009,0,0,0 +ADJUDGED,0,0,0,2009,0,0,0 +ADJUDICATED,0,0,0,2009,0,0,0 +ADJUDICATIONS,0,0,0,2009,0,0,0 +ADJUDICATORY,0,0,0,2009,0,0,0 +ADMISSION,0,0,0,2009,0,0,0 +ADULTERATING,2009,0,0,0,0,0,0 +ADVANCEMENTS,0,2009,0,0,0,0,0 +ADVANTAGED,0,2009,0,0,0,0,0 +ADVERSARIAL,2009,0,0,0,0,0,0 +ADVERSELY,2009,0,0,0,0,0,0 +AFFIDAVITS,0,0,0,2009,0,0,0 +AFOREMENTIONED,0,0,0,2009,0,0,0 +AFTERMATHS,2009,0,0,0,0,0,0 +AGGRAVATES,2009,0,0,0,0,0,0 +AGGRIEVED,0,0,0,2009,0,0,0 +ALIENATED,2009,0,0,0,0,0,0 +ALIENATIONS,2009,0,0,0,0,0,0 +ALLEGED,2009,0,0,2009,0,0,0 +ALLIANCE,0,2009,0,0,0,0,0 +ALTERATIONS,0,0,2009,0,0,0,0 +AMBIGUOUS,0,0,2009,0,0,0,0 +AMENDED,0,0,0,2009,0,0,0 +AMENDS,0,0,0,2009,0,0,0 +ANNOYED,2009,0,0,0,0,0,0 +ANNULLED,2009,0,0,0,0,0,0 +ANNULS,2009,0,0,0,0,0,0 +ANOMALY,2009,0,2009,0,0,0,0 +ANTICIPATED,0,0,2009,0,0,0,0 +ANTICIPATIONS,0,0,2009,0,0,0,0 +ANYWISE,0,0,0,2009,0,0,0 +APPEALABLE,0,0,0,2009,0,0,0 +APPEAR,0,0,2009,0,0,0,0 +APPELLANT,0,0,0,2009,0,0,0 +APPOINTOR,0,0,0,2011,0,0,0 +APPROXIMATES,0,0,2009,0,0,0,0 +APPURTENANCE,0,0,0,2009,0,0,0 +ARBITRAL,0,0,0,2009,0,0,0 +ARBITRATE,0,0,0,2009,0,0,0 +ARBITRATION,0,0,0,2009,0,0,0 +ARBITRATOR,0,0,0,2009,0,0,0 +ARGUING,2009,0,0,0,0,0,0 +ARREARAGE,2009,0,0,2009,0,0,0 +ARRESTED,2009,0,0,0,0,0,0 +RETROCEDE,0,0,0,2011,0,0,0 +BOLSTERED,0,2009,0,0,0,0,0 +BONAFIDE,0,0,0,2009,0,0,0 +BOOSTED,0,2009,0,0,0,0,0 +BOUNDED,0,0,0,0,0,0,2009 +BOYCOTTS,2009,0,0,0,0,0,0 +BREACHING,2009,0,0,2009,0,0,0 +BREAKDOWN,2009,0,0,0,0,0,0 +BREAKTHROUGH,0,2009,0,0,0,0,0 +BRIBERIES,2009,0,0,0,0,0,0 +BRIDGE,-2020,0,0,0,0,0,0 +BURDENED,2009,0,0,0,0,0,0 +BURNED,2009,0,0,0,0,0,0 +CANCEL,2009,0,0,0,0,0,0 +CANCELLATIONS,2009,0,0,0,0,0,0 +CARELESS,2009,0,0,0,0,0,0 +CATASTROPHE,2009,0,0,0,0,0,0 +CAUTION,2009,0,0,0,0,0,0 +CAUTIONS,2009,0,0,0,0,0,0 +CEASE,2009,0,0,0,0,0,0 +CEDANT,0,0,0,2012,0,0,0 +CENSURES,2009,0,0,0,0,0,0 +CHALLENGE,2009,0,0,0,0,0,0 +CHARGEOFFS,2009,0,0,0,0,0,0 +CHOATE,0,0,0,2011,0,0,0 +CIRCUMVENTION,2009,0,0,0,0,0,0 +SHOCKED,2009,0,0,0,0,0,0 +SHORTFALLS,2009,0,0,0,0,0,0 +SHUTDOWN,2009,0,0,0,0,0,0 +SLANDER,2009,0,0,0,0,0,0 +REPUDIATED,2009,0,0,0,0,0,0 +REPUDIATIONS,2009,0,0,0,0,0,0 +REQUIRED,0,0,0,0,0,0,2009 +REQUIRING,0,0,0,0,0,0,2009 +RESCINDING,0,0,0,2009,0,0,0 +RESIGN,2009,0,0,0,0,0,0 +RESIGNING,2009,0,0,0,0,0,0 +RESTATED,2009,0,0,0,0,0,0 +RESTATING,2009,0,0,0,0,0,0 +NONUSURIOUS,0,0,0,2011,0,0,0 +NOTARIZATIONS,0,0,0,2009,0,0,0 +NOTARY,0,0,0,2009,0,0,0 +NUISANCES,2009,0,0,0,0,0,0 +NULLIFIES,2009,0,0,2009,0,0,0 +NULLITY,0,0,0,2009,0,0,0 +OBJECTIONABLE,2009,0,0,0,0,0,0 +OBLIGATED,0,0,0,0,0,0,2009 +OBLIGATIONS,0,0,0,0,0,0,2009 +OBLIGEE,0,0,0,2009,0,0,0 +OBLIGORS,0,0,0,2009,0,0,0 +OBSOLETE,2009,0,0,0,0,0,0 +OBSTRUCTED,2009,0,0,0,0,0,0 +OCCASIONALLY,0,0,2009,0,0,2009,0 +OFFENDED,2009,0,0,0,0,0,0 +OFFENDS,2009,0,0,0,0,0,0 +OFFEROR,0,0,0,2009,0,0,0 +OMIT,2009,0,0,0,0,0,0 +ONEROUS,2009,0,0,0,0,0,0 +OPPORTUNITY,0,2009,0,0,0,0,0 +OPPOSING,2009,0,0,0,0,0,0 +OPTIONEE,0,0,0,2009,0,0,0 +OUTAGES,2009,0,0,0,0,0,0 +OUTPERFORMED,0,2009,0,0,0,0,0 +OVERAGES,2009,0,0,0,0,0,0 +OVERBUILT,2009,0,0,0,0,0,0 +OVERCAPACITIES,2009,0,0,0,0,0,0 +OVERCHARGES,2009,0,0,0,0,0,0 +OVERCOMING,2009,0,0,0,0,0,0 +OVERESTIMATES,2009,0,0,0,0,0,0 +OVERLOAD,2009,0,0,0,0,0,0 +OVERLOOK,2009,0,0,0,0,0,0 +OVERPAID,2009,0,0,0,0,0,0 +OVERPRODUCES,2009,0,0,0,0,0,0 +OVERRULED,0,0,0,2009,0,0,0 +OVERRUNNING,2009,0,0,0,0,0,0 +OVERSHADOWING,2009,0,0,0,0,0,0 +OVERSTATEMENT,2009,0,0,0,0,0,0 +OVERSUPPLIED,2009,0,0,0,0,0,0 +OVERTLY,2009,0,0,0,0,0,0 +OVERTURNS,2009,0,0,0,0,0,0 +PANIC,2009,0,0,0,0,0,0 +NEARLY,0,0,2009,0,0,2009,0 +NECESSITATING,0,0,0,0,0,0,2009 +NEGLECT,2009,0,0,0,0,0,0 +NEGLECTS,2009,0,0,0,0,0,0 +NEGLIGENTLY,2009,0,0,0,0,0,0 +BARRIER,2009,0,0,0,0,0,0 +BELIEVE,0,0,2009,0,0,0,0 +IDLED,2009,0,0,0,0,0,0 +IGNORES,2009,0,0,0,0,0,0 +ILLEGALITIES,2009,0,0,0,0,0,0 +ILLICIT,2009,0,0,0,0,0,0 +IMBALANCE,2009,0,0,0,0,0,0 +IMMORAL,2009,0,0,0,0,0,0 +IMPAIRMENT,2009,0,0,0,0,0,2011 +IMPASSES,2009,0,0,0,0,0,0 +IMPEDIMENT,2009,0,0,0,0,0,0 +IMPERATIVE,2009,0,0,0,0,0,0 +IMPERMISSIBLE,2009,0,0,0,0,0,0 +IMPLICATES,2009,0,0,0,0,0,0 +IMPOSES,0,0,0,0,0,0,2009 +IMPOSSIBILITY,2009,0,0,0,0,0,0 +IMPOUNDING,2009,0,0,0,0,0,0 +IMPRACTICALITIES,2009,0,0,0,0,0,0 +IMPRECISIONS,0,0,2009,0,0,0,0 +IMPRESSING,0,2009,0,0,0,0,0 +IMPROBABILITY,0,0,2009,0,0,0,0 +IMPROPRIETIES,2009,0,0,0,0,0,0 +IMPROVEMENT,0,2009,0,0,0,0,0 +IMPRUDENT,2009,0,0,0,0,0,0 +INACCURACIES,2009,0,0,0,0,0,0 +INACTION,2009,0,0,0,0,0,0 +INACTIVATES,2009,0,0,0,0,0,0 +INACTIVITY,2009,0,0,0,0,0,0 +INADEQUATELY,2009,0,0,0,0,0,0 +INADVISABLE,2009,0,0,0,0,0,0 +INATTENTION,2009,0,0,0,0,0,0 +INCARCERATE,2009,0,0,2009,0,0,0 +INCARCERATION,2009,0,0,2009,0,0,0 +INCIDENCES,2009,0,0,0,0,0,0 +INCOMPATIBILITY,2009,0,0,0,0,0,0 +INCOMPETENT,2009,0,0,0,0,0,0 +INCOMPLETELY,2009,0,0,0,0,0,0 +INCONSISTENCY,2009,0,0,0,0,0,0 +INCONTESTABLE,0,0,0,2009,0,0,0 +INCORRECT,2009,0,0,0,0,0,0 +INCREDIBLY,0,2009,0,0,0,0,0 +INDEFEASIBLE,2009,0,0,0,0,0,0 +INDEFINITENESS,0,0,2009,0,0,0,0 +INDEMNIFIED,0,0,0,2009,0,0,0 +INDEMNITEE,0,0,0,2009,0,0,0 +INDEMNITORS,0,0,0,2009,0,0,0 +INDICT,2009,0,0,2009,0,0,0 +INDICTMENT,2009,0,0,2009,0,0,0 +INEFFECTIVELY,2009,0,0,0,0,0,0 +INEFFICIENT,2009,0,0,0,0,0,0 +INEQUITABLE,2009,0,0,0,0,0,0 +INEVITABLE,2009,0,0,0,0,0,0 +INEXPERIENCED,2009,0,0,0,0,0,0 +INFORCE,0,0,0,2009,0,0,0 +INFRINGE,2009,0,0,0,0,0,0 +INFRINGER,0,0,0,2009,0,0,0 +INHIBIT,0,0,0,0,0,0,2011 +INIMICAL,2009,0,0,0,0,0,0 +INJURE,2009,0,0,0,0,0,0 +INJURING,2009,0,0,0,0,0,0 +INNOVATED,0,2009,0,0,0,0,0 +INNOVATIONS,0,2009,0,0,0,0,0 +INNOVATORS,0,2009,0,0,0,0,0 +INSECURE,2009,0,0,0,0,0,0 +INSISTED,0,0,0,0,0,0,2009 +INSOFAR,0,0,0,2009,0,0,0 +INSPIRATION,0,2009,0,0,0,0,0 +INSUBORDINATION,2009,0,0,0,0,0,0 +INSURRECTION,2009,0,0,0,0,0,0 +INTEGRITY,0,2009,0,0,0,0,0 +INTERFERENCE,2009,0,0,0,0,0,0 +INTERLOCUTORY,0,0,0,2009,0,0,0 +INTERPOSE,0,0,0,2009,0,0,0 +INTERPOSITION,0,0,0,2009,0,0,0 +INTERROGATES,0,0,0,2009,0,0,0 +INTERROGATOR,0,0,0,2009,0,0,0 +INTERRUPT,2009,0,0,0,0,0,0 +INTERRUPTIONS,2009,0,0,0,0,0,0 +INTIMIDATION,2009,0,0,0,0,0,0 +SPORADICALLY,0,0,2009,0,0,0,0 +STABILIZE,0,2009,0,0,0,0,0 +STABLE,0,2009,0,0,0,0,0 +STAGNATED,2009,0,0,0,0,0,0 +STANDSTILL,2009,0,0,0,0,0,0 +STATUTORILY,0,0,0,2009,0,0,0 +STIPULATES,0,0,0,0,0,0,2009 +STOLEN,2009,0,0,0,0,0,0 +STOPPING,2009,0,0,0,0,0,0 +STRAINING,2009,0,0,0,0,0,0 +STRENGTHENED,0,2009,0,0,0,0,0 +STRESS,2009,0,0,0,0,0,0 +STRESSING,2009,0,0,0,0,0,0 +STRICTLY,0,0,0,0,0,0,2009 +STRONGEST,0,2009,0,0,0,0,0 +SUBDOCKET,0,0,0,2012,0,0,0 +SUBLEASEE,0,0,0,2011,0,0,0 +SUBLICENSOR,0,0,0,2011,0,0,0 +SUBPOENAED,2009,0,0,2009,0,0,0 +SUBSTANDARD,2009,0,0,0,0,0,0 +SUCCEEDED,0,2009,0,0,0,0,0 +SUCCESSES,0,2009,0,0,0,0,0 +SUDDENLY,0,0,2009,0,0,0,0 +SUFFER,2009,0,0,0,0,0,0 +SUGGEST,0,0,2009,0,0,2009,0 +SUING,2009,0,0,2009,0,0,0 +SUMMONSES,2009,0,0,2009,0,0,0 +SUPERSEDED,0,0,0,2009,0,0,0 +SURETY,0,0,0,2009,0,0,0 +SURPASSING,0,2009,0,0,0,0,0 +SUSPECTED,2009,0,0,0,0,0,0 +SUSPENDING,2009,0,0,0,0,0,0 +SUSPICION,2009,0,0,0,0,0,0 +REVISED,0,0,2009,0,0,0,0 +REVOKE,2009,0,0,0,0,0,0 +REVOLUTIONIZE,0,2009,0,0,0,0,0 +REWARD,0,2009,0,0,0,0,0 +RIDICULE,2009,0,0,0,0,0,0 +RISK,0,0,2009,0,0,0,0 +RISKINESS,0,0,2009,0,0,0,0 +ROUGHLY,0,0,2009,0,0,0,0 +SABOTAGE,2009,0,0,0,0,0,0 +SACRIFICIAL,2009,0,0,0,0,0,0 +SATISFACTORY,0,2009,0,0,0,0,0 +SATISFYING,0,2009,0,0,0,0,0 +SCRUTINIZED,2009,0,0,0,0,0,0 +SECRECY,-2020,0,0,0,0,0,0 +SEIZES,2009,0,0,0,0,0,0 +SENTENCED,2009,0,0,2009,0,0,0 +SERIOUSLY,2009,0,0,0,0,0,0 +SETTLEMENT,0,0,0,2009,0,0,0 +SEVERABLE,0,0,0,2009,0,0,0 +SEVERE,2009,0,0,0,0,0,0 +SEVERITY,2009,0,0,0,0,0,0 +ENTHUSIASTIC,0,2009,0,0,0,0,0 +ERODE,2009,0,0,0,0,0,0 +EROSION,2009,0,0,0,0,0,0 +ERRING,2009,0,0,0,0,0,0 +ERRORS,2009,0,0,0,0,0,0 +ESCALATES,2009,0,0,0,0,0,0 +ESCHEATMENT,0,0,0,2011,0,0,0 +ESCROWS,0,0,0,0,0,0,2009 +EVADES,2009,0,0,0,0,0,0 +EVASIVE,2009,0,0,0,0,0,0 +EVICTION,2009,0,0,0,0,0,0 +EVIDENTIARY,0,0,0,2009,0,0,0 +EXACERBATING,2009,0,0,0,0,0,0 +EXAGGERATED,2009,0,0,0,0,0,0 +EXCEEDANCE,0,0,0,2011,0,0,0 +EXCELLENT,0,2009,0,0,0,0,0 +EXCEPTIONALLY,0,2009,0,0,0,0,0 +EXCITED,0,2009,0,0,0,0,0 +EXCLUSIVELY,0,2009,0,0,0,0,0 +EXCULPATE,2009,0,0,2009,0,0,0 +EXCULPATION,2009,0,0,2009,0,0,0 +EXECUTORS,0,0,0,2009,0,0,0 +EXECUTRIXES,0,0,0,2009,0,0,0 +EXONERATES,2009,0,0,0,0,0,0 +EXPLOIT,2009,0,0,0,0,0,0 +EXPLOITED,2009,0,0,0,0,0,0 +EXPOSED,2009,0,0,0,0,0,0 +EXPOSURES,0,0,2009,0,0,0,0 +EXPROPRIATING,2009,0,0,0,0,0,0 +EXPULSIONS,2009,0,0,0,0,0,0 +EXTRAJUDICIAL,0,0,0,2014,0,0,0 +SOLVES,0,2009,0,0,0,0,0 +SOMEWHAT,0,0,2009,0,0,2009,0 +SPAMMING,2014,0,0,0,0,0,0 +TREMENDOUSLY,0,2009,0,0,0,0,0 +LOCKOUTS,2009,0,0,0,0,0,0 +LOSS,2009,0,0,0,0,0,0 +LOYAL,0,2009,0,0,0,0,0 +MALFEASANCE,2009,0,0,0,0,0,0 +MALFUNCTIONS,2009,0,0,0,0,0,0 +MALPRACTICE,2009,0,0,0,0,0,0 +MANDATES,0,0,0,0,0,0,2009 +MANIPULATE,2009,0,0,0,0,0,0 +MANIPULATION,2009,0,0,0,0,0,0 +MARKDOWNS,2009,0,0,0,0,0,0 +MEDIATED,0,0,0,2009,0,0,0 +MEDIATIONS,0,0,0,2009,0,0,0 +MIGHT,0,0,2009,0,0,2009,0 +MISAPPLIES,2009,0,0,0,0,0,0 +MISAPPROPRIATED,2009,0,0,0,0,0,0 +MISAPPROPRIATIONS,2009,0,0,0,0,0,0 +MISCALCULATES,2009,0,0,0,0,0,0 +MISCHARACTERIZATION,2014,0,0,0,0,0,0 +MISCLASSIFIED,2011,0,0,0,0,0,0 +MISDATED,2011,0,0,0,0,0,0 +MISFEASANCE,0,0,0,2009,0,0,0 +MISHANDLING,2009,0,0,0,0,0,0 +MISINFORMING,2009,0,0,0,0,0,0 +MISINTERPRETATIONS,2009,0,0,0,0,0,0 +MISJUDGE,2009,0,0,0,0,0,0 +MISJUDGMENT,2009,0,0,0,0,0,0 +MISLABELING,2009,0,0,0,0,0,0 +MISLEADING,2009,0,0,0,0,0,0 +MISMANAGE,2009,0,0,0,0,0,0 +MISMANAGING,2009,0,0,0,0,0,0 +MISMATCHING,2009,0,0,0,0,0,0 +MISPRICINGS,2014,0,0,0,0,0,0 +MISREPRESENTED,2009,0,0,0,0,0,0 +MISSED,2009,0,0,0,0,0,0 +WORRIES,2009,0,0,0,0,0,0 +WORSEN,2009,0,0,0,0,0,0 +WORST,2009,0,0,0,0,0,0 +TIGHTENING,2009,0,0,0,0,0,0 +TOLERATING,2009,0,0,0,0,0,0 +TORTIOUSLY,0,0,0,2009,0,0,0 +FINES,2009,0,0,0,0,0,0 +NONASSESSABLE,0,0,2009,0,0,0,0 +NONCANCELLABLE,0,0,0,0,0,0,2009 +NONCOMPLIANT,2009,0,0,0,0,0,0 +NONCONFORMITY,2009,0,0,0,0,0,0 +NONCONTRIBUTORY,0,0,0,2009,0,0,0 +NONFORFEITABILITY,0,0,0,2011,0,0,0 +NONGUARANTOR,0,0,0,2011,0,0,0 +DISCIPLINARY,2009,0,0,0,0,0,0 +DISCLAIMERS,2009,0,0,0,0,0,0 +DISCLOSED,2009,0,0,0,0,0,0 +DISCONTINUANCES,2009,0,0,0,0,0,0 +DISCONTINUED,2009,0,0,0,0,0,0 +DISCOURAGED,2009,0,0,0,0,0,0 +DISCREDITED,2009,0,0,0,0,0,0 +DISCREPANCY,2009,0,0,0,0,0,0 +SPECULATED,0,0,2009,0,0,0,0 +SPECULATIONS,0,0,2009,0,0,0,0 +TRAGICALLY,2009,0,0,0,0,0,0 +LEGISLATED,0,0,0,2009,0,0,0 +LEGISLATIONS,0,0,0,2009,0,0,0 +LEGISLATORS,0,0,0,2009,0,0,0 +LIBELED,0,0,0,2009,0,0,0 +LIE,2009,0,0,0,0,0,0 +COMPULSION,2009,0,0,0,0,0,2009 +CONCEDE,2009,0,0,0,0,0,0 +CONCEIVABLE,0,0,2009,0,0,2009,0 +CONCERNS,2009,0,0,0,0,0,0 +CONCLUSIVE,0,2009,0,0,0,0,0 +CONDEMNATIONS,2009,0,0,0,0,0,0 +CONDEMNS,2009,0,0,0,0,0,0 +CONDONED,2009,0,0,0,0,0,0 +CONFESSES,2009,0,0,0,0,0,0 +CONFINE,2009,0,0,0,0,0,2009 +CONFINES,2009,0,0,0,0,0,2009 +CONFISCATES,2009,0,0,0,0,0,0 +CONFISCATORY,0,0,0,2009,0,0,0 +CONFLICTS,2009,0,0,0,0,0,0 +CONFRONTATIONS,2009,0,0,0,0,0,0 +CONFUSE,2009,0,0,0,0,0,0 +CONFUSINGLY,2009,0,2009,0,0,0,0 +CONSENTING,0,0,0,2009,0,0,0 +CONSPIRACY,2009,0,0,0,0,0,0 +CONSPIRE,2009,0,0,0,0,0,0 +CONSTITUTION,0,0,0,2009,0,0,0 +CONSTITUTIONS,0,0,0,2009,0,0,0 +CONSTRAINING,0,0,0,0,0,0,2009 +CONSTRUCTIVE,0,2009,0,0,0,0,0 +CONSTRUES,0,0,0,2012,0,0,0 +CONTENDED,2009,0,0,0,0,0,0 +CONTENTIONS,2009,0,0,0,0,0,0 +CONTESTATION,0,0,0,2011,0,0,0 +CONTINGENCY,0,0,2009,0,0,0,0 +CONTRACT,0,0,0,2009,0,0,0 +CONTRACTIBLE,0,0,0,2009,0,0,0 +CONTRACTIONS,2009,0,0,0,0,0,0 +CONTRADICT,2009,0,0,0,0,0,0 +CONTRADICTIONS,2009,0,0,0,0,0,0 +CONTRAVENE,0,0,0,2009,0,0,0 +CONTRAVENTION,0,0,0,2009,0,0,0 +CONTROVERSY,2009,0,0,0,0,0,0 +CONVENIENS,0,0,0,2012,0,0,0 +CONVICTED,2009,0,0,2009,0,0,0 +CORRECTED,2009,0,0,0,0,0,0 +CORRECTS,2009,0,0,0,0,0,0 +CORRUPTION,2009,0,0,0,0,0,0 +COSTLY,2009,0,0,0,0,0,0 +COUNSELED,0,0,0,2009,0,0,0 +COUNTERCLAIMED,2009,0,0,0,0,0,0 +COUNTERFEITED,2009,0,0,0,0,0,0 +COUNTERFEITS,2009,0,0,0,0,0,0 +COUNTERSUED,0,0,0,2011,0,0,0 +COURTEOUS,0,2009,0,0,0,0,0 +COVENANTED,0,0,0,0,0,0,2011 +CREATIVELY,0,2009,0,0,0,0,0 +CRIMES,2009,0,0,2009,0,0,0 +CRIMINALIZING,0,0,0,2014,0,0,0 +CRISIS,2009,0,0,0,0,0,0 +CRITICISMS,2009,0,0,0,0,0,0 +CRITICIZING,2009,0,0,0,0,0,0 +CROSSROADS,0,0,2009,0,0,0,0 +CULPABLE,2009,0,0,0,0,0,0 +CURTAILED,2009,0,0,0,0,0,0 +CURTAILS,2009,0,0,0,0,0,0 +CYBERATTACK,2014,0,0,0,0,0,0 +CYBERCRIMES,2014,0,0,0,0,0,0 +TAINTS,2009,0,0,0,0,0,0 +TENSE,2009,0,0,0,0,0,0 +TERMINATE,2009,0,0,0,0,0,0 +TERMINATION,2009,0,0,0,0,0,0 +TESTIFY,2009,0,0,2009,0,0,0 +THENCEFORTH,0,0,0,2009,0,0,0 +THEREFROM,0,0,0,2009,0,0,0 +THEREON,0,0,0,2009,0,0,0 +THERETOFORE,0,0,0,2009,0,0,0 +THEREWITH,0,0,0,2009,0,0,0 +THREATENING,2009,0,0,0,0,0,0 +FACIE,0,0,0,2009,0,0,0 +FAILING,2009,0,0,0,0,0,0 +FAILURES,2009,0,0,0,0,0,0 +FALSIFICATION,2009,0,0,0,0,0,0 +FALSIFY,2009,0,0,0,0,0,0 +FATALITIES,2009,0,0,0,0,0,0 +FAULTED,2009,0,0,0,0,0,0 +FAVORABLY,0,2009,0,0,0,0,0 +FAVORITES,0,2009,0,0,0,0,0 +FELONIOUS,2009,0,0,2009,0,0,0 +DAMAGES,2009,0,0,0,0,0,0 +DANGER,2009,0,0,0,0,0,0 +DEADLOCK,2009,0,0,0,0,0,0 +DEADWEIGHT,2009,0,0,0,0,0,0 +DEBARRED,2009,0,0,0,0,0,0 +DECEIT,2009,0,0,0,0,0,0 +DECEIVED,2009,0,0,0,0,0,0 +DECEPTIONS,2009,0,0,0,0,0,0 +DECLINE,2009,0,0,0,0,0,0 +DECREE,0,0,0,2009,0,0,0 +DEFACE,2009,0,0,0,0,0,0 +DEFALCATIONS,0,0,0,2009,0,0,0 +DEFAME,2009,0,0,0,0,0,0 +DEFAULT,2009,0,0,0,0,0,0 +DEFEASANCE,0,0,0,2009,0,0,0 +DEFEASEMENT,0,0,0,2014,0,0,0 +DEFEATED,2009,0,0,0,0,0,0 +DEFECTIVE,2009,0,0,0,0,0,0 +DEFENDABLE,0,0,0,2014,0,0,0 +DEFENDING,2009,0,0,0,0,0,0 +DEFERENCE,0,0,0,2009,0,0,0 +DEFICIT,2009,0,0,0,0,0,0 +DEFRAUD,2009,0,0,0,0,0,0 +DEFUNCT,2009,0,0,0,0,0,0 +DEGRADED,2009,0,0,0,0,0,0 +DELAYED,2009,0,0,0,0,0,0 +DELEGATABLE,0,0,0,2011,0,0,0 +DELIBERATE,2009,0,0,0,0,0,0 +DELIGHTED,0,2009,0,0,0,0,0 +DELIGHTS,0,2009,0,0,0,0,0 +DELINQUENTLY,2009,0,0,0,0,0,0 +DELISTING,2009,0,0,0,0,0,0 +DEMISES,2009,0,0,0,0,0,0 +DEMOLISHES,2009,0,0,0,0,0,0 +DEMOTE,2009,0,0,0,0,0,0 +DEMOTION,2009,0,0,0,0,0,0 +DEMURRERS,0,0,0,2009,0,0,0 +DENIALS,2009,0,0,0,0,0,0 +DENIGRATED,2009,0,0,0,0,0,0 +DENY,2009,0,0,0,0,0,0 +DEPENDABLE,0,2009,0,0,0,0,0 +DEPENDED,0,0,2009,0,0,2009,0 +DEPENDENT,0,0,2009,0,0,0,2011 +DEPLETED,2009,0,0,0,0,0,0 +DEPLETIONS,2009,0,0,0,0,0,0 +DEPOSING,0,0,0,2009,0,0,0 +INVALID,2009,0,0,0,0,0,0 +INVALIDATING,2009,0,0,0,0,0,0 +INVENTED,0,2009,0,0,0,0,0 +INVENTIVE,0,2009,0,0,0,0,0 +INVESTIGATE,2009,0,0,0,0,0,0 +INVESTIGATION,2009,0,0,0,0,0,0 +IRRECONCILABLE,2009,0,0,0,0,0,0 +IRREGULAR,2009,0,0,0,0,0,0 +IRREPARABLE,2009,0,0,0,0,0,0 +IRREVOCABLE,0,0,0,2009,0,0,2009 +JOINDER,0,0,0,2009,0,0,0 +JUDICIARY,0,0,0,2009,0,0,0 +JURISDICTIONAL,0,0,0,2009,0,0,0 +JURIST,0,0,0,2009,0,0,0 +JURY,0,0,0,2009,0,0,0 +JUSTIFIABLE,2009,0,0,0,0,0,0 +DILIGENTLY,0,2009,0,0,0,0,0 +DIMINISHING,2009,0,0,0,0,0,0 +DISADVANTAGE,2009,0,0,0,0,0,0 +DISAFFILIATION,2009,0,0,2009,0,0,0 +DISAFFIRMS,0,0,0,2011,0,0,0 +DISAGREEING,2009,0,0,0,0,0,0 +DISALLOW,2009,0,0,0,0,0,0 +DISALLOWING,2009,0,0,0,0,0,0 +DISAPPEARANCES,2009,0,0,0,0,0,0 +DISAPPOINT,2009,0,0,0,0,0,0 +DISAPPOINTMENT,2009,0,0,0,0,0,0 +DISAPPROVALS,2009,0,0,0,0,0,0 +DISAPPROVING,2009,0,0,0,0,0,0 +DISASSOCIATIONS,2009,0,0,0,0,0,0 +DISASTROUSLY,2009,0,0,0,0,0,0 +DISAVOWING,2009,0,0,0,0,0,0 +GREATER,0,-2020,0,0,0,0,0 +GRIEVANCE,2009,0,0,0,0,0,0 +GUILTY,2009,0,0,0,0,0,0 +HAMPERED,2009,0,0,0,0,0,0 +HAPPILY,0,2009,0,0,0,0,0 +HARASSED,2009,0,0,0,0,0,0 +HARDSHIPS,2009,0,0,0,0,0,0 +HARMFULLY,2009,0,0,0,0,0,0 +HARSHER,2009,0,0,0,0,0,0 +HAZARD,2009,0,0,0,0,0,0 +NONJUDICIALLY,0,0,0,2011,0,0,0 +NONPERFORMANCE,2009,0,0,0,0,0,0 +HURT,2009,0,0,0,0,0,0 +DISFAVORED,2009,0,0,0,0,0,0 +DISGORGED,2009,0,0,0,0,0,0 +COMPLICATING,2009,0,0,0,0,0,0 +COMPLIMENTARY,0,2009,0,0,0,0,0 +COMPLY,0,0,0,0,0,0,2009 +MISSTATED,2009,0,0,0,0,0,0 +MISSTATING,2009,0,0,0,0,0,0 +MISTAKEN,2009,0,0,0,0,0,0 +MISTRIAL,2009,0,0,2009,0,0,0 +MISUNDERSTANDINGS,2009,0,0,0,0,0,0 +MISUSES,2009,0,0,0,0,0,0 +MONOPOLIZATION,2009,0,0,0,0,0,0 +MONOPOLIZING,2009,0,0,0,0,0,0 +MORATORIUMS,2009,0,0,0,0,0,0 +MOTIONS,0,0,0,2009,0,0,0 +SLOWDOWN,2009,0,0,0,0,0,0 +SLOWEST,2009,0,0,0,0,0,0 +SLUGGISH,2009,0,0,0,0,0,0 +SMOOTHING,0,2009,0,0,0,0,0 +DEPRESS,2009,0,0,0,0,0,0 +DEPRIVATION,2009,0,0,0,0,0,0 +DEPRIVING,2009,0,0,0,0,0,0 +DEROGATED,0,0,0,2009,0,0,0 +DEROGATIONS,0,0,0,2009,0,0,0 +DESIRED,0,2009,0,0,0,0,0 +DESTABILIZE,2009,0,0,0,0,0,0 +DESTROY,2009,0,0,0,0,0,0 +DESTRUCTION,2009,0,0,0,0,0,0 +DETAINER,0,0,0,2009,0,0,0 +DETERIORATE,2009,0,0,0,0,0,0 +DETERIORATION,2009,0,0,0,0,0,0 +DETERRENCES,2009,0,0,0,0,0,0 +DETERS,2009,0,0,0,0,0,0 +DETRIMENT,2009,0,0,0,0,0,0 +DEVALUE,2009,0,0,0,0,0,0 +DEVASTATE,2009,0,0,0,0,0,0 +DEVIATE,2009,0,2009,0,0,0,0 +DEVIATION,2009,0,2009,0,0,0,0 +DEVOLVED,2009,0,0,0,0,0,0 +DICTATED,0,0,0,0,0,0,2009 +DIFFERED,0,0,2009,0,0,0,0 +DIFFICULTIES,2009,0,0,0,0,0,0 +ASCENDANCY,0,0,0,2009,0,0,0 +ASSAULTED,2009,0,0,0,0,0,0 +ASSERTIONS,2009,0,0,0,0,0,0 +ASSUME,0,0,2009,0,0,0,0 +ASSUMPTION,0,0,2009,0,0,0,0 +ASSURES,0,2009,0,0,0,0,0 +ATTAINING,0,2009,0,0,0,0,0 +ATTEST,0,0,0,2009,0,0,0 +ATTESTING,0,0,0,2009,0,0,0 +ATTORNMENT,0,0,0,2009,0,0,0 +ATTRITION,2009,0,0,0,0,0,0 +BAIL,2009,0,0,2009,0,0,0 +BAILIFF,0,0,0,2009,0,0,0 +BALK,2009,0,0,0,0,0,0 +BANKRUPTCY,2009,0,0,0,0,0,0 +BANS,2009,0,0,0,0,0,0 +CLAIMABLE,0,0,0,2009,0,0,0 +CLAIMING,2009,0,0,0,0,0,0 +CLAWBACK,2009,0,0,0,0,0,0 +CLOSEOUT,2009,0,0,0,0,0,0 +CLOSURE,2009,0,0,0,0,0,0 +CODICIL,0,0,0,2009,0,0,0 +CODIFIED,0,0,0,2009,0,0,0 +COERCE,2009,0,0,0,0,0,0 +COERCION,2009,0,0,0,0,0,0 +COLLABORATING,0,2009,0,0,0,0,0 +COLLABORATOR,0,2009,0,0,0,0,0 +COLLAPSES,2009,0,0,0,0,0,0 +COLLUDE,2009,0,0,0,0,0,0 +COLLUSION,2009,0,0,2009,0,0,0 +POSING,2009,0,0,0,0,0,0 +POSSIBILITIES,0,0,2009,0,0,0,0 +POSTCLOSING,0,0,0,2011,0,0,0 +POSTPONE,2009,0,0,0,0,0,0 +POSTPONES,2009,0,0,0,0,0,0 +WRITEOFF,2009,0,0,0,0,0,0 +WRONGDOING,2009,0,0,0,0,0,0 +WRONGLY,2009,0,0,0,0,0,0 +HONORED,0,2009,0,0,0,0,0 +HOSTILITY,2009,0,0,0,0,0,0 +REASSESSED,0,0,2009,0,0,0,0 +REASSESSMENTS,2009,0,2009,0,0,0,0 +REASSIGNMENT,2009,0,0,0,0,0,0 +REBOUNDED,0,2009,0,0,0,0,0 +REBUTTABLE,0,0,0,2009,0,0,0 +REBUTTED,0,0,0,2009,0,0,0 +RECALCULATES,0,0,2009,0,0,0,0 +RECALL,2009,0,0,0,0,0,0 +RECEPTIVE,0,2009,0,0,0,0,0 +RECKLESS,2009,0,0,0,0,0,0 +RECONSIDERED,0,0,2009,0,0,0,0 +RECOUPABLE,0,0,0,2009,0,0,0 +RECOURSES,0,0,0,2009,0,0,0 +RECUSE,0,0,0,2011,0,0,0 +REDACT,2009,0,0,2009,0,0,0 +REDACTIONS,2009,0,0,2009,0,0,0 +REDRESS,2009,0,0,0,0,0,0 +REEXAMINATION,0,0,2009,0,0,0,0 +REFERENDUM,0,0,0,2009,0,0,0 +REFILES,0,0,0,2009,0,0,0 +REFRAINS,0,0,0,0,0,0,2009 +REFUSED,2009,0,0,0,0,0,0 +REGAINED,0,2009,0,0,0,0,0 +REGULATES,0,0,0,2009,0,0,0 +REGULATIVE,0,0,0,2009,0,0,0 +REHEAR,0,0,0,2009,0,0,0 +VARIABLE,0,0,2009,0,0,0,0 +VARIANCES,0,0,2009,0,0,0,0 +VARIATIONS,0,0,2009,0,0,0,0 +VARYING,0,0,2009,0,0,0,0 +VERDICTS,2009,0,0,2009,0,0,0 +VIATICAL,0,0,0,2011,0,0,0 +VIOLATE,2009,0,0,0,0,0,0 +VIOLATION,2009,0,0,0,0,0,0 +VIOLATORS,2009,0,0,0,0,0,0 +VITIATE,2009,0,0,0,0,0,0 +VITIATION,2009,0,0,0,0,0,0 +VOLATILE,2009,0,2009,0,0,0,0 +VULNERABILITY,2009,0,0,0,0,0,0 +WARNED,2009,0,0,0,0,0,0 +WARRANTEES,0,0,0,2011,0,0,0 +WASTING,2009,0,0,0,0,0,0 +WEAKENING,2009,0,0,0,0,0,0 +WEAKLY,2009,0,0,0,0,0,0 +DISHONESTY,2009,0,0,0,0,0,0 +DISHONORED,2009,0,0,0,0,0,0 +DISLOYAL,2009,0,0,0,0,0,0 +DISMALLY,2009,0,0,0,0,0,0 +DISMISSED,2009,0,0,0,0,0,0 +DISPARAGE,2009,0,0,0,0,0,0 +DISPARAGES,2009,0,0,0,0,0,0 +DISPARITY,2009,0,0,0,0,0,0 +DISPLACEMENTS,2009,0,0,0,0,0,0 +DISPOSITIVE,0,0,0,2009,0,0,0 +DISPOSSESSING,2009,0,0,0,0,0,0 +DISPROPORTIONAL,2009,0,0,0,0,0,0 +DISPUTED,2009,0,0,0,0,0,0 +DISQUALIFICATIONS,2009,0,0,0,0,0,0 +DISQUALIFYING,2009,0,0,0,0,0,0 +DISREGARDS,2009,0,0,0,0,0,0 +DISRUPTED,2009,0,0,0,0,0,0 +DISRUPTIVE,2009,0,0,0,0,0,0 +DISSENT,2009,0,0,0,0,0,0 +DISSENTING,2009,0,0,0,0,0,0 +DISSOLUTION,2009,0,0,0,0,0,0 +DISTINCTIVE,0,2009,0,0,0,0,0 +DISTORTED,2009,0,0,0,0,0,0 +DISTORTS,2009,0,0,0,0,0,0 +DISTRACTION,2009,0,0,0,0,0,0 +DISTRESS,2009,0,0,0,0,0,0 +DISTURB,2009,0,0,0,0,0,0 +DISTURBING,2009,0,0,0,0,0,0 +DIVERTED,2009,0,0,0,0,0,0 +DIVESTED,2009,0,0,0,0,0,0 +DIVESTMENT,2009,0,0,0,0,0,0 +DIVORCED,2009,0,0,0,0,0,0 +DIVULGING,2009,0,0,0,0,0,0 +DOCKETS,0,0,0,2009,0,0,0 +DOUBTFUL,2009,0,2009,0,0,0,0 +DOWNGRADES,2009,0,0,0,0,0,0 +DOWNSIZES,2009,0,0,0,0,0,0 +DOWNTIMES,2009,0,0,0,0,0,0 +DOWNWARDS,2009,0,0,0,0,0,0 +DRAWBACK,2009,0,0,0,0,0,0 +DROUGHT,2009,0,0,0,0,0,0 +DYSFUNCTION,2009,0,0,0,0,0,0 +EARMARKED,0,0,0,0,0,0,2009 +EASILY,0,2009,0,0,0,0,0 +EFFICIENCIES,0,2009,0,0,0,0,0 +EGREGIOUS,2009,0,0,0,0,0,0 +EMBARGOED,2009,0,0,0,0,0,0 +EMBARRASSED,2009,0,0,0,0,0,0 +EMBARRASSMENTS,2009,0,0,0,0,0,0 +EMBEZZLEMENTS,2009,0,0,0,0,0,0 +EMPOWER,0,2009,0,0,0,0,0 +ENABLE,0,2009,0,0,0,0,0 +ENCOURAGED,0,2009,0,0,0,0,0 +ENCROACH,2009,0,0,0,0,0,0 +ENCROACHMENT,2009,0,0,0,0,0,0 +ENCUMBERING,2009,0,0,2009,0,0,2009 +ENCUMBRANCERS,0,0,0,2011,0,0,0 +ENDANGERING,2009,0,0,0,0,0,0 +ENFORCEABILITY,0,0,0,2009,0,0,0 +ENHANCED,0,2009,0,0,0,0,0 +ENHANCING,0,2009,0,0,0,0,0 +ENJOINS,2009,0,0,0,0,0,0 +ENJOYED,0,2009,0,0,0,0,0 +ENTAIL,0,0,0,0,0,0,2009 +PECUNIARILY,0,0,0,2009,0,0,0 +PENALIZING,2009,0,0,0,0,0,0 +PERFECT,0,2009,0,0,0,0,0 +PERHAPS,0,0,2009,0,0,2009,0 +PERMISSIBLE,0,0,0,0,0,0,2009 +PERMITTEE,0,0,0,2011,0,0,0 +PERPETRATED,2009,0,0,2009,0,0,0 +PERSIST,2009,0,0,0,0,0,0 +PERSISTENTLY,2009,0,0,0,0,0,0 +PERVASIVE,2009,0,0,0,0,0,0 +PETITIONED,0,0,0,2009,0,0,0 +PETITIONS,0,0,0,2009,0,0,0 +PICKETING,2009,0,0,0,0,0,0 +HENCEFORWARD,0,0,0,2009,0,0,0 +HEREFOR,0,0,0,2009,0,0,0 +HEREINABOVE,0,0,0,2009,0,0,0 +HEREOF,0,0,0,2009,0,0,0 +HEREUNDER,0,0,0,2009,0,0,0 +HEREWITHIN,0,0,0,2014,0,0,0 +HINDERED,2009,0,0,0,0,0,0 +HINDRANCES,2009,0,0,0,0,0,0 +WHENSOEVER,0,0,0,2009,0,0,0 +WHEREBY,0,0,0,2009,0,0,0 +WHEREON,0,0,0,2009,0,0,0 +WHEREWITH,0,0,0,2009,0,0,0 +WHOSOEVER,0,0,0,2009,0,0,0 +WILLFULLY,2009,0,0,2009,0,0,0 +WINNERS,0,2009,0,0,0,0,0 +FLUCTUATING,0,0,2009,0,0,0,0 +FORBEAR,0,0,0,2009,0,0,0 +FORBEARS,0,0,0,2009,0,0,0 +FORBIDS,2009,0,0,0,0,0,2009 +FOREBEAR,0,0,0,2009,0,0,0 +FORECLOSED,2009,0,0,0,0,0,0 +FORECLOSURES,2009,0,0,0,0,0,0 +FORESTALL,2009,0,0,0,0,0,0 +FORFEIT,2009,0,0,0,0,0,0 +FORFEITING,2009,0,0,0,0,0,0 +FORGERS,2009,0,0,0,0,0,0 +FRAUD,2009,0,0,0,0,0,0 +FRAUDULENTLY,2009,0,0,0,0,0,0 +FRUSTRATE,2009,0,0,0,0,0,0 +FRUSTRATINGLY,2009,0,0,0,0,0,0 +FUGITIVES,2009,0,0,2009,0,0,0 +GAINING,0,2009,0,0,0,0,0 +GRANTORS,0,0,0,2009,0,0,0 +DISGRACE,2009,0,0,0,0,0,0 +LITIGANTS,2009,0,0,2009,0,0,0 +LITIGATING,2009,0,0,2009,0,0,0 +LITIGATORS,0,0,0,2009,0,0,0 +LACK,2009,0,0,0,0,0,0 +LACKS,2009,0,0,0,0,0,0 +LAGS,2009,0,0,0,0,0,0 +LAPSING,2009,0,0,0,0,0,0 +LAWFUL,0,0,0,2009,0,0,0 +LAWMAKING,0,0,0,2009,0,0,0 +LAWYER,0,0,0,2009,0,0,0 +LEADERSHIP,0,2009,0,0,0,0,0 +LEGALITY,0,0,0,2009,0,0,0 +LEGALIZED,0,0,0,2009,0,0,0 +LEGALS,0,0,0,2009,0,0,0 +FLAW,2009,0,0,0,0,0,0 +NONPRODUCTIVE,2009,0,0,0,0,0,0 +LIQUIDATED,2009,0,0,0,0,0,0 +LIQUIDATIONS,2009,0,0,0,0,0,0 +BENEFICIATED,0,0,0,2014,0,0,0 +BENEFITING,0,2009,0,0,0,0,0 +BETTER,0,2009,0,0,0,0,0 +POPULARITY,0,2009,0,0,0,0,0 +REINTERPRET,0,0,2009,0,0,0,0 +REINTERPRETING,0,0,2009,0,0,0,0 +REJECTING,2009,0,0,0,0,0,0 +RELEASEES,0,0,0,2011,0,0,0 +RELINQUISHING,2009,0,0,0,0,0,0 +RELUCTANT,2009,0,0,0,0,0,0 +REMANDS,0,0,0,2009,0,0,0 +REMEDIATION,0,0,0,2009,0,0,0 +RENEGOTIATE,2009,0,0,0,0,0,0 +RENEGOTIATION,2009,0,0,0,0,0,0 +RENOUNCEMENT,2009,0,0,0,0,0,0 +REPARATION,2009,0,0,0,0,0,0 +REPOSSESSED,2009,0,0,0,0,0,0 +REPOSSESSIONS,2009,0,0,0,0,0,0 +TROUBLE,2009,0,0,0,0,0,0 +TURMOIL,2009,0,0,0,0,0,0 +UNACCOUNTED,2009,0,0,0,0,0,0 +UNAPPEALABLE,0,0,0,2009,0,0,0 +UNAUTHORIZED,2009,0,0,0,0,0,0 +UNAVOIDABLY,2009,0,0,0,0,0,0 +UNCERTAINTIES,0,0,2009,0,0,0,0 +UNCOLLECTED,2009,0,0,0,0,0,0 +UNCOMPETITIVE,2009,0,0,0,0,0,0 +UNCONSCIONABLE,2009,0,0,0,0,0,0 +UNCONSTITUTIONALLY,0,0,0,2009,0,0,0 +UNCONTROLLED,2009,0,0,0,0,0,0 +UNCOVERING,2009,0,0,0,0,0,0 +UNDEFINED,0,0,2009,0,0,0,0 +UNDERCUT,2009,0,0,0,0,0,0 +UNDERESTIMATED,2009,0,0,0,0,0,0 +UNDERFUNDED,2009,0,0,0,0,0,0 +UNDERMINES,2009,0,0,0,0,0,0 +UNDERPAYMENTS,2009,0,0,0,0,0,0 +UNDERPERFORMED,2011,0,0,0,0,0,0 +UNDERPRODUCTION,2009,0,0,0,0,0,0 +UNDERSTATEMENT,2009,0,0,0,0,0,0 +UNDERUTILIZATION,2009,0,0,0,0,0,0 +UNDESIRED,2009,0,0,0,0,0,0 +UNDETERMINED,2009,0,2009,0,0,0,0 +UNDOCUMENTED,2009,0,2009,0,0,0,0 +UNECONOMIC,2009,0,0,0,0,0,0 +UNEMPLOYMENT,2009,0,0,0,0,0,0 +UNENFORCEABLE,0,0,0,2009,0,0,0 +UNETHICALLY,2009,0,0,0,0,0,0 +UNFAIR,2009,0,0,0,0,0,0 +UNFAVORABILITY,2014,0,0,0,0,0,0 +UNFEASIBLE,2009,0,0,0,0,0,0 +UNFORESEEABLE,2009,0,0,0,0,0,0 +UNFORTUNATELY,2009,0,0,0,0,0,0 +UNFUNDED,2009,0,0,0,0,0,0 +UNIDENTIFIED,0,0,2009,0,0,0,0 +UNINTENTIONALLY,2009,0,0,0,0,0,0 +UNJUSTIFIED,2009,0,0,0,0,0,0 +UNKNOWN,0,0,2009,0,0,0,0 +UNLAWFULNESS,0,0,0,2009,0,0,0 +UNMATCHED,0,2009,0,0,0,0,0 +UNNECESSARY,2009,0,0,0,0,0,0 +UNOCCUPIED,2009,0,0,0,0,0,0 +UNPLANNED,2009,0,2009,0,0,0,0 +UNPREDICTABLY,2009,0,2009,0,0,0,0 +UNPROFITABLE,2009,0,0,0,0,0,0 +UNQUANTIFIABLE,0,0,2011,0,0,0,0 +UNREASONABLENESS,2009,0,0,0,0,0,0 +UNRECOVERABLE,2009,0,0,0,0,0,0 +UNREMEDIATED,0,0,0,2011,0,0,0 +UNREST,2009,0,0,0,0,0,0 +UNSATISFACTORY,2009,0,0,0,0,0,0 +UNSEASONABLE,0,0,2009,0,0,0,0 +UNSOLD,2009,0,0,0,0,0,0 +UNSTABILIZED,2014,0,0,0,0,0,0 +UNSUCCESSFUL,2009,0,0,0,0,0,0 +UNSUITABLY,2009,0,0,0,0,0,0 +UNSUSPECTED,2009,0,0,0,0,0,0 +UNTESTED,0,0,2009,0,0,0,0 +UNTRUTH,2009,0,0,0,0,0,0 +UNTRUTHS,2009,0,0,0,0,0,0 +UNWANTED,2009,0,0,0,0,0,0 +UNWILLINGNESS,2009,0,0,0,0,0,0 +UPTURNS,0,2009,0,0,0,0,0 +USURP,2009,0,0,2009,0,0,0 +USURPS,2009,0,0,2009,0,0,0 +VAGUELY,0,0,2012,0,0,0,0 +VAGUEST,0,0,2012,0,0,0,0 +PLEA,2009,0,0,0,0,0,0 +PLEADINGS,2009,0,0,2009,0,0,0 +PLEASANTLY,0,2009,0,0,0,0,0 +PLEDGE,0,0,0,0,0,0,2009 +PLEDGES,0,0,0,0,0,0,2009 +PLENTIFUL,0,2009,0,0,0,0,0 +COMPELS,0,0,0,0,0,0,2009 +COMPLAINANTS,0,0,0,2009,0,0,0 +COMPLAINT,2009,0,0,0,0,0,0 +COMMIT,0,0,0,0,0,0,2009 +COMMITTED,0,0,0,0,0,0,2009 +LIMIT,0,0,0,0,0,0,2009 +LIMITS,0,0,0,0,0,0,2009 +RESTRAINED,0,0,0,0,0,0,2009 +RESTRAINTS,0,0,0,0,0,0,2009 +RESTRICTION,0,0,0,0,0,0,2009 +RESTRICTIVENESS,0,0,0,0,0,0,2009 +RESTRUCTURES,2009,0,0,0,0,0,0 +RETALIATED,2009,0,0,0,0,0,0 +RETALIATIONS,2009,0,0,0,0,0,0 +PRECAUTIONARY,0,0,2009,0,0,0,0 +PRECIPITOUSLY,2009,0,0,0,0,0,0 +PRECLUDING,2009,0,0,0,0,0,2009 +PREDECEASE,0,0,0,2009,0,0,0 +PREDICT,0,0,2009,0,0,0,0 +PREDICTION,0,0,2009,0,0,0,0 +PREDICTORS,0,0,2009,0,0,0,0 +PREHEARING,0,0,0,2011,0,0,0 +PREJUDICIAL,2009,0,0,2009,0,0,0 +PREMATURE,2009,0,0,0,0,0,0 +PREPETITION,0,0,0,2009,0,0,0 +PRESTIGIOUS,0,2009,0,0,0,0,0 +PRESUMES,0,0,2009,0,0,0,0 +PRESUMPTIVELY,0,0,0,2009,0,0,0 +PREVENTING,2009,0,0,0,0,0,2009 +PRIVITY,0,0,0,2011,0,0,0 +PROBABILITIES,0,0,2009,0,0,0,0 +PROBATE,0,0,0,2009,0,0,0 +PROBATION,0,0,0,2009,0,0,0 +PROBATIONERS,0,0,0,2009,0,0,0 +PROBLEMATICAL,2009,0,0,0,0,0,0 +PROFICIENTLY,0,2009,0,0,0,0,0 +PROGRESS,0,2009,0,0,0,0,0 +PROHIBIT,0,0,0,0,0,0,2009 +PROHIBITIONS,0,0,0,0,0,0,2009 +PROHIBITS,0,0,0,0,0,0,2009 +PROLONGED,2009,0,0,0,0,0,0 +PROMULGATED,0,0,0,2009,0,0,0 +PROMULGATIONS,0,0,0,2009,0,0,0 +PRORATA,0,0,0,2009,0,0,0 +PROSECUTES,2009,0,0,2009,0,0,0 +PROSECUTOR,0,0,0,2009,0,0,0 +PROSPERING,0,2009,0,0,0,0,0 +PROTEST,2009,0,0,0,0,0,0 +PROTESTING,2009,0,0,0,0,0,0 +PROTRACTED,2009,0,0,0,0,0,0 +PROVISOS,0,0,0,2009,0,0,0 +PROVOKING,2009,0,0,0,0,0,0 +PUNISHING,2009,0,0,0,0,0,0 +PURPORT,2009,0,0,0,0,0,0 +PURPORTS,2009,0,0,0,0,0,0 +QUESTIONED,2009,0,0,0,0,0,0 +QUITCLAIM,0,0,0,2009,0,0,0 +RACKETEERING,2009,0,0,0,0,0,0 +RANDOMIZES,0,0,2009,0,0,0,0 +RATA,0,0,0,2009,0,0,0 +RATIONALIZATIONS,2009,0,0,0,0,0,0 +RATIONALIZING,2009,0,0,0,0,0,0 +ABANDON,2009,0,0,0,0,0,0 +ABANDONMENTS,2009,0,0,0,0,0,0 +ABDICATING,2009,0,0,0,0,0,0 +ABERRATION,2009,0,0,0,0,0,0 +ABEYANCE,0,0,2009,0,0,0,0 +ABLE,0,2009,0,0,0,0,0 +ABNORMALLY,2009,0,0,0,0,0,0 +ABOLISHING,2009,0,0,0,0,0,0 +ABROGATES,2009,0,0,2009,0,0,0 +ABRUPT,2009,0,0,0,0,0,0 +ABSENCES,2009,0,0,0,0,0,0 +ABSOLVES,0,0,0,2009,0,0,0 +ABUSE,2009,0,0,0,0,0,0 +ABUSIVE,2009,0,0,0,0,0,0 +ACCESSIONS,0,0,0,2009,0,0,0 +ACCIDENTS,2009,0,0,0,0,0,0 +ACCOMPLISHES,0,2009,0,0,0,0,0 +ACCUSATION,2009,0,0,0,0,0,0 +ACCUSES,2009,0,0,0,0,0,0 +ACHIEVEMENT,0,2009,0,0,0,0,0 +ACQUIESCE,2009,0,0,0,0,0,0 +ACQUIREES,0,0,0,2011,0,0,0 +ACQUITTAL,2009,0,0,2009,0,0,0 +ACQUITTED,2009,0,0,2009,0,0,0 +ADJOURN,0,0,0,2009,0,0,0 +ADJOURNMENTS,0,0,0,2009,0,0,0 +ADJUDGES,0,0,0,2009,0,0,0 +ADJUDICATES,0,0,0,2009,0,0,0 +ADJUDICATIVE,0,0,0,2009,0,0,0 +ADMISSIBILITY,0,0,0,2009,0,0,0 +ADMISSIONS,0,0,0,2009,0,0,0 +ADULTERATION,2009,0,0,0,0,0,0 +ADVANCES,0,2009,0,0,0,0,0 +ADVANTAGEOUS,0,2009,0,0,0,0,0 +ADVERSARIES,2009,0,0,0,0,0,0 +ADVERSITIES,2009,0,0,0,0,0,0 +AFFIRMANCE,0,0,0,2011,0,0,0 +AFORESAID,0,0,0,2009,0,0,0 +AGAINST,2009,0,0,0,0,0,0 +AGGRAVATING,2009,0,0,0,0,0,0 +ALERTED,2009,0,0,0,0,0,0 +ALIENATES,2009,0,0,0,0,0,0 +ALLEGATION,2009,0,0,2009,0,0,0 +ALLEGEDLY,2009,0,0,2009,0,0,0 +ALLIANCES,0,2009,0,0,0,0,0 +ALWAYS,0,0,0,0,2009,0,0 +AMEND,0,0,0,2009,0,0,0 +AMENDING,0,0,0,2009,0,0,0 +ANNOY,2009,0,0,0,0,0,0 +ANNOYING,2009,0,0,0,0,0,0 +ANNULLING,2009,0,0,0,0,0,0 +ANOMALIES,2009,0,2009,0,0,0,0 +ANTECEDENT,0,0,0,2009,0,0,0 +ANTICIPATES,0,0,2009,0,0,0,0 +ANTICOMPETITIVE,2009,0,0,0,0,0,0 +APPARENT,0,0,2009,0,0,0,0 +APPEALED,0,0,0,2009,0,0,0 +APPEARED,0,0,2009,0,0,2009,0 +APPELLANTS,0,0,0,2009,0,0,0 +APPROXIMATE,0,0,2009,0,0,0,0 +APPROXIMATING,0,0,2009,0,0,0,0 +APPURTENANCES,0,0,0,2009,0,0,0 +ARBITRARILY,0,0,2009,0,0,0,0 +ARBITRATED,0,0,0,2009,0,0,0 +ARBITRATIONAL,0,0,0,2011,0,0,0 +ARBITRATORS,0,0,0,2009,0,0,0 +ARGUMENT,2009,0,0,0,0,0,0 +ARREARAGES,2009,0,0,2009,0,0,0 +ARRESTS,2009,0,0,0,0,0,0 +RETROCEDED,0,0,0,2011,0,0,0 +BOLSTERING,0,2009,0,0,0,0,0 +BOOM,0,2009,0,0,0,0,0 +BOTTLENECK,2009,0,0,0,0,0,0 +BOYCOTT,2009,0,0,0,0,0,0 +BREACH,2009,0,0,2009,0,0,0 +BREAK,2009,0,0,0,0,0,0 +BREAKDOWNS,2009,0,0,0,0,0,0 +BREAKTHROUGHS,0,2009,0,0,0,0,0 +BRIBERY,2009,0,0,0,0,0,0 +BRILLIANT,0,2009,0,0,0,0,0 +BURDENING,2009,0,0,0,0,0,0 +CALAMITIES,2009,0,0,0,0,0,0 +CANCELED,2009,0,0,0,0,0,0 +CANCELLED,2009,0,0,0,0,0,0 +CARELESSLY,2009,0,0,0,0,0,0 +CATASTROPHES,2009,0,0,0,0,0,0 +CAUTIONARY,2009,0,0,0,0,0,0 +CAUTIOUS,0,0,2009,0,0,0,0 +CEASED,2009,0,0,0,0,0,0 +CEDANTS,0,0,0,2012,0,0,0 +CENSURING,2009,0,0,0,0,0,0 +CHALLENGED,2009,0,0,0,0,0,0 +CHARITABLE,0,2009,0,0,0,0,0 +CIRCUMVENT,2009,0,0,0,0,0,0 +CIRCUMVENTIONS,2009,0,0,0,0,0,0 +SHORTAGE,2009,0,0,0,0,0,0 +SHRINKAGE,2009,0,0,0,0,0,0 +SHUTDOWNS,2009,0,0,0,0,0,0 +SLANDERED,2009,0,0,0,0,0,0 +REPUDIATES,2009,0,0,0,0,0,0 +REQUESTER,0,0,0,2011,0,0,0 +REQUIREMENT,0,0,0,0,0,0,2009 +REREGULATION,0,0,0,2011,0,0,0 +RESCINDS,0,0,0,2009,0,0,0 +RESIGNATION,2009,0,0,0,0,0,0 +RESIGNS,2009,0,0,0,0,0,0 +RESTATEMENT,2009,0,0,0,0,0,0 +RESTITUTIONARY,0,0,0,2011,0,0,0 +NOTARIAL,0,0,0,2009,0,0,0 +NOTARIZE,0,0,0,2009,0,0,0 +NOTWITHSTANDING,0,0,0,2009,0,0,0 +NULLIFICATION,2009,0,0,2009,0,0,0 +NULLIFY,2009,0,0,2009,0,0,0 +OBJECTED,2009,0,0,0,0,0,0 +OBJECTIONABLY,2009,0,0,0,0,0,0 +OBLIGATES,0,0,0,0,0,0,2009 +OBLIGATORY,0,0,0,0,0,0,2009 +OBLIGEES,0,0,0,2011,0,0,0 +OBSCENE,2009,0,0,0,0,0,0 +OBSTACLE,2009,0,0,0,0,0,0 +OBSTRUCTING,2009,0,0,0,0,0,0 +OFFENCE,2009,0,0,0,0,0,0 +OFFENDER,2009,0,0,0,0,0,0 +OFFENSE,0,0,0,2009,0,0,0 +OFFERORS,0,0,0,2011,0,0,0 +OMITS,2009,0,0,0,0,0,0 +OPPORTUNISTIC,2009,0,0,0,0,0,0 +OPPOSE,2009,0,0,0,0,0,0 +OPPOSITION,2009,0,0,0,0,0,0 +OPTIONEES,0,0,0,2009,0,0,0 +OUTDATED,2009,0,0,0,0,0,0 +OUTPERFORMING,0,2009,0,0,0,0,0 +OVERBUILD,2009,0,0,0,0,0,0 +OVERBURDEN,2009,0,0,0,0,0,0 +OVERCAPACITY,2009,0,0,0,0,0,0 +OVERCHARGING,2009,0,0,0,0,0,0 +OVERDUE,2009,0,0,0,0,0,0 +OVERESTIMATING,2009,0,0,0,0,0,0 +OVERLOADED,2009,0,0,0,0,0,0 +OVERLOOKED,2009,0,0,0,0,0,0 +OVERPAYMENT,2009,0,0,0,0,0,0 +OVERPRODUCING,2009,0,0,0,0,0,0 +OVERRULES,0,0,0,2009,0,0,0 +OVERRUNS,2009,0,0,0,0,0,0 +OVERSHADOWS,2009,0,0,0,0,0,0 +OVERSTATEMENTS,2009,0,0,0,0,0,0 +OVERSUPPLIES,2009,0,0,0,0,0,0 +OVERTURN,2009,0,0,0,0,0,0 +OVERVALUE,2009,0,0,0,0,0,0 +PANICS,2009,0,0,0,0,0,0 +NECESSITATE,0,0,0,0,0,0,2009 +NEGATIVE,2009,0,0,0,0,0,0 +NEGLECTED,2009,0,0,0,0,0,0 +NEGLIGENCE,2009,0,0,0,0,0,0 +NEVER,0,0,0,0,2009,0,0 +BARRIERS,2009,0,0,0,0,0,0 +BELIEVED,0,0,2009,0,0,0,0 +IDLING,2009,0,0,0,0,0,0 +IGNORING,2009,0,0,0,0,0,0 +ILLEGALITY,2009,0,0,0,0,0,0 +ILLICITLY,2009,0,0,0,0,0,0 +IMBALANCES,2009,0,0,0,0,0,0 +IMPAIR,2009,0,0,0,0,0,2011 +IMPAIRMENTS,2009,0,0,0,0,0,2011 +IMPEDE,2009,0,0,0,0,0,0 +IMPEDIMENTS,2009,0,0,0,0,0,0 +IMPERFECTION,2009,0,0,0,0,0,0 +IMPLEADED,0,0,0,2009,0,0,0 +IMPLICATING,2009,0,0,0,0,0,0 +IMPOSING,0,0,0,0,0,0,2009 +IMPOSSIBLE,2009,0,0,0,0,0,0 +IMPOUNDS,2009,0,0,0,0,0,0 +IMPRACTICALITY,2009,0,0,0,0,0,0 +IMPRESS,0,2009,0,0,0,0,0 +IMPRESSIVE,0,2009,0,0,0,0,0 +IMPROBABLE,0,0,2009,0,0,0,0 +IMPROPRIETY,2009,0,0,0,0,0,0 +IMPROVEMENTS,0,2009,0,0,0,0,0 +IMPRUDENTLY,2009,0,0,0,0,0,0 +INACCURACY,2009,0,0,0,0,0,0 +INACTIONS,2009,0,0,0,0,0,0 +INACTIVATING,2009,0,0,0,0,0,0 +INADEQUACIES,2009,0,0,0,0,0,0 +INADVERTENT,2009,0,0,0,0,0,0 +INAPPROPRIATE,2009,0,0,0,0,0,0 +INCAPABLE,2009,0,0,0,0,0,0 +INCARCERATED,2009,0,0,2009,0,0,0 +INCARCERATIONS,2009,0,0,2009,0,0,0 +INCIDENT,2009,0,0,0,0,0,0 +INCOMPATIBLE,2009,0,0,0,0,0,0 +INCOMPETENTLY,2009,0,0,0,0,0,0 +INCOMPLETENESS,2009,0,2009,0,0,0,0 +INCONSISTENT,2009,0,0,0,0,0,0 +INCONVENIENCE,2009,0,0,0,0,0,0 +INCORRECTLY,2009,0,0,0,0,0,0 +INDEBTED,0,0,0,0,0,0,2009 +INDEFEASIBLY,2009,0,0,0,0,0,0 +INDEMNIFIABLE,0,0,0,2009,0,0,0 +INDEMNIFIES,0,0,0,2009,0,0,0 +INDEMNITEES,0,0,0,2009,0,0,0 +INDEMNITY,0,0,0,2009,0,0,0 +INDICTABLE,2009,0,0,2009,0,0,0 +INDICTMENTS,2009,0,0,2009,0,0,0 +INEFFECTIVENESS,2009,0,0,0,0,0,0 +INEFFICIENTLY,2009,0,0,0,0,0,0 +INEQUITABLY,2009,0,0,0,0,0,0 +INEXACT,0,0,2009,0,0,0,0 +INFERIOR,2009,0,0,0,0,0,0 +INFORMATIVE,0,2009,0,0,0,0,0 +INFRINGED,2009,0,0,0,0,0,0 +INFRINGES,2009,0,0,0,0,0,0 +INHIBITED,2009,0,0,0,0,0,2009 +INJUNCTION,2009,0,0,2009,0,0,0 +INJURED,2009,0,0,0,0,0,0 +INJURIOUS,2009,0,0,0,0,0,0 +INNOVATES,0,2009,0,0,0,0,0 +INNOVATIVE,0,2009,0,0,0,0,0 +INORDINATE,2009,0,0,0,0,0,0 +INSENSITIVE,2009,0,0,0,0,0,0 +INSISTENCE,0,0,0,0,0,0,2009 +INSOLVENCIES,2009,0,0,0,0,0,0 +INSPIRATIONAL,0,2009,0,0,0,0,0 +INSUFFICIENCY,2009,0,0,0,0,0,0 +INSURRECTIONS,2009,0,0,0,0,0,0 +INTENTIONAL,2009,0,0,0,0,0,0 +INTERFERENCES,2009,0,0,0,0,0,0 +INTERMITTENT,2009,0,0,0,0,0,0 +INTERPOSED,0,0,0,2009,0,0,0 +INTERPOSITIONS,0,0,0,2009,0,0,0 +INTERROGATING,0,0,0,2009,0,0,0 +INTERROGATORIES,0,0,0,2009,0,0,0 +INTERRUPTED,2009,0,0,0,0,0,0 +INTERRUPTS,2009,0,0,0,0,0,0 +STABILITY,0,2009,0,0,0,0,0 +STABILIZED,0,2009,0,0,0,0,0 +STAGGERING,2009,0,0,0,0,0,0 +STAGNATES,2009,0,0,0,0,0,0 +STANDSTILLS,2009,0,0,0,0,0,0 +STATUTORY,0,0,0,2009,0,0,0 +STIPULATING,0,0,0,0,0,0,2009 +STOPPAGE,2009,0,0,0,0,0,0 +STOPS,2009,0,0,0,0,0,0 +STRAINS,2009,0,0,0,0,0,0 +STRENGTHENING,0,2009,0,0,0,0,0 +STRESSED,2009,0,0,0,0,0,0 +STRICT,0,0,0,0,0,0,2009 +STRINGENT,2009,0,0,0,0,0,0 +STRONGLY,0,0,0,0,2009,0,0 +SUBJECTED,2009,0,0,0,0,0,0 +SUBLEASEHOLD,0,0,0,2011,0,0,0 +SUBPARAGRAPH,0,0,0,2009,0,0,0 +SUBPOENAS,2009,0,0,2009,0,0,0 +SUBTRUST,0,0,0,2011,0,0,0 +SUCCEEDING,0,2009,0,0,0,0,0 +SUCCESSFUL,0,2009,0,0,0,0,0 +SUE,2009,0,0,2009,0,0,0 +SUFFERED,2009,0,0,0,0,0,0 +SUGGESTED,0,0,2009,0,0,0,0 +SUMMONED,2009,0,0,2009,0,0,0 +SUPERIOR,0,2009,0,0,0,0,0 +SUPERSEDES,0,0,0,2009,0,0,0 +SURPASS,0,2009,0,0,0,0,0 +SUSCEPTIBILITY,2009,0,2009,0,0,0,0 +SUSPECTS,2009,0,0,0,0,0,0 +SUSPENDS,2009,0,0,0,0,0,0 +SUSPICIONS,2009,0,0,0,0,0,0 +REVOCABILITY,0,0,0,2011,0,0,0 +REVOKED,2009,0,0,0,0,0,0 +REVOLUTIONIZED,0,2009,0,0,0,0,0 +REWARDED,0,2009,0,0,0,0,0 +RIDICULED,2009,0,0,0,0,0,0 +RISKED,0,0,2009,0,0,0,0 +RISKING,0,0,2009,0,0,0,0 +RULING,0,0,0,2009,0,0,0 +SACRIFICE,2009,0,0,0,0,0,0 +SACRIFICING,2009,0,0,0,0,0,0 +SATISFIED,0,2009,0,0,0,0,0 +SCANDALOUS,2009,0,0,0,0,0,0 +SCRUTINIZES,2009,0,0,0,0,0,0 +SEEMS,0,0,2009,0,0,0,0 +SEIZING,2009,0,0,0,0,0,0 +SENTENCING,2009,0,0,2009,0,0,0 +SERIOUSNESS,2009,0,0,0,0,0,0 +SETTLEMENTS,0,0,0,2009,0,0,0 +SEVERALLY,0,0,0,2009,0,0,0 +SEVERED,2009,0,0,0,0,0,0 +ENTHUSIASTICALLY,0,2009,0,0,0,0,0 +ERODED,2009,0,0,0,0,0,0 +ERRATIC,2009,0,0,0,0,0,0 +ERRONEOUS,2009,0,0,0,0,0,0 +ERRS,2009,0,0,0,0,0,0 +ESCALATING,2009,0,0,0,0,0,0 +ESCROW,0,0,0,0,0,0,2009 +ESTOPPEL,0,0,0,2011,0,0,0 +EVADING,2009,0,0,0,0,0,0 +EVICT,2009,0,0,0,0,0,0 +EVICTIONS,2009,0,0,0,0,0,0 +EXACERBATE,2009,0,0,0,0,0,0 +EXACERBATION,2009,0,0,0,0,0,0 +EXAGGERATES,2009,0,0,0,0,0,0 +EXCEEDANCES,0,0,0,2011,0,0,0 +EXCELLING,0,2009,0,0,0,0,0 +EXCESSIVE,2009,0,0,0,0,0,0 +EXCITEMENT,0,2009,0,0,0,0,0 +EXCLUSIVENESS,0,2009,0,0,0,0,0 +EXCULPATED,2009,0,0,2009,0,0,0 +EXCULPATIONS,2009,0,0,2009,0,0,0 +EXECUTORY,0,0,0,2009,0,0,0 +EXEMPLARY,0,2009,0,0,0,0,0 +EXONERATING,2009,0,0,0,0,0,0 +EXPLOITATION,2009,0,0,0,0,0,0 +EXPLOITING,2009,0,0,0,0,0,0 +EXPOSES,2009,0,0,0,0,0,0 +EXPROPRIATE,2009,0,0,0,0,0,0 +EXPROPRIATION,2009,0,0,0,0,0,0 +EXTENUATING,2009,0,0,0,0,0,0 +SOLVING,0,2009,0,0,0,0,0 +SOMEWHERE,0,0,2009,0,0,0,0 +LOSE,2009,0,0,0,0,0,0 +LOSSES,2009,0,0,0,0,0,0 +LUCRATIVE,0,2009,0,0,0,0,0 +MALFUNCTION,2009,0,0,0,0,0,0 +MALICE,2009,0,0,0,0,0,0 +MANDAMUS,0,0,0,2009,0,0,0 +MANDATING,0,0,0,0,0,0,2009 +MANIPULATED,2009,0,0,0,0,0,0 +MANIPULATIONS,2009,0,0,0,0,0,0 +MAY,0,0,2009,0,0,2009,0 +MEDIATES,0,0,0,2009,0,0,0 +MEDIATOR,0,0,0,2009,0,0,0 +MISAPPLICATION,2009,0,0,0,0,0,0 +MISAPPLY,2009,0,0,0,0,0,0 +MISAPPROPRIATES,2009,0,0,0,0,0,0 +MISBRANDED,2009,0,0,0,0,0,0 +MISCALCULATING,2009,0,0,0,0,0,0 +MISCHIEF,2009,0,0,0,0,0,0 +MISCLASSIFY,2014,0,0,0,0,0,0 +MISDEMEANOR,2009,0,0,2009,0,0,0 +MISHANDLE,2009,0,0,0,0,0,0 +MISINFORM,2009,0,0,0,0,0,0 +MISINFORMS,2009,0,0,0,0,0,0 +MISINTERPRETED,2009,0,0,0,0,0,0 +MISJUDGED,2009,0,0,0,0,0,0 +MISJUDGMENTS,2009,0,0,0,0,0,0 +MISLABELLED,2009,0,0,0,0,0,0 +MISLEADINGLY,2009,0,0,0,0,0,0 +MISMANAGED,2009,0,0,0,0,0,0 +MISMATCH,2009,0,0,0,0,0,0 +MISPLACED,2009,0,0,0,0,0,0 +MISREPRESENT,2009,0,0,0,0,0,0 +MISREPRESENTING,2009,0,0,0,0,0,0 +MISSES,2009,0,0,0,0,0,0 +WORRY,2009,0,0,0,0,0,0 +WORSENED,2009,0,0,0,0,0,0 +WORTHLESS,2009,0,0,0,0,0,0 +TOLERATE,2009,0,0,0,0,0,0 +TOLERATION,2009,0,0,0,0,0,0 +TORTS,0,0,0,2009,0,0,0 +FICTITIOUS,2009,0,0,0,0,0,0 +FIRED,2009,0,0,0,0,0,0 +NONATTAINMENT,2009,0,0,0,0,0,0 +NONCOMPETITIVE,2009,0,0,0,0,0,0 +NONCOMPLYING,2009,0,0,0,0,0,0 +NONCONTINGENT,0,0,0,2011,0,0,0 +NONDISCLOSURE,2009,0,0,0,0,0,0 +NONFORFEITABLE,0,0,0,2009,0,0,0 +DISCLAIM,2009,0,0,0,0,0,0 +DISCLAIMING,2009,0,0,0,0,0,0 +DISCLOSES,2009,0,0,0,0,0,0 +DISCONTINUATION,2009,0,0,0,0,0,0 +DISCONTINUES,2009,0,0,0,0,0,0 +DISCOURAGES,2009,0,0,0,0,0,0 +DISCREDITING,2009,0,0,0,0,0,0 +SPECTACULAR,0,2009,0,0,0,0,0 +SPECULATES,0,0,2009,0,0,0,0 +SPECULATIVE,0,0,2009,0,0,0,0 +TRAGEDIES,2009,0,0,0,0,0,0 +TRANSFEROR,0,0,0,2009,0,0,0 +LEGISLATES,0,0,0,2009,0,0,0 +LEGISLATIVE,0,0,0,2009,0,0,0 +LEGISLATURE,0,0,0,2009,0,0,0 +LIBELOUS,0,0,0,2009,0,0,0 +LIENHOLDERS,0,0,0,2011,0,0,0 +COMPULSORY,0,0,0,0,0,0,2009 +CONCEDED,2009,0,0,0,0,0,0 +CONCEIVABLY,0,0,2009,0,0,0,0 +CONCILIATING,2009,0,0,0,0,0,0 +CONCLUSIVELY,0,2009,0,0,0,0,0 +CONDEMNED,2009,0,0,0,0,0,0 +CONDITIONAL,0,0,2009,0,0,0,0 +CONDUCIVE,0,2009,0,0,0,0,0 +CONFESSING,2009,0,0,0,0,0,0 +CONFINED,2009,0,0,0,0,0,2009 +CONFINING,2009,0,0,0,0,0,2009 +CONFISCATING,2009,0,0,0,0,0,0 +CONFLICT,2009,0,0,0,0,0,0 +CONFRONT,2009,0,0,0,0,0,0 +CONFRONTED,2009,0,0,0,0,0,0 +CONFUSED,2009,0,0,0,0,0,0 +CONFUSION,2009,0,2009,0,0,0,0 +CONSENTS,0,0,0,2009,0,0,0 +CONSPIRATOR,2009,0,0,0,0,0,0 +CONSPIRED,2009,0,0,0,0,0,0 +CONSTITUTIONAL,0,0,0,2009,0,0,0 +CONSTITUTIVE,0,0,0,2009,0,0,0 +CONSTRAINS,0,0,0,0,0,0,2009 +CONSTRUCTIVELY,0,2009,0,0,0,0,0 +CONSTRUING,0,0,0,2012,0,0,0 +CONTENDING,2009,0,0,0,0,0,0 +CONTENTIOUS,2009,0,0,0,0,0,0 +CONTESTED,2009,0,0,0,0,0,0 +CONTINGENT,0,0,2009,0,0,0,0 +CONTRACTED,0,0,0,2009,0,0,0 +CONTRACTILE,0,0,0,2009,0,0,0 +CONTRACTS,0,0,0,2009,0,0,0 +CONTRADICTED,2009,0,0,0,0,0,0 +CONTRADICTORY,2009,0,0,0,0,0,0 +CONTRAVENED,0,0,0,2009,0,0,0 +CONTRAVENTIONS,0,0,0,2009,0,0,0 +CONTROVERT,0,0,0,2009,0,0,0 +CONVEYANCE,0,0,0,2009,0,0,0 +CONVICTING,2009,0,0,2009,0,0,0 +CORRECTING,2009,0,0,0,0,0,0 +CORRUPT,2009,0,0,0,0,0,0 +CORRUPTIONS,2009,0,0,0,0,0,0 +COTERMINOUS,0,0,0,2009,0,0,0 +COUNSELLED,0,0,0,2009,0,0,0 +COUNTERCLAIMING,2009,0,0,0,0,0,0 +COUNTERFEITER,2009,0,0,0,0,0,0 +COUNTERMEASURE,2009,0,0,0,0,0,0 +COUNTERSUIT,0,0,0,2011,0,0,0 +COURTROOM,0,0,0,2009,0,0,0 +COVENANTING,0,0,0,0,0,0,2011 +CREATIVENESS,0,2009,0,0,0,0,0 +CRIMINAL,2009,0,0,2009,0,0,0 +CRIMINALLY,2009,0,0,2009,0,0,0 +CRITICAL,-2020,0,0,0,0,0,0 +CRITICIZE,2009,0,0,0,0,0,0 +CROSSCLAIM,0,0,0,2011,0,0,0 +CRUCIAL,2009,0,0,0,0,0,0 +CULPABLY,2009,0,0,0,0,0,0 +CURTAILING,2009,0,0,0,0,0,0 +CUT,2009,0,0,0,0,0,0 +CYBERATTACKS,2014,0,0,0,0,0,0 +CYBERCRIMINAL,2014,0,0,0,0,0,0 +TAINT,2009,0,0,0,0,0,0 +TAMPERED,2009,0,0,0,0,0,0 +TENTATIVE,0,0,2009,0,0,0,0 +TERMINATED,2009,0,0,0,0,0,0 +TERMINATIONS,2009,0,0,0,0,0,0 +TESTIFYING,2009,0,0,2009,0,0,0 +THENCEFORWARD,0,0,0,2009,0,0,0 +THEREIN,0,0,0,2009,0,0,0 +THEREOVER,0,0,0,2011,0,0,0 +THEREUNDER,0,0,0,2009,0,0,0 +THREAT,2009,0,0,0,0,0,0 +THREATENS,2009,0,0,0,0,0,0 +FACTO,0,0,0,2011,0,0,0 +FAILINGS,2009,0,0,0,0,0,0 +FALLOUT,2009,0,0,0,0,0,0 +FALSIFICATIONS,2009,0,0,0,0,0,0 +FALSIFYING,2009,0,0,0,0,0,0 +FATALITY,2009,0,0,0,0,0,0 +FAULTS,2009,0,0,0,0,0,0 +FAVORED,0,2009,0,0,0,0,0 +FEAR,2009,0,0,0,0,0,0 +FELONY,2009,0,0,2009,0,0,0 +DAMAGING,2009,0,0,0,0,0,0 +DANGEROUS,2009,0,0,0,0,0,0 +DEADLOCKED,2009,0,0,0,0,0,0 +DEADWEIGHTS,2009,0,0,0,0,0,0 +DECEASED,2009,0,0,0,0,0,0 +DECEITFUL,2009,0,0,0,0,0,0 +DECEIVES,2009,0,0,0,0,0,0 +DECEPTIVE,2009,0,0,0,0,0,0 +DECLINED,2009,0,0,0,0,0,0 +DECREED,0,0,0,2009,0,0,0 +DEFACED,2009,0,0,0,0,0,0 +DEFAMATION,2009,0,0,0,0,0,0 +DEFAMED,2009,0,0,0,0,0,0 +DEFAULTED,2009,0,0,0,0,0,0 +DEFEASANCES,0,0,0,2011,0,0,0 +DEFEASES,0,0,0,2014,0,0,0 +DEFEATING,2009,0,0,0,0,0,0 +DEFECTIVELY,0,0,0,2009,0,0,0 +DEFENDANT,2009,0,0,2009,0,0,0 +DEFENDS,2009,0,0,0,0,0,0 +DEFICIENCIES,2009,0,0,0,0,0,0 +DEFICITS,2009,0,0,0,0,0,0 +DEFRAUDED,2009,0,0,0,0,0,0 +DEGRADATION,2009,0,0,0,0,0,0 +DEGRADES,2009,0,0,0,0,0,0 +DELAYING,2009,0,0,0,0,0,0 +DELEGATEE,0,0,0,2011,0,0,0 +DELIBERATED,2009,0,0,0,0,0,0 +DELIGHTFUL,0,2009,0,0,0,0,0 +DELINQUENCIES,2009,0,0,0,0,0,0 +DELINQUENTS,2009,0,0,0,0,0,0 +DELISTS,2011,0,0,0,0,0,0 +DEMISING,2009,0,0,0,0,0,0 +DEMOLISHING,2009,0,0,0,0,0,0 +DEMOTED,2009,0,0,0,0,0,0 +DEMOTIONS,2009,0,0,0,0,0,0 +DEMURRING,0,0,0,2009,0,0,0 +DENIED,2009,0,0,0,0,0,0 +DENIGRATES,2009,0,0,0,0,0,0 +DENYING,2009,0,0,0,0,0,0 +DEPENDANCE,0,0,0,0,0,0,2011 +DEPENDENCE,0,0,2009,0,0,0,0 +DEPENDING,0,0,2009,0,0,2009,2011 +DEPLETES,2009,0,0,0,0,0,0 +DEPOSE,0,0,0,2009,0,0,0 +DEPOSITION,0,0,0,2009,0,0,0 +INVALIDATE,2009,0,0,0,0,0,0 +INVALIDATION,2009,0,0,0,0,0,0 +INVENTING,0,2009,0,0,0,0,0 +INVENTIVENESS,0,2009,0,0,0,0,0 +INVESTIGATED,2009,0,0,0,0,0,0 +INVESTIGATIONS,2009,0,0,0,0,0,0 +IRRECONCILABLY,2009,0,0,0,0,0,0 +IRREGULARITIES,2009,0,0,0,0,0,0 +IRREPARABLY,2009,0,0,0,0,0,0 +IRREVOCABLY,0,0,0,2009,0,0,2009 +JUDICIAL,0,0,0,2009,0,0,0 +JURIES,0,0,0,2009,0,0,0 +JURISDICTIONALLY,0,0,0,2011,0,0,0 +JURISTS,0,0,0,2009,0,0,0 +JURYMAN,0,0,0,2009,0,0,0 +KICKBACK,2009,0,0,0,0,0,0 +DIMINISH,2009,0,0,0,0,0,0 +DIMINUTION,2009,0,0,0,0,0,0 +DISADVANTAGED,2009,0,0,0,0,0,0 +DISAFFIRM,0,0,0,2009,0,0,0 +DISAGREE,2009,0,0,0,0,0,0 +DISAGREEMENT,2009,0,0,0,0,0,0 +DISALLOWANCE,2009,0,0,0,0,0,0 +DISALLOWS,2009,0,0,0,0,0,0 +DISAPPEARED,2009,0,0,0,0,0,0 +DISAPPOINTED,2009,0,0,0,0,0,0 +DISAPPOINTMENTS,2009,0,0,0,0,0,0 +DISAPPROVE,2009,0,0,0,0,0,0 +DISASSOCIATES,2009,0,0,0,0,0,0 +DISASTER,2009,0,0,0,0,0,0 +DISAVOW,2009,0,0,0,0,0,0 +DISAVOWS,2009,0,0,0,0,0,0 +GRATUITOUS,2009,0,0,0,0,0,0 +GREATEST,0,2009,0,0,0,0,0 +GRIEVANCES,2009,0,0,0,0,0,0 +HALT,2009,0,0,0,0,0,0 +HAMPERING,2009,0,0,0,0,0,0 +HAPPINESS,0,2009,0,0,0,0,0 +HARASSING,2009,0,0,0,0,0,0 +HARM,2009,0,0,0,0,0,0 +HARMING,2009,0,0,0,0,0,0 +HARSHEST,2009,0,0,0,0,0,0 +HAZARDOUS,2009,0,0,0,0,0,0 +NONINFRINGEMENT,0,0,0,2011,0,0,0 +NONJURISDICTIONAL,0,0,0,2011,0,0,0 +NONPERFORMANCES,2009,0,0,0,0,0,0 +HURTING,2009,0,0,0,0,0,0 +DISFAVORING,2009,0,0,0,0,0,0 +DISGORGEMENT,2009,0,0,0,0,0,0 +COMPLICATE,2009,0,0,0,0,0,0 +COMPLICATION,2009,0,0,0,0,0,0 +COMPLIMENTED,0,2009,0,0,0,0,0 +MISSTATEMENT,2009,0,0,0,0,0,0 +MISSTEP,2009,0,0,0,0,0,0 +MISTAKENLY,2009,0,0,0,0,0,0 +MISTRIALS,2009,0,0,2009,0,0,0 +MISUNDERSTOOD,2009,0,0,0,0,0,0 +MISUSING,2009,0,0,0,0,0,0 +MONOPOLIZE,2009,0,0,0,0,0,0 +MONOPOLY,2009,0,0,0,0,0,0 +MOREOVER,0,0,0,2009,0,0,0 +MUST,0,0,0,0,2009,0,0 +SLIPPAGE,2009,0,0,0,0,0,0 +SLOWDOWNS,2009,0,0,0,0,0,0 +SLOWING,2009,0,0,0,0,0,0 +SLUGGISHLY,2009,0,0,0,0,0,0 +SMOOTHLY,0,2009,0,0,0,0,0 +DEPRESSED,2009,0,0,0,0,0,0 +DEPRIVE,2009,0,0,0,0,0,0 +DERELICT,2009,0,0,0,0,0,0 +DEROGATES,0,0,0,2009,0,0,0 +DEROGATORY,2009,0,0,0,0,0,0 +DESIST,0,0,0,2009,0,0,0 +DESTABILIZED,2009,0,0,0,0,0,0 +DESTROYED,2009,0,0,0,0,0,0 +DESTRUCTIVE,2009,0,0,0,0,0,0 +DETENTION,2009,0,0,0,0,0,0 +DETERIORATED,2009,0,0,0,0,0,0 +DETERIORATIONS,2009,0,0,0,0,0,0 +DETERRENT,2009,0,0,0,0,0,0 +DETRACT,2009,0,0,0,0,0,0 +DETRIMENTAL,2009,0,0,0,0,0,0 +DEVALUED,2009,0,0,0,0,0,0 +DEVASTATED,2009,0,0,0,0,0,0 +DEVIATED,2009,0,2009,0,0,0,0 +DEVIATIONS,2009,0,2009,0,0,0,0 +DEVOLVES,2009,0,0,0,0,0,0 +DICTATES,0,0,0,0,0,0,2009 +DIFFERING,0,0,2009,0,0,0,0 +DIFFICULTLY,2009,0,0,0,0,0,0 +ASCENDANT,0,0,0,2009,0,0,0 +ASSAULTING,2009,0,0,0,0,0,0 +ASSIGNATION,0,0,0,2009,0,0,0 +ASSUMED,0,0,2009,0,0,0,0 +ASSUMPTIONS,0,0,2009,0,0,0,0 +ASSURING,0,2009,0,0,0,0,0 +ATTAINMENT,0,2009,0,0,0,0,0 +ATTESTATION,0,0,0,2009,0,0,0 +ATTORN,0,0,0,2011,0,0,0 +ATTORNS,0,0,0,2011,0,0,0 +AVERSELY,2011,0,0,0,0,0,0 +BAILED,0,0,0,2009,0,0,0 +BAILIFFS,0,0,0,2009,0,0,0 +BALKED,2009,0,0,0,0,0,0 +BANKRUPTED,2009,0,0,0,0,0,0 +CLAIMANT,0,0,0,2009,0,0,0 +CLAIMS,2009,0,0,2009,0,0,0 +CLAWBACKS,0,0,0,2014,0,0,0 +CLOSEOUTS,2009,0,0,0,0,0,0 +CLOSURES,2009,0,0,0,0,0,0 +CODICILS,0,0,0,2009,0,0,0 +CODIFIES,0,0,0,2009,0,0,0 +COERCED,2009,0,0,0,0,0,0 +COERCIVE,2009,0,0,0,0,0,0 +DISINCENTIVES,2009,0,0,0,0,0,0 +COLLABORATE,0,2009,0,0,0,0,0 +COLLABORATION,0,2009,0,0,0,0,0 +COLLABORATORS,0,2009,0,0,0,0,0 +COLLAPSING,2009,0,0,0,0,0,0 +COLLUDED,2009,0,0,0,0,0,0 +COLLUSIONS,2009,0,0,0,0,0,0 +POSITIVE,0,2009,0,0,0,0,0 +POSSIBILITY,0,0,2009,0,0,0,0 +POSTCLOSURE,0,0,0,2011,0,0,0 +POSTPONED,2009,0,0,0,0,0,0 +POSTPONING,2009,0,0,0,0,0,0 +WRIT,0,0,0,2009,0,0,0 +WRITEOFFS,2009,0,0,0,0,0,0 +WRONGDOINGS,2009,0,0,0,0,0,0 +HONORING,0,2009,0,0,0,0,0 +REASSESSES,0,0,2009,0,0,0,0 +REASSIGN,2009,0,0,0,0,0,0 +REASSIGNMENTS,2009,0,0,0,0,0,0 +REBOUNDING,0,2009,0,0,0,0,0 +REBUTTABLY,0,0,0,2011,0,0,0 +REBUTTING,0,0,0,2009,0,0,0 +RECALCULATING,0,0,2009,0,0,0,0 +RECALLED,2009,0,0,0,0,0,0 +RECESSION,2009,0,0,0,0,0,0 +RECKLESSLY,2009,0,0,0,0,0,0 +RECONSIDERING,0,0,2009,0,0,0,0 +RECOUPMENT,0,0,0,2009,0,0,0 +RECTIFICATION,0,0,0,2009,0,0,0 +RECUSED,0,0,0,2011,0,0,0 +REDACTED,2009,0,0,2009,0,0,0 +REDEFAULT,2014,0,0,0,0,0,0 +REDRESSED,2009,0,0,0,0,0,0 +REEXAMINE,0,0,2009,0,0,0,0 +REFERENDUMS,0,0,0,2009,0,0,0 +REFILING,0,0,0,2009,0,0,0 +REFUSAL,2009,0,0,0,0,0,0 +REFUSES,2009,0,0,0,0,0,0 +REGAINING,0,2009,0,0,0,0,0 +REGULATING,0,0,0,2009,0,0,0 +REGULATOR,0,0,0,2009,0,0,0 +REHEARD,0,0,0,2009,0,0,0 +VARIABLES,0,0,2009,0,0,0,0 +VARIANT,0,0,2009,0,0,0,0 +VARIED,0,0,2009,0,0,0,0 +VENDEE,0,0,0,2011,0,0,0 +VERSATILE,0,2009,0,0,0,0,0 +VIBRANCY,0,2009,0,0,0,0,0 +VIOLATED,2009,0,0,0,0,0,0 +VIOLATIONS,2009,0,0,0,0,0,0 +VIOLENCE,2009,0,0,0,0,0,0 +VITIATED,2009,0,0,0,0,0,0 +VOIDABLE,0,0,0,2009,0,0,0 +VOLATILITIES,0,0,2009,0,0,0,0 +VULNERABLE,2009,0,0,0,0,0,0 +WARNING,2009,0,0,0,0,0,0 +WARRANTOR,0,0,0,2011,0,0,0 +WEAK,2009,0,0,0,0,0,0 +WEAKENS,2009,0,0,0,0,0,0 +WEAKNESS,2009,0,0,0,0,0,0 +DISHONOR,2009,0,0,0,0,0,0 +DISHONORING,2009,0,0,0,0,0,0 +DISINTERESTED,2009,0,0,0,0,0,0 +DISLOYALLY,2009,0,0,0,0,0,0 +DISMISS,2009,0,0,0,0,0,0 +DISMISSES,2009,0,0,0,0,0,0 +DISPARAGED,2009,0,0,0,0,0,0 +DISPARAGING,2009,0,0,0,0,0,0 +DISPLACE,2009,0,0,0,0,0,0 +DISPLACES,2009,0,0,0,0,0,0 +DISPOSSESS,2009,0,0,0,0,0,0 +DISPOSSESSION,0,0,0,2009,0,0,0 +DISPROPORTIONATE,2009,0,0,0,0,0,0 +DISPUTES,2009,0,0,0,0,0,0 +DISQUALIFIED,2009,0,0,0,0,0,0 +DISREGARD,2009,0,0,0,0,0,0 +DISREPUTABLE,2009,0,0,0,0,0,0 +DISRUPTING,2009,0,0,0,0,0,0 +DISRUPTS,2009,0,0,0,0,0,0 +DISSENTED,2009,0,0,0,0,0,0 +DISSENTS,2009,0,0,0,0,0,0 +DISSOLUTIONS,2009,0,0,0,0,0,0 +DISTINCTIVELY,0,2009,0,0,0,0,0 +DISTORTING,2009,0,0,0,0,0,0 +DISTRACT,2009,0,0,0,0,0,0 +DISTRACTIONS,2009,0,0,0,0,0,0 +DISTRESSED,2009,0,0,0,0,0,0 +DISTURBANCE,2009,0,0,0,0,0,0 +DISTURBS,2009,0,0,0,0,0,0 +DIVERTING,2009,0,0,0,0,0,0 +DIVESTING,2009,0,0,0,0,0,0 +DIVESTMENTS,2009,0,0,0,0,0,0 +DIVULGE,2009,0,0,0,0,0,0 +DOCKET,0,0,0,2009,0,0,0 +DONEES,0,0,0,2011,0,0,0 +DOUBTS,2009,0,2009,0,0,0,0 +DOWNGRADING,2009,0,0,0,0,0,0 +DOWNSIZING,2009,0,0,0,0,0,0 +DOWNTURN,2009,0,0,0,0,0,0 +DRAG,2009,0,0,0,0,0,0 +DRAWBACKS,2009,0,0,0,0,0,0 +DROUGHTS,2009,0,0,0,0,0,0 +DYSFUNCTIONAL,2009,0,0,0,0,0,0 +EARMARKING,0,0,0,0,0,0,2009 +EASING,2009,0,0,0,0,0,0 +EFFICIENCY,0,2009,0,0,0,0,0 +EGREGIOUSLY,2009,0,0,0,0,0,0 +EMBARGOES,2009,0,0,0,0,0,0 +EMBARRASSES,2009,0,0,0,0,0,0 +EMBEZZLE,2009,0,0,0,0,0,0 +EMBEZZLER,2009,0,0,0,0,0,0 +EMPOWERED,0,2009,0,0,0,0,0 +ENABLED,0,2009,0,0,0,0,0 +ENCOURAGEMENT,0,2009,0,0,0,0,0 +ENCROACHED,2009,0,0,0,0,0,0 +ENCROACHMENTS,2009,0,0,0,0,0,0 +ENCUMBERS,2009,0,0,2009,0,0,2009 +ENCUMBRANCES,2009,0,0,2009,0,0,2009 +ENDANGERMENT,2009,0,0,0,0,0,0 +ENFORCEABLE,0,0,0,2009,0,0,0 +ENHANCEMENT,0,2009,0,0,0,0,0 +ENJOIN,2009,0,0,0,0,0,0 +ENJOY,0,2009,0,0,0,0,0 +ENJOYING,0,2009,0,0,0,0,0 +ENTAILED,0,0,0,0,0,0,2009 +PENALIZE,2009,0,0,0,0,0,0 +PENALTIES,2009,0,0,0,0,0,0 +PERFECTED,0,2009,0,0,0,0,0 +PERIL,2009,0,0,0,0,0,0 +PERMISSION,0,0,0,0,0,0,2009 +PERMITTEES,0,0,0,2011,0,0,0 +PERPETRATES,2009,0,0,2009,0,0,0 +PERSISTED,2009,0,0,0,0,0,0 +PERSISTING,2009,0,0,0,0,0,0 +PERVASIVELY,2009,0,0,0,0,0,0 +PETITIONER,0,0,0,2009,0,0,0 +PETTY,2009,0,0,0,0,0,0 +HEREAFTER,0,0,0,2009,0,0,0 +HEREFORE,0,0,0,2014,0,0,0 +HEREINAFTER,0,0,0,2009,0,0,0 +HEREON,0,0,0,2009,0,0,0 +HEREUNTO,0,0,0,2009,0,0,0 +HIDDEN,0,0,2009,0,0,0,0 +HINDERING,2009,0,0,0,0,0,0 +HINGES,0,0,2009,0,0,0,0 +WHEREABOUTS,0,0,0,2009,0,0,0 +WHEREFORE,0,0,0,2009,0,0,0 +WHERETO,0,0,0,2009,0,0,0 +WHISTLEBLOWERS,0,0,0,2011,0,0,0 +WILFUL,0,0,0,2009,0,0,0 +WILLFULNESS,0,0,0,2009,0,0,0 +WINNING,0,2009,0,0,0,0,0 +FLUCTUATE,0,0,2009,0,0,0,0 +FLUCTUATION,0,0,2009,0,0,0,0 +FORBEARANCE,0,0,0,2009,0,0,0 +FORBID,2009,0,0,0,0,0,2009 +FORCE,-2020,0,0,0,0,0,0 +FOREBEARANCE,0,0,0,2011,0,0,0 +FORECLOSES,2009,0,0,0,0,0,0 +FOREGO,2009,0,0,0,0,0,0 +FORESTALLED,2009,0,0,0,0,0,0 +FORFEITABILITY,0,0,0,2011,0,0,0 +FORFEITS,2009,0,0,0,0,0,0 +FORGERY,2009,0,0,0,0,0,0 +FRAUDS,2009,0,0,0,0,0,0 +FRIENDLY,0,2009,0,0,0,0,0 +FRUSTRATED,2009,0,0,0,0,0,0 +FRUSTRATION,2009,0,0,0,0,0,0 +FURTHERANCE,0,0,0,2009,0,0,0 +GAINS,0,2009,0,0,0,0,0 +DISGORGEMENTS,2009,0,0,0,0,0,0 +DISGRACEFUL,2009,0,0,0,0,0,0 +LITIGATE,2009,0,0,2009,0,0,0 +LITIGATION,2009,0,0,2009,0,0,0 +LITIGIOUS,0,0,0,2009,0,0,0 +LACKED,2009,0,0,0,0,0,0 +LAG,2009,0,0,0,0,0,0 +LAPSE,2009,0,0,0,0,0,0 +LATE,-2020,0,0,0,0,0,0 +LAWFULLY,0,0,0,2009,0,0,0 +LAWS,0,0,0,2009,0,0,0 +LAWYERS,0,0,0,2009,0,0,0 +LEADING,0,2009,0,0,0,0,0 +LEGALIZATION,0,0,0,2009,0,0,0 +LEGALIZES,0,0,0,2009,0,0,0 +LEGATEE,0,0,0,2009,0,0,0 +FLAWED,2009,0,0,0,0,0,0 +NONRECOVERABLE,2009,0,0,0,0,0,0 +LIQUIDATES,2009,0,0,0,0,0,0 +LIQUIDATOR,2009,0,0,0,0,0,0 +BENEFICIATION,0,0,0,2011,0,0,0 +BENEFITTED,0,2009,0,0,0,0,0 +POOR,2009,0,0,0,0,0,0 +REINTERPRETATION,0,0,2009,0,0,0,0 +REINTERPRETS,0,0,2009,0,0,0,0 +REJECTION,2009,0,0,0,0,0,0 +RELINQUISH,2009,0,0,0,0,0,0 +RELINQUISHMENT,2009,0,0,0,0,0,0 +REMAND,0,0,0,2009,0,0,0 +REMEDIATE,0,0,0,2009,0,0,0 +REMEDIATIONS,0,0,0,2009,0,0,0 +RENEGOTIATED,2009,0,0,0,0,0,0 +RENEGOTIATIONS,2009,0,0,0,0,0,0 +RENOUNCEMENTS,2009,0,0,0,0,0,0 +REPARATIONS,2009,0,0,0,0,0,0 +REPOSSESSES,2009,0,0,0,0,0,0 +REPRORATED,0,0,0,2018,0,0,0 +TROUBLED,2009,0,0,0,0,0,0 +UNABLE,2009,0,0,0,0,0,0 +UNAMBIGUOUSLY,0,0,0,0,2009,0,0 +UNAPPEALED,0,0,0,2011,0,0,0 +UNAVAILABILITY,2009,0,0,0,0,0,2011 +UNAWARE,2009,0,0,0,0,0,0 +UNCERTAINTY,0,0,2009,0,0,0,0 +UNCOLLECTIBILITY,2009,0,0,0,0,0,0 +UNCOMPLETED,2009,0,0,0,0,0,0 +UNCONSCIONABLY,2009,0,0,0,0,0,0 +UNCONTRACTED,0,0,0,2011,0,0,0 +UNCORRECTED,2009,0,0,0,0,0,0 +UNCOVERS,2009,0,0,0,0,0,0 +UNDELIVERABLE,2009,0,0,0,0,0,0 +UNDERCUTS,2009,0,0,0,0,0,0 +UNDERESTIMATES,2009,0,0,0,0,0,0 +UNDERINSURED,2009,0,0,0,0,0,0 +UNDERMINING,2009,0,0,0,0,0,0 +UNDERPAYS,2009,0,0,0,0,0,0 +UNDERPERFORMING,2009,0,0,0,0,0,0 +UNDERREPORTING,2011,0,0,0,0,0,0 +UNDERSTATEMENTS,2009,0,0,0,0,0,0 +UNDERUTILIZED,2009,0,0,0,0,0,0 +UNDETECTABLE,0,0,2009,0,0,0,0 +UNDISCHARGED,0,0,0,2009,0,0,0 +UNDOUBTEDLY,0,0,0,0,2009,0,0 +UNECONOMICAL,2009,0,0,0,0,0,0 +UNENCUMBER,0,0,0,2014,0,0,0 +UNEQUIVOCAL,0,0,0,0,2009,0,0 +UNEXCUSED,2009,0,0,0,0,0,0 +UNFAIRLY,2009,0,0,0,0,0,0 +UNFAVORABLE,2009,0,0,0,0,0,0 +UNFIT,2009,0,0,0,0,0,0 +UNFORESEEN,2009,0,0,0,0,0,0 +UNFOUNDED,2009,0,0,0,0,0,0 +UNGUARANTEED,0,0,2009,0,0,0,0 +UNINSURED,2009,0,0,0,0,0,0 +UNJUST,2009,0,0,0,0,0,0 +UNJUSTLY,2009,0,0,0,0,0,0 +UNKNOWNS,0,0,2009,0,0,0,0 +UNLICENSED,2009,0,0,0,0,0,0 +UNMERCHANTABLE,2011,0,0,0,0,0,0 +UNNEEDED,2009,0,0,0,0,0,0 +UNPAID,2009,0,0,0,0,0,0 +UNPOPULAR,2009,0,0,0,0,0,0 +UNPREDICTED,2011,0,2011,0,0,0,0 +UNPROVED,0,0,2009,0,0,0,0 +UNQUANTIFIED,0,0,2011,0,0,0,0 +UNREASONABLY,2009,0,0,0,0,0,0 +UNRECOVERED,2009,0,0,0,0,0,0 +UNREMEDIED,2009,0,0,0,0,0,0 +UNSAFE,2009,0,0,0,0,0,0 +UNSATISFIED,2009,0,0,0,0,0,0 +UNSEASONABLY,0,0,2009,0,0,0,0 +UNSOUND,2009,0,0,0,0,0,0 +UNSTABLE,2009,0,0,0,0,0,0 +UNSUCCESSFULLY,2009,0,0,0,0,0,0 +UNSUITED,2009,0,0,0,0,0,0 +UNSUSPECTING,2009,0,0,0,0,0,0 +UNTIMELY,2009,0,0,0,0,0,0 +UNTRUTHFUL,2009,0,0,0,0,0,0 +UNUSABLE,2009,0,0,0,0,0,0 +UNWARRANTED,2009,0,0,0,0,0,0 +UNWRITTEN,0,0,2009,0,0,0,0 +URGENCY,2009,0,0,0,0,0,0 +USURPATION,0,0,0,2009,0,0,0 +USURY,2009,0,0,2009,0,0,0 +VAGUENESS,0,0,2012,0,0,0,0 +VALUABLE,0,2009,0,0,0,0,0 +PLEAD,2009,0,0,0,0,0,0 +PLEADS,2009,0,0,2009,0,0,0 +PLEASED,0,2009,0,0,0,0,0 +PLEDGED,0,0,0,0,0,0,2009 +PLEDGING,0,0,0,0,0,0,2009 +COMPEL,0,0,0,0,0,0,2011 +COMPENSATORY,0,0,0,2009,0,0,0 +COMPLAINED,2009,0,0,0,0,0,0 +COMPLAINTS,2009,0,0,0,0,0,0 +COMMITMENT,0,0,0,0,0,0,2009 +COMMITTING,0,0,0,0,0,0,2009 +LIMITATION,2009,0,0,0,0,0,0 +LINGERING,2009,0,0,0,0,0,0 +RESTRAINING,0,0,0,0,0,0,2009 +RESTRICT,0,0,0,0,0,0,2009 +RESTRICTIONS,0,0,0,0,0,0,2009 +RESTRICTS,0,0,0,0,0,0,2009 +RESTRUCTURING,2009,0,0,0,0,0,0 +RETALIATES,2009,0,0,0,0,0,0 +RETALIATORY,2009,0,0,0,0,0,0 +PRECAUTIONS,0,0,2009,0,0,0,0 +PRECLUDE,2009,0,0,0,0,0,2009 +PRECONDITION,0,0,0,0,0,0,2009 +PREDECEASED,0,0,0,2009,0,0,0 +PREDICTABILITY,0,0,2009,0,0,0,0 +PREDICTIONS,0,0,2009,0,0,0,0 +PREDICTS,0,0,2009,0,0,0,0 +PREJUDICE,2009,0,0,2009,0,0,0 +PREJUDICING,2009,0,0,2009,0,0,0 +PREMATURELY,2009,0,0,0,0,0,0 +PRESET,0,0,0,0,0,0,2009 +PRESUMABLY,0,0,2009,0,0,0,0 +PRESUMING,0,0,2009,0,0,0,0 +PRETRIAL,2009,0,0,2009,0,0,0 +PREVENTION,2009,0,0,0,0,0,0 +PROACTIVE,0,2009,0,0,0,0,0 +PROBABILITY,0,0,2009,0,0,0,0 +PROBATED,0,0,0,2009,0,0,0 +PROBATIONAL,0,0,0,2009,0,0,0 +PROBATIONS,0,0,0,2009,0,0,0 +PROBLEMS,2009,0,0,0,0,0,0 +PROFITABILITY,0,2009,0,0,0,0,0 +PROGRESSED,0,2009,0,0,0,0,0 +PROHIBITED,0,0,0,0,0,0,2009 +PROHIBITIVE,0,0,0,0,0,0,2009 +PROLONG,2009,0,0,0,0,0,0 +PROLONGING,2009,0,0,0,0,0,0 +PROMULGATES,0,0,0,2009,0,0,0 +PROMULGATOR,0,0,0,2009,0,0,0 +PRORATION,0,0,0,2009,0,0,0 +PROSECUTING,2009,0,0,2009,0,0,0 +PROSECUTORIAL,0,0,0,2011,0,0,0 +PROSPERITY,0,2009,0,0,0,0,0 +PROTESTED,2009,0,0,0,0,0,0 +PROTESTOR,2009,0,0,0,0,0,0 +PROTRACTION,2009,0,0,0,0,0,0 +PROVOKE,2009,0,0,0,0,0,0 +PUNISHABLE,0,0,0,2009,0,0,0 +PUNISHMENT,2009,0,0,0,0,0,0 +PURPORTED,2009,0,0,0,0,0,0 +QUESTION,2009,0,0,0,0,0,0 +QUESTIONING,2009,0,0,0,0,0,0 +QUITCLAIMS,0,0,0,2009,0,0,0 +RANDOM,0,0,2009,0,0,0,0 +RANDOMIZING,0,0,2009,0,0,0,0 +RATABLE,0,0,0,2009,0,0,0 +RATIONALIZE,2009,0,0,0,0,0,0 +ABANDONED,2009,0,0,0,0,0,0 +ABANDONS,2009,0,0,0,0,0,0 +ABDICATION,2009,0,0,0,0,0,0 +ABERRATIONAL,2009,0,0,0,0,0,0 +ABEYANCES,0,0,2009,0,0,0,0 +ABNORMAL,2009,0,0,0,0,0,0 +ABOLISH,2009,0,0,0,0,0,0 +ABOVEMENTIONED,0,0,0,2011,0,0,0 +ABROGATING,2009,0,0,2009,0,0,0 +ABRUPTLY,2009,0,0,0,0,0,0 +ABSENTEEISM,2009,0,0,0,0,0,0 +ABSOLVING,0,0,0,2009,0,0,0 +ABUSED,2009,0,0,0,0,0,0 +ABUSIVELY,2009,0,0,0,0,0,0 +ACCIDENT,2009,0,0,0,0,0,0 +ACCLAIMED,0,2009,0,0,0,0,0 +ACCOMPLISHING,0,2009,0,0,0,0,0 +ACCUSATIONS,2009,0,0,0,0,0,0 +ACCUSING,2009,0,0,0,0,0,0 +ACHIEVEMENTS,0,2009,0,0,0,0,0 +ACQUIESCED,2009,0,0,0,0,0,0 +ACQUIRORS,0,0,0,2011,0,0,0 +ACQUITTALS,2009,0,0,2009,0,0,0 +ACQUITTING,2009,0,0,2009,0,0,0 +ADJOURNED,0,0,0,2009,0,0,0 +ADJOURNS,0,0,0,2009,0,0,0 +ADJUDGING,0,0,0,2009,0,0,0 +ADJUDICATING,0,0,0,2009,0,0,0 +ADJUDICATOR,0,0,0,2009,0,0,0 +ADMISSIBLE,0,0,0,2009,0,0,0 +ADULTERATE,2009,0,0,0,0,0,0 +ADULTERATIONS,2009,0,0,0,0,0,0 +ADVANCING,0,2009,0,0,0,0,0 +ADVANTAGEOUSLY,0,2009,0,0,0,0,0 +ADVERSARY,2009,0,0,0,0,0,0 +ADVERSITY,2009,0,0,0,0,0,0 +AFFREIGHTMENT,0,0,0,2012,0,0,0 +AFORESTATED,0,0,0,2011,0,0,0 +AGGRAVATE,2009,0,0,0,0,0,0 +AGGRAVATION,2009,0,0,0,0,0,0 +ALERTING,2009,0,0,0,0,0,0 +ALIENATING,2009,0,0,0,0,0,0 +ALLEGATIONS,2009,0,0,2009,0,0,0 +ALLEGES,2009,0,0,2009,0,0,0 +ALMOST,0,0,2009,0,0,2009,0 +AMBIGUITIES,0,0,2009,0,0,0,0 +AMENDABLE,0,0,0,2009,0,0,0 +AMENDMENT,0,0,0,2009,0,0,0 +ANNOYANCE,2009,0,0,0,0,0,0 +ANNOYS,2009,0,0,0,0,0,0 +ANNULMENT,2009,0,0,0,0,0,0 +ANOMALOUS,2009,0,2009,0,0,0,0 +ANTECEDENTS,0,0,0,2009,0,0,0 +ANTICIPATING,0,0,2009,0,0,0,0 +ANTICORRUPTION,0,0,0,2014,0,0,0 +APPARENTLY,0,0,2009,0,0,2009,0 +APPEALING,0,0,0,2009,0,0,0 +APPEARING,0,0,2009,0,0,2009,0 +APPELLATE,0,0,0,2009,0,0,0 +APPROXIMATED,0,0,2009,0,0,0,0 +APPROXIMATION,0,0,2009,0,0,0,0 +APPURTENANT,0,0,0,2009,0,0,0 +ARBITRARINESS,0,0,2009,0,0,0,0 +ARBITRATES,0,0,0,2009,0,0,0 +ARBITRATIONS,0,0,0,2009,0,0,0 +ARGUE,2009,0,0,0,0,0,0 +ARGUMENTATIVE,2009,0,0,0,0,0,0 +ARREARS,2009,0,0,0,0,0,0 +ARTIFICIALLY,2009,0,0,0,0,0,0 +RETRIBUTION,2009,0,0,0,0,0,0 +RETROCESSIONAIRES,0,0,0,2011,0,0,0 +BOLSTERS,0,2009,0,0,0,0,0 +BOOMING,0,2009,0,0,0,0,0 +BOTTLENECKS,2009,0,0,0,0,0,0 +BOYCOTTED,2009,0,0,0,0,0,0 +BREACHED,2009,0,0,2009,0,0,0 +BREAKAGE,2009,0,0,0,0,0,0 +BREAKING,-2020,0,0,0,0,0,0 +BRIBE,2009,0,0,0,0,0,0 +BRIBES,2009,0,0,0,0,0,0 +BROKEN,-2020,0,0,0,0,0,0 +BURDENS,2009,0,0,0,0,0,0 +CALAMITOUS,2009,0,0,0,0,0,0 +CANCELING,2009,0,0,0,0,0,0 +CANCELLING,2009,0,0,0,0,0,0 +CARELESSNESS,2009,0,0,0,0,0,0 +CATASTROPHIC,2009,0,0,0,0,0,0 +CAUTIONED,2009,0,0,0,0,0,0 +CAUTIOUSLY,0,0,2009,0,0,0,0 +CEASES,2009,0,0,0,0,0,0 +CENSURE,2009,0,0,0,0,0,0 +CERTIORARI,0,0,0,2011,0,0,0 +CHALLENGES,2009,0,0,0,0,0,0 +CHATTEL,0,0,0,2009,0,0,0 +CIRCUMVENTED,2009,0,0,0,0,0,0 +CIRCUMVENTS,2009,0,0,0,0,0,0 +SHALL,0,0,0,2009,0,0,0 +SHORTAGES,2009,0,0,0,0,0,0 +SHRINKAGES,2009,0,0,0,0,0,0 +SHUTS,2009,0,0,0,0,0,0 +SLANDEROUS,2009,0,0,0,0,0,0 +REPUDIATING,2009,0,0,0,0,0,0 +REQUESTOR,0,0,0,2011,0,0,0 +REQUIREMENTS,0,0,0,0,0,0,2009 +RESCIND,0,0,0,2009,0,0,0 +RESCISSION,0,0,0,2009,0,0,0 +RESIGNATIONS,2009,0,0,0,0,0,0 +RESOLVE,0,2009,0,0,0,0,0 +RESTATEMENTS,2009,0,0,0,0,0,0 +NOTARIES,0,0,0,2009,0,0,0 +NOTARIZED,0,0,0,2009,0,0,0 +NOVO,0,0,0,2011,0,0,0 +NULLIFICATIONS,2009,0,0,2009,0,0,0 +NULLIFYING,2009,0,0,2009,0,0,0 +OBJECTING,2009,0,0,0,0,0,0 +OBJECTIONS,2009,0,0,0,0,0,0 +OBLIGATING,0,0,0,0,0,0,2009 +OBLIGE,0,0,0,0,0,0,2009 +OBLIGES,0,0,0,0,0,0,2009 +OBSCENITY,2009,0,0,0,0,0,0 +OBSTACLES,2009,0,0,0,0,0,0 +OBSTRUCTION,2009,0,0,0,0,0,0 +OFFENCES,2009,0,0,0,0,0,0 +OFFENDERS,2009,0,0,0,0,0,0 +OFFEREE,0,0,0,2009,0,0,0 +OMISSION,2009,0,0,0,0,0,0 +OMITTED,2009,0,0,0,0,0,0 +OPPORTUNISTICALLY,2009,0,0,0,0,0,0 +OPPOSED,2009,0,0,0,0,0,0 +OPPOSITIONS,2009,0,0,0,0,0,0 +ORDINARILY,0,0,2009,0,0,0,0 +OUTMODED,2009,0,0,0,0,0,0 +OUTPERFORMS,0,2009,0,0,0,0,0 +OVERBUILDING,2009,0,0,0,0,0,0 +OVERBURDENED,2009,0,0,0,0,0,0 +OVERCHARGE,2009,0,0,0,0,0,0 +OVERCOME,2009,0,0,0,0,0,0 +OVERESTIMATE,2009,0,0,0,0,0,0 +OVERESTIMATION,2009,0,0,0,0,0,0 +OVERLOADING,2009,0,0,0,0,0,0 +OVERLOOKING,2009,0,0,0,0,0,0 +OVERPAYMENTS,2009,0,0,0,0,0,0 +OVERPRODUCTION,2009,0,0,0,0,0,0 +OVERRULING,0,0,0,2009,0,0,0 +OVERSHADOW,2009,0,0,0,0,0,0 +OVERSTATE,2009,0,0,0,0,0,0 +OVERSTATES,2009,0,0,0,0,0,0 +OVERSUPPLY,2009,0,0,0,0,0,0 +OVERTURNED,2009,0,0,0,0,0,0 +OVERVALUED,2009,0,0,0,0,0,0 +PARA,0,0,0,-2020,0,0,0 +NECESSITATED,0,0,0,0,0,0,2009 +NEGATIVELY,2009,0,0,0,0,0,0 +NEGLECTFUL,2009,0,0,0,0,0,0 +NEGLIGENCES,2009,0,0,0,0,0,0 +NOLO,0,0,0,2011,0,0,0 +BEAUTIFUL,0,2009,0,0,0,0,0 +BELIEVES,0,0,2009,0,0,0,0 +IDEAL,0,2009,0,0,0,0,0 +IGNORE,2009,0,0,0,0,0,0 +ILL,2009,0,0,0,0,0,0 +ILLEGALLY,2009,0,0,0,0,0,0 +ILLIQUID,2009,0,0,0,0,0,0 +IMMATERIALITY,0,0,0,2009,0,0,0 +IMPAIRED,2009,0,0,0,0,0,2011 +IMPAIRS,2009,0,0,0,0,0,2011 +IMPEDED,2009,0,0,0,0,0,0 +IMPEDING,2009,0,0,0,0,0,0 +IMPERFECTIONS,2009,0,0,0,0,0,0 +IMPLICATE,2009,0,0,0,0,0,0 +IMPOSE,0,0,0,0,0,0,2009 +IMPOSITION,0,0,0,0,0,0,2009 +IMPOUND,2009,0,0,0,0,0,0 +IMPRACTICABLE,2009,0,0,0,0,0,0 +IMPRECISE,0,0,2009,0,0,0,0 +IMPRESSED,0,2009,0,0,0,0,0 +IMPRESSIVELY,0,2009,0,0,0,0,0 +IMPROPER,2009,0,0,0,0,0,0 +IMPROVE,0,2009,0,0,0,0,0 +IMPROVES,0,2009,0,0,0,0,0 +INABILITY,2009,0,0,0,0,0,0 +INACCURATE,2009,0,0,0,0,0,0 +INACTIVATE,2009,0,0,0,0,0,0 +INACTIVATION,2009,0,0,0,0,0,0 +INADEQUACY,2009,0,0,0,0,0,0 +INADVERTENTLY,2009,0,0,0,0,0,0 +INAPPROPRIATELY,2009,0,0,0,0,0,0 +INCAPACITATED,2009,0,0,0,0,0,0 +INCARCERATES,2009,0,0,2009,0,0,0 +INCHOATE,0,0,0,2009,0,0,0 +INCIDENTS,2009,0,0,0,0,0,0 +INCOMPETENCE,2009,0,0,0,0,0,0 +INCOMPETENTS,2009,0,0,0,0,0,0 +INCONCLUSIVE,2009,0,0,0,0,0,0 +INCONSISTENTLY,2009,0,0,0,0,0,0 +INCONVENIENCES,2009,0,0,0,0,0,0 +INCORRECTNESS,2009,0,0,0,0,0,0 +INDECENCY,2009,0,0,0,0,0,0 +INDEFINITE,0,0,2009,0,0,0,0 +INDEMNIFICATION,0,0,0,2009,0,0,0 +INDEMNIFY,0,0,0,2009,0,0,0 +INDEMNITIES,0,0,0,2009,0,0,0 +INDETERMINABLE,0,0,2009,0,0,0,0 +INDICTED,2009,0,0,2009,0,0,0 +INDORSEES,0,0,0,2011,0,0,0 +INEFFICIENCIES,2009,0,0,0,0,0,0 +INELIGIBILITY,2009,0,0,0,0,0,0 +INEQUITIES,2009,0,0,0,0,0,0 +INEXACTNESS,0,0,2009,0,0,0,0 +INFLICTED,2009,0,0,0,0,0,0 +INFRACTION,2009,0,0,2009,0,0,0 +INFRINGEMENT,2009,0,0,0,0,0,0 +INFRINGING,2009,0,0,0,0,0,0 +INHIBITING,0,0,0,0,0,0,2011 +INJUNCTIONS,2009,0,0,2009,0,0,0 +INJURES,2009,0,0,0,0,0,0 +INJURY,2009,0,0,0,0,0,0 +INNOVATING,0,2009,0,0,0,0,0 +INNOVATIVENESS,0,2011,0,0,0,0,0 +INORDINATELY,2009,0,0,0,0,0,0 +INSIGHTFUL,0,2009,0,0,0,0,0 +INSISTING,0,0,0,0,0,0,2009 +INSOLVENCY,2009,0,0,0,0,0,0 +INSTABILITIES,0,0,2009,0,0,0,0 +INSUFFICIENT,2009,0,0,0,0,0,0 +INTANGIBLE,0,0,2009,0,0,0,0 +INTERFERE,2009,0,0,0,0,0,0 +INTERFERES,2009,0,0,0,0,0,0 +INTERMITTENTLY,2009,0,0,0,0,0,0 +INTERPOSES,0,0,0,2009,0,0,0 +INTERROGATE,0,0,0,2009,0,0,0 +INTERROGATION,0,0,0,2009,0,0,0 +INTERROGATORS,0,0,0,2009,0,0,0 +INTERRUPTING,2009,0,0,0,0,0,0 +INTESTACY,0,0,0,2009,0,0,0 +STABILIZATION,0,2009,0,0,0,0,0 +STABILIZES,0,2009,0,0,0,0,0 +STAGNANT,2009,0,0,0,0,0,0 +STAGNATING,2009,0,0,0,0,0,0 +STATUTE,0,0,0,2009,0,0,0 +STIPULATE,0,0,0,0,0,0,2009 +STIPULATION,0,0,0,0,0,0,2009 +STOPPAGES,2009,0,0,0,0,0,0 +STRAIN,2009,0,0,0,0,0,0 +STRENGTH,0,2009,0,0,0,0,0 +STRENGTHENS,0,2009,0,0,0,0,0 +STRESSES,2009,0,0,0,0,0,0 +STRICTER,0,0,0,0,0,0,2009 +STRONG,0,2009,0,0,0,0,0 +SUBCLAUSE,0,0,0,2009,0,0,0 +SUBJECTING,2009,0,0,0,0,0,0 +SUBLESSORS,0,0,0,2011,0,0,0 +SUBPARAGRAPHS,0,0,0,2009,0,0,0 +SUBROGATED,0,0,0,2009,0,0,0 +SUBTRUSTS,0,0,0,2011,0,0,0 +SUCCEEDS,0,2009,0,0,0,0,0 +SUCCESSFULLY,0,2009,0,0,0,0,0 +SUED,2009,0,0,2009,0,0,0 +SUFFERING,2009,0,0,0,0,0,0 +SUGGESTING,0,0,2009,0,0,0,0 +SUMMONING,2009,0,0,2009,0,0,0 +SUPERSEDE,0,0,0,2009,0,0,0 +SUPERSEDING,0,0,0,2009,0,0,0 +SURPASSED,0,2009,0,0,0,0,0 +SUSCEPTIBLE,2009,0,0,0,0,0,0 +SUSPEND,2009,0,0,0,0,0,0 +SUSPENSION,2009,0,0,0,0,0,0 +SUSPICIOUS,2009,0,0,0,0,0,0 +REVOCATION,2009,0,0,2009,0,0,0 +REVOKES,2009,0,0,0,0,0,0 +REVOLUTIONIZES,0,2009,0,0,0,0,0 +REWARDING,0,2009,0,0,0,0,0 +RIDICULES,2009,0,0,0,0,0,0 +RISKIER,2009,0,2009,0,0,0,0 +RISKS,0,0,2009,0,0,0,0 +RULINGS,0,0,0,2009,0,0,0 +SACRIFICED,2009,0,0,0,0,0,0 +SATISFACTION,0,2009,0,0,0,0,0 +SATISFIES,0,2009,0,0,0,0,0 +SCANDALS,2009,0,0,0,0,0,0 +SCRUTINIZING,2009,0,0,0,0,0,0 +SEIZE,2009,0,0,0,0,0,0 +SELDOM,0,0,2009,0,0,2009,0 +SEQUESTRATOR,0,0,0,2009,0,0,0 +SETBACK,2009,0,0,0,0,0,0 +SEVER,2009,0,0,0,0,0,0 +SEVERANCE,0,0,0,2009,0,0,0 +SEVERELY,2009,0,0,0,0,0,0 +ENTRENCH,0,0,0,0,0,0,2009 +ERODES,2009,0,0,0,0,0,0 +ERRATICALLY,2009,0,0,0,0,0,0 +ERRONEOUSLY,2009,0,0,0,0,0,0 +ESCALATE,2009,0,0,0,0,0,0 +ESCHEAT,0,0,0,2011,0,0,0 +ESCROWED,0,0,0,0,0,0,2009 +EVADE,2009,0,0,0,0,0,0 +EVASION,2009,0,0,0,0,0,0 +EVICTED,2009,0,0,0,0,0,0 +EVICTS,2009,0,0,0,0,0,0 +EXACERBATED,2009,0,0,0,0,0,0 +EXACERBATIONS,2009,0,0,0,0,0,0 +EXAGGERATING,2009,0,0,0,0,0,0 +EXCEEDENCES,0,0,0,2011,0,0,0 +EXCELS,0,2009,0,0,0,0,0 +EXCESSIVELY,2009,0,0,0,0,0,0 +EXCITING,0,2009,0,0,0,0,0 +EXCLUSIVES,0,2009,0,0,0,0,0 +EXCULPATES,2009,0,0,2009,0,0,0 +EXCULPATORY,2009,0,0,2009,0,0,0 +EXECUTRICES,0,0,0,2009,0,0,0 +EXONERATE,2009,0,0,0,0,0,0 +EXONERATION,2009,0,0,0,0,0,0 +EXPLOITATIONS,2009,0,0,0,0,0,0 +EXPLOITS,2009,0,0,0,0,0,0 +EXPOSING,2009,0,0,0,0,0,0 +EXPROPRIATED,2009,0,0,0,0,0,0 +EXPROPRIATIONS,2009,0,0,0,0,0,0 +EXTRACONTRACTUAL,0,0,0,2011,0,0,0 +SOLVENCIES,2009,0,0,0,0,0,0 +SOMETIME,0,0,2009,0,0,0,0 +SPAM,2014,0,0,0,0,0,0 +TRAUMATIC,2009,0,0,0,0,0,0 +LOSES,2009,0,0,0,0,0,0 +LOST,2009,0,0,0,0,0,0 +LYING,2009,0,0,0,0,0,0 +MALFUNCTIONED,2009,0,0,0,0,0,0 +MALICIOUS,2009,0,0,0,0,0,0 +MANDATE,0,0,0,0,0,0,2009 +MANDATORY,0,0,0,0,0,0,2009 +MANIPULATES,2009,0,0,0,0,0,0 +MANIPULATIVE,2009,0,0,0,0,0,0 +MAYBE,0,0,2009,0,0,2009,0 +MEDIATING,0,0,0,2009,0,0,0 +MEDIATORS,0,0,0,2009,0,0,0 +MISAPPLICATIONS,2009,0,0,0,0,0,0 +MISAPPLYING,2009,0,0,0,0,0,0 +MISAPPROPRIATING,2009,0,0,0,0,0,0 +MISCALCULATE,2009,0,0,0,0,0,0 +MISCALCULATION,2009,0,0,0,0,0,0 +MISCLASSIFICATION,2011,0,0,0,0,0,0 +MISCOMMUNICATION,2014,0,0,0,0,0,0 +MISDEMEANORS,2009,0,0,0,0,0,0 +MISHANDLED,2009,0,0,0,0,0,0 +MISINFORMATION,2009,0,0,0,0,0,0 +MISINTERPRET,2009,0,0,0,0,0,0 +MISINTERPRETING,2009,0,0,0,0,0,0 +MISJUDGES,2009,0,0,0,0,0,0 +MISLABEL,2009,0,0,0,0,0,0 +MISLABELS,2009,0,0,0,0,0,0 +MISLEADS,2009,0,0,0,0,0,0 +MISMANAGEMENT,2009,0,0,0,0,0,0 +MISMATCHED,2009,0,0,0,0,0,0 +MISPRICE,2014,0,0,0,0,0,0 +MISREPRESENTATION,2009,0,0,0,0,0,0 +MISREPRESENTS,2009,0,0,0,0,0,0 +WORRYING,2009,0,0,0,0,0,0 +WORSENING,2009,0,0,0,0,0,0 +WORTHY,0,2009,0,0,0,0,0 +TOLERATED,2009,0,0,0,0,0,0 +TORT,0,0,0,2009,0,0,0 +TORTUOUS,2009,0,0,0,0,0,0 +FIDE,0,0,0,2009,0,0,0 +FIRING,2009,0,0,0,0,0,0 +NONBREACHING,0,0,0,2011,0,0,0 +NONCOMPLIANCE,2009,0,0,0,0,0,0 +NONCONFORMING,2009,0,0,0,0,0,0 +NONCONTRACT,0,0,0,2012,0,0,0 +NONFEASANCE,0,0,0,2011,0,0,0 +NONFORFEITURE,0,0,0,2011,0,0,0 +DISCLAIMED,2009,0,0,0,0,0,0 +DISCLAIMS,2009,0,0,0,0,0,0 +DISCLOSING,2009,0,0,0,0,0,0 +DISCONTINUATIONS,2009,0,0,0,0,0,0 +DISCONTINUING,2009,0,0,0,0,0,0 +DISCOURAGING,2009,0,0,0,0,0,0 +DISCREDITS,2009,0,0,0,0,0,0 +SPECTACULARLY,0,2009,0,0,0,0,0 +SPECULATING,0,0,2009,0,0,0,0 +SPECULATIVELY,0,0,2009,0,0,0,0 +TRAGEDY,2009,0,0,0,0,0,0 +TRANSFERORS,0,0,0,2012,0,0,0 +LEGISLATING,0,0,0,2009,0,0,0 +LEGISLATIVELY,0,0,0,2009,0,0,0 +LEGISLATURES,0,0,0,2009,0,0,0 +LIBELS,0,0,0,2009,0,0,0 +CONCEALED,2009,0,0,0,0,0,0 +CONCEDES,2009,0,0,0,0,0,0 +CONCERN,2009,0,0,0,0,0,0 +CONCILIATION,2009,0,0,0,0,0,0 +CONDEMN,2009,0,0,0,0,0,0 +CONDEMNING,2009,0,0,0,0,0,0 +CONDITIONALLY,0,0,2009,0,0,0,0 +CONFESS,2009,0,0,0,0,0,0 +CONFESSION,2009,0,0,0,0,0,0 +CONFINEMENT,2009,0,0,0,0,0,2009 +CONFISCATE,2009,0,0,0,0,0,0 +CONFISCATION,2009,0,0,0,0,0,0 +CONFLICTED,2009,0,0,0,0,0,0 +CONFRONTATION,2009,0,0,0,0,0,0 +CONFRONTING,2009,0,0,0,0,0,0 +CONFUSES,2009,0,2009,0,0,0,0 +CONSENT,0,0,0,2009,0,0,0 +CONSERVATORSHIPS,0,0,0,2011,0,0,0 +CONSPIRATORIAL,2009,0,0,0,0,0,0 +CONSPIRES,2009,0,0,0,0,0,0 +CONSTITUTIONALITY,0,0,0,2009,0,0,0 +CONSTRAIN,0,0,0,0,0,0,2009 +CONSTRAINT,0,0,0,0,0,0,2009 +CONSTRUE,0,0,0,2012,0,0,0 +CONTEMPT,2009,0,0,0,0,0,0 +CONTENDS,2009,0,0,0,0,0,0 +CONTENTIOUSLY,2009,0,0,0,0,0,0 +CONTESTING,2009,0,0,0,0,0,0 +CONTINGENTLY,0,0,2009,0,0,0,0 +CONTRACTHOLDER,0,0,0,2009,0,0,0 +CONTRACTING,0,0,0,2009,0,0,0 +CONTRACTUAL,0,0,0,2009,0,0,0 +CONTRADICTING,2009,0,0,0,0,0,0 +CONTRADICTS,2009,0,0,0,0,0,0 +CONTRAVENES,0,0,0,2009,0,0,0 +CONTROVERSIAL,2009,0,0,0,0,0,0 +CONTROVERTED,0,0,0,2009,0,0,0 +CONVEYANCES,0,0,0,2009,0,0,0 +CONVICTION,2009,0,0,2009,0,0,0 +CORRECTION,2009,0,0,0,0,0,0 +CORRUPTED,2009,0,0,0,0,0,0 +CORRUPTLY,2009,0,0,0,0,0,0 +COULD,0,0,2009,0,0,2009,0 +COUNSELS,0,0,0,2009,0,0,0 +COUNTERCLAIMS,2009,0,0,0,0,0,0 +COUNTERFEITERS,2009,0,0,0,0,0,0 +COUNTERMEASURES,2009,0,0,0,0,0,0 +COUNTERSUITS,0,0,0,2014,0,0,0 +COURTS,0,0,0,2009,0,0,0 +COVENANTS,0,0,0,0,0,0,2011 +CREATIVITY,0,2009,0,0,0,0,0 +CRIMINALITY,0,0,0,2009,0,0,0 +CRIMINALS,2009,0,0,2009,0,0,0 +CRITICALLY,2009,0,0,0,0,0,0 +CRITICIZED,2009,0,0,0,0,0,0 +CROSSCLAIMS,0,0,0,2011,0,0,0 +CRUCIALLY,2009,0,0,0,0,0,0 +CUMBERSOME,2009,0,0,0,0,0,0 +CURTAILMENT,2009,0,0,0,0,0,0 +CUTBACK,2009,0,0,0,0,0,0 +CYBERBULLYING,2014,0,0,0,0,0,0 +CYBERCRIMINALS,2014,0,0,0,0,0,0 +TAINTED,2009,0,0,0,0,0,0 +TENANTABILITY,0,0,0,2011,0,0,0 +TENTATIVELY,0,0,2009,0,0,0,0 +TERMINATES,2009,0,0,0,0,0,0 +TERMINUS,0,0,0,-2020,0,0,0 +TESTIMONY,0,0,0,2009,0,0,0 +THEREAFTER,0,0,0,2009,0,0,0 +THEREINAFTER,0,0,0,2011,0,0,0 +THERETO,0,0,0,2009,0,0,0 +THEREUNTO,0,0,0,2009,0,0,0 +THREATEN,2009,0,0,0,0,0,0 +THREATS,2009,0,0,0,0,0,0 +FAIL,2009,0,0,0,0,0,0 +FAILS,2009,0,0,0,0,0,0 +FALSE,2009,0,0,0,0,0,0 +FALSIFIED,2009,0,0,0,0,0,0 +FALSITY,2009,0,0,0,0,0,0 +FATALLY,2009,0,0,0,0,0,0 +FAULTY,2009,0,0,0,0,0,0 +FAVORING,0,2009,0,0,0,0,0 +FEARS,2009,0,0,0,0,0,0 +DAMAGE,2009,0,0,0,0,0,0 +DAMPEN,2009,0,0,0,0,0,0 +DANGEROUSLY,2009,0,0,0,0,0,0 +DEADLOCKING,2009,0,0,0,0,0,0 +DEBARMENT,2009,0,0,0,0,0,0 +DECEDENT,0,0,0,2009,0,0,0 +DECEITFULNESS,2009,0,0,0,0,0,0 +DECEIVING,2009,0,0,0,0,0,0 +DECEPTIVELY,2009,0,0,0,0,0,0 +DECLINES,2009,0,0,0,0,0,0 +DECREEING,0,0,0,2009,0,0,0 +DEFACEMENT,2009,0,0,0,0,0,0 +DEFAMATIONS,2009,0,0,0,0,0,0 +DEFAMES,2009,0,0,0,0,0,0 +DEFAULTING,2009,0,0,0,0,0,0 +DEFEASE,0,0,0,2009,0,0,0 +DEFEASING,0,0,0,2011,0,0,0 +DEFEATS,2009,0,0,0,0,0,0 +DEFECTS,2009,0,0,0,0,0,0 +DEFENDANTS,2009,0,0,2009,0,0,0 +DEFENSIVE,2009,0,0,0,0,0,0 +DEFICIENCY,2009,0,0,0,0,0,0 +DEFINITELY,0,0,0,0,2009,0,0 +DEFRAUDING,2009,0,0,0,0,0,0 +DEGRADATIONS,2009,0,0,0,0,0,0 +DEGRADING,2009,0,0,0,0,0,0 +DELAYS,2009,0,0,0,0,0,0 +DELEGEES,0,0,0,2011,0,0,0 +DELIBERATELY,2009,0,0,0,0,0,0 +DELIGHTFULLY,0,2009,0,0,0,0,0 +DELINQUENCY,2009,0,0,0,0,0,0 +DELIST,2009,0,0,0,0,0,0 +DEMISE,2009,0,0,0,0,0,0 +DEMOLISH,2009,0,0,0,0,0,0 +DEMOLITION,2009,0,0,0,0,0,0 +DEMOTES,2009,0,0,0,0,0,0 +DEMURRED,0,0,0,2009,0,0,0 +DEMURS,0,0,0,2009,0,0,0 +DENIES,2009,0,0,0,0,0,0 +DENIGRATING,2009,0,0,0,0,0,0 +DEPEND,0,0,2009,0,0,2009,2011 +DEPENDANCES,0,0,0,0,0,0,2011 +DEPENDENCIES,0,0,2009,0,0,0,2011 +DEPENDS,0,0,2009,0,0,2009,2011 +DEPLETING,2009,0,0,0,0,0,0 +DEPOSED,0,0,0,2009,0,0,0 +DEPOSITIONAL,0,0,0,2011,0,0,0 +INVALIDATED,2009,0,0,0,0,0,0 +INVALIDITY,2009,0,0,0,0,0,0 +INVENTION,0,2009,0,0,0,0,0 +INVENTOR,0,2009,0,0,0,0,0 +INVESTIGATES,2009,0,0,0,0,0,0 +INVOLUNTARILY,2009,0,0,0,0,0,0 +IRRECOVERABLE,2009,0,0,0,0,0,0 +IRREGULARITY,2009,0,0,0,0,0,0 +IRREVERSIBLE,2009,0,0,0,0,0,0 +JEOPARDIZE,2009,0,0,0,0,0,0 +JUDICIALLY,0,0,0,2009,0,0,0 +JURIS,0,0,0,2011,0,0,0 +JURISDICTIONS,0,0,0,2009,0,0,0 +JUROR,0,0,0,2009,0,0,0 +JUSTICE,0,0,0,2009,0,0,0 +KICKBACKS,2009,0,0,0,0,0,0 +DIMINISHED,2009,0,0,0,0,0,0 +DIRECTIVE,0,0,0,0,0,0,2009 +DISADVANTAGEOUS,2009,0,0,0,0,0,0 +DISAFFIRMANCE,0,0,0,2009,0,0,0 +DISAGREEABLE,2009,0,0,0,0,0,0 +DISAGREEMENTS,2009,0,0,0,0,0,0 +DISALLOWANCES,2009,0,0,0,0,0,0 +DISAPPEAR,2009,0,0,0,0,0,0 +DISAPPEARING,2009,0,0,0,0,0,0 +DISAPPOINTING,2009,0,0,0,0,0,0 +DISAPPOINTS,2009,0,0,0,0,0,0 +DISAPPROVED,2009,0,0,0,0,0,0 +DISASSOCIATING,2009,0,0,0,0,0,0 +DISASTERS,2009,0,0,0,0,0,0 +DISAVOWAL,2009,0,0,0,0,0,0 +GRATUITOUSLY,2009,0,0,0,0,0,0 +GREATLY,0,2009,0,0,0,0,0 +GROSSLY,2009,0,0,0,0,0,0 +HALTED,2009,0,0,0,0,0,0 +HAMPERS,2009,0,0,0,0,0,0 +HAPPY,0,2009,0,0,0,0,0 +HARASSMENT,2009,0,0,0,0,0,0 +HARMED,2009,0,0,0,0,0,0 +HARMS,2009,0,0,0,0,0,0 +HARSHLY,2009,0,0,0,0,0,0 +HAZARDS,2009,0,0,0,0,0,0 +NONINFRINGING,0,0,0,2011,0,0,0 +NONPAYMENT,2009,0,0,0,0,0,0 +NONPERFORMING,2009,0,0,0,0,0,0 +DISFAVORS,2009,0,0,0,0,0,0 +PREAMENDMENT,0,0,0,2011,0,0,0 +COMPLICATED,2009,0,0,0,0,0,0 +COMPLICATIONS,2009,0,0,0,0,0,0 +COMPLIMENTING,0,2009,0,0,0,0,0 +MISSTATEMENTS,2009,0,0,0,0,0,0 +MISSTEPS,2009,0,0,0,0,0,0 +MISTAKES,2009,0,0,0,0,0,0 +MISUNDERSTAND,2009,0,0,0,0,0,0 +MISUSE,2009,0,0,0,0,0,0 +MONOPOLISTIC,2009,0,0,0,0,0,0 +MONOPOLIZED,2009,0,0,0,0,0,0 +MORATORIA,2009,0,0,0,0,0,0 +MOTHBALLED,2009,0,0,0,0,0,0 +MUTANDIS,0,0,0,2011,0,0,0 +SLIPPAGES,2009,0,0,0,0,0,0 +SLOWED,2009,0,0,0,0,0,0 +SLOWLY,2009,0,0,0,0,0,0 +SLUGGISHNESS,2009,0,0,0,0,0,0 +SMOOTHS,0,2009,0,0,0,0,0 +DEPRESSES,2009,0,0,0,0,0,0 +DEPRIVED,2009,0,0,0,0,0,0 +DERELICTION,2009,0,0,0,0,0,0 +DEROGATING,0,0,0,2009,0,0,0 +DESIGNATOR,0,0,0,2011,0,0,0 +DESPITE,0,2009,0,0,0,0,0 +DESTABILIZING,2009,0,2009,0,0,0,0 +DESTROYING,2009,0,0,0,0,0,0 +DETAIN,2009,0,0,0,0,0,0 +DETENTIONS,2009,0,0,0,0,0,0 +DETERIORATES,2009,0,0,0,0,0,0 +DETERRED,2009,0,0,0,0,0,0 +DETERRENTS,2009,0,0,0,0,0,0 +DETRACTED,2009,0,0,0,0,0,0 +DETRIMENTALLY,2009,0,0,0,0,0,0 +DEVALUES,2009,0,0,0,0,0,0 +DEVASTATING,2009,0,0,0,0,0,0 +DEVIATES,2009,0,2009,0,0,0,0 +DEVISEES,0,0,0,2011,0,0,0 +DEVOLVING,2009,0,0,0,0,0,0 +DICTATING,0,0,0,0,0,0,2009 +DIFFERS,0,0,2009,0,0,0,0 +DIFFICULTY,2009,0,0,0,0,0,0 +ASCENDANTS,0,0,0,2009,0,0,0 +ASSAULTS,2009,0,0,0,0,0,0 +ASSIGNATIONS,0,0,0,2009,0,0,0 +ASSUMES,0,0,2009,0,0,0,0 +ASSURE,0,2009,0,0,0,0,0 +ATTAIN,0,2009,0,0,0,0,0 +ATTAINMENTS,0,2009,0,0,0,0,0 +ATTESTATIONS,0,0,0,2009,0,0,0 +ATTORNEY,0,0,0,2009,0,0,0 +ATTRACTIVE,0,2009,0,0,0,0,0 +BACKDATING,2009,0,0,0,0,0,0 +BAILEE,0,0,0,2009,0,0,0 +BAILMENT,0,0,0,2011,0,0,0 +BANKRUPT,2009,0,0,0,0,0,0 +BANKRUPTING,2009,0,0,0,0,0,0 +CLAIMANTS,0,0,0,2009,0,0,0 +CLARIFICATION,0,0,2009,0,0,0,0 +CLEARLY,0,0,0,0,2009,0,0 +CLOSING,-2020,0,0,0,0,0,0 +CODEFENDANT,0,0,0,2011,0,0,0 +CODIFICATION,0,0,0,2009,0,0,0 +CODIFY,0,0,0,2009,0,0,0 +COERCES,2009,0,0,0,0,0,0 +COLLABORATED,0,2009,0,0,0,0,0 +COLLABORATIONS,0,2009,0,0,0,0,0 +COLLAPSE,2009,0,0,0,0,0,0 +COLLISION,2009,0,0,0,0,0,0 +COLLUDES,2009,0,0,0,0,0,0 +COLLUSIVE,2009,0,0,0,0,0,0 +POSITIVELY,0,2009,0,0,0,0,0 +POSSIBLE,0,0,2009,0,0,2009,0 +POSTCONTRACT,0,0,0,2011,0,0,0 +POSTPONEMENT,2009,0,0,0,0,0,0 +WRITEDOWN,2009,0,0,0,0,0,0 +WRITS,0,0,0,2009,0,0,0 +WRONGFUL,2009,0,0,0,0,0,0 +HONOR,0,2009,0,0,0,0,0 +HONORS,0,2009,0,0,0,0,0 +REARGUMENT,0,0,0,2011,0,0,0 +REASSESSING,0,0,2009,0,0,0,0 +REASSIGNED,2009,0,0,0,0,0,0 +REASSIGNS,2009,0,0,0,0,0,0 +REBUT,0,0,0,2009,0,0,0 +REBUTTAL,0,0,0,2009,0,0,0 +RECALCULATE,0,0,2009,0,0,0,0 +RECALCULATION,0,0,2009,0,0,0,0 +RECALLING,2009,0,0,0,0,0,0 +RECESSIONARY,2009,0,0,0,0,0,0 +RECKLESSNESS,2009,0,0,0,0,0,0 +RECONSIDERS,0,0,2009,0,0,0,0 +RECOUPMENTS,0,0,0,2009,0,0,0 +RECTIFICATIONS,0,0,0,2009,0,0,0 +RECUSES,0,0,0,2014,0,0,0 +REDACTING,2009,0,0,2009,0,0,0 +REDEFAULTED,2012,0,0,0,0,0,0 +REDRESSES,2009,0,0,0,0,0,0 +REEXAMINING,0,0,2009,0,0,0,0 +REFILE,0,0,0,2009,0,0,0 +REFRAIN,0,0,0,0,0,0,2009 +REFUSALS,2009,0,0,0,0,0,0 +REFUSING,2009,0,0,0,0,0,0 +REGULATE,0,0,0,2009,0,0,0 +REGULATION,0,0,0,2009,0,0,0 +REGULATORS,0,0,0,2009,0,0,0 +REHEARING,0,0,0,2009,0,0,0 +VARIABLY,0,0,2009,0,0,0,0 +VARIANTS,0,0,2009,0,0,0,0 +VARIES,0,0,2009,0,0,0,0 +VENDEES,0,0,0,2011,0,0,0 +VERSATILITY,0,2009,0,0,0,0,0 +VIBRANT,0,2009,0,0,0,0,0 +VIOLATES,2009,0,0,0,0,0,0 +VIOLATIVE,2009,0,0,2009,0,0,0 +VIOLENT,2009,0,0,0,0,0,0 +VITIATES,2009,0,0,0,0,0,0 +VOIDED,2009,0,0,2009,0,0,0 +VOLATILITY,2009,0,2009,0,0,0,0 +VULNERABLY,2009,0,0,0,0,0,0 +WARNINGS,2009,0,0,0,0,0,0 +WASTED,2009,0,0,0,0,0,0 +WEAKEN,2009,0,0,0,0,0,0 +WEAKER,2009,0,0,0,0,0,0 +WEAKNESSES,2009,0,0,0,0,0,0 +DISHONORABLE,2009,0,0,0,0,0,0 +DISHONORS,2009,0,0,0,0,0,0 +DISINTERESTEDLY,2009,0,0,0,0,0,0 +DISLOYALTY,2009,0,0,0,0,0,0 +DISMISSAL,2009,0,0,0,0,0,0 +DISMISSING,2009,0,0,0,0,0,0 +DISPARAGEMENT,2009,0,0,0,0,0,0 +DISPARAGINGLY,2009,0,0,0,0,0,0 +DISPLACED,2009,0,0,0,0,0,0 +DISPLACING,2009,0,0,0,0,0,0 +DISPOSSESSED,2009,0,0,0,0,0,0 +DISPOSSESSORY,0,0,0,2011,0,0,0 +DISPROPORTIONATELY,2009,0,0,0,0,0,0 +DISPUTING,2009,0,0,0,0,0,0 +DISQUALIFIES,2009,0,0,0,0,0,0 +DISREGARDED,2009,0,0,0,0,0,0 +DISREPUTE,2009,0,0,0,0,0,0 +DISRUPTION,2009,0,0,0,0,0,0 +DISSATISFACTION,2009,0,0,0,0,0,0 +DISSENTER,2009,0,0,0,0,0,0 +DISSIDENT,2009,0,0,0,0,0,0 +DISTINCTION,0,2009,0,0,0,0,0 +DISTINCTIVENESS,0,2009,0,0,0,0,0 +DISTORTION,2009,0,0,0,0,0,0 +DISTRACTED,2009,0,0,0,0,0,0 +DISTRACTS,2009,0,0,0,0,0,0 +DISTRIBUTEE,0,0,0,2009,0,0,0 +DISTURBANCES,2009,0,0,0,0,0,0 +DIVERSION,2009,0,0,0,0,0,0 +DIVERTS,2009,0,0,0,0,0,0 +DIVESTITURE,2009,0,0,0,0,0,0 +DIVESTS,2009,0,0,0,0,0,0 +DIVULGED,2009,0,0,0,0,0,0 +DOCKETED,0,0,0,2009,0,0,0 +DOUBT,2009,0,2009,0,0,0,0 +DOWNGRADE,2009,0,0,0,0,0,0 +DOWNSIZE,2009,0,0,0,0,0,0 +DOWNSIZINGS,2009,0,0,0,0,0,0 +DOWNTURNS,2009,0,0,0,0,0,0 +DRASTIC,2009,0,0,0,0,0,0 +DREAM,0,2009,0,0,0,0,0 +DULY,0,0,0,2009,0,0,0 +DYSFUNCTIONS,2009,0,0,0,0,0,0 +EARMARKS,0,0,0,0,0,0,2009 +EASY,0,2009,0,0,0,0,0 +EFFICIENT,0,2009,0,0,0,0,0 +EJECTMENT,0,0,0,2011,0,0,0 +EMBARGOING,2009,0,0,0,0,0,0 +EMBARRASSING,2009,0,0,0,0,0,0 +EMBEZZLED,2009,0,0,0,0,0,0 +EMBEZZLES,2009,0,0,0,0,0,0 +EMPOWERING,0,2009,0,0,0,0,0 +ENABLES,0,2009,0,0,0,0,0 +ENCOURAGES,0,2009,0,0,0,0,0 +ENCROACHES,2009,0,0,0,0,0,0 +ENCUMBER,2009,0,0,2009,0,0,2009 +ENCUMBRANCE,2009,0,0,2009,0,0,2009 +ENDANGER,2009,0,0,0,0,0,0 +ENDANGERS,2009,0,0,0,0,0,0 +ENFORCEABLY,0,0,0,2011,0,0,0 +ENHANCEMENTS,0,2009,0,0,0,0,0 +ENJOINED,2009,0,0,0,0,0,0 +ENJOYABLE,0,2009,0,0,0,0,0 +ENJOYMENT,0,2009,0,0,0,0,0 +ENTAILING,0,0,0,0,0,0,2009 +PASSU,0,0,0,2009,0,0,0 +PENALIZED,2009,0,0,0,0,0,0 +PENALTY,2009,0,0,0,0,0,0 +PERFECTLY,0,2009,0,0,0,0,0 +PERILS,2009,0,0,0,0,0,0 +PERMISSIONS,0,0,0,0,0,0,2009 +PERMITTING,0,0,0,0,0,0,2009 +PERPETRATING,2009,0,0,2009,0,0,0 +PERSISTENCE,2009,0,0,0,0,0,0 +PERSISTS,2009,0,0,0,0,0,0 +PERVASIVENESS,2009,0,0,0,0,0,0 +PETITIONERS,0,0,0,2009,0,0,0 +PICKET,2009,0,0,0,0,0,0 +HEREBY,0,0,0,2009,0,0,0 +HEREFROM,0,0,0,2009,0,0,0 +HEREINBEFORE,0,0,0,2009,0,0,0 +HERETO,0,0,0,2009,0,0,0 +HEREUPON,0,0,0,2009,0,0,0 +HIGHEST,0,2009,0,0,2009,0,0 +HINDERS,2009,0,0,0,0,0,0 +WHATEVER,0,0,0,2009,0,0,0 +WHEREAS,0,0,0,2009,0,0,0 +WHEREIN,0,0,0,2009,0,0,0 +WHEREUNDER,0,0,0,2011,0,0,0 +WHOMEVER,0,0,0,2009,0,0,0 +WILL,0,0,0,0,2009,0,0 +WIN,0,2009,0,0,0,0,0 +WITNESS,0,0,0,2009,0,0,0 +FLUCTUATED,0,0,2009,0,0,0,0 +FLUCTUATIONS,0,0,2009,0,0,0,0 +FORBEARANCES,0,0,0,2009,0,0,0 +FORBIDDEN,2009,0,0,0,0,0,2009 +FORCED,2009,0,0,0,0,0,0 +FOREBEARS,0,0,0,2009,0,0,0 +FORECLOSING,2009,0,0,0,0,0,0 +FOREGOES,2009,0,0,0,0,0,0 +FORESTALLING,2009,0,0,0,0,0,0 +FORFEITABLE,0,0,0,2009,0,0,0 +FORFEITURE,2009,0,0,0,0,0,0 +FORTHWITH,0,0,0,2009,0,0,0 +FRAUDULENCE,2009,0,0,0,0,0,0 +FRIVOLOUS,2009,0,0,0,0,0,0 +FRUSTRATES,2009,0,0,0,0,0,0 +FRUSTRATIONS,2009,0,0,0,0,0,0 +GAIN,0,2009,0,0,0,0,0 +GOOD,0,2009,0,0,0,0,0 +DISGORGES,2009,0,0,0,0,0,0 +DISGRACEFULLY,2009,0,0,0,0,0,0 +LITIGATED,2009,0,0,2009,0,0,0 +LITIGATIONS,2009,0,0,2009,0,0,0 +LITIGIOUSNESS,0,0,0,2009,0,0,0 +LACKING,2009,0,0,0,0,0,0 +LAGGED,2009,0,0,0,0,0,0 +LAPSED,2009,0,0,0,0,0,0 +LAUNDERING,2009,0,0,0,0,0,0 +LAWFULNESS,0,0,0,2009,0,0,0 +LAWSUIT,0,0,0,2009,0,0,0 +LAYOFF,2009,0,0,0,0,0,0 +LEGAL,0,0,0,2009,0,0,0 +LEGALIZATIONS,0,0,0,2009,0,0,0 +LEGALIZING,0,0,0,2009,0,0,0 +LEGATEES,0,0,0,2009,0,0,0 +FLAWS,2009,0,0,0,0,0,0 +NONRENEWAL,2009,0,0,0,0,0,0 +LIQUIDATING,2009,0,0,0,0,0,0 +LIQUIDATORS,2009,0,0,0,0,0,0 +BENEFICIAL,0,-2020,0,2020,0,0,0 +BENEFIT,0,-2020,0,0,0,0,0 +BENEFITTING,0,2009,0,0,0,0,0 +POORLY,2009,0,0,0,0,0,0 +REINTERPRETATIONS,0,0,2009,0,0,0,0 +REJECT,2009,0,0,0,0,0,0 +REJECTIONS,2009,0,0,0,0,0,0 +RELINQUISHED,2009,0,0,0,0,0,0 +RELINQUISHMENTS,2009,0,0,0,0,0,0 +REMANDED,0,0,0,2009,0,0,0 +REMEDIATED,0,0,0,2009,0,0,0 +REMEDIED,0,0,0,2009,0,0,0 +RENEGOTIATES,2009,0,0,0,0,0,0 +RENOUNCE,2009,0,0,0,0,0,0 +RENOUNCES,2009,0,0,0,0,0,0 +REPLEDGED,0,0,0,2012,0,0,0 +REPOSSESSING,2009,0,0,0,0,0,0 +TROUBLES,2009,0,0,0,0,0,0 +UNACCEPTABLE,2009,0,0,0,0,0,0 +UNANNOUNCED,2009,0,0,0,0,0,0 +UNAPPROVED,2009,0,0,0,0,0,0 +UNAVAILABLE,2009,0,0,0,0,0,2011 +UNCERTAIN,0,0,2009,0,0,2009,0 +UNCLEAR,0,0,2009,0,0,0,0 +UNCOLLECTIBLE,2009,0,0,0,0,0,0 +UNCOMPROMISING,0,0,0,0,2009,0,0 +UNCONSTITUTIONAL,0,0,0,2009,0,0,0 +UNCONTROLLABLE,2009,0,0,0,0,0,0 +UNCOVER,2009,0,0,0,0,0,0 +UNDECIDED,0,0,2009,0,0,0,0 +UNDELIVERED,2009,0,0,0,0,0,0 +UNDERCUTTING,2009,0,0,0,0,0,0 +UNDERESTIMATING,2009,0,0,0,0,0,0 +UNDERMINE,2009,0,0,0,0,0,0 +UNDERPAID,2009,0,0,0,0,0,0 +UNDERPERFORM,2011,0,0,0,0,0,0 +UNDERPERFORMS,2014,0,0,0,0,0,0 +UNDERSTATE,2009,0,0,0,0,0,0 +UNDERSTATES,2009,0,0,0,0,0,0 +UNDESIGNATED,0,0,2009,0,0,0,0 +UNDETECTED,2009,0,0,0,0,0,0 +UNDISCLOSED,2009,0,0,0,0,0,0 +UNDUE,2009,0,0,0,0,0,0 +UNECONOMICALLY,2009,0,0,0,0,0,0 +UNENCUMBERED,0,0,0,2009,0,0,0 +UNEQUIVOCALLY,0,0,0,0,2009,0,0 +UNEXPECTED,2009,0,2009,0,0,0,0 +UNFAMILIAR,0,0,2009,0,0,0,0 +UNFAVORABLY,2009,0,0,0,0,0,0 +UNFITNESS,2009,0,0,0,0,0,0 +UNFORSEEN,2011,0,2011,0,0,0,0 +UNFRIENDLY,2009,0,0,0,0,0,0 +UNHEDGED,0,0,2009,0,0,0,0 +UNINTENDED,2009,0,0,0,0,0,0 +UNJUSTIFIABLE,2009,0,0,0,0,0,0 +UNKNOWING,2009,0,0,0,0,0,0 +UNLAWFUL,2009,0,0,2009,0,0,0 +UNLIQUIDATED,2009,0,0,0,0,0,0 +UNMERITORIOUS,2014,0,0,0,0,0,0 +UNOBSERVABLE,0,0,2009,0,0,0,0 +UNPARALLELED,0,2009,0,0,2009,0,0 +UNPREDICTABILITY,2009,0,2009,0,0,0,0 +UNPRODUCTIVE,2009,0,0,0,0,0,0 +UNPROVEN,0,0,2009,0,0,0,0 +UNREALISTIC,2009,0,0,0,0,0,0 +UNRECEPTIVE,2014,0,0,0,0,0,0 +UNREIMBURSED,2009,0,0,0,0,0,0 +UNREPORTED,2009,0,0,0,0,0,0 +UNSALABLE,2009,0,0,0,0,0,0 +UNSAVORY,2009,0,0,0,0,0,0 +UNSELLABLE,2014,0,0,0,0,0,0 +UNSPECIFIC,0,0,2009,0,0,0,0 +UNSTAYED,0,0,0,2009,0,0,0 +UNSUITABILITY,2009,0,0,0,0,0,0 +UNSURE,2009,0,0,0,0,0,0 +UNSUSTAINABLE,2009,0,0,0,0,0,0 +UNTO,0,0,0,2009,0,0,0 +UNTRUTHFULLY,2009,0,0,0,0,0,0 +UNUSUAL,0,0,2009,0,0,0,0 +UNWELCOME,2009,0,0,0,0,0,0 +UPSET,2009,0,0,0,0,0,0 +URGENT,2009,0,0,0,0,0,0 +USURPED,2009,0,0,2009,0,0,0 +VAGARIES,0,0,2009,0,0,0,0 +VAGUENESSES,0,0,2012,0,0,0,0 +VANDALISM,2009,0,0,0,0,0,0 +PLAINTIFF,2009,0,0,2009,0,0,0 +PLEADED,2009,0,0,0,0,0,0 +PLEAS,2009,0,0,2009,0,0,0 +PLEASURE,0,2009,0,0,0,0,0 +PLEDGEE,0,0,0,2009,0,0,0 +PLEDGOR,0,0,0,2009,0,0,0 +COMPELLED,0,0,0,0,0,0,2009 +COMPLAIN,2009,0,0,0,0,0,0 +COMPLAINING,2009,0,0,0,0,0,0 +COMMITMENTS,0,0,0,0,0,0,2009 +LIMITATIONS,2009,0,0,0,0,0,0 +RESTRAINS,0,0,0,0,0,0,2009 +RESTRICTED,0,0,0,0,0,0,2009 +RESTRICTIVE,0,0,0,0,0,0,2009 +RESTRUCTURE,2009,0,0,0,0,0,0 +RESTRUCTURINGS,2009,0,0,0,0,0,0 +RETALIATING,2009,0,0,0,0,0,0 +RETENDERING,0,0,0,2011,0,0,0 +PRECIPITATED,2009,0,0,0,0,0,0 +PRECLUDED,2009,0,0,0,0,0,2009 +PRECONDITIONS,0,0,0,0,0,0,2009 +PREDECEASES,0,0,0,2009,0,0,0 +PREDICTED,0,0,2009,0,0,0,0 +PREDICTIVE,0,0,2009,0,0,0,0 +PREEMINENCE,0,2009,0,0,0,0,0 +PREJUDICED,2009,0,0,2009,0,0,0 +PRELIMINARILY,0,0,2009,0,0,0,0 +PREMIER,0,2009,0,0,0,0,0 +PRESSING,2009,0,0,0,0,0,0 +PRESUME,0,0,2009,0,0,0,0 +PRESUMPTION,0,0,2009,0,0,0,0 +PREVENT,0,0,0,0,0,0,2009 +PREVENTS,2009,0,0,0,0,0,2009 +PROACTIVELY,0,2009,0,0,0,0,0 +PROBABLE,0,0,2009,0,0,0,0 +PROBATES,0,0,0,2009,0,0,0 +PROBATIONARY,0,0,0,2009,0,0,0 +PROBLEM,2009,0,0,0,0,0,0 +PROFICIENCY,0,2009,0,0,0,0,0 +PROFITABLE,0,2009,0,0,0,0,0 +PROGRESSES,0,2009,0,0,0,0,0 +PROHIBITING,0,0,0,0,0,0,2009 +PROHIBITIVELY,0,0,0,0,0,0,2009 +PROLONGATION,2009,0,0,0,0,0,0 +PROLONGS,2009,0,0,0,0,0,0 +PROMULGATING,0,0,0,2009,0,0,0 +PROMULGATORS,0,0,0,2009,0,0,0 +PROSECUTE,2009,0,0,2009,0,0,0 +PROSECUTION,2009,0,0,2009,0,0,0 +PROSECUTORS,0,0,0,2009,0,0,0 +PROSPEROUS,0,2009,0,0,0,0,0 +PROTESTER,2009,0,0,0,0,0,0 +PROTESTORS,2009,0,0,0,0,0,0 +PROVISO,0,0,0,2009,0,0,0 +PROVOKED,2009,0,0,0,0,0,0 +PUNISHED,2009,0,0,0,0,0,0 +PUNISHMENTS,2009,0,0,0,0,0,0 +PURPORTEDLY,2009,0,0,0,0,0,0 +QUESTIONABLE,2009,0,0,0,0,0,0 +QUESTIONS,2009,0,0,0,0,0,0 +QUITTING,2009,0,0,0,0,0,0 +RANDOMIZE,0,0,2009,0,0,0,0 +RANDOMLY,0,0,2009,0,0,0,0 +RATABLY,0,0,0,2009,0,0,0 +RATIONALIZED,2009,0,0,0,0,0,0 +ABANDONING,2009,0,0,0,0,0,0 +ABDICATED,2009,0,0,0,0,0,0 +ABDICATIONS,2009,0,0,0,0,0,0 +ABERRATIONS,2009,0,0,0,0,0,0 +ABIDE,0,0,0,0,0,0,2009 +ABNORMALITIES,2009,0,0,0,0,0,0 +ABOLISHED,2009,0,0,0,0,0,0 +ABROGATE,2009,0,0,2009,0,0,0 +ABROGATION,2009,0,0,2009,0,0,0 +ABRUPTNESS,2009,0,0,0,0,0,0 +ABSOLVE,0,0,0,2009,0,0,0 +ABUNDANCE,0,2009,0,0,0,0,0 +ABUSES,2009,0,0,0,0,0,0 +ABUSIVENESS,2009,0,0,0,0,0,0 +ACCIDENTAL,2009,0,0,0,0,0,0 +ACCOMPLISH,0,2009,0,0,0,0,0 +ACCOMPLISHMENT,0,2009,0,0,0,0,0 +ACCUSE,2009,0,0,0,0,0,0 +ACHIEVE,0,2009,0,0,0,0,0 +ACHIEVES,0,2009,0,0,0,0,0 +ACQUIESCES,2009,0,0,0,0,0,0 +ACQUIT,2009,0,0,2009,0,0,0 +ACQUITTANCE,0,0,0,2009,0,0,0 +ADDENDUMS,0,0,0,2011,0,0,0 +ADJOURNING,0,0,0,2009,0,0,0 +ADJUDGE,0,0,0,2009,0,0,0 +ADJUDICATE,0,0,0,2009,0,0,0 +ADJUDICATION,0,0,0,2009,0,0,0 +ADJUDICATORS,0,0,0,2009,0,0,0 +ADMISSIBLY,0,0,0,2009,0,0,0 +ADULTERATED,2009,0,0,0,0,0,0 +ADVANCEMENT,0,2009,0,0,0,0,0 +ADVANTAGE,0,2009,0,0,0,0,0 +ADVANTAGES,0,2009,0,0,0,0,0 +ADVERSE,2009,0,0,0,0,0,0 +AFFIDAVIT,0,0,0,2009,0,0,0 +AFOREDESCRIBED,0,0,0,2011,0,0,0 +AFTERMATH,2009,0,0,0,0,0,0 +AGGRAVATED,2009,0,0,0,0,0,0 +AGGRAVATIONS,2009,0,0,0,0,0,0 +ALIENATE,2009,0,0,0,0,0,0 +ALIENATION,2009,0,0,0,0,0,0 +ALLEGE,2009,0,0,2009,0,0,0 +ALLEGING,2009,0,0,2009,0,0,0 +ALTERATION,0,0,2009,0,0,0,0 +AMBIGUITY,0,0,2009,0,0,0,0 +AMENDATORY,0,0,0,2009,0,0,0 +AMENDMENTS,0,0,0,2009,0,0,0 +ANNOYANCES,2009,0,0,0,0,0,0 +ANNUL,2009,0,0,0,0,0,0 +ANNULMENTS,2009,0,0,0,0,0,0 +ANOMALOUSLY,2009,0,2009,0,0,0,0 +ANTICIPATE,0,0,2009,0,0,0,0 +ANTICIPATION,0,0,2009,0,0,0,0 +ANTITRUST,2009,0,0,2009,0,0,0 +APPEAL,0,0,0,2009,0,0,0 +APPEALS,0,0,0,2009,0,0,0 +APPEARS,0,0,2009,0,0,2009,0 +APPELLEES,0,0,0,2011,0,0,0 +APPROXIMATELY,0,0,2009,0,0,0,0 +APPROXIMATIONS,0,0,2009,0,0,0,0 +ARBITRABILITY,0,0,0,2011,0,0,0 +ARBITRARY,0,0,2009,0,0,0,0 +ARBITRATING,0,0,0,2009,0,0,0 +ARBITRATIVE,0,0,0,2011,0,0,0 +ARGUED,2009,0,0,0,0,0,0 +ARGUMENTS,2009,0,0,0,0,0,0 +ARREST,2009,0,0,0,0,0,0 +RETRIBUTIONS,2009,0,0,0,0,0,0 +BONA,0,0,0,2009,0,0,0 +BOOST,0,2009,0,0,0,0,0 +BOUND,0,0,0,0,0,0,2011 +BOYCOTTING,2009,0,0,0,0,0,0 +BREACHES,2009,0,0,2009,0,0,0 +BREAKAGES,2009,0,0,0,0,0,0 +BREAKS,2009,0,0,0,0,0,0 +BRIBED,2009,0,0,0,0,0,0 +BRIBING,2009,0,0,0,0,0,0 +BURDEN,2009,0,0,0,0,0,0 +BURDENSOME,2009,0,0,0,0,0,0 +CALAMITY,2009,0,0,0,0,0,0 +CANCELLATION,2009,0,0,0,0,0,0 +CANCELS,2009,0,0,0,0,0,0 +CATASTROPHICALLY,2009,0,0,0,0,0,0 +CAUTIONING,2009,0,0,0,0,0,0 +CAUTIOUSNESS,0,0,2009,0,0,0,0 +CEASING,2009,0,0,0,0,0,0 +CENSURED,2009,0,0,0,0,0,0 +CESSION,0,0,0,2009,0,0,0 +CHALLENGING,2009,0,0,0,0,0,0 +CHATTELS,0,0,0,2009,0,0,0 +CIRCUMVENTING,2009,0,0,0,0,0,0 +SHARPLY,2009,0,0,0,0,0,0 +SHORTFALL,2009,0,0,0,0,0,0 +SHUT,2009,0,0,0,0,0,0 +SHUTTING,2009,0,0,0,0,0,0 +SLANDERS,2009,0,0,0,0,0,0 +REPUDIATE,2009,0,0,0,0,0,0 +REPUDIATION,2009,0,0,0,0,0,0 +REQUIRE,0,0,0,0,0,0,2009 +REQUIRES,0,0,0,0,0,0,2009 +RESCINDED,0,0,0,2009,0,0,0 +RESCISSIONS,0,0,0,2009,0,0,0 +RESIGNED,2009,0,0,0,0,0,0 +RESTATE,2009,0,0,0,0,0,0 +RESTATES,2009,0,0,0,0,0,0 +NONTERMINABLE,0,0,0,2011,0,0,0 +NOTARIZATION,0,0,0,2009,0,0,0 +NOTARIZING,0,0,0,2009,0,0,0 +NUISANCE,2009,0,0,0,0,0,0 +NULLIFIED,2009,0,0,2009,0,0,0 +NULLITIES,0,0,0,2009,0,0,0 +OBJECTION,2009,0,0,0,0,0,0 +OBLIGATE,0,0,0,0,0,0,2009 +OBLIGATION,0,0,0,0,0,0,2009 +OBLIGED,0,0,0,0,0,0,2009 +OBLIGOR,0,0,0,2009,0,0,0 +OBSOLESCENCE,2009,0,0,0,0,0,0 +OBSTRUCT,2009,0,0,0,0,0,0 +OBSTRUCTIONS,2009,0,0,0,0,0,0 +OFFEND,2009,0,0,0,0,0,0 +OFFENDING,2009,0,0,0,0,0,0 +OFFEREES,0,0,0,2011,0,0,0 +OMISSIONS,2009,0,0,0,0,0,0 +OMITTING,2009,0,0,0,0,0,0 +OPPORTUNITIES,0,2009,0,0,0,0,0 +OPPOSES,2009,0,0,0,0,0,0 +OPTIMISTIC,0,2009,0,0,0,0,0 +OUTAGE,2009,0,0,0,0,0,0 +OUTPERFORM,0,2009,0,0,0,0,0 +OVERAGE,2009,0,0,0,0,0,0 +OVERBUILDS,2009,0,0,0,0,0,0 +OVERBURDENING,2009,0,0,0,0,0,0 +OVERCHARGED,2009,0,0,0,0,0,0 +OVERCOMES,2009,0,0,0,0,0,0 +OVERESTIMATED,2009,0,0,0,0,0,0 +OVERESTIMATIONS,2009,0,0,0,0,0,0 +OVERLOADS,2009,0,0,0,0,0,0 +OVERLOOKS,2009,0,0,0,0,0,0 +OVERPRODUCED,2009,0,0,0,0,0,0 +OVERRULE,0,0,0,2009,0,0,0 +OVERRUN,2009,0,0,0,0,0,0 +OVERSHADOWED,2009,0,0,0,0,0,0 +OVERSTATED,2009,0,0,0,0,0,0 +OVERSTATING,2009,0,0,0,0,0,0 +OVERSUPPLYING,2009,0,0,0,0,0,0 +OVERTURNING,2009,0,0,0,0,0,0 +OVERVALUING,2009,0,0,0,0,0,0 +PARI,0,0,0,2009,0,0,0 +NECESSITATES,0,0,0,0,0,0,2009 +NEGATIVES,2009,0,0,0,0,0,0 +NEGLECTING,2009,0,0,0,0,0,0 +NEGLIGENT,2009,0,0,0,0,0,0 +BARRED,2009,0,0,0,0,0,0 +BEAUTIFULLY,0,2009,0,0,0,0,0 +BELIEVING,0,0,2009,0,0,0,0 +IDLE,2009,0,0,0,0,0,0 +IGNORED,2009,0,0,0,0,0,0 +ILLEGAL,2009,0,0,0,0,0,0 +ILLEGIBLE,2009,0,0,0,0,0,0 +ILLIQUIDITY,2009,0,0,0,0,0,0 +IMMATURE,2009,0,0,0,0,0,0 +IMPAIRING,2009,0,0,0,0,0,2011 +IMPASSE,2009,0,0,0,0,0,0 +IMPEDES,2009,0,0,0,0,0,0 +IMPENDING,2009,0,0,0,0,0,0 +IMPERIL,2009,0,0,0,0,0,0 +IMPLICATED,2009,0,0,0,0,0,0 +IMPOSED,0,0,0,0,0,0,2009 +IMPOSITIONS,0,0,0,0,0,0,2009 +IMPOUNDED,2009,0,0,0,0,0,0 +IMPRACTICAL,2009,0,0,0,0,0,0 +IMPRECISION,0,0,2009,0,0,0,0 +IMPRESSES,0,2009,0,0,0,0,0 +IMPRISONMENT,2009,0,0,0,0,0,0 +IMPROPERLY,2009,0,0,0,0,0,0 +IMPROVED,0,2009,0,0,0,0,0 +IMPROVING,0,2009,0,0,0,0,0 +INACCESSIBLE,2009,0,0,0,0,0,0 +INACCURATELY,2009,0,0,0,0,0,0 +INACTIVATED,2009,0,0,0,0,0,0 +INACTIVATIONS,2009,0,0,0,0,0,0 +INADEQUATE,2009,0,0,0,0,0,0 +INADVISABILITY,2009,0,0,0,0,0,0 +INASMUCH,0,0,0,2009,0,0,0 +INCAPACITY,2009,0,0,2009,0,0,0 +INCARCERATING,2009,0,0,2009,0,0,0 +INCIDENCE,2009,0,0,0,0,0,0 +INCOMPATIBILITIES,2009,0,0,0,0,0,0 +INCOMPETENCY,2009,0,0,0,0,0,0 +INCOMPLETE,2009,0,0,0,0,0,0 +INCONSISTENCIES,2009,0,0,0,0,0,0 +INCONTESTABILITY,0,0,0,2009,0,0,0 +INCONVENIENT,2009,0,0,0,0,0,0 +INCREDIBLE,0,2009,0,0,0,0,0 +INDECENT,2009,0,0,0,0,0,0 +INDEFINITELY,0,0,2009,0,0,0,0 +INDEMNIFICATIONS,0,0,0,2009,0,0,0 +INDEMNIFYING,0,0,0,2009,0,0,0 +INDEMNITOR,0,0,0,2009,0,0,0 +INDETERMINATE,0,0,2009,0,0,0,0 +INDICTING,2009,0,0,2009,0,0,0 +INEFFECTIVE,2009,0,0,0,0,0,0 +INEFFICIENCY,2009,0,0,0,0,0,0 +INELIGIBLE,2009,0,0,0,0,0,0 +INEQUITY,2009,0,0,0,0,0,0 +INEXPERIENCE,2009,0,0,0,0,0,0 +INFLUENTIAL,0,2009,0,0,0,0,0 +INFRACTIONS,2009,0,0,2009,0,0,0 +INFRINGEMENTS,2009,0,0,0,0,0,0 +INGENUITY,0,2009,0,0,0,0,0 +INHIBITS,0,0,0,0,0,0,2011 +INJUNCTIVE,0,0,0,2009,0,0,0 +INJURIES,2009,0,0,0,0,0,0 +INNOVATE,0,2009,0,0,0,0,0 +INNOVATION,0,2009,0,0,0,0,0 +INNOVATOR,0,2009,0,0,0,0,0 +INQUIRY,2009,0,0,0,0,0,0 +INSIST,0,0,0,0,0,0,2009 +INSISTS,0,0,0,0,0,0,2009 +INSOLVENT,2009,0,0,0,0,0,0 +INSTABILITY,2009,0,2009,0,0,0,0 +INSUFFICIENTLY,2009,0,0,0,0,0,0 +INTANGIBLES,0,0,2009,0,0,0,0 +INTERFERED,2009,0,0,0,0,0,0 +INTERFERING,2009,0,0,0,0,0,0 +INTERPLEADER,0,0,0,2009,0,0,0 +INTERPOSING,0,0,0,2009,0,0,0 +INTERROGATED,0,0,0,2009,0,0,0 +INTERROGATIONS,0,0,0,2009,0,0,0 +INTERROGATORY,0,0,0,2009,0,0,0 +INTERRUPTION,2009,0,0,0,0,0,0 +INTESTATE,0,0,0,2009,0,0,0 +SPORADIC,0,0,2009,0,0,0,0 +STABILIZATIONS,0,2009,0,0,0,0,0 +STABILIZING,0,2009,0,0,0,0,0 +STAGNATE,2009,0,0,0,0,0,0 +STAGNATION,2009,0,0,0,0,0,0 +STATUTES,0,0,0,2009,0,0,0 +STIPULATED,0,0,0,0,0,0,2009 +STIPULATIONS,0,0,0,0,0,0,2009 +STOPPED,2009,0,0,0,0,0,0 +STRAINED,2009,0,0,0,0,0,0 +STRENGTHEN,0,2009,0,0,0,0,0 +STRENGTHS,0,2009,0,0,0,0,0 +STRESSFUL,2009,0,0,0,0,0,0 +STRICTEST,0,0,0,0,0,0,2009 +STRONGER,0,2009,0,0,0,0,0 +SUBCLAUSES,0,0,0,2009,0,0,0 +SUBJECTION,2009,0,0,0,0,0,0 +SUBLICENSEE,0,0,0,2009,0,0,0 +SUBPOENA,2009,0,0,2009,0,0,0 +SUBROGATION,0,0,0,2009,0,0,0 +SUCCEED,0,2009,0,0,0,0,0 +SUCCESS,0,2009,0,0,0,0,0 +SUDDEN,0,0,2009,0,0,0,0 +SUES,2009,0,0,2009,0,0,0 +SUFFERS,2009,0,0,0,0,0,0 +SUGGESTS,0,0,2009,0,0,2009,0 +SUMMONS,2009,0,0,2009,0,0,0 +SUPERSEDEAS,0,0,0,2011,0,0,0 +SURETIES,0,0,0,2009,0,0,0 +SURPASSES,0,2009,0,0,0,0,0 +SUSPECT,2009,0,0,0,0,0,0 +SUSPENDED,2009,0,0,0,0,0,0 +SUSPENSIONS,2009,0,0,0,0,0,0 +SUSPICIOUSLY,2009,0,0,0,0,0,0 +REVISE,0,0,2009,0,0,0,0 +REVOCATIONS,2009,0,0,2009,0,0,0 +REVOKING,2009,0,0,0,0,0,0 +REVOLUTIONIZING,0,2009,0,0,0,0,0 +REWARDS,0,-2020,0,0,0,0,0 +RIDICULING,2009,0,0,0,0,0,0 +RISKIEST,2009,0,2009,0,0,0,0 +RISKY,2009,0,2009,0,0,0,0 +RUMORS,0,0,2009,0,0,0,0 +SACRIFICES,2009,0,0,0,0,0,0 +SATISFACTORILY,0,2009,0,0,0,0,0 +SATISFY,0,2009,0,0,0,0,0 +SCRUTINIZE,2009,0,0,0,0,0,0 +SCRUTINY,2009,0,0,0,0,0,0 +SEIZED,2009,0,0,0,0,0,0 +SELDOMLY,0,0,2009,0,0,2009,0 +SERIOUS,2009,0,0,0,0,0,0 +SETBACKS,2009,0,0,0,0,0,0 +SEVERABILITY,0,0,0,2009,0,0,0 +SEVERANCES,0,0,0,2009,0,0,0 +SEVERITIES,2009,0,0,0,0,0,0 +ENTHUSIASM,0,2009,0,0,0,0,0 +ENTRENCHED,0,0,0,0,0,0,2009 +ERODING,2009,0,0,0,0,0,0 +ERRED,2009,0,0,0,0,0,0 +ERROR,2009,0,0,0,0,0,0 +ESCALATED,2009,0,0,0,0,0,0 +ESCHEATED,0,0,0,2011,0,0,0 +ESCROWING,0,0,0,2011,0,0,0 +EVADED,2009,0,0,0,0,0,0 +EVASIONS,2009,0,0,0,0,0,0 +EVICTING,2009,0,0,0,0,0,0 +EVIDENTIAL,0,0,0,2011,0,0,0 +EXACERBATES,2009,0,0,0,0,0,0 +EXAGGERATE,2009,0,0,0,0,0,0 +EXAGGERATION,2009,0,0,0,0,0,0 +EXCELLENCE,0,2009,0,0,0,0,0 +EXCEPTIONAL,0,2009,0,0,0,0,0 +EXCISED,0,0,0,2009,0,0,0 +EXCLUSIVE,0,2009,0,0,0,0,0 +EXCLUSIVITY,0,2009,0,0,0,0,0 +EXCULPATING,2009,0,0,2009,0,0,0 +EXECUTOR,0,0,0,2009,0,0,0 +EXECUTRIX,0,0,0,2009,0,0,0 +EXONERATED,2009,0,0,0,0,0,0 +EXONERATIONS,2009,0,0,0,0,0,0 +EXPLOITATIVE,2009,0,0,0,0,0,0 +EXPOSE,2009,0,0,0,0,0,0 +EXPOSURE,0,0,2009,0,0,0,0 +EXPROPRIATES,2009,0,0,0,0,0,0 +EXPULSION,2009,0,0,0,0,0,0 +EXTRACORPOREAL,0,0,0,2011,0,0,0 +SOLVENCY,2009,0,0,0,0,0,0 +SOMETIMES,0,0,2009,0,0,2009,0 +SPAMMERS,2014,0,0,0,0,0,0 +TREMENDOUS,0,2009,0,0,0,0,0 +LOCKOUT,2009,0,0,0,0,0,0 +LOSING,2009,0,0,0,0,0,0 +LOWEST,0,0,0,0,2009,0,0 +MAJEURE,0,0,0,2011,0,0,0 +MALFUNCTIONING,2009,0,0,0,0,0,0 +MALICIOUSLY,2009,0,0,0,0,0,0 +MANDATED,0,0,0,0,0,0,2009 +MANDITORILY,0,0,0,0,0,0,2011 +MANIPULATING,2009,0,0,0,0,0,0 +MARKDOWN,2009,0,0,0,0,0,0 +MEDIATE,0,0,0,2009,0,0,0 +MEDIATION,0,0,0,2009,0,0,0 +MERITORIOUS,0,2009,0,0,0,0,0 +MISAPPLIED,2009,0,0,0,0,0,0 +MISAPPROPRIATE,2009,0,0,0,0,0,0 +MISAPPROPRIATION,2009,0,0,0,0,0,0 +MISCALCULATED,2009,0,0,0,0,0,0 +MISCALCULATIONS,2009,0,0,0,0,0,0 +MISCLASSIFICATIONS,2014,0,0,0,0,0,0 +MISCONDUCT,2009,0,0,0,0,0,0 +MISDIRECTED,2009,0,0,0,0,0,0 +MISHANDLES,2009,0,0,0,0,0,0 +MISINFORMED,2009,0,0,0,0,0,0 +MISINTERPRETATION,2009,0,0,0,0,0,0 +MISINTERPRETS,2009,0,0,0,0,0,0 +MISJUDGING,2009,0,0,0,0,0,0 +MISLABELED,2009,0,0,0,0,0,0 +MISLEAD,2009,0,0,0,0,0,0 +MISLED,2009,0,0,0,0,0,0 +MISMANAGES,2009,0,0,0,0,0,0 +MISMATCHES,2009,0,0,0,0,0,0 +MISPRICING,2014,0,0,0,0,0,0 +MISREPRESENTATIONS,2009,0,0,0,0,0,0 +MISS,2009,0,0,0,0,0,0 +WORSE,2009,0,0,0,0,0,0 +WORSENS,2009,0,0,0,0,0,0 +TOLERATES,2009,0,0,0,0,0,0 +TORTIOUS,0,0,0,2009,0,0,0 +TORTUOUSLY,2009,0,0,0,0,0,0 +FINED,2009,0,0,0,0,0,0 +NONAPPEALABLE,0,0,0,2009,0,0,0 +NONCANCELABLE,0,0,0,0,0,0,2009 +NONCOMPLIANCES,2009,0,0,0,0,0,0 +NONCONFORMITIES,2009,0,0,0,0,0,0 +NONCONTRACTUAL,0,0,0,2011,0,0,0 +NONFIDUCIARY,0,0,0,2011,0,0,0 +NONFUNCTIONAL,2009,0,0,0,0,0,0 +DISCLAIMER,2009,0,0,0,0,0,0 +DISCLOSE,2009,0,0,0,0,0,0 +DISCONTINUANCE,2009,0,0,0,0,0,0 +DISCONTINUE,2009,0,0,0,0,0,0 +DISCOURAGE,2009,0,0,0,0,0,0 +DISCREDIT,2009,0,0,0,0,0,0 +DISCREPANCIES,2009,0,0,0,0,0,0 +SPECULATE,0,0,2009,0,0,0,0 +SPECULATION,0,0,2009,0,0,0,0 +TRAGIC,2009,0,0,0,0,0,0 +TRANSPARENCY,0,2009,0,0,0,0,0 +LEGISLATE,0,0,0,2009,0,0,0 +LEGISLATION,0,0,0,2009,0,0,0 +LEGISLATOR,0,0,0,2009,0,0,0 +LIBEL,0,0,0,2009,0,0,0 +LICENSABLE,0,0,0,2011,0,0,0 +CONCEALING,2009,0,0,0,0,0,0 +CONCEDING,2009,0,0,0,0,0,0 +CONCERNED,2009,0,0,0,0,0,0 +CONCILIATIONS,2009,0,0,0,0,0,0 +CONDEMNATION,2009,0,0,0,0,0,0 +CONDEMNOR,0,0,0,2011,0,0,0 +CONDONE,2009,0,0,0,0,0,0 +CONFESSED,2009,0,0,0,0,0,0 +CONFIDENT,0,2009,0,0,0,0,0 +CONFINEMENTS,2009,0,0,0,0,0,0 +CONFISCATED,2009,0,0,0,0,0,0 +CONFISCATIONS,2009,0,0,0,0,0,0 +CONFLICTING,2009,0,0,0,0,0,0 +CONFRONTATIONAL,2009,0,0,0,0,0,0 +CONFRONTS,2009,0,0,0,0,0,0 +CONFUSING,2009,0,2009,0,0,0,0 +CONSENTED,0,0,0,2009,0,0,0 +CONSPIRACIES,2009,0,0,0,0,0,0 +CONSPIRATORS,2009,0,0,0,0,0,0 +CONSPIRING,2009,0,0,0,0,0,0 +CONSTITUTIONALLY,0,0,0,2009,0,0,0 +CONSTRAINED,0,0,0,0,0,0,2009 +CONSTRAINTS,0,0,0,0,0,0,2009 +CONSTRUED,0,0,0,2012,0,0,0 +CONTEND,2009,0,0,0,0,0,0 +CONTENTION,2009,0,0,0,0,0,0 +CONTESTABILITY,0,0,0,2014,0,0,0 +CONTINGENCIES,0,0,2009,0,0,0,0 +CONTINGENTS,0,0,2009,0,0,0,0 +CONTRACTHOLDERS,0,0,0,2009,0,0,0 +CONTRACTION,2009,0,0,0,0,0,0 +CONTRACTUALLY,0,0,0,2009,0,0,0 +CONTRADICTION,2009,0,0,0,0,0,0 +CONTRARY,2009,0,0,0,0,0,0 +CONTRAVENING,0,0,0,2009,0,0,0 +CONTROVERSIES,2009,0,0,0,0,0,0 +CONTROVERTING,0,0,0,2009,0,0,0 +CONVICT,2009,0,0,2009,0,0,0 +CONVICTIONS,2009,0,0,2009,0,0,0 +CORRECTIONS,2009,0,0,0,0,0,0 +CORRUPTING,2009,0,0,0,0,0,0 +CORRUPTNESS,2009,0,0,0,0,0,0 +COUNSEL,0,0,0,2009,0,0,0 +COUNTERCLAIM,2009,0,0,0,0,0,0 +COUNTERFEIT,2009,0,0,0,0,0,0 +COUNTERFEITING,2009,0,0,0,0,0,0 +COUNTERSIGNOR,0,0,0,2011,0,0,0 +COURT,0,0,0,2009,0,0,0 +COVENANT,0,0,0,0,0,0,2011 +CREATIVE,0,2009,0,0,0,0,0 +CRIME,2009,0,0,2009,0,0,0 +CRIMINALIZE,0,0,0,2014,0,0,0 +CRISES,2009,0,0,0,0,0,0 +CRITICISM,2009,0,0,0,0,0,0 +CRITICIZES,2009,0,0,0,0,0,0 +CROSSROAD,0,0,2009,0,0,0,0 +CULPABILITY,2009,0,0,0,0,0,0 +CURTAIL,2009,0,0,0,0,0,0 +CURTAILMENTS,2009,0,0,0,0,0,0 +CUTBACKS,2009,0,0,0,0,0,0 +CYBERCRIME,2014,0,0,0,0,0,0 +TAINTING,2009,0,0,0,0,0,0 +TENDING,0,0,2009,0,0,0,0 +TERMINABLE,0,0,0,2009,0,0,0 +TERMINATING,2009,0,0,0,0,0,0 +TESTAMENTARY,0,0,0,2009,0,0,0 +THENCE,0,0,0,2009,0,0,0 +THEREAT,0,0,0,2009,0,0,0 +THEREOF,0,0,0,2009,0,0,0 +THERETOFOR,0,0,0,2011,0,0,0 +THEREUPON,0,0,0,2009,0,0,0 +THREATENED,2009,0,0,0,0,0,0 +FAILED,2009,0,0,0,0,0,0 +FAILURE,2009,0,0,0,0,0,0 +FALSELY,2009,0,0,0,0,0,0 +FALSIFIES,2009,0,0,0,0,0,0 +FANTASTIC,0,2009,0,0,0,0,0 +FAULT,2009,0,0,0,0,0,0 +FAVORABLE,0,2009,0,0,0,0,0 +FAVORITE,0,2009,0,0,0,0,0 +FELONIES,2009,0,0,2009,0,0,0 +DAMAGED,2009,0,0,0,0,0,0 +DAMPENED,2009,0,0,0,0,0,0 +DANGERS,2009,0,0,0,0,0,0 +DEADLOCKS,2009,0,0,0,0,0,0 +DEBARMENTS,2009,0,0,0,0,0,0 +DECEDENTS,0,0,0,2009,0,0,0 +DECEIVE,2009,0,0,0,0,0,0 +DECEPTION,2009,0,0,0,0,0,0 +DECLARANT,0,0,0,2011,0,0,0 +DECLINING,2009,0,0,0,0,0,0 +DECREES,0,0,0,2009,0,0,0 +DEFALCATION,0,0,0,2009,0,0,0 +DEFAMATORY,2009,0,0,0,0,0,0 +DEFAMING,2009,0,0,0,0,0,0 +DEFAULTS,2009,0,0,0,0,0,0 +DEFEASED,0,0,0,2009,0,0,0 +DEFEAT,2009,0,0,0,0,0,0 +DEFECT,2009,0,0,0,0,0,0 +DEFEND,2009,0,0,0,0,0,0 +DEFENDED,2009,0,0,0,0,0,0 +DEFER,2009,0,0,0,0,0,0 +DEFICIENT,2009,0,0,0,0,0,0 +DEFINITIVELY,0,0,0,0,2009,0,0 +DEFRAUDS,2009,0,0,0,0,0,0 +DEGRADE,2009,0,0,0,0,0,0 +DELAY,2009,0,0,0,0,0,0 +DELEGABLE,0,0,0,2009,0,0,0 +DELETERIOUS,2009,0,0,0,0,0,0 +DELIGHT,0,2009,0,0,0,0,0 +DELIGHTING,0,2009,0,0,0,0,0 +DELINQUENT,2009,0,0,0,0,0,0 +DELISTED,2009,0,0,0,0,0,0 +DEMISED,2009,0,0,0,0,0,0 +DEMOLISHED,2009,0,0,0,0,0,0 +DEMOLITIONS,2009,0,0,0,0,0,0 +DEMOTING,2009,0,0,0,0,0,0 +DEMURRER,0,0,0,2009,0,0,0 +DENIAL,2009,0,0,0,0,0,0 +DENIGRATE,2009,0,0,0,0,0,0 +DENIGRATION,2009,0,0,0,0,0,0 +DEPENDABILITY,0,2009,0,0,0,0,0 +DEPENDANT,0,0,0,0,0,0,2011 +DEPENDENCY,0,0,2009,0,0,0,0 +DEPLETE,2009,0,0,0,0,0,0 +DEPLETION,2009,0,0,0,0,0,0 +DEPOSES,0,0,0,2009,0,0,0 +DEPOSITIONS,0,0,0,2009,0,0,0 +INTRUSION,2009,0,0,0,0,0,0 +INVALIDATES,2009,0,0,0,0,0,0 +INVENT,0,2009,0,0,0,0,0 +INVENTIONS,0,2009,0,0,0,0,0 +INVENTORS,0,2009,0,0,0,0,0 +INVESTIGATING,2009,0,0,0,0,0,0 +INVOLUNTARY,2009,0,0,0,0,0,0 +IRRECOVERABLY,2009,0,0,0,0,0,0 +IRREGULARLY,2009,0,0,0,0,0,0 +IRREVOCABILITY,0,0,0,2011,0,0,0 +JEOPARDIZED,2009,0,0,0,0,0,0 +JUDICIARIES,0,0,0,2009,0,0,0 +JURISDICTION,0,0,0,2009,0,0,0 +JURISPRUDENCE,0,0,0,2009,0,0,0 +JURORS,0,0,0,2009,0,0,0 +JUSTICES,0,0,0,2009,0,0,0 +KNOWINGLY,2009,0,0,0,0,0,0 +DILIGENT,0,2009,0,0,0,0,0 +DIMINISHES,2009,0,0,0,0,0,0 +DIRECTIVES,0,0,0,0,0,0,2009 +DISADVANTAGES,2009,0,0,0,0,0,0 +DISAFFIRMED,0,0,0,2011,0,0,0 +DISAGREED,2009,0,0,0,0,0,0 +DISAGREES,2009,0,0,0,0,0,0 +DISALLOWED,2009,0,0,0,0,0,0 +DISAPPEARANCE,2009,0,0,0,0,0,0 +DISAPPEARS,2009,0,0,0,0,0,0 +DISAPPOINTINGLY,2009,0,0,0,0,0,0 +DISAPPROVAL,2009,0,0,0,0,0,0 +DISAPPROVES,2009,0,0,0,0,0,0 +DISASSOCIATION,2009,0,0,0,0,0,0 +DISASTROUS,2009,0,0,0,0,0,0 +DISAVOWED,2009,0,0,0,0,0,0 +GREAT,0,-2020,0,0,0,0,0 +GREATNESS,0,2009,0,0,0,0,0 +GROUNDLESS,2009,0,0,0,0,0,0 +HAMPER,2009,0,0,0,0,0,0 +HAPPIEST,0,2009,0,0,0,0,0 +HARASS,2009,0,0,0,0,0,0 +HARDSHIP,2009,0,0,0,0,0,0 +HARMFUL,2009,0,0,0,0,0,0 +HARSH,2009,0,0,0,0,0,0 +HARSHNESS,2009,0,0,0,0,0,0 +NONJUDICIAL,0,0,0,2009,0,0,0 +NONPAYMENTS,2009,0,0,0,0,0,0 \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/python-challenge/analysis.csv b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/python-challenge/analysis.csv new file mode 100644 index 0000000000000..5c4a1246021eb --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/python-challenge/analysis.csv @@ -0,0 +1,3877 @@ +Word,Negative,Positive,Uncertainty,Litigious,Strong_Modal,Weak_Modal,Constraining +NONSEVERABLE,0,0,0,2011,0,0,0 +DISFAVOR,2009,0,0,0,0,0,0 +DISGORGE,2009,0,0,0,0,0,0 +COMPLICATES,2009,0,0,0,0,0,0 +COMPLIMENT,0,2009,0,0,0,0,0 +COMPLIMENTS,0,2009,0,0,0,0,0 +MISSTATE,2009,0,0,0,0,0,0 +MISSTATES,2009,0,0,0,0,0,0 +MISTAKE,2009,0,0,0,0,0,0 +MISTAKING,2009,0,0,0,0,0,0 +MISUNDERSTANDING,2009,0,0,0,0,0,0 +MISUSED,2009,0,0,0,0,0,0 +MONOPOLISTS,2009,0,0,0,0,0,0 +MONOPOLIZES,2009,0,0,0,0,0,0 +MORATORIUM,2009,0,0,0,0,0,0 +MOTHBALLING,2009,0,0,0,0,0,0 +SLOW,2009,0,0,0,0,0,0 +SLOWER,2009,0,0,0,0,0,0 +SLOWNESS,2009,0,0,0,0,0,0 +SMOOTH,0,2009,0,0,0,0,0 +DEPRECATION,2009,0,0,0,0,0,0 +DEPRESSING,2009,0,0,0,0,0,0 +DEPRIVES,2009,0,0,0,0,0,0 +DEROGATE,0,0,0,2009,0,0,0 +DEROGATION,0,0,0,2009,0,0,0 +DESIRABLE,0,2009,0,0,0,0,0 +DESTABILIZATION,2009,0,0,0,0,0,0 +DESTINED,0,2009,0,0,0,0,0 +DESTROYS,2009,0,0,0,0,0,0 +DETAINED,2009,0,0,0,0,0,0 +DETER,2009,0,0,0,0,0,0 +DETERIORATING,2009,0,0,0,0,0,0 +DETERRENCE,2009,0,0,0,0,0,0 +DETERRING,2009,0,0,0,0,0,0 +DETRACTING,2009,0,0,0,0,0,0 +DETRIMENTS,2009,0,0,0,0,0,0 +DEVALUING,2009,0,0,0,0,0,0 +DEVASTATION,2009,0,0,0,0,0,0 +DEVIATING,2009,0,2009,0,0,0,0 +DEVOLVE,2009,0,0,0,0,0,0 +DICTATE,0,0,0,0,0,0,2009 +DIFFER,0,0,2009,0,0,0,0 +DIFFICULT,2009,0,0,0,0,0,0 +ASSAULT,2009,0,0,0,0,0,0 +ASSERTABLE,0,0,0,2011,0,0,0 +ASSUMABLE,0,0,0,2009,0,0,0 +ASSUMING,0,0,2009,0,0,0,0 +ASSURED,0,2009,0,0,0,0,0 +ATTAINED,0,2009,0,0,0,0,0 +ATTAINS,0,2009,0,0,0,0,0 +ATTESTED,0,0,0,2009,0,0,0 +ATTORNEYS,0,0,0,2009,0,0,0 +ATTRACTIVENESS,0,2009,0,0,0,0,0 +BAD,2009,0,0,0,0,0,0 +BAILEES,0,0,0,2011,0,0,0 +BAILOUT,2009,0,0,0,0,0,0 +BANKRUPTCIES,2009,0,0,0,0,0,0 +BANKRUPTS,2009,0,0,0,0,0,0 +CLAIM,0,0,0,2009,0,0,0 +CLAIMHOLDER,0,0,0,2014,0,0,0 +CLARIFICATIONS,0,0,2009,0,0,0,0 +CLOSED,-2020,0,0,0,0,0,0 +CLOSINGS,2009,0,0,0,0,0,0 +CODEFENDANTS,0,0,0,2011,0,0,0 +CODIFICATIONS,0,0,0,2009,0,0,0 +CODIFYING,0,0,0,2009,0,0,0 +COERCING,2009,0,0,0,0,0,0 +COLLABORATES,0,2009,0,0,0,0,0 +COLLABORATIVE,0,2009,0,0,0,0,0 +COLLAPSED,2009,0,0,0,0,0,0 +COLLISIONS,2009,0,0,0,0,0,0 +COLLUDING,2009,0,0,0,0,0,0 +POSES,2009,0,0,0,0,0,0 +POSSESSORY,0,0,0,2009,0,0,0 +POSSIBLY,0,0,2009,0,0,2009,0 +POSTJUDGMENT,0,0,0,2011,0,0,0 +POSTPONEMENTS,2009,0,0,0,0,0,0 +WRITEDOWNS,2009,0,0,0,0,0,0 +WRONG,2009,0,0,0,0,0,0 +WRONGFULLY,2009,0,0,0,0,0,0 +HONORABLE,0,-2020,0,0,0,0,0 +HOSTILE,2009,0,0,0,0,0,0 +REASSESS,0,0,2009,0,0,0,0 +REASSESSMENT,2009,0,2009,0,0,0,0 +REASSIGNING,2009,0,0,0,0,0,0 +REBOUND,0,2009,0,0,0,0,0 +REBUTS,0,0,0,2009,0,0,0 +REBUTTALS,0,0,0,2009,0,0,0 +RECALCULATED,0,0,2009,0,0,0,0 +RECALCULATIONS,0,0,2009,0,0,0,0 +RECALLS,2009,0,0,0,0,0,0 +RECESSIONS,2009,0,0,0,0,0,0 +RECONSIDER,0,0,2009,0,0,0,0 +RECORDATION,0,0,0,2009,0,0,0 +RECOURSE,0,0,0,2009,0,0,0 +RECUSAL,0,0,0,2011,0,0,0 +RECUSING,0,0,0,2011,0,0,0 +REDACTION,2009,0,0,2009,0,0,0 +REDEFAULTS,2014,0,0,0,0,0,0 +REDRESSING,2009,0,0,0,0,0,0 +REFERENDA,0,0,0,2009,0,0,0 +REFILED,0,0,0,2009,0,0,0 +REFRAINING,0,0,0,0,0,0,2009 +REFUSE,2009,0,0,0,0,0,0 +REGAIN,0,2009,0,0,0,0,0 +REGULATED,0,0,0,2009,0,0,0 +REGULATIONS,0,0,0,2009,0,0,0 +REGULATORY,0,0,0,2009,0,0,0 +REHEARINGS,0,0,0,2009,0,0,0 +VARIABILITY,0,0,2009,0,0,0,0 +VARIANCE,0,0,2009,0,0,0,0 +VARIATION,0,0,2009,0,0,0,0 +VARY,0,0,2009,0,0,0,0 +VERDICT,2009,0,0,2009,0,0,0 +VETOED,2009,0,0,0,0,0,0 +VICTIMS,2009,0,0,0,0,0,0 +VIOLATING,2009,0,0,0,0,0,0 +VIOLATOR,2009,0,0,0,0,0,0 +VIOLENTLY,2009,0,0,0,0,0,0 +VITIATING,2009,0,0,0,0,0,0 +VOIDING,2009,0,0,2009,0,0,0 +VULNERABILITIES,2009,0,0,0,0,0,0 +WARN,2009,0,0,0,0,0,0 +WARNS,2009,0,0,0,0,0,0 +WASTEFUL,2009,0,0,0,0,0,0 +WEAKENED,2009,0,0,0,0,0,0 +WEAKEST,2009,0,0,0,0,0,0 +DISHONESTLY,2009,0,0,0,0,0,0 +DISHONORABLY,2009,0,0,0,0,0,0 +DISINTERESTEDNESS,2009,0,0,0,0,0,0 +DISMAL,2009,0,0,0,0,0,0 +DISMISSALS,2009,0,0,0,0,0,0 +DISORDERLY,2009,0,0,0,0,0,0 +DISPARAGEMENTS,2009,0,0,0,0,0,0 +DISPARITIES,2009,0,0,0,0,0,0 +DISPLACEMENT,2009,0,0,0,0,0,0 +DISPOSE,2009,0,0,0,0,0,0 +DISPOSSESSES,2009,0,0,0,0,0,0 +DISPROPORTION,2009,0,0,0,0,0,0 +DISPUTE,2009,0,0,0,0,0,0 +DISQUALIFICATION,2009,0,0,0,0,0,0 +DISQUALIFY,2009,0,0,0,0,0,0 +DISREGARDING,2009,0,0,0,0,0,0 +DISRUPT,2009,0,0,0,0,0,0 +DISRUPTIONS,2009,0,0,0,0,0,0 +DISSATISFIED,2009,0,0,0,0,0,0 +DISSENTERS,2009,0,0,0,0,0,0 +DISSIDENTS,2009,0,0,0,0,0,0 +DISTINCTIONS,0,2009,0,0,0,0,0 +DISTORT,2009,0,0,0,0,0,0 +DISTORTIONS,2009,0,0,0,0,0,0 +DISTRACTING,2009,0,0,0,0,0,0 +DISTRAINT,0,0,0,2009,0,0,0 +DISTRIBUTEES,0,0,0,2009,0,0,0 +DISTURBED,2009,0,0,0,0,0,0 +DIVERT,2009,0,0,0,0,0,0 +DIVEST,2009,0,0,0,0,0,0 +DIVESTITURES,2009,0,0,0,0,0,0 +DIVORCE,2009,0,0,0,0,0,0 +DIVULGES,2009,0,0,0,0,0,0 +DOCKETING,0,0,0,2009,0,0,0 +DOUBTED,2009,0,2009,0,0,0,0 +DOWNGRADED,2009,0,0,0,0,0,0 +DOWNSIZED,2009,0,0,0,0,0,0 +DOWNTIME,2009,0,0,0,0,0,0 +DOWNWARD,2009,0,0,0,0,0,0 +DRASTICALLY,2009,0,0,0,0,0,0 +DROPPED,2009,0,0,0,0,0,0 +DURESS,2009,0,0,0,0,0,0 +EARMARK,0,0,0,0,0,0,2009 +EASIER,0,2009,0,0,0,0,0 +EFFECTIVE,0,-2020,0,0,0,0,0 +EFFICIENTLY,0,2009,0,0,0,0,0 +EMBARGO,2009,0,0,0,0,0,0 +EMBARRASS,2009,0,0,0,0,0,0 +EMBARRASSMENT,2009,0,0,0,0,0,0 +EMBEZZLEMENT,2009,0,0,0,0,0,0 +EMBEZZLING,2009,0,0,0,0,0,0 +EMPOWERS,0,2009,0,0,0,0,0 +ENABLING,0,2009,0,0,0,0,0 +ENCOURAGING,0,2009,0,0,0,0,0 +ENCROACHING,2009,0,0,0,0,0,0 +ENCUMBERED,2009,0,0,2009,0,0,2009 +ENCUMBRANCER,0,0,0,2009,0,0,0 +ENDANGERED,2009,0,0,0,0,0,0 +ENDORSEE,0,0,0,2011,0,0,0 +ENHANCE,0,2009,0,0,0,0,0 +ENHANCES,0,2009,0,0,0,0,0 +ENJOINING,2009,0,0,0,0,0,0 +ENJOYABLY,0,2009,0,0,0,0,0 +ENJOYS,0,2009,0,0,0,0,0 +ENTAILS,0,0,0,0,0,0,2009 +PATENTEE,0,0,0,2014,0,0,0 +PENALIZES,2009,0,0,0,0,0,0 +PENDING,0,0,2009,0,0,0,0 +PERFECTS,0,2009,0,0,0,0,0 +PERJURY,2009,0,0,2009,0,0,0 +PERMITTED,0,0,0,0,0,0,2009 +PERPETRATE,2009,0,0,2009,0,0,0 +PERPETRATION,2009,0,0,2009,0,0,0 +PERSISTENT,2009,0,0,0,0,0,0 +PERSONAM,0,0,0,2011,0,0,0 +PETITION,0,0,0,2009,0,0,0 +PETITIONING,0,0,0,2009,0,0,0 +PICKETED,2009,0,0,0,0,0,0 +HENCEFORTH,0,0,0,2009,0,0,0 +HEREDITAMENTS,0,0,0,2009,0,0,0 +HEREIN,0,0,0,2009,0,0,0 +HEREINBELOW,0,0,0,2009,0,0,0 +HERETOFORE,0,0,0,2009,0,0,0 +HEREWITH,0,0,0,2009,0,0,0 +HINDER,2009,0,0,0,0,0,0 +HINDRANCE,2009,0,0,0,0,0,0 +WHATSOEVER,0,0,0,2009,0,0,0 +WHEREAT,0,0,0,2009,0,0,0 +WHEREOF,0,0,0,2009,0,0,0 +WHEREUPON,0,0,0,2009,0,0,0 +WHOMSOEVER,0,0,0,2009,0,0,0 +WILLFUL,0,0,0,2009,0,0,0 +WINNER,0,2009,0,0,0,0,0 +WITNESSES,0,0,0,2009,0,0,0 +FLUCTUATES,0,0,2009,0,0,0,0 +FORBADE,0,0,0,2009,0,0,2011 +FORBEARING,0,0,0,2009,0,0,0 +FORBIDDING,2009,0,0,0,0,0,2009 +FORCING,2009,0,0,0,0,0,0 +FORECLOSE,2009,0,0,0,0,0,0 +FORECLOSURE,2009,0,0,0,0,0,0 +FOREGONE,2009,0,0,0,0,0,0 +FORESTALLS,2009,0,0,0,0,0,0 +FORFEITED,2009,0,0,0,0,0,0 +FORFEITURES,2009,0,0,0,0,0,0 +FORWHICH,0,0,0,2012,0,0,0 +FRAUDULENT,2009,0,0,0,0,0,0 +FRIVOLOUSLY,2009,0,0,0,0,0,0 +FRUSTRATING,2009,0,0,0,0,0,0 +FUGITIVE,-2020,0,0,2009,0,0,0 +GAINED,0,2009,0,0,0,0,0 +GRANTOR,0,0,0,2009,0,0,0 +DISGORGING,2009,0,0,0,0,0,0 +DISHONEST,2009,0,0,0,0,0,0 +LITIGANT,2009,0,0,2009,0,0,0 +LITIGATES,2009,0,0,2009,0,0,0 +LITIGATOR,0,0,0,2009,0,0,0 +LACKLUSTER,2009,0,0,0,0,0,0 +LAGGING,2009,0,0,0,0,0,0 +LAPSES,2009,0,0,0,0,0,0 +LAW,0,0,0,2009,0,0,0 +LAWMAKERS,0,0,0,2009,0,0,0 +LAWSUITS,0,0,0,2009,0,0,0 +LAYOFFS,2009,0,0,0,0,0,0 +LEGALESE,0,0,0,2009,0,0,0 +LEGALIZE,0,0,0,2009,0,0,0 +LEGALLY,0,0,0,2009,0,0,0 +NONPRODUCING,2009,0,0,0,0,0,0 +LIQUIDATE,2009,0,0,0,0,0,0 +LIQUIDATION,2009,0,0,0,0,0,0 +BENEFICIALLY,0,2009,0,0,0,0,0 +BENEFITED,0,2009,0,0,0,0,0 +BEST,0,2012,0,0,2009,0,0 +POPULAR,0,2009,0,0,0,0,0 +REINTERPRETED,0,0,2009,0,0,0,0 +REJECTED,2009,0,0,0,0,0,0 +REJECTS,2009,0,0,0,0,0,0 +RELINQUISHES,2009,0,0,0,0,0,0 +RELUCTANCE,2009,0,0,0,0,0,0 +REMANDING,0,0,0,2009,0,0,0 +REMEDIATING,0,0,0,2009,0,0,0 +REMISED,0,0,0,2011,0,0,0 +RENEGOTIATING,2009,0,0,0,0,0,0 +RENOUNCED,2009,0,0,0,0,0,0 +RENOUNCING,2009,0,0,0,0,0,0 +REPLEVIN,0,0,0,2009,0,0,0 +REPOSSESSION,2009,0,0,0,0,0,0 +TURBULENCE,2009,0,2009,0,0,0,0 +UNACCEPTABLY,2009,0,0,0,0,0,0 +UNANTICIPATED,2009,0,0,0,0,0,0 +UNATTRACTIVE,2009,0,0,0,0,0,0 +UNAVOIDABLE,2009,0,0,0,0,0,0 +UNCERTAINLY,0,0,2009,0,0,2009,0 +UNCOLLECTABLE,2009,0,0,0,0,0,0 +UNCOLLECTIBLES,2009,0,0,0,0,0,0 +UNCONFIRMED,0,0,2009,0,0,0,0 +UNCONSTITUTIONALITY,0,0,0,2009,0,0,0 +UNCONTROLLABLY,2009,0,0,0,0,0,0 +UNCOVERED,2009,0,0,0,0,0,0 +UNDEFEASED,0,0,0,2012,0,0,0 +UNDERCAPITALIZED,2009,0,0,0,0,0,0 +UNDERESTIMATE,2009,0,0,0,0,0,0 +UNDERESTIMATION,2009,0,0,0,0,0,0 +UNDERMINED,2009,0,0,0,0,0,0 +UNDERPAYMENT,2009,0,0,0,0,0,0 +UNDERPERFORMANCE,2009,0,0,0,0,0,0 +UNDERPRODUCED,2009,0,0,0,0,0,0 +UNDERSTATED,2009,0,0,0,0,0,0 +UNDERSTATING,2009,0,0,0,0,0,0 +UNDESIRABLE,2009,0,0,0,0,0,0 +UNDETERMINABLE,0,0,2009,0,0,0,0 +UNDISPUTED,0,0,0,0,2009,0,0 +UNDULY,2009,0,0,0,0,0,0 +UNEMPLOYED,2009,0,0,0,0,0,0 +UNENFORCEABILITY,0,0,0,2009,0,0,0 +UNETHICAL,2009,0,0,0,0,0,0 +UNEXPECTEDLY,2009,0,2009,0,0,0,0 +UNFAMILIARITY,0,0,2009,0,0,0,0 +UNFAVOURABLE,2011,0,0,0,0,0,0 +UNFORECASTED,0,0,2011,0,0,0,0 +UNFORTUNATE,2009,0,0,0,0,0,0 +UNFULFILLED,2009,0,0,0,0,0,0 +UNIDENTIFIABLE,0,0,2009,0,0,0,0 +UNINTENTIONAL,2009,0,0,0,0,0,0 +UNJUSTIFIABLY,2009,0,0,0,0,0,0 +UNKNOWINGLY,2009,0,0,0,0,0,0 +UNLAWFULLY,2009,0,0,2009,0,0,0 +UNMARKETABLE,2009,0,0,0,0,0,0 +UNNECESSARILY,2009,0,0,0,0,0,0 +UNOBTAINABLE,2009,0,0,0,0,0,0 +UNPERFORMED,2009,0,0,0,0,0,0 +UNPREDICTABLE,2009,0,2009,0,0,0,0 +UNPROFITABILITY,2011,0,0,0,0,0,0 +UNQUALIFIED,2009,0,0,0,0,0,0 +UNREASONABLE,2009,0,0,0,0,0,0 +UNRECONCILED,0,0,2011,0,0,0,0 +UNRELIABLE,2009,0,0,0,0,0,0 +UNRESOLVED,2009,0,0,0,0,0,0 +UNSALEABLE,2009,0,0,0,0,0,0 +UNSCHEDULED,2009,0,0,0,0,0,0 +UNSETTLED,0,0,2009,0,0,0,0 +UNSPECIFIED,0,0,2009,0,0,0,0 +UNSUBSTANTIATED,2009,0,0,0,0,0,0 +UNSUITABLE,2009,0,0,0,0,0,0 +UNSURPASSED,0,2009,0,0,2009,0,0 +UNTENABLE,2009,0,0,0,0,0,0 +UNTRUSTED,2014,0,0,0,0,0,0 +UNTRUTHFULNESS,2009,0,0,0,0,0,0 +UNUSUALLY,0,0,2009,0,0,0,0 +UNWILLING,2009,0,0,0,0,0,0 +UPTURN,0,2009,0,0,0,0,0 +USURIOUS,2009,0,0,2009,0,0,0 +USURPING,2009,0,0,2009,0,0,0 +VAGUE,0,0,2012,0,0,0,0 +VAGUER,0,0,2012,0,0,0,0 +PLAINTIFFS,2009,0,0,2009,0,0,0 +PLEADING,2009,0,0,2009,0,0,0 +PLEASANT,0,2009,0,0,0,0,0 +PLED,2009,0,0,0,0,0,0 +PLEDGEES,0,0,0,2011,0,0,0 +PLEDGORS,0,0,0,2012,0,0,0 +COMPELLING,0,0,0,0,0,0,2011 +COMPLAINANT,0,0,0,2009,0,0,0 +COMPLAINS,2009,0,0,0,0,0,0 +COMMITS,0,0,0,0,0,0,2009 +LIKELIHOOD,0,0,2009,0,0,0,0 +LIMITING,0,0,0,0,0,0,2009 +RESTRAIN,0,0,0,0,0,0,2009 +RESTRAINT,0,0,0,0,0,0,2009 +RESTRICTING,0,0,0,0,0,0,2009 +RESTRICTIVELY,0,0,0,0,0,0,2009 +RESTRUCTURED,2009,0,0,0,0,0,0 +RETALIATE,2009,0,0,0,0,0,0 +RETALIATION,2009,0,0,0,0,0,0 +PRECAUTION,0,0,2009,0,0,0,0 +PRECIPITOUS,2009,0,0,0,0,0,0 +PRECLUDES,2009,0,0,0,0,0,2009 +PREDATORY,2009,0,0,0,0,0,0 +PREDECEASING,0,0,0,2009,0,0,0 +PREDICTING,0,0,2009,0,0,0,0 +PREDICTOR,0,0,2009,0,0,0,0 +PREEMINENT,0,2009,0,0,0,0,0 +PREJUDICES,2009,0,0,2009,0,0,0 +PRELIMINARY,0,0,2009,0,0,0,0 +PREMIERE,0,2009,0,0,0,0,0 +PRESTIGE,0,2009,0,0,0,0,0 +PRESUMED,0,0,2009,0,0,0,0 +PRESUMPTIONS,0,0,2009,0,0,0,0 +PREVENTED,0,0,0,0,0,0,2009 +PRIMA,0,0,0,2009,0,0,0 +PROBABILISTIC,0,0,2009,0,0,0,0 +PROBABLY,0,0,2009,0,0,0,0 +PROBATING,0,0,0,2009,0,0,0 +PROBATIONER,0,0,0,2009,0,0,0 +PROBLEMATIC,2009,0,0,0,0,0,0 +PROFICIENT,0,2009,0,0,0,0,0 +PROFITABLY,0,2009,0,0,0,0,0 +PROGRESSING,0,2009,0,0,0,0,0 +PROHIBITION,0,0,0,0,0,0,2009 +PROHIBITORY,0,0,0,0,0,0,2009 +PROLONGATIONS,2009,0,0,0,0,0,0 +PROMULGATE,0,0,0,2009,0,0,0 +PROMULGATION,0,0,0,2009,0,0,0 +PRONE,2009,0,0,0,0,0,0 +PROSECUTED,2009,0,0,2009,0,0,0 +PROSECUTIONS,2009,0,0,2009,0,0,0 +PROSPERED,0,2009,0,0,0,0,0 +PROSPERS,0,2009,0,0,0,0,0 +PROTESTERS,2009,0,0,0,0,0,0 +PROTESTS,2009,0,0,0,0,0,0 +PROVISOES,0,0,0,2009,0,0,0 +PROVOKES,2009,0,0,0,0,0,0 +PUNISHES,2009,0,0,0,0,0,0 +PUNITIVE,2009,0,0,0,0,0,0 +PURPORTING,2009,0,0,0,0,0,0 +QUESTIONABLY,2009,0,0,0,0,0,0 +QUIT,2009,0,0,0,0,0,0 +RACKETEER,2009,0,0,0,0,0,0 +RANDOMIZED,0,0,2009,0,0,0,0 +RANDOMNESS,0,0,2009,0,0,0,0 +RATIONALIZATION,2009,0,0,0,0,0,0 +RATIONALIZES,2009,0,0,0,0,0,0 +ABANDONMENT,2009,0,0,0,0,0,0 +ABDICATES,2009,0,0,0,0,0,0 +ABERRANT,2009,0,0,0,0,0,0 +ABETTING,2009,0,0,0,0,0,0 +ABIDING,0,0,0,0,0,0,2009 +ABNORMALITY,2009,0,0,0,0,0,0 +ABOLISHES,2009,0,0,0,0,0,0 +ABROGATED,2009,0,0,2009,0,0,0 +ABROGATIONS,2009,0,0,2009,0,0,0 +ABSENCE,2009,0,0,0,0,0,0 +ABSOLVED,0,0,0,2009,0,0,0 +ABUNDANT,0,2009,0,0,0,0,0 +ABUSING,2009,0,0,0,0,0,0 +ACCESSION,0,0,0,2009,0,0,0 +ACCIDENTALLY,2009,0,0,0,0,0,0 +ACCOMPLISHED,0,2009,0,0,0,0,0 +ACCOMPLISHMENTS,0,2009,0,0,0,0,0 +ACCUSED,2009,0,0,0,0,0,0 +ACHIEVED,0,2009,0,0,0,0,0 +ACHIEVING,0,2009,0,0,0,0,0 +ACQUIESCING,2009,0,0,0,0,0,0 +ACQUITS,2009,0,0,2009,0,0,0 +ACQUITTANCES,0,0,0,2011,0,0,0 +ADEQUATELY,0,2009,0,0,0,0,0 +ADJOURNMENT,0,0,0,2009,0,0,0 +ADJUDGED,0,0,0,2009,0,0,0 +ADJUDICATED,0,0,0,2009,0,0,0 +ADJUDICATIONS,0,0,0,2009,0,0,0 +ADJUDICATORY,0,0,0,2009,0,0,0 +ADMISSION,0,0,0,2009,0,0,0 +ADULTERATING,2009,0,0,0,0,0,0 +ADVANCEMENTS,0,2009,0,0,0,0,0 +ADVANTAGED,0,2009,0,0,0,0,0 +ADVERSARIAL,2009,0,0,0,0,0,0 +ADVERSELY,2009,0,0,0,0,0,0 +AFFIDAVITS,0,0,0,2009,0,0,0 +AFOREMENTIONED,0,0,0,2009,0,0,0 +AFTERMATHS,2009,0,0,0,0,0,0 +AGGRAVATES,2009,0,0,0,0,0,0 +AGGRIEVED,0,0,0,2009,0,0,0 +ALIENATED,2009,0,0,0,0,0,0 +ALIENATIONS,2009,0,0,0,0,0,0 +ALLEGED,2009,0,0,2009,0,0,0 +ALLIANCE,0,2009,0,0,0,0,0 +ALTERATIONS,0,0,2009,0,0,0,0 +AMBIGUOUS,0,0,2009,0,0,0,0 +AMENDED,0,0,0,2009,0,0,0 +AMENDS,0,0,0,2009,0,0,0 +ANNOYED,2009,0,0,0,0,0,0 +ANNULLED,2009,0,0,0,0,0,0 +ANNULS,2009,0,0,0,0,0,0 +ANOMALY,2009,0,2009,0,0,0,0 +ANTICIPATED,0,0,2009,0,0,0,0 +ANTICIPATIONS,0,0,2009,0,0,0,0 +ANYWISE,0,0,0,2009,0,0,0 +APPEALABLE,0,0,0,2009,0,0,0 +APPEAR,0,0,2009,0,0,0,0 +APPELLANT,0,0,0,2009,0,0,0 +APPOINTOR,0,0,0,2011,0,0,0 +APPROXIMATES,0,0,2009,0,0,0,0 +APPURTENANCE,0,0,0,2009,0,0,0 +ARBITRAL,0,0,0,2009,0,0,0 +ARBITRATE,0,0,0,2009,0,0,0 +ARBITRATION,0,0,0,2009,0,0,0 +ARBITRATOR,0,0,0,2009,0,0,0 +ARGUING,2009,0,0,0,0,0,0 +ARREARAGE,2009,0,0,2009,0,0,0 +ARRESTED,2009,0,0,0,0,0,0 +RETROCEDE,0,0,0,2011,0,0,0 +BOLSTERED,0,2009,0,0,0,0,0 +BONAFIDE,0,0,0,2009,0,0,0 +BOOSTED,0,2009,0,0,0,0,0 +BOUNDED,0,0,0,0,0,0,2009 +BOYCOTTS,2009,0,0,0,0,0,0 +BREACHING,2009,0,0,2009,0,0,0 +BREAKDOWN,2009,0,0,0,0,0,0 +BREAKTHROUGH,0,2009,0,0,0,0,0 +BRIBERIES,2009,0,0,0,0,0,0 +BRIDGE,-2020,0,0,0,0,0,0 +BURDENED,2009,0,0,0,0,0,0 +BURNED,2009,0,0,0,0,0,0 +CANCEL,2009,0,0,0,0,0,0 +CANCELLATIONS,2009,0,0,0,0,0,0 +CARELESS,2009,0,0,0,0,0,0 +CATASTROPHE,2009,0,0,0,0,0,0 +CAUTION,2009,0,0,0,0,0,0 +CAUTIONS,2009,0,0,0,0,0,0 +CEASE,2009,0,0,0,0,0,0 +CEDANT,0,0,0,2012,0,0,0 +CENSURES,2009,0,0,0,0,0,0 +CHALLENGE,2009,0,0,0,0,0,0 +CHARGEOFFS,2009,0,0,0,0,0,0 +CHOATE,0,0,0,2011,0,0,0 +CIRCUMVENTION,2009,0,0,0,0,0,0 +SHOCKED,2009,0,0,0,0,0,0 +SHORTFALLS,2009,0,0,0,0,0,0 +SHUTDOWN,2009,0,0,0,0,0,0 +SLANDER,2009,0,0,0,0,0,0 +REPUDIATED,2009,0,0,0,0,0,0 +REPUDIATIONS,2009,0,0,0,0,0,0 +REQUIRED,0,0,0,0,0,0,2009 +REQUIRING,0,0,0,0,0,0,2009 +RESCINDING,0,0,0,2009,0,0,0 +RESIGN,2009,0,0,0,0,0,0 +RESIGNING,2009,0,0,0,0,0,0 +RESTATED,2009,0,0,0,0,0,0 +RESTATING,2009,0,0,0,0,0,0 +NONUSURIOUS,0,0,0,2011,0,0,0 +NOTARIZATIONS,0,0,0,2009,0,0,0 +NOTARY,0,0,0,2009,0,0,0 +NUISANCES,2009,0,0,0,0,0,0 +NULLIFIES,2009,0,0,2009,0,0,0 +NULLITY,0,0,0,2009,0,0,0 +OBJECTIONABLE,2009,0,0,0,0,0,0 +OBLIGATED,0,0,0,0,0,0,2009 +OBLIGATIONS,0,0,0,0,0,0,2009 +OBLIGEE,0,0,0,2009,0,0,0 +OBLIGORS,0,0,0,2009,0,0,0 +OBSOLETE,2009,0,0,0,0,0,0 +OBSTRUCTED,2009,0,0,0,0,0,0 +OCCASIONALLY,0,0,2009,0,0,2009,0 +OFFENDED,2009,0,0,0,0,0,0 +OFFENDS,2009,0,0,0,0,0,0 +OFFEROR,0,0,0,2009,0,0,0 +OMIT,2009,0,0,0,0,0,0 +ONEROUS,2009,0,0,0,0,0,0 +OPPORTUNITY,0,2009,0,0,0,0,0 +OPPOSING,2009,0,0,0,0,0,0 +OPTIONEE,0,0,0,2009,0,0,0 +OUTAGES,2009,0,0,0,0,0,0 +OUTPERFORMED,0,2009,0,0,0,0,0 +OVERAGES,2009,0,0,0,0,0,0 +OVERBUILT,2009,0,0,0,0,0,0 +OVERCAPACITIES,2009,0,0,0,0,0,0 +OVERCHARGES,2009,0,0,0,0,0,0 +OVERCOMING,2009,0,0,0,0,0,0 +OVERESTIMATES,2009,0,0,0,0,0,0 +OVERLOAD,2009,0,0,0,0,0,0 +OVERLOOK,2009,0,0,0,0,0,0 +OVERPAID,2009,0,0,0,0,0,0 +OVERPRODUCES,2009,0,0,0,0,0,0 +OVERRULED,0,0,0,2009,0,0,0 +OVERRUNNING,2009,0,0,0,0,0,0 +OVERSHADOWING,2009,0,0,0,0,0,0 +OVERSTATEMENT,2009,0,0,0,0,0,0 +OVERSUPPLIED,2009,0,0,0,0,0,0 +OVERTLY,2009,0,0,0,0,0,0 +OVERTURNS,2009,0,0,0,0,0,0 +PANIC,2009,0,0,0,0,0,0 +NEARLY,0,0,2009,0,0,2009,0 +NECESSITATING,0,0,0,0,0,0,2009 +NEGLECT,2009,0,0,0,0,0,0 +NEGLECTS,2009,0,0,0,0,0,0 +NEGLIGENTLY,2009,0,0,0,0,0,0 +BARRIER,2009,0,0,0,0,0,0 +BELIEVE,0,0,2009,0,0,0,0 +IDLED,2009,0,0,0,0,0,0 +IGNORES,2009,0,0,0,0,0,0 +ILLEGALITIES,2009,0,0,0,0,0,0 +ILLICIT,2009,0,0,0,0,0,0 +IMBALANCE,2009,0,0,0,0,0,0 +IMMORAL,2009,0,0,0,0,0,0 +IMPAIRMENT,2009,0,0,0,0,0,2011 +IMPASSES,2009,0,0,0,0,0,0 +IMPEDIMENT,2009,0,0,0,0,0,0 +IMPERATIVE,2009,0,0,0,0,0,0 +IMPERMISSIBLE,2009,0,0,0,0,0,0 +IMPLICATES,2009,0,0,0,0,0,0 +IMPOSES,0,0,0,0,0,0,2009 +IMPOSSIBILITY,2009,0,0,0,0,0,0 +IMPOUNDING,2009,0,0,0,0,0,0 +IMPRACTICALITIES,2009,0,0,0,0,0,0 +IMPRECISIONS,0,0,2009,0,0,0,0 +IMPRESSING,0,2009,0,0,0,0,0 +IMPROBABILITY,0,0,2009,0,0,0,0 +IMPROPRIETIES,2009,0,0,0,0,0,0 +IMPROVEMENT,0,2009,0,0,0,0,0 +IMPRUDENT,2009,0,0,0,0,0,0 +INACCURACIES,2009,0,0,0,0,0,0 +INACTION,2009,0,0,0,0,0,0 +INACTIVATES,2009,0,0,0,0,0,0 +INACTIVITY,2009,0,0,0,0,0,0 +INADEQUATELY,2009,0,0,0,0,0,0 +INADVISABLE,2009,0,0,0,0,0,0 +INATTENTION,2009,0,0,0,0,0,0 +INCARCERATE,2009,0,0,2009,0,0,0 +INCARCERATION,2009,0,0,2009,0,0,0 +INCIDENCES,2009,0,0,0,0,0,0 +INCOMPATIBILITY,2009,0,0,0,0,0,0 +INCOMPETENT,2009,0,0,0,0,0,0 +INCOMPLETELY,2009,0,0,0,0,0,0 +INCONSISTENCY,2009,0,0,0,0,0,0 +INCONTESTABLE,0,0,0,2009,0,0,0 +INCORRECT,2009,0,0,0,0,0,0 +INCREDIBLY,0,2009,0,0,0,0,0 +INDEFEASIBLE,2009,0,0,0,0,0,0 +INDEFINITENESS,0,0,2009,0,0,0,0 +INDEMNIFIED,0,0,0,2009,0,0,0 +INDEMNITEE,0,0,0,2009,0,0,0 +INDEMNITORS,0,0,0,2009,0,0,0 +INDICT,2009,0,0,2009,0,0,0 +INDICTMENT,2009,0,0,2009,0,0,0 +INEFFECTIVELY,2009,0,0,0,0,0,0 +INEFFICIENT,2009,0,0,0,0,0,0 +INEQUITABLE,2009,0,0,0,0,0,0 +INEVITABLE,2009,0,0,0,0,0,0 +INEXPERIENCED,2009,0,0,0,0,0,0 +INFORCE,0,0,0,2009,0,0,0 +INFRINGE,2009,0,0,0,0,0,0 +INFRINGER,0,0,0,2009,0,0,0 +INHIBIT,0,0,0,0,0,0,2011 +INIMICAL,2009,0,0,0,0,0,0 +INJURE,2009,0,0,0,0,0,0 +INJURING,2009,0,0,0,0,0,0 +INNOVATED,0,2009,0,0,0,0,0 +INNOVATIONS,0,2009,0,0,0,0,0 +INNOVATORS,0,2009,0,0,0,0,0 +INSECURE,2009,0,0,0,0,0,0 +INSISTED,0,0,0,0,0,0,2009 +INSOFAR,0,0,0,2009,0,0,0 +INSPIRATION,0,2009,0,0,0,0,0 +INSUBORDINATION,2009,0,0,0,0,0,0 +INSURRECTION,2009,0,0,0,0,0,0 +INTEGRITY,0,2009,0,0,0,0,0 +INTERFERENCE,2009,0,0,0,0,0,0 +INTERLOCUTORY,0,0,0,2009,0,0,0 +INTERPOSE,0,0,0,2009,0,0,0 +INTERPOSITION,0,0,0,2009,0,0,0 +INTERROGATES,0,0,0,2009,0,0,0 +INTERROGATOR,0,0,0,2009,0,0,0 +INTERRUPT,2009,0,0,0,0,0,0 +INTERRUPTIONS,2009,0,0,0,0,0,0 +INTIMIDATION,2009,0,0,0,0,0,0 +SPORADICALLY,0,0,2009,0,0,0,0 +STABILIZE,0,2009,0,0,0,0,0 +STABLE,0,2009,0,0,0,0,0 +STAGNATED,2009,0,0,0,0,0,0 +STANDSTILL,2009,0,0,0,0,0,0 +STATUTORILY,0,0,0,2009,0,0,0 +STIPULATES,0,0,0,0,0,0,2009 +STOLEN,2009,0,0,0,0,0,0 +STOPPING,2009,0,0,0,0,0,0 +STRAINING,2009,0,0,0,0,0,0 +STRENGTHENED,0,2009,0,0,0,0,0 +STRESS,2009,0,0,0,0,0,0 +STRESSING,2009,0,0,0,0,0,0 +STRICTLY,0,0,0,0,0,0,2009 +STRONGEST,0,2009,0,0,0,0,0 +SUBDOCKET,0,0,0,2012,0,0,0 +SUBLEASEE,0,0,0,2011,0,0,0 +SUBLICENSOR,0,0,0,2011,0,0,0 +SUBPOENAED,2009,0,0,2009,0,0,0 +SUBSTANDARD,2009,0,0,0,0,0,0 +SUCCEEDED,0,2009,0,0,0,0,0 +SUCCESSES,0,2009,0,0,0,0,0 +SUDDENLY,0,0,2009,0,0,0,0 +SUFFER,2009,0,0,0,0,0,0 +SUGGEST,0,0,2009,0,0,2009,0 +SUING,2009,0,0,2009,0,0,0 +SUMMONSES,2009,0,0,2009,0,0,0 +SUPERSEDED,0,0,0,2009,0,0,0 +SURETY,0,0,0,2009,0,0,0 +SURPASSING,0,2009,0,0,0,0,0 +SUSPECTED,2009,0,0,0,0,0,0 +SUSPENDING,2009,0,0,0,0,0,0 +SUSPICION,2009,0,0,0,0,0,0 +REVISED,0,0,2009,0,0,0,0 +REVOKE,2009,0,0,0,0,0,0 +REVOLUTIONIZE,0,2009,0,0,0,0,0 +REWARD,0,2009,0,0,0,0,0 +RIDICULE,2009,0,0,0,0,0,0 +RISK,0,0,2009,0,0,0,0 +RISKINESS,0,0,2009,0,0,0,0 +ROUGHLY,0,0,2009,0,0,0,0 +SABOTAGE,2009,0,0,0,0,0,0 +SACRIFICIAL,2009,0,0,0,0,0,0 +SATISFACTORY,0,2009,0,0,0,0,0 +SATISFYING,0,2009,0,0,0,0,0 +SCRUTINIZED,2009,0,0,0,0,0,0 +SECRECY,-2020,0,0,0,0,0,0 +SEIZES,2009,0,0,0,0,0,0 +SENTENCED,2009,0,0,2009,0,0,0 +SERIOUSLY,2009,0,0,0,0,0,0 +SETTLEMENT,0,0,0,2009,0,0,0 +SEVERABLE,0,0,0,2009,0,0,0 +SEVERE,2009,0,0,0,0,0,0 +SEVERITY,2009,0,0,0,0,0,0 +ENTHUSIASTIC,0,2009,0,0,0,0,0 +ERODE,2009,0,0,0,0,0,0 +EROSION,2009,0,0,0,0,0,0 +ERRING,2009,0,0,0,0,0,0 +ERRORS,2009,0,0,0,0,0,0 +ESCALATES,2009,0,0,0,0,0,0 +ESCHEATMENT,0,0,0,2011,0,0,0 +ESCROWS,0,0,0,0,0,0,2009 +EVADES,2009,0,0,0,0,0,0 +EVASIVE,2009,0,0,0,0,0,0 +EVICTION,2009,0,0,0,0,0,0 +EVIDENTIARY,0,0,0,2009,0,0,0 +EXACERBATING,2009,0,0,0,0,0,0 +EXAGGERATED,2009,0,0,0,0,0,0 +EXCEEDANCE,0,0,0,2011,0,0,0 +EXCELLENT,0,2009,0,0,0,0,0 +EXCEPTIONALLY,0,2009,0,0,0,0,0 +EXCITED,0,2009,0,0,0,0,0 +EXCLUSIVELY,0,2009,0,0,0,0,0 +EXCULPATE,2009,0,0,2009,0,0,0 +EXCULPATION,2009,0,0,2009,0,0,0 +EXECUTORS,0,0,0,2009,0,0,0 +EXECUTRIXES,0,0,0,2009,0,0,0 +EXONERATES,2009,0,0,0,0,0,0 +EXPLOIT,2009,0,0,0,0,0,0 +EXPLOITED,2009,0,0,0,0,0,0 +EXPOSED,2009,0,0,0,0,0,0 +EXPOSURES,0,0,2009,0,0,0,0 +EXPROPRIATING,2009,0,0,0,0,0,0 +EXPULSIONS,2009,0,0,0,0,0,0 +EXTRAJUDICIAL,0,0,0,2014,0,0,0 +SOLVES,0,2009,0,0,0,0,0 +SOMEWHAT,0,0,2009,0,0,2009,0 +SPAMMING,2014,0,0,0,0,0,0 +TREMENDOUSLY,0,2009,0,0,0,0,0 +LOCKOUTS,2009,0,0,0,0,0,0 +LOSS,2009,0,0,0,0,0,0 +LOYAL,0,2009,0,0,0,0,0 +MALFEASANCE,2009,0,0,0,0,0,0 +MALFUNCTIONS,2009,0,0,0,0,0,0 +MALPRACTICE,2009,0,0,0,0,0,0 +MANDATES,0,0,0,0,0,0,2009 +MANIPULATE,2009,0,0,0,0,0,0 +MANIPULATION,2009,0,0,0,0,0,0 +MARKDOWNS,2009,0,0,0,0,0,0 +MEDIATED,0,0,0,2009,0,0,0 +MEDIATIONS,0,0,0,2009,0,0,0 +MIGHT,0,0,2009,0,0,2009,0 +MISAPPLIES,2009,0,0,0,0,0,0 +MISAPPROPRIATED,2009,0,0,0,0,0,0 +MISAPPROPRIATIONS,2009,0,0,0,0,0,0 +MISCALCULATES,2009,0,0,0,0,0,0 +MISCHARACTERIZATION,2014,0,0,0,0,0,0 +MISCLASSIFIED,2011,0,0,0,0,0,0 +MISDATED,2011,0,0,0,0,0,0 +MISFEASANCE,0,0,0,2009,0,0,0 +MISHANDLING,2009,0,0,0,0,0,0 +MISINFORMING,2009,0,0,0,0,0,0 +MISINTERPRETATIONS,2009,0,0,0,0,0,0 +MISJUDGE,2009,0,0,0,0,0,0 +MISJUDGMENT,2009,0,0,0,0,0,0 +MISLABELING,2009,0,0,0,0,0,0 +MISLEADING,2009,0,0,0,0,0,0 +MISMANAGE,2009,0,0,0,0,0,0 +MISMANAGING,2009,0,0,0,0,0,0 +MISMATCHING,2009,0,0,0,0,0,0 +MISPRICINGS,2014,0,0,0,0,0,0 +MISREPRESENTED,2009,0,0,0,0,0,0 +MISSED,2009,0,0,0,0,0,0 +WORRIES,2009,0,0,0,0,0,0 +WORSEN,2009,0,0,0,0,0,0 +WORST,2009,0,0,0,0,0,0 +TIGHTENING,2009,0,0,0,0,0,0 +TOLERATING,2009,0,0,0,0,0,0 +TORTIOUSLY,0,0,0,2009,0,0,0 +FINES,2009,0,0,0,0,0,0 +NONASSESSABLE,0,0,2009,0,0,0,0 +NONCANCELLABLE,0,0,0,0,0,0,2009 +NONCOMPLIANT,2009,0,0,0,0,0,0 +NONCONFORMITY,2009,0,0,0,0,0,0 +NONCONTRIBUTORY,0,0,0,2009,0,0,0 +NONFORFEITABILITY,0,0,0,2011,0,0,0 +NONGUARANTOR,0,0,0,2011,0,0,0 +DISCIPLINARY,2009,0,0,0,0,0,0 +DISCLAIMERS,2009,0,0,0,0,0,0 +DISCLOSED,2009,0,0,0,0,0,0 +DISCONTINUANCES,2009,0,0,0,0,0,0 +DISCONTINUED,2009,0,0,0,0,0,0 +DISCOURAGED,2009,0,0,0,0,0,0 +DISCREDITED,2009,0,0,0,0,0,0 +DISCREPANCY,2009,0,0,0,0,0,0 +SPECULATED,0,0,2009,0,0,0,0 +SPECULATIONS,0,0,2009,0,0,0,0 +TRAGICALLY,2009,0,0,0,0,0,0 +LEGISLATED,0,0,0,2009,0,0,0 +LEGISLATIONS,0,0,0,2009,0,0,0 +LEGISLATORS,0,0,0,2009,0,0,0 +LIBELED,0,0,0,2009,0,0,0 +LIE,2009,0,0,0,0,0,0 +COMPULSION,2009,0,0,0,0,0,2009 +CONCEDE,2009,0,0,0,0,0,0 +CONCEIVABLE,0,0,2009,0,0,2009,0 +CONCERNS,2009,0,0,0,0,0,0 +CONCLUSIVE,0,2009,0,0,0,0,0 +CONDEMNATIONS,2009,0,0,0,0,0,0 +CONDEMNS,2009,0,0,0,0,0,0 +CONDONED,2009,0,0,0,0,0,0 +CONFESSES,2009,0,0,0,0,0,0 +CONFINE,2009,0,0,0,0,0,2009 +CONFINES,2009,0,0,0,0,0,2009 +CONFISCATES,2009,0,0,0,0,0,0 +CONFISCATORY,0,0,0,2009,0,0,0 +CONFLICTS,2009,0,0,0,0,0,0 +CONFRONTATIONS,2009,0,0,0,0,0,0 +CONFUSE,2009,0,0,0,0,0,0 +CONFUSINGLY,2009,0,2009,0,0,0,0 +CONSENTING,0,0,0,2009,0,0,0 +CONSPIRACY,2009,0,0,0,0,0,0 +CONSPIRE,2009,0,0,0,0,0,0 +CONSTITUTION,0,0,0,2009,0,0,0 +CONSTITUTIONS,0,0,0,2009,0,0,0 +CONSTRAINING,0,0,0,0,0,0,2009 +CONSTRUCTIVE,0,2009,0,0,0,0,0 +CONSTRUES,0,0,0,2012,0,0,0 +CONTENDED,2009,0,0,0,0,0,0 +CONTENTIONS,2009,0,0,0,0,0,0 +CONTESTATION,0,0,0,2011,0,0,0 +CONTINGENCY,0,0,2009,0,0,0,0 +CONTRACT,0,0,0,2009,0,0,0 +CONTRACTIBLE,0,0,0,2009,0,0,0 +CONTRACTIONS,2009,0,0,0,0,0,0 +CONTRADICT,2009,0,0,0,0,0,0 +CONTRADICTIONS,2009,0,0,0,0,0,0 +CONTRAVENE,0,0,0,2009,0,0,0 +CONTRAVENTION,0,0,0,2009,0,0,0 +CONTROVERSY,2009,0,0,0,0,0,0 +CONVENIENS,0,0,0,2012,0,0,0 +CONVICTED,2009,0,0,2009,0,0,0 +CORRECTED,2009,0,0,0,0,0,0 +CORRECTS,2009,0,0,0,0,0,0 +CORRUPTION,2009,0,0,0,0,0,0 +COSTLY,2009,0,0,0,0,0,0 +COUNSELED,0,0,0,2009,0,0,0 +COUNTERCLAIMED,2009,0,0,0,0,0,0 +COUNTERFEITED,2009,0,0,0,0,0,0 +COUNTERFEITS,2009,0,0,0,0,0,0 +COUNTERSUED,0,0,0,2011,0,0,0 +COURTEOUS,0,2009,0,0,0,0,0 +COVENANTED,0,0,0,0,0,0,2011 +CREATIVELY,0,2009,0,0,0,0,0 +CRIMES,2009,0,0,2009,0,0,0 +CRIMINALIZING,0,0,0,2014,0,0,0 +CRISIS,2009,0,0,0,0,0,0 +CRITICISMS,2009,0,0,0,0,0,0 +CRITICIZING,2009,0,0,0,0,0,0 +CROSSROADS,0,0,2009,0,0,0,0 +CULPABLE,2009,0,0,0,0,0,0 +CURTAILED,2009,0,0,0,0,0,0 +CURTAILS,2009,0,0,0,0,0,0 +CYBERATTACK,2014,0,0,0,0,0,0 +CYBERCRIMES,2014,0,0,0,0,0,0 +TAINTS,2009,0,0,0,0,0,0 +TENSE,2009,0,0,0,0,0,0 +TERMINATE,2009,0,0,0,0,0,0 +TERMINATION,2009,0,0,0,0,0,0 +TESTIFY,2009,0,0,2009,0,0,0 +THENCEFORTH,0,0,0,2009,0,0,0 +THEREFROM,0,0,0,2009,0,0,0 +THEREON,0,0,0,2009,0,0,0 +THERETOFORE,0,0,0,2009,0,0,0 +THEREWITH,0,0,0,2009,0,0,0 +THREATENING,2009,0,0,0,0,0,0 +FACIE,0,0,0,2009,0,0,0 +FAILING,2009,0,0,0,0,0,0 +FAILURES,2009,0,0,0,0,0,0 +FALSIFICATION,2009,0,0,0,0,0,0 +FALSIFY,2009,0,0,0,0,0,0 +FATALITIES,2009,0,0,0,0,0,0 +FAULTED,2009,0,0,0,0,0,0 +FAVORABLY,0,2009,0,0,0,0,0 +FAVORITES,0,2009,0,0,0,0,0 +FELONIOUS,2009,0,0,2009,0,0,0 +DAMAGES,2009,0,0,0,0,0,0 +DANGER,2009,0,0,0,0,0,0 +DEADLOCK,2009,0,0,0,0,0,0 +DEADWEIGHT,2009,0,0,0,0,0,0 +DEBARRED,2009,0,0,0,0,0,0 +DECEIT,2009,0,0,0,0,0,0 +DECEIVED,2009,0,0,0,0,0,0 +DECEPTIONS,2009,0,0,0,0,0,0 +DECLINE,2009,0,0,0,0,0,0 +DECREE,0,0,0,2009,0,0,0 +DEFACE,2009,0,0,0,0,0,0 +DEFALCATIONS,0,0,0,2009,0,0,0 +DEFAME,2009,0,0,0,0,0,0 +DEFAULT,2009,0,0,0,0,0,0 +DEFEASANCE,0,0,0,2009,0,0,0 +DEFEASEMENT,0,0,0,2014,0,0,0 +DEFEATED,2009,0,0,0,0,0,0 +DEFECTIVE,2009,0,0,0,0,0,0 +DEFENDABLE,0,0,0,2014,0,0,0 +DEFENDING,2009,0,0,0,0,0,0 +DEFERENCE,0,0,0,2009,0,0,0 +DEFICIT,2009,0,0,0,0,0,0 +DEFRAUD,2009,0,0,0,0,0,0 +DEFUNCT,2009,0,0,0,0,0,0 +DEGRADED,2009,0,0,0,0,0,0 +DELAYED,2009,0,0,0,0,0,0 +DELEGATABLE,0,0,0,2011,0,0,0 +DELIBERATE,2009,0,0,0,0,0,0 +DELIGHTED,0,2009,0,0,0,0,0 +DELIGHTS,0,2009,0,0,0,0,0 +DELINQUENTLY,2009,0,0,0,0,0,0 +DELISTING,2009,0,0,0,0,0,0 +DEMISES,2009,0,0,0,0,0,0 +DEMOLISHES,2009,0,0,0,0,0,0 +DEMOTE,2009,0,0,0,0,0,0 +DEMOTION,2009,0,0,0,0,0,0 +DEMURRERS,0,0,0,2009,0,0,0 +DENIALS,2009,0,0,0,0,0,0 +DENIGRATED,2009,0,0,0,0,0,0 +DENY,2009,0,0,0,0,0,0 +DEPENDABLE,0,2009,0,0,0,0,0 +DEPENDED,0,0,2009,0,0,2009,0 +DEPENDENT,0,0,2009,0,0,0,2011 +DEPLETED,2009,0,0,0,0,0,0 +DEPLETIONS,2009,0,0,0,0,0,0 +DEPOSING,0,0,0,2009,0,0,0 +INVALID,2009,0,0,0,0,0,0 +INVALIDATING,2009,0,0,0,0,0,0 +INVENTED,0,2009,0,0,0,0,0 +INVENTIVE,0,2009,0,0,0,0,0 +INVESTIGATE,2009,0,0,0,0,0,0 +INVESTIGATION,2009,0,0,0,0,0,0 +IRRECONCILABLE,2009,0,0,0,0,0,0 +IRREGULAR,2009,0,0,0,0,0,0 +IRREPARABLE,2009,0,0,0,0,0,0 +IRREVOCABLE,0,0,0,2009,0,0,2009 +JOINDER,0,0,0,2009,0,0,0 +JUDICIARY,0,0,0,2009,0,0,0 +JURISDICTIONAL,0,0,0,2009,0,0,0 +JURIST,0,0,0,2009,0,0,0 +JURY,0,0,0,2009,0,0,0 +JUSTIFIABLE,2009,0,0,0,0,0,0 +DILIGENTLY,0,2009,0,0,0,0,0 +DIMINISHING,2009,0,0,0,0,0,0 +DISADVANTAGE,2009,0,0,0,0,0,0 +DISAFFILIATION,2009,0,0,2009,0,0,0 +DISAFFIRMS,0,0,0,2011,0,0,0 +DISAGREEING,2009,0,0,0,0,0,0 +DISALLOW,2009,0,0,0,0,0,0 +DISALLOWING,2009,0,0,0,0,0,0 +DISAPPEARANCES,2009,0,0,0,0,0,0 +DISAPPOINT,2009,0,0,0,0,0,0 +DISAPPOINTMENT,2009,0,0,0,0,0,0 +DISAPPROVALS,2009,0,0,0,0,0,0 +DISAPPROVING,2009,0,0,0,0,0,0 +DISASSOCIATIONS,2009,0,0,0,0,0,0 +DISASTROUSLY,2009,0,0,0,0,0,0 +DISAVOWING,2009,0,0,0,0,0,0 +GREATER,0,-2020,0,0,0,0,0 +GRIEVANCE,2009,0,0,0,0,0,0 +GUILTY,2009,0,0,0,0,0,0 +HAMPERED,2009,0,0,0,0,0,0 +HAPPILY,0,2009,0,0,0,0,0 +HARASSED,2009,0,0,0,0,0,0 +HARDSHIPS,2009,0,0,0,0,0,0 +HARMFULLY,2009,0,0,0,0,0,0 +HARSHER,2009,0,0,0,0,0,0 +HAZARD,2009,0,0,0,0,0,0 +NONJUDICIALLY,0,0,0,2011,0,0,0 +NONPERFORMANCE,2009,0,0,0,0,0,0 +HURT,2009,0,0,0,0,0,0 +DISFAVORED,2009,0,0,0,0,0,0 +DISGORGED,2009,0,0,0,0,0,0 +COMPLICATING,2009,0,0,0,0,0,0 +COMPLIMENTARY,0,2009,0,0,0,0,0 +COMPLY,0,0,0,0,0,0,2009 +MISSTATED,2009,0,0,0,0,0,0 +MISSTATING,2009,0,0,0,0,0,0 +MISTAKEN,2009,0,0,0,0,0,0 +MISTRIAL,2009,0,0,2009,0,0,0 +MISUNDERSTANDINGS,2009,0,0,0,0,0,0 +MISUSES,2009,0,0,0,0,0,0 +MONOPOLIZATION,2009,0,0,0,0,0,0 +MONOPOLIZING,2009,0,0,0,0,0,0 +MORATORIUMS,2009,0,0,0,0,0,0 +MOTIONS,0,0,0,2009,0,0,0 +SLOWDOWN,2009,0,0,0,0,0,0 +SLOWEST,2009,0,0,0,0,0,0 +SLUGGISH,2009,0,0,0,0,0,0 +SMOOTHING,0,2009,0,0,0,0,0 +DEPRESS,2009,0,0,0,0,0,0 +DEPRIVATION,2009,0,0,0,0,0,0 +DEPRIVING,2009,0,0,0,0,0,0 +DEROGATED,0,0,0,2009,0,0,0 +DEROGATIONS,0,0,0,2009,0,0,0 +DESIRED,0,2009,0,0,0,0,0 +DESTABILIZE,2009,0,0,0,0,0,0 +DESTROY,2009,0,0,0,0,0,0 +DESTRUCTION,2009,0,0,0,0,0,0 +DETAINER,0,0,0,2009,0,0,0 +DETERIORATE,2009,0,0,0,0,0,0 +DETERIORATION,2009,0,0,0,0,0,0 +DETERRENCES,2009,0,0,0,0,0,0 +DETERS,2009,0,0,0,0,0,0 +DETRIMENT,2009,0,0,0,0,0,0 +DEVALUE,2009,0,0,0,0,0,0 +DEVASTATE,2009,0,0,0,0,0,0 +DEVIATE,2009,0,2009,0,0,0,0 +DEVIATION,2009,0,2009,0,0,0,0 +DEVOLVED,2009,0,0,0,0,0,0 +DICTATED,0,0,0,0,0,0,2009 +DIFFERED,0,0,2009,0,0,0,0 +DIFFICULTIES,2009,0,0,0,0,0,0 +ASCENDANCY,0,0,0,2009,0,0,0 +ASSAULTED,2009,0,0,0,0,0,0 +ASSERTIONS,2009,0,0,0,0,0,0 +ASSUME,0,0,2009,0,0,0,0 +ASSUMPTION,0,0,2009,0,0,0,0 +ASSURES,0,2009,0,0,0,0,0 +ATTAINING,0,2009,0,0,0,0,0 +ATTEST,0,0,0,2009,0,0,0 +ATTESTING,0,0,0,2009,0,0,0 +ATTORNMENT,0,0,0,2009,0,0,0 +ATTRITION,2009,0,0,0,0,0,0 +BAIL,2009,0,0,2009,0,0,0 +BAILIFF,0,0,0,2009,0,0,0 +BALK,2009,0,0,0,0,0,0 +BANKRUPTCY,2009,0,0,0,0,0,0 +BANS,2009,0,0,0,0,0,0 +CLAIMABLE,0,0,0,2009,0,0,0 +CLAIMING,2009,0,0,0,0,0,0 +CLAWBACK,2009,0,0,0,0,0,0 +CLOSEOUT,2009,0,0,0,0,0,0 +CLOSURE,2009,0,0,0,0,0,0 +CODICIL,0,0,0,2009,0,0,0 +CODIFIED,0,0,0,2009,0,0,0 +COERCE,2009,0,0,0,0,0,0 +COERCION,2009,0,0,0,0,0,0 +COLLABORATING,0,2009,0,0,0,0,0 +COLLABORATOR,0,2009,0,0,0,0,0 +COLLAPSES,2009,0,0,0,0,0,0 +COLLUDE,2009,0,0,0,0,0,0 +COLLUSION,2009,0,0,2009,0,0,0 +POSING,2009,0,0,0,0,0,0 +POSSIBILITIES,0,0,2009,0,0,0,0 +POSTCLOSING,0,0,0,2011,0,0,0 +POSTPONE,2009,0,0,0,0,0,0 +POSTPONES,2009,0,0,0,0,0,0 +WRITEOFF,2009,0,0,0,0,0,0 +WRONGDOING,2009,0,0,0,0,0,0 +WRONGLY,2009,0,0,0,0,0,0 +HONORED,0,2009,0,0,0,0,0 +HOSTILITY,2009,0,0,0,0,0,0 +REASSESSED,0,0,2009,0,0,0,0 +REASSESSMENTS,2009,0,2009,0,0,0,0 +REASSIGNMENT,2009,0,0,0,0,0,0 +REBOUNDED,0,2009,0,0,0,0,0 +REBUTTABLE,0,0,0,2009,0,0,0 +REBUTTED,0,0,0,2009,0,0,0 +RECALCULATES,0,0,2009,0,0,0,0 +RECALL,2009,0,0,0,0,0,0 +RECEPTIVE,0,2009,0,0,0,0,0 +RECKLESS,2009,0,0,0,0,0,0 +RECONSIDERED,0,0,2009,0,0,0,0 +RECOUPABLE,0,0,0,2009,0,0,0 +RECOURSES,0,0,0,2009,0,0,0 +RECUSE,0,0,0,2011,0,0,0 +REDACT,2009,0,0,2009,0,0,0 +REDACTIONS,2009,0,0,2009,0,0,0 +REDRESS,2009,0,0,0,0,0,0 +REEXAMINATION,0,0,2009,0,0,0,0 +REFERENDUM,0,0,0,2009,0,0,0 +REFILES,0,0,0,2009,0,0,0 +REFRAINS,0,0,0,0,0,0,2009 +REFUSED,2009,0,0,0,0,0,0 +REGAINED,0,2009,0,0,0,0,0 +REGULATES,0,0,0,2009,0,0,0 +REGULATIVE,0,0,0,2009,0,0,0 +REHEAR,0,0,0,2009,0,0,0 +VARIABLE,0,0,2009,0,0,0,0 +VARIANCES,0,0,2009,0,0,0,0 +VARIATIONS,0,0,2009,0,0,0,0 +VARYING,0,0,2009,0,0,0,0 +VERDICTS,2009,0,0,2009,0,0,0 +VIATICAL,0,0,0,2011,0,0,0 +VIOLATE,2009,0,0,0,0,0,0 +VIOLATION,2009,0,0,0,0,0,0 +VIOLATORS,2009,0,0,0,0,0,0 +VITIATE,2009,0,0,0,0,0,0 +VITIATION,2009,0,0,0,0,0,0 +VOLATILE,2009,0,2009,0,0,0,0 +VULNERABILITY,2009,0,0,0,0,0,0 +WARNED,2009,0,0,0,0,0,0 +WARRANTEES,0,0,0,2011,0,0,0 +WASTING,2009,0,0,0,0,0,0 +WEAKENING,2009,0,0,0,0,0,0 +WEAKLY,2009,0,0,0,0,0,0 +DISHONESTY,2009,0,0,0,0,0,0 +DISHONORED,2009,0,0,0,0,0,0 +DISLOYAL,2009,0,0,0,0,0,0 +DISMALLY,2009,0,0,0,0,0,0 +DISMISSED,2009,0,0,0,0,0,0 +DISPARAGE,2009,0,0,0,0,0,0 +DISPARAGES,2009,0,0,0,0,0,0 +DISPARITY,2009,0,0,0,0,0,0 +DISPLACEMENTS,2009,0,0,0,0,0,0 +DISPOSITIVE,0,0,0,2009,0,0,0 +DISPOSSESSING,2009,0,0,0,0,0,0 +DISPROPORTIONAL,2009,0,0,0,0,0,0 +DISPUTED,2009,0,0,0,0,0,0 +DISQUALIFICATIONS,2009,0,0,0,0,0,0 +DISQUALIFYING,2009,0,0,0,0,0,0 +DISREGARDS,2009,0,0,0,0,0,0 +DISRUPTED,2009,0,0,0,0,0,0 +DISRUPTIVE,2009,0,0,0,0,0,0 +DISSENT,2009,0,0,0,0,0,0 +DISSENTING,2009,0,0,0,0,0,0 +DISSOLUTION,2009,0,0,0,0,0,0 +DISTINCTIVE,0,2009,0,0,0,0,0 +DISTORTED,2009,0,0,0,0,0,0 +DISTORTS,2009,0,0,0,0,0,0 +DISTRACTION,2009,0,0,0,0,0,0 +DISTRESS,2009,0,0,0,0,0,0 +DISTURB,2009,0,0,0,0,0,0 +DISTURBING,2009,0,0,0,0,0,0 +DIVERTED,2009,0,0,0,0,0,0 +DIVESTED,2009,0,0,0,0,0,0 +DIVESTMENT,2009,0,0,0,0,0,0 +DIVORCED,2009,0,0,0,0,0,0 +DIVULGING,2009,0,0,0,0,0,0 +DOCKETS,0,0,0,2009,0,0,0 +DOUBTFUL,2009,0,2009,0,0,0,0 +DOWNGRADES,2009,0,0,0,0,0,0 +DOWNSIZES,2009,0,0,0,0,0,0 +DOWNTIMES,2009,0,0,0,0,0,0 +DOWNWARDS,2009,0,0,0,0,0,0 +DRAWBACK,2009,0,0,0,0,0,0 +DROUGHT,2009,0,0,0,0,0,0 +DYSFUNCTION,2009,0,0,0,0,0,0 +EARMARKED,0,0,0,0,0,0,2009 +EASILY,0,2009,0,0,0,0,0 +EFFICIENCIES,0,2009,0,0,0,0,0 +EGREGIOUS,2009,0,0,0,0,0,0 +EMBARGOED,2009,0,0,0,0,0,0 +EMBARRASSED,2009,0,0,0,0,0,0 +EMBARRASSMENTS,2009,0,0,0,0,0,0 +EMBEZZLEMENTS,2009,0,0,0,0,0,0 +EMPOWER,0,2009,0,0,0,0,0 +ENABLE,0,2009,0,0,0,0,0 +ENCOURAGED,0,2009,0,0,0,0,0 +ENCROACH,2009,0,0,0,0,0,0 +ENCROACHMENT,2009,0,0,0,0,0,0 +ENCUMBERING,2009,0,0,2009,0,0,2009 +ENCUMBRANCERS,0,0,0,2011,0,0,0 +ENDANGERING,2009,0,0,0,0,0,0 +ENFORCEABILITY,0,0,0,2009,0,0,0 +ENHANCED,0,2009,0,0,0,0,0 +ENHANCING,0,2009,0,0,0,0,0 +ENJOINS,2009,0,0,0,0,0,0 +ENJOYED,0,2009,0,0,0,0,0 +ENTAIL,0,0,0,0,0,0,2009 +PECUNIARILY,0,0,0,2009,0,0,0 +PENALIZING,2009,0,0,0,0,0,0 +PERFECT,0,2009,0,0,0,0,0 +PERHAPS,0,0,2009,0,0,2009,0 +PERMISSIBLE,0,0,0,0,0,0,2009 +PERMITTEE,0,0,0,2011,0,0,0 +PERPETRATED,2009,0,0,2009,0,0,0 +PERSIST,2009,0,0,0,0,0,0 +PERSISTENTLY,2009,0,0,0,0,0,0 +PERVASIVE,2009,0,0,0,0,0,0 +PETITIONED,0,0,0,2009,0,0,0 +PETITIONS,0,0,0,2009,0,0,0 +PICKETING,2009,0,0,0,0,0,0 +HENCEFORWARD,0,0,0,2009,0,0,0 +HEREFOR,0,0,0,2009,0,0,0 +HEREINABOVE,0,0,0,2009,0,0,0 +HEREOF,0,0,0,2009,0,0,0 +HEREUNDER,0,0,0,2009,0,0,0 +HEREWITHIN,0,0,0,2014,0,0,0 +HINDERED,2009,0,0,0,0,0,0 +HINDRANCES,2009,0,0,0,0,0,0 +WHENSOEVER,0,0,0,2009,0,0,0 +WHEREBY,0,0,0,2009,0,0,0 +WHEREON,0,0,0,2009,0,0,0 +WHEREWITH,0,0,0,2009,0,0,0 +WHOSOEVER,0,0,0,2009,0,0,0 +WILLFULLY,2009,0,0,2009,0,0,0 +WINNERS,0,2009,0,0,0,0,0 +FLUCTUATING,0,0,2009,0,0,0,0 +FORBEAR,0,0,0,2009,0,0,0 +FORBEARS,0,0,0,2009,0,0,0 +FORBIDS,2009,0,0,0,0,0,2009 +FOREBEAR,0,0,0,2009,0,0,0 +FORECLOSED,2009,0,0,0,0,0,0 +FORECLOSURES,2009,0,0,0,0,0,0 +FORESTALL,2009,0,0,0,0,0,0 +FORFEIT,2009,0,0,0,0,0,0 +FORFEITING,2009,0,0,0,0,0,0 +FORGERS,2009,0,0,0,0,0,0 +FRAUD,2009,0,0,0,0,0,0 +FRAUDULENTLY,2009,0,0,0,0,0,0 +FRUSTRATE,2009,0,0,0,0,0,0 +FRUSTRATINGLY,2009,0,0,0,0,0,0 +FUGITIVES,2009,0,0,2009,0,0,0 +GAINING,0,2009,0,0,0,0,0 +GRANTORS,0,0,0,2009,0,0,0 +DISGRACE,2009,0,0,0,0,0,0 +LITIGANTS,2009,0,0,2009,0,0,0 +LITIGATING,2009,0,0,2009,0,0,0 +LITIGATORS,0,0,0,2009,0,0,0 +LACK,2009,0,0,0,0,0,0 +LACKS,2009,0,0,0,0,0,0 +LAGS,2009,0,0,0,0,0,0 +LAPSING,2009,0,0,0,0,0,0 +LAWFUL,0,0,0,2009,0,0,0 +LAWMAKING,0,0,0,2009,0,0,0 +LAWYER,0,0,0,2009,0,0,0 +LEADERSHIP,0,2009,0,0,0,0,0 +LEGALITY,0,0,0,2009,0,0,0 +LEGALIZED,0,0,0,2009,0,0,0 +LEGALS,0,0,0,2009,0,0,0 +FLAW,2009,0,0,0,0,0,0 +NONPRODUCTIVE,2009,0,0,0,0,0,0 +LIQUIDATED,2009,0,0,0,0,0,0 +LIQUIDATIONS,2009,0,0,0,0,0,0 +BENEFICIATED,0,0,0,2014,0,0,0 +BENEFITING,0,2009,0,0,0,0,0 +BETTER,0,2009,0,0,0,0,0 +POPULARITY,0,2009,0,0,0,0,0 +REINTERPRET,0,0,2009,0,0,0,0 +REINTERPRETING,0,0,2009,0,0,0,0 +REJECTING,2009,0,0,0,0,0,0 +RELEASEES,0,0,0,2011,0,0,0 +RELINQUISHING,2009,0,0,0,0,0,0 +RELUCTANT,2009,0,0,0,0,0,0 +REMANDS,0,0,0,2009,0,0,0 +REMEDIATION,0,0,0,2009,0,0,0 +RENEGOTIATE,2009,0,0,0,0,0,0 +RENEGOTIATION,2009,0,0,0,0,0,0 +RENOUNCEMENT,2009,0,0,0,0,0,0 +REPARATION,2009,0,0,0,0,0,0 +REPOSSESSED,2009,0,0,0,0,0,0 +REPOSSESSIONS,2009,0,0,0,0,0,0 +TROUBLE,2009,0,0,0,0,0,0 +TURMOIL,2009,0,0,0,0,0,0 +UNACCOUNTED,2009,0,0,0,0,0,0 +UNAPPEALABLE,0,0,0,2009,0,0,0 +UNAUTHORIZED,2009,0,0,0,0,0,0 +UNAVOIDABLY,2009,0,0,0,0,0,0 +UNCERTAINTIES,0,0,2009,0,0,0,0 +UNCOLLECTED,2009,0,0,0,0,0,0 +UNCOMPETITIVE,2009,0,0,0,0,0,0 +UNCONSCIONABLE,2009,0,0,0,0,0,0 +UNCONSTITUTIONALLY,0,0,0,2009,0,0,0 +UNCONTROLLED,2009,0,0,0,0,0,0 +UNCOVERING,2009,0,0,0,0,0,0 +UNDEFINED,0,0,2009,0,0,0,0 +UNDERCUT,2009,0,0,0,0,0,0 +UNDERESTIMATED,2009,0,0,0,0,0,0 +UNDERFUNDED,2009,0,0,0,0,0,0 +UNDERMINES,2009,0,0,0,0,0,0 +UNDERPAYMENTS,2009,0,0,0,0,0,0 +UNDERPERFORMED,2011,0,0,0,0,0,0 +UNDERPRODUCTION,2009,0,0,0,0,0,0 +UNDERSTATEMENT,2009,0,0,0,0,0,0 +UNDERUTILIZATION,2009,0,0,0,0,0,0 +UNDESIRED,2009,0,0,0,0,0,0 +UNDETERMINED,2009,0,2009,0,0,0,0 +UNDOCUMENTED,2009,0,2009,0,0,0,0 +UNECONOMIC,2009,0,0,0,0,0,0 +UNEMPLOYMENT,2009,0,0,0,0,0,0 +UNENFORCEABLE,0,0,0,2009,0,0,0 +UNETHICALLY,2009,0,0,0,0,0,0 +UNFAIR,2009,0,0,0,0,0,0 +UNFAVORABILITY,2014,0,0,0,0,0,0 +UNFEASIBLE,2009,0,0,0,0,0,0 +UNFORESEEABLE,2009,0,0,0,0,0,0 +UNFORTUNATELY,2009,0,0,0,0,0,0 +UNFUNDED,2009,0,0,0,0,0,0 +UNIDENTIFIED,0,0,2009,0,0,0,0 +UNINTENTIONALLY,2009,0,0,0,0,0,0 +UNJUSTIFIED,2009,0,0,0,0,0,0 +UNKNOWN,0,0,2009,0,0,0,0 +UNLAWFULNESS,0,0,0,2009,0,0,0 +UNMATCHED,0,2009,0,0,0,0,0 +UNNECESSARY,2009,0,0,0,0,0,0 +UNOCCUPIED,2009,0,0,0,0,0,0 +UNPLANNED,2009,0,2009,0,0,0,0 +UNPREDICTABLY,2009,0,2009,0,0,0,0 +UNPROFITABLE,2009,0,0,0,0,0,0 +UNQUANTIFIABLE,0,0,2011,0,0,0,0 +UNREASONABLENESS,2009,0,0,0,0,0,0 +UNRECOVERABLE,2009,0,0,0,0,0,0 +UNREMEDIATED,0,0,0,2011,0,0,0 +UNREST,2009,0,0,0,0,0,0 +UNSATISFACTORY,2009,0,0,0,0,0,0 +UNSEASONABLE,0,0,2009,0,0,0,0 +UNSOLD,2009,0,0,0,0,0,0 +UNSTABILIZED,2014,0,0,0,0,0,0 +UNSUCCESSFUL,2009,0,0,0,0,0,0 +UNSUITABLY,2009,0,0,0,0,0,0 +UNSUSPECTED,2009,0,0,0,0,0,0 +UNTESTED,0,0,2009,0,0,0,0 +UNTRUTH,2009,0,0,0,0,0,0 +UNTRUTHS,2009,0,0,0,0,0,0 +UNWANTED,2009,0,0,0,0,0,0 +UNWILLINGNESS,2009,0,0,0,0,0,0 +UPTURNS,0,2009,0,0,0,0,0 +USURP,2009,0,0,2009,0,0,0 +USURPS,2009,0,0,2009,0,0,0 +VAGUELY,0,0,2012,0,0,0,0 +VAGUEST,0,0,2012,0,0,0,0 +PLEA,2009,0,0,0,0,0,0 +PLEADINGS,2009,0,0,2009,0,0,0 +PLEASANTLY,0,2009,0,0,0,0,0 +PLEDGE,0,0,0,0,0,0,2009 +PLEDGES,0,0,0,0,0,0,2009 +PLENTIFUL,0,2009,0,0,0,0,0 +COMPELS,0,0,0,0,0,0,2009 +COMPLAINANTS,0,0,0,2009,0,0,0 +COMPLAINT,2009,0,0,0,0,0,0 +COMMIT,0,0,0,0,0,0,2009 +COMMITTED,0,0,0,0,0,0,2009 +LIMIT,0,0,0,0,0,0,2009 +LIMITS,0,0,0,0,0,0,2009 +RESTRAINED,0,0,0,0,0,0,2009 +RESTRAINTS,0,0,0,0,0,0,2009 +RESTRICTION,0,0,0,0,0,0,2009 +RESTRICTIVENESS,0,0,0,0,0,0,2009 +RESTRUCTURES,2009,0,0,0,0,0,0 +RETALIATED,2009,0,0,0,0,0,0 +RETALIATIONS,2009,0,0,0,0,0,0 +PRECAUTIONARY,0,0,2009,0,0,0,0 +PRECIPITOUSLY,2009,0,0,0,0,0,0 +PRECLUDING,2009,0,0,0,0,0,2009 +PREDECEASE,0,0,0,2009,0,0,0 +PREDICT,0,0,2009,0,0,0,0 +PREDICTION,0,0,2009,0,0,0,0 +PREDICTORS,0,0,2009,0,0,0,0 +PREHEARING,0,0,0,2011,0,0,0 +PREJUDICIAL,2009,0,0,2009,0,0,0 +PREMATURE,2009,0,0,0,0,0,0 +PREPETITION,0,0,0,2009,0,0,0 +PRESTIGIOUS,0,2009,0,0,0,0,0 +PRESUMES,0,0,2009,0,0,0,0 +PRESUMPTIVELY,0,0,0,2009,0,0,0 +PREVENTING,2009,0,0,0,0,0,2009 +PRIVITY,0,0,0,2011,0,0,0 +PROBABILITIES,0,0,2009,0,0,0,0 +PROBATE,0,0,0,2009,0,0,0 +PROBATION,0,0,0,2009,0,0,0 +PROBATIONERS,0,0,0,2009,0,0,0 +PROBLEMATICAL,2009,0,0,0,0,0,0 +PROFICIENTLY,0,2009,0,0,0,0,0 +PROGRESS,0,2009,0,0,0,0,0 +PROHIBIT,0,0,0,0,0,0,2009 +PROHIBITIONS,0,0,0,0,0,0,2009 +PROHIBITS,0,0,0,0,0,0,2009 +PROLONGED,2009,0,0,0,0,0,0 +PROMULGATED,0,0,0,2009,0,0,0 +PROMULGATIONS,0,0,0,2009,0,0,0 +PRORATA,0,0,0,2009,0,0,0 +PROSECUTES,2009,0,0,2009,0,0,0 +PROSECUTOR,0,0,0,2009,0,0,0 +PROSPERING,0,2009,0,0,0,0,0 +PROTEST,2009,0,0,0,0,0,0 +PROTESTING,2009,0,0,0,0,0,0 +PROTRACTED,2009,0,0,0,0,0,0 +PROVISOS,0,0,0,2009,0,0,0 +PROVOKING,2009,0,0,0,0,0,0 +PUNISHING,2009,0,0,0,0,0,0 +PURPORT,2009,0,0,0,0,0,0 +PURPORTS,2009,0,0,0,0,0,0 +QUESTIONED,2009,0,0,0,0,0,0 +QUITCLAIM,0,0,0,2009,0,0,0 +RACKETEERING,2009,0,0,0,0,0,0 +RANDOMIZES,0,0,2009,0,0,0,0 +RATA,0,0,0,2009,0,0,0 +RATIONALIZATIONS,2009,0,0,0,0,0,0 +RATIONALIZING,2009,0,0,0,0,0,0 +ABANDON,2009,0,0,0,0,0,0 +ABANDONMENTS,2009,0,0,0,0,0,0 +ABDICATING,2009,0,0,0,0,0,0 +ABERRATION,2009,0,0,0,0,0,0 +ABEYANCE,0,0,2009,0,0,0,0 +ABLE,0,2009,0,0,0,0,0 +ABNORMALLY,2009,0,0,0,0,0,0 +ABOLISHING,2009,0,0,0,0,0,0 +ABROGATES,2009,0,0,2009,0,0,0 +ABRUPT,2009,0,0,0,0,0,0 +ABSENCES,2009,0,0,0,0,0,0 +ABSOLVES,0,0,0,2009,0,0,0 +ABUSE,2009,0,0,0,0,0,0 +ABUSIVE,2009,0,0,0,0,0,0 +ACCESSIONS,0,0,0,2009,0,0,0 +ACCIDENTS,2009,0,0,0,0,0,0 +ACCOMPLISHES,0,2009,0,0,0,0,0 +ACCUSATION,2009,0,0,0,0,0,0 +ACCUSES,2009,0,0,0,0,0,0 +ACHIEVEMENT,0,2009,0,0,0,0,0 +ACQUIESCE,2009,0,0,0,0,0,0 +ACQUIREES,0,0,0,2011,0,0,0 +ACQUITTAL,2009,0,0,2009,0,0,0 +ACQUITTED,2009,0,0,2009,0,0,0 +ADJOURN,0,0,0,2009,0,0,0 +ADJOURNMENTS,0,0,0,2009,0,0,0 +ADJUDGES,0,0,0,2009,0,0,0 +ADJUDICATES,0,0,0,2009,0,0,0 +ADJUDICATIVE,0,0,0,2009,0,0,0 +ADMISSIBILITY,0,0,0,2009,0,0,0 +ADMISSIONS,0,0,0,2009,0,0,0 +ADULTERATION,2009,0,0,0,0,0,0 +ADVANCES,0,2009,0,0,0,0,0 +ADVANTAGEOUS,0,2009,0,0,0,0,0 +ADVERSARIES,2009,0,0,0,0,0,0 +ADVERSITIES,2009,0,0,0,0,0,0 +AFFIRMANCE,0,0,0,2011,0,0,0 +AFORESAID,0,0,0,2009,0,0,0 +AGAINST,2009,0,0,0,0,0,0 +AGGRAVATING,2009,0,0,0,0,0,0 +ALERTED,2009,0,0,0,0,0,0 +ALIENATES,2009,0,0,0,0,0,0 +ALLEGATION,2009,0,0,2009,0,0,0 +ALLEGEDLY,2009,0,0,2009,0,0,0 +ALLIANCES,0,2009,0,0,0,0,0 +ALWAYS,0,0,0,0,2009,0,0 +AMEND,0,0,0,2009,0,0,0 +AMENDING,0,0,0,2009,0,0,0 +ANNOY,2009,0,0,0,0,0,0 +ANNOYING,2009,0,0,0,0,0,0 +ANNULLING,2009,0,0,0,0,0,0 +ANOMALIES,2009,0,2009,0,0,0,0 +ANTECEDENT,0,0,0,2009,0,0,0 +ANTICIPATES,0,0,2009,0,0,0,0 +ANTICOMPETITIVE,2009,0,0,0,0,0,0 +APPARENT,0,0,2009,0,0,0,0 +APPEALED,0,0,0,2009,0,0,0 +APPEARED,0,0,2009,0,0,2009,0 +APPELLANTS,0,0,0,2009,0,0,0 +APPROXIMATE,0,0,2009,0,0,0,0 +APPROXIMATING,0,0,2009,0,0,0,0 +APPURTENANCES,0,0,0,2009,0,0,0 +ARBITRARILY,0,0,2009,0,0,0,0 +ARBITRATED,0,0,0,2009,0,0,0 +ARBITRATIONAL,0,0,0,2011,0,0,0 +ARBITRATORS,0,0,0,2009,0,0,0 +ARGUMENT,2009,0,0,0,0,0,0 +ARREARAGES,2009,0,0,2009,0,0,0 +ARRESTS,2009,0,0,0,0,0,0 +RETROCEDED,0,0,0,2011,0,0,0 +BOLSTERING,0,2009,0,0,0,0,0 +BOOM,0,2009,0,0,0,0,0 +BOTTLENECK,2009,0,0,0,0,0,0 +BOYCOTT,2009,0,0,0,0,0,0 +BREACH,2009,0,0,2009,0,0,0 +BREAK,2009,0,0,0,0,0,0 +BREAKDOWNS,2009,0,0,0,0,0,0 +BREAKTHROUGHS,0,2009,0,0,0,0,0 +BRIBERY,2009,0,0,0,0,0,0 +BRILLIANT,0,2009,0,0,0,0,0 +BURDENING,2009,0,0,0,0,0,0 +CALAMITIES,2009,0,0,0,0,0,0 +CANCELED,2009,0,0,0,0,0,0 +CANCELLED,2009,0,0,0,0,0,0 +CARELESSLY,2009,0,0,0,0,0,0 +CATASTROPHES,2009,0,0,0,0,0,0 +CAUTIONARY,2009,0,0,0,0,0,0 +CAUTIOUS,0,0,2009,0,0,0,0 +CEASED,2009,0,0,0,0,0,0 +CEDANTS,0,0,0,2012,0,0,0 +CENSURING,2009,0,0,0,0,0,0 +CHALLENGED,2009,0,0,0,0,0,0 +CHARITABLE,0,2009,0,0,0,0,0 +CIRCUMVENT,2009,0,0,0,0,0,0 +CIRCUMVENTIONS,2009,0,0,0,0,0,0 +SHORTAGE,2009,0,0,0,0,0,0 +SHRINKAGE,2009,0,0,0,0,0,0 +SHUTDOWNS,2009,0,0,0,0,0,0 +SLANDERED,2009,0,0,0,0,0,0 +REPUDIATES,2009,0,0,0,0,0,0 +REQUESTER,0,0,0,2011,0,0,0 +REQUIREMENT,0,0,0,0,0,0,2009 +REREGULATION,0,0,0,2011,0,0,0 +RESCINDS,0,0,0,2009,0,0,0 +RESIGNATION,2009,0,0,0,0,0,0 +RESIGNS,2009,0,0,0,0,0,0 +RESTATEMENT,2009,0,0,0,0,0,0 +RESTITUTIONARY,0,0,0,2011,0,0,0 +NOTARIAL,0,0,0,2009,0,0,0 +NOTARIZE,0,0,0,2009,0,0,0 +NOTWITHSTANDING,0,0,0,2009,0,0,0 +NULLIFICATION,2009,0,0,2009,0,0,0 +NULLIFY,2009,0,0,2009,0,0,0 +OBJECTED,2009,0,0,0,0,0,0 +OBJECTIONABLY,2009,0,0,0,0,0,0 +OBLIGATES,0,0,0,0,0,0,2009 +OBLIGATORY,0,0,0,0,0,0,2009 +OBLIGEES,0,0,0,2011,0,0,0 +OBSCENE,2009,0,0,0,0,0,0 +OBSTACLE,2009,0,0,0,0,0,0 +OBSTRUCTING,2009,0,0,0,0,0,0 +OFFENCE,2009,0,0,0,0,0,0 +OFFENDER,2009,0,0,0,0,0,0 +OFFENSE,0,0,0,2009,0,0,0 +OFFERORS,0,0,0,2011,0,0,0 +OMITS,2009,0,0,0,0,0,0 +OPPORTUNISTIC,2009,0,0,0,0,0,0 +OPPOSE,2009,0,0,0,0,0,0 +OPPOSITION,2009,0,0,0,0,0,0 +OPTIONEES,0,0,0,2009,0,0,0 +OUTDATED,2009,0,0,0,0,0,0 +OUTPERFORMING,0,2009,0,0,0,0,0 +OVERBUILD,2009,0,0,0,0,0,0 +OVERBURDEN,2009,0,0,0,0,0,0 +OVERCAPACITY,2009,0,0,0,0,0,0 +OVERCHARGING,2009,0,0,0,0,0,0 +OVERDUE,2009,0,0,0,0,0,0 +OVERESTIMATING,2009,0,0,0,0,0,0 +OVERLOADED,2009,0,0,0,0,0,0 +OVERLOOKED,2009,0,0,0,0,0,0 +OVERPAYMENT,2009,0,0,0,0,0,0 +OVERPRODUCING,2009,0,0,0,0,0,0 +OVERRULES,0,0,0,2009,0,0,0 +OVERRUNS,2009,0,0,0,0,0,0 +OVERSHADOWS,2009,0,0,0,0,0,0 +OVERSTATEMENTS,2009,0,0,0,0,0,0 +OVERSUPPLIES,2009,0,0,0,0,0,0 +OVERTURN,2009,0,0,0,0,0,0 +OVERVALUE,2009,0,0,0,0,0,0 +PANICS,2009,0,0,0,0,0,0 +NECESSITATE,0,0,0,0,0,0,2009 +NEGATIVE,2009,0,0,0,0,0,0 +NEGLECTED,2009,0,0,0,0,0,0 +NEGLIGENCE,2009,0,0,0,0,0,0 +NEVER,0,0,0,0,2009,0,0 +BARRIERS,2009,0,0,0,0,0,0 +BELIEVED,0,0,2009,0,0,0,0 +IDLING,2009,0,0,0,0,0,0 +IGNORING,2009,0,0,0,0,0,0 +ILLEGALITY,2009,0,0,0,0,0,0 +ILLICITLY,2009,0,0,0,0,0,0 +IMBALANCES,2009,0,0,0,0,0,0 +IMPAIR,2009,0,0,0,0,0,2011 +IMPAIRMENTS,2009,0,0,0,0,0,2011 +IMPEDE,2009,0,0,0,0,0,0 +IMPEDIMENTS,2009,0,0,0,0,0,0 +IMPERFECTION,2009,0,0,0,0,0,0 +IMPLEADED,0,0,0,2009,0,0,0 +IMPLICATING,2009,0,0,0,0,0,0 +IMPOSING,0,0,0,0,0,0,2009 +IMPOSSIBLE,2009,0,0,0,0,0,0 +IMPOUNDS,2009,0,0,0,0,0,0 +IMPRACTICALITY,2009,0,0,0,0,0,0 +IMPRESS,0,2009,0,0,0,0,0 +IMPRESSIVE,0,2009,0,0,0,0,0 +IMPROBABLE,0,0,2009,0,0,0,0 +IMPROPRIETY,2009,0,0,0,0,0,0 +IMPROVEMENTS,0,2009,0,0,0,0,0 +IMPRUDENTLY,2009,0,0,0,0,0,0 +INACCURACY,2009,0,0,0,0,0,0 +INACTIONS,2009,0,0,0,0,0,0 +INACTIVATING,2009,0,0,0,0,0,0 +INADEQUACIES,2009,0,0,0,0,0,0 +INADVERTENT,2009,0,0,0,0,0,0 +INAPPROPRIATE,2009,0,0,0,0,0,0 +INCAPABLE,2009,0,0,0,0,0,0 +INCARCERATED,2009,0,0,2009,0,0,0 +INCARCERATIONS,2009,0,0,2009,0,0,0 +INCIDENT,2009,0,0,0,0,0,0 +INCOMPATIBLE,2009,0,0,0,0,0,0 +INCOMPETENTLY,2009,0,0,0,0,0,0 +INCOMPLETENESS,2009,0,2009,0,0,0,0 +INCONSISTENT,2009,0,0,0,0,0,0 +INCONVENIENCE,2009,0,0,0,0,0,0 +INCORRECTLY,2009,0,0,0,0,0,0 +INDEBTED,0,0,0,0,0,0,2009 +INDEFEASIBLY,2009,0,0,0,0,0,0 +INDEMNIFIABLE,0,0,0,2009,0,0,0 +INDEMNIFIES,0,0,0,2009,0,0,0 +INDEMNITEES,0,0,0,2009,0,0,0 +INDEMNITY,0,0,0,2009,0,0,0 +INDICTABLE,2009,0,0,2009,0,0,0 +INDICTMENTS,2009,0,0,2009,0,0,0 +INEFFECTIVENESS,2009,0,0,0,0,0,0 +INEFFICIENTLY,2009,0,0,0,0,0,0 +INEQUITABLY,2009,0,0,0,0,0,0 +INEXACT,0,0,2009,0,0,0,0 +INFERIOR,2009,0,0,0,0,0,0 +INFORMATIVE,0,2009,0,0,0,0,0 +INFRINGED,2009,0,0,0,0,0,0 +INFRINGES,2009,0,0,0,0,0,0 +INHIBITED,2009,0,0,0,0,0,2009 +INJUNCTION,2009,0,0,2009,0,0,0 +INJURED,2009,0,0,0,0,0,0 +INJURIOUS,2009,0,0,0,0,0,0 +INNOVATES,0,2009,0,0,0,0,0 +INNOVATIVE,0,2009,0,0,0,0,0 +INORDINATE,2009,0,0,0,0,0,0 +INSENSITIVE,2009,0,0,0,0,0,0 +INSISTENCE,0,0,0,0,0,0,2009 +INSOLVENCIES,2009,0,0,0,0,0,0 +INSPIRATIONAL,0,2009,0,0,0,0,0 +INSUFFICIENCY,2009,0,0,0,0,0,0 +INSURRECTIONS,2009,0,0,0,0,0,0 +INTENTIONAL,2009,0,0,0,0,0,0 +INTERFERENCES,2009,0,0,0,0,0,0 +INTERMITTENT,2009,0,0,0,0,0,0 +INTERPOSED,0,0,0,2009,0,0,0 +INTERPOSITIONS,0,0,0,2009,0,0,0 +INTERROGATING,0,0,0,2009,0,0,0 +INTERROGATORIES,0,0,0,2009,0,0,0 +INTERRUPTED,2009,0,0,0,0,0,0 +INTERRUPTS,2009,0,0,0,0,0,0 +STABILITY,0,2009,0,0,0,0,0 +STABILIZED,0,2009,0,0,0,0,0 +STAGGERING,2009,0,0,0,0,0,0 +STAGNATES,2009,0,0,0,0,0,0 +STANDSTILLS,2009,0,0,0,0,0,0 +STATUTORY,0,0,0,2009,0,0,0 +STIPULATING,0,0,0,0,0,0,2009 +STOPPAGE,2009,0,0,0,0,0,0 +STOPS,2009,0,0,0,0,0,0 +STRAINS,2009,0,0,0,0,0,0 +STRENGTHENING,0,2009,0,0,0,0,0 +STRESSED,2009,0,0,0,0,0,0 +STRICT,0,0,0,0,0,0,2009 +STRINGENT,2009,0,0,0,0,0,0 +STRONGLY,0,0,0,0,2009,0,0 +SUBJECTED,2009,0,0,0,0,0,0 +SUBLEASEHOLD,0,0,0,2011,0,0,0 +SUBPARAGRAPH,0,0,0,2009,0,0,0 +SUBPOENAS,2009,0,0,2009,0,0,0 +SUBTRUST,0,0,0,2011,0,0,0 +SUCCEEDING,0,2009,0,0,0,0,0 +SUCCESSFUL,0,2009,0,0,0,0,0 +SUE,2009,0,0,2009,0,0,0 +SUFFERED,2009,0,0,0,0,0,0 +SUGGESTED,0,0,2009,0,0,0,0 +SUMMONED,2009,0,0,2009,0,0,0 +SUPERIOR,0,2009,0,0,0,0,0 +SUPERSEDES,0,0,0,2009,0,0,0 +SURPASS,0,2009,0,0,0,0,0 +SUSCEPTIBILITY,2009,0,2009,0,0,0,0 +SUSPECTS,2009,0,0,0,0,0,0 +SUSPENDS,2009,0,0,0,0,0,0 +SUSPICIONS,2009,0,0,0,0,0,0 +REVOCABILITY,0,0,0,2011,0,0,0 +REVOKED,2009,0,0,0,0,0,0 +REVOLUTIONIZED,0,2009,0,0,0,0,0 +REWARDED,0,2009,0,0,0,0,0 +RIDICULED,2009,0,0,0,0,0,0 +RISKED,0,0,2009,0,0,0,0 +RISKING,0,0,2009,0,0,0,0 +RULING,0,0,0,2009,0,0,0 +SACRIFICE,2009,0,0,0,0,0,0 +SACRIFICING,2009,0,0,0,0,0,0 +SATISFIED,0,2009,0,0,0,0,0 +SCANDALOUS,2009,0,0,0,0,0,0 +SCRUTINIZES,2009,0,0,0,0,0,0 +SEEMS,0,0,2009,0,0,0,0 +SEIZING,2009,0,0,0,0,0,0 +SENTENCING,2009,0,0,2009,0,0,0 +SERIOUSNESS,2009,0,0,0,0,0,0 +SETTLEMENTS,0,0,0,2009,0,0,0 +SEVERALLY,0,0,0,2009,0,0,0 +SEVERED,2009,0,0,0,0,0,0 +ENTHUSIASTICALLY,0,2009,0,0,0,0,0 +ERODED,2009,0,0,0,0,0,0 +ERRATIC,2009,0,0,0,0,0,0 +ERRONEOUS,2009,0,0,0,0,0,0 +ERRS,2009,0,0,0,0,0,0 +ESCALATING,2009,0,0,0,0,0,0 +ESCROW,0,0,0,0,0,0,2009 +ESTOPPEL,0,0,0,2011,0,0,0 +EVADING,2009,0,0,0,0,0,0 +EVICT,2009,0,0,0,0,0,0 +EVICTIONS,2009,0,0,0,0,0,0 +EXACERBATE,2009,0,0,0,0,0,0 +EXACERBATION,2009,0,0,0,0,0,0 +EXAGGERATES,2009,0,0,0,0,0,0 +EXCEEDANCES,0,0,0,2011,0,0,0 +EXCELLING,0,2009,0,0,0,0,0 +EXCESSIVE,2009,0,0,0,0,0,0 +EXCITEMENT,0,2009,0,0,0,0,0 +EXCLUSIVENESS,0,2009,0,0,0,0,0 +EXCULPATED,2009,0,0,2009,0,0,0 +EXCULPATIONS,2009,0,0,2009,0,0,0 +EXECUTORY,0,0,0,2009,0,0,0 +EXEMPLARY,0,2009,0,0,0,0,0 +EXONERATING,2009,0,0,0,0,0,0 +EXPLOITATION,2009,0,0,0,0,0,0 +EXPLOITING,2009,0,0,0,0,0,0 +EXPOSES,2009,0,0,0,0,0,0 +EXPROPRIATE,2009,0,0,0,0,0,0 +EXPROPRIATION,2009,0,0,0,0,0,0 +EXTENUATING,2009,0,0,0,0,0,0 +SOLVING,0,2009,0,0,0,0,0 +SOMEWHERE,0,0,2009,0,0,0,0 +LOSE,2009,0,0,0,0,0,0 +LOSSES,2009,0,0,0,0,0,0 +LUCRATIVE,0,2009,0,0,0,0,0 +MALFUNCTION,2009,0,0,0,0,0,0 +MALICE,2009,0,0,0,0,0,0 +MANDAMUS,0,0,0,2009,0,0,0 +MANDATING,0,0,0,0,0,0,2009 +MANIPULATED,2009,0,0,0,0,0,0 +MANIPULATIONS,2009,0,0,0,0,0,0 +MAY,0,0,2009,0,0,2009,0 +MEDIATES,0,0,0,2009,0,0,0 +MEDIATOR,0,0,0,2009,0,0,0 +MISAPPLICATION,2009,0,0,0,0,0,0 +MISAPPLY,2009,0,0,0,0,0,0 +MISAPPROPRIATES,2009,0,0,0,0,0,0 +MISBRANDED,2009,0,0,0,0,0,0 +MISCALCULATING,2009,0,0,0,0,0,0 +MISCHIEF,2009,0,0,0,0,0,0 +MISCLASSIFY,2014,0,0,0,0,0,0 +MISDEMEANOR,2009,0,0,2009,0,0,0 +MISHANDLE,2009,0,0,0,0,0,0 +MISINFORM,2009,0,0,0,0,0,0 +MISINFORMS,2009,0,0,0,0,0,0 +MISINTERPRETED,2009,0,0,0,0,0,0 +MISJUDGED,2009,0,0,0,0,0,0 +MISJUDGMENTS,2009,0,0,0,0,0,0 +MISLABELLED,2009,0,0,0,0,0,0 +MISLEADINGLY,2009,0,0,0,0,0,0 +MISMANAGED,2009,0,0,0,0,0,0 +MISMATCH,2009,0,0,0,0,0,0 +MISPLACED,2009,0,0,0,0,0,0 +MISREPRESENT,2009,0,0,0,0,0,0 +MISREPRESENTING,2009,0,0,0,0,0,0 +MISSES,2009,0,0,0,0,0,0 +WORRY,2009,0,0,0,0,0,0 +WORSENED,2009,0,0,0,0,0,0 +WORTHLESS,2009,0,0,0,0,0,0 +TOLERATE,2009,0,0,0,0,0,0 +TOLERATION,2009,0,0,0,0,0,0 +TORTS,0,0,0,2009,0,0,0 +FICTITIOUS,2009,0,0,0,0,0,0 +FIRED,2009,0,0,0,0,0,0 +NONATTAINMENT,2009,0,0,0,0,0,0 +NONCOMPETITIVE,2009,0,0,0,0,0,0 +NONCOMPLYING,2009,0,0,0,0,0,0 +NONCONTINGENT,0,0,0,2011,0,0,0 +NONDISCLOSURE,2009,0,0,0,0,0,0 +NONFORFEITABLE,0,0,0,2009,0,0,0 +DISCLAIM,2009,0,0,0,0,0,0 +DISCLAIMING,2009,0,0,0,0,0,0 +DISCLOSES,2009,0,0,0,0,0,0 +DISCONTINUATION,2009,0,0,0,0,0,0 +DISCONTINUES,2009,0,0,0,0,0,0 +DISCOURAGES,2009,0,0,0,0,0,0 +DISCREDITING,2009,0,0,0,0,0,0 +SPECTACULAR,0,2009,0,0,0,0,0 +SPECULATES,0,0,2009,0,0,0,0 +SPECULATIVE,0,0,2009,0,0,0,0 +TRAGEDIES,2009,0,0,0,0,0,0 +TRANSFEROR,0,0,0,2009,0,0,0 +LEGISLATES,0,0,0,2009,0,0,0 +LEGISLATIVE,0,0,0,2009,0,0,0 +LEGISLATURE,0,0,0,2009,0,0,0 +LIBELOUS,0,0,0,2009,0,0,0 +LIENHOLDERS,0,0,0,2011,0,0,0 +COMPULSORY,0,0,0,0,0,0,2009 +CONCEDED,2009,0,0,0,0,0,0 +CONCEIVABLY,0,0,2009,0,0,0,0 +CONCILIATING,2009,0,0,0,0,0,0 +CONCLUSIVELY,0,2009,0,0,0,0,0 +CONDEMNED,2009,0,0,0,0,0,0 +CONDITIONAL,0,0,2009,0,0,0,0 +CONDUCIVE,0,2009,0,0,0,0,0 +CONFESSING,2009,0,0,0,0,0,0 +CONFINED,2009,0,0,0,0,0,2009 +CONFINING,2009,0,0,0,0,0,2009 +CONFISCATING,2009,0,0,0,0,0,0 +CONFLICT,2009,0,0,0,0,0,0 +CONFRONT,2009,0,0,0,0,0,0 +CONFRONTED,2009,0,0,0,0,0,0 +CONFUSED,2009,0,0,0,0,0,0 +CONFUSION,2009,0,2009,0,0,0,0 +CONSENTS,0,0,0,2009,0,0,0 +CONSPIRATOR,2009,0,0,0,0,0,0 +CONSPIRED,2009,0,0,0,0,0,0 +CONSTITUTIONAL,0,0,0,2009,0,0,0 +CONSTITUTIVE,0,0,0,2009,0,0,0 +CONSTRAINS,0,0,0,0,0,0,2009 +CONSTRUCTIVELY,0,2009,0,0,0,0,0 +CONSTRUING,0,0,0,2012,0,0,0 +CONTENDING,2009,0,0,0,0,0,0 +CONTENTIOUS,2009,0,0,0,0,0,0 +CONTESTED,2009,0,0,0,0,0,0 +CONTINGENT,0,0,2009,0,0,0,0 +CONTRACTED,0,0,0,2009,0,0,0 +CONTRACTILE,0,0,0,2009,0,0,0 +CONTRACTS,0,0,0,2009,0,0,0 +CONTRADICTED,2009,0,0,0,0,0,0 +CONTRADICTORY,2009,0,0,0,0,0,0 +CONTRAVENED,0,0,0,2009,0,0,0 +CONTRAVENTIONS,0,0,0,2009,0,0,0 +CONTROVERT,0,0,0,2009,0,0,0 +CONVEYANCE,0,0,0,2009,0,0,0 +CONVICTING,2009,0,0,2009,0,0,0 +CORRECTING,2009,0,0,0,0,0,0 +CORRUPT,2009,0,0,0,0,0,0 +CORRUPTIONS,2009,0,0,0,0,0,0 +COTERMINOUS,0,0,0,2009,0,0,0 +COUNSELLED,0,0,0,2009,0,0,0 +COUNTERCLAIMING,2009,0,0,0,0,0,0 +COUNTERFEITER,2009,0,0,0,0,0,0 +COUNTERMEASURE,2009,0,0,0,0,0,0 +COUNTERSUIT,0,0,0,2011,0,0,0 +COURTROOM,0,0,0,2009,0,0,0 +COVENANTING,0,0,0,0,0,0,2011 +CREATIVENESS,0,2009,0,0,0,0,0 +CRIMINAL,2009,0,0,2009,0,0,0 +CRIMINALLY,2009,0,0,2009,0,0,0 +CRITICAL,-2020,0,0,0,0,0,0 +CRITICIZE,2009,0,0,0,0,0,0 +CROSSCLAIM,0,0,0,2011,0,0,0 +CRUCIAL,2009,0,0,0,0,0,0 +CULPABLY,2009,0,0,0,0,0,0 +CURTAILING,2009,0,0,0,0,0,0 +CUT,2009,0,0,0,0,0,0 +CYBERATTACKS,2014,0,0,0,0,0,0 +CYBERCRIMINAL,2014,0,0,0,0,0,0 +TAINT,2009,0,0,0,0,0,0 +TAMPERED,2009,0,0,0,0,0,0 +TENTATIVE,0,0,2009,0,0,0,0 +TERMINATED,2009,0,0,0,0,0,0 +TERMINATIONS,2009,0,0,0,0,0,0 +TESTIFYING,2009,0,0,2009,0,0,0 +THENCEFORWARD,0,0,0,2009,0,0,0 +THEREIN,0,0,0,2009,0,0,0 +THEREOVER,0,0,0,2011,0,0,0 +THEREUNDER,0,0,0,2009,0,0,0 +THREAT,2009,0,0,0,0,0,0 +THREATENS,2009,0,0,0,0,0,0 +FACTO,0,0,0,2011,0,0,0 +FAILINGS,2009,0,0,0,0,0,0 +FALLOUT,2009,0,0,0,0,0,0 +FALSIFICATIONS,2009,0,0,0,0,0,0 +FALSIFYING,2009,0,0,0,0,0,0 +FATALITY,2009,0,0,0,0,0,0 +FAULTS,2009,0,0,0,0,0,0 +FAVORED,0,2009,0,0,0,0,0 +FEAR,2009,0,0,0,0,0,0 +FELONY,2009,0,0,2009,0,0,0 +DAMAGING,2009,0,0,0,0,0,0 +DANGEROUS,2009,0,0,0,0,0,0 +DEADLOCKED,2009,0,0,0,0,0,0 +DEADWEIGHTS,2009,0,0,0,0,0,0 +DECEASED,2009,0,0,0,0,0,0 +DECEITFUL,2009,0,0,0,0,0,0 +DECEIVES,2009,0,0,0,0,0,0 +DECEPTIVE,2009,0,0,0,0,0,0 +DECLINED,2009,0,0,0,0,0,0 +DECREED,0,0,0,2009,0,0,0 +DEFACED,2009,0,0,0,0,0,0 +DEFAMATION,2009,0,0,0,0,0,0 +DEFAMED,2009,0,0,0,0,0,0 +DEFAULTED,2009,0,0,0,0,0,0 +DEFEASANCES,0,0,0,2011,0,0,0 +DEFEASES,0,0,0,2014,0,0,0 +DEFEATING,2009,0,0,0,0,0,0 +DEFECTIVELY,0,0,0,2009,0,0,0 +DEFENDANT,2009,0,0,2009,0,0,0 +DEFENDS,2009,0,0,0,0,0,0 +DEFICIENCIES,2009,0,0,0,0,0,0 +DEFICITS,2009,0,0,0,0,0,0 +DEFRAUDED,2009,0,0,0,0,0,0 +DEGRADATION,2009,0,0,0,0,0,0 +DEGRADES,2009,0,0,0,0,0,0 +DELAYING,2009,0,0,0,0,0,0 +DELEGATEE,0,0,0,2011,0,0,0 +DELIBERATED,2009,0,0,0,0,0,0 +DELIGHTFUL,0,2009,0,0,0,0,0 +DELINQUENCIES,2009,0,0,0,0,0,0 +DELINQUENTS,2009,0,0,0,0,0,0 +DELISTS,2011,0,0,0,0,0,0 +DEMISING,2009,0,0,0,0,0,0 +DEMOLISHING,2009,0,0,0,0,0,0 +DEMOTED,2009,0,0,0,0,0,0 +DEMOTIONS,2009,0,0,0,0,0,0 +DEMURRING,0,0,0,2009,0,0,0 +DENIED,2009,0,0,0,0,0,0 +DENIGRATES,2009,0,0,0,0,0,0 +DENYING,2009,0,0,0,0,0,0 +DEPENDANCE,0,0,0,0,0,0,2011 +DEPENDENCE,0,0,2009,0,0,0,0 +DEPENDING,0,0,2009,0,0,2009,2011 +DEPLETES,2009,0,0,0,0,0,0 +DEPOSE,0,0,0,2009,0,0,0 +DEPOSITION,0,0,0,2009,0,0,0 +INVALIDATE,2009,0,0,0,0,0,0 +INVALIDATION,2009,0,0,0,0,0,0 +INVENTING,0,2009,0,0,0,0,0 +INVENTIVENESS,0,2009,0,0,0,0,0 +INVESTIGATED,2009,0,0,0,0,0,0 +INVESTIGATIONS,2009,0,0,0,0,0,0 +IRRECONCILABLY,2009,0,0,0,0,0,0 +IRREGULARITIES,2009,0,0,0,0,0,0 +IRREPARABLY,2009,0,0,0,0,0,0 +IRREVOCABLY,0,0,0,2009,0,0,2009 +JUDICIAL,0,0,0,2009,0,0,0 +JURIES,0,0,0,2009,0,0,0 +JURISDICTIONALLY,0,0,0,2011,0,0,0 +JURISTS,0,0,0,2009,0,0,0 +JURYMAN,0,0,0,2009,0,0,0 +KICKBACK,2009,0,0,0,0,0,0 +DIMINISH,2009,0,0,0,0,0,0 +DIMINUTION,2009,0,0,0,0,0,0 +DISADVANTAGED,2009,0,0,0,0,0,0 +DISAFFIRM,0,0,0,2009,0,0,0 +DISAGREE,2009,0,0,0,0,0,0 +DISAGREEMENT,2009,0,0,0,0,0,0 +DISALLOWANCE,2009,0,0,0,0,0,0 +DISALLOWS,2009,0,0,0,0,0,0 +DISAPPEARED,2009,0,0,0,0,0,0 +DISAPPOINTED,2009,0,0,0,0,0,0 +DISAPPOINTMENTS,2009,0,0,0,0,0,0 +DISAPPROVE,2009,0,0,0,0,0,0 +DISASSOCIATES,2009,0,0,0,0,0,0 +DISASTER,2009,0,0,0,0,0,0 +DISAVOW,2009,0,0,0,0,0,0 +DISAVOWS,2009,0,0,0,0,0,0 +GRATUITOUS,2009,0,0,0,0,0,0 +GREATEST,0,2009,0,0,0,0,0 +GRIEVANCES,2009,0,0,0,0,0,0 +HALT,2009,0,0,0,0,0,0 +HAMPERING,2009,0,0,0,0,0,0 +HAPPINESS,0,2009,0,0,0,0,0 +HARASSING,2009,0,0,0,0,0,0 +HARM,2009,0,0,0,0,0,0 +HARMING,2009,0,0,0,0,0,0 +HARSHEST,2009,0,0,0,0,0,0 +HAZARDOUS,2009,0,0,0,0,0,0 +NONINFRINGEMENT,0,0,0,2011,0,0,0 +NONJURISDICTIONAL,0,0,0,2011,0,0,0 +NONPERFORMANCES,2009,0,0,0,0,0,0 +HURTING,2009,0,0,0,0,0,0 +DISFAVORING,2009,0,0,0,0,0,0 +DISGORGEMENT,2009,0,0,0,0,0,0 +COMPLICATE,2009,0,0,0,0,0,0 +COMPLICATION,2009,0,0,0,0,0,0 +COMPLIMENTED,0,2009,0,0,0,0,0 +MISSTATEMENT,2009,0,0,0,0,0,0 +MISSTEP,2009,0,0,0,0,0,0 +MISTAKENLY,2009,0,0,0,0,0,0 +MISTRIALS,2009,0,0,2009,0,0,0 +MISUNDERSTOOD,2009,0,0,0,0,0,0 +MISUSING,2009,0,0,0,0,0,0 +MONOPOLIZE,2009,0,0,0,0,0,0 +MONOPOLY,2009,0,0,0,0,0,0 +MOREOVER,0,0,0,2009,0,0,0 +MUST,0,0,0,0,2009,0,0 +SLIPPAGE,2009,0,0,0,0,0,0 +SLOWDOWNS,2009,0,0,0,0,0,0 +SLOWING,2009,0,0,0,0,0,0 +SLUGGISHLY,2009,0,0,0,0,0,0 +SMOOTHLY,0,2009,0,0,0,0,0 +DEPRESSED,2009,0,0,0,0,0,0 +DEPRIVE,2009,0,0,0,0,0,0 +DERELICT,2009,0,0,0,0,0,0 +DEROGATES,0,0,0,2009,0,0,0 +DEROGATORY,2009,0,0,0,0,0,0 +DESIST,0,0,0,2009,0,0,0 +DESTABILIZED,2009,0,0,0,0,0,0 +DESTROYED,2009,0,0,0,0,0,0 +DESTRUCTIVE,2009,0,0,0,0,0,0 +DETENTION,2009,0,0,0,0,0,0 +DETERIORATED,2009,0,0,0,0,0,0 +DETERIORATIONS,2009,0,0,0,0,0,0 +DETERRENT,2009,0,0,0,0,0,0 +DETRACT,2009,0,0,0,0,0,0 +DETRIMENTAL,2009,0,0,0,0,0,0 +DEVALUED,2009,0,0,0,0,0,0 +DEVASTATED,2009,0,0,0,0,0,0 +DEVIATED,2009,0,2009,0,0,0,0 +DEVIATIONS,2009,0,2009,0,0,0,0 +DEVOLVES,2009,0,0,0,0,0,0 +DICTATES,0,0,0,0,0,0,2009 +DIFFERING,0,0,2009,0,0,0,0 +DIFFICULTLY,2009,0,0,0,0,0,0 +ASCENDANT,0,0,0,2009,0,0,0 +ASSAULTING,2009,0,0,0,0,0,0 +ASSIGNATION,0,0,0,2009,0,0,0 +ASSUMED,0,0,2009,0,0,0,0 +ASSUMPTIONS,0,0,2009,0,0,0,0 +ASSURING,0,2009,0,0,0,0,0 +ATTAINMENT,0,2009,0,0,0,0,0 +ATTESTATION,0,0,0,2009,0,0,0 +ATTORN,0,0,0,2011,0,0,0 +ATTORNS,0,0,0,2011,0,0,0 +AVERSELY,2011,0,0,0,0,0,0 +BAILED,0,0,0,2009,0,0,0 +BAILIFFS,0,0,0,2009,0,0,0 +BALKED,2009,0,0,0,0,0,0 +BANKRUPTED,2009,0,0,0,0,0,0 +CLAIMANT,0,0,0,2009,0,0,0 +CLAIMS,2009,0,0,2009,0,0,0 +CLAWBACKS,0,0,0,2014,0,0,0 +CLOSEOUTS,2009,0,0,0,0,0,0 +CLOSURES,2009,0,0,0,0,0,0 +CODICILS,0,0,0,2009,0,0,0 +CODIFIES,0,0,0,2009,0,0,0 +COERCED,2009,0,0,0,0,0,0 +COERCIVE,2009,0,0,0,0,0,0 +DISINCENTIVES,2009,0,0,0,0,0,0 +COLLABORATE,0,2009,0,0,0,0,0 +COLLABORATION,0,2009,0,0,0,0,0 +COLLABORATORS,0,2009,0,0,0,0,0 +COLLAPSING,2009,0,0,0,0,0,0 +COLLUDED,2009,0,0,0,0,0,0 +COLLUSIONS,2009,0,0,0,0,0,0 +POSITIVE,0,2009,0,0,0,0,0 +POSSIBILITY,0,0,2009,0,0,0,0 +POSTCLOSURE,0,0,0,2011,0,0,0 +POSTPONED,2009,0,0,0,0,0,0 +POSTPONING,2009,0,0,0,0,0,0 +WRIT,0,0,0,2009,0,0,0 +WRITEOFFS,2009,0,0,0,0,0,0 +WRONGDOINGS,2009,0,0,0,0,0,0 +HONORING,0,2009,0,0,0,0,0 +REASSESSES,0,0,2009,0,0,0,0 +REASSIGN,2009,0,0,0,0,0,0 +REASSIGNMENTS,2009,0,0,0,0,0,0 +REBOUNDING,0,2009,0,0,0,0,0 +REBUTTABLY,0,0,0,2011,0,0,0 +REBUTTING,0,0,0,2009,0,0,0 +RECALCULATING,0,0,2009,0,0,0,0 +RECALLED,2009,0,0,0,0,0,0 +RECESSION,2009,0,0,0,0,0,0 +RECKLESSLY,2009,0,0,0,0,0,0 +RECONSIDERING,0,0,2009,0,0,0,0 +RECOUPMENT,0,0,0,2009,0,0,0 +RECTIFICATION,0,0,0,2009,0,0,0 +RECUSED,0,0,0,2011,0,0,0 +REDACTED,2009,0,0,2009,0,0,0 +REDEFAULT,2014,0,0,0,0,0,0 +REDRESSED,2009,0,0,0,0,0,0 +REEXAMINE,0,0,2009,0,0,0,0 +REFERENDUMS,0,0,0,2009,0,0,0 +REFILING,0,0,0,2009,0,0,0 +REFUSAL,2009,0,0,0,0,0,0 +REFUSES,2009,0,0,0,0,0,0 +REGAINING,0,2009,0,0,0,0,0 +REGULATING,0,0,0,2009,0,0,0 +REGULATOR,0,0,0,2009,0,0,0 +REHEARD,0,0,0,2009,0,0,0 +VARIABLES,0,0,2009,0,0,0,0 +VARIANT,0,0,2009,0,0,0,0 +VARIED,0,0,2009,0,0,0,0 +VENDEE,0,0,0,2011,0,0,0 +VERSATILE,0,2009,0,0,0,0,0 +VIBRANCY,0,2009,0,0,0,0,0 +VIOLATED,2009,0,0,0,0,0,0 +VIOLATIONS,2009,0,0,0,0,0,0 +VIOLENCE,2009,0,0,0,0,0,0 +VITIATED,2009,0,0,0,0,0,0 +VOIDABLE,0,0,0,2009,0,0,0 +VOLATILITIES,0,0,2009,0,0,0,0 +VULNERABLE,2009,0,0,0,0,0,0 +WARNING,2009,0,0,0,0,0,0 +WARRANTOR,0,0,0,2011,0,0,0 +WEAK,2009,0,0,0,0,0,0 +WEAKENS,2009,0,0,0,0,0,0 +WEAKNESS,2009,0,0,0,0,0,0 +DISHONOR,2009,0,0,0,0,0,0 +DISHONORING,2009,0,0,0,0,0,0 +DISINTERESTED,2009,0,0,0,0,0,0 +DISLOYALLY,2009,0,0,0,0,0,0 +DISMISS,2009,0,0,0,0,0,0 +DISMISSES,2009,0,0,0,0,0,0 +DISPARAGED,2009,0,0,0,0,0,0 +DISPARAGING,2009,0,0,0,0,0,0 +DISPLACE,2009,0,0,0,0,0,0 +DISPLACES,2009,0,0,0,0,0,0 +DISPOSSESS,2009,0,0,0,0,0,0 +DISPOSSESSION,0,0,0,2009,0,0,0 +DISPROPORTIONATE,2009,0,0,0,0,0,0 +DISPUTES,2009,0,0,0,0,0,0 +DISQUALIFIED,2009,0,0,0,0,0,0 +DISREGARD,2009,0,0,0,0,0,0 +DISREPUTABLE,2009,0,0,0,0,0,0 +DISRUPTING,2009,0,0,0,0,0,0 +DISRUPTS,2009,0,0,0,0,0,0 +DISSENTED,2009,0,0,0,0,0,0 +DISSENTS,2009,0,0,0,0,0,0 +DISSOLUTIONS,2009,0,0,0,0,0,0 +DISTINCTIVELY,0,2009,0,0,0,0,0 +DISTORTING,2009,0,0,0,0,0,0 +DISTRACT,2009,0,0,0,0,0,0 +DISTRACTIONS,2009,0,0,0,0,0,0 +DISTRESSED,2009,0,0,0,0,0,0 +DISTURBANCE,2009,0,0,0,0,0,0 +DISTURBS,2009,0,0,0,0,0,0 +DIVERTING,2009,0,0,0,0,0,0 +DIVESTING,2009,0,0,0,0,0,0 +DIVESTMENTS,2009,0,0,0,0,0,0 +DIVULGE,2009,0,0,0,0,0,0 +DOCKET,0,0,0,2009,0,0,0 +DONEES,0,0,0,2011,0,0,0 +DOUBTS,2009,0,2009,0,0,0,0 +DOWNGRADING,2009,0,0,0,0,0,0 +DOWNSIZING,2009,0,0,0,0,0,0 +DOWNTURN,2009,0,0,0,0,0,0 +DRAG,2009,0,0,0,0,0,0 +DRAWBACKS,2009,0,0,0,0,0,0 +DROUGHTS,2009,0,0,0,0,0,0 +DYSFUNCTIONAL,2009,0,0,0,0,0,0 +EARMARKING,0,0,0,0,0,0,2009 +EASING,2009,0,0,0,0,0,0 +EFFICIENCY,0,2009,0,0,0,0,0 +EGREGIOUSLY,2009,0,0,0,0,0,0 +EMBARGOES,2009,0,0,0,0,0,0 +EMBARRASSES,2009,0,0,0,0,0,0 +EMBEZZLE,2009,0,0,0,0,0,0 +EMBEZZLER,2009,0,0,0,0,0,0 +EMPOWERED,0,2009,0,0,0,0,0 +ENABLED,0,2009,0,0,0,0,0 +ENCOURAGEMENT,0,2009,0,0,0,0,0 +ENCROACHED,2009,0,0,0,0,0,0 +ENCROACHMENTS,2009,0,0,0,0,0,0 +ENCUMBERS,2009,0,0,2009,0,0,2009 +ENCUMBRANCES,2009,0,0,2009,0,0,2009 +ENDANGERMENT,2009,0,0,0,0,0,0 +ENFORCEABLE,0,0,0,2009,0,0,0 +ENHANCEMENT,0,2009,0,0,0,0,0 +ENJOIN,2009,0,0,0,0,0,0 +ENJOY,0,2009,0,0,0,0,0 +ENJOYING,0,2009,0,0,0,0,0 +ENTAILED,0,0,0,0,0,0,2009 +PENALIZE,2009,0,0,0,0,0,0 +PENALTIES,2009,0,0,0,0,0,0 +PERFECTED,0,2009,0,0,0,0,0 +PERIL,2009,0,0,0,0,0,0 +PERMISSION,0,0,0,0,0,0,2009 +PERMITTEES,0,0,0,2011,0,0,0 +PERPETRATES,2009,0,0,2009,0,0,0 +PERSISTED,2009,0,0,0,0,0,0 +PERSISTING,2009,0,0,0,0,0,0 +PERVASIVELY,2009,0,0,0,0,0,0 +PETITIONER,0,0,0,2009,0,0,0 +PETTY,2009,0,0,0,0,0,0 +HEREAFTER,0,0,0,2009,0,0,0 +HEREFORE,0,0,0,2014,0,0,0 +HEREINAFTER,0,0,0,2009,0,0,0 +HEREON,0,0,0,2009,0,0,0 +HEREUNTO,0,0,0,2009,0,0,0 +HIDDEN,0,0,2009,0,0,0,0 +HINDERING,2009,0,0,0,0,0,0 +HINGES,0,0,2009,0,0,0,0 +WHEREABOUTS,0,0,0,2009,0,0,0 +WHEREFORE,0,0,0,2009,0,0,0 +WHERETO,0,0,0,2009,0,0,0 +WHISTLEBLOWERS,0,0,0,2011,0,0,0 +WILFUL,0,0,0,2009,0,0,0 +WILLFULNESS,0,0,0,2009,0,0,0 +WINNING,0,2009,0,0,0,0,0 +FLUCTUATE,0,0,2009,0,0,0,0 +FLUCTUATION,0,0,2009,0,0,0,0 +FORBEARANCE,0,0,0,2009,0,0,0 +FORBID,2009,0,0,0,0,0,2009 +FORCE,-2020,0,0,0,0,0,0 +FOREBEARANCE,0,0,0,2011,0,0,0 +FORECLOSES,2009,0,0,0,0,0,0 +FOREGO,2009,0,0,0,0,0,0 +FORESTALLED,2009,0,0,0,0,0,0 +FORFEITABILITY,0,0,0,2011,0,0,0 +FORFEITS,2009,0,0,0,0,0,0 +FORGERY,2009,0,0,0,0,0,0 +FRAUDS,2009,0,0,0,0,0,0 +FRIENDLY,0,2009,0,0,0,0,0 +FRUSTRATED,2009,0,0,0,0,0,0 +FRUSTRATION,2009,0,0,0,0,0,0 +FURTHERANCE,0,0,0,2009,0,0,0 +GAINS,0,2009,0,0,0,0,0 +DISGORGEMENTS,2009,0,0,0,0,0,0 +DISGRACEFUL,2009,0,0,0,0,0,0 +LITIGATE,2009,0,0,2009,0,0,0 +LITIGATION,2009,0,0,2009,0,0,0 +LITIGIOUS,0,0,0,2009,0,0,0 +LACKED,2009,0,0,0,0,0,0 +LAG,2009,0,0,0,0,0,0 +LAPSE,2009,0,0,0,0,0,0 +LATE,-2020,0,0,0,0,0,0 +LAWFULLY,0,0,0,2009,0,0,0 +LAWS,0,0,0,2009,0,0,0 +LAWYERS,0,0,0,2009,0,0,0 +LEADING,0,2009,0,0,0,0,0 +LEGALIZATION,0,0,0,2009,0,0,0 +LEGALIZES,0,0,0,2009,0,0,0 +LEGATEE,0,0,0,2009,0,0,0 +FLAWED,2009,0,0,0,0,0,0 +NONRECOVERABLE,2009,0,0,0,0,0,0 +LIQUIDATES,2009,0,0,0,0,0,0 +LIQUIDATOR,2009,0,0,0,0,0,0 +BENEFICIATION,0,0,0,2011,0,0,0 +BENEFITTED,0,2009,0,0,0,0,0 +POOR,2009,0,0,0,0,0,0 +REINTERPRETATION,0,0,2009,0,0,0,0 +REINTERPRETS,0,0,2009,0,0,0,0 +REJECTION,2009,0,0,0,0,0,0 +RELINQUISH,2009,0,0,0,0,0,0 +RELINQUISHMENT,2009,0,0,0,0,0,0 +REMAND,0,0,0,2009,0,0,0 +REMEDIATE,0,0,0,2009,0,0,0 +REMEDIATIONS,0,0,0,2009,0,0,0 +RENEGOTIATED,2009,0,0,0,0,0,0 +RENEGOTIATIONS,2009,0,0,0,0,0,0 +RENOUNCEMENTS,2009,0,0,0,0,0,0 +REPARATIONS,2009,0,0,0,0,0,0 +REPOSSESSES,2009,0,0,0,0,0,0 +REPRORATED,0,0,0,2018,0,0,0 +TROUBLED,2009,0,0,0,0,0,0 +UNABLE,2009,0,0,0,0,0,0 +UNAMBIGUOUSLY,0,0,0,0,2009,0,0 +UNAPPEALED,0,0,0,2011,0,0,0 +UNAVAILABILITY,2009,0,0,0,0,0,2011 +UNAWARE,2009,0,0,0,0,0,0 +UNCERTAINTY,0,0,2009,0,0,0,0 +UNCOLLECTIBILITY,2009,0,0,0,0,0,0 +UNCOMPLETED,2009,0,0,0,0,0,0 +UNCONSCIONABLY,2009,0,0,0,0,0,0 +UNCONTRACTED,0,0,0,2011,0,0,0 +UNCORRECTED,2009,0,0,0,0,0,0 +UNCOVERS,2009,0,0,0,0,0,0 +UNDELIVERABLE,2009,0,0,0,0,0,0 +UNDERCUTS,2009,0,0,0,0,0,0 +UNDERESTIMATES,2009,0,0,0,0,0,0 +UNDERINSURED,2009,0,0,0,0,0,0 +UNDERMINING,2009,0,0,0,0,0,0 +UNDERPAYS,2009,0,0,0,0,0,0 +UNDERPERFORMING,2009,0,0,0,0,0,0 +UNDERREPORTING,2011,0,0,0,0,0,0 +UNDERSTATEMENTS,2009,0,0,0,0,0,0 +UNDERUTILIZED,2009,0,0,0,0,0,0 +UNDETECTABLE,0,0,2009,0,0,0,0 +UNDISCHARGED,0,0,0,2009,0,0,0 +UNDOUBTEDLY,0,0,0,0,2009,0,0 +UNECONOMICAL,2009,0,0,0,0,0,0 +UNENCUMBER,0,0,0,2014,0,0,0 +UNEQUIVOCAL,0,0,0,0,2009,0,0 +UNEXCUSED,2009,0,0,0,0,0,0 +UNFAIRLY,2009,0,0,0,0,0,0 +UNFAVORABLE,2009,0,0,0,0,0,0 +UNFIT,2009,0,0,0,0,0,0 +UNFORESEEN,2009,0,0,0,0,0,0 +UNFOUNDED,2009,0,0,0,0,0,0 +UNGUARANTEED,0,0,2009,0,0,0,0 +UNINSURED,2009,0,0,0,0,0,0 +UNJUST,2009,0,0,0,0,0,0 +UNJUSTLY,2009,0,0,0,0,0,0 +UNKNOWNS,0,0,2009,0,0,0,0 +UNLICENSED,2009,0,0,0,0,0,0 +UNMERCHANTABLE,2011,0,0,0,0,0,0 +UNNEEDED,2009,0,0,0,0,0,0 +UNPAID,2009,0,0,0,0,0,0 +UNPOPULAR,2009,0,0,0,0,0,0 +UNPREDICTED,2011,0,2011,0,0,0,0 +UNPROVED,0,0,2009,0,0,0,0 +UNQUANTIFIED,0,0,2011,0,0,0,0 +UNREASONABLY,2009,0,0,0,0,0,0 +UNRECOVERED,2009,0,0,0,0,0,0 +UNREMEDIED,2009,0,0,0,0,0,0 +UNSAFE,2009,0,0,0,0,0,0 +UNSATISFIED,2009,0,0,0,0,0,0 +UNSEASONABLY,0,0,2009,0,0,0,0 +UNSOUND,2009,0,0,0,0,0,0 +UNSTABLE,2009,0,0,0,0,0,0 +UNSUCCESSFULLY,2009,0,0,0,0,0,0 +UNSUITED,2009,0,0,0,0,0,0 +UNSUSPECTING,2009,0,0,0,0,0,0 +UNTIMELY,2009,0,0,0,0,0,0 +UNTRUTHFUL,2009,0,0,0,0,0,0 +UNUSABLE,2009,0,0,0,0,0,0 +UNWARRANTED,2009,0,0,0,0,0,0 +UNWRITTEN,0,0,2009,0,0,0,0 +URGENCY,2009,0,0,0,0,0,0 +USURPATION,0,0,0,2009,0,0,0 +USURY,2009,0,0,2009,0,0,0 +VAGUENESS,0,0,2012,0,0,0,0 +VALUABLE,0,2009,0,0,0,0,0 +PLEAD,2009,0,0,0,0,0,0 +PLEADS,2009,0,0,2009,0,0,0 +PLEASED,0,2009,0,0,0,0,0 +PLEDGED,0,0,0,0,0,0,2009 +PLEDGING,0,0,0,0,0,0,2009 +COMPEL,0,0,0,0,0,0,2011 +COMPENSATORY,0,0,0,2009,0,0,0 +COMPLAINED,2009,0,0,0,0,0,0 +COMPLAINTS,2009,0,0,0,0,0,0 +COMMITMENT,0,0,0,0,0,0,2009 +COMMITTING,0,0,0,0,0,0,2009 +LIMITATION,2009,0,0,0,0,0,0 +LINGERING,2009,0,0,0,0,0,0 +RESTRAINING,0,0,0,0,0,0,2009 +RESTRICT,0,0,0,0,0,0,2009 +RESTRICTIONS,0,0,0,0,0,0,2009 +RESTRICTS,0,0,0,0,0,0,2009 +RESTRUCTURING,2009,0,0,0,0,0,0 +RETALIATES,2009,0,0,0,0,0,0 +RETALIATORY,2009,0,0,0,0,0,0 +PRECAUTIONS,0,0,2009,0,0,0,0 +PRECLUDE,2009,0,0,0,0,0,2009 +PRECONDITION,0,0,0,0,0,0,2009 +PREDECEASED,0,0,0,2009,0,0,0 +PREDICTABILITY,0,0,2009,0,0,0,0 +PREDICTIONS,0,0,2009,0,0,0,0 +PREDICTS,0,0,2009,0,0,0,0 +PREJUDICE,2009,0,0,2009,0,0,0 +PREJUDICING,2009,0,0,2009,0,0,0 +PREMATURELY,2009,0,0,0,0,0,0 +PRESET,0,0,0,0,0,0,2009 +PRESUMABLY,0,0,2009,0,0,0,0 +PRESUMING,0,0,2009,0,0,0,0 +PRETRIAL,2009,0,0,2009,0,0,0 +PREVENTION,2009,0,0,0,0,0,0 +PROACTIVE,0,2009,0,0,0,0,0 +PROBABILITY,0,0,2009,0,0,0,0 +PROBATED,0,0,0,2009,0,0,0 +PROBATIONAL,0,0,0,2009,0,0,0 +PROBATIONS,0,0,0,2009,0,0,0 +PROBLEMS,2009,0,0,0,0,0,0 +PROFITABILITY,0,2009,0,0,0,0,0 +PROGRESSED,0,2009,0,0,0,0,0 +PROHIBITED,0,0,0,0,0,0,2009 +PROHIBITIVE,0,0,0,0,0,0,2009 +PROLONG,2009,0,0,0,0,0,0 +PROLONGING,2009,0,0,0,0,0,0 +PROMULGATES,0,0,0,2009,0,0,0 +PROMULGATOR,0,0,0,2009,0,0,0 +PRORATION,0,0,0,2009,0,0,0 +PROSECUTING,2009,0,0,2009,0,0,0 +PROSECUTORIAL,0,0,0,2011,0,0,0 +PROSPERITY,0,2009,0,0,0,0,0 +PROTESTED,2009,0,0,0,0,0,0 +PROTESTOR,2009,0,0,0,0,0,0 +PROTRACTION,2009,0,0,0,0,0,0 +PROVOKE,2009,0,0,0,0,0,0 +PUNISHABLE,0,0,0,2009,0,0,0 +PUNISHMENT,2009,0,0,0,0,0,0 +PURPORTED,2009,0,0,0,0,0,0 +QUESTION,2009,0,0,0,0,0,0 +QUESTIONING,2009,0,0,0,0,0,0 +QUITCLAIMS,0,0,0,2009,0,0,0 +RANDOM,0,0,2009,0,0,0,0 +RANDOMIZING,0,0,2009,0,0,0,0 +RATABLE,0,0,0,2009,0,0,0 +RATIONALIZE,2009,0,0,0,0,0,0 +ABANDONED,2009,0,0,0,0,0,0 +ABANDONS,2009,0,0,0,0,0,0 +ABDICATION,2009,0,0,0,0,0,0 +ABERRATIONAL,2009,0,0,0,0,0,0 +ABEYANCES,0,0,2009,0,0,0,0 +ABNORMAL,2009,0,0,0,0,0,0 +ABOLISH,2009,0,0,0,0,0,0 +ABOVEMENTIONED,0,0,0,2011,0,0,0 +ABROGATING,2009,0,0,2009,0,0,0 +ABRUPTLY,2009,0,0,0,0,0,0 +ABSENTEEISM,2009,0,0,0,0,0,0 +ABSOLVING,0,0,0,2009,0,0,0 +ABUSED,2009,0,0,0,0,0,0 +ABUSIVELY,2009,0,0,0,0,0,0 +ACCIDENT,2009,0,0,0,0,0,0 +ACCLAIMED,0,2009,0,0,0,0,0 +ACCOMPLISHING,0,2009,0,0,0,0,0 +ACCUSATIONS,2009,0,0,0,0,0,0 +ACCUSING,2009,0,0,0,0,0,0 +ACHIEVEMENTS,0,2009,0,0,0,0,0 +ACQUIESCED,2009,0,0,0,0,0,0 +ACQUIRORS,0,0,0,2011,0,0,0 +ACQUITTALS,2009,0,0,2009,0,0,0 +ACQUITTING,2009,0,0,2009,0,0,0 +ADJOURNED,0,0,0,2009,0,0,0 +ADJOURNS,0,0,0,2009,0,0,0 +ADJUDGING,0,0,0,2009,0,0,0 +ADJUDICATING,0,0,0,2009,0,0,0 +ADJUDICATOR,0,0,0,2009,0,0,0 +ADMISSIBLE,0,0,0,2009,0,0,0 +ADULTERATE,2009,0,0,0,0,0,0 +ADULTERATIONS,2009,0,0,0,0,0,0 +ADVANCING,0,2009,0,0,0,0,0 +ADVANTAGEOUSLY,0,2009,0,0,0,0,0 +ADVERSARY,2009,0,0,0,0,0,0 +ADVERSITY,2009,0,0,0,0,0,0 +AFFREIGHTMENT,0,0,0,2012,0,0,0 +AFORESTATED,0,0,0,2011,0,0,0 +AGGRAVATE,2009,0,0,0,0,0,0 +AGGRAVATION,2009,0,0,0,0,0,0 +ALERTING,2009,0,0,0,0,0,0 +ALIENATING,2009,0,0,0,0,0,0 +ALLEGATIONS,2009,0,0,2009,0,0,0 +ALLEGES,2009,0,0,2009,0,0,0 +ALMOST,0,0,2009,0,0,2009,0 +AMBIGUITIES,0,0,2009,0,0,0,0 +AMENDABLE,0,0,0,2009,0,0,0 +AMENDMENT,0,0,0,2009,0,0,0 +ANNOYANCE,2009,0,0,0,0,0,0 +ANNOYS,2009,0,0,0,0,0,0 +ANNULMENT,2009,0,0,0,0,0,0 +ANOMALOUS,2009,0,2009,0,0,0,0 +ANTECEDENTS,0,0,0,2009,0,0,0 +ANTICIPATING,0,0,2009,0,0,0,0 +ANTICORRUPTION,0,0,0,2014,0,0,0 +APPARENTLY,0,0,2009,0,0,2009,0 +APPEALING,0,0,0,2009,0,0,0 +APPEARING,0,0,2009,0,0,2009,0 +APPELLATE,0,0,0,2009,0,0,0 +APPROXIMATED,0,0,2009,0,0,0,0 +APPROXIMATION,0,0,2009,0,0,0,0 +APPURTENANT,0,0,0,2009,0,0,0 +ARBITRARINESS,0,0,2009,0,0,0,0 +ARBITRATES,0,0,0,2009,0,0,0 +ARBITRATIONS,0,0,0,2009,0,0,0 +ARGUE,2009,0,0,0,0,0,0 +ARGUMENTATIVE,2009,0,0,0,0,0,0 +ARREARS,2009,0,0,0,0,0,0 +ARTIFICIALLY,2009,0,0,0,0,0,0 +RETRIBUTION,2009,0,0,0,0,0,0 +RETROCESSIONAIRES,0,0,0,2011,0,0,0 +BOLSTERS,0,2009,0,0,0,0,0 +BOOMING,0,2009,0,0,0,0,0 +BOTTLENECKS,2009,0,0,0,0,0,0 +BOYCOTTED,2009,0,0,0,0,0,0 +BREACHED,2009,0,0,2009,0,0,0 +BREAKAGE,2009,0,0,0,0,0,0 +BREAKING,-2020,0,0,0,0,0,0 +BRIBE,2009,0,0,0,0,0,0 +BRIBES,2009,0,0,0,0,0,0 +BROKEN,-2020,0,0,0,0,0,0 +BURDENS,2009,0,0,0,0,0,0 +CALAMITOUS,2009,0,0,0,0,0,0 +CANCELING,2009,0,0,0,0,0,0 +CANCELLING,2009,0,0,0,0,0,0 +CARELESSNESS,2009,0,0,0,0,0,0 +CATASTROPHIC,2009,0,0,0,0,0,0 +CAUTIONED,2009,0,0,0,0,0,0 +CAUTIOUSLY,0,0,2009,0,0,0,0 +CEASES,2009,0,0,0,0,0,0 +CENSURE,2009,0,0,0,0,0,0 +CERTIORARI,0,0,0,2011,0,0,0 +CHALLENGES,2009,0,0,0,0,0,0 +CHATTEL,0,0,0,2009,0,0,0 +CIRCUMVENTED,2009,0,0,0,0,0,0 +CIRCUMVENTS,2009,0,0,0,0,0,0 +SHALL,0,0,0,2009,0,0,0 +SHORTAGES,2009,0,0,0,0,0,0 +SHRINKAGES,2009,0,0,0,0,0,0 +SHUTS,2009,0,0,0,0,0,0 +SLANDEROUS,2009,0,0,0,0,0,0 +REPUDIATING,2009,0,0,0,0,0,0 +REQUESTOR,0,0,0,2011,0,0,0 +REQUIREMENTS,0,0,0,0,0,0,2009 +RESCIND,0,0,0,2009,0,0,0 +RESCISSION,0,0,0,2009,0,0,0 +RESIGNATIONS,2009,0,0,0,0,0,0 +RESOLVE,0,2009,0,0,0,0,0 +RESTATEMENTS,2009,0,0,0,0,0,0 +NOTARIES,0,0,0,2009,0,0,0 +NOTARIZED,0,0,0,2009,0,0,0 +NOVO,0,0,0,2011,0,0,0 +NULLIFICATIONS,2009,0,0,2009,0,0,0 +NULLIFYING,2009,0,0,2009,0,0,0 +OBJECTING,2009,0,0,0,0,0,0 +OBJECTIONS,2009,0,0,0,0,0,0 +OBLIGATING,0,0,0,0,0,0,2009 +OBLIGE,0,0,0,0,0,0,2009 +OBLIGES,0,0,0,0,0,0,2009 +OBSCENITY,2009,0,0,0,0,0,0 +OBSTACLES,2009,0,0,0,0,0,0 +OBSTRUCTION,2009,0,0,0,0,0,0 +OFFENCES,2009,0,0,0,0,0,0 +OFFENDERS,2009,0,0,0,0,0,0 +OFFEREE,0,0,0,2009,0,0,0 +OMISSION,2009,0,0,0,0,0,0 +OMITTED,2009,0,0,0,0,0,0 +OPPORTUNISTICALLY,2009,0,0,0,0,0,0 +OPPOSED,2009,0,0,0,0,0,0 +OPPOSITIONS,2009,0,0,0,0,0,0 +ORDINARILY,0,0,2009,0,0,0,0 +OUTMODED,2009,0,0,0,0,0,0 +OUTPERFORMS,0,2009,0,0,0,0,0 +OVERBUILDING,2009,0,0,0,0,0,0 +OVERBURDENED,2009,0,0,0,0,0,0 +OVERCHARGE,2009,0,0,0,0,0,0 +OVERCOME,2009,0,0,0,0,0,0 +OVERESTIMATE,2009,0,0,0,0,0,0 +OVERESTIMATION,2009,0,0,0,0,0,0 +OVERLOADING,2009,0,0,0,0,0,0 +OVERLOOKING,2009,0,0,0,0,0,0 +OVERPAYMENTS,2009,0,0,0,0,0,0 +OVERPRODUCTION,2009,0,0,0,0,0,0 +OVERRULING,0,0,0,2009,0,0,0 +OVERSHADOW,2009,0,0,0,0,0,0 +OVERSTATE,2009,0,0,0,0,0,0 +OVERSTATES,2009,0,0,0,0,0,0 +OVERSUPPLY,2009,0,0,0,0,0,0 +OVERTURNED,2009,0,0,0,0,0,0 +OVERVALUED,2009,0,0,0,0,0,0 +PARA,0,0,0,-2020,0,0,0 +NECESSITATED,0,0,0,0,0,0,2009 +NEGATIVELY,2009,0,0,0,0,0,0 +NEGLECTFUL,2009,0,0,0,0,0,0 +NEGLIGENCES,2009,0,0,0,0,0,0 +NOLO,0,0,0,2011,0,0,0 +BEAUTIFUL,0,2009,0,0,0,0,0 +BELIEVES,0,0,2009,0,0,0,0 +IDEAL,0,2009,0,0,0,0,0 +IGNORE,2009,0,0,0,0,0,0 +ILL,2009,0,0,0,0,0,0 +ILLEGALLY,2009,0,0,0,0,0,0 +ILLIQUID,2009,0,0,0,0,0,0 +IMMATERIALITY,0,0,0,2009,0,0,0 +IMPAIRED,2009,0,0,0,0,0,2011 +IMPAIRS,2009,0,0,0,0,0,2011 +IMPEDED,2009,0,0,0,0,0,0 +IMPEDING,2009,0,0,0,0,0,0 +IMPERFECTIONS,2009,0,0,0,0,0,0 +IMPLICATE,2009,0,0,0,0,0,0 +IMPOSE,0,0,0,0,0,0,2009 +IMPOSITION,0,0,0,0,0,0,2009 +IMPOUND,2009,0,0,0,0,0,0 +IMPRACTICABLE,2009,0,0,0,0,0,0 +IMPRECISE,0,0,2009,0,0,0,0 +IMPRESSED,0,2009,0,0,0,0,0 +IMPRESSIVELY,0,2009,0,0,0,0,0 +IMPROPER,2009,0,0,0,0,0,0 +IMPROVE,0,2009,0,0,0,0,0 +IMPROVES,0,2009,0,0,0,0,0 +INABILITY,2009,0,0,0,0,0,0 +INACCURATE,2009,0,0,0,0,0,0 +INACTIVATE,2009,0,0,0,0,0,0 +INACTIVATION,2009,0,0,0,0,0,0 +INADEQUACY,2009,0,0,0,0,0,0 +INADVERTENTLY,2009,0,0,0,0,0,0 +INAPPROPRIATELY,2009,0,0,0,0,0,0 +INCAPACITATED,2009,0,0,0,0,0,0 +INCARCERATES,2009,0,0,2009,0,0,0 +INCHOATE,0,0,0,2009,0,0,0 +INCIDENTS,2009,0,0,0,0,0,0 +INCOMPETENCE,2009,0,0,0,0,0,0 +INCOMPETENTS,2009,0,0,0,0,0,0 +INCONCLUSIVE,2009,0,0,0,0,0,0 +INCONSISTENTLY,2009,0,0,0,0,0,0 +INCONVENIENCES,2009,0,0,0,0,0,0 +INCORRECTNESS,2009,0,0,0,0,0,0 +INDECENCY,2009,0,0,0,0,0,0 +INDEFINITE,0,0,2009,0,0,0,0 +INDEMNIFICATION,0,0,0,2009,0,0,0 +INDEMNIFY,0,0,0,2009,0,0,0 +INDEMNITIES,0,0,0,2009,0,0,0 +INDETERMINABLE,0,0,2009,0,0,0,0 +INDICTED,2009,0,0,2009,0,0,0 +INDORSEES,0,0,0,2011,0,0,0 +INEFFICIENCIES,2009,0,0,0,0,0,0 +INELIGIBILITY,2009,0,0,0,0,0,0 +INEQUITIES,2009,0,0,0,0,0,0 +INEXACTNESS,0,0,2009,0,0,0,0 +INFLICTED,2009,0,0,0,0,0,0 +INFRACTION,2009,0,0,2009,0,0,0 +INFRINGEMENT,2009,0,0,0,0,0,0 +INFRINGING,2009,0,0,0,0,0,0 +INHIBITING,0,0,0,0,0,0,2011 +INJUNCTIONS,2009,0,0,2009,0,0,0 +INJURES,2009,0,0,0,0,0,0 +INJURY,2009,0,0,0,0,0,0 +INNOVATING,0,2009,0,0,0,0,0 +INNOVATIVENESS,0,2011,0,0,0,0,0 +INORDINATELY,2009,0,0,0,0,0,0 +INSIGHTFUL,0,2009,0,0,0,0,0 +INSISTING,0,0,0,0,0,0,2009 +INSOLVENCY,2009,0,0,0,0,0,0 +INSTABILITIES,0,0,2009,0,0,0,0 +INSUFFICIENT,2009,0,0,0,0,0,0 +INTANGIBLE,0,0,2009,0,0,0,0 +INTERFERE,2009,0,0,0,0,0,0 +INTERFERES,2009,0,0,0,0,0,0 +INTERMITTENTLY,2009,0,0,0,0,0,0 +INTERPOSES,0,0,0,2009,0,0,0 +INTERROGATE,0,0,0,2009,0,0,0 +INTERROGATION,0,0,0,2009,0,0,0 +INTERROGATORS,0,0,0,2009,0,0,0 +INTERRUPTING,2009,0,0,0,0,0,0 +INTESTACY,0,0,0,2009,0,0,0 +STABILIZATION,0,2009,0,0,0,0,0 +STABILIZES,0,2009,0,0,0,0,0 +STAGNANT,2009,0,0,0,0,0,0 +STAGNATING,2009,0,0,0,0,0,0 +STATUTE,0,0,0,2009,0,0,0 +STIPULATE,0,0,0,0,0,0,2009 +STIPULATION,0,0,0,0,0,0,2009 +STOPPAGES,2009,0,0,0,0,0,0 +STRAIN,2009,0,0,0,0,0,0 +STRENGTH,0,2009,0,0,0,0,0 +STRENGTHENS,0,2009,0,0,0,0,0 +STRESSES,2009,0,0,0,0,0,0 +STRICTER,0,0,0,0,0,0,2009 +STRONG,0,2009,0,0,0,0,0 +SUBCLAUSE,0,0,0,2009,0,0,0 +SUBJECTING,2009,0,0,0,0,0,0 +SUBLESSORS,0,0,0,2011,0,0,0 +SUBPARAGRAPHS,0,0,0,2009,0,0,0 +SUBROGATED,0,0,0,2009,0,0,0 +SUBTRUSTS,0,0,0,2011,0,0,0 +SUCCEEDS,0,2009,0,0,0,0,0 +SUCCESSFULLY,0,2009,0,0,0,0,0 +SUED,2009,0,0,2009,0,0,0 +SUFFERING,2009,0,0,0,0,0,0 +SUGGESTING,0,0,2009,0,0,0,0 +SUMMONING,2009,0,0,2009,0,0,0 +SUPERSEDE,0,0,0,2009,0,0,0 +SUPERSEDING,0,0,0,2009,0,0,0 +SURPASSED,0,2009,0,0,0,0,0 +SUSCEPTIBLE,2009,0,0,0,0,0,0 +SUSPEND,2009,0,0,0,0,0,0 +SUSPENSION,2009,0,0,0,0,0,0 +SUSPICIOUS,2009,0,0,0,0,0,0 +REVOCATION,2009,0,0,2009,0,0,0 +REVOKES,2009,0,0,0,0,0,0 +REVOLUTIONIZES,0,2009,0,0,0,0,0 +REWARDING,0,2009,0,0,0,0,0 +RIDICULES,2009,0,0,0,0,0,0 +RISKIER,2009,0,2009,0,0,0,0 +RISKS,0,0,2009,0,0,0,0 +RULINGS,0,0,0,2009,0,0,0 +SACRIFICED,2009,0,0,0,0,0,0 +SATISFACTION,0,2009,0,0,0,0,0 +SATISFIES,0,2009,0,0,0,0,0 +SCANDALS,2009,0,0,0,0,0,0 +SCRUTINIZING,2009,0,0,0,0,0,0 +SEIZE,2009,0,0,0,0,0,0 +SELDOM,0,0,2009,0,0,2009,0 +SEQUESTRATOR,0,0,0,2009,0,0,0 +SETBACK,2009,0,0,0,0,0,0 +SEVER,2009,0,0,0,0,0,0 +SEVERANCE,0,0,0,2009,0,0,0 +SEVERELY,2009,0,0,0,0,0,0 +ENTRENCH,0,0,0,0,0,0,2009 +ERODES,2009,0,0,0,0,0,0 +ERRATICALLY,2009,0,0,0,0,0,0 +ERRONEOUSLY,2009,0,0,0,0,0,0 +ESCALATE,2009,0,0,0,0,0,0 +ESCHEAT,0,0,0,2011,0,0,0 +ESCROWED,0,0,0,0,0,0,2009 +EVADE,2009,0,0,0,0,0,0 +EVASION,2009,0,0,0,0,0,0 +EVICTED,2009,0,0,0,0,0,0 +EVICTS,2009,0,0,0,0,0,0 +EXACERBATED,2009,0,0,0,0,0,0 +EXACERBATIONS,2009,0,0,0,0,0,0 +EXAGGERATING,2009,0,0,0,0,0,0 +EXCEEDENCES,0,0,0,2011,0,0,0 +EXCELS,0,2009,0,0,0,0,0 +EXCESSIVELY,2009,0,0,0,0,0,0 +EXCITING,0,2009,0,0,0,0,0 +EXCLUSIVES,0,2009,0,0,0,0,0 +EXCULPATES,2009,0,0,2009,0,0,0 +EXCULPATORY,2009,0,0,2009,0,0,0 +EXECUTRICES,0,0,0,2009,0,0,0 +EXONERATE,2009,0,0,0,0,0,0 +EXONERATION,2009,0,0,0,0,0,0 +EXPLOITATIONS,2009,0,0,0,0,0,0 +EXPLOITS,2009,0,0,0,0,0,0 +EXPOSING,2009,0,0,0,0,0,0 +EXPROPRIATED,2009,0,0,0,0,0,0 +EXPROPRIATIONS,2009,0,0,0,0,0,0 +EXTRACONTRACTUAL,0,0,0,2011,0,0,0 +SOLVENCIES,2009,0,0,0,0,0,0 +SOMETIME,0,0,2009,0,0,0,0 +SPAM,2014,0,0,0,0,0,0 +TRAUMATIC,2009,0,0,0,0,0,0 +LOSES,2009,0,0,0,0,0,0 +LOST,2009,0,0,0,0,0,0 +LYING,2009,0,0,0,0,0,0 +MALFUNCTIONED,2009,0,0,0,0,0,0 +MALICIOUS,2009,0,0,0,0,0,0 +MANDATE,0,0,0,0,0,0,2009 +MANDATORY,0,0,0,0,0,0,2009 +MANIPULATES,2009,0,0,0,0,0,0 +MANIPULATIVE,2009,0,0,0,0,0,0 +MAYBE,0,0,2009,0,0,2009,0 +MEDIATING,0,0,0,2009,0,0,0 +MEDIATORS,0,0,0,2009,0,0,0 +MISAPPLICATIONS,2009,0,0,0,0,0,0 +MISAPPLYING,2009,0,0,0,0,0,0 +MISAPPROPRIATING,2009,0,0,0,0,0,0 +MISCALCULATE,2009,0,0,0,0,0,0 +MISCALCULATION,2009,0,0,0,0,0,0 +MISCLASSIFICATION,2011,0,0,0,0,0,0 +MISCOMMUNICATION,2014,0,0,0,0,0,0 +MISDEMEANORS,2009,0,0,0,0,0,0 +MISHANDLED,2009,0,0,0,0,0,0 +MISINFORMATION,2009,0,0,0,0,0,0 +MISINTERPRET,2009,0,0,0,0,0,0 +MISINTERPRETING,2009,0,0,0,0,0,0 +MISJUDGES,2009,0,0,0,0,0,0 +MISLABEL,2009,0,0,0,0,0,0 +MISLABELS,2009,0,0,0,0,0,0 +MISLEADS,2009,0,0,0,0,0,0 +MISMANAGEMENT,2009,0,0,0,0,0,0 +MISMATCHED,2009,0,0,0,0,0,0 +MISPRICE,2014,0,0,0,0,0,0 +MISREPRESENTATION,2009,0,0,0,0,0,0 +MISREPRESENTS,2009,0,0,0,0,0,0 +WORRYING,2009,0,0,0,0,0,0 +WORSENING,2009,0,0,0,0,0,0 +WORTHY,0,2009,0,0,0,0,0 +TOLERATED,2009,0,0,0,0,0,0 +TORT,0,0,0,2009,0,0,0 +TORTUOUS,2009,0,0,0,0,0,0 +FIDE,0,0,0,2009,0,0,0 +FIRING,2009,0,0,0,0,0,0 +NONBREACHING,0,0,0,2011,0,0,0 +NONCOMPLIANCE,2009,0,0,0,0,0,0 +NONCONFORMING,2009,0,0,0,0,0,0 +NONCONTRACT,0,0,0,2012,0,0,0 +NONFEASANCE,0,0,0,2011,0,0,0 +NONFORFEITURE,0,0,0,2011,0,0,0 +DISCLAIMED,2009,0,0,0,0,0,0 +DISCLAIMS,2009,0,0,0,0,0,0 +DISCLOSING,2009,0,0,0,0,0,0 +DISCONTINUATIONS,2009,0,0,0,0,0,0 +DISCONTINUING,2009,0,0,0,0,0,0 +DISCOURAGING,2009,0,0,0,0,0,0 +DISCREDITS,2009,0,0,0,0,0,0 +SPECTACULARLY,0,2009,0,0,0,0,0 +SPECULATING,0,0,2009,0,0,0,0 +SPECULATIVELY,0,0,2009,0,0,0,0 +TRAGEDY,2009,0,0,0,0,0,0 +TRANSFERORS,0,0,0,2012,0,0,0 +LEGISLATING,0,0,0,2009,0,0,0 +LEGISLATIVELY,0,0,0,2009,0,0,0 +LEGISLATURES,0,0,0,2009,0,0,0 +LIBELS,0,0,0,2009,0,0,0 +CONCEALED,2009,0,0,0,0,0,0 +CONCEDES,2009,0,0,0,0,0,0 +CONCERN,2009,0,0,0,0,0,0 +CONCILIATION,2009,0,0,0,0,0,0 +CONDEMN,2009,0,0,0,0,0,0 +CONDEMNING,2009,0,0,0,0,0,0 +CONDITIONALLY,0,0,2009,0,0,0,0 +CONFESS,2009,0,0,0,0,0,0 +CONFESSION,2009,0,0,0,0,0,0 +CONFINEMENT,2009,0,0,0,0,0,2009 +CONFISCATE,2009,0,0,0,0,0,0 +CONFISCATION,2009,0,0,0,0,0,0 +CONFLICTED,2009,0,0,0,0,0,0 +CONFRONTATION,2009,0,0,0,0,0,0 +CONFRONTING,2009,0,0,0,0,0,0 +CONFUSES,2009,0,2009,0,0,0,0 +CONSENT,0,0,0,2009,0,0,0 +CONSERVATORSHIPS,0,0,0,2011,0,0,0 +CONSPIRATORIAL,2009,0,0,0,0,0,0 +CONSPIRES,2009,0,0,0,0,0,0 +CONSTITUTIONALITY,0,0,0,2009,0,0,0 +CONSTRAIN,0,0,0,0,0,0,2009 +CONSTRAINT,0,0,0,0,0,0,2009 +CONSTRUE,0,0,0,2012,0,0,0 +CONTEMPT,2009,0,0,0,0,0,0 +CONTENDS,2009,0,0,0,0,0,0 +CONTENTIOUSLY,2009,0,0,0,0,0,0 +CONTESTING,2009,0,0,0,0,0,0 +CONTINGENTLY,0,0,2009,0,0,0,0 +CONTRACTHOLDER,0,0,0,2009,0,0,0 +CONTRACTING,0,0,0,2009,0,0,0 +CONTRACTUAL,0,0,0,2009,0,0,0 +CONTRADICTING,2009,0,0,0,0,0,0 +CONTRADICTS,2009,0,0,0,0,0,0 +CONTRAVENES,0,0,0,2009,0,0,0 +CONTROVERSIAL,2009,0,0,0,0,0,0 +CONTROVERTED,0,0,0,2009,0,0,0 +CONVEYANCES,0,0,0,2009,0,0,0 +CONVICTION,2009,0,0,2009,0,0,0 +CORRECTION,2009,0,0,0,0,0,0 +CORRUPTED,2009,0,0,0,0,0,0 +CORRUPTLY,2009,0,0,0,0,0,0 +COULD,0,0,2009,0,0,2009,0 +COUNSELS,0,0,0,2009,0,0,0 +COUNTERCLAIMS,2009,0,0,0,0,0,0 +COUNTERFEITERS,2009,0,0,0,0,0,0 +COUNTERMEASURES,2009,0,0,0,0,0,0 +COUNTERSUITS,0,0,0,2014,0,0,0 +COURTS,0,0,0,2009,0,0,0 +COVENANTS,0,0,0,0,0,0,2011 +CREATIVITY,0,2009,0,0,0,0,0 +CRIMINALITY,0,0,0,2009,0,0,0 +CRIMINALS,2009,0,0,2009,0,0,0 +CRITICALLY,2009,0,0,0,0,0,0 +CRITICIZED,2009,0,0,0,0,0,0 +CROSSCLAIMS,0,0,0,2011,0,0,0 +CRUCIALLY,2009,0,0,0,0,0,0 +CUMBERSOME,2009,0,0,0,0,0,0 +CURTAILMENT,2009,0,0,0,0,0,0 +CUTBACK,2009,0,0,0,0,0,0 +CYBERBULLYING,2014,0,0,0,0,0,0 +CYBERCRIMINALS,2014,0,0,0,0,0,0 +TAINTED,2009,0,0,0,0,0,0 +TENANTABILITY,0,0,0,2011,0,0,0 +TENTATIVELY,0,0,2009,0,0,0,0 +TERMINATES,2009,0,0,0,0,0,0 +TERMINUS,0,0,0,-2020,0,0,0 +TESTIMONY,0,0,0,2009,0,0,0 +THEREAFTER,0,0,0,2009,0,0,0 +THEREINAFTER,0,0,0,2011,0,0,0 +THERETO,0,0,0,2009,0,0,0 +THEREUNTO,0,0,0,2009,0,0,0 +THREATEN,2009,0,0,0,0,0,0 +THREATS,2009,0,0,0,0,0,0 +FAIL,2009,0,0,0,0,0,0 +FAILS,2009,0,0,0,0,0,0 +FALSE,2009,0,0,0,0,0,0 +FALSIFIED,2009,0,0,0,0,0,0 +FALSITY,2009,0,0,0,0,0,0 +FATALLY,2009,0,0,0,0,0,0 +FAULTY,2009,0,0,0,0,0,0 +FAVORING,0,2009,0,0,0,0,0 +FEARS,2009,0,0,0,0,0,0 +DAMAGE,2009,0,0,0,0,0,0 +DAMPEN,2009,0,0,0,0,0,0 +DANGEROUSLY,2009,0,0,0,0,0,0 +DEADLOCKING,2009,0,0,0,0,0,0 +DEBARMENT,2009,0,0,0,0,0,0 +DECEDENT,0,0,0,2009,0,0,0 +DECEITFULNESS,2009,0,0,0,0,0,0 +DECEIVING,2009,0,0,0,0,0,0 +DECEPTIVELY,2009,0,0,0,0,0,0 +DECLINES,2009,0,0,0,0,0,0 +DECREEING,0,0,0,2009,0,0,0 +DEFACEMENT,2009,0,0,0,0,0,0 +DEFAMATIONS,2009,0,0,0,0,0,0 +DEFAMES,2009,0,0,0,0,0,0 +DEFAULTING,2009,0,0,0,0,0,0 +DEFEASE,0,0,0,2009,0,0,0 +DEFEASING,0,0,0,2011,0,0,0 +DEFEATS,2009,0,0,0,0,0,0 +DEFECTS,2009,0,0,0,0,0,0 +DEFENDANTS,2009,0,0,2009,0,0,0 +DEFENSIVE,2009,0,0,0,0,0,0 +DEFICIENCY,2009,0,0,0,0,0,0 +DEFINITELY,0,0,0,0,2009,0,0 +DEFRAUDING,2009,0,0,0,0,0,0 +DEGRADATIONS,2009,0,0,0,0,0,0 +DEGRADING,2009,0,0,0,0,0,0 +DELAYS,2009,0,0,0,0,0,0 +DELEGEES,0,0,0,2011,0,0,0 +DELIBERATELY,2009,0,0,0,0,0,0 +DELIGHTFULLY,0,2009,0,0,0,0,0 +DELINQUENCY,2009,0,0,0,0,0,0 +DELIST,2009,0,0,0,0,0,0 +DEMISE,2009,0,0,0,0,0,0 +DEMOLISH,2009,0,0,0,0,0,0 +DEMOLITION,2009,0,0,0,0,0,0 +DEMOTES,2009,0,0,0,0,0,0 +DEMURRED,0,0,0,2009,0,0,0 +DEMURS,0,0,0,2009,0,0,0 +DENIES,2009,0,0,0,0,0,0 +DENIGRATING,2009,0,0,0,0,0,0 +DEPEND,0,0,2009,0,0,2009,2011 +DEPENDANCES,0,0,0,0,0,0,2011 +DEPENDENCIES,0,0,2009,0,0,0,2011 +DEPENDS,0,0,2009,0,0,2009,2011 +DEPLETING,2009,0,0,0,0,0,0 +DEPOSED,0,0,0,2009,0,0,0 +DEPOSITIONAL,0,0,0,2011,0,0,0 +INVALIDATED,2009,0,0,0,0,0,0 +INVALIDITY,2009,0,0,0,0,0,0 +INVENTION,0,2009,0,0,0,0,0 +INVENTOR,0,2009,0,0,0,0,0 +INVESTIGATES,2009,0,0,0,0,0,0 +INVOLUNTARILY,2009,0,0,0,0,0,0 +IRRECOVERABLE,2009,0,0,0,0,0,0 +IRREGULARITY,2009,0,0,0,0,0,0 +IRREVERSIBLE,2009,0,0,0,0,0,0 +JEOPARDIZE,2009,0,0,0,0,0,0 +JUDICIALLY,0,0,0,2009,0,0,0 +JURIS,0,0,0,2011,0,0,0 +JURISDICTIONS,0,0,0,2009,0,0,0 +JUROR,0,0,0,2009,0,0,0 +JUSTICE,0,0,0,2009,0,0,0 +KICKBACKS,2009,0,0,0,0,0,0 +DIMINISHED,2009,0,0,0,0,0,0 +DIRECTIVE,0,0,0,0,0,0,2009 +DISADVANTAGEOUS,2009,0,0,0,0,0,0 +DISAFFIRMANCE,0,0,0,2009,0,0,0 +DISAGREEABLE,2009,0,0,0,0,0,0 +DISAGREEMENTS,2009,0,0,0,0,0,0 +DISALLOWANCES,2009,0,0,0,0,0,0 +DISAPPEAR,2009,0,0,0,0,0,0 +DISAPPEARING,2009,0,0,0,0,0,0 +DISAPPOINTING,2009,0,0,0,0,0,0 +DISAPPOINTS,2009,0,0,0,0,0,0 +DISAPPROVED,2009,0,0,0,0,0,0 +DISASSOCIATING,2009,0,0,0,0,0,0 +DISASTERS,2009,0,0,0,0,0,0 +DISAVOWAL,2009,0,0,0,0,0,0 +GRATUITOUSLY,2009,0,0,0,0,0,0 +GREATLY,0,2009,0,0,0,0,0 +GROSSLY,2009,0,0,0,0,0,0 +HALTED,2009,0,0,0,0,0,0 +HAMPERS,2009,0,0,0,0,0,0 +HAPPY,0,2009,0,0,0,0,0 +HARASSMENT,2009,0,0,0,0,0,0 +HARMED,2009,0,0,0,0,0,0 +HARMS,2009,0,0,0,0,0,0 +HARSHLY,2009,0,0,0,0,0,0 +HAZARDS,2009,0,0,0,0,0,0 +NONINFRINGING,0,0,0,2011,0,0,0 +NONPAYMENT,2009,0,0,0,0,0,0 +NONPERFORMING,2009,0,0,0,0,0,0 +DISFAVORS,2009,0,0,0,0,0,0 +PREAMENDMENT,0,0,0,2011,0,0,0 +COMPLICATED,2009,0,0,0,0,0,0 +COMPLICATIONS,2009,0,0,0,0,0,0 +COMPLIMENTING,0,2009,0,0,0,0,0 +MISSTATEMENTS,2009,0,0,0,0,0,0 +MISSTEPS,2009,0,0,0,0,0,0 +MISTAKES,2009,0,0,0,0,0,0 +MISUNDERSTAND,2009,0,0,0,0,0,0 +MISUSE,2009,0,0,0,0,0,0 +MONOPOLISTIC,2009,0,0,0,0,0,0 +MONOPOLIZED,2009,0,0,0,0,0,0 +MORATORIA,2009,0,0,0,0,0,0 +MOTHBALLED,2009,0,0,0,0,0,0 +MUTANDIS,0,0,0,2011,0,0,0 +SLIPPAGES,2009,0,0,0,0,0,0 +SLOWED,2009,0,0,0,0,0,0 +SLOWLY,2009,0,0,0,0,0,0 +SLUGGISHNESS,2009,0,0,0,0,0,0 +SMOOTHS,0,2009,0,0,0,0,0 +DEPRESSES,2009,0,0,0,0,0,0 +DEPRIVED,2009,0,0,0,0,0,0 +DERELICTION,2009,0,0,0,0,0,0 +DEROGATING,0,0,0,2009,0,0,0 +DESIGNATOR,0,0,0,2011,0,0,0 +DESPITE,0,2009,0,0,0,0,0 +DESTABILIZING,2009,0,2009,0,0,0,0 +DESTROYING,2009,0,0,0,0,0,0 +DETAIN,2009,0,0,0,0,0,0 +DETENTIONS,2009,0,0,0,0,0,0 +DETERIORATES,2009,0,0,0,0,0,0 +DETERRED,2009,0,0,0,0,0,0 +DETERRENTS,2009,0,0,0,0,0,0 +DETRACTED,2009,0,0,0,0,0,0 +DETRIMENTALLY,2009,0,0,0,0,0,0 +DEVALUES,2009,0,0,0,0,0,0 +DEVASTATING,2009,0,0,0,0,0,0 +DEVIATES,2009,0,2009,0,0,0,0 +DEVISEES,0,0,0,2011,0,0,0 +DEVOLVING,2009,0,0,0,0,0,0 +DICTATING,0,0,0,0,0,0,2009 +DIFFERS,0,0,2009,0,0,0,0 +DIFFICULTY,2009,0,0,0,0,0,0 +ASCENDANTS,0,0,0,2009,0,0,0 +ASSAULTS,2009,0,0,0,0,0,0 +ASSIGNATIONS,0,0,0,2009,0,0,0 +ASSUMES,0,0,2009,0,0,0,0 +ASSURE,0,2009,0,0,0,0,0 +ATTAIN,0,2009,0,0,0,0,0 +ATTAINMENTS,0,2009,0,0,0,0,0 +ATTESTATIONS,0,0,0,2009,0,0,0 +ATTORNEY,0,0,0,2009,0,0,0 +ATTRACTIVE,0,2009,0,0,0,0,0 +BACKDATING,2009,0,0,0,0,0,0 +BAILEE,0,0,0,2009,0,0,0 +BAILMENT,0,0,0,2011,0,0,0 +BANKRUPT,2009,0,0,0,0,0,0 +BANKRUPTING,2009,0,0,0,0,0,0 +CLAIMANTS,0,0,0,2009,0,0,0 +CLARIFICATION,0,0,2009,0,0,0,0 +CLEARLY,0,0,0,0,2009,0,0 +CLOSING,-2020,0,0,0,0,0,0 +CODEFENDANT,0,0,0,2011,0,0,0 +CODIFICATION,0,0,0,2009,0,0,0 +CODIFY,0,0,0,2009,0,0,0 +COERCES,2009,0,0,0,0,0,0 +COLLABORATED,0,2009,0,0,0,0,0 +COLLABORATIONS,0,2009,0,0,0,0,0 +COLLAPSE,2009,0,0,0,0,0,0 +COLLISION,2009,0,0,0,0,0,0 +COLLUDES,2009,0,0,0,0,0,0 +COLLUSIVE,2009,0,0,0,0,0,0 +POSITIVELY,0,2009,0,0,0,0,0 +POSSIBLE,0,0,2009,0,0,2009,0 +POSTCONTRACT,0,0,0,2011,0,0,0 +POSTPONEMENT,2009,0,0,0,0,0,0 +WRITEDOWN,2009,0,0,0,0,0,0 +WRITS,0,0,0,2009,0,0,0 +WRONGFUL,2009,0,0,0,0,0,0 +HONOR,0,2009,0,0,0,0,0 +HONORS,0,2009,0,0,0,0,0 +REARGUMENT,0,0,0,2011,0,0,0 +REASSESSING,0,0,2009,0,0,0,0 +REASSIGNED,2009,0,0,0,0,0,0 +REASSIGNS,2009,0,0,0,0,0,0 +REBUT,0,0,0,2009,0,0,0 +REBUTTAL,0,0,0,2009,0,0,0 +RECALCULATE,0,0,2009,0,0,0,0 +RECALCULATION,0,0,2009,0,0,0,0 +RECALLING,2009,0,0,0,0,0,0 +RECESSIONARY,2009,0,0,0,0,0,0 +RECKLESSNESS,2009,0,0,0,0,0,0 +RECONSIDERS,0,0,2009,0,0,0,0 +RECOUPMENTS,0,0,0,2009,0,0,0 +RECTIFICATIONS,0,0,0,2009,0,0,0 +RECUSES,0,0,0,2014,0,0,0 +REDACTING,2009,0,0,2009,0,0,0 +REDEFAULTED,2012,0,0,0,0,0,0 +REDRESSES,2009,0,0,0,0,0,0 +REEXAMINING,0,0,2009,0,0,0,0 +REFILE,0,0,0,2009,0,0,0 +REFRAIN,0,0,0,0,0,0,2009 +REFUSALS,2009,0,0,0,0,0,0 +REFUSING,2009,0,0,0,0,0,0 +REGULATE,0,0,0,2009,0,0,0 +REGULATION,0,0,0,2009,0,0,0 +REGULATORS,0,0,0,2009,0,0,0 +REHEARING,0,0,0,2009,0,0,0 +VARIABLY,0,0,2009,0,0,0,0 +VARIANTS,0,0,2009,0,0,0,0 +VARIES,0,0,2009,0,0,0,0 +VENDEES,0,0,0,2011,0,0,0 +VERSATILITY,0,2009,0,0,0,0,0 +VIBRANT,0,2009,0,0,0,0,0 +VIOLATES,2009,0,0,0,0,0,0 +VIOLATIVE,2009,0,0,2009,0,0,0 +VIOLENT,2009,0,0,0,0,0,0 +VITIATES,2009,0,0,0,0,0,0 +VOIDED,2009,0,0,2009,0,0,0 +VOLATILITY,2009,0,2009,0,0,0,0 +VULNERABLY,2009,0,0,0,0,0,0 +WARNINGS,2009,0,0,0,0,0,0 +WASTED,2009,0,0,0,0,0,0 +WEAKEN,2009,0,0,0,0,0,0 +WEAKER,2009,0,0,0,0,0,0 +WEAKNESSES,2009,0,0,0,0,0,0 +DISHONORABLE,2009,0,0,0,0,0,0 +DISHONORS,2009,0,0,0,0,0,0 +DISINTERESTEDLY,2009,0,0,0,0,0,0 +DISLOYALTY,2009,0,0,0,0,0,0 +DISMISSAL,2009,0,0,0,0,0,0 +DISMISSING,2009,0,0,0,0,0,0 +DISPARAGEMENT,2009,0,0,0,0,0,0 +DISPARAGINGLY,2009,0,0,0,0,0,0 +DISPLACED,2009,0,0,0,0,0,0 +DISPLACING,2009,0,0,0,0,0,0 +DISPOSSESSED,2009,0,0,0,0,0,0 +DISPOSSESSORY,0,0,0,2011,0,0,0 +DISPROPORTIONATELY,2009,0,0,0,0,0,0 +DISPUTING,2009,0,0,0,0,0,0 +DISQUALIFIES,2009,0,0,0,0,0,0 +DISREGARDED,2009,0,0,0,0,0,0 +DISREPUTE,2009,0,0,0,0,0,0 +DISRUPTION,2009,0,0,0,0,0,0 +DISSATISFACTION,2009,0,0,0,0,0,0 +DISSENTER,2009,0,0,0,0,0,0 +DISSIDENT,2009,0,0,0,0,0,0 +DISTINCTION,0,2009,0,0,0,0,0 +DISTINCTIVENESS,0,2009,0,0,0,0,0 +DISTORTION,2009,0,0,0,0,0,0 +DISTRACTED,2009,0,0,0,0,0,0 +DISTRACTS,2009,0,0,0,0,0,0 +DISTRIBUTEE,0,0,0,2009,0,0,0 +DISTURBANCES,2009,0,0,0,0,0,0 +DIVERSION,2009,0,0,0,0,0,0 +DIVERTS,2009,0,0,0,0,0,0 +DIVESTITURE,2009,0,0,0,0,0,0 +DIVESTS,2009,0,0,0,0,0,0 +DIVULGED,2009,0,0,0,0,0,0 +DOCKETED,0,0,0,2009,0,0,0 +DOUBT,2009,0,2009,0,0,0,0 +DOWNGRADE,2009,0,0,0,0,0,0 +DOWNSIZE,2009,0,0,0,0,0,0 +DOWNSIZINGS,2009,0,0,0,0,0,0 +DOWNTURNS,2009,0,0,0,0,0,0 +DRASTIC,2009,0,0,0,0,0,0 +DREAM,0,2009,0,0,0,0,0 +DULY,0,0,0,2009,0,0,0 +DYSFUNCTIONS,2009,0,0,0,0,0,0 +EARMARKS,0,0,0,0,0,0,2009 +EASY,0,2009,0,0,0,0,0 +EFFICIENT,0,2009,0,0,0,0,0 +EJECTMENT,0,0,0,2011,0,0,0 +EMBARGOING,2009,0,0,0,0,0,0 +EMBARRASSING,2009,0,0,0,0,0,0 +EMBEZZLED,2009,0,0,0,0,0,0 +EMBEZZLES,2009,0,0,0,0,0,0 +EMPOWERING,0,2009,0,0,0,0,0 +ENABLES,0,2009,0,0,0,0,0 +ENCOURAGES,0,2009,0,0,0,0,0 +ENCROACHES,2009,0,0,0,0,0,0 +ENCUMBER,2009,0,0,2009,0,0,2009 +ENCUMBRANCE,2009,0,0,2009,0,0,2009 +ENDANGER,2009,0,0,0,0,0,0 +ENDANGERS,2009,0,0,0,0,0,0 +ENFORCEABLY,0,0,0,2011,0,0,0 +ENHANCEMENTS,0,2009,0,0,0,0,0 +ENJOINED,2009,0,0,0,0,0,0 +ENJOYABLE,0,2009,0,0,0,0,0 +ENJOYMENT,0,2009,0,0,0,0,0 +ENTAILING,0,0,0,0,0,0,2009 +PASSU,0,0,0,2009,0,0,0 +PENALIZED,2009,0,0,0,0,0,0 +PENALTY,2009,0,0,0,0,0,0 +PERFECTLY,0,2009,0,0,0,0,0 +PERILS,2009,0,0,0,0,0,0 +PERMISSIONS,0,0,0,0,0,0,2009 +PERMITTING,0,0,0,0,0,0,2009 +PERPETRATING,2009,0,0,2009,0,0,0 +PERSISTENCE,2009,0,0,0,0,0,0 +PERSISTS,2009,0,0,0,0,0,0 +PERVASIVENESS,2009,0,0,0,0,0,0 +PETITIONERS,0,0,0,2009,0,0,0 +PICKET,2009,0,0,0,0,0,0 +HEREBY,0,0,0,2009,0,0,0 +HEREFROM,0,0,0,2009,0,0,0 +HEREINBEFORE,0,0,0,2009,0,0,0 +HERETO,0,0,0,2009,0,0,0 +HEREUPON,0,0,0,2009,0,0,0 +HIGHEST,0,2009,0,0,2009,0,0 +HINDERS,2009,0,0,0,0,0,0 +WHATEVER,0,0,0,2009,0,0,0 +WHEREAS,0,0,0,2009,0,0,0 +WHEREIN,0,0,0,2009,0,0,0 +WHEREUNDER,0,0,0,2011,0,0,0 +WHOMEVER,0,0,0,2009,0,0,0 +WILL,0,0,0,0,2009,0,0 +WIN,0,2009,0,0,0,0,0 +WITNESS,0,0,0,2009,0,0,0 +FLUCTUATED,0,0,2009,0,0,0,0 +FLUCTUATIONS,0,0,2009,0,0,0,0 +FORBEARANCES,0,0,0,2009,0,0,0 +FORBIDDEN,2009,0,0,0,0,0,2009 +FORCED,2009,0,0,0,0,0,0 +FOREBEARS,0,0,0,2009,0,0,0 +FORECLOSING,2009,0,0,0,0,0,0 +FOREGOES,2009,0,0,0,0,0,0 +FORESTALLING,2009,0,0,0,0,0,0 +FORFEITABLE,0,0,0,2009,0,0,0 +FORFEITURE,2009,0,0,0,0,0,0 +FORTHWITH,0,0,0,2009,0,0,0 +FRAUDULENCE,2009,0,0,0,0,0,0 +FRIVOLOUS,2009,0,0,0,0,0,0 +FRUSTRATES,2009,0,0,0,0,0,0 +FRUSTRATIONS,2009,0,0,0,0,0,0 +GAIN,0,2009,0,0,0,0,0 +GOOD,0,2009,0,0,0,0,0 +DISGORGES,2009,0,0,0,0,0,0 +DISGRACEFULLY,2009,0,0,0,0,0,0 +LITIGATED,2009,0,0,2009,0,0,0 +LITIGATIONS,2009,0,0,2009,0,0,0 +LITIGIOUSNESS,0,0,0,2009,0,0,0 +LACKING,2009,0,0,0,0,0,0 +LAGGED,2009,0,0,0,0,0,0 +LAPSED,2009,0,0,0,0,0,0 +LAUNDERING,2009,0,0,0,0,0,0 +LAWFULNESS,0,0,0,2009,0,0,0 +LAWSUIT,0,0,0,2009,0,0,0 +LAYOFF,2009,0,0,0,0,0,0 +LEGAL,0,0,0,2009,0,0,0 +LEGALIZATIONS,0,0,0,2009,0,0,0 +LEGALIZING,0,0,0,2009,0,0,0 +LEGATEES,0,0,0,2009,0,0,0 +FLAWS,2009,0,0,0,0,0,0 +NONRENEWAL,2009,0,0,0,0,0,0 +LIQUIDATING,2009,0,0,0,0,0,0 +LIQUIDATORS,2009,0,0,0,0,0,0 +BENEFICIAL,0,-2020,0,2020,0,0,0 +BENEFIT,0,-2020,0,0,0,0,0 +BENEFITTING,0,2009,0,0,0,0,0 +POORLY,2009,0,0,0,0,0,0 +REINTERPRETATIONS,0,0,2009,0,0,0,0 +REJECT,2009,0,0,0,0,0,0 +REJECTIONS,2009,0,0,0,0,0,0 +RELINQUISHED,2009,0,0,0,0,0,0 +RELINQUISHMENTS,2009,0,0,0,0,0,0 +REMANDED,0,0,0,2009,0,0,0 +REMEDIATED,0,0,0,2009,0,0,0 +REMEDIED,0,0,0,2009,0,0,0 +RENEGOTIATES,2009,0,0,0,0,0,0 +RENOUNCE,2009,0,0,0,0,0,0 +RENOUNCES,2009,0,0,0,0,0,0 +REPLEDGED,0,0,0,2012,0,0,0 +REPOSSESSING,2009,0,0,0,0,0,0 +TROUBLES,2009,0,0,0,0,0,0 +UNACCEPTABLE,2009,0,0,0,0,0,0 +UNANNOUNCED,2009,0,0,0,0,0,0 +UNAPPROVED,2009,0,0,0,0,0,0 +UNAVAILABLE,2009,0,0,0,0,0,2011 +UNCERTAIN,0,0,2009,0,0,2009,0 +UNCLEAR,0,0,2009,0,0,0,0 +UNCOLLECTIBLE,2009,0,0,0,0,0,0 +UNCOMPROMISING,0,0,0,0,2009,0,0 +UNCONSTITUTIONAL,0,0,0,2009,0,0,0 +UNCONTROLLABLE,2009,0,0,0,0,0,0 +UNCOVER,2009,0,0,0,0,0,0 +UNDECIDED,0,0,2009,0,0,0,0 +UNDELIVERED,2009,0,0,0,0,0,0 +UNDERCUTTING,2009,0,0,0,0,0,0 +UNDERESTIMATING,2009,0,0,0,0,0,0 +UNDERMINE,2009,0,0,0,0,0,0 +UNDERPAID,2009,0,0,0,0,0,0 +UNDERPERFORM,2011,0,0,0,0,0,0 +UNDERPERFORMS,2014,0,0,0,0,0,0 +UNDERSTATE,2009,0,0,0,0,0,0 +UNDERSTATES,2009,0,0,0,0,0,0 +UNDESIGNATED,0,0,2009,0,0,0,0 +UNDETECTED,2009,0,0,0,0,0,0 +UNDISCLOSED,2009,0,0,0,0,0,0 +UNDUE,2009,0,0,0,0,0,0 +UNECONOMICALLY,2009,0,0,0,0,0,0 +UNENCUMBERED,0,0,0,2009,0,0,0 +UNEQUIVOCALLY,0,0,0,0,2009,0,0 +UNEXPECTED,2009,0,2009,0,0,0,0 +UNFAMILIAR,0,0,2009,0,0,0,0 +UNFAVORABLY,2009,0,0,0,0,0,0 +UNFITNESS,2009,0,0,0,0,0,0 +UNFORSEEN,2011,0,2011,0,0,0,0 +UNFRIENDLY,2009,0,0,0,0,0,0 +UNHEDGED,0,0,2009,0,0,0,0 +UNINTENDED,2009,0,0,0,0,0,0 +UNJUSTIFIABLE,2009,0,0,0,0,0,0 +UNKNOWING,2009,0,0,0,0,0,0 +UNLAWFUL,2009,0,0,2009,0,0,0 +UNLIQUIDATED,2009,0,0,0,0,0,0 +UNMERITORIOUS,2014,0,0,0,0,0,0 +UNOBSERVABLE,0,0,2009,0,0,0,0 +UNPARALLELED,0,2009,0,0,2009,0,0 +UNPREDICTABILITY,2009,0,2009,0,0,0,0 +UNPRODUCTIVE,2009,0,0,0,0,0,0 +UNPROVEN,0,0,2009,0,0,0,0 +UNREALISTIC,2009,0,0,0,0,0,0 +UNRECEPTIVE,2014,0,0,0,0,0,0 +UNREIMBURSED,2009,0,0,0,0,0,0 +UNREPORTED,2009,0,0,0,0,0,0 +UNSALABLE,2009,0,0,0,0,0,0 +UNSAVORY,2009,0,0,0,0,0,0 +UNSELLABLE,2014,0,0,0,0,0,0 +UNSPECIFIC,0,0,2009,0,0,0,0 +UNSTAYED,0,0,0,2009,0,0,0 +UNSUITABILITY,2009,0,0,0,0,0,0 +UNSURE,2009,0,0,0,0,0,0 +UNSUSTAINABLE,2009,0,0,0,0,0,0 +UNTO,0,0,0,2009,0,0,0 +UNTRUTHFULLY,2009,0,0,0,0,0,0 +UNUSUAL,0,0,2009,0,0,0,0 +UNWELCOME,2009,0,0,0,0,0,0 +UPSET,2009,0,0,0,0,0,0 +URGENT,2009,0,0,0,0,0,0 +USURPED,2009,0,0,2009,0,0,0 +VAGARIES,0,0,2009,0,0,0,0 +VAGUENESSES,0,0,2012,0,0,0,0 +VANDALISM,2009,0,0,0,0,0,0 +PLAINTIFF,2009,0,0,2009,0,0,0 +PLEADED,2009,0,0,0,0,0,0 +PLEAS,2009,0,0,2009,0,0,0 +PLEASURE,0,2009,0,0,0,0,0 +PLEDGEE,0,0,0,2009,0,0,0 +PLEDGOR,0,0,0,2009,0,0,0 +COMPELLED,0,0,0,0,0,0,2009 +COMPLAIN,2009,0,0,0,0,0,0 +COMPLAINING,2009,0,0,0,0,0,0 +COMMITMENTS,0,0,0,0,0,0,2009 +LIMITATIONS,2009,0,0,0,0,0,0 +RESTRAINS,0,0,0,0,0,0,2009 +RESTRICTED,0,0,0,0,0,0,2009 +RESTRICTIVE,0,0,0,0,0,0,2009 +RESTRUCTURE,2009,0,0,0,0,0,0 +RESTRUCTURINGS,2009,0,0,0,0,0,0 +RETALIATING,2009,0,0,0,0,0,0 +RETENDERING,0,0,0,2011,0,0,0 +PRECIPITATED,2009,0,0,0,0,0,0 +PRECLUDED,2009,0,0,0,0,0,2009 +PRECONDITIONS,0,0,0,0,0,0,2009 +PREDECEASES,0,0,0,2009,0,0,0 +PREDICTED,0,0,2009,0,0,0,0 +PREDICTIVE,0,0,2009,0,0,0,0 +PREEMINENCE,0,2009,0,0,0,0,0 +PREJUDICED,2009,0,0,2009,0,0,0 +PRELIMINARILY,0,0,2009,0,0,0,0 +PREMIER,0,2009,0,0,0,0,0 +PRESSING,2009,0,0,0,0,0,0 +PRESUME,0,0,2009,0,0,0,0 +PRESUMPTION,0,0,2009,0,0,0,0 +PREVENT,0,0,0,0,0,0,2009 +PREVENTS,2009,0,0,0,0,0,2009 +PROACTIVELY,0,2009,0,0,0,0,0 +PROBABLE,0,0,2009,0,0,0,0 +PROBATES,0,0,0,2009,0,0,0 +PROBATIONARY,0,0,0,2009,0,0,0 +PROBLEM,2009,0,0,0,0,0,0 +PROFICIENCY,0,2009,0,0,0,0,0 +PROFITABLE,0,2009,0,0,0,0,0 +PROGRESSES,0,2009,0,0,0,0,0 +PROHIBITING,0,0,0,0,0,0,2009 +PROHIBITIVELY,0,0,0,0,0,0,2009 +PROLONGATION,2009,0,0,0,0,0,0 +PROLONGS,2009,0,0,0,0,0,0 +PROMULGATING,0,0,0,2009,0,0,0 +PROMULGATORS,0,0,0,2009,0,0,0 +PROSECUTE,2009,0,0,2009,0,0,0 +PROSECUTION,2009,0,0,2009,0,0,0 +PROSECUTORS,0,0,0,2009,0,0,0 +PROSPEROUS,0,2009,0,0,0,0,0 +PROTESTER,2009,0,0,0,0,0,0 +PROTESTORS,2009,0,0,0,0,0,0 +PROVISO,0,0,0,2009,0,0,0 +PROVOKED,2009,0,0,0,0,0,0 +PUNISHED,2009,0,0,0,0,0,0 +PUNISHMENTS,2009,0,0,0,0,0,0 +PURPORTEDLY,2009,0,0,0,0,0,0 +QUESTIONABLE,2009,0,0,0,0,0,0 +QUESTIONS,2009,0,0,0,0,0,0 +QUITTING,2009,0,0,0,0,0,0 +RANDOMIZE,0,0,2009,0,0,0,0 +RANDOMLY,0,0,2009,0,0,0,0 +RATABLY,0,0,0,2009,0,0,0 +RATIONALIZED,2009,0,0,0,0,0,0 +ABANDONING,2009,0,0,0,0,0,0 +ABDICATED,2009,0,0,0,0,0,0 +ABDICATIONS,2009,0,0,0,0,0,0 +ABERRATIONS,2009,0,0,0,0,0,0 +ABIDE,0,0,0,0,0,0,2009 +ABNORMALITIES,2009,0,0,0,0,0,0 +ABOLISHED,2009,0,0,0,0,0,0 +ABROGATE,2009,0,0,2009,0,0,0 +ABROGATION,2009,0,0,2009,0,0,0 +ABRUPTNESS,2009,0,0,0,0,0,0 +ABSOLVE,0,0,0,2009,0,0,0 +ABUNDANCE,0,2009,0,0,0,0,0 +ABUSES,2009,0,0,0,0,0,0 +ABUSIVENESS,2009,0,0,0,0,0,0 +ACCIDENTAL,2009,0,0,0,0,0,0 +ACCOMPLISH,0,2009,0,0,0,0,0 +ACCOMPLISHMENT,0,2009,0,0,0,0,0 +ACCUSE,2009,0,0,0,0,0,0 +ACHIEVE,0,2009,0,0,0,0,0 +ACHIEVES,0,2009,0,0,0,0,0 +ACQUIESCES,2009,0,0,0,0,0,0 +ACQUIT,2009,0,0,2009,0,0,0 +ACQUITTANCE,0,0,0,2009,0,0,0 +ADDENDUMS,0,0,0,2011,0,0,0 +ADJOURNING,0,0,0,2009,0,0,0 +ADJUDGE,0,0,0,2009,0,0,0 +ADJUDICATE,0,0,0,2009,0,0,0 +ADJUDICATION,0,0,0,2009,0,0,0 +ADJUDICATORS,0,0,0,2009,0,0,0 +ADMISSIBLY,0,0,0,2009,0,0,0 +ADULTERATED,2009,0,0,0,0,0,0 +ADVANCEMENT,0,2009,0,0,0,0,0 +ADVANTAGE,0,2009,0,0,0,0,0 +ADVANTAGES,0,2009,0,0,0,0,0 +ADVERSE,2009,0,0,0,0,0,0 +AFFIDAVIT,0,0,0,2009,0,0,0 +AFOREDESCRIBED,0,0,0,2011,0,0,0 +AFTERMATH,2009,0,0,0,0,0,0 +AGGRAVATED,2009,0,0,0,0,0,0 +AGGRAVATIONS,2009,0,0,0,0,0,0 +ALIENATE,2009,0,0,0,0,0,0 +ALIENATION,2009,0,0,0,0,0,0 +ALLEGE,2009,0,0,2009,0,0,0 +ALLEGING,2009,0,0,2009,0,0,0 +ALTERATION,0,0,2009,0,0,0,0 +AMBIGUITY,0,0,2009,0,0,0,0 +AMENDATORY,0,0,0,2009,0,0,0 +AMENDMENTS,0,0,0,2009,0,0,0 +ANNOYANCES,2009,0,0,0,0,0,0 +ANNUL,2009,0,0,0,0,0,0 +ANNULMENTS,2009,0,0,0,0,0,0 +ANOMALOUSLY,2009,0,2009,0,0,0,0 +ANTICIPATE,0,0,2009,0,0,0,0 +ANTICIPATION,0,0,2009,0,0,0,0 +ANTITRUST,2009,0,0,2009,0,0,0 +APPEAL,0,0,0,2009,0,0,0 +APPEALS,0,0,0,2009,0,0,0 +APPEARS,0,0,2009,0,0,2009,0 +APPELLEES,0,0,0,2011,0,0,0 +APPROXIMATELY,0,0,2009,0,0,0,0 +APPROXIMATIONS,0,0,2009,0,0,0,0 +ARBITRABILITY,0,0,0,2011,0,0,0 +ARBITRARY,0,0,2009,0,0,0,0 +ARBITRATING,0,0,0,2009,0,0,0 +ARBITRATIVE,0,0,0,2011,0,0,0 +ARGUED,2009,0,0,0,0,0,0 +ARGUMENTS,2009,0,0,0,0,0,0 +ARREST,2009,0,0,0,0,0,0 +RETRIBUTIONS,2009,0,0,0,0,0,0 +BONA,0,0,0,2009,0,0,0 +BOOST,0,2009,0,0,0,0,0 +BOUND,0,0,0,0,0,0,2011 +BOYCOTTING,2009,0,0,0,0,0,0 +BREACHES,2009,0,0,2009,0,0,0 +BREAKAGES,2009,0,0,0,0,0,0 +BREAKS,2009,0,0,0,0,0,0 +BRIBED,2009,0,0,0,0,0,0 +BRIBING,2009,0,0,0,0,0,0 +BURDEN,2009,0,0,0,0,0,0 +BURDENSOME,2009,0,0,0,0,0,0 +CALAMITY,2009,0,0,0,0,0,0 +CANCELLATION,2009,0,0,0,0,0,0 +CANCELS,2009,0,0,0,0,0,0 +CATASTROPHICALLY,2009,0,0,0,0,0,0 +CAUTIONING,2009,0,0,0,0,0,0 +CAUTIOUSNESS,0,0,2009,0,0,0,0 +CEASING,2009,0,0,0,0,0,0 +CENSURED,2009,0,0,0,0,0,0 +CESSION,0,0,0,2009,0,0,0 +CHALLENGING,2009,0,0,0,0,0,0 +CHATTELS,0,0,0,2009,0,0,0 +CIRCUMVENTING,2009,0,0,0,0,0,0 +SHARPLY,2009,0,0,0,0,0,0 +SHORTFALL,2009,0,0,0,0,0,0 +SHUT,2009,0,0,0,0,0,0 +SHUTTING,2009,0,0,0,0,0,0 +SLANDERS,2009,0,0,0,0,0,0 +REPUDIATE,2009,0,0,0,0,0,0 +REPUDIATION,2009,0,0,0,0,0,0 +REQUIRE,0,0,0,0,0,0,2009 +REQUIRES,0,0,0,0,0,0,2009 +RESCINDED,0,0,0,2009,0,0,0 +RESCISSIONS,0,0,0,2009,0,0,0 +RESIGNED,2009,0,0,0,0,0,0 +RESTATE,2009,0,0,0,0,0,0 +RESTATES,2009,0,0,0,0,0,0 +NONTERMINABLE,0,0,0,2011,0,0,0 +NOTARIZATION,0,0,0,2009,0,0,0 +NOTARIZING,0,0,0,2009,0,0,0 +NUISANCE,2009,0,0,0,0,0,0 +NULLIFIED,2009,0,0,2009,0,0,0 +NULLITIES,0,0,0,2009,0,0,0 +OBJECTION,2009,0,0,0,0,0,0 +OBLIGATE,0,0,0,0,0,0,2009 +OBLIGATION,0,0,0,0,0,0,2009 +OBLIGED,0,0,0,0,0,0,2009 +OBLIGOR,0,0,0,2009,0,0,0 +OBSOLESCENCE,2009,0,0,0,0,0,0 +OBSTRUCT,2009,0,0,0,0,0,0 +OBSTRUCTIONS,2009,0,0,0,0,0,0 +OFFEND,2009,0,0,0,0,0,0 +OFFENDING,2009,0,0,0,0,0,0 +OFFEREES,0,0,0,2011,0,0,0 +OMISSIONS,2009,0,0,0,0,0,0 +OMITTING,2009,0,0,0,0,0,0 +OPPORTUNITIES,0,2009,0,0,0,0,0 +OPPOSES,2009,0,0,0,0,0,0 +OPTIMISTIC,0,2009,0,0,0,0,0 +OUTAGE,2009,0,0,0,0,0,0 +OUTPERFORM,0,2009,0,0,0,0,0 +OVERAGE,2009,0,0,0,0,0,0 +OVERBUILDS,2009,0,0,0,0,0,0 +OVERBURDENING,2009,0,0,0,0,0,0 +OVERCHARGED,2009,0,0,0,0,0,0 +OVERCOMES,2009,0,0,0,0,0,0 +OVERESTIMATED,2009,0,0,0,0,0,0 +OVERESTIMATIONS,2009,0,0,0,0,0,0 +OVERLOADS,2009,0,0,0,0,0,0 +OVERLOOKS,2009,0,0,0,0,0,0 +OVERPRODUCED,2009,0,0,0,0,0,0 +OVERRULE,0,0,0,2009,0,0,0 +OVERRUN,2009,0,0,0,0,0,0 +OVERSHADOWED,2009,0,0,0,0,0,0 +OVERSTATED,2009,0,0,0,0,0,0 +OVERSTATING,2009,0,0,0,0,0,0 +OVERSUPPLYING,2009,0,0,0,0,0,0 +OVERTURNING,2009,0,0,0,0,0,0 +OVERVALUING,2009,0,0,0,0,0,0 +PARI,0,0,0,2009,0,0,0 +NECESSITATES,0,0,0,0,0,0,2009 +NEGATIVES,2009,0,0,0,0,0,0 +NEGLECTING,2009,0,0,0,0,0,0 +NEGLIGENT,2009,0,0,0,0,0,0 +BARRED,2009,0,0,0,0,0,0 +BEAUTIFULLY,0,2009,0,0,0,0,0 +BELIEVING,0,0,2009,0,0,0,0 +IDLE,2009,0,0,0,0,0,0 +IGNORED,2009,0,0,0,0,0,0 +ILLEGAL,2009,0,0,0,0,0,0 +ILLEGIBLE,2009,0,0,0,0,0,0 +ILLIQUIDITY,2009,0,0,0,0,0,0 +IMMATURE,2009,0,0,0,0,0,0 +IMPAIRING,2009,0,0,0,0,0,2011 +IMPASSE,2009,0,0,0,0,0,0 +IMPEDES,2009,0,0,0,0,0,0 +IMPENDING,2009,0,0,0,0,0,0 +IMPERIL,2009,0,0,0,0,0,0 +IMPLICATED,2009,0,0,0,0,0,0 +IMPOSED,0,0,0,0,0,0,2009 +IMPOSITIONS,0,0,0,0,0,0,2009 +IMPOUNDED,2009,0,0,0,0,0,0 +IMPRACTICAL,2009,0,0,0,0,0,0 +IMPRECISION,0,0,2009,0,0,0,0 +IMPRESSES,0,2009,0,0,0,0,0 +IMPRISONMENT,2009,0,0,0,0,0,0 +IMPROPERLY,2009,0,0,0,0,0,0 +IMPROVED,0,2009,0,0,0,0,0 +IMPROVING,0,2009,0,0,0,0,0 +INACCESSIBLE,2009,0,0,0,0,0,0 +INACCURATELY,2009,0,0,0,0,0,0 +INACTIVATED,2009,0,0,0,0,0,0 +INACTIVATIONS,2009,0,0,0,0,0,0 +INADEQUATE,2009,0,0,0,0,0,0 +INADVISABILITY,2009,0,0,0,0,0,0 +INASMUCH,0,0,0,2009,0,0,0 +INCAPACITY,2009,0,0,2009,0,0,0 +INCARCERATING,2009,0,0,2009,0,0,0 +INCIDENCE,2009,0,0,0,0,0,0 +INCOMPATIBILITIES,2009,0,0,0,0,0,0 +INCOMPETENCY,2009,0,0,0,0,0,0 +INCOMPLETE,2009,0,0,0,0,0,0 +INCONSISTENCIES,2009,0,0,0,0,0,0 +INCONTESTABILITY,0,0,0,2009,0,0,0 +INCONVENIENT,2009,0,0,0,0,0,0 +INCREDIBLE,0,2009,0,0,0,0,0 +INDECENT,2009,0,0,0,0,0,0 +INDEFINITELY,0,0,2009,0,0,0,0 +INDEMNIFICATIONS,0,0,0,2009,0,0,0 +INDEMNIFYING,0,0,0,2009,0,0,0 +INDEMNITOR,0,0,0,2009,0,0,0 +INDETERMINATE,0,0,2009,0,0,0,0 +INDICTING,2009,0,0,2009,0,0,0 +INEFFECTIVE,2009,0,0,0,0,0,0 +INEFFICIENCY,2009,0,0,0,0,0,0 +INELIGIBLE,2009,0,0,0,0,0,0 +INEQUITY,2009,0,0,0,0,0,0 +INEXPERIENCE,2009,0,0,0,0,0,0 +INFLUENTIAL,0,2009,0,0,0,0,0 +INFRACTIONS,2009,0,0,2009,0,0,0 +INFRINGEMENTS,2009,0,0,0,0,0,0 +INGENUITY,0,2009,0,0,0,0,0 +INHIBITS,0,0,0,0,0,0,2011 +INJUNCTIVE,0,0,0,2009,0,0,0 +INJURIES,2009,0,0,0,0,0,0 +INNOVATE,0,2009,0,0,0,0,0 +INNOVATION,0,2009,0,0,0,0,0 +INNOVATOR,0,2009,0,0,0,0,0 +INQUIRY,2009,0,0,0,0,0,0 +INSIST,0,0,0,0,0,0,2009 +INSISTS,0,0,0,0,0,0,2009 +INSOLVENT,2009,0,0,0,0,0,0 +INSTABILITY,2009,0,2009,0,0,0,0 +INSUFFICIENTLY,2009,0,0,0,0,0,0 +INTANGIBLES,0,0,2009,0,0,0,0 +INTERFERED,2009,0,0,0,0,0,0 +INTERFERING,2009,0,0,0,0,0,0 +INTERPLEADER,0,0,0,2009,0,0,0 +INTERPOSING,0,0,0,2009,0,0,0 +INTERROGATED,0,0,0,2009,0,0,0 +INTERROGATIONS,0,0,0,2009,0,0,0 +INTERROGATORY,0,0,0,2009,0,0,0 +INTERRUPTION,2009,0,0,0,0,0,0 +INTESTATE,0,0,0,2009,0,0,0 +SPORADIC,0,0,2009,0,0,0,0 +STABILIZATIONS,0,2009,0,0,0,0,0 +STABILIZING,0,2009,0,0,0,0,0 +STAGNATE,2009,0,0,0,0,0,0 +STAGNATION,2009,0,0,0,0,0,0 +STATUTES,0,0,0,2009,0,0,0 +STIPULATED,0,0,0,0,0,0,2009 +STIPULATIONS,0,0,0,0,0,0,2009 +STOPPED,2009,0,0,0,0,0,0 +STRAINED,2009,0,0,0,0,0,0 +STRENGTHEN,0,2009,0,0,0,0,0 +STRENGTHS,0,2009,0,0,0,0,0 +STRESSFUL,2009,0,0,0,0,0,0 +STRICTEST,0,0,0,0,0,0,2009 +STRONGER,0,2009,0,0,0,0,0 +SUBCLAUSES,0,0,0,2009,0,0,0 +SUBJECTION,2009,0,0,0,0,0,0 +SUBLICENSEE,0,0,0,2009,0,0,0 +SUBPOENA,2009,0,0,2009,0,0,0 +SUBROGATION,0,0,0,2009,0,0,0 +SUCCEED,0,2009,0,0,0,0,0 +SUCCESS,0,2009,0,0,0,0,0 +SUDDEN,0,0,2009,0,0,0,0 +SUES,2009,0,0,2009,0,0,0 +SUFFERS,2009,0,0,0,0,0,0 +SUGGESTS,0,0,2009,0,0,2009,0 +SUMMONS,2009,0,0,2009,0,0,0 +SUPERSEDEAS,0,0,0,2011,0,0,0 +SURETIES,0,0,0,2009,0,0,0 +SURPASSES,0,2009,0,0,0,0,0 +SUSPECT,2009,0,0,0,0,0,0 +SUSPENDED,2009,0,0,0,0,0,0 +SUSPENSIONS,2009,0,0,0,0,0,0 +SUSPICIOUSLY,2009,0,0,0,0,0,0 +REVISE,0,0,2009,0,0,0,0 +REVOCATIONS,2009,0,0,2009,0,0,0 +REVOKING,2009,0,0,0,0,0,0 +REVOLUTIONIZING,0,2009,0,0,0,0,0 +REWARDS,0,-2020,0,0,0,0,0 +RIDICULING,2009,0,0,0,0,0,0 +RISKIEST,2009,0,2009,0,0,0,0 +RISKY,2009,0,2009,0,0,0,0 +RUMORS,0,0,2009,0,0,0,0 +SACRIFICES,2009,0,0,0,0,0,0 +SATISFACTORILY,0,2009,0,0,0,0,0 +SATISFY,0,2009,0,0,0,0,0 +SCRUTINIZE,2009,0,0,0,0,0,0 +SCRUTINY,2009,0,0,0,0,0,0 +SEIZED,2009,0,0,0,0,0,0 +SELDOMLY,0,0,2009,0,0,2009,0 +SERIOUS,2009,0,0,0,0,0,0 +SETBACKS,2009,0,0,0,0,0,0 +SEVERABILITY,0,0,0,2009,0,0,0 +SEVERANCES,0,0,0,2009,0,0,0 +SEVERITIES,2009,0,0,0,0,0,0 +ENTHUSIASM,0,2009,0,0,0,0,0 +ENTRENCHED,0,0,0,0,0,0,2009 +ERODING,2009,0,0,0,0,0,0 +ERRED,2009,0,0,0,0,0,0 +ERROR,2009,0,0,0,0,0,0 +ESCALATED,2009,0,0,0,0,0,0 +ESCHEATED,0,0,0,2011,0,0,0 +ESCROWING,0,0,0,2011,0,0,0 +EVADED,2009,0,0,0,0,0,0 +EVASIONS,2009,0,0,0,0,0,0 +EVICTING,2009,0,0,0,0,0,0 +EVIDENTIAL,0,0,0,2011,0,0,0 +EXACERBATES,2009,0,0,0,0,0,0 +EXAGGERATE,2009,0,0,0,0,0,0 +EXAGGERATION,2009,0,0,0,0,0,0 +EXCELLENCE,0,2009,0,0,0,0,0 +EXCEPTIONAL,0,2009,0,0,0,0,0 +EXCISED,0,0,0,2009,0,0,0 +EXCLUSIVE,0,2009,0,0,0,0,0 +EXCLUSIVITY,0,2009,0,0,0,0,0 +EXCULPATING,2009,0,0,2009,0,0,0 +EXECUTOR,0,0,0,2009,0,0,0 +EXECUTRIX,0,0,0,2009,0,0,0 +EXONERATED,2009,0,0,0,0,0,0 +EXONERATIONS,2009,0,0,0,0,0,0 +EXPLOITATIVE,2009,0,0,0,0,0,0 +EXPOSE,2009,0,0,0,0,0,0 +EXPOSURE,0,0,2009,0,0,0,0 +EXPROPRIATES,2009,0,0,0,0,0,0 +EXPULSION,2009,0,0,0,0,0,0 +EXTRACORPOREAL,0,0,0,2011,0,0,0 +SOLVENCY,2009,0,0,0,0,0,0 +SOMETIMES,0,0,2009,0,0,2009,0 +SPAMMERS,2014,0,0,0,0,0,0 +TREMENDOUS,0,2009,0,0,0,0,0 +LOCKOUT,2009,0,0,0,0,0,0 +LOSING,2009,0,0,0,0,0,0 +LOWEST,0,0,0,0,2009,0,0 +MAJEURE,0,0,0,2011,0,0,0 +MALFUNCTIONING,2009,0,0,0,0,0,0 +MALICIOUSLY,2009,0,0,0,0,0,0 +MANDATED,0,0,0,0,0,0,2009 +MANDITORILY,0,0,0,0,0,0,2011 +MANIPULATING,2009,0,0,0,0,0,0 +MARKDOWN,2009,0,0,0,0,0,0 +MEDIATE,0,0,0,2009,0,0,0 +MEDIATION,0,0,0,2009,0,0,0 +MERITORIOUS,0,2009,0,0,0,0,0 +MISAPPLIED,2009,0,0,0,0,0,0 +MISAPPROPRIATE,2009,0,0,0,0,0,0 +MISAPPROPRIATION,2009,0,0,0,0,0,0 +MISCALCULATED,2009,0,0,0,0,0,0 +MISCALCULATIONS,2009,0,0,0,0,0,0 +MISCLASSIFICATIONS,2014,0,0,0,0,0,0 +MISCONDUCT,2009,0,0,0,0,0,0 +MISDIRECTED,2009,0,0,0,0,0,0 +MISHANDLES,2009,0,0,0,0,0,0 +MISINFORMED,2009,0,0,0,0,0,0 +MISINTERPRETATION,2009,0,0,0,0,0,0 +MISINTERPRETS,2009,0,0,0,0,0,0 +MISJUDGING,2009,0,0,0,0,0,0 +MISLABELED,2009,0,0,0,0,0,0 +MISLEAD,2009,0,0,0,0,0,0 +MISLED,2009,0,0,0,0,0,0 +MISMANAGES,2009,0,0,0,0,0,0 +MISMATCHES,2009,0,0,0,0,0,0 +MISPRICING,2014,0,0,0,0,0,0 +MISREPRESENTATIONS,2009,0,0,0,0,0,0 +MISS,2009,0,0,0,0,0,0 +WORSE,2009,0,0,0,0,0,0 +WORSENS,2009,0,0,0,0,0,0 +TOLERATES,2009,0,0,0,0,0,0 +TORTIOUS,0,0,0,2009,0,0,0 +TORTUOUSLY,2009,0,0,0,0,0,0 +FINED,2009,0,0,0,0,0,0 +NONAPPEALABLE,0,0,0,2009,0,0,0 +NONCANCELABLE,0,0,0,0,0,0,2009 +NONCOMPLIANCES,2009,0,0,0,0,0,0 +NONCONFORMITIES,2009,0,0,0,0,0,0 +NONCONTRACTUAL,0,0,0,2011,0,0,0 +NONFIDUCIARY,0,0,0,2011,0,0,0 +NONFUNCTIONAL,2009,0,0,0,0,0,0 +DISCLAIMER,2009,0,0,0,0,0,0 +DISCLOSE,2009,0,0,0,0,0,0 +DISCONTINUANCE,2009,0,0,0,0,0,0 +DISCONTINUE,2009,0,0,0,0,0,0 +DISCOURAGE,2009,0,0,0,0,0,0 +DISCREDIT,2009,0,0,0,0,0,0 +DISCREPANCIES,2009,0,0,0,0,0,0 +SPECULATE,0,0,2009,0,0,0,0 +SPECULATION,0,0,2009,0,0,0,0 +TRAGIC,2009,0,0,0,0,0,0 +TRANSPARENCY,0,2009,0,0,0,0,0 +LEGISLATE,0,0,0,2009,0,0,0 +LEGISLATION,0,0,0,2009,0,0,0 +LEGISLATOR,0,0,0,2009,0,0,0 +LIBEL,0,0,0,2009,0,0,0 +LICENSABLE,0,0,0,2011,0,0,0 +CONCEALING,2009,0,0,0,0,0,0 +CONCEDING,2009,0,0,0,0,0,0 +CONCERNED,2009,0,0,0,0,0,0 +CONCILIATIONS,2009,0,0,0,0,0,0 +CONDEMNATION,2009,0,0,0,0,0,0 +CONDEMNOR,0,0,0,2011,0,0,0 +CONDONE,2009,0,0,0,0,0,0 +CONFESSED,2009,0,0,0,0,0,0 +CONFIDENT,0,2009,0,0,0,0,0 +CONFINEMENTS,2009,0,0,0,0,0,0 +CONFISCATED,2009,0,0,0,0,0,0 +CONFISCATIONS,2009,0,0,0,0,0,0 +CONFLICTING,2009,0,0,0,0,0,0 +CONFRONTATIONAL,2009,0,0,0,0,0,0 +CONFRONTS,2009,0,0,0,0,0,0 +CONFUSING,2009,0,2009,0,0,0,0 +CONSENTED,0,0,0,2009,0,0,0 +CONSPIRACIES,2009,0,0,0,0,0,0 +CONSPIRATORS,2009,0,0,0,0,0,0 +CONSPIRING,2009,0,0,0,0,0,0 +CONSTITUTIONALLY,0,0,0,2009,0,0,0 +CONSTRAINED,0,0,0,0,0,0,2009 +CONSTRAINTS,0,0,0,0,0,0,2009 +CONSTRUED,0,0,0,2012,0,0,0 +CONTEND,2009,0,0,0,0,0,0 +CONTENTION,2009,0,0,0,0,0,0 +CONTESTABILITY,0,0,0,2014,0,0,0 +CONTINGENCIES,0,0,2009,0,0,0,0 +CONTINGENTS,0,0,2009,0,0,0,0 +CONTRACTHOLDERS,0,0,0,2009,0,0,0 +CONTRACTION,2009,0,0,0,0,0,0 +CONTRACTUALLY,0,0,0,2009,0,0,0 +CONTRADICTION,2009,0,0,0,0,0,0 +CONTRARY,2009,0,0,0,0,0,0 +CONTRAVENING,0,0,0,2009,0,0,0 +CONTROVERSIES,2009,0,0,0,0,0,0 +CONTROVERTING,0,0,0,2009,0,0,0 +CONVICT,2009,0,0,2009,0,0,0 +CONVICTIONS,2009,0,0,2009,0,0,0 +CORRECTIONS,2009,0,0,0,0,0,0 +CORRUPTING,2009,0,0,0,0,0,0 +CORRUPTNESS,2009,0,0,0,0,0,0 +COUNSEL,0,0,0,2009,0,0,0 +COUNTERCLAIM,2009,0,0,0,0,0,0 +COUNTERFEIT,2009,0,0,0,0,0,0 +COUNTERFEITING,2009,0,0,0,0,0,0 +COUNTERSIGNOR,0,0,0,2011,0,0,0 +COURT,0,0,0,2009,0,0,0 +COVENANT,0,0,0,0,0,0,2011 +CREATIVE,0,2009,0,0,0,0,0 +CRIME,2009,0,0,2009,0,0,0 +CRIMINALIZE,0,0,0,2014,0,0,0 +CRISES,2009,0,0,0,0,0,0 +CRITICISM,2009,0,0,0,0,0,0 +CRITICIZES,2009,0,0,0,0,0,0 +CROSSROAD,0,0,2009,0,0,0,0 +CULPABILITY,2009,0,0,0,0,0,0 +CURTAIL,2009,0,0,0,0,0,0 +CURTAILMENTS,2009,0,0,0,0,0,0 +CUTBACKS,2009,0,0,0,0,0,0 +CYBERCRIME,2014,0,0,0,0,0,0 +TAINTING,2009,0,0,0,0,0,0 +TENDING,0,0,2009,0,0,0,0 +TERMINABLE,0,0,0,2009,0,0,0 +TERMINATING,2009,0,0,0,0,0,0 +TESTAMENTARY,0,0,0,2009,0,0,0 +THENCE,0,0,0,2009,0,0,0 +THEREAT,0,0,0,2009,0,0,0 +THEREOF,0,0,0,2009,0,0,0 +THERETOFOR,0,0,0,2011,0,0,0 +THEREUPON,0,0,0,2009,0,0,0 +THREATENED,2009,0,0,0,0,0,0 +FAILED,2009,0,0,0,0,0,0 +FAILURE,2009,0,0,0,0,0,0 +FALSELY,2009,0,0,0,0,0,0 +FALSIFIES,2009,0,0,0,0,0,0 +FANTASTIC,0,2009,0,0,0,0,0 +FAULT,2009,0,0,0,0,0,0 +FAVORABLE,0,2009,0,0,0,0,0 +FAVORITE,0,2009,0,0,0,0,0 +FELONIES,2009,0,0,2009,0,0,0 +DAMAGED,2009,0,0,0,0,0,0 +DAMPENED,2009,0,0,0,0,0,0 +DANGERS,2009,0,0,0,0,0,0 +DEADLOCKS,2009,0,0,0,0,0,0 +DEBARMENTS,2009,0,0,0,0,0,0 +DECEDENTS,0,0,0,2009,0,0,0 +DECEIVE,2009,0,0,0,0,0,0 +DECEPTION,2009,0,0,0,0,0,0 +DECLARANT,0,0,0,2011,0,0,0 +DECLINING,2009,0,0,0,0,0,0 +DECREES,0,0,0,2009,0,0,0 +DEFALCATION,0,0,0,2009,0,0,0 +DEFAMATORY,2009,0,0,0,0,0,0 +DEFAMING,2009,0,0,0,0,0,0 +DEFAULTS,2009,0,0,0,0,0,0 +DEFEASED,0,0,0,2009,0,0,0 +DEFEAT,2009,0,0,0,0,0,0 +DEFECT,2009,0,0,0,0,0,0 +DEFEND,2009,0,0,0,0,0,0 +DEFENDED,2009,0,0,0,0,0,0 +DEFER,2009,0,0,0,0,0,0 +DEFICIENT,2009,0,0,0,0,0,0 +DEFINITIVELY,0,0,0,0,2009,0,0 +DEFRAUDS,2009,0,0,0,0,0,0 +DEGRADE,2009,0,0,0,0,0,0 +DELAY,2009,0,0,0,0,0,0 +DELEGABLE,0,0,0,2009,0,0,0 +DELETERIOUS,2009,0,0,0,0,0,0 +DELIGHT,0,2009,0,0,0,0,0 +DELIGHTING,0,2009,0,0,0,0,0 +DELINQUENT,2009,0,0,0,0,0,0 +DELISTED,2009,0,0,0,0,0,0 +DEMISED,2009,0,0,0,0,0,0 +DEMOLISHED,2009,0,0,0,0,0,0 +DEMOLITIONS,2009,0,0,0,0,0,0 +DEMOTING,2009,0,0,0,0,0,0 +DEMURRER,0,0,0,2009,0,0,0 +DENIAL,2009,0,0,0,0,0,0 +DENIGRATE,2009,0,0,0,0,0,0 +DENIGRATION,2009,0,0,0,0,0,0 +DEPENDABILITY,0,2009,0,0,0,0,0 +DEPENDANT,0,0,0,0,0,0,2011 +DEPENDENCY,0,0,2009,0,0,0,0 +DEPLETE,2009,0,0,0,0,0,0 +DEPLETION,2009,0,0,0,0,0,0 +DEPOSES,0,0,0,2009,0,0,0 +DEPOSITIONS,0,0,0,2009,0,0,0 +INTRUSION,2009,0,0,0,0,0,0 +INVALIDATES,2009,0,0,0,0,0,0 +INVENT,0,2009,0,0,0,0,0 +INVENTIONS,0,2009,0,0,0,0,0 +INVENTORS,0,2009,0,0,0,0,0 +INVESTIGATING,2009,0,0,0,0,0,0 +INVOLUNTARY,2009,0,0,0,0,0,0 +IRRECOVERABLY,2009,0,0,0,0,0,0 +IRREGULARLY,2009,0,0,0,0,0,0 +IRREVOCABILITY,0,0,0,2011,0,0,0 +JEOPARDIZED,2009,0,0,0,0,0,0 +JUDICIARIES,0,0,0,2009,0,0,0 +JURISDICTION,0,0,0,2009,0,0,0 +JURISPRUDENCE,0,0,0,2009,0,0,0 +JURORS,0,0,0,2009,0,0,0 +JUSTICES,0,0,0,2009,0,0,0 +KNOWINGLY,2009,0,0,0,0,0,0 +DILIGENT,0,2009,0,0,0,0,0 +DIMINISHES,2009,0,0,0,0,0,0 +DIRECTIVES,0,0,0,0,0,0,2009 +DISADVANTAGES,2009,0,0,0,0,0,0 +DISAFFIRMED,0,0,0,2011,0,0,0 +DISAGREED,2009,0,0,0,0,0,0 +DISAGREES,2009,0,0,0,0,0,0 +DISALLOWED,2009,0,0,0,0,0,0 +DISAPPEARANCE,2009,0,0,0,0,0,0 +DISAPPEARS,2009,0,0,0,0,0,0 +DISAPPOINTINGLY,2009,0,0,0,0,0,0 +DISAPPROVAL,2009,0,0,0,0,0,0 +DISAPPROVES,2009,0,0,0,0,0,0 +DISASSOCIATION,2009,0,0,0,0,0,0 +DISASTROUS,2009,0,0,0,0,0,0 +DISAVOWED,2009,0,0,0,0,0,0 +GREAT,0,-2020,0,0,0,0,0 +GREATNESS,0,2009,0,0,0,0,0 +GROUNDLESS,2009,0,0,0,0,0,0 +HAMPER,2009,0,0,0,0,0,0 +HAPPIEST,0,2009,0,0,0,0,0 +HARASS,2009,0,0,0,0,0,0 +HARDSHIP,2009,0,0,0,0,0,0 +HARMFUL,2009,0,0,0,0,0,0 +HARSH,2009,0,0,0,0,0,0 +HARSHNESS,2009,0,0,0,0,0,0 +NONJUDICIAL,0,0,0,2009,0,0,0 +NONPAYMENTS,2009,0,0,0,0,0,0 \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/python-challenge/task.py b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/python-challenge/task.py new file mode 100644 index 0000000000000..a7e0702e011ab --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/python-challenge/task.py @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# beam-playground: +# name: FinalChallenge2 +# description: Final challenge 2. +# multifile: true +# files: +# - name: analysis.csv +# context_line: 50 +# categories: +# - Quickstart +# complexity: ADVANCED +# tags: +# - hellobeam + +import re +import apache_beam as beam +from apache_beam.io import ReadFromText +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.transforms import window, trigger +from apache_beam.transforms.combiners import CountCombineFn + + +class SplitWords(beam.DoFn): + def process(self, element): + return element.lower().split(" ") + + +class Analysis: + def __init__(self, word, negative, positive, uncertainty, litigious, strong, weak, constraining): + self.word = word + self.negative = negative + self.positive = positive + self.uncertainty = uncertainty + self.litigious = litigious + self.strong = strong + self.weak = weak + self.constraining = constraining + + def __str__(self): + return (f'Analysis(word={self.word}, negative={self.negative}, positive={self.positive}, ' + f'uncertainty={self.uncertainty}, litigious={self.litigious}, strong={self.strong}, ' + f'weak={self.weak}, constraining={self.constraining})') + +def run(): + pipeline_options = PipelineOptions() + with beam.Pipeline(options=pipeline_options) as p: + shakespeare = (p + | 'Read from text file' >> ReadFromText('gs://apache-beam-samples/shakespeare/kinglear.txt') + | 'Split into words' >> beam.ParDo(SplitWords()) + | 'Filter empty words' >> beam.Filter(bool)) + +if __name__ == "__main__": + run() diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/python-solution/analysis.csv b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/python-solution/analysis.csv new file mode 100644 index 0000000000000..5c4a1246021eb --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/python-solution/analysis.csv @@ -0,0 +1,3877 @@ +Word,Negative,Positive,Uncertainty,Litigious,Strong_Modal,Weak_Modal,Constraining +NONSEVERABLE,0,0,0,2011,0,0,0 +DISFAVOR,2009,0,0,0,0,0,0 +DISGORGE,2009,0,0,0,0,0,0 +COMPLICATES,2009,0,0,0,0,0,0 +COMPLIMENT,0,2009,0,0,0,0,0 +COMPLIMENTS,0,2009,0,0,0,0,0 +MISSTATE,2009,0,0,0,0,0,0 +MISSTATES,2009,0,0,0,0,0,0 +MISTAKE,2009,0,0,0,0,0,0 +MISTAKING,2009,0,0,0,0,0,0 +MISUNDERSTANDING,2009,0,0,0,0,0,0 +MISUSED,2009,0,0,0,0,0,0 +MONOPOLISTS,2009,0,0,0,0,0,0 +MONOPOLIZES,2009,0,0,0,0,0,0 +MORATORIUM,2009,0,0,0,0,0,0 +MOTHBALLING,2009,0,0,0,0,0,0 +SLOW,2009,0,0,0,0,0,0 +SLOWER,2009,0,0,0,0,0,0 +SLOWNESS,2009,0,0,0,0,0,0 +SMOOTH,0,2009,0,0,0,0,0 +DEPRECATION,2009,0,0,0,0,0,0 +DEPRESSING,2009,0,0,0,0,0,0 +DEPRIVES,2009,0,0,0,0,0,0 +DEROGATE,0,0,0,2009,0,0,0 +DEROGATION,0,0,0,2009,0,0,0 +DESIRABLE,0,2009,0,0,0,0,0 +DESTABILIZATION,2009,0,0,0,0,0,0 +DESTINED,0,2009,0,0,0,0,0 +DESTROYS,2009,0,0,0,0,0,0 +DETAINED,2009,0,0,0,0,0,0 +DETER,2009,0,0,0,0,0,0 +DETERIORATING,2009,0,0,0,0,0,0 +DETERRENCE,2009,0,0,0,0,0,0 +DETERRING,2009,0,0,0,0,0,0 +DETRACTING,2009,0,0,0,0,0,0 +DETRIMENTS,2009,0,0,0,0,0,0 +DEVALUING,2009,0,0,0,0,0,0 +DEVASTATION,2009,0,0,0,0,0,0 +DEVIATING,2009,0,2009,0,0,0,0 +DEVOLVE,2009,0,0,0,0,0,0 +DICTATE,0,0,0,0,0,0,2009 +DIFFER,0,0,2009,0,0,0,0 +DIFFICULT,2009,0,0,0,0,0,0 +ASSAULT,2009,0,0,0,0,0,0 +ASSERTABLE,0,0,0,2011,0,0,0 +ASSUMABLE,0,0,0,2009,0,0,0 +ASSUMING,0,0,2009,0,0,0,0 +ASSURED,0,2009,0,0,0,0,0 +ATTAINED,0,2009,0,0,0,0,0 +ATTAINS,0,2009,0,0,0,0,0 +ATTESTED,0,0,0,2009,0,0,0 +ATTORNEYS,0,0,0,2009,0,0,0 +ATTRACTIVENESS,0,2009,0,0,0,0,0 +BAD,2009,0,0,0,0,0,0 +BAILEES,0,0,0,2011,0,0,0 +BAILOUT,2009,0,0,0,0,0,0 +BANKRUPTCIES,2009,0,0,0,0,0,0 +BANKRUPTS,2009,0,0,0,0,0,0 +CLAIM,0,0,0,2009,0,0,0 +CLAIMHOLDER,0,0,0,2014,0,0,0 +CLARIFICATIONS,0,0,2009,0,0,0,0 +CLOSED,-2020,0,0,0,0,0,0 +CLOSINGS,2009,0,0,0,0,0,0 +CODEFENDANTS,0,0,0,2011,0,0,0 +CODIFICATIONS,0,0,0,2009,0,0,0 +CODIFYING,0,0,0,2009,0,0,0 +COERCING,2009,0,0,0,0,0,0 +COLLABORATES,0,2009,0,0,0,0,0 +COLLABORATIVE,0,2009,0,0,0,0,0 +COLLAPSED,2009,0,0,0,0,0,0 +COLLISIONS,2009,0,0,0,0,0,0 +COLLUDING,2009,0,0,0,0,0,0 +POSES,2009,0,0,0,0,0,0 +POSSESSORY,0,0,0,2009,0,0,0 +POSSIBLY,0,0,2009,0,0,2009,0 +POSTJUDGMENT,0,0,0,2011,0,0,0 +POSTPONEMENTS,2009,0,0,0,0,0,0 +WRITEDOWNS,2009,0,0,0,0,0,0 +WRONG,2009,0,0,0,0,0,0 +WRONGFULLY,2009,0,0,0,0,0,0 +HONORABLE,0,-2020,0,0,0,0,0 +HOSTILE,2009,0,0,0,0,0,0 +REASSESS,0,0,2009,0,0,0,0 +REASSESSMENT,2009,0,2009,0,0,0,0 +REASSIGNING,2009,0,0,0,0,0,0 +REBOUND,0,2009,0,0,0,0,0 +REBUTS,0,0,0,2009,0,0,0 +REBUTTALS,0,0,0,2009,0,0,0 +RECALCULATED,0,0,2009,0,0,0,0 +RECALCULATIONS,0,0,2009,0,0,0,0 +RECALLS,2009,0,0,0,0,0,0 +RECESSIONS,2009,0,0,0,0,0,0 +RECONSIDER,0,0,2009,0,0,0,0 +RECORDATION,0,0,0,2009,0,0,0 +RECOURSE,0,0,0,2009,0,0,0 +RECUSAL,0,0,0,2011,0,0,0 +RECUSING,0,0,0,2011,0,0,0 +REDACTION,2009,0,0,2009,0,0,0 +REDEFAULTS,2014,0,0,0,0,0,0 +REDRESSING,2009,0,0,0,0,0,0 +REFERENDA,0,0,0,2009,0,0,0 +REFILED,0,0,0,2009,0,0,0 +REFRAINING,0,0,0,0,0,0,2009 +REFUSE,2009,0,0,0,0,0,0 +REGAIN,0,2009,0,0,0,0,0 +REGULATED,0,0,0,2009,0,0,0 +REGULATIONS,0,0,0,2009,0,0,0 +REGULATORY,0,0,0,2009,0,0,0 +REHEARINGS,0,0,0,2009,0,0,0 +VARIABILITY,0,0,2009,0,0,0,0 +VARIANCE,0,0,2009,0,0,0,0 +VARIATION,0,0,2009,0,0,0,0 +VARY,0,0,2009,0,0,0,0 +VERDICT,2009,0,0,2009,0,0,0 +VETOED,2009,0,0,0,0,0,0 +VICTIMS,2009,0,0,0,0,0,0 +VIOLATING,2009,0,0,0,0,0,0 +VIOLATOR,2009,0,0,0,0,0,0 +VIOLENTLY,2009,0,0,0,0,0,0 +VITIATING,2009,0,0,0,0,0,0 +VOIDING,2009,0,0,2009,0,0,0 +VULNERABILITIES,2009,0,0,0,0,0,0 +WARN,2009,0,0,0,0,0,0 +WARNS,2009,0,0,0,0,0,0 +WASTEFUL,2009,0,0,0,0,0,0 +WEAKENED,2009,0,0,0,0,0,0 +WEAKEST,2009,0,0,0,0,0,0 +DISHONESTLY,2009,0,0,0,0,0,0 +DISHONORABLY,2009,0,0,0,0,0,0 +DISINTERESTEDNESS,2009,0,0,0,0,0,0 +DISMAL,2009,0,0,0,0,0,0 +DISMISSALS,2009,0,0,0,0,0,0 +DISORDERLY,2009,0,0,0,0,0,0 +DISPARAGEMENTS,2009,0,0,0,0,0,0 +DISPARITIES,2009,0,0,0,0,0,0 +DISPLACEMENT,2009,0,0,0,0,0,0 +DISPOSE,2009,0,0,0,0,0,0 +DISPOSSESSES,2009,0,0,0,0,0,0 +DISPROPORTION,2009,0,0,0,0,0,0 +DISPUTE,2009,0,0,0,0,0,0 +DISQUALIFICATION,2009,0,0,0,0,0,0 +DISQUALIFY,2009,0,0,0,0,0,0 +DISREGARDING,2009,0,0,0,0,0,0 +DISRUPT,2009,0,0,0,0,0,0 +DISRUPTIONS,2009,0,0,0,0,0,0 +DISSATISFIED,2009,0,0,0,0,0,0 +DISSENTERS,2009,0,0,0,0,0,0 +DISSIDENTS,2009,0,0,0,0,0,0 +DISTINCTIONS,0,2009,0,0,0,0,0 +DISTORT,2009,0,0,0,0,0,0 +DISTORTIONS,2009,0,0,0,0,0,0 +DISTRACTING,2009,0,0,0,0,0,0 +DISTRAINT,0,0,0,2009,0,0,0 +DISTRIBUTEES,0,0,0,2009,0,0,0 +DISTURBED,2009,0,0,0,0,0,0 +DIVERT,2009,0,0,0,0,0,0 +DIVEST,2009,0,0,0,0,0,0 +DIVESTITURES,2009,0,0,0,0,0,0 +DIVORCE,2009,0,0,0,0,0,0 +DIVULGES,2009,0,0,0,0,0,0 +DOCKETING,0,0,0,2009,0,0,0 +DOUBTED,2009,0,2009,0,0,0,0 +DOWNGRADED,2009,0,0,0,0,0,0 +DOWNSIZED,2009,0,0,0,0,0,0 +DOWNTIME,2009,0,0,0,0,0,0 +DOWNWARD,2009,0,0,0,0,0,0 +DRASTICALLY,2009,0,0,0,0,0,0 +DROPPED,2009,0,0,0,0,0,0 +DURESS,2009,0,0,0,0,0,0 +EARMARK,0,0,0,0,0,0,2009 +EASIER,0,2009,0,0,0,0,0 +EFFECTIVE,0,-2020,0,0,0,0,0 +EFFICIENTLY,0,2009,0,0,0,0,0 +EMBARGO,2009,0,0,0,0,0,0 +EMBARRASS,2009,0,0,0,0,0,0 +EMBARRASSMENT,2009,0,0,0,0,0,0 +EMBEZZLEMENT,2009,0,0,0,0,0,0 +EMBEZZLING,2009,0,0,0,0,0,0 +EMPOWERS,0,2009,0,0,0,0,0 +ENABLING,0,2009,0,0,0,0,0 +ENCOURAGING,0,2009,0,0,0,0,0 +ENCROACHING,2009,0,0,0,0,0,0 +ENCUMBERED,2009,0,0,2009,0,0,2009 +ENCUMBRANCER,0,0,0,2009,0,0,0 +ENDANGERED,2009,0,0,0,0,0,0 +ENDORSEE,0,0,0,2011,0,0,0 +ENHANCE,0,2009,0,0,0,0,0 +ENHANCES,0,2009,0,0,0,0,0 +ENJOINING,2009,0,0,0,0,0,0 +ENJOYABLY,0,2009,0,0,0,0,0 +ENJOYS,0,2009,0,0,0,0,0 +ENTAILS,0,0,0,0,0,0,2009 +PATENTEE,0,0,0,2014,0,0,0 +PENALIZES,2009,0,0,0,0,0,0 +PENDING,0,0,2009,0,0,0,0 +PERFECTS,0,2009,0,0,0,0,0 +PERJURY,2009,0,0,2009,0,0,0 +PERMITTED,0,0,0,0,0,0,2009 +PERPETRATE,2009,0,0,2009,0,0,0 +PERPETRATION,2009,0,0,2009,0,0,0 +PERSISTENT,2009,0,0,0,0,0,0 +PERSONAM,0,0,0,2011,0,0,0 +PETITION,0,0,0,2009,0,0,0 +PETITIONING,0,0,0,2009,0,0,0 +PICKETED,2009,0,0,0,0,0,0 +HENCEFORTH,0,0,0,2009,0,0,0 +HEREDITAMENTS,0,0,0,2009,0,0,0 +HEREIN,0,0,0,2009,0,0,0 +HEREINBELOW,0,0,0,2009,0,0,0 +HERETOFORE,0,0,0,2009,0,0,0 +HEREWITH,0,0,0,2009,0,0,0 +HINDER,2009,0,0,0,0,0,0 +HINDRANCE,2009,0,0,0,0,0,0 +WHATSOEVER,0,0,0,2009,0,0,0 +WHEREAT,0,0,0,2009,0,0,0 +WHEREOF,0,0,0,2009,0,0,0 +WHEREUPON,0,0,0,2009,0,0,0 +WHOMSOEVER,0,0,0,2009,0,0,0 +WILLFUL,0,0,0,2009,0,0,0 +WINNER,0,2009,0,0,0,0,0 +WITNESSES,0,0,0,2009,0,0,0 +FLUCTUATES,0,0,2009,0,0,0,0 +FORBADE,0,0,0,2009,0,0,2011 +FORBEARING,0,0,0,2009,0,0,0 +FORBIDDING,2009,0,0,0,0,0,2009 +FORCING,2009,0,0,0,0,0,0 +FORECLOSE,2009,0,0,0,0,0,0 +FORECLOSURE,2009,0,0,0,0,0,0 +FOREGONE,2009,0,0,0,0,0,0 +FORESTALLS,2009,0,0,0,0,0,0 +FORFEITED,2009,0,0,0,0,0,0 +FORFEITURES,2009,0,0,0,0,0,0 +FORWHICH,0,0,0,2012,0,0,0 +FRAUDULENT,2009,0,0,0,0,0,0 +FRIVOLOUSLY,2009,0,0,0,0,0,0 +FRUSTRATING,2009,0,0,0,0,0,0 +FUGITIVE,-2020,0,0,2009,0,0,0 +GAINED,0,2009,0,0,0,0,0 +GRANTOR,0,0,0,2009,0,0,0 +DISGORGING,2009,0,0,0,0,0,0 +DISHONEST,2009,0,0,0,0,0,0 +LITIGANT,2009,0,0,2009,0,0,0 +LITIGATES,2009,0,0,2009,0,0,0 +LITIGATOR,0,0,0,2009,0,0,0 +LACKLUSTER,2009,0,0,0,0,0,0 +LAGGING,2009,0,0,0,0,0,0 +LAPSES,2009,0,0,0,0,0,0 +LAW,0,0,0,2009,0,0,0 +LAWMAKERS,0,0,0,2009,0,0,0 +LAWSUITS,0,0,0,2009,0,0,0 +LAYOFFS,2009,0,0,0,0,0,0 +LEGALESE,0,0,0,2009,0,0,0 +LEGALIZE,0,0,0,2009,0,0,0 +LEGALLY,0,0,0,2009,0,0,0 +NONPRODUCING,2009,0,0,0,0,0,0 +LIQUIDATE,2009,0,0,0,0,0,0 +LIQUIDATION,2009,0,0,0,0,0,0 +BENEFICIALLY,0,2009,0,0,0,0,0 +BENEFITED,0,2009,0,0,0,0,0 +BEST,0,2012,0,0,2009,0,0 +POPULAR,0,2009,0,0,0,0,0 +REINTERPRETED,0,0,2009,0,0,0,0 +REJECTED,2009,0,0,0,0,0,0 +REJECTS,2009,0,0,0,0,0,0 +RELINQUISHES,2009,0,0,0,0,0,0 +RELUCTANCE,2009,0,0,0,0,0,0 +REMANDING,0,0,0,2009,0,0,0 +REMEDIATING,0,0,0,2009,0,0,0 +REMISED,0,0,0,2011,0,0,0 +RENEGOTIATING,2009,0,0,0,0,0,0 +RENOUNCED,2009,0,0,0,0,0,0 +RENOUNCING,2009,0,0,0,0,0,0 +REPLEVIN,0,0,0,2009,0,0,0 +REPOSSESSION,2009,0,0,0,0,0,0 +TURBULENCE,2009,0,2009,0,0,0,0 +UNACCEPTABLY,2009,0,0,0,0,0,0 +UNANTICIPATED,2009,0,0,0,0,0,0 +UNATTRACTIVE,2009,0,0,0,0,0,0 +UNAVOIDABLE,2009,0,0,0,0,0,0 +UNCERTAINLY,0,0,2009,0,0,2009,0 +UNCOLLECTABLE,2009,0,0,0,0,0,0 +UNCOLLECTIBLES,2009,0,0,0,0,0,0 +UNCONFIRMED,0,0,2009,0,0,0,0 +UNCONSTITUTIONALITY,0,0,0,2009,0,0,0 +UNCONTROLLABLY,2009,0,0,0,0,0,0 +UNCOVERED,2009,0,0,0,0,0,0 +UNDEFEASED,0,0,0,2012,0,0,0 +UNDERCAPITALIZED,2009,0,0,0,0,0,0 +UNDERESTIMATE,2009,0,0,0,0,0,0 +UNDERESTIMATION,2009,0,0,0,0,0,0 +UNDERMINED,2009,0,0,0,0,0,0 +UNDERPAYMENT,2009,0,0,0,0,0,0 +UNDERPERFORMANCE,2009,0,0,0,0,0,0 +UNDERPRODUCED,2009,0,0,0,0,0,0 +UNDERSTATED,2009,0,0,0,0,0,0 +UNDERSTATING,2009,0,0,0,0,0,0 +UNDESIRABLE,2009,0,0,0,0,0,0 +UNDETERMINABLE,0,0,2009,0,0,0,0 +UNDISPUTED,0,0,0,0,2009,0,0 +UNDULY,2009,0,0,0,0,0,0 +UNEMPLOYED,2009,0,0,0,0,0,0 +UNENFORCEABILITY,0,0,0,2009,0,0,0 +UNETHICAL,2009,0,0,0,0,0,0 +UNEXPECTEDLY,2009,0,2009,0,0,0,0 +UNFAMILIARITY,0,0,2009,0,0,0,0 +UNFAVOURABLE,2011,0,0,0,0,0,0 +UNFORECASTED,0,0,2011,0,0,0,0 +UNFORTUNATE,2009,0,0,0,0,0,0 +UNFULFILLED,2009,0,0,0,0,0,0 +UNIDENTIFIABLE,0,0,2009,0,0,0,0 +UNINTENTIONAL,2009,0,0,0,0,0,0 +UNJUSTIFIABLY,2009,0,0,0,0,0,0 +UNKNOWINGLY,2009,0,0,0,0,0,0 +UNLAWFULLY,2009,0,0,2009,0,0,0 +UNMARKETABLE,2009,0,0,0,0,0,0 +UNNECESSARILY,2009,0,0,0,0,0,0 +UNOBTAINABLE,2009,0,0,0,0,0,0 +UNPERFORMED,2009,0,0,0,0,0,0 +UNPREDICTABLE,2009,0,2009,0,0,0,0 +UNPROFITABILITY,2011,0,0,0,0,0,0 +UNQUALIFIED,2009,0,0,0,0,0,0 +UNREASONABLE,2009,0,0,0,0,0,0 +UNRECONCILED,0,0,2011,0,0,0,0 +UNRELIABLE,2009,0,0,0,0,0,0 +UNRESOLVED,2009,0,0,0,0,0,0 +UNSALEABLE,2009,0,0,0,0,0,0 +UNSCHEDULED,2009,0,0,0,0,0,0 +UNSETTLED,0,0,2009,0,0,0,0 +UNSPECIFIED,0,0,2009,0,0,0,0 +UNSUBSTANTIATED,2009,0,0,0,0,0,0 +UNSUITABLE,2009,0,0,0,0,0,0 +UNSURPASSED,0,2009,0,0,2009,0,0 +UNTENABLE,2009,0,0,0,0,0,0 +UNTRUSTED,2014,0,0,0,0,0,0 +UNTRUTHFULNESS,2009,0,0,0,0,0,0 +UNUSUALLY,0,0,2009,0,0,0,0 +UNWILLING,2009,0,0,0,0,0,0 +UPTURN,0,2009,0,0,0,0,0 +USURIOUS,2009,0,0,2009,0,0,0 +USURPING,2009,0,0,2009,0,0,0 +VAGUE,0,0,2012,0,0,0,0 +VAGUER,0,0,2012,0,0,0,0 +PLAINTIFFS,2009,0,0,2009,0,0,0 +PLEADING,2009,0,0,2009,0,0,0 +PLEASANT,0,2009,0,0,0,0,0 +PLED,2009,0,0,0,0,0,0 +PLEDGEES,0,0,0,2011,0,0,0 +PLEDGORS,0,0,0,2012,0,0,0 +COMPELLING,0,0,0,0,0,0,2011 +COMPLAINANT,0,0,0,2009,0,0,0 +COMPLAINS,2009,0,0,0,0,0,0 +COMMITS,0,0,0,0,0,0,2009 +LIKELIHOOD,0,0,2009,0,0,0,0 +LIMITING,0,0,0,0,0,0,2009 +RESTRAIN,0,0,0,0,0,0,2009 +RESTRAINT,0,0,0,0,0,0,2009 +RESTRICTING,0,0,0,0,0,0,2009 +RESTRICTIVELY,0,0,0,0,0,0,2009 +RESTRUCTURED,2009,0,0,0,0,0,0 +RETALIATE,2009,0,0,0,0,0,0 +RETALIATION,2009,0,0,0,0,0,0 +PRECAUTION,0,0,2009,0,0,0,0 +PRECIPITOUS,2009,0,0,0,0,0,0 +PRECLUDES,2009,0,0,0,0,0,2009 +PREDATORY,2009,0,0,0,0,0,0 +PREDECEASING,0,0,0,2009,0,0,0 +PREDICTING,0,0,2009,0,0,0,0 +PREDICTOR,0,0,2009,0,0,0,0 +PREEMINENT,0,2009,0,0,0,0,0 +PREJUDICES,2009,0,0,2009,0,0,0 +PRELIMINARY,0,0,2009,0,0,0,0 +PREMIERE,0,2009,0,0,0,0,0 +PRESTIGE,0,2009,0,0,0,0,0 +PRESUMED,0,0,2009,0,0,0,0 +PRESUMPTIONS,0,0,2009,0,0,0,0 +PREVENTED,0,0,0,0,0,0,2009 +PRIMA,0,0,0,2009,0,0,0 +PROBABILISTIC,0,0,2009,0,0,0,0 +PROBABLY,0,0,2009,0,0,0,0 +PROBATING,0,0,0,2009,0,0,0 +PROBATIONER,0,0,0,2009,0,0,0 +PROBLEMATIC,2009,0,0,0,0,0,0 +PROFICIENT,0,2009,0,0,0,0,0 +PROFITABLY,0,2009,0,0,0,0,0 +PROGRESSING,0,2009,0,0,0,0,0 +PROHIBITION,0,0,0,0,0,0,2009 +PROHIBITORY,0,0,0,0,0,0,2009 +PROLONGATIONS,2009,0,0,0,0,0,0 +PROMULGATE,0,0,0,2009,0,0,0 +PROMULGATION,0,0,0,2009,0,0,0 +PRONE,2009,0,0,0,0,0,0 +PROSECUTED,2009,0,0,2009,0,0,0 +PROSECUTIONS,2009,0,0,2009,0,0,0 +PROSPERED,0,2009,0,0,0,0,0 +PROSPERS,0,2009,0,0,0,0,0 +PROTESTERS,2009,0,0,0,0,0,0 +PROTESTS,2009,0,0,0,0,0,0 +PROVISOES,0,0,0,2009,0,0,0 +PROVOKES,2009,0,0,0,0,0,0 +PUNISHES,2009,0,0,0,0,0,0 +PUNITIVE,2009,0,0,0,0,0,0 +PURPORTING,2009,0,0,0,0,0,0 +QUESTIONABLY,2009,0,0,0,0,0,0 +QUIT,2009,0,0,0,0,0,0 +RACKETEER,2009,0,0,0,0,0,0 +RANDOMIZED,0,0,2009,0,0,0,0 +RANDOMNESS,0,0,2009,0,0,0,0 +RATIONALIZATION,2009,0,0,0,0,0,0 +RATIONALIZES,2009,0,0,0,0,0,0 +ABANDONMENT,2009,0,0,0,0,0,0 +ABDICATES,2009,0,0,0,0,0,0 +ABERRANT,2009,0,0,0,0,0,0 +ABETTING,2009,0,0,0,0,0,0 +ABIDING,0,0,0,0,0,0,2009 +ABNORMALITY,2009,0,0,0,0,0,0 +ABOLISHES,2009,0,0,0,0,0,0 +ABROGATED,2009,0,0,2009,0,0,0 +ABROGATIONS,2009,0,0,2009,0,0,0 +ABSENCE,2009,0,0,0,0,0,0 +ABSOLVED,0,0,0,2009,0,0,0 +ABUNDANT,0,2009,0,0,0,0,0 +ABUSING,2009,0,0,0,0,0,0 +ACCESSION,0,0,0,2009,0,0,0 +ACCIDENTALLY,2009,0,0,0,0,0,0 +ACCOMPLISHED,0,2009,0,0,0,0,0 +ACCOMPLISHMENTS,0,2009,0,0,0,0,0 +ACCUSED,2009,0,0,0,0,0,0 +ACHIEVED,0,2009,0,0,0,0,0 +ACHIEVING,0,2009,0,0,0,0,0 +ACQUIESCING,2009,0,0,0,0,0,0 +ACQUITS,2009,0,0,2009,0,0,0 +ACQUITTANCES,0,0,0,2011,0,0,0 +ADEQUATELY,0,2009,0,0,0,0,0 +ADJOURNMENT,0,0,0,2009,0,0,0 +ADJUDGED,0,0,0,2009,0,0,0 +ADJUDICATED,0,0,0,2009,0,0,0 +ADJUDICATIONS,0,0,0,2009,0,0,0 +ADJUDICATORY,0,0,0,2009,0,0,0 +ADMISSION,0,0,0,2009,0,0,0 +ADULTERATING,2009,0,0,0,0,0,0 +ADVANCEMENTS,0,2009,0,0,0,0,0 +ADVANTAGED,0,2009,0,0,0,0,0 +ADVERSARIAL,2009,0,0,0,0,0,0 +ADVERSELY,2009,0,0,0,0,0,0 +AFFIDAVITS,0,0,0,2009,0,0,0 +AFOREMENTIONED,0,0,0,2009,0,0,0 +AFTERMATHS,2009,0,0,0,0,0,0 +AGGRAVATES,2009,0,0,0,0,0,0 +AGGRIEVED,0,0,0,2009,0,0,0 +ALIENATED,2009,0,0,0,0,0,0 +ALIENATIONS,2009,0,0,0,0,0,0 +ALLEGED,2009,0,0,2009,0,0,0 +ALLIANCE,0,2009,0,0,0,0,0 +ALTERATIONS,0,0,2009,0,0,0,0 +AMBIGUOUS,0,0,2009,0,0,0,0 +AMENDED,0,0,0,2009,0,0,0 +AMENDS,0,0,0,2009,0,0,0 +ANNOYED,2009,0,0,0,0,0,0 +ANNULLED,2009,0,0,0,0,0,0 +ANNULS,2009,0,0,0,0,0,0 +ANOMALY,2009,0,2009,0,0,0,0 +ANTICIPATED,0,0,2009,0,0,0,0 +ANTICIPATIONS,0,0,2009,0,0,0,0 +ANYWISE,0,0,0,2009,0,0,0 +APPEALABLE,0,0,0,2009,0,0,0 +APPEAR,0,0,2009,0,0,0,0 +APPELLANT,0,0,0,2009,0,0,0 +APPOINTOR,0,0,0,2011,0,0,0 +APPROXIMATES,0,0,2009,0,0,0,0 +APPURTENANCE,0,0,0,2009,0,0,0 +ARBITRAL,0,0,0,2009,0,0,0 +ARBITRATE,0,0,0,2009,0,0,0 +ARBITRATION,0,0,0,2009,0,0,0 +ARBITRATOR,0,0,0,2009,0,0,0 +ARGUING,2009,0,0,0,0,0,0 +ARREARAGE,2009,0,0,2009,0,0,0 +ARRESTED,2009,0,0,0,0,0,0 +RETROCEDE,0,0,0,2011,0,0,0 +BOLSTERED,0,2009,0,0,0,0,0 +BONAFIDE,0,0,0,2009,0,0,0 +BOOSTED,0,2009,0,0,0,0,0 +BOUNDED,0,0,0,0,0,0,2009 +BOYCOTTS,2009,0,0,0,0,0,0 +BREACHING,2009,0,0,2009,0,0,0 +BREAKDOWN,2009,0,0,0,0,0,0 +BREAKTHROUGH,0,2009,0,0,0,0,0 +BRIBERIES,2009,0,0,0,0,0,0 +BRIDGE,-2020,0,0,0,0,0,0 +BURDENED,2009,0,0,0,0,0,0 +BURNED,2009,0,0,0,0,0,0 +CANCEL,2009,0,0,0,0,0,0 +CANCELLATIONS,2009,0,0,0,0,0,0 +CARELESS,2009,0,0,0,0,0,0 +CATASTROPHE,2009,0,0,0,0,0,0 +CAUTION,2009,0,0,0,0,0,0 +CAUTIONS,2009,0,0,0,0,0,0 +CEASE,2009,0,0,0,0,0,0 +CEDANT,0,0,0,2012,0,0,0 +CENSURES,2009,0,0,0,0,0,0 +CHALLENGE,2009,0,0,0,0,0,0 +CHARGEOFFS,2009,0,0,0,0,0,0 +CHOATE,0,0,0,2011,0,0,0 +CIRCUMVENTION,2009,0,0,0,0,0,0 +SHOCKED,2009,0,0,0,0,0,0 +SHORTFALLS,2009,0,0,0,0,0,0 +SHUTDOWN,2009,0,0,0,0,0,0 +SLANDER,2009,0,0,0,0,0,0 +REPUDIATED,2009,0,0,0,0,0,0 +REPUDIATIONS,2009,0,0,0,0,0,0 +REQUIRED,0,0,0,0,0,0,2009 +REQUIRING,0,0,0,0,0,0,2009 +RESCINDING,0,0,0,2009,0,0,0 +RESIGN,2009,0,0,0,0,0,0 +RESIGNING,2009,0,0,0,0,0,0 +RESTATED,2009,0,0,0,0,0,0 +RESTATING,2009,0,0,0,0,0,0 +NONUSURIOUS,0,0,0,2011,0,0,0 +NOTARIZATIONS,0,0,0,2009,0,0,0 +NOTARY,0,0,0,2009,0,0,0 +NUISANCES,2009,0,0,0,0,0,0 +NULLIFIES,2009,0,0,2009,0,0,0 +NULLITY,0,0,0,2009,0,0,0 +OBJECTIONABLE,2009,0,0,0,0,0,0 +OBLIGATED,0,0,0,0,0,0,2009 +OBLIGATIONS,0,0,0,0,0,0,2009 +OBLIGEE,0,0,0,2009,0,0,0 +OBLIGORS,0,0,0,2009,0,0,0 +OBSOLETE,2009,0,0,0,0,0,0 +OBSTRUCTED,2009,0,0,0,0,0,0 +OCCASIONALLY,0,0,2009,0,0,2009,0 +OFFENDED,2009,0,0,0,0,0,0 +OFFENDS,2009,0,0,0,0,0,0 +OFFEROR,0,0,0,2009,0,0,0 +OMIT,2009,0,0,0,0,0,0 +ONEROUS,2009,0,0,0,0,0,0 +OPPORTUNITY,0,2009,0,0,0,0,0 +OPPOSING,2009,0,0,0,0,0,0 +OPTIONEE,0,0,0,2009,0,0,0 +OUTAGES,2009,0,0,0,0,0,0 +OUTPERFORMED,0,2009,0,0,0,0,0 +OVERAGES,2009,0,0,0,0,0,0 +OVERBUILT,2009,0,0,0,0,0,0 +OVERCAPACITIES,2009,0,0,0,0,0,0 +OVERCHARGES,2009,0,0,0,0,0,0 +OVERCOMING,2009,0,0,0,0,0,0 +OVERESTIMATES,2009,0,0,0,0,0,0 +OVERLOAD,2009,0,0,0,0,0,0 +OVERLOOK,2009,0,0,0,0,0,0 +OVERPAID,2009,0,0,0,0,0,0 +OVERPRODUCES,2009,0,0,0,0,0,0 +OVERRULED,0,0,0,2009,0,0,0 +OVERRUNNING,2009,0,0,0,0,0,0 +OVERSHADOWING,2009,0,0,0,0,0,0 +OVERSTATEMENT,2009,0,0,0,0,0,0 +OVERSUPPLIED,2009,0,0,0,0,0,0 +OVERTLY,2009,0,0,0,0,0,0 +OVERTURNS,2009,0,0,0,0,0,0 +PANIC,2009,0,0,0,0,0,0 +NEARLY,0,0,2009,0,0,2009,0 +NECESSITATING,0,0,0,0,0,0,2009 +NEGLECT,2009,0,0,0,0,0,0 +NEGLECTS,2009,0,0,0,0,0,0 +NEGLIGENTLY,2009,0,0,0,0,0,0 +BARRIER,2009,0,0,0,0,0,0 +BELIEVE,0,0,2009,0,0,0,0 +IDLED,2009,0,0,0,0,0,0 +IGNORES,2009,0,0,0,0,0,0 +ILLEGALITIES,2009,0,0,0,0,0,0 +ILLICIT,2009,0,0,0,0,0,0 +IMBALANCE,2009,0,0,0,0,0,0 +IMMORAL,2009,0,0,0,0,0,0 +IMPAIRMENT,2009,0,0,0,0,0,2011 +IMPASSES,2009,0,0,0,0,0,0 +IMPEDIMENT,2009,0,0,0,0,0,0 +IMPERATIVE,2009,0,0,0,0,0,0 +IMPERMISSIBLE,2009,0,0,0,0,0,0 +IMPLICATES,2009,0,0,0,0,0,0 +IMPOSES,0,0,0,0,0,0,2009 +IMPOSSIBILITY,2009,0,0,0,0,0,0 +IMPOUNDING,2009,0,0,0,0,0,0 +IMPRACTICALITIES,2009,0,0,0,0,0,0 +IMPRECISIONS,0,0,2009,0,0,0,0 +IMPRESSING,0,2009,0,0,0,0,0 +IMPROBABILITY,0,0,2009,0,0,0,0 +IMPROPRIETIES,2009,0,0,0,0,0,0 +IMPROVEMENT,0,2009,0,0,0,0,0 +IMPRUDENT,2009,0,0,0,0,0,0 +INACCURACIES,2009,0,0,0,0,0,0 +INACTION,2009,0,0,0,0,0,0 +INACTIVATES,2009,0,0,0,0,0,0 +INACTIVITY,2009,0,0,0,0,0,0 +INADEQUATELY,2009,0,0,0,0,0,0 +INADVISABLE,2009,0,0,0,0,0,0 +INATTENTION,2009,0,0,0,0,0,0 +INCARCERATE,2009,0,0,2009,0,0,0 +INCARCERATION,2009,0,0,2009,0,0,0 +INCIDENCES,2009,0,0,0,0,0,0 +INCOMPATIBILITY,2009,0,0,0,0,0,0 +INCOMPETENT,2009,0,0,0,0,0,0 +INCOMPLETELY,2009,0,0,0,0,0,0 +INCONSISTENCY,2009,0,0,0,0,0,0 +INCONTESTABLE,0,0,0,2009,0,0,0 +INCORRECT,2009,0,0,0,0,0,0 +INCREDIBLY,0,2009,0,0,0,0,0 +INDEFEASIBLE,2009,0,0,0,0,0,0 +INDEFINITENESS,0,0,2009,0,0,0,0 +INDEMNIFIED,0,0,0,2009,0,0,0 +INDEMNITEE,0,0,0,2009,0,0,0 +INDEMNITORS,0,0,0,2009,0,0,0 +INDICT,2009,0,0,2009,0,0,0 +INDICTMENT,2009,0,0,2009,0,0,0 +INEFFECTIVELY,2009,0,0,0,0,0,0 +INEFFICIENT,2009,0,0,0,0,0,0 +INEQUITABLE,2009,0,0,0,0,0,0 +INEVITABLE,2009,0,0,0,0,0,0 +INEXPERIENCED,2009,0,0,0,0,0,0 +INFORCE,0,0,0,2009,0,0,0 +INFRINGE,2009,0,0,0,0,0,0 +INFRINGER,0,0,0,2009,0,0,0 +INHIBIT,0,0,0,0,0,0,2011 +INIMICAL,2009,0,0,0,0,0,0 +INJURE,2009,0,0,0,0,0,0 +INJURING,2009,0,0,0,0,0,0 +INNOVATED,0,2009,0,0,0,0,0 +INNOVATIONS,0,2009,0,0,0,0,0 +INNOVATORS,0,2009,0,0,0,0,0 +INSECURE,2009,0,0,0,0,0,0 +INSISTED,0,0,0,0,0,0,2009 +INSOFAR,0,0,0,2009,0,0,0 +INSPIRATION,0,2009,0,0,0,0,0 +INSUBORDINATION,2009,0,0,0,0,0,0 +INSURRECTION,2009,0,0,0,0,0,0 +INTEGRITY,0,2009,0,0,0,0,0 +INTERFERENCE,2009,0,0,0,0,0,0 +INTERLOCUTORY,0,0,0,2009,0,0,0 +INTERPOSE,0,0,0,2009,0,0,0 +INTERPOSITION,0,0,0,2009,0,0,0 +INTERROGATES,0,0,0,2009,0,0,0 +INTERROGATOR,0,0,0,2009,0,0,0 +INTERRUPT,2009,0,0,0,0,0,0 +INTERRUPTIONS,2009,0,0,0,0,0,0 +INTIMIDATION,2009,0,0,0,0,0,0 +SPORADICALLY,0,0,2009,0,0,0,0 +STABILIZE,0,2009,0,0,0,0,0 +STABLE,0,2009,0,0,0,0,0 +STAGNATED,2009,0,0,0,0,0,0 +STANDSTILL,2009,0,0,0,0,0,0 +STATUTORILY,0,0,0,2009,0,0,0 +STIPULATES,0,0,0,0,0,0,2009 +STOLEN,2009,0,0,0,0,0,0 +STOPPING,2009,0,0,0,0,0,0 +STRAINING,2009,0,0,0,0,0,0 +STRENGTHENED,0,2009,0,0,0,0,0 +STRESS,2009,0,0,0,0,0,0 +STRESSING,2009,0,0,0,0,0,0 +STRICTLY,0,0,0,0,0,0,2009 +STRONGEST,0,2009,0,0,0,0,0 +SUBDOCKET,0,0,0,2012,0,0,0 +SUBLEASEE,0,0,0,2011,0,0,0 +SUBLICENSOR,0,0,0,2011,0,0,0 +SUBPOENAED,2009,0,0,2009,0,0,0 +SUBSTANDARD,2009,0,0,0,0,0,0 +SUCCEEDED,0,2009,0,0,0,0,0 +SUCCESSES,0,2009,0,0,0,0,0 +SUDDENLY,0,0,2009,0,0,0,0 +SUFFER,2009,0,0,0,0,0,0 +SUGGEST,0,0,2009,0,0,2009,0 +SUING,2009,0,0,2009,0,0,0 +SUMMONSES,2009,0,0,2009,0,0,0 +SUPERSEDED,0,0,0,2009,0,0,0 +SURETY,0,0,0,2009,0,0,0 +SURPASSING,0,2009,0,0,0,0,0 +SUSPECTED,2009,0,0,0,0,0,0 +SUSPENDING,2009,0,0,0,0,0,0 +SUSPICION,2009,0,0,0,0,0,0 +REVISED,0,0,2009,0,0,0,0 +REVOKE,2009,0,0,0,0,0,0 +REVOLUTIONIZE,0,2009,0,0,0,0,0 +REWARD,0,2009,0,0,0,0,0 +RIDICULE,2009,0,0,0,0,0,0 +RISK,0,0,2009,0,0,0,0 +RISKINESS,0,0,2009,0,0,0,0 +ROUGHLY,0,0,2009,0,0,0,0 +SABOTAGE,2009,0,0,0,0,0,0 +SACRIFICIAL,2009,0,0,0,0,0,0 +SATISFACTORY,0,2009,0,0,0,0,0 +SATISFYING,0,2009,0,0,0,0,0 +SCRUTINIZED,2009,0,0,0,0,0,0 +SECRECY,-2020,0,0,0,0,0,0 +SEIZES,2009,0,0,0,0,0,0 +SENTENCED,2009,0,0,2009,0,0,0 +SERIOUSLY,2009,0,0,0,0,0,0 +SETTLEMENT,0,0,0,2009,0,0,0 +SEVERABLE,0,0,0,2009,0,0,0 +SEVERE,2009,0,0,0,0,0,0 +SEVERITY,2009,0,0,0,0,0,0 +ENTHUSIASTIC,0,2009,0,0,0,0,0 +ERODE,2009,0,0,0,0,0,0 +EROSION,2009,0,0,0,0,0,0 +ERRING,2009,0,0,0,0,0,0 +ERRORS,2009,0,0,0,0,0,0 +ESCALATES,2009,0,0,0,0,0,0 +ESCHEATMENT,0,0,0,2011,0,0,0 +ESCROWS,0,0,0,0,0,0,2009 +EVADES,2009,0,0,0,0,0,0 +EVASIVE,2009,0,0,0,0,0,0 +EVICTION,2009,0,0,0,0,0,0 +EVIDENTIARY,0,0,0,2009,0,0,0 +EXACERBATING,2009,0,0,0,0,0,0 +EXAGGERATED,2009,0,0,0,0,0,0 +EXCEEDANCE,0,0,0,2011,0,0,0 +EXCELLENT,0,2009,0,0,0,0,0 +EXCEPTIONALLY,0,2009,0,0,0,0,0 +EXCITED,0,2009,0,0,0,0,0 +EXCLUSIVELY,0,2009,0,0,0,0,0 +EXCULPATE,2009,0,0,2009,0,0,0 +EXCULPATION,2009,0,0,2009,0,0,0 +EXECUTORS,0,0,0,2009,0,0,0 +EXECUTRIXES,0,0,0,2009,0,0,0 +EXONERATES,2009,0,0,0,0,0,0 +EXPLOIT,2009,0,0,0,0,0,0 +EXPLOITED,2009,0,0,0,0,0,0 +EXPOSED,2009,0,0,0,0,0,0 +EXPOSURES,0,0,2009,0,0,0,0 +EXPROPRIATING,2009,0,0,0,0,0,0 +EXPULSIONS,2009,0,0,0,0,0,0 +EXTRAJUDICIAL,0,0,0,2014,0,0,0 +SOLVES,0,2009,0,0,0,0,0 +SOMEWHAT,0,0,2009,0,0,2009,0 +SPAMMING,2014,0,0,0,0,0,0 +TREMENDOUSLY,0,2009,0,0,0,0,0 +LOCKOUTS,2009,0,0,0,0,0,0 +LOSS,2009,0,0,0,0,0,0 +LOYAL,0,2009,0,0,0,0,0 +MALFEASANCE,2009,0,0,0,0,0,0 +MALFUNCTIONS,2009,0,0,0,0,0,0 +MALPRACTICE,2009,0,0,0,0,0,0 +MANDATES,0,0,0,0,0,0,2009 +MANIPULATE,2009,0,0,0,0,0,0 +MANIPULATION,2009,0,0,0,0,0,0 +MARKDOWNS,2009,0,0,0,0,0,0 +MEDIATED,0,0,0,2009,0,0,0 +MEDIATIONS,0,0,0,2009,0,0,0 +MIGHT,0,0,2009,0,0,2009,0 +MISAPPLIES,2009,0,0,0,0,0,0 +MISAPPROPRIATED,2009,0,0,0,0,0,0 +MISAPPROPRIATIONS,2009,0,0,0,0,0,0 +MISCALCULATES,2009,0,0,0,0,0,0 +MISCHARACTERIZATION,2014,0,0,0,0,0,0 +MISCLASSIFIED,2011,0,0,0,0,0,0 +MISDATED,2011,0,0,0,0,0,0 +MISFEASANCE,0,0,0,2009,0,0,0 +MISHANDLING,2009,0,0,0,0,0,0 +MISINFORMING,2009,0,0,0,0,0,0 +MISINTERPRETATIONS,2009,0,0,0,0,0,0 +MISJUDGE,2009,0,0,0,0,0,0 +MISJUDGMENT,2009,0,0,0,0,0,0 +MISLABELING,2009,0,0,0,0,0,0 +MISLEADING,2009,0,0,0,0,0,0 +MISMANAGE,2009,0,0,0,0,0,0 +MISMANAGING,2009,0,0,0,0,0,0 +MISMATCHING,2009,0,0,0,0,0,0 +MISPRICINGS,2014,0,0,0,0,0,0 +MISREPRESENTED,2009,0,0,0,0,0,0 +MISSED,2009,0,0,0,0,0,0 +WORRIES,2009,0,0,0,0,0,0 +WORSEN,2009,0,0,0,0,0,0 +WORST,2009,0,0,0,0,0,0 +TIGHTENING,2009,0,0,0,0,0,0 +TOLERATING,2009,0,0,0,0,0,0 +TORTIOUSLY,0,0,0,2009,0,0,0 +FINES,2009,0,0,0,0,0,0 +NONASSESSABLE,0,0,2009,0,0,0,0 +NONCANCELLABLE,0,0,0,0,0,0,2009 +NONCOMPLIANT,2009,0,0,0,0,0,0 +NONCONFORMITY,2009,0,0,0,0,0,0 +NONCONTRIBUTORY,0,0,0,2009,0,0,0 +NONFORFEITABILITY,0,0,0,2011,0,0,0 +NONGUARANTOR,0,0,0,2011,0,0,0 +DISCIPLINARY,2009,0,0,0,0,0,0 +DISCLAIMERS,2009,0,0,0,0,0,0 +DISCLOSED,2009,0,0,0,0,0,0 +DISCONTINUANCES,2009,0,0,0,0,0,0 +DISCONTINUED,2009,0,0,0,0,0,0 +DISCOURAGED,2009,0,0,0,0,0,0 +DISCREDITED,2009,0,0,0,0,0,0 +DISCREPANCY,2009,0,0,0,0,0,0 +SPECULATED,0,0,2009,0,0,0,0 +SPECULATIONS,0,0,2009,0,0,0,0 +TRAGICALLY,2009,0,0,0,0,0,0 +LEGISLATED,0,0,0,2009,0,0,0 +LEGISLATIONS,0,0,0,2009,0,0,0 +LEGISLATORS,0,0,0,2009,0,0,0 +LIBELED,0,0,0,2009,0,0,0 +LIE,2009,0,0,0,0,0,0 +COMPULSION,2009,0,0,0,0,0,2009 +CONCEDE,2009,0,0,0,0,0,0 +CONCEIVABLE,0,0,2009,0,0,2009,0 +CONCERNS,2009,0,0,0,0,0,0 +CONCLUSIVE,0,2009,0,0,0,0,0 +CONDEMNATIONS,2009,0,0,0,0,0,0 +CONDEMNS,2009,0,0,0,0,0,0 +CONDONED,2009,0,0,0,0,0,0 +CONFESSES,2009,0,0,0,0,0,0 +CONFINE,2009,0,0,0,0,0,2009 +CONFINES,2009,0,0,0,0,0,2009 +CONFISCATES,2009,0,0,0,0,0,0 +CONFISCATORY,0,0,0,2009,0,0,0 +CONFLICTS,2009,0,0,0,0,0,0 +CONFRONTATIONS,2009,0,0,0,0,0,0 +CONFUSE,2009,0,0,0,0,0,0 +CONFUSINGLY,2009,0,2009,0,0,0,0 +CONSENTING,0,0,0,2009,0,0,0 +CONSPIRACY,2009,0,0,0,0,0,0 +CONSPIRE,2009,0,0,0,0,0,0 +CONSTITUTION,0,0,0,2009,0,0,0 +CONSTITUTIONS,0,0,0,2009,0,0,0 +CONSTRAINING,0,0,0,0,0,0,2009 +CONSTRUCTIVE,0,2009,0,0,0,0,0 +CONSTRUES,0,0,0,2012,0,0,0 +CONTENDED,2009,0,0,0,0,0,0 +CONTENTIONS,2009,0,0,0,0,0,0 +CONTESTATION,0,0,0,2011,0,0,0 +CONTINGENCY,0,0,2009,0,0,0,0 +CONTRACT,0,0,0,2009,0,0,0 +CONTRACTIBLE,0,0,0,2009,0,0,0 +CONTRACTIONS,2009,0,0,0,0,0,0 +CONTRADICT,2009,0,0,0,0,0,0 +CONTRADICTIONS,2009,0,0,0,0,0,0 +CONTRAVENE,0,0,0,2009,0,0,0 +CONTRAVENTION,0,0,0,2009,0,0,0 +CONTROVERSY,2009,0,0,0,0,0,0 +CONVENIENS,0,0,0,2012,0,0,0 +CONVICTED,2009,0,0,2009,0,0,0 +CORRECTED,2009,0,0,0,0,0,0 +CORRECTS,2009,0,0,0,0,0,0 +CORRUPTION,2009,0,0,0,0,0,0 +COSTLY,2009,0,0,0,0,0,0 +COUNSELED,0,0,0,2009,0,0,0 +COUNTERCLAIMED,2009,0,0,0,0,0,0 +COUNTERFEITED,2009,0,0,0,0,0,0 +COUNTERFEITS,2009,0,0,0,0,0,0 +COUNTERSUED,0,0,0,2011,0,0,0 +COURTEOUS,0,2009,0,0,0,0,0 +COVENANTED,0,0,0,0,0,0,2011 +CREATIVELY,0,2009,0,0,0,0,0 +CRIMES,2009,0,0,2009,0,0,0 +CRIMINALIZING,0,0,0,2014,0,0,0 +CRISIS,2009,0,0,0,0,0,0 +CRITICISMS,2009,0,0,0,0,0,0 +CRITICIZING,2009,0,0,0,0,0,0 +CROSSROADS,0,0,2009,0,0,0,0 +CULPABLE,2009,0,0,0,0,0,0 +CURTAILED,2009,0,0,0,0,0,0 +CURTAILS,2009,0,0,0,0,0,0 +CYBERATTACK,2014,0,0,0,0,0,0 +CYBERCRIMES,2014,0,0,0,0,0,0 +TAINTS,2009,0,0,0,0,0,0 +TENSE,2009,0,0,0,0,0,0 +TERMINATE,2009,0,0,0,0,0,0 +TERMINATION,2009,0,0,0,0,0,0 +TESTIFY,2009,0,0,2009,0,0,0 +THENCEFORTH,0,0,0,2009,0,0,0 +THEREFROM,0,0,0,2009,0,0,0 +THEREON,0,0,0,2009,0,0,0 +THERETOFORE,0,0,0,2009,0,0,0 +THEREWITH,0,0,0,2009,0,0,0 +THREATENING,2009,0,0,0,0,0,0 +FACIE,0,0,0,2009,0,0,0 +FAILING,2009,0,0,0,0,0,0 +FAILURES,2009,0,0,0,0,0,0 +FALSIFICATION,2009,0,0,0,0,0,0 +FALSIFY,2009,0,0,0,0,0,0 +FATALITIES,2009,0,0,0,0,0,0 +FAULTED,2009,0,0,0,0,0,0 +FAVORABLY,0,2009,0,0,0,0,0 +FAVORITES,0,2009,0,0,0,0,0 +FELONIOUS,2009,0,0,2009,0,0,0 +DAMAGES,2009,0,0,0,0,0,0 +DANGER,2009,0,0,0,0,0,0 +DEADLOCK,2009,0,0,0,0,0,0 +DEADWEIGHT,2009,0,0,0,0,0,0 +DEBARRED,2009,0,0,0,0,0,0 +DECEIT,2009,0,0,0,0,0,0 +DECEIVED,2009,0,0,0,0,0,0 +DECEPTIONS,2009,0,0,0,0,0,0 +DECLINE,2009,0,0,0,0,0,0 +DECREE,0,0,0,2009,0,0,0 +DEFACE,2009,0,0,0,0,0,0 +DEFALCATIONS,0,0,0,2009,0,0,0 +DEFAME,2009,0,0,0,0,0,0 +DEFAULT,2009,0,0,0,0,0,0 +DEFEASANCE,0,0,0,2009,0,0,0 +DEFEASEMENT,0,0,0,2014,0,0,0 +DEFEATED,2009,0,0,0,0,0,0 +DEFECTIVE,2009,0,0,0,0,0,0 +DEFENDABLE,0,0,0,2014,0,0,0 +DEFENDING,2009,0,0,0,0,0,0 +DEFERENCE,0,0,0,2009,0,0,0 +DEFICIT,2009,0,0,0,0,0,0 +DEFRAUD,2009,0,0,0,0,0,0 +DEFUNCT,2009,0,0,0,0,0,0 +DEGRADED,2009,0,0,0,0,0,0 +DELAYED,2009,0,0,0,0,0,0 +DELEGATABLE,0,0,0,2011,0,0,0 +DELIBERATE,2009,0,0,0,0,0,0 +DELIGHTED,0,2009,0,0,0,0,0 +DELIGHTS,0,2009,0,0,0,0,0 +DELINQUENTLY,2009,0,0,0,0,0,0 +DELISTING,2009,0,0,0,0,0,0 +DEMISES,2009,0,0,0,0,0,0 +DEMOLISHES,2009,0,0,0,0,0,0 +DEMOTE,2009,0,0,0,0,0,0 +DEMOTION,2009,0,0,0,0,0,0 +DEMURRERS,0,0,0,2009,0,0,0 +DENIALS,2009,0,0,0,0,0,0 +DENIGRATED,2009,0,0,0,0,0,0 +DENY,2009,0,0,0,0,0,0 +DEPENDABLE,0,2009,0,0,0,0,0 +DEPENDED,0,0,2009,0,0,2009,0 +DEPENDENT,0,0,2009,0,0,0,2011 +DEPLETED,2009,0,0,0,0,0,0 +DEPLETIONS,2009,0,0,0,0,0,0 +DEPOSING,0,0,0,2009,0,0,0 +INVALID,2009,0,0,0,0,0,0 +INVALIDATING,2009,0,0,0,0,0,0 +INVENTED,0,2009,0,0,0,0,0 +INVENTIVE,0,2009,0,0,0,0,0 +INVESTIGATE,2009,0,0,0,0,0,0 +INVESTIGATION,2009,0,0,0,0,0,0 +IRRECONCILABLE,2009,0,0,0,0,0,0 +IRREGULAR,2009,0,0,0,0,0,0 +IRREPARABLE,2009,0,0,0,0,0,0 +IRREVOCABLE,0,0,0,2009,0,0,2009 +JOINDER,0,0,0,2009,0,0,0 +JUDICIARY,0,0,0,2009,0,0,0 +JURISDICTIONAL,0,0,0,2009,0,0,0 +JURIST,0,0,0,2009,0,0,0 +JURY,0,0,0,2009,0,0,0 +JUSTIFIABLE,2009,0,0,0,0,0,0 +DILIGENTLY,0,2009,0,0,0,0,0 +DIMINISHING,2009,0,0,0,0,0,0 +DISADVANTAGE,2009,0,0,0,0,0,0 +DISAFFILIATION,2009,0,0,2009,0,0,0 +DISAFFIRMS,0,0,0,2011,0,0,0 +DISAGREEING,2009,0,0,0,0,0,0 +DISALLOW,2009,0,0,0,0,0,0 +DISALLOWING,2009,0,0,0,0,0,0 +DISAPPEARANCES,2009,0,0,0,0,0,0 +DISAPPOINT,2009,0,0,0,0,0,0 +DISAPPOINTMENT,2009,0,0,0,0,0,0 +DISAPPROVALS,2009,0,0,0,0,0,0 +DISAPPROVING,2009,0,0,0,0,0,0 +DISASSOCIATIONS,2009,0,0,0,0,0,0 +DISASTROUSLY,2009,0,0,0,0,0,0 +DISAVOWING,2009,0,0,0,0,0,0 +GREATER,0,-2020,0,0,0,0,0 +GRIEVANCE,2009,0,0,0,0,0,0 +GUILTY,2009,0,0,0,0,0,0 +HAMPERED,2009,0,0,0,0,0,0 +HAPPILY,0,2009,0,0,0,0,0 +HARASSED,2009,0,0,0,0,0,0 +HARDSHIPS,2009,0,0,0,0,0,0 +HARMFULLY,2009,0,0,0,0,0,0 +HARSHER,2009,0,0,0,0,0,0 +HAZARD,2009,0,0,0,0,0,0 +NONJUDICIALLY,0,0,0,2011,0,0,0 +NONPERFORMANCE,2009,0,0,0,0,0,0 +HURT,2009,0,0,0,0,0,0 +DISFAVORED,2009,0,0,0,0,0,0 +DISGORGED,2009,0,0,0,0,0,0 +COMPLICATING,2009,0,0,0,0,0,0 +COMPLIMENTARY,0,2009,0,0,0,0,0 +COMPLY,0,0,0,0,0,0,2009 +MISSTATED,2009,0,0,0,0,0,0 +MISSTATING,2009,0,0,0,0,0,0 +MISTAKEN,2009,0,0,0,0,0,0 +MISTRIAL,2009,0,0,2009,0,0,0 +MISUNDERSTANDINGS,2009,0,0,0,0,0,0 +MISUSES,2009,0,0,0,0,0,0 +MONOPOLIZATION,2009,0,0,0,0,0,0 +MONOPOLIZING,2009,0,0,0,0,0,0 +MORATORIUMS,2009,0,0,0,0,0,0 +MOTIONS,0,0,0,2009,0,0,0 +SLOWDOWN,2009,0,0,0,0,0,0 +SLOWEST,2009,0,0,0,0,0,0 +SLUGGISH,2009,0,0,0,0,0,0 +SMOOTHING,0,2009,0,0,0,0,0 +DEPRESS,2009,0,0,0,0,0,0 +DEPRIVATION,2009,0,0,0,0,0,0 +DEPRIVING,2009,0,0,0,0,0,0 +DEROGATED,0,0,0,2009,0,0,0 +DEROGATIONS,0,0,0,2009,0,0,0 +DESIRED,0,2009,0,0,0,0,0 +DESTABILIZE,2009,0,0,0,0,0,0 +DESTROY,2009,0,0,0,0,0,0 +DESTRUCTION,2009,0,0,0,0,0,0 +DETAINER,0,0,0,2009,0,0,0 +DETERIORATE,2009,0,0,0,0,0,0 +DETERIORATION,2009,0,0,0,0,0,0 +DETERRENCES,2009,0,0,0,0,0,0 +DETERS,2009,0,0,0,0,0,0 +DETRIMENT,2009,0,0,0,0,0,0 +DEVALUE,2009,0,0,0,0,0,0 +DEVASTATE,2009,0,0,0,0,0,0 +DEVIATE,2009,0,2009,0,0,0,0 +DEVIATION,2009,0,2009,0,0,0,0 +DEVOLVED,2009,0,0,0,0,0,0 +DICTATED,0,0,0,0,0,0,2009 +DIFFERED,0,0,2009,0,0,0,0 +DIFFICULTIES,2009,0,0,0,0,0,0 +ASCENDANCY,0,0,0,2009,0,0,0 +ASSAULTED,2009,0,0,0,0,0,0 +ASSERTIONS,2009,0,0,0,0,0,0 +ASSUME,0,0,2009,0,0,0,0 +ASSUMPTION,0,0,2009,0,0,0,0 +ASSURES,0,2009,0,0,0,0,0 +ATTAINING,0,2009,0,0,0,0,0 +ATTEST,0,0,0,2009,0,0,0 +ATTESTING,0,0,0,2009,0,0,0 +ATTORNMENT,0,0,0,2009,0,0,0 +ATTRITION,2009,0,0,0,0,0,0 +BAIL,2009,0,0,2009,0,0,0 +BAILIFF,0,0,0,2009,0,0,0 +BALK,2009,0,0,0,0,0,0 +BANKRUPTCY,2009,0,0,0,0,0,0 +BANS,2009,0,0,0,0,0,0 +CLAIMABLE,0,0,0,2009,0,0,0 +CLAIMING,2009,0,0,0,0,0,0 +CLAWBACK,2009,0,0,0,0,0,0 +CLOSEOUT,2009,0,0,0,0,0,0 +CLOSURE,2009,0,0,0,0,0,0 +CODICIL,0,0,0,2009,0,0,0 +CODIFIED,0,0,0,2009,0,0,0 +COERCE,2009,0,0,0,0,0,0 +COERCION,2009,0,0,0,0,0,0 +COLLABORATING,0,2009,0,0,0,0,0 +COLLABORATOR,0,2009,0,0,0,0,0 +COLLAPSES,2009,0,0,0,0,0,0 +COLLUDE,2009,0,0,0,0,0,0 +COLLUSION,2009,0,0,2009,0,0,0 +POSING,2009,0,0,0,0,0,0 +POSSIBILITIES,0,0,2009,0,0,0,0 +POSTCLOSING,0,0,0,2011,0,0,0 +POSTPONE,2009,0,0,0,0,0,0 +POSTPONES,2009,0,0,0,0,0,0 +WRITEOFF,2009,0,0,0,0,0,0 +WRONGDOING,2009,0,0,0,0,0,0 +WRONGLY,2009,0,0,0,0,0,0 +HONORED,0,2009,0,0,0,0,0 +HOSTILITY,2009,0,0,0,0,0,0 +REASSESSED,0,0,2009,0,0,0,0 +REASSESSMENTS,2009,0,2009,0,0,0,0 +REASSIGNMENT,2009,0,0,0,0,0,0 +REBOUNDED,0,2009,0,0,0,0,0 +REBUTTABLE,0,0,0,2009,0,0,0 +REBUTTED,0,0,0,2009,0,0,0 +RECALCULATES,0,0,2009,0,0,0,0 +RECALL,2009,0,0,0,0,0,0 +RECEPTIVE,0,2009,0,0,0,0,0 +RECKLESS,2009,0,0,0,0,0,0 +RECONSIDERED,0,0,2009,0,0,0,0 +RECOUPABLE,0,0,0,2009,0,0,0 +RECOURSES,0,0,0,2009,0,0,0 +RECUSE,0,0,0,2011,0,0,0 +REDACT,2009,0,0,2009,0,0,0 +REDACTIONS,2009,0,0,2009,0,0,0 +REDRESS,2009,0,0,0,0,0,0 +REEXAMINATION,0,0,2009,0,0,0,0 +REFERENDUM,0,0,0,2009,0,0,0 +REFILES,0,0,0,2009,0,0,0 +REFRAINS,0,0,0,0,0,0,2009 +REFUSED,2009,0,0,0,0,0,0 +REGAINED,0,2009,0,0,0,0,0 +REGULATES,0,0,0,2009,0,0,0 +REGULATIVE,0,0,0,2009,0,0,0 +REHEAR,0,0,0,2009,0,0,0 +VARIABLE,0,0,2009,0,0,0,0 +VARIANCES,0,0,2009,0,0,0,0 +VARIATIONS,0,0,2009,0,0,0,0 +VARYING,0,0,2009,0,0,0,0 +VERDICTS,2009,0,0,2009,0,0,0 +VIATICAL,0,0,0,2011,0,0,0 +VIOLATE,2009,0,0,0,0,0,0 +VIOLATION,2009,0,0,0,0,0,0 +VIOLATORS,2009,0,0,0,0,0,0 +VITIATE,2009,0,0,0,0,0,0 +VITIATION,2009,0,0,0,0,0,0 +VOLATILE,2009,0,2009,0,0,0,0 +VULNERABILITY,2009,0,0,0,0,0,0 +WARNED,2009,0,0,0,0,0,0 +WARRANTEES,0,0,0,2011,0,0,0 +WASTING,2009,0,0,0,0,0,0 +WEAKENING,2009,0,0,0,0,0,0 +WEAKLY,2009,0,0,0,0,0,0 +DISHONESTY,2009,0,0,0,0,0,0 +DISHONORED,2009,0,0,0,0,0,0 +DISLOYAL,2009,0,0,0,0,0,0 +DISMALLY,2009,0,0,0,0,0,0 +DISMISSED,2009,0,0,0,0,0,0 +DISPARAGE,2009,0,0,0,0,0,0 +DISPARAGES,2009,0,0,0,0,0,0 +DISPARITY,2009,0,0,0,0,0,0 +DISPLACEMENTS,2009,0,0,0,0,0,0 +DISPOSITIVE,0,0,0,2009,0,0,0 +DISPOSSESSING,2009,0,0,0,0,0,0 +DISPROPORTIONAL,2009,0,0,0,0,0,0 +DISPUTED,2009,0,0,0,0,0,0 +DISQUALIFICATIONS,2009,0,0,0,0,0,0 +DISQUALIFYING,2009,0,0,0,0,0,0 +DISREGARDS,2009,0,0,0,0,0,0 +DISRUPTED,2009,0,0,0,0,0,0 +DISRUPTIVE,2009,0,0,0,0,0,0 +DISSENT,2009,0,0,0,0,0,0 +DISSENTING,2009,0,0,0,0,0,0 +DISSOLUTION,2009,0,0,0,0,0,0 +DISTINCTIVE,0,2009,0,0,0,0,0 +DISTORTED,2009,0,0,0,0,0,0 +DISTORTS,2009,0,0,0,0,0,0 +DISTRACTION,2009,0,0,0,0,0,0 +DISTRESS,2009,0,0,0,0,0,0 +DISTURB,2009,0,0,0,0,0,0 +DISTURBING,2009,0,0,0,0,0,0 +DIVERTED,2009,0,0,0,0,0,0 +DIVESTED,2009,0,0,0,0,0,0 +DIVESTMENT,2009,0,0,0,0,0,0 +DIVORCED,2009,0,0,0,0,0,0 +DIVULGING,2009,0,0,0,0,0,0 +DOCKETS,0,0,0,2009,0,0,0 +DOUBTFUL,2009,0,2009,0,0,0,0 +DOWNGRADES,2009,0,0,0,0,0,0 +DOWNSIZES,2009,0,0,0,0,0,0 +DOWNTIMES,2009,0,0,0,0,0,0 +DOWNWARDS,2009,0,0,0,0,0,0 +DRAWBACK,2009,0,0,0,0,0,0 +DROUGHT,2009,0,0,0,0,0,0 +DYSFUNCTION,2009,0,0,0,0,0,0 +EARMARKED,0,0,0,0,0,0,2009 +EASILY,0,2009,0,0,0,0,0 +EFFICIENCIES,0,2009,0,0,0,0,0 +EGREGIOUS,2009,0,0,0,0,0,0 +EMBARGOED,2009,0,0,0,0,0,0 +EMBARRASSED,2009,0,0,0,0,0,0 +EMBARRASSMENTS,2009,0,0,0,0,0,0 +EMBEZZLEMENTS,2009,0,0,0,0,0,0 +EMPOWER,0,2009,0,0,0,0,0 +ENABLE,0,2009,0,0,0,0,0 +ENCOURAGED,0,2009,0,0,0,0,0 +ENCROACH,2009,0,0,0,0,0,0 +ENCROACHMENT,2009,0,0,0,0,0,0 +ENCUMBERING,2009,0,0,2009,0,0,2009 +ENCUMBRANCERS,0,0,0,2011,0,0,0 +ENDANGERING,2009,0,0,0,0,0,0 +ENFORCEABILITY,0,0,0,2009,0,0,0 +ENHANCED,0,2009,0,0,0,0,0 +ENHANCING,0,2009,0,0,0,0,0 +ENJOINS,2009,0,0,0,0,0,0 +ENJOYED,0,2009,0,0,0,0,0 +ENTAIL,0,0,0,0,0,0,2009 +PECUNIARILY,0,0,0,2009,0,0,0 +PENALIZING,2009,0,0,0,0,0,0 +PERFECT,0,2009,0,0,0,0,0 +PERHAPS,0,0,2009,0,0,2009,0 +PERMISSIBLE,0,0,0,0,0,0,2009 +PERMITTEE,0,0,0,2011,0,0,0 +PERPETRATED,2009,0,0,2009,0,0,0 +PERSIST,2009,0,0,0,0,0,0 +PERSISTENTLY,2009,0,0,0,0,0,0 +PERVASIVE,2009,0,0,0,0,0,0 +PETITIONED,0,0,0,2009,0,0,0 +PETITIONS,0,0,0,2009,0,0,0 +PICKETING,2009,0,0,0,0,0,0 +HENCEFORWARD,0,0,0,2009,0,0,0 +HEREFOR,0,0,0,2009,0,0,0 +HEREINABOVE,0,0,0,2009,0,0,0 +HEREOF,0,0,0,2009,0,0,0 +HEREUNDER,0,0,0,2009,0,0,0 +HEREWITHIN,0,0,0,2014,0,0,0 +HINDERED,2009,0,0,0,0,0,0 +HINDRANCES,2009,0,0,0,0,0,0 +WHENSOEVER,0,0,0,2009,0,0,0 +WHEREBY,0,0,0,2009,0,0,0 +WHEREON,0,0,0,2009,0,0,0 +WHEREWITH,0,0,0,2009,0,0,0 +WHOSOEVER,0,0,0,2009,0,0,0 +WILLFULLY,2009,0,0,2009,0,0,0 +WINNERS,0,2009,0,0,0,0,0 +FLUCTUATING,0,0,2009,0,0,0,0 +FORBEAR,0,0,0,2009,0,0,0 +FORBEARS,0,0,0,2009,0,0,0 +FORBIDS,2009,0,0,0,0,0,2009 +FOREBEAR,0,0,0,2009,0,0,0 +FORECLOSED,2009,0,0,0,0,0,0 +FORECLOSURES,2009,0,0,0,0,0,0 +FORESTALL,2009,0,0,0,0,0,0 +FORFEIT,2009,0,0,0,0,0,0 +FORFEITING,2009,0,0,0,0,0,0 +FORGERS,2009,0,0,0,0,0,0 +FRAUD,2009,0,0,0,0,0,0 +FRAUDULENTLY,2009,0,0,0,0,0,0 +FRUSTRATE,2009,0,0,0,0,0,0 +FRUSTRATINGLY,2009,0,0,0,0,0,0 +FUGITIVES,2009,0,0,2009,0,0,0 +GAINING,0,2009,0,0,0,0,0 +GRANTORS,0,0,0,2009,0,0,0 +DISGRACE,2009,0,0,0,0,0,0 +LITIGANTS,2009,0,0,2009,0,0,0 +LITIGATING,2009,0,0,2009,0,0,0 +LITIGATORS,0,0,0,2009,0,0,0 +LACK,2009,0,0,0,0,0,0 +LACKS,2009,0,0,0,0,0,0 +LAGS,2009,0,0,0,0,0,0 +LAPSING,2009,0,0,0,0,0,0 +LAWFUL,0,0,0,2009,0,0,0 +LAWMAKING,0,0,0,2009,0,0,0 +LAWYER,0,0,0,2009,0,0,0 +LEADERSHIP,0,2009,0,0,0,0,0 +LEGALITY,0,0,0,2009,0,0,0 +LEGALIZED,0,0,0,2009,0,0,0 +LEGALS,0,0,0,2009,0,0,0 +FLAW,2009,0,0,0,0,0,0 +NONPRODUCTIVE,2009,0,0,0,0,0,0 +LIQUIDATED,2009,0,0,0,0,0,0 +LIQUIDATIONS,2009,0,0,0,0,0,0 +BENEFICIATED,0,0,0,2014,0,0,0 +BENEFITING,0,2009,0,0,0,0,0 +BETTER,0,2009,0,0,0,0,0 +POPULARITY,0,2009,0,0,0,0,0 +REINTERPRET,0,0,2009,0,0,0,0 +REINTERPRETING,0,0,2009,0,0,0,0 +REJECTING,2009,0,0,0,0,0,0 +RELEASEES,0,0,0,2011,0,0,0 +RELINQUISHING,2009,0,0,0,0,0,0 +RELUCTANT,2009,0,0,0,0,0,0 +REMANDS,0,0,0,2009,0,0,0 +REMEDIATION,0,0,0,2009,0,0,0 +RENEGOTIATE,2009,0,0,0,0,0,0 +RENEGOTIATION,2009,0,0,0,0,0,0 +RENOUNCEMENT,2009,0,0,0,0,0,0 +REPARATION,2009,0,0,0,0,0,0 +REPOSSESSED,2009,0,0,0,0,0,0 +REPOSSESSIONS,2009,0,0,0,0,0,0 +TROUBLE,2009,0,0,0,0,0,0 +TURMOIL,2009,0,0,0,0,0,0 +UNACCOUNTED,2009,0,0,0,0,0,0 +UNAPPEALABLE,0,0,0,2009,0,0,0 +UNAUTHORIZED,2009,0,0,0,0,0,0 +UNAVOIDABLY,2009,0,0,0,0,0,0 +UNCERTAINTIES,0,0,2009,0,0,0,0 +UNCOLLECTED,2009,0,0,0,0,0,0 +UNCOMPETITIVE,2009,0,0,0,0,0,0 +UNCONSCIONABLE,2009,0,0,0,0,0,0 +UNCONSTITUTIONALLY,0,0,0,2009,0,0,0 +UNCONTROLLED,2009,0,0,0,0,0,0 +UNCOVERING,2009,0,0,0,0,0,0 +UNDEFINED,0,0,2009,0,0,0,0 +UNDERCUT,2009,0,0,0,0,0,0 +UNDERESTIMATED,2009,0,0,0,0,0,0 +UNDERFUNDED,2009,0,0,0,0,0,0 +UNDERMINES,2009,0,0,0,0,0,0 +UNDERPAYMENTS,2009,0,0,0,0,0,0 +UNDERPERFORMED,2011,0,0,0,0,0,0 +UNDERPRODUCTION,2009,0,0,0,0,0,0 +UNDERSTATEMENT,2009,0,0,0,0,0,0 +UNDERUTILIZATION,2009,0,0,0,0,0,0 +UNDESIRED,2009,0,0,0,0,0,0 +UNDETERMINED,2009,0,2009,0,0,0,0 +UNDOCUMENTED,2009,0,2009,0,0,0,0 +UNECONOMIC,2009,0,0,0,0,0,0 +UNEMPLOYMENT,2009,0,0,0,0,0,0 +UNENFORCEABLE,0,0,0,2009,0,0,0 +UNETHICALLY,2009,0,0,0,0,0,0 +UNFAIR,2009,0,0,0,0,0,0 +UNFAVORABILITY,2014,0,0,0,0,0,0 +UNFEASIBLE,2009,0,0,0,0,0,0 +UNFORESEEABLE,2009,0,0,0,0,0,0 +UNFORTUNATELY,2009,0,0,0,0,0,0 +UNFUNDED,2009,0,0,0,0,0,0 +UNIDENTIFIED,0,0,2009,0,0,0,0 +UNINTENTIONALLY,2009,0,0,0,0,0,0 +UNJUSTIFIED,2009,0,0,0,0,0,0 +UNKNOWN,0,0,2009,0,0,0,0 +UNLAWFULNESS,0,0,0,2009,0,0,0 +UNMATCHED,0,2009,0,0,0,0,0 +UNNECESSARY,2009,0,0,0,0,0,0 +UNOCCUPIED,2009,0,0,0,0,0,0 +UNPLANNED,2009,0,2009,0,0,0,0 +UNPREDICTABLY,2009,0,2009,0,0,0,0 +UNPROFITABLE,2009,0,0,0,0,0,0 +UNQUANTIFIABLE,0,0,2011,0,0,0,0 +UNREASONABLENESS,2009,0,0,0,0,0,0 +UNRECOVERABLE,2009,0,0,0,0,0,0 +UNREMEDIATED,0,0,0,2011,0,0,0 +UNREST,2009,0,0,0,0,0,0 +UNSATISFACTORY,2009,0,0,0,0,0,0 +UNSEASONABLE,0,0,2009,0,0,0,0 +UNSOLD,2009,0,0,0,0,0,0 +UNSTABILIZED,2014,0,0,0,0,0,0 +UNSUCCESSFUL,2009,0,0,0,0,0,0 +UNSUITABLY,2009,0,0,0,0,0,0 +UNSUSPECTED,2009,0,0,0,0,0,0 +UNTESTED,0,0,2009,0,0,0,0 +UNTRUTH,2009,0,0,0,0,0,0 +UNTRUTHS,2009,0,0,0,0,0,0 +UNWANTED,2009,0,0,0,0,0,0 +UNWILLINGNESS,2009,0,0,0,0,0,0 +UPTURNS,0,2009,0,0,0,0,0 +USURP,2009,0,0,2009,0,0,0 +USURPS,2009,0,0,2009,0,0,0 +VAGUELY,0,0,2012,0,0,0,0 +VAGUEST,0,0,2012,0,0,0,0 +PLEA,2009,0,0,0,0,0,0 +PLEADINGS,2009,0,0,2009,0,0,0 +PLEASANTLY,0,2009,0,0,0,0,0 +PLEDGE,0,0,0,0,0,0,2009 +PLEDGES,0,0,0,0,0,0,2009 +PLENTIFUL,0,2009,0,0,0,0,0 +COMPELS,0,0,0,0,0,0,2009 +COMPLAINANTS,0,0,0,2009,0,0,0 +COMPLAINT,2009,0,0,0,0,0,0 +COMMIT,0,0,0,0,0,0,2009 +COMMITTED,0,0,0,0,0,0,2009 +LIMIT,0,0,0,0,0,0,2009 +LIMITS,0,0,0,0,0,0,2009 +RESTRAINED,0,0,0,0,0,0,2009 +RESTRAINTS,0,0,0,0,0,0,2009 +RESTRICTION,0,0,0,0,0,0,2009 +RESTRICTIVENESS,0,0,0,0,0,0,2009 +RESTRUCTURES,2009,0,0,0,0,0,0 +RETALIATED,2009,0,0,0,0,0,0 +RETALIATIONS,2009,0,0,0,0,0,0 +PRECAUTIONARY,0,0,2009,0,0,0,0 +PRECIPITOUSLY,2009,0,0,0,0,0,0 +PRECLUDING,2009,0,0,0,0,0,2009 +PREDECEASE,0,0,0,2009,0,0,0 +PREDICT,0,0,2009,0,0,0,0 +PREDICTION,0,0,2009,0,0,0,0 +PREDICTORS,0,0,2009,0,0,0,0 +PREHEARING,0,0,0,2011,0,0,0 +PREJUDICIAL,2009,0,0,2009,0,0,0 +PREMATURE,2009,0,0,0,0,0,0 +PREPETITION,0,0,0,2009,0,0,0 +PRESTIGIOUS,0,2009,0,0,0,0,0 +PRESUMES,0,0,2009,0,0,0,0 +PRESUMPTIVELY,0,0,0,2009,0,0,0 +PREVENTING,2009,0,0,0,0,0,2009 +PRIVITY,0,0,0,2011,0,0,0 +PROBABILITIES,0,0,2009,0,0,0,0 +PROBATE,0,0,0,2009,0,0,0 +PROBATION,0,0,0,2009,0,0,0 +PROBATIONERS,0,0,0,2009,0,0,0 +PROBLEMATICAL,2009,0,0,0,0,0,0 +PROFICIENTLY,0,2009,0,0,0,0,0 +PROGRESS,0,2009,0,0,0,0,0 +PROHIBIT,0,0,0,0,0,0,2009 +PROHIBITIONS,0,0,0,0,0,0,2009 +PROHIBITS,0,0,0,0,0,0,2009 +PROLONGED,2009,0,0,0,0,0,0 +PROMULGATED,0,0,0,2009,0,0,0 +PROMULGATIONS,0,0,0,2009,0,0,0 +PRORATA,0,0,0,2009,0,0,0 +PROSECUTES,2009,0,0,2009,0,0,0 +PROSECUTOR,0,0,0,2009,0,0,0 +PROSPERING,0,2009,0,0,0,0,0 +PROTEST,2009,0,0,0,0,0,0 +PROTESTING,2009,0,0,0,0,0,0 +PROTRACTED,2009,0,0,0,0,0,0 +PROVISOS,0,0,0,2009,0,0,0 +PROVOKING,2009,0,0,0,0,0,0 +PUNISHING,2009,0,0,0,0,0,0 +PURPORT,2009,0,0,0,0,0,0 +PURPORTS,2009,0,0,0,0,0,0 +QUESTIONED,2009,0,0,0,0,0,0 +QUITCLAIM,0,0,0,2009,0,0,0 +RACKETEERING,2009,0,0,0,0,0,0 +RANDOMIZES,0,0,2009,0,0,0,0 +RATA,0,0,0,2009,0,0,0 +RATIONALIZATIONS,2009,0,0,0,0,0,0 +RATIONALIZING,2009,0,0,0,0,0,0 +ABANDON,2009,0,0,0,0,0,0 +ABANDONMENTS,2009,0,0,0,0,0,0 +ABDICATING,2009,0,0,0,0,0,0 +ABERRATION,2009,0,0,0,0,0,0 +ABEYANCE,0,0,2009,0,0,0,0 +ABLE,0,2009,0,0,0,0,0 +ABNORMALLY,2009,0,0,0,0,0,0 +ABOLISHING,2009,0,0,0,0,0,0 +ABROGATES,2009,0,0,2009,0,0,0 +ABRUPT,2009,0,0,0,0,0,0 +ABSENCES,2009,0,0,0,0,0,0 +ABSOLVES,0,0,0,2009,0,0,0 +ABUSE,2009,0,0,0,0,0,0 +ABUSIVE,2009,0,0,0,0,0,0 +ACCESSIONS,0,0,0,2009,0,0,0 +ACCIDENTS,2009,0,0,0,0,0,0 +ACCOMPLISHES,0,2009,0,0,0,0,0 +ACCUSATION,2009,0,0,0,0,0,0 +ACCUSES,2009,0,0,0,0,0,0 +ACHIEVEMENT,0,2009,0,0,0,0,0 +ACQUIESCE,2009,0,0,0,0,0,0 +ACQUIREES,0,0,0,2011,0,0,0 +ACQUITTAL,2009,0,0,2009,0,0,0 +ACQUITTED,2009,0,0,2009,0,0,0 +ADJOURN,0,0,0,2009,0,0,0 +ADJOURNMENTS,0,0,0,2009,0,0,0 +ADJUDGES,0,0,0,2009,0,0,0 +ADJUDICATES,0,0,0,2009,0,0,0 +ADJUDICATIVE,0,0,0,2009,0,0,0 +ADMISSIBILITY,0,0,0,2009,0,0,0 +ADMISSIONS,0,0,0,2009,0,0,0 +ADULTERATION,2009,0,0,0,0,0,0 +ADVANCES,0,2009,0,0,0,0,0 +ADVANTAGEOUS,0,2009,0,0,0,0,0 +ADVERSARIES,2009,0,0,0,0,0,0 +ADVERSITIES,2009,0,0,0,0,0,0 +AFFIRMANCE,0,0,0,2011,0,0,0 +AFORESAID,0,0,0,2009,0,0,0 +AGAINST,2009,0,0,0,0,0,0 +AGGRAVATING,2009,0,0,0,0,0,0 +ALERTED,2009,0,0,0,0,0,0 +ALIENATES,2009,0,0,0,0,0,0 +ALLEGATION,2009,0,0,2009,0,0,0 +ALLEGEDLY,2009,0,0,2009,0,0,0 +ALLIANCES,0,2009,0,0,0,0,0 +ALWAYS,0,0,0,0,2009,0,0 +AMEND,0,0,0,2009,0,0,0 +AMENDING,0,0,0,2009,0,0,0 +ANNOY,2009,0,0,0,0,0,0 +ANNOYING,2009,0,0,0,0,0,0 +ANNULLING,2009,0,0,0,0,0,0 +ANOMALIES,2009,0,2009,0,0,0,0 +ANTECEDENT,0,0,0,2009,0,0,0 +ANTICIPATES,0,0,2009,0,0,0,0 +ANTICOMPETITIVE,2009,0,0,0,0,0,0 +APPARENT,0,0,2009,0,0,0,0 +APPEALED,0,0,0,2009,0,0,0 +APPEARED,0,0,2009,0,0,2009,0 +APPELLANTS,0,0,0,2009,0,0,0 +APPROXIMATE,0,0,2009,0,0,0,0 +APPROXIMATING,0,0,2009,0,0,0,0 +APPURTENANCES,0,0,0,2009,0,0,0 +ARBITRARILY,0,0,2009,0,0,0,0 +ARBITRATED,0,0,0,2009,0,0,0 +ARBITRATIONAL,0,0,0,2011,0,0,0 +ARBITRATORS,0,0,0,2009,0,0,0 +ARGUMENT,2009,0,0,0,0,0,0 +ARREARAGES,2009,0,0,2009,0,0,0 +ARRESTS,2009,0,0,0,0,0,0 +RETROCEDED,0,0,0,2011,0,0,0 +BOLSTERING,0,2009,0,0,0,0,0 +BOOM,0,2009,0,0,0,0,0 +BOTTLENECK,2009,0,0,0,0,0,0 +BOYCOTT,2009,0,0,0,0,0,0 +BREACH,2009,0,0,2009,0,0,0 +BREAK,2009,0,0,0,0,0,0 +BREAKDOWNS,2009,0,0,0,0,0,0 +BREAKTHROUGHS,0,2009,0,0,0,0,0 +BRIBERY,2009,0,0,0,0,0,0 +BRILLIANT,0,2009,0,0,0,0,0 +BURDENING,2009,0,0,0,0,0,0 +CALAMITIES,2009,0,0,0,0,0,0 +CANCELED,2009,0,0,0,0,0,0 +CANCELLED,2009,0,0,0,0,0,0 +CARELESSLY,2009,0,0,0,0,0,0 +CATASTROPHES,2009,0,0,0,0,0,0 +CAUTIONARY,2009,0,0,0,0,0,0 +CAUTIOUS,0,0,2009,0,0,0,0 +CEASED,2009,0,0,0,0,0,0 +CEDANTS,0,0,0,2012,0,0,0 +CENSURING,2009,0,0,0,0,0,0 +CHALLENGED,2009,0,0,0,0,0,0 +CHARITABLE,0,2009,0,0,0,0,0 +CIRCUMVENT,2009,0,0,0,0,0,0 +CIRCUMVENTIONS,2009,0,0,0,0,0,0 +SHORTAGE,2009,0,0,0,0,0,0 +SHRINKAGE,2009,0,0,0,0,0,0 +SHUTDOWNS,2009,0,0,0,0,0,0 +SLANDERED,2009,0,0,0,0,0,0 +REPUDIATES,2009,0,0,0,0,0,0 +REQUESTER,0,0,0,2011,0,0,0 +REQUIREMENT,0,0,0,0,0,0,2009 +REREGULATION,0,0,0,2011,0,0,0 +RESCINDS,0,0,0,2009,0,0,0 +RESIGNATION,2009,0,0,0,0,0,0 +RESIGNS,2009,0,0,0,0,0,0 +RESTATEMENT,2009,0,0,0,0,0,0 +RESTITUTIONARY,0,0,0,2011,0,0,0 +NOTARIAL,0,0,0,2009,0,0,0 +NOTARIZE,0,0,0,2009,0,0,0 +NOTWITHSTANDING,0,0,0,2009,0,0,0 +NULLIFICATION,2009,0,0,2009,0,0,0 +NULLIFY,2009,0,0,2009,0,0,0 +OBJECTED,2009,0,0,0,0,0,0 +OBJECTIONABLY,2009,0,0,0,0,0,0 +OBLIGATES,0,0,0,0,0,0,2009 +OBLIGATORY,0,0,0,0,0,0,2009 +OBLIGEES,0,0,0,2011,0,0,0 +OBSCENE,2009,0,0,0,0,0,0 +OBSTACLE,2009,0,0,0,0,0,0 +OBSTRUCTING,2009,0,0,0,0,0,0 +OFFENCE,2009,0,0,0,0,0,0 +OFFENDER,2009,0,0,0,0,0,0 +OFFENSE,0,0,0,2009,0,0,0 +OFFERORS,0,0,0,2011,0,0,0 +OMITS,2009,0,0,0,0,0,0 +OPPORTUNISTIC,2009,0,0,0,0,0,0 +OPPOSE,2009,0,0,0,0,0,0 +OPPOSITION,2009,0,0,0,0,0,0 +OPTIONEES,0,0,0,2009,0,0,0 +OUTDATED,2009,0,0,0,0,0,0 +OUTPERFORMING,0,2009,0,0,0,0,0 +OVERBUILD,2009,0,0,0,0,0,0 +OVERBURDEN,2009,0,0,0,0,0,0 +OVERCAPACITY,2009,0,0,0,0,0,0 +OVERCHARGING,2009,0,0,0,0,0,0 +OVERDUE,2009,0,0,0,0,0,0 +OVERESTIMATING,2009,0,0,0,0,0,0 +OVERLOADED,2009,0,0,0,0,0,0 +OVERLOOKED,2009,0,0,0,0,0,0 +OVERPAYMENT,2009,0,0,0,0,0,0 +OVERPRODUCING,2009,0,0,0,0,0,0 +OVERRULES,0,0,0,2009,0,0,0 +OVERRUNS,2009,0,0,0,0,0,0 +OVERSHADOWS,2009,0,0,0,0,0,0 +OVERSTATEMENTS,2009,0,0,0,0,0,0 +OVERSUPPLIES,2009,0,0,0,0,0,0 +OVERTURN,2009,0,0,0,0,0,0 +OVERVALUE,2009,0,0,0,0,0,0 +PANICS,2009,0,0,0,0,0,0 +NECESSITATE,0,0,0,0,0,0,2009 +NEGATIVE,2009,0,0,0,0,0,0 +NEGLECTED,2009,0,0,0,0,0,0 +NEGLIGENCE,2009,0,0,0,0,0,0 +NEVER,0,0,0,0,2009,0,0 +BARRIERS,2009,0,0,0,0,0,0 +BELIEVED,0,0,2009,0,0,0,0 +IDLING,2009,0,0,0,0,0,0 +IGNORING,2009,0,0,0,0,0,0 +ILLEGALITY,2009,0,0,0,0,0,0 +ILLICITLY,2009,0,0,0,0,0,0 +IMBALANCES,2009,0,0,0,0,0,0 +IMPAIR,2009,0,0,0,0,0,2011 +IMPAIRMENTS,2009,0,0,0,0,0,2011 +IMPEDE,2009,0,0,0,0,0,0 +IMPEDIMENTS,2009,0,0,0,0,0,0 +IMPERFECTION,2009,0,0,0,0,0,0 +IMPLEADED,0,0,0,2009,0,0,0 +IMPLICATING,2009,0,0,0,0,0,0 +IMPOSING,0,0,0,0,0,0,2009 +IMPOSSIBLE,2009,0,0,0,0,0,0 +IMPOUNDS,2009,0,0,0,0,0,0 +IMPRACTICALITY,2009,0,0,0,0,0,0 +IMPRESS,0,2009,0,0,0,0,0 +IMPRESSIVE,0,2009,0,0,0,0,0 +IMPROBABLE,0,0,2009,0,0,0,0 +IMPROPRIETY,2009,0,0,0,0,0,0 +IMPROVEMENTS,0,2009,0,0,0,0,0 +IMPRUDENTLY,2009,0,0,0,0,0,0 +INACCURACY,2009,0,0,0,0,0,0 +INACTIONS,2009,0,0,0,0,0,0 +INACTIVATING,2009,0,0,0,0,0,0 +INADEQUACIES,2009,0,0,0,0,0,0 +INADVERTENT,2009,0,0,0,0,0,0 +INAPPROPRIATE,2009,0,0,0,0,0,0 +INCAPABLE,2009,0,0,0,0,0,0 +INCARCERATED,2009,0,0,2009,0,0,0 +INCARCERATIONS,2009,0,0,2009,0,0,0 +INCIDENT,2009,0,0,0,0,0,0 +INCOMPATIBLE,2009,0,0,0,0,0,0 +INCOMPETENTLY,2009,0,0,0,0,0,0 +INCOMPLETENESS,2009,0,2009,0,0,0,0 +INCONSISTENT,2009,0,0,0,0,0,0 +INCONVENIENCE,2009,0,0,0,0,0,0 +INCORRECTLY,2009,0,0,0,0,0,0 +INDEBTED,0,0,0,0,0,0,2009 +INDEFEASIBLY,2009,0,0,0,0,0,0 +INDEMNIFIABLE,0,0,0,2009,0,0,0 +INDEMNIFIES,0,0,0,2009,0,0,0 +INDEMNITEES,0,0,0,2009,0,0,0 +INDEMNITY,0,0,0,2009,0,0,0 +INDICTABLE,2009,0,0,2009,0,0,0 +INDICTMENTS,2009,0,0,2009,0,0,0 +INEFFECTIVENESS,2009,0,0,0,0,0,0 +INEFFICIENTLY,2009,0,0,0,0,0,0 +INEQUITABLY,2009,0,0,0,0,0,0 +INEXACT,0,0,2009,0,0,0,0 +INFERIOR,2009,0,0,0,0,0,0 +INFORMATIVE,0,2009,0,0,0,0,0 +INFRINGED,2009,0,0,0,0,0,0 +INFRINGES,2009,0,0,0,0,0,0 +INHIBITED,2009,0,0,0,0,0,2009 +INJUNCTION,2009,0,0,2009,0,0,0 +INJURED,2009,0,0,0,0,0,0 +INJURIOUS,2009,0,0,0,0,0,0 +INNOVATES,0,2009,0,0,0,0,0 +INNOVATIVE,0,2009,0,0,0,0,0 +INORDINATE,2009,0,0,0,0,0,0 +INSENSITIVE,2009,0,0,0,0,0,0 +INSISTENCE,0,0,0,0,0,0,2009 +INSOLVENCIES,2009,0,0,0,0,0,0 +INSPIRATIONAL,0,2009,0,0,0,0,0 +INSUFFICIENCY,2009,0,0,0,0,0,0 +INSURRECTIONS,2009,0,0,0,0,0,0 +INTENTIONAL,2009,0,0,0,0,0,0 +INTERFERENCES,2009,0,0,0,0,0,0 +INTERMITTENT,2009,0,0,0,0,0,0 +INTERPOSED,0,0,0,2009,0,0,0 +INTERPOSITIONS,0,0,0,2009,0,0,0 +INTERROGATING,0,0,0,2009,0,0,0 +INTERROGATORIES,0,0,0,2009,0,0,0 +INTERRUPTED,2009,0,0,0,0,0,0 +INTERRUPTS,2009,0,0,0,0,0,0 +STABILITY,0,2009,0,0,0,0,0 +STABILIZED,0,2009,0,0,0,0,0 +STAGGERING,2009,0,0,0,0,0,0 +STAGNATES,2009,0,0,0,0,0,0 +STANDSTILLS,2009,0,0,0,0,0,0 +STATUTORY,0,0,0,2009,0,0,0 +STIPULATING,0,0,0,0,0,0,2009 +STOPPAGE,2009,0,0,0,0,0,0 +STOPS,2009,0,0,0,0,0,0 +STRAINS,2009,0,0,0,0,0,0 +STRENGTHENING,0,2009,0,0,0,0,0 +STRESSED,2009,0,0,0,0,0,0 +STRICT,0,0,0,0,0,0,2009 +STRINGENT,2009,0,0,0,0,0,0 +STRONGLY,0,0,0,0,2009,0,0 +SUBJECTED,2009,0,0,0,0,0,0 +SUBLEASEHOLD,0,0,0,2011,0,0,0 +SUBPARAGRAPH,0,0,0,2009,0,0,0 +SUBPOENAS,2009,0,0,2009,0,0,0 +SUBTRUST,0,0,0,2011,0,0,0 +SUCCEEDING,0,2009,0,0,0,0,0 +SUCCESSFUL,0,2009,0,0,0,0,0 +SUE,2009,0,0,2009,0,0,0 +SUFFERED,2009,0,0,0,0,0,0 +SUGGESTED,0,0,2009,0,0,0,0 +SUMMONED,2009,0,0,2009,0,0,0 +SUPERIOR,0,2009,0,0,0,0,0 +SUPERSEDES,0,0,0,2009,0,0,0 +SURPASS,0,2009,0,0,0,0,0 +SUSCEPTIBILITY,2009,0,2009,0,0,0,0 +SUSPECTS,2009,0,0,0,0,0,0 +SUSPENDS,2009,0,0,0,0,0,0 +SUSPICIONS,2009,0,0,0,0,0,0 +REVOCABILITY,0,0,0,2011,0,0,0 +REVOKED,2009,0,0,0,0,0,0 +REVOLUTIONIZED,0,2009,0,0,0,0,0 +REWARDED,0,2009,0,0,0,0,0 +RIDICULED,2009,0,0,0,0,0,0 +RISKED,0,0,2009,0,0,0,0 +RISKING,0,0,2009,0,0,0,0 +RULING,0,0,0,2009,0,0,0 +SACRIFICE,2009,0,0,0,0,0,0 +SACRIFICING,2009,0,0,0,0,0,0 +SATISFIED,0,2009,0,0,0,0,0 +SCANDALOUS,2009,0,0,0,0,0,0 +SCRUTINIZES,2009,0,0,0,0,0,0 +SEEMS,0,0,2009,0,0,0,0 +SEIZING,2009,0,0,0,0,0,0 +SENTENCING,2009,0,0,2009,0,0,0 +SERIOUSNESS,2009,0,0,0,0,0,0 +SETTLEMENTS,0,0,0,2009,0,0,0 +SEVERALLY,0,0,0,2009,0,0,0 +SEVERED,2009,0,0,0,0,0,0 +ENTHUSIASTICALLY,0,2009,0,0,0,0,0 +ERODED,2009,0,0,0,0,0,0 +ERRATIC,2009,0,0,0,0,0,0 +ERRONEOUS,2009,0,0,0,0,0,0 +ERRS,2009,0,0,0,0,0,0 +ESCALATING,2009,0,0,0,0,0,0 +ESCROW,0,0,0,0,0,0,2009 +ESTOPPEL,0,0,0,2011,0,0,0 +EVADING,2009,0,0,0,0,0,0 +EVICT,2009,0,0,0,0,0,0 +EVICTIONS,2009,0,0,0,0,0,0 +EXACERBATE,2009,0,0,0,0,0,0 +EXACERBATION,2009,0,0,0,0,0,0 +EXAGGERATES,2009,0,0,0,0,0,0 +EXCEEDANCES,0,0,0,2011,0,0,0 +EXCELLING,0,2009,0,0,0,0,0 +EXCESSIVE,2009,0,0,0,0,0,0 +EXCITEMENT,0,2009,0,0,0,0,0 +EXCLUSIVENESS,0,2009,0,0,0,0,0 +EXCULPATED,2009,0,0,2009,0,0,0 +EXCULPATIONS,2009,0,0,2009,0,0,0 +EXECUTORY,0,0,0,2009,0,0,0 +EXEMPLARY,0,2009,0,0,0,0,0 +EXONERATING,2009,0,0,0,0,0,0 +EXPLOITATION,2009,0,0,0,0,0,0 +EXPLOITING,2009,0,0,0,0,0,0 +EXPOSES,2009,0,0,0,0,0,0 +EXPROPRIATE,2009,0,0,0,0,0,0 +EXPROPRIATION,2009,0,0,0,0,0,0 +EXTENUATING,2009,0,0,0,0,0,0 +SOLVING,0,2009,0,0,0,0,0 +SOMEWHERE,0,0,2009,0,0,0,0 +LOSE,2009,0,0,0,0,0,0 +LOSSES,2009,0,0,0,0,0,0 +LUCRATIVE,0,2009,0,0,0,0,0 +MALFUNCTION,2009,0,0,0,0,0,0 +MALICE,2009,0,0,0,0,0,0 +MANDAMUS,0,0,0,2009,0,0,0 +MANDATING,0,0,0,0,0,0,2009 +MANIPULATED,2009,0,0,0,0,0,0 +MANIPULATIONS,2009,0,0,0,0,0,0 +MAY,0,0,2009,0,0,2009,0 +MEDIATES,0,0,0,2009,0,0,0 +MEDIATOR,0,0,0,2009,0,0,0 +MISAPPLICATION,2009,0,0,0,0,0,0 +MISAPPLY,2009,0,0,0,0,0,0 +MISAPPROPRIATES,2009,0,0,0,0,0,0 +MISBRANDED,2009,0,0,0,0,0,0 +MISCALCULATING,2009,0,0,0,0,0,0 +MISCHIEF,2009,0,0,0,0,0,0 +MISCLASSIFY,2014,0,0,0,0,0,0 +MISDEMEANOR,2009,0,0,2009,0,0,0 +MISHANDLE,2009,0,0,0,0,0,0 +MISINFORM,2009,0,0,0,0,0,0 +MISINFORMS,2009,0,0,0,0,0,0 +MISINTERPRETED,2009,0,0,0,0,0,0 +MISJUDGED,2009,0,0,0,0,0,0 +MISJUDGMENTS,2009,0,0,0,0,0,0 +MISLABELLED,2009,0,0,0,0,0,0 +MISLEADINGLY,2009,0,0,0,0,0,0 +MISMANAGED,2009,0,0,0,0,0,0 +MISMATCH,2009,0,0,0,0,0,0 +MISPLACED,2009,0,0,0,0,0,0 +MISREPRESENT,2009,0,0,0,0,0,0 +MISREPRESENTING,2009,0,0,0,0,0,0 +MISSES,2009,0,0,0,0,0,0 +WORRY,2009,0,0,0,0,0,0 +WORSENED,2009,0,0,0,0,0,0 +WORTHLESS,2009,0,0,0,0,0,0 +TOLERATE,2009,0,0,0,0,0,0 +TOLERATION,2009,0,0,0,0,0,0 +TORTS,0,0,0,2009,0,0,0 +FICTITIOUS,2009,0,0,0,0,0,0 +FIRED,2009,0,0,0,0,0,0 +NONATTAINMENT,2009,0,0,0,0,0,0 +NONCOMPETITIVE,2009,0,0,0,0,0,0 +NONCOMPLYING,2009,0,0,0,0,0,0 +NONCONTINGENT,0,0,0,2011,0,0,0 +NONDISCLOSURE,2009,0,0,0,0,0,0 +NONFORFEITABLE,0,0,0,2009,0,0,0 +DISCLAIM,2009,0,0,0,0,0,0 +DISCLAIMING,2009,0,0,0,0,0,0 +DISCLOSES,2009,0,0,0,0,0,0 +DISCONTINUATION,2009,0,0,0,0,0,0 +DISCONTINUES,2009,0,0,0,0,0,0 +DISCOURAGES,2009,0,0,0,0,0,0 +DISCREDITING,2009,0,0,0,0,0,0 +SPECTACULAR,0,2009,0,0,0,0,0 +SPECULATES,0,0,2009,0,0,0,0 +SPECULATIVE,0,0,2009,0,0,0,0 +TRAGEDIES,2009,0,0,0,0,0,0 +TRANSFEROR,0,0,0,2009,0,0,0 +LEGISLATES,0,0,0,2009,0,0,0 +LEGISLATIVE,0,0,0,2009,0,0,0 +LEGISLATURE,0,0,0,2009,0,0,0 +LIBELOUS,0,0,0,2009,0,0,0 +LIENHOLDERS,0,0,0,2011,0,0,0 +COMPULSORY,0,0,0,0,0,0,2009 +CONCEDED,2009,0,0,0,0,0,0 +CONCEIVABLY,0,0,2009,0,0,0,0 +CONCILIATING,2009,0,0,0,0,0,0 +CONCLUSIVELY,0,2009,0,0,0,0,0 +CONDEMNED,2009,0,0,0,0,0,0 +CONDITIONAL,0,0,2009,0,0,0,0 +CONDUCIVE,0,2009,0,0,0,0,0 +CONFESSING,2009,0,0,0,0,0,0 +CONFINED,2009,0,0,0,0,0,2009 +CONFINING,2009,0,0,0,0,0,2009 +CONFISCATING,2009,0,0,0,0,0,0 +CONFLICT,2009,0,0,0,0,0,0 +CONFRONT,2009,0,0,0,0,0,0 +CONFRONTED,2009,0,0,0,0,0,0 +CONFUSED,2009,0,0,0,0,0,0 +CONFUSION,2009,0,2009,0,0,0,0 +CONSENTS,0,0,0,2009,0,0,0 +CONSPIRATOR,2009,0,0,0,0,0,0 +CONSPIRED,2009,0,0,0,0,0,0 +CONSTITUTIONAL,0,0,0,2009,0,0,0 +CONSTITUTIVE,0,0,0,2009,0,0,0 +CONSTRAINS,0,0,0,0,0,0,2009 +CONSTRUCTIVELY,0,2009,0,0,0,0,0 +CONSTRUING,0,0,0,2012,0,0,0 +CONTENDING,2009,0,0,0,0,0,0 +CONTENTIOUS,2009,0,0,0,0,0,0 +CONTESTED,2009,0,0,0,0,0,0 +CONTINGENT,0,0,2009,0,0,0,0 +CONTRACTED,0,0,0,2009,0,0,0 +CONTRACTILE,0,0,0,2009,0,0,0 +CONTRACTS,0,0,0,2009,0,0,0 +CONTRADICTED,2009,0,0,0,0,0,0 +CONTRADICTORY,2009,0,0,0,0,0,0 +CONTRAVENED,0,0,0,2009,0,0,0 +CONTRAVENTIONS,0,0,0,2009,0,0,0 +CONTROVERT,0,0,0,2009,0,0,0 +CONVEYANCE,0,0,0,2009,0,0,0 +CONVICTING,2009,0,0,2009,0,0,0 +CORRECTING,2009,0,0,0,0,0,0 +CORRUPT,2009,0,0,0,0,0,0 +CORRUPTIONS,2009,0,0,0,0,0,0 +COTERMINOUS,0,0,0,2009,0,0,0 +COUNSELLED,0,0,0,2009,0,0,0 +COUNTERCLAIMING,2009,0,0,0,0,0,0 +COUNTERFEITER,2009,0,0,0,0,0,0 +COUNTERMEASURE,2009,0,0,0,0,0,0 +COUNTERSUIT,0,0,0,2011,0,0,0 +COURTROOM,0,0,0,2009,0,0,0 +COVENANTING,0,0,0,0,0,0,2011 +CREATIVENESS,0,2009,0,0,0,0,0 +CRIMINAL,2009,0,0,2009,0,0,0 +CRIMINALLY,2009,0,0,2009,0,0,0 +CRITICAL,-2020,0,0,0,0,0,0 +CRITICIZE,2009,0,0,0,0,0,0 +CROSSCLAIM,0,0,0,2011,0,0,0 +CRUCIAL,2009,0,0,0,0,0,0 +CULPABLY,2009,0,0,0,0,0,0 +CURTAILING,2009,0,0,0,0,0,0 +CUT,2009,0,0,0,0,0,0 +CYBERATTACKS,2014,0,0,0,0,0,0 +CYBERCRIMINAL,2014,0,0,0,0,0,0 +TAINT,2009,0,0,0,0,0,0 +TAMPERED,2009,0,0,0,0,0,0 +TENTATIVE,0,0,2009,0,0,0,0 +TERMINATED,2009,0,0,0,0,0,0 +TERMINATIONS,2009,0,0,0,0,0,0 +TESTIFYING,2009,0,0,2009,0,0,0 +THENCEFORWARD,0,0,0,2009,0,0,0 +THEREIN,0,0,0,2009,0,0,0 +THEREOVER,0,0,0,2011,0,0,0 +THEREUNDER,0,0,0,2009,0,0,0 +THREAT,2009,0,0,0,0,0,0 +THREATENS,2009,0,0,0,0,0,0 +FACTO,0,0,0,2011,0,0,0 +FAILINGS,2009,0,0,0,0,0,0 +FALLOUT,2009,0,0,0,0,0,0 +FALSIFICATIONS,2009,0,0,0,0,0,0 +FALSIFYING,2009,0,0,0,0,0,0 +FATALITY,2009,0,0,0,0,0,0 +FAULTS,2009,0,0,0,0,0,0 +FAVORED,0,2009,0,0,0,0,0 +FEAR,2009,0,0,0,0,0,0 +FELONY,2009,0,0,2009,0,0,0 +DAMAGING,2009,0,0,0,0,0,0 +DANGEROUS,2009,0,0,0,0,0,0 +DEADLOCKED,2009,0,0,0,0,0,0 +DEADWEIGHTS,2009,0,0,0,0,0,0 +DECEASED,2009,0,0,0,0,0,0 +DECEITFUL,2009,0,0,0,0,0,0 +DECEIVES,2009,0,0,0,0,0,0 +DECEPTIVE,2009,0,0,0,0,0,0 +DECLINED,2009,0,0,0,0,0,0 +DECREED,0,0,0,2009,0,0,0 +DEFACED,2009,0,0,0,0,0,0 +DEFAMATION,2009,0,0,0,0,0,0 +DEFAMED,2009,0,0,0,0,0,0 +DEFAULTED,2009,0,0,0,0,0,0 +DEFEASANCES,0,0,0,2011,0,0,0 +DEFEASES,0,0,0,2014,0,0,0 +DEFEATING,2009,0,0,0,0,0,0 +DEFECTIVELY,0,0,0,2009,0,0,0 +DEFENDANT,2009,0,0,2009,0,0,0 +DEFENDS,2009,0,0,0,0,0,0 +DEFICIENCIES,2009,0,0,0,0,0,0 +DEFICITS,2009,0,0,0,0,0,0 +DEFRAUDED,2009,0,0,0,0,0,0 +DEGRADATION,2009,0,0,0,0,0,0 +DEGRADES,2009,0,0,0,0,0,0 +DELAYING,2009,0,0,0,0,0,0 +DELEGATEE,0,0,0,2011,0,0,0 +DELIBERATED,2009,0,0,0,0,0,0 +DELIGHTFUL,0,2009,0,0,0,0,0 +DELINQUENCIES,2009,0,0,0,0,0,0 +DELINQUENTS,2009,0,0,0,0,0,0 +DELISTS,2011,0,0,0,0,0,0 +DEMISING,2009,0,0,0,0,0,0 +DEMOLISHING,2009,0,0,0,0,0,0 +DEMOTED,2009,0,0,0,0,0,0 +DEMOTIONS,2009,0,0,0,0,0,0 +DEMURRING,0,0,0,2009,0,0,0 +DENIED,2009,0,0,0,0,0,0 +DENIGRATES,2009,0,0,0,0,0,0 +DENYING,2009,0,0,0,0,0,0 +DEPENDANCE,0,0,0,0,0,0,2011 +DEPENDENCE,0,0,2009,0,0,0,0 +DEPENDING,0,0,2009,0,0,2009,2011 +DEPLETES,2009,0,0,0,0,0,0 +DEPOSE,0,0,0,2009,0,0,0 +DEPOSITION,0,0,0,2009,0,0,0 +INVALIDATE,2009,0,0,0,0,0,0 +INVALIDATION,2009,0,0,0,0,0,0 +INVENTING,0,2009,0,0,0,0,0 +INVENTIVENESS,0,2009,0,0,0,0,0 +INVESTIGATED,2009,0,0,0,0,0,0 +INVESTIGATIONS,2009,0,0,0,0,0,0 +IRRECONCILABLY,2009,0,0,0,0,0,0 +IRREGULARITIES,2009,0,0,0,0,0,0 +IRREPARABLY,2009,0,0,0,0,0,0 +IRREVOCABLY,0,0,0,2009,0,0,2009 +JUDICIAL,0,0,0,2009,0,0,0 +JURIES,0,0,0,2009,0,0,0 +JURISDICTIONALLY,0,0,0,2011,0,0,0 +JURISTS,0,0,0,2009,0,0,0 +JURYMAN,0,0,0,2009,0,0,0 +KICKBACK,2009,0,0,0,0,0,0 +DIMINISH,2009,0,0,0,0,0,0 +DIMINUTION,2009,0,0,0,0,0,0 +DISADVANTAGED,2009,0,0,0,0,0,0 +DISAFFIRM,0,0,0,2009,0,0,0 +DISAGREE,2009,0,0,0,0,0,0 +DISAGREEMENT,2009,0,0,0,0,0,0 +DISALLOWANCE,2009,0,0,0,0,0,0 +DISALLOWS,2009,0,0,0,0,0,0 +DISAPPEARED,2009,0,0,0,0,0,0 +DISAPPOINTED,2009,0,0,0,0,0,0 +DISAPPOINTMENTS,2009,0,0,0,0,0,0 +DISAPPROVE,2009,0,0,0,0,0,0 +DISASSOCIATES,2009,0,0,0,0,0,0 +DISASTER,2009,0,0,0,0,0,0 +DISAVOW,2009,0,0,0,0,0,0 +DISAVOWS,2009,0,0,0,0,0,0 +GRATUITOUS,2009,0,0,0,0,0,0 +GREATEST,0,2009,0,0,0,0,0 +GRIEVANCES,2009,0,0,0,0,0,0 +HALT,2009,0,0,0,0,0,0 +HAMPERING,2009,0,0,0,0,0,0 +HAPPINESS,0,2009,0,0,0,0,0 +HARASSING,2009,0,0,0,0,0,0 +HARM,2009,0,0,0,0,0,0 +HARMING,2009,0,0,0,0,0,0 +HARSHEST,2009,0,0,0,0,0,0 +HAZARDOUS,2009,0,0,0,0,0,0 +NONINFRINGEMENT,0,0,0,2011,0,0,0 +NONJURISDICTIONAL,0,0,0,2011,0,0,0 +NONPERFORMANCES,2009,0,0,0,0,0,0 +HURTING,2009,0,0,0,0,0,0 +DISFAVORING,2009,0,0,0,0,0,0 +DISGORGEMENT,2009,0,0,0,0,0,0 +COMPLICATE,2009,0,0,0,0,0,0 +COMPLICATION,2009,0,0,0,0,0,0 +COMPLIMENTED,0,2009,0,0,0,0,0 +MISSTATEMENT,2009,0,0,0,0,0,0 +MISSTEP,2009,0,0,0,0,0,0 +MISTAKENLY,2009,0,0,0,0,0,0 +MISTRIALS,2009,0,0,2009,0,0,0 +MISUNDERSTOOD,2009,0,0,0,0,0,0 +MISUSING,2009,0,0,0,0,0,0 +MONOPOLIZE,2009,0,0,0,0,0,0 +MONOPOLY,2009,0,0,0,0,0,0 +MOREOVER,0,0,0,2009,0,0,0 +MUST,0,0,0,0,2009,0,0 +SLIPPAGE,2009,0,0,0,0,0,0 +SLOWDOWNS,2009,0,0,0,0,0,0 +SLOWING,2009,0,0,0,0,0,0 +SLUGGISHLY,2009,0,0,0,0,0,0 +SMOOTHLY,0,2009,0,0,0,0,0 +DEPRESSED,2009,0,0,0,0,0,0 +DEPRIVE,2009,0,0,0,0,0,0 +DERELICT,2009,0,0,0,0,0,0 +DEROGATES,0,0,0,2009,0,0,0 +DEROGATORY,2009,0,0,0,0,0,0 +DESIST,0,0,0,2009,0,0,0 +DESTABILIZED,2009,0,0,0,0,0,0 +DESTROYED,2009,0,0,0,0,0,0 +DESTRUCTIVE,2009,0,0,0,0,0,0 +DETENTION,2009,0,0,0,0,0,0 +DETERIORATED,2009,0,0,0,0,0,0 +DETERIORATIONS,2009,0,0,0,0,0,0 +DETERRENT,2009,0,0,0,0,0,0 +DETRACT,2009,0,0,0,0,0,0 +DETRIMENTAL,2009,0,0,0,0,0,0 +DEVALUED,2009,0,0,0,0,0,0 +DEVASTATED,2009,0,0,0,0,0,0 +DEVIATED,2009,0,2009,0,0,0,0 +DEVIATIONS,2009,0,2009,0,0,0,0 +DEVOLVES,2009,0,0,0,0,0,0 +DICTATES,0,0,0,0,0,0,2009 +DIFFERING,0,0,2009,0,0,0,0 +DIFFICULTLY,2009,0,0,0,0,0,0 +ASCENDANT,0,0,0,2009,0,0,0 +ASSAULTING,2009,0,0,0,0,0,0 +ASSIGNATION,0,0,0,2009,0,0,0 +ASSUMED,0,0,2009,0,0,0,0 +ASSUMPTIONS,0,0,2009,0,0,0,0 +ASSURING,0,2009,0,0,0,0,0 +ATTAINMENT,0,2009,0,0,0,0,0 +ATTESTATION,0,0,0,2009,0,0,0 +ATTORN,0,0,0,2011,0,0,0 +ATTORNS,0,0,0,2011,0,0,0 +AVERSELY,2011,0,0,0,0,0,0 +BAILED,0,0,0,2009,0,0,0 +BAILIFFS,0,0,0,2009,0,0,0 +BALKED,2009,0,0,0,0,0,0 +BANKRUPTED,2009,0,0,0,0,0,0 +CLAIMANT,0,0,0,2009,0,0,0 +CLAIMS,2009,0,0,2009,0,0,0 +CLAWBACKS,0,0,0,2014,0,0,0 +CLOSEOUTS,2009,0,0,0,0,0,0 +CLOSURES,2009,0,0,0,0,0,0 +CODICILS,0,0,0,2009,0,0,0 +CODIFIES,0,0,0,2009,0,0,0 +COERCED,2009,0,0,0,0,0,0 +COERCIVE,2009,0,0,0,0,0,0 +DISINCENTIVES,2009,0,0,0,0,0,0 +COLLABORATE,0,2009,0,0,0,0,0 +COLLABORATION,0,2009,0,0,0,0,0 +COLLABORATORS,0,2009,0,0,0,0,0 +COLLAPSING,2009,0,0,0,0,0,0 +COLLUDED,2009,0,0,0,0,0,0 +COLLUSIONS,2009,0,0,0,0,0,0 +POSITIVE,0,2009,0,0,0,0,0 +POSSIBILITY,0,0,2009,0,0,0,0 +POSTCLOSURE,0,0,0,2011,0,0,0 +POSTPONED,2009,0,0,0,0,0,0 +POSTPONING,2009,0,0,0,0,0,0 +WRIT,0,0,0,2009,0,0,0 +WRITEOFFS,2009,0,0,0,0,0,0 +WRONGDOINGS,2009,0,0,0,0,0,0 +HONORING,0,2009,0,0,0,0,0 +REASSESSES,0,0,2009,0,0,0,0 +REASSIGN,2009,0,0,0,0,0,0 +REASSIGNMENTS,2009,0,0,0,0,0,0 +REBOUNDING,0,2009,0,0,0,0,0 +REBUTTABLY,0,0,0,2011,0,0,0 +REBUTTING,0,0,0,2009,0,0,0 +RECALCULATING,0,0,2009,0,0,0,0 +RECALLED,2009,0,0,0,0,0,0 +RECESSION,2009,0,0,0,0,0,0 +RECKLESSLY,2009,0,0,0,0,0,0 +RECONSIDERING,0,0,2009,0,0,0,0 +RECOUPMENT,0,0,0,2009,0,0,0 +RECTIFICATION,0,0,0,2009,0,0,0 +RECUSED,0,0,0,2011,0,0,0 +REDACTED,2009,0,0,2009,0,0,0 +REDEFAULT,2014,0,0,0,0,0,0 +REDRESSED,2009,0,0,0,0,0,0 +REEXAMINE,0,0,2009,0,0,0,0 +REFERENDUMS,0,0,0,2009,0,0,0 +REFILING,0,0,0,2009,0,0,0 +REFUSAL,2009,0,0,0,0,0,0 +REFUSES,2009,0,0,0,0,0,0 +REGAINING,0,2009,0,0,0,0,0 +REGULATING,0,0,0,2009,0,0,0 +REGULATOR,0,0,0,2009,0,0,0 +REHEARD,0,0,0,2009,0,0,0 +VARIABLES,0,0,2009,0,0,0,0 +VARIANT,0,0,2009,0,0,0,0 +VARIED,0,0,2009,0,0,0,0 +VENDEE,0,0,0,2011,0,0,0 +VERSATILE,0,2009,0,0,0,0,0 +VIBRANCY,0,2009,0,0,0,0,0 +VIOLATED,2009,0,0,0,0,0,0 +VIOLATIONS,2009,0,0,0,0,0,0 +VIOLENCE,2009,0,0,0,0,0,0 +VITIATED,2009,0,0,0,0,0,0 +VOIDABLE,0,0,0,2009,0,0,0 +VOLATILITIES,0,0,2009,0,0,0,0 +VULNERABLE,2009,0,0,0,0,0,0 +WARNING,2009,0,0,0,0,0,0 +WARRANTOR,0,0,0,2011,0,0,0 +WEAK,2009,0,0,0,0,0,0 +WEAKENS,2009,0,0,0,0,0,0 +WEAKNESS,2009,0,0,0,0,0,0 +DISHONOR,2009,0,0,0,0,0,0 +DISHONORING,2009,0,0,0,0,0,0 +DISINTERESTED,2009,0,0,0,0,0,0 +DISLOYALLY,2009,0,0,0,0,0,0 +DISMISS,2009,0,0,0,0,0,0 +DISMISSES,2009,0,0,0,0,0,0 +DISPARAGED,2009,0,0,0,0,0,0 +DISPARAGING,2009,0,0,0,0,0,0 +DISPLACE,2009,0,0,0,0,0,0 +DISPLACES,2009,0,0,0,0,0,0 +DISPOSSESS,2009,0,0,0,0,0,0 +DISPOSSESSION,0,0,0,2009,0,0,0 +DISPROPORTIONATE,2009,0,0,0,0,0,0 +DISPUTES,2009,0,0,0,0,0,0 +DISQUALIFIED,2009,0,0,0,0,0,0 +DISREGARD,2009,0,0,0,0,0,0 +DISREPUTABLE,2009,0,0,0,0,0,0 +DISRUPTING,2009,0,0,0,0,0,0 +DISRUPTS,2009,0,0,0,0,0,0 +DISSENTED,2009,0,0,0,0,0,0 +DISSENTS,2009,0,0,0,0,0,0 +DISSOLUTIONS,2009,0,0,0,0,0,0 +DISTINCTIVELY,0,2009,0,0,0,0,0 +DISTORTING,2009,0,0,0,0,0,0 +DISTRACT,2009,0,0,0,0,0,0 +DISTRACTIONS,2009,0,0,0,0,0,0 +DISTRESSED,2009,0,0,0,0,0,0 +DISTURBANCE,2009,0,0,0,0,0,0 +DISTURBS,2009,0,0,0,0,0,0 +DIVERTING,2009,0,0,0,0,0,0 +DIVESTING,2009,0,0,0,0,0,0 +DIVESTMENTS,2009,0,0,0,0,0,0 +DIVULGE,2009,0,0,0,0,0,0 +DOCKET,0,0,0,2009,0,0,0 +DONEES,0,0,0,2011,0,0,0 +DOUBTS,2009,0,2009,0,0,0,0 +DOWNGRADING,2009,0,0,0,0,0,0 +DOWNSIZING,2009,0,0,0,0,0,0 +DOWNTURN,2009,0,0,0,0,0,0 +DRAG,2009,0,0,0,0,0,0 +DRAWBACKS,2009,0,0,0,0,0,0 +DROUGHTS,2009,0,0,0,0,0,0 +DYSFUNCTIONAL,2009,0,0,0,0,0,0 +EARMARKING,0,0,0,0,0,0,2009 +EASING,2009,0,0,0,0,0,0 +EFFICIENCY,0,2009,0,0,0,0,0 +EGREGIOUSLY,2009,0,0,0,0,0,0 +EMBARGOES,2009,0,0,0,0,0,0 +EMBARRASSES,2009,0,0,0,0,0,0 +EMBEZZLE,2009,0,0,0,0,0,0 +EMBEZZLER,2009,0,0,0,0,0,0 +EMPOWERED,0,2009,0,0,0,0,0 +ENABLED,0,2009,0,0,0,0,0 +ENCOURAGEMENT,0,2009,0,0,0,0,0 +ENCROACHED,2009,0,0,0,0,0,0 +ENCROACHMENTS,2009,0,0,0,0,0,0 +ENCUMBERS,2009,0,0,2009,0,0,2009 +ENCUMBRANCES,2009,0,0,2009,0,0,2009 +ENDANGERMENT,2009,0,0,0,0,0,0 +ENFORCEABLE,0,0,0,2009,0,0,0 +ENHANCEMENT,0,2009,0,0,0,0,0 +ENJOIN,2009,0,0,0,0,0,0 +ENJOY,0,2009,0,0,0,0,0 +ENJOYING,0,2009,0,0,0,0,0 +ENTAILED,0,0,0,0,0,0,2009 +PENALIZE,2009,0,0,0,0,0,0 +PENALTIES,2009,0,0,0,0,0,0 +PERFECTED,0,2009,0,0,0,0,0 +PERIL,2009,0,0,0,0,0,0 +PERMISSION,0,0,0,0,0,0,2009 +PERMITTEES,0,0,0,2011,0,0,0 +PERPETRATES,2009,0,0,2009,0,0,0 +PERSISTED,2009,0,0,0,0,0,0 +PERSISTING,2009,0,0,0,0,0,0 +PERVASIVELY,2009,0,0,0,0,0,0 +PETITIONER,0,0,0,2009,0,0,0 +PETTY,2009,0,0,0,0,0,0 +HEREAFTER,0,0,0,2009,0,0,0 +HEREFORE,0,0,0,2014,0,0,0 +HEREINAFTER,0,0,0,2009,0,0,0 +HEREON,0,0,0,2009,0,0,0 +HEREUNTO,0,0,0,2009,0,0,0 +HIDDEN,0,0,2009,0,0,0,0 +HINDERING,2009,0,0,0,0,0,0 +HINGES,0,0,2009,0,0,0,0 +WHEREABOUTS,0,0,0,2009,0,0,0 +WHEREFORE,0,0,0,2009,0,0,0 +WHERETO,0,0,0,2009,0,0,0 +WHISTLEBLOWERS,0,0,0,2011,0,0,0 +WILFUL,0,0,0,2009,0,0,0 +WILLFULNESS,0,0,0,2009,0,0,0 +WINNING,0,2009,0,0,0,0,0 +FLUCTUATE,0,0,2009,0,0,0,0 +FLUCTUATION,0,0,2009,0,0,0,0 +FORBEARANCE,0,0,0,2009,0,0,0 +FORBID,2009,0,0,0,0,0,2009 +FORCE,-2020,0,0,0,0,0,0 +FOREBEARANCE,0,0,0,2011,0,0,0 +FORECLOSES,2009,0,0,0,0,0,0 +FOREGO,2009,0,0,0,0,0,0 +FORESTALLED,2009,0,0,0,0,0,0 +FORFEITABILITY,0,0,0,2011,0,0,0 +FORFEITS,2009,0,0,0,0,0,0 +FORGERY,2009,0,0,0,0,0,0 +FRAUDS,2009,0,0,0,0,0,0 +FRIENDLY,0,2009,0,0,0,0,0 +FRUSTRATED,2009,0,0,0,0,0,0 +FRUSTRATION,2009,0,0,0,0,0,0 +FURTHERANCE,0,0,0,2009,0,0,0 +GAINS,0,2009,0,0,0,0,0 +DISGORGEMENTS,2009,0,0,0,0,0,0 +DISGRACEFUL,2009,0,0,0,0,0,0 +LITIGATE,2009,0,0,2009,0,0,0 +LITIGATION,2009,0,0,2009,0,0,0 +LITIGIOUS,0,0,0,2009,0,0,0 +LACKED,2009,0,0,0,0,0,0 +LAG,2009,0,0,0,0,0,0 +LAPSE,2009,0,0,0,0,0,0 +LATE,-2020,0,0,0,0,0,0 +LAWFULLY,0,0,0,2009,0,0,0 +LAWS,0,0,0,2009,0,0,0 +LAWYERS,0,0,0,2009,0,0,0 +LEADING,0,2009,0,0,0,0,0 +LEGALIZATION,0,0,0,2009,0,0,0 +LEGALIZES,0,0,0,2009,0,0,0 +LEGATEE,0,0,0,2009,0,0,0 +FLAWED,2009,0,0,0,0,0,0 +NONRECOVERABLE,2009,0,0,0,0,0,0 +LIQUIDATES,2009,0,0,0,0,0,0 +LIQUIDATOR,2009,0,0,0,0,0,0 +BENEFICIATION,0,0,0,2011,0,0,0 +BENEFITTED,0,2009,0,0,0,0,0 +POOR,2009,0,0,0,0,0,0 +REINTERPRETATION,0,0,2009,0,0,0,0 +REINTERPRETS,0,0,2009,0,0,0,0 +REJECTION,2009,0,0,0,0,0,0 +RELINQUISH,2009,0,0,0,0,0,0 +RELINQUISHMENT,2009,0,0,0,0,0,0 +REMAND,0,0,0,2009,0,0,0 +REMEDIATE,0,0,0,2009,0,0,0 +REMEDIATIONS,0,0,0,2009,0,0,0 +RENEGOTIATED,2009,0,0,0,0,0,0 +RENEGOTIATIONS,2009,0,0,0,0,0,0 +RENOUNCEMENTS,2009,0,0,0,0,0,0 +REPARATIONS,2009,0,0,0,0,0,0 +REPOSSESSES,2009,0,0,0,0,0,0 +REPRORATED,0,0,0,2018,0,0,0 +TROUBLED,2009,0,0,0,0,0,0 +UNABLE,2009,0,0,0,0,0,0 +UNAMBIGUOUSLY,0,0,0,0,2009,0,0 +UNAPPEALED,0,0,0,2011,0,0,0 +UNAVAILABILITY,2009,0,0,0,0,0,2011 +UNAWARE,2009,0,0,0,0,0,0 +UNCERTAINTY,0,0,2009,0,0,0,0 +UNCOLLECTIBILITY,2009,0,0,0,0,0,0 +UNCOMPLETED,2009,0,0,0,0,0,0 +UNCONSCIONABLY,2009,0,0,0,0,0,0 +UNCONTRACTED,0,0,0,2011,0,0,0 +UNCORRECTED,2009,0,0,0,0,0,0 +UNCOVERS,2009,0,0,0,0,0,0 +UNDELIVERABLE,2009,0,0,0,0,0,0 +UNDERCUTS,2009,0,0,0,0,0,0 +UNDERESTIMATES,2009,0,0,0,0,0,0 +UNDERINSURED,2009,0,0,0,0,0,0 +UNDERMINING,2009,0,0,0,0,0,0 +UNDERPAYS,2009,0,0,0,0,0,0 +UNDERPERFORMING,2009,0,0,0,0,0,0 +UNDERREPORTING,2011,0,0,0,0,0,0 +UNDERSTATEMENTS,2009,0,0,0,0,0,0 +UNDERUTILIZED,2009,0,0,0,0,0,0 +UNDETECTABLE,0,0,2009,0,0,0,0 +UNDISCHARGED,0,0,0,2009,0,0,0 +UNDOUBTEDLY,0,0,0,0,2009,0,0 +UNECONOMICAL,2009,0,0,0,0,0,0 +UNENCUMBER,0,0,0,2014,0,0,0 +UNEQUIVOCAL,0,0,0,0,2009,0,0 +UNEXCUSED,2009,0,0,0,0,0,0 +UNFAIRLY,2009,0,0,0,0,0,0 +UNFAVORABLE,2009,0,0,0,0,0,0 +UNFIT,2009,0,0,0,0,0,0 +UNFORESEEN,2009,0,0,0,0,0,0 +UNFOUNDED,2009,0,0,0,0,0,0 +UNGUARANTEED,0,0,2009,0,0,0,0 +UNINSURED,2009,0,0,0,0,0,0 +UNJUST,2009,0,0,0,0,0,0 +UNJUSTLY,2009,0,0,0,0,0,0 +UNKNOWNS,0,0,2009,0,0,0,0 +UNLICENSED,2009,0,0,0,0,0,0 +UNMERCHANTABLE,2011,0,0,0,0,0,0 +UNNEEDED,2009,0,0,0,0,0,0 +UNPAID,2009,0,0,0,0,0,0 +UNPOPULAR,2009,0,0,0,0,0,0 +UNPREDICTED,2011,0,2011,0,0,0,0 +UNPROVED,0,0,2009,0,0,0,0 +UNQUANTIFIED,0,0,2011,0,0,0,0 +UNREASONABLY,2009,0,0,0,0,0,0 +UNRECOVERED,2009,0,0,0,0,0,0 +UNREMEDIED,2009,0,0,0,0,0,0 +UNSAFE,2009,0,0,0,0,0,0 +UNSATISFIED,2009,0,0,0,0,0,0 +UNSEASONABLY,0,0,2009,0,0,0,0 +UNSOUND,2009,0,0,0,0,0,0 +UNSTABLE,2009,0,0,0,0,0,0 +UNSUCCESSFULLY,2009,0,0,0,0,0,0 +UNSUITED,2009,0,0,0,0,0,0 +UNSUSPECTING,2009,0,0,0,0,0,0 +UNTIMELY,2009,0,0,0,0,0,0 +UNTRUTHFUL,2009,0,0,0,0,0,0 +UNUSABLE,2009,0,0,0,0,0,0 +UNWARRANTED,2009,0,0,0,0,0,0 +UNWRITTEN,0,0,2009,0,0,0,0 +URGENCY,2009,0,0,0,0,0,0 +USURPATION,0,0,0,2009,0,0,0 +USURY,2009,0,0,2009,0,0,0 +VAGUENESS,0,0,2012,0,0,0,0 +VALUABLE,0,2009,0,0,0,0,0 +PLEAD,2009,0,0,0,0,0,0 +PLEADS,2009,0,0,2009,0,0,0 +PLEASED,0,2009,0,0,0,0,0 +PLEDGED,0,0,0,0,0,0,2009 +PLEDGING,0,0,0,0,0,0,2009 +COMPEL,0,0,0,0,0,0,2011 +COMPENSATORY,0,0,0,2009,0,0,0 +COMPLAINED,2009,0,0,0,0,0,0 +COMPLAINTS,2009,0,0,0,0,0,0 +COMMITMENT,0,0,0,0,0,0,2009 +COMMITTING,0,0,0,0,0,0,2009 +LIMITATION,2009,0,0,0,0,0,0 +LINGERING,2009,0,0,0,0,0,0 +RESTRAINING,0,0,0,0,0,0,2009 +RESTRICT,0,0,0,0,0,0,2009 +RESTRICTIONS,0,0,0,0,0,0,2009 +RESTRICTS,0,0,0,0,0,0,2009 +RESTRUCTURING,2009,0,0,0,0,0,0 +RETALIATES,2009,0,0,0,0,0,0 +RETALIATORY,2009,0,0,0,0,0,0 +PRECAUTIONS,0,0,2009,0,0,0,0 +PRECLUDE,2009,0,0,0,0,0,2009 +PRECONDITION,0,0,0,0,0,0,2009 +PREDECEASED,0,0,0,2009,0,0,0 +PREDICTABILITY,0,0,2009,0,0,0,0 +PREDICTIONS,0,0,2009,0,0,0,0 +PREDICTS,0,0,2009,0,0,0,0 +PREJUDICE,2009,0,0,2009,0,0,0 +PREJUDICING,2009,0,0,2009,0,0,0 +PREMATURELY,2009,0,0,0,0,0,0 +PRESET,0,0,0,0,0,0,2009 +PRESUMABLY,0,0,2009,0,0,0,0 +PRESUMING,0,0,2009,0,0,0,0 +PRETRIAL,2009,0,0,2009,0,0,0 +PREVENTION,2009,0,0,0,0,0,0 +PROACTIVE,0,2009,0,0,0,0,0 +PROBABILITY,0,0,2009,0,0,0,0 +PROBATED,0,0,0,2009,0,0,0 +PROBATIONAL,0,0,0,2009,0,0,0 +PROBATIONS,0,0,0,2009,0,0,0 +PROBLEMS,2009,0,0,0,0,0,0 +PROFITABILITY,0,2009,0,0,0,0,0 +PROGRESSED,0,2009,0,0,0,0,0 +PROHIBITED,0,0,0,0,0,0,2009 +PROHIBITIVE,0,0,0,0,0,0,2009 +PROLONG,2009,0,0,0,0,0,0 +PROLONGING,2009,0,0,0,0,0,0 +PROMULGATES,0,0,0,2009,0,0,0 +PROMULGATOR,0,0,0,2009,0,0,0 +PRORATION,0,0,0,2009,0,0,0 +PROSECUTING,2009,0,0,2009,0,0,0 +PROSECUTORIAL,0,0,0,2011,0,0,0 +PROSPERITY,0,2009,0,0,0,0,0 +PROTESTED,2009,0,0,0,0,0,0 +PROTESTOR,2009,0,0,0,0,0,0 +PROTRACTION,2009,0,0,0,0,0,0 +PROVOKE,2009,0,0,0,0,0,0 +PUNISHABLE,0,0,0,2009,0,0,0 +PUNISHMENT,2009,0,0,0,0,0,0 +PURPORTED,2009,0,0,0,0,0,0 +QUESTION,2009,0,0,0,0,0,0 +QUESTIONING,2009,0,0,0,0,0,0 +QUITCLAIMS,0,0,0,2009,0,0,0 +RANDOM,0,0,2009,0,0,0,0 +RANDOMIZING,0,0,2009,0,0,0,0 +RATABLE,0,0,0,2009,0,0,0 +RATIONALIZE,2009,0,0,0,0,0,0 +ABANDONED,2009,0,0,0,0,0,0 +ABANDONS,2009,0,0,0,0,0,0 +ABDICATION,2009,0,0,0,0,0,0 +ABERRATIONAL,2009,0,0,0,0,0,0 +ABEYANCES,0,0,2009,0,0,0,0 +ABNORMAL,2009,0,0,0,0,0,0 +ABOLISH,2009,0,0,0,0,0,0 +ABOVEMENTIONED,0,0,0,2011,0,0,0 +ABROGATING,2009,0,0,2009,0,0,0 +ABRUPTLY,2009,0,0,0,0,0,0 +ABSENTEEISM,2009,0,0,0,0,0,0 +ABSOLVING,0,0,0,2009,0,0,0 +ABUSED,2009,0,0,0,0,0,0 +ABUSIVELY,2009,0,0,0,0,0,0 +ACCIDENT,2009,0,0,0,0,0,0 +ACCLAIMED,0,2009,0,0,0,0,0 +ACCOMPLISHING,0,2009,0,0,0,0,0 +ACCUSATIONS,2009,0,0,0,0,0,0 +ACCUSING,2009,0,0,0,0,0,0 +ACHIEVEMENTS,0,2009,0,0,0,0,0 +ACQUIESCED,2009,0,0,0,0,0,0 +ACQUIRORS,0,0,0,2011,0,0,0 +ACQUITTALS,2009,0,0,2009,0,0,0 +ACQUITTING,2009,0,0,2009,0,0,0 +ADJOURNED,0,0,0,2009,0,0,0 +ADJOURNS,0,0,0,2009,0,0,0 +ADJUDGING,0,0,0,2009,0,0,0 +ADJUDICATING,0,0,0,2009,0,0,0 +ADJUDICATOR,0,0,0,2009,0,0,0 +ADMISSIBLE,0,0,0,2009,0,0,0 +ADULTERATE,2009,0,0,0,0,0,0 +ADULTERATIONS,2009,0,0,0,0,0,0 +ADVANCING,0,2009,0,0,0,0,0 +ADVANTAGEOUSLY,0,2009,0,0,0,0,0 +ADVERSARY,2009,0,0,0,0,0,0 +ADVERSITY,2009,0,0,0,0,0,0 +AFFREIGHTMENT,0,0,0,2012,0,0,0 +AFORESTATED,0,0,0,2011,0,0,0 +AGGRAVATE,2009,0,0,0,0,0,0 +AGGRAVATION,2009,0,0,0,0,0,0 +ALERTING,2009,0,0,0,0,0,0 +ALIENATING,2009,0,0,0,0,0,0 +ALLEGATIONS,2009,0,0,2009,0,0,0 +ALLEGES,2009,0,0,2009,0,0,0 +ALMOST,0,0,2009,0,0,2009,0 +AMBIGUITIES,0,0,2009,0,0,0,0 +AMENDABLE,0,0,0,2009,0,0,0 +AMENDMENT,0,0,0,2009,0,0,0 +ANNOYANCE,2009,0,0,0,0,0,0 +ANNOYS,2009,0,0,0,0,0,0 +ANNULMENT,2009,0,0,0,0,0,0 +ANOMALOUS,2009,0,2009,0,0,0,0 +ANTECEDENTS,0,0,0,2009,0,0,0 +ANTICIPATING,0,0,2009,0,0,0,0 +ANTICORRUPTION,0,0,0,2014,0,0,0 +APPARENTLY,0,0,2009,0,0,2009,0 +APPEALING,0,0,0,2009,0,0,0 +APPEARING,0,0,2009,0,0,2009,0 +APPELLATE,0,0,0,2009,0,0,0 +APPROXIMATED,0,0,2009,0,0,0,0 +APPROXIMATION,0,0,2009,0,0,0,0 +APPURTENANT,0,0,0,2009,0,0,0 +ARBITRARINESS,0,0,2009,0,0,0,0 +ARBITRATES,0,0,0,2009,0,0,0 +ARBITRATIONS,0,0,0,2009,0,0,0 +ARGUE,2009,0,0,0,0,0,0 +ARGUMENTATIVE,2009,0,0,0,0,0,0 +ARREARS,2009,0,0,0,0,0,0 +ARTIFICIALLY,2009,0,0,0,0,0,0 +RETRIBUTION,2009,0,0,0,0,0,0 +RETROCESSIONAIRES,0,0,0,2011,0,0,0 +BOLSTERS,0,2009,0,0,0,0,0 +BOOMING,0,2009,0,0,0,0,0 +BOTTLENECKS,2009,0,0,0,0,0,0 +BOYCOTTED,2009,0,0,0,0,0,0 +BREACHED,2009,0,0,2009,0,0,0 +BREAKAGE,2009,0,0,0,0,0,0 +BREAKING,-2020,0,0,0,0,0,0 +BRIBE,2009,0,0,0,0,0,0 +BRIBES,2009,0,0,0,0,0,0 +BROKEN,-2020,0,0,0,0,0,0 +BURDENS,2009,0,0,0,0,0,0 +CALAMITOUS,2009,0,0,0,0,0,0 +CANCELING,2009,0,0,0,0,0,0 +CANCELLING,2009,0,0,0,0,0,0 +CARELESSNESS,2009,0,0,0,0,0,0 +CATASTROPHIC,2009,0,0,0,0,0,0 +CAUTIONED,2009,0,0,0,0,0,0 +CAUTIOUSLY,0,0,2009,0,0,0,0 +CEASES,2009,0,0,0,0,0,0 +CENSURE,2009,0,0,0,0,0,0 +CERTIORARI,0,0,0,2011,0,0,0 +CHALLENGES,2009,0,0,0,0,0,0 +CHATTEL,0,0,0,2009,0,0,0 +CIRCUMVENTED,2009,0,0,0,0,0,0 +CIRCUMVENTS,2009,0,0,0,0,0,0 +SHALL,0,0,0,2009,0,0,0 +SHORTAGES,2009,0,0,0,0,0,0 +SHRINKAGES,2009,0,0,0,0,0,0 +SHUTS,2009,0,0,0,0,0,0 +SLANDEROUS,2009,0,0,0,0,0,0 +REPUDIATING,2009,0,0,0,0,0,0 +REQUESTOR,0,0,0,2011,0,0,0 +REQUIREMENTS,0,0,0,0,0,0,2009 +RESCIND,0,0,0,2009,0,0,0 +RESCISSION,0,0,0,2009,0,0,0 +RESIGNATIONS,2009,0,0,0,0,0,0 +RESOLVE,0,2009,0,0,0,0,0 +RESTATEMENTS,2009,0,0,0,0,0,0 +NOTARIES,0,0,0,2009,0,0,0 +NOTARIZED,0,0,0,2009,0,0,0 +NOVO,0,0,0,2011,0,0,0 +NULLIFICATIONS,2009,0,0,2009,0,0,0 +NULLIFYING,2009,0,0,2009,0,0,0 +OBJECTING,2009,0,0,0,0,0,0 +OBJECTIONS,2009,0,0,0,0,0,0 +OBLIGATING,0,0,0,0,0,0,2009 +OBLIGE,0,0,0,0,0,0,2009 +OBLIGES,0,0,0,0,0,0,2009 +OBSCENITY,2009,0,0,0,0,0,0 +OBSTACLES,2009,0,0,0,0,0,0 +OBSTRUCTION,2009,0,0,0,0,0,0 +OFFENCES,2009,0,0,0,0,0,0 +OFFENDERS,2009,0,0,0,0,0,0 +OFFEREE,0,0,0,2009,0,0,0 +OMISSION,2009,0,0,0,0,0,0 +OMITTED,2009,0,0,0,0,0,0 +OPPORTUNISTICALLY,2009,0,0,0,0,0,0 +OPPOSED,2009,0,0,0,0,0,0 +OPPOSITIONS,2009,0,0,0,0,0,0 +ORDINARILY,0,0,2009,0,0,0,0 +OUTMODED,2009,0,0,0,0,0,0 +OUTPERFORMS,0,2009,0,0,0,0,0 +OVERBUILDING,2009,0,0,0,0,0,0 +OVERBURDENED,2009,0,0,0,0,0,0 +OVERCHARGE,2009,0,0,0,0,0,0 +OVERCOME,2009,0,0,0,0,0,0 +OVERESTIMATE,2009,0,0,0,0,0,0 +OVERESTIMATION,2009,0,0,0,0,0,0 +OVERLOADING,2009,0,0,0,0,0,0 +OVERLOOKING,2009,0,0,0,0,0,0 +OVERPAYMENTS,2009,0,0,0,0,0,0 +OVERPRODUCTION,2009,0,0,0,0,0,0 +OVERRULING,0,0,0,2009,0,0,0 +OVERSHADOW,2009,0,0,0,0,0,0 +OVERSTATE,2009,0,0,0,0,0,0 +OVERSTATES,2009,0,0,0,0,0,0 +OVERSUPPLY,2009,0,0,0,0,0,0 +OVERTURNED,2009,0,0,0,0,0,0 +OVERVALUED,2009,0,0,0,0,0,0 +PARA,0,0,0,-2020,0,0,0 +NECESSITATED,0,0,0,0,0,0,2009 +NEGATIVELY,2009,0,0,0,0,0,0 +NEGLECTFUL,2009,0,0,0,0,0,0 +NEGLIGENCES,2009,0,0,0,0,0,0 +NOLO,0,0,0,2011,0,0,0 +BEAUTIFUL,0,2009,0,0,0,0,0 +BELIEVES,0,0,2009,0,0,0,0 +IDEAL,0,2009,0,0,0,0,0 +IGNORE,2009,0,0,0,0,0,0 +ILL,2009,0,0,0,0,0,0 +ILLEGALLY,2009,0,0,0,0,0,0 +ILLIQUID,2009,0,0,0,0,0,0 +IMMATERIALITY,0,0,0,2009,0,0,0 +IMPAIRED,2009,0,0,0,0,0,2011 +IMPAIRS,2009,0,0,0,0,0,2011 +IMPEDED,2009,0,0,0,0,0,0 +IMPEDING,2009,0,0,0,0,0,0 +IMPERFECTIONS,2009,0,0,0,0,0,0 +IMPLICATE,2009,0,0,0,0,0,0 +IMPOSE,0,0,0,0,0,0,2009 +IMPOSITION,0,0,0,0,0,0,2009 +IMPOUND,2009,0,0,0,0,0,0 +IMPRACTICABLE,2009,0,0,0,0,0,0 +IMPRECISE,0,0,2009,0,0,0,0 +IMPRESSED,0,2009,0,0,0,0,0 +IMPRESSIVELY,0,2009,0,0,0,0,0 +IMPROPER,2009,0,0,0,0,0,0 +IMPROVE,0,2009,0,0,0,0,0 +IMPROVES,0,2009,0,0,0,0,0 +INABILITY,2009,0,0,0,0,0,0 +INACCURATE,2009,0,0,0,0,0,0 +INACTIVATE,2009,0,0,0,0,0,0 +INACTIVATION,2009,0,0,0,0,0,0 +INADEQUACY,2009,0,0,0,0,0,0 +INADVERTENTLY,2009,0,0,0,0,0,0 +INAPPROPRIATELY,2009,0,0,0,0,0,0 +INCAPACITATED,2009,0,0,0,0,0,0 +INCARCERATES,2009,0,0,2009,0,0,0 +INCHOATE,0,0,0,2009,0,0,0 +INCIDENTS,2009,0,0,0,0,0,0 +INCOMPETENCE,2009,0,0,0,0,0,0 +INCOMPETENTS,2009,0,0,0,0,0,0 +INCONCLUSIVE,2009,0,0,0,0,0,0 +INCONSISTENTLY,2009,0,0,0,0,0,0 +INCONVENIENCES,2009,0,0,0,0,0,0 +INCORRECTNESS,2009,0,0,0,0,0,0 +INDECENCY,2009,0,0,0,0,0,0 +INDEFINITE,0,0,2009,0,0,0,0 +INDEMNIFICATION,0,0,0,2009,0,0,0 +INDEMNIFY,0,0,0,2009,0,0,0 +INDEMNITIES,0,0,0,2009,0,0,0 +INDETERMINABLE,0,0,2009,0,0,0,0 +INDICTED,2009,0,0,2009,0,0,0 +INDORSEES,0,0,0,2011,0,0,0 +INEFFICIENCIES,2009,0,0,0,0,0,0 +INELIGIBILITY,2009,0,0,0,0,0,0 +INEQUITIES,2009,0,0,0,0,0,0 +INEXACTNESS,0,0,2009,0,0,0,0 +INFLICTED,2009,0,0,0,0,0,0 +INFRACTION,2009,0,0,2009,0,0,0 +INFRINGEMENT,2009,0,0,0,0,0,0 +INFRINGING,2009,0,0,0,0,0,0 +INHIBITING,0,0,0,0,0,0,2011 +INJUNCTIONS,2009,0,0,2009,0,0,0 +INJURES,2009,0,0,0,0,0,0 +INJURY,2009,0,0,0,0,0,0 +INNOVATING,0,2009,0,0,0,0,0 +INNOVATIVENESS,0,2011,0,0,0,0,0 +INORDINATELY,2009,0,0,0,0,0,0 +INSIGHTFUL,0,2009,0,0,0,0,0 +INSISTING,0,0,0,0,0,0,2009 +INSOLVENCY,2009,0,0,0,0,0,0 +INSTABILITIES,0,0,2009,0,0,0,0 +INSUFFICIENT,2009,0,0,0,0,0,0 +INTANGIBLE,0,0,2009,0,0,0,0 +INTERFERE,2009,0,0,0,0,0,0 +INTERFERES,2009,0,0,0,0,0,0 +INTERMITTENTLY,2009,0,0,0,0,0,0 +INTERPOSES,0,0,0,2009,0,0,0 +INTERROGATE,0,0,0,2009,0,0,0 +INTERROGATION,0,0,0,2009,0,0,0 +INTERROGATORS,0,0,0,2009,0,0,0 +INTERRUPTING,2009,0,0,0,0,0,0 +INTESTACY,0,0,0,2009,0,0,0 +STABILIZATION,0,2009,0,0,0,0,0 +STABILIZES,0,2009,0,0,0,0,0 +STAGNANT,2009,0,0,0,0,0,0 +STAGNATING,2009,0,0,0,0,0,0 +STATUTE,0,0,0,2009,0,0,0 +STIPULATE,0,0,0,0,0,0,2009 +STIPULATION,0,0,0,0,0,0,2009 +STOPPAGES,2009,0,0,0,0,0,0 +STRAIN,2009,0,0,0,0,0,0 +STRENGTH,0,2009,0,0,0,0,0 +STRENGTHENS,0,2009,0,0,0,0,0 +STRESSES,2009,0,0,0,0,0,0 +STRICTER,0,0,0,0,0,0,2009 +STRONG,0,2009,0,0,0,0,0 +SUBCLAUSE,0,0,0,2009,0,0,0 +SUBJECTING,2009,0,0,0,0,0,0 +SUBLESSORS,0,0,0,2011,0,0,0 +SUBPARAGRAPHS,0,0,0,2009,0,0,0 +SUBROGATED,0,0,0,2009,0,0,0 +SUBTRUSTS,0,0,0,2011,0,0,0 +SUCCEEDS,0,2009,0,0,0,0,0 +SUCCESSFULLY,0,2009,0,0,0,0,0 +SUED,2009,0,0,2009,0,0,0 +SUFFERING,2009,0,0,0,0,0,0 +SUGGESTING,0,0,2009,0,0,0,0 +SUMMONING,2009,0,0,2009,0,0,0 +SUPERSEDE,0,0,0,2009,0,0,0 +SUPERSEDING,0,0,0,2009,0,0,0 +SURPASSED,0,2009,0,0,0,0,0 +SUSCEPTIBLE,2009,0,0,0,0,0,0 +SUSPEND,2009,0,0,0,0,0,0 +SUSPENSION,2009,0,0,0,0,0,0 +SUSPICIOUS,2009,0,0,0,0,0,0 +REVOCATION,2009,0,0,2009,0,0,0 +REVOKES,2009,0,0,0,0,0,0 +REVOLUTIONIZES,0,2009,0,0,0,0,0 +REWARDING,0,2009,0,0,0,0,0 +RIDICULES,2009,0,0,0,0,0,0 +RISKIER,2009,0,2009,0,0,0,0 +RISKS,0,0,2009,0,0,0,0 +RULINGS,0,0,0,2009,0,0,0 +SACRIFICED,2009,0,0,0,0,0,0 +SATISFACTION,0,2009,0,0,0,0,0 +SATISFIES,0,2009,0,0,0,0,0 +SCANDALS,2009,0,0,0,0,0,0 +SCRUTINIZING,2009,0,0,0,0,0,0 +SEIZE,2009,0,0,0,0,0,0 +SELDOM,0,0,2009,0,0,2009,0 +SEQUESTRATOR,0,0,0,2009,0,0,0 +SETBACK,2009,0,0,0,0,0,0 +SEVER,2009,0,0,0,0,0,0 +SEVERANCE,0,0,0,2009,0,0,0 +SEVERELY,2009,0,0,0,0,0,0 +ENTRENCH,0,0,0,0,0,0,2009 +ERODES,2009,0,0,0,0,0,0 +ERRATICALLY,2009,0,0,0,0,0,0 +ERRONEOUSLY,2009,0,0,0,0,0,0 +ESCALATE,2009,0,0,0,0,0,0 +ESCHEAT,0,0,0,2011,0,0,0 +ESCROWED,0,0,0,0,0,0,2009 +EVADE,2009,0,0,0,0,0,0 +EVASION,2009,0,0,0,0,0,0 +EVICTED,2009,0,0,0,0,0,0 +EVICTS,2009,0,0,0,0,0,0 +EXACERBATED,2009,0,0,0,0,0,0 +EXACERBATIONS,2009,0,0,0,0,0,0 +EXAGGERATING,2009,0,0,0,0,0,0 +EXCEEDENCES,0,0,0,2011,0,0,0 +EXCELS,0,2009,0,0,0,0,0 +EXCESSIVELY,2009,0,0,0,0,0,0 +EXCITING,0,2009,0,0,0,0,0 +EXCLUSIVES,0,2009,0,0,0,0,0 +EXCULPATES,2009,0,0,2009,0,0,0 +EXCULPATORY,2009,0,0,2009,0,0,0 +EXECUTRICES,0,0,0,2009,0,0,0 +EXONERATE,2009,0,0,0,0,0,0 +EXONERATION,2009,0,0,0,0,0,0 +EXPLOITATIONS,2009,0,0,0,0,0,0 +EXPLOITS,2009,0,0,0,0,0,0 +EXPOSING,2009,0,0,0,0,0,0 +EXPROPRIATED,2009,0,0,0,0,0,0 +EXPROPRIATIONS,2009,0,0,0,0,0,0 +EXTRACONTRACTUAL,0,0,0,2011,0,0,0 +SOLVENCIES,2009,0,0,0,0,0,0 +SOMETIME,0,0,2009,0,0,0,0 +SPAM,2014,0,0,0,0,0,0 +TRAUMATIC,2009,0,0,0,0,0,0 +LOSES,2009,0,0,0,0,0,0 +LOST,2009,0,0,0,0,0,0 +LYING,2009,0,0,0,0,0,0 +MALFUNCTIONED,2009,0,0,0,0,0,0 +MALICIOUS,2009,0,0,0,0,0,0 +MANDATE,0,0,0,0,0,0,2009 +MANDATORY,0,0,0,0,0,0,2009 +MANIPULATES,2009,0,0,0,0,0,0 +MANIPULATIVE,2009,0,0,0,0,0,0 +MAYBE,0,0,2009,0,0,2009,0 +MEDIATING,0,0,0,2009,0,0,0 +MEDIATORS,0,0,0,2009,0,0,0 +MISAPPLICATIONS,2009,0,0,0,0,0,0 +MISAPPLYING,2009,0,0,0,0,0,0 +MISAPPROPRIATING,2009,0,0,0,0,0,0 +MISCALCULATE,2009,0,0,0,0,0,0 +MISCALCULATION,2009,0,0,0,0,0,0 +MISCLASSIFICATION,2011,0,0,0,0,0,0 +MISCOMMUNICATION,2014,0,0,0,0,0,0 +MISDEMEANORS,2009,0,0,0,0,0,0 +MISHANDLED,2009,0,0,0,0,0,0 +MISINFORMATION,2009,0,0,0,0,0,0 +MISINTERPRET,2009,0,0,0,0,0,0 +MISINTERPRETING,2009,0,0,0,0,0,0 +MISJUDGES,2009,0,0,0,0,0,0 +MISLABEL,2009,0,0,0,0,0,0 +MISLABELS,2009,0,0,0,0,0,0 +MISLEADS,2009,0,0,0,0,0,0 +MISMANAGEMENT,2009,0,0,0,0,0,0 +MISMATCHED,2009,0,0,0,0,0,0 +MISPRICE,2014,0,0,0,0,0,0 +MISREPRESENTATION,2009,0,0,0,0,0,0 +MISREPRESENTS,2009,0,0,0,0,0,0 +WORRYING,2009,0,0,0,0,0,0 +WORSENING,2009,0,0,0,0,0,0 +WORTHY,0,2009,0,0,0,0,0 +TOLERATED,2009,0,0,0,0,0,0 +TORT,0,0,0,2009,0,0,0 +TORTUOUS,2009,0,0,0,0,0,0 +FIDE,0,0,0,2009,0,0,0 +FIRING,2009,0,0,0,0,0,0 +NONBREACHING,0,0,0,2011,0,0,0 +NONCOMPLIANCE,2009,0,0,0,0,0,0 +NONCONFORMING,2009,0,0,0,0,0,0 +NONCONTRACT,0,0,0,2012,0,0,0 +NONFEASANCE,0,0,0,2011,0,0,0 +NONFORFEITURE,0,0,0,2011,0,0,0 +DISCLAIMED,2009,0,0,0,0,0,0 +DISCLAIMS,2009,0,0,0,0,0,0 +DISCLOSING,2009,0,0,0,0,0,0 +DISCONTINUATIONS,2009,0,0,0,0,0,0 +DISCONTINUING,2009,0,0,0,0,0,0 +DISCOURAGING,2009,0,0,0,0,0,0 +DISCREDITS,2009,0,0,0,0,0,0 +SPECTACULARLY,0,2009,0,0,0,0,0 +SPECULATING,0,0,2009,0,0,0,0 +SPECULATIVELY,0,0,2009,0,0,0,0 +TRAGEDY,2009,0,0,0,0,0,0 +TRANSFERORS,0,0,0,2012,0,0,0 +LEGISLATING,0,0,0,2009,0,0,0 +LEGISLATIVELY,0,0,0,2009,0,0,0 +LEGISLATURES,0,0,0,2009,0,0,0 +LIBELS,0,0,0,2009,0,0,0 +CONCEALED,2009,0,0,0,0,0,0 +CONCEDES,2009,0,0,0,0,0,0 +CONCERN,2009,0,0,0,0,0,0 +CONCILIATION,2009,0,0,0,0,0,0 +CONDEMN,2009,0,0,0,0,0,0 +CONDEMNING,2009,0,0,0,0,0,0 +CONDITIONALLY,0,0,2009,0,0,0,0 +CONFESS,2009,0,0,0,0,0,0 +CONFESSION,2009,0,0,0,0,0,0 +CONFINEMENT,2009,0,0,0,0,0,2009 +CONFISCATE,2009,0,0,0,0,0,0 +CONFISCATION,2009,0,0,0,0,0,0 +CONFLICTED,2009,0,0,0,0,0,0 +CONFRONTATION,2009,0,0,0,0,0,0 +CONFRONTING,2009,0,0,0,0,0,0 +CONFUSES,2009,0,2009,0,0,0,0 +CONSENT,0,0,0,2009,0,0,0 +CONSERVATORSHIPS,0,0,0,2011,0,0,0 +CONSPIRATORIAL,2009,0,0,0,0,0,0 +CONSPIRES,2009,0,0,0,0,0,0 +CONSTITUTIONALITY,0,0,0,2009,0,0,0 +CONSTRAIN,0,0,0,0,0,0,2009 +CONSTRAINT,0,0,0,0,0,0,2009 +CONSTRUE,0,0,0,2012,0,0,0 +CONTEMPT,2009,0,0,0,0,0,0 +CONTENDS,2009,0,0,0,0,0,0 +CONTENTIOUSLY,2009,0,0,0,0,0,0 +CONTESTING,2009,0,0,0,0,0,0 +CONTINGENTLY,0,0,2009,0,0,0,0 +CONTRACTHOLDER,0,0,0,2009,0,0,0 +CONTRACTING,0,0,0,2009,0,0,0 +CONTRACTUAL,0,0,0,2009,0,0,0 +CONTRADICTING,2009,0,0,0,0,0,0 +CONTRADICTS,2009,0,0,0,0,0,0 +CONTRAVENES,0,0,0,2009,0,0,0 +CONTROVERSIAL,2009,0,0,0,0,0,0 +CONTROVERTED,0,0,0,2009,0,0,0 +CONVEYANCES,0,0,0,2009,0,0,0 +CONVICTION,2009,0,0,2009,0,0,0 +CORRECTION,2009,0,0,0,0,0,0 +CORRUPTED,2009,0,0,0,0,0,0 +CORRUPTLY,2009,0,0,0,0,0,0 +COULD,0,0,2009,0,0,2009,0 +COUNSELS,0,0,0,2009,0,0,0 +COUNTERCLAIMS,2009,0,0,0,0,0,0 +COUNTERFEITERS,2009,0,0,0,0,0,0 +COUNTERMEASURES,2009,0,0,0,0,0,0 +COUNTERSUITS,0,0,0,2014,0,0,0 +COURTS,0,0,0,2009,0,0,0 +COVENANTS,0,0,0,0,0,0,2011 +CREATIVITY,0,2009,0,0,0,0,0 +CRIMINALITY,0,0,0,2009,0,0,0 +CRIMINALS,2009,0,0,2009,0,0,0 +CRITICALLY,2009,0,0,0,0,0,0 +CRITICIZED,2009,0,0,0,0,0,0 +CROSSCLAIMS,0,0,0,2011,0,0,0 +CRUCIALLY,2009,0,0,0,0,0,0 +CUMBERSOME,2009,0,0,0,0,0,0 +CURTAILMENT,2009,0,0,0,0,0,0 +CUTBACK,2009,0,0,0,0,0,0 +CYBERBULLYING,2014,0,0,0,0,0,0 +CYBERCRIMINALS,2014,0,0,0,0,0,0 +TAINTED,2009,0,0,0,0,0,0 +TENANTABILITY,0,0,0,2011,0,0,0 +TENTATIVELY,0,0,2009,0,0,0,0 +TERMINATES,2009,0,0,0,0,0,0 +TERMINUS,0,0,0,-2020,0,0,0 +TESTIMONY,0,0,0,2009,0,0,0 +THEREAFTER,0,0,0,2009,0,0,0 +THEREINAFTER,0,0,0,2011,0,0,0 +THERETO,0,0,0,2009,0,0,0 +THEREUNTO,0,0,0,2009,0,0,0 +THREATEN,2009,0,0,0,0,0,0 +THREATS,2009,0,0,0,0,0,0 +FAIL,2009,0,0,0,0,0,0 +FAILS,2009,0,0,0,0,0,0 +FALSE,2009,0,0,0,0,0,0 +FALSIFIED,2009,0,0,0,0,0,0 +FALSITY,2009,0,0,0,0,0,0 +FATALLY,2009,0,0,0,0,0,0 +FAULTY,2009,0,0,0,0,0,0 +FAVORING,0,2009,0,0,0,0,0 +FEARS,2009,0,0,0,0,0,0 +DAMAGE,2009,0,0,0,0,0,0 +DAMPEN,2009,0,0,0,0,0,0 +DANGEROUSLY,2009,0,0,0,0,0,0 +DEADLOCKING,2009,0,0,0,0,0,0 +DEBARMENT,2009,0,0,0,0,0,0 +DECEDENT,0,0,0,2009,0,0,0 +DECEITFULNESS,2009,0,0,0,0,0,0 +DECEIVING,2009,0,0,0,0,0,0 +DECEPTIVELY,2009,0,0,0,0,0,0 +DECLINES,2009,0,0,0,0,0,0 +DECREEING,0,0,0,2009,0,0,0 +DEFACEMENT,2009,0,0,0,0,0,0 +DEFAMATIONS,2009,0,0,0,0,0,0 +DEFAMES,2009,0,0,0,0,0,0 +DEFAULTING,2009,0,0,0,0,0,0 +DEFEASE,0,0,0,2009,0,0,0 +DEFEASING,0,0,0,2011,0,0,0 +DEFEATS,2009,0,0,0,0,0,0 +DEFECTS,2009,0,0,0,0,0,0 +DEFENDANTS,2009,0,0,2009,0,0,0 +DEFENSIVE,2009,0,0,0,0,0,0 +DEFICIENCY,2009,0,0,0,0,0,0 +DEFINITELY,0,0,0,0,2009,0,0 +DEFRAUDING,2009,0,0,0,0,0,0 +DEGRADATIONS,2009,0,0,0,0,0,0 +DEGRADING,2009,0,0,0,0,0,0 +DELAYS,2009,0,0,0,0,0,0 +DELEGEES,0,0,0,2011,0,0,0 +DELIBERATELY,2009,0,0,0,0,0,0 +DELIGHTFULLY,0,2009,0,0,0,0,0 +DELINQUENCY,2009,0,0,0,0,0,0 +DELIST,2009,0,0,0,0,0,0 +DEMISE,2009,0,0,0,0,0,0 +DEMOLISH,2009,0,0,0,0,0,0 +DEMOLITION,2009,0,0,0,0,0,0 +DEMOTES,2009,0,0,0,0,0,0 +DEMURRED,0,0,0,2009,0,0,0 +DEMURS,0,0,0,2009,0,0,0 +DENIES,2009,0,0,0,0,0,0 +DENIGRATING,2009,0,0,0,0,0,0 +DEPEND,0,0,2009,0,0,2009,2011 +DEPENDANCES,0,0,0,0,0,0,2011 +DEPENDENCIES,0,0,2009,0,0,0,2011 +DEPENDS,0,0,2009,0,0,2009,2011 +DEPLETING,2009,0,0,0,0,0,0 +DEPOSED,0,0,0,2009,0,0,0 +DEPOSITIONAL,0,0,0,2011,0,0,0 +INVALIDATED,2009,0,0,0,0,0,0 +INVALIDITY,2009,0,0,0,0,0,0 +INVENTION,0,2009,0,0,0,0,0 +INVENTOR,0,2009,0,0,0,0,0 +INVESTIGATES,2009,0,0,0,0,0,0 +INVOLUNTARILY,2009,0,0,0,0,0,0 +IRRECOVERABLE,2009,0,0,0,0,0,0 +IRREGULARITY,2009,0,0,0,0,0,0 +IRREVERSIBLE,2009,0,0,0,0,0,0 +JEOPARDIZE,2009,0,0,0,0,0,0 +JUDICIALLY,0,0,0,2009,0,0,0 +JURIS,0,0,0,2011,0,0,0 +JURISDICTIONS,0,0,0,2009,0,0,0 +JUROR,0,0,0,2009,0,0,0 +JUSTICE,0,0,0,2009,0,0,0 +KICKBACKS,2009,0,0,0,0,0,0 +DIMINISHED,2009,0,0,0,0,0,0 +DIRECTIVE,0,0,0,0,0,0,2009 +DISADVANTAGEOUS,2009,0,0,0,0,0,0 +DISAFFIRMANCE,0,0,0,2009,0,0,0 +DISAGREEABLE,2009,0,0,0,0,0,0 +DISAGREEMENTS,2009,0,0,0,0,0,0 +DISALLOWANCES,2009,0,0,0,0,0,0 +DISAPPEAR,2009,0,0,0,0,0,0 +DISAPPEARING,2009,0,0,0,0,0,0 +DISAPPOINTING,2009,0,0,0,0,0,0 +DISAPPOINTS,2009,0,0,0,0,0,0 +DISAPPROVED,2009,0,0,0,0,0,0 +DISASSOCIATING,2009,0,0,0,0,0,0 +DISASTERS,2009,0,0,0,0,0,0 +DISAVOWAL,2009,0,0,0,0,0,0 +GRATUITOUSLY,2009,0,0,0,0,0,0 +GREATLY,0,2009,0,0,0,0,0 +GROSSLY,2009,0,0,0,0,0,0 +HALTED,2009,0,0,0,0,0,0 +HAMPERS,2009,0,0,0,0,0,0 +HAPPY,0,2009,0,0,0,0,0 +HARASSMENT,2009,0,0,0,0,0,0 +HARMED,2009,0,0,0,0,0,0 +HARMS,2009,0,0,0,0,0,0 +HARSHLY,2009,0,0,0,0,0,0 +HAZARDS,2009,0,0,0,0,0,0 +NONINFRINGING,0,0,0,2011,0,0,0 +NONPAYMENT,2009,0,0,0,0,0,0 +NONPERFORMING,2009,0,0,0,0,0,0 +DISFAVORS,2009,0,0,0,0,0,0 +PREAMENDMENT,0,0,0,2011,0,0,0 +COMPLICATED,2009,0,0,0,0,0,0 +COMPLICATIONS,2009,0,0,0,0,0,0 +COMPLIMENTING,0,2009,0,0,0,0,0 +MISSTATEMENTS,2009,0,0,0,0,0,0 +MISSTEPS,2009,0,0,0,0,0,0 +MISTAKES,2009,0,0,0,0,0,0 +MISUNDERSTAND,2009,0,0,0,0,0,0 +MISUSE,2009,0,0,0,0,0,0 +MONOPOLISTIC,2009,0,0,0,0,0,0 +MONOPOLIZED,2009,0,0,0,0,0,0 +MORATORIA,2009,0,0,0,0,0,0 +MOTHBALLED,2009,0,0,0,0,0,0 +MUTANDIS,0,0,0,2011,0,0,0 +SLIPPAGES,2009,0,0,0,0,0,0 +SLOWED,2009,0,0,0,0,0,0 +SLOWLY,2009,0,0,0,0,0,0 +SLUGGISHNESS,2009,0,0,0,0,0,0 +SMOOTHS,0,2009,0,0,0,0,0 +DEPRESSES,2009,0,0,0,0,0,0 +DEPRIVED,2009,0,0,0,0,0,0 +DERELICTION,2009,0,0,0,0,0,0 +DEROGATING,0,0,0,2009,0,0,0 +DESIGNATOR,0,0,0,2011,0,0,0 +DESPITE,0,2009,0,0,0,0,0 +DESTABILIZING,2009,0,2009,0,0,0,0 +DESTROYING,2009,0,0,0,0,0,0 +DETAIN,2009,0,0,0,0,0,0 +DETENTIONS,2009,0,0,0,0,0,0 +DETERIORATES,2009,0,0,0,0,0,0 +DETERRED,2009,0,0,0,0,0,0 +DETERRENTS,2009,0,0,0,0,0,0 +DETRACTED,2009,0,0,0,0,0,0 +DETRIMENTALLY,2009,0,0,0,0,0,0 +DEVALUES,2009,0,0,0,0,0,0 +DEVASTATING,2009,0,0,0,0,0,0 +DEVIATES,2009,0,2009,0,0,0,0 +DEVISEES,0,0,0,2011,0,0,0 +DEVOLVING,2009,0,0,0,0,0,0 +DICTATING,0,0,0,0,0,0,2009 +DIFFERS,0,0,2009,0,0,0,0 +DIFFICULTY,2009,0,0,0,0,0,0 +ASCENDANTS,0,0,0,2009,0,0,0 +ASSAULTS,2009,0,0,0,0,0,0 +ASSIGNATIONS,0,0,0,2009,0,0,0 +ASSUMES,0,0,2009,0,0,0,0 +ASSURE,0,2009,0,0,0,0,0 +ATTAIN,0,2009,0,0,0,0,0 +ATTAINMENTS,0,2009,0,0,0,0,0 +ATTESTATIONS,0,0,0,2009,0,0,0 +ATTORNEY,0,0,0,2009,0,0,0 +ATTRACTIVE,0,2009,0,0,0,0,0 +BACKDATING,2009,0,0,0,0,0,0 +BAILEE,0,0,0,2009,0,0,0 +BAILMENT,0,0,0,2011,0,0,0 +BANKRUPT,2009,0,0,0,0,0,0 +BANKRUPTING,2009,0,0,0,0,0,0 +CLAIMANTS,0,0,0,2009,0,0,0 +CLARIFICATION,0,0,2009,0,0,0,0 +CLEARLY,0,0,0,0,2009,0,0 +CLOSING,-2020,0,0,0,0,0,0 +CODEFENDANT,0,0,0,2011,0,0,0 +CODIFICATION,0,0,0,2009,0,0,0 +CODIFY,0,0,0,2009,0,0,0 +COERCES,2009,0,0,0,0,0,0 +COLLABORATED,0,2009,0,0,0,0,0 +COLLABORATIONS,0,2009,0,0,0,0,0 +COLLAPSE,2009,0,0,0,0,0,0 +COLLISION,2009,0,0,0,0,0,0 +COLLUDES,2009,0,0,0,0,0,0 +COLLUSIVE,2009,0,0,0,0,0,0 +POSITIVELY,0,2009,0,0,0,0,0 +POSSIBLE,0,0,2009,0,0,2009,0 +POSTCONTRACT,0,0,0,2011,0,0,0 +POSTPONEMENT,2009,0,0,0,0,0,0 +WRITEDOWN,2009,0,0,0,0,0,0 +WRITS,0,0,0,2009,0,0,0 +WRONGFUL,2009,0,0,0,0,0,0 +HONOR,0,2009,0,0,0,0,0 +HONORS,0,2009,0,0,0,0,0 +REARGUMENT,0,0,0,2011,0,0,0 +REASSESSING,0,0,2009,0,0,0,0 +REASSIGNED,2009,0,0,0,0,0,0 +REASSIGNS,2009,0,0,0,0,0,0 +REBUT,0,0,0,2009,0,0,0 +REBUTTAL,0,0,0,2009,0,0,0 +RECALCULATE,0,0,2009,0,0,0,0 +RECALCULATION,0,0,2009,0,0,0,0 +RECALLING,2009,0,0,0,0,0,0 +RECESSIONARY,2009,0,0,0,0,0,0 +RECKLESSNESS,2009,0,0,0,0,0,0 +RECONSIDERS,0,0,2009,0,0,0,0 +RECOUPMENTS,0,0,0,2009,0,0,0 +RECTIFICATIONS,0,0,0,2009,0,0,0 +RECUSES,0,0,0,2014,0,0,0 +REDACTING,2009,0,0,2009,0,0,0 +REDEFAULTED,2012,0,0,0,0,0,0 +REDRESSES,2009,0,0,0,0,0,0 +REEXAMINING,0,0,2009,0,0,0,0 +REFILE,0,0,0,2009,0,0,0 +REFRAIN,0,0,0,0,0,0,2009 +REFUSALS,2009,0,0,0,0,0,0 +REFUSING,2009,0,0,0,0,0,0 +REGULATE,0,0,0,2009,0,0,0 +REGULATION,0,0,0,2009,0,0,0 +REGULATORS,0,0,0,2009,0,0,0 +REHEARING,0,0,0,2009,0,0,0 +VARIABLY,0,0,2009,0,0,0,0 +VARIANTS,0,0,2009,0,0,0,0 +VARIES,0,0,2009,0,0,0,0 +VENDEES,0,0,0,2011,0,0,0 +VERSATILITY,0,2009,0,0,0,0,0 +VIBRANT,0,2009,0,0,0,0,0 +VIOLATES,2009,0,0,0,0,0,0 +VIOLATIVE,2009,0,0,2009,0,0,0 +VIOLENT,2009,0,0,0,0,0,0 +VITIATES,2009,0,0,0,0,0,0 +VOIDED,2009,0,0,2009,0,0,0 +VOLATILITY,2009,0,2009,0,0,0,0 +VULNERABLY,2009,0,0,0,0,0,0 +WARNINGS,2009,0,0,0,0,0,0 +WASTED,2009,0,0,0,0,0,0 +WEAKEN,2009,0,0,0,0,0,0 +WEAKER,2009,0,0,0,0,0,0 +WEAKNESSES,2009,0,0,0,0,0,0 +DISHONORABLE,2009,0,0,0,0,0,0 +DISHONORS,2009,0,0,0,0,0,0 +DISINTERESTEDLY,2009,0,0,0,0,0,0 +DISLOYALTY,2009,0,0,0,0,0,0 +DISMISSAL,2009,0,0,0,0,0,0 +DISMISSING,2009,0,0,0,0,0,0 +DISPARAGEMENT,2009,0,0,0,0,0,0 +DISPARAGINGLY,2009,0,0,0,0,0,0 +DISPLACED,2009,0,0,0,0,0,0 +DISPLACING,2009,0,0,0,0,0,0 +DISPOSSESSED,2009,0,0,0,0,0,0 +DISPOSSESSORY,0,0,0,2011,0,0,0 +DISPROPORTIONATELY,2009,0,0,0,0,0,0 +DISPUTING,2009,0,0,0,0,0,0 +DISQUALIFIES,2009,0,0,0,0,0,0 +DISREGARDED,2009,0,0,0,0,0,0 +DISREPUTE,2009,0,0,0,0,0,0 +DISRUPTION,2009,0,0,0,0,0,0 +DISSATISFACTION,2009,0,0,0,0,0,0 +DISSENTER,2009,0,0,0,0,0,0 +DISSIDENT,2009,0,0,0,0,0,0 +DISTINCTION,0,2009,0,0,0,0,0 +DISTINCTIVENESS,0,2009,0,0,0,0,0 +DISTORTION,2009,0,0,0,0,0,0 +DISTRACTED,2009,0,0,0,0,0,0 +DISTRACTS,2009,0,0,0,0,0,0 +DISTRIBUTEE,0,0,0,2009,0,0,0 +DISTURBANCES,2009,0,0,0,0,0,0 +DIVERSION,2009,0,0,0,0,0,0 +DIVERTS,2009,0,0,0,0,0,0 +DIVESTITURE,2009,0,0,0,0,0,0 +DIVESTS,2009,0,0,0,0,0,0 +DIVULGED,2009,0,0,0,0,0,0 +DOCKETED,0,0,0,2009,0,0,0 +DOUBT,2009,0,2009,0,0,0,0 +DOWNGRADE,2009,0,0,0,0,0,0 +DOWNSIZE,2009,0,0,0,0,0,0 +DOWNSIZINGS,2009,0,0,0,0,0,0 +DOWNTURNS,2009,0,0,0,0,0,0 +DRASTIC,2009,0,0,0,0,0,0 +DREAM,0,2009,0,0,0,0,0 +DULY,0,0,0,2009,0,0,0 +DYSFUNCTIONS,2009,0,0,0,0,0,0 +EARMARKS,0,0,0,0,0,0,2009 +EASY,0,2009,0,0,0,0,0 +EFFICIENT,0,2009,0,0,0,0,0 +EJECTMENT,0,0,0,2011,0,0,0 +EMBARGOING,2009,0,0,0,0,0,0 +EMBARRASSING,2009,0,0,0,0,0,0 +EMBEZZLED,2009,0,0,0,0,0,0 +EMBEZZLES,2009,0,0,0,0,0,0 +EMPOWERING,0,2009,0,0,0,0,0 +ENABLES,0,2009,0,0,0,0,0 +ENCOURAGES,0,2009,0,0,0,0,0 +ENCROACHES,2009,0,0,0,0,0,0 +ENCUMBER,2009,0,0,2009,0,0,2009 +ENCUMBRANCE,2009,0,0,2009,0,0,2009 +ENDANGER,2009,0,0,0,0,0,0 +ENDANGERS,2009,0,0,0,0,0,0 +ENFORCEABLY,0,0,0,2011,0,0,0 +ENHANCEMENTS,0,2009,0,0,0,0,0 +ENJOINED,2009,0,0,0,0,0,0 +ENJOYABLE,0,2009,0,0,0,0,0 +ENJOYMENT,0,2009,0,0,0,0,0 +ENTAILING,0,0,0,0,0,0,2009 +PASSU,0,0,0,2009,0,0,0 +PENALIZED,2009,0,0,0,0,0,0 +PENALTY,2009,0,0,0,0,0,0 +PERFECTLY,0,2009,0,0,0,0,0 +PERILS,2009,0,0,0,0,0,0 +PERMISSIONS,0,0,0,0,0,0,2009 +PERMITTING,0,0,0,0,0,0,2009 +PERPETRATING,2009,0,0,2009,0,0,0 +PERSISTENCE,2009,0,0,0,0,0,0 +PERSISTS,2009,0,0,0,0,0,0 +PERVASIVENESS,2009,0,0,0,0,0,0 +PETITIONERS,0,0,0,2009,0,0,0 +PICKET,2009,0,0,0,0,0,0 +HEREBY,0,0,0,2009,0,0,0 +HEREFROM,0,0,0,2009,0,0,0 +HEREINBEFORE,0,0,0,2009,0,0,0 +HERETO,0,0,0,2009,0,0,0 +HEREUPON,0,0,0,2009,0,0,0 +HIGHEST,0,2009,0,0,2009,0,0 +HINDERS,2009,0,0,0,0,0,0 +WHATEVER,0,0,0,2009,0,0,0 +WHEREAS,0,0,0,2009,0,0,0 +WHEREIN,0,0,0,2009,0,0,0 +WHEREUNDER,0,0,0,2011,0,0,0 +WHOMEVER,0,0,0,2009,0,0,0 +WILL,0,0,0,0,2009,0,0 +WIN,0,2009,0,0,0,0,0 +WITNESS,0,0,0,2009,0,0,0 +FLUCTUATED,0,0,2009,0,0,0,0 +FLUCTUATIONS,0,0,2009,0,0,0,0 +FORBEARANCES,0,0,0,2009,0,0,0 +FORBIDDEN,2009,0,0,0,0,0,2009 +FORCED,2009,0,0,0,0,0,0 +FOREBEARS,0,0,0,2009,0,0,0 +FORECLOSING,2009,0,0,0,0,0,0 +FOREGOES,2009,0,0,0,0,0,0 +FORESTALLING,2009,0,0,0,0,0,0 +FORFEITABLE,0,0,0,2009,0,0,0 +FORFEITURE,2009,0,0,0,0,0,0 +FORTHWITH,0,0,0,2009,0,0,0 +FRAUDULENCE,2009,0,0,0,0,0,0 +FRIVOLOUS,2009,0,0,0,0,0,0 +FRUSTRATES,2009,0,0,0,0,0,0 +FRUSTRATIONS,2009,0,0,0,0,0,0 +GAIN,0,2009,0,0,0,0,0 +GOOD,0,2009,0,0,0,0,0 +DISGORGES,2009,0,0,0,0,0,0 +DISGRACEFULLY,2009,0,0,0,0,0,0 +LITIGATED,2009,0,0,2009,0,0,0 +LITIGATIONS,2009,0,0,2009,0,0,0 +LITIGIOUSNESS,0,0,0,2009,0,0,0 +LACKING,2009,0,0,0,0,0,0 +LAGGED,2009,0,0,0,0,0,0 +LAPSED,2009,0,0,0,0,0,0 +LAUNDERING,2009,0,0,0,0,0,0 +LAWFULNESS,0,0,0,2009,0,0,0 +LAWSUIT,0,0,0,2009,0,0,0 +LAYOFF,2009,0,0,0,0,0,0 +LEGAL,0,0,0,2009,0,0,0 +LEGALIZATIONS,0,0,0,2009,0,0,0 +LEGALIZING,0,0,0,2009,0,0,0 +LEGATEES,0,0,0,2009,0,0,0 +FLAWS,2009,0,0,0,0,0,0 +NONRENEWAL,2009,0,0,0,0,0,0 +LIQUIDATING,2009,0,0,0,0,0,0 +LIQUIDATORS,2009,0,0,0,0,0,0 +BENEFICIAL,0,-2020,0,2020,0,0,0 +BENEFIT,0,-2020,0,0,0,0,0 +BENEFITTING,0,2009,0,0,0,0,0 +POORLY,2009,0,0,0,0,0,0 +REINTERPRETATIONS,0,0,2009,0,0,0,0 +REJECT,2009,0,0,0,0,0,0 +REJECTIONS,2009,0,0,0,0,0,0 +RELINQUISHED,2009,0,0,0,0,0,0 +RELINQUISHMENTS,2009,0,0,0,0,0,0 +REMANDED,0,0,0,2009,0,0,0 +REMEDIATED,0,0,0,2009,0,0,0 +REMEDIED,0,0,0,2009,0,0,0 +RENEGOTIATES,2009,0,0,0,0,0,0 +RENOUNCE,2009,0,0,0,0,0,0 +RENOUNCES,2009,0,0,0,0,0,0 +REPLEDGED,0,0,0,2012,0,0,0 +REPOSSESSING,2009,0,0,0,0,0,0 +TROUBLES,2009,0,0,0,0,0,0 +UNACCEPTABLE,2009,0,0,0,0,0,0 +UNANNOUNCED,2009,0,0,0,0,0,0 +UNAPPROVED,2009,0,0,0,0,0,0 +UNAVAILABLE,2009,0,0,0,0,0,2011 +UNCERTAIN,0,0,2009,0,0,2009,0 +UNCLEAR,0,0,2009,0,0,0,0 +UNCOLLECTIBLE,2009,0,0,0,0,0,0 +UNCOMPROMISING,0,0,0,0,2009,0,0 +UNCONSTITUTIONAL,0,0,0,2009,0,0,0 +UNCONTROLLABLE,2009,0,0,0,0,0,0 +UNCOVER,2009,0,0,0,0,0,0 +UNDECIDED,0,0,2009,0,0,0,0 +UNDELIVERED,2009,0,0,0,0,0,0 +UNDERCUTTING,2009,0,0,0,0,0,0 +UNDERESTIMATING,2009,0,0,0,0,0,0 +UNDERMINE,2009,0,0,0,0,0,0 +UNDERPAID,2009,0,0,0,0,0,0 +UNDERPERFORM,2011,0,0,0,0,0,0 +UNDERPERFORMS,2014,0,0,0,0,0,0 +UNDERSTATE,2009,0,0,0,0,0,0 +UNDERSTATES,2009,0,0,0,0,0,0 +UNDESIGNATED,0,0,2009,0,0,0,0 +UNDETECTED,2009,0,0,0,0,0,0 +UNDISCLOSED,2009,0,0,0,0,0,0 +UNDUE,2009,0,0,0,0,0,0 +UNECONOMICALLY,2009,0,0,0,0,0,0 +UNENCUMBERED,0,0,0,2009,0,0,0 +UNEQUIVOCALLY,0,0,0,0,2009,0,0 +UNEXPECTED,2009,0,2009,0,0,0,0 +UNFAMILIAR,0,0,2009,0,0,0,0 +UNFAVORABLY,2009,0,0,0,0,0,0 +UNFITNESS,2009,0,0,0,0,0,0 +UNFORSEEN,2011,0,2011,0,0,0,0 +UNFRIENDLY,2009,0,0,0,0,0,0 +UNHEDGED,0,0,2009,0,0,0,0 +UNINTENDED,2009,0,0,0,0,0,0 +UNJUSTIFIABLE,2009,0,0,0,0,0,0 +UNKNOWING,2009,0,0,0,0,0,0 +UNLAWFUL,2009,0,0,2009,0,0,0 +UNLIQUIDATED,2009,0,0,0,0,0,0 +UNMERITORIOUS,2014,0,0,0,0,0,0 +UNOBSERVABLE,0,0,2009,0,0,0,0 +UNPARALLELED,0,2009,0,0,2009,0,0 +UNPREDICTABILITY,2009,0,2009,0,0,0,0 +UNPRODUCTIVE,2009,0,0,0,0,0,0 +UNPROVEN,0,0,2009,0,0,0,0 +UNREALISTIC,2009,0,0,0,0,0,0 +UNRECEPTIVE,2014,0,0,0,0,0,0 +UNREIMBURSED,2009,0,0,0,0,0,0 +UNREPORTED,2009,0,0,0,0,0,0 +UNSALABLE,2009,0,0,0,0,0,0 +UNSAVORY,2009,0,0,0,0,0,0 +UNSELLABLE,2014,0,0,0,0,0,0 +UNSPECIFIC,0,0,2009,0,0,0,0 +UNSTAYED,0,0,0,2009,0,0,0 +UNSUITABILITY,2009,0,0,0,0,0,0 +UNSURE,2009,0,0,0,0,0,0 +UNSUSTAINABLE,2009,0,0,0,0,0,0 +UNTO,0,0,0,2009,0,0,0 +UNTRUTHFULLY,2009,0,0,0,0,0,0 +UNUSUAL,0,0,2009,0,0,0,0 +UNWELCOME,2009,0,0,0,0,0,0 +UPSET,2009,0,0,0,0,0,0 +URGENT,2009,0,0,0,0,0,0 +USURPED,2009,0,0,2009,0,0,0 +VAGARIES,0,0,2009,0,0,0,0 +VAGUENESSES,0,0,2012,0,0,0,0 +VANDALISM,2009,0,0,0,0,0,0 +PLAINTIFF,2009,0,0,2009,0,0,0 +PLEADED,2009,0,0,0,0,0,0 +PLEAS,2009,0,0,2009,0,0,0 +PLEASURE,0,2009,0,0,0,0,0 +PLEDGEE,0,0,0,2009,0,0,0 +PLEDGOR,0,0,0,2009,0,0,0 +COMPELLED,0,0,0,0,0,0,2009 +COMPLAIN,2009,0,0,0,0,0,0 +COMPLAINING,2009,0,0,0,0,0,0 +COMMITMENTS,0,0,0,0,0,0,2009 +LIMITATIONS,2009,0,0,0,0,0,0 +RESTRAINS,0,0,0,0,0,0,2009 +RESTRICTED,0,0,0,0,0,0,2009 +RESTRICTIVE,0,0,0,0,0,0,2009 +RESTRUCTURE,2009,0,0,0,0,0,0 +RESTRUCTURINGS,2009,0,0,0,0,0,0 +RETALIATING,2009,0,0,0,0,0,0 +RETENDERING,0,0,0,2011,0,0,0 +PRECIPITATED,2009,0,0,0,0,0,0 +PRECLUDED,2009,0,0,0,0,0,2009 +PRECONDITIONS,0,0,0,0,0,0,2009 +PREDECEASES,0,0,0,2009,0,0,0 +PREDICTED,0,0,2009,0,0,0,0 +PREDICTIVE,0,0,2009,0,0,0,0 +PREEMINENCE,0,2009,0,0,0,0,0 +PREJUDICED,2009,0,0,2009,0,0,0 +PRELIMINARILY,0,0,2009,0,0,0,0 +PREMIER,0,2009,0,0,0,0,0 +PRESSING,2009,0,0,0,0,0,0 +PRESUME,0,0,2009,0,0,0,0 +PRESUMPTION,0,0,2009,0,0,0,0 +PREVENT,0,0,0,0,0,0,2009 +PREVENTS,2009,0,0,0,0,0,2009 +PROACTIVELY,0,2009,0,0,0,0,0 +PROBABLE,0,0,2009,0,0,0,0 +PROBATES,0,0,0,2009,0,0,0 +PROBATIONARY,0,0,0,2009,0,0,0 +PROBLEM,2009,0,0,0,0,0,0 +PROFICIENCY,0,2009,0,0,0,0,0 +PROFITABLE,0,2009,0,0,0,0,0 +PROGRESSES,0,2009,0,0,0,0,0 +PROHIBITING,0,0,0,0,0,0,2009 +PROHIBITIVELY,0,0,0,0,0,0,2009 +PROLONGATION,2009,0,0,0,0,0,0 +PROLONGS,2009,0,0,0,0,0,0 +PROMULGATING,0,0,0,2009,0,0,0 +PROMULGATORS,0,0,0,2009,0,0,0 +PROSECUTE,2009,0,0,2009,0,0,0 +PROSECUTION,2009,0,0,2009,0,0,0 +PROSECUTORS,0,0,0,2009,0,0,0 +PROSPEROUS,0,2009,0,0,0,0,0 +PROTESTER,2009,0,0,0,0,0,0 +PROTESTORS,2009,0,0,0,0,0,0 +PROVISO,0,0,0,2009,0,0,0 +PROVOKED,2009,0,0,0,0,0,0 +PUNISHED,2009,0,0,0,0,0,0 +PUNISHMENTS,2009,0,0,0,0,0,0 +PURPORTEDLY,2009,0,0,0,0,0,0 +QUESTIONABLE,2009,0,0,0,0,0,0 +QUESTIONS,2009,0,0,0,0,0,0 +QUITTING,2009,0,0,0,0,0,0 +RANDOMIZE,0,0,2009,0,0,0,0 +RANDOMLY,0,0,2009,0,0,0,0 +RATABLY,0,0,0,2009,0,0,0 +RATIONALIZED,2009,0,0,0,0,0,0 +ABANDONING,2009,0,0,0,0,0,0 +ABDICATED,2009,0,0,0,0,0,0 +ABDICATIONS,2009,0,0,0,0,0,0 +ABERRATIONS,2009,0,0,0,0,0,0 +ABIDE,0,0,0,0,0,0,2009 +ABNORMALITIES,2009,0,0,0,0,0,0 +ABOLISHED,2009,0,0,0,0,0,0 +ABROGATE,2009,0,0,2009,0,0,0 +ABROGATION,2009,0,0,2009,0,0,0 +ABRUPTNESS,2009,0,0,0,0,0,0 +ABSOLVE,0,0,0,2009,0,0,0 +ABUNDANCE,0,2009,0,0,0,0,0 +ABUSES,2009,0,0,0,0,0,0 +ABUSIVENESS,2009,0,0,0,0,0,0 +ACCIDENTAL,2009,0,0,0,0,0,0 +ACCOMPLISH,0,2009,0,0,0,0,0 +ACCOMPLISHMENT,0,2009,0,0,0,0,0 +ACCUSE,2009,0,0,0,0,0,0 +ACHIEVE,0,2009,0,0,0,0,0 +ACHIEVES,0,2009,0,0,0,0,0 +ACQUIESCES,2009,0,0,0,0,0,0 +ACQUIT,2009,0,0,2009,0,0,0 +ACQUITTANCE,0,0,0,2009,0,0,0 +ADDENDUMS,0,0,0,2011,0,0,0 +ADJOURNING,0,0,0,2009,0,0,0 +ADJUDGE,0,0,0,2009,0,0,0 +ADJUDICATE,0,0,0,2009,0,0,0 +ADJUDICATION,0,0,0,2009,0,0,0 +ADJUDICATORS,0,0,0,2009,0,0,0 +ADMISSIBLY,0,0,0,2009,0,0,0 +ADULTERATED,2009,0,0,0,0,0,0 +ADVANCEMENT,0,2009,0,0,0,0,0 +ADVANTAGE,0,2009,0,0,0,0,0 +ADVANTAGES,0,2009,0,0,0,0,0 +ADVERSE,2009,0,0,0,0,0,0 +AFFIDAVIT,0,0,0,2009,0,0,0 +AFOREDESCRIBED,0,0,0,2011,0,0,0 +AFTERMATH,2009,0,0,0,0,0,0 +AGGRAVATED,2009,0,0,0,0,0,0 +AGGRAVATIONS,2009,0,0,0,0,0,0 +ALIENATE,2009,0,0,0,0,0,0 +ALIENATION,2009,0,0,0,0,0,0 +ALLEGE,2009,0,0,2009,0,0,0 +ALLEGING,2009,0,0,2009,0,0,0 +ALTERATION,0,0,2009,0,0,0,0 +AMBIGUITY,0,0,2009,0,0,0,0 +AMENDATORY,0,0,0,2009,0,0,0 +AMENDMENTS,0,0,0,2009,0,0,0 +ANNOYANCES,2009,0,0,0,0,0,0 +ANNUL,2009,0,0,0,0,0,0 +ANNULMENTS,2009,0,0,0,0,0,0 +ANOMALOUSLY,2009,0,2009,0,0,0,0 +ANTICIPATE,0,0,2009,0,0,0,0 +ANTICIPATION,0,0,2009,0,0,0,0 +ANTITRUST,2009,0,0,2009,0,0,0 +APPEAL,0,0,0,2009,0,0,0 +APPEALS,0,0,0,2009,0,0,0 +APPEARS,0,0,2009,0,0,2009,0 +APPELLEES,0,0,0,2011,0,0,0 +APPROXIMATELY,0,0,2009,0,0,0,0 +APPROXIMATIONS,0,0,2009,0,0,0,0 +ARBITRABILITY,0,0,0,2011,0,0,0 +ARBITRARY,0,0,2009,0,0,0,0 +ARBITRATING,0,0,0,2009,0,0,0 +ARBITRATIVE,0,0,0,2011,0,0,0 +ARGUED,2009,0,0,0,0,0,0 +ARGUMENTS,2009,0,0,0,0,0,0 +ARREST,2009,0,0,0,0,0,0 +RETRIBUTIONS,2009,0,0,0,0,0,0 +BONA,0,0,0,2009,0,0,0 +BOOST,0,2009,0,0,0,0,0 +BOUND,0,0,0,0,0,0,2011 +BOYCOTTING,2009,0,0,0,0,0,0 +BREACHES,2009,0,0,2009,0,0,0 +BREAKAGES,2009,0,0,0,0,0,0 +BREAKS,2009,0,0,0,0,0,0 +BRIBED,2009,0,0,0,0,0,0 +BRIBING,2009,0,0,0,0,0,0 +BURDEN,2009,0,0,0,0,0,0 +BURDENSOME,2009,0,0,0,0,0,0 +CALAMITY,2009,0,0,0,0,0,0 +CANCELLATION,2009,0,0,0,0,0,0 +CANCELS,2009,0,0,0,0,0,0 +CATASTROPHICALLY,2009,0,0,0,0,0,0 +CAUTIONING,2009,0,0,0,0,0,0 +CAUTIOUSNESS,0,0,2009,0,0,0,0 +CEASING,2009,0,0,0,0,0,0 +CENSURED,2009,0,0,0,0,0,0 +CESSION,0,0,0,2009,0,0,0 +CHALLENGING,2009,0,0,0,0,0,0 +CHATTELS,0,0,0,2009,0,0,0 +CIRCUMVENTING,2009,0,0,0,0,0,0 +SHARPLY,2009,0,0,0,0,0,0 +SHORTFALL,2009,0,0,0,0,0,0 +SHUT,2009,0,0,0,0,0,0 +SHUTTING,2009,0,0,0,0,0,0 +SLANDERS,2009,0,0,0,0,0,0 +REPUDIATE,2009,0,0,0,0,0,0 +REPUDIATION,2009,0,0,0,0,0,0 +REQUIRE,0,0,0,0,0,0,2009 +REQUIRES,0,0,0,0,0,0,2009 +RESCINDED,0,0,0,2009,0,0,0 +RESCISSIONS,0,0,0,2009,0,0,0 +RESIGNED,2009,0,0,0,0,0,0 +RESTATE,2009,0,0,0,0,0,0 +RESTATES,2009,0,0,0,0,0,0 +NONTERMINABLE,0,0,0,2011,0,0,0 +NOTARIZATION,0,0,0,2009,0,0,0 +NOTARIZING,0,0,0,2009,0,0,0 +NUISANCE,2009,0,0,0,0,0,0 +NULLIFIED,2009,0,0,2009,0,0,0 +NULLITIES,0,0,0,2009,0,0,0 +OBJECTION,2009,0,0,0,0,0,0 +OBLIGATE,0,0,0,0,0,0,2009 +OBLIGATION,0,0,0,0,0,0,2009 +OBLIGED,0,0,0,0,0,0,2009 +OBLIGOR,0,0,0,2009,0,0,0 +OBSOLESCENCE,2009,0,0,0,0,0,0 +OBSTRUCT,2009,0,0,0,0,0,0 +OBSTRUCTIONS,2009,0,0,0,0,0,0 +OFFEND,2009,0,0,0,0,0,0 +OFFENDING,2009,0,0,0,0,0,0 +OFFEREES,0,0,0,2011,0,0,0 +OMISSIONS,2009,0,0,0,0,0,0 +OMITTING,2009,0,0,0,0,0,0 +OPPORTUNITIES,0,2009,0,0,0,0,0 +OPPOSES,2009,0,0,0,0,0,0 +OPTIMISTIC,0,2009,0,0,0,0,0 +OUTAGE,2009,0,0,0,0,0,0 +OUTPERFORM,0,2009,0,0,0,0,0 +OVERAGE,2009,0,0,0,0,0,0 +OVERBUILDS,2009,0,0,0,0,0,0 +OVERBURDENING,2009,0,0,0,0,0,0 +OVERCHARGED,2009,0,0,0,0,0,0 +OVERCOMES,2009,0,0,0,0,0,0 +OVERESTIMATED,2009,0,0,0,0,0,0 +OVERESTIMATIONS,2009,0,0,0,0,0,0 +OVERLOADS,2009,0,0,0,0,0,0 +OVERLOOKS,2009,0,0,0,0,0,0 +OVERPRODUCED,2009,0,0,0,0,0,0 +OVERRULE,0,0,0,2009,0,0,0 +OVERRUN,2009,0,0,0,0,0,0 +OVERSHADOWED,2009,0,0,0,0,0,0 +OVERSTATED,2009,0,0,0,0,0,0 +OVERSTATING,2009,0,0,0,0,0,0 +OVERSUPPLYING,2009,0,0,0,0,0,0 +OVERTURNING,2009,0,0,0,0,0,0 +OVERVALUING,2009,0,0,0,0,0,0 +PARI,0,0,0,2009,0,0,0 +NECESSITATES,0,0,0,0,0,0,2009 +NEGATIVES,2009,0,0,0,0,0,0 +NEGLECTING,2009,0,0,0,0,0,0 +NEGLIGENT,2009,0,0,0,0,0,0 +BARRED,2009,0,0,0,0,0,0 +BEAUTIFULLY,0,2009,0,0,0,0,0 +BELIEVING,0,0,2009,0,0,0,0 +IDLE,2009,0,0,0,0,0,0 +IGNORED,2009,0,0,0,0,0,0 +ILLEGAL,2009,0,0,0,0,0,0 +ILLEGIBLE,2009,0,0,0,0,0,0 +ILLIQUIDITY,2009,0,0,0,0,0,0 +IMMATURE,2009,0,0,0,0,0,0 +IMPAIRING,2009,0,0,0,0,0,2011 +IMPASSE,2009,0,0,0,0,0,0 +IMPEDES,2009,0,0,0,0,0,0 +IMPENDING,2009,0,0,0,0,0,0 +IMPERIL,2009,0,0,0,0,0,0 +IMPLICATED,2009,0,0,0,0,0,0 +IMPOSED,0,0,0,0,0,0,2009 +IMPOSITIONS,0,0,0,0,0,0,2009 +IMPOUNDED,2009,0,0,0,0,0,0 +IMPRACTICAL,2009,0,0,0,0,0,0 +IMPRECISION,0,0,2009,0,0,0,0 +IMPRESSES,0,2009,0,0,0,0,0 +IMPRISONMENT,2009,0,0,0,0,0,0 +IMPROPERLY,2009,0,0,0,0,0,0 +IMPROVED,0,2009,0,0,0,0,0 +IMPROVING,0,2009,0,0,0,0,0 +INACCESSIBLE,2009,0,0,0,0,0,0 +INACCURATELY,2009,0,0,0,0,0,0 +INACTIVATED,2009,0,0,0,0,0,0 +INACTIVATIONS,2009,0,0,0,0,0,0 +INADEQUATE,2009,0,0,0,0,0,0 +INADVISABILITY,2009,0,0,0,0,0,0 +INASMUCH,0,0,0,2009,0,0,0 +INCAPACITY,2009,0,0,2009,0,0,0 +INCARCERATING,2009,0,0,2009,0,0,0 +INCIDENCE,2009,0,0,0,0,0,0 +INCOMPATIBILITIES,2009,0,0,0,0,0,0 +INCOMPETENCY,2009,0,0,0,0,0,0 +INCOMPLETE,2009,0,0,0,0,0,0 +INCONSISTENCIES,2009,0,0,0,0,0,0 +INCONTESTABILITY,0,0,0,2009,0,0,0 +INCONVENIENT,2009,0,0,0,0,0,0 +INCREDIBLE,0,2009,0,0,0,0,0 +INDECENT,2009,0,0,0,0,0,0 +INDEFINITELY,0,0,2009,0,0,0,0 +INDEMNIFICATIONS,0,0,0,2009,0,0,0 +INDEMNIFYING,0,0,0,2009,0,0,0 +INDEMNITOR,0,0,0,2009,0,0,0 +INDETERMINATE,0,0,2009,0,0,0,0 +INDICTING,2009,0,0,2009,0,0,0 +INEFFECTIVE,2009,0,0,0,0,0,0 +INEFFICIENCY,2009,0,0,0,0,0,0 +INELIGIBLE,2009,0,0,0,0,0,0 +INEQUITY,2009,0,0,0,0,0,0 +INEXPERIENCE,2009,0,0,0,0,0,0 +INFLUENTIAL,0,2009,0,0,0,0,0 +INFRACTIONS,2009,0,0,2009,0,0,0 +INFRINGEMENTS,2009,0,0,0,0,0,0 +INGENUITY,0,2009,0,0,0,0,0 +INHIBITS,0,0,0,0,0,0,2011 +INJUNCTIVE,0,0,0,2009,0,0,0 +INJURIES,2009,0,0,0,0,0,0 +INNOVATE,0,2009,0,0,0,0,0 +INNOVATION,0,2009,0,0,0,0,0 +INNOVATOR,0,2009,0,0,0,0,0 +INQUIRY,2009,0,0,0,0,0,0 +INSIST,0,0,0,0,0,0,2009 +INSISTS,0,0,0,0,0,0,2009 +INSOLVENT,2009,0,0,0,0,0,0 +INSTABILITY,2009,0,2009,0,0,0,0 +INSUFFICIENTLY,2009,0,0,0,0,0,0 +INTANGIBLES,0,0,2009,0,0,0,0 +INTERFERED,2009,0,0,0,0,0,0 +INTERFERING,2009,0,0,0,0,0,0 +INTERPLEADER,0,0,0,2009,0,0,0 +INTERPOSING,0,0,0,2009,0,0,0 +INTERROGATED,0,0,0,2009,0,0,0 +INTERROGATIONS,0,0,0,2009,0,0,0 +INTERROGATORY,0,0,0,2009,0,0,0 +INTERRUPTION,2009,0,0,0,0,0,0 +INTESTATE,0,0,0,2009,0,0,0 +SPORADIC,0,0,2009,0,0,0,0 +STABILIZATIONS,0,2009,0,0,0,0,0 +STABILIZING,0,2009,0,0,0,0,0 +STAGNATE,2009,0,0,0,0,0,0 +STAGNATION,2009,0,0,0,0,0,0 +STATUTES,0,0,0,2009,0,0,0 +STIPULATED,0,0,0,0,0,0,2009 +STIPULATIONS,0,0,0,0,0,0,2009 +STOPPED,2009,0,0,0,0,0,0 +STRAINED,2009,0,0,0,0,0,0 +STRENGTHEN,0,2009,0,0,0,0,0 +STRENGTHS,0,2009,0,0,0,0,0 +STRESSFUL,2009,0,0,0,0,0,0 +STRICTEST,0,0,0,0,0,0,2009 +STRONGER,0,2009,0,0,0,0,0 +SUBCLAUSES,0,0,0,2009,0,0,0 +SUBJECTION,2009,0,0,0,0,0,0 +SUBLICENSEE,0,0,0,2009,0,0,0 +SUBPOENA,2009,0,0,2009,0,0,0 +SUBROGATION,0,0,0,2009,0,0,0 +SUCCEED,0,2009,0,0,0,0,0 +SUCCESS,0,2009,0,0,0,0,0 +SUDDEN,0,0,2009,0,0,0,0 +SUES,2009,0,0,2009,0,0,0 +SUFFERS,2009,0,0,0,0,0,0 +SUGGESTS,0,0,2009,0,0,2009,0 +SUMMONS,2009,0,0,2009,0,0,0 +SUPERSEDEAS,0,0,0,2011,0,0,0 +SURETIES,0,0,0,2009,0,0,0 +SURPASSES,0,2009,0,0,0,0,0 +SUSPECT,2009,0,0,0,0,0,0 +SUSPENDED,2009,0,0,0,0,0,0 +SUSPENSIONS,2009,0,0,0,0,0,0 +SUSPICIOUSLY,2009,0,0,0,0,0,0 +REVISE,0,0,2009,0,0,0,0 +REVOCATIONS,2009,0,0,2009,0,0,0 +REVOKING,2009,0,0,0,0,0,0 +REVOLUTIONIZING,0,2009,0,0,0,0,0 +REWARDS,0,-2020,0,0,0,0,0 +RIDICULING,2009,0,0,0,0,0,0 +RISKIEST,2009,0,2009,0,0,0,0 +RISKY,2009,0,2009,0,0,0,0 +RUMORS,0,0,2009,0,0,0,0 +SACRIFICES,2009,0,0,0,0,0,0 +SATISFACTORILY,0,2009,0,0,0,0,0 +SATISFY,0,2009,0,0,0,0,0 +SCRUTINIZE,2009,0,0,0,0,0,0 +SCRUTINY,2009,0,0,0,0,0,0 +SEIZED,2009,0,0,0,0,0,0 +SELDOMLY,0,0,2009,0,0,2009,0 +SERIOUS,2009,0,0,0,0,0,0 +SETBACKS,2009,0,0,0,0,0,0 +SEVERABILITY,0,0,0,2009,0,0,0 +SEVERANCES,0,0,0,2009,0,0,0 +SEVERITIES,2009,0,0,0,0,0,0 +ENTHUSIASM,0,2009,0,0,0,0,0 +ENTRENCHED,0,0,0,0,0,0,2009 +ERODING,2009,0,0,0,0,0,0 +ERRED,2009,0,0,0,0,0,0 +ERROR,2009,0,0,0,0,0,0 +ESCALATED,2009,0,0,0,0,0,0 +ESCHEATED,0,0,0,2011,0,0,0 +ESCROWING,0,0,0,2011,0,0,0 +EVADED,2009,0,0,0,0,0,0 +EVASIONS,2009,0,0,0,0,0,0 +EVICTING,2009,0,0,0,0,0,0 +EVIDENTIAL,0,0,0,2011,0,0,0 +EXACERBATES,2009,0,0,0,0,0,0 +EXAGGERATE,2009,0,0,0,0,0,0 +EXAGGERATION,2009,0,0,0,0,0,0 +EXCELLENCE,0,2009,0,0,0,0,0 +EXCEPTIONAL,0,2009,0,0,0,0,0 +EXCISED,0,0,0,2009,0,0,0 +EXCLUSIVE,0,2009,0,0,0,0,0 +EXCLUSIVITY,0,2009,0,0,0,0,0 +EXCULPATING,2009,0,0,2009,0,0,0 +EXECUTOR,0,0,0,2009,0,0,0 +EXECUTRIX,0,0,0,2009,0,0,0 +EXONERATED,2009,0,0,0,0,0,0 +EXONERATIONS,2009,0,0,0,0,0,0 +EXPLOITATIVE,2009,0,0,0,0,0,0 +EXPOSE,2009,0,0,0,0,0,0 +EXPOSURE,0,0,2009,0,0,0,0 +EXPROPRIATES,2009,0,0,0,0,0,0 +EXPULSION,2009,0,0,0,0,0,0 +EXTRACORPOREAL,0,0,0,2011,0,0,0 +SOLVENCY,2009,0,0,0,0,0,0 +SOMETIMES,0,0,2009,0,0,2009,0 +SPAMMERS,2014,0,0,0,0,0,0 +TREMENDOUS,0,2009,0,0,0,0,0 +LOCKOUT,2009,0,0,0,0,0,0 +LOSING,2009,0,0,0,0,0,0 +LOWEST,0,0,0,0,2009,0,0 +MAJEURE,0,0,0,2011,0,0,0 +MALFUNCTIONING,2009,0,0,0,0,0,0 +MALICIOUSLY,2009,0,0,0,0,0,0 +MANDATED,0,0,0,0,0,0,2009 +MANDITORILY,0,0,0,0,0,0,2011 +MANIPULATING,2009,0,0,0,0,0,0 +MARKDOWN,2009,0,0,0,0,0,0 +MEDIATE,0,0,0,2009,0,0,0 +MEDIATION,0,0,0,2009,0,0,0 +MERITORIOUS,0,2009,0,0,0,0,0 +MISAPPLIED,2009,0,0,0,0,0,0 +MISAPPROPRIATE,2009,0,0,0,0,0,0 +MISAPPROPRIATION,2009,0,0,0,0,0,0 +MISCALCULATED,2009,0,0,0,0,0,0 +MISCALCULATIONS,2009,0,0,0,0,0,0 +MISCLASSIFICATIONS,2014,0,0,0,0,0,0 +MISCONDUCT,2009,0,0,0,0,0,0 +MISDIRECTED,2009,0,0,0,0,0,0 +MISHANDLES,2009,0,0,0,0,0,0 +MISINFORMED,2009,0,0,0,0,0,0 +MISINTERPRETATION,2009,0,0,0,0,0,0 +MISINTERPRETS,2009,0,0,0,0,0,0 +MISJUDGING,2009,0,0,0,0,0,0 +MISLABELED,2009,0,0,0,0,0,0 +MISLEAD,2009,0,0,0,0,0,0 +MISLED,2009,0,0,0,0,0,0 +MISMANAGES,2009,0,0,0,0,0,0 +MISMATCHES,2009,0,0,0,0,0,0 +MISPRICING,2014,0,0,0,0,0,0 +MISREPRESENTATIONS,2009,0,0,0,0,0,0 +MISS,2009,0,0,0,0,0,0 +WORSE,2009,0,0,0,0,0,0 +WORSENS,2009,0,0,0,0,0,0 +TOLERATES,2009,0,0,0,0,0,0 +TORTIOUS,0,0,0,2009,0,0,0 +TORTUOUSLY,2009,0,0,0,0,0,0 +FINED,2009,0,0,0,0,0,0 +NONAPPEALABLE,0,0,0,2009,0,0,0 +NONCANCELABLE,0,0,0,0,0,0,2009 +NONCOMPLIANCES,2009,0,0,0,0,0,0 +NONCONFORMITIES,2009,0,0,0,0,0,0 +NONCONTRACTUAL,0,0,0,2011,0,0,0 +NONFIDUCIARY,0,0,0,2011,0,0,0 +NONFUNCTIONAL,2009,0,0,0,0,0,0 +DISCLAIMER,2009,0,0,0,0,0,0 +DISCLOSE,2009,0,0,0,0,0,0 +DISCONTINUANCE,2009,0,0,0,0,0,0 +DISCONTINUE,2009,0,0,0,0,0,0 +DISCOURAGE,2009,0,0,0,0,0,0 +DISCREDIT,2009,0,0,0,0,0,0 +DISCREPANCIES,2009,0,0,0,0,0,0 +SPECULATE,0,0,2009,0,0,0,0 +SPECULATION,0,0,2009,0,0,0,0 +TRAGIC,2009,0,0,0,0,0,0 +TRANSPARENCY,0,2009,0,0,0,0,0 +LEGISLATE,0,0,0,2009,0,0,0 +LEGISLATION,0,0,0,2009,0,0,0 +LEGISLATOR,0,0,0,2009,0,0,0 +LIBEL,0,0,0,2009,0,0,0 +LICENSABLE,0,0,0,2011,0,0,0 +CONCEALING,2009,0,0,0,0,0,0 +CONCEDING,2009,0,0,0,0,0,0 +CONCERNED,2009,0,0,0,0,0,0 +CONCILIATIONS,2009,0,0,0,0,0,0 +CONDEMNATION,2009,0,0,0,0,0,0 +CONDEMNOR,0,0,0,2011,0,0,0 +CONDONE,2009,0,0,0,0,0,0 +CONFESSED,2009,0,0,0,0,0,0 +CONFIDENT,0,2009,0,0,0,0,0 +CONFINEMENTS,2009,0,0,0,0,0,0 +CONFISCATED,2009,0,0,0,0,0,0 +CONFISCATIONS,2009,0,0,0,0,0,0 +CONFLICTING,2009,0,0,0,0,0,0 +CONFRONTATIONAL,2009,0,0,0,0,0,0 +CONFRONTS,2009,0,0,0,0,0,0 +CONFUSING,2009,0,2009,0,0,0,0 +CONSENTED,0,0,0,2009,0,0,0 +CONSPIRACIES,2009,0,0,0,0,0,0 +CONSPIRATORS,2009,0,0,0,0,0,0 +CONSPIRING,2009,0,0,0,0,0,0 +CONSTITUTIONALLY,0,0,0,2009,0,0,0 +CONSTRAINED,0,0,0,0,0,0,2009 +CONSTRAINTS,0,0,0,0,0,0,2009 +CONSTRUED,0,0,0,2012,0,0,0 +CONTEND,2009,0,0,0,0,0,0 +CONTENTION,2009,0,0,0,0,0,0 +CONTESTABILITY,0,0,0,2014,0,0,0 +CONTINGENCIES,0,0,2009,0,0,0,0 +CONTINGENTS,0,0,2009,0,0,0,0 +CONTRACTHOLDERS,0,0,0,2009,0,0,0 +CONTRACTION,2009,0,0,0,0,0,0 +CONTRACTUALLY,0,0,0,2009,0,0,0 +CONTRADICTION,2009,0,0,0,0,0,0 +CONTRARY,2009,0,0,0,0,0,0 +CONTRAVENING,0,0,0,2009,0,0,0 +CONTROVERSIES,2009,0,0,0,0,0,0 +CONTROVERTING,0,0,0,2009,0,0,0 +CONVICT,2009,0,0,2009,0,0,0 +CONVICTIONS,2009,0,0,2009,0,0,0 +CORRECTIONS,2009,0,0,0,0,0,0 +CORRUPTING,2009,0,0,0,0,0,0 +CORRUPTNESS,2009,0,0,0,0,0,0 +COUNSEL,0,0,0,2009,0,0,0 +COUNTERCLAIM,2009,0,0,0,0,0,0 +COUNTERFEIT,2009,0,0,0,0,0,0 +COUNTERFEITING,2009,0,0,0,0,0,0 +COUNTERSIGNOR,0,0,0,2011,0,0,0 +COURT,0,0,0,2009,0,0,0 +COVENANT,0,0,0,0,0,0,2011 +CREATIVE,0,2009,0,0,0,0,0 +CRIME,2009,0,0,2009,0,0,0 +CRIMINALIZE,0,0,0,2014,0,0,0 +CRISES,2009,0,0,0,0,0,0 +CRITICISM,2009,0,0,0,0,0,0 +CRITICIZES,2009,0,0,0,0,0,0 +CROSSROAD,0,0,2009,0,0,0,0 +CULPABILITY,2009,0,0,0,0,0,0 +CURTAIL,2009,0,0,0,0,0,0 +CURTAILMENTS,2009,0,0,0,0,0,0 +CUTBACKS,2009,0,0,0,0,0,0 +CYBERCRIME,2014,0,0,0,0,0,0 +TAINTING,2009,0,0,0,0,0,0 +TENDING,0,0,2009,0,0,0,0 +TERMINABLE,0,0,0,2009,0,0,0 +TERMINATING,2009,0,0,0,0,0,0 +TESTAMENTARY,0,0,0,2009,0,0,0 +THENCE,0,0,0,2009,0,0,0 +THEREAT,0,0,0,2009,0,0,0 +THEREOF,0,0,0,2009,0,0,0 +THERETOFOR,0,0,0,2011,0,0,0 +THEREUPON,0,0,0,2009,0,0,0 +THREATENED,2009,0,0,0,0,0,0 +FAILED,2009,0,0,0,0,0,0 +FAILURE,2009,0,0,0,0,0,0 +FALSELY,2009,0,0,0,0,0,0 +FALSIFIES,2009,0,0,0,0,0,0 +FANTASTIC,0,2009,0,0,0,0,0 +FAULT,2009,0,0,0,0,0,0 +FAVORABLE,0,2009,0,0,0,0,0 +FAVORITE,0,2009,0,0,0,0,0 +FELONIES,2009,0,0,2009,0,0,0 +DAMAGED,2009,0,0,0,0,0,0 +DAMPENED,2009,0,0,0,0,0,0 +DANGERS,2009,0,0,0,0,0,0 +DEADLOCKS,2009,0,0,0,0,0,0 +DEBARMENTS,2009,0,0,0,0,0,0 +DECEDENTS,0,0,0,2009,0,0,0 +DECEIVE,2009,0,0,0,0,0,0 +DECEPTION,2009,0,0,0,0,0,0 +DECLARANT,0,0,0,2011,0,0,0 +DECLINING,2009,0,0,0,0,0,0 +DECREES,0,0,0,2009,0,0,0 +DEFALCATION,0,0,0,2009,0,0,0 +DEFAMATORY,2009,0,0,0,0,0,0 +DEFAMING,2009,0,0,0,0,0,0 +DEFAULTS,2009,0,0,0,0,0,0 +DEFEASED,0,0,0,2009,0,0,0 +DEFEAT,2009,0,0,0,0,0,0 +DEFECT,2009,0,0,0,0,0,0 +DEFEND,2009,0,0,0,0,0,0 +DEFENDED,2009,0,0,0,0,0,0 +DEFER,2009,0,0,0,0,0,0 +DEFICIENT,2009,0,0,0,0,0,0 +DEFINITIVELY,0,0,0,0,2009,0,0 +DEFRAUDS,2009,0,0,0,0,0,0 +DEGRADE,2009,0,0,0,0,0,0 +DELAY,2009,0,0,0,0,0,0 +DELEGABLE,0,0,0,2009,0,0,0 +DELETERIOUS,2009,0,0,0,0,0,0 +DELIGHT,0,2009,0,0,0,0,0 +DELIGHTING,0,2009,0,0,0,0,0 +DELINQUENT,2009,0,0,0,0,0,0 +DELISTED,2009,0,0,0,0,0,0 +DEMISED,2009,0,0,0,0,0,0 +DEMOLISHED,2009,0,0,0,0,0,0 +DEMOLITIONS,2009,0,0,0,0,0,0 +DEMOTING,2009,0,0,0,0,0,0 +DEMURRER,0,0,0,2009,0,0,0 +DENIAL,2009,0,0,0,0,0,0 +DENIGRATE,2009,0,0,0,0,0,0 +DENIGRATION,2009,0,0,0,0,0,0 +DEPENDABILITY,0,2009,0,0,0,0,0 +DEPENDANT,0,0,0,0,0,0,2011 +DEPENDENCY,0,0,2009,0,0,0,0 +DEPLETE,2009,0,0,0,0,0,0 +DEPLETION,2009,0,0,0,0,0,0 +DEPOSES,0,0,0,2009,0,0,0 +DEPOSITIONS,0,0,0,2009,0,0,0 +INTRUSION,2009,0,0,0,0,0,0 +INVALIDATES,2009,0,0,0,0,0,0 +INVENT,0,2009,0,0,0,0,0 +INVENTIONS,0,2009,0,0,0,0,0 +INVENTORS,0,2009,0,0,0,0,0 +INVESTIGATING,2009,0,0,0,0,0,0 +INVOLUNTARY,2009,0,0,0,0,0,0 +IRRECOVERABLY,2009,0,0,0,0,0,0 +IRREGULARLY,2009,0,0,0,0,0,0 +IRREVOCABILITY,0,0,0,2011,0,0,0 +JEOPARDIZED,2009,0,0,0,0,0,0 +JUDICIARIES,0,0,0,2009,0,0,0 +JURISDICTION,0,0,0,2009,0,0,0 +JURISPRUDENCE,0,0,0,2009,0,0,0 +JURORS,0,0,0,2009,0,0,0 +JUSTICES,0,0,0,2009,0,0,0 +KNOWINGLY,2009,0,0,0,0,0,0 +DILIGENT,0,2009,0,0,0,0,0 +DIMINISHES,2009,0,0,0,0,0,0 +DIRECTIVES,0,0,0,0,0,0,2009 +DISADVANTAGES,2009,0,0,0,0,0,0 +DISAFFIRMED,0,0,0,2011,0,0,0 +DISAGREED,2009,0,0,0,0,0,0 +DISAGREES,2009,0,0,0,0,0,0 +DISALLOWED,2009,0,0,0,0,0,0 +DISAPPEARANCE,2009,0,0,0,0,0,0 +DISAPPEARS,2009,0,0,0,0,0,0 +DISAPPOINTINGLY,2009,0,0,0,0,0,0 +DISAPPROVAL,2009,0,0,0,0,0,0 +DISAPPROVES,2009,0,0,0,0,0,0 +DISASSOCIATION,2009,0,0,0,0,0,0 +DISASTROUS,2009,0,0,0,0,0,0 +DISAVOWED,2009,0,0,0,0,0,0 +GREAT,0,-2020,0,0,0,0,0 +GREATNESS,0,2009,0,0,0,0,0 +GROUNDLESS,2009,0,0,0,0,0,0 +HAMPER,2009,0,0,0,0,0,0 +HAPPIEST,0,2009,0,0,0,0,0 +HARASS,2009,0,0,0,0,0,0 +HARDSHIP,2009,0,0,0,0,0,0 +HARMFUL,2009,0,0,0,0,0,0 +HARSH,2009,0,0,0,0,0,0 +HARSHNESS,2009,0,0,0,0,0,0 +NONJUDICIAL,0,0,0,2009,0,0,0 +NONPAYMENTS,2009,0,0,0,0,0,0 \ No newline at end of file diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/python-solution/task.py b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/python-solution/task.py new file mode 100644 index 0000000000000..5965768403e78 --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/python-solution/task.py @@ -0,0 +1,135 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# beam-playground: +# name: FinalSolution2 +# description: Final challenge solution 2. +# multifile: true +# files: +# - name: analysis.csv +# context_line: 57 +# categories: +# - Quickstart +# complexity: ADVANCED +# tags: +# - hellobeam + +import re +import apache_beam as beam +from apache_beam.io import ReadFromText +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.transforms.combiners import CountCombineFn + + +class SplitWords(beam.DoFn): + def process(self, element): + return element.lower().split(" ") + + +class Analysis: + def __init__(self, word, negative, positive, uncertainty, litigious, strong, weak, constraining): + self.word = word + self.negative = negative + self.positive = positive + self.uncertainty = uncertainty + self.litigious = litigious + self.strong = strong + self.weak = weak + self.constraining = constraining + + def __str__(self): + return (f'Analysis(word={self.word}, negative={self.negative}, positive={self.positive}, ' + f'uncertainty={self.uncertainty}, litigious={self.litigious}, strong={self.strong}, ' + f'weak={self.weak}, constraining={self.constraining})') + + +class ExtractAnalysis(beam.DoFn): + def process(self, element): + items = re.split(r',(?=(?:[^"]*"[^"]*")*[^"]*$)', element) + if items[1] != 'Negative': + yield Analysis(items[0].lower(), items[1], items[2], items[3], items[4], items[5], items[6], items[7]) + + +class Partition(beam.PTransform): + def expand(self, pcoll): + return pcoll | beam.Partition(self._analysis_partition_fn, 3) + + @staticmethod + def _analysis_partition_fn(analysis, num_partitions): + if analysis.positive != "0": + return 0 + elif analysis.negative != "0": + return 1 + else: + return 2 + + +class LogOutput(beam.DoFn): + def __init__(self, message): + self.message = message + + def process(self, element): + print(f"{self.message}: {element}") + + +class MatchWordDoFn(beam.DoFn): + def process(self, element, analysis): + for a in analysis: + if a.word == element: + yield a + + +def run(): + pipeline_options = PipelineOptions() + with beam.Pipeline(options=pipeline_options) as p: + shakespeare = (p + | 'Read from text file' >> ReadFromText('gs://apache-beam-samples/shakespeare/kinglear.txt') + | 'Split into words' >> beam.ParDo(SplitWords()) + | 'Filter empty words' >> beam.Filter(bool)) + + analysis = (p + | 'Read from csv file' >> ReadFromText('analysis.csv') + | 'Extract Analysis' >> beam.ParDo(ExtractAnalysis())) + + matches = shakespeare | beam.ParDo(MatchWordDoFn(), beam.pvalue.AsList(analysis)) + + result = matches | Partition() + + positive_words = result[0] + negative_words = result[1] + + (positive_words + | 'Count Positive Words' >> beam.CombineGlobally(CountCombineFn()).without_defaults() + | 'Log Positive Words' >> beam.ParDo(LogOutput('Positive word count'))) + + (positive_words + | 'Filter Strong or Weak Positive Words' >> beam.Filter( + lambda analysis: analysis.strong != '0' or analysis.weak != '0') + | 'Count Strong or Weak Positive Words' >> beam.CombineGlobally(CountCombineFn()).without_defaults() + | 'Log Strong or Weak Positive Words' >> beam.ParDo(LogOutput('Positive words with enhanced effect count'))) + + (negative_words + | 'Count Negative Words' >> beam.CombineGlobally(CountCombineFn()).without_defaults() + | 'Log Negative Words' >> beam.ParDo(LogOutput('Negative word count'))) + + (negative_words + | 'Filter Strong or Weak Negative Words' >> beam.Filter( + lambda analysis: analysis.strong != '0' or analysis.weak != '0') + | 'Count Strong or Weak Negative Words' >> beam.CombineGlobally(CountCombineFn()).without_defaults() + | 'Log Strong or Weak Negative Words' >> beam.ParDo(LogOutput('Negative words with enhanced effect count'))) + +if __name__ == "__main__": + run() diff --git a/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/unit-info.yaml b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/unit-info.yaml new file mode 100644 index 0000000000000..9bd80fe93f335 --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/unit-info.yaml @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +sdk: + - Java + - Python + - Go +id: final-challenge-2 +name: Final Challenge 2 +taskName: FinalChallenge2 +solutionName: FinalSolution2 diff --git a/learning/tour-of-beam/learning-content/final-challenge/module-info.yaml b/learning/tour-of-beam/learning-content/final-challenge/module-info.yaml new file mode 100644 index 0000000000000..e031bd36da09c --- /dev/null +++ b/learning/tour-of-beam/learning-content/final-challenge/module-info.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +sdk: + - Java + - Python + - Go +id: final-challenge +name: Final challenge +complexity: ADVANCED +content: +- final-challenge-1 +- final-challenge-2 diff --git a/learning/tour-of-beam/learning-content/introduction/introduction-concepts/pipeline-concepts/setting-pipeline/python-example/task.py b/learning/tour-of-beam/learning-content/introduction/introduction-concepts/pipeline-concepts/setting-pipeline/python-example/task.py index 91ce88c6fa5ec..29fc5997b4c7c 100644 --- a/learning/tour-of-beam/learning-content/introduction/introduction-concepts/pipeline-concepts/setting-pipeline/python-example/task.py +++ b/learning/tour-of-beam/learning-content/introduction/introduction-concepts/pipeline-concepts/setting-pipeline/python-example/task.py @@ -69,18 +69,13 @@ def main(argv=None, save_main_session=True): pipeline_options.view_as(SetupOptions).save_main_session = save_main_session with beam.Pipeline(options=pipeline_options) as p: - - # Read the text file[pattern] into a PCollection. - lines = p | 'Read' >> ReadFromText(known_args.input) \ + # Read the text file[pattern] into a PCollection. + lines = p | 'Read' >> ReadFromText(known_args.input) \ | beam.Filter(lambda line: line != "") - - # Write the output using a "Write" transform that has side effects. - # pylint: disable=expression-not-assigned - output = lines | 'Write' >> WriteToText(known_args.output) - - - result = p.run() - result.wait_until_finish() + + # Write the output using a "Write" transform that has side effects. + # pylint: disable=expression-not-assigned + output = lines | 'Write' >> WriteToText(known_args.output) if __name__ == '__main__': diff --git a/learning/tour-of-beam/learning-content/io/big-query-io/read-query/go-example/main.go b/learning/tour-of-beam/learning-content/io/big-query-io/read-query/go-example/main.go index fec979ad7eda0..49ab6057bac25 100644 --- a/learning/tour-of-beam/learning-content/io/big-query-io/read-query/go-example/main.go +++ b/learning/tour-of-beam/learning-content/io/big-query-io/read-query/go-example/main.go @@ -18,9 +18,11 @@ // beam-playground: // name: read-query -// description: BigQuery read query example. +// description: BigQueryIO read query example. // multifile: false -// context_line: 40 +// context_line: 42 +// never_run: true +// always_run: true // categories: // - Quickstart // complexity: ADVANCED @@ -29,47 +31,49 @@ package main import ( - _ "context" - _ "flag" - _ "github.com/apache/beam/sdks/v2/go/pkg/beam" - _ "github.com/apache/beam/sdks/v2/go/pkg/beam/io/bigqueryio" - _ "github.com/apache/beam/sdks/v2/go/pkg/beam/log" - _ "github.com/apache/beam/sdks/v2/go/pkg/beam/x/beamx" - _ "github.com/apache/beam/sdks/v2/go/pkg/beam/x/debug" + "context" + "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/io/bigqueryio" + "github.com/apache/beam/sdks/v2/go/pkg/beam/log" + "github.com/apache/beam/sdks/v2/go/pkg/beam/transforms/top" + "github.com/apache/beam/sdks/v2/go/pkg/beam/x/beamx" + "github.com/apache/beam/sdks/v2/go/pkg/beam/x/debug" + + "cloud.google.com/go/bigquery" internal_log "log" - _ "reflect" + "reflect" ) -// Define the data model: The CommentRow struct is defined, which models one row of HackerNews comments. -//The bigquery tag in the struct field is used to map the struct field to the BigQuery column. -type CommentRow struct { - Text string `bigquery:"text"` +type Game struct { + GameID bigquery.NullString `bigquery:"gameId"` + GameNumber bigquery.NullInt64 `bigquery:"gameNumber"` + SeasonID bigquery.NullString `bigquery:"seasonId"` + Year bigquery.NullInt64 `bigquery:"year"` + Type bigquery.NullString `bigquery:"type"` + DayNight bigquery.NullString `bigquery:"dayNight"` + Duration bigquery.NullString `bigquery:"duration"` } -// Construct the BigQuery query: A constant query is defined that selects the text column -// from the bigquery-public-data.hacker_news.comments table for a certain time range. -const query = `SELECT text -FROM ` + "`bigquery-public-data.hacker_news.comments`" + ` -WHERE time_ts BETWEEN '2013-01-01' AND '2014-01-01' -LIMIT 1000 -` - func main() { internal_log.Println("Running Task") - /* - ctx := context.Background() - p := beam.NewPipeline() - s := p.Root() - project := "tess-372508" - // Build a PCollection by querying BigQuery. - rows := bigqueryio.Query(s, project, query, - reflect.TypeOf(CommentRow{}), bigqueryio.UseStandardSQL()) + ctx := context.Background() + p := beam.NewPipeline() + s := p.Root() + project := "apache-beam-testing" - debug.Print(s, rows) + // Build a PCollection by querying BigQuery. + rows := bigqueryio.Query(s, project, "select * from `bigquery-public-data.baseball.schedules`", + reflect.TypeOf(Game{}), bigqueryio.UseStandardSQL()) - // Now that the pipeline is fully constructed, we execute it. - if err := beamx.Run(ctx, p); err != nil { - log.Exitf(ctx, "Failed to execute job: %v", err) - }*/ + fixedSizeLines := top.Largest(s, rows, 5, less) + + debug.Print(s, fixedSizeLines) + // Now that the pipeline is fully constructed, we execute it. + if err := beamx.Run(ctx, p); err != nil { + log.Exitf(ctx, "Failed to execute job: %v", err) + } +} +func less(a, b Game) bool { + return true } diff --git a/learning/tour-of-beam/learning-content/io/big-query-io/read-query/java-example/Task.java b/learning/tour-of-beam/learning-content/io/big-query-io/read-query/java-example/Task.java index 256c70919ce77..12c1fbcd9b48f 100644 --- a/learning/tour-of-beam/learning-content/io/big-query-io/read-query/java-example/Task.java +++ b/learning/tour-of-beam/learning-content/io/big-query-io/read-query/java-example/Task.java @@ -27,11 +27,10 @@ // tags: // - hellobeam +import com.google.api.services.bigquery.model.TableRow; import org.apache.beam.sdk.Pipeline; -import org.apache.beam.sdk.coders.DoubleCoder; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryOptions; -import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.transforms.DoFn; @@ -40,52 +39,49 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class Task { +public class Task { private static final Logger LOG = LoggerFactory.getLogger(Task.class); - public static void main(String[] args) { - LOG.info("Running Task"); - System.setProperty("GOOGLE_APPLICATION_CREDENTIALS", "to\\path\\credential.json"); - PipelineOptions options = PipelineOptionsFactory.fromArgs(args).create(); - options.setTempLocation("gs://bucket"); - options.as(BigQueryOptions.class).setProject("project-id"); + private static final String WEATHER_SAMPLES_QUERY = + "select * from `clouddataflow-readonly.samples.weather_stations`"; - Pipeline pipeline = Pipeline.create(options); + public static void applyBigQueryTornadoes(Pipeline p) { + /*TypedRead bigqueryIO = + BigQueryIO.readTableRows() + .fromQuery(WEATHER_SAMPLES_QUERY) + .usingStandardSql(); - // pCollection.apply(BigQueryIO.read(... - This part of the pipeline reads from a BigQuery table using a SQL query and stores the result in a PCollection. - // The BigQueryIO.read() function is used to read from BigQuery. It is configured with a lambda function to extract a field from each record. - // The .fromQuery("SELECT field FROM project-id.dataset.table") - // specifies the SQL query used to read from BigQuery. You should replace "field", "project-id", "dataset", and "table" with your specific field name, project id, dataset name, and table name, respectively. -/* - PCollection pCollection = pipeline - .apply(BigQueryIO.read( - (SchemaAndRecord elem) -> (Double) elem.getRecord().get("field")) - .fromQuery( - "SELECT field FROM `project-id.dataset.table`") - .usingStandardSql() - .withCoder(DoubleCoder.of())); - pCollection - .apply("Log words", ParDo.of(new LogOutput<>())); -*/ - pipeline.run(); + PCollection rowsFromBigQuery = p.apply(bigqueryIO); + + rowsFromBigQuery + .apply(ParDo.of(new LogOutput<>("Result: ")));*/ + } + + public static void runBigQueryTornadoes(PipelineOptions options) { + Pipeline p = Pipeline.create(options); + applyBigQueryTornadoes(p); + p.run().waitUntilFinish(); + } + + public static void main(String[] args) { + PipelineOptions options = + PipelineOptionsFactory.fromArgs(args).withValidation().as(PipelineOptions.class); + runBigQueryTornadoes(options); } static class LogOutput extends DoFn { private final String prefix; - LogOutput() { - this.prefix = "Processing element"; - } - LogOutput(String prefix) { this.prefix = prefix; } @ProcessElement - public void processElement(ProcessContext c) throws Exception { - LOG.info(prefix + ": {}", c.element()); + public void processElement(ProcessContext c) { + LOG.info(prefix + c.element()); + c.output(c.element()); } } -} \ No newline at end of file +} diff --git a/learning/tour-of-beam/learning-content/io/big-query-io/read-query/python-example/task.py b/learning/tour-of-beam/learning-content/io/big-query-io/read-query/python-example/task.py index 6204635c630f3..fbb1e1e302d19 100644 --- a/learning/tour-of-beam/learning-content/io/big-query-io/read-query/python-example/task.py +++ b/learning/tour-of-beam/learning-content/io/big-query-io/read-query/python-example/task.py @@ -16,8 +16,10 @@ # beam-playground: # name: read-query -# description: TextIO read query example. +# description: BigQueryIO read query example. # multifile: false +# never_run: true +# always_run: true # context_line: 34 # categories: # - Quickstart @@ -26,39 +28,43 @@ # - hellobeam import argparse +import os +import warnings + import apache_beam as beam -from apache_beam.io import ReadFromText -from apache_beam.io import WriteToBigQuery -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.options.pipeline_options import PipelineOptions, GoogleCloudOptions, SetupOptions +from apache_beam.io.gcp.bigquery import ReadFromBigQueryRequest, ReadAllFromBigQuery + +class WeatherData: + def __init__(self, station_number, wban_number, year, month, day): + self.station_number = station_number + self.wban_number = wban_number + self.year = year + self.month = month + self.day = day + def __str__(self): + return f"Weather Data: Station {self.station_number} (WBAN {self.wban_number}), Date: {self.year}-{self.month}-{self.day}" def run(argv=None): parser = argparse.ArgumentParser() - parser.add_argument('--input', - dest='input', - default='gs://bucket', - help='Input file to process.') known_args, pipeline_args = parser.parse_known_args(argv) pipeline_options = PipelineOptions(pipeline_args) - pipeline_options.view_as(SetupOptions).save_main_session = True + pipeline_options.view_as(PipelineOptions) - """ - (p | 'ReadTable' >> ReadFromBigQuery(query='SELECT * FROM project-id.dataset.table') - This part of the - pipeline reads from a BigQuery table using a SQL query and processes the result. The ReadFromBigQuery( - query='SELECT * FROM project-id.dataset.table') function is used to read from BigQuery. 'LogOutput' >> - beam.Map(lambda elem: print(f"Processing element: {elem['field']}"))) - This part of the pipeline processes the - PCollection and logs the output to the console. It prints the 'field' column from each row in the table. - """ - with beam.Pipeline(options=pipeline_options) as p: - (p #| 'ReadTable' >> beam.io.Read(beam.io.BigQuerySource(query='SELECT * FROM `project-id.dataset.table`'))) - # Each row is a dictionary where the keys are the BigQuery columns - #| beam.Map(lambda elem: elem['field']) - ) + with beam.Pipeline(options=pipeline_options, argv=argv) as p: + (p + # | 'ReadFromBigQuery' >> beam.io.ReadFromBigQuery(query='select * from `apache-beam-testing.clouddataflow_samples.weather_stations`',use_standard_sql=True, + # method=beam.io.ReadFromBigQuery.Method.DIRECT_READ) + # | beam.combiners.Sample.FixedSizeGlobally(5) + # | beam.FlatMap(lambda line: line) + # | beam.Map(lambda element: WeatherData(element['station_number'],element['wban_number'],element['year'],element['month'],element['day'])) + # | beam.Map(print) + ) if __name__ == '__main__': - run() + run() diff --git a/learning/tour-of-beam/learning-content/io/big-query-io/read-table/description.md b/learning/tour-of-beam/learning-content/io/big-query-io/read-table/description.md index 23344989d0aa8..a3f1c1993d965 100644 --- a/learning/tour-of-beam/learning-content/io/big-query-io/read-table/description.md +++ b/learning/tour-of-beam/learning-content/io/big-query-io/read-table/description.md @@ -35,7 +35,7 @@ The `logOutput` struct is defined as a custom `DoFn` that implements the Process {{if (eq .Sdk "java")}} ``` PCollection pCollection = pipeline - .apply("ReadFromBigQuery", BigQueryIO.readTableRows().from("clouddataflow-readonly:samples.weather_stations").withMethod(TypedRead.Method.DIRECT_READ)) + .apply("ReadFromBigQuery", BigQueryIO.readTableRows().from("apache-beam-testing.samples.weather_stations").withMethod(TypedRead.Method.DIRECT_READ)) ``` The `BigQueryIO.readTableRows()` method is called to create a `BigQueryIO.Read` transform that will read data from a `BigQuery` table. diff --git a/learning/tour-of-beam/learning-content/io/big-query-io/read-table/go-example/main.go b/learning/tour-of-beam/learning-content/io/big-query-io/read-table/go-example/main.go index 1751beb191e77..ef2462f4de362 100644 --- a/learning/tour-of-beam/learning-content/io/big-query-io/read-table/go-example/main.go +++ b/learning/tour-of-beam/learning-content/io/big-query-io/read-table/go-example/main.go @@ -28,6 +28,7 @@ // complexity: ADVANCED // tags: // - hellobeam + package main import ( @@ -62,7 +63,7 @@ func main() { s := p.Root() project := "apache-beam-testing" - // Build a PCollection by querying BigQuery. + // Build a PCollection by querying BigQuery. rows := bigqueryio.Read(s, project, "bigquery-public-data:baseball.schedules", reflect.TypeOf(Game{})) fixedSizeLines := top.Largest(s, rows, 5, less) diff --git a/learning/tour-of-beam/learning-content/io/big-query-io/read-table/java-example/Task.java b/learning/tour-of-beam/learning-content/io/big-query-io/read-table/java-example/Task.java index 835954382a9d9..63f5afd23575e 100644 --- a/learning/tour-of-beam/learning-content/io/big-query-io/read-table/java-example/Task.java +++ b/learning/tour-of-beam/learning-content/io/big-query-io/read-table/java-example/Task.java @@ -35,7 +35,11 @@ import org.apache.beam.sdk.io.gcp.bigquery.BigQueryOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.sdk.transforms.*; +import org.apache.beam.sdk.transforms.Flatten; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.Sample; import org.apache.beam.sdk.values.PCollection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,7 +65,7 @@ public static void main(String[] args) { */ PCollection pCollection = pipeline - .apply("ReadFromBigQuery", BigQueryIO.readTableRows().from("clouddataflow-readonly:samples.weather_stations").withMethod(TypedRead.Method.DIRECT_READ)); + .apply("ReadFromBigQuery", BigQueryIO.readTableRows().from("apache-beam-testing.samples.weather_stations").withMethod(TypedRead.Method.DIRECT_READ)); final PTransform, PCollection>> sample = Sample.fixedSizeGlobally(5); diff --git a/learning/tour-of-beam/learning-content/io/big-query-io/read-table/python-example/task.py b/learning/tour-of-beam/learning-content/io/big-query-io/read-table/python-example/task.py index d57d9a145b874..e89779e5a26bd 100644 --- a/learning/tour-of-beam/learning-content/io/big-query-io/read-table/python-example/task.py +++ b/learning/tour-of-beam/learning-content/io/big-query-io/read-table/python-example/task.py @@ -16,7 +16,7 @@ # beam-playground: # name: read-table -# description: TextIO read table example. +# description: BigQueryIO read table example. # multifile: false # never_run: true # always_run: true @@ -28,13 +28,8 @@ # - hellobeam import argparse -import os -import warnings - import apache_beam as beam from apache_beam.options.pipeline_options import PipelineOptions, GoogleCloudOptions, SetupOptions -#from google.cloud import bigquery -from apache_beam.io.gcp.bigquery import ReadFromBigQueryRequest, ReadAllFromBigQuery class WeatherData: def __init__(self, station_number, wban_number, year, month, day): @@ -57,7 +52,7 @@ def run(argv=None): with beam.Pipeline(options=pipeline_options, argv=argv) as p: - (p | 'ReadFromBigQuery' >> beam.io.ReadFromBigQuery(table='apache-beam-testing:clouddataflow_samples.weather_stations', + (p | 'ReadFromBigQuery' >> beam.io.ReadFromBigQuery(table='apache-beam-testing:clouddataflow_samples.weather_stations', method=beam.io.ReadFromBigQuery.Method.DIRECT_READ) | beam.combiners.Sample.FixedSizeGlobally(5) | beam.FlatMap(lambda line: line) diff --git a/learning/tour-of-beam/learning-content/io/rest-api/description.md b/learning/tour-of-beam/learning-content/io/rest-api/description.md index 3ebdd2d0afe3c..8f7d9ed7f567b 100644 --- a/learning/tour-of-beam/learning-content/io/rest-api/description.md +++ b/learning/tour-of-beam/learning-content/io/rest-api/description.md @@ -31,7 +31,7 @@ PCollection weatherData = }) .fromQuery( "SELECT year, month, day, max_temperature " - + "FROM [clouddataflow-readonly:samples.weather_stations] " + + "FROM [apache-beam-testing.samples.weather_stations] " + "WHERE year BETWEEN 2007 AND 2009") .withCoder(AvroCoder.of(WeatherData.class))); diff --git a/learning/tour-of-beam/learning-content/splittable-dofn/splittable/unit-info.yaml b/learning/tour-of-beam/learning-content/splittable-dofn/splittable/unit-info.yaml index 73660fef37bab..c657dc64c351d 100644 --- a/learning/tour-of-beam/learning-content/splittable-dofn/splittable/unit-info.yaml +++ b/learning/tour-of-beam/learning-content/splittable-dofn/splittable/unit-info.yaml @@ -21,7 +21,7 @@ sdk: - Java - Python - Go -id: splittable-dofn +id: splittable name: Splittable doFn complexity: ADVANCED taskName: splittable-dofn diff --git a/learning/tour-of-beam/terraform/build.gradle.kts b/learning/tour-of-beam/terraform/build.gradle.kts index 8c320dd242a7c..30d30309273ca 100644 --- a/learning/tour-of-beam/terraform/build.gradle.kts +++ b/learning/tour-of-beam/terraform/build.gradle.kts @@ -49,7 +49,7 @@ tasks.register("terraformRef") { tasks.register("terraformApplyBackend") { group = "backend-deploy" - + val pg_router_host = if (project.extensions.extraProperties.has("pg_router_host")) { project.extensions.extraProperties["pg_router_host"] as String } else { @@ -64,7 +64,7 @@ tasks.register("terraformApplyBackend") { "-var=project_id=$(gcloud config get-value project)", "-var-file=./common.tfvars" ) - + tasks.getByName("uploadLearningMaterials").mustRunAfter(this) } @@ -174,7 +174,7 @@ tasks.register("firebaseHostingCreate") { }.assertNormalExitValue() println("Firebase hosting site has been added to project $projectId.") } - + exec { executable("firebase") args("target:apply", "hosting", webapp_id , webapp_id) @@ -184,12 +184,12 @@ tasks.register("firebaseHostingCreate") { val file = project.file("../frontend/firebase.json") val content = file.readText() - + val oldContent = """"public": "build/web",""" val newContent = """"public": "build/web", "target": "$webapp_id",""" val updatedContent = content.replace(oldContent, newContent) - + file.writeText(updatedContent) } } diff --git a/local-env-setup.sh b/local-env-setup.sh index cc041e872cd39..6cd1092023a59 100755 --- a/local-env-setup.sh +++ b/local-env-setup.sh @@ -55,7 +55,7 @@ if [ "$kernelname" = "Linux" ]; then exit fi - for ver in 3.7 3.8 3.9 3.10 3; do + for ver in 3.8 3.9 3.10 3.11 3; do apt install --yes python$ver-venv done @@ -89,7 +89,7 @@ elif [ "$kernelname" = "Darwin" ]; then echo "Installing openjdk@8" brew install openjdk@8 fi - for ver in 3.7 3.8 3.9; do + for ver in 3.8 3.9 3.10 3.11; do if brew ls --versions python@$ver > /dev/null; then echo "python@$ver already installed. Skipping" brew info python@$ver diff --git a/model/OWNERS b/model/OWNERS deleted file mode 100644 index a5b32dc72cff4..0000000000000 --- a/model/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - herohde - - lukecwik diff --git a/model/fn-execution/OWNERS b/model/fn-execution/OWNERS deleted file mode 100644 index 23065b82b3f3a..0000000000000 --- a/model/fn-execution/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - herohde - - robertwb - - lukecwik diff --git a/model/fn-execution/src/main/proto/org/apache/beam/model/fn_execution/v1/beam_fn_api.proto b/model/fn-execution/src/main/proto/org/apache/beam/model/fn_execution/v1/beam_fn_api.proto index ec6c176e5a376..777ce8636b7be 100644 --- a/model/fn-execution/src/main/proto/org/apache/beam/model/fn_execution/v1/beam_fn_api.proto +++ b/model/fn-execution/src/main/proto/org/apache/beam/model/fn_execution/v1/beam_fn_api.proto @@ -46,6 +46,52 @@ import "google/protobuf/struct.proto"; import "google/protobuf/timestamp.proto"; import "google/protobuf/duration.proto"; + +// Describes transforms necessary to execute Beam over the FnAPI but are +// implementation details rather than part of the core model. +message FnApiTransforms { + enum Runner { + // DataSource is a Root Transform, and a source of data for downstream + // transforms in the same ProcessBundleDescriptor. + // It represents a stream of values coming in from an external source/over + // a data channel, typically from the runner. It's not the PCollection itself + // but a description of how to get the portion of the PCollection for a given + // bundle. + // + // The DataSource transform is implemented in each SDK and not explicitly + // provided during pipeline construction. A runner inserts the transform + // in ProcessBundleDescriptors to indicate where the bundle + // can retrieve data for an associated ProcessBundleRequest. + // Data for the same request will be retrieved with the matching instruction ID, + // and transform ID determined by the runner. + // + // The DataSource transform will take a stream of bytes from the remote + // source for the matching instruction ID and decode them as windowed + // values using the provided coder ID, which must be a windowed value coder. + // + // Payload: RemoteGrpcPort + DATA_SOURCE = 0 [(org.apache.beam.model.pipeline.v1.beam_urn) = "beam:runner:source:v1"]; + + // DataSink is a transform that sends PCollection elements to a remote + // port using the Data API. + // + // The DataSink transform is implemented in each SDK and not explicitly + // provided during pipeline construction. A runner inserts the transform in + // ProcessBundleDescriptors to indicate where the bundle can send + // data for each associated ProcessBundleRequest. Data for the same + // request will be sent with the matching instruction ID and transform ID. + // Each PCollection that exits the ProcessBundleDescriptor subgraph will have + // it's own DataSink, keyed by a transform ID determined by the runner. + // + // The DataSink will take in a stream of elements for a given instruction ID + // and encode them for transmission to the remote sink. The coder ID must be + // for a windowed value coder. + // + // Payload: RemoteGrpcPort + DATA_SINK = 1 [(org.apache.beam.model.pipeline.v1.beam_urn) = "beam:runner:sink:v1"]; + } +} + // A descriptor for connecting to a remote port using the Beam Fn Data API. // Allows for communication between two environments (for example between the // runner and the SDK). @@ -155,20 +201,32 @@ message SampleDataRequest { repeated string pcollection_ids = 1; } - // An element sampled when the SDK is processing a bundle. This is a proto // message to allow for additional per-element metadata. message SampledElement { - // Required. Sampled raw bytes for an element. This is a + // (Required) Sampled raw bytes for an element. This is a // single encoded element in the nested context. bytes element = 1; - // FUTURE WORK: Capture lull detections and exceptions. - // - // Optional. Present if there was an exception - // processing the above element. - // - // LogEntry exception_entry = 2; + // (Required) Timestamp of when the sample was taken. + google.protobuf.Timestamp sample_timestamp = 2; + + message Exception { + // (Required) The instruction ID of the associated ProcessBundleRequest. + string instruction_id = 1; + + // (Required) The transform ID of the executing PTransform during the + // exception. + string transform_id = 2; + + // (Required) The error message to be displayed to the user. Can use the + // other fields to query for contextual logs. + string error = 3; + } + + // (Optional) This will be set if this element was sampled because of a user + // exception. + Exception exception = 3; } // If supported, the `SampleDataResponse` will contain samples from PCollections @@ -181,27 +239,6 @@ message SampleDataResponse { // Map from PCollection id to sampled elements. map element_samples = 1; - - // FUTURE WORK: Investigate ways of storing multiple interesting types of - // sampled elements. There are two ways of accomplishing this: - // 1) Maps of typed elements: include multiple maps here with typed element - // proto messages, ex. - // - // message SlowElement {...} - // message ErroredElement {...} - // map slow_elements - // map errored_elements - // - // However, this forces an element into a single category. It disallows - // classification across multiple characteristics (like a slow and errored - // element). - // - // 2) Compositional types: allow for URN and payloads on the base - // SampledElement message for interesting characteristics. The base class - // can then be queried for specific URNs. This allows for multiple - // attributes on the same element. - // - // For a longer conversation, see https://github.com/apache/beam/pull/25065. } // A request to provide full MonitoringInfo associated with the entire SDK @@ -530,30 +567,57 @@ message ProcessBundleSplitRequest { // first_residual_element. // - The current bundle, if no further splits happen, will have done exactly // the work under primary_roots and all elements up to and including the -// channel splits last_primary_element. +// channel split's last_primary_element. // // This allows the SDK to relinquish ownership of and commit to not process some // of the elements that it may have been sent (the residual) while retaining // ownership and commitment to finish the other portion (the primary). // -// For example, lets say the SDK is processing elements A B C D E and a split -// request comes in. The SDK could return a response with a channel split -// representing a last_primary_element of 3 (D) and first_residual_element of 4 -// (E). The SDK is now responsible for processing A B C D and the runner must -// process E in the future. A future split request could have the SDK split the -// elements B into B1 and B2 and C into C1 and C2 representing their primary and -// residual roots. The SDK would return a response with a channel split -// representing a last_primary_element of 0 (A) and first_residual_element of 3 -// (D) with primary_roots (B1, C1) and residual_roots (B2, C2). The SDK is now -// responsible for processing A B1 C1 and the runner must process C2 D2 (and E -// from the prior split) in the future. Yet another future split request could -// have the SDK could split B1 further into B1a and B1b primary and residuals -// and return C2 as a residual (assuming C2 was left unprocessed). The SDK would -// return a response with a channel split representing a last_primary_element of -// 0 (A) and first_residual_element of 4 (E) with primary_roots (B1a) and -// residual_roots (B1b, C1). The SDK is now responsible for processing A B1a the -// runner must process B1b C1 (in addition to C2, D, E from prior splits) in the -// future. +// Example with three splits of a single bundle: +// Let's say the SDK is processing elements [A B C D E]. These elements make +// up the 0-indexed channel. +// +// ** First Split ** +// Channel Split = [ A B C D <> E ] +// Primary Roots = [] (No elements were split) +// Residual Roots = [] +// +// Say a split request comes in. The SDK could return a response with a channel +// split representing a last_primary_element of 3 (D) and +// first_residual_element of 4 (E). The SDK is now responsible for processing A +// B C D and the runner must process E in the future. +// +// (A B C D) | (E) +// +// ** Second Split ** +// Channel Split = [ A < B C > D E ] +// Primary Roots = [B1 C1] +// Residual Roots = [B2 C2] +// +// A future split request could have the SDK split the elements B into B1 and +// B2 and C into C1 and C2 representing their primary and residual roots. The +// +// (A B1 C1) | (B2 C2 D) +// +// SDK would return a response with a channel split representing a +// last_primary_element of 0 (A) and first_residual_element of 3 (D) with +// primary_roots (B1, C1) and residual_roots (B2, C2). The SDK is now +// responsible for processing A B1 C1 and the runner must process B2 C2 D (and +// E from the prior split) in the future. +// +// ** Third Split ** +// Channel Split = [ A < B C > D E ] +// Primary Roots = [B1a] +// Residual Roots [B1b C1] +// Yet another future split request could have the SDK could split B1 further +// into B1a and B1b primary and residuals and return C1 as a residual (assuming +// C1 was left unprocessed). The SDK would return a response with a channel +// split representing a last_primary_element of 0 (A) and +// first_residual_element of 3 (E) with primary_roots (B1a) and residual_roots +// (B1b, C1). The SDK is now responsible for processing A B1a the runner must +// process B1b C1 (in addition to C2, D, E from prior splits) in the future. +// +// (A B1a) | (B1b C1) // // For more rigorous definitions see https://s.apache.org/beam-breaking-fusion message ProcessBundleSplitResponse { diff --git a/model/interactive/OWNERS b/model/interactive/OWNERS deleted file mode 100644 index 5413f506e36d8..0000000000000 --- a/model/interactive/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - srohde diff --git a/model/job-management/OWNERS b/model/job-management/OWNERS deleted file mode 100644 index 23065b82b3f3a..0000000000000 --- a/model/job-management/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - herohde - - robertwb - - lukecwik diff --git a/model/pipeline/OWNERS b/model/pipeline/OWNERS deleted file mode 100644 index 23065b82b3f3a..0000000000000 --- a/model/pipeline/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - herohde - - robertwb - - lukecwik diff --git a/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/beam_runner_api.proto b/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/beam_runner_api.proto index 2483103b5794d..db958f183c453 100644 --- a/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/beam_runner_api.proto +++ b/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/beam_runner_api.proto @@ -1982,5 +1982,9 @@ message StandardResourceHints { // SDKs should convert the size to bytes, but can allow users to specify human-friendly units (e.g. GiB). // Payload: ASCII encoded string of the base 10 representation of an integer number of bytes. MIN_RAM_BYTES = 1 [(beam_urn) = "beam:resources:min_ram_bytes:v1"]; + // Describes desired number of CPUs available in transform's execution environment. + // SDKs should accept and validate a positive integer count. + // Payload: ASCII encoded string of the base 10 representation of an integer number of CPUs. + CPU_COUNT = 2 [(beam_urn) = "beam:resources:cpu_count:v1"]; } } diff --git a/ownership/JAVA_DEPENDENCY_OWNERS.yaml b/ownership/JAVA_DEPENDENCY_OWNERS.yaml deleted file mode 100644 index eb43b8a49fd68..0000000000000 --- a/ownership/JAVA_DEPENDENCY_OWNERS.yaml +++ /dev/null @@ -1,1013 +0,0 @@ -#!/usr/bin/env python -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Beam Java SDK dependency ownership -# Please update if new dependencies are introduced to the Beam. -# Sign up with you JIRA username to take the ownership of a package. -# Separate names by comma. - ---- - -deps: - args4j:args4j: - group: args4j - artifact: args4j - owners: - - biz.aQute:bndlib: - group: biz.aQute - artifact: bndlib - owners: - - ca.coglinc:javacc-gradle-plugin: - group: ca.coglinc - artifact: javacc-gradle-plugin - owners: - - com.alibaba:fastjson: - group: com.alibaba - artifact: fastjson - owners: - - com.amazonaws:amazon-kinesis-client: - group: com.amazonaws - artifact: amazon-kinesis-client - owners: aromanenko-dev - - com.amazonaws:amazon-kinesis-producer: - group: com.amazonaws - artifact: amazon-kinesis-producer - owners: aromanenko-dev - - com.amazonaws:aws-java-sdk-cloudwatch: - group: com.amazonaws - artifact: aws-java-sdk-cloudwatch - owners: - - com.amazonaws:aws-java-sdk-core: - group: com.amazonaws - artifact: aws-java-sdk-core - owners: - - com.amazonaws:aws-java-sdk-kinesis: - group: com.amazonaws - artifact: aws-java-sdk-kinesis - owners: - - com.amazonaws:aws-java-sdk-s3: - group: com.amazonaws - artifact: aws-java-sdk-s3 - owners: - - com.carrotsearch.randomizedtesting:randomizedtesting-runner: - group: com.carrotsearch.randomizedtesting - artifact: randomizedtesting-runner - owners: - - com.clearspring.analytics:stream: - group: com.clearspring.analytics - artifact: stream - owners: - - com.commercehub.gradle.plugin:gradle-avro-plugin: - group: com.commercehub.gradle.plugin - artifact: gradle-avro-plugin - owners: - - com.datastax.cassandra:cassandra-driver-core: - group: com.datastax.cassandra - artifact: cassandra-driver-core - owners: - - com.datastax.cassandra:cassandra-driver-mapping: - group: com.datastax.cassandra - artifact: cassandra-driver-mapping - owners: - - com.diffplug.spotless:spotless-plugin-gradle: - group: com.diffplug.spotless - artifact: spotless-plugin-gradle - owners: - - com.esotericsoftware.kryo:kryo: - group: com.esotericsoftware.kryo - artifact: kryo - owners: - - com.fasterxml.jackson.core:jackson-annotations: - group: com.fasterxml.jackson.core - artifact: jackson-annotations - owners: - - com.fasterxml.jackson.core:jackson-core: - group: com.fasterxml.jackson.core - artifact: jackson-core - owners: - - com.fasterxml.jackson.core:jackson-databind: - group: com.fasterxml.jackson.core - artifact: jackson-databind - owners: - - com.fasterxml.jackson.dataformat:jackson-dataformat-cbor: - group: com.fasterxml.jackson.dataformat - artifact: jackson-dataformat-cbor - owners: - - com.fasterxml.jackson.dataformat:jackson-dataformat-yaml: - group: com.fasterxml.jackson.dataformat - artifact: jackson-dataformat-yaml - owners: - - com.fasterxml.jackson.datatype:jackson-datatype-joda: - group: com.fasterxml.jackson.datatype - artifact: jackson-datatype-joda - owners: - - com.fasterxml.jackson.module:jackson-module-scala_2.11: - group: com.fasterxml.jackson.module - artifact: jackson-module-scala_2.11 - owners: - - com.github.jengelman.gradle.plugins:shadow: - group: com.github.jengelman.gradle.plugins - artifact: shadow - owners: - - com.google.api:gax-grpc: - group: com.google.api - artifact: gax-grpc - owners: - - com.google.api:api-common: - group: com.google.api - artifact: api-common - owners: - - com.google.api-client:google-api-client: - group: com.google.api-client - artifact: google-api-client - owners: - - com.google.api-client:google-api-client-jackson2: - group: com.google.api-client - artifact: google-api-client-jackson2 - owners: - - com.google.api-client:google-api-client-java6: - group: com.google.api-client - artifact: google-api-client-java6 - owners: - - com.google.api.grpc:grpc-google-cloud-pubsub-v1: - group: com.google.api.grpc - artifact: grpc-google-cloud-pubsub-v1 - owners: - - com.google.api.grpc:proto-google-cloud-pubsub-v1: - group: com.google.api.grpc - artifact: proto-google-cloud-pubsub-v1 - owners: - - com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1: - group: com.google.api.grpc - artifact: proto-google-cloud-spanner-admin-database-v1 - owners: - - com.google.api.grpc:proto-google-common-protos: - group: com.google.api.grpc - artifact: proto-google-common-protos - owners: - - com.google.apis:google-api-services-bigquery: - group: com.google.apis - artifact: google-api-services-bigquery - owners: chamikara - - com.google.apis:google-api-services-clouddebugger: - group: com.google.apis - artifact: google-api-services-clouddebugger - owners: chamikara - - com.google.apis:google-api-services-cloudresourcemanager: - group: com.google.apis - artifact: google-api-services-cloudresourcemanager - owners: chamikara - - com.google.apis:google-api-services-dataflow: - group: com.google.apis - artifact: google-api-services-dataflow - owners: chamikara - - com.google.apis:google-api-services-pubsub: - group: com.google.apis - artifact: google-api-services-pubsub - owners: chamikara - - com.google.apis:google-api-services-storage: - group: com.google.apis - artifact: google-api-services-storage - owners: chamikara - - com.google.auth:google-auth-library-credentials: - group: com.google.auth - artifact: google-auth-library-credentials - owners: - - com.google.auth:google-auth-library-oauth2-http: - group: com.google.auth - artifact: google-auth-library-oauth2-http - owners: - - com.google.auto.service:auto-service: - group: com.google.auto.service - artifact: auto-service - owners: - - com.google.auto.value:auto-value: - group: com.google.auto.value - artifact: auto-value - owners: - - com.google.cloud:google-cloud-core: - group: com.google.cloud - artifact: google-cloud-core - owners: - - com.google.cloud:google-cloud-core-grpc: - group: com.google.cloud - artifact: google-cloud-core-grpc - owners: - - com.google.cloud:google-cloud-spanner: - group: com.google.cloud - artifact: google-cloud-spanner - owners: - - com.google.cloud.bigdataoss:gcsio: - group: com.google.cloud.bigdataoss - artifact: gcsio - owners: - - com.google.cloud.bigdataoss:util: - group: com.google.cloud.bigdataoss - artifact: util - owners: - - com.google.cloud.bigtable:bigtable-client-core: - group: com.google.cloud.bigtable - artifact: bigtable-client-core - owners: - - com.google.cloud.bigtable:bigtable-protos: - group: com.google.cloud.bigtable - artifact: bigtable-protos - owners: - - com.google.cloud.dataflow:google-cloud-dataflow-java-proto-library-all: - group: com.google.cloud.dataflow - artifact: google-cloud-dataflow-java-proto-library-all - owners: - - com.google.cloud.datastore:datastore-v1-proto-client: - group: com.google.cloud.datastore - artifact: datastore-v1-proto-client - owners: - - com.google.cloud.datastore:datastore-v1-protos: - group: com.google.cloud.datastore - artifact: datastore-v1-protos - owners: - - com.google.code.findbugs:findbugs-annotations: - group: com.google.code.findbugs - artifact: findbugs-annotations - owners: - - com.google.code.findbugs:jsr305: - group: com.google.code.findbugs - artifact: jsr305 - owners: - - com.google.errorprone:error_prone_annotations: - group: com.google.errorprone - artifact: error_prone_annotations - owners: - - com.google.guava:guava: - group: com.google.guava - artifact: guava - owners: - - com.google.guava:guava-testlib: - group: com.google.guava - artifact: guava-testlib - owners: - - com.google.http-client:google-http-client: - group: com.google.http-client - artifact: google-http-client - owners: - - com.google.http-client:google-http-client-jackson2: - group: com.google.http-client - artifact: google-http-client-jackson2 - owners: - - com.google.http-client:google-http-client-jackson: - group: com.google.http-client - artifact: google-http-client-jackson - owners: - - com.google.http-client:google-http-client-protobuf: - group: com.google.http-client - artifact: google-http-client-protobuf - owners: - - com.google.oauth-client:google-oauth-client-java6: - group: com.google.oauth-client - artifact: google-oauth-client-java6 - owners: - - com.google.oauth-client:google-oauth-client: - group: com.google.oauth-client - artifact: google-oauth-client - owners: - - com.google.protobuf:protobuf-gradle-plugin: - group: com.google.protobuf - artifact: protobuf-gradle-plugin - owners: pabloem - - com.google.protobuf:protobuf-java: - group: com.google.protobuf - artifact: protobuf-java - owners: chamikara - - com.google.protobuf:protobuf-java-util: - group: com.google.protobuf - artifact: protobuf-java-util - owners: chamikara - - com.google.protobuf:protoc: - group: com.google.protobuf - artifact: protoc - owners: chamikara - - com.gradle:build-scan-plugin: - group: com.gradle - artifact: build-scan-plugin - owners: - - com.pholser:junit-quickcheck-core: - group: com.pholser - artifact: junit-quickcheck-core - owners: - - com.tdunning:t-digest: - group: com.tdunning - artifact: t-digest - owners: - - com.typesafe:config: - group: com.typesafe - artifact: config - owners: - - commons-cli:commons-cli: - group: commons-cli - artifact: commons-cli - owners: - - commons-codec:commons-codec: - group: commons-codec - artifact: commons-codec - owners: - - commons-io:commons-io: - group: commons-io - artifact: commons-io - owners: - - cz.malohlava:visteg: - group: cz.malohlava - artifact: visteg - owners: - - de.flapdoodle.embed:de.flapdoodle.embed.mongo: - group: de.flapdoodle.embed - artifact: de.flapdoodle.embed.mongo - owners: - - de.flapdoodle.embed:de.flapdoodle.embed.process: - group: de.flapdoodle.embed - artifact: de.flapdoodle.embed.process - owners: - - gradle.plugin.com.github.blindpirate:gogradle: - group: gradle.plugin.com.github.blindpirate - artifact: gogradle - owners: - - gradle.plugin.com.palantir.gradle.docker:gradle-docker: - group: gradle.plugin.com.palantir.gradle.docker - artifact: gradle-docker - owners: - - gradle.plugin.io.pry.gradle.offline_dependencies:gradle-offline-dependencies-plugin: - group: gradle.plugin.io.pry.gradle.offline_dependencies - artifact: gradle-offline-dependencies-plugin - owners: - - gradle.plugin.org.nosphere.apache:creadur-rat-gradle: - group: gradle.plugin.org.nosphere.apache - artifact: creadur-rat-gradle - owners: - - io.dropwizard.metrics:metrics-core: - group: io.dropwizard.metrics - artifact: metrics-core - owners: - - io.grpc:grpc-all: - group: io.grpc - artifact: grpc-all - owners: chamikara - - io.grpc:grpc-auth: - group: io.grpc - artifact: grpc-auth - owners: chamikara - - io.grpc:grpc-core: - group: io.grpc - artifact: grpc-core - owners: chamikara - - io.grpc:grpc-netty: - group: io.grpc - artifact: grpc-netty - owners: chamikara - - io.grpc:grpc-protobuf: - group: io.grpc - artifact: grpc-protobuf - owners: chamikara - - io.grpc:grpc-stub: - group: io.grpc - artifact: grpc-stub - owners: chamikara - - io.grpc:protoc-gen-grpc-java: - group: io.grpc - artifact: protoc-gen-grpc-java - owners: chamikara - - io.netty:netty-all: - group: io.netty - artifact: netty-all - owners: chamikara - - io.netty:netty-handler: - group: io.netty - artifact: netty-handler - owners: chamikara - - io.netty:netty-tcnative-boringssl-static: - group: io.netty - artifact: netty-tcnative-boringssl-static - owners: chamikara - - io.netty:netty-transport-native-epoll: - group: io.netty - artifact: netty-transport-native-epoll - owners: chamikara - - io.opencensus:opencensus-api: - group: io.opencensus - artifact: opencensus-api - owners: - - io.opencensus:opencensus-contrib-grpc-metrics: - group: io.opencensus - artifact: opencensus-contrib-grpc-metrics - owners: - - javax.xml.bind:jaxb-api: - group: javax.xml.bind - artifact: jaxb-api - owners: - - jline:jline: - group: jline - artifact: jline - owners: kenn,kedin,apilloud,amaliujia - - joda-time:joda-time: - group: joda-time - artifact: joda-time - owners: - - net.bytebuddy:byte-buddy: - group: net.bytebuddy - artifact: byte-buddy - owners: - - net.java.dev.javacc:javacc: - group: net.java.dev.javacc - artifact: javacc - owners: - - net.java.dev.jna:jna: - group: net.java.dev.jna - artifact: jna - owners: - - net.ltgt.gradle:gradle-errorprone-plugin: - group: net.ltgt.gradle - artifact: gradle-errorprone-plugin - owners: - - net.researchgate:gradle-release: - group: net.researchgate - artifact: gradle-release - owners: pabloem - - org.apache.activemq:activemq-amqp: - group: org.apache.activemq - artifact: activemq-amqp - owners: - - org.apache.activemq:activemq-broker: - group: org.apache.activemq - artifact: activemq-broker - owners: - - org.apache.activemq:activemq-client: - group: org.apache.activemq - artifact: activemq-client - owners: - - org.apache.activemq:activemq-jaas: - group: org.apache.activemq - artifact: activemq-jaas - owners: - - org.apache.activemq:activemq-kahadb-store: - group: org.apache.activemq - artifact: activemq-kahadb-store - owners: - - org.apache.activemq:activemq-mqtt: - group: org.apache.activemq - artifact: activemq-mqtt - owners: - - org.apache.activemq.tooling:activemq-junit: - group: org.apache.activemq.tooling - artifact: activemq-junit - owners: - - org.apache.avro:avro: - group: org.apache.avro - artifact: avro - owners: - - org.apache.calcite:calcite-core: - group: org.apache.calcite - artifact: calcite-core - owners: kenn,kedin,apilloud,amaliujia,mingmxu - - org.apache.calcite:calcite-linq4j: - group: org.apache.calcite - artifact: calcite-linq4j - owners: kenn,kedin,apilloud,amaliujia,mingmxu - - org.apache.calcite.avatica:avatica-core: - group: org.apache.calcite.avatica - artifact: avatica-core - owners: kenn,kedin,apilloud,amaliujia,mingmxu - - org.apache.cassandra:cassandra-all: - group: org.apache.cassandra - artifact: cassandra-all - owners: - - org.apache.commons:commons-compress: - group: org.apache.commons - artifact: commons-compress - owners: - - org.apache.commons:commons-csv: - group: org.apache.commons - artifact: commons-csv - owners: kenn,kedin,apilloud,amaliujia,mingmxu - - org.apache.commons:commons-dbcp2: - group: org.apache.commons - artifact: commons-dbcp2 - owners: - - org.apache.commons:commons-lang3: - group: org.apache.commons - artifact: commons-lang3 - owners: - - org.apache.derby:derby: - group: org.apache.derby - artifact: derby - owners: - - org.apache.derby:derbyclient: - group: org.apache.derby - artifact: derbyclient - owners: - - org.apache.derby:derbynet: - group: org.apache.derby - artifact: derbynet - owners: - - org.apache.flink:flink-clients_2.11: - group: org.apache.flink - artifact: flink-clients_2.11 - owners: - - org.apache.flink:flink-core: - group: org.apache.flink - artifact: flink-core - owners: - - org.apache.flink:flink-java: - group: org.apache.flink - artifact: flink-java - owners: - - org.apache.flink:flink-metrics-core: - group: org.apache.flink - artifact: flink-metrics-core - owners: - - org.apache.flink:flink-runtime_2.11: - group: org.apache.flink - artifact: flink-runtime_2.11 - owners: - - org.apache.flink:flink-streaming-java_2.11: - group: org.apache.flink - artifact: flink-streaming-java_2.11 - owners: - - org.apache.flink:flink-test-utils_2.11: - group: org.apache.flink - artifact: flink-test-utils_2.11 - owners: - - org.apache.hadoop:hadoop-client: - group: org.apache.hadoop - artifact: hadoop-client - owners: timrobertson100 - - org.apache.hadoop:hadoop-common: - group: org.apache.hadoop - artifact: hadoop-common - owners: timrobertson100 - - org.apache.hadoop:hadoop-hdfs: - group: org.apache.hadoop - artifact: hadoop-hdfs - owners: timrobertson100 - - org.apache.hadoop:hadoop-mapreduce-client-core: - group: org.apache.hadoop - artifact: hadoop-mapreduce-client-core - owners: timrobertson100 - - org.apache.hadoop:hadoop-minicluster: - group: org.apache.hadoop - artifact: hadoop-minicluster - owners: timrobertson100 - - org.apache.hbase:hbase-common: - group: org.apache.hbase - artifact: hbase-common - owners: timrobertson100 - - org.apache.hbase:hbase-hadoop-compat: - group: org.apache.hbase - artifact: hbase-hadoop-compat - owners: timrobertson100 - - org.apache.hbase:hbase-hadoop2-compat: - group: org.apache.hbase - artifact: hbase-hadoop2-compat - owners: timrobertson100 - - org.apache.hbase:hbase-server: - group: org.apache.hbase - artifact: hbase-server - owners: timrobertson100 - - org.apache.hbase:hbase-shaded-client: - group: org.apache.hbase - artifact: hbase-shaded-client - owners: timrobertson100 - - org.apache.hbase:hbase-shaded-server: - group: org.apache.hbase - artifact: hbase-shaded-server - owners: timrobertson100 - - org.apache.hive:hive-cli: - group: org.apache.hive - artifact: hive-cli - owners: timrobertson100 - - org.apache.hive:hive-common: - group: org.apache.hive - artifact: hive-common - owners: timrobertson100 - - org.apache.hive:hive-exec: - group: org.apache.hive - artifact: hive-exec - owners: timrobertson100 - - org.apache.hive.hcatalog:hive-hcatalog-core: - group: org.apache.hive.hcatalog - artifact: hive-hcatalog-core - owners: timrobertson100 - - org.apache.httpcomponents:httpasyncclient: - group: org.apache.httpcomponents - artifact: httpasyncclient - owners: - - org.apache.httpcomponents:httpclient: - group: org.apache.httpcomponents - artifact: httpclient - owners: - - org.apache.httpcomponents:httpcore: - group: org.apache.httpcomponents - artifact: httpcore - owners: - - org.apache.httpcomponents:httpcore-nio: - group: org.apache.httpcomponents - artifact: httpcore-nio - owners: - - org.apache.kafka:kafka_2.11: - group: org.apache.kafka - artifact: kafka_2.11 - owners: - - org.apache.kafka:kafka-clients: - group: org.apache.kafka - artifact: kafka-clients - owners: - - org.apache.kudu:kudu-client: - group: org.apache.kudu - artifact: kudu-client - owners: - - org.apache.logging.log4j:log4j-api: - group: org.apache.logging.log4j - artifact: log4j-api - owners: - - org.apache.logging.log4j:log4j-core: - group: org.apache.logging.log4j - artifact: log4j-core - owners: - - org.apache.parquet:parquet-avro: - group: org.apache.parquet - artifact: parquet-avro - owners: - - org.apache.parquet:parquet-common: - group: org.apache.parquet - artifact: parquet-common - owners: - - org.apache.parquet:parquet-hadoop: - group: org.apache.parquet - artifact: parquet-hadoop - owners: timrobertson100 - - org.apache.qpid:proton-j: - group: org.apache.qpid - artifact: proton-j - owners: - - org.apache.rat:apache-rat-tasks: - group: org.apache.rat - artifact: apache-rat-tasks - owners: - - org.apache.solr:solr-core: - group: org.apache.solr - artifact: solr-core - owners: timrobertson100 - - org.apache.solr:solr-solrj: - group: org.apache.solr - artifact: solr-solrj - owners: timrobertson100 - - org.apache.solr:solr-test-framework: - group: org.apache.solr - artifact: solr-test-framework - owners: timrobertson100 - - org.apache.spark:spark-core_2.11: - group: org.apache.spark - artifact: spark-core_2.11 - owners: - - org.apache.spark:spark-network-common_2.11: - group: org.apache.spark - artifact: spark-network-common_2.11 - owners: - - org.apache.spark:spark-streaming_2.11: - group: org.apache.spark - artifact: spark-streaming_2.11 - owners: - - org.apache.tika:tika-core: - group: org.apache.tika - artifact: tika-core - owners: - - org.apache.tika:tika-parsers: - group: org.apache.tika - artifact: tika-parsers - owners: - - org.apache.zookeeper:zookeeper: - group: org.apache.zookeeper - artifact: zookeeper - owners: timrobertson100 - - org.assertj:assertj-core: - group: org.assertj - artifact: assertj-core - owners: - - org.codehaus.groovy:groovy-all: - group: org.codehaus.groovy - artifact: groovy-all - owners: - - org.codehaus.woodstox:stax2-api: - group: org.codehaus.woodstox - artifact: stax2-api - owners: - - org.codehaus.woodstox:woodstox-core-asl: - group: org.codehaus.woodstox - artifact: woodstox-core-asl - owners: - - org.elasticsearch:elasticsearch: - group: org.elasticsearch - artifact: elasticsearch - owners: echauchot, timrobertson100 - - org.elasticsearch:elasticsearch-hadoop: - group: org.elasticsearch - artifact: elasticsearch-hadoop - owners: timrobertson100 - - org.elasticsearch.client:elasticsearch-rest-client: - group: org.elasticsearch.client - artifact: elasticsearch-rest-client - owners: echauchot, timrobertson100 - - org.elasticsearch.client:transport: - group: org.elasticsearch.client - artifact: transport - owners: echauchot, timrobertson100 - - org.elasticsearch.plugin:transport-netty4-client: - group: org.elasticsearch.plugin - artifact: transport-netty4-client - owners: echauchot, timrobertson100 - - org.elasticsearch.test:framework: - group: org.elasticsearch.test - artifact: framework - owners: echauchot, timrobertson100 - - org.freemarker:freemarker: - group: org.freemarker - artifact: freemarker - owners: - - org.fusesource.mqtt-client:mqtt-client: - group: org.fusesource.mqtt-client - artifact: mqtt-client - owners: - - org.hamcrest:hamcrest-core: - group: org.hamcrest - artifact: hamcrest-core - owners: - - org.hamcrest:hamcrest-library: - group: org.hamcrest - artifact: hamcrest-library - owners: - - org.mockito:mockito-core: - group: org.mockito - artifact: mockito-core - owners: - - org.mongodb:mongo-java-driver: - group: org.mongodb - artifact: mongo-java-driver - owners: - - org.postgresql:postgresql: - group: org.postgresql - artifact: postgresql - owners: timrobertson100 - - org.powermock:powermock-mockito-release-full: - group: org.powermock - artifact: powermock-mockito-release-full - owners: boyuanzz - - org.scala-lang:scala-library: - group: org.scala-lang - artifact: scala-library - owners: - - org.slf4j:slf4j-api: - group: org.slf4j - artifact: slf4j-api - owners: - - org.slf4j:slf4j-jdk14: - group: org.slf4j - artifact: slf4j-jdk14 - owners: - - org.slf4j:slf4j-simple: - group: org.slf4j - artifact: slf4j-simple - owners: - - org.slf4j:slf4j-log4j12: - group: org.slf4j - artifact: slf4j-log4j12 - owners: - - org.springframework:spring-expression: - group: org.springframework - artifact: spring-expression - owners: - - org.tukaani:xz: - group: org.tukaani - artifact: xz - owners: - - org.xerial.snappy:snappy-java: - group: org.xerial.snappy - artifact: snappy-java - owners: - - redis.clients:jedis: - group: redis.clients - artifact: jedis - owners: - - sqlline:sqlline: - group: sqlline - artifact: sqlline - owners: kenn,kedin,apilloud,amaliujia - -# No dependency added below this line. -... diff --git a/ownership/PYTHON_DEPENDENCY_OWNERS.yaml b/ownership/PYTHON_DEPENDENCY_OWNERS.yaml deleted file mode 100644 index 182a966fd5618..0000000000000 --- a/ownership/PYTHON_DEPENDENCY_OWNERS.yaml +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Beam Python SDK dependency ownership -# Please update if new dependencies are introduced to the Beam. -# Sign up with you JIRA username to take the ownership of a package. -# Separate names by comma. - ---- - -deps: - avro: - owners: chamikara - - fastavro: - owners: rdub, chamikara - - dill: - owners: robertwb - - google-apitools: - owners: - - google-cloud-core: - owners: - - google-cloud-bigquery: - owners: markflyhigh - - google-cloud-bigtable: - owners: - - google-cloud-pubsub: - owners: markflyhigh - - grpcio: - owners: robertwb - - hdfs: - owners: - - httplib2: - owners: - - mock: - owners: - - pymongo: - owners: yichi - - proto-google-cloud-pubsub-v1: - owners: - - protobuf: - owners: - - pyhamcrest: - owners: markflyhigh - - pytz: - owners: - - pyvcf: - owners: - - pyyaml: - owners: robertwb - - typing: - owners: robertwb - - pandas: - owners: bhulette, robertwb - - pyarrow: - owners: bhulette - - numpy: - owners: bhulette - -# No dependency added below this line. -... diff --git a/playground/README.md b/playground/README.md index 6f69a59d0551f..eb6ed3619bcaa 100644 --- a/playground/README.md +++ b/playground/README.md @@ -41,11 +41,11 @@ build, test, and deploy the frontend and backend services. > - buf > - sbt -1. Install Go 1.18+ +1. Install Go 1.20+ **Ubuntu 22.04 and newer:** ```shell - sudo apt install golang` + sudo apt install golang ``` **Other Linux variants:** Follow manual at https://go.dev/doc/install @@ -224,4 +224,4 @@ Several directories in this repository are used for the Beam Playground project. # Contribution guide - Backend: see [backend/README.md](/playground/backend/README.md) and [backend/CONTRIBUTE.md](/playground/backend/CONTRIBUTE.md) -- Frontend: see [frontend/README.md](/playground/frontend/README.md) and [frontend/CONTRIBUTE.md](/playground/frontend/CONTRIBUTE.md) \ No newline at end of file +- Frontend: see [frontend/README.md](/playground/frontend/README.md) and [frontend/CONTRIBUTE.md](/playground/frontend/CONTRIBUTE.md) diff --git a/playground/backend/containers/go/build.gradle b/playground/backend/containers/go/build.gradle index ee9d82faa5b06..39d79103d6325 100644 --- a/playground/backend/containers/go/build.gradle +++ b/playground/backend/containers/go/build.gradle @@ -88,7 +88,7 @@ docker { buildArgs( ['BASE_IMAGE' : project.rootProject.hasProperty(["base-image"]) ? project.rootProject["base-image"] : - "golang:1.18-bullseye", + "golang:1.20-bullseye", 'SDK_TAG' : project.rootProject.hasProperty(["sdk-tag"]) ? project.rootProject["sdk-tag"] : project.rootProject.sdk_version, 'SDK_TAG_LOCAL': project.rootProject.sdk_version, diff --git a/playground/backend/containers/java/Dockerfile b/playground/backend/containers/java/Dockerfile index 6cd16fab4e7c8..f9d1b24b96d26 100644 --- a/playground/backend/containers/java/Dockerfile +++ b/playground/backend/containers/java/Dockerfile @@ -16,7 +16,7 @@ # limitations under the License. ############################################################################### ARG BEAM_VERSION=2.44.0 -FROM golang:1.18-bullseye AS build +FROM golang:1.20-bullseye AS build ARG BEAM_VERSION ARG GIT_COMMIT="" ARG GIT_TIMESTAMP="0" @@ -65,7 +65,6 @@ ENV BEAM_SDK="SDK_JAVA" ENV PROPERTY_PATH=/opt/playground/backend/properties.yaml ARG CALCITE_VERSION=1_28_0 ARG BYTEBUDDY_VERSION=1.12.14 -ARG FASTJSON_VERSION=1.2.69 ARG JANINO_VERSION=3.0.11 # Copy build result @@ -103,9 +102,6 @@ RUN wget https://repo1.maven.org/maven2/org/apache/beam/beam-vendor-calcite-$CAL RUN wget https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy/$BYTEBUDDY_VERSION/byte-buddy-$BYTEBUDDY_VERSION.jar &&\ mv byte-buddy-$BYTEBUDDY_VERSION.jar /opt/apache/beam/jars/byte-buddy-$BYTEBUDDY_VERSION.jar -RUN wget https://repo1.maven.org/maven2/com/alibaba/fastjson/$FASTJSON_VERSION/fastjson-$FASTJSON_VERSION.jar &&\ - mv fastjson-$FASTJSON_VERSION.jar /opt/apache/beam/jars/fastjson-$FASTJSON_VERSION.jar - RUN wget https://repo1.maven.org/maven2/org/codehaus/janino/janino/$JANINO_VERSION/janino-$JANINO_VERSION.jar &&\ mv janino-$JANINO_VERSION.jar /opt/apache/beam/jars/janino-$JANINO_VERSION.jar diff --git a/playground/backend/containers/python/Dockerfile b/playground/backend/containers/python/Dockerfile index 59454b6e97cb1..60a76f14f7321 100644 --- a/playground/backend/containers/python/Dockerfile +++ b/playground/backend/containers/python/Dockerfile @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### -ARG GO_BASE_IMAGE=golang:1.18-bullseye +ARG GO_BASE_IMAGE=golang:1.20-bullseye ARG SDK_TAG ARG BASE_IMAGE=apache/beam_python3.10_sdk:$SDK_TAG FROM $GO_BASE_IMAGE AS build diff --git a/playground/backend/containers/python/build.gradle b/playground/backend/containers/python/build.gradle index 9b11b19b16b77..4a845b516b801 100644 --- a/playground/backend/containers/python/build.gradle +++ b/playground/backend/containers/python/build.gradle @@ -75,7 +75,7 @@ docker { buildArgs( ['GO_BASE_IMAGE': project.rootProject.hasProperty(["go-base-image"]) ? project.rootProject["go-base-image"] : - "golang:1.18-bullseye", + "golang:1.20-bullseye", 'SDK_TAG' : project.rootProject.hasProperty(["sdk-tag"]) ? project.rootProject["sdk-tag"] : default_beam_version, 'GIT_COMMIT' : getGitCommitHash(), diff --git a/playground/backend/containers/router/Dockerfile b/playground/backend/containers/router/Dockerfile index 2aab1d579fd50..57717d09f8cb0 100644 --- a/playground/backend/containers/router/Dockerfile +++ b/playground/backend/containers/router/Dockerfile @@ -16,7 +16,7 @@ # limitations under the License. ############################################################################### #Dokerfile to set up the Beam Go SDK -ARG BASE_IMAGE=golang:1.18-bullseye +ARG BASE_IMAGE=golang:1.20-bullseye #Two-stage assembly FROM $BASE_IMAGE AS build ARG GIT_COMMIT="" diff --git a/playground/backend/containers/router/build.gradle b/playground/backend/containers/router/build.gradle index ad5b5a6b1c49a..49ae2dc5d4017 100644 --- a/playground/backend/containers/router/build.gradle +++ b/playground/backend/containers/router/build.gradle @@ -70,7 +70,7 @@ docker { tags containerImageTags() buildArgs(['BASE_IMAGE' : project.rootProject.hasProperty(["base-image"]) ? project.rootProject["base-image"] : - "golang:1.18-bullseye", + "golang:1.20-bullseye", 'GIT_COMMIT' : getGitCommitHash(), 'GIT_TIMESTAMP': getGitCommitTimestamp()]) } diff --git a/playground/backend/containers/scio/Dockerfile b/playground/backend/containers/scio/Dockerfile index 13bf5f8c494f2..ff57f6978b50a 100644 --- a/playground/backend/containers/scio/Dockerfile +++ b/playground/backend/containers/scio/Dockerfile @@ -16,7 +16,7 @@ # limitations under the License. ############################################################################### ARG BASE_IMAGE=openjdk:11 -FROM golang:1.18-bullseye AS build +FROM golang:1.20-bullseye AS build ARG GIT_COMMIT="" ARG GIT_TIMESTAMP="0" diff --git a/playground/backend/go.mod b/playground/backend/go.mod index b9f9c653a93a4..9f5fb433ab7e6 100644 --- a/playground/backend/go.mod +++ b/playground/backend/go.mod @@ -15,7 +15,7 @@ module beam.apache.org/playground/backend -go 1.18 +go 1.20 require ( cloud.google.com/go/datastore v1.10.0 diff --git a/playground/backend/internal/code_processing/code_processing.go b/playground/backend/internal/code_processing/code_processing.go index f874a048ac95c..b1fc191ceb35a 100644 --- a/playground/backend/internal/code_processing/code_processing.go +++ b/playground/backend/internal/code_processing/code_processing.go @@ -111,8 +111,11 @@ func Process(ctx context.Context, cacheService cache.Cache, lc *fs_tool.LifeCycl err = runStep(pipelineLifeCycleCtx, cacheService, &lc.Paths, pipelineId, isUnitTest, sdkEnv, pipelineOptions) if err != nil { var pipelineCanceledError perrors.PipelineCanceledError + var runError perrors.RunError if errors.As(err, &pipelineCanceledError) { logger.Warnf("%s: pipeline execution has been canceled: %s", pipelineId, pipelineCanceledError.Error()) + } else if errors.As(err, &runError) { + logger.Warnf("%s: pipeline execution ended with error from child process: %s", runError.Error()) } else { logger.Errorf("%s: error during run step: %s", pipelineId, err.Error()) } @@ -190,7 +193,7 @@ func runStep(ctx context.Context, cacheService cache.Cache, paths *fs_tool.LifeC if processingErr != nil { return processingErr } - return fmt.Errorf("run error: %s", runError.String()) + return perrors.RunError{Log: runError} } // Run step is finished and code is executed err = processRunSuccess(pipelineId, cacheService, stopReadLogsChannel, finishReadLogsChannel) diff --git a/playground/backend/internal/errors/lifecycle_error.go b/playground/backend/internal/errors/lifecycle_error.go index a467bfd1a0054..25de738a5432d 100644 --- a/playground/backend/internal/errors/lifecycle_error.go +++ b/playground/backend/internal/errors/lifecycle_error.go @@ -15,6 +15,11 @@ package errors +import ( + "bytes" + "fmt" +) + type PipelineCanceledError struct { Reason string } @@ -23,6 +28,14 @@ func (e PipelineCanceledError) Error() string { return e.Reason } +type RunError struct { + Log bytes.Buffer +} + +func (e RunError) Error() string { + return fmt.Sprintf("run error: %s", e.Log.String()) +} + type CompilationError struct { Reason string } diff --git a/playground/backend/internal/fs_tool/ExampleData.scala b/playground/backend/internal/fs_tool/ExampleData.scala index 4283394c400fd..e7cdfabce4ba5 100644 --- a/playground/backend/internal/fs_tool/ExampleData.scala +++ b/playground/backend/internal/fs_tool/ExampleData.scala @@ -26,8 +26,8 @@ object ExampleData { "gs://apache-beam-samples/traffic_sensor/Freeways-5Minaa2010-01-01_to_2010-02-15_test2.csv" val GAMING = "gs://apache-beam-samples/game/gaming_data*.csv" - val WEATHER_SAMPLES_TABLE = "clouddataflow-readonly:samples.weather_stations" + val WEATHER_SAMPLES_TABLE = "apache-beam-testing.samples.weather_stations" val SHAKESPEARE_TABLE = "bigquery-public-data:samples.shakespeare" - val EVENT_TABLE = "clouddataflow-readonly:samples.gdelt_sample" + val EVENT_TABLE = "apache-beam-testing.samples.gdelt_sample" val COUNTRY_TABLE = "gdelt-bq:full.crosswalk_geocountrycodetohuman" } diff --git a/playground/frontend/playground_components/lib/src/constants/colors.dart b/playground/frontend/playground_components/lib/src/constants/colors.dart index 6db92295c16aa..e548b31e4e4a2 100644 --- a/playground/frontend/playground_components/lib/src/constants/colors.dart +++ b/playground/frontend/playground_components/lib/src/constants/colors.dart @@ -60,9 +60,9 @@ class BeamLightThemeColors { static const primary = Color(0xffE74D1A); static const icon = BeamColors.grey3; - static const code1 = Color(0xffDA2833); + static const code1 = primary; static const code2 = Color(0xff5929B4); - static const codeComment = Color(0xff4C6B60); + static const codeComment = Color(0xff1B5E20); static const codeBackground = Color(0xffFEF6F3); } @@ -79,8 +79,8 @@ class BeamDarkThemeColors { static const primary = Color(0xffF26628); static const icon = Color(0xff606772); - static const code1 = Color(0xffDA2833); - static const code2 = Color(0xff5929B4); - static const codeComment = Color(0xff4C6B60); + static const code1 = Color(0xffFFEB3B); + static const code2 = primary; + static const codeComment = Color(0xff689F38); static const codeBackground = Color(0xff231B1B); } diff --git a/playground/frontend/playground_components/tools/extract_symbols_go/go.mod b/playground/frontend/playground_components/tools/extract_symbols_go/go.mod index 3542725fa6ec9..eb4d9437f1308 100644 --- a/playground/frontend/playground_components/tools/extract_symbols_go/go.mod +++ b/playground/frontend/playground_components/tools/extract_symbols_go/go.mod @@ -15,6 +15,6 @@ module beam.apache.org/playground/extract_symbols_go -go 1.18 +go 1.20 require gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/playground/frontend/pubspec.lock b/playground/frontend/pubspec.lock index 425e33e4de7df..e49850e7a820d 100644 --- a/playground/frontend/pubspec.lock +++ b/playground/frontend/pubspec.lock @@ -45,10 +45,10 @@ packages: dependency: transitive description: name: archive - sha256: "0c8368c9b3f0abbc193b9d6133649a614204b528982bebc7026372d61677ce3a" + sha256: "49b1fad315e57ab0bbc15bcbb874e83116a1d78f77ebd500a4af6c9407d6b28e" url: "https://pub.dev" source: hosted - version: "3.3.7" + version: "3.3.8" args: dependency: transitive description: @@ -189,10 +189,10 @@ packages: dependency: "direct main" description: name: collection - sha256: "4a07be6cb69c84d677a6c3096fcf960cc3285a8330b4603e0d463d15d9bd934c" + sha256: f092b211a4319e98e5ff58223576de6c2803db36221657b46c82574721240687 url: "https://pub.dev" source: hosted - version: "1.17.1" + version: "1.17.2" connectivity_plus: dependency: transitive description: @@ -600,10 +600,10 @@ packages: dependency: "direct main" description: name: intl - sha256: a3715e3bc90294e971cb7dc063fbf3cd9ee0ebf8604ffeafabd9e6f16abbdbe6 + sha256: "3bc132a9dbce73a7e4a21a17d06e1878839ffbf975568bc875c60537824b0c4d" url: "https://pub.dev" source: hosted - version: "0.18.0" + version: "0.18.1" io: dependency: transitive description: @@ -672,18 +672,18 @@ packages: dependency: transitive description: name: matcher - sha256: "6501fbd55da300384b768785b83e5ce66991266cec21af89ab9ae7f5ce1c4cbb" + sha256: "1803e76e6653768d64ed8ff2e1e67bea3ad4b923eb5c56a295c3e634bad5960e" url: "https://pub.dev" source: hosted - version: "0.12.15" + version: "0.12.16" material_color_utilities: dependency: transitive description: name: material_color_utilities - sha256: d92141dc6fe1dad30722f9aa826c7fbc896d021d792f80678280601aff8cf724 + sha256: "9528f2f296073ff54cb9fee677df673ace1218163c3bc7628093e7eed5203d41" url: "https://pub.dev" source: hosted - version: "0.2.0" + version: "0.5.0" meta: dependency: transitive description: @@ -1059,10 +1059,10 @@ packages: dependency: transitive description: name: source_span - sha256: dd904f795d4b4f3b870833847c461801f6750a9fa8e61ea5ac53f9422b31f250 + sha256: "53e943d4206a5e30df338fd4c6e7a077e02254531b138a15aec3bd143c1a8b3c" url: "https://pub.dev" source: hosted - version: "1.9.1" + version: "1.10.0" stack_trace: dependency: transitive description: @@ -1115,26 +1115,26 @@ packages: dependency: transitive description: name: test - sha256: "3dac9aecf2c3991d09b9cdde4f98ded7b30804a88a0d7e4e7e1678e78d6b97f4" + sha256: "13b41f318e2a5751c3169137103b60c584297353d4b1761b66029bae6411fe46" url: "https://pub.dev" source: hosted - version: "1.24.1" + version: "1.24.3" test_api: dependency: transitive description: name: test_api - sha256: eb6ac1540b26de412b3403a163d919ba86f6a973fe6cc50ae3541b80092fdcfb + sha256: "75760ffd7786fffdfb9597c35c5b27eaeec82be8edfb6d71d32651128ed7aab8" url: "https://pub.dev" source: hosted - version: "0.5.1" + version: "0.6.0" test_core: dependency: transitive description: name: test_core - sha256: "5138dbffb77b2289ecb12b81c11ba46036590b72a64a7a90d6ffb880f1a29e93" + sha256: "99806e9e6d95c7b059b7a0fc08f07fc53fabe54a829497f0d9676299f1e8637e" url: "https://pub.dev" source: hosted - version: "0.5.1" + version: "0.5.3" timing: dependency: transitive description: @@ -1299,10 +1299,10 @@ packages: dependency: transitive description: name: vm_service - sha256: f6deed8ed625c52864792459709183da231ebf66ff0cf09e69b573227c377efe + sha256: c620a6f783fa22436da68e42db7ebbf18b8c44b9a46ab911f666ff09ffd9153f url: "https://pub.dev" source: hosted - version: "11.3.0" + version: "11.7.1" watcher: dependency: transitive description: @@ -1311,6 +1311,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.1.0" + web: + dependency: transitive + description: + name: web + sha256: dc8ccd225a2005c1be616fe02951e2e342092edf968cf0844220383757ef8f10 + url: "https://pub.dev" + source: hosted + version: "0.1.4-beta" web_browser_detect: dependency: transitive description: diff --git a/playground/infrastructure/cloudbuild/playground_ci_examples.sh b/playground/infrastructure/cloudbuild/playground_ci_examples.sh old mode 100644 new mode 100755 index 6123c6b4fa124..962d18a0f4752 --- a/playground/infrastructure/cloudbuild/playground_ci_examples.sh +++ b/playground/infrastructure/cloudbuild/playground_ci_examples.sh @@ -84,7 +84,7 @@ export STEP=CI export SDK_CONFIG="$BEAM_ROOT_DIR/playground/sdks.yaml" export BEAM_EXAMPLE_CATEGORIES="$BEAM_ROOT_DIR/playground/categories.yaml" export GRADLE_VERSION=7.5.1 -export GO_VERSION=1.18 +export GO_VERSION=1.20 LogOutput "Installing python java8 and dependencies" apt-get update > /dev/null diff --git a/playground/kafka-emulator/build.gradle b/playground/kafka-emulator/build.gradle index 486a232f9b99e..2d3f70aa9883f 100644 --- a/playground/kafka-emulator/build.gradle +++ b/playground/kafka-emulator/build.gradle @@ -24,11 +24,11 @@ plugins { applyJavaNature(exportJavadoc: false, publish: false) distZip { - archiveName "${baseName}.zip" + archiveFileName = "${archiveBaseName}.zip" } distTar { - archiveName "${baseName}.tar" + archiveFileName = "${archiveBaseName}.tar" } dependencies { diff --git a/plugins/beam-code-completion-plugin/build.gradle.kts b/plugins/beam-code-completion-plugin/build.gradle.kts index cff54b9458fde..264f2a67aa1d4 100644 --- a/plugins/beam-code-completion-plugin/build.gradle.kts +++ b/plugins/beam-code-completion-plugin/build.gradle.kts @@ -68,3 +68,4 @@ tasks.test { systemProperty("idea.home.path", System.getenv("INTELLIJ_IDEA_SOURCES")) useJUnit() } + diff --git a/plugins/beam-code-completion-plugin/gradlew b/plugins/beam-code-completion-plugin/gradlew old mode 100644 new mode 100755 diff --git a/plugins/beam-code-completion-plugin/src/main/java/BeamCompletionContributor.java b/plugins/beam-code-completion-plugin/src/main/java/BeamCompletionContributor.java index 579aa750fd2fc..348668e7df796 100644 --- a/plugins/beam-code-completion-plugin/src/main/java/BeamCompletionContributor.java +++ b/plugins/beam-code-completion-plugin/src/main/java/BeamCompletionContributor.java @@ -33,6 +33,15 @@ * suggestions specifically for Apache Beam pipelines. */ public class BeamCompletionContributor extends CompletionContributor { + public static final String[] beamJavaSDKTransforms = { + "Filter", "FlatMapElements", "Keys", "KvSwap", "MapElements", "ParDo", + "Partition", "Regex", "Reify", "ToString", "WithKeys", "WithTimestamps", + "Values", "ApproximateQuantiles", "ApproximateUnique", "CoGroupByKey", "Combine", + "CombineWithContext", "Count", "Distinct", "GroupByKey", "GroupIntoBatches", + "HllCount", "Latest", "Max", "Mean", "Min", "Sample", "Sum", "Top", + "Create", "Flatten", "PAssert", "View", "Window" + }; + /** * A pattern condition that matches method call expressions with the name "apply" in the context of the * Apache Beam `Pipeline` class. @@ -75,15 +84,14 @@ public boolean accepts(@NotNull PsiMethodCallExpression psiMethodCallExpression, ); /** - * Fills the completion variants for the given completion parameters and result set. + * Fills the completion variants with Transforms from Java SDK. */ @Override public void fillCompletionVariants(@NotNull final CompletionParameters parameters, @NotNull CompletionResultSet result) { - final PsiElement position = parameters.getPosition(); - PrefixMatcher matcher = result.getPrefixMatcher(); - PsiElement parent = position.getParent(); if (AFTER_METHOD_CALL_PATTERN.accepts(parameters.getPosition())){ - result.addElement(LookupElementBuilder.create("TestCompletionElement")); + for (String transform: beamJavaSDKTransforms) { + result.addElement(LookupElementBuilder.create(transform).appendTailText(" org.apache.beam.sdk.transforms", true)); + } } } } diff --git a/plugins/beam-code-completion-plugin/src/test/java/BeamCompletionContributorTestCase.java b/plugins/beam-code-completion-plugin/src/test/java/BeamCompletionContributorTestCase.java index 8237238755d03..95a756ae37496 100644 --- a/plugins/beam-code-completion-plugin/src/test/java/BeamCompletionContributorTestCase.java +++ b/plugins/beam-code-completion-plugin/src/test/java/BeamCompletionContributorTestCase.java @@ -15,14 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import com.intellij.codeInsight.completion.CompletionType; +import com.intellij.codeInsight.lookup.LookupElement; import com.intellij.testFramework.fixtures.LightJavaCodeInsightFixtureTestCase; import org.jetbrains.annotations.NotNull; import com.intellij.testFramework.LightProjectDescriptor; import com.intellij.testFramework.fixtures.DefaultLightProjectDescriptor; -import java.util.List; +// Each test method test a feature as a whole rather than each function or method. public class BeamCompletionContributorTestCase extends LightJavaCodeInsightFixtureTestCase { @Override protected String getTestDataPath() { @@ -34,27 +34,16 @@ protected String getTestDataPath() { return new DefaultLightProjectDescriptor().withRepositoryLibrary("org.apache.beam:beam-sdks-java-core:2.48.0"); } - public void testElementPatternIsTriggered() { + public void testCompletionsSuccess() throws Throwable { myFixture.configureByFile("TestCompletions.java"); - myFixture.complete(CompletionType.BASIC); - List lookupElementStrings = myFixture.getLookupElementStrings(); - assertNotNull(lookupElementStrings); - assertEquals(lookupElementStrings.get(0), "TestCompletionElement"); - } - - public void testElementPatternWrongClass() { - myFixture.configureByFile("TestCompletionsWrongClass.java"); - myFixture.complete(CompletionType.BASIC); - List lookupElementStrings = myFixture.getLookupElementStrings(); - assertNotNull(lookupElementStrings); - assertNotSame(lookupElementStrings.get(0), "TestCompletionElement"); - } - - public void testElementPatternWrongMethod() { - myFixture.configureByFile("TestCompletionsWrongMethod.java"); - myFixture.complete(CompletionType.BASIC); - List lookupElementStrings = myFixture.getLookupElementStrings(); - assertNotNull(lookupElementStrings); - assertNotSame(lookupElementStrings.get(0), "TestCompletionElement"); + LookupElement[] result = myFixture.completeBasic(); + LookupElement scenarioOutlineLookupElement = null; + for (LookupElement lookupElement : result) { + if (lookupElement.getLookupString().equals("Filter")) { + scenarioOutlineLookupElement = lookupElement; + break; + } + } + assert scenarioOutlineLookupElement != null; } } diff --git a/plugins/beam-code-completion-plugin/src/test/testData/TestCompletions.java b/plugins/beam-code-completion-plugin/src/test/testData/TestCompletions.java index c211f12c8a290..17274abd3b011 100644 --- a/plugins/beam-code-completion-plugin/src/test/testData/TestCompletions.java +++ b/plugins/beam-code-completion-plugin/src/test/testData/TestCompletions.java @@ -32,7 +32,7 @@ public class TestCompletions { public static void main(String[] args) { PipelineOptions options = PipelineOptionsFactory.create(); Pipeline p = Pipeline.create(options); - p.apply(T); + p.apply(Fi); } } diff --git a/plugins/beam-code-completion-plugin/src/test/testData/TestCompletionsWrongClass.java b/plugins/beam-code-completion-plugin/src/test/testData/TestCompletionsWrongClass.java deleted file mode 100644 index 5f882b3d5b3e4..0000000000000 --- a/plugins/beam-code-completion-plugin/src/test/testData/TestCompletionsWrongClass.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.Arrays; -import org.apache.beam.sdk.Pipeline; -import org.apache.beam.sdk.io.TextIO; -import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.sdk.transforms.Count; -import org.apache.beam.sdk.transforms.Filter; -import org.apache.beam.sdk.transforms.FlatMapElements; -import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.TypeDescriptors; - -public class TestCompletionsWrongClass { - public static void main(String[] args) { - String test = "Test"; - test.apply(T); - } -} diff --git a/plugins/beam-code-completion-plugin/src/test/testData/TestCompletionsWrongMethod.java b/plugins/beam-code-completion-plugin/src/test/testData/TestCompletionsWrongMethod.java deleted file mode 100644 index 74feeaa9ded66..0000000000000 --- a/plugins/beam-code-completion-plugin/src/test/testData/TestCompletionsWrongMethod.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.Arrays; -import org.apache.beam.sdk.Pipeline; -import org.apache.beam.sdk.io.TextIO; -import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.sdk.transforms.Count; -import org.apache.beam.sdk.transforms.Filter; -import org.apache.beam.sdk.transforms.FlatMapElements; -import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.TypeDescriptors; - -public class TestCompletions { - public static void main(String[] args) { - PipelineOptions options = PipelineOptionsFactory.create(); - Pipeline p = Pipeline.create(o); - } -} - diff --git a/release/OWNERS b/release/OWNERS deleted file mode 100644 index 8b0b05531cd10..0000000000000 --- a/release/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - jbonofre diff --git a/release/go-licenses/common.gradle b/release/go-licenses/common.gradle index 970d53cfe19a9..273ce9260b2d2 100644 --- a/release/go-licenses/common.gradle +++ b/release/go-licenses/common.gradle @@ -24,6 +24,10 @@ docker { files '../get-licenses.sh', '../go.mod' noCache false buildArgs(['sdk_location': "github.com/apache/beam/sdks/v2/${sdkName}/container"]) + // If Docker buildx builder is used, explicitly load the image into the local repository. + // Set the buildx flag to pick up the load flag. + buildx true + load true } // The mkdir happens at configuration time to allow dockerRun volume to be declared. diff --git a/release/src/main/Dockerfile b/release/src/main/Dockerfile index 0dc80450a2a5b..04d39765cfc20 100644 --- a/release/src/main/Dockerfile +++ b/release/src/main/Dockerfile @@ -42,12 +42,11 @@ RUN curl https://pyenv.run | bash && \ echo 'command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"' >> /root/.bashrc && \ echo ''eval "$(pyenv init -)"'' >> /root/.bashrc && \ source /root/.bashrc && \ - pyenv install 3.7.10 && \ pyenv install 3.8.9 && \ pyenv install 3.9.4 && \ pyenv install 3.10.7 && \ pyenv install 3.11.3 && \ - pyenv global 3.8.9 3.7.10 3.9.4 3.10.7 3.11.3 + pyenv global 3.8.9 3.9.4 3.10.7 3.11.3 # Install a Go version >= 1.16 so we can bootstrap higher # Go versions diff --git a/release/src/main/python-release/python_release_automation.sh b/release/src/main/python-release/python_release_automation.sh index 7c036caa58b9a..e245406b57d07 100755 --- a/release/src/main/python-release/python_release_automation.sh +++ b/release/src/main/python-release/python_release_automation.sh @@ -19,7 +19,7 @@ source release/src/main/python-release/run_release_candidate_python_quickstart.sh source release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh -for version in 3.7 3.8 3.9 3.10 3.11 +for version in 3.8 3.9 3.10 3.11 do run_release_candidate_python_quickstart "tar" "python${version}" run_release_candidate_python_mobile_gaming "tar" "python${version}" diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh index 90678f6363019..d0e6310f50aae 100755 --- a/release/src/main/scripts/build_release_candidate.sh +++ b/release/src/main/scripts/build_release_candidate.sh @@ -214,7 +214,7 @@ if [[ $confirmation = "y" ]]; then rm -r "$RELEASE_DIR" echo "----Signing Source Release ${SOURCE_RELEASE_ZIP}-----" - gpg --local-user ${SIGNING_KEY} --armor --detach-sig "${SOURCE_RELEASE_ZIP}" + gpg --local-user ${SIGNING_KEY} --armor --batch --yes --detach-sig "${SOURCE_RELEASE_ZIP}" echo "----Creating Hash Value for ${SOURCE_RELEASE_ZIP}----" sha512sum ${SOURCE_RELEASE_ZIP} > ${SOURCE_RELEASE_ZIP}.sha512 @@ -281,7 +281,7 @@ if [[ $confirmation = "y" ]]; then for artifact in *.whl; do echo "------------------Signing ${artifact} wheel-------------------" - gpg --local-user "${SIGNING_KEY}" --armor --detach-sig "${artifact}" + gpg --local-user "${SIGNING_KEY}" --armor --batch --yes --detach-sig "${artifact}" done cd .. @@ -346,7 +346,7 @@ if [[ $confirmation = "y" ]]; then cd ${BEAM_ROOT_DIR} RELEASE_COMMIT=$(git rev-list -n 1 "tags/${RC_TAG}") # TODO(https://github.com/apache/beam/issues/20209): Don't hardcode py version in this file. - cd sdks/python && pip install -r build-requirements.txt && tox -e py38-docs + cd sdks/python && tox -e py38-docs GENERATED_PYDOC=~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_PYTHON_DOC}/${BEAM_ROOT_DIR}/sdks/python/target/docs/_build rm -rf ${GENERATED_PYDOC}/.doctrees diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py index 181fd0c8b92ba..5fbeb51a10cdb 100644 --- a/release/src/main/scripts/download_github_actions_artifacts.py +++ b/release/src/main/scripts/download_github_actions_artifacts.py @@ -44,7 +44,7 @@ def parse_arguments(): description= "Script for downloading GitHub Actions artifacts from 'Build python wheels' workflow." ) - parser.add_argument("--github-user", required=True) + parser.add_argument("--github-token-var", required=False, default='GITHUB_TOKEN') parser.add_argument("--repo-url", required=True) parser.add_argument("--rc-tag", required=True) parser.add_argument("--release-commit", required=True) @@ -52,7 +52,7 @@ def parse_arguments(): parser.add_argument("--rc_number", required=False, default="") args = parser.parse_args() - github_token = ask_for_github_token() + github_token = get_github_token(args.github_token_var) print("You passed following arguments:") pprint.pprint({**vars(args), **{"github_token": github_token}}) @@ -61,7 +61,6 @@ def parse_arguments(): print("You said NO. Quitting ...") sys.exit(1) - user_github_id = args.github_user repo_url = args.repo_url rc_tag = args.rc_tag release_commit = args.release_commit @@ -69,11 +68,14 @@ def parse_arguments(): else os.path.abspath(args.artifacts_dir) rc_number = args.rc_number - return github_token, user_github_id, repo_url, rc_tag, release_commit, artifacts_dir, rc_number + return github_token, repo_url, rc_tag, release_commit, artifacts_dir, rc_number -def ask_for_github_token(): - """Ask for github token and print basic information about it.""" +def get_github_token(github_token_var): + """Get GitHub token from env or ask for it and print basic information about it.""" + if github_token_var in os.environ: + return os.environ[github_token_var] + url = "https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token" message = ( f"You need to have a github access token with public_repo scope. " @@ -321,7 +323,6 @@ def extract_single_artifact(file_path, output_dir): ) ( github_token, - user_github_id, repo_url, rc_tag, release_commit, diff --git a/release/src/main/scripts/github_actions_jobs.txt b/release/src/main/scripts/github_actions_jobs.txt new file mode 100644 index 0000000000000..6f200454afe06 --- /dev/null +++ b/release/src/main/scripts/github_actions_jobs.txt @@ -0,0 +1,25 @@ +Run PythonDocs PreCommit,beam_PreCommit_PythonDocs +Run PythonLint PreCommit,beam_PreCommit_PythonLint +Run RAT PreCommit,beam_PreCommit_RAT +Run Spotless PreCommit,beam_PreCommit_Spotless +Run Website PreCommit,beam_PreCommit_Website +Run Website_Stage_GCS PreCommit,beam_PreCommit_Website_Stage_GCS +Run Whitespace PreCommit,beam_PreCommit_Whitespace +Run PythonFormatter PreCommit,beam_PreCommit_PythonFormatter +Run Kotlin_Examples PreCommit,beam_PreCommit_Kotlin_Examples +Run Go PreCommit,beam_PreCommit_Go +Run GoPortable PreCommit,beam_PreCommit_GoPortable +Run GoPrism PreCommit,beam_PreCommit_GoPrism +Run Typescript PreCommit,beam_PreCommit_Typescript +Run CommunityMetrics PreCommit,beam_PreCommit_CommunityMetrics +Run Java_Flink_Versions PreCommit,beam_PreCommit_Java_Flink_Versions +Run It_Framework PreCommit,beam_PreCommit_ItFramework +Run Java examples on Dataflow,beam_PostCommit_Java_Examples_Dataflow +Run Go PostCommit Dataflow ARM,beam_PostCommit_Go_Dataflow_ARM +Run Java_Examples_Dataflow_ARM PostCommit 8,beam_PostCommit_Java_Examples_Dataflow_ARM +Run Java_Examples_Dataflow_ARM PostCommit 11,beam_PostCommit_Java_Examples_Dataflow_ARM +Run Java_Examples_Dataflow_ARM PostCommit 17,beam_PostCommit_Java_Examples_Dataflow_ARM +Run Python ValidatesContainer Dataflow ARM 3.8,beam_Python_ValidatesContainer_Dataflow_ARM +Run Python ValidatesContainer Dataflow ARM 3.9,beam_Python_ValidatesContainer_Dataflow_ARM +Run Python ValidatesContainer Dataflow ARM 3.10,beam_Python_ValidatesContainer_Dataflow_ARM +Run Python ValidatesContainer Dataflow ARM 3.11,beam_Python_ValidatesContainer_Dataflow_ARM diff --git a/release/src/main/scripts/jenkins_jobs.txt b/release/src/main/scripts/jenkins_jobs.txt index deefde0a280c2..65ccc4d9dec2e 100644 --- a/release/src/main/scripts/jenkins_jobs.txt +++ b/release/src/main/scripts/jenkins_jobs.txt @@ -1,7 +1,5 @@ -Run Beam Metrics deployment,beam_PostCommit_BeamMetrics_Publish_PR Run Chicago Taxi on Dataflow,beam_PostCommit_Python_Chicago_Taxi_Dataflow_PR Run Chicago Taxi on Flink,beam_PostCommit_Python_Chicago_Taxi_Flink_PR -Run CommunityMetrics PreCommit,beam_PreCommit_CommunityMetrics_Phrase Run Dataflow Runner Nexmark Tests,beam_PostCommit_Java_Nexmark_Dataflow_PR Run Dataflow Runner Tpcds Tests,beam_PostCommit_Java_Tpcds_Dataflow_PR Run Dataflow Runner V2 Java 11 Nexmark Tests,beam_PostCommit_Java_Nexmark_DataflowV2_Java11_PR @@ -14,7 +12,6 @@ Run Dataflow ValidatesRunner,beam_PostCommit_Java_ValidatesRunner_Dataflow_PR Run Direct Runner Nexmark Tests,beam_PostCommit_Java_Nexmark_Direct_PR Run Direct ValidatesRunner Java 11,beam_PostCommit_Java_ValidatesRunner_Direct_Java11_PR Run Direct ValidatesRunner Java 17,beam_PostCommit_Java_ValidatesRunner_Direct_Java17_PR -Run Direct ValidatesRunner in Java 11,beam_PostCommit_Java11_ValidatesRunner_Direct_PR Run Direct ValidatesRunner,beam_PostCommit_Java_ValidatesRunner_Direct_PR Run Flink Runner Nexmark Tests,beam_PostCommit_Java_Nexmark_Flink_PR Run Flink Runner Tpcds Tests,beam_PostCommit_Java_Tpcds_Flink_PR @@ -22,10 +19,8 @@ Run Flink ValidatesRunner Java 11,beam_PostCommit_Java_ValidatesRunner_Flink_Jav Run Flink ValidatesRunner,beam_PostCommit_Java_ValidatesRunner_Flink_PR Run Go Flink ValidatesRunner,beam_PostCommit_Go_VR_Flink_PR Run Go PostCommit,beam_PostCommit_Go_PR -Run Go PreCommit,beam_PreCommit_Go_Phrase Run Go Samza ValidatesRunner,beam_PostCommit_Go_VR_Samza_PR Run Go Spark ValidatesRunner,beam_PostCommit_Go_VR_Spark_PR -Run GoPortable PreCommit,beam_PreCommit_GoPortable_Phrase Run Java 11 Examples on Dataflow Runner V2,beam_PostCommit_Java_Examples_Dataflow_V2_java11_PR Run Java 17 Examples on Dataflow Runner V2,beam_PostCommit_Java_Examples_Dataflow_V2_java17_PR Run Java Dataflow V2 ValidatesRunner Streaming,beam_PostCommit_Java_VR_Dataflow_V2_Streaming_PR @@ -36,7 +31,6 @@ Run Java Examples_Flink,beam_PostCommit_Java_Examples_Flink_PR Run Java Examples_Spark,beam_PostCommit_Java_Examples_Spark_PR Run Java Flink PortableValidatesRunner Streaming,beam_PostCommit_Java_PVR_Flink_Streaming_PR Run Java InfluxDbIO_IT,beam_PostCommit_Java_InfluxDbIO_IT_PR -Run Java Portability examples on Dataflow with Java 11,beam_PostCommit_Java11_Examples_Dataflow_Portability_PR Run Java PostCommit,beam_PostCommit_Java_PR Run Java PreCommit,beam_PreCommit_Java_Phrase Run Java Samza PortableValidatesRunner,beam_PostCommit_Java_PVR_Samza_PR @@ -46,7 +40,6 @@ Run Java Spark PortableValidatesRunner Batch,beam_PostCommit_Java_PVR_Spark_Batc Run Java Spark v3 PortableValidatesRunner Streaming,beam_PostCommit_Java_PVR_Spark3_Streaming_PR Run Java examples on Dataflow Java 11,beam_PostCommit_Java_Examples_Dataflow_Java11_PR Run Java examples on Dataflow Java 17,beam_PostCommit_Java_Examples_Dataflow_Java17_PR -Run Java examples on Dataflow with Java 11,beam_PostCommit_Java11_Examples_Dataflow_PR Run Java_Amazon-Web-Services2_IO_Direct PreCommit,beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct_Phrase Run Java_Amazon-Web-Services_IO_Direct PreCommit,beam_PreCommit_Java_Amazon-Web-Services_IO_Direct_Phrase Run Java_Amqp_IO_Direct PreCommit,beam_PreCommit_Java_Amqp_IO_Direct_Phrase @@ -62,6 +55,7 @@ Run Java_Examples_Dataflow_Java11 PreCommit,beam_PreCommit_Java_Examples_Dataflo Run Java_Examples_Dataflow_Java17 PreCommit,beam_PreCommit_Java_Examples_Dataflow_Java17_Phrase Run Java_File-schema-transform_IO_Direct PreCommit,beam_PreCommit_Java_File-schema-transform_IO_Direct_Phrase Run Java_GCP_IO_Direct PreCommit,beam_PreCommit_Java_GCP_IO_Direct_Phrase +Run Java_Google-ads_IO_Direct PreCommit,beam_PreCommit_Java_Google-ads_IO_Direct_Phrase Run Java_Hbase_IO_Direct PreCommit,beam_PreCommit_Java_HBase_IO_Direct_Phrase Run Java_Hcatalog_IO_Direct PreCommit,beam_PreCommit_Java_HCatalog_IO_Direct_Phrase Run Java_IOs_Direct PreCommit,beam_PreCommit_Java_IOs_Direct_Phrase @@ -95,7 +89,6 @@ Run Jpms Direct Java 11 PostCommit,beam_PostCommit_Java_Jpms_Direct_Java11_PR Run Jpms Direct Java 17 PostCommit,beam_PostCommit_Java_Jpms_Direct_Java17_PR Run Jpms Flink Java 11 PostCommit,beam_PostCommit_Java_Jpms_Flink_Java11_PR Run Jpms Spark Java 11 PostCommit,beam_PostCommit_Java_Jpms_Spark_Java11_PR -Run Kotlin_Examples PreCommit,beam_PreCommit_Kotlin_Examples_Phrase Run PortableJar_Flink PostCommit,beam_PostCommit_PortableJar_Flink_PR Run PortableJar_Spark PostCommit,beam_PostCommit_PortableJar_Spark_PR Run Portable_Python PreCommit,beam_PreCommit_Portable_Python_Phrase @@ -107,13 +100,10 @@ Run Python 3.10 PostCommit Sickbay,beam_PostCommit_Sickbay_Python310_PR Run Python 3.10 PostCommit,beam_PostCommit_Python310_PR Run Python 3.11 PostCommit Sickbay,beam_PostCommit_Sickbay_Python311_PR Run Python 3.11 PostCommit,beam_PostCommit_Python311_PR -Run Python 3.7 PostCommit Sickbay,beam_PostCommit_Sickbay_Python37_PR -Run Python 3.7 PostCommit,beam_PostCommit_Python37_PR Run Python 3.8 PostCommit Sickbay,beam_PostCommit_Sickbay_Python38_PR Run Python 3.8 PostCommit,beam_PostCommit_Python38_PR Run Python 3.9 PostCommit Sickbay,beam_PostCommit_Sickbay_Python39_PR Run Python 3.9 PostCommit,beam_PostCommit_Python39_PR -Run Python Dataflow V2 ValidatesRunner,beam_PostCommit_Py_VR_Dataflow_V2_PR Run Python Dataflow ValidatesContainer,beam_PostCommit_Py_ValCont_PR Run Python Dataflow ValidatesRunner,beam_PostCommit_Py_VR_Dataflow_PR Run Python Direct Runner Nexmark Tests,beam_PostCommit_Python_Nexmark_Direct_PR @@ -128,9 +118,6 @@ Run Python RC Dataflow ValidatesContainer,beam_PostCommit_Py_ValCont_with_RC_PR Run Python Samza ValidatesRunner,beam_PostCommit_Python_VR_Samza_PR Run Python Spark ValidatesRunner,beam_PostCommit_Python_VR_Spark_PR Run PythonDocker PreCommit,beam_PreCommit_PythonDocker_Phrase -Run PythonDocs PreCommit,beam_PreCommit_PythonDocs_Phrase -Run PythonFormatter PreCommit,beam_PreCommit_PythonFormatter_Phrase -Run PythonLint PreCommit,beam_PreCommit_PythonLint_Phrase Run Python_Coverage PreCommit,beam_PreCommit_Python_Coverage_Phrase Run Python_Dataframes PreCommit,beam_PreCommit_Python_Dataframes_Phrase Run Python_Examples PreCommit,beam_PreCommit_Python_Examples_Phrase @@ -140,8 +127,7 @@ Run Python_Runners PreCommit,beam_PreCommit_Python_Runners_Phrase Run Python_Transforms PreCommit,beam_PreCommit_Python_Transforms_Phrase Run Python_Xlang_Gcp_Dataflow PostCommit,beam_PostCommit_Python_Xlang_Gcp_Dataflow_PR Run Python_Xlang_Gcp_Direct PostCommit,beam_PostCommit_Python_Xlang_Gcp_Direct_PR -Run RAT PreCommit,beam_PreCommit_RAT_Phrase -Run Release Gradle Build,beam_Release_Gradle_Build +Run Python_Xlang_IO_Dataflow PostCommit,beam_PostCommit_Python_Xlang_IO_Dataflow_PR Run SQL PostCommit,beam_PostCommit_SQL_PR Run SQL PreCommit,beam_PreCommit_SQL_Phrase Run SQL_Java11 PreCommit,beam_PreCommit_SQL_Java11_Phrase @@ -152,11 +138,9 @@ Run Spark Runner Tpcds Tests,beam_PostCommit_Java_Tpcds_Spark_PR Run Spark StructuredStreaming ValidatesRunner,beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming_PR Run Spark ValidatesRunner Java 11,beam_PostCommit_Java_ValidatesRunner_Spark_Java11_PR Run Spark ValidatesRunner,beam_PostCommit_Java_ValidatesRunner_Spark_PR -Run Spotless PreCommit,beam_PreCommit_Spotless_Phrase +Run TransformService_Direct PostCommit,beam_PostCommit_TransformService_Direct_PR Run Twister2 ValidatesRunner,beam_PostCommit_Java_ValidatesRunner_Twister2_PR -Run Typescript PreCommit,beam_PreCommit_Typescript_Phrase Run ULR Loopback ValidatesRunner,beam_PostCommit_Java_ValidatesRunner_ULR_PR -Run Whitespace PreCommit,beam_PreCommit_Whitespace_Phrase Run XVR_Direct PostCommit,beam_PostCommit_XVR_Direct_PR Run XVR_Flink PostCommit,beam_PostCommit_XVR_Flink_PR Run XVR_GoUsingJava_Dataflow PostCommit,beam_PostCommit_XVR_GoUsingJava_Dataflow_PR diff --git a/release/src/main/scripts/mass_comment.py b/release/src/main/scripts/mass_comment.py index 2a41e43cb2861..6d020f2af8e3d 100644 --- a/release/src/main/scripts/mass_comment.py +++ b/release/src/main/scripts/mass_comment.py @@ -120,11 +120,25 @@ def getRemainingComments(accessToken, pr, initialComments): for comment in initialComments: if f'/{comment[1]}_Phrase/' not in check_urls and f'/{comment[1]}_PR/' not in check_urls \ and f'/{comment[1]}_Commit/' not in check_urls and f'/{comment[1]}/' not in check_urls \ - and 'Sickbay' not in check_urls: + and 'Sickbay' not in comment[1]: print(comment) remainingComments.append(comment) return remainingComments +def getGithubActionsTriggerCommands(dirname): + ''' + Returns all trigger commands that will start PostCommit Dataflow ARM Github Actions test suites. + ''' + gha_trigger_commands = [] + + with open(os.path.join(dirname, 'github_actions_jobs.txt')) as file: + comments = [line.strip() for line in file if len(line.strip()) > 0] + + for i in range(len(comments)): + parts = comments[i].split(',') + gha_trigger_commands.append((parts[0], parts[1])) + + return gha_trigger_commands ################################################################################ if __name__ == '__main__': @@ -142,7 +156,10 @@ def getRemainingComments(accessToken, pr, initialComments): for i in range(len(comments)): parts = comments[i].split(',') comments[i] = (parts[0], parts[1]) - + + gha_comments = getGithubActionsTriggerCommands(dirname) + comments.extend(gha_comments) + if not probeGitHubIsUp(): print("GitHub is unavailable, skipping fetching data.") exit() @@ -173,4 +190,4 @@ def getRemainingComments(accessToken, pr, initialComments): print(f'{len(remainingComments)} comments still must be reposted: {str(remainingComments)}') print('Trying to repost comments.') - print('Done.') \ No newline at end of file + print('Done.') diff --git a/release/src/main/scripts/publish_docker_images.sh b/release/src/main/scripts/publish_docker_images.sh index b189c9c1bfdc3..b191b7656fa8e 100755 --- a/release/src/main/scripts/publish_docker_images.sh +++ b/release/src/main/scripts/publish_docker_images.sh @@ -37,25 +37,32 @@ echo "Which release candidate will be the source of final docker images? (ex: 1) read RC_NUM RC_VERSION="rc${RC_NUM}" -echo "================Confirming Release and RC version===========" +echo "================Pull RC Containers from DockerHub===========" +IMAGES=$(docker search ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX} --format "{{.Name}}" --limit 100) +KNOWN_IMAGES=() echo "We are using ${RC_VERSION} to push docker images for ${RELEASE}." +while read IMAGE; do + # Try pull verified RC from dockerhub. + if docker pull "${IMAGE}:${RELEASE}${RC_VERSION}" 2>/dev/null ; then + KNOWN_IMAGES+=( $IMAGE ) + fi +done < <(echo "${IMAGES}") + +echo "================Confirming Release and RC version===========" echo "Publishing the following images:" -IMAGES=$(docker images --filter "reference=apache/beam_*:${RELEASE}${RC_VERSION}" --format "{{.Repository}}") -echo "${IMAGES}" +# Sort by name for easy examination +IFS=$'\n' KNOWN_IMAGES=($(sort <<<"${KNOWN_IMAGES[*]}")) +unset IFS +printf "%s\n" ${KNOWN_IMAGES[@]} echo "Do you want to proceed? [y|N]" read confirmation if [[ $confirmation = "y" ]]; then - echo "${IMAGES}" | while read IMAGE; do - # Pull verified RC from dockerhub. - docker pull "${IMAGE}:${RELEASE}${RC_VERSION}" - - # Tag with ${RELEASE} and push to dockerhub. - docker tag "${IMAGE}:${RELEASE}${RC_VERSION}" "${IMAGE}:${RELEASE}" - docker push "${IMAGE}:${RELEASE}" + for IMAGE in "${KNOWN_IMAGES[@]}"; do + # Perform a carbon copy of ${RC_VERSION} to dockerhub with a new tag as ${RELEASE}. + docker buildx imagetools create --tag "${IMAGE}:${RELEASE}" "${IMAGE}:${RELEASE}${RC_VERSION}" - # Tag with latest and push to dockerhub. - docker tag "${IMAGE}:${RELEASE}${RC_VERSION}" "${IMAGE}:latest" - docker push "${IMAGE}:latest" + # Perform a carbon copy of ${RC_VERSION} to dockerhub with a new tag as latest. + docker buildx imagetools create --tag "${IMAGE}:latest" "${IMAGE}:${RELEASE}" done -fi \ No newline at end of file +fi diff --git a/release/src/main/scripts/run_rc_validation.sh b/release/src/main/scripts/run_rc_validation.sh index 4ad8af16723b9..7f32c2979660b 100755 --- a/release/src/main/scripts/run_rc_validation.sh +++ b/release/src/main/scripts/run_rc_validation.sh @@ -271,8 +271,8 @@ echo "This task will create a PR against apache/beam, trigger a jenkins job to r echo "1. Python quickstart validations(batch & streaming)" echo "2. Python MobileGame validations(UserScore, HourlyTeamScore)" if [[ "$python_quickstart_mobile_game" = true && ! -z `which hub` ]]; then - touch empty_file.txt - git add empty_file.txt + touch empty_file.json + git add empty_file.json git commit -m "Add empty file in order to create PR" --quiet git push -f ${GITHUB_USERNAME} --quiet # Create a test PR diff --git a/release/src/main/scripts/verify_release_build.sh b/release/src/main/scripts/verify_release_build.sh index 6d16a232449d7..214c65cc9ef64 100755 --- a/release/src/main/scripts/verify_release_build.sh +++ b/release/src/main/scripts/verify_release_build.sh @@ -97,7 +97,17 @@ fi echo "====================== 2.2 Checking hub ========================" HUB_VERSION=2.12.0 -HUB_ARTIFACTS_NAME=hub-linux-amd64-${HUB_VERSION} + +if [[ "$(uname)" == "Linux" ]]; then + ARCITECTURE=linux-amd64 +elif [[ "$(uname)" == "Darwin" ]]; then + ARCITECTURE=darwin-amd64 +else + echo "Error: unsupported architecture for Hub" + exit +fi +HUB_ARTIFACTS_NAME=hub-${ARCITECTURE}-${HUB_VERSION} + if [[ -z `which hub` ]]; then echo "There is no hub installed on your machine." if [[ "${INSTALL_HUB}" = true ]]; then @@ -115,8 +125,8 @@ hub version echo "" -echo "==================== 3 Run Gradle Release Build & PostCommit Tests on Jenkins ===================" -echo "[Current Task] Run Gradle release build and all PostCommit Tests against Release Branch on Jenkins." +echo "==================== 3 Run PostCommit Tests on Jenkins ===================" +echo "[Current Task] Run all PostCommit Tests against Release Branch on Jenkins." echo "This task will create a PR against apache/beam." echo "After PR created, you need to comment phrases listed in description in the created PR:" diff --git a/runners/core-construction-java/OWNERS b/runners/core-construction-java/OWNERS deleted file mode 100644 index af32c311923a2..0000000000000 --- a/runners/core-construction-java/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lukecwik diff --git a/runners/core-construction-java/build.gradle b/runners/core-construction-java/build.gradle index 90b64cddc60c6..f593865b3fe97 100644 --- a/runners/core-construction-java/build.gradle +++ b/runners/core-construction-java/build.gradle @@ -55,8 +55,9 @@ dependencies { implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(path: ":sdks:java:extensions:avro") implementation project(path: ":sdks:java:fn-execution") + implementation project(path: ":sdks:java:transform-service:launcher") implementation library.java.vendored_grpc_1_54_0 - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.classgraph implementation library.java.jackson_core implementation library.java.jackson_databind diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/AvroGenericCoderRegistrar.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/AvroGenericCoderRegistrar.java index 1797718d6ed11..1ec54f27bce6e 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/AvroGenericCoderRegistrar.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/AvroGenericCoderRegistrar.java @@ -21,7 +21,7 @@ import java.util.Map; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.extensions.avro.coders.AvroGenericCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** Coder registrar for AvroGenericCoder. */ @AutoService(CoderTranslatorRegistrar.class) diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/AvroGenericCoderTranslator.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/AvroGenericCoderTranslator.java index 614810abbdbc7..e332e60e9ab05 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/AvroGenericCoderTranslator.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/AvroGenericCoderTranslator.java @@ -23,7 +23,7 @@ import org.apache.beam.runners.core.construction.CoderTranslation.TranslationContext; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.extensions.avro.coders.AvroGenericCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; /** Coder translator for AvroGenericCoder. */ public class AvroGenericCoderTranslator implements CoderTranslator { diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java index 798ae5acb3d58..5e4c2652bfe6d 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.util.ArrayList; @@ -29,10 +29,10 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableBiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableBiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** Converts to and from Beam Runner API representations of {@link Coder Coders}. */ @SuppressWarnings({ diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslators.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslators.java index a7a20b0d3d6d3..769778e0eca49 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslators.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslators.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Collections; import java.util.List; @@ -38,7 +38,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** {@link CoderTranslator} implementations for known coder types. */ class CoderTranslators { diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java index 278808becf0f7..fbe4876b414fb 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java @@ -40,9 +40,9 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** * Methods for translating between {@link Combine} {@link PTransform PTransforms} and {@link @@ -61,7 +61,7 @@ public static class CombinePerKeyPayloadTranslator private CombinePerKeyPayloadTranslator() {} @Override - public String getUrn(Combine.PerKey transform) { + public String getUrn() { return COMBINE_PER_KEY_TRANSFORM_URN; } @@ -108,7 +108,7 @@ public static class CombineGloballyPayloadTranslator private CombineGloballyPayloadTranslator() {} @Override - public String getUrn(Combine.Globally transform) { + public String getUrn() { return COMBINE_GLOBALLY_TRANSFORM_URN; } @@ -165,7 +165,7 @@ public static class CombineGroupedValuesPayloadTranslator private CombineGroupedValuesPayloadTranslator() {} @Override - public String getUrn(Combine.GroupedValues transform) { + public String getUrn() { return COMBINE_GROUPED_VALUES_TRANSFORM_URN; } diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java index 4161674836f67..71038564ec4ca 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; import java.io.IOException; @@ -90,7 +90,7 @@ public static PCollectionView getView( static class CreatePCollectionViewTranslator implements TransformPayloadTranslator> { @Override - public String getUrn(View.CreatePCollectionView transform) { + public String getUrn() { return PTransformTranslation.CREATE_VIEW_TRANSFORM_URN; } diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DeduplicatedFlattenFactory.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DeduplicatedFlattenFactory.java index 5328ee900055b..03f3d42a64910 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DeduplicatedFlattenFactory.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DeduplicatedFlattenFactory.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.values.PCollectionList; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** * A {@link PTransformOverrideFactory} that will apply a flatten where no element appears in the diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DefaultArtifactResolver.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DefaultArtifactResolver.java index 0996022a52d7a..7b20c2e6ca195 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DefaultArtifactResolver.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DefaultArtifactResolver.java @@ -25,9 +25,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.beam.model.pipeline.v1.RunnerApi; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; /** * A default artifact resolver. This resolver applies {@link ResolutionFn} in the reversed order diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DisplayDataTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DisplayDataTranslation.java index 1cb6b2ca6a806..e031274e419f7 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DisplayDataTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DisplayDataTranslation.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.List; import java.util.Map; @@ -26,8 +26,8 @@ import org.apache.beam.model.pipeline.v1.RunnerApi.StandardDisplayData; import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** Utilities for going to/from DisplayData protos. */ @SuppressWarnings({ diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/EmptyFlattenAsCreateFactory.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/EmptyFlattenAsCreateFactory.java index 5462909b3e493..cbfb1f9c08422 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/EmptyFlattenAsCreateFactory.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/EmptyFlattenAsCreateFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Map; import org.apache.beam.sdk.coders.VoidCoder; diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java index c4e0ca208fe01..31a555989afdc 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java @@ -49,15 +49,15 @@ import org.apache.beam.sdk.util.common.ReflectHelpers; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashCode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.HashCode; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -328,51 +328,65 @@ public static List getArtifacts(List stagingFiles) } else { file = new File(path); } - // Spurious items get added to the classpath. Filter by just those that exist. - if (file.exists()) { - ArtifactInformation.Builder artifactBuilder = ArtifactInformation.newBuilder(); - artifactBuilder.setTypeUrn(BeamUrns.getUrn(StandardArtifacts.Types.FILE)); - artifactBuilder.setRoleUrn(BeamUrns.getUrn(StandardArtifacts.Roles.STAGING_TO)); - HashCode hashCode; - if (file.isDirectory()) { - File zippedFile; - try { - zippedFile = zipDirectory(file); - hashCode = Files.asByteSource(zippedFile).hash(Hashing.sha256()); - } catch (IOException e) { - throw new RuntimeException(e); - } - - artifactBuilder.setTypePayload( - RunnerApi.ArtifactFilePayload.newBuilder() - .setPath(zippedFile.getPath()) - .setSha256(hashCode.toString()) - .build() - .toByteString()); + // Spurious items get added to the classpath, but ignoring silently can cause confusion. + // Therefore, issue logs if a file does not exist before ignoring. The level will be warning + // if they have a staged name, as those are likely to cause problems or unintended behavior + // (e.g., dataflow-worker.jar, windmill_main). + if (!file.exists()) { + if (stagedName != null) { + LOG.warn( + "Stage Artifact '{}' with the name '{}' was not found, staging will be ignored.", + file, + stagedName); } else { - try { - hashCode = Files.asByteSource(file).hash(Hashing.sha256()); - } catch (IOException e) { - throw new RuntimeException(e); - } - artifactBuilder.setTypePayload( - RunnerApi.ArtifactFilePayload.newBuilder() - .setPath(file.getPath()) - .setSha256(hashCode.toString()) - .build() - .toByteString()); + LOG.info("Stage Artifact '{}' was not found, staging will be ignored.", file); + } + continue; + } + + ArtifactInformation.Builder artifactBuilder = ArtifactInformation.newBuilder(); + artifactBuilder.setTypeUrn(BeamUrns.getUrn(StandardArtifacts.Types.FILE)); + artifactBuilder.setRoleUrn(BeamUrns.getUrn(StandardArtifacts.Roles.STAGING_TO)); + HashCode hashCode; + if (file.isDirectory()) { + File zippedFile; + try { + zippedFile = zipDirectory(file); + hashCode = Files.asByteSource(zippedFile).hash(Hashing.sha256()); + } catch (IOException e) { + throw new RuntimeException(e); } - if (stagedName == null) { - stagedName = createStagingFileName(file, hashCode); + + artifactBuilder.setTypePayload( + RunnerApi.ArtifactFilePayload.newBuilder() + .setPath(zippedFile.getPath()) + .setSha256(hashCode.toString()) + .build() + .toByteString()); + + } else { + try { + hashCode = Files.asByteSource(file).hash(Hashing.sha256()); + } catch (IOException e) { + throw new RuntimeException(e); } - artifactBuilder.setRolePayload( - RunnerApi.ArtifactStagingToRolePayload.newBuilder() - .setStagedName(stagedName) + artifactBuilder.setTypePayload( + RunnerApi.ArtifactFilePayload.newBuilder() + .setPath(file.getPath()) + .setSha256(hashCode.toString()) .build() .toByteString()); - artifactsBuilder.add(artifactBuilder.build()); } + if (stagedName == null) { + stagedName = createStagingFileName(file, hashCode); + } + artifactBuilder.setRolePayload( + RunnerApi.ArtifactStagingToRolePayload.newBuilder() + .setStagedName(stagedName) + .build() + .toByteString()); + artifactsBuilder.add(artifactBuilder.build()); } return artifactsBuilder.build(); } diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ExecutableStageTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ExecutableStageTranslation.java index b445e0bb3136a..822418c0ff4d9 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ExecutableStageTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ExecutableStageTranslation.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.util.ArrayList; @@ -28,10 +28,10 @@ import org.apache.beam.model.pipeline.v1.RunnerApi.ExecutableStagePayload; import org.apache.beam.runners.core.construction.graph.ExecutableStage; import org.apache.beam.sdk.runners.AppliedPTransform; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.LinkedHashMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.LinkedHashMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; /** * Utilities for converting {@link ExecutableStage}s to and from {@link RunnerApi} protocol buffers. diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/External.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/External.java index e87183f3af650..534a2b5fe0e60 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/External.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/External.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.FileOutputStream; import java.io.IOException; @@ -53,10 +53,10 @@ import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ManagedChannel; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ManagedChannelBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; /** @@ -295,7 +295,7 @@ public OutputT expand(InputT input) { response .getComponents() .toBuilder() - .putAllEnvironments(resolveArtifacts(newEnvironmentsWithDependencies)) + .putAllEnvironments(resolveArtifacts(newEnvironmentsWithDependencies, endpoint)) .build(); expandedTransform = response.getTransform(); expandedRequirements = response.getRequirementsList(); @@ -338,8 +338,8 @@ public OutputT expand(InputT input) { return toOutputCollection(outputMapBuilder.build()); } - private Map resolveArtifacts( - Map environments) { + static Map resolveArtifacts( + Map environments, Endpoints.ApiServiceDescriptor endpoint) { if (environments.size() == 0) { return environments; } @@ -367,7 +367,7 @@ private Map resolveArtifacts( } } - private RunnerApi.Environment resolveArtifacts( + private static RunnerApi.Environment resolveArtifacts( ArtifactRetrievalServiceGrpc.ArtifactRetrievalServiceBlockingStub retrievalStub, RunnerApi.Environment environment) throws IOException { @@ -378,7 +378,7 @@ private RunnerApi.Environment resolveArtifacts( .build(); } - private List resolveArtifacts( + private static List resolveArtifacts( ArtifactRetrievalServiceGrpc.ArtifactRetrievalServiceBlockingStub retrievalStub, List artifacts) throws IOException { diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ExternalTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ExternalTranslation.java index 7348d368a427c..eb4c9f8a0beea 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ExternalTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ExternalTranslation.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.util.List; @@ -28,7 +28,8 @@ import org.apache.beam.sdk.runners.AppliedPTransform; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** Translating External transforms to proto. */ @@ -120,7 +121,8 @@ public RunnerApi.PTransform translate( entry .getValue() .toBuilder() - .setCoderId(coderRenameMap.getOrDefault(coderId, coderId)) + .setCoderId( + Preconditions.checkNotNull(coderRenameMap.getOrDefault(coderId, coderId))) .build()); } } @@ -141,12 +143,14 @@ public RunnerApi.PTransform translate( for (Map.Entry inputEntry : proto.getInputsMap().entrySet()) { transformBuilder.putInputs( inputEntry.getKey(), - pColRenameMap.getOrDefault(inputEntry.getValue(), inputEntry.getValue())); + Preconditions.checkNotNull( + pColRenameMap.getOrDefault(inputEntry.getValue(), inputEntry.getValue()))); } for (Map.Entry outputEntry : proto.getOutputsMap().entrySet()) { transformBuilder.putOutputs( outputEntry.getKey(), - pColRenameMap.getOrDefault(outputEntry.getValue(), outputEntry.getValue())); + Preconditions.checkNotNull( + pColRenameMap.getOrDefault(outputEntry.getValue(), outputEntry.getValue()))); } mergingComponentsBuilder.putTransforms(entry.getKey(), transformBuilder.build()); } @@ -161,7 +165,8 @@ public RunnerApi.PTransform translate( for (Map.Entry outputEntry : expandedTransform.getOutputsMap().entrySet()) { rootTransformBuilder.putOutputs( outputEntry.getKey(), - pColRenameMap.getOrDefault(outputEntry.getValue(), outputEntry.getValue())); + Preconditions.checkNotNull( + pColRenameMap.getOrDefault(outputEntry.getValue(), outputEntry.getValue()))); } components.mergeFrom(mergingComponentsBuilder.build(), null); diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ExternalTranslationOptions.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ExternalTranslationOptions.java new file mode 100644 index 0000000000000..4b3ef24ca1d2a --- /dev/null +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ExternalTranslationOptions.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.core.construction; + +import java.util.List; +import org.apache.beam.sdk.options.Default; +import org.apache.beam.sdk.options.Description; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.transforms.resourcehints.ResourceHintsOptions.EmptyListDefault; + +public interface ExternalTranslationOptions extends PipelineOptions { + + @Description("Set of URNs of transforms to be overriden using the transform service.") + @Default.InstanceFactory(EmptyListDefault.class) + List getTransformsToOverride(); + + void setTransformsToOverride(List transformsToOverride); + + @Description("Address of an already available transform service.") + String getTransformServiceAddress(); + + void setTransformServiceAddress(String transformServiceAddress); + + @Description("An available Beam version which will be used to start a transform service.") + String getTransformServiceBeamVersion(); + + void setTransformServiceBeamVersion(String transformServiceBeamVersion); +} diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ExternalTranslationOptionsRegistrar.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ExternalTranslationOptionsRegistrar.java new file mode 100644 index 0000000000000..6296f4c83775f --- /dev/null +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ExternalTranslationOptionsRegistrar.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.core.construction; + +import com.google.auto.service.AutoService; +import org.apache.beam.sdk.annotations.Internal; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.PipelineOptionsRegistrar; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; + +/** A registrar for ExternalTranslationOptions. */ +@AutoService(PipelineOptionsRegistrar.class) +@Internal +public class ExternalTranslationOptionsRegistrar implements PipelineOptionsRegistrar { + @Override + public Iterable> getPipelineOptions() { + return ImmutableList.>builder() + .add(ExternalTranslationOptions.class) + .build(); + } +} diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/FlattenTranslator.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/FlattenTranslator.java index 37c09663c5a24..201a65e6233c8 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/FlattenTranslator.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/FlattenTranslator.java @@ -43,7 +43,7 @@ public static TransformPayloadTranslator create() { private FlattenTranslator() {} @Override - public String getUrn(Flatten.PCollections transform) { + public String getUrn() { return PTransformTranslation.FLATTEN_TRANSFORM_URN; } diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/GroupByKeyTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/GroupByKeyTranslation.java index e6bbbf0767a54..183fa7ffcdc93 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/GroupByKeyTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/GroupByKeyTranslation.java @@ -38,7 +38,7 @@ public class GroupByKeyTranslation { static class GroupByKeyTranslator implements TransformPayloadTranslator> { @Override - public String getUrn(GroupByKey transform) { + public String getUrn() { return PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN; } diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/GroupIntoBatchesTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/GroupIntoBatchesTranslation.java index 88b0c5c9d69f5..7c81afd8ae071 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/GroupIntoBatchesTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/GroupIntoBatchesTranslation.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.runners.AppliedPTransform; import org.apache.beam.sdk.transforms.GroupIntoBatches; import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; @SuppressWarnings({ "rawtypes", // TODO(https://github.com/apache/beam/issues/20447) @@ -39,7 +39,7 @@ public class GroupIntoBatchesTranslation { static class GroupIntoBatchesTranslator implements TransformPayloadTranslator> { @Override - public String getUrn(GroupIntoBatches transform) { + public String getUrn() { return PTransformTranslation.GROUP_INTO_BATCHES_URN; } @@ -61,7 +61,7 @@ public RunnerApi.FunctionSpec translate( static class ShardedGroupIntoBatchesTranslator implements TransformPayloadTranslator.WithShardedKey> { @Override - public String getUrn(GroupIntoBatches.WithShardedKey transform) { + public String getUrn() { return PTransformTranslation.GROUP_INTO_BATCHES_WITH_SHARDED_KEY_URN; } diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ImpulseTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ImpulseTranslation.java index 3de0ce9de8acb..25f0cd7749b5d 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ImpulseTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ImpulseTranslation.java @@ -37,7 +37,7 @@ public class ImpulseTranslation { private static class ImpulseTranslator implements TransformPayloadTranslator { @Override - public String getUrn(Impulse transform) { + public String getUrn() { return PTransformTranslation.IMPULSE_TRANSFORM_URN; } diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ModelCoderRegistrar.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ModelCoderRegistrar.java index 8078b7c1a5975..01a69f4412902 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ModelCoderRegistrar.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ModelCoderRegistrar.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.service.AutoService; import java.util.Collections; @@ -39,10 +39,10 @@ import org.apache.beam.sdk.util.ShardedKey; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableBiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableBiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; /** The {@link CoderTranslatorRegistrar} for coders which are shared across languages. */ @AutoService(CoderTranslatorRegistrar.class) diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ModelCoders.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ModelCoders.java index c2b45d354fb00..e428d58d48030 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ModelCoders.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ModelCoders.java @@ -18,8 +18,8 @@ package org.apache.beam.runners.core.construction; import static org.apache.beam.runners.core.construction.BeamUrns.getUrn; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.util.Set; @@ -27,7 +27,7 @@ import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec; import org.apache.beam.model.pipeline.v1.RunnerApi.StandardCoders; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; /** Utilities and constants ot interact with coders that are part of the Beam Model. */ public class ModelCoders { diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionViewTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionViewTranslation.java index ae224c92a5485..6ccf673af0d1a 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionViewTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PCollectionViewTranslation.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import org.apache.beam.model.pipeline.v1.RunnerApi; diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java index bf3f135dae558..aec269b5929db 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java @@ -38,7 +38,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollection.IsBounded; import org.apache.beam.sdk.values.PValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformReplacements.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformReplacements.java index 37a2e4b385cdb..7dbc0b964f19b 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformReplacements.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformReplacements.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Map; import java.util.Set; @@ -25,7 +25,7 @@ import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** */ @SuppressWarnings({"nullness", "keyfor"}) // TODO(https://github.com/apache/beam/issues/20497) diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java index 485350715c9ec..8f415e718e957 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.core.construction; import static org.apache.beam.runners.core.construction.BeamUrns.getUrn; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.util.Collection; @@ -37,23 +37,30 @@ import org.apache.beam.runners.core.construction.ExternalTranslation.ExternalTranslator; import org.apache.beam.runners.core.construction.ParDoTranslation.ParDoTranslator; import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.coders.RowCoder; import org.apache.beam.sdk.io.Read; import org.apache.beam.sdk.runners.AppliedPTransform; +import org.apache.beam.sdk.schemas.SchemaTranslation; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.display.DisplayData; +import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.util.common.ReflectHelpers.ObjectsClassComparator; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PInput; import org.apache.beam.sdk.values.POutput; +import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSortedSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSortedSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Utilities for converting {@link PTransform PTransforms} to {@link RunnerApi Runner API protocol @@ -65,10 +72,14 @@ "keyfor" }) // TODO(https://github.com/apache/beam/issues/20497) public class PTransformTranslation { + + private static final Logger LOG = LoggerFactory.getLogger(PTransformTranslation.class); + // We specifically copy the values here so that they can be used in switch case statements // and we validate that the value matches the actual URN in the static block below. // Primitives + public static final String CREATE_TRANSFORM_URN = "beam:transform:create:v1"; public static final String PAR_DO_TRANSFORM_URN = "beam:transform:pardo:v1"; public static final String FLATTEN_TRANSFORM_URN = "beam:transform:flatten:v1"; public static final String GROUP_BY_KEY_TRANSFORM_URN = "beam:transform:group_by_key:v1"; @@ -83,6 +94,10 @@ public class PTransformTranslation { public static final ImmutableSet RUNNER_IMPLEMENTED_TRANSFORMS = ImmutableSet.of(GROUP_BY_KEY_TRANSFORM_URN, IMPULSE_TRANSFORM_URN); + public static final String CONFIG_ROW_KEY = "config_row"; + + public static final String CONFIG_ROW_SCHEMA_KEY = "config_row_schema"; + // DeprecatedPrimitives /** * @deprecated SDKs should move away from creating `Read` transforms and migrate to using Impulse @@ -435,10 +450,9 @@ public RunnerApi.PTransform translate( RunnerApi.PTransform.Builder transformBuilder = translateAppliedPTransform(appliedPTransform, subtransforms, components); - FunctionSpec spec = - KNOWN_PAYLOAD_TRANSLATORS - .get(appliedPTransform.getTransform().getClass()) - .translate(appliedPTransform, components); + TransformPayloadTranslator payloadTranslator = + KNOWN_PAYLOAD_TRANSLATORS.get(appliedPTransform.getTransform().getClass()); + FunctionSpec spec = payloadTranslator.translate(appliedPTransform, components); if (spec != null) { transformBuilder.setSpec(spec); @@ -461,6 +475,33 @@ public RunnerApi.PTransform translate( } } } + + Row configRow = null; + try { + configRow = payloadTranslator.toConfigRow(appliedPTransform.getTransform()); + } catch (UnsupportedOperationException e) { + // Optional toConfigRow() has not been implemented. We can just ignore. + } catch (Exception e) { + LOG.warn( + "Could not attach the config row for transform " + + appliedPTransform.getTransform().getName() + + ": " + + e); + // Ignoring the error and continuing with the translation since attaching config rows is + // optional. + } + if (configRow != null) { + transformBuilder.putAnnotations( + CONFIG_ROW_KEY, + ByteString.copyFrom( + CoderUtils.encodeToByteArray(RowCoder.of(configRow.getSchema()), configRow))); + + transformBuilder.putAnnotations( + CONFIG_ROW_SCHEMA_KEY, + ByteString.copyFrom( + SchemaTranslation.schemaToProto(configRow.getSchema(), true).toByteArray())); + } + return transformBuilder.build(); } } @@ -508,14 +549,63 @@ static RunnerApi.PTransform.Builder translateAppliedPTransform( * *

    When going to a protocol buffer message, the translator produces a payload corresponding to * the Java representation while registering components that payload references. + * + *

    Also, provides methods for generating a Row-based constructor config for the transform that + * can be later used to re-construct the transform. */ public interface TransformPayloadTranslator> { - String getUrn(T transform); + /** + * Provides a unique URN for transforms represented by this {@code TransformPayloadTranslator}. + */ + String getUrn(); + + /** + * Same as {@link #getUrn()} but the returned URN may depend on the transform provided. + * + *

    Only override this if the same {@code TransformPayloadTranslator} used for multiple + * transforms. Otherwise, use {@link #getUrn()}. + */ + default String getUrn(T transform) { + return getUrn(); + } + + /** */ + /** + * Translates the given transform represented by the provided {@code AppliedPTransform} to a + * {@code FunctionSpec} with a URN and a payload. + * + * @param application an {@code AppliedPTransform} that includes the transform to be expanded. + * @param components components of the pipeline that includes the transform. + * @return a generated spec for the transform to be included in the pipeline proto. If return + * value is null, transform should include an empty spec. + * @throws IOException + */ @Nullable FunctionSpec translate(AppliedPTransform application, SdkComponents components) throws IOException; + /** + * Generates a Row-based construction configuration for the provided transform. + * + * @param transform a transform represented by the current {@code TransformPayloadTranslator}. + * @return + */ + default Row toConfigRow(T transform) { + throw new UnsupportedOperationException("Not implemented"); + } + + /** + * Construts a transform from a provided Row-based construction configuration. + * + * @param configRow a construction configuration similar to what would be generated by the + * {@link #toConfigRow(PTransform)} method. + * @return a transform represented by the current {@code TransformPayloadTranslator}. + */ + default T fromConfigRow(Row configRow) { + throw new UnsupportedOperationException("Not implemented"); + } + /** * A {@link TransformPayloadTranslator} for transforms that contain no references to components, * so they do not need a specialized rehydration. @@ -526,7 +616,7 @@ abstract class NotSerializable> public static NotSerializable forUrn(final String urn) { return new NotSerializable>() { @Override - public String getUrn(PTransform transform) { + public String getUrn() { return urn; } }; diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java index d8bb5bb3f1ac5..5ef11a747e878 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ParDoTranslation.java @@ -26,9 +26,9 @@ import static org.apache.beam.runners.core.construction.PTransformTranslation.SPLITTABLE_TRUNCATE_SIZED_RESTRICTION_URN; import static org.apache.beam.sdk.transforms.reflect.DoFnSignatures.getStateSpecOrThrow; import static org.apache.beam.sdk.transforms.reflect.DoFnSignatures.getTimerSpecOrThrow; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.io.IOException; @@ -81,9 +81,9 @@ import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; /** Utilities for interacting with {@link ParDo} instances and {@link ParDoPayload} protos. */ @SuppressWarnings({ diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineOptionsTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineOptionsTranslation.java index 0c38790e2a2ba..4a23c62372671 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineOptionsTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineOptionsTranslation.java @@ -30,8 +30,8 @@ import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Struct; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.util.JsonFormat; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.CaseFormat; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CaseFormat; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * Utilities for going to/from Runner API pipeline options. diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineTranslation.java index f324686154a3d..e39a38a74c2ce 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineTranslation.java @@ -32,9 +32,9 @@ import org.apache.beam.sdk.Pipeline.PipelineVisitor; import org.apache.beam.sdk.runners.AppliedPTransform; import org.apache.beam.sdk.runners.TransformHierarchy.Node; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ListMultimap; /** Utilities for going to/from Runner API pipelines. */ public class PipelineTranslation { @@ -102,6 +102,21 @@ public void visitPrimitiveTransform(Node node) { // TODO(JIRA-5649): Don't even emit these transforms in the generated protos. res = elideDeprecatedViews(res); } + + ExternalTranslationOptions externalTranslationOptions = + pipeline.getOptions().as(ExternalTranslationOptions.class); + List urnsToOverride = externalTranslationOptions.getTransformsToOverride(); + if (urnsToOverride.size() > 0) { + try (TransformUpgrader upgrader = TransformUpgrader.of()) { + res = + upgrader.upgradeTransformsViaTransformService( + res, urnsToOverride, externalTranslationOptions); + } catch (Exception e) { + throw new RuntimeException( + "Could not override the transforms with URNs " + urnsToOverride, e); + } + } + // Validate that translation didn't produce an invalid pipeline. PipelineValidator.validate(res); return res; diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PrimitiveCreate.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PrimitiveCreate.java index 2cd62d5a2b8e9..d91ddbc079bc4 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PrimitiveCreate.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PrimitiveCreate.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.values.PCollection.IsBounded; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** An implementation of {@link Create} that returns a primitive {@link PCollection}. */ public class PrimitiveCreate extends PTransform> { diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java index 7a4caef51181f..f04d25509593a 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReadTranslation.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; import java.io.IOException; @@ -38,7 +38,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * Methods for translating {@link SplittableParDo.PrimitiveBoundedRead} and {@link @@ -154,7 +154,7 @@ public static TransformPayloadTranslator create() { private UnboundedReadPayloadTranslator() {} @Override - public String getUrn(SplittableParDo.PrimitiveUnboundedRead transform) { + public String getUrn() { return PTransformTranslation.READ_TRANSFORM_URN; } @@ -181,7 +181,7 @@ public static TransformPayloadTranslator create() { private BoundedReadPayloadTranslator() {} @Override - public String getUrn(SplittableParDo.PrimitiveBoundedRead transform) { + public String getUrn() { return PTransformTranslation.READ_TRANSFORM_URN; } diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RehydratedComponents.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RehydratedComponents.java index 5d59855d75c50..f1bbc521a096e 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RehydratedComponents.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RehydratedComponents.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.util.Collection; @@ -31,9 +31,9 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LoadingCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReplacementOutputs.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReplacementOutputs.java index 553f05f0d26a1..6ff0d8c0ec030 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReplacementOutputs.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReplacementOutputs.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Collections; import java.util.HashMap; @@ -29,8 +29,8 @@ import org.apache.beam.sdk.values.PValues; import org.apache.beam.sdk.values.TaggedPValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** Utility methods for creating {@link ReplacementOutput} for known styles of {@link POutput}. */ @SuppressWarnings({ diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReshuffleTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReshuffleTranslation.java index 98d39c8ff0ac4..bd91673f38181 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReshuffleTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/ReshuffleTranslation.java @@ -38,7 +38,7 @@ public class ReshuffleTranslation { static class ReshuffleTranslator implements TransformPayloadTranslator> { @Override - public String getUrn(Reshuffle transform) { + public String getUrn() { return PTransformTranslation.RESHUFFLE_URN; } diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java index accf86424e2b2..feeb5feda1b1e 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SdkComponents.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.text.Normalizer; @@ -42,12 +42,12 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashBiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; /** SDK objects that will be represented at some later point within a {@link Components} object. */ diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java index 8302b87e4f22a..5ea2c4968dd98 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; import java.io.IOException; @@ -75,9 +75,9 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; @@ -395,7 +395,7 @@ public static TransformPayloadTranslator create() { private ProcessKeyedElementsTranslator() {} @Override - public String getUrn(ProcessKeyedElements transform) { + public String getUrn() { return PTransformTranslation.SPLITTABLE_PROCESS_KEYED_URN; } diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDoNaiveBounded.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDoNaiveBounded.java index 7cccab966f41e..061e1cb11b5c1 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDoNaiveBounded.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDoNaiveBounded.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -53,7 +53,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java index 360ea94c0fc9e..53bb324d03fa0 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.core.construction; import static org.apache.beam.runners.core.construction.PTransformTranslation.TEST_STREAM_TRANSFORM_URN; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; import java.io.IOException; @@ -168,7 +168,7 @@ static TestStream.Event eventFromProto( /** A translator registered to translate {@link TestStream} objects to protobuf representation. */ static class TestStreamTranslator implements TransformPayloadTranslator> { @Override - public String getUrn(TestStream transform) { + public String getUrn() { return TEST_STREAM_TRANSFORM_URN; } diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java index 850b8822d5e5c..52fcd3fe031e4 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Collection; import java.util.Map; @@ -26,7 +26,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Utilities for extracting subsets of inputs from an {@link AppliedPTransform}. */ public class TransformInputs { diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformUpgrader.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformUpgrader.java new file mode 100644 index 0000000000000..5e1609f27a399 --- /dev/null +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformUpgrader.java @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.core.construction; + +import java.io.IOException; +import java.net.ServerSocket; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import org.apache.beam.model.expansion.v1.ExpansionApi; +import org.apache.beam.model.pipeline.v1.Endpoints; +import org.apache.beam.model.pipeline.v1.ExternalTransforms; +import org.apache.beam.model.pipeline.v1.RunnerApi; +import org.apache.beam.model.pipeline.v1.RunnerApi.PTransform; +import org.apache.beam.model.pipeline.v1.SchemaApi; +import org.apache.beam.sdk.transformservice.launcher.TransformServiceLauncher; +import org.apache.beam.sdk.values.PInput; +import org.apache.beam.sdk.values.POutput; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ManagedChannelBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; + +/** + * A utility class that allows upgrading transforms of a given pipeline using the Beam Transform + * Service. + */ +public class TransformUpgrader implements AutoCloseable { + private static final String UPGRADE_NAMESPACE = "transform:upgrade:"; + + private ExpansionServiceClientFactory clientFactory; + + private TransformUpgrader() { + // Creating a default 'ExpansionServiceClientFactory' instance per 'TransformUpgrader' instance + // so that each instance can maintain a set of live channels and close them independently. + clientFactory = + DefaultExpansionServiceClientFactory.create( + endPoint -> ManagedChannelBuilder.forTarget(endPoint.getUrl()).usePlaintext().build()); + } + + private TransformUpgrader(ExpansionServiceClientFactory clientFactory) { + this.clientFactory = clientFactory; + } + + public static TransformUpgrader of() { + return new TransformUpgrader(); + } + + @VisibleForTesting + static TransformUpgrader of(ExpansionServiceClientFactory clientFactory) { + return new TransformUpgrader(clientFactory); + } + + /** + * Upgrade identified transforms in a given pipeline using the Transform Service. + * + * @param pipeline the pipeline proto. + * @param urnsToOverride URNs of the transforms to be overridden. + * @param options options for determining the transform service to use. + * @return pipelines with transforms upgraded using the Transform Service. + * @throws Exception + */ + public RunnerApi.Pipeline upgradeTransformsViaTransformService( + RunnerApi.Pipeline pipeline, List urnsToOverride, ExternalTranslationOptions options) + throws IOException, TimeoutException { + List transformsToOverride = + pipeline.getComponents().getTransformsMap().entrySet().stream() + .filter( + entry -> { + String urn = entry.getValue().getSpec().getUrn(); + if (urn != null && urnsToOverride.contains(urn)) { + return true; + } + return false; + }) + .map( + entry -> { + return entry.getKey(); + }) + .collect(Collectors.toList()); + + String serviceAddress; + TransformServiceLauncher service = null; + + if (options.getTransformServiceAddress() != null) { + serviceAddress = options.getTransformServiceAddress(); + } else if (options.getTransformServiceBeamVersion() != null) { + String projectName = UUID.randomUUID().toString(); + int port = findAvailablePort(); + service = TransformServiceLauncher.forProject(projectName, port, null); + service.setBeamVersion(options.getTransformServiceBeamVersion()); + + // Starting the transform service. + service.start(); + service.waitTillUp(-1); + serviceAddress = "localhost:" + Integer.toString(port); + } else { + throw new IllegalArgumentException( + "Either option TransformServiceAddress or option TransformServiceBeamVersion should be " + + "provided to override a transform using the transform service"); + } + + Endpoints.ApiServiceDescriptor expansionServiceEndpoint = + Endpoints.ApiServiceDescriptor.newBuilder().setUrl(serviceAddress).build(); + + for (String transformId : transformsToOverride) { + pipeline = + updateTransformViaTransformService(pipeline, transformId, expansionServiceEndpoint); + } + + if (service != null) { + service.shutdown(); + } + + return pipeline; + } + + private < + InputT extends PInput, + OutputT extends POutput, + TransformT extends org.apache.beam.sdk.transforms.PTransform> + RunnerApi.Pipeline updateTransformViaTransformService( + RunnerApi.Pipeline runnerAPIpipeline, + String transformId, + Endpoints.ApiServiceDescriptor transformServiceEndpoint) + throws IOException { + PTransform transformToUpgrade = + runnerAPIpipeline.getComponents().getTransformsMap().get(transformId); + if (transformToUpgrade == null) { + throw new IllegalArgumentException("Could not find a transform with the ID " + transformId); + } + + ByteString configRowBytes = + transformToUpgrade.getAnnotationsOrThrow(PTransformTranslation.CONFIG_ROW_KEY); + ByteString configRowSchemaBytes = + transformToUpgrade.getAnnotationsOrThrow(PTransformTranslation.CONFIG_ROW_SCHEMA_KEY); + SchemaApi.Schema configRowSchemaProto = + SchemaApi.Schema.parseFrom(configRowSchemaBytes.toByteArray()); + + ExternalTransforms.ExternalConfigurationPayload payload = + ExternalTransforms.ExternalConfigurationPayload.newBuilder() + .setSchema(configRowSchemaProto) + .setPayload(configRowBytes) + .build(); + + RunnerApi.PTransform.Builder ptransformBuilder = + RunnerApi.PTransform.newBuilder() + .setUniqueName(transformToUpgrade.getUniqueName() + "_external") + .setSpec( + RunnerApi.FunctionSpec.newBuilder() + .setUrn(transformToUpgrade.getSpec().getUrn()) + .setPayload(ByteString.copyFrom(payload.toByteArray())) + .build()); + + for (Map.Entry entry : transformToUpgrade.getInputsMap().entrySet()) { + ptransformBuilder.putInputs(entry.getKey(), entry.getValue()); + } + for (Map.Entry entry : transformToUpgrade.getOutputsMap().entrySet()) { + ptransformBuilder.putOutputs(entry.getKey(), entry.getValue()); + } + + ExpansionApi.ExpansionRequest.Builder requestBuilder = + ExpansionApi.ExpansionRequest.newBuilder(); + ExpansionApi.ExpansionRequest request = + requestBuilder + .setComponents(runnerAPIpipeline.getComponents()) + .setTransform(ptransformBuilder.build()) + .setNamespace(UPGRADE_NAMESPACE) + .build(); + + ExpansionApi.ExpansionResponse response = + clientFactory.getExpansionServiceClient(transformServiceEndpoint).expand(request); + + if (!Strings.isNullOrEmpty(response.getError())) { + throw new RuntimeException(String.format("expansion service error: %s", response.getError())); + } + + Map newEnvironmentsWithDependencies = + response.getComponents().getEnvironmentsMap().entrySet().stream() + .filter( + kv -> + !runnerAPIpipeline.getComponents().getEnvironmentsMap().containsKey(kv.getKey()) + && kv.getValue().getDependenciesCount() != 0) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + RunnerApi.Components expandedComponents = + response + .getComponents() + .toBuilder() + .putAllEnvironments( + External.ExpandableTransform.resolveArtifacts( + newEnvironmentsWithDependencies, transformServiceEndpoint)) + .build(); + RunnerApi.PTransform expandedTransform = response.getTransform(); + List expandedRequirements = response.getRequirementsList(); + + RunnerApi.Components.Builder newComponentsBuilder = expandedComponents.toBuilder(); + + // We record transforms that consume outputs of the old transform and update them to consume + // outputs of the new (upgraded) transform. + Collection oldOutputs = transformToUpgrade.getOutputsMap().values(); + Map inputReplacements = new HashMap<>(); + if (transformToUpgrade.getOutputsMap().size() == 1) { + inputReplacements.put( + oldOutputs.iterator().next(), + expandedTransform.getOutputsMap().values().iterator().next()); + } else { + for (Map.Entry entry : transformToUpgrade.getOutputsMap().entrySet()) { + if (expandedTransform.getOutputsMap().keySet().contains(entry.getKey())) { + throw new IllegalArgumentException( + "Original transform did not have an output with tag " + + entry.getKey() + + " but upgraded transform did."); + } + String newOutput = expandedTransform.getOutputsMap().get(entry.getKey()); + if (newOutput == null) { + throw new IllegalArgumentException( + "Could not find an output with tag " + + entry.getKey() + + " for the transform " + + expandedTransform); + } + inputReplacements.put(entry.getValue(), newOutput); + } + } + + // The list of obsolete (overridden) transforms that should be removed from the pipeline + // produced by this method. + List transformsToRemove = new ArrayList<>(); + recursivelyFindSubTransforms( + transformId, runnerAPIpipeline.getComponents(), transformsToRemove); + + Map updatedExpandedTransformMap = + expandedComponents.getTransformsMap().entrySet().stream() + .filter( + entry -> { + // Do not include already overridden transforms. + return !transformsToRemove.contains(entry.getKey()); + }) + .collect( + Collectors.toMap( + entry -> entry.getKey(), + entry -> { + // Fix inputs + Map inputsMap = entry.getValue().getInputsMap(); + PTransform.Builder transformBuilder = entry.getValue().toBuilder(); + if (!Collections.disjoint(inputsMap.values(), inputReplacements.keySet())) { + Map updatedInputsMap = new HashMap<>(); + for (Map.Entry inputEntry : inputsMap.entrySet()) { + String updaterValue = + inputReplacements.containsKey(inputEntry.getValue()) + ? inputReplacements.get(inputEntry.getValue()) + : inputEntry.getValue(); + updatedInputsMap.put(inputEntry.getKey(), updaterValue); + } + transformBuilder.clearInputs(); + transformBuilder.putAllInputs(updatedInputsMap); + } + return transformBuilder.build(); + })); + + newComponentsBuilder.clearTransforms(); + newComponentsBuilder.putAllTransforms(updatedExpandedTransformMap); + newComponentsBuilder.putTransforms(transformId, expandedTransform); + + RunnerApi.Pipeline.Builder newRunnerAPIPipelineBuilder = runnerAPIpipeline.toBuilder(); + newRunnerAPIPipelineBuilder.clearComponents(); + newRunnerAPIPipelineBuilder.setComponents(newComponentsBuilder.build()); + + newRunnerAPIPipelineBuilder.addAllRequirements(expandedRequirements); + + return newRunnerAPIPipelineBuilder.build(); + } + + private static void recursivelyFindSubTransforms( + String transformId, RunnerApi.Components components, List results) { + results.add(transformId); + PTransform transform = components.getTransformsMap().get(transformId); + if (transform == null) { + throw new IllegalArgumentException("Could not find a transform with id " + transformId); + } + List subTransforms = transform.getSubtransformsList(); + if (subTransforms != null) { + for (String subTransformId : subTransforms) { + recursivelyFindSubTransforms(subTransformId, components, results); + } + } + } + + private static int findAvailablePort() throws IOException { + ServerSocket s = new ServerSocket(0); + try { + return s.getLocalPort(); + } finally { + s.close(); + try { + // Some systems don't free the port for future use immediately. + Thread.sleep(100); + } catch (InterruptedException exn) { + // ignore + } + } + } + + @Override + public void close() throws Exception { + clientFactory.close(); + } +} diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TriggerTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TriggerTranslation.java index 31871b1747981..d4b5da958f314 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TriggerTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TriggerTranslation.java @@ -40,8 +40,8 @@ import org.apache.beam.sdk.transforms.windowing.TimestampTransform; import org.apache.beam.sdk.transforms.windowing.Trigger; import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java index 0761b45086a29..53fad782da968 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSource.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.io.InputStream; @@ -47,9 +47,9 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; import org.slf4j.Logger; @@ -288,6 +288,15 @@ private void init( residualElementsList == null ? new ResidualElements(Collections.emptyList()) : new ResidualElements(residualElementsList); + + if (this.residualSource != null) { + // close current residualSource to avoid leak of reader.close() in ResidualSource + try { + this.residualSource.close(); + } catch (IOException e) { + LOG.warn("Ignore error at closing ResidualSource", e); + } + } this.residualSource = residualSource == null ? null : new ResidualSource(residualSource, options); } @@ -465,7 +474,7 @@ public ResidualSource(BoundedSource residualSource, PipelineOptions options) } private boolean advance() throws IOException { - checkArgument(!closed, "advance() call on closed %s", getClass().getName()); + checkState(!closed, "advance() call on closed %s", getClass().getName()); if (readerDone) { return false; } @@ -505,6 +514,7 @@ BoundedSource getSource() { } Checkpoint getCheckpointMark() { + checkState(!closed, "getCheckpointMark() call on closed %s", getClass().getName()); if (reader == null) { // Reader hasn't started, checkpoint the residualSource. return new Checkpoint<>(null /* residualElements */, residualSource); diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java index 2eb1f4fcdaf79..294d89308a31e 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowIntoTranslation.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; import java.io.IOException; @@ -30,7 +30,6 @@ import org.apache.beam.sdk.runners.AppliedPTransform; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.windowing.Window; -import org.apache.beam.sdk.transforms.windowing.Window.Assign; import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; import org.checkerframework.checker.nullness.qual.Nullable; @@ -47,7 +46,7 @@ public class WindowIntoTranslation { static class WindowAssignTranslator implements TransformPayloadTranslator> { @Override - public String getUrn(Assign transform) { + public String getUrn() { return PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN; } @@ -116,7 +115,7 @@ public static TransformPayloadTranslator create() { private WindowIntoPayloadTranslator() {} @Override - public String getUrn(Window.Assign transform) { + public String getUrn() { return PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN; } diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java index 5b1453bacae80..0acf8c97b763b 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslation.java @@ -47,7 +47,7 @@ import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.util.Durations; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.util.Timestamps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.joda.time.Duration; /** Utilities for working with {@link WindowingStrategy WindowingStrategies}. */ diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WriteFilesTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WriteFilesTranslation.java index cef8a708fc728..3a23ed073776b 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WriteFilesTranslation.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/WriteFilesTranslation.java @@ -18,8 +18,8 @@ package org.apache.beam.runners.core.construction; import static org.apache.beam.runners.core.construction.PTransformTranslation.WRITE_FILES_TRANSFORM_URN; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.service.AutoService; import java.io.IOException; @@ -46,9 +46,9 @@ import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; /** * Utility methods for translating a {@link WriteFiles} to and from {@link RunnerApi} @@ -276,7 +276,7 @@ public boolean isRunnerDeterminedSharding() { static class WriteFilesTranslator implements TransformPayloadTranslator> { @Override - public String getUrn(WriteFiles transform) { + public String getUrn() { return WRITE_FILES_TRANSFORM_URN; } diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/FieldAccessVisitor.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/FieldAccessVisitor.java index d63c71b361078..ac9d505ae8655 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/FieldAccessVisitor.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/FieldAccessVisitor.java @@ -30,9 +30,9 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.ParDo.MultiOutput; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** Computes which Schema fields are (or conversely, are not) accessed in a pipeline. */ class FieldAccessVisitor extends PipelineVisitor.Defaults { diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/FusedPipeline.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/FusedPipeline.java index f9af79018cfb4..a8f8743c9ff9f 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/FusedPipeline.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/FusedPipeline.java @@ -30,7 +30,7 @@ import org.apache.beam.model.pipeline.v1.RunnerApi.Pipeline; import org.apache.beam.runners.core.construction.SyntheticComponents; import org.apache.beam.runners.core.construction.graph.PipelineNode.PTransformNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; /** A {@link Pipeline} which has been separated into collections of executable components. */ @AutoValue diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/GreedyPCollectionFusers.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/GreedyPCollectionFusers.java index 784d52cb6a0f3..4265264f6b5e6 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/GreedyPCollectionFusers.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/GreedyPCollectionFusers.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction.graph; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Collection; import java.util.Map; @@ -32,8 +32,8 @@ import org.apache.beam.runners.core.construction.graph.PipelineNode.PTransformNode; import org.apache.beam.sdk.transforms.Flatten; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/GreedyPipelineFuser.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/GreedyPipelineFuser.java index d00f2c1c16fd2..0d53ca7c18b76 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/GreedyPipelineFuser.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/GreedyPipelineFuser.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.core.construction.graph; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.util.ArrayDeque; @@ -43,12 +43,12 @@ import org.apache.beam.runners.core.construction.graph.OutputDeduplicator.DeduplicationResult; import org.apache.beam.runners.core.construction.graph.PipelineNode.PCollectionNode; import org.apache.beam.runners.core.construction.graph.PipelineNode.PTransformNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ComparisonChain; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ComparisonChain; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/GreedyStageFuser.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/GreedyStageFuser.java index eb02aad3a9859..6245577f14d14 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/GreedyStageFuser.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/GreedyStageFuser.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.core.construction.graph; import static org.apache.beam.runners.core.construction.graph.ExecutableStage.DEFAULT_WIRE_CODER_SETTINGS; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.ArrayDeque; import java.util.LinkedHashSet; @@ -29,7 +29,7 @@ import org.apache.beam.model.pipeline.v1.RunnerApi.Environment; import org.apache.beam.runners.core.construction.graph.PipelineNode.PCollectionNode; import org.apache.beam.runners.core.construction.graph.PipelineNode.PTransformNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ImmutableExecutableStage.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ImmutableExecutableStage.java index a66958f662337..56bd2ef6f4d3d 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ImmutableExecutableStage.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ImmutableExecutableStage.java @@ -25,7 +25,7 @@ import org.apache.beam.model.pipeline.v1.RunnerApi.ExecutableStagePayload.WireCoderSetting; import org.apache.beam.runners.core.construction.graph.PipelineNode.PCollectionNode; import org.apache.beam.runners.core.construction.graph.PipelineNode.PTransformNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; /** An {@link ExecutableStage} which is constructed with all of its initial state. */ @AutoValue diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/Networks.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/Networks.java index fcb781fd42a23..d9fdd8140d62d 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/Networks.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/Networks.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.core.construction.graph; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.ArrayDeque; @@ -35,16 +35,16 @@ import java.util.Set; import java.util.function.Function; import javax.annotation.Nonnull; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.ElementOrder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.EndpointPair; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.Graphs; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.MutableNetwork; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.Network; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.NetworkBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.ElementOrder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.EndpointPair; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.Graphs; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.MutableNetwork; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.Network; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.NetworkBuilder; /** Static utility methods for {@link Network} instances that are directed. */ @SuppressWarnings({ diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/OutputDeduplicator.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/OutputDeduplicator.java index cb8cbeb329870..d23bb2719020c 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/OutputDeduplicator.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/OutputDeduplicator.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction.graph; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.util.ArrayList; @@ -38,8 +38,8 @@ import org.apache.beam.runners.core.construction.SyntheticComponents; import org.apache.beam.runners.core.construction.graph.PipelineNode.PCollectionNode; import org.apache.beam.runners.core.construction.graph.PipelineNode.PTransformNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/PCollectionOutputTagVisitor.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/PCollectionOutputTagVisitor.java index 0a221e9cf4721..0d40154b09f2d 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/PCollectionOutputTagVisitor.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/PCollectionOutputTagVisitor.java @@ -30,9 +30,9 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableBiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableBiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * {@link PipelineVisitor} to convert projection pushdown targets from {@link PCollection} to {@link diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/PipelineValidator.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/PipelineValidator.java index 04a12d66fffa3..022548aaff043 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/PipelineValidator.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/PipelineValidator.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction.graph; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Map; import java.util.Set; @@ -33,10 +33,10 @@ import org.apache.beam.model.pipeline.v1.RunnerApi.WindowingStrategy; import org.apache.beam.runners.core.construction.PTransformTranslation; import org.apache.beam.runners.core.construction.ParDoTranslation; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; /** * Validates well-formedness of a pipeline. It is recommended to use this class on any user-supplied diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProjectionProducerVisitor.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProjectionProducerVisitor.java index 4e0fa3fb734e4..8a8bf125544f9 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProjectionProducerVisitor.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProjectionProducerVisitor.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.schemas.ProjectionProducer; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** A {@link PipelineVisitor} to discover projection pushdown opportunities. */ class ProjectionProducerVisitor extends PipelineVisitor.Defaults { diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProjectionPushdownOptimizer.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProjectionPushdownOptimizer.java index a3c6bb20cc258..3713a16b309ea 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProjectionPushdownOptimizer.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProjectionPushdownOptimizer.java @@ -35,8 +35,8 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.TaggedPValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProtoOverrides.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProtoOverrides.java index 6a7c3adf6cc7b..89b1e33657925 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProtoOverrides.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/ProtoOverrides.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction.graph; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.List; import java.util.Map; diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/QueryablePipeline.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/QueryablePipeline.java index 28cc10d3a98fa..1b89f282e7d58 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/QueryablePipeline.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/QueryablePipeline.java @@ -34,7 +34,7 @@ import static org.apache.beam.runners.core.construction.PTransformTranslation.SPLITTABLE_PROCESS_SIZED_ELEMENTS_AND_RESTRICTIONS_URN; import static org.apache.beam.runners.core.construction.PTransformTranslation.SPLITTABLE_SPLIT_AND_SIZE_RESTRICTIONS_URN; import static org.apache.beam.runners.core.construction.PTransformTranslation.TEST_STREAM_TRANSFORM_URN; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.ArrayDeque; import java.util.Collection; @@ -62,13 +62,13 @@ import org.apache.beam.runners.core.construction.graph.PipelineNode.PCollectionNode; import org.apache.beam.runners.core.construction.graph.PipelineNode.PTransformNode; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.MutableNetwork; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.Network; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.NetworkBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.MutableNetwork; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.Network; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.NetworkBuilder; /** * A {@link Pipeline} which has additional methods to relate nodes in the graph relative to each diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/SideInputReference.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/SideInputReference.java index 75369a5e08ae6..80584bf56d468 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/SideInputReference.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/SideInputReference.java @@ -24,7 +24,7 @@ import org.apache.beam.model.pipeline.v1.RunnerApi.PTransform; import org.apache.beam.runners.core.construction.graph.PipelineNode.PCollectionNode; import org.apache.beam.runners.core.construction.graph.PipelineNode.PTransformNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; /** * A reference to a side input. This includes the PTransform that references the side input as well diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/SplittableParDoExpander.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/SplittableParDoExpander.java index 434fd2c2ab732..0edcc1ab13e62 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/SplittableParDoExpander.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/graph/SplittableParDoExpander.java @@ -33,7 +33,7 @@ import org.apache.beam.runners.core.construction.PTransformTranslation; import org.apache.beam.runners.core.construction.ParDoTranslation; import org.apache.beam.runners.core.construction.graph.ProtoOverrides.TransformReplacement; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; /** * A set of transform replacements for expanding a splittable ParDo into various sub components. diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/resources/PipelineResources.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/resources/PipelineResources.java index 37ce403f909ed..433760a6562b1 100644 --- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/resources/PipelineResources.java +++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/resources/PipelineResources.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction.resources; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.File; import java.io.IOException; @@ -29,12 +29,12 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.ZipFiles; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Funnels; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hasher; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Funnels; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hasher; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java index b68887350d49d..b558c8b6d516f 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CoderTranslationTest.java @@ -57,8 +57,8 @@ import org.apache.beam.sdk.util.ShardedKey; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.hamcrest.Matchers; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CombineTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CombineTranslationTest.java index e036bfe97d2f4..a3a2bb5ba533d 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CombineTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CombineTranslationTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.junit.Assert.assertEquals; import java.io.IOException; @@ -43,7 +43,7 @@ import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Rule; import org.junit.Test; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslationTest.java index fccf1bd3af307..6974c01ccb12f 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslationTest.java @@ -39,7 +39,7 @@ import org.apache.beam.sdk.values.PCollectionViews.ValueOrMetadataCoder; import org.apache.beam.sdk.values.PValues; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.Matchers; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/DefaultArtifactResolverTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/DefaultArtifactResolverTest.java index d9cbcba058427..53c22af2ccafa 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/DefaultArtifactResolverTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/DefaultArtifactResolverTest.java @@ -24,7 +24,7 @@ import java.util.Optional; import org.apache.beam.model.pipeline.v1.RunnerApi; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EnvironmentsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EnvironmentsTest.java index 9a3ed7db6bc7e..ae429fb1fe6d7 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EnvironmentsTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EnvironmentsTest.java @@ -21,14 +21,18 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; +import java.io.File; import java.io.IOException; import java.io.Serializable; +import java.util.List; import java.util.Optional; import org.apache.beam.model.pipeline.v1.Endpoints; import org.apache.beam.model.pipeline.v1.RunnerApi; +import org.apache.beam.model.pipeline.v1.RunnerApi.ArtifactInformation; import org.apache.beam.model.pipeline.v1.RunnerApi.DockerPayload; import org.apache.beam.model.pipeline.v1.RunnerApi.Environment; import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec; @@ -41,6 +45,7 @@ import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.options.PortablePipelineOptions; +import org.apache.beam.sdk.testing.ExpectedLogs; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.DoFnSchemaInformation; import org.apache.beam.sdk.transforms.ParDo; @@ -49,7 +54,7 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -60,6 +65,7 @@ @RunWith(JUnit4.class) public class EnvironmentsTest implements Serializable { @Rule public transient ExpectedException thrown = ExpectedException.none(); + @Rule public transient ExpectedLogs expectedLogs = ExpectedLogs.none(Environments.class); @Test public void createEnvironmentDockerFromEnvironmentConfig() throws IOException { @@ -304,4 +310,43 @@ public void testNonLtsJavaVersion() { public void testJavaVersionStrictInvalid() { assertEquals(JavaVersion.java8, JavaVersion.forSpecificationStrict("invalid")); } + + @Test + public void testGetArtifactsExistingNoLogs() throws Exception { + File file1 = File.createTempFile("file1-", ".txt"); + file1.deleteOnExit(); + File file2 = File.createTempFile("file2-", ".txt"); + file2.deleteOnExit(); + + List artifacts = + Environments.getArtifacts(ImmutableList.of(file1.getAbsolutePath(), "file2=" + file2)); + + assertThat(artifacts, hasSize(2)); + expectedLogs.verifyNotLogged("was not found"); + } + + @Test + public void testGetArtifactsBadFileLogsInfo() throws Exception { + File file1 = File.createTempFile("file1-", ".txt"); + file1.deleteOnExit(); + + List artifacts = + Environments.getArtifacts(ImmutableList.of(file1.getAbsolutePath(), "spurious_file")); + + assertThat(artifacts, hasSize(1)); + expectedLogs.verifyInfo("'spurious_file' was not found"); + } + + @Test + public void testGetArtifactsBadNamedFileLogsWarn() throws Exception { + File file1 = File.createTempFile("file1-", ".txt"); + file1.deleteOnExit(); + + List artifacts = + Environments.getArtifacts( + ImmutableList.of(file1.getAbsolutePath(), "file_name=spurious_file")); + + assertThat(artifacts, hasSize(1)); + expectedLogs.verifyWarn("name 'file_name' was not found"); + } } diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ExternalTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ExternalTranslationTest.java index 2c9fab4173c9e..8020cfa5ab540 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ExternalTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ExternalTranslationTest.java @@ -27,8 +27,8 @@ import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ForwardingPTransformTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ForwardingPTransformTest.java index 1ecf04fe4124b..6c94d91f630a4 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ForwardingPTransformTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ForwardingPTransformTest.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/GroupIntoBatchesTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/GroupIntoBatchesTranslationTest.java index bb52e35b910e2..6760fa1f714d0 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/GroupIntoBatchesTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/GroupIntoBatchesTranslationTest.java @@ -33,9 +33,9 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PValues; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.joda.time.Duration; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/MorePipelineTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/MorePipelineTest.java index 193b495f5e525..32fb164d85c76 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/MorePipelineTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/MorePipelineTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.not; @@ -43,7 +43,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.PCollectionViews; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.Matchers; import org.junit.Rule; import org.junit.Test; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionTranslationTest.java index b521b682fb381..2c5f53fad1f7a 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PCollectionTranslationTest.java @@ -52,7 +52,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollection.IsBounded; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Test; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java index 2acc8c81db002..c7a2ae4a4a670 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java @@ -70,8 +70,8 @@ import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.TypeDescriptors; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.hamcrest.Matchers; import org.joda.time.Duration; import org.junit.Rule; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformReplacementsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformReplacementsTest.java index 7006a555b5355..d841de4581ba8 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformReplacementsTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformReplacementsTest.java @@ -33,7 +33,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.PValues; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformTranslationTest.java index a0158e3db8274..4c3c3e072ce96 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformTranslationTest.java @@ -53,7 +53,7 @@ import org.apache.beam.sdk.values.PValues; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java index c207b288546b6..d6d335b562919 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ParDoTranslationTest.java @@ -69,7 +69,7 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Test; import org.junit.experimental.runners.Enclosed; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineOptionsTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineOptionsTranslationTest.java index fdf58427f4882..497a7388dfbee 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineOptionsTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineOptionsTranslationTest.java @@ -33,7 +33,7 @@ import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.NullValue; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Struct; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Value; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineTranslationTest.java index 78b36653101bc..c17a2866f3c1f 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineTranslationTest.java @@ -63,7 +63,7 @@ import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslationTest.java index 7f3076a631bd3..7ebe55d6917c8 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReadTranslationTest.java @@ -40,7 +40,7 @@ import org.apache.beam.sdk.io.UnboundedSource; import org.apache.beam.sdk.io.UnboundedSource.CheckpointMark; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReplacementOutputsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReplacementOutputsTest.java index 0666c1468a4bc..8a6942535db3a 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReplacementOutputsTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ReplacementOutputsTest.java @@ -34,8 +34,8 @@ import org.apache.beam.sdk.values.TaggedPValue; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.hamcrest.Matchers; import org.junit.Rule; import org.junit.Test; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TestStreamTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TestStreamTranslationTest.java index 01833a83eeb74..459b48ea95a7e 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TestStreamTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TestStreamTranslationTest.java @@ -33,7 +33,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PValues; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Test; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformUpgraderTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformUpgraderTest.java new file mode 100644 index 0000000000000..6620e780bc166 --- /dev/null +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformUpgraderTest.java @@ -0,0 +1,369 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.core.construction; + +import static org.junit.Assert.assertEquals; + +import com.google.auto.service.AutoService; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.nio.charset.Charset; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.apache.beam.model.expansion.v1.ExpansionApi; +import org.apache.beam.model.pipeline.v1.Endpoints; +import org.apache.beam.model.pipeline.v1.RunnerApi; +import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.io.TextIO; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.runners.AppliedPTransform; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.SimpleFunction; +import org.apache.beam.sdk.transforms.ToString; +import org.apache.beam.sdk.util.ByteStringOutputStream; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for TransformServiceBasedOverride. */ +@RunWith(JUnit4.class) +public class TransformUpgraderTest { + static class TestTransform extends PTransform, PCollection> { + private int testParam; + + public TestTransform(int testParam) { + this.testParam = testParam; + } + + @Override + public PCollection expand(PCollection input) { + return input.apply( + MapElements.via( + new SimpleFunction() { + @Override + public Integer apply(Integer input) { + return input * testParam; + } + })); + } + + public Integer getTestParam() { + return testParam; + } + } + + static class TestTransformPayloadTranslator + implements PTransformTranslation.TransformPayloadTranslator { + + static final String URN = "beam:transform:test:transform_to_update"; + + Schema configRowSchema = Schema.builder().addInt32Field("multiplier").build(); + + @Override + public String getUrn() { + return URN; + } + + @Override + public TestTransform fromConfigRow(Row configRow) { + return new TestTransform(configRow.getInt32("multiplier")); + } + + @Override + public Row toConfigRow(TestTransform transform) { + return Row.withSchema(configRowSchema) + .withFieldValue("multiplier", transform.getTestParam()) + .build(); + } + + @Override + public RunnerApi.@Nullable FunctionSpec translate( + AppliedPTransform application, SdkComponents components) + throws IOException { + + int testParam = application.getTransform().getTestParam(); + + FunctionSpec.Builder specBuilder = FunctionSpec.newBuilder(); + specBuilder.setUrn(getUrn()); + + ByteStringOutputStream byteStringOut = new ByteStringOutputStream(); + ObjectOutputStream objectOutputStream = new ObjectOutputStream(byteStringOut); + objectOutputStream.writeObject(testParam); + objectOutputStream.flush(); + specBuilder.setPayload(byteStringOut.toByteString()); + + return specBuilder.build(); + } + } + + @AutoService(TransformPayloadTranslatorRegistrar.class) + public static class Registrar implements TransformPayloadTranslatorRegistrar { + @Override + public Map, TestTransformPayloadTranslator> + getTransformPayloadTranslators() { + return Collections.singletonMap(TestTransform.class, new TestTransformPayloadTranslator()); + } + } + + static class TestTransform2 extends TestTransform { + public TestTransform2(int testParam) { + super(testParam); + } + } + + static class TestTransformPayloadTranslator2 extends TestTransformPayloadTranslator { + static final String URN = "beam:transform:test:transform_to_update2"; + + @Override + public String getUrn() { + return URN; + } + } + + @AutoService(TransformPayloadTranslatorRegistrar.class) + public static class Registrar2 implements TransformPayloadTranslatorRegistrar { + @Override + public Map, TestTransformPayloadTranslator2> + getTransformPayloadTranslators() { + return Collections.singletonMap(TestTransform2.class, new TestTransformPayloadTranslator2()); + } + } + + static class TestExpansionServiceClientFactory implements ExpansionServiceClientFactory { + ExpansionApi.ExpansionResponse response; + + @Override + public ExpansionServiceClient getExpansionServiceClient( + Endpoints.ApiServiceDescriptor endpoint) { + return new ExpansionServiceClient() { + @Override + public ExpansionApi.ExpansionResponse expand(ExpansionApi.ExpansionRequest request) { + RunnerApi.Components.Builder responseComponents = request.getComponents().toBuilder(); + RunnerApi.PTransform transformToUpgrade = + request.getComponents().getTransformsMap().get("TransformUpgraderTest-TestTransform"); + ByteString alreadyUpgraded = ByteString.empty(); + try { + alreadyUpgraded = transformToUpgrade.getAnnotationsOrThrow("already_upgraded"); + } catch (Exception e) { + // Ignore + } + if (!alreadyUpgraded.isEmpty()) { + transformToUpgrade = + request + .getComponents() + .getTransformsMap() + .get("TransformUpgraderTest-TestTransform2"); + } + if (!transformToUpgrade + .getSpec() + .getUrn() + .equals(request.getTransform().getSpec().getUrn())) { + throw new RuntimeException("Could not find a valid transform to upgrade"); + } + + Integer oldParam; + try { + ByteArrayInputStream byteArrayInputStream = + new ByteArrayInputStream(transformToUpgrade.getSpec().getPayload().toByteArray()); + ObjectInputStream objectInputStream = new ObjectInputStream(byteArrayInputStream); + oldParam = (Integer) objectInputStream.readObject(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + RunnerApi.PTransform.Builder upgradedTransform = transformToUpgrade.toBuilder(); + FunctionSpec.Builder specBuilder = upgradedTransform.getSpecBuilder(); + + ByteStringOutputStream byteStringOutputStream = new ByteStringOutputStream(); + try { + ObjectOutputStream objectOutputStream = new ObjectOutputStream(byteStringOutputStream); + objectOutputStream.writeObject(oldParam * 2); + objectOutputStream.flush(); + specBuilder.setPayload(byteStringOutputStream.toByteString()); + } catch (IOException e) { + throw new RuntimeException(e); + } + + upgradedTransform.setSpec(specBuilder.build()); + upgradedTransform.putAnnotations( + "already_upgraded", + ByteString.copyFrom("dummyvalue".getBytes(Charset.defaultCharset()))); + + response = + ExpansionApi.ExpansionResponse.newBuilder() + .setComponents(responseComponents.build()) + .setTransform(upgradedTransform.build()) + .build(); + return response; + } + + @Override + public ExpansionApi.DiscoverSchemaTransformResponse discover( + ExpansionApi.DiscoverSchemaTransformRequest request) { + return null; + } + + @Override + public void close() throws Exception { + // do nothing + } + }; + } + + @Override + public void close() throws Exception { + // do nothing + } + } + + private void validateTestParam(RunnerApi.PTransform updatedTestTransform, Integer expectedValue) { + Integer updatedParam; + try { + ByteArrayInputStream byteArrayInputStream = + new ByteArrayInputStream(updatedTestTransform.getSpec().getPayload().toByteArray()); + ObjectInputStream objectInputStream = new ObjectInputStream(byteArrayInputStream); + updatedParam = (Integer) objectInputStream.readObject(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertEquals(Integer.valueOf(expectedValue), updatedParam); + } + + @Test + public void testTransformUpgrade() throws Exception { + Pipeline pipeline = Pipeline.create(); + pipeline + .apply(Create.of(1, 2, 3)) + .apply(new TestTransform(2)) + .apply(ToString.elements()) + .apply(TextIO.write().to("dummyfilename")); + + RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(pipeline, false); + ExternalTranslationOptions options = + PipelineOptionsFactory.create().as(ExternalTranslationOptions.class); + List urnsToOverride = ImmutableList.of(TestTransformPayloadTranslator.URN); + options.setTransformsToOverride(urnsToOverride); + options.setTransformServiceAddress("dummyaddress"); + + RunnerApi.Pipeline upgradedPipelineProto = + TransformUpgrader.of(new TestExpansionServiceClientFactory()) + .upgradeTransformsViaTransformService(pipelineProto, urnsToOverride, options); + + RunnerApi.PTransform upgradedTransform = + upgradedPipelineProto + .getComponents() + .getTransformsMap() + .get("TransformUpgraderTest-TestTransform"); + + validateTestParam(upgradedTransform, 4); + } + + @Test + public void testTransformUpgradeMultipleOccurrences() throws Exception { + Pipeline pipeline = Pipeline.create(); + pipeline + .apply(Create.of(1, 2, 3)) + .apply(new TestTransform(2)) + .apply(ToString.elements()) + .apply(TextIO.write().to("dummyfilename")); + pipeline + .apply(Create.of(1, 2, 3)) + .apply(new TestTransform(2)) + .apply(ToString.elements()) + .apply(TextIO.write().to("dummyfilename")); + + RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(pipeline, false); + ExternalTranslationOptions options = + PipelineOptionsFactory.create().as(ExternalTranslationOptions.class); + List urnsToOverride = ImmutableList.of(TestTransformPayloadTranslator.URN); + options.setTransformsToOverride(urnsToOverride); + options.setTransformServiceAddress("dummyaddress"); + + RunnerApi.Pipeline upgradedPipelineProto = + TransformUpgrader.of(new TestExpansionServiceClientFactory()) + .upgradeTransformsViaTransformService(pipelineProto, urnsToOverride, options); + + RunnerApi.PTransform upgradedTransform1 = + upgradedPipelineProto + .getComponents() + .getTransformsMap() + .get("TransformUpgraderTest-TestTransform"); + validateTestParam(upgradedTransform1, 4); + + RunnerApi.PTransform upgradedTransform2 = + upgradedPipelineProto + .getComponents() + .getTransformsMap() + .get("TransformUpgraderTest-TestTransform2"); + validateTestParam(upgradedTransform2, 4); + } + + @Test + public void testTransformUpgradeMultipleURNs() throws Exception { + Pipeline pipeline = Pipeline.create(); + pipeline + .apply(Create.of(1, 2, 3)) + .apply(new TestTransform(2)) + .apply(ToString.elements()) + .apply(TextIO.write().to("dummyfilename")); + pipeline + .apply(Create.of(1, 2, 3)) + .apply(new TestTransform2(2)) + .apply(ToString.elements()) + .apply(TextIO.write().to("dummyfilename")); + + RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(pipeline, false); + ExternalTranslationOptions options = + PipelineOptionsFactory.create().as(ExternalTranslationOptions.class); + List urnsToOverride = + ImmutableList.of(TestTransformPayloadTranslator.URN, TestTransformPayloadTranslator2.URN); + options.setTransformsToOverride(urnsToOverride); + options.setTransformServiceAddress("dummyaddress"); + + RunnerApi.Pipeline upgradedPipelineProto = + TransformUpgrader.of(new TestExpansionServiceClientFactory()) + .upgradeTransformsViaTransformService(pipelineProto, urnsToOverride, options); + + RunnerApi.PTransform upgradedTransform1 = + upgradedPipelineProto + .getComponents() + .getTransformsMap() + .get("TransformUpgraderTest-TestTransform"); + validateTestParam(upgradedTransform1, 4); + + RunnerApi.PTransform upgradedTransform2 = + upgradedPipelineProto + .getComponents() + .getTransformsMap() + .get("TransformUpgraderTest-TestTransform2"); + validateTestParam(upgradedTransform2, 4); + } +} diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TriggerTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TriggerTranslationTest.java index d2f108afd601e..176f67531ce5a 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TriggerTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TriggerTranslationTest.java @@ -32,7 +32,7 @@ import org.apache.beam.sdk.transforms.windowing.Never; import org.apache.beam.sdk.transforms.windowing.Repeatedly; import org.apache.beam.sdk.transforms.windowing.Trigger; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Test; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSourceTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSourceTest.java index 57aadc7bb0c64..31f6842a42bc3 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSourceTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/UnboundedReadFromBoundedSourceTest.java @@ -26,9 +26,15 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.lang.ref.Reference; +import java.lang.ref.ReferenceQueue; +import java.lang.ref.WeakReference; import java.nio.ByteBuffer; import java.nio.channels.ReadableByteChannel; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.NoSuchElementException; import java.util.Random; import org.apache.beam.runners.core.construction.UnboundedReadFromBoundedSource.BoundedToUnboundedSourceAdapter; @@ -58,9 +64,9 @@ import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.joda.time.Instant; import org.junit.Rule; import org.junit.Test; @@ -69,10 +75,14 @@ import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** Unit tests for {@link UnboundedReadFromBoundedSource}. */ @RunWith(JUnit4.class) public class UnboundedReadFromBoundedSourceTest { + private static final Logger LOG = + LoggerFactory.getLogger(UnboundedReadFromBoundedSourceTest.class); @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); @@ -280,6 +290,38 @@ public void testReadFromCheckpointBeforeStart() throws Exception { unboundedSource.createReader(options, checkpoint).getCurrent(); } + @Test + public void testReadersClosedProperly() throws IOException { + ManagedReaderBoundedSource boundedSource = new ManagedReaderBoundedSource(0, 10); + BoundedToUnboundedSourceAdapter unboundedSource = + new BoundedToUnboundedSourceAdapter<>(boundedSource); + PipelineOptions options = PipelineOptionsFactory.create(); + + BoundedToUnboundedSourceAdapter.Reader reader = + unboundedSource.createReader(options, new Checkpoint(null, boundedSource)); + + for (int i = 0; i < 3; ++i) { + if (i == 0) { + assertTrue(reader.start()); + } else { + assertTrue(reader.advance()); + } + assertEquals(i, (int) reader.getCurrent()); + } + Checkpoint checkpoint = reader.getCheckpointMark(); + List> residualElements = checkpoint.getResidualElements(); + for (int i = 0; i < 7; ++i) { + TimestampedValue element = residualElements.get(i); + assertEquals(i + 3, (int) element.getValue()); + } + for (int i = 0; i < 100; ++i) { + // A WeakReference of an object that no other objects reference are not immediately added to + // ReferenceQueue. To test this, we should run System.gc() multiple times. + // If a reader is GCed without closing, `cleanQueue` throws a RuntimeException. + boundedSource.cleanQueue(); + } + } + /** Generate byte array of given size. */ private static byte[] generateInput(int size) { // Arbitrary but fixed seed @@ -298,6 +340,7 @@ private static void writeFile(File file, byte[] input) throws IOException { /** Unsplittable source for use in tests. */ private static class UnsplittableSource extends FileBasedSource { + public UnsplittableSource(String fileOrPatternSpec, long minBundleSize) { super(StaticValueProvider.of(fileOrPatternSpec), minBundleSize); } @@ -323,6 +366,7 @@ public Coder getOutputCoder() { } private static class UnsplittableReader extends FileBasedReader { + ByteBuffer buff = ByteBuffer.allocate(1); Byte current; long offset; @@ -370,4 +414,140 @@ protected long getCurrentOffset() { } } } + + /** + * An integer generating bounded source. This source class checks if readers are closed properly. + * For that, it manages weak references of readers, and checks at `createReader` and `cleanQueue` + * if readers were closed before GCed. The `cleanQueue` does not change the state in + * `ManagedReaderBoundedSource`, but throws an exception if it finds a reader GCed without + * closing. + */ + private static class ManagedReaderBoundedSource extends BoundedSource { + + private final int from; + private final int to; // exclusive + + private transient ReferenceQueue refQueue; + private transient Map, CloseStatus> cloesStatusMap; + + public ManagedReaderBoundedSource(int from, int to) { + if (from > to) { + throw new RuntimeException( + String.format("`from` <= `to`, but got from: %d, to: %d", from, to)); + } + this.from = from; + this.to = to; + } + + @Override + public List> split( + long desiredBundleSizeBytes, PipelineOptions options) { + return Collections.singletonList(this); + } + + @Override + public long getEstimatedSizeBytes(PipelineOptions options) { + return (to - from) * 4L; + } + + @Override + public BoundedReader createReader(PipelineOptions options) { + // Add weak reference to queue to monitor GCed readers. If `CloseStatus` associated with + // reader is not closed, it means a reader was GCed without closing properly. The CloseStatus + // check for GCed readers are done at cleanQueue(). + if (refQueue == null) { + refQueue = new ReferenceQueue<>(); + cloesStatusMap = new HashMap<>(); + } + cleanQueue(); + + CloseStatus status = new CloseStatus(); + ManagedReader reader = new ManagedReader(status); + WeakReference reference = new WeakReference<>(reader, refQueue); + cloesStatusMap.put(reference, status); + LOG.info("Add reference {} for reader {}", reference, reader); + return reader; + } + + public void cleanQueue() { + System.gc(); + + Reference reference; + while ((reference = refQueue.poll()) != null) { + CloseStatus closeStatus = cloesStatusMap.get(reference); + LOG.info("Poll reference: {}, closed: {}", reference, closeStatus.closed); + closeStatus.throwIfNotClosed(); + } + } + + class CloseStatus { + + private final RuntimeException allocationStacktrace; + + private boolean closed; + + public CloseStatus() { + allocationStacktrace = + new RuntimeException("Previous reader was not closed properly. Reader allocation was"); + closed = false; + } + + void close() { + cleanQueue(); + closed = true; + } + + void throwIfNotClosed() { + if (!closed) { + throw allocationStacktrace; + } + } + } + + class ManagedReader extends BoundedReader { + + private final CloseStatus status; + + int current; + + public ManagedReader(CloseStatus status) { + this.status = status; + } + + @Override + public boolean start() { + if (from < to) { + current = from; + return true; + } else { + return false; + } + } + + @Override + public boolean advance() { + if (current + 1 < to) { + ++current; + return true; + } else { + return false; + } + } + + @Override + public Integer getCurrent() { + return current; + } + + @Override + public void close() { + status.close(); + } + + @Override + public BoundedSource getCurrentSource() { + return ManagedReaderBoundedSource.this; + } + } + } } diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ValidateRunnerXlangTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ValidateRunnerXlangTest.java index 1ecd9041a75fb..efefa77f508a0 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ValidateRunnerXlangTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/ValidateRunnerXlangTest.java @@ -41,7 +41,7 @@ import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslationTest.java index 6eb7822cba759..113afd4b0f40d 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowIntoTranslationTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.junit.Assert.assertEquals; import java.util.concurrent.atomic.AtomicReference; @@ -39,7 +39,7 @@ import org.apache.beam.sdk.transforms.windowing.Window.Assign; import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslationTest.java index f73694fc6d6a3..6981e74db7820 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WindowingStrategyTranslationTest.java @@ -48,7 +48,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WriteFilesTranslationTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WriteFilesTranslationTest.java index c8861fdb28f26..cc6daee00591e 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WriteFilesTranslationTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/WriteFilesTranslationTest.java @@ -40,7 +40,7 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PValues; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ExecutableStageMatcher.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ExecutableStageMatcher.java index a76b5dc51e868..83bde34a0a9f3 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ExecutableStageMatcher.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ExecutableStageMatcher.java @@ -26,7 +26,7 @@ import org.apache.beam.model.pipeline.v1.RunnerApi.PTransform; import org.apache.beam.runners.core.construction.graph.PipelineNode.PCollectionNode; import org.apache.beam.runners.core.construction.graph.PipelineNode.PTransformNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.Description; import org.hamcrest.Matcher; import org.hamcrest.Matchers; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ExecutableStageTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ExecutableStageTest.java index f0732b5ad3654..b46769cd7267f 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ExecutableStageTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ExecutableStageTest.java @@ -41,7 +41,7 @@ import org.apache.beam.runners.core.construction.Environments; import org.apache.beam.runners.core.construction.PTransformTranslation; import org.apache.beam.runners.core.construction.graph.PipelineNode.PTransformNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/FusedPipelineTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/FusedPipelineTest.java index fff804b41d887..db6752dcc5037 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/FusedPipelineTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/FusedPipelineTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction.graph; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.equalTo; @@ -41,7 +41,7 @@ import org.apache.beam.sdk.transforms.Values; import org.apache.beam.sdk.transforms.WithKeys; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/GreedyPipelineFuserTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/GreedyPipelineFuserTest.java index aef3dddf22ce0..dffd15077d429 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/GreedyPipelineFuserTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/GreedyPipelineFuserTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction.graph; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.getOnlyElement; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.getOnlyElement; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.contains; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/GreedyStageFuserTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/GreedyStageFuserTest.java index b0d1e82cd0f7b..af4ea7e79f1cc 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/GreedyStageFuserTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/GreedyStageFuserTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction.graph; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.getOnlyElement; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.getOnlyElement; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.emptyIterable; @@ -42,7 +42,7 @@ import org.apache.beam.runners.core.construction.PTransformTranslation; import org.apache.beam.runners.core.construction.graph.PipelineNode.PCollectionNode; import org.apache.beam.runners.core.construction.graph.PipelineNode.PTransformNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.hamcrest.Description; import org.hamcrest.TypeSafeMatcher; import org.junit.Before; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/NetworksTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/NetworksTest.java index 9a13909a0f258..af0368c52b099 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/NetworksTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/NetworksTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction.graph; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.emptyIterable; @@ -34,15 +34,15 @@ import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.ElementOrder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.EndpointPair; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.MutableNetwork; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.NetworkBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.ElementOrder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.EndpointPair; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.MutableNetwork; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.NetworkBuilder; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/OutputDeduplicatorTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/OutputDeduplicatorTest.java index 2a40b56cf5fdd..27a222f3a82b4 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/OutputDeduplicatorTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/OutputDeduplicatorTest.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.core.construction.graph; import static org.apache.beam.runners.core.construction.graph.ExecutableStage.DEFAULT_WIRE_CODER_SETTINGS; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.getOnlyElement; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.getOnlyElement; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.empty; @@ -42,7 +42,7 @@ import org.apache.beam.runners.core.construction.graph.OutputDeduplicator.DeduplicationResult; import org.apache.beam.runners.core.construction.graph.PipelineNode.PCollectionNode; import org.apache.beam.runners.core.construction.graph.PipelineNode.PTransformNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ProjectionProducerVisitorTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ProjectionProducerVisitorTest.java index 493beaec0a576..89c1cf8904ae4 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ProjectionProducerVisitorTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ProjectionProducerVisitorTest.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ProjectionPushdownOptimizerTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ProjectionPushdownOptimizerTest.java index 81843d6825649..c9b6ba89fd1c6 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ProjectionPushdownOptimizerTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ProjectionPushdownOptimizerTest.java @@ -37,7 +37,7 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ProtoOverridesTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ProtoOverridesTest.java index d6531e4c55794..17b2a5039c41f 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ProtoOverridesTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/ProtoOverridesTest.java @@ -37,7 +37,7 @@ import org.apache.beam.model.pipeline.v1.RunnerApi.WindowingStrategy; import org.apache.beam.runners.core.construction.graph.ProtoOverrides.TransformReplacement; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/QueryablePipelineTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/QueryablePipelineTest.java index c3864d6a162d6..90e6af182e63f 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/QueryablePipelineTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/QueryablePipelineTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.construction.graph; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.getOnlyElement; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.getOnlyElement; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.emptyIterable; @@ -64,7 +64,7 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.joda.time.Duration; import org.junit.Rule; import org.junit.Test; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/SplittableParDoExpanderTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/SplittableParDoExpanderTest.java index bee413874f1c6..548f5feab271f 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/SplittableParDoExpanderTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/graph/SplittableParDoExpanderTest.java @@ -33,8 +33,8 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/renderer/PipelineDotRendererTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/renderer/PipelineDotRendererTest.java index eca57b9e8b195..8d7e02d70c8d7 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/renderer/PipelineDotRendererTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/renderer/PipelineDotRendererTest.java @@ -52,7 +52,10 @@ public void testEmptyPipeline() { @Test public void testCompositePipeline() { - p.apply(Create.timestamped(TimestampedValue.of(KV.of(1, 1), new Instant(1)))) + p.apply( + Create.timestamped( + TimestampedValue.of(KV.of(1, 1), new Instant(1)), + TimestampedValue.of(KV.of(2, 2), new Instant(2)))) .apply(Window.into(FixedWindows.of(Duration.millis(10)))) .apply(Sum.integersPerKey()); assertEquals( diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/resources/ClasspathScanningResourcesDetectorTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/resources/ClasspathScanningResourcesDetectorTest.java index afd0472946b7f..07c009ef14512 100644 --- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/resources/ClasspathScanningResourcesDetectorTest.java +++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/resources/ClasspathScanningResourcesDetectorTest.java @@ -120,7 +120,7 @@ public void shouldDetectClassPathResourceFromJavaClassPathEnvVariable() throws I @Test public void shouldNotDetectClassPathResourceThatIsNotAFile() throws Exception { - String url = "http://www.google.com/all-the-secrets.jar"; + String url = "http://www.example.com/all-the-secrets.jar"; ClassLoader classLoader = new URLClassLoader(new URL[] {new URL(url)}); ClasspathScanningResourcesDetector detector = new ClasspathScanningResourcesDetector(new ClassGraph()); diff --git a/runners/core-java/OWNERS b/runners/core-java/OWNERS deleted file mode 100644 index d1c065101f155..0000000000000 --- a/runners/core-java/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - kennknowles - - lukecwik diff --git a/runners/core-java/build.gradle b/runners/core-java/build.gradle index 4381fd0fa7ffb..f8663e215fb6c 100644 --- a/runners/core-java/build.gradle +++ b/runners/core-java/build.gradle @@ -44,7 +44,7 @@ dependencies { implementation project(path: ":model:job-management", configuration: "shadow") implementation project(":runners:core-construction-java") implementation project(":sdks:java:fn-execution") - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.joda_time implementation library.java.vendored_grpc_1_54_0 implementation library.java.slf4j_api diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ActiveWindowSet.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ActiveWindowSet.java index 03992e357e016..597d04a4c862e 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ActiveWindowSet.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ActiveWindowSet.java @@ -21,7 +21,7 @@ import java.util.Set; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.WindowFn; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** * Track which windows are active, and the state address window(s) under which their diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/GlobalCombineFnRunners.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/GlobalCombineFnRunners.java index ea0ea3e47e98b..a24da21ebb7a9 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/GlobalCombineFnRunners.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/GlobalCombineFnRunners.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** * Static utility methods that provide {@link GlobalCombineFnRunner} implementations for different diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupByKeyViaGroupByKeyOnly.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupByKeyViaGroupByKeyOnly.java index f4d861107b2a2..803128e24d73c 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupByKeyViaGroupByKeyOnly.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/GroupByKeyViaGroupByKeyOnly.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.ArrayList; import java.util.Comparator; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryBundleFinalizer.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryBundleFinalizer.java index 60282fbc49584..bde7d625ffcca 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryBundleFinalizer.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryBundleFinalizer.java @@ -21,7 +21,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.beam.sdk.transforms.DoFn.BundleFinalizer; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; /** An in-memory {@link BundleFinalizer} that keeps track of any pending finalization requests. */ diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryMultimapSideInputView.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryMultimapSideInputView.java index 9ce1157c16969..881af66cd6324 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryMultimapSideInputView.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryMultimapSideInputView.java @@ -26,8 +26,8 @@ import org.apache.beam.sdk.transforms.Materializations; import org.apache.beam.sdk.transforms.Materializations.MultimapView; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; /** An in-memory representation of {@link MultimapView}. */ diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryStateInternals.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryStateInternals.java index 8d7820d4be8e1..2f15f88b954e2 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryStateInternals.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryStateInternals.java @@ -54,13 +54,13 @@ import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.util.CombineFnUtil; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.UnknownKeyFor; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryTimerInternals.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryTimerInternals.java index 038d3152fd1de..7e84ccbbccfeb 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryTimerInternals.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/InMemoryTimerInternals.java @@ -17,9 +17,9 @@ */ package org.apache.beam.runners.core; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.NavigableSet; import java.util.NoSuchElementException; @@ -27,10 +27,10 @@ import org.apache.beam.sdk.state.TimeDomain; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowTracing; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashBasedTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Table; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBasedTable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Table; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java index fc395de8228f8..756a89c88aca0 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItemCoder.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** A {@link Coder} for {@link KeyedWorkItem KeyedWorkItems}. */ public class KeyedWorkItemCoder extends StructuredCoder> { diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItems.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItems.java index e04743b52ea31..4c6df0c21051c 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItems.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/KeyedWorkItems.java @@ -21,8 +21,8 @@ import java.util.Objects; import org.apache.beam.runners.core.TimerInternals.TimerData; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; /** Static utility methods that provide {@link KeyedWorkItem} implementations. */ diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java index 15653817612c0..ca1a0393b95d9 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.Instant; /** diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java index eb017cd79aece..d6f237bf571ce 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java @@ -23,8 +23,8 @@ import org.apache.beam.sdk.util.WindowTracing; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/MergingActiveWindowSet.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/MergingActiveWindowSet.java index 24e21ac74a336..96ab8ea3450a5 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/MergingActiveWindowSet.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/MergingActiveWindowSet.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.core; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.ArrayList; import java.util.Collection; @@ -34,9 +34,9 @@ import org.apache.beam.sdk.state.ValueState; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.WindowFn; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; /** An {@link ActiveWindowSet} for merging {@link WindowFn} implementations. */ diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/NonMergingActiveWindowSet.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/NonMergingActiveWindowSet.java index 03efe29017f3d..5fdf79fdddfd0 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/NonMergingActiveWindowSet.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/NonMergingActiveWindowSet.java @@ -21,8 +21,8 @@ import java.util.Set; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.WindowFn; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; /** * Implementation of {@link ActiveWindowSet} used with {@link WindowFn WindowFns} that don't support diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/NullSideInputReader.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/NullSideInputReader.java index 186a207ebf05d..a13deb4b49413 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/NullSideInputReader.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/NullSideInputReader.java @@ -21,7 +21,7 @@ import java.util.Set; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; /** * A {@link SideInputReader} representing a well-defined set of views, but not storing any values diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java index 38eff394f05d5..85a46eb7dc042 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.core; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Map; import java.util.concurrent.CancellationException; @@ -48,8 +48,8 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Futures; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Futures; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/PaneInfoTracker.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/PaneInfoTracker.java index 34d3451d29443..12ceca976417e 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/PaneInfoTracker.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/PaneInfoTracker.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.beam.sdk.state.ReadableState; @@ -27,7 +27,7 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo.PaneInfoCoder; import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing; import org.apache.beam.sdk.util.WindowTracing; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.Instant; /** diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/PeekingReiterator.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/PeekingReiterator.java index 668887fda6fc5..3a316a88546fd 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/PeekingReiterator.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/PeekingReiterator.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.core; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.NoSuchElementException; import org.apache.beam.sdk.util.common.Reiterator; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java index 3678012226180..28bdf45abd929 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ProcessFnRunner.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Collection; import java.util.Collections; @@ -30,7 +30,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Instant; /** diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java index 5b4b760c6d26f..2479451c2b467 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnContextFactory.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.core; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.Collection; import java.util.Map; @@ -35,7 +35,7 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java index 8e5c1a5b8ccaa..b9eef50ed59f4 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ReduceFnRunner.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.core; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.ArrayList; import java.util.Collection; @@ -52,9 +52,9 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java index 97aa0ecc985b3..8b2cb5c3db2dc 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Collection; import java.util.Collections; @@ -39,7 +39,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java index 48bc22c448437..16986fdf8d52d 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.core; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.Collection; import java.util.HashMap; @@ -58,10 +58,10 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunner.java index a6a6455d2d2d3..fb04b536b4c06 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunner.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunner.java @@ -26,8 +26,8 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDoViaKeyedWorkItems.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDoViaKeyedWorkItems.java index 2fa8337e1e4dc..9a666d8043e57 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDoViaKeyedWorkItems.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDoViaKeyedWorkItems.java @@ -54,8 +54,8 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableProcessElementInvoker.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableProcessElementInvoker.java index 0b22c349da510..00a72b9197bb2 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableProcessElementInvoker.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableProcessElementInvoker.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.Map; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateNamespaces.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateNamespaces.java index 69bfdb3a1585b..6c0ed7740489a 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateNamespaces.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateNamespaces.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.coders.CoderException; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.CoderUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; import org.checkerframework.checker.nullness.qual.Nullable; /** Factory methods for creating the {@link StateNamespace StateNamespaces}. */ diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTable.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTable.java index d285c8e24717f..3aa359681eebd 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTable.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTable.java @@ -23,9 +23,9 @@ import org.apache.beam.runners.core.StateTag.StateBinder; import org.apache.beam.sdk.state.State; import org.apache.beam.sdk.state.StateContext; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Equivalence; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashBasedTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Table; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Equivalence; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBasedTable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Table; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTags.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTags.java index 793b10048bcaa..7ffb10c85c0aa 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTags.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StateTags.java @@ -37,8 +37,8 @@ import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.TimestampCombiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Equivalence; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Equivalence; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; /** Static utility methods for creating {@link StateTag} instances. */ diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java index d4faaf1d3bd13..0b4e980b7f5f9 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java @@ -40,7 +40,7 @@ import org.apache.beam.sdk.util.WindowTracing; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SystemReduceFn.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SystemReduceFn.java index 53c0c7146ef6a..b25d72c93948d 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SystemReduceFn.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SystemReduceFn.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.transforms.GroupByKey; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.AppliedCombineFn; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** * {@link ReduceFn} implementing the default reduction behaviors of {@link GroupByKey}. diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java index 143254a5b37b2..6c92d234b86fc 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/TimerInternals.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.state.TimeDomain; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ComparisonChain; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ComparisonChain; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java index 60a30039914a8..e5a5f90587c15 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/WatermarkHold.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.Serializable; @@ -31,7 +31,7 @@ import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.util.WindowTracing; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DefaultMetricResults.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DefaultMetricResults.java index 56283eb1f7bd3..a77f3947b529d 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DefaultMetricResults.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/DefaultMetricResults.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.metrics.MetricResult; import org.apache.beam.sdk.metrics.MetricResults; import org.apache.beam.sdk.metrics.MetricsFilter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ExecutionStateSampler.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ExecutionStateSampler.java index 6689d3ca31db5..a2b745c74edea 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ExecutionStateSampler.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ExecutionStateSampler.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.metrics; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.Closeable; import java.util.Set; @@ -29,8 +29,8 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTimeUtils.MillisProvider; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ExecutionStateTracker.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ExecutionStateTracker.java index 78640177fc9eb..0eae4a93245cd 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ExecutionStateTracker.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ExecutionStateTracker.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.metrics; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.Closeable; @@ -26,8 +26,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; /** Tracks the current state of a single execution thread. */ diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricUpdates.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricUpdates.java index e09471266c727..7ef936c8552d7 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricUpdates.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricUpdates.java @@ -21,7 +21,7 @@ import java.io.Serializable; import java.util.Collections; import org.apache.beam.sdk.metrics.MetricKey; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** Representation of multiple metric updates. */ @AutoValue diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java index 7a9507dc6b103..af18044ce42ce 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java @@ -26,7 +26,7 @@ import static org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.encodeInt64Counter; import static org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.encodeInt64Distribution; import static org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.encodeInt64Gauge; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.Serializable; import java.util.ArrayList; @@ -48,9 +48,9 @@ import org.apache.beam.sdk.util.HistogramData; import org.apache.beam.sdk.values.KV; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerStepMap.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerStepMap.java index e89edfd73078b..7f49ebda2e209 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerStepMap.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerStepMap.java @@ -37,7 +37,7 @@ import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.util.JsonFormat; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsMap.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsMap.java index 2cb3d7f45b8eb..f8751ece2dcf1 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsMap.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsMap.java @@ -24,8 +24,8 @@ import java.util.concurrent.ConcurrentMap; import java.util.function.BiConsumer; import java.util.function.Consumer; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsPusher.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsPusher.java index a0acbc4fcf3fc..f0aa1a116e989 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsPusher.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsPusher.java @@ -30,8 +30,8 @@ import org.apache.beam.sdk.metrics.MetricsOptions; import org.apache.beam.sdk.metrics.MetricsSink; import org.apache.beam.sdk.util.InstanceBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.checkerframework.checker.nullness.qual.Nullable; /** Component that regularly merges metrics and pushes them to a metrics sink. */ diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MonitoringInfoConstants.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MonitoringInfoConstants.java index 968c58ab33142..d6b2f94738cff 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MonitoringInfoConstants.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MonitoringInfoConstants.java @@ -20,13 +20,13 @@ import static org.apache.beam.model.pipeline.v1.MetricsApi.labelProps; import static org.apache.beam.model.pipeline.v1.MetricsApi.monitoringInfoSpec; import static org.apache.beam.runners.core.construction.BeamUrns.getUrn; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo; import org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo.MonitoringInfoLabels; import org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfoSpecs; import org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfoTypeUrns; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** This static class fetches MonitoringInfo related values from metrics.proto. */ public final class MonitoringInfoConstants { diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MonitoringInfoMetricName.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MonitoringInfoMetricName.java index c6dda30354d55..adf3ad84f9e82 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MonitoringInfoMetricName.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MonitoringInfoMetricName.java @@ -17,14 +17,14 @@ */ package org.apache.beam.runners.core.metrics; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Map; import java.util.Objects; import org.apache.beam.model.pipeline.v1.MetricsApi; import org.apache.beam.sdk.metrics.MetricName; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/OWNERS b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/OWNERS deleted file mode 100644 index 883da4e3f34e5..0000000000000 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - echauchot diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ServiceCallMetric.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ServiceCallMetric.java index 17f55ba35f01e..b3b6aebe4621d 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ServiceCallMetric.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ServiceCallMetric.java @@ -20,7 +20,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.beam.sdk.metrics.Counter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /* * Metric class which records Service API call metrics. diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ShortIdMap.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ShortIdMap.java index df856560a8d64..d035be86008f2 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ShortIdMap.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ShortIdMap.java @@ -24,9 +24,9 @@ import java.util.NoSuchElementException; import org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashBiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBiMap; /** A Class for registering short ids for MonitoringInfos. */ public class ShortIdMap { diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/SimpleExecutionState.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/SimpleExecutionState.java index ee4773d130e0f..4729e8163bc51 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/SimpleExecutionState.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/SimpleExecutionState.java @@ -26,7 +26,7 @@ import org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo; import org.apache.beam.runners.core.metrics.ExecutionStateTracker.ExecutionState; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.Duration; import org.joda.time.format.PeriodFormatter; import org.joda.time.format.PeriodFormatterBuilder; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/SimpleMonitoringInfoBuilder.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/SimpleMonitoringInfoBuilder.java index 329139ba9cf71..c44a2621ee6c7 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/SimpleMonitoringInfoBuilder.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/SimpleMonitoringInfoBuilder.java @@ -23,7 +23,7 @@ import static org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.encodeInt64Counter; import static org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.encodeInt64Distribution; import static org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.encodeInt64Gauge; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.HashMap; import java.util.Map; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/SpecMonitoringInfoValidator.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/SpecMonitoringInfoValidator.java index 620a5720f00b7..d78600f003594 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/SpecMonitoringInfoValidator.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/SpecMonitoringInfoValidator.java @@ -27,7 +27,7 @@ import org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo; import org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfoSpec; import org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfoSpecs; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; /** Class implements validation of MonitoringInfos against MonitoringInfoSpecs. */ public class SpecMonitoringInfoValidator { diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterAllStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterAllStateMachine.java index 2d5f71114b291..e50f95a6b4a43 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterAllStateMachine.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterAllStateMachine.java @@ -17,12 +17,12 @@ */ package org.apache.beam.runners.core.triggers; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Arrays; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * A {@link TriggerStateMachine} that fires and finishes once after all of its sub-triggers have diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterDelayFromFirstElementStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterDelayFromFirstElementStateMachine.java index e577ac89fe716..e8ca0990559b0 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterDelayFromFirstElementStateMachine.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterDelayFromFirstElementStateMachine.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.transforms.Combine.Holder; import org.apache.beam.sdk.transforms.Min; import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterEachStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterEachStateMachine.java index 0fac4c89e97c4..e16e859ba66ff 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterEachStateMachine.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterEachStateMachine.java @@ -17,12 +17,12 @@ */ package org.apache.beam.runners.core.triggers; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Arrays; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * A composite {@link TriggerStateMachine} that executes its sub-triggers in order. Only one diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterFirstStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterFirstStateMachine.java index 207a2fa31e47a..8d2056540d8e7 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterFirstStateMachine.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterFirstStateMachine.java @@ -17,12 +17,12 @@ */ package org.apache.beam.runners.core.triggers; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Arrays; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * Create a composite {@link TriggerStateMachine} that fires once after at least one of its diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterSynchronizedProcessingTimeStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterSynchronizedProcessingTimeStateMachine.java index f7c64242c7a83..c7e69f3342554 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterSynchronizedProcessingTimeStateMachine.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterSynchronizedProcessingTimeStateMachine.java @@ -21,7 +21,7 @@ import java.util.List; import org.apache.beam.sdk.state.TimeDomain; import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterWatermarkStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterWatermarkStateMachine.java index 6bb24a9598530..0073727c08bcf 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterWatermarkStateMachine.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/AfterWatermarkStateMachine.java @@ -17,11 +17,11 @@ */ package org.apache.beam.runners.core.triggers; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.Objects; import org.apache.beam.sdk.state.TimeDomain; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/ExecutableTriggerStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/ExecutableTriggerStateMachine.java index 7cb59608565e3..006c34fe153c8 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/ExecutableTriggerStateMachine.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/ExecutableTriggerStateMachine.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.core.triggers; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.Serializable; import java.util.ArrayList; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/FinishedTriggersSet.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/FinishedTriggersSet.java index 99c6ac57824a3..056a816ea0a02 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/FinishedTriggersSet.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/FinishedTriggersSet.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.core.triggers; import java.util.Set; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; /** An implementation of {@link FinishedTriggers} atop a user-provided mutable {@link Set}. */ public class FinishedTriggersSet implements FinishedTriggers { diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/OrFinallyStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/OrFinallyStateMachine.java index fe21ef8c5bf14..e7f5f8754174c 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/OrFinallyStateMachine.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/OrFinallyStateMachine.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.core.triggers; import java.util.Arrays; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** * Executes the {@code actual} trigger until it finishes or until the {@code until} trigger fires. diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachine.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachine.java index 3721ff34d26e4..8216c674ba806 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachine.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachine.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.transforms.windowing.WindowFn; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineContextFactory.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineContextFactory.java index 4468e77b520b5..9e491c17b73ee 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineContextFactory.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineContextFactory.java @@ -34,9 +34,9 @@ import org.apache.beam.sdk.state.Timers; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.WindowFn; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineRunner.java index 5a05895379bad..cf29646ebaa3c 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineRunner.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/triggers/TriggerStateMachineRunner.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.triggers; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.BitSet; import java.util.Collection; @@ -30,8 +30,8 @@ import org.apache.beam.sdk.state.Timers; import org.apache.beam.sdk.state.ValueState; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/InMemoryMultimapSideInputViewTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/InMemoryMultimapSideInputViewTest.java index 313b90b6d6430..869febf5601aa 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/InMemoryMultimapSideInputViewTest.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/InMemoryMultimapSideInputViewTest.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.transforms.Materializations.MultimapView; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.Description; import org.hamcrest.Matchers; import org.hamcrest.TypeSafeMatcher; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/KeyedWorkItemCoderTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/KeyedWorkItemCoderTest.java index 5de57491c0a52..65c9d099d62b7 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/KeyedWorkItemCoderTest.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/KeyedWorkItemCoderTest.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunnerTest.java index d579611ec3489..89cfb4df2486a 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunnerTest.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunnerTest.java @@ -31,8 +31,8 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Before; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/MergingActiveWindowSetTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/MergingActiveWindowSetTest.java index 88ca862647d7e..c822f3c77d42d 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/MergingActiveWindowSetTest.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/MergingActiveWindowSetTest.java @@ -29,8 +29,8 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.transforms.windowing.IntervalWindow; import org.apache.beam.sdk.transforms.windowing.Sessions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.After; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvokerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvokerTest.java index f44c3d70e43ae..14a52128f1a9c 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvokerTest.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvokerTest.java @@ -43,7 +43,7 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java index 3e82852b337a5..e57186c2b4aeb 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnRunnerTest.java @@ -83,7 +83,7 @@ import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Before; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java index d54fa4aa2a2bd..56e3736086971 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java @@ -64,11 +64,11 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Equivalence; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Equivalence; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SideInputHandlerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SideInputHandlerTest.java index c8ebfb69f571f..b76d8fe6b8234 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SideInputHandlerTest.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SideInputHandlerTest.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Test; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java index 791f2d17dd114..a50eb7d9647ee 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java @@ -50,8 +50,8 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ListMultimap; import org.joda.time.Duration; import org.joda.time.Instant; import org.joda.time.format.PeriodFormat; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunnerTest.java index e2518e75c568f..7887faccbb067 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunnerTest.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimplePushbackSideInputDoFnRunnerTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -61,8 +61,8 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Before; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoProcessFnTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoProcessFnTest.java index e64ef3edb8b3f..c6fe4e9bc6711 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoProcessFnTest.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoProcessFnTest.java @@ -19,7 +19,7 @@ import static org.apache.beam.sdk.transforms.DoFn.ProcessContinuation.resume; import static org.apache.beam.sdk.transforms.DoFn.ProcessContinuation.stop; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.greaterThanOrEqualTo; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/StateInternalsTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/StateInternalsTest.java index 4284519104add..a4cd504eee716 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/StateInternalsTest.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/StateInternalsTest.java @@ -50,7 +50,7 @@ import org.apache.beam.sdk.transforms.CombineWithContext; import org.apache.beam.sdk.transforms.Sum; import org.apache.beam.sdk.transforms.windowing.TimestampCombiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matchers; import org.joda.time.Instant; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java index 7afb1f3f878e2..b65e6af84d965 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/StatefulDoFnRunnerTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.when; @@ -48,7 +48,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/WindowMatchers.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/WindowMatchers.java index d934f72d9f6be..ee33eb28f69da 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/WindowMatchers.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/WindowMatchers.java @@ -23,7 +23,7 @@ import org.apache.beam.sdk.transforms.windowing.IntervalWindow; import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.hamcrest.Description; import org.hamcrest.Matcher; import org.hamcrest.Matchers; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/WindowMatchersTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/WindowMatchersTest.java index ffd79ce606b69..a5f36be1134bc 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/WindowMatchersTest.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/WindowMatchersTest.java @@ -22,7 +22,7 @@ import org.apache.beam.sdk.transforms.windowing.IntervalWindow; import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MonitoringInfoConstantsTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MonitoringInfoConstantsTest.java index b748eb8e00a3e..d42751e716e9e 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MonitoringInfoConstantsTest.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/MonitoringInfoConstantsTest.java @@ -21,9 +21,9 @@ import static org.hamcrest.MatcherAssert.assertThat; import org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfoSpecs; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; import org.hamcrest.Matchers; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/SimpleMonitoringInfoBuilderTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/SimpleMonitoringInfoBuilderTest.java index f747808d723f1..d6f726e8bec97 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/SimpleMonitoringInfoBuilderTest.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/metrics/SimpleMonitoringInfoBuilderTest.java @@ -23,7 +23,7 @@ import static org.junit.Assert.assertTrue; import org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/StubTriggerStateMachine.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/StubTriggerStateMachine.java index 4e25ff9e9da15..5c3330b585b1d 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/StubTriggerStateMachine.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/StubTriggerStateMachine.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.triggers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; /** No-op {@link TriggerStateMachine} implementation for testing. */ abstract class StubTriggerStateMachine extends TriggerStateMachine { diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java index ab6e6538f8c84..096f3ac608b62 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.core.triggers; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import static org.junit.Assert.assertTrue; import java.util.ArrayList; @@ -50,8 +50,8 @@ import org.apache.beam.sdk.util.WindowTracing; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachinesTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachinesTest.java index 41772c2618ef2..ef7de559fceb9 100644 --- a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachinesTest.java +++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachinesTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.core.triggers; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; diff --git a/runners/direct-java/OWNERS b/runners/direct-java/OWNERS deleted file mode 100644 index af32c311923a2..0000000000000 --- a/runners/direct-java/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lukecwik diff --git a/runners/direct-java/build.gradle b/runners/direct-java/build.gradle index c8088cb557e5a..8bb8933051b75 100644 --- a/runners/direct-java/build.gradle +++ b/runners/direct-java/build.gradle @@ -67,7 +67,7 @@ configurations { } dependencies { - shadow library.java.vendored_guava_26_0_jre + shadow library.java.vendored_guava_32_1_2_jre shadow project(path: ":model:pipeline", configuration: "shadow") dependOnProjects.each { implementation project(it) diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java index 8f8d73d237bd2..97eedb06bc3fa 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java @@ -38,12 +38,12 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.SettableFuture; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.SettableFuture; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CloningBundleFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CloningBundleFactory.java index c5f2a5d087754..7295b9227d838 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CloningBundleFactory.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CloningBundleFactory.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.util.UserCodeException; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.joda.time.Instant; /** diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternals.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternals.java index 6a465c47490a7..720ce77c388d2 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternals.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternals.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.direct; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Collection; import java.util.HashSet; @@ -55,7 +55,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.TimestampCombiner; import org.apache.beam.sdk.util.CombineFnUtil; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java index acaed7773dd88..a5066cbf52fbe 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.PInput; import org.apache.beam.sdk.values.PValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ListMultimap; /** * Methods for interacting with the underlying structure of a {@link Pipeline} that is being diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java index 37edb835dd020..eb12ff0e3daa5 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.direct; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Collection; import java.util.HashMap; @@ -36,9 +36,9 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.PInput; import org.apache.beam.sdk.values.PValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java index ae6f34acfde9b..a8f881433a3c2 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.direct; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import org.apache.beam.runners.core.KeyedWorkItem; import org.apache.beam.runners.core.KeyedWorkItemCoder; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java index ef4a3aa570d3e..b12b21a95a90b 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** A {@link PTransformOverrideFactory} for {@link GroupByKey} PTransforms. */ final class DirectGroupByKeyOverrideFactory diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java index b00212e21e764..5b286dc0b2e0f 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectMetrics.java @@ -41,7 +41,7 @@ import org.apache.beam.sdk.metrics.MetricResult; import org.apache.beam.sdk.metrics.MetricResults; import org.apache.beam.sdk.metrics.MetricsFilter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; /** Implementation of {@link MetricResults} for the Direct Runner. */ diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java index f208c02df20bb..2f72e98020d87 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java @@ -22,7 +22,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; import org.apache.beam.sdk.runners.PipelineRunnerRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * Contains the {@link PipelineRunnerRegistrar} and {@link PipelineOptionsRegistrar} for the {@link diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java index 34ce76ed7c9fc..8c0faedd857ab 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java @@ -45,13 +45,13 @@ import org.apache.beam.sdk.util.UserCodeException; import org.apache.beam.sdk.util.common.ReflectHelpers; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.joda.time.Duration; /** @@ -322,6 +322,9 @@ private DirectPipelineResult(PipelineExecutor executor, EvaluationContext evalua @Override public State getState() { + if (this.state == State.RUNNING) { + this.state = executor.getPipelineState(); + } return state; } diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTimerInternals.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTimerInternals.java index 1cca631e07a6a..db1112f738851 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTimerInternals.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTimerInternals.java @@ -26,8 +26,8 @@ import org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks; import org.apache.beam.sdk.state.TimeDomain; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTransformExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTransformExecutor.java index 7515f7c294a45..ded87d72e6d00 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTransformExecutor.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTransformExecutor.java @@ -29,8 +29,8 @@ import org.apache.beam.sdk.metrics.MetricsEnvironment; import org.apache.beam.sdk.runners.AppliedPTransform; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectWriteViewVisitor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectWriteViewVisitor.java index 8fab93752c14a..8bb1680978539 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectWriteViewVisitor.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectWriteViewVisitor.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.PCollectionViews; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; /** * Adds a {@link DirectRunner}-specific {@link WriteView} step for each {@link PCollectionView} for diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DoFnLifecycleManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DoFnLifecycleManager.java index fdd46673a459d..e863ccb09d884 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DoFnLifecycleManager.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DoFnLifecycleManager.java @@ -17,6 +17,8 @@ */ package org.apache.beam.runners.direct; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; + import java.util.Collection; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -27,11 +29,11 @@ import org.apache.beam.sdk.transforms.DoFn.Teardown; import org.apache.beam.sdk.transforms.reflect.DoFnInvokers; import org.apache.beam.sdk.util.SerializableUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LoadingCache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.RemovalListener; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.RemovalNotification; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.RemovalListener; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.RemovalNotification; /** * Manages {@link DoFn} setup, teardown, and serialization. @@ -113,9 +115,9 @@ private class TeardownRemovedFnListener implements RemovalListener> notification) { try { - DoFnInvokers.invokerFor(notification.getValue()).invokeTeardown(); + DoFnInvokers.invokerFor(checkNotNull(notification.getValue())).invokeTeardown(); } catch (Exception e) { - thrownOnTeardown.put(notification.getKey(), e); + thrownOnTeardown.put(checkNotNull(notification.getKey()), e); } } } diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java index ebd0104650830..0fca686bced72 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.direct; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.Collection; import java.util.Collections; @@ -47,10 +47,10 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.joda.time.Instant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java index 3441c037966d2..59dc736693d06 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java @@ -39,14 +39,14 @@ import org.apache.beam.sdk.util.UserCodeException; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LoadingCache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.RemovalListener; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Queues; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.RemovalListener; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Queues; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FlattenEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FlattenEvaluatorFactory.java index c8080627ae641..92a65fe289e30 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FlattenEvaluatorFactory.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FlattenEvaluatorFactory.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** * The {@link DirectRunner} {@link TransformEvaluatorFactory} for the {@link Flatten} {@link diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java index e3c1aaf64d4ca..04b57ffd8514a 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java @@ -50,8 +50,8 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Instant; /** diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactory.java index 882cf6eb05d98..3371d477fe2e9 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactory.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.direct; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.ArrayList; import java.util.HashMap; @@ -36,7 +36,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** * The {@link DirectRunner} {@link TransformEvaluatorFactory} for the {@link GroupByKeyOnly} {@link diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactory.java index 13ae9c985c41e..3faaa986bee3b 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactory.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.direct; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import org.apache.beam.runners.direct.DirectRunner.Enforcement; import org.apache.beam.runners.local.StructuralKey; @@ -29,8 +29,8 @@ import org.apache.beam.sdk.util.MutationDetectors; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.SetMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.SetMultimap; import org.joda.time.Instant; /** diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutableListBundleFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutableListBundleFactory.java index d2af98a087c3d..b601f41a61717 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutableListBundleFactory.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutableListBundleFactory.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.direct; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.util.Iterator; @@ -27,8 +27,8 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImpulseEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImpulseEvaluatorFactory.java index 958b8bcca0141..09337f3728b42 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImpulseEvaluatorFactory.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImpulseEvaluatorFactory.java @@ -26,8 +26,8 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; /** The evaluator for the {@link Impulse} transform. Produces only empty byte arrays. */ diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitor.java index c8ee392fa1551..38938b3ff6973 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitor.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/KeyedPValueTrackingVisitor.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.direct; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.HashSet; import java.util.Map; @@ -33,7 +33,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; /** * A pipeline visitor that tracks all keyed {@link PValue PValues}. A {@link PValue} is keyed if it diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/MultiStepCombine.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/MultiStepCombine.java index b526fa65f626a..e88145c5bb7e1 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/MultiStepCombine.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/MultiStepCombine.java @@ -17,9 +17,9 @@ */ package org.apache.beam.runners.direct; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Collections; import java.util.LinkedHashMap; @@ -54,7 +54,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java index 49c2cb05db584..8370f95a419ed 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.direct; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.HashMap; import java.util.List; @@ -47,7 +47,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; @SuppressWarnings({ "rawtypes", // TODO(https://github.com/apache/beam/issues/20447) diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java index 84fc129d79307..7a11343396c46 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java @@ -32,9 +32,9 @@ import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LoadingCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java index 27ba955e588f8..b16bb63883a5f 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiOverrideFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.direct; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.IOException; @@ -57,7 +57,7 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** * A {@link PTransformOverrideFactory} that provides overrides for applications of a {@link ParDo} diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/QuiescenceDriver.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/QuiescenceDriver.java index 62e0cf169da8c..13cf2ebfe8441 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/QuiescenceDriver.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/QuiescenceDriver.java @@ -37,8 +37,8 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootProviderRegistry.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootProviderRegistry.java index 253c1bac0dfd3..560da714a4c63 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootProviderRegistry.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/RootProviderRegistry.java @@ -20,7 +20,7 @@ import static org.apache.beam.runners.core.construction.PTransformTranslation.FLATTEN_TRANSFORM_URN; import static org.apache.beam.runners.core.construction.PTransformTranslation.IMPULSE_TRANSFORM_URN; import static org.apache.beam.runners.direct.TestStreamEvaluatorFactory.DirectTestStreamFactory.DIRECT_TEST_STREAM_URN; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.Collection; import java.util.Map; @@ -29,7 +29,7 @@ import org.apache.beam.sdk.runners.AppliedPTransform; import org.apache.beam.sdk.transforms.Impulse; import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * A {@link RootInputProvider} that delegates to primitive {@link RootInputProvider} implementations diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java index 54734216c301b..62b0a06ddff18 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.direct; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.ArrayList; import java.util.Collection; @@ -42,13 +42,13 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LoadingCache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java index f0471b0d8d885..a5bf31e406d26 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.direct; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Collection; import java.util.concurrent.Executors; @@ -41,9 +41,9 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java index e00a2b7ee9acc..9d8a815ede6be 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java @@ -41,8 +41,8 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionTuple; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Instant; /** A {@link TransformEvaluatorFactory} for stateful {@link ParDo}. */ diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepAndKey.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepAndKey.java index e0149138ba2c7..2cff63259c0d1 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepAndKey.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepAndKey.java @@ -20,7 +20,7 @@ import java.util.Objects; import org.apache.beam.runners.local.StructuralKey; import org.apache.beam.sdk.runners.AppliedPTransform; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java index 9a58e2d856fef..ae4da1ec07172 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java @@ -32,7 +32,7 @@ import org.apache.beam.sdk.runners.AppliedPTransform; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; /** An immutable {@link TransformResult}. */ diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java index 827ae47b14e28..6831639177261 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java @@ -42,9 +42,9 @@ import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java index f3ad7de875850..f30ff8325d7e2 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java @@ -30,8 +30,8 @@ import static org.apache.beam.runners.direct.MultiStepCombine.DIRECT_MERGE_ACCUMULATORS_EXTRACT_OUTPUT_URN; import static org.apache.beam.runners.direct.ParDoMultiOverrideFactory.DIRECT_STATEFUL_PAR_DO_URN; import static org.apache.beam.runners.direct.TestStreamEvaluatorFactory.DirectTestStreamFactory.DIRECT_TEST_STREAM_URN; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.service.AutoService; import java.util.ArrayList; @@ -47,7 +47,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.runners.AppliedPTransform; import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutorServices.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutorServices.java index 7ec618f451e67..1ec5b1c3f17e5 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutorServices.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutorServices.java @@ -23,7 +23,7 @@ import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadDeduplicator.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadDeduplicator.java index b2226980ae790..65b49de8debf5 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadDeduplicator.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadDeduplicator.java @@ -23,9 +23,9 @@ import org.apache.beam.runners.local.StructuralKey; import org.apache.beam.sdk.coders.ByteArrayCoder; import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LoadingCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache; import org.joda.time.Duration; /** diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java index e413bdd5bfdc2..f362e7bdf1e98 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.direct; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.getOnlyElement; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.getOnlyElement; import com.google.auto.value.AutoValue; import java.io.IOException; @@ -40,8 +40,8 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java index 4aca03fdd8403..d5ea6f06b95b9 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.runners.AppliedPTransform; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** * The {@link DirectRunner} {@link TransformEvaluatorFactory} for the {@link DirectRunner-specific} diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkCallbackExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkCallbackExecutor.java index d2f56d363c9c5..cc1606a0e01b4 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkCallbackExecutor.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkCallbackExecutor.java @@ -30,8 +30,8 @@ import org.apache.beam.sdk.runners.AppliedPTransform; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ComparisonChain; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ComparisonChain; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java index b239d24e76ca8..e2db5550ff4ac 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.direct; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -54,18 +54,18 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowTracing; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ComparisonChain; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Queues; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.SortedMultiset; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.TreeMultiset; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ComparisonChain; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Queues; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.SortedMultiset; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.TreeMultiset; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java index ab4626c426bb6..5f0d5071acb31 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java index f570b04aac647..1b77e8bc309fa 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java @@ -38,9 +38,9 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Suppliers; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Suppliers; /** * A {@link PTransformOverrideFactory} that overrides {@link WriteFiles} {@link PTransform diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java index 55b57eea7ae12..1683ea8e142c4 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java @@ -56,8 +56,8 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.hamcrest.Matchers; import org.joda.time.Instant; import org.junit.Before; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CloningBundleFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CloningBundleFactoryTest.java index 6d7a975e2b9d6..e03f453e21ad0 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CloningBundleFactoryTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CloningBundleFactoryTest.java @@ -42,8 +42,8 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Instant; import org.junit.Rule; import org.junit.Test; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java index fab33250e1ab8..87f6d773c9dd8 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java @@ -37,7 +37,7 @@ import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.PValues; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.Matchers; import org.joda.time.Instant; import org.junit.Rule; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternalsTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternalsTest.java index aa3e8d7b96ef5..ae2de1dc67f1d 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternalsTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CopyOnAccessInMemoryStateInternalsTest.java @@ -50,7 +50,7 @@ import org.apache.beam.sdk.transforms.Sum; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.TimestampCombiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Instant; import org.junit.Rule; import org.junit.Test; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java index 606029e7641f8..110f5a97b9e05 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java @@ -42,7 +42,7 @@ import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.PInput; import org.apache.beam.sdk.values.POutput; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.hamcrest.Matchers; import org.junit.Rule; import org.junit.Test; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectMetricsTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectMetricsTest.java index 69e922e1cb34b..46f74d6b7e051 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectMetricsTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectMetricsTest.java @@ -36,7 +36,7 @@ import org.apache.beam.sdk.metrics.MetricName; import org.apache.beam.sdk.metrics.MetricQueryResults; import org.apache.beam.sdk.metrics.MetricsFilter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; import org.junit.After; import org.junit.Before; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRegistrarTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRegistrarTest.java index d7275ecb43a83..86384c7fe0822 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRegistrarTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRegistrarTest.java @@ -25,8 +25,8 @@ import org.apache.beam.runners.direct.DirectRegistrar.Runner; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; import org.apache.beam.sdk.runners.PipelineRunnerRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerApiSurfaceTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerApiSurfaceTest.java index 899713f45351c..63ad5d988ea4f 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerApiSurfaceTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerApiSurfaceTest.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.options.PipelineOptionsRegistrar; import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.util.ApiSurface; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java index f2e69d8b9e530..183f0d2b468ee 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.direct; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -40,9 +40,13 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.apache.beam.runners.direct.DirectRunner.DirectPipelineResult; @@ -91,8 +95,8 @@ import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matchers; import org.joda.time.Duration; @@ -336,6 +340,36 @@ public void hang(ProcessContext context) throws InterruptedException { assertEquals(null, result.waitUntilFinish(Duration.millis(1L))); } + @Test + public void testNoBlockOnRunState() { + + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + final Future handler = + executor.submit( + () -> { + PipelineOptions options = + PipelineOptionsFactory.fromArgs("--blockOnRun=false").create(); + Pipeline pipeline = Pipeline.create(options); + pipeline.apply(GenerateSequence.from(0).to(100)); + + PipelineResult result = pipeline.run(); + while (true) { + if (result.getState() == State.DONE) { + return result; + } + Thread.sleep(100); + } + }); + try { + handler.get(10, TimeUnit.SECONDS); + } catch (ExecutionException | InterruptedException | TimeoutException e) { + // timeout means it never reaches DONE state + throw new RuntimeException(e); + } finally { + executor.shutdownNow(); + } + } + private static final AtomicLong TEARDOWN_CALL = new AtomicLong(-1); @Test diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectTransformExecutorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectTransformExecutorTest.java index a3d0e4be8f90c..196840c9d8605 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectTransformExecutorTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectTransformExecutorTest.java @@ -43,8 +43,8 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.hamcrest.Matchers; import org.joda.time.Instant; import org.junit.Before; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DoFnLifecycleManagerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DoFnLifecycleManagerTest.java index 61e62bd42e705..fc019b708c4b3 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DoFnLifecycleManagerTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DoFnLifecycleManagerTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.direct; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DoFnLifecycleManagersTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DoFnLifecycleManagersTest.java index a47ed55430d0b..49a4023b3b746 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DoFnLifecycleManagersTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DoFnLifecycleManagersTest.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.util.UserCodeException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.hamcrest.Matcher; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java index 68a2040b74dce..b3f3ab529bdf3 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java @@ -67,8 +67,8 @@ import org.apache.beam.sdk.values.PCollection.IsBounded; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Instant; import org.junit.Before; import org.junit.Rule; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutorTest.java index f5e9388ac895d..52781399d1b1f 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutorTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutorTest.java @@ -34,8 +34,8 @@ import org.apache.beam.sdk.testing.ThreadLeakTracker; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.LinkedListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.LinkedListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.joda.time.Instant; import org.junit.Ignore; import org.junit.Rule; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/FlattenEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/FlattenEvaluatorFactoryTest.java index fb02a60a5e0f4..1aea14dac6141 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/FlattenEvaluatorFactoryTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/FlattenEvaluatorFactoryTest.java @@ -33,7 +33,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.hamcrest.Matchers; import org.joda.time.Instant; import org.junit.Rule; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyEvaluatorFactoryTest.java index 5f41929506c0f..63b6fb5c29e9d 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyEvaluatorFactoryTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyEvaluatorFactoryTest.java @@ -34,9 +34,9 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashMultiset; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multiset; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashMultiset; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multiset; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.joda.time.Instant; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactoryTest.java index 7555f8a94ed45..135f42b013236 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactoryTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactoryTest.java @@ -34,9 +34,9 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashMultiset; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multiset; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashMultiset; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multiset; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.joda.time.Instant; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutableListBundleFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutableListBundleFactoryTest.java index 81812ff49d1c6..5b008c3235efb 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutableListBundleFactoryTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutableListBundleFactoryTest.java @@ -40,7 +40,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.Matcher; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImpulseEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImpulseEvaluatorFactoryTest.java index bc5e06e390d24..758b6811c496d 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImpulseEvaluatorFactoryTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImpulseEvaluatorFactoryTest.java @@ -34,7 +34,7 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Instant; import org.junit.Before; import org.junit.Test; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/MockClock.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/MockClock.java index faa5a40cefcdd..0c3028c7141c9 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/MockClock.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/MockClock.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.direct; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java index 93e6686d4fa4b..d7f82c4e045b9 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java @@ -50,9 +50,9 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matchers; import org.joda.time.Instant; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java index f2d1409797ec8..cb8d655aa6028 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java @@ -48,8 +48,8 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Before; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java index 41a602495ba40..2d7b6308816aa 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactoryTest.java @@ -69,9 +69,9 @@ import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Before; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactoryTest.java index 6c393c6858ae8..188c5e66b1c43 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactoryTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactoryTest.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.hamcrest.Matchers; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorServicesTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorServicesTest.java index 41056168b6f55..f1b7986ef6158 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorServicesTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorServicesTest.java @@ -22,7 +22,7 @@ import static org.mockito.Mockito.verify; import java.util.concurrent.ExecutorService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadDeduplicatorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadDeduplicatorTest.java index 50706c3816162..18b5733a335ec 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadDeduplicatorTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadDeduplicatorTest.java @@ -31,10 +31,10 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.beam.runners.direct.UnboundedReadDeduplicator.CachedIdDeduplicator; import org.apache.beam.runners.direct.UnboundedReadDeduplicator.NeverDeduplicator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Futures; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListenableFuture; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Futures; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListenableFuture; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java index a22a829c2faad..a5ca66dd5901a 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java @@ -21,7 +21,7 @@ import static java.util.Collections.emptySet; import static java.util.Collections.singletonMap; import static org.apache.beam.runners.direct.DirectGraphs.getProducer; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; @@ -77,12 +77,12 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ContiguousSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.DiscreteDomain; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.LinkedListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Range; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ContiguousSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.DiscreteDomain; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.LinkedListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Range; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matchers; import org.joda.time.DateTime; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java index c271c6f227b10..91e094927e700 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java @@ -36,7 +36,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; import org.junit.Rule; import org.junit.Test; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java index ca103073cb9ce..847acd4935533 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java @@ -66,9 +66,9 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Before; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java index c6ca05e017a17..b334c8431683f 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java @@ -43,9 +43,9 @@ import org.apache.beam.sdk.transforms.windowing.WindowMappingFn; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.hamcrest.Matchers; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java index 890cd4f1f4219..7e823eabd889e 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java @@ -62,7 +62,7 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PValues; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Rule; import org.junit.Test; diff --git a/runners/extensions-java/metrics/src/main/java/org/apache/beam/runners/extensions/metrics/OWNERS b/runners/extensions-java/metrics/src/main/java/org/apache/beam/runners/extensions/metrics/OWNERS deleted file mode 100644 index 883da4e3f34e5..0000000000000 --- a/runners/extensions-java/metrics/src/main/java/org/apache/beam/runners/extensions/metrics/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - echauchot diff --git a/runners/flink/1.12/src/test/java/org/apache/beam/runners/flink/RemoteMiniClusterImpl.java b/runners/flink/1.12/src/test/java/org/apache/beam/runners/flink/RemoteMiniClusterImpl.java index 290b6f8b8c426..5d8bebd6dff9d 100644 --- a/runners/flink/1.12/src/test/java/org/apache/beam/runners/flink/RemoteMiniClusterImpl.java +++ b/runners/flink/1.12/src/test/java/org/apache/beam/runners/flink/RemoteMiniClusterImpl.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.flink; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.akka.AkkaUtils; import org.apache.flink.runtime.minicluster.MiniCluster; diff --git a/runners/flink/OWNERS b/runners/flink/OWNERS deleted file mode 100644 index 240dda0cded57..0000000000000 --- a/runners/flink/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - angoenka - - mxm - - tweise diff --git a/runners/flink/flink_runner.gradle b/runners/flink/flink_runner.gradle index ef2562df86f31..b1a459337e513 100644 --- a/runners/flink/flink_runner.gradle +++ b/runners/flink/flink_runner.gradle @@ -46,36 +46,51 @@ evaluationDependsOn(":examples:java") /* * Copy & merge source overrides into build directory. */ -def sourceOverridesBase = "${project.buildDir}/source-overrides/src" +def sourceOverridesBase = project.layout.buildDirectory.dir('source-overrides/src').get() def copySourceOverrides = tasks.register('copySourceOverrides', Copy) { it.from main_source_overrides it.into "${sourceOverridesBase}/main/java" it.duplicatesStrategy DuplicatesStrategy.INCLUDE } -compileJava.dependsOn copySourceOverrides def copyResourcesOverrides = tasks.register('copyResourcesOverrides', Copy) { it.from main_resources_overrides it.into "${sourceOverridesBase}/main/resources" it.duplicatesStrategy DuplicatesStrategy.INCLUDE } -processResources.dependsOn copyResourcesOverrides def copyTestSourceOverrides = tasks.register('copyTestSourceOverrides', Copy) { it.from test_source_overrides it.into "${sourceOverridesBase}/test/java" it.duplicatesStrategy DuplicatesStrategy.INCLUDE } -compileTestJava.dependsOn copyTestSourceOverrides def copyTestResourcesOverrides = tasks.register('copyTestResourcesOverrides', Copy) { it.from test_resources_overrides it.into "${sourceOverridesBase}/test/resources" it.duplicatesStrategy DuplicatesStrategy.INCLUDE } + +// add dependency to gradle Java plugin defined tasks +compileJava.dependsOn copySourceOverrides +processResources.dependsOn copyResourcesOverrides +compileTestJava.dependsOn copyTestSourceOverrides processTestResources.dependsOn copyTestResourcesOverrides +// add dependency BeamModulePlugin defined custom tasks +// they are defined only when certain flags are provided (e.g. -Prelease; -Ppublishing, etc) +def sourcesJar = project.tasks.findByName('sourcesJar') +if (sourcesJar != null) { + sourcesJar.dependsOn copySourceOverrides + sourcesJar.dependsOn copyResourcesOverrides +} +def testSourcesJar = project.tasks.findByName('testSourcesJar') +if (testSourcesJar != null) { + testSourcesJar.dependsOn copyTestSourceOverrides + testSourcesJar.dependsOn copyTestResourcesOverrides +} + /* * We have to explicitly set all directories here to make sure each * version of Flink has the correct overrides set. @@ -129,7 +144,7 @@ configurations { dependencies { compileOnly project(":sdks:java:build-tools") - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(":runners:core-java") implementation project(":runners:core-construction-java") diff --git a/runners/flink/job-server-container/flink_job_server_container.gradle b/runners/flink/job-server-container/flink_job_server_container.gradle index b74918104c2bc..f970698e21807 100644 --- a/runners/flink/job-server-container/flink_job_server_container.gradle +++ b/runners/flink/job-server-container/flink_job_server_container.gradle @@ -52,6 +52,9 @@ task copyDockerfileDependencies(type: Copy) { into "build" } +def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] +def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers") + docker { name containerImageName( name: project.docker_image_default_repo_prefix + "flink${project.parent.name}_job_server", @@ -63,6 +66,10 @@ docker { // tags used by dockerTag task tags containerImageTags() files "./build/" + buildx useBuildx + platform(*project.containerPlatforms()) + load useBuildx && !pushContainers + push pushContainers } // Ensure that we build the required resources and copy and file dependencies from related projects diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/CreateStreamingFlinkView.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/CreateStreamingFlinkView.java index eae54b886f12f..815623814b339 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/CreateStreamingFlinkView.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/CreateStreamingFlinkView.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** Flink streaming overrides for various view (side input) transforms. */ @SuppressWarnings({ diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchPortablePipelineTranslator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchPortablePipelineTranslator.java index 0fcf4998dd10e..92c43ccf75b4b 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchPortablePipelineTranslator.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchPortablePipelineTranslator.java @@ -22,7 +22,7 @@ import static org.apache.beam.runners.fnexecution.translation.PipelineTranslatorUtils.createOutputMap; import static org.apache.beam.runners.fnexecution.translation.PipelineTranslatorUtils.getWindowingStrategy; import static org.apache.beam.runners.fnexecution.translation.PipelineTranslatorUtils.instantiateCoder; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; import java.io.IOException; @@ -72,11 +72,11 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.api.common.operators.Order; import org.apache.flink.api.common.typeinfo.TypeInformation; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java index de2a567965b62..91ca8e8fe107d 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.flink; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.util.Collections; @@ -77,10 +77,10 @@ import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.MultimapBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.MultimapBuilder; import org.apache.flink.api.common.functions.RichGroupReduceFunction; import org.apache.flink.api.common.operators.Order; import org.apache.flink.api.common.typeinfo.TypeInformation; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkDetachedRunnerResult.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkDetachedRunnerResult.java index 2f289968cf650..77d0e7d3434ca 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkDetachedRunnerResult.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkDetachedRunnerResult.java @@ -18,9 +18,16 @@ package org.apache.beam.runners.flink; import java.io.IOException; +import java.util.concurrent.ExecutionException; +import org.apache.beam.runners.core.metrics.MetricsContainerStepMap; +import org.apache.beam.runners.flink.metrics.FlinkMetricContainer; import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.metrics.MetricResults; +import org.apache.flink.api.common.JobStatus; +import org.apache.flink.core.execution.JobClient; import org.joda.time.Duration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Result of a detached execution of a {@link org.apache.beam.sdk.Pipeline} with Flink. In detached @@ -28,31 +35,102 @@ */ public class FlinkDetachedRunnerResult implements PipelineResult { - FlinkDetachedRunnerResult() {} + private static final Logger LOG = LoggerFactory.getLogger(FlinkDetachedRunnerResult.class); + + private JobClient jobClient; + private int jobCheckIntervalInSecs; + + FlinkDetachedRunnerResult(JobClient jobClient, int jobCheckIntervalInSecs) { + this.jobClient = jobClient; + this.jobCheckIntervalInSecs = jobCheckIntervalInSecs; + } @Override public State getState() { - return State.UNKNOWN; + try { + return toBeamJobState(jobClient.getJobStatus().get()); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException("Fail to get flink job state", e); + } } @Override public MetricResults metrics() { - throw new UnsupportedOperationException("The FlinkRunner does not currently support metrics."); + return MetricsContainerStepMap.asAttemptedOnlyMetricResults(getMetricsContainerStepMap()); + } + + private MetricsContainerStepMap getMetricsContainerStepMap() { + try { + return (MetricsContainerStepMap) + jobClient + .getAccumulators() + .get() + .getOrDefault(FlinkMetricContainer.ACCUMULATOR_NAME, new MetricsContainerStepMap()); + } catch (InterruptedException | ExecutionException e) { + LOG.warn("Fail to get flink job accumulators", e); + return new MetricsContainerStepMap(); + } } @Override public State cancel() throws IOException { - throw new UnsupportedOperationException("Cancelling is not yet supported."); + try { + this.jobClient.cancel().get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException("Fail to cancel flink job", e); + } + return getState(); } @Override public State waitUntilFinish() { - return State.UNKNOWN; + return waitUntilFinish(Duration.millis(Long.MAX_VALUE)); } @Override public State waitUntilFinish(Duration duration) { - return State.UNKNOWN; + long start = System.currentTimeMillis(); + long durationInMillis = duration.getMillis(); + State state = State.UNKNOWN; + while (durationInMillis < 1 || (System.currentTimeMillis() - start) < durationInMillis) { + state = getState(); + if (state.isTerminal()) { + return state; + } + try { + Thread.sleep(jobCheckIntervalInSecs * 1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + if (state != null && !state.isTerminal()) { + LOG.warn("Job is not finished in {} seconds", duration.getStandardSeconds()); + } + return state; + } + + private State toBeamJobState(JobStatus flinkJobStatus) { + switch (flinkJobStatus) { + case CANCELLING: + case CREATED: + case INITIALIZING: + case FAILING: + case RECONCILING: + case RESTARTING: + case RUNNING: + return State.RUNNING; + case FINISHED: + return State.DONE; + case CANCELED: + return State.CANCELLED; + case FAILED: + return State.FAILED; + case SUSPENDED: + return State.STOPPED; + default: + return State.UNKNOWN; + } } @Override diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkExecutionEnvironments.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkExecutionEnvironments.java index fb4d90884d241..7c1bc87ced036 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkExecutionEnvironments.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkExecutionEnvironments.java @@ -29,19 +29,21 @@ import org.apache.beam.runners.core.construction.SerializablePipelineOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.InstanceBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Streams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.net.HostAndPort; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Streams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.net.HostAndPort; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.ExecutionMode; import org.apache.flink.api.java.CollectionEnvironment; import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.api.java.LocalEnvironment; +import org.apache.flink.api.java.RemoteEnvironment; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.CoreOptions; +import org.apache.flink.configuration.DeploymentOptions; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.configuration.RestOptions; import org.apache.flink.configuration.TaskManagerOptions; @@ -128,6 +130,17 @@ static ExecutionEnvironment createBatchExecutionEnvironment( if (options.getParallelism() != -1 && !(flinkBatchEnv instanceof CollectionEnvironment)) { flinkBatchEnv.setParallelism(options.getParallelism()); } + + // Only RemoteEnvironment support detached mode, other batch environment enforce to use attached + // mode + if (!options.getAttachedMode()) { + if (flinkBatchEnv instanceof RemoteEnvironment) { + flinkBatchEnv.getConfiguration().set(DeploymentOptions.ATTACHED, options.getAttachedMode()); + } else { + LOG.warn("Detached mode is only supported in RemoteEnvironment for batch"); + } + } + // Set the correct parallelism, required by UnboundedSourceWrapper to generate consistent // splits. final int parallelism; @@ -238,6 +251,18 @@ public static StreamExecutionEnvironment createStreamExecutionEnvironment( flinkStreamEnv.disableOperatorChaining(); } + // Only RemoteStreamEnvironment support detached mode, other stream environment enforce to use + // attached mode. + if (!options.getAttachedMode()) { + if (flinkStreamEnv instanceof RemoteStreamEnvironment) { + ((RemoteStreamEnvironment) flinkStreamEnv) + .getClientConfiguration() + .set(DeploymentOptions.ATTACHED, options.getAttachedMode()); + } else { + LOG.warn("Detached mode is only supported in RemoteStreamEnvironment for streaming"); + } + } + // default to event time flinkStreamEnv.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkJobInvoker.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkJobInvoker.java index b6bbe92c97556..b6e65cdc66290 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkJobInvoker.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkJobInvoker.java @@ -29,8 +29,8 @@ import org.apache.beam.runners.jobsubmission.PortablePipelineRunner; import org.apache.beam.sdk.options.PortablePipelineOptions; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Struct; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineExecutionEnvironment.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineExecutionEnvironment.java index 07bb3e9d66192..7961bea6069d9 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineExecutionEnvironment.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineExecutionEnvironment.java @@ -17,13 +17,18 @@ */ package org.apache.beam.runners.flink; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import java.util.Map; import org.apache.beam.runners.core.construction.resources.PipelineResources; +import org.apache.beam.runners.core.metrics.MetricsPusher; import org.apache.beam.sdk.Pipeline; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.sdk.PipelineResult; +import org.apache.beam.sdk.metrics.MetricsOptions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.api.java.ExecutionEnvironment; +import org.apache.flink.core.execution.JobClient; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.graph.StreamGraph; @@ -127,18 +132,56 @@ private static void prepareFilesToStageForRemoteClusterExecution(FlinkPipelineOp } /** Launches the program execution. */ - public JobExecutionResult executePipeline() throws Exception { + public PipelineResult executePipeline() throws Exception { final String jobName = options.getJobName(); if (flinkBatchEnv != null) { - return flinkBatchEnv.execute(jobName); + if (options.getAttachedMode()) { + JobExecutionResult jobExecutionResult = flinkBatchEnv.execute(jobName); + return createAttachedPipelineResult(jobExecutionResult); + } else { + JobClient jobClient = flinkBatchEnv.executeAsync(jobName); + return createDetachedPipelineResult(jobClient, options); + } } else if (flinkStreamEnv != null) { - return flinkStreamEnv.execute(jobName); + if (options.getAttachedMode()) { + JobExecutionResult jobExecutionResult = flinkStreamEnv.execute(jobName); + return createAttachedPipelineResult(jobExecutionResult); + } else { + JobClient jobClient = flinkStreamEnv.executeAsync(jobName); + return createDetachedPipelineResult(jobClient, options); + } } else { throw new IllegalStateException("The Pipeline has not yet been translated."); } } + private FlinkDetachedRunnerResult createDetachedPipelineResult( + JobClient jobClient, FlinkPipelineOptions options) { + LOG.info("Pipeline submitted in detached mode"); + return new FlinkDetachedRunnerResult(jobClient, options.getJobCheckIntervalInSecs()); + } + + private FlinkRunnerResult createAttachedPipelineResult(JobExecutionResult result) { + LOG.info("Execution finished in {} msecs", result.getNetRuntime()); + Map accumulators = result.getAllAccumulatorResults(); + if (accumulators != null && !accumulators.isEmpty()) { + LOG.info("Final accumulator values:"); + for (Map.Entry entry : result.getAllAccumulatorResults().entrySet()) { + LOG.info("{} : {}", entry.getKey(), entry.getValue()); + } + } + FlinkRunnerResult flinkRunnerResult = + new FlinkRunnerResult(accumulators, result.getNetRuntime()); + MetricsPusher metricsPusher = + new MetricsPusher( + flinkRunnerResult.getMetricsContainerStepMap(), + options.as(MetricsOptions.class), + flinkRunnerResult); + metricsPusher.start(); + return flinkRunnerResult; + } + /** * Retrieves the generated JobGraph which can be submitted against the cluster. For testing * purposes. diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java index 68c50960794a7..2435907ecec94 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java @@ -142,6 +142,20 @@ public interface FlinkPipelineOptions void setNumberOfExecutionRetries(Integer retries); + @Description( + "Set job check interval in seconds under detached mode in method waitUntilFinish, " + + "by default it is 5 seconds") + @Default.Integer(5) + int getJobCheckIntervalInSecs(); + + void setJobCheckIntervalInSecs(int seconds); + + @Description("Specifies if the pipeline is submitted in attached or detached mode") + @Default.Boolean(true) + boolean getAttachedMode(); + + void setAttachedMode(boolean attachedMode); + @Description( "Sets the delay in milliseconds between executions. A value of {@code -1} " + "indicates that the default value should be used.") @@ -306,6 +320,13 @@ public interface FlinkPipelineOptions void setEnableStableInputDrain(Boolean enableStableInputDrain); + @Description( + "Set the maximum size of input split when data is read from a filesystem. 0 implies no max size.") + @Default.Long(0) + Long getFileInputSplitMaxSizeMB(); + + void setFileInputSplitMaxSizeMB(Long fileInputSplitMaxSizeMB); + static FlinkPipelineOptions defaults() { return PipelineOptionsFactory.as(FlinkPipelineOptions.class); } diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineRunner.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineRunner.java index b22f51b57f47f..5f6074e2546f3 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineRunner.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineRunner.java @@ -45,7 +45,7 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.options.SdkHarnessOptions; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Struct; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.apache.flink.api.common.JobExecutionResult; import org.checkerframework.checker.nullness.qual.Nullable; import org.kohsuke.args4j.CmdLineException; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPortableClientEntryPoint.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPortableClientEntryPoint.java index 4ccae59c6f118..47d3959ad18cc 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPortableClientEntryPoint.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPortableClientEntryPoint.java @@ -33,9 +33,9 @@ import org.apache.beam.runners.jobsubmission.JobInvoker; import org.apache.beam.runners.jobsubmission.PortablePipelineResult; import org.apache.beam.runners.jobsubmission.PortablePipelineRunner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; import org.apache.flink.api.common.time.Deadline; import org.kohsuke.args4j.CmdLineException; import org.kohsuke.args4j.CmdLineParser; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPortableRunnerResult.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPortableRunnerResult.java index f9b697708b1ce..de19f3dfe7489 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPortableRunnerResult.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPortableRunnerResult.java @@ -17,10 +17,13 @@ */ package org.apache.beam.runners.flink; +import java.io.IOException; import java.util.Map; import org.apache.beam.model.jobmanagement.v1.JobApi; import org.apache.beam.model.pipeline.v1.MetricsApi; import org.apache.beam.runners.jobsubmission.PortablePipelineResult; +import org.apache.beam.sdk.metrics.MetricResults; +import org.joda.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,7 +47,11 @@ public JobApi.MetricResults portableMetrics() throws UnsupportedOperationExcepti .build(); } - static class Detached extends FlinkDetachedRunnerResult implements PortablePipelineResult { + static class Detached implements PortablePipelineResult { + + Detached() { + super(); + } @Override public JobApi.MetricResults portableMetrics() throws UnsupportedOperationException { @@ -52,5 +59,31 @@ public JobApi.MetricResults portableMetrics() throws UnsupportedOperationExcepti "Collecting monitoring infos is not implemented yet in Flink portable runner (detached mode)."); return JobApi.MetricResults.newBuilder().build(); } + + @Override + public State getState() { + return State.UNKNOWN; + } + + @Override + public State cancel() throws IOException { + throw new UnsupportedOperationException("Cancelling is not yet supported."); + } + + @Override + public State waitUntilFinish(Duration duration) { + return State.UNKNOWN; + } + + @Override + public State waitUntilFinish() { + return State.UNKNOWN; + } + + @Override + public MetricResults metrics() { + throw new UnsupportedOperationException( + "The FlinkRunner does not currently support metrics."); + } } } diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java index d047e17eba517..295ad2f98a2f1 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunner.java @@ -18,26 +18,22 @@ package org.apache.beam.runners.flink; import java.util.HashSet; -import java.util.Map; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import org.apache.beam.runners.core.construction.SplittableParDo; import org.apache.beam.runners.core.construction.graph.ProjectionPushdownOptimizer; -import org.apache.beam.runners.core.metrics.MetricsPusher; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.PipelineRunner; import org.apache.beam.sdk.metrics.MetricsEnvironment; -import org.apache.beam.sdk.metrics.MetricsOptions; import org.apache.beam.sdk.options.ExperimentalOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsValidator; import org.apache.beam.sdk.runners.TransformHierarchy; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.View; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.flink.api.common.JobExecutionResult; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.apache.flink.runtime.jobgraph.JobGraph; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -103,42 +99,13 @@ public PipelineResult run(Pipeline pipeline) { LOG.info("Translating pipeline to Flink program."); env.translate(pipeline); - JobExecutionResult result; try { LOG.info("Starting execution of Flink program."); - result = env.executePipeline(); + return env.executePipeline(); } catch (Exception e) { LOG.error("Pipeline execution failed", e); throw new RuntimeException("Pipeline execution failed", e); } - return createPipelineResult(result, options); - } - - static PipelineResult createPipelineResult(JobExecutionResult result, PipelineOptions options) { - String resultClassName = result.getClass().getCanonicalName(); - if (resultClassName.equals("org.apache.flink.core.execution.DetachedJobExecutionResult")) { - LOG.info("Pipeline submitted in Detached mode"); - // no metricsPusher because metrics are not supported in detached mode - return new FlinkDetachedRunnerResult(); - } else { - LOG.info("Execution finished in {} msecs", result.getNetRuntime()); - Map accumulators = result.getAllAccumulatorResults(); - if (accumulators != null && !accumulators.isEmpty()) { - LOG.info("Final accumulator values:"); - for (Map.Entry entry : result.getAllAccumulatorResults().entrySet()) { - LOG.info("{} : {}", entry.getKey(), entry.getValue()); - } - } - FlinkRunnerResult flinkRunnerResult = - new FlinkRunnerResult(accumulators, result.getNetRuntime()); - MetricsPusher metricsPusher = - new MetricsPusher( - flinkRunnerResult.getMetricsContainerStepMap(), - options.as(MetricsOptions.class), - flinkRunnerResult); - metricsPusher.start(); - return flinkRunnerResult; - } } /** For testing. */ diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunnerRegistrar.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunnerRegistrar.java index 49b1bb0a13ab8..7388f37381ddd 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunnerRegistrar.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkRunnerRegistrar.java @@ -22,7 +22,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; import org.apache.beam.sdk.runners.PipelineRunnerRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * AutoService registrar - will register FlinkRunner and FlinkOptions as possible pipeline runner diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java index 8a74735cd83e1..e9f3f7fe9176c 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java @@ -54,9 +54,9 @@ import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.ShardedKey; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; import org.apache.flink.runtime.state.KeyGroupRangeAssignment; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.Preconditions; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPortablePipelineTranslator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPortablePipelineTranslator.java index d1fbd12e186b9..f6c25b0cce684 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPortablePipelineTranslator.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPortablePipelineTranslator.java @@ -99,14 +99,14 @@ import org.apache.beam.sdk.values.ValueWithRecordId; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashMultiset; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashMultiset; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.api.common.functions.FlatMapFunction; import org.apache.flink.api.common.functions.RichMapFunction; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java index ec97bb7337762..6d42d0c3b485b 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java @@ -93,9 +93,9 @@ import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.ValueWithRecordId; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.apache.flink.api.common.functions.FlatMapFunction; import org.apache.flink.api.common.functions.RichFlatMapFunction; import org.apache.flink.api.common.functions.RichMapFunction; @@ -1446,7 +1446,7 @@ private static class CreateStreamingFlinkViewPayloadTranslator private CreateStreamingFlinkViewPayloadTranslator() {} @Override - public String getUrn(CreateStreamingFlinkView.CreateFlinkPCollectionView transform) { + public String getUrn() { return CreateStreamingFlinkView.CREATE_STREAMING_FLINK_VIEW_URN; } } diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java index cf235da1a0252..9791eaeb4ac1d 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.flink; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.HashMap; import java.util.Map; @@ -34,8 +34,8 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkTransformOverrides.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkTransformOverrides.java index 3766e1ede1aa6..b53864d968c75 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkTransformOverrides.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkTransformOverrides.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.flink; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.List; import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems; @@ -27,7 +27,7 @@ import org.apache.beam.runners.core.construction.SplittableParDoNaiveBounded; import org.apache.beam.sdk.runners.PTransformOverride; import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** {@link PTransform} overrides for Flink runner. */ @SuppressWarnings({ diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/FlinkMetricContainer.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/FlinkMetricContainer.java index e389be5f15370..c05db9e0b156b 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/FlinkMetricContainer.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/FlinkMetricContainer.java @@ -33,7 +33,7 @@ import org.apache.beam.sdk.metrics.MetricResult; import org.apache.beam.sdk.metrics.MetricResults; import org.apache.beam.sdk.metrics.MetricsFilter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.apache.flink.api.common.accumulators.Accumulator; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.configuration.GlobalConfiguration; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/AbstractFlinkCombineRunner.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/AbstractFlinkCombineRunner.java index 133c944ca13cc..c4be4736104f4 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/AbstractFlinkCombineRunner.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/AbstractFlinkCombineRunner.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.flink.util.Collector; /** diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkAssignContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkAssignContext.java index 3b52095319131..eb31fcac6eb15 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkAssignContext.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkAssignContext.java @@ -20,7 +20,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Instant; /** {@link org.apache.beam.sdk.transforms.windowing.WindowFn.AssignContext} for Flink functions. */ diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunction.java index af94106542d66..f437daf86e718 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunction.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunction.java @@ -40,7 +40,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.flink.api.common.functions.AbstractRichFunction; import org.apache.flink.api.common.functions.FlatMapFunction; import org.apache.flink.api.common.functions.RuntimeContext; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkExecutableStageFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkExecutableStageFunction.java index afc5ae106d920..7693fb94490d0 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkExecutableStageFunction.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkExecutableStageFunction.java @@ -62,7 +62,7 @@ import org.apache.beam.sdk.transforms.join.RawUnionValue; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.flink.api.common.functions.AbstractRichFunction; import org.apache.flink.api.common.functions.GroupReduceFunction; import org.apache.flink.api.common.functions.MapPartitionFunction; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNonMergingReduceFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNonMergingReduceFunction.java index da350f8e0f208..fff44e327d029 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNonMergingReduceFunction.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkNonMergingReduceFunction.java @@ -27,9 +27,9 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.PeekingIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.PeekingIterator; import org.apache.flink.api.common.functions.GroupReduceFunction; import org.apache.flink.util.Collector; import org.joda.time.Instant; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkSideInputReader.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkSideInputReader.java index ed0e06d1fb066..438688015c4fb 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkSideInputReader.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkSideInputReader.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.flink.translation.functions; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.Collections; import java.util.HashMap; @@ -34,7 +34,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.apache.flink.api.common.functions.RuntimeContext; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunction.java index 76cce7beb67fd..16eac410f2784 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunction.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunction.java @@ -54,7 +54,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.flink.api.common.functions.RichGroupReduceFunction; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.configuration.Configuration; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/HashingFlinkCombineRunner.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/HashingFlinkCombineRunner.java index e11eadd3f989b..455a8540e505f 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/HashingFlinkCombineRunner.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/HashingFlinkCombineRunner.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.util.Collector; import org.joda.time.Instant; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/ImpulseSourceFunction.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/ImpulseSourceFunction.java index 444e8fbc3c7be..577949bcbc8d3 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/ImpulseSourceFunction.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/ImpulseSourceFunction.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.flink.translation.functions; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.flink.api.common.state.ListState; import org.apache.flink.api.common.state.ListStateDescriptor; import org.apache.flink.api.common.typeutils.base.BooleanSerializer; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SideInputInitializer.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SideInputInitializer.java index 63f6fc13607b9..d44428a84f517 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SideInputInitializer.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SideInputInitializer.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.flink.translation.functions; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.ArrayList; import java.util.HashMap; @@ -35,7 +35,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.apache.flink.api.common.functions.BroadcastVariableInitializer; /** diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SingleWindowFlinkCombineRunner.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SingleWindowFlinkCombineRunner.java index 15c65922ca2cd..58ecfdf96e804 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SingleWindowFlinkCombineRunner.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SingleWindowFlinkCombineRunner.java @@ -27,10 +27,10 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.PeekingIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.PeekingIterator; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.util.Collector; import org.joda.time.Instant; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SortingFlinkCombineRunner.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SortingFlinkCombineRunner.java index 22cb949b61f31..149c3e284032a 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SortingFlinkCombineRunner.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/functions/SortingFlinkCombineRunner.java @@ -29,8 +29,8 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.flink.util.Collector; import org.joda.time.Instant; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeInformation.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeInformation.java index b64d8bde095ca..9f4da7cea8e5a 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeInformation.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeInformation.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.flink.translation.types; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import org.apache.beam.runners.core.construction.SerializablePipelineOptions; import org.apache.beam.sdk.coders.Coder; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeSerializer.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeSerializer.java index de460ea7c2153..2195ecdf1ab7d 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeSerializer.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeSerializer.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.CoderException; import org.apache.beam.sdk.util.CoderUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/WindowedKvKeySelector.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/WindowedKvKeySelector.java index 4158c026e2038..13f55a4c14567 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/WindowedKvKeySelector.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/types/WindowedKvKeySelector.java @@ -22,8 +22,8 @@ import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Bytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Bytes; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.typeutils.ResultTypeQueryable; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/LookupPipelineVisitor.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/LookupPipelineVisitor.java index 56b36a38de94a..9cf71d4e78e93 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/LookupPipelineVisitor.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/LookupPipelineVisitor.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** * Pipeline visitor that fills lookup table of {@link PTransform} to {@link AppliedPTransform} for diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/ImpulseInputFormat.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/ImpulseInputFormat.java index 8db4df8b17ba6..153c875ca1436 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/ImpulseInputFormat.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/ImpulseInputFormat.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.flink.translation.wrappers; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import org.apache.beam.sdk.util.WindowedValue; import org.apache.flink.api.common.io.DefaultInputSplitAssigner; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/SourceInputFormat.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/SourceInputFormat.java index 7e1835eac721b..a1b8bced7a1dd 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/SourceInputFormat.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/SourceInputFormat.java @@ -20,9 +20,11 @@ import java.io.IOException; import java.util.List; import org.apache.beam.runners.core.construction.SerializablePipelineOptions; +import org.apache.beam.runners.flink.FlinkPipelineOptions; import org.apache.beam.runners.flink.metrics.FlinkMetricContainer; import org.apache.beam.runners.flink.metrics.ReaderInvocationUtil; import org.apache.beam.sdk.io.BoundedSource; +import org.apache.beam.sdk.io.FileBasedSource; import org.apache.beam.sdk.io.Source; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.transforms.windowing.GlobalWindow; @@ -108,12 +110,30 @@ public float getAverageRecordWidth() { return null; } + private long getDesiredSizeBytes(int numSplits) throws Exception { + long totalSize = initialSource.getEstimatedSizeBytes(options); + long defaultSplitSize = totalSize / numSplits; + long maxSplitSize = 0; + if (options != null) { + maxSplitSize = options.as(FlinkPipelineOptions.class).getFileInputSplitMaxSizeMB(); + } + if (initialSource instanceof FileBasedSource && maxSplitSize > 0) { + // Most of the time parallelism is < number of files in source. + // Each file becomes a unique split which commonly create skew. + // This limits the size of splits to reduce skew. + return Math.min(defaultSplitSize, maxSplitSize * 1024 * 1024); + } else { + return defaultSplitSize; + } + } + @Override @SuppressWarnings("unchecked") public SourceInputSplit[] createInputSplits(int numSplits) throws IOException { try { - long desiredSizeBytes = initialSource.getEstimatedSizeBytes(options) / numSplits; + long desiredSizeBytes = getDesiredSizeBytes(numSplits); List> shards = initialSource.split(desiredSizeBytes, options); + int numShards = shards.size(); SourceInputSplit[] sourceInputSplits = new SourceInputSplit[numShards]; for (int i = 0; i < numShards; i++) { diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java index 394aaf50c06d9..3686734c563b1 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java @@ -88,11 +88,11 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.flink.api.common.state.ListState; import org.apache.flink.api.common.state.ListStateDescriptor; import org.apache.flink.api.common.state.MapState; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ExecutableStageDoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ExecutableStageDoFnOperator.java index 5060fa66a424c..d21316c16160b 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ExecutableStageDoFnOperator.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ExecutableStageDoFnOperator.java @@ -108,8 +108,8 @@ import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.StatusRuntimeException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.flink.api.common.state.ListStateDescriptor; import org.apache.flink.api.common.typeutils.base.StringSerializer; import org.apache.flink.api.java.functions.KeySelector; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/FlinkKeyUtils.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/FlinkKeyUtils.java index 060205cd543e1..6c81df0bfcd36 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/FlinkKeyUtils.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/FlinkKeyUtils.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.flink.translation.wrappers.streaming; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.io.InputStream; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/NonKeyedPushedBackElementsHandler.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/NonKeyedPushedBackElementsHandler.java index 1686f50a844c7..71a49d6b18666 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/NonKeyedPushedBackElementsHandler.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/NonKeyedPushedBackElementsHandler.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.flink.translation.wrappers.streaming; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.stream.Stream; import java.util.stream.StreamSupport; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java index e06bafe851950..747bd9f71894e 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.coders.StructuredCoder; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Singleton keyed work item coder. */ public class SingletonKeyedWorkItemCoder diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java index d0d0019be66d5..59d09ae999664 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SplittableDoFnOperator.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.flink.translation.wrappers.streaming; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Collection; import java.util.Collections; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/StreamingImpulseSource.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/StreamingImpulseSource.java index 116cca3d2761b..8f21e42d61e66 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/StreamingImpulseSource.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/StreamingImpulseSource.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.flink.translation.wrappers.streaming.io; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java index c0e196e813058..961d31a753700 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java @@ -42,7 +42,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.ValueWithRecordId; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.state.ListState; import org.apache.flink.api.common.state.ListStateDescriptor; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/bounded/FlinkBoundedSourceReader.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/bounded/FlinkBoundedSourceReader.java index 9cea73f6a4a32..7fb5fcc714c91 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/bounded/FlinkBoundedSourceReader.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/bounded/FlinkBoundedSourceReader.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.apache.flink.api.connector.source.ReaderOutput; import org.apache.flink.api.connector.source.SourceReaderContext; import org.apache.flink.core.io.InputStatus; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/unbounded/FlinkUnboundedSourceReader.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/unbounded/FlinkUnboundedSourceReader.java index 3c596360efd7c..8f3595b9729d1 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/unbounded/FlinkUnboundedSourceReader.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/unbounded/FlinkUnboundedSourceReader.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.ValueWithRecordId; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.apache.flink.api.common.eventtime.Watermark; import org.apache.flink.api.connector.source.ReaderOutput; import org.apache.flink.api.connector.source.SourceOutput; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/BufferedElements.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/BufferedElements.java index ad3e37fa5a752..91887dd04ed73 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/BufferedElements.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/BufferedElements.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.state.TimeDomain; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/BufferingDoFnRunner.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/BufferingDoFnRunner.java index 2a9b176796efb..ef32af1d318a6 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/BufferingDoFnRunner.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/BufferingDoFnRunner.java @@ -33,8 +33,8 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.flink.api.common.state.ListState; import org.apache.flink.api.common.state.ListStateDescriptor; import org.apache.flink.runtime.state.KeyedStateBackend; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/NonKeyedBufferingElementsHandler.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/NonKeyedBufferingElementsHandler.java index f5f195e4c8657..18fe6d84b79dd 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/NonKeyedBufferingElementsHandler.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/NonKeyedBufferingElementsHandler.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.flink.translation.wrappers.streaming.stableinput; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.stream.Stream; import java.util.stream.StreamSupport; diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java index 5c70bc3979910..205270c22332c 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java @@ -60,14 +60,14 @@ import org.apache.beam.sdk.util.CombineContextFactory; import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TimestampedValue.TimestampedValueCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.TreeMultiset; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.TreeMultiset; import org.apache.flink.api.common.state.ListState; import org.apache.flink.api.common.state.ListStateDescriptor; import org.apache.flink.api.common.state.MapStateDescriptor; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkJobServerDriverTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkJobServerDriverTest.java index 47b2e0f11221c..4a628eeb4fdf0 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkJobServerDriverTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkJobServerDriverTest.java @@ -25,7 +25,7 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; import org.junit.Test; /** Tests for {@link FlinkJobServerDriver}. */ diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkPipelineExecutionEnvironmentTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkPipelineExecutionEnvironmentTest.java index 8af13c4c7fe35..d8c4c6f6c8ecb 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkPipelineExecutionEnvironmentTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkPipelineExecutionEnvironmentTest.java @@ -53,8 +53,8 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.windowing.FixedWindows; import org.apache.beam.sdk.transforms.windowing.Window; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.api.java.RemoteEnvironment; import org.apache.flink.configuration.Configuration; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkRequiresStableInputTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkRequiresStableInputTest.java index bac201ff56cca..91d6aca3507cf 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkRequiresStableInputTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkRequiresStableInputTest.java @@ -59,8 +59,8 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.joda.time.Instant; import org.junit.BeforeClass; import org.junit.ClassRule; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSavepointTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSavepointTest.java index 32fd7f8a1060f..fef5df4021714 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSavepointTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSavepointTest.java @@ -51,8 +51,8 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.JobStatus; import org.apache.flink.configuration.CheckpointingOptions; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslatorTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslatorTest.java index 849f8be952cbe..5d56e6ddbf675 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslatorTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslatorTest.java @@ -46,8 +46,8 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.ShardedKey; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.state.KeyGroupRangeAssignment; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslatorsTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslatorsTest.java index 45cfb30f9118c..451070c1c1643 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslatorsTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslatorsTest.java @@ -47,7 +47,7 @@ import org.apache.beam.sdk.values.PValues; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.transformations.LegacySourceTransformation; import org.apache.flink.streaming.api.transformations.OneInputTransformation; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSubmissionTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSubmissionTest.java index a2c04d88f713c..601dbc66b1a22 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSubmissionTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSubmissionTest.java @@ -30,10 +30,10 @@ import org.apache.beam.sdk.io.GenerateSequence; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.flink.client.cli.CliFrontend; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; @@ -184,6 +184,7 @@ private static void prepareEnvironment() throws Exception { flinkCluster.getClusterPort(), RestOptions.PORT.key(), flinkCluster.getRestPort()); + Files.write(file.toPath(), config.getBytes(Charsets.UTF_8)); // Create a new environment with the location of the Flink config for CliFrontend diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableExecutionTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableExecutionTest.java index 9a9c4125090ee..4da94e5a13a04 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableExecutionTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableExecutionTest.java @@ -42,9 +42,9 @@ import org.apache.beam.sdk.transforms.WithKeys; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableStateExecutionTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableStateExecutionTest.java index 8468893855350..aa40ec0e9f3a6 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableStateExecutionTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableStateExecutionTest.java @@ -41,8 +41,8 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableTimersExecutionTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableTimersExecutionTest.java index 828e23a96ee24..594bf55d73de1 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableTimersExecutionTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableTimersExecutionTest.java @@ -51,8 +51,8 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourcePortableTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourcePortableTest.java index 84f41824fd484..193f291ab9fd5 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourcePortableTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourcePortableTest.java @@ -49,9 +49,9 @@ import org.apache.beam.sdk.transforms.windowing.FixedWindows; import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourceStreamingTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourceStreamingTest.java index 2921065c15474..8da44d4b3a83e 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourceStreamingTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourceStreamingTest.java @@ -23,7 +23,7 @@ import org.apache.beam.sdk.io.TextIO; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; import org.apache.flink.test.util.AbstractTestBase; import org.apache.flink.test.util.TestBaseUtils; import org.junit.After; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourceTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourceTest.java index 2974780f049b6..b4e6e707b8553 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourceTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourceTest.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; import org.apache.flink.test.util.JavaProgramTestBase; import org.apache.flink.test.util.TestBaseUtils; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/metrics/FlinkMetricContainerTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/metrics/FlinkMetricContainerTest.java index d0efaa64332b9..0947ddda8d0b4 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/metrics/FlinkMetricContainerTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/metrics/FlinkMetricContainerTest.java @@ -46,7 +46,7 @@ import org.apache.beam.sdk.metrics.MetricKey; import org.apache.beam.sdk.metrics.MetricName; import org.apache.beam.sdk.metrics.MetricsContainer; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.metrics.SimpleCounter; import org.junit.Before; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/GroupByNullKeyTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/GroupByNullKeyTest.java index e3f3fda464695..5b3a338546022 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/GroupByNullKeyTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/GroupByNullKeyTest.java @@ -33,7 +33,7 @@ import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; import org.apache.flink.test.util.AbstractTestBase; import org.apache.flink.test.util.TestBaseUtils; import org.joda.time.Duration; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/TopWikipediaSessionsTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/TopWikipediaSessionsTest.java index faa35ca4e0a70..f6fd654bbcef7 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/TopWikipediaSessionsTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/TopWikipediaSessionsTest.java @@ -32,7 +32,7 @@ import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; import org.apache.flink.test.util.AbstractTestBase; import org.apache.flink.test.util.TestBaseUtils; import org.joda.time.Duration; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/functions/FlinkExecutableStageFunctionTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/functions/FlinkExecutableStageFunctionTest.java index 9cc91be3f4b63..88488d82884fe 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/functions/FlinkExecutableStageFunctionTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/functions/FlinkExecutableStageFunctionTest.java @@ -52,7 +52,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Struct; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.flink.api.common.cache.DistributedCache; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.configuration.Configuration; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperatorTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperatorTest.java index ee5bc90093e8d..17cc16cc76e07 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperatorTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperatorTest.java @@ -89,11 +89,11 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ExecutableStageDoFnOperatorTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ExecutableStageDoFnOperatorTest.java index fdb966686332a..1aedd1af673b3 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ExecutableStageDoFnOperatorTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/ExecutableStageDoFnOperatorTest.java @@ -103,10 +103,10 @@ import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Struct; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.flink.api.common.cache.DistributedCache; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.api.common.typeinfo.TypeInformation; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/FlinkKeyUtilsTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/FlinkKeyUtilsTest.java index 18c4035a14464..d1751c047ef18 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/FlinkKeyUtilsTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/FlinkKeyUtilsTest.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.coders.VoidCoder; import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; import org.junit.Test; /** Tests for {@link FlinkKeyUtils}. */ diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/StreamRecordStripper.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/StreamRecordStripper.java index 4f3a5f64f012c..004dc83164c2a 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/StreamRecordStripper.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/StreamRecordStripper.java @@ -18,8 +18,8 @@ package org.apache.beam.runners.flink.translation.wrappers.streaming; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WindowDoFnOperatorTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WindowDoFnOperatorTest.java index 4383094cb3369..8fab1bc6c1678 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WindowDoFnOperatorTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WindowDoFnOperatorTest.java @@ -50,7 +50,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.typeutils.GenericTypeInfo; import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapperTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapperTest.java index 3eb1ef6983606..0634695b1c465 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapperTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapperTest.java @@ -54,7 +54,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.ValueWithRecordId; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/BufferedElementsTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/BufferedElementsTest.java index 1259c2a696334..5652d23662fd5 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/BufferedElementsTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/BufferedElementsTest.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.Matchers; import org.joda.time.Instant; import org.junit.Test; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/BufferingDoFnRunnerTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/BufferingDoFnRunnerTest.java index 573e4b76ede09..58f6fd8484bf0 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/BufferingDoFnRunnerTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/BufferingDoFnRunnerTest.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.coders.VarIntCoder; import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.flink.api.common.state.ListState; import org.apache.flink.runtime.state.OperatorStateBackend; import org.junit.Assert; diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/website/PipelineOptionsTableGenerator.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/website/PipelineOptionsTableGenerator.java index adfc7af642e81..6994de2ed6fe5 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/website/PipelineOptionsTableGenerator.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/website/PipelineOptionsTableGenerator.java @@ -26,8 +26,8 @@ import org.apache.beam.runners.flink.FlinkPipelineOptions; import org.apache.beam.sdk.options.Default; import org.apache.beam.sdk.options.Description; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.CaseFormat; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CaseFormat; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/OWNERS b/runners/google-cloud-dataflow-java/OWNERS deleted file mode 100644 index 20c3f13857d9b..0000000000000 --- a/runners/google-cloud-dataflow-java/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - pabloem - - lukecwik diff --git a/runners/google-cloud-dataflow-java/arm/build.gradle b/runners/google-cloud-dataflow-java/arm/build.gradle new file mode 100644 index 0000000000000..e79eeedcd8284 --- /dev/null +++ b/runners/google-cloud-dataflow-java/arm/build.gradle @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import groovy.json.JsonOutput + +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.runners.dataflow', + publish: false, + classesTriggerCheckerBugs: [ + 'PrimitiveParDoSingleFactory': 'https://github.com/typetools/checker-framework/issues/3791', + // TODO(https://github.com/apache/beam/issues/21068): This currently crashes with checkerframework 3.10.0 + // when compiling :runners:google-cloud-dataflow-java:compileJava with: + // message: class file for com.google.api.services.bigquery.model.TableRow not found + // ; The Checker Framework crashed. Please report the crash. + // Compilation unit: /usr/local/google/home/lcwik/git/beam/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DefaultCoderCloudObjectTranslatorRegistrar.java + // Last visited tree at line 57 column 1: + // @AutoService(CoderCloudObjectTranslatorRegistrar.class) + // Exception: com.sun.tools.javac.code.Symbol$CompletionFailure: class file for com.google.api.services.bigquery.model.TableRow not found; com.sun.tools.javac.code.Symbol$CompletionFailure: class file for com.google.api.services.bigquery.model.TableRow not found + 'DefaultCoderCloudObjectTranslatorRegistrar': 'TODO(https://github.com/apache/beam/issues/21068): Report the crash if still occurring on newest version', + ], +) + +description = "Apache Beam :: Runners :: Google Cloud Dataflow" + +/* + * We need to rely on manually specifying these evaluationDependsOn to ensure that + * the following projects are evaluated before we evaluate this project. This is because + * we are attempting to reference parameters such as "sourceSets.test.output" directly. + */ +evaluationDependsOn(":sdks:java:io:google-cloud-platform") +evaluationDependsOn(":sdks:java:core") +evaluationDependsOn(":examples:java") +evaluationDependsOn(":runners:google-cloud-dataflow-java:worker") +evaluationDependsOn(":sdks:java:container:java8") +evaluationDependsOn(":sdks:java:container:java11") + +processResources { + filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [ + 'dataflow.fnapi_environment_major_version' : project(':runners:google-cloud-dataflow-java').ext.dataflowFnapiEnvironmentMajorVersion, + 'dataflow.fnapi_container_version' : project(':runners:google-cloud-dataflow-java').ext.dataflowFnapiContainerVersion, + 'dataflow.container_base_repository' : project(':runners:google-cloud-dataflow-java').ext.dataflowContainerBaseRepository, + ] +} + +// Exclude tests that need a runner +test { + systemProperty "beamTestPipelineOptions", "" + systemProperty "beamUseDummyRunner", "true" + useJUnit { + excludeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' + } +} + +configurations { examplesJavaIntegrationTest } + +dependencies { + examplesJavaIntegrationTest project(project.path) + examplesJavaIntegrationTest project(path: ":runners:google-cloud-dataflow-java", configuration: "testRuntimeMigration") + examplesJavaIntegrationTest project(path: ":examples:java", configuration: "testRuntimeMigration") +} + +def javaVer = "java8" +if(project.hasProperty('compileAndRunTestsWithJava17')) { + javaVer = "java17" +} else if(project.hasProperty('compileAndRunTestsWithJava11')) { + javaVer = "java11" +} +def dataflowProject = project.findProperty('dataflowProject') ?: 'apache-beam-testing' +def dataflowRegion = project.findProperty('dataflowRegion') ?: 'us-central1' +def dataflowValidatesTempRoot = project.findProperty('dataflowTempRoot') ?: 'gs://temp-storage-for-validates-runner-tests' +def firestoreDb = project.findProperty('firestoreDb') ?: 'firestoredb' +def dockerImageRoot = project.findProperty('docker-repository-root') ?: "us.gcr.io/${dataflowProject}/java-postcommit-it" +def DockerJavaMultiarchImageContainer = "${dockerImageRoot}/${project.docker_image_default_repo_prefix}${javaVer}_sdk" +def dockerTag = project.findProperty('docker-tag') ?: new Date().format('yyyyMMddHHmmss') +ext.DockerJavaMultiarchImageName = "${DockerJavaMultiarchImageContainer}:${dockerTag}" + +def runnerV2PipelineOptionsARM = [ + "--runner=TestDataflowRunner", + "--project=${dataflowProject}", + "--region=${dataflowRegion}", + "--tempRoot=${dataflowValidatesTempRoot}", + "--sdkContainerImage=${DockerJavaMultiarchImageContainer}:${dockerTag}", + "--experiments=use_unified_worker,use_runner_v2", + "--firestoreDb=${firestoreDb}", + "--workerMachineType=t2a-standard-1", +] + +// Build and push multi-arch docker images to a container registry for use within tests. +// NB: Tasks which consume docker images from the registry should depend on this +// task directly ('dependsOn buildAndPushDockerJavaMultiarchContainer'). +// Note: we don't delete the multi-arch containers here because this command only +// deletes the manifest list with the tag, the associated container images can't be +// deleted because they are not tagged. However, multi-arch containers that are older +// than 6 weeks old are deleted by stale_dataflow_prebuilt_image_cleaner.sh that runs +// daily. +def buildAndPushDockerJavaMultiarchContainer = tasks.register("buildAndPushDockerJavaMultiarchContainer") { + dependsOn ":sdks:java:container:${javaVer}:docker" +} + +task printrunnerV2PipelineOptionsARM { + dependsOn buildAndPushDockerJavaMultiarchContainer + + doLast { + println "To run a Dataflow job with runner V2 on ARM, add the following pipeline options to your command-line:" + println runnerV2PipelineOptionsARM.join(' ') + } +} + +task examplesJavaRunnerV2IntegrationTestARM(type: Test) { + group = "Verification" + dependsOn buildAndPushDockerJavaMultiarchContainer + + systemProperty "beamTestPipelineOptions", JsonOutput.toJson(runnerV2PipelineOptionsARM) + + include '**/*IT.class' + // TODO(https://github.com/apache/beam/issues/20593): test times out. + exclude '**/FhirIOReadIT.class' + + maxParallelForks 4 + classpath = configurations.examplesJavaIntegrationTest + testClassesDirs = files(project(":examples:java").sourceSets.test.output.classesDirs) + useJUnit { } +} diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index a39f05eea5bd1..adc1f2e09bc4e 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -49,13 +49,19 @@ evaluationDependsOn(":runners:google-cloud-dataflow-java:worker") evaluationDependsOn(":sdks:java:container:java8") evaluationDependsOn(":sdks:java:container:java11") +ext.dataflowLegacyEnvironmentMajorVersion = '8' +ext.dataflowFnapiEnvironmentMajorVersion = '8' +ext.dataflowLegacyContainerVersion = 'beam-master-20230809' +ext.dataflowFnapiContainerVersion = 'beam-master-20230809' +ext.dataflowContainerBaseRepository = 'gcr.io/cloud-dataflow/v1beta3' + processResources { filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [ - 'dataflow.legacy_environment_major_version' : '8', - 'dataflow.fnapi_environment_major_version' : '8', - 'dataflow.legacy_container_version' : 'beam-master-20230426', - 'dataflow.fnapi_container_version' : 'beam-master-20230426', - 'dataflow.container_base_repository' : 'gcr.io/cloud-dataflow/v1beta3', + 'dataflow.legacy_environment_major_version' : project.ext.dataflowFnapiEnvironmentMajorVersion, + 'dataflow.fnapi_environment_major_version' : project.ext.dataflowFnapiEnvironmentMajorVersion, + 'dataflow.legacy_container_version' : project.ext.dataflowLegacyContainerVersion, + 'dataflow.fnapi_container_version' : project.ext.dataflowFnapiContainerVersion, + 'dataflow.container_base_repository' : project.ext.dataflowContainerBaseRepository, ] } @@ -78,7 +84,7 @@ configurations { dependencies { implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) permitUnusedDeclared enforcedPlatform(library.java.google_cloud_platform_libraries_bom) - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":model:pipeline", configuration: "shadow") implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(":sdks:java:extensions:avro") @@ -606,6 +612,9 @@ task googleCloudPlatformRunnerV2IntegrationTest(type: Test) { exclude '**/FhirIOLROIT.class' exclude '**/FhirIOSearchIT.class' exclude '**/FhirIOPatientEverythingIT.class' + // failing due to pane index not incrementing after Reshuffle: + // https://github.com/apache/beam/issues/28219 + exclude '**/FileLoadsStreamingIT.class' maxParallelForks 4 classpath = configurations.googleCloudPlatformIntegrationTest @@ -613,24 +622,13 @@ task googleCloudPlatformRunnerV2IntegrationTest(type: Test) { useJUnit { } } -task examplesJavaLegacyWorkerIntegrationTest(type: Test) { +task examplesJavaRunnerV2PreCommit(type: Test) { group = "Verification" - dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" - - systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ - "--runner=TestDataflowRunner", - "--project=${dataflowProject}", - "--region=${dataflowRegion}", - "--tempRoot=${dataflowPostCommitTempRoot}", - "--dataflowWorkerJar=${dataflowLegacyWorkerJar}", - "--workerHarnessContainerImage=", - ]) + dependsOn buildAndPushDockerJavaContainer + systemProperty "beamTestPipelineOptions", JsonOutput.toJson(runnerV2PipelineOptions) + include '**/WordCountIT.class' + include '**/WindowedWordCountIT.class' - // The examples/java preCommit task already covers running WordCountIT/WindowedWordCountIT so - // this postCommit integration test excludes them. - include '**/*IT.class' - exclude '**/WordCountIT.class' - exclude '**/WindowedWordCountIT.class' maxParallelForks 4 classpath = configurations.examplesJavaIntegrationTest testClassesDirs = files(project(":examples:java").sourceSets.test.output.classesDirs) @@ -643,7 +641,7 @@ task examplesJavaRunnerV2IntegrationTest(type: Test) { systemProperty "beamTestPipelineOptions", JsonOutput.toJson(runnerV2PipelineOptions) - // The examples/java preCommit task already covers running WordCountIT/WindowedWordCountIT so + // examplesJavaRunnerV2PreCommit task already covers running WordCountIT/WindowedWordCountIT so // this postCommit integration test excludes them. include '**/*IT.class' exclude '**/WordCountIT.class' @@ -701,7 +699,6 @@ task postCommit { description = "Various integration tests using the Dataflow runner." dependsOn googleCloudPlatformLegacyWorkerIntegrationTest dependsOn googleCloudPlatformLegacyWorkerKmsIntegrationTest - dependsOn examplesJavaLegacyWorkerIntegrationTest dependsOn coreSDKJavaLegacyWorkerIntegrationTest } @@ -709,7 +706,6 @@ task postCommitRunnerV2 { group = "Verification" description = "Various integration tests using the Dataflow runner V2." dependsOn googleCloudPlatformRunnerV2IntegrationTest - dependsOn examplesJavaRunnerV2IntegrationTest dependsOn coreSDKJavaRunnerV2IntegrationTest } diff --git a/runners/google-cloud-dataflow-java/examples/build.gradle b/runners/google-cloud-dataflow-java/examples/build.gradle index b7be85134ba4e..20bc50dea5a70 100644 --- a/runners/google-cloud-dataflow-java/examples/build.gradle +++ b/runners/google-cloud-dataflow-java/examples/build.gradle @@ -53,11 +53,25 @@ def dataflowWorkerImpersonationServiceAccount = project.findProperty('dataflowWo def impersonationTempRoot = project.findProperty('gcpImpersonationTempRoot') ?: 'gs://impersonation-test-bucket/tmproot' +/* + * Set common configurations for test tasks. + * + * @param dataflowWorkerJar (required) the path te dataflow worker Jar. + * @param gcsTempRoot gcs temp root + * @additionalOptions additional options + * @workerHarnessContainerImage worker harness container image + * @param runWordCount + * "only" - only run wordcount; + * "exclude" - run example without wordcount; + * "include" - run full example + */ def commonConfig = { Map args -> if (!args.dataflowWorkerJar) { throw new GradleException("Dataflow integration test configuration requires dataflowWorkerJar parameter") } + def runWordCount = args?.runWordCount ?: 'only' + def actualDataflowWorkerJar = args.dataflowWorkerJar def actualWorkerHarnessContainerImage = args.workerHarnessContainerImage ?: '' def actualGcsTempRoot = args.gcsTempRoot ?: gcsTempRoot @@ -66,8 +80,16 @@ def commonConfig = { Map args -> // return the preevaluated configuration closure return { testClassesDirs = files(project(":examples:java").sourceSets.test.output.classesDirs) - include "**/WordCountIT.class" - include "**/WindowedWordCountIT.class" + if (runWordCount == 'only') { + include "**/WordCountIT.class" + include "**/WindowedWordCountIT.class" + } else { + include "**/IT.class" + if (runWordCount == 'exclude') { + exclude "**/WordCountIT.class" + exclude "**/WindowedWordCountIT.class" + } + } forkEvery 1 maxParallelForks 4 @@ -115,11 +137,21 @@ task verifyFnApiWorker(type: Test) { } } +task postCommitLegacyWorker(type: Test) { + dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + with commonConfig(dataflowWorkerJar: dataflowWorkerJar, runWordCount: 'exclude') +} + +task javaPostCommit() { + dependsOn postCommitLegacyWorker +} + task postCommitLegacyWorkerJava11(type: Test) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath systemProperty "java.specification.version", "11" - with commonConfig(dataflowWorkerJar: dataflowWorkerJar) + with commonConfig(dataflowWorkerJar: dataflowWorkerJar, runWordCount: 'only') } task java11PostCommit() { @@ -130,7 +162,7 @@ task postCommitLegacyWorkerJava17(type: Test) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath systemProperty "java.specification.version", "17" - with commonConfig(dataflowWorkerJar: dataflowWorkerJar) + with commonConfig(dataflowWorkerJar: dataflowWorkerJar, runWordCount: 'only') } task java17PostCommit() { diff --git a/runners/google-cloud-dataflow-java/scripts/cleanup_untagged_gcr_images.sh b/runners/google-cloud-dataflow-java/scripts/cleanup_untagged_gcr_images.sh index 5bf8197f8f98d..67ee242ee0d1b 100755 --- a/runners/google-cloud-dataflow-java/scripts/cleanup_untagged_gcr_images.sh +++ b/runners/google-cloud-dataflow-java/scripts/cleanup_untagged_gcr_images.sh @@ -27,6 +27,7 @@ echo "${DIGESTS}" | while read -r digest; do if [[ ! -z "${digest}" ]]; then img="${IMAGE_NAME}@${digest}" echo "Removing untagged image ${img}" - gcloud container images delete --quiet "${img}" + # Tolerate 404 as tags may target to same image and already removed. + DELETE_RESULT="$(gcloud container images delete --quiet "${img}" 2>&1)" || [[ "$DELETE_RESULT" == *"'status': 404"* ]] fi done diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverrides.java index 1b0008a69636d..838b00f02c518 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverrides.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverrides.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Map; import org.apache.beam.runners.core.construction.PTransformReplacements; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java index 9013ca2b3499c..6af6c499cfad5 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.IOException; @@ -75,15 +75,15 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ForwardingMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ForwardingMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowClient.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowClient.java index 325148cb9f594..d643c9e40cde0 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowClient.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowClient.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.dataflow.Dataflow; import com.google.api.services.dataflow.Dataflow.Projects.Locations.Jobs; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java index 3c92aebe1851d..9e6f70a54213f 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowMetrics.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.dataflow; import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; import com.google.api.client.util.ArrayMap; import com.google.api.services.dataflow.model.JobMetrics; @@ -39,9 +39,9 @@ import org.apache.beam.sdk.metrics.MetricResults; import org.apache.beam.sdk.metrics.MetricsFilter; import org.apache.beam.sdk.runners.AppliedPTransform; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPTransformMatchers.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPTransformMatchers.java index c0d4fef138316..1a03c1666eaf4 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPTransformMatchers.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPTransformMatchers.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.toStringHelper; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.toStringHelper; import java.util.ArrayDeque; import org.apache.beam.sdk.Pipeline; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java index 4090ac0cd9fbd..3838534c6aee4 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.dataflow; import static org.apache.beam.runners.dataflow.util.TimeUtil.fromCloudTime; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.util.BackOff; @@ -43,10 +43,10 @@ import org.apache.beam.sdk.metrics.MetricResults; import org.apache.beam.sdk.runners.AppliedPTransform; import org.apache.beam.sdk.util.FluentBackoff; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashBiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.slf4j.Logger; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java index f4baa5c2a8b20..2650df56ed814 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java @@ -23,7 +23,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; import org.apache.beam.sdk.runners.PipelineRunnerRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * Contains the {@link PipelineOptionsRegistrar} and {@link PipelineRunnerRegistrar} for the {@link diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java index f5b493c7a3f22..1660f6ee61940 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java @@ -27,10 +27,10 @@ import static org.apache.beam.sdk.options.ExperimentalOptions.hasExperiment; import static org.apache.beam.sdk.util.SerializableUtils.serializeToByteArray; import static org.apache.beam.sdk.util.StringUtils.byteArrayToJsonString; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings.isNullOrEmpty; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings.isNullOrEmpty; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -108,9 +108,9 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.commons.codec.EncoderException; import org.apache.commons.codec.net.PercentCodec; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java index 60ebe8d8846a7..26548038a1dfc 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java @@ -22,10 +22,10 @@ import static org.apache.beam.sdk.util.CoderUtils.encodeToByteArray; import static org.apache.beam.sdk.util.SerializableUtils.serializeToByteArray; import static org.apache.beam.sdk.util.StringUtils.byteArrayToJsonString; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings.isNullOrEmpty; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings.isNullOrEmpty; import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; @@ -129,6 +129,7 @@ import org.apache.beam.sdk.transforms.Combine; import org.apache.beam.sdk.transforms.Combine.CombineFn; import org.apache.beam.sdk.transforms.Combine.GroupedValues; +import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.GroupIntoBatches; import org.apache.beam.sdk.transforms.Impulse; @@ -163,18 +164,18 @@ import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.TextFormat; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Utf8; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashCode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Utf8; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.HashCode; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; import org.joda.time.DateTimeUtils; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; @@ -495,6 +496,23 @@ protected DataflowRunner(DataflowPipelineOptions options) { this.ptransformViewsWithNonDeterministicKeyCoders = new HashSet<>(); } + private static class AlwaysCreateViaRead + implements PTransformOverrideFactory, Create.Values> { + @Override + public PTransformOverrideFactory.PTransformReplacement> + getReplacementTransform( + AppliedPTransform, Create.Values> appliedTransform) { + return PTransformOverrideFactory.PTransformReplacement.of( + appliedTransform.getPipeline().begin(), appliedTransform.getTransform().alwaysUseRead()); + } + + @Override + public final Map, ReplacementOutput> mapOutputs( + Map, PCollection> outputs, PCollection newOutput) { + return ReplacementOutputs.singleton(outputs, newOutput); + } + } + private List getOverrides(boolean streaming) { ImmutableList.Builder overridesBuilder = ImmutableList.builder(); @@ -509,6 +527,13 @@ private List getOverrides(boolean streaming) { PTransformOverride.of( PTransformMatchers.emptyFlatten(), EmptyFlattenAsCreateFactory.instance())); + if (streaming) { + // For update compatibility, always use a Read for Create in streaming mode. + overridesBuilder.add( + PTransformOverride.of( + PTransformMatchers.classEqualTo(Create.Values.class), new AlwaysCreateViaRead())); + } + // By default Dataflow runner replaces single-output ParDo with a ParDoSingle override. // However, we want a different expansion for single-output splittable ParDo. overridesBuilder @@ -661,6 +686,29 @@ private List getOverrides(boolean streaming) { PTransformOverride.of( PTransformMatchers.classEqualTo(ParDo.SingleOutput.class), new PrimitiveParDoSingleFactory())); + + if (streaming) { + // For update compatibility, always use a Read for Create in streaming mode. + overridesBuilder + .add( + PTransformOverride.of( + PTransformMatchers.classEqualTo(Create.Values.class), new AlwaysCreateViaRead())) + // Create is implemented in terms of BoundedRead. + .add( + PTransformOverride.of( + PTransformMatchers.classEqualTo(Read.Bounded.class), + new StreamingBoundedReadOverrideFactory())) + // Streaming Bounded Read is implemented in terms of Streaming Unbounded Read. + .add( + PTransformOverride.of( + PTransformMatchers.classEqualTo(Read.Unbounded.class), + new StreamingUnboundedReadOverrideFactory())) + .add( + PTransformOverride.of( + PTransformMatchers.classEqualTo(ParDo.SingleOutput.class), + new PrimitiveParDoSingleFactory())); + } + return overridesBuilder.build(); } @@ -1341,6 +1389,7 @@ public DataflowPipelineJob run(Pipeline pipeline) { newJob.setReplaceJobId(jobIdToUpdate); } if (options.getCreateFromSnapshot() != null && !options.getCreateFromSnapshot().isEmpty()) { + newJob.setTransformNameMapping(options.getTransformNameMapping()); newJob.setCreatedFromSnapshotId(options.getCreateFromSnapshot()); } @@ -1467,7 +1516,11 @@ static void configureSdkHarnessContainerImages( SdkHarnessContainerImage image = new SdkHarnessContainerImage(); image.setEnvironmentId(environmentInfo.environmentId()); image.setContainerImage(environmentInfo.containerUrl()); - if (environmentInfo.containerUrl().toLowerCase().contains("python")) { + if (!environmentInfo + .capabilities() + .contains( + BeamUrns.getUrn( + RunnerApi.StandardProtocols.Enum.MULTI_CORE_BUNDLE_PROCESSING))) { image.setUseSingleCorePerContainer(true); } image.setCapabilities(environmentInfo.capabilities()); @@ -1702,7 +1755,7 @@ void maybeRecordPCollectionWithAutoSharding(PCollection pcol) { options.isEnableStreamingEngine(), "Runner determined sharding not available in Dataflow for GroupIntoBatches for" + " non-Streaming-Engine jobs. In order to use runner determined sharding, please use" - + " --streaming --enable_streaming_engine"); + + " --streaming --experiments=enable_streaming_engine"); pCollectionsPreservedKeys.add(pcol); pcollectionsRequiringAutoSharding.add(pcol); } @@ -2446,10 +2499,10 @@ static boolean useStreamingEngine(DataflowPipelineOptions options) { static void verifyDoFnSupported( DoFn fn, boolean streaming, DataflowPipelineOptions options) { - if (DoFnSignatures.usesMultimapState(fn)) { + if (!streaming && DoFnSignatures.usesMultimapState(fn)) { throw new UnsupportedOperationException( String.format( - "%s does not currently support %s", + "%s does not currently support %s in batch mode", DataflowRunner.class.getSimpleName(), MultimapState.class.getSimpleName())); } if (streaming && DoFnSignatures.requiresTimeSortedInput(fn)) { @@ -2461,6 +2514,13 @@ static void verifyDoFnSupported( boolean streamingEngine = useStreamingEngine(options); boolean isUnifiedWorker = useUnifiedWorker(options); + + if (DoFnSignatures.usesMultimapState(fn) && isUnifiedWorker) { + throw new UnsupportedOperationException( + String.format( + "%s does not currently support %s running using streaming on unified worker", + DataflowRunner.class.getSimpleName(), MultimapState.class.getSimpleName())); + } if (DoFnSignatures.usesSetState(fn)) { if (streaming && (isUnifiedWorker || streamingEngine)) { throw new UnsupportedOperationException( @@ -2512,6 +2572,12 @@ public String getUrn(PTransform transform) { return "dataflow_stub:" + transform.getClass().getName(); } + @Override + public String getUrn() { + throw new UnsupportedOperationException( + "URN of DataflowPayloadTranslator depends on the transform. Please use 'getUrn(PTransform transform)' instead."); + } + @Override public RunnerApi.FunctionSpec translate( AppliedPTransform> application, SdkComponents components) diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunnerInfo.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunnerInfo.java index 911a8935a6ba8..70cdc46552f18 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunnerInfo.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunnerInfo.java @@ -17,14 +17,14 @@ */ package org.apache.beam.runners.dataflow; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.io.InputStream; import java.util.Map; import java.util.Properties; import org.apache.beam.sdk.util.ReleaseInfo; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/GroupIntoBatchesOverride.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/GroupIntoBatchesOverride.java index 101e674bb48f0..b7ba74c2cb8a7 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/GroupIntoBatchesOverride.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/GroupIntoBatchesOverride.java @@ -40,7 +40,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; @SuppressWarnings({ "rawtypes" // TODO(https://github.com/apache/beam/issues/20447) diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.java index 732de76305951..140858d88c04f 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.java @@ -23,7 +23,7 @@ import static org.apache.beam.runners.core.construction.ParDoTranslation.translateTimerFamilySpec; import static org.apache.beam.sdk.transforms.reflect.DoFnSignatures.getStateSpecOrThrow; import static org.apache.beam.sdk.transforms.reflect.DoFnSignatures.getTimerSpecOrThrow; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; import java.io.IOException; @@ -66,8 +66,8 @@ import org.apache.beam.sdk.values.PCollectionViews; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; /** * A {@link PTransformOverrideFactory} that produces {@link ParDoSingle} instances from {@link @@ -157,7 +157,7 @@ public static PTransformTranslation.TransformPayloadTranslator create() { private PayloadTranslator() {} @Override - public String getUrn(ParDoSingle transform) { + public String getUrn() { return PAR_DO_TRANSFORM_URN; } diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/StreamingViewOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/StreamingViewOverrides.java index 93fdc51e1d19d..633c0ec08c999 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/StreamingViewOverrides.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/StreamingViewOverrides.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.PCollectionViews; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; /** * Dataflow streaming overrides for {@link CreatePCollectionView}, specialized for different view diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TestDataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TestDataflowRunner.java index 2a32c86a519d5..efb725f6a67bd 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TestDataflowRunner.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TestDataflowRunner.java @@ -37,10 +37,10 @@ import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.testing.TestPipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.slf4j.Logger; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java index ff0a1a0dfaf96..e68dde045763c 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java @@ -21,7 +21,7 @@ import static org.apache.beam.runners.dataflow.util.Structs.addString; import static org.apache.beam.runners.dataflow.util.Structs.addStringList; import static org.apache.beam.sdk.util.SerializableUtils.serializeToByteArray; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.services.dataflow.model.SourceMetadata; import java.util.ArrayList; @@ -32,7 +32,7 @@ import org.apache.beam.sdk.io.Source; import org.apache.beam.sdk.io.UnboundedSource; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java index 5db857a5423fa..ce3861f26c2f8 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java @@ -17,9 +17,9 @@ */ package org.apache.beam.runners.dataflow.internal; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.io.DataInputStream; @@ -43,9 +43,9 @@ import org.apache.beam.sdk.coders.VarLongCoder; import org.apache.beam.sdk.util.VarInt; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.HashFunction; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java index c4a2941f24d42..709009f8c8069 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java @@ -28,6 +28,7 @@ import org.apache.beam.sdk.options.Description; import org.apache.beam.sdk.options.ExperimentalOptions; import org.apache.beam.sdk.options.Hidden; +import org.apache.beam.sdk.options.MemoryMonitorOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.InstanceBuilder; @@ -39,7 +40,8 @@ "[Internal] Options used to control execution of the Dataflow SDK for " + "debugging and testing purposes.") @Hidden -public interface DataflowPipelineDebugOptions extends ExperimentalOptions, PipelineOptions { +public interface DataflowPipelineDebugOptions + extends ExperimentalOptions, MemoryMonitorOptions, PipelineOptions { /** * The root URL for the Dataflow API. {@code dataflowEndpoint} can override this value if it @@ -234,22 +236,6 @@ public Dataflow create(PipelineOptions options) { void setJfrRecordingDurationSec(int value); - /** - * The GC thrashing threshold percentage. A given period of time is considered "thrashing" if this - * percentage of CPU time is spent in garbage collection. Dataflow will force fail tasks after - * sustained periods of thrashing. - * - *

    If {@literal 100} is given as the value, MemoryMonitor will be disabled. - */ - @Description( - "The GC thrashing threshold percentage. A given period of time is considered \"thrashing\" if this " - + "percentage of CPU time is spent in garbage collection. Dataflow will force fail tasks after " - + "sustained periods of thrashing.") - @Default.Double(50.0) - Double getGCThrashingPercentagePerPeriod(); - - void setGCThrashingPercentagePerPeriod(Double value); - /** * The size of the worker's in-memory cache, in megabytes. * diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java index f87af28ca61b4..985e1736dcb03 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java @@ -17,6 +17,7 @@ */ package org.apache.beam.runners.dataflow.options; +import com.google.api.services.dataflow.Dataflow; import java.util.List; import java.util.Map; import org.apache.beam.runners.dataflow.DataflowRunner; @@ -136,6 +137,25 @@ public interface DataflowPipelineOptions void setRegion(String region); + /** + * Dataflow endpoint to use. + * + *

    Defaults to the current version of the Google Cloud Dataflow API, at the time the current + * SDK version was released. + * + *

    If the string contains "://", then this is treated as a URL, otherwise {@link + * #getApiRootUrl()} is used as the root URL. + */ + @Description( + "The URL for the Dataflow API. If the string contains \"://\", this" + + " will be treated as the entire URL, otherwise will be treated relative to apiRootUrl.") + @Override + @Default.String(Dataflow.DEFAULT_SERVICE_PATH) + String getDataflowEndpoint(); + + @Override + void setDataflowEndpoint(String value); + /** Labels that will be applied to the billing records for this job. */ @Description("Labels that will be applied to the billing records for this job.") Map getLabels(); diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowWorkerLoggingOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowWorkerLoggingOptions.java index c2fde4857d8c3..5322539b80e67 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowWorkerLoggingOptions.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowWorkerLoggingOptions.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.options; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.fasterxml.jackson.annotation.JsonCreator; import java.util.Arrays; @@ -32,8 +32,8 @@ * * @deprecated This interface will no longer be the source of truth for worker logging configuration * once jobs are executed using a dedicated SDK harness instead of user code being co-located - * alongside Dataflow worker code. Please set the option below and also the corresponding option - * within {@link org.apache.beam.sdk.options.SdkHarnessOptions} to ensure forward compatibility. + * alongside Dataflow worker code. Consider set corresponding options within {@link + * org.apache.beam.sdk.options.SdkHarnessOptions} to ensure forward compatibility. */ @Description("Options that are used to control logging configuration on the Dataflow worker.") @Deprecated diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DefaultGcpRegionFactory.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DefaultGcpRegionFactory.java index a3f1011275b8c..8c88b478fb210 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DefaultGcpRegionFactory.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DefaultGcpRegionFactory.java @@ -27,8 +27,8 @@ import java.util.stream.Collectors; import org.apache.beam.sdk.options.DefaultValueFactory; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudKnownType.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudKnownType.java index 413a5ab51d0a9..807c8fc8855b1 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudKnownType.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudKnownType.java @@ -20,7 +20,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; /** A utility for manipulating well-known cloud types. */ diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObject.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObject.java index 4898062d14fe3..fa88c32c9c097 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObject.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObject.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.client.json.GenericJson; import com.google.api.client.util.Key; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObjectTranslators.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObjectTranslators.java index c6029662e9ff2..d158b8d5a7a52 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObjectTranslators.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObjectTranslators.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.ArrayList; import java.util.Collections; @@ -45,8 +45,9 @@ import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.util.StringUtils; import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder; +import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Utilities for creating {@link CloudObjectTranslator} instances for {@link Coder Coders}. */ @SuppressWarnings({ @@ -501,6 +502,39 @@ public String cloudObjectClassName() { }; } + public static CloudObjectTranslator timestampedValue() { + return new CloudObjectTranslator() { + @Override + public CloudObject toCloudObject( + TimestampedValue.TimestampedValueCoder target, SdkComponents sdkComponents) { + CloudObject base = CloudObject.forClass(TimestampedValue.TimestampedValueCoder.class); + return addComponents( + base, ImmutableList.>of(target.getValueCoder()), sdkComponents); + } + + @Override + public TimestampedValue.TimestampedValueCoder fromCloudObject(CloudObject cloudObject) { + List> components = getComponents(cloudObject); + checkArgument( + components.size() == 1, + "Expected 1 components for %s, got %s", + TimestampedValue.TimestampedValueCoder.class.getSimpleName(), + components.size()); + return TimestampedValue.TimestampedValueCoder.of(components.get(0)); + } + + @Override + public Class getSupportedClass() { + return TimestampedValue.TimestampedValueCoder.class; + } + + @Override + public String cloudObjectClassName() { + return CloudObject.forClass(TimestampedValue.TimestampedValueCoder.class).getClassName(); + } + }; + } + public static CloudObjectTranslator nullable() { return new CloudObjectTranslator() { @Override diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObjects.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObjects.java index 214f06785fd45..1376f21cd15db 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObjects.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/CloudObjects.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.dataflow.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.Map; import java.util.ServiceLoader; @@ -36,8 +36,8 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.transforms.windowing.IntervalWindow.IntervalWindowCoder; import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; /** Utilities for converting an object to a {@link CloudObject}. */ diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DataflowTemplateJob.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DataflowTemplateJob.java index bac54d088381f..d8ecdb9d42a7f 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DataflowTemplateJob.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DataflowTemplateJob.java @@ -21,7 +21,7 @@ import com.google.api.client.util.Sleeper; import org.apache.beam.runners.dataflow.DataflowPipelineJob; import org.apache.beam.sdk.metrics.MetricResults; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DataflowTransport.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DataflowTransport.java index e76bb13492905..ad579333ebf4f 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DataflowTransport.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DataflowTransport.java @@ -30,7 +30,7 @@ import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.sdk.extensions.gcp.auth.NullCredentialInitializer; import org.apache.beam.sdk.extensions.gcp.util.RetryHttpRequestInitializer; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Helpers for cloud communication. */ public class DataflowTransport { diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DefaultCoderCloudObjectTranslatorRegistrar.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DefaultCoderCloudObjectTranslatorRegistrar.java index d5dbb6bcbd944..2d83bd77526eb 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DefaultCoderCloudObjectTranslatorRegistrar.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DefaultCoderCloudObjectTranslatorRegistrar.java @@ -46,10 +46,10 @@ import org.apache.beam.sdk.io.gcp.bigquery.TableDestinationCoderV2; import org.apache.beam.sdk.io.gcp.bigquery.TableDestinationCoderV3; import org.apache.beam.sdk.io.gcp.bigquery.TableRowJsonCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; /** * The {@link CoderCloudObjectTranslatorRegistrar} containing the default collection of {@link @@ -80,6 +80,7 @@ public class DefaultCoderCloudObjectTranslatorRegistrar CloudObjectTranslators.iterableLike(ListCoder.class), CloudObjectTranslators.iterableLike(SetCoder.class), CloudObjectTranslators.map(), + CloudObjectTranslators.timestampedValue(), CloudObjectTranslators.nullable(), CloudObjectTranslators.union(), CloudObjectTranslators.coGroupByKeyResult(), diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/GcsStager.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/GcsStager.java index d1045c467f016..bf34e007c40cc 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/GcsStager.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/GcsStager.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.dataflow.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.services.dataflow.model.DataflowPackage; import java.util.List; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java index 4122262c704c6..a7dd5155b663c 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java @@ -33,7 +33,7 @@ import org.apache.beam.runners.dataflow.DataflowClient; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.sdk.PipelineResult.State; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; import org.slf4j.Logger; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/OutputReference.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/OutputReference.java index 21732e6cc0cc1..024fbc19c4609 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/OutputReference.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/OutputReference.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.client.json.GenericJson; import com.google.api.client.util.Key; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PackageUtil.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PackageUtil.java index 1f59b72341162..df6a3255f5b6f 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PackageUtil.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/PackageUtil.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.dataflow.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.client.util.BackOff; import com.google.api.client.util.Sleeper; @@ -52,12 +52,12 @@ import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.MimeTypes; import org.apache.beam.sdk.util.MoreFutures; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashCode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteSource; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.HashCode; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteSource; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java index 09a4d4b1d427b..a757f8c334cad 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.dataflow.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -34,9 +34,9 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.CoderException; import org.apache.beam.sdk.util.VarInt; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.UnsignedBytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedBytes; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RowCoderCloudObjectTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RowCoderCloudObjectTranslator.java index 19afdacfc9397..e6b1a053b75d9 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RowCoderCloudObjectTranslator.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RowCoderCloudObjectTranslator.java @@ -18,8 +18,6 @@ package org.apache.beam.runners.dataflow.util; import java.io.IOException; -import java.util.UUID; -import javax.annotation.Nullable; import org.apache.beam.model.pipeline.v1.SchemaApi; import org.apache.beam.runners.core.construction.SdkComponents; import org.apache.beam.sdk.coders.RowCoder; @@ -58,10 +56,7 @@ public RowCoder fromCloudObject(CloudObject cloudObject) { SchemaApi.Schema.Builder schemaBuilder = SchemaApi.Schema.newBuilder(); JsonFormat.parser().merge(Structs.getString(cloudObject, SCHEMA), schemaBuilder); Schema schema = SchemaTranslation.schemaFromProto(schemaBuilder.build()); - @Nullable UUID uuid = schema.getUUID(); - if (schema.isEncodingPositionsOverridden() && uuid != null) { - RowCoder.overrideEncodingPositions(uuid, schema.getEncodingPositions()); - } + SchemaCoderCloudObjectTranslator.overrideEncodingPositions(schema); return RowCoder.of(schema); } catch (IOException e) { throw new RuntimeException(e); diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/SchemaCoderCloudObjectTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/SchemaCoderCloudObjectTranslator.java index 8abbc05f65e2f..afe38019e1439 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/SchemaCoderCloudObjectTranslator.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/SchemaCoderCloudObjectTranslator.java @@ -22,10 +22,12 @@ import javax.annotation.Nullable; import org.apache.beam.model.pipeline.v1.SchemaApi; import org.apache.beam.runners.core.construction.SdkComponents; +import org.apache.beam.sdk.coders.RowCoder; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.SchemaCoder; import org.apache.beam.sdk.schemas.SchemaTranslation; import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.beam.sdk.util.Preconditions; import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.util.StringUtils; import org.apache.beam.sdk.values.TypeDescriptor; @@ -100,16 +102,50 @@ public SchemaCoder fromCloudObject(CloudObject cloudObject) { SchemaApi.Schema.Builder schemaBuilder = SchemaApi.Schema.newBuilder(); JsonFormat.parser().merge(Structs.getString(cloudObject, SCHEMA), schemaBuilder); Schema schema = SchemaTranslation.schemaFromProto(schemaBuilder.build()); - @Nullable UUID uuid = schema.getUUID(); - if (schema.isEncodingPositionsOverridden() && uuid != null) { - SchemaCoder.overrideEncodingPositions(uuid, schema.getEncodingPositions()); - } + overrideEncodingPositions(schema); return SchemaCoder.of(schema, typeDescriptor, toRowFunction, fromRowFunction); } catch (IOException e) { throw new RuntimeException(e); } } + static void overrideEncodingPositions(Schema schema) { + @Nullable UUID uuid = schema.getUUID(); + if (schema.isEncodingPositionsOverridden() && uuid != null) { + RowCoder.overrideEncodingPositions(uuid, schema.getEncodingPositions()); + } + schema.getFields().stream() + .map(Schema.Field::getType) + .forEach(SchemaCoderCloudObjectTranslator::overrideEncodingPositions); + } + + private static void overrideEncodingPositions(Schema.FieldType fieldType) { + switch (fieldType.getTypeName()) { + case ROW: + overrideEncodingPositions(Preconditions.checkArgumentNotNull(fieldType.getRowSchema())); + break; + case ARRAY: + case ITERABLE: + overrideEncodingPositions( + Preconditions.checkArgumentNotNull(fieldType.getCollectionElementType())); + break; + case MAP: + overrideEncodingPositions(Preconditions.checkArgumentNotNull(fieldType.getMapKeyType())); + overrideEncodingPositions(Preconditions.checkArgumentNotNull(fieldType.getMapValueType())); + break; + case LOGICAL_TYPE: + Schema.LogicalType logicalType = + Preconditions.checkArgumentNotNull(fieldType.getLogicalType()); + @Nullable Schema.FieldType argumentType = logicalType.getArgumentType(); + if (argumentType != null) { + overrideEncodingPositions(argumentType); + } + overrideEncodingPositions(logicalType.getBaseType()); + break; + default: + } + } + @Override public Class getSupportedClass() { return SchemaCoder.class; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/SerializableCoderCloudObjectTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/SerializableCoderCloudObjectTranslator.java index a981f65fc2bcd..be902caf1f422 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/SerializableCoderCloudObjectTranslator.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/SerializableCoderCloudObjectTranslator.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.Serializable; import org.apache.beam.runners.core.construction.SdkComponents; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/TimeUtil.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/TimeUtil.java index 94dd4e0813f4d..4086b69c9574b 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/TimeUtil.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/TimeUtil.java @@ -19,7 +19,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTime; import org.joda.time.Duration; diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverridesTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverridesTest.java index de326f71fb73b..cc6a4a7eac57e 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverridesTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/BatchStatefulParDoOverridesTest.java @@ -50,7 +50,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Ignore; import org.junit.Test; diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/BatchViewOverridesTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/BatchViewOverridesTest.java index 9ce81bf5764f1..72d6786c02d02 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/BatchViewOverridesTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/BatchViewOverridesTest.java @@ -42,8 +42,8 @@ import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Instant; import org.junit.Rule; import org.junit.Test; diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java index c9247a8c36f92..527273abb42ed 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowMetricsTest.java @@ -50,10 +50,10 @@ import org.apache.beam.sdk.metrics.MetricsFilter; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.runners.AppliedPTransform; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashBiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java index b8bc272280970..eb8014982a30e 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java @@ -51,8 +51,8 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.testing.ExpectedLogs; import org.apache.beam.sdk.util.FastNanoClockAndSleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Before; diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrarTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrarTest.java index ac80ec6979238..c91ad665cd06e 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrarTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrarTest.java @@ -24,8 +24,8 @@ import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; import org.apache.beam.sdk.runners.PipelineRunnerRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java index 27b98a9c7591f..e5fb13875cded 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java @@ -19,7 +19,7 @@ import static org.apache.beam.runners.dataflow.util.Structs.getString; import static org.apache.beam.sdk.util.StringUtils.jsonStringToByteArray; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; @@ -123,10 +123,10 @@ import org.apache.beam.sdk.values.TypeDescriptors; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.hamcrest.Matchers; import org.joda.time.Duration; import org.junit.Assert; @@ -723,7 +723,7 @@ public void testTaggedNamesOverridden() throws Exception { PCollectionTuple outputs = pipeline - .apply(Create.of(3)) + .apply(Create.of(3, 4)) .apply( ParDo.of( new DoFn() { @@ -775,7 +775,7 @@ public void testBatchStatefulParDoTranslation() throws Exception { TupleTag mainOutputTag = new TupleTag() {}; pipeline - .apply(Create.of(KV.of(1, 1))) + .apply(Create.of(KV.of(1, 1), KV.of(2, 3))) .apply( ParDo.of( new DoFn, Integer>() { @@ -917,7 +917,7 @@ public void processElement(ProcessContext c) { // No need to actually check the pipeline as the ValidatesRunner tests // ensure translation is correct. This is just a quick check to see that translation // does not crash. - assertEquals(24, steps.size()); + assertEquals(25, steps.size()); } /** Smoke test to fail fast if translation of a splittable ParDo in streaming breaks. */ @@ -1057,7 +1057,7 @@ public void testToSingletonTranslationWithIsmSideInput() throws Exception { assertAllStepOutputsHaveUniqueIds(job); List steps = job.getSteps(); - assertEquals(9, steps.size()); + assertEquals(10, steps.size()); @SuppressWarnings("unchecked") List> toIsmRecordOutputs = diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java index 84f1069f59218..078f25e0e38e8 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.dataflow; import static org.apache.beam.runners.dataflow.DataflowRunner.getContainerImageForJob; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files.getFileExtension; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files.getFileExtension; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; @@ -171,9 +171,9 @@ import org.apache.beam.sdk.values.TypeDescriptors; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; @@ -1632,7 +1632,7 @@ public PCollection expand(PCollection input) { private static class TestTransformTranslator implements TransformPayloadTranslator { @Override - public String getUrn(TestTransform transform) { + public String getUrn() { return "test_transform"; } diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactoryTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactoryTest.java index c09b2233d2feb..d9736fcbe3140 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactoryTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactoryTest.java @@ -39,7 +39,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.PValues; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Rule; import org.junit.Test; diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java index 5e86fd3e6637d..ff48656da50bd 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/TestDataflowRunnerTest.java @@ -56,9 +56,9 @@ import org.apache.beam.sdk.testing.TestPipelineOptions; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.joda.time.Duration; diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java index 8c19ab955a354..25c939fce6be4 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.testing.ResetDateTimeProvider; import org.apache.beam.sdk.testing.RestoreSystemProperties; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowWorkerLoggingOptionsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowWorkerLoggingOptionsTest.java index 1fa5b143f7195..3d5d0e16890d8 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowWorkerLoggingOptionsTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowWorkerLoggingOptionsTest.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.beam.runners.dataflow.options.DataflowWorkerLoggingOptions.WorkerLogLevelOverrides; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/CloudObjectTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/CloudObjectTest.java index 118bab8a1eed8..1044f683ba281 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/CloudObjectTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/CloudObjectTest.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.dataflow.util; import com.google.common.testing.EqualsTester; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/CloudObjectsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/CloudObjectsTest.java index 7bacd2a0ac25f..f49a288ac70f4 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/CloudObjectsTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/CloudObjectsTest.java @@ -67,10 +67,11 @@ import org.apache.beam.sdk.util.InstanceBuilder; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Test; import org.junit.experimental.runners.Enclosed; @@ -167,6 +168,7 @@ public static Iterable> data() { .add(SetCoder.of(VarLongCoder.of())) .add(MapCoder.of(VarLongCoder.of(), ByteArrayCoder.of())) .add(NullableCoder.of(IntervalWindow.getCoder())) + .add(TimestampedValue.TimestampedValueCoder.of(VarLongCoder.of())) .add( UnionCoder.of( ImmutableList.of( diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/GCSUploadMain.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/GCSUploadMain.java index 468ec958334e7..e940d40f0a8e1 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/GCSUploadMain.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/GCSUploadMain.java @@ -25,9 +25,9 @@ import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.sdk.io.FileSystems; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashCode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.HashCode; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; /** Standalone program to upload files to GCS, for testing in isolation. */ public class GCSUploadMain { diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java index df54652cc61bc..5364b8d52b428 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java @@ -81,14 +81,14 @@ import org.apache.beam.sdk.util.FastNanoClockAndSleeper; import org.apache.beam.sdk.util.MimeTypes; import org.apache.beam.sdk.util.ZipFiles; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashCode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.LineReader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.HashCode; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.LineReader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matchers; import org.junit.After; @@ -131,7 +131,7 @@ public void teardown() { private File makeFileWithContents(String name, String contents) throws Exception { File tmpFile = tmpFolder.newFile(name); - Files.write(contents, tmpFile, StandardCharsets.UTF_8); + Files.asCharSink(tmpFile, StandardCharsets.UTF_8).write(contents); assertTrue(tmpFile.setLastModified(0)); // required for determinism with directories return tmpFile; } diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/RandomAccessDataTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/RandomAccessDataTest.java index cf0807680679f..2051c484aed98 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/RandomAccessDataTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/RandomAccessDataTest.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.coders.Coder.Context; import org.apache.beam.sdk.coders.CoderException; import org.apache.beam.sdk.testing.CoderProperties; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.UnsignedBytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedBytes; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/runners/google-cloud-dataflow-java/worker/build.gradle b/runners/google-cloud-dataflow-java/worker/build.gradle index 332d7cdef2a71..ce06063c9b52d 100644 --- a/runners/google-cloud-dataflow-java/worker/build.gradle +++ b/runners/google-cloud-dataflow-java/worker/build.gradle @@ -67,90 +67,91 @@ def excluded_dependencies = [ library.java.error_prone_annotations, // Provided scope added in worker library.java.hamcrest, // Test only library.java.junit, // Test only - library.java.jsonassert // Test only + library.java.jsonassert, // Test only + library.java.truth // Test only ] applyJavaNature( automaticModuleName: 'org.apache.beam.runners.dataflow.worker', archivesBaseName: 'beam-runners-google-cloud-dataflow-java-legacy-worker', classesTriggerCheckerBugs: [ - 'BatchGroupAlsoByWindowAndCombineFn': 'TODO: file a bug report', - 'AssignWindowsParDoFnFactory': 'TODO: file a bug report', - 'FetchAndFilterStreamingSideInputsOperation': 'https://github.com/typetools/checker-framework/issues/5436', + 'BatchGroupAlsoByWindowAndCombineFn' : 'TODO: file a bug report', + 'AssignWindowsParDoFnFactory' : 'TODO: file a bug report', + 'FetchAndFilterStreamingSideInputsOperation': 'https://github.com/typetools/checker-framework/issues/5436', ], exportJavadoc: false, enableSpotbugs: false /* TODO(BEAM-5658): enable spotbugs */, shadowJarValidationExcludes: [ - "org/apache/beam/runners/dataflow/worker/**", - "org/apache/beam/repackaged/beam_runners_google_cloud_dataflow_java_legacy_worker/**", - // TODO(https://github.com/apache/beam/issues/19114): Move DataflowRunnerHarness class under org.apache.beam.runners.dataflow.worker namespace - "com/google/cloud/dataflow/worker/DataflowRunnerHarness.class", - // Allow slf4j implementation worker for logging during pipeline execution - "org/slf4j/impl/**" + "org/apache/beam/runners/dataflow/worker/**", + "org/apache/beam/repackaged/beam_runners_google_cloud_dataflow_java_legacy_worker/**", + // TODO(https://github.com/apache/beam/issues/19114): Move DataflowRunnerHarness class under org.apache.beam.runners.dataflow.worker namespace + "com/google/cloud/dataflow/worker/DataflowRunnerHarness.class", + // Allow slf4j implementation worker for logging during pipeline execution + "org/slf4j/impl/**" ], shadowClosure: { - // Each included dependency must also include all of its necessary transitive dependencies - // or have them provided by the users pipeline during job submission. Typically a users - // pipeline includes :runners:google-cloud-dataflow-java and its transitive dependencies - // so those dependencies don't need to be shaded (bundled and relocated) away. All other - // dependencies needed to run the worker must be shaded (bundled and relocated) to prevent - // ClassNotFound and/or MethodNotFound errors during pipeline execution. - // - // Each included dependency should have a matching relocation rule below that ensures - // that the shaded jar is correctly built. + // Each included dependency must also include all of its necessary transitive dependencies + // or have them provided by the users pipeline during job submission. Typically a users + // pipeline includes :runners:google-cloud-dataflow-java and its transitive dependencies + // so those dependencies don't need to be shaded (bundled and relocated) away. All other + // dependencies needed to run the worker must be shaded (bundled and relocated) to prevent + // ClassNotFound and/or MethodNotFound errors during pipeline execution. + // + // Each included dependency should have a matching relocation rule below that ensures + // that the shaded jar is correctly built. - dependencies { - include(dependency(library.java.slf4j_jdk14)) - } + dependencies { + include(dependency(library.java.slf4j_jdk14)) + } - dependencies { - include(project(path: ":model:fn-execution", configuration: "shadow")) - } - relocate("org.apache.beam.model.fnexecution.v1", getWorkerRelocatedPath("org.apache.beam.model.fnexecution.v1")) + dependencies { + include(project(path: ":model:fn-execution", configuration: "shadow")) + } + relocate("org.apache.beam.model.fnexecution.v1", getWorkerRelocatedPath("org.apache.beam.model.fnexecution.v1")) - dependencies { - include(project(":runners:core-construction-java")) - include(project(":runners:core-java")) - } - relocate("org.apache.beam.runners.core", getWorkerRelocatedPath("org.apache.beam.runners.core")) - relocate("org.apache.beam.repackaged.beam_runners_core_construction_java", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_runners_core_construction_java")) - relocate("org.apache.beam.repackaged.beam_runners_core_java", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_runners_core_java")) + dependencies { + include(project(":runners:core-construction-java")) + include(project(":runners:core-java")) + } + relocate("org.apache.beam.runners.core", getWorkerRelocatedPath("org.apache.beam.runners.core")) + relocate("org.apache.beam.repackaged.beam_runners_core_construction_java", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_runners_core_construction_java")) + relocate("org.apache.beam.repackaged.beam_runners_core_java", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_runners_core_java")) - dependencies { - include(project(":runners:java-fn-execution")) - } - relocate("org.apache.beam.runners.fnexecution", getWorkerRelocatedPath("org.apache.beam.runners.fnexecution")) - relocate("org.apache.beam.repackaged.beam_runners_java_fn_execution", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_runners_java_fn_execution")) + dependencies { + include(project(":runners:java-fn-execution")) + } + relocate("org.apache.beam.runners.fnexecution", getWorkerRelocatedPath("org.apache.beam.runners.fnexecution")) + relocate("org.apache.beam.repackaged.beam_runners_java_fn_execution", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_runners_java_fn_execution")) - dependencies { - include(project(":sdks:java:fn-execution")) - } - relocate("org.apache.beam.sdk.fn", getWorkerRelocatedPath("org.apache.beam.sdk.fn")) - relocate("org.apache.beam.repackaged.beam_sdks_java_fn_execution", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_sdks_java_fn_execution")) + dependencies { + include(project(":sdks:java:fn-execution")) + } + relocate("org.apache.beam.sdk.fn", getWorkerRelocatedPath("org.apache.beam.sdk.fn")) + relocate("org.apache.beam.repackaged.beam_sdks_java_fn_execution", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_sdks_java_fn_execution")) - dependencies { - // We have to include jetty-server/jetty-servlet and all of its transitive dependencies - // which includes several org.eclipse.jetty artifacts + servlet-api - include(dependency("org.eclipse.jetty:.*:9.2.10.v20150310")) - include(dependency("javax.servlet:javax.servlet-api:3.1.0")) - } - relocate("org.eclipse.jetty", getWorkerRelocatedPath("org.eclipse.jetty")) - relocate("javax.servlet", getWorkerRelocatedPath("javax.servlet")) + dependencies { + // We have to include jetty-server/jetty-servlet and all of its transitive dependencies + // which includes several org.eclipse.jetty artifacts + servlet-api + include(dependency("org.eclipse.jetty:.*:9.2.10.v20150310")) + include(dependency("javax.servlet:javax.servlet-api:3.1.0")) + } + relocate("org.eclipse.jetty", getWorkerRelocatedPath("org.eclipse.jetty")) + relocate("javax.servlet", getWorkerRelocatedPath("javax.servlet")) - // We don't relocate windmill since it is already underneath the org.apache.beam.runners.dataflow.worker namespace and never - // expect a user pipeline to include it. There is also a JNI component that windmill server relies on which makes - // arbitrary relocation more difficult. - dependencies { - include(project(path: ":runners:google-cloud-dataflow-java:worker:windmill", configuration: "shadow")) - } + // We don't relocate windmill since it is already underneath the org.apache.beam.runners.dataflow.worker namespace and never + // expect a user pipeline to include it. There is also a JNI component that windmill server relies on which makes + // arbitrary relocation more difficult. + dependencies { + include(project(path: ":runners:google-cloud-dataflow-java:worker:windmill", configuration: "shadow")) + } - // Include original source files extracted under - // '$buildDir/original_sources_to_package' to jar - from "$buildDir/original_sources_to_package" + // Include original source files extracted under + // '$buildDir/original_sources_to_package' to jar + from "$buildDir/original_sources_to_package" - exclude "META-INF/LICENSE.txt" - exclude "about.html" -}) + exclude "META-INF/LICENSE.txt" + exclude "about.html" + }) /******************************************************************************/ // Configure the worker root project @@ -191,7 +192,7 @@ dependencies { implementation project(":runners:java-fn-execution") implementation project(":sdks:java:fn-execution") implementation project(path: ":runners:google-cloud-dataflow-java:worker:windmill", configuration: "shadow") - shadow library.java.vendored_guava_26_0_jre + shadow library.java.vendored_guava_32_1_2_jre implementation library.java.google_auth_library_credentials implementation library.java.proto_google_common_protos @@ -219,6 +220,10 @@ dependencies { // as well and placed within the testImplementation configuration. Otherwise we can place it within // the shadowTest configuration. testImplementation project(path: ":runners:core-java", configuration: "testRuntimeMigration") + // TODO: excluding Guava until Truth updates it to >32.1.x + testImplementation(library.java.truth) { + exclude group: 'com.google.guava', module: 'guava' + } shadowTest project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "testRuntimeMigration") shadowTest project(path: ":runners:direct-java", configuration: "shadow") shadowTest project(path: ":sdks:java:harness", configuration: "shadowTest") @@ -232,8 +237,8 @@ dependencies { project.task('validateShadedJarContainsSlf4jJdk14', dependsOn: 'shadowJar') { ext.outFile = project.file("${project.reportsDir}/${name}.out") inputs.files(project.configurations.shadow.artifacts.files) - .withPropertyName("shadowArtifactsFiles") - .withPathSensitivity(PathSensitivity.RELATIVE) + .withPropertyName("shadowArtifactsFiles") + .withPathSensitivity(PathSensitivity.RELATIVE) outputs.files outFile doLast { project.configurations.shadow.artifacts.files.each { diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ApplianceShuffleEntryReader.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ApplianceShuffleEntryReader.java index abcf3c3b99068..9822c27074aa6 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ApplianceShuffleEntryReader.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ApplianceShuffleEntryReader.java @@ -24,7 +24,7 @@ import org.apache.beam.runners.dataflow.worker.util.common.worker.ShuffleEntryReader; import org.apache.beam.runners.dataflow.worker.util.common.worker.ShufflePosition; import org.apache.beam.sdk.util.common.Reiterator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.checkerframework.checker.nullness.qual.Nullable; /** An implementation of ShuffleEntryReader that uses ApplianceShuffleReader. */ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AssignWindowsParDoFnFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AssignWindowsParDoFnFactory.java index b4943ba77491d..7697271792a8e 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AssignWindowsParDoFnFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AssignWindowsParDoFnFactory.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.dataflow.worker; import static org.apache.beam.runners.dataflow.util.Structs.getBytes; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.services.dataflow.model.SideInputInfo; import java.util.Collection; @@ -34,7 +34,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AvroByteReader.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AvroByteReader.java index 4ff7cc6c2bcee..c20373afe3bd6 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AvroByteReader.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AvroByteReader.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.services.dataflow.model.ApproximateReportedProgress; import com.google.api.services.dataflow.model.ApproximateSplitRequest; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AvroByteReaderFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AvroByteReaderFactory.java index 938bf3f83936c..fca0e833bfef8 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AvroByteReaderFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AvroByteReaderFactory.java @@ -28,7 +28,7 @@ import org.apache.beam.runners.dataflow.worker.util.common.worker.NativeReader; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** Creates an {@link AvroByteReader} from a CloudObject spec. */ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AvroByteSink.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AvroByteSink.java index 9e0a2e98c0a6a..7d47d85bded94 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AvroByteSink.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AvroByteSink.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.io.fs.ResourceId; import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.util.MimeTypes; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** * A sink that writes Avro files. Records are written to the Avro file as a series of byte arrays. diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AvroByteSinkFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AvroByteSinkFactory.java index bece350c3cddf..eee801d3c35c8 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AvroByteSinkFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/AvroByteSinkFactory.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.io.FileSystems; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** Creates an {@link AvroByteSink} from a {@link CloudObject} spec. */ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/BatchDataflowWorker.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/BatchDataflowWorker.java index 00b6089dde5bb..7407c97619b4d 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/BatchDataflowWorker.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/BatchDataflowWorker.java @@ -40,11 +40,11 @@ import org.apache.beam.sdk.fn.IdGenerators; import org.apache.beam.sdk.util.Weighted; import org.apache.beam.sdk.util.WeightedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.MutableNetwork; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.MutableNetwork; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,22 +60,8 @@ }) public class BatchDataflowWorker implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(BatchDataflowWorker.class); - - /** A client to get and update work items. */ - private final WorkUnitClient workUnitClient; - - /** - * Pipeline options, initially provided via the constructor and partially provided via each work - * work unit. - */ - private final DataflowWorkerHarnessOptions options; - - /** The factory to create {@link DataflowMapTaskExecutor DataflowMapTaskExecutors}. */ - private final DataflowMapTaskExecutorFactory mapTaskExecutorFactory; - /** The idGenerator to generate unique id globally. */ private static final IdGenerator idGenerator = IdGenerators.decrementingLongs(); - /** * Function which converts map tasks to their network representation for execution. * @@ -90,30 +76,7 @@ public class BatchDataflowWorker implements Closeable { new FixMultiOutputInfosOnParDoInstructions(idGenerator) .andThen(new MapTaskToNetworkFunction(idGenerator)); - /** Registry of known {@link ReaderFactory ReaderFactories}. */ - private final ReaderRegistry readerRegistry = ReaderRegistry.defaultRegistry(); - - /** Registry of known {@link SinkFactory SinkFactories}. */ - private final SinkRegistry sinkRegistry = SinkRegistry.defaultRegistry(); - - /** A side input cache shared between all execution contexts. */ - private final Cache> sideInputDataCache; - - /** - * A side input cache shared between all execution contexts. This cache is meant to store values - * as weak references. This allows for insertion of logical keys with zero weight since they will - * only be scoped to the lifetime of the value being cached. - */ - private final Cache sideInputWeakReferenceCache; - private static final int DEFAULT_STATUS_PORT = 8081; - - /** Status pages returning health of worker. */ - private WorkerStatusPages statusPages; - - /** Periodic sender of debug information to the debug capture service. */ - private DebugCapture.Manager debugCaptureManager = null; - /** * A weight in "bytes" for the overhead of a {@link Weighted} wrapper in the cache. It is just an * approximation so it is OK for it to be fairly arbitrary as long as it is nonzero. @@ -121,33 +84,42 @@ public class BatchDataflowWorker implements Closeable { private static final int OVERHEAD_WEIGHT = 8; private static final long MEGABYTES = 1024 * 1024; - /** * Limit the number of logical references. Weak references may never be cleared if the object is * long lived irrespective if the user actually is interested in the key lookup anymore. */ private static final int MAX_LOGICAL_REFERENCES = 1_000_000; - /** How many concurrent write operations to a cache should we allow. */ private static final int CACHE_CONCURRENCY_LEVEL = 4 * Runtime.getRuntime().availableProcessors(); + /** A client to get and update work items. */ + private final WorkUnitClient workUnitClient; + /** + * Pipeline options, initially provided via the constructor and partially provided via each work + * work unit. + */ + private final DataflowWorkerHarnessOptions options; + /** The factory to create {@link DataflowMapTaskExecutor DataflowMapTaskExecutors}. */ + private final DataflowMapTaskExecutorFactory mapTaskExecutorFactory; + /** Registry of known {@link ReaderFactory ReaderFactories}. */ + private final ReaderRegistry readerRegistry = ReaderRegistry.defaultRegistry(); + /** Registry of known {@link SinkFactory SinkFactories}. */ + private final SinkRegistry sinkRegistry = SinkRegistry.defaultRegistry(); + /** A side input cache shared between all execution contexts. */ + private final Cache> sideInputDataCache; + /** + * A side input cache shared between all execution contexts. This cache is meant to store values + * as weak references. This allows for insertion of logical keys with zero weight since they will + * only be scoped to the lifetime of the value being cached. + */ + private final Cache sideInputWeakReferenceCache; private final Function> mapTaskToNetwork; - private final MemoryMonitor memoryMonitor; private final Thread memoryMonitorThread; - - /** - * Returns a {@link BatchDataflowWorker} configured to execute user functions via intrinsic Java - * execution. - * - *

    This is also known as the "legacy" or "pre-portability" approach. It is not yet deprecated - * as there is not a compatible path forward for users. - */ - static BatchDataflowWorker forBatchIntrinsicWorkerHarness( - WorkUnitClient workUnitClient, DataflowWorkerHarnessOptions options) { - return new BatchDataflowWorker( - workUnitClient, IntrinsicMapTaskExecutorFactory.defaultFactory(), options); - } + /** Status pages returning health of worker. */ + private final WorkerStatusPages statusPages; + /** Periodic sender of debug information to the debug capture service. */ + private DebugCapture.Manager debugCaptureManager = null; protected BatchDataflowWorker( WorkUnitClient workUnitClient, @@ -188,6 +160,19 @@ protected BatchDataflowWorker( ExecutionStateSampler.instance().start(); } + /** + * Returns a {@link BatchDataflowWorker} configured to execute user functions via intrinsic Java + * execution. + * + *

    This is also known as the "legacy" or "pre-portability" approach. It is not yet deprecated + * as there is not a compatible path forward for users. + */ + static BatchDataflowWorker forBatchIntrinsicWorkerHarness( + WorkUnitClient workUnitClient, DataflowWorkerHarnessOptions options) { + return new BatchDataflowWorker( + workUnitClient, IntrinsicMapTaskExecutorFactory.defaultFactory(), options); + } + private static DebugCapture.Manager initializeAndStartDebugCaptureManager( DataflowWorkerHarnessOptions options, Collection debugCapturePages) { DebugCapture.Manager result = new DebugCapture.Manager(options, debugCapturePages); @@ -215,7 +200,7 @@ private static Thread startMemoryMonitorThread(MemoryMonitor memoryMonitor) { */ public boolean getAndPerformWork() throws IOException { while (true) { - Optional work = workUnitClient.getWorkItem(); + Optional work = Optional.fromJavaUtil(workUnitClient.getWorkItem()); if (work.isPresent()) { WorkItemStatusClient statusProvider = new WorkItemStatusClient(workUnitClient, work.get()); return doWork(work.get(), statusProvider); @@ -243,7 +228,7 @@ boolean doWork(WorkItem workItem, WorkItemStatusClient workItemStatusClient) thr } else if (workItem.getSourceOperationTask() != null) { stageName = workItem.getSourceOperationTask().getStageName(); } else { - throw new RuntimeException("Unknown kind of work item: " + workItem.toString()); + throw new RuntimeException("Unknown kind of work item: " + workItem); } CounterSet counterSet = new CounterSet(); diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/BatchModeExecutionContext.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/BatchModeExecutionContext.java index efdfea0de0100..901e305b22b14 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/BatchModeExecutionContext.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/BatchModeExecutionContext.java @@ -45,11 +45,11 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WeightedValue; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ByteArrayReader.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ByteArrayReader.java index 586d86fae9887..e3de27baa5188 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ByteArrayReader.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ByteArrayReader.java @@ -19,7 +19,7 @@ import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.UnsafeByteOperations; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Ints; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Ints; class ByteArrayReader { diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ByteStringCoder.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ByteStringCoder.java index 5c426f71d6bd0..d48d9ae77a4ed 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ByteStringCoder.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ByteStringCoder.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.util.VarInt; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; /** * A simplified {@link Coder} for {@link ByteString}, to avoid a dependency on diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/CombineValuesFnFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/CombineValuesFnFactory.java index b914cc5fbda8e..50d518f782162 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/CombineValuesFnFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/CombineValuesFnFactory.java @@ -45,7 +45,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ConcatReader.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ConcatReader.java index d46e2120e0846..1f49591d162e2 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ConcatReader.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ConcatReader.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.dataflow.model.ApproximateReportedProgress; import com.google.api.services.dataflow.model.ApproximateSplitRequest; @@ -33,8 +33,8 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.io.range.OffsetRangeTracker; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ContextActivationObserverRegistry.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ContextActivationObserverRegistry.java index b7558af044741..e19ad600cc70e 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ContextActivationObserverRegistry.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ContextActivationObserverRegistry.java @@ -23,9 +23,9 @@ import java.util.Set; import org.apache.beam.sdk.util.common.ReflectHelpers; import org.apache.beam.sdk.util.common.ReflectHelpers.ObjectsClassComparator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.slf4j.LoggerFactory; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/CounterShortIdCache.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/CounterShortIdCache.java index 3afed7754bc56..80ad2bf0758eb 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/CounterShortIdCache.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/CounterShortIdCache.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.dataflow.model.CounterStructuredName; import com.google.api.services.dataflow.model.CounterUpdate; @@ -28,7 +28,7 @@ import com.google.api.services.dataflow.model.WorkItemServiceState; import com.google.api.services.dataflow.model.WorkItemStatus; import java.util.concurrent.ConcurrentHashMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/CreateIsmShardKeyAndSortKeyDoFnFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/CreateIsmShardKeyAndSortKeyDoFnFactory.java index d7293a4b72338..e917a150e0a24 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/CreateIsmShardKeyAndSortKeyDoFnFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/CreateIsmShardKeyAndSortKeyDoFnFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.services.dataflow.model.SideInputInfo; import java.util.List; @@ -35,7 +35,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * A {@link ParDoFnFactory} that creates a system {@link ParDoFn} responsible for limiting the users diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowApiUtils.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowApiUtils.java index 1900a5105ce16..4546a5a46d02f 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowApiUtils.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowApiUtils.java @@ -23,8 +23,8 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import org.apache.beam.sdk.extensions.gcp.util.Transport; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CountingOutputStream; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CountingOutputStream; /** A utility class for generic interactions with the Google Cloud Dataflow API. */ public final class DataflowApiUtils { diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowBatchWorkerHarness.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowBatchWorkerHarness.java index a06d1d7d1e7fc..cc79ac6dbc0d1 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowBatchWorkerHarness.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowBatchWorkerHarness.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.util.ArrayList; @@ -32,7 +32,7 @@ import org.apache.beam.sdk.util.BackOffUtils; import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowElementExecutionTracker.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowElementExecutionTracker.java index c446833955c15..ccc98fa87e6aa 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowElementExecutionTracker.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowElementExecutionTracker.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.time.Duration; import java.util.ArrayDeque; @@ -36,10 +36,10 @@ import org.apache.beam.runners.dataflow.worker.counters.NameContext; import org.apache.beam.runners.dataflow.worker.util.common.worker.ElementExecutionTracker; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.PeekingIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.PeekingIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionContext.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionContext.java index d02637c7efa92..d1de0833d87d6 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionContext.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionContext.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.dataflow.model.SideInputInfo; import java.io.Closeable; @@ -43,8 +43,8 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Closer; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Closer; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateKey.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateKey.java index 7c435472c4f62..de3be8715c1b2 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateKey.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateKey.java @@ -19,8 +19,8 @@ import com.google.auto.value.AutoValue; import org.apache.beam.runners.dataflow.worker.counters.NameContext; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ComparisonChain; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ComparisonChain; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateRegistry.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateRegistry.java index 88299790c332f..5f26991dda073 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateRegistry.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateRegistry.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates.notNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates.notNull; import com.google.api.services.dataflow.model.CounterUpdate; import java.util.Map; @@ -26,7 +26,7 @@ import org.apache.beam.runners.dataflow.worker.counters.NameContext; import org.apache.beam.runners.dataflow.worker.profiler.ScopedProfiler.ProfileScope; import org.apache.beam.sdk.metrics.MetricsContainer; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; import org.checkerframework.checker.nullness.qual.Nullable; /** Manages the instances of {@link ExecutionState} */ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowMapTaskExecutorFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowMapTaskExecutorFactory.java index a1db1493df1fd..bc2be66ac106c 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowMapTaskExecutorFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowMapTaskExecutorFactory.java @@ -23,7 +23,7 @@ import org.apache.beam.runners.dataflow.worker.graph.Nodes.Node; import org.apache.beam.sdk.fn.IdGenerator; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.MutableNetwork; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.MutableNetwork; /** Creates a {@link DataflowMapTaskExecutor} from a {@link MapTask} definition. */ public interface DataflowMapTaskExecutorFactory { diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowMetricsContainer.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowMetricsContainer.java index 81517129c8e91..c3e4fb1388b06 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowMetricsContainer.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowMetricsContainer.java @@ -22,9 +22,11 @@ import org.apache.beam.sdk.metrics.Counter; import org.apache.beam.sdk.metrics.Distribution; import org.apache.beam.sdk.metrics.Gauge; +import org.apache.beam.sdk.metrics.Histogram; import org.apache.beam.sdk.metrics.MetricName; import org.apache.beam.sdk.metrics.MetricsContainer; import org.apache.beam.sdk.metrics.MetricsEnvironment; +import org.apache.beam.sdk.util.HistogramData; /** * An implementation of {@link MetricsContainer} that reads the current execution state (tracked in @@ -56,6 +58,11 @@ public Counter getCounter(MetricName metricName) { return getCurrentContainer().getCounter(metricName); } + @Override + public Counter getPerWorkerCounter(MetricName metricName) { + return getCurrentContainer().getPerWorkerCounter(metricName); + } + @Override public Distribution getDistribution(MetricName metricName) { return getCurrentContainer().getDistribution(metricName); @@ -65,4 +72,10 @@ public Distribution getDistribution(MetricName metricName) { public Gauge getGauge(MetricName metricName) { return getCurrentContainer().getGauge(metricName); } + + @Override + public Histogram getPerWorkerHistogram( + MetricName metricName, HistogramData.BucketType bucketType) { + return getCurrentContainer().getPerWorkerHistogram(metricName, bucketType); + } } diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowOperationContext.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowOperationContext.java index 2062913e04773..df520ebd3923c 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowOperationContext.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowOperationContext.java @@ -40,9 +40,9 @@ import org.apache.beam.runners.dataflow.worker.profiler.ScopedProfiler.ProfileScope; import org.apache.beam.runners.dataflow.worker.util.common.worker.OperationContext; import org.apache.beam.sdk.metrics.MetricsContainer; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.format.PeriodFormatter; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowOutputCounter.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowOutputCounter.java index 4e24df0819ef9..81ef0a1c9d95f 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowOutputCounter.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowOutputCounter.java @@ -25,7 +25,7 @@ import org.apache.beam.runners.dataflow.worker.util.common.worker.ElementCounter; import org.apache.beam.runners.dataflow.worker.util.common.worker.OutputObjectAndByteCounter; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** * A Dataflow-specific version of {@link ElementCounter}, which specifies the object counter name diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowProcessFnRunner.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowProcessFnRunner.java index c3a8d30b1194f..22718b745105e 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowProcessFnRunner.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowProcessFnRunner.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.dataflow.worker; import static org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems.ProcessFn; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Collection; import org.apache.beam.runners.core.DoFnRunner; @@ -32,7 +32,7 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Instant; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowSideInputReadCounter.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowSideInputReadCounter.java index ab2838b0482cd..cd4b11b460a44 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowSideInputReadCounter.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowSideInputReadCounter.java @@ -25,7 +25,7 @@ import org.apache.beam.runners.dataflow.worker.counters.Counter; import org.apache.beam.runners.dataflow.worker.counters.CounterName; import org.apache.beam.runners.dataflow.worker.counters.NameContext; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; /** * This class tracks time and bytes spent when consuming side inputs. diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowSystemMetrics.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowSystemMetrics.java index 0dae356b754b6..ee2a04af9982b 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowSystemMetrics.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowSystemMetrics.java @@ -20,7 +20,7 @@ import org.apache.beam.runners.dataflow.worker.counters.CounterName; import org.apache.beam.runners.dataflow.worker.counters.NameContext; import org.apache.beam.sdk.metrics.MetricName; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; /** This holds system metrics related constants used in Batch and Streaming. */ public class DataflowSystemMetrics { @@ -39,6 +39,9 @@ public enum StreamingSystemCounterNames { JAVA_HARNESS_USED_MEMORY("dataflow_java_harness_used_memory"), JAVA_HARNESS_MAX_MEMORY("dataflow_java_harness_max_memory"), JAVA_HARNESS_RESTARTS("dataflow_java_harness_restarts"), + TIME_AT_MAX_ACTIVE_THREADS("dataflow_time_at_max_active_threads"), + ACTIVE_THREADS("dataflow_active_threads"), + TOTAL_ALLOCATED_THREADS("dataflow_total_allocated_threads"), WINDMILL_QUOTA_THROTTLING("dataflow_streaming_engine_throttled_msecs"), MEMORY_THRASHING("dataflow_streaming_engine_user_worker_thrashing"); diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowWorkProgressUpdater.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowWorkProgressUpdater.java index f16104295622d..f901b01ed5663 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowWorkProgressUpdater.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowWorkProgressUpdater.java @@ -31,7 +31,7 @@ import org.apache.beam.runners.dataflow.util.TimeUtil; import org.apache.beam.runners.dataflow.worker.util.common.worker.WorkExecutor; import org.apache.beam.runners.dataflow.worker.util.common.worker.WorkProgressUpdater; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowWorkUnitClient.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowWorkUnitClient.java index 53d33ddd0ba6a..ffa377fd3f828 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowWorkUnitClient.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowWorkUnitClient.java @@ -24,7 +24,7 @@ import static org.apache.beam.runners.dataflow.worker.util.WorkerPropertyNames.WORK_ITEM_TYPE_REMOTE_SOURCE_TASK; import static org.apache.beam.runners.dataflow.worker.util.WorkerPropertyNames.WORK_ITEM_TYPE_SEQ_MAP_TASK; import static org.apache.beam.runners.dataflow.worker.util.WorkerPropertyNames.WORK_ITEM_TYPE_STREAMING_CONFIG_TASK; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; import com.google.api.services.dataflow.Dataflow; import com.google.api.services.dataflow.model.LeaseWorkItemRequest; @@ -39,15 +39,15 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; import javax.annotation.concurrent.ThreadSafe; import org.apache.beam.runners.dataflow.options.DataflowWorkerHarnessOptions; import org.apache.beam.runners.dataflow.util.PropertyNames; import org.apache.beam.runners.dataflow.worker.logging.DataflowWorkerLoggingMDC; import org.apache.beam.runners.dataflow.worker.util.common.worker.WorkProgressUpdater; import org.apache.beam.sdk.extensions.gcp.util.Transport; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.DateTime; import org.joda.time.Duration; import org.slf4j.Logger; @@ -87,7 +87,7 @@ class DataflowWorkUnitClient implements WorkUnitClient { } /** - * Gets a {@link WorkItem} from the Dataflow service, or returns {@link Optional#absent()} if no + * Gets a {@link WorkItem} from the Dataflow service, or returns {@link Optional#empty()} if no * work was found. * *

    If work is returned, the calling thread should call reportWorkItemStatus after completing it @@ -116,11 +116,11 @@ public Optional getWorkItem() throws IOException { if (!workItem.isPresent()) { // Normal case, this means that the response contained no work, i.e. no work is available // at this time. - return Optional.absent(); + return Optional.empty(); } - if (workItem.isPresent() && workItem.get().getId() == null) { - logger.debug("Discarding invalid work item {}", workItem.orNull()); - return Optional.absent(); + if (workItem.get().getId() == null) { + logger.debug("Discarding invalid work item {}", workItem.get()); + return Optional.empty(); } WorkItem work = workItem.get(); @@ -148,7 +148,7 @@ public Optional getWorkItem() throws IOException { /** * Gets a global streaming config {@link WorkItem} from the Dataflow service, or returns {@link - * Optional#absent()} if no work was found. + * Optional#empty()} if no work was found. */ @Override public Optional getGlobalStreamingConfigWorkItem() throws IOException { @@ -158,7 +158,7 @@ public Optional getGlobalStreamingConfigWorkItem() throws IOException /** * Gets a streaming config {@link WorkItem} for the given computation from the Dataflow service, - * or returns {@link Optional#absent()} if no work was found. + * or returns {@link Optional#empty()} if no work was found. */ @Override public Optional getStreamingConfigWorkItem(String computationId) throws IOException { @@ -197,7 +197,7 @@ private Optional getWorkItemInternal( List workItems = response.getWorkItems(); if (workItems == null || workItems.isEmpty()) { // We didn't lease any work. - return Optional.absent(); + return Optional.empty(); } else if (workItems.size() > 1) { throw new IOException( "This version of the SDK expects no more than one work item from the service: " diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowWorkerHarnessHelper.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowWorkerHarnessHelper.java index c6d35ef04e5eb..4280dbb6a5cbf 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowWorkerHarnessHelper.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowWorkerHarnessHelper.java @@ -30,7 +30,7 @@ import org.apache.beam.runners.dataflow.worker.logging.DataflowWorkerLoggingInitializer; import org.apache.beam.runners.dataflow.worker.logging.DataflowWorkerLoggingMDC; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.TextFormat; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.checkerframework.checker.nullness.qual.Nullable; import org.conscrypt.OpenSSLProvider; import org.slf4j.Logger; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DefaultParDoFnFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DefaultParDoFnFactory.java index 9dc8b1eaf5fa0..b455bfb144061 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DefaultParDoFnFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DefaultParDoFnFactory.java @@ -24,7 +24,7 @@ import org.apache.beam.runners.dataflow.worker.util.common.worker.ParDoFn; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * A factory that dispatches to all known factories in the Dataflow SDK based on the value of {@link diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ExperimentContext.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ExperimentContext.java index be05d453f9722..9ad17d0e833ea 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ExperimentContext.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ExperimentContext.java @@ -21,8 +21,8 @@ import java.util.Set; import org.apache.beam.runners.dataflow.options.DataflowPipelineDebugOptions; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; /** * A convenient class to provide fast lookup of enabled experiments in the worker code. diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/Filepatterns.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/Filepatterns.java index 70a0093616ce0..a4d710d5e3995 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/Filepatterns.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/Filepatterns.java @@ -19,7 +19,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Utilities for handling filepatterns. */ @SuppressWarnings({ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupAlsoByWindowParDoFnFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupAlsoByWindowParDoFnFactory.java index 1bfb756ef7003..3d49626911050 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupAlsoByWindowParDoFnFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupAlsoByWindowParDoFnFactory.java @@ -20,7 +20,7 @@ import static org.apache.beam.runners.dataflow.util.Structs.getBytes; import static org.apache.beam.runners.dataflow.util.Structs.getObject; import static org.apache.beam.runners.dataflow.util.Structs.getString; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.services.dataflow.model.SideInputInfo; import java.util.ArrayList; @@ -56,7 +56,7 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupAlsoByWindowsParDoFn.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupAlsoByWindowsParDoFn.java index 43665f0378781..a611769eb5ee7 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupAlsoByWindowsParDoFn.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupAlsoByWindowsParDoFn.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.ArrayList; import java.util.List; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupingShuffleReader.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupingShuffleReader.java index 2fefc9c6084af..8af6068f2a3a0 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupingShuffleReader.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupingShuffleReader.java @@ -21,7 +21,7 @@ import static org.apache.beam.runners.dataflow.worker.SourceTranslationUtils.cloudPositionToReaderPosition; import static org.apache.beam.runners.dataflow.worker.SourceTranslationUtils.cloudProgressToReaderProgress; import static org.apache.beam.runners.dataflow.worker.SourceTranslationUtils.splitRequestToApproximateSplitRequest; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.services.dataflow.model.ApproximateReportedProgress; import com.google.api.services.dataflow.model.ApproximateSplitRequest; @@ -56,7 +56,7 @@ import org.apache.beam.sdk.util.common.Reiterable; import org.apache.beam.sdk.util.common.Reiterator; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupingShuffleReaderFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupingShuffleReaderFactory.java index abb35ff16f4d5..daf1e87f42962 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupingShuffleReaderFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupingShuffleReaderFactory.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** Creates a GroupingShuffleReader from a CloudObject spec. */ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupingShuffleReaderWithFaultyBytesReadCounter.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupingShuffleReaderWithFaultyBytesReadCounter.java index 2ae27f1dd41e5..1540431bb1785 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupingShuffleReaderWithFaultyBytesReadCounter.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/GroupingShuffleReaderWithFaultyBytesReadCounter.java @@ -23,7 +23,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/InMemoryReader.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/InMemoryReader.java index 0c2edc7218260..9ce5fad93d996 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/InMemoryReader.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/InMemoryReader.java @@ -18,8 +18,8 @@ package org.apache.beam.runners.dataflow.worker; import static com.google.api.client.util.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.services.dataflow.model.ApproximateReportedProgress; import java.io.IOException; @@ -32,8 +32,8 @@ import org.apache.beam.sdk.io.range.OffsetRangeTracker; import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.util.StringUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/InMemoryReaderFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/InMemoryReaderFactory.java index ba6526f9e329b..955e0893b3e36 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/InMemoryReaderFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/InMemoryReaderFactory.java @@ -28,7 +28,7 @@ import org.apache.beam.runners.dataflow.worker.util.common.worker.NativeReader; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** Creates an InMemoryReader from a CloudObject spec. */ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IntrinsicMapTaskExecutorFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IntrinsicMapTaskExecutorFactory.java index 48405d2a6b187..499dcaa4e77c6 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IntrinsicMapTaskExecutorFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IntrinsicMapTaskExecutorFactory.java @@ -66,10 +66,10 @@ import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder; import org.apache.beam.sdk.util.common.ElementByteSizeObserver; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.MutableNetwork; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.Network; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.MutableNetwork; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.Network; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmReader.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmReader.java index a22656695426d..b19e77cde8711 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmReader.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmReader.java @@ -26,7 +26,7 @@ import org.apache.beam.runners.dataflow.worker.util.common.worker.NativeReader; import org.apache.beam.sdk.io.fs.ResourceId; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; /** * A {@link NativeReader} that reads Ism files. diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmReaderFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmReaderFactory.java index 5418ef9d0dc2e..d0bf0c96f9a02 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmReaderFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmReaderFactory.java @@ -19,7 +19,7 @@ import static org.apache.beam.runners.dataflow.util.Structs.getString; import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; import java.util.Map; @@ -38,8 +38,8 @@ import org.apache.beam.sdk.util.WeightedValue; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmReaderImpl.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmReaderImpl.java index 01d67e9cd7077..86aab6cd1f973 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmReaderImpl.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmReaderImpl.java @@ -17,9 +17,9 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.Closeable; @@ -62,17 +62,17 @@ import org.apache.beam.sdk.util.VarInt; import org.apache.beam.sdk.util.WeightedValue; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSortedMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Ints; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSortedMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Ints; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Longs; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmSideInputReader.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmSideInputReader.java index 5fff72c5be28b..3060244415348 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmSideInputReader.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmSideInputReader.java @@ -20,9 +20,9 @@ import static org.apache.beam.runners.dataflow.util.Structs.addString; import static org.apache.beam.runners.dataflow.util.Structs.getString; import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.services.dataflow.model.SideInputInfo; import com.google.api.services.dataflow.model.Source; @@ -81,16 +81,16 @@ import org.apache.beam.sdk.values.PCollectionViews.SingletonViewFn; import org.apache.beam.sdk.values.PCollectionViews.SingletonViewFn2; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Ints; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Ints; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmSink.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmSink.java index 620e70444ad9a..748e3a1542af1 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmSink.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmSink.java @@ -17,9 +17,9 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.nio.channels.Channels; @@ -46,9 +46,9 @@ import org.apache.beam.sdk.util.MimeTypes; import org.apache.beam.sdk.util.VarInt; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CountingOutputStream; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CountingOutputStream; /** * A {@link Sink} that writes Ism files. diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmSinkFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmSinkFactory.java index b5fd69b8ac4c1..99d38246b42f2 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmSinkFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/IsmSinkFactory.java @@ -19,7 +19,7 @@ import static org.apache.beam.runners.dataflow.util.Structs.getString; import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; import java.math.RoundingMode; @@ -35,8 +35,8 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.math.DoubleMath; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.math.DoubleMath; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/LazilyInitializedSideInputReader.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/LazilyInitializedSideInputReader.java index 5bd34c05b94c5..08a1847a639d8 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/LazilyInitializedSideInputReader.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/LazilyInitializedSideInputReader.java @@ -24,8 +24,8 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Suppliers; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Suppliers; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/MetricTrackingWindmillServerStub.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/MetricTrackingWindmillServerStub.java index b7a1f6aec8d29..33b55647213aa 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/MetricTrackingWindmillServerStub.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/MetricTrackingWindmillServerStub.java @@ -29,8 +29,9 @@ import org.apache.beam.runners.dataflow.worker.windmill.Windmill; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataRequest; import org.apache.beam.runners.dataflow.worker.windmill.WindmillServerStub; -import org.apache.beam.runners.dataflow.worker.windmill.WindmillServerStub.GetDataStream; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.SettableFuture; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetDataStream; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStreamPool; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.SettableFuture; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; @@ -47,6 +48,10 @@ }) public class MetricTrackingWindmillServerStub { + private static final int MAX_READS_PER_BATCH = 60; + private static final int MAX_ACTIVE_READS = 10; + private static final int NUM_STREAMS = 1; + private static final Duration STREAM_TIMEOUT = Duration.standardSeconds(30); private final AtomicInteger activeSideInputs = new AtomicInteger(); private final AtomicInteger activeStateReads = new AtomicInteger(); private final AtomicInteger activeHeartbeats = new AtomicInteger(); @@ -54,39 +59,13 @@ public class MetricTrackingWindmillServerStub { private final MemoryMonitor gcThrashingMonitor; private final boolean useStreamingRequests; - private static final class ReadBatch { - ArrayList reads = new ArrayList<>(); - SettableFuture startRead = SettableFuture.create(); - } - @GuardedBy("this") private final List pendingReadBatches; @GuardedBy("this") private int activeReadThreads = 0; - private WindmillServerStub.StreamPool streamPool; - - private static final int MAX_READS_PER_BATCH = 60; - private static final int MAX_ACTIVE_READS = 10; - private static final int NUM_STREAMS = 1; - private static final Duration STREAM_TIMEOUT = Duration.standardSeconds(30); - - private static final class QueueEntry { - - final String computation; - final Windmill.KeyedGetDataRequest request; - final SettableFuture response; - - QueueEntry( - String computation, - Windmill.KeyedGetDataRequest request, - SettableFuture response) { - this.computation = computation; - this.request = request; - this.response = response; - } - } + private WindmillStreamPool streamPool; public MetricTrackingWindmillServerStub( WindmillServerStub server, MemoryMonitor gcThrashingMonitor, boolean useStreamingRequests) { @@ -100,8 +79,7 @@ public MetricTrackingWindmillServerStub( public void start() { if (useStreamingRequests) { streamPool = - new WindmillServerStub.StreamPool<>( - NUM_STREAMS, STREAM_TIMEOUT, this.server::getDataStream); + WindmillStreamPool.create(NUM_STREAMS, STREAM_TIMEOUT, this.server::getDataStream); } } @@ -300,4 +278,25 @@ public void printHtml(PrintWriter writer) { } writer.println("Heartbeat Keys Active: " + activeHeartbeats.get()); } + + private static final class ReadBatch { + ArrayList reads = new ArrayList<>(); + SettableFuture startRead = SettableFuture.create(); + } + + private static final class QueueEntry { + + final String computation; + final Windmill.KeyedGetDataRequest request; + final SettableFuture response; + + QueueEntry( + String computation, + Windmill.KeyedGetDataRequest request, + SettableFuture response) { + this.computation = computation; + this.request = request; + this.response = response; + } + } } diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/MetricsContainerRegistry.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/MetricsContainerRegistry.java index 8120788353d3e..c89032e05ae90 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/MetricsContainerRegistry.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/MetricsContainerRegistry.java @@ -20,7 +20,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentSkipListMap; import org.apache.beam.sdk.metrics.MetricsContainer; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; /** * Manages the instances of {@link MetricsContainer} that have been created for a specific context. diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/MetricsEnvironmentContextActivationObserverRegistration.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/MetricsEnvironmentContextActivationObserverRegistration.java index 9500170f24e83..25d0308124f56 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/MetricsEnvironmentContextActivationObserverRegistration.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/MetricsEnvironmentContextActivationObserverRegistration.java @@ -22,7 +22,7 @@ import java.io.IOException; import org.apache.beam.runners.core.metrics.ExecutionStateTracker; import org.apache.beam.sdk.metrics.MetricsEnvironment; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** * Registration of a {@link ContextActivationObserver} for the {@link MetricsEnvironment} with the diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/OrderedCode.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/OrderedCode.java index 1adf19b5a98ec..89217c72ad19b 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/OrderedCode.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/OrderedCode.java @@ -20,8 +20,8 @@ import java.math.RoundingMode; import java.util.ArrayList; import java.util.Arrays; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.math.LongMath; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.math.LongMath; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Longs; /** * This module provides routines for encoding a sequence of typed entities into a byte array. The diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PairWithConstantKeyDoFnFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PairWithConstantKeyDoFnFactory.java index 84c719cc5de5b..2ca04d9168af0 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PairWithConstantKeyDoFnFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PairWithConstantKeyDoFnFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.services.dataflow.model.SideInputInfo; import java.util.List; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PartialGroupByKeyParDoFns.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PartialGroupByKeyParDoFns.java index fee5127066cd1..ca709de7effbd 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PartialGroupByKeyParDoFns.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PartialGroupByKeyParDoFns.java @@ -47,9 +47,9 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.common.ElementByteSizeObserver; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CountingOutputStream; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CountingOutputStream; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PartitioningShuffleReader.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PartitioningShuffleReader.java index ed12052cb1b10..81dfd59ff2b41 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PartitioningShuffleReader.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PartitioningShuffleReader.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** * A source that reads from a key-sharded dataset, and returns KVs without any values grouping. diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PartitioningShuffleReaderFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PartitioningShuffleReaderFactory.java index d6f85c6003887..1a8c7e7d49ff0 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PartitioningShuffleReaderFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PartitioningShuffleReaderFactory.java @@ -19,7 +19,7 @@ import static com.google.api.client.util.Base64.decodeBase64; import static org.apache.beam.runners.dataflow.util.Structs.getString; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; import java.util.Map; @@ -30,7 +30,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** Creates a PartitioningShuffleReader from a CloudObject spec. */ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PubsubDynamicSink.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PubsubDynamicSink.java index 70a74c3f15632..b86f48717890d 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PubsubDynamicSink.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PubsubDynamicSink.java @@ -34,9 +34,9 @@ import org.apache.beam.sdk.util.ByteStringOutputStream; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; @SuppressWarnings({ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PubsubReader.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PubsubReader.java index 5dc047e933476..d0931e02cc87f 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PubsubReader.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PubsubReader.java @@ -36,7 +36,7 @@ import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** A Reader that receives elements from Pubsub, via a Windmill server. */ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PubsubSink.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PubsubSink.java index 3173dabeb8b8a..c1cc4d2bac627 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PubsubSink.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/PubsubSink.java @@ -19,7 +19,7 @@ import static org.apache.beam.runners.dataflow.util.Structs.getBytes; import static org.apache.beam.runners.dataflow.util.Structs.getString; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.service.AutoService; import java.io.IOException; @@ -38,7 +38,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ReaderCache.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ReaderCache.java index 466aac63d9377..01010863f1eeb 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ReaderCache.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ReaderCache.java @@ -22,11 +22,11 @@ import java.util.concurrent.TimeUnit; import javax.annotation.concurrent.ThreadSafe; import org.apache.beam.sdk.io.UnboundedSource; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.RemovalCause; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.RemovalNotification; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.RemovalCause; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.RemovalNotification; import org.joda.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ReaderRegistry.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ReaderRegistry.java index dfff9722f5a24..2e574c07b879f 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ReaderRegistry.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ReaderRegistry.java @@ -29,10 +29,10 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ReifyTimestampAndWindowsParDoFnFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ReifyTimestampAndWindowsParDoFnFactory.java index 9fd519bb7f440..a5971bb6bdaf7 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ReifyTimestampAndWindowsParDoFnFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ReifyTimestampAndWindowsParDoFnFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.services.dataflow.model.SideInputInfo; import java.util.List; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/RunnerHarnessCoderCloudObjectTranslatorRegistrar.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/RunnerHarnessCoderCloudObjectTranslatorRegistrar.java index 77fb7145772f9..cb90a320ceb83 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/RunnerHarnessCoderCloudObjectTranslatorRegistrar.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/RunnerHarnessCoderCloudObjectTranslatorRegistrar.java @@ -37,7 +37,7 @@ import org.apache.beam.sdk.coders.VarLongCoder; import org.apache.beam.sdk.coders.VoidCoder; import org.apache.beam.sdk.util.InstanceBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * A registrar for {@link CloudObjectTranslator}s for the Dataflow runner harness. diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ShuffleSink.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ShuffleSink.java index 7f1dd75ad0fd7..02e2d48c369a3 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ShuffleSink.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ShuffleSink.java @@ -37,8 +37,8 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Ints; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Ints; /** * A sink that writes to a shuffle dataset. diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ShuffleSinkFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ShuffleSinkFactory.java index 1beb2f15bd72d..53175eaf91e5d 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ShuffleSinkFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ShuffleSinkFactory.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** Creates a {@link ShuffleSink} from a {@link CloudObject} spec. */ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SimpleParDoFn.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SimpleParDoFn.java index d48ea9d58caa6..a413c2c03dbef 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SimpleParDoFn.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SimpleParDoFn.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.Closeable; import java.util.Collection; @@ -57,8 +57,8 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SinkRegistry.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SinkRegistry.java index 80cd1aa2fd3d2..582257e789e31 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SinkRegistry.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SinkRegistry.java @@ -26,9 +26,9 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SizeReportingSinkWrapper.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SizeReportingSinkWrapper.java index 4005202c66323..ed6a77f6d9c42 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SizeReportingSinkWrapper.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SizeReportingSinkWrapper.java @@ -19,7 +19,7 @@ import java.io.IOException; import org.apache.beam.runners.dataflow.worker.util.common.worker.Sink; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** * A wrapper for Sink that reports bytes buffered (or written) to {@link DataflowExecutionContext}. diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SourceOperationExecutorFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SourceOperationExecutorFactory.java index 90fdf2a47c4b5..a5dd7790f057b 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SourceOperationExecutorFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SourceOperationExecutorFactory.java @@ -21,7 +21,7 @@ import org.apache.beam.runners.dataflow.worker.counters.CounterSet; import org.apache.beam.runners.dataflow.worker.counters.NameContext; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; /** Creates a SourceOperationExecutor from a SourceOperation. */ public class SourceOperationExecutorFactory { diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SplittableProcessFnFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SplittableProcessFnFactory.java index 872acc7c35d96..f31d71374983a 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SplittableProcessFnFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SplittableProcessFnFactory.java @@ -22,7 +22,7 @@ import static org.apache.beam.runners.dataflow.util.Structs.getBytes; import static org.apache.beam.runners.dataflow.util.Structs.getObject; import static org.apache.beam.sdk.util.SerializableUtils.deserializeFromByteArray; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Collection; import java.util.List; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StateFetcher.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StateFetcher.java index 82ca67c8fb396..0cbcd2e830120 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StateFetcher.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StateFetcher.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.Closeable; import java.util.Collections; @@ -39,12 +39,12 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Weigher; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Weigher; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorker.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorker.java index 92d0710bbe757..4c1693d613874 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorker.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorker.java @@ -19,10 +19,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.beam.runners.dataflow.DataflowRunner.hasExperiment; -import static org.apache.beam.runners.dataflow.worker.DataflowSystemMetrics.THROTTLING_MSECS_METRIC_NAME; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; -import com.google.api.services.dataflow.model.CounterStructuredName; import com.google.api.services.dataflow.model.CounterUpdate; import com.google.api.services.dataflow.model.MapTask; import com.google.api.services.dataflow.model.Status; @@ -30,23 +28,19 @@ import com.google.api.services.dataflow.model.StreamingConfigTask; import com.google.api.services.dataflow.model.WorkItem; import com.google.api.services.dataflow.model.WorkItemStatus; -import com.google.auto.value.AutoValue; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.File; import java.io.IOException; import java.io.PrintWriter; -import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.Deque; -import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Queue; +import java.util.Optional; import java.util.Random; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -54,9 +48,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -66,17 +58,13 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.beam.runners.core.metrics.ExecutionStateSampler; -import org.apache.beam.runners.core.metrics.ExecutionStateTracker; import org.apache.beam.runners.core.metrics.MetricsLogger; import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.runners.dataflow.internal.CustomSources; import org.apache.beam.runners.dataflow.options.DataflowWorkerHarnessOptions; import org.apache.beam.runners.dataflow.util.CloudObject; import org.apache.beam.runners.dataflow.util.CloudObjects; -import org.apache.beam.runners.dataflow.worker.DataflowSystemMetrics.StreamingPerStageSystemCounterNames; import org.apache.beam.runners.dataflow.worker.DataflowSystemMetrics.StreamingSystemCounterNames; -import org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker.Work.State; -import org.apache.beam.runners.dataflow.worker.StreamingModeExecutionContext.StreamingModeExecutionStateRegistry; import org.apache.beam.runners.dataflow.worker.apiary.FixMultiOutputInfosOnParDoInstructions; import org.apache.beam.runners.dataflow.worker.counters.Counter; import org.apache.beam.runners.dataflow.worker.counters.CounterSet; @@ -97,17 +85,29 @@ import org.apache.beam.runners.dataflow.worker.status.LastExceptionDataProvider; import org.apache.beam.runners.dataflow.worker.status.StatusDataProvider; import org.apache.beam.runners.dataflow.worker.status.WorkerStatusPages; +import org.apache.beam.runners.dataflow.worker.streaming.Commit; +import org.apache.beam.runners.dataflow.worker.streaming.ComputationState; +import org.apache.beam.runners.dataflow.worker.streaming.ExecutionState; +import org.apache.beam.runners.dataflow.worker.streaming.KeyCommitTooLargeException; +import org.apache.beam.runners.dataflow.worker.streaming.ShardedKey; +import org.apache.beam.runners.dataflow.worker.streaming.StageInfo; +import org.apache.beam.runners.dataflow.worker.streaming.WeightedBoundedQueue; +import org.apache.beam.runners.dataflow.worker.streaming.Work; +import org.apache.beam.runners.dataflow.worker.streaming.Work.State; import org.apache.beam.runners.dataflow.worker.util.BoundedQueueExecutor; import org.apache.beam.runners.dataflow.worker.util.MemoryMonitor; import org.apache.beam.runners.dataflow.worker.util.common.worker.ElementCounter; import org.apache.beam.runners.dataflow.worker.util.common.worker.OutputObjectAndByteCounter; import org.apache.beam.runners.dataflow.worker.util.common.worker.ReadOperation; import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.LatencyAttribution; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.WorkItemCommitRequest; import org.apache.beam.runners.dataflow.worker.windmill.WindmillServerStub; -import org.apache.beam.runners.dataflow.worker.windmill.WindmillServerStub.CommitWorkStream; -import org.apache.beam.runners.dataflow.worker.windmill.WindmillServerStub.GetWorkStream; -import org.apache.beam.runners.dataflow.worker.windmill.WindmillServerStub.StreamPool; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.CommitWorkStream; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetWorkStream; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStreamPool; +import org.apache.beam.runners.dataflow.worker.windmill.state.WindmillStateCache; +import org.apache.beam.runners.dataflow.worker.windmill.state.WindmillStateReader; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.KvCoder; import org.apache.beam.sdk.extensions.gcp.util.Transport; @@ -124,22 +124,20 @@ import org.apache.beam.sdk.util.UserCodeException; import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.TextFormat; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.EvictingQueue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.MultimapBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.MutableNetwork; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.net.HostAndPort; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.EvictingQueue; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.MultimapBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.MutableNetwork; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.net.HostAndPort; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; @@ -153,8 +151,29 @@ }) public class StreamingDataflowWorker { - private static final Logger LOG = LoggerFactory.getLogger(StreamingDataflowWorker.class); + // TODO(https://github.com/apache/beam/issues/19632): Update throttling counters to use generic + // throttling-msecs metric. + public static final MetricName BIGQUERY_STREAMING_INSERT_THROTTLE_TIME = + MetricName.named( + "org.apache.beam.sdk.io.gcp.bigquery.BigQueryServicesImpl$DatasetServiceImpl", + "throttling-msecs"); + // Maximum number of threads for processing. Currently each thread processes one key at a time. + static final int MAX_PROCESSING_THREADS = 300; + static final long THREAD_EXPIRATION_TIME_SEC = 60; + static final long TARGET_COMMIT_BUNDLE_BYTES = 32 << 20; + static final int MAX_COMMIT_QUEUE_BYTES = 500 << 20; // 500MB + static final int NUM_COMMIT_STREAMS = 1; + static final int GET_WORK_STREAM_TIMEOUT_MINUTES = 3; + static final Duration COMMIT_STREAM_TIMEOUT = Duration.standardMinutes(1); + /** + * Sinks are marked 'full' in {@link StreamingModeExecutionContext} once the amount of data sinked + * (across all the sinks, if there are more than one) reaches this limit. This serves as hint for + * readers to stop producing more. This can be disabled with 'disable_limiting_bundle_sink_bytes' + * experiment. + */ + static final int MAX_SINK_BYTES = 10_000_000; + private static final Logger LOG = LoggerFactory.getLogger(StreamingDataflowWorker.class); /** The idGenerator to generate unique id globally. */ private static final IdGenerator idGenerator = IdGenerators.decrementingLongs(); /** @@ -163,7 +182,6 @@ public class StreamingDataflowWorker { */ private static final Function fixMultiOutputInfos = new FixMultiOutputInfosOnParDoInstructions(idGenerator); - /** * Function which converts map tasks to their network representation for execution. * @@ -175,235 +193,35 @@ public class StreamingDataflowWorker { private static final Function> mapTaskToBaseNetwork = new MapTaskToNetworkFunction(idGenerator); - private static Random clientIdGenerator = new Random(); - - // Maximum number of threads for processing. Currently each thread processes one key at a time. - static final int MAX_PROCESSING_THREADS = 300; - static final long THREAD_EXPIRATION_TIME_SEC = 60; - static final long TARGET_COMMIT_BUNDLE_BYTES = 32 << 20; - static final int MAX_COMMIT_QUEUE_BYTES = 500 << 20; // 500MB - static final int NUM_COMMIT_STREAMS = 1; - static final int GET_WORK_STREAM_TIMEOUT_MINUTES = 3; - static final Duration COMMIT_STREAM_TIMEOUT = Duration.standardMinutes(1); - private static final int DEFAULT_STATUS_PORT = 8081; - // Maximum size of the result of a GetWork request. private static final long MAX_GET_WORK_FETCH_BYTES = 64L << 20; // 64m - // Reserved ID for counter updates. // Matches kWindmillCounterUpdate in workflow_worker_service_multi_hubs.cc. private static final String WINDMILL_COUNTER_UPDATE_WORK_ID = "3"; - /** Maximum number of failure stacktraces to report in each update sent to backend. */ private static final int MAX_FAILURES_TO_REPORT_IN_UPDATE = 1000; - // TODO(https://github.com/apache/beam/issues/19632): Update throttling counters to use generic - // throttling-msecs metric. - public static final MetricName BIGQUERY_STREAMING_INSERT_THROTTLE_TIME = - MetricName.named( - "org.apache.beam.sdk.io.gcp.bigquery.BigQueryServicesImpl$DatasetServiceImpl", - "throttling-msecs"); - private static final Duration MAX_LOCAL_PROCESSING_RETRY_DURATION = Duration.standardMinutes(5); - - /** Returns whether an exception was caused by a {@link OutOfMemoryError}. */ - private static boolean isOutOfMemoryError(Throwable t) { - while (t != null) { - if (t instanceof OutOfMemoryError) { - return true; - } - t = t.getCause(); - } - return false; - } - - private static class KeyCommitTooLargeException extends Exception { - - public static KeyCommitTooLargeException causedBy( - String computationId, long byteLimit, WorkItemCommitRequest request) { - StringBuilder message = new StringBuilder(); - message.append("Commit request for stage "); - message.append(computationId); - message.append(" and key "); - message.append(request.getKey().toStringUtf8()); - if (request.getSerializedSize() > 0) { - message.append( - " has size " - + request.getSerializedSize() - + " which is more than the limit of " - + byteLimit); - } else { - message.append(" is larger than 2GB and cannot be processed"); - } - message.append( - ". This may be caused by grouping a very " - + "large amount of data in a single window without using Combine," - + " or by producing a large amount of data from a single input element."); - return new KeyCommitTooLargeException(message.toString()); - } - - private KeyCommitTooLargeException(String message) { - super(message); - } - } - - private static MapTask parseMapTask(String input) throws IOException { - return Transport.getJsonFactory().fromString(input, MapTask.class); - } - - public static void main(String[] args) throws Exception { - JvmInitializers.runOnStartup(); - - DataflowWorkerHarnessHelper.initializeLogging(StreamingDataflowWorker.class); - DataflowWorkerHarnessOptions options = - DataflowWorkerHarnessHelper.initializeGlobalStateAndPipelineOptions( - StreamingDataflowWorker.class); - DataflowWorkerHarnessHelper.configureLogging(options); - checkArgument( - options.isStreaming(), - "%s instantiated with options indicating batch use", - StreamingDataflowWorker.class.getName()); - - checkArgument( - !DataflowRunner.hasExperiment(options, "beam_fn_api"), - "%s cannot be main() class with beam_fn_api enabled", - StreamingDataflowWorker.class.getSimpleName()); - - StreamingDataflowWorker worker = - StreamingDataflowWorker.fromDataflowWorkerHarnessOptions(options); - - // Use the MetricsLogger container which is used by BigQueryIO to periodically log process-wide - // metrics. - MetricsEnvironment.setProcessWideContainer(new MetricsLogger(null)); - - JvmInitializers.runBeforeProcessing(options); - worker.startStatusPages(); - worker.start(); - } - - /** Bounded set of queues, with a maximum total weight. */ - private static class WeightedBoundedQueue { - - private final LinkedBlockingQueue queue = new LinkedBlockingQueue<>(); - private final int maxWeight; - private final Semaphore limit; - private final Function weigher; - - public WeightedBoundedQueue(int maxWeight, Function weigher) { - this.maxWeight = maxWeight; - this.limit = new Semaphore(maxWeight, true); - this.weigher = weigher; - } - - /** - * Adds the value to the queue, blocking if this would cause the overall weight to exceed the - * limit. - */ - public void put(V value) { - limit.acquireUninterruptibly(weigher.apply(value)); - queue.add(value); - } - - /** Returns and removes the next value, or null if there is no such value. */ - public @Nullable V poll() { - V result = queue.poll(); - if (result != null) { - limit.release(weigher.apply(result)); - } - return result; - } - - /** - * Retrieves and removes the head of this queue, waiting up to the specified wait time if - * necessary for an element to become available. - * - * @param timeout how long to wait before giving up, in units of {@code unit} - * @param unit a {@code TimeUnit} determining how to interpret the {@code timeout} parameter - * @return the head of this queue, or {@code null} if the specified waiting time elapses before - * an element is available - * @throws InterruptedException if interrupted while waiting - */ - public @Nullable V poll(long timeout, TimeUnit unit) throws InterruptedException { - V result = queue.poll(timeout, unit); - if (result != null) { - limit.release(weigher.apply(result)); - } - return result; - } - - /** Returns and removes the next value, or blocks until one is available. */ - public @Nullable V take() throws InterruptedException { - V result = queue.take(); - limit.release(weigher.apply(result)); - return result; - } - - /** Returns the current weight of the queue. */ - public int weight() { - return maxWeight - limit.availablePermits(); - } - - public int size() { - return queue.size(); - } - } - - // Value class for a queued commit. - static class Commit { - - private Windmill.WorkItemCommitRequest request; - private ComputationState computationState; - private Work work; - - public Commit( - Windmill.WorkItemCommitRequest request, ComputationState computationState, Work work) { - this.request = request; - assert request.getSerializedSize() > 0; - this.computationState = computationState; - this.work = work; - } - - public Windmill.WorkItemCommitRequest getRequest() { - return request; - } - - public ComputationState getComputationState() { - return computationState; - } - - public Work getWork() { - return work; - } - - public int getSize() { - return request.getSerializedSize(); - } - } - + private static final Random clientIdGenerator = new Random(); + final WindmillStateCache stateCache; // Maps from computation ids to per-computation state. private final ConcurrentMap computationMap = new ConcurrentHashMap<>(); private final WeightedBoundedQueue commitQueue = - new WeightedBoundedQueue<>( + WeightedBoundedQueue.create( MAX_COMMIT_QUEUE_BYTES, commit -> Math.min(MAX_COMMIT_QUEUE_BYTES, commit.getSize())); - // Cache of tokens to commit callbacks. // Using Cache with time eviction policy helps us to prevent memory leak when callback ids are // discarded by Dataflow service and calling commitCallback is best-effort. private final Cache commitCallbacks = CacheBuilder.newBuilder().expireAfterWrite(5L, TimeUnit.MINUTES).build(); - // Map of user state names to system state names. // TODO(drieber): obsolete stateNameMap. Use transformUserNameToStateFamily in // ComputationState instead. private final ConcurrentMap stateNameMap = new ConcurrentHashMap<>(); private final ConcurrentMap systemNameToComputationIdMap = new ConcurrentHashMap<>(); - - final WindmillStateCache stateCache; - private final ThreadFactory threadFactory; - private DataflowMapTaskExecutorFactory mapTaskExecutorFactory; private final BoundedQueueExecutor workUnitExecutor; private final WindmillServerStub windmillServer; private final Thread dispatchThread; @@ -414,16 +232,13 @@ public int getSize() { private final StreamingDataflowWorkerOptions options; private final boolean windmillServiceEnabled; private final long clientId; - private final MetricTrackingWindmillServerStub metricTrackingWindmillServer; private final CounterSet pendingDeltaCounters = new CounterSet(); private final CounterSet pendingCumulativeCounters = new CounterSet(); private final java.util.concurrent.ConcurrentLinkedQueue pendingMonitoringInfos = new ConcurrentLinkedQueue<>(); - // Map from stage name to StageInfo containing metrics container registry and per stage counters. private final ConcurrentMap stageInfoMap = new ConcurrentHashMap(); - // Built-in delta counters. private final Counter windmillShuffleBytesRead; private final Counter windmillStateBytesRead; @@ -432,136 +247,40 @@ public int getSize() { // Built-in cumulative counters. private final Counter javaHarnessUsedMemory; private final Counter javaHarnessMaxMemory; + private final Counter timeAtMaxActiveThreads; + private final Counter activeThreads; + private final Counter totalAllocatedThreads; private final Counter windmillMaxObservedWorkItemCommitBytes; private final Counter memoryThrashing; - private ScheduledExecutorService refreshWorkTimer; - private ScheduledExecutorService statusPageTimer; - private final boolean publishCounters; - private ScheduledExecutorService globalWorkerUpdatesTimer; - private int retryLocallyDelayMs = 10000; - - // Periodically fires a global config request to dataflow service. Only used when windmill service - // is enabled. - private ScheduledExecutorService globalConfigRefreshTimer; - private final MemoryMonitor memoryMonitor; private final Thread memoryMonitorThread; - private final WorkerStatusPages statusPages; - // Periodic sender of debug information to the debug capture service. - private DebugCapture.Manager debugCaptureManager = null; - // Limit on bytes sinked (committed) in a work item. private final long maxSinkBytes; // = MAX_SINK_BYTES unless disabled in options. - // Possibly overridden by streaming engine config. - private int maxWorkItemCommitBytes = Integer.MAX_VALUE; - private final EvictingQueue pendingFailuresToReport = - EvictingQueue.create(MAX_FAILURES_TO_REPORT_IN_UPDATE); - + EvictingQueue.create(MAX_FAILURES_TO_REPORT_IN_UPDATE); private final ReaderCache readerCache; - private final WorkUnitClient workUnitClient; private final CompletableFuture isDoneFuture; private final Function> mapTaskToNetwork; - - /** - * Sinks are marked 'full' in {@link StreamingModeExecutionContext} once the amount of data sinked - * (across all the sinks, if there are more than one) reaches this limit. This serves as hint for - * readers to stop producing more. This can be disabled with 'disable_limiting_bundle_sink_bytes' - * experiment. - */ - static final int MAX_SINK_BYTES = 10_000_000; - private final ReaderRegistry readerRegistry = ReaderRegistry.defaultRegistry(); private final SinkRegistry sinkRegistry = SinkRegistry.defaultRegistry(); - - private HotKeyLogger hotKeyLogger; - private final Supplier clock; private final Function executorSupplier; - - /** Contains a few of the stage specific fields. E.g. metrics container registry, counters etc. */ - private static class StageInfo { - - final String stageName; - final String systemName; - final MetricsContainerRegistry metricsContainerRegistry; - final StreamingModeExecutionStateRegistry executionStateRegistry; - final CounterSet deltaCounters; - final Counter throttledMsecs; - final Counter totalProcessingMsecs; - final Counter timerProcessingMsecs; - - StageInfo(String stageName, String systemName, StreamingDataflowWorker worker) { - this.stageName = stageName; - this.systemName = systemName; - metricsContainerRegistry = StreamingStepMetricsContainer.createRegistry(); - executionStateRegistry = new StreamingModeExecutionStateRegistry(worker); - NameContext nameContext = NameContext.create(stageName, null, systemName, null); - deltaCounters = new CounterSet(); - throttledMsecs = - deltaCounters.longSum( - StreamingPerStageSystemCounterNames.THROTTLED_MSECS.counterName(nameContext)); - totalProcessingMsecs = - deltaCounters.longSum( - StreamingPerStageSystemCounterNames.TOTAL_PROCESSING_MSECS.counterName(nameContext)); - timerProcessingMsecs = - deltaCounters.longSum( - StreamingPerStageSystemCounterNames.TIMER_PROCESSING_MSECS.counterName(nameContext)); - } - - List extractCounterUpdates() { - List counterUpdates = new ArrayList<>(); - Iterables.addAll( - counterUpdates, - StreamingStepMetricsContainer.extractMetricUpdates(metricsContainerRegistry)); - Iterables.addAll(counterUpdates, executionStateRegistry.extractUpdates(false)); - for (CounterUpdate counterUpdate : counterUpdates) { - translateKnownStepCounters(counterUpdate); - } - counterUpdates.addAll( - deltaCounters.extractModifiedDeltaUpdates(DataflowCounterUpdateExtractor.INSTANCE)); - return counterUpdates; - } - - // Checks if the step counter affects any per-stage counters. Currently 'throttled_millis' - // is the only counter updated. - private void translateKnownStepCounters(CounterUpdate stepCounterUpdate) { - CounterStructuredName structuredName = - stepCounterUpdate.getStructuredNameAndMetadata().getName(); - if ((THROTTLING_MSECS_METRIC_NAME.getNamespace().equals(structuredName.getOriginNamespace()) - && THROTTLING_MSECS_METRIC_NAME.getName().equals(structuredName.getName())) - || (BIGQUERY_STREAMING_INSERT_THROTTLE_TIME - .getNamespace() - .equals(structuredName.getOriginNamespace()) - && BIGQUERY_STREAMING_INSERT_THROTTLE_TIME - .getName() - .equals(structuredName.getName()))) { - long msecs = DataflowCounterUpdateExtractor.splitIntToLong(stepCounterUpdate.getInteger()); - if (msecs > 0) { - throttledMsecs.addValue(msecs); - } - } - } - } - - public static StreamingDataflowWorker fromDataflowWorkerHarnessOptions( - DataflowWorkerHarnessOptions options) throws IOException { - - return new StreamingDataflowWorker( - Collections.emptyList(), - IntrinsicMapTaskExecutorFactory.defaultFactory(), - new DataflowWorkUnitClient(options, LOG), - options.as(StreamingDataflowWorkerOptions.class), - true, - new HotKeyLogger(), - Instant::now, - (threadName) -> - Executors.newSingleThreadScheduledExecutor( - new ThreadFactoryBuilder().setNameFormat(threadName).build())); - } + private final DataflowMapTaskExecutorFactory mapTaskExecutorFactory; + private final HotKeyLogger hotKeyLogger; + // Periodic sender of debug information to the debug capture service. + private final DebugCapture.@Nullable Manager debugCaptureManager; + private ScheduledExecutorService refreshWorkTimer; + private ScheduledExecutorService statusPageTimer; + private ScheduledExecutorService globalWorkerUpdatesTimer; + private int retryLocallyDelayMs = 10000; + // Periodically fires a global config request to dataflow service. Only used when windmill service + // is enabled. + private ScheduledExecutorService globalConfigRefreshTimer; + // Possibly overridden by streaming engine config. + private int maxWorkItemCommitBytes = Integer.MAX_VALUE; @VisibleForTesting StreamingDataflowWorker( @@ -591,6 +310,8 @@ public static StreamingDataflowWorker fromDataflowWorkerHarnessOptions( if (windmillServiceEnabled) { this.debugCaptureManager = new DebugCapture.Manager(options, statusPages.getDebugCapturePages()); + } else { + this.debugCaptureManager = null; } this.windmillShuffleBytesRead = pendingDeltaCounters.longSum( @@ -610,6 +331,14 @@ public static StreamingDataflowWorker fromDataflowWorkerHarnessOptions( this.javaHarnessMaxMemory = pendingCumulativeCounters.longSum( StreamingSystemCounterNames.JAVA_HARNESS_MAX_MEMORY.counterName()); + this.timeAtMaxActiveThreads = + pendingCumulativeCounters.longSum( + StreamingSystemCounterNames.TIME_AT_MAX_ACTIVE_THREADS.counterName()); + this.activeThreads = + pendingCumulativeCounters.intSum(StreamingSystemCounterNames.ACTIVE_THREADS.counterName()); + this.totalAllocatedThreads = + pendingCumulativeCounters.intSum( + StreamingSystemCounterNames.TOTAL_ALLOCATED_THREADS.counterName()); this.windmillMaxObservedWorkItemCommitBytes = pendingCumulativeCounters.intMax( StreamingSystemCounterNames.WINDMILL_MAX_WORK_ITEM_COMMIT_BYTES.counterName()); @@ -696,6 +425,86 @@ public void run() { LOG.debug("maxWorkItemCommitBytes: {}", maxWorkItemCommitBytes); } + /** Returns whether an exception was caused by a {@link OutOfMemoryError}. */ + private static boolean isOutOfMemoryError(Throwable t) { + while (t != null) { + if (t instanceof OutOfMemoryError) { + return true; + } + t = t.getCause(); + } + return false; + } + + private static MapTask parseMapTask(String input) throws IOException { + return Transport.getJsonFactory().fromString(input, MapTask.class); + } + + public static void main(String[] args) throws Exception { + JvmInitializers.runOnStartup(); + + DataflowWorkerHarnessHelper.initializeLogging(StreamingDataflowWorker.class); + DataflowWorkerHarnessOptions options = + DataflowWorkerHarnessHelper.initializeGlobalStateAndPipelineOptions( + StreamingDataflowWorker.class); + DataflowWorkerHarnessHelper.configureLogging(options); + checkArgument( + options.isStreaming(), + "%s instantiated with options indicating batch use", + StreamingDataflowWorker.class.getName()); + + checkArgument( + !DataflowRunner.hasExperiment(options, "beam_fn_api"), + "%s cannot be main() class with beam_fn_api enabled", + StreamingDataflowWorker.class.getSimpleName()); + + StreamingDataflowWorker worker = + StreamingDataflowWorker.fromDataflowWorkerHarnessOptions(options); + + // Use the MetricsLogger container which is used by BigQueryIO to periodically log process-wide + // metrics. + MetricsEnvironment.setProcessWideContainer(new MetricsLogger(null)); + + // When enabled, the Pipeline will record Per-Worker metrics that will be piped to WMW. + StreamingStepMetricsContainer.setEnablePerWorkerMetrics( + options.isEnableStreamingEngine() + && DataflowRunner.hasExperiment(options, "enable_per_worker_metrics")); + + JvmInitializers.runBeforeProcessing(options); + worker.startStatusPages(); + worker.start(); + } + + public static StreamingDataflowWorker fromDataflowWorkerHarnessOptions( + DataflowWorkerHarnessOptions options) throws IOException { + + return new StreamingDataflowWorker( + Collections.emptyList(), + IntrinsicMapTaskExecutorFactory.defaultFactory(), + new DataflowWorkUnitClient(options, LOG), + options.as(StreamingDataflowWorkerOptions.class), + true, + new HotKeyLogger(), + Instant::now, + (threadName) -> + Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat(threadName).build())); + } + + private static void sleep(int millis) { + Uninterruptibles.sleepUninterruptibly(millis, TimeUnit.MILLISECONDS); + } + + /** Sets the stage name and workId of the current Thread for logging. */ + private static void setUpWorkLoggingContext(Windmill.WorkItem workItem, String computationId) { + String workIdBuilder = + Long.toHexString(workItem.getShardingKey()) + + '-' + + Long.toHexString(workItem.getWorkToken()); + DataflowWorkerLoggingMDC.setWorkId(workIdBuilder); + DataflowWorkerLoggingMDC.setStageName(computationId); + } + private int chooseMaximumNumberOfThreads() { if (options.getNumberOfWorkerHarnessThreads() != 0) { return options.getNumberOfWorkerHarnessThreads(); @@ -805,7 +614,7 @@ public void run() { + options.getWorkerId() + "_" + page.pageName() - + timestamp.toString()) + + timestamp) .replaceAll("/", "_")); writer = new PrintWriter(outputFile, UTF_8.name()); page.captureData(writer); @@ -933,10 +742,6 @@ private synchronized void addComputation( } } - private static void sleep(int millis) { - Uninterruptibles.sleepUninterruptibly(millis, TimeUnit.MILLISECONDS); - } - /** * If the computation is not yet known about, configuration for it will be fetched. This can still * return null if there is no configuration fetched for the computation. @@ -987,7 +792,11 @@ private void dispatchLoop() { computationWork.getDependentRealtimeInputWatermark()); for (final Windmill.WorkItem workItem : computationWork.getWorkList()) { scheduleWorkItem( - computationState, inputDataWatermark, synchronizedProcessingTime, workItem); + computationState, + inputDataWatermark, + synchronizedProcessingTime, + workItem, + /* getWorkStreamLatencies= */ Collections.emptyList()); } } } @@ -1005,13 +814,15 @@ void streamingDispatchLoop() { (String computation, Instant inputDataWatermark, Instant synchronizedProcessingTime, - Windmill.WorkItem workItem) -> { + Windmill.WorkItem workItem, + Collection getWorkStreamLatencies) -> { memoryMonitor.waitForResources("GetWork"); scheduleWorkItem( getComputationState(computation), inputDataWatermark, synchronizedProcessingTime, - workItem); + workItem, + getWorkStreamLatencies); }); try { // Reconnect every now and again to enable better load balancing. @@ -1030,128 +841,28 @@ private void scheduleWorkItem( final ComputationState computationState, final Instant inputDataWatermark, final Instant synchronizedProcessingTime, - final Windmill.WorkItem workItem) { + final Windmill.WorkItem workItem, + final Collection getWorkStreamLatencies) { Preconditions.checkNotNull(inputDataWatermark); // May be null if output watermark not yet known. final @Nullable Instant outputDataWatermark = WindmillTimeUtils.windmillToHarnessWatermark(workItem.getOutputDataWatermark()); Preconditions.checkState( outputDataWatermark == null || !outputDataWatermark.isAfter(inputDataWatermark)); - Work work = - new Work(workItem, clock) { - @Override - public void run() { - process( - computationState, - inputDataWatermark, - outputDataWatermark, - synchronizedProcessingTime, - this); - } - }; + Work scheduledWork = + Work.create( + workItem, + clock, + getWorkStreamLatencies, + work -> + process( + computationState, + inputDataWatermark, + outputDataWatermark, + synchronizedProcessingTime, + work)); computationState.activateWork( - ShardedKey.create(workItem.getKey(), workItem.getShardingKey()), work); - } - - @AutoValue - abstract static class ShardedKey { - - public static ShardedKey create(ByteString key, long shardingKey) { - return new AutoValue_StreamingDataflowWorker_ShardedKey(key, shardingKey); - } - - public abstract ByteString key(); - - public abstract long shardingKey(); - - @Override - public final String toString() { - ByteString keyToDisplay = key(); - if (keyToDisplay.size() > 100) { - keyToDisplay = keyToDisplay.substring(0, 100); - } - return String.format("%016x-%s", shardingKey(), TextFormat.escapeBytes(keyToDisplay)); - } - } - - abstract static class Work implements Runnable { - - enum State { - QUEUED(Windmill.LatencyAttribution.State.QUEUED), - PROCESSING(Windmill.LatencyAttribution.State.ACTIVE), - READING(Windmill.LatencyAttribution.State.READING), - COMMIT_QUEUED(Windmill.LatencyAttribution.State.COMMITTING), - COMMITTING(Windmill.LatencyAttribution.State.COMMITTING); - - private final Windmill.LatencyAttribution.State latencyAttributionState; - - private State(Windmill.LatencyAttribution.State latencyAttributionState) { - this.latencyAttributionState = latencyAttributionState; - } - - Windmill.LatencyAttribution.State toLatencyAttributionState() { - return latencyAttributionState; - } - } - - private final Windmill.WorkItem workItem; - private final Supplier clock; - private final Instant startTime; - private Instant stateStartTime; - private State state; - private Map totalDurationPerState; - - public Work(Windmill.WorkItem workItem, Supplier clock) { - this.workItem = workItem; - this.clock = clock; - this.startTime = this.stateStartTime = clock.get(); - this.state = State.QUEUED; - this.totalDurationPerState = new EnumMap<>(Windmill.LatencyAttribution.State.class); - } - - public Windmill.WorkItem getWorkItem() { - return workItem; - } - - public Instant getStartTime() { - return startTime; - } - - public State getState() { - return state; - } - - public void setState(State state) { - Instant now = clock.get(); - totalDurationPerState.compute( - this.state.toLatencyAttributionState(), - (s, d) -> new Duration(this.stateStartTime, now).plus(d == null ? Duration.ZERO : d)); - this.state = state; - this.stateStartTime = now; - } - - public Instant getStateStartTime() { - return stateStartTime; - } - - public Iterable getLatencyAttributionList() { - List list = new ArrayList<>(); - for (Windmill.LatencyAttribution.State state : Windmill.LatencyAttribution.State.values()) { - Duration duration = totalDurationPerState.getOrDefault(state, Duration.ZERO); - if (state == this.state.toLatencyAttributionState()) { - duration = duration.plus(new Duration(this.stateStartTime, clock.get())); - } - if (duration.equals(Duration.ZERO)) { - continue; - } - list.add( - Windmill.LatencyAttribution.newBuilder() - .setState(state) - .setTotalDurationMillis(duration.getMillis()) - .build()); - } - return list; - } + ShardedKey.create(workItem.getKey(), workItem.getShardingKey()), scheduledWork); } /** @@ -1221,15 +932,9 @@ private void process( final String computationId = computationState.getComputationId(); final ByteString key = workItem.getKey(); work.setState(State.PROCESSING); - { - StringBuilder workIdBuilder = new StringBuilder(33); - workIdBuilder.append(Long.toHexString(workItem.getShardingKey())); - workIdBuilder.append('-'); - workIdBuilder.append(Long.toHexString(workItem.getWorkToken())); - DataflowWorkerLoggingMDC.setWorkId(workIdBuilder.toString()); - } - DataflowWorkerLoggingMDC.setStageName(computationId); + setUpWorkLoggingContext(workItem, computationId); + LOG.debug("Starting processing for {}:\n{}", computationId, work); Windmill.WorkItemCommitRequest.Builder outputBuilder = initializeOutputBuilder(key, workItem); @@ -1240,7 +945,7 @@ private void process( if (workItem.getSourceState().getOnlyFinalize()) { outputBuilder.setSourceStateUpdates(Windmill.SourceState.newBuilder().setOnlyFinalize(true)); work.setState(State.COMMIT_QUEUED); - commitQueue.put(new Commit(outputBuilder.build(), computationState, work)); + commitQueue.put(Commit.create(outputBuilder.build(), computationState, work)); return; } @@ -1250,7 +955,7 @@ private void process( StageInfo stageInfo = stageInfoMap.computeIfAbsent( - mapTask.getStageName(), s -> new StageInfo(s, mapTask.getSystemName(), this)); + mapTask.getStageName(), s -> StageInfo.create(s, mapTask.getSystemName(), this)); ExecutionState executionState = null; String counterName = "dataflow_source_bytes_processed-" + mapTask.getSystemName(); @@ -1275,12 +980,14 @@ private void process( DataflowExecutionContext.DataflowExecutionStateTracker executionStateTracker = new DataflowExecutionContext.DataflowExecutionStateTracker( ExecutionStateSampler.instance(), - stageInfo.executionStateRegistry.getState( - NameContext.forStage(mapTask.getStageName()), - "other", - null, - ScopedProfiler.INSTANCE.emptyScope()), - stageInfo.deltaCounters, + stageInfo + .executionStateRegistry() + .getState( + NameContext.forStage(mapTask.getStageName()), + "other", + null, + ScopedProfiler.INSTANCE.emptyScope()), + stageInfo.deltaCounters(), options, computationId); StreamingModeExecutionContext context = @@ -1292,9 +999,9 @@ private void process( ? computationState.getTransformUserNameToStateFamily() : stateNameMap, stateCache.forComputation(computationId), - stageInfo.metricsContainerRegistry, + stageInfo.metricsContainerRegistry(), executionStateTracker, - stageInfo.executionStateRegistry, + stageInfo.executionStateRegistry(), maxSinkBytes); DataflowMapTaskExecutor mapTaskExecutor = mapTaskExecutorFactory.create( @@ -1341,8 +1048,18 @@ private void process( .setSamplingPeriod(100) .countBytes(counterName)); } - executionState = - new ExecutionState(mapTaskExecutor, context, keyCoder, executionStateTracker); + + ExecutionState.Builder executionStateBuilder = + ExecutionState.builder() + .setWorkExecutor(mapTaskExecutor) + .setContext(context) + .setExecutionStateTracker(executionStateTracker); + + if (keyCoder != null) { + executionStateBuilder.setKeyCoder(keyCoder); + } + + executionState = executionStateBuilder.build(); } WindmillStateReader stateReader = @@ -1369,10 +1086,10 @@ public void close() { // // The coder type that will be present is: // WindowedValueCoder(TimerOrElementCoder(KvCoder)) - @Nullable Coder keyCoder = executionState.getKeyCoder(); + Optional> keyCoder = executionState.keyCoder(); @Nullable Object executionKey = - keyCoder == null ? null : keyCoder.decode(key.newInput(), Coder.Context.OUTER); + !keyCoder.isPresent() ? null : keyCoder.get().decode(key.newInput(), Coder.Context.OUTER); if (workItem.hasHotKeyInfo()) { Windmill.HotKeyInfo hotKeyInfo = workItem.getHotKeyInfo(); @@ -1381,7 +1098,7 @@ public void close() { // The MapTask instruction is ordered by dependencies, such that the first element is // always going to be the shuffle task. String stepName = computationState.getMapTask().getInstructions().get(0).getName(); - if (options.isHotKeyLoggingEnabled() && keyCoder != null) { + if (options.isHotKeyLoggingEnabled() && keyCoder.isPresent()) { hotKeyLogger.logHotKeyDetection(stepName, hotKeyAge, executionKey); } else { hotKeyLogger.logHotKeyDetection(stepName, hotKeyAge); @@ -1389,7 +1106,7 @@ public void close() { } executionState - .getContext() + .context() .start( executionKey, workItem, @@ -1401,13 +1118,13 @@ public void close() { outputBuilder); // Blocks while executing work. - executionState.getWorkExecutor().execute(); + executionState.workExecutor().execute(); // Reports source bytes processed to workitemcommitrequest if available. try { long sourceBytesProcessed = 0; HashMap counters = - ((DataflowMapTaskExecutor) executionState.getWorkExecutor()) + ((DataflowMapTaskExecutor) executionState.workExecutor()) .getReadOperation() .receivers[0] .getOutputCounters(); @@ -1421,9 +1138,9 @@ public void close() { } Iterables.addAll( - this.pendingMonitoringInfos, executionState.getWorkExecutor().extractMetricUpdates()); + this.pendingMonitoringInfos, executionState.workExecutor().extractMetricUpdates()); - commitCallbacks.putAll(executionState.getContext().flushState()); + commitCallbacks.putAll(executionState.context().flushState()); // Release the execution state for another thread to use. computationState.getExecutionStateQueue().offer(executionState); @@ -1431,6 +1148,7 @@ public void close() { // Add the output to the commit queue. work.setState(State.COMMIT_QUEUED); + outputBuilder.addAllPerWorkItemLatencyAttributions(work.getLatencyAttributions()); WorkItemCommitRequest commitRequest = outputBuilder.build(); int byteLimit = maxWorkItemCommitBytes; @@ -1451,10 +1169,15 @@ public void close() { commitRequest = buildWorkItemTruncationRequest(key, workItem, estimatedCommitSize); } - commitQueue.put(new Commit(commitRequest, computationState, work)); + commitQueue.put(Commit.create(commitRequest, computationState, work)); // Compute shuffle and state byte statistics these will be flushed asynchronously. - long stateBytesWritten = outputBuilder.clearOutputMessages().build().getSerializedSize(); + long stateBytesWritten = + outputBuilder + .clearOutputMessages() + .clearPerWorkItemLatencyAttributions() + .build() + .getSerializedSize(); long shuffleBytesRead = 0; for (Windmill.InputMessageBundle bundle : workItem.getMessageBundlesList()) { for (Windmill.Message message : bundle.getMessagesList()) { @@ -1470,8 +1193,8 @@ public void close() { } catch (Throwable t) { if (executionState != null) { try { - executionState.getContext().invalidateCache(); - executionState.getWorkExecutor().close(); + executionState.context().invalidateCache(); + executionState.workExecutor().close(); } catch (Exception e) { LOG.warn("Failed to close map task executor: ", e); } finally { @@ -1492,7 +1215,7 @@ public void close() { } else { LastExceptionDataProvider.reportException(t); LOG.debug("Failed work: {}", work); - Duration elapsedTimeSinceStart = new Duration(clock.get(), work.getStartTime()); + Duration elapsedTimeSinceStart = new Duration(work.getStartTime(), clock.get()); if (!reportFailure(computationId, workItem, t)) { LOG.error( "Execution of work for computation '{}' on key '{}' failed with uncaught exception, " @@ -1536,7 +1259,7 @@ public void close() { } else { // Consider the item invalid. It will eventually be retried by Windmill if it still needs to // be processed. - computationState.completeWork( + computationState.completeWorkAndScheduleNextWorkForKey( ShardedKey.create(key, workItem.getShardingKey()), workItem.getWorkToken()); } } finally { @@ -1544,7 +1267,7 @@ public void close() { // work items causing exceptions are also accounted in time spent. long processingTimeMsecs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - processingStartTimeNanos); - stageInfo.totalProcessingMsecs.addValue(processingTimeMsecs); + stageInfo.totalProcessingMsecs().addValue(processingTimeMsecs); // Attribute all the processing to timers if the work item contains any timers. // Tests show that work items rarely contain both timers and message bundles. It should @@ -1552,7 +1275,7 @@ public void close() { // Another option: Derive time split between messages and timers based on recent totals. // either here or in DFE. if (work.getWorkItem().hasTimers()) { - stageInfo.timerProcessingMsecs.addValue(processingTimeMsecs); + stageInfo.timerProcessingMsecs().addValue(processingTimeMsecs); } DataflowWorkerLoggingMDC.setWorkId(null); @@ -1585,8 +1308,8 @@ private void commitLoop() { continue; } while (commit != null) { - ComputationState computationState = commit.getComputationState(); - commit.getWork().setState(State.COMMITTING); + ComputationState computationState = commit.computationState(); + commit.work().setState(Work.State.COMMITTING); Windmill.ComputationCommitWorkRequest.Builder computationRequestBuilder = computationRequestMap.get(computationState); if (computationRequestBuilder == null) { @@ -1594,7 +1317,7 @@ private void commitLoop() { computationRequestBuilder.setComputationId(computationState.getComputationId()); computationRequestMap.put(computationState, computationRequestBuilder); } - computationRequestBuilder.addRequests(commit.getRequest()); + computationRequestBuilder.addRequests(commit.request()); // Send the request if we've exceeded the bytes or there is no more // pending work. commitBytes is a long, so this cannot overflow. commitBytes += commit.getSize(); @@ -1612,7 +1335,7 @@ private void commitLoop() { computationRequestMap.entrySet()) { ComputationState computationState = entry.getKey(); for (Windmill.WorkItemCommitRequest workRequest : entry.getValue().getRequestsList()) { - computationState.completeWork( + computationState.completeWorkAndScheduleNextWorkForKey( ShardedKey.create(workRequest.getKey(), workRequest.getShardingKey()), workRequest.getWorkToken()); } @@ -1623,34 +1346,34 @@ private void commitLoop() { // Adds the commit to the commitStream if it fits, returning true iff it is consumed. private boolean addCommitToStream(Commit commit, CommitWorkStream commitStream) { Preconditions.checkNotNull(commit); - final ComputationState state = commit.getComputationState(); - final Windmill.WorkItemCommitRequest request = commit.getRequest(); + final ComputationState state = commit.computationState(); + final Windmill.WorkItemCommitRequest request = commit.request(); final int size = commit.getSize(); - commit.getWork().setState(State.COMMITTING); + commit.work().setState(Work.State.COMMITTING); activeCommitBytes.addAndGet(size); if (commitStream.commitWorkItem( - state.computationId, + state.getComputationId(), request, (Windmill.CommitStatus status) -> { if (status != Windmill.CommitStatus.OK) { readerCache.invalidateReader( WindmillComputationKey.create( - state.computationId, request.getKey(), request.getShardingKey())); + state.getComputationId(), request.getKey(), request.getShardingKey())); stateCache - .forComputation(state.computationId) + .forComputation(state.getComputationId()) .invalidate(request.getKey(), request.getShardingKey()); } activeCommitBytes.addAndGet(-size); // This may throw an exception if the commit was not active, which is possible if it // was deemed stuck. - state.completeWork( + state.completeWorkAndScheduleNextWorkForKey( ShardedKey.create(request.getKey(), request.getShardingKey()), request.getWorkToken()); })) { return true; } else { // Back out the stats changes since the commit wasn't consumed. - commit.getWork().setState(State.COMMIT_QUEUED); + commit.work().setState(Work.State.COMMIT_QUEUED); activeCommitBytes.addAndGet(-size); return false; } @@ -1664,7 +1387,7 @@ private Commit batchCommitsToStream(CommitWorkStream commitStream) { Commit commit; try { if (commits < 5) { - commit = commitQueue.poll(10 - 2 * commits, TimeUnit.MILLISECONDS); + commit = commitQueue.poll(10 - 2L * commits, TimeUnit.MILLISECONDS); } else { commit = commitQueue.poll(); } @@ -1681,8 +1404,8 @@ private Commit batchCommitsToStream(CommitWorkStream commitStream) { } private void streamingCommitLoop() { - StreamPool streamPool = - new StreamPool<>( + WindmillStreamPool streamPool = + WindmillStreamPool.create( NUM_COMMIT_STREAMS, COMMIT_STREAM_TIMEOUT, windmillServer::commitWorkStream); Commit initialCommit = null; while (running.get()) { @@ -1751,7 +1474,8 @@ private void getConfigFromWindmill(String computation) { addComputation( computationId, mapTask, - transformUserNameToStateFamilyByComputationId.get(computationId)); + transformUserNameToStateFamilyByComputationId.getOrDefault( + computationId, ImmutableMap.of())); } catch (IOException e) { LOG.warn("Parsing MapTask failed: {}", serializedMapTask); LOG.warn("Error: ", e); @@ -1769,13 +1493,12 @@ private void getConfigFromWindmill(String computation) { * @throws IOException if the RPC fails. */ private void getConfigFromDataflowService(@Nullable String computation) throws IOException { - Optional workItem; - if (computation != null) { - workItem = workUnitClient.getStreamingConfigWorkItem(computation); - } else { - workItem = workUnitClient.getGlobalStreamingConfigWorkItem(); - } - if (workItem == null || !workItem.isPresent() || workItem.get() == null) { + Optional workItem = + computation != null + ? workUnitClient.getStreamingConfigWorkItem(computation) + : workUnitClient.getGlobalStreamingConfigWorkItem(); + + if (!workItem.isPresent()) { return; } StreamingConfigTask config = workItem.get().getStreamingConfigTask(); @@ -1802,7 +1525,8 @@ private void getConfigFromDataflowService(@Nullable String computation) throws I addComputation( computationConfig.getComputationId(), mapTask, - computationConfig.getTransformUserNameToStateFamily()); + Optional.ofNullable(computationConfig.getTransformUserNameToStateFamily()) + .orElseGet(ImmutableMap::of)); } } @@ -1989,9 +1713,19 @@ private void updateVMMetrics() { javaHarnessMaxMemory.addValue(maxMemory); } + private void updateThreadMetrics() { + timeAtMaxActiveThreads.getAndReset(); + timeAtMaxActiveThreads.addValue(workUnitExecutor.allThreadsActiveTime()); + activeThreads.getAndReset(); + activeThreads.addValue(workUnitExecutor.activeCount()); + totalAllocatedThreads.getAndReset(); + totalAllocatedThreads.addValue(chooseMaximumNumberOfThreads()); + } + @VisibleForTesting public void reportPeriodicWorkerUpdates() { updateVMMetrics(); + updateThreadMetrics(); try { sendWorkerUpdatesToDataflowService(pendingDeltaCounters, pendingCumulativeCounters); } catch (IOException e) { @@ -2132,277 +1866,6 @@ private void invalidateStuckCommits() { } } - /** - * Class representing the state of a computation. - * - *

    This class is synchronized, but only used from the dispatch and commit threads, so should - * not be heavily contended. Still, blocking work should not be done by it. - */ - static class ComputationState implements AutoCloseable { - - private final String computationId; - private final MapTask mapTask; - private final ImmutableMap transformUserNameToStateFamily; - // Map from key to work for the key. The first item in the queue is - // actively processing. Synchronized by itself. - private final Map> activeWork = new HashMap<>(); - private final BoundedQueueExecutor executor; - private final ConcurrentLinkedQueue executionStateQueue = - new ConcurrentLinkedQueue<>(); - private final WindmillStateCache.ForComputation computationStateCache; - - public ComputationState( - String computationId, - MapTask mapTask, - BoundedQueueExecutor executor, - Map transformUserNameToStateFamily, - WindmillStateCache.ForComputation computationStateCache) { - this.computationId = computationId; - this.mapTask = mapTask; - this.executor = executor; - this.transformUserNameToStateFamily = - transformUserNameToStateFamily != null - ? ImmutableMap.copyOf(transformUserNameToStateFamily) - : ImmutableMap.of(); - this.computationStateCache = computationStateCache; - Preconditions.checkNotNull(mapTask.getStageName()); - Preconditions.checkNotNull(mapTask.getSystemName()); - } - - public String getComputationId() { - return computationId; - } - - public MapTask getMapTask() { - return mapTask; - } - - public ImmutableMap getTransformUserNameToStateFamily() { - return transformUserNameToStateFamily; - } - - public ConcurrentLinkedQueue getExecutionStateQueue() { - return executionStateQueue; - } - - /** Mark the given shardedKey and work as active. */ - public boolean activateWork(ShardedKey shardedKey, Work work) { - synchronized (activeWork) { - Deque queue = activeWork.get(shardedKey); - if (queue != null) { - Preconditions.checkState(!queue.isEmpty()); - // Ensure we don't already have this work token queueud. - for (Work queuedWork : queue) { - if (queuedWork.getWorkItem().getWorkToken() == work.getWorkItem().getWorkToken()) { - return false; - } - } - // Queue the work for later processing. - queue.addLast(work); - return true; - } else { - queue = new ArrayDeque<>(); - queue.addLast(work); - activeWork.put(shardedKey, queue); - // Fall through to execute without the lock held. - } - } - executor.execute(work, work.getWorkItem().getSerializedSize()); - return true; - } - - /** - * Marks the work for the given shardedKey as complete. Schedules queued work for the key if - * any. - */ - public void completeWork(ShardedKey shardedKey, long workToken) { - Work nextWork; - synchronized (activeWork) { - Queue queue = activeWork.get(shardedKey); - if (queue == null) { - // Work may have been completed due to clearing of stuck commits. - LOG.warn( - "Unable to complete inactive work for key {} and token {}.", shardedKey, workToken); - return; - } - Work completedWork = queue.peek(); - // avoid Preconditions.checkState here to prevent eagerly evaluating the - // format string parameters for the error message. - if (completedWork == null) { - throw new IllegalStateException( - String.format( - "Active key %s without work, expected token %d", shardedKey, workToken)); - } - if (completedWork.getWorkItem().getWorkToken() != workToken) { - // Work may have been completed due to clearing of stuck commits. - LOG.warn( - "Unable to complete due to token mismatch for key {} and token {}, actual token was {}.", - shardedKey, - workToken, - completedWork.getWorkItem().getWorkToken()); - return; - } - queue.remove(); // We consumed the matching work item. - nextWork = queue.peek(); - if (nextWork == null) { - Preconditions.checkState(queue == activeWork.remove(shardedKey)); - } - } - if (nextWork != null) { - executor.forceExecute(nextWork, nextWork.getWorkItem().getSerializedSize()); - } - } - - public void invalidateStuckCommits(Instant stuckCommitDeadline) { - synchronized (activeWork) { - // Determine the stuck commit keys but complete them outside of iterating over - // activeWork as completeWork may delete the entry from activeWork. - Map stuckCommits = new HashMap<>(); - for (Map.Entry> entry : activeWork.entrySet()) { - ShardedKey shardedKey = entry.getKey(); - Work work = entry.getValue().peek(); - if (work.getState() == State.COMMITTING - && work.getStateStartTime().isBefore(stuckCommitDeadline)) { - LOG.error( - "Detected key {} stuck in COMMITTING state since {}, completing it with error.", - shardedKey, - work.getStateStartTime()); - stuckCommits.put(shardedKey, work.getWorkItem().getWorkToken()); - } - } - for (Map.Entry stuckCommit : stuckCommits.entrySet()) { - computationStateCache.invalidate( - stuckCommit.getKey().key(), stuckCommit.getKey().shardingKey()); - completeWork(stuckCommit.getKey(), stuckCommit.getValue()); - } - } - } - - /** Adds any work started before the refreshDeadline to the GetDataRequest builder. */ - public List getKeysToRefresh(Instant refreshDeadline) { - List result = new ArrayList<>(); - synchronized (activeWork) { - for (Map.Entry> entry : activeWork.entrySet()) { - ShardedKey shardedKey = entry.getKey(); - for (Work work : entry.getValue()) { - if (work.getStartTime().isBefore(refreshDeadline)) { - result.add( - Windmill.KeyedGetDataRequest.newBuilder() - .setKey(shardedKey.key()) - .setShardingKey(shardedKey.shardingKey()) - .setWorkToken(work.getWorkItem().getWorkToken()) - .addAllLatencyAttribution(work.getLatencyAttributionList()) - .build()); - } - } - } - } - return result; - } - - private String elapsedString(Instant start, Instant end) { - Duration activeFor = new Duration(start, end); - // Duration's toString always starts with "PT"; remove that here. - return activeFor.toString().substring(2); - } - - public void printActiveWork(PrintWriter writer) { - final Instant now = Instant.now(); - // The max number of keys in COMMITTING or COMMIT_QUEUED status to be shown. - final int maxCommitPending = 50; - int commitPendingCount = 0; - writer.println( - "

    "); - writer.println( - ""); - // We use a StringBuilder in the synchronized section to buffer writes since the provided - // PrintWriter may block when flushing. - StringBuilder builder = new StringBuilder(); - synchronized (activeWork) { - for (Map.Entry> entry : activeWork.entrySet()) { - Queue queue = entry.getValue(); - Preconditions.checkNotNull(queue); - Work work = queue.peek(); - Preconditions.checkNotNull(work); - Windmill.WorkItem workItem = work.getWorkItem(); - State state = work.getState(); - if (state == State.COMMITTING || state == State.COMMIT_QUEUED) { - if (++commitPendingCount >= maxCommitPending) { - continue; - } - } - builder.append(""); - builder.append("\n"); - } - } - writer.print(builder.toString()); - writer.println("
    KeyTokenQueuedActive ForStateState Active For
    "); - builder.append(String.format("%016x", workItem.getShardingKey())); - builder.append(""); - builder.append(String.format("%016x", workItem.getWorkToken())); - builder.append(""); - builder.append(queue.size() - 1); - builder.append(""); - builder.append(elapsedString(work.getStartTime(), now)); - builder.append(""); - builder.append(state); - builder.append(""); - builder.append(elapsedString(work.getStateStartTime(), now)); - builder.append("
    "); - if (commitPendingCount >= maxCommitPending) { - writer.println("
    "); - writer.print("Skipped keys in COMMITTING/COMMIT_QUEUED: "); - writer.println(commitPendingCount - maxCommitPending); - writer.println("
    "); - } - } - - @Override - public void close() throws Exception { - ExecutionState executionState; - while ((executionState = executionStateQueue.poll()) != null) { - executionState.getWorkExecutor().close(); - } - executionStateQueue.clear(); - } - } - - private static class ExecutionState { - - public final DataflowWorkExecutor workExecutor; - public final StreamingModeExecutionContext context; - public final @Nullable Coder keyCoder; - private final ExecutionStateTracker executionStateTracker; - - public ExecutionState( - DataflowWorkExecutor workExecutor, - StreamingModeExecutionContext context, - Coder keyCoder, - ExecutionStateTracker executionStateTracker) { - this.workExecutor = workExecutor; - this.context = context; - this.keyCoder = keyCoder; - this.executionStateTracker = executionStateTracker; - } - - public DataflowWorkExecutor getWorkExecutor() { - return workExecutor; - } - - public StreamingModeExecutionContext getContext() { - return context; - } - - public ExecutionStateTracker getExecutionStateTracker() { - return executionStateTracker; - } - - public @Nullable Coder getKeyCoder() { - return keyCoder; - } - } - private class HarnessDataProvider implements StatusDataProvider { @Override diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingGroupAlsoByWindowsDoFns.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingGroupAlsoByWindowsDoFns.java index dc098e7c942dc..370b21b49348f 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingGroupAlsoByWindowsDoFns.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingGroupAlsoByWindowsDoFns.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.util.AppliedCombineFn; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; /** {@link GroupAlsoByWindowFn}'s that merge windows and groups elements in those windows. */ public abstract class StreamingGroupAlsoByWindowsDoFns { diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingKeyedWorkItemSideInputDoFnRunner.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingKeyedWorkItemSideInputDoFnRunner.java index 53db340670da0..36a93f6282bd4 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingKeyedWorkItemSideInputDoFnRunner.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingKeyedWorkItemSideInputDoFnRunner.java @@ -35,9 +35,9 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.joda.time.Instant; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingModeExecutionContext.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingModeExecutionContext.java index c27a3804b5ebd..c8fa6e6dfb783 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingModeExecutionContext.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingModeExecutionContext.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.services.dataflow.model.CounterUpdate; import com.google.api.services.dataflow.model.SideInputInfo; @@ -49,6 +49,9 @@ import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GlobalDataId; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GlobalDataRequest; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.Timer; +import org.apache.beam.runners.dataflow.worker.windmill.state.WindmillStateCache; +import org.apache.beam.runners.dataflow.worker.windmill.state.WindmillStateInternals; +import org.apache.beam.runners.dataflow.worker.windmill.state.WindmillStateReader; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.io.UnboundedSource; import org.apache.beam.sdk.metrics.MetricsContainer; @@ -58,17 +61,17 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashBasedTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.PeekingIterator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Table; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBasedTable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.PeekingIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Table; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; @@ -82,7 +85,12 @@ public class StreamingModeExecutionContext extends DataflowExecutionContext { private static final Logger LOG = LoggerFactory.getLogger(StreamingModeExecutionContext.class); - + private final String computationId; + private final Map, Map> sideInputCache; + // Per-key cache of active Reader objects in use by this process. + private final ImmutableMap stateNameMap; + private final WindmillStateCache.ForComputation stateCache; + private final ReaderCache readerCache; /** * The current user-facing key for this execution context. * @@ -94,20 +102,12 @@ public class StreamingModeExecutionContext extends DataflowExecutionContext, Map> sideInputCache; - - // Per-key cache of active Reader objects in use by this process. - private final ImmutableMap stateNameMap; - private final WindmillStateCache.ForComputation stateCache; - private Windmill.WorkItem work; private WindmillComputationKey computationKey; private StateFetcher stateFetcher; private Windmill.WorkItemCommitRequest.Builder outputBuilder; private UnboundedSource.UnboundedReader activeReader; private volatile long backlogBytes; - private final ReaderCache readerCache; public StreamingModeExecutionContext( CounterFactory counterFactory, @@ -133,86 +133,6 @@ public StreamingModeExecutionContext( this.backlogBytes = UnboundedSource.UnboundedReader.BACKLOG_UNKNOWN; } - /** - * Execution states in Streaming are shared between multiple map-task executors. Thus this class - * needs to be thread safe for multiple writers. A single stage could have have multiple executors - * running concurrently. - */ - public static class StreamingModeExecutionState - extends DataflowOperationContext.DataflowExecutionState { - - // AtomicLong is used because this value is written in two places: - // 1. The sampling thread calls takeSample to increment the time spent in this state - // 2. The reporting thread calls extractUpdate which reads the current sum *AND* sets it to 0. - private final AtomicLong totalMillisInState = new AtomicLong(); - - // The worker that created this state. Used to report lulls back to the worker. - @SuppressWarnings("unused") // Affects a public api - private final StreamingDataflowWorker worker; - - public StreamingModeExecutionState( - NameContext nameContext, - String stateName, - MetricsContainer metricsContainer, - ProfileScope profileScope, - StreamingDataflowWorker worker) { - // TODO: Take in the requesting step name and side input index for streaming. - super(nameContext, stateName, null, null, metricsContainer, profileScope); - this.worker = worker; - } - - /** - * Take sample is only called by the ExecutionStateSampler thread. It is the only place that - * increments totalMillisInState, however the reporting thread periodically calls extractUpdate - * which will read the sum and reset it to 0, so totalMillisInState does have multiple writers. - */ - @Override - public void takeSample(long millisSinceLastSample) { - totalMillisInState.addAndGet(millisSinceLastSample); - } - - /** - * Extract updates in the form of a {@link CounterUpdate}. - * - *

    Non-final updates are extracted periodically and report the physical value as a delta. - * This requires setting the totalMillisInState back to 0. - * - *

    Final updates should never be requested from a Streaming job since the work unit never - * completes. - */ - @Override - public @Nullable CounterUpdate extractUpdate(boolean isFinalUpdate) { - // Streaming reports deltas, so isFinalUpdate doesn't matter, and should never be true. - long sum = totalMillisInState.getAndSet(0); - return sum == 0 ? null : createUpdate(false, sum); - } - } - - /** - * Implementation of DataflowExecutionStateRegistry that creates Streaming versions of - * ExecutionState. - */ - public static class StreamingModeExecutionStateRegistry extends DataflowExecutionStateRegistry { - - private final StreamingDataflowWorker worker; - - public StreamingModeExecutionStateRegistry(StreamingDataflowWorker worker) { - this.worker = worker; - } - - @Override - protected DataflowOperationContext.DataflowExecutionState createState( - NameContext nameContext, - String stateName, - String requestingStepName, - Integer inputIndex, - MetricsContainer container, - ProfileScope profileScope) { - return new StreamingModeExecutionState( - nameContext, stateName, container, profileScope, worker); - } - } - @VisibleForTesting public long getBacklogBytes() { return backlogBytes; @@ -304,11 +224,8 @@ protected SideInputReader getSideInputReaderForViews( String stateFamily, StateFetcher.SideInputState state, Supplier scopedReadStateSupplier) { - Map tagCache = sideInputCache.get(view.getTagInternal()); - if (tagCache == null) { - tagCache = new HashMap<>(); - sideInputCache.put(view.getTagInternal(), tagCache); - } + Map tagCache = + sideInputCache.computeIfAbsent(view.getTagInternal(), k -> new HashMap<>()); if (tagCache.containsKey(sideInputWindow)) { @SuppressWarnings("unchecked") @@ -455,6 +372,10 @@ public Map flushState() { return callbacks; } + String getStateFamily(NameContext nameContext) { + return nameContext.userName() == null ? null : stateNameMap.get(nameContext.userName()); + } + interface StreamingModeStepContext { boolean issueSideInputFetch( @@ -478,8 +399,84 @@ void writePCollectionViewData( throws IOException; } - String getStateFamily(NameContext nameContext) { - return nameContext.userName() == null ? null : stateNameMap.get(nameContext.userName()); + /** + * Execution states in Streaming are shared between multiple map-task executors. Thus this class + * needs to be thread safe for multiple writers. A single stage could have have multiple executors + * running concurrently. + */ + public static class StreamingModeExecutionState + extends DataflowOperationContext.DataflowExecutionState { + + // AtomicLong is used because this value is written in two places: + // 1. The sampling thread calls takeSample to increment the time spent in this state + // 2. The reporting thread calls extractUpdate which reads the current sum *AND* sets it to 0. + private final AtomicLong totalMillisInState = new AtomicLong(); + + // The worker that created this state. Used to report lulls back to the worker. + @SuppressWarnings("unused") // Affects a public api + private final StreamingDataflowWorker worker; + + public StreamingModeExecutionState( + NameContext nameContext, + String stateName, + MetricsContainer metricsContainer, + ProfileScope profileScope, + StreamingDataflowWorker worker) { + // TODO: Take in the requesting step name and side input index for streaming. + super(nameContext, stateName, null, null, metricsContainer, profileScope); + this.worker = worker; + } + + /** + * Take sample is only called by the ExecutionStateSampler thread. It is the only place that + * increments totalMillisInState, however the reporting thread periodically calls extractUpdate + * which will read the sum and reset it to 0, so totalMillisInState does have multiple writers. + */ + @Override + public void takeSample(long millisSinceLastSample) { + totalMillisInState.addAndGet(millisSinceLastSample); + } + + /** + * Extract updates in the form of a {@link CounterUpdate}. + * + *

    Non-final updates are extracted periodically and report the physical value as a delta. + * This requires setting the totalMillisInState back to 0. + * + *

    Final updates should never be requested from a Streaming job since the work unit never + * completes. + */ + @Override + public @Nullable CounterUpdate extractUpdate(boolean isFinalUpdate) { + // Streaming reports deltas, so isFinalUpdate doesn't matter, and should never be true. + long sum = totalMillisInState.getAndSet(0); + return sum == 0 ? null : createUpdate(false, sum); + } + } + + /** + * Implementation of DataflowExecutionStateRegistry that creates Streaming versions of + * ExecutionState. + */ + public static class StreamingModeExecutionStateRegistry extends DataflowExecutionStateRegistry { + + private final StreamingDataflowWorker worker; + + public StreamingModeExecutionStateRegistry(StreamingDataflowWorker worker) { + this.worker = worker; + } + + @Override + protected DataflowOperationContext.DataflowExecutionState createState( + NameContext nameContext, + String stateName, + String requestingStepName, + Integer inputIndex, + MetricsContainer container, + ProfileScope profileScope) { + return new StreamingModeExecutionState( + nameContext, stateName, container, profileScope, worker); + } } private static class ScopedReadStateSupplier implements Supplier { @@ -501,15 +498,156 @@ public Closeable get() { } } + /** + * A specialized {@link StepContext} that uses provided {@link StateInternals} and {@link + * TimerInternals} for user state and timers. + */ + private static class UserStepContext extends DataflowStepContext + implements StreamingModeStepContext { + + private final StreamingModeExecutionContext.StepContext wrapped; + + public UserStepContext(StreamingModeExecutionContext.StepContext wrapped) { + super(wrapped.getNameContext()); + this.wrapped = wrapped; + } + + @Override + public boolean issueSideInputFetch( + PCollectionView view, BoundedWindow w, StateFetcher.SideInputState s) { + return wrapped.issueSideInputFetch(view, w, s); + } + + @Override + public void addBlockingSideInput(GlobalDataRequest blocked) { + wrapped.addBlockingSideInput(blocked); + } + + @Override + public void addBlockingSideInputs(Iterable blocked) { + wrapped.addBlockingSideInputs(blocked); + } + + @Override + public StateInternals stateInternals() { + return wrapped.stateInternals(); + } + + @Override + public Iterable getSideInputNotifications() { + return wrapped.getSideInputNotifications(); + } + + @Override + public void writePCollectionViewData( + TupleTag tag, + Iterable data, + Coder> dataCoder, + W window, + Coder windowCoder) + throws IOException { + throw new IllegalStateException("User DoFns cannot write PCollectionView data"); + } + + @Override + public TimerInternals timerInternals() { + return wrapped.userTimerInternals(); + } + + @Override + public TimerData getNextFiredTimer(Coder windowCoder) { + return wrapped.getNextFiredUserTimer(windowCoder); + } + + @Override + public void setStateCleanupTimer( + String timerId, + W window, + Coder windowCoder, + Instant cleanupTime, + Instant cleanupOutputTimestamp) { + throw new UnsupportedOperationException( + String.format( + "setStateCleanupTimer should not be called on %s, only on a system %s", + getClass().getSimpleName(), + StreamingModeExecutionContext.StepContext.class.getSimpleName())); + } + + @Override + public DataflowStepContext namespacedToUser() { + return this; + } + } + + /** A {@link SideInputReader} that fetches side inputs from the streaming worker's cache. */ + public static class StreamingModeSideInputReader implements SideInputReader { + + private final StreamingModeExecutionContext context; + private final Set> viewSet; + + private StreamingModeSideInputReader( + Iterable> views, StreamingModeExecutionContext context) { + this.context = context; + this.viewSet = ImmutableSet.copyOf(views); + } + + public static StreamingModeSideInputReader of( + Iterable> views, StreamingModeExecutionContext context) { + return new StreamingModeSideInputReader(views, context); + } + + @Override + public T get(PCollectionView view, BoundedWindow window) { + if (!contains(view)) { + throw new RuntimeException("get() called with unknown view"); + } + + // We are only fetching the cached value here, so we don't need stateFamily or + // readStateSupplier. + return context + .fetchSideInput( + view, + window, + null /* unused stateFamily */, + StateFetcher.SideInputState.CACHED_IN_WORKITEM, + null /* unused readStateSupplier */) + .orNull(); + } + + @Override + public boolean contains(PCollectionView view) { + return viewSet.contains(view); + } + + @Override + public boolean isEmpty() { + return viewSet.isEmpty(); + } + } + class StepContext extends DataflowExecutionContext.DataflowStepContext implements StreamingModeStepContext { + private final String stateFamily; + private final Supplier scopedReadStateSupplier; private WindmillStateInternals stateInternals; - private WindmillTimerInternals systemTimerInternals; private WindmillTimerInternals userTimerInternals; - private final String stateFamily; - private final Supplier scopedReadStateSupplier; + // Lazily initialized + private Iterator cachedFiredSystemTimers = null; + // Lazily initialized + private PeekingIterator cachedFiredUserTimers = null; + // An ordered list of any timers that were set or modified by user processing earlier in this + // bundle. + // We use a NavigableSet instead of a priority queue to prevent duplicate elements from ending + // up in the queue. + private NavigableSet modifiedUserEventTimersOrdered = null; + private NavigableSet modifiedUserProcessingTimersOrdered = null; + private NavigableSet modifiedUserSynchronizedProcessingTimersOrdered = null; + // A list of timer keys that were modified by user processing earlier in this bundle. This + // serves a tombstone, so + // that we know not to fire any bundle tiemrs that were moddified. + private Table modifiedUserTimerKeys = null; public StepContext(DataflowOperationContext operationContext) { super(operationContext.nameContext()); @@ -570,14 +708,11 @@ public void flushState() { userTimerInternals.persistTo(outputBuilder); } - // Lazily initialized - private Iterator cachedFiredSystemTimers = null; - @Override public TimerData getNextFiredTimer(Coder windowCoder) { if (cachedFiredSystemTimers == null) { cachedFiredSystemTimers = - FluentIterable.from(StreamingModeExecutionContext.this.getFiredTimers()) + FluentIterable.from(StreamingModeExecutionContext.this.getFiredTimers()) .filter( timer -> WindmillTimerInternals.isSystemTimer(timer) @@ -601,16 +736,6 @@ public TimerData getNextFiredTimer(Coder windowCode return nextTimer; } - // Lazily initialized - private PeekingIterator cachedFiredUserTimers = null; - // An ordered list of any timers that were set or modified by user processing earlier in this - // bundle. - // We use a NavigableSet instead of a priority queue to prevent duplicate elements from ending - // up in the queue. - private NavigableSet modifiedUserEventTimersOrdered = null; - private NavigableSet modifiedUserProcessingTimersOrdered = null; - private NavigableSet modifiedUserSynchronizedProcessingTimersOrdered = null; - private NavigableSet getModifiedUserTimersOrdered(TimeDomain timeDomain) { switch (timeDomain) { case EVENT_TIME: @@ -624,11 +749,6 @@ private NavigableSet getModifiedUserTimersOrdered(TimeDomain timeDoma } } - // A list of timer keys that were modified by user processing earlier in this bundle. This - // serves a tombstone, so - // that we know not to fire any bundle tiemrs that were moddified. - private Table modifiedUserTimerKeys = null; - private void onUserTimerModified(TimerData timerData) { if (!timerData.getDeleted()) { getModifiedUserTimersOrdered(timerData.getDomain()).add(timerData); @@ -804,131 +924,4 @@ public TimerInternals userTimerInternals() { return checkNotNull(userTimerInternals); } } - - /** - * A specialized {@link StepContext} that uses provided {@link StateInternals} and {@link - * TimerInternals} for user state and timers. - */ - private static class UserStepContext extends DataflowStepContext - implements StreamingModeStepContext { - - private final StreamingModeExecutionContext.StepContext wrapped; - - public UserStepContext(StreamingModeExecutionContext.StepContext wrapped) { - super(wrapped.getNameContext()); - this.wrapped = wrapped; - } - - @Override - public boolean issueSideInputFetch( - PCollectionView view, BoundedWindow w, StateFetcher.SideInputState s) { - return wrapped.issueSideInputFetch(view, w, s); - } - - @Override - public void addBlockingSideInput(GlobalDataRequest blocked) { - wrapped.addBlockingSideInput(blocked); - } - - @Override - public void addBlockingSideInputs(Iterable blocked) { - wrapped.addBlockingSideInputs(blocked); - } - - @Override - public StateInternals stateInternals() { - return wrapped.stateInternals(); - } - - @Override - public Iterable getSideInputNotifications() { - return wrapped.getSideInputNotifications(); - } - - @Override - public void writePCollectionViewData( - TupleTag tag, - Iterable data, - Coder> dataCoder, - W window, - Coder windowCoder) - throws IOException { - throw new IllegalStateException("User DoFns cannot write PCollectionView data"); - } - - @Override - public TimerInternals timerInternals() { - return wrapped.userTimerInternals(); - } - - @Override - public TimerData getNextFiredTimer(Coder windowCoder) { - return wrapped.getNextFiredUserTimer(windowCoder); - } - - @Override - public void setStateCleanupTimer( - String timerId, - W window, - Coder windowCoder, - Instant cleanupTime, - Instant cleanupOutputTimestamp) { - throw new UnsupportedOperationException( - String.format( - "setStateCleanupTimer should not be called on %s, only on a system %s", - getClass().getSimpleName(), - StreamingModeExecutionContext.StepContext.class.getSimpleName())); - } - - @Override - public DataflowStepContext namespacedToUser() { - return this; - } - } - - /** A {@link SideInputReader} that fetches side inputs from the streaming worker's cache. */ - public static class StreamingModeSideInputReader implements SideInputReader { - - private StreamingModeExecutionContext context; - private Set> viewSet; - - private StreamingModeSideInputReader( - Iterable> views, StreamingModeExecutionContext context) { - this.context = context; - this.viewSet = ImmutableSet.copyOf(views); - } - - public static StreamingModeSideInputReader of( - Iterable> views, StreamingModeExecutionContext context) { - return new StreamingModeSideInputReader(views, context); - } - - @Override - public T get(PCollectionView view, BoundedWindow window) { - if (!contains(view)) { - throw new RuntimeException("get() called with unknown view"); - } - - // We are only fetching the cached value here, so we don't need stateFamily or - // readStateSupplier. - return context - .fetchSideInput( - view, - window, - null /* unused stateFamily */, - StateFetcher.SideInputState.CACHED_IN_WORKITEM, - null /* unused readStateSupplier */) - .orNull(); - } - - @Override - public boolean contains(PCollectionView view) { - return viewSet.contains(view); - } - - @Override - public boolean isEmpty() { - return viewSet.isEmpty(); - } - } } diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingPCollectionViewWriterDoFnFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingPCollectionViewWriterDoFnFactory.java index 238d6d6abd714..b9bf203e529ee 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingPCollectionViewWriterDoFnFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingPCollectionViewWriterDoFnFactory.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.services.dataflow.model.SideInputInfo; import java.util.List; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingPCollectionViewWriterParDoFn.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingPCollectionViewWriterParDoFn.java index 30d9209e792e7..c04ccc383c73f 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingPCollectionViewWriterParDoFn.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingPCollectionViewWriterParDoFn.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import org.apache.beam.runners.dataflow.worker.util.common.worker.ParDoFn; import org.apache.beam.runners.dataflow.worker.util.common.worker.Receiver; @@ -26,7 +26,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** * A {@link ParDoFn} that writes side input data using {@link diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputFetcher.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputFetcher.java index 844b3f1d57056..2b551acd2d8c7 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputFetcher.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputFetcher.java @@ -50,9 +50,9 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Parser; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; /** A class that handles streaming side inputs in a {@link DoFnRunner}. */ @SuppressWarnings({"keyfor", "nullness"}) // TODO(https://github.com/apache/beam/issues/20497) diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainer.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainer.java index 2ed9a05110e1c..875a2d649ece2 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainer.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainer.java @@ -24,16 +24,20 @@ import javax.annotation.Nonnull; import org.apache.beam.runners.core.metrics.DistributionData; import org.apache.beam.runners.core.metrics.GaugeCell; +import org.apache.beam.runners.core.metrics.HistogramCell; import org.apache.beam.runners.core.metrics.MetricsMap; import org.apache.beam.sdk.metrics.Counter; import org.apache.beam.sdk.metrics.Distribution; import org.apache.beam.sdk.metrics.Gauge; +import org.apache.beam.sdk.metrics.Histogram; import org.apache.beam.sdk.metrics.MetricKey; import org.apache.beam.sdk.metrics.MetricName; import org.apache.beam.sdk.metrics.MetricsContainer; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.sdk.util.HistogramData; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; import org.checkerframework.checker.nullness.qual.Nullable; /** @@ -47,14 +51,22 @@ public class StreamingStepMetricsContainer implements MetricsContainer { private final String stepName; + private static Boolean enablePerWorkerMetrics; + private MetricsMap counters = new MetricsMap<>(DeltaCounterCell::new); + private MetricsMap perWorkerCounters = + new MetricsMap<>(DeltaCounterCell::new); + private MetricsMap gauges = new MetricsMap<>(GaugeCell::new); private MetricsMap distributions = new MetricsMap<>(DeltaDistributionCell::new); + private MetricsMap, HistogramCell> perWorkerHistograms = + new MetricsMap<>(HistogramCell::new); + private StreamingStepMetricsContainer(String stepName) { this.stepName = stepName; } @@ -73,6 +85,15 @@ public Counter getCounter(MetricName metricName) { return counters.get(metricName); } + @Override + public Counter getPerWorkerCounter(MetricName metricName) { + if (enablePerWorkerMetrics) { + return perWorkerCounters.get(metricName); + } else { + return MetricsContainer.super.getPerWorkerCounter(metricName); + } + } + @Override public Distribution getDistribution(MetricName metricName) { return distributions.get(metricName); @@ -83,6 +104,16 @@ public Gauge getGauge(MetricName metricName) { return gauges.get(metricName); } + @Override + public Histogram getPerWorkerHistogram( + MetricName metricName, HistogramData.BucketType bucketType) { + if (enablePerWorkerMetrics) { + return perWorkerHistograms.get(KV.of(metricName, bucketType)); + } else { + return MetricsContainer.super.getPerWorkerHistogram(metricName, bucketType); + } + } + public Iterable extractUpdates() { return counterUpdates().append(distributionUpdates()); } @@ -142,4 +173,8 @@ public static Iterable extractMetricUpdates( .getContainers() .transformAndConcat(StreamingStepMetricsContainer::extractUpdates); } + + public static void setEnablePerWorkerMetrics(Boolean enablePerWorkerMetrics) { + StreamingStepMetricsContainer.enablePerWorkerMetrics = enablePerWorkerMetrics; + } } diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ToIsmRecordForMultimapDoFnFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ToIsmRecordForMultimapDoFnFactory.java index c8be02db34882..2627441ce39ae 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ToIsmRecordForMultimapDoFnFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ToIsmRecordForMultimapDoFnFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.services.dataflow.model.SideInputInfo; import java.util.Iterator; @@ -38,7 +38,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * A {@link ParDoFnFactory} that creates a system {@link ParDoFn} responsible for transforming diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/UngroupedShuffleReader.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/UngroupedShuffleReader.java index 00aa35be5f044..27f3cb9735e83 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/UngroupedShuffleReader.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/UngroupedShuffleReader.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/UngroupedShuffleReaderFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/UngroupedShuffleReaderFactory.java index aef4abaffb201..6f25b78b1fd95 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/UngroupedShuffleReaderFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/UngroupedShuffleReaderFactory.java @@ -27,7 +27,7 @@ import org.apache.beam.runners.dataflow.worker.util.common.worker.NativeReader; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** Creates an UngroupedShuffleReader from a CloudObject spec. */ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/UngroupedWindmillReader.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/UngroupedWindmillReader.java index 9ceb8c914d559..e4e56a96c15aa 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/UngroupedWindmillReader.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/UngroupedWindmillReader.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/UserParDoFnFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/UserParDoFnFactory.java index 81e36bfbb58da..a8d5975e45eac 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/UserParDoFnFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/UserParDoFnFactory.java @@ -19,7 +19,7 @@ import static org.apache.beam.runners.dataflow.DataflowRunner.StreamingPCollectionViewWriterFn; import static org.apache.beam.runners.dataflow.util.Structs.getBytes; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.services.dataflow.model.SideInputInfo; import java.util.List; @@ -38,8 +38,8 @@ import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ValuesDoFnFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ValuesDoFnFactory.java index 9a1f8ca60859d..3fe48870d2462 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ValuesDoFnFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ValuesDoFnFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.services.dataflow.model.SideInputInfo; import java.util.List; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/Weighers.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/Weighers.java index d7c3847f34888..eb4e0f4885a7c 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/Weighers.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/Weighers.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.dataflow.worker; import org.apache.beam.sdk.util.Weighted; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Weigher; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Weigher; /** * A {@code Weigher} @@ -28,7 +28,7 @@ *

    Package-private here so that the dependency on Guava does not leak into the public API * surface. */ -class Weighers { +public class Weighers { public static Weigher fixedWeightKeys(final int keyWeight) { return (key, value) -> (int) Math.min(keyWeight + value.getWeight(), Integer.MAX_VALUE); } diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillKeyedWorkItem.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillKeyedWorkItem.java index ec260bc2a6727..40ee075f5fb4c 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillKeyedWorkItem.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillKeyedWorkItem.java @@ -36,10 +36,10 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.common.ElementByteSizeObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicate; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicate; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillReaderIteratorBase.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillReaderIteratorBase.java index d12f6cdfae7c5..405440ba1193c 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillReaderIteratorBase.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillReaderIteratorBase.java @@ -22,7 +22,7 @@ import org.apache.beam.runners.dataflow.worker.util.common.worker.NativeReader; import org.apache.beam.runners.dataflow.worker.windmill.Windmill; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; /** * Base class for iterators that decode messages from bundles inside a {@link Windmill.WorkItem}. diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillSink.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillSink.java index f472abaecf4b6..e9b3f252d457b 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillSink.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillSink.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.dataflow.worker; import static org.apache.beam.runners.dataflow.util.Structs.getString; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.service.AutoService; import java.io.IOException; @@ -42,7 +42,7 @@ import org.apache.beam.sdk.values.ValueWithRecordId; import org.apache.beam.sdk.values.ValueWithRecordId.ValueWithRecordIdCoder; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; @SuppressWarnings({ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillStateInternals.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillStateInternals.java deleted file mode 100644 index 1a4fe8fca06ab..0000000000000 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillStateInternals.java +++ /dev/null @@ -1,2161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.runners.dataflow.worker; - -import com.google.auto.value.AutoValue; -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.nio.charset.StandardCharsets; -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Random; -import java.util.Set; -import java.util.SortedSet; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.function.BiConsumer; -import java.util.function.Function; -import javax.annotation.concurrent.NotThreadSafe; -import org.apache.beam.runners.core.StateInternals; -import org.apache.beam.runners.core.StateNamespace; -import org.apache.beam.runners.core.StateTable; -import org.apache.beam.runners.core.StateTag; -import org.apache.beam.runners.core.StateTag.StateBinder; -import org.apache.beam.runners.core.StateTags; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.SortedListEntry; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.SortedListRange; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.TagSortedListDeleteRequest; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.TagSortedListInsertRequest; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.TagSortedListUpdateRequest; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.WorkItemCommitRequest; -import org.apache.beam.sdk.coders.BooleanCoder; -import org.apache.beam.sdk.coders.Coder; -import org.apache.beam.sdk.coders.Coder.Context; -import org.apache.beam.sdk.coders.CoderException; -import org.apache.beam.sdk.coders.CustomCoder; -import org.apache.beam.sdk.coders.InstantCoder; -import org.apache.beam.sdk.coders.MapCoder; -import org.apache.beam.sdk.coders.NullableCoder; -import org.apache.beam.sdk.coders.SetCoder; -import org.apache.beam.sdk.coders.StructuredCoder; -import org.apache.beam.sdk.coders.VarLongCoder; -import org.apache.beam.sdk.state.BagState; -import org.apache.beam.sdk.state.CombiningState; -import org.apache.beam.sdk.state.MapState; -import org.apache.beam.sdk.state.MultimapState; -import org.apache.beam.sdk.state.OrderedListState; -import org.apache.beam.sdk.state.ReadableState; -import org.apache.beam.sdk.state.ReadableStates; -import org.apache.beam.sdk.state.SetState; -import org.apache.beam.sdk.state.State; -import org.apache.beam.sdk.state.StateContext; -import org.apache.beam.sdk.state.StateContexts; -import org.apache.beam.sdk.state.ValueState; -import org.apache.beam.sdk.state.WatermarkHoldState; -import org.apache.beam.sdk.transforms.Combine.CombineFn; -import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext; -import org.apache.beam.sdk.transforms.windowing.TimestampCombiner; -import org.apache.beam.sdk.util.ByteStringOutputStream; -import org.apache.beam.sdk.util.CombineFnUtil; -import org.apache.beam.sdk.util.Weighted; -import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BoundType; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Range; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.RangeSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.TreeRangeSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Futures; -import org.checkerframework.checker.initialization.qual.Initialized; -import org.checkerframework.checker.nullness.qual.NonNull; -import org.checkerframework.checker.nullness.qual.Nullable; -import org.checkerframework.checker.nullness.qual.UnknownKeyFor; -import org.joda.time.Duration; -import org.joda.time.Instant; - -/** Implementation of {@link StateInternals} using Windmill to manage the underlying data. */ -@SuppressWarnings({ - "rawtypes", // TODO(https://github.com/apache/beam/issues/20447) - "nullness" // TODO(https://github.com/apache/beam/issues/20497) -}) -class WindmillStateInternals implements StateInternals { - - /** - * The key will be null when not in a keyed context, from the users perspective. There is still a - * "key" for the Windmill computation, but it cannot be meaningfully deserialized. - */ - private final @Nullable K key; - - @Override - public @Nullable K getKey() { - return key; - } - - private static class CachingStateTable extends StateTable { - private final String stateFamily; - private final WindmillStateReader reader; - private final WindmillStateCache.ForKeyAndFamily cache; - private final boolean isSystemTable; - boolean isNewKey; - private final Supplier scopedReadStateSupplier; - private final StateTable derivedStateTable; - - public CachingStateTable( - @Nullable K key, - String stateFamily, - WindmillStateReader reader, - WindmillStateCache.ForKeyAndFamily cache, - boolean isSystemTable, - boolean isNewKey, - Supplier scopedReadStateSupplier, - StateTable derivedStateTable) { - this.stateFamily = stateFamily; - this.reader = reader; - this.cache = cache; - this.isSystemTable = isSystemTable; - this.isNewKey = isNewKey; - this.scopedReadStateSupplier = scopedReadStateSupplier; - this.derivedStateTable = derivedStateTable != null ? derivedStateTable : this; - } - - @Override - protected StateBinder binderForNamespace( - final StateNamespace namespace, final StateContext c) { - // Look up state objects in the cache or create new ones if not found. The state will - // be added to the cache in persist(). - return new StateBinder() { - @Override - public BagState bindBag(StateTag> address, Coder elemCoder) { - if (isSystemTable) { - address = StateTags.makeSystemTagInternal(address); - } - WindmillBag result = (WindmillBag) cache.get(namespace, address); - if (result == null) { - result = new WindmillBag<>(namespace, address, stateFamily, elemCoder, isNewKey); - } - result.initializeForWorkItem(reader, scopedReadStateSupplier); - return result; - } - - @Override - public SetState bindSet(StateTag> spec, Coder elemCoder) { - WindmillSet result = - new WindmillSet(namespace, spec, stateFamily, elemCoder, cache, isNewKey); - result.initializeForWorkItem(reader, scopedReadStateSupplier); - return result; - } - - @Override - public MapState bindMap( - StateTag> spec, Coder keyCoder, Coder valueCoder) { - WindmillMap result = (WindmillMap) cache.get(namespace, spec); - if (result == null) { - result = - new WindmillMap( - namespace, spec, stateFamily, keyCoder, valueCoder, isNewKey); - } - result.initializeForWorkItem(reader, scopedReadStateSupplier); - return result; - } - - @Override - public MultimapState bindMultimap( - StateTag> spec, - Coder keyCoder, - Coder valueCoder) { - throw new UnsupportedOperationException( - String.format("%s is not supported", MultimapState.class.getSimpleName())); - } - - @Override - public OrderedListState bindOrderedList( - StateTag> spec, Coder elemCoder) { - if (isSystemTable) { - spec = StateTags.makeSystemTagInternal(spec); - } - WindmillOrderedList result = (WindmillOrderedList) cache.get(namespace, spec); - if (result == null) { - result = - new WindmillOrderedList<>( - derivedStateTable, namespace, spec, stateFamily, elemCoder, isNewKey); - } - result.initializeForWorkItem(reader, scopedReadStateSupplier); - return result; - } - - @Override - public WatermarkHoldState bindWatermark( - StateTag address, TimestampCombiner timestampCombiner) { - if (isSystemTable) { - address = StateTags.makeSystemTagInternal(address); - } - WindmillWatermarkHold result = (WindmillWatermarkHold) cache.get(namespace, address); - if (result == null) { - result = - new WindmillWatermarkHold( - namespace, address, stateFamily, timestampCombiner, isNewKey); - } - result.initializeForWorkItem(reader, scopedReadStateSupplier); - return result; - } - - @Override - public CombiningState bindCombiningValue( - StateTag> address, - Coder accumCoder, - CombineFn combineFn) { - if (isSystemTable) { - address = StateTags.makeSystemTagInternal(address); - } - WindmillCombiningState result = - new WindmillCombiningState<>( - namespace, address, stateFamily, accumCoder, combineFn, cache, isNewKey); - result.initializeForWorkItem(reader, scopedReadStateSupplier); - return result; - } - - @Override - public - CombiningState bindCombiningValueWithContext( - StateTag> address, - Coder accumCoder, - CombineFnWithContext combineFn) { - if (isSystemTable) { - address = StateTags.makeSystemTagInternal(address); - } - return bindCombiningValue(address, accumCoder, CombineFnUtil.bindContext(combineFn, c)); - } - - @Override - public ValueState bindValue(StateTag> address, Coder coder) { - if (isSystemTable) { - address = StateTags.makeSystemTagInternal(address); - } - WindmillValue result = (WindmillValue) cache.get(namespace, address); - if (result == null) { - result = new WindmillValue<>(namespace, address, stateFamily, coder, isNewKey); - } - result.initializeForWorkItem(reader, scopedReadStateSupplier); - return result; - } - }; - } - } - - private WindmillStateCache.ForKeyAndFamily cache; - Supplier scopedReadStateSupplier; - private StateTable workItemState; - private StateTable workItemDerivedState; - - public WindmillStateInternals( - @Nullable K key, - String stateFamily, - WindmillStateReader reader, - boolean isNewKey, - WindmillStateCache.ForKeyAndFamily cache, - Supplier scopedReadStateSupplier) { - this.key = key; - this.cache = cache; - this.scopedReadStateSupplier = scopedReadStateSupplier; - this.workItemDerivedState = - new CachingStateTable<>( - key, stateFamily, reader, cache, true, isNewKey, scopedReadStateSupplier, null); - this.workItemState = - new CachingStateTable<>( - key, - stateFamily, - reader, - cache, - false, - isNewKey, - scopedReadStateSupplier, - workItemDerivedState); - } - - private void persist(List> commitsToMerge, StateTable stateTable) { - for (State location : stateTable.values()) { - if (!(location instanceof WindmillState)) { - throw new IllegalStateException( - String.format( - "%s wasn't created by %s -- unable to persist it", - location.getClass().getSimpleName(), getClass().getSimpleName())); - } - - try { - commitsToMerge.add(((WindmillState) location).persist(cache)); - } catch (IOException e) { - throw new RuntimeException("Unable to persist state", e); - } - } - - // All cached State objects now have known values. - // Clear any references to the underlying reader to prevent space leaks. - // The next work unit to use these cached State objects will reset the - // reader to a current reader in case those values are modified. - for (State location : stateTable.values()) { - ((WindmillState) location).cleanupAfterWorkItem(); - } - - // Clear out the map of already retrieved state instances. - stateTable.clear(); - } - - public void persist(final Windmill.WorkItemCommitRequest.Builder commitBuilder) { - List> commitsToMerge = new ArrayList<>(); - - // Call persist on each first, which may schedule some futures for reading. - persist(commitsToMerge, workItemState); - persist(commitsToMerge, workItemDerivedState); - - try (Closeable scope = scopedReadStateSupplier.get()) { - for (Future commitFuture : commitsToMerge) { - commitBuilder.mergeFrom(commitFuture.get()); - } - } catch (ExecutionException | InterruptedException | IOException exc) { - if (exc instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw new RuntimeException("Failed to retrieve Windmill state during persist()", exc); - } - - cache.persist(); - } - - /** Encodes the given namespace and address as {@code <namespace>+<address>}. */ - @VisibleForTesting - static ByteString encodeKey(StateNamespace namespace, StateTag address) { - try { - // Use ByteStringOutputStream rather than concatenation and String.format. We build these keys - // a lot, and this leads to better performance results. See associated benchmarks. - ByteStringOutputStream stream = new ByteStringOutputStream(); - OutputStreamWriter writer = new OutputStreamWriter(stream, StandardCharsets.UTF_8); - - // stringKey starts and ends with a slash. We separate it from the - // StateTag ID by a '+' (which is guaranteed not to be in the stringKey) because the - // ID comes from the user. - namespace.appendTo(writer); - writer.write('+'); - address.appendTo(writer); - writer.flush(); - return stream.toByteString(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** - * Abstract base class for all Windmill state. - * - *

    Note that these are not thread safe; each state object is associated with a key and thus - * only accessed by a single thread at once. - */ - @NotThreadSafe - private abstract static class WindmillState { - protected Supplier scopedReadStateSupplier; - protected WindmillStateReader reader; - - /** - * Return an asynchronously computed {@link WorkItemCommitRequest}. The request should be of a - * form that can be merged with others (only add to repeated fields). - */ - abstract Future persist(WindmillStateCache.ForKeyAndFamily cache) - throws IOException; - - /** - * Prepare this (possibly reused from cache) state for reading from {@code reader} if needed. - */ - void initializeForWorkItem( - WindmillStateReader reader, Supplier scopedReadStateSupplier) { - this.reader = reader; - this.scopedReadStateSupplier = scopedReadStateSupplier; - } - - /** - * This (now cached) state should never need to interact with the reader until the next work - * item. Clear it to prevent space leaks. The reader will be reset by {@link - * #initializeForWorkItem} upon the next work item. - */ - void cleanupAfterWorkItem() { - this.reader = null; - this.scopedReadStateSupplier = null; - } - - Closeable scopedReadState() { - return scopedReadStateSupplier.get(); - } - } - - /** - * Base class for implementations of {@link WindmillState} where the {@link #persist} call does - * not require any asynchronous reading. - */ - private abstract static class SimpleWindmillState extends WindmillState { - @Override - public final Future persist(WindmillStateCache.ForKeyAndFamily cache) - throws IOException { - return Futures.immediateFuture(persistDirectly(cache)); - } - - /** - * Returns a {@link WorkItemCommitRequest} that can be used to persist this state to Windmill. - */ - protected abstract WorkItemCommitRequest persistDirectly( - WindmillStateCache.ForKeyAndFamily cache) throws IOException; - } - - @Override - public T state(StateNamespace namespace, StateTag address) { - return workItemState.get(namespace, address, StateContexts.nullContext()); - } - - @Override - public T state( - StateNamespace namespace, StateTag address, StateContext c) { - return workItemState.get(namespace, address, c); - } - - private static class WindmillValue extends SimpleWindmillState implements ValueState { - private final StateNamespace namespace; - private final StateTag> address; - private final ByteString stateKey; - private final String stateFamily; - private final Coder coder; - - /** Whether we've modified the value since creation of this state. */ - private boolean modified = false; - /** Whether the in memory value is the true value. */ - private boolean valueIsKnown = false; - /** The size of the encoded value */ - private long cachedSize = -1; - - private T value; - - private WindmillValue( - StateNamespace namespace, - StateTag> address, - String stateFamily, - Coder coder, - boolean isNewKey) { - this.namespace = namespace; - this.address = address; - this.stateKey = encodeKey(namespace, address); - this.stateFamily = stateFamily; - this.coder = coder; - if (isNewKey) { - this.valueIsKnown = true; - this.value = null; - } - } - - @Override - public void clear() { - modified = true; - valueIsKnown = true; - value = null; - } - - @Override - @SuppressWarnings("FutureReturnValueIgnored") - public WindmillValue readLater() { - getFuture(); - return this; - } - - @Override - public T read() { - try (Closeable scope = scopedReadState()) { - if (!valueIsKnown) { - cachedSize = -1; - } - value = getFuture().get(); - valueIsKnown = true; - return value; - } catch (InterruptedException | ExecutionException | IOException e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw new RuntimeException("Unable to read value from state", e); - } - } - - @Override - public void write(T value) { - modified = true; - valueIsKnown = true; - cachedSize = -1; - this.value = value; - } - - @Override - protected WorkItemCommitRequest persistDirectly(WindmillStateCache.ForKeyAndFamily cache) - throws IOException { - if (!valueIsKnown) { - // The value was never read, written or cleared. - // Thus nothing to update in Windmill. - // And no need to add to global cache. - return WorkItemCommitRequest.newBuilder().buildPartial(); - } - - ByteString encoded = null; - if (cachedSize == -1 || modified) { - ByteStringOutputStream stream = new ByteStringOutputStream(); - if (value != null) { - coder.encode(value, stream, Coder.Context.OUTER); - } - encoded = stream.toByteString(); - cachedSize = encoded.size(); - } - - // Place in cache to avoid a future read. - cache.put(namespace, address, this, cachedSize); - - if (!modified) { - // The value was read, but never written or cleared. - // But nothing to update in Windmill. - return WorkItemCommitRequest.newBuilder().buildPartial(); - } - - // The value was written or cleared. Commit that change to Windmill. - modified = false; - WorkItemCommitRequest.Builder commitBuilder = WorkItemCommitRequest.newBuilder(); - commitBuilder - .addValueUpdatesBuilder() - .setTag(stateKey) - .setStateFamily(stateFamily) - .getValueBuilder() - .setData(encoded) - .setTimestamp(Long.MAX_VALUE); - return commitBuilder.buildPartial(); - } - - private Future getFuture() { - // WindmillStateReader guarantees that we can ask for a future for a particular tag multiple - // times and it will efficiently be reused. - return valueIsKnown - ? Futures.immediateFuture(value) - : reader.valueFuture(stateKey, stateFamily, coder); - } - } - - // Coder for closed-open ranges. - private static class RangeCoder extends StructuredCoder> { - private Coder boundCoder; - - RangeCoder(Coder boundCoder) { - this.boundCoder = NullableCoder.of(boundCoder); - } - - @Override - public List> getCoderArguments() { - return Lists.newArrayList(boundCoder); - } - - @Override - public void verifyDeterministic() throws NonDeterministicException { - boundCoder.verifyDeterministic(); - ; - } - - @Override - public void encode(Range value, OutputStream outStream) throws CoderException, IOException { - Preconditions.checkState( - value.lowerBoundType().equals(BoundType.CLOSED), "unexpected range " + value); - Preconditions.checkState( - value.upperBoundType().equals(BoundType.OPEN), "unexpected range " + value); - boundCoder.encode(value.hasLowerBound() ? value.lowerEndpoint() : null, outStream); - boundCoder.encode(value.hasUpperBound() ? value.upperEndpoint() : null, outStream); - } - - @Override - public Range decode(InputStream inStream) throws CoderException, IOException { - @Nullable T lower = boundCoder.decode(inStream); - @Nullable T upper = boundCoder.decode(inStream); - if (lower == null) { - return upper != null ? Range.lessThan(upper) : Range.all(); - } else if (upper == null) { - return Range.atLeast(lower); - } else { - return Range.closedOpen(lower, upper); - } - } - } - - private static class RangeSetCoder extends CustomCoder> { - private SetCoder> rangesCoder; - - RangeSetCoder(Coder boundCoder) { - this.rangesCoder = SetCoder.of(new RangeCoder<>(boundCoder)); - } - - @Override - public void encode(RangeSet value, OutputStream outStream) throws IOException { - rangesCoder.encode(value.asRanges(), outStream); - } - - @Override - public RangeSet decode(InputStream inStream) throws CoderException, IOException { - return TreeRangeSet.create(rangesCoder.decode(inStream)); - } - } - - /** - * Tracker for the ids used in an ordered list. - * - *

    Windmill accepts an int64 id for each timestamped-element in the list. Unique elements are - * identified by the pair of timestamp and id. This means that tow unique elements e1, e2 must - * have different (ts1, id1), (ts2, id2) pairs. To accomplish this we bucket time into five-minute - * buckets, and store a free list of ids available for each bucket. - * - *

    When a timestamp range is deleted, we remove id tracking for elements in that range. In - * order to handle the case where a range is deleted piecemeal, we track sub-range deletions for - * each range. For example: - * - *

    12:00 - 12:05 ids 12:05 - 12:10 ids - * - *

    delete 12:00-12:06 - * - *

    12:00 - 12:05 *removed* 12:05 - 12:10 ids subranges deleted 12:05-12:06 - * - *

    delete 12:06 - 12:07 - * - *

    12:05 - 12:10 ids subranges deleted 12:05-12:07 - * - *

    delete 12:07 - 12:10 - * - *

    12:05 - 12:10 *removed* - */ - static final class IdTracker { - static final String IDS_AVAILABLE_STR = "IdsAvailable"; - static final String DELETIONS_STR = "Deletions"; - - // Note that this previously was Long.MIN_VALUE but ids are unsigned when - // sending to windmill for Streaming Engine. For updated appliance - // pipelines with existing state, there may be negative ids. - static final long NEW_RANGE_MIN_ID = 0; - static final long NEW_RANGE_MAX_ID = Long.MAX_VALUE; - - // We track ids on five-minute boundaries. - private static final Duration RESOLUTION = Duration.standardMinutes(5); - static final MapCoder, RangeSet> IDS_AVAILABLE_CODER = - MapCoder.of(new RangeCoder<>(InstantCoder.of()), new RangeSetCoder<>(VarLongCoder.of())); - static final MapCoder, RangeSet> SUBRANGE_DELETIONS_CODER = - MapCoder.of(new RangeCoder<>(InstantCoder.of()), new RangeSetCoder<>(InstantCoder.of())); - private final StateTag, RangeSet>>> idsAvailableTag; - // A map from five-minute ranges to the set of ids available in that interval. - final ValueState, RangeSet>> idsAvailableValue; - private final StateTag, RangeSet>>> subRangeDeletionsTag; - // If a timestamp-range in the map has been partially cleared, the cleared intervals are stored - // here. - final ValueState, RangeSet>> subRangeDeletionsValue; - - IdTracker( - StateTable stateTable, - StateNamespace namespace, - StateTag spec, - String stateFamily, - boolean complete) { - this.idsAvailableTag = - StateTags.makeSystemTagInternal( - StateTags.value(spec.getId() + IDS_AVAILABLE_STR, IDS_AVAILABLE_CODER)); - this.idsAvailableValue = - stateTable.get(namespace, idsAvailableTag, StateContexts.nullContext()); - this.subRangeDeletionsTag = - StateTags.makeSystemTagInternal( - StateTags.value(spec.getId() + DELETIONS_STR, SUBRANGE_DELETIONS_CODER)); - this.subRangeDeletionsValue = - stateTable.get(namespace, subRangeDeletionsTag, StateContexts.nullContext()); - } - - static > - Map, RangeSet> newSortedRangeMap(Class valueClass) { - return Maps.newTreeMap( - Comparator., Instant>comparing(Range::lowerEndpoint) - .thenComparing(Range::upperEndpoint)); - } - - private Range getTrackedRange(Instant ts) { - Instant snapped = - new Instant(ts.getMillis() - ts.plus(RESOLUTION).getMillis() % RESOLUTION.getMillis()); - return Range.closedOpen(snapped, snapped.plus(RESOLUTION)); - } - - @SuppressWarnings("FutureReturnValueIgnored") - void readLater() { - idsAvailableValue.readLater(); - subRangeDeletionsValue.readLater(); - } - - Map, RangeSet> readIdsAvailable() { - Map, RangeSet> idsAvailable = idsAvailableValue.read(); - return idsAvailable != null ? idsAvailable : newSortedRangeMap(Long.class); - } - - Map, RangeSet> readSubRangeDeletions() { - Map, RangeSet> subRangeDeletions = subRangeDeletionsValue.read(); - return subRangeDeletions != null ? subRangeDeletions : newSortedRangeMap(Instant.class); - } - - void clear() throws ExecutionException, InterruptedException { - idsAvailableValue.clear(); - subRangeDeletionsValue.clear(); - } - - void add( - SortedSet> elements, BiConsumer, Long> output) - throws ExecutionException, InterruptedException { - Range currentIdRange = null; - long currentId = 0; - - Range currentTsRange = null; - RangeSet currentTsRangeDeletions = null; - - Map, RangeSet> idsAvailable = readIdsAvailable(); - Map, RangeSet> subRangeDeletions = readSubRangeDeletions(); - - RangeSet availableIdsForTsRange = null; - Iterator> idRangeIter = null; - RangeSet idsUsed = TreeRangeSet.create(); - for (TimestampedValueWithId pendingAdd : elements) { - // Since elements are in increasing ts order, often we'll be able to reuse the previous - // iteration's range. - if (currentTsRange == null - || !currentTsRange.contains(pendingAdd.getValue().getTimestamp())) { - if (availableIdsForTsRange != null) { - // We're moving onto a new ts range. Remove all used ids - availableIdsForTsRange.removeAll(idsUsed); - idsUsed = TreeRangeSet.create(); - } - - // Lookup the range for the current timestamp. - currentTsRange = getTrackedRange(pendingAdd.getValue().getTimestamp()); - // Lookup available ids for this timestamp range. If nothing there, we default to all ids - // available. - availableIdsForTsRange = - idsAvailable.computeIfAbsent( - currentTsRange, - r -> - TreeRangeSet.create( - ImmutableList.of(Range.closedOpen(NEW_RANGE_MIN_ID, NEW_RANGE_MAX_ID)))); - idRangeIter = availableIdsForTsRange.asRanges().iterator(); - currentIdRange = null; - currentTsRangeDeletions = subRangeDeletions.get(currentTsRange); - } - - if (currentIdRange == null || currentId >= currentIdRange.upperEndpoint()) { - // Move to the next range of free ids, and start assigning ranges from there. - currentIdRange = idRangeIter.next(); - currentId = currentIdRange.lowerEndpoint(); - } - - if (currentTsRangeDeletions != null) { - currentTsRangeDeletions.remove( - Range.closedOpen( - pendingAdd.getValue().getTimestamp(), - pendingAdd.getValue().getTimestamp().plus(Duration.millis(1)))); - } - idsUsed.add(Range.closedOpen(currentId, currentId + 1)); - output.accept(pendingAdd.getValue(), currentId++); - } - if (availableIdsForTsRange != null) { - availableIdsForTsRange.removeAll(idsUsed); - } - writeValues(idsAvailable, subRangeDeletions); - } - - // Remove a timestamp range. Returns ids freed up. - void remove(Range tsRange) throws ExecutionException, InterruptedException { - Map, RangeSet> idsAvailable = readIdsAvailable(); - Map, RangeSet> subRangeDeletions = readSubRangeDeletions(); - - for (Range current = getTrackedRange(tsRange.lowerEndpoint()); - current.lowerEndpoint().isBefore(tsRange.upperEndpoint()); - current = getTrackedRange(current.lowerEndpoint().plus(RESOLUTION))) { - // TODO(reuvenlax): shouldn't need to iterate over all ranges. - boolean rangeCleared; - if (!tsRange.encloses(current)) { - // This can happen if the beginning or the end of tsRange doesn't fall on a RESOLUTION - // boundary. Since we - // are deleting a portion of a tracked range, track what we are deleting. - RangeSet rangeDeletions = - subRangeDeletions.computeIfAbsent(current, r -> TreeRangeSet.create()); - rangeDeletions.add(tsRange.intersection(current)); - // If we ended up deleting the whole range, than we can simply remove it from the tracking - // map. - rangeCleared = rangeDeletions.encloses(current); - } else { - rangeCleared = true; - } - if (rangeCleared) { - // Remove the range from both maps. - idsAvailable.remove(current); - subRangeDeletions.remove(current); - } - } - writeValues(idsAvailable, subRangeDeletions); - } - - private void writeValues( - Map, RangeSet> idsAvailable, - Map, RangeSet> subRangeDeletions) { - if (idsAvailable.isEmpty()) { - idsAvailable.clear(); - } else { - idsAvailableValue.write(idsAvailable); - } - if (subRangeDeletions.isEmpty()) { - subRangeDeletionsValue.clear(); - } else { - subRangeDeletionsValue.write(subRangeDeletions); - } - } - } - - @AutoValue - abstract static class TimestampedValueWithId { - private static final Comparator> COMPARATOR = - Comparator., Instant>comparing(v -> v.getValue().getTimestamp()) - .thenComparingLong(TimestampedValueWithId::getId); - - abstract TimestampedValue getValue(); - - abstract long getId(); - - static TimestampedValueWithId of(TimestampedValue value, long id) { - return new AutoValue_WindmillStateInternals_TimestampedValueWithId<>(value, id); - } - - static TimestampedValueWithId bound(Instant ts) { - return of(TimestampedValue.of(null, ts), Long.MIN_VALUE); - } - } - - static class WindmillOrderedList extends SimpleWindmillState implements OrderedListState { - private final ByteString stateKey; - private final String stateFamily; - private final Coder elemCoder; - private boolean complete; - private boolean cleared = false; - // We need to sort based on timestamp, but we need objects with the same timestamp to be treated - // as unique. We can't use a MultiSet as we can't construct a comparator that uniquely - // identifies objects, - // so we construct a unique in-memory long ids for each element. - private SortedSet> pendingAdds = - Sets.newTreeSet(TimestampedValueWithId.COMPARATOR); - - private RangeSet pendingDeletes = TreeRangeSet.create(); - private IdTracker idTracker; - - // The default proto values for SortedListRange correspond to the minimum and maximum - // timestamps. - static final long MIN_TS_MICROS = SortedListRange.getDefaultInstance().getStart(); - static final long MAX_TS_MICROS = SortedListRange.getDefaultInstance().getLimit(); - - private WindmillOrderedList( - StateTable derivedStateTable, - StateNamespace namespace, - StateTag> spec, - String stateFamily, - Coder elemCoder, - boolean isNewKey) { - - this.stateKey = encodeKey(namespace, spec); - this.stateFamily = stateFamily; - this.elemCoder = elemCoder; - this.complete = isNewKey; - this.idTracker = new IdTracker(derivedStateTable, namespace, spec, stateFamily, complete); - } - - @Override - public Iterable> read() { - return readRange(null, null); - } - - private SortedSet> getPendingAddRange( - @Nullable Instant minTimestamp, @Nullable Instant limitTimestamp) { - SortedSet> pendingInRange = pendingAdds; - if (minTimestamp != null && limitTimestamp != null) { - pendingInRange = - pendingInRange.subSet( - TimestampedValueWithId.bound(minTimestamp), - TimestampedValueWithId.bound(limitTimestamp)); - } else if (minTimestamp == null && limitTimestamp != null) { - pendingInRange = pendingInRange.headSet(TimestampedValueWithId.bound(limitTimestamp)); - } else if (limitTimestamp == null && minTimestamp != null) { - pendingInRange = pendingInRange.tailSet(TimestampedValueWithId.bound(minTimestamp)); - } - return pendingInRange; - } - - @Override - public Iterable> readRange( - @Nullable Instant minTimestamp, @Nullable Instant limitTimestamp) { - idTracker.readLater(); - - final Future>> future = getFuture(minTimestamp, limitTimestamp); - try (Closeable scope = scopedReadState()) { - SortedSet> pendingInRange = - getPendingAddRange(minTimestamp, limitTimestamp); - - // Transform the return iterator so it has the same type as pendingAdds. We need to ensure - // that the ids don't overlap with any in pendingAdds, so begin with pendingAdds.size(). - Iterable> data = - new Iterable>() { - private Iterable> iterable = future.get(); - - @Override - public Iterator> iterator() { - return new Iterator>() { - private Iterator> iter = iterable.iterator(); - private long currentId = pendingAdds.size(); - - @Override - public boolean hasNext() { - return iter.hasNext(); - } - - @Override - public TimestampedValueWithId next() { - return TimestampedValueWithId.of(iter.next(), currentId++); - } - }; - } - }; - - Iterable> includingAdds = - Iterables.mergeSorted( - ImmutableList.of(data, pendingInRange), TimestampedValueWithId.COMPARATOR); - Iterable> fullIterable = - Iterables.filter( - Iterables.transform(includingAdds, TimestampedValueWithId::getValue), - tv -> !pendingDeletes.contains(tv.getTimestamp())); - // TODO(reuvenlax): If we have a known bounded amount of data, cache known ranges. - return fullIterable; - } catch (InterruptedException | ExecutionException | IOException e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw new RuntimeException("Unable to read state", e); - } - } - - @Override - public void clear() { - cleared = true; - complete = true; - pendingAdds.clear(); - pendingDeletes.clear(); - try { - idTracker.clear(); - } catch (ExecutionException | InterruptedException e) { - throw new RuntimeException(e); - } - } - - @Override - public void clearRange(Instant minTimestamp, Instant limitTimestamp) { - getPendingAddRange(minTimestamp, limitTimestamp).clear(); - pendingDeletes.add(Range.closedOpen(minTimestamp, limitTimestamp)); - } - - @Override - public void add(TimestampedValue value) { - // We use the current size of the container as the in-memory id. This works because - // pendingAdds is completely - // cleared when it is processed (otherwise we could end up with duplicate elements in the same - // container). These - // are not the ids that will be sent to windmill. - pendingAdds.add(TimestampedValueWithId.of(value, pendingAdds.size())); - // Leave pendingDeletes alone. Since we can have multiple values with the same timestamp, we - // may still need - // overlapping deletes to remove previous entries at this timestamp. - } - - @Override - public ReadableState isEmpty() { - return new ReadableState() { - @Override - public ReadableState readLater() { - WindmillOrderedList.this.readLater(); - return this; - } - - @Override - public Boolean read() { - return Iterables.isEmpty(WindmillOrderedList.this.read()); - } - }; - } - - @Override - public OrderedListState readLater() { - return readRangeLater(null, null); - } - - @Override - @SuppressWarnings("FutureReturnValueIgnored") - public OrderedListState readRangeLater( - @Nullable Instant minTimestamp, @Nullable Instant limitTimestamp) { - idTracker.readLater(); - getFuture(minTimestamp, limitTimestamp); - return this; - } - - @Override - public WorkItemCommitRequest persistDirectly(WindmillStateCache.ForKeyAndFamily cache) - throws IOException { - WorkItemCommitRequest.Builder commitBuilder = WorkItemCommitRequest.newBuilder(); - TagSortedListUpdateRequest.Builder updatesBuilder = - commitBuilder - .addSortedListUpdatesBuilder() - .setStateFamily(cache.getStateFamily()) - .setTag(stateKey); - try { - if (cleared) { - // Default range. - updatesBuilder.addDeletesBuilder().build(); - cleared = false; - } - - if (!pendingAdds.isEmpty()) { - // TODO(reuvenlax): Once we start caching data, we should remove this line. We have it - // here now - // because once we persist - // added data we forget about it from the cache, so the object is no longer complete. - complete = false; - - TagSortedListInsertRequest.Builder insertBuilder = updatesBuilder.addInsertsBuilder(); - idTracker.add( - pendingAdds, - (elem, id) -> { - try { - ByteStringOutputStream elementStream = new ByteStringOutputStream(); - elemCoder.encode(elem.getValue(), elementStream, Context.OUTER); - insertBuilder.addEntries( - SortedListEntry.newBuilder() - .setValue(elementStream.toByteString()) - .setSortKey( - WindmillTimeUtils.harnessToWindmillTimestamp(elem.getTimestamp())) - .setId(id)); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - pendingAdds.clear(); - insertBuilder.build(); - } - - if (!pendingDeletes.isEmpty()) { - for (Range range : pendingDeletes.asRanges()) { - TagSortedListDeleteRequest.Builder deletesBuilder = updatesBuilder.addDeletesBuilder(); - deletesBuilder.setRange( - SortedListRange.newBuilder() - .setStart(WindmillTimeUtils.harnessToWindmillTimestamp(range.lowerEndpoint())) - .setLimit(WindmillTimeUtils.harnessToWindmillTimestamp(range.upperEndpoint()))); - deletesBuilder.build(); - idTracker.remove(range); - } - pendingDeletes.clear(); - } - } catch (ExecutionException | InterruptedException e) { - throw new RuntimeException(e); - } - return commitBuilder.buildPartial(); - } - - private Future>> getFuture( - @Nullable Instant minTimestamp, @Nullable Instant limitTimestamp) { - long startSortKey = - minTimestamp != null - ? WindmillTimeUtils.harnessToWindmillTimestamp(minTimestamp) - : MIN_TS_MICROS; - long limitSortKey = - limitTimestamp != null - ? WindmillTimeUtils.harnessToWindmillTimestamp(limitTimestamp) - : MAX_TS_MICROS; - - if (complete) { - // Right now we don't cache any data, so complete means an empty list. - // TODO(reuvenlax): change this once we start caching data. - return Futures.immediateFuture(Collections.emptyList()); - } - return reader.orderedListFuture( - Range.closedOpen(startSortKey, limitSortKey), stateKey, stateFamily, elemCoder); - } - } - - static class WindmillSet extends SimpleWindmillState implements SetState { - WindmillMap windmillMap; - - WindmillSet( - StateNamespace namespace, - StateTag> address, - String stateFamily, - Coder keyCoder, - WindmillStateCache.ForKeyAndFamily cache, - boolean isNewKey) { - StateTag> internalMapAddress = - StateTags.convertToMapTagInternal(address); - WindmillMap cachedMap = - (WindmillMap) cache.get(namespace, internalMapAddress); - this.windmillMap = - (cachedMap != null) - ? cachedMap - : new WindmillMap<>( - namespace, - internalMapAddress, - stateFamily, - keyCoder, - BooleanCoder.of(), - isNewKey); - } - - @Override - protected WorkItemCommitRequest persistDirectly(WindmillStateCache.ForKeyAndFamily cache) - throws IOException { - return windmillMap.persistDirectly(cache); - } - - @Override - public @UnknownKeyFor @NonNull @Initialized ReadableState< - @UnknownKeyFor @NonNull @Initialized Boolean> - contains(K k) { - return windmillMap.getOrDefault(k, false); - } - - @Override - public @UnknownKeyFor @NonNull @Initialized ReadableState< - @UnknownKeyFor @NonNull @Initialized Boolean> - addIfAbsent(K k) { - return new ReadableState() { - ReadableState putState = windmillMap.putIfAbsent(k, true); - - @Override - public @Nullable Boolean read() { - Boolean result = putState.read(); - return (result != null) ? result : false; - } - - @Override - public @UnknownKeyFor @NonNull @Initialized ReadableState readLater() { - putState = putState.readLater(); - return this; - } - }; - } - - @Override - public void remove(K k) { - windmillMap.remove(k); - } - - @Override - public void add(K value) { - windmillMap.put(value, true); - } - - @Override - public @UnknownKeyFor @NonNull @Initialized ReadableState< - @UnknownKeyFor @NonNull @Initialized Boolean> - isEmpty() { - return windmillMap.isEmpty(); - } - - @Override - public @Nullable Iterable read() { - return windmillMap.keys().read(); - } - - @Override - public @UnknownKeyFor @NonNull @Initialized SetState readLater() { - windmillMap.keys().readLater(); - return this; - } - - @Override - public void clear() { - windmillMap.clear(); - } - - @Override - void initializeForWorkItem( - WindmillStateReader reader, Supplier scopedReadStateSupplier) { - windmillMap.initializeForWorkItem(reader, scopedReadStateSupplier); - } - - @Override - void cleanupAfterWorkItem() { - windmillMap.cleanupAfterWorkItem(); - } - } - - static class WindmillMap extends SimpleWindmillState implements MapState { - private final StateNamespace namespace; - private final StateTag> address; - private final ByteString stateKeyPrefix; - private final String stateFamily; - private final Coder keyCoder; - private final Coder valueCoder; - private boolean complete; - - // TODO(reuvenlax): Should we evict items from the cache? We would have to make sure - // that anything in the cache that is not committed is not evicted. negativeCache could be - // evicted whenever we want. - private Map cachedValues = Maps.newHashMap(); - private Set negativeCache = Sets.newHashSet(); - private boolean cleared = false; - - private Set localAdditions = Sets.newHashSet(); - private Set localRemovals = Sets.newHashSet(); - - WindmillMap( - StateNamespace namespace, - StateTag> address, - String stateFamily, - Coder keyCoder, - Coder valueCoder, - boolean isNewKey) { - this.namespace = namespace; - this.address = address; - this.stateKeyPrefix = encodeKey(namespace, address); - this.stateFamily = stateFamily; - this.keyCoder = keyCoder; - this.valueCoder = valueCoder; - this.complete = isNewKey; - } - - private K userKeyFromProtoKey(ByteString tag) throws IOException { - Preconditions.checkState(tag.startsWith(stateKeyPrefix)); - ByteString keyBytes = tag.substring(stateKeyPrefix.size()); - return keyCoder.decode(keyBytes.newInput(), Context.OUTER); - } - - private ByteString protoKeyFromUserKey(K key) throws IOException { - ByteStringOutputStream keyStream = new ByteStringOutputStream(); - stateKeyPrefix.writeTo(keyStream); - keyCoder.encode(key, keyStream, Context.OUTER); - return keyStream.toByteString(); - } - - @Override - protected WorkItemCommitRequest persistDirectly(WindmillStateCache.ForKeyAndFamily cache) - throws IOException { - if (!cleared && localAdditions.isEmpty() && localRemovals.isEmpty()) { - // No changes, so return directly. - return WorkItemCommitRequest.newBuilder().buildPartial(); - } - - WorkItemCommitRequest.Builder commitBuilder = WorkItemCommitRequest.newBuilder(); - - if (cleared) { - commitBuilder - .addTagValuePrefixDeletesBuilder() - .setStateFamily(stateFamily) - .setTagPrefix(stateKeyPrefix); - } - cleared = false; - - for (K key : localAdditions) { - ByteString keyBytes = protoKeyFromUserKey(key); - ByteStringOutputStream valueStream = new ByteStringOutputStream(); - valueCoder.encode(cachedValues.get(key), valueStream, Context.OUTER); - ByteString valueBytes = valueStream.toByteString(); - - commitBuilder - .addValueUpdatesBuilder() - .setTag(keyBytes) - .setStateFamily(stateFamily) - .getValueBuilder() - .setData(valueBytes) - .setTimestamp(Long.MAX_VALUE); - } - localAdditions.clear(); - - for (K key : localRemovals) { - ByteStringOutputStream keyStream = new ByteStringOutputStream(); - stateKeyPrefix.writeTo(keyStream); - keyCoder.encode(key, keyStream, Context.OUTER); - ByteString keyBytes = keyStream.toByteString(); - // Leaving data blank means that we delete the tag. - commitBuilder - .addValueUpdatesBuilder() - .setTag(keyBytes) - .setStateFamily(stateFamily) - .getValueBuilder() - .setTimestamp(Long.MAX_VALUE); - - V cachedValue = cachedValues.remove(key); - if (cachedValue != null) { - ByteStringOutputStream valueStream = new ByteStringOutputStream(); - valueCoder.encode(cachedValues.get(key), valueStream, Context.OUTER); - } - } - negativeCache.addAll(localRemovals); - localRemovals.clear(); - - // TODO(reuvenlax): We should store in the cache parameter, as that would enable caching the - // map - // between work items, reducing fetches to Windmill. To do so, we need keep track of the - // encoded size - // of the map, and to do so efficiently (i.e. without iterating over the entire map on every - // persist) - // we need to track the sizes of each map entry. - cache.put(namespace, address, this, 1); - return commitBuilder.buildPartial(); - } - - @Override - public @UnknownKeyFor @NonNull @Initialized ReadableState get(K key) { - return getOrDefault(key, null); - } - - @Override - public @UnknownKeyFor @NonNull @Initialized ReadableState getOrDefault( - K key, @Nullable V defaultValue) { - return new ReadableState() { - @Override - public @Nullable V read() { - Future persistedData = getFutureForKey(key); - try (Closeable scope = scopedReadState()) { - if (localRemovals.contains(key) || negativeCache.contains(key)) { - return null; - } - @Nullable V cachedValue = cachedValues.get(key); - if (cachedValue != null || complete) { - return cachedValue; - } - - V persistedValue = persistedData.get(); - if (persistedValue == null) { - negativeCache.add(key); - return defaultValue; - } - // TODO: Don't do this if it was already in cache. - cachedValues.put(key, persistedValue); - return persistedValue; - } catch (InterruptedException | ExecutionException | IOException e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw new RuntimeException("Unable to read state", e); - } - } - - @Override - @SuppressWarnings("FutureReturnValueIgnored") - public @UnknownKeyFor @NonNull @Initialized ReadableState readLater() { - WindmillMap.this.getFutureForKey(key); - return this; - } - }; - } - - @Override - public @UnknownKeyFor @NonNull @Initialized ReadableState< - @UnknownKeyFor @NonNull @Initialized Iterable> - keys() { - ReadableState>> entries = entries(); - return new ReadableState>() { - @Override - public @Nullable Iterable read() { - return Iterables.transform(entries.read(), e -> e.getKey()); - } - - @Override - public @UnknownKeyFor @NonNull @Initialized ReadableState> readLater() { - entries.readLater(); - return this; - } - }; - } - - @Override - public @UnknownKeyFor @NonNull @Initialized ReadableState< - @UnknownKeyFor @NonNull @Initialized Iterable> - values() { - ReadableState>> entries = entries(); - return new ReadableState>() { - @Override - public @Nullable Iterable read() { - return Iterables.transform(entries.read(), e -> e.getValue()); - } - - @Override - public @UnknownKeyFor @NonNull @Initialized ReadableState> readLater() { - entries.readLater(); - return this; - } - }; - } - - @Override - public @UnknownKeyFor @NonNull @Initialized ReadableState< - @UnknownKeyFor @NonNull @Initialized Iterable< - @UnknownKeyFor @NonNull @Initialized Entry>> - entries() { - return new ReadableState>>() { - @Override - public Iterable> read() { - if (complete) { - return Iterables.unmodifiableIterable(cachedValues.entrySet()); - } - Future>> persistedData = getFuture(); - try (Closeable scope = scopedReadState()) { - Iterable> data = persistedData.get(); - Iterable> transformedData = - Iterables., Map.Entry>transform( - data, - entry -> { - try { - return new AbstractMap.SimpleEntry<>( - userKeyFromProtoKey(entry.getKey()), entry.getValue()); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - - if (data instanceof Weighted) { - // This is a known amount of data. Cache it all. - transformedData.forEach( - e -> { - // The cached data overrides what is read from state, so call putIfAbsent. - cachedValues.putIfAbsent(e.getKey(), e.getValue()); - }); - complete = true; - return Iterables.unmodifiableIterable(cachedValues.entrySet()); - } else { - // This means that the result might be too large to cache, so don't add it to the - // local cache. Instead merge the iterables, giving priority to any local additions - // (represented in cachedValued and localRemovals) that may not have been committed - // yet. - return Iterables.unmodifiableIterable( - Iterables.concat( - cachedValues.entrySet(), - Iterables.filter( - transformedData, - e -> - !cachedValues.containsKey(e.getKey()) - && !localRemovals.contains(e.getKey())))); - } - - } catch (InterruptedException | ExecutionException | IOException e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw new RuntimeException("Unable to read state", e); - } - } - - @Override - @SuppressWarnings("FutureReturnValueIgnored") - public @UnknownKeyFor @NonNull @Initialized ReadableState>> - readLater() { - WindmillMap.this.getFuture(); - return this; - } - }; - } - - @Override - public ReadableState isEmpty() { - return new ReadableState() { - // TODO(reuvenlax): Can we find a more efficient way of implementing isEmpty than reading - // the entire map? - ReadableState> keys = WindmillMap.this.keys(); - - @Override - public @Nullable Boolean read() { - return Iterables.isEmpty(keys.read()); - } - - @Override - public @UnknownKeyFor @NonNull @Initialized ReadableState readLater() { - keys.readLater(); - return this; - } - }; - } - - @Override - public void put(K key, V value) { - V oldValue = cachedValues.put(key, value); - if (valueCoder.consistentWithEquals() && value.equals(oldValue)) { - return; - } - localAdditions.add(key); - localRemovals.remove(key); - negativeCache.remove(key); - } - - @Override - public @UnknownKeyFor @NonNull @Initialized ReadableState computeIfAbsent( - K key, Function mappingFunction) { - Future persistedData = getFutureForKey(key); - try (Closeable scope = scopedReadState()) { - if (localRemovals.contains(key) || negativeCache.contains(key)) { - return ReadableStates.immediate(null); - } - @Nullable V cachedValue = cachedValues.get(key); - if (cachedValue != null || complete) { - return ReadableStates.immediate(cachedValue); - } - - V persistedValue = persistedData.get(); - if (persistedValue == null) { - // This is a new value. Add it to the map and return null. - put(key, mappingFunction.apply(key)); - return ReadableStates.immediate(null); - } - // TODO: Don't do this if it was already in cache. - cachedValues.put(key, persistedValue); - return ReadableStates.immediate(persistedValue); - } catch (InterruptedException | ExecutionException | IOException e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw new RuntimeException("Unable to read state", e); - } - } - - @Override - public void remove(K key) { - if (localRemovals.add(key)) { - cachedValues.remove(key); - localAdditions.remove(key); - } - } - - @Override - public void clear() { - cachedValues.clear(); - localAdditions.clear(); - localRemovals.clear(); - negativeCache.clear(); - cleared = true; - complete = true; - } - - private Future getFutureForKey(K key) { - try { - ByteStringOutputStream keyStream = new ByteStringOutputStream(); - stateKeyPrefix.writeTo(keyStream); - keyCoder.encode(key, keyStream, Context.OUTER); - return reader.valueFuture(keyStream.toByteString(), stateFamily, valueCoder); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private Future>> getFuture() { - if (complete) { - // The caller will merge in local cached values. - return Futures.immediateFuture(Collections.emptyList()); - } else { - return reader.valuePrefixFuture(stateKeyPrefix, stateFamily, valueCoder); - } - } - }; - - private static class WindmillBag extends SimpleWindmillState implements BagState { - - private final StateNamespace namespace; - private final StateTag> address; - private final ByteString stateKey; - private final String stateFamily; - private final Coder elemCoder; - - private boolean cleared = false; - /** - * If non-{@literal null}, this contains the complete contents of the bag, except for any local - * additions. If {@literal null} then we don't know if Windmill contains additional values which - * should be part of the bag. We'll need to read them if the work item actually wants the bag - * contents. - */ - private ConcatIterables cachedValues = null; - - private List localAdditions = new ArrayList<>(); - private long encodedSize = 0; - - private WindmillBag( - StateNamespace namespace, - StateTag> address, - String stateFamily, - Coder elemCoder, - boolean isNewKey) { - this.namespace = namespace; - this.address = address; - this.stateKey = encodeKey(namespace, address); - this.stateFamily = stateFamily; - this.elemCoder = elemCoder; - if (isNewKey) { - this.cachedValues = new ConcatIterables<>(); - } - } - - @Override - public void clear() { - cleared = true; - cachedValues = new ConcatIterables<>(); - localAdditions = new ArrayList<>(); - encodedSize = 0; - } - - /** - * Return iterable over all bag values in Windmill which should contribute to overall bag - * contents. - */ - private Iterable fetchData(Future> persistedData) { - try (Closeable scope = scopedReadState()) { - if (cachedValues != null) { - return cachedValues.snapshot(); - } - Iterable data = persistedData.get(); - if (data instanceof Weighted) { - // We have a known bounded amount of data; cache it. - cachedValues = new ConcatIterables<>(); - cachedValues.extendWith(data); - encodedSize = ((Weighted) data).getWeight(); - return cachedValues.snapshot(); - } else { - // This is an iterable that may not fit in memory at once; don't cache it. - return data; - } - } catch (InterruptedException | ExecutionException | IOException e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw new RuntimeException("Unable to read state", e); - } - } - - public boolean valuesAreCached() { - return cachedValues != null; - } - - @Override - @SuppressWarnings("FutureReturnValueIgnored") - public WindmillBag readLater() { - getFuture(); - return this; - } - - @Override - public Iterable read() { - return Iterables.concat( - fetchData(getFuture()), Iterables.limit(localAdditions, localAdditions.size())); - } - - @Override - public ReadableState isEmpty() { - return new ReadableState() { - @Override - public ReadableState readLater() { - WindmillBag.this.readLater(); - return this; - } - - @Override - public Boolean read() { - return Iterables.isEmpty(fetchData(getFuture())) && localAdditions.isEmpty(); - } - }; - } - - @Override - public void add(T input) { - localAdditions.add(input); - } - - @Override - public WorkItemCommitRequest persistDirectly(WindmillStateCache.ForKeyAndFamily cache) - throws IOException { - WorkItemCommitRequest.Builder commitBuilder = WorkItemCommitRequest.newBuilder(); - - Windmill.TagBag.Builder bagUpdatesBuilder = null; - - if (cleared) { - bagUpdatesBuilder = commitBuilder.addBagUpdatesBuilder(); - bagUpdatesBuilder.setDeleteAll(true); - cleared = false; - } - - if (!localAdditions.isEmpty()) { - // Tell Windmill to capture the local additions. - if (bagUpdatesBuilder == null) { - bagUpdatesBuilder = commitBuilder.addBagUpdatesBuilder(); - } - for (T value : localAdditions) { - ByteStringOutputStream stream = new ByteStringOutputStream(); - // Encode the value - elemCoder.encode(value, stream, Coder.Context.OUTER); - ByteString encoded = stream.toByteString(); - if (cachedValues != null) { - // We'll capture this value in the cache below. - // Capture the value's size now since we have it. - encodedSize += encoded.size(); - } - bagUpdatesBuilder.addValues(encoded); - } - } - - if (bagUpdatesBuilder != null) { - bagUpdatesBuilder.setTag(stateKey).setStateFamily(stateFamily); - } - - if (cachedValues != null) { - if (!localAdditions.isEmpty()) { - // Capture the local additions in the cached value since we and - // Windmill are now in agreement. - cachedValues.extendWith(localAdditions); - } - // We now know the complete bag contents, and any read on it will yield a - // cached value, so cache it for future reads. - cache.put(namespace, address, this, encodedSize); - } - - // Don't reuse the localAdditions object; we don't want future changes to it to - // modify the value of cachedValues. - localAdditions = new ArrayList<>(); - - return commitBuilder.buildPartial(); - } - - private Future> getFuture() { - return cachedValues != null ? null : reader.bagFuture(stateKey, stateFamily, elemCoder); - } - } - - private static class ConcatIterables implements Iterable { - // List of component iterables. Should only be appended to in order to support snapshot(). - List> iterables; - - public ConcatIterables() { - this.iterables = new ArrayList<>(); - } - - public void extendWith(Iterable iterable) { - iterables.add(iterable); - } - - @Override - public Iterator iterator() { - return Iterators.concat(Iterables.transform(iterables, Iterable::iterator).iterator()); - } - - /** - * Returns a view of the current state of this iterable. Remembers the current length of - * iterables so that the returned value Will not change due to future extendWith() calls. - */ - public Iterable snapshot() { - final int limit = iterables.size(); - final List> iterablesList = iterables; - return () -> - Iterators.concat( - Iterators.transform( - Iterators.limit(iterablesList.iterator(), limit), Iterable::iterator)); - } - } - - private static class WindmillWatermarkHold extends WindmillState implements WatermarkHoldState { - // The encoded size of an Instant. - private static final int ENCODED_SIZE = 8; - - private final TimestampCombiner timestampCombiner; - private final StateNamespace namespace; - private final StateTag address; - private final ByteString stateKey; - private final String stateFamily; - - private boolean cleared = false; - /** - * If non-{@literal null}, the known current hold value, or absent if we know there are no - * output watermark holds. If {@literal null}, the current hold value could depend on holds in - * Windmill we do not yet know. - */ - private Optional cachedValue = null; - - private Instant localAdditions = null; - - private WindmillWatermarkHold( - StateNamespace namespace, - StateTag address, - String stateFamily, - TimestampCombiner timestampCombiner, - boolean isNewKey) { - this.namespace = namespace; - this.address = address; - this.stateKey = encodeKey(namespace, address); - this.stateFamily = stateFamily; - this.timestampCombiner = timestampCombiner; - if (isNewKey) { - cachedValue = Optional.absent(); - } - } - - @Override - public void clear() { - cleared = true; - cachedValue = Optional.absent(); - localAdditions = null; - } - - @Override - @SuppressWarnings("FutureReturnValueIgnored") - public WindmillWatermarkHold readLater() { - getFuture(); - return this; - } - - @Override - public Instant read() { - try (Closeable scope = scopedReadState()) { - Instant persistedHold = getFuture().get(); - if (persistedHold == null) { - cachedValue = Optional.absent(); - } else { - cachedValue = Optional.of(persistedHold); - } - } catch (InterruptedException | ExecutionException | IOException e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw new RuntimeException("Unable to read state", e); - } - - if (localAdditions == null) { - return cachedValue.orNull(); - } else if (!cachedValue.isPresent()) { - return localAdditions; - } else { - return timestampCombiner.combine(localAdditions, cachedValue.get()); - } - } - - @Override - public ReadableState isEmpty() { - throw new UnsupportedOperationException(); - } - - @Override - public void add(Instant outputTime) { - localAdditions = - (localAdditions == null) - ? outputTime - : timestampCombiner.combine(outputTime, localAdditions); - } - - @Override - public TimestampCombiner getTimestampCombiner() { - return timestampCombiner; - } - - @Override - public Future persist(final WindmillStateCache.ForKeyAndFamily cache) { - - Future result; - - if (!cleared && localAdditions == null) { - // No changes, so no need to update Windmill and no need to cache any value. - return Futures.immediateFuture(WorkItemCommitRequest.newBuilder().buildPartial()); - } - - if (cleared && localAdditions == null) { - // Just clearing the persisted state; blind delete - WorkItemCommitRequest.Builder commitBuilder = WorkItemCommitRequest.newBuilder(); - commitBuilder - .addWatermarkHoldsBuilder() - .setTag(stateKey) - .setStateFamily(stateFamily) - .setReset(true); - - result = Futures.immediateFuture(commitBuilder.buildPartial()); - } else if (cleared && localAdditions != null) { - // Since we cleared before adding, we can do a blind overwrite of persisted state - WorkItemCommitRequest.Builder commitBuilder = WorkItemCommitRequest.newBuilder(); - commitBuilder - .addWatermarkHoldsBuilder() - .setTag(stateKey) - .setStateFamily(stateFamily) - .setReset(true) - .addTimestamps(WindmillTimeUtils.harnessToWindmillTimestamp(localAdditions)); - - cachedValue = Optional.of(localAdditions); - - result = Futures.immediateFuture(commitBuilder.buildPartial()); - } else if (!cleared && localAdditions != null) { - // Otherwise, we need to combine the local additions with the already persisted data - result = combineWithPersisted(); - } else { - throw new IllegalStateException("Unreachable condition"); - } - - return Futures.lazyTransform( - result, - result1 -> { - cleared = false; - localAdditions = null; - if (cachedValue != null) { - cache.put(namespace, address, WindmillWatermarkHold.this, ENCODED_SIZE); - } - return result1; - }); - } - - private Future getFuture() { - return cachedValue != null - ? Futures.immediateFuture(cachedValue.orNull()) - : reader.watermarkFuture(stateKey, stateFamily); - } - - /** - * Combines local additions with persisted data and mutates the {@code commitBuilder} to write - * the result. - */ - private Future combineWithPersisted() { - boolean windmillCanCombine = false; - - // If the combined output time depends only on the window, then we are just blindly adding - // the same value that may or may not already be present. This depends on the state only being - // used for one window. - windmillCanCombine |= timestampCombiner.dependsOnlyOnWindow(); - - // If the combined output time depends only on the earliest input timestamp, then because - // assignOutputTime is monotonic, the hold only depends on the earliest output timestamp - // (which is the value submitted as a watermark hold). The only way holds for later inputs - // can be redundant is if the are later (or equal) to the earliest. So taking the MIN - // implicitly, as Windmill does, has the desired behavior. - windmillCanCombine |= timestampCombiner.dependsOnlyOnEarliestTimestamp(); - - if (windmillCanCombine) { - // We do a blind write and let Windmill take the MIN - WorkItemCommitRequest.Builder commitBuilder = WorkItemCommitRequest.newBuilder(); - commitBuilder - .addWatermarkHoldsBuilder() - .setTag(stateKey) - .setStateFamily(stateFamily) - .addTimestamps(WindmillTimeUtils.harnessToWindmillTimestamp(localAdditions)); - - if (cachedValue != null) { - cachedValue = - Optional.of( - cachedValue.isPresent() - ? timestampCombiner.combine(cachedValue.get(), localAdditions) - : localAdditions); - } - - return Futures.immediateFuture(commitBuilder.buildPartial()); - } else { - // The non-fast path does a read-modify-write - return Futures.lazyTransform( - (cachedValue != null) - ? Futures.immediateFuture(cachedValue.orNull()) - : reader.watermarkFuture(stateKey, stateFamily), - priorHold -> { - cachedValue = - Optional.of( - (priorHold != null) - ? timestampCombiner.combine(priorHold, localAdditions) - : localAdditions); - WorkItemCommitRequest.Builder commitBuilder = WorkItemCommitRequest.newBuilder(); - commitBuilder - .addWatermarkHoldsBuilder() - .setTag(stateKey) - .setStateFamily(stateFamily) - .setReset(true) - .addTimestamps(WindmillTimeUtils.harnessToWindmillTimestamp(cachedValue.get())); - - return commitBuilder.buildPartial(); - }); - } - } - } - - private static class WindmillCombiningState extends WindmillState - implements CombiningState { - - private final WindmillBag bag; - private final CombineFn combineFn; - - /* We use a separate, in-memory AccumT rather than relying on the WindmillWatermarkBag's - * localAdditions, because we want to combine multiple InputT's to a single AccumT - * before adding it. - */ - private AccumT localAdditionsAccum; - private boolean hasLocalAdditions = false; - - private WindmillCombiningState( - StateNamespace namespace, - StateTag> address, - String stateFamily, - Coder accumCoder, - CombineFn combineFn, - WindmillStateCache.ForKeyAndFamily cache, - boolean isNewKey) { - StateTag> internalBagAddress = StateTags.convertToBagTagInternal(address); - WindmillBag cachedBag = - (WindmillBag) cache.get(namespace, internalBagAddress); - this.bag = - (cachedBag != null) - ? cachedBag - : new WindmillBag<>(namespace, internalBagAddress, stateFamily, accumCoder, isNewKey); - this.combineFn = combineFn; - this.localAdditionsAccum = combineFn.createAccumulator(); - } - - @Override - void initializeForWorkItem( - WindmillStateReader reader, Supplier scopedReadStateSupplier) { - super.initializeForWorkItem(reader, scopedReadStateSupplier); - this.bag.initializeForWorkItem(reader, scopedReadStateSupplier); - } - - @Override - void cleanupAfterWorkItem() { - super.cleanupAfterWorkItem(); - bag.cleanupAfterWorkItem(); - } - - @Override - public WindmillCombiningState readLater() { - bag.readLater(); - return this; - } - - @Override - public OutputT read() { - return combineFn.extractOutput(getAccum()); - } - - @Override - public void add(InputT input) { - hasLocalAdditions = true; - localAdditionsAccum = combineFn.addInput(localAdditionsAccum, input); - } - - @Override - public void clear() { - bag.clear(); - localAdditionsAccum = combineFn.createAccumulator(); - hasLocalAdditions = false; - } - - @Override - public Future persist(WindmillStateCache.ForKeyAndFamily cache) - throws IOException { - if (hasLocalAdditions) { - if (COMPACT_NOW.get().get() || bag.valuesAreCached()) { - // Implicitly clears the bag and combines local and persisted accumulators. - localAdditionsAccum = getAccum(); - } - bag.add(combineFn.compact(localAdditionsAccum)); - localAdditionsAccum = combineFn.createAccumulator(); - hasLocalAdditions = false; - } - - return bag.persist(cache); - } - - @Override - public AccumT getAccum() { - Iterable accums = - Iterables.concat(bag.read(), Collections.singleton(localAdditionsAccum)); - - // Compact things - AccumT merged = combineFn.mergeAccumulators(accums); - bag.clear(); - localAdditionsAccum = merged; - hasLocalAdditions = true; - return merged; - } - - @Override - public ReadableState isEmpty() { - final ReadableState bagIsEmpty = bag.isEmpty(); - return new ReadableState() { - @Override - public ReadableState readLater() { - bagIsEmpty.readLater(); - return this; - } - - @Override - public Boolean read() { - return !hasLocalAdditions && bagIsEmpty.read(); - } - }; - } - - @Override - public void addAccum(AccumT accum) { - hasLocalAdditions = true; - localAdditionsAccum = combineFn.mergeAccumulators(Arrays.asList(localAdditionsAccum, accum)); - } - - @Override - public AccumT mergeAccumulators(Iterable accumulators) { - return combineFn.mergeAccumulators(accumulators); - } - } - - @VisibleForTesting - static final ThreadLocal> COMPACT_NOW = - ThreadLocal.withInitial( - () -> - new Supplier() { - /* The rate at which, on average, this will return true. */ - static final double RATE = 0.002; - Random random = new Random(); - long counter = nextSample(); - - private long nextSample() { - // Use geometric distribution to find next true value. - // This lets us avoid invoking random.nextDouble() on every call. - return (long) Math.floor(Math.log(random.nextDouble()) / Math.log(1 - RATE)); - } - - @Override - public Boolean get() { - counter--; - if (counter < 0) { - counter = nextSample(); - return true; - } else { - return false; - } - } - }); -} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillStateReader.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillStateReader.java deleted file mode 100644 index 4339e173f9bbf..0000000000000 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillStateReader.java +++ /dev/null @@ -1,1007 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.runners.dataflow.worker; - -import com.google.api.client.util.Lists; -import com.google.auto.value.AutoValue; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.io.IOException; -import java.io.InputStream; -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.function.Supplier; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import org.apache.beam.runners.dataflow.worker.WindmillStateReader.StateTag.Kind; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.SortedListEntry; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.SortedListRange; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.TagBag; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.TagSortedListFetchRequest; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.TagValue; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.TagValuePrefixRequest; -import org.apache.beam.sdk.coders.Coder; -import org.apache.beam.sdk.coders.Coder.Context; -import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.sdk.util.Weighted; -import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.AbstractIterator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ForwardingList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Range; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ForwardingFuture; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Futures; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.SettableFuture; -import org.joda.time.Instant; - -/** - * Reads persistent state from {@link Windmill}. Returns {@code Future}s containing the data that - * has been read. Will not initiate a read until {@link Future#get} is called, at which point all - * the pending futures will be read. - * - *

    CAUTION Watch out for escaping references to the reader ending up inside {@link - * WindmillStateCache}. - */ -@SuppressWarnings({ - "rawtypes", // TODO(https://github.com/apache/beam/issues/20447) - "nullness" // TODO(https://github.com/apache/beam/issues/20497) -}) -class WindmillStateReader { - /** - * Ideal maximum bytes in a TagBag response. However, Windmill will always return at least one - * value if possible irrespective of this limit. - */ - public static final long INITIAL_MAX_BAG_BYTES = 8L << 20; // 8MB - - public static final long CONTINUATION_MAX_BAG_BYTES = 32L << 20; // 32MB - - /** - * Ideal maximum bytes in a TagSortedList response. However, Windmill will always return at least - * one value if possible irrespective of this limit. - */ - public static final long MAX_ORDERED_LIST_BYTES = 8L << 20; // 8MB - - /** - * Ideal maximum bytes in a tag-value prefix response. However, Windmill will always return at - * least one value if possible irrespective of this limit. - */ - public static final long MAX_TAG_VALUE_PREFIX_BYTES = 8L << 20; // 8MB - - /** - * Ideal maximum bytes in a KeyedGetDataResponse. However, Windmill will always return at least - * one value if possible irrespective of this limit. - */ - public static final long MAX_KEY_BYTES = 16L << 20; // 16MB - - public static final long MAX_CONTINUATION_KEY_BYTES = 72L << 20; // 72MB - - /** - * When combined with a key and computationId, represents the unique address for state managed by - * Windmill. - */ - @AutoValue - abstract static class StateTag { - enum Kind { - VALUE, - BAG, - WATERMARK, - ORDERED_LIST, - VALUE_PREFIX - } - - abstract Kind getKind(); - - abstract ByteString getTag(); - - abstract String getStateFamily(); - - /** - * For {@link Kind#BAG, Kind#ORDERED_LIST, Kind#VALUE_PREFIX} kinds: A previous - * 'continuation_position' returned by Windmill to signal the resulting bag was incomplete. - * Sending that position will request the next page of values. Null for first request. - * - *

    Null for other kinds. - */ - @Nullable - abstract RequestPositionT getRequestPosition(); - - /** For {@link Kind#ORDERED_LIST} kinds: the range to fetch or delete. */ - @Nullable - abstract Range getSortedListRange(); - - static StateTag of( - Kind kind, ByteString tag, String stateFamily, @Nullable RequestPositionT requestPosition) { - return new AutoValue_WindmillStateReader_StateTag.Builder() - .setKind(kind) - .setTag(tag) - .setStateFamily(stateFamily) - .setRequestPosition(requestPosition) - .build(); - } - - static StateTag of( - Kind kind, ByteString tag, String stateFamily) { - return of(kind, tag, stateFamily, null); - } - - abstract Builder toBuilder(); - - @AutoValue.Builder - abstract static class Builder { - abstract Builder setKind(Kind kind); - - abstract Builder setTag(ByteString tag); - - abstract Builder setStateFamily(String stateFamily); - - abstract Builder setRequestPosition( - @Nullable RequestPositionT requestPosition); - - abstract Builder setSortedListRange(@Nullable Range sortedListRange); - - abstract StateTag build(); - } - } - - /** - * An in-memory collection of deserialized values and an optional continuation position to pass to - * Windmill when fetching the next page of values. - */ - private static class ValuesAndContPosition { - private final List values; - - /** Position to pass to next request for next page of values. Null if done. */ - private final @Nullable ContinuationT continuationPosition; - - public ValuesAndContPosition(List values, @Nullable ContinuationT continuationPosition) { - this.values = values; - this.continuationPosition = continuationPosition; - } - } - - private final String computation; - private final ByteString key; - private final long shardingKey; - private final long workToken; - - // WindmillStateReader should only perform blocking i/o in a try-with-resources block that - // declares an AutoCloseable vended by readWrapperSupplier. - private final Supplier readWrapperSupplier; - - private final MetricTrackingWindmillServerStub server; - - private long bytesRead = 0L; - - public WindmillStateReader( - MetricTrackingWindmillServerStub server, - String computation, - ByteString key, - long shardingKey, - long workToken, - Supplier readWrapperSupplier) { - this.server = server; - this.computation = computation; - this.key = key; - this.shardingKey = shardingKey; - this.workToken = workToken; - this.readWrapperSupplier = readWrapperSupplier; - } - - public WindmillStateReader( - MetricTrackingWindmillServerStub server, - String computation, - ByteString key, - long shardingKey, - long workToken) { - this(server, computation, key, shardingKey, workToken, () -> null); - } - - private static final class CoderAndFuture { - private Coder coder = null; - private final SettableFuture future; - - private CoderAndFuture(Coder coder, SettableFuture future) { - this.coder = coder; - this.future = future; - } - - private SettableFuture getFuture() { - return future; - } - - private SettableFuture getNonDoneFuture(StateTag stateTag) { - if (future.isDone()) { - throw new IllegalStateException("Future for " + stateTag + " is already done"); - } - return future; - } - - private Coder getAndClearCoder() { - if (coder == null) { - throw new IllegalStateException("Coder has already been cleared from cache"); - } - Coder result = (Coder) coder; - if (result == null) { - throw new IllegalStateException("Coder has already been cleared from cache"); - } - coder = null; - return result; - } - - private void checkNoCoder() { - if (coder != null) { - throw new IllegalStateException("Unexpected coder"); - } - } - } - - @VisibleForTesting - ConcurrentLinkedQueue> pendingLookups = new ConcurrentLinkedQueue<>(); - - private ConcurrentHashMap, CoderAndFuture> waiting = new ConcurrentHashMap<>(); - - private Future stateFuture(StateTag stateTag, @Nullable Coder coder) { - CoderAndFuture coderAndFuture = new CoderAndFuture<>(coder, SettableFuture.create()); - CoderAndFuture existingCoderAndFutureWildcard = - waiting.putIfAbsent(stateTag, coderAndFuture); - if (existingCoderAndFutureWildcard == null) { - // Schedule a new request. It's response is guaranteed to find the future and coder. - pendingLookups.add(stateTag); - } else { - // Piggy-back on the pending or already answered request. - @SuppressWarnings("unchecked") - CoderAndFuture existingCoderAndFuture = - (CoderAndFuture) existingCoderAndFutureWildcard; - coderAndFuture = existingCoderAndFuture; - } - - return wrappedFuture(coderAndFuture.getFuture()); - } - - private CoderAndFuture getWaiting(StateTag stateTag, boolean shouldRemove) { - CoderAndFuture coderAndFutureWildcard; - if (shouldRemove) { - coderAndFutureWildcard = waiting.remove(stateTag); - } else { - coderAndFutureWildcard = waiting.get(stateTag); - } - if (coderAndFutureWildcard == null) { - throw new IllegalStateException("Missing future for " + stateTag); - } - @SuppressWarnings("unchecked") - CoderAndFuture coderAndFuture = (CoderAndFuture) coderAndFutureWildcard; - return coderAndFuture; - } - - public Future watermarkFuture(ByteString encodedTag, String stateFamily) { - return stateFuture(StateTag.of(StateTag.Kind.WATERMARK, encodedTag, stateFamily), null); - } - - public Future valueFuture(ByteString encodedTag, String stateFamily, Coder coder) { - return stateFuture(StateTag.of(StateTag.Kind.VALUE, encodedTag, stateFamily), coder); - } - - public Future> bagFuture( - ByteString encodedTag, String stateFamily, Coder elemCoder) { - // First request has no continuation position. - StateTag stateTag = StateTag.of(StateTag.Kind.BAG, encodedTag, stateFamily); - // Convert the ValuesAndContPosition to Iterable. - return valuesToPagingIterableFuture(stateTag, elemCoder, this.stateFuture(stateTag, elemCoder)); - } - - public Future>> orderedListFuture( - Range range, ByteString encodedTag, String stateFamily, Coder elemCoder) { - // First request has no continuation position. - StateTag stateTag = - StateTag.of(StateTag.Kind.ORDERED_LIST, encodedTag, stateFamily) - .toBuilder() - .setSortedListRange(Preconditions.checkNotNull(range)) - .build(); - return Preconditions.checkNotNull( - valuesToPagingIterableFuture(stateTag, elemCoder, this.stateFuture(stateTag, elemCoder))); - } - - public Future>> valuePrefixFuture( - ByteString prefix, String stateFamily, Coder valueCoder) { - // First request has no continuation position. - StateTag stateTag = - StateTag.of(Kind.VALUE_PREFIX, prefix, stateFamily).toBuilder().build(); - return Preconditions.checkNotNull( - valuesToPagingIterableFuture(stateTag, valueCoder, this.stateFuture(stateTag, valueCoder))); - } - - /** - * Internal request to fetch the next 'page' of values. Return null if no continuation position is - * in {@code contStateTag}, which signals there are no more pages. - */ - private @Nullable - Future> continuationFuture( - StateTag contStateTag, Coder coder) { - if (contStateTag.getRequestPosition() == null) { - // We're done. - return null; - } - return stateFuture(contStateTag, coder); - } - - /** - * A future which will trigger a GetData request to Windmill for all outstanding futures on the - * first {@link #get}. - */ - private static class WrappedFuture extends ForwardingFuture.SimpleForwardingFuture { - /** - * The reader we'll use to service the eventual read. Null if read has been fulfilled. - * - *

    NOTE: We must clear this after the read is fulfilled to prevent space leaks. - */ - private @Nullable WindmillStateReader reader; - - public WrappedFuture(WindmillStateReader reader, Future delegate) { - super(delegate); - this.reader = reader; - } - - @Override - public T get() throws InterruptedException, ExecutionException { - if (!delegate().isDone() && reader != null) { - // Only one thread per reader, so no race here. - reader.startBatchAndBlock(); - } - reader = null; - return super.get(); - } - - @Override - public T get(long timeout, TimeUnit unit) - throws InterruptedException, ExecutionException, TimeoutException { - if (!delegate().isDone() && reader != null) { - // Only one thread per reader, so no race here. - reader.startBatchAndBlock(); - } - reader = null; - return super.get(timeout, unit); - } - } - - private Future wrappedFuture(final Future future) { - if (future.isDone()) { - // If the underlying lookup is already complete, we don't need to create the wrapper. - return future; - } else { - // Otherwise, wrap the true future so we know when to trigger a GetData. - return new WrappedFuture<>(this, future); - } - } - - /** Function to extract an {@link Iterable} from the continuation-supporting page read future. */ - private static class ToIterableFunction - implements Function, Iterable> { - /** - * Reader to request continuation pages from, or {@literal null} if no continuation pages - * required. - */ - private @Nullable WindmillStateReader reader; - - private final StateTag stateTag; - private final Coder coder; - - public ToIterableFunction( - WindmillStateReader reader, StateTag stateTag, Coder coder) { - this.reader = reader; - this.stateTag = stateTag; - this.coder = coder; - } - - @SuppressFBWarnings( - value = "NP_METHOD_PARAMETER_TIGHTENS_ANNOTATION", - justification = "https://github.com/google/guava/issues/920") - @Override - public Iterable apply( - @Nonnull ValuesAndContPosition valuesAndContPosition) { - if (valuesAndContPosition.continuationPosition == null) { - // Number of values is small enough Windmill sent us the entire bag in one response. - reader = null; - return valuesAndContPosition.values; - } else { - // Return an iterable which knows how to come back for more. - StateTag contStateTag = - StateTag.of( - stateTag.getKind(), - stateTag.getTag(), - stateTag.getStateFamily(), - valuesAndContPosition.continuationPosition); - if (stateTag.getSortedListRange() != null) { - contStateTag = - contStateTag.toBuilder().setSortedListRange(stateTag.getSortedListRange()).build(); - } - return new PagingIterable( - reader, valuesAndContPosition.values, contStateTag, coder); - } - } - } - - /** - * Return future which transforms a {@code ValuesAndContPosition} result into the initial - * Iterable result expected from the external caller. - */ - private Future> valuesToPagingIterableFuture( - final StateTag stateTag, - final Coder coder, - final Future> future) { - Function, Iterable> toIterable = - new ToIterableFunction<>(this, stateTag, coder); - return Futures.lazyTransform(future, toIterable); - } - - public void startBatchAndBlock() { - // First, drain work out of the pending lookups into a set. These will be the items we fetch. - HashSet> toFetch = Sets.newHashSet(); - try { - while (!pendingLookups.isEmpty()) { - StateTag stateTag = pendingLookups.poll(); - if (stateTag == null) { - break; - } - - if (!toFetch.add(stateTag)) { - throw new IllegalStateException("Duplicate tags being fetched."); - } - } - - // If we failed to drain anything, some other thread pulled it off the queue. We have no work - // to do. - if (toFetch.isEmpty()) { - return; - } - - Windmill.KeyedGetDataRequest request = createRequest(toFetch); - Windmill.KeyedGetDataResponse response; - try (AutoCloseable readWrapper = readWrapperSupplier.get()) { - response = server.getStateData(computation, request); - } - if (response == null) { - throw new RuntimeException("Windmill unexpectedly returned null for request " + request); - } - - // Removes tags from toFetch as they are processed. - consumeResponse(response, toFetch); - } catch (Exception e) { - // Set up all the remaining futures for this key to throw an exception. This ensures that if - // the exception is caught that all futures have been completed and do not block. - for (StateTag stateTag : toFetch) { - waiting.get(stateTag).future.setException(e); - } - throw new RuntimeException(e); - } - } - - public long getBytesRead() { - return bytesRead; - } - - private Windmill.KeyedGetDataRequest createRequest(Iterable> toFetch) { - Windmill.KeyedGetDataRequest.Builder keyedDataBuilder = - Windmill.KeyedGetDataRequest.newBuilder() - .setKey(key) - .setShardingKey(shardingKey) - .setWorkToken(workToken); - - boolean continuation = false; - List> orderedListsToFetch = Lists.newArrayList(); - for (StateTag stateTag : toFetch) { - switch (stateTag.getKind()) { - case BAG: - TagBag.Builder bag = - keyedDataBuilder - .addBagsToFetchBuilder() - .setTag(stateTag.getTag()) - .setStateFamily(stateTag.getStateFamily()); - if (stateTag.getRequestPosition() == null) { - bag.setFetchMaxBytes(INITIAL_MAX_BAG_BYTES); - } else { - // We're asking for the next page. - bag.setFetchMaxBytes(CONTINUATION_MAX_BAG_BYTES); - bag.setRequestPosition((Long) stateTag.getRequestPosition()); - continuation = true; - } - break; - - case ORDERED_LIST: - orderedListsToFetch.add(stateTag); - break; - - case WATERMARK: - keyedDataBuilder - .addWatermarkHoldsToFetchBuilder() - .setTag(stateTag.getTag()) - .setStateFamily(stateTag.getStateFamily()); - break; - - case VALUE: - keyedDataBuilder - .addValuesToFetchBuilder() - .setTag(stateTag.getTag()) - .setStateFamily(stateTag.getStateFamily()); - break; - - case VALUE_PREFIX: - TagValuePrefixRequest.Builder prefixFetchBuilder = - keyedDataBuilder - .addTagValuePrefixesToFetchBuilder() - .setTagPrefix(stateTag.getTag()) - .setStateFamily(stateTag.getStateFamily()) - .setFetchMaxBytes(MAX_TAG_VALUE_PREFIX_BYTES); - if (stateTag.getRequestPosition() != null) { - prefixFetchBuilder.setRequestPosition((ByteString) stateTag.getRequestPosition()); - } - break; - - default: - throw new RuntimeException("Unknown kind of tag requested: " + stateTag.getKind()); - } - } - orderedListsToFetch.sort( - Comparator.>comparingLong(s -> s.getSortedListRange().lowerEndpoint()) - .thenComparingLong(s -> s.getSortedListRange().upperEndpoint())); - for (StateTag stateTag : orderedListsToFetch) { - Range range = Preconditions.checkNotNull(stateTag.getSortedListRange()); - TagSortedListFetchRequest.Builder sorted_list = - keyedDataBuilder - .addSortedListsToFetchBuilder() - .setTag(stateTag.getTag()) - .setStateFamily(stateTag.getStateFamily()) - .setFetchMaxBytes(MAX_ORDERED_LIST_BYTES); - sorted_list.addFetchRanges( - SortedListRange.newBuilder() - .setStart(range.lowerEndpoint()) - .setLimit(range.upperEndpoint()) - .build()); - if (stateTag.getRequestPosition() != null) { - // We're asking for the next page. - sorted_list.setRequestPosition((ByteString) stateTag.getRequestPosition()); - } - } - if (continuation) { - keyedDataBuilder.setMaxBytes(MAX_CONTINUATION_KEY_BYTES); - } else { - keyedDataBuilder.setMaxBytes(MAX_KEY_BYTES); - } - - return keyedDataBuilder.build(); - } - - private void consumeResponse(Windmill.KeyedGetDataResponse response, Set> toFetch) { - bytesRead += response.getSerializedSize(); - if (response.getFailed()) { - throw new KeyTokenInvalidException(key.toStringUtf8()); - } - - if (!key.equals(response.getKey())) { - throw new RuntimeException("Expected data for key " + key + " but was " + response.getKey()); - } - - for (Windmill.TagBag bag : response.getBagsList()) { - StateTag stateTag = - StateTag.of( - StateTag.Kind.BAG, - bag.getTag(), - bag.getStateFamily(), - bag.hasRequestPosition() ? bag.getRequestPosition() : null); - if (!toFetch.remove(stateTag)) { - throw new IllegalStateException( - "Received response for unrequested tag " + stateTag + ". Pending tags: " + toFetch); - } - consumeBag(bag, stateTag); - } - - for (Windmill.WatermarkHold hold : response.getWatermarkHoldsList()) { - StateTag stateTag = - StateTag.of(StateTag.Kind.WATERMARK, hold.getTag(), hold.getStateFamily()); - if (!toFetch.remove(stateTag)) { - throw new IllegalStateException( - "Received response for unrequested tag " + stateTag + ". Pending tags: " + toFetch); - } - consumeWatermark(hold, stateTag); - } - - for (Windmill.TagValue value : response.getValuesList()) { - StateTag stateTag = - StateTag.of(StateTag.Kind.VALUE, value.getTag(), value.getStateFamily()); - if (!toFetch.remove(stateTag)) { - throw new IllegalStateException( - "Received response for unrequested tag " + stateTag + ". Pending tags: " + toFetch); - } - consumeTagValue(value, stateTag); - } - for (Windmill.TagValuePrefixResponse prefix_response : response.getTagValuePrefixesList()) { - StateTag stateTag = - StateTag.of( - Kind.VALUE_PREFIX, - prefix_response.getTagPrefix(), - prefix_response.getStateFamily(), - prefix_response.hasRequestPosition() ? prefix_response.getRequestPosition() : null); - if (!toFetch.remove(stateTag)) { - throw new IllegalStateException( - "Received response for unrequested tag " + stateTag + ". Pending tags: " + toFetch); - } - consumeTagPrefixResponse(prefix_response, stateTag); - } - for (Windmill.TagSortedListFetchResponse sorted_list : response.getTagSortedListsList()) { - SortedListRange sortedListRange = Iterables.getOnlyElement(sorted_list.getFetchRangesList()); - Range range = Range.closedOpen(sortedListRange.getStart(), sortedListRange.getLimit()); - StateTag stateTag = - StateTag.of( - StateTag.Kind.ORDERED_LIST, - sorted_list.getTag(), - sorted_list.getStateFamily(), - sorted_list.hasRequestPosition() ? sorted_list.getRequestPosition() : null) - .toBuilder() - .setSortedListRange(range) - .build(); - if (!toFetch.remove(stateTag)) { - throw new IllegalStateException( - "Received response for unrequested tag " + stateTag + ". Pending tags: " + toFetch); - } - - consumeSortedList(sorted_list, stateTag); - } - - if (!toFetch.isEmpty()) { - throw new IllegalStateException( - "Didn't receive responses for all pending fetches. Missing: " + toFetch); - } - } - - @VisibleForTesting - static class WeightedList extends ForwardingList implements Weighted { - private List delegate; - long weight; - - WeightedList(List delegate) { - this.delegate = delegate; - this.weight = 0; - } - - @Override - protected List delegate() { - return delegate; - } - - @Override - public boolean add(T elem) { - throw new UnsupportedOperationException("Must use AddWeighted()"); - } - - @Override - public long getWeight() { - return weight; - } - - public void addWeighted(T elem, long weight) { - delegate.add(elem); - this.weight += weight; - } - } - - /** The deserialized values in {@code bag} as a read-only array list. */ - private List bagPageValues(TagBag bag, Coder elemCoder) { - if (bag.getValuesCount() == 0) { - return new WeightedList(Collections.emptyList()); - } - - WeightedList valueList = new WeightedList<>(new ArrayList(bag.getValuesCount())); - for (ByteString value : bag.getValuesList()) { - try { - valueList.addWeighted( - elemCoder.decode(value.newInput(), Coder.Context.OUTER), value.size()); - } catch (IOException e) { - throw new IllegalStateException("Unable to decode tag list using " + elemCoder, e); - } - } - return valueList; - } - - private List> sortedListPageValues( - Windmill.TagSortedListFetchResponse sortedListFetchResponse, Coder elemCoder) { - if (sortedListFetchResponse.getEntriesCount() == 0) { - return new WeightedList<>(Collections.emptyList()); - } - - WeightedList> entryList = - new WeightedList<>(new ArrayList<>(sortedListFetchResponse.getEntriesCount())); - for (SortedListEntry entry : sortedListFetchResponse.getEntriesList()) { - try { - T value = elemCoder.decode(entry.getValue().newInput(), Coder.Context.OUTER); - entryList.addWeighted( - TimestampedValue.of( - value, WindmillTimeUtils.windmillToHarnessTimestamp(entry.getSortKey())), - entry.getValue().size() + 8); - } catch (IOException e) { - throw new IllegalStateException("Unable to decode tag sorted list using " + elemCoder, e); - } - } - return entryList; - } - - private List> tagPrefixPageTagValues( - Windmill.TagValuePrefixResponse tagValuePrefixResponse, Coder valueCoder) { - if (tagValuePrefixResponse.getTagValuesCount() == 0) { - return new WeightedList<>(Collections.emptyList()); - } - - WeightedList> entryList = - new WeightedList>( - new ArrayList<>(tagValuePrefixResponse.getTagValuesCount())); - for (TagValue entry : tagValuePrefixResponse.getTagValuesList()) { - try { - V value = valueCoder.decode(entry.getValue().getData().newInput(), Context.OUTER); - entryList.addWeighted( - new AbstractMap.SimpleEntry<>(entry.getTag(), value), - entry.getTag().size() + entry.getValue().getData().size()); - } catch (IOException e) { - throw new IllegalStateException("Unable to decode tag value " + e); - } - } - return entryList; - } - - private void consumeBag(TagBag bag, StateTag stateTag) { - boolean shouldRemove; - if (stateTag.getRequestPosition() == null) { - // This is the response for the first page. - // Leave the future in the cache so subsequent requests for the first page - // can return immediately. - shouldRemove = false; - } else { - // This is a response for a subsequent page. - // Don't cache the future since we may need to make multiple requests with different - // continuation positions. - shouldRemove = true; - } - CoderAndFuture> coderAndFuture = - getWaiting(stateTag, shouldRemove); - SettableFuture> future = - coderAndFuture.getNonDoneFuture(stateTag); - try { - Coder coder = coderAndFuture.getAndClearCoder(); - List values = this.bagPageValues(bag, coder); - future.set( - new ValuesAndContPosition<>( - values, bag.hasContinuationPosition() ? bag.getContinuationPosition() : null)); - } catch (Exception e) { - future.setException(new RuntimeException("Error parsing bag response", e)); - } - } - - private void consumeWatermark(Windmill.WatermarkHold watermarkHold, StateTag stateTag) { - CoderAndFuture coderAndFuture = getWaiting(stateTag, false); - SettableFuture future = coderAndFuture.getNonDoneFuture(stateTag); - // No coders for watermarks - coderAndFuture.checkNoCoder(); - - Instant hold = null; - for (long timestamp : watermarkHold.getTimestampsList()) { - Instant instant = new Instant(TimeUnit.MICROSECONDS.toMillis(timestamp)); - // TIMESTAMP_MAX_VALUE represents infinity, and windmill will return it if no hold is set, so - // don't treat it as a hold here. - if (instant.isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE) - && (hold == null || instant.isBefore(hold))) { - hold = instant; - } - } - - future.set(hold); - } - - private void consumeTagValue(TagValue tagValue, StateTag stateTag) { - CoderAndFuture coderAndFuture = getWaiting(stateTag, false); - SettableFuture future = coderAndFuture.getNonDoneFuture(stateTag); - Coder coder = coderAndFuture.getAndClearCoder(); - - if (tagValue.hasValue() - && tagValue.getValue().hasData() - && !tagValue.getValue().getData().isEmpty()) { - InputStream inputStream = tagValue.getValue().getData().newInput(); - try { - T value = coder.decode(inputStream, Coder.Context.OUTER); - future.set(value); - } catch (IOException e) { - future.setException(new IllegalStateException("Unable to decode value using " + coder, e)); - } - } else { - future.set(null); - } - } - - private void consumeTagPrefixResponse( - Windmill.TagValuePrefixResponse tagValuePrefixResponse, StateTag stateTag) { - boolean shouldRemove; - if (stateTag.getRequestPosition() == null) { - // This is the response for the first page.// Leave the future in the cache so subsequent - // requests for the first page - // can return immediately. - shouldRemove = false; - } else { - // This is a response for a subsequent page. - // Don't cache the future since we may need to make multiple requests with different - // continuation positions. - shouldRemove = true; - } - - CoderAndFuture, ByteString>> coderAndFuture = - getWaiting(stateTag, shouldRemove); - SettableFuture, ByteString>> future = - coderAndFuture.getNonDoneFuture(stateTag); - Coder valueCoder = coderAndFuture.getAndClearCoder(); - try { - List> values = - this.tagPrefixPageTagValues(tagValuePrefixResponse, valueCoder); - future.set( - new ValuesAndContPosition<>( - values, - tagValuePrefixResponse.hasContinuationPosition() - ? tagValuePrefixResponse.getContinuationPosition() - : null)); - } catch (Exception e) { - future.setException(new RuntimeException("Error parsing tag value prefix", e)); - } - } - - private void consumeSortedList( - Windmill.TagSortedListFetchResponse sortedListFetchResponse, StateTag stateTag) { - boolean shouldRemove; - if (stateTag.getRequestPosition() == null) { - // This is the response for the first page.// Leave the future in the cache so subsequent - // requests for the first page - // can return immediately. - shouldRemove = false; - } else { - // This is a response for a subsequent page. - // Don't cache the future since we may need to make multiple requests with different - // continuation positions. - shouldRemove = true; - } - - CoderAndFuture, ByteString>> coderAndFuture = - getWaiting(stateTag, shouldRemove); - SettableFuture, ByteString>> future = - coderAndFuture.getNonDoneFuture(stateTag); - Coder coder = coderAndFuture.getAndClearCoder(); - try { - List> values = this.sortedListPageValues(sortedListFetchResponse, coder); - future.set( - new ValuesAndContPosition<>( - values, - sortedListFetchResponse.hasContinuationPosition() - ? sortedListFetchResponse.getContinuationPosition() - : null)); - } catch (Exception e) { - future.setException(new RuntimeException("Error parsing ordered list", e)); - } - } - /** - * An iterable over elements backed by paginated GetData requests to Windmill. The iterable may be - * iterated over an arbitrary number of times and multiple iterators may be active simultaneously. - * - *

    There are two pattern we wish to support with low -memory and -latency: - * - *

      - *
    1. Re-iterate over the initial elements multiple times (eg Iterables.first). We'll cache the - * initial 'page' of values returned by Windmill from our first request for the lifetime of - * the iterable. - *
    2. Iterate through all elements of a very large collection. We'll send the GetData request - * for the next page when the current page is begun. We'll discard intermediate pages and - * only retain the first. Thus the maximum memory pressure is one page plus one page per - * call to iterator. - *
    - */ - private static class PagingIterable implements Iterable { - /** - * The reader we will use for scheduling continuation pages. - * - *

    NOTE We've made this explicit to remind us to be careful not to cache the iterable. - */ - private final WindmillStateReader reader; - - /** Initial values returned for the first page. Never reclaimed. */ - private final List firstPage; - - /** State tag with continuation position set for second page. */ - private final StateTag secondPagePos; - - /** Coder for elements. */ - private final Coder coder; - - private PagingIterable( - WindmillStateReader reader, - List firstPage, - StateTag secondPagePos, - Coder coder) { - this.reader = reader; - this.firstPage = firstPage; - this.secondPagePos = secondPagePos; - this.coder = coder; - } - - @Override - public Iterator iterator() { - return new AbstractIterator() { - private Iterator currentPage = firstPage.iterator(); - private StateTag nextPagePos = secondPagePos; - private Future> pendingNextPage = - // NOTE: The results of continuation page reads are never cached. - reader.continuationFuture(nextPagePos, coder); - - @Override - protected ResultT computeNext() { - while (true) { - if (currentPage.hasNext()) { - return currentPage.next(); - } - if (pendingNextPage == null) { - return endOfData(); - } - - ValuesAndContPosition valuesAndContPosition; - try { - valuesAndContPosition = pendingNextPage.get(); - } catch (InterruptedException | ExecutionException e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw new RuntimeException("Unable to read value from state", e); - } - currentPage = valuesAndContPosition.values.iterator(); - StateTag.Builder nextPageBuilder = - StateTag.of( - nextPagePos.getKind(), - nextPagePos.getTag(), - nextPagePos.getStateFamily(), - valuesAndContPosition.continuationPosition) - .toBuilder(); - if (secondPagePos.getSortedListRange() != null) { - nextPageBuilder.setSortedListRange(secondPagePos.getSortedListRange()); - } - nextPagePos = nextPageBuilder.build(); - pendingNextPage = - // NOTE: The results of continuation page reads are never cached. - reader.continuationFuture(nextPagePos, coder); - } - } - }; - } - } -} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillTimeUtils.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillTimeUtils.java index 9732826bdd6b1..e3ceddda6375f 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillTimeUtils.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillTimeUtils.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.dataflow.worker; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillTimerInternals.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillTimerInternals.java index c2d2d3b67d242..5d56186ad94cd 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillTimerInternals.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillTimerInternals.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -36,10 +36,10 @@ import org.apache.beam.sdk.util.ExposedByteArrayOutputStream; import org.apache.beam.sdk.util.VarInt; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashBasedTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Table; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Table.Cell; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBasedTable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Table; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Table.Cell; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindowingWindmillReader.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindowingWindmillReader.java index 462b979ffc6c8..38d00319a4c3a 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindowingWindmillReader.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindowingWindmillReader.java @@ -34,8 +34,8 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkItemStatusClient.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkItemStatusClient.java index fcf227334cd80..85cf877481a1c 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkItemStatusClient.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkItemStatusClient.java @@ -17,9 +17,9 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.services.dataflow.model.CounterUpdate; import com.google.api.services.dataflow.model.MetricStructuredName; @@ -49,8 +49,8 @@ import org.apache.beam.runners.dataflow.worker.util.common.worker.NativeReader.Progress; import org.apache.beam.runners.dataflow.worker.util.common.worker.ReadOperation; import org.apache.beam.sdk.util.UserCodeException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.slf4j.Logger; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkUnitClient.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkUnitClient.java index 12d85069c5203..82fbcd82c131a 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkUnitClient.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkUnitClient.java @@ -21,7 +21,7 @@ import com.google.api.services.dataflow.model.WorkItemServiceState; import com.google.api.services.dataflow.model.WorkItemStatus; import java.io.IOException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; +import java.util.Optional; /** Abstract base class describing a client for WorkItem work units. */ interface WorkUnitClient { @@ -31,14 +31,14 @@ interface WorkUnitClient { Optional getWorkItem() throws IOException; /** - * Returns a new global streaming config WorkItem, or returns {@link Optional#absent()} if no work + * Returns a new global streaming config WorkItem, or returns {@link Optional#empty()} if no work * was found. */ Optional getGlobalStreamingConfigWorkItem() throws IOException; /** * Returns a streaming config WorkItem for the given computation, or returns {@link - * Optional#absent()} if no work was found. + * Optional#empty()} if no work was found. */ Optional getStreamingConfigWorkItem(String computationId) throws IOException; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java index f986a9b0a90c5..a9050236efc80 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java @@ -24,7 +24,7 @@ import static org.apache.beam.runners.dataflow.util.Structs.getStrings; import static org.apache.beam.sdk.util.SerializableUtils.deserializeFromByteArray; import static org.apache.beam.sdk.util.SerializableUtils.serializeToByteArray; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.client.util.Base64; import com.google.api.services.dataflow.model.ApproximateReportedProgress; @@ -62,10 +62,10 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.ValueWithRecordId; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; @@ -776,6 +776,9 @@ public double getRemainingParallelism() { private static class UnboundedReaderIterator extends NativeReader.NativeReaderIterator>> { + // Do not close reader. The reader is cached in StreamingModeExecutionContext.readerCache, and + // will be reused until the cache is evicted, expired or invalidated. + // See UnboundedReader#iterator(). private final UnboundedSource.UnboundedReader reader; private final StreamingModeExecutionContext context; private final boolean started; @@ -862,7 +865,9 @@ public WindowedValue> getCurrent() throws NoSuchElementExce } @Override - public void close() {} + public void close() { + // Don't close reader. + } @Override public NativeReader.Progress getProgress() { diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerUncaughtExceptionHandler.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerUncaughtExceptionHandler.java index fa3910c695b91..5a8e87d23ab99 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerUncaughtExceptionHandler.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerUncaughtExceptionHandler.java @@ -21,7 +21,7 @@ import java.lang.Thread.UncaughtExceptionHandler; import org.apache.beam.runners.dataflow.worker.logging.DataflowWorkerLoggingInitializer; import org.apache.beam.runners.dataflow.worker.util.common.worker.JvmRuntime; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/apiary/Apiary.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/apiary/Apiary.java index 4d0dfe2a88fc6..2c918c2571f5b 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/apiary/Apiary.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/apiary/Apiary.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.dataflow.worker.apiary; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * Static convenience methods to work around default encodings done by Apiary for default fields. diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/apiary/FixMultiOutputInfosOnParDoInstructions.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/apiary/FixMultiOutputInfosOnParDoInstructions.java index e7e0c18486339..80ebd771faf0a 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/apiary/FixMultiOutputInfosOnParDoInstructions.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/apiary/FixMultiOutputInfosOnParDoInstructions.java @@ -24,7 +24,7 @@ import java.util.List; import java.util.function.Function; import org.apache.beam.sdk.fn.IdGenerator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * {@link ParDoInstruction}s are meant to always have {@link MultiOutputInfo}s which give names to diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/Counter.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/Counter.java index 2f010efad7702..62ffd855812ba 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/Counter.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/Counter.java @@ -21,7 +21,7 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.beam.runners.dataflow.worker.counters.CounterFactory.CounterDistribution; import org.apache.beam.runners.dataflow.worker.counters.CounterFactory.CounterMean; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/CounterFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/CounterFactory.java index 1c6b5bf00cf6d..f056d578229ee 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/CounterFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/CounterFactory.java @@ -27,11 +27,11 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.beam.runners.dataflow.worker.counters.Counter.AtomicCounterValue; import org.apache.beam.runners.dataflow.worker.counters.Counter.CounterUpdateExtractor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.math.LongMath; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.AtomicDouble; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.math.LongMath; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.AtomicDouble; import org.checkerframework.checker.nullness.qual.Nullable; /** Factory interface for creating counters. */ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/CounterName.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/CounterName.java index 94f3d4ec3daeb..fbade122c1ad1 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/CounterName.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/CounterName.java @@ -17,11 +17,11 @@ */ package org.apache.beam.runners.dataflow.worker.counters; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.ToStringHelper; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.ToStringHelper; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/CounterSet.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/CounterSet.java index 6c50b99ca169e..a298228dbe420 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/CounterSet.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/CounterSet.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker.counters; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.ArrayList; import java.util.Collections; @@ -25,7 +25,7 @@ import java.util.concurrent.ConcurrentHashMap; import org.apache.beam.runners.dataflow.worker.counters.Counter.AtomicCounterValue; import org.apache.beam.runners.dataflow.worker.counters.Counter.CounterUpdateExtractor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/DataflowCounterUpdateExtractor.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/DataflowCounterUpdateExtractor.java index effe802938c71..ffb0cde884782 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/DataflowCounterUpdateExtractor.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/DataflowCounterUpdateExtractor.java @@ -29,7 +29,7 @@ import com.google.api.services.dataflow.model.SplitInt64; import org.apache.beam.runners.dataflow.worker.counters.Counter.CounterUpdateExtractor; import org.apache.beam.runners.dataflow.worker.counters.CounterFactory.CounterMean; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Factory methods for extracting {@link CounterUpdate} updates from counters. */ @SuppressWarnings({ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/NameContext.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/NameContext.java index 6188386a4e67a..4f4a1c3834e5e 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/NameContext.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/counters/NameContext.java @@ -31,7 +31,10 @@ public abstract class NameContext { * systemName} and a {@code userName}. */ public static NameContext create( - String stageName, String originalName, String systemName, String userName) { + String stageName, + @Nullable String originalName, + String systemName, + @Nullable String userName) { return new AutoValue_NameContext(stageName, originalName, systemName, userName); } @@ -44,7 +47,7 @@ public static NameContext forStage(String stageName) { } /** Returns the name of the stage this instruction is executing in. */ - public abstract @Nullable String stageName(); + public abstract String stageName(); /** * Returns the "original" name of this instruction. This name is a short name assigned by the SDK diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/Edges.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/Edges.java index 59d673cf1141a..1cdd9c95cf1c7 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/Edges.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/Edges.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker.graph; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.dataflow.model.MultiOutputInfo; import com.google.auto.value.AutoValue; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/LengthPrefixUnknownCoders.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/LengthPrefixUnknownCoders.java index 01b57aec19d01..ff90c64601853 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/LengthPrefixUnknownCoders.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/LengthPrefixUnknownCoders.java @@ -41,10 +41,10 @@ import org.apache.beam.runners.dataflow.worker.util.WorkerPropertyNames; import org.apache.beam.sdk.coders.ByteArrayCoder; import org.apache.beam.sdk.coders.LengthPrefixCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.MutableNetwork; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.MutableNetwork; /** Utility for replacing or wrapping unknown coders with {@link LengthPrefixCoder}. */ public class LengthPrefixUnknownCoders { diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/MapTaskToNetworkFunction.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/MapTaskToNetworkFunction.java index cc4aa1aefc82d..8e1705461b255 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/MapTaskToNetworkFunction.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/MapTaskToNetworkFunction.java @@ -35,9 +35,9 @@ import org.apache.beam.runners.dataflow.worker.graph.Nodes.ParallelInstructionNode; import org.apache.beam.sdk.extensions.gcp.util.Transport; import org.apache.beam.sdk.fn.IdGenerator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.MutableNetwork; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.NetworkBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.MutableNetwork; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.NetworkBuilder; /** * Creates a directed bipartite network of {@link ParallelInstructionNode}s and {@link diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/Networks.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/Networks.java index e28ddc472a681..da3272f53313c 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/Networks.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/Networks.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.dataflow.worker.graph; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.ArrayDeque; import java.util.ArrayList; @@ -33,12 +33,12 @@ import java.util.Set; import java.util.function.Function; import org.apache.beam.runners.dataflow.worker.graph.Nodes.Node; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.EndpointPair; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.MutableNetwork; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.Network; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.EndpointPair; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.MutableNetwork; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.Network; /** Static utility methods for {@link Network} instances that are directed. */ @SuppressWarnings({ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/Nodes.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/Nodes.java index 0085f831190d0..6092d0d64de5a 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/Nodes.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/graph/Nodes.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker.graph; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.client.json.GenericJson; import com.google.api.client.json.JsonFactory; @@ -33,8 +33,8 @@ import org.apache.beam.runners.dataflow.worker.util.common.worker.OutputReceiver; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.extensions.gcp.util.Transport; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; /** Container class for different types of network nodes. All nodes only have reference equality. */ @SuppressWarnings({ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingHandler.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingHandler.java index 1de5f1fa6e1c3..44daec2379a17 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingHandler.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingHandler.java @@ -43,9 +43,9 @@ import org.apache.beam.runners.core.metrics.ExecutionStateTracker.ExecutionState; import org.apache.beam.runners.dataflow.worker.DataflowOperationContext.DataflowExecutionState; import org.apache.beam.runners.dataflow.worker.counters.NameContext; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CountingOutputStream; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CountingOutputStream; /** * Formats {@link LogRecord} into JSON format for Cloud Logging. Any exception is represented using diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingInitializer.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingInitializer.java index b7527be558039..0673ae790eaf9 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingInitializer.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingInitializer.java @@ -35,9 +35,12 @@ import java.util.logging.LogManager; import java.util.logging.Logger; import org.apache.beam.runners.dataflow.options.DataflowWorkerLoggingOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableBiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.sdk.options.SdkHarnessOptions; +import org.apache.beam.sdk.options.SdkHarnessOptions.LogLevel; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableBiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.slf4j.LoggerFactory; /** * Sets up {@link java.util.logging} configuration on the Dataflow worker with a rotating file @@ -54,6 +57,8 @@ "ForbidDefaultCharset" }) public class DataflowWorkerLoggingInitializer { + private static final org.slf4j.Logger LOG = + LoggerFactory.getLogger(DataflowWorkerLoggingInitializer.class); private static final String ROOT_LOGGER_NAME = ""; @VisibleForTesting @@ -188,10 +193,20 @@ public static synchronized void configure(DataflowWorkerLoggingOptions options) if (!initialized) { throw new RuntimeException("configure() called before initialize()"); } - if (options.getDefaultWorkerLogLevel() != null) { - Level defaultLevel = getJulLevel(options.getDefaultWorkerLogLevel()); - LogManager.getLogManager().getLogger(ROOT_LOGGER_NAME).setLevel(defaultLevel); + + // For compatibility reason, we do not call SdkHarnessOptions.getConfiguredLoggerFromOptions + // to config the logging for legacy worker, instead replicate the config steps used for + // DataflowWorkerLoggingOptions for default log level and log level overrides. + SdkHarnessOptions harnessOptions = options.as(SdkHarnessOptions.class); + boolean usedDeprecated = false; + + // default value for both DefaultSdkHarnessLogLevel and DefaultWorkerLogLevel are INFO + Level overrideLevel = getJulLevel(harnessOptions.getDefaultSdkHarnessLogLevel()); + if (options.getDefaultWorkerLogLevel() != null && options.getDefaultWorkerLogLevel() != INFO) { + overrideLevel = getJulLevel(options.getDefaultWorkerLogLevel()); + usedDeprecated = true; } + LogManager.getLogManager().getLogger(ROOT_LOGGER_NAME).setLevel(overrideLevel); if (options.getWorkerLogLevelOverrides() != null) { for (Map.Entry loggerOverride : @@ -200,6 +215,14 @@ public static synchronized void configure(DataflowWorkerLoggingOptions options) logger.setLevel(getJulLevel(loggerOverride.getValue())); configuredLoggers.add(logger); } + usedDeprecated = true; + } else if (harnessOptions.getSdkHarnessLogLevelOverrides() != null) { + for (Map.Entry loggerOverride : + harnessOptions.getSdkHarnessLogLevelOverrides().entrySet()) { + Logger logger = Logger.getLogger(loggerOverride.getKey()); + logger.setLevel(getJulLevel(loggerOverride.getValue())); + configuredLoggers.add(logger); + } } // If the options specify a level for messages logged to System.out/err, we need to reconfigure @@ -223,6 +246,12 @@ public static synchronized void configure(DataflowWorkerLoggingOptions options) getJulLevel(options.getWorkerSystemErrMessageLevel()), Charset.defaultCharset())); } + + if (usedDeprecated) { + LOG.warn( + "Deprecated DataflowWorkerLoggingOptions are used for log level settings." + + "Consider using options defined in SdkHarnessOptions for forward compatibility."); + } } /** @@ -253,6 +282,10 @@ private static Level getJulLevel(DataflowWorkerLoggingOptions.Level level) { return LEVELS.inverse().get(level); } + private static Level getJulLevel(SdkHarnessOptions.LogLevel level) { + return LogLevel.LEVEL_CONFIGURATION.get(level); + } + @VisibleForTesting public static synchronized void reset() { if (!initialized) { diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/logging/JulHandlerPrintStreamAdapterFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/logging/JulHandlerPrintStreamAdapterFactory.java index 436860180050d..7e4f3a2cb184c 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/logging/JulHandlerPrintStreamAdapterFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/logging/JulHandlerPrintStreamAdapterFactory.java @@ -36,7 +36,7 @@ import java.util.logging.LogRecord; import java.util.logging.Logger; import javax.annotation.concurrent.GuardedBy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** * A {@link PrintStream} factory that creates {@link PrintStream}s which output to the specified JUL diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/options/StreamingDataflowWorkerOptions.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/options/StreamingDataflowWorkerOptions.java index 908221973fae8..cc5b3302b01bb 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/options/StreamingDataflowWorkerOptions.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/options/StreamingDataflowWorkerOptions.java @@ -19,9 +19,9 @@ import java.io.IOException; import org.apache.beam.runners.dataflow.options.DataflowWorkerHarnessOptions; -import org.apache.beam.runners.dataflow.worker.windmill.GrpcWindmillServer; -import org.apache.beam.runners.dataflow.worker.windmill.WindmillServer; import org.apache.beam.runners.dataflow.worker.windmill.WindmillServerStub; +import org.apache.beam.runners.dataflow.worker.windmill.appliance.JniWindmillApplianceServer; +import org.apache.beam.runners.dataflow.worker.windmill.grpcclient.GrpcWindmillServer; import org.apache.beam.sdk.options.Default; import org.apache.beam.sdk.options.DefaultValueFactory; import org.apache.beam.sdk.options.Description; @@ -207,12 +207,12 @@ public WindmillServerStub create(PipelineOptions options) { || streamingOptions.isEnableStreamingEngine() || streamingOptions.getLocalWindmillHostport().startsWith("grpc:")) { try { - return new GrpcWindmillServer(streamingOptions); + return GrpcWindmillServer.create(streamingOptions); } catch (IOException e) { throw new RuntimeException("Failed to create GrpcWindmillServer: ", e); } } else { - return new WindmillServer(streamingOptions.getLocalWindmillHostport()); + return new JniWindmillApplianceServer(streamingOptions.getLocalWindmillHostport()); } } } diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/profiler/ScopedProfiler.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/profiler/ScopedProfiler.java index b4c8f5f4af081..274f6b2e0e073 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/profiler/ScopedProfiler.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/profiler/ScopedProfiler.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker.profiler; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/BaseStatusServlet.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/BaseStatusServlet.java index 41580fe594e91..a198a34e1d238 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/BaseStatusServlet.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/BaseStatusServlet.java @@ -22,7 +22,7 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; /** Base class for status servlets. */ public abstract class BaseStatusServlet extends HttpServlet { diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/DebugCapture.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/DebugCapture.java index fc4501f4e7c35..c859b10717145 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/DebugCapture.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/DebugCapture.java @@ -36,7 +36,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.beam.runners.dataflow.options.DataflowWorkerHarnessOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/HeapzServlet.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/HeapzServlet.java index 9b2c613188cd8..1cf8ee26e0fee 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/HeapzServlet.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/HeapzServlet.java @@ -28,7 +28,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.beam.runners.dataflow.worker.util.MemoryMonitor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; /** * Respond to /heapz with a page allowing downloading of the heap dumps. diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/JfrzServlet.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/JfrzServlet.java index 0b7ba27c5a365..a4d6136d198f0 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/JfrzServlet.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/JfrzServlet.java @@ -26,7 +26,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.beam.runners.dataflow.worker.util.MemoryMonitor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; /** * Respond to /jfrz with a page allowing downloading of the JFR profiles. diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/SdkWorkerStatusServlet.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/SdkWorkerStatusServlet.java index 22bf351181370..1517ac73d4546 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/SdkWorkerStatusServlet.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/SdkWorkerStatusServlet.java @@ -29,7 +29,7 @@ import javax.servlet.http.HttpServletResponse; import org.apache.beam.runners.dataflow.worker.status.DebugCapture.Capturable; import org.apache.beam.runners.fnexecution.status.BeamWorkerStatusGrpcService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; /** * Servlet dedicated to provide live status info retrieved from SDK Harness. Note this is different diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/ThreadzServlet.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/ThreadzServlet.java index b79729c61cf66..b0bc52d78ca1c 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/ThreadzServlet.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/ThreadzServlet.java @@ -29,7 +29,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.beam.runners.dataflow.worker.status.DebugCapture.Capturable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; /** Respond to /threadz with the stack traces of all running threads. */ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/WorkerStatusPages.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/WorkerStatusPages.java index 34a26688bd375..08515a1fb33ef 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/WorkerStatusPages.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/WorkerStatusPages.java @@ -28,7 +28,7 @@ import org.apache.beam.runners.core.construction.Environments; import org.apache.beam.runners.dataflow.worker.status.DebugCapture.Capturable; import org.apache.beam.runners.dataflow.worker.util.MemoryMonitor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.servlet.ServletHandler; import org.eclipse.jetty.servlet.ServletHolder; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ActiveWorkState.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ActiveWorkState.java new file mode 100644 index 0000000000000..9858666c40a23 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ActiveWorkState.java @@ -0,0 +1,292 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.streaming; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList.toImmutableList; + +import java.io.PrintWriter; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Queue; +import java.util.function.BiConsumer; +import java.util.stream.Stream; +import javax.annotation.Nullable; +import javax.annotation.concurrent.GuardedBy; +import javax.annotation.concurrent.ThreadSafe; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataRequest; +import org.apache.beam.runners.dataflow.worker.windmill.state.WindmillStateCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.joda.time.Duration; +import org.joda.time.Instant; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manages the active {@link Work} queues for their {@link ShardedKey}(s). Gives an interface to + * activate, queue, and complete {@link Work} (including invalidating stuck {@link Work}). + */ +@ThreadSafe +final class ActiveWorkState { + private static final Logger LOG = LoggerFactory.getLogger(ActiveWorkState.class); + + /* The max number of keys in COMMITTING or COMMIT_QUEUED status to be shown.*/ + private static final int MAX_PRINTABLE_COMMIT_PENDING_KEYS = 50; + + /** + * Map from {@link ShardedKey} to {@link Work} for the key. The first item in the {@link + * Queue} is actively processing. + */ + @GuardedBy("this") + private final Map> activeWork; + + @GuardedBy("this") + private final WindmillStateCache.ForComputation computationStateCache; + + private ActiveWorkState( + Map> activeWork, + WindmillStateCache.ForComputation computationStateCache) { + this.activeWork = activeWork; + this.computationStateCache = computationStateCache; + } + + static ActiveWorkState create(WindmillStateCache.ForComputation computationStateCache) { + return new ActiveWorkState(new HashMap<>(), computationStateCache); + } + + @VisibleForTesting + static ActiveWorkState forTesting( + Map> activeWork, + WindmillStateCache.ForComputation computationStateCache) { + return new ActiveWorkState(activeWork, computationStateCache); + } + + /** + * Activates {@link Work} for the {@link ShardedKey}. Outcome can be 1 of 3 {@link + * ActivateWorkResult} + * + *

    1. EXECUTE: The {@link ShardedKey} has not been seen before, create a {@link Queue} + * for the key. The caller should execute the work. + * + *

    2. DUPLICATE: A work queue for the {@link ShardedKey} exists, and the work already exists in + * the {@link ShardedKey}'s work queue, mark the {@link Work} as a duplicate. + * + *

    3. QUEUED: A work queue for the {@link ShardedKey} exists, and the work is not in the key's + * work queue, queue the work for later processing. + */ + synchronized ActivateWorkResult activateWorkForKey(ShardedKey shardedKey, Work work) { + Deque workQueue = activeWork.getOrDefault(shardedKey, new ArrayDeque<>()); + + // This key does not have any work queued up on it. Create one, insert Work, and mark the work + // to be executed. + if (!activeWork.containsKey(shardedKey) || workQueue.isEmpty()) { + workQueue.addLast(work); + activeWork.put(shardedKey, workQueue); + return ActivateWorkResult.EXECUTE; + } + + // Ensure we don't already have this work token queued. + for (Work queuedWork : workQueue) { + if (queuedWork.getWorkItem().getWorkToken() == work.getWorkItem().getWorkToken()) { + return ActivateWorkResult.DUPLICATE; + } + } + + // Queue the work for later processing. + workQueue.addLast(work); + return ActivateWorkResult.QUEUED; + } + + /** + * Removes the complete work from the {@link Queue}. The {@link Work} is marked as completed + * if its workToken matches the one that is passed in. Returns the next {@link Work} in the {@link + * ShardedKey}'s work queue, if one exists else removes the {@link ShardedKey} from {@link + * #activeWork}. + */ + synchronized Optional completeWorkAndGetNextWorkForKey( + ShardedKey shardedKey, long workToken) { + @Nullable Queue workQueue = activeWork.get(shardedKey); + if (workQueue == null) { + // Work may have been completed due to clearing of stuck commits. + LOG.warn("Unable to complete inactive work for key {} and token {}.", shardedKey, workToken); + return Optional.empty(); + } + removeCompletedWorkFromQueue(workQueue, shardedKey, workToken); + return getNextWork(workQueue, shardedKey); + } + + private synchronized void removeCompletedWorkFromQueue( + Queue workQueue, ShardedKey shardedKey, long workToken) { + // avoid Preconditions.checkState here to prevent eagerly evaluating the + // format string parameters for the error message. + Work completedWork = + Optional.ofNullable(workQueue.peek()) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Active key %s without work, expected token %d", + shardedKey, workToken))); + + if (completedWork.getWorkItem().getWorkToken() != workToken) { + // Work may have been completed due to clearing of stuck commits. + LOG.warn( + "Unable to complete due to token mismatch for key {} and token {}, actual token was {}.", + shardedKey, + workToken, + completedWork.getWorkItem().getWorkToken()); + return; + } + + // We consumed the matching work item. + workQueue.remove(); + } + + private synchronized Optional getNextWork(Queue workQueue, ShardedKey shardedKey) { + Optional nextWork = Optional.ofNullable(workQueue.peek()); + if (!nextWork.isPresent()) { + Preconditions.checkState(workQueue == activeWork.remove(shardedKey)); + } + + return nextWork; + } + + /** + * Invalidates all {@link Work} that is in the {@link Work.State#COMMITTING} state which started + * before the stuckCommitDeadline. + */ + synchronized void invalidateStuckCommits( + Instant stuckCommitDeadline, BiConsumer shardedKeyAndWorkTokenConsumer) { + for (Entry shardedKeyAndWorkToken : + getStuckCommitsAt(stuckCommitDeadline).entrySet()) { + ShardedKey shardedKey = shardedKeyAndWorkToken.getKey(); + long workToken = shardedKeyAndWorkToken.getValue(); + computationStateCache.invalidate(shardedKey.key(), shardedKey.shardingKey()); + shardedKeyAndWorkTokenConsumer.accept(shardedKey, workToken); + } + } + + private synchronized ImmutableMap getStuckCommitsAt( + Instant stuckCommitDeadline) { + // Determine the stuck commit keys but complete them outside the loop iterating over + // activeWork as completeWork may delete the entry from activeWork. + ImmutableMap.Builder stuckCommits = ImmutableMap.builder(); + for (Entry> entry : activeWork.entrySet()) { + ShardedKey shardedKey = entry.getKey(); + @Nullable Work work = entry.getValue().peek(); + if (work != null) { + if (work.isStuckCommittingAt(stuckCommitDeadline)) { + LOG.error( + "Detected key {} stuck in COMMITTING state since {}, completing it with error.", + shardedKey, + work.getStateStartTime()); + stuckCommits.put(shardedKey, work.getWorkItem().getWorkToken()); + } + } + } + + return stuckCommits.build(); + } + + synchronized ImmutableList getKeysToRefresh(Instant refreshDeadline) { + return activeWork.entrySet().stream() + .flatMap(entry -> toKeyedGetDataRequestStream(entry, refreshDeadline)) + .collect(toImmutableList()); + } + + private static Stream toKeyedGetDataRequestStream( + Entry> shardedKeyAndWorkQueue, Instant refreshDeadline) { + ShardedKey shardedKey = shardedKeyAndWorkQueue.getKey(); + Deque workQueue = shardedKeyAndWorkQueue.getValue(); + + return workQueue.stream() + .filter(work -> work.getStartTime().isBefore(refreshDeadline)) + .map( + work -> + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(shardedKey.key()) + .setShardingKey(shardedKey.shardingKey()) + .setWorkToken(work.getWorkItem().getWorkToken()) + .addAllLatencyAttribution(work.getLatencyAttributions()) + .build()); + } + + synchronized void printActiveWork(PrintWriter writer, Instant now) { + writer.println( + ""); + writer.println( + ""); + // Use StringBuilder because we are appending in loop. + StringBuilder activeWorkStatus = new StringBuilder(); + int commitsPendingCount = 0; + for (Map.Entry> entry : activeWork.entrySet()) { + Queue workQueue = Preconditions.checkNotNull(entry.getValue()); + Work activeWork = Preconditions.checkNotNull(workQueue.peek()); + Windmill.WorkItem workItem = activeWork.getWorkItem(); + if (activeWork.isCommitPending()) { + if (++commitsPendingCount >= MAX_PRINTABLE_COMMIT_PENDING_KEYS) { + continue; + } + } + activeWorkStatus.append(""); + activeWorkStatus.append("\n"); + } + + writer.print(activeWorkStatus); + writer.println("
    KeyTokenQueuedActive ForStateState Active For
    "); + activeWorkStatus.append(String.format("%016x", workItem.getShardingKey())); + activeWorkStatus.append(""); + activeWorkStatus.append(String.format("%016x", workItem.getWorkToken())); + activeWorkStatus.append(""); + activeWorkStatus.append(workQueue.size() - 1); + activeWorkStatus.append(""); + activeWorkStatus.append(elapsedString(activeWork.getStartTime(), now)); + activeWorkStatus.append(""); + activeWorkStatus.append(activeWork.getState()); + activeWorkStatus.append(""); + activeWorkStatus.append(elapsedString(activeWork.getStateStartTime(), now)); + activeWorkStatus.append("
    "); + + if (commitsPendingCount >= MAX_PRINTABLE_COMMIT_PENDING_KEYS) { + writer.println("
    "); + writer.print("Skipped keys in COMMITTING/COMMIT_QUEUED: "); + writer.println(commitsPendingCount - MAX_PRINTABLE_COMMIT_PENDING_KEYS); + writer.println("
    "); + } + } + + private static String elapsedString(Instant start, Instant end) { + Duration activeFor = new Duration(start, end); + // Duration's toString always starts with "PT"; remove that here. + return activeFor.toString().substring(2); + } + + enum ActivateWorkResult { + QUEUED, + EXECUTE, + DUPLICATE + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/Commit.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/Commit.java new file mode 100644 index 0000000000000..946897967561e --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/Commit.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.streaming; + +import com.google.auto.value.AutoValue; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.WorkItemCommitRequest; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; + +/** Value class for a queued commit. */ +@AutoValue +public abstract class Commit { + + public static Commit create( + WorkItemCommitRequest request, ComputationState computationState, Work work) { + Preconditions.checkArgument(request.getSerializedSize() > 0); + return new AutoValue_Commit(request, computationState, work); + } + + public abstract WorkItemCommitRequest request(); + + public abstract ComputationState computationState(); + + public abstract Work work(); + + public final int getSize() { + return request().getSerializedSize(); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ComputationState.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ComputationState.java new file mode 100644 index 0000000000000..9d7a9131f5849 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ComputationState.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.streaming; + +import com.google.api.services.dataflow.model.MapTask; +import java.io.PrintWriter; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentLinkedQueue; +import javax.annotation.Nullable; +import org.apache.beam.runners.dataflow.worker.util.BoundedQueueExecutor; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.runners.dataflow.worker.windmill.state.WindmillStateCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.joda.time.Instant; + +/** + * Class representing the state of a computation. + * + *

    This class is synchronized, but only used from the dispatch and commit threads, so should not + * be heavily contended. Still, blocking work should not be done by it. + */ +public class ComputationState implements AutoCloseable { + private final String computationId; + private final MapTask mapTask; + private final ImmutableMap transformUserNameToStateFamily; + private final ActiveWorkState activeWorkState; + private final BoundedQueueExecutor executor; + private final ConcurrentLinkedQueue executionStateQueue; + + public ComputationState( + String computationId, + MapTask mapTask, + BoundedQueueExecutor executor, + Map transformUserNameToStateFamily, + WindmillStateCache.ForComputation computationStateCache) { + Preconditions.checkNotNull(mapTask.getStageName()); + Preconditions.checkNotNull(mapTask.getSystemName()); + this.computationId = computationId; + this.mapTask = mapTask; + this.executor = executor; + this.transformUserNameToStateFamily = ImmutableMap.copyOf(transformUserNameToStateFamily); + this.executionStateQueue = new ConcurrentLinkedQueue<>(); + this.activeWorkState = ActiveWorkState.create(computationStateCache); + } + + public String getComputationId() { + return computationId; + } + + public MapTask getMapTask() { + return mapTask; + } + + public ImmutableMap getTransformUserNameToStateFamily() { + return transformUserNameToStateFamily; + } + + public ConcurrentLinkedQueue getExecutionStateQueue() { + return executionStateQueue; + } + + /** + * Mark the given {@link ShardedKey} and {@link Work} as active, and schedules execution of {@link + * Work} if there is no active {@link Work} for the {@link ShardedKey} already processing. + */ + public boolean activateWork(ShardedKey shardedKey, Work work) { + switch (activeWorkState.activateWorkForKey(shardedKey, work)) { + case DUPLICATE: + return false; + case QUEUED: + return true; + case EXECUTE: + { + execute(work); + return true; + } + default: + // This will never happen, the switch is exhaustive. + throw new IllegalStateException("Unrecognized ActivateWorkResult"); + } + } + + /** + * Marks the work for the given shardedKey as complete. Schedules queued work for the key if any. + */ + public void completeWorkAndScheduleNextWorkForKey(ShardedKey shardedKey, long workToken) { + activeWorkState + .completeWorkAndGetNextWorkForKey(shardedKey, workToken) + .ifPresent(this::forceExecute); + } + + public void invalidateStuckCommits(Instant stuckCommitDeadline) { + activeWorkState.invalidateStuckCommits( + stuckCommitDeadline, this::completeWorkAndScheduleNextWorkForKey); + } + + private void execute(Work work) { + executor.execute(work, work.getWorkItem().getSerializedSize()); + } + + private void forceExecute(Work work) { + executor.forceExecute(work, work.getWorkItem().getSerializedSize()); + } + + /** Adds any work started before the refreshDeadline to the GetDataRequest builder. */ + public List getKeysToRefresh(Instant refreshDeadline) { + return activeWorkState.getKeysToRefresh(refreshDeadline); + } + + public void printActiveWork(PrintWriter writer) { + activeWorkState.printActiveWork(writer, Instant.now()); + } + + @Override + public void close() throws Exception { + @Nullable ExecutionState executionState; + while ((executionState = executionStateQueue.poll()) != null) { + executionState.workExecutor().close(); + } + executionStateQueue.clear(); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ExecutionState.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ExecutionState.java new file mode 100644 index 0000000000000..ba35179a75b35 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ExecutionState.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.streaming; + +import com.google.auto.value.AutoValue; +import java.util.Optional; +import org.apache.beam.runners.core.metrics.ExecutionStateTracker; +import org.apache.beam.runners.dataflow.worker.DataflowWorkExecutor; +import org.apache.beam.runners.dataflow.worker.StreamingModeExecutionContext; +import org.apache.beam.sdk.coders.Coder; + +@AutoValue +public abstract class ExecutionState { + + public abstract DataflowWorkExecutor workExecutor(); + + public abstract StreamingModeExecutionContext context(); + + public abstract Optional> keyCoder(); + + public abstract ExecutionStateTracker executionStateTracker(); + + public static ExecutionState.Builder builder() { + return new AutoValue_ExecutionState.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setWorkExecutor(DataflowWorkExecutor workExecutor); + + public abstract Builder setContext(StreamingModeExecutionContext context); + + public abstract Builder setKeyCoder(Coder keyCoder); + + public abstract Builder setExecutionStateTracker(ExecutionStateTracker executionStateTracker); + + public abstract ExecutionState build(); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/KeyCommitTooLargeException.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/KeyCommitTooLargeException.java new file mode 100644 index 0000000000000..090d9981309e5 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/KeyCommitTooLargeException.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.streaming; + +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; + +public final class KeyCommitTooLargeException extends Exception { + + public static KeyCommitTooLargeException causedBy( + String computationId, long byteLimit, Windmill.WorkItemCommitRequest request) { + StringBuilder message = new StringBuilder(); + message.append("Commit request for stage "); + message.append(computationId); + message.append(" and key "); + message.append(request.getKey().toStringUtf8()); + if (request.getSerializedSize() > 0) { + message.append( + " has size " + + request.getSerializedSize() + + " which is more than the limit of " + + byteLimit); + } else { + message.append(" is larger than 2GB and cannot be processed"); + } + message.append( + ". This may be caused by grouping a very " + + "large amount of data in a single window without using Combine," + + " or by producing a large amount of data from a single input element."); + return new KeyCommitTooLargeException(message.toString()); + } + + private KeyCommitTooLargeException(String message) { + super(message); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ShardedKey.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ShardedKey.java new file mode 100644 index 0000000000000..86433d9e6752e --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ShardedKey.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.streaming; + +import com.google.auto.value.AutoValue; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; + +@AutoValue +public abstract class ShardedKey { + + public static ShardedKey create(ByteString key, long shardingKey) { + return new AutoValue_ShardedKey(key, shardingKey); + } + + public abstract ByteString key(); + + public abstract long shardingKey(); + + @Override + public final String toString() { + return String.format("%016x", shardingKey()); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/StageInfo.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/StageInfo.java new file mode 100644 index 0000000000000..b514dfc84bb93 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/StageInfo.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.streaming; + +import static org.apache.beam.runners.dataflow.worker.DataflowSystemMetrics.THROTTLING_MSECS_METRIC_NAME; + +import com.google.api.services.dataflow.model.CounterStructuredName; +import com.google.api.services.dataflow.model.CounterUpdate; +import com.google.auto.value.AutoValue; +import java.util.ArrayList; +import java.util.List; +import org.apache.beam.runners.dataflow.worker.DataflowSystemMetrics; +import org.apache.beam.runners.dataflow.worker.MetricsContainerRegistry; +import org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker; +import org.apache.beam.runners.dataflow.worker.StreamingModeExecutionContext.StreamingModeExecutionStateRegistry; +import org.apache.beam.runners.dataflow.worker.StreamingStepMetricsContainer; +import org.apache.beam.runners.dataflow.worker.counters.Counter; +import org.apache.beam.runners.dataflow.worker.counters.CounterSet; +import org.apache.beam.runners.dataflow.worker.counters.DataflowCounterUpdateExtractor; +import org.apache.beam.runners.dataflow.worker.counters.NameContext; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; + +/** Contains a few of the stage specific fields. E.g. metrics container registry, counters etc. */ +@AutoValue +public abstract class StageInfo { + public static StageInfo create( + String stageName, String systemName, StreamingDataflowWorker worker) { + NameContext nameContext = NameContext.create(stageName, null, systemName, null); + CounterSet deltaCounters = new CounterSet(); + return new AutoValue_StageInfo( + stageName, + systemName, + StreamingStepMetricsContainer.createRegistry(), + new StreamingModeExecutionStateRegistry(worker), + deltaCounters, + deltaCounters.longSum( + DataflowSystemMetrics.StreamingPerStageSystemCounterNames.THROTTLED_MSECS.counterName( + nameContext)), + deltaCounters.longSum( + DataflowSystemMetrics.StreamingPerStageSystemCounterNames.TOTAL_PROCESSING_MSECS + .counterName(nameContext)), + deltaCounters.longSum( + DataflowSystemMetrics.StreamingPerStageSystemCounterNames.TIMER_PROCESSING_MSECS + .counterName(nameContext))); + } + + public abstract String stageName(); + + public abstract String systemName(); + + public abstract MetricsContainerRegistry + metricsContainerRegistry(); + + public abstract StreamingModeExecutionStateRegistry executionStateRegistry(); + + public abstract CounterSet deltaCounters(); + + public abstract Counter throttledMsecs(); + + public abstract Counter totalProcessingMsecs(); + + public abstract Counter timerProcessingMsecs(); + + public List extractCounterUpdates() { + List counterUpdates = new ArrayList<>(); + Iterables.addAll( + counterUpdates, + StreamingStepMetricsContainer.extractMetricUpdates(metricsContainerRegistry())); + Iterables.addAll(counterUpdates, executionStateRegistry().extractUpdates(false)); + for (CounterUpdate counterUpdate : counterUpdates) { + translateKnownStepCounters(counterUpdate); + } + counterUpdates.addAll( + deltaCounters().extractModifiedDeltaUpdates(DataflowCounterUpdateExtractor.INSTANCE)); + return counterUpdates; + } + + /** + * Checks if the step counter affects any per-stage counters. Currently 'throttled_millis' is the + * only counter updated. + */ + private void translateKnownStepCounters(CounterUpdate stepCounterUpdate) { + CounterStructuredName structuredName = + stepCounterUpdate.getStructuredNameAndMetadata().getName(); + if ((THROTTLING_MSECS_METRIC_NAME.getNamespace().equals(structuredName.getOriginNamespace()) + && THROTTLING_MSECS_METRIC_NAME.getName().equals(structuredName.getName())) + || (StreamingDataflowWorker.BIGQUERY_STREAMING_INSERT_THROTTLE_TIME + .getNamespace() + .equals(structuredName.getOriginNamespace()) + && StreamingDataflowWorker.BIGQUERY_STREAMING_INSERT_THROTTLE_TIME + .getName() + .equals(structuredName.getName()))) { + long msecs = DataflowCounterUpdateExtractor.splitIntToLong(stepCounterUpdate.getInteger()); + if (msecs > 0) { + throttledMsecs().addValue(msecs); + } + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/WeightedBoundedQueue.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/WeightedBoundedQueue.java new file mode 100644 index 0000000000000..f2893f3e71914 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/WeightedBoundedQueue.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.streaming; + +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** Bounded set of queues, with a maximum total weight. */ +public final class WeightedBoundedQueue { + + private final LinkedBlockingQueue queue; + private final int maxWeight; + private final Semaphore limit; + private final Function weigher; + + private WeightedBoundedQueue( + LinkedBlockingQueue linkedBlockingQueue, + int maxWeight, + Semaphore limit, + Function weigher) { + this.queue = linkedBlockingQueue; + this.maxWeight = maxWeight; + this.limit = limit; + this.weigher = weigher; + } + + public static WeightedBoundedQueue create(int maxWeight, Function weigherFn) { + return new WeightedBoundedQueue<>( + new LinkedBlockingQueue<>(), maxWeight, new Semaphore(maxWeight, true), weigherFn); + } + + /** + * Adds the value to the queue, blocking if this would cause the overall weight to exceed the + * limit. + */ + public void put(V value) { + limit.acquireUninterruptibly(weigher.apply(value)); + queue.add(value); + } + + /** Returns and removes the next value, or null if there is no such value. */ + public @Nullable V poll() { + V result = queue.poll(); + if (result != null) { + limit.release(weigher.apply(result)); + } + return result; + } + + /** + * Retrieves and removes the head of this queue, waiting up to the specified wait time if + * necessary for an element to become available. + * + * @param timeout how long to wait before giving up, in units of {@code unit} + * @param unit a {@code TimeUnit} determining how to interpret the {@code timeout} parameter + * @return the head of this queue, or {@code null} if the specified waiting time elapses before an + * element is available + * @throws InterruptedException if interrupted while waiting + */ + public @Nullable V poll(long timeout, TimeUnit unit) throws InterruptedException { + V result = queue.poll(timeout, unit); + if (result != null) { + limit.release(weigher.apply(result)); + } + return result; + } + + /** Returns and removes the next value, or blocks until one is available. */ + public @Nullable V take() throws InterruptedException { + V result = queue.take(); + limit.release(weigher.apply(result)); + return result; + } + + /** Returns the current weight of the queue. */ + public int queuedElementsWeight() { + return maxWeight - limit.availablePermits(); + } + + public int size() { + return queue.size(); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/Work.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/Work.java new file mode 100644 index 0000000000000..cc3f6d1b12b21 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/Work.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.streaming; + +import com.google.auto.value.AutoValue; +import java.util.ArrayList; +import java.util.Collection; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Supplier; +import javax.annotation.concurrent.NotThreadSafe; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.joda.time.Duration; +import org.joda.time.Instant; + +@NotThreadSafe +public class Work implements Runnable { + + private final Windmill.WorkItem workItem; + private final Supplier clock; + private final Instant startTime; + private final Map totalDurationPerState; + private final Consumer processWorkFn; + private TimedState currentState; + + private Work(Windmill.WorkItem workItem, Supplier clock, Consumer processWorkFn) { + this.workItem = workItem; + this.clock = clock; + this.processWorkFn = processWorkFn; + this.startTime = clock.get(); + this.totalDurationPerState = new EnumMap<>(Windmill.LatencyAttribution.State.class); + this.currentState = TimedState.initialState(startTime); + } + + public static Work create( + Windmill.WorkItem workItem, + Supplier clock, + Collection getWorkStreamLatencies, + Consumer processWorkFn) { + Work work = new Work(workItem, clock, processWorkFn); + work.recordGetWorkStreamLatencies(getWorkStreamLatencies); + return work; + } + + @Override + public void run() { + processWorkFn.accept(this); + } + + public Windmill.WorkItem getWorkItem() { + return workItem; + } + + public Instant getStartTime() { + return startTime; + } + + public State getState() { + return currentState.state(); + } + + public void setState(State state) { + Instant now = clock.get(); + totalDurationPerState.compute( + this.currentState.state().toLatencyAttributionState(), + (s, d) -> + new Duration(this.currentState.startTime(), now).plus(d == null ? Duration.ZERO : d)); + this.currentState = TimedState.create(state, now); + } + + public boolean isCommitPending() { + return currentState.isCommitPending(); + } + + public Instant getStateStartTime() { + return currentState.startTime(); + } + + private void recordGetWorkStreamLatencies( + Collection getWorkStreamLatencies) { + for (Windmill.LatencyAttribution latency : getWorkStreamLatencies) { + totalDurationPerState.put( + latency.getState(), Duration.millis(latency.getTotalDurationMillis())); + } + } + + public Collection getLatencyAttributions() { + List list = new ArrayList<>(); + for (Windmill.LatencyAttribution.State state : Windmill.LatencyAttribution.State.values()) { + Duration duration = totalDurationPerState.getOrDefault(state, Duration.ZERO); + if (state == this.currentState.state().toLatencyAttributionState()) { + duration = duration.plus(new Duration(this.currentState.startTime(), clock.get())); + } + if (duration.equals(Duration.ZERO)) { + continue; + } + list.add( + Windmill.LatencyAttribution.newBuilder() + .setState(state) + .setTotalDurationMillis(duration.getMillis()) + .build()); + } + return list; + } + + boolean isStuckCommittingAt(Instant stuckCommitDeadline) { + return currentState.state() == Work.State.COMMITTING + && currentState.startTime().isBefore(stuckCommitDeadline); + } + + public enum State { + QUEUED(Windmill.LatencyAttribution.State.QUEUED), + PROCESSING(Windmill.LatencyAttribution.State.ACTIVE), + READING(Windmill.LatencyAttribution.State.READING), + COMMIT_QUEUED(Windmill.LatencyAttribution.State.COMMITTING), + COMMITTING(Windmill.LatencyAttribution.State.COMMITTING), + GET_WORK_IN_WINDMILL_WORKER(Windmill.LatencyAttribution.State.GET_WORK_IN_WINDMILL_WORKER), + GET_WORK_IN_TRANSIT_TO_DISPATCHER( + Windmill.LatencyAttribution.State.GET_WORK_IN_TRANSIT_TO_DISPATCHER), + GET_WORK_IN_TRANSIT_TO_USER_WORKER( + Windmill.LatencyAttribution.State.GET_WORK_IN_TRANSIT_TO_USER_WORKER); + + private final Windmill.LatencyAttribution.State latencyAttributionState; + + State(Windmill.LatencyAttribution.State latencyAttributionState) { + this.latencyAttributionState = latencyAttributionState; + } + + Windmill.LatencyAttribution.State toLatencyAttributionState() { + return latencyAttributionState; + } + } + + /** + * Represents the current state of an instance of {@link Work}. Contains the {@link State} and + * {@link Instant} when it started. + */ + @AutoValue + abstract static class TimedState { + private static TimedState create(State state, Instant startTime) { + return new AutoValue_Work_TimedState(state, startTime); + } + + private static TimedState initialState(Instant startTime) { + return create(State.QUEUED, startTime); + } + + private boolean isCommitPending() { + return state() == Work.State.COMMITTING || state() == Work.State.COMMIT_QUEUED; + } + + abstract State state(); + + abstract Instant startTime(); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BatchGroupAlsoByWindowAndCombineFn.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BatchGroupAlsoByWindowAndCombineFn.java index 12fd94e00f078..c3f1af8d6637b 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BatchGroupAlsoByWindowAndCombineFn.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BatchGroupAlsoByWindowAndCombineFn.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Arrays; import java.util.Collection; @@ -40,8 +40,8 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.joda.time.Instant; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BatchGroupAlsoByWindowViaIteratorsFn.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BatchGroupAlsoByWindowViaIteratorsFn.java index 9c16882f7c02d..5e9e79a40bc37 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BatchGroupAlsoByWindowViaIteratorsFn.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BatchGroupAlsoByWindowViaIteratorsFn.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.ArrayList; import java.util.Arrays; @@ -39,10 +39,10 @@ import org.apache.beam.sdk.util.common.Reiterator; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ListMultimap; import org.joda.time.Instant; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BatchGroupAlsoByWindowsDoFns.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BatchGroupAlsoByWindowsDoFns.java index d7b2dca69e239..47d06b8817415 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BatchGroupAlsoByWindowsDoFns.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BatchGroupAlsoByWindowsDoFns.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import org.apache.beam.runners.core.StateInternalsFactory; import org.apache.beam.runners.core.SystemReduceFn; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BoundedQueueExecutor.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BoundedQueueExecutor.java index 1784bbf8e3bae..a160b0e6ad036 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BoundedQueueExecutor.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/BoundedQueueExecutor.java @@ -21,8 +21,9 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Monitor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Monitor.Guard; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Monitor; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Monitor.Guard; /** An executor for executing work on windmill items. */ @SuppressWarnings({ @@ -36,6 +37,9 @@ public class BoundedQueueExecutor { private final Monitor monitor = new Monitor(); private int elementsOutstanding = 0; private long bytesOutstanding = 0; + private final AtomicInteger activeCount = new AtomicInteger(); + private long startTimeMaxActiveThreadsUsed; + private long totalTimeMaxActiveThreadsUsed; public BoundedQueueExecutor( int maximumPoolSize, @@ -51,7 +55,29 @@ public BoundedQueueExecutor( keepAliveTime, unit, new LinkedBlockingQueue<>(), - threadFactory); + threadFactory) { + @Override + protected void beforeExecute(Thread t, Runnable r) { + super.beforeExecute(t, r); + synchronized (this) { + if (activeCount.getAndIncrement() >= maximumPoolSize - 1) { + startTimeMaxActiveThreadsUsed = System.currentTimeMillis(); + } + } + } + + @Override + protected void afterExecute(Runnable r, Throwable t) { + super.afterExecute(r, t); + synchronized (this) { + if (activeCount.getAndDecrement() == maximumPoolSize) { + totalTimeMaxActiveThreadsUsed += + (System.currentTimeMillis() - startTimeMaxActiveThreadsUsed); + startTimeMaxActiveThreadsUsed = 0; + } + } + } + }; executor.allowCoreThreadTimeOut(true); this.maximumElementsOutstanding = maximumElementsOutstanding; this.maximumBytesOutstanding = maximumBytesOutstanding; @@ -89,6 +115,14 @@ public boolean executorQueueIsEmpty() { return executor.getQueue().isEmpty(); } + public long allThreadsActiveTime() { + return totalTimeMaxActiveThreadsUsed; + } + + public int activeCount() { + return activeCount.intValue(); + } + public String summaryHtml() { monitor.enter(); try { diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/JfrInterop.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/JfrInterop.java index 116e7a235d467..936b310071f3f 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/JfrInterop.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/JfrInterop.java @@ -25,7 +25,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; /** * Exposes methods to interop with JFR. This is only supported on java 9 and up, java 8 does not diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/MemoryMonitor.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/MemoryMonitor.java index eac36d15abfc2..84b391055df94 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/MemoryMonitor.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/MemoryMonitor.java @@ -56,12 +56,13 @@ import org.apache.beam.sdk.io.fs.CreateOptions.StandardCreateOptions; import org.apache.beam.sdk.io.fs.ResourceId; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Closeables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.AtomicDouble; +import org.apache.beam.sdk.options.SdkHarnessOptions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Closeables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.AtomicDouble; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -237,10 +238,11 @@ public static MemoryMonitor fromOptions(PipelineOptions options) { DataflowPipelineDebugOptions debugOptions = options.as(DataflowPipelineDebugOptions.class); DataflowWorkerHarnessOptions workerHarnessOptions = options.as(DataflowWorkerHarnessOptions.class); + SdkHarnessOptions sdkHarnessOptions = options.as(SdkHarnessOptions.class); String uploadToGCSPath = debugOptions.getSaveHeapDumpsToGcsPath(); String workerId = workerHarnessOptions.getWorkerId(); boolean canDumpHeap = uploadToGCSPath != null || debugOptions.getDumpHeapOnOOM(); - double gcThrashingPercentagePerPeriod = debugOptions.getGCThrashingPercentagePerPeriod(); + double gcThrashingPercentagePerPeriod = sdkHarnessOptions.getGCThrashingPercentagePerPeriod(); Duration jfrProfileDuration; if (uploadToGCSPath != null && debugOptions.getRecordJfrOnGcThrashing()) { diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/ScalableBloomFilter.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/ScalableBloomFilter.java index 711fc729354fc..784ff785c6583 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/ScalableBloomFilter.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/ScalableBloomFilter.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.io.InputStream; @@ -31,12 +31,12 @@ import org.apache.beam.sdk.coders.AtomicCoder; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.CoderException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.BloomFilter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Funnel; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.PrimitiveSink; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.math.DoubleMath; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.math.LongMath; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.BloomFilter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Funnel; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.PrimitiveSink; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.math.DoubleMath; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.math.LongMath; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/TimerOrElement.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/TimerOrElement.java index 7108610702301..f54957ff07009 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/TimerOrElement.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/TimerOrElement.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -34,7 +34,7 @@ import org.apache.beam.runners.dataflow.util.Structs; import org.apache.beam.runners.dataflow.worker.WindmillKeyedWorkItem.FakeKeyedWorkItemCoder; import org.apache.beam.sdk.coders.Coder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * Empty class which exists because the back end will sometimes insert uses of {@code diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/ValueInEmptyWindows.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/ValueInEmptyWindows.java index a1f2fa71e5999..5084af0d187e1 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/ValueInEmptyWindows.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/ValueInEmptyWindows.java @@ -23,7 +23,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/ForwardingReiterator.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/ForwardingReiterator.java index 2ffa8491cdd9d..ffbac3df6f1bf 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/ForwardingReiterator.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/ForwardingReiterator.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker.util.common; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import org.apache.beam.sdk.util.common.Reiterator; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/BatchingShuffleEntryReader.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/BatchingShuffleEntryReader.java index 85982fb0ce063..31dea074aa602 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/BatchingShuffleEntryReader.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/BatchingShuffleEntryReader.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.dataflow.worker.util.common.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.ListIterator; import java.util.NoSuchElementException; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ByteArrayShufflePosition.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ByteArrayShufflePosition.java index 8be53f0fcb085..d74c2fb1b8b7a 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ByteArrayShufflePosition.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ByteArrayShufflePosition.java @@ -22,7 +22,7 @@ import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.UnsafeByteOperations; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/CachingShuffleBatchReader.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/CachingShuffleBatchReader.java index 6981c6dac1200..7dd637b093f6f 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/CachingShuffleBatchReader.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/CachingShuffleBatchReader.java @@ -20,12 +20,12 @@ import java.io.IOException; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LoadingCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache; import org.checkerframework.checker.nullness.qual.Nullable; /** A {@link ShuffleBatchReader} that caches batches as they're read. */ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/FlattenOperation.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/FlattenOperation.java index fd839e8aa776d..4847e9f2ea9c5 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/FlattenOperation.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/FlattenOperation.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.dataflow.worker.util.common.worker; import java.io.Closeable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** A flatten operation. */ public class FlattenOperation extends ReceivingOperation { diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingShuffleEntryIterator.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingShuffleEntryIterator.java index 513f6b735b1a6..c7ed7e88e8256 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingShuffleEntryIterator.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingShuffleEntryIterator.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.dataflow.worker.util.common.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.NoSuchElementException; import org.apache.beam.sdk.util.common.ElementByteSizeObservableIterable; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingShuffleRangeTracker.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingShuffleRangeTracker.java index 00e0ad25a4c92..c9a767c42f808 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingShuffleRangeTracker.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingShuffleRangeTracker.java @@ -17,11 +17,11 @@ */ package org.apache.beam.runners.dataflow.worker.util.common.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.beam.sdk.io.range.RangeTracker; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingTables.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingTables.java index f883f2f3e75f4..40fd498a1da20 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingTables.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingTables.java @@ -23,7 +23,7 @@ import java.util.List; import java.util.Map; import java.util.Random; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** Static utility methods that provide {@link GroupingTable} implementations. */ @SuppressWarnings({ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/MapTaskExecutor.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/MapTaskExecutor.java index 16133e9201e85..e364b00395743 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/MapTaskExecutor.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/MapTaskExecutor.java @@ -22,8 +22,8 @@ import java.util.ListIterator; import org.apache.beam.runners.core.metrics.ExecutionStateTracker; import org.apache.beam.runners.dataflow.worker.counters.CounterSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ParDoOperation.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ParDoOperation.java index 4b6b2b3bb94fd..27b6e9d1fb350 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ParDoOperation.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ParDoOperation.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.dataflow.worker.util.common.worker; import java.io.Closeable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** A ParDo mapping function. */ public class ParDoOperation extends ReceivingOperation { diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ProgressTrackingReiterator.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ProgressTrackingReiterator.java index 3130d24abda7c..0b0e09d0cd3e7 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ProgressTrackingReiterator.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ProgressTrackingReiterator.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker.util.common.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import org.apache.beam.runners.dataflow.worker.util.common.ForwardingReiterator; import org.apache.beam.sdk.util.common.Reiterator; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ReadOperation.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ReadOperation.java index 5a5348a771254..1ee8f2bc843ea 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ReadOperation.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ReadOperation.java @@ -30,9 +30,9 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.beam.runners.dataflow.worker.counters.Counter; import org.apache.beam.runners.dataflow.worker.counters.CounterName; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ShuffleReadCounter.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ShuffleReadCounter.java index 47e7d59454334..58cfae8eeb66e 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ShuffleReadCounter.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ShuffleReadCounter.java @@ -23,7 +23,7 @@ import org.apache.beam.runners.dataflow.worker.counters.Counter; import org.apache.beam.runners.dataflow.worker.counters.CounterName; import org.apache.beam.runners.dataflow.worker.counters.CounterSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** Counts the Bytes and MSECS spent within a shuffle read. */ @SuppressWarnings({ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/WorkExecutor.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/WorkExecutor.java index e607144615bcf..2f836dfea3b81 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/WorkExecutor.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/WorkExecutor.java @@ -19,7 +19,7 @@ import java.util.List; import org.apache.beam.runners.dataflow.worker.counters.CounterSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; /** Abstract executor for WorkItem tasks. */ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/WorkProgressUpdater.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/WorkProgressUpdater.java index 7b450f5e2ac86..8ed15373ffacd 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/WorkProgressUpdater.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/WorkProgressUpdater.java @@ -24,8 +24,8 @@ import java.util.concurrent.TimeUnit; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.NotThreadSafe; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/WriteOperation.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/WriteOperation.java index e7dc431b28887..673140d58d894 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/WriteOperation.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/WriteOperation.java @@ -20,7 +20,7 @@ import java.io.Closeable; import org.apache.beam.runners.dataflow.worker.counters.Counter; import org.apache.beam.runners.dataflow.worker.counters.CounterName; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; /** A write operation. */ @SuppressWarnings({ diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/AbstractWindmillStream.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/AbstractWindmillStream.java new file mode 100644 index 0000000000000..ea7efff7a06d9 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/AbstractWindmillStream.java @@ -0,0 +1,332 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.function.Supplier; +import org.apache.beam.sdk.util.BackOff; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Status; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.StatusRuntimeException; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Instant; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Base class for persistent streams connecting to Windmill. + * + *

    This class handles the underlying gRPC StreamObservers, and automatically reconnects the + * stream if it is broken. Subclasses are responsible for retrying requests that have been lost on a + * broken stream. + * + *

    Subclasses should override onResponse to handle responses from the server, and onNewStream to + * perform any work that must be done when a new stream is created, such as sending headers or + * retrying requests. + * + *

    send and startStream should not be called from onResponse; use executor() instead. + * + *

    Synchronization on this is used to synchronize the gRpc stream state and internal data + * structures. Since grpc channel operations may block, synchronization on this stream may also + * block. This is generally not a problem since streams are used in a single-threaded manner. + * However, some accessors used for status page and other debugging need to take care not to require + * synchronizing on this. + */ +public abstract class AbstractWindmillStream implements WindmillStream { + public static final long DEFAULT_STREAM_RPC_DEADLINE_SECONDS = 300; + // Default gRPC streams to 2MB chunks, which has shown to be a large enough chunk size to reduce + // per-chunk overhead, and small enough that we can still perform granular flow-control. + protected static final int RPC_STREAM_CHUNK_SIZE = 2 << 20; + private static final Logger LOG = LoggerFactory.getLogger(AbstractWindmillStream.class); + protected final AtomicBoolean clientClosed; + private final AtomicLong lastSendTimeMs; + private final Executor executor; + private final BackOff backoff; + private final AtomicLong startTimeMs; + private final AtomicLong lastResponseTimeMs; + private final AtomicInteger errorCount; + private final AtomicReference lastError; + private final AtomicLong sleepUntil; + private final CountDownLatch finishLatch; + private final Set> streamRegistry; + private final int logEveryNStreamFailures; + private final Supplier> requestObserverSupplier; + // Indicates if the current stream in requestObserver is closed by calling close() method + private final AtomicBoolean streamClosed; + private @Nullable StreamObserver requestObserver; + + protected AbstractWindmillStream( + Function, StreamObserver> clientFactory, + BackOff backoff, + StreamObserverFactory streamObserverFactory, + Set> streamRegistry, + int logEveryNStreamFailures) { + this.executor = + Executors.newSingleThreadExecutor( + new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat("WindmillStream-thread") + .build()); + this.backoff = backoff; + this.streamRegistry = streamRegistry; + this.logEveryNStreamFailures = logEveryNStreamFailures; + this.clientClosed = new AtomicBoolean(); + this.streamClosed = new AtomicBoolean(); + this.startTimeMs = new AtomicLong(); + this.lastSendTimeMs = new AtomicLong(); + this.lastResponseTimeMs = new AtomicLong(); + this.errorCount = new AtomicInteger(); + this.lastError = new AtomicReference<>(); + this.sleepUntil = new AtomicLong(); + this.finishLatch = new CountDownLatch(1); + this.requestObserverSupplier = + () -> + streamObserverFactory.from( + clientFactory, new AbstractWindmillStream.ResponseObserver()); + } + + private static long debugDuration(long nowMs, long startMs) { + if (startMs <= 0) { + return -1; + } + return Math.max(0, nowMs - startMs); + } + + /** Called on each response from the server. */ + protected abstract void onResponse(ResponseT response); + + /** Called when a new underlying stream to the server has been opened. */ + protected abstract void onNewStream(); + + /** Returns whether there are any pending requests that should be retried on a stream break. */ + protected abstract boolean hasPendingRequests(); + + /** + * Called when the client side stream is throttled due to resource exhausted errors. Will be + * called for each resource exhausted error not just the first. onResponse() must stop throttling + * on receipt of the first good message. + */ + protected abstract void startThrottleTimer(); + + private StreamObserver requestObserver() { + if (requestObserver == null) { + throw new NullPointerException( + "requestObserver cannot be null. Missing a call to startStream() to initialize."); + } + + return requestObserver; + } + + /** Send a request to the server. */ + protected final void send(RequestT request) { + lastSendTimeMs.set(Instant.now().getMillis()); + synchronized (this) { + if (streamClosed.get()) { + throw new IllegalStateException("Send called on a client closed stream."); + } + + requestObserver().onNext(request); + } + } + + /** Starts the underlying stream. */ + protected final void startStream() { + // Add the stream to the registry after it has been fully constructed. + streamRegistry.add(this); + while (true) { + try { + synchronized (this) { + startTimeMs.set(Instant.now().getMillis()); + lastResponseTimeMs.set(0); + streamClosed.set(false); + // lazily initialize the requestObserver. Gets reset whenever the stream is reopened. + requestObserver = requestObserverSupplier.get(); + onNewStream(); + if (clientClosed.get()) { + close(); + } + return; + } + } catch (Exception e) { + LOG.error("Failed to create new stream, retrying: ", e); + try { + long sleep = backoff.nextBackOffMillis(); + sleepUntil.set(Instant.now().getMillis() + sleep); + Thread.sleep(sleep); + } catch (InterruptedException | IOException i) { + // Keep trying to create the stream. + } + } + } + } + + protected final Executor executor() { + return executor; + } + + public final synchronized void maybeSendHealthCheck(Instant lastSendThreshold) { + if (lastSendTimeMs.get() < lastSendThreshold.getMillis() && !clientClosed.get()) { + try { + sendHealthCheck(); + } catch (RuntimeException e) { + LOG.debug("Received exception sending health check.", e); + } + } + } + + protected abstract void sendHealthCheck(); + + // Care is taken that synchronization on this is unnecessary for all status page information. + // Blocking sends are made beneath this stream object's lock which could block status page + // rendering. + public final void appendSummaryHtml(PrintWriter writer) { + appendSpecificHtml(writer); + if (errorCount.get() > 0) { + writer.format(", %d errors, last error [ %s ]", errorCount.get(), lastError.get()); + } + if (clientClosed.get()) { + writer.write(", client closed"); + } + long nowMs = Instant.now().getMillis(); + long sleepLeft = sleepUntil.get() - nowMs; + if (sleepLeft > 0) { + writer.format(", %dms backoff remaining", sleepLeft); + } + writer.format( + ", current stream is %dms old, last send %dms, last response %dms, closed: %s", + debugDuration(nowMs, startTimeMs.get()), + debugDuration(nowMs, lastSendTimeMs.get()), + debugDuration(nowMs, lastResponseTimeMs.get()), + streamClosed.get()); + } + + // Don't require synchronization on stream, see the appendSummaryHtml comment. + protected abstract void appendSpecificHtml(PrintWriter writer); + + @Override + public final synchronized void close() { + // Synchronization of close and onCompleted necessary for correct retry logic in onNewStream. + clientClosed.set(true); + requestObserver().onCompleted(); + streamClosed.set(true); + } + + @Override + public final boolean awaitTermination(int time, TimeUnit unit) throws InterruptedException { + return finishLatch.await(time, unit); + } + + @Override + public final Instant startTime() { + return new Instant(startTimeMs.get()); + } + + private class ResponseObserver implements StreamObserver { + @Override + public void onNext(ResponseT response) { + try { + backoff.reset(); + } catch (IOException e) { + // Ignore. + } + lastResponseTimeMs.set(Instant.now().getMillis()); + onResponse(response); + } + + @Override + public void onError(Throwable t) { + onStreamFinished(t); + } + + @Override + public void onCompleted() { + onStreamFinished(null); + } + + private void onStreamFinished(@Nullable Throwable t) { + synchronized (this) { + if (clientClosed.get() && !hasPendingRequests()) { + streamRegistry.remove(AbstractWindmillStream.this); + finishLatch.countDown(); + return; + } + } + if (t != null) { + Status status = null; + if (t instanceof StatusRuntimeException) { + status = ((StatusRuntimeException) t).getStatus(); + } + String statusError = status == null ? "" : status.toString(); + lastError.set(statusError); + if (errorCount.getAndIncrement() % logEveryNStreamFailures == 0) { + long nowMillis = Instant.now().getMillis(); + String responseDebug; + if (lastResponseTimeMs.get() == 0) { + responseDebug = "never received response"; + } else { + responseDebug = + "received response " + (nowMillis - lastResponseTimeMs.get()) + "ms ago"; + } + LOG.debug( + "{} streaming Windmill RPC errors for {}, last was: {} with status {}." + + " created {}ms ago, {}. This is normal with autoscaling.", + AbstractWindmillStream.this.getClass(), + errorCount.get(), + t, + statusError, + nowMillis - startTimeMs.get(), + responseDebug); + } + // If the stream was stopped due to a resource exhausted error then we are throttled. + if (status != null && status.getCode() == Status.Code.RESOURCE_EXHAUSTED) { + startThrottleTimer(); + } + + try { + long sleep = backoff.nextBackOffMillis(); + sleepUntil.set(Instant.now().getMillis() + sleep); + Thread.sleep(sleep); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (IOException e) { + // Ignore. + } + } else { + errorCount.incrementAndGet(); + String error = + "Stream completed successfully but did not complete requested operations, " + + "recreating"; + LOG.warn(error); + lastError.set(error); + } + executor.execute(AbstractWindmillStream.this::startStream); + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/ForwardingClientResponseObserver.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/ForwardingClientResponseObserver.java index 3737e29efb133..a1f80598d89a8 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/ForwardingClientResponseObserver.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/ForwardingClientResponseObserver.java @@ -27,23 +27,23 @@ *

    Used to wrap existing {@link StreamObserver}s to be able to install an {@link * ClientCallStreamObserver#setOnReadyHandler(Runnable) onReadyHandler}. * - *

    This is as thread-safe as the undering stream observer that is being wrapped. + *

    This is as thread-safe as the underlying stream observer that is being wrapped. */ -final class ForwardingClientResponseObserver - implements ClientResponseObserver { +final class ForwardingClientResponseObserver + implements ClientResponseObserver { private final Runnable onReadyHandler; private final Runnable onDoneHandler; - private final StreamObserver inboundObserver; + private final StreamObserver inboundObserver; ForwardingClientResponseObserver( - StreamObserver inboundObserver, Runnable onReadyHandler, Runnable onDoneHandler) { + StreamObserver inboundObserver, Runnable onReadyHandler, Runnable onDoneHandler) { this.inboundObserver = inboundObserver; this.onReadyHandler = onReadyHandler; this.onDoneHandler = onDoneHandler; } @Override - public void onNext(ReqT value) { + public void onNext(ResponseT value) { inboundObserver.onNext(value); } @@ -60,7 +60,7 @@ public void onCompleted() { } @Override - public void beforeStart(ClientCallStreamObserver stream) { + public void beforeStart(ClientCallStreamObserver stream) { stream.setOnReadyHandler(onReadyHandler); } } diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/GrpcWindmillServer.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/GrpcWindmillServer.java deleted file mode 100644 index 7fe1a7b544079..0000000000000 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/GrpcWindmillServer.java +++ /dev/null @@ -1,1685 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.runners.dataflow.worker.windmill; - -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintWriter; -import java.io.SequenceInputStream; -import java.net.URI; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Random; -import java.util.Set; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.BlockingDeque; -import java.util.concurrent.CancellationException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingDeque; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.Supplier; -import org.apache.beam.runners.dataflow.worker.WindmillTimeUtils; -import org.apache.beam.runners.dataflow.worker.options.StreamingDataflowWorkerOptions; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.CommitStatus; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.CommitWorkRequest; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.CommitWorkResponse; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.ComputationGetDataRequest; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetConfigRequest; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetConfigResponse; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetDataRequest; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetDataResponse; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetWorkRequest; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetWorkResponse; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GlobalData; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GlobalDataRequest; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.JobHeader; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataRequest; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataResponse; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.ReportStatsRequest; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.ReportStatsResponse; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingCommitRequestChunk; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingCommitResponse; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingCommitWorkRequest; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingGetDataRequest; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingGetDataResponse; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingGetWorkRequest; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingGetWorkRequestExtension; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingGetWorkResponseChunk; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.WorkItemCommitRequest; -import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.sdk.util.BackOff; -import org.apache.beam.sdk.util.BackOffUtils; -import org.apache.beam.sdk.util.FluentBackoff; -import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.CallCredentials; -import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Channel; -import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Status; -import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.StatusRuntimeException; -import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.auth.MoreCallCredentials; -import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessChannelBuilder; -import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.netty.GrpcSslContexts; -import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.netty.NegotiationType; -import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.netty.NettyChannelBuilder; -import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Verify; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.net.HostAndPort; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.checkerframework.checker.nullness.qual.Nullable; -import org.joda.time.Duration; -import org.joda.time.Instant; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** gRPC client for communicating with Windmill Service. */ -// Very likely real potential for bugs - https://github.com/apache/beam/issues/19273 -// Very likely real potential for bugs - https://github.com/apache/beam/issues/19271 -@SuppressFBWarnings({"JLM_JSR166_UTILCONCURRENT_MONITORENTER", "IS2_INCONSISTENT_SYNC"}) -@SuppressWarnings({ - "nullness" // TODO(https://github.com/apache/beam/issues/20497) -}) -public class GrpcWindmillServer extends WindmillServerStub { - private static final Logger LOG = LoggerFactory.getLogger(GrpcWindmillServer.class); - - // If a connection cannot be established, gRPC will fail fast so this deadline can be relatively - // high. - private static final long DEFAULT_UNARY_RPC_DEADLINE_SECONDS = 300; - private static final long DEFAULT_STREAM_RPC_DEADLINE_SECONDS = 300; - - private static final Duration MIN_BACKOFF = Duration.millis(1); - private static final Duration MAX_BACKOFF = Duration.standardSeconds(30); - // Default gRPC streams to 2MB chunks, which has shown to be a large enough chunk size to reduce - // per-chunk overhead, and small enough that we can still granularly flow-control. - private static final int COMMIT_STREAM_CHUNK_SIZE = 2 << 20; - private static final int GET_DATA_STREAM_CHUNK_SIZE = 2 << 20; - - private static final long HEARTBEAT_REQUEST_ID = Long.MAX_VALUE; - private static final AtomicLong nextId = new AtomicLong(0); - - private final StreamingDataflowWorkerOptions options; - private final int streamingRpcBatchLimit; - private final List stubList = - new ArrayList<>(); - private final List - syncStubList = new ArrayList<>(); - private WindmillApplianceGrpc.WindmillApplianceBlockingStub syncApplianceStub = null; - private long unaryDeadlineSeconds = DEFAULT_UNARY_RPC_DEADLINE_SECONDS; - private long streamDeadlineSeconds = DEFAULT_STREAM_RPC_DEADLINE_SECONDS; - private ImmutableSet endpoints; - private int logEveryNStreamFailures = 20; - private Duration maxBackoff = MAX_BACKOFF; - private final ThrottleTimer getWorkThrottleTimer = new ThrottleTimer(); - private final ThrottleTimer getDataThrottleTimer = new ThrottleTimer(); - private final ThrottleTimer commitWorkThrottleTimer = new ThrottleTimer(); - private final Random rand = new Random(); - - private final Set> streamRegistry = - Collections.newSetFromMap(new ConcurrentHashMap, Boolean>()); - - private final Timer healthCheckTimer; - - public GrpcWindmillServer(StreamingDataflowWorkerOptions options) throws IOException { - this.options = options; - this.streamingRpcBatchLimit = options.getWindmillServiceStreamingRpcBatchLimit(); - this.logEveryNStreamFailures = options.getWindmillServiceStreamingLogEveryNStreamFailures(); - this.endpoints = ImmutableSet.of(); - if (options.getWindmillServiceEndpoint() != null) { - Set endpoints = new HashSet<>(); - for (String endpoint : Splitter.on(',').split(options.getWindmillServiceEndpoint())) { - endpoints.add( - HostAndPort.fromString(endpoint).withDefaultPort(options.getWindmillServicePort())); - } - initializeWindmillService(endpoints); - } else if (!streamingEngineEnabled() && options.getLocalWindmillHostport() != null) { - int portStart = options.getLocalWindmillHostport().lastIndexOf(':'); - String endpoint = options.getLocalWindmillHostport().substring(0, portStart); - assert ("grpc:localhost".equals(endpoint)); - int port = Integer.parseInt(options.getLocalWindmillHostport().substring(portStart + 1)); - this.endpoints = ImmutableSet.of(HostAndPort.fromParts("localhost", port)); - initializeLocalHost(port); - } - if (options.getWindmillServiceStreamingRpcHealthCheckPeriodMs() > 0) { - this.healthCheckTimer = new Timer("WindmillHealthCheckTimer"); - this.healthCheckTimer.schedule( - new TimerTask() { - @Override - public void run() { - Instant reportThreshold = - Instant.now() - .minus( - Duration.millis( - options.getWindmillServiceStreamingRpcHealthCheckPeriodMs())); - for (AbstractWindmillStream stream : streamRegistry) { - stream.maybeSendHealthCheck(reportThreshold); - } - } - }, - 0, - options.getWindmillServiceStreamingRpcHealthCheckPeriodMs()); - } else { - this.healthCheckTimer = null; - } - } - - private GrpcWindmillServer(String name, boolean enableStreamingEngine) { - this.options = PipelineOptionsFactory.create().as(StreamingDataflowWorkerOptions.class); - this.streamingRpcBatchLimit = Integer.MAX_VALUE; - options.setProject("project"); - options.setJobId("job"); - options.setWorkerId("worker"); - if (enableStreamingEngine) { - List experiments = this.options.getExperiments(); - if (experiments == null) { - experiments = new ArrayList<>(); - } - experiments.add(GcpOptions.STREAMING_ENGINE_EXPERIMENT); - options.setExperiments(experiments); - } - this.stubList.add(CloudWindmillServiceV1Alpha1Grpc.newStub(inProcessChannel(name))); - this.healthCheckTimer = null; - } - - private boolean streamingEngineEnabled() { - return options.isEnableStreamingEngine(); - } - - @Override - public synchronized void setWindmillServiceEndpoints(Set endpoints) - throws IOException { - Preconditions.checkNotNull(endpoints); - if (endpoints.equals(this.endpoints)) { - // The endpoints are equal don't recreate the stubs. - return; - } - LOG.info("Creating a new windmill stub, endpoints: {}", endpoints); - if (this.endpoints != null) { - LOG.info("Previous windmill stub endpoints: {}", this.endpoints); - } - initializeWindmillService(endpoints); - } - - @Override - public synchronized boolean isReady() { - return !stubList.isEmpty(); - } - - private synchronized void initializeLocalHost(int port) throws IOException { - this.logEveryNStreamFailures = 1; - this.maxBackoff = Duration.millis(500); - this.unaryDeadlineSeconds = 10; // For local testing use short deadlines. - Channel channel = localhostChannel(port); - if (streamingEngineEnabled()) { - this.stubList.add(CloudWindmillServiceV1Alpha1Grpc.newStub(channel)); - this.syncStubList.add(CloudWindmillServiceV1Alpha1Grpc.newBlockingStub(channel)); - } else { - this.syncApplianceStub = WindmillApplianceGrpc.newBlockingStub(channel); - } - } - - /** - * Create a wrapper around credentials callback that delegates to the underlying vendored {@link - * com.google.auth.RequestMetadataCallback}. Note that this class should override every method - * that is not final and not static and call the delegate directly. - * - *

    TODO: Replace this with an auto generated proxy which calls the underlying implementation - * delegate to reduce maintenance burden. - */ - private static class VendoredRequestMetadataCallbackAdapter - implements com.google.auth.RequestMetadataCallback { - private final org.apache.beam.vendor.grpc.v1p54p0.com.google.auth.RequestMetadataCallback - callback; - - private VendoredRequestMetadataCallbackAdapter( - org.apache.beam.vendor.grpc.v1p54p0.com.google.auth.RequestMetadataCallback callback) { - this.callback = callback; - } - - @Override - public void onSuccess(Map> metadata) { - callback.onSuccess(metadata); - } - - @Override - public void onFailure(Throwable exception) { - callback.onFailure(exception); - } - } - - /** - * Create a wrapper around credentials that delegates to the underlying {@link - * com.google.auth.Credentials}. Note that this class should override every method that is not - * final and not static and call the delegate directly. - * - *

    TODO: Replace this with an auto generated proxy which calls the underlying implementation - * delegate to reduce maintenance burden. - */ - private static class VendoredCredentialsAdapter - extends org.apache.beam.vendor.grpc.v1p54p0.com.google.auth.Credentials { - private final com.google.auth.Credentials credentials; - - private VendoredCredentialsAdapter(com.google.auth.Credentials credentials) { - this.credentials = credentials; - } - - @Override - public String getAuthenticationType() { - return credentials.getAuthenticationType(); - } - - @Override - public Map> getRequestMetadata() throws IOException { - return credentials.getRequestMetadata(); - } - - @Override - public void getRequestMetadata( - final URI uri, - Executor executor, - final org.apache.beam.vendor.grpc.v1p54p0.com.google.auth.RequestMetadataCallback - callback) { - credentials.getRequestMetadata( - uri, executor, new VendoredRequestMetadataCallbackAdapter(callback)); - } - - @Override - public Map> getRequestMetadata(URI uri) throws IOException { - return credentials.getRequestMetadata(uri); - } - - @Override - public boolean hasRequestMetadata() { - return credentials.hasRequestMetadata(); - } - - @Override - public boolean hasRequestMetadataOnly() { - return credentials.hasRequestMetadataOnly(); - } - - @Override - public void refresh() throws IOException { - credentials.refresh(); - } - } - - private synchronized void initializeWindmillService(Set endpoints) - throws IOException { - LOG.info("Initializing Streaming Engine GRPC client for endpoints: {}", endpoints); - this.stubList.clear(); - this.syncStubList.clear(); - this.endpoints = ImmutableSet.copyOf(endpoints); - for (HostAndPort endpoint : this.endpoints) { - if ("localhost".equals(endpoint.getHost())) { - initializeLocalHost(endpoint.getPort()); - } else { - CallCredentials creds = - MoreCallCredentials.from(new VendoredCredentialsAdapter(options.getGcpCredential())); - this.stubList.add( - CloudWindmillServiceV1Alpha1Grpc.newStub(remoteChannel(endpoint)) - .withCallCredentials(creds)); - this.syncStubList.add( - CloudWindmillServiceV1Alpha1Grpc.newBlockingStub(remoteChannel(endpoint)) - .withCallCredentials(creds)); - } - } - } - - @VisibleForTesting - static GrpcWindmillServer newTestInstance(String name, boolean enableStreamingEngine) { - return new GrpcWindmillServer(name, enableStreamingEngine); - } - - private Channel inProcessChannel(String name) { - return InProcessChannelBuilder.forName(name).directExecutor().build(); - } - - private Channel localhostChannel(int port) { - return NettyChannelBuilder.forAddress("localhost", port) - .maxInboundMessageSize(java.lang.Integer.MAX_VALUE) - .negotiationType(NegotiationType.PLAINTEXT) - .build(); - } - - private Channel remoteChannel(HostAndPort endpoint) throws IOException { - NettyChannelBuilder builder = - NettyChannelBuilder.forAddress(endpoint.getHost(), endpoint.getPort()); - int timeoutSec = options.getWindmillServiceRpcChannelAliveTimeoutSec(); - if (timeoutSec > 0) { - builder = - builder - .keepAliveTime(timeoutSec, TimeUnit.SECONDS) - .keepAliveTimeout(timeoutSec, TimeUnit.SECONDS) - .keepAliveWithoutCalls(true); - } - return builder - .flowControlWindow(10 * 1024 * 1024) - .maxInboundMessageSize(java.lang.Integer.MAX_VALUE) - .maxInboundMetadataSize(1024 * 1024) - .negotiationType(NegotiationType.TLS) - // Set ciphers(null) to not use GCM, which is disabled for Dataflow - // due to it being horribly slow. - .sslContext(GrpcSslContexts.forClient().ciphers(null).build()) - .build(); - } - - private synchronized CloudWindmillServiceV1Alpha1Grpc.CloudWindmillServiceV1Alpha1Stub stub() { - if (stubList.isEmpty()) { - throw new RuntimeException("windmillServiceEndpoint has not been set"); - } - if (stubList.size() == 1) { - return stubList.get(0); - } - return stubList.get(rand.nextInt(stubList.size())); - } - - private synchronized CloudWindmillServiceV1Alpha1Grpc.CloudWindmillServiceV1Alpha1BlockingStub - syncStub() { - if (syncStubList.isEmpty()) { - throw new RuntimeException("windmillServiceEndpoint has not been set"); - } - if (syncStubList.size() == 1) { - return syncStubList.get(0); - } - return syncStubList.get(rand.nextInt(syncStubList.size())); - } - - @Override - public void appendSummaryHtml(PrintWriter writer) { - writer.write("Active Streams:
    "); - for (AbstractWindmillStream stream : streamRegistry) { - stream.appendSummaryHtml(writer); - writer.write("
    "); - } - } - - // Configure backoff to retry calls forever, with a maximum sane retry interval. - private BackOff grpcBackoff() { - return FluentBackoff.DEFAULT - .withInitialBackoff(MIN_BACKOFF) - .withMaxBackoff(maxBackoff) - .backoff(); - } - - private ResponseT callWithBackoff(Supplier function) { - BackOff backoff = grpcBackoff(); - int rpcErrors = 0; - while (true) { - try { - return function.get(); - } catch (StatusRuntimeException e) { - try { - if (++rpcErrors % 20 == 0) { - LOG.warn( - "Many exceptions calling gRPC. Last exception: {} with status {}", - e, - e.getStatus()); - } - if (!BackOffUtils.next(Sleeper.DEFAULT, backoff)) { - throw new WindmillServerStub.RpcException(e); - } - } catch (IOException | InterruptedException i) { - if (i instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - WindmillServerStub.RpcException rpcException = new WindmillServerStub.RpcException(e); - rpcException.addSuppressed(i); - throw rpcException; - } - } - } - } - - @Override - public GetWorkResponse getWork(GetWorkRequest request) { - if (syncApplianceStub == null) { - return callWithBackoff( - () -> - syncStub() - .withDeadlineAfter(unaryDeadlineSeconds, TimeUnit.SECONDS) - .getWork( - request - .toBuilder() - .setJobId(options.getJobId()) - .setProjectId(options.getProject()) - .setWorkerId(options.getWorkerId()) - .build())); - } else { - return callWithBackoff( - () -> - syncApplianceStub - .withDeadlineAfter(unaryDeadlineSeconds, TimeUnit.SECONDS) - .getWork(request)); - } - } - - @Override - public GetDataResponse getData(GetDataRequest request) { - if (syncApplianceStub == null) { - return callWithBackoff( - () -> - syncStub() - .withDeadlineAfter(unaryDeadlineSeconds, TimeUnit.SECONDS) - .getData( - request - .toBuilder() - .setJobId(options.getJobId()) - .setProjectId(options.getProject()) - .build())); - } else { - return callWithBackoff( - () -> - syncApplianceStub - .withDeadlineAfter(unaryDeadlineSeconds, TimeUnit.SECONDS) - .getData(request)); - } - } - - @Override - public CommitWorkResponse commitWork(CommitWorkRequest request) { - if (syncApplianceStub == null) { - return callWithBackoff( - () -> - syncStub() - .withDeadlineAfter(unaryDeadlineSeconds, TimeUnit.SECONDS) - .commitWork( - request - .toBuilder() - .setJobId(options.getJobId()) - .setProjectId(options.getProject()) - .build())); - } else { - return callWithBackoff( - () -> - syncApplianceStub - .withDeadlineAfter(unaryDeadlineSeconds, TimeUnit.SECONDS) - .commitWork(request)); - } - } - - @Override - public GetWorkStream getWorkStream(GetWorkRequest request, WorkItemReceiver receiver) { - return new GrpcGetWorkStream( - GetWorkRequest.newBuilder(request) - .setJobId(options.getJobId()) - .setProjectId(options.getProject()) - .setWorkerId(options.getWorkerId()) - .build(), - receiver); - } - - @Override - public GetDataStream getDataStream() { - return new GrpcGetDataStream(); - } - - @Override - public CommitWorkStream commitWorkStream() { - return new GrpcCommitWorkStream(); - } - - @Override - public GetConfigResponse getConfig(GetConfigRequest request) { - if (syncApplianceStub == null) { - throw new RpcException( - new UnsupportedOperationException("GetConfig not supported with windmill service.")); - } else { - return callWithBackoff( - () -> - syncApplianceStub - .withDeadlineAfter(unaryDeadlineSeconds, TimeUnit.SECONDS) - .getConfig(request)); - } - } - - @Override - public ReportStatsResponse reportStats(ReportStatsRequest request) { - if (syncApplianceStub == null) { - throw new RpcException( - new UnsupportedOperationException("ReportStats not supported with windmill service.")); - } else { - return callWithBackoff( - () -> - syncApplianceStub - .withDeadlineAfter(unaryDeadlineSeconds, TimeUnit.SECONDS) - .reportStats(request)); - } - } - - @Override - public long getAndResetThrottleTime() { - return getWorkThrottleTimer.getAndResetThrottleTime() - + getDataThrottleTimer.getAndResetThrottleTime() - + commitWorkThrottleTimer.getAndResetThrottleTime(); - } - - private JobHeader makeHeader() { - return JobHeader.newBuilder() - .setJobId(options.getJobId()) - .setProjectId(options.getProject()) - .setWorkerId(options.getWorkerId()) - .build(); - } - - /** Returns a long that is unique to this process. */ - private static long uniqueId() { - return nextId.incrementAndGet(); - } - - /** - * Base class for persistent streams connecting to Windmill. - * - *

    This class handles the underlying gRPC StreamObservers, and automatically reconnects the - * stream if it is broken. Subclasses are responsible for retrying requests that have been lost on - * a broken stream. - * - *

    Subclasses should override onResponse to handle responses from the server, and onNewStream - * to perform any work that must be done when a new stream is created, such as sending headers or - * retrying requests. - * - *

    send and startStream should not be called from onResponse; use executor() instead. - * - *

    Synchronization on this is used to synchronize the gRpc stream state and internal data - * structures. Since grpc channel operations may block, synchronization on this stream may also - * block. This is generally not a problem since streams are used in a single-threaded manner. - * However some accessors used for status page and other debugging need to take care not to - * require synchronizing on this. - */ - private abstract class AbstractWindmillStream implements WindmillStream { - private final StreamObserverFactory streamObserverFactory = - StreamObserverFactory.direct( - streamDeadlineSeconds * 2, options.getWindmillMessagesBetweenIsReadyChecks()); - private final Function, StreamObserver> clientFactory; - private final Executor executor = - Executors.newSingleThreadExecutor( - new ThreadFactoryBuilder() - .setDaemon(true) - .setNameFormat("WindmillStream-thread") - .build()); - - // The following should be protected by synchronizing on this, except for - // the atomics which may be read atomically for status pages. - private StreamObserver requestObserver; - // Indicates if the current stream in requestObserver is closed by calling close() method - private final AtomicBoolean streamClosed = new AtomicBoolean(); - private final AtomicLong startTimeMs = new AtomicLong(); - private final AtomicLong lastSendTimeMs = new AtomicLong(); - private final AtomicLong lastResponseTimeMs = new AtomicLong(); - private final AtomicInteger errorCount = new AtomicInteger(); - private final AtomicReference lastError = new AtomicReference<>(); - private final BackOff backoff = grpcBackoff(); - private final AtomicLong sleepUntil = new AtomicLong(); - protected final AtomicBoolean clientClosed = new AtomicBoolean(); - private final CountDownLatch finishLatch = new CountDownLatch(1); - - protected AbstractWindmillStream( - Function, StreamObserver> clientFactory) { - this.clientFactory = clientFactory; - } - - /** Called on each response from the server. */ - protected abstract void onResponse(ResponseT response); - /** Called when a new underlying stream to the server has been opened. */ - protected abstract void onNewStream(); - /** Returns whether there are any pending requests that should be retried on a stream break. */ - protected abstract boolean hasPendingRequests(); - /** - * Called when the stream is throttled due to resource exhausted errors. Will be called for each - * resource exhausted error not just the first. onResponse() must stop throttling on receipt of - * the first good message. - */ - protected abstract void startThrottleTimer(); - /** Send a request to the server. */ - protected final void send(RequestT request) { - lastSendTimeMs.set(Instant.now().getMillis()); - synchronized (this) { - if (streamClosed.get()) { - throw new IllegalStateException("Send called on a client closed stream."); - } - requestObserver.onNext(request); - } - } - - /** Starts the underlying stream. */ - protected final void startStream() { - // Add the stream to the registry after it has been fully constructed. - streamRegistry.add(this); - BackOff backoff = grpcBackoff(); - while (true) { - try { - synchronized (this) { - startTimeMs.set(Instant.now().getMillis()); - lastResponseTimeMs.set(0); - requestObserver = streamObserverFactory.from(clientFactory, new ResponseObserver()); - streamClosed.set(false); - onNewStream(); - if (clientClosed.get()) { - close(); - } - return; - } - } catch (Exception e) { - LOG.error("Failed to create new stream, retrying: ", e); - try { - long sleep = backoff.nextBackOffMillis(); - sleepUntil.set(Instant.now().getMillis() + sleep); - Thread.sleep(sleep); - } catch (InterruptedException i) { - // Keep trying to create the stream. - } catch (IOException i) { - // Ignore. - } - } - } - } - - protected final Executor executor() { - return executor; - } - - public final synchronized void maybeSendHealthCheck(Instant lastSendThreshold) { - if (lastSendTimeMs.get() < lastSendThreshold.getMillis() && !clientClosed.get()) { - try { - sendHealthCheck(); - } catch (RuntimeException e) { - LOG.debug("Received exception sending health check.", e); - } - } - } - - protected abstract void sendHealthCheck(); - - protected final long debugDuration(long nowMs, long startMs) { - if (startMs <= 0) { - return -1; - } - return Math.max(0, nowMs - startMs); - } - - // Care is taken that synchronization on this is unnecessary for all status page information. - // Blocking sends are made beneath this stream object's lock which could block status page - // rendering. - public final void appendSummaryHtml(PrintWriter writer) { - appendSpecificHtml(writer); - if (errorCount.get() > 0) { - writer.format(", %d errors, last error [ %s ]", errorCount.get(), lastError.get()); - } - if (clientClosed.get()) { - writer.write(", client closed"); - } - long nowMs = Instant.now().getMillis(); - long sleepLeft = sleepUntil.get() - nowMs; - if (sleepLeft > 0) { - writer.format(", %dms backoff remaining", sleepLeft); - } - writer.format( - ", current stream is %dms old, last send %dms, last response %dms, closed: %s", - debugDuration(nowMs, startTimeMs.get()), - debugDuration(nowMs, lastSendTimeMs.get()), - debugDuration(nowMs, lastResponseTimeMs.get()), - streamClosed.get()); - } - - // Don't require synchronization on stream, see the appendSummaryHtml comment. - protected abstract void appendSpecificHtml(PrintWriter writer); - - private class ResponseObserver implements StreamObserver { - @Override - public void onNext(ResponseT response) { - try { - backoff.reset(); - } catch (IOException e) { - // Ignore. - } - lastResponseTimeMs.set(Instant.now().getMillis()); - onResponse(response); - } - - @Override - public void onError(Throwable t) { - onStreamFinished(t); - } - - @Override - public void onCompleted() { - onStreamFinished(null); - } - - private void onStreamFinished(@Nullable Throwable t) { - synchronized (this) { - if (clientClosed.get() && !hasPendingRequests()) { - streamRegistry.remove(AbstractWindmillStream.this); - finishLatch.countDown(); - return; - } - } - if (t != null) { - Status status = null; - if (t instanceof StatusRuntimeException) { - status = ((StatusRuntimeException) t).getStatus(); - } - String statusError = status.toString(); - lastError.set(statusError); - if (errorCount.getAndIncrement() % logEveryNStreamFailures == 0) { - long nowMillis = Instant.now().getMillis(); - String responseDebug; - if (lastResponseTimeMs.get() == 0) { - responseDebug = "never received response"; - } else { - responseDebug = - "received response " + (nowMillis - lastResponseTimeMs.get()) + "ms ago"; - } - LOG.debug( - "{} streaming Windmill RPC errors for {}, last was: {} with status {}." - + " created {}ms ago, {}. This is normal with autoscaling.", - AbstractWindmillStream.this.getClass(), - errorCount.get(), - t.toString(), - statusError, - nowMillis - startTimeMs.get(), - responseDebug); - } - // If the stream was stopped due to a resource exhausted error then we are throttled. - if (status != null && status.getCode() == Status.Code.RESOURCE_EXHAUSTED) { - startThrottleTimer(); - } - - try { - long sleep = backoff.nextBackOffMillis(); - sleepUntil.set(Instant.now().getMillis() + sleep); - Thread.sleep(sleep); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (IOException e) { - // Ignore. - } - } else { - errorCount.incrementAndGet(); - String error = - "Stream completed successfully but did not complete requested operations, " - + "recreating"; - LOG.warn(error); - lastError.set(error); - } - executor.execute(AbstractWindmillStream.this::startStream); - } - } - - @Override - public final synchronized void close() { - // Synchronization of close and onCompleted necessary for correct retry logic in onNewStream. - clientClosed.set(true); - requestObserver.onCompleted(); - streamClosed.set(true); - } - - @Override - public final boolean awaitTermination(int time, TimeUnit unit) throws InterruptedException { - return finishLatch.await(time, unit); - } - - @Override - public final Instant startTime() { - return new Instant(startTimeMs.get()); - } - } - - private class GrpcGetWorkStream - extends AbstractWindmillStream - implements GetWorkStream { - private final GetWorkRequest request; - private final WorkItemReceiver receiver; - private final Map buffers = new ConcurrentHashMap<>(); - private final AtomicLong inflightMessages = new AtomicLong(); - private final AtomicLong inflightBytes = new AtomicLong(); - - private GrpcGetWorkStream(GetWorkRequest request, WorkItemReceiver receiver) { - super( - responseObserver -> - stub() - .withDeadlineAfter(streamDeadlineSeconds, TimeUnit.SECONDS) - .getWorkStream(responseObserver)); - this.request = request; - this.receiver = receiver; - startStream(); - } - - @Override - protected synchronized void onNewStream() { - buffers.clear(); - inflightMessages.set(request.getMaxItems()); - inflightBytes.set(request.getMaxBytes()); - send(StreamingGetWorkRequest.newBuilder().setRequest(request).build()); - } - - @Override - protected boolean hasPendingRequests() { - return false; - } - - @Override - public void appendSpecificHtml(PrintWriter writer) { - // Number of buffers is same as distict workers that sent work on this stream. - writer.format( - "GetWorkStream: %d buffers, %d inflight messages allowed, %d inflight bytes allowed", - buffers.size(), inflightMessages.intValue(), inflightBytes.intValue()); - } - - @Override - public void sendHealthCheck() { - send( - StreamingGetWorkRequest.newBuilder() - .setRequestExtension( - StreamingGetWorkRequestExtension.newBuilder() - .setMaxItems(0) - .setMaxBytes(0) - .build()) - .build()); - } - - @Override - protected void onResponse(StreamingGetWorkResponseChunk chunk) { - getWorkThrottleTimer.stop(); - long id = chunk.getStreamId(); - - WorkItemBuffer buffer = buffers.computeIfAbsent(id, (Long l) -> new WorkItemBuffer()); - buffer.append(chunk); - - if (chunk.getRemainingBytesForWorkItem() == 0) { - long size = buffer.bufferedSize(); - buffer.runAndReset(); - - // Record the fact that there are now fewer outstanding messages and bytes on the stream. - long numInflight = inflightMessages.decrementAndGet(); - long bytesInflight = inflightBytes.addAndGet(-size); - - // If the outstanding items or bytes limit has gotten too low, top both off with a - // GetWorkExtension. The goal is to keep the limits relatively close to their maximum - // values without sending too many extension requests. - if (numInflight < request.getMaxItems() / 2 || bytesInflight < request.getMaxBytes() / 2) { - long moreItems = request.getMaxItems() - numInflight; - long moreBytes = request.getMaxBytes() - bytesInflight; - inflightMessages.getAndAdd(moreItems); - inflightBytes.getAndAdd(moreBytes); - final StreamingGetWorkRequest extension = - StreamingGetWorkRequest.newBuilder() - .setRequestExtension( - StreamingGetWorkRequestExtension.newBuilder() - .setMaxItems(moreItems) - .setMaxBytes(moreBytes)) - .build(); - executor() - .execute( - () -> { - try { - send(extension); - } catch (IllegalStateException e) { - // Stream was closed. - } - }); - } - } - } - - @Override - protected void startThrottleTimer() { - getWorkThrottleTimer.start(); - } - - private class WorkItemBuffer { - private String computation; - private Instant inputDataWatermark; - private Instant synchronizedProcessingTime; - private ByteString data = ByteString.EMPTY; - private long bufferedSize = 0; - - private void setMetadata(Windmill.ComputationWorkItemMetadata metadata) { - this.computation = metadata.getComputationId(); - this.inputDataWatermark = - WindmillTimeUtils.windmillToHarnessWatermark(metadata.getInputDataWatermark()); - this.synchronizedProcessingTime = - WindmillTimeUtils.windmillToHarnessWatermark( - metadata.getDependentRealtimeInputWatermark()); - } - - public void append(StreamingGetWorkResponseChunk chunk) { - if (chunk.hasComputationMetadata()) { - setMetadata(chunk.getComputationMetadata()); - } - - this.data = data.concat(chunk.getSerializedWorkItem()); - this.bufferedSize += chunk.getSerializedWorkItem().size(); - } - - public long bufferedSize() { - return bufferedSize; - } - - public void runAndReset() { - try { - receiver.receiveWork( - computation, - inputDataWatermark, - synchronizedProcessingTime, - Windmill.WorkItem.parseFrom(data.newInput())); - } catch (IOException e) { - LOG.error("Failed to parse work item from stream: ", e); - } - data = ByteString.EMPTY; - bufferedSize = 0; - } - } - } - - private class GrpcGetDataStream - extends AbstractWindmillStream - implements GetDataStream { - private class QueuedRequest { - public QueuedRequest(String computation, KeyedGetDataRequest request) { - this.id = uniqueId(); - this.globalDataRequest = null; - this.dataRequest = - ComputationGetDataRequest.newBuilder() - .setComputationId(computation) - .addRequests(request) - .build(); - this.byteSize = this.dataRequest.getSerializedSize(); - } - - public QueuedRequest(GlobalDataRequest request) { - this.id = uniqueId(); - this.globalDataRequest = request; - this.dataRequest = null; - this.byteSize = this.globalDataRequest.getSerializedSize(); - } - - final long id; - final long byteSize; - final GlobalDataRequest globalDataRequest; - final ComputationGetDataRequest dataRequest; - AppendableInputStream responseStream = null; - } - - private class QueuedBatch { - public QueuedBatch() {} - - final List requests = new ArrayList<>(); - long byteSize = 0; - boolean finalized = false; - final CountDownLatch sent = new CountDownLatch(1); - }; - - private final Deque batches = new ConcurrentLinkedDeque<>(); - private final Map pending = new ConcurrentHashMap<>(); - - @Override - public void appendSpecificHtml(PrintWriter writer) { - writer.format( - "GetDataStream: %d queued batches, %d pending requests [", - batches.size(), pending.size()); - for (Map.Entry entry : pending.entrySet()) { - writer.format("Stream %d ", entry.getKey()); - if (entry.getValue().cancelled.get()) { - writer.append("cancelled "); - } - if (entry.getValue().complete.get()) { - writer.append("complete "); - } - int queueSize = entry.getValue().queue.size(); - if (queueSize > 0) { - writer.format("%d queued responses ", queueSize); - } - long blockedMs = entry.getValue().blockedStartMs.get(); - if (blockedMs > 0) { - writer.format("blocked for %dms", Instant.now().getMillis() - blockedMs); - } - } - writer.append("]"); - } - - GrpcGetDataStream() { - super( - responseObserver -> - stub() - .withDeadlineAfter(streamDeadlineSeconds, TimeUnit.SECONDS) - .getDataStream(responseObserver)); - startStream(); - } - - @Override - protected synchronized void onNewStream() { - send(StreamingGetDataRequest.newBuilder().setHeader(makeHeader()).build()); - - if (clientClosed.get()) { - // We rely on close only occurring after all methods on the stream have returned. - // Since the requestKeyedData and requestGlobalData methods are blocking this - // means there should be no pending requests. - Verify.verify(!hasPendingRequests()); - } else { - for (AppendableInputStream responseStream : pending.values()) { - responseStream.cancel(); - } - } - } - - @Override - protected boolean hasPendingRequests() { - return !pending.isEmpty() || !batches.isEmpty(); - } - - @Override - protected void onResponse(StreamingGetDataResponse chunk) { - Preconditions.checkArgument(chunk.getRequestIdCount() == chunk.getSerializedResponseCount()); - Preconditions.checkArgument( - chunk.getRemainingBytesForResponse() == 0 || chunk.getRequestIdCount() == 1); - getDataThrottleTimer.stop(); - - for (int i = 0; i < chunk.getRequestIdCount(); ++i) { - AppendableInputStream responseStream = pending.get(chunk.getRequestId(i)); - Verify.verify(responseStream != null, "No pending response stream"); - responseStream.append(chunk.getSerializedResponse(i).newInput()); - if (chunk.getRemainingBytesForResponse() == 0) { - responseStream.complete(); - } - } - } - - @Override - protected void startThrottleTimer() { - getDataThrottleTimer.start(); - } - - @Override - public KeyedGetDataResponse requestKeyedData(String computation, KeyedGetDataRequest request) { - return issueRequest(new QueuedRequest(computation, request), KeyedGetDataResponse::parseFrom); - } - - @Override - public GlobalData requestGlobalData(GlobalDataRequest request) { - return issueRequest(new QueuedRequest(request), GlobalData::parseFrom); - } - - @Override - public void refreshActiveWork(Map> active) { - long builderBytes = 0; - StreamingGetDataRequest.Builder builder = StreamingGetDataRequest.newBuilder(); - for (Map.Entry> entry : active.entrySet()) { - for (KeyedGetDataRequest request : entry.getValue()) { - // Calculate the bytes with some overhead for proto encoding. - long bytes = (long) entry.getKey().length() + request.getSerializedSize() + 10; - if (builderBytes > 0 - && (builderBytes + bytes > GET_DATA_STREAM_CHUNK_SIZE - || builder.getRequestIdCount() >= streamingRpcBatchLimit)) { - send(builder.build()); - builderBytes = 0; - builder.clear(); - } - builderBytes += bytes; - builder.addStateRequest( - ComputationGetDataRequest.newBuilder() - .setComputationId(entry.getKey()) - .addRequests(request)); - } - } - if (builderBytes > 0) { - send(builder.build()); - } - } - - @Override - public void sendHealthCheck() { - if (hasPendingRequests()) { - send(StreamingGetDataRequest.newBuilder().build()); - } - } - - private ResponseT issueRequest(QueuedRequest request, ParseFn parseFn) { - while (true) { - request.responseStream = new AppendableInputStream(); - try { - queueRequestAndWait(request); - return parseFn.parse(request.responseStream); - } catch (CancellationException e) { - // Retry issuing the request since the response stream was cancelled. - continue; - } catch (IOException e) { - LOG.error("Parsing GetData response failed: ", e); - continue; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } finally { - pending.remove(request.id); - } - } - } - - private void queueRequestAndWait(QueuedRequest request) throws InterruptedException { - QueuedBatch batch; - boolean responsibleForSend = false; - CountDownLatch waitForSendLatch = null; - synchronized (batches) { - batch = batches.isEmpty() ? null : batches.getLast(); - if (batch == null - || batch.finalized - || batch.requests.size() >= streamingRpcBatchLimit - || batch.byteSize + request.byteSize > GET_DATA_STREAM_CHUNK_SIZE) { - if (batch != null) { - waitForSendLatch = batch.sent; - } - batch = new QueuedBatch(); - batches.addLast(batch); - responsibleForSend = true; - } - batch.requests.add(request); - batch.byteSize += request.byteSize; - } - if (responsibleForSend) { - if (waitForSendLatch == null) { - // If there was not a previous batch wait a little while to improve - // batching. - Thread.sleep(1); - } else { - waitForSendLatch.await(); - } - // Finalize the batch so that no additional requests will be added. Leave the batch in the - // queue so that a subsequent batch will wait for it's completion. - synchronized (batches) { - Verify.verify(batch == batches.peekFirst()); - batch.finalized = true; - } - sendBatch(batch.requests); - synchronized (batches) { - Verify.verify(batch == batches.pollFirst()); - } - // Notify all waiters with requests in this batch as well as the sender - // of the next batch (if one exists). - batch.sent.countDown(); - } else { - // Wait for this batch to be sent before parsing the response. - batch.sent.await(); - } - } - - private void sendBatch(List requests) { - StreamingGetDataRequest batchedRequest = flushToBatch(requests); - synchronized (this) { - // Synchronization of pending inserts is necessary with send to ensure duplicates are not - // sent on stream reconnect. - for (QueuedRequest request : requests) { - Verify.verify(pending.put(request.id, request.responseStream) == null); - } - try { - send(batchedRequest); - } catch (IllegalStateException e) { - // The stream broke before this call went through; onNewStream will retry the fetch. - LOG.warn("GetData stream broke before call started.", e); - } - } - } - - private StreamingGetDataRequest flushToBatch(List requests) { - // Put all global data requests first because there is only a single repeated field for - // request ids and the initial ids correspond to global data requests if they are present. - requests.sort( - (QueuedRequest r1, QueuedRequest r2) -> { - boolean r1gd = r1.globalDataRequest != null; - boolean r2gd = r2.globalDataRequest != null; - return r1gd == r2gd ? 0 : (r1gd ? -1 : 1); - }); - StreamingGetDataRequest.Builder builder = StreamingGetDataRequest.newBuilder(); - for (QueuedRequest request : requests) { - builder.addRequestId(request.id); - if (request.globalDataRequest == null) { - builder.addStateRequest(request.dataRequest); - } else { - builder.addGlobalDataRequest(request.globalDataRequest); - } - } - return builder.build(); - } - } - - private class GrpcCommitWorkStream - extends AbstractWindmillStream - implements CommitWorkStream { - private class PendingRequest { - private final String computation; - private final WorkItemCommitRequest request; - private final Consumer onDone; - - PendingRequest( - String computation, WorkItemCommitRequest request, Consumer onDone) { - this.computation = computation; - this.request = request; - this.onDone = onDone; - } - - long getBytes() { - return (long) request.getSerializedSize() + computation.length(); - } - } - - private final Map pending = new ConcurrentHashMap<>(); - - private class Batcher { - long queuedBytes = 0; - final Map queue = new HashMap<>(); - - boolean canAccept(PendingRequest request) { - return queue.isEmpty() - || (queue.size() < streamingRpcBatchLimit - && (request.getBytes() + queuedBytes) < COMMIT_STREAM_CHUNK_SIZE); - } - - void add(long id, PendingRequest request) { - assert (canAccept(request)); - queuedBytes += request.getBytes(); - queue.put(id, request); - } - - void flush() { - flushInternal(queue); - queuedBytes = 0; - queue.clear(); - } - } - - private final Batcher batcher = new Batcher(); - - GrpcCommitWorkStream() { - super( - responseObserver -> - stub() - .withDeadlineAfter(streamDeadlineSeconds, TimeUnit.SECONDS) - .commitWorkStream(responseObserver)); - startStream(); - } - - @Override - public void appendSpecificHtml(PrintWriter writer) { - writer.format("CommitWorkStream: %d pending", pending.size()); - } - - @Override - protected synchronized void onNewStream() { - send(StreamingCommitWorkRequest.newBuilder().setHeader(makeHeader()).build()); - Batcher resendBatcher = new Batcher(); - for (Map.Entry entry : pending.entrySet()) { - if (!resendBatcher.canAccept(entry.getValue())) { - resendBatcher.flush(); - } - resendBatcher.add(entry.getKey(), entry.getValue()); - } - resendBatcher.flush(); - } - - @Override - protected boolean hasPendingRequests() { - return !pending.isEmpty(); - } - - @Override - public void sendHealthCheck() { - if (hasPendingRequests()) { - StreamingCommitWorkRequest.Builder builder = StreamingCommitWorkRequest.newBuilder(); - builder.addCommitChunkBuilder().setRequestId(HEARTBEAT_REQUEST_ID); - send(builder.build()); - } - } - - @Override - protected void onResponse(StreamingCommitResponse response) { - commitWorkThrottleTimer.stop(); - - RuntimeException finalException = null; - for (int i = 0; i < response.getRequestIdCount(); ++i) { - long requestId = response.getRequestId(i); - if (requestId == HEARTBEAT_REQUEST_ID) { - continue; - } - PendingRequest done = pending.remove(requestId); - if (done == null) { - LOG.error("Got unknown commit request ID: {}", requestId); - } else { - try { - done.onDone.accept( - (i < response.getStatusCount()) ? response.getStatus(i) : CommitStatus.OK); - } catch (RuntimeException e) { - // Catch possible exceptions to ensure that an exception for one commit does not prevent - // other commits from being processed. - LOG.warn("Exception while processing commit response.", e); - finalException = e; - } - } - } - if (finalException != null) { - throw finalException; - } - } - - @Override - protected void startThrottleTimer() { - commitWorkThrottleTimer.start(); - } - - @Override - public boolean commitWorkItem( - String computation, WorkItemCommitRequest commitRequest, Consumer onDone) { - PendingRequest request = new PendingRequest(computation, commitRequest, onDone); - if (!batcher.canAccept(request)) { - return false; - } - batcher.add(uniqueId(), request); - return true; - } - - @Override - public void flush() { - batcher.flush(); - } - - private void flushInternal(Map requests) { - if (requests.isEmpty()) { - return; - } - if (requests.size() == 1) { - Map.Entry elem = requests.entrySet().iterator().next(); - if (elem.getValue().request.getSerializedSize() > COMMIT_STREAM_CHUNK_SIZE) { - issueMultiChunkRequest(elem.getKey(), elem.getValue()); - } else { - issueSingleRequest(elem.getKey(), elem.getValue()); - } - } else { - issueBatchedRequest(requests); - } - } - - private void issueSingleRequest(final long id, PendingRequest pendingRequest) { - StreamingCommitWorkRequest.Builder requestBuilder = StreamingCommitWorkRequest.newBuilder(); - requestBuilder - .addCommitChunkBuilder() - .setComputationId(pendingRequest.computation) - .setRequestId(id) - .setShardingKey(pendingRequest.request.getShardingKey()) - .setSerializedWorkItemCommit(pendingRequest.request.toByteString()); - StreamingCommitWorkRequest chunk = requestBuilder.build(); - synchronized (this) { - pending.put(id, pendingRequest); - try { - send(chunk); - } catch (IllegalStateException e) { - // Stream was broken, request will be retried when stream is reopened. - } - } - } - - private void issueBatchedRequest(Map requests) { - StreamingCommitWorkRequest.Builder requestBuilder = StreamingCommitWorkRequest.newBuilder(); - String lastComputation = null; - for (Map.Entry entry : requests.entrySet()) { - PendingRequest request = entry.getValue(); - StreamingCommitRequestChunk.Builder chunkBuilder = requestBuilder.addCommitChunkBuilder(); - if (lastComputation == null || !lastComputation.equals(request.computation)) { - chunkBuilder.setComputationId(request.computation); - lastComputation = request.computation; - } - chunkBuilder.setRequestId(entry.getKey()); - chunkBuilder.setShardingKey(request.request.getShardingKey()); - chunkBuilder.setSerializedWorkItemCommit(request.request.toByteString()); - } - StreamingCommitWorkRequest request = requestBuilder.build(); - synchronized (this) { - pending.putAll(requests); - try { - send(request); - } catch (IllegalStateException e) { - // Stream was broken, request will be retried when stream is reopened. - } - } - } - - private void issueMultiChunkRequest(final long id, PendingRequest pendingRequest) { - Preconditions.checkNotNull(pendingRequest.computation); - final ByteString serializedCommit = pendingRequest.request.toByteString(); - - synchronized (this) { - pending.put(id, pendingRequest); - for (int i = 0; i < serializedCommit.size(); i += COMMIT_STREAM_CHUNK_SIZE) { - int end = i + COMMIT_STREAM_CHUNK_SIZE; - ByteString chunk = serializedCommit.substring(i, Math.min(end, serializedCommit.size())); - - StreamingCommitRequestChunk.Builder chunkBuilder = - StreamingCommitRequestChunk.newBuilder() - .setRequestId(id) - .setSerializedWorkItemCommit(chunk) - .setComputationId(pendingRequest.computation) - .setShardingKey(pendingRequest.request.getShardingKey()); - int remaining = serializedCommit.size() - end; - if (remaining > 0) { - chunkBuilder.setRemainingBytesForWorkItem(remaining); - } - - StreamingCommitWorkRequest requestChunk = - StreamingCommitWorkRequest.newBuilder().addCommitChunk(chunkBuilder).build(); - try { - send(requestChunk); - } catch (IllegalStateException e) { - // Stream was broken, request will be retried when stream is reopened. - break; - } - } - } - } - } - - @FunctionalInterface - private interface ParseFn { - ResponseT parse(InputStream input) throws IOException; - } - - /** An InputStream that can be dynamically extended with additional InputStreams. */ - @SuppressWarnings("JdkObsolete") - private static class AppendableInputStream extends InputStream { - private static final InputStream POISON_PILL = ByteString.EMPTY.newInput(); - private final AtomicBoolean cancelled = new AtomicBoolean(false); - private final AtomicBoolean complete = new AtomicBoolean(false); - private final AtomicLong blockedStartMs = new AtomicLong(); - private final BlockingDeque queue = new LinkedBlockingDeque<>(10); - private final InputStream stream = - new SequenceInputStream( - new Enumeration() { - // The first stream is eagerly read on SequenceInputStream creation. For this reason - // we use an empty element as the first input to avoid blocking from the queue when - // creating the AppendableInputStream. - InputStream current = ByteString.EMPTY.newInput(); - - @Override - public boolean hasMoreElements() { - if (current != null) { - return true; - } - - try { - blockedStartMs.set(Instant.now().getMillis()); - current = queue.poll(180, TimeUnit.SECONDS); - if (current != null && current != POISON_PILL) { - return true; - } - if (cancelled.get()) { - throw new CancellationException(); - } - if (complete.get()) { - return false; - } - throw new IllegalStateException( - "Got poison pill or timeout but stream is not done."); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new CancellationException(); - } - } - - @Override - public InputStream nextElement() { - if (!hasMoreElements()) { - throw new NoSuchElementException(); - } - blockedStartMs.set(0); - InputStream next = current; - current = null; - return next; - } - }); - - /** Appends a new InputStream to the tail of this stream. */ - public synchronized void append(InputStream chunk) { - try { - queue.put(chunk); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.debug("interrupted append"); - } - } - - /** Cancels the stream. Future calls to InputStream methods will throw CancellationException. */ - public synchronized void cancel() { - cancelled.set(true); - try { - // Put the poison pill at the head of the queue to cancel as quickly as possible. - queue.clear(); - queue.putFirst(POISON_PILL); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.debug("interrupted cancel"); - } - } - - /** Signals that no new InputStreams will be added to this stream. */ - public synchronized void complete() { - complete.set(true); - try { - queue.put(POISON_PILL); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.debug("interrupted complete"); - } - } - - @Override - public int read() throws IOException { - if (cancelled.get()) { - throw new CancellationException(); - } - return stream.read(); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - if (cancelled.get()) { - throw new CancellationException(); - } - return stream.read(b, off, len); - } - - @Override - public int available() throws IOException { - if (cancelled.get()) { - throw new CancellationException(); - } - return stream.available(); - } - - @Override - public void close() throws IOException { - stream.close(); - } - } - - /** - * A stopwatch used to track the amount of time spent throttled due to Resource Exhausted errors. - * Throttle time is cumulative for all three rpcs types but not for all streams. So if GetWork and - * CommitWork are both blocked for x, totalTime will be 2x. However, if 2 GetWork streams are both - * blocked for x totalTime will be x. All methods are thread safe. - */ - private static class ThrottleTimer { - - // This is -1 if not currently being throttled or the time in - // milliseconds when throttling for this type started. - private long startTime = -1; - // This is the collected total throttle times since the last poll. Throttle times are - // reported as a delta so this is cleared whenever it gets reported. - private long totalTime = 0; - - /** - * Starts the timer if it has not been started and does nothing if it has already been started. - */ - public synchronized void start() { - if (!throttled()) { // This timer is not started yet so start it now. - startTime = Instant.now().getMillis(); - } - } - - /** Stops the timer if it has been started and does nothing if it has not been started. */ - public synchronized void stop() { - if (throttled()) { // This timer has been started already so stop it now. - totalTime += Instant.now().getMillis() - startTime; - startTime = -1; - } - } - - /** Returns if the specified type is currently being throttled. */ - public synchronized boolean throttled() { - return startTime != -1; - } - - /** Returns the combined total of all throttle times and resets those times to 0. */ - public synchronized long getAndResetThrottleTime() { - if (throttled()) { - stop(); - start(); - } - long toReturn = totalTime; - totalTime = 0; - return toReturn; - } - } -} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/StreamObserverFactory.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/StreamObserverFactory.java index a046f2fd46ac3..e0878b7b0b91b 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/StreamObserverFactory.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/StreamObserverFactory.java @@ -33,9 +33,9 @@ public static StreamObserverFactory direct( return new Direct(deadlineSeconds, messagesBetweenIsReadyChecks); } - public abstract StreamObserver from( - Function, StreamObserver> clientFactory, - StreamObserver responseObserver); + public abstract StreamObserver from( + Function, StreamObserver> clientFactory, + StreamObserver responseObserver); private static class Direct extends StreamObserverFactory { private final long deadlineSeconds; @@ -47,14 +47,14 @@ private static class Direct extends StreamObserverFactory { } @Override - public StreamObserver from( - Function, StreamObserver> clientFactory, - StreamObserver inboundObserver) { + public StreamObserver from( + Function, StreamObserver> clientFactory, + StreamObserver inboundObserver) { AdvancingPhaser phaser = new AdvancingPhaser(1); - CallStreamObserver outboundObserver = - (CallStreamObserver) + CallStreamObserver outboundObserver = + (CallStreamObserver) clientFactory.apply( - new ForwardingClientResponseObserver( + new ForwardingClientResponseObserver( inboundObserver, phaser::arrive, phaser::forceTermination)); return new DirectStreamObserver<>( phaser, outboundObserver, deadlineSeconds, messagesBetweenIsReadyChecks); diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillEndpoints.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillEndpoints.java new file mode 100644 index 0000000000000..64b6e675ef5ff --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillEndpoints.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList.toImmutableList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap.toImmutableMap; + +import com.google.auto.value.AutoValue; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Map; +import java.util.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.net.HostAndPort; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Value class for holding endpoints used for communicating with Windmill service. Corresponds + * directly with {@link Windmill.WorkerMetadataResponse}. + */ +@AutoValue +public abstract class WindmillEndpoints { + private static final Logger LOG = LoggerFactory.getLogger(WindmillEndpoints.class); + + /** + * Used by GetData GlobalDataRequest(s) to support Beam side inputs. Returns a map where the key + * is a global data tag and the value is the endpoint where the data associated with the global + * data tag resides. + * + * @see Beam Side + * Inputs + */ + public abstract ImmutableMap globalDataEndpoints(); + + /** + * Used by GetWork/GetData/CommitWork calls to send, receive, and commit work directly to/from + * Windmill servers. Returns a list of endpoints used to communicate with the corresponding + * Windmill servers. + */ + public abstract ImmutableList windmillEndpoints(); + + public static WindmillEndpoints from( + Windmill.WorkerMetadataResponse workerMetadataResponseProto) { + ImmutableMap globalDataServers = + workerMetadataResponseProto.getGlobalDataEndpointsMap().entrySet().stream() + .collect( + toImmutableMap( + Map.Entry::getKey, // global data key + endpoint -> WindmillEndpoints.Endpoint.from(endpoint.getValue()))); + + ImmutableList windmillServers = + workerMetadataResponseProto.getWorkEndpointsList().stream() + .map(WindmillEndpoints.Endpoint::from) + .collect(toImmutableList()); + + return WindmillEndpoints.builder() + .setGlobalDataEndpoints(globalDataServers) + .setWindmillEndpoints(windmillServers) + .build(); + } + + public static WindmillEndpoints.Builder builder() { + return new AutoValue_WindmillEndpoints.Builder(); + } + + /** + * Representation of an endpoint in {@link Windmill.WorkerMetadataResponse.Endpoint} proto with + * the worker_token field, and direct_endpoint field parsed into a {@link WindmillServiceAddress} + * which holds either a {@link Inet6Address} or {@link HostAndPort} used to connect to Streaming + * Engine. {@link Inet6Address}(s) represent direct Windmill worker connections, and {@link + * HostAndPort}(s) represent connections to the Windmill Dispatcher. + */ + @AutoValue + public abstract static class Endpoint { + /** + * {@link WindmillServiceAddress} representation of {@link + * Windmill.WorkerMetadataResponse.Endpoint#getDirectEndpoint()}. The proto's direct_endpoint + * string can be converted to either {@link Inet6Address} or {@link HostAndPort}. + */ + public abstract Optional directEndpoint(); + + /** + * Corresponds to {@link Windmill.WorkerMetadataResponse.Endpoint#getWorkerToken()} in the + * windmill.proto file. + */ + public abstract Optional workerToken(); + + public static Endpoint.Builder builder() { + return new AutoValue_WindmillEndpoints_Endpoint.Builder(); + } + + public static Endpoint from(Windmill.WorkerMetadataResponse.Endpoint endpointProto) { + Endpoint.Builder endpointBuilder = Endpoint.builder(); + if (endpointProto.hasDirectEndpoint() && !endpointProto.getDirectEndpoint().isEmpty()) { + parseDirectEndpoint(endpointProto.getDirectEndpoint()) + .ifPresent(endpointBuilder::setDirectEndpoint); + } + if (endpointProto.hasWorkerToken() && !endpointProto.getWorkerToken().isEmpty()) { + endpointBuilder.setWorkerToken(endpointProto.getWorkerToken()); + } + + Endpoint endpoint = endpointBuilder.build(); + + if (!endpoint.directEndpoint().isPresent() && !endpoint.workerToken().isPresent()) { + throw new IllegalArgumentException( + String.format( + "direct_endpoint=[%s] not present or could not be parsed, and worker_token" + + " not present. At least one of these fields is required.", + endpointProto.getDirectEndpoint())); + } + + return endpoint; + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setDirectEndpoint(WindmillServiceAddress directEndpoint); + + public abstract Builder setWorkerToken(String workerToken); + + public abstract Endpoint build(); + } + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setGlobalDataEndpoints( + ImmutableMap globalDataServers); + + public abstract Builder setWindmillEndpoints( + ImmutableList windmillServers); + + abstract ImmutableList.Builder windmillEndpointsBuilder(); + + public final Builder addWindmillEndpoint(WindmillEndpoints.Endpoint endpoint) { + windmillEndpointsBuilder().add(endpoint); + return this; + } + + public final Builder addAllWindmillEndpoints(Iterable endpoints) { + windmillEndpointsBuilder().addAll(endpoints); + return this; + } + + abstract ImmutableMap.Builder globalDataEndpointsBuilder(); + + public final Builder addGlobalDataEndpoint( + String globalDataKey, WindmillEndpoints.Endpoint endpoint) { + globalDataEndpointsBuilder().put(globalDataKey, endpoint); + return this; + } + + public final Builder addAllGlobalDataEndpoints( + Map globalDataEndpoints) { + globalDataEndpointsBuilder().putAll(globalDataEndpoints); + return this; + } + + public abstract WindmillEndpoints build(); + } + + private static Optional parseDirectEndpoint(String directEndpoint) { + Optional directEndpointIpV6Address = + tryParseDirectEndpointIntoIpV6Address(directEndpoint).map(WindmillServiceAddress::create); + + return directEndpointIpV6Address.isPresent() + ? directEndpointIpV6Address + : tryParseEndpointIntoHostAndPort(directEndpoint).map(WindmillServiceAddress::create); + } + + private static Optional tryParseEndpointIntoHostAndPort(String directEndpoint) { + try { + return Optional.of(HostAndPort.fromString(directEndpoint)); + } catch (IllegalArgumentException e) { + LOG.warn("{} cannot be parsed into a gcpServiceAddress", directEndpoint); + return Optional.empty(); + } + } + + private static Optional tryParseDirectEndpointIntoIpV6Address( + String directEndpoint) { + InetAddress directEndpointAddress = null; + try { + directEndpointAddress = Inet6Address.getByName(directEndpoint); + } catch (UnknownHostException e) { + LOG.warn( + "Error occurred trying to parse direct_endpoint={} into IPv6 address. Exception={}", + directEndpoint, + e.toString()); + } + + // Inet6Address.getByAddress returns either an IPv4 or an IPv6 address depending on the format + // of the direct_endpoint string. + if (!(directEndpointAddress instanceof Inet6Address)) { + LOG.warn( + "{} is not an IPv6 address. Direct endpoints are expected to be in IPv6 format.", + directEndpoint); + return Optional.empty(); + } + + return Optional.ofNullable((Inet6Address) directEndpointAddress); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillServer.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillServer.java deleted file mode 100644 index fbc67987811ce..0000000000000 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillServer.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.runners.dataflow.worker.windmill; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.StandardCopyOption; - -/** Implementation of a WindmillServerBase. */ -@SuppressWarnings({ - "nullness" // TODO(https://github.com/apache/beam/issues/20497) -}) -public class WindmillServer extends WindmillServerBase { - private static final String WINDMILL_SERVER_JNI_LIBRARY_PROPERTY = "windmill.jni_library"; - private static final String DEFAULT_SHUFFLE_CLIENT_LIBRARY = "libwindmill_service_jni.so"; - - static { - try { - // TODO: Remove the use of JNI here - File tempfile = File.createTempFile("libwindmill_service_jni", ".so"); - InputStream input = - ClassLoader.getSystemResourceAsStream( - System.getProperty( - WINDMILL_SERVER_JNI_LIBRARY_PROPERTY, DEFAULT_SHUFFLE_CLIENT_LIBRARY)); - Files.copy(input, tempfile.toPath(), StandardCopyOption.REPLACE_EXISTING); - System.load(tempfile.getAbsolutePath()); - } catch (IOException e) { - throw new RuntimeException("Loading windmill_service failed:", e); - } - } - - /** - * The host should be specified as protocol://address:port to connect to a windmill server through - * rpcz. - */ - public WindmillServer(String host) { - super(host); - } -} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillServerBase.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillServerBase.java index 8907d86d92cd9..fe81eece13830 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillServerBase.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillServerBase.java @@ -19,11 +19,17 @@ import java.io.IOException; import java.util.Set; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.net.HostAndPort; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.CommitWorkStream; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetDataStream; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetWorkStream; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetWorkStream.WorkItemReceiver; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.net.HostAndPort; /** - * Implementation of a WindmillServerStub which communcates with an actual windmill server at the - * specified location. + * Implementation of a WindmillServerStub which communicates with a Windmill appliance server. + * + * @implNote This is only for use in Streaming Appliance. Please do not change the name or path of + * this class as this will break JNI. */ @SuppressWarnings({ "nullness" // TODO(https://github.com/apache/beam/issues/20497) @@ -31,9 +37,9 @@ public class WindmillServerBase extends WindmillServerStub { /** Pointer to the underlying native windmill client object. */ - private long nativePointer; + private final long nativePointer; - WindmillServerBase(String host) { + protected WindmillServerBase(String host) { this.nativePointer = create(host); } diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillServerStub.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillServerStub.java index 238f22aa643c9..1bb5359e06f48 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillServerStub.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillServerStub.java @@ -19,24 +19,13 @@ import java.io.IOException; import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; -import java.util.function.Supplier; -import javax.annotation.concurrent.ThreadSafe; import org.apache.beam.runners.dataflow.worker.status.StatusDataProvider; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.CommitStatus; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataRequest; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataResponse; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.net.HostAndPort; -import org.checkerframework.checker.nullness.qual.Nullable; -import org.joda.time.Duration; -import org.joda.time.Instant; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.CommitWorkStream; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetDataStream; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetWorkStream; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetWorkStream.WorkItemReceiver; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.net.HostAndPort; /** Stub for communicating with a Windmill server. */ @SuppressWarnings({ @@ -69,16 +58,6 @@ public abstract class WindmillServerStub implements StatusDataProvider { /** Report execution information to the server. */ public abstract Windmill.ReportStatsResponse reportStats(Windmill.ReportStatsRequest request); - /** Functional interface for receiving WorkItems. */ - @FunctionalInterface - public interface WorkItemReceiver { - void receiveWork( - String computation, - @Nullable Instant inputDataWatermark, - Instant synchronizedProcessingTime, - Windmill.WorkItem workItem); - } - /** * Gets work to process, returned as a stream. * @@ -100,137 +79,10 @@ public abstract GetWorkStream getWorkStream( @Override public void appendSummaryHtml(PrintWriter writer) {} - /** Superclass for streams returned by streaming Windmill methods. */ - @ThreadSafe - public interface WindmillStream { - /** Indicates that no more requests will be sent. */ - void close(); - - /** Waits for the server to close its end of the connection, with timeout. */ - boolean awaitTermination(int time, TimeUnit unit) throws InterruptedException; - - /** Returns when the stream was opened. */ - Instant startTime(); - } - - /** Handle representing a stream of GetWork responses. */ - @ThreadSafe - public interface GetWorkStream extends WindmillStream {} - - /** Interface for streaming GetDataRequests to Windmill. */ - @ThreadSafe - public interface GetDataStream extends WindmillStream { - /** Issues a keyed GetData fetch, blocking until the result is ready. */ - KeyedGetDataResponse requestKeyedData(String computation, Windmill.KeyedGetDataRequest request); - - /** Issues a global GetData fetch, blocking until the result is ready. */ - Windmill.GlobalData requestGlobalData(Windmill.GlobalDataRequest request); - - /** Tells windmill processing is ongoing for the given keys. */ - void refreshActiveWork(Map> active); - } - - /** Interface for streaming CommitWorkRequests to Windmill. */ - @ThreadSafe - public interface CommitWorkStream extends WindmillStream { - /** - * Commits a work item and running onDone when the commit has been processed by the server. - * Returns true if the request was accepted. If false is returned the stream should be flushed - * and the request recommitted. - * - *

    onDone will be called with the status of the commit. - */ - boolean commitWorkItem( - String computation, Windmill.WorkItemCommitRequest request, Consumer onDone); - - /** Flushes any pending work items to the wire. */ - void flush(); - } - - /** - * Pool of homogeneous streams to Windmill. - * - *

    The pool holds a fixed total number of streams, and keeps each stream open for a specified - * time to allow for better load-balancing. - */ - @ThreadSafe - public static class StreamPool { - - private final Duration streamTimeout; - - private final class StreamData { - final S stream = supplier.get(); - int holds = 1; - }; - - private final List streams; - private final Supplier supplier; - private final HashMap holds; - - public StreamPool(int numStreams, Duration streamTimeout, Supplier supplier) { - this.streams = new ArrayList<>(numStreams); - for (int i = 0; i < numStreams; i++) { - streams.add(null); - } - this.streamTimeout = streamTimeout; - this.supplier = supplier; - this.holds = new HashMap<>(); - } - - // Returns a stream for use that may be cached from a previous call. Each call of getStream - // must be matched with a call of releaseStream. - public S getStream() { - int index = ThreadLocalRandom.current().nextInt(streams.size()); - S result; - S closeStream = null; - synchronized (this) { - StreamData streamData = streams.get(index); - if (streamData == null - || streamData.stream.startTime().isBefore(Instant.now().minus(streamTimeout))) { - if (streamData != null && --streamData.holds == 0) { - holds.remove(streamData.stream); - closeStream = streamData.stream; - } - streamData = new StreamData(); - streams.set(index, streamData); - holds.put(streamData.stream, streamData); - } - streamData.holds++; - result = streamData.stream; - } - if (closeStream != null) { - closeStream.close(); - } - return result; - } - - // Releases a stream that was obtained with getStream. - public void releaseStream(S stream) { - boolean closeStream = false; - synchronized (this) { - if (--holds.get(stream).holds == 0) { - closeStream = true; - holds.remove(stream); - } - } - if (closeStream) { - stream.close(); - } - } - } - /** Generic Exception type for implementors to use to represent errors while making RPCs. */ - public static class RpcException extends RuntimeException { - public RpcException() { - super(); - } - + public static final class RpcException extends RuntimeException { public RpcException(Throwable cause) { super(cause); } - - public RpcException(String message, Throwable cause) { - super(message, cause); - } } } diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillServiceAddress.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillServiceAddress.java new file mode 100644 index 0000000000000..3ebda8fab8edb --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillServiceAddress.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill; + +import com.google.auto.value.AutoOneOf; +import java.net.Inet6Address; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.net.HostAndPort; + +/** Used to create channels to communicate with Streaming Engine via gRpc. */ +@AutoOneOf(WindmillServiceAddress.Kind.class) +public abstract class WindmillServiceAddress { + public static WindmillServiceAddress create(Inet6Address ipv6Address) { + return AutoOneOf_WindmillServiceAddress.ipv6(ipv6Address); + } + + public static WindmillServiceAddress create(HostAndPort gcpServiceAddress) { + return AutoOneOf_WindmillServiceAddress.gcpServiceAddress(gcpServiceAddress); + } + + public abstract Kind getKind(); + + public abstract Inet6Address ipv6(); + + public abstract HostAndPort gcpServiceAddress(); + + public enum Kind { + IPV6, + GCP_SERVICE_ADDRESS + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillStream.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillStream.java new file mode 100644 index 0000000000000..4dd4164fc4efd --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillStream.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import javax.annotation.concurrent.ThreadSafe; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Instant; + +/** Superclass for streams returned by streaming Windmill methods. */ +@ThreadSafe +public interface WindmillStream { + /** Indicates that no more requests will be sent. */ + void close(); + + /** Waits for the server to close its end of the connection, with timeout. */ + boolean awaitTermination(int time, TimeUnit unit) throws InterruptedException; + + /** Returns when the stream was opened. */ + Instant startTime(); + + /** Handle representing a stream of GetWork responses. */ + @ThreadSafe + interface GetWorkStream extends WindmillStream { + /** Functional interface for receiving WorkItems. */ + @FunctionalInterface + interface WorkItemReceiver { + void receiveWork( + String computation, + @Nullable Instant inputDataWatermark, + @Nullable Instant synchronizedProcessingTime, + Windmill.WorkItem workItem, + Collection getWorkStreamLatencies); + } + } + + /** Interface for streaming GetDataRequests to Windmill. */ + @ThreadSafe + interface GetDataStream extends WindmillStream { + /** Issues a keyed GetData fetch, blocking until the result is ready. */ + Windmill.KeyedGetDataResponse requestKeyedData( + String computation, Windmill.KeyedGetDataRequest request); + + /** Issues a global GetData fetch, blocking until the result is ready. */ + Windmill.GlobalData requestGlobalData(Windmill.GlobalDataRequest request); + + /** Tells windmill processing is ongoing for the given keys. */ + void refreshActiveWork(Map> active); + } + + /** Interface for streaming CommitWorkRequests to Windmill. */ + @ThreadSafe + interface CommitWorkStream extends WindmillStream { + + /** + * Commits a work item and running onDone when the commit has been processed by the server. + * Returns true if the request was accepted. If false is returned the stream should be flushed + * and the request recommitted. + * + *

    onDone will be called with the status of the commit. + */ + boolean commitWorkItem( + String computation, + Windmill.WorkItemCommitRequest request, + Consumer onDone); + + /** Flushes any pending work items to the wire. */ + void flush(); + } + + /** Interface for streaming GetWorkerMetadata requests to Windmill. */ + @ThreadSafe + interface GetWorkerMetadataStream extends WindmillStream {} +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillStreamPool.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillStreamPool.java new file mode 100644 index 0000000000000..9cd4ab0ea4a5b --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillStreamPool.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; +import java.util.function.Supplier; +import javax.annotation.concurrent.GuardedBy; +import javax.annotation.concurrent.ThreadSafe; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Duration; +import org.joda.time.Instant; + +/** + * Pool of homogeneous streams to Windmill. + * + *

    The pool holds a fixed total number of streams, and keeps each stream open for a specified + * time to allow for better load-balancing. + */ +@ThreadSafe +public class WindmillStreamPool { + + private final Duration streamTimeout; + private final Supplier streamSupplier; + + /** @implNote Size of streams never changes once initialized. */ + private final List<@Nullable StreamData> streams; + + @GuardedBy("this") + private final Map> holds; + + private WindmillStreamPool( + Duration streamTimeout, + Supplier streamSupplier, + List<@Nullable StreamData> streams, + Map> holds) { + this.streams = streams; + this.streamTimeout = streamTimeout; + this.streamSupplier = streamSupplier; + this.holds = holds; + } + + public static WindmillStreamPool create( + int numStreams, Duration streamTimeout, Supplier streamSupplier) { + return new WindmillStreamPool<>( + streamTimeout, streamSupplier, newStreamList(numStreams), new HashMap<>()); + } + + @VisibleForTesting + static WindmillStreamPool forTesting( + Duration streamTimeout, + Supplier streamSupplier, + List<@Nullable StreamData> streamPool, + Map> holds) { + return new WindmillStreamPool<>(streamTimeout, streamSupplier, streamPool, holds); + } + + /** + * Creates a new list of streams of the given capacity with all values initialized to null. This + * is because we randomly load balance across all the streams in the pool. + */ + @VisibleForTesting + static List<@Nullable StreamData> newStreamList( + int numStreams) { + List<@Nullable StreamData> streamPool = new ArrayList<>(numStreams); + for (int i = 0; i < numStreams; i++) { + streamPool.add(null); + } + return streamPool; + } + + /** + * Returns a stream for use that may be cached from a previous call. Each call of getStream must + * be matched with a call of {@link WindmillStreamPool#releaseStream(WindmillStream)}. If the + * stream has been cached but has timed out and drained (no longer has any holds), the stream will + * be closed. + */ + public StreamT getStream() { + int index = streams.size() == 1 ? 0 : ThreadLocalRandom.current().nextInt(streams.size()); + // We will return this stream + StreamT resultStream; + StreamT closeThisStream = null; + try { + synchronized (this) { + WindmillStreamPool.StreamData existingStreamData = streams.get(index); + // There are 3 possible states that can result from fetching the stream from the cache. + if (existingStreamData == null) { + // 1. Stream doesn't exist create and cache a new one. + resultStream = createAndCacheStream(index).stream; + } else if (existingStreamData.hasTimedOut(streamTimeout)) { + // 2. The stream exists, but has timed out. The timed out stream is not returned (a new + // one is created and returned here) and evicted from the cache if the stream has + // completely drained. Every call to getStream(), is matched with a call to + // releaseStream(), so the stream will eventually drain and be closed. + if (--existingStreamData.holds == 0) { + holds.remove(existingStreamData.stream); + closeThisStream = existingStreamData.stream; + } + // Create and cache a new stream at the timed out stream's index. + resultStream = createAndCacheStream(index).stream; + } else { + // 3. The stream exists and is in a valid state. + existingStreamData.holds++; + resultStream = existingStreamData.stream; + } + } + return resultStream; + } finally { + if (closeThisStream != null) { + closeThisStream.close(); + } + } + } + + private synchronized WindmillStreamPool.StreamData createAndCacheStream(int cacheKey) { + WindmillStreamPool.StreamData newStreamData = + new WindmillStreamPool.StreamData<>(streamSupplier.get()); + newStreamData.holds++; + streams.set(cacheKey, newStreamData); + holds.put(newStreamData.stream, newStreamData); + return newStreamData; + } + + /** Releases a stream that was obtained with {@link WindmillStreamPool#getStream()}. */ + public void releaseStream(StreamT stream) { + boolean closeStream = false; + synchronized (this) { + StreamData streamData = holds.get(stream); + // All streams that are created by an instance of a pool will be present. + if (streamData == null) { + throw new IllegalStateException( + "Attempted to release stream that does not exist in this pool. This stream " + + "may not have been created by this pool or may have been released more " + + "times than acquired."); + } + if (--streamData.holds == 0) { + closeStream = true; + holds.remove(stream); + } + } + + if (closeStream) { + stream.close(); + } + } + + @VisibleForTesting + static final class StreamData { + final StreamT stream; + int holds; + + @VisibleForTesting + StreamData(StreamT stream) { + this.stream = stream; + holds = 1; + } + + private boolean hasTimedOut(Duration timeout) { + return stream.startTime().isBefore(Instant.now().minus(timeout)); + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/appliance/JniWindmillApplianceServer.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/appliance/JniWindmillApplianceServer.java new file mode 100644 index 0000000000000..8385c7cb59710 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/appliance/JniWindmillApplianceServer.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.appliance; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillServerBase; + +/** + * JNI Implementation of a {@link WindmillServerBase}. + * + * @implNote This is only for use in Streaming Appliance. + */ +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) +public class JniWindmillApplianceServer extends WindmillServerBase { + private static final String WINDMILL_SERVER_JNI_LIBRARY_PROPERTY = "windmill.jni_library"; + private static final String DEFAULT_SHUFFLE_CLIENT_LIBRARY = "libwindmill_service_jni.so"; + + static { + try { + // TODO: Remove the use of JNI here + File tempfile = File.createTempFile("libwindmill_service_jni", ".so"); + InputStream input = + ClassLoader.getSystemResourceAsStream( + System.getProperty( + WINDMILL_SERVER_JNI_LIBRARY_PROPERTY, DEFAULT_SHUFFLE_CLIENT_LIBRARY)); + Files.copy(input, tempfile.toPath(), StandardCopyOption.REPLACE_EXISTING); + System.load(tempfile.getAbsolutePath()); + } catch (IOException e) { + throw new RuntimeException("Loading windmill_service failed:", e); + } + } + + /** + * The host should be specified as protocol://address:port to connect to a windmill server through + * rpcz. + */ + public JniWindmillApplianceServer(String host) { + super(host); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/AppendableInputStream.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/AppendableInputStream.java new file mode 100644 index 0000000000000..dbd3613ee4c29 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/AppendableInputStream.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.grpcclient; + +import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.util.Enumeration; +import java.util.NoSuchElementException; +import java.util.concurrent.BlockingDeque; +import java.util.concurrent.CancellationException; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.Nullable; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.joda.time.Instant; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** An InputStream that can be dynamically extended with additional InputStreams. */ +@SuppressWarnings("JdkObsolete") +final class AppendableInputStream extends InputStream { + private static final Logger LOG = LoggerFactory.getLogger(AppendableInputStream.class); + private static final int QUEUE_MAX_CAPACITY = 10; + private static final InputStream POISON_PILL = ByteString.EMPTY.newInput(); + + private final AtomicBoolean cancelled; + private final AtomicBoolean complete; + private final AtomicLong blockedStartMs; + private final BlockingDeque queue; + private final InputStream stream; + + AppendableInputStream() { + this.cancelled = new AtomicBoolean(false); + this.complete = new AtomicBoolean(false); + this.blockedStartMs = new AtomicLong(); + this.queue = new LinkedBlockingDeque<>(QUEUE_MAX_CAPACITY); + this.stream = new SequenceInputStream(new InputStreamEnumeration()); + } + + long getBlockedStartMs() { + return blockedStartMs.get(); + } + + boolean isComplete() { + return complete.get(); + } + + boolean isCancelled() { + return cancelled.get(); + } + + int size() { + return queue.size(); + } + + /** Appends a new InputStream to the tail of this stream. */ + synchronized void append(InputStream chunk) { + try { + queue.put(chunk); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.debug("interrupted append"); + } + } + + /** Cancels the stream. Future calls to InputStream methods will throw CancellationException. */ + synchronized void cancel() { + cancelled.set(true); + try { + // Put the poison pill at the head of the queue to cancel as quickly as possible. + queue.clear(); + queue.putFirst(POISON_PILL); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.debug("interrupted cancel"); + } + } + + /** Signals that no new InputStreams will be added to this stream. */ + synchronized void complete() { + complete.set(true); + try { + queue.put(POISON_PILL); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.debug("interrupted complete"); + } + } + + @Override + public int read() throws IOException { + if (cancelled.get()) { + throw new CancellationException(); + } + return stream.read(); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (cancelled.get()) { + throw new CancellationException(); + } + return stream.read(b, off, len); + } + + @Override + public int available() throws IOException { + if (cancelled.get()) { + throw new CancellationException(); + } + return stream.available(); + } + + @Override + public void close() throws IOException { + stream.close(); + } + + @SuppressWarnings("NullableProblems") + private class InputStreamEnumeration implements Enumeration { + // The first stream is eagerly read on SequenceInputStream creation. For this reason + // we use an empty element as the first input to avoid blocking from the queue when + // creating the AppendableInputStream. + private @Nullable InputStream current = POISON_PILL; + + @Override + public boolean hasMoreElements() { + if (current != null) { + return true; + } + + try { + blockedStartMs.set(Instant.now().getMillis()); + current = queue.poll(180, TimeUnit.SECONDS); + if (current != null && current != POISON_PILL) { + return true; + } + if (cancelled.get()) { + throw new CancellationException(); + } + if (complete.get()) { + return false; + } + throw new IllegalStateException("Got poison pill or timeout but stream is not done."); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new CancellationException(); + } + } + + @SuppressWarnings("return") + @Override + public InputStream nextElement() { + if (!hasMoreElements()) { + throw new NoSuchElementException(); + } + blockedStartMs.set(0); + InputStream next = current; + current = null; + return next; + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GetWorkTimingInfosTracker.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GetWorkTimingInfosTracker.java new file mode 100644 index 0000000000000..e6710993af9b4 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GetWorkTimingInfosTracker.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.grpcclient; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetWorkStreamTimingInfo; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetWorkStreamTimingInfo.Event; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.LatencyAttribution; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.LatencyAttribution.State; +import org.joda.time.DateTimeUtils.MillisProvider; +import org.joda.time.Duration; +import org.joda.time.Instant; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class GetWorkTimingInfosTracker { + + private static final Logger LOG = LoggerFactory.getLogger(GetWorkTimingInfosTracker.class); + + private final Map aggregatedGetWorkStreamLatencies; + private final MillisProvider clock; + private Instant workItemCreationEndTime; + private Instant workItemLastChunkReceivedByWorkerTime; + private @Nullable LatencyAttribution workItemCreationLatency; + + GetWorkTimingInfosTracker(MillisProvider clock) { + this.aggregatedGetWorkStreamLatencies = new EnumMap<>(State.class); + this.clock = clock; + this.workItemCreationEndTime = Instant.EPOCH; + workItemLastChunkReceivedByWorkerTime = Instant.EPOCH; + workItemCreationLatency = null; + } + + public void addTimingInfo(Collection infos) { + // We want to record duration for each stage and also be reflective on total work item + // processing time. It can be tricky because timings of different + // StreamingGetWorkResponseChunks can be interleaved. Current strategy is to record the + // sum duration in each transmission stage across different chunks, then divide the total + // duration (start from the chunk creation end in the windmill worker to the end of last chunk + // reception by the user worker) proportionally according the sum duration values across the + // many stages, the final latency is also capped by the corresponding stage maximum latency + // seen across multiple chunks. This should allow us to identify the slow stage meanwhile + // avoid confusions for comparing the stage duration to the total processing elapsed wall + // time. + Map getWorkStreamTimings = new HashMap<>(); + for (GetWorkStreamTimingInfo info : infos) { + getWorkStreamTimings.putIfAbsent( + info.getEvent(), Instant.ofEpochMilli(info.getTimestampUsec() / 1000)); + } + + // Record the difference between starting to get work and the first chunk being sent as the + // work creation time. + Instant workItemCreationStart = getWorkStreamTimings.get(Event.GET_WORK_CREATION_START); + Instant workItemCreationEnd = getWorkStreamTimings.get(Event.GET_WORK_CREATION_END); + if (workItemCreationStart != null + && workItemCreationEnd != null + && workItemCreationLatency == null) { + workItemCreationLatency = + LatencyAttribution.newBuilder() + .setState(State.GET_WORK_IN_WINDMILL_WORKER) + .setTotalDurationMillis( + new Duration(workItemCreationStart, workItemCreationEnd).getMillis()) + .build(); + } + // Record the work item creation end time as the start of transmission stages. + if (workItemCreationEnd != null && workItemCreationEnd.isAfter(workItemCreationEndTime)) { + workItemCreationEndTime = workItemCreationEnd; + } + + // Record the latency of each chunk between send on worker and arrival on dispatcher. + Instant receivedByDispatcherTiming = + getWorkStreamTimings.get(Event.GET_WORK_RECEIVED_BY_DISPATCHER); + if (workItemCreationEnd != null && receivedByDispatcherTiming != null) { + Duration newDuration = new Duration(workItemCreationEnd, receivedByDispatcherTiming); + aggregatedGetWorkStreamLatencies.compute( + State.GET_WORK_IN_TRANSIT_TO_DISPATCHER, + (stateKey, duration) -> { + if (duration == null) { + return new SumAndMaxDurations(newDuration, newDuration); + } + duration.max = newDuration.isLongerThan(duration.max) ? newDuration : duration.max; + duration.sum = duration.sum.plus(newDuration); + return duration; + }); + } + + // Record the latency of each chunk between send on dispatcher and arrival on worker. + Instant forwardedByDispatcherTiming = + getWorkStreamTimings.get(Event.GET_WORK_FORWARDED_BY_DISPATCHER); + Instant now = Instant.ofEpochMilli(clock.getMillis()); + if (forwardedByDispatcherTiming != null) { + Duration newDuration = new Duration(forwardedByDispatcherTiming, now); + aggregatedGetWorkStreamLatencies.compute( + State.GET_WORK_IN_TRANSIT_TO_USER_WORKER, + (stateKey, duration) -> { + if (duration == null) { + return new SumAndMaxDurations(newDuration, newDuration); + } + duration.max = newDuration.isLongerThan(duration.max) ? newDuration : duration.max; + duration.sum = duration.sum.plus(newDuration); + return duration; + }); + } + workItemLastChunkReceivedByWorkerTime = now; + } + + List getLatencyAttributions() { + if (workItemCreationLatency == null && aggregatedGetWorkStreamLatencies.isEmpty()) { + return Collections.emptyList(); + } + List latencyAttributions = + new ArrayList<>(aggregatedGetWorkStreamLatencies.size() + 1); + if (workItemCreationLatency != null) { + latencyAttributions.add(workItemCreationLatency); + } + if (workItemCreationEndTime.isAfter(workItemLastChunkReceivedByWorkerTime)) { + LOG.warn( + "Work item creation time {} is after the work received time {}, " + + "one or more GetWorkStream timing infos are missing.", + workItemCreationEndTime, + workItemLastChunkReceivedByWorkerTime); + return latencyAttributions; + } + long totalTransmissionDurationElapsedTime = + new Duration(workItemCreationEndTime, workItemLastChunkReceivedByWorkerTime).getMillis(); + long totalSumDurationTimeMills = 0; + for (SumAndMaxDurations duration : aggregatedGetWorkStreamLatencies.values()) { + totalSumDurationTimeMills += duration.sum.getMillis(); + } + final long finalTotalSumDurationTimeMills = totalSumDurationTimeMills; + + aggregatedGetWorkStreamLatencies.forEach( + (state, duration) -> { + long scaledDuration = + (long) + (((double) duration.sum.getMillis() / finalTotalSumDurationTimeMills) + * totalTransmissionDurationElapsedTime); + // Cap final duration by the max state duration across different chunks. This ensures + // the sum of final durations does not exceed the total elapsed time and the duration + // for each stage does not exceed the stage maximum. + long durationMills = Math.min(duration.max.getMillis(), scaledDuration); + latencyAttributions.add( + LatencyAttribution.newBuilder() + .setState(state) + .setTotalDurationMillis(durationMills) + .build()); + }); + return latencyAttributions; + } + + public void reset() { + this.aggregatedGetWorkStreamLatencies.clear(); + this.workItemCreationEndTime = Instant.EPOCH; + this.workItemLastChunkReceivedByWorkerTime = Instant.EPOCH; + this.workItemCreationLatency = null; + } + + private static class SumAndMaxDurations { + + private Duration sum; + private Duration max; + + public SumAndMaxDurations(Duration sum, Duration max) { + this.sum = sum; + this.max = max; + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcCommitWorkStream.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcCommitWorkStream.java new file mode 100644 index 0000000000000..1bba40805dec4 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcCommitWorkStream.java @@ -0,0 +1,335 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.grpcclient; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; + +import java.io.PrintWriter; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; +import java.util.function.Function; +import org.apache.beam.runners.dataflow.worker.windmill.AbstractWindmillStream; +import org.apache.beam.runners.dataflow.worker.windmill.StreamObserverFactory; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.CommitStatus; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.JobHeader; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingCommitRequestChunk; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingCommitResponse; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingCommitWorkRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.WorkItemCommitRequest; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.CommitWorkStream; +import org.apache.beam.sdk.util.BackOff; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +final class GrpcCommitWorkStream + extends AbstractWindmillStream + implements CommitWorkStream { + private static final Logger LOG = LoggerFactory.getLogger(GrpcCommitWorkStream.class); + + private static final long HEARTBEAT_REQUEST_ID = Long.MAX_VALUE; + + private final Map pending; + private final Batcher batcher; + private final AtomicLong idGenerator; + private final JobHeader jobHeader; + private final ThrottleTimer commitWorkThrottleTimer; + private final int streamingRpcBatchLimit; + + private GrpcCommitWorkStream( + Function, StreamObserver> + startCommitWorkRpcFn, + BackOff backoff, + StreamObserverFactory streamObserverFactory, + Set> streamRegistry, + int logEveryNStreamFailures, + ThrottleTimer commitWorkThrottleTimer, + JobHeader jobHeader, + AtomicLong idGenerator, + int streamingRpcBatchLimit) { + super( + startCommitWorkRpcFn, + backoff, + streamObserverFactory, + streamRegistry, + logEveryNStreamFailures); + pending = new ConcurrentHashMap<>(); + batcher = new Batcher(); + this.idGenerator = idGenerator; + this.jobHeader = jobHeader; + this.commitWorkThrottleTimer = commitWorkThrottleTimer; + this.streamingRpcBatchLimit = streamingRpcBatchLimit; + } + + static GrpcCommitWorkStream create( + Function, StreamObserver> + startCommitWorkRpcFn, + BackOff backoff, + StreamObserverFactory streamObserverFactory, + Set> streamRegistry, + int logEveryNStreamFailures, + ThrottleTimer commitWorkThrottleTimer, + JobHeader jobHeader, + AtomicLong idGenerator, + int streamingRpcBatchLimit) { + GrpcCommitWorkStream commitWorkStream = + new GrpcCommitWorkStream( + startCommitWorkRpcFn, + backoff, + streamObserverFactory, + streamRegistry, + logEveryNStreamFailures, + commitWorkThrottleTimer, + jobHeader, + idGenerator, + streamingRpcBatchLimit); + commitWorkStream.startStream(); + return commitWorkStream; + } + + @Override + public void appendSpecificHtml(PrintWriter writer) { + writer.format("CommitWorkStream: %d pending", pending.size()); + } + + @Override + protected synchronized void onNewStream() { + send(StreamingCommitWorkRequest.newBuilder().setHeader(jobHeader).build()); + Batcher resendBatcher = new Batcher(); + for (Map.Entry entry : pending.entrySet()) { + if (!resendBatcher.canAccept(entry.getValue())) { + resendBatcher.flush(); + } + resendBatcher.add(entry.getKey(), entry.getValue()); + } + resendBatcher.flush(); + } + + @Override + protected boolean hasPendingRequests() { + return !pending.isEmpty(); + } + + @Override + public void sendHealthCheck() { + if (hasPendingRequests()) { + StreamingCommitWorkRequest.Builder builder = StreamingCommitWorkRequest.newBuilder(); + builder.addCommitChunkBuilder().setRequestId(HEARTBEAT_REQUEST_ID); + send(builder.build()); + } + } + + @Override + protected void onResponse(StreamingCommitResponse response) { + commitWorkThrottleTimer.stop(); + + RuntimeException finalException = null; + for (int i = 0; i < response.getRequestIdCount(); ++i) { + long requestId = response.getRequestId(i); + if (requestId == HEARTBEAT_REQUEST_ID) { + continue; + } + PendingRequest done = pending.remove(requestId); + if (done == null) { + LOG.error("Got unknown commit request ID: {}", requestId); + } else { + try { + done.onDone.accept( + (i < response.getStatusCount()) ? response.getStatus(i) : CommitStatus.OK); + } catch (RuntimeException e) { + // Catch possible exceptions to ensure that an exception for one commit does not prevent + // other commits from being processed. + LOG.warn("Exception while processing commit response.", e); + finalException = e; + } + } + } + if (finalException != null) { + throw finalException; + } + } + + @Override + protected void startThrottleTimer() { + commitWorkThrottleTimer.start(); + } + + @Override + public boolean commitWorkItem( + String computation, WorkItemCommitRequest commitRequest, Consumer onDone) { + PendingRequest request = new PendingRequest(computation, commitRequest, onDone); + if (!batcher.canAccept(request)) { + return false; + } + batcher.add(idGenerator.incrementAndGet(), request); + return true; + } + + @Override + public void flush() { + batcher.flush(); + } + + private void flushInternal(Map requests) { + if (requests.isEmpty()) { + return; + } + if (requests.size() == 1) { + Map.Entry elem = requests.entrySet().iterator().next(); + if (elem.getValue().request.getSerializedSize() + > AbstractWindmillStream.RPC_STREAM_CHUNK_SIZE) { + issueMultiChunkRequest(elem.getKey(), elem.getValue()); + } else { + issueSingleRequest(elem.getKey(), elem.getValue()); + } + } else { + issueBatchedRequest(requests); + } + } + + private void issueSingleRequest(final long id, PendingRequest pendingRequest) { + StreamingCommitWorkRequest.Builder requestBuilder = StreamingCommitWorkRequest.newBuilder(); + requestBuilder + .addCommitChunkBuilder() + .setComputationId(pendingRequest.computation) + .setRequestId(id) + .setShardingKey(pendingRequest.request.getShardingKey()) + .setSerializedWorkItemCommit(pendingRequest.request.toByteString()); + StreamingCommitWorkRequest chunk = requestBuilder.build(); + synchronized (this) { + pending.put(id, pendingRequest); + try { + send(chunk); + } catch (IllegalStateException e) { + // Stream was broken, request will be retried when stream is reopened. + } + } + } + + private void issueBatchedRequest(Map requests) { + StreamingCommitWorkRequest.Builder requestBuilder = StreamingCommitWorkRequest.newBuilder(); + String lastComputation = null; + for (Map.Entry entry : requests.entrySet()) { + PendingRequest request = entry.getValue(); + StreamingCommitRequestChunk.Builder chunkBuilder = requestBuilder.addCommitChunkBuilder(); + if (lastComputation == null || !lastComputation.equals(request.computation)) { + chunkBuilder.setComputationId(request.computation); + lastComputation = request.computation; + } + chunkBuilder.setRequestId(entry.getKey()); + chunkBuilder.setShardingKey(request.request.getShardingKey()); + chunkBuilder.setSerializedWorkItemCommit(request.request.toByteString()); + } + StreamingCommitWorkRequest request = requestBuilder.build(); + synchronized (this) { + pending.putAll(requests); + try { + send(request); + } catch (IllegalStateException e) { + // Stream was broken, request will be retried when stream is reopened. + } + } + } + + private void issueMultiChunkRequest(final long id, PendingRequest pendingRequest) { + checkNotNull(pendingRequest.computation); + final ByteString serializedCommit = pendingRequest.request.toByteString(); + + synchronized (this) { + pending.put(id, pendingRequest); + for (int i = 0; + i < serializedCommit.size(); + i += AbstractWindmillStream.RPC_STREAM_CHUNK_SIZE) { + int end = i + AbstractWindmillStream.RPC_STREAM_CHUNK_SIZE; + ByteString chunk = serializedCommit.substring(i, Math.min(end, serializedCommit.size())); + + StreamingCommitRequestChunk.Builder chunkBuilder = + StreamingCommitRequestChunk.newBuilder() + .setRequestId(id) + .setSerializedWorkItemCommit(chunk) + .setComputationId(pendingRequest.computation) + .setShardingKey(pendingRequest.request.getShardingKey()); + int remaining = serializedCommit.size() - end; + if (remaining > 0) { + chunkBuilder.setRemainingBytesForWorkItem(remaining); + } + + StreamingCommitWorkRequest requestChunk = + StreamingCommitWorkRequest.newBuilder().addCommitChunk(chunkBuilder).build(); + try { + send(requestChunk); + } catch (IllegalStateException e) { + // Stream was broken, request will be retried when stream is reopened. + break; + } + } + } + } + + private static class PendingRequest { + + private final String computation; + private final WorkItemCommitRequest request; + private final Consumer onDone; + + PendingRequest( + String computation, WorkItemCommitRequest request, Consumer onDone) { + this.computation = computation; + this.request = request; + this.onDone = onDone; + } + + long getBytes() { + return (long) request.getSerializedSize() + computation.length(); + } + } + + private class Batcher { + + private final Map queue; + private long queuedBytes; + + private Batcher() { + this.queuedBytes = 0; + this.queue = new HashMap<>(); + } + + boolean canAccept(PendingRequest request) { + return queue.isEmpty() + || (queue.size() < streamingRpcBatchLimit + && (request.getBytes() + queuedBytes) < AbstractWindmillStream.RPC_STREAM_CHUNK_SIZE); + } + + void add(long id, PendingRequest request) { + assert (canAccept(request)); + queuedBytes += request.getBytes(); + queue.put(id, request); + } + + void flush() { + flushInternal(queue); + queuedBytes = 0; + queue.clear(); + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcDeadlineClientInterceptor.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcDeadlineClientInterceptor.java new file mode 100644 index 0000000000000..6b0e19cbb4802 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcDeadlineClientInterceptor.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.grpcclient; + +import java.util.concurrent.TimeUnit; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.CallOptions; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Channel; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ClientCall; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ClientInterceptor; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.MethodDescriptor; + +/** + * Intercepts outgoing calls and attaches a relative deadline to the call. Deadlines are absolute, + * so they must be created and attached with every RPC call. The alternative to registering an + * {@link ClientInterceptor} would be to remember to call stub.withDeadlineAfter(...) at every call + * site. For more context see grpc-java/issues/1495 and grpc-java/issues/4305. + * + * @see Official gRPC Deadline Documentation. + */ +class GrpcDeadlineClientInterceptor implements ClientInterceptor { + private static final int DEFAULT_UNARY_RPC_DEADLINE_SECONDS = 10; + private static final TimeUnit DEFAULT_TIME_UNIT = TimeUnit.SECONDS; + + private final int duration; + private final TimeUnit timeUnit; + + GrpcDeadlineClientInterceptor(int duration, TimeUnit timeUnit) { + this.duration = duration; + this.timeUnit = timeUnit; + } + + static GrpcDeadlineClientInterceptor withDefaultUnaryRpcDeadline() { + return new GrpcDeadlineClientInterceptor(DEFAULT_UNARY_RPC_DEADLINE_SECONDS, DEFAULT_TIME_UNIT); + } + + @Override + public ClientCall interceptCall( + MethodDescriptor methodDescriptor, CallOptions callOptions, Channel next) { + return next.newCall(methodDescriptor, callOptions.withDeadlineAfter(duration, timeUnit)); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcGetDataStream.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcGetDataStream.java new file mode 100644 index 0000000000000..238cc771dce8b --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcGetDataStream.java @@ -0,0 +1,333 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.grpcclient; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Verify.verify; + +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintWriter; +import java.util.Deque; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; +import org.apache.beam.runners.dataflow.worker.windmill.AbstractWindmillStream; +import org.apache.beam.runners.dataflow.worker.windmill.StreamObserverFactory; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.ComputationGetDataRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GlobalData; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GlobalDataRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.JobHeader; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataResponse; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingGetDataRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingGetDataResponse; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetDataStream; +import org.apache.beam.runners.dataflow.worker.windmill.grpcclient.GrpcGetDataStreamRequests.QueuedBatch; +import org.apache.beam.runners.dataflow.worker.windmill.grpcclient.GrpcGetDataStreamRequests.QueuedRequest; +import org.apache.beam.sdk.util.BackOff; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; +import org.joda.time.Instant; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +final class GrpcGetDataStream + extends AbstractWindmillStream + implements GetDataStream { + private static final Logger LOG = LoggerFactory.getLogger(GrpcGetDataStream.class); + + private final Deque batches; + private final Map pending; + private final AtomicLong idGenerator; + private final ThrottleTimer getDataThrottleTimer; + private final JobHeader jobHeader; + private final int streamingRpcBatchLimit; + + private GrpcGetDataStream( + Function, StreamObserver> + startGetDataRpcFn, + BackOff backoff, + StreamObserverFactory streamObserverFactory, + Set> streamRegistry, + int logEveryNStreamFailures, + ThrottleTimer getDataThrottleTimer, + JobHeader jobHeader, + AtomicLong idGenerator, + int streamingRpcBatchLimit) { + super( + startGetDataRpcFn, backoff, streamObserverFactory, streamRegistry, logEveryNStreamFailures); + this.idGenerator = idGenerator; + this.getDataThrottleTimer = getDataThrottleTimer; + this.jobHeader = jobHeader; + this.streamingRpcBatchLimit = streamingRpcBatchLimit; + this.batches = new ConcurrentLinkedDeque<>(); + this.pending = new ConcurrentHashMap<>(); + } + + static GrpcGetDataStream create( + Function, StreamObserver> + startGetDataRpcFn, + BackOff backoff, + StreamObserverFactory streamObserverFactory, + Set> streamRegistry, + int logEveryNStreamFailures, + ThrottleTimer getDataThrottleTimer, + JobHeader jobHeader, + AtomicLong idGenerator, + int streamingRpcBatchLimit) { + GrpcGetDataStream getDataStream = + new GrpcGetDataStream( + startGetDataRpcFn, + backoff, + streamObserverFactory, + streamRegistry, + logEveryNStreamFailures, + getDataThrottleTimer, + jobHeader, + idGenerator, + streamingRpcBatchLimit); + getDataStream.startStream(); + return getDataStream; + } + + @Override + protected synchronized void onNewStream() { + send(StreamingGetDataRequest.newBuilder().setHeader(jobHeader).build()); + if (clientClosed.get()) { + // We rely on close only occurring after all methods on the stream have returned. + // Since the requestKeyedData and requestGlobalData methods are blocking this + // means there should be no pending requests. + verify(!hasPendingRequests()); + } else { + for (AppendableInputStream responseStream : pending.values()) { + responseStream.cancel(); + } + } + } + + @Override + protected boolean hasPendingRequests() { + return !pending.isEmpty() || !batches.isEmpty(); + } + + @Override + @SuppressWarnings("dereference.of.nullable") + protected void onResponse(StreamingGetDataResponse chunk) { + checkArgument(chunk.getRequestIdCount() == chunk.getSerializedResponseCount()); + checkArgument(chunk.getRemainingBytesForResponse() == 0 || chunk.getRequestIdCount() == 1); + getDataThrottleTimer.stop(); + + for (int i = 0; i < chunk.getRequestIdCount(); ++i) { + AppendableInputStream responseStream = pending.get(chunk.getRequestId(i)); + verify(responseStream != null, "No pending response stream"); + responseStream.append(chunk.getSerializedResponse(i).newInput()); + if (chunk.getRemainingBytesForResponse() == 0) { + responseStream.complete(); + } + } + } + + @Override + protected void startThrottleTimer() { + getDataThrottleTimer.start(); + } + + private long uniqueId() { + return idGenerator.incrementAndGet(); + } + + @Override + public KeyedGetDataResponse requestKeyedData(String computation, KeyedGetDataRequest request) { + return issueRequest( + QueuedRequest.forComputation(uniqueId(), computation, request), + KeyedGetDataResponse::parseFrom); + } + + @Override + public GlobalData requestGlobalData(GlobalDataRequest request) { + return issueRequest(QueuedRequest.global(uniqueId(), request), GlobalData::parseFrom); + } + + @Override + public void refreshActiveWork(Map> active) { + long builderBytes = 0; + StreamingGetDataRequest.Builder builder = StreamingGetDataRequest.newBuilder(); + for (Map.Entry> entry : active.entrySet()) { + for (KeyedGetDataRequest request : entry.getValue()) { + // Calculate the bytes with some overhead for proto encoding. + long bytes = (long) entry.getKey().length() + request.getSerializedSize() + 10; + if (builderBytes > 0 + && (builderBytes + bytes > AbstractWindmillStream.RPC_STREAM_CHUNK_SIZE + || builder.getRequestIdCount() >= streamingRpcBatchLimit)) { + send(builder.build()); + builderBytes = 0; + builder.clear(); + } + builderBytes += bytes; + builder.addStateRequest( + ComputationGetDataRequest.newBuilder() + .setComputationId(entry.getKey()) + .addRequests(request)); + } + } + if (builderBytes > 0) { + send(builder.build()); + } + } + + @Override + public void sendHealthCheck() { + if (hasPendingRequests()) { + send(StreamingGetDataRequest.newBuilder().build()); + } + } + + @Override + public void appendSpecificHtml(PrintWriter writer) { + writer.format( + "GetDataStream: %d queued batches, %d pending requests [", batches.size(), pending.size()); + for (Map.Entry entry : pending.entrySet()) { + writer.format("Stream %d ", entry.getKey()); + if (entry.getValue().isCancelled()) { + writer.append("cancelled "); + } + if (entry.getValue().isComplete()) { + writer.append("complete "); + } + int queueSize = entry.getValue().size(); + if (queueSize > 0) { + writer.format("%d queued responses ", queueSize); + } + long blockedMs = entry.getValue().getBlockedStartMs(); + if (blockedMs > 0) { + writer.format("blocked for %dms", Instant.now().getMillis() - blockedMs); + } + } + writer.append("]"); + } + + private ResponseT issueRequest(QueuedRequest request, ParseFn parseFn) { + while (true) { + request.resetResponseStream(); + try { + queueRequestAndWait(request); + return parseFn.parse(request.getResponseStream()); + } catch (CancellationException e) { + // Retry issuing the request since the response stream was cancelled. + continue; + } catch (IOException e) { + LOG.error("Parsing GetData response failed: ", e); + continue; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } finally { + pending.remove(request.id()); + } + } + } + + private void queueRequestAndWait(QueuedRequest request) throws InterruptedException { + QueuedBatch batch; + boolean responsibleForSend = false; + CountDownLatch waitForSendLatch = null; + synchronized (batches) { + batch = batches.isEmpty() ? null : batches.getLast(); + if (batch == null + || batch.isFinalized() + || batch.requests().size() >= streamingRpcBatchLimit + || batch.byteSize() + request.byteSize() > AbstractWindmillStream.RPC_STREAM_CHUNK_SIZE) { + if (batch != null) { + waitForSendLatch = batch.getLatch(); + } + batch = new QueuedBatch(); + batches.addLast(batch); + responsibleForSend = true; + } + batch.addRequest(request); + } + if (responsibleForSend) { + if (waitForSendLatch == null) { + // If there was not a previous batch wait a little while to improve + // batching. + Thread.sleep(1); + } else { + waitForSendLatch.await(); + } + // Finalize the batch so that no additional requests will be added. Leave the batch in the + // queue so that a subsequent batch will wait for it's completion. + synchronized (batches) { + verify(batch == batches.peekFirst()); + batch.markFinalized(); + } + sendBatch(batch.requests()); + synchronized (batches) { + verify(batch == batches.pollFirst()); + } + // Notify all waiters with requests in this batch as well as the sender + // of the next batch (if one exists). + batch.countDown(); + } else { + // Wait for this batch to be sent before parsing the response. + batch.await(); + } + } + + @SuppressWarnings("NullableProblems") + private void sendBatch(List requests) { + StreamingGetDataRequest batchedRequest = flushToBatch(requests); + synchronized (this) { + // Synchronization of pending inserts is necessary with send to ensure duplicates are not + // sent on stream reconnect. + for (QueuedRequest request : requests) { + // Map#put returns null if there was no previous mapping for the key, meaning we have not + // seen it before. + verify(pending.put(request.id(), request.getResponseStream()) == null); + } + try { + send(batchedRequest); + } catch (IllegalStateException e) { + // The stream broke before this call went through; onNewStream will retry the fetch. + LOG.warn("GetData stream broke before call started.", e); + } + } + } + + @SuppressWarnings("argument") + private StreamingGetDataRequest flushToBatch(List requests) { + // Put all global data requests first because there is only a single repeated field for + // request ids and the initial ids correspond to global data requests if they are present. + requests.sort(QueuedRequest.globalRequestsFirst()); + StreamingGetDataRequest.Builder builder = StreamingGetDataRequest.newBuilder(); + for (QueuedRequest request : requests) { + request.addToStreamingGetDataRequest(builder); + } + return builder.build(); + } + + @FunctionalInterface + private interface ParseFn { + ResponseT parse(InputStream input) throws IOException; + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcGetDataStreamRequests.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcGetDataStreamRequests.java new file mode 100644 index 0000000000000..7da7b13958b9b --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcGetDataStreamRequests.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.grpcclient; + +import com.google.auto.value.AutoOneOf; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.ComputationGetDataRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GlobalDataRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataRequest; + +/** Utility data classes for {@link GrpcGetDataStream}. */ +final class GrpcGetDataStreamRequests { + private GrpcGetDataStreamRequests() {} + + static class QueuedRequest { + private final long id; + private final ComputationOrGlobalDataRequest dataRequest; + private AppendableInputStream responseStream; + + private QueuedRequest(long id, ComputationOrGlobalDataRequest dataRequest) { + this.id = id; + this.dataRequest = dataRequest; + responseStream = new AppendableInputStream(); + } + + static QueuedRequest forComputation( + long id, String computation, KeyedGetDataRequest keyedGetDataRequest) { + ComputationGetDataRequest computationGetDataRequest = + ComputationGetDataRequest.newBuilder() + .setComputationId(computation) + .addRequests(keyedGetDataRequest) + .build(); + return new QueuedRequest( + id, ComputationOrGlobalDataRequest.computation(computationGetDataRequest)); + } + + static QueuedRequest global(long id, GlobalDataRequest globalDataRequest) { + return new QueuedRequest(id, ComputationOrGlobalDataRequest.global(globalDataRequest)); + } + + static Comparator globalRequestsFirst() { + return (QueuedRequest r1, QueuedRequest r2) -> { + boolean r1gd = r1.dataRequest.isGlobal(); + boolean r2gd = r2.dataRequest.isGlobal(); + return r1gd == r2gd ? 0 : (r1gd ? -1 : 1); + }; + } + + long id() { + return id; + } + + long byteSize() { + return dataRequest.serializedSize(); + } + + AppendableInputStream getResponseStream() { + return responseStream; + } + + void resetResponseStream() { + this.responseStream = new AppendableInputStream(); + } + + void addToStreamingGetDataRequest(Windmill.StreamingGetDataRequest.Builder builder) { + builder.addRequestId(id); + if (dataRequest.isForComputation()) { + builder.addStateRequest(dataRequest.computation()); + } else { + builder.addGlobalDataRequest(dataRequest.global()); + } + } + } + + static class QueuedBatch { + private final List requests = new ArrayList<>(); + private final CountDownLatch sent = new CountDownLatch(1); + private long byteSize = 0; + private boolean finalized = false; + + CountDownLatch getLatch() { + return sent; + } + + List requests() { + return requests; + } + + long byteSize() { + return byteSize; + } + + boolean isFinalized() { + return finalized; + } + + void markFinalized() { + finalized = true; + } + + void addRequest(QueuedRequest request) { + requests.add(request); + byteSize += request.byteSize(); + } + + void countDown() { + sent.countDown(); + } + + void await() throws InterruptedException { + sent.await(); + } + } + + @AutoOneOf(ComputationOrGlobalDataRequest.Kind.class) + abstract static class ComputationOrGlobalDataRequest { + static ComputationOrGlobalDataRequest computation( + ComputationGetDataRequest computationGetDataRequest) { + return AutoOneOf_GrpcGetDataStreamRequests_ComputationOrGlobalDataRequest.computation( + computationGetDataRequest); + } + + static ComputationOrGlobalDataRequest global(GlobalDataRequest globalDataRequest) { + return AutoOneOf_GrpcGetDataStreamRequests_ComputationOrGlobalDataRequest.global( + globalDataRequest); + } + + abstract Kind getKind(); + + abstract ComputationGetDataRequest computation(); + + abstract GlobalDataRequest global(); + + boolean isGlobal() { + return getKind() == Kind.GLOBAL; + } + + boolean isForComputation() { + return getKind() == Kind.COMPUTATION; + } + + long serializedSize() { + switch (getKind()) { + case GLOBAL: + return global().getSerializedSize(); + case COMPUTATION: + return computation().getSerializedSize(); + // this will never happen since the switch is exhaustive. + default: + throw new UnsupportedOperationException("unknown dataRequest type."); + } + } + + enum Kind { + COMPUTATION, + GLOBAL + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcGetWorkStream.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcGetWorkStream.java new file mode 100644 index 0000000000000..4660fe25b13b3 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcGetWorkStream.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.grpcclient; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; +import javax.annotation.Nullable; +import org.apache.beam.runners.dataflow.worker.WindmillTimeUtils; +import org.apache.beam.runners.dataflow.worker.windmill.AbstractWindmillStream; +import org.apache.beam.runners.dataflow.worker.windmill.StreamObserverFactory; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetWorkRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.LatencyAttribution; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingGetWorkRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingGetWorkRequestExtension; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingGetWorkResponseChunk; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetWorkStream; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetWorkStream.WorkItemReceiver; +import org.apache.beam.sdk.util.BackOff; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; +import org.joda.time.Instant; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +final class GrpcGetWorkStream + extends AbstractWindmillStream + implements GetWorkStream { + + private static final Logger LOG = LoggerFactory.getLogger(GrpcGetWorkStream.class); + + private final GetWorkRequest request; + private final WorkItemReceiver receiver; + private final ThrottleTimer getWorkThrottleTimer; + private final Map buffers; + private final AtomicLong inflightMessages; + private final AtomicLong inflightBytes; + + private GrpcGetWorkStream( + Function< + StreamObserver, + StreamObserver> + startGetWorkRpcFn, + GetWorkRequest request, + BackOff backoff, + StreamObserverFactory streamObserverFactory, + Set> streamRegistry, + int logEveryNStreamFailures, + ThrottleTimer getWorkThrottleTimer, + WorkItemReceiver receiver) { + super( + startGetWorkRpcFn, backoff, streamObserverFactory, streamRegistry, logEveryNStreamFailures); + this.request = request; + this.getWorkThrottleTimer = getWorkThrottleTimer; + this.receiver = receiver; + this.buffers = new ConcurrentHashMap<>(); + this.inflightMessages = new AtomicLong(); + this.inflightBytes = new AtomicLong(); + } + + static GrpcGetWorkStream create( + Function< + StreamObserver, + StreamObserver> + startGetWorkRpcFn, + GetWorkRequest request, + BackOff backoff, + StreamObserverFactory streamObserverFactory, + Set> streamRegistry, + int logEveryNStreamFailures, + ThrottleTimer getWorkThrottleTimer, + WorkItemReceiver receiver) { + GrpcGetWorkStream getWorkStream = + new GrpcGetWorkStream( + startGetWorkRpcFn, + request, + backoff, + streamObserverFactory, + streamRegistry, + logEveryNStreamFailures, + getWorkThrottleTimer, + receiver); + getWorkStream.startStream(); + return getWorkStream; + } + + private void sendRequestExtension(long moreItems, long moreBytes) { + final StreamingGetWorkRequest extension = + StreamingGetWorkRequest.newBuilder() + .setRequestExtension( + StreamingGetWorkRequestExtension.newBuilder() + .setMaxItems(moreItems) + .setMaxBytes(moreBytes)) + .build(); + + executor() + .execute( + () -> { + try { + send(extension); + } catch (IllegalStateException e) { + // Stream was closed. + } + }); + } + + @Override + protected synchronized void onNewStream() { + buffers.clear(); + inflightMessages.set(request.getMaxItems()); + inflightBytes.set(request.getMaxBytes()); + send(StreamingGetWorkRequest.newBuilder().setRequest(request).build()); + } + + @Override + protected boolean hasPendingRequests() { + return false; + } + + @Override + public void appendSpecificHtml(PrintWriter writer) { + // Number of buffers is same as distinct workers that sent work on this stream. + writer.format( + "GetWorkStream: %d buffers, %d inflight messages allowed, %d inflight bytes allowed", + buffers.size(), inflightMessages.intValue(), inflightBytes.intValue()); + } + + @Override + public void sendHealthCheck() { + send( + StreamingGetWorkRequest.newBuilder() + .setRequestExtension( + StreamingGetWorkRequestExtension.newBuilder().setMaxItems(0).setMaxBytes(0).build()) + .build()); + } + + @Override + protected void onResponse(StreamingGetWorkResponseChunk chunk) { + getWorkThrottleTimer.stop(); + + GrpcGetWorkStream.WorkItemBuffer buffer = + buffers.computeIfAbsent( + chunk.getStreamId(), unused -> new GrpcGetWorkStream.WorkItemBuffer()); + buffer.append(chunk); + + if (chunk.getRemainingBytesForWorkItem() == 0) { + long size = buffer.bufferedSize(); + buffer.runAndReset(); + + // Record the fact that there are now fewer outstanding messages and bytes on the stream. + long numInflight = inflightMessages.decrementAndGet(); + long bytesInflight = inflightBytes.addAndGet(-size); + + // If the outstanding items or bytes limit has gotten too low, top both off with a + // GetWorkExtension. The goal is to keep the limits relatively close to their maximum + // values without sending too many extension requests. + if (numInflight < request.getMaxItems() / 2 || bytesInflight < request.getMaxBytes() / 2) { + long moreItems = request.getMaxItems() - numInflight; + long moreBytes = request.getMaxBytes() - bytesInflight; + inflightMessages.getAndAdd(moreItems); + inflightBytes.getAndAdd(moreBytes); + sendRequestExtension(moreItems, moreBytes); + } + } + } + + @Override + protected void startThrottleTimer() { + getWorkThrottleTimer.start(); + } + + private class WorkItemBuffer { + private final GetWorkTimingInfosTracker workTimingInfosTracker; + private String computation; + @Nullable private Instant inputDataWatermark; + @Nullable private Instant synchronizedProcessingTime; + private ByteString data; + private long bufferedSize; + + @SuppressWarnings("initialization.fields.uninitialized") + WorkItemBuffer() { + workTimingInfosTracker = new GetWorkTimingInfosTracker(System::currentTimeMillis); + data = ByteString.EMPTY; + bufferedSize = 0; + } + + @SuppressWarnings("NullableProblems") + private void setMetadata(Windmill.ComputationWorkItemMetadata metadata) { + this.computation = metadata.getComputationId(); + this.inputDataWatermark = + WindmillTimeUtils.windmillToHarnessWatermark(metadata.getInputDataWatermark()); + this.synchronizedProcessingTime = + WindmillTimeUtils.windmillToHarnessWatermark( + metadata.getDependentRealtimeInputWatermark()); + } + + private void append(StreamingGetWorkResponseChunk chunk) { + if (chunk.hasComputationMetadata()) { + setMetadata(chunk.getComputationMetadata()); + } + + this.data = data.concat(chunk.getSerializedWorkItem()); + this.bufferedSize += chunk.getSerializedWorkItem().size(); + workTimingInfosTracker.addTimingInfo(chunk.getPerWorkItemTimingInfosList()); + } + + private long bufferedSize() { + return bufferedSize; + } + + private void runAndReset() { + try { + Windmill.WorkItem workItem = Windmill.WorkItem.parseFrom(data.newInput()); + List getWorkStreamLatencies = + workTimingInfosTracker.getLatencyAttributions(); + receiver.receiveWork( + computation, + inputDataWatermark, + synchronizedProcessingTime, + workItem, + getWorkStreamLatencies); + } catch (IOException e) { + LOG.error("Failed to parse work item from stream: ", e); + } + workTimingInfosTracker.reset(); + data = ByteString.EMPTY; + bufferedSize = 0; + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcGetWorkerMetadataStream.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcGetWorkerMetadataStream.java new file mode 100644 index 0000000000000..427fd412ec7f4 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcGetWorkerMetadataStream.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.grpcclient; + +import com.google.errorprone.annotations.concurrent.GuardedBy; +import java.io.PrintWriter; +import java.util.Optional; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Function; +import org.apache.beam.runners.dataflow.worker.windmill.AbstractWindmillStream; +import org.apache.beam.runners.dataflow.worker.windmill.StreamObserverFactory; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.JobHeader; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.WorkerMetadataRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.WorkerMetadataResponse; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillEndpoints; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetWorkerMetadataStream; +import org.apache.beam.sdk.util.BackOff; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +final class GrpcGetWorkerMetadataStream + extends AbstractWindmillStream + implements GetWorkerMetadataStream { + private static final Logger LOG = LoggerFactory.getLogger(GrpcGetWorkerMetadataStream.class); + private static final WorkerMetadataRequest HEALTH_CHECK_REQUEST = + WorkerMetadataRequest.getDefaultInstance(); + private final WorkerMetadataRequest workerMetadataRequest; + private final ThrottleTimer getWorkerMetadataThrottleTimer; + private final Consumer serverMappingConsumer; + private final Object metadataLock; + + @GuardedBy("metadataLock") + private long metadataVersion; + + @GuardedBy("metadataLock") + private WorkerMetadataResponse latestResponse; + + private GrpcGetWorkerMetadataStream( + Function, StreamObserver> + startGetWorkerMetadataRpcFn, + BackOff backoff, + StreamObserverFactory streamObserverFactory, + Set> streamRegistry, + int logEveryNStreamFailures, + JobHeader jobHeader, + long metadataVersion, + ThrottleTimer getWorkerMetadataThrottleTimer, + Consumer serverMappingConsumer) { + super( + startGetWorkerMetadataRpcFn, + backoff, + streamObserverFactory, + streamRegistry, + logEveryNStreamFailures); + this.workerMetadataRequest = WorkerMetadataRequest.newBuilder().setHeader(jobHeader).build(); + this.metadataVersion = metadataVersion; + this.getWorkerMetadataThrottleTimer = getWorkerMetadataThrottleTimer; + this.serverMappingConsumer = serverMappingConsumer; + this.latestResponse = WorkerMetadataResponse.getDefaultInstance(); + this.metadataLock = new Object(); + } + + public static GrpcGetWorkerMetadataStream create( + Function, StreamObserver> + startGetWorkerMetadataRpcFn, + BackOff backoff, + StreamObserverFactory streamObserverFactory, + Set> streamRegistry, + int logEveryNStreamFailures, + JobHeader jobHeader, + int metadataVersion, + ThrottleTimer getWorkerMetadataThrottleTimer, + Consumer serverMappingUpdater) { + GrpcGetWorkerMetadataStream getWorkerMetadataStream = + new GrpcGetWorkerMetadataStream( + startGetWorkerMetadataRpcFn, + backoff, + streamObserverFactory, + streamRegistry, + logEveryNStreamFailures, + jobHeader, + metadataVersion, + getWorkerMetadataThrottleTimer, + serverMappingUpdater); + getWorkerMetadataStream.startStream(); + return getWorkerMetadataStream; + } + + /** + * Each instance of {@link AbstractWindmillStream} owns its own responseObserver that calls + * onResponse(). + */ + @Override + protected void onResponse(WorkerMetadataResponse response) { + extractWindmillEndpointsFrom(response).ifPresent(serverMappingConsumer); + } + + /** + * Acquires the {@link #metadataLock} Returns {@link Optional} if the + * metadataVersion in the response is not stale (older or equal to {@link #metadataVersion}), else + * returns empty {@link Optional}. + */ + private Optional extractWindmillEndpointsFrom( + WorkerMetadataResponse response) { + synchronized (metadataLock) { + if (response.getMetadataVersion() > this.metadataVersion) { + this.metadataVersion = response.getMetadataVersion(); + this.latestResponse = response; + return Optional.of(WindmillEndpoints.from(response)); + } else { + // If the currentMetadataVersion is greater than or equal to one in the response, the + // response data is stale, and we do not want to do anything. + LOG.info( + "Received WorkerMetadataResponse={}; Received metadata version={}; Current metadata version={}. " + + "Skipping update because received stale metadata", + response, + response.getMetadataVersion(), + this.metadataVersion); + } + } + + return Optional.empty(); + } + + @Override + protected synchronized void onNewStream() { + send(workerMetadataRequest); + } + + @Override + protected boolean hasPendingRequests() { + return false; + } + + @Override + protected void startThrottleTimer() { + getWorkerMetadataThrottleTimer.start(); + } + + @Override + protected void sendHealthCheck() { + send(HEALTH_CHECK_REQUEST); + } + + @Override + protected void appendSpecificHtml(PrintWriter writer) { + synchronized (metadataLock) { + writer.format( + "GetWorkerMetadataStream: version=[%d] , job_header=[%s], latest_response=[%s]", + this.metadataVersion, workerMetadataRequest.getHeader(), this.latestResponse); + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcWindmillServer.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcWindmillServer.java new file mode 100644 index 0000000000000..19cb90297df5b --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcWindmillServer.java @@ -0,0 +1,607 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.grpcclient; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.io.IOException; +import java.io.PrintWriter; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; +import org.apache.beam.runners.dataflow.worker.options.StreamingDataflowWorkerOptions; +import org.apache.beam.runners.dataflow.worker.windmill.AbstractWindmillStream; +import org.apache.beam.runners.dataflow.worker.windmill.CloudWindmillServiceV1Alpha1Grpc; +import org.apache.beam.runners.dataflow.worker.windmill.StreamObserverFactory; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.CommitWorkRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.CommitWorkResponse; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetConfigRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetConfigResponse; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetDataRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetDataResponse; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetWorkRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetWorkResponse; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.JobHeader; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.ReportStatsRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.ReportStatsResponse; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillApplianceGrpc; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillServerStub; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.CommitWorkStream; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetDataStream; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetWorkStream; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetWorkStream.WorkItemReceiver; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.util.BackOff; +import org.apache.beam.sdk.util.BackOffUtils; +import org.apache.beam.sdk.util.FluentBackoff; +import org.apache.beam.sdk.util.Sleeper; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Channel; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.StatusRuntimeException; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.auth.MoreCallCredentials; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessChannelBuilder; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.netty.GrpcSslContexts; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.netty.NegotiationType; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.netty.NettyChannelBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.net.HostAndPort; +import org.joda.time.Duration; +import org.joda.time.Instant; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** gRPC client for communicating with Streaming Engine. */ +// Very likely real potential for bugs - https://github.com/apache/beam/issues/19273 +// Very likely real potential for bugs - https://github.com/apache/beam/issues/19271 +@SuppressFBWarnings({"JLM_JSR166_UTILCONCURRENT_MONITORENTER", "IS2_INCONSISTENT_SYNC"}) +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) +public final class GrpcWindmillServer extends WindmillServerStub { + private static final Logger LOG = LoggerFactory.getLogger(GrpcWindmillServer.class); + + // If a connection cannot be established, gRPC will fail fast so this deadline can be relatively + // high. + private static final long DEFAULT_STREAM_RPC_DEADLINE_SECONDS = 300; + private static final int DEFAULT_LOG_EVERY_N_FAILURES = 20; + private static final String LOCALHOST = "localhost"; + private static final Duration MIN_BACKOFF = Duration.millis(1); + private static final Duration MAX_BACKOFF = Duration.standardSeconds(30); + private static final AtomicLong nextId = new AtomicLong(0); + private static final int NO_HEALTH_CHECK = -1; + + private final StreamingDataflowWorkerOptions options; + private final int streamingRpcBatchLimit; + private final List stubList; + private final ThrottleTimer getWorkThrottleTimer; + private final ThrottleTimer getDataThrottleTimer; + private final ThrottleTimer commitWorkThrottleTimer; + private final Random rand; + private final Set> streamRegistry; + private ImmutableSet endpoints; + private int logEveryNStreamFailures; + private Duration maxBackoff = MAX_BACKOFF; + private WindmillApplianceGrpc.WindmillApplianceBlockingStub syncApplianceStub = null; + + private GrpcWindmillServer(StreamingDataflowWorkerOptions options) { + this.options = options; + this.streamingRpcBatchLimit = options.getWindmillServiceStreamingRpcBatchLimit(); + this.stubList = new ArrayList<>(); + this.logEveryNStreamFailures = options.getWindmillServiceStreamingLogEveryNStreamFailures(); + this.endpoints = ImmutableSet.of(); + this.getWorkThrottleTimer = new ThrottleTimer(); + this.getDataThrottleTimer = new ThrottleTimer(); + this.commitWorkThrottleTimer = new ThrottleTimer(); + this.rand = new Random(); + this.streamRegistry = Collections.newSetFromMap(new ConcurrentHashMap<>()); + } + + private static StreamingDataflowWorkerOptions testOptions(boolean enableStreamingEngine) { + StreamingDataflowWorkerOptions options = + PipelineOptionsFactory.create().as(StreamingDataflowWorkerOptions.class); + options.setProject("project"); + options.setJobId("job"); + options.setWorkerId("worker"); + List experiments = + options.getExperiments() == null ? new ArrayList<>() : options.getExperiments(); + if (enableStreamingEngine) { + experiments.add(GcpOptions.STREAMING_ENGINE_EXPERIMENT); + } + options.setExperiments(experiments); + + options.setWindmillServiceStreamingRpcBatchLimit(Integer.MAX_VALUE); + options.setWindmillServiceStreamingRpcHealthCheckPeriodMs(NO_HEALTH_CHECK); + options.setWindmillServiceStreamingLogEveryNStreamFailures(DEFAULT_LOG_EVERY_N_FAILURES); + + return options; + } + + /** Create new instance of {@link GrpcWindmillServer}. */ + public static GrpcWindmillServer create(StreamingDataflowWorkerOptions workerOptions) + throws IOException { + GrpcWindmillServer grpcWindmillServer = new GrpcWindmillServer(workerOptions); + if (workerOptions.getWindmillServiceEndpoint() != null) { + grpcWindmillServer.configureWindmillServiceEndpoints(); + } else if (!workerOptions.isEnableStreamingEngine() + && workerOptions.getLocalWindmillHostport() != null) { + grpcWindmillServer.configureLocalHost(); + } + + if (workerOptions.getWindmillServiceStreamingRpcHealthCheckPeriodMs() > 0) { + grpcWindmillServer.scheduleHealthCheckTimer( + workerOptions, () -> grpcWindmillServer.streamRegistry); + } + + return grpcWindmillServer; + } + + @VisibleForTesting + static GrpcWindmillServer newTestInstance(String name) { + GrpcWindmillServer testServer = + new GrpcWindmillServer(testOptions(/* enableStreamingEngine= */ true)); + testServer.stubList.add(CloudWindmillServiceV1Alpha1Grpc.newStub(inProcessChannel(name))); + return testServer; + } + + @VisibleForTesting + static GrpcWindmillServer newApplianceTestInstance(Channel channel) { + GrpcWindmillServer testServer = + new GrpcWindmillServer(testOptions(/* enableStreamingEngine= */ false)); + testServer.syncApplianceStub = createWindmillApplianceStubWithDeadlineInterceptor(channel); + return testServer; + } + + private static WindmillApplianceGrpc.WindmillApplianceBlockingStub + createWindmillApplianceStubWithDeadlineInterceptor(Channel channel) { + return WindmillApplianceGrpc.newBlockingStub(channel) + .withInterceptors(GrpcDeadlineClientInterceptor.withDefaultUnaryRpcDeadline()); + } + + private static Channel inProcessChannel(String name) { + return InProcessChannelBuilder.forName(name).directExecutor().build(); + } + + private static Channel localhostChannel(int port) { + return NettyChannelBuilder.forAddress(LOCALHOST, port) + .maxInboundMessageSize(Integer.MAX_VALUE) + .negotiationType(NegotiationType.PLAINTEXT) + .build(); + } + + private static UnsupportedOperationException unsupportedUnaryRequestInStreamingEngineException( + String rpcName) { + return new UnsupportedOperationException( + String.format("Unary %s calls are not supported in Streaming Engine.", rpcName)); + } + + private void scheduleHealthCheckTimer( + StreamingDataflowWorkerOptions options, Supplier>> streams) { + new Timer("WindmillHealthCheckTimer") + .schedule( + new HealthCheckTimerTask(options, streams), + 0, + options.getWindmillServiceStreamingRpcHealthCheckPeriodMs()); + } + + private void configureWindmillServiceEndpoints() throws IOException { + Set endpoints = new HashSet<>(); + for (String endpoint : Splitter.on(',').split(options.getWindmillServiceEndpoint())) { + endpoints.add( + HostAndPort.fromString(endpoint).withDefaultPort(options.getWindmillServicePort())); + } + initializeWindmillService(endpoints); + } + + private void configureLocalHost() { + int portStart = options.getLocalWindmillHostport().lastIndexOf(':'); + String endpoint = options.getLocalWindmillHostport().substring(0, portStart); + assert ("grpc:localhost".equals(endpoint)); + int port = Integer.parseInt(options.getLocalWindmillHostport().substring(portStart + 1)); + this.endpoints = ImmutableSet.of(HostAndPort.fromParts(LOCALHOST, port)); + initializeLocalHost(port); + } + + @Override + public synchronized void setWindmillServiceEndpoints(Set endpoints) + throws IOException { + Preconditions.checkNotNull(endpoints); + if (endpoints.equals(this.endpoints)) { + // The endpoints are equal don't recreate the stubs. + return; + } + LOG.info("Creating a new windmill stub, endpoints: {}", endpoints); + if (this.endpoints != null) { + LOG.info("Previous windmill stub endpoints: {}", this.endpoints); + } + initializeWindmillService(endpoints); + } + + @Override + public synchronized boolean isReady() { + return !stubList.isEmpty(); + } + + private synchronized void initializeLocalHost(int port) { + this.logEveryNStreamFailures = 1; + this.maxBackoff = Duration.millis(500); + Channel channel = localhostChannel(port); + if (options.isEnableStreamingEngine()) { + this.stubList.add(CloudWindmillServiceV1Alpha1Grpc.newStub(channel)); + } else { + this.syncApplianceStub = createWindmillApplianceStubWithDeadlineInterceptor(channel); + } + } + + private synchronized void initializeWindmillService(Set endpoints) + throws IOException { + LOG.info("Initializing Streaming Engine GRPC client for endpoints: {}", endpoints); + this.stubList.clear(); + this.endpoints = ImmutableSet.copyOf(endpoints); + for (HostAndPort endpoint : this.endpoints) { + if (LOCALHOST.equals(endpoint.getHost())) { + initializeLocalHost(endpoint.getPort()); + } else { + this.stubList.add( + CloudWindmillServiceV1Alpha1Grpc.newStub(remoteChannel(endpoint)) + .withCallCredentials( + MoreCallCredentials.from( + new VendoredCredentialsAdapter(options.getGcpCredential())))); + } + } + } + + private Channel remoteChannel(HostAndPort endpoint) throws IOException { + NettyChannelBuilder builder = + NettyChannelBuilder.forAddress(endpoint.getHost(), endpoint.getPort()); + int timeoutSec = options.getWindmillServiceRpcChannelAliveTimeoutSec(); + if (timeoutSec > 0) { + builder + .keepAliveTime(timeoutSec, TimeUnit.SECONDS) + .keepAliveTimeout(timeoutSec, TimeUnit.SECONDS) + .keepAliveWithoutCalls(true); + } + return builder + .flowControlWindow(10 * 1024 * 1024) + .maxInboundMessageSize(Integer.MAX_VALUE) + .maxInboundMetadataSize(1024 * 1024) + .negotiationType(NegotiationType.TLS) + // Set ciphers(null) to not use GCM, which is disabled for Dataflow + // due to it being horribly slow. + .sslContext(GrpcSslContexts.forClient().ciphers(null).build()) + .build(); + } + + /** + * Stubs returned from this method do not (and should not) have {@link + * org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Deadline}(s) set since they represent an absolute + * point in time. {@link org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Deadline}(s) should not be + * treated as a timeout which represents a relative point in time. + * + * @see Official gRPC deadline documentation for more + * details. + */ + private synchronized CloudWindmillServiceV1Alpha1Grpc.CloudWindmillServiceV1Alpha1Stub stub() { + if (stubList.isEmpty()) { + throw new RuntimeException("windmillServiceEndpoint has not been set"); + } + + return stubList.size() == 1 ? stubList.get(0) : stubList.get(rand.nextInt(stubList.size())); + } + + @Override + public void appendSummaryHtml(PrintWriter writer) { + writer.write("Active Streams:
    "); + for (AbstractWindmillStream stream : streamRegistry) { + stream.appendSummaryHtml(writer); + writer.write("
    "); + } + } + + // Configure backoff to retry calls forever, with a maximum sane retry interval. + private BackOff grpcBackoff() { + return FluentBackoff.DEFAULT + .withInitialBackoff(MIN_BACKOFF) + .withMaxBackoff(maxBackoff) + .backoff(); + } + + private ResponseT callWithBackoff(Supplier function) { + BackOff backoff = grpcBackoff(); + int rpcErrors = 0; + while (true) { + try { + return function.get(); + } catch (StatusRuntimeException e) { + try { + if (++rpcErrors % 20 == 0) { + LOG.warn( + "Many exceptions calling gRPC. Last exception: {} with status {}", + e, + e.getStatus()); + } + if (!BackOffUtils.next(Sleeper.DEFAULT, backoff)) { + throw new RpcException(e); + } + } catch (IOException | InterruptedException i) { + if (i instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + RpcException rpcException = new RpcException(e); + rpcException.addSuppressed(i); + throw rpcException; + } + } + } + } + + @Override + public GetWorkResponse getWork(GetWorkRequest request) { + if (syncApplianceStub != null) { + return callWithBackoff(() -> syncApplianceStub.getWork(request)); + } + + throw new RpcException(unsupportedUnaryRequestInStreamingEngineException("GetWork")); + } + + @Override + public GetDataResponse getData(GetDataRequest request) { + if (syncApplianceStub != null) { + return callWithBackoff(() -> syncApplianceStub.getData(request)); + } + + throw new RpcException(unsupportedUnaryRequestInStreamingEngineException("GetData")); + } + + @Override + public CommitWorkResponse commitWork(CommitWorkRequest request) { + if (syncApplianceStub != null) { + return callWithBackoff(() -> syncApplianceStub.commitWork(request)); + } + throw new RpcException(unsupportedUnaryRequestInStreamingEngineException("CommitWork")); + } + + private StreamObserverFactory newStreamObserverFactory() { + return StreamObserverFactory.direct( + DEFAULT_STREAM_RPC_DEADLINE_SECONDS * 2, options.getWindmillMessagesBetweenIsReadyChecks()); + } + + @Override + public GetWorkStream getWorkStream(GetWorkRequest request, WorkItemReceiver receiver) { + GetWorkRequest getWorkRequest = + GetWorkRequest.newBuilder(request) + .setJobId(options.getJobId()) + .setProjectId(options.getProject()) + .setWorkerId(options.getWorkerId()) + .build(); + + return GrpcGetWorkStream.create( + responseObserver -> + stub() + // Deadlines are absolute points in time, so generate a new one everytime this + // function is called. + .withDeadlineAfter( + AbstractWindmillStream.DEFAULT_STREAM_RPC_DEADLINE_SECONDS, TimeUnit.SECONDS) + .getWorkStream(responseObserver), + getWorkRequest, + grpcBackoff(), + newStreamObserverFactory(), + streamRegistry, + logEveryNStreamFailures, + getWorkThrottleTimer, + receiver); + } + + @Override + public GetDataStream getDataStream() { + return GrpcGetDataStream.create( + responseObserver -> + stub() + // Deadlines are absolute points in time, so generate a new one everytime this + // function is called. + .withDeadlineAfter( + AbstractWindmillStream.DEFAULT_STREAM_RPC_DEADLINE_SECONDS, TimeUnit.SECONDS) + .getDataStream(responseObserver), + grpcBackoff(), + newStreamObserverFactory(), + streamRegistry, + logEveryNStreamFailures, + getDataThrottleTimer, + makeHeader(), + nextId, + streamingRpcBatchLimit); + } + + @Override + public CommitWorkStream commitWorkStream() { + return GrpcCommitWorkStream.create( + responseObserver -> + stub() + // Deadlines are absolute points in time, so generate a new one everytime this + // function is called. + .withDeadlineAfter( + AbstractWindmillStream.DEFAULT_STREAM_RPC_DEADLINE_SECONDS, TimeUnit.SECONDS) + .commitWorkStream(responseObserver), + grpcBackoff(), + newStreamObserverFactory(), + streamRegistry, + logEveryNStreamFailures, + commitWorkThrottleTimer, + makeHeader(), + nextId, + streamingRpcBatchLimit); + } + + @Override + public GetConfigResponse getConfig(GetConfigRequest request) { + if (syncApplianceStub != null) { + return callWithBackoff(() -> syncApplianceStub.getConfig(request)); + } + + throw new RpcException( + new UnsupportedOperationException("GetConfig not supported in Streaming Engine.")); + } + + @Override + public ReportStatsResponse reportStats(ReportStatsRequest request) { + if (syncApplianceStub != null) { + return callWithBackoff(() -> syncApplianceStub.reportStats(request)); + } + + throw new RpcException( + new UnsupportedOperationException("ReportStats not supported in Streaming Engine.")); + } + + @Override + public long getAndResetThrottleTime() { + return getWorkThrottleTimer.getAndResetThrottleTime() + + getDataThrottleTimer.getAndResetThrottleTime() + + commitWorkThrottleTimer.getAndResetThrottleTime(); + } + + private JobHeader makeHeader() { + return JobHeader.newBuilder() + .setJobId(options.getJobId()) + .setProjectId(options.getProject()) + .setWorkerId(options.getWorkerId()) + .build(); + } + + /** + * Create a wrapper around credentials callback that delegates to the underlying vendored {@link + * com.google.auth.RequestMetadataCallback}. Note that this class should override every method + * that is not final and not static and call the delegate directly. + * + *

    TODO: Replace this with an auto generated proxy which calls the underlying implementation + * delegate to reduce maintenance burden. + */ + private static class VendoredRequestMetadataCallbackAdapter + implements com.google.auth.RequestMetadataCallback { + + private final org.apache.beam.vendor.grpc.v1p54p0.com.google.auth.RequestMetadataCallback + callback; + + private VendoredRequestMetadataCallbackAdapter( + org.apache.beam.vendor.grpc.v1p54p0.com.google.auth.RequestMetadataCallback callback) { + this.callback = callback; + } + + @Override + public void onSuccess(Map> metadata) { + callback.onSuccess(metadata); + } + + @Override + public void onFailure(Throwable exception) { + callback.onFailure(exception); + } + } + + /** + * Create a wrapper around credentials that delegates to the underlying {@link + * com.google.auth.Credentials}. Note that this class should override every method that is not + * final and not static and call the delegate directly. + * + *

    TODO: Replace this with an auto generated proxy which calls the underlying implementation + * delegate to reduce maintenance burden. + */ + private static class VendoredCredentialsAdapter + extends org.apache.beam.vendor.grpc.v1p54p0.com.google.auth.Credentials { + + private final com.google.auth.Credentials credentials; + + private VendoredCredentialsAdapter(com.google.auth.Credentials credentials) { + this.credentials = credentials; + } + + @Override + public String getAuthenticationType() { + return credentials.getAuthenticationType(); + } + + @Override + public Map> getRequestMetadata() throws IOException { + return credentials.getRequestMetadata(); + } + + @Override + public void getRequestMetadata( + final URI uri, + Executor executor, + final org.apache.beam.vendor.grpc.v1p54p0.com.google.auth.RequestMetadataCallback + callback) { + credentials.getRequestMetadata( + uri, executor, new VendoredRequestMetadataCallbackAdapter(callback)); + } + + @Override + public Map> getRequestMetadata(URI uri) throws IOException { + return credentials.getRequestMetadata(uri); + } + + @Override + public boolean hasRequestMetadata() { + return credentials.hasRequestMetadata(); + } + + @Override + public boolean hasRequestMetadataOnly() { + return credentials.hasRequestMetadataOnly(); + } + + @Override + public void refresh() throws IOException { + credentials.refresh(); + } + } + + private static class HealthCheckTimerTask extends TimerTask { + private final StreamingDataflowWorkerOptions options; + private final Supplier>> streams; + + public HealthCheckTimerTask( + StreamingDataflowWorkerOptions options, + Supplier>> streams) { + this.options = options; + this.streams = streams; + } + + @Override + public void run() { + Instant reportThreshold = + Instant.now() + .minus(Duration.millis(options.getWindmillServiceStreamingRpcHealthCheckPeriodMs())); + for (AbstractWindmillStream stream : streams.get()) { + stream.maybeSendHealthCheck(reportThreshold); + } + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/ThrottleTimer.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/ThrottleTimer.java new file mode 100644 index 0000000000000..237339aff3993 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/ThrottleTimer.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.grpcclient; + +import org.joda.time.Instant; + +/** + * A stopwatch used to track the amount of time spent throttled due to Resource Exhausted errors. + * Throttle time is cumulative for all three rpcs types but not for all streams. So if GetWork and + * CommitWork are both blocked for x, totalTime will be 2x. However, if 2 GetWork streams are both + * blocked for x totalTime will be x. All methods are thread safe. + */ +class ThrottleTimer { + // This is -1 if not currently being throttled or the time in + // milliseconds when throttling for this type started. + private long startTime = -1; + // This is the collected total throttle times since the last poll. Throttle times are + // reported as a delta so this is cleared whenever it gets reported. + private long totalTime = 0; + + /** + * Starts the timer if it has not been started and does nothing if it has already been started. + */ + synchronized void start() { + if (!throttled()) { // This timer is not started yet so start it now. + startTime = Instant.now().getMillis(); + } + } + + /** Stops the timer if it has been started and does nothing if it has not been started. */ + public synchronized void stop() { + if (throttled()) { // This timer has been started already so stop it now. + totalTime += Instant.now().getMillis() - startTime; + startTime = -1; + } + } + + /** Returns if the specified type is currently being throttled. */ + public synchronized boolean throttled() { + return startTime != -1; + } + + /** Returns the combined total of all throttle times and resets those times to 0. */ + public synchronized long getAndResetThrottleTime() { + if (throttled()) { + stop(); + start(); + } + long toReturn = totalTime; + totalTime = 0; + return toReturn; + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/CachingStateTable.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/CachingStateTable.java new file mode 100644 index 0000000000000..bcaf8bf21a2da --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/CachingStateTable.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import java.io.Closeable; +import java.util.Optional; +import javax.annotation.Nullable; +import org.apache.beam.runners.core.StateNamespace; +import org.apache.beam.runners.core.StateTable; +import org.apache.beam.runners.core.StateTag; +import org.apache.beam.runners.core.StateTags; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.state.BagState; +import org.apache.beam.sdk.state.CombiningState; +import org.apache.beam.sdk.state.MapState; +import org.apache.beam.sdk.state.MultimapState; +import org.apache.beam.sdk.state.OrderedListState; +import org.apache.beam.sdk.state.SetState; +import org.apache.beam.sdk.state.State; +import org.apache.beam.sdk.state.StateContext; +import org.apache.beam.sdk.state.ValueState; +import org.apache.beam.sdk.state.WatermarkHoldState; +import org.apache.beam.sdk.transforms.Combine; +import org.apache.beam.sdk.transforms.CombineWithContext; +import org.apache.beam.sdk.transforms.windowing.TimestampCombiner; +import org.apache.beam.sdk.util.CombineFnUtil; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; + +final class CachingStateTable extends StateTable { + private final String stateFamily; + private final WindmillStateReader reader; + private final WindmillStateCache.ForKeyAndFamily cache; + private final boolean isSystemTable; + private final Supplier scopedReadStateSupplier; + private final @Nullable StateTable derivedStateTable; + private final boolean isNewKey; + + private CachingStateTable(Builder builder) { + this.stateFamily = builder.stateFamily; + this.reader = builder.reader; + this.cache = builder.cache; + this.isSystemTable = builder.isSystemTable; + this.isNewKey = builder.isNewKey; + this.scopedReadStateSupplier = builder.scopedReadStateSupplier; + this.derivedStateTable = builder.derivedStateTable; + + if (this.isSystemTable) { + Preconditions.checkState(derivedStateTable == null); + } else { + Preconditions.checkNotNull(this.derivedStateTable); + } + } + + static CachingStateTable.Builder builder( + String stateFamily, + WindmillStateReader reader, + WindmillStateCache.ForKeyAndFamily cache, + boolean isNewKey, + Supplier scopedReadStateSupplier) { + return new CachingStateTable.Builder( + stateFamily, reader, cache, scopedReadStateSupplier, isNewKey); + } + + @Override + @SuppressWarnings("deprecation") + protected StateTag.StateBinder binderForNamespace(StateNamespace namespace, StateContext c) { + // Look up state objects in the cache or create new ones if not found. The state will + // be added to the cache in persist(). + return new StateTag.StateBinder() { + @Override + public BagState bindBag(StateTag> address, Coder elemCoder) { + StateTag> resolvedAddress = + isSystemTable ? StateTags.makeSystemTagInternal(address) : address; + + WindmillBag result = + cache + .get(namespace, resolvedAddress) + .map(bagState -> (WindmillBag) bagState) + .orElseGet( + () -> + new WindmillBag<>( + namespace, resolvedAddress, stateFamily, elemCoder, isNewKey)); + + result.initializeForWorkItem(reader, scopedReadStateSupplier); + return result; + } + + @Override + public SetState bindSet(StateTag> spec, Coder elemCoder) { + WindmillSet result = + new WindmillSet<>(namespace, spec, stateFamily, elemCoder, cache, isNewKey); + result.initializeForWorkItem(reader, scopedReadStateSupplier); + return result; + } + + @Override + public MapState bindMap( + StateTag> spec, Coder keyCoder, Coder valueCoder) { + WindmillMap result = + cache + .get(namespace, spec) + .map(mapState -> (WindmillMap) mapState) + .orElseGet( + () -> + new WindmillMap<>( + namespace, spec, stateFamily, keyCoder, valueCoder, isNewKey)); + + result.initializeForWorkItem(reader, scopedReadStateSupplier); + return result; + } + + @Override + public MultimapState bindMultimap( + StateTag> spec, + Coder keyCoder, + Coder valueCoder) { + WindmillMultimap result = + cache + .get(namespace, spec) + .map(multimapState -> (WindmillMultimap) multimapState) + .orElseGet( + () -> + new WindmillMultimap<>( + namespace, spec, stateFamily, keyCoder, valueCoder, isNewKey)); + result.initializeForWorkItem(reader, scopedReadStateSupplier); + return result; + } + + @Override + public OrderedListState bindOrderedList( + StateTag> spec, Coder elemCoder) { + StateTag> specOrInternalTag = addressOrInternalTag(spec); + + WindmillOrderedList result = + cache + .get(namespace, specOrInternalTag) + .map(orderedList -> (WindmillOrderedList) orderedList) + .orElseGet( + () -> + new WindmillOrderedList<>( + Optional.ofNullable(derivedStateTable).orElse(CachingStateTable.this), + namespace, + specOrInternalTag, + stateFamily, + elemCoder, + isNewKey)); + + result.initializeForWorkItem(reader, scopedReadStateSupplier); + return result; + } + + @Override + public WatermarkHoldState bindWatermark( + StateTag address, TimestampCombiner timestampCombiner) { + StateTag addressOrInternalTag = addressOrInternalTag(address); + + WindmillWatermarkHold result = + cache + .get(namespace, addressOrInternalTag) + .map(watermarkHold -> (WindmillWatermarkHold) watermarkHold) + .orElseGet( + () -> + new WindmillWatermarkHold( + namespace, address, stateFamily, timestampCombiner, isNewKey)); + + result.initializeForWorkItem(reader, scopedReadStateSupplier); + return result; + } + + @Override + public CombiningState bindCombiningValue( + StateTag> address, + Coder accumCoder, + Combine.CombineFn combineFn) { + StateTag> addressOrInternalTag = + addressOrInternalTag(address); + + WindmillCombiningState result = + new WindmillCombiningState<>( + namespace, + addressOrInternalTag, + stateFamily, + accumCoder, + combineFn, + cache, + isNewKey); + + result.initializeForWorkItem(reader, scopedReadStateSupplier); + return result; + } + + @Override + public + CombiningState bindCombiningValueWithContext( + StateTag> address, + Coder accumCoder, + CombineWithContext.CombineFnWithContext combineFn) { + return bindCombiningValue( + addressOrInternalTag(address), accumCoder, CombineFnUtil.bindContext(combineFn, c)); + } + + @Override + public ValueState bindValue(StateTag> address, Coder coder) { + StateTag> addressOrInternalTag = addressOrInternalTag(address); + + WindmillValue result = + cache + .get(namespace, addressOrInternalTag) + .map(value -> (WindmillValue) value) + .orElseGet( + () -> + new WindmillValue<>( + namespace, addressOrInternalTag, stateFamily, coder, isNewKey)); + + result.initializeForWorkItem(reader, scopedReadStateSupplier); + return result; + } + + private StateTag addressOrInternalTag(StateTag address) { + return isSystemTable ? StateTags.makeSystemTagInternal(address) : address; + } + }; + } + + static class Builder { + private final String stateFamily; + private final WindmillStateReader reader; + private final WindmillStateCache.ForKeyAndFamily cache; + private final Supplier scopedReadStateSupplier; + private final boolean isNewKey; + private boolean isSystemTable; + private @Nullable StateTable derivedStateTable; + + private Builder( + String stateFamily, + WindmillStateReader reader, + WindmillStateCache.ForKeyAndFamily cache, + Supplier scopedReadStateSupplier, + boolean isNewKey) { + this.stateFamily = stateFamily; + this.reader = reader; + this.cache = cache; + this.scopedReadStateSupplier = scopedReadStateSupplier; + this.isNewKey = isNewKey; + this.isSystemTable = true; + this.derivedStateTable = null; + } + + Builder withDerivedState(StateTable derivedStateTable) { + this.isSystemTable = false; + this.derivedStateTable = derivedStateTable; + return this; + } + + CachingStateTable build() { + return new CachingStateTable(this); + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/ConcatIterables.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/ConcatIterables.java new file mode 100644 index 0000000000000..4bb806bd70fd5 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/ConcatIterables.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; + +class ConcatIterables implements Iterable { + // List of component iterables. Should only be appended to in order to support snapshot(). + private final List> iterables; + + public ConcatIterables() { + this.iterables = new ArrayList<>(); + } + + public void extendWith(Iterable iterable) { + iterables.add(iterable); + } + + @Override + public Iterator iterator() { + return Iterators.concat(Iterables.transform(iterables, Iterable::iterator).iterator()); + } + + /** + * Returns a view of the current state of this iterable. Remembers the current length of iterables + * so that the returned value Will not change due to future extendWith() calls. + */ + public Iterable snapshot() { + final int limit = iterables.size(); + final List> iterablesList = iterables; + return () -> + Iterators.concat( + Iterators.transform( + Iterators.limit(iterablesList.iterator(), limit), Iterable::iterator)); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/IdTracker.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/IdTracker.java new file mode 100644 index 0000000000000..5090626ae8eeb --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/IdTracker.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import java.util.Comparator; +import java.util.Iterator; +import java.util.Map; +import java.util.SortedSet; +import java.util.concurrent.ExecutionException; +import java.util.function.BiConsumer; +import org.apache.beam.runners.core.StateNamespace; +import org.apache.beam.runners.core.StateTable; +import org.apache.beam.runners.core.StateTag; +import org.apache.beam.runners.core.StateTags; +import org.apache.beam.sdk.coders.InstantCoder; +import org.apache.beam.sdk.coders.MapCoder; +import org.apache.beam.sdk.coders.VarLongCoder; +import org.apache.beam.sdk.state.StateContexts; +import org.apache.beam.sdk.state.ValueState; +import org.apache.beam.sdk.values.TimestampedValue; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Range; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.RangeSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.TreeRangeSet; +import org.joda.time.Duration; +import org.joda.time.Instant; + +/** + * Tracker for the ids used in an ordered list. + * + *

    Windmill accepts an int64 id for each timestamped-element in the list. Unique elements are + * identified by the pair of timestamp and id. This means that tow unique elements e1, e2 must have + * different (ts1, id1), (ts2, id2) pairs. To accomplish this we bucket time into five-minute + * buckets, and store a free list of ids available for each bucket. + * + *

    When a timestamp range is deleted, we remove id tracking for elements in that range. In order + * to handle the case where a range is deleted piecemeal, we track sub-range deletions for each + * range. For example: + * + *

    12:00 - 12:05 ids 12:05 - 12:10 ids + * + *

    delete 12:00-12:06 + * + *

    12:00 - 12:05 *removed* 12:05 - 12:10 ids subranges deleted 12:05-12:06 + * + *

    delete 12:06 - 12:07 + * + *

    12:05 - 12:10 ids subranges deleted 12:05-12:07 + * + *

    delete 12:07 - 12:10 + * + *

    12:05 - 12:10 *removed* + */ +@SuppressWarnings("nullness" // TODO(https://github.com/apache/beam/issues/20497) +) +final class IdTracker { + @VisibleForTesting static final String IDS_AVAILABLE_STR = "IdsAvailable"; + @VisibleForTesting static final String DELETIONS_STR = "Deletions"; + // Note that this previously was Long.MIN_VALUE but ids are unsigned when + // sending to windmill for Streaming Engine. For updated appliance + // pipelines with existing state, there may be negative ids. + @VisibleForTesting static final long NEW_RANGE_MIN_ID = 0; + + @VisibleForTesting + static final MapCoder, RangeSet> IDS_AVAILABLE_CODER = + MapCoder.of(new RangeCoder<>(InstantCoder.of()), new RangeSetCoder<>(VarLongCoder.of())); + + @VisibleForTesting + static final MapCoder, RangeSet> SUBRANGE_DELETIONS_CODER = + MapCoder.of(new RangeCoder<>(InstantCoder.of()), new RangeSetCoder<>(InstantCoder.of())); + + private static final long NEW_RANGE_MAX_ID = Long.MAX_VALUE; + // We track ids on five-minute boundaries. + private static final Duration RESOLUTION = Duration.standardMinutes(5); + // A map from five-minute ranges to the set of ids available in that interval. + private final ValueState, RangeSet>> idsAvailableValue; + // If a timestamp-range in the map has been partially cleared, the cleared intervals are stored + // here. + private final ValueState, RangeSet>> subRangeDeletionsValue; + + IdTracker(StateTable stateTable, StateNamespace namespace, StateTag spec) { + StateTag, RangeSet>>> idsAvailableTag = + StateTags.makeSystemTagInternal( + StateTags.value(spec.getId() + IDS_AVAILABLE_STR, IDS_AVAILABLE_CODER)); + StateTag, RangeSet>>> subRangeDeletionsTag = + StateTags.makeSystemTagInternal( + StateTags.value(spec.getId() + DELETIONS_STR, SUBRANGE_DELETIONS_CODER)); + + this.idsAvailableValue = + stateTable.get(namespace, idsAvailableTag, StateContexts.nullContext()); + this.subRangeDeletionsValue = + stateTable.get(namespace, subRangeDeletionsTag, StateContexts.nullContext()); + } + + static > + Map, RangeSet> newSortedRangeMap() { + return Maps.newTreeMap( + Comparator., Instant>comparing(Range::lowerEndpoint) + .thenComparing(Range::upperEndpoint)); + } + + private Range getTrackedRange(Instant ts) { + Instant snapped = + new Instant(ts.getMillis() - ts.plus(RESOLUTION).getMillis() % RESOLUTION.getMillis()); + return Range.closedOpen(snapped, snapped.plus(RESOLUTION)); + } + + @SuppressWarnings("FutureReturnValueIgnored") + void readLater() { + idsAvailableValue.readLater(); + subRangeDeletionsValue.readLater(); + } + + Map, RangeSet> readIdsAvailable() { + Map, RangeSet> idsAvailable = idsAvailableValue.read(); + return idsAvailable != null ? idsAvailable : newSortedRangeMap(); + } + + Map, RangeSet> readSubRangeDeletions() { + Map, RangeSet> subRangeDeletions = subRangeDeletionsValue.read(); + return subRangeDeletions != null ? subRangeDeletions : newSortedRangeMap(); + } + + void clear() throws ExecutionException, InterruptedException { + idsAvailableValue.clear(); + subRangeDeletionsValue.clear(); + } + + void add( + SortedSet> elements, BiConsumer, Long> output) + throws ExecutionException, InterruptedException { + Range currentIdRange = null; + long currentId = 0; + + Range currentTsRange = null; + RangeSet currentTsRangeDeletions = null; + + Map, RangeSet> idsAvailable = readIdsAvailable(); + Map, RangeSet> subRangeDeletions = readSubRangeDeletions(); + + RangeSet availableIdsForTsRange = null; + Iterator> idRangeIter = null; + RangeSet idsUsed = TreeRangeSet.create(); + for (TimestampedValueWithId pendingAdd : elements) { + // Since elements are in increasing ts order, often we'll be able to reuse the previous + // iteration's range. + if (currentTsRange == null + || !currentTsRange.contains(pendingAdd.getValue().getTimestamp())) { + if (availableIdsForTsRange != null) { + // We're moving onto a new ts range. Remove all used ids + availableIdsForTsRange.removeAll(idsUsed); + idsUsed = TreeRangeSet.create(); + } + + // Lookup the range for the current timestamp. + currentTsRange = getTrackedRange(pendingAdd.getValue().getTimestamp()); + // Lookup available ids for this timestamp range. If nothing there, we default to all ids + // available. + availableIdsForTsRange = + idsAvailable.computeIfAbsent( + currentTsRange, + r -> + TreeRangeSet.create( + ImmutableList.of(Range.closedOpen(NEW_RANGE_MIN_ID, NEW_RANGE_MAX_ID)))); + idRangeIter = availableIdsForTsRange.asRanges().iterator(); + currentIdRange = null; + currentTsRangeDeletions = subRangeDeletions.get(currentTsRange); + } + + if (currentIdRange == null || currentId >= currentIdRange.upperEndpoint()) { + // Move to the next range of free ids, and start assigning ranges from there. + currentIdRange = idRangeIter.next(); + currentId = currentIdRange.lowerEndpoint(); + } + + if (currentTsRangeDeletions != null) { + currentTsRangeDeletions.remove( + Range.closedOpen( + pendingAdd.getValue().getTimestamp(), + pendingAdd.getValue().getTimestamp().plus(Duration.millis(1)))); + } + idsUsed.add(Range.closedOpen(currentId, currentId + 1)); + output.accept(pendingAdd.getValue(), currentId++); + } + if (availableIdsForTsRange != null) { + availableIdsForTsRange.removeAll(idsUsed); + } + writeValues(idsAvailable, subRangeDeletions); + } + + // Remove a timestamp range. Returns ids freed up. + void remove(Range tsRange) throws ExecutionException, InterruptedException { + Map, RangeSet> idsAvailable = readIdsAvailable(); + Map, RangeSet> subRangeDeletions = readSubRangeDeletions(); + + for (Range current = getTrackedRange(tsRange.lowerEndpoint()); + current.lowerEndpoint().isBefore(tsRange.upperEndpoint()); + current = getTrackedRange(current.lowerEndpoint().plus(RESOLUTION))) { + // TODO(reuvenlax): shouldn't need to iterate over all ranges. + boolean rangeCleared; + if (!tsRange.encloses(current)) { + // This can happen if the beginning or the end of tsRange doesn't fall on a RESOLUTION + // boundary. Since we are deleting a portion of a tracked range, track what we are deleting. + RangeSet rangeDeletions = + subRangeDeletions.computeIfAbsent(current, r -> TreeRangeSet.create()); + rangeDeletions.add(tsRange.intersection(current)); + // If we ended up deleting the whole range, then we can simply remove it from the tracking + // map. + rangeCleared = rangeDeletions.encloses(current); + } else { + rangeCleared = true; + } + if (rangeCleared) { + // Remove the range from both maps. + idsAvailable.remove(current); + subRangeDeletions.remove(current); + } + } + writeValues(idsAvailable, subRangeDeletions); + } + + private void writeValues( + Map, RangeSet> idsAvailable, + Map, RangeSet> subRangeDeletions) { + if (idsAvailable.isEmpty()) { + idsAvailable.clear(); + } else { + idsAvailableValue.write(idsAvailable); + } + if (subRangeDeletions.isEmpty()) { + subRangeDeletionsValue.clear(); + } else { + subRangeDeletionsValue.write(subRangeDeletions); + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/PagingIterable.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/PagingIterable.java new file mode 100644 index 0000000000000..73f076d920134 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/PagingIterable.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.AbstractIterator; + +/** + * An iterable over elements backed by paginated GetData requests to Windmill. The iterable may be + * iterated over an arbitrary number of times and multiple iterators may be active simultaneously. + * + *

    There are two pattern we wish to support with low -memory and -latency: + * + *

      + *
    1. Re-iterate over the initial elements multiple times (eg Iterables.first). We'll cache the + * initial 'page' of values returned by Windmill from our first request for the lifetime of + * the iterable. + *
    2. Iterate through all elements of a very large collection. We'll send the GetData request for + * the next page when the current page is begun. We'll discard intermediate pages and only + * retain the first. Thus the maximum memory pressure is one page plus one page per call to + * iterator. + *
    + */ +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) +class PagingIterable implements Iterable { + /** + * The reader we will use for scheduling continuation pages. + * + *

    NOTE We've made this explicit to remind us to be careful not to cache the iterable. + */ + private final WindmillStateReader reader; + + /** Initial values returned for the first page. Never reclaimed. */ + private final List firstPage; + + /** State tag with continuation position set for second page. */ + private final StateTag secondPagePos; + + /** Coder for elements. */ + private final Coder coder; + + PagingIterable( + WindmillStateReader reader, + List firstPage, + StateTag secondPagePos, + Coder coder) { + this.reader = reader; + this.firstPage = firstPage; + this.secondPagePos = secondPagePos; + this.coder = coder; + } + + @Override + public Iterator iterator() { + return new PagingIterableIterator(); + } + + private class PagingIterableIterator extends AbstractIterator { + private Iterator currentPage = firstPage.iterator(); + private StateTag nextPagePos = secondPagePos; + private Future> pendingNextPage = + // NOTE: The results of continuation page reads are never cached. + reader.continuationFuture(nextPagePos, coder); + + @Override + protected ResultT computeNext() { + while (true) { + if (currentPage.hasNext()) { + return currentPage.next(); + } + if (pendingNextPage == null) { + return endOfData(); + } + + ValuesAndContPosition valuesAndContPosition; + try { + valuesAndContPosition = pendingNextPage.get(); + } catch (InterruptedException | ExecutionException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new RuntimeException("Unable to read value from state", e); + } + currentPage = valuesAndContPosition.getValues().iterator(); + StateTag.Builder nextPageBuilder = + StateTag.of( + nextPagePos.getKind(), + nextPagePos.getTag(), + nextPagePos.getStateFamily(), + valuesAndContPosition.getContinuationPosition()) + .toBuilder(); + if (secondPagePos.getSortedListRange() != null) { + nextPageBuilder.setSortedListRange(secondPagePos.getSortedListRange()); + } + if (secondPagePos.getOmitValues() != null) { + nextPageBuilder.setOmitValues(secondPagePos.getOmitValues()); + } + if (secondPagePos.getMultimapKey() != null) { + nextPageBuilder.setMultimapKey(secondPagePos.getMultimapKey()); + } + nextPagePos = nextPageBuilder.build(); + pendingNextPage = + // NOTE: The results of continuation page reads are never cached. + reader.continuationFuture(nextPagePos, coder); + } + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/RangeCoder.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/RangeCoder.java new file mode 100644 index 0000000000000..0e11531226f75 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/RangeCoder.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.List; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.coders.NullableCoder; +import org.apache.beam.sdk.coders.StructuredCoder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BoundType; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Range; +import org.checkerframework.checker.nullness.qual.Nullable; + +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) +/** Coder for closed-open ranges. */ +class RangeCoder> extends StructuredCoder> { + private final Coder boundCoder; + + RangeCoder(Coder boundCoder) { + this.boundCoder = NullableCoder.of(boundCoder); + } + + @Override + public List> getCoderArguments() { + return Lists.newArrayList(boundCoder); + } + + @Override + public void verifyDeterministic() throws NonDeterministicException { + boundCoder.verifyDeterministic(); + } + + @Override + public void encode(Range value, OutputStream outStream) throws IOException { + Preconditions.checkState( + value.lowerBoundType().equals(BoundType.CLOSED), "unexpected range " + value); + Preconditions.checkState( + value.upperBoundType().equals(BoundType.OPEN), "unexpected range " + value); + boundCoder.encode(value.hasLowerBound() ? value.lowerEndpoint() : null, outStream); + boundCoder.encode(value.hasUpperBound() ? value.upperEndpoint() : null, outStream); + } + + @Override + public Range decode(InputStream inStream) throws IOException { + @Nullable T lower = boundCoder.decode(inStream); + @Nullable T upper = boundCoder.decode(inStream); + if (lower == null) { + return upper != null ? Range.lessThan(upper) : Range.all(); + } else if (upper == null) { + return Range.atLeast(lower); + } else { + return Range.closedOpen(lower, upper); + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/RangeSetCoder.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/RangeSetCoder.java new file mode 100644 index 0000000000000..291a83e81ee60 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/RangeSetCoder.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.coders.CustomCoder; +import org.apache.beam.sdk.coders.SetCoder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Range; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.RangeSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.TreeRangeSet; + +class RangeSetCoder> extends CustomCoder> { + private final SetCoder> rangesCoder; + + RangeSetCoder(Coder boundCoder) { + this.rangesCoder = SetCoder.of(new RangeCoder<>(boundCoder)); + } + + @Override + public void encode(RangeSet value, OutputStream outStream) throws IOException { + rangesCoder.encode(value.asRanges(), outStream); + } + + @Override + public RangeSet decode(InputStream inStream) throws IOException { + return TreeRangeSet.create(rangesCoder.decode(inStream)); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/SimpleWindmillState.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/SimpleWindmillState.java new file mode 100644 index 0000000000000..bd7f8041c6800 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/SimpleWindmillState.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import java.io.IOException; +import java.util.concurrent.Future; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Futures; + +/** + * Base class for implementations of {@link WindmillState} where the {@link #persist} call does not + * require any asynchronous reading. + */ +abstract class SimpleWindmillState extends WindmillState { + @Override + public final Future persist( + WindmillStateCache.ForKeyAndFamily cache) throws IOException { + return Futures.immediateFuture(persistDirectly(cache)); + } + + /** + * Returns a {@link Windmill.WorkItemCommitRequest} that can be used to persist this state to + * Windmill. + */ + protected abstract Windmill.WorkItemCommitRequest persistDirectly( + WindmillStateCache.ForKeyAndFamily cache) throws IOException; +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/StateTag.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/StateTag.java new file mode 100644 index 0000000000000..13c2a9e66baa3 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/StateTag.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import com.google.auto.value.AutoValue; +import javax.annotation.Nullable; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Range; + +/** + * When combined with a key and computationId, represents the unique address for state managed by + * Windmill. + */ +@AutoValue +public abstract class StateTag { + static StateTag of( + Kind kind, ByteString tag, String stateFamily, @Nullable RequestPositionT requestPosition) { + return new AutoValue_StateTag.Builder() + .setKind(kind) + .setTag(tag) + .setStateFamily(stateFamily) + .setRequestPosition(requestPosition) + .build(); + } + + public static StateTag of( + Kind kind, ByteString tag, String stateFamily) { + return of(kind, tag, stateFamily, null); + } + + abstract Kind getKind(); + + abstract ByteString getTag(); + + abstract String getStateFamily(); + + /** + * For {@link Kind#BAG, Kind#ORDERED_LIST, Kind#VALUE_PREFIX, KIND#MULTIMAP_SINGLE_ENTRY, + * KIND#MULTIMAP_ALL} kinds: A previous 'continuation_position' returned by Windmill to signal the + * resulting state was incomplete. Sending that position will request the next page of values. + * Null for first request. + * + *

    Null for other kinds. + */ + @Nullable + public abstract RequestPositionT getRequestPosition(); + + /** For {@link Kind#ORDERED_LIST} kinds: the range to fetch or delete. */ + @Nullable + abstract Range getSortedListRange(); + + /** For {@link Kind#MULTIMAP_SINGLE_ENTRY} kinds: the key in the multimap to fetch or delete. */ + @Nullable + abstract ByteString getMultimapKey(); + + /** + * For {@link Kind#MULTIMAP_ALL} kinds: will only return the keys of the multimap and not the + * values if true. + */ + @Nullable + abstract Boolean getOmitValues(); + + public abstract Builder toBuilder(); + + public enum Kind { + VALUE, + BAG, + WATERMARK, + ORDERED_LIST, + VALUE_PREFIX, + MULTIMAP_SINGLE_ENTRY, + MULTIMAP_ALL + } + + @AutoValue.Builder + abstract static class Builder { + abstract Builder setKind(Kind kind); + + abstract Builder setTag(ByteString tag); + + abstract Builder setStateFamily(String stateFamily); + + abstract Builder setRequestPosition( + @Nullable RequestPositionT requestPosition); + + abstract Builder setSortedListRange(@Nullable Range sortedListRange); + + abstract Builder setMultimapKey(@Nullable ByteString encodedMultimapKey); + + abstract Builder setOmitValues(Boolean omitValues); + + abstract StateTag build(); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/TimestampedValueWithId.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/TimestampedValueWithId.java new file mode 100644 index 0000000000000..e180efafb65bc --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/TimestampedValueWithId.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import com.google.auto.value.AutoValue; +import java.util.Comparator; +import org.apache.beam.sdk.values.TimestampedValue; +import org.joda.time.Instant; + +@AutoValue +abstract class TimestampedValueWithId { + public static final Comparator> COMPARATOR = + Comparator., Instant>comparing(v -> v.getValue().getTimestamp()) + .thenComparingLong(TimestampedValueWithId::getId); + + static TimestampedValueWithId of(TimestampedValue value, long id) { + return new AutoValue_TimestampedValueWithId<>(value, id); + } + + static TimestampedValueWithId bound(Instant ts) { + return of(TimestampedValue.of(null, ts), Long.MIN_VALUE); + } + + abstract TimestampedValue getValue(); + + abstract long getId(); +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/ToIterableFunction.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/ToIterableFunction.java new file mode 100644 index 0000000000000..3db058c79a03b --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/ToIterableFunction.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function; + +/** Function to extract an {@link Iterable} from the continuation-supporting page read future. */ +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) +public class ToIterableFunction + implements Function, Iterable> { + private final StateTag stateTag; + private final Coder coder; + /** + * Reader to request continuation pages from, or {@literal null} if no continuation pages + * required. + */ + private @Nullable WindmillStateReader reader; + + public ToIterableFunction( + WindmillStateReader reader, StateTag stateTag, Coder coder) { + this.reader = reader; + this.stateTag = stateTag; + this.coder = coder; + } + + @SuppressFBWarnings( + value = "NP_METHOD_PARAMETER_TIGHTENS_ANNOTATION", + justification = "https://github.com/google/guava/issues/920") + @Override + public Iterable apply( + @Nonnull ValuesAndContPosition valuesAndContPosition) { + if (valuesAndContPosition.getContinuationPosition() == null) { + // Number of values is small enough Windmill sent us the entire bag in one response. + reader = null; + return valuesAndContPosition.getValues(); + } else { + // Return an iterable which knows how to come back for more. + StateTag.Builder continuationTBuilder = + StateTag.of( + stateTag.getKind(), + stateTag.getTag(), + stateTag.getStateFamily(), + valuesAndContPosition.getContinuationPosition()) + .toBuilder(); + if (stateTag.getSortedListRange() != null) { + continuationTBuilder.setSortedListRange(stateTag.getSortedListRange()).build(); + } + if (stateTag.getMultimapKey() != null) { + continuationTBuilder.setMultimapKey(stateTag.getMultimapKey()).build(); + } + if (stateTag.getOmitValues() != null) { + continuationTBuilder.setOmitValues(stateTag.getOmitValues()).build(); + } + return new PagingIterable<>( + reader, valuesAndContPosition.getValues(), continuationTBuilder.build(), coder); + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/ValuesAndContPosition.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/ValuesAndContPosition.java new file mode 100644 index 0000000000000..a1002fee43806 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/ValuesAndContPosition.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import java.util.List; +import javax.annotation.Nullable; + +/** + * An in-memory collection of deserialized values and an optional continuation position to pass to + * Windmill when fetching the next page of values. + */ +public class ValuesAndContPosition { + private final List values; + + /** Position to pass to next request for next page of values. Null if done. */ + private final @Nullable ContinuationT continuationPosition; + + public ValuesAndContPosition(List values, @Nullable ContinuationT continuationPosition) { + this.values = values; + this.continuationPosition = continuationPosition; + } + + public List getValues() { + return values; + } + + @Nullable + public ContinuationT getContinuationPosition() { + return continuationPosition; + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WeightedList.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WeightedList.java new file mode 100644 index 0000000000000..1b39d07c6fe7b --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WeightedList.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import java.util.List; +import org.apache.beam.sdk.util.Weighted; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ForwardingList; + +@VisibleForTesting +public class WeightedList extends ForwardingList implements Weighted { + private final List delegate; + long weight; + + WeightedList(List delegate) { + this.delegate = delegate; + this.weight = 0; + } + + @Override + protected List delegate() { + return delegate; + } + + @Override + public boolean add(T elem) { + throw new UnsupportedOperationException("Must use AddWeighted()"); + } + + @Override + public long getWeight() { + return weight; + } + + public void addWeighted(T elem, long weight) { + delegate.add(elem); + this.weight += weight; + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillBag.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillBag.java new file mode 100644 index 0000000000000..7cdb3776dfa18 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillBag.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import org.apache.beam.runners.core.StateNamespace; +import org.apache.beam.runners.core.StateTag; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.state.BagState; +import org.apache.beam.sdk.state.ReadableState; +import org.apache.beam.sdk.util.ByteStringOutputStream; +import org.apache.beam.sdk.util.Weighted; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; + +@SuppressWarnings({ + "rawtypes", // TODO(https://github.com/apache/beam/issues/20447) + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) +public class WindmillBag extends SimpleWindmillState implements BagState { + + private final StateNamespace namespace; + private final StateTag> address; + private final ByteString stateKey; + private final String stateFamily; + private final Coder elemCoder; + + private boolean cleared = false; + /** + * If non-{@literal null}, this contains the complete contents of the bag, except for any local + * additions. If {@literal null} then we don't know if Windmill contains additional values which + * should be part of the bag. We'll need to read them if the work item actually wants the bag + * contents. + */ + private ConcatIterables cachedValues = null; + + private List localAdditions = new ArrayList<>(); + private long encodedSize = 0; + + WindmillBag( + StateNamespace namespace, + StateTag> address, + String stateFamily, + Coder elemCoder, + boolean isNewKey) { + this.namespace = namespace; + this.address = address; + this.stateKey = WindmillStateUtil.encodeKey(namespace, address); + this.stateFamily = stateFamily; + this.elemCoder = elemCoder; + if (isNewKey) { + this.cachedValues = new ConcatIterables<>(); + } + } + + @Override + public void clear() { + cleared = true; + cachedValues = new ConcatIterables<>(); + localAdditions = new ArrayList<>(); + encodedSize = 0; + } + + /** + * Return iterable over all bag values in Windmill which should contribute to overall bag + * contents. + */ + private Iterable fetchData(Future> persistedData) { + try (Closeable scope = scopedReadState()) { + if (cachedValues != null) { + return cachedValues.snapshot(); + } + Iterable data = persistedData.get(); + if (data instanceof Weighted) { + // We have a known bounded amount of data; cache it. + cachedValues = new ConcatIterables<>(); + cachedValues.extendWith(data); + encodedSize = ((Weighted) data).getWeight(); + return cachedValues.snapshot(); + } else { + // This is an iterable that may not fit in memory at once; don't cache it. + return data; + } + } catch (InterruptedException | ExecutionException | IOException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new RuntimeException("Unable to read state", e); + } + } + + public boolean valuesAreCached() { + return cachedValues != null; + } + + @Override + @SuppressWarnings("FutureReturnValueIgnored") + public WindmillBag readLater() { + getFuture(); + return this; + } + + @Override + public Iterable read() { + return Iterables.concat( + fetchData(getFuture()), Iterables.limit(localAdditions, localAdditions.size())); + } + + @Override + public ReadableState isEmpty() { + return new ReadableState() { + @Override + public ReadableState readLater() { + WindmillBag.this.readLater(); + return this; + } + + @Override + public Boolean read() { + return Iterables.isEmpty(fetchData(getFuture())) && localAdditions.isEmpty(); + } + }; + } + + @Override + public void add(T input) { + localAdditions.add(input); + } + + @Override + public Windmill.WorkItemCommitRequest persistDirectly(WindmillStateCache.ForKeyAndFamily cache) + throws IOException { + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + + Windmill.TagBag.Builder bagUpdatesBuilder = null; + + if (cleared) { + bagUpdatesBuilder = commitBuilder.addBagUpdatesBuilder(); + bagUpdatesBuilder.setDeleteAll(true); + cleared = false; + } + + if (!localAdditions.isEmpty()) { + // Tell Windmill to capture the local additions. + if (bagUpdatesBuilder == null) { + bagUpdatesBuilder = commitBuilder.addBagUpdatesBuilder(); + } + for (T value : localAdditions) { + ByteStringOutputStream stream = new ByteStringOutputStream(); + // Encode the value + elemCoder.encode(value, stream, Coder.Context.OUTER); + ByteString encoded = stream.toByteString(); + if (cachedValues != null) { + // We'll capture this value in the cache below. + // Capture the value's size now since we have it. + encodedSize += encoded.size(); + } + bagUpdatesBuilder.addValues(encoded); + } + } + + if (bagUpdatesBuilder != null) { + bagUpdatesBuilder.setTag(stateKey).setStateFamily(stateFamily); + } + + if (cachedValues != null) { + if (!localAdditions.isEmpty()) { + // Capture the local additions in the cached value since we and + // Windmill are now in agreement. + cachedValues.extendWith(localAdditions); + } + // We now know the complete bag contents, and any read on it will yield a + // cached value, so cache it for future reads. + cache.put(namespace, address, this, encodedSize); + } + + // Don't reuse the localAdditions object; we don't want future changes to it to + // modify the value of cachedValues. + localAdditions = new ArrayList<>(); + + return commitBuilder.buildPartial(); + } + + private Future> getFuture() { + return cachedValues != null ? null : reader.bagFuture(stateKey, stateFamily, elemCoder); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillCombiningState.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillCombiningState.java new file mode 100644 index 0000000000000..98359913c7033 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillCombiningState.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.Future; +import javax.annotation.concurrent.NotThreadSafe; +import org.apache.beam.runners.core.StateNamespace; +import org.apache.beam.runners.core.StateTag; +import org.apache.beam.runners.core.StateTags; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.state.BagState; +import org.apache.beam.sdk.state.CombiningState; +import org.apache.beam.sdk.state.ReadableState; +import org.apache.beam.sdk.transforms.Combine; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; + +@NotThreadSafe +class WindmillCombiningState extends WindmillState + implements CombiningState { + + private final WindmillBag bag; + private final Combine.CombineFn combineFn; + + /* We use a separate, in-memory AccumT rather than relying on the WindmillWatermarkBag's + * localAdditions, because we want to combine multiple InputT's to a single AccumT + * before adding it. + */ + private AccumT localAdditionsAccumulator; + private boolean hasLocalAdditions; + + WindmillCombiningState( + StateNamespace namespace, + StateTag> address, + String stateFamily, + Coder accumCoder, + Combine.CombineFn combineFn, + WindmillStateCache.ForKeyAndFamily cache, + boolean isNewKey) { + StateTag> internalBagAddress = StateTags.convertToBagTagInternal(address); + this.bag = + cache + .get(namespace, internalBagAddress) + .map(state -> (WindmillBag) state) + .orElseGet( + () -> + new WindmillBag<>( + namespace, internalBagAddress, stateFamily, accumCoder, isNewKey)); + + this.combineFn = combineFn; + this.localAdditionsAccumulator = combineFn.createAccumulator(); + this.hasLocalAdditions = false; + } + + @Override + void initializeForWorkItem( + WindmillStateReader reader, Supplier scopedReadStateSupplier) { + super.initializeForWorkItem(reader, scopedReadStateSupplier); + this.bag.initializeForWorkItem(reader, scopedReadStateSupplier); + } + + @Override + void cleanupAfterWorkItem() { + super.cleanupAfterWorkItem(); + bag.cleanupAfterWorkItem(); + } + + @Override + public WindmillCombiningState readLater() { + bag.readLater(); + return this; + } + + @Override + @SuppressWarnings("nullness") + public OutputT read() { + return combineFn.extractOutput(getAccum()); + } + + @Override + public void add(InputT input) { + hasLocalAdditions = true; + localAdditionsAccumulator = combineFn.addInput(localAdditionsAccumulator, input); + } + + @Override + public void clear() { + bag.clear(); + localAdditionsAccumulator = combineFn.createAccumulator(); + hasLocalAdditions = false; + } + + @Override + public Future persist(WindmillStateCache.ForKeyAndFamily cache) + throws IOException { + if (hasLocalAdditions) { + if (WindmillStateInternals.COMPACT_NOW.get().get() || bag.valuesAreCached()) { + // Implicitly clears the bag and combines local and persisted accumulators. + localAdditionsAccumulator = getAccum(); + } + bag.add(combineFn.compact(localAdditionsAccumulator)); + localAdditionsAccumulator = combineFn.createAccumulator(); + hasLocalAdditions = false; + } + + return bag.persist(cache); + } + + @Override + public AccumT getAccum() { + Iterable accumulators = + Iterables.concat(bag.read(), Collections.singleton(localAdditionsAccumulator)); + + // Compact things + AccumT merged = combineFn.mergeAccumulators(accumulators); + bag.clear(); + localAdditionsAccumulator = merged; + hasLocalAdditions = true; + return merged; + } + + @Override + public ReadableState isEmpty() { + final ReadableState bagIsEmpty = bag.isEmpty(); + return new ReadableState() { + @Override + public ReadableState readLater() { + bagIsEmpty.readLater(); + return this; + } + + @Override + public Boolean read() { + return !hasLocalAdditions && bagIsEmpty.read(); + } + }; + } + + @Override + public void addAccum(AccumT accumulator) { + hasLocalAdditions = true; + localAdditionsAccumulator = + combineFn.mergeAccumulators(Arrays.asList(localAdditionsAccumulator, accumulator)); + } + + @Override + public AccumT mergeAccumulators(Iterable accumulators) { + return combineFn.mergeAccumulators(accumulators); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillMap.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillMap.java new file mode 100644 index 0000000000000..43490a725ac4f --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillMap.java @@ -0,0 +1,449 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import static org.apache.beam.runners.dataflow.worker.windmill.state.WindmillStateUtil.encodeKey; + +import java.io.Closeable; +import java.io.IOException; +import java.util.AbstractMap; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.function.Function; +import javax.annotation.Nullable; +import org.apache.beam.runners.core.StateNamespace; +import org.apache.beam.runners.core.StateTag; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.state.MapState; +import org.apache.beam.sdk.state.ReadableState; +import org.apache.beam.sdk.state.ReadableStates; +import org.apache.beam.sdk.util.ByteStringOutputStream; +import org.apache.beam.sdk.util.Weighted; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Futures; +import org.checkerframework.checker.initialization.qual.Initialized; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.UnknownKeyFor; + +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) +public class WindmillMap extends SimpleWindmillState implements MapState { + private final StateNamespace namespace; + private final StateTag> address; + private final ByteString stateKeyPrefix; + private final String stateFamily; + private final Coder keyCoder; + private final Coder valueCoder; + // TODO(reuvenlax): Should we evict items from the cache? We would have to make sure + // that anything in the cache that is not committed is not evicted. negativeCache could be + // evicted whenever we want. + private final Map cachedValues = Maps.newHashMap(); + private final Set negativeCache = Sets.newHashSet(); + private final Set localAdditions = Sets.newHashSet(); + private final Set localRemovals = Sets.newHashSet(); + private boolean complete; + private boolean cleared = false; + + WindmillMap( + StateNamespace namespace, + StateTag> address, + String stateFamily, + Coder keyCoder, + Coder valueCoder, + boolean isNewKey) { + this.namespace = namespace; + this.address = address; + this.stateKeyPrefix = encodeKey(namespace, address); + this.stateFamily = stateFamily; + this.keyCoder = keyCoder; + this.valueCoder = valueCoder; + this.complete = isNewKey; + } + + private K userKeyFromProtoKey(ByteString tag) throws IOException { + Preconditions.checkState(tag.startsWith(stateKeyPrefix)); + ByteString keyBytes = tag.substring(stateKeyPrefix.size()); + return keyCoder.decode(keyBytes.newInput(), Coder.Context.OUTER); + } + + private ByteString protoKeyFromUserKey(K key) throws IOException { + ByteStringOutputStream keyStream = new ByteStringOutputStream(); + stateKeyPrefix.writeTo(keyStream); + keyCoder.encode(key, keyStream, Coder.Context.OUTER); + return keyStream.toByteString(); + } + + @Override + protected Windmill.WorkItemCommitRequest persistDirectly(WindmillStateCache.ForKeyAndFamily cache) + throws IOException { + if (!cleared && localAdditions.isEmpty() && localRemovals.isEmpty()) { + // No changes, so return directly. + return Windmill.WorkItemCommitRequest.newBuilder().buildPartial(); + } + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + + if (cleared) { + commitBuilder + .addTagValuePrefixDeletesBuilder() + .setStateFamily(stateFamily) + .setTagPrefix(stateKeyPrefix); + } + cleared = false; + + for (K key : localAdditions) { + ByteString keyBytes = protoKeyFromUserKey(key); + ByteStringOutputStream valueStream = new ByteStringOutputStream(); + valueCoder.encode(cachedValues.get(key), valueStream, Coder.Context.OUTER); + ByteString valueBytes = valueStream.toByteString(); + + commitBuilder + .addValueUpdatesBuilder() + .setTag(keyBytes) + .setStateFamily(stateFamily) + .getValueBuilder() + .setData(valueBytes) + .setTimestamp(Long.MAX_VALUE); + } + localAdditions.clear(); + + for (K key : localRemovals) { + ByteStringOutputStream keyStream = new ByteStringOutputStream(); + stateKeyPrefix.writeTo(keyStream); + keyCoder.encode(key, keyStream, Coder.Context.OUTER); + ByteString keyBytes = keyStream.toByteString(); + // Leaving data blank means that we delete the tag. + commitBuilder + .addValueUpdatesBuilder() + .setTag(keyBytes) + .setStateFamily(stateFamily) + .getValueBuilder() + .setTimestamp(Long.MAX_VALUE); + + V cachedValue = cachedValues.remove(key); + if (cachedValue != null) { + ByteStringOutputStream valueStream = new ByteStringOutputStream(); + valueCoder.encode(cachedValues.get(key), valueStream, Coder.Context.OUTER); + } + } + negativeCache.addAll(localRemovals); + localRemovals.clear(); + + // TODO(reuvenlax): We should store in the cache parameter, as that would enable caching the + // map + // between work items, reducing fetches to Windmill. To do so, we need keep track of the + // encoded size + // of the map, and to do so efficiently (i.e. without iterating over the entire map on every + // persist) + // we need to track the sizes of each map entry. + cache.put(namespace, address, this, 1); + return commitBuilder.buildPartial(); + } + + @Override + public @UnknownKeyFor @NonNull @Initialized ReadableState get(K key) { + return getOrDefault(key, null); + } + + @Override + public @UnknownKeyFor @NonNull @Initialized ReadableState getOrDefault( + K key, @Nullable V defaultValue) { + return new WindmillMapReadResultReadableState(key, defaultValue); + } + + @Override + public @UnknownKeyFor @NonNull @Initialized ReadableState< + @UnknownKeyFor @NonNull @Initialized Iterable> + keys() { + ReadableState>> entries = entries(); + return new WindmillMapKeysReadableState(entries); + } + + @Override + public @UnknownKeyFor @NonNull @Initialized ReadableState< + @UnknownKeyFor @NonNull @Initialized Iterable> + values() { + ReadableState>> entries = entries(); + return new WindmillMapValuesReadableState(entries); + } + + @Override + public @UnknownKeyFor @NonNull @Initialized ReadableState< + @UnknownKeyFor @NonNull @Initialized Iterable< + Map.@UnknownKeyFor @NonNull @Initialized Entry>> + entries() { + return new WindmillMapEntriesReadableState(); + } + + @Override + public ReadableState isEmpty() { + return new WindmillMapIsEmptyReadableState(); + } + + @Override + public void put(K key, V value) { + V oldValue = cachedValues.put(key, value); + if (valueCoder.consistentWithEquals() && value.equals(oldValue)) { + return; + } + localAdditions.add(key); + localRemovals.remove(key); + negativeCache.remove(key); + } + + @Override + public @UnknownKeyFor @NonNull @Initialized ReadableState computeIfAbsent( + K key, Function mappingFunction) { + Future persistedData = getFutureForKey(key); + try (Closeable scope = scopedReadState()) { + if (localRemovals.contains(key) || negativeCache.contains(key)) { + return ReadableStates.immediate(null); + } + @Nullable V cachedValue = cachedValues.get(key); + if (cachedValue != null || complete) { + return ReadableStates.immediate(cachedValue); + } + + V persistedValue = persistedData.get(); + if (persistedValue == null) { + // This is a new value. Add it to the map and return null. + put(key, mappingFunction.apply(key)); + return ReadableStates.immediate(null); + } + // TODO: Don't do this if it was already in cache. + cachedValues.put(key, persistedValue); + return ReadableStates.immediate(persistedValue); + } catch (InterruptedException | ExecutionException | IOException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new RuntimeException("Unable to read state", e); + } + } + + @Override + public void remove(K key) { + if (localRemovals.add(key)) { + cachedValues.remove(key); + localAdditions.remove(key); + } + } + + @Override + public void clear() { + cachedValues.clear(); + localAdditions.clear(); + localRemovals.clear(); + negativeCache.clear(); + cleared = true; + complete = true; + } + + private Future getFutureForKey(K key) { + try { + ByteStringOutputStream keyStream = new ByteStringOutputStream(); + stateKeyPrefix.writeTo(keyStream); + keyCoder.encode(key, keyStream, Coder.Context.OUTER); + return reader.valueFuture(keyStream.toByteString(), stateFamily, valueCoder); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private Future>> getFuture() { + if (complete) { + // The caller will merge in local cached values. + return Futures.immediateFuture(Collections.emptyList()); + } else { + return reader.valuePrefixFuture(stateKeyPrefix, stateFamily, valueCoder); + } + } + + private class WindmillMapKeysReadableState implements ReadableState> { + private final ReadableState>> entries; + + public WindmillMapKeysReadableState(ReadableState>> entries) { + this.entries = entries; + } + + @Override + public Iterable read() { + return Iterables.transform(entries.read(), Map.Entry::getKey); + } + + @Override + public @UnknownKeyFor @NonNull @Initialized ReadableState> readLater() { + entries.readLater(); + return this; + } + } + + private class WindmillMapValuesReadableState implements ReadableState> { + private final ReadableState>> entries; + + public WindmillMapValuesReadableState(ReadableState>> entries) { + this.entries = entries; + } + + @Override + public @Nullable Iterable read() { + return Iterables.transform(entries.read(), Map.Entry::getValue); + } + + @Override + public @UnknownKeyFor @NonNull @Initialized ReadableState> readLater() { + entries.readLater(); + return this; + } + } + + private class WindmillMapEntriesReadableState + implements ReadableState>> { + @Override + public Iterable> read() { + if (complete) { + return Iterables.unmodifiableIterable(cachedValues.entrySet()); + } + Future>> persistedData = getFuture(); + try (Closeable scope = scopedReadState()) { + Iterable> data = persistedData.get(); + Iterable> transformedData = + Iterables.transform( + data, + entry -> { + try { + return new AbstractMap.SimpleEntry<>( + userKeyFromProtoKey(entry.getKey()), entry.getValue()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + if (data instanceof Weighted) { + // This is a known amount of data. Cache it all. + transformedData.forEach( + e -> { + // The cached data overrides what is read from state, so call putIfAbsent. + cachedValues.putIfAbsent(e.getKey(), e.getValue()); + }); + complete = true; + return Iterables.unmodifiableIterable(cachedValues.entrySet()); + } else { + // This means that the result might be too large to cache, so don't add it to the + // local cache. Instead merge the iterables, giving priority to any local additions + // (represented in cachedValued and localRemovals) that may not have been committed + // yet. + return Iterables.unmodifiableIterable( + Iterables.concat( + cachedValues.entrySet(), + Iterables.filter( + transformedData, + e -> + !cachedValues.containsKey(e.getKey()) + && !localRemovals.contains(e.getKey())))); + } + + } catch (InterruptedException | ExecutionException | IOException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new RuntimeException("Unable to read state", e); + } + } + + @Override + @SuppressWarnings("FutureReturnValueIgnored") + public @UnknownKeyFor @NonNull @Initialized ReadableState>> + readLater() { + WindmillMap.this.getFuture(); + return this; + } + } + + private class WindmillMapIsEmptyReadableState implements ReadableState { + // TODO(reuvenlax): Can we find a more efficient way of implementing isEmpty than reading + // the entire map? + final ReadableState> keys = WindmillMap.this.keys(); + + @Override + public @Nullable Boolean read() { + return Iterables.isEmpty(keys.read()); + } + + @Override + public @UnknownKeyFor @NonNull @Initialized ReadableState readLater() { + keys.readLater(); + return this; + } + } + + private class WindmillMapReadResultReadableState implements ReadableState { + private final K key; + private final @Nullable V defaultValue; + + public WindmillMapReadResultReadableState(K key, @Nullable V defaultValue) { + this.key = key; + this.defaultValue = defaultValue; + } + + @Override + public @Nullable V read() { + Future persistedData = getFutureForKey(key); + try (Closeable scope = scopedReadState()) { + if (localRemovals.contains(key) || negativeCache.contains(key)) { + return null; + } + @Nullable V cachedValue = cachedValues.get(key); + if (cachedValue != null || complete) { + return cachedValue; + } + + V persistedValue = persistedData.get(); + if (persistedValue == null) { + negativeCache.add(key); + return defaultValue; + } + // TODO: Don't do this if it was already in cache. + cachedValues.put(key, persistedValue); + return persistedValue; + } catch (InterruptedException | ExecutionException | IOException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new RuntimeException("Unable to read state", e); + } + } + + @Override + @SuppressWarnings("FutureReturnValueIgnored") + public @UnknownKeyFor @NonNull @Initialized ReadableState readLater() { + WindmillMap.this.getFutureForKey(key); + return this; + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillMultimap.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillMultimap.java new file mode 100644 index 0000000000000..1c0b3df44c21f --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillMultimap.java @@ -0,0 +1,732 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import static org.apache.beam.runners.dataflow.worker.windmill.state.WindmillStateUtil.encodeKey; + +import java.io.Closeable; +import java.io.IOException; +import java.util.AbstractMap; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.stream.Collectors; +import org.apache.beam.repackaged.core.org.apache.commons.lang3.tuple.Triple; +import org.apache.beam.runners.core.StateNamespace; +import org.apache.beam.runners.core.StateTag; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.state.MultimapState; +import org.apache.beam.sdk.state.ReadableState; +import org.apache.beam.sdk.util.ByteStringOutputStream; +import org.apache.beam.sdk.util.Weighted; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Futures; + +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) +public class WindmillMultimap extends SimpleWindmillState implements MultimapState { + + private final StateNamespace namespace; + private final StateTag> address; + private final ByteString stateKey; + private final String stateFamily; + private final Coder keyCoder; + private final Coder valueCoder; + // Set to true when user clears the entire multimap, so that we can later send delete request to + // the windmill backend. + private boolean cleared = false; + // We use the structural value of the keys as the key in keyStateMap, so that different java + // Objects with the same content will be treated as the same Multimap key. + private Map keyStateMap = Maps.newHashMap(); + // If true, all keys are cached in keyStateMap with existence == KNOWN_EXIST. + private boolean allKeysKnown; + // True if all contents of this multimap are cached in this object. + private boolean complete; + // hasLocalAdditions and hasLocalRemovals track whether there are local changes that needs to be + // propagated to windmill. + private boolean hasLocalAdditions = false; + private boolean hasLocalRemovals = false; + + WindmillMultimap( + StateNamespace namespace, + StateTag> address, + String stateFamily, + Coder keyCoder, + Coder valueCoder, + boolean isNewShardingKey) { + this.namespace = namespace; + this.address = address; + this.stateKey = encodeKey(namespace, address); + this.stateFamily = stateFamily; + this.keyCoder = keyCoder; + this.valueCoder = valueCoder; + this.complete = isNewShardingKey; + this.allKeysKnown = isNewShardingKey; + } + + private static Iterable> unnestCachedEntries( + Iterable>>> cachedEntries) { + return Iterables.concat( + Iterables.transform( + cachedEntries, + entry -> + Iterables.transform( + entry.getValue().getRight(), + v -> new AbstractMap.SimpleEntry<>(entry.getValue().getLeft(), v)))); + } + + @Override + public void put(K key, V value) { + final Object structuralKey = keyCoder.structuralValue(key); + hasLocalAdditions = true; + keyStateMap.compute( + structuralKey, + (k, v) -> { + if (v == null) v = new KeyState(key); + v.existence = KeyExistence.KNOWN_EXIST; + v.localAdditions.add(value); + return v; + }); + } + + // Initiates a backend state read to fetch all entries if necessary. + private Future>>> necessaryEntriesFromStorageFuture( + boolean omitValues) { + if (complete) { + // Since we're complete, even if there are entries in storage we don't need to read them. + return Futures.immediateFuture(Collections.emptyList()); + } else { + return reader.multimapFetchAllFuture(omitValues, stateKey, stateFamily, valueCoder); + } + } + + // Initiates a backend state read to fetch a single entry if necessary. + private Future> necessaryKeyEntriesFromStorageFuture(K key) { + try { + ByteStringOutputStream keyStream = new ByteStringOutputStream(); + keyCoder.encode(key, keyStream, Coder.Context.OUTER); + return reader.multimapFetchSingleEntryFuture( + keyStream.toByteString(), stateKey, stateFamily, valueCoder); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public ReadableState> get(K key) { + return new ReadResultReadableState(key); + } + + @Override + protected Windmill.WorkItemCommitRequest persistDirectly(WindmillStateCache.ForKeyAndFamily cache) + throws IOException { + if (!cleared && !hasLocalAdditions && !hasLocalRemovals) { + cache.put(namespace, address, this, 1); + return Windmill.WorkItemCommitRequest.newBuilder().buildPartial(); + } + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + Windmill.TagMultimapUpdateRequest.Builder builder = commitBuilder.addMultimapUpdatesBuilder(); + builder.setTag(stateKey).setStateFamily(stateFamily); + + if (cleared) { + builder.setDeleteAll(true); + } + if (hasLocalRemovals || hasLocalAdditions) { + ByteStringOutputStream keyStream = new ByteStringOutputStream(); + ByteStringOutputStream valueStream = new ByteStringOutputStream(); + Iterator> iterator = keyStateMap.entrySet().iterator(); + while (iterator.hasNext()) { + KeyState keyState = iterator.next().getValue(); + if (!keyState.removedLocally && keyState.localAdditions.isEmpty()) { + if (keyState.existence == KeyExistence.KNOWN_NONEXISTENT) iterator.remove(); + continue; + } + keyCoder.encode(keyState.originalKey, keyStream, Coder.Context.OUTER); + ByteString encodedKey = keyStream.toByteStringAndReset(); + Windmill.TagMultimapEntry.Builder entryBuilder = builder.addUpdatesBuilder(); + entryBuilder.setEntryName(encodedKey); + if (keyState.removedLocally) entryBuilder.setDeleteAll(true); + keyState.removedLocally = false; + if (!keyState.localAdditions.isEmpty()) { + for (V value : keyState.localAdditions) { + valueCoder.encode(value, valueStream, Coder.Context.OUTER); + ByteString encodedValue = valueStream.toByteStringAndReset(); + entryBuilder.addValues(encodedValue); + } + // Move newly added values from localAdditions to keyState.values as those new values + // now + // are also persisted in Windmill. If a key now has no more values and is not + // KNOWN_EXIST, + // remove it from cache. + if (keyState.valuesCached) { + keyState.values.extendWith(keyState.localAdditions); + keyState.valuesSize += keyState.localAdditions.size(); + } + // Create a new localAdditions so that the cached values are unaffected. + keyState.localAdditions = Lists.newArrayList(); + } + if (!keyState.valuesCached && keyState.existence != KeyExistence.KNOWN_EXIST) { + iterator.remove(); + } + } + } + + hasLocalAdditions = false; + hasLocalRemovals = false; + cleared = false; + + cache.put(namespace, address, this, 1); + return commitBuilder.buildPartial(); + } + + @Override + public void remove(K key) { + final Object structuralKey = keyCoder.structuralValue(key); + // does not insert key if allKeysKnown. + KeyState keyState = + keyStateMap.computeIfAbsent(structuralKey, k -> allKeysKnown ? null : new KeyState(key)); + if (keyState == null || keyState.existence == KeyExistence.KNOWN_NONEXISTENT) { + return; + } + if (keyState.valuesCached && keyState.valuesSize == 0) { + // no data in windmill, deleting from local cache is sufficient. + keyStateMap.remove(structuralKey); + } else { + // there may be data in windmill that need to be removed. + hasLocalRemovals = true; + keyState.removedLocally = true; + keyState.values = new ConcatIterables<>(); + keyState.valuesSize = 0; + keyState.existence = KeyExistence.KNOWN_NONEXISTENT; + } + if (!keyState.localAdditions.isEmpty()) { + keyState.localAdditions = Lists.newArrayList(); + } + keyState.valuesCached = true; + } + + @Override + public void clear() { + keyStateMap = Maps.newHashMap(); + cleared = true; + complete = true; + allKeysKnown = true; + hasLocalAdditions = false; + hasLocalRemovals = false; + } + + @Override + public ReadableState> keys() { + return new KeysReadableState(); + } + + @Override + public ReadableState>> entries() { + return new EntriesReadableState(); + } + + @Override + public ReadableState containsKey(K key) { + return new ContainsKeyReadableState(key); + } + + // Currently, isEmpty is implemented by reading all keys and could potentially be optimized. + // But note that if isEmpty is often followed by iterating over keys then maybe not too bad; if + // isEmpty is followed by iterating over both keys and values then it won't help much. + @Override + public ReadableState isEmpty() { + return new IsEmptyReadableState(); + } + + private enum KeyExistence { + // this key is known to exist, it has at least 1 value in either localAdditions or windmill + KNOWN_EXIST, + // this key is known to be nonexistent, it has 0 value in both localAdditions and windmill + KNOWN_NONEXISTENT, + // we don't know if this key is in this multimap, it has exact 0 value in localAddition, but + // may have no or any number of values in windmill. This is just to provide a mapping between + // the original key and the structural key. + UNKNOWN_EXISTENCE + } + + private class KeyState { + final K originalKey; + KeyExistence existence; + // valuesCached can be true if only existence == KNOWN_EXIST and all values of this key are + // cached (both values and localAdditions). + boolean valuesCached; + // Represents the values in windmill. When new values are added during user processing, they + // are added to localAdditions but not values. Those new values will be added to values only + // after they are persisted into windmill and removed from localAdditions + ConcatIterables values; + int valuesSize; + + // When new values are added during user processing, they are added to localAdditions, so that + // we can later try to persist them in windmill. When a key is removed during user processing, + // we mark removedLocally to be true so that we can later try to delete it from windmill. If + // localAdditions is not empty and removedLocally is true, values in localAdditions will be + // added to windmill after old values in windmill are removed. + List localAdditions; + boolean removedLocally; + + KeyState(K originalKey) { + this.originalKey = originalKey; + existence = KeyExistence.UNKNOWN_EXISTENCE; + valuesCached = complete; + values = new ConcatIterables<>(); + valuesSize = 0; + localAdditions = Lists.newArrayList(); + removedLocally = false; + } + } + + private class ReadResultReadableState implements ReadableState> { + final Object structuralKey; + private final K key; + + public ReadResultReadableState(K key) { + this.key = key; + structuralKey = keyCoder.structuralValue(key); + } + + @Override + public Iterable read() { + KeyState keyState = null; + if (allKeysKnown) { + keyState = keyStateMap.get(structuralKey); + if (keyState == null || keyState.existence == KeyExistence.UNKNOWN_EXISTENCE) { + if (keyState != null) keyStateMap.remove(structuralKey); + return Collections.emptyList(); + } + } else { + keyState = keyStateMap.computeIfAbsent(structuralKey, k -> new KeyState(key)); + } + if (keyState.existence == KeyExistence.KNOWN_NONEXISTENT) { + return Collections.emptyList(); + } + Iterable localNewValues = + Iterables.limit(keyState.localAdditions, keyState.localAdditions.size()); + if (keyState.removedLocally) { + // this key has been removed locally but the removal hasn't been sent to windmill, + // thus values in windmill(if any) are obsolete, and we only care about local values. + return Iterables.unmodifiableIterable(localNewValues); + } + if (keyState.valuesCached || complete) { + return Iterables.unmodifiableIterable( + Iterables.concat( + Iterables.limit(keyState.values, keyState.valuesSize), localNewValues)); + } + Future> persistedData = necessaryKeyEntriesFromStorageFuture(key); + try (Closeable scope = scopedReadState()) { + final Iterable persistedValues = persistedData.get(); + // Iterables.isEmpty() is O(1). + if (Iterables.isEmpty(persistedValues)) { + if (keyState.localAdditions.isEmpty()) { + // empty in both cache and windmill, mark key as KNOWN_NONEXISTENT. + keyState.existence = KeyExistence.KNOWN_NONEXISTENT; + return Collections.emptyList(); + } + return Iterables.unmodifiableIterable(localNewValues); + } + keyState.existence = KeyExistence.KNOWN_EXIST; + if (persistedValues instanceof Weighted) { + keyState.valuesCached = true; + ConcatIterables it = new ConcatIterables<>(); + it.extendWith(persistedValues); + keyState.values = it; + keyState.valuesSize = Iterables.size(persistedValues); + } + return Iterables.unmodifiableIterable(Iterables.concat(persistedValues, localNewValues)); + } catch (InterruptedException | ExecutionException | IOException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new RuntimeException("Unable to read Multimap state", e); + } + } + + @Override + @SuppressWarnings("FutureReturnValueIgnored") + public ReadableState> readLater() { + WindmillMultimap.this.necessaryKeyEntriesFromStorageFuture(key); + return this; + } + } + + private class KeysReadableState implements ReadableState> { + + private Map cachedExistKeys() { + return keyStateMap.entrySet().stream() + .filter(entry -> entry.getValue().existence == KeyExistence.KNOWN_EXIST) + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().originalKey)); + } + + @Override + public Iterable read() { + if (allKeysKnown) { + return Iterables.unmodifiableIterable(cachedExistKeys().values()); + } + Future>>> persistedData = + necessaryEntriesFromStorageFuture(true); + try (Closeable scope = scopedReadState()) { + Iterable>> entries = persistedData.get(); + if (entries instanceof Weighted) { + // This is a known amount of data, cache them all. + entries.forEach( + entry -> { + try { + K originalKey = keyCoder.decode(entry.getKey().newInput(), Coder.Context.OUTER); + KeyState keyState = + keyStateMap.computeIfAbsent( + keyCoder.structuralValue(originalKey), stk -> new KeyState(originalKey)); + if (keyState.existence == KeyExistence.UNKNOWN_EXISTENCE) { + keyState.existence = KeyExistence.KNOWN_EXIST; + } + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + allKeysKnown = true; + keyStateMap + .values() + .removeIf( + keyState -> + keyState.existence != KeyExistence.KNOWN_EXIST && !keyState.removedLocally); + return Iterables.unmodifiableIterable(cachedExistKeys().values()); + } else { + Map cachedExistKeys = Maps.newHashMap(); + Set cachedNonExistKeys = Sets.newHashSet(); + keyStateMap.forEach( + (structuralKey, keyState) -> { + switch (keyState.existence) { + case KNOWN_EXIST: + cachedExistKeys.put(structuralKey, keyState.originalKey); + break; + case KNOWN_NONEXISTENT: + cachedNonExistKeys.add(structuralKey); + break; + default: + break; + } + }); + // keysOnlyInWindmill is lazily loaded. + Iterable keysOnlyInWindmill = + Iterables.filter( + Iterables.transform( + entries, + entry -> { + try { + K originalKey = + keyCoder.decode(entry.getKey().newInput(), Coder.Context.OUTER); + Object structuralKey = keyCoder.structuralValue(originalKey); + if (cachedExistKeys.containsKey(structuralKey) + || cachedNonExistKeys.contains(structuralKey)) return null; + return originalKey; + } catch (IOException e) { + throw new RuntimeException(e); + } + }), + Objects::nonNull); + return Iterables.unmodifiableIterable( + Iterables.concat(cachedExistKeys.values(), keysOnlyInWindmill)); + } + } catch (InterruptedException | ExecutionException | IOException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new RuntimeException("Unable to read state", e); + } + } + + @Override + @SuppressWarnings("FutureReturnValueIgnored") + public ReadableState> readLater() { + WindmillMultimap.this.necessaryEntriesFromStorageFuture(true); + return this; + } + } + + private class EntriesReadableState implements ReadableState>> { + @Override + public Iterable> read() { + if (complete) { + return Iterables.unmodifiableIterable( + unnestCachedEntries(mergedCachedEntries(null).entrySet())); + } + Future>>> persistedData = + necessaryEntriesFromStorageFuture(false); + try (Closeable scope = scopedReadState()) { + Iterable>> entries = persistedData.get(); + if (Iterables.isEmpty(entries)) { + complete = true; + allKeysKnown = true; + return Iterables.unmodifiableIterable( + unnestCachedEntries(mergedCachedEntries(null).entrySet())); + } + if (!(entries instanceof Weighted)) { + return nonWeightedEntries(entries); + } + // This is a known amount of data, cache them all. + entries.forEach( + entry -> { + try { + final K originalKey = + keyCoder.decode(entry.getKey().newInput(), Coder.Context.OUTER); + final Object structuralKey = keyCoder.structuralValue(originalKey); + KeyState keyState = + keyStateMap.computeIfAbsent(structuralKey, k -> new KeyState(originalKey)); + // Ignore any key from windmill that has been marked pending deletion or is + // fully cached. + if (keyState.existence == KeyExistence.KNOWN_NONEXISTENT + || (keyState.existence == KeyExistence.KNOWN_EXIST && keyState.valuesCached)) + return; + // Or else cache contents from windmill. + keyState.existence = KeyExistence.KNOWN_EXIST; + keyState.values.extendWith(entry.getValue()); + keyState.valuesSize += Iterables.size(entry.getValue()); + keyState.valuesCached = true; + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + allKeysKnown = true; + complete = true; + return Iterables.unmodifiableIterable( + unnestCachedEntries(mergedCachedEntries(null).entrySet())); + } catch (InterruptedException | ExecutionException | IOException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new RuntimeException("Unable to read state", e); + } + } + + @Override + @SuppressWarnings("FutureReturnValueIgnored") + public ReadableState>> readLater() { + WindmillMultimap.this.necessaryEntriesFromStorageFuture(false); + return this; + } + + /** + * Collect all cached entries into a map and all KNOWN_NONEXISTENT keys to + * knownNonexistentKeys(if not null). Note that this method is not side-effect-free: it unloads + * any key that is not KNOWN_EXIST and not pending deletion from cache; also if complete it + * marks the valuesCached of any key that is KNOWN_EXIST to true, entries() depends on this + * behavior when the fetched result is weighted to iterate the whole keyStateMap one less time. + * For each cached key, returns its structural key and a tuple of . + */ + private Map>> mergedCachedEntries( + Set knownNonexistentKeys) { + Map>> cachedEntries = Maps.newHashMap(); + keyStateMap + .entrySet() + .removeIf( + (entry -> { + Object structuralKey = entry.getKey(); + KeyState keyState = entry.getValue(); + if (complete && keyState.existence == KeyExistence.KNOWN_EXIST) { + keyState.valuesCached = true; + } + ConcatIterables it = null; + if (!keyState.localAdditions.isEmpty()) { + it = new ConcatIterables<>(); + it.extendWith( + Iterables.limit(keyState.localAdditions, keyState.localAdditions.size())); + } + if (keyState.valuesCached) { + if (it == null) it = new ConcatIterables<>(); + it.extendWith(Iterables.limit(keyState.values, keyState.valuesSize)); + } + if (it != null) { + cachedEntries.put( + structuralKey, Triple.of(keyState.originalKey, keyState.valuesCached, it)); + } + if (knownNonexistentKeys != null + && keyState.existence == KeyExistence.KNOWN_NONEXISTENT) + knownNonexistentKeys.add(structuralKey); + return (keyState.existence == KeyExistence.KNOWN_NONEXISTENT + && !keyState.removedLocally) + || keyState.existence == KeyExistence.UNKNOWN_EXISTENCE; + })); + return cachedEntries; + } + + private Iterable> nonWeightedEntries( + Iterable>> lazyWindmillEntries) { + class ResultIterable implements Iterable> { + private final Iterable>> lazyWindmillEntries; + private final Map>> cachedEntries; + private final Set knownNonexistentKeys; + + ResultIterable( + Map>> cachedEntries, + Iterable>> lazyWindmillEntries, + Set knownNonexistentKeys) { + this.cachedEntries = cachedEntries; + this.lazyWindmillEntries = lazyWindmillEntries; + this.knownNonexistentKeys = knownNonexistentKeys; + } + + @Override + public Iterator> iterator() { + // Each time when the Iterable returned by entries() is iterated, a new Iterator is + // created. Every iterator must keep its own copy of seenCachedKeys so that if a key + // is paginated into multiple iterables from windmill, the cached values of this key + // will only be returned once. + Set seenCachedKeys = Sets.newHashSet(); + // notFullyCachedEntries returns all entries from windmill that are not fully cached + // and combines them with localAdditions. If a key is fully cached, contents of this + // key from windmill are ignored. + Iterable>> notFullyCachedEntries = + Iterables.filter( + Iterables.transform( + lazyWindmillEntries, + entry -> { + try { + final K key = + keyCoder.decode(entry.getKey().newInput(), Coder.Context.OUTER); + final Object structuralKey = keyCoder.structuralValue(key); + // key is deleted in cache thus fully cached. + if (knownNonexistentKeys.contains(structuralKey)) return null; + Triple> triple = + cachedEntries.get(structuralKey); + // no record of key in cache, return content in windmill. + if (triple == null) { + return Triple.of(structuralKey, key, entry.getValue()); + } + // key is fully cached in cache. + if (triple.getMiddle()) return null; + + // key is not fully cached, combine the content in windmill with local + // additions with only the first observed page for the key to ensure + // it is not repeated. + if (!seenCachedKeys.add(structuralKey)) { + return Triple.of(structuralKey, key, entry.getValue()); + } else { + ConcatIterables it = new ConcatIterables<>(); + it.extendWith(triple.getRight()); + it.extendWith(entry.getValue()); + return Triple.of(structuralKey, key, it); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + }), + Objects::nonNull); + Iterator> unnestWindmill = + Iterators.concat( + Iterables.transform( + notFullyCachedEntries, + entry -> + Iterables.transform( + entry.getRight(), + v -> new AbstractMap.SimpleEntry<>(entry.getMiddle(), v)) + .iterator()) + .iterator()); + Iterator> fullyCached = + unnestCachedEntries( + Iterables.filter( + cachedEntries.entrySet(), + entry -> !seenCachedKeys.contains(entry.getKey()))) + .iterator(); + return Iterators.concat(unnestWindmill, fullyCached); + } + } + + Set knownNonexistentKeys = Sets.newHashSet(); + Map>> cachedEntries = + mergedCachedEntries(knownNonexistentKeys); + return Iterables.unmodifiableIterable( + new ResultIterable(cachedEntries, lazyWindmillEntries, knownNonexistentKeys)); + } + } + + private class ContainsKeyReadableState implements ReadableState { + final Object structuralKey; + private final K key; + ReadableState> values; + + public ContainsKeyReadableState(K key) { + this.key = key; + structuralKey = keyCoder.structuralValue(key); + values = null; + } + + @Override + public Boolean read() { + KeyState keyState = keyStateMap.getOrDefault(structuralKey, null); + if (keyState != null && keyState.existence != KeyExistence.UNKNOWN_EXISTENCE) { + return keyState.existence == KeyExistence.KNOWN_EXIST; + } + if (values == null) { + values = WindmillMultimap.this.get(key); + } + return !Iterables.isEmpty(values.read()); + } + + @Override + public ReadableState readLater() { + if (values == null) { + values = WindmillMultimap.this.get(key); + } + values.readLater(); + return this; + } + } + + private class IsEmptyReadableState implements ReadableState { + ReadableState> keys = null; + + @Override + public Boolean read() { + for (KeyState keyState : keyStateMap.values()) { + if (keyState.existence == KeyExistence.KNOWN_EXIST) { + return false; + } + } + if (keys == null) { + keys = WindmillMultimap.this.keys(); + } + return Iterables.isEmpty(keys.read()); + } + + @Override + public ReadableState readLater() { + if (keys == null) { + keys = WindmillMultimap.this.keys(); + } + keys.readLater(); + return this; + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillOrderedList.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillOrderedList.java new file mode 100644 index 0000000000000..c92e2e93ddfec --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillOrderedList.java @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import static org.apache.beam.runners.dataflow.worker.windmill.state.WindmillStateUtil.encodeKey; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; +import java.util.SortedSet; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import org.apache.beam.runners.core.StateNamespace; +import org.apache.beam.runners.core.StateTable; +import org.apache.beam.runners.core.StateTag; +import org.apache.beam.runners.dataflow.worker.WindmillTimeUtils; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.state.OrderedListState; +import org.apache.beam.sdk.state.ReadableState; +import org.apache.beam.sdk.util.ByteStringOutputStream; +import org.apache.beam.sdk.values.TimestampedValue; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Range; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.RangeSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.TreeRangeSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Futures; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Instant; + +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) +public class WindmillOrderedList extends SimpleWindmillState implements OrderedListState { + // The default proto values for SortedListRange correspond to the minimum and maximum + // timestamps. + static final long MIN_TS_MICROS = Windmill.SortedListRange.getDefaultInstance().getStart(); + static final long MAX_TS_MICROS = Windmill.SortedListRange.getDefaultInstance().getLimit(); + private final ByteString stateKey; + private final String stateFamily; + private final Coder elemCoder; + // We need to sort based on timestamp, but we need objects with the same timestamp to be treated + // as unique. We can't use a MultiSet as we can't construct a comparator that uniquely + // identifies objects, + // so we construct a unique in-memory long ids for each element. + private final SortedSet> pendingAdds = + Sets.newTreeSet(TimestampedValueWithId.COMPARATOR); + private final RangeSet pendingDeletes = TreeRangeSet.create(); + private final IdTracker idTracker; + private boolean complete; + private boolean cleared = false; + + WindmillOrderedList( + StateTable derivedStateTable, + StateNamespace namespace, + StateTag> spec, + String stateFamily, + Coder elemCoder, + boolean isNewKey) { + + this.stateKey = encodeKey(namespace, spec); + this.stateFamily = stateFamily; + this.elemCoder = elemCoder; + this.complete = isNewKey; + this.idTracker = new IdTracker(derivedStateTable, namespace, spec); + } + + @Override + public Iterable> read() { + return readRange(null, null); + } + + private SortedSet> getPendingAddRange( + @Nullable Instant minTimestamp, @Nullable Instant limitTimestamp) { + SortedSet> pendingInRange = pendingAdds; + if (minTimestamp != null && limitTimestamp != null) { + pendingInRange = + pendingInRange.subSet( + TimestampedValueWithId.bound(minTimestamp), + TimestampedValueWithId.bound(limitTimestamp)); + } else if (minTimestamp == null && limitTimestamp != null) { + pendingInRange = pendingInRange.headSet(TimestampedValueWithId.bound(limitTimestamp)); + } else if (limitTimestamp == null && minTimestamp != null) { + pendingInRange = pendingInRange.tailSet(TimestampedValueWithId.bound(minTimestamp)); + } + return pendingInRange; + } + + @Override + public Iterable> readRange( + @Nullable Instant minTimestamp, @Nullable Instant limitTimestamp) { + idTracker.readLater(); + + final Future>> future = getFuture(minTimestamp, limitTimestamp); + try (Closeable ignored = scopedReadState()) { + SortedSet> pendingInRange = + getPendingAddRange(minTimestamp, limitTimestamp); + + // Transform the return iterator, so it has the same type as pendingAdds. We need to ensure + // that the ids don't overlap with any in pendingAdds, so begin with pendingAdds.size(). + Iterable> data = + new Iterable>() { + // Anything returned from windmill that has been deleted should be ignored. + private final Iterable> iterable = + Iterables.filter(future.get(), tv -> !pendingDeletes.contains(tv.getTimestamp())); + + @Override + public Iterator> iterator() { + return new Iterator>() { + private final Iterator> iter = iterable.iterator(); + private long currentId = pendingAdds.size(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public TimestampedValueWithId next() { + return TimestampedValueWithId.of(iter.next(), currentId++); + } + }; + } + }; + + Iterable> includingAdds = + Iterables.mergeSorted( + ImmutableList.of(data, pendingInRange), TimestampedValueWithId.COMPARATOR); + + // TODO(reuvenlax): If we have a known bounded amount of data, cache known ranges. + return Iterables.transform(includingAdds, TimestampedValueWithId::getValue); + } catch (InterruptedException | ExecutionException | IOException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new RuntimeException("Unable to read state", e); + } + } + + @Override + public void clear() { + cleared = true; + complete = true; + pendingAdds.clear(); + pendingDeletes.clear(); + try { + idTracker.clear(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + @Override + public void clearRange(Instant minTimestamp, Instant limitTimestamp) { + getPendingAddRange(minTimestamp, limitTimestamp).clear(); + pendingDeletes.add(Range.closedOpen(minTimestamp, limitTimestamp)); + } + + @Override + public void add(TimestampedValue value) { + // We use the current size of the container as the in-memory id. This works because + // pendingAdds is completely + // cleared when it is processed (otherwise we could end up with duplicate elements in the same + // container). These + // are not the ids that will be sent to windmill. + pendingAdds.add(TimestampedValueWithId.of(value, pendingAdds.size())); + // Leave pendingDeletes alone. Since we can have multiple values with the same timestamp, we + // may still need + // overlapping deletes to remove previous entries at this timestamp. + } + + @Override + public ReadableState isEmpty() { + return new ReadableState() { + @Override + public ReadableState readLater() { + WindmillOrderedList.this.readLater(); + return this; + } + + @Override + public Boolean read() { + return Iterables.isEmpty(WindmillOrderedList.this.read()); + } + }; + } + + @Override + public OrderedListState readLater() { + return readRangeLater(null, null); + } + + @Override + @SuppressWarnings("FutureReturnValueIgnored") + public OrderedListState readRangeLater( + @Nullable Instant minTimestamp, @Nullable Instant limitTimestamp) { + idTracker.readLater(); + getFuture(minTimestamp, limitTimestamp); + return this; + } + + @Override + public Windmill.WorkItemCommitRequest persistDirectly(WindmillStateCache.ForKeyAndFamily cache) + throws IOException { + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + Windmill.TagSortedListUpdateRequest.Builder updatesBuilder = + commitBuilder + .addSortedListUpdatesBuilder() + .setStateFamily(cache.getStateFamily()) + .setTag(stateKey); + try { + if (cleared) { + // Default range. + updatesBuilder.addDeletesBuilder().build(); + cleared = false; + } + + if (!pendingAdds.isEmpty()) { + // TODO(reuvenlax): Once we start caching data, we should remove this line. We have it + // here now + // because once we persist + // added data we forget about it from the cache, so the object is no longer complete. + complete = false; + + Windmill.TagSortedListInsertRequest.Builder insertBuilder = + updatesBuilder.addInsertsBuilder(); + idTracker.add( + pendingAdds, + (elem, id) -> { + try { + ByteStringOutputStream elementStream = new ByteStringOutputStream(); + elemCoder.encode(elem.getValue(), elementStream, Coder.Context.OUTER); + insertBuilder.addEntries( + Windmill.SortedListEntry.newBuilder() + .setValue(elementStream.toByteString()) + .setSortKey( + WindmillTimeUtils.harnessToWindmillTimestamp(elem.getTimestamp())) + .setId(id)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + pendingAdds.clear(); + insertBuilder.build(); + } + + if (!pendingDeletes.isEmpty()) { + for (Range range : pendingDeletes.asRanges()) { + Windmill.TagSortedListDeleteRequest.Builder deletesBuilder = + updatesBuilder.addDeletesBuilder(); + deletesBuilder.setRange( + Windmill.SortedListRange.newBuilder() + .setStart(WindmillTimeUtils.harnessToWindmillTimestamp(range.lowerEndpoint())) + .setLimit(WindmillTimeUtils.harnessToWindmillTimestamp(range.upperEndpoint()))); + deletesBuilder.build(); + idTracker.remove(range); + } + pendingDeletes.clear(); + } + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException(e); + } + return commitBuilder.buildPartial(); + } + + private Future>> getFuture( + @Nullable Instant minTimestamp, @Nullable Instant limitTimestamp) { + long startSortKey = + minTimestamp != null + ? WindmillTimeUtils.harnessToWindmillTimestamp(minTimestamp) + : MIN_TS_MICROS; + long limitSortKey = + limitTimestamp != null + ? WindmillTimeUtils.harnessToWindmillTimestamp(limitTimestamp) + : MAX_TS_MICROS; + + if (complete) { + // Right now we don't cache any data, so complete means an empty list. + // TODO(reuvenlax): change this once we start caching data. + return Futures.immediateFuture(Collections.emptyList()); + } + return reader.orderedListFuture( + Range.closedOpen(startSortKey, limitSortKey), stateKey, stateFamily, elemCoder); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillSet.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillSet.java new file mode 100644 index 0000000000000..4afb879e722e9 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillSet.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Optional; +import org.apache.beam.runners.core.StateNamespace; +import org.apache.beam.runners.core.StateTag; +import org.apache.beam.runners.core.StateTags; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.sdk.coders.BooleanCoder; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.state.MapState; +import org.apache.beam.sdk.state.ReadableState; +import org.apache.beam.sdk.state.SetState; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.checkerframework.checker.initialization.qual.Initialized; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.UnknownKeyFor; + +public class WindmillSet extends SimpleWindmillState implements SetState { + private final WindmillMap windmillMap; + + WindmillSet( + StateNamespace namespace, + StateTag> address, + String stateFamily, + Coder keyCoder, + WindmillStateCache.ForKeyAndFamily cache, + boolean isNewKey) { + StateTag> internalMapAddress = StateTags.convertToMapTagInternal(address); + + this.windmillMap = + cache + .get(namespace, internalMapAddress) + .map(map -> (WindmillMap) map) + .orElseGet( + () -> + new WindmillMap<>( + namespace, + internalMapAddress, + stateFamily, + keyCoder, + BooleanCoder.of(), + isNewKey)); + } + + @Override + protected Windmill.WorkItemCommitRequest persistDirectly(WindmillStateCache.ForKeyAndFamily cache) + throws IOException { + return windmillMap.persistDirectly(cache); + } + + @Override + public @UnknownKeyFor @NonNull @Initialized ReadableState< + @UnknownKeyFor @NonNull @Initialized Boolean> + contains(K k) { + return windmillMap.getOrDefault(k, false); + } + + @Override + public @UnknownKeyFor @NonNull @Initialized ReadableState< + @UnknownKeyFor @NonNull @Initialized Boolean> + addIfAbsent(K k) { + return new WindmillSetAddIfAbsentReadableState(k); + } + + @Override + public void remove(K k) { + windmillMap.remove(k); + } + + @Override + public void add(K value) { + windmillMap.put(value, true); + } + + @Override + public @UnknownKeyFor @NonNull @Initialized ReadableState< + @UnknownKeyFor @NonNull @Initialized Boolean> + isEmpty() { + return windmillMap.isEmpty(); + } + + @Override + public Iterable read() { + return windmillMap.keys().read(); + } + + @Override + public @UnknownKeyFor @NonNull @Initialized SetState readLater() { + windmillMap.keys().readLater(); + return this; + } + + @Override + public void clear() { + windmillMap.clear(); + } + + @Override + void initializeForWorkItem( + WindmillStateReader reader, Supplier scopedReadStateSupplier) { + windmillMap.initializeForWorkItem(reader, scopedReadStateSupplier); + } + + @Override + void cleanupAfterWorkItem() { + windmillMap.cleanupAfterWorkItem(); + } + + private class WindmillSetAddIfAbsentReadableState implements ReadableState { + ReadableState putState; + + public WindmillSetAddIfAbsentReadableState(K k) { + putState = windmillMap.putIfAbsent(k, true); + } + + @Override + public Boolean read() { + return Optional.ofNullable(putState.read()).orElse(false); + } + + @Override + public @UnknownKeyFor @NonNull @Initialized ReadableState readLater() { + putState = putState.readLater(); + return this; + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillState.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillState.java new file mode 100644 index 0000000000000..59fd3f8a1b379 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillState.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import java.io.Closeable; +import java.io.IOException; +import java.util.concurrent.Future; +import javax.annotation.concurrent.NotThreadSafe; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; + +/** + * Abstract base class for all Windmill state. + * + *

    Note that these are not thread safe; each state object is associated with a key and thus only + * accessed by a single thread at once. + */ +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) +@NotThreadSafe +abstract class WindmillState { + protected Supplier scopedReadStateSupplier; + protected WindmillStateReader reader; + + /** + * Return an asynchronously computed {@link Windmill.WorkItemCommitRequest}. The request should be + * of a form that can be merged with others (only add to repeated fields). + */ + abstract Future persist(WindmillStateCache.ForKeyAndFamily cache) + throws IOException; + + /** Prepare this (possibly reused from cache) state for reading from {@code reader} if needed. */ + void initializeForWorkItem( + WindmillStateReader reader, Supplier scopedReadStateSupplier) { + this.reader = reader; + this.scopedReadStateSupplier = scopedReadStateSupplier; + } + + /** + * This (now cached) state should never need to interact with the reader until the next work item. + * Clear it to prevent space leaks. The reader will be reset by {@link #initializeForWorkItem} + * upon the next work item. + */ + void cleanupAfterWorkItem() { + this.reader = null; + this.scopedReadStateSupplier = null; + } + + Closeable scopedReadState() { + return scopedReadStateSupplier.get(); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillStateCache.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateCache.java similarity index 88% rename from runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillStateCache.java rename to runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateCache.java index 6ae0031cb1795..6c1239d6ebd2f 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillStateCache.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateCache.java @@ -15,12 +15,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.runners.dataflow.worker; +package org.apache.beam.runners.dataflow.worker.windmill.state; import java.io.IOException; import java.io.PrintWriter; import java.util.HashMap; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ConcurrentMap; import java.util.function.BiConsumer; import javax.servlet.http.HttpServletRequest; @@ -28,18 +29,21 @@ import org.apache.beam.runners.core.StateNamespace; import org.apache.beam.runners.core.StateTag; import org.apache.beam.runners.core.StateTags; +import org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker; +import org.apache.beam.runners.dataflow.worker.Weighers; +import org.apache.beam.runners.dataflow.worker.WindmillComputationKey; import org.apache.beam.runners.dataflow.worker.status.BaseStatusServlet; import org.apache.beam.runners.dataflow.worker.status.StatusDataProvider; import org.apache.beam.sdk.state.State; import org.apache.beam.sdk.util.Weighted; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Equivalence; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheStats; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Weigher; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.MapMaker; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Equivalence; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheStats; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Weigher; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.MapMaker; import org.checkerframework.checker.nullness.qual.Nullable; /** @@ -84,14 +88,6 @@ public WindmillStateCache(long workerCacheMb) { .build(); } - private static class EntryStats { - long entries; - long idWeight; - long entryWeight; - long entryValues; - long maxEntryValues; - } - private EntryStats calculateEntryStats() { EntryStats stats = new EntryStats(); BiConsumer consumer = @@ -119,130 +115,52 @@ public CacheStats getCacheStats() { return stateCache.stats(); } - /** Per-computation view of the state cache. */ - public class ForComputation { - - private final String computation; - - private ForComputation(String computation) { - this.computation = computation; - } - - /** Invalidate all cache entries for this computation and {@code processingKey}. */ - public void invalidate(ByteString processingKey, long shardingKey) { - WindmillComputationKey key = - WindmillComputationKey.create(computation, processingKey, shardingKey); - // By removing the ForKey object, all state for the key is orphaned in the cache and will - // be removed by normal cache cleanup. - keyIndex.remove(key); - } - - /** - * Returns a per-computation, per-key view of the state cache. Access to the cached data for - * this key is not thread-safe. Callers should ensure that there is only a single ForKey object - * in use at a time and that access to it is synchronized or single-threaded. - */ - public ForKey forKey(WindmillComputationKey computationKey, long cacheToken, long workToken) { - ForKey forKey = keyIndex.get(computationKey); - if (forKey == null || !forKey.updateTokens(cacheToken, workToken)) { - forKey = new ForKey(computationKey, cacheToken, workToken); - // We prefer this implementation to using compute because that is implemented similarly for - // ConcurrentHashMap with the downside of it performing inserts for unchanged existing - // values as well. - keyIndex.put(computationKey, forKey); - } - return forKey; - } + /** Returns a per-computation view of the state cache. */ + public ForComputation forComputation(String computation) { + return new ForComputation(computation); } - /** Per-computation, per-key view of the state cache. */ - // Note that we utilize the default equality and hashCode for this class based upon the instance - // (instead of the fields) to optimize cache invalidation. - public class ForKey { - private final WindmillComputationKey computationKey; - // Cache token must be consistent for the key for the cache to be valid. - private final long cacheToken; - - // The work token for processing must be greater than the last work token. As work items are - // increasing for a key, a less-than or equal to work token indicates that the current token is - // for stale processing. - private long workToken; - - /** - * Returns a per-computation, per-key, per-family view of the state cache. Access to the cached - * data for this key is not thread-safe. Callers should ensure that there is only a single - * ForKeyAndFamily object in use at a time for a given computation, key, family tuple and that - * access to it is synchronized or single-threaded. - */ - public ForKeyAndFamily forFamily(String stateFamily) { - return new ForKeyAndFamily(this, stateFamily); - } - - private ForKey(WindmillComputationKey computationKey, long cacheToken, long workToken) { - this.computationKey = computationKey; - this.cacheToken = cacheToken; - this.workToken = workToken; - } - - private boolean updateTokens(long cacheToken, long workToken) { - if (this.cacheToken != cacheToken || workToken <= this.workToken) { - return false; - } - this.workToken = workToken; - return true; - } + /** Print summary statistics of the cache to the given {@link PrintWriter}. */ + @Override + public void appendSummaryHtml(PrintWriter response) { + response.println("Cache Stats:
    "); + response.println( + "" + + "" + + "" + + ""); + CacheStats cacheStats = stateCache.stats(); + EntryStats entryStats = calculateEntryStats(); + response.println(""); + response.println(""); + response.println(""); + response.println(""); + response.println(""); + response.println(""); + response.println(""); + response.println(""); + response.println(""); + response.println("
    Hit RatioEvictionsEntriesEntry ValuesMax Entry ValuesId WeightEntry WeightMax WeightKeys
    " + cacheStats.hitRate() + "" + cacheStats.evictionCount() + "" + entryStats.entries + "(" + stateCache.size() + " inc. weak) " + entryStats.entryValues + "" + entryStats.maxEntryValues + "" + entryStats.idWeight / MEGABYTES + "MB" + entryStats.entryWeight / MEGABYTES + "MB" + getMaxWeight() / MEGABYTES + "MB" + keyIndex.size() + "

    "); } - /** - * Per-computation, per-key, per-family view of the state cache. Modifications are cached locally - * and must be flushed to the cache by calling persist. This class is not thread-safe. - */ - public class ForKeyAndFamily { - final ForKey forKey; - final String stateFamily; - private final HashMap localCache; - - private ForKeyAndFamily(ForKey forKey, String stateFamily) { - this.forKey = forKey; - this.stateFamily = stateFamily; - localCache = new HashMap<>(); - } - - public String getStateFamily() { - return stateFamily; - } - - public @Nullable T get(StateNamespace namespace, StateTag address) { - StateId id = new StateId(forKey, stateFamily, namespace); - @SuppressWarnings("nullness") // Unsure how to annotate lambda return allowing null. - @Nullable - StateCacheEntry entry = localCache.computeIfAbsent(id, key -> stateCache.getIfPresent(key)); - return entry == null ? null : entry.get(namespace, address); - } - - public void put( - StateNamespace namespace, StateTag address, T value, long weight) { - StateId id = new StateId(forKey, stateFamily, namespace); - @Nullable StateCacheEntry entry = localCache.get(id); - if (entry == null) { - entry = stateCache.getIfPresent(id); - if (entry == null) { - entry = new StateCacheEntry(); - } - boolean hadValue = localCache.putIfAbsent(id, entry) != null; - Preconditions.checkState(!hadValue); + public BaseStatusServlet statusServlet() { + return new BaseStatusServlet("/cachez") { + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws IOException { + PrintWriter writer = response.getWriter(); + writer.println("

    Cache Information

    "); + appendSummaryHtml(writer); } - entry.put(namespace, address, value, weight); - } - - public void persist() { - localCache.forEach((id, entry) -> stateCache.put(id, entry)); - } + }; } - /** Returns a per-computation view of the state cache. */ - public ForComputation forComputation(String computation) { - return new ForComputation(computation); + private static class EntryStats { + long entries; + long idWeight; + long entryWeight; + long entryValues; + long maxEntryValues; } /** @@ -297,12 +215,10 @@ public StateCacheEntry() { this.weight = 0; } - public @Nullable T get(StateNamespace namespace, StateTag tag) { - @SuppressWarnings("unchecked") - @Nullable - WeightedValue weightedValue = - (WeightedValue) values.get(new NamespacedTag<>(namespace, tag)); - return weightedValue == null ? null : weightedValue.value; + @SuppressWarnings("unchecked") + public Optional get(StateNamespace namespace, StateTag tag) { + return Optional.ofNullable((WeightedValue) values.get(new NamespacedTag<>(namespace, tag))) + .flatMap(WeightedValue::value); } public void put( @@ -362,43 +278,137 @@ public int hashCode() { } private static class WeightedValue { - public long weight; - public @Nullable T value; + private long weight; + private @Nullable T value; + + private Optional value() { + return Optional.ofNullable(this.value); + } } } - /** Print summary statistics of the cache to the given {@link PrintWriter}. */ - @Override - public void appendSummaryHtml(PrintWriter response) { - response.println("Cache Stats:
    "); - response.println( - "" - + "" - + "" - + ""); - CacheStats cacheStats = stateCache.stats(); - EntryStats entryStats = calculateEntryStats(); - response.println(""); - response.println(""); - response.println(""); - response.println(""); - response.println(""); - response.println(""); - response.println(""); - response.println(""); - response.println(""); - response.println("
    Hit RatioEvictionsEntriesEntry ValuesMax Entry ValuesId WeightEntry WeightMax WeightKeys
    " + cacheStats.hitRate() + "" + cacheStats.evictionCount() + "" + entryStats.entries + "(" + stateCache.size() + " inc. weak) " + entryStats.entryValues + "" + entryStats.maxEntryValues + "" + entryStats.idWeight / MEGABYTES + "MB" + entryStats.entryWeight / MEGABYTES + "MB" + getMaxWeight() / MEGABYTES + "MB" + keyIndex.size() + "

    "); + /** Per-computation view of the state cache. */ + public class ForComputation { + + private final String computation; + + private ForComputation(String computation) { + this.computation = computation; + } + + /** Invalidate all cache entries for this computation and {@code processingKey}. */ + public void invalidate(ByteString processingKey, long shardingKey) { + WindmillComputationKey key = + WindmillComputationKey.create(computation, processingKey, shardingKey); + // By removing the ForKey object, all state for the key is orphaned in the cache and will + // be removed by normal cache cleanup. + keyIndex.remove(key); + } + + /** + * Returns a per-computation, per-key view of the state cache. Access to the cached data for + * this key is not thread-safe. Callers should ensure that there is only a single ForKey object + * in use at a time and that access to it is synchronized or single-threaded. + */ + public ForKey forKey(WindmillComputationKey computationKey, long cacheToken, long workToken) { + ForKey forKey = keyIndex.get(computationKey); + if (forKey == null || !forKey.updateTokens(cacheToken, workToken)) { + forKey = new ForKey(computationKey, cacheToken, workToken); + // We prefer this implementation to using compute because that is implemented similarly for + // ConcurrentHashMap with the downside of it performing inserts for unchanged existing + // values as well. + keyIndex.put(computationKey, forKey); + } + return forKey; + } } - public BaseStatusServlet statusServlet() { - return new BaseStatusServlet("/cachez") { - @Override - protected void doGet(HttpServletRequest request, HttpServletResponse response) - throws IOException { - PrintWriter writer = response.getWriter(); - writer.println("

    Cache Information

    "); - appendSummaryHtml(writer); + /** Per-computation, per-key view of the state cache. */ + // Note that we utilize the default equality and hashCode for this class based upon the instance + // (instead of the fields) to optimize cache invalidation. + public class ForKey { + private final WindmillComputationKey computationKey; + // Cache token must be consistent for the key for the cache to be valid. + private final long cacheToken; + + // The work token for processing must be greater than the last work token. As work items are + // increasing for a key, a less-than or equal to work token indicates that the current token is + // for stale processing. + private long workToken; + + private ForKey(WindmillComputationKey computationKey, long cacheToken, long workToken) { + this.computationKey = computationKey; + this.cacheToken = cacheToken; + this.workToken = workToken; + } + + /** + * Returns a per-computation, per-key, per-family view of the state cache. Access to the cached + * data for this key is not thread-safe. Callers should ensure that there is only a single + * ForKeyAndFamily object in use at a time for a given computation, key, family tuple and that + * access to it is synchronized or single-threaded. + */ + public ForKeyAndFamily forFamily(String stateFamily) { + return new ForKeyAndFamily(this, stateFamily); + } + + private boolean updateTokens(long cacheToken, long workToken) { + if (this.cacheToken != cacheToken || workToken <= this.workToken) { + return false; } - }; + this.workToken = workToken; + return true; + } + } + + /** + * Per-computation, per-key, per-family view of the state cache. Modifications are cached locally + * and must be flushed to the cache by calling persist. This class is not thread-safe. + */ + public class ForKeyAndFamily { + final ForKey forKey; + final String stateFamily; + private final HashMap localCache; + + private ForKeyAndFamily(ForKey forKey, String stateFamily) { + this.forKey = forKey; + this.stateFamily = stateFamily; + localCache = new HashMap<>(); + } + + public String getStateFamily() { + return stateFamily; + } + + public Optional get(StateNamespace namespace, StateTag address) { + @SuppressWarnings("nullness") + // the mapping function for localCache.computeIfAbsent (i.e stateCache.getIfPresent) is + // nullable. + Optional stateCacheEntry = + Optional.ofNullable( + localCache.computeIfAbsent( + new StateId(forKey, stateFamily, namespace), stateCache::getIfPresent)); + + return stateCacheEntry.flatMap(entry -> entry.get(namespace, address)); + } + + public void put( + StateNamespace namespace, StateTag address, T value, long weight) { + StateId id = new StateId(forKey, stateFamily, namespace); + @Nullable StateCacheEntry entry = localCache.get(id); + if (entry == null) { + entry = stateCache.getIfPresent(id); + if (entry == null) { + entry = new StateCacheEntry(); + } + boolean hadValue = localCache.putIfAbsent(id, entry) != null; + Preconditions.checkState(!hadValue); + } + entry.put(namespace, address, value, weight); + } + + public void persist() { + localCache.forEach(stateCache::put); + } } } diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateInternals.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateInternals.java new file mode 100644 index 0000000000000..c900228e86b02 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateInternals.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import org.apache.beam.runners.core.StateInternals; +import org.apache.beam.runners.core.StateNamespace; +import org.apache.beam.runners.core.StateTable; +import org.apache.beam.runners.core.StateTag; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.WorkItemCommitRequest; +import org.apache.beam.sdk.state.State; +import org.apache.beam.sdk.state.StateContext; +import org.apache.beam.sdk.state.StateContexts; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** Implementation of {@link StateInternals} using Windmill to manage the underlying data. */ +@SuppressWarnings("nullness" // TODO(https://github.com/apache/beam/issues/20497) +) +public class WindmillStateInternals implements StateInternals { + + @VisibleForTesting + static final ThreadLocal> COMPACT_NOW = + ThreadLocal.withInitial(ShouldCompactNowFn::new); + /** + * The key will be null when not in a keyed context, from the users perspective. There is still a + * "key" for the Windmill computation, but it cannot be meaningfully deserialized. + */ + private final @Nullable K key; + + private final WindmillStateCache.ForKeyAndFamily cache; + private final StateTable workItemState; + private final StateTable workItemDerivedState; + private final Supplier scopedReadStateSupplier; + + public WindmillStateInternals( + @Nullable K key, + String stateFamily, + WindmillStateReader reader, + boolean isNewKey, + WindmillStateCache.ForKeyAndFamily cache, + Supplier scopedReadStateSupplier) { + this.key = key; + this.cache = cache; + this.scopedReadStateSupplier = scopedReadStateSupplier; + this.workItemDerivedState = + CachingStateTable.builder(stateFamily, reader, cache, isNewKey, scopedReadStateSupplier) + .build(); + this.workItemState = + CachingStateTable.builder(stateFamily, reader, cache, isNewKey, scopedReadStateSupplier) + .withDerivedState(workItemDerivedState) + .build(); + } + + @Override + public @Nullable K getKey() { + return key; + } + + private void persist(List> commitsToMerge, StateTable stateTable) { + for (State location : stateTable.values()) { + if (!(location instanceof WindmillState)) { + throw new IllegalStateException( + String.format( + "%s wasn't created by %s -- unable to persist it", + location.getClass().getSimpleName(), getClass().getSimpleName())); + } + + try { + commitsToMerge.add(((WindmillState) location).persist(cache)); + } catch (IOException e) { + throw new RuntimeException("Unable to persist state", e); + } + } + + // All cached State objects now have known values. + // Clear any references to the underlying reader to prevent space leaks. + // The next work unit to use these cached State objects will reset the + // reader to a current reader in case those values are modified. + for (State location : stateTable.values()) { + ((WindmillState) location).cleanupAfterWorkItem(); + } + + // Clear out the map of already retrieved state instances. + stateTable.clear(); + } + + public void persist(final Windmill.WorkItemCommitRequest.Builder commitBuilder) { + List> commitsToMerge = new ArrayList<>(); + + // Call persist on each first, which may schedule some futures for reading. + persist(commitsToMerge, workItemState); + persist(commitsToMerge, workItemDerivedState); + + try (Closeable ignored = scopedReadStateSupplier.get()) { + for (Future commitFuture : commitsToMerge) { + commitBuilder.mergeFrom(commitFuture.get()); + } + } catch (ExecutionException | InterruptedException | IOException exc) { + if (exc instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new RuntimeException("Failed to retrieve Windmill state during persist()", exc); + } + + cache.persist(); + } + + @Override + public T state(StateNamespace namespace, StateTag address) { + return workItemState.get(namespace, address, StateContexts.nullContext()); + } + + @Override + public T state( + StateNamespace namespace, StateTag address, StateContext c) { + return workItemState.get(namespace, address, c); + } + + private static class ShouldCompactNowFn implements Supplier { + /* The rate at which, on average, this will return true. */ + private static final double RATE = 0.002; + private final Random random; + private long counter; + + private ShouldCompactNowFn() { + this.random = new Random(); + this.counter = nextSample(random); + } + + private static long nextSample(Random random) { + // Use geometric distribution to find next true value. + // This lets us avoid invoking random.nextDouble() on every call. + return (long) Math.floor(Math.log(random.nextDouble()) / Math.log(1 - RATE)); + } + + @Override + public Boolean get() { + counter--; + if (counter < 0) { + counter = nextSample(random); + return true; + } else { + return false; + } + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateReader.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateReader.java new file mode 100644 index 0000000000000..07d652992c1cc --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateReader.java @@ -0,0 +1,950 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import com.google.api.client.util.Lists; +import java.io.IOException; +import java.io.InputStream; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.apache.beam.runners.dataflow.worker.KeyTokenInvalidException; +import org.apache.beam.runners.dataflow.worker.MetricTrackingWindmillServerStub; +import org.apache.beam.runners.dataflow.worker.WindmillTimeUtils; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataResponse; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.SortedListEntry; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.SortedListRange; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.TagBag; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.TagSortedListFetchRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.TagValue; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.TagValuePrefixRequest; +import org.apache.beam.runners.dataflow.worker.windmill.state.StateTag.Kind; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.coders.Coder.Context; +import org.apache.beam.sdk.transforms.windowing.BoundedWindow; +import org.apache.beam.sdk.values.TimestampedValue; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Range; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Futures; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.SettableFuture; +import org.joda.time.Instant; + +/** + * Reads persistent state from {@link Windmill}. Returns {@code Future}s containing the data that + * has been read. Will not initiate a read until {@link Future#get} is called, at which point all + * the pending futures will be read. + * + *

    CAUTION Watch out for escaping references to the reader ending up inside {@link + * WindmillStateCache}. + */ +@SuppressWarnings({ + "rawtypes", // TODO(https://github.com/apache/beam/issues/20447) + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) +public class WindmillStateReader { + /** + * Ideal maximum bytes in a TagBag response. However, Windmill will always return at least one + * value if possible irrespective of this limit. + */ + public static final long INITIAL_MAX_BAG_BYTES = 8L << 20; // 8MB + + public static final long CONTINUATION_MAX_BAG_BYTES = 32L << 20; // 32MB + + /** + * Ideal maximum bytes in a TagMultimapFetchResponse response. However, Windmill will always + * return at least one value if possible irrespective of this limit. + */ + public static final long INITIAL_MAX_MULTIMAP_BYTES = 8L << 20; // 8MB + + public static final long CONTINUATION_MAX_MULTIMAP_BYTES = 32L << 20; // 32MB + + /** + * Ideal maximum bytes in a TagSortedList response. However, Windmill will always return at least + * one value if possible irrespective of this limit. + */ + public static final long MAX_ORDERED_LIST_BYTES = 8L << 20; // 8MB + + /** + * Ideal maximum bytes in a tag-value prefix response. However, Windmill will always return at + * least one value if possible irrespective of this limit. + */ + public static final long MAX_TAG_VALUE_PREFIX_BYTES = 8L << 20; // 8MB + + /** + * Ideal maximum bytes in a KeyedGetDataResponse. However, Windmill will always return at least + * one value if possible irrespective of this limit. + */ + public static final long MAX_KEY_BYTES = 16L << 20; // 16MB + + public static final long MAX_CONTINUATION_KEY_BYTES = 72L << 20; // 72MB + @VisibleForTesting final ConcurrentLinkedQueue> pendingLookups; + private final String computation; + private final ByteString key; + private final long shardingKey; + private final long workToken; + // WindmillStateReader should only perform blocking i/o in a try-with-resources block that + // declares an AutoCloseable vended by readWrapperSupplier. + private final Supplier readWrapperSupplier; + private final MetricTrackingWindmillServerStub metricTrackingWindmillServerStub; + private final ConcurrentHashMap, CoderAndFuture> waiting; + private long bytesRead = 0L; + + public WindmillStateReader( + MetricTrackingWindmillServerStub metricTrackingWindmillServerStub, + String computation, + ByteString key, + long shardingKey, + long workToken, + Supplier readWrapperSupplier) { + this.metricTrackingWindmillServerStub = metricTrackingWindmillServerStub; + this.computation = computation; + this.key = key; + this.shardingKey = shardingKey; + this.workToken = workToken; + this.readWrapperSupplier = readWrapperSupplier; + this.waiting = new ConcurrentHashMap<>(); + this.pendingLookups = new ConcurrentLinkedQueue<>(); + } + + public WindmillStateReader( + MetricTrackingWindmillServerStub metricTrackingWindmillServerStub, + String computation, + ByteString key, + long shardingKey, + long workToken) { + this(metricTrackingWindmillServerStub, computation, key, shardingKey, workToken, () -> null); + } + + private Future stateFuture(StateTag stateTag, @Nullable Coder coder) { + CoderAndFuture coderAndFuture = new CoderAndFuture<>(coder, SettableFuture.create()); + CoderAndFuture existingCoderAndFutureWildcard = + waiting.putIfAbsent(stateTag, coderAndFuture); + if (existingCoderAndFutureWildcard == null) { + // Schedule a new request. Its response is guaranteed to find the future and coder. + pendingLookups.add(stateTag); + } else { + // Piggy-back on the pending or already answered request. + @SuppressWarnings("unchecked") + CoderAndFuture existingCoderAndFuture = + (CoderAndFuture) existingCoderAndFutureWildcard; + coderAndFuture = existingCoderAndFuture; + } + + return wrappedFuture(coderAndFuture.getFuture()); + } + + private CoderAndFuture getWaiting(StateTag stateTag, boolean shouldRemove) { + CoderAndFuture coderAndFutureWildcard; + if (shouldRemove) { + coderAndFutureWildcard = waiting.remove(stateTag); + } else { + coderAndFutureWildcard = waiting.get(stateTag); + } + if (coderAndFutureWildcard == null) { + throw new IllegalStateException("Missing future for " + stateTag); + } + @SuppressWarnings("unchecked") + CoderAndFuture coderAndFuture = (CoderAndFuture) coderAndFutureWildcard; + return coderAndFuture; + } + + public Future watermarkFuture(ByteString encodedTag, String stateFamily) { + return stateFuture(StateTag.of(StateTag.Kind.WATERMARK, encodedTag, stateFamily), null); + } + + public Future valueFuture(ByteString encodedTag, String stateFamily, Coder coder) { + return stateFuture(StateTag.of(StateTag.Kind.VALUE, encodedTag, stateFamily), coder); + } + + public Future> bagFuture( + ByteString encodedTag, String stateFamily, Coder elemCoder) { + // First request has no continuation position. + StateTag stateTag = StateTag.of(StateTag.Kind.BAG, encodedTag, stateFamily); + // Convert the ValuesAndContPosition to Iterable. + return valuesToPagingIterableFuture(stateTag, elemCoder, this.stateFuture(stateTag, elemCoder)); + } + + public Future>> orderedListFuture( + Range range, ByteString encodedTag, String stateFamily, Coder elemCoder) { + // First request has no continuation position. + StateTag stateTag = + StateTag.of(StateTag.Kind.ORDERED_LIST, encodedTag, stateFamily) + .toBuilder() + .setSortedListRange(Preconditions.checkNotNull(range)) + .build(); + return valuesToPagingIterableFuture(stateTag, elemCoder, this.stateFuture(stateTag, elemCoder)); + } + + public Future>>> multimapFetchAllFuture( + boolean omitValues, ByteString encodedTag, String stateFamily, Coder elemCoder) { + StateTag stateTag = + StateTag.of(Kind.MULTIMAP_ALL, encodedTag, stateFamily) + .toBuilder() + .setOmitValues(omitValues) + .build(); + return valuesToPagingIterableFuture(stateTag, elemCoder, this.stateFuture(stateTag, elemCoder)); + } + + public Future> multimapFetchSingleEntryFuture( + ByteString encodedKey, ByteString encodedTag, String stateFamily, Coder elemCoder) { + StateTag stateTag = + StateTag.of(Kind.MULTIMAP_SINGLE_ENTRY, encodedTag, stateFamily) + .toBuilder() + .setMultimapKey(encodedKey) + .build(); + return valuesToPagingIterableFuture(stateTag, elemCoder, this.stateFuture(stateTag, elemCoder)); + } + + public Future>> valuePrefixFuture( + ByteString prefix, String stateFamily, Coder valueCoder) { + // First request has no continuation position. + StateTag stateTag = + StateTag.of(Kind.VALUE_PREFIX, prefix, stateFamily).toBuilder().build(); + return valuesToPagingIterableFuture( + stateTag, valueCoder, this.stateFuture(stateTag, valueCoder)); + } + + /** + * Internal request to fetch the next 'page' of values. Return null if no continuation position is + * in {@code contStateTag}, which signals there are no more pages. + */ + @Nullable + Future> continuationFuture( + StateTag contStateTag, Coder coder) { + if (contStateTag.getRequestPosition() == null) { + // We're done. + return null; + } + return stateFuture(contStateTag, coder); + } + + private Future wrappedFuture(final Future future) { + if (future.isDone()) { + // If the underlying lookup is already complete, we don't need to create the wrapper. + return future; + } else { + // Otherwise, wrap the true future so we know when to trigger a GetData. + return new WrappedFuture<>(this, future); + } + } + + /** + * Return future which transforms a {@code ValuesAndContPosition} result into the initial + * Iterable result expected from the external caller. + */ + private Future> valuesToPagingIterableFuture( + final StateTag stateTag, + final Coder coder, + final Future> future) { + Function, Iterable> toIterable = + new ToIterableFunction<>(this, stateTag, coder); + return Futures.lazyTransform(future, toIterable); + } + + private void delayUnbatchableMultimapFetches( + List> multimapTags, HashSet> toFetch) { + // Each KeyedGetDataRequest can have at most 1 TagMultimapFetchRequest per + // pair, thus we need to delay unbatchable multimap requests of the same stateFamily and tag + // into later batches. There's no priority between get()/entries()/keys(), they will be fetched + // based on the order they occur in pendingLookups, so that all requests can eventually be + // fetched and none starves. + + Map>>> groupedTags = + multimapTags.stream() + .collect( + Collectors.groupingBy( + StateTag::getStateFamily, Collectors.groupingBy(StateTag::getTag))); + + for (Map>> familyTags : groupedTags.values()) { + for (List> tags : familyTags.values()) { + StateTag first = tags.remove(0); + toFetch.add(first); + if (tags.isEmpty()) continue; + + if (first.getKind() == Kind.MULTIMAP_ALL) { + // first is keys()/entries(), no more TagMultimapFetchRequests allowed in current batch. + pendingLookups.addAll(tags); + continue; + } + // first is get(key), no keys()/entries() allowed in current batch; each different key can + // have at most one fetch request in this batch. + Set addedKeys = Sets.newHashSet(); + addedKeys.add(first.getMultimapKey()); + for (StateTag other : tags) { + if (other.getKind() == Kind.MULTIMAP_ALL || addedKeys.contains(other.getMultimapKey())) { + pendingLookups.add(other); + } else { + toFetch.add(other); + addedKeys.add(other.getMultimapKey()); + } + } + } + } + } + + public void startBatchAndBlock() { + // First, drain work out of the pending lookups into a set. These will be the items we fetch. + HashSet> toFetch = Sets.newHashSet(); + try { + List> multimapTags = Lists.newArrayList(); + while (!pendingLookups.isEmpty()) { + StateTag stateTag = pendingLookups.poll(); + if (stateTag == null) { + break; + } + if (stateTag.getKind() == Kind.MULTIMAP_ALL + || stateTag.getKind() == Kind.MULTIMAP_SINGLE_ENTRY) { + multimapTags.add(stateTag); + continue; + } + if (!toFetch.add(stateTag)) { + throw new IllegalStateException("Duplicate tags being fetched."); + } + } + if (!multimapTags.isEmpty()) { + delayUnbatchableMultimapFetches(multimapTags, toFetch); + } + + // If we failed to drain anything, some other thread pulled it off the queue. We have no work + // to do. + if (toFetch.isEmpty()) { + return; + } + + KeyedGetDataResponse response = tryGetDataFromWindmill(toFetch); + + // Removes tags from toFetch as they are processed. + consumeResponse(response, toFetch); + } catch (Exception e) { + // Set up all the remaining futures for this key to throw an exception. This ensures that if + // the exception is caught that all futures have been completed and do not block. + for (StateTag stateTag : toFetch) { + waiting.get(stateTag).future.setException(e); + } + + throw new RuntimeException(e); + } + } + + private KeyedGetDataResponse tryGetDataFromWindmill(HashSet> stateTags) + throws Exception { + KeyedGetDataRequest keyedGetDataRequest = createRequest(stateTags); + try (AutoCloseable ignored = readWrapperSupplier.get()) { + return Optional.ofNullable( + metricTrackingWindmillServerStub.getStateData(computation, keyedGetDataRequest)) + .orElseThrow( + () -> + new RuntimeException( + "Windmill unexpectedly returned null for request " + keyedGetDataRequest)); + } + } + + public long getBytesRead() { + return bytesRead; + } + + private KeyedGetDataRequest createRequest(Iterable> toFetch) { + KeyedGetDataRequest.Builder keyedDataBuilder = + KeyedGetDataRequest.newBuilder() + .setKey(key) + .setShardingKey(shardingKey) + .setWorkToken(workToken); + + boolean continuation = false; + List> orderedListsToFetch = Lists.newArrayList(); + List> multimapSingleEntryToFetch = Lists.newArrayList(); + for (StateTag stateTag : toFetch) { + switch (stateTag.getKind()) { + case BAG: + TagBag.Builder bag = + keyedDataBuilder + .addBagsToFetchBuilder() + .setTag(stateTag.getTag()) + .setStateFamily(stateTag.getStateFamily()); + if (stateTag.getRequestPosition() == null) { + bag.setFetchMaxBytes(INITIAL_MAX_BAG_BYTES); + } else { + // We're asking for the next page. + bag.setFetchMaxBytes(CONTINUATION_MAX_BAG_BYTES); + bag.setRequestPosition((Long) stateTag.getRequestPosition()); + continuation = true; + } + break; + + case ORDERED_LIST: + orderedListsToFetch.add(stateTag); + break; + + case WATERMARK: + keyedDataBuilder + .addWatermarkHoldsToFetchBuilder() + .setTag(stateTag.getTag()) + .setStateFamily(stateTag.getStateFamily()); + break; + + case VALUE: + keyedDataBuilder + .addValuesToFetchBuilder() + .setTag(stateTag.getTag()) + .setStateFamily(stateTag.getStateFamily()); + break; + + case VALUE_PREFIX: + TagValuePrefixRequest.Builder prefixFetchBuilder = + keyedDataBuilder + .addTagValuePrefixesToFetchBuilder() + .setTagPrefix(stateTag.getTag()) + .setStateFamily(stateTag.getStateFamily()) + .setFetchMaxBytes(MAX_TAG_VALUE_PREFIX_BYTES); + if (stateTag.getRequestPosition() != null) { + prefixFetchBuilder.setRequestPosition((ByteString) stateTag.getRequestPosition()); + } + break; + + case MULTIMAP_SINGLE_ENTRY: + multimapSingleEntryToFetch.add(stateTag); + break; + + case MULTIMAP_ALL: + Windmill.TagMultimapFetchRequest.Builder multimapFetchBuilder = + keyedDataBuilder + .addMultimapsToFetchBuilder() + .setTag(stateTag.getTag()) + .setStateFamily(stateTag.getStateFamily()) + .setFetchEntryNamesOnly(stateTag.getOmitValues()); + if (stateTag.getRequestPosition() == null) { + multimapFetchBuilder.setFetchMaxBytes(INITIAL_MAX_MULTIMAP_BYTES); + } else { + multimapFetchBuilder.setFetchMaxBytes(CONTINUATION_MAX_MULTIMAP_BYTES); + multimapFetchBuilder.setRequestPosition((ByteString) stateTag.getRequestPosition()); + continuation = true; + } + break; + + default: + throw new RuntimeException("Unknown kind of tag requested: " + stateTag.getKind()); + } + } + if (!multimapSingleEntryToFetch.isEmpty()) { + Map>>> multimapTags = + multimapSingleEntryToFetch.stream() + .collect( + Collectors.groupingBy( + StateTag::getStateFamily, Collectors.groupingBy(StateTag::getTag))); + for (Map.Entry>>> entry : multimapTags.entrySet()) { + String stateFamily = entry.getKey(); + Map>> familyTags = multimapTags.get(stateFamily); + for (Map.Entry>> tagEntry : familyTags.entrySet()) { + ByteString tag = tagEntry.getKey(); + List> stateTags = tagEntry.getValue(); + Windmill.TagMultimapFetchRequest.Builder multimapFetchBuilder = + keyedDataBuilder + .addMultimapsToFetchBuilder() + .setTag(tag) + .setStateFamily(stateFamily) + .setFetchEntryNamesOnly(false); + for (StateTag stateTag : stateTags) { + Windmill.TagMultimapEntry.Builder entryBuilder = + multimapFetchBuilder + .addEntriesToFetchBuilder() + .setEntryName(stateTag.getMultimapKey()); + if (stateTag.getRequestPosition() == null) { + entryBuilder.setFetchMaxBytes(INITIAL_MAX_BAG_BYTES); + } else { + entryBuilder.setFetchMaxBytes(CONTINUATION_MAX_BAG_BYTES); + entryBuilder.setRequestPosition((Long) stateTag.getRequestPosition()); + continuation = true; + } + } + } + } + } + + orderedListsToFetch.sort( + Comparator.>comparingLong(s -> s.getSortedListRange().lowerEndpoint()) + .thenComparingLong(s -> s.getSortedListRange().upperEndpoint())); + for (StateTag stateTag : orderedListsToFetch) { + Range range = Preconditions.checkNotNull(stateTag.getSortedListRange()); + TagSortedListFetchRequest.Builder sorted_list = + keyedDataBuilder + .addSortedListsToFetchBuilder() + .setTag(stateTag.getTag()) + .setStateFamily(stateTag.getStateFamily()) + .setFetchMaxBytes(MAX_ORDERED_LIST_BYTES); + sorted_list.addFetchRanges( + SortedListRange.newBuilder() + .setStart(range.lowerEndpoint()) + .setLimit(range.upperEndpoint()) + .build()); + if (stateTag.getRequestPosition() != null) { + // We're asking for the next page. + sorted_list.setRequestPosition((ByteString) stateTag.getRequestPosition()); + } + } + if (continuation) { + keyedDataBuilder.setMaxBytes(MAX_CONTINUATION_KEY_BYTES); + } else { + keyedDataBuilder.setMaxBytes(MAX_KEY_BYTES); + } + + return keyedDataBuilder.build(); + } + + private void consumeResponse(KeyedGetDataResponse response, Set> toFetch) { + bytesRead += response.getSerializedSize(); + if (response.getFailed()) { + throw new KeyTokenInvalidException(key.toStringUtf8()); + } + + if (!key.equals(response.getKey())) { + throw new RuntimeException("Expected data for key " + key + " but was " + response.getKey()); + } + + for (Windmill.TagBag bag : response.getBagsList()) { + StateTag stateTag = + StateTag.of( + StateTag.Kind.BAG, + bag.getTag(), + bag.getStateFamily(), + bag.hasRequestPosition() ? bag.getRequestPosition() : null); + if (!toFetch.remove(stateTag)) { + throw new IllegalStateException( + "Received response for unrequested tag " + stateTag + ". Pending tags: " + toFetch); + } + consumeBag(bag, stateTag); + } + + for (Windmill.WatermarkHold hold : response.getWatermarkHoldsList()) { + StateTag stateTag = + StateTag.of(StateTag.Kind.WATERMARK, hold.getTag(), hold.getStateFamily()); + if (!toFetch.remove(stateTag)) { + throw new IllegalStateException( + "Received response for unrequested tag " + stateTag + ". Pending tags: " + toFetch); + } + consumeWatermark(hold, stateTag); + } + + for (Windmill.TagValue value : response.getValuesList()) { + StateTag stateTag = + StateTag.of(StateTag.Kind.VALUE, value.getTag(), value.getStateFamily()); + if (!toFetch.remove(stateTag)) { + throw new IllegalStateException( + "Received response for unrequested tag " + stateTag + ". Pending tags: " + toFetch); + } + consumeTagValue(value, stateTag); + } + for (Windmill.TagValuePrefixResponse prefix_response : response.getTagValuePrefixesList()) { + StateTag stateTag = + StateTag.of( + Kind.VALUE_PREFIX, + prefix_response.getTagPrefix(), + prefix_response.getStateFamily(), + prefix_response.hasRequestPosition() ? prefix_response.getRequestPosition() : null); + if (!toFetch.remove(stateTag)) { + throw new IllegalStateException( + "Received response for unrequested tag " + stateTag + ". Pending tags: " + toFetch); + } + consumeTagPrefixResponse(prefix_response, stateTag); + } + for (Windmill.TagSortedListFetchResponse sorted_list : response.getTagSortedListsList()) { + SortedListRange sortedListRange = Iterables.getOnlyElement(sorted_list.getFetchRangesList()); + Range range = Range.closedOpen(sortedListRange.getStart(), sortedListRange.getLimit()); + StateTag stateTag = + StateTag.of( + StateTag.Kind.ORDERED_LIST, + sorted_list.getTag(), + sorted_list.getStateFamily(), + sorted_list.hasRequestPosition() ? sorted_list.getRequestPosition() : null) + .toBuilder() + .setSortedListRange(range) + .build(); + if (!toFetch.remove(stateTag)) { + throw new IllegalStateException( + "Received response for unrequested tag " + stateTag + ". Pending tags: " + toFetch); + } + + consumeSortedList(sorted_list, stateTag); + } + + for (Windmill.TagMultimapFetchResponse tagMultimap : response.getTagMultimapsList()) { + // First check if it's keys()/entries() + StateTag.Builder builder = + StateTag.of( + Kind.MULTIMAP_ALL, + tagMultimap.getTag(), + tagMultimap.getStateFamily(), + tagMultimap.hasRequestPosition() ? tagMultimap.getRequestPosition() : null) + .toBuilder(); + StateTag tag = builder.setOmitValues(true).build(); + if (toFetch.contains(tag)) { + // this is keys() + toFetch.remove(tag); + consumeMultimapAll(tagMultimap, tag); + continue; + } + tag = builder.setOmitValues(false).build(); + if (toFetch.contains(tag)) { + // this is entries() + toFetch.remove(tag); + consumeMultimapAll(tagMultimap, tag); + continue; + } + // this is get() + StateTag.Builder entryTagBuilder = + StateTag.of( + Kind.MULTIMAP_SINGLE_ENTRY, tagMultimap.getTag(), tagMultimap.getStateFamily()) + .toBuilder(); + StateTag entryTag = null; + for (Windmill.TagMultimapEntry entry : tagMultimap.getEntriesList()) { + entryTag = + entryTagBuilder + .setMultimapKey(entry.getEntryName()) + .setRequestPosition(entry.hasRequestPosition() ? entry.getRequestPosition() : null) + .build(); + if (!toFetch.remove(entryTag)) { + throw new IllegalStateException( + "Received response for unrequested tag " + entryTag + ". Pending tags: " + toFetch); + } + consumeMultimapSingleEntry(entry, entryTag); + } + } + + if (!toFetch.isEmpty()) { + throw new IllegalStateException( + "Didn't receive responses for all pending fetches. Missing: " + toFetch); + } + } + + /** The deserialized values in {@code bag} as a read-only array list. */ + private List bagPageValues(TagBag bag, Coder elemCoder) { + if (bag.getValuesCount() == 0) { + return new WeightedList(Collections.emptyList()); + } + + WeightedList valueList = new WeightedList<>(new ArrayList(bag.getValuesCount())); + for (ByteString value : bag.getValuesList()) { + try { + valueList.addWeighted( + elemCoder.decode(value.newInput(), Coder.Context.OUTER), value.size()); + } catch (IOException e) { + throw new IllegalStateException("Unable to decode tag list using " + elemCoder, e); + } + } + return valueList; + } + + private List> sortedListPageValues( + Windmill.TagSortedListFetchResponse sortedListFetchResponse, Coder elemCoder) { + if (sortedListFetchResponse.getEntriesCount() == 0) { + return new WeightedList<>(Collections.emptyList()); + } + + WeightedList> entryList = + new WeightedList<>(new ArrayList<>(sortedListFetchResponse.getEntriesCount())); + for (SortedListEntry entry : sortedListFetchResponse.getEntriesList()) { + try { + T value = elemCoder.decode(entry.getValue().newInput(), Coder.Context.OUTER); + entryList.addWeighted( + TimestampedValue.of( + value, WindmillTimeUtils.windmillToHarnessTimestamp(entry.getSortKey())), + entry.getValue().size() + 8); + } catch (IOException e) { + throw new IllegalStateException("Unable to decode tag sorted list using " + elemCoder, e); + } + } + return entryList; + } + + private List> tagPrefixPageTagValues( + Windmill.TagValuePrefixResponse tagValuePrefixResponse, Coder valueCoder) { + if (tagValuePrefixResponse.getTagValuesCount() == 0) { + return new WeightedList<>(Collections.emptyList()); + } + + WeightedList> entryList = + new WeightedList>( + new ArrayList<>(tagValuePrefixResponse.getTagValuesCount())); + for (TagValue entry : tagValuePrefixResponse.getTagValuesList()) { + try { + V value = valueCoder.decode(entry.getValue().getData().newInput(), Context.OUTER); + entryList.addWeighted( + new AbstractMap.SimpleEntry<>(entry.getTag(), value), + entry.getTag().size() + entry.getValue().getData().size()); + } catch (IOException e) { + throw new IllegalStateException("Unable to decode tag value " + e); + } + } + return entryList; + } + + private WeightedList multimapEntryPageValues( + Windmill.TagMultimapEntry entry, Coder valueCoder) { + if (entry.getValuesCount() == 0) { + return new WeightedList<>(Collections.emptyList()); + } + WeightedList valuesList = new WeightedList<>(new ArrayList<>(entry.getValuesCount())); + for (ByteString value : entry.getValuesList()) { + try { + V decoded = valueCoder.decode(value.newInput(), Context.OUTER); + valuesList.addWeighted(decoded, value.size()); + } catch (IOException e) { + throw new IllegalStateException("Unable to decode multimap value " + e); + } + } + return valuesList; + } + + private List>> multimapPageValues( + Windmill.TagMultimapFetchResponse response, Coder valueCoder) { + if (response.getEntriesCount() == 0) { + return new WeightedList<>(Collections.emptyList()); + } + WeightedList>> entriesList = + new WeightedList<>(new ArrayList<>(response.getEntriesCount())); + for (Windmill.TagMultimapEntry entry : response.getEntriesList()) { + WeightedList values = multimapEntryPageValues(entry, valueCoder); + entriesList.addWeighted( + new AbstractMap.SimpleEntry<>(entry.getEntryName(), values), + entry.getEntryName().size() + values.getWeight()); + } + return entriesList; + } + + private void consumeBag(TagBag bag, StateTag stateTag) { + boolean shouldRemove; + // This is the response for the first page. + // Leave the future in the cache so subsequent requests for the first page + // can return immediately. + // This is a response for a subsequent page. + // Don't cache the future since we may need to make multiple requests with different + // continuation positions. + shouldRemove = stateTag.getRequestPosition() != null; + CoderAndFuture> coderAndFuture = + getWaiting(stateTag, shouldRemove); + SettableFuture> future = + coderAndFuture.getNonDoneFuture(stateTag); + try { + Coder coder = coderAndFuture.getAndClearCoder(); + List values = this.bagPageValues(bag, coder); + future.set( + new ValuesAndContPosition<>( + values, bag.hasContinuationPosition() ? bag.getContinuationPosition() : null)); + } catch (Exception e) { + future.setException(new RuntimeException("Error parsing bag response", e)); + } + } + + private void consumeWatermark(Windmill.WatermarkHold watermarkHold, StateTag stateTag) { + CoderAndFuture coderAndFuture = getWaiting(stateTag, false); + SettableFuture future = coderAndFuture.getNonDoneFuture(stateTag); + // No coders for watermarks + coderAndFuture.checkNoCoder(); + + Instant hold = null; + for (long timestamp : watermarkHold.getTimestampsList()) { + Instant instant = new Instant(TimeUnit.MICROSECONDS.toMillis(timestamp)); + // TIMESTAMP_MAX_VALUE represents infinity, and windmill will return it if no hold is set, so + // don't treat it as a hold here. + if (instant.isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE) + && (hold == null || instant.isBefore(hold))) { + hold = instant; + } + } + + future.set(hold); + } + + private void consumeTagValue(TagValue tagValue, StateTag stateTag) { + CoderAndFuture coderAndFuture = getWaiting(stateTag, false); + SettableFuture future = coderAndFuture.getNonDoneFuture(stateTag); + Coder coder = coderAndFuture.getAndClearCoder(); + + if (tagValue.hasValue() + && tagValue.getValue().hasData() + && !tagValue.getValue().getData().isEmpty()) { + InputStream inputStream = tagValue.getValue().getData().newInput(); + try { + T value = coder.decode(inputStream, Coder.Context.OUTER); + future.set(value); + } catch (IOException e) { + future.setException(new IllegalStateException("Unable to decode value using " + coder, e)); + } + } else { + future.set(null); + } + } + + private void consumeTagPrefixResponse( + Windmill.TagValuePrefixResponse tagValuePrefixResponse, StateTag stateTag) { + boolean shouldRemove; + // This is the response for the first page. + // Leave the future in the cache so subsequent + // requests for the first page + // can return immediately. + // This is a response for a subsequent page. + // Don't cache the future since we may need to make multiple requests with different + // continuation positions. + shouldRemove = stateTag.getRequestPosition() != null; + + CoderAndFuture, ByteString>> coderAndFuture = + getWaiting(stateTag, shouldRemove); + SettableFuture, ByteString>> future = + coderAndFuture.getNonDoneFuture(stateTag); + Coder valueCoder = coderAndFuture.getAndClearCoder(); + try { + List> values = + this.tagPrefixPageTagValues(tagValuePrefixResponse, valueCoder); + future.set( + new ValuesAndContPosition<>( + values, + tagValuePrefixResponse.hasContinuationPosition() + ? tagValuePrefixResponse.getContinuationPosition() + : null)); + } catch (Exception e) { + future.setException(new RuntimeException("Error parsing tag value prefix", e)); + } + } + + private void consumeSortedList( + Windmill.TagSortedListFetchResponse sortedListFetchResponse, StateTag stateTag) { + boolean shouldRemove; + // This is the response for the first page.// Leave the future in the cache so subsequent + // requests for the first page + // can return immediately. + // This is a response for a subsequent page. + // Don't cache the future since we may need to make multiple requests with different + // continuation positions. + shouldRemove = stateTag.getRequestPosition() != null; + + CoderAndFuture, ByteString>> coderAndFuture = + getWaiting(stateTag, shouldRemove); + SettableFuture, ByteString>> future = + coderAndFuture.getNonDoneFuture(stateTag); + Coder coder = coderAndFuture.getAndClearCoder(); + try { + List> values = this.sortedListPageValues(sortedListFetchResponse, coder); + future.set( + new ValuesAndContPosition<>( + values, + sortedListFetchResponse.hasContinuationPosition() + ? sortedListFetchResponse.getContinuationPosition() + : null)); + } catch (Exception e) { + future.setException(new RuntimeException("Error parsing ordered list", e)); + } + } + + private void consumeMultimapAll( + Windmill.TagMultimapFetchResponse response, StateTag stateTag) { + // Leave the future in the cache for the first page; do not cache for subsequent pages. + boolean shouldRemove = stateTag.getRequestPosition() != null; + CoderAndFuture>, ByteString>> + coderAndFuture = getWaiting(stateTag, shouldRemove); + SettableFuture>, ByteString>> future = + coderAndFuture.getNonDoneFuture(stateTag); + Coder valueCoder = coderAndFuture.getAndClearCoder(); + try { + List>> entries = + this.multimapPageValues(response, valueCoder); + future.set( + new ValuesAndContPosition<>( + entries, + response.hasContinuationPosition() ? response.getContinuationPosition() : null)); + } catch (Exception e) { + future.setException(new RuntimeException("Error parsing multimap fetch all result", e)); + } + } + + private void consumeMultimapSingleEntry( + Windmill.TagMultimapEntry entry, StateTag stateTag) { + // Leave the future in the cache for the first page; do not cache for subsequent pages. + boolean shouldRemove = stateTag.getRequestPosition() != null; + CoderAndFuture> coderAndFuture = + getWaiting(stateTag, shouldRemove); + SettableFuture> future = + coderAndFuture.getNonDoneFuture(stateTag); + Coder valueCoder = coderAndFuture.getAndClearCoder(); + try { + List values = this.multimapEntryPageValues(entry, valueCoder); + Long continuationToken = + entry.hasContinuationPosition() ? entry.getContinuationPosition() : null; + future.set(new ValuesAndContPosition<>(values, continuationToken)); + } catch (Exception e) { + future.setException(new RuntimeException("Error parsing multimap single entry result", e)); + } + } + + private static final class CoderAndFuture { + private final SettableFuture future; + private Coder coder = null; + + private CoderAndFuture(Coder coder, SettableFuture future) { + this.coder = coder; + this.future = future; + } + + private SettableFuture getFuture() { + return future; + } + + private SettableFuture getNonDoneFuture(StateTag stateTag) { + if (future.isDone()) { + throw new IllegalStateException("Future for " + stateTag + " is already done"); + } + return future; + } + + private Coder getAndClearCoder() { + if (coder == null) { + throw new IllegalStateException("Coder has already been cleared from cache"); + } + Coder result = (Coder) coder; + if (result == null) { + throw new IllegalStateException("Coder has already been cleared from cache"); + } + coder = null; + return result; + } + + private void checkNoCoder() { + if (coder != null) { + throw new IllegalStateException("Unexpected coder"); + } + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateUtil.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateUtil.java new file mode 100644 index 0000000000000..3cac5c3c5724b --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateUtil.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import org.apache.beam.runners.core.StateNamespace; +import org.apache.beam.runners.core.StateTag; +import org.apache.beam.sdk.util.ByteStringOutputStream; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; + +class WindmillStateUtil { + /** Encodes the given namespace and address as {@code <namespace>+<address>}. */ + @VisibleForTesting + static ByteString encodeKey(StateNamespace namespace, StateTag address) { + try { + // Use ByteStringOutputStream rather than concatenation and String.format. We build these keys + // a lot, and this leads to better performance results. See associated benchmarks. + ByteStringOutputStream stream = new ByteStringOutputStream(); + OutputStreamWriter writer = new OutputStreamWriter(stream, StandardCharsets.UTF_8); + + // stringKey starts and ends with a slash. We separate it from the + // StateTag ID by a '+' (which is guaranteed not to be in the stringKey) because the + // ID comes from the user. + namespace.appendTo(writer); + writer.write('+'); + address.appendTo(writer); + writer.flush(); + return stream.toByteString(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillValue.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillValue.java new file mode 100644 index 0000000000000..1ea6e56435d26 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillValue.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import static org.apache.beam.runners.dataflow.worker.windmill.state.WindmillStateUtil.encodeKey; + +import java.io.Closeable; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import org.apache.beam.runners.core.StateNamespace; +import org.apache.beam.runners.core.StateTag; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.state.ValueState; +import org.apache.beam.sdk.util.ByteStringOutputStream; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Futures; + +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) +public class WindmillValue extends SimpleWindmillState implements ValueState { + private final StateNamespace namespace; + private final StateTag> address; + private final ByteString stateKey; + private final String stateFamily; + private final Coder coder; + + /** Whether we've modified the value since creation of this state. */ + private boolean modified = false; + /** Whether the in memory value is the true value. */ + private boolean valueIsKnown = false; + /** The size of the encoded value */ + private long cachedSize = -1; + + private T value; + + WindmillValue( + StateNamespace namespace, + StateTag> address, + String stateFamily, + Coder coder, + boolean isNewKey) { + this.namespace = namespace; + this.address = address; + this.stateKey = encodeKey(namespace, address); + this.stateFamily = stateFamily; + this.coder = coder; + if (isNewKey) { + this.valueIsKnown = true; + this.value = null; + } + } + + @Override + public void clear() { + modified = true; + valueIsKnown = true; + value = null; + } + + @Override + @SuppressWarnings("FutureReturnValueIgnored") + public WindmillValue readLater() { + getFuture(); + return this; + } + + @Override + public T read() { + try (Closeable scope = scopedReadState()) { + if (!valueIsKnown) { + cachedSize = -1; + } + value = getFuture().get(); + valueIsKnown = true; + return value; + } catch (InterruptedException | ExecutionException | IOException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new RuntimeException("Unable to read value from state", e); + } + } + + @Override + public void write(T value) { + modified = true; + valueIsKnown = true; + cachedSize = -1; + this.value = value; + } + + @Override + protected Windmill.WorkItemCommitRequest persistDirectly(WindmillStateCache.ForKeyAndFamily cache) + throws IOException { + if (!valueIsKnown) { + // The value was never read, written or cleared. + // Thus nothing to update in Windmill. + // And no need to add to global cache. + return Windmill.WorkItemCommitRequest.newBuilder().buildPartial(); + } + + ByteString encoded = null; + if (cachedSize == -1 || modified) { + ByteStringOutputStream stream = new ByteStringOutputStream(); + if (value != null) { + coder.encode(value, stream, Coder.Context.OUTER); + } + encoded = stream.toByteString(); + cachedSize = encoded.size(); + } + + // Place in cache to avoid a future read. + cache.put(namespace, address, this, cachedSize); + + if (!modified) { + // The value was read, but never written or cleared. + // But nothing to update in Windmill. + return Windmill.WorkItemCommitRequest.newBuilder().buildPartial(); + } + + // The value was written or cleared. Commit that change to Windmill. + modified = false; + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + commitBuilder + .addValueUpdatesBuilder() + .setTag(stateKey) + .setStateFamily(stateFamily) + .getValueBuilder() + .setData(encoded) + .setTimestamp(Long.MAX_VALUE); + return commitBuilder.buildPartial(); + } + + private Future getFuture() { + // WindmillStateReader guarantees that we can ask for a future for a particular tag multiple + // times and it will efficiently be reused. + return valueIsKnown + ? Futures.immediateFuture(value) + : reader.valueFuture(stateKey, stateFamily, coder); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillWatermarkHold.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillWatermarkHold.java new file mode 100644 index 0000000000000..a800c2eb6dadb --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillWatermarkHold.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import static org.apache.beam.runners.dataflow.worker.windmill.state.WindmillStateUtil.encodeKey; + +import java.io.Closeable; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import org.apache.beam.runners.core.StateNamespace; +import org.apache.beam.runners.core.StateTag; +import org.apache.beam.runners.dataflow.worker.WindmillTimeUtils; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.sdk.state.ReadableState; +import org.apache.beam.sdk.state.WatermarkHoldState; +import org.apache.beam.sdk.transforms.windowing.TimestampCombiner; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Futures; +import org.joda.time.Instant; + +@SuppressWarnings({ + "rawtypes", // TODO(https://github.com/apache/beam/issues/20447) + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) +public class WindmillWatermarkHold extends WindmillState implements WatermarkHoldState { + // The encoded size of an Instant. + private static final int ENCODED_SIZE = 8; + + private final TimestampCombiner timestampCombiner; + private final StateNamespace namespace; + private final StateTag address; + private final ByteString stateKey; + private final String stateFamily; + + private boolean cleared = false; + /** + * If non-{@literal null}, the known current hold value, or absent if we know there are no output + * watermark holds. If {@literal null}, the current hold value could depend on holds in Windmill + * we do not yet know. + */ + private Optional cachedValue = null; + + private Instant localAdditions = null; + + WindmillWatermarkHold( + StateNamespace namespace, + StateTag address, + String stateFamily, + TimestampCombiner timestampCombiner, + boolean isNewKey) { + this.namespace = namespace; + this.address = address; + this.stateKey = encodeKey(namespace, address); + this.stateFamily = stateFamily; + this.timestampCombiner = timestampCombiner; + if (isNewKey) { + cachedValue = Optional.absent(); + } + } + + @Override + public void clear() { + cleared = true; + cachedValue = Optional.absent(); + localAdditions = null; + } + + @Override + @SuppressWarnings("FutureReturnValueIgnored") + public WindmillWatermarkHold readLater() { + getFuture(); + return this; + } + + @Override + public Instant read() { + try (Closeable scope = scopedReadState()) { + Instant persistedHold = getFuture().get(); + if (persistedHold == null) { + cachedValue = Optional.absent(); + } else { + cachedValue = Optional.of(persistedHold); + } + } catch (InterruptedException | ExecutionException | IOException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new RuntimeException("Unable to read state", e); + } + + if (localAdditions == null) { + return cachedValue.orNull(); + } else if (!cachedValue.isPresent()) { + return localAdditions; + } else { + return timestampCombiner.combine(localAdditions, cachedValue.get()); + } + } + + @Override + public ReadableState isEmpty() { + throw new UnsupportedOperationException(); + } + + @Override + public void add(Instant outputTime) { + localAdditions = + (localAdditions == null) + ? outputTime + : timestampCombiner.combine(outputTime, localAdditions); + } + + @Override + public TimestampCombiner getTimestampCombiner() { + return timestampCombiner; + } + + @Override + public Future persist( + final WindmillStateCache.ForKeyAndFamily cache) { + + Future result; + + if (!cleared && localAdditions == null) { + // No changes, so no need to update Windmill and no need to cache any value. + return Futures.immediateFuture(Windmill.WorkItemCommitRequest.newBuilder().buildPartial()); + } + + if (cleared && localAdditions == null) { + // Just clearing the persisted state; blind delete + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + commitBuilder + .addWatermarkHoldsBuilder() + .setTag(stateKey) + .setStateFamily(stateFamily) + .setReset(true); + + result = Futures.immediateFuture(commitBuilder.buildPartial()); + } else if (cleared && localAdditions != null) { + // Since we cleared before adding, we can do a blind overwrite of persisted state + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + commitBuilder + .addWatermarkHoldsBuilder() + .setTag(stateKey) + .setStateFamily(stateFamily) + .setReset(true) + .addTimestamps(WindmillTimeUtils.harnessToWindmillTimestamp(localAdditions)); + + cachedValue = Optional.of(localAdditions); + + result = Futures.immediateFuture(commitBuilder.buildPartial()); + } else if (!cleared && localAdditions != null) { + // Otherwise, we need to combine the local additions with the already persisted data + result = combineWithPersisted(); + } else { + throw new IllegalStateException("Unreachable condition"); + } + + return Futures.lazyTransform( + result, + result1 -> { + cleared = false; + localAdditions = null; + if (cachedValue != null) { + cache.put(namespace, address, WindmillWatermarkHold.this, ENCODED_SIZE); + } + return result1; + }); + } + + private Future getFuture() { + return cachedValue != null + ? Futures.immediateFuture(cachedValue.orNull()) + : reader.watermarkFuture(stateKey, stateFamily); + } + + /** + * Combines local additions with persisted data and mutates the {@code commitBuilder} to write the + * result. + */ + private Future combineWithPersisted() { + boolean windmillCanCombine = false; + + // If the combined output time depends only on the window, then we are just blindly adding + // the same value that may or may not already be present. This depends on the state only being + // used for one window. + windmillCanCombine |= timestampCombiner.dependsOnlyOnWindow(); + + // If the combined output time depends only on the earliest input timestamp, then because + // assignOutputTime is monotonic, the hold only depends on the earliest output timestamp + // (which is the value submitted as a watermark hold). The only way holds for later inputs + // can be redundant is if the are later (or equal) to the earliest. So taking the MIN + // implicitly, as Windmill does, has the desired behavior. + windmillCanCombine |= timestampCombiner.dependsOnlyOnEarliestTimestamp(); + + if (windmillCanCombine) { + // We do a blind write and let Windmill take the MIN + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + commitBuilder + .addWatermarkHoldsBuilder() + .setTag(stateKey) + .setStateFamily(stateFamily) + .addTimestamps(WindmillTimeUtils.harnessToWindmillTimestamp(localAdditions)); + + if (cachedValue != null) { + cachedValue = + Optional.of( + cachedValue.isPresent() + ? timestampCombiner.combine(cachedValue.get(), localAdditions) + : localAdditions); + } + + return Futures.immediateFuture(commitBuilder.buildPartial()); + } else { + // The non-fast path does a read-modify-write + return Futures.lazyTransform( + (cachedValue != null) + ? Futures.immediateFuture(cachedValue.orNull()) + : reader.watermarkFuture(stateKey, stateFamily), + priorHold -> { + cachedValue = + Optional.of( + (priorHold != null) + ? timestampCombiner.combine(priorHold, localAdditions) + : localAdditions); + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + commitBuilder + .addWatermarkHoldsBuilder() + .setTag(stateKey) + .setStateFamily(stateFamily) + .setReset(true) + .addTimestamps(WindmillTimeUtils.harnessToWindmillTimestamp(cachedValue.get())); + + return commitBuilder.buildPartial(); + }); + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WrappedFuture.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WrappedFuture.java new file mode 100644 index 0000000000000..035f6ec8e93de --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WrappedFuture.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import javax.annotation.Nullable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ForwardingFuture; + +/** + * A future which will trigger a GetData request to Windmill for all outstanding futures on the + * first {@link #get}. + */ +public class WrappedFuture extends ForwardingFuture.SimpleForwardingFuture { + /** + * The reader we'll use to service the eventual read. Null if read has been fulfilled. + * + *

    NOTE: We must clear this after the read is fulfilled to prevent space leaks. + */ + private @Nullable WindmillStateReader reader; + + public WrappedFuture(WindmillStateReader reader, Future delegate) { + super(delegate); + this.reader = reader; + } + + @Override + public T get() throws InterruptedException, ExecutionException { + if (!delegate().isDone() && reader != null) { + // Only one thread per reader, so no race here. + reader.startBatchAndBlock(); + } + reader = null; + return super.get(); + } + + @Override + public T get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + if (!delegate().isDone() && reader != null) { + // Only one thread per reader, so no race here. + reader.startBatchAndBlock(); + } + reader = null; + return super.get(timeout, unit); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/harness/test/TestExecutors.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/harness/test/TestExecutors.java index 57c53a3d42e4f..ada137877b35b 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/harness/test/TestExecutors.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/harness/test/TestExecutors.java @@ -20,7 +20,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ForwardingExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ForwardingExecutorService; import org.junit.rules.TestRule; import org.junit.runner.Description; import org.junit.runners.model.Statement; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/BatchDataflowWorkerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/BatchDataflowWorkerTest.java index 60cad0a006070..b4f544129db67 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/BatchDataflowWorkerTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/BatchDataflowWorkerTest.java @@ -37,12 +37,12 @@ import com.google.api.services.dataflow.model.WorkItemStatus; import java.io.IOException; import java.util.ArrayList; +import java.util.Optional; import org.apache.beam.runners.dataflow.options.DataflowWorkerHarnessOptions; import org.apache.beam.runners.dataflow.util.TimeUtil; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.util.FastNanoClockAndSleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.Description; import org.hamcrest.TypeSafeMatcher; import org.joda.time.Duration; @@ -61,16 +61,10 @@ @RunWith(JUnit4.class) public class BatchDataflowWorkerTest { - private static class WorkerException extends Exception {} - @Rule public FastNanoClockAndSleeper clockAndSleeper = new FastNanoClockAndSleeper(); - @Mock WorkUnitClient mockWorkUnitClient; - @Mock DataflowWorkProgressUpdater mockProgressUpdater; - @Mock DataflowWorkExecutor mockWorkExecutor; - DataflowWorkerHarnessOptions options; @Before @@ -98,7 +92,7 @@ public void testWhenNoWorkIsReturnedThatWeImmediatelyRetry() throws Exception { workItem.setReportStatusInterval(TimeUtil.toCloudDuration(Duration.standardMinutes(1))); when(mockWorkUnitClient.getWorkItem()) - .thenReturn(Optional.absent()) + .thenReturn(Optional.empty()) .thenReturn(Optional.of(workItem)); assertTrue(worker.getAndPerformWork()); @@ -138,7 +132,7 @@ public void testWhenProcessingWorkUnitFailsWeReportStatus() throws Exception { Throwable error = errorCaptor.getValue(); assertThat(error, notNullValue()); - assertThat(error.getMessage(), equalTo("Unknown kind of work item: " + workItem.toString())); + assertThat(error.getMessage(), equalTo("Unknown kind of work item: " + workItem)); } @Test @@ -168,8 +162,9 @@ public void testStopProgressReportInCaseOfFailure() throws Exception { @Test public void testIsSplitResponseTooLarge() throws IOException { SourceSplitResponse splitResponse = new SourceSplitResponse(); - splitResponse.setShards( - ImmutableList.of(new SourceSplitShard(), new SourceSplitShard())); + splitResponse.setShards(ImmutableList.of(new SourceSplitShard(), new SourceSplitShard())); assertThat(DataflowApiUtils.computeSerializedSizeBytes(splitResponse), greaterThan(0L)); } + + private static class WorkerException extends Exception {} } diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/CombineValuesFnFactoryTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/CombineValuesFnFactoryTest.java index b45df0aa18549..786f3ae76b837 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/CombineValuesFnFactoryTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/CombineValuesFnFactoryTest.java @@ -56,9 +56,9 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/CreateIsmShardKeyAndSortKeyDoFnFactoryTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/CreateIsmShardKeyAndSortKeyDoFnFactoryTest.java index 9798315609e8b..e95cfc0a0ff5b 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/CreateIsmShardKeyAndSortKeyDoFnFactoryTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/CreateIsmShardKeyAndSortKeyDoFnFactoryTest.java @@ -31,8 +31,8 @@ import org.apache.beam.runners.dataflow.worker.util.common.worker.ParDoFn; import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowElementExecutionTrackerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowElementExecutionTrackerTest.java index bdb714dd53d89..16ba5e6c49fe7 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowElementExecutionTrackerTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowElementExecutionTrackerTest.java @@ -32,7 +32,7 @@ import org.apache.beam.runners.dataflow.worker.counters.NameContext; import org.apache.beam.runners.dataflow.worker.util.common.worker.ElementExecutionTracker; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Before; import org.junit.Rule; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateTrackerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateTrackerTest.java index 0185ffda28194..551937c355954 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateTrackerTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateTrackerTest.java @@ -38,7 +38,7 @@ import org.apache.beam.runners.dataflow.worker.util.common.worker.ElementExecutionTracker; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.DateTimeUtils.MillisProvider; import org.junit.Before; import org.junit.Test; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowOperationContextTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowOperationContextTest.java index 715c15de2e352..34c3b3d5373c9 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowOperationContextTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowOperationContextTest.java @@ -49,8 +49,8 @@ import org.apache.beam.runners.dataflow.worker.profiler.ScopedProfiler.ProfileScope; import org.apache.beam.sdk.metrics.MetricsContainer; import org.apache.beam.sdk.testing.RestoreSystemProperties; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matchers; import org.joda.time.Duration; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowWorkUnitClientTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowWorkUnitClientTest.java index a7b53956e65af..3c63f3cc19d29 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowWorkUnitClientTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowWorkUnitClientTest.java @@ -34,6 +34,7 @@ import com.google.api.services.dataflow.model.SeqMapTask; import com.google.api.services.dataflow.model.WorkItem; import java.io.IOException; +import java.util.Optional; import org.apache.beam.runners.dataflow.options.DataflowWorkerHarnessOptions; import org.apache.beam.runners.dataflow.worker.logging.DataflowWorkerLoggingMDC; import org.apache.beam.runners.dataflow.worker.testing.RestoreDataflowLoggingMDC; @@ -42,9 +43,8 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.testing.RestoreSystemProperties; import org.apache.beam.sdk.util.FastNanoClockAndSleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -61,6 +61,9 @@ @RunWith(JUnit4.class) public class DataflowWorkUnitClientTest { private static final Logger LOG = LoggerFactory.getLogger(DataflowWorkUnitClientTest.class); + private static final String PROJECT_ID = "TEST_PROJECT_ID"; + private static final String JOB_ID = "TEST_JOB_ID"; + private static final String WORKER_ID = "TEST_WORKER_ID"; @Rule public TestRule restoreSystemProperties = new RestoreSystemProperties(); @Rule public TestRule restoreLogging = new RestoreDataflowLoggingMDC(); @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -69,10 +72,6 @@ public class DataflowWorkUnitClientTest { @Mock private MockLowLevelHttpRequest request; private DataflowWorkerHarnessOptions pipelineOptions; - private static final String PROJECT_ID = "TEST_PROJECT_ID"; - private static final String JOB_ID = "TEST_JOB_ID"; - private static final String WORKER_ID = "TEST_WORKER_ID"; - @Before public void setUp() throws Exception { MockitoAnnotations.initMocks(this); @@ -104,10 +103,10 @@ public void testCloudServiceCall() throws Exception { .fromString(request.getContentAsString(), LeaseWorkItemRequest.class); assertEquals(WORKER_ID, actualRequest.getWorkerId()); assertEquals( - ImmutableList.of(WORKER_ID, "remote_source", "custom_source"), + ImmutableList.of(WORKER_ID, "remote_source", "custom_source"), actualRequest.getWorkerCapabilities()); assertEquals( - ImmutableList.of("map_task", "seq_map_task", "remote_source_task"), + ImmutableList.of("map_task", "seq_map_task", "remote_source_task"), actualRequest.getWorkItemTypes()); assertEquals("1234", DataflowWorkerLoggingMDC.getWorkId()); } @@ -151,17 +150,17 @@ public void testCloudServiceCallNoWorkPresent() throws Exception { WorkUnitClient client = new DataflowWorkUnitClient(pipelineOptions, LOG); - assertEquals(Optional.absent(), client.getWorkItem()); + assertEquals(Optional.empty(), client.getWorkItem()); LeaseWorkItemRequest actualRequest = Transport.getJsonFactory() .fromString(request.getContentAsString(), LeaseWorkItemRequest.class); assertEquals(WORKER_ID, actualRequest.getWorkerId()); assertEquals( - ImmutableList.of(WORKER_ID, "remote_source", "custom_source"), + ImmutableList.of(WORKER_ID, "remote_source", "custom_source"), actualRequest.getWorkerCapabilities()); assertEquals( - ImmutableList.of("map_task", "seq_map_task", "remote_source_task"), + ImmutableList.of("map_task", "seq_map_task", "remote_source_task"), actualRequest.getWorkItemTypes()); } @@ -175,17 +174,17 @@ public void testCloudServiceCallNoWorkId() throws Exception { WorkUnitClient client = new DataflowWorkUnitClient(pipelineOptions, LOG); - assertEquals(Optional.absent(), client.getWorkItem()); + assertEquals(Optional.empty(), client.getWorkItem()); LeaseWorkItemRequest actualRequest = Transport.getJsonFactory() .fromString(request.getContentAsString(), LeaseWorkItemRequest.class); assertEquals(WORKER_ID, actualRequest.getWorkerId()); assertEquals( - ImmutableList.of(WORKER_ID, "remote_source", "custom_source"), + ImmutableList.of(WORKER_ID, "remote_source", "custom_source"), actualRequest.getWorkerCapabilities()); assertEquals( - ImmutableList.of("map_task", "seq_map_task", "remote_source_task"), + ImmutableList.of("map_task", "seq_map_task", "remote_source_task"), actualRequest.getWorkItemTypes()); } @@ -195,17 +194,17 @@ public void testCloudServiceCallNoWorkItem() throws Exception { WorkUnitClient client = new DataflowWorkUnitClient(pipelineOptions, LOG); - assertEquals(Optional.absent(), client.getWorkItem()); + assertEquals(Optional.empty(), client.getWorkItem()); LeaseWorkItemRequest actualRequest = Transport.getJsonFactory() .fromString(request.getContentAsString(), LeaseWorkItemRequest.class); assertEquals(WORKER_ID, actualRequest.getWorkerId()); assertEquals( - ImmutableList.of(WORKER_ID, "remote_source", "custom_source"), + ImmutableList.of(WORKER_ID, "remote_source", "custom_source"), actualRequest.getWorkerCapabilities()); assertEquals( - ImmutableList.of("map_task", "seq_map_task", "remote_source_task"), + ImmutableList.of("map_task", "seq_map_task", "remote_source_task"), actualRequest.getWorkItemTypes()); } diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DefaultParDoFnFactoryTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DefaultParDoFnFactoryTest.java index 8b279d27f88b9..371fce54ac65d 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DefaultParDoFnFactoryTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DefaultParDoFnFactoryTest.java @@ -40,7 +40,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.hamcrest.Matchers; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ExperimentContextTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ExperimentContextTest.java index 56a65755b6923..7c1f1ee75a9f8 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ExperimentContextTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ExperimentContextTest.java @@ -26,7 +26,7 @@ import org.apache.beam.runners.dataflow.worker.ExperimentContext.Experiment; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/FakeWindmillServer.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/FakeWindmillServer.java index b0a64c82d035d..4700217dc8a4e 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/FakeWindmillServer.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/FakeWindmillServer.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Queue; @@ -48,10 +49,15 @@ import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetDataRequest; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetDataResponse; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.LatencyAttribution; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.LatencyAttribution.State; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.WorkItemCommitRequest; import org.apache.beam.runners.dataflow.worker.windmill.WindmillServerStub; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.net.HostAndPort; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.CommitWorkStream; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetDataStream; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetWorkStream; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.net.HostAndPort; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.rules.ErrorCollector; @@ -61,58 +67,6 @@ /** An in-memory Windmill server that offers provided work and data. */ class FakeWindmillServer extends WindmillServerStub { private static final Logger LOG = LoggerFactory.getLogger(FakeWindmillServer.class); - - static class ResponseQueue { - private final Queue> responses = new ConcurrentLinkedQueue<>(); - private Function defaultResponse; - Duration sleep = Duration.ZERO; - - // (Fluent) interface for response producers, accessible from tests. - - ResponseQueue thenAnswer(Function mapFun) { - responses.add(mapFun); - return this; - } - - ResponseQueue thenReturn(U response) { - return thenAnswer((request) -> response); - } - - ResponseQueue answerByDefault(Function mapFun) { - defaultResponse = mapFun; - return this; - } - - ResponseQueue returnByDefault(U response) { - return answerByDefault((request) -> response); - } - - ResponseQueue delayEachResponseBy(Duration sleep) { - this.sleep = sleep; - return this; - } - - // Interface for response consumers, accessible from the enclosing class. - - private U getOrDefault(T request) { - Function mapFun = responses.poll(); - U response = mapFun == null ? defaultResponse.apply(request) : mapFun.apply(request); - Uninterruptibles.sleepUninterruptibly(sleep.getMillis(), TimeUnit.MILLISECONDS); - return response; - } - - private U get(T request) { - Function mapFun = responses.poll(); - U response = mapFun == null ? null : mapFun.apply(request); - Uninterruptibles.sleepUninterruptibly(sleep.getMillis(), TimeUnit.MILLISECONDS); - return response; - } - - private boolean isEmpty() { - return responses.isEmpty(); - } - } - private final ResponseQueue workToOffer; private final ResponseQueue dataToOffer; private final ResponseQueue commitsToOffer; @@ -120,13 +74,13 @@ private boolean isEmpty() { private final Map commitsReceived; private final ArrayList statsReceived; private final LinkedBlockingQueue exceptions; - private int commitsRequested = 0; - private int numGetDataRequests = 0; private final AtomicInteger expectedExceptionCount; private final ErrorCollector errorCollector; + private final ConcurrentHashMap> droppedStreamingCommits; + private int commitsRequested = 0; + private int numGetDataRequests = 0; private boolean isReady = true; private boolean dropStreamingCommits = false; - private final ConcurrentHashMap> droppedStreamingCommits; public FakeWindmillServer(ErrorCollector errorCollector) { workToOffer = @@ -240,11 +194,12 @@ public Windmill.ReportStatsResponse reportStats(Windmill.ReportStatsRequest requ @Override public long getAndResetThrottleTime() { - return (long) 0; + return 0; } @Override - public GetWorkStream getWorkStream(Windmill.GetWorkRequest request, WorkItemReceiver receiver) { + public GetWorkStream getWorkStream( + Windmill.GetWorkRequest request, GetWorkStream.WorkItemReceiver receiver) { LOG.debug("getWorkStream: {}", request.toString()); Instant startTime = Instant.now(); final CountDownLatch done = new CountDownLatch(1); @@ -273,7 +228,15 @@ public boolean awaitTermination(int time, TimeUnit unit) throws InterruptedExcep computationWork.getInputDataWatermark()); for (Windmill.WorkItem workItem : computationWork.getWorkList()) { receiver.receiveWork( - computationWork.getComputationId(), inputDataWatermark, Instant.now(), workItem); + computationWork.getComputationId(), + inputDataWatermark, + Instant.now(), + workItem, + Collections.singletonList( + LatencyAttribution.newBuilder() + .setState(State.GET_WORK_IN_TRANSIT_TO_USER_WORKER) + .setTotalDurationMillis(1000) + .build())); } } } @@ -474,4 +437,55 @@ public boolean isReady() { public void setIsReady(boolean ready) { this.isReady = ready; } + + static class ResponseQueue { + private final Queue> responses = new ConcurrentLinkedQueue<>(); + Duration sleep = Duration.ZERO; + private Function defaultResponse; + + // (Fluent) interface for response producers, accessible from tests. + + ResponseQueue thenAnswer(Function mapFun) { + responses.add(mapFun); + return this; + } + + ResponseQueue thenReturn(U response) { + return thenAnswer((request) -> response); + } + + ResponseQueue answerByDefault(Function mapFun) { + defaultResponse = mapFun; + return this; + } + + ResponseQueue returnByDefault(U response) { + return answerByDefault((request) -> response); + } + + ResponseQueue delayEachResponseBy(Duration sleep) { + this.sleep = sleep; + return this; + } + + // Interface for response consumers, accessible from the enclosing class. + + private U getOrDefault(T request) { + Function mapFun = responses.poll(); + U response = mapFun == null ? defaultResponse.apply(request) : mapFun.apply(request); + Uninterruptibles.sleepUninterruptibly(sleep.getMillis(), TimeUnit.MILLISECONDS); + return response; + } + + private U get(T request) { + Function mapFun = responses.poll(); + U response = mapFun == null ? null : mapFun.apply(request); + Uninterruptibles.sleepUninterruptibly(sleep.getMillis(), TimeUnit.MILLISECONDS); + return response; + } + + private boolean isEmpty() { + return responses.isEmpty(); + } + } } diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/GroupAlsoByWindowParDoFnFactoryTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/GroupAlsoByWindowParDoFnFactoryTest.java index fb748ff0d192d..7aff323c9e833 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/GroupAlsoByWindowParDoFnFactoryTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/GroupAlsoByWindowParDoFnFactoryTest.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.transforms.windowing.FixedWindows; import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.junit.Rule; import org.junit.Test; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/GroupingShuffleReaderTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/GroupingShuffleReaderTest.java index 69b7304d104ec..49f53ef8e4d00 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/GroupingShuffleReaderTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/GroupingShuffleReaderTest.java @@ -81,7 +81,7 @@ import org.apache.beam.sdk.util.common.Reiterator; import org.apache.beam.sdk.values.KV; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IntrinsicMapTaskExecutorFactoryTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IntrinsicMapTaskExecutorFactoryTest.java index aa308f4ba3c4c..bb0e396a27d84 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IntrinsicMapTaskExecutorFactoryTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IntrinsicMapTaskExecutorFactoryTest.java @@ -99,11 +99,11 @@ import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.MutableNetwork; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.Network; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.MutableNetwork; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.Network; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Before; import org.junit.Test; @@ -117,6 +117,7 @@ @RunWith(JUnit4.class) @SuppressWarnings({ "rawtypes", // TODO(https://github.com/apache/beam/issues/20447) + "DoNotMock", // TODO: Use NetworkBuilder to create a real instance }) public class IntrinsicMapTaskExecutorFactoryTest { private static final String STAGE = "test"; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IntrinsicMapTaskExecutorTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IntrinsicMapTaskExecutorTest.java index 092f4f21dd531..d60af9ebdc9dc 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IntrinsicMapTaskExecutorTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IntrinsicMapTaskExecutorTest.java @@ -69,7 +69,7 @@ import org.apache.beam.sdk.metrics.Metrics; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmFormatTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmFormatTest.java index 8ffb5a440f8c0..9b6c0536b6fe2 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmFormatTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmFormatTest.java @@ -45,7 +45,7 @@ import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.testing.CoderPropertiesTest.NonDeterministicCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmReaderFactoryTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmReaderFactoryTest.java index 680822c8aec23..09dba0fe5fd5c 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmReaderFactoryTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmReaderFactoryTest.java @@ -39,8 +39,8 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.After; import org.junit.Before; import org.junit.Rule; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmReaderTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmReaderTest.java index 549f409acd257..cabf7388875a7 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmReaderTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmReaderTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -71,15 +71,15 @@ import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.util.WeightedValue; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicate; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Ints; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.UnsignedBytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicate; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Ints; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedBytes; import org.junit.After; import org.junit.Before; import org.junit.Rule; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmSideInputReaderTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmSideInputReaderTest.java index 5d4a058c3b2cc..42bc9eb53891e 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmSideInputReaderTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmSideInputReaderTest.java @@ -19,8 +19,8 @@ import static org.apache.beam.runners.dataflow.util.Structs.getString; import static org.apache.beam.sdk.util.WindowedValue.valueInGlobalWindow; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.concat; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.concat; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.empty; @@ -106,19 +106,19 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.TreeMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Closer; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.TreeMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Closer; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.hamcrest.Matcher; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmSinkTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmSinkTest.java index f64e8b8c691ac..d395ec8acf6d4 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmSinkTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/IsmSinkTest.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.io.FileSystems; import org.apache.beam.sdk.testing.CoderPropertiesTest.NonDeterministicCoder; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/LazilyInitializedSideInputReaderTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/LazilyInitializedSideInputReaderTest.java index fbdb0b6dbae93..4ccc0f1a42f4b 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/LazilyInitializedSideInputReaderTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/LazilyInitializedSideInputReaderTest.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/LogRecordMatcherTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/LogRecordMatcherTest.java index f78304e97aca7..3ff2e9df89acc 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/LogRecordMatcherTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/LogRecordMatcherTest.java @@ -24,7 +24,7 @@ import java.util.Set; import java.util.logging.Level; import java.util.logging.LogRecord; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/LogSaverTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/LogSaverTest.java index 31047e45ba716..b91539eeae936 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/LogSaverTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/LogSaverTest.java @@ -25,7 +25,7 @@ import java.util.logging.Level; import java.util.logging.LogRecord; import java.util.logging.Logger; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.hamcrest.Matchers; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/OrderedCodeTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/OrderedCodeTest.java index 311e00851b9fa..58e2413e88757 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/OrderedCodeTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/OrderedCodeTest.java @@ -23,8 +23,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Bytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Bytes; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/PairWithConstantKeyDoFnFactoryTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/PairWithConstantKeyDoFnFactoryTest.java index e04d2ea80d4b9..be1ed3c270c4a 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/PairWithConstantKeyDoFnFactoryTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/PairWithConstantKeyDoFnFactoryTest.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.util.StringUtils; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/PartialGroupByKeyParDoFnsTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/PartialGroupByKeyParDoFnsTest.java index 0d392e822c051..17ad1de03d880 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/PartialGroupByKeyParDoFnsTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/PartialGroupByKeyParDoFnsTest.java @@ -71,9 +71,9 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.hamcrest.collection.IsIterableContainingInAnyOrder; import org.junit.Before; import org.junit.Test; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/PartitioningShuffleReaderTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/PartitioningShuffleReaderTest.java index 788439381d9ec..864346278da69 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/PartitioningShuffleReaderTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/PartitioningShuffleReaderTest.java @@ -36,7 +36,7 @@ import org.apache.beam.sdk.transforms.windowing.IntervalWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Assert; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/PubsubDynamicSinkTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/PubsubDynamicSinkTest.java index 71d6e649427aa..d0af1caa5f7ad 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/PubsubDynamicSinkTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/PubsubDynamicSinkTest.java @@ -34,7 +34,7 @@ import org.apache.beam.sdk.transforms.windowing.IntervalWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Instant; import org.junit.Before; import org.junit.Test; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ReaderCacheTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ReaderCacheTest.java index 1418512ff3bf1..861e786642ca8 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ReaderCacheTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ReaderCacheTest.java @@ -27,7 +27,7 @@ import java.util.concurrent.TimeUnit; import org.apache.beam.sdk.io.UnboundedSource; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Stopwatch; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Stopwatch; import org.joda.time.Duration; import org.junit.Before; import org.junit.Test; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ReifyTimestampAndWindowsParDoFnFactoryTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ReifyTimestampAndWindowsParDoFnFactoryTest.java index 860c953d94e6d..55b8d62261c78 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ReifyTimestampAndWindowsParDoFnFactoryTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ReifyTimestampAndWindowsParDoFnFactoryTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; @@ -27,7 +27,7 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; import org.junit.Test; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/RunnerHarnessCoderCloudObjectTranslatorRegistrarTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/RunnerHarnessCoderCloudObjectTranslatorRegistrarTest.java index 9aa015f72706a..9c25259345d9d 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/RunnerHarnessCoderCloudObjectTranslatorRegistrarTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/RunnerHarnessCoderCloudObjectTranslatorRegistrarTest.java @@ -32,8 +32,8 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.VarIntCoder; import org.apache.beam.sdk.coders.VoidCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ShuffleSinkTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ShuffleSinkTest.java index 4f125520877f8..530744af5b0af 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ShuffleSinkTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ShuffleSinkTest.java @@ -39,7 +39,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Assert; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/SimpleParDoFnTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/SimpleParDoFnTest.java index d798bd7186464..76cc74673fb02 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/SimpleParDoFnTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/SimpleParDoFnTest.java @@ -59,9 +59,9 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StateFetcherTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StateFetcherTest.java index 0d13dfa86fb7f..13d8a9bd3ffbd 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StateFetcherTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StateFetcherTest.java @@ -45,9 +45,9 @@ import org.apache.beam.sdk.util.ByteStringOutputStream; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorkerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorkerTest.java index d52335698e534..24e6e2795c683 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorkerTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorkerTest.java @@ -28,6 +28,8 @@ import static org.hamcrest.Matchers.lessThan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.eq; @@ -65,6 +67,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.PriorityQueue; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; @@ -92,8 +95,10 @@ import org.apache.beam.runners.dataflow.util.CloudObjects; import org.apache.beam.runners.dataflow.util.PropertyNames; import org.apache.beam.runners.dataflow.util.Structs; -import org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker.ShardedKey; import org.apache.beam.runners.dataflow.worker.options.StreamingDataflowWorkerOptions; +import org.apache.beam.runners.dataflow.worker.streaming.ComputationState; +import org.apache.beam.runners.dataflow.worker.streaming.ShardedKey; +import org.apache.beam.runners.dataflow.worker.streaming.Work; import org.apache.beam.runners.dataflow.worker.testing.RestoreDataflowLoggingMDC; import org.apache.beam.runners.dataflow.worker.testing.TestCountingSource; import org.apache.beam.runners.dataflow.worker.util.BoundedQueueExecutor; @@ -110,13 +115,13 @@ import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataResponse; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedMessageBundle; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.LatencyAttribution; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.LatencyAttribution.State; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.Timer; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.Timer.Type; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.WatermarkHold; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.WorkItemCommitRequest; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.Coder.Context; -import org.apache.beam.sdk.coders.CoderException; import org.apache.beam.sdk.coders.CollectionCoder; import org.apache.beam.sdk.coders.KvCoder; import org.apache.beam.sdk.coders.ListCoder; @@ -157,17 +162,18 @@ import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.TextFormat; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheStats; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.UnsignedLong; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheStats; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedLong; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.hamcrest.Matcher; import org.hamcrest.Matchers; import org.joda.time.Duration; import org.joda.time.Instant; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ErrorCollector; @@ -188,36 +194,16 @@ @SuppressWarnings("unused") public class StreamingDataflowWorkerTest { - private final boolean streamingEngine; - - @Parameterized.Parameters(name = "{index}: [streamingEngine={0}]") - public static Iterable data() { - return Arrays.asList(new Object[][] {{false}, {true}}); - } - - public StreamingDataflowWorkerTest(Boolean streamingEngine) { - this.streamingEngine = streamingEngine; - } - private static final Logger LOG = LoggerFactory.getLogger(StreamingDataflowWorkerTest.class); - private static final IntervalWindow DEFAULT_WINDOW = new IntervalWindow(new Instant(1234), Duration.millis(1000)); - private static final IntervalWindow WINDOW_AT_ZERO = new IntervalWindow(new Instant(0), new Instant(1000)); - private static final IntervalWindow WINDOW_AT_ONE_SECOND = new IntervalWindow(new Instant(1000), new Instant(2000)); - private static final Coder DEFAULT_WINDOW_CODER = IntervalWindow.getCoder(); private static final Coder> DEFAULT_WINDOW_COLLECTION_CODER = CollectionCoder.of(DEFAULT_WINDOW_CODER); - - private byte[] intervalWindowBytes(IntervalWindow window) throws Exception { - return CoderUtils.encodeToByteArray(DEFAULT_WINDOW_COLLECTION_CODER, Arrays.asList(window)); - } - // Default values that are unimportant for correctness, but must be consistent // between pieces of this test suite private static final String DEFAULT_COMPUTATION_ID = "computation"; @@ -239,14 +225,26 @@ private byte[] intervalWindowBytes(IntervalWindow window) throws Exception { private static final ByteString DEFAULT_KEY_BYTES = ByteString.copyFromUtf8(DEFAULT_KEY_STRING); private static final String DEFAULT_DATA_STRING = "data"; private static final String DEFAULT_DESTINATION_STREAM_ID = "out"; - - @Rule public BlockingFn blockingFn = new BlockingFn(); - @Rule public TestRule restoreMDC = new RestoreDataflowLoggingMDC(); - @Rule public ErrorCollector errorCollector = new ErrorCollector(); - - WorkUnitClient mockWorkUnitClient = mock(WorkUnitClient.class); - HotKeyLogger hotKeyLogger = mock(HotKeyLogger.class); - + private static final Function EMPTY_DATA_RESPONDER = + (GetDataRequest request) -> { + GetDataResponse.Builder builder = GetDataResponse.newBuilder(); + for (ComputationGetDataRequest compRequest : request.getRequestsList()) { + ComputationGetDataResponse.Builder compBuilder = + builder.addDataBuilder().setComputationId(compRequest.getComputationId()); + for (KeyedGetDataRequest keyRequest : compRequest.getRequestsList()) { + KeyedGetDataResponse.Builder keyBuilder = + compBuilder + .addDataBuilder() + .setKey(keyRequest.getKey()) + .setShardingKey(keyRequest.getShardingKey()); + keyBuilder.addAllValues(keyRequest.getValuesToFetchList()); + keyBuilder.addAllBags(keyRequest.getBagsToFetchList()); + keyBuilder.addAllWatermarkHolds(keyRequest.getWatermarkHoldsToFetchList()); + } + } + return builder.build(); + }; + private final boolean streamingEngine; private final Supplier idGenerator = new Supplier() { private final AtomicLong idGenerator = new AtomicLong(1L); @@ -256,6 +254,50 @@ public Long get() { return idGenerator.getAndIncrement(); } }; + @Rule public BlockingFn blockingFn = new BlockingFn(); + @Rule public TestRule restoreMDC = new RestoreDataflowLoggingMDC(); + @Rule public ErrorCollector errorCollector = new ErrorCollector(); + WorkUnitClient mockWorkUnitClient = mock(WorkUnitClient.class); + HotKeyLogger hotKeyLogger = mock(HotKeyLogger.class); + + public StreamingDataflowWorkerTest(Boolean streamingEngine) { + this.streamingEngine = streamingEngine; + } + + @Parameterized.Parameters(name = "{index}: [streamingEngine={0}]") + public static Iterable data() { + return Arrays.asList(new Object[][] {{false}, {true}}); + } + + private static CounterUpdate getCounter(Iterable counters, String name) { + for (CounterUpdate counter : counters) { + if (counter.getNameAndKind().getName().equals(name)) { + return counter; + } + } + return null; + } + + static Work createMockWork(long workToken) { + return Work.create( + Windmill.WorkItem.newBuilder().setKey(ByteString.EMPTY).setWorkToken(workToken).build(), + Instant::now, + Collections.emptyList(), + work -> {}); + } + + static Work createMockWork(long workToken, Consumer processWorkFn) { + return Work.create( + Windmill.WorkItem.newBuilder().setKey(ByteString.EMPTY).setWorkToken(workToken).build(), + Instant::now, + Collections.emptyList(), + processWorkFn); + } + + private byte[] intervalWindowBytes(IntervalWindow window) throws Exception { + return CoderUtils.encodeToByteArray( + DEFAULT_WINDOW_COLLECTION_CODER, Collections.singletonList(window)); + } private String keyStringForIndex(int index) { return DEFAULT_KEY_STRING + index; @@ -270,7 +312,7 @@ private ParallelInstruction makeWindowingSourceInstruction(Coder coder) { CloudObject.forClassName( "com.google.cloud.dataflow.sdk.util.TimerOrElement$TimerOrElementCoder"); List component = - Collections.singletonList(CloudObjects.asCloudObject(coder, /*sdkComponents=*/ null)); + Collections.singletonList(CloudObjects.asCloudObject(coder, /* sdkComponents= */ null)); Structs.addList(timerCloudObject, PropertyNames.COMPONENT_ENCODINGS, component); CloudObject encodedCoder = CloudObject.forClassName("kind:windowed_value"); @@ -280,7 +322,7 @@ private ParallelInstruction makeWindowingSourceInstruction(Coder coder) { PropertyNames.COMPONENT_ENCODINGS, ImmutableList.of( timerCloudObject, - CloudObjects.asCloudObject(IntervalWindowCoder.of(), /*sdkComponents=*/ null))); + CloudObjects.asCloudObject(IntervalWindowCoder.of(), /* sdkComponents= */ null))); return new ParallelInstruction() .setSystemName(DEFAULT_SOURCE_SYSTEM_NAME) @@ -292,7 +334,7 @@ private ParallelInstruction makeWindowingSourceInstruction(Coder coder) { .setSpec(CloudObject.forClass(WindowingWindmillReader.class)) .setCodec(encodedCoder))) .setOutputs( - Arrays.asList( + Collections.singletonList( new InstructionOutput() .setName(Long.toString(idGenerator.get())) .setCodec(encodedCoder) @@ -312,9 +354,9 @@ private ParallelInstruction makeSourceInstruction(Coder coder) { .setCodec( CloudObjects.asCloudObject( WindowedValue.getFullCoder(coder, IntervalWindow.getCoder()), - /*sdkComponents=*/ null)))) + /* sdkComponents= */ null)))) .setOutputs( - Arrays.asList( + Collections.singletonList( new InstructionOutput() .setName(Long.toString(idGenerator.get())) .setOriginalName(DEFAULT_OUTPUT_ORIGINAL_NAME) @@ -322,7 +364,7 @@ private ParallelInstruction makeSourceInstruction(Coder coder) { .setCodec( CloudObjects.asCloudObject( WindowedValue.getFullCoder(coder, IntervalWindow.getCoder()), - /*sdkComponents=*/ null)))); + /* sdkComponents= */ null)))); } private ParallelInstruction makeDoFnInstruction( @@ -357,9 +399,9 @@ private ParallelInstruction makeDoFnInstruction( .setNumOutputs(1) .setUserFn(spec) .setMultiOutputInfos( - Arrays.asList(new MultiOutputInfo().setTag(PropertyNames.OUTPUT)))) + Collections.singletonList(new MultiOutputInfo().setTag(PropertyNames.OUTPUT)))) .setOutputs( - Arrays.asList( + Collections.singletonList( new InstructionOutput() .setName(PropertyNames.OUTPUT) .setOriginalName(DEFAULT_OUTPUT_ORIGINAL_NAME) @@ -368,7 +410,7 @@ private ParallelInstruction makeDoFnInstruction( CloudObjects.asCloudObject( WindowedValue.getFullCoder( outputCoder, windowingStrategy.getWindowFn().windowCoder()), - /*sdkComponents=*/ null)))); + /* sdkComponents= */ null)))); } private ParallelInstruction makeDoFnInstruction( @@ -404,7 +446,7 @@ private ParallelInstruction makeSinkInstruction( .setCodec( CloudObjects.asCloudObject( WindowedValue.getFullCoder(coder, windowCoder), - /*sdkComponents=*/ null)))); + /* sdkComponents= */ null)))); } private ParallelInstruction makeSinkInstruction( @@ -490,26 +532,6 @@ private Windmill.GetWorkResponse buildSessionInput( .build(); } - private static final Function EMPTY_DATA_RESPONDER = - (GetDataRequest request) -> { - GetDataResponse.Builder builder = GetDataResponse.newBuilder(); - for (ComputationGetDataRequest compRequest : request.getRequestsList()) { - ComputationGetDataResponse.Builder compBuilder = - builder.addDataBuilder().setComputationId(compRequest.getComputationId()); - for (KeyedGetDataRequest keyRequest : compRequest.getRequestsList()) { - KeyedGetDataResponse.Builder keyBuilder = - compBuilder - .addDataBuilder() - .setKey(keyRequest.getKey()) - .setShardingKey(keyRequest.getShardingKey()); - keyBuilder.addAllValues(keyRequest.getValuesToFetchList()); - keyBuilder.addAllBags(keyRequest.getBagsToFetchList()); - keyBuilder.addAllWatermarkHolds(keyRequest.getWatermarkHoldsToFetchList()); - } - } - return builder.build(); - }; - private Windmill.GetWorkResponse makeInput(int index, long timestamp) throws Exception { return makeInput(index, timestamp, keyStringForIndex(index), DEFAULT_SHARDING_KEY); } @@ -549,7 +571,8 @@ private Windmill.GetWorkResponse makeInput( + " }" + "}", CoderUtils.encodeToByteArray( - CollectionCoder.of(IntervalWindow.getCoder()), Arrays.asList(DEFAULT_WINDOW))); + CollectionCoder.of(IntervalWindow.getCoder()), + Collections.singletonList(DEFAULT_WINDOW))); } /** @@ -600,6 +623,11 @@ private WorkItemCommitRequest.Builder makeExpectedOutput( parseCommitRequest(expectedCommitRequestBuilder.toString())); } + private WorkItemCommitRequest removeDynamicFields(WorkItemCommitRequest request) { + // Throw away per_work_item_attribution because it is dynamic in tests. + return request.toBuilder().clearPerWorkItemLatencyAttributions().build(); + } + private WorkItemCommitRequest.Builder makeExpectedTruncationRequestOutput( int index, String key, long shardingKey, long estimatedSize) throws Exception { StringBuilder expectedCommitRequestBuilder = @@ -676,8 +704,7 @@ private StreamingComputationConfig makeDefaultStreamingComputationConfig( return config; } - private ByteString addPaneTag(PaneInfo pane, byte[] windowBytes) - throws CoderException, IOException { + private ByteString addPaneTag(PaneInfo pane, byte[] windowBytes) throws IOException { ByteStringOutputStream output = new ByteStringOutputStream(); PaneInfo.PaneInfoCoder.INSTANCE.encode(pane, output, Context.OUTER); output.write(windowBytes); @@ -710,7 +737,7 @@ private StreamingDataflowWorker makeWorker( throws Exception { StreamingDataflowWorker worker = new StreamingDataflowWorker( - Arrays.asList(defaultMapTask(instructions)), + Collections.singletonList(defaultMapTask(instructions)), IntrinsicMapTaskExecutorFactory.defaultFactory(), mockWorkUnitClient, options, @@ -759,7 +786,8 @@ public void testBasicHarness() throws Exception { for (int i = 0; i < numIters; ++i) { assertTrue(result.containsKey((long) i)); assertEquals( - makeExpectedOutput(i, TimeUnit.MILLISECONDS.toMicros(i)).build(), result.get((long) i)); + makeExpectedOutput(i, TimeUnit.MILLISECONDS.toMicros(i)).build(), + removeDynamicFields(result.get((long) i))); } verify(hotKeyLogger, atLeastOnce()).logHotKeyDetection(nullable(String.class), any()); @@ -798,7 +826,8 @@ public void testBasic() throws Exception { for (int i = 0; i < numIters; ++i) { assertTrue(result.containsKey((long) i)); assertEquals( - makeExpectedOutput(i, TimeUnit.MILLISECONDS.toMicros(i)).build(), result.get((long) i)); + makeExpectedOutput(i, TimeUnit.MILLISECONDS.toMicros(i)).build(), + removeDynamicFields(result.get((long) i))); } verify(hotKeyLogger, atLeastOnce()).logHotKeyDetection(nullable(String.class), any()); @@ -878,34 +907,6 @@ public void testHotKeyLoggingNotEnabled() throws Exception { verify(hotKeyLogger, atLeastOnce()).logHotKeyDetection(nullable(String.class), any()); } - static class BlockingFn extends DoFn implements TestRule { - - public static CountDownLatch blocker = new CountDownLatch(1); - public static Semaphore counter = new Semaphore(0); - public static AtomicInteger callCounter = new AtomicInteger(0); - - @ProcessElement - public void processElement(ProcessContext c) throws InterruptedException { - callCounter.incrementAndGet(); - counter.release(); - blocker.await(); - c.output(c.element()); - } - - @Override - public Statement apply(final Statement base, final Description description) { - return new Statement() { - @Override - public void evaluate() throws Throwable { - blocker = new CountDownLatch(1); - counter = new Semaphore(0); - callCounter = new AtomicInteger(); - base.evaluate(); - } - }; - } - } - @Test public void testIgnoreRetriedKeys() throws Exception { final int numIters = 4; @@ -979,7 +980,8 @@ public void testIgnoreRetriedKeys() throws Exception { for (int i = 0; i < numIters; ++i) { assertTrue(result.containsKey((long) i)); assertEquals( - makeExpectedOutput(i, TimeUnit.MILLISECONDS.toMicros(i)).build(), result.get((long) i)); + makeExpectedOutput(i, TimeUnit.MILLISECONDS.toMicros(i)).build(), + removeDynamicFields(result.get((long) i))); assertTrue(result.containsKey((long) i + 1000)); assertEquals( makeExpectedOutput( @@ -989,7 +991,7 @@ public void testIgnoreRetriedKeys() throws Exception { DEFAULT_SHARDING_KEY + 1, keyStringForIndex(i)) .build(), - result.get((long) i + 1000)); + removeDynamicFields(result.get((long) i + 1000))); assertTrue(result.containsKey((long) i + numIters)); assertEquals( makeExpectedOutput( @@ -999,7 +1001,7 @@ public void testIgnoreRetriedKeys() throws Exception { DEFAULT_SHARDING_KEY, keyStringForIndex(i)) .build(), - result.get((long) i + numIters)); + removeDynamicFields(result.get((long) i + numIters))); } // Re-add the work, it should process due to the keys no longer being active. @@ -1025,7 +1027,7 @@ public void testIgnoreRetriedKeys() throws Exception { DEFAULT_SHARDING_KEY, keyStringForIndex(i)) .build(), - result.get((long) i + numIters * 2)); + removeDynamicFields(result.get((long) i + numIters * 2))); } } @@ -1068,21 +1070,6 @@ public void testNumberOfWorkerHarnessThreadsIsHonored() throws Exception { BlockingFn.blocker.countDown(); } - static class KeyTokenInvalidFn extends DoFn, KV> { - - static boolean thrown = false; - - @ProcessElement - public void processElement(ProcessContext c) { - if (!thrown) { - thrown = true; - throw new KeyTokenInvalidException("key"); - } else { - c.output(c.element()); - } - } - } - @Test public void testKeyTokenInvalidException() throws Exception { if (streamingEngine) { @@ -1117,26 +1104,10 @@ public void testKeyTokenInvalidException() throws Exception { assertEquals( makeExpectedOutput(1, 0, DEFAULT_KEY_STRING, DEFAULT_SHARDING_KEY, DEFAULT_KEY_STRING) .build(), - result.get(1L)); + removeDynamicFields(result.get(1L))); assertEquals(1, result.size()); } - static class LargeCommitFn extends DoFn, KV> { - - @ProcessElement - public void processElement(ProcessContext c) { - if (c.element().getKey().equals("large_key")) { - StringBuilder s = new StringBuilder(); - for (int i = 0; i < 100; ++i) { - s.append("large_commit"); - } - c.output(KV.of(c.element().getKey(), s.toString())); - } else { - c.output(c.element()); - } - } - } - @Test public void testKeyCommitTooLargeException() throws Exception { KvCoder kvCoder = KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()); @@ -1165,7 +1136,8 @@ public void testKeyCommitTooLargeException() throws Exception { assertEquals(2, result.size()); assertEquals( - makeExpectedOutput(2, 0, "key", DEFAULT_SHARDING_KEY, "key").build(), result.get(2L)); + makeExpectedOutput(2, 0, "key", DEFAULT_SHARDING_KEY, "key").build(), + removeDynamicFields(result.get(2L))); assertTrue(result.containsKey(1L)); WorkItemCommitRequest largeCommit = result.get(1L); @@ -1204,15 +1176,6 @@ public void testKeyCommitTooLargeException() throws Exception { assertTrue(foundErrors); } - static class ChangeKeysFn extends DoFn, KV> { - - @ProcessElement - public void processElement(ProcessContext c) { - KV elem = c.element(); - c.output(KV.of(elem.getKey() + "_" + elem.getValue(), elem.getValue())); - } - } - @Test public void testKeyChange() throws Exception { KvCoder kvCoder = KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()); @@ -1254,7 +1217,7 @@ public void testKeyChange() throws Exception { DEFAULT_SHARDING_KEY, keyStringForIndex(i) + "_data" + i) .build(), - result.get((long) i)); + removeDynamicFields(result.get((long) i))); assertTrue(result.containsKey((long) i + 1000)); assertEquals( makeExpectedOutput( @@ -1264,25 +1227,8 @@ public void testKeyChange() throws Exception { DEFAULT_SHARDING_KEY + i, keyStringForIndex(i) + "_data" + (i + 1000)) .build(), - result.get((long) i + 1000)); - } - } - - static class TestExceptionFn extends DoFn { - - @ProcessElement - public void processElement(ProcessContext c) throws Exception { - if (firstTime) { - firstTime = false; - try { - throw new Exception("Exception!"); - } catch (Exception e) { - throw new Exception("Another exception!", e); - } - } + removeDynamicFields(result.get((long) i + 1000))); } - - boolean firstTime = true; } @Test(timeout = 30000) @@ -1328,7 +1274,8 @@ public void testExceptions() throws Exception { + " }" + "}", CoderUtils.encodeToByteArray( - CollectionCoder.of(IntervalWindow.getCoder()), Arrays.asList(DEFAULT_WINDOW)))); + CollectionCoder.of(IntervalWindow.getCoder()), + Collections.singletonList(DEFAULT_WINDOW)))); StreamingDataflowWorker worker = makeWorker(instructions, createTestingPipelineOptions(server), true /* publishCounters */); @@ -1410,7 +1357,7 @@ public void testAssignWindows() throws Exception { .setNumOutputs(1) .setUserFn(spec)) .setOutputs( - Arrays.asList( + Collections.singletonList( new InstructionOutput() .setOriginalName(DEFAULT_OUTPUT_ORIGINAL_NAME) .setSystemName(DEFAULT_OUTPUT_SYSTEM_NAME) @@ -1419,7 +1366,7 @@ public void testAssignWindows() throws Exception { CloudObjects.asCloudObject( WindowedValue.getFullCoder( StringUtf8Coder.of(), IntervalWindow.getCoder()), - /*sdkComponents=*/ null)))); + /* sdkComponents= */ null)))); List instructions = Arrays.asList( @@ -1444,7 +1391,7 @@ public void testAssignWindows() throws Exception { Map result = server.waitForAndGetCommits(2); assertThat( - result.get((long) timestamp1), + removeDynamicFields(result.get((long) timestamp1)), equalTo( setMessagesMetadata( PaneInfo.NO_FIRING, @@ -1453,7 +1400,7 @@ public void testAssignWindows() throws Exception { .build())); assertThat( - result.get((long) timestamp2), + removeDynamicFields(result.get((long) timestamp2)), equalTo( setMessagesMetadata( PaneInfo.NO_FIRING, @@ -1527,7 +1474,7 @@ public void testMergeWindows() throws Exception { addObject( spec, WorkerPropertyNames.INPUT_CODER, - CloudObjects.asCloudObject(windowedKvCoder, /*sdkComponents=*/ null)); + CloudObjects.asCloudObject(windowedKvCoder, /* sdkComponents= */ null)); ParallelInstruction mergeWindowsInstruction = new ParallelInstruction() @@ -1540,14 +1487,14 @@ public void testMergeWindows() throws Exception { .setNumOutputs(1) .setUserFn(spec)) .setOutputs( - Arrays.asList( + Collections.singletonList( new InstructionOutput() .setOriginalName(DEFAULT_OUTPUT_ORIGINAL_NAME) .setSystemName(DEFAULT_OUTPUT_SYSTEM_NAME) .setName("output") .setCodec( CloudObjects.asCloudObject( - windowedGroupedCoder, /*sdkComponents=*/ null)))); + windowedGroupedCoder, /* sdkComponents= */ null)))); List instructions = Arrays.asList( @@ -1654,6 +1601,7 @@ public void testMergeWindows() throws Exception { Windmill.WorkItemCommitRequest.newBuilder(actualOutput) .clearCounterUpdates() .clearOutputMessages() + .clearPerWorkItemLatencyAttributions() .build() .getSerializedSize(), splitIntToLong(getCounter(counters, "WindmillStateBytesWritten").getInteger())); @@ -1736,7 +1684,7 @@ public void testMergeWindows() throws Exception { assertEquals( PaneInfo.createPane(true, true, Timing.ON_TIME), PaneInfoCoder.INSTANCE.decode(inStream)); assertEquals( - Arrays.asList(WINDOW_AT_ZERO), + Collections.singletonList(WINDOW_AT_ZERO), DEFAULT_WINDOW_COLLECTION_CODER.decode(inStream, Coder.Context.OUTER)); // Data was deleted @@ -1776,7 +1724,7 @@ public void testMergeWindows() throws Exception { splitIntToLong(getCounter(counters, "WindmillStateBytesRead").getInteger())); // State updates to clear state assertEquals( - Windmill.WorkItemCommitRequest.newBuilder(actualOutput) + Windmill.WorkItemCommitRequest.newBuilder(removeDynamicFields(actualOutput)) .clearCounterUpdates() .clearOutputMessages() .build() @@ -1786,15 +1734,6 @@ public void testMergeWindows() throws Exception { assertEquals(0L, splitIntToLong(getCounter(counters, "WindmillShuffleBytesRead").getInteger())); } - static class PassthroughDoFn - extends DoFn>, KV>> { - - @ProcessElement - public void processElement(ProcessContext c) { - c.output(c.element()); - } - } - @Test // Runs a merging windows test verifying stored state, holds and timers with caching due to // the first processing having is_new_key set. @@ -1822,7 +1761,7 @@ public void testMergeWindowsCaching() throws Exception { addObject( spec, WorkerPropertyNames.INPUT_CODER, - CloudObjects.asCloudObject(windowedKvCoder, /*sdkComponents=*/ null)); + CloudObjects.asCloudObject(windowedKvCoder, /* sdkComponents= */ null)); ParallelInstruction mergeWindowsInstruction = new ParallelInstruction() @@ -1835,14 +1774,14 @@ public void testMergeWindowsCaching() throws Exception { .setNumOutputs(1) .setUserFn(spec)) .setOutputs( - Arrays.asList( + Collections.singletonList( new InstructionOutput() .setOriginalName(DEFAULT_OUTPUT_ORIGINAL_NAME) .setSystemName(DEFAULT_OUTPUT_SYSTEM_NAME) .setName("output") .setCodec( CloudObjects.asCloudObject( - windowedGroupedCoder, /*sdkComponents=*/ null)))); + windowedGroupedCoder, /* sdkComponents= */ null)))); List instructions = Arrays.asList( @@ -1949,7 +1888,7 @@ public void testMergeWindowsCaching() throws Exception { assertEquals(0L, splitIntToLong(getCounter(counters, "WindmillStateBytesRead").getInteger())); // Timer + buffer + watermark hold assertEquals( - Windmill.WorkItemCommitRequest.newBuilder(actualOutput) + Windmill.WorkItemCommitRequest.newBuilder(removeDynamicFields(actualOutput)) .clearCounterUpdates() .clearOutputMessages() .build() @@ -2035,7 +1974,7 @@ public void testMergeWindowsCaching() throws Exception { assertEquals( PaneInfo.createPane(true, true, Timing.ON_TIME), PaneInfoCoder.INSTANCE.decode(inStream)); assertEquals( - Arrays.asList(WINDOW_AT_ZERO), + Collections.singletonList(WINDOW_AT_ZERO), DEFAULT_WINDOW_COLLECTION_CODER.decode(inStream, Coder.Context.OUTER)); // Data was deleted @@ -2075,7 +2014,7 @@ public void testMergeWindowsCaching() throws Exception { splitIntToLong(getCounter(counters, "WindmillStateBytesRead").getInteger())); // State updates to clear state assertEquals( - Windmill.WorkItemCommitRequest.newBuilder(actualOutput) + Windmill.WorkItemCommitRequest.newBuilder(removeDynamicFields(actualOutput)) .clearCounterUpdates() .clearOutputMessages() .build() @@ -2090,27 +2029,6 @@ public void testMergeWindowsCaching() throws Exception { assertEquals(4, stats.missCount()); } - static class Action { - - public Action(GetWorkResponse response) { - this.response = response; - } - - Action withHolds(WatermarkHold... holds) { - this.expectedHolds = holds; - return this; - } - - Action withTimers(Timer... timers) { - this.expectedTimers = timers; - return this; - } - - GetWorkResponse response; - Timer[] expectedTimers = new Timer[] {}; - WatermarkHold[] expectedHolds = new WatermarkHold[] {}; - } - // Helper for running tests for merging sessions based upon Actions consisting of GetWorkResponse // and expected timers and holds in the corresponding commit. All GetData requests are responded // to with empty state, relying on user worker caching to keep data written. @@ -2143,7 +2061,7 @@ private void runMergeSessionsActions(List actions) throws Exception { addObject( spec, WorkerPropertyNames.INPUT_CODER, - CloudObjects.asCloudObject(windowedKvCoder, /*sdkComponents=*/ null)); + CloudObjects.asCloudObject(windowedKvCoder, /* sdkComponents= */ null)); ParallelInstruction mergeWindowsInstruction = new ParallelInstruction() @@ -2156,14 +2074,14 @@ private void runMergeSessionsActions(List actions) throws Exception { .setNumOutputs(1) .setUserFn(spec)) .setOutputs( - Arrays.asList( + Collections.singletonList( new InstructionOutput() .setOriginalName(DEFAULT_OUTPUT_ORIGINAL_NAME) .setSystemName(DEFAULT_OUTPUT_SYSTEM_NAME) .setName("output") .setCodec( CloudObjects.asCloudObject( - windowedGroupedCoder, /*sdkComponents=*/ null)))); + windowedGroupedCoder, /* sdkComponents= */ null)))); List instructions = Arrays.asList( @@ -2198,8 +2116,10 @@ private void runMergeSessionsActions(List actions) throws Exception { public void testMergeSessionWindows() throws Exception { // Test a single late window. runMergeSessionsActions( - Arrays.asList( - new Action(buildSessionInput(1, 40, 0, Arrays.asList(1L), Collections.EMPTY_LIST)) + Collections.singletonList( + new Action( + buildSessionInput( + 1, 40, 0, Collections.singletonList(1L), Collections.EMPTY_LIST)) .withHolds( buildHold("/gAAAAAAAAAsK/+uhold", -1, true), buildHold("/gAAAAAAAAAsK/+uextra", -1, true)) @@ -2213,7 +2133,9 @@ public void testMergeSessionWindows() throws Exception { // elements runMergeSessionsActions( Arrays.asList( - new Action(buildSessionInput(1, 0, 0, Arrays.asList(1L), Collections.EMPTY_LIST)) + new Action( + buildSessionInput( + 1, 0, 0, Collections.singletonList(1L), Collections.EMPTY_LIST)) .withHolds(buildHold("/gAAAAAAAAAsK/+uhold", 10, false)) .withTimers( buildWatermarkTimer("/s/gAAAAAAAAAsK/+0", 10), @@ -2224,12 +2146,14 @@ public void testMergeSessionWindows() throws Exception { 30, 0, Collections.EMPTY_LIST, - Arrays.asList(buildWatermarkTimer("/s/gAAAAAAAAAsK/+0", 10)))) + Collections.singletonList(buildWatermarkTimer("/s/gAAAAAAAAAsK/+0", 10)))) .withTimers(buildWatermarkTimer("/s/gAAAAAAAAAsK/+0", 3600010)) .withHolds( buildHold("/gAAAAAAAAAsK/+uhold", -1, true), buildHold("/gAAAAAAAAAsK/+uextra", -1, true)), - new Action(buildSessionInput(3, 30, 0, Arrays.asList(8L), Collections.EMPTY_LIST)) + new Action( + buildSessionInput( + 3, 30, 0, Collections.singletonList(8L), Collections.EMPTY_LIST)) .withTimers( buildWatermarkTimer("/s/gAAAAAAAABIR/+0", 3600017), buildWatermarkTimer("/s/gAAAAAAAAAsK/+0", 10, true), @@ -2237,7 +2161,9 @@ public void testMergeSessionWindows() throws Exception { .withHolds( buildHold("/gAAAAAAAAAsK/+uhold", -1, true), buildHold("/gAAAAAAAAAsK/+uextra", -1, true)), - new Action(buildSessionInput(4, 30, 0, Arrays.asList(31L), Collections.EMPTY_LIST)) + new Action( + buildSessionInput( + 4, 30, 0, Collections.singletonList(31L), Collections.EMPTY_LIST)) .withTimers( buildWatermarkTimer("/s/gAAAAAAAACkK/+0", 3600040), buildWatermarkTimer("/s/gAAAAAAAACkK/+0", 40)) @@ -2261,31 +2187,13 @@ public void testMergeSessionWindows() throws Exception { 50, 0, Collections.EMPTY_LIST, - Arrays.asList(buildWatermarkTimer("/s/gAAAAAAAACko/+0", 40)))) + Collections.singletonList(buildWatermarkTimer("/s/gAAAAAAAACko/+0", 40)))) .withTimers(buildWatermarkTimer("/s/gAAAAAAAACko/+0", 3600040)) .withHolds( buildHold("/gAAAAAAAAAsK/+uhold", -1, true), buildHold("/gAAAAAAAAAsK/+uextra", -1, true)))); } - private static CounterUpdate getCounter(Iterable counters, String name) { - for (CounterUpdate counter : counters) { - if (counter.getNameAndKind().getName().equals(name)) { - return counter; - } - } - return null; - } - - static class PrintFn extends DoFn>, String> { - - @ProcessElement - public void processElement(ProcessContext c) { - KV elem = c.element().getValue(); - c.output(elem.getKey() + ":" + elem.getValue()); - } - } - private List makeUnboundedSourcePipeline() throws Exception { return makeUnboundedSourcePipeline(1, new PrintFn()); } @@ -2303,7 +2211,7 @@ private List makeUnboundedSourcePipeline( ValueWithRecordId.ValueWithRecordIdCoder.of( KvCoder.of(VarIntCoder.of(), VarIntCoder.of())), GlobalWindow.Coder.INSTANCE), - /*sdkComponents=*/ null); + /* sdkComponents= */ null); return Arrays.asList( new ParallelInstruction() @@ -2316,7 +2224,7 @@ private List makeUnboundedSourcePipeline( new TestCountingSource(numMessagesPerShard), options) .setCodec(codec))) .setOutputs( - Arrays.asList( + Collections.singletonList( new InstructionOutput() .setName("read_output") .setOriginalName(DEFAULT_OUTPUT_ORIGINAL_NAME) @@ -2363,13 +2271,13 @@ public void testUnboundedSources() throws Exception { UnsignedLong.fromLongBits(commit.getSourceStateUpdates().getFinalizeIds(0)); assertThat( - commit, + removeDynamicFields(commit), equalTo( setMessagesMetadata( PaneInfo.NO_FIRING, CoderUtils.encodeToByteArray( CollectionCoder.of(GlobalWindow.Coder.INSTANCE), - Arrays.asList(GlobalWindow.INSTANCE)), + Collections.singletonList(GlobalWindow.INSTANCE)), parseCommitRequest( "key: \"0000000000000001\" " + "sharding_key: 1 " @@ -2425,7 +2333,7 @@ public void testUnboundedSources() throws Exception { finalizeId = UnsignedLong.fromLongBits(commit.getSourceStateUpdates().getFinalizeIds(0)); assertThat( - commit, + removeDynamicFields(commit), equalTo( parseCommitRequest( "key: \"0000000000000001\" " @@ -2444,7 +2352,7 @@ public void testUnboundedSources() throws Exception { assertThat(finalizeTracker, contains(0)); - assertEquals(null, getCounter(counters, "dataflow_input_size-computation")); + assertNull(getCounter(counters, "dataflow_input_size-computation")); // Test recovery (on a new key so fresh reader state). Counter is done. server @@ -2473,7 +2381,7 @@ public void testUnboundedSources() throws Exception { finalizeId = UnsignedLong.fromLongBits(commit.getSourceStateUpdates().getFinalizeIds(0)); assertThat( - commit, + removeDynamicFields(commit), equalTo( parseCommitRequest( "key: \"0000000000000002\" " @@ -2490,7 +2398,7 @@ public void testUnboundedSources() throws Exception { + "source_watermark: 1000") .build())); - assertEquals(null, getCounter(counters, "dataflow_input_size-computation")); + assertNull(getCounter(counters, "dataflow_input_size-computation")); } @Test @@ -2530,13 +2438,13 @@ public void testUnboundedSourcesDrain() throws Exception { UnsignedLong.fromLongBits(commit.getSourceStateUpdates().getFinalizeIds(0)); assertThat( - commit, + removeDynamicFields(commit), equalTo( setMessagesMetadata( PaneInfo.NO_FIRING, CoderUtils.encodeToByteArray( CollectionCoder.of(GlobalWindow.Coder.INSTANCE), - Arrays.asList(GlobalWindow.INSTANCE)), + Collections.singletonList(GlobalWindow.INSTANCE)), parseCommitRequest( "key: \"0000000000000001\" " + "sharding_key: 1 " @@ -2646,7 +2554,7 @@ public void testUnboundedSourceWorkRetry() throws Exception { PaneInfo.NO_FIRING, CoderUtils.encodeToByteArray( CollectionCoder.of(GlobalWindow.Coder.INSTANCE), - Arrays.asList(GlobalWindow.INSTANCE)), + Collections.singletonList(GlobalWindow.INSTANCE)), parseCommitRequest( "key: \"0000000000000001\" " + "sharding_key: 1 " @@ -2673,7 +2581,7 @@ public void testUnboundedSourceWorkRetry() throws Exception { + "source_watermark: 1000")) .build(); - assertThat(commit, equalTo(expectedCommit)); + assertThat(removeDynamicFields(commit), equalTo(expectedCommit)); // Test retry of work item, it should return the same result and not start the reader from the // position it was left at. @@ -2687,7 +2595,7 @@ public void testUnboundedSourceWorkRetry() throws Exception { .getSourceStateUpdatesBuilder() .setFinalizeIds(0, commit.getSourceStateUpdates().getFinalizeIds(0)); expectedCommit = commitBuilder.build(); - assertThat(commit, equalTo(expectedCommit)); + assertThat(removeDynamicFields(commit), equalTo(expectedCommit)); // Continue with processing. server @@ -2717,7 +2625,7 @@ public void testUnboundedSourceWorkRetry() throws Exception { finalizeId = UnsignedLong.fromLongBits(commit.getSourceStateUpdates().getFinalizeIds(0)); assertThat( - commit, + removeDynamicFields(commit), equalTo( parseCommitRequest( "key: \"0000000000000001\" " @@ -2737,25 +2645,13 @@ public void testUnboundedSourceWorkRetry() throws Exception { assertThat(finalizeTracker, contains(0)); } - private static class MockWork extends StreamingDataflowWorker.Work { - - public MockWork(long workToken) { - super( - Windmill.WorkItem.newBuilder().setKey(ByteString.EMPTY).setWorkToken(workToken).build(), - Instant::now); - } - - @Override - public void run() {} - } - @Test public void testActiveWork() throws Exception { BoundedQueueExecutor mockExecutor = Mockito.mock(BoundedQueueExecutor.class); - StreamingDataflowWorker.ComputationState computationState = - new StreamingDataflowWorker.ComputationState( + ComputationState computationState = + new ComputationState( "computation", - defaultMapTask(Arrays.asList(makeSourceInstruction(StringUtf8Coder.of()))), + defaultMapTask(Collections.singletonList(makeSourceInstruction(StringUtf8Coder.of()))), mockExecutor, ImmutableMap.of(), null); @@ -2763,49 +2659,49 @@ public void testActiveWork() throws Exception { ShardedKey key1 = ShardedKey.create(ByteString.copyFromUtf8("key1"), 1); ShardedKey key2 = ShardedKey.create(ByteString.copyFromUtf8("key2"), 2); - MockWork m1 = new MockWork(1); + Work m1 = createMockWork(1); assertTrue(computationState.activateWork(key1, m1)); Mockito.verify(mockExecutor).execute(m1, m1.getWorkItem().getSerializedSize()); - computationState.completeWork(key1, 1); + computationState.completeWorkAndScheduleNextWorkForKey(key1, 1); Mockito.verifyNoMoreInteractions(mockExecutor); // Verify work queues. - MockWork m2 = new MockWork(2); + Work m2 = createMockWork(2); assertTrue(computationState.activateWork(key1, m2)); Mockito.verify(mockExecutor).execute(m2, m2.getWorkItem().getSerializedSize()); - MockWork m3 = new MockWork(3); + Work m3 = createMockWork(3); assertTrue(computationState.activateWork(key1, m3)); Mockito.verifyNoMoreInteractions(mockExecutor); // Verify another key is a separate queue. - MockWork m4 = new MockWork(4); + Work m4 = createMockWork(4); assertTrue(computationState.activateWork(key2, m4)); Mockito.verify(mockExecutor).execute(m4, m4.getWorkItem().getSerializedSize()); - computationState.completeWork(key2, 4); + computationState.completeWorkAndScheduleNextWorkForKey(key2, 4); Mockito.verifyNoMoreInteractions(mockExecutor); - computationState.completeWork(key1, 2); + computationState.completeWorkAndScheduleNextWorkForKey(key1, 2); Mockito.verify(mockExecutor).forceExecute(m3, m3.getWorkItem().getSerializedSize()); - computationState.completeWork(key1, 3); + computationState.completeWorkAndScheduleNextWorkForKey(key1, 3); Mockito.verifyNoMoreInteractions(mockExecutor); // Verify duplicate work dropped. - MockWork m5 = new MockWork(5); + Work m5 = createMockWork(5); computationState.activateWork(key1, m5); Mockito.verify(mockExecutor).execute(m5, m5.getWorkItem().getSerializedSize()); assertFalse(computationState.activateWork(key1, m5)); Mockito.verifyNoMoreInteractions(mockExecutor); - computationState.completeWork(key1, 5); + computationState.completeWorkAndScheduleNextWorkForKey(key1, 5); Mockito.verifyNoMoreInteractions(mockExecutor); } @Test public void testActiveWorkForShardedKeys() throws Exception { BoundedQueueExecutor mockExecutor = Mockito.mock(BoundedQueueExecutor.class); - StreamingDataflowWorker.ComputationState computationState = - new StreamingDataflowWorker.ComputationState( + ComputationState computationState = + new ComputationState( "computation", - defaultMapTask(Arrays.asList(makeSourceInstruction(StringUtf8Coder.of()))), + defaultMapTask(Collections.singletonList(makeSourceInstruction(StringUtf8Coder.of()))), mockExecutor, ImmutableMap.of(), null); @@ -2813,22 +2709,22 @@ public void testActiveWorkForShardedKeys() throws Exception { ShardedKey key1Shard1 = ShardedKey.create(ByteString.copyFromUtf8("key1"), 1); ShardedKey key1Shard2 = ShardedKey.create(ByteString.copyFromUtf8("key1"), 2); - MockWork m1 = new MockWork(1); + Work m1 = createMockWork(1); assertTrue(computationState.activateWork(key1Shard1, m1)); Mockito.verify(mockExecutor).execute(m1, m1.getWorkItem().getSerializedSize()); - computationState.completeWork(key1Shard1, 1); + computationState.completeWorkAndScheduleNextWorkForKey(key1Shard1, 1); Mockito.verifyNoMoreInteractions(mockExecutor); // Verify work queues. - MockWork m2 = new MockWork(2); + Work m2 = createMockWork(2); assertTrue(computationState.activateWork(key1Shard1, m2)); Mockito.verify(mockExecutor).execute(m2, m2.getWorkItem().getSerializedSize()); - MockWork m3 = new MockWork(3); + Work m3 = createMockWork(3); assertTrue(computationState.activateWork(key1Shard1, m3)); Mockito.verifyNoMoreInteractions(mockExecutor); // Verify a different shard of key is a separate queue. - MockWork m4 = new MockWork(3); + Work m4 = createMockWork(3); assertFalse(computationState.activateWork(key1Shard1, m4)); Mockito.verifyNoMoreInteractions(mockExecutor); assertTrue(computationState.activateWork(key1Shard2, m4)); @@ -2836,43 +2732,138 @@ public void testActiveWorkForShardedKeys() throws Exception { // Verify duplicate work dropped assertFalse(computationState.activateWork(key1Shard2, m4)); - computationState.completeWork(key1Shard2, 3); + computationState.completeWorkAndScheduleNextWorkForKey(key1Shard2, 3); Mockito.verifyNoMoreInteractions(mockExecutor); } - static class TestExceptionInvalidatesCacheFn - extends DoFn>, String> { + @Test + @Ignore // Test is flaky on Jenkins (#27555) + public void testMaxThreadMetric() throws Exception { + int maxThreads = 2; + int threadExpiration = 60; + // setting up actual implementation of executor instead of mocking to keep track of + // active thread count. + BoundedQueueExecutor executor = + new BoundedQueueExecutor( + maxThreads, + threadExpiration, + TimeUnit.SECONDS, + maxThreads, + 10000000, + new ThreadFactoryBuilder() + .setNameFormat("DataflowWorkUnits-%d") + .setDaemon(true) + .build()); + + ComputationState computationState = + new ComputationState( + "computation", + defaultMapTask(Collections.singletonList(makeSourceInstruction(StringUtf8Coder.of()))), + executor, + ImmutableMap.of(), + null); - static boolean thrown = false; + ShardedKey key1Shard1 = ShardedKey.create(ByteString.copyFromUtf8("key1"), 1); - @StateId("int") - private final StateSpec> counter = StateSpecs.value(VarIntCoder.of()); + // overriding definition of MockWork to add sleep, which will help us keep track of how + // long each work item takes to process and therefore let us manipulate how long the time + // at which we're at max threads is. + Consumer sleepProcessWorkFn = + unused -> { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }; - @ProcessElement - public void processElement(ProcessContext c, @StateId("int") ValueState state) - throws Exception { - KV elem = c.element().getValue(); - if (elem.getValue() == 0) { - LOG.error("**** COUNTER 0 ****"); - assertEquals(null, state.read()); - state.write(42); - assertEquals((Integer) 42, state.read()); - } else if (elem.getValue() == 1) { - LOG.error("**** COUNTER 1 ****"); - assertEquals((Integer) 42, state.read()); - } else if (elem.getValue() == 2) { - if (!thrown) { - LOG.error("**** COUNTER 2 (will throw) ****"); - thrown = true; - throw new Exception("Exception!"); - } - LOG.error("**** COUNTER 2 (retry) ****"); - assertEquals((Integer) 42, state.read()); - } else { - throw new RuntimeException("only expecting values [0,2]"); - } - c.output(elem.getKey() + ":" + elem.getValue()); + Work m2 = createMockWork(2, sleepProcessWorkFn); + Work m3 = createMockWork(3, sleepProcessWorkFn); + + assertTrue(computationState.activateWork(key1Shard1, m2)); + assertTrue(computationState.activateWork(key1Shard1, m3)); + executor.execute(m2, m2.getWorkItem().getSerializedSize()); + + executor.execute(m3, m3.getWorkItem().getSerializedSize()); + + // Will get close to 1000ms that both work items are processing (sleeping, really) + // give or take a few ms. + long i = 990L; + assertTrue(executor.allThreadsActiveTime() >= i); + executor.shutdown(); + } + + volatile boolean stop = false; + + @Test + public void testActiveThreadMetric() throws Exception { + int maxThreads = 5; + int threadExpirationSec = 60; + // setting up actual implementation of executor instead of mocking to keep track of + // active thread count. + BoundedQueueExecutor executor = + new BoundedQueueExecutor( + maxThreads, + threadExpirationSec, + TimeUnit.SECONDS, + maxThreads, + 10000000, + new ThreadFactoryBuilder() + .setNameFormat("DataflowWorkUnits-%d") + .setDaemon(true) + .build()); + + ComputationState computationState = + new ComputationState( + "computation", + defaultMapTask(Arrays.asList(makeSourceInstruction(StringUtf8Coder.of()))), + executor, + ImmutableMap.of(), + null); + + ShardedKey key1Shard1 = ShardedKey.create(ByteString.copyFromUtf8("key1"), 1); + + Consumer sleepProcessWorkFn = + unused -> { + synchronized (this) { + this.notify(); + } + int count = 0; + while (!stop) { + count += 1; + } + }; + + Work m2 = createMockWork(2, sleepProcessWorkFn); + + Work m3 = createMockWork(3, sleepProcessWorkFn); + + Work m4 = createMockWork(4, sleepProcessWorkFn); + assertEquals(0, executor.activeCount()); + + assertTrue(computationState.activateWork(key1Shard1, m2)); + synchronized (this) { + executor.execute(m2, m2.getWorkItem().getSerializedSize()); + this.wait(); + // Seems current executor executes the initial work item twice + this.wait(); + } + assertEquals(2, executor.activeCount()); + + assertTrue(computationState.activateWork(key1Shard1, m3)); + assertTrue(computationState.activateWork(key1Shard1, m4)); + synchronized (this) { + executor.execute(m3, m3.getWorkItem().getSerializedSize()); + this.wait(); } + assertEquals(3, executor.activeCount()); + synchronized (this) { + executor.execute(m4, m4.getWorkItem().getSerializedSize()); + this.wait(); + } + assertEquals(4, executor.activeCount()); + stop = true; + executor.shutdown(); } @Test @@ -2920,7 +2911,7 @@ public void testExceptionInvalidatesCache() throws Exception { ValueWithRecordId.ValueWithRecordIdCoder.of( KvCoder.of(VarIntCoder.of(), VarIntCoder.of())), GlobalWindow.Coder.INSTANCE), - /*sdkComponents=*/ null); + /* sdkComponents= */ null); TestCountingSource counter = new TestCountingSource(3).withThrowOnFirstSnapshot(true); @@ -2935,7 +2926,7 @@ public void testExceptionInvalidatesCache() throws Exception { .setSource( CustomSources.serializeToCloudSource(counter, options).setCodec(codec))) .setOutputs( - Arrays.asList( + Collections.singletonList( new InstructionOutput() .setName("read_output") .setOriginalName(DEFAULT_OUTPUT_ORIGINAL_NAME) @@ -3068,7 +3059,11 @@ public void testExceptionInvalidatesCache() throws Exception { assertThat( // The commit will include a timer to clean up state - this timer is irrelevant // for the current test. Also remove source_bytes_processed because it's dynamic. - setValuesTimestamps(commit.toBuilder().clearOutputTimers().clearSourceBytesProcessed()) + setValuesTimestamps( + removeDynamicFields(commit) + .toBuilder() + .clearOutputTimers() + .clearSourceBytesProcessed()) .build(), equalTo( setMessagesMetadata( @@ -3081,21 +3076,6 @@ public void testExceptionInvalidatesCache() throws Exception { } } - private static class FanoutFn extends DoFn { - - @ProcessElement - public void processElement(ProcessContext c) { - StringBuilder builder = new StringBuilder(1000000); - for (int i = 0; i < 1000000; i++) { - builder.append(' '); - } - String largeString = builder.toString(); - for (int i = 0; i < 3000; i++) { - c.output(largeString); - } - } - } - @Test public void testHugeCommits() throws Exception { List instructions = @@ -3115,15 +3095,6 @@ public void testHugeCommits() throws Exception { worker.stop(); } - private static class SlowDoFn extends DoFn { - - @ProcessElement - public void processElement(ProcessContext c) throws Exception { - Thread.sleep(1000); - c.output(c.element()); - } - } - @Test public void testActiveWorkRefresh() throws Exception { List instructions = @@ -3148,297 +3119,135 @@ public void testActiveWorkRefresh() throws Exception { assertThat(server.numGetDataRequests(), greaterThan(0)); } - static class FakeClock implements Supplier { - private class FakeScheduledExecutor implements ScheduledExecutorService { - @Override - public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { - return true; - } + @Test + public void testLatencyAttributionProtobufsPopulated() { + FakeClock clock = new FakeClock(); + Work work = Work.create(null, clock, Collections.emptyList(), unused -> {}); - @Override - public void execute(Runnable command) { - throw new UnsupportedOperationException("Not implemented yet"); - } + clock.sleep(Duration.millis(10)); + work.setState(Work.State.PROCESSING); + clock.sleep(Duration.millis(20)); + work.setState(Work.State.READING); + clock.sleep(Duration.millis(30)); + work.setState(Work.State.PROCESSING); + clock.sleep(Duration.millis(40)); + work.setState(Work.State.COMMIT_QUEUED); + clock.sleep(Duration.millis(50)); + work.setState(Work.State.COMMITTING); + clock.sleep(Duration.millis(60)); - @Override - public List> invokeAll(Collection> tasks) - throws InterruptedException { - throw new UnsupportedOperationException("Not implemented yet"); - } + Iterator it = work.getLatencyAttributions().iterator(); + assertTrue(it.hasNext()); + LatencyAttribution lat = it.next(); + assertSame(State.QUEUED, lat.getState()); + assertEquals(10, lat.getTotalDurationMillis()); + assertTrue(it.hasNext()); + lat = it.next(); + assertSame(State.ACTIVE, lat.getState()); + assertEquals(60, lat.getTotalDurationMillis()); + assertTrue(it.hasNext()); + lat = it.next(); + assertSame(State.READING, lat.getState()); + assertEquals(30, lat.getTotalDurationMillis()); + assertTrue(it.hasNext()); + lat = it.next(); + assertSame(State.COMMITTING, lat.getState()); + assertEquals(110, lat.getTotalDurationMillis()); + assertFalse(it.hasNext()); + } - @Override - public List> invokeAll( - Collection> tasks, long timeout, TimeUnit unit) - throws InterruptedException { - throw new UnsupportedOperationException("Not implemented yet"); - } + @Test + public void testLatencyAttributionToQueuedState() throws Exception { + final int workToken = 3232; // A unique id makes it easier to search logs. - @Override - public T invokeAny(Collection> tasks) - throws ExecutionException, InterruptedException { - throw new UnsupportedOperationException("Not implemented yet"); - } + FakeClock clock = new FakeClock(); + List instructions = + Arrays.asList( + makeSourceInstruction(StringUtf8Coder.of()), + makeDoFnInstruction( + new FakeSlowDoFn(clock, Duration.millis(1000)), 0, StringUtf8Coder.of()), + makeSinkInstruction(StringUtf8Coder.of(), 0)); - @Override - public T invokeAny(Collection> tasks, long timeout, TimeUnit unit) - throws ExecutionException, InterruptedException, TimeoutException { - throw new UnsupportedOperationException("Not implemented yet"); - } + FakeWindmillServer server = new FakeWindmillServer(errorCollector); + StreamingDataflowWorkerOptions options = createTestingPipelineOptions(server); + options.setActiveWorkRefreshPeriodMillis(100); + // A single-threaded worker processes work sequentially, leaving a second work item in state + // QUEUED until the first work item is committed. + options.setNumberOfWorkerHarnessThreads(1); + StreamingDataflowWorker worker = + makeWorker( + instructions, + options, + false /* publishCounters */, + clock, + clock::newFakeScheduledExecutor); + worker.start(); - @Override - public boolean isShutdown() { - throw new UnsupportedOperationException("Not implemented yet"); - } + ActiveWorkRefreshSink awrSink = new ActiveWorkRefreshSink(EMPTY_DATA_RESPONDER); + server.whenGetDataCalled().answerByDefault(awrSink::getData).delayEachResponseBy(Duration.ZERO); + server + .whenGetWorkCalled() + .thenReturn(makeInput(workToken + 1, 0 /* timestamp */)) + .thenReturn(makeInput(workToken, 1 /* timestamp */)); + server.waitForAndGetCommits(2); - @Override - public boolean isTerminated() { - throw new UnsupportedOperationException("Not implemented yet"); - } + worker.stop(); - @Override - public void shutdown() {} + assertEquals( + awrSink.getLatencyAttributionDuration(workToken, State.QUEUED), Duration.millis(1000)); + assertEquals(awrSink.getLatencyAttributionDuration(workToken + 1, State.QUEUED), Duration.ZERO); + } - @Override - public List shutdownNow() { - throw new UnsupportedOperationException("Not implemented yet"); - } + @Test + public void testLatencyAttributionToActiveState() throws Exception { + final int workToken = 4242; // A unique id makes it easier to search logs. - @Override - public Future submit(Callable task) { - throw new UnsupportedOperationException("Not implemented yet"); - } + FakeClock clock = new FakeClock(); + // Inject processing latency on the fake clock in the worker via FakeSlowDoFn. + List instructions = + Arrays.asList( + makeSourceInstruction(StringUtf8Coder.of()), + makeDoFnInstruction( + new FakeSlowDoFn(clock, Duration.millis(1000)), 0, StringUtf8Coder.of()), + makeSinkInstruction(StringUtf8Coder.of(), 0)); - @Override - public Future submit(Runnable task) { - throw new UnsupportedOperationException("Not implemented yet"); - } + FakeWindmillServer server = new FakeWindmillServer(errorCollector); + StreamingDataflowWorkerOptions options = createTestingPipelineOptions(server); + options.setActiveWorkRefreshPeriodMillis(100); + StreamingDataflowWorker worker = + makeWorker( + instructions, + options, + false /* publishCounters */, + clock, + clock::newFakeScheduledExecutor); + worker.start(); - @Override - public Future submit(Runnable task, T result) { - throw new UnsupportedOperationException("Not implemented yet"); - } + ActiveWorkRefreshSink awrSink = new ActiveWorkRefreshSink(EMPTY_DATA_RESPONDER); + server.whenGetDataCalled().answerByDefault(awrSink::getData).delayEachResponseBy(Duration.ZERO); + server.whenGetWorkCalled().thenReturn(makeInput(workToken, 0 /* timestamp */)); + server.waitForAndGetCommits(1); - @Override - public ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit) { - throw new UnsupportedOperationException("Not implemented yet"); - } + worker.stop(); - @Override - public ScheduledFuture schedule(Runnable command, long delay, TimeUnit unit) { - throw new UnsupportedOperationException("Not implemented yet"); - } + assertEquals( + awrSink.getLatencyAttributionDuration(workToken, State.ACTIVE), Duration.millis(1000)); + } - @Override - public ScheduledFuture scheduleAtFixedRate( - Runnable command, long initialDelay, long period, TimeUnit unit) { - throw new UnsupportedOperationException("Not implemented yet"); - } - - @Override - public ScheduledFuture scheduleWithFixedDelay( - Runnable command, long initialDelay, long delay, TimeUnit unit) { - if (delay <= 0) { - throw new UnsupportedOperationException( - "Please supply a delay > 0 to scheduleWithFixedDelay"); - } - FakeClock.this.schedule( - Duration.millis(unit.toMillis(initialDelay)), - new Runnable() { - @Override - public void run() { - command.run(); - FakeClock.this.schedule(Duration.millis(unit.toMillis(delay)), this); - } - }); - FakeClock.this.sleep(Duration.ZERO); // Execute work that has an intial delay of zero. - return null; - } - } - - private static class Job implements Comparable { - final Instant when; - final Runnable work; - - Job(Instant when, Runnable work) { - this.when = when; - this.work = work; - } - - @Override - public int compareTo(Job job) { - return when.compareTo(job.when); - } - } - - private final PriorityQueue jobs = new PriorityQueue<>(); - private Instant now = Instant.now(); - - public ScheduledExecutorService newFakeScheduledExecutor(String unused) { - return new FakeScheduledExecutor(); - } - - @Override - public synchronized Instant get() { - return now; - } - - public synchronized void clear() { - jobs.clear(); - } - - public synchronized void sleep(Duration duration) { - if (duration.isShorterThan(Duration.ZERO)) { - throw new UnsupportedOperationException("Cannot sleep backwards in time"); - } - Instant endOfSleep = now.plus(duration); - while (true) { - Job job = jobs.peek(); - if (job == null || job.when.isAfter(endOfSleep)) { - break; - } - jobs.remove(); - now = job.when; - job.work.run(); - } - now = endOfSleep; - } - - private synchronized void schedule(Duration fromNow, Runnable work) { - jobs.add(new Job(now.plus(fromNow), work)); - } - } - - private static class FakeSlowDoFn extends DoFn { - private static FakeClock clock; // A static variable keeps this DoFn serializable. - private final Duration sleep; - - FakeSlowDoFn(FakeClock clock, Duration sleep) { - FakeSlowDoFn.clock = clock; - this.sleep = sleep; - } - - @ProcessElement - public void processElement(ProcessContext c) throws Exception { - clock.sleep(sleep); - c.output(c.element()); - } - } - - @Test - public void testLatencyAttributionProtobufsPopulated() throws Exception { - FakeClock clock = new FakeClock(); - StreamingDataflowWorker.Work work = - new StreamingDataflowWorker.Work(null, clock) { - @Override - public void run() {} - }; - - clock.sleep(Duration.millis(10)); - work.setState(StreamingDataflowWorker.Work.State.PROCESSING); - clock.sleep(Duration.millis(20)); - work.setState(StreamingDataflowWorker.Work.State.READING); - clock.sleep(Duration.millis(30)); - work.setState(StreamingDataflowWorker.Work.State.PROCESSING); - clock.sleep(Duration.millis(40)); - work.setState(StreamingDataflowWorker.Work.State.COMMIT_QUEUED); - clock.sleep(Duration.millis(50)); - work.setState(StreamingDataflowWorker.Work.State.COMMITTING); - clock.sleep(Duration.millis(60)); - - Iterator it = work.getLatencyAttributionList().iterator(); - assertTrue(it.hasNext()); - LatencyAttribution lat = it.next(); - assertTrue(lat.getState() == LatencyAttribution.State.QUEUED); - assertTrue(lat.getTotalDurationMillis() == 10); - assertTrue(it.hasNext()); - lat = it.next(); - assertTrue(lat.getState() == LatencyAttribution.State.ACTIVE); - assertTrue(lat.getTotalDurationMillis() == 60); - assertTrue(it.hasNext()); - lat = it.next(); - assertTrue(lat.getState() == LatencyAttribution.State.READING); - assertTrue(lat.getTotalDurationMillis() == 30); - assertTrue(it.hasNext()); - lat = it.next(); - assertTrue(lat.getState() == LatencyAttribution.State.COMMITTING); - assertTrue(lat.getTotalDurationMillis() == 110); - assertTrue(!it.hasNext()); - } - - // Aggregates LatencyAttribution data from active work refresh requests. - static class ActiveWorkRefreshSink { - private final Function responder; - private final Map> totalDurations = - new HashMap<>(); - - ActiveWorkRefreshSink(Function responder) { - this.responder = responder; - } - - Duration getLatencyAttributionDuration(long workToken, LatencyAttribution.State state) { - EnumMap durations = totalDurations.get(workToken); - return durations == null ? Duration.ZERO : durations.getOrDefault(state, Duration.ZERO); - } - - boolean isActiveWorkRefresh(GetDataRequest request) { - for (ComputationGetDataRequest computationRequest : request.getRequestsList()) { - if (!computationRequest.getComputationId().equals(DEFAULT_COMPUTATION_ID)) { - return false; - } - for (KeyedGetDataRequest keyedRequest : computationRequest.getRequestsList()) { - if (keyedRequest.getWorkToken() == 0 - || keyedRequest.getShardingKey() != DEFAULT_SHARDING_KEY - || keyedRequest.getValuesToFetchCount() != 0 - || keyedRequest.getBagsToFetchCount() != 0 - || keyedRequest.getTagValuePrefixesToFetchCount() != 0 - || keyedRequest.getWatermarkHoldsToFetchCount() != 0) { - return false; - } - } - } - return true; - } - - GetDataResponse getData(GetDataRequest request) { - if (!isActiveWorkRefresh(request)) { - return responder.apply(request); - } - for (ComputationGetDataRequest computationRequest : request.getRequestsList()) { - for (KeyedGetDataRequest keyedRequest : computationRequest.getRequestsList()) { - for (LatencyAttribution la : keyedRequest.getLatencyAttributionList()) { - EnumMap durations = - totalDurations.computeIfAbsent( - keyedRequest.getWorkToken(), - (Long workToken) -> - new EnumMap( - LatencyAttribution.State.class)); - Duration cur = Duration.millis(la.getTotalDurationMillis()); - durations.compute(la.getState(), (s, d) -> d == null || d.isShorterThan(cur) ? cur : d); - } - } - } - return EMPTY_DATA_RESPONDER.apply(request); - } - } - - @Test - public void testLatencyAttributionToQueuedState() throws Exception { - final int workToken = 3232; // A unique id makes it easier to search logs. + @Test + public void testLatencyAttributionToReadingState() throws Exception { + final int workToken = 5454; // A unique id makes it easier to find logs. FakeClock clock = new FakeClock(); List instructions = Arrays.asList( makeSourceInstruction(StringUtf8Coder.of()), - makeDoFnInstruction( - new FakeSlowDoFn(clock, Duration.millis(1000)), 0, StringUtf8Coder.of()), + makeDoFnInstruction(new ReadingDoFn(), 0, StringUtf8Coder.of()), makeSinkInstruction(StringUtf8Coder.of(), 0)); FakeWindmillServer server = new FakeWindmillServer(errorCollector); StreamingDataflowWorkerOptions options = createTestingPipelineOptions(server); options.setActiveWorkRefreshPeriodMillis(100); - // A single-threaded worker processes work sequentially, leaving a second work item in state - // QUEUED until the first work item is committed. - options.setNumberOfWorkerHarnessThreads(1); StreamingDataflowWorker worker = makeWorker( instructions, @@ -3448,40 +3257,43 @@ public void testLatencyAttributionToQueuedState() throws Exception { clock::newFakeScheduledExecutor); worker.start(); - ActiveWorkRefreshSink awrSink = new ActiveWorkRefreshSink(EMPTY_DATA_RESPONDER); + // Inject latency on the fake clock when the server receives a GetData call that isn't + // only for refreshing active work. + ActiveWorkRefreshSink awrSink = + new ActiveWorkRefreshSink( + (request) -> { + clock.sleep(Duration.millis(1000)); + return EMPTY_DATA_RESPONDER.apply(request); + }); server.whenGetDataCalled().answerByDefault(awrSink::getData).delayEachResponseBy(Duration.ZERO); - server - .whenGetWorkCalled() - .thenReturn(makeInput(workToken + 1, 0 /* timestamp */)) - .thenReturn(makeInput(workToken, 1 /* timestamp */)); - server.waitForAndGetCommits(2); + server.whenGetWorkCalled().thenReturn(makeInput(workToken, 0 /* timestamp */)); + server.waitForAndGetCommits(1); worker.stop(); - assertTrue( - awrSink - .getLatencyAttributionDuration(workToken, LatencyAttribution.State.QUEUED) - .equals(Duration.millis(1000))); - assertTrue( - awrSink - .getLatencyAttributionDuration(workToken + 1, LatencyAttribution.State.QUEUED) - .equals(Duration.ZERO)); + assertEquals( + awrSink.getLatencyAttributionDuration(workToken, State.READING), Duration.millis(1000)); } @Test - public void testLatencyAttributionToActiveState() throws Exception { - final int workToken = 4242; // A unique id makes it easier to search logs. + public void testLatencyAttributionToCommittingState() throws Exception { + final int workToken = 6464; // A unique id makes it easier to find logs. FakeClock clock = new FakeClock(); - // Inject processing latency on the fake clock in the worker via FakeSlowDoFn. List instructions = Arrays.asList( makeSourceInstruction(StringUtf8Coder.of()), - makeDoFnInstruction( - new FakeSlowDoFn(clock, Duration.millis(1000)), 0, StringUtf8Coder.of()), makeSinkInstruction(StringUtf8Coder.of(), 0)); + // Inject latency on the fake clock when the server receives a CommitWork call. FakeWindmillServer server = new FakeWindmillServer(errorCollector); + server + .whenCommitWorkCalled() + .answerByDefault( + (request) -> { + clock.sleep(Duration.millis(1000)); + return Windmill.CommitWorkResponse.getDefaultInstance(); + }); StreamingDataflowWorkerOptions options = createTestingPipelineOptions(server); options.setActiveWorkRefreshPeriodMillis(100); StreamingDataflowWorker worker = @@ -3495,43 +3307,32 @@ public void testLatencyAttributionToActiveState() throws Exception { ActiveWorkRefreshSink awrSink = new ActiveWorkRefreshSink(EMPTY_DATA_RESPONDER); server.whenGetDataCalled().answerByDefault(awrSink::getData).delayEachResponseBy(Duration.ZERO); - server.whenGetWorkCalled().thenReturn(makeInput(workToken, 0 /* timestamp */)); + server.whenGetWorkCalled().thenReturn(makeInput(workToken, TimeUnit.MILLISECONDS.toMicros(0))); server.waitForAndGetCommits(1); worker.stop(); - assertTrue( - awrSink - .getLatencyAttributionDuration(workToken, LatencyAttribution.State.ACTIVE) - .equals(Duration.millis(1000))); - } - - // A DoFn that triggers a GetData request. - static class ReadingDoFn extends DoFn { - @StateId("int") - private final StateSpec> counter = StateSpecs.value(VarIntCoder.of()); - - @ProcessElement - public void processElement(ProcessContext c, @StateId("int") ValueState state) { - state.read(); - c.output(c.element()); - } + assertEquals( + awrSink.getLatencyAttributionDuration(workToken, State.COMMITTING), Duration.millis(1000)); } @Test - public void testLatencyAttributionToReadingState() throws Exception { - final int workToken = 5454; // A unique id makes it easier to find logs. + public void testLatencyAttributionPopulatedInCommitRequest() throws Exception { + final int workToken = 7272; // A unique id makes it easier to search logs. + long dofnWaitTimeMs = 1000; FakeClock clock = new FakeClock(); List instructions = Arrays.asList( makeSourceInstruction(StringUtf8Coder.of()), - makeDoFnInstruction(new ReadingDoFn(), 0, StringUtf8Coder.of()), + makeDoFnInstruction( + new FakeSlowDoFn(clock, Duration.millis(dofnWaitTimeMs)), 0, StringUtf8Coder.of()), makeSinkInstruction(StringUtf8Coder.of(), 0)); FakeWindmillServer server = new FakeWindmillServer(errorCollector); StreamingDataflowWorkerOptions options = createTestingPipelineOptions(server); options.setActiveWorkRefreshPeriodMillis(100); + options.setNumberOfWorkerHarnessThreads(1); StreamingDataflowWorker worker = makeWorker( instructions, @@ -3541,96 +3342,40 @@ public void testLatencyAttributionToReadingState() throws Exception { clock::newFakeScheduledExecutor); worker.start(); - // Inject latency on the fake clock when the server receives a GetData call that isn't - // only for refreshing active work. - ActiveWorkRefreshSink awrSink = - new ActiveWorkRefreshSink( - (request) -> { - clock.sleep(Duration.millis(1000)); - return EMPTY_DATA_RESPONDER.apply(request); - }); + ActiveWorkRefreshSink awrSink = new ActiveWorkRefreshSink(EMPTY_DATA_RESPONDER); server.whenGetDataCalled().answerByDefault(awrSink::getData).delayEachResponseBy(Duration.ZERO); - server.whenGetWorkCalled().thenReturn(makeInput(workToken, 0 /* timestamp */)); - server.waitForAndGetCommits(1); + server.whenGetWorkCalled().thenReturn(makeInput(workToken, 1 /* timestamp */)); + Map workItemCommitRequest = server.waitForAndGetCommits(1); worker.stop(); - assertTrue( - awrSink - .getLatencyAttributionDuration(workToken, LatencyAttribution.State.READING) - .equals(Duration.millis(1000))); + assertEquals( + workItemCommitRequest.get((long) workToken).getPerWorkItemLatencyAttributions(0), + LatencyAttribution.newBuilder() + .setState(State.ACTIVE) + .setTotalDurationMillis(dofnWaitTimeMs) + .build()); + if (streamingEngine) { + // Initial fake latency provided to FakeWindmillServer when invoke receiveWork in + // GetWorkStream(). + assertEquals( + workItemCommitRequest.get((long) workToken).getPerWorkItemLatencyAttributions(1), + LatencyAttribution.newBuilder() + .setState(State.GET_WORK_IN_TRANSIT_TO_USER_WORKER) + .setTotalDurationMillis(1000) + .build()); + } } @Test - public void testLatencyAttributionToCommittingState() throws Exception { - final int workToken = 6464; // A unique id makes it easier to find logs. + public void testLimitOnOutputBundleSize() throws Exception { + // This verifies that ReadOperation, StreamingModeExecutionContext, and windmill sinks + // coordinate to limit size of an output bundle. + List finalizeTracker = Lists.newArrayList(); + TestCountingSource.setFinalizeTracker(finalizeTracker); - FakeClock clock = new FakeClock(); - List instructions = - Arrays.asList( - makeSourceInstruction(StringUtf8Coder.of()), - makeSinkInstruction(StringUtf8Coder.of(), 0)); - - // Inject latency on the fake clock when the server receives a CommitWork call. - FakeWindmillServer server = new FakeWindmillServer(errorCollector); - server - .whenCommitWorkCalled() - .answerByDefault( - (request) -> { - clock.sleep(Duration.millis(1000)); - return Windmill.CommitWorkResponse.getDefaultInstance(); - }); - StreamingDataflowWorkerOptions options = createTestingPipelineOptions(server); - options.setActiveWorkRefreshPeriodMillis(100); - StreamingDataflowWorker worker = - makeWorker( - instructions, - options, - false /* publishCounters */, - clock, - clock::newFakeScheduledExecutor); - worker.start(); - - ActiveWorkRefreshSink awrSink = new ActiveWorkRefreshSink(EMPTY_DATA_RESPONDER); - server.whenGetDataCalled().answerByDefault(awrSink::getData).delayEachResponseBy(Duration.ZERO); - server.whenGetWorkCalled().thenReturn(makeInput(workToken, TimeUnit.MILLISECONDS.toMicros(0))); - server.waitForAndGetCommits(1); - - worker.stop(); - - assertTrue( - awrSink - .getLatencyAttributionDuration(workToken, LatencyAttribution.State.COMMITTING) - .equals(Duration.millis(1000))); - } - - /** For each input element, emits a large string. */ - private static class InflateDoFn extends DoFn>, String> { - - final int inflatedSize; - - /** For each input elements, outputs a string of this length */ - InflateDoFn(int inflatedSize) { - this.inflatedSize = inflatedSize; - } - - @ProcessElement - public void processElement(ProcessContext c) { - char[] chars = new char[inflatedSize]; - Arrays.fill(chars, ' '); - c.output(new String(chars)); - } - } - - @Test - public void testLimitOnOutputBundleSize() throws Exception { - // This verifies that ReadOperation, StreamingModeExecutionContext, and windmill sinks - // coordinate to limit size of an output bundle. - List finalizeTracker = Lists.newArrayList(); - TestCountingSource.setFinalizeTracker(finalizeTracker); - - final int numMessagesInCustomSourceShard = 100000; // 100K input messages. - final int inflatedSizePerMessage = 10000; // x10k => 1GB total output size. + final int numMessagesInCustomSourceShard = 100000; // 100K input messages. + final int inflatedSizePerMessage = 10000; // x10k => 1GB total output size. FakeWindmillServer server = new FakeWindmillServer(errorCollector); StreamingDataflowWorker worker = @@ -3818,6 +3563,461 @@ public void testStuckCommit() throws Exception { makeExpectedOutput( 1, TimeUnit.MILLISECONDS.toMicros(1), DEFAULT_KEY_STRING, 1, DEFAULT_KEY_STRING) .build(), - result.get(1L)); + removeDynamicFields(result.get(1L))); + } + + static class BlockingFn extends DoFn implements TestRule { + + public static CountDownLatch blocker = new CountDownLatch(1); + public static Semaphore counter = new Semaphore(0); + public static AtomicInteger callCounter = new AtomicInteger(0); + + @ProcessElement + public void processElement(ProcessContext c) throws InterruptedException { + callCounter.incrementAndGet(); + counter.release(); + blocker.await(); + c.output(c.element()); + } + + @Override + public Statement apply(final Statement base, final Description description) { + return new Statement() { + @Override + public void evaluate() throws Throwable { + blocker = new CountDownLatch(1); + counter = new Semaphore(0); + callCounter = new AtomicInteger(); + base.evaluate(); + } + }; + } + } + + static class KeyTokenInvalidFn extends DoFn, KV> { + + static boolean thrown = false; + + @ProcessElement + public void processElement(ProcessContext c) { + if (!thrown) { + thrown = true; + throw new KeyTokenInvalidException("key"); + } else { + c.output(c.element()); + } + } + } + + static class LargeCommitFn extends DoFn, KV> { + + @ProcessElement + public void processElement(ProcessContext c) { + if (c.element().getKey().equals("large_key")) { + StringBuilder s = new StringBuilder(); + for (int i = 0; i < 100; ++i) { + s.append("large_commit"); + } + c.output(KV.of(c.element().getKey(), s.toString())); + } else { + c.output(c.element()); + } + } + } + + static class ChangeKeysFn extends DoFn, KV> { + + @ProcessElement + public void processElement(ProcessContext c) { + KV elem = c.element(); + c.output(KV.of(elem.getKey() + "_" + elem.getValue(), elem.getValue())); + } + } + + static class TestExceptionFn extends DoFn { + + boolean firstTime = true; + + @ProcessElement + public void processElement(ProcessContext c) throws Exception { + if (firstTime) { + firstTime = false; + try { + throw new Exception("Exception!"); + } catch (Exception e) { + throw new Exception("Another exception!", e); + } + } + } + } + + static class PassthroughDoFn + extends DoFn>, KV>> { + + @ProcessElement + public void processElement(ProcessContext c) { + c.output(c.element()); + } + } + + static class Action { + + GetWorkResponse response; + Timer[] expectedTimers = new Timer[] {}; + WatermarkHold[] expectedHolds = new WatermarkHold[] {}; + + public Action(GetWorkResponse response) { + this.response = response; + } + + Action withHolds(WatermarkHold... holds) { + this.expectedHolds = holds; + return this; + } + + Action withTimers(Timer... timers) { + this.expectedTimers = timers; + return this; + } + } + + static class PrintFn extends DoFn>, String> { + + @ProcessElement + public void processElement(ProcessContext c) { + KV elem = c.element().getValue(); + c.output(elem.getKey() + ":" + elem.getValue()); + } + } + + private static class MockWork { + Work create(long workToken) { + return Work.create( + Windmill.WorkItem.newBuilder().setKey(ByteString.EMPTY).setWorkToken(workToken).build(), + Instant::now, + Collections.emptyList(), + work -> {}); + } + } + + static class TestExceptionInvalidatesCacheFn + extends DoFn>, String> { + + static boolean thrown = false; + + @StateId("int") + private final StateSpec> counter = StateSpecs.value(VarIntCoder.of()); + + @ProcessElement + public void processElement(ProcessContext c, @StateId("int") ValueState state) + throws Exception { + KV elem = c.element().getValue(); + if (elem.getValue() == 0) { + LOG.error("**** COUNTER 0 ****"); + assertNull(state.read()); + state.write(42); + assertEquals((Integer) 42, state.read()); + } else if (elem.getValue() == 1) { + LOG.error("**** COUNTER 1 ****"); + assertEquals((Integer) 42, state.read()); + } else if (elem.getValue() == 2) { + if (!thrown) { + LOG.error("**** COUNTER 2 (will throw) ****"); + thrown = true; + throw new Exception("Exception!"); + } + LOG.error("**** COUNTER 2 (retry) ****"); + assertEquals((Integer) 42, state.read()); + } else { + throw new RuntimeException("only expecting values [0,2]"); + } + c.output(elem.getKey() + ":" + elem.getValue()); + } + } + + private static class FanoutFn extends DoFn { + + @ProcessElement + public void processElement(ProcessContext c) { + StringBuilder builder = new StringBuilder(1000000); + for (int i = 0; i < 1000000; i++) { + builder.append(' '); + } + String largeString = builder.toString(); + for (int i = 0; i < 3000; i++) { + c.output(largeString); + } + } + } + + private static class SlowDoFn extends DoFn { + + @ProcessElement + public void processElement(ProcessContext c) throws Exception { + Thread.sleep(1000); + c.output(c.element()); + } + } + + static class FakeClock implements Supplier { + private final PriorityQueue jobs = new PriorityQueue<>(); + private Instant now = Instant.now(); + + public ScheduledExecutorService newFakeScheduledExecutor(String unused) { + return new FakeScheduledExecutor(); + } + + @Override + public synchronized Instant get() { + return now; + } + + public synchronized void clear() { + jobs.clear(); + } + + public synchronized void sleep(Duration duration) { + if (duration.isShorterThan(Duration.ZERO)) { + throw new UnsupportedOperationException("Cannot sleep backwards in time"); + } + Instant endOfSleep = now.plus(duration); + while (true) { + Job job = jobs.peek(); + if (job == null || job.when.isAfter(endOfSleep)) { + break; + } + jobs.remove(); + now = job.when; + job.work.run(); + } + now = endOfSleep; + } + + private synchronized void schedule(Duration fromNow, Runnable work) { + jobs.add(new Job(now.plus(fromNow), work)); + } + + private static class Job implements Comparable { + final Instant when; + final Runnable work; + + Job(Instant when, Runnable work) { + this.when = when; + this.work = work; + } + + @Override + public int compareTo(Job job) { + return when.compareTo(job.when); + } + } + + private class FakeScheduledExecutor implements ScheduledExecutorService { + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return true; + } + + @Override + public void execute(Runnable command) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public List> invokeAll(Collection> tasks) + throws InterruptedException { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public List> invokeAll( + Collection> tasks, long timeout, TimeUnit unit) + throws InterruptedException { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public T invokeAny(Collection> tasks) + throws ExecutionException, InterruptedException { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public T invokeAny(Collection> tasks, long timeout, TimeUnit unit) + throws ExecutionException, InterruptedException, TimeoutException { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public boolean isShutdown() { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public boolean isTerminated() { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public void shutdown() {} + + @Override + public List shutdownNow() { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public Future submit(Callable task) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public Future submit(Runnable task) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public Future submit(Runnable task, T result) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public ScheduledFuture schedule(Runnable command, long delay, TimeUnit unit) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public ScheduledFuture scheduleAtFixedRate( + Runnable command, long initialDelay, long period, TimeUnit unit) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public ScheduledFuture scheduleWithFixedDelay( + Runnable command, long initialDelay, long delay, TimeUnit unit) { + if (delay <= 0) { + throw new UnsupportedOperationException( + "Please supply a delay > 0 to scheduleWithFixedDelay"); + } + FakeClock.this.schedule( + Duration.millis(unit.toMillis(initialDelay)), + new Runnable() { + @Override + public void run() { + command.run(); + FakeClock.this.schedule(Duration.millis(unit.toMillis(delay)), this); + } + }); + FakeClock.this.sleep(Duration.ZERO); // Execute work that has an intial delay of zero. + return null; + } + } + } + + private static class FakeSlowDoFn extends DoFn { + private static FakeClock clock; // A static variable keeps this DoFn serializable. + private final Duration sleep; + + FakeSlowDoFn(FakeClock clock, Duration sleep) { + FakeSlowDoFn.clock = clock; + this.sleep = sleep; + } + + @ProcessElement + public void processElement(ProcessContext c) throws Exception { + clock.sleep(sleep); + c.output(c.element()); + } + } + + // Aggregates LatencyAttribution data from active work refresh requests. + static class ActiveWorkRefreshSink { + private final Function responder; + private final Map> totalDurations = + new HashMap<>(); + + ActiveWorkRefreshSink(Function responder) { + this.responder = responder; + } + + Duration getLatencyAttributionDuration(long workToken, LatencyAttribution.State state) { + EnumMap durations = totalDurations.get(workToken); + return durations == null ? Duration.ZERO : durations.getOrDefault(state, Duration.ZERO); + } + + boolean isActiveWorkRefresh(GetDataRequest request) { + for (ComputationGetDataRequest computationRequest : request.getRequestsList()) { + if (!computationRequest.getComputationId().equals(DEFAULT_COMPUTATION_ID)) { + return false; + } + for (KeyedGetDataRequest keyedRequest : computationRequest.getRequestsList()) { + if (keyedRequest.getWorkToken() == 0 + || keyedRequest.getShardingKey() != DEFAULT_SHARDING_KEY + || keyedRequest.getValuesToFetchCount() != 0 + || keyedRequest.getBagsToFetchCount() != 0 + || keyedRequest.getTagValuePrefixesToFetchCount() != 0 + || keyedRequest.getWatermarkHoldsToFetchCount() != 0) { + return false; + } + } + } + return true; + } + + GetDataResponse getData(GetDataRequest request) { + if (!isActiveWorkRefresh(request)) { + return responder.apply(request); + } + for (ComputationGetDataRequest computationRequest : request.getRequestsList()) { + for (KeyedGetDataRequest keyedRequest : computationRequest.getRequestsList()) { + for (LatencyAttribution la : keyedRequest.getLatencyAttributionList()) { + EnumMap durations = + totalDurations.computeIfAbsent( + keyedRequest.getWorkToken(), + (Long workToken) -> + new EnumMap( + LatencyAttribution.State.class)); + Duration cur = Duration.millis(la.getTotalDurationMillis()); + durations.compute(la.getState(), (s, d) -> d == null || d.isShorterThan(cur) ? cur : d); + } + } + } + return EMPTY_DATA_RESPONDER.apply(request); + } + } + + // A DoFn that triggers a GetData request. + static class ReadingDoFn extends DoFn { + @StateId("int") + private final StateSpec> counter = StateSpecs.value(VarIntCoder.of()); + + @ProcessElement + public void processElement(ProcessContext c, @StateId("int") ValueState state) { + state.read(); + c.output(c.element()); + } + } + + /** For each input element, emits a large string. */ + private static class InflateDoFn extends DoFn>, String> { + + final int inflatedSize; + + /** For each input elements, outputs a string of this length */ + InflateDoFn(int inflatedSize) { + this.inflatedSize = inflatedSize; + } + + @ProcessElement + public void processElement(ProcessContext c) { + char[] chars = new char[inflatedSize]; + Arrays.fill(chars, ' '); + c.output(new String(chars)); + } } } diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingKeyedWorkItemSideInputDoFnRunnerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingKeyedWorkItemSideInputDoFnRunnerTest.java index 5d6f1e7099f19..c23a5d142be63 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingKeyedWorkItemSideInputDoFnRunnerTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingKeyedWorkItemSideInputDoFnRunnerTest.java @@ -54,8 +54,8 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Before; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingModeExecutionContextTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingModeExecutionContextTest.java index e8209e43d38b9..6620dbdaab79d 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingModeExecutionContextTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingModeExecutionContextTest.java @@ -56,6 +56,8 @@ import org.apache.beam.runners.dataflow.worker.profiler.ScopedProfiler.NoopProfileScope; import org.apache.beam.runners.dataflow.worker.profiler.ScopedProfiler.ProfileScope; import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.runners.dataflow.worker.windmill.state.WindmillStateCache; +import org.apache.beam.runners.dataflow.worker.windmill.state.WindmillStateReader; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.metrics.MetricsContainer; import org.apache.beam.sdk.options.PipelineOptionsFactory; @@ -65,7 +67,7 @@ import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.hamcrest.Matchers; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingPCollectionViewWriterDoFnFactoryTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingPCollectionViewWriterDoFnFactoryTest.java index 4c2c434a40692..374d4ac6308dd 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingPCollectionViewWriterDoFnFactoryTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingPCollectionViewWriterDoFnFactoryTest.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.coders.BigEndianIntegerCoder; import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputDoFnRunnerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputDoFnRunnerTest.java index cb5fef770804c..05e0ff4176155 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputDoFnRunnerTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputDoFnRunnerTest.java @@ -64,7 +64,7 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.hamcrest.Matchers; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputFetcherTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputFetcherTest.java index 7f80a3e5fecc0..9ce462be32115 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputFetcherTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputFetcherTest.java @@ -50,8 +50,8 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.hamcrest.Matchers; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainerTest.java index 1a4c43905d20a..9e6d45a2351be 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainerTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainerTest.java @@ -20,6 +20,7 @@ import static org.apache.beam.runners.dataflow.worker.counters.DataflowCounterUpdateExtractor.longToSplitInt; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.sameInstance; @@ -33,6 +34,9 @@ import org.apache.beam.sdk.metrics.Distribution; import org.apache.beam.sdk.metrics.MetricName; import org.apache.beam.sdk.metrics.MetricsContainer; +import org.apache.beam.sdk.metrics.NoOpCounter; +import org.apache.beam.sdk.metrics.NoOpHistogram; +import org.apache.beam.sdk.util.HistogramData; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -178,4 +182,22 @@ public void testDistributionUpdateExtraction() { .setMin(longToSplitInt(3)) .setSum(longToSplitInt(3))))); } + + @Test + public void testPerWorkerMetrics() { + StreamingStepMetricsContainer.setEnablePerWorkerMetrics(false); + MetricsContainer metricsContainer = registry.getContainer("test_step"); + assertThat( + metricsContainer.getPerWorkerCounter(name1), sameInstance(NoOpCounter.getInstance())); + HistogramData.BucketType testBucket = HistogramData.LinearBuckets.of(1, 1, 1); + assertThat( + metricsContainer.getPerWorkerHistogram(name1, testBucket), + sameInstance(NoOpHistogram.getInstance())); + + StreamingStepMetricsContainer.setEnablePerWorkerMetrics(true); + assertThat(metricsContainer.getPerWorkerCounter(name1), not(instanceOf(NoOpCounter.class))); + assertThat( + metricsContainer.getPerWorkerHistogram(name1, testBucket), + not(instanceOf(NoOpHistogram.class))); + } } diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ToIsmRecordForMultimapDoFnFactoryTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ToIsmRecordForMultimapDoFnFactoryTest.java index 7d273d63351af..02ca8e923a23c 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ToIsmRecordForMultimapDoFnFactoryTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ToIsmRecordForMultimapDoFnFactoryTest.java @@ -30,8 +30,8 @@ import org.apache.beam.runners.dataflow.worker.util.common.worker.ParDoFn; import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/UngroupedShuffleReaderTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/UngroupedShuffleReaderTest.java index 241029e06e259..1e504d97b81d5 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/UngroupedShuffleReaderTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/UngroupedShuffleReaderTest.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.transforms.windowing.IntervalWindow; import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Assert; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/UserParDoFnFactoryTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/UserParDoFnFactoryTest.java index c8c46bf6caec3..ff114ef2f0789 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/UserParDoFnFactoryTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/UserParDoFnFactoryTest.java @@ -68,7 +68,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Test; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ValuesDoFnFactoryTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ValuesDoFnFactoryTest.java index d360753456cd5..c0e5a301ea2a4 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ValuesDoFnFactoryTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ValuesDoFnFactoryTest.java @@ -27,7 +27,7 @@ import org.apache.beam.runners.dataflow.util.PropertyNames; import org.apache.beam.runners.dataflow.worker.util.common.worker.ParDoFn; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WindmillStateInternalsTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WindmillStateInternalsTest.java deleted file mode 100644 index 2e81b37c01627..0000000000000 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WindmillStateInternalsTest.java +++ /dev/null @@ -1,1862 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.runners.dataflow.worker; - -import static org.apache.beam.runners.dataflow.worker.DataflowMatchers.ByteStringMatcher.byteStringEq; -import static org.apache.beam.sdk.testing.SystemNanoTimeSleeper.sleepMillis; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.when; - -import java.io.Closeable; -import java.io.IOException; -import java.util.AbstractMap; -import java.util.AbstractMap.SimpleEntry; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import javax.annotation.Nullable; -import org.apache.beam.runners.core.StateNamespace; -import org.apache.beam.runners.core.StateNamespaceForTest; -import org.apache.beam.runners.core.StateTag; -import org.apache.beam.runners.core.StateTags; -import org.apache.beam.runners.dataflow.options.DataflowWorkerHarnessOptions; -import org.apache.beam.runners.dataflow.worker.WindmillStateInternals.IdTracker; -import org.apache.beam.runners.dataflow.worker.WindmillStateInternals.WindmillOrderedList; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.TagBag; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.TagSortedListUpdateRequest; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.TagValue; -import org.apache.beam.sdk.coders.Coder; -import org.apache.beam.sdk.coders.Coder.Context; -import org.apache.beam.sdk.coders.StringUtf8Coder; -import org.apache.beam.sdk.coders.VarIntCoder; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.sdk.state.BagState; -import org.apache.beam.sdk.state.CombiningState; -import org.apache.beam.sdk.state.GroupingState; -import org.apache.beam.sdk.state.MapState; -import org.apache.beam.sdk.state.OrderedListState; -import org.apache.beam.sdk.state.ReadableState; -import org.apache.beam.sdk.state.ValueState; -import org.apache.beam.sdk.state.WatermarkHoldState; -import org.apache.beam.sdk.transforms.Sum; -import org.apache.beam.sdk.transforms.windowing.TimestampCombiner; -import org.apache.beam.sdk.util.ByteStringOutputStream; -import org.apache.beam.sdk.util.CoderUtils; -import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Range; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.RangeSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Futures; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.SettableFuture; -import org.hamcrest.Matchers; -import org.joda.time.Instant; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; - -/** Tests for {@link WindmillStateInternals}. */ -@RunWith(JUnit4.class) -@SuppressWarnings({ - "rawtypes", // TODO(https://github.com/apache/beam/issues/20447) -}) -public class WindmillStateInternalsTest { - - private static final StateNamespace NAMESPACE = new StateNamespaceForTest("ns"); - private static final String STATE_FAMILY = "family"; - - private static final StateTag> COMBINING_ADDR = - StateTags.combiningValueFromInputInternal("combining", VarIntCoder.of(), Sum.ofIntegers()); - private static final ByteString COMBINING_KEY = key(NAMESPACE, "combining"); - private final Coder accumCoder = - Sum.ofIntegers().getAccumulatorCoder(null, VarIntCoder.of()); - private long workToken = 0; - - DataflowWorkerHarnessOptions options; - - @Mock private WindmillStateReader mockReader; - - private WindmillStateInternals underTest; - private WindmillStateInternals underTestNewKey; - private WindmillStateCache cache; - - @Mock private Supplier readStateSupplier; - - private static ByteString key(StateNamespace namespace, String addrId) { - return ByteString.copyFromUtf8(namespace.stringKey() + "+u" + addrId); - } - - private static ByteString systemKey(StateNamespace namespace, String addrId) { - return ByteString.copyFromUtf8(namespace.stringKey() + "+s" + addrId); - } - - @Before - public void setUp() { - MockitoAnnotations.initMocks(this); - options = PipelineOptionsFactory.as(DataflowWorkerHarnessOptions.class); - cache = new WindmillStateCache(options.getWorkerCacheMb()); - resetUnderTest(); - } - - public void resetUnderTest() { - workToken++; - underTest = - new WindmillStateInternals<>( - "dummyKey", - STATE_FAMILY, - mockReader, - false, - cache - .forComputation("comp") - .forKey( - WindmillComputationKey.create( - "comp", ByteString.copyFrom("dummyKey", Charsets.UTF_8), 123), - 17L, - workToken) - .forFamily(STATE_FAMILY), - readStateSupplier); - underTestNewKey = - new WindmillStateInternals( - "dummyNewKey", - STATE_FAMILY, - mockReader, - true, - cache - .forComputation("comp") - .forKey( - WindmillComputationKey.create( - "comp", ByteString.copyFrom("dummyNewKey", Charsets.UTF_8), 123), - 17L, - workToken) - .forFamily(STATE_FAMILY), - readStateSupplier); - } - - @After - public void tearDown() throws Exception { - // Make sure no WindmillStateReader (a per-WorkItem object) escapes into the cache - // (a global object). - WindmillStateTestUtils.assertNoReference(cache, WindmillStateReader.class); - } - - private void waitAndSet(final SettableFuture future, final T value, final long millis) { - new Thread( - () -> { - try { - sleepMillis(millis); - } catch (InterruptedException e) { - throw new RuntimeException("Interrupted before setting", e); - } - future.set(value); - }) - .run(); - } - - private WindmillStateReader.WeightedList weightedList(String... elems) { - WindmillStateReader.WeightedList result = - new WindmillStateReader.WeightedList<>(new ArrayList(elems.length)); - for (String elem : elems) { - result.addWeighted(elem, elem.length()); - } - return result; - } - - private ByteString protoKeyFromUserKey(@Nullable K tag, Coder keyCoder) - throws IOException { - ByteStringOutputStream keyStream = new ByteStringOutputStream(); - key(NAMESPACE, "map").writeTo(keyStream); - if (tag != null) { - keyCoder.encode(tag, keyStream, Context.OUTER); - } - return keyStream.toByteString(); - } - - private K userKeyFromProtoKey(ByteString tag, Coder keyCoder) throws IOException { - ByteString keyBytes = tag.substring(key(NAMESPACE, "map").size()); - return keyCoder.decode(keyBytes.newInput(), Context.OUTER); - } - - @Test - public void testMapAddBeforeGet() throws Exception { - StateTag> addr = - StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); - MapState mapState = underTest.state(NAMESPACE, addr); - - final String tag = "tag"; - SettableFuture future = SettableFuture.create(); - when(mockReader.valueFuture( - protoKeyFromUserKey(tag, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) - .thenReturn(future); - - ReadableState result = mapState.get("tag"); - result = result.readLater(); - waitAndSet(future, 1, 200); - assertEquals(1, (int) result.read()); - mapState.put("tag", 2); - assertEquals(2, (int) result.read()); - } - - @Test - public void testMapAddClearBeforeGet() throws Exception { - StateTag> addr = - StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); - MapState mapState = underTest.state(NAMESPACE, addr); - - final String tag = "tag"; - - SettableFuture>> prefixFuture = SettableFuture.create(); - when(mockReader.valuePrefixFuture( - protoKeyFromUserKey(null, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) - .thenReturn(prefixFuture); - - ReadableState result = mapState.get("tag"); - result = result.readLater(); - waitAndSet( - prefixFuture, - ImmutableList.of( - new AbstractMap.SimpleEntry<>(protoKeyFromUserKey(tag, StringUtf8Coder.of()), 1)), - 50); - assertFalse(mapState.isEmpty().read()); - mapState.clear(); - assertTrue(mapState.isEmpty().read()); - assertNull(mapState.get("tag").read()); - mapState.put("tag", 2); - assertFalse(mapState.isEmpty().read()); - assertEquals(2, (int) result.read()); - } - - @Test - public void testMapLocalAddOverridesStorage() throws Exception { - StateTag> addr = - StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); - MapState mapState = underTest.state(NAMESPACE, addr); - - final String tag = "tag"; - - SettableFuture future = SettableFuture.create(); - when(mockReader.valueFuture( - protoKeyFromUserKey(tag, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) - .thenReturn(future); - SettableFuture>> prefixFuture = SettableFuture.create(); - when(mockReader.valuePrefixFuture( - protoKeyFromUserKey(null, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) - .thenReturn(prefixFuture); - - waitAndSet(future, 1, 50); - waitAndSet( - prefixFuture, - ImmutableList.of( - new AbstractMap.SimpleEntry<>(protoKeyFromUserKey(tag, StringUtf8Coder.of()), 1)), - 50); - mapState.put(tag, 42); - assertEquals(42, (int) mapState.get(tag).read()); - assertThat( - mapState.entries().read(), - Matchers.containsInAnyOrder(new AbstractMap.SimpleEntry<>(tag, 42))); - } - - @Test - public void testMapLocalRemoveOverridesStorage() throws Exception { - StateTag> addr = - StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); - MapState mapState = underTest.state(NAMESPACE, addr); - - final String tag1 = "tag1"; - final String tag2 = "tag2"; - - SettableFuture future = SettableFuture.create(); - when(mockReader.valueFuture( - protoKeyFromUserKey(tag1, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) - .thenReturn(future); - SettableFuture>> prefixFuture = SettableFuture.create(); - when(mockReader.valuePrefixFuture( - protoKeyFromUserKey(null, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) - .thenReturn(prefixFuture); - - waitAndSet(future, 1, 50); - waitAndSet( - prefixFuture, - ImmutableList.of( - new AbstractMap.SimpleEntry<>(protoKeyFromUserKey(tag1, StringUtf8Coder.of()), 1), - new AbstractMap.SimpleEntry<>(protoKeyFromUserKey(tag2, StringUtf8Coder.of()), 2)), - 50); - mapState.remove(tag1); - assertNull(mapState.get(tag1).read()); - assertThat( - mapState.entries().read(), - Matchers.containsInAnyOrder(new AbstractMap.SimpleEntry<>(tag2, 2))); - - mapState.remove(tag2); - assertTrue(mapState.isEmpty().read()); - } - - @Test - public void testMapLocalClearOverridesStorage() throws Exception { - StateTag> addr = - StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); - MapState mapState = underTest.state(NAMESPACE, addr); - - final String tag1 = "tag1"; - final String tag2 = "tag2"; - - SettableFuture future1 = SettableFuture.create(); - SettableFuture future2 = SettableFuture.create(); - - when(mockReader.valueFuture( - protoKeyFromUserKey(tag1, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) - .thenReturn(future1); - when(mockReader.valueFuture( - protoKeyFromUserKey(tag2, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) - .thenReturn(future2); - SettableFuture>> prefixFuture = SettableFuture.create(); - when(mockReader.valuePrefixFuture( - protoKeyFromUserKey(null, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) - .thenReturn(prefixFuture); - - waitAndSet(future1, 1, 50); - waitAndSet(future2, 2, 50); - waitAndSet( - prefixFuture, - ImmutableList.of( - new AbstractMap.SimpleEntry<>(protoKeyFromUserKey(tag1, StringUtf8Coder.of()), 1), - new AbstractMap.SimpleEntry<>(protoKeyFromUserKey(tag2, StringUtf8Coder.of()), 2)), - 50); - mapState.clear(); - assertNull(mapState.get(tag1).read()); - assertNull(mapState.get(tag2).read()); - assertThat(mapState.entries().read(), Matchers.emptyIterable()); - assertTrue(mapState.isEmpty().read()); - } - - @Test - public void testMapAddBeforeRead() throws Exception { - StateTag> addr = - StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); - MapState mapState = underTest.state(NAMESPACE, addr); - - final String tag1 = "tag1"; - final String tag2 = "tag2"; - final String tag3 = "tag3"; - SettableFuture>> prefixFuture = SettableFuture.create(); - when(mockReader.valuePrefixFuture( - protoKeyFromUserKey(null, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) - .thenReturn(prefixFuture); - - ReadableState>> result = mapState.entries(); - result = result.readLater(); - - mapState.put(tag1, 1); - waitAndSet( - prefixFuture, - ImmutableList.of( - new AbstractMap.SimpleEntry<>(protoKeyFromUserKey(tag2, StringUtf8Coder.of()), 2)), - 200); - Iterable> readData = result.read(); - assertThat( - readData, - Matchers.containsInAnyOrder( - new AbstractMap.SimpleEntry<>(tag1, 1), new AbstractMap.SimpleEntry<>(tag2, 2))); - - mapState.put(tag3, 3); - assertThat( - result.read(), - Matchers.containsInAnyOrder( - new AbstractMap.SimpleEntry<>(tag1, 1), - new AbstractMap.SimpleEntry<>(tag2, 2), - new AbstractMap.SimpleEntry<>(tag3, 3))); - } - - @Test - public void testMapPutIfAbsentSucceeds() throws Exception { - StateTag> addr = - StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); - MapState mapState = underTest.state(NAMESPACE, addr); - - final String tag1 = "tag1"; - SettableFuture future = SettableFuture.create(); - when(mockReader.valueFuture( - protoKeyFromUserKey(tag1, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) - .thenReturn(future); - waitAndSet(future, null, 50); - - assertNull(mapState.putIfAbsent(tag1, 42).read()); - assertEquals(42, (int) mapState.get(tag1).read()); - } - - @Test - public void testMapPutIfAbsentFails() throws Exception { - StateTag> addr = - StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); - MapState mapState = underTest.state(NAMESPACE, addr); - - final String tag1 = "tag1"; - mapState.put(tag1, 1); - assertEquals(1, (int) mapState.putIfAbsent(tag1, 42).read()); - assertEquals(1, (int) mapState.get(tag1).read()); - - final String tag2 = "tag2"; - SettableFuture future = SettableFuture.create(); - when(mockReader.valueFuture( - protoKeyFromUserKey(tag2, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) - .thenReturn(future); - waitAndSet(future, 2, 50); - assertEquals(2, (int) mapState.putIfAbsent(tag2, 42).read()); - assertEquals(2, (int) mapState.get(tag2).read()); - } - - @Test - public void testMapPutIfAbsentNoReadSucceeds() throws Exception { - StateTag> addr = - StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); - MapState mapState = underTest.state(NAMESPACE, addr); - - final String tag1 = "tag1"; - SettableFuture future = SettableFuture.create(); - when(mockReader.valueFuture( - protoKeyFromUserKey(tag1, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) - .thenReturn(future); - waitAndSet(future, null, 50); - ReadableState readableState = mapState.putIfAbsent(tag1, 42); - assertEquals(42, (int) mapState.get(tag1).read()); - assertNull(readableState.read()); - } - - @Test - public void testMapPutIfAbsentNoReadFails() throws Exception { - StateTag> addr = - StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); - MapState mapState = underTest.state(NAMESPACE, addr); - - final String tag1 = "tag1"; - mapState.put(tag1, 1); - ReadableState readableState = mapState.putIfAbsent(tag1, 42); - assertEquals(1, (int) mapState.get(tag1).read()); - assertEquals(1, (int) readableState.read()); - - final String tag2 = "tag2"; - SettableFuture future = SettableFuture.create(); - when(mockReader.valueFuture( - protoKeyFromUserKey(tag2, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) - .thenReturn(future); - waitAndSet(future, 2, 50); - readableState = mapState.putIfAbsent(tag2, 42); - assertEquals(2, (int) mapState.get(tag2).read()); - assertEquals(2, (int) readableState.read()); - } - - @Test - public void testMapMultiplePutIfAbsentNoRead() throws Exception { - StateTag> addr = - StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); - MapState mapState = underTest.state(NAMESPACE, addr); - - final String tag1 = "tag1"; - SettableFuture future = SettableFuture.create(); - when(mockReader.valueFuture( - protoKeyFromUserKey(tag1, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) - .thenReturn(future); - waitAndSet(future, null, 50); - ReadableState readableState = mapState.putIfAbsent(tag1, 42); - assertEquals(42, (int) mapState.get(tag1).read()); - ReadableState readableState2 = mapState.putIfAbsent(tag1, 43); - mapState.put(tag1, 1); - ReadableState readableState3 = mapState.putIfAbsent(tag1, 44); - assertEquals(1, (int) mapState.get(tag1).read()); - assertNull(readableState.read()); - assertEquals(42, (int) readableState2.read()); - assertEquals(1, (int) readableState3.read()); - } - - @Test - public void testMapNegativeCache() throws Exception { - StateTag> addr = - StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); - MapState mapState = underTest.state(NAMESPACE, addr); - - final String tag = "tag"; - SettableFuture future = SettableFuture.create(); - when(mockReader.valueFuture( - protoKeyFromUserKey(tag, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) - .thenReturn(future); - waitAndSet(future, null, 200); - assertNull(mapState.get(tag).read()); - future.set(42); - assertNull(mapState.get(tag).read()); - } - - private Map.Entry fromTagValue( - TagValue tagValue, Coder keyCoder, Coder valueCoder) { - try { - V value = - !tagValue.getValue().getData().isEmpty() - ? valueCoder.decode(tagValue.getValue().getData().newInput()) - : null; - return new AbstractMap.SimpleEntry<>(userKeyFromProtoKey(tagValue.getTag(), keyCoder), value); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Test - public void testMapAddPersist() throws Exception { - StateTag> addr = - StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); - MapState mapState = underTest.state(NAMESPACE, addr); - - final String tag1 = "tag1"; - final String tag2 = "tag2"; - mapState.put(tag1, 1); - mapState.put(tag2, 2); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(2, commitBuilder.getValueUpdatesCount()); - assertThat( - commitBuilder.getValueUpdatesList().stream() - .map(tv -> fromTagValue(tv, StringUtf8Coder.of(), VarIntCoder.of())) - .collect(Collectors.toList()), - Matchers.containsInAnyOrder(new SimpleEntry<>(tag1, 1), new SimpleEntry<>(tag2, 2))); - } - - @Test - public void testMapRemovePersist() throws Exception { - StateTag> addr = - StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); - MapState mapState = underTest.state(NAMESPACE, addr); - - final String tag1 = "tag1"; - final String tag2 = "tag2"; - mapState.remove(tag1); - mapState.remove(tag2); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(2, commitBuilder.getValueUpdatesCount()); - assertThat( - commitBuilder.getValueUpdatesList().stream() - .map(tv -> fromTagValue(tv, StringUtf8Coder.of(), VarIntCoder.of())) - .collect(Collectors.toList()), - Matchers.containsInAnyOrder(new SimpleEntry<>(tag1, null), new SimpleEntry<>(tag2, null))); - } - - @Test - public void testMapClearPersist() throws Exception { - StateTag> addr = - StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); - MapState mapState = underTest.state(NAMESPACE, addr); - - final String tag1 = "tag1"; - final String tag2 = "tag2"; - mapState.put(tag1, 1); - mapState.put(tag2, 2); - mapState.clear(); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(0, commitBuilder.getValueUpdatesCount()); - assertEquals(1, commitBuilder.getTagValuePrefixDeletesCount()); - System.err.println(commitBuilder); - assertEquals(STATE_FAMILY, commitBuilder.getTagValuePrefixDeletes(0).getStateFamily()); - assertEquals( - protoKeyFromUserKey(null, StringUtf8Coder.of()), - commitBuilder.getTagValuePrefixDeletes(0).getTagPrefix()); - } - - @Test - public void testMapComplexPersist() throws Exception { - StateTag> addr = - StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); - MapState mapState = underTest.state(NAMESPACE, addr); - - final String tag1 = "tag1"; - final String tag2 = "tag2"; - final String tag3 = "tag3"; - final String tag4 = "tag4"; - - mapState.put(tag1, 1); - mapState.clear(); - mapState.put(tag2, 2); - mapState.put(tag3, 3); - mapState.remove(tag2); - mapState.remove(tag4); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - assertEquals(1, commitBuilder.getTagValuePrefixDeletesCount()); - assertEquals(STATE_FAMILY, commitBuilder.getTagValuePrefixDeletes(0).getStateFamily()); - assertEquals( - protoKeyFromUserKey(null, StringUtf8Coder.of()), - commitBuilder.getTagValuePrefixDeletes(0).getTagPrefix()); - assertThat( - commitBuilder.getValueUpdatesList().stream() - .map(tv -> fromTagValue(tv, StringUtf8Coder.of(), VarIntCoder.of())) - .collect(Collectors.toList()), - Matchers.containsInAnyOrder( - new SimpleEntry<>(tag3, 3), - new SimpleEntry<>(tag2, null), - new SimpleEntry<>(tag4, null))); - - // Once persist has been called, calling persist again should be a noop. - commitBuilder = Windmill.WorkItemCommitRequest.newBuilder(); - assertEquals(0, commitBuilder.getTagValuePrefixDeletesCount()); - assertEquals(0, commitBuilder.getValueUpdatesCount()); - } - - public static final Range FULL_ORDERED_LIST_RANGE = - Range.closedOpen(WindmillOrderedList.MIN_TS_MICROS, WindmillOrderedList.MAX_TS_MICROS); - - @Test - public void testOrderedListAddBeforeRead() throws Exception { - StateTag> addr = - StateTags.orderedList("orderedList", StringUtf8Coder.of()); - OrderedListState orderedList = underTest.state(NAMESPACE, addr); - - SettableFuture>> future = SettableFuture.create(); - when(mockReader.orderedListFuture( - FULL_ORDERED_LIST_RANGE, - key(NAMESPACE, "orderedList"), - STATE_FAMILY, - StringUtf8Coder.of())) - .thenReturn(future); - - orderedList.readLater(); - - final TimestampedValue helloValue = - TimestampedValue.of("hello", Instant.ofEpochMilli(100)); - final TimestampedValue worldValue = - TimestampedValue.of("world", Instant.ofEpochMilli(75)); - final TimestampedValue goodbyeValue = - TimestampedValue.of("goodbye", Instant.ofEpochMilli(50)); - - orderedList.add(helloValue); - waitAndSet(future, Arrays.asList(worldValue), 200); - assertThat(orderedList.read(), Matchers.contains(worldValue, helloValue)); - - orderedList.add(goodbyeValue); - assertThat(orderedList.read(), Matchers.contains(goodbyeValue, worldValue, helloValue)); - } - - @Test - public void testOrderedListClearBeforeRead() throws Exception { - StateTag> addr = - StateTags.orderedList("orderedList", StringUtf8Coder.of()); - OrderedListState orderedListState = underTest.state(NAMESPACE, addr); - - final TimestampedValue helloElement = TimestampedValue.of("hello", Instant.EPOCH); - orderedListState.clear(); - orderedListState.add(helloElement); - assertThat(orderedListState.read(), Matchers.containsInAnyOrder(helloElement)); - - // Shouldn't need to read from windmill for this. - Mockito.verifyZeroInteractions(mockReader); - } - - @Test - public void testOrderedListIsEmptyFalse() throws Exception { - StateTag> addr = - StateTags.orderedList("orderedList", StringUtf8Coder.of()); - OrderedListState orderedList = underTest.state(NAMESPACE, addr); - - SettableFuture>> future = SettableFuture.create(); - when(mockReader.orderedListFuture( - FULL_ORDERED_LIST_RANGE, - key(NAMESPACE, "orderedList"), - STATE_FAMILY, - StringUtf8Coder.of())) - .thenReturn(future); - ReadableState result = orderedList.isEmpty().readLater(); - Mockito.verify(mockReader) - .orderedListFuture( - FULL_ORDERED_LIST_RANGE, - key(NAMESPACE, "orderedList"), - STATE_FAMILY, - StringUtf8Coder.of()); - - waitAndSet(future, Arrays.asList(TimestampedValue.of("world", Instant.EPOCH)), 200); - assertThat(result.read(), Matchers.is(false)); - } - - @Test - public void testOrderedListIsEmptyTrue() throws Exception { - StateTag> addr = - StateTags.orderedList("orderedList", StringUtf8Coder.of()); - OrderedListState orderedList = underTest.state(NAMESPACE, addr); - - SettableFuture>> future = SettableFuture.create(); - when(mockReader.orderedListFuture( - FULL_ORDERED_LIST_RANGE, - key(NAMESPACE, "orderedList"), - STATE_FAMILY, - StringUtf8Coder.of())) - .thenReturn(future); - ReadableState result = orderedList.isEmpty().readLater(); - Mockito.verify(mockReader) - .orderedListFuture( - FULL_ORDERED_LIST_RANGE, - key(NAMESPACE, "orderedList"), - STATE_FAMILY, - StringUtf8Coder.of()); - - waitAndSet(future, Collections.emptyList(), 200); - assertThat(result.read(), Matchers.is(true)); - } - - @Test - public void testOrderedListIsEmptyAfterClear() throws Exception { - StateTag> addr = - StateTags.orderedList("orderedList", StringUtf8Coder.of()); - OrderedListState orderedList = underTest.state(NAMESPACE, addr); - - orderedList.clear(); - ReadableState result = orderedList.isEmpty(); - Mockito.verify(mockReader, never()) - .orderedListFuture( - FULL_ORDERED_LIST_RANGE, - key(NAMESPACE, "orderedList"), - STATE_FAMILY, - StringUtf8Coder.of()); - assertThat(result.read(), Matchers.is(true)); - - orderedList.add(TimestampedValue.of("hello", Instant.EPOCH)); - assertThat(result.read(), Matchers.is(false)); - } - - @Test - public void testOrderedListAddPersist() throws Exception { - StateTag> addr = - StateTags.orderedList("orderedList", StringUtf8Coder.of()); - OrderedListState orderedList = underTest.state(NAMESPACE, addr); - - SettableFuture, RangeSet>> orderedListFuture = SettableFuture.create(); - orderedListFuture.set(null); - SettableFuture, RangeSet>> deletionsFuture = - SettableFuture.create(); - deletionsFuture.set(null); - when(mockReader.valueFuture( - systemKey(NAMESPACE, "orderedList" + IdTracker.IDS_AVAILABLE_STR), - STATE_FAMILY, - IdTracker.IDS_AVAILABLE_CODER)) - .thenReturn(orderedListFuture); - when(mockReader.valueFuture( - systemKey(NAMESPACE, "orderedList" + IdTracker.DELETIONS_STR), - STATE_FAMILY, - IdTracker.SUBRANGE_DELETIONS_CODER)) - .thenReturn(deletionsFuture); - - orderedList.add(TimestampedValue.of("hello", Instant.ofEpochMilli(1))); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(1, commitBuilder.getSortedListUpdatesCount()); - TagSortedListUpdateRequest updates = commitBuilder.getSortedListUpdates(0); - assertEquals(key(NAMESPACE, "orderedList"), updates.getTag()); - assertEquals(1, updates.getInsertsCount()); - assertEquals(1, updates.getInserts(0).getEntriesCount()); - - assertEquals("hello", updates.getInserts(0).getEntries(0).getValue().toStringUtf8()); - assertEquals(1000, updates.getInserts(0).getEntries(0).getSortKey()); - assertEquals(IdTracker.NEW_RANGE_MIN_ID, updates.getInserts(0).getEntries(0).getId()); - } - - @Test - public void testOrderedListClearPersist() throws Exception { - StateTag> addr = - StateTags.orderedList("orderedList", StringUtf8Coder.of()); - OrderedListState orderedListState = underTest.state(NAMESPACE, addr); - - orderedListState.add(TimestampedValue.of("hello", Instant.ofEpochMilli(1))); - orderedListState.clear(); - orderedListState.add(TimestampedValue.of("world", Instant.ofEpochMilli(2))); - orderedListState.add(TimestampedValue.of("world", Instant.ofEpochMilli(2))); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(1, commitBuilder.getSortedListUpdatesCount()); - TagSortedListUpdateRequest updates = commitBuilder.getSortedListUpdates(0); - assertEquals(STATE_FAMILY, updates.getStateFamily()); - assertEquals(key(NAMESPACE, "orderedList"), updates.getTag()); - assertEquals(1, updates.getInsertsCount()); - assertEquals(2, updates.getInserts(0).getEntriesCount()); - - assertEquals("world", updates.getInserts(0).getEntries(0).getValue().toStringUtf8()); - assertEquals("world", updates.getInserts(0).getEntries(1).getValue().toStringUtf8()); - assertEquals(2000, updates.getInserts(0).getEntries(0).getSortKey()); - assertEquals(2000, updates.getInserts(0).getEntries(1).getSortKey()); - assertEquals(IdTracker.NEW_RANGE_MIN_ID, updates.getInserts(0).getEntries(0).getId()); - assertEquals(IdTracker.NEW_RANGE_MIN_ID + 1, updates.getInserts(0).getEntries(1).getId()); - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testOrderedListDeleteRangePersist() { - SettableFuture, RangeSet>> orderedListFuture = SettableFuture.create(); - orderedListFuture.set(null); - SettableFuture, RangeSet>> deletionsFuture = - SettableFuture.create(); - deletionsFuture.set(null); - when(mockReader.valueFuture( - systemKey(NAMESPACE, "orderedList" + IdTracker.IDS_AVAILABLE_STR), - STATE_FAMILY, - IdTracker.IDS_AVAILABLE_CODER)) - .thenReturn(orderedListFuture); - when(mockReader.valueFuture( - systemKey(NAMESPACE, "orderedList" + IdTracker.DELETIONS_STR), - STATE_FAMILY, - IdTracker.SUBRANGE_DELETIONS_CODER)) - .thenReturn(deletionsFuture); - - StateTag> addr = - StateTags.orderedList("orderedList", StringUtf8Coder.of()); - OrderedListState orderedListState = underTest.state(NAMESPACE, addr); - - orderedListState.add(TimestampedValue.of("hello", Instant.ofEpochMilli(1))); - orderedListState.add(TimestampedValue.of("hello", Instant.ofEpochMilli(2))); - orderedListState.add(TimestampedValue.of("hello", Instant.ofEpochMilli(2))); - orderedListState.add(TimestampedValue.of("world", Instant.ofEpochMilli(3))); - orderedListState.add(TimestampedValue.of("world", Instant.ofEpochMilli(4))); - orderedListState.clearRange(Instant.ofEpochMilli(2), Instant.ofEpochMilli(4)); - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(1, commitBuilder.getSortedListUpdatesCount()); - TagSortedListUpdateRequest updates = commitBuilder.getSortedListUpdates(0); - assertEquals(STATE_FAMILY, updates.getStateFamily()); - assertEquals(key(NAMESPACE, "orderedList"), updates.getTag()); - assertEquals(1, updates.getInsertsCount()); - assertEquals(2, updates.getInserts(0).getEntriesCount()); - - assertEquals("hello", updates.getInserts(0).getEntries(0).getValue().toStringUtf8()); - assertEquals("world", updates.getInserts(0).getEntries(1).getValue().toStringUtf8()); - assertEquals(1000, updates.getInserts(0).getEntries(0).getSortKey()); - assertEquals(4000, updates.getInserts(0).getEntries(1).getSortKey()); - assertEquals(IdTracker.NEW_RANGE_MIN_ID, updates.getInserts(0).getEntries(0).getId()); - assertEquals(IdTracker.NEW_RANGE_MIN_ID + 1, updates.getInserts(0).getEntries(1).getId()); - } - - @Test - public void testOrderedListMergePendingAdds() { - SettableFuture, RangeSet>> orderedListFuture = SettableFuture.create(); - orderedListFuture.set(null); - SettableFuture, RangeSet>> deletionsFuture = - SettableFuture.create(); - deletionsFuture.set(null); - when(mockReader.valueFuture( - systemKey(NAMESPACE, "orderedList" + IdTracker.IDS_AVAILABLE_STR), - STATE_FAMILY, - IdTracker.IDS_AVAILABLE_CODER)) - .thenReturn(orderedListFuture); - when(mockReader.valueFuture( - systemKey(NAMESPACE, "orderedList" + IdTracker.DELETIONS_STR), - STATE_FAMILY, - IdTracker.SUBRANGE_DELETIONS_CODER)) - .thenReturn(deletionsFuture); - - SettableFuture>> fromStorage = SettableFuture.create(); - when(mockReader.orderedListFuture( - FULL_ORDERED_LIST_RANGE, - key(NAMESPACE, "orderedList"), - STATE_FAMILY, - StringUtf8Coder.of())) - .thenReturn(fromStorage); - - StateTag> addr = - StateTags.orderedList("orderedList", StringUtf8Coder.of()); - OrderedListState orderedListState = underTest.state(NAMESPACE, addr); - - orderedListState.add(TimestampedValue.of("second", Instant.ofEpochMilli(1))); - orderedListState.add(TimestampedValue.of("third", Instant.ofEpochMilli(2))); - orderedListState.add(TimestampedValue.of("fourth", Instant.ofEpochMilli(2))); - orderedListState.add(TimestampedValue.of("eighth", Instant.ofEpochMilli(10))); - orderedListState.add(TimestampedValue.of("ninth", Instant.ofEpochMilli(15))); - - fromStorage.set( - ImmutableList.of( - TimestampedValue.of("first", Instant.ofEpochMilli(-1)), - TimestampedValue.of("fifth", Instant.ofEpochMilli(5)), - TimestampedValue.of("sixth", Instant.ofEpochMilli(5)), - TimestampedValue.of("seventh", Instant.ofEpochMilli(5)), - TimestampedValue.of("tenth", Instant.ofEpochMilli(20)))); - - TimestampedValue[] expected = - Iterables.toArray( - ImmutableList.of( - TimestampedValue.of("first", Instant.ofEpochMilli(-1)), - TimestampedValue.of("second", Instant.ofEpochMilli(1)), - TimestampedValue.of("third", Instant.ofEpochMilli(2)), - TimestampedValue.of("fourth", Instant.ofEpochMilli(2)), - TimestampedValue.of("fifth", Instant.ofEpochMilli(5)), - TimestampedValue.of("sixth", Instant.ofEpochMilli(5)), - TimestampedValue.of("seventh", Instant.ofEpochMilli(5)), - TimestampedValue.of("eighth", Instant.ofEpochMilli(10)), - TimestampedValue.of("ninth", Instant.ofEpochMilli(15)), - TimestampedValue.of("tenth", Instant.ofEpochMilli(20))), - TimestampedValue.class); - - TimestampedValue[] read = Iterables.toArray(orderedListState.read(), TimestampedValue.class); - assertArrayEquals(expected, read); - } - - @Test - public void testOrderedListPersistEmpty() throws Exception { - StateTag> addr = - StateTags.orderedList("orderedList", StringUtf8Coder.of()); - OrderedListState orderedListState = underTest.state(NAMESPACE, addr); - - orderedListState.clear(); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - // 1 bag update = the clear - assertEquals(1, commitBuilder.getSortedListUpdatesCount()); - TagSortedListUpdateRequest updates = commitBuilder.getSortedListUpdates(0); - assertEquals(1, updates.getDeletesCount()); - assertEquals(WindmillOrderedList.MIN_TS_MICROS, updates.getDeletes(0).getRange().getStart()); - assertEquals(WindmillOrderedList.MAX_TS_MICROS, updates.getDeletes(0).getRange().getLimit()); - } - - @Test - public void testNewOrderedListNoFetch() throws Exception { - StateTag> addr = - StateTags.orderedList("orderedList", StringUtf8Coder.of()); - OrderedListState orderedList = underTestNewKey.state(NAMESPACE, addr); - - assertThat(orderedList.read(), Matchers.emptyIterable()); - - // Shouldn't need to read from windmill for this. - Mockito.verifyZeroInteractions(mockReader); - } - - // test ordered list cleared before read - // test fetch + add + read - // test ids - - @Test - public void testBagAddBeforeRead() throws Exception { - StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); - BagState bag = underTest.state(NAMESPACE, addr); - - SettableFuture> future = SettableFuture.create(); - when(mockReader.bagFuture(key(NAMESPACE, "bag"), STATE_FAMILY, StringUtf8Coder.of())) - .thenReturn(future); - - bag.readLater(); - - bag.add("hello"); - waitAndSet(future, Arrays.asList("world"), 200); - assertThat(bag.read(), Matchers.containsInAnyOrder("hello", "world")); - - bag.add("goodbye"); - assertThat(bag.read(), Matchers.containsInAnyOrder("hello", "world", "goodbye")); - } - - @Test - public void testBagClearBeforeRead() throws Exception { - StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); - BagState bag = underTest.state(NAMESPACE, addr); - - bag.clear(); - bag.add("hello"); - assertThat(bag.read(), Matchers.containsInAnyOrder("hello")); - - // Shouldn't need to read from windmill for this. - Mockito.verifyZeroInteractions(mockReader); - } - - @Test - public void testBagIsEmptyFalse() throws Exception { - StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); - BagState bag = underTest.state(NAMESPACE, addr); - - SettableFuture> future = SettableFuture.create(); - when(mockReader.bagFuture(key(NAMESPACE, "bag"), STATE_FAMILY, StringUtf8Coder.of())) - .thenReturn(future); - ReadableState result = bag.isEmpty().readLater(); - Mockito.verify(mockReader).bagFuture(key(NAMESPACE, "bag"), STATE_FAMILY, StringUtf8Coder.of()); - - waitAndSet(future, Arrays.asList("world"), 200); - assertThat(result.read(), Matchers.is(false)); - } - - @Test - public void testBagIsEmptyTrue() throws Exception { - StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); - BagState bag = underTest.state(NAMESPACE, addr); - - SettableFuture> future = SettableFuture.create(); - when(mockReader.bagFuture(key(NAMESPACE, "bag"), STATE_FAMILY, StringUtf8Coder.of())) - .thenReturn(future); - ReadableState result = bag.isEmpty().readLater(); - Mockito.verify(mockReader).bagFuture(key(NAMESPACE, "bag"), STATE_FAMILY, StringUtf8Coder.of()); - - waitAndSet(future, Arrays.asList(), 200); - assertThat(result.read(), Matchers.is(true)); - } - - @Test - public void testBagIsEmptyAfterClear() throws Exception { - StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); - BagState bag = underTest.state(NAMESPACE, addr); - - bag.clear(); - ReadableState result = bag.isEmpty(); - Mockito.verify(mockReader, never()) - .bagFuture(key(NAMESPACE, "bag"), STATE_FAMILY, StringUtf8Coder.of()); - assertThat(result.read(), Matchers.is(true)); - - bag.add("hello"); - assertThat(result.read(), Matchers.is(false)); - } - - @Test - public void testBagAddPersist() throws Exception { - StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); - BagState bag = underTest.state(NAMESPACE, addr); - - bag.add("hello"); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(1, commitBuilder.getBagUpdatesCount()); - - TagBag bagUpdates = commitBuilder.getBagUpdates(0); - assertEquals(key(NAMESPACE, "bag"), bagUpdates.getTag()); - assertEquals(1, bagUpdates.getValuesCount()); - assertEquals("hello", bagUpdates.getValues(0).toStringUtf8()); - - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testBagClearPersist() throws Exception { - StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); - BagState bag = underTest.state(NAMESPACE, addr); - - bag.add("hello"); - bag.clear(); - bag.add("world"); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(1, commitBuilder.getBagUpdatesCount()); - - TagBag tagBag = commitBuilder.getBagUpdates(0); - assertEquals(key(NAMESPACE, "bag"), tagBag.getTag()); - assertEquals(STATE_FAMILY, tagBag.getStateFamily()); - assertTrue(tagBag.getDeleteAll()); - assertEquals(1, tagBag.getValuesCount()); - assertEquals("world", tagBag.getValues(0).toStringUtf8()); - - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testBagPersistEmpty() throws Exception { - StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); - BagState bag = underTest.state(NAMESPACE, addr); - - bag.clear(); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - // 1 bag update = the clear - assertEquals(1, commitBuilder.getBagUpdatesCount()); - } - - @Test - public void testNewBagNoFetch() throws Exception { - StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); - BagState bag = underTestNewKey.state(NAMESPACE, addr); - - assertThat(bag.read(), Matchers.emptyIterable()); - - // Shouldn't need to read from windmill for this. - Mockito.verifyZeroInteractions(mockReader); - } - - @Test - @SuppressWarnings("ArraysAsListPrimitiveArray") - public void testCombiningAddBeforeRead() throws Exception { - GroupingState value = underTest.state(NAMESPACE, COMBINING_ADDR); - - SettableFuture> future = SettableFuture.create(); - when(mockReader.bagFuture(eq(COMBINING_KEY), eq(STATE_FAMILY), Mockito.>any())) - .thenReturn(future); - - value.readLater(); - - value.add(5); - value.add(6); - waitAndSet(future, Arrays.asList(new int[] {8}, new int[] {10}), 200); - assertThat(value.read(), Matchers.equalTo(29)); - - // That get "compressed" the combiner. So, the underlying future should change: - future.set(Arrays.asList(new int[] {29})); - - value.add(2); - assertThat(value.read(), Matchers.equalTo(31)); - } - - @Test - public void testCombiningClearBeforeRead() throws Exception { - GroupingState value = underTest.state(NAMESPACE, COMBINING_ADDR); - - value.clear(); - value.readLater(); - - value.add(5); - value.add(6); - assertThat(value.read(), Matchers.equalTo(11)); - - value.add(2); - assertThat(value.read(), Matchers.equalTo(13)); - - // Shouldn't need to read from windmill for this because we immediately cleared.. - Mockito.verifyZeroInteractions(mockReader); - } - - @Test - @SuppressWarnings("ArraysAsListPrimitiveArray") - public void testCombiningIsEmpty() throws Exception { - GroupingState value = underTest.state(NAMESPACE, COMBINING_ADDR); - - SettableFuture> future = SettableFuture.create(); - when(mockReader.bagFuture(eq(COMBINING_KEY), eq(STATE_FAMILY), Mockito.>any())) - .thenReturn(future); - ReadableState result = value.isEmpty().readLater(); - ArgumentCaptor byteString = ArgumentCaptor.forClass(ByteString.class); - - // Note that we do expect the third argument - the coder - to be equal to accumCoder, but that - // is possibly overspecified and currently trips an issue in the SDK where identical coders are - // not #equals(). - // - // What matters is that a future is created, hence a Windmill RPC sent. - Mockito.verify(mockReader) - .bagFuture(byteString.capture(), eq(STATE_FAMILY), Mockito.>any()); - assertThat(byteString.getValue(), byteStringEq(COMBINING_KEY)); - - waitAndSet(future, Arrays.asList(new int[] {29}), 200); - assertThat(result.read(), Matchers.is(false)); - } - - @Test - public void testCombiningIsEmptyAfterClear() throws Exception { - GroupingState value = underTest.state(NAMESPACE, COMBINING_ADDR); - - value.clear(); - ReadableState result = value.isEmpty(); - Mockito.verify(mockReader, never()).bagFuture(COMBINING_KEY, STATE_FAMILY, accumCoder); - assertThat(result.read(), Matchers.is(true)); - - value.add(87); - assertThat(result.read(), Matchers.is(false)); - } - - @Test - public void testCombiningAddPersist() throws Exception { - disableCompactOnWrite(); - - GroupingState value = underTest.state(NAMESPACE, COMBINING_ADDR); - - value.add(5); - value.add(6); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(1, commitBuilder.getBagUpdatesCount()); - - TagBag bagUpdates = commitBuilder.getBagUpdates(0); - assertEquals(COMBINING_KEY, bagUpdates.getTag()); - assertEquals(1, bagUpdates.getValuesCount()); - assertEquals( - 11, CoderUtils.decodeFromByteArray(accumCoder, bagUpdates.getValues(0).toByteArray())[0]); - - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testCombiningAddPersistWithCompact() throws Exception { - forceCompactOnWrite(); - - Mockito.when( - mockReader.bagFuture( - org.mockito.Matchers.any(), - org.mockito.Matchers.any(), - org.mockito.Matchers.>any())) - .thenReturn( - Futures.>immediateFuture( - ImmutableList.of(new int[] {40}, new int[] {60}))); - - GroupingState value = underTest.state(NAMESPACE, COMBINING_ADDR); - - value.add(5); - value.add(6); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(1, commitBuilder.getBagUpdatesCount()); - TagBag bagUpdates = commitBuilder.getBagUpdates(0); - assertEquals(COMBINING_KEY, bagUpdates.getTag()); - assertEquals(1, bagUpdates.getValuesCount()); - assertTrue(bagUpdates.getDeleteAll()); - assertEquals( - 111, CoderUtils.decodeFromByteArray(accumCoder, bagUpdates.getValues(0).toByteArray())[0]); - } - - @Test - public void testCombiningClearPersist() throws Exception { - disableCompactOnWrite(); - - GroupingState value = underTest.state(NAMESPACE, COMBINING_ADDR); - - value.clear(); - value.add(5); - value.add(6); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(1, commitBuilder.getBagUpdatesCount()); - - TagBag tagBag = commitBuilder.getBagUpdates(0); - assertEquals(COMBINING_KEY, tagBag.getTag()); - assertEquals(STATE_FAMILY, tagBag.getStateFamily()); - assertTrue(tagBag.getDeleteAll()); - assertEquals(1, tagBag.getValuesCount()); - assertEquals( - 11, CoderUtils.decodeFromByteArray(accumCoder, tagBag.getValues(0).toByteArray())[0]); - - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testNewCombiningNoFetch() throws Exception { - GroupingState value = underTestNewKey.state(NAMESPACE, COMBINING_ADDR); - - assertThat(value.isEmpty().read(), Matchers.is(true)); - assertThat(value.read(), Matchers.is(Sum.ofIntegers().identity())); - assertThat(value.isEmpty().read(), Matchers.is(false)); - - // Shouldn't need to read from windmill for this. - Mockito.verifyZeroInteractions(mockReader); - } - - @Test - public void testWatermarkAddBeforeReadEarliest() throws Exception { - StateTag addr = - StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST); - WatermarkHoldState bag = underTest.state(NAMESPACE, addr); - - SettableFuture future = SettableFuture.create(); - when(mockReader.watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY)).thenReturn(future); - - bag.readLater(); - - bag.add(new Instant(3000)); - waitAndSet(future, new Instant(2000), 200); - assertThat(bag.read(), Matchers.equalTo(new Instant(2000))); - - Mockito.verify(mockReader, times(2)).watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY); - Mockito.verifyNoMoreInteractions(mockReader); - - // Adding another value doesn't create another future, but does update the result. - bag.add(new Instant(1000)); - assertThat(bag.read(), Matchers.equalTo(new Instant(1000))); - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testWatermarkAddBeforeReadLatest() throws Exception { - StateTag addr = - StateTags.watermarkStateInternal("watermark", TimestampCombiner.LATEST); - WatermarkHoldState bag = underTest.state(NAMESPACE, addr); - - SettableFuture future = SettableFuture.create(); - when(mockReader.watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY)).thenReturn(future); - - // Suggesting we will read it later should get a future from the underlying WindmillStateReader - bag.readLater(); - - // Actually reading it will request another future, and get the same one, from - // WindmillStateReader - bag.add(new Instant(3000)); - waitAndSet(future, new Instant(2000), 200); - assertThat(bag.read(), Matchers.equalTo(new Instant(3000))); - - Mockito.verify(mockReader, times(2)).watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY); - Mockito.verifyNoMoreInteractions(mockReader); - - // Adding another value doesn't create another future, but does update the result. - bag.add(new Instant(3000)); - assertThat(bag.read(), Matchers.equalTo(new Instant(3000))); - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testWatermarkAddBeforeReadEndOfWindow() throws Exception { - StateTag addr = - StateTags.watermarkStateInternal("watermark", TimestampCombiner.END_OF_WINDOW); - WatermarkHoldState bag = underTest.state(NAMESPACE, addr); - - SettableFuture future = SettableFuture.create(); - when(mockReader.watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY)).thenReturn(future); - - // Requests a future once - bag.readLater(); - - bag.add(new Instant(3000)); - waitAndSet(future, new Instant(3000), 200); - // read() requests a future again, receiving the same one - assertThat(bag.read(), Matchers.equalTo(new Instant(3000))); - - Mockito.verify(mockReader, times(2)).watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY); - Mockito.verifyNoMoreInteractions(mockReader); - - // Adding another value doesn't create another future, but does update the result. - bag.add(new Instant(3000)); - assertThat(bag.read(), Matchers.equalTo(new Instant(3000))); - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testWatermarkClearBeforeRead() throws Exception { - StateTag addr = - StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST); - - WatermarkHoldState bag = underTest.state(NAMESPACE, addr); - - bag.clear(); - assertThat(bag.read(), Matchers.nullValue()); - - bag.add(new Instant(300)); - assertThat(bag.read(), Matchers.equalTo(new Instant(300))); - - // Shouldn't need to read from windmill because the value is already available. - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testWatermarkPersistEarliest() throws Exception { - StateTag addr = - StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST); - WatermarkHoldState bag = underTest.state(NAMESPACE, addr); - - bag.add(new Instant(1000)); - bag.add(new Instant(2000)); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(1, commitBuilder.getWatermarkHoldsCount()); - - Windmill.WatermarkHold watermarkHold = commitBuilder.getWatermarkHolds(0); - assertEquals(key(NAMESPACE, "watermark"), watermarkHold.getTag()); - assertEquals(TimeUnit.MILLISECONDS.toMicros(1000), watermarkHold.getTimestamps(0)); - - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testWatermarkPersistLatestEmpty() throws Exception { - StateTag addr = - StateTags.watermarkStateInternal("watermark", TimestampCombiner.LATEST); - WatermarkHoldState hold = underTest.state(NAMESPACE, addr); - - hold.add(new Instant(1000)); - hold.add(new Instant(2000)); - - when(mockReader.watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY)) - .thenReturn(Futures.immediateFuture(null)); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(1, commitBuilder.getWatermarkHoldsCount()); - - Windmill.WatermarkHold watermarkHold = commitBuilder.getWatermarkHolds(0); - assertEquals(key(NAMESPACE, "watermark"), watermarkHold.getTag()); - assertEquals(TimeUnit.MILLISECONDS.toMicros(2000), watermarkHold.getTimestamps(0)); - - Mockito.verify(mockReader).watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY); - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testWatermarkPersistLatestWindmillWins() throws Exception { - StateTag addr = - StateTags.watermarkStateInternal("watermark", TimestampCombiner.LATEST); - WatermarkHoldState hold = underTest.state(NAMESPACE, addr); - - hold.add(new Instant(1000)); - hold.add(new Instant(2000)); - - when(mockReader.watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY)) - .thenReturn(Futures.immediateFuture(new Instant(4000))); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(1, commitBuilder.getWatermarkHoldsCount()); - - Windmill.WatermarkHold watermarkHold = commitBuilder.getWatermarkHolds(0); - assertEquals(key(NAMESPACE, "watermark"), watermarkHold.getTag()); - assertEquals(TimeUnit.MILLISECONDS.toMicros(4000), watermarkHold.getTimestamps(0)); - - Mockito.verify(mockReader).watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY); - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testWatermarkPersistLatestLocalAdditionsWin() throws Exception { - StateTag addr = - StateTags.watermarkStateInternal("watermark", TimestampCombiner.LATEST); - WatermarkHoldState hold = underTest.state(NAMESPACE, addr); - - hold.add(new Instant(1000)); - hold.add(new Instant(2000)); - - when(mockReader.watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY)) - .thenReturn(Futures.immediateFuture(new Instant(500))); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(1, commitBuilder.getWatermarkHoldsCount()); - - Windmill.WatermarkHold watermarkHold = commitBuilder.getWatermarkHolds(0); - assertEquals(key(NAMESPACE, "watermark"), watermarkHold.getTag()); - assertEquals(TimeUnit.MILLISECONDS.toMicros(2000), watermarkHold.getTimestamps(0)); - - Mockito.verify(mockReader).watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY); - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testWatermarkPersistEndOfWindow() throws Exception { - StateTag addr = - StateTags.watermarkStateInternal("watermark", TimestampCombiner.END_OF_WINDOW); - WatermarkHoldState hold = underTest.state(NAMESPACE, addr); - - hold.add(new Instant(2000)); - hold.add(new Instant(2000)); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(1, commitBuilder.getWatermarkHoldsCount()); - - Windmill.WatermarkHold watermarkHold = commitBuilder.getWatermarkHolds(0); - assertEquals(key(NAMESPACE, "watermark"), watermarkHold.getTag()); - assertEquals(TimeUnit.MILLISECONDS.toMicros(2000), watermarkHold.getTimestamps(0)); - - // Blind adds should not need to read the future. - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testWatermarkClearPersist() throws Exception { - StateTag addr = - StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST); - WatermarkHoldState hold = underTest.state(NAMESPACE, addr); - - hold.add(new Instant(500)); - hold.clear(); - hold.add(new Instant(1000)); - hold.add(new Instant(2000)); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(1, commitBuilder.getWatermarkHoldsCount()); - - Windmill.WatermarkHold clearAndUpdate = commitBuilder.getWatermarkHolds(0); - assertEquals(key(NAMESPACE, "watermark"), clearAndUpdate.getTag()); - assertEquals(1, clearAndUpdate.getTimestampsCount()); - assertEquals(TimeUnit.MILLISECONDS.toMicros(1000), clearAndUpdate.getTimestamps(0)); - - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testWatermarkPersistEmpty() throws Exception { - StateTag addr = - StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST); - WatermarkHoldState bag = underTest.state(NAMESPACE, addr); - - bag.add(new Instant(500)); - bag.clear(); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - // 1 bag update corresponds to deletion. There shouldn't be a bag update adding items. - assertEquals(1, commitBuilder.getWatermarkHoldsCount()); - } - - @Test - public void testNewWatermarkNoFetch() throws Exception { - StateTag addr = - StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST); - - WatermarkHoldState bag = underTestNewKey.state(NAMESPACE, addr); - assertThat(bag.read(), Matchers.nullValue()); - - // Shouldn't need to read from windmill for this. - Mockito.verifyZeroInteractions(mockReader); - } - - @Test - public void testValueSetBeforeRead() throws Exception { - StateTag> addr = StateTags.value("value", StringUtf8Coder.of()); - ValueState value = underTest.state(NAMESPACE, addr); - - value.write("Hello"); - - assertEquals("Hello", value.read()); - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testValueClearBeforeRead() throws Exception { - StateTag> addr = StateTags.value("value", StringUtf8Coder.of()); - ValueState value = underTest.state(NAMESPACE, addr); - - value.clear(); - - assertEquals(null, value.read()); - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testValueRead() throws Exception { - StateTag> addr = StateTags.value("value", StringUtf8Coder.of()); - ValueState value = underTest.state(NAMESPACE, addr); - - SettableFuture future = SettableFuture.create(); - when(mockReader.valueFuture(key(NAMESPACE, "value"), STATE_FAMILY, StringUtf8Coder.of())) - .thenReturn(future); - waitAndSet(future, "World", 200); - - assertEquals("World", value.read()); - } - - @Test - public void testValueSetPersist() throws Exception { - StateTag> addr = StateTags.value("value", StringUtf8Coder.of()); - ValueState value = underTest.state(NAMESPACE, addr); - - value.write("Hi"); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(1, commitBuilder.getValueUpdatesCount()); - TagValue valueUpdate = commitBuilder.getValueUpdates(0); - assertEquals(key(NAMESPACE, "value"), valueUpdate.getTag()); - assertEquals("Hi", valueUpdate.getValue().getData().toStringUtf8()); - assertTrue(valueUpdate.isInitialized()); - - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testValueClearPersist() throws Exception { - StateTag> addr = StateTags.value("value", StringUtf8Coder.of()); - ValueState value = underTest.state(NAMESPACE, addr); - - value.write("Hi"); - value.clear(); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(1, commitBuilder.getValueUpdatesCount()); - TagValue valueUpdate = commitBuilder.getValueUpdates(0); - assertEquals(key(NAMESPACE, "value"), valueUpdate.getTag()); - assertEquals(0, valueUpdate.getValue().getData().size()); - - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testValueNoChangePersist() throws Exception { - StateTag> addr = StateTags.value("value", StringUtf8Coder.of()); - underTest.state(NAMESPACE, addr); - - Windmill.WorkItemCommitRequest.Builder commitBuilder = - Windmill.WorkItemCommitRequest.newBuilder(); - underTest.persist(commitBuilder); - - assertEquals(0, commitBuilder.getValueUpdatesCount()); - - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testNewValueNoFetch() throws Exception { - StateTag> addr = StateTags.value("value", StringUtf8Coder.of()); - ValueState value = underTestNewKey.state(NAMESPACE, addr); - - assertEquals(null, value.read()); - - // Shouldn't need to read from windmill for this. - Mockito.verifyZeroInteractions(mockReader); - } - - @Test - public void testCachedValue() throws Exception { - StateTag> addr = StateTags.value("value", StringUtf8Coder.of()); - ValueState value = underTest.state(NAMESPACE, addr); - - assertEquals(0, cache.getWeight()); - - value.write("Hi"); - underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); - - assertEquals(132, cache.getWeight()); - - resetUnderTest(); - value = underTest.state(NAMESPACE, addr); - assertEquals("Hi", value.read()); - value.clear(); - underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); - - assertEquals(130, cache.getWeight()); - - resetUnderTest(); - value = underTest.state(NAMESPACE, addr); - assertEquals(null, value.read()); - underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); - - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testCachedBag() throws Exception { - StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); - BagState bag = underTest.state(NAMESPACE, addr); - - assertEquals(0, cache.getWeight()); - - SettableFuture> future = SettableFuture.create(); - when(mockReader.bagFuture(key(NAMESPACE, "bag"), STATE_FAMILY, StringUtf8Coder.of())) - .thenReturn(future); - - bag.readLater(); - - assertEquals(0, cache.getWeight()); - - bag.add("hello"); - waitAndSet(future, weightedList("world"), 200); - Iterable readResult1 = bag.read(); - assertThat(readResult1, Matchers.containsInAnyOrder("hello", "world")); - - underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); - - assertEquals(140, cache.getWeight()); - - resetUnderTest(); - bag = underTest.state(NAMESPACE, addr); - bag.add("goodbye"); - - // Make sure that cached iterables have not changed after persist+add. - assertThat(readResult1, Matchers.containsInAnyOrder("hello", "world")); - - Iterable readResult2 = bag.read(); - assertThat(readResult2, Matchers.containsInAnyOrder("hello", "world", "goodbye")); - bag.clear(); - // Make sure that cached iterables have not changed after clear. - assertThat(readResult2, Matchers.containsInAnyOrder("hello", "world", "goodbye")); - bag.add("new"); - // Make sure that cached iterables have not changed after clear+add. - assertThat(readResult2, Matchers.containsInAnyOrder("hello", "world", "goodbye")); - - underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); - - assertEquals(133, cache.getWeight()); - - resetUnderTest(); - bag = underTest.state(NAMESPACE, addr); - bag.add("new2"); - assertThat(bag.read(), Matchers.containsInAnyOrder("new", "new2")); - bag.clear(); - bag.add("new3"); - - underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); - - assertEquals(134, cache.getWeight()); - - resetUnderTest(); - bag = underTest.state(NAMESPACE, addr); - assertThat(bag.read(), Matchers.containsInAnyOrder("new3")); - underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); - - Mockito.verify(mockReader, times(2)) - .bagFuture(key(NAMESPACE, "bag"), STATE_FAMILY, StringUtf8Coder.of()); - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - public void testCachedWatermarkHold() throws Exception { - StateTag addr = - StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST); - WatermarkHoldState hold = underTest.state(NAMESPACE, addr); - - SettableFuture future = SettableFuture.create(); - when(mockReader.watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY)).thenReturn(future); - - assertEquals(0, cache.getWeight()); - - hold.readLater(); - - hold.add(new Instant(3000)); - waitAndSet(future, new Instant(2000), 200); - assertThat(hold.read(), Matchers.equalTo(new Instant(2000))); - - underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); - - assertEquals(138, cache.getWeight()); - - resetUnderTest(); - hold = underTest.state(NAMESPACE, addr); - assertThat(hold.read(), Matchers.equalTo(new Instant(2000))); - hold.clear(); - - underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); - - assertEquals(138, cache.getWeight()); - - resetUnderTest(); - hold = underTest.state(NAMESPACE, addr); - assertEquals(null, hold.read()); - underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); - - Mockito.verify(mockReader, times(2)).watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY); - Mockito.verifyNoMoreInteractions(mockReader); - } - - @Test - @SuppressWarnings("ArraysAsListPrimitiveArray") - public void testCachedCombining() throws Exception { - GroupingState value = underTest.state(NAMESPACE, COMBINING_ADDR); - - SettableFuture> future = SettableFuture.create(); - when(mockReader.bagFuture( - eq(key(NAMESPACE, "combining")), eq(STATE_FAMILY), Mockito.>any())) - .thenReturn(future); - - assertEquals(0, cache.getWeight()); - - value.readLater(); - - value.add(1); - waitAndSet(future, Arrays.asList(new int[] {2}), 200); - assertThat(value.read(), Matchers.equalTo(3)); - - underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); - - assertEquals(131, cache.getWeight()); - - resetUnderTest(); - value = underTest.state(NAMESPACE, COMBINING_ADDR); - assertThat(value.read(), Matchers.equalTo(3)); - value.add(3); - assertThat(value.read(), Matchers.equalTo(6)); - value.clear(); - - underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); - - assertEquals(130, cache.getWeight()); - - resetUnderTest(); - value = underTest.state(NAMESPACE, COMBINING_ADDR); - assertThat(value.read(), Matchers.equalTo(0)); - underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); - - // Note that we do expect the third argument - the coder - to be equal to accumCoder, but that - // is possibly overspecified and currently trips an issue in the SDK where identical coders are - // not #equals(). - // - // What matters is the number of futures created, hence Windmill RPCs. - Mockito.verify(mockReader, times(2)) - .bagFuture(eq(key(NAMESPACE, "combining")), eq(STATE_FAMILY), Mockito.>any()); - Mockito.verifyNoMoreInteractions(mockReader); - } - - private void disableCompactOnWrite() { - WindmillStateInternals.COMPACT_NOW.set(() -> false); - } - - private void forceCompactOnWrite() { - WindmillStateInternals.COMPACT_NOW.set(() -> true); - } -} diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WindmillStateReaderTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WindmillStateReaderTest.java deleted file mode 100644 index e80131ff16f7b..0000000000000 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WindmillStateReaderTest.java +++ /dev/null @@ -1,1003 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.runners.dataflow.worker; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.util.AbstractMap; -import java.util.Map; -import java.util.concurrent.Future; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataRequest; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.SortedListEntry; -import org.apache.beam.runners.dataflow.worker.windmill.Windmill.SortedListRange; -import org.apache.beam.sdk.coders.Coder; -import org.apache.beam.sdk.coders.VarIntCoder; -import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.sdk.util.ByteStringOutputStream; -import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Range; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; -import org.hamcrest.Matchers; -import org.joda.time.Instant; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; - -/** Tests for {@link WindmillStateReader}. */ -@RunWith(JUnit4.class) -@SuppressWarnings({ - "FutureReturnValueIgnored", -}) -public class WindmillStateReaderTest { - private static final VarIntCoder INT_CODER = VarIntCoder.of(); - - private static final String COMPUTATION = "computation"; - private static final ByteString DATA_KEY = ByteString.copyFromUtf8("DATA_KEY"); - private static final long SHARDING_KEY = 17L; - private static final long WORK_TOKEN = 5043L; - private static final long CONT_POSITION = 1391631351L; - - private static final ByteString STATE_KEY_PREFIX = ByteString.copyFromUtf8("key"); - private static final ByteString STATE_KEY_1 = ByteString.copyFromUtf8("key1"); - private static final ByteString STATE_KEY_2 = ByteString.copyFromUtf8("key2"); - private static final String STATE_FAMILY = "family"; - - private static void assertNoReader(Object obj) throws Exception { - WindmillStateTestUtils.assertNoReference(obj, WindmillStateReader.class); - } - - @Mock private MetricTrackingWindmillServerStub mockWindmill; - - private WindmillStateReader underTest; - - @Before - public void setUp() { - MockitoAnnotations.initMocks(this); - - underTest = - new WindmillStateReader(mockWindmill, COMPUTATION, DATA_KEY, SHARDING_KEY, WORK_TOKEN); - } - - private Windmill.Value intValue(int value) throws IOException { - return Windmill.Value.newBuilder() - .setData(intData(value)) - .setTimestamp( - WindmillTimeUtils.harnessToWindmillTimestamp(BoundedWindow.TIMESTAMP_MAX_VALUE)) - .build(); - } - - private ByteString intData(int value) throws IOException { - ByteStringOutputStream output = new ByteStringOutputStream(); - INT_CODER.encode(value, output, Coder.Context.OUTER); - return output.toByteString(); - } - - @Test - public void testReadBag() throws Exception { - Future> future = underTest.bagFuture(STATE_KEY_1, STATE_FAMILY, INT_CODER); - Mockito.verifyNoMoreInteractions(mockWindmill); - - Windmill.KeyedGetDataRequest.Builder expectedRequest = - Windmill.KeyedGetDataRequest.newBuilder() - .setKey(DATA_KEY) - .setShardingKey(SHARDING_KEY) - .setWorkToken(WORK_TOKEN) - .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) - .addBagsToFetch( - Windmill.TagBag.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .setFetchMaxBytes(WindmillStateReader.INITIAL_MAX_BAG_BYTES)); - - Windmill.KeyedGetDataResponse.Builder response = - Windmill.KeyedGetDataResponse.newBuilder() - .setKey(DATA_KEY) - .addBags( - Windmill.TagBag.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addValues(intData(5)) - .addValues(intData(6))); - - Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) - .thenReturn(response.build()); - - Iterable results = future.get(); - Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); - for (Integer unused : results) { - // Iterate over the results to force loading all the pages. - } - Mockito.verifyNoMoreInteractions(mockWindmill); - - assertThat(results, Matchers.contains(5, 6)); - assertNoReader(future); - } - - @Test - public void testReadBagWithContinuations() throws Exception { - Future> future = underTest.bagFuture(STATE_KEY_1, STATE_FAMILY, INT_CODER); - Mockito.verifyNoMoreInteractions(mockWindmill); - - Windmill.KeyedGetDataRequest.Builder expectedRequest1 = - Windmill.KeyedGetDataRequest.newBuilder() - .setKey(DATA_KEY) - .setShardingKey(SHARDING_KEY) - .setWorkToken(WORK_TOKEN) - .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) - .addBagsToFetch( - Windmill.TagBag.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .setFetchMaxBytes(WindmillStateReader.INITIAL_MAX_BAG_BYTES)); - - Windmill.KeyedGetDataResponse.Builder response1 = - Windmill.KeyedGetDataResponse.newBuilder() - .setKey(DATA_KEY) - .addBags( - Windmill.TagBag.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .setContinuationPosition(CONT_POSITION) - .addValues(intData(5)) - .addValues(intData(6))); - - Windmill.KeyedGetDataRequest.Builder expectedRequest2 = - Windmill.KeyedGetDataRequest.newBuilder() - .setKey(DATA_KEY) - .setShardingKey(SHARDING_KEY) - .setWorkToken(WORK_TOKEN) - .setMaxBytes(WindmillStateReader.MAX_CONTINUATION_KEY_BYTES) - .addBagsToFetch( - Windmill.TagBag.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .setFetchMaxBytes(WindmillStateReader.CONTINUATION_MAX_BAG_BYTES) - .setRequestPosition(CONT_POSITION)); - - Windmill.KeyedGetDataResponse.Builder response2 = - Windmill.KeyedGetDataResponse.newBuilder() - .setKey(DATA_KEY) - .addBags( - Windmill.TagBag.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .setRequestPosition(CONT_POSITION) - .addValues(intData(7)) - .addValues(intData(8))); - - Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest1.build())) - .thenReturn(response1.build()); - Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest2.build())) - .thenReturn(response2.build()); - - Iterable results = future.get(); - Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest1.build()); - for (Integer unused : results) { - // Iterate over the results to force loading all the pages. - } - Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest2.build()); - Mockito.verifyNoMoreInteractions(mockWindmill); - - assertThat(results, Matchers.contains(5, 6, 7, 8)); - // NOTE: The future will still contain a reference to the underlying reader. - } - - @Test - public void testReadSortedList() throws Exception { - long beginning = SortedListRange.getDefaultInstance().getStart(); - long end = SortedListRange.getDefaultInstance().getLimit(); - Future>> future = - underTest.orderedListFuture( - Range.closedOpen(beginning, end), STATE_KEY_1, STATE_FAMILY, INT_CODER); - Mockito.verifyNoMoreInteractions(mockWindmill); - - // Fetch the entire list. - Windmill.KeyedGetDataRequest.Builder expectedRequest = - Windmill.KeyedGetDataRequest.newBuilder() - .setKey(DATA_KEY) - .setShardingKey(SHARDING_KEY) - .setWorkToken(WORK_TOKEN) - .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) - .addSortedListsToFetch( - Windmill.TagSortedListFetchRequest.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addFetchRanges(SortedListRange.newBuilder().setStart(beginning).setLimit(end)) - .setFetchMaxBytes(WindmillStateReader.MAX_ORDERED_LIST_BYTES)); - - Windmill.KeyedGetDataResponse.Builder response = - Windmill.KeyedGetDataResponse.newBuilder() - .setKey(DATA_KEY) - .addTagSortedLists( - Windmill.TagSortedListFetchResponse.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addEntries( - SortedListEntry.newBuilder().setValue(intData(5)).setSortKey(5000).setId(5)) - .addEntries( - SortedListEntry.newBuilder().setValue(intData(6)).setSortKey(6000).setId(5)) - .addEntries( - SortedListEntry.newBuilder().setValue(intData(7)).setSortKey(7000).setId(7)) - .addEntries( - SortedListEntry.newBuilder().setValue(intData(8)).setSortKey(8000).setId(8)) - .addFetchRanges( - SortedListRange.newBuilder().setStart(beginning).setLimit(end))); - - Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) - .thenReturn(response.build()); - - Iterable> results = future.get(); - Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); - for (TimestampedValue unused : results) { - // Iterate over the results to force loading all the pages. - } - Mockito.verifyNoMoreInteractions(mockWindmill); - - assertThat( - results, - Matchers.contains( - TimestampedValue.of(5, Instant.ofEpochMilli(5)), - TimestampedValue.of(6, Instant.ofEpochMilli(6)), - TimestampedValue.of(7, Instant.ofEpochMilli(7)), - TimestampedValue.of(8, Instant.ofEpochMilli(8)))); - assertNoReader(future); - } - - @Test - public void testReadSortedListRanges() throws Exception { - Future>> future1 = - underTest.orderedListFuture(Range.closedOpen(0L, 5L), STATE_KEY_1, STATE_FAMILY, INT_CODER); - Future>> future2 = - underTest.orderedListFuture(Range.closedOpen(5L, 6L), STATE_KEY_1, STATE_FAMILY, INT_CODER); - Future>> future3 = - underTest.orderedListFuture( - Range.closedOpen(6L, 10L), STATE_KEY_1, STATE_FAMILY, INT_CODER); - Mockito.verifyNoMoreInteractions(mockWindmill); - - // Fetch the entire list. - Windmill.KeyedGetDataRequest.Builder expectedRequest = - Windmill.KeyedGetDataRequest.newBuilder() - .setKey(DATA_KEY) - .setShardingKey(SHARDING_KEY) - .setWorkToken(WORK_TOKEN) - .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) - .addSortedListsToFetch( - Windmill.TagSortedListFetchRequest.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addFetchRanges(SortedListRange.newBuilder().setStart(0).setLimit(5)) - .setFetchMaxBytes(WindmillStateReader.MAX_ORDERED_LIST_BYTES)) - .addSortedListsToFetch( - Windmill.TagSortedListFetchRequest.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addFetchRanges(SortedListRange.newBuilder().setStart(5).setLimit(6)) - .setFetchMaxBytes(WindmillStateReader.MAX_ORDERED_LIST_BYTES)) - .addSortedListsToFetch( - Windmill.TagSortedListFetchRequest.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addFetchRanges(SortedListRange.newBuilder().setStart(6).setLimit(10)) - .setFetchMaxBytes(WindmillStateReader.MAX_ORDERED_LIST_BYTES)); - - Windmill.KeyedGetDataResponse.Builder response = - Windmill.KeyedGetDataResponse.newBuilder() - .setKey(DATA_KEY) - .addTagSortedLists( - Windmill.TagSortedListFetchResponse.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addEntries( - SortedListEntry.newBuilder().setValue(intData(5)).setSortKey(5000).setId(5)) - .addFetchRanges(SortedListRange.newBuilder().setStart(0).setLimit(5))) - .addTagSortedLists( - Windmill.TagSortedListFetchResponse.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addEntries( - SortedListEntry.newBuilder().setValue(intData(6)).setSortKey(6000).setId(5)) - .addEntries( - SortedListEntry.newBuilder().setValue(intData(7)).setSortKey(7000).setId(7)) - .addFetchRanges(SortedListRange.newBuilder().setStart(5).setLimit(6))) - .addTagSortedLists( - Windmill.TagSortedListFetchResponse.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addEntries( - SortedListEntry.newBuilder().setValue(intData(8)).setSortKey(8000).setId(8)) - .addFetchRanges(SortedListRange.newBuilder().setStart(6).setLimit(10))); - - Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) - .thenReturn(response.build()); - - { - Iterable> results = future1.get(); - Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); - for (TimestampedValue unused : results) { - // Iterate over the results to force loading all the pages. - } - Mockito.verifyNoMoreInteractions(mockWindmill); - assertThat(results, Matchers.contains(TimestampedValue.of(5, Instant.ofEpochMilli(5)))); - assertNoReader(future1); - } - - { - Iterable> results = future2.get(); - Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); - for (TimestampedValue unused : results) { - // Iterate over the results to force loading all the pages. - } - Mockito.verifyNoMoreInteractions(mockWindmill); - assertThat( - results, - Matchers.contains( - TimestampedValue.of(6, Instant.ofEpochMilli(6)), - TimestampedValue.of(7, Instant.ofEpochMilli(7)))); - assertNoReader(future2); - } - - { - Iterable> results = future3.get(); - Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); - for (TimestampedValue unused : results) { - // Iterate over the results to force loading all the pages. - } - Mockito.verifyNoMoreInteractions(mockWindmill); - assertThat(results, Matchers.contains(TimestampedValue.of(8, Instant.ofEpochMilli(8)))); - assertNoReader(future3); - } - } - - @Test - public void testReadSortedListWithContinuations() throws Exception { - long beginning = SortedListRange.getDefaultInstance().getStart(); - long end = SortedListRange.getDefaultInstance().getLimit(); - - Future>> future = - underTest.orderedListFuture( - Range.closedOpen(beginning, end), STATE_KEY_1, STATE_FAMILY, INT_CODER); - - Mockito.verifyNoMoreInteractions(mockWindmill); - - Windmill.KeyedGetDataRequest.Builder expectedRequest1 = - Windmill.KeyedGetDataRequest.newBuilder() - .setKey(DATA_KEY) - .setShardingKey(SHARDING_KEY) - .setWorkToken(WORK_TOKEN) - .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) - .addSortedListsToFetch( - Windmill.TagSortedListFetchRequest.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addFetchRanges(SortedListRange.newBuilder().setStart(beginning).setLimit(end)) - .setFetchMaxBytes(WindmillStateReader.MAX_ORDERED_LIST_BYTES)); - - final ByteString CONT_1 = ByteString.copyFrom("CONTINUATION_1", Charsets.UTF_8); - final ByteString CONT_2 = ByteString.copyFrom("CONTINUATION_2", Charsets.UTF_8); - Windmill.KeyedGetDataResponse.Builder response1 = - Windmill.KeyedGetDataResponse.newBuilder() - .setKey(DATA_KEY) - .addTagSortedLists( - Windmill.TagSortedListFetchResponse.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addEntries( - SortedListEntry.newBuilder().setValue(intData(1)).setSortKey(1000).setId(1)) - .setContinuationPosition(CONT_1) - .addFetchRanges( - SortedListRange.newBuilder().setStart(beginning).setLimit(end))); - - Windmill.KeyedGetDataRequest.Builder expectedRequest2 = - Windmill.KeyedGetDataRequest.newBuilder() - .setKey(DATA_KEY) - .setShardingKey(SHARDING_KEY) - .setWorkToken(WORK_TOKEN) - .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) - .addSortedListsToFetch( - Windmill.TagSortedListFetchRequest.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addFetchRanges(SortedListRange.newBuilder().setStart(beginning).setLimit(end)) - .setRequestPosition(CONT_1) - .setFetchMaxBytes(WindmillStateReader.MAX_ORDERED_LIST_BYTES)); - - Windmill.KeyedGetDataResponse.Builder response2 = - Windmill.KeyedGetDataResponse.newBuilder() - .setKey(DATA_KEY) - .addTagSortedLists( - Windmill.TagSortedListFetchResponse.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addEntries( - SortedListEntry.newBuilder().setValue(intData(2)).setSortKey(2000).setId(2)) - .addEntries( - SortedListEntry.newBuilder().setValue(intData(3)).setSortKey(3000).setId(3)) - .addEntries( - SortedListEntry.newBuilder().setValue(intData(4)).setSortKey(4000).setId(4)) - .setContinuationPosition(CONT_2) - .addFetchRanges(SortedListRange.newBuilder().setStart(beginning).setLimit(end)) - .setRequestPosition(CONT_1)); - - Windmill.KeyedGetDataRequest.Builder expectedRequest3 = - Windmill.KeyedGetDataRequest.newBuilder() - .setKey(DATA_KEY) - .setShardingKey(SHARDING_KEY) - .setWorkToken(WORK_TOKEN) - .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) - .addSortedListsToFetch( - Windmill.TagSortedListFetchRequest.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addFetchRanges(SortedListRange.newBuilder().setStart(beginning).setLimit(end)) - .setRequestPosition(CONT_2) - .setFetchMaxBytes(WindmillStateReader.MAX_ORDERED_LIST_BYTES)); - - Windmill.KeyedGetDataResponse.Builder response3 = - Windmill.KeyedGetDataResponse.newBuilder() - .setKey(DATA_KEY) - .addTagSortedLists( - Windmill.TagSortedListFetchResponse.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addEntries( - SortedListEntry.newBuilder().setValue(intData(5)).setSortKey(5000).setId(5)) - .addEntries( - SortedListEntry.newBuilder().setValue(intData(6)).setSortKey(6000).setId(7)) - .addEntries( - SortedListEntry.newBuilder().setValue(intData(7)).setSortKey(7000).setId(7)) - .addFetchRanges(SortedListRange.newBuilder().setStart(beginning).setLimit(end)) - .setRequestPosition(CONT_2)); - - Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest1.build())) - .thenReturn(response1.build()); - Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest2.build())) - .thenReturn(response2.build()); - Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest3.build())) - .thenReturn(response3.build()); - - Iterable> results = future.get(); - Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest1.build()); - for (TimestampedValue unused : results) { - // Iterate over the results to force loading all the pages. - } - Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest2.build()); - Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest3.build()); - Mockito.verifyNoMoreInteractions(mockWindmill); - - assertThat( - results, - Matchers.contains( - TimestampedValue.of(1, Instant.ofEpochMilli(1)), - TimestampedValue.of(2, Instant.ofEpochMilli(2)), - TimestampedValue.of(3, Instant.ofEpochMilli(3)), - TimestampedValue.of(4, Instant.ofEpochMilli(4)), - TimestampedValue.of(5, Instant.ofEpochMilli(5)), - TimestampedValue.of(6, Instant.ofEpochMilli(6)), - TimestampedValue.of(7, Instant.ofEpochMilli(7)))); - // NOTE: The future will still contain a reference to the underlying reader. - } - - @Test - public void testReadTagValuePrefix() throws Exception { - Future>> future = - underTest.valuePrefixFuture(STATE_KEY_PREFIX, STATE_FAMILY, INT_CODER); - Mockito.verifyNoMoreInteractions(mockWindmill); - - Windmill.KeyedGetDataRequest.Builder expectedRequest = - Windmill.KeyedGetDataRequest.newBuilder() - .setKey(DATA_KEY) - .setShardingKey(SHARDING_KEY) - .setWorkToken(WORK_TOKEN) - .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) - .addTagValuePrefixesToFetch( - Windmill.TagValuePrefixRequest.newBuilder() - .setTagPrefix(STATE_KEY_PREFIX) - .setStateFamily(STATE_FAMILY) - .setFetchMaxBytes(WindmillStateReader.MAX_TAG_VALUE_PREFIX_BYTES)); - - Windmill.KeyedGetDataResponse.Builder response = - Windmill.KeyedGetDataResponse.newBuilder() - .setKey(DATA_KEY) - .addTagValuePrefixes( - Windmill.TagValuePrefixResponse.newBuilder() - .setTagPrefix(STATE_KEY_PREFIX) - .setStateFamily(STATE_FAMILY) - .addTagValues( - Windmill.TagValue.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .setValue(intValue(8))) - .addTagValues( - Windmill.TagValue.newBuilder() - .setTag(STATE_KEY_2) - .setStateFamily(STATE_FAMILY) - .setValue(intValue(9)))); - - Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) - .thenReturn(response.build()); - - Iterable> result = future.get(); - Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); - Mockito.verifyNoMoreInteractions(mockWindmill); - - assertThat( - result, - Matchers.containsInAnyOrder( - new AbstractMap.SimpleEntry<>(STATE_KEY_1, 8), - new AbstractMap.SimpleEntry<>(STATE_KEY_2, 9))); - - assertNoReader(future); - } - - @Test - public void testReadTagValuePrefixWithContinuations() throws Exception { - Future>> future = - underTest.valuePrefixFuture(STATE_KEY_PREFIX, STATE_FAMILY, INT_CODER); - Mockito.verifyNoMoreInteractions(mockWindmill); - - Windmill.KeyedGetDataRequest.Builder expectedRequest1 = - Windmill.KeyedGetDataRequest.newBuilder() - .setKey(DATA_KEY) - .setShardingKey(SHARDING_KEY) - .setWorkToken(WORK_TOKEN) - .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) - .addTagValuePrefixesToFetch( - Windmill.TagValuePrefixRequest.newBuilder() - .setTagPrefix(STATE_KEY_PREFIX) - .setStateFamily(STATE_FAMILY) - .setFetchMaxBytes(WindmillStateReader.MAX_TAG_VALUE_PREFIX_BYTES)); - - final ByteString CONT = ByteString.copyFrom("CONTINUATION", Charsets.UTF_8); - Windmill.KeyedGetDataResponse.Builder response1 = - Windmill.KeyedGetDataResponse.newBuilder() - .setKey(DATA_KEY) - .addTagValuePrefixes( - Windmill.TagValuePrefixResponse.newBuilder() - .setTagPrefix(STATE_KEY_PREFIX) - .setStateFamily(STATE_FAMILY) - .setContinuationPosition(CONT) - .addTagValues( - Windmill.TagValue.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .setValue(intValue(8)))); - - Windmill.KeyedGetDataRequest.Builder expectedRequest2 = - Windmill.KeyedGetDataRequest.newBuilder() - .setKey(DATA_KEY) - .setShardingKey(SHARDING_KEY) - .setWorkToken(WORK_TOKEN) - .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) - .addTagValuePrefixesToFetch( - Windmill.TagValuePrefixRequest.newBuilder() - .setTagPrefix(STATE_KEY_PREFIX) - .setStateFamily(STATE_FAMILY) - .setRequestPosition(CONT) - .setFetchMaxBytes(WindmillStateReader.MAX_TAG_VALUE_PREFIX_BYTES)); - - Windmill.KeyedGetDataResponse.Builder response2 = - Windmill.KeyedGetDataResponse.newBuilder() - .setKey(DATA_KEY) - .addTagValuePrefixes( - Windmill.TagValuePrefixResponse.newBuilder() - .setTagPrefix(STATE_KEY_PREFIX) - .setStateFamily(STATE_FAMILY) - .setRequestPosition(CONT) - .addTagValues( - Windmill.TagValue.newBuilder() - .setTag(STATE_KEY_2) - .setStateFamily(STATE_FAMILY) - .setValue(intValue(9)))); - - Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest1.build())) - .thenReturn(response1.build()); - Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest2.build())) - .thenReturn(response2.build()); - - Iterable> results = future.get(); - Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest1.build()); - for (Map.Entry unused : results) { - // Iterate over the results to force loading all the pages. - } - Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest2.build()); - Mockito.verifyNoMoreInteractions(mockWindmill); - - assertThat( - results, - Matchers.containsInAnyOrder( - new AbstractMap.SimpleEntry<>(STATE_KEY_1, 8), - new AbstractMap.SimpleEntry<>(STATE_KEY_2, 9))); - // NOTE: The future will still contain a reference to the underlying reader. - } - - @Test - public void testReadValue() throws Exception { - Future future = underTest.valueFuture(STATE_KEY_1, STATE_FAMILY, INT_CODER); - Mockito.verifyNoMoreInteractions(mockWindmill); - - Windmill.KeyedGetDataRequest.Builder expectedRequest = - Windmill.KeyedGetDataRequest.newBuilder() - .setKey(DATA_KEY) - .setShardingKey(SHARDING_KEY) - .setWorkToken(WORK_TOKEN) - .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) - .addValuesToFetch( - Windmill.TagValue.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .build()); - Windmill.KeyedGetDataResponse.Builder response = - Windmill.KeyedGetDataResponse.newBuilder() - .setKey(DATA_KEY) - .addValues( - Windmill.TagValue.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .setValue(intValue(8))); - - Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) - .thenReturn(response.build()); - - Integer result = future.get(); - Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); - Mockito.verifyNoMoreInteractions(mockWindmill); - - assertThat(result, Matchers.equalTo(8)); - assertNoReader(future); - } - - @Test - public void testReadWatermark() throws Exception { - Future future = underTest.watermarkFuture(STATE_KEY_1, STATE_FAMILY); - Mockito.verifyNoMoreInteractions(mockWindmill); - - Windmill.KeyedGetDataRequest.Builder expectedRequest = - Windmill.KeyedGetDataRequest.newBuilder() - .setKey(DATA_KEY) - .setShardingKey(SHARDING_KEY) - .setWorkToken(WORK_TOKEN) - .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) - .addWatermarkHoldsToFetch( - Windmill.WatermarkHold.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY)); - - Windmill.KeyedGetDataResponse.Builder response = - Windmill.KeyedGetDataResponse.newBuilder() - .setKey(DATA_KEY) - .addWatermarkHolds( - Windmill.WatermarkHold.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addTimestamps(5000000) - .addTimestamps(6000000)); - - Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) - .thenReturn(response.build()); - - Instant result = future.get(); - Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); - - assertThat(result, Matchers.equalTo(new Instant(5000))); - assertNoReader(future); - } - - @Test - public void testBatching() throws Exception { - // Reads two bags and verifies that we batch them up correctly. - Future watermarkFuture = underTest.watermarkFuture(STATE_KEY_2, STATE_FAMILY); - Future> bagFuture = underTest.bagFuture(STATE_KEY_1, STATE_FAMILY, INT_CODER); - - Mockito.verifyNoMoreInteractions(mockWindmill); - - ArgumentCaptor request = - ArgumentCaptor.forClass(Windmill.KeyedGetDataRequest.class); - - Windmill.KeyedGetDataResponse.Builder response = - Windmill.KeyedGetDataResponse.newBuilder() - .setKey(DATA_KEY) - .addWatermarkHolds( - Windmill.WatermarkHold.newBuilder() - .setTag(STATE_KEY_2) - .setStateFamily(STATE_FAMILY) - .addTimestamps(5000000) - .addTimestamps(6000000)) - .addBags( - Windmill.TagBag.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addValues(intData(5)) - .addValues(intData(100))); - - Mockito.when( - mockWindmill.getStateData( - Mockito.eq(COMPUTATION), Mockito.isA(Windmill.KeyedGetDataRequest.class))) - .thenReturn(response.build()); - Instant result = watermarkFuture.get(); - Mockito.verify(mockWindmill).getStateData(Mockito.eq(COMPUTATION), request.capture()); - - // Verify the request looks right. - KeyedGetDataRequest keyedRequest = request.getValue(); - assertThat(keyedRequest.getKey(), Matchers.equalTo(DATA_KEY)); - assertThat(keyedRequest.getWorkToken(), Matchers.equalTo(WORK_TOKEN)); - assertThat(keyedRequest.getBagsToFetchCount(), Matchers.equalTo(1)); - assertThat(keyedRequest.getBagsToFetch(0).getDeleteAll(), Matchers.equalTo(false)); - assertThat(keyedRequest.getBagsToFetch(0).getTag(), Matchers.equalTo(STATE_KEY_1)); - assertThat(keyedRequest.getWatermarkHoldsToFetchCount(), Matchers.equalTo(1)); - assertThat(keyedRequest.getWatermarkHoldsToFetch(0).getTag(), Matchers.equalTo(STATE_KEY_2)); - - // Verify the values returned to the user. - assertThat(result, Matchers.equalTo(new Instant(5000))); - Mockito.verifyNoMoreInteractions(mockWindmill); - - assertThat(bagFuture.get(), Matchers.contains(5, 100)); - Mockito.verifyNoMoreInteractions(mockWindmill); - - // And verify that getting a future again returns the already completed future. - Future watermarkFuture2 = underTest.watermarkFuture(STATE_KEY_2, STATE_FAMILY); - assertTrue(watermarkFuture2.isDone()); - assertNoReader(watermarkFuture); - assertNoReader(watermarkFuture2); - } - - @Test - public void testNoStateFamily() throws Exception { - Future future = underTest.valueFuture(STATE_KEY_1, "", INT_CODER); - Mockito.verifyNoMoreInteractions(mockWindmill); - - Windmill.KeyedGetDataRequest.Builder expectedRequest = - Windmill.KeyedGetDataRequest.newBuilder() - .setKey(DATA_KEY) - .setShardingKey(SHARDING_KEY) - .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) - .setWorkToken(WORK_TOKEN) - .addValuesToFetch( - Windmill.TagValue.newBuilder().setTag(STATE_KEY_1).setStateFamily("").build()); - Windmill.KeyedGetDataResponse.Builder response = - Windmill.KeyedGetDataResponse.newBuilder() - .setKey(DATA_KEY) - .addValues( - Windmill.TagValue.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily("") - .setValue(intValue(8))); - - Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) - .thenReturn(response.build()); - - Integer result = future.get(); - Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); - Mockito.verifyNoMoreInteractions(mockWindmill); - - assertThat(result, Matchers.equalTo(8)); - assertNoReader(future); - } - - @Test - public void testKeyTokenInvalid() throws Exception { - // Reads two states and verifies that we batch them up correctly. - Future watermarkFuture = underTest.watermarkFuture(STATE_KEY_2, STATE_FAMILY); - Future> bagFuture = underTest.bagFuture(STATE_KEY_1, STATE_FAMILY, INT_CODER); - - Mockito.verifyNoMoreInteractions(mockWindmill); - - Windmill.KeyedGetDataResponse.Builder response = - Windmill.KeyedGetDataResponse.newBuilder().setKey(DATA_KEY).setFailed(true); - - Mockito.when( - mockWindmill.getStateData( - Mockito.eq(COMPUTATION), Mockito.isA(Windmill.KeyedGetDataRequest.class))) - .thenReturn(response.build()); - - try { - watermarkFuture.get(); - fail("Expected KeyTokenInvalidException"); - } catch (Exception e) { - assertTrue(KeyTokenInvalidException.isKeyTokenInvalidException(e)); - } - - try { - bagFuture.get(); - fail("Expected KeyTokenInvalidException"); - } catch (Exception e) { - assertTrue(KeyTokenInvalidException.isKeyTokenInvalidException(e)); - } - } - - @Test - public void testBatchingReadException() throws Exception { - // Reads two states and verifies that we batch them up correctly and propagate the read - // exception to both, not just the issuing future. - Future watermarkFuture = underTest.watermarkFuture(STATE_KEY_2, STATE_FAMILY); - Future> bagFuture = underTest.bagFuture(STATE_KEY_1, STATE_FAMILY, INT_CODER); - - Mockito.verifyNoMoreInteractions(mockWindmill); - - RuntimeException expectedException = new RuntimeException("expected exception"); - - Mockito.when( - mockWindmill.getStateData( - Mockito.eq(COMPUTATION), Mockito.isA(Windmill.KeyedGetDataRequest.class))) - .thenThrow(expectedException); - - try { - watermarkFuture.get(); - fail("Expected RuntimeException"); - } catch (Exception e) { - assertThat(e.toString(), Matchers.containsString("expected exception")); - } - - try { - bagFuture.get(); - fail("Expected RuntimeException"); - } catch (Exception e) { - assertThat(e.toString(), Matchers.containsString("expected exception")); - } - } - - @Test - public void testBatchingCoderExceptions() throws Exception { - // Read a batch of states with coder errors and verify it only affects the - // relevant futures. - Future valueFuture = underTest.valueFuture(STATE_KEY_1, STATE_FAMILY, INT_CODER); - Future> bagFuture = underTest.bagFuture(STATE_KEY_1, STATE_FAMILY, INT_CODER); - Future>> valuePrefixFuture = - underTest.valuePrefixFuture(STATE_KEY_PREFIX, STATE_FAMILY, INT_CODER); - long beginning = SortedListRange.getDefaultInstance().getStart(); - long end = SortedListRange.getDefaultInstance().getLimit(); - Future>> orderedListFuture = - underTest.orderedListFuture( - Range.closedOpen(beginning, end), STATE_KEY_1, STATE_FAMILY, INT_CODER); - // This should be the final part of the response, and we should process it successfully. - Future watermarkFuture = underTest.watermarkFuture(STATE_KEY_1, STATE_FAMILY); - - Mockito.verifyNoMoreInteractions(mockWindmill); - - ByteString invalidByteString = ByteString.copyFrom(BaseEncoding.base16().decode("FFFF")); - Windmill.Value invalidValue = intValue(0).toBuilder().setData(invalidByteString).build(); - - Windmill.KeyedGetDataRequest.Builder expectedRequest = - Windmill.KeyedGetDataRequest.newBuilder() - .setKey(DATA_KEY) - .setShardingKey(SHARDING_KEY) - .setWorkToken(WORK_TOKEN) - .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) - .addValuesToFetch( - Windmill.TagValue.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .build()) - .addBagsToFetch( - Windmill.TagBag.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .setFetchMaxBytes(WindmillStateReader.INITIAL_MAX_BAG_BYTES)) - .addTagValuePrefixesToFetch( - Windmill.TagValuePrefixRequest.newBuilder() - .setTagPrefix(STATE_KEY_PREFIX) - .setStateFamily(STATE_FAMILY) - .setFetchMaxBytes(WindmillStateReader.MAX_TAG_VALUE_PREFIX_BYTES)) - .addSortedListsToFetch( - Windmill.TagSortedListFetchRequest.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addFetchRanges(SortedListRange.newBuilder().setStart(beginning).setLimit(end)) - .setFetchMaxBytes(WindmillStateReader.MAX_ORDERED_LIST_BYTES)) - .addWatermarkHoldsToFetch( - Windmill.WatermarkHold.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY)); - - Windmill.KeyedGetDataResponse.Builder response = - Windmill.KeyedGetDataResponse.newBuilder() - .setKey(DATA_KEY) - .addValues( - Windmill.TagValue.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .setValue(invalidValue)) - .addBags( - Windmill.TagBag.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addValues(invalidByteString)) - .addTagValuePrefixes( - Windmill.TagValuePrefixResponse.newBuilder() - .setTagPrefix(STATE_KEY_PREFIX) - .setStateFamily(STATE_FAMILY) - .addTagValues( - Windmill.TagValue.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .setValue(invalidValue))) - .addTagSortedLists( - Windmill.TagSortedListFetchResponse.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addEntries( - SortedListEntry.newBuilder() - .setValue(invalidByteString) - .setSortKey(5000) - .setId(5)) - .addFetchRanges(SortedListRange.newBuilder().setStart(beginning).setLimit(end))) - .addWatermarkHolds( - Windmill.WatermarkHold.newBuilder() - .setTag(STATE_KEY_1) - .setStateFamily(STATE_FAMILY) - .addTimestamps(5000000) - .addTimestamps(6000000)); - - Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) - .thenReturn(response.build()); - Mockito.verifyNoMoreInteractions(mockWindmill); - - try { - valueFuture.get(); - fail("Expected RuntimeException"); - } catch (Exception e) { - assertThat(e.toString(), Matchers.containsString("Unable to decode value")); - } - - try { - bagFuture.get(); - fail("Expected RuntimeException"); - } catch (Exception e) { - assertThat(e.toString(), Matchers.containsString("Error parsing bag")); - } - - try { - orderedListFuture.get(); - fail("Expected RuntimeException"); - } catch (Exception e) { - assertThat(e.toString(), Matchers.containsString("Error parsing ordered list")); - } - - try { - valuePrefixFuture.get(); - fail("Expected RuntimeException"); - } catch (Exception e) { - assertThat(e.toString(), Matchers.containsString("Error parsing tag value prefix")); - } - - assertThat(watermarkFuture.get(), Matchers.equalTo(new Instant(5000))); - } - - /** - * Tests that multiple reads for the same tag in the same batch are cached. We can't compare the - * futures since we've wrapped the delegate aronud them, so we just verify there is only one - * queued lookup. - */ - @Test - public void testCachingWithinBatch() throws Exception { - underTest.watermarkFuture(STATE_KEY_1, STATE_FAMILY); - underTest.watermarkFuture(STATE_KEY_1, STATE_FAMILY); - assertEquals(1, underTest.pendingLookups.size()); - } -} diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WindmillTimerInternalsTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WindmillTimerInternalsTest.java index 8632034a9b20e..ec8672b6a75fc 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WindmillTimerInternalsTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WindmillTimerInternalsTest.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.transforms.windowing.IntervalWindow; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkItemStatusClientTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkItemStatusClientTest.java index 8dbd90285d28e..ec75f22a5e6d1 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkItemStatusClientTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkItemStatusClientTest.java @@ -67,7 +67,7 @@ import org.apache.beam.sdk.metrics.MetricName; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.junit.Before; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesSplitOnlySourceTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesSplitOnlySourceTest.java index 7e197a3aee776..938d53a0c057c 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesSplitOnlySourceTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesSplitOnlySourceTest.java @@ -33,7 +33,7 @@ import org.apache.beam.sdk.io.CountingSource; import org.apache.beam.sdk.io.OffsetBasedSource; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java index c326c8a247d1d..6fa2ffe711f83 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java @@ -31,8 +31,8 @@ import static org.apache.beam.sdk.util.CoderUtils.encodeToByteArray; import static org.apache.beam.sdk.util.SerializableUtils.deserializeFromByteArray; import static org.apache.beam.sdk.util.WindowedValue.valueInGlobalWindow; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables.getStackTraceAsString; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables.getStackTraceAsString; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.contains; @@ -114,10 +114,10 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.ValueWithRecordId; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerPipelineOptionsFactoryTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerPipelineOptionsFactoryTest.java index 9c789dc33ed43..f8684edfa2e44 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerPipelineOptionsFactoryTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerPipelineOptionsFactoryTest.java @@ -25,7 +25,7 @@ import java.nio.file.Paths; import org.apache.beam.runners.dataflow.options.DataflowWorkerHarnessOptions; import org.apache.beam.sdk.testing.RestoreSystemProperties; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/apiary/FixMultiOutputInfosOnParDoInstructionsTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/apiary/FixMultiOutputInfosOnParDoInstructionsTest.java index 233009c24f500..17bea4317ee74 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/apiary/FixMultiOutputInfosOnParDoInstructionsTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/apiary/FixMultiOutputInfosOnParDoInstructionsTest.java @@ -26,7 +26,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.beam.sdk.fn.IdGenerators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/counters/CounterFactoryTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/counters/CounterFactoryTest.java index 6a13fb291b94a..7197621609616 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/counters/CounterFactoryTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/counters/CounterFactoryTest.java @@ -21,7 +21,7 @@ import java.util.List; import org.apache.beam.runners.dataflow.worker.counters.CounterFactory.CounterDistribution; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/counters/CounterTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/counters/CounterTest.java index fcd44f834eef6..095f8c880c921 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/counters/CounterTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/counters/CounterTest.java @@ -26,7 +26,7 @@ import org.apache.beam.runners.dataflow.worker.counters.Counter.CommitState; import org.apache.beam.runners.dataflow.worker.counters.CounterFactory.CounterDistribution; import org.apache.beam.runners.dataflow.worker.counters.CounterFactory.CounterMean; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/graph/LengthPrefixUnknownCodersTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/graph/LengthPrefixUnknownCodersTest.java index 6883a4951597b..7a87cdac7d65c 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/graph/LengthPrefixUnknownCodersTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/graph/LengthPrefixUnknownCodersTest.java @@ -55,9 +55,9 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.MutableNetwork; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.NetworkBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.MutableNetwork; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.NetworkBuilder; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/graph/MapTaskToNetworkFunctionTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/graph/MapTaskToNetworkFunctionTest.java index bde42837225c6..d747e77bf3738 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/graph/MapTaskToNetworkFunctionTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/graph/MapTaskToNetworkFunctionTest.java @@ -46,9 +46,9 @@ import org.apache.beam.runners.dataflow.worker.graph.Nodes.ParallelInstructionNode; import org.apache.beam.sdk.extensions.gcp.util.Transport; import org.apache.beam.sdk.fn.IdGenerators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.Network; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.Network; import org.hamcrest.Matchers; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/graph/NetworksTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/graph/NetworksTest.java index 9c2dc0300ea83..79df82d099b73 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/graph/NetworksTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/graph/NetworksTest.java @@ -34,11 +34,11 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.MutableNetwork; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.graph.NetworkBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.MutableNetwork; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.graph.NetworkBuilder; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingHandlerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingHandlerTest.java index f901034ed61df..b7d6a53f37d55 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingHandlerTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingHandlerTest.java @@ -38,7 +38,7 @@ import org.apache.beam.runners.dataflow.worker.TestOperationContext.TestDataflowExecutionState; import org.apache.beam.runners.dataflow.worker.testing.RestoreDataflowLoggingMDC; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Timestamp; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; import org.junit.After; import org.junit.Before; import org.junit.Rule; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingInitializerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingInitializerTest.java index 72501ce4b3c0c..c1b134cafa1c4 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingInitializerTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingInitializerTest.java @@ -46,8 +46,10 @@ import org.apache.beam.runners.dataflow.options.DataflowWorkerLoggingOptions; import org.apache.beam.runners.dataflow.options.DataflowWorkerLoggingOptions.WorkerLogLevelOverrides; import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.options.SdkHarnessOptions; +import org.apache.beam.sdk.options.SdkHarnessOptions.SdkHarnessLogLevelOverrides; import org.apache.beam.sdk.testing.RestoreSystemProperties; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -102,7 +104,7 @@ public void testWithDefaults() { } @Test - public void testWithConfigurationOverride() { + public void testWithWorkerConfigurationOverride() { DataflowWorkerLoggingOptions options = PipelineOptionsFactory.as(DataflowWorkerLoggingOptions.class); options.setDefaultWorkerLogLevel(DataflowWorkerLoggingOptions.Level.WARN); @@ -116,7 +118,20 @@ public void testWithConfigurationOverride() { } @Test - public void testWithCustomLogLevels() { + public void testWithSdkHarnessConfigurationOverride() { + SdkHarnessOptions options = PipelineOptionsFactory.as(SdkHarnessOptions.class); + options.setDefaultSdkHarnessLogLevel(SdkHarnessOptions.LogLevel.WARN); + + DataflowWorkerLoggingInitializer.configure(options.as(DataflowWorkerLoggingOptions.class)); + + Logger rootLogger = LogManager.getLogManager().getLogger(""); + assertEquals(1, rootLogger.getHandlers().length); + assertEquals(Level.WARNING, rootLogger.getLevel()); + assertIsDataflowWorkerLoggingHandler(rootLogger.getHandlers()[0], Level.ALL); + } + + @Test + public void testWithWorkerCustomLogLevels() { DataflowWorkerLoggingOptions options = PipelineOptionsFactory.as(DataflowWorkerLoggingOptions.class); options.setWorkerLogLevelOverrides( @@ -137,6 +152,27 @@ public void testWithCustomLogLevels() { assertTrue(aLogger.getUseParentHandlers()); } + @Test + public void testWithSdkHarnessCustomLogLevels() { + SdkHarnessOptions options = PipelineOptionsFactory.as(SdkHarnessOptions.class); + options.setSdkHarnessLogLevelOverrides( + new SdkHarnessLogLevelOverrides() + .addOverrideForName("C", SdkHarnessOptions.LogLevel.DEBUG) + .addOverrideForName("D", SdkHarnessOptions.LogLevel.ERROR)); + + DataflowWorkerLoggingInitializer.configure(options.as(DataflowWorkerLoggingOptions.class)); + + Logger aLogger = LogManager.getLogManager().getLogger("C"); + assertEquals(0, aLogger.getHandlers().length); + assertEquals(Level.FINE, aLogger.getLevel()); + assertTrue(aLogger.getUseParentHandlers()); + + Logger bLogger = LogManager.getLogManager().getLogger("D"); + assertEquals(Level.SEVERE, bLogger.getLevel()); + assertEquals(0, bLogger.getHandlers().length); + assertTrue(aLogger.getUseParentHandlers()); + } + private void assertIsDataflowWorkerLoggingHandler(Handler handler, Level level) { assertThat(handler, instanceOf(DataflowWorkerLoggingHandler.class)); assertEquals(level, handler.getLevel()); diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/logging/JulHandlerPrintStreamAdapterFactoryTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/logging/JulHandlerPrintStreamAdapterFactoryTest.java index c62bc7cd7342d..62df1342f1db9 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/logging/JulHandlerPrintStreamAdapterFactoryTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/logging/JulHandlerPrintStreamAdapterFactoryTest.java @@ -31,7 +31,7 @@ import java.util.logging.Level; import java.util.logging.LogRecord; import org.apache.beam.runners.dataflow.worker.LogSaver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/profiler/ScopedProfilerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/profiler/ScopedProfilerTest.java index 9bb5b97e62330..2ad5c3415da11 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/profiler/ScopedProfilerTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/profiler/ScopedProfilerTest.java @@ -23,7 +23,7 @@ import java.util.HashMap; import org.apache.beam.runners.dataflow.worker.profiler.ScopedProfiler.ProfileScope; import org.apache.beam.runners.dataflow.worker.profiler.ScopedProfiler.ProfilerWrapper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; import org.hamcrest.Matchers; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/status/ThreadzServletTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/status/ThreadzServletTest.java index 9658e53075419..7737f7d405b5c 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/status/ThreadzServletTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/status/ThreadzServletTest.java @@ -24,7 +24,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.hamcrest.Matchers; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/ActiveWorkStateTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/ActiveWorkStateTest.java new file mode 100644 index 0000000000000..12ae816de8292 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/ActiveWorkStateTest.java @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.streaming; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList.toImmutableList; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import com.google.auto.value.AutoValue; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import javax.annotation.Nullable; +import org.apache.beam.runners.dataflow.worker.streaming.ActiveWorkState.ActivateWorkResult; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataRequest; +import org.apache.beam.runners.dataflow.worker.windmill.state.WindmillStateCache; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.joda.time.Instant; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class ActiveWorkStateTest { + + private final WindmillStateCache.ForComputation computationStateCache = + mock(WindmillStateCache.ForComputation.class); + private Map> readOnlyActiveWork; + + private ActiveWorkState activeWorkState; + + private static ShardedKey shardedKey(String str, long shardKey) { + return ShardedKey.create(ByteString.copyFromUtf8(str), shardKey); + } + + private static Work emptyWork() { + return createWork(null); + } + + private static Work createWork(@Nullable Windmill.WorkItem workItem) { + return Work.create(workItem, Instant::now, Collections.emptyList(), unused -> {}); + } + + private static Work expiredWork(Windmill.WorkItem workItem) { + return Work.create(workItem, () -> Instant.EPOCH, Collections.emptyList(), unused -> {}); + } + + private static Windmill.WorkItem createWorkItem(long workToken) { + return Windmill.WorkItem.newBuilder() + .setKey(ByteString.copyFromUtf8("")) + .setShardingKey(1) + .setWorkToken(workToken) + .build(); + } + + @Before + public void setup() { + Map> readWriteActiveWorkMap = new HashMap<>(); + // Only use readOnlyActiveWork to verify internal behavior in reaction to exposed API calls. + readOnlyActiveWork = Collections.unmodifiableMap(readWriteActiveWorkMap); + activeWorkState = ActiveWorkState.forTesting(readWriteActiveWorkMap, computationStateCache); + } + + @Test + public void testActivateWorkForKey_EXECUTE_unknownKey() { + ActivateWorkResult activateWorkResult = + activeWorkState.activateWorkForKey(shardedKey("someKey", 1L), emptyWork()); + + assertEquals(ActivateWorkResult.EXECUTE, activateWorkResult); + } + + @Test + public void testActivateWorkForKey_EXECUTE_emptyWorkQueueForKey() { + ShardedKey shardedKey = shardedKey("someKey", 1L); + long workToken = 1L; + + ActivateWorkResult activateWorkResult = + activeWorkState.activateWorkForKey(shardedKey, createWork(createWorkItem(workToken))); + + Optional nextWorkForKey = + activeWorkState.completeWorkAndGetNextWorkForKey(shardedKey, workToken); + + assertEquals(ActivateWorkResult.EXECUTE, activateWorkResult); + assertEquals(Optional.empty(), nextWorkForKey); + assertThat(readOnlyActiveWork).doesNotContainKey(shardedKey); + } + + @Test + public void testActivateWorkForKey_DUPLICATE() { + long workToken = 10L; + ShardedKey shardedKey = shardedKey("someKey", 1L); + + // ActivateWork with the same shardedKey, and the same workTokens. + activeWorkState.activateWorkForKey(shardedKey, createWork(createWorkItem(workToken))); + ActivateWorkResult activateWorkResult = + activeWorkState.activateWorkForKey(shardedKey, createWork(createWorkItem(workToken))); + + assertEquals(ActivateWorkResult.DUPLICATE, activateWorkResult); + } + + @Test + public void testActivateWorkForKey_QUEUED() { + ShardedKey shardedKey = shardedKey("someKey", 1L); + + // ActivateWork with the same shardedKey, but different workTokens. + activeWorkState.activateWorkForKey(shardedKey, createWork(createWorkItem(1L))); + ActivateWorkResult activateWorkResult = + activeWorkState.activateWorkForKey(shardedKey, createWork(createWorkItem(2L))); + + assertEquals(ActivateWorkResult.QUEUED, activateWorkResult); + } + + @Test + public void testCompleteWorkAndGetNextWorkForKey_noWorkQueueForKey() { + assertEquals( + Optional.empty(), + activeWorkState.completeWorkAndGetNextWorkForKey(shardedKey("someKey", 1L), 10L)); + } + + @Test + public void testCompleteWorkAndGetNextWorkForKey_currentWorkInQueueDoesNotMatchWorkToComplete() { + long workTokenToComplete = 1L; + + Work workInQueue = createWork(createWorkItem(2L)); + ShardedKey shardedKey = shardedKey("someKey", 1L); + + activeWorkState.activateWorkForKey(shardedKey, workInQueue); + activeWorkState.completeWorkAndGetNextWorkForKey(shardedKey, workTokenToComplete); + + assertEquals(1, readOnlyActiveWork.get(shardedKey).size()); + assertEquals(workInQueue, readOnlyActiveWork.get(shardedKey).peek()); + } + + @Test + public void testCompleteWorkAndGetNextWorkForKey_removesWorkFromQueueWhenComplete() { + long workTokenToComplete = 1L; + + Work activeWork = createWork(createWorkItem(workTokenToComplete)); + Work nextWork = createWork(createWorkItem(2L)); + ShardedKey shardedKey = shardedKey("someKey", 1L); + + activeWorkState.activateWorkForKey(shardedKey, activeWork); + activeWorkState.activateWorkForKey(shardedKey, nextWork); + activeWorkState.completeWorkAndGetNextWorkForKey(shardedKey, workTokenToComplete); + + assertEquals(nextWork, readOnlyActiveWork.get(shardedKey).peek()); + assertEquals(1, readOnlyActiveWork.get(shardedKey).size()); + assertFalse(readOnlyActiveWork.get(shardedKey).contains(activeWork)); + } + + @Test + public void testCompleteWorkAndGetNextWorkForKey_removesQueueIfNoWorkPresent() { + Work workInQueue = createWork(createWorkItem(1L)); + ShardedKey shardedKey = shardedKey("someKey", 1L); + + activeWorkState.activateWorkForKey(shardedKey, workInQueue); + activeWorkState.completeWorkAndGetNextWorkForKey( + shardedKey, workInQueue.getWorkItem().getWorkToken()); + + assertFalse(readOnlyActiveWork.containsKey(shardedKey)); + } + + @Test + public void testCompleteWorkAndGetNextWorkForKey_returnsWorkIfPresent() { + Work workToBeCompleted = createWork(createWorkItem(1L)); + Work nextWork = createWork(createWorkItem(2L)); + ShardedKey shardedKey = shardedKey("someKey", 1L); + + activeWorkState.activateWorkForKey(shardedKey, workToBeCompleted); + activeWorkState.activateWorkForKey(shardedKey, nextWork); + activeWorkState.completeWorkAndGetNextWorkForKey( + shardedKey, workToBeCompleted.getWorkItem().getWorkToken()); + + Optional nextWorkOpt = + activeWorkState.completeWorkAndGetNextWorkForKey( + shardedKey, workToBeCompleted.getWorkItem().getWorkToken()); + + assertTrue(nextWorkOpt.isPresent()); + assertSame(nextWork, nextWorkOpt.get()); + + Optional endOfWorkQueue = + activeWorkState.completeWorkAndGetNextWorkForKey( + shardedKey, nextWork.getWorkItem().getWorkToken()); + + assertFalse(endOfWorkQueue.isPresent()); + assertFalse(readOnlyActiveWork.containsKey(shardedKey)); + } + + @Test + public void testInvalidateStuckCommits() { + Map invalidatedCommits = new HashMap<>(); + + Work stuckWork1 = expiredWork(createWorkItem(1L)); + stuckWork1.setState(Work.State.COMMITTING); + Work stuckWork2 = expiredWork(createWorkItem(2L)); + stuckWork2.setState(Work.State.COMMITTING); + ShardedKey shardedKey1 = shardedKey("someKey", 1L); + ShardedKey shardedKey2 = shardedKey("anotherKey", 2L); + + activeWorkState.activateWorkForKey(shardedKey1, stuckWork1); + activeWorkState.activateWorkForKey(shardedKey2, stuckWork2); + + activeWorkState.invalidateStuckCommits(Instant.now(), invalidatedCommits::put); + + assertThat(invalidatedCommits) + .containsEntry(shardedKey1, stuckWork1.getWorkItem().getWorkToken()); + assertThat(invalidatedCommits) + .containsEntry(shardedKey2, stuckWork2.getWorkItem().getWorkToken()); + verify(computationStateCache).invalidate(shardedKey1.key(), shardedKey1.shardingKey()); + verify(computationStateCache).invalidate(shardedKey2.key(), shardedKey2.shardingKey()); + } + + @Test + public void testGetKeysToRefresh() { + Instant refreshDeadline = Instant.now(); + + Work freshWork = createWork(createWorkItem(3L)); + Work refreshableWork1 = expiredWork(createWorkItem(1L)); + refreshableWork1.setState(Work.State.COMMITTING); + Work refreshableWork2 = expiredWork(createWorkItem(2L)); + refreshableWork2.setState(Work.State.COMMITTING); + ShardedKey shardedKey1 = shardedKey("someKey", 1L); + ShardedKey shardedKey2 = shardedKey("anotherKey", 2L); + + activeWorkState.activateWorkForKey(shardedKey1, refreshableWork1); + activeWorkState.activateWorkForKey(shardedKey1, freshWork); + activeWorkState.activateWorkForKey(shardedKey2, refreshableWork2); + + ImmutableList requests = activeWorkState.getKeysToRefresh(refreshDeadline); + + ImmutableList expected = + ImmutableList.of( + GetDataRequestKeyShardingKeyAndWorkToken.from(shardedKey1, refreshableWork1), + GetDataRequestKeyShardingKeyAndWorkToken.from(shardedKey2, refreshableWork2)); + + ImmutableList actual = + requests.stream() + .map(GetDataRequestKeyShardingKeyAndWorkToken::from) + .collect(toImmutableList()); + + assertThat(actual).containsExactlyElementsIn(expected); + } + + @AutoValue + abstract static class GetDataRequestKeyShardingKeyAndWorkToken { + + private static GetDataRequestKeyShardingKeyAndWorkToken create( + ByteString key, long shardingKey, long workToken) { + return new AutoValue_ActiveWorkStateTest_GetDataRequestKeyShardingKeyAndWorkToken( + key, shardingKey, workToken); + } + + private static GetDataRequestKeyShardingKeyAndWorkToken from( + KeyedGetDataRequest keyedGetDataRequest) { + return create( + keyedGetDataRequest.getKey(), + keyedGetDataRequest.getShardingKey(), + keyedGetDataRequest.getWorkToken()); + } + + private static GetDataRequestKeyShardingKeyAndWorkToken from(ShardedKey shardedKey, Work work) { + return create(shardedKey.key(), shardedKey.shardingKey(), work.getWorkItem().getWorkToken()); + } + + abstract ByteString key(); + + abstract long shardingKey(); + + abstract long workToken(); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/WeightBoundedQueueTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/WeightBoundedQueueTest.java new file mode 100644 index 0000000000000..b2d98fb0e9543 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/WeightBoundedQueueTest.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.streaming; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class WeightBoundedQueueTest { + private static final int MAX_WEIGHT = 10; + + @Test + public void testPut_hasCapacity() { + WeightedBoundedQueue queue = + WeightedBoundedQueue.create(MAX_WEIGHT, i -> Math.min(MAX_WEIGHT, i)); + + int insertedValue = 1; + + queue.put(insertedValue); + + assertEquals(insertedValue, queue.queuedElementsWeight()); + assertEquals(1, queue.size()); + assertEquals(insertedValue, (int) queue.poll()); + } + + @Test + public void testPut_noCapacity() throws InterruptedException { + WeightedBoundedQueue queue = + WeightedBoundedQueue.create(MAX_WEIGHT, i -> Math.min(MAX_WEIGHT, i)); + + // Insert value that takes all the capacity into the queue. + queue.put(MAX_WEIGHT); + + // Try to insert another value into the queue. This will block since there is no capacity in the + // queue. + Thread putThread = + new Thread( + () -> { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + queue.put(MAX_WEIGHT); + }); + putThread.start(); + + // Should only see the first value in the queue, since the queue is at capacity. thread2 + // should be blocked. + assertEquals(MAX_WEIGHT, queue.queuedElementsWeight()); + assertEquals(1, queue.size()); + + // Poll the queue, pulling off the only value inside and freeing up the capacity in the queue. + queue.poll(); + + // Wait for the putThread which was previously blocked due to the queue being at capacity. + putThread.join(); + + assertEquals(MAX_WEIGHT, queue.queuedElementsWeight()); + assertEquals(1, queue.size()); + } + + @Test + public void testPoll() { + WeightedBoundedQueue queue = + WeightedBoundedQueue.create(MAX_WEIGHT, i -> Math.min(MAX_WEIGHT, i)); + + int insertedValue1 = 1; + int insertedValue2 = 2; + + queue.put(insertedValue1); + queue.put(insertedValue2); + + assertEquals(insertedValue1 + insertedValue2, queue.queuedElementsWeight()); + assertEquals(2, queue.size()); + assertEquals(insertedValue1, (int) queue.poll()); + assertEquals(1, queue.size()); + } + + @Test + public void testPoll_withTimeout() throws InterruptedException { + WeightedBoundedQueue queue = + WeightedBoundedQueue.create(MAX_WEIGHT, i -> Math.min(MAX_WEIGHT, i)); + int pollWaitTimeMillis = 10000; + int insertedValue1 = 1; + + AtomicInteger pollResult = new AtomicInteger(); + Thread pollThread = + new Thread( + () -> { + int polled; + try { + polled = queue.poll(pollWaitTimeMillis, TimeUnit.MILLISECONDS); + pollResult.set(polled); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + + pollThread.start(); + Thread.sleep(pollWaitTimeMillis / 100); + queue.put(insertedValue1); + pollThread.join(); + + assertEquals(insertedValue1, pollResult.get()); + } + + @Test + public void testPoll_withTimeout_timesOut() throws InterruptedException { + WeightedBoundedQueue queue = + WeightedBoundedQueue.create(MAX_WEIGHT, i -> Math.min(MAX_WEIGHT, i)); + int defaultPollResult = -10; + int pollWaitTimeMillis = 100; + int insertedValue1 = 1; + + // AtomicInteger default isn't null, so set it to a negative value and verify that it doesn't + // change. + AtomicInteger pollResult = new AtomicInteger(defaultPollResult); + + Thread pollThread = + new Thread( + () -> { + int polled; + try { + polled = queue.poll(pollWaitTimeMillis, TimeUnit.MILLISECONDS); + pollResult.set(polled); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + + pollThread.start(); + Thread.sleep(pollWaitTimeMillis * 100); + queue.put(insertedValue1); + pollThread.join(); + + assertEquals(defaultPollResult, pollResult.get()); + } + + @Test + public void testPoll_emptyQueue() { + WeightedBoundedQueue queue = + WeightedBoundedQueue.create(MAX_WEIGHT, i -> Math.min(MAX_WEIGHT, i)); + + assertNull(queue.poll()); + } + + @Test + public void testTake() throws InterruptedException { + WeightedBoundedQueue queue = + WeightedBoundedQueue.create(MAX_WEIGHT, i -> Math.min(MAX_WEIGHT, i)); + + AtomicInteger value = new AtomicInteger(); + // Should block until value is available + Thread takeThread = + new Thread( + () -> { + try { + value.set(queue.take()); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + takeThread.start(); + + Thread.sleep(100); + queue.put(MAX_WEIGHT); + + takeThread.join(); + + assertEquals(MAX_WEIGHT, value.get()); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/CounterHamcrestMatchers.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/CounterHamcrestMatchers.java index 2cb0833c2e0f8..3332532f29830 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/CounterHamcrestMatchers.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/CounterHamcrestMatchers.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.worker.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.services.dataflow.model.CounterMetadata; import com.google.api.services.dataflow.model.CounterStructuredName; @@ -30,7 +30,7 @@ import org.apache.beam.runners.dataflow.worker.counters.CounterFactory; import org.apache.beam.runners.dataflow.worker.counters.CounterName; import org.apache.beam.runners.dataflow.worker.counters.DataflowCounterUpdateExtractor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Description; import org.hamcrest.TypeSafeMatcher; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/DataflowCounterUpdateExtractorTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/DataflowCounterUpdateExtractorTest.java index fb9bcd20d8493..b5b530260266a 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/DataflowCounterUpdateExtractorTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/DataflowCounterUpdateExtractorTest.java @@ -43,7 +43,7 @@ import org.apache.beam.runners.dataflow.worker.counters.CounterName; import org.apache.beam.runners.dataflow.worker.counters.DataflowCounterUpdateExtractor; import org.apache.beam.runners.dataflow.worker.util.common.worker.ShuffleReadCounter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/GroupAlsoByWindowProperties.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/GroupAlsoByWindowProperties.java index 79c2b9a1a2e4f..8a42107e09f4d 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/GroupAlsoByWindowProperties.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/GroupAlsoByWindowProperties.java @@ -48,11 +48,11 @@ import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LoadingCache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/ListOutputManager.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/ListOutputManager.java index 7445ebd79285c..742ba68cf6344 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/ListOutputManager.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/ListOutputManager.java @@ -23,8 +23,8 @@ import org.apache.beam.runners.core.DoFnRunners.OutputManager; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; /** * An implementation of {@code OutputManager} using simple lists, for testing and in-memory contexts diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ExecutorTestUtils.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ExecutorTestUtils.java index ee891b48bfdb9..2c35f4bf99dbd 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ExecutorTestUtils.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ExecutorTestUtils.java @@ -29,7 +29,7 @@ import org.apache.beam.runners.dataflow.worker.IntrinsicMapTaskExecutorFactory.ElementByteSizeObservableCoder; import org.apache.beam.runners.dataflow.worker.counters.CounterSet; import org.apache.beam.sdk.coders.StringUtf8Coder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Utilities for tests. */ @SuppressWarnings({ diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingShuffleEntryIteratorTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingShuffleEntryIteratorTest.java index 4ab7e2053d514..4a4b37d93316b 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingShuffleEntryIteratorTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/GroupingShuffleEntryIteratorTest.java @@ -42,8 +42,8 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.util.common.Reiterator; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.After; import org.junit.Before; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/MapTaskExecutorTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/MapTaskExecutorTest.java index 260c9e12e71bc..5c34441766c3d 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/MapTaskExecutorTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/MapTaskExecutorTest.java @@ -63,7 +63,7 @@ import org.apache.beam.sdk.metrics.Metrics; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ReadOperationTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ReadOperationTest.java index f6f5f1b7bdc0a..dd7d3effdf354 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ReadOperationTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ReadOperationTest.java @@ -65,7 +65,7 @@ import org.apache.beam.runners.dataflow.worker.util.common.worker.ExecutorTestUtils.TestReader; import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.io.range.OffsetRangeTracker; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/TestOutputReceiver.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/TestOutputReceiver.java index afd86aa995e67..62d0623d39964 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/TestOutputReceiver.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/TestOutputReceiver.java @@ -27,7 +27,7 @@ import org.apache.beam.runners.dataflow.worker.counters.NameContext; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.StringUtf8Coder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** An OutputReceiver that allows the output elements to be retrieved. */ public class TestOutputReceiver extends OutputReceiver { diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillStreamPoolTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillStreamPoolTest.java new file mode 100644 index 0000000000000..9924bb7d2b2bc --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/WindmillStreamPoolTest.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Duration; +import org.joda.time.Instant; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class WindmillStreamPoolTest { + private static final int DEFAULT_NUM_STREAMS = 10; + private static final int NEW_STREAM_HOLDS = 2; + private final ConcurrentHashMap< + TestWindmillStream, WindmillStreamPool.StreamData> + holds = new ConcurrentHashMap<>(); + private List> streams; + + @Before + public void setUp() { + streams = WindmillStreamPool.newStreamList(DEFAULT_NUM_STREAMS); + holds.clear(); + } + + @Test + public void testGetStream_returnsAndCachesNewStream() { + Duration streamTimeout = Duration.standardSeconds(1); + WindmillStreamPool streamPool = + WindmillStreamPool.forTesting( + streamTimeout, () -> new TestWindmillStream(Instant.now()), streams, holds); + TestWindmillStream stream = streamPool.getStream(); + assertTrue(holds.containsKey(stream)); + assertEquals(2, holds.get(stream).holds); + assertTrue(streams.contains(holds.get(stream))); + } + + @Test + public void testGetStream_returnsCachedStreamAndIncrementsHolds() { + Duration streamTimeout = Duration.standardDays(1); + int cachedStreamHolds = 2; + // Populate the stream data. + for (int i = 0; i < DEFAULT_NUM_STREAMS; i++) { + WindmillStreamPool.StreamData streamData = + new WindmillStreamPool.StreamData<>(new TestWindmillStream(Instant.now())); + streamData.holds = cachedStreamHolds; + streams.set(i, streamData); + holds.put(streamData.stream, streamData); + } + + WindmillStreamPool streamPool = + WindmillStreamPool.forTesting( + streamTimeout, () -> new TestWindmillStream(Instant.now()), streams, holds); + TestWindmillStream stream = streamPool.getStream(); + assertEquals(cachedStreamHolds + 1, holds.get(stream).holds); + } + + @Test + public void testGetStream_returnsAndCachesNewStream_whenOldStreamTimedOutAndDrained() { + Duration streamTimeout = Duration.ZERO; + Instant expired = Instant.EPOCH; + // Populate the stream data. + for (int i = 0; i < DEFAULT_NUM_STREAMS; i++) { + WindmillStreamPool.StreamData streamData = + new WindmillStreamPool.StreamData<>(new TestWindmillStream(expired)); + streams.set(i, streamData); + holds.put(streamData.stream, streamData); + } + + WindmillStreamPool streamPool = + WindmillStreamPool.forTesting( + streamTimeout, () -> new TestWindmillStream(Instant.now()), streams, holds); + TestWindmillStream stream = streamPool.getStream(); + assertEquals(NEW_STREAM_HOLDS, holds.get(stream).holds); + } + + @Test + public void testGetStream_closesInvalidStream() { + Duration streamTimeout = Duration.ZERO; + Instant expired = Instant.EPOCH; + WindmillStreamPool.StreamData streamData = + new WindmillStreamPool.StreamData<>(new TestWindmillStream(expired)); + List> streams = + WindmillStreamPool.newStreamList(1); + streams.set(0, streamData); + holds.put(streamData.stream, streamData); + + WindmillStreamPool streamPool = + WindmillStreamPool.forTesting( + streamTimeout, () -> new TestWindmillStream(Instant.now()), streams, holds); + + TestWindmillStream stream = streamPool.getStream(); + assertEquals(NEW_STREAM_HOLDS, holds.get(stream).holds); + assertTrue(streamData.stream.closed); + assertEquals(1, streams.size()); + assertEquals(1, holds.size()); + } + + @Test + public void testGetStream_returnsNewAndCachesNewStream_whenOldStreamTimedOutAndNotDrained() { + int notDrained = 4; + Duration streamTimeout = Duration.ZERO; + Instant expired = Instant.EPOCH; + + // Populate the stream data. + List> streams = + WindmillStreamPool.newStreamList(1); + WindmillStreamPool.StreamData expiredStreamData = + new WindmillStreamPool.StreamData<>(new TestWindmillStream(expired)); + expiredStreamData.holds = notDrained; + streams.set(0, expiredStreamData); + holds.put(expiredStreamData.stream, expiredStreamData); + + WindmillStreamPool streamPool = + WindmillStreamPool.forTesting( + streamTimeout, () -> new TestWindmillStream(Instant.now()), streams, holds); + TestWindmillStream newStream = streamPool.getStream(); + + assertEquals(NEW_STREAM_HOLDS, holds.get(newStream).holds); + assertEquals(2, holds.size()); + assertEquals(1, streams.size()); + } + + @Test + public void testGetStream_doesNotCloseExpiredStream_whenNotDrained() { + int notDrained = 4; + Duration streamTimeout = Duration.ZERO; + Instant expired = Instant.EPOCH; + + // Populate the stream data. + List> streams = + WindmillStreamPool.newStreamList(1); + WindmillStreamPool.StreamData expiredStreamData = + new WindmillStreamPool.StreamData<>(new TestWindmillStream(expired)); + expiredStreamData.holds = notDrained; + streams.set(0, expiredStreamData); + holds.put(expiredStreamData.stream, expiredStreamData); + + WindmillStreamPool streamPool = + WindmillStreamPool.forTesting( + streamTimeout, () -> new TestWindmillStream(Instant.now()), streams, holds); + TestWindmillStream newStream = streamPool.getStream(); + + assertNotSame(expiredStreamData.stream, newStream); + assertFalse(expiredStreamData.stream.closed); + assertEquals(notDrained - 1, expiredStreamData.holds); + assertEquals(2, holds.size()); + assertEquals(1, streams.size()); + assertTrue(holds.containsKey(expiredStreamData.stream)); + assertFalse(streams.contains(expiredStreamData)); + } + + @Test + public void testReleaseStream_closesStream() { + Duration streamTimeout = Duration.standardDays(1); + WindmillStreamPool.StreamData streamData = + new WindmillStreamPool.StreamData<>(new TestWindmillStream(Instant.now())); + List> streams = + WindmillStreamPool.newStreamList(1); + streams.set(0, streamData); + holds.put(streamData.stream, streamData); + + WindmillStreamPool streamPool = + WindmillStreamPool.forTesting( + streamTimeout, () -> new TestWindmillStream(Instant.now()), streams, holds); + TestWindmillStream stream = streamPool.getStream(); + holds.get(stream).holds = 1; + streamPool.releaseStream(stream); + assertFalse(holds.containsKey(stream)); + assertTrue(stream.closed); + } + + @Test + public void testReleaseStream_doesNotCloseStream_ifStreamHasHolds() { + Duration streamTimeout = Duration.standardDays(1); + WindmillStreamPool.StreamData streamData = + new WindmillStreamPool.StreamData<>(new TestWindmillStream(Instant.now())); + List> streams = + WindmillStreamPool.newStreamList(1); + streams.set(0, streamData); + holds.put(streamData.stream, streamData); + + WindmillStreamPool streamPool = + WindmillStreamPool.forTesting( + streamTimeout, () -> new TestWindmillStream(Instant.now()), streams, holds); + TestWindmillStream stream = streamPool.getStream(); + streamPool.releaseStream(stream); + assertTrue(holds.containsKey(stream)); + assertFalse(stream.closed); + } + + @Test + public void testReleaseStream_throwsExceptionWhenAttemptingToReleaseUnheldStream() { + WindmillStreamPool streamPool = + WindmillStreamPool.forTesting( + Duration.ZERO, () -> new TestWindmillStream(Instant.now()), streams, holds); + TestWindmillStream unheldStream = new TestWindmillStream(Instant.now()); + assertThrows(IllegalStateException.class, () -> streamPool.releaseStream(unheldStream)); + } + + private static class TestWindmillStream implements WindmillStream { + private final Instant startTime; + private boolean closed; + + private TestWindmillStream(Instant startTime) { + this.startTime = startTime; + this.closed = false; + } + + @Override + public void close() { + closed = true; + } + + @Override + public boolean awaitTermination(int time, TimeUnit unit) { + return false; + } + + @Override + public Instant startTime() { + return startTime; + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcGetWorkerMetadataStreamTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcGetWorkerMetadataStreamTest.java new file mode 100644 index 0000000000000..45ed3381a8bfe --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcGetWorkerMetadataStreamTest.java @@ -0,0 +1,328 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.grpcclient; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.runners.dataflow.worker.windmill.AbstractWindmillStream.DEFAULT_STREAM_RPC_DEADLINE_SECONDS; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.apache.beam.runners.dataflow.worker.windmill.AbstractWindmillStream; +import org.apache.beam.runners.dataflow.worker.windmill.CloudWindmillServiceV1Alpha1Grpc; +import org.apache.beam.runners.dataflow.worker.windmill.StreamObserverFactory; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.JobHeader; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.WorkerMetadataRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.WorkerMetadataResponse; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillEndpoints; +import org.apache.beam.sdk.util.FluentBackoff; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ManagedChannel; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Server; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessChannelBuilder; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessServerBuilder; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.testing.GrpcCleanupRule; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.util.MutableHandlerRegistry; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mockito; + +@RunWith(JUnit4.class) +public class GrpcGetWorkerMetadataStreamTest { + private static final String IPV6_ADDRESS_1 = "2001:db8:0000:bac5:0000:0000:fed0:81a2"; + private static final String IPV6_ADDRESS_2 = "2001:db8:0000:bac5:0000:0000:fed0:82a3"; + private static final List DIRECT_PATH_ENDPOINTS = + Lists.newArrayList( + WorkerMetadataResponse.Endpoint.newBuilder() + .setDirectEndpoint(IPV6_ADDRESS_1) + .setWorkerToken("worker_token") + .build()); + private static final Map GLOBAL_DATA_ENDPOINTS = + Maps.newHashMap(); + private static final JobHeader TEST_JOB_HEADER = + JobHeader.newBuilder() + .setJobId("test_job") + .setWorkerId("test_worker") + .setProjectId("test_project") + .build(); + private static final String FAKE_SERVER_NAME = "Fake server for GrpcGetWorkerMetadataStreamTest"; + @Rule public final GrpcCleanupRule grpcCleanup = new GrpcCleanupRule(); + private final MutableHandlerRegistry serviceRegistry = new MutableHandlerRegistry(); + private final Set> streamRegistry = new HashSet<>(); + private ManagedChannel inProcessChannel; + private GrpcGetWorkerMetadataStream stream; + + private GrpcGetWorkerMetadataStream getWorkerMetadataTestStream( + GetWorkerMetadataTestStub getWorkerMetadataTestStub, + int metadataVersion, + Consumer endpointsConsumer) { + serviceRegistry.addService(getWorkerMetadataTestStub); + return GrpcGetWorkerMetadataStream.create( + responseObserver -> + CloudWindmillServiceV1Alpha1Grpc.newStub(inProcessChannel) + .getWorkerMetadataStream(responseObserver), + FluentBackoff.DEFAULT.backoff(), + StreamObserverFactory.direct(DEFAULT_STREAM_RPC_DEADLINE_SECONDS * 2, 1), + streamRegistry, + 1, // logEveryNStreamFailures + TEST_JOB_HEADER, + metadataVersion, + new ThrottleTimer(), + endpointsConsumer); + } + + @Before + public void setUp() throws IOException { + Server server = + InProcessServerBuilder.forName(FAKE_SERVER_NAME) + .fallbackHandlerRegistry(serviceRegistry) + .directExecutor() + .build() + .start(); + + inProcessChannel = + grpcCleanup.register( + InProcessChannelBuilder.forName(FAKE_SERVER_NAME).directExecutor().build()); + grpcCleanup.register(server); + grpcCleanup.register(inProcessChannel); + GLOBAL_DATA_ENDPOINTS.put( + "global_data", + WorkerMetadataResponse.Endpoint.newBuilder() + .setDirectEndpoint(IPV6_ADDRESS_1) + .setWorkerToken("worker_token") + .build()); + } + + @After + public void cleanUp() { + inProcessChannel.shutdownNow(); + } + + @Test + public void testGetWorkerMetadata() { + WorkerMetadataResponse mockResponse = + WorkerMetadataResponse.newBuilder() + .setMetadataVersion(1) + .addAllWorkEndpoints(DIRECT_PATH_ENDPOINTS) + .putAllGlobalDataEndpoints(GLOBAL_DATA_ENDPOINTS) + .build(); + TestWindmillEndpointsConsumer testWindmillEndpointsConsumer = + new TestWindmillEndpointsConsumer(); + GetWorkerMetadataTestStub testStub = + new GetWorkerMetadataTestStub(new TestGetWorkMetadataRequestObserver()); + int metadataVersion = -1; + stream = getWorkerMetadataTestStream(testStub, metadataVersion, testWindmillEndpointsConsumer); + testStub.injectWorkerMetadata(mockResponse); + + assertThat(testWindmillEndpointsConsumer.globalDataEndpoints.keySet()) + .containsExactlyElementsIn(GLOBAL_DATA_ENDPOINTS.keySet()); + assertThat(testWindmillEndpointsConsumer.windmillEndpoints) + .containsExactlyElementsIn( + DIRECT_PATH_ENDPOINTS.stream() + .map(WindmillEndpoints.Endpoint::from) + .collect(Collectors.toList())); + } + + @Test + public void testGetWorkerMetadata_consumesSubsequentResponseMetadata() { + WorkerMetadataResponse initialResponse = + WorkerMetadataResponse.newBuilder() + .setMetadataVersion(1) + .addAllWorkEndpoints(DIRECT_PATH_ENDPOINTS) + .putAllGlobalDataEndpoints(GLOBAL_DATA_ENDPOINTS) + .build(); + TestWindmillEndpointsConsumer testWindmillEndpointsConsumer = + Mockito.spy(new TestWindmillEndpointsConsumer()); + + GetWorkerMetadataTestStub testStub = + new GetWorkerMetadataTestStub(new TestGetWorkMetadataRequestObserver()); + int metadataVersion = 0; + stream = getWorkerMetadataTestStream(testStub, metadataVersion, testWindmillEndpointsConsumer); + testStub.injectWorkerMetadata(initialResponse); + + List newDirectPathEndpoints = + Lists.newArrayList( + WorkerMetadataResponse.Endpoint.newBuilder().setDirectEndpoint(IPV6_ADDRESS_2).build()); + Map newGlobalDataEndpoints = new HashMap<>(); + newGlobalDataEndpoints.put( + "new_global_data", + WorkerMetadataResponse.Endpoint.newBuilder().setDirectEndpoint(IPV6_ADDRESS_2).build()); + + WorkerMetadataResponse newWorkMetadataResponse = + WorkerMetadataResponse.newBuilder() + .setMetadataVersion(initialResponse.getMetadataVersion() + 1) + .addAllWorkEndpoints(newDirectPathEndpoints) + .putAllGlobalDataEndpoints(newGlobalDataEndpoints) + .build(); + + testStub.injectWorkerMetadata(newWorkMetadataResponse); + + assertThat(newGlobalDataEndpoints.keySet()) + .containsExactlyElementsIn(testWindmillEndpointsConsumer.globalDataEndpoints.keySet()); + assertThat(testWindmillEndpointsConsumer.windmillEndpoints) + .containsExactlyElementsIn( + newDirectPathEndpoints.stream() + .map(WindmillEndpoints.Endpoint::from) + .collect(Collectors.toList())); + } + + @Test + public void testGetWorkerMetadata_doesNotConsumeResponseIfMetadataStale() { + WorkerMetadataResponse freshEndpoints = + WorkerMetadataResponse.newBuilder() + .setMetadataVersion(2) + .addAllWorkEndpoints(DIRECT_PATH_ENDPOINTS) + .putAllGlobalDataEndpoints(GLOBAL_DATA_ENDPOINTS) + .build(); + + TestWindmillEndpointsConsumer testWindmillEndpointsConsumer = + Mockito.spy(new TestWindmillEndpointsConsumer()); + GetWorkerMetadataTestStub testStub = + new GetWorkerMetadataTestStub(new TestGetWorkMetadataRequestObserver()); + int metadataVersion = 0; + stream = getWorkerMetadataTestStream(testStub, metadataVersion, testWindmillEndpointsConsumer); + testStub.injectWorkerMetadata(freshEndpoints); + + List staleDirectPathEndpoints = + Lists.newArrayList( + WorkerMetadataResponse.Endpoint.newBuilder() + .setDirectEndpoint("staleWindmillEndpoint") + .build()); + Map staleGlobalDataEndpoints = new HashMap<>(); + staleGlobalDataEndpoints.put( + "stale_global_data", + WorkerMetadataResponse.Endpoint.newBuilder().setDirectEndpoint("staleGlobalData").build()); + + testStub.injectWorkerMetadata( + WorkerMetadataResponse.newBuilder() + .setMetadataVersion(1) + .addAllWorkEndpoints(staleDirectPathEndpoints) + .putAllGlobalDataEndpoints(staleGlobalDataEndpoints) + .build()); + + // Should have ignored the stale update and only used initial. + verify(testWindmillEndpointsConsumer).accept(WindmillEndpoints.from(freshEndpoints)); + verifyNoMoreInteractions(testWindmillEndpointsConsumer); + } + + @Test + public void testGetWorkerMetadata_correctlyAddsAndRemovesStreamFromRegistry() { + GetWorkerMetadataTestStub testStub = + new GetWorkerMetadataTestStub(new TestGetWorkMetadataRequestObserver()); + stream = getWorkerMetadataTestStream(testStub, 0, new TestWindmillEndpointsConsumer()); + testStub.injectWorkerMetadata( + WorkerMetadataResponse.newBuilder() + .setMetadataVersion(1) + .addAllWorkEndpoints(DIRECT_PATH_ENDPOINTS) + .putAllGlobalDataEndpoints(GLOBAL_DATA_ENDPOINTS) + .build()); + + assertTrue(streamRegistry.contains(stream)); + stream.close(); + assertFalse(streamRegistry.contains(stream)); + } + + @Test + public void testSendHealthCheck() { + TestGetWorkMetadataRequestObserver requestObserver = + Mockito.spy(new TestGetWorkMetadataRequestObserver()); + GetWorkerMetadataTestStub testStub = new GetWorkerMetadataTestStub(requestObserver); + stream = getWorkerMetadataTestStream(testStub, 0, new TestWindmillEndpointsConsumer()); + stream.sendHealthCheck(); + + verify(requestObserver).onNext(WorkerMetadataRequest.getDefaultInstance()); + } + + private static class GetWorkerMetadataTestStub + extends CloudWindmillServiceV1Alpha1Grpc.CloudWindmillServiceV1Alpha1ImplBase { + private final TestGetWorkMetadataRequestObserver requestObserver; + private @Nullable StreamObserver responseObserver; + + private GetWorkerMetadataTestStub(TestGetWorkMetadataRequestObserver requestObserver) { + this.requestObserver = requestObserver; + } + + @Override + public StreamObserver getWorkerMetadataStream( + StreamObserver responseObserver) { + if (this.responseObserver == null) { + this.responseObserver = responseObserver; + requestObserver.responseObserver = this.responseObserver; + } + + return requestObserver; + } + + private void injectWorkerMetadata(WorkerMetadataResponse response) { + if (responseObserver != null) { + responseObserver.onNext(response); + } + } + } + + @SuppressWarnings("UnusedVariable") + private static class TestGetWorkMetadataRequestObserver + implements StreamObserver { + private @Nullable StreamObserver responseObserver; + + @Override + public void onNext(WorkerMetadataRequest workerMetadataRequest) {} + + @Override + public void onError(Throwable throwable) {} + + @Override + public void onCompleted() { + responseObserver.onCompleted(); + } + } + + private static class TestWindmillEndpointsConsumer implements Consumer { + private final Map globalDataEndpoints; + private final Set windmillEndpoints; + + private TestWindmillEndpointsConsumer() { + this.globalDataEndpoints = new HashMap<>(); + this.windmillEndpoints = new HashSet<>(); + } + + @Override + public void accept(WindmillEndpoints windmillEndpoints) { + this.globalDataEndpoints.clear(); + this.windmillEndpoints.clear(); + this.globalDataEndpoints.putAll(windmillEndpoints.globalDataEndpoints()); + this.windmillEndpoints.addAll(windmillEndpoints.windmillEndpoints()); + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/GrpcWindmillServerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcWindmillServerTest.java similarity index 85% rename from runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/GrpcWindmillServerTest.java rename to runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcWindmillServerTest.java index 182c05f68cc31..53afc6990e433 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/GrpcWindmillServerTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/grpcclient/GrpcWindmillServerTest.java @@ -15,15 +15,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.runners.dataflow.worker.windmill; +package org.apache.beam.runners.dataflow.worker.windmill.grpcclient; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.InputStream; import java.io.SequenceInputStream; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -38,16 +41,21 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.beam.runners.dataflow.worker.windmill.CloudWindmillServiceV1Alpha1Grpc.CloudWindmillServiceV1Alpha1ImplBase; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.CommitStatus; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.ComputationGetDataRequest; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.ComputationWorkItemMetadata; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetWorkRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetWorkStreamTimingInfo; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GetWorkStreamTimingInfo.Event; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GlobalData; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GlobalDataId; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.GlobalDataRequest; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.JobHeader; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataRequest; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataResponse; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.LatencyAttribution; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.LatencyAttribution.State; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingCommitRequestChunk; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingCommitResponse; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.StreamingCommitWorkRequest; @@ -59,13 +67,22 @@ import org.apache.beam.runners.dataflow.worker.windmill.Windmill.Value; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.WorkItem; import org.apache.beam.runners.dataflow.worker.windmill.Windmill.WorkItemCommitRequest; -import org.apache.beam.runners.dataflow.worker.windmill.WindmillServerStub.CommitWorkStream; -import org.apache.beam.runners.dataflow.worker.windmill.WindmillServerStub.GetDataStream; -import org.apache.beam.runners.dataflow.worker.windmill.WindmillServerStub.GetWorkStream; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillApplianceGrpc; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.CommitWorkStream; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetDataStream; +import org.apache.beam.runners.dataflow.worker.windmill.WindmillStream.GetWorkStream; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.CallOptions; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Channel; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ClientCall; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ClientInterceptor; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ClientInterceptors; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Deadline; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.MethodDescriptor; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Server; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Status; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.StatusRuntimeException; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessChannelBuilder; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessServerBuilder; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.util.MutableHandlerRegistry; @@ -82,19 +99,22 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** Unit tests for {@link GrpcWindmillServer}. */ +/** + * Unit tests for {@link + * org.apache.beam.runners.dataflow.worker.windmill.grpcclient.GrpcWindmillServer}. + */ @RunWith(JUnit4.class) @SuppressWarnings({ "rawtypes", // TODO(https://github.com/apache/beam/issues/20447) }) public class GrpcWindmillServerTest { - private static final Logger LOG = LoggerFactory.getLogger(GrpcWindmillServerTest.class); + private static final Logger LOG = LoggerFactory.getLogger(GrpcWindmillServerTest.class); + private static final int STREAM_CHUNK_SIZE = 2 << 20; private final MutableHandlerRegistry serviceRegistry = new MutableHandlerRegistry(); @Rule public ErrorCollector errorCollector = new ErrorCollector(); private Server server; - private GrpcWindmillServer client; - private static final int STREAM_CHUNK_SIZE = 2 << 20; + private org.apache.beam.runners.dataflow.worker.windmill.grpcclient.GrpcWindmillServer client; private int remainingErrors = 20; @Before @@ -108,7 +128,7 @@ public void setUp() throws Exception { .build() .start(); - this.client = GrpcWindmillServer.newTestInstance(name, true); + this.client = GrpcWindmillServer.newTestInstance(name); } @After @@ -127,48 +147,64 @@ private void maybeInjectError(Stream stream) { } } - class ResponseErrorInjector { - private Stream stream; - private Thread errorThread; - private boolean cancelled = false; + @Test + public void testWindmillApplianceGrpcDeadlineInterceptor() { + Windmill.GetWorkResponse getWorkResponse = + Windmill.GetWorkResponse.newBuilder() + .addWork( + Windmill.ComputationWorkItems.newBuilder() + .setComputationId("someComputation1") + .build()) + .build(); - public ResponseErrorInjector(Stream stream) { - this.stream = stream; - errorThread = new Thread(this::errorThreadBody); - errorThread.start(); - } + serviceRegistry.addService( + new WindmillApplianceGrpc.WindmillApplianceImplBase() { + @Override + public void getWork( + Windmill.GetWorkRequest request, + StreamObserver responseObserver) { + responseObserver.onNext(getWorkResponse); + responseObserver.onCompleted(); + } + }); - private void errorThreadBody() { - int i = 0; - while (true) { - try { - Thread.sleep(ThreadLocalRandom.current().nextInt(++i * 10)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; - } - synchronized (this) { - if (cancelled) { - break; + ClientInterceptor testInterceptor = + new ClientInterceptor() { + Deadline deadline; + + @Override + public ClientCall interceptCall( + MethodDescriptor methodDescriptor, + CallOptions callOptions, + Channel channel) { + assertNotNull(callOptions.getDeadline()); + if (deadline != null) { + // We want to assert that a new deadline is being created on every call since + // deadlines are absolute. + assertNotEquals(deadline, callOptions.getDeadline()); + } + + deadline = callOptions.getDeadline(); + // Just forward the call once we record the deadline + return channel.newCall(methodDescriptor, callOptions); } - } - maybeInjectError(stream); - } - } + }; - public void cancel() { - LOG.info("Starting cancel of error injector."); - synchronized (this) { - cancelled = true; - } - errorThread.interrupt(); - try { - errorThread.join(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - LOG.info("Done cancelling."); - } + Channel inprocessChannel = + ClientInterceptors.intercept( + InProcessChannelBuilder.forName("Fake server for " + getClass()) + .directExecutor() + .build(), + testInterceptor); + + this.client = GrpcWindmillServer.newApplianceTestInstance(inprocessChannel); + + Windmill.GetWorkResponse response1 = client.getWork(GetWorkRequest.getDefaultInstance()); + Windmill.GetWorkResponse response2 = client.getWork(GetWorkRequest.getDefaultInstance()); + + // Assert that the request/responses are being forwarded correctly. + assertEquals(response1, getWorkResponse); + assertEquals(response2, getWorkResponse); } @Test @@ -181,8 +217,8 @@ public void testStreamingGetWork() throws Exception { public StreamObserver getWorkStream( StreamObserver responseObserver) { return new StreamObserver() { + final ResponseErrorInjector injector = new ResponseErrorInjector(responseObserver); boolean sawHeader = false; - ResponseErrorInjector injector = new ResponseErrorInjector(responseObserver); @Override public void onNext(StreamingGetWorkRequest request) { @@ -268,7 +304,8 @@ public void onCompleted() { (String computation, @Nullable Instant inputDataWatermark, Instant synchronizedProcessingTime, - Windmill.WorkItem workItem) -> { + WorkItem workItem, + Collection getWorkStreamLatencies) -> { latch.countDown(); assertEquals(inputDataWatermark, new Instant(18)); assertEquals(synchronizedProcessingTime, new Instant(17)); @@ -290,11 +327,11 @@ public void testStreamingGetData() throws Exception { public StreamObserver getDataStream( StreamObserver responseObserver) { return new StreamObserver() { - boolean sawHeader = false; - HashSet seenIds = new HashSet<>(); - ResponseErrorInjector injector = new ResponseErrorInjector(responseObserver); - StreamingGetDataResponse.Builder responseBuilder = + final HashSet seenIds = new HashSet<>(); + final ResponseErrorInjector injector = new ResponseErrorInjector(responseObserver); + final StreamingGetDataResponse.Builder responseBuilder = StreamingGetDataResponse.newBuilder(); + boolean sawHeader = false; @Override public void onNext(StreamingGetDataRequest chunk) { @@ -501,10 +538,10 @@ private StreamObserver getTestCommitStreamObserver( StreamObserver responseObserver, Map commitRequests) { return new StreamObserver() { + final ResponseErrorInjector injector = new ResponseErrorInjector(responseObserver); boolean sawHeader = false; InputStream buffer = null; long remainingBytes = 0; - ResponseErrorInjector injector = new ResponseErrorInjector(responseObserver); @Override public void onNext(StreamingCommitWorkRequest request) { @@ -945,7 +982,8 @@ public void onCompleted() { (String computation, @Nullable Instant inputDataWatermark, Instant synchronizedProcessingTime, - Windmill.WorkItem workItem) -> { + Windmill.WorkItem workItem, + Collection getWorkStreamLatencies) -> { latch.countDown(); }); // Wait for 100 items or 30 seconds. @@ -957,4 +995,100 @@ public void onCompleted() { stream.close(); assertTrue(stream.awaitTermination(30, TimeUnit.SECONDS)); } + + @Test + public void testGetWorkTimingInfosTracker() throws Exception { + GetWorkTimingInfosTracker tracker = new GetWorkTimingInfosTracker(() -> 50); + List infos = new ArrayList<>(); + for (int i = 0; i <= 3; i++) { + infos.add( + GetWorkStreamTimingInfo.newBuilder() + .setEvent(Event.GET_WORK_CREATION_START) + .setTimestampUsec(0) + .build()); + infos.add( + GetWorkStreamTimingInfo.newBuilder() + .setEvent(Event.GET_WORK_CREATION_END) + .setTimestampUsec(10000) + .build()); + infos.add( + GetWorkStreamTimingInfo.newBuilder() + .setEvent(Event.GET_WORK_RECEIVED_BY_DISPATCHER) + .setTimestampUsec((i + 11) * 1000) + .build()); + infos.add( + GetWorkStreamTimingInfo.newBuilder() + .setEvent(Event.GET_WORK_FORWARDED_BY_DISPATCHER) + .setTimestampUsec((i + 16) * 1000) + .build()); + tracker.addTimingInfo(infos); + infos.clear(); + } + // durations for each chunk: + // GET_WORK_IN_WINDMILL_WORKER: 10, 10, 10, 10 + // GET_WORK_IN_TRANSIT_TO_DISPATCHER: 1, 2, 3, 4 -> sum to 10 + // GET_WORK_IN_TRANSIT_TO_USER_WORKER: 34, 33, 32, 31 -> sum to 130 + Map latencies = new HashMap<>(); + List attributions = tracker.getLatencyAttributions(); + assertEquals(3, attributions.size()); + for (LatencyAttribution attribution : attributions) { + latencies.put(attribution.getState(), attribution); + } + assertEquals(10L, latencies.get(State.GET_WORK_IN_WINDMILL_WORKER).getTotalDurationMillis()); + // elapsed time from 10 -> 50; + long elapsedTime = 40; + // sumDurations: 1 + 2 + 3 + 4 + 34 + 33 + 32 + 31; + long sumDurations = 140; + assertEquals( + Math.min(4, (long) (elapsedTime * (10.0 / sumDurations))), + latencies.get(State.GET_WORK_IN_TRANSIT_TO_DISPATCHER).getTotalDurationMillis()); + assertEquals( + Math.min(34, (long) (elapsedTime * (130.0 / sumDurations))), + latencies.get(State.GET_WORK_IN_TRANSIT_TO_USER_WORKER).getTotalDurationMillis()); + } + + class ResponseErrorInjector { + + private final Stream stream; + private final Thread errorThread; + private boolean cancelled = false; + + public ResponseErrorInjector(Stream stream) { + this.stream = stream; + errorThread = new Thread(this::errorThreadBody); + errorThread.start(); + } + + private void errorThreadBody() { + int i = 0; + while (true) { + try { + Thread.sleep(ThreadLocalRandom.current().nextInt(++i * 10)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + synchronized (this) { + if (cancelled) { + break; + } + } + maybeInjectError(stream); + } + } + + public void cancel() { + LOG.info("Starting cancel of error injector."); + synchronized (this) { + cancelled = true; + } + errorThread.interrupt(); + try { + errorThread.join(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + LOG.info("Done cancelling."); + } + } } diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WindmillStateCacheTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateCacheTest.java similarity index 75% rename from runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WindmillStateCacheTest.java rename to runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateCacheTest.java index eca431af11a79..cc6633f1b704e 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WindmillStateCacheTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateCacheTest.java @@ -15,17 +15,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.runners.dataflow.worker; +package org.apache.beam.runners.dataflow.worker.windmill.state; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; import java.io.IOException; import java.util.Objects; +import java.util.Optional; import org.apache.beam.runners.core.StateNamespace; import org.apache.beam.runners.core.StateNamespaces; import org.apache.beam.runners.core.StateTag; import org.apache.beam.runners.dataflow.options.DataflowWorkerHarnessOptions; +import org.apache.beam.runners.dataflow.worker.WindmillComputationKey; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.state.State; import org.apache.beam.sdk.state.StateSpec; @@ -38,7 +39,7 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -/** Tests for {@link WindmillStateCache}. */ +/** Tests for {@link org.apache.beam.runners.dataflow.worker.windmill.state.WindmillStateCache}. */ @RunWith(JUnit4.class) public class WindmillStateCacheTest { @@ -153,10 +154,11 @@ public void setUp() { public void testBasic() throws Exception { WindmillStateCache.ForKeyAndFamily keyCache = cache.forComputation(COMPUTATION).forKey(COMPUTATION_KEY, 0L, 1L).forFamily(STATE_FAMILY); - assertNull(keyCache.get(StateNamespaces.global(), new TestStateTag("tag1"))); - assertNull(keyCache.get(windowNamespace(0), new TestStateTag("tag2"))); - assertNull(keyCache.get(triggerNamespace(0, 0), new TestStateTag("tag3"))); - assertNull(keyCache.get(triggerNamespace(0, 0), new TestStateTag("tag2"))); + assertEquals( + Optional.empty(), keyCache.get(StateNamespaces.global(), new TestStateTag("tag1"))); + assertEquals(Optional.empty(), keyCache.get(windowNamespace(0), new TestStateTag("tag2"))); + assertEquals(Optional.empty(), keyCache.get(triggerNamespace(0, 0), new TestStateTag("tag3"))); + assertEquals(Optional.empty(), keyCache.get(triggerNamespace(0, 0), new TestStateTag("tag2"))); assertEquals(0, cache.getWeight()); keyCache.put(StateNamespaces.global(), new TestStateTag("tag1"), new TestState("g1"), 2); @@ -177,12 +179,17 @@ public void testBasic() throws Exception { keyCache = cache.forComputation(COMPUTATION).forKey(COMPUTATION_KEY, 0L, 2L).forFamily(STATE_FAMILY); assertEquals( - new TestState("g1"), keyCache.get(StateNamespaces.global(), new TestStateTag("tag1"))); - assertEquals(new TestState("w2"), keyCache.get(windowNamespace(0), new TestStateTag("tag2"))); + Optional.of(new TestState("g1")), + keyCache.get(StateNamespaces.global(), new TestStateTag("tag1"))); + assertEquals( + Optional.of(new TestState("w2")), + keyCache.get(windowNamespace(0), new TestStateTag("tag2"))); assertEquals( - new TestState("t3"), keyCache.get(triggerNamespace(0, 0), new TestStateTag("tag3"))); + Optional.of(new TestState("t3")), + keyCache.get(triggerNamespace(0, 0), new TestStateTag("tag3"))); assertEquals( - new TestState("t2"), keyCache.get(triggerNamespace(0, 0), new TestStateTag("tag2"))); + Optional.of(new TestState("t2")), + keyCache.get(triggerNamespace(0, 0), new TestStateTag("tag2"))); } /** Verifies that max weight is set */ @@ -196,7 +203,8 @@ public void testMaxWeight() throws Exception { public void testInvalidation() throws Exception { WindmillStateCache.ForKeyAndFamily keyCache = cache.forComputation(COMPUTATION).forKey(COMPUTATION_KEY, 0L, 1L).forFamily(STATE_FAMILY); - assertNull(keyCache.get(StateNamespaces.global(), new TestStateTag("tag1"))); + assertEquals( + Optional.empty(), keyCache.get(StateNamespaces.global(), new TestStateTag("tag1"))); keyCache.put(StateNamespaces.global(), new TestStateTag("tag1"), new TestState("g1"), 2); keyCache.persist(); @@ -204,11 +212,13 @@ public void testInvalidation() throws Exception { cache.forComputation(COMPUTATION).forKey(COMPUTATION_KEY, 0L, 2L).forFamily(STATE_FAMILY); assertEquals(127, cache.getWeight()); assertEquals( - new TestState("g1"), keyCache.get(StateNamespaces.global(), new TestStateTag("tag1"))); + Optional.of(new TestState("g1")), + keyCache.get(StateNamespaces.global(), new TestStateTag("tag1"))); keyCache = cache.forComputation(COMPUTATION).forKey(COMPUTATION_KEY, 1L, 3L).forFamily(STATE_FAMILY); - assertNull(keyCache.get(StateNamespaces.global(), new TestStateTag("tag1"))); + assertEquals( + Optional.empty(), keyCache.get(StateNamespaces.global(), new TestStateTag("tag1"))); assertEquals(127, cache.getWeight()); } @@ -225,8 +235,8 @@ public void testEviction() throws Exception { // Eviction is atomic across the whole window. keyCache = cache.forComputation(COMPUTATION).forKey(COMPUTATION_KEY, 0L, 2L).forFamily(STATE_FAMILY); - assertNull(keyCache.get(windowNamespace(0), new TestStateTag("tag2"))); - assertNull(keyCache.get(triggerNamespace(0, 0), new TestStateTag("tag3"))); + assertEquals(Optional.empty(), keyCache.get(windowNamespace(0), new TestStateTag("tag2"))); + assertEquals(Optional.empty(), keyCache.get(triggerNamespace(0, 0), new TestStateTag("tag3"))); } /** Verifies that the cache does not vend for stale work tokens. */ @@ -239,35 +249,35 @@ public void testStaleWorkItem() throws Exception { keyCache.put(windowNamespace(0), tag, new TestState("w2"), 2); // Same cache. - assertEquals(new TestState("w2"), keyCache.get(windowNamespace(0), tag)); + assertEquals(Optional.of(new TestState("w2")), keyCache.get(windowNamespace(0), tag)); assertEquals(0, cache.getWeight()); keyCache.persist(); assertEquals(127, cache.getWeight()); - assertEquals(new TestState("w2"), keyCache.get(windowNamespace(0), tag)); + assertEquals(Optional.of(new TestState("w2")), keyCache.get(windowNamespace(0), tag)); // Previous work token. keyCache = cache.forComputation(COMPUTATION).forKey(COMPUTATION_KEY, 0L, 1L).forFamily(STATE_FAMILY); - assertNull(keyCache.get(windowNamespace(0), tag)); + assertEquals(Optional.empty(), keyCache.get(windowNamespace(0), tag)); // Retry of work token that inserted. keyCache = cache.forComputation(COMPUTATION).forKey(COMPUTATION_KEY, 0L, 2L).forFamily(STATE_FAMILY); - assertNull(keyCache.get(windowNamespace(0), tag)); + assertEquals(Optional.empty(), keyCache.get(windowNamespace(0), tag)); keyCache = cache.forComputation(COMPUTATION).forKey(COMPUTATION_KEY, 0L, 10L).forFamily(STATE_FAMILY); - assertNull(keyCache.get(windowNamespace(0), tag)); + assertEquals(Optional.empty(), keyCache.get(windowNamespace(0), tag)); keyCache.put(windowNamespace(0), tag, new TestState("w3"), 2); // Ensure that second put updated work token. keyCache = cache.forComputation(COMPUTATION).forKey(COMPUTATION_KEY, 0L, 5L).forFamily(STATE_FAMILY); - assertNull(keyCache.get(windowNamespace(0), tag)); + assertEquals(Optional.empty(), keyCache.get(windowNamespace(0), tag)); keyCache = cache.forComputation(COMPUTATION).forKey(COMPUTATION_KEY, 0L, 15L).forFamily(STATE_FAMILY); - assertNull(keyCache.get(windowNamespace(0), tag)); + assertEquals(Optional.empty(), keyCache.get(windowNamespace(0), tag)); } /** Verifies that caches are kept independently per-key. */ @@ -293,7 +303,7 @@ public void testMultipleKeys() throws Exception { TestState state1 = new TestState("g1"); keyCache1.put(StateNamespaces.global(), tag, state1, 2); - assertEquals(state1, keyCache1.get(StateNamespaces.global(), tag)); + assertEquals(Optional.of(state1), keyCache1.get(StateNamespaces.global(), tag)); keyCache1.persist(); keyCache1 = @@ -301,22 +311,22 @@ public void testMultipleKeys() throws Exception { .forComputation("comp1") .forKey(computationKey("comp1", "key1", SHARDING_KEY), 0L, 1L) .forFamily(STATE_FAMILY); - assertEquals(state1, keyCache1.get(StateNamespaces.global(), tag)); - assertNull(keyCache2.get(StateNamespaces.global(), tag)); - assertNull(keyCache3.get(StateNamespaces.global(), tag)); + assertEquals(Optional.of(state1), keyCache1.get(StateNamespaces.global(), tag)); + assertEquals(Optional.empty(), keyCache2.get(StateNamespaces.global(), tag)); + assertEquals(Optional.empty(), keyCache3.get(StateNamespaces.global(), tag)); TestState state2 = new TestState("g2"); keyCache2.put(StateNamespaces.global(), tag, state2, 2); keyCache2.persist(); - assertEquals(state2, keyCache2.get(StateNamespaces.global(), tag)); + assertEquals(Optional.of(state2), keyCache2.get(StateNamespaces.global(), tag)); keyCache2 = cache .forComputation("comp1") .forKey(computationKey("comp1", "key2", SHARDING_KEY), 0L, 20L) .forFamily(STATE_FAMILY); - assertEquals(state2, keyCache2.get(StateNamespaces.global(), tag)); - assertEquals(state1, keyCache1.get(StateNamespaces.global(), tag)); - assertNull(keyCache3.get(StateNamespaces.global(), tag)); + assertEquals(Optional.of(state2), keyCache2.get(StateNamespaces.global(), tag)); + assertEquals(Optional.of(state1), keyCache1.get(StateNamespaces.global(), tag)); + assertEquals(Optional.empty(), keyCache3.get(StateNamespaces.global(), tag)); } /** Verifies that caches are kept independently per shard of key. */ @@ -343,28 +353,28 @@ public void testMultipleShardsOfKey() throws Exception { TestState state1 = new TestState("g1"); key1CacheShard1.put(StateNamespaces.global(), tag, state1, 2); key1CacheShard1.persist(); - assertEquals(state1, key1CacheShard1.get(StateNamespaces.global(), tag)); + assertEquals(Optional.of(state1), key1CacheShard1.get(StateNamespaces.global(), tag)); key1CacheShard1 = cache .forComputation(COMPUTATION) .forKey(computationKey(COMPUTATION, "key1", 1), 0L, 1L) .forFamily(STATE_FAMILY); - assertEquals(state1, key1CacheShard1.get(StateNamespaces.global(), tag)); - assertNull(key1CacheShard2.get(StateNamespaces.global(), tag)); - assertNull(key2CacheShard1.get(StateNamespaces.global(), tag)); + assertEquals(Optional.of(state1), key1CacheShard1.get(StateNamespaces.global(), tag)); + assertEquals(Optional.empty(), key1CacheShard2.get(StateNamespaces.global(), tag)); + assertEquals(Optional.empty(), key2CacheShard1.get(StateNamespaces.global(), tag)); TestState state2 = new TestState("g2"); key1CacheShard2.put(StateNamespaces.global(), tag, state2, 2); - assertEquals(state2, key1CacheShard2.get(StateNamespaces.global(), tag)); + assertEquals(Optional.of(state2), key1CacheShard2.get(StateNamespaces.global(), tag)); key1CacheShard2.persist(); key1CacheShard2 = cache .forComputation(COMPUTATION) .forKey(computationKey(COMPUTATION, "key1", 2), 0L, 20L) .forFamily(STATE_FAMILY); - assertEquals(state2, key1CacheShard2.get(StateNamespaces.global(), tag)); - assertEquals(state1, key1CacheShard1.get(StateNamespaces.global(), tag)); - assertNull(key2CacheShard1.get(StateNamespaces.global(), tag)); + assertEquals(Optional.of(state2), key1CacheShard2.get(StateNamespaces.global(), tag)); + assertEquals(Optional.of(state1), key1CacheShard1.get(StateNamespaces.global(), tag)); + assertEquals(Optional.empty(), key2CacheShard1.get(StateNamespaces.global(), tag)); } /** Verifies that caches are kept independently per-family. */ @@ -379,22 +389,22 @@ public void testMultipleFamilies() throws Exception { TestState state1 = new TestState("g1"); family1.put(StateNamespaces.global(), tag, state1, 2); - assertEquals(state1, family1.get(StateNamespaces.global(), tag)); + assertEquals(Optional.of(state1), family1.get(StateNamespaces.global(), tag)); family1.persist(); TestState state2 = new TestState("g2"); family2.put(StateNamespaces.global(), tag, state2, 2); family2.persist(); - assertEquals(state2, family2.get(StateNamespaces.global(), tag)); + assertEquals(Optional.of(state2), family2.get(StateNamespaces.global(), tag)); keyCache = cache.forComputation("comp1").forKey(computationKey("comp1", "key1", SHARDING_KEY), 0L, 1L); family1 = keyCache.forFamily("family1"); family2 = keyCache.forFamily("family2"); WindmillStateCache.ForKeyAndFamily family3 = keyCache.forFamily("family3"); - assertEquals(state1, family1.get(StateNamespaces.global(), tag)); - assertEquals(state2, family2.get(StateNamespaces.global(), tag)); - assertNull(family3.get(StateNamespaces.global(), tag)); + assertEquals(Optional.of(state1), family1.get(StateNamespaces.global(), tag)); + assertEquals(Optional.of(state2), family2.get(StateNamespaces.global(), tag)); + assertEquals(Optional.empty(), family3.get(StateNamespaces.global(), tag)); } /** Verifies explicit invalidation does indeed invalidate the correct entries. */ @@ -450,13 +460,17 @@ public void testExplicitInvalidation() throws Exception { .forKey(computationKey("comp1", "key1", 2), 0L, 1L) .forFamily(STATE_FAMILY); assertEquals( - new TestState("g1"), keyCache1.get(StateNamespaces.global(), new TestStateTag("tag1"))); + Optional.of(new TestState("g1")), + keyCache1.get(StateNamespaces.global(), new TestStateTag("tag1"))); assertEquals( - new TestState("g2"), keyCache2.get(StateNamespaces.global(), new TestStateTag("tag2"))); + Optional.of(new TestState("g2")), + keyCache2.get(StateNamespaces.global(), new TestStateTag("tag2"))); assertEquals( - new TestState("g3"), keyCache3.get(StateNamespaces.global(), new TestStateTag("tag3"))); + Optional.of(new TestState("g3")), + keyCache3.get(StateNamespaces.global(), new TestStateTag("tag3"))); assertEquals( - new TestState("g4"), keyCache4.get(StateNamespaces.global(), new TestStateTag("tag4"))); + Optional.of(new TestState("g4")), + keyCache4.get(StateNamespaces.global(), new TestStateTag("tag4"))); // Invalidation of key 1 shard 1 does not affect another shard of key 1 or other keys. cache.forComputation("comp1").invalidate(ByteString.copyFromUtf8("key1"), 1); @@ -466,23 +480,30 @@ public void testExplicitInvalidation() throws Exception { .forKey(computationKey("comp1", "key1", 1), 0L, 2L) .forFamily(STATE_FAMILY); - assertNull(keyCache1.get(StateNamespaces.global(), new TestStateTag("tag1"))); assertEquals( - new TestState("g2"), keyCache2.get(StateNamespaces.global(), new TestStateTag("tag2"))); + Optional.empty(), keyCache1.get(StateNamespaces.global(), new TestStateTag("tag1"))); + assertEquals( + Optional.of(new TestState("g2")), + keyCache2.get(StateNamespaces.global(), new TestStateTag("tag2"))); assertEquals( - new TestState("g3"), keyCache3.get(StateNamespaces.global(), new TestStateTag("tag3"))); + Optional.of(new TestState("g3")), + keyCache3.get(StateNamespaces.global(), new TestStateTag("tag3"))); assertEquals( - new TestState("g4"), keyCache4.get(StateNamespaces.global(), new TestStateTag("tag4"))); + Optional.of(new TestState("g4")), + keyCache4.get(StateNamespaces.global(), new TestStateTag("tag4"))); // Invalidation of an non-existing key affects nothing. cache.forComputation("comp1").invalidate(ByteString.copyFromUtf8("key1"), 3); assertEquals( - new TestState("g2"), keyCache2.get(StateNamespaces.global(), new TestStateTag("tag2"))); + Optional.of(new TestState("g2")), + keyCache2.get(StateNamespaces.global(), new TestStateTag("tag2"))); assertEquals( - new TestState("g3"), keyCache3.get(StateNamespaces.global(), new TestStateTag("tag3"))); + Optional.of(new TestState("g3")), + keyCache3.get(StateNamespaces.global(), new TestStateTag("tag3"))); assertEquals( - new TestState("g4"), keyCache4.get(StateNamespaces.global(), new TestStateTag("tag4"))); + Optional.of(new TestState("g4")), + keyCache4.get(StateNamespaces.global(), new TestStateTag("tag4"))); } private static class TestStateTagWithBadEquality extends TestStateTag { @@ -517,9 +538,9 @@ public void testBadCoderEquality() throws Exception { keyCache1 = cache.forComputation(COMPUTATION).forKey(COMPUTATION_KEY, 0L, 1L).forFamily(STATE_FAMILY); - assertEquals(new TestState("g1"), keyCache1.get(StateNamespaces.global(), tag)); + assertEquals(Optional.of(new TestState("g1")), keyCache1.get(StateNamespaces.global(), tag)); assertEquals( - new TestState("g1"), + Optional.of(new TestState("g1")), keyCache1.get(StateNamespaces.global(), new TestStateTagWithBadEquality("tag1"))); } } diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateInternalsTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateInternalsTest.java new file mode 100644 index 0000000000000..8971c39ccaa1f --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateInternalsTest.java @@ -0,0 +1,3142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import static org.apache.beam.runners.dataflow.worker.DataflowMatchers.ByteStringMatcher.byteStringEq; +import static org.apache.beam.sdk.testing.SystemNanoTimeSleeper.sleepMillis; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.AbstractMap; +import java.util.AbstractMap.SimpleEntry; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.apache.beam.runners.core.StateNamespace; +import org.apache.beam.runners.core.StateNamespaceForTest; +import org.apache.beam.runners.core.StateTag; +import org.apache.beam.runners.core.StateTags; +import org.apache.beam.runners.dataflow.options.DataflowWorkerHarnessOptions; +import org.apache.beam.runners.dataflow.worker.WindmillComputationKey; +import org.apache.beam.runners.dataflow.worker.WindmillStateTestUtils; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.TagBag; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.TagSortedListUpdateRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.TagValue; +import org.apache.beam.sdk.coders.ByteArrayCoder; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.coders.Coder.Context; +import org.apache.beam.sdk.coders.StringUtf8Coder; +import org.apache.beam.sdk.coders.VarIntCoder; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.state.BagState; +import org.apache.beam.sdk.state.CombiningState; +import org.apache.beam.sdk.state.GroupingState; +import org.apache.beam.sdk.state.MapState; +import org.apache.beam.sdk.state.MultimapState; +import org.apache.beam.sdk.state.OrderedListState; +import org.apache.beam.sdk.state.ReadableState; +import org.apache.beam.sdk.state.ValueState; +import org.apache.beam.sdk.state.WatermarkHoldState; +import org.apache.beam.sdk.transforms.Sum; +import org.apache.beam.sdk.transforms.windowing.TimestampCombiner; +import org.apache.beam.sdk.util.ByteStringOutputStream; +import org.apache.beam.sdk.util.CoderUtils; +import org.apache.beam.sdk.values.TimestampedValue; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Range; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.RangeSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Futures; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.SettableFuture; +import org.hamcrest.Matchers; +import org.hamcrest.core.CombinableMatcher; +import org.joda.time.Instant; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +/** Tests for {@link WindmillStateInternals}. */ +@RunWith(JUnit4.class) +@SuppressWarnings({ + "rawtypes", // TODO(https://github.com/apache/beam/issues/20447) +}) +public class WindmillStateInternalsTest { + + public static final Range FULL_ORDERED_LIST_RANGE = + Range.closedOpen(WindmillOrderedList.MIN_TS_MICROS, WindmillOrderedList.MAX_TS_MICROS); + private static final StateNamespace NAMESPACE = new StateNamespaceForTest("ns"); + private static final String STATE_FAMILY = "family"; + private static final StateTag> COMBINING_ADDR = + StateTags.combiningValueFromInputInternal("combining", VarIntCoder.of(), Sum.ofIntegers()); + private static final ByteString COMBINING_KEY = key(NAMESPACE, "combining"); + private final Coder accumCoder = + Sum.ofIntegers().getAccumulatorCoder(null, VarIntCoder.of()); + DataflowWorkerHarnessOptions options; + private long workToken = 0; + @Mock private WindmillStateReader mockReader; + private WindmillStateInternals underTest; + private WindmillStateInternals underTestNewKey; + private WindmillStateCache cache; + @Mock private Supplier readStateSupplier; + + private static ByteString key(StateNamespace namespace, String addrId) { + return ByteString.copyFromUtf8(namespace.stringKey() + "+u" + addrId); + } + + private static ByteString systemKey(StateNamespace namespace, String addrId) { + return ByteString.copyFromUtf8(namespace.stringKey() + "+s" + addrId); + } + + private static ByteString encodeWithCoder(T key, Coder coder) { + ByteStringOutputStream out = new ByteStringOutputStream(); + try { + coder.encode(key, out, Context.OUTER); + } catch (IOException e) { + throw new RuntimeException(e); + } + return out.toByteString(); + } + + // We use the structural value of the Multimap keys to differentiate between different keys. So we + // mix using the original key object and a duplicate but same key object so make sure the + // correctness. + private static byte[] dup(byte[] key) { + byte[] res = new byte[key.length]; + System.arraycopy(key, 0, res, 0, key.length); + return res; + } + + private static Map.Entry> multimapEntry( + byte[] key, Integer... values) { + return new AbstractMap.SimpleEntry<>( + encodeWithCoder(key, ByteArrayCoder.of()), Arrays.asList(values)); + } + + @SafeVarargs + private static List weightedList(T... entries) { + WeightedList list = new WeightedList<>(new ArrayList<>()); + for (T entry : entries) { + list.addWeighted(entry, 1); + } + return list; + } + + private static CombinableMatcher multimapEntryMatcher(byte[] key, Integer value) { + return Matchers.both(Matchers.hasProperty("key", Matchers.equalTo(key))) + .and(Matchers.hasProperty("value", Matchers.equalTo(value))); + } + + private static MultimapEntryUpdate decodeTagMultimapEntry(Windmill.TagMultimapEntry entryProto) { + try { + String key = StringUtf8Coder.of().decode(entryProto.getEntryName().newInput(), Context.OUTER); + List values = new ArrayList<>(); + for (ByteString value : entryProto.getValuesList()) { + values.add(VarIntCoder.of().decode(value.newInput(), Context.OUTER)); + } + return new MultimapEntryUpdate(key, values, entryProto.getDeleteAll()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static void assertTagMultimapUpdates( + Windmill.TagMultimapUpdateRequest.Builder updates, MultimapEntryUpdate... expected) { + assertThat( + updates.getUpdatesList().stream() + .map(WindmillStateInternalsTest::decodeTagMultimapEntry) + .collect(Collectors.toList()), + Matchers.containsInAnyOrder(expected)); + } + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + options = PipelineOptionsFactory.as(DataflowWorkerHarnessOptions.class); + cache = new WindmillStateCache(options.getWorkerCacheMb()); + resetUnderTest(); + } + + public void resetUnderTest() { + workToken++; + underTest = + new WindmillStateInternals<>( + "dummyKey", + STATE_FAMILY, + mockReader, + false, + cache + .forComputation("comp") + .forKey( + WindmillComputationKey.create( + "comp", ByteString.copyFrom("dummyKey", Charsets.UTF_8), 123), + 17L, + workToken) + .forFamily(STATE_FAMILY), + readStateSupplier); + underTestNewKey = + new WindmillStateInternals( + "dummyNewKey", + STATE_FAMILY, + mockReader, + true, + cache + .forComputation("comp") + .forKey( + WindmillComputationKey.create( + "comp", ByteString.copyFrom("dummyNewKey", Charsets.UTF_8), 123), + 17L, + workToken) + .forFamily(STATE_FAMILY), + readStateSupplier); + } + + @After + public void tearDown() throws Exception { + // Make sure no WindmillStateReader (a per-WorkItem object) escapes into the cache + // (a global object). + WindmillStateTestUtils.assertNoReference(cache, WindmillStateReader.class); + } + + private void waitAndSet(final SettableFuture future, final T value, final long millis) { + new Thread( + () -> { + try { + sleepMillis(millis); + } catch (InterruptedException e) { + throw new RuntimeException("Interrupted before setting", e); + } + future.set(value); + }) + .run(); + } + + private WeightedList weightedList(String... elems) { + WeightedList result = new WeightedList<>(new ArrayList(elems.length)); + for (String elem : elems) { + result.addWeighted(elem, elem.length()); + } + return result; + } + + private ByteString protoKeyFromUserKey(@Nullable K tag, Coder keyCoder) + throws IOException { + ByteStringOutputStream keyStream = new ByteStringOutputStream(); + key(NAMESPACE, "map").writeTo(keyStream); + if (tag != null) { + keyCoder.encode(tag, keyStream, Context.OUTER); + } + return keyStream.toByteString(); + } + + private K userKeyFromProtoKey(ByteString tag, Coder keyCoder) throws IOException { + ByteString keyBytes = tag.substring(key(NAMESPACE, "map").size()); + return keyCoder.decode(keyBytes.newInput(), Context.OUTER); + } + + @Test + public void testMapAddBeforeGet() throws Exception { + StateTag> addr = + StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); + MapState mapState = underTest.state(NAMESPACE, addr); + + final String tag = "tag"; + SettableFuture future = SettableFuture.create(); + when(mockReader.valueFuture( + protoKeyFromUserKey(tag, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(future); + + ReadableState result = mapState.get("tag"); + result = result.readLater(); + waitAndSet(future, 1, 200); + assertEquals(1, (int) result.read()); + mapState.put("tag", 2); + assertEquals(2, (int) result.read()); + } + + @Test + public void testMapAddClearBeforeGet() throws Exception { + StateTag> addr = + StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); + MapState mapState = underTest.state(NAMESPACE, addr); + + final String tag = "tag"; + + SettableFuture>> prefixFuture = SettableFuture.create(); + when(mockReader.valuePrefixFuture( + protoKeyFromUserKey(null, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(prefixFuture); + + ReadableState result = mapState.get("tag"); + result = result.readLater(); + waitAndSet( + prefixFuture, + ImmutableList.of( + new AbstractMap.SimpleEntry<>(protoKeyFromUserKey(tag, StringUtf8Coder.of()), 1)), + 50); + assertFalse(mapState.isEmpty().read()); + mapState.clear(); + assertTrue(mapState.isEmpty().read()); + assertNull(mapState.get("tag").read()); + mapState.put("tag", 2); + assertFalse(mapState.isEmpty().read()); + assertEquals(2, (int) result.read()); + } + + @Test + public void testMapLocalAddOverridesStorage() throws Exception { + StateTag> addr = + StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); + MapState mapState = underTest.state(NAMESPACE, addr); + + final String tag = "tag"; + + SettableFuture future = SettableFuture.create(); + when(mockReader.valueFuture( + protoKeyFromUserKey(tag, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(future); + SettableFuture>> prefixFuture = SettableFuture.create(); + when(mockReader.valuePrefixFuture( + protoKeyFromUserKey(null, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(prefixFuture); + + waitAndSet(future, 1, 50); + waitAndSet( + prefixFuture, + ImmutableList.of( + new AbstractMap.SimpleEntry<>(protoKeyFromUserKey(tag, StringUtf8Coder.of()), 1)), + 50); + mapState.put(tag, 42); + assertEquals(42, (int) mapState.get(tag).read()); + assertThat( + mapState.entries().read(), + Matchers.containsInAnyOrder(new AbstractMap.SimpleEntry<>(tag, 42))); + } + + @Test + public void testMapLocalRemoveOverridesStorage() throws Exception { + StateTag> addr = + StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); + MapState mapState = underTest.state(NAMESPACE, addr); + + final String tag1 = "tag1"; + final String tag2 = "tag2"; + + SettableFuture future = SettableFuture.create(); + when(mockReader.valueFuture( + protoKeyFromUserKey(tag1, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(future); + SettableFuture>> prefixFuture = SettableFuture.create(); + when(mockReader.valuePrefixFuture( + protoKeyFromUserKey(null, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(prefixFuture); + + waitAndSet(future, 1, 50); + waitAndSet( + prefixFuture, + ImmutableList.of( + new AbstractMap.SimpleEntry<>(protoKeyFromUserKey(tag1, StringUtf8Coder.of()), 1), + new AbstractMap.SimpleEntry<>(protoKeyFromUserKey(tag2, StringUtf8Coder.of()), 2)), + 50); + mapState.remove(tag1); + assertNull(mapState.get(tag1).read()); + assertThat( + mapState.entries().read(), + Matchers.containsInAnyOrder(new AbstractMap.SimpleEntry<>(tag2, 2))); + + mapState.remove(tag2); + assertTrue(mapState.isEmpty().read()); + } + + @Test + public void testMapLocalClearOverridesStorage() throws Exception { + StateTag> addr = + StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); + MapState mapState = underTest.state(NAMESPACE, addr); + + final String tag1 = "tag1"; + final String tag2 = "tag2"; + + SettableFuture future1 = SettableFuture.create(); + SettableFuture future2 = SettableFuture.create(); + + when(mockReader.valueFuture( + protoKeyFromUserKey(tag1, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(future1); + when(mockReader.valueFuture( + protoKeyFromUserKey(tag2, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(future2); + SettableFuture>> prefixFuture = SettableFuture.create(); + when(mockReader.valuePrefixFuture( + protoKeyFromUserKey(null, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(prefixFuture); + + waitAndSet(future1, 1, 50); + waitAndSet(future2, 2, 50); + waitAndSet( + prefixFuture, + ImmutableList.of( + new AbstractMap.SimpleEntry<>(protoKeyFromUserKey(tag1, StringUtf8Coder.of()), 1), + new AbstractMap.SimpleEntry<>(protoKeyFromUserKey(tag2, StringUtf8Coder.of()), 2)), + 50); + mapState.clear(); + assertNull(mapState.get(tag1).read()); + assertNull(mapState.get(tag2).read()); + assertThat(mapState.entries().read(), Matchers.emptyIterable()); + assertTrue(mapState.isEmpty().read()); + } + + @Test + public void testMapAddBeforeRead() throws Exception { + StateTag> addr = + StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); + MapState mapState = underTest.state(NAMESPACE, addr); + + final String tag1 = "tag1"; + final String tag2 = "tag2"; + final String tag3 = "tag3"; + SettableFuture>> prefixFuture = SettableFuture.create(); + when(mockReader.valuePrefixFuture( + protoKeyFromUserKey(null, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(prefixFuture); + + ReadableState>> result = mapState.entries(); + result = result.readLater(); + + mapState.put(tag1, 1); + waitAndSet( + prefixFuture, + ImmutableList.of( + new AbstractMap.SimpleEntry<>(protoKeyFromUserKey(tag2, StringUtf8Coder.of()), 2)), + 200); + Iterable> readData = result.read(); + assertThat( + readData, + Matchers.containsInAnyOrder( + new AbstractMap.SimpleEntry<>(tag1, 1), new AbstractMap.SimpleEntry<>(tag2, 2))); + + mapState.put(tag3, 3); + assertThat( + result.read(), + Matchers.containsInAnyOrder( + new AbstractMap.SimpleEntry<>(tag1, 1), + new AbstractMap.SimpleEntry<>(tag2, 2), + new AbstractMap.SimpleEntry<>(tag3, 3))); + } + + @Test + public void testMapPutIfAbsentSucceeds() throws Exception { + StateTag> addr = + StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); + MapState mapState = underTest.state(NAMESPACE, addr); + + final String tag1 = "tag1"; + SettableFuture future = SettableFuture.create(); + when(mockReader.valueFuture( + protoKeyFromUserKey(tag1, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(future); + waitAndSet(future, null, 50); + + assertNull(mapState.putIfAbsent(tag1, 42).read()); + assertEquals(42, (int) mapState.get(tag1).read()); + } + + @Test + public void testMapPutIfAbsentFails() throws Exception { + StateTag> addr = + StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); + MapState mapState = underTest.state(NAMESPACE, addr); + + final String tag1 = "tag1"; + mapState.put(tag1, 1); + assertEquals(1, (int) mapState.putIfAbsent(tag1, 42).read()); + assertEquals(1, (int) mapState.get(tag1).read()); + + final String tag2 = "tag2"; + SettableFuture future = SettableFuture.create(); + when(mockReader.valueFuture( + protoKeyFromUserKey(tag2, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(future); + waitAndSet(future, 2, 50); + assertEquals(2, (int) mapState.putIfAbsent(tag2, 42).read()); + assertEquals(2, (int) mapState.get(tag2).read()); + } + + @Test + public void testMapPutIfAbsentNoReadSucceeds() throws Exception { + StateTag> addr = + StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); + MapState mapState = underTest.state(NAMESPACE, addr); + + final String tag1 = "tag1"; + SettableFuture future = SettableFuture.create(); + when(mockReader.valueFuture( + protoKeyFromUserKey(tag1, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(future); + waitAndSet(future, null, 50); + ReadableState readableState = mapState.putIfAbsent(tag1, 42); + assertEquals(42, (int) mapState.get(tag1).read()); + assertNull(readableState.read()); + } + + @Test + public void testMapPutIfAbsentNoReadFails() throws Exception { + StateTag> addr = + StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); + MapState mapState = underTest.state(NAMESPACE, addr); + + final String tag1 = "tag1"; + mapState.put(tag1, 1); + ReadableState readableState = mapState.putIfAbsent(tag1, 42); + assertEquals(1, (int) mapState.get(tag1).read()); + assertEquals(1, (int) readableState.read()); + + final String tag2 = "tag2"; + SettableFuture future = SettableFuture.create(); + when(mockReader.valueFuture( + protoKeyFromUserKey(tag2, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(future); + waitAndSet(future, 2, 50); + readableState = mapState.putIfAbsent(tag2, 42); + assertEquals(2, (int) mapState.get(tag2).read()); + assertEquals(2, (int) readableState.read()); + } + + @Test + public void testMapMultiplePutIfAbsentNoRead() throws Exception { + StateTag> addr = + StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); + MapState mapState = underTest.state(NAMESPACE, addr); + + final String tag1 = "tag1"; + SettableFuture future = SettableFuture.create(); + when(mockReader.valueFuture( + protoKeyFromUserKey(tag1, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(future); + waitAndSet(future, null, 50); + ReadableState readableState = mapState.putIfAbsent(tag1, 42); + assertEquals(42, (int) mapState.get(tag1).read()); + ReadableState readableState2 = mapState.putIfAbsent(tag1, 43); + mapState.put(tag1, 1); + ReadableState readableState3 = mapState.putIfAbsent(tag1, 44); + assertEquals(1, (int) mapState.get(tag1).read()); + assertNull(readableState.read()); + assertEquals(42, (int) readableState2.read()); + assertEquals(1, (int) readableState3.read()); + } + + @Test + public void testMapNegativeCache() throws Exception { + StateTag> addr = + StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); + MapState mapState = underTest.state(NAMESPACE, addr); + + final String tag = "tag"; + SettableFuture future = SettableFuture.create(); + when(mockReader.valueFuture( + protoKeyFromUserKey(tag, StringUtf8Coder.of()), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(future); + waitAndSet(future, null, 200); + assertNull(mapState.get(tag).read()); + future.set(42); + assertNull(mapState.get(tag).read()); + } + + private Map.Entry fromTagValue( + TagValue tagValue, Coder keyCoder, Coder valueCoder) { + try { + V value = + !tagValue.getValue().getData().isEmpty() + ? valueCoder.decode(tagValue.getValue().getData().newInput()) + : null; + return new AbstractMap.SimpleEntry<>(userKeyFromProtoKey(tagValue.getTag(), keyCoder), value); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Test + public void testMapAddPersist() throws Exception { + StateTag> addr = + StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); + MapState mapState = underTest.state(NAMESPACE, addr); + + final String tag1 = "tag1"; + final String tag2 = "tag2"; + mapState.put(tag1, 1); + mapState.put(tag2, 2); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(2, commitBuilder.getValueUpdatesCount()); + assertThat( + commitBuilder.getValueUpdatesList().stream() + .map(tv -> fromTagValue(tv, StringUtf8Coder.of(), VarIntCoder.of())) + .collect(Collectors.toList()), + Matchers.containsInAnyOrder(new SimpleEntry<>(tag1, 1), new SimpleEntry<>(tag2, 2))); + } + + @Test + public void testMapRemovePersist() throws Exception { + StateTag> addr = + StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); + MapState mapState = underTest.state(NAMESPACE, addr); + + final String tag1 = "tag1"; + final String tag2 = "tag2"; + mapState.remove(tag1); + mapState.remove(tag2); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(2, commitBuilder.getValueUpdatesCount()); + assertThat( + commitBuilder.getValueUpdatesList().stream() + .map(tv -> fromTagValue(tv, StringUtf8Coder.of(), VarIntCoder.of())) + .collect(Collectors.toList()), + Matchers.containsInAnyOrder(new SimpleEntry<>(tag1, null), new SimpleEntry<>(tag2, null))); + } + + @Test + public void testMapClearPersist() throws Exception { + StateTag> addr = + StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); + MapState mapState = underTest.state(NAMESPACE, addr); + + final String tag1 = "tag1"; + final String tag2 = "tag2"; + mapState.put(tag1, 1); + mapState.put(tag2, 2); + mapState.clear(); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(0, commitBuilder.getValueUpdatesCount()); + assertEquals(1, commitBuilder.getTagValuePrefixDeletesCount()); + System.err.println(commitBuilder); + assertEquals(STATE_FAMILY, commitBuilder.getTagValuePrefixDeletes(0).getStateFamily()); + assertEquals( + protoKeyFromUserKey(null, StringUtf8Coder.of()), + commitBuilder.getTagValuePrefixDeletes(0).getTagPrefix()); + } + + @Test + public void testMapComplexPersist() throws Exception { + StateTag> addr = + StateTags.map("map", StringUtf8Coder.of(), VarIntCoder.of()); + MapState mapState = underTest.state(NAMESPACE, addr); + + final String tag1 = "tag1"; + final String tag2 = "tag2"; + final String tag3 = "tag3"; + final String tag4 = "tag4"; + + mapState.put(tag1, 1); + mapState.clear(); + mapState.put(tag2, 2); + mapState.put(tag3, 3); + mapState.remove(tag2); + mapState.remove(tag4); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + assertEquals(1, commitBuilder.getTagValuePrefixDeletesCount()); + assertEquals(STATE_FAMILY, commitBuilder.getTagValuePrefixDeletes(0).getStateFamily()); + assertEquals( + protoKeyFromUserKey(null, StringUtf8Coder.of()), + commitBuilder.getTagValuePrefixDeletes(0).getTagPrefix()); + assertThat( + commitBuilder.getValueUpdatesList().stream() + .map(tv -> fromTagValue(tv, StringUtf8Coder.of(), VarIntCoder.of())) + .collect(Collectors.toList()), + Matchers.containsInAnyOrder( + new SimpleEntry<>(tag3, 3), + new SimpleEntry<>(tag2, null), + new SimpleEntry<>(tag4, null))); + + // Once persist has been called, calling persist again should be a noop. + commitBuilder = Windmill.WorkItemCommitRequest.newBuilder(); + assertEquals(0, commitBuilder.getTagValuePrefixDeletesCount()); + assertEquals(0, commitBuilder.getValueUpdatesCount()); + } + + @Test + public void testMultimapGet() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, ByteArrayCoder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final byte[] key = "key".getBytes(StandardCharsets.UTF_8); + SettableFuture> future = SettableFuture.create(); + when(mockReader.multimapFetchSingleEntryFuture( + encodeWithCoder(key, ByteArrayCoder.of()), + key(NAMESPACE, tag), + STATE_FAMILY, + VarIntCoder.of())) + .thenReturn(future); + + ReadableState> result = multimapState.get(dup(key)).readLater(); + waitAndSet(future, Arrays.asList(1, 2, 3), 30); + assertThat(result.read(), Matchers.containsInAnyOrder(1, 2, 3)); + } + + @Test + public void testMultimapPutAndGet() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, ByteArrayCoder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final byte[] key = "key".getBytes(StandardCharsets.UTF_8); + SettableFuture> future = SettableFuture.create(); + when(mockReader.multimapFetchSingleEntryFuture( + encodeWithCoder(key, ByteArrayCoder.of()), + key(NAMESPACE, tag), + STATE_FAMILY, + VarIntCoder.of())) + .thenReturn(future); + + multimapState.put(key, 1); + ReadableState> result = multimapState.get(dup(key)).readLater(); + waitAndSet(future, Arrays.asList(1, 2, 3), 30); + assertThat(result.read(), Matchers.containsInAnyOrder(1, 1, 2, 3)); + } + + @Test + public void testMultimapRemoveAndGet() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, ByteArrayCoder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final byte[] key = "key".getBytes(StandardCharsets.UTF_8); + SettableFuture> future = SettableFuture.create(); + when(mockReader.multimapFetchSingleEntryFuture( + encodeWithCoder(key, ByteArrayCoder.of()), + key(NAMESPACE, tag), + STATE_FAMILY, + VarIntCoder.of())) + .thenReturn(future); + + ReadableState> result1 = multimapState.get(key).readLater(); + ReadableState> result2 = multimapState.get(dup(key)).readLater(); + waitAndSet(future, Arrays.asList(1, 2, 3), 30); + + assertTrue(multimapState.containsKey(key).read()); + assertThat(result1.read(), Matchers.containsInAnyOrder(1, 2, 3)); + + multimapState.remove(key); + assertFalse(multimapState.containsKey(dup(key)).read()); + assertThat(result2.read(), Matchers.emptyIterable()); + } + + @Test + public void testMultimapRemoveThenPut() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, ByteArrayCoder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final byte[] key = "key".getBytes(StandardCharsets.UTF_8); + SettableFuture> future = SettableFuture.create(); + when(mockReader.multimapFetchSingleEntryFuture( + encodeWithCoder(key, ByteArrayCoder.of()), + key(NAMESPACE, tag), + STATE_FAMILY, + VarIntCoder.of())) + .thenReturn(future); + + ReadableState> result = multimapState.get(key).readLater(); + waitAndSet(future, Arrays.asList(1, 2, 3), 30); + multimapState.remove(dup(key)); + multimapState.put(key, 4); + multimapState.put(dup(key), 5); + assertThat(result.read(), Matchers.containsInAnyOrder(4, 5)); + } + + @Test + public void testMultimapRemovePersistPut() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, StringUtf8Coder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final String key = "key"; + multimapState.put(key, 1); + multimapState.put(key, 2); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + + // After key is removed, this key is cache complete and no need to read backend. + multimapState.remove(key); + multimapState.put(key, 4); + // Since key is cache complete, value 4 in localAdditions should be added to cached values, + /// instead of being cleared from cache after persisted. + underTest.persist(commitBuilder); + assertTagMultimapUpdates( + Iterables.getOnlyElement(commitBuilder.getMultimapUpdatesBuilderList()), + new MultimapEntryUpdate(key, Collections.singletonList(4), true)); + + multimapState.put(key, 5); + assertThat(multimapState.get(key).read(), Matchers.containsInAnyOrder(4, 5)); + } + + @Test + public void testMultimapGetLocalCombineStorage() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, ByteArrayCoder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final byte[] key = "key".getBytes(StandardCharsets.UTF_8); + SettableFuture> future = SettableFuture.create(); + when(mockReader.multimapFetchSingleEntryFuture( + encodeWithCoder(key, ByteArrayCoder.of()), + key(NAMESPACE, tag), + STATE_FAMILY, + VarIntCoder.of())) + .thenReturn(future); + + ReadableState> result = multimapState.get(dup(key)).readLater(); + waitAndSet(future, Arrays.asList(1, 2), 30); + multimapState.put(key, 3); + multimapState.put(dup(key), 4); + assertFalse(multimapState.isEmpty().read()); + assertThat(result.read(), Matchers.containsInAnyOrder(1, 2, 3, 4)); + } + + @Test + public void testMultimapLocalRemoveOverrideStorage() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, ByteArrayCoder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final byte[] key = "key".getBytes(StandardCharsets.UTF_8); + SettableFuture> future = SettableFuture.create(); + when(mockReader.multimapFetchSingleEntryFuture( + encodeWithCoder(key, ByteArrayCoder.of()), + key(NAMESPACE, tag), + STATE_FAMILY, + VarIntCoder.of())) + .thenReturn(future); + + ReadableState> result = multimapState.get(key).readLater(); + waitAndSet(future, Arrays.asList(1, 2), 30); + multimapState.remove(dup(key)); + assertThat(result.read(), Matchers.emptyIterable()); + multimapState.put(key, 3); + multimapState.put(dup(key), 4); + assertFalse(multimapState.isEmpty().read()); + assertThat(result.read(), Matchers.containsInAnyOrder(3, 4)); + } + + @Test + public void testMultimapLocalClearOverrideStorage() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, ByteArrayCoder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final byte[] key1 = "key1".getBytes(StandardCharsets.UTF_8); + final byte[] key2 = "key2".getBytes(StandardCharsets.UTF_8); + SettableFuture> future = SettableFuture.create(); + when(mockReader.multimapFetchSingleEntryFuture( + encodeWithCoder(key1, ByteArrayCoder.of()), + key(NAMESPACE, tag), + STATE_FAMILY, + VarIntCoder.of())) + .thenReturn(future); + SettableFuture> future2 = SettableFuture.create(); + when(mockReader.multimapFetchSingleEntryFuture( + encodeWithCoder(key2, ByteArrayCoder.of()), + key(NAMESPACE, tag), + STATE_FAMILY, + VarIntCoder.of())) + .thenReturn(future2); + + ReadableState> result1 = multimapState.get(key1).readLater(); + ReadableState> result2 = multimapState.get(dup(key2)).readLater(); + multimapState.clear(); + waitAndSet(future, Arrays.asList(1, 2), 30); + assertThat(result1.read(), Matchers.emptyIterable()); + assertThat(result2.read(), Matchers.emptyIterable()); + assertThat(multimapState.keys().read(), Matchers.emptyIterable()); + assertThat(multimapState.entries().read(), Matchers.emptyIterable()); + assertTrue(multimapState.isEmpty().read()); + } + + @Test + public void testMultimapBasicEntriesAndKeys() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, ByteArrayCoder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final byte[] key1 = "key1".getBytes(StandardCharsets.UTF_8); + final byte[] key2 = "key2".getBytes(StandardCharsets.UTF_8); + + SettableFuture>>> entriesFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + false, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(entriesFuture); + SettableFuture>>> keysFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + true, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(keysFuture); + + ReadableState>> entriesResult = + multimapState.entries().readLater(); + ReadableState> keysResult = multimapState.keys().readLater(); + waitAndSet( + entriesFuture, + Arrays.asList(multimapEntry(key1, 1, 2, 3), multimapEntry(key2, 2, 3, 4)), + 30); + waitAndSet(keysFuture, Arrays.asList(multimapEntry(key1), multimapEntry(key2)), 30); + + Iterable> entries = entriesResult.read(); + assertEquals(6, Iterables.size(entries)); + assertThat( + entries, + Matchers.containsInAnyOrder( + multimapEntryMatcher(key1, 1), + multimapEntryMatcher(key1, 2), + multimapEntryMatcher(key1, 3), + multimapEntryMatcher(key2, 4), + multimapEntryMatcher(key2, 2), + multimapEntryMatcher(key2, 3))); + + Iterable keys = keysResult.read(); + assertEquals(2, Iterables.size(keys)); + assertThat(keys, Matchers.containsInAnyOrder(key1, key2)); + } + + @Test + public void testMultimapEntriesAndKeysMergeLocalAdd() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, ByteArrayCoder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final byte[] key1 = "key1".getBytes(StandardCharsets.UTF_8); + final byte[] key2 = "key2".getBytes(StandardCharsets.UTF_8); + final byte[] key3 = "key3".getBytes(StandardCharsets.UTF_8); + + SettableFuture>>> entriesFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + false, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(entriesFuture); + SettableFuture>>> keysFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + true, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(keysFuture); + + ReadableState>> entriesResult = + multimapState.entries().readLater(); + ReadableState> keysResult = multimapState.keys().readLater(); + waitAndSet( + entriesFuture, + Arrays.asList(multimapEntry(key1, 1, 2, 3), multimapEntry(key2, 2, 3, 4)), + 30); + waitAndSet(keysFuture, Arrays.asList(multimapEntry(key1), multimapEntry(key2)), 30); + + multimapState.put(key1, 7); + multimapState.put(dup(key2), 8); + multimapState.put(dup(key3), 8); + + Iterable> entries = entriesResult.read(); + assertEquals(9, Iterables.size(entries)); + assertThat( + entries, + Matchers.containsInAnyOrder( + multimapEntryMatcher(key1, 1), + multimapEntryMatcher(key1, 2), + multimapEntryMatcher(key1, 3), + multimapEntryMatcher(key1, 7), + multimapEntryMatcher(key2, 4), + multimapEntryMatcher(key2, 2), + multimapEntryMatcher(key2, 3), + multimapEntryMatcher(key2, 8), + multimapEntryMatcher(key3, 8))); + + Iterable keys = keysResult.read(); + assertEquals(3, Iterables.size(keys)); + assertThat(keys, Matchers.containsInAnyOrder(key1, key2, key3)); + } + + @Test + public void testMultimapEntriesAndKeysMergeLocalRemove() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, ByteArrayCoder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final byte[] key1 = "key1".getBytes(StandardCharsets.UTF_8); + final byte[] key2 = "key2".getBytes(StandardCharsets.UTF_8); + final byte[] key3 = "key3".getBytes(StandardCharsets.UTF_8); + + SettableFuture>>> entriesFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + false, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(entriesFuture); + SettableFuture>>> keysFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + true, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(keysFuture); + + ReadableState>> entriesResult = + multimapState.entries().readLater(); + ReadableState> keysResult = multimapState.keys().readLater(); + waitAndSet( + entriesFuture, + Arrays.asList(multimapEntry(key1, 1, 2, 3), multimapEntry(key2, 2, 3, 4)), + 30); + waitAndSet(keysFuture, Arrays.asList(multimapEntry(key1), multimapEntry(key2)), 30); + + multimapState.remove(dup(key1)); + multimapState.put(key2, 8); + multimapState.put(dup(key3), 8); + + Iterable> entries = entriesResult.read(); + assertEquals(5, Iterables.size(entries)); + assertThat( + entries, + Matchers.containsInAnyOrder( + multimapEntryMatcher(key2, 4), + multimapEntryMatcher(key2, 2), + multimapEntryMatcher(key2, 3), + multimapEntryMatcher(key2, 8), + multimapEntryMatcher(key3, 8))); + + Iterable keys = keysResult.read(); + assertThat(keys, Matchers.containsInAnyOrder(key2, key3)); + } + + @Test + public void testMultimapCacheComplete() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, ByteArrayCoder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final byte[] key = "key".getBytes(StandardCharsets.UTF_8); + + SettableFuture>>> entriesFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + false, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(entriesFuture); + + // to set up the multimap as cache complete + waitAndSet(entriesFuture, weightedList(multimapEntry(key, 1, 2, 3)), 30); + multimapState.entries().read(); + + multimapState.put(key, 2); + + when(mockReader.multimapFetchAllFuture( + anyBoolean(), eq(key(NAMESPACE, tag)), eq(STATE_FAMILY), eq(VarIntCoder.of()))) + .thenThrow( + new RuntimeException( + "The multimap is cache complete and should not perform any windmill read.")); + when(mockReader.multimapFetchSingleEntryFuture( + any(), eq(key(NAMESPACE, tag)), eq(STATE_FAMILY), eq(VarIntCoder.of()))) + .thenThrow( + new RuntimeException( + "The multimap is cache complete and should not perform any windmill read.")); + + Iterable> entries = multimapState.entries().read(); + assertEquals(4, Iterables.size(entries)); + assertThat( + entries, + Matchers.containsInAnyOrder( + multimapEntryMatcher(key, 1), + multimapEntryMatcher(key, 2), + multimapEntryMatcher(key, 3), + multimapEntryMatcher(key, 2))); + + Iterable keys = multimapState.keys().read(); + assertThat(keys, Matchers.containsInAnyOrder(key)); + + Iterable values = multimapState.get(dup(key)).read(); + assertThat(values, Matchers.containsInAnyOrder(1, 2, 2, 3)); + } + + @Test + public void testMultimapCachedSingleEntry() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, ByteArrayCoder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final byte[] key = "key".getBytes(StandardCharsets.UTF_8); + + SettableFuture> entryFuture = SettableFuture.create(); + when(mockReader.multimapFetchSingleEntryFuture( + encodeWithCoder(key, ByteArrayCoder.of()), + key(NAMESPACE, tag), + STATE_FAMILY, + VarIntCoder.of())) + .thenReturn(entryFuture); + + // to set up the entry key as cache complete and add some local changes + waitAndSet(entryFuture, weightedList(1, 2, 3), 30); + multimapState.get(key).read(); + multimapState.put(key, 2); + + when(mockReader.multimapFetchSingleEntryFuture( + eq(encodeWithCoder(key, ByteArrayCoder.of())), + eq(key(NAMESPACE, tag)), + eq(STATE_FAMILY), + eq(VarIntCoder.of()))) + .thenThrow( + new RuntimeException( + "The multimap is cache complete for " + + Arrays.toString(key) + + " and should not perform any windmill read.")); + + Iterable values = multimapState.get(dup(key)).read(); + assertThat(values, Matchers.containsInAnyOrder(1, 2, 2, 3)); + assertTrue(multimapState.containsKey(key).read()); + } + + @Test + public void testMultimapCachedPartialEntry() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, ByteArrayCoder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final byte[] key1 = "key1".getBytes(StandardCharsets.UTF_8); + final byte[] key2 = "key2".getBytes(StandardCharsets.UTF_8); + final byte[] key3 = "key3".getBytes(StandardCharsets.UTF_8); + + SettableFuture> entryFuture = SettableFuture.create(); + when(mockReader.multimapFetchSingleEntryFuture( + encodeWithCoder(key1, ByteArrayCoder.of()), + key(NAMESPACE, tag), + STATE_FAMILY, + VarIntCoder.of())) + .thenReturn(entryFuture); + + // to set up the entry key1 as cache complete and add some local changes + waitAndSet(entryFuture, weightedList(1, 2, 3), 30); + multimapState.get(key1).read(); + multimapState.put(key1, 2); + multimapState.put(key3, 20); + + SettableFuture>>> entriesFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + false, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(entriesFuture); + + // windmill contains extra entry key2 + waitAndSet( + entriesFuture, + weightedList(multimapEntry(key1, 1, 2, 3), multimapEntry(key2, 4, 5, 6)), + 30); + + // key1 exist in both cache and windmill; key2 exists only in windmill; key3 exists only in + // cache. They should all be merged. + Iterable> entries = multimapState.entries().read(); + + assertEquals(8, Iterables.size(entries)); + assertThat( + entries, + Matchers.containsInAnyOrder( + multimapEntryMatcher(key1, 1), + multimapEntryMatcher(key1, 2), + multimapEntryMatcher(key1, 2), + multimapEntryMatcher(key1, 3), + multimapEntryMatcher(key2, 4), + multimapEntryMatcher(key2, 5), + multimapEntryMatcher(key2, 6), + multimapEntryMatcher(key3, 20))); + + assertThat(multimapState.keys().read(), Matchers.containsInAnyOrder(key1, key2, key3)); + } + + @Test + public void testMultimapEntriesCombineCacheAndWindmill() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, ByteArrayCoder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final byte[] key1 = "key1".getBytes(StandardCharsets.UTF_8); + final byte[] key2 = "key2".getBytes(StandardCharsets.UTF_8); + final byte[] key3 = "key3".getBytes(StandardCharsets.UTF_8); + + SettableFuture> entryFuture = SettableFuture.create(); + when(mockReader.multimapFetchSingleEntryFuture( + encodeWithCoder(key1, ByteArrayCoder.of()), + key(NAMESPACE, tag), + STATE_FAMILY, + VarIntCoder.of())) + .thenReturn(entryFuture); + + // to set up the entry key1 as cache complete and add some local changes + waitAndSet(entryFuture, weightedList(1, 2, 3), 30); + multimapState.get(key1).read(); + multimapState.put(dup(key1), 2); + multimapState.put(dup(key3), 20); + + SettableFuture>>> entriesFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + false, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(entriesFuture); + SettableFuture>>> keysFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + true, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(keysFuture); + + // windmill contains extra entry key2, and this time the entries returned should not be cached. + waitAndSet( + entriesFuture, + Arrays.asList(multimapEntry(key1, 1, 2, 3), multimapEntry(key2, 4, 5, 6)), + 30); + waitAndSet(keysFuture, Arrays.asList(multimapEntry(key1), multimapEntry(key2)), 30); + + // key1 exist in both cache and windmill; key2 exists only in windmill; key3 exists only in + // cache. They should all be merged. + Iterable> entries = multimapState.entries().read(); + + assertEquals(8, Iterables.size(entries)); + assertThat( + entries, + Matchers.containsInAnyOrder( + multimapEntryMatcher(key1, 1), + multimapEntryMatcher(key1, 2), + multimapEntryMatcher(key1, 2), + multimapEntryMatcher(key1, 3), + multimapEntryMatcher(key2, 4), + multimapEntryMatcher(key2, 5), + multimapEntryMatcher(key2, 6), + multimapEntryMatcher(key3, 20))); + + assertThat(multimapState.keys().read(), Matchers.containsInAnyOrder(key1, key2, key3)); + } + + @Test + public void testMultimapModifyAfterReadDoesNotAffectResult() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, ByteArrayCoder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final byte[] key1 = "key1".getBytes(StandardCharsets.UTF_8); + final byte[] key2 = "key2".getBytes(StandardCharsets.UTF_8); + final byte[] key3 = "key3".getBytes(StandardCharsets.UTF_8); + final byte[] key4 = "key4".getBytes(StandardCharsets.UTF_8); + + SettableFuture>>> entriesFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + false, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(entriesFuture); + SettableFuture>>> keysFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + true, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(keysFuture); + SettableFuture> getKey1Future = SettableFuture.create(); + SettableFuture> getKey2Future = SettableFuture.create(); + SettableFuture> getKey4Future = SettableFuture.create(); + when(mockReader.multimapFetchSingleEntryFuture( + encodeWithCoder(key1, ByteArrayCoder.of()), + key(NAMESPACE, tag), + STATE_FAMILY, + VarIntCoder.of())) + .thenReturn(getKey1Future); + when(mockReader.multimapFetchSingleEntryFuture( + encodeWithCoder(key2, ByteArrayCoder.of()), + key(NAMESPACE, tag), + STATE_FAMILY, + VarIntCoder.of())) + .thenReturn(getKey2Future); + when(mockReader.multimapFetchSingleEntryFuture( + encodeWithCoder(key4, ByteArrayCoder.of()), + key(NAMESPACE, tag), + STATE_FAMILY, + VarIntCoder.of())) + .thenReturn(getKey4Future); + + ReadableState>> entriesResult = + multimapState.entries().readLater(); + ReadableState> keysResult = multimapState.keys().readLater(); + waitAndSet( + entriesFuture, + Arrays.asList(multimapEntry(key1, 1, 2, 3), multimapEntry(key2, 2, 3, 4)), + 200); + waitAndSet(keysFuture, Arrays.asList(multimapEntry(key1), multimapEntry(key2)), 200); + + // make key4 to be known nonexistent. + multimapState.remove(key4); + + ReadableState> key1Future = multimapState.get(key1).readLater(); + waitAndSet(getKey1Future, Arrays.asList(1, 2, 3), 200); + ReadableState> key2Future = multimapState.get(key2).readLater(); + waitAndSet(getKey2Future, Arrays.asList(2, 3, 4), 200); + ReadableState> key4Future = multimapState.get(key4).readLater(); + waitAndSet(getKey4Future, Collections.emptyList(), 200); + + multimapState.put(key1, 7); + multimapState.put(dup(key2), 8); + multimapState.put(dup(key3), 8); + + Iterable> entries = entriesResult.read(); + Iterable keys = keysResult.read(); + Iterable key1Values = key1Future.read(); + Iterable key2Values = key2Future.read(); + Iterable key4Values = key4Future.read(); + + // values added/removed after read should not be reflected in result + multimapState.remove(key1); + multimapState.put(key2, 9); + multimapState.put(key4, 10); + + assertEquals(9, Iterables.size(entries)); + assertThat( + entries, + Matchers.containsInAnyOrder( + multimapEntryMatcher(key1, 1), + multimapEntryMatcher(key1, 2), + multimapEntryMatcher(key1, 3), + multimapEntryMatcher(key1, 7), + multimapEntryMatcher(key2, 4), + multimapEntryMatcher(key2, 2), + multimapEntryMatcher(key2, 3), + multimapEntryMatcher(key2, 8), + multimapEntryMatcher(key3, 8))); + + assertEquals(3, Iterables.size(keys)); + assertThat(keys, Matchers.containsInAnyOrder(key1, key2, key3)); + + assertEquals(4, Iterables.size(key1Values)); + assertThat(key1Values, Matchers.containsInAnyOrder(1, 2, 3, 7)); + + assertEquals(4, Iterables.size(key2Values)); + assertThat(key2Values, Matchers.containsInAnyOrder(2, 3, 4, 8)); + + assertTrue(Iterables.isEmpty(key4Values)); + } + + @Test + public void testMultimapLazyIterateHugeEntriesResult() { + // A multimap with 1 million keys with a total of 10GBs data + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, ByteArrayCoder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + SettableFuture>>> entriesFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + false, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(entriesFuture); + + waitAndSet( + entriesFuture, + () -> + new Iterator>>() { + final int targetEntries = 1_000_000; // return 1 million entries, which is 10 GBs + final byte[] entryKey = new byte[10_000]; // each key is 10KB + final Random rand = new Random(); + int returnedEntries = 0; + + @Override + public boolean hasNext() { + return returnedEntries < targetEntries; + } + + @Override + public Map.Entry> next() { + returnedEntries++; + rand.nextBytes(entryKey); + return multimapEntry(entryKey, 1); + } + }, + 200); + Iterable> entries = multimapState.entries().read(); + assertEquals(1_000_000, Iterables.size(entries)); + } + + @Test + public void testMultimapLazyIterateHugeKeysResult() { + // A multimap with 1 million keys with a total of 10GBs data + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, ByteArrayCoder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + SettableFuture>>> keysFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + true, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(keysFuture); + + waitAndSet( + keysFuture, + () -> + new Iterator>>() { + final int targetEntries = 1_000_000; // return 1 million entries, which is 10 GBs + final byte[] entryKey = new byte[10_000]; // each key is 10KB + final Random rand = new Random(); + int returnedEntries = 0; + + @Override + public boolean hasNext() { + return returnedEntries < targetEntries; + } + + @Override + public Map.Entry> next() { + returnedEntries++; + rand.nextBytes(entryKey); + return multimapEntry(entryKey); + } + }, + 200); + Iterable keys = multimapState.keys().read(); + assertEquals(1_000_000, Iterables.size(keys)); + } + + @Test + public void testMultimapLazyIterateHugeEntriesResultSingleEntry() { + // A multimap with 1 key and 1 million values and a total of 10GBs data + final String tag = "multimap"; + final Integer key = 100; + StateTag> addr = + StateTags.multimap(tag, VarIntCoder.of(), ByteArrayCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + SettableFuture>>> entriesFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + false, key(NAMESPACE, tag), STATE_FAMILY, ByteArrayCoder.of())) + .thenReturn(entriesFuture); + SettableFuture> getKeyFuture = SettableFuture.create(); + when(mockReader.multimapFetchSingleEntryFuture( + encodeWithCoder(key, VarIntCoder.of()), + key(NAMESPACE, tag), + STATE_FAMILY, + ByteArrayCoder.of())) + .thenReturn(getKeyFuture); + + // a not weighted iterators that returns tons of data + Iterable values = + () -> + new Iterator() { + final int targetValues = 1_000_000; // return 1 million values, which is 10 GBs + final byte[] value = new byte[10_000]; // each value is 10KB + final Random rand = new Random(); + int returnedValues = 0; + + @Override + public boolean hasNext() { + return returnedValues < targetValues; + } + + @Override + public byte[] next() { + returnedValues++; + rand.nextBytes(value); + return value; + } + }; + + waitAndSet( + entriesFuture, + Collections.singletonList( + new SimpleEntry<>(encodeWithCoder(key, VarIntCoder.of()), values)), + 200); + waitAndSet(getKeyFuture, values, 200); + + Iterable> entries = multimapState.entries().read(); + assertEquals(1_000_000, Iterables.size(entries)); + + Iterable valueResult = multimapState.get(key).read(); + assertEquals(1_000_000, Iterables.size(valueResult)); + } + + @Test + public void testMultimapPutAndPersist() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, StringUtf8Coder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final String key1 = "key1"; + final String key2 = "key2"; + + multimapState.put(key1, 1); + multimapState.put(key1, 2); + multimapState.put(key2, 2); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getMultimapUpdatesCount()); + Windmill.TagMultimapUpdateRequest.Builder builder = + Iterables.getOnlyElement(commitBuilder.getMultimapUpdatesBuilderList()); + assertTagMultimapUpdates( + builder, + new MultimapEntryUpdate(key1, Arrays.asList(1, 2), false), + new MultimapEntryUpdate(key2, Collections.singletonList(2), false)); + } + + @Test + public void testMultimapRemovePutAndPersist() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, StringUtf8Coder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final String key1 = "key1"; + final String key2 = "key2"; + + // we should add 1 and 2 to key1 + multimapState.remove(key1); + multimapState.put(key1, 1); + multimapState.put(key1, 2); + // we should not add 2 to key 2 + multimapState.put(key2, 2); + multimapState.remove(key2); + // we should add 4 to key 2 + multimapState.put(key2, 4); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getMultimapUpdatesCount()); + Windmill.TagMultimapUpdateRequest.Builder builder = + Iterables.getOnlyElement(commitBuilder.getMultimapUpdatesBuilderList()); + assertTagMultimapUpdates( + builder, + new MultimapEntryUpdate(key1, Arrays.asList(1, 2), true), + new MultimapEntryUpdate(key2, Collections.singletonList(4), true)); + } + + @Test + public void testMultimapRemoveAndPersist() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, StringUtf8Coder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final String key1 = "key1"; + final String key2 = "key2"; + + multimapState.remove(key1); + multimapState.remove(key2); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getMultimapUpdatesCount()); + Windmill.TagMultimapUpdateRequest.Builder builder = + Iterables.getOnlyElement(commitBuilder.getMultimapUpdatesBuilderList()); + assertTagMultimapUpdates( + builder, + new MultimapEntryUpdate(key1, Collections.emptyList(), true), + new MultimapEntryUpdate(key2, Collections.emptyList(), true)); + } + + @Test + public void testMultimapPutRemoveClearAndPersist() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, StringUtf8Coder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final String key1 = "key1"; + final String key2 = "key2"; + final String key3 = "key3"; + + // no need to send any put/remove if clear is called later + multimapState.put(key1, 1); + multimapState.put(key2, 2); + multimapState.remove(key2); + multimapState.clear(); + // remove without put sent after clear should also not be added: we are cache complete after + // clear, so we know we can skip unnecessary remove. + multimapState.remove(key3); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getMultimapUpdatesCount()); + Windmill.TagMultimapUpdateRequest.Builder builder = + Iterables.getOnlyElement(commitBuilder.getMultimapUpdatesBuilderList()); + assertEquals(0, builder.getUpdatesCount()); + assertTrue(builder.getDeleteAll()); + } + + @Test + public void testMultimapPutRemoveAndPersistWhenComplete() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, StringUtf8Coder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + SettableFuture>>> entriesFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + false, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(entriesFuture); + + // to set up the multimap as cache complete + waitAndSet(entriesFuture, Collections.emptyList(), 30); + multimapState.entries().read(); + + final String key1 = "key1"; + final String key2 = "key2"; + + // put when complete should be sent + multimapState.put(key1, 4); + + // put-then-remove when complete should not be sent + multimapState.put(key2, 5); + multimapState.remove(key2); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getMultimapUpdatesCount()); + Windmill.TagMultimapUpdateRequest.Builder builder = + Iterables.getOnlyElement(commitBuilder.getMultimapUpdatesBuilderList()); + assertTagMultimapUpdates( + builder, new MultimapEntryUpdate(key1, Collections.singletonList(4), false)); + } + + @Test + public void testMultimapRemoveAndKeysAndPersist() throws IOException { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, ByteArrayCoder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + + final byte[] key1 = "key1".getBytes(StandardCharsets.UTF_8); + final byte[] key2 = "key2".getBytes(StandardCharsets.UTF_8); + + SettableFuture>>> keysFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + true, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(keysFuture); + + ReadableState> keysResult = multimapState.keys().readLater(); + waitAndSet( + keysFuture, + new WeightedList<>(Arrays.asList(multimapEntry(key1), multimapEntry(key2))), + 30); + + multimapState.remove(key1); + + Iterable keys = keysResult.read(); + assertEquals(1, Iterables.size(keys)); + assertThat(keys, Matchers.containsInAnyOrder(key2)); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getMultimapUpdatesCount()); + Windmill.TagMultimapUpdateRequest.Builder builder = + Iterables.getOnlyElement(commitBuilder.getMultimapUpdatesBuilderList()); + assertEquals(1, builder.getUpdatesCount()); + assertFalse(builder.getDeleteAll()); + Windmill.TagMultimapEntry entryUpdate = Iterables.getOnlyElement(builder.getUpdatesList()); + byte[] decodedKey = + ByteArrayCoder.of().decode(entryUpdate.getEntryName().newInput(), Context.OUTER); + assertArrayEquals(key1, decodedKey); + assertTrue(entryUpdate.getDeleteAll()); + } + + @Test + public void testMultimapFuzzTest() { + final String tag = "multimap"; + StateTag> addr = + StateTags.multimap(tag, StringUtf8Coder.of(), VarIntCoder.of()); + MultimapState multimapState = underTest.state(NAMESPACE, addr); + Random rand = new Random(); + final int ROUNDS = 100; + + SettableFuture>>> entriesFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + false, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(entriesFuture); + + // to set up the multimap as initially empty and cache complete + waitAndSet(entriesFuture, Collections.emptyList(), 30); + multimapState.entries().read(); + + // mirror mimics all operations on multimapState and is used to verify the correctness. + Multimap mirror = ArrayListMultimap.create(); + + final BiConsumer, MultimapState> operateFn = + (Multimap expected, MultimapState actual) -> { + final int OPS_PER_ROUND = 2000; + final int NUM_KEY = 20; + for (int j = 0; j < OPS_PER_ROUND; j++) { + int op = rand.nextInt(100); + String key = "key" + rand.nextInt(NUM_KEY); + if (op < 50) { + // 50% put operation + Integer value = rand.nextInt(); + actual.put(key, value); + expected.put(key, value); + } else if (op < 95) { + // 45% remove key operation + actual.remove(key); + expected.removeAll(key); + } else { + // 5% clear operation + actual.clear(); + expected.clear(); + } + } + }; + + final BiConsumer, MultimapState> validateFn = + (Multimap expected, MultimapState actual) -> { + Iterable read = actual.keys().read(); + Set bytes = expected.keySet(); + assertThat(read, Matchers.containsInAnyOrder(bytes.toArray())); + for (String key : actual.keys().read()) { + assertThat( + actual.get(key).read(), Matchers.containsInAnyOrder(expected.get(key).toArray())); + } + }; + + for (int i = 0; i < ROUNDS; i++) { + operateFn.accept(mirror, multimapState); + validateFn.accept(mirror, multimapState); + } + + // clear cache and recreate multimapState + cache.forComputation("comp").invalidate(ByteString.copyFrom("dummyKey", Charsets.UTF_8), 123); + resetUnderTest(); + multimapState = underTest.state(NAMESPACE, addr); + + // create corresponding GetData response based on mirror + entriesFuture = SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + false, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(entriesFuture); + SettableFuture>>> keysFuture = + SettableFuture.create(); + when(mockReader.multimapFetchAllFuture( + true, key(NAMESPACE, tag), STATE_FAMILY, VarIntCoder.of())) + .thenReturn(keysFuture); + + List>> entriesFutureContent = new ArrayList<>(); + List>> keysFutureContent = new ArrayList<>(); + + // multimapState is not cache complete, state backend should return content in mirror. + for (Map.Entry> entry : mirror.asMap().entrySet()) { + entriesFutureContent.add( + new AbstractMap.SimpleEntry<>( + encodeWithCoder(entry.getKey(), StringUtf8Coder.of()), entry.getValue())); + keysFutureContent.add( + new AbstractMap.SimpleEntry<>( + encodeWithCoder(entry.getKey(), StringUtf8Coder.of()), Collections.emptyList())); + SettableFuture> getKeyFuture = SettableFuture.create(); + when(mockReader.multimapFetchSingleEntryFuture( + encodeWithCoder(entry.getKey(), StringUtf8Coder.of()), + key(NAMESPACE, tag), + STATE_FAMILY, + VarIntCoder.of())) + .thenReturn(getKeyFuture); + waitAndSet(getKeyFuture, entry.getValue(), 20); + } + waitAndSet(entriesFuture, entriesFutureContent, 20); + waitAndSet(keysFuture, keysFutureContent, 20); + + validateFn.accept(mirror, multimapState); + + // merge cache and new changes + for (int i = 0; i < ROUNDS; i++) { + operateFn.accept(mirror, multimapState); + validateFn.accept(mirror, multimapState); + } + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + } + + @Test + public void testOrderedListAddBeforeRead() throws Exception { + StateTag> addr = + StateTags.orderedList("orderedList", StringUtf8Coder.of()); + OrderedListState orderedList = underTest.state(NAMESPACE, addr); + + SettableFuture>> future = SettableFuture.create(); + when(mockReader.orderedListFuture( + FULL_ORDERED_LIST_RANGE, + key(NAMESPACE, "orderedList"), + STATE_FAMILY, + StringUtf8Coder.of())) + .thenReturn(future); + + orderedList.readLater(); + + final TimestampedValue helloValue = + TimestampedValue.of("hello", Instant.ofEpochMilli(100)); + final TimestampedValue worldValue = + TimestampedValue.of("world", Instant.ofEpochMilli(75)); + final TimestampedValue goodbyeValue = + TimestampedValue.of("goodbye", Instant.ofEpochMilli(50)); + + orderedList.add(helloValue); + waitAndSet(future, Collections.singletonList(worldValue), 200); + assertThat(orderedList.read(), Matchers.contains(worldValue, helloValue)); + + orderedList.add(goodbyeValue); + assertThat(orderedList.read(), Matchers.contains(goodbyeValue, worldValue, helloValue)); + } + + @Test + public void testOrderedListClearBeforeRead() throws Exception { + StateTag> addr = + StateTags.orderedList("orderedList", StringUtf8Coder.of()); + OrderedListState orderedListState = underTest.state(NAMESPACE, addr); + + final TimestampedValue helloElement = TimestampedValue.of("hello", Instant.EPOCH); + orderedListState.clear(); + orderedListState.add(helloElement); + assertThat(orderedListState.read(), Matchers.containsInAnyOrder(helloElement)); + + // Shouldn't need to read from windmill for this. + Mockito.verifyZeroInteractions(mockReader); + } + + @Test + public void testOrderedListIsEmptyFalse() throws Exception { + StateTag> addr = + StateTags.orderedList("orderedList", StringUtf8Coder.of()); + OrderedListState orderedList = underTest.state(NAMESPACE, addr); + + SettableFuture>> future = SettableFuture.create(); + when(mockReader.orderedListFuture( + FULL_ORDERED_LIST_RANGE, + key(NAMESPACE, "orderedList"), + STATE_FAMILY, + StringUtf8Coder.of())) + .thenReturn(future); + ReadableState result = orderedList.isEmpty().readLater(); + Mockito.verify(mockReader) + .orderedListFuture( + FULL_ORDERED_LIST_RANGE, + key(NAMESPACE, "orderedList"), + STATE_FAMILY, + StringUtf8Coder.of()); + + waitAndSet(future, Collections.singletonList(TimestampedValue.of("world", Instant.EPOCH)), 200); + assertThat(result.read(), Matchers.is(false)); + } + + @Test + public void testOrderedListIsEmptyTrue() throws Exception { + StateTag> addr = + StateTags.orderedList("orderedList", StringUtf8Coder.of()); + OrderedListState orderedList = underTest.state(NAMESPACE, addr); + + SettableFuture>> future = SettableFuture.create(); + when(mockReader.orderedListFuture( + FULL_ORDERED_LIST_RANGE, + key(NAMESPACE, "orderedList"), + STATE_FAMILY, + StringUtf8Coder.of())) + .thenReturn(future); + ReadableState result = orderedList.isEmpty().readLater(); + Mockito.verify(mockReader) + .orderedListFuture( + FULL_ORDERED_LIST_RANGE, + key(NAMESPACE, "orderedList"), + STATE_FAMILY, + StringUtf8Coder.of()); + + waitAndSet(future, Collections.emptyList(), 200); + assertThat(result.read(), Matchers.is(true)); + } + + @Test + public void testOrderedListIsEmptyAfterClear() throws Exception { + StateTag> addr = + StateTags.orderedList("orderedList", StringUtf8Coder.of()); + OrderedListState orderedList = underTest.state(NAMESPACE, addr); + + orderedList.clear(); + ReadableState result = orderedList.isEmpty(); + Mockito.verify(mockReader, never()) + .orderedListFuture( + FULL_ORDERED_LIST_RANGE, + key(NAMESPACE, "orderedList"), + STATE_FAMILY, + StringUtf8Coder.of()); + assertThat(result.read(), Matchers.is(true)); + + orderedList.add(TimestampedValue.of("hello", Instant.EPOCH)); + assertThat(result.read(), Matchers.is(false)); + } + + @Test + public void testOrderedListAddPersist() throws Exception { + StateTag> addr = + StateTags.orderedList("orderedList", StringUtf8Coder.of()); + OrderedListState orderedList = underTest.state(NAMESPACE, addr); + + SettableFuture, RangeSet>> orderedListFuture = SettableFuture.create(); + orderedListFuture.set(null); + SettableFuture, RangeSet>> deletionsFuture = + SettableFuture.create(); + deletionsFuture.set(null); + when(mockReader.valueFuture( + systemKey(NAMESPACE, "orderedList" + IdTracker.IDS_AVAILABLE_STR), + STATE_FAMILY, + IdTracker.IDS_AVAILABLE_CODER)) + .thenReturn(orderedListFuture); + when(mockReader.valueFuture( + systemKey(NAMESPACE, "orderedList" + IdTracker.DELETIONS_STR), + STATE_FAMILY, + IdTracker.SUBRANGE_DELETIONS_CODER)) + .thenReturn(deletionsFuture); + + orderedList.add(TimestampedValue.of("hello", Instant.ofEpochMilli(1))); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getSortedListUpdatesCount()); + TagSortedListUpdateRequest updates = commitBuilder.getSortedListUpdates(0); + assertEquals(key(NAMESPACE, "orderedList"), updates.getTag()); + assertEquals(1, updates.getInsertsCount()); + assertEquals(1, updates.getInserts(0).getEntriesCount()); + + assertEquals("hello", updates.getInserts(0).getEntries(0).getValue().toStringUtf8()); + assertEquals(1000, updates.getInserts(0).getEntries(0).getSortKey()); + assertEquals(IdTracker.NEW_RANGE_MIN_ID, updates.getInserts(0).getEntries(0).getId()); + } + + @Test + public void testOrderedListClearPersist() throws Exception { + StateTag> addr = + StateTags.orderedList("orderedList", StringUtf8Coder.of()); + OrderedListState orderedListState = underTest.state(NAMESPACE, addr); + + orderedListState.add(TimestampedValue.of("hello", Instant.ofEpochMilli(1))); + orderedListState.clear(); + orderedListState.add(TimestampedValue.of("world", Instant.ofEpochMilli(2))); + orderedListState.add(TimestampedValue.of("world", Instant.ofEpochMilli(2))); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getSortedListUpdatesCount()); + TagSortedListUpdateRequest updates = commitBuilder.getSortedListUpdates(0); + assertEquals(STATE_FAMILY, updates.getStateFamily()); + assertEquals(key(NAMESPACE, "orderedList"), updates.getTag()); + assertEquals(1, updates.getInsertsCount()); + assertEquals(2, updates.getInserts(0).getEntriesCount()); + + assertEquals("world", updates.getInserts(0).getEntries(0).getValue().toStringUtf8()); + assertEquals("world", updates.getInserts(0).getEntries(1).getValue().toStringUtf8()); + assertEquals(2000, updates.getInserts(0).getEntries(0).getSortKey()); + assertEquals(2000, updates.getInserts(0).getEntries(1).getSortKey()); + assertEquals(IdTracker.NEW_RANGE_MIN_ID, updates.getInserts(0).getEntries(0).getId()); + assertEquals(IdTracker.NEW_RANGE_MIN_ID + 1, updates.getInserts(0).getEntries(1).getId()); + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testOrderedListDeleteRangePersist() { + SettableFuture, RangeSet>> orderedListFuture = SettableFuture.create(); + orderedListFuture.set(null); + SettableFuture, RangeSet>> deletionsFuture = + SettableFuture.create(); + deletionsFuture.set(null); + when(mockReader.valueFuture( + systemKey(NAMESPACE, "orderedList" + IdTracker.IDS_AVAILABLE_STR), + STATE_FAMILY, + IdTracker.IDS_AVAILABLE_CODER)) + .thenReturn(orderedListFuture); + when(mockReader.valueFuture( + systemKey(NAMESPACE, "orderedList" + IdTracker.DELETIONS_STR), + STATE_FAMILY, + IdTracker.SUBRANGE_DELETIONS_CODER)) + .thenReturn(deletionsFuture); + + StateTag> addr = + StateTags.orderedList("orderedList", StringUtf8Coder.of()); + OrderedListState orderedListState = underTest.state(NAMESPACE, addr); + + orderedListState.add(TimestampedValue.of("hello", Instant.ofEpochMilli(1))); + orderedListState.add(TimestampedValue.of("hello", Instant.ofEpochMilli(2))); + orderedListState.add(TimestampedValue.of("hello", Instant.ofEpochMilli(2))); + orderedListState.add(TimestampedValue.of("world", Instant.ofEpochMilli(3))); + orderedListState.add(TimestampedValue.of("world", Instant.ofEpochMilli(4))); + orderedListState.clearRange(Instant.ofEpochMilli(2), Instant.ofEpochMilli(4)); + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getSortedListUpdatesCount()); + TagSortedListUpdateRequest updates = commitBuilder.getSortedListUpdates(0); + assertEquals(STATE_FAMILY, updates.getStateFamily()); + assertEquals(key(NAMESPACE, "orderedList"), updates.getTag()); + assertEquals(1, updates.getInsertsCount()); + assertEquals(2, updates.getInserts(0).getEntriesCount()); + + assertEquals("hello", updates.getInserts(0).getEntries(0).getValue().toStringUtf8()); + assertEquals("world", updates.getInserts(0).getEntries(1).getValue().toStringUtf8()); + assertEquals(1000, updates.getInserts(0).getEntries(0).getSortKey()); + assertEquals(4000, updates.getInserts(0).getEntries(1).getSortKey()); + assertEquals(IdTracker.NEW_RANGE_MIN_ID, updates.getInserts(0).getEntries(0).getId()); + assertEquals(IdTracker.NEW_RANGE_MIN_ID + 1, updates.getInserts(0).getEntries(1).getId()); + } + + @Test + public void testOrderedListMergePendingAdds() { + SettableFuture, RangeSet>> orderedListFuture = SettableFuture.create(); + orderedListFuture.set(null); + SettableFuture, RangeSet>> deletionsFuture = + SettableFuture.create(); + deletionsFuture.set(null); + when(mockReader.valueFuture( + systemKey(NAMESPACE, "orderedList" + IdTracker.IDS_AVAILABLE_STR), + STATE_FAMILY, + IdTracker.IDS_AVAILABLE_CODER)) + .thenReturn(orderedListFuture); + when(mockReader.valueFuture( + systemKey(NAMESPACE, "orderedList" + IdTracker.DELETIONS_STR), + STATE_FAMILY, + IdTracker.SUBRANGE_DELETIONS_CODER)) + .thenReturn(deletionsFuture); + + SettableFuture>> fromStorage = SettableFuture.create(); + when(mockReader.orderedListFuture( + FULL_ORDERED_LIST_RANGE, + key(NAMESPACE, "orderedList"), + STATE_FAMILY, + StringUtf8Coder.of())) + .thenReturn(fromStorage); + + StateTag> addr = + StateTags.orderedList("orderedList", StringUtf8Coder.of()); + OrderedListState orderedListState = underTest.state(NAMESPACE, addr); + + orderedListState.add(TimestampedValue.of("second", Instant.ofEpochMilli(1))); + orderedListState.add(TimestampedValue.of("third", Instant.ofEpochMilli(2))); + orderedListState.add(TimestampedValue.of("fourth", Instant.ofEpochMilli(2))); + orderedListState.add(TimestampedValue.of("eighth", Instant.ofEpochMilli(10))); + orderedListState.add(TimestampedValue.of("ninth", Instant.ofEpochMilli(15))); + + fromStorage.set( + ImmutableList.of( + TimestampedValue.of("first", Instant.ofEpochMilli(-1)), + TimestampedValue.of("fifth", Instant.ofEpochMilli(5)), + TimestampedValue.of("sixth", Instant.ofEpochMilli(5)), + TimestampedValue.of("seventh", Instant.ofEpochMilli(5)), + TimestampedValue.of("tenth", Instant.ofEpochMilli(20)))); + + TimestampedValue[] expected = + Iterables.toArray( + ImmutableList.of( + TimestampedValue.of("first", Instant.ofEpochMilli(-1)), + TimestampedValue.of("second", Instant.ofEpochMilli(1)), + TimestampedValue.of("third", Instant.ofEpochMilli(2)), + TimestampedValue.of("fourth", Instant.ofEpochMilli(2)), + TimestampedValue.of("fifth", Instant.ofEpochMilli(5)), + TimestampedValue.of("sixth", Instant.ofEpochMilli(5)), + TimestampedValue.of("seventh", Instant.ofEpochMilli(5)), + TimestampedValue.of("eighth", Instant.ofEpochMilli(10)), + TimestampedValue.of("ninth", Instant.ofEpochMilli(15)), + TimestampedValue.of("tenth", Instant.ofEpochMilli(20))), + TimestampedValue.class); + + TimestampedValue[] read = Iterables.toArray(orderedListState.read(), TimestampedValue.class); + assertArrayEquals(expected, read); + } + + @Test + public void testOrderedListMergePendingAddsAndDeletes() { + SettableFuture, RangeSet>> orderedListFuture = SettableFuture.create(); + orderedListFuture.set(null); + SettableFuture, RangeSet>> deletionsFuture = + SettableFuture.create(); + deletionsFuture.set(null); + when(mockReader.valueFuture( + systemKey(NAMESPACE, "orderedList" + IdTracker.IDS_AVAILABLE_STR), + STATE_FAMILY, + IdTracker.IDS_AVAILABLE_CODER)) + .thenReturn(orderedListFuture); + when(mockReader.valueFuture( + systemKey(NAMESPACE, "orderedList" + IdTracker.DELETIONS_STR), + STATE_FAMILY, + IdTracker.SUBRANGE_DELETIONS_CODER)) + .thenReturn(deletionsFuture); + + SettableFuture>> fromStorage = SettableFuture.create(); + when(mockReader.orderedListFuture( + FULL_ORDERED_LIST_RANGE, + key(NAMESPACE, "orderedList"), + STATE_FAMILY, + StringUtf8Coder.of())) + .thenReturn(fromStorage); + + StateTag> addr = + StateTags.orderedList("orderedList", StringUtf8Coder.of()); + OrderedListState orderedListState = underTest.state(NAMESPACE, addr); + + orderedListState.add(TimestampedValue.of("second", Instant.ofEpochMilli(1))); + orderedListState.add(TimestampedValue.of("third", Instant.ofEpochMilli(2))); + orderedListState.add(TimestampedValue.of("fourth", Instant.ofEpochMilli(2))); + orderedListState.add(TimestampedValue.of("eighth", Instant.ofEpochMilli(10))); + orderedListState.add(TimestampedValue.of("ninth", Instant.ofEpochMilli(15))); + + orderedListState.clearRange(Instant.ofEpochMilli(2), Instant.ofEpochMilli(5)); + orderedListState.add(TimestampedValue.of("fourth", Instant.ofEpochMilli(4))); + + fromStorage.set( + ImmutableList.of( + TimestampedValue.of("first", Instant.ofEpochMilli(-1)), + TimestampedValue.of("fifth", Instant.ofEpochMilli(5)), + TimestampedValue.of("sixth", Instant.ofEpochMilli(5)), + TimestampedValue.of("seventh", Instant.ofEpochMilli(5)), + TimestampedValue.of("tenth", Instant.ofEpochMilli(20)))); + + TimestampedValue[] expected = + Iterables.toArray( + ImmutableList.of( + TimestampedValue.of("first", Instant.ofEpochMilli(-1)), + TimestampedValue.of("second", Instant.ofEpochMilli(1)), + TimestampedValue.of("fourth", Instant.ofEpochMilli(4)), + TimestampedValue.of("fifth", Instant.ofEpochMilli(5)), + TimestampedValue.of("sixth", Instant.ofEpochMilli(5)), + TimestampedValue.of("seventh", Instant.ofEpochMilli(5)), + TimestampedValue.of("eighth", Instant.ofEpochMilli(10)), + TimestampedValue.of("ninth", Instant.ofEpochMilli(15)), + TimestampedValue.of("tenth", Instant.ofEpochMilli(20))), + TimestampedValue.class); + + TimestampedValue[] read = Iterables.toArray(orderedListState.read(), TimestampedValue.class); + assertArrayEquals(expected, read); + } + + @Test + public void testOrderedListPersistEmpty() throws Exception { + StateTag> addr = + StateTags.orderedList("orderedList", StringUtf8Coder.of()); + OrderedListState orderedListState = underTest.state(NAMESPACE, addr); + + orderedListState.clear(); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + // 1 bag update = the clear + assertEquals(1, commitBuilder.getSortedListUpdatesCount()); + TagSortedListUpdateRequest updates = commitBuilder.getSortedListUpdates(0); + assertEquals(1, updates.getDeletesCount()); + assertEquals(WindmillOrderedList.MIN_TS_MICROS, updates.getDeletes(0).getRange().getStart()); + assertEquals(WindmillOrderedList.MAX_TS_MICROS, updates.getDeletes(0).getRange().getLimit()); + } + + @Test + public void testNewOrderedListNoFetch() throws Exception { + StateTag> addr = + StateTags.orderedList("orderedList", StringUtf8Coder.of()); + OrderedListState orderedList = underTestNewKey.state(NAMESPACE, addr); + + assertThat(orderedList.read(), Matchers.emptyIterable()); + + // Shouldn't need to read from windmill for this. + Mockito.verifyZeroInteractions(mockReader); + } + + @Test + public void testBagAddBeforeRead() throws Exception { + StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); + BagState bag = underTest.state(NAMESPACE, addr); + + SettableFuture> future = SettableFuture.create(); + when(mockReader.bagFuture(key(NAMESPACE, "bag"), STATE_FAMILY, StringUtf8Coder.of())) + .thenReturn(future); + + bag.readLater(); + + bag.add("hello"); + waitAndSet(future, Collections.singletonList("world"), 200); + assertThat(bag.read(), Matchers.containsInAnyOrder("hello", "world")); + + bag.add("goodbye"); + assertThat(bag.read(), Matchers.containsInAnyOrder("hello", "world", "goodbye")); + } + + // test ordered list cleared before read + // test fetch + add + read + // test ids + + @Test + public void testBagClearBeforeRead() throws Exception { + StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); + BagState bag = underTest.state(NAMESPACE, addr); + + bag.clear(); + bag.add("hello"); + assertThat(bag.read(), Matchers.containsInAnyOrder("hello")); + + // Shouldn't need to read from windmill for this. + Mockito.verifyZeroInteractions(mockReader); + } + + @Test + public void testBagIsEmptyFalse() throws Exception { + StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); + BagState bag = underTest.state(NAMESPACE, addr); + + SettableFuture> future = SettableFuture.create(); + when(mockReader.bagFuture(key(NAMESPACE, "bag"), STATE_FAMILY, StringUtf8Coder.of())) + .thenReturn(future); + ReadableState result = bag.isEmpty().readLater(); + Mockito.verify(mockReader).bagFuture(key(NAMESPACE, "bag"), STATE_FAMILY, StringUtf8Coder.of()); + + waitAndSet(future, Collections.singletonList("world"), 200); + assertThat(result.read(), Matchers.is(false)); + } + + @Test + public void testBagIsEmptyTrue() throws Exception { + StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); + BagState bag = underTest.state(NAMESPACE, addr); + + SettableFuture> future = SettableFuture.create(); + when(mockReader.bagFuture(key(NAMESPACE, "bag"), STATE_FAMILY, StringUtf8Coder.of())) + .thenReturn(future); + ReadableState result = bag.isEmpty().readLater(); + Mockito.verify(mockReader).bagFuture(key(NAMESPACE, "bag"), STATE_FAMILY, StringUtf8Coder.of()); + + waitAndSet(future, Collections.emptyList(), 200); + assertThat(result.read(), Matchers.is(true)); + } + + @Test + public void testBagIsEmptyAfterClear() throws Exception { + StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); + BagState bag = underTest.state(NAMESPACE, addr); + + bag.clear(); + ReadableState result = bag.isEmpty(); + Mockito.verify(mockReader, never()) + .bagFuture(key(NAMESPACE, "bag"), STATE_FAMILY, StringUtf8Coder.of()); + assertThat(result.read(), Matchers.is(true)); + + bag.add("hello"); + assertThat(result.read(), Matchers.is(false)); + } + + @Test + public void testBagAddPersist() throws Exception { + StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); + BagState bag = underTest.state(NAMESPACE, addr); + + bag.add("hello"); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getBagUpdatesCount()); + + TagBag bagUpdates = commitBuilder.getBagUpdates(0); + assertEquals(key(NAMESPACE, "bag"), bagUpdates.getTag()); + assertEquals(1, bagUpdates.getValuesCount()); + assertEquals("hello", bagUpdates.getValues(0).toStringUtf8()); + + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testBagClearPersist() throws Exception { + StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); + BagState bag = underTest.state(NAMESPACE, addr); + + bag.add("hello"); + bag.clear(); + bag.add("world"); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getBagUpdatesCount()); + + TagBag tagBag = commitBuilder.getBagUpdates(0); + assertEquals(key(NAMESPACE, "bag"), tagBag.getTag()); + assertEquals(STATE_FAMILY, tagBag.getStateFamily()); + assertTrue(tagBag.getDeleteAll()); + assertEquals(1, tagBag.getValuesCount()); + assertEquals("world", tagBag.getValues(0).toStringUtf8()); + + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testBagPersistEmpty() throws Exception { + StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); + BagState bag = underTest.state(NAMESPACE, addr); + + bag.clear(); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + // 1 bag update = the clear + assertEquals(1, commitBuilder.getBagUpdatesCount()); + } + + @Test + public void testNewBagNoFetch() throws Exception { + StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); + BagState bag = underTestNewKey.state(NAMESPACE, addr); + + assertThat(bag.read(), Matchers.emptyIterable()); + + // Shouldn't need to read from windmill for this. + Mockito.verifyZeroInteractions(mockReader); + } + + @Test + @SuppressWarnings("ArraysAsListPrimitiveArray") + public void testCombiningAddBeforeRead() throws Exception { + GroupingState value = underTest.state(NAMESPACE, COMBINING_ADDR); + + SettableFuture> future = SettableFuture.create(); + when(mockReader.bagFuture(eq(COMBINING_KEY), eq(STATE_FAMILY), Mockito.>any())) + .thenReturn(future); + + value.readLater(); + + value.add(5); + value.add(6); + waitAndSet(future, Arrays.asList(new int[] {8}, new int[] {10}), 200); + assertThat(value.read(), Matchers.equalTo(29)); + + // That get "compressed" the combiner. So, the underlying future should change: + future.set(Collections.singletonList(new int[] {29})); + + value.add(2); + assertThat(value.read(), Matchers.equalTo(31)); + } + + @Test + public void testCombiningClearBeforeRead() throws Exception { + GroupingState value = underTest.state(NAMESPACE, COMBINING_ADDR); + + value.clear(); + value.readLater(); + + value.add(5); + value.add(6); + assertThat(value.read(), Matchers.equalTo(11)); + + value.add(2); + assertThat(value.read(), Matchers.equalTo(13)); + + // Shouldn't need to read from windmill for this because we immediately cleared.. + Mockito.verifyZeroInteractions(mockReader); + } + + @Test + @SuppressWarnings("ArraysAsListPrimitiveArray") + public void testCombiningIsEmpty() throws Exception { + GroupingState value = underTest.state(NAMESPACE, COMBINING_ADDR); + + SettableFuture> future = SettableFuture.create(); + when(mockReader.bagFuture(eq(COMBINING_KEY), eq(STATE_FAMILY), Mockito.>any())) + .thenReturn(future); + ReadableState result = value.isEmpty().readLater(); + ArgumentCaptor byteString = ArgumentCaptor.forClass(ByteString.class); + + // Note that we do expect the third argument - the coder - to be equal to accumCoder, but that + // is possibly overspecified and currently trips an issue in the SDK where identical coders are + // not #equals(). + // + // What matters is that a future is created, hence a Windmill RPC sent. + Mockito.verify(mockReader) + .bagFuture(byteString.capture(), eq(STATE_FAMILY), Mockito.>any()); + assertThat(byteString.getValue(), byteStringEq(COMBINING_KEY)); + + waitAndSet(future, Collections.singletonList(new int[] {29}), 200); + assertThat(result.read(), Matchers.is(false)); + } + + @Test + public void testCombiningIsEmptyAfterClear() throws Exception { + GroupingState value = underTest.state(NAMESPACE, COMBINING_ADDR); + + value.clear(); + ReadableState result = value.isEmpty(); + Mockito.verify(mockReader, never()).bagFuture(COMBINING_KEY, STATE_FAMILY, accumCoder); + assertThat(result.read(), Matchers.is(true)); + + value.add(87); + assertThat(result.read(), Matchers.is(false)); + } + + @Test + public void testCombiningAddPersist() throws Exception { + disableCompactOnWrite(); + + GroupingState value = underTest.state(NAMESPACE, COMBINING_ADDR); + + value.add(5); + value.add(6); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getBagUpdatesCount()); + + TagBag bagUpdates = commitBuilder.getBagUpdates(0); + assertEquals(COMBINING_KEY, bagUpdates.getTag()); + assertEquals(1, bagUpdates.getValuesCount()); + assertEquals( + 11, CoderUtils.decodeFromByteArray(accumCoder, bagUpdates.getValues(0).toByteArray())[0]); + + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testCombiningAddPersistWithCompact() throws Exception { + forceCompactOnWrite(); + + Mockito.when( + mockReader.bagFuture( + org.mockito.Matchers.any(), + org.mockito.Matchers.any(), + org.mockito.Matchers.>any())) + .thenReturn(Futures.immediateFuture(ImmutableList.of(new int[] {40}, new int[] {60}))); + + GroupingState value = underTest.state(NAMESPACE, COMBINING_ADDR); + + value.add(5); + value.add(6); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getBagUpdatesCount()); + TagBag bagUpdates = commitBuilder.getBagUpdates(0); + assertEquals(COMBINING_KEY, bagUpdates.getTag()); + assertEquals(1, bagUpdates.getValuesCount()); + assertTrue(bagUpdates.getDeleteAll()); + assertEquals( + 111, CoderUtils.decodeFromByteArray(accumCoder, bagUpdates.getValues(0).toByteArray())[0]); + } + + @Test + public void testCombiningClearPersist() throws Exception { + disableCompactOnWrite(); + + GroupingState value = underTest.state(NAMESPACE, COMBINING_ADDR); + + value.clear(); + value.add(5); + value.add(6); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getBagUpdatesCount()); + + TagBag tagBag = commitBuilder.getBagUpdates(0); + assertEquals(COMBINING_KEY, tagBag.getTag()); + assertEquals(STATE_FAMILY, tagBag.getStateFamily()); + assertTrue(tagBag.getDeleteAll()); + assertEquals(1, tagBag.getValuesCount()); + assertEquals( + 11, CoderUtils.decodeFromByteArray(accumCoder, tagBag.getValues(0).toByteArray())[0]); + + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testNewCombiningNoFetch() throws Exception { + GroupingState value = underTestNewKey.state(NAMESPACE, COMBINING_ADDR); + + assertThat(value.isEmpty().read(), Matchers.is(true)); + assertThat(value.read(), Matchers.is(Sum.ofIntegers().identity())); + assertThat(value.isEmpty().read(), Matchers.is(false)); + + // Shouldn't need to read from windmill for this. + Mockito.verifyZeroInteractions(mockReader); + } + + @Test + public void testWatermarkAddBeforeReadEarliest() throws Exception { + StateTag addr = + StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST); + WatermarkHoldState bag = underTest.state(NAMESPACE, addr); + + SettableFuture future = SettableFuture.create(); + when(mockReader.watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY)).thenReturn(future); + + bag.readLater(); + + bag.add(new Instant(3000)); + waitAndSet(future, new Instant(2000), 200); + assertThat(bag.read(), Matchers.equalTo(new Instant(2000))); + + Mockito.verify(mockReader, times(2)).watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY); + Mockito.verifyNoMoreInteractions(mockReader); + + // Adding another value doesn't create another future, but does update the result. + bag.add(new Instant(1000)); + assertThat(bag.read(), Matchers.equalTo(new Instant(1000))); + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testWatermarkAddBeforeReadLatest() throws Exception { + StateTag addr = + StateTags.watermarkStateInternal("watermark", TimestampCombiner.LATEST); + WatermarkHoldState bag = underTest.state(NAMESPACE, addr); + + SettableFuture future = SettableFuture.create(); + when(mockReader.watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY)).thenReturn(future); + + // Suggesting we will read it later should get a future from the underlying WindmillStateReader + bag.readLater(); + + // Actually reading it will request another future, and get the same one, from + // WindmillStateReader + bag.add(new Instant(3000)); + waitAndSet(future, new Instant(2000), 200); + assertThat(bag.read(), Matchers.equalTo(new Instant(3000))); + + Mockito.verify(mockReader, times(2)).watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY); + Mockito.verifyNoMoreInteractions(mockReader); + + // Adding another value doesn't create another future, but does update the result. + bag.add(new Instant(3000)); + assertThat(bag.read(), Matchers.equalTo(new Instant(3000))); + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testWatermarkAddBeforeReadEndOfWindow() throws Exception { + StateTag addr = + StateTags.watermarkStateInternal("watermark", TimestampCombiner.END_OF_WINDOW); + WatermarkHoldState bag = underTest.state(NAMESPACE, addr); + + SettableFuture future = SettableFuture.create(); + when(mockReader.watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY)).thenReturn(future); + + // Requests a future once + bag.readLater(); + + bag.add(new Instant(3000)); + waitAndSet(future, new Instant(3000), 200); + // read() requests a future again, receiving the same one + assertThat(bag.read(), Matchers.equalTo(new Instant(3000))); + + Mockito.verify(mockReader, times(2)).watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY); + Mockito.verifyNoMoreInteractions(mockReader); + + // Adding another value doesn't create another future, but does update the result. + bag.add(new Instant(3000)); + assertThat(bag.read(), Matchers.equalTo(new Instant(3000))); + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testWatermarkClearBeforeRead() throws Exception { + StateTag addr = + StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST); + + WatermarkHoldState bag = underTest.state(NAMESPACE, addr); + + bag.clear(); + assertThat(bag.read(), Matchers.nullValue()); + + bag.add(new Instant(300)); + assertThat(bag.read(), Matchers.equalTo(new Instant(300))); + + // Shouldn't need to read from windmill because the value is already available. + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testWatermarkPersistEarliest() throws Exception { + StateTag addr = + StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST); + WatermarkHoldState bag = underTest.state(NAMESPACE, addr); + + bag.add(new Instant(1000)); + bag.add(new Instant(2000)); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getWatermarkHoldsCount()); + + Windmill.WatermarkHold watermarkHold = commitBuilder.getWatermarkHolds(0); + assertEquals(key(NAMESPACE, "watermark"), watermarkHold.getTag()); + assertEquals(TimeUnit.MILLISECONDS.toMicros(1000), watermarkHold.getTimestamps(0)); + + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testWatermarkPersistLatestEmpty() throws Exception { + StateTag addr = + StateTags.watermarkStateInternal("watermark", TimestampCombiner.LATEST); + WatermarkHoldState hold = underTest.state(NAMESPACE, addr); + + hold.add(new Instant(1000)); + hold.add(new Instant(2000)); + + when(mockReader.watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY)) + .thenReturn(Futures.immediateFuture(null)); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getWatermarkHoldsCount()); + + Windmill.WatermarkHold watermarkHold = commitBuilder.getWatermarkHolds(0); + assertEquals(key(NAMESPACE, "watermark"), watermarkHold.getTag()); + assertEquals(TimeUnit.MILLISECONDS.toMicros(2000), watermarkHold.getTimestamps(0)); + + Mockito.verify(mockReader).watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY); + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testWatermarkPersistLatestWindmillWins() throws Exception { + StateTag addr = + StateTags.watermarkStateInternal("watermark", TimestampCombiner.LATEST); + WatermarkHoldState hold = underTest.state(NAMESPACE, addr); + + hold.add(new Instant(1000)); + hold.add(new Instant(2000)); + + when(mockReader.watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY)) + .thenReturn(Futures.immediateFuture(new Instant(4000))); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getWatermarkHoldsCount()); + + Windmill.WatermarkHold watermarkHold = commitBuilder.getWatermarkHolds(0); + assertEquals(key(NAMESPACE, "watermark"), watermarkHold.getTag()); + assertEquals(TimeUnit.MILLISECONDS.toMicros(4000), watermarkHold.getTimestamps(0)); + + Mockito.verify(mockReader).watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY); + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testWatermarkPersistLatestLocalAdditionsWin() throws Exception { + StateTag addr = + StateTags.watermarkStateInternal("watermark", TimestampCombiner.LATEST); + WatermarkHoldState hold = underTest.state(NAMESPACE, addr); + + hold.add(new Instant(1000)); + hold.add(new Instant(2000)); + + when(mockReader.watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY)) + .thenReturn(Futures.immediateFuture(new Instant(500))); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getWatermarkHoldsCount()); + + Windmill.WatermarkHold watermarkHold = commitBuilder.getWatermarkHolds(0); + assertEquals(key(NAMESPACE, "watermark"), watermarkHold.getTag()); + assertEquals(TimeUnit.MILLISECONDS.toMicros(2000), watermarkHold.getTimestamps(0)); + + Mockito.verify(mockReader).watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY); + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testWatermarkPersistEndOfWindow() throws Exception { + StateTag addr = + StateTags.watermarkStateInternal("watermark", TimestampCombiner.END_OF_WINDOW); + WatermarkHoldState hold = underTest.state(NAMESPACE, addr); + + hold.add(new Instant(2000)); + hold.add(new Instant(2000)); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getWatermarkHoldsCount()); + + Windmill.WatermarkHold watermarkHold = commitBuilder.getWatermarkHolds(0); + assertEquals(key(NAMESPACE, "watermark"), watermarkHold.getTag()); + assertEquals(TimeUnit.MILLISECONDS.toMicros(2000), watermarkHold.getTimestamps(0)); + + // Blind adds should not need to read the future. + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testWatermarkClearPersist() throws Exception { + StateTag addr = + StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST); + WatermarkHoldState hold = underTest.state(NAMESPACE, addr); + + hold.add(new Instant(500)); + hold.clear(); + hold.add(new Instant(1000)); + hold.add(new Instant(2000)); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getWatermarkHoldsCount()); + + Windmill.WatermarkHold clearAndUpdate = commitBuilder.getWatermarkHolds(0); + assertEquals(key(NAMESPACE, "watermark"), clearAndUpdate.getTag()); + assertEquals(1, clearAndUpdate.getTimestampsCount()); + assertEquals(TimeUnit.MILLISECONDS.toMicros(1000), clearAndUpdate.getTimestamps(0)); + + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testWatermarkPersistEmpty() throws Exception { + StateTag addr = + StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST); + WatermarkHoldState bag = underTest.state(NAMESPACE, addr); + + bag.add(new Instant(500)); + bag.clear(); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + // 1 bag update corresponds to deletion. There shouldn't be a bag update adding items. + assertEquals(1, commitBuilder.getWatermarkHoldsCount()); + } + + @Test + public void testNewWatermarkNoFetch() throws Exception { + StateTag addr = + StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST); + + WatermarkHoldState bag = underTestNewKey.state(NAMESPACE, addr); + assertThat(bag.read(), Matchers.nullValue()); + + // Shouldn't need to read from windmill for this. + Mockito.verifyZeroInteractions(mockReader); + } + + @Test + public void testValueSetBeforeRead() throws Exception { + StateTag> addr = StateTags.value("value", StringUtf8Coder.of()); + ValueState value = underTest.state(NAMESPACE, addr); + + value.write("Hello"); + + assertEquals("Hello", value.read()); + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testValueClearBeforeRead() throws Exception { + StateTag> addr = StateTags.value("value", StringUtf8Coder.of()); + ValueState value = underTest.state(NAMESPACE, addr); + + value.clear(); + + assertNull(value.read()); + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testValueRead() throws Exception { + StateTag> addr = StateTags.value("value", StringUtf8Coder.of()); + ValueState value = underTest.state(NAMESPACE, addr); + + SettableFuture future = SettableFuture.create(); + when(mockReader.valueFuture(key(NAMESPACE, "value"), STATE_FAMILY, StringUtf8Coder.of())) + .thenReturn(future); + waitAndSet(future, "World", 200); + + assertEquals("World", value.read()); + } + + @Test + public void testValueSetPersist() throws Exception { + StateTag> addr = StateTags.value("value", StringUtf8Coder.of()); + ValueState value = underTest.state(NAMESPACE, addr); + + value.write("Hi"); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getValueUpdatesCount()); + TagValue valueUpdate = commitBuilder.getValueUpdates(0); + assertEquals(key(NAMESPACE, "value"), valueUpdate.getTag()); + assertEquals("Hi", valueUpdate.getValue().getData().toStringUtf8()); + assertTrue(valueUpdate.isInitialized()); + + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testValueClearPersist() throws Exception { + StateTag> addr = StateTags.value("value", StringUtf8Coder.of()); + ValueState value = underTest.state(NAMESPACE, addr); + + value.write("Hi"); + value.clear(); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(1, commitBuilder.getValueUpdatesCount()); + TagValue valueUpdate = commitBuilder.getValueUpdates(0); + assertEquals(key(NAMESPACE, "value"), valueUpdate.getTag()); + assertEquals(0, valueUpdate.getValue().getData().size()); + + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testValueNoChangePersist() throws Exception { + StateTag> addr = StateTags.value("value", StringUtf8Coder.of()); + underTest.state(NAMESPACE, addr); + + Windmill.WorkItemCommitRequest.Builder commitBuilder = + Windmill.WorkItemCommitRequest.newBuilder(); + underTest.persist(commitBuilder); + + assertEquals(0, commitBuilder.getValueUpdatesCount()); + + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testNewValueNoFetch() throws Exception { + StateTag> addr = StateTags.value("value", StringUtf8Coder.of()); + ValueState value = underTestNewKey.state(NAMESPACE, addr); + + assertNull(value.read()); + + // Shouldn't need to read from windmill for this. + Mockito.verifyZeroInteractions(mockReader); + } + + @Test + public void testCachedValue() throws Exception { + StateTag> addr = StateTags.value("value", StringUtf8Coder.of()); + ValueState value = underTest.state(NAMESPACE, addr); + + assertEquals(0, cache.getWeight()); + + value.write("Hi"); + underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); + + assertEquals(132, cache.getWeight()); + + resetUnderTest(); + value = underTest.state(NAMESPACE, addr); + assertEquals("Hi", value.read()); + value.clear(); + underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); + + assertEquals(130, cache.getWeight()); + + resetUnderTest(); + value = underTest.state(NAMESPACE, addr); + assertNull(value.read()); + underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); + + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testCachedBag() throws Exception { + StateTag> addr = StateTags.bag("bag", StringUtf8Coder.of()); + BagState bag = underTest.state(NAMESPACE, addr); + + assertEquals(0, cache.getWeight()); + + SettableFuture> future = SettableFuture.create(); + when(mockReader.bagFuture(key(NAMESPACE, "bag"), STATE_FAMILY, StringUtf8Coder.of())) + .thenReturn(future); + + bag.readLater(); + + assertEquals(0, cache.getWeight()); + + bag.add("hello"); + waitAndSet(future, weightedList("world"), 200); + Iterable readResult1 = bag.read(); + assertThat(readResult1, Matchers.containsInAnyOrder("hello", "world")); + + underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); + + assertEquals(140, cache.getWeight()); + + resetUnderTest(); + bag = underTest.state(NAMESPACE, addr); + bag.add("goodbye"); + + // Make sure that cached iterables have not changed after persist+add. + assertThat(readResult1, Matchers.containsInAnyOrder("hello", "world")); + + Iterable readResult2 = bag.read(); + assertThat(readResult2, Matchers.containsInAnyOrder("hello", "world", "goodbye")); + bag.clear(); + // Make sure that cached iterables have not changed after clear. + assertThat(readResult2, Matchers.containsInAnyOrder("hello", "world", "goodbye")); + bag.add("new"); + // Make sure that cached iterables have not changed after clear+add. + assertThat(readResult2, Matchers.containsInAnyOrder("hello", "world", "goodbye")); + + underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); + + assertEquals(133, cache.getWeight()); + + resetUnderTest(); + bag = underTest.state(NAMESPACE, addr); + bag.add("new2"); + assertThat(bag.read(), Matchers.containsInAnyOrder("new", "new2")); + bag.clear(); + bag.add("new3"); + + underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); + + assertEquals(134, cache.getWeight()); + + resetUnderTest(); + bag = underTest.state(NAMESPACE, addr); + assertThat(bag.read(), Matchers.containsInAnyOrder("new3")); + underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); + + Mockito.verify(mockReader, times(2)) + .bagFuture(key(NAMESPACE, "bag"), STATE_FAMILY, StringUtf8Coder.of()); + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + public void testCachedWatermarkHold() throws Exception { + StateTag addr = + StateTags.watermarkStateInternal("watermark", TimestampCombiner.EARLIEST); + WatermarkHoldState hold = underTest.state(NAMESPACE, addr); + + SettableFuture future = SettableFuture.create(); + when(mockReader.watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY)).thenReturn(future); + + assertEquals(0, cache.getWeight()); + + hold.readLater(); + + hold.add(new Instant(3000)); + waitAndSet(future, new Instant(2000), 200); + assertThat(hold.read(), Matchers.equalTo(new Instant(2000))); + + underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); + + assertEquals(138, cache.getWeight()); + + resetUnderTest(); + hold = underTest.state(NAMESPACE, addr); + assertThat(hold.read(), Matchers.equalTo(new Instant(2000))); + hold.clear(); + + underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); + + assertEquals(138, cache.getWeight()); + + resetUnderTest(); + hold = underTest.state(NAMESPACE, addr); + assertNull(hold.read()); + underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); + + Mockito.verify(mockReader, times(2)).watermarkFuture(key(NAMESPACE, "watermark"), STATE_FAMILY); + Mockito.verifyNoMoreInteractions(mockReader); + } + + @Test + @SuppressWarnings("ArraysAsListPrimitiveArray") + public void testCachedCombining() throws Exception { + GroupingState value = underTest.state(NAMESPACE, COMBINING_ADDR); + + SettableFuture> future = SettableFuture.create(); + when(mockReader.bagFuture( + eq(key(NAMESPACE, "combining")), eq(STATE_FAMILY), Mockito.>any())) + .thenReturn(future); + + assertEquals(0, cache.getWeight()); + + value.readLater(); + + value.add(1); + waitAndSet(future, Collections.singletonList(new int[] {2}), 200); + assertThat(value.read(), Matchers.equalTo(3)); + + underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); + + assertEquals(131, cache.getWeight()); + + resetUnderTest(); + value = underTest.state(NAMESPACE, COMBINING_ADDR); + assertThat(value.read(), Matchers.equalTo(3)); + value.add(3); + assertThat(value.read(), Matchers.equalTo(6)); + value.clear(); + + underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); + + assertEquals(130, cache.getWeight()); + + resetUnderTest(); + value = underTest.state(NAMESPACE, COMBINING_ADDR); + assertThat(value.read(), Matchers.equalTo(0)); + underTest.persist(Windmill.WorkItemCommitRequest.newBuilder()); + + // Note that we do expect the third argument - the coder - to be equal to accumCoder, but that + // is possibly overspecified and currently trips an issue in the SDK where identical coders are + // not #equals(). + // + // What matters is the number of futures created, hence Windmill RPCs. + Mockito.verify(mockReader, times(2)) + .bagFuture(eq(key(NAMESPACE, "combining")), eq(STATE_FAMILY), Mockito.>any()); + Mockito.verifyNoMoreInteractions(mockReader); + } + + private void disableCompactOnWrite() { + WindmillStateInternals.COMPACT_NOW.set(() -> false); + } + + private void forceCompactOnWrite() { + WindmillStateInternals.COMPACT_NOW.set(() -> true); + } + + private static class MultimapEntryUpdate { + String key; + Iterable values; + boolean deleteAll; + + public MultimapEntryUpdate(String key, Iterable values, boolean deleteAll) { + this.key = key; + this.values = values; + this.deleteAll = deleteAll; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof MultimapEntryUpdate)) return false; + MultimapEntryUpdate that = (MultimapEntryUpdate) o; + return deleteAll == that.deleteAll + && Objects.equals(key, that.key) + && Objects.equals(values, that.values); + } + + @Override + public int hashCode() { + return Objects.hash(key, values, deleteAll); + } + } +} diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateReaderTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateReaderTest.java new file mode 100644 index 0000000000000..b8c4803a8f341 --- /dev/null +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateReaderTest.java @@ -0,0 +1,1724 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.dataflow.worker.windmill.state; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; + +import com.google.api.client.util.Lists; +import com.google.common.collect.Maps; +import java.io.IOException; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Future; +import org.apache.beam.runners.dataflow.worker.KeyTokenInvalidException; +import org.apache.beam.runners.dataflow.worker.MetricTrackingWindmillServerStub; +import org.apache.beam.runners.dataflow.worker.WindmillStateTestUtils; +import org.apache.beam.runners.dataflow.worker.WindmillTimeUtils; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataRequest; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.SortedListEntry; +import org.apache.beam.runners.dataflow.worker.windmill.Windmill.SortedListRange; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.coders.VarIntCoder; +import org.apache.beam.sdk.transforms.windowing.BoundedWindow; +import org.apache.beam.sdk.util.ByteStringOutputStream; +import org.apache.beam.sdk.values.TimestampedValue; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Range; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; +import org.hamcrest.Matchers; +import org.joda.time.Instant; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +/** Tests for {@link WindmillStateReader}. */ +@RunWith(JUnit4.class) +@SuppressWarnings({ + "FutureReturnValueIgnored", +}) +public class WindmillStateReaderTest { + private static final VarIntCoder INT_CODER = VarIntCoder.of(); + + private static final String COMPUTATION = "computation"; + private static final ByteString DATA_KEY = ByteString.copyFromUtf8("DATA_KEY"); + private static final long SHARDING_KEY = 17L; + private static final long WORK_TOKEN = 5043L; + private static final long CONT_POSITION = 1391631351L; + + private static final ByteString STATE_KEY_PREFIX = ByteString.copyFromUtf8("key"); + private static final ByteString STATE_MULTIMAP_KEY_1 = ByteString.copyFromUtf8("multimapkey1"); + private static final ByteString STATE_MULTIMAP_KEY_2 = ByteString.copyFromUtf8("multimapkey2"); + private static final ByteString STATE_MULTIMAP_KEY_3 = ByteString.copyFromUtf8("multimapkey3"); + private static final ByteString STATE_MULTIMAP_CONT_1 = ByteString.copyFromUtf8("continuation_1"); + private static final ByteString STATE_MULTIMAP_CONT_2 = ByteString.copyFromUtf8("continuation_2"); + private static final ByteString STATE_KEY_1 = ByteString.copyFromUtf8("key1"); + private static final ByteString STATE_KEY_2 = ByteString.copyFromUtf8("key2"); + private static final String STATE_FAMILY = "family"; + + private static void assertNoReader(Object obj) throws Exception { + WindmillStateTestUtils.assertNoReference(obj, WindmillStateReader.class); + } + + @Mock private MetricTrackingWindmillServerStub mockWindmill; + + private WindmillStateReader underTest; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + + underTest = + new WindmillStateReader(mockWindmill, COMPUTATION, DATA_KEY, SHARDING_KEY, WORK_TOKEN); + } + + private Windmill.Value intValue(int value) throws IOException { + return Windmill.Value.newBuilder() + .setData(intData(value)) + .setTimestamp( + WindmillTimeUtils.harnessToWindmillTimestamp(BoundedWindow.TIMESTAMP_MAX_VALUE)) + .build(); + } + + private ByteString intData(int value) throws IOException { + ByteStringOutputStream output = new ByteStringOutputStream(); + INT_CODER.encode(value, output, Coder.Context.OUTER); + return output.toByteString(); + } + + @Test + public void testReadMultimapSingleEntry() throws Exception { + Future> future = + underTest.multimapFetchSingleEntryFuture( + STATE_MULTIMAP_KEY_1, STATE_KEY_1, STATE_FAMILY, INT_CODER); + Mockito.verifyNoMoreInteractions(mockWindmill); + + Windmill.KeyedGetDataRequest.Builder expectedRequest = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addMultimapsToFetch( + Windmill.TagMultimapFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchEntryNamesOnly(false) + .addEntriesToFetch( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_1) + .setFetchMaxBytes(WindmillStateReader.INITIAL_MAX_MULTIMAP_BYTES) + .build())); + + Windmill.KeyedGetDataResponse.Builder response = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagMultimaps( + Windmill.TagMultimapFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_1) + .addAllValues(Arrays.asList(intData(5), intData(6))))); + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) + .thenReturn(response.build()); + + Iterable results = future.get(); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); + + assertThat(results, Matchers.containsInAnyOrder(5, 6)); + Mockito.verifyNoMoreInteractions(mockWindmill); + assertNoReader(future); + } + + @Test + public void testReadMultimapSingleEntryPaginated() throws Exception { + Future> future = + underTest.multimapFetchSingleEntryFuture( + STATE_MULTIMAP_KEY_1, STATE_KEY_1, STATE_FAMILY, INT_CODER); + Mockito.verifyNoMoreInteractions(mockWindmill); + + Windmill.KeyedGetDataRequest.Builder expectedRequest1 = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addMultimapsToFetch( + Windmill.TagMultimapFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchEntryNamesOnly(false) + .addEntriesToFetch( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_1) + .setFetchMaxBytes(WindmillStateReader.INITIAL_MAX_MULTIMAP_BYTES) + .build())); + + Windmill.KeyedGetDataResponse.Builder response1 = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagMultimaps( + Windmill.TagMultimapFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_1) + .addAllValues(Arrays.asList(intData(5), intData(6))) + .setContinuationPosition(500))); + Windmill.KeyedGetDataRequest.Builder expectedRequest2 = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_CONTINUATION_KEY_BYTES) + .addMultimapsToFetch( + Windmill.TagMultimapFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchEntryNamesOnly(false) + .addEntriesToFetch( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_1) + .setFetchMaxBytes(WindmillStateReader.CONTINUATION_MAX_MULTIMAP_BYTES) + .setRequestPosition(500) + .build())); + + Windmill.KeyedGetDataResponse.Builder response2 = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagMultimaps( + Windmill.TagMultimapFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_1) + .addAllValues(Arrays.asList(intData(7), intData(8))) + .setContinuationPosition(800) + .setRequestPosition(500))); + Windmill.KeyedGetDataRequest.Builder expectedRequest3 = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_CONTINUATION_KEY_BYTES) + .addMultimapsToFetch( + Windmill.TagMultimapFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchEntryNamesOnly(false) + .addEntriesToFetch( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_1) + .setFetchMaxBytes(WindmillStateReader.CONTINUATION_MAX_MULTIMAP_BYTES) + .setRequestPosition(800) + .build())); + + Windmill.KeyedGetDataResponse.Builder response3 = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagMultimaps( + Windmill.TagMultimapFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_1) + .addAllValues(Arrays.asList(intData(9), intData(10))) + .setRequestPosition(800))); + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest1.build())) + .thenReturn(response1.build()); + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest2.build())) + .thenReturn(response2.build()); + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest3.build())) + .thenReturn(response3.build()); + + Iterable results = future.get(); + + assertThat(results, Matchers.contains(5, 6, 7, 8, 9, 10)); + + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest1.build()); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest2.build()); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest3.build()); + Mockito.verifyNoMoreInteractions(mockWindmill); + // NOTE: The future will still contain a reference to the underlying reader, thus not calling + // assertNoReader(future). + } + + // check whether the two TagMultimapFetchRequests equal to each other, ignoring the order of + // entries and the order of values in each entry. + private static void assertMultimapFetchRequestEqual( + Windmill.TagMultimapFetchRequest req1, Windmill.TagMultimapFetchRequest req2) { + assertMultimapEntriesEqual(req1.getEntriesToFetchList(), req2.getEntriesToFetchList()); + assertEquals( + req1.toBuilder().clearEntriesToFetch().build(), + req2.toBuilder().clearEntriesToFetch().build()); + } + + private static void assertMultimapEntriesEqual( + List left, List right) { + Map map = Maps.newHashMap(); + for (Windmill.TagMultimapEntry entry : left) { + map.put(entry.getEntryName(), entry); + } + for (Windmill.TagMultimapEntry entry : right) { + assertTrue(map.containsKey(entry.getEntryName())); + Windmill.TagMultimapEntry that = map.remove(entry.getEntryName()); + if (entry.getValuesCount() == 0) { + assertEquals(0, that.getValuesCount()); + } else { + assertThat(entry.getValuesList(), Matchers.containsInAnyOrder(that.getValuesList())); + } + assertEquals(entry.toBuilder().clearValues().build(), that.toBuilder().clearValues().build()); + } + assertTrue(map.isEmpty()); + } + + @Test + public void testReadMultimapMultipleEntries() throws Exception { + Future> future1 = + underTest.multimapFetchSingleEntryFuture( + STATE_MULTIMAP_KEY_1, STATE_KEY_1, STATE_FAMILY, INT_CODER); + Future> future2 = + underTest.multimapFetchSingleEntryFuture( + STATE_MULTIMAP_KEY_2, STATE_KEY_1, STATE_FAMILY, INT_CODER); + Mockito.verifyNoMoreInteractions(mockWindmill); + + Windmill.KeyedGetDataRequest.Builder expectedRequest = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addMultimapsToFetch( + Windmill.TagMultimapFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchEntryNamesOnly(false) + .addEntriesToFetch( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_1) + .setFetchMaxBytes(WindmillStateReader.INITIAL_MAX_MULTIMAP_BYTES) + .build()) + .addEntriesToFetch( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_2) + .setFetchMaxBytes(WindmillStateReader.INITIAL_MAX_MULTIMAP_BYTES) + .build())); + + Windmill.KeyedGetDataResponse.Builder response = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagMultimaps( + Windmill.TagMultimapFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_1) + .addAllValues(Arrays.asList(intData(5), intData(6)))) + .addEntries( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_2) + .addAllValues(Arrays.asList(intData(15), intData(16))))); + when(mockWindmill.getStateData(ArgumentMatchers.eq(COMPUTATION), ArgumentMatchers.any())) + .thenReturn(response.build()); + + Iterable results1 = future1.get(); + Iterable results2 = future2.get(); + + final ArgumentCaptor requestCaptor = + ArgumentCaptor.forClass(Windmill.KeyedGetDataRequest.class); + Mockito.verify(mockWindmill) + .getStateData(ArgumentMatchers.eq(COMPUTATION), requestCaptor.capture()); + assertMultimapFetchRequestEqual( + expectedRequest.build().getMultimapsToFetch(0), + requestCaptor.getValue().getMultimapsToFetch(0)); + + assertThat(results1, Matchers.containsInAnyOrder(5, 6)); + assertThat(results2, Matchers.containsInAnyOrder(15, 16)); + + Mockito.verifyNoMoreInteractions(mockWindmill); + assertNoReader(future1); + assertNoReader(future2); + } + + @Test + public void testReadMultimapMultipleEntriesWithPagination() throws Exception { + Future> future1 = + underTest.multimapFetchSingleEntryFuture( + STATE_MULTIMAP_KEY_1, STATE_KEY_1, STATE_FAMILY, INT_CODER); + Future> future2 = + underTest.multimapFetchSingleEntryFuture( + STATE_MULTIMAP_KEY_2, STATE_KEY_1, STATE_FAMILY, INT_CODER); + Mockito.verifyNoMoreInteractions(mockWindmill); + + Windmill.KeyedGetDataRequest.Builder expectedRequest1 = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addMultimapsToFetch( + Windmill.TagMultimapFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchEntryNamesOnly(false) + .addEntriesToFetch( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_1) + .setFetchMaxBytes(WindmillStateReader.INITIAL_MAX_MULTIMAP_BYTES) + .build()) + .addEntriesToFetch( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_2) + .setFetchMaxBytes(WindmillStateReader.INITIAL_MAX_MULTIMAP_BYTES) + .build())); + + Windmill.KeyedGetDataResponse.Builder response1 = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagMultimaps( + Windmill.TagMultimapFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_1) + .addAllValues(Arrays.asList(intData(5), intData(6))) + .setContinuationPosition(800)) + .addEntries( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_2) + .addAllValues(Arrays.asList(intData(15), intData(16))))); + Windmill.KeyedGetDataRequest.Builder expectedRequest2 = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_CONTINUATION_KEY_BYTES) + .addMultimapsToFetch( + Windmill.TagMultimapFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchEntryNamesOnly(false) + .addEntriesToFetch( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_1) + .setFetchMaxBytes(WindmillStateReader.CONTINUATION_MAX_MULTIMAP_BYTES) + .setRequestPosition(800) + .build())); + Windmill.KeyedGetDataResponse.Builder response2 = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagMultimaps( + Windmill.TagMultimapFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_1) + .addAllValues(Arrays.asList(intData(7), intData(8))) + .setRequestPosition(800))); + when(mockWindmill.getStateData(ArgumentMatchers.eq(COMPUTATION), ArgumentMatchers.any())) + .thenReturn(response1.build()) + .thenReturn(response2.build()); + + Iterable results1 = future1.get(); + Iterable results2 = future2.get(); + + assertThat(results1, Matchers.containsInAnyOrder(5, 6, 7, 8)); + assertThat(results2, Matchers.containsInAnyOrder(15, 16)); + + final ArgumentCaptor requestCaptor = + ArgumentCaptor.forClass(Windmill.KeyedGetDataRequest.class); + Mockito.verify(mockWindmill, times(2)) + .getStateData(ArgumentMatchers.eq(COMPUTATION), requestCaptor.capture()); + assertMultimapFetchRequestEqual( + expectedRequest1.build().getMultimapsToFetch(0), + requestCaptor.getAllValues().get(0).getMultimapsToFetch(0)); + assertMultimapFetchRequestEqual( + expectedRequest2.build().getMultimapsToFetch(0), + requestCaptor.getAllValues().get(1).getMultimapsToFetch(0)); + Mockito.verifyNoMoreInteractions(mockWindmill); + // NOTE: The future will still contain a reference to the underlying reader, thus not calling + // assertNoReader(future). + } + + @Test + public void testReadMultimapKeys() throws Exception { + Future>>> future = + underTest.multimapFetchAllFuture(true, STATE_KEY_1, STATE_FAMILY, INT_CODER); + Mockito.verifyNoMoreInteractions(mockWindmill); + + Windmill.KeyedGetDataRequest.Builder expectedRequest = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addMultimapsToFetch( + Windmill.TagMultimapFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchEntryNamesOnly(true) + .setFetchMaxBytes(WindmillStateReader.INITIAL_MAX_MULTIMAP_BYTES)); + + Windmill.KeyedGetDataResponse.Builder response = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagMultimaps( + Windmill.TagMultimapFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + Windmill.TagMultimapEntry.newBuilder().setEntryName(STATE_MULTIMAP_KEY_1)) + .addEntries( + Windmill.TagMultimapEntry.newBuilder().setEntryName(STATE_MULTIMAP_KEY_2))); + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) + .thenReturn(response.build()); + + Iterable>> results = future.get(); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); + List keys = Lists.newArrayList(); + for (Map.Entry> entry : results) { + keys.add(entry.getKey()); + assertEquals(0, Iterables.size(entry.getValue())); + } + Mockito.verifyNoMoreInteractions(mockWindmill); + + assertThat(keys, Matchers.containsInAnyOrder(STATE_MULTIMAP_KEY_1, STATE_MULTIMAP_KEY_2)); + assertNoReader(future); + } + + @Test + public void testReadMultimapKeysPaginated() throws Exception { + Future>>> future = + underTest.multimapFetchAllFuture(true, STATE_KEY_1, STATE_FAMILY, INT_CODER); + Mockito.verifyNoMoreInteractions(mockWindmill); + + Windmill.KeyedGetDataRequest.Builder expectedRequest1 = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addMultimapsToFetch( + Windmill.TagMultimapFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchEntryNamesOnly(true) + .setFetchMaxBytes(WindmillStateReader.INITIAL_MAX_MULTIMAP_BYTES)); + + Windmill.KeyedGetDataResponse.Builder response1 = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagMultimaps( + Windmill.TagMultimapFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + Windmill.TagMultimapEntry.newBuilder().setEntryName(STATE_MULTIMAP_KEY_1)) + .setContinuationPosition(STATE_MULTIMAP_CONT_1)); + + Windmill.KeyedGetDataRequest.Builder expectedRequest2 = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_CONTINUATION_KEY_BYTES) + .addMultimapsToFetch( + Windmill.TagMultimapFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchEntryNamesOnly(true) + .setFetchMaxBytes(WindmillStateReader.CONTINUATION_MAX_MULTIMAP_BYTES) + .setRequestPosition(STATE_MULTIMAP_CONT_1)); + + Windmill.KeyedGetDataResponse.Builder response2 = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagMultimaps( + Windmill.TagMultimapFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + Windmill.TagMultimapEntry.newBuilder().setEntryName(STATE_MULTIMAP_KEY_2)) + .setRequestPosition(STATE_MULTIMAP_CONT_1) + .setContinuationPosition(STATE_MULTIMAP_CONT_2)); + Windmill.KeyedGetDataRequest.Builder expectedRequest3 = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_CONTINUATION_KEY_BYTES) + .addMultimapsToFetch( + Windmill.TagMultimapFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchEntryNamesOnly(true) + .setFetchMaxBytes(WindmillStateReader.CONTINUATION_MAX_MULTIMAP_BYTES) + .setRequestPosition(STATE_MULTIMAP_CONT_2)); + + Windmill.KeyedGetDataResponse.Builder response3 = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagMultimaps( + Windmill.TagMultimapFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + Windmill.TagMultimapEntry.newBuilder().setEntryName(STATE_MULTIMAP_KEY_3)) + .setRequestPosition(STATE_MULTIMAP_CONT_2)); + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest1.build())) + .thenReturn(response1.build()); + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest2.build())) + .thenReturn(response2.build()); + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest3.build())) + .thenReturn(response3.build()); + + Iterable>> results = future.get(); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest1.build()); + List keys = Lists.newArrayList(); + for (Map.Entry> entry : results) { + keys.add(entry.getKey()); + assertEquals(0, Iterables.size(entry.getValue())); + } + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest2.build()); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest3.build()); + Mockito.verifyNoMoreInteractions(mockWindmill); + + assertThat( + keys, + Matchers.containsInAnyOrder( + STATE_MULTIMAP_KEY_1, STATE_MULTIMAP_KEY_2, STATE_MULTIMAP_KEY_3)); + // NOTE: The future will still contain a reference to the underlying reader, thus not calling + // assertNoReader(future). + } + + @Test + public void testReadMultimapAllEntries() throws Exception { + Future>>> future = + underTest.multimapFetchAllFuture(false, STATE_KEY_1, STATE_FAMILY, INT_CODER); + Mockito.verifyNoMoreInteractions(mockWindmill); + + Windmill.KeyedGetDataRequest.Builder expectedRequest = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addMultimapsToFetch( + Windmill.TagMultimapFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchEntryNamesOnly(false) + .setFetchMaxBytes(WindmillStateReader.INITIAL_MAX_MULTIMAP_BYTES)); + + Windmill.KeyedGetDataResponse.Builder response = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagMultimaps( + Windmill.TagMultimapFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_1) + .addValues(intData(1)) + .addValues(intData(2))) + .addEntries( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_2) + .addValues(intData(10)) + .addValues(intData(20)))); + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) + .thenReturn(response.build()); + + Iterable>> results = future.get(); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); + int foundEntries = 0; + for (Map.Entry> entry : results) { + if (entry.getKey().equals(STATE_MULTIMAP_KEY_1)) { + foundEntries++; + assertThat(entry.getValue(), Matchers.containsInAnyOrder(1, 2)); + } else { + foundEntries++; + assertEquals(STATE_MULTIMAP_KEY_2, entry.getKey()); + assertThat(entry.getValue(), Matchers.containsInAnyOrder(10, 20)); + } + } + assertEquals(2, foundEntries); + Mockito.verifyNoMoreInteractions(mockWindmill); + assertNoReader(future); + } + + private static void assertMultimapEntries( + Iterable>> expected, + List>> actual) { + Map> expectedMap = Maps.newHashMap(); + for (Map.Entry> entry : expected) { + ByteString key = entry.getKey(); + if (!expectedMap.containsKey(key)) expectedMap.put(key, new ArrayList<>()); + entry.getValue().forEach(expectedMap.get(key)::add); + } + for (Map.Entry> entry : actual) { + assertTrue(expectedMap.containsKey(entry.getKey())); + assertThat( + entry.getValue(), + Matchers.containsInAnyOrder(expectedMap.remove(entry.getKey()).toArray())); + } + assertTrue(expectedMap.isEmpty()); + } + + @Test + public void testReadMultimapEntriesPaginated() throws Exception { + Future>>> future = + underTest.multimapFetchAllFuture(false, STATE_KEY_1, STATE_FAMILY, INT_CODER); + Mockito.verifyNoMoreInteractions(mockWindmill); + + Windmill.KeyedGetDataRequest.Builder expectedRequest1 = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addMultimapsToFetch( + Windmill.TagMultimapFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchEntryNamesOnly(false) + .setFetchMaxBytes(WindmillStateReader.INITIAL_MAX_MULTIMAP_BYTES)); + + Windmill.KeyedGetDataResponse.Builder response1 = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagMultimaps( + Windmill.TagMultimapFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_1) + .addValues(intData(1)) + .addValues(intData(2))) + .addEntries( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_2) + .addValues(intData(3)) + .addValues(intData(3))) + .setContinuationPosition(STATE_MULTIMAP_CONT_1)); + + Windmill.KeyedGetDataRequest.Builder expectedRequest2 = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_CONTINUATION_KEY_BYTES) + .addMultimapsToFetch( + Windmill.TagMultimapFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchEntryNamesOnly(false) + .setFetchMaxBytes(WindmillStateReader.CONTINUATION_MAX_MULTIMAP_BYTES) + .setRequestPosition(STATE_MULTIMAP_CONT_1)); + + Windmill.KeyedGetDataResponse.Builder response2 = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagMultimaps( + Windmill.TagMultimapFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_2) + .addValues(intData(2))) + .setRequestPosition(STATE_MULTIMAP_CONT_1) + .setContinuationPosition(STATE_MULTIMAP_CONT_2)); + Windmill.KeyedGetDataRequest.Builder expectedRequest3 = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_CONTINUATION_KEY_BYTES) + .addMultimapsToFetch( + Windmill.TagMultimapFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchEntryNamesOnly(false) + .setFetchMaxBytes(WindmillStateReader.CONTINUATION_MAX_MULTIMAP_BYTES) + .setRequestPosition(STATE_MULTIMAP_CONT_2)); + + Windmill.KeyedGetDataResponse.Builder response3 = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagMultimaps( + Windmill.TagMultimapFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + Windmill.TagMultimapEntry.newBuilder() + .setEntryName(STATE_MULTIMAP_KEY_2) + .addValues(intData(4))) + .setRequestPosition(STATE_MULTIMAP_CONT_2)); + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest1.build())) + .thenReturn(response1.build()); + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest2.build())) + .thenReturn(response2.build()); + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest3.build())) + .thenReturn(response3.build()); + + Iterable>> results = future.get(); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest1.build()); + assertMultimapEntries( + results, + Arrays.asList( + new AbstractMap.SimpleEntry<>(STATE_MULTIMAP_KEY_1, Arrays.asList(1, 2)), + new AbstractMap.SimpleEntry<>(STATE_MULTIMAP_KEY_2, Arrays.asList(3, 3, 2, 4)))); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest2.build()); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest3.build()); + Mockito.verifyNoMoreInteractions(mockWindmill); + // NOTE: The future will still contain a reference to the underlying reader , thus not calling + // assertNoReader(future). + } + + @Test + public void testReadBag() throws Exception { + Future> future = underTest.bagFuture(STATE_KEY_1, STATE_FAMILY, INT_CODER); + Mockito.verifyNoMoreInteractions(mockWindmill); + + Windmill.KeyedGetDataRequest.Builder expectedRequest = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addBagsToFetch( + Windmill.TagBag.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchMaxBytes(WindmillStateReader.INITIAL_MAX_BAG_BYTES)); + + Windmill.KeyedGetDataResponse.Builder response = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addBags( + Windmill.TagBag.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addValues(intData(5)) + .addValues(intData(6))); + + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) + .thenReturn(response.build()); + + Iterable results = future.get(); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); + for (Integer unused : results) { + // Iterate over the results to force loading all the pages. + } + Mockito.verifyNoMoreInteractions(mockWindmill); + + assertThat(results, Matchers.contains(5, 6)); + assertNoReader(future); + } + + @Test + public void testReadBagWithContinuations() throws Exception { + Future> future = underTest.bagFuture(STATE_KEY_1, STATE_FAMILY, INT_CODER); + Mockito.verifyNoMoreInteractions(mockWindmill); + + Windmill.KeyedGetDataRequest.Builder expectedRequest1 = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addBagsToFetch( + Windmill.TagBag.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchMaxBytes(WindmillStateReader.INITIAL_MAX_BAG_BYTES)); + + Windmill.KeyedGetDataResponse.Builder response1 = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addBags( + Windmill.TagBag.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setContinuationPosition(CONT_POSITION) + .addValues(intData(5)) + .addValues(intData(6))); + + Windmill.KeyedGetDataRequest.Builder expectedRequest2 = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_CONTINUATION_KEY_BYTES) + .addBagsToFetch( + Windmill.TagBag.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchMaxBytes(WindmillStateReader.CONTINUATION_MAX_BAG_BYTES) + .setRequestPosition(CONT_POSITION)); + + Windmill.KeyedGetDataResponse.Builder response2 = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addBags( + Windmill.TagBag.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setRequestPosition(CONT_POSITION) + .addValues(intData(7)) + .addValues(intData(8))); + + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest1.build())) + .thenReturn(response1.build()); + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest2.build())) + .thenReturn(response2.build()); + + Iterable results = future.get(); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest1.build()); + for (Integer unused : results) { + // Iterate over the results to force loading all the pages. + } + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest2.build()); + Mockito.verifyNoMoreInteractions(mockWindmill); + + assertThat(results, Matchers.contains(5, 6, 7, 8)); + // NOTE: The future will still contain a reference to the underlying reader , thus not calling + // assertNoReader(future). + } + + @Test + public void testReadSortedList() throws Exception { + long beginning = SortedListRange.getDefaultInstance().getStart(); + long end = SortedListRange.getDefaultInstance().getLimit(); + Future>> future = + underTest.orderedListFuture( + Range.closedOpen(beginning, end), STATE_KEY_1, STATE_FAMILY, INT_CODER); + Mockito.verifyNoMoreInteractions(mockWindmill); + + // Fetch the entire list. + Windmill.KeyedGetDataRequest.Builder expectedRequest = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addSortedListsToFetch( + Windmill.TagSortedListFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addFetchRanges(SortedListRange.newBuilder().setStart(beginning).setLimit(end)) + .setFetchMaxBytes(WindmillStateReader.MAX_ORDERED_LIST_BYTES)); + + Windmill.KeyedGetDataResponse.Builder response = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagSortedLists( + Windmill.TagSortedListFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + SortedListEntry.newBuilder().setValue(intData(5)).setSortKey(5000).setId(5)) + .addEntries( + SortedListEntry.newBuilder().setValue(intData(6)).setSortKey(6000).setId(5)) + .addEntries( + SortedListEntry.newBuilder().setValue(intData(7)).setSortKey(7000).setId(7)) + .addEntries( + SortedListEntry.newBuilder().setValue(intData(8)).setSortKey(8000).setId(8)) + .addFetchRanges( + SortedListRange.newBuilder().setStart(beginning).setLimit(end))); + + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) + .thenReturn(response.build()); + + Iterable> results = future.get(); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); + for (TimestampedValue unused : results) { + // Iterate over the results to force loading all the pages. + } + Mockito.verifyNoMoreInteractions(mockWindmill); + + assertThat( + results, + Matchers.contains( + TimestampedValue.of(5, Instant.ofEpochMilli(5)), + TimestampedValue.of(6, Instant.ofEpochMilli(6)), + TimestampedValue.of(7, Instant.ofEpochMilli(7)), + TimestampedValue.of(8, Instant.ofEpochMilli(8)))); + assertNoReader(future); + } + + @Test + public void testReadSortedListRanges() throws Exception { + Future>> future1 = + underTest.orderedListFuture(Range.closedOpen(0L, 5L), STATE_KEY_1, STATE_FAMILY, INT_CODER); + Future>> future2 = + underTest.orderedListFuture(Range.closedOpen(5L, 6L), STATE_KEY_1, STATE_FAMILY, INT_CODER); + Future>> future3 = + underTest.orderedListFuture( + Range.closedOpen(6L, 10L), STATE_KEY_1, STATE_FAMILY, INT_CODER); + Mockito.verifyNoMoreInteractions(mockWindmill); + + // Fetch the entire list. + Windmill.KeyedGetDataRequest.Builder expectedRequest = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addSortedListsToFetch( + Windmill.TagSortedListFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addFetchRanges(SortedListRange.newBuilder().setStart(0).setLimit(5)) + .setFetchMaxBytes(WindmillStateReader.MAX_ORDERED_LIST_BYTES)) + .addSortedListsToFetch( + Windmill.TagSortedListFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addFetchRanges(SortedListRange.newBuilder().setStart(5).setLimit(6)) + .setFetchMaxBytes(WindmillStateReader.MAX_ORDERED_LIST_BYTES)) + .addSortedListsToFetch( + Windmill.TagSortedListFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addFetchRanges(SortedListRange.newBuilder().setStart(6).setLimit(10)) + .setFetchMaxBytes(WindmillStateReader.MAX_ORDERED_LIST_BYTES)); + + Windmill.KeyedGetDataResponse.Builder response = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagSortedLists( + Windmill.TagSortedListFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + SortedListEntry.newBuilder().setValue(intData(5)).setSortKey(5000).setId(5)) + .addFetchRanges(SortedListRange.newBuilder().setStart(0).setLimit(5))) + .addTagSortedLists( + Windmill.TagSortedListFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + SortedListEntry.newBuilder().setValue(intData(6)).setSortKey(6000).setId(5)) + .addEntries( + SortedListEntry.newBuilder().setValue(intData(7)).setSortKey(7000).setId(7)) + .addFetchRanges(SortedListRange.newBuilder().setStart(5).setLimit(6))) + .addTagSortedLists( + Windmill.TagSortedListFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + SortedListEntry.newBuilder().setValue(intData(8)).setSortKey(8000).setId(8)) + .addFetchRanges(SortedListRange.newBuilder().setStart(6).setLimit(10))); + + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) + .thenReturn(response.build()); + + { + Iterable> results = future1.get(); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); + for (TimestampedValue unused : results) { + // Iterate over the results to force loading all the pages. + } + Mockito.verifyNoMoreInteractions(mockWindmill); + assertThat(results, Matchers.contains(TimestampedValue.of(5, Instant.ofEpochMilli(5)))); + assertNoReader(future1); + } + + { + Iterable> results = future2.get(); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); + for (TimestampedValue unused : results) { + // Iterate over the results to force loading all the pages. + } + Mockito.verifyNoMoreInteractions(mockWindmill); + assertThat( + results, + Matchers.contains( + TimestampedValue.of(6, Instant.ofEpochMilli(6)), + TimestampedValue.of(7, Instant.ofEpochMilli(7)))); + assertNoReader(future2); + } + + { + Iterable> results = future3.get(); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); + for (TimestampedValue unused : results) { + // Iterate over the results to force loading all the pages. + } + Mockito.verifyNoMoreInteractions(mockWindmill); + assertThat(results, Matchers.contains(TimestampedValue.of(8, Instant.ofEpochMilli(8)))); + assertNoReader(future3); + } + } + + @Test + public void testReadSortedListWithContinuations() throws Exception { + long beginning = SortedListRange.getDefaultInstance().getStart(); + long end = SortedListRange.getDefaultInstance().getLimit(); + + Future>> future = + underTest.orderedListFuture( + Range.closedOpen(beginning, end), STATE_KEY_1, STATE_FAMILY, INT_CODER); + + Mockito.verifyNoMoreInteractions(mockWindmill); + + Windmill.KeyedGetDataRequest.Builder expectedRequest1 = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addSortedListsToFetch( + Windmill.TagSortedListFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addFetchRanges(SortedListRange.newBuilder().setStart(beginning).setLimit(end)) + .setFetchMaxBytes(WindmillStateReader.MAX_ORDERED_LIST_BYTES)); + + final ByteString CONT_1 = ByteString.copyFrom("CONTINUATION_1", Charsets.UTF_8); + final ByteString CONT_2 = ByteString.copyFrom("CONTINUATION_2", Charsets.UTF_8); + Windmill.KeyedGetDataResponse.Builder response1 = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagSortedLists( + Windmill.TagSortedListFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + SortedListEntry.newBuilder().setValue(intData(1)).setSortKey(1000).setId(1)) + .setContinuationPosition(CONT_1) + .addFetchRanges( + SortedListRange.newBuilder().setStart(beginning).setLimit(end))); + + Windmill.KeyedGetDataRequest.Builder expectedRequest2 = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addSortedListsToFetch( + Windmill.TagSortedListFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addFetchRanges(SortedListRange.newBuilder().setStart(beginning).setLimit(end)) + .setRequestPosition(CONT_1) + .setFetchMaxBytes(WindmillStateReader.MAX_ORDERED_LIST_BYTES)); + + Windmill.KeyedGetDataResponse.Builder response2 = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagSortedLists( + Windmill.TagSortedListFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + SortedListEntry.newBuilder().setValue(intData(2)).setSortKey(2000).setId(2)) + .addEntries( + SortedListEntry.newBuilder().setValue(intData(3)).setSortKey(3000).setId(3)) + .addEntries( + SortedListEntry.newBuilder().setValue(intData(4)).setSortKey(4000).setId(4)) + .setContinuationPosition(CONT_2) + .addFetchRanges(SortedListRange.newBuilder().setStart(beginning).setLimit(end)) + .setRequestPosition(CONT_1)); + + Windmill.KeyedGetDataRequest.Builder expectedRequest3 = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addSortedListsToFetch( + Windmill.TagSortedListFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addFetchRanges(SortedListRange.newBuilder().setStart(beginning).setLimit(end)) + .setRequestPosition(CONT_2) + .setFetchMaxBytes(WindmillStateReader.MAX_ORDERED_LIST_BYTES)); + + Windmill.KeyedGetDataResponse.Builder response3 = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagSortedLists( + Windmill.TagSortedListFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + SortedListEntry.newBuilder().setValue(intData(5)).setSortKey(5000).setId(5)) + .addEntries( + SortedListEntry.newBuilder().setValue(intData(6)).setSortKey(6000).setId(7)) + .addEntries( + SortedListEntry.newBuilder().setValue(intData(7)).setSortKey(7000).setId(7)) + .addFetchRanges(SortedListRange.newBuilder().setStart(beginning).setLimit(end)) + .setRequestPosition(CONT_2)); + + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest1.build())) + .thenReturn(response1.build()); + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest2.build())) + .thenReturn(response2.build()); + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest3.build())) + .thenReturn(response3.build()); + + Iterable> results = future.get(); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest1.build()); + for (TimestampedValue unused : results) { + // Iterate over the results to force loading all the pages. + } + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest2.build()); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest3.build()); + Mockito.verifyNoMoreInteractions(mockWindmill); + + assertThat( + results, + Matchers.contains( + TimestampedValue.of(1, Instant.ofEpochMilli(1)), + TimestampedValue.of(2, Instant.ofEpochMilli(2)), + TimestampedValue.of(3, Instant.ofEpochMilli(3)), + TimestampedValue.of(4, Instant.ofEpochMilli(4)), + TimestampedValue.of(5, Instant.ofEpochMilli(5)), + TimestampedValue.of(6, Instant.ofEpochMilli(6)), + TimestampedValue.of(7, Instant.ofEpochMilli(7)))); + // NOTE: The future will still contain a reference to the underlying reader , thus not calling + // assertNoReader(future). + } + + @Test + public void testReadTagValuePrefix() throws Exception { + Future>> future = + underTest.valuePrefixFuture(STATE_KEY_PREFIX, STATE_FAMILY, INT_CODER); + Mockito.verifyNoMoreInteractions(mockWindmill); + + Windmill.KeyedGetDataRequest.Builder expectedRequest = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addTagValuePrefixesToFetch( + Windmill.TagValuePrefixRequest.newBuilder() + .setTagPrefix(STATE_KEY_PREFIX) + .setStateFamily(STATE_FAMILY) + .setFetchMaxBytes(WindmillStateReader.MAX_TAG_VALUE_PREFIX_BYTES)); + + Windmill.KeyedGetDataResponse.Builder response = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagValuePrefixes( + Windmill.TagValuePrefixResponse.newBuilder() + .setTagPrefix(STATE_KEY_PREFIX) + .setStateFamily(STATE_FAMILY) + .addTagValues( + Windmill.TagValue.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setValue(intValue(8))) + .addTagValues( + Windmill.TagValue.newBuilder() + .setTag(STATE_KEY_2) + .setStateFamily(STATE_FAMILY) + .setValue(intValue(9)))); + + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) + .thenReturn(response.build()); + + Iterable> result = future.get(); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); + Mockito.verifyNoMoreInteractions(mockWindmill); + + assertThat( + result, + Matchers.containsInAnyOrder( + new AbstractMap.SimpleEntry<>(STATE_KEY_1, 8), + new AbstractMap.SimpleEntry<>(STATE_KEY_2, 9))); + + assertNoReader(future); + } + + @Test + public void testReadTagValuePrefixWithContinuations() throws Exception { + Future>> future = + underTest.valuePrefixFuture(STATE_KEY_PREFIX, STATE_FAMILY, INT_CODER); + Mockito.verifyNoMoreInteractions(mockWindmill); + + Windmill.KeyedGetDataRequest.Builder expectedRequest1 = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addTagValuePrefixesToFetch( + Windmill.TagValuePrefixRequest.newBuilder() + .setTagPrefix(STATE_KEY_PREFIX) + .setStateFamily(STATE_FAMILY) + .setFetchMaxBytes(WindmillStateReader.MAX_TAG_VALUE_PREFIX_BYTES)); + + final ByteString CONT = ByteString.copyFrom("CONTINUATION", Charsets.UTF_8); + Windmill.KeyedGetDataResponse.Builder response1 = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagValuePrefixes( + Windmill.TagValuePrefixResponse.newBuilder() + .setTagPrefix(STATE_KEY_PREFIX) + .setStateFamily(STATE_FAMILY) + .setContinuationPosition(CONT) + .addTagValues( + Windmill.TagValue.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setValue(intValue(8)))); + + Windmill.KeyedGetDataRequest.Builder expectedRequest2 = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addTagValuePrefixesToFetch( + Windmill.TagValuePrefixRequest.newBuilder() + .setTagPrefix(STATE_KEY_PREFIX) + .setStateFamily(STATE_FAMILY) + .setRequestPosition(CONT) + .setFetchMaxBytes(WindmillStateReader.MAX_TAG_VALUE_PREFIX_BYTES)); + + Windmill.KeyedGetDataResponse.Builder response2 = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addTagValuePrefixes( + Windmill.TagValuePrefixResponse.newBuilder() + .setTagPrefix(STATE_KEY_PREFIX) + .setStateFamily(STATE_FAMILY) + .setRequestPosition(CONT) + .addTagValues( + Windmill.TagValue.newBuilder() + .setTag(STATE_KEY_2) + .setStateFamily(STATE_FAMILY) + .setValue(intValue(9)))); + + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest1.build())) + .thenReturn(response1.build()); + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest2.build())) + .thenReturn(response2.build()); + + Iterable> results = future.get(); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest1.build()); + for (Map.Entry unused : results) { + // Iterate over the results to force loading all the pages. + } + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest2.build()); + Mockito.verifyNoMoreInteractions(mockWindmill); + + assertThat( + results, + Matchers.containsInAnyOrder( + new AbstractMap.SimpleEntry<>(STATE_KEY_1, 8), + new AbstractMap.SimpleEntry<>(STATE_KEY_2, 9))); + // NOTE: The future will still contain a reference to the underlying reader , thus not calling + // assertNoReader(future). + } + + @Test + public void testReadValue() throws Exception { + Future future = underTest.valueFuture(STATE_KEY_1, STATE_FAMILY, INT_CODER); + Mockito.verifyNoMoreInteractions(mockWindmill); + + Windmill.KeyedGetDataRequest.Builder expectedRequest = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addValuesToFetch( + Windmill.TagValue.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .build()); + Windmill.KeyedGetDataResponse.Builder response = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addValues( + Windmill.TagValue.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setValue(intValue(8))); + + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) + .thenReturn(response.build()); + + Integer result = future.get(); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); + Mockito.verifyNoMoreInteractions(mockWindmill); + + assertThat(result, Matchers.equalTo(8)); + assertNoReader(future); + } + + @Test + public void testReadWatermark() throws Exception { + Future future = underTest.watermarkFuture(STATE_KEY_1, STATE_FAMILY); + Mockito.verifyNoMoreInteractions(mockWindmill); + + Windmill.KeyedGetDataRequest.Builder expectedRequest = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addWatermarkHoldsToFetch( + Windmill.WatermarkHold.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY)); + + Windmill.KeyedGetDataResponse.Builder response = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addWatermarkHolds( + Windmill.WatermarkHold.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addTimestamps(5000000) + .addTimestamps(6000000)); + + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) + .thenReturn(response.build()); + + Instant result = future.get(); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); + + assertThat(result, Matchers.equalTo(new Instant(5000))); + assertNoReader(future); + } + + @Test + public void testBatching() throws Exception { + // Reads two bags and verifies that we batch them up correctly. + Future watermarkFuture = underTest.watermarkFuture(STATE_KEY_2, STATE_FAMILY); + Future> bagFuture = underTest.bagFuture(STATE_KEY_1, STATE_FAMILY, INT_CODER); + + Mockito.verifyNoMoreInteractions(mockWindmill); + + ArgumentCaptor request = + ArgumentCaptor.forClass(Windmill.KeyedGetDataRequest.class); + + Windmill.KeyedGetDataResponse.Builder response = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addWatermarkHolds( + Windmill.WatermarkHold.newBuilder() + .setTag(STATE_KEY_2) + .setStateFamily(STATE_FAMILY) + .addTimestamps(5000000) + .addTimestamps(6000000)) + .addBags( + Windmill.TagBag.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addValues(intData(5)) + .addValues(intData(100))); + + Mockito.when( + mockWindmill.getStateData( + Mockito.eq(COMPUTATION), Mockito.isA(Windmill.KeyedGetDataRequest.class))) + .thenReturn(response.build()); + Instant result = watermarkFuture.get(); + Mockito.verify(mockWindmill).getStateData(Mockito.eq(COMPUTATION), request.capture()); + + // Verify the request looks right. + KeyedGetDataRequest keyedRequest = request.getValue(); + assertThat(keyedRequest.getKey(), Matchers.equalTo(DATA_KEY)); + assertThat(keyedRequest.getWorkToken(), Matchers.equalTo(WORK_TOKEN)); + assertThat(keyedRequest.getBagsToFetchCount(), Matchers.equalTo(1)); + assertThat(keyedRequest.getBagsToFetch(0).getDeleteAll(), Matchers.equalTo(false)); + assertThat(keyedRequest.getBagsToFetch(0).getTag(), Matchers.equalTo(STATE_KEY_1)); + assertThat(keyedRequest.getWatermarkHoldsToFetchCount(), Matchers.equalTo(1)); + assertThat(keyedRequest.getWatermarkHoldsToFetch(0).getTag(), Matchers.equalTo(STATE_KEY_2)); + + // Verify the values returned to the user. + assertThat(result, Matchers.equalTo(new Instant(5000))); + Mockito.verifyNoMoreInteractions(mockWindmill); + + assertThat(bagFuture.get(), Matchers.contains(5, 100)); + Mockito.verifyNoMoreInteractions(mockWindmill); + + // And verify that getting a future again returns the already completed future. + Future watermarkFuture2 = underTest.watermarkFuture(STATE_KEY_2, STATE_FAMILY); + assertTrue(watermarkFuture2.isDone()); + assertNoReader(watermarkFuture); + assertNoReader(watermarkFuture2); + } + + @Test + public void testNoStateFamily() throws Exception { + Future future = underTest.valueFuture(STATE_KEY_1, "", INT_CODER); + Mockito.verifyNoMoreInteractions(mockWindmill); + + Windmill.KeyedGetDataRequest.Builder expectedRequest = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .setWorkToken(WORK_TOKEN) + .addValuesToFetch( + Windmill.TagValue.newBuilder().setTag(STATE_KEY_1).setStateFamily("").build()); + Windmill.KeyedGetDataResponse.Builder response = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addValues( + Windmill.TagValue.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily("") + .setValue(intValue(8))); + + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) + .thenReturn(response.build()); + + Integer result = future.get(); + Mockito.verify(mockWindmill).getStateData(COMPUTATION, expectedRequest.build()); + Mockito.verifyNoMoreInteractions(mockWindmill); + + assertThat(result, Matchers.equalTo(8)); + assertNoReader(future); + } + + @Test + public void testKeyTokenInvalid() throws Exception { + // Reads two states and verifies that we batch them up correctly. + Future watermarkFuture = underTest.watermarkFuture(STATE_KEY_2, STATE_FAMILY); + Future> bagFuture = underTest.bagFuture(STATE_KEY_1, STATE_FAMILY, INT_CODER); + + Mockito.verifyNoMoreInteractions(mockWindmill); + + Windmill.KeyedGetDataResponse.Builder response = + Windmill.KeyedGetDataResponse.newBuilder().setKey(DATA_KEY).setFailed(true); + + Mockito.when( + mockWindmill.getStateData( + Mockito.eq(COMPUTATION), Mockito.isA(Windmill.KeyedGetDataRequest.class))) + .thenReturn(response.build()); + + try { + watermarkFuture.get(); + fail("Expected KeyTokenInvalidException"); + } catch (Exception e) { + assertTrue(KeyTokenInvalidException.isKeyTokenInvalidException(e)); + } + + try { + bagFuture.get(); + fail("Expected KeyTokenInvalidException"); + } catch (Exception e) { + assertTrue(KeyTokenInvalidException.isKeyTokenInvalidException(e)); + } + } + + @Test + public void testBatchingReadException() throws Exception { + // Reads two states and verifies that we batch them up correctly and propagate the read + // exception to both, not just the issuing future. + Future watermarkFuture = underTest.watermarkFuture(STATE_KEY_2, STATE_FAMILY); + Future> bagFuture = underTest.bagFuture(STATE_KEY_1, STATE_FAMILY, INT_CODER); + + Mockito.verifyNoMoreInteractions(mockWindmill); + + RuntimeException expectedException = new RuntimeException("expected exception"); + + Mockito.when( + mockWindmill.getStateData( + Mockito.eq(COMPUTATION), Mockito.isA(Windmill.KeyedGetDataRequest.class))) + .thenThrow(expectedException); + + try { + watermarkFuture.get(); + fail("Expected RuntimeException"); + } catch (Exception e) { + assertThat(e.toString(), Matchers.containsString("expected exception")); + } + + try { + bagFuture.get(); + fail("Expected RuntimeException"); + } catch (Exception e) { + assertThat(e.toString(), Matchers.containsString("expected exception")); + } + } + + @Test + public void testBatchingCoderExceptions() throws Exception { + // Read a batch of states with coder errors and verify it only affects the + // relevant futures. + Future valueFuture = underTest.valueFuture(STATE_KEY_1, STATE_FAMILY, INT_CODER); + Future> bagFuture = underTest.bagFuture(STATE_KEY_1, STATE_FAMILY, INT_CODER); + Future>> valuePrefixFuture = + underTest.valuePrefixFuture(STATE_KEY_PREFIX, STATE_FAMILY, INT_CODER); + long beginning = SortedListRange.getDefaultInstance().getStart(); + long end = SortedListRange.getDefaultInstance().getLimit(); + Future>> orderedListFuture = + underTest.orderedListFuture( + Range.closedOpen(beginning, end), STATE_KEY_1, STATE_FAMILY, INT_CODER); + // This should be the final part of the response, and we should process it successfully. + Future watermarkFuture = underTest.watermarkFuture(STATE_KEY_1, STATE_FAMILY); + + Mockito.verifyNoMoreInteractions(mockWindmill); + + ByteString invalidByteString = ByteString.copyFrom(BaseEncoding.base16().decode("FFFF")); + Windmill.Value invalidValue = intValue(0).toBuilder().setData(invalidByteString).build(); + + Windmill.KeyedGetDataRequest.Builder expectedRequest = + Windmill.KeyedGetDataRequest.newBuilder() + .setKey(DATA_KEY) + .setShardingKey(SHARDING_KEY) + .setWorkToken(WORK_TOKEN) + .setMaxBytes(WindmillStateReader.MAX_KEY_BYTES) + .addValuesToFetch( + Windmill.TagValue.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .build()) + .addBagsToFetch( + Windmill.TagBag.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setFetchMaxBytes(WindmillStateReader.INITIAL_MAX_BAG_BYTES)) + .addTagValuePrefixesToFetch( + Windmill.TagValuePrefixRequest.newBuilder() + .setTagPrefix(STATE_KEY_PREFIX) + .setStateFamily(STATE_FAMILY) + .setFetchMaxBytes(WindmillStateReader.MAX_TAG_VALUE_PREFIX_BYTES)) + .addSortedListsToFetch( + Windmill.TagSortedListFetchRequest.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addFetchRanges(SortedListRange.newBuilder().setStart(beginning).setLimit(end)) + .setFetchMaxBytes(WindmillStateReader.MAX_ORDERED_LIST_BYTES)) + .addWatermarkHoldsToFetch( + Windmill.WatermarkHold.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY)); + + Windmill.KeyedGetDataResponse.Builder response = + Windmill.KeyedGetDataResponse.newBuilder() + .setKey(DATA_KEY) + .addValues( + Windmill.TagValue.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setValue(invalidValue)) + .addBags( + Windmill.TagBag.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addValues(invalidByteString)) + .addTagValuePrefixes( + Windmill.TagValuePrefixResponse.newBuilder() + .setTagPrefix(STATE_KEY_PREFIX) + .setStateFamily(STATE_FAMILY) + .addTagValues( + Windmill.TagValue.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .setValue(invalidValue))) + .addTagSortedLists( + Windmill.TagSortedListFetchResponse.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addEntries( + SortedListEntry.newBuilder() + .setValue(invalidByteString) + .setSortKey(5000) + .setId(5)) + .addFetchRanges(SortedListRange.newBuilder().setStart(beginning).setLimit(end))) + .addWatermarkHolds( + Windmill.WatermarkHold.newBuilder() + .setTag(STATE_KEY_1) + .setStateFamily(STATE_FAMILY) + .addTimestamps(5000000) + .addTimestamps(6000000)); + + Mockito.when(mockWindmill.getStateData(COMPUTATION, expectedRequest.build())) + .thenReturn(response.build()); + Mockito.verifyNoMoreInteractions(mockWindmill); + + try { + valueFuture.get(); + fail("Expected RuntimeException"); + } catch (Exception e) { + assertThat(e.toString(), Matchers.containsString("Unable to decode value")); + } + + try { + bagFuture.get(); + fail("Expected RuntimeException"); + } catch (Exception e) { + assertThat(e.toString(), Matchers.containsString("Error parsing bag")); + } + + try { + orderedListFuture.get(); + fail("Expected RuntimeException"); + } catch (Exception e) { + assertThat(e.toString(), Matchers.containsString("Error parsing ordered list")); + } + + try { + valuePrefixFuture.get(); + fail("Expected RuntimeException"); + } catch (Exception e) { + assertThat(e.toString(), Matchers.containsString("Error parsing tag value prefix")); + } + + assertThat(watermarkFuture.get(), Matchers.equalTo(new Instant(5000))); + } + + /** + * Tests that multiple reads for the same tag in the same batch are cached. We can't compare the + * futures since we've wrapped the delegate aronud them, so we just verify there is only one + * queued lookup. + */ + @Test + public void testCachingWithinBatch() throws Exception { + underTest.watermarkFuture(STATE_KEY_1, STATE_FAMILY); + underTest.watermarkFuture(STATE_KEY_1, STATE_FAMILY); + assertEquals(1, underTest.pendingLookups.size()); + } +} diff --git a/runners/google-cloud-dataflow-java/worker/windmill/src/main/proto/windmill.proto b/runners/google-cloud-dataflow-java/worker/windmill/src/main/proto/windmill.proto index b0e4dba698bcf..1759185911d49 100644 --- a/runners/google-cloud-dataflow-java/worker/windmill/src/main/proto/windmill.proto +++ b/runners/google-cloud-dataflow-java/worker/windmill/src/main/proto/windmill.proto @@ -27,14 +27,14 @@ option java_outer_classname = "Windmill"; // API Data types message Message { - required int64 timestamp = 1 [default=-0x8000000000000000]; + required int64 timestamp = 1 [default = -0x8000000000000000]; required bytes data = 2; optional bytes metadata = 3; } message Timer { required bytes tag = 1; - optional int64 timestamp = 2 [default=-0x8000000000000000]; + optional int64 timestamp = 2 [default = -0x8000000000000000]; enum Type { WATERMARK = 0; REALTIME = 1; @@ -65,11 +65,44 @@ message LatencyAttribution { ACTIVE = 2; READING = 3; COMMITTING = 4; + // State which starts with the Windmill Worker receiving the GetWorkRequest + // and ends with the Windmill Worker sending the GetWorkResponse to the + // Windmill Dispatcher. + GET_WORK_IN_WINDMILL_WORKER = 5; + // State which starts with the Windmill Worker sending the GetWorkResponse + // and ends with the Windmill Dispatcher receiving the GetWorkResponse. + GET_WORK_IN_TRANSIT_TO_DISPATCHER = 6; + // State which starts with the Windmill Dispatcher sending the + // GetWorkResponse and ends with the user worker receiving the + // GetWorkResponse. + GET_WORK_IN_TRANSIT_TO_USER_WORKER = 7; } optional State state = 1; optional int64 total_duration_millis = 2; } +message GetWorkStreamTimingInfo { + enum Event { + UNKNOWN = 0; + // Work item creation started by the Windmill Worker. + GET_WORK_CREATION_START = 1; + // Work item creation finished by the Windmill Worker. + GET_WORK_CREATION_END = 2; + // The GetWorkResponse containing this work item is received by the Windmill + // Dispatcher. + GET_WORK_RECEIVED_BY_DISPATCHER = 3; + // The GetWorkResponse containing this work item is forwarded by the + // Windmill Dispatcher to the user worker. + GET_WORK_FORWARDED_BY_DISPATCHER = 4; + } + + // Critical event of the work item processing. + optional Event event = 1; + + // Timestamp of the event. + optional int64 timestamp_usec = 2; +} + message OutputMessageBundle { optional string destination_computation_id = 1; optional string destination_stream_id = 3; @@ -91,7 +124,7 @@ message TimerBundle { } message Value { - required int64 timestamp = 1 [default=-0x8000000000000000]; + required int64 timestamp = 1 [default = -0x8000000000000000]; required bytes data = 2; } @@ -146,6 +179,79 @@ message TagBag { optional int64 fetch_max_bytes = 6 [default = 0x7fffffffffffffff]; } +// For a given sharding key and state family, a TagMultimap is a collection of +// `entry_name, bag of values` pairs, each pair is a TagMultimapEntry. +// +// The request_position, continuation_position and fetch_max_bytes fields in +// TagMultimapEntry are used for the pagination and byte limiting of individual +// entry fetch requests get(entry_name); while those fields in +// TagMultimapFetchRequest and TagMultimapFetchResponse are used for full +// multimap fetch requests entry_names() and entries(). +// Do not set both in a TagMultimapFetchRequest at the same time. +message TagMultimapEntry { + optional bytes entry_name = 1; + // In update request: if true all values associated with this entry_name will + // be deleted. If new values are present they will be written. + optional bool delete_all = 2; + // In update request: The given values will be added to the collection and + // associated with entry_name. + // In fetch response: Values that are associated with this entry_name in the + // multimap. + repeated bytes values = 3; + // In fetch request: A previously returned continuation_position from an + // earlier read response. Indicates we wish to fetch the next page of values. + // If this is the first request, set to empty. + // In fetch response: copied from request. + optional int64 request_position = 4; + // In fetch response: Set when there are values after those returned above, + // but they were suppressed to respect the fetch_max_bytes limit. Subsequent + // requests should copy this to request_position to retrieve the next page of + // values. + optional int64 continuation_position = 5; + // In fetch request: Limits the size of the fetched values to this byte limit. + // A lower limit may be imposed by the service. + optional int64 fetch_max_bytes = 6 [default = 0x7fffffffffffffff]; +} + +message TagMultimapFetchRequest { + optional bytes tag = 1; + optional string state_family = 2; + // If true, values will be omitted in the response. + optional bool fetch_entry_names_only = 3; + // Limits the size of the fetched entries to this byte limit. A lower limit + // may be imposed by the service. + optional int64 fetch_max_bytes = 4 [default = 0x7fffffffffffffff]; + // A previously returned continuation_position from an earlier fetch response. + // Indicates we wish to fetch the next page of entries. If this is the first + // request, set to empty. + optional bytes request_position = 5; + // Fetch the requested subset of entries only. Will fetch all entries if left + // empty. Entries in entries_to_fetch should only have the entry_name, + // request_position and fetch_max_bytes set. + repeated TagMultimapEntry entries_to_fetch = 6; +} + +message TagMultimapFetchResponse { + optional bytes tag = 1; + optional string state_family = 2; + repeated TagMultimapEntry entries = 3; + // Will be set only if the entries_to_fetch in the request is empty when + // we want to fetch all entries. + optional bytes continuation_position = 4; + // Request position copied from request. + optional bytes request_position = 5; +} + +message TagMultimapUpdateRequest { + optional bytes tag = 1; + optional string state_family = 2; + // All entries including the values in the multimap will be deleted. + optional bool delete_all = 3; + // If delete_all is true and updates are not empty, the multimap will first be + // cleared and then those updates will be applied. + repeated TagMultimapEntry updates = 4; +} + // A single entry in a sorted list message SortedListEntry { // The value payload. @@ -187,7 +293,7 @@ message TagSortedListFetchResponse { repeated SortedListRange fetch_ranges = 5; // Request position copied from request. optional bytes request_position = 6; - } +} message TagSortedListUpdateRequest { optional bytes tag = 1; @@ -226,7 +332,7 @@ message SourceState { message WatermarkHold { required bytes tag = 1; - repeated int64 timestamps = 2 [packed=true]; + repeated int64 timestamps = 2 [packed = true]; optional bool reset = 3; optional string state_family = 4; } @@ -248,7 +354,7 @@ message WorkItem { optional TimerBundle timers = 4; repeated GlobalDataId global_data_id_notifications = 5; optional SourceState source_state = 6; - optional int64 output_data_watermark = 8 [default=-0x8000000000000000]; + optional int64 output_data_watermark = 8 [default = -0x8000000000000000]; // Indicates that this is a new key with no data associated. This allows // the harness to optimize data fetching. optional bool is_new_key = 10; @@ -262,9 +368,9 @@ message WorkItem { message ComputationWorkItems { required string computation_id = 1; repeated WorkItem work = 2; - optional int64 input_data_watermark = 3 [default=-0x8000000000000000]; + optional int64 input_data_watermark = 3 [default = -0x8000000000000000]; optional int64 dependent_realtime_input_watermark = 4 - [default = -0x8000000000000000]; + [default = -0x8000000000000000]; } //////////////////////////////////////////////////////////////////////////////// @@ -297,6 +403,8 @@ message KeyedGetDataRequest { repeated TagBag bags_to_fetch = 8; // Must be at most one sorted_list_to_fetch for a given state family and tag. repeated TagSortedListFetchRequest sorted_lists_to_fetch = 9; + // Must be at most one multimaps_to_fetch for a given state family and tag. + repeated TagMultimapFetchRequest multimaps_to_fetch = 12; repeated WatermarkHold watermark_holds_to_fetch = 5; repeated LatencyAttribution latency_attribution = 13; @@ -329,6 +437,8 @@ message KeyedGetDataResponse { repeated TagBag bags = 6; // There is one TagSortedListFetchResponse per state-family, tag pair. repeated TagSortedListFetchResponse tag_sorted_lists = 8; + // There is one TagMultimapFetchResponse per state-family, tag pair. + repeated TagMultimapFetchResponse tag_multimaps = 10; repeated WatermarkHold watermark_holds = 5; reserved 4; @@ -368,16 +478,16 @@ message Counter { // value accumulated since the worker started working on this WorkItem. // By default this is false, indicating that this metric is reported // as a delta that is not associated with any WorkItem. - optional bool cumulative = 7; + optional bool cumulative = 7; } message GlobalDataRequest { required GlobalDataId data_id = 1; - optional int64 existence_watermark_deadline = 2 [default=0x7FFFFFFFFFFFFFFF]; + optional int64 existence_watermark_deadline = 2 [default = 0x7FFFFFFFFFFFFFFF]; optional string state_family = 3; } -// next id: 24 +// next id: 28 message WorkItemCommitRequest { required bytes key = 1; required fixed64 work_token = 2; @@ -394,16 +504,20 @@ message WorkItemCommitRequest { repeated TagValuePrefix tag_value_prefix_deletes = 25; repeated TagBag bag_updates = 18; repeated TagSortedListUpdateRequest sorted_list_updates = 24; + repeated TagMultimapUpdateRequest multimap_updates = 26; repeated Counter counter_updates = 8; repeated GlobalDataRequest global_data_requests = 11; repeated GlobalData global_data_updates = 10; optional SourceState source_state_updates = 12; - optional int64 source_watermark = 13 [default=-0x8000000000000000]; - optional int64 source_backlog_bytes = 17 [default=-1]; + optional int64 source_watermark = 13 [default = -0x8000000000000000]; + optional int64 source_backlog_bytes = 17 [default = -1]; optional int64 source_bytes_processed = 22; repeated WatermarkHold watermark_holds = 14; + // Collected work item processing state durations. + repeated LatencyAttribution per_work_item_latency_attributions = 27; + // DEPRECATED repeated GlobalDataId global_data_id_requests = 9; @@ -478,7 +592,7 @@ message GetConfigResponse { optional string computation_id = 2; } repeated SystemNameToComputationIdMapEntry - system_name_to_computation_id_map = 3; + system_name_to_computation_id_map = 3; // Map of computation id to ComputationConfig. message ComputationConfigMapEntry { @@ -513,10 +627,10 @@ message ReportStatsResponse { // Streaming API message StreamingGetWorkRequest { - oneof chunk_type { + oneof chunk_type { GetWorkRequest request = 1; StreamingGetWorkRequestExtension request_extension = 2; - } + } } message StreamingGetWorkRequestExtension { @@ -538,12 +652,16 @@ message StreamingGetWorkResponseChunk { // from other stream_ids may be interleaved on the physical stream. optional fixed64 stream_id = 4; + // Timing infos for the work item. Windmill Dispatcher and user worker should + // propagate critical event timings if the list is not empty. + repeated GetWorkStreamTimingInfo per_work_item_timing_infos = 8; + // reserved field 5 } message ComputationWorkItemMetadata { optional string computation_id = 1; - optional int64 input_data_watermark = 2 [default=-0x8000000000000000]; + optional int64 input_data_watermark = 2 [default = -0x8000000000000000]; optional int64 dependent_realtime_input_watermark = 3 [default = -0x8000000000000000]; } @@ -624,6 +742,37 @@ message StreamingCommitResponse { repeated CommitStatus status = 2; } +message WorkerMetadataRequest { + optional JobHeader header = 1; +} + +// Converted into org.apache.beam.runners.dataflow.worker.windmill.WindmillEndpoints +// used to connect to Streaming Engine. +message WorkerMetadataResponse { + // The metadata version increases with every modification. Within a single + // stream it will always be increasing. The version may be used across streams + // to ensure that the view of the metadata does not move backwards. + optional int64 metadata_version = 1; + + // Endpoints that should be used for requesting work with GetWorkStream. + // Additional data for returned work should be fetched from the endpoint with + // GetDataStream. The work should be committed to the endpoint with + // CommitWorkStream. Each response on this stream replaces the previous, and + // connections to endpoints that are no longer present should be closed. + message Endpoint { + // IPv6 address of a streaming engine windmill worker. + optional string direct_endpoint = 1; + optional string worker_token = 2; + } + repeated Endpoint work_endpoints = 2; + + // Maps from GlobalData tag to the endpoint that should be used for GetData + // calls to retrieve that global data. + map global_data_endpoints = 3; + + reserved 4; +} + service WindmillAppliance { // Gets streaming Dataflow work. rpc GetWork(.windmill.GetWorkRequest) returns (.windmill.GetWorkResponse); diff --git a/runners/google-cloud-dataflow-java/worker/windmill/src/main/proto/windmill_service.proto b/runners/google-cloud-dataflow-java/worker/windmill/src/main/proto/windmill_service.proto index bef819d699017..d9183e54e0dd3 100644 --- a/runners/google-cloud-dataflow-java/worker/windmill/src/main/proto/windmill_service.proto +++ b/runners/google-cloud-dataflow-java/worker/windmill/src/main/proto/windmill_service.proto @@ -33,6 +33,10 @@ service CloudWindmillServiceV1Alpha1 { rpc GetWorkStream(stream .windmill.StreamingGetWorkRequest) returns (stream .windmill.StreamingGetWorkResponseChunk); + // Gets worker metadata. Response is a stream. + rpc GetWorkerMetadataStream(stream .windmill.WorkerMetadataRequest) + returns (stream .windmill.WorkerMetadataResponse); + // Gets data from Windmill. rpc GetData(.windmill.GetDataRequest) returns(.windmill.GetDataResponse); diff --git a/runners/java-fn-execution/OWNERS b/runners/java-fn-execution/OWNERS deleted file mode 100644 index 201e7cb37505c..0000000000000 --- a/runners/java-fn-execution/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lukecwik - - angoenka diff --git a/runners/java-fn-execution/build.gradle b/runners/java-fn-execution/build.gradle index 4cb3a08960942..02795add47d40 100644 --- a/runners/java-fn-execution/build.gradle +++ b/runners/java-fn-execution/build.gradle @@ -23,7 +23,7 @@ applyJavaNature( description = "Apache Beam :: Runners :: Java Fn Execution" dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(":runners:core-construction-java") implementation project(":runners:core-java") compileOnly project(":sdks:java:harness") diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/ArtifactRetrievalService.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/ArtifactRetrievalService.java index df191b40bc0fb..5668870bd7497 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/ArtifactRetrievalService.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/ArtifactRetrievalService.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.fnexecution.artifact; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.io.InputStream; diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/ArtifactStagingService.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/ArtifactStagingService.java index bfcac4437de1b..1ae3bdbfa0a10 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/ArtifactStagingService.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/ArtifactStagingService.java @@ -56,11 +56,11 @@ import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Status; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.StatusException; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/BundleCheckpointHandlers.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/BundleCheckpointHandlers.java index 79e6fd0acf94e..c1a9ae0d60712 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/BundleCheckpointHandlers.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/BundleCheckpointHandlers.java @@ -33,7 +33,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/DefaultJobBundleFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/DefaultJobBundleFactory.java index 1ac50922a2923..824c2c78cc505 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/DefaultJobBundleFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/DefaultJobBundleFactory.java @@ -71,16 +71,16 @@ import org.apache.beam.sdk.options.PortablePipelineOptions; import org.apache.beam.sdk.util.NoopLock; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LoadingCache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientPoolService.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientPoolService.java index 2db9fb473a32a..a2c61f3363b3a 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientPoolService.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientPoolService.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.fnexecution.control; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.HashMap; import java.util.Map; diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/MapControlClientPool.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/MapControlClientPool.java index 420d7fd3f96f7..3fc7ac6b027bb 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/MapControlClientPool.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/MapControlClientPool.java @@ -23,7 +23,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; /** * A {@link ControlClientPool} backed by a client map. It is expected that a given client id will be diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/ProcessBundleDescriptors.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/ProcessBundleDescriptors.java index 703bbf9539ee5..34c95335ad3e5 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/ProcessBundleDescriptors.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/ProcessBundleDescriptors.java @@ -18,8 +18,8 @@ package org.apache.beam.runners.fnexecution.control; import static org.apache.beam.runners.core.construction.SyntheticComponents.uniqueId; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.io.IOException; @@ -59,9 +59,9 @@ import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder; import org.apache.beam.sdk.values.KV; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableTable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableTable; import org.checkerframework.checker.nullness.qual.Nullable; /** Utility methods for creating {@link ProcessBundleDescriptor} instances. */ diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/ReferenceCountingExecutableStageContextFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/ReferenceCountingExecutableStageContextFactory.java index 35ac37bcff34f..c18180dfe3ab8 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/ReferenceCountingExecutableStageContextFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/ReferenceCountingExecutableStageContextFactory.java @@ -31,9 +31,9 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PortablePipelineOptions; import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClient.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClient.java index 123128814f9ed..c9faf8508aa39 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClient.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClient.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.fnexecution.control; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.ArrayList; import java.util.Collections; @@ -60,7 +60,7 @@ import org.apache.beam.sdk.fn.data.TimerEndpoint; import org.apache.beam.sdk.util.MoreFutures; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/SingleEnvironmentInstanceJobBundleFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/SingleEnvironmentInstanceJobBundleFactory.java index 17e669e7651ee..3a71bcfae72f2 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/SingleEnvironmentInstanceJobBundleFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/SingleEnvironmentInstanceJobBundleFactory.java @@ -37,7 +37,7 @@ import org.apache.beam.sdk.fn.data.FnDataReceiver; import org.apache.beam.sdk.fn.server.GrpcFnServer; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** * A {@link JobBundleFactory} which can manage a single instance of an {@link Environment}. diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/TimerReceiverFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/TimerReceiverFactory.java index 85df1310da9f4..bbc75d94839f2 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/TimerReceiverFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/TimerReceiverFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.fnexecution.control; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.HashMap; import java.util.Map; diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/data/GrpcDataService.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/data/GrpcDataService.java index 07975a29360a3..15a17e98fee27 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/data/GrpcDataService.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/data/GrpcDataService.java @@ -34,7 +34,7 @@ import org.apache.beam.sdk.fn.stream.OutboundObserverFactory; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.SettableFuture; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.SettableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerCommand.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerCommand.java index bdd995b4b7c0c..79484fea9fa76 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerCommand.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerCommand.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.fnexecution.environment; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.BufferedReader; @@ -33,7 +33,7 @@ import java.util.concurrent.TimeoutException; import java.util.regex.Pattern; import java.util.stream.Collectors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,20 +53,28 @@ class DockerCommand { // but we _should_ always capture full ids. private static final Pattern CONTAINER_ID_PATTERN = Pattern.compile("\\p{XDigit}{64}"); + /** + * Return a DockerCommand instance with default timeout settings: pull timeout 10 min and other + * command timeout 2 min. + */ public static DockerCommand getDefault() { - return forExecutable(DEFAULT_DOCKER_COMMAND, Duration.ofMinutes(2)); + return forExecutable(DEFAULT_DOCKER_COMMAND, Duration.ofMinutes(2), Duration.ofMinutes(10)); } - static DockerCommand forExecutable(String dockerExecutable, Duration commandTimeout) { - return new DockerCommand(dockerExecutable, commandTimeout); + static DockerCommand forExecutable( + String dockerExecutable, Duration commandTimeout, Duration pullTimeout) { + return new DockerCommand(dockerExecutable, commandTimeout, pullTimeout); } private final String dockerExecutable; private final Duration commandTimeout; + // pull remote image can take longer time + private final Duration pullTimeout; - private DockerCommand(String dockerExecutable, Duration commandTimeout) { + private DockerCommand(String dockerExecutable, Duration commandTimeout, Duration pullTimeout) { this.dockerExecutable = dockerExecutable; this.commandTimeout = commandTimeout; + this.pullTimeout = pullTimeout; } /** @@ -83,7 +91,8 @@ public String runImage(String imageTag, List dockerOpts, List ar // Pull the image from docker repo. This will be no-op if the image already exists. try { runShortCommand( - ImmutableList.builder().add(dockerExecutable).add("pull").add(imageTag).build()); + ImmutableList.builder().add(dockerExecutable).add("pull").add(imageTag).build(), + pullTimeout); } catch (IOException | TimeoutException | InterruptedException e) { if (LOG.isDebugEnabled()) { LOG.debug("Unable to pull docker image {}", imageTag, e); @@ -134,7 +143,8 @@ public String getContainerLogs(String containerId) checkArgument( CONTAINER_ID_PATTERN.matcher(containerId).matches(), "Container ID must be a 64-character hexadecimal string"); - return runShortCommand(Arrays.asList(dockerExecutable, "logs", containerId), true, "\n"); + return runShortCommand( + Arrays.asList(dockerExecutable, "logs", containerId), true, "\n", commandTimeout); } /** @@ -168,7 +178,12 @@ public void removeContainer(String containerId) private String runShortCommand(List invocation) throws IOException, TimeoutException, InterruptedException { - return runShortCommand(invocation, false, ""); + return runShortCommand(invocation, false, "", commandTimeout); + } + + private String runShortCommand(List invocation, Duration timeout) + throws IOException, TimeoutException, InterruptedException { + return runShortCommand(invocation, false, "", timeout); } /** @@ -182,7 +197,10 @@ private String runShortCommand(List invocation) * @throws TimeoutException if command has not finished by {@link DockerCommand#commandTimeout} */ private String runShortCommand( - List invocation, boolean redirectErrorStream, CharSequence delimiter) + List invocation, + boolean redirectErrorStream, + CharSequence delimiter, + Duration timeout) throws IOException, TimeoutException, InterruptedException { ProcessBuilder pb = new ProcessBuilder(invocation); pb.redirectErrorStream(redirectErrorStream); @@ -216,7 +234,7 @@ private String runShortCommand( }); } // TODO: Retry on interrupt? - boolean processDone = process.waitFor(commandTimeout.toMillis(), TimeUnit.MILLISECONDS); + boolean processDone = process.waitFor(timeout.toMillis(), TimeUnit.MILLISECONDS); if (!processDone) { process.destroy(); throw new TimeoutException( @@ -228,7 +246,7 @@ private String runShortCommand( if (exitCode != 0) { String errorString; try { - errorString = errorFuture.get(commandTimeout.toMillis(), TimeUnit.MILLISECONDS); + errorString = errorFuture.get(timeout.toMillis(), TimeUnit.MILLISECONDS); } catch (Exception stderrEx) { errorString = String.format("Error capturing %s: %s", errorStringName, stderrEx.getMessage()); @@ -242,8 +260,7 @@ private String runShortCommand( errorString)); } try { - // TODO: Consider a stricter timeout. - return resultString.get(commandTimeout.toMillis(), TimeUnit.MILLISECONDS); + return resultString.get(timeout.toMillis(), TimeUnit.MILLISECONDS); } catch (ExecutionException e) { Throwable cause = e.getCause(); // Recast any exceptions in reading output as IOExceptions. diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java index ee816a944e5a4..a1713c69d9204 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.fnexecution.environment; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; import java.nio.file.Files; import java.nio.file.Paths; @@ -40,9 +40,9 @@ import org.apache.beam.sdk.options.ManualDockerEnvironmentOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.RemoteEnvironmentOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.net.HostAndPort; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.net.HostAndPort; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/EmbeddedEnvironmentFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/EmbeddedEnvironmentFactory.java index 1dbf6b6c3ac82..2a3db0b4e0518 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/EmbeddedEnvironmentFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/EmbeddedEnvironmentFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.fnexecution.environment; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.time.Duration; import java.util.Collections; @@ -45,7 +45,7 @@ import org.apache.beam.sdk.fn.server.ServerFactory; import org.apache.beam.sdk.fn.stream.OutboundObserverFactory; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/ExternalEnvironmentFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/ExternalEnvironmentFactory.java index 8fb3ac0d4fcb6..e6d30f6403f70 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/ExternalEnvironmentFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/ExternalEnvironmentFactory.java @@ -36,7 +36,7 @@ import org.apache.beam.sdk.fn.server.GrpcFnServer; import org.apache.beam.sdk.fn.server.ServerFactory; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ManagedChannel; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/ProcessEnvironmentFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/ProcessEnvironmentFactory.java index 98825d5914daa..fe026e5e5659e 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/ProcessEnvironmentFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/ProcessEnvironmentFactory.java @@ -32,8 +32,8 @@ import org.apache.beam.sdk.fn.server.GrpcFnServer; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.RemoteEnvironmentOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/ProcessManager.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/ProcessManager.java index 00ffc5b9b1ad8..3e28ac64083e0 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/ProcessManager.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/ProcessManager.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.fnexecution.environment; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.File; @@ -28,8 +28,8 @@ import java.util.List; import java.util.Map; import javax.annotation.concurrent.ThreadSafe; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/logging/GrpcLoggingService.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/logging/GrpcLoggingService.java index c8a9b347e7216..37e59f5e58e5e 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/logging/GrpcLoggingService.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/logging/GrpcLoggingService.java @@ -25,7 +25,7 @@ import org.apache.beam.model.fnexecution.v1.BeamFnLoggingGrpc; import org.apache.beam.sdk.fn.server.FnService; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/state/GrpcStateService.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/state/GrpcStateService.java index 532f9e038f2de..42fbe58fddd7c 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/state/GrpcStateService.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/state/GrpcStateService.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.fnexecution.state; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables.getStackTraceAsString; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables.getStackTraceAsString; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/state/StateRequestHandlers.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/state/StateRequestHandlers.java index 35ee3e966f3a9..0de40dc703657 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/state/StateRequestHandlers.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/state/StateRequestHandlers.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.fnexecution.state; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.ArrayList; import java.util.Collections; @@ -54,8 +54,8 @@ import org.apache.beam.sdk.util.common.Reiterable; import org.apache.beam.sdk.values.KV; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * A set of utility methods which construct {@link StateRequestHandler}s. diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/status/BeamWorkerStatusGrpcService.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/status/BeamWorkerStatusGrpcService.java index 64ad6f57ba901..1c3e453f893f4 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/status/BeamWorkerStatusGrpcService.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/status/BeamWorkerStatusGrpcService.java @@ -36,9 +36,9 @@ import org.apache.beam.sdk.fn.server.FnService; import org.apache.beam.sdk.fn.server.HeaderAccessor; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/translation/BatchSideInputHandlerFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/translation/BatchSideInputHandlerFactory.java index 75e52683122e6..dba4af3bc3462 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/translation/BatchSideInputHandlerFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/translation/BatchSideInputHandlerFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.fnexecution.translation; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.ArrayList; import java.util.Collections; @@ -37,10 +37,10 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** {@link StateRequestHandler} that uses a {@link SideInputGetter} to access side inputs. */ @SuppressWarnings({ diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/translation/PipelineTranslatorUtils.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/translation/PipelineTranslatorUtils.java index a807b6d336db4..eaf157e0bd672 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/translation/PipelineTranslatorUtils.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/translation/PipelineTranslatorUtils.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.fnexecution.translation; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.util.Collection; @@ -44,11 +44,11 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableBiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableBiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.joda.time.Instant; /** Utilities for pipeline translation. */ diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/translation/StreamingSideInputHandlerFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/translation/StreamingSideInputHandlerFactory.java index 1ae95c77eff88..3b649759099d8 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/translation/StreamingSideInputHandlerFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/translation/StreamingSideInputHandlerFactory.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.fnexecution.translation; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.ArrayList; import java.util.Collections; @@ -36,7 +36,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * {@link StateRequestHandler} that uses {@link org.apache.beam.runners.core.SideInputHandler} to diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/wire/ByteStringCoder.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/wire/ByteStringCoder.java index 883a44741c48c..4319c23fcb2de 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/wire/ByteStringCoder.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/wire/ByteStringCoder.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.UnsafeByteOperations; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; /** * A duplicate of {@link ByteStringCoder} that uses the Apache Beam vendored protobuf. diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/wire/WireCoders.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/wire/WireCoders.java index 8cf7cd3f8ffa9..de328d5a96421 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/wire/WireCoders.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/wire/WireCoders.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.fnexecution.wire; import static org.apache.beam.runners.core.construction.BeamUrns.getUrn; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import org.apache.beam.model.pipeline.v1.RunnerApi; diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/EmbeddedSdkHarness.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/EmbeddedSdkHarness.java index 172863930dab8..e9428cdc2c941 100644 --- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/EmbeddedSdkHarness.java +++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/EmbeddedSdkHarness.java @@ -36,7 +36,7 @@ import org.apache.beam.sdk.fn.server.InProcessServerFactory; import org.apache.beam.sdk.fn.stream.OutboundObserverFactory; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.junit.rules.ExternalResource; import org.junit.rules.TestRule; diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/GrpcContextHeaderAccessorProviderTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/GrpcContextHeaderAccessorProviderTest.java index 2d6fdb0480745..187c68decc9d1 100644 --- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/GrpcContextHeaderAccessorProviderTest.java +++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/GrpcContextHeaderAccessorProviderTest.java @@ -36,7 +36,7 @@ import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessChannelBuilder; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.testing.GrpcCleanupRule; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/ServerFactoryTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/ServerFactoryTest.java index c02bb01c49c52..6e19f9d6a7b46 100644 --- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/ServerFactoryTest.java +++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/ServerFactoryTest.java @@ -50,9 +50,9 @@ import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.testing.GrpcCleanupRule; import org.apache.beam.vendor.grpc.v1p54p0.io.netty.channel.epoll.Epoll; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.net.HostAndPort; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.net.HostAndPort; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.junit.Rule; import org.junit.Test; diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/artifact/ArtifactRetrievalServiceTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/artifact/ArtifactRetrievalServiceTest.java index 940a559dd16d7..bcce5f55a994a 100644 --- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/artifact/ArtifactRetrievalServiceTest.java +++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/artifact/ArtifactRetrievalServiceTest.java @@ -33,9 +33,9 @@ import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessChannelBuilder; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessServerBuilder; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.testing.GrpcCleanupRule; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -113,6 +113,7 @@ private String getArtifact(RunnerApi.ArtifactInformation artifact) { return all.toStringUtf8(); } + @SuppressWarnings("InlineMeInliner") // inline `Strings.repeat()` - Java 11+ API only @Test public void testRetrieveArtifacts() throws IOException, InterruptedException { Map artifacts = diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/artifact/ArtifactStagingServiceTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/artifact/ArtifactStagingServiceTest.java index 95be0cc516ade..3f80c97f4336e 100644 --- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/artifact/ArtifactStagingServiceTest.java +++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/artifact/ArtifactStagingServiceTest.java @@ -35,10 +35,10 @@ import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessServerBuilder; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.testing.GrpcCleanupRule; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -145,6 +145,7 @@ private String getArtifact(RunnerApi.ArtifactInformation artifact) { return all.toStringUtf8(); } + @SuppressWarnings("InlineMeInliner") // inline `Strings.repeat()` - Java 11+ API only @Test public void testStageArtifacts() throws InterruptedException, ExecutionException { List contentsList = diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/DefaultJobBundleFactoryTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/DefaultJobBundleFactoryTest.java index 5fb2b0c52b8a1..7adc78a22ef81 100644 --- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/DefaultJobBundleFactoryTest.java +++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/DefaultJobBundleFactoryTest.java @@ -71,7 +71,7 @@ import org.apache.beam.sdk.options.PortablePipelineOptions; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Struct; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/ProcessBundleDescriptorsTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/ProcessBundleDescriptorsTest.java index 788fc38af54b8..47df7b7c9735b 100644 --- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/ProcessBundleDescriptorsTest.java +++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/ProcessBundleDescriptorsTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.fnexecution.control; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; @@ -58,8 +58,8 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Test; /** Tests for {@link ProcessBundleDescriptors}. */ diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/RemoteExecutionTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/RemoteExecutionTest.java index 8edf8a94036af..ea23e28ddb666 100644 --- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/RemoteExecutionTest.java +++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/RemoteExecutionTest.java @@ -19,7 +19,7 @@ import static org.apache.beam.sdk.options.ExperimentalOptions.addExperiment; import static org.apache.beam.sdk.util.WindowedValue.valueInGlobalWindow; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -143,11 +143,11 @@ import org.apache.beam.sdk.values.PCollectionList; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.hamcrest.Matcher; import org.hamcrest.Matchers; import org.hamcrest.collection.IsEmptyIterable; diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClientTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClientTest.java index fffd0eef411af..6ebe5eca3e651 100644 --- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClientTest.java +++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClientTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.fnexecution.control; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.getOnlyElement; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.getOnlyElement; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.is; @@ -89,8 +89,8 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SingleEnvironmentInstanceJobBundleFactoryTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SingleEnvironmentInstanceJobBundleFactoryTest.java index a68aeda49d33d..11973385fe166 100644 --- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SingleEnvironmentInstanceJobBundleFactoryTest.java +++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SingleEnvironmentInstanceJobBundleFactoryTest.java @@ -48,7 +48,7 @@ import org.apache.beam.sdk.options.ExperimentalOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/DockerCommandTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/DockerCommandTest.java index 6c27975e61435..e7d294fffc5d3 100644 --- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/DockerCommandTest.java +++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/DockerCommandTest.java @@ -26,8 +26,8 @@ import java.io.IOException; import java.util.concurrent.TimeUnit; import org.apache.beam.runners.fnexecution.environment.testing.NeedsDocker; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Stopwatch; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Stopwatch; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/state/StateRequestHandlersTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/state/StateRequestHandlersTest.java index 8314ab343449a..5529133b5b496 100644 --- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/state/StateRequestHandlersTest.java +++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/state/StateRequestHandlersTest.java @@ -38,7 +38,7 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/status/BeamWorkerStatusGrpcServiceTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/status/BeamWorkerStatusGrpcServiceTest.java index 438e8145191ea..e1d6401d5e538 100644 --- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/status/BeamWorkerStatusGrpcServiceTest.java +++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/status/BeamWorkerStatusGrpcServiceTest.java @@ -43,7 +43,7 @@ import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessChannelBuilder; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.testing.GrpcCleanupRule; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.junit.After; import org.junit.Before; import org.junit.Rule; diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/translation/PipelineTranslatorUtilsTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/translation/PipelineTranslatorUtilsTest.java index 36997d66eaaf5..a0fbd26d72f12 100644 --- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/translation/PipelineTranslatorUtilsTest.java +++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/translation/PipelineTranslatorUtilsTest.java @@ -23,8 +23,8 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; /** Tests for {@link PipelineTranslatorUtils}. */ diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/ByteStringCoderTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/ByteStringCoderTest.java index 8535ad525d8e7..4442674f4e6eb 100644 --- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/ByteStringCoderTest.java +++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/ByteStringCoderTest.java @@ -32,7 +32,7 @@ import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/CommonCoderTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/CommonCoderTest.java index f0e16b3c156d4..389cdff232a2f 100644 --- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/CommonCoderTest.java +++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/CommonCoderTest.java @@ -18,10 +18,10 @@ package org.apache.beam.runners.fnexecution.wire; import static org.apache.beam.runners.core.construction.BeamUrns.getUrn; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList.toImmutableList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList.toImmutableList; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; @@ -89,13 +89,13 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableBiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CharStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableBiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CharStreams; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/LengthPrefixUnknownCodersTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/LengthPrefixUnknownCodersTest.java index a423b3686cd93..ddce63a19c991 100644 --- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/LengthPrefixUnknownCodersTest.java +++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/LengthPrefixUnknownCodersTest.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.coders.LengthPrefixCoder; import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/java-job-service/build.gradle b/runners/java-job-service/build.gradle index 60585838c15e8..3c3ff3ad92eb8 100644 --- a/runners/java-job-service/build.gradle +++ b/runners/java-job-service/build.gradle @@ -23,7 +23,7 @@ applyJavaNature( description = "Apache Beam :: Runners :: Java Job Service" dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(":runners:core-construction-java") implementation project(path: ":model:pipeline", configuration: "shadow") implementation project(path: ":sdks:java:core", configuration: "shadow") diff --git a/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/InMemoryJobService.java b/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/InMemoryJobService.java index 18c6437edeb82..17efbf9a06ec5 100644 --- a/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/InMemoryJobService.java +++ b/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/InMemoryJobService.java @@ -61,8 +61,8 @@ import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.StatusException; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.StatusRuntimeException; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/JobInvocation.java b/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/JobInvocation.java index 44144dee72f13..b7a5dbf035187 100644 --- a/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/JobInvocation.java +++ b/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/JobInvocation.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.jobsubmission; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables.getRootCause; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables.getStackTraceAsString; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables.getRootCause; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables.getStackTraceAsString; import java.io.IOException; import java.util.ArrayList; @@ -36,10 +36,10 @@ import org.apache.beam.runners.fnexecution.provisioning.JobInfo; import org.apache.beam.sdk.PipelineResult; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.util.Timestamps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.FutureCallback; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Futures; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListenableFuture; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.FutureCallback; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Futures; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListenableFuture; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/JobInvoker.java b/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/JobInvoker.java index 3682fe6939fd0..96157421f7942 100644 --- a/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/JobInvoker.java +++ b/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/JobInvoker.java @@ -23,9 +23,9 @@ import javax.annotation.Nullable; import org.apache.beam.model.pipeline.v1.RunnerApi; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Struct; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; /** Factory to create {@link JobInvocation} instances. */ @SuppressWarnings({ diff --git a/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/JobServerDriver.java b/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/JobServerDriver.java index 3b1dfe64db30f..a342593b21f76 100644 --- a/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/JobServerDriver.java +++ b/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/JobServerDriver.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.expansion.service.ExpansionService; import org.apache.beam.sdk.fn.server.GrpcFnServer; import org.apache.beam.sdk.fn.server.ServerFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.kohsuke.args4j.Option; import org.kohsuke.args4j.spi.ExplicitBooleanOptionHandler; import org.slf4j.Logger; diff --git a/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/PortablePipelineJarCreator.java b/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/PortablePipelineJarCreator.java index 62b37f138a2d8..b5463354bed59 100644 --- a/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/PortablePipelineJarCreator.java +++ b/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/PortablePipelineJarCreator.java @@ -48,9 +48,9 @@ import org.apache.beam.sdk.options.PortablePipelineOptions; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.MessageOrBuilder; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.util.JsonFormat; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; import org.apache.commons.compress.utils.IOUtils; import org.joda.time.Duration; import org.slf4j.Logger; diff --git a/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/PortablePipelineJarUtils.java b/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/PortablePipelineJarUtils.java index 785d88349e8e2..02333021e53e9 100644 --- a/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/PortablePipelineJarUtils.java +++ b/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/PortablePipelineJarUtils.java @@ -30,7 +30,7 @@ import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Message; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Struct; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.util.JsonFormat; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; /** * Contains common code for writing and reading portable pipeline jars. diff --git a/runners/java-job-service/src/test/java/org/apache/beam/runners/jobsubmission/InMemoryJobServiceTest.java b/runners/java-job-service/src/test/java/org/apache/beam/runners/jobsubmission/InMemoryJobServiceTest.java index 60e95053db7d9..45d164c6e605a 100644 --- a/runners/java-job-service/src/test/java/org/apache/beam/runners/jobsubmission/InMemoryJobServiceTest.java +++ b/runners/java-job-service/src/test/java/org/apache/beam/runners/jobsubmission/InMemoryJobServiceTest.java @@ -40,7 +40,7 @@ import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Struct; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.StatusException; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/java-job-service/src/test/java/org/apache/beam/runners/jobsubmission/JobInvocationTest.java b/runners/java-job-service/src/test/java/org/apache/beam/runners/jobsubmission/JobInvocationTest.java index f294ec7770ecd..e45db3a0119c3 100644 --- a/runners/java-job-service/src/test/java/org/apache/beam/runners/jobsubmission/JobInvocationTest.java +++ b/runners/java-job-service/src/test/java/org/apache/beam/runners/jobsubmission/JobInvocationTest.java @@ -34,8 +34,8 @@ import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.metrics.MetricResults; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Struct; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.joda.time.Duration; import org.junit.After; import org.junit.Before; diff --git a/runners/java-job-service/src/test/java/org/apache/beam/runners/jobsubmission/PortablePipelineJarCreatorTest.java b/runners/java-job-service/src/test/java/org/apache/beam/runners/jobsubmission/PortablePipelineJarCreatorTest.java index 48b6dd8bfa26a..8acf07d3c795f 100644 --- a/runners/java-job-service/src/test/java/org/apache/beam/runners/jobsubmission/PortablePipelineJarCreatorTest.java +++ b/runners/java-job-service/src/test/java/org/apache/beam/runners/jobsubmission/PortablePipelineJarCreatorTest.java @@ -42,7 +42,7 @@ import org.apache.beam.model.pipeline.v1.RunnerApi; import org.apache.beam.runners.fnexecution.artifact.ArtifactRetrievalService; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/runners/jet/build.gradle b/runners/jet/build.gradle index db1c6515d9b03..4b87fe940407d 100644 --- a/runners/jet/build.gradle +++ b/runners/jet/build.gradle @@ -46,7 +46,7 @@ dependencies { implementation project(":runners:core-construction-java") implementation "com.hazelcast.jet:hazelcast-jet:$jet_version" implementation library.java.joda_time - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.slf4j_api testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") diff --git a/runners/jet/src/main/java/org/apache/beam/runners/jet/JetRunnerRegistrar.java b/runners/jet/src/main/java/org/apache/beam/runners/jet/JetRunnerRegistrar.java index 203d1cb6dc8aa..6b9a6bc64f8af 100644 --- a/runners/jet/src/main/java/org/apache/beam/runners/jet/JetRunnerRegistrar.java +++ b/runners/jet/src/main/java/org/apache/beam/runners/jet/JetRunnerRegistrar.java @@ -22,7 +22,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; import org.apache.beam.sdk.runners.PipelineRunnerRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * Contains the {@link PipelineRunnerRegistrar} and {@link PipelineOptionsRegistrar} for the {@link diff --git a/runners/jet/src/main/java/org/apache/beam/runners/jet/Utils.java b/runners/jet/src/main/java/org/apache/beam/runners/jet/Utils.java index 1c34cb98f2547..aa19fb2fa444c 100644 --- a/runners/jet/src/main/java/org/apache/beam/runners/jet/Utils.java +++ b/runners/jet/src/main/java/org/apache/beam/runners/jet/Utils.java @@ -53,7 +53,7 @@ import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; /** Various common methods used by the Jet based runner. */ diff --git a/runners/jet/src/main/java/org/apache/beam/runners/jet/metrics/JetMetricResults.java b/runners/jet/src/main/java/org/apache/beam/runners/jet/metrics/JetMetricResults.java index a448542f1819b..8e28f3fda0e83 100644 --- a/runners/jet/src/main/java/org/apache/beam/runners/jet/metrics/JetMetricResults.java +++ b/runners/jet/src/main/java/org/apache/beam/runners/jet/metrics/JetMetricResults.java @@ -33,8 +33,8 @@ import org.apache.beam.sdk.metrics.MetricResult; import org.apache.beam.sdk.metrics.MetricResults; import org.apache.beam.sdk.metrics.MetricsFilter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicate; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicate; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; import org.checkerframework.checker.nullness.qual.Nullable; /** Jet specific {@link MetricResults}. */ diff --git a/runners/jet/src/main/java/org/apache/beam/runners/jet/metrics/JetMetricsContainer.java b/runners/jet/src/main/java/org/apache/beam/runners/jet/metrics/JetMetricsContainer.java index 967f1e8255180..5441d05dcf76d 100644 --- a/runners/jet/src/main/java/org/apache/beam/runners/jet/metrics/JetMetricsContainer.java +++ b/runners/jet/src/main/java/org/apache/beam/runners/jet/metrics/JetMetricsContainer.java @@ -32,7 +32,7 @@ import org.apache.beam.sdk.metrics.MetricKey; import org.apache.beam.sdk.metrics.MetricName; import org.apache.beam.sdk.metrics.MetricsContainer; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Jet specific implementation of {@link MetricsContainer}. */ public class JetMetricsContainer implements MetricsContainer { diff --git a/runners/jet/src/main/java/org/apache/beam/runners/jet/processors/AbstractParDoP.java b/runners/jet/src/main/java/org/apache/beam/runners/jet/processors/AbstractParDoP.java index b9858ff65627a..c5bd4ecaf2ef0 100644 --- a/runners/jet/src/main/java/org/apache/beam/runners/jet/processors/AbstractParDoP.java +++ b/runners/jet/src/main/java/org/apache/beam/runners/jet/processors/AbstractParDoP.java @@ -63,7 +63,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; @SuppressWarnings({ "rawtypes", // TODO(https://github.com/apache/beam/issues/20447) diff --git a/runners/jet/src/main/java/org/apache/beam/runners/jet/processors/AssignWindowP.java b/runners/jet/src/main/java/org/apache/beam/runners/jet/processors/AssignWindowP.java index f459769db45a8..82151a812d63d 100644 --- a/runners/jet/src/main/java/org/apache/beam/runners/jet/processors/AssignWindowP.java +++ b/runners/jet/src/main/java/org/apache/beam/runners/jet/processors/AssignWindowP.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Instant; /** diff --git a/runners/local-java/OWNERS b/runners/local-java/OWNERS deleted file mode 100644 index af32c311923a2..0000000000000 --- a/runners/local-java/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lukecwik diff --git a/runners/portability/OWNERS b/runners/portability/OWNERS deleted file mode 100644 index af32c311923a2..0000000000000 --- a/runners/portability/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lukecwik diff --git a/runners/portability/java/build.gradle b/runners/portability/java/build.gradle index 8c39b8aaf58b4..ee6fe3b8fd8db 100644 --- a/runners/portability/java/build.gradle +++ b/runners/portability/java/build.gradle @@ -42,7 +42,7 @@ dependencies { implementation library.java.joda_time implementation library.java.slf4j_api implementation library.java.vendored_grpc_1_54_0 - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre testImplementation project(path: ":runners:core-construction-java", configuration: "testRuntimeMigration") testImplementation library.java.hamcrest diff --git a/runners/portability/java/src/main/java/org/apache/beam/runners/portability/CloseableResource.java b/runners/portability/java/src/main/java/org/apache/beam/runners/portability/CloseableResource.java index 52d42d54e7f9b..8f4756b4aed77 100644 --- a/runners/portability/java/src/main/java/org/apache/beam/runners/portability/CloseableResource.java +++ b/runners/portability/java/src/main/java/org/apache/beam/runners/portability/CloseableResource.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.portability; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/runners/portability/java/src/main/java/org/apache/beam/runners/portability/PortableMetrics.java b/runners/portability/java/src/main/java/org/apache/beam/runners/portability/PortableMetrics.java index 946f757ccb223..fc94e408bfd33 100644 --- a/runners/portability/java/src/main/java/org/apache/beam/runners/portability/PortableMetrics.java +++ b/runners/portability/java/src/main/java/org/apache/beam/runners/portability/PortableMetrics.java @@ -41,7 +41,7 @@ import org.apache.beam.sdk.metrics.MetricResult; import org.apache.beam.sdk.metrics.MetricResults; import org.apache.beam.sdk.metrics.MetricsFilter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; @SuppressWarnings({ "nullness" // TODO(https://github.com/apache/beam/issues/20497) diff --git a/runners/portability/java/src/main/java/org/apache/beam/runners/portability/PortableRunner.java b/runners/portability/java/src/main/java/org/apache/beam/runners/portability/PortableRunner.java index 23d99d88e21ac..3a3034d922634 100644 --- a/runners/portability/java/src/main/java/org/apache/beam/runners/portability/PortableRunner.java +++ b/runners/portability/java/src/main/java/org/apache/beam/runners/portability/PortableRunner.java @@ -52,8 +52,8 @@ import org.apache.beam.sdk.options.PortablePipelineOptions; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ManagedChannel; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/portability/java/src/main/java/org/apache/beam/runners/portability/PortableRunnerRegistrar.java b/runners/portability/java/src/main/java/org/apache/beam/runners/portability/PortableRunnerRegistrar.java index cf6904fb7c579..588d8d6a85803 100644 --- a/runners/portability/java/src/main/java/org/apache/beam/runners/portability/PortableRunnerRegistrar.java +++ b/runners/portability/java/src/main/java/org/apache/beam/runners/portability/PortableRunnerRegistrar.java @@ -20,7 +20,7 @@ import com.google.auto.service.AutoService; import org.apache.beam.sdk.PipelineRunner; import org.apache.beam.sdk.runners.PipelineRunnerRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Registrar for the portable runner. */ @AutoService(PipelineRunnerRegistrar.class) diff --git a/runners/portability/java/src/main/java/org/apache/beam/runners/portability/testing/TestPortablePipelineOptions.java b/runners/portability/java/src/main/java/org/apache/beam/runners/portability/testing/TestPortablePipelineOptions.java index 5e9324a74102b..a7808899ca641 100644 --- a/runners/portability/java/src/main/java/org/apache/beam/runners/portability/testing/TestPortablePipelineOptions.java +++ b/runners/portability/java/src/main/java/org/apache/beam/runners/portability/testing/TestPortablePipelineOptions.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.options.PortablePipelineOptions; import org.apache.beam.sdk.options.Validation.Required; import org.apache.beam.sdk.testing.TestPipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Options for {@link TestPortableRunner}. */ public interface TestPortablePipelineOptions extends TestPipelineOptions, PortablePipelineOptions { diff --git a/runners/portability/java/src/main/java/org/apache/beam/runners/portability/testing/TestUniversalRunner.java b/runners/portability/java/src/main/java/org/apache/beam/runners/portability/testing/TestUniversalRunner.java index 145f39d1b653b..533106869c62b 100644 --- a/runners/portability/java/src/main/java/org/apache/beam/runners/portability/testing/TestUniversalRunner.java +++ b/runners/portability/java/src/main/java/org/apache/beam/runners/portability/testing/TestUniversalRunner.java @@ -33,8 +33,8 @@ import org.apache.beam.sdk.options.PortablePipelineOptions; import org.apache.beam.sdk.runners.PipelineRunnerRegistrar; import org.apache.beam.sdk.testing.TestPipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.Matchers; /** A {@link PipelineRunner} a {@link Pipeline} against a {@code JobService}. */ diff --git a/runners/portability/java/src/test/java/org/apache/beam/runners/portability/PortableRunnerTest.java b/runners/portability/java/src/test/java/org/apache/beam/runners/portability/PortableRunnerTest.java index 1d260949b1bbe..c3895e4c35cf6 100644 --- a/runners/portability/java/src/test/java/org/apache/beam/runners/portability/PortableRunnerTest.java +++ b/runners/portability/java/src/test/java/org/apache/beam/runners/portability/PortableRunnerTest.java @@ -49,8 +49,8 @@ import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Server; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessServerBuilder; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.testing.GrpcCleanupRule; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/runners/samza/OWNERS b/runners/samza/OWNERS deleted file mode 100644 index fbd140d90fbbc..0000000000000 --- a/runners/samza/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - alnzng - - mynameborat - - xinyuiscool diff --git a/runners/samza/build.gradle b/runners/samza/build.gradle index d0579b48ae067..1dc4332f8ad44 100644 --- a/runners/samza/build.gradle +++ b/runners/samza/build.gradle @@ -43,7 +43,7 @@ configurations { def samza_version = "1.6.0" dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(":runners:core-java") implementation project(":runners:core-construction-java") diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaJobInvoker.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaJobInvoker.java index a6f252bf98648..5cd4a8ede33d5 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaJobInvoker.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaJobInvoker.java @@ -27,8 +27,8 @@ import org.apache.beam.runners.jobsubmission.PortablePipelineJarCreator; import org.apache.beam.runners.jobsubmission.PortablePipelineRunner; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Struct; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaPipelineOptions.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaPipelineOptions.java index fc359c1d4ca9f..a34303d925522 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaPipelineOptions.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaPipelineOptions.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.options.Description; import org.apache.beam.sdk.options.Hidden; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.samza.config.ConfigLoaderFactory; import org.apache.samza.config.loaders.PropertiesConfigLoaderFactory; import org.apache.samza.metrics.MetricsReporter; @@ -173,4 +173,10 @@ public ExecutorService create(PipelineOptions options) { new ThreadFactoryBuilder().setNameFormat("Process Element Thread-%d").build()); } } + + @Description("Enable/disable late data dropping in GroupByKey/Combine transforms") + @Default.Boolean(false) + boolean getDropLateData(); + + void setDropLateData(boolean dropLateData); } diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaPipelineOptionsValidator.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaPipelineOptionsValidator.java index 5beb9fe0e562e..1db0974b5d303 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaPipelineOptionsValidator.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaPipelineOptionsValidator.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.samza; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.apache.samza.config.JobConfig.JOB_CONTAINER_THREAD_POOL_SIZE; import java.util.HashMap; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaRunner.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaRunner.java index 3ee9197d2f388..26745114b43d7 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaRunner.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaRunner.java @@ -45,7 +45,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsValidator; import org.apache.beam.sdk.values.PValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; import org.apache.samza.application.StreamApplication; import org.apache.samza.config.Config; import org.apache.samza.context.ExternalContext; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaRunnerRegistrar.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaRunnerRegistrar.java index dd44b86664b64..102838975d138 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaRunnerRegistrar.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaRunnerRegistrar.java @@ -22,7 +22,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; import org.apache.beam.sdk.runners.PipelineRunnerRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * AutoService registrar - will register SamzaRunner and SamzaOptions as possible pipeline runner diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/TestSamzaRunner.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/TestSamzaRunner.java index f9bbae85410d6..810fc0c983f79 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/TestSamzaRunner.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/TestSamzaRunner.java @@ -21,8 +21,9 @@ import static org.apache.samza.config.JobConfig.JOB_LOGGED_STORE_BASE_DIR; import static org.apache.samza.config.JobConfig.JOB_NON_LOGGED_STORE_BASE_DIR; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.File; -import java.nio.file.Paths; +import java.nio.file.Files; import java.util.HashMap; import java.util.Map; import org.apache.beam.runners.samza.translation.ConfigBuilder; @@ -37,24 +38,18 @@ public class TestSamzaRunner extends PipelineRunner { private final SamzaRunner delegate; + private final File storeDir; public static TestSamzaRunner fromOptions(PipelineOptions options) { - return new TestSamzaRunner(createSamzaPipelineOptions(options)); + return new TestSamzaRunner(options); } - public static SamzaPipelineOptions createSamzaPipelineOptions(PipelineOptions options) { + public static SamzaPipelineOptions createSamzaPipelineOptions( + PipelineOptions options, File storeDir) { try { final SamzaPipelineOptions samzaOptions = PipelineOptionsValidator.validate(SamzaPipelineOptions.class, options); final Map config = new HashMap<>(ConfigBuilder.localRunConfig()); - final File storeDir = - Paths.get(System.getProperty("java.io.tmpdir"), "beam-samza-test").toFile(); - // Re-create the folder for test stores - FileUtils.deleteDirectory(storeDir); - if (!storeDir.mkdir()) { - // ignore - } - config.put(JOB_LOGGED_STORE_BASE_DIR, storeDir.getAbsolutePath()); config.put(JOB_NON_LOGGED_STORE_BASE_DIR, storeDir.getAbsolutePath()); config.put(JOB_JMX_ENABLED, "false"); @@ -69,15 +64,26 @@ public static SamzaPipelineOptions createSamzaPipelineOptions(PipelineOptions op } } - public TestSamzaRunner(SamzaPipelineOptions options) { - this.delegate = SamzaRunner.fromOptions(options); + private static File createStoreDir() { + try { + return Files.createTempDirectory("beam-samza-test").toFile(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public TestSamzaRunner(PipelineOptions options) { + this.storeDir = createStoreDir(); + this.delegate = SamzaRunner.fromOptions(createSamzaPipelineOptions(options, storeDir)); } @Override + @SuppressFBWarnings(value = "DE_MIGHT_IGNORE") public PipelineResult run(Pipeline pipeline) { try { final PipelineResult result = delegate.run(pipeline); result.waitUntilFinish(); + return result; } catch (Throwable t) { // Search for AssertionError. If present use it as the cause of the pipeline failure. @@ -91,6 +97,13 @@ public PipelineResult run(Pipeline pipeline) { } throw t; + } finally { + try { + // delete the store folder + FileUtils.deleteDirectory(storeDir); + } catch (Exception ignore) { + // Ignore + } } } } diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/adapter/BoundedSourceSystem.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/adapter/BoundedSourceSystem.java index a5b663fa3b17a..eb2b423a1171f 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/adapter/BoundedSourceSystem.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/adapter/BoundedSourceSystem.java @@ -43,7 +43,7 @@ import org.apache.beam.sdk.io.BoundedSource.BoundedReader; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.samza.Partition; import org.apache.samza.SamzaException; import org.apache.samza.config.Config; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/adapter/UnboundedSourceSystem.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/adapter/UnboundedSourceSystem.java index 0c6b6cee6aa35..3b53894954d7e 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/adapter/UnboundedSourceSystem.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/adapter/UnboundedSourceSystem.java @@ -47,8 +47,8 @@ import org.apache.beam.sdk.io.UnboundedSource.UnboundedReader; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.samza.Partition; import org.apache.samza.SamzaException; import org.apache.samza.config.Config; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/metrics/SamzaMetricOp.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/metrics/SamzaMetricOp.java index b624006fd988f..8738d90c66c55 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/metrics/SamzaMetricOp.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/metrics/SamzaMetricOp.java @@ -28,7 +28,7 @@ import org.apache.beam.runners.samza.runtime.OpEmitter; import org.apache.beam.runners.samza.util.PipelineJsonRenderer; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.apache.samza.config.Config; import org.apache.samza.context.Context; import org.apache.samza.operators.Scheduler; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/metrics/SamzaTransformMetricRegistry.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/metrics/SamzaTransformMetricRegistry.java index e4849c31fa6ea..add207752f06a 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/metrics/SamzaTransformMetricRegistry.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/metrics/SamzaTransformMetricRegistry.java @@ -24,7 +24,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.apache.samza.context.Context; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/ClassicBundleManager.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/ClassicBundleManager.java index 47899dd43c02d..1d75afbb4c575 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/ClassicBundleManager.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/ClassicBundleManager.java @@ -30,8 +30,8 @@ import org.apache.beam.sdk.state.TimeDomain; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.apache.samza.operators.Scheduler; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/DoFnOp.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/DoFnOp.java index c5fda797bddd7..b963f75f71d56 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/DoFnOp.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/DoFnOp.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.samza.runtime; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.ArrayList; import java.util.Collection; @@ -58,7 +58,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; import org.apache.samza.config.Config; import org.apache.samza.context.Context; import org.apache.samza.operators.Scheduler; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/FutureCollectorImpl.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/FutureCollectorImpl.java index 10f915a8baf9c..f54f5e441b84b 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/FutureCollectorImpl.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/FutureCollectorImpl.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.samza.runtime; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.ArrayList; import java.util.Collection; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/GroupByKeyOp.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/GroupByKeyOp.java index 3ecd406da615f..1b19275dd9675 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/GroupByKeyOp.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/GroupByKeyOp.java @@ -180,11 +180,19 @@ public TimerInternals timerInternals() { DoFnSchemaInformation.create(), Collections.emptyMap()); + final DoFnRunner, KV> dropLateDataRunner = + pipelineOptions.getDropLateData() + ? DoFnRunners.lateDataDroppingRunner( + doFnRunner, keyedInternals.timerInternals(), windowingStrategy) + : doFnRunner; + final SamzaExecutionContext executionContext = (SamzaExecutionContext) context.getApplicationContainerContext(); - this.fnRunner = + final DoFnRunner, KV> doFnRunnerWithMetrics = DoFnRunnerWithMetrics.wrap( - doFnRunner, executionContext.getMetricsContainer(), transformFullName); + dropLateDataRunner, executionContext.getMetricsContainer(), transformFullName); + + this.fnRunner = new DoFnRunnerWithKeyedInternals<>(doFnRunnerWithMetrics, keyedInternals); } @Override diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/KeyedInternals.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/KeyedInternals.java index c661be1096cc0..dc442d88ac326 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/KeyedInternals.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/KeyedInternals.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.samza.runtime; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.ArrayList; import java.util.List; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/PortableDoFnOp.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/PortableDoFnOp.java index c66c004103964..fcac505f8cf9c 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/PortableDoFnOp.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/PortableDoFnOp.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.samza.runtime; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.ArrayList; import java.util.Collection; @@ -54,7 +54,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; import org.apache.samza.config.Config; import org.apache.samza.context.Context; import org.apache.samza.operators.Scheduler; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaAssignContext.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaAssignContext.java index 97e64b31def54..1a0caaa99c6d0 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaAssignContext.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaAssignContext.java @@ -20,7 +20,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Instant; @SuppressWarnings({"keyfor", "nullness"}) // TODO(https://github.com/apache/beam/issues/20497) diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaDoFnRunners.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaDoFnRunners.java index 6f658262e9635..758a341b2a5fc 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaDoFnRunners.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaDoFnRunners.java @@ -66,7 +66,7 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.samza.context.Context; import org.joda.time.Instant; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaStateRequestHandlers.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaStateRequestHandlers.java index 148b08d66a2a9..5af189b7e102a 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaStateRequestHandlers.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaStateRequestHandlers.java @@ -41,7 +41,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.apache.samza.context.TaskContext; /** diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaStoreStateInternals.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaStoreStateInternals.java index 44e8055ea82ed..2ba60a168b99a 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaStoreStateInternals.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaStoreStateInternals.java @@ -66,10 +66,10 @@ import org.apache.beam.sdk.transforms.CombineWithContext; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.windowing.TimestampCombiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Ints; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.UnsignedBytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Ints; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedBytes; import org.apache.samza.config.Config; import org.apache.samza.context.TaskContext; import org.apache.samza.serializers.Serde; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/ConfigBuilder.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/ConfigBuilder.java index c40476532bb64..c9832ecbabd3b 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/ConfigBuilder.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/ConfigBuilder.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.samza.translation; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.apache.samza.config.JobConfig.JOB_AUTOSIZING_CONTAINER_THREAD_POOL_SIZE; import static org.apache.samza.config.JobConfig.JOB_CONTAINER_THREAD_POOL_SIZE; import static org.apache.samza.config.JobConfig.JOB_ID; @@ -41,8 +41,8 @@ import org.apache.beam.runners.samza.runtime.SamzaStoreStateInternals; import org.apache.beam.runners.samza.util.ConfigUtils; import org.apache.beam.runners.samza.util.PortableConfigUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.samza.config.ApplicationConfig; import org.apache.samza.config.Config; import org.apache.samza.config.ConfigLoaderFactory; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/ConfigContext.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/ConfigContext.java index 3fc405d15a5b8..954f95a593ee4 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/ConfigContext.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/ConfigContext.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.runners.TransformHierarchy; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** Helper that provides context data such as output for config generation. */ @SuppressWarnings({ diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/FlattenPCollectionsTranslator.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/FlattenPCollectionsTranslator.java index 9d529137628c7..03fd613f85e9d 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/FlattenPCollectionsTranslator.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/FlattenPCollectionsTranslator.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.samza.translation; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.ArrayList; import java.util.HashSet; @@ -33,7 +33,7 @@ import org.apache.beam.sdk.transforms.Flatten; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.samza.operators.MessageStream; /** diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/GroupByKeyTranslator.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/GroupByKeyTranslator.java index 6a4f3fb5207de..f88c36c171c85 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/GroupByKeyTranslator.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/GroupByKeyTranslator.java @@ -51,7 +51,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.samza.operators.MessageStream; import org.apache.samza.serializers.KVSerde; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/ParDoBoundMultiTranslator.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/ParDoBoundMultiTranslator.java index 608e5f7238ce9..2c998417b403d 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/ParDoBoundMultiTranslator.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/ParDoBoundMultiTranslator.java @@ -69,7 +69,7 @@ import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; import org.apache.samza.operators.MessageStream; import org.apache.samza.operators.functions.FlatMapFunction; import org.apache.samza.operators.functions.WatermarkFunction; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/PortableTranslationContext.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/PortableTranslationContext.java index 2195489ca13fb..f64c8dc616e3b 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/PortableTranslationContext.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/PortableTranslationContext.java @@ -27,7 +27,7 @@ import org.apache.beam.runners.fnexecution.provisioning.JobInfo; import org.apache.beam.runners.samza.SamzaPipelineOptions; import org.apache.beam.runners.samza.runtime.OpMessage; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.samza.application.descriptors.StreamApplicationDescriptor; import org.apache.samza.operators.KV; import org.apache.samza.operators.MessageStream; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/ReadTranslator.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/ReadTranslator.java index 174294291bc15..e1530ead60adb 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/ReadTranslator.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/ReadTranslator.java @@ -32,7 +32,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.samza.operators.KV; import org.apache.samza.serializers.KVSerde; import org.apache.samza.serializers.NoOpSerde; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaPipelineTranslator.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaPipelineTranslator.java index 9ab8a8fe2500b..74eb6d71e7f0e 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaPipelineTranslator.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaPipelineTranslator.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.samza.translation; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; import java.util.HashMap; @@ -32,7 +32,7 @@ import org.apache.beam.sdk.transforms.Combine; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** This class knows all the translators from a primitive BEAM transform to a Samza operator. */ @SuppressWarnings({ diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaPortablePipelineTranslator.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaPortablePipelineTranslator.java index 055dc4f2d721c..a5ec212ed3deb 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaPortablePipelineTranslator.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaPortablePipelineTranslator.java @@ -28,7 +28,7 @@ import org.apache.beam.runners.core.construction.graph.PipelineNode; import org.apache.beam.runners.core.construction.graph.QueryablePipeline; import org.apache.beam.runners.samza.SamzaPipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaPublishView.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaPublishView.java index 9a50d3d579ac9..a3ebbffef9a8f 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaPublishView.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaPublishView.java @@ -59,7 +59,7 @@ static class SamzaPublishViewPayloadTranslator SamzaPublishViewPayloadTranslator() {} @Override - public String getUrn(SamzaPublishView transform) { + public String getUrn() { return SAMZA_PUBLISH_VIEW_URN; } } diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaPublishViewTransformOverride.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaPublishViewTransformOverride.java index 1f8fbcc142570..839f4d6145ca3 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaPublishViewTransformOverride.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaPublishViewTransformOverride.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.transforms.View; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** Samza override for {@link View} (side input) transforms. */ class SamzaPublishViewTransformOverride diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaTestStreamSystemFactory.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaTestStreamSystemFactory.java index 052d03a6a223a..7287994e9e8c9 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaTestStreamSystemFactory.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaTestStreamSystemFactory.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.samza.Partition; import org.apache.samza.SamzaException; import org.apache.samza.config.Config; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaTestStreamTranslator.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaTestStreamTranslator.java index 5afe34ac8cc00..77f3b885a09d8 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaTestStreamTranslator.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaTestStreamTranslator.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.samza.operators.KV; import org.apache.samza.serializers.KVSerde; import org.apache.samza.serializers.NoOpSerde; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaTransformOverrides.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaTransformOverrides.java index df18ada4984fb..b7063df11ae69 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaTransformOverrides.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaTransformOverrides.java @@ -24,7 +24,7 @@ import org.apache.beam.runners.core.construction.SplittableParDoNaiveBounded; import org.apache.beam.runners.core.construction.UnsupportedOverrideFactory; import org.apache.beam.sdk.runners.PTransformOverride; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** {@link org.apache.beam.sdk.transforms.PTransform} overrides for Samza runner. */ @SuppressWarnings({ diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/TranslationContext.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/TranslationContext.java index e3e4bb46fce73..49402393cf70f 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/TranslationContext.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/TranslationContext.java @@ -45,8 +45,8 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.samza.application.descriptors.StreamApplicationDescriptor; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/util/DoFnUtils.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/util/DoFnUtils.java index c0901111e44ef..de212367c5145 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/util/DoFnUtils.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/util/DoFnUtils.java @@ -21,8 +21,8 @@ import java.util.stream.Collectors; import org.apache.beam.runners.core.construction.graph.ExecutableStage; import org.apache.beam.runners.core.construction.graph.PipelineNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.commons.collections.CollectionUtils; /** Utils for {@link org.apache.beam.runners.samza.runtime.DoFnOp}. */ diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/util/PipelineJsonRenderer.java b/runners/samza/src/main/java/org/apache/beam/runners/samza/util/PipelineJsonRenderer.java index befe7547d0214..4c42daab68333 100644 --- a/runners/samza/src/main/java/org/apache/beam/runners/samza/util/PipelineJsonRenderer.java +++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/util/PipelineJsonRenderer.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.samza.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.errorprone.annotations.DoNotCall; import com.google.errorprone.annotations.FormatMethod; @@ -47,8 +47,8 @@ import org.apache.beam.vendor.grpc.v1p54p0.com.google.gson.JsonArray; import org.apache.beam.vendor.grpc.v1p54p0.com.google.gson.JsonObject; import org.apache.beam.vendor.grpc.v1p54p0.com.google.gson.JsonParser; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; import org.apache.samza.config.Config; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/samza/src/test/java/org/apache/beam/runners/samza/SamzaPipelineOptionsValidatorTest.java b/runners/samza/src/test/java/org/apache/beam/runners/samza/SamzaPipelineOptionsValidatorTest.java index a5b03a23de36f..2ef56c7ce63c0 100644 --- a/runners/samza/src/test/java/org/apache/beam/runners/samza/SamzaPipelineOptionsValidatorTest.java +++ b/runners/samza/src/test/java/org/apache/beam/runners/samza/SamzaPipelineOptionsValidatorTest.java @@ -24,7 +24,7 @@ import java.util.Collections; import java.util.Map; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; /** Test for {@link SamzaPipelineOptionsValidator}. */ diff --git a/runners/samza/src/test/java/org/apache/beam/runners/samza/adapter/BoundedSourceSystemTest.java b/runners/samza/src/test/java/org/apache/beam/runners/samza/adapter/BoundedSourceSystemTest.java index ca31f8686aa38..4325c29b9b3c2 100644 --- a/runners/samza/src/test/java/org/apache/beam/runners/samza/adapter/BoundedSourceSystemTest.java +++ b/runners/samza/src/test/java/org/apache/beam/runners/samza/adapter/BoundedSourceSystemTest.java @@ -39,7 +39,7 @@ import org.apache.beam.sdk.io.BoundedSource; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.apache.samza.Partition; import org.apache.samza.metrics.MetricsRegistryMap; import org.apache.samza.system.IncomingMessageEnvelope; diff --git a/runners/samza/src/test/java/org/apache/beam/runners/samza/metrics/TestSamzaRunnerWithTransformMetrics.java b/runners/samza/src/test/java/org/apache/beam/runners/samza/metrics/TestSamzaRunnerWithTransformMetrics.java index a605f4fc27b73..e0bcbed1577cd 100644 --- a/runners/samza/src/test/java/org/apache/beam/runners/samza/metrics/TestSamzaRunnerWithTransformMetrics.java +++ b/runners/samza/src/test/java/org/apache/beam/runners/samza/metrics/TestSamzaRunnerWithTransformMetrics.java @@ -45,7 +45,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.samza.context.Context; import org.apache.samza.metrics.Counter; import org.apache.samza.metrics.Gauge; diff --git a/runners/samza/src/test/java/org/apache/beam/runners/samza/metrics/TestSamzaTransformMetricsRegistry.java b/runners/samza/src/test/java/org/apache/beam/runners/samza/metrics/TestSamzaTransformMetricsRegistry.java index f8dacc3aec688..f94543ef809e8 100644 --- a/runners/samza/src/test/java/org/apache/beam/runners/samza/metrics/TestSamzaTransformMetricsRegistry.java +++ b/runners/samza/src/test/java/org/apache/beam/runners/samza/metrics/TestSamzaTransformMetricsRegistry.java @@ -28,7 +28,7 @@ import static org.mockito.Mockito.when; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.samza.context.Context; import org.apache.samza.metrics.Gauge; import org.apache.samza.metrics.Timer; diff --git a/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/AsyncDoFnRunnerTest.java b/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/AsyncDoFnRunnerTest.java index 983275c7b4728..f6f88eb8b2809 100644 --- a/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/AsyncDoFnRunnerTest.java +++ b/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/AsyncDoFnRunnerTest.java @@ -50,7 +50,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; diff --git a/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/FutureCollectorImplTest.java b/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/FutureCollectorImplTest.java index 58d0fbd3f1b62..a808fd2239600 100644 --- a/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/FutureCollectorImplTest.java +++ b/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/FutureCollectorImplTest.java @@ -26,7 +26,7 @@ import java.util.concurrent.CompletionStage; import java.util.stream.Collectors; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Assert; import org.junit.Before; import org.junit.Test; diff --git a/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/GroupByKeyOpTest.java b/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/GroupByKeyOpTest.java new file mode 100644 index 0000000000000..8670d9a46eac0 --- /dev/null +++ b/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/GroupByKeyOpTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.samza.runtime; + +import java.io.Serializable; +import java.util.Arrays; +import org.apache.beam.sdk.coders.KvCoder; +import org.apache.beam.sdk.coders.StringUtf8Coder; +import org.apache.beam.sdk.coders.VarIntCoder; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.testing.TestStream; +import org.apache.beam.sdk.transforms.Combine; +import org.apache.beam.sdk.transforms.Sum; +import org.apache.beam.sdk.transforms.windowing.FixedWindows; +import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.TimestampedValue; +import org.joda.time.Duration; +import org.joda.time.Instant; +import org.junit.Rule; +import org.junit.Test; + +/** Tests for GroupByKeyOp. */ +public class GroupByKeyOpTest implements Serializable { + @Rule + public final transient TestPipeline pipeline = + TestPipeline.fromOptions( + PipelineOptionsFactory.fromArgs("--runner=TestSamzaRunner").create()); + + @Rule + public final transient TestPipeline dropLateDataPipeline = + TestPipeline.fromOptions( + PipelineOptionsFactory.fromArgs("--runner=TestSamzaRunner", "--dropLateData=true") + .create()); + + @Test + public void testDefaultGbk() { + TestStream.Builder testStream = + TestStream.create(VarIntCoder.of()) + .addElements(TimestampedValue.of(1, new Instant(1000))) + .addElements(TimestampedValue.of(2, new Instant(2000))) + .advanceWatermarkTo(new Instant(3000)) + .addElements(TimestampedValue.of(10, new Instant(1000))) + .advanceWatermarkTo(new Instant(10000)); + + PCollection aggregated = + pipeline + .apply(testStream.advanceWatermarkToInfinity()) + .apply( + Window.into(FixedWindows.of(Duration.standardSeconds(3))) + .accumulatingFiredPanes()) + .apply(Combine.globally(Sum.ofIntegers()).withoutDefaults()); + + PAssert.that(aggregated).containsInAnyOrder(Arrays.asList(3, 10)); + + pipeline.run().waitUntilFinish(); + } + + @Test + public void testDropLateDataNonKeyed() { + TestStream.Builder testStream = + TestStream.create(VarIntCoder.of()) + .addElements(TimestampedValue.of(1, new Instant(1000))) + .addElements(TimestampedValue.of(2, new Instant(2000))) + .advanceWatermarkTo(new Instant(3000)) + .addElements(TimestampedValue.of(10, new Instant(1000))) + .advanceWatermarkTo(new Instant(10000)); + + PCollection aggregated = + dropLateDataPipeline + .apply(testStream.advanceWatermarkToInfinity()) + .apply( + Window.into(FixedWindows.of(Duration.standardSeconds(3))) + .accumulatingFiredPanes()) + .apply(Combine.globally(Sum.ofIntegers()).withoutDefaults()); + + PAssert.that(aggregated).containsInAnyOrder(3); + + dropLateDataPipeline.run().waitUntilFinish(); + } + + @Test + public void testDropLateDataKeyed() { + TestStream.Builder> testStream = + TestStream.create(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())) + .addElements(TimestampedValue.of(KV.of("a", 1), new Instant(1000))) + .addElements(TimestampedValue.of(KV.of("b", 2), new Instant(2000))) + .addElements(TimestampedValue.of(KV.of("a", 3), new Instant(2500))) + .advanceWatermarkTo(new Instant(3000)) + .addElements(TimestampedValue.of(KV.of("a", 10), new Instant(1000))) + .advanceWatermarkTo(new Instant(10000)); + + PCollection> aggregated = + dropLateDataPipeline + .apply(testStream.advanceWatermarkToInfinity()) + .apply( + Window.>into(FixedWindows.of(Duration.standardSeconds(3))) + .accumulatingFiredPanes()) + .apply(Sum.integersPerKey()); + + PAssert.that(aggregated).containsInAnyOrder(Arrays.asList(KV.of("a", 4), KV.of("b", 2))); + + dropLateDataPipeline.run().waitUntilFinish(); + } +} diff --git a/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/SamzaStoreStateInternalsTest.java b/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/SamzaStoreStateInternalsTest.java index 97d19a7c73398..0041626001791 100644 --- a/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/SamzaStoreStateInternalsTest.java +++ b/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/SamzaStoreStateInternalsTest.java @@ -59,10 +59,10 @@ import org.apache.beam.sdk.transforms.Sum; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.apache.samza.context.ContainerContext; import org.apache.samza.context.JobContext; import org.apache.samza.metrics.MetricsRegistry; diff --git a/runners/samza/src/test/java/org/apache/beam/runners/samza/translation/ConfigGeneratorTest.java b/runners/samza/src/test/java/org/apache/beam/runners/samza/translation/ConfigGeneratorTest.java index 601831845e83e..9fbc515979b08 100644 --- a/runners/samza/src/test/java/org/apache/beam/runners/samza/translation/ConfigGeneratorTest.java +++ b/runners/samza/src/test/java/org/apache/beam/runners/samza/translation/ConfigGeneratorTest.java @@ -42,8 +42,8 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.apache.samza.config.Config; import org.apache.samza.config.JobConfig; import org.apache.samza.config.JobCoordinatorConfig; diff --git a/runners/samza/src/test/java/org/apache/beam/runners/samza/util/DoFnUtilsTest.java b/runners/samza/src/test/java/org/apache/beam/runners/samza/util/DoFnUtilsTest.java index 1e31616bd86ce..ea74f078c75d1 100644 --- a/runners/samza/src/test/java/org/apache/beam/runners/samza/util/DoFnUtilsTest.java +++ b/runners/samza/src/test/java/org/apache/beam/runners/samza/util/DoFnUtilsTest.java @@ -32,7 +32,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Test; public class DoFnUtilsTest implements Serializable { diff --git a/runners/samza/src/test/java/org/apache/beam/runners/samza/util/FutureUtilsTest.java b/runners/samza/src/test/java/org/apache/beam/runners/samza/util/FutureUtilsTest.java index 357fc6fa23965..2adc9f9e65e92 100644 --- a/runners/samza/src/test/java/org/apache/beam/runners/samza/util/FutureUtilsTest.java +++ b/runners/samza/src/test/java/org/apache/beam/runners/samza/util/FutureUtilsTest.java @@ -22,7 +22,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.CountDownLatch; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Assert; import org.junit.Test; diff --git a/runners/samza/src/test/java/org/apache/beam/runners/samza/util/PipelineJsonRendererTest.java b/runners/samza/src/test/java/org/apache/beam/runners/samza/util/PipelineJsonRendererTest.java index 3860fb2296320..0a4f532808b10 100644 --- a/runners/samza/src/test/java/org/apache/beam/runners/samza/util/PipelineJsonRendererTest.java +++ b/runners/samza/src/test/java/org/apache/beam/runners/samza/util/PipelineJsonRendererTest.java @@ -85,7 +85,10 @@ public void testCompositePipeline() throws IOException { Pipeline p = Pipeline.create(options); - p.apply(Create.timestamped(TimestampedValue.of(KV.of(1, 1), new Instant(1)))) + p.apply( + Create.timestamped( + TimestampedValue.of(KV.of(1, 1), new Instant(1)), + TimestampedValue.of(KV.of(2, 2), new Instant(2)))) .apply(Window.into(FixedWindows.of(Duration.millis(10)))) .apply(Sum.integersPerKey()); diff --git a/runners/samza/src/test/java/org/apache/beam/runners/samza/util/TestHashIdGenerator.java b/runners/samza/src/test/java/org/apache/beam/runners/samza/util/TestHashIdGenerator.java index 881ce7f91c434..cf765e3db221a 100644 --- a/runners/samza/src/test/java/org/apache/beam/runners/samza/util/TestHashIdGenerator.java +++ b/runners/samza/src/test/java/org/apache/beam/runners/samza/util/TestHashIdGenerator.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.transforms.Max; import org.apache.beam.sdk.transforms.Min; import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.junit.Assert; import org.junit.Test; diff --git a/runners/spark/3/build.gradle b/runners/spark/3/build.gradle index 494d367131b44..5380146044d5a 100644 --- a/runners/spark/3/build.gradle +++ b/runners/spark/3/build.gradle @@ -34,8 +34,16 @@ createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner: 'Spark') // Additional supported Spark versions (used in compatibility tests) def sparkVersions = [ + "341": "3.4.1", + "340": "3.4.0", + "332": "3.3.2", + "331": "3.3.1", "330": "3.3.0", - "321": "3.2.1" + "324": "3.2.4", + "323": "3.2.3", + "321": "3.2.1", + "312": "3.1.2", + "311": "3.1.1" ] sparkVersions.each { kv -> diff --git a/runners/spark/3/job-server/build.gradle b/runners/spark/3/job-server/build.gradle index d11a1a8edb13b..68bb8d9a10e11 100644 --- a/runners/spark/3/job-server/build.gradle +++ b/runners/spark/3/job-server/build.gradle @@ -28,13 +28,4 @@ project.ext { } // Load the main build script which contains all build logic. -apply from: "$basePath/spark_job_server.gradle" - - -configurations.runtimeClasspath { - resolutionStrategy { - // Downgrade the Scala version of the job-server to match the Scala version of a Spark 3.1.2 cluster to prevent - // a Scala bug (InvalidClassException when deserializing WrappedArray), see https://github.com/apache/beam/issues/21092 - force "org.scala-lang:scala-library:2.12.10" - } -} \ No newline at end of file +apply from: "$basePath/spark_job_server.gradle" \ No newline at end of file diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunner.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunner.java index 65c8befc699d7..024c854fe814e 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunner.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunner.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.spark.structuredstreaming; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -42,7 +42,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.options.PipelineOptionsValidator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.spark.SparkEnv$; import org.apache.spark.metrics.MetricsSystem; import org.apache.spark.sql.SparkSession; diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrar.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrar.java index fb759da960ce4..a1dc3ad3a9bef 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrar.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrar.java @@ -22,7 +22,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; import org.apache.beam.sdk.runners.PipelineRunnerRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * Contains the {@link PipelineRunnerRegistrar} and {@link PipelineOptionsRegistrar} for the {@link diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/io/BoundedDatasetFactory.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/io/BoundedDatasetFactory.java index 4cee1c7e897b8..c290a40287a57 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/io/BoundedDatasetFactory.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/io/BoundedDatasetFactory.java @@ -20,7 +20,7 @@ import static java.util.stream.Collectors.toList; import static org.apache.beam.runners.spark.structuredstreaming.translation.utils.ScalaInterop.emptyList; import static org.apache.beam.sdk.util.WindowedValue.timestampedValueInGlobalWindow; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static scala.collection.JavaConverters.asScalaIterator; import java.io.Closeable; @@ -31,13 +31,14 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.IntSupplier; import java.util.function.Supplier; +import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.apache.beam.sdk.io.BoundedSource; import org.apache.beam.sdk.io.BoundedSource.BoundedReader; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.AbstractIterator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.AbstractIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.apache.spark.InterruptibleIterator; import org.apache.spark.Partition; import org.apache.spark.SparkContext; @@ -288,7 +289,7 @@ public void close() throws IOException { } @Override - protected WindowedValue computeNext() { + protected @CheckForNull WindowedValue computeNext() { try { if (started ? reader.advance() : start()) { return timestampedValueInGlobalWindow(reader.getCurrent(), reader.getCurrentTimestamp()); diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/MetricsAccumulator.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/MetricsAccumulator.java index 6edddff5831c6..63407b9f14d89 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/MetricsAccumulator.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/MetricsAccumulator.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.spark.structuredstreaming.metrics; import org.apache.beam.runners.core.metrics.MetricsContainerStepMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.apache.spark.sql.SparkSession; import org.apache.spark.util.AccumulatorV2; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetric.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetric.java index 1754ac4d1678e..19ba92956e70f 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetric.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetric.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.spark.structuredstreaming.metrics; import static org.apache.beam.runners.core.metrics.MetricsContainerStepMap.asAttemptedOnlyMetricResults; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates.not; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates.not; import com.codahale.metrics.Gauge; import com.codahale.metrics.Metric; @@ -36,9 +36,9 @@ import org.apache.beam.sdk.metrics.MetricQueryResults; import org.apache.beam.sdk.metrics.MetricResult; import org.apache.beam.sdk.metrics.MetricResults; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Streams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Streams; /** * An adapter between the {@link MetricsContainerStepMap} and the Dropwizard {@link Metric} diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/WithMetricsSupport.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/WithMetricsSupport.java index c9233a128c165..f632f7a6aa1af 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/WithMetricsSupport.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/WithMetricsSupport.java @@ -26,8 +26,8 @@ import com.codahale.metrics.Timer; import java.util.Map; import java.util.SortedMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSortedMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSortedMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; /** * A {@link MetricRegistry} decorator-like that supports {@link BeamMetricSet}s as {@link Gauge diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/EvaluationContext.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/EvaluationContext.java index a7a17ebb677ac..da89ece0a2f73 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/EvaluationContext.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/EvaluationContext.java @@ -21,7 +21,7 @@ import javax.annotation.Nullable; import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.execution.ExplainMode; diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkSessionFactory.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkSessionFactory.java index 34d42c76031f0..6e2d21bfb9488 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkSessionFactory.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkSessionFactory.java @@ -79,8 +79,8 @@ import org.apache.beam.sdk.values.PCollectionViews; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Collections2; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Collections2; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.spark.SparkConf; import org.apache.spark.serializer.KryoRegistrator; import org.apache.spark.serializer.KryoSerializer; diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkTransformOverrides.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkTransformOverrides.java index 996f60cb74788..b12816b9b19bf 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkTransformOverrides.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkTransformOverrides.java @@ -25,7 +25,7 @@ import org.apache.beam.runners.core.construction.UnsupportedOverrideFactory; import org.apache.beam.sdk.runners.PTransformOverride; import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** {@link PTransform} overrides for Spark runner. */ @SuppressWarnings({ diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java index 8f362bb3da6ff..58a596d1c2670 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java @@ -18,8 +18,8 @@ package org.apache.beam.runners.spark.structuredstreaming.translation; import static org.apache.beam.runners.spark.structuredstreaming.translation.helpers.EncoderHelpers.windowedValueEncoder; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.getOnlyElement; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.getOnlyElement; import java.io.IOException; import java.util.List; diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/Aggregators.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/Aggregators.java index 250242bbeadca..24d0f2eced69f 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/Aggregators.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/Aggregators.java @@ -22,8 +22,8 @@ import static org.apache.beam.runners.spark.structuredstreaming.translation.helpers.EncoderHelpers.mapEncoder; import static org.apache.beam.runners.spark.structuredstreaming.translation.helpers.EncoderHelpers.mutablePairEncoder; import static org.apache.beam.sdk.transforms.windowing.PaneInfo.NO_FIRING; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators.peekingIterator; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators.peekingIterator; import java.util.Collection; import java.util.HashMap; @@ -45,11 +45,11 @@ import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Collections2; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.PeekingIterator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Collections2; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.PeekingIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.apache.spark.sql.Encoder; import org.apache.spark.sql.expressions.Aggregator; import org.apache.spark.util.MutablePair; diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnPartitionIteratorFactory.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnPartitionIteratorFactory.java index 33994e9a1efe0..907b847c7e8a5 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnPartitionIteratorFactory.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnPartitionIteratorFactory.java @@ -25,13 +25,14 @@ import java.util.Deque; import java.util.Map; import java.util.function.Supplier; +import javax.annotation.CheckForNull; import org.apache.beam.runners.core.DoFnRunners; import org.apache.beam.runners.spark.structuredstreaming.metrics.MetricsAccumulator; import org.apache.beam.runners.spark.structuredstreaming.translation.batch.DoFnRunnerFactory.DoFnRunnerWithTeardown; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.AbstractIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.AbstractIterator; import org.checkerframework.checker.nullness.qual.NonNull; import scala.Function1; import scala.Tuple2; @@ -169,7 +170,7 @@ private DoFnPartitionIt(Iterator> partitionIt) { } @Override - protected OutT computeNext() { + protected @CheckForNull OutT computeNext() { try { while (true) { if (!buffer.isEmpty()) { diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnRunnerFactory.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnRunnerFactory.java index a065df946357c..48caa938994ff 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnRunnerFactory.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnRunnerFactory.java @@ -43,9 +43,9 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.joda.time.Instant; import scala.Serializable; diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenTranslatorBatch.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenTranslatorBatch.java index 397a3f4bae118..fd02a3ef52425 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenTranslatorBatch.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenTranslatorBatch.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoder; diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java index 5770d93094c96..9e75e22a30870 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java @@ -36,7 +36,7 @@ import static org.apache.beam.runners.spark.structuredstreaming.translation.utils.ScalaInterop.listOf; import static org.apache.beam.runners.spark.structuredstreaming.translation.utils.ScalaInterop.seqOf; import static org.apache.beam.sdk.transforms.windowing.PaneInfo.NO_FIRING; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.apache.spark.sql.functions.col; import static org.apache.spark.sql.functions.collect_list; import static org.apache.spark.sql.functions.explode; @@ -167,7 +167,7 @@ && eligibleForGroupByWindow(windowing, false) result = input .select(explode(col("windows")).as("window"), col("value"), col("timestamp")) - .groupBy(col("value.key"), col("window")) + .groupBy(col("value.key").as("key"), col("window")) .agg(collect_list(col("value.value")).as("values"), timestampAggregator(tsCombiner)) .select( inSingleWindow( diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ImpulseTranslatorBatch.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ImpulseTranslatorBatch.java index 1219173b4f23b..28f929679dd63 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ImpulseTranslatorBatch.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ImpulseTranslatorBatch.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.spark.sql.Dataset; class ImpulseTranslatorBatch extends TransformTranslator, Impulse> { diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java index 570fa0c01ab43..712be0f4f8da5 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java @@ -20,7 +20,7 @@ import static org.apache.beam.runners.spark.structuredstreaming.translation.batch.DoFnRunnerFactory.simple; import static org.apache.beam.runners.spark.structuredstreaming.translation.helpers.EncoderHelpers.oneOfEncoder; import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.apache.spark.sql.functions.col; import java.io.IOException; @@ -51,7 +51,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.apache.spark.broadcast.Broadcast; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoder; diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java index f3442ecd7196f..140f2f88d8cd3 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.sql.Dataset; import org.checkerframework.checker.nullness.qual.NonNull; diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/CachedSideInputReader.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/CachedSideInputReader.java index b489be4772d3e..1db08d935fcab 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/CachedSideInputReader.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/CachedSideInputReader.java @@ -30,10 +30,10 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.GlobalWindows; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheStats; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheStats; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SideInputValues.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SideInputValues.java index 8ff925561b865..57d97cff16259 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SideInputValues.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SideInputValues.java @@ -20,7 +20,7 @@ import static org.apache.beam.runners.spark.structuredstreaming.translation.utils.ScalaInterop.fun1; import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; import static org.apache.beam.sdk.util.WindowedValue.getFullCoder; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.apache.spark.sql.Encoders.BINARY; import com.esotericsoftware.kryo.Kryo; @@ -44,7 +44,7 @@ import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.apache.spark.sql.Dataset; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.NonNull; diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SparkSideInputReader.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SparkSideInputReader.java index bf3c2cd66796a..50c0b8a50b14b 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SparkSideInputReader.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SparkSideInputReader.java @@ -20,7 +20,7 @@ import static org.apache.beam.sdk.transforms.Materializations.ITERABLE_MATERIALIZATION_URN; import static org.apache.beam.sdk.transforms.Materializations.MULTIMAP_MATERIALIZATION_URN; import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.Serializable; import java.util.Collections; @@ -38,7 +38,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.apache.spark.broadcast.Broadcast; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderFactory.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderFactory.java index e70cc7253f8da..ceafc1642baa3 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderFactory.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderFactory.java @@ -69,6 +69,10 @@ private static Expression invoke( // Spark 3.1.x return STATIC_INVOKE_CONSTRUCTOR.newInstance( cls, type, fun, seqOf(args), propagateNull, true); + case 7: + // Spark 3.2.0 + return STATIC_INVOKE_CONSTRUCTOR.newInstance( + cls, type, fun, seqOf(args), emptyList(), propagateNull, true); case 8: // Spark 3.2.x, 3.3.x return STATIC_INVOKE_CONSTRUCTOR.newInstance( @@ -89,8 +93,14 @@ static Expression invoke( // created reflectively. This is fine as it's just needed once to create the query plan. switch (STATIC_INVOKE_CONSTRUCTOR.getParameterCount()) { case 6: + // Spark 3.1.x return INVOKE_CONSTRUCTOR.newInstance(obj, fun, type, seqOf(args), false, nullable); + case 7: + // Spark 3.2.0 + return INVOKE_CONSTRUCTOR.newInstance( + obj, fun, type, seqOf(args), emptyList(), false, nullable); case 8: + // Spark 3.2.x, 3.3.x return INVOKE_CONSTRUCTOR.newInstance( obj, fun, type, seqOf(args), emptyList(), false, nullable, true); default: diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpers.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpers.java index bc6eec02054ec..7a19c1c0e0277 100644 --- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpers.java +++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpers.java @@ -47,10 +47,10 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo.PaneInfoCoder; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.apache.spark.sql.Encoder; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.catalyst.SerializerBuildHelper; diff --git a/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrarTest.java b/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrarTest.java index 30d8297809b97..bc7b561eea2d7 100644 --- a/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrarTest.java +++ b/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrarTest.java @@ -23,8 +23,8 @@ import java.util.ServiceLoader; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; import org.apache.beam.sdk.runners.PipelineRunnerRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/InMemoryMetrics.java b/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/InMemoryMetrics.java index 7f2eaa10e8092..69f6abee1f676 100644 --- a/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/InMemoryMetrics.java +++ b/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/InMemoryMetrics.java @@ -23,7 +23,7 @@ import java.util.Collection; import java.util.Properties; import org.apache.beam.runners.spark.structuredstreaming.metrics.WithMetricsSupport; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.spark.metrics.sink.Sink; /** An in-memory {@link Sink} implementation for tests. */ diff --git a/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/SparkMetricsSinkTest.java b/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/SparkMetricsSinkTest.java index 2f02656dc37eb..603221de078ba 100644 --- a/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/SparkMetricsSinkTest.java +++ b/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/SparkMetricsSinkTest.java @@ -30,8 +30,8 @@ import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; diff --git a/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/AggregatorsTest.java b/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/AggregatorsTest.java index b5b07db0e38bb..d642a6e9866cc 100644 --- a/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/AggregatorsTest.java +++ b/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/AggregatorsTest.java @@ -42,7 +42,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Streams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Streams; import org.apache.spark.sql.Encoder; import org.apache.spark.sql.expressions.Aggregator; import org.apache.spark.util.MutablePair; diff --git a/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombineGroupedValuesTest.java b/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombineGroupedValuesTest.java index 774186c1821c4..5b23a6ac9ea5d 100644 --- a/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombineGroupedValuesTest.java +++ b/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombineGroupedValuesTest.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.transforms.Sum; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; diff --git a/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTest.java b/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTest.java index 5a2335a154eb1..41c032cd85be1 100644 --- a/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTest.java +++ b/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTest.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.ClassRule; diff --git a/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTest.java b/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTest.java index b1aa300fc27a3..bf1799bb13b20 100644 --- a/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTest.java +++ b/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTest.java @@ -44,8 +44,8 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.ClassRule; diff --git a/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SideInputValuesTest.java b/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SideInputValuesTest.java index 7c214e5640f56..3b828ee7d1810 100644 --- a/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SideInputValuesTest.java +++ b/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SideInputValuesTest.java @@ -37,7 +37,7 @@ import org.apache.beam.sdk.transforms.windowing.IntervalWindow; import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.spark.serializer.KryoSerializer; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoder; diff --git a/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpersTest.java b/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpersTest.java index ab6e3083c54a7..ecb79d97b3e2c 100644 --- a/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpersTest.java +++ b/runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpersTest.java @@ -28,7 +28,7 @@ import static org.apache.beam.runners.spark.structuredstreaming.translation.helpers.EncoderHelpers.oneOfEncoder; import static org.apache.beam.runners.spark.structuredstreaming.translation.helpers.EncoderHelpers.windowedValueEncoder; import static org.apache.beam.runners.spark.structuredstreaming.translation.utils.ScalaInterop.tuple; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates.notNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates.notNull; import static org.apache.spark.sql.types.DataTypes.IntegerType; import static org.apache.spark.sql.types.DataTypes.StringType; import static org.apache.spark.sql.types.DataTypes.createStructField; @@ -67,9 +67,9 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoder; import org.apache.spark.sql.catalyst.InternalRow; diff --git a/runners/spark/OWNERS b/runners/spark/OWNERS deleted file mode 100644 index 8b0b05531cd10..0000000000000 --- a/runners/spark/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - jbonofre diff --git a/runners/spark/job-server/container/spark_job_server_container.gradle b/runners/spark/job-server/container/spark_job_server_container.gradle index 9b8ba3f31579b..3f3cd74bc10fe 100644 --- a/runners/spark/job-server/container/spark_job_server_container.gradle +++ b/runners/spark/job-server/container/spark_job_server_container.gradle @@ -51,6 +51,9 @@ task copyDockerfileDependencies(type: Copy) { into "build" } +def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] +def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers") + docker { name containerImageName(name: project.docker_image_default_repo_prefix + spark_job_server_image, root: project.rootProject.hasProperty(["docker-repository-root"]) ? @@ -61,6 +64,10 @@ docker { // tags used by dockerTag task tags containerImageTags() files "./build/" + buildx useBuildx + platform(*project.containerPlatforms()) + load useBuildx && !pushContainers + push pushContainers } // Ensure that we build the required resources and copy and file dependencies from related projects diff --git a/runners/spark/spark_runner.gradle b/runners/spark/spark_runner.gradle index 0e3028d3843c9..d0dbe453ddfb0 100644 --- a/runners/spark/spark_runner.gradle +++ b/runners/spark/spark_runner.gradle @@ -171,7 +171,7 @@ dependencies { implementation project(path: ":model:job-management", configuration: "shadow") implementation project(":sdks:java:fn-execution") implementation library.java.vendored_grpc_1_54_0 - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre spark.components.each { component -> provided "$component:$spark_version" } @@ -181,7 +181,7 @@ dependencies { runtimeOnly library.java.jackson_module_scala_2_12 // Force paranamer 2.8 to avoid issues when using Scala 2.12 runtimeOnly "com.thoughtworks.paranamer:paranamer:2.8" - provided library.java.hadoop_common + provided "org.apache.hadoop:hadoop-client-api:3.3.1" provided library.java.commons_io provided library.java.hamcrest provided "com.esotericsoftware:kryo-shaded:4.0.2" @@ -364,7 +364,7 @@ tasks.register("validatesStructuredStreamingRunnerBatch", Test) { // Register various other classes used in tests systemProperty 'spark.kryo.classesToRegister', 'org.apache.beam.sdk.transforms.ViewTest$NonDeterministicStringCoder,' + - 'org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.RegularImmutableList' + 'org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.RegularImmutableList' jvmArgs += sparkTestJvmArgs() jvmArgs '-Xmx7g' // Increase memory heap in order to avoid OOM errors diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkJobInvoker.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkJobInvoker.java index ee0777b425041..2d7767ee3bba1 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkJobInvoker.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkJobInvoker.java @@ -27,8 +27,8 @@ import org.apache.beam.runners.jobsubmission.PortablePipelineRunner; import org.apache.beam.sdk.options.PortablePipelineOptions; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Struct; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkNativePipelineVisitor.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkNativePipelineVisitor.java index 86fef0d81412b..143eab8be85f0 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkNativePipelineVisitor.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkNativePipelineVisitor.java @@ -31,8 +31,8 @@ import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PInput; import org.apache.beam.sdk.values.POutput; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; /** * Pipeline visitor for translating a Beam pipeline into equivalent Spark operations. Used for @@ -183,7 +183,7 @@ private String replaceFnString( String doFnName; Class enclosingClass = fnClass.getEnclosingClass(); if (enclosingClass != null && enclosingClass.equals(MapElements.class)) { - Field parent = fnClass.getDeclaredField("this$0"); + Field parent = fnClass.getSuperclass().getDeclaredField("outer"); parent.setAccessible(true); Field fnField = enclosingClass.getDeclaredField(fnFieldName); fnField.setAccessible(true); diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineRunner.java index 2a6ea379bd3f7..159cb32a8c5e4 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineRunner.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineRunner.java @@ -53,8 +53,8 @@ import org.apache.beam.sdk.metrics.MetricsOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Struct; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.streaming.api.java.JavaStreamingContext; import org.apache.spark.streaming.api.java.JavaStreamingListener; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java index d10209ba14ee8..f5720365b260d 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java @@ -60,7 +60,7 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.spark.SparkEnv$; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.metrics.MetricsSystem; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerRegistrar.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerRegistrar.java index cfc07377b9ceb..30f9c29ae6cd2 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerRegistrar.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunnerRegistrar.java @@ -22,7 +22,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; import org.apache.beam.sdk.runners.PipelineRunnerRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * Contains the {@link PipelineRunnerRegistrar} and {@link PipelineOptionsRegistrar} for the {@link diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkTransformOverrides.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkTransformOverrides.java index 748e754364b72..95a0080ce0bc4 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkTransformOverrides.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkTransformOverrides.java @@ -25,7 +25,7 @@ import org.apache.beam.runners.core.construction.UnsupportedOverrideFactory; import org.apache.beam.sdk.runners.PTransformOverride; import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** {@link PTransform} overrides for Spark runner. */ @SuppressWarnings({ diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java index 8288dd49c3985..e46b6c5f5e0be 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.spark; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.isOneOf; @@ -33,7 +33,7 @@ import org.apache.beam.sdk.PipelineRunner; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsValidator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.apache.commons.io.FileUtils; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/CoderHelpers.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/CoderHelpers.java index 52c29b8abf7ad..2725a57f7dc06 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/CoderHelpers.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/CoderHelpers.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.spark.coders; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.ByteArrayInputStream; @@ -201,7 +201,7 @@ public static PairFunction, ByteArray, byte[]> toByteFunctio */ public static class FromByteFunction implements PairFunction, K, V>, - org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function< + org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function< Tuple2, Tuple2> { private final Coder keyCoder; private final Coder valueCoder; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/SparkRunnerKryoRegistrator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/SparkRunnerKryoRegistrator.java index 440cd9b6858d0..619d2d16173d5 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/SparkRunnerKryoRegistrator.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/SparkRunnerKryoRegistrator.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashBasedTable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBasedTable; import org.apache.spark.serializer.KryoRegistrator; import scala.collection.mutable.WrappedArray; @@ -68,7 +68,7 @@ public void registerClasses(Kryo kryo) { Class.forName("org.apache.beam.sdk.util.WindowedValue$TimestampedValueInGlobalWindow")); kryo.register( Class.forName( - "org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashBasedTable$Factory")); + "org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBasedTable$Factory")); } catch (ClassNotFoundException e) { throw new IllegalStateException("Unable to register classes with kryo.", e); } diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java index 2ebed0acf73e4..2609f75d14715 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.spark.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.ArrayDeque; import java.util.Arrays; @@ -33,7 +33,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/MicrobatchSource.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/MicrobatchSource.java index 9da520f6b9c04..8d3231fe31c81 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/MicrobatchSource.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/MicrobatchSource.java @@ -30,12 +30,12 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.BackOff; import org.apache.beam.sdk.util.FluentBackoff; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.RemovalListener; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.RemovalNotification; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.RemovalListener; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.RemovalNotification; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceDStream.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceDStream.java index 4b20f4e871388..e5e3446690706 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceDStream.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceDStream.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.spark.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import org.apache.beam.runners.core.construction.SerializablePipelineOptions; import org.apache.beam.runners.spark.SparkPipelineOptions; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java index 6f5febd2c0928..c846ac20a626e 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SourceRDD.java @@ -17,8 +17,8 @@ */ package org.apache.beam.runners.spark.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.Closeable; import java.io.IOException; @@ -36,7 +36,7 @@ import org.apache.beam.sdk.metrics.MetricsContainer; import org.apache.beam.sdk.metrics.MetricsEnvironment; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.apache.spark.Dependency; import org.apache.spark.HashPartitioner; import org.apache.spark.InterruptibleIterator; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java index 398e2c03fe8be..34ef3331ae49a 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java @@ -42,7 +42,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext$; import org.apache.spark.api.java.function.FlatMapFunction; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulator.java index c0a43f3569394..dbdfe11a585c4 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulator.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulator.java @@ -22,8 +22,8 @@ import org.apache.beam.runners.spark.SparkPipelineOptions; import org.apache.beam.runners.spark.translation.streaming.Checkpoint; import org.apache.beam.runners.spark.translation.streaming.Checkpoint.CheckpointDir; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.spark.api.java.JavaSparkContext; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetric.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetric.java index 1eb8349a513fd..c210d06c9401f 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetric.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkBeamMetric.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.spark.metrics; import static org.apache.beam.runners.core.metrics.MetricsContainerStepMap.asAttemptedOnlyMetricResults; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates.not; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates.not; import com.codahale.metrics.Gauge; import com.codahale.metrics.Metric; @@ -36,9 +36,9 @@ import org.apache.beam.sdk.metrics.MetricQueryResults; import org.apache.beam.sdk.metrics.MetricResult; import org.apache.beam.sdk.metrics.MetricResults; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Streams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Streams; /** * An adapter between the {@link MetricsContainerStepMap} and the Dropwizard {@link Metric} diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/WithMetricsSupport.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/WithMetricsSupport.java index a0fc714100a4e..ae557c0089b37 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/WithMetricsSupport.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/WithMetricsSupport.java @@ -26,8 +26,8 @@ import com.codahale.metrics.Timer; import java.util.Map; import java.util.SortedMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSortedMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSortedMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; /** * A {@link MetricRegistry} decorator-like that supports {@link AggregatorMetric} and {@link diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java index 81bd254678e2f..930dd4d15e256 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java @@ -55,12 +55,12 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicate; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.AbstractIterator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Table; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicate; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.AbstractIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Table; import org.apache.spark.api.java.JavaSparkContext$; import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.streaming.Duration; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java index 34e0ba29a26f2..3ad955e78aa74 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java @@ -43,8 +43,8 @@ import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext; import org.apache.beam.sdk.transforms.windowing.TimestampCombiner; import org.apache.beam.sdk.util.CombineFnUtil; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashBasedTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Table; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBasedTable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Table; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java index 597c18b4cacee..de9820e1255c4 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.spark.stateful; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Collection; import java.util.Collections; @@ -31,8 +31,8 @@ import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.SparkWatermarks; import org.apache.beam.sdk.state.TimeDomain; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java index ed0b0a4e2ea6b..b72481128f531 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java @@ -38,10 +38,10 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Stopwatch; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Stopwatch; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.spark.streaming.State; import org.apache.spark.streaming.StateSpec; import org.joda.time.Instant; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/BoundedDataset.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/BoundedDataset.java index 7ac36a5726773..5cebc513e8b9b 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/BoundedDataset.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/BoundedDataset.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDDLike; import org.apache.spark.api.java.JavaSparkContext; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java index 210b2db44bb36..d6895216c2f87 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.spark.translation; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.HashMap; import java.util.LinkedHashMap; @@ -40,7 +40,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.streaming.api.java.JavaStreamingContext; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java index 2b06d87132f5b..62c5e25794276 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.spark.Partitioner; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupNonMergingWindowsFunctions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupNonMergingWindowsFunctions.java index 0a28248e45fa2..0a8e7d8a159b5 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupNonMergingWindowsFunctions.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupNonMergingWindowsFunctions.java @@ -32,11 +32,11 @@ import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.AbstractIterator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.PeekingIterator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Bytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.AbstractIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.PeekingIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Bytes; import org.apache.spark.HashPartitioner; import org.apache.spark.Partitioner; import org.apache.spark.api.java.JavaPairRDD; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java index 713c866a89e73..df36d24531a64 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/MultiDoFnFunction.java @@ -45,7 +45,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; import org.apache.spark.api.java.function.PairFlatMapFunction; import org.apache.spark.util.AccumulatorV2; import scala.Tuple2; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAssignWindowFn.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAssignWindowFn.java index 5a6ce6f5a80e8..fd00cadf3d40a 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAssignWindowFn.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkAssignWindowFn.java @@ -23,7 +23,7 @@ import org.apache.beam.sdk.transforms.windowing.Window.Assign; import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.spark.api.java.function.Function; import org.joda.time.Instant; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkBatchPortablePipelineTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkBatchPortablePipelineTranslator.java index 269dea95f3616..3ef215834dec2 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkBatchPortablePipelineTranslator.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkBatchPortablePipelineTranslator.java @@ -59,8 +59,8 @@ import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.spark.HashPartitioner; import org.apache.spark.Partitioner; import org.apache.spark.api.java.JavaPairRDD; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkCombineFn.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkCombineFn.java index cb46e1ddea54f..ddf4b12bae130 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkCombineFn.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkCombineFn.java @@ -56,10 +56,10 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.spark.api.java.function.Function; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java index 9f9465ccde8fe..88db3b3f158b6 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.spark.translation; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import javax.annotation.Nullable; import org.apache.beam.runners.spark.SparkContextOptions; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkExecutableStageFunction.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkExecutableStageFunction.java index 9cb07366ea761..a3d3186ad4797 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkExecutableStageFunction.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkExecutableStageFunction.java @@ -63,7 +63,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.broadcast.Broadcast; import org.joda.time.Instant; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkInputDataProcessor.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkInputDataProcessor.java index ac0296ba8cd9f..0af480a2ff02a 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkInputDataProcessor.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkInputDataProcessor.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.spark.translation; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.ArrayDeque; import java.util.Iterator; @@ -27,6 +27,7 @@ import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import javax.annotation.CheckForNull; import org.apache.beam.runners.core.DoFnRunners.OutputManager; import org.apache.beam.runners.core.StateNamespace; import org.apache.beam.runners.core.StateNamespaces; @@ -35,9 +36,9 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.AbstractIterator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.AbstractIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.Nullable; import scala.Tuple2; @@ -141,7 +142,7 @@ private class UnboundedInOutIterator } @Override - protected Tuple2, WindowedValue> computeNext() { + protected @CheckForNull Tuple2, WindowedValue> computeNext() { try { // Process each element from the (input) iterator, which produces, zero, one or more // output elements (of type V) in the output iterator. Note that the output @@ -304,7 +305,7 @@ private class BoundedInOutIterator } @Override - protected Tuple2, WindowedValue> computeNext() { + protected @CheckForNull Tuple2, WindowedValue> computeNext() { if (outputProducerTask == null) { outputProducerTask = startOutputProducerTask(); diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkStreamingPortablePipelineTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkStreamingPortablePipelineTranslator.java index 657ad0c42b640..dea82d8e3da0c 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkStreamingPortablePipelineTranslator.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkStreamingPortablePipelineTranslator.java @@ -57,8 +57,8 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.broadcast.Broadcast; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java index 7fd0f9155d3ea..878447e7f8d5c 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.spark.translation; import static org.apache.beam.runners.spark.translation.TranslationUtils.canAvoidRddSerialization; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Arrays; import java.util.Collection; @@ -71,11 +71,11 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.AbstractIterator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.AbstractIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.spark.HashPartitioner; import org.apache.spark.Partitioner; import org.apache.spark.api.java.JavaPairRDD; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java index 099d1c2fcfa36..23af6f71b9384 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java @@ -44,9 +44,9 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.Function; @@ -150,7 +150,7 @@ public static PairFlatMapFunction>, K, V> toPairFlatMap /** A pair to {@link KV} function . */ static class FromPairFunction implements Function, KV>, - org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function< + org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function< Tuple2, KV> { @Override public KV call(Tuple2 t2) { @@ -184,7 +184,7 @@ static FlatMapFunction>, KV> fromPairFlatMapF /** Extract window from a {@link KV} with {@link WindowedValue} value. */ static class ToKVByWindowInValueFunction implements Function>, WindowedValue>>, - org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function< + org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function< KV>, WindowedValue>> { @Override diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/ValueAndCoderLazySerializable.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/ValueAndCoderLazySerializable.java index a2b2186679617..530de84b6acd9 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/ValueAndCoderLazySerializable.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/ValueAndCoderLazySerializable.java @@ -28,8 +28,8 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.util.VarInt; import org.apache.beam.sdk.util.common.ElementByteSizeObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; /** * A holder object that lets you serialize an element with a Coder with minimal wasted space. diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/SparkRunnerStreamingContextFactory.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/SparkRunnerStreamingContextFactory.java index 4c4d575a324c0..c50c59dc33f27 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/SparkRunnerStreamingContextFactory.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/SparkRunnerStreamingContextFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.spark.translation.streaming; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import org.apache.beam.runners.spark.SparkPipelineOptions; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java index 2abeeb7c33ff1..266b67798a226 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java @@ -18,8 +18,8 @@ package org.apache.beam.runners.spark.translation.streaming; import static org.apache.beam.runners.spark.translation.TranslationUtils.rejectStateAndTimers; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.service.AutoService; import java.util.ArrayList; @@ -84,7 +84,7 @@ import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.JavaSparkContext$; @@ -636,7 +636,7 @@ private static class SparkConsoleIOWriteUnboundedPayloadTranslator ConsoleIO.Write.Unbound> { @Override - public String getUrn(ConsoleIO.Write.Unbound transform) { + public String getUrn() { return ConsoleIO.Write.Unbound.TRANSFORM_URN; } } @@ -645,7 +645,7 @@ private static class SparkCreateStreamPayloadTranslator extends PTransformTranslation.TransformPayloadTranslator.NotSerializable> { @Override - public String getUrn(CreateStream transform) { + public String getUrn() { return CreateStream.TRANSFORM_URN; } } diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/TestDStream.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/TestDStream.java index ec4937d439c5a..e0bda8ba9491f 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/TestDStream.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/TestDStream.java @@ -34,7 +34,7 @@ import org.apache.beam.sdk.util.Preconditions; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.JavaSparkContext$; import org.apache.spark.rdd.RDD; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/WatermarkSyncedDStream.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/WatermarkSyncedDStream.java index 24b5a0f867fa2..fef2d2ec675ed 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/WatermarkSyncedDStream.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/WatermarkSyncedDStream.java @@ -17,14 +17,14 @@ */ package org.apache.beam.runners.spark.translation.streaming; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Queue; import java.util.concurrent.TimeUnit; import org.apache.beam.runners.spark.util.GlobalWatermarkHolder; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Stopwatch; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Stopwatch; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext$; import org.apache.spark.rdd.RDD; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/ByteArray.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/ByteArray.java index 10220dfac6889..a7ff4b8fcd285 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/ByteArray.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/ByteArray.java @@ -19,7 +19,7 @@ import java.io.Serializable; import java.util.Arrays; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.UnsignedBytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedBytes; import org.checkerframework.checker.nullness.qual.Nullable; /** Serializable byte array. */ diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/CachedSideInputReader.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/CachedSideInputReader.java index 14fabcce03211..a18aa2c25d722 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/CachedSideInputReader.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/CachedSideInputReader.java @@ -23,7 +23,7 @@ import org.apache.beam.runners.spark.util.SideInputStorage.Value; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; import org.apache.spark.util.SizeEstimator; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java index 0257c75f7a4e7..f589e36bd7893 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.spark.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.Serializable; import java.util.HashMap; @@ -28,11 +28,11 @@ import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LoadingCache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.apache.spark.SparkEnv; import org.apache.spark.broadcast.Broadcast; import org.apache.spark.storage.BlockId; diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SideInputStorage.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SideInputStorage.java index f4070f14bc10c..9753197f6fa1f 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SideInputStorage.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SideInputStorage.java @@ -21,8 +21,8 @@ import java.util.concurrent.TimeUnit; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SparkSideInputReader.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SparkSideInputReader.java index 459e72dde96da..ce32be569ae49 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SparkSideInputReader.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/SparkSideInputReader.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.spark.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.Map; import java.util.stream.Collectors; @@ -36,7 +36,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; /** A {@link SideInputReader} for the SparkRunner. */ diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/ProvidedSparkContextTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/ProvidedSparkContextTest.java index 0ef6bb0e078c9..01acc25dd32bf 100644 --- a/runners/spark/src/test/java/org/apache/beam/runners/spark/ProvidedSparkContextTest.java +++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/ProvidedSparkContextTest.java @@ -29,8 +29,8 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.junit.ClassRule; import org.junit.FixMethodOrder; import org.junit.Test; diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerDebuggerTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerDebuggerTest.java index 157b3cb946e97..6e4acc6fb9306 100644 --- a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerDebuggerTest.java +++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerDebuggerTest.java @@ -17,13 +17,17 @@ */ package org.apache.beam.runners.spark; +import static org.apache.beam.sdk.transforms.Contextful.fn; +import static org.apache.beam.sdk.transforms.Requirements.requiresSideInputs; import static org.hamcrest.MatcherAssert.assertThat; +import java.util.Arrays; import java.util.Collections; import org.apache.beam.runners.spark.examples.WordCount; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.coders.KvCoder; import org.apache.beam.sdk.coders.StringUtf8Coder; +import org.apache.beam.sdk.coders.VarIntCoder; import org.apache.beam.sdk.io.TextIO; import org.apache.beam.sdk.io.kafka.KafkaIO; import org.apache.beam.sdk.options.PipelineOptions; @@ -39,12 +43,15 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.SimpleFunction; import org.apache.beam.sdk.transforms.Sum; +import org.apache.beam.sdk.transforms.View; import org.apache.beam.sdk.transforms.WithKeys; import org.apache.beam.sdk.transforms.windowing.FixedWindows; import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; +import org.apache.beam.sdk.values.PCollectionView; +import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; import org.hamcrest.Matchers; @@ -83,7 +90,8 @@ public void debugBatchPipeline() { .apply(TextIO.write().to("!!PLACEHOLDER-OUTPUT-DIR!!").withNumShards(3).withSuffix(".txt")); final String expectedPipeline = - "sparkContext.()\n" + "sparkContext.()\n" + + "_.mapPartitions(new org.apache.beam.sdk.transforms.FlatMapElements$3())\n" + "_.mapPartitions(" + "new org.apache.beam.runners.spark.examples.WordCount$ExtractWordsFn())\n" + "_.mapPartitions(new org.apache.beam.sdk.transforms.Count$PerElement$1())\n" @@ -159,6 +167,40 @@ public void debugStreamingPipeline() { Matchers.equalTo(expectedPipeline)); } + @Test + public void debugBatchPipelineWithContextfulTransform() { + PipelineOptions options = contextRule.configure(PipelineOptionsFactory.create()); + options.setRunner(SparkRunnerDebugger.class); + Pipeline pipeline = Pipeline.create(options); + + final PCollectionView view = + pipeline.apply("Dummy", Create.of(0)).apply(View.asSingleton()); + + pipeline + .apply(Create.of(Arrays.asList(0))) + .setCoder(VarIntCoder.of()) + .apply( + MapElements.into(new TypeDescriptor() {}) + .via(fn((element, c) -> element, requiresSideInputs(view)))); + + SparkRunnerDebugger.DebugSparkPipelineResult result = + (SparkRunnerDebugger.DebugSparkPipelineResult) pipeline.run(); + + final String expectedPipeline = + "sparkContext.()\n" + + "_.mapPartitions(new org.apache.beam.sdk.transforms.Create$Values$2())\n" + + "_.aggregate(..., new org.apache.beam.sdk.transforms.View$SingletonCombineFn(), ...)\n" + + "_.\n" + + "sparkContext.()\n" + + "_.mapPartitions(new org.apache.beam.sdk.transforms.Create$Values$2())\n" + + "_.mapPartitions(new org.apache.beam.sdk.transforms.Contextful())"; + + assertThat( + "Debug pipeline did not equal expected", + result.getDebugString(), + Matchers.equalTo(expectedPipeline)); + } + private static class FormatKVFn extends DoFn, String> { @SuppressWarnings("unused") @ProcessElement diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerRegistrarTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerRegistrarTest.java index 2276ef8e0cdad..c0b62be045fe7 100644 --- a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerRegistrarTest.java +++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerRegistrarTest.java @@ -23,8 +23,8 @@ import java.util.ServiceLoader; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; import org.apache.beam.sdk.runners.PipelineRunnerRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/TestSparkPipelineOptionsRegistrar.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/TestSparkPipelineOptionsRegistrar.java index 2199e1daee8df..5015fb040e5dd 100644 --- a/runners/spark/src/test/java/org/apache/beam/runners/spark/TestSparkPipelineOptionsRegistrar.java +++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/TestSparkPipelineOptionsRegistrar.java @@ -20,7 +20,7 @@ import com.google.auto.service.AutoService; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * A registrar for {@link TestSparkPipelineOptions} to temporarily work around some complexities in diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/InMemoryMetrics.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/InMemoryMetrics.java index b69275b2e3924..db040bbfcc45e 100644 --- a/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/InMemoryMetrics.java +++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/InMemoryMetrics.java @@ -23,7 +23,7 @@ import java.util.Collection; import java.util.Properties; import org.apache.beam.runners.spark.metrics.WithMetricsSupport; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.spark.metrics.sink.Sink; /** An in-memory {@link Sink} implementation for tests. */ diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/SparkMetricsSinkTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/SparkMetricsSinkTest.java index c632a9800a8ba..6c53ba8056ad9 100644 --- a/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/SparkMetricsSinkTest.java +++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/SparkMetricsSinkTest.java @@ -37,8 +37,8 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.ClassRule; diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/AvroPipelineTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/AvroPipelineTest.java index b46f5fafd42e4..4ee93a84d92ec 100644 --- a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/AvroPipelineTest.java +++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/AvroPipelineTest.java @@ -33,8 +33,8 @@ import org.apache.beam.sdk.extensions.avro.io.AvroIO; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Resources; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Resources; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/NumShardsTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/NumShardsTest.java index 777d1d4dc5f03..ad95c17b5a1e4 100644 --- a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/NumShardsTest.java +++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/NumShardsTest.java @@ -33,8 +33,8 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/ReaderToIteratorAdapterTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/ReaderToIteratorAdapterTest.java index 5b43e1b2aa8ff..9892dd467b855 100644 --- a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/ReaderToIteratorAdapterTest.java +++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/ReaderToIteratorAdapterTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.spark.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/GroupNonMergingWindowsFunctionsTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/GroupNonMergingWindowsFunctionsTest.java index c2896e7f5f2ce..fd299924af915 100644 --- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/GroupNonMergingWindowsFunctionsTest.java +++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/GroupNonMergingWindowsFunctionsTest.java @@ -38,7 +38,7 @@ import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Bytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Bytes; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Assert; diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkCombineFnTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkCombineFnTest.java index adf8d23d7a6aa..295b7ef2b948d 100644 --- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkCombineFnTest.java +++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkCombineFnTest.java @@ -41,8 +41,8 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Before; diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkExecutableStageFunctionTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkExecutableStageFunctionTest.java index 44109068aca06..2068e272051a9 100644 --- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkExecutableStageFunctionTest.java +++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkExecutableStageFunctionTest.java @@ -58,7 +58,7 @@ import org.apache.beam.sdk.transforms.join.RawUnionValue; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkInputDataProcessorTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkInputDataProcessorTest.java index 82041d94300b7..be2b788f12f50 100644 --- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkInputDataProcessorTest.java +++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkInputDataProcessorTest.java @@ -37,8 +37,8 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Instant; import org.junit.Rule; import org.junit.Test; diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/TransformTranslatorTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/TransformTranslatorTest.java index f55f6a921a9fd..c271eae426cab 100644 --- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/TransformTranslatorTest.java +++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/TransformTranslatorTest.java @@ -30,8 +30,8 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Test; diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java index 46c106e75727f..f3e41911cee0b 100644 --- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java +++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java @@ -67,10 +67,10 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.PDone; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.serialization.Serializer; diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SparkCoGroupByKeyStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SparkCoGroupByKeyStreamingTest.java index ac66be0439696..b64fa8e33eb9f 100644 --- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SparkCoGroupByKeyStreamingTest.java +++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SparkCoGroupByKeyStreamingTest.java @@ -41,7 +41,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/runners/twister2/build.gradle b/runners/twister2/build.gradle index 0b7c248f1ada2..c9a5e526f3379 100644 --- a/runners/twister2/build.gradle +++ b/runners/twister2/build.gradle @@ -31,7 +31,7 @@ configurations { description = "Apache Beam :: Runners :: Twister2" def twister2_version = '0.6.0' dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation(project(path: ":runners:core-java")){ exclude group: 'com.esotericsoftware.kryo', module: 'kryo' diff --git a/runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2Runner.java b/runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2Runner.java index 890172f89343f..5a390b4741f44 100644 --- a/runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2Runner.java +++ b/runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2Runner.java @@ -55,7 +55,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsValidator; import org.apache.beam.sdk.runners.PTransformOverride; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * A {@link PipelineRunner} that executes the operations in the pipeline by first translating them diff --git a/runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2RunnerRegistrar.java b/runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2RunnerRegistrar.java index ef84313cae3b4..e0bb8ea94efe4 100644 --- a/runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2RunnerRegistrar.java +++ b/runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2RunnerRegistrar.java @@ -22,7 +22,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; import org.apache.beam.sdk.runners.PipelineRunnerRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * AutoService registrar - will register Twister2Runner and Twister2Options as possible pipeline diff --git a/runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2TranslationContext.java b/runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2TranslationContext.java index e8541517c1fb0..a5b03cd628653 100644 --- a/runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2TranslationContext.java +++ b/runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2TranslationContext.java @@ -34,7 +34,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** Twister2TranslationContext. */ @SuppressWarnings({ diff --git a/runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/ParDoMultiOutputTranslatorBatch.java b/runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/ParDoMultiOutputTranslatorBatch.java index 0c98dc335eae0..1a452e8b372ba 100644 --- a/runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/ParDoMultiOutputTranslatorBatch.java +++ b/runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/ParDoMultiOutputTranslatorBatch.java @@ -41,7 +41,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; /** ParDo translator. */ @SuppressWarnings({ diff --git a/runners/twister2/src/main/java/org/apache/beam/runners/twister2/utils/Twister2AssignContext.java b/runners/twister2/src/main/java/org/apache/beam/runners/twister2/utils/Twister2AssignContext.java index cb5c4d91c89d9..491917a8839c0 100644 --- a/runners/twister2/src/main/java/org/apache/beam/runners/twister2/utils/Twister2AssignContext.java +++ b/runners/twister2/src/main/java/org/apache/beam/runners/twister2/utils/Twister2AssignContext.java @@ -20,7 +20,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Instant; /** doc. */ diff --git a/runners/twister2/src/main/java/org/apache/beam/runners/twister2/utils/Twister2SideInputReader.java b/runners/twister2/src/main/java/org/apache/beam/runners/twister2/utils/Twister2SideInputReader.java index 77afdc23222d0..a4735080f8f5c 100644 --- a/runners/twister2/src/main/java/org/apache/beam/runners/twister2/utils/Twister2SideInputReader.java +++ b/runners/twister2/src/main/java/org/apache/beam/runners/twister2/utils/Twister2SideInputReader.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.twister2.utils; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import edu.iu.dsc.tws.api.dataset.DataPartition; import edu.iu.dsc.tws.api.dataset.DataPartitionConsumer; diff --git a/scripts/ci/release/test/resources/mass_comment.txt b/scripts/ci/release/test/resources/mass_comment.txt index b3a307c9e698c..93468b0c961b4 100644 --- a/scripts/ci/release/test/resources/mass_comment.txt +++ b/scripts/ci/release/test/resources/mass_comment.txt @@ -59,9 +59,10 @@ Run Portable_Python PreCommit Run PostCommit_Java_Dataflow Run PostCommit_Java_DataflowV2 Run PostCommit_Java_Hadoop_Versions -Run Python 3.7 PostCommit Run Python 3.8 PostCommit Run Python 3.9 PostCommit +Run Python 3.10 PostCommit +Run Python 3.11 PostCommit Run Python Dataflow V2 ValidatesRunner Run Python Dataflow ValidatesContainer Run Python Dataflow ValidatesRunner diff --git a/sdks/go.mod b/sdks/go.mod index fedb04f904852..192802f77eaa9 100644 --- a/sdks/go.mod +++ b/sdks/go.mod @@ -20,95 +20,108 @@ // directory. module github.com/apache/beam/sdks/v2 -go 1.19 +go 1.20 require ( - cloud.google.com/go/bigquery v1.51.2 - cloud.google.com/go/bigtable v1.18.1 - cloud.google.com/go/datastore v1.11.0 + cloud.google.com/go/bigquery v1.56.0 + cloud.google.com/go/bigtable v1.20.0 + cloud.google.com/go/datastore v1.15.0 cloud.google.com/go/profiler v0.3.1 - cloud.google.com/go/pubsub v1.31.0 - cloud.google.com/go/spanner v1.47.0 - cloud.google.com/go/storage v1.30.1 - github.com/aws/aws-sdk-go-v2 v1.18.1 - github.com/aws/aws-sdk-go-v2/config v1.18.27 - github.com/aws/aws-sdk-go-v2/credentials v1.13.26 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.67 - github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0 - github.com/aws/smithy-go v1.13.5 + cloud.google.com/go/pubsub v1.33.0 + cloud.google.com/go/spanner v1.50.0 + cloud.google.com/go/storage v1.33.0 + github.com/aws/aws-sdk-go-v2 v1.21.2 + github.com/aws/aws-sdk-go-v2/config v1.19.0 + github.com/aws/aws-sdk-go-v2/credentials v1.13.43 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.91 + github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2 + github.com/aws/smithy-go v1.15.0 github.com/docker/go-connections v0.4.0 github.com/dustin/go-humanize v1.0.1 github.com/go-sql-driver/mysql v1.7.1 github.com/golang/protobuf v1.5.3 // TODO(danoliveira): Fully replace this with google.golang.org/protobuf - github.com/google/go-cmp v0.5.9 - github.com/google/uuid v1.3.0 + github.com/google/go-cmp v0.6.0 + github.com/google/uuid v1.3.1 github.com/johannesboyne/gofakes3 v0.0.0-20221110173912-32fb85c5aed6 github.com/lib/pq v1.10.9 github.com/linkedin/goavro/v2 v2.12.0 - github.com/proullon/ramsql v0.0.0-20211120092837-c8d0a408b939 + github.com/proullon/ramsql v0.1.3 github.com/spf13/cobra v1.7.0 - github.com/testcontainers/testcontainers-go v0.20.1 - github.com/tetratelabs/wazero v1.2.1 + github.com/testcontainers/testcontainers-go v0.25.0 + github.com/tetratelabs/wazero v1.5.0 github.com/xitongsys/parquet-go v1.6.2 github.com/xitongsys/parquet-go-source v0.0.0-20220315005136-aec0fe3e777c - go.mongodb.org/mongo-driver v1.11.7 - golang.org/x/net v0.11.0 - golang.org/x/oauth2 v0.9.0 - golang.org/x/sync v0.3.0 - golang.org/x/sys v0.9.0 - golang.org/x/text v0.10.0 - google.golang.org/api v0.128.0 - google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc - google.golang.org/grpc v1.55.0 - google.golang.org/protobuf v1.30.0 + go.mongodb.org/mongo-driver v1.12.1 + golang.org/x/net v0.17.0 + golang.org/x/oauth2 v0.13.0 + golang.org/x/sync v0.4.0 + golang.org/x/sys v0.13.0 + golang.org/x/text v0.13.0 + google.golang.org/api v0.147.0 + google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 + google.golang.org/grpc v1.59.0 + google.golang.org/protobuf v1.31.0 gopkg.in/retry.v1 v1.0.3 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/fsouza/fake-gcs-server v1.45.2 - golang.org/x/exp v0.0.0-20230321023759-10a507213a29 + github.com/fsouza/fake-gcs-server v1.47.5 + golang.org/x/exp v0.0.0-20230807204917-050eac23e9de ) require ( - cloud.google.com/go v0.110.2 // indirect - cloud.google.com/go/compute v1.19.3 // indirect + dario.cat/mergo v1.0.0 // indirect + github.com/Microsoft/hcsshim v0.11.0 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/shirou/gopsutil/v3 v3.23.8 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect +) + +require ( + cloud.google.com/go v0.110.8 // indirect + cloud.google.com/go/compute v1.23.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.0 // indirect - cloud.google.com/go/longrunning v0.5.0 // indirect + cloud.google.com/go/iam v1.1.2 // indirect + cloud.google.com/go/longrunning v0.5.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/Microsoft/go-winio v0.5.2 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect github.com/apache/arrow/go/v12 v12.0.0 // indirect github.com/apache/thrift v0.16.0 // indirect github.com/aws/aws-sdk-go v1.34.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect - github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe // indirect - github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195 // indirect - github.com/containerd/containerd v1.6.19 // indirect + github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 // indirect + github.com/containerd/containerd v1.7.6 // indirect github.com/cpuguy83/dockercfg v0.3.1 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker v23.0.5+incompatible // indirect + github.com/docker/docker v24.0.6+incompatible // but required to resolve issue docker has with go1.20 github.com/docker/go-units v0.5.0 // indirect - github.com/envoyproxy/go-control-plane v0.11.0 // indirect - github.com/envoyproxy/protoc-gen-validate v0.10.0 // indirect + github.com/envoyproxy/go-control-plane v0.11.1 // indirect + github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect github.com/felixge/httpsnoop v1.0.2 // indirect github.com/goccy/go-json v0.9.11 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -117,47 +130,46 @@ require ( github.com/google/flatbuffers v2.0.8+incompatible // indirect github.com/google/pprof v0.0.0-20221103000818-d260c55eee4c // indirect github.com/google/renameio/v2 v2.0.0 // indirect - github.com/google/s2a-go v0.1.4 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect - github.com/googleapis/gax-go/v2 v2.11.0 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/mux v1.8.0 // indirect - github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect - github.com/klauspost/compress v1.16.5 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kr/text v0.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/moby/patternmatcher v0.5.0 // indirect github.com/moby/sys/sequential v0.5.0 // indirect - github.com/moby/term v0.0.0-20221128092401-c43b287e0e0f // indirect + github.com/moby/term v0.5.0 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc2 // indirect + github.com/opencontainers/image-spec v1.1.0-rc4 // indirect github.com/opencontainers/runc v1.1.5 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pkg/xattr v0.4.9 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 // indirect - github.com/sirupsen/logrus v1.9.2 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.1.1 // indirect - github.com/xdg-go/stringprep v1.0.3 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/crypto v0.10.0 // indirect - golang.org/x/mod v0.9.0 // indirect - golang.org/x/tools v0.7.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/mod v0.11.0 // indirect + golang.org/x/tools v0.10.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect ) diff --git a/sdks/go.sum b/sdks/go.sum index 3b74e68750200..457fe2bf1ffe8 100644 --- a/sdks/go.sum +++ b/sdks/go.sum @@ -8,44 +8,47 @@ cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA= -cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= +cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME= +cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.51.2 h1:p6SZQJBh64rNJB/9V5O0jvMBI8O/XV5rJKlhmmCU+2o= -cloud.google.com/go/bigquery v1.51.2/go.mod h1:6YYSJ37dAY1HyMDq/+XByPmzsC52MgzNXhxjlTzIVCM= -cloud.google.com/go/bigtable v1.18.1 h1:SxQk9Bj6OKxeiuvevG/KBjqGn/7X8heZbWfK0tYkFd8= -cloud.google.com/go/bigtable v1.18.1/go.mod h1:NAVyfJot9jlo+KmgWLUJ5DJGwNDoChzAcrecLpmuAmY= -cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds= -cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= +cloud.google.com/go/bigquery v1.56.0 h1:LHIc9E7Kw+ftFpQFKzZYBB88IAFz7qONawXXx0F3QBo= +cloud.google.com/go/bigquery v1.56.0/go.mod h1:KDcsploXTEY7XT3fDQzMUZlpQLHzE4itubHrnmhUrZA= +cloud.google.com/go/bigtable v1.20.0 h1:NqZC/WcesSn4O8L0I2JmuNsUigSyBQifVLYgM9LMQeQ= +cloud.google.com/go/bigtable v1.20.0/go.mod h1:upJDn8frsjzpRMfybiWkD1PG6WCCL7CRl26MgVeoXY4= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/datacatalog v1.13.0 h1:4H5IJiyUE0X6ShQBqgFFZvGGcrwGVndTwUSLP4c52gw= +cloud.google.com/go/datacatalog v1.17.1 h1:qGWrlYvWtK+8jD1jhwq5BsGoSr7S4/LOroV7LwXi00g= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/datastore v1.11.0 h1:iF6I/HaLs3Ado8uRKMvZRvF/ZLkWaWE9i8AiHzbC774= -cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= -cloud.google.com/go/iam v1.1.0 h1:67gSqaPukx7O8WLLHMa0PNs3EBGd2eE4d+psbO/CO94= -cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= -cloud.google.com/go/kms v1.10.2 h1:8UePKEypK3SQ6g+4mn/s/VgE5L7XOh+FwGGRUqvY3Hw= -cloud.google.com/go/longrunning v0.5.0 h1:DK8BH0+hS+DIvc9a2TPnteUievsTCH4ORMAASSb7JcQ= -cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPkwP0YF1oVEchc= +cloud.google.com/go/datastore v1.15.0 h1:0P9WcsQeTWjuD1H14JIY7XQscIPQ4Laje8ti96IC5vg= +cloud.google.com/go/datastore v1.15.0/go.mod h1:GAeStMBIt9bPS7jMJA85kgkpsMkvseWWXiaHya9Jes8= +cloud.google.com/go/iam v1.1.2 h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4= +cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/kms v1.15.2 h1:lh6qra6oC4AyWe5fUUUBe/S27k12OHAleOOOw6KakdE= +cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= +cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= cloud.google.com/go/profiler v0.3.1 h1:b5got9Be9Ia0HVvyt7PavWxXEht15B9lWnigdvHtxOc= cloud.google.com/go/profiler v0.3.1/go.mod h1:GsG14VnmcMFQ9b+kq71wh3EKMZr3WRMgLzNiFRpW7tE= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.31.0 h1:aXdyyJz90kA+bor9+6+xHAciMD5mj8v15WqFZ5E0sek= -cloud.google.com/go/pubsub v1.31.0/go.mod h1:dYmJ3K97NCQ/e4OwZ20rD4Ym3Bu8Gu9m/aJdWQjdcks= -cloud.google.com/go/spanner v1.47.0 h1:aqiMP8dhsEXgn9K5EZBWxPG7dxIiyM2VaikqeU4iteg= -cloud.google.com/go/spanner v1.47.0/go.mod h1:IXsJwVW2j4UKs0eYDqodab6HgGuA1bViSqW4uH9lfUI= +cloud.google.com/go/pubsub v1.33.0 h1:6SPCPvWav64tj0sVX/+npCBKhUi/UjJehy9op/V3p2g= +cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= +cloud.google.com/go/spanner v1.50.0 h1:QrJFOpaxCXdXF+GkiruLz642PHxkdj68PbbnLw3O2Zw= +cloud.google.com/go/spanner v1.50.0/go.mod h1:eGj9mQGK8+hkgSVbHNQ06pQ4oS+cyc4tXXd6Dif1KoM= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= -cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= +cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M= +cloud.google.com/go/storage v1.33.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= @@ -59,12 +62,12 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= -github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/hcsshim v0.9.7 h1:mKNHW/Xvv1aFH87Jb6ERDzXTJTLPlmzfZ28VBFD/bfg= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/hcsshim v0.11.0 h1:7EFNIY4igHEXUdj1zXgAyU3fLc7QfOKHbkldRVTBdiM= +github.com/Microsoft/hcsshim v0.11.0/go.mod h1:OEthFdQv/AD2RAdzR6Mm1N1KPCztGKDurW1Z8b8VGMM= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 h1:byKBBF2CKWBjjA4J1ZL2JXttJULvWSl50LegTyRZ728= github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0= github.com/apache/arrow/go/v12 v12.0.0 h1:xtZE63VWl7qLdB0JObIXvvhGjoVNrQ9ciIHG2OK5cmc= @@ -78,77 +81,61 @@ github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZve github.com/aws/aws-sdk-go v1.34.0 h1:brux2dRrlwCF5JhTL7MUT3WUwo9zfDHZZp3+g3Mvlmo= github.com/aws/aws-sdk-go v1.34.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go-v2 v1.7.1/go.mod h1:L5LuPC1ZgDr2xQS7AmIec/Jlc7O/Y1u2KxJyNVab250= -github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo= -github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= +github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA= +github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 h1:Sc82v7tDQ/vdU1WtuSyzZ1I7y/68j//HJ6uozND1IDs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14/go.mod h1:9NCTOURS8OpxvoAVHq79LK81/zC78hfRWFn+aL0SPcY= github.com/aws/aws-sdk-go-v2/config v1.5.0/go.mod h1:RWlPOAW3E3tbtNAqTwvSW54Of/yP3oiZXMI0xfUdjyA= -github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= -github.com/aws/aws-sdk-go-v2/config v1.18.27 h1:Az9uLwmssTE6OGTpsFqOnaGpLnKDqNYOJzWuC6UAYzA= -github.com/aws/aws-sdk-go-v2/config v1.18.27/go.mod h1:0My+YgmkGxeqjXZb5BYme5pc4drjTnM+x1GJ3zv42Nw= +github.com/aws/aws-sdk-go-v2/config v1.19.0 h1:AdzDvwH6dWuVARCl3RTLGRc4Ogy+N7yLFxVxXe1ClQ0= +github.com/aws/aws-sdk-go-v2/config v1.19.0/go.mod h1:ZwDUgFnQgsazQTnWfeLWk5GjeqTQTL8lMkoE1UXzxdE= github.com/aws/aws-sdk-go-v2/credentials v1.3.1/go.mod h1:r0n73xwsIVagq8RsxmZbGSRQFj9As3je72C2WzUIToc= -github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= -github.com/aws/aws-sdk-go-v2/credentials v1.13.26 h1:qmU+yhKmOCyujmuPY7tf5MxR/RKyZrOPO3V4DobiTUk= -github.com/aws/aws-sdk-go-v2/credentials v1.13.26/go.mod h1:GoXt2YC8jHUBbA4jr+W3JiemnIbkXOfxSXcisUsZ3os= +github.com/aws/aws-sdk-go-v2/credentials v1.13.43 h1:LU8vo40zBlo3R7bAvBVy/ku4nxGEyZe9N8MqAeFTzF8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.43/go.mod h1:zWJBz1Yf1ZtX5NGax9ZdNjhhI4rgjfgsyk6vTY1yfVg= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.3.0/go.mod h1:2LAuqPx1I6jNfaGDucWfA2zqQCYCOMCDHiCOciALyNw= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 h1:LxK/bitrAr4lnh9LnIS6i7zWbCOdMsfzKFBI6LUCS0I= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4/go.mod h1:E1hLXN/BL2e6YizK1zFlYd8vsfi2GTjbjBazinMmeaM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 h1:PIktER+hwIG286DqXyvVENjgLTAwGgoeriLDD5C+YlQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13/go.mod h1:f/Ib/qYjhV2/qdsf79H3QP/eRE4AkVyEf6sk7XfZ1tg= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.3.2/go.mod h1:qaqQiHSrOUVOfKe6fhgQ6UzhxjwqVW8aHNegd6Ws4w4= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.67 h1:fI9/5BDEaAv/pv1VO1X1n3jfP9it+IGqWsCuuBQI8wM= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.67/go.mod h1:zQClPRIwQZfJlZq6WZve+s4Tb4JW+3V6eS+4+KrYeP8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 h1:srIVS45eQuewqz6fKKu6ZGXaq6FuFg5NzgQBAM6g8Y4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.91 h1:haAyxKHwoE+y/TJt+qHcPQf1dCViyyGbWcKjjYUllTE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.91/go.mod h1:ACQ6ta5YFlfSOz2c9A+EVYawLxFMZ0rI3Q0A0tGieKo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 h1:nFBQlGtkbPzp/NjZLuFxRqmT91rLJkgvsEQs68h962Y= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 h1:JRVhO25+r3ar2mKGP7E0LDl8K9/G36gjlqca5iQbaqc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8= github.com/aws/aws-sdk-go-v2/internal/ini v1.1.1/go.mod h1:Zy8smImhTdOETZqfyn01iNOe0CNggVbPjCajyaz6Gvg= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35/go.mod h1:0Eg1YjxE0Bhn56lx+SHJwCzhW+2JGtizsrx+lCqrfm0= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.25/go.mod h1:SUbB4wcbSEyCvqBxv/O/IBf93RbEze7U7OnoTlpPB+g= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 h1:wscW+pnn3J1OYnanMnza5ZVYXLX4cKk5rAvUAl4Qu+c= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26/go.mod h1:MtYiox5gvyB+OyP0Mr0Sm/yzbEAIPL9eijj/ouHAPw0= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 h1:hze8YsjSh8Wl1rYa1CJpRmXP21BvOBuc76YhW0HsuQ4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45/go.mod h1:lD5M20o09/LCuQ2mE62Mb/iSdSlCNuj6H5ci7tW7OsE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6 h1:wmGLw2i8ZTlHLw7a9ULGfQbuccw8uIiNr6sol5bFzc8= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6/go.mod h1:Q0Hq2X/NuL7z8b1Dww8rmOFl+jzusKEcyvkKspwdpyc= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.1/go.mod h1:v33JQ57i2nekYTA70Mb+O18KeH4KqhdqxTJZNK1zdRE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.28/go.mod h1:spfrICMD6wCAhjhzHuy6DOZZ+LAIY10UxhUmLzpJTTs= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 h1:zZSLP3v3riMOP14H7b4XP0uyfREDQOYv2cqIrvTXDNQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29/go.mod h1:z7EjRjVwZ6pWcWdI2H64dKttvzaP99jRIj5hphW0M5U= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 h1:7R8uRYyXzdD71KWVCL78lJZltah6VVznXBazvKjfH58= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15/go.mod h1:26SQUPcTNgV1Tapwdt4a1rOsYRsnBsJHLMPoxK2b0d8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 h1:skaFGzv+3kA+v2BPKhuekeb1Hbb105+44r8ASC+q5SE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38/go.mod h1:epIZoRSSbRIwLPJU5F+OldHhwZPBdpDeQkRdCeY3+00= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.1/go.mod h1:zceowr5Z1Nh2WVP8bf/3ikB41IZW59E4yIYbg+pC6mw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 h1:bkRyG4a929RCnpVSTvLM2j/T4ls015ZhhYApbmYs15s= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28/go.mod h1:jj7znCIg05jXlaGBlFMGP8+7UN3VtCkRBG2spnmRQkU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 h1:WWZA/I2K4ptBS1kg0kV1JbBtG/umed0vwHRrmcr9z7k= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37/go.mod h1:vBmDnwWXWxNPFRMmG2m/3MKOe+xEcMDo1tanpaWCcck= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.1/go.mod h1:6EQZIwNNvHpq/2/QSJnp4+ECvqIy55w95Ofs0ze+nGQ= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2/go.mod h1:4tfW5l4IAB32VWCDEBxCRtR9T4BWy4I4kr1spr8NgZM= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 h1:dBL3StFxHtpBzJJ/mNEsjXVgfO+7jR0dAIEwLqMapEA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3/go.mod h1:f1QyiAsvIv4B49DmCqrhlXqyaR+0IxMmyX+1P+AnzOM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 h1:9ulSU5ClouoPIYhDQdg9tpl83d5Yb91PXTKK+17q+ow= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6/go.mod h1:lnc2taBsR9nTlz9meD+lhFZZ9EWY712QHrRflWpTcOA= github.com/aws/aws-sdk-go-v2/service/s3 v1.11.1/go.mod h1:XLAGFrEjbvMCLvAtWLLP32yTv8GpBquCApZEycDLunI= -github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1/go.mod h1:J9kLNzEiHSeGMyN7238EjJmBpCniVzFda75Gxl/NqB8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0 h1:ya7fmrN2fE7s1P2gaPbNg5MTkERVWfsH8ToP1YC4Z9o= -github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2 h1:Ll5/YVCOzRB+gxPqs2uD0R7/MyATC0w85626glSKmp4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2/go.mod h1:Zjfqt7KhQK+PO1bbOsFNzKgaq7TcxzmEoDWN8lM0qzQ= github.com/aws/aws-sdk-go-v2/service/sso v1.3.1/go.mod h1:J3A3RGUvuCZjvSuZEcOpHDnzZP/sKbhDWV2T1EOzFIM= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 h1:nneMBM2p79PGWBQovYO/6Xnc2ryRMw3InnDJq1FHkSY= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.12/go.mod h1:HuCOxYsF21eKrerARYO6HapNeh9GBNq7fius2AcwodY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 h1:2qTR7IFk7/0IN/adSFhYu9Xthr0zVFTgBrmPldILn80= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12/go.mod h1:E4VrHCPzmVB/KFXtqBGKb3c8zpbNBgKe3fisDNLAW5w= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 h1:JuPGc7IkOP4AaqcZSIcyqLpFSqBWK32rM9+a1g6u73k= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.2/go.mod h1:gsL4keucRCgW+xA85ALBpRFfdSLH4kHOVSnLMSuBECo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 h1:HFiiRkf1SdaAmV3/BHOFZ9DjFynPHj8G/UIO1lQS+fk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3/go.mod h1:a7bHA82fyUXOm+ZSWKU6PIoBxrjSprdLoM8xPYvzYVg= github.com/aws/aws-sdk-go-v2/service/sts v1.6.0/go.mod h1:q7o0j7d7HrJk/vr9uUt3BVRASvcU7gYZB9PUgPiByXg= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 h1:XFJ2Z6sNUUcAz9poj+245DMkrHE4h2j5I9/xD50RHfE= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.2/go.mod h1:dp0yLPsLBOi++WTxzCjA/oZqi6NPIhoR+uF7GeMU9eg= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwFYTCZVhlsSSBvlbU= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ= github.com/aws/smithy-go v1.6.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= -github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= -github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= +github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= @@ -158,35 +145,30 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe h1:QQ3GSy+MqSHxm/d8nCtnAiZdYFd45cYZPs8vOOIYKfk= github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195 h1:58f1tJ1ra+zFINPlwLWvQsR9CzAKt2e+EWV2yX9oXQ4= -github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.6.19 h1:F0qgQPrG0P2JPgwpxWxYavrVeXAG0ezUIB9Z/4FTUAU= -github.com/containerd/containerd v1.6.19/go.mod h1:HZCDMn4v/Xl2579/MvtOC2M206i+JJ6VxFWU/NetrGY= -github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= +github.com/containerd/containerd v1.7.6 h1:oNAVsnhPoy4BTPQivLgTzI9Oleml9l/+eYIDYXRCYo8= +github.com/containerd/containerd v1.7.6/go.mod h1:SY6lrkkuJT40BVNO37tlYTSnKJnP5AXBc0fhx0q+TJ4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v23.0.5+incompatible h1:DaxtlTJjFSnLOXVNUBU1+6kXGz2lpDoEAH6QoxaSg8k= -github.com/docker/docker v23.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE= +github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -197,13 +179,11 @@ github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+m github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.11.0 h1:jtLewhRR2vMRNnq2ZZUoCjUlgut+Y0+sDDWPOfwOi1o= -github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= +github.com/envoyproxy/go-control-plane v0.11.1 h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM= +github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v0.10.0 h1:oIfnZFdC0YhpNNEX+SuIqko4cqqVZeN9IGTrhZje83Y= -github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o= github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= @@ -211,14 +191,14 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/frankban/quicktest v1.2.2/go.mod h1:Qh/WofXFeiAFII1aEBu529AtJo6Zg2VHscnEsbBnJ20= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/fsouza/fake-gcs-server v1.45.2 h1:m4hkghNBY7lmPtpgE41nZa1mOo2PedAZnw3Hd2RfsGk= -github.com/fsouza/fake-gcs-server v1.45.2/go.mod h1:JDINLKL72GbpnqrtS5cptlcIUDQJlI4iNj4lmh7EvmQ= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/fsouza/fake-gcs-server v1.47.5 h1:o+wL01s01j/2OdkIaduDogXw2bZveq9TFb8f+BqEHtM= +github.com/fsouza/fake-gcs-server v1.47.5/go.mod h1:PhN8F1rHAOCL5jWyXcw8nPfLfHnka6D9fT7ctL9nbkA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gorp/gorp v2.0.0+incompatible h1:dIQPsBtl6/H1MjVseWuWPXa7ET4p6Dve4j3Hg+UjqYw= -github.com/go-gorp/gorp v2.0.0+incompatible/go.mod h1:7IfkAQnO7jfT/9IQ3R9wL1dFhukN6aQxzKTHnkxzA/E= +github.com/go-gorp/gorp v2.2.0+incompatible h1:xAUh4QgEeqPPhK3vxZN+bzrim1z5Av6q837gtjUlshc= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= @@ -252,10 +232,8 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -281,8 +259,9 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= @@ -295,32 +274,34 @@ github.com/google/pprof v0.0.0-20221103000818-d260c55eee4c/go.mod h1:dDKJzRmX4S3 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= -github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= -github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.4 h1:uGy6JWR/uMIILU8wbf+OkstIrNiMjGpEIyhx8f6W7s4= -github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= -github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/go-uuid v0.0.0-20180228145832-27454136f036/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgx/v5 v5.3.1 h1:Fcr8QJ1ZeLi5zsPZqQeUZhNhxfkkKBOgJuYkJHoBOtU= github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -339,10 +320,10 @@ github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= @@ -350,11 +331,12 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/linkedin/goavro/v2 v2.12.0 h1:rIQQSj8jdAUlKQh6DttK8wCRv4t4QO09g1C4aBWXslg= github.com/linkedin/goavro/v2 v2.12.0/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= @@ -363,15 +345,15 @@ github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcs github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/minio-go/v7 v7.0.55 h1:ZXqUO/8cgfHzI+08h/zGuTTFpISSA32BZmBE3FCLJas= +github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo= github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= -github.com/moby/term v0.0.0-20221128092401-c43b287e0e0f h1:J/7hjLaHLD7epG0m6TBMGmp4NQ+ibBYLfeyJWdAIFLA= -github.com/moby/term v0.0.0-20221128092401-c43b287e0e0f/go.mod h1:15ce4BGCFxt7I5NQKT+HV0yEDxmf6fSysfEDiVo3zFM= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= @@ -382,8 +364,8 @@ github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2 github.com/ncw/swift v1.0.52/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= -github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= +github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -398,12 +380,13 @@ github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE= github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/proullon/ramsql v0.0.0-20211120092837-c8d0a408b939 h1:mtMU7aT8cTAyNL3O4RyOfe/OOUxwCN525SIbKQoUvw0= -github.com/proullon/ramsql v0.0.0-20211120092837-c8d0a408b939/go.mod h1:jG8oAQG0ZPHPyxg5QlMERS31airDC+ZuqiAe8DUvFVo= +github.com/proullon/ramsql v0.1.3 h1:/LRcXJf4lEmhdb4tYcci473I2VynjcZSzh2hsjJ8rSk= +github.com/proullon/ramsql v0.1.3/go.mod h1:CFGqeQHQpdRfWqYmWD3yXqPTEaHkF4zgXy1C6qDWc9E= github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a h1:3QH7VyOaaiUHNrA9Se4YQIRkDTCw1EJls9xTUCaCeRM= github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a/go.mod h1:4r5QyqhjIWCcK8DO4KMclc5Iknq5qVBAlbYYzAbUScQ= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= @@ -414,11 +397,16 @@ github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5P github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 h1:J6qvD6rbmOil46orKqJaRPG+zTpoGlBTUdyv8ki63L0= github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63/go.mod h1:n+VKSARF5y/tS9XFSP7vWDfS+GUC5vs/YT7M5XDTUEM= +github.com/shirou/gopsutil/v3 v3.23.8 h1:xnATPiybo6GgdRoC4YoGnxXZFRc3dqQTGi73oLvvBrE= +github.com/shirou/gopsutil/v3 v3.23.8/go.mod h1:7hmCaBn+2ZwaZOr6jmPBZDfawwMGuo1id3C6aM8EDqQ= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= -github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= @@ -433,29 +421,31 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/testcontainers/testcontainers-go v0.20.1 h1:mK15UPJ8c5P+NsQKmkqzs/jMdJt6JMs5vlw2y4j92c0= -github.com/testcontainers/testcontainers-go v0.20.1/go.mod h1:zb+NOlCQBkZ7RQp4QI+YMIHyO2CQ/qsXzNF5eLJ24SY= -github.com/tetratelabs/wazero v1.2.1 h1:J4X2hrGzJvt+wqltuvcSjHQ7ujQxA9gb6PeMs4qlUWs= -github.com/tetratelabs/wazero v1.2.1/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/testcontainers/testcontainers-go v0.25.0 h1:erH6cQjsaJrH+rJDU9qIf89KFdhK0Bft0aEZHlYC3Vs= +github.com/testcontainers/testcontainers-go v0.25.0/go.mod h1:4sC9SiJyzD1XFi59q8umTQYWxnkweEc5OjVtTUlJzqQ= +github.com/tetratelabs/wazero v1.5.0 h1:Yz3fZHivfDiZFUXnWMPUoiW7s8tC1sjdBtlJn08qYa0= +github.com/tetratelabs/wazero v1.5.0/go.mod h1:0U0G41+ochRKoPKCJlh0jMg1CHkyfK8kDqiirMmKY8A= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E= -github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= -github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs= -github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww= github.com/xitongsys/parquet-go v1.6.2 h1:MhCaXii4eqceKPu9BwrjLqyK10oX9WF+xGhwvwbw7xM= github.com/xitongsys/parquet-go v1.6.2/go.mod h1:IulAQyalCm0rPiZVNnCgm/PCL64X2tdSVGMQ/UeKqWA= @@ -468,19 +458,20 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.mongodb.org/mongo-driver v1.11.7 h1:LIwYxASDLGUg/8wOhgOOZhX8tQa/9tgZPgzZoVqJvcs= -go.mongodb.org/mongo-driver v1.11.7/go.mod h1:G9TgswdsWjX4tmDA5zfs2+6AEPpYJwqblyjsfuh8oXY= +go.mongodb.org/mongo-driver v1.12.1 h1:nLkghSU8fQNaK7oUmDhQFsnrtcoNy7Z6LVFKsEecqgE= +go.mongodb.org/mongo-driver v1.12.1/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -490,10 +481,9 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -504,8 +494,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230807204917-050eac23e9de h1:l5Za6utMv/HsBWWqzt4S8X17j+kt1uVETUX5UFhn2rE= +golang.org/x/exp v0.0.0-20230807204917-050eac23e9de/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -526,8 +516,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= -golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -546,22 +536,21 @@ golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs= -golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -569,10 +558,9 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -583,6 +571,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -594,10 +583,10 @@ golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -607,11 +596,13 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -622,8 +613,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -661,8 +653,8 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -679,8 +671,8 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.128.0 h1:RjPESny5CnQRn9V6siglged+DZCgfu9l6mO9dkX9VOg= -google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750= +google.golang.org/api v0.147.0 h1:Can3FaQo9LlVqxJCodNmeZW/ib3/qKAY3rFeXiHo5gc= +google.golang.org/api v0.147.0/go.mod h1:pQ/9j83DcmPd/5C9e2nFOdjjNkDZ1G+zkbK2uvdkJMs= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -705,14 +697,13 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao= -google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0= +google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk= +google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 h1:W18sezcAYs+3tDZX4F80yctqa12jcP1PUS2gQu1zTPU= +google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97/go.mod h1:iargEX0SFPm3xcfMI0d1domjg0ZF4Aa0p2awqyxhvF0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c h1:jHkCUWkseRf+W+edG5hMzr/Uh1xkDREY4caybAq4dpY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -721,12 +712,9 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -739,8 +727,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -756,15 +744,15 @@ gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3M gopkg.in/retry.v1 v1.0.3 h1:a9CArYczAVv6Qs6VGoLMio99GEs7kY9UzSF9+LD+iGs= gopkg.in/retry.v1 v1.0.3/go.mod h1:FJkXmWiMaAo7xB+xhvDF59zhfjDWyzmyAxiT4dB688g= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gorm.io/driver/postgres v1.5.2 h1:ytTDxxEv+MplXOfFe3Lzm7SjG09fcdb3Z/c056DTBx0= +gorm.io/gorm v1.25.2 h1:gs1o6Vsa+oVKG/a9ElL3XgyGfghFfkKA2SInQaCyMho= +gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/sdks/go/OWNERS b/sdks/go/OWNERS deleted file mode 100644 index dafa6adc96880..0000000000000 --- a/sdks/go/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - herohde # emeritus - - lostluck diff --git a/sdks/go/README.md b/sdks/go/README.md index a3b03c2e61841..7734d58d9eb95 100644 --- a/sdks/go/README.md +++ b/sdks/go/README.md @@ -131,6 +131,7 @@ Executing all unit tests for the SDK is possible from the `\sdks\go` To test your change as Jenkins would execute it from a PR, from the beam root directory, run: * `./gradlew :sdks:go:goTest` executes the unit tests. + * `./gradlew :sdks:go:test:prismValidatesRunner` validates the SDK against the Go Prism runner as a stand alone binary, with containers. * `./gradlew :sdks:go:test:ulrValidatesRunner` validates the SDK against the Portable Python runner. * `./gradlew :sdks:go:test:flinkValidatesRunner` validates the SDK against the Flink runner. diff --git a/sdks/go/cmd/prism/prism.go b/sdks/go/cmd/prism/prism.go index 32c6fbadd5296..804ae0c2ab2d0 100644 --- a/sdks/go/cmd/prism/prism.go +++ b/sdks/go/cmd/prism/prism.go @@ -30,19 +30,21 @@ import ( ) var ( + jobPort = flag.Int("job_port", 8073, "specify the job management service port") + webPort = flag.Int("web_port", 8074, "specify the web ui port") jobManagerEndpoint = flag.String("jm_override", "", "set to only stand up a web ui that refers to a seperate JobManagement endpoint") - serveHttp = flag.Bool("serve_http", true, "enable or disable the web ui") + serveHTTP = flag.Bool("serve_http", true, "enable or disable the web ui") ) func main() { flag.Parse() ctx := context.Background() - cli, err := makeJobClient(ctx, *jobManagerEndpoint) + cli, err := makeJobClient(ctx, prism.Options{Port: *jobPort}, *jobManagerEndpoint) if err != nil { log.Fatalf("error creating job server: %v", err) } - if *serveHttp { - if err := prism.CreateWebServer(ctx, cli, prism.Options{Port: 8074}); err != nil { + if *serveHTTP { + if err := prism.CreateWebServer(ctx, cli, prism.Options{Port: *webPort}); err != nil { log.Fatalf("error creating web server: %v", err) } } else { @@ -51,7 +53,7 @@ func main() { } } -func makeJobClient(ctx context.Context, endpoint string) (jobpb.JobServiceClient, error) { +func makeJobClient(ctx context.Context, opts prism.Options, endpoint string) (jobpb.JobServiceClient, error) { if endpoint != "" { clientConn, err := grpc.DialContext(ctx, endpoint, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock()) if err != nil { @@ -59,7 +61,7 @@ func makeJobClient(ctx context.Context, endpoint string) (jobpb.JobServiceClient } return jobpb.NewJobServiceClient(clientConn), nil } - cli, err := prism.CreateJobServer(ctx, prism.Options{Port: 8073}) + cli, err := prism.CreateJobServer(ctx, opts) if err != nil { return nil, fmt.Errorf("error creating local job server: %v", err) } diff --git a/sdks/go/container/build.gradle b/sdks/go/container/build.gradle index 9a9bbb32debbd..8429de133ee7c 100644 --- a/sdks/go/container/build.gradle +++ b/sdks/go/container/build.gradle @@ -27,6 +27,9 @@ goBuild { outputLocation = './build/target/${GOOS}_${GOARCH}/boot' } +def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] +def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers") + docker { name containerImageName( name: project.docker_image_default_repo_prefix + "go_sdk", @@ -38,8 +41,10 @@ docker { files "./build/" buildArgs(['pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) || project.rootProject.hasProperty(["isRelease"])]) - buildx project.containerPlatforms() != [project.nativeArchitecture()] + buildx useBuildx platform(*project.containerPlatforms()) + load useBuildx && !pushContainers + push pushContainers } dockerPrepare.dependsOn tasks.named("goBuild") @@ -63,5 +68,5 @@ if (project.rootProject.hasProperty(["docker-pull-licenses"])) { } task pushAll { - dependsOn ":sdks:go:container:dockerPush" + dependsOn ":sdks:go:container:docker" } \ No newline at end of file diff --git a/sdks/go/container/tools/buffered_logging.go b/sdks/go/container/tools/buffered_logging.go new file mode 100644 index 0000000000000..445d19fabfdc8 --- /dev/null +++ b/sdks/go/container/tools/buffered_logging.go @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tools + +import ( + "context" + "log" + "math" + "os" + "strings" + "time" +) + +const initialLogSize int = 255 + +// BufferedLogger is a wrapper around the FnAPI logging client meant to be used +// in place of stdout and stderr in bootloader subprocesses. Not intended for +// Beam end users. +type BufferedLogger struct { + logger *Logger + builder strings.Builder + logs []string + lastFlush time.Time + flushInterval time.Duration + periodicFlushContext context.Context + now func() time.Time +} + +// NewBufferedLogger returns a new BufferedLogger type by reference. +func NewBufferedLogger(logger *Logger) *BufferedLogger { + return &BufferedLogger{logger: logger, lastFlush: time.Now(), flushInterval: time.Duration(math.MaxInt64), periodicFlushContext: context.Background(), now: time.Now} +} + +// NewBufferedLoggerWithFlushInterval returns a new BufferedLogger type by reference. This type will +// flush logs periodically on Write() calls as well as when Flush*() functions are called. +func NewBufferedLoggerWithFlushInterval(ctx context.Context, logger *Logger, interval time.Duration) *BufferedLogger { + return &BufferedLogger{logger: logger, lastFlush: time.Now(), flushInterval: interval, periodicFlushContext: ctx, now: time.Now} +} + +// Write implements the io.Writer interface, converting input to a string +// and storing it in the BufferedLogger's buffer. If a logger is not provided, +// the output is sent directly to os.Stderr. +func (b *BufferedLogger) Write(p []byte) (int, error) { + if b.logger == nil { + return os.Stderr.Write(p) + } + n, err := b.builder.Write(p) + if b.logs == nil { + b.logs = make([]string, 0, initialLogSize) + } + b.logs = append(b.logs, b.builder.String()) + b.builder.Reset() + if b.now().Sub(b.lastFlush) > b.flushInterval { + b.FlushAtDebug(b.periodicFlushContext) + } + return n, err +} + +// FlushAtError flushes the contents of the buffer to the logging +// service at Error. +func (b *BufferedLogger) FlushAtError(ctx context.Context) { + if b.logger == nil { + return + } + for _, message := range b.logs { + b.logger.Errorf(ctx, message) + } + b.logs = nil + b.lastFlush = time.Now() +} + +// FlushAtDebug flushes the contents of the buffer to the logging +// service at Debug. +func (b *BufferedLogger) FlushAtDebug(ctx context.Context) { + if b.logger == nil { + return + } + for _, message := range b.logs { + b.logger.Printf(ctx, message) + } + b.logs = nil + b.lastFlush = time.Now() +} + +// Prints directly to the logging service. If the logger is nil, prints directly to the +// console. Used for the container pre-build workflow. +func (b *BufferedLogger) Printf(ctx context.Context, format string, args ...any) { + if b.logger == nil { + log.Printf(format, args...) + return + } + b.logger.Printf(ctx, format, args...) +} diff --git a/sdks/go/container/tools/buffered_logging_test.go b/sdks/go/container/tools/buffered_logging_test.go new file mode 100644 index 0000000000000..9f542d2d5ab6b --- /dev/null +++ b/sdks/go/container/tools/buffered_logging_test.go @@ -0,0 +1,241 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tools + +import ( + "context" + "testing" + "time" + + fnpb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/fnexecution_v1" +) + +func TestBufferedLogger(t *testing.T) { + ctx := context.Background() + + t.Run("write", func(t *testing.T) { + catcher := &logCatcher{} + l := &Logger{client: catcher} + bl := NewBufferedLogger(l) + + message := []byte("test message") + n, err := bl.Write(message) + if err != nil { + t.Errorf("got error %v", err) + } + if got, want := n, len(message); got != want { + t.Errorf("got %d bytes written, want %d", got, want) + } + if got, want := bl.logs[0], "test message"; got != want { + t.Errorf("got message %q, want %q", got, want) + } + }) + + t.Run("flush single message", func(t *testing.T) { + catcher := &logCatcher{} + l := &Logger{client: catcher} + bl := NewBufferedLogger(l) + + message := []byte("test message") + n, err := bl.Write(message) + + if err != nil { + t.Errorf("got error %v", err) + } + if got, want := n, len(message); got != want { + t.Errorf("got %d bytes written, want %d", got, want) + } + + bl.FlushAtDebug(ctx) + + received := catcher.msgs[0].GetLogEntries()[0] + + if got, want := received.Message, "test message"; got != want { + t.Errorf("got message %q, want %q", got, want) + } + + if got, want := received.Severity, fnpb.LogEntry_Severity_DEBUG; got != want { + t.Errorf("got severity %v, want %v", got, want) + } + }) + + t.Run("flush multiple messages", func(t *testing.T) { + catcher := &logCatcher{} + l := &Logger{client: catcher} + bl := NewBufferedLogger(l) + + messages := []string{"foo", "bar", "baz"} + + for _, message := range messages { + messBytes := []byte(message) + n, err := bl.Write(messBytes) + + if err != nil { + t.Errorf("got error %v", err) + } + if got, want := n, len(messBytes); got != want { + t.Errorf("got %d bytes written, want %d", got, want) + } + } + + bl.FlushAtDebug(ctx) + + received := catcher.msgs[0].GetLogEntries() + + for i, message := range received { + if got, want := message.Message, messages[i]; got != want { + t.Errorf("got message %q, want %q", got, want) + } + + if got, want := message.Severity, fnpb.LogEntry_Severity_DEBUG; got != want { + t.Errorf("got severity %v, want %v", got, want) + } + } + }) + + t.Run("flush single message at error", func(t *testing.T) { + catcher := &logCatcher{} + l := &Logger{client: catcher} + bl := NewBufferedLogger(l) + + message := []byte("test error") + n, err := bl.Write(message) + + if err != nil { + t.Errorf("got error %v", err) + } + if got, want := n, len(message); got != want { + t.Errorf("got %d bytes written, want %d", got, want) + } + + bl.FlushAtError(ctx) + + received := catcher.msgs[0].GetLogEntries()[0] + + if got, want := received.Message, "test error"; got != want { + t.Errorf("got message %q, want %q", got, want) + } + + if got, want := received.Severity, fnpb.LogEntry_Severity_ERROR; got != want { + t.Errorf("got severity %v, want %v", got, want) + } + }) + + t.Run("flush multiple messages at error", func(t *testing.T) { + catcher := &logCatcher{} + l := &Logger{client: catcher} + bl := NewBufferedLogger(l) + + messages := []string{"foo", "bar", "baz"} + + for _, message := range messages { + messBytes := []byte(message) + n, err := bl.Write(messBytes) + + if err != nil { + t.Errorf("got error %v", err) + } + if got, want := n, len(messBytes); got != want { + t.Errorf("got %d bytes written, want %d", got, want) + } + } + + bl.FlushAtError(ctx) + + received := catcher.msgs[0].GetLogEntries() + + for i, message := range received { + if got, want := message.Message, messages[i]; got != want { + t.Errorf("got message %q, want %q", got, want) + } + + if got, want := message.Severity, fnpb.LogEntry_Severity_ERROR; got != want { + t.Errorf("got severity %v, want %v", got, want) + } + } + }) + + t.Run("direct print", func(t *testing.T) { + catcher := &logCatcher{} + l := &Logger{client: catcher} + bl := NewBufferedLogger(l) + + bl.Printf(ctx, "foo %v", "bar") + + received := catcher.msgs[0].GetLogEntries()[0] + + if got, want := received.Message, "foo bar"; got != want { + t.Errorf("l.Printf(\"foo %%v\", \"bar\"): got message %q, want %q", got, want) + } + + if got, want := received.Severity, fnpb.LogEntry_Severity_DEBUG; got != want { + t.Errorf("l.Printf(\"foo %%v\", \"bar\"): got severity %v, want %v", got, want) + } + }) + + t.Run("debug flush at interval", func(t *testing.T) { + catcher := &logCatcher{} + l := &Logger{client: catcher} + interval := 5 * time.Second + bl := NewBufferedLoggerWithFlushInterval(context.Background(), l, interval) + + startTime := time.Now() + bl.now = func() time.Time { return startTime } + + messages := []string{"foo", "bar"} + + for i, message := range messages { + if i > 1 { + bl.now = func() time.Time { return startTime.Add(6 * time.Second) } + } + messBytes := []byte(message) + n, err := bl.Write(messBytes) + + if err != nil { + t.Errorf("got error %v", err) + } + if got, want := n, len(messBytes); got != want { + t.Errorf("got %d bytes written, want %d", got, want) + } + } + + lastMessage := "baz" + bl.now = func() time.Time { return startTime.Add(6 * time.Second) } + messBytes := []byte(lastMessage) + n, err := bl.Write(messBytes) + + if err != nil { + t.Errorf("got error %v", err) + } + if got, want := n, len(messBytes); got != want { + t.Errorf("got %d bytes written, want %d", got, want) + } + + // Type should have auto-flushed at debug after the third message + received := catcher.msgs[0].GetLogEntries() + messages = append(messages, lastMessage) + + for i, message := range received { + if got, want := message.Message, messages[i]; got != want { + t.Errorf("got message %q, want %q", got, want) + } + + if got, want := message.Severity, fnpb.LogEntry_Severity_DEBUG; got != want { + t.Errorf("got severity %v, want %v", got, want) + } + } + }) +} diff --git a/sdks/go/container/tools/logging.go b/sdks/go/container/tools/logging.go index 012efdb376ba7..ced13b744d454 100644 --- a/sdks/go/container/tools/logging.go +++ b/sdks/go/container/tools/logging.go @@ -111,6 +111,11 @@ func (l *Logger) Warnf(ctx context.Context, format string, args ...any) { l.Log(ctx, fnpb.LogEntry_Severity_WARN, fmt.Sprintf(format, args...)) } +// Errorf logs the message with Error severity. +func (l *Logger) Errorf(ctx context.Context, format string, args ...any) { + l.Log(ctx, fnpb.LogEntry_Severity_ERROR, fmt.Sprintf(format, args...)) +} + // Fatalf logs the message with Critical severity, and then calls os.Exit(1). func (l *Logger) Fatalf(ctx context.Context, format string, args ...any) { l.Log(ctx, fnpb.LogEntry_Severity_CRITICAL, fmt.Sprintf(format, args...)) diff --git a/sdks/go/container/tools/logging_test.go b/sdks/go/container/tools/logging_test.go index e33b3c075a6d2..8730a0fe9c193 100644 --- a/sdks/go/container/tools/logging_test.go +++ b/sdks/go/container/tools/logging_test.go @@ -58,6 +58,22 @@ func TestLogger(t *testing.T) { t.Errorf("l.Printf(\"foo %%v\", \"bar\"): got severity %v, want %v", got, want) } }) + t.Run("SuccessfulLoggingAtError", func(t *testing.T) { + catcher := &logCatcher{} + l := &Logger{client: catcher} + + l.Errorf(ctx, "failed to install dependency %v", "bar") + + received := catcher.msgs[0].GetLogEntries()[0] + + if got, want := received.Message, "failed to install dependency bar"; got != want { + t.Errorf("l.Printf(\"foo %%v\", \"bar\"): got message %q, want %q", got, want) + } + + if got, want := received.Severity, fnpb.LogEntry_Severity_ERROR; got != want { + t.Errorf("l.Errorf(\"failed to install dependency %%v\", \"bar\"): got severity %v, want %v", got, want) + } + }) t.Run("backup path", func(t *testing.T) { catcher := &logCatcher{} l := &Logger{client: catcher} diff --git a/sdks/go/examples/cookbook/filter/filter.go b/sdks/go/examples/cookbook/filter/filter.go index 68d81af98bbf0..56b18390a70b8 100644 --- a/sdks/go/examples/cookbook/filter/filter.go +++ b/sdks/go/examples/cookbook/filter/filter.go @@ -32,7 +32,7 @@ import ( ) var ( - input = flag.String("input", "clouddataflow-readonly:samples.weather_stations", "Weather data BQ table.") + input = flag.String("input", "apache-beam-testing.samples.weather_stations", "Weather data BQ table.") output = flag.String("output", "", "Output BQ table.") month = flag.Int("month_filter", 7, "Numerical month to analyze") ) diff --git a/sdks/go/examples/cookbook/join/join.go b/sdks/go/examples/cookbook/join/join.go index 25c9fb71c07fe..e2e8fb019b80b 100644 --- a/sdks/go/examples/cookbook/join/join.go +++ b/sdks/go/examples/cookbook/join/join.go @@ -34,7 +34,7 @@ import ( // See: https://github.com/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/cookbook/JoinExamples.java const ( - gdeltEventsTable = "clouddataflow-readonly:samples.gdelt_sample" + gdeltEventsTable = "apache-beam-testing.samples.gdelt_sample" countryCodesTable = "gdelt-bq:full.crosswalk_geocountrycodetohuman" ) diff --git a/sdks/go/examples/cookbook/max/max.go b/sdks/go/examples/cookbook/max/max.go index fa6c0e2c53594..89b1ca24400f6 100644 --- a/sdks/go/examples/cookbook/max/max.go +++ b/sdks/go/examples/cookbook/max/max.go @@ -32,7 +32,7 @@ import ( ) var ( - input = flag.String("input", "clouddataflow-readonly:samples.weather_stations", "Weather data BQ table.") + input = flag.String("input", "apache-beam-testing.samples.weather_stations", "Weather data BQ table.") output = flag.String("output", "", "Output BQ table.") ) diff --git a/sdks/go/examples/cookbook/tornadoes/tornadoes.go b/sdks/go/examples/cookbook/tornadoes/tornadoes.go index 1810f63e35356..bd327bfba1217 100644 --- a/sdks/go/examples/cookbook/tornadoes/tornadoes.go +++ b/sdks/go/examples/cookbook/tornadoes/tornadoes.go @@ -29,7 +29,7 @@ // // --output=YOUR_PROJECT_ID:DATASET_ID.TABLE_ID // -// The BigQuery input table defaults to clouddataflow-readonly:samples.weather_stations +// The BigQuery input table defaults to apache-beam-testing.samples.weather_stations // and can be overridden with {@code --input}. package main @@ -48,7 +48,7 @@ import ( ) var ( - input = flag.String("input", "clouddataflow-readonly:samples.weather_stations", "BigQuery table with weather data to read from, specified as :.") + input = flag.String("input", "apache-beam-testing.samples.weather_stations", "BigQuery table with weather data to read from, specified as :.") output = flag.String("output", "", "BigQuery table to write to, specified as :.. The dataset must already exist") ) diff --git a/sdks/go/examples/large_wordcount/large_wordcount.go b/sdks/go/examples/large_wordcount/large_wordcount.go index df04b19a38386..eb9cf3010e750 100644 --- a/sdks/go/examples/large_wordcount/large_wordcount.go +++ b/sdks/go/examples/large_wordcount/large_wordcount.go @@ -73,6 +73,7 @@ import ( _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/direct" _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/dot" _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/flink" + _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism" _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/samza" _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/spark" _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/universal" diff --git a/sdks/go/examples/minimal_wordcount/minimal_wordcount.go b/sdks/go/examples/minimal_wordcount/minimal_wordcount.go index f25f07a96d7b5..f5f22cae1d653 100644 --- a/sdks/go/examples/minimal_wordcount/minimal_wordcount.go +++ b/sdks/go/examples/minimal_wordcount/minimal_wordcount.go @@ -62,7 +62,7 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam" "github.com/apache/beam/sdks/v2/go/pkg/beam/io/textio" - "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/direct" + "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism" "github.com/apache/beam/sdks/v2/go/pkg/beam/transforms/stats" _ "github.com/apache/beam/sdks/v2/go/pkg/beam/io/filesystem/gcs" @@ -119,6 +119,6 @@ func main() { // formatted strings) to a text file. textio.Write(s, "wordcounts.txt", formatted) - // Run the pipeline on the direct runner. - direct.Execute(context.Background(), p) + // Run the pipeline on the prism runner. + prism.Execute(context.Background(), p) } diff --git a/sdks/go/examples/snippets/04transforms.go b/sdks/go/examples/snippets/04transforms.go index bb21e9e317e4c..bb3abe3cf83af 100644 --- a/sdks/go/examples/snippets/04transforms.go +++ b/sdks/go/examples/snippets/04transforms.go @@ -27,6 +27,7 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/window" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/sdf" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/state" + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/timers" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex" "github.com/apache/beam/sdks/v2/go/pkg/beam/io/rtrackers/offsetrange" "github.com/apache/beam/sdks/v2/go/pkg/beam/register" @@ -64,17 +65,17 @@ func applyWordLen(s beam.Scope, words beam.PCollection) beam.PCollection { return wordLengths } +// [START model_pardo_apply_anon] + +func wordLengths(word string) int { return len(word) } +func init() { register.Function1x1(wordLengths) } + func applyWordLenAnon(s beam.Scope, words beam.PCollection) beam.PCollection { - // [START model_pardo_apply_anon] - // Apply an anonymous function as a DoFn PCollection words. - // Save the result as the PCollection wordLengths. - wordLengths := beam.ParDo(s, func(word string) int { - return len(word) - }, words) - // [END model_pardo_apply_anon] - return wordLengths + return beam.ParDo(s, wordLengths, words) } +// [END model_pardo_apply_anon] + func applyGbk(s beam.Scope, input []stringPair) beam.PCollection { // [START groupbykey] // CreateAndSplit creates and returns a PCollection with @@ -344,26 +345,29 @@ func globallyAverage(s beam.Scope, ints beam.PCollection) beam.PCollection { return average } +// [START combine_global_with_default] + +func returnSideOrDefault(d float64, iter func(*float64) bool) float64 { + var c float64 + if iter(&c) { + // Side input has a value, so return it. + return c + } + // Otherwise, return the default + return d +} +func init() { register.Function2x1(returnSideOrDefault) } + func globallyAverageWithDefault(s beam.Scope, ints beam.PCollection) beam.PCollection { - // [START combine_global_with_default] // Setting combine defaults has requires no helper function in the Go SDK. average := beam.Combine(s, &averageFn{}, ints) // To add a default value: defaultValue := beam.Create(s, float64(0)) - avgWithDefault := beam.ParDo(s, func(d float64, iter func(*float64) bool) float64 { - var c float64 - if iter(&c) { - // Side input has a value, so return it. - return c - } - // Otherwise, return the default - return d - }, defaultValue, beam.SideInput{Input: average}) - // [END combine_global_with_default] - return avgWithDefault + return beam.ParDo(s, returnSideOrDefault, defaultValue, beam.SideInput{Input: average}) } +// [END combine_global_with_default] func perKeyAverage(s beam.Scope, playerAccuracies beam.PCollection) beam.PCollection { // [START combine_per_key] avgAccuracyPerPlayer := stats.MeanPerKey(s, playerAccuracies) @@ -548,58 +552,114 @@ func contains(s []string, e string) bool { return false } -// TODO(https://github.com/apache/beam/issues/22737): Update state_and_timers to a good example to demonstrate both state and timers. -// Rename this to bag_state and update the bag state example in the programming guide at that point. // [START state_and_timers] +// stateAndTimersFn is an example stateful DoFn with state and a timer. +type stateAndTimersFn struct { + Buffer1 state.Bag[string] + Buffer2 state.Bag[int64] + Watermark timers.EventTime +} + +func (s *stateAndTimersFn) ProcessElement(sp state.Provider, tp timers.Provider, w beam.Window, key string, value int64, emit func(string, int64)) error { + // ... handle processing elements here, set a callback timer... + + // Read all the data from Buffer1 in this window. + vals, ok, err := s.Buffer1.Read(sp) + if err != nil { + return err + } + if ok && s.shouldClearBuffer(vals) { + // clear the buffer data if required conditions are met. + s.Buffer1.Clear(sp) + } + + // Add the value to Buffer2. + s.Buffer2.Add(sp, value) + + if s.allConditionsMet() { + // Clear the timer if certain condition met and you don't want to trigger + // the callback method. + s.Watermark.Clear(tp) + } + + emit(key, value) + + return nil +} + +func (s *stateAndTimersFn) OnTimer(sp state.Provider, tp timers.Provider, w beam.Window, key string, timer timers.Context, emit func(string, int64)) error { + // Window and key parameters are really useful especially for debugging issues. + switch timer.Family { + case s.Watermark.Family: + // timer expired, emit a different signal + emit(key, -1) + } + return nil +} + +func (s *stateAndTimersFn) shouldClearBuffer([]string) bool { + // some business logic + return false +} + +func (s *stateAndTimersFn) allConditionsMet() bool { + // other business logic + return true +} + +// [END state_and_timers] + +// [START bag_state] + // bagStateFn only emits words that haven't been seen type bagStateFn struct { - bag state.Bag[string] + Bag state.Bag[string] } -func (s *bagStateFn) ProcessElement(p state.Provider, book string, word string, emitWords func(string)) error { +func (s *bagStateFn) ProcessElement(p state.Provider, book, word string, emitWords func(string)) error { // Get all values we've written to this bag state in this window. - vals, ok, err := s.bag.Read(p) + vals, ok, err := s.Bag.Read(p) if err != nil { return err } if !ok || !contains(vals, word) { emitWords(word) - s.bag.Add(p, word) + s.Bag.Add(p, word) } if len(vals) > 10000 { // Example of clearing and starting again with an empty bag - s.bag.Clear(p) + s.Bag.Clear(p) } return nil } -// [END state_and_timers] +// [END bag_state] // [START value_state] // valueStateFn keeps track of the number of elements seen. type valueStateFn struct { - val state.Value[int] + Val state.Value[int] } func (s *valueStateFn) ProcessElement(p state.Provider, book string, word string, emitWords func(string)) error { // Get the value stored in our state - val, ok, err := s.val.Read(p) + val, ok, err := s.Val.Read(p) if err != nil { return err } if !ok { - s.val.Write(p, 1) + s.Val.Write(p, 1) } else { - s.val.Write(p, val+1) + s.Val.Write(p, val+1) } if val > 10000 { // Example of clearing and starting again with an empty bag - s.val.Clear(p) + s.Val.Clear(p) } return nil @@ -620,7 +680,7 @@ func (m MyCustomType) FromBytes(_ []byte) MyCustomType { // [START value_state_coder] type valueStateDoFn struct { - val state.Value[MyCustomType] + Val state.Value[MyCustomType] } func encode(m MyCustomType) []byte { @@ -644,40 +704,422 @@ type combineFn struct{} // combiningStateFn keeps track of the number of elements seen. type combiningStateFn struct { // types are the types of the accumulator, input, and output respectively - val state.Combining[int, int, int] + Val state.Combining[int, int, int] } func (s *combiningStateFn) ProcessElement(p state.Provider, book string, word string, emitWords func(string)) error { // Get the value stored in our state - val, _, err := s.val.Read(p) + val, _, err := s.Val.Read(p) if err != nil { return err } - s.val.Add(p, 1) + s.Val.Add(p, 1) if val > 10000 { // Example of clearing and starting again with an empty bag - s.val.Clear(p) + s.Val.Clear(p) } return nil } -func main() { +func combineState(s beam.Scope, input beam.PCollection) beam.PCollection { // ... // CombineFn param can be a simple fn like this or a structural CombineFn cFn := state.MakeCombiningState[int, int, int]("stateKey", func(a, b int) int { return a + b }) + combined := beam.ParDo(s, combiningStateFn{Val: cFn}, input) + // ... // [END combining_state] - fmt.Print(cFn) + return combined +} + +// [START event_time_timer] + +type eventTimerDoFn struct { + State state.Value[int64] + Timer timers.EventTime +} + +func (fn *eventTimerDoFn) ProcessElement(ts beam.EventTime, sp state.Provider, tp timers.Provider, book, word string, emitWords func(string)) { + // ... + + // Set an event-time timer to the element timestamp. + fn.Timer.Set(tp, ts.ToTime()) + + // ... +} + +func (fn *eventTimerDoFn) OnTimer(sp state.Provider, tp timers.Provider, w beam.Window, key string, timer timers.Context, emitWords func(string)) { + switch timer.Family { + case fn.Timer.Family: + // process callback for this timer + } +} + +func AddEventTimeDoFn(s beam.Scope, in beam.PCollection) beam.PCollection { + return beam.ParDo(s, &eventTimerDoFn{ + // Timers are given family names so their callbacks can be handled independantly. + Timer: timers.InEventTime("processWatermark"), + State: state.MakeValueState[int64]("latest"), + }, in) +} + +// [END event_time_timer] + +// [START processing_time_timer] + +type processingTimerDoFn struct { + Timer timers.ProcessingTime +} + +func (fn *processingTimerDoFn) ProcessElement(sp state.Provider, tp timers.Provider, book, word string, emitWords func(string)) { + // ... + + // Set a timer to go off 30 seconds in the future. + fn.Timer.Set(tp, time.Now().Add(30*time.Second)) + + // ... +} + +func (fn *processingTimerDoFn) OnTimer(sp state.Provider, tp timers.Provider, w beam.Window, key string, timer timers.Context, emitWords func(string)) { + switch timer.Family { + case fn.Timer.Family: + // process callback for this timer + } +} + +func AddProcessingTimeDoFn(s beam.Scope, in beam.PCollection) beam.PCollection { + return beam.ParDo(s, &processingTimerDoFn{ + // Timers are given family names so their callbacks can be handled independantly. + Timer: timers.InProcessingTime("timer"), + }, in) +} + +// [END processing_time_timer] + +// [START dynamic_timer_tags] + +type hasAction interface { + Action() string +} + +type dynamicTagsDoFn[V hasAction] struct { + Timer timers.EventTime +} + +func (fn *dynamicTagsDoFn[V]) ProcessElement(ts beam.EventTime, tp timers.Provider, key string, value V, emitWords func(string)) { + // ... + + // Set a timer to go off 30 seconds in the future. + fn.Timer.Set(tp, ts.ToTime(), timers.WithTag(value.Action())) + + // ... } +func (fn *dynamicTagsDoFn[V]) OnTimer(tp timers.Provider, w beam.Window, key string, timer timers.Context, emitWords func(string)) { + switch timer.Family { + case fn.Timer.Family: + tag := timer.Tag // Do something with fired tag + _ = tag + } +} + +func AddDynamicTimerTagsDoFn[V hasAction](s beam.Scope, in beam.PCollection) beam.PCollection { + return beam.ParDo(s, &dynamicTagsDoFn[V]{ + Timer: timers.InEventTime("actionTimers"), + }, in) +} + +// [END dynamic_timer_tags] + +// [START timer_output_timestamps_bad] + +type badTimerOutputTimestampsFn[V any] struct { + ElementBag state.Bag[V] + TimerSet state.Value[bool] + OutputState timers.ProcessingTime +} + +func (fn *badTimerOutputTimestampsFn[V]) ProcessElement(sp state.Provider, tp timers.Provider, key string, value V, emit func(string)) error { + // Add the current element to the bag for this key. + if err := fn.ElementBag.Add(sp, value); err != nil { + return err + } + set, _, err := fn.TimerSet.Read(sp) + if err != nil { + return err + } + if !set { + fn.OutputState.Set(tp, time.Now().Add(1*time.Minute)) + fn.TimerSet.Write(sp, true) + } + return nil +} + +func (fn *badTimerOutputTimestampsFn[V]) OnTimer(sp state.Provider, tp timers.Provider, w beam.Window, key string, timer timers.Context, emit func(string)) error { + switch timer.Family { + case fn.OutputState.Family: + vs, _, err := fn.ElementBag.Read(sp) + if err != nil { + return err + } + for _, v := range vs { + // Output each element + emit(fmt.Sprintf("%v", v)) + } + + fn.ElementBag.Clear(sp) + // Note that the timer has now fired. + fn.TimerSet.Clear(sp) + } + return nil +} + +// [END timer_output_timestamps_bad] + +// [START timer_output_timestamps_good] + +type element[V any] struct { + Timestamp int64 + Value V +} + +type goodTimerOutputTimestampsFn[V any] struct { + ElementBag state.Bag[element[V]] // The bag of elements accumulated. + TimerTimerstamp state.Value[int64] // The timestamp of the timer set. + MinTimestampInBag state.Combining[int64, int64, int64] // The minimum timestamp stored in the bag. + OutputState timers.ProcessingTime // The timestamp of the timer. +} + +func (fn *goodTimerOutputTimestampsFn[V]) ProcessElement(et beam.EventTime, sp state.Provider, tp timers.Provider, key string, value V, emit func(beam.EventTime, string)) error { + // ... + // Add the current element to the bag for this key, and preserve the event time. + if err := fn.ElementBag.Add(sp, element[V]{Timestamp: et.Milliseconds(), Value: value}); err != nil { + return err + } + + // Keep track of the minimum element timestamp currently stored in the bag. + fn.MinTimestampInBag.Add(sp, et.Milliseconds()) + + // If the timer is already set, then reset it at the same time but with an updated output timestamp (otherwise + // we would keep resetting the timer to the future). If there is no timer set, then set one to expire in a minute. + ts, ok, _ := fn.TimerTimerstamp.Read(sp) + var tsToSet time.Time + if ok { + tsToSet = time.UnixMilli(ts) + } else { + tsToSet = time.Now().Add(1 * time.Minute) + } + + minTs, _, _ := fn.MinTimestampInBag.Read(sp) + outputTs := time.UnixMilli(minTs) + + // Setting the outputTimestamp to the minimum timestamp in the bag holds the watermark to that timestamp until the + // timer fires. This allows outputting all the elements with their timestamp. + fn.OutputState.Set(tp, tsToSet, timers.WithOutputTimestamp(outputTs)) + fn.TimerTimerstamp.Write(sp, tsToSet.UnixMilli()) + + return nil +} + +func (fn *goodTimerOutputTimestampsFn[V]) OnTimer(sp state.Provider, tp timers.Provider, w beam.Window, key string, timer timers.Context, emit func(beam.EventTime, string)) error { + switch timer.Family { + case fn.OutputState.Family: + vs, _, err := fn.ElementBag.Read(sp) + if err != nil { + return err + } + for _, v := range vs { + // Output each element with their timestamp + emit(beam.EventTime(v.Timestamp), fmt.Sprintf("%v", v.Value)) + } + + fn.ElementBag.Clear(sp) + // Note that the timer has now fired. + fn.TimerTimerstamp.Clear(sp) + } + return nil +} + +func AddTimedOutputBatching[V any](s beam.Scope, in beam.PCollection) beam.PCollection { + return beam.ParDo(s, &goodTimerOutputTimestampsFn[V]{ + ElementBag: state.MakeBagState[element[V]]("elementBag"), + TimerTimerstamp: state.MakeValueState[int64]("timerTimestamp"), + MinTimestampInBag: state.MakeCombiningState[int64, int64, int64]("minTimestampInBag", func(a, b int64) int64 { + if a < b { + return a + } + return b + }), + OutputState: timers.InProcessingTime("outputState"), + }, in) +} + +// [END timer_output_timestamps_good] + +// updateState exists for example purposes only +func updateState(sp, state, k, v any) {} + +// [START timer_garbage_collection] + +type timerGarbageCollectionFn[V any] struct { + State state.Value[V] // The state for the key. + MaxTimestampInBag state.Combining[int64, int64, int64] // The maximum element timestamp seen so far. + GcTimer timers.EventTime // The timestamp of the timer. +} + +func (fn *timerGarbageCollectionFn[V]) ProcessElement(et beam.EventTime, sp state.Provider, tp timers.Provider, key string, value V, emit func(beam.EventTime, string)) { + updateState(sp, fn.State, key, value) + fn.MaxTimestampInBag.Add(sp, et.Milliseconds()) + + // Set the timer to be one hour after the maximum timestamp seen. This will keep overwriting the same timer, so + // as long as there is activity on this key the state will stay active. Once the key goes inactive for one hour's + // worth of event time (as measured by the watermark), then the gc timer will fire. + maxTs, _, _ := fn.MaxTimestampInBag.Read(sp) + expirationTime := time.UnixMilli(maxTs).Add(1 * time.Hour) + fn.GcTimer.Set(tp, expirationTime) +} + +func (fn *timerGarbageCollectionFn[V]) OnTimer(sp state.Provider, tp timers.Provider, w beam.Window, key string, timer timers.Context, emit func(beam.EventTime, string)) { + switch timer.Family { + case fn.GcTimer.Family: + // Clear all the state for the key + fn.State.Clear(sp) + fn.MaxTimestampInBag.Clear(sp) + } +} + +func AddTimerGarbageCollection[V any](s beam.Scope, in beam.PCollection) beam.PCollection { + return beam.ParDo(s, &timerGarbageCollectionFn[V]{ + State: state.MakeValueState[V]("timerTimestamp"), + MaxTimestampInBag: state.MakeCombiningState[int64, int64, int64]("maxTimestampInBag", func(a, b int64) int64 { + if a > b { + return a + } + return b + }), + GcTimer: timers.InEventTime("gcTimer"), + }, in) +} + +// [END timer_garbage_collection] + +type Event struct{} + +func (*Event) isClick() bool { return false } + +// [START join_dofn_example] + +type JoinedEvent struct { + View, Click *Event +} + +type joinDoFn struct { + View state.Value[*Event] // Store the view event. + Click state.Value[*Event] // Store the click event. + + MaxTimestampSeen state.Combining[int64, int64, int64] // The maximum element timestamp seen so far. + GcTimer timers.EventTime // The timestamp of the timer. +} + +func (fn *joinDoFn) ProcessElement(et beam.EventTime, sp state.Provider, tp timers.Provider, key string, event *Event, emit func(JoinedEvent)) { + valueState := fn.View + if event.isClick() { + valueState = fn.Click + } + valueState.Write(sp, event) + + view, _, _ := fn.View.Read(sp) + click, _, _ := fn.Click.Read(sp) + if view != nil && click != nil { + emit(JoinedEvent{View: view, Click: click}) + fn.clearState(sp) + return + } + + fn.MaxTimestampSeen.Add(sp, et.Milliseconds()) + expTs, _, _ := fn.MaxTimestampSeen.Read(sp) + fn.GcTimer.Set(tp, time.UnixMilli(expTs).Add(1*time.Hour)) +} + +func (fn *joinDoFn) OnTimer(sp state.Provider, tp timers.Provider, w beam.Window, key string, timer timers.Context, emit func(beam.EventTime, string)) { + switch timer.Family { + case fn.GcTimer.Family: + fn.clearState(sp) + } +} + +func (fn *joinDoFn) clearState(sp state.Provider) { + fn.View.Clear(sp) + fn.Click.Clear(sp) + fn.MaxTimestampSeen.Clear(sp) +} + +func AddJoinDoFn(s beam.Scope, in beam.PCollection) beam.PCollection { + return beam.ParDo(s, &joinDoFn{ + View: state.MakeValueState[*Event]("view"), + Click: state.MakeValueState[*Event]("click"), + MaxTimestampSeen: state.MakeCombiningState[int64, int64, int64]("maxTimestampSeen", func(a, b int64) int64 { + if a > b { + return a + } + return b + }), + GcTimer: timers.InEventTime("gcTimer"), + }, in) +} + +// [END join_dofn_example] + +func sendRpc(...any) {} + +// [START batching_dofn_example] + +type bufferDoFn[V any] struct { + Elements state.Bag[V] // Store the elements buffered so far. + IsTimerSet state.Value[bool] // Keep track of whether a timer is currently set or not. + + OutputElements timers.ProcessingTime // The processing-time timer user to publish the RPC. +} + +func (fn *bufferDoFn[V]) ProcessElement(et beam.EventTime, sp state.Provider, tp timers.Provider, key string, value V) { + fn.Elements.Add(sp, value) + + isSet, _, _ := fn.IsTimerSet.Read(sp) + if !isSet { + fn.OutputElements.Set(tp, time.Now().Add(10*time.Second)) + fn.IsTimerSet.Write(sp, true) + } +} + +func (fn *bufferDoFn[V]) OnTimer(sp state.Provider, tp timers.Provider, w beam.Window, key string, timer timers.Context) { + switch timer.Family { + case fn.OutputElements.Family: + elements, _, _ := fn.Elements.Read(sp) + sendRpc(elements) + fn.Elements.Clear(sp) + fn.IsTimerSet.Clear(sp) + } +} + +func AddBufferDoFn[V any](s beam.Scope, in beam.PCollection) beam.PCollection { + return beam.ParDo(s, &bufferDoFn[V]{ + Elements: state.MakeBagState[V]("elements"), + IsTimerSet: state.MakeValueState[bool]("isTimerSet"), + + OutputElements: timers.InProcessingTime("outputElements"), + }, in) +} + +// [END batching_dofn_example] + type statefulDoFn struct { - s state.Value[int] + S state.Value[int] } func statefulPipeline() beam.PCollection { @@ -686,7 +1128,9 @@ func statefulPipeline() beam.PCollection { // [START windowed_state] - items := beam.ParDo(s, statefulDoFn{}, elements) + items := beam.ParDo(s, statefulDoFn{ + S: state.MakeValueState[int]("S"), + }, elements) out := beam.WindowInto(s, window.NewFixedWindows(24*time.Hour), items) // [END windowed_state] diff --git a/sdks/go/examples/snippets/04transforms_test.go b/sdks/go/examples/snippets/04transforms_test.go index 8d888e028562b..509da6d5065ae 100644 --- a/sdks/go/examples/snippets/04transforms_test.go +++ b/sdks/go/examples/snippets/04transforms_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/passert" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest" ) @@ -205,6 +206,14 @@ func TestSideInputs(t *testing.T) { ptest.RunAndValidate(t, p) } +func emitOnTestKey(k string, v int, emit func(int)) { + if k == "test" { + emit(v) + } +} + +func init() { register.Function3x0(emitOnTestKey) } + func TestComposite(t *testing.T) { p, s, lines := ptest.CreateList([]string{ "this test dataset has the word test", @@ -215,11 +224,7 @@ func TestComposite(t *testing.T) { // A Composite PTransform function is called like any other function. wordCounts := CountWords(s, lines) // returns a PCollection> // [END countwords_composite_call] - testCount := beam.ParDo(s, func(k string, v int, emit func(int)) { - if k == "test" { - emit(v) - } - }, wordCounts) + testCount := beam.ParDo(s, emitOnTestKey, wordCounts) passert.Equals(s, testCount, 4) ptest.RunAndValidate(t, p) } diff --git a/sdks/go/examples/snippets/10metrics.go b/sdks/go/examples/snippets/10metrics.go index 34d8b113d7d8c..c69a03c444d5e 100644 --- a/sdks/go/examples/snippets/10metrics.go +++ b/sdks/go/examples/snippets/10metrics.go @@ -34,7 +34,7 @@ func queryMetrics(pr beam.PipelineResult, ns, n string) metrics.QueryResults { // [END metrics_query] -var runner = "direct" +var runner = "prism" // [START metrics_pipeline] diff --git a/sdks/go/pkg/beam/beam.shims.go b/sdks/go/pkg/beam/beam.shims.go index 6653fb0129f7d..29ebaf2ca6811 100644 --- a/sdks/go/pkg/beam/beam.shims.go +++ b/sdks/go/pkg/beam/beam.shims.go @@ -25,7 +25,6 @@ import ( // Library imports "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/exec" - "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/graphx/schema" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/sdf" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/util/reflectx" @@ -44,13 +43,6 @@ func init() { runtime.RegisterFunction(schemaDec) runtime.RegisterFunction(schemaEnc) runtime.RegisterFunction(swapKVFn) - runtime.RegisterType(reflect.TypeOf((*createFn)(nil)).Elem()) - schema.RegisterType(reflect.TypeOf((*createFn)(nil)).Elem()) - runtime.RegisterType(reflect.TypeOf((*reflect.Type)(nil)).Elem()) - schema.RegisterType(reflect.TypeOf((*reflect.Type)(nil)).Elem()) - runtime.RegisterType(reflect.TypeOf((*reflectx.Func)(nil)).Elem()) - schema.RegisterType(reflect.TypeOf((*reflectx.Func)(nil)).Elem()) - reflectx.RegisterStructWrapper(reflect.TypeOf((*createFn)(nil)).Elem(), wrapMakerCreateFn) reflectx.RegisterFunc(reflect.TypeOf((*func(reflect.Type, []byte) (typex.T, error))(nil)).Elem(), funcMakerReflect۰TypeSliceOfByteГTypex۰TError) reflectx.RegisterFunc(reflect.TypeOf((*func(reflect.Type, typex.T) ([]byte, error))(nil)).Elem(), funcMakerReflect۰TypeTypex۰TГSliceOfByteError) reflectx.RegisterFunc(reflect.TypeOf((*func([]byte, func(typex.T)) error)(nil)).Elem(), funcMakerSliceOfByteEmitTypex۰TГError) @@ -64,13 +56,6 @@ func init() { exec.RegisterEmitter(reflect.TypeOf((*func(typex.T))(nil)).Elem(), emitMakerTypex۰T) } -func wrapMakerCreateFn(fn any) map[string]reflectx.Func { - dfn := fn.(*createFn) - return map[string]reflectx.Func{ - "ProcessElement": reflectx.MakeFunc(func(a0 []byte, a1 func(typex.T)) error { return dfn.ProcessElement(a0, a1) }), - } -} - type callerReflect۰TypeSliceOfByteГTypex۰TError struct { fn func(reflect.Type, []byte) (typex.T, error) } diff --git a/sdks/go/pkg/beam/core/core.go b/sdks/go/pkg/beam/core/core.go index f401ffd74165a..ed62a2e9eac01 100644 --- a/sdks/go/pkg/beam/core/core.go +++ b/sdks/go/pkg/beam/core/core.go @@ -27,7 +27,7 @@ const ( // SdkName is the human readable name of the SDK for UserAgents. SdkName = "Apache Beam SDK for Go" // SdkVersion is the current version of the SDK. - SdkVersion = "2.49.0.dev" + SdkVersion = "2.52.0.dev" // DefaultDockerImage represents the associated image for this release. DefaultDockerImage = "apache/beam_go_sdk:" + SdkVersion diff --git a/sdks/go/pkg/beam/core/graph/fn.go b/sdks/go/pkg/beam/core/graph/fn.go index e316645cd6a88..80f647abf5e62 100644 --- a/sdks/go/pkg/beam/core/graph/fn.go +++ b/sdks/go/pkg/beam/core/graph/fn.go @@ -233,7 +233,7 @@ func init() { } } -// lifecycleMethodName returns if the passed in string is one of the lifecycle +// IsLifecycleMethod return true if the passed in string is one of the lifecycle // method names used by the Go SDK as DoFn or CombineFn lifecycle methods. These // are the only methods that need shims generated for them. func IsLifecycleMethod(n string) bool { diff --git a/sdks/go/pkg/beam/core/runtime/exec/datasource.go b/sdks/go/pkg/beam/core/runtime/exec/datasource.go index 4c31929491397..401cdbef7a374 100644 --- a/sdks/go/pkg/beam/core/runtime/exec/datasource.go +++ b/sdks/go/pkg/beam/core/runtime/exec/datasource.go @@ -83,8 +83,7 @@ func (n *DataSource) ID() UnitID { // Up initializes this datasource. func (n *DataSource) Up(ctx context.Context) error { - // TODO(https://github.com/apache/beam/issues/23043) - Reenable single iteration or more fully rip this out. - safeToSingleIterate := false + safeToSingleIterate := true switch n.Out.(type) { case *Expand, *Multiplex: // CoGBK Expands aren't safe, as they may re-iterate the GBK stream. diff --git a/sdks/go/pkg/beam/core/runtime/exec/fullvalue.go b/sdks/go/pkg/beam/core/runtime/exec/fullvalue.go index 16bc228944ea1..0a9343199a1ca 100644 --- a/sdks/go/pkg/beam/core/runtime/exec/fullvalue.go +++ b/sdks/go/pkg/beam/core/runtime/exec/fullvalue.go @@ -251,6 +251,9 @@ func (s *decodeStream) Read() (*FullValue, error) { } err := s.d.DecodeTo(s.r, &s.ret) if err != nil { + if err == io.EOF { + return nil, io.EOF + } return nil, errors.Wrap(err, "decodeStream value decode failed") } s.next++ @@ -296,12 +299,20 @@ func (s *decodeMultiChunkStream) Close() error { // TODO(https://github.com/apache/beam/issues/22901): // Optimize the case where we have length prefixed values // so we can avoid allocating the values in the first place. - for s.next < s.chunk { - err := s.d.DecodeTo(s.r, &s.ret) - if err != nil { - return errors.Wrap(err, "decodeStream value decode failed on close") + + for { + // If we have a stream, we're finished with the available bytes from the reader, + // so we move to close it after this loop. + if s.stream != nil { + break + } + // Drain the whole available iterable to ensure the reader is in the right position. + _, err := s.Read() + if err == io.EOF { + break + } else if err != nil { + return err } - s.next++ } if s.stream != nil { s.stream.Close() @@ -334,6 +345,9 @@ func (s *decodeMultiChunkStream) Read() (*FullValue, error) { if s.chunk == 0 && s.next == 0 { chunk, err := coder.DecodeVarInt(s.r.reader) if err != nil { + if err == io.EOF { + return nil, io.EOF + } return nil, errors.Wrap(err, "decodeMultiChunkStream chunk size decoding failed") } s.chunk = chunk diff --git a/sdks/go/pkg/beam/core/runtime/exec/fullvalue_test.go b/sdks/go/pkg/beam/core/runtime/exec/fullvalue_test.go index d1482d59d160a..f8fc002d94662 100644 --- a/sdks/go/pkg/beam/core/runtime/exec/fullvalue_test.go +++ b/sdks/go/pkg/beam/core/runtime/exec/fullvalue_test.go @@ -397,11 +397,14 @@ func TestDecodeMultiChunkStream(t *testing.T) { if fv, err := ds.Read(); err != io.EOF { t.Errorf("unexpected error on decodeStream.Read after close: %v, %v", fv, err) } - // Check that next was iterated to equal size + // Check that next was iterated to an empty stream. dds := ds.(*decodeMultiChunkStream) - if got, want := dds.next, int64(size); got != want { + if got, want := dds.next, int64(0); got != want { t.Errorf("unexpected configuration after decodeStream.Close: got %v, want %v", got, want) } + if dds.stream != nil { + t.Errorf("got non-nil stream after close: %#v", dds.stream) + } // Check that a 2nd stream will fail: if s, err := drs.Open(); err == nil || s != nil { diff --git a/sdks/go/pkg/beam/core/runtime/exec/pardo.go b/sdks/go/pkg/beam/core/runtime/exec/pardo.go index 212ff53b6dd84..b93835264507d 100644 --- a/sdks/go/pkg/beam/core/runtime/exec/pardo.go +++ b/sdks/go/pkg/beam/core/runtime/exec/pardo.go @@ -552,5 +552,5 @@ func (n *ParDo) fail(err error) error { } func (n *ParDo) String() string { - return fmt.Sprintf("ParDo[%v] Out:%v Sig: %v", path.Base(n.Fn.Name()), IDs(n.Out...), n.Fn.ProcessElementFn().Fn.Type()) + return fmt.Sprintf("ParDo[%v] Out:%v Sig: %v, SideInputs: %v", path.Base(n.Fn.Name()), IDs(n.Out...), n.Fn.ProcessElementFn().Fn.Type(), n.Side) } diff --git a/sdks/go/pkg/beam/core/runtime/exec/plan.go b/sdks/go/pkg/beam/core/runtime/exec/plan.go index 77063ce18df82..8c27191b35ab5 100644 --- a/sdks/go/pkg/beam/core/runtime/exec/plan.go +++ b/sdks/go/pkg/beam/core/runtime/exec/plan.go @@ -233,7 +233,8 @@ func (p *Plan) Down(ctx context.Context) error { func (p *Plan) String() string { var units []string - for _, u := range p.units { + for i := len(p.units) - 1; i >= 0; i-- { + u := p.units[i] units = append(units, fmt.Sprintf("%v: %v", u.ID(), u)) } return fmt.Sprintf("Plan[%v]:\n%v", p.ID(), strings.Join(units, "\n")) diff --git a/sdks/go/pkg/beam/core/runtime/exec/sdf.go b/sdks/go/pkg/beam/core/runtime/exec/sdf.go index 6482bfb3a6ae7..b21b47b20ae2b 100644 --- a/sdks/go/pkg/beam/core/runtime/exec/sdf.go +++ b/sdks/go/pkg/beam/core/runtime/exec/sdf.go @@ -661,10 +661,8 @@ type SplittableUnit interface { // the singleWindowSplit and multiWindowSplit methods. func (n *ProcessSizedElementsAndRestrictions) Split(ctx context.Context, f float64) ([]*FullValue, []*FullValue, error) { // Get the watermark state immediately so that we don't overestimate our current watermark. - var pWeState any - var rWeState any - rWeState = n.wesInv.Invoke(n.PDo.we) - pWeState = rWeState + rWeState := n.wesInv.Invoke(n.PDo.we) + pWeState := rWeState // If we've processed elements, the initial watermark estimator state will be set. // In that case we should hold the output watermark at that initial state so that we don't // Advance past where the current elements are holding the watermark diff --git a/sdks/go/pkg/beam/core/runtime/exec/sideinput.go b/sdks/go/pkg/beam/core/runtime/exec/sideinput.go index 1af4e71689b13..c3ceeee5d8b83 100644 --- a/sdks/go/pkg/beam/core/runtime/exec/sideinput.go +++ b/sdks/go/pkg/beam/core/runtime/exec/sideinput.go @@ -140,7 +140,7 @@ func (s *sideInputAdapter) NewKeyedIterable(ctx context.Context, reader StateRea } func (s *sideInputAdapter) String() string { - return fmt.Sprintf("SideInputAdapter[%v, %v]", s.sid, s.sideInputID) + return fmt.Sprintf("SideInputAdapter[%v, %v] - Coder %v", s.sid, s.sideInputID, s.c) } // proxyReStream is a simple wrapper of an open function. diff --git a/sdks/go/pkg/beam/core/runtime/exec/to_string.go b/sdks/go/pkg/beam/core/runtime/exec/to_string.go new file mode 100644 index 0000000000000..2196fd9518066 --- /dev/null +++ b/sdks/go/pkg/beam/core/runtime/exec/to_string.go @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package exec + +import ( + "context" + "fmt" +) + +type ToString struct { + // UID is the unit identifier. + UID UnitID + // Out is the output node. + Out Node +} + +func (m *ToString) ID() UnitID { + return m.UID +} + +func (m *ToString) Up(ctx context.Context) error { + return nil +} + +func (m *ToString) StartBundle(ctx context.Context, id string, data DataContext) error { + return m.Out.StartBundle(ctx, id, data) +} + +func (m *ToString) ProcessElement(ctx context.Context, elm *FullValue, values ...ReStream) error { + ret := FullValue{ + Windows: elm.Windows, + Elm: elm.Elm, + Elm2: fmt.Sprintf("%v", elm.Elm2), + Timestamp: elm.Timestamp, + } + + return m.Out.ProcessElement(ctx, &ret, values...) +} + +func (m *ToString) FinishBundle(ctx context.Context) error { + return m.Out.FinishBundle(ctx) +} + +func (m *ToString) Down(ctx context.Context) error { + return nil +} + +func (m *ToString) String() string { + return fmt.Sprintf("ToStringFn. Out:%v", m.Out) +} diff --git a/sdks/go/pkg/beam/core/runtime/exec/to_string_test.go b/sdks/go/pkg/beam/core/runtime/exec/to_string_test.go new file mode 100644 index 0000000000000..7f9942a43bd58 --- /dev/null +++ b/sdks/go/pkg/beam/core/runtime/exec/to_string_test.go @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package exec + +import ( + "context" + "testing" +) + +var toStringTestCases = []struct { + Input []any + Expected []any +}{ + {Input: strInput, Expected: strInput}, + {Input: intInput, Expected: strInput}, + {Input: int64Input, Expected: strInput}, +} + +func TestToString(t *testing.T) { + for _, testCase := range toStringTestCases { + ctx := context.Background() + out := &CaptureNode{UID: 1} + toString := &ToString{UID: 2, Out: out} + a := &FixedRoot{UID: 3, Elements: makeKVInput("key", testCase.Input...), Out: toString} + + p, err := NewPlan("a", []Unit{a, toString, out}) + if err != nil { + t.Fatalf("failed to construct plan: %v", err) + } + + if err := p.Execute(ctx, "1", DataContext{}); err != nil { + t.Fatalf("execute failed: %v", err) + } + + if err := p.Down(ctx); err != nil { + t.Fatalf("down failed: %v", err) + } + + expected := makeKVValues("key", testCase.Expected...) + if !equalList(out.Elements, expected) { + t.Errorf("tostring returned %v, want %v", extractKeyedValues(out.Elements...), extractKeyedValues(expected...)) + } + } +} diff --git a/sdks/go/pkg/beam/core/runtime/exec/translate.go b/sdks/go/pkg/beam/core/runtime/exec/translate.go index 65827d0583871..4f078092a310f 100644 --- a/sdks/go/pkg/beam/core/runtime/exec/translate.go +++ b/sdks/go/pkg/beam/core/runtime/exec/translate.go @@ -193,7 +193,11 @@ func newBuilder(desc *fnpb.ProcessBundleDescriptor) (*builder, error) { input := unmarshalKeyedValues(transform.GetInputs()) for i, from := range input { - succ[from] = append(succ[from], linkID{id, i}) + // We don't need to multiplex successors for pardo side inputs. + // so we only do so for SDK side Flattens. + if i == 0 || transform.GetSpec().GetUrn() == graphx.URNFlatten { + succ[from] = append(succ[from], linkID{id, i}) + } } output := unmarshalKeyedValues(transform.GetOutputs()) for _, to := range output { @@ -731,7 +735,10 @@ func (b *builder) makeLink(from string, id linkID) (Node, error) { } // Strip PCollections from Expand nodes, as CoGBK metrics are handled by // the DataSource that preceeds them. - trueOut := out[0].(*PCollection).Out + trueOut := out[0] + if pcol, ok := trueOut.(*PCollection); ok { + trueOut = pcol.Out + } b.units = b.units[:len(b.units)-1] u = &Expand{UID: b.idgen.New(), ValueDecoders: decoders, Out: trueOut} @@ -816,6 +823,9 @@ func (b *builder) makeLink(from string, id linkID) (Node, error) { } u = sink + case graphx.URNToString: + u = &ToString{UID: b.idgen.New(), Out: out[0]} + default: panic(fmt.Sprintf("Unexpected transform URN: %v", urn)) } diff --git a/sdks/go/pkg/beam/core/runtime/graphx/translate.go b/sdks/go/pkg/beam/core/runtime/graphx/translate.go index ef46fe1e43da5..ad76703e3001c 100644 --- a/sdks/go/pkg/beam/core/runtime/graphx/translate.go +++ b/sdks/go/pkg/beam/core/runtime/graphx/translate.go @@ -47,6 +47,7 @@ const ( URNCombinePerKey = "beam:transform:combine_per_key:v1" URNWindow = "beam:transform:window_into:v1" URNMapWindows = "beam:transform:map_windows:v1" + URNToString = "beam:transform:to_string:v1" URNIterableSideInput = "beam:side_input:iterable:v1" URNMultimapSideInput = "beam:side_input:multimap:v1" @@ -106,6 +107,7 @@ func goCapabilities() []string { URNWorkerStatus, URNMonitoringInfoShortID, URNBaseVersionGo, + URNToString, } return append(capabilities, knownStandardCoders()...) } diff --git a/sdks/go/pkg/beam/core/runtime/harness/datamgr.go b/sdks/go/pkg/beam/core/runtime/harness/datamgr.go index d8c0f4d1d8527..ed57e3eca59b3 100644 --- a/sdks/go/pkg/beam/core/runtime/harness/datamgr.go +++ b/sdks/go/pkg/beam/core/runtime/harness/datamgr.go @@ -27,6 +27,8 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam/internal/errors" "github.com/apache/beam/sdks/v2/go/pkg/beam/log" fnpb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/fnexecution_v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const ( @@ -128,7 +130,12 @@ func (m *DataChannelManager) Open(ctx context.Context, port exec.Port) (*DataCha return nil, err } ch.forceRecreate = func(id string, err error) { - log.Warnf(ctx, "forcing DataChannel[%v] reconnection on port %v due to %v", id, port, err) + switch status.Code(err) { + case codes.Canceled: + // Don't log on context canceled path. + default: + log.Warnf(ctx, "forcing DataChannel[%v] reconnection on port %v due to %v", id, port, err) + } m.mu.Lock() delete(m.ports, port.URL) m.mu.Unlock() @@ -371,7 +378,8 @@ func (c *DataChannel) read(ctx context.Context) { c.terminateStreamOnError(err) c.mu.Unlock() - if err == io.EOF { + st := status.Code(err) + if st == codes.Canceled || err == io.EOF { return } log.Errorf(ctx, "DataChannel.read %v bad: %v", c.id, err) @@ -672,8 +680,7 @@ func (w *timerWriter) Close() error { w.ch.mu.Lock() defer w.ch.mu.Unlock() delete(w.ch.timerWriters[w.id.instID], timerKey{w.id.ptransformID, w.timerFamilyID}) - var msg *fnpb.Elements - msg = &fnpb.Elements{ + msg := &fnpb.Elements{ Timers: []*fnpb.Elements_Timers{ { InstructionId: string(w.id.instID), diff --git a/sdks/go/pkg/beam/core/runtime/harness/harness.go b/sdks/go/pkg/beam/core/runtime/harness/harness.go index 3f0e82c8265ff..c5db9a85f3677 100644 --- a/sdks/go/pkg/beam/core/runtime/harness/harness.go +++ b/sdks/go/pkg/beam/core/runtime/harness/harness.go @@ -393,7 +393,8 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe c.mu.Unlock() if err != nil { - return fail(ctx, instID, "Failed: %v", err) + c.failed[instID] = err + return fail(ctx, instID, "ProcessBundle failed: %v", err) } tokens := msg.GetCacheTokens() @@ -427,6 +428,7 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe // If there was an error on the data channel reads, fail this bundle // since we may have had a short read. c.failed[instID] = dataError + err = dataError } else { // Non failure plans should either be moved to the finalized state // or to plans so they can be re-used. @@ -706,6 +708,6 @@ func fail(ctx context.Context, id instructionID, format string, args ...any) *fn // dial to the specified endpoint. if timeout <=0, call blocks until // grpc.Dial succeeds. func dial(ctx context.Context, endpoint, purpose string, timeout time.Duration) (*grpc.ClientConn, error) { - log.Infof(ctx, "Connecting via grpc @ %s for %s ...", endpoint, purpose) + log.Output(ctx, log.SevDebug, 1, fmt.Sprintf("Connecting via grpc @ %s for %s ...", endpoint, purpose)) return grpcx.Dial(ctx, endpoint, timeout) } diff --git a/sdks/go/pkg/beam/core/runtime/harness/statemgr.go b/sdks/go/pkg/beam/core/runtime/harness/statemgr.go index f10f0d92e84ea..76d4e1f32c23a 100644 --- a/sdks/go/pkg/beam/core/runtime/harness/statemgr.go +++ b/sdks/go/pkg/beam/core/runtime/harness/statemgr.go @@ -29,6 +29,8 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam/log" fnpb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/fnexecution_v1" "github.com/golang/protobuf/proto" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type writeTypeEnum int32 @@ -525,7 +527,12 @@ func (m *StateChannelManager) Open(ctx context.Context, port exec.Port) (*StateC return nil, err } ch.forceRecreate = func(id string, err error) { - log.Warnf(ctx, "forcing StateChannel[%v] reconnection on port %v due to %v", id, port, err) + switch status.Code(err) { + case codes.Canceled: + // Don't log on context canceled path. + default: + log.Warnf(ctx, "forcing StateChannel[%v] reconnection on port %v due to %v", id, port, err) + } m.mu.Lock() delete(m.ports, port.URL) m.mu.Unlock() diff --git a/sdks/go/pkg/beam/core/runtime/metricsx/metricsx.go b/sdks/go/pkg/beam/core/runtime/metricsx/metricsx.go index 4a872e291c6ad..c71ead2083640 100644 --- a/sdks/go/pkg/beam/core/runtime/metricsx/metricsx.go +++ b/sdks/go/pkg/beam/core/runtime/metricsx/metricsx.go @@ -24,6 +24,7 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/coder" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/metrics" pipepb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/pipeline_v1" + "golang.org/x/exp/slog" ) // FromMonitoringInfos extracts metrics from monitored states and @@ -139,7 +140,7 @@ func groupByType(p *pipepb.Pipeline, minfos []*pipepb.MonitoringInfo) ( } } if len(errs) > 0 { - log.Printf("Warning: %v errors during metrics processing: %v\n", len(errs), errs) + slog.Debug("errors during metrics processing", "count", len(errs), "errors", errs) } return counters, distributions, gauges, msecs, pcols } diff --git a/sdks/go/pkg/beam/core/runtime/symbols.go b/sdks/go/pkg/beam/core/runtime/symbols.go index e8ff532e76373..84afe9b769af3 100644 --- a/sdks/go/pkg/beam/core/runtime/symbols.go +++ b/sdks/go/pkg/beam/core/runtime/symbols.go @@ -105,5 +105,5 @@ func ResolveFunction(name string, t reflect.Type) (any, error) { type failResolver bool func (p failResolver) Sym2Addr(name string) (uintptr, error) { - return 0, errors.Errorf("%v not found. Use runtime.RegisterFunction in unit tests", name) + return 0, errors.Errorf("%v not found. Register DoFns and functions with the the beam/register package.", name) } diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/expansionx/download.go b/sdks/go/pkg/beam/core/runtime/xlangx/expansionx/download.go index af3b495720b72..e5fff10396758 100644 --- a/sdks/go/pkg/beam/core/runtime/xlangx/expansionx/download.go +++ b/sdks/go/pkg/beam/core/runtime/xlangx/expansionx/download.go @@ -382,20 +382,29 @@ func jarExists(jarPath string) bool { return err == nil } -func getPythonVersion() (string, error) { +// GetPythonVersion returns the Python version to use. It checks for +// env variable PYTHON_PATH and returns that it if set. +// If no PYTHON_PATH is defined then it checks for python or python3 +// and returns that. Otherwise it returns an error. +func GetPythonVersion() (string, error) { + if pythonPath := os.Getenv("PYTHON_PATH"); pythonPath != "" { + return pythonPath, nil + } for _, v := range []string{"python", "python3"} { cmd := exec.Command(v, "--version") if err := cmd.Run(); err == nil { return v, nil } } - return "", fmt.Errorf("no python installation found") + return "", errors.New("no python installation found. If you use a " + + "custom container image, please check if python/python3 is available or specify the " + + "full path to the python interpreter in PYTHON_PATH environment variable") } // SetUpPythonEnvironment sets up the virtual ennvironment required for the // Apache Beam Python SDK to run an expansion service module. func SetUpPythonEnvironment(extraPackage string) (string, error) { - py, err := getPythonVersion() + py, err := GetPythonVersion() if err != nil { return "", fmt.Errorf("no python installation found: %v", err) } diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/expansionx/download_test.go b/sdks/go/pkg/beam/core/runtime/xlangx/expansionx/download_test.go index 9779c2361899f..65e72342a9b9f 100644 --- a/sdks/go/pkg/beam/core/runtime/xlangx/expansionx/download_test.go +++ b/sdks/go/pkg/beam/core/runtime/xlangx/expansionx/download_test.go @@ -217,3 +217,35 @@ func TestGetJar_dev(t *testing.T) { t.Errorf("error message does not contain gradle command %v for user, got message: %v", gradleTarget, err) } } + +func TestGetPythonVersion(t *testing.T) { + tests := []struct { + name string + PYTHON_PATH string + }{ + { + name: "PYTHON_PATH set", + PYTHON_PATH: "/bin/python", + }, + { + name: "PYTHON_PATH not set", + PYTHON_PATH: "", + }, + } + + for _, test := range tests { + if test.PYTHON_PATH != "" { + os.Setenv("PYTHON_PATH", test.PYTHON_PATH) + } + pythonVersion, err := GetPythonVersion() + if err != nil { + t.Errorf("python installation not found: %v, when PYTHON_PATH=%v", err, test.PYTHON_PATH) + } + if test.PYTHON_PATH != "" && pythonVersion != test.PYTHON_PATH { + t.Errorf("incorrect PYTHON_PATH, want: %v, got: %v", test.PYTHON_PATH, pythonVersion) + } + if test.PYTHON_PATH != "" { + os.Unsetenv(test.PYTHON_PATH) + } + } +} diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/payload.go b/sdks/go/pkg/beam/core/runtime/xlangx/payload.go index 959b3f659fc2a..8c01ffbd34ffa 100644 --- a/sdks/go/pkg/beam/core/runtime/xlangx/payload.go +++ b/sdks/go/pkg/beam/core/runtime/xlangx/payload.go @@ -26,30 +26,29 @@ import ( "google.golang.org/protobuf/proto" ) -// EncodeStructPayload takes a native Go struct and returns a marshaled -// ExternalConfigurationPayload proto, containing a Schema representation of -// the original type and the original value encoded as a Row. This is intended -// to be used as the expansion payload for an External transform. -func EncodeStructPayload(pl any) ([]byte, error) { +// CreateExternalConfigurationPayload takes a native Go struct and returns an +// ExternalConfigurationPayload proto with the struct encoded as a Row and its +// associated schema. +func CreateExternalConfigurationPayload(pl any) (*pipepb.ExternalConfigurationPayload, error) { rt := reflect.TypeOf(pl) // Encode payload value as a Row. enc, err := coder.RowEncoderForStruct(rt) if err != nil { err = errors.WithContext(err, "creating Row encoder for payload") - return []byte{}, errors.WithContextf(err, "encoding external payload %v", pl) + return nil, errors.WithContextf(err, "encoding external payload %v", pl) } var buf bytes.Buffer if err := enc(pl, &buf); err != nil { err = errors.WithContext(err, "encoding payload as Row") - return []byte{}, errors.WithContextf(err, "encoding external payload %v", pl) + return nil, errors.WithContextf(err, "encoding external payload %v", pl) } // Convert payload type into Schema representation. scm, err := schema.FromType(rt) if err != nil { err = errors.WithContext(err, "creating schema for payload") - return []byte{}, errors.WithContextf(err, "encoding external payload %v", pl) + return nil, errors.WithContextf(err, "encoding external payload %v", pl) } // Put schema and row into payload proto, and marshal it. @@ -57,6 +56,19 @@ func EncodeStructPayload(pl any) ([]byte, error) { Schema: scm, Payload: buf.Bytes(), } + return ecp, nil +} + +// EncodeStructPayload takes a native Go struct and returns a marshaled +// ExternalConfigurationPayload proto, containing a Schema representation of +// the original type and the original value encoded as a Row. This is intended +// to be used as the expansion payload for an External transform. +func EncodeStructPayload(pl any) ([]byte, error) { + ecp, err := CreateExternalConfigurationPayload(pl) + if err != nil { + return []byte{}, err + } + plBytes, err := proto.Marshal(ecp) if err != nil { err = errors.Wrapf(err, "failed to marshal payload as proto") diff --git a/sdks/go/pkg/beam/create.go b/sdks/go/pkg/beam/create.go index 4ddc5396c7240..d2bd554963ee6 100644 --- a/sdks/go/pkg/beam/create.go +++ b/sdks/go/pkg/beam/create.go @@ -20,8 +20,14 @@ import ( "reflect" "github.com/apache/beam/sdks/v2/go/pkg/beam/internal/errors" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" ) +func init() { + register.DoFn2x1[[]byte, func(T), error]((*createFn)(nil)) + register.Emitter1[T]() +} + // Create inserts a fixed non-empty set of values into the pipeline. The values must // be of the same type 'A' and the returned PCollection is of type A. // diff --git a/sdks/go/pkg/beam/create_test.go b/sdks/go/pkg/beam/create_test.go index 3acfe779bba15..785c3b33db621 100644 --- a/sdks/go/pkg/beam/create_test.go +++ b/sdks/go/pkg/beam/create_test.go @@ -26,6 +26,15 @@ import ( "github.com/golang/protobuf/proto" ) +func TestMain(m *testing.M) { + ptest.Main(m) +} + +func init() { + beam.RegisterType(reflect.TypeOf((*wc)(nil)).Elem()) + beam.RegisterType(reflect.TypeOf((*testProto)(nil)).Elem()) +} + type wc struct { K string V int @@ -66,8 +75,8 @@ func TestCreateList(t *testing.T) { {[]float64{float64(0.1), float64(0.2), float64(0.3)}}, {[]uint{uint(1), uint(2), uint(3)}}, {[]bool{false, true, true, false, true}}, - {[]wc{wc{"a", 23}, wc{"b", 42}, wc{"c", 5}}}, - {[]*testProto{&testProto{}, &testProto{stringValue("test")}}}, // Test for BEAM-4401 + {[]wc{{"a", 23}, {"b", 42}, {"c", 5}}}, + {[]*testProto{{}, {stringValue("test")}}}, // Test for BEAM-4401 } for _, test := range tests { diff --git a/sdks/go/pkg/beam/io/avroio/avroio.go b/sdks/go/pkg/beam/io/avroio/avroio.go index b282c4aa30472..809c9479f7a4e 100644 --- a/sdks/go/pkg/beam/io/avroio/avroio.go +++ b/sdks/go/pkg/beam/io/avroio/avroio.go @@ -20,18 +20,20 @@ import ( "context" "encoding/json" "reflect" - "strings" "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/io/fileio" "github.com/apache/beam/sdks/v2/go/pkg/beam/io/filesystem" "github.com/apache/beam/sdks/v2/go/pkg/beam/log" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" "github.com/linkedin/goavro/v2" ) func init() { - beam.RegisterFunction(expandFn) - beam.RegisterType(reflect.TypeOf((*avroReadFn)(nil)).Elem()) - beam.RegisterType(reflect.TypeOf((*writeAvroFn)(nil)).Elem()) + register.DoFn3x1[context.Context, fileio.ReadableFile, func(beam.X), error]((*avroReadFn)(nil)) + register.DoFn3x1[context.Context, int, func(*string) bool, error]((*writeAvroFn)(nil)) + register.Emitter1[beam.X]() + register.Iter1[string]() } // Read reads a set of files and returns lines as a PCollection @@ -46,7 +48,8 @@ func Read(s beam.Scope, glob string, t reflect.Type) beam.PCollection { } func read(s beam.Scope, t reflect.Type, col beam.PCollection) beam.PCollection { - files := beam.ParDo(s, expandFn, col) + matches := fileio.MatchAll(s, col, fileio.MatchEmptyAllow()) + files := fileio.ReadMatches(s, matches, fileio.ReadUncompressed()) return beam.ParDo(s, &avroReadFn{Type: beam.EncodedType{T: t}}, files, @@ -54,42 +57,15 @@ func read(s beam.Scope, t reflect.Type, col beam.PCollection) beam.PCollection { ) } -func expandFn(ctx context.Context, glob string, emit func(string)) error { - if strings.TrimSpace(glob) == "" { - return nil // ignore empty string elements here - } - - fs, err := filesystem.New(ctx, glob) - if err != nil { - return err - } - defer fs.Close() - - files, err := fs.List(ctx, glob) - if err != nil { - return err - } - for _, filename := range files { - emit(filename) - } - return nil -} - type avroReadFn struct { // Avro schema type Type beam.EncodedType } -func (f *avroReadFn) ProcessElement(ctx context.Context, filename string, emit func(beam.X)) (err error) { - log.Infof(ctx, "Reading AVRO from %v", filename) - - fs, err := filesystem.New(ctx, filename) - if err != nil { - return - } - defer fs.Close() +func (f *avroReadFn) ProcessElement(ctx context.Context, file fileio.ReadableFile, emit func(beam.X)) (err error) { + log.Infof(ctx, "Reading AVRO from %v", file.Metadata.Path) - fd, err := fs.OpenRead(ctx, filename) + fd, err := file.Open(ctx) if err != nil { return } diff --git a/sdks/go/pkg/beam/io/avroio/avroio_test.go b/sdks/go/pkg/beam/io/avroio/avroio_test.go index 8e2894133bfed..403a818755578 100644 --- a/sdks/go/pkg/beam/io/avroio/avroio_test.go +++ b/sdks/go/pkg/beam/io/avroio/avroio_test.go @@ -25,12 +25,30 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam" _ "github.com/apache/beam/sdks/v2/go/pkg/beam/io/filesystem/local" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/passert" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest" "github.com/linkedin/goavro/v2" ) +func TestMain(m *testing.M) { + ptest.Main(m) +} + +func init() { + beam.RegisterType(reflect.TypeOf((*Tweet)(nil)).Elem()) + beam.RegisterType(reflect.TypeOf((*NullableFloat64)(nil)).Elem()) + beam.RegisterType(reflect.TypeOf((*NullableString)(nil)).Elem()) + beam.RegisterType(reflect.TypeOf((*NullableTweet)(nil)).Elem()) + register.Function2x0(toJSONString) +} + +func toJSONString(user TwitterUser, emit func(string)) { + b, _ := json.Marshal(user) + emit(string(b)) +} + type Tweet struct { Stamp int64 `json:"timestamp"` Tweet string `json:"tweet"` @@ -126,16 +144,11 @@ func TestWrite(t *testing.T) { avroFile := "./user.avro" testUsername := "user1" testInfo := "userInfo" - p, s, sequence := ptest.CreateList([]string{testUsername}) - format := beam.ParDo(s, func(username string, emit func(string)) { - newUser := TwitterUser{ - User: username, - Info: testInfo, - } - - b, _ := json.Marshal(newUser) - emit(string(b)) - }, sequence) + p, s, sequence := ptest.CreateList([]TwitterUser{{ + User: testUsername, + Info: testInfo, + }}) + format := beam.ParDo(s, toJSONString, sequence) Write(s, avroFile, userSchema, format) t.Cleanup(func() { os.Remove(avroFile) diff --git a/sdks/go/pkg/beam/io/bigqueryio/bigquery.go b/sdks/go/pkg/beam/io/bigqueryio/bigquery.go index 4ca64be87800a..1a5e8650052b7 100644 --- a/sdks/go/pkg/beam/io/bigqueryio/bigquery.go +++ b/sdks/go/pkg/beam/io/bigqueryio/bigquery.go @@ -145,9 +145,9 @@ func query(s beam.Scope, project, query string, t reflect.Type, options ...func( } type queryFn struct { - // Project is the project + // Project is the project. Project string `json:"project"` - // Table is the table identifier. + // Query is the query statement. Query string `json:"query"` // Type is the encoded schema type. Type beam.EncodedType `json:"type"` @@ -337,10 +337,9 @@ func (f *writeFn) ProcessElement(ctx context.Context, _ int, iter func(*beam.X) } data = nil size = writeOverheadBytes - } else { - data = append(data, reflect.ValueOf(val.(any))) - size += current } + data = append(data, reflect.ValueOf(val.(any))) + size += current } if len(data) == 0 { return nil diff --git a/sdks/go/pkg/beam/io/bigtableio/bigtable_test.go b/sdks/go/pkg/beam/io/bigtableio/bigtable_test.go index 4d2dc1b333802..2f41f4ff615e9 100644 --- a/sdks/go/pkg/beam/io/bigtableio/bigtable_test.go +++ b/sdks/go/pkg/beam/io/bigtableio/bigtable_test.go @@ -23,8 +23,13 @@ import ( "cloud.google.com/go/bigtable" "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest" ) +func TestMain(m *testing.M) { + ptest.Main(m) +} + func TestHashStringToInt(t *testing.T) { equalVal := "equal" diff --git a/sdks/go/pkg/beam/io/databaseio/database_test.go b/sdks/go/pkg/beam/io/databaseio/database_test.go index 1876f57012159..f6c1355e851a3 100644 --- a/sdks/go/pkg/beam/io/databaseio/database_test.go +++ b/sdks/go/pkg/beam/io/databaseio/database_test.go @@ -22,11 +22,16 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam" _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/direct" + _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/passert" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest" _ "github.com/proullon/ramsql/driver" ) +func TestMain(m *testing.M) { + ptest.Main(m) +} + type Address struct { Street string Street_number int diff --git a/sdks/go/pkg/beam/io/databaseio/writer.go b/sdks/go/pkg/beam/io/databaseio/writer.go index 4719831581ff3..ebd805b080d7a 100644 --- a/sdks/go/pkg/beam/io/databaseio/writer.go +++ b/sdks/go/pkg/beam/io/databaseio/writer.go @@ -108,7 +108,7 @@ type valueTemplateGenerator struct { func (v *valueTemplateGenerator) generate(rowCount int, columnColunt int) string { switch v.driver { - case "postgres": + case "postgres", "pgx": // the point is to generate ($1,$2),($3,$4) valueTemplates := make([]string, rowCount) for i := 0; i < rowCount; i++ { diff --git a/sdks/go/pkg/beam/io/databaseio/writer_test.go b/sdks/go/pkg/beam/io/databaseio/writer_test.go index 24f617a9b8578..91d7fb9246dd3 100644 --- a/sdks/go/pkg/beam/io/databaseio/writer_test.go +++ b/sdks/go/pkg/beam/io/databaseio/writer_test.go @@ -39,6 +39,12 @@ func TestValueTemplateGenerator_generate(t *testing.T) { columnCount: 10, expected: "", }, + { + generator: &valueTemplateGenerator{"pgx"}, + rowCount: 4, + columnCount: 3, + expected: "($1,$2,$3),($4,$5,$6),($7,$8,$9),($10,$11,$12)", + }, { generator: &valueTemplateGenerator{"mysql"}, rowCount: 4, diff --git a/sdks/go/pkg/beam/io/datastoreio/datastore.go b/sdks/go/pkg/beam/io/datastoreio/datastore.go index f4391059aef64..23f81a6d33866 100644 --- a/sdks/go/pkg/beam/io/datastoreio/datastore.go +++ b/sdks/go/pkg/beam/io/datastoreio/datastore.go @@ -221,8 +221,8 @@ func (s *queryFn) Setup() error { return nil } -func (f *queryFn) ProcessElement(ctx context.Context, _ string, v func(*string) bool, emit func(beam.X)) error { - client, err := f.newClientFunc(ctx, f.Project) +func (s *queryFn) ProcessElement(ctx context.Context, _ string, v func(*string) bool, emit func(beam.X)) error { + client, err := s.newClientFunc(ctx, s.Project) if err != nil { return err } @@ -238,13 +238,13 @@ func (f *queryFn) ProcessElement(ctx context.Context, _ string, v func(*string) } // lookup type - t, ok := runtime.LookupType(f.Type) + t, ok := runtime.LookupType(s.Type) if !ok { - return errors.Errorf("No type registered %s", f.Type) + return errors.Errorf("No type registered %s", s.Type) } // Translate BoundedQuery to datastore.Query - dq := datastore.NewQuery(f.Kind) + dq := datastore.NewQuery(s.Kind) if q.Start != nil { dq = dq.Filter("__key__ >=", q.Start) } diff --git a/sdks/go/pkg/beam/io/datastoreio/datastore_test.go b/sdks/go/pkg/beam/io/datastoreio/datastore_test.go index a18891bfd14d6..b95439e2d56dc 100644 --- a/sdks/go/pkg/beam/io/datastoreio/datastore_test.go +++ b/sdks/go/pkg/beam/io/datastoreio/datastore_test.go @@ -29,6 +29,17 @@ import ( "google.golang.org/api/option" ) +func TestMain(m *testing.M) { + // TODO(https://github.com/apache/beam/issues/27549): Make tests compatible with portable runners. + // To work on this change, replace call with `ptest.Main(m)` + ptest.MainWithDefault(m, "direct") +} + +func init() { + beam.RegisterType(reflect.TypeOf((*Foo)(nil)).Elem()) + beam.RegisterType(reflect.TypeOf((*Bar)(nil)).Elem()) +} + // fake client type implements datastoreio.clientType type fakeClient struct { runCounter int @@ -53,6 +64,11 @@ type Foo struct { type Bar struct { } +func init() { + beam.RegisterType(reflect.TypeOf((*Foo)(nil)).Elem()) + beam.RegisterType(reflect.TypeOf((*Bar)(nil)).Elem()) +} + func Test_query(t *testing.T) { testCases := []struct { v any @@ -75,7 +91,7 @@ func Test_query(t *testing.T) { } itemType := reflect.TypeOf(tc.v) - itemKey := runtime.RegisterType(itemType) + itemKey, _ := runtime.TypeKey(itemType) p, s := beam.NewPipelineWithRoot() query(s, "project", "Item", tc.shard, itemType, itemKey, newClient) @@ -93,7 +109,12 @@ func Test_query(t *testing.T) { } } +// Baz is intentionally unregistered. +type Baz struct { +} + func Test_query_Bad(t *testing.T) { + fooKey, _ := runtime.TypeKey(reflect.TypeOf(Foo{})) testCases := []struct { v any itemType reflect.Type @@ -103,8 +124,8 @@ func Test_query_Bad(t *testing.T) { }{ // mismatch typeKey parameter { - Foo{}, - reflect.TypeOf(Foo{}), + Baz{}, + reflect.TypeOf(Baz{}), "MismatchType", "No type registered MismatchType", nil, @@ -113,7 +134,7 @@ func Test_query_Bad(t *testing.T) { { Foo{}, reflect.TypeOf(Foo{}), - runtime.RegisterType(reflect.TypeOf(Foo{})), + fooKey, "fake client error", errors.New("fake client error"), }, diff --git a/sdks/go/pkg/beam/io/fhirio/deidentify_test.go b/sdks/go/pkg/beam/io/fhirio/deidentify_test.go index 10f281cd1ed6f..caa5b88d7c836 100644 --- a/sdks/go/pkg/beam/io/fhirio/deidentify_test.go +++ b/sdks/go/pkg/beam/io/fhirio/deidentify_test.go @@ -24,6 +24,12 @@ import ( "google.golang.org/api/healthcare/v1" ) +func TestMain(m *testing.M) { + // TODO(https://github.com/apache/beam/issues/27547): Make tests compatible with portable runners. + // To work on this change, replace call with `ptest.Main(m)` + ptest.MainWithDefault(m, "direct") +} + func TestDeidentify_Error(t *testing.T) { p, s := beam.NewPipelineWithRoot() out := deidentify(s, "src", "dst", nil, requestReturnErrorFakeClient) diff --git a/sdks/go/pkg/beam/io/fileio/match_test.go b/sdks/go/pkg/beam/io/fileio/match_test.go index 5bc849e5057e8..69e17e9181a4a 100644 --- a/sdks/go/pkg/beam/io/fileio/match_test.go +++ b/sdks/go/pkg/beam/io/fileio/match_test.go @@ -27,6 +27,10 @@ import ( "github.com/google/go-cmp/cmp" ) +func TestMain(m *testing.M) { + ptest.Main(m) +} + type testFile struct { filename string data []byte diff --git a/sdks/go/pkg/beam/io/parquetio/parquetio.go b/sdks/go/pkg/beam/io/parquetio/parquetio.go index 0d5a4fbb8316c..eb2a611f68365 100644 --- a/sdks/go/pkg/beam/io/parquetio/parquetio.go +++ b/sdks/go/pkg/beam/io/parquetio/parquetio.go @@ -18,11 +18,10 @@ package parquetio import ( "context" - "io" "reflect" - "strings" "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/io/fileio" "github.com/apache/beam/sdks/v2/go/pkg/beam/io/filesystem" "github.com/apache/beam/sdks/v2/go/pkg/beam/register" "github.com/xitongsys/parquet-go-source/buffer" @@ -31,14 +30,11 @@ import ( ) func init() { - register.Function3x1(expandFn) register.Emitter1[string]() - beam.RegisterType(reflect.TypeOf((*parquetReadFn)(nil)).Elem()) - register.DoFn3x1[context.Context, string, func(beam.X), error](&parquetReadFn{}) + register.DoFn3x1[context.Context, fileio.ReadableFile, func(beam.X), error](&parquetReadFn{}) register.Emitter1[beam.X]() - beam.RegisterType(reflect.TypeOf((*parquetWriteFn)(nil)).Elem()) register.DoFn3x1[context.Context, int, func(*beam.X) bool, error](&parquetWriteFn{}) register.Iter1[beam.X]() } @@ -63,7 +59,8 @@ func Read(s beam.Scope, glob string, t reflect.Type) beam.PCollection { } func read(s beam.Scope, t reflect.Type, col beam.PCollection) beam.PCollection { - files := beam.ParDo(s, expandFn, col) + matches := fileio.MatchAll(s, col, fileio.MatchEmptyAllow()) + files := fileio.ReadMatches(s, matches, fileio.ReadUncompressed()) return beam.ParDo(s, &parquetReadFn{Type: beam.EncodedType{T: t}}, files, @@ -71,45 +68,12 @@ func read(s beam.Scope, t reflect.Type, col beam.PCollection) beam.PCollection { ) } -func expandFn(ctx context.Context, glob string, emit func(string)) error { - if strings.TrimSpace(glob) == "" { - return nil // ignore empty string elements here - } - - fs, err := filesystem.New(ctx, glob) - if err != nil { - return err - } - defer fs.Close() - - files, err := fs.List(ctx, glob) - if err != nil { - return err - } - for _, filename := range files { - emit(filename) - } - return nil -} - type parquetReadFn struct { Type beam.EncodedType } -func (a *parquetReadFn) ProcessElement(ctx context.Context, filename string, emit func(beam.X)) error { - fs, err := filesystem.New(ctx, filename) - if err != nil { - return err - } - defer fs.Close() - - fd, err := fs.OpenRead(ctx, filename) - if err != nil { - return err - } - defer fd.Close() - - data, err := io.ReadAll(fd) +func (a *parquetReadFn) ProcessElement(ctx context.Context, file fileio.ReadableFile, emit func(beam.X)) error { + data, err := file.Read(ctx) if err != nil { return err } @@ -132,7 +96,7 @@ func (a *parquetReadFn) ProcessElement(ctx context.Context, filename string, emi } // Write writes a PCollection to .parquet file. -// Write expects a type t of struct with parquet tags +// Write expects elements of a struct type with parquet tags // For example: // // type Student struct { @@ -144,7 +108,8 @@ func (a *parquetReadFn) ProcessElement(ctx context.Context, filename string, emi // Day int32 `parquet:"name=day, type=INT32, convertedtype=DATE"` // Ignored int32 //without parquet tag and won't write // } -func Write(s beam.Scope, filename string, t reflect.Type, col beam.PCollection) { +func Write(s beam.Scope, filename string, col beam.PCollection) { + t := col.Type().Type() s = s.Scope("parquetio.Write") filesystem.ValidateScheme(filename) pre := beam.AddFixedKey(s, col) diff --git a/sdks/go/pkg/beam/io/parquetio/parquetio_test.go b/sdks/go/pkg/beam/io/parquetio/parquetio_test.go index f5f966ab16932..f3c9013956096 100644 --- a/sdks/go/pkg/beam/io/parquetio/parquetio_test.go +++ b/sdks/go/pkg/beam/io/parquetio/parquetio_test.go @@ -29,6 +29,10 @@ import ( "github.com/xitongsys/parquet-go/reader" ) +func TestMain(m *testing.M) { + ptest.Main(m) +} + type Student struct { Name string `parquet:"name=name, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` Age int32 `parquet:"name=age, type=INT32, encoding=PLAIN"` @@ -91,7 +95,7 @@ func TestWrite(t *testing.T) { } p, s, sequence := ptest.CreateList(studentList) parquetFile := "./write_student.parquet" - Write(s, parquetFile, reflect.TypeOf(Student{}), sequence) + Write(s, parquetFile, sequence) t.Cleanup(func() { os.Remove(parquetFile) }) diff --git a/sdks/go/pkg/beam/io/spannerio/common.go b/sdks/go/pkg/beam/io/spannerio/common.go index 04cc2154a6048..743a70d2fcff9 100644 --- a/sdks/go/pkg/beam/io/spannerio/common.go +++ b/sdks/go/pkg/beam/io/spannerio/common.go @@ -18,9 +18,10 @@ package spannerio import ( - "cloud.google.com/go/spanner" "context" "fmt" + + "cloud.google.com/go/spanner" "google.golang.org/api/option" "google.golang.org/api/option/internaloption" "google.golang.org/grpc" @@ -28,9 +29,9 @@ import ( ) type spannerFn struct { - Database string `json:"database"` // Database is the spanner connection string - endpoint string // Override spanner endpoint in tests - client *spanner.Client // Spanner Client + Database string `json:"database"` // Database is the spanner connection string + TestEndpoint string // Optional endpoint override for local testing. Not required for production pipelines. + client *spanner.Client // Spanner Client } func newSpannerFn(db string) spannerFn { @@ -48,9 +49,9 @@ func (f *spannerFn) Setup(ctx context.Context) error { var opts []option.ClientOption // Append emulator options assuming endpoint is local (for testing). - if f.endpoint != "" { + if f.TestEndpoint != "" { opts = []option.ClientOption{ - option.WithEndpoint(f.endpoint), + option.WithEndpoint(f.TestEndpoint), option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())), option.WithoutAuthentication(), internaloption.SkipDialSettingsValidation(), diff --git a/sdks/go/pkg/beam/io/spannerio/read_test.go b/sdks/go/pkg/beam/io/spannerio/read_test.go index 1a7705b1aca2d..7e1a65d0fe8a8 100644 --- a/sdks/go/pkg/beam/io/spannerio/read_test.go +++ b/sdks/go/pkg/beam/io/spannerio/read_test.go @@ -27,6 +27,10 @@ import ( spannertest "github.com/apache/beam/sdks/v2/go/test/integration/io/spannerio" ) +func TestMain(m *testing.M) { + ptest.Main(m) +} + func TestRead(t *testing.T) { ctx := context.Background() @@ -102,7 +106,7 @@ func TestRead(t *testing.T) { p, s := beam.NewPipelineWithRoot() fn := newQueryFn(testCase.database, "SELECT * from "+testCase.table, reflect.TypeOf(TestDto{}), queryOptions{}) - fn.endpoint = srv.Addr + fn.TestEndpoint = srv.Addr imp := beam.Impulse(s) rows := beam.ParDo(s, fn, imp, beam.TypeDefinition{Var: beam.XType, T: reflect.TypeOf(TestDto{})}) diff --git a/sdks/go/pkg/beam/io/spannerio/write_test.go b/sdks/go/pkg/beam/io/spannerio/write_test.go index f273315ba119a..3c2c1f591519b 100644 --- a/sdks/go/pkg/beam/io/spannerio/write_test.go +++ b/sdks/go/pkg/beam/io/spannerio/write_test.go @@ -17,12 +17,12 @@ package spannerio import ( "context" - spannertest "github.com/apache/beam/sdks/v2/go/test/integration/io/spannerio" "testing" "cloud.google.com/go/spanner" "github.com/apache/beam/sdks/v2/go/pkg/beam" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest" + spannertest "github.com/apache/beam/sdks/v2/go/test/integration/io/spannerio" "google.golang.org/api/iterator" ) @@ -77,7 +77,7 @@ func TestWrite(t *testing.T) { p, s, col := ptest.CreateList(testCase.rows) fn := newWriteFn(testCase.database, testCase.table, col.Type().Type()) - fn.endpoint = srv.Addr + fn.TestEndpoint = srv.Addr beam.ParDo0(s, fn, col) diff --git a/sdks/go/pkg/beam/io/textio/textio_test.go b/sdks/go/pkg/beam/io/textio/textio_test.go index 10b0f5f4b1b59..ff4579b0db8dd 100644 --- a/sdks/go/pkg/beam/io/textio/textio_test.go +++ b/sdks/go/pkg/beam/io/textio/textio_test.go @@ -17,7 +17,6 @@ package textio import ( - "context" "errors" "os" "path/filepath" @@ -25,10 +24,19 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam" _ "github.com/apache/beam/sdks/v2/go/pkg/beam/io/filesystem/local" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/passert" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest" ) +func TestMain(m *testing.M) { + ptest.Main(m) +} + +func init() { + register.Function2x1(toKV) +} + const testDir = "../../../../data" var ( @@ -144,9 +152,7 @@ func TestReadSdf(t *testing.T) { lines := ReadSdf(s, testFilePath) passert.Count(s, lines, "NumLines", 1) - if _, err := beam.Run(context.Background(), "direct", p); err != nil { - t.Fatalf("Failed to execute job: %v", err) - } + ptest.RunAndValidate(t, p) } func TestReadAllSdf(t *testing.T) { @@ -155,7 +161,5 @@ func TestReadAllSdf(t *testing.T) { lines := ReadAllSdf(s, files) passert.Count(s, lines, "NumLines", 1) - if _, err := beam.Run(context.Background(), "direct", p); err != nil { - t.Fatalf("Failed to execute job: %v", err) - } + ptest.RunAndValidate(t, p) } diff --git a/sdks/go/pkg/beam/io/xlang/bigtableio/bigtable.go b/sdks/go/pkg/beam/io/xlang/bigtableio/bigtable.go new file mode 100644 index 0000000000000..5b6d7d9163108 --- /dev/null +++ b/sdks/go/pkg/beam/io/xlang/bigtableio/bigtable.go @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package bigtableio contains cross-language functionality for using Google Cloud BigQuery +// (https://cloud.google.com/bigquery). These transforms only work on runners that support +// cross-language transforms. +// +// # Setup +// +// Transforms specified here are cross-language transforms implemented in a +// different SDK (listed below). During pipeline construction, the Go SDK will +// need to connect to an expansion service containing information on these +// transforms in their native SDK. If an expansion service address is not +// provided, an appropriate expansion service will be automatically started; +// however this is slower than having a persistent expansion service running. +// +// To use a persistent expansion service, it must be run as a separate process +// accessible during pipeline construction. The address of that process must be +// passed to the transforms in this package. +// +// The version of the expansion service should match the version of the Beam SDK +// being used. For numbered releases of Beam, these expansions services are +// released to the Maven repository as modules. For development versions of +// Beam, it is recommended to build and run it from source using Gradle. +// +// Current supported SDKs, including expansion service modules and reference +// documentation: +// +// Java: +// - Vendored Module: beam-sdks-java-io-google-cloud-platform-expansion-service +// - Run via Gradle: ./gradlew :sdks:java:io:google-cloud-platform:expansion-service:runExpansionService +// - Reference Class: org.apache.beam.sdk.io.gcp.bigtable.BigtableReadSchemaTransformProvider and +// org.apache.beam.sdk.io.gcp.bigtable.BigtableIO +// +// # Note On Documentation +// +// This cross-language implementation relies on the behavior of external SDKs. In order to keep +// documentation up-to-date and consistent, Bigtable functionality will not be described in detail +// in this package. Instead, references to relevant documentation in other SDKs is included where +// relevant. +package bigtableio + +import ( + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex" + "reflect" + + "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/xlangx" + xlschema "github.com/apache/beam/sdks/v2/go/pkg/beam/transforms/xlang/schema" +) + +type bigtableConfig struct { + InstanceId string `beam:"instanceId"` + ProjectId string `beam:"projectId"` + TableId string `beam:"tableId"` +} + +// Cell represents a single cell in a Bigtable row. +// TODO(https://github.com/apache/beam/issues/21784): Change back to a named struct once resolved. +type Cell = struct { + Value []uint8 `beam:"value"` + Timestamp_micros int64 `beam:"timestamp_micros"` +} + +// Row represents a row in Bigtable. +type Row struct { + Key []uint8 `beam:"key"` + Column_families map[string]map[string][]Cell `beam:"column_families"` +} + +// AddCell adds cell to a Row. Note, this method does not deduplicate cells with the same +// (family, qualifier, timestamp) nor does it maintain cells sorted in timestamp-descending order. +func (row *Row) AddCell(family string, qualifier string, value []byte, timestamp int64) { + if row.Column_families == nil { + row.Column_families = make(map[string]map[string][]Cell) + } + + cf, found := row.Column_families[family] + if !found { + cf = make(map[string][]Cell) + row.Column_families[family] = cf + } + cf[qualifier] = append(cf[qualifier], Cell{ + Value: value, + Timestamp_micros: timestamp, + }) +} + +const ( + readURN = "beam:schematransform:org.apache.beam:bigtable_read:v1" + serviceGradleTarget = ":sdks:java:io:google-cloud-platform:expansion-service:runExpansionService" +) + +var autoStartupAddress = xlangx.UseAutomatedJavaExpansionService(serviceGradleTarget) + +type ReadOption func(*readConfig) +type readConfig struct { + addr string +} + +// ReadExpansionAddr specifies the address of a persistent expansion service to use for a Read +// transform. If this is not provided, or if an empty string is provided, the transform will +// automatically start an appropriate expansion service instead. +func ReadExpansionAddr(addr string) ReadOption { + return func(rc *readConfig) { + rc.addr = addr + } +} + +// Read reads rows from a Bigtable table. +func Read(s beam.Scope, projectId string, instanceId string, table string, opts ...ReadOption) beam.PCollection { + rc := readConfig{} + for _, opt := range opts { + opt(&rc) + } + + addr := rc.addr + if addr == "" { + addr = autoStartupAddress + } + btConfig := bigtableConfig{InstanceId: instanceId, ProjectId: projectId, TableId: table} + + outs := xlschema.Transform(s, btConfig, readURN, xlschema.ExpansionAddr(addr), xlschema.UnnamedOutputType(typex.New(reflect.TypeOf(Row{})))) + return outs[beam.UnnamedOutputTag()] +} diff --git a/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api.pb.go b/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api.pb.go index 7baa5d80bb6b0..9d14cff3c7d61 100644 --- a/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api.pb.go +++ b/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api.pb.go @@ -27,7 +27,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.27.1 -// protoc v3.21.9 +// protoc v4.24.4 // source: org/apache/beam/model/fn_execution/v1/beam_fn_api.proto // TODO: Consider consolidating common components in another package @@ -41,6 +41,7 @@ import ( protoimpl "google.golang.org/protobuf/runtime/protoimpl" _ "google.golang.org/protobuf/types/descriptorpb" durationpb "google.golang.org/protobuf/types/known/durationpb" + structpb "google.golang.org/protobuf/types/known/structpb" timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" @@ -53,6 +54,87 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type FnApiTransforms_Runner int32 + +const ( + // DataSource is a Root Transform, and a source of data for downstream + // transforms in the same ProcessBundleDescriptor. + // It represents a stream of values coming in from an external source/over + // a data channel, typically from the runner. It's not the PCollection itself + // but a description of how to get the portion of the PCollection for a given + // bundle. + // + // The DataSource transform is implemented in each SDK and not explicitly + // provided during pipeline construction. A runner inserts the transform + // in ProcessBundleDescriptors to indicate where the bundle + // can retrieve data for an associated ProcessBundleRequest. + // Data for the same request will be retrieved with the matching instruction ID, + // and transform ID determined by the runner. + // + // The DataSource transform will take a stream of bytes from the remote + // source for the matching instruction ID and decode them as windowed + // values using the provided coder ID, which must be a windowed value coder. + // + // Payload: RemoteGrpcPort + FnApiTransforms_DATA_SOURCE FnApiTransforms_Runner = 0 + // DataSink is a transform that sends PCollection elements to a remote + // port using the Data API. + // + // The DataSink transform is implemented in each SDK and not explicitly + // provided during pipeline construction. A runner inserts the transform in + // ProcessBundleDescriptors to indicate where the bundle can send + // data for each associated ProcessBundleRequest. Data for the same + // request will be sent with the matching instruction ID and transform ID. + // Each PCollection that exits the ProcessBundleDescriptor subgraph will have + // it's own DataSink, keyed by a transform ID determined by the runner. + // + // The DataSink will take in a stream of elements for a given instruction ID + // and encode them for transmission to the remote sink. The coder ID must be + // for a windowed value coder. + // + // Payload: RemoteGrpcPort + FnApiTransforms_DATA_SINK FnApiTransforms_Runner = 1 +) + +// Enum value maps for FnApiTransforms_Runner. +var ( + FnApiTransforms_Runner_name = map[int32]string{ + 0: "DATA_SOURCE", + 1: "DATA_SINK", + } + FnApiTransforms_Runner_value = map[string]int32{ + "DATA_SOURCE": 0, + "DATA_SINK": 1, + } +) + +func (x FnApiTransforms_Runner) Enum() *FnApiTransforms_Runner { + p := new(FnApiTransforms_Runner) + *p = x + return p +} + +func (x FnApiTransforms_Runner) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (FnApiTransforms_Runner) Descriptor() protoreflect.EnumDescriptor { + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_enumTypes[0].Descriptor() +} + +func (FnApiTransforms_Runner) Type() protoreflect.EnumType { + return &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_enumTypes[0] +} + +func (x FnApiTransforms_Runner) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use FnApiTransforms_Runner.Descriptor instead. +func (FnApiTransforms_Runner) EnumDescriptor() ([]byte, []int) { + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{0, 0} +} + type LogEntry_Severity_Enum int32 const ( @@ -110,11 +192,11 @@ func (x LogEntry_Severity_Enum) String() string { } func (LogEntry_Severity_Enum) Descriptor() protoreflect.EnumDescriptor { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_enumTypes[0].Descriptor() + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_enumTypes[1].Descriptor() } func (LogEntry_Severity_Enum) Type() protoreflect.EnumType { - return &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_enumTypes[0] + return &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_enumTypes[1] } func (x LogEntry_Severity_Enum) Number() protoreflect.EnumNumber { @@ -123,7 +205,47 @@ func (x LogEntry_Severity_Enum) Number() protoreflect.EnumNumber { // Deprecated: Use LogEntry_Severity_Enum.Descriptor instead. func (LogEntry_Severity_Enum) EnumDescriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{31, 1, 0} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{35, 1, 0} +} + +// Describes transforms necessary to execute Beam over the FnAPI but are +// implementation details rather than part of the core model. +type FnApiTransforms struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *FnApiTransforms) Reset() { + *x = FnApiTransforms{} + if protoimpl.UnsafeEnabled { + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FnApiTransforms) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FnApiTransforms) ProtoMessage() {} + +func (x *FnApiTransforms) ProtoReflect() protoreflect.Message { + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FnApiTransforms.ProtoReflect.Descriptor instead. +func (*FnApiTransforms) Descriptor() ([]byte, []int) { + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{0} } // A descriptor for connecting to a remote port using the Beam Fn Data API. @@ -146,7 +268,7 @@ type RemoteGrpcPort struct { func (x *RemoteGrpcPort) Reset() { *x = RemoteGrpcPort{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[0] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -159,7 +281,7 @@ func (x *RemoteGrpcPort) String() string { func (*RemoteGrpcPort) ProtoMessage() {} func (x *RemoteGrpcPort) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[0] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -172,7 +294,7 @@ func (x *RemoteGrpcPort) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoteGrpcPort.ProtoReflect.Descriptor instead. func (*RemoteGrpcPort) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{0} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{1} } func (x *RemoteGrpcPort) GetApiServiceDescriptor() *pipeline_v1.ApiServiceDescriptor { @@ -201,7 +323,7 @@ type GetProcessBundleDescriptorRequest struct { func (x *GetProcessBundleDescriptorRequest) Reset() { *x = GetProcessBundleDescriptorRequest{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[1] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -214,7 +336,7 @@ func (x *GetProcessBundleDescriptorRequest) String() string { func (*GetProcessBundleDescriptorRequest) ProtoMessage() {} func (x *GetProcessBundleDescriptorRequest) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[1] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -227,7 +349,7 @@ func (x *GetProcessBundleDescriptorRequest) ProtoReflect() protoreflect.Message // Deprecated: Use GetProcessBundleDescriptorRequest.ProtoReflect.Descriptor instead. func (*GetProcessBundleDescriptorRequest) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{1} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{2} } func (x *GetProcessBundleDescriptorRequest) GetProcessBundleDescriptorId() string { @@ -259,6 +381,7 @@ type InstructionRequest struct { // *InstructionRequest_FinalizeBundle // *InstructionRequest_MonitoringInfos // *InstructionRequest_HarnessMonitoringInfos + // *InstructionRequest_SampleData // *InstructionRequest_Register Request isInstructionRequest_Request `protobuf_oneof:"request"` } @@ -266,7 +389,7 @@ type InstructionRequest struct { func (x *InstructionRequest) Reset() { *x = InstructionRequest{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[2] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -279,7 +402,7 @@ func (x *InstructionRequest) String() string { func (*InstructionRequest) ProtoMessage() {} func (x *InstructionRequest) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[2] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -292,7 +415,7 @@ func (x *InstructionRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use InstructionRequest.ProtoReflect.Descriptor instead. func (*InstructionRequest) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{2} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{3} } func (x *InstructionRequest) GetInstructionId() string { @@ -351,6 +474,13 @@ func (x *InstructionRequest) GetHarnessMonitoringInfos() *HarnessMonitoringInfos return nil } +func (x *InstructionRequest) GetSampleData() *SampleDataRequest { + if x, ok := x.GetRequest().(*InstructionRequest_SampleData); ok { + return x.SampleData + } + return nil +} + func (x *InstructionRequest) GetRegister() *RegisterRequest { if x, ok := x.GetRequest().(*InstructionRequest_Register); ok { return x.Register @@ -386,6 +516,10 @@ type InstructionRequest_HarnessMonitoringInfos struct { HarnessMonitoringInfos *HarnessMonitoringInfosRequest `protobuf:"bytes,1006,opt,name=harness_monitoring_infos,json=harnessMonitoringInfos,proto3,oneof"` } +type InstructionRequest_SampleData struct { + SampleData *SampleDataRequest `protobuf:"bytes,1007,opt,name=sample_data,json=sampleData,proto3,oneof"` +} + type InstructionRequest_Register struct { // DEPRECATED Register *RegisterRequest `protobuf:"bytes,1000,opt,name=register,proto3,oneof"` @@ -403,6 +537,8 @@ func (*InstructionRequest_MonitoringInfos) isInstructionRequest_Request() {} func (*InstructionRequest_HarnessMonitoringInfos) isInstructionRequest_Request() {} +func (*InstructionRequest_SampleData) isInstructionRequest_Request() {} + func (*InstructionRequest_Register) isInstructionRequest_Request() {} // The response for an associated request the SDK had been asked to fulfill. @@ -431,6 +567,7 @@ type InstructionResponse struct { // *InstructionResponse_FinalizeBundle // *InstructionResponse_MonitoringInfos // *InstructionResponse_HarnessMonitoringInfos + // *InstructionResponse_SampleData // *InstructionResponse_Register Response isInstructionResponse_Response `protobuf_oneof:"response"` } @@ -438,7 +575,7 @@ type InstructionResponse struct { func (x *InstructionResponse) Reset() { *x = InstructionResponse{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[3] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -451,7 +588,7 @@ func (x *InstructionResponse) String() string { func (*InstructionResponse) ProtoMessage() {} func (x *InstructionResponse) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[3] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -464,7 +601,7 @@ func (x *InstructionResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use InstructionResponse.ProtoReflect.Descriptor instead. func (*InstructionResponse) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{3} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{4} } func (x *InstructionResponse) GetInstructionId() string { @@ -530,6 +667,13 @@ func (x *InstructionResponse) GetHarnessMonitoringInfos() *HarnessMonitoringInfo return nil } +func (x *InstructionResponse) GetSampleData() *SampleDataResponse { + if x, ok := x.GetResponse().(*InstructionResponse_SampleData); ok { + return x.SampleData + } + return nil +} + func (x *InstructionResponse) GetRegister() *RegisterResponse { if x, ok := x.GetResponse().(*InstructionResponse_Register); ok { return x.Register @@ -565,6 +709,10 @@ type InstructionResponse_HarnessMonitoringInfos struct { HarnessMonitoringInfos *HarnessMonitoringInfosResponse `protobuf:"bytes,1006,opt,name=harness_monitoring_infos,json=harnessMonitoringInfos,proto3,oneof"` } +type InstructionResponse_SampleData struct { + SampleData *SampleDataResponse `protobuf:"bytes,1007,opt,name=sample_data,json=sampleData,proto3,oneof"` +} + type InstructionResponse_Register struct { // DEPRECATED Register *RegisterResponse `protobuf:"bytes,1000,opt,name=register,proto3,oneof"` @@ -582,8 +730,182 @@ func (*InstructionResponse_MonitoringInfos) isInstructionResponse_Response() {} func (*InstructionResponse_HarnessMonitoringInfos) isInstructionResponse_Response() {} +func (*InstructionResponse_SampleData) isInstructionResponse_Response() {} + func (*InstructionResponse_Register) isInstructionResponse_Response() {} +// If supported, the `SampleDataRequest` will respond with a +// `SampleDataResponse`. The SDK being queried must have the +// "beam:protocol:data_sampling:v1" capability. Samples are taken only from the +// specified PCollection ids. An empty list will return everything. +type SampleDataRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // (Optional) The PCollection ids to filter for. + PcollectionIds []string `protobuf:"bytes,1,rep,name=pcollection_ids,json=pcollectionIds,proto3" json:"pcollection_ids,omitempty"` +} + +func (x *SampleDataRequest) Reset() { + *x = SampleDataRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SampleDataRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SampleDataRequest) ProtoMessage() {} + +func (x *SampleDataRequest) ProtoReflect() protoreflect.Message { + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SampleDataRequest.ProtoReflect.Descriptor instead. +func (*SampleDataRequest) Descriptor() ([]byte, []int) { + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{5} +} + +func (x *SampleDataRequest) GetPcollectionIds() []string { + if x != nil { + return x.PcollectionIds + } + return nil +} + +// An element sampled when the SDK is processing a bundle. This is a proto +// message to allow for additional per-element metadata. +type SampledElement struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // (Required) Sampled raw bytes for an element. This is a + // single encoded element in the nested context. + Element []byte `protobuf:"bytes,1,opt,name=element,proto3" json:"element,omitempty"` + // (Required) Timestamp of when the sample was taken. + SampleTimestamp *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=sample_timestamp,json=sampleTimestamp,proto3" json:"sample_timestamp,omitempty"` + // (Optional) This will be set if this element was sampled because of a user + // exception. + Exception *SampledElement_Exception `protobuf:"bytes,3,opt,name=exception,proto3" json:"exception,omitempty"` +} + +func (x *SampledElement) Reset() { + *x = SampledElement{} + if protoimpl.UnsafeEnabled { + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SampledElement) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SampledElement) ProtoMessage() {} + +func (x *SampledElement) ProtoReflect() protoreflect.Message { + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SampledElement.ProtoReflect.Descriptor instead. +func (*SampledElement) Descriptor() ([]byte, []int) { + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{6} +} + +func (x *SampledElement) GetElement() []byte { + if x != nil { + return x.Element + } + return nil +} + +func (x *SampledElement) GetSampleTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.SampleTimestamp + } + return nil +} + +func (x *SampledElement) GetException() *SampledElement_Exception { + if x != nil { + return x.Exception + } + return nil +} + +// If supported, the `SampleDataResponse` will contain samples from PCollections +// based upon the filters specified in the request. +type SampleDataResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Map from PCollection id to sampled elements. + ElementSamples map[string]*SampleDataResponse_ElementList `protobuf:"bytes,1,rep,name=element_samples,json=elementSamples,proto3" json:"element_samples,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *SampleDataResponse) Reset() { + *x = SampleDataResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SampleDataResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SampleDataResponse) ProtoMessage() {} + +func (x *SampleDataResponse) ProtoReflect() protoreflect.Message { + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SampleDataResponse.ProtoReflect.Descriptor instead. +func (*SampleDataResponse) Descriptor() ([]byte, []int) { + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{7} +} + +func (x *SampleDataResponse) GetElementSamples() map[string]*SampleDataResponse_ElementList { + if x != nil { + return x.ElementSamples + } + return nil +} + // A request to provide full MonitoringInfo associated with the entire SDK // harness process, not specific to a bundle. // @@ -605,7 +927,7 @@ type HarnessMonitoringInfosRequest struct { func (x *HarnessMonitoringInfosRequest) Reset() { *x = HarnessMonitoringInfosRequest{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[4] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -618,7 +940,7 @@ func (x *HarnessMonitoringInfosRequest) String() string { func (*HarnessMonitoringInfosRequest) ProtoMessage() {} func (x *HarnessMonitoringInfosRequest) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[4] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -631,7 +953,7 @@ func (x *HarnessMonitoringInfosRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use HarnessMonitoringInfosRequest.ProtoReflect.Descriptor instead. func (*HarnessMonitoringInfosRequest) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{4} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{8} } type HarnessMonitoringInfosResponse struct { @@ -657,7 +979,7 @@ type HarnessMonitoringInfosResponse struct { func (x *HarnessMonitoringInfosResponse) Reset() { *x = HarnessMonitoringInfosResponse{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[5] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -670,7 +992,7 @@ func (x *HarnessMonitoringInfosResponse) String() string { func (*HarnessMonitoringInfosResponse) ProtoMessage() {} func (x *HarnessMonitoringInfosResponse) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[5] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -683,7 +1005,7 @@ func (x *HarnessMonitoringInfosResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use HarnessMonitoringInfosResponse.ProtoReflect.Descriptor instead. func (*HarnessMonitoringInfosResponse) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{5} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{9} } func (x *HarnessMonitoringInfosResponse) GetMonitoringData() map[string][]byte { @@ -708,7 +1030,7 @@ type RegisterRequest struct { func (x *RegisterRequest) Reset() { *x = RegisterRequest{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[6] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -721,7 +1043,7 @@ func (x *RegisterRequest) String() string { func (*RegisterRequest) ProtoMessage() {} func (x *RegisterRequest) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[6] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -734,7 +1056,7 @@ func (x *RegisterRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterRequest.ProtoReflect.Descriptor instead. func (*RegisterRequest) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{6} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{10} } func (x *RegisterRequest) GetProcessBundleDescriptor() []*ProcessBundleDescriptor { @@ -754,7 +1076,7 @@ type RegisterResponse struct { func (x *RegisterResponse) Reset() { *x = RegisterResponse{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[7] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -767,7 +1089,7 @@ func (x *RegisterResponse) String() string { func (*RegisterResponse) ProtoMessage() {} func (x *RegisterResponse) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[7] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -780,7 +1102,7 @@ func (x *RegisterResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterResponse.ProtoReflect.Descriptor instead. func (*RegisterResponse) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{7} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{11} } // Definitions that should be used to construct the bundle processing graph. @@ -814,7 +1136,7 @@ type ProcessBundleDescriptor struct { func (x *ProcessBundleDescriptor) Reset() { *x = ProcessBundleDescriptor{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[8] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -827,7 +1149,7 @@ func (x *ProcessBundleDescriptor) String() string { func (*ProcessBundleDescriptor) ProtoMessage() {} func (x *ProcessBundleDescriptor) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[8] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -840,7 +1162,7 @@ func (x *ProcessBundleDescriptor) ProtoReflect() protoreflect.Message { // Deprecated: Use ProcessBundleDescriptor.ProtoReflect.Descriptor instead. func (*ProcessBundleDescriptor) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{8} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{12} } func (x *ProcessBundleDescriptor) GetId() string { @@ -931,7 +1253,7 @@ type BundleApplication struct { func (x *BundleApplication) Reset() { *x = BundleApplication{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[9] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -944,7 +1266,7 @@ func (x *BundleApplication) String() string { func (*BundleApplication) ProtoMessage() {} func (x *BundleApplication) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[9] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -957,7 +1279,7 @@ func (x *BundleApplication) ProtoReflect() protoreflect.Message { // Deprecated: Use BundleApplication.ProtoReflect.Descriptor instead. func (*BundleApplication) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{9} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{13} } func (x *BundleApplication) GetTransformId() string { @@ -1014,7 +1336,7 @@ type DelayedBundleApplication struct { func (x *DelayedBundleApplication) Reset() { *x = DelayedBundleApplication{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[10] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1027,7 +1349,7 @@ func (x *DelayedBundleApplication) String() string { func (*DelayedBundleApplication) ProtoMessage() {} func (x *DelayedBundleApplication) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[10] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1040,7 +1362,7 @@ func (x *DelayedBundleApplication) ProtoReflect() protoreflect.Message { // Deprecated: Use DelayedBundleApplication.ProtoReflect.Descriptor instead. func (*DelayedBundleApplication) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{10} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{14} } func (x *DelayedBundleApplication) GetApplication() *BundleApplication { @@ -1091,7 +1413,7 @@ type ProcessBundleRequest struct { func (x *ProcessBundleRequest) Reset() { *x = ProcessBundleRequest{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[11] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1104,7 +1426,7 @@ func (x *ProcessBundleRequest) String() string { func (*ProcessBundleRequest) ProtoMessage() {} func (x *ProcessBundleRequest) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[11] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1117,7 +1439,7 @@ func (x *ProcessBundleRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ProcessBundleRequest.ProtoReflect.Descriptor instead. func (*ProcessBundleRequest) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{11} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{15} } func (x *ProcessBundleRequest) GetProcessBundleDescriptorId() string { @@ -1186,7 +1508,7 @@ type ProcessBundleResponse struct { func (x *ProcessBundleResponse) Reset() { *x = ProcessBundleResponse{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[12] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1199,7 +1521,7 @@ func (x *ProcessBundleResponse) String() string { func (*ProcessBundleResponse) ProtoMessage() {} func (x *ProcessBundleResponse) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[12] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1212,7 +1534,7 @@ func (x *ProcessBundleResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ProcessBundleResponse.ProtoReflect.Descriptor instead. func (*ProcessBundleResponse) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{12} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{16} } func (x *ProcessBundleResponse) GetResidualRoots() []*DelayedBundleApplication { @@ -1266,7 +1588,7 @@ type ProcessBundleProgressRequest struct { func (x *ProcessBundleProgressRequest) Reset() { *x = ProcessBundleProgressRequest{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[13] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1279,7 +1601,7 @@ func (x *ProcessBundleProgressRequest) String() string { func (*ProcessBundleProgressRequest) ProtoMessage() {} func (x *ProcessBundleProgressRequest) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[13] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1292,7 +1614,7 @@ func (x *ProcessBundleProgressRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ProcessBundleProgressRequest.ProtoReflect.Descriptor instead. func (*ProcessBundleProgressRequest) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{13} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{17} } func (x *ProcessBundleProgressRequest) GetInstructionId() string { @@ -1325,7 +1647,7 @@ type MonitoringInfosMetadataRequest struct { func (x *MonitoringInfosMetadataRequest) Reset() { *x = MonitoringInfosMetadataRequest{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[14] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1338,7 +1660,7 @@ func (x *MonitoringInfosMetadataRequest) String() string { func (*MonitoringInfosMetadataRequest) ProtoMessage() {} func (x *MonitoringInfosMetadataRequest) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[14] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1351,7 +1673,7 @@ func (x *MonitoringInfosMetadataRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MonitoringInfosMetadataRequest.ProtoReflect.Descriptor instead. func (*MonitoringInfosMetadataRequest) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{14} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{18} } func (x *MonitoringInfosMetadataRequest) GetMonitoringInfoId() []string { @@ -1386,7 +1708,7 @@ type ProcessBundleProgressResponse struct { func (x *ProcessBundleProgressResponse) Reset() { *x = ProcessBundleProgressResponse{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[15] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1399,7 +1721,7 @@ func (x *ProcessBundleProgressResponse) String() string { func (*ProcessBundleProgressResponse) ProtoMessage() {} func (x *ProcessBundleProgressResponse) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[15] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1412,7 +1734,7 @@ func (x *ProcessBundleProgressResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ProcessBundleProgressResponse.ProtoReflect.Descriptor instead. func (*ProcessBundleProgressResponse) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{15} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{19} } func (x *ProcessBundleProgressResponse) GetMonitoringInfos() []*pipeline_v1.MonitoringInfo { @@ -1453,7 +1775,7 @@ type MonitoringInfosMetadataResponse struct { func (x *MonitoringInfosMetadataResponse) Reset() { *x = MonitoringInfosMetadataResponse{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[16] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1466,7 +1788,7 @@ func (x *MonitoringInfosMetadataResponse) String() string { func (*MonitoringInfosMetadataResponse) ProtoMessage() {} func (x *MonitoringInfosMetadataResponse) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[16] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[20] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1479,7 +1801,7 @@ func (x *MonitoringInfosMetadataResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MonitoringInfosMetadataResponse.ProtoReflect.Descriptor instead. func (*MonitoringInfosMetadataResponse) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{16} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{20} } func (x *MonitoringInfosMetadataResponse) GetMonitoringInfo() map[string]*pipeline_v1.MonitoringInfo { @@ -1509,7 +1831,7 @@ type ProcessBundleSplitRequest struct { func (x *ProcessBundleSplitRequest) Reset() { *x = ProcessBundleSplitRequest{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[17] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1522,7 +1844,7 @@ func (x *ProcessBundleSplitRequest) String() string { func (*ProcessBundleSplitRequest) ProtoMessage() {} func (x *ProcessBundleSplitRequest) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[17] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[21] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1535,7 +1857,7 @@ func (x *ProcessBundleSplitRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ProcessBundleSplitRequest.ProtoReflect.Descriptor instead. func (*ProcessBundleSplitRequest) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{17} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{21} } func (x *ProcessBundleSplitRequest) GetInstructionId() string { @@ -1561,30 +1883,57 @@ func (x *ProcessBundleSplitRequest) GetDesiredSplits() map[string]*ProcessBundle // first_residual_element. // - The current bundle, if no further splits happen, will have done exactly // the work under primary_roots and all elements up to and including the -// channel splits last_primary_element. +// channel split's last_primary_element. // // This allows the SDK to relinquish ownership of and commit to not process some // of the elements that it may have been sent (the residual) while retaining // ownership and commitment to finish the other portion (the primary). // -// For example, lets say the SDK is processing elements A B C D E and a split -// request comes in. The SDK could return a response with a channel split -// representing a last_primary_element of 3 (D) and first_residual_element of 4 -// (E). The SDK is now responsible for processing A B C D and the runner must -// process E in the future. A future split request could have the SDK split the -// elements B into B1 and B2 and C into C1 and C2 representing their primary and -// residual roots. The SDK would return a response with a channel split -// representing a last_primary_element of 0 (A) and first_residual_element of 3 -// (D) with primary_roots (B1, C1) and residual_roots (B2, C2). The SDK is now -// responsible for processing A B1 C1 and the runner must process C2 D2 (and E -// from the prior split) in the future. Yet another future split request could -// have the SDK could split B1 further into B1a and B1b primary and residuals -// and return C2 as a residual (assuming C2 was left unprocessed). The SDK would -// return a response with a channel split representing a last_primary_element of -// 0 (A) and first_residual_element of 4 (E) with primary_roots (B1a) and -// residual_roots (B1b, C1). The SDK is now responsible for processing A B1a the -// runner must process B1b C1 (in addition to C2, D, E from prior splits) in the -// future. +// Example with three splits of a single bundle: +// Let's say the SDK is processing elements [A B C D E]. These elements make +// up the 0-indexed channel. +// +// ** First Split ** +// Channel Split = [ A B C D <> E ] +// Primary Roots = [] (No elements were split) +// Residual Roots = [] +// +// Say a split request comes in. The SDK could return a response with a channel +// split representing a last_primary_element of 3 (D) and +// first_residual_element of 4 (E). The SDK is now responsible for processing A +// B C D and the runner must process E in the future. +// +// (A B C D) | (E) +// +// ** Second Split ** +// Channel Split = [ A < B C > D E ] +// Primary Roots = [B1 C1] +// Residual Roots = [B2 C2] +// +// A future split request could have the SDK split the elements B into B1 and +// B2 and C into C1 and C2 representing their primary and residual roots. The +// +// (A B1 C1) | (B2 C2 D) +// +// SDK would return a response with a channel split representing a +// last_primary_element of 0 (A) and first_residual_element of 3 (D) with +// primary_roots (B1, C1) and residual_roots (B2, C2). The SDK is now +// responsible for processing A B1 C1 and the runner must process B2 C2 D (and +// E from the prior split) in the future. +// +// ** Third Split ** +// Channel Split = [ A < B C > D E ] +// Primary Roots = [B1a] +// Residual Roots [B1b C1] +// Yet another future split request could have the SDK could split B1 further +// into B1a and B1b primary and residuals and return C1 as a residual (assuming +// C1 was left unprocessed). The SDK would return a response with a channel +// split representing a last_primary_element of 0 (A) and +// first_residual_element of 3 (E) with primary_roots (B1a) and residual_roots +// (B1b, C1). The SDK is now responsible for processing A B1a the runner must +// process B1b C1 (in addition to C2, D, E from prior splits) in the future. +// +// (A B1a) | (B1b C1) // // For more rigorous definitions see https://s.apache.org/beam-breaking-fusion type ProcessBundleSplitResponse struct { @@ -1624,7 +1973,7 @@ type ProcessBundleSplitResponse struct { func (x *ProcessBundleSplitResponse) Reset() { *x = ProcessBundleSplitResponse{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[18] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1637,7 +1986,7 @@ func (x *ProcessBundleSplitResponse) String() string { func (*ProcessBundleSplitResponse) ProtoMessage() {} func (x *ProcessBundleSplitResponse) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[18] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1650,7 +1999,7 @@ func (x *ProcessBundleSplitResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ProcessBundleSplitResponse.ProtoReflect.Descriptor instead. func (*ProcessBundleSplitResponse) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{18} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{22} } func (x *ProcessBundleSplitResponse) GetPrimaryRoots() []*BundleApplication { @@ -1687,7 +2036,7 @@ type FinalizeBundleRequest struct { func (x *FinalizeBundleRequest) Reset() { *x = FinalizeBundleRequest{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[19] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1700,7 +2049,7 @@ func (x *FinalizeBundleRequest) String() string { func (*FinalizeBundleRequest) ProtoMessage() {} func (x *FinalizeBundleRequest) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[19] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1713,7 +2062,7 @@ func (x *FinalizeBundleRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FinalizeBundleRequest.ProtoReflect.Descriptor instead. func (*FinalizeBundleRequest) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{19} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{23} } func (x *FinalizeBundleRequest) GetInstructionId() string { @@ -1732,7 +2081,7 @@ type FinalizeBundleResponse struct { func (x *FinalizeBundleResponse) Reset() { *x = FinalizeBundleResponse{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[20] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1745,7 +2094,7 @@ func (x *FinalizeBundleResponse) String() string { func (*FinalizeBundleResponse) ProtoMessage() {} func (x *FinalizeBundleResponse) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[20] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1758,7 +2107,7 @@ func (x *FinalizeBundleResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use FinalizeBundleResponse.ProtoReflect.Descriptor instead. func (*FinalizeBundleResponse) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{20} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{24} } // Messages used to represent logical byte streams. @@ -1777,7 +2126,7 @@ type Elements struct { func (x *Elements) Reset() { *x = Elements{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[21] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1790,7 +2139,7 @@ func (x *Elements) String() string { func (*Elements) ProtoMessage() {} func (x *Elements) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[21] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1803,7 +2152,7 @@ func (x *Elements) ProtoReflect() protoreflect.Message { // Deprecated: Use Elements.ProtoReflect.Descriptor instead. func (*Elements) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{21} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{25} } func (x *Elements) GetData() []*Elements_Data { @@ -1848,7 +2197,7 @@ type StateRequest struct { func (x *StateRequest) Reset() { *x = StateRequest{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[22] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1861,7 +2210,7 @@ func (x *StateRequest) String() string { func (*StateRequest) ProtoMessage() {} func (x *StateRequest) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[22] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1874,7 +2223,7 @@ func (x *StateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StateRequest.ProtoReflect.Descriptor instead. func (*StateRequest) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{22} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{26} } func (x *StateRequest) GetId() string { @@ -1977,7 +2326,7 @@ type StateResponse struct { func (x *StateResponse) Reset() { *x = StateResponse{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[23] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1990,7 +2339,7 @@ func (x *StateResponse) String() string { func (*StateResponse) ProtoMessage() {} func (x *StateResponse) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[23] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2003,7 +2352,7 @@ func (x *StateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use StateResponse.ProtoReflect.Descriptor instead. func (*StateResponse) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{23} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{27} } func (x *StateResponse) GetId() string { @@ -2095,7 +2444,7 @@ type StateKey struct { func (x *StateKey) Reset() { *x = StateKey{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[24] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2108,7 +2457,7 @@ func (x *StateKey) String() string { func (*StateKey) ProtoMessage() {} func (x *StateKey) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[24] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2121,7 +2470,7 @@ func (x *StateKey) ProtoReflect() protoreflect.Message { // Deprecated: Use StateKey.ProtoReflect.Descriptor instead. func (*StateKey) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{24} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{28} } func (m *StateKey) GetType() isStateKey_Type { @@ -2243,7 +2592,7 @@ type StateGetRequest struct { func (x *StateGetRequest) Reset() { *x = StateGetRequest{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[25] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2256,7 +2605,7 @@ func (x *StateGetRequest) String() string { func (*StateGetRequest) ProtoMessage() {} func (x *StateGetRequest) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[25] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2269,7 +2618,7 @@ func (x *StateGetRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StateGetRequest.ProtoReflect.Descriptor instead. func (*StateGetRequest) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{25} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{29} } func (x *StateGetRequest) GetContinuationToken() []byte { @@ -2299,7 +2648,7 @@ type StateGetResponse struct { func (x *StateGetResponse) Reset() { *x = StateGetResponse{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[26] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2312,7 +2661,7 @@ func (x *StateGetResponse) String() string { func (*StateGetResponse) ProtoMessage() {} func (x *StateGetResponse) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[26] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[30] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2325,7 +2674,7 @@ func (x *StateGetResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use StateGetResponse.ProtoReflect.Descriptor instead. func (*StateGetResponse) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{26} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{30} } func (x *StateGetResponse) GetContinuationToken() []byte { @@ -2357,7 +2706,7 @@ type StateAppendRequest struct { func (x *StateAppendRequest) Reset() { *x = StateAppendRequest{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[27] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2370,7 +2719,7 @@ func (x *StateAppendRequest) String() string { func (*StateAppendRequest) ProtoMessage() {} func (x *StateAppendRequest) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[27] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[31] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2383,7 +2732,7 @@ func (x *StateAppendRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StateAppendRequest.ProtoReflect.Descriptor instead. func (*StateAppendRequest) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{27} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{31} } func (x *StateAppendRequest) GetData() []byte { @@ -2403,7 +2752,7 @@ type StateAppendResponse struct { func (x *StateAppendResponse) Reset() { *x = StateAppendResponse{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[28] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2416,7 +2765,7 @@ func (x *StateAppendResponse) String() string { func (*StateAppendResponse) ProtoMessage() {} func (x *StateAppendResponse) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[28] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[32] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2429,7 +2778,7 @@ func (x *StateAppendResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use StateAppendResponse.ProtoReflect.Descriptor instead. func (*StateAppendResponse) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{28} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{32} } // A request to clear state. @@ -2442,7 +2791,7 @@ type StateClearRequest struct { func (x *StateClearRequest) Reset() { *x = StateClearRequest{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[29] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2455,7 +2804,7 @@ func (x *StateClearRequest) String() string { func (*StateClearRequest) ProtoMessage() {} func (x *StateClearRequest) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[29] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2468,7 +2817,7 @@ func (x *StateClearRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StateClearRequest.ProtoReflect.Descriptor instead. func (*StateClearRequest) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{29} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{33} } // A response to clear state. @@ -2481,7 +2830,7 @@ type StateClearResponse struct { func (x *StateClearResponse) Reset() { *x = StateClearResponse{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[30] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2494,7 +2843,7 @@ func (x *StateClearResponse) String() string { func (*StateClearResponse) ProtoMessage() {} func (x *StateClearResponse) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[30] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2507,7 +2856,7 @@ func (x *StateClearResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use StateClearResponse.ProtoReflect.Descriptor instead. func (*StateClearResponse) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{30} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{34} } // A log entry @@ -2542,12 +2891,15 @@ type LogEntry struct { LogLocation string `protobuf:"bytes,7,opt,name=log_location,json=logLocation,proto3" json:"log_location,omitempty"` // (Optional) The name of the thread this log statement is associated with. Thread string `protobuf:"bytes,8,opt,name=thread,proto3" json:"thread,omitempty"` + // (Optional) Additional structured data to log. + // Keys are limited to these characters: [a-zA-Z_-] + CustomData *structpb.Struct `protobuf:"bytes,9,opt,name=custom_data,json=customData,proto3" json:"custom_data,omitempty"` } func (x *LogEntry) Reset() { *x = LogEntry{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[31] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2560,7 +2912,7 @@ func (x *LogEntry) String() string { func (*LogEntry) ProtoMessage() {} func (x *LogEntry) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[31] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2573,7 +2925,7 @@ func (x *LogEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use LogEntry.ProtoReflect.Descriptor instead. func (*LogEntry) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{31} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{35} } func (x *LogEntry) GetSeverity() LogEntry_Severity_Enum { @@ -2632,6 +2984,13 @@ func (x *LogEntry) GetThread() string { return "" } +func (x *LogEntry) GetCustomData() *structpb.Struct { + if x != nil { + return x.CustomData + } + return nil +} + type LogControl struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2641,7 +3000,7 @@ type LogControl struct { func (x *LogControl) Reset() { *x = LogControl{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[32] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2654,7 +3013,7 @@ func (x *LogControl) String() string { func (*LogControl) ProtoMessage() {} func (x *LogControl) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[32] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[36] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2667,7 +3026,7 @@ func (x *LogControl) ProtoReflect() protoreflect.Message { // Deprecated: Use LogControl.ProtoReflect.Descriptor instead. func (*LogControl) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{32} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{36} } type StartWorkerRequest struct { @@ -2686,7 +3045,7 @@ type StartWorkerRequest struct { func (x *StartWorkerRequest) Reset() { *x = StartWorkerRequest{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[33] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2699,7 +3058,7 @@ func (x *StartWorkerRequest) String() string { func (*StartWorkerRequest) ProtoMessage() {} func (x *StartWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[33] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[37] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2712,7 +3071,7 @@ func (x *StartWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StartWorkerRequest.ProtoReflect.Descriptor instead. func (*StartWorkerRequest) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{33} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{37} } func (x *StartWorkerRequest) GetWorkerId() string { @@ -2768,7 +3127,7 @@ type StartWorkerResponse struct { func (x *StartWorkerResponse) Reset() { *x = StartWorkerResponse{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[34] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2781,7 +3140,7 @@ func (x *StartWorkerResponse) String() string { func (*StartWorkerResponse) ProtoMessage() {} func (x *StartWorkerResponse) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[34] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2794,7 +3153,7 @@ func (x *StartWorkerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use StartWorkerResponse.ProtoReflect.Descriptor instead. func (*StartWorkerResponse) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{34} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{38} } func (x *StartWorkerResponse) GetError() string { @@ -2815,7 +3174,7 @@ type StopWorkerRequest struct { func (x *StopWorkerRequest) Reset() { *x = StopWorkerRequest{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[35] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2828,7 +3187,7 @@ func (x *StopWorkerRequest) String() string { func (*StopWorkerRequest) ProtoMessage() {} func (x *StopWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[35] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2841,7 +3200,7 @@ func (x *StopWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StopWorkerRequest.ProtoReflect.Descriptor instead. func (*StopWorkerRequest) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{35} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{39} } func (x *StopWorkerRequest) GetWorkerId() string { @@ -2862,7 +3221,7 @@ type StopWorkerResponse struct { func (x *StopWorkerResponse) Reset() { *x = StopWorkerResponse{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[36] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2875,7 +3234,7 @@ func (x *StopWorkerResponse) String() string { func (*StopWorkerResponse) ProtoMessage() {} func (x *StopWorkerResponse) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[36] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2888,7 +3247,7 @@ func (x *StopWorkerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use StopWorkerResponse.ProtoReflect.Descriptor instead. func (*StopWorkerResponse) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{36} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{40} } func (x *StopWorkerResponse) GetError() string { @@ -2912,7 +3271,7 @@ type WorkerStatusRequest struct { func (x *WorkerStatusRequest) Reset() { *x = WorkerStatusRequest{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[37] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2925,7 +3284,7 @@ func (x *WorkerStatusRequest) String() string { func (*WorkerStatusRequest) ProtoMessage() {} func (x *WorkerStatusRequest) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[37] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2938,7 +3297,7 @@ func (x *WorkerStatusRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerStatusRequest.ProtoReflect.Descriptor instead. func (*WorkerStatusRequest) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{37} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{41} } func (x *WorkerStatusRequest) GetId() string { @@ -2969,7 +3328,7 @@ type WorkerStatusResponse struct { func (x *WorkerStatusResponse) Reset() { *x = WorkerStatusResponse{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[38] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2982,7 +3341,7 @@ func (x *WorkerStatusResponse) String() string { func (*WorkerStatusResponse) ProtoMessage() {} func (x *WorkerStatusResponse) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[38] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[42] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2995,7 +3354,7 @@ func (x *WorkerStatusResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerStatusResponse.ProtoReflect.Descriptor instead. func (*WorkerStatusResponse) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{38} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{42} } func (x *WorkerStatusResponse) GetId() string { @@ -3019,45 +3378,38 @@ func (x *WorkerStatusResponse) GetStatusInfo() string { return "" } -// Contains the cache token and also defines the scope of what the token applies to. -// -// See https://s.apache.org/beam-fn-state-api-and-bundle-processing#heading=h.7ghoih5aig5m -// for additional details on how to use the cache token with the State API -// to cache data across bundle boundaries. -type ProcessBundleRequest_CacheToken struct { +type SampledElement_Exception struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // The scope of a cache token. - // - // Types that are assignable to Type: - // - // *ProcessBundleRequest_CacheToken_UserState_ - // *ProcessBundleRequest_CacheToken_SideInput_ - Type isProcessBundleRequest_CacheToken_Type `protobuf_oneof:"type"` - // An opaque token used with the StateKey to create a globally unique - // identifier. - Token []byte `protobuf:"bytes,10,opt,name=token,proto3" json:"token,omitempty"` + // (Required) The instruction ID of the associated ProcessBundleRequest. + InstructionId string `protobuf:"bytes,1,opt,name=instruction_id,json=instructionId,proto3" json:"instruction_id,omitempty"` + // (Required) The transform ID of the executing PTransform during the + // exception. + TransformId string `protobuf:"bytes,2,opt,name=transform_id,json=transformId,proto3" json:"transform_id,omitempty"` + // (Required) The error message to be displayed to the user. Can use the + // other fields to query for contextual logs. + Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"` } -func (x *ProcessBundleRequest_CacheToken) Reset() { - *x = ProcessBundleRequest_CacheToken{} +func (x *SampledElement_Exception) Reset() { + *x = SampledElement_Exception{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[46] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *ProcessBundleRequest_CacheToken) String() string { +func (x *SampledElement_Exception) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ProcessBundleRequest_CacheToken) ProtoMessage() {} +func (*SampledElement_Exception) ProtoMessage() {} -func (x *ProcessBundleRequest_CacheToken) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[46] +func (x *SampledElement_Exception) ProtoReflect() protoreflect.Message { + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[43] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3068,79 +3420,58 @@ func (x *ProcessBundleRequest_CacheToken) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ProcessBundleRequest_CacheToken.ProtoReflect.Descriptor instead. -func (*ProcessBundleRequest_CacheToken) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{11, 0} -} - -func (m *ProcessBundleRequest_CacheToken) GetType() isProcessBundleRequest_CacheToken_Type { - if m != nil { - return m.Type - } - return nil +// Deprecated: Use SampledElement_Exception.ProtoReflect.Descriptor instead. +func (*SampledElement_Exception) Descriptor() ([]byte, []int) { + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{6, 0} } -func (x *ProcessBundleRequest_CacheToken) GetUserState() *ProcessBundleRequest_CacheToken_UserState { - if x, ok := x.GetType().(*ProcessBundleRequest_CacheToken_UserState_); ok { - return x.UserState +func (x *SampledElement_Exception) GetInstructionId() string { + if x != nil { + return x.InstructionId } - return nil + return "" } -func (x *ProcessBundleRequest_CacheToken) GetSideInput() *ProcessBundleRequest_CacheToken_SideInput { - if x, ok := x.GetType().(*ProcessBundleRequest_CacheToken_SideInput_); ok { - return x.SideInput +func (x *SampledElement_Exception) GetTransformId() string { + if x != nil { + return x.TransformId } - return nil + return "" } -func (x *ProcessBundleRequest_CacheToken) GetToken() []byte { +func (x *SampledElement_Exception) GetError() string { if x != nil { - return x.Token + return x.Error } - return nil -} - -type isProcessBundleRequest_CacheToken_Type interface { - isProcessBundleRequest_CacheToken_Type() -} - -type ProcessBundleRequest_CacheToken_UserState_ struct { - UserState *ProcessBundleRequest_CacheToken_UserState `protobuf:"bytes,1,opt,name=user_state,json=userState,proto3,oneof"` -} - -type ProcessBundleRequest_CacheToken_SideInput_ struct { - SideInput *ProcessBundleRequest_CacheToken_SideInput `protobuf:"bytes,2,opt,name=side_input,json=sideInput,proto3,oneof"` + return "" } -func (*ProcessBundleRequest_CacheToken_UserState_) isProcessBundleRequest_CacheToken_Type() {} - -func (*ProcessBundleRequest_CacheToken_SideInput_) isProcessBundleRequest_CacheToken_Type() {} - -// A flag to indicate a cache token is valid for all user state. -type ProcessBundleRequest_CacheToken_UserState struct { +type SampleDataResponse_ElementList struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + + // Required. The individual elements sampled from a PCollection. + Elements []*SampledElement `protobuf:"bytes,1,rep,name=elements,proto3" json:"elements,omitempty"` } -func (x *ProcessBundleRequest_CacheToken_UserState) Reset() { - *x = ProcessBundleRequest_CacheToken_UserState{} +func (x *SampleDataResponse_ElementList) Reset() { + *x = SampleDataResponse_ElementList{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[47] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *ProcessBundleRequest_CacheToken_UserState) String() string { +func (x *SampleDataResponse_ElementList) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ProcessBundleRequest_CacheToken_UserState) ProtoMessage() {} +func (*SampleDataResponse_ElementList) ProtoMessage() {} -func (x *ProcessBundleRequest_CacheToken_UserState) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[47] +func (x *SampleDataResponse_ElementList) ProtoReflect() protoreflect.Message { + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[44] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3151,12 +3482,156 @@ func (x *ProcessBundleRequest_CacheToken_UserState) ProtoReflect() protoreflect. return mi.MessageOf(x) } -// Deprecated: Use ProcessBundleRequest_CacheToken_UserState.ProtoReflect.Descriptor instead. -func (*ProcessBundleRequest_CacheToken_UserState) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{11, 0, 0} +// Deprecated: Use SampleDataResponse_ElementList.ProtoReflect.Descriptor instead. +func (*SampleDataResponse_ElementList) Descriptor() ([]byte, []int) { + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{7, 0} } -// A flag to indicate a cache token is valid for a side input. +func (x *SampleDataResponse_ElementList) GetElements() []*SampledElement { + if x != nil { + return x.Elements + } + return nil +} + +// Contains the cache token and also defines the scope of what the token applies to. +// +// See https://s.apache.org/beam-fn-state-api-and-bundle-processing#heading=h.7ghoih5aig5m +// for additional details on how to use the cache token with the State API +// to cache data across bundle boundaries. +type ProcessBundleRequest_CacheToken struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The scope of a cache token. + // + // Types that are assignable to Type: + // + // *ProcessBundleRequest_CacheToken_UserState_ + // *ProcessBundleRequest_CacheToken_SideInput_ + Type isProcessBundleRequest_CacheToken_Type `protobuf_oneof:"type"` + // An opaque token used with the StateKey to create a globally unique + // identifier. + Token []byte `protobuf:"bytes,10,opt,name=token,proto3" json:"token,omitempty"` +} + +func (x *ProcessBundleRequest_CacheToken) Reset() { + *x = ProcessBundleRequest_CacheToken{} + if protoimpl.UnsafeEnabled { + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[53] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ProcessBundleRequest_CacheToken) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProcessBundleRequest_CacheToken) ProtoMessage() {} + +func (x *ProcessBundleRequest_CacheToken) ProtoReflect() protoreflect.Message { + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[53] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProcessBundleRequest_CacheToken.ProtoReflect.Descriptor instead. +func (*ProcessBundleRequest_CacheToken) Descriptor() ([]byte, []int) { + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{15, 0} +} + +func (m *ProcessBundleRequest_CacheToken) GetType() isProcessBundleRequest_CacheToken_Type { + if m != nil { + return m.Type + } + return nil +} + +func (x *ProcessBundleRequest_CacheToken) GetUserState() *ProcessBundleRequest_CacheToken_UserState { + if x, ok := x.GetType().(*ProcessBundleRequest_CacheToken_UserState_); ok { + return x.UserState + } + return nil +} + +func (x *ProcessBundleRequest_CacheToken) GetSideInput() *ProcessBundleRequest_CacheToken_SideInput { + if x, ok := x.GetType().(*ProcessBundleRequest_CacheToken_SideInput_); ok { + return x.SideInput + } + return nil +} + +func (x *ProcessBundleRequest_CacheToken) GetToken() []byte { + if x != nil { + return x.Token + } + return nil +} + +type isProcessBundleRequest_CacheToken_Type interface { + isProcessBundleRequest_CacheToken_Type() +} + +type ProcessBundleRequest_CacheToken_UserState_ struct { + UserState *ProcessBundleRequest_CacheToken_UserState `protobuf:"bytes,1,opt,name=user_state,json=userState,proto3,oneof"` +} + +type ProcessBundleRequest_CacheToken_SideInput_ struct { + SideInput *ProcessBundleRequest_CacheToken_SideInput `protobuf:"bytes,2,opt,name=side_input,json=sideInput,proto3,oneof"` +} + +func (*ProcessBundleRequest_CacheToken_UserState_) isProcessBundleRequest_CacheToken_Type() {} + +func (*ProcessBundleRequest_CacheToken_SideInput_) isProcessBundleRequest_CacheToken_Type() {} + +// A flag to indicate a cache token is valid for all user state. +type ProcessBundleRequest_CacheToken_UserState struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *ProcessBundleRequest_CacheToken_UserState) Reset() { + *x = ProcessBundleRequest_CacheToken_UserState{} + if protoimpl.UnsafeEnabled { + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[54] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ProcessBundleRequest_CacheToken_UserState) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProcessBundleRequest_CacheToken_UserState) ProtoMessage() {} + +func (x *ProcessBundleRequest_CacheToken_UserState) ProtoReflect() protoreflect.Message { + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[54] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProcessBundleRequest_CacheToken_UserState.ProtoReflect.Descriptor instead. +func (*ProcessBundleRequest_CacheToken_UserState) Descriptor() ([]byte, []int) { + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{15, 0, 0} +} + +// A flag to indicate a cache token is valid for a side input. type ProcessBundleRequest_CacheToken_SideInput struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -3171,7 +3646,7 @@ type ProcessBundleRequest_CacheToken_SideInput struct { func (x *ProcessBundleRequest_CacheToken_SideInput) Reset() { *x = ProcessBundleRequest_CacheToken_SideInput{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[48] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3184,7 +3659,7 @@ func (x *ProcessBundleRequest_CacheToken_SideInput) String() string { func (*ProcessBundleRequest_CacheToken_SideInput) ProtoMessage() {} func (x *ProcessBundleRequest_CacheToken_SideInput) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[48] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[55] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3197,7 +3672,7 @@ func (x *ProcessBundleRequest_CacheToken_SideInput) ProtoReflect() protoreflect. // Deprecated: Use ProcessBundleRequest_CacheToken_SideInput.ProtoReflect.Descriptor instead. func (*ProcessBundleRequest_CacheToken_SideInput) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{11, 0, 1} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{15, 0, 1} } func (x *ProcessBundleRequest_CacheToken_SideInput) GetTransformId() string { @@ -3238,7 +3713,7 @@ type ProcessBundleSplitRequest_DesiredSplit struct { func (x *ProcessBundleSplitRequest_DesiredSplit) Reset() { *x = ProcessBundleSplitRequest_DesiredSplit{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[52] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3251,7 +3726,7 @@ func (x *ProcessBundleSplitRequest_DesiredSplit) String() string { func (*ProcessBundleSplitRequest_DesiredSplit) ProtoMessage() {} func (x *ProcessBundleSplitRequest_DesiredSplit) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[52] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[59] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3264,7 +3739,7 @@ func (x *ProcessBundleSplitRequest_DesiredSplit) ProtoReflect() protoreflect.Mes // Deprecated: Use ProcessBundleSplitRequest_DesiredSplit.ProtoReflect.Descriptor instead. func (*ProcessBundleSplitRequest_DesiredSplit) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{17, 0} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{21, 0} } func (x *ProcessBundleSplitRequest_DesiredSplit) GetFractionOfRemainder() float64 { @@ -3331,7 +3806,7 @@ type ProcessBundleSplitResponse_ChannelSplit struct { func (x *ProcessBundleSplitResponse_ChannelSplit) Reset() { *x = ProcessBundleSplitResponse_ChannelSplit{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[54] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3344,7 +3819,7 @@ func (x *ProcessBundleSplitResponse_ChannelSplit) String() string { func (*ProcessBundleSplitResponse_ChannelSplit) ProtoMessage() {} func (x *ProcessBundleSplitResponse_ChannelSplit) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[54] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[61] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3357,7 +3832,7 @@ func (x *ProcessBundleSplitResponse_ChannelSplit) ProtoReflect() protoreflect.Me // Deprecated: Use ProcessBundleSplitResponse_ChannelSplit.ProtoReflect.Descriptor instead. func (*ProcessBundleSplitResponse_ChannelSplit) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{18, 0} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{22, 0} } func (x *ProcessBundleSplitResponse_ChannelSplit) GetTransformId() string { @@ -3412,7 +3887,7 @@ type Elements_Data struct { func (x *Elements_Data) Reset() { *x = Elements_Data{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[55] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3425,7 +3900,7 @@ func (x *Elements_Data) String() string { func (*Elements_Data) ProtoMessage() {} func (x *Elements_Data) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[55] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[62] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3438,7 +3913,7 @@ func (x *Elements_Data) ProtoReflect() protoreflect.Message { // Deprecated: Use Elements_Data.ProtoReflect.Descriptor instead. func (*Elements_Data) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{21, 0} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{25, 0} } func (x *Elements_Data) GetInstructionId() string { @@ -3498,7 +3973,7 @@ type Elements_Timers struct { func (x *Elements_Timers) Reset() { *x = Elements_Timers{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[56] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[63] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3511,7 +3986,7 @@ func (x *Elements_Timers) String() string { func (*Elements_Timers) ProtoMessage() {} func (x *Elements_Timers) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[56] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[63] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3524,7 +3999,7 @@ func (x *Elements_Timers) ProtoReflect() protoreflect.Message { // Deprecated: Use Elements_Timers.ProtoReflect.Descriptor instead. func (*Elements_Timers) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{21, 1} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{25, 1} } func (x *Elements_Timers) GetInstructionId() string { @@ -3580,7 +4055,7 @@ type StateKey_Runner struct { func (x *StateKey_Runner) Reset() { *x = StateKey_Runner{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[57] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[64] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3593,7 +4068,7 @@ func (x *StateKey_Runner) String() string { func (*StateKey_Runner) ProtoMessage() {} func (x *StateKey_Runner) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[57] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[64] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3606,7 +4081,7 @@ func (x *StateKey_Runner) ProtoReflect() protoreflect.Message { // Deprecated: Use StateKey_Runner.ProtoReflect.Descriptor instead. func (*StateKey_Runner) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{24, 0} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{28, 0} } func (x *StateKey_Runner) GetKey() []byte { @@ -3644,7 +4119,7 @@ type StateKey_IterableSideInput struct { func (x *StateKey_IterableSideInput) Reset() { *x = StateKey_IterableSideInput{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[58] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[65] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3657,7 +4132,7 @@ func (x *StateKey_IterableSideInput) String() string { func (*StateKey_IterableSideInput) ProtoMessage() {} func (x *StateKey_IterableSideInput) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[58] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[65] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3670,7 +4145,7 @@ func (x *StateKey_IterableSideInput) ProtoReflect() protoreflect.Message { // Deprecated: Use StateKey_IterableSideInput.ProtoReflect.Descriptor instead. func (*StateKey_IterableSideInput) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{24, 1} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{28, 1} } func (x *StateKey_IterableSideInput) GetTransformId() string { @@ -3725,7 +4200,7 @@ type StateKey_MultimapSideInput struct { func (x *StateKey_MultimapSideInput) Reset() { *x = StateKey_MultimapSideInput{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[59] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[66] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3738,7 +4213,7 @@ func (x *StateKey_MultimapSideInput) String() string { func (*StateKey_MultimapSideInput) ProtoMessage() {} func (x *StateKey_MultimapSideInput) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[59] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[66] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3751,7 +4226,7 @@ func (x *StateKey_MultimapSideInput) ProtoReflect() protoreflect.Message { // Deprecated: Use StateKey_MultimapSideInput.ProtoReflect.Descriptor instead. func (*StateKey_MultimapSideInput) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{24, 2} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{28, 2} } func (x *StateKey_MultimapSideInput) GetTransformId() string { @@ -3810,7 +4285,7 @@ type StateKey_MultimapKeysSideInput struct { func (x *StateKey_MultimapKeysSideInput) Reset() { *x = StateKey_MultimapKeysSideInput{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[60] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[67] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3823,7 +4298,7 @@ func (x *StateKey_MultimapKeysSideInput) String() string { func (*StateKey_MultimapKeysSideInput) ProtoMessage() {} func (x *StateKey_MultimapKeysSideInput) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[60] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[67] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3836,7 +4311,7 @@ func (x *StateKey_MultimapKeysSideInput) ProtoReflect() protoreflect.Message { // Deprecated: Use StateKey_MultimapKeysSideInput.ProtoReflect.Descriptor instead. func (*StateKey_MultimapKeysSideInput) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{24, 3} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{28, 3} } func (x *StateKey_MultimapKeysSideInput) GetTransformId() string { @@ -3888,7 +4363,7 @@ type StateKey_BagUserState struct { func (x *StateKey_BagUserState) Reset() { *x = StateKey_BagUserState{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[61] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[68] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3901,7 +4376,7 @@ func (x *StateKey_BagUserState) String() string { func (*StateKey_BagUserState) ProtoMessage() {} func (x *StateKey_BagUserState) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[61] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[68] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3914,7 +4389,7 @@ func (x *StateKey_BagUserState) ProtoReflect() protoreflect.Message { // Deprecated: Use StateKey_BagUserState.ProtoReflect.Descriptor instead. func (*StateKey_BagUserState) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{24, 4} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{28, 4} } func (x *StateKey_BagUserState) GetTransformId() string { @@ -3976,7 +4451,7 @@ type StateKey_MultimapKeysUserState struct { func (x *StateKey_MultimapKeysUserState) Reset() { *x = StateKey_MultimapKeysUserState{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[62] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[69] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3989,7 +4464,7 @@ func (x *StateKey_MultimapKeysUserState) String() string { func (*StateKey_MultimapKeysUserState) ProtoMessage() {} func (x *StateKey_MultimapKeysUserState) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[62] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[69] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4002,7 +4477,7 @@ func (x *StateKey_MultimapKeysUserState) ProtoReflect() protoreflect.Message { // Deprecated: Use StateKey_MultimapKeysUserState.ProtoReflect.Descriptor instead. func (*StateKey_MultimapKeysUserState) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{24, 5} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{28, 5} } func (x *StateKey_MultimapKeysUserState) GetTransformId() string { @@ -4063,7 +4538,7 @@ type StateKey_MultimapUserState struct { func (x *StateKey_MultimapUserState) Reset() { *x = StateKey_MultimapUserState{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[63] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[70] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4076,7 +4551,7 @@ func (x *StateKey_MultimapUserState) String() string { func (*StateKey_MultimapUserState) ProtoMessage() {} func (x *StateKey_MultimapUserState) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[63] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[70] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4089,7 +4564,7 @@ func (x *StateKey_MultimapUserState) ProtoReflect() protoreflect.Message { // Deprecated: Use StateKey_MultimapUserState.ProtoReflect.Descriptor instead. func (*StateKey_MultimapUserState) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{24, 6} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{28, 6} } func (x *StateKey_MultimapUserState) GetTransformId() string { @@ -4141,7 +4616,7 @@ type LogEntry_List struct { func (x *LogEntry_List) Reset() { *x = LogEntry_List{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[64] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[71] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4154,7 +4629,7 @@ func (x *LogEntry_List) String() string { func (*LogEntry_List) ProtoMessage() {} func (x *LogEntry_List) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[64] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[71] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4167,7 +4642,7 @@ func (x *LogEntry_List) ProtoReflect() protoreflect.Message { // Deprecated: Use LogEntry_List.ProtoReflect.Descriptor instead. func (*LogEntry_List) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{31, 0} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{35, 0} } func (x *LogEntry_List) GetLogEntries() []*LogEntry { @@ -4199,7 +4674,7 @@ type LogEntry_Severity struct { func (x *LogEntry_Severity) Reset() { *x = LogEntry_Severity{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[65] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4212,7 +4687,7 @@ func (x *LogEntry_Severity) String() string { func (*LogEntry_Severity) ProtoMessage() {} func (x *LogEntry_Severity) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[65] + mi := &file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[72] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4225,7 +4700,7 @@ func (x *LogEntry_Severity) ProtoReflect() protoreflect.Message { // Deprecated: Use LogEntry_Severity.ProtoReflect.Descriptor instead. func (*LogEntry_Severity) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{31, 1} + return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP(), []int{35, 1} } var File_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto protoreflect.FileDescriptor @@ -4249,853 +4724,925 @@ var file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDesc = []byt 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x20, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, - 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x22, 0x9a, 0x01, 0x0a, 0x0e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x47, 0x72, 0x70, 0x63, 0x50, - 0x6f, 0x72, 0x74, 0x12, 0x6d, 0x0a, 0x16, 0x61, 0x70, 0x69, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, - 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, - 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x14, 0x61, 0x70, - 0x69, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, - 0x6f, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x22, 0x64, 0x0a, - 0x21, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, - 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x3f, 0x0a, 0x1c, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x62, 0x75, - 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x5f, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, - 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, - 0x72, 0x49, 0x64, 0x22, 0xde, 0x06, 0x0a, 0x12, 0x49, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, - 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, - 0x64, 0x12, 0x65, 0x0a, 0x0e, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x62, 0x75, 0x6e, - 0x64, 0x6c, 0x65, 0x18, 0xe9, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x6f, 0x72, 0x67, - 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, - 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x70, 0x72, 0x6f, 0x63, 0x65, - 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x7e, 0x0a, 0x17, 0x70, 0x72, 0x6f, 0x63, - 0x65, 0x73, 0x73, 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x67, 0x72, - 0x65, 0x73, 0x73, 0x18, 0xea, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x43, 0x2e, 0x6f, 0x72, 0x67, + 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2f, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, + 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x73, + 0x0a, 0x0f, 0x46, 0x6e, 0x41, 0x70, 0x69, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, + 0x73, 0x22, 0x60, 0x0a, 0x06, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x12, 0x2c, 0x0a, 0x0b, 0x44, + 0x41, 0x54, 0x41, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x10, 0x00, 0x1a, 0x1b, 0xa2, 0xb4, + 0xfa, 0xc2, 0x05, 0x15, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x3a, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x3a, 0x76, 0x31, 0x12, 0x28, 0x0a, 0x09, 0x44, 0x41, 0x54, + 0x41, 0x5f, 0x53, 0x49, 0x4e, 0x4b, 0x10, 0x01, 0x1a, 0x19, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x13, + 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x3a, 0x73, 0x69, 0x6e, 0x6b, + 0x3a, 0x76, 0x31, 0x22, 0x9a, 0x01, 0x0a, 0x0e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x47, 0x72, + 0x70, 0x63, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x6d, 0x0a, 0x16, 0x61, 0x70, 0x69, 0x5f, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, + 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, + 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x70, 0x69, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, + 0x14, 0x61, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x6f, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, + 0x22, 0x64, 0x0a, 0x21, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, + 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3f, 0x0a, 0x1c, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, + 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, + 0x6f, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x70, 0x72, 0x6f, + 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x6f, 0x72, 0x49, 0x64, 0x22, 0xbc, 0x07, 0x0a, 0x12, 0x49, 0x6e, 0x73, 0x74, 0x72, + 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x25, 0x0a, + 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x65, 0x0a, 0x0e, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, + 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x18, 0xe9, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, 0x2e, + 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, + 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, + 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x70, 0x72, + 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x7e, 0x0a, 0x17, 0x70, + 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x70, 0x72, + 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0xea, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x43, 0x2e, + 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, + 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, + 0x64, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x48, 0x00, 0x52, 0x15, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, + 0x64, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x75, 0x0a, 0x14, 0x70, + 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x73, 0x70, + 0x6c, 0x69, 0x74, 0x18, 0xeb, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, - 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, - 0x00, 0x52, 0x15, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, - 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x75, 0x0a, 0x14, 0x70, 0x72, 0x6f, 0x63, - 0x65, 0x73, 0x73, 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x73, 0x70, 0x6c, 0x69, 0x74, - 0x18, 0xeb, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, + 0x53, 0x70, 0x6c, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x12, + 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x53, 0x70, 0x6c, + 0x69, 0x74, 0x12, 0x68, 0x0a, 0x0f, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x5f, 0x62, + 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x18, 0xec, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x6f, + 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x42, 0x75, 0x6e, + 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0e, 0x66, 0x69, + 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x73, 0x0a, 0x10, + 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, + 0x18, 0xed, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, - 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x53, 0x70, 0x6c, - 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x12, 0x70, 0x72, 0x6f, - 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x12, - 0x68, 0x0a, 0x0f, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x5f, 0x62, 0x75, 0x6e, 0x64, - 0x6c, 0x65, 0x18, 0xec, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, - 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, - 0x31, 0x2e, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0e, 0x66, 0x69, 0x6e, 0x61, 0x6c, - 0x69, 0x7a, 0x65, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x73, 0x0a, 0x10, 0x6d, 0x6f, 0x6e, - 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0xed, 0x07, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, - 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, - 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x6e, - 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x4d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0f, 0x6d, - 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x81, - 0x01, 0x0a, 0x18, 0x68, 0x61, 0x72, 0x6e, 0x65, 0x73, 0x73, 0x5f, 0x6d, 0x6f, 0x6e, 0x69, 0x74, - 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0xee, 0x07, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, - 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x61, 0x72, 0x6e, 0x65, - 0x73, 0x73, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x16, 0x68, 0x61, 0x72, 0x6e, - 0x65, 0x73, 0x73, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, - 0x6f, 0x73, 0x12, 0x55, 0x0a, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x18, 0xe8, - 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, + 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x4d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, + 0x52, 0x0f, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, + 0x73, 0x12, 0x81, 0x01, 0x0a, 0x18, 0x68, 0x61, 0x72, 0x6e, 0x65, 0x73, 0x73, 0x5f, 0x6d, 0x6f, + 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0xee, + 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, - 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, - 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, - 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x42, 0x09, 0x0a, 0x07, 0x72, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x22, 0xfd, 0x06, 0x0a, 0x13, 0x49, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x0e, - 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x66, 0x0a, 0x0e, 0x70, 0x72, 0x6f, - 0x63, 0x65, 0x73, 0x73, 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x18, 0xe9, 0x07, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, - 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, - 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x48, 0x00, 0x52, 0x0d, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, - 0x65, 0x12, 0x7f, 0x0a, 0x17, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x62, 0x75, 0x6e, - 0x64, 0x6c, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0xea, 0x07, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, + 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x61, + 0x72, 0x6e, 0x65, 0x73, 0x73, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, + 0x6e, 0x66, 0x6f, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x16, 0x68, + 0x61, 0x72, 0x6e, 0x65, 0x73, 0x73, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, + 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x5c, 0x0a, 0x0b, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x5f, + 0x64, 0x61, 0x74, 0x61, 0x18, 0xef, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x6f, 0x72, + 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, + 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0a, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x44, + 0x61, 0x74, 0x61, 0x12, 0x55, 0x0a, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x18, + 0xe8, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, + 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, + 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, + 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, + 0x52, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x42, 0x09, 0x0a, 0x07, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xdc, 0x07, 0x0a, 0x13, 0x49, 0x6e, 0x73, 0x74, 0x72, 0x75, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, + 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x66, 0x0a, 0x0e, 0x70, 0x72, + 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x18, 0xe9, 0x07, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, - 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x15, 0x70, 0x72, 0x6f, + 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x48, 0x00, 0x52, 0x0d, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, + 0x6c, 0x65, 0x12, 0x7f, 0x0a, 0x17, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x62, 0x75, + 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0xea, 0x07, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, + 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, - 0x73, 0x73, 0x12, 0x76, 0x0a, 0x14, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x62, 0x75, - 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x18, 0xeb, 0x07, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x41, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, + 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x15, 0x70, 0x72, + 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x67, 0x72, + 0x65, 0x73, 0x73, 0x12, 0x76, 0x0a, 0x14, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x62, + 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x18, 0xeb, 0x07, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, + 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, + 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x12, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, + 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x12, 0x69, 0x0a, 0x0f, 0x66, + 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x18, 0xec, + 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, + 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, + 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, + 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0e, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, + 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x74, 0x0a, 0x10, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, + 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0xed, 0x07, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x46, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, - 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x12, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, - 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x12, 0x69, 0x0a, 0x0f, 0x66, 0x69, - 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x18, 0xec, 0x07, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, - 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, - 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6e, - 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0e, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x42, - 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x74, 0x0a, 0x10, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, - 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0xed, 0x07, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x46, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, + 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0f, 0x6d, 0x6f, 0x6e, + 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x82, 0x01, 0x0a, + 0x18, 0x68, 0x61, 0x72, 0x6e, 0x65, 0x73, 0x73, 0x5f, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, + 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0xee, 0x07, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x45, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, - 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0f, 0x6d, 0x6f, 0x6e, 0x69, - 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x82, 0x01, 0x0a, 0x18, - 0x68, 0x61, 0x72, 0x6e, 0x65, 0x73, 0x73, 0x5f, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, - 0x6e, 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0xee, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x45, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, - 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x61, 0x72, 0x6e, 0x65, 0x73, 0x73, 0x4d, - 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x16, 0x68, 0x61, 0x72, 0x6e, 0x65, 0x73, - 0x73, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x61, 0x72, 0x6e, 0x65, 0x73, 0x73, + 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x16, 0x68, 0x61, 0x72, 0x6e, 0x65, + 0x73, 0x73, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, + 0x73, 0x12, 0x5d, 0x0a, 0x0b, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, + 0x18, 0xef, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, + 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, + 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x48, 0x00, 0x52, 0x0a, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x44, 0x61, 0x74, 0x61, 0x12, 0x56, 0x0a, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x18, 0xe8, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x42, 0x0a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1f, 0x0a, 0x1d, 0x48, 0x61, 0x72, 0x6e, 0x65, 0x73, 0x73, 0x4d, - 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xe8, 0x01, 0x0a, 0x1e, 0x48, 0x61, 0x72, 0x6e, 0x65, 0x73, - 0x73, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x82, 0x01, 0x0a, 0x0f, 0x6d, 0x6f, 0x6e, - 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x59, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x3c, 0x0a, 0x11, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x44, 0x61, + 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x70, 0x63, 0x6f, + 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0e, 0x70, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, + 0x64, 0x73, 0x22, 0xbd, 0x02, 0x0a, 0x0e, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x64, 0x45, 0x6c, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, + 0x45, 0x0a, 0x10, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0f, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x5d, 0x0a, 0x09, 0x65, 0x78, 0x63, 0x65, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, + 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x64, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x2e, 0x45, 0x78, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x65, 0x78, 0x63, 0x65, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x6b, 0x0a, 0x09, 0x45, 0x78, 0x63, 0x65, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, 0x73, 0x74, + 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, + 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x22, 0xf9, 0x02, 0x0a, 0x12, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x44, 0x61, 0x74, + 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x76, 0x0a, 0x0f, 0x65, 0x6c, 0x65, + 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x4d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x61, 0x72, 0x6e, 0x65, - 0x73, 0x73, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, - 0x72, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0e, 0x6d, - 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x1a, 0x41, 0x0a, - 0x13, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x22, 0x8d, 0x01, 0x0a, 0x0f, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x7a, 0x0a, 0x19, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, - 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, - 0x72, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, - 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, - 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, - 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x17, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, - 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, - 0x22, 0x12, 0x0a, 0x10, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x9d, 0x0b, 0x0a, 0x17, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x61, 0x6d, 0x70, 0x6c, + 0x65, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x45, 0x6c, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x0e, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, + 0x73, 0x1a, 0x60, 0x0a, 0x0b, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, 0x74, + 0x12, 0x51, 0x0a, 0x08, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, + 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x61, 0x6d, 0x70, 0x6c, + 0x65, 0x64, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x08, 0x65, 0x6c, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x1a, 0x88, 0x01, 0x0a, 0x13, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, + 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x5b, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x6f, + 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x44, 0x61, 0x74, 0x61, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x4c, + 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x1f, + 0x0a, 0x1d, 0x48, 0x61, 0x72, 0x6e, 0x65, 0x73, 0x73, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, + 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, + 0xe8, 0x01, 0x0a, 0x1e, 0x48, 0x61, 0x72, 0x6e, 0x65, 0x73, 0x73, 0x4d, 0x6f, 0x6e, 0x69, 0x74, + 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x82, 0x01, 0x0a, 0x0f, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, + 0x67, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x59, 0x2e, 0x6f, + 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x61, 0x72, 0x6e, 0x65, 0x73, 0x73, 0x4d, 0x6f, 0x6e, 0x69, + 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x61, + 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0e, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, + 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x1a, 0x41, 0x0a, 0x13, 0x4d, 0x6f, 0x6e, 0x69, 0x74, + 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x8d, 0x01, 0x0a, 0x0f, 0x52, + 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x7a, + 0x0a, 0x19, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, + 0x5f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x3e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, + 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, + 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, + 0x72, 0x52, 0x17, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, + 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x22, 0x12, 0x0a, 0x10, 0x52, 0x65, + 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x9d, + 0x0b, 0x0a, 0x17, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, + 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x6e, 0x0a, 0x0a, 0x74, 0x72, + 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4e, + 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, + 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, + 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, + 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x12, 0x74, 0x0a, 0x0c, 0x70, 0x63, + 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x50, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, + 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, - 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, - 0x12, 0x6e, 0x0a, 0x0a, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, - 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, - 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, - 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, - 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x73, - 0x12, 0x74, 0x0a, 0x0c, 0x70, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x50, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, - 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, - 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, - 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, - 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x50, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x70, 0x63, 0x6f, 0x6c, 0x6c, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x8a, 0x01, 0x0a, 0x14, 0x77, 0x69, 0x6e, 0x64, 0x6f, - 0x77, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x69, 0x65, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x57, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, - 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, - 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, - 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x69, 0x6e, 0x67, 0x53, - 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x13, - 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x69, 0x6e, 0x67, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, - 0x69, 0x65, 0x73, 0x12, 0x62, 0x0a, 0x06, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x4a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, - 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, - 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, - 0x74, 0x6f, 0x72, 0x2e, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x06, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x73, 0x12, 0x74, 0x0a, 0x0c, 0x65, 0x6e, 0x76, 0x69, 0x72, - 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x50, 0x2e, + 0x2e, 0x50, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x0c, 0x70, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x12, 0x8a, 0x01, 0x0a, 0x14, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x69, 0x6e, 0x67, 0x5f, 0x73, + 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x57, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, + 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, + 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, + 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x69, 0x6e, 0x67, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, + 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x13, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, + 0x69, 0x6e, 0x67, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x69, 0x65, 0x73, 0x12, 0x62, 0x0a, + 0x06, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, - 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x45, 0x6e, - 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x0c, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x78, 0x0a, - 0x1c, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x61, 0x70, 0x69, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, - 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, - 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x19, 0x73, 0x74, - 0x61, 0x74, 0x65, 0x41, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x12, 0x78, 0x0a, 0x1c, 0x74, 0x69, 0x6d, 0x65, 0x72, + 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x43, 0x6f, + 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x63, 0x6f, 0x64, 0x65, 0x72, + 0x73, 0x12, 0x74, 0x0a, 0x0c, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x50, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, + 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, + 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x65, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x78, 0x0a, 0x1c, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x61, 0x70, 0x69, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x64, 0x65, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, - 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x19, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x41, 0x70, 0x69, + 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x19, 0x73, 0x74, 0x61, 0x74, 0x65, 0x41, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, - 0x72, 0x1a, 0x6c, 0x0a, 0x0f, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x43, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, - 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, - 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x54, 0x72, 0x61, 0x6e, 0x73, - 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, - 0x6f, 0x0a, 0x11, 0x50, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x44, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, - 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, - 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x43, 0x6f, 0x6c, 0x6c, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x1a, 0x7c, 0x0a, 0x18, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x69, 0x6e, 0x67, 0x53, 0x74, 0x72, - 0x61, 0x74, 0x65, 0x67, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x4a, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, + 0x72, 0x12, 0x78, 0x0a, 0x1c, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x5f, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, + 0x72, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, + 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, + 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x70, 0x69, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, + 0x52, 0x19, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x41, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x1a, 0x6c, 0x0a, 0x0f, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x43, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x2d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, + 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x6f, 0x0a, 0x11, 0x50, 0x63, 0x6f, + 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x44, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x2e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, + 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x7c, 0x0a, 0x18, 0x57, 0x69, + 0x6e, 0x64, 0x6f, 0x77, 0x69, 0x6e, 0x67, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x69, 0x65, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x4a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, + 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, + 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x69, 0x6e, 0x64, + 0x6f, 0x77, 0x69, 0x6e, 0x67, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x63, 0x0a, 0x0b, 0x43, 0x6f, 0x64, 0x65, + 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x3e, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, + 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, + 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x64, + 0x65, 0x72, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x6f, 0x0a, + 0x11, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x44, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, + 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, + 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x9b, + 0x03, 0x0a, 0x11, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, + 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, + 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x07, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x7b, 0x0a, 0x11, + 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, + 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, + 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x57, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, + 0x6b, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x57, + 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x73, 0x12, 0x50, 0x0a, 0x0a, 0x69, 0x73, 0x5f, + 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x31, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x69, 0x6e, 0x67, 0x53, 0x74, 0x72, 0x61, 0x74, - 0x65, 0x67, 0x79, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x63, - 0x0a, 0x0b, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x3e, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, + 0x31, 0x2e, 0x49, 0x73, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x65, 0x64, 0x2e, 0x45, 0x6e, 0x75, 0x6d, + 0x52, 0x09, 0x69, 0x73, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x65, 0x64, 0x1a, 0x5f, 0x0a, 0x15, 0x4f, + 0x75, 0x74, 0x70, 0x75, 0x74, 0x57, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x30, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xc3, 0x01, 0x0a, + 0x18, 0x44, 0x65, 0x6c, 0x61, 0x79, 0x65, 0x64, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x41, 0x70, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x5a, 0x0a, 0x0b, 0x61, 0x70, 0x70, + 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, - 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x1a, 0x6f, 0x0a, 0x11, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, - 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x44, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, + 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x41, 0x70, 0x70, + 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4b, 0x0a, 0x14, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x12, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x44, 0x65, 0x6c, + 0x61, 0x79, 0x22, 0x83, 0x05, 0x0a, 0x14, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, + 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3f, 0x0a, 0x1c, 0x70, + 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x64, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x19, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, + 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x49, 0x64, 0x12, 0x69, 0x0a, 0x0c, + 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x46, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, + 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, + 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x43, 0x61, 0x63, 0x68, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x0b, 0x63, 0x61, 0x63, 0x68, + 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x4b, 0x0a, 0x08, 0x65, 0x6c, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, - 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, - 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x22, 0x9b, 0x03, 0x0a, 0x11, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x41, - 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, - 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x19, 0x0a, - 0x08, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x07, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6c, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x65, 0x6c, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x12, 0x7b, 0x0a, 0x11, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x77, 0x61, 0x74, - 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4e, 0x2e, - 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, - 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x41, 0x70, 0x70, 0x6c, - 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x57, 0x61, - 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, 0x6f, - 0x75, 0x74, 0x70, 0x75, 0x74, 0x57, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x73, 0x12, - 0x50, 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x65, 0x64, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x31, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, - 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, - 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x73, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x65, - 0x64, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x09, 0x69, 0x73, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x65, - 0x64, 0x1a, 0x5f, 0x0a, 0x15, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x57, 0x61, 0x74, 0x65, 0x72, - 0x6d, 0x61, 0x72, 0x6b, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x30, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x22, 0xc3, 0x01, 0x0a, 0x18, 0x44, 0x65, 0x6c, 0x61, 0x79, 0x65, 0x64, 0x42, 0x75, - 0x6e, 0x64, 0x6c, 0x65, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x5a, 0x0a, 0x0b, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, - 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, - 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x6e, - 0x64, 0x6c, 0x65, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0b, - 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4b, 0x0a, 0x14, 0x72, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x64, 0x65, - 0x6c, 0x61, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x12, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x54, - 0x69, 0x6d, 0x65, 0x44, 0x65, 0x6c, 0x61, 0x79, 0x22, 0x83, 0x05, 0x0a, 0x14, 0x50, 0x72, 0x6f, - 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x3f, 0x0a, 0x1c, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x62, 0x75, 0x6e, - 0x64, 0x6c, 0x65, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, - 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, - 0x49, 0x64, 0x12, 0x69, 0x0a, 0x0c, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, - 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x46, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, + 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x08, 0x65, 0x6c, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x73, 0x1a, 0xf1, 0x02, 0x0a, 0x0a, 0x43, 0x61, 0x63, 0x68, 0x65, 0x54, 0x6f, + 0x6b, 0x65, 0x6e, 0x12, 0x71, 0x0a, 0x0a, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x50, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, + 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, + 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x2e, + 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x48, 0x00, 0x52, 0x09, 0x75, 0x73, 0x65, + 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x71, 0x0a, 0x0a, 0x73, 0x69, 0x64, 0x65, 0x5f, 0x69, + 0x6e, 0x70, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x50, 0x2e, 0x6f, 0x72, 0x67, + 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x54, 0x6f, 0x6b, + 0x65, 0x6e, 0x2e, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x48, 0x00, 0x52, 0x09, + 0x73, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, + 0x65, 0x6e, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x1a, + 0x0b, 0x0a, 0x09, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x1a, 0x52, 0x0a, 0x09, + 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, + 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x0d, + 0x73, 0x69, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x49, 0x64, + 0x42, 0x06, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0xa3, 0x04, 0x0a, 0x15, 0x50, 0x72, 0x6f, + 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x66, 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x69, 0x64, 0x75, 0x61, 0x6c, 0x5f, 0x72, + 0x6f, 0x6f, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x6f, 0x72, 0x67, + 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x44, 0x65, 0x6c, 0x61, 0x79, 0x65, 0x64, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, + 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0d, 0x72, 0x65, 0x73, + 0x69, 0x64, 0x75, 0x61, 0x6c, 0x52, 0x6f, 0x6f, 0x74, 0x73, 0x12, 0x5c, 0x0a, 0x10, 0x6d, 0x6f, + 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, + 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, + 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, + 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0f, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, + 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x33, 0x0a, 0x15, 0x72, 0x65, 0x71, 0x75, + 0x69, 0x72, 0x65, 0x73, 0x5f, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, + 0x73, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x79, 0x0a, + 0x0f, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x64, 0x61, 0x74, 0x61, + 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x50, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, + 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, + 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, + 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x44, + 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0e, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, + 0x72, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x12, 0x4b, 0x0a, 0x08, 0x65, 0x6c, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, + 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x08, 0x65, 0x6c, 0x65, + 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x1a, 0x41, 0x0a, 0x13, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, + 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x22, 0x45, + 0x0a, 0x1c, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, + 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x25, + 0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0x4e, 0x0a, 0x1e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, + 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x6d, 0x6f, 0x6e, 0x69, 0x74, + 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x10, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, + 0x6e, 0x66, 0x6f, 0x49, 0x64, 0x22, 0xd6, 0x02, 0x0a, 0x1d, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, + 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5c, 0x0a, 0x10, 0x6d, 0x6f, 0x6e, 0x69, 0x74, + 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x31, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, + 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, + 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, + 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0f, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, + 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x81, 0x01, 0x0a, 0x0f, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, + 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x58, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, + 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, + 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, + 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0e, 0x6d, 0x6f, 0x6e, 0x69, 0x74, + 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x1a, 0x41, 0x0a, 0x13, 0x4d, 0x6f, 0x6e, + 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a, 0x04, 0x08, 0x01, + 0x10, 0x02, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0x9d, + 0x02, 0x0a, 0x1f, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, + 0x6f, 0x73, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x83, 0x01, 0x0a, 0x0f, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, + 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x5a, 0x2e, 0x6f, + 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, + 0x6e, 0x66, 0x6f, 0x73, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, + 0x6e, 0x66, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0e, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, + 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x74, 0x0a, 0x13, 0x4d, 0x6f, 0x6e, 0x69, + 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x47, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x31, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, + 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x81, + 0x04, 0x0a, 0x19, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, + 0x53, 0x70, 0x6c, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, + 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x49, 0x64, 0x12, 0x7a, 0x0a, 0x0e, 0x64, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x5f, 0x73, + 0x70, 0x6c, 0x69, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x53, 0x2e, 0x6f, 0x72, + 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, + 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, + 0x65, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x44, 0x65, + 0x73, 0x69, 0x72, 0x65, 0x64, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x0d, 0x64, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x1a, + 0xae, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x53, 0x70, 0x6c, 0x69, 0x74, + 0x12, 0x32, 0x0a, 0x15, 0x66, 0x72, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x66, 0x5f, + 0x72, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, + 0x13, 0x66, 0x72, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x66, 0x52, 0x65, 0x6d, 0x61, 0x69, + 0x6e, 0x64, 0x65, 0x72, 0x12, 0x30, 0x0a, 0x14, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x5f, + 0x73, 0x70, 0x6c, 0x69, 0x74, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x03, 0x52, 0x12, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x53, 0x70, 0x6c, 0x69, 0x74, + 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x12, 0x38, 0x0a, 0x18, 0x65, 0x73, 0x74, 0x69, 0x6d, 0x61, + 0x74, 0x65, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x16, 0x65, 0x73, 0x74, 0x69, 0x6d, 0x61, + 0x74, 0x65, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, + 0x1a, 0x8f, 0x01, 0x0a, 0x12, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x53, 0x70, 0x6c, 0x69, + 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x63, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, - 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, - 0x52, 0x0b, 0x63, 0x61, 0x63, 0x68, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x4b, 0x0a, - 0x08, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, - 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, - 0x52, 0x08, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x1a, 0xf1, 0x02, 0x0a, 0x0a, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x71, 0x0a, 0x0a, 0x75, 0x73, 0x65, - 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x50, 0x2e, - 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, - 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, - 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, - 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x2e, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x48, - 0x00, 0x52, 0x09, 0x75, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x71, 0x0a, 0x0a, - 0x73, 0x69, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x50, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, + 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x53, 0x70, + 0x6c, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x44, 0x65, 0x73, 0x69, 0x72, + 0x65, 0x64, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x22, 0xf6, 0x03, 0x0a, 0x1a, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, + 0x6e, 0x64, 0x6c, 0x65, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x5d, 0x0a, 0x0d, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x72, 0x6f, 0x6f, + 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, + 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, + 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x0c, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x52, 0x6f, 0x6f, 0x74, 0x73, + 0x12, 0x66, 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x69, 0x64, 0x75, 0x61, 0x6c, 0x5f, 0x72, 0x6f, 0x6f, + 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, + 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, + 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x44, 0x65, 0x6c, 0x61, 0x79, 0x65, 0x64, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x41, 0x70, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0d, 0x72, 0x65, 0x73, 0x69, 0x64, + 0x75, 0x61, 0x6c, 0x52, 0x6f, 0x6f, 0x74, 0x73, 0x12, 0x75, 0x0a, 0x0e, 0x63, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x5f, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x4e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, - 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x43, 0x61, - 0x63, 0x68, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x2e, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, - 0x75, 0x74, 0x48, 0x00, 0x52, 0x09, 0x73, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, - 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, - 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x1a, 0x0b, 0x0a, 0x09, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x1a, 0x52, 0x0a, 0x09, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, - 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, - 0x49, 0x64, 0x12, 0x22, 0x0a, 0x0d, 0x73, 0x69, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, - 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x69, 0x64, 0x65, 0x49, - 0x6e, 0x70, 0x75, 0x74, 0x49, 0x64, 0x42, 0x06, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0xa3, - 0x04, 0x0a, 0x15, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x69, - 0x64, 0x75, 0x61, 0x6c, 0x5f, 0x72, 0x6f, 0x6f, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x3f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, + 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x53, 0x70, 0x6c, 0x69, 0x74, + 0x52, 0x0d, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x1a, + 0x99, 0x01, 0x0a, 0x0c, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x53, 0x70, 0x6c, 0x69, 0x74, + 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, + 0x6d, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x70, 0x72, 0x69, 0x6d, + 0x61, 0x72, 0x79, 0x5f, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x12, 0x6c, 0x61, 0x73, 0x74, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x45, 0x6c, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x34, 0x0a, 0x16, 0x66, 0x69, 0x72, 0x73, 0x74, 0x5f, 0x72, + 0x65, 0x73, 0x69, 0x64, 0x75, 0x61, 0x6c, 0x5f, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x14, 0x66, 0x69, 0x72, 0x73, 0x74, 0x52, 0x65, 0x73, 0x69, + 0x64, 0x75, 0x61, 0x6c, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x22, 0x3e, 0x0a, 0x15, 0x46, + 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, + 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0x18, 0x0a, 0x16, 0x46, + 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xd6, 0x03, 0x0a, 0x08, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x73, 0x12, 0x48, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x34, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x6c, 0x61, 0x79, 0x65, 0x64, - 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x0d, 0x72, 0x65, 0x73, 0x69, 0x64, 0x75, 0x61, 0x6c, 0x52, 0x6f, 0x6f, 0x74, 0x73, - 0x12, 0x5c, 0x0a, 0x10, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x69, - 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6f, 0x72, 0x67, - 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, - 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, - 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0f, 0x6d, - 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x33, - 0x0a, 0x15, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x73, 0x5f, 0x66, 0x69, 0x6e, 0x61, 0x6c, - 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x72, - 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x73, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x79, 0x0a, 0x0f, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, - 0x67, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x50, 0x2e, 0x6f, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x73, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x4e, 0x0a, 0x06, + 0x74, 0x69, 0x6d, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, - 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, - 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0e, - 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x12, 0x4b, - 0x0a, 0x08, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, - 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x73, 0x52, 0x08, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x1a, 0x41, 0x0a, 0x13, 0x4d, - 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a, 0x04, - 0x08, 0x01, 0x10, 0x02, 0x22, 0x45, 0x0a, 0x1c, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, - 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, - 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0x4e, 0x0a, 0x1e, 0x4d, - 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x4d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2c, 0x0a, - 0x12, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x6d, 0x6f, 0x6e, 0x69, 0x74, - 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x49, 0x64, 0x22, 0xd6, 0x02, 0x0a, 0x1d, - 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x72, 0x6f, - 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5c, 0x0a, - 0x10, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, - 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, - 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, - 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0f, 0x6d, 0x6f, 0x6e, 0x69, - 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x81, 0x01, 0x0a, 0x0f, - 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, - 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x58, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, - 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, - 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, - 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x67, 0x72, - 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, - 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x0e, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x1a, - 0x41, 0x0a, 0x13, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, - 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x4a, 0x04, - 0x08, 0x04, 0x10, 0x05, 0x22, 0x9d, 0x02, 0x0a, 0x1f, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, - 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x83, 0x01, 0x0a, 0x0f, 0x6d, 0x6f, 0x6e, - 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x5a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, - 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, - 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, - 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0e, - 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x74, - 0x0a, 0x13, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x47, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, - 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, - 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, - 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x22, 0x81, 0x04, 0x0a, 0x19, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, - 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, 0x73, 0x74, - 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x7a, 0x0a, 0x0e, 0x64, 0x65, 0x73, - 0x69, 0x72, 0x65, 0x64, 0x5f, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x53, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, - 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, - 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x53, 0x70, 0x6c, 0x69, 0x74, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x64, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x53, - 0x70, 0x6c, 0x69, 0x74, 0x73, 0x1a, 0xae, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, - 0x64, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x12, 0x32, 0x0a, 0x15, 0x66, 0x72, 0x61, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x5f, 0x6f, 0x66, 0x5f, 0x72, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x64, 0x65, 0x72, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x13, 0x66, 0x72, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, - 0x66, 0x52, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x64, 0x65, 0x72, 0x12, 0x30, 0x0a, 0x14, 0x61, 0x6c, - 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x5f, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x5f, 0x70, 0x6f, 0x69, 0x6e, - 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x03, 0x52, 0x12, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, - 0x64, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x12, 0x38, 0x0a, 0x18, - 0x65, 0x73, 0x74, 0x69, 0x6d, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, - 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x16, - 0x65, 0x73, 0x74, 0x69, 0x6d, 0x61, 0x74, 0x65, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x45, 0x6c, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x1a, 0x8f, 0x01, 0x0a, 0x12, 0x44, 0x65, 0x73, 0x69, 0x72, - 0x65, 0x64, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x63, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4d, - 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, - 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, - 0x6e, 0x64, 0x6c, 0x65, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x2e, 0x44, 0x65, 0x73, 0x69, 0x72, 0x65, 0x64, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xf6, 0x03, 0x0a, 0x1a, 0x50, 0x72, 0x6f, - 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5d, 0x0a, 0x0d, 0x70, 0x72, 0x69, 0x6d, 0x61, - 0x72, 0x79, 0x5f, 0x72, 0x6f, 0x6f, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, - 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, - 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x41, 0x70, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, - 0x79, 0x52, 0x6f, 0x6f, 0x74, 0x73, 0x12, 0x66, 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x69, 0x64, 0x75, - 0x61, 0x6c, 0x5f, 0x72, 0x6f, 0x6f, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3f, - 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, - 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x6c, 0x61, 0x79, 0x65, 0x64, 0x42, 0x75, - 0x6e, 0x64, 0x6c, 0x65, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x0d, 0x72, 0x65, 0x73, 0x69, 0x64, 0x75, 0x61, 0x6c, 0x52, 0x6f, 0x6f, 0x74, 0x73, 0x12, 0x75, - 0x0a, 0x0e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x73, - 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, - 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, - 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, - 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x53, 0x70, 0x6c, 0x69, - 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, - 0x6c, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x52, 0x0d, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x53, - 0x70, 0x6c, 0x69, 0x74, 0x73, 0x1a, 0x99, 0x01, 0x0a, 0x0c, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, - 0x6c, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, - 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, - 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x6c, 0x61, 0x73, - 0x74, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x12, 0x6c, 0x61, 0x73, 0x74, 0x50, 0x72, 0x69, - 0x6d, 0x61, 0x72, 0x79, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x34, 0x0a, 0x16, 0x66, - 0x69, 0x72, 0x73, 0x74, 0x5f, 0x72, 0x65, 0x73, 0x69, 0x64, 0x75, 0x61, 0x6c, 0x5f, 0x65, 0x6c, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x14, 0x66, 0x69, 0x72, - 0x73, 0x74, 0x52, 0x65, 0x73, 0x69, 0x64, 0x75, 0x61, 0x6c, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x22, 0x3e, 0x0a, 0x15, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x42, 0x75, 0x6e, - 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, - 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, - 0x64, 0x22, 0x18, 0x0a, 0x16, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x42, 0x75, 0x6e, - 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xd1, 0x03, 0x0a, 0x08, - 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x48, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, - 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, - 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, - 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x52, 0x04, 0x64, 0x61, - 0x74, 0x61, 0x12, 0x4e, 0x0a, 0x06, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x72, 0x73, 0x52, 0x06, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x73, 0x1a, 0x81, 0x01, 0x0a, + 0x04, 0x44, 0x61, 0x74, 0x61, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x69, + 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, + 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, + 0x16, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x42, 0x02, 0x08, + 0x01, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x73, 0x5f, 0x6c, 0x61, + 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x69, 0x73, 0x4c, 0x61, 0x73, 0x74, + 0x1a, 0xab, 0x01, 0x0a, 0x06, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x69, + 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, + 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x26, 0x0a, 0x0f, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x5f, 0x66, + 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, + 0x74, 0x69, 0x6d, 0x65, 0x72, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x49, 0x64, 0x12, 0x16, 0x0a, + 0x06, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x74, + 0x69, 0x6d, 0x65, 0x72, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x73, 0x5f, 0x6c, 0x61, 0x73, 0x74, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x69, 0x73, 0x4c, 0x61, 0x73, 0x74, 0x22, 0x94, + 0x03, 0x0a, 0x0c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, + 0x25, 0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x4c, 0x0a, 0x09, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, + 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, + 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x52, 0x08, 0x73, 0x74, 0x61, 0x74, + 0x65, 0x4b, 0x65, 0x79, 0x12, 0x4b, 0x0a, 0x03, 0x67, 0x65, 0x74, 0x18, 0xe8, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6c, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x73, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x73, 0x52, 0x06, 0x74, 0x69, 0x6d, 0x65, - 0x72, 0x73, 0x1a, 0x7d, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, - 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, - 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, - 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, - 0x72, 0x6d, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x73, 0x5f, 0x6c, - 0x61, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x69, 0x73, 0x4c, 0x61, 0x73, - 0x74, 0x1a, 0xab, 0x01, 0x0a, 0x06, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x73, 0x12, 0x25, 0x0a, 0x0e, - 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, - 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, - 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x26, 0x0a, 0x0f, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x5f, - 0x66, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0d, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x49, 0x64, 0x12, 0x16, - 0x0a, 0x06, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, - 0x74, 0x69, 0x6d, 0x65, 0x72, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x73, 0x5f, 0x6c, 0x61, 0x73, - 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x69, 0x73, 0x4c, 0x61, 0x73, 0x74, 0x22, - 0x94, 0x03, 0x0a, 0x0c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, - 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x4c, 0x0a, 0x09, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, - 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, - 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x52, 0x08, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x4b, 0x0a, 0x03, 0x67, 0x65, 0x74, 0x18, 0xe8, 0x07, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, - 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x03, 0x67, - 0x65, 0x74, 0x12, 0x54, 0x0a, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x18, 0xe9, 0x07, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, - 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, - 0x52, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x12, 0x51, 0x0a, 0x05, 0x63, 0x6c, 0x65, 0x61, - 0x72, 0x18, 0xea, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, - 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, - 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, - 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x43, 0x6c, 0x65, 0x61, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x48, 0x00, 0x52, 0x05, 0x63, 0x6c, 0x65, 0x61, 0x72, 0x42, 0x09, 0x0a, 0x07, 0x72, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xba, 0x02, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x4c, - 0x0a, 0x03, 0x67, 0x65, 0x74, 0x18, 0xe8, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x03, 0x67, 0x65, + 0x74, 0x12, 0x54, 0x0a, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x18, 0xe9, 0x07, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, + 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, + 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x12, 0x51, 0x0a, 0x05, 0x63, 0x6c, 0x65, 0x61, 0x72, + 0x18, 0xea, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, + 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, + 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x43, 0x6c, 0x65, 0x61, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x48, 0x00, 0x52, 0x05, 0x63, 0x6c, 0x65, 0x61, 0x72, 0x42, 0x09, 0x0a, 0x07, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xba, 0x02, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x4c, 0x0a, + 0x03, 0x67, 0x65, 0x74, 0x18, 0xe8, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x72, + 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, + 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x03, 0x67, 0x65, 0x74, 0x12, 0x55, 0x0a, 0x06, 0x61, + 0x70, 0x70, 0x65, 0x6e, 0x64, 0x18, 0xe9, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x03, 0x67, 0x65, 0x74, 0x12, 0x55, 0x0a, 0x06, - 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x18, 0xe9, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, - 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, - 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x41, 0x70, 0x70, 0x65, 0x6e, - 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x06, 0x61, 0x70, 0x70, - 0x65, 0x6e, 0x64, 0x12, 0x52, 0x0a, 0x05, 0x63, 0x6c, 0x65, 0x61, 0x72, 0x18, 0xea, 0x07, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, - 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x43, 0x6c, 0x65, 0x61, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, - 0x52, 0x05, 0x63, 0x6c, 0x65, 0x61, 0x72, 0x42, 0x0a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0xea, 0x0c, 0x0a, 0x08, 0x53, 0x74, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, - 0x12, 0x50, 0x0a, 0x06, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x36, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, - 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x4b, 0x65, - 0x79, 0x2e, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x72, 0x75, 0x6e, 0x6e, - 0x65, 0x72, 0x12, 0x73, 0x0a, 0x13, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x5f, 0x73, - 0x69, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x41, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, - 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, - 0x2e, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, - 0x75, 0x74, 0x48, 0x00, 0x52, 0x11, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x53, 0x69, - 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x64, 0x0a, 0x0e, 0x62, 0x61, 0x67, 0x5f, 0x75, - 0x73, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x3c, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x06, 0x61, 0x70, 0x70, 0x65, + 0x6e, 0x64, 0x12, 0x52, 0x0a, 0x05, 0x63, 0x6c, 0x65, 0x61, 0x72, 0x18, 0xea, 0x07, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, + 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x43, 0x6c, 0x65, 0x61, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, + 0x05, 0x63, 0x6c, 0x65, 0x61, 0x72, 0x42, 0x0a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0xea, 0x0c, 0x0a, 0x08, 0x53, 0x74, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x12, + 0x50, 0x0a, 0x06, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x36, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, - 0x2e, 0x42, 0x61, 0x67, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x48, 0x00, 0x52, - 0x0c, 0x62, 0x61, 0x67, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x73, 0x0a, - 0x13, 0x69, 0x74, 0x65, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x64, 0x65, 0x5f, 0x69, - 0x6e, 0x70, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x6f, 0x72, 0x67, - 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, - 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x2e, 0x49, 0x74, 0x65, 0x72, - 0x61, 0x62, 0x6c, 0x65, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x48, 0x00, 0x52, - 0x11, 0x69, 0x74, 0x65, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, - 0x75, 0x74, 0x12, 0x80, 0x01, 0x0a, 0x18, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x5f, - 0x6b, 0x65, 0x79, 0x73, 0x5f, 0x73, 0x69, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, - 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, - 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x2e, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x4b, - 0x65, 0x79, 0x73, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x48, 0x00, 0x52, 0x15, - 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x4b, 0x65, 0x79, 0x73, 0x53, 0x69, 0x64, 0x65, - 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x80, 0x01, 0x0a, 0x18, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x6d, - 0x61, 0x70, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, - 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, - 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, - 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x2e, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x6d, - 0x61, 0x70, 0x4b, 0x65, 0x79, 0x73, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x48, - 0x00, 0x52, 0x15, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x4b, 0x65, 0x79, 0x73, 0x55, - 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x73, 0x0a, 0x13, 0x6d, 0x75, 0x6c, 0x74, - 0x69, 0x6d, 0x61, 0x70, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, - 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, - 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x2e, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x55, - 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x48, 0x00, 0x52, 0x11, 0x6d, 0x75, 0x6c, 0x74, - 0x69, 0x6d, 0x61, 0x70, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x1a, 0x1a, 0x0a, - 0x06, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x1a, 0x72, 0x0a, 0x11, 0x49, 0x74, 0x65, - 0x72, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x21, - 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, - 0x64, 0x12, 0x22, 0x0a, 0x0d, 0x73, 0x69, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, - 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x69, 0x64, 0x65, 0x49, 0x6e, - 0x70, 0x75, 0x74, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x1a, 0x84, 0x01, - 0x0a, 0x11, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, - 0x70, 0x75, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, - 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x0d, 0x73, 0x69, 0x64, 0x65, 0x5f, 0x69, - 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, - 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x77, 0x69, - 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x77, 0x69, 0x6e, 0x64, - 0x6f, 0x77, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x1a, 0x76, 0x0a, 0x15, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, - 0x4b, 0x65, 0x79, 0x73, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x21, 0x0a, + 0x2e, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x72, 0x75, 0x6e, 0x6e, 0x65, + 0x72, 0x12, 0x73, 0x0a, 0x13, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x5f, 0x73, 0x69, + 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, + 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, + 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x2e, + 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, + 0x74, 0x48, 0x00, 0x52, 0x11, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x53, 0x69, 0x64, + 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x64, 0x0a, 0x0e, 0x62, 0x61, 0x67, 0x5f, 0x75, 0x73, + 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, + 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, + 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x2e, + 0x42, 0x61, 0x67, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x48, 0x00, 0x52, 0x0c, + 0x62, 0x61, 0x67, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x73, 0x0a, 0x13, + 0x69, 0x74, 0x65, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x64, 0x65, 0x5f, 0x69, 0x6e, + 0x70, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x6f, 0x72, 0x67, 0x2e, + 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x2e, 0x49, 0x74, 0x65, 0x72, 0x61, + 0x62, 0x6c, 0x65, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x48, 0x00, 0x52, 0x11, + 0x69, 0x74, 0x65, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, + 0x74, 0x12, 0x80, 0x01, 0x0a, 0x18, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x5f, 0x6b, + 0x65, 0x79, 0x73, 0x5f, 0x73, 0x69, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, + 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x4b, 0x65, 0x79, 0x2e, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x4b, 0x65, + 0x79, 0x73, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x48, 0x00, 0x52, 0x15, 0x6d, + 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x4b, 0x65, 0x79, 0x73, 0x53, 0x69, 0x64, 0x65, 0x49, + 0x6e, 0x70, 0x75, 0x74, 0x12, 0x80, 0x01, 0x0a, 0x18, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, + 0x70, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, + 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, + 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, + 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x2e, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, + 0x70, 0x4b, 0x65, 0x79, 0x73, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x48, 0x00, + 0x52, 0x15, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x4b, 0x65, 0x79, 0x73, 0x55, 0x73, + 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x73, 0x0a, 0x13, 0x6d, 0x75, 0x6c, 0x74, 0x69, + 0x6d, 0x61, 0x70, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, + 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x4b, 0x65, 0x79, 0x2e, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x55, 0x73, + 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x48, 0x00, 0x52, 0x11, 0x6d, 0x75, 0x6c, 0x74, 0x69, + 0x6d, 0x61, 0x70, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x1a, 0x1a, 0x0a, 0x06, + 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x1a, 0x72, 0x0a, 0x11, 0x49, 0x74, 0x65, 0x72, + 0x61, 0x62, 0x6c, 0x65, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x0d, 0x73, 0x69, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x1a, 0x7f, 0x0a, 0x0c, - 0x42, 0x61, 0x67, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x21, 0x0a, 0x0c, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x1a, 0x84, 0x01, 0x0a, + 0x11, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, + 0x75, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, + 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x0d, 0x73, 0x69, 0x64, 0x65, 0x5f, 0x69, 0x6e, + 0x70, 0x75, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x69, + 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x77, 0x69, 0x6e, + 0x64, 0x6f, 0x77, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, + 0x77, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x1a, 0x76, 0x0a, 0x15, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x4b, + 0x65, 0x79, 0x73, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, - 0x22, 0x0a, 0x0d, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x75, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x1a, 0x88, 0x01, - 0x0a, 0x15, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x4b, 0x65, 0x79, 0x73, 0x55, 0x73, - 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, - 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, - 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x0d, 0x75, 0x73, - 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0b, 0x75, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x49, 0x64, 0x12, 0x16, - 0x0a, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, - 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x1a, 0x9d, 0x01, 0x0a, 0x11, 0x4d, 0x75, 0x6c, - 0x74, 0x69, 0x6d, 0x61, 0x70, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x21, - 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, - 0x64, 0x12, 0x22, 0x0a, 0x0d, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, - 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x75, 0x73, 0x65, 0x72, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x17, 0x0a, 0x07, 0x6d, 0x61, 0x70, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x06, 0x6d, 0x61, 0x70, 0x4b, 0x65, 0x79, 0x42, 0x06, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, - 0x22, 0x40, 0x0a, 0x0f, 0x53, 0x74, 0x61, 0x74, 0x65, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x12, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x11, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, - 0x65, 0x6e, 0x22, 0x55, 0x0a, 0x10, 0x53, 0x74, 0x61, 0x74, 0x65, 0x47, 0x65, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2d, 0x0a, 0x12, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, - 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x11, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x28, 0x0a, 0x12, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, - 0x61, 0x74, 0x61, 0x22, 0x15, 0x0a, 0x13, 0x53, 0x74, 0x61, 0x74, 0x65, 0x41, 0x70, 0x70, 0x65, - 0x6e, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x13, 0x0a, 0x11, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x43, 0x6c, 0x65, 0x61, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, - 0x14, 0x0a, 0x12, 0x53, 0x74, 0x61, 0x74, 0x65, 0x43, 0x6c, 0x65, 0x61, 0x72, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xa2, 0x04, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x59, 0x0a, 0x08, 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, - 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, - 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x67, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x2e, 0x45, - 0x6e, 0x75, 0x6d, 0x52, 0x08, 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x12, 0x38, 0x0a, - 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x72, 0x61, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x74, 0x72, 0x61, 0x63, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x72, - 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0d, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x21, - 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, - 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6c, 0x6f, 0x67, 0x4c, 0x6f, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x08, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x1a, 0x58, 0x0a, 0x04, - 0x4c, 0x69, 0x73, 0x74, 0x12, 0x50, 0x0a, 0x0b, 0x6c, 0x6f, 0x67, 0x5f, 0x65, 0x6e, 0x74, 0x72, - 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, - 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, - 0x31, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x6c, 0x6f, 0x67, 0x45, - 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x1a, 0x72, 0x0a, 0x08, 0x53, 0x65, 0x76, 0x65, 0x72, 0x69, - 0x74, 0x79, 0x22, 0x66, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, - 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x54, - 0x52, 0x41, 0x43, 0x45, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x44, 0x45, 0x42, 0x55, 0x47, 0x10, - 0x02, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x46, 0x4f, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x4e, - 0x4f, 0x54, 0x49, 0x43, 0x45, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x41, 0x52, 0x4e, 0x10, - 0x05, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x06, 0x12, 0x0c, 0x0a, 0x08, - 0x43, 0x52, 0x49, 0x54, 0x49, 0x43, 0x41, 0x4c, 0x10, 0x07, 0x22, 0x0c, 0x0a, 0x0a, 0x4c, 0x6f, - 0x67, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x22, 0xe1, 0x04, 0x0a, 0x12, 0x53, 0x74, 0x61, - 0x72, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, 0x64, 0x12, 0x62, 0x0a, 0x10, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, - 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, - 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x70, 0x69, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, - 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x12, 0x62, 0x0a, 0x10, 0x6c, 0x6f, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x65, 0x6e, 0x64, 0x70, - 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x72, 0x67, - 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, - 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, - 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, - 0x74, 0x6f, 0x72, 0x52, 0x0f, 0x6c, 0x6f, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x45, 0x6e, 0x64, 0x70, - 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x64, 0x0a, 0x11, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x22, 0x0a, 0x0d, 0x73, 0x69, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, + 0x74, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x1a, 0x7f, 0x0a, 0x0c, 0x42, + 0x61, 0x67, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x74, + 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x22, + 0x0a, 0x0d, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x69, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x75, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x1a, 0x88, 0x01, 0x0a, + 0x15, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x4b, 0x65, 0x79, 0x73, 0x55, 0x73, 0x65, + 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, + 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, + 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x0d, 0x75, 0x73, 0x65, + 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0b, 0x75, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, + 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x77, + 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x1a, 0x9d, 0x01, 0x0a, 0x11, 0x4d, 0x75, 0x6c, 0x74, + 0x69, 0x6d, 0x61, 0x70, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x21, 0x0a, + 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, + 0x12, 0x22, 0x0a, 0x0d, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x69, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x75, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x17, + 0x0a, 0x07, 0x6d, 0x61, 0x70, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x06, 0x6d, 0x61, 0x70, 0x4b, 0x65, 0x79, 0x42, 0x06, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, + 0x40, 0x0a, 0x0f, 0x53, 0x74, 0x61, 0x74, 0x65, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x12, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x11, + 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, + 0x6e, 0x22, 0x55, 0x0a, 0x10, 0x53, 0x74, 0x61, 0x74, 0x65, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2d, 0x0a, 0x12, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x11, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, + 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x28, 0x0a, 0x12, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, + 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, + 0x74, 0x61, 0x22, 0x15, 0x0a, 0x13, 0x53, 0x74, 0x61, 0x74, 0x65, 0x41, 0x70, 0x70, 0x65, 0x6e, + 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x13, 0x0a, 0x11, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x43, 0x6c, 0x65, 0x61, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x14, + 0x0a, 0x12, 0x53, 0x74, 0x61, 0x74, 0x65, 0x43, 0x6c, 0x65, 0x61, 0x72, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xdc, 0x04, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x59, 0x0a, 0x08, 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x3d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, + 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x67, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x53, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x2e, 0x45, 0x6e, + 0x75, 0x6d, 0x52, 0x08, 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x12, 0x38, 0x0a, 0x09, + 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x12, 0x14, 0x0a, 0x05, 0x74, 0x72, 0x61, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x74, 0x72, 0x61, 0x63, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, + 0x69, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x21, 0x0a, + 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, + 0x12, 0x21, 0x0a, 0x0c, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6c, 0x6f, 0x67, 0x4c, 0x6f, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x38, 0x0a, 0x0b, 0x63, + 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x17, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x52, 0x0a, 0x63, 0x75, 0x73, 0x74, 0x6f, + 0x6d, 0x44, 0x61, 0x74, 0x61, 0x1a, 0x58, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x50, 0x0a, + 0x0b, 0x6c, 0x6f, 0x67, 0x5f, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, + 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x0a, 0x6c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x1a, + 0x72, 0x0a, 0x08, 0x53, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x22, 0x66, 0x0a, 0x04, 0x45, + 0x6e, 0x75, 0x6d, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x54, 0x52, 0x41, 0x43, 0x45, 0x10, 0x01, 0x12, + 0x09, 0x0a, 0x05, 0x44, 0x45, 0x42, 0x55, 0x47, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, + 0x46, 0x4f, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x4e, 0x4f, 0x54, 0x49, 0x43, 0x45, 0x10, 0x04, + 0x12, 0x08, 0x0a, 0x04, 0x57, 0x41, 0x52, 0x4e, 0x10, 0x05, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, + 0x52, 0x4f, 0x52, 0x10, 0x06, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x52, 0x49, 0x54, 0x49, 0x43, 0x41, + 0x4c, 0x10, 0x07, 0x22, 0x0c, 0x0a, 0x0a, 0x4c, 0x6f, 0x67, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x22, 0xe1, 0x04, 0x0a, 0x12, 0x53, 0x74, 0x61, 0x72, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, + 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, + 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, + 0x6b, 0x65, 0x72, 0x49, 0x64, 0x12, 0x62, 0x0a, 0x10, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, - 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x10, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, - 0x63, 0x74, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x66, 0x0a, 0x12, 0x70, 0x72, - 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, - 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, - 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x70, 0x69, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, - 0x11, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, - 0x6e, 0x74, 0x12, 0x5d, 0x0a, 0x06, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x0a, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, - 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, - 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, - 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x70, 0x61, 0x72, 0x61, 0x6d, - 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x2b, 0x0a, 0x13, - 0x53, 0x74, 0x61, 0x72, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x30, 0x0a, 0x11, 0x53, 0x74, 0x6f, - 0x70, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, - 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, 0x64, 0x22, 0x2a, 0x0a, 0x12, 0x53, - 0x74, 0x6f, 0x70, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x25, 0x0a, 0x13, 0x57, 0x6f, 0x72, 0x6b, 0x65, - 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, - 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x5d, - 0x0a, 0x14, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x1f, 0x0a, 0x0b, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0a, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x32, 0xc3, 0x02, - 0x0a, 0x0d, 0x42, 0x65, 0x61, 0x6d, 0x46, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x12, - 0x86, 0x01, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x3a, 0x2e, 0x6f, 0x72, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x62, 0x0a, 0x10, 0x6c, 0x6f, 0x67, + 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, + 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, + 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x0f, 0x6c, 0x6f, + 0x67, 0x67, 0x69, 0x6e, 0x67, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x64, 0x0a, + 0x11, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, + 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, + 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, + 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x70, 0x69, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, + 0x72, 0x52, 0x10, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x45, 0x6e, 0x64, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x12, 0x66, 0x0a, 0x12, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, + 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, + 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x11, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, + 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x5d, 0x0a, 0x06, 0x70, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0x39, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, - 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, - 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, - 0x49, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0xa8, 0x01, 0x0a, 0x1a, 0x47, 0x65, 0x74, - 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x12, 0x48, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, - 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, - 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, - 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, - 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x3e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, - 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, - 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, - 0x72, 0x22, 0x00, 0x32, 0x7c, 0x0a, 0x0a, 0x42, 0x65, 0x61, 0x6d, 0x46, 0x6e, 0x44, 0x61, 0x74, - 0x61, 0x12, 0x6e, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61, 0x12, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, - 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, - 0x31, 0x2e, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x1a, 0x2f, 0x2e, 0x6f, 0x72, 0x67, - 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, - 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x76, 0x31, 0x2e, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x00, 0x28, 0x01, 0x30, - 0x01, 0x32, 0x87, 0x01, 0x0a, 0x0b, 0x42, 0x65, 0x61, 0x6d, 0x46, 0x6e, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x78, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x33, 0x2e, 0x6f, 0x72, 0x67, + 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x06, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x50, 0x61, + 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x2b, 0x0a, 0x13, 0x53, 0x74, 0x61, 0x72, 0x74, 0x57, 0x6f, + 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x22, 0x30, 0x0a, 0x11, 0x53, 0x74, 0x6f, 0x70, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x65, + 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, + 0x65, 0x72, 0x49, 0x64, 0x22, 0x2a, 0x0a, 0x12, 0x53, 0x74, 0x6f, 0x70, 0x57, 0x6f, 0x72, 0x6b, + 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, + 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x22, 0x25, 0x0a, 0x13, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x5d, 0x0a, 0x14, 0x57, 0x6f, 0x72, 0x6b, 0x65, + 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, + 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, + 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x32, 0xc3, 0x02, 0x0a, 0x0d, 0x42, 0x65, 0x61, 0x6d, 0x46, + 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x86, 0x01, 0x0a, 0x07, 0x43, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x3a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, + 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x73, + 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x1a, 0x39, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, + 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x00, 0x28, 0x01, 0x30, + 0x01, 0x12, 0xa8, 0x01, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, + 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, + 0x12, 0x48, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, + 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x63, + 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, + 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x34, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, - 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x32, 0x89, 0x01, 0x0a, 0x0d, - 0x42, 0x65, 0x61, 0x6d, 0x46, 0x6e, 0x4c, 0x6f, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x12, 0x78, 0x0a, - 0x07, 0x4c, 0x6f, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x12, 0x34, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, + 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, + 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x22, 0x00, 0x32, 0x7c, 0x0a, 0x0a, + 0x42, 0x65, 0x61, 0x6d, 0x46, 0x6e, 0x44, 0x61, 0x74, 0x61, 0x12, 0x6e, 0x0a, 0x04, 0x44, 0x61, + 0x74, 0x61, 0x12, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, + 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6c, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x1a, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, + 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6c, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x73, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x32, 0x87, 0x01, 0x0a, 0x0b, 0x42, + 0x65, 0x61, 0x6d, 0x46, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x78, 0x0a, 0x05, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x12, 0x33, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, + 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x34, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, - 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x1a, 0x31, - 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, - 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x67, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x32, 0xa9, 0x02, 0x0a, 0x18, 0x42, 0x65, 0x61, 0x6d, - 0x46, 0x6e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, - 0x50, 0x6f, 0x6f, 0x6c, 0x12, 0x86, 0x01, 0x0a, 0x0b, 0x53, 0x74, 0x61, 0x72, 0x74, 0x57, 0x6f, - 0x72, 0x6b, 0x65, 0x72, 0x12, 0x39, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, - 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, - 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, - 0x72, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x3a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, - 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x57, 0x6f, 0x72, - 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x83, 0x01, - 0x0a, 0x0a, 0x53, 0x74, 0x6f, 0x70, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x12, 0x38, 0x2e, 0x6f, - 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, - 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x39, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, - 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, - 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, - 0x74, 0x6f, 0x70, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x32, 0xa4, 0x01, 0x0a, 0x12, 0x42, 0x65, 0x61, 0x6d, 0x46, 0x6e, 0x57, 0x6f, - 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x8d, 0x01, 0x0a, 0x0c, 0x57, - 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x3b, 0x2e, 0x6f, 0x72, - 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, - 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0x3a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, + 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x28, 0x01, 0x30, 0x01, 0x32, 0x89, 0x01, 0x0a, 0x0d, 0x42, 0x65, 0x61, 0x6d, 0x46, 0x6e, 0x4c, + 0x6f, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x12, 0x78, 0x0a, 0x07, 0x4c, 0x6f, 0x67, 0x67, 0x69, 0x6e, + 0x67, 0x12, 0x34, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, + 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, + 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, + 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x4c, 0x6f, 0x67, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, + 0x32, 0xa9, 0x02, 0x0a, 0x18, 0x42, 0x65, 0x61, 0x6d, 0x46, 0x6e, 0x45, 0x78, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x50, 0x6f, 0x6f, 0x6c, 0x12, 0x86, 0x01, + 0x0a, 0x0b, 0x53, 0x74, 0x61, 0x72, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x12, 0x39, 0x2e, + 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, + 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, + 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, - 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x81, 0x01, 0x0a, 0x24, 0x6f, - 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, - 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x2e, 0x76, 0x31, 0x42, 0x09, 0x42, 0x65, 0x61, 0x6d, 0x46, 0x6e, 0x41, 0x70, 0x69, 0x5a, 0x4e, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x70, 0x61, 0x63, 0x68, - 0x65, 0x2f, 0x62, 0x65, 0x61, 0x6d, 0x2f, 0x73, 0x64, 0x6b, 0x73, 0x2f, 0x76, 0x32, 0x2f, 0x67, - 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x62, 0x65, 0x61, 0x6d, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, - 0x2f, 0x66, 0x6e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x76, 0x31, 0x3b, - 0x66, 0x6e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x76, 0x31, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x83, 0x01, 0x0a, 0x0a, 0x53, 0x74, 0x6f, 0x70, 0x57, + 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x12, 0x38, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, + 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, + 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, + 0x6f, 0x70, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x39, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, + 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x57, 0x6f, 0x72, 0x6b, + 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x32, 0xa4, 0x01, 0x0a, + 0x12, 0x42, 0x65, 0x61, 0x6d, 0x46, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x8d, 0x01, 0x0a, 0x0c, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x3b, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, + 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x6f, 0x72, + 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x1a, 0x3a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, + 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, 0x5f, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x00, 0x28, + 0x01, 0x30, 0x01, 0x42, 0x81, 0x01, 0x0a, 0x24, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, + 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x66, 0x6e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x42, 0x09, 0x42, 0x65, + 0x61, 0x6d, 0x46, 0x6e, 0x41, 0x70, 0x69, 0x5a, 0x4e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2f, 0x62, 0x65, 0x61, 0x6d, 0x2f, + 0x73, 0x64, 0x6b, 0x73, 0x2f, 0x76, 0x32, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x62, + 0x65, 0x61, 0x6d, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2f, 0x66, 0x6e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x76, 0x31, 0x3b, 0x66, 0x6e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -5110,185 +5657,202 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescGZIP() return file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDescData } -var file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes = make([]protoimpl.MessageInfo, 67) +var file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes = make([]protoimpl.MessageInfo, 74) var file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_goTypes = []interface{}{ - (LogEntry_Severity_Enum)(0), // 0: org.apache.beam.model.fn_execution.v1.LogEntry.Severity.Enum - (*RemoteGrpcPort)(nil), // 1: org.apache.beam.model.fn_execution.v1.RemoteGrpcPort - (*GetProcessBundleDescriptorRequest)(nil), // 2: org.apache.beam.model.fn_execution.v1.GetProcessBundleDescriptorRequest - (*InstructionRequest)(nil), // 3: org.apache.beam.model.fn_execution.v1.InstructionRequest - (*InstructionResponse)(nil), // 4: org.apache.beam.model.fn_execution.v1.InstructionResponse - (*HarnessMonitoringInfosRequest)(nil), // 5: org.apache.beam.model.fn_execution.v1.HarnessMonitoringInfosRequest - (*HarnessMonitoringInfosResponse)(nil), // 6: org.apache.beam.model.fn_execution.v1.HarnessMonitoringInfosResponse - (*RegisterRequest)(nil), // 7: org.apache.beam.model.fn_execution.v1.RegisterRequest - (*RegisterResponse)(nil), // 8: org.apache.beam.model.fn_execution.v1.RegisterResponse - (*ProcessBundleDescriptor)(nil), // 9: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor - (*BundleApplication)(nil), // 10: org.apache.beam.model.fn_execution.v1.BundleApplication - (*DelayedBundleApplication)(nil), // 11: org.apache.beam.model.fn_execution.v1.DelayedBundleApplication - (*ProcessBundleRequest)(nil), // 12: org.apache.beam.model.fn_execution.v1.ProcessBundleRequest - (*ProcessBundleResponse)(nil), // 13: org.apache.beam.model.fn_execution.v1.ProcessBundleResponse - (*ProcessBundleProgressRequest)(nil), // 14: org.apache.beam.model.fn_execution.v1.ProcessBundleProgressRequest - (*MonitoringInfosMetadataRequest)(nil), // 15: org.apache.beam.model.fn_execution.v1.MonitoringInfosMetadataRequest - (*ProcessBundleProgressResponse)(nil), // 16: org.apache.beam.model.fn_execution.v1.ProcessBundleProgressResponse - (*MonitoringInfosMetadataResponse)(nil), // 17: org.apache.beam.model.fn_execution.v1.MonitoringInfosMetadataResponse - (*ProcessBundleSplitRequest)(nil), // 18: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitRequest - (*ProcessBundleSplitResponse)(nil), // 19: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitResponse - (*FinalizeBundleRequest)(nil), // 20: org.apache.beam.model.fn_execution.v1.FinalizeBundleRequest - (*FinalizeBundleResponse)(nil), // 21: org.apache.beam.model.fn_execution.v1.FinalizeBundleResponse - (*Elements)(nil), // 22: org.apache.beam.model.fn_execution.v1.Elements - (*StateRequest)(nil), // 23: org.apache.beam.model.fn_execution.v1.StateRequest - (*StateResponse)(nil), // 24: org.apache.beam.model.fn_execution.v1.StateResponse - (*StateKey)(nil), // 25: org.apache.beam.model.fn_execution.v1.StateKey - (*StateGetRequest)(nil), // 26: org.apache.beam.model.fn_execution.v1.StateGetRequest - (*StateGetResponse)(nil), // 27: org.apache.beam.model.fn_execution.v1.StateGetResponse - (*StateAppendRequest)(nil), // 28: org.apache.beam.model.fn_execution.v1.StateAppendRequest - (*StateAppendResponse)(nil), // 29: org.apache.beam.model.fn_execution.v1.StateAppendResponse - (*StateClearRequest)(nil), // 30: org.apache.beam.model.fn_execution.v1.StateClearRequest - (*StateClearResponse)(nil), // 31: org.apache.beam.model.fn_execution.v1.StateClearResponse - (*LogEntry)(nil), // 32: org.apache.beam.model.fn_execution.v1.LogEntry - (*LogControl)(nil), // 33: org.apache.beam.model.fn_execution.v1.LogControl - (*StartWorkerRequest)(nil), // 34: org.apache.beam.model.fn_execution.v1.StartWorkerRequest - (*StartWorkerResponse)(nil), // 35: org.apache.beam.model.fn_execution.v1.StartWorkerResponse - (*StopWorkerRequest)(nil), // 36: org.apache.beam.model.fn_execution.v1.StopWorkerRequest - (*StopWorkerResponse)(nil), // 37: org.apache.beam.model.fn_execution.v1.StopWorkerResponse - (*WorkerStatusRequest)(nil), // 38: org.apache.beam.model.fn_execution.v1.WorkerStatusRequest - (*WorkerStatusResponse)(nil), // 39: org.apache.beam.model.fn_execution.v1.WorkerStatusResponse - nil, // 40: org.apache.beam.model.fn_execution.v1.HarnessMonitoringInfosResponse.MonitoringDataEntry - nil, // 41: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.TransformsEntry - nil, // 42: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.PcollectionsEntry - nil, // 43: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.WindowingStrategiesEntry - nil, // 44: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.CodersEntry - nil, // 45: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.EnvironmentsEntry - nil, // 46: org.apache.beam.model.fn_execution.v1.BundleApplication.OutputWatermarksEntry - (*ProcessBundleRequest_CacheToken)(nil), // 47: org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.CacheToken - (*ProcessBundleRequest_CacheToken_UserState)(nil), // 48: org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.CacheToken.UserState - (*ProcessBundleRequest_CacheToken_SideInput)(nil), // 49: org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.CacheToken.SideInput - nil, // 50: org.apache.beam.model.fn_execution.v1.ProcessBundleResponse.MonitoringDataEntry - nil, // 51: org.apache.beam.model.fn_execution.v1.ProcessBundleProgressResponse.MonitoringDataEntry - nil, // 52: org.apache.beam.model.fn_execution.v1.MonitoringInfosMetadataResponse.MonitoringInfoEntry - (*ProcessBundleSplitRequest_DesiredSplit)(nil), // 53: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitRequest.DesiredSplit - nil, // 54: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitRequest.DesiredSplitsEntry - (*ProcessBundleSplitResponse_ChannelSplit)(nil), // 55: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitResponse.ChannelSplit - (*Elements_Data)(nil), // 56: org.apache.beam.model.fn_execution.v1.Elements.Data - (*Elements_Timers)(nil), // 57: org.apache.beam.model.fn_execution.v1.Elements.Timers - (*StateKey_Runner)(nil), // 58: org.apache.beam.model.fn_execution.v1.StateKey.Runner - (*StateKey_IterableSideInput)(nil), // 59: org.apache.beam.model.fn_execution.v1.StateKey.IterableSideInput - (*StateKey_MultimapSideInput)(nil), // 60: org.apache.beam.model.fn_execution.v1.StateKey.MultimapSideInput - (*StateKey_MultimapKeysSideInput)(nil), // 61: org.apache.beam.model.fn_execution.v1.StateKey.MultimapKeysSideInput - (*StateKey_BagUserState)(nil), // 62: org.apache.beam.model.fn_execution.v1.StateKey.BagUserState - (*StateKey_MultimapKeysUserState)(nil), // 63: org.apache.beam.model.fn_execution.v1.StateKey.MultimapKeysUserState - (*StateKey_MultimapUserState)(nil), // 64: org.apache.beam.model.fn_execution.v1.StateKey.MultimapUserState - (*LogEntry_List)(nil), // 65: org.apache.beam.model.fn_execution.v1.LogEntry.List - (*LogEntry_Severity)(nil), // 66: org.apache.beam.model.fn_execution.v1.LogEntry.Severity - nil, // 67: org.apache.beam.model.fn_execution.v1.StartWorkerRequest.ParamsEntry - (*pipeline_v1.ApiServiceDescriptor)(nil), // 68: org.apache.beam.model.pipeline.v1.ApiServiceDescriptor - (pipeline_v1.IsBounded_Enum)(0), // 69: org.apache.beam.model.pipeline.v1.IsBounded.Enum - (*durationpb.Duration)(nil), // 70: google.protobuf.Duration - (*pipeline_v1.MonitoringInfo)(nil), // 71: org.apache.beam.model.pipeline.v1.MonitoringInfo - (*timestamppb.Timestamp)(nil), // 72: google.protobuf.Timestamp - (*pipeline_v1.PTransform)(nil), // 73: org.apache.beam.model.pipeline.v1.PTransform - (*pipeline_v1.PCollection)(nil), // 74: org.apache.beam.model.pipeline.v1.PCollection - (*pipeline_v1.WindowingStrategy)(nil), // 75: org.apache.beam.model.pipeline.v1.WindowingStrategy - (*pipeline_v1.Coder)(nil), // 76: org.apache.beam.model.pipeline.v1.Coder - (*pipeline_v1.Environment)(nil), // 77: org.apache.beam.model.pipeline.v1.Environment + (FnApiTransforms_Runner)(0), // 0: org.apache.beam.model.fn_execution.v1.FnApiTransforms.Runner + (LogEntry_Severity_Enum)(0), // 1: org.apache.beam.model.fn_execution.v1.LogEntry.Severity.Enum + (*FnApiTransforms)(nil), // 2: org.apache.beam.model.fn_execution.v1.FnApiTransforms + (*RemoteGrpcPort)(nil), // 3: org.apache.beam.model.fn_execution.v1.RemoteGrpcPort + (*GetProcessBundleDescriptorRequest)(nil), // 4: org.apache.beam.model.fn_execution.v1.GetProcessBundleDescriptorRequest + (*InstructionRequest)(nil), // 5: org.apache.beam.model.fn_execution.v1.InstructionRequest + (*InstructionResponse)(nil), // 6: org.apache.beam.model.fn_execution.v1.InstructionResponse + (*SampleDataRequest)(nil), // 7: org.apache.beam.model.fn_execution.v1.SampleDataRequest + (*SampledElement)(nil), // 8: org.apache.beam.model.fn_execution.v1.SampledElement + (*SampleDataResponse)(nil), // 9: org.apache.beam.model.fn_execution.v1.SampleDataResponse + (*HarnessMonitoringInfosRequest)(nil), // 10: org.apache.beam.model.fn_execution.v1.HarnessMonitoringInfosRequest + (*HarnessMonitoringInfosResponse)(nil), // 11: org.apache.beam.model.fn_execution.v1.HarnessMonitoringInfosResponse + (*RegisterRequest)(nil), // 12: org.apache.beam.model.fn_execution.v1.RegisterRequest + (*RegisterResponse)(nil), // 13: org.apache.beam.model.fn_execution.v1.RegisterResponse + (*ProcessBundleDescriptor)(nil), // 14: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor + (*BundleApplication)(nil), // 15: org.apache.beam.model.fn_execution.v1.BundleApplication + (*DelayedBundleApplication)(nil), // 16: org.apache.beam.model.fn_execution.v1.DelayedBundleApplication + (*ProcessBundleRequest)(nil), // 17: org.apache.beam.model.fn_execution.v1.ProcessBundleRequest + (*ProcessBundleResponse)(nil), // 18: org.apache.beam.model.fn_execution.v1.ProcessBundleResponse + (*ProcessBundleProgressRequest)(nil), // 19: org.apache.beam.model.fn_execution.v1.ProcessBundleProgressRequest + (*MonitoringInfosMetadataRequest)(nil), // 20: org.apache.beam.model.fn_execution.v1.MonitoringInfosMetadataRequest + (*ProcessBundleProgressResponse)(nil), // 21: org.apache.beam.model.fn_execution.v1.ProcessBundleProgressResponse + (*MonitoringInfosMetadataResponse)(nil), // 22: org.apache.beam.model.fn_execution.v1.MonitoringInfosMetadataResponse + (*ProcessBundleSplitRequest)(nil), // 23: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitRequest + (*ProcessBundleSplitResponse)(nil), // 24: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitResponse + (*FinalizeBundleRequest)(nil), // 25: org.apache.beam.model.fn_execution.v1.FinalizeBundleRequest + (*FinalizeBundleResponse)(nil), // 26: org.apache.beam.model.fn_execution.v1.FinalizeBundleResponse + (*Elements)(nil), // 27: org.apache.beam.model.fn_execution.v1.Elements + (*StateRequest)(nil), // 28: org.apache.beam.model.fn_execution.v1.StateRequest + (*StateResponse)(nil), // 29: org.apache.beam.model.fn_execution.v1.StateResponse + (*StateKey)(nil), // 30: org.apache.beam.model.fn_execution.v1.StateKey + (*StateGetRequest)(nil), // 31: org.apache.beam.model.fn_execution.v1.StateGetRequest + (*StateGetResponse)(nil), // 32: org.apache.beam.model.fn_execution.v1.StateGetResponse + (*StateAppendRequest)(nil), // 33: org.apache.beam.model.fn_execution.v1.StateAppendRequest + (*StateAppendResponse)(nil), // 34: org.apache.beam.model.fn_execution.v1.StateAppendResponse + (*StateClearRequest)(nil), // 35: org.apache.beam.model.fn_execution.v1.StateClearRequest + (*StateClearResponse)(nil), // 36: org.apache.beam.model.fn_execution.v1.StateClearResponse + (*LogEntry)(nil), // 37: org.apache.beam.model.fn_execution.v1.LogEntry + (*LogControl)(nil), // 38: org.apache.beam.model.fn_execution.v1.LogControl + (*StartWorkerRequest)(nil), // 39: org.apache.beam.model.fn_execution.v1.StartWorkerRequest + (*StartWorkerResponse)(nil), // 40: org.apache.beam.model.fn_execution.v1.StartWorkerResponse + (*StopWorkerRequest)(nil), // 41: org.apache.beam.model.fn_execution.v1.StopWorkerRequest + (*StopWorkerResponse)(nil), // 42: org.apache.beam.model.fn_execution.v1.StopWorkerResponse + (*WorkerStatusRequest)(nil), // 43: org.apache.beam.model.fn_execution.v1.WorkerStatusRequest + (*WorkerStatusResponse)(nil), // 44: org.apache.beam.model.fn_execution.v1.WorkerStatusResponse + (*SampledElement_Exception)(nil), // 45: org.apache.beam.model.fn_execution.v1.SampledElement.Exception + (*SampleDataResponse_ElementList)(nil), // 46: org.apache.beam.model.fn_execution.v1.SampleDataResponse.ElementList + nil, // 47: org.apache.beam.model.fn_execution.v1.SampleDataResponse.ElementSamplesEntry + nil, // 48: org.apache.beam.model.fn_execution.v1.HarnessMonitoringInfosResponse.MonitoringDataEntry + nil, // 49: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.TransformsEntry + nil, // 50: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.PcollectionsEntry + nil, // 51: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.WindowingStrategiesEntry + nil, // 52: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.CodersEntry + nil, // 53: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.EnvironmentsEntry + nil, // 54: org.apache.beam.model.fn_execution.v1.BundleApplication.OutputWatermarksEntry + (*ProcessBundleRequest_CacheToken)(nil), // 55: org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.CacheToken + (*ProcessBundleRequest_CacheToken_UserState)(nil), // 56: org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.CacheToken.UserState + (*ProcessBundleRequest_CacheToken_SideInput)(nil), // 57: org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.CacheToken.SideInput + nil, // 58: org.apache.beam.model.fn_execution.v1.ProcessBundleResponse.MonitoringDataEntry + nil, // 59: org.apache.beam.model.fn_execution.v1.ProcessBundleProgressResponse.MonitoringDataEntry + nil, // 60: org.apache.beam.model.fn_execution.v1.MonitoringInfosMetadataResponse.MonitoringInfoEntry + (*ProcessBundleSplitRequest_DesiredSplit)(nil), // 61: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitRequest.DesiredSplit + nil, // 62: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitRequest.DesiredSplitsEntry + (*ProcessBundleSplitResponse_ChannelSplit)(nil), // 63: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitResponse.ChannelSplit + (*Elements_Data)(nil), // 64: org.apache.beam.model.fn_execution.v1.Elements.Data + (*Elements_Timers)(nil), // 65: org.apache.beam.model.fn_execution.v1.Elements.Timers + (*StateKey_Runner)(nil), // 66: org.apache.beam.model.fn_execution.v1.StateKey.Runner + (*StateKey_IterableSideInput)(nil), // 67: org.apache.beam.model.fn_execution.v1.StateKey.IterableSideInput + (*StateKey_MultimapSideInput)(nil), // 68: org.apache.beam.model.fn_execution.v1.StateKey.MultimapSideInput + (*StateKey_MultimapKeysSideInput)(nil), // 69: org.apache.beam.model.fn_execution.v1.StateKey.MultimapKeysSideInput + (*StateKey_BagUserState)(nil), // 70: org.apache.beam.model.fn_execution.v1.StateKey.BagUserState + (*StateKey_MultimapKeysUserState)(nil), // 71: org.apache.beam.model.fn_execution.v1.StateKey.MultimapKeysUserState + (*StateKey_MultimapUserState)(nil), // 72: org.apache.beam.model.fn_execution.v1.StateKey.MultimapUserState + (*LogEntry_List)(nil), // 73: org.apache.beam.model.fn_execution.v1.LogEntry.List + (*LogEntry_Severity)(nil), // 74: org.apache.beam.model.fn_execution.v1.LogEntry.Severity + nil, // 75: org.apache.beam.model.fn_execution.v1.StartWorkerRequest.ParamsEntry + (*pipeline_v1.ApiServiceDescriptor)(nil), // 76: org.apache.beam.model.pipeline.v1.ApiServiceDescriptor + (*timestamppb.Timestamp)(nil), // 77: google.protobuf.Timestamp + (pipeline_v1.IsBounded_Enum)(0), // 78: org.apache.beam.model.pipeline.v1.IsBounded.Enum + (*durationpb.Duration)(nil), // 79: google.protobuf.Duration + (*pipeline_v1.MonitoringInfo)(nil), // 80: org.apache.beam.model.pipeline.v1.MonitoringInfo + (*structpb.Struct)(nil), // 81: google.protobuf.Struct + (*pipeline_v1.PTransform)(nil), // 82: org.apache.beam.model.pipeline.v1.PTransform + (*pipeline_v1.PCollection)(nil), // 83: org.apache.beam.model.pipeline.v1.PCollection + (*pipeline_v1.WindowingStrategy)(nil), // 84: org.apache.beam.model.pipeline.v1.WindowingStrategy + (*pipeline_v1.Coder)(nil), // 85: org.apache.beam.model.pipeline.v1.Coder + (*pipeline_v1.Environment)(nil), // 86: org.apache.beam.model.pipeline.v1.Environment } var file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_depIdxs = []int32{ - 68, // 0: org.apache.beam.model.fn_execution.v1.RemoteGrpcPort.api_service_descriptor:type_name -> org.apache.beam.model.pipeline.v1.ApiServiceDescriptor - 12, // 1: org.apache.beam.model.fn_execution.v1.InstructionRequest.process_bundle:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleRequest - 14, // 2: org.apache.beam.model.fn_execution.v1.InstructionRequest.process_bundle_progress:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleProgressRequest - 18, // 3: org.apache.beam.model.fn_execution.v1.InstructionRequest.process_bundle_split:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleSplitRequest - 20, // 4: org.apache.beam.model.fn_execution.v1.InstructionRequest.finalize_bundle:type_name -> org.apache.beam.model.fn_execution.v1.FinalizeBundleRequest - 15, // 5: org.apache.beam.model.fn_execution.v1.InstructionRequest.monitoring_infos:type_name -> org.apache.beam.model.fn_execution.v1.MonitoringInfosMetadataRequest - 5, // 6: org.apache.beam.model.fn_execution.v1.InstructionRequest.harness_monitoring_infos:type_name -> org.apache.beam.model.fn_execution.v1.HarnessMonitoringInfosRequest - 7, // 7: org.apache.beam.model.fn_execution.v1.InstructionRequest.register:type_name -> org.apache.beam.model.fn_execution.v1.RegisterRequest - 13, // 8: org.apache.beam.model.fn_execution.v1.InstructionResponse.process_bundle:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleResponse - 16, // 9: org.apache.beam.model.fn_execution.v1.InstructionResponse.process_bundle_progress:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleProgressResponse - 19, // 10: org.apache.beam.model.fn_execution.v1.InstructionResponse.process_bundle_split:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleSplitResponse - 21, // 11: org.apache.beam.model.fn_execution.v1.InstructionResponse.finalize_bundle:type_name -> org.apache.beam.model.fn_execution.v1.FinalizeBundleResponse - 17, // 12: org.apache.beam.model.fn_execution.v1.InstructionResponse.monitoring_infos:type_name -> org.apache.beam.model.fn_execution.v1.MonitoringInfosMetadataResponse - 6, // 13: org.apache.beam.model.fn_execution.v1.InstructionResponse.harness_monitoring_infos:type_name -> org.apache.beam.model.fn_execution.v1.HarnessMonitoringInfosResponse - 8, // 14: org.apache.beam.model.fn_execution.v1.InstructionResponse.register:type_name -> org.apache.beam.model.fn_execution.v1.RegisterResponse - 40, // 15: org.apache.beam.model.fn_execution.v1.HarnessMonitoringInfosResponse.monitoring_data:type_name -> org.apache.beam.model.fn_execution.v1.HarnessMonitoringInfosResponse.MonitoringDataEntry - 9, // 16: org.apache.beam.model.fn_execution.v1.RegisterRequest.process_bundle_descriptor:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor - 41, // 17: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.transforms:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.TransformsEntry - 42, // 18: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.pcollections:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.PcollectionsEntry - 43, // 19: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.windowing_strategies:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.WindowingStrategiesEntry - 44, // 20: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.coders:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.CodersEntry - 45, // 21: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.environments:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.EnvironmentsEntry - 68, // 22: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.state_api_service_descriptor:type_name -> org.apache.beam.model.pipeline.v1.ApiServiceDescriptor - 68, // 23: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.timer_api_service_descriptor:type_name -> org.apache.beam.model.pipeline.v1.ApiServiceDescriptor - 46, // 24: org.apache.beam.model.fn_execution.v1.BundleApplication.output_watermarks:type_name -> org.apache.beam.model.fn_execution.v1.BundleApplication.OutputWatermarksEntry - 69, // 25: org.apache.beam.model.fn_execution.v1.BundleApplication.is_bounded:type_name -> org.apache.beam.model.pipeline.v1.IsBounded.Enum - 10, // 26: org.apache.beam.model.fn_execution.v1.DelayedBundleApplication.application:type_name -> org.apache.beam.model.fn_execution.v1.BundleApplication - 70, // 27: org.apache.beam.model.fn_execution.v1.DelayedBundleApplication.requested_time_delay:type_name -> google.protobuf.Duration - 47, // 28: org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.cache_tokens:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.CacheToken - 22, // 29: org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.elements:type_name -> org.apache.beam.model.fn_execution.v1.Elements - 11, // 30: org.apache.beam.model.fn_execution.v1.ProcessBundleResponse.residual_roots:type_name -> org.apache.beam.model.fn_execution.v1.DelayedBundleApplication - 71, // 31: org.apache.beam.model.fn_execution.v1.ProcessBundleResponse.monitoring_infos:type_name -> org.apache.beam.model.pipeline.v1.MonitoringInfo - 50, // 32: org.apache.beam.model.fn_execution.v1.ProcessBundleResponse.monitoring_data:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleResponse.MonitoringDataEntry - 22, // 33: org.apache.beam.model.fn_execution.v1.ProcessBundleResponse.elements:type_name -> org.apache.beam.model.fn_execution.v1.Elements - 71, // 34: org.apache.beam.model.fn_execution.v1.ProcessBundleProgressResponse.monitoring_infos:type_name -> org.apache.beam.model.pipeline.v1.MonitoringInfo - 51, // 35: org.apache.beam.model.fn_execution.v1.ProcessBundleProgressResponse.monitoring_data:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleProgressResponse.MonitoringDataEntry - 52, // 36: org.apache.beam.model.fn_execution.v1.MonitoringInfosMetadataResponse.monitoring_info:type_name -> org.apache.beam.model.fn_execution.v1.MonitoringInfosMetadataResponse.MonitoringInfoEntry - 54, // 37: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitRequest.desired_splits:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleSplitRequest.DesiredSplitsEntry - 10, // 38: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitResponse.primary_roots:type_name -> org.apache.beam.model.fn_execution.v1.BundleApplication - 11, // 39: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitResponse.residual_roots:type_name -> org.apache.beam.model.fn_execution.v1.DelayedBundleApplication - 55, // 40: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitResponse.channel_splits:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleSplitResponse.ChannelSplit - 56, // 41: org.apache.beam.model.fn_execution.v1.Elements.data:type_name -> org.apache.beam.model.fn_execution.v1.Elements.Data - 57, // 42: org.apache.beam.model.fn_execution.v1.Elements.timers:type_name -> org.apache.beam.model.fn_execution.v1.Elements.Timers - 25, // 43: org.apache.beam.model.fn_execution.v1.StateRequest.state_key:type_name -> org.apache.beam.model.fn_execution.v1.StateKey - 26, // 44: org.apache.beam.model.fn_execution.v1.StateRequest.get:type_name -> org.apache.beam.model.fn_execution.v1.StateGetRequest - 28, // 45: org.apache.beam.model.fn_execution.v1.StateRequest.append:type_name -> org.apache.beam.model.fn_execution.v1.StateAppendRequest - 30, // 46: org.apache.beam.model.fn_execution.v1.StateRequest.clear:type_name -> org.apache.beam.model.fn_execution.v1.StateClearRequest - 27, // 47: org.apache.beam.model.fn_execution.v1.StateResponse.get:type_name -> org.apache.beam.model.fn_execution.v1.StateGetResponse - 29, // 48: org.apache.beam.model.fn_execution.v1.StateResponse.append:type_name -> org.apache.beam.model.fn_execution.v1.StateAppendResponse - 31, // 49: org.apache.beam.model.fn_execution.v1.StateResponse.clear:type_name -> org.apache.beam.model.fn_execution.v1.StateClearResponse - 58, // 50: org.apache.beam.model.fn_execution.v1.StateKey.runner:type_name -> org.apache.beam.model.fn_execution.v1.StateKey.Runner - 60, // 51: org.apache.beam.model.fn_execution.v1.StateKey.multimap_side_input:type_name -> org.apache.beam.model.fn_execution.v1.StateKey.MultimapSideInput - 62, // 52: org.apache.beam.model.fn_execution.v1.StateKey.bag_user_state:type_name -> org.apache.beam.model.fn_execution.v1.StateKey.BagUserState - 59, // 53: org.apache.beam.model.fn_execution.v1.StateKey.iterable_side_input:type_name -> org.apache.beam.model.fn_execution.v1.StateKey.IterableSideInput - 61, // 54: org.apache.beam.model.fn_execution.v1.StateKey.multimap_keys_side_input:type_name -> org.apache.beam.model.fn_execution.v1.StateKey.MultimapKeysSideInput - 63, // 55: org.apache.beam.model.fn_execution.v1.StateKey.multimap_keys_user_state:type_name -> org.apache.beam.model.fn_execution.v1.StateKey.MultimapKeysUserState - 64, // 56: org.apache.beam.model.fn_execution.v1.StateKey.multimap_user_state:type_name -> org.apache.beam.model.fn_execution.v1.StateKey.MultimapUserState - 0, // 57: org.apache.beam.model.fn_execution.v1.LogEntry.severity:type_name -> org.apache.beam.model.fn_execution.v1.LogEntry.Severity.Enum - 72, // 58: org.apache.beam.model.fn_execution.v1.LogEntry.timestamp:type_name -> google.protobuf.Timestamp - 68, // 59: org.apache.beam.model.fn_execution.v1.StartWorkerRequest.control_endpoint:type_name -> org.apache.beam.model.pipeline.v1.ApiServiceDescriptor - 68, // 60: org.apache.beam.model.fn_execution.v1.StartWorkerRequest.logging_endpoint:type_name -> org.apache.beam.model.pipeline.v1.ApiServiceDescriptor - 68, // 61: org.apache.beam.model.fn_execution.v1.StartWorkerRequest.artifact_endpoint:type_name -> org.apache.beam.model.pipeline.v1.ApiServiceDescriptor - 68, // 62: org.apache.beam.model.fn_execution.v1.StartWorkerRequest.provision_endpoint:type_name -> org.apache.beam.model.pipeline.v1.ApiServiceDescriptor - 67, // 63: org.apache.beam.model.fn_execution.v1.StartWorkerRequest.params:type_name -> org.apache.beam.model.fn_execution.v1.StartWorkerRequest.ParamsEntry - 73, // 64: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.TransformsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.PTransform - 74, // 65: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.PcollectionsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.PCollection - 75, // 66: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.WindowingStrategiesEntry.value:type_name -> org.apache.beam.model.pipeline.v1.WindowingStrategy - 76, // 67: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.CodersEntry.value:type_name -> org.apache.beam.model.pipeline.v1.Coder - 77, // 68: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.EnvironmentsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.Environment - 72, // 69: org.apache.beam.model.fn_execution.v1.BundleApplication.OutputWatermarksEntry.value:type_name -> google.protobuf.Timestamp - 48, // 70: org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.CacheToken.user_state:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.CacheToken.UserState - 49, // 71: org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.CacheToken.side_input:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.CacheToken.SideInput - 71, // 72: org.apache.beam.model.fn_execution.v1.MonitoringInfosMetadataResponse.MonitoringInfoEntry.value:type_name -> org.apache.beam.model.pipeline.v1.MonitoringInfo - 53, // 73: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitRequest.DesiredSplitsEntry.value:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleSplitRequest.DesiredSplit - 32, // 74: org.apache.beam.model.fn_execution.v1.LogEntry.List.log_entries:type_name -> org.apache.beam.model.fn_execution.v1.LogEntry - 4, // 75: org.apache.beam.model.fn_execution.v1.BeamFnControl.Control:input_type -> org.apache.beam.model.fn_execution.v1.InstructionResponse - 2, // 76: org.apache.beam.model.fn_execution.v1.BeamFnControl.GetProcessBundleDescriptor:input_type -> org.apache.beam.model.fn_execution.v1.GetProcessBundleDescriptorRequest - 22, // 77: org.apache.beam.model.fn_execution.v1.BeamFnData.Data:input_type -> org.apache.beam.model.fn_execution.v1.Elements - 23, // 78: org.apache.beam.model.fn_execution.v1.BeamFnState.State:input_type -> org.apache.beam.model.fn_execution.v1.StateRequest - 65, // 79: org.apache.beam.model.fn_execution.v1.BeamFnLogging.Logging:input_type -> org.apache.beam.model.fn_execution.v1.LogEntry.List - 34, // 80: org.apache.beam.model.fn_execution.v1.BeamFnExternalWorkerPool.StartWorker:input_type -> org.apache.beam.model.fn_execution.v1.StartWorkerRequest - 36, // 81: org.apache.beam.model.fn_execution.v1.BeamFnExternalWorkerPool.StopWorker:input_type -> org.apache.beam.model.fn_execution.v1.StopWorkerRequest - 39, // 82: org.apache.beam.model.fn_execution.v1.BeamFnWorkerStatus.WorkerStatus:input_type -> org.apache.beam.model.fn_execution.v1.WorkerStatusResponse - 3, // 83: org.apache.beam.model.fn_execution.v1.BeamFnControl.Control:output_type -> org.apache.beam.model.fn_execution.v1.InstructionRequest - 9, // 84: org.apache.beam.model.fn_execution.v1.BeamFnControl.GetProcessBundleDescriptor:output_type -> org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor - 22, // 85: org.apache.beam.model.fn_execution.v1.BeamFnData.Data:output_type -> org.apache.beam.model.fn_execution.v1.Elements - 24, // 86: org.apache.beam.model.fn_execution.v1.BeamFnState.State:output_type -> org.apache.beam.model.fn_execution.v1.StateResponse - 33, // 87: org.apache.beam.model.fn_execution.v1.BeamFnLogging.Logging:output_type -> org.apache.beam.model.fn_execution.v1.LogControl - 35, // 88: org.apache.beam.model.fn_execution.v1.BeamFnExternalWorkerPool.StartWorker:output_type -> org.apache.beam.model.fn_execution.v1.StartWorkerResponse - 37, // 89: org.apache.beam.model.fn_execution.v1.BeamFnExternalWorkerPool.StopWorker:output_type -> org.apache.beam.model.fn_execution.v1.StopWorkerResponse - 38, // 90: org.apache.beam.model.fn_execution.v1.BeamFnWorkerStatus.WorkerStatus:output_type -> org.apache.beam.model.fn_execution.v1.WorkerStatusRequest - 83, // [83:91] is the sub-list for method output_type - 75, // [75:83] is the sub-list for method input_type - 75, // [75:75] is the sub-list for extension type_name - 75, // [75:75] is the sub-list for extension extendee - 0, // [0:75] is the sub-list for field type_name + 76, // 0: org.apache.beam.model.fn_execution.v1.RemoteGrpcPort.api_service_descriptor:type_name -> org.apache.beam.model.pipeline.v1.ApiServiceDescriptor + 17, // 1: org.apache.beam.model.fn_execution.v1.InstructionRequest.process_bundle:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleRequest + 19, // 2: org.apache.beam.model.fn_execution.v1.InstructionRequest.process_bundle_progress:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleProgressRequest + 23, // 3: org.apache.beam.model.fn_execution.v1.InstructionRequest.process_bundle_split:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleSplitRequest + 25, // 4: org.apache.beam.model.fn_execution.v1.InstructionRequest.finalize_bundle:type_name -> org.apache.beam.model.fn_execution.v1.FinalizeBundleRequest + 20, // 5: org.apache.beam.model.fn_execution.v1.InstructionRequest.monitoring_infos:type_name -> org.apache.beam.model.fn_execution.v1.MonitoringInfosMetadataRequest + 10, // 6: org.apache.beam.model.fn_execution.v1.InstructionRequest.harness_monitoring_infos:type_name -> org.apache.beam.model.fn_execution.v1.HarnessMonitoringInfosRequest + 7, // 7: org.apache.beam.model.fn_execution.v1.InstructionRequest.sample_data:type_name -> org.apache.beam.model.fn_execution.v1.SampleDataRequest + 12, // 8: org.apache.beam.model.fn_execution.v1.InstructionRequest.register:type_name -> org.apache.beam.model.fn_execution.v1.RegisterRequest + 18, // 9: org.apache.beam.model.fn_execution.v1.InstructionResponse.process_bundle:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleResponse + 21, // 10: org.apache.beam.model.fn_execution.v1.InstructionResponse.process_bundle_progress:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleProgressResponse + 24, // 11: org.apache.beam.model.fn_execution.v1.InstructionResponse.process_bundle_split:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleSplitResponse + 26, // 12: org.apache.beam.model.fn_execution.v1.InstructionResponse.finalize_bundle:type_name -> org.apache.beam.model.fn_execution.v1.FinalizeBundleResponse + 22, // 13: org.apache.beam.model.fn_execution.v1.InstructionResponse.monitoring_infos:type_name -> org.apache.beam.model.fn_execution.v1.MonitoringInfosMetadataResponse + 11, // 14: org.apache.beam.model.fn_execution.v1.InstructionResponse.harness_monitoring_infos:type_name -> org.apache.beam.model.fn_execution.v1.HarnessMonitoringInfosResponse + 9, // 15: org.apache.beam.model.fn_execution.v1.InstructionResponse.sample_data:type_name -> org.apache.beam.model.fn_execution.v1.SampleDataResponse + 13, // 16: org.apache.beam.model.fn_execution.v1.InstructionResponse.register:type_name -> org.apache.beam.model.fn_execution.v1.RegisterResponse + 77, // 17: org.apache.beam.model.fn_execution.v1.SampledElement.sample_timestamp:type_name -> google.protobuf.Timestamp + 45, // 18: org.apache.beam.model.fn_execution.v1.SampledElement.exception:type_name -> org.apache.beam.model.fn_execution.v1.SampledElement.Exception + 47, // 19: org.apache.beam.model.fn_execution.v1.SampleDataResponse.element_samples:type_name -> org.apache.beam.model.fn_execution.v1.SampleDataResponse.ElementSamplesEntry + 48, // 20: org.apache.beam.model.fn_execution.v1.HarnessMonitoringInfosResponse.monitoring_data:type_name -> org.apache.beam.model.fn_execution.v1.HarnessMonitoringInfosResponse.MonitoringDataEntry + 14, // 21: org.apache.beam.model.fn_execution.v1.RegisterRequest.process_bundle_descriptor:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor + 49, // 22: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.transforms:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.TransformsEntry + 50, // 23: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.pcollections:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.PcollectionsEntry + 51, // 24: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.windowing_strategies:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.WindowingStrategiesEntry + 52, // 25: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.coders:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.CodersEntry + 53, // 26: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.environments:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.EnvironmentsEntry + 76, // 27: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.state_api_service_descriptor:type_name -> org.apache.beam.model.pipeline.v1.ApiServiceDescriptor + 76, // 28: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.timer_api_service_descriptor:type_name -> org.apache.beam.model.pipeline.v1.ApiServiceDescriptor + 54, // 29: org.apache.beam.model.fn_execution.v1.BundleApplication.output_watermarks:type_name -> org.apache.beam.model.fn_execution.v1.BundleApplication.OutputWatermarksEntry + 78, // 30: org.apache.beam.model.fn_execution.v1.BundleApplication.is_bounded:type_name -> org.apache.beam.model.pipeline.v1.IsBounded.Enum + 15, // 31: org.apache.beam.model.fn_execution.v1.DelayedBundleApplication.application:type_name -> org.apache.beam.model.fn_execution.v1.BundleApplication + 79, // 32: org.apache.beam.model.fn_execution.v1.DelayedBundleApplication.requested_time_delay:type_name -> google.protobuf.Duration + 55, // 33: org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.cache_tokens:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.CacheToken + 27, // 34: org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.elements:type_name -> org.apache.beam.model.fn_execution.v1.Elements + 16, // 35: org.apache.beam.model.fn_execution.v1.ProcessBundleResponse.residual_roots:type_name -> org.apache.beam.model.fn_execution.v1.DelayedBundleApplication + 80, // 36: org.apache.beam.model.fn_execution.v1.ProcessBundleResponse.monitoring_infos:type_name -> org.apache.beam.model.pipeline.v1.MonitoringInfo + 58, // 37: org.apache.beam.model.fn_execution.v1.ProcessBundleResponse.monitoring_data:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleResponse.MonitoringDataEntry + 27, // 38: org.apache.beam.model.fn_execution.v1.ProcessBundleResponse.elements:type_name -> org.apache.beam.model.fn_execution.v1.Elements + 80, // 39: org.apache.beam.model.fn_execution.v1.ProcessBundleProgressResponse.monitoring_infos:type_name -> org.apache.beam.model.pipeline.v1.MonitoringInfo + 59, // 40: org.apache.beam.model.fn_execution.v1.ProcessBundleProgressResponse.monitoring_data:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleProgressResponse.MonitoringDataEntry + 60, // 41: org.apache.beam.model.fn_execution.v1.MonitoringInfosMetadataResponse.monitoring_info:type_name -> org.apache.beam.model.fn_execution.v1.MonitoringInfosMetadataResponse.MonitoringInfoEntry + 62, // 42: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitRequest.desired_splits:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleSplitRequest.DesiredSplitsEntry + 15, // 43: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitResponse.primary_roots:type_name -> org.apache.beam.model.fn_execution.v1.BundleApplication + 16, // 44: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitResponse.residual_roots:type_name -> org.apache.beam.model.fn_execution.v1.DelayedBundleApplication + 63, // 45: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitResponse.channel_splits:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleSplitResponse.ChannelSplit + 64, // 46: org.apache.beam.model.fn_execution.v1.Elements.data:type_name -> org.apache.beam.model.fn_execution.v1.Elements.Data + 65, // 47: org.apache.beam.model.fn_execution.v1.Elements.timers:type_name -> org.apache.beam.model.fn_execution.v1.Elements.Timers + 30, // 48: org.apache.beam.model.fn_execution.v1.StateRequest.state_key:type_name -> org.apache.beam.model.fn_execution.v1.StateKey + 31, // 49: org.apache.beam.model.fn_execution.v1.StateRequest.get:type_name -> org.apache.beam.model.fn_execution.v1.StateGetRequest + 33, // 50: org.apache.beam.model.fn_execution.v1.StateRequest.append:type_name -> org.apache.beam.model.fn_execution.v1.StateAppendRequest + 35, // 51: org.apache.beam.model.fn_execution.v1.StateRequest.clear:type_name -> org.apache.beam.model.fn_execution.v1.StateClearRequest + 32, // 52: org.apache.beam.model.fn_execution.v1.StateResponse.get:type_name -> org.apache.beam.model.fn_execution.v1.StateGetResponse + 34, // 53: org.apache.beam.model.fn_execution.v1.StateResponse.append:type_name -> org.apache.beam.model.fn_execution.v1.StateAppendResponse + 36, // 54: org.apache.beam.model.fn_execution.v1.StateResponse.clear:type_name -> org.apache.beam.model.fn_execution.v1.StateClearResponse + 66, // 55: org.apache.beam.model.fn_execution.v1.StateKey.runner:type_name -> org.apache.beam.model.fn_execution.v1.StateKey.Runner + 68, // 56: org.apache.beam.model.fn_execution.v1.StateKey.multimap_side_input:type_name -> org.apache.beam.model.fn_execution.v1.StateKey.MultimapSideInput + 70, // 57: org.apache.beam.model.fn_execution.v1.StateKey.bag_user_state:type_name -> org.apache.beam.model.fn_execution.v1.StateKey.BagUserState + 67, // 58: org.apache.beam.model.fn_execution.v1.StateKey.iterable_side_input:type_name -> org.apache.beam.model.fn_execution.v1.StateKey.IterableSideInput + 69, // 59: org.apache.beam.model.fn_execution.v1.StateKey.multimap_keys_side_input:type_name -> org.apache.beam.model.fn_execution.v1.StateKey.MultimapKeysSideInput + 71, // 60: org.apache.beam.model.fn_execution.v1.StateKey.multimap_keys_user_state:type_name -> org.apache.beam.model.fn_execution.v1.StateKey.MultimapKeysUserState + 72, // 61: org.apache.beam.model.fn_execution.v1.StateKey.multimap_user_state:type_name -> org.apache.beam.model.fn_execution.v1.StateKey.MultimapUserState + 1, // 62: org.apache.beam.model.fn_execution.v1.LogEntry.severity:type_name -> org.apache.beam.model.fn_execution.v1.LogEntry.Severity.Enum + 77, // 63: org.apache.beam.model.fn_execution.v1.LogEntry.timestamp:type_name -> google.protobuf.Timestamp + 81, // 64: org.apache.beam.model.fn_execution.v1.LogEntry.custom_data:type_name -> google.protobuf.Struct + 76, // 65: org.apache.beam.model.fn_execution.v1.StartWorkerRequest.control_endpoint:type_name -> org.apache.beam.model.pipeline.v1.ApiServiceDescriptor + 76, // 66: org.apache.beam.model.fn_execution.v1.StartWorkerRequest.logging_endpoint:type_name -> org.apache.beam.model.pipeline.v1.ApiServiceDescriptor + 76, // 67: org.apache.beam.model.fn_execution.v1.StartWorkerRequest.artifact_endpoint:type_name -> org.apache.beam.model.pipeline.v1.ApiServiceDescriptor + 76, // 68: org.apache.beam.model.fn_execution.v1.StartWorkerRequest.provision_endpoint:type_name -> org.apache.beam.model.pipeline.v1.ApiServiceDescriptor + 75, // 69: org.apache.beam.model.fn_execution.v1.StartWorkerRequest.params:type_name -> org.apache.beam.model.fn_execution.v1.StartWorkerRequest.ParamsEntry + 8, // 70: org.apache.beam.model.fn_execution.v1.SampleDataResponse.ElementList.elements:type_name -> org.apache.beam.model.fn_execution.v1.SampledElement + 46, // 71: org.apache.beam.model.fn_execution.v1.SampleDataResponse.ElementSamplesEntry.value:type_name -> org.apache.beam.model.fn_execution.v1.SampleDataResponse.ElementList + 82, // 72: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.TransformsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.PTransform + 83, // 73: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.PcollectionsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.PCollection + 84, // 74: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.WindowingStrategiesEntry.value:type_name -> org.apache.beam.model.pipeline.v1.WindowingStrategy + 85, // 75: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.CodersEntry.value:type_name -> org.apache.beam.model.pipeline.v1.Coder + 86, // 76: org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor.EnvironmentsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.Environment + 77, // 77: org.apache.beam.model.fn_execution.v1.BundleApplication.OutputWatermarksEntry.value:type_name -> google.protobuf.Timestamp + 56, // 78: org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.CacheToken.user_state:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.CacheToken.UserState + 57, // 79: org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.CacheToken.side_input:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleRequest.CacheToken.SideInput + 80, // 80: org.apache.beam.model.fn_execution.v1.MonitoringInfosMetadataResponse.MonitoringInfoEntry.value:type_name -> org.apache.beam.model.pipeline.v1.MonitoringInfo + 61, // 81: org.apache.beam.model.fn_execution.v1.ProcessBundleSplitRequest.DesiredSplitsEntry.value:type_name -> org.apache.beam.model.fn_execution.v1.ProcessBundleSplitRequest.DesiredSplit + 37, // 82: org.apache.beam.model.fn_execution.v1.LogEntry.List.log_entries:type_name -> org.apache.beam.model.fn_execution.v1.LogEntry + 6, // 83: org.apache.beam.model.fn_execution.v1.BeamFnControl.Control:input_type -> org.apache.beam.model.fn_execution.v1.InstructionResponse + 4, // 84: org.apache.beam.model.fn_execution.v1.BeamFnControl.GetProcessBundleDescriptor:input_type -> org.apache.beam.model.fn_execution.v1.GetProcessBundleDescriptorRequest + 27, // 85: org.apache.beam.model.fn_execution.v1.BeamFnData.Data:input_type -> org.apache.beam.model.fn_execution.v1.Elements + 28, // 86: org.apache.beam.model.fn_execution.v1.BeamFnState.State:input_type -> org.apache.beam.model.fn_execution.v1.StateRequest + 73, // 87: org.apache.beam.model.fn_execution.v1.BeamFnLogging.Logging:input_type -> org.apache.beam.model.fn_execution.v1.LogEntry.List + 39, // 88: org.apache.beam.model.fn_execution.v1.BeamFnExternalWorkerPool.StartWorker:input_type -> org.apache.beam.model.fn_execution.v1.StartWorkerRequest + 41, // 89: org.apache.beam.model.fn_execution.v1.BeamFnExternalWorkerPool.StopWorker:input_type -> org.apache.beam.model.fn_execution.v1.StopWorkerRequest + 44, // 90: org.apache.beam.model.fn_execution.v1.BeamFnWorkerStatus.WorkerStatus:input_type -> org.apache.beam.model.fn_execution.v1.WorkerStatusResponse + 5, // 91: org.apache.beam.model.fn_execution.v1.BeamFnControl.Control:output_type -> org.apache.beam.model.fn_execution.v1.InstructionRequest + 14, // 92: org.apache.beam.model.fn_execution.v1.BeamFnControl.GetProcessBundleDescriptor:output_type -> org.apache.beam.model.fn_execution.v1.ProcessBundleDescriptor + 27, // 93: org.apache.beam.model.fn_execution.v1.BeamFnData.Data:output_type -> org.apache.beam.model.fn_execution.v1.Elements + 29, // 94: org.apache.beam.model.fn_execution.v1.BeamFnState.State:output_type -> org.apache.beam.model.fn_execution.v1.StateResponse + 38, // 95: org.apache.beam.model.fn_execution.v1.BeamFnLogging.Logging:output_type -> org.apache.beam.model.fn_execution.v1.LogControl + 40, // 96: org.apache.beam.model.fn_execution.v1.BeamFnExternalWorkerPool.StartWorker:output_type -> org.apache.beam.model.fn_execution.v1.StartWorkerResponse + 42, // 97: org.apache.beam.model.fn_execution.v1.BeamFnExternalWorkerPool.StopWorker:output_type -> org.apache.beam.model.fn_execution.v1.StopWorkerResponse + 43, // 98: org.apache.beam.model.fn_execution.v1.BeamFnWorkerStatus.WorkerStatus:output_type -> org.apache.beam.model.fn_execution.v1.WorkerStatusRequest + 91, // [91:99] is the sub-list for method output_type + 83, // [83:91] is the sub-list for method input_type + 83, // [83:83] is the sub-list for extension type_name + 83, // [83:83] is the sub-list for extension extendee + 0, // [0:83] is the sub-list for field type_name } func init() { file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() } @@ -5298,7 +5862,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } if !protoimpl.UnsafeEnabled { file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoteGrpcPort); i { + switch v := v.(*FnApiTransforms); i { case 0: return &v.state case 1: @@ -5310,7 +5874,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetProcessBundleDescriptorRequest); i { + switch v := v.(*RemoteGrpcPort); i { case 0: return &v.state case 1: @@ -5322,7 +5886,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*InstructionRequest); i { + switch v := v.(*GetProcessBundleDescriptorRequest); i { case 0: return &v.state case 1: @@ -5334,7 +5898,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*InstructionResponse); i { + switch v := v.(*InstructionRequest); i { case 0: return &v.state case 1: @@ -5346,7 +5910,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HarnessMonitoringInfosRequest); i { + switch v := v.(*InstructionResponse); i { case 0: return &v.state case 1: @@ -5358,7 +5922,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HarnessMonitoringInfosResponse); i { + switch v := v.(*SampleDataRequest); i { case 0: return &v.state case 1: @@ -5370,7 +5934,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RegisterRequest); i { + switch v := v.(*SampledElement); i { case 0: return &v.state case 1: @@ -5382,7 +5946,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RegisterResponse); i { + switch v := v.(*SampleDataResponse); i { case 0: return &v.state case 1: @@ -5394,7 +5958,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProcessBundleDescriptor); i { + switch v := v.(*HarnessMonitoringInfosRequest); i { case 0: return &v.state case 1: @@ -5406,7 +5970,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BundleApplication); i { + switch v := v.(*HarnessMonitoringInfosResponse); i { case 0: return &v.state case 1: @@ -5418,7 +5982,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DelayedBundleApplication); i { + switch v := v.(*RegisterRequest); i { case 0: return &v.state case 1: @@ -5430,7 +5994,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProcessBundleRequest); i { + switch v := v.(*RegisterResponse); i { case 0: return &v.state case 1: @@ -5442,7 +6006,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProcessBundleResponse); i { + switch v := v.(*ProcessBundleDescriptor); i { case 0: return &v.state case 1: @@ -5454,7 +6018,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProcessBundleProgressRequest); i { + switch v := v.(*BundleApplication); i { case 0: return &v.state case 1: @@ -5466,7 +6030,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MonitoringInfosMetadataRequest); i { + switch v := v.(*DelayedBundleApplication); i { case 0: return &v.state case 1: @@ -5478,7 +6042,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProcessBundleProgressResponse); i { + switch v := v.(*ProcessBundleRequest); i { case 0: return &v.state case 1: @@ -5490,7 +6054,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MonitoringInfosMetadataResponse); i { + switch v := v.(*ProcessBundleResponse); i { case 0: return &v.state case 1: @@ -5502,7 +6066,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProcessBundleSplitRequest); i { + switch v := v.(*ProcessBundleProgressRequest); i { case 0: return &v.state case 1: @@ -5514,7 +6078,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProcessBundleSplitResponse); i { + switch v := v.(*MonitoringInfosMetadataRequest); i { case 0: return &v.state case 1: @@ -5526,7 +6090,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FinalizeBundleRequest); i { + switch v := v.(*ProcessBundleProgressResponse); i { case 0: return &v.state case 1: @@ -5538,7 +6102,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FinalizeBundleResponse); i { + switch v := v.(*MonitoringInfosMetadataResponse); i { case 0: return &v.state case 1: @@ -5550,7 +6114,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Elements); i { + switch v := v.(*ProcessBundleSplitRequest); i { case 0: return &v.state case 1: @@ -5562,7 +6126,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StateRequest); i { + switch v := v.(*ProcessBundleSplitResponse); i { case 0: return &v.state case 1: @@ -5574,7 +6138,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StateResponse); i { + switch v := v.(*FinalizeBundleRequest); i { case 0: return &v.state case 1: @@ -5586,7 +6150,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StateKey); i { + switch v := v.(*FinalizeBundleResponse); i { case 0: return &v.state case 1: @@ -5598,7 +6162,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StateGetRequest); i { + switch v := v.(*Elements); i { case 0: return &v.state case 1: @@ -5610,7 +6174,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StateGetResponse); i { + switch v := v.(*StateRequest); i { case 0: return &v.state case 1: @@ -5622,7 +6186,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StateAppendRequest); i { + switch v := v.(*StateResponse); i { case 0: return &v.state case 1: @@ -5634,7 +6198,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StateAppendResponse); i { + switch v := v.(*StateKey); i { case 0: return &v.state case 1: @@ -5646,7 +6210,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StateClearRequest); i { + switch v := v.(*StateGetRequest); i { case 0: return &v.state case 1: @@ -5658,7 +6222,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StateClearResponse); i { + switch v := v.(*StateGetResponse); i { case 0: return &v.state case 1: @@ -5670,7 +6234,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LogEntry); i { + switch v := v.(*StateAppendRequest); i { case 0: return &v.state case 1: @@ -5682,7 +6246,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LogControl); i { + switch v := v.(*StateAppendResponse); i { case 0: return &v.state case 1: @@ -5694,7 +6258,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartWorkerRequest); i { + switch v := v.(*StateClearRequest); i { case 0: return &v.state case 1: @@ -5706,7 +6270,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartWorkerResponse); i { + switch v := v.(*StateClearResponse); i { case 0: return &v.state case 1: @@ -5718,7 +6282,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopWorkerRequest); i { + switch v := v.(*LogEntry); i { case 0: return &v.state case 1: @@ -5730,7 +6294,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopWorkerResponse); i { + switch v := v.(*LogControl); i { case 0: return &v.state case 1: @@ -5742,7 +6306,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*WorkerStatusRequest); i { + switch v := v.(*StartWorkerRequest); i { case 0: return &v.state case 1: @@ -5754,6 +6318,54 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StartWorkerResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StopWorkerRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StopWorkerResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*WorkerStatusRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*WorkerStatusResponse); i { case 0: return &v.state @@ -5765,7 +6377,31 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { return nil } } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SampledElement_Exception); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SampleDataResponse_ElementList); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ProcessBundleRequest_CacheToken); i { case 0: return &v.state @@ -5777,7 +6413,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { return nil } } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ProcessBundleRequest_CacheToken_UserState); i { case 0: return &v.state @@ -5789,7 +6425,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { return nil } } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ProcessBundleRequest_CacheToken_SideInput); i { case 0: return &v.state @@ -5801,7 +6437,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { return nil } } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ProcessBundleSplitRequest_DesiredSplit); i { case 0: return &v.state @@ -5813,7 +6449,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { return nil } } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ProcessBundleSplitResponse_ChannelSplit); i { case 0: return &v.state @@ -5825,7 +6461,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { return nil } } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Elements_Data); i { case 0: return &v.state @@ -5837,7 +6473,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { return nil } } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Elements_Timers); i { case 0: return &v.state @@ -5849,7 +6485,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { return nil } } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[64].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*StateKey_Runner); i { case 0: return &v.state @@ -5861,7 +6497,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { return nil } } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[65].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*StateKey_IterableSideInput); i { case 0: return &v.state @@ -5873,7 +6509,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { return nil } } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[66].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*StateKey_MultimapSideInput); i { case 0: return &v.state @@ -5885,7 +6521,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { return nil } } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[67].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*StateKey_MultimapKeysSideInput); i { case 0: return &v.state @@ -5897,7 +6533,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { return nil } } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[68].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*StateKey_BagUserState); i { case 0: return &v.state @@ -5909,7 +6545,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { return nil } } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[69].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*StateKey_MultimapKeysUserState); i { case 0: return &v.state @@ -5921,7 +6557,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { return nil } } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[70].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*StateKey_MultimapUserState); i { case 0: return &v.state @@ -5933,7 +6569,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { return nil } } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[64].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[71].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*LogEntry_List); i { case 0: return &v.state @@ -5945,7 +6581,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { return nil } } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[65].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[72].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*LogEntry_Severity); i { case 0: return &v.state @@ -5958,35 +6594,37 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { } } } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[2].OneofWrappers = []interface{}{ + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[3].OneofWrappers = []interface{}{ (*InstructionRequest_ProcessBundle)(nil), (*InstructionRequest_ProcessBundleProgress)(nil), (*InstructionRequest_ProcessBundleSplit)(nil), (*InstructionRequest_FinalizeBundle)(nil), (*InstructionRequest_MonitoringInfos)(nil), (*InstructionRequest_HarnessMonitoringInfos)(nil), + (*InstructionRequest_SampleData)(nil), (*InstructionRequest_Register)(nil), } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[3].OneofWrappers = []interface{}{ + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[4].OneofWrappers = []interface{}{ (*InstructionResponse_ProcessBundle)(nil), (*InstructionResponse_ProcessBundleProgress)(nil), (*InstructionResponse_ProcessBundleSplit)(nil), (*InstructionResponse_FinalizeBundle)(nil), (*InstructionResponse_MonitoringInfos)(nil), (*InstructionResponse_HarnessMonitoringInfos)(nil), + (*InstructionResponse_SampleData)(nil), (*InstructionResponse_Register)(nil), } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[22].OneofWrappers = []interface{}{ + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[26].OneofWrappers = []interface{}{ (*StateRequest_Get)(nil), (*StateRequest_Append)(nil), (*StateRequest_Clear)(nil), } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[23].OneofWrappers = []interface{}{ + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[27].OneofWrappers = []interface{}{ (*StateResponse_Get)(nil), (*StateResponse_Append)(nil), (*StateResponse_Clear)(nil), } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[24].OneofWrappers = []interface{}{ + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[28].OneofWrappers = []interface{}{ (*StateKey_Runner_)(nil), (*StateKey_MultimapSideInput_)(nil), (*StateKey_BagUserState_)(nil), @@ -5995,7 +6633,7 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { (*StateKey_MultimapKeysUserState_)(nil), (*StateKey_MultimapUserState_)(nil), } - file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[46].OneofWrappers = []interface{}{ + file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_msgTypes[53].OneofWrappers = []interface{}{ (*ProcessBundleRequest_CacheToken_UserState_)(nil), (*ProcessBundleRequest_CacheToken_SideInput_)(nil), } @@ -6004,8 +6642,8 @@ func file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_org_apache_beam_model_fn_execution_v1_beam_fn_api_proto_rawDesc, - NumEnums: 1, - NumMessages: 67, + NumEnums: 2, + NumMessages: 74, NumExtensions: 0, NumServices: 6, }, diff --git a/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api_grpc.pb.go b/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api_grpc.pb.go index 699bb99fa5551..cd53ea805705e 100644 --- a/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api_grpc.pb.go +++ b/sdks/go/pkg/beam/model/fnexecution_v1/beam_fn_api_grpc.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.1.0 -// - protoc v3.21.9 +// - protoc v4.24.4 // source: org/apache/beam/model/fn_execution/v1/beam_fn_api.proto package fnexecution_v1 diff --git a/sdks/go/pkg/beam/model/fnexecution_v1/beam_provision_api.pb.go b/sdks/go/pkg/beam/model/fnexecution_v1/beam_provision_api.pb.go index f5649728069a0..26cf245f72069 100644 --- a/sdks/go/pkg/beam/model/fnexecution_v1/beam_provision_api.pb.go +++ b/sdks/go/pkg/beam/model/fnexecution_v1/beam_provision_api.pb.go @@ -22,7 +22,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.27.1 -// protoc v3.21.9 +// protoc v4.24.4 // source: org/apache/beam/model/fn_execution/v1/beam_provision_api.proto package fnexecution_v1 diff --git a/sdks/go/pkg/beam/model/fnexecution_v1/beam_provision_api_grpc.pb.go b/sdks/go/pkg/beam/model/fnexecution_v1/beam_provision_api_grpc.pb.go index fa58bfd2c5ef5..9064b348b4c04 100644 --- a/sdks/go/pkg/beam/model/fnexecution_v1/beam_provision_api_grpc.pb.go +++ b/sdks/go/pkg/beam/model/fnexecution_v1/beam_provision_api_grpc.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.1.0 -// - protoc v3.21.9 +// - protoc v4.24.4 // source: org/apache/beam/model/fn_execution/v1/beam_provision_api.proto package fnexecution_v1 diff --git a/sdks/go/pkg/beam/model/jobmanagement_v1/beam_artifact_api.pb.go b/sdks/go/pkg/beam/model/jobmanagement_v1/beam_artifact_api.pb.go index c59a8983d0fcd..85bb2e368970d 100644 --- a/sdks/go/pkg/beam/model/jobmanagement_v1/beam_artifact_api.pb.go +++ b/sdks/go/pkg/beam/model/jobmanagement_v1/beam_artifact_api.pb.go @@ -22,7 +22,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.27.1 -// protoc v3.21.9 +// protoc v4.24.4 // source: org/apache/beam/model/job_management/v1/beam_artifact_api.proto package jobmanagement_v1 diff --git a/sdks/go/pkg/beam/model/jobmanagement_v1/beam_artifact_api_grpc.pb.go b/sdks/go/pkg/beam/model/jobmanagement_v1/beam_artifact_api_grpc.pb.go index 960a44f4044a1..28e43e21fbbdf 100644 --- a/sdks/go/pkg/beam/model/jobmanagement_v1/beam_artifact_api_grpc.pb.go +++ b/sdks/go/pkg/beam/model/jobmanagement_v1/beam_artifact_api_grpc.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.1.0 -// - protoc v3.21.9 +// - protoc v4.24.4 // source: org/apache/beam/model/job_management/v1/beam_artifact_api.proto package jobmanagement_v1 diff --git a/sdks/go/pkg/beam/model/jobmanagement_v1/beam_expansion_api.pb.go b/sdks/go/pkg/beam/model/jobmanagement_v1/beam_expansion_api.pb.go index bcd03e0301bc4..8f7ca43ec0f5b 100644 --- a/sdks/go/pkg/beam/model/jobmanagement_v1/beam_expansion_api.pb.go +++ b/sdks/go/pkg/beam/model/jobmanagement_v1/beam_expansion_api.pb.go @@ -22,7 +22,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.27.1 -// protoc v3.21.9 +// protoc v4.24.4 // source: org/apache/beam/model/job_management/v1/beam_expansion_api.proto package jobmanagement_v1 diff --git a/sdks/go/pkg/beam/model/jobmanagement_v1/beam_expansion_api_grpc.pb.go b/sdks/go/pkg/beam/model/jobmanagement_v1/beam_expansion_api_grpc.pb.go index d42341ccd9e7a..f1c3782f5fb80 100644 --- a/sdks/go/pkg/beam/model/jobmanagement_v1/beam_expansion_api_grpc.pb.go +++ b/sdks/go/pkg/beam/model/jobmanagement_v1/beam_expansion_api_grpc.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.1.0 -// - protoc v3.21.9 +// - protoc v4.24.4 // source: org/apache/beam/model/job_management/v1/beam_expansion_api.proto package jobmanagement_v1 diff --git a/sdks/go/pkg/beam/model/jobmanagement_v1/beam_job_api.pb.go b/sdks/go/pkg/beam/model/jobmanagement_v1/beam_job_api.pb.go index 1c588d3b27610..62e0b313ec2da 100644 --- a/sdks/go/pkg/beam/model/jobmanagement_v1/beam_job_api.pb.go +++ b/sdks/go/pkg/beam/model/jobmanagement_v1/beam_job_api.pb.go @@ -22,7 +22,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.27.1 -// protoc v3.21.9 +// protoc v4.24.4 // source: org/apache/beam/model/job_management/v1/beam_job_api.proto package jobmanagement_v1 diff --git a/sdks/go/pkg/beam/model/jobmanagement_v1/beam_job_api_grpc.pb.go b/sdks/go/pkg/beam/model/jobmanagement_v1/beam_job_api_grpc.pb.go index fb5ba496faf07..38f2c85a1c1cd 100644 --- a/sdks/go/pkg/beam/model/jobmanagement_v1/beam_job_api_grpc.pb.go +++ b/sdks/go/pkg/beam/model/jobmanagement_v1/beam_job_api_grpc.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.1.0 -// - protoc v3.21.9 +// - protoc v4.24.4 // source: org/apache/beam/model/job_management/v1/beam_job_api.proto package jobmanagement_v1 diff --git a/sdks/go/pkg/beam/model/pipeline_v1/beam_runner_api.pb.go b/sdks/go/pkg/beam/model/pipeline_v1/beam_runner_api.pb.go index f45436293611d..49df2b5c2e597 100644 --- a/sdks/go/pkg/beam/model/pipeline_v1/beam_runner_api.pb.go +++ b/sdks/go/pkg/beam/model/pipeline_v1/beam_runner_api.pb.go @@ -22,7 +22,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.27.1 -// protoc v3.21.9 +// protoc v4.24.4 // source: org/apache/beam/model/pipeline/v1/beam_runner_api.proto package pipeline_v1 @@ -752,7 +752,7 @@ func (x IsBounded_Enum) Number() protoreflect.EnumNumber { // Deprecated: Use IsBounded_Enum.Descriptor instead. func (IsBounded_Enum) EnumDescriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{17, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{18, 0} } type StandardCoders_Enum int32 @@ -897,7 +897,6 @@ const ( // remainder of the iterable (e.g. over the state API). // // Components: Coder for a single element. - // Experimental. StandardCoders_STATE_BACKED_ITERABLE StandardCoders_Enum = 9 // Encodes an arbitrary user defined window and its max timestamp (inclusive). // The encoding format is: @@ -974,7 +973,6 @@ const ( // // The payload for RowCoder is an instance of Schema. // Components: None - // Experimental. StandardCoders_ROW StandardCoders_Enum = 13 // Encodes a user key and a shard id which is an opaque byte string. // @@ -997,7 +995,6 @@ const ( // encode(user_key) // // Components: the user key coder. - // Experimental. StandardCoders_SHARDED_KEY StandardCoders_Enum = 15 // Wraps a coder of a potentially null value // A Nullable Type is encoded by: @@ -1078,7 +1075,7 @@ func (x StandardCoders_Enum) Number() protoreflect.EnumNumber { // Deprecated: Use StandardCoders_Enum.Descriptor instead. func (StandardCoders_Enum) EnumDescriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{28, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{29, 0} } type MergeStatus_Enum int32 @@ -1138,7 +1135,7 @@ func (x MergeStatus_Enum) Number() protoreflect.EnumNumber { // Deprecated: Use MergeStatus_Enum.Descriptor instead. func (MergeStatus_Enum) EnumDescriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{30, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{31, 0} } type AccumulationMode_Enum int32 @@ -1193,7 +1190,7 @@ func (x AccumulationMode_Enum) Number() protoreflect.EnumNumber { // Deprecated: Use AccumulationMode_Enum.Descriptor instead. func (AccumulationMode_Enum) EnumDescriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{31, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{32, 0} } type ClosingBehavior_Enum int32 @@ -1245,7 +1242,7 @@ func (x ClosingBehavior_Enum) Number() protoreflect.EnumNumber { // Deprecated: Use ClosingBehavior_Enum.Descriptor instead. func (ClosingBehavior_Enum) EnumDescriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{32, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{33, 0} } type OnTimeBehavior_Enum int32 @@ -1297,7 +1294,7 @@ func (x OnTimeBehavior_Enum) Number() protoreflect.EnumNumber { // Deprecated: Use OnTimeBehavior_Enum.Descriptor instead. func (OnTimeBehavior_Enum) EnumDescriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{33, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{34, 0} } type OutputTime_Enum int32 @@ -1354,7 +1351,7 @@ func (x OutputTime_Enum) Number() protoreflect.EnumNumber { // Deprecated: Use OutputTime_Enum.Descriptor instead. func (OutputTime_Enum) EnumDescriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{34, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{35, 0} } type TimeDomain_Enum int32 @@ -1406,7 +1403,7 @@ func (x TimeDomain_Enum) Number() protoreflect.EnumNumber { // Deprecated: Use TimeDomain_Enum.Descriptor instead. func (TimeDomain_Enum) EnumDescriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{35, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{36, 0} } type StandardArtifacts_Types int32 @@ -1476,7 +1473,7 @@ func (x StandardArtifacts_Types) Number() protoreflect.EnumNumber { // Deprecated: Use StandardArtifacts_Types.Descriptor instead. func (StandardArtifacts_Types) EnumDescriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{39, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{40, 0} } type StandardArtifacts_Roles int32 @@ -1533,7 +1530,7 @@ func (x StandardArtifacts_Roles) Number() protoreflect.EnumNumber { // Deprecated: Use StandardArtifacts_Roles.Descriptor instead. func (StandardArtifacts_Roles) EnumDescriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{39, 1} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{40, 1} } type StandardEnvironments_Environments int32 @@ -1585,7 +1582,7 @@ func (x StandardEnvironments_Environments) Number() protoreflect.EnumNumber { // Deprecated: Use StandardEnvironments_Environments.Descriptor instead. func (StandardEnvironments_Environments) EnumDescriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{49, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{50, 0} } type StandardProtocols_Enum int32 @@ -1622,6 +1619,12 @@ const ( // SDKs ability to store the data in memory reducing the amount of memory // used overall. StandardProtocols_STATE_CACHING StandardProtocols_Enum = 7 + // Indicates that this SDK can sample in-flight elements. These samples can + // then be queried using the SampleDataRequest. Samples are uniquely associated + // with a PCollection. Meaning, samples are taken for each PCollection + // during bundle processing. This is disabled by default and enabled with the + // `enable_data_sampling` experiment. + StandardProtocols_DATA_SAMPLING StandardProtocols_Enum = 8 ) // Enum value maps for StandardProtocols_Enum. @@ -1635,6 +1638,7 @@ var ( 4: "HARNESS_MONITORING_INFOS", 6: "CONTROL_REQUEST_ELEMENTS_EMBEDDING", 7: "STATE_CACHING", + 8: "DATA_SAMPLING", } StandardProtocols_Enum_value = map[string]int32{ "LEGACY_PROGRESS_REPORTING": 0, @@ -1645,6 +1649,7 @@ var ( "HARNESS_MONITORING_INFOS": 4, "CONTROL_REQUEST_ELEMENTS_EMBEDDING": 6, "STATE_CACHING": 7, + "DATA_SAMPLING": 8, } ) @@ -1672,7 +1677,7 @@ func (x StandardProtocols_Enum) Number() protoreflect.EnumNumber { // Deprecated: Use StandardProtocols_Enum.Descriptor instead. func (StandardProtocols_Enum) EnumDescriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{53, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{54, 0} } type StandardRunnerProtocols_Enum int32 @@ -1722,7 +1727,7 @@ func (x StandardRunnerProtocols_Enum) Number() protoreflect.EnumNumber { // Deprecated: Use StandardRunnerProtocols_Enum.Descriptor instead. func (StandardRunnerProtocols_Enum) EnumDescriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{54, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{55, 0} } type StandardRequirements_Enum int32 @@ -1792,7 +1797,7 @@ func (x StandardRequirements_Enum) Number() protoreflect.EnumNumber { // Deprecated: Use StandardRequirements_Enum.Descriptor instead. func (StandardRequirements_Enum) EnumDescriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{55, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{56, 0} } type StandardDisplayData_DisplayData int32 @@ -1837,7 +1842,7 @@ func (x StandardDisplayData_DisplayData) Number() protoreflect.EnumNumber { // Deprecated: Use StandardDisplayData_DisplayData.Descriptor instead. func (StandardDisplayData_DisplayData) EnumDescriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{57, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{58, 0} } type StandardResourceHints_Enum int32 @@ -1852,6 +1857,10 @@ const ( // SDKs should convert the size to bytes, but can allow users to specify human-friendly units (e.g. GiB). // Payload: ASCII encoded string of the base 10 representation of an integer number of bytes. StandardResourceHints_MIN_RAM_BYTES StandardResourceHints_Enum = 1 + // Describes desired number of CPUs available in transform's execution environment. + // SDKs should accept and validate a positive integer count. + // Payload: ASCII encoded string of the base 10 representation of an integer number of CPUs. + StandardResourceHints_CPU_COUNT StandardResourceHints_Enum = 2 ) // Enum value maps for StandardResourceHints_Enum. @@ -1859,10 +1868,12 @@ var ( StandardResourceHints_Enum_name = map[int32]string{ 0: "ACCELERATOR", 1: "MIN_RAM_BYTES", + 2: "CPU_COUNT", } StandardResourceHints_Enum_value = map[string]int32{ "ACCELERATOR": 0, "MIN_RAM_BYTES": 1, + "CPU_COUNT": 2, } ) @@ -1890,7 +1901,7 @@ func (x StandardResourceHints_Enum) Number() protoreflect.EnumNumber { // Deprecated: Use StandardResourceHints_Enum.Descriptor instead. func (StandardResourceHints_Enum) EnumDescriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{62, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{63, 0} } type BeamConstants struct { @@ -2654,6 +2665,7 @@ type StateSpec struct { // *StateSpec_MapSpec // *StateSpec_SetSpec // *StateSpec_OrderedListSpec + // *StateSpec_MultimapSpec Spec isStateSpec_Spec `protobuf_oneof:"spec"` // (Required) URN of the protocol required by this state specification to present // the desired SDK-specific interface to a UDF. @@ -2747,6 +2759,13 @@ func (x *StateSpec) GetOrderedListSpec() *OrderedListStateSpec { return nil } +func (x *StateSpec) GetMultimapSpec() *MultimapStateSpec { + if x, ok := x.GetSpec().(*StateSpec_MultimapSpec); ok { + return x.MultimapSpec + } + return nil +} + func (x *StateSpec) GetProtocol() *FunctionSpec { if x != nil { return x.Protocol @@ -2782,6 +2801,10 @@ type StateSpec_OrderedListSpec struct { OrderedListSpec *OrderedListStateSpec `protobuf:"bytes,6,opt,name=ordered_list_spec,json=orderedListSpec,proto3,oneof"` } +type StateSpec_MultimapSpec struct { + MultimapSpec *MultimapStateSpec `protobuf:"bytes,8,opt,name=multimap_spec,json=multimapSpec,proto3,oneof"` +} + func (*StateSpec_ReadModifyWriteSpec) isStateSpec_Spec() {} func (*StateSpec_BagSpec) isStateSpec_Spec() {} @@ -2794,6 +2817,8 @@ func (*StateSpec_SetSpec) isStateSpec_Spec() {} func (*StateSpec_OrderedListSpec) isStateSpec_Spec() {} +func (*StateSpec_MultimapSpec) isStateSpec_Spec() {} + type ReadModifyWriteStateSpec struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -3045,6 +3070,61 @@ func (x *MapStateSpec) GetValueCoderId() string { return "" } +type MultimapStateSpec struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + KeyCoderId string `protobuf:"bytes,1,opt,name=key_coder_id,json=keyCoderId,proto3" json:"key_coder_id,omitempty"` + ValueCoderId string `protobuf:"bytes,2,opt,name=value_coder_id,json=valueCoderId,proto3" json:"value_coder_id,omitempty"` +} + +func (x *MultimapStateSpec) Reset() { + *x = MultimapStateSpec{} + if protoimpl.UnsafeEnabled { + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MultimapStateSpec) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MultimapStateSpec) ProtoMessage() {} + +func (x *MultimapStateSpec) ProtoReflect() protoreflect.Message { + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[15] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MultimapStateSpec.ProtoReflect.Descriptor instead. +func (*MultimapStateSpec) Descriptor() ([]byte, []int) { + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{15} +} + +func (x *MultimapStateSpec) GetKeyCoderId() string { + if x != nil { + return x.KeyCoderId + } + return "" +} + +func (x *MultimapStateSpec) GetValueCoderId() string { + if x != nil { + return x.ValueCoderId + } + return "" +} + type SetStateSpec struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -3056,7 +3136,7 @@ type SetStateSpec struct { func (x *SetStateSpec) Reset() { *x = SetStateSpec{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[15] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3069,7 +3149,7 @@ func (x *SetStateSpec) String() string { func (*SetStateSpec) ProtoMessage() {} func (x *SetStateSpec) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[15] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3082,7 +3162,7 @@ func (x *SetStateSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use SetStateSpec.ProtoReflect.Descriptor instead. func (*SetStateSpec) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{15} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{16} } func (x *SetStateSpec) GetElementCoderId() string { @@ -3104,7 +3184,7 @@ type TimerFamilySpec struct { func (x *TimerFamilySpec) Reset() { *x = TimerFamilySpec{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[16] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3117,7 +3197,7 @@ func (x *TimerFamilySpec) String() string { func (*TimerFamilySpec) ProtoMessage() {} func (x *TimerFamilySpec) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[16] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3130,7 +3210,7 @@ func (x *TimerFamilySpec) ProtoReflect() protoreflect.Message { // Deprecated: Use TimerFamilySpec.ProtoReflect.Descriptor instead. func (*TimerFamilySpec) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{16} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{17} } func (x *TimerFamilySpec) GetTimeDomain() TimeDomain_Enum { @@ -3156,7 +3236,7 @@ type IsBounded struct { func (x *IsBounded) Reset() { *x = IsBounded{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[17] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3169,7 +3249,7 @@ func (x *IsBounded) String() string { func (*IsBounded) ProtoMessage() {} func (x *IsBounded) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[17] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3182,7 +3262,7 @@ func (x *IsBounded) ProtoReflect() protoreflect.Message { // Deprecated: Use IsBounded.ProtoReflect.Descriptor instead. func (*IsBounded) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{17} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{18} } // The payload for the primitive Read transform. @@ -3200,7 +3280,7 @@ type ReadPayload struct { func (x *ReadPayload) Reset() { *x = ReadPayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[18] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3213,7 +3293,7 @@ func (x *ReadPayload) String() string { func (*ReadPayload) ProtoMessage() {} func (x *ReadPayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[18] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3226,7 +3306,7 @@ func (x *ReadPayload) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadPayload.ProtoReflect.Descriptor instead. func (*ReadPayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{18} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{19} } func (x *ReadPayload) GetSource() *FunctionSpec { @@ -3256,7 +3336,7 @@ type WindowIntoPayload struct { func (x *WindowIntoPayload) Reset() { *x = WindowIntoPayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[19] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3269,7 +3349,7 @@ func (x *WindowIntoPayload) String() string { func (*WindowIntoPayload) ProtoMessage() {} func (x *WindowIntoPayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[19] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[20] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3282,7 +3362,7 @@ func (x *WindowIntoPayload) ProtoReflect() protoreflect.Message { // Deprecated: Use WindowIntoPayload.ProtoReflect.Descriptor instead. func (*WindowIntoPayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{19} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{20} } func (x *WindowIntoPayload) GetWindowFn() *FunctionSpec { @@ -3307,7 +3387,7 @@ type CombinePayload struct { func (x *CombinePayload) Reset() { *x = CombinePayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[20] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3320,7 +3400,7 @@ func (x *CombinePayload) String() string { func (*CombinePayload) ProtoMessage() {} func (x *CombinePayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[20] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[21] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3333,7 +3413,7 @@ func (x *CombinePayload) ProtoReflect() protoreflect.Message { // Deprecated: Use CombinePayload.ProtoReflect.Descriptor instead. func (*CombinePayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{20} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{21} } func (x *CombinePayload) GetCombineFn() *FunctionSpec { @@ -3368,7 +3448,7 @@ type TestStreamPayload struct { func (x *TestStreamPayload) Reset() { *x = TestStreamPayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[21] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3381,7 +3461,7 @@ func (x *TestStreamPayload) String() string { func (*TestStreamPayload) ProtoMessage() {} func (x *TestStreamPayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[21] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3394,7 +3474,7 @@ func (x *TestStreamPayload) ProtoReflect() protoreflect.Message { // Deprecated: Use TestStreamPayload.ProtoReflect.Descriptor instead. func (*TestStreamPayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{21} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{22} } func (x *TestStreamPayload) GetCoderId() string { @@ -3433,7 +3513,7 @@ type EventsRequest struct { func (x *EventsRequest) Reset() { *x = EventsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[22] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3446,7 +3526,7 @@ func (x *EventsRequest) String() string { func (*EventsRequest) ProtoMessage() {} func (x *EventsRequest) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[22] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3459,7 +3539,7 @@ func (x *EventsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use EventsRequest.ProtoReflect.Descriptor instead. func (*EventsRequest) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{22} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{23} } func (x *EventsRequest) GetOutputIds() []string { @@ -3487,7 +3567,7 @@ type WriteFilesPayload struct { func (x *WriteFilesPayload) Reset() { *x = WriteFilesPayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[23] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3500,7 +3580,7 @@ func (x *WriteFilesPayload) String() string { func (*WriteFilesPayload) ProtoMessage() {} func (x *WriteFilesPayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[23] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3513,7 +3593,7 @@ func (x *WriteFilesPayload) ProtoReflect() protoreflect.Message { // Deprecated: Use WriteFilesPayload.ProtoReflect.Descriptor instead. func (*WriteFilesPayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{23} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{24} } func (x *WriteFilesPayload) GetSink() *FunctionSpec { @@ -3584,7 +3664,7 @@ type PubSubReadPayload struct { func (x *PubSubReadPayload) Reset() { *x = PubSubReadPayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[24] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3597,7 +3677,7 @@ func (x *PubSubReadPayload) String() string { func (*PubSubReadPayload) ProtoMessage() {} func (x *PubSubReadPayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[24] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3610,7 +3690,7 @@ func (x *PubSubReadPayload) ProtoReflect() protoreflect.Message { // Deprecated: Use PubSubReadPayload.ProtoReflect.Descriptor instead. func (*PubSubReadPayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{24} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{25} } func (x *PubSubReadPayload) GetTopic() string { @@ -3688,7 +3768,7 @@ type PubSubWritePayload struct { func (x *PubSubWritePayload) Reset() { *x = PubSubWritePayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[25] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3701,7 +3781,7 @@ func (x *PubSubWritePayload) String() string { func (*PubSubWritePayload) ProtoMessage() {} func (x *PubSubWritePayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[25] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3714,7 +3794,7 @@ func (x *PubSubWritePayload) ProtoReflect() protoreflect.Message { // Deprecated: Use PubSubWritePayload.ProtoReflect.Descriptor instead. func (*PubSubWritePayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{25} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{26} } func (x *PubSubWritePayload) GetTopic() string { @@ -3762,7 +3842,7 @@ type GroupIntoBatchesPayload struct { func (x *GroupIntoBatchesPayload) Reset() { *x = GroupIntoBatchesPayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[26] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3775,7 +3855,7 @@ func (x *GroupIntoBatchesPayload) String() string { func (*GroupIntoBatchesPayload) ProtoMessage() {} func (x *GroupIntoBatchesPayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[26] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3788,7 +3868,7 @@ func (x *GroupIntoBatchesPayload) ProtoReflect() protoreflect.Message { // Deprecated: Use GroupIntoBatchesPayload.ProtoReflect.Descriptor instead. func (*GroupIntoBatchesPayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{26} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{27} } func (x *GroupIntoBatchesPayload) GetBatchSize() int64 { @@ -3834,7 +3914,7 @@ type Coder struct { func (x *Coder) Reset() { *x = Coder{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[27] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3847,7 +3927,7 @@ func (x *Coder) String() string { func (*Coder) ProtoMessage() {} func (x *Coder) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[27] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3860,7 +3940,7 @@ func (x *Coder) ProtoReflect() protoreflect.Message { // Deprecated: Use Coder.ProtoReflect.Descriptor instead. func (*Coder) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{27} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{28} } func (x *Coder) GetSpec() *FunctionSpec { @@ -3886,7 +3966,7 @@ type StandardCoders struct { func (x *StandardCoders) Reset() { *x = StandardCoders{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[28] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3899,7 +3979,7 @@ func (x *StandardCoders) String() string { func (*StandardCoders) ProtoMessage() {} func (x *StandardCoders) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[28] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3912,7 +3992,7 @@ func (x *StandardCoders) ProtoReflect() protoreflect.Message { // Deprecated: Use StandardCoders.ProtoReflect.Descriptor instead. func (*StandardCoders) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{28} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{29} } // A windowing strategy describes the window function, triggering, allowed @@ -3970,7 +4050,7 @@ type WindowingStrategy struct { func (x *WindowingStrategy) Reset() { *x = WindowingStrategy{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[29] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3983,7 +4063,7 @@ func (x *WindowingStrategy) String() string { func (*WindowingStrategy) ProtoMessage() {} func (x *WindowingStrategy) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[29] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[30] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3996,7 +4076,7 @@ func (x *WindowingStrategy) ProtoReflect() protoreflect.Message { // Deprecated: Use WindowingStrategy.ProtoReflect.Descriptor instead. func (*WindowingStrategy) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{29} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{30} } func (x *WindowingStrategy) GetWindowFn() *FunctionSpec { @@ -4088,7 +4168,7 @@ type MergeStatus struct { func (x *MergeStatus) Reset() { *x = MergeStatus{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[30] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4101,7 +4181,7 @@ func (x *MergeStatus) String() string { func (*MergeStatus) ProtoMessage() {} func (x *MergeStatus) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[30] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[31] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4114,7 +4194,7 @@ func (x *MergeStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use MergeStatus.ProtoReflect.Descriptor instead. func (*MergeStatus) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{30} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{31} } // Whether or not subsequent outputs of aggregations should be entire @@ -4129,7 +4209,7 @@ type AccumulationMode struct { func (x *AccumulationMode) Reset() { *x = AccumulationMode{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[31] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4142,7 +4222,7 @@ func (x *AccumulationMode) String() string { func (*AccumulationMode) ProtoMessage() {} func (x *AccumulationMode) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[31] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[32] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4155,7 +4235,7 @@ func (x *AccumulationMode) ProtoReflect() protoreflect.Message { // Deprecated: Use AccumulationMode.ProtoReflect.Descriptor instead. func (*AccumulationMode) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{31} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{32} } // Controls whether or not an aggregating transform should output data @@ -4169,7 +4249,7 @@ type ClosingBehavior struct { func (x *ClosingBehavior) Reset() { *x = ClosingBehavior{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[32] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4182,7 +4262,7 @@ func (x *ClosingBehavior) String() string { func (*ClosingBehavior) ProtoMessage() {} func (x *ClosingBehavior) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[32] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4195,7 +4275,7 @@ func (x *ClosingBehavior) ProtoReflect() protoreflect.Message { // Deprecated: Use ClosingBehavior.ProtoReflect.Descriptor instead. func (*ClosingBehavior) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{32} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{33} } // Controls whether or not an aggregating transform should output data @@ -4209,7 +4289,7 @@ type OnTimeBehavior struct { func (x *OnTimeBehavior) Reset() { *x = OnTimeBehavior{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[33] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4222,7 +4302,7 @@ func (x *OnTimeBehavior) String() string { func (*OnTimeBehavior) ProtoMessage() {} func (x *OnTimeBehavior) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[33] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4235,7 +4315,7 @@ func (x *OnTimeBehavior) ProtoReflect() protoreflect.Message { // Deprecated: Use OnTimeBehavior.ProtoReflect.Descriptor instead. func (*OnTimeBehavior) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{33} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{34} } // When a number of windowed, timestamped inputs are aggregated, the timestamp @@ -4249,7 +4329,7 @@ type OutputTime struct { func (x *OutputTime) Reset() { *x = OutputTime{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[34] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4262,7 +4342,7 @@ func (x *OutputTime) String() string { func (*OutputTime) ProtoMessage() {} func (x *OutputTime) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[34] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4275,7 +4355,7 @@ func (x *OutputTime) ProtoReflect() protoreflect.Message { // Deprecated: Use OutputTime.ProtoReflect.Descriptor instead. func (*OutputTime) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{34} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{35} } // The different time domains in the Beam model. @@ -4288,7 +4368,7 @@ type TimeDomain struct { func (x *TimeDomain) Reset() { *x = TimeDomain{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[35] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4301,7 +4381,7 @@ func (x *TimeDomain) String() string { func (*TimeDomain) ProtoMessage() {} func (x *TimeDomain) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[35] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[36] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4314,7 +4394,7 @@ func (x *TimeDomain) ProtoReflect() protoreflect.Message { // Deprecated: Use TimeDomain.ProtoReflect.Descriptor instead. func (*TimeDomain) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{35} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{36} } // A small DSL for expressing when to emit new aggregations @@ -4348,7 +4428,7 @@ type Trigger struct { func (x *Trigger) Reset() { *x = Trigger{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[36] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4361,7 +4441,7 @@ func (x *Trigger) String() string { func (*Trigger) ProtoMessage() {} func (x *Trigger) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[36] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[37] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4374,7 +4454,7 @@ func (x *Trigger) ProtoReflect() protoreflect.Message { // Deprecated: Use Trigger.ProtoReflect.Descriptor instead. func (*Trigger) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{36} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{37} } func (m *Trigger) GetTrigger() isTrigger_Trigger { @@ -4563,7 +4643,7 @@ type TimestampTransform struct { func (x *TimestampTransform) Reset() { *x = TimestampTransform{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[37] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4576,7 +4656,7 @@ func (x *TimestampTransform) String() string { func (*TimestampTransform) ProtoMessage() {} func (x *TimestampTransform) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[37] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4589,7 +4669,7 @@ func (x *TimestampTransform) ProtoReflect() protoreflect.Message { // Deprecated: Use TimestampTransform.ProtoReflect.Descriptor instead. func (*TimestampTransform) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{37} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{38} } func (m *TimestampTransform) GetTimestampTransform() isTimestampTransform_TimestampTransform { @@ -4662,7 +4742,7 @@ type SideInput struct { func (x *SideInput) Reset() { *x = SideInput{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[38] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4675,7 +4755,7 @@ func (x *SideInput) String() string { func (*SideInput) ProtoMessage() {} func (x *SideInput) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[38] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4688,7 +4768,7 @@ func (x *SideInput) ProtoReflect() protoreflect.Message { // Deprecated: Use SideInput.ProtoReflect.Descriptor instead. func (*SideInput) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{38} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{39} } func (x *SideInput) GetAccessPattern() *FunctionSpec { @@ -4721,7 +4801,7 @@ type StandardArtifacts struct { func (x *StandardArtifacts) Reset() { *x = StandardArtifacts{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[39] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4734,7 +4814,7 @@ func (x *StandardArtifacts) String() string { func (*StandardArtifacts) ProtoMessage() {} func (x *StandardArtifacts) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[39] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4747,7 +4827,7 @@ func (x *StandardArtifacts) ProtoReflect() protoreflect.Message { // Deprecated: Use StandardArtifacts.ProtoReflect.Descriptor instead. func (*StandardArtifacts) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{39} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{40} } type ArtifactFilePayload struct { @@ -4764,7 +4844,7 @@ type ArtifactFilePayload struct { func (x *ArtifactFilePayload) Reset() { *x = ArtifactFilePayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[40] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4777,7 +4857,7 @@ func (x *ArtifactFilePayload) String() string { func (*ArtifactFilePayload) ProtoMessage() {} func (x *ArtifactFilePayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[40] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4790,7 +4870,7 @@ func (x *ArtifactFilePayload) ProtoReflect() protoreflect.Message { // Deprecated: Use ArtifactFilePayload.ProtoReflect.Descriptor instead. func (*ArtifactFilePayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{40} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{41} } func (x *ArtifactFilePayload) GetPath() string { @@ -4821,7 +4901,7 @@ type ArtifactUrlPayload struct { func (x *ArtifactUrlPayload) Reset() { *x = ArtifactUrlPayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[41] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4834,7 +4914,7 @@ func (x *ArtifactUrlPayload) String() string { func (*ArtifactUrlPayload) ProtoMessage() {} func (x *ArtifactUrlPayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[41] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[42] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4847,7 +4927,7 @@ func (x *ArtifactUrlPayload) ProtoReflect() protoreflect.Message { // Deprecated: Use ArtifactUrlPayload.ProtoReflect.Descriptor instead. func (*ArtifactUrlPayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{41} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{42} } func (x *ArtifactUrlPayload) GetUrl() string { @@ -4876,7 +4956,7 @@ type EmbeddedFilePayload struct { func (x *EmbeddedFilePayload) Reset() { *x = EmbeddedFilePayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[42] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4889,7 +4969,7 @@ func (x *EmbeddedFilePayload) String() string { func (*EmbeddedFilePayload) ProtoMessage() {} func (x *EmbeddedFilePayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[42] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[43] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4902,7 +4982,7 @@ func (x *EmbeddedFilePayload) ProtoReflect() protoreflect.Message { // Deprecated: Use EmbeddedFilePayload.ProtoReflect.Descriptor instead. func (*EmbeddedFilePayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{42} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{43} } func (x *EmbeddedFilePayload) GetData() []byte { @@ -4926,7 +5006,7 @@ type PyPIPayload struct { func (x *PyPIPayload) Reset() { *x = PyPIPayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[43] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4939,7 +5019,7 @@ func (x *PyPIPayload) String() string { func (*PyPIPayload) ProtoMessage() {} func (x *PyPIPayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[43] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[44] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4952,7 +5032,7 @@ func (x *PyPIPayload) ProtoReflect() protoreflect.Message { // Deprecated: Use PyPIPayload.ProtoReflect.Descriptor instead. func (*PyPIPayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{43} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{44} } func (x *PyPIPayload) GetArtifactId() string { @@ -4984,7 +5064,7 @@ type MavenPayload struct { func (x *MavenPayload) Reset() { *x = MavenPayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[44] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4997,7 +5077,7 @@ func (x *MavenPayload) String() string { func (*MavenPayload) ProtoMessage() {} func (x *MavenPayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[44] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[45] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5010,7 +5090,7 @@ func (x *MavenPayload) ProtoReflect() protoreflect.Message { // Deprecated: Use MavenPayload.ProtoReflect.Descriptor instead. func (*MavenPayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{44} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{45} } func (x *MavenPayload) GetArtifact() string { @@ -5042,7 +5122,7 @@ type DeferredArtifactPayload struct { func (x *DeferredArtifactPayload) Reset() { *x = DeferredArtifactPayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[45] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5055,7 +5135,7 @@ func (x *DeferredArtifactPayload) String() string { func (*DeferredArtifactPayload) ProtoMessage() {} func (x *DeferredArtifactPayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[45] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[46] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5068,7 +5148,7 @@ func (x *DeferredArtifactPayload) ProtoReflect() protoreflect.Message { // Deprecated: Use DeferredArtifactPayload.ProtoReflect.Descriptor instead. func (*DeferredArtifactPayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{45} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{46} } func (x *DeferredArtifactPayload) GetKey() string { @@ -5097,7 +5177,7 @@ type ArtifactStagingToRolePayload struct { func (x *ArtifactStagingToRolePayload) Reset() { *x = ArtifactStagingToRolePayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[46] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5110,7 +5190,7 @@ func (x *ArtifactStagingToRolePayload) String() string { func (*ArtifactStagingToRolePayload) ProtoMessage() {} func (x *ArtifactStagingToRolePayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[46] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[47] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5123,7 +5203,7 @@ func (x *ArtifactStagingToRolePayload) ProtoReflect() protoreflect.Message { // Deprecated: Use ArtifactStagingToRolePayload.ProtoReflect.Descriptor instead. func (*ArtifactStagingToRolePayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{46} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{47} } func (x *ArtifactStagingToRolePayload) GetStagedName() string { @@ -5149,7 +5229,7 @@ type ArtifactInformation struct { func (x *ArtifactInformation) Reset() { *x = ArtifactInformation{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[47] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5162,7 +5242,7 @@ func (x *ArtifactInformation) String() string { func (*ArtifactInformation) ProtoMessage() {} func (x *ArtifactInformation) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[47] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[48] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5175,7 +5255,7 @@ func (x *ArtifactInformation) ProtoReflect() protoreflect.Message { // Deprecated: Use ArtifactInformation.ProtoReflect.Descriptor instead. func (*ArtifactInformation) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{47} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{48} } func (x *ArtifactInformation) GetTypeUrn() string { @@ -5238,7 +5318,7 @@ type Environment struct { func (x *Environment) Reset() { *x = Environment{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[48] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5251,7 +5331,7 @@ func (x *Environment) String() string { func (*Environment) ProtoMessage() {} func (x *Environment) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[48] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[49] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5264,7 +5344,7 @@ func (x *Environment) ProtoReflect() protoreflect.Message { // Deprecated: Use Environment.ProtoReflect.Descriptor instead. func (*Environment) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{48} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{49} } func (x *Environment) GetUrn() string { @@ -5318,7 +5398,7 @@ type StandardEnvironments struct { func (x *StandardEnvironments) Reset() { *x = StandardEnvironments{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[49] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5331,7 +5411,7 @@ func (x *StandardEnvironments) String() string { func (*StandardEnvironments) ProtoMessage() {} func (x *StandardEnvironments) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[49] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[50] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5344,7 +5424,7 @@ func (x *StandardEnvironments) ProtoReflect() protoreflect.Message { // Deprecated: Use StandardEnvironments.ProtoReflect.Descriptor instead. func (*StandardEnvironments) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{49} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{50} } // The payload of a Docker image @@ -5359,7 +5439,7 @@ type DockerPayload struct { func (x *DockerPayload) Reset() { *x = DockerPayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[50] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5372,7 +5452,7 @@ func (x *DockerPayload) String() string { func (*DockerPayload) ProtoMessage() {} func (x *DockerPayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[50] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[51] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5385,7 +5465,7 @@ func (x *DockerPayload) ProtoReflect() protoreflect.Message { // Deprecated: Use DockerPayload.ProtoReflect.Descriptor instead. func (*DockerPayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{50} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{51} } func (x *DockerPayload) GetContainerImage() string { @@ -5409,7 +5489,7 @@ type ProcessPayload struct { func (x *ProcessPayload) Reset() { *x = ProcessPayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[51] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5422,7 +5502,7 @@ func (x *ProcessPayload) String() string { func (*ProcessPayload) ProtoMessage() {} func (x *ProcessPayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[51] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[52] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5435,7 +5515,7 @@ func (x *ProcessPayload) ProtoReflect() protoreflect.Message { // Deprecated: Use ProcessPayload.ProtoReflect.Descriptor instead. func (*ProcessPayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{51} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{52} } func (x *ProcessPayload) GetOs() string { @@ -5478,7 +5558,7 @@ type ExternalPayload struct { func (x *ExternalPayload) Reset() { *x = ExternalPayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[52] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5491,7 +5571,7 @@ func (x *ExternalPayload) String() string { func (*ExternalPayload) ProtoMessage() {} func (x *ExternalPayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[52] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[53] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5504,7 +5584,7 @@ func (x *ExternalPayload) ProtoReflect() protoreflect.Message { // Deprecated: Use ExternalPayload.ProtoReflect.Descriptor instead. func (*ExternalPayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{52} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{53} } func (x *ExternalPayload) GetEndpoint() *ApiServiceDescriptor { @@ -5533,7 +5613,7 @@ type StandardProtocols struct { func (x *StandardProtocols) Reset() { *x = StandardProtocols{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[53] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5546,7 +5626,7 @@ func (x *StandardProtocols) String() string { func (*StandardProtocols) ProtoMessage() {} func (x *StandardProtocols) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[53] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[54] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5559,7 +5639,7 @@ func (x *StandardProtocols) ProtoReflect() protoreflect.Message { // Deprecated: Use StandardProtocols.ProtoReflect.Descriptor instead. func (*StandardProtocols) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{53} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{54} } // These URNs are used to indicate capabilities of runner that an environment @@ -5573,7 +5653,7 @@ type StandardRunnerProtocols struct { func (x *StandardRunnerProtocols) Reset() { *x = StandardRunnerProtocols{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[54] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5586,7 +5666,7 @@ func (x *StandardRunnerProtocols) String() string { func (*StandardRunnerProtocols) ProtoMessage() {} func (x *StandardRunnerProtocols) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[54] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[55] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5599,7 +5679,7 @@ func (x *StandardRunnerProtocols) ProtoReflect() protoreflect.Message { // Deprecated: Use StandardRunnerProtocols.ProtoReflect.Descriptor instead. func (*StandardRunnerProtocols) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{54} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{55} } // These URNs are used to indicate requirements of a pipeline that cannot @@ -5616,7 +5696,7 @@ type StandardRequirements struct { func (x *StandardRequirements) Reset() { *x = StandardRequirements{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[55] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5629,7 +5709,7 @@ func (x *StandardRequirements) String() string { func (*StandardRequirements) ProtoMessage() {} func (x *StandardRequirements) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[55] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[56] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5642,7 +5722,7 @@ func (x *StandardRequirements) ProtoReflect() protoreflect.Message { // Deprecated: Use StandardRequirements.ProtoReflect.Descriptor instead. func (*StandardRequirements) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{55} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{56} } // A URN along with a parameter object whose schema is determined by the @@ -5690,7 +5770,7 @@ type FunctionSpec struct { func (x *FunctionSpec) Reset() { *x = FunctionSpec{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[56] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5703,7 +5783,7 @@ func (x *FunctionSpec) String() string { func (*FunctionSpec) ProtoMessage() {} func (x *FunctionSpec) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[56] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[57] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5716,7 +5796,7 @@ func (x *FunctionSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use FunctionSpec.ProtoReflect.Descriptor instead. func (*FunctionSpec) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{56} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{57} } func (x *FunctionSpec) GetUrn() string { @@ -5748,7 +5828,7 @@ type StandardDisplayData struct { func (x *StandardDisplayData) Reset() { *x = StandardDisplayData{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[57] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5761,7 +5841,7 @@ func (x *StandardDisplayData) String() string { func (*StandardDisplayData) ProtoMessage() {} func (x *StandardDisplayData) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[57] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[58] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5774,7 +5854,7 @@ func (x *StandardDisplayData) ProtoReflect() protoreflect.Message { // Deprecated: Use StandardDisplayData.ProtoReflect.Descriptor instead. func (*StandardDisplayData) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{57} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{58} } type LabelledPayload struct { @@ -5802,7 +5882,7 @@ type LabelledPayload struct { func (x *LabelledPayload) Reset() { *x = LabelledPayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[58] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5815,7 +5895,7 @@ func (x *LabelledPayload) String() string { func (*LabelledPayload) ProtoMessage() {} func (x *LabelledPayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[58] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[59] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5828,7 +5908,7 @@ func (x *LabelledPayload) ProtoReflect() protoreflect.Message { // Deprecated: Use LabelledPayload.ProtoReflect.Descriptor instead. func (*LabelledPayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{58} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{59} } func (x *LabelledPayload) GetLabel() string { @@ -5935,7 +6015,7 @@ type DisplayData struct { func (x *DisplayData) Reset() { *x = DisplayData{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[59] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5948,7 +6028,7 @@ func (x *DisplayData) String() string { func (*DisplayData) ProtoMessage() {} func (x *DisplayData) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[59] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[60] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5961,7 +6041,7 @@ func (x *DisplayData) ProtoReflect() protoreflect.Message { // Deprecated: Use DisplayData.ProtoReflect.Descriptor instead. func (*DisplayData) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{59} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{60} } func (x *DisplayData) GetUrn() string { @@ -6012,7 +6092,7 @@ type MessageWithComponents struct { func (x *MessageWithComponents) Reset() { *x = MessageWithComponents{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[60] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6025,7 +6105,7 @@ func (x *MessageWithComponents) String() string { func (*MessageWithComponents) ProtoMessage() {} func (x *MessageWithComponents) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[60] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[61] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6038,7 +6118,7 @@ func (x *MessageWithComponents) ProtoReflect() protoreflect.Message { // Deprecated: Use MessageWithComponents.ProtoReflect.Descriptor instead. func (*MessageWithComponents) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{60} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{61} } func (x *MessageWithComponents) GetComponents() *Components { @@ -6232,7 +6312,7 @@ type ExecutableStagePayload struct { func (x *ExecutableStagePayload) Reset() { *x = ExecutableStagePayload{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[61] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6245,7 +6325,7 @@ func (x *ExecutableStagePayload) String() string { func (*ExecutableStagePayload) ProtoMessage() {} func (x *ExecutableStagePayload) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[61] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[62] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6258,7 +6338,7 @@ func (x *ExecutableStagePayload) ProtoReflect() protoreflect.Message { // Deprecated: Use ExecutableStagePayload.ProtoReflect.Descriptor instead. func (*ExecutableStagePayload) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{61} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{62} } func (x *ExecutableStagePayload) GetEnvironment() *Environment { @@ -6342,7 +6422,7 @@ type StandardResourceHints struct { func (x *StandardResourceHints) Reset() { *x = StandardResourceHints{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[62] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[63] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6355,7 +6435,7 @@ func (x *StandardResourceHints) String() string { func (*StandardResourceHints) ProtoMessage() {} func (x *StandardResourceHints) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[62] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[63] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6368,7 +6448,7 @@ func (x *StandardResourceHints) ProtoReflect() protoreflect.Message { // Deprecated: Use StandardResourceHints.ProtoReflect.Descriptor instead. func (*StandardResourceHints) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{62} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{63} } type TestStreamPayload_Event struct { @@ -6387,7 +6467,7 @@ type TestStreamPayload_Event struct { func (x *TestStreamPayload_Event) Reset() { *x = TestStreamPayload_Event{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[74] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[75] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6400,7 +6480,7 @@ func (x *TestStreamPayload_Event) String() string { func (*TestStreamPayload_Event) ProtoMessage() {} func (x *TestStreamPayload_Event) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[74] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[75] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6413,7 +6493,7 @@ func (x *TestStreamPayload_Event) ProtoReflect() protoreflect.Message { // Deprecated: Use TestStreamPayload_Event.ProtoReflect.Descriptor instead. func (*TestStreamPayload_Event) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{21, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{22, 0} } func (m *TestStreamPayload_Event) GetEvent() isTestStreamPayload_Event_Event { @@ -6482,7 +6562,7 @@ type TestStreamPayload_TimestampedElement struct { func (x *TestStreamPayload_TimestampedElement) Reset() { *x = TestStreamPayload_TimestampedElement{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[75] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[76] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6495,7 +6575,7 @@ func (x *TestStreamPayload_TimestampedElement) String() string { func (*TestStreamPayload_TimestampedElement) ProtoMessage() {} func (x *TestStreamPayload_TimestampedElement) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[75] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[76] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6508,7 +6588,7 @@ func (x *TestStreamPayload_TimestampedElement) ProtoReflect() protoreflect.Messa // Deprecated: Use TestStreamPayload_TimestampedElement.ProtoReflect.Descriptor instead. func (*TestStreamPayload_TimestampedElement) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{21, 1} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{22, 1} } func (x *TestStreamPayload_TimestampedElement) GetEncodedElement() []byte { @@ -6542,7 +6622,7 @@ type TestStreamPayload_Event_AdvanceWatermark struct { func (x *TestStreamPayload_Event_AdvanceWatermark) Reset() { *x = TestStreamPayload_Event_AdvanceWatermark{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[76] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[77] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6555,7 +6635,7 @@ func (x *TestStreamPayload_Event_AdvanceWatermark) String() string { func (*TestStreamPayload_Event_AdvanceWatermark) ProtoMessage() {} func (x *TestStreamPayload_Event_AdvanceWatermark) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[76] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[77] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6568,7 +6648,7 @@ func (x *TestStreamPayload_Event_AdvanceWatermark) ProtoReflect() protoreflect.M // Deprecated: Use TestStreamPayload_Event_AdvanceWatermark.ProtoReflect.Descriptor instead. func (*TestStreamPayload_Event_AdvanceWatermark) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{21, 0, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{22, 0, 0} } func (x *TestStreamPayload_Event_AdvanceWatermark) GetNewWatermark() int64 { @@ -6598,7 +6678,7 @@ type TestStreamPayload_Event_AdvanceProcessingTime struct { func (x *TestStreamPayload_Event_AdvanceProcessingTime) Reset() { *x = TestStreamPayload_Event_AdvanceProcessingTime{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[77] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[78] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6611,7 +6691,7 @@ func (x *TestStreamPayload_Event_AdvanceProcessingTime) String() string { func (*TestStreamPayload_Event_AdvanceProcessingTime) ProtoMessage() {} func (x *TestStreamPayload_Event_AdvanceProcessingTime) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[77] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[78] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6624,7 +6704,7 @@ func (x *TestStreamPayload_Event_AdvanceProcessingTime) ProtoReflect() protorefl // Deprecated: Use TestStreamPayload_Event_AdvanceProcessingTime.ProtoReflect.Descriptor instead. func (*TestStreamPayload_Event_AdvanceProcessingTime) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{21, 0, 1} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{22, 0, 1} } func (x *TestStreamPayload_Event_AdvanceProcessingTime) GetAdvanceDuration() int64 { @@ -6651,7 +6731,7 @@ type TestStreamPayload_Event_AddElements struct { func (x *TestStreamPayload_Event_AddElements) Reset() { *x = TestStreamPayload_Event_AddElements{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[78] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[79] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6664,7 +6744,7 @@ func (x *TestStreamPayload_Event_AddElements) String() string { func (*TestStreamPayload_Event_AddElements) ProtoMessage() {} func (x *TestStreamPayload_Event_AddElements) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[78] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[79] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6677,7 +6757,7 @@ func (x *TestStreamPayload_Event_AddElements) ProtoReflect() protoreflect.Messag // Deprecated: Use TestStreamPayload_Event_AddElements.ProtoReflect.Descriptor instead. func (*TestStreamPayload_Event_AddElements) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{21, 0, 2} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{22, 0, 2} } func (x *TestStreamPayload_Event_AddElements) GetElements() []*TestStreamPayload_TimestampedElement { @@ -6706,7 +6786,7 @@ type Trigger_AfterAll struct { func (x *Trigger_AfterAll) Reset() { *x = Trigger_AfterAll{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[80] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[81] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6719,7 +6799,7 @@ func (x *Trigger_AfterAll) String() string { func (*Trigger_AfterAll) ProtoMessage() {} func (x *Trigger_AfterAll) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[80] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[81] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6732,7 +6812,7 @@ func (x *Trigger_AfterAll) ProtoReflect() protoreflect.Message { // Deprecated: Use Trigger_AfterAll.ProtoReflect.Descriptor instead. func (*Trigger_AfterAll) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{36, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{37, 0} } func (x *Trigger_AfterAll) GetSubtriggers() []*Trigger { @@ -6754,7 +6834,7 @@ type Trigger_AfterAny struct { func (x *Trigger_AfterAny) Reset() { *x = Trigger_AfterAny{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[81] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[82] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6767,7 +6847,7 @@ func (x *Trigger_AfterAny) String() string { func (*Trigger_AfterAny) ProtoMessage() {} func (x *Trigger_AfterAny) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[81] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[82] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6780,7 +6860,7 @@ func (x *Trigger_AfterAny) ProtoReflect() protoreflect.Message { // Deprecated: Use Trigger_AfterAny.ProtoReflect.Descriptor instead. func (*Trigger_AfterAny) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{36, 1} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{37, 1} } func (x *Trigger_AfterAny) GetSubtriggers() []*Trigger { @@ -6803,7 +6883,7 @@ type Trigger_AfterEach struct { func (x *Trigger_AfterEach) Reset() { *x = Trigger_AfterEach{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[82] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[83] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6816,7 +6896,7 @@ func (x *Trigger_AfterEach) String() string { func (*Trigger_AfterEach) ProtoMessage() {} func (x *Trigger_AfterEach) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[82] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[83] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6829,7 +6909,7 @@ func (x *Trigger_AfterEach) ProtoReflect() protoreflect.Message { // Deprecated: Use Trigger_AfterEach.ProtoReflect.Descriptor instead. func (*Trigger_AfterEach) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{36, 2} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{37, 2} } func (x *Trigger_AfterEach) GetSubtriggers() []*Trigger { @@ -6858,7 +6938,7 @@ type Trigger_AfterEndOfWindow struct { func (x *Trigger_AfterEndOfWindow) Reset() { *x = Trigger_AfterEndOfWindow{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[83] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[84] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6871,7 +6951,7 @@ func (x *Trigger_AfterEndOfWindow) String() string { func (*Trigger_AfterEndOfWindow) ProtoMessage() {} func (x *Trigger_AfterEndOfWindow) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[83] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[84] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6884,7 +6964,7 @@ func (x *Trigger_AfterEndOfWindow) ProtoReflect() protoreflect.Message { // Deprecated: Use Trigger_AfterEndOfWindow.ProtoReflect.Descriptor instead. func (*Trigger_AfterEndOfWindow) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{36, 3} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{37, 3} } func (x *Trigger_AfterEndOfWindow) GetEarlyFirings() *Trigger { @@ -6915,7 +6995,7 @@ type Trigger_AfterProcessingTime struct { func (x *Trigger_AfterProcessingTime) Reset() { *x = Trigger_AfterProcessingTime{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[84] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[85] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6928,7 +7008,7 @@ func (x *Trigger_AfterProcessingTime) String() string { func (*Trigger_AfterProcessingTime) ProtoMessage() {} func (x *Trigger_AfterProcessingTime) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[84] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[85] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6941,7 +7021,7 @@ func (x *Trigger_AfterProcessingTime) ProtoReflect() protoreflect.Message { // Deprecated: Use Trigger_AfterProcessingTime.ProtoReflect.Descriptor instead. func (*Trigger_AfterProcessingTime) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{36, 4} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{37, 4} } func (x *Trigger_AfterProcessingTime) GetTimestampTransforms() []*TimestampTransform { @@ -6962,7 +7042,7 @@ type Trigger_AfterSynchronizedProcessingTime struct { func (x *Trigger_AfterSynchronizedProcessingTime) Reset() { *x = Trigger_AfterSynchronizedProcessingTime{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[85] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[86] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6975,7 +7055,7 @@ func (x *Trigger_AfterSynchronizedProcessingTime) String() string { func (*Trigger_AfterSynchronizedProcessingTime) ProtoMessage() {} func (x *Trigger_AfterSynchronizedProcessingTime) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[85] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[86] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6988,7 +7068,7 @@ func (x *Trigger_AfterSynchronizedProcessingTime) ProtoReflect() protoreflect.Me // Deprecated: Use Trigger_AfterSynchronizedProcessingTime.ProtoReflect.Descriptor instead. func (*Trigger_AfterSynchronizedProcessingTime) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{36, 5} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{37, 5} } // The default trigger. Equivalent to Repeat { AfterEndOfWindow } but @@ -7002,7 +7082,7 @@ type Trigger_Default struct { func (x *Trigger_Default) Reset() { *x = Trigger_Default{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[86] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[87] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7015,7 +7095,7 @@ func (x *Trigger_Default) String() string { func (*Trigger_Default) ProtoMessage() {} func (x *Trigger_Default) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[86] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[87] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7028,7 +7108,7 @@ func (x *Trigger_Default) ProtoReflect() protoreflect.Message { // Deprecated: Use Trigger_Default.ProtoReflect.Descriptor instead. func (*Trigger_Default) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{36, 6} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{37, 6} } // Ready whenever the requisite number of input elements have arrived @@ -7043,7 +7123,7 @@ type Trigger_ElementCount struct { func (x *Trigger_ElementCount) Reset() { *x = Trigger_ElementCount{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[87] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[88] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7056,7 +7136,7 @@ func (x *Trigger_ElementCount) String() string { func (*Trigger_ElementCount) ProtoMessage() {} func (x *Trigger_ElementCount) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[87] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[88] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7069,7 +7149,7 @@ func (x *Trigger_ElementCount) ProtoReflect() protoreflect.Message { // Deprecated: Use Trigger_ElementCount.ProtoReflect.Descriptor instead. func (*Trigger_ElementCount) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{36, 7} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{37, 7} } func (x *Trigger_ElementCount) GetElementCount() int32 { @@ -7090,7 +7170,7 @@ type Trigger_Never struct { func (x *Trigger_Never) Reset() { *x = Trigger_Never{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[88] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[89] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7103,7 +7183,7 @@ func (x *Trigger_Never) String() string { func (*Trigger_Never) ProtoMessage() {} func (x *Trigger_Never) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[88] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[89] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7116,7 +7196,7 @@ func (x *Trigger_Never) ProtoReflect() protoreflect.Message { // Deprecated: Use Trigger_Never.ProtoReflect.Descriptor instead. func (*Trigger_Never) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{36, 8} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{37, 8} } // Always ready. This can also be expressed as ElementCount(1) but @@ -7130,7 +7210,7 @@ type Trigger_Always struct { func (x *Trigger_Always) Reset() { *x = Trigger_Always{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[89] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[90] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7143,7 +7223,7 @@ func (x *Trigger_Always) String() string { func (*Trigger_Always) ProtoMessage() {} func (x *Trigger_Always) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[89] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[90] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7156,7 +7236,7 @@ func (x *Trigger_Always) ProtoReflect() protoreflect.Message { // Deprecated: Use Trigger_Always.ProtoReflect.Descriptor instead. func (*Trigger_Always) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{36, 9} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{37, 9} } // Ready whenever either of its subtriggers are ready, but finishes output @@ -7175,7 +7255,7 @@ type Trigger_OrFinally struct { func (x *Trigger_OrFinally) Reset() { *x = Trigger_OrFinally{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[90] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[91] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7188,7 +7268,7 @@ func (x *Trigger_OrFinally) String() string { func (*Trigger_OrFinally) ProtoMessage() {} func (x *Trigger_OrFinally) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[90] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[91] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7201,7 +7281,7 @@ func (x *Trigger_OrFinally) ProtoReflect() protoreflect.Message { // Deprecated: Use Trigger_OrFinally.ProtoReflect.Descriptor instead. func (*Trigger_OrFinally) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{36, 10} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{37, 10} } func (x *Trigger_OrFinally) GetMain() *Trigger { @@ -7232,7 +7312,7 @@ type Trigger_Repeat struct { func (x *Trigger_Repeat) Reset() { *x = Trigger_Repeat{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[91] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[92] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7245,7 +7325,7 @@ func (x *Trigger_Repeat) String() string { func (*Trigger_Repeat) ProtoMessage() {} func (x *Trigger_Repeat) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[91] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[92] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7258,7 +7338,7 @@ func (x *Trigger_Repeat) ProtoReflect() protoreflect.Message { // Deprecated: Use Trigger_Repeat.ProtoReflect.Descriptor instead. func (*Trigger_Repeat) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{36, 11} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{37, 11} } func (x *Trigger_Repeat) GetSubtrigger() *Trigger { @@ -7280,7 +7360,7 @@ type TimestampTransform_Delay struct { func (x *TimestampTransform_Delay) Reset() { *x = TimestampTransform_Delay{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[92] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[93] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7293,7 +7373,7 @@ func (x *TimestampTransform_Delay) String() string { func (*TimestampTransform_Delay) ProtoMessage() {} func (x *TimestampTransform_Delay) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[92] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[93] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7306,7 +7386,7 @@ func (x *TimestampTransform_Delay) ProtoReflect() protoreflect.Message { // Deprecated: Use TimestampTransform_Delay.ProtoReflect.Descriptor instead. func (*TimestampTransform_Delay) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{37, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{38, 0} } func (x *TimestampTransform_Delay) GetDelayMillis() int64 { @@ -7332,7 +7412,7 @@ type TimestampTransform_AlignTo struct { func (x *TimestampTransform_AlignTo) Reset() { *x = TimestampTransform_AlignTo{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[93] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[94] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7345,7 +7425,7 @@ func (x *TimestampTransform_AlignTo) String() string { func (*TimestampTransform_AlignTo) ProtoMessage() {} func (x *TimestampTransform_AlignTo) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[93] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[94] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7358,7 +7438,7 @@ func (x *TimestampTransform_AlignTo) ProtoReflect() protoreflect.Message { // Deprecated: Use TimestampTransform_AlignTo.ProtoReflect.Descriptor instead. func (*TimestampTransform_AlignTo) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{37, 1} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{38, 1} } func (x *TimestampTransform_AlignTo) GetPeriod() int64 { @@ -7391,7 +7471,7 @@ type ExecutableStagePayload_SideInputId struct { func (x *ExecutableStagePayload_SideInputId) Reset() { *x = ExecutableStagePayload_SideInputId{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[97] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[98] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7404,7 +7484,7 @@ func (x *ExecutableStagePayload_SideInputId) String() string { func (*ExecutableStagePayload_SideInputId) ProtoMessage() {} func (x *ExecutableStagePayload_SideInputId) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[97] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[98] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7417,7 +7497,7 @@ func (x *ExecutableStagePayload_SideInputId) ProtoReflect() protoreflect.Message // Deprecated: Use ExecutableStagePayload_SideInputId.ProtoReflect.Descriptor instead. func (*ExecutableStagePayload_SideInputId) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{61, 0} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{62, 0} } func (x *ExecutableStagePayload_SideInputId) GetTransformId() string { @@ -7450,7 +7530,7 @@ type ExecutableStagePayload_UserStateId struct { func (x *ExecutableStagePayload_UserStateId) Reset() { *x = ExecutableStagePayload_UserStateId{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[98] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[99] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7463,7 +7543,7 @@ func (x *ExecutableStagePayload_UserStateId) String() string { func (*ExecutableStagePayload_UserStateId) ProtoMessage() {} func (x *ExecutableStagePayload_UserStateId) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[98] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[99] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7476,7 +7556,7 @@ func (x *ExecutableStagePayload_UserStateId) ProtoReflect() protoreflect.Message // Deprecated: Use ExecutableStagePayload_UserStateId.ProtoReflect.Descriptor instead. func (*ExecutableStagePayload_UserStateId) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{61, 1} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{62, 1} } func (x *ExecutableStagePayload_UserStateId) GetTransformId() string { @@ -7509,7 +7589,7 @@ type ExecutableStagePayload_TimerId struct { func (x *ExecutableStagePayload_TimerId) Reset() { *x = ExecutableStagePayload_TimerId{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[99] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[100] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7522,7 +7602,7 @@ func (x *ExecutableStagePayload_TimerId) String() string { func (*ExecutableStagePayload_TimerId) ProtoMessage() {} func (x *ExecutableStagePayload_TimerId) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[99] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[100] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7535,7 +7615,7 @@ func (x *ExecutableStagePayload_TimerId) ProtoReflect() protoreflect.Message { // Deprecated: Use ExecutableStagePayload_TimerId.ProtoReflect.Descriptor instead. func (*ExecutableStagePayload_TimerId) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{61, 2} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{62, 2} } func (x *ExecutableStagePayload_TimerId) GetTransformId() string { @@ -7568,7 +7648,7 @@ type ExecutableStagePayload_TimerFamilyId struct { func (x *ExecutableStagePayload_TimerFamilyId) Reset() { *x = ExecutableStagePayload_TimerFamilyId{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[100] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[101] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7581,7 +7661,7 @@ func (x *ExecutableStagePayload_TimerFamilyId) String() string { func (*ExecutableStagePayload_TimerFamilyId) ProtoMessage() {} func (x *ExecutableStagePayload_TimerFamilyId) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[100] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[101] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7594,7 +7674,7 @@ func (x *ExecutableStagePayload_TimerFamilyId) ProtoReflect() protoreflect.Messa // Deprecated: Use ExecutableStagePayload_TimerFamilyId.ProtoReflect.Descriptor instead. func (*ExecutableStagePayload_TimerFamilyId) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{61, 3} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{62, 3} } func (x *ExecutableStagePayload_TimerFamilyId) GetTransformId() string { @@ -7639,7 +7719,7 @@ type ExecutableStagePayload_WireCoderSetting struct { func (x *ExecutableStagePayload_WireCoderSetting) Reset() { *x = ExecutableStagePayload_WireCoderSetting{} if protoimpl.UnsafeEnabled { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[101] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[102] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7652,7 +7732,7 @@ func (x *ExecutableStagePayload_WireCoderSetting) String() string { func (*ExecutableStagePayload_WireCoderSetting) ProtoMessage() {} func (x *ExecutableStagePayload_WireCoderSetting) ProtoReflect() protoreflect.Message { - mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[101] + mi := &file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[102] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7665,7 +7745,7 @@ func (x *ExecutableStagePayload_WireCoderSetting) ProtoReflect() protoreflect.Me // Deprecated: Use ExecutableStagePayload_WireCoderSetting.ProtoReflect.Descriptor instead. func (*ExecutableStagePayload_WireCoderSetting) Descriptor() ([]byte, []int) { - return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{61, 4} + return file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP(), []int{62, 4} } func (x *ExecutableStagePayload_WireCoderSetting) GetUrn() string { @@ -8166,7 +8246,7 @@ var file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDesc = []byt 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x53, 0x70, 0x65, 0x63, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x4a, 0x04, 0x08, 0x06, 0x10, 0x07, 0x22, 0x85, 0x05, 0x0a, 0x09, 0x53, 0x74, 0x61, 0x74, + 0x01, 0x4a, 0x04, 0x08, 0x06, 0x10, 0x07, 0x22, 0xe2, 0x05, 0x0a, 0x09, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x72, 0x0a, 0x16, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x79, 0x5f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, @@ -8201,974 +8281,993 @@ var file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDesc = []byt 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x65, 0x64, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x70, 0x65, 0x63, 0x48, 0x00, 0x52, 0x0f, 0x6f, 0x72, 0x64, 0x65, 0x72, - 0x65, 0x64, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x4b, 0x0a, 0x08, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, - 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, - 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x08, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x42, 0x06, 0x0a, 0x04, 0x73, 0x70, 0x65, 0x63, 0x22, - 0x35, 0x0a, 0x18, 0x52, 0x65, 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x79, 0x57, 0x72, 0x69, - 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x19, 0x0a, 0x08, 0x63, - 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, - 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x22, 0x38, 0x0a, 0x0c, 0x42, 0x61, 0x67, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0e, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, - 0x22, 0x40, 0x0a, 0x14, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x65, 0x64, 0x4c, 0x69, 0x73, 0x74, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x6c, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0e, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x72, - 0x49, 0x64, 0x22, 0x96, 0x01, 0x0a, 0x12, 0x43, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x69, 0x6e, 0x67, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x30, 0x0a, 0x14, 0x61, 0x63, 0x63, - 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x61, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, - 0x61, 0x74, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x12, 0x4e, 0x0a, 0x0a, 0x63, - 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x5f, 0x66, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, - 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, - 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x46, 0x6e, 0x22, 0x56, 0x0a, 0x0c, 0x4d, - 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x20, 0x0a, 0x0c, 0x6b, - 0x65, 0x79, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0a, 0x6b, 0x65, 0x79, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x12, 0x24, 0x0a, - 0x0e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x43, 0x6f, 0x64, 0x65, - 0x72, 0x49, 0x64, 0x22, 0x38, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, + 0x65, 0x64, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x5b, 0x0a, 0x0d, 0x6d, 0x75, + 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x08, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x34, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, + 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, + 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x53, 0x70, 0x65, 0x63, 0x48, 0x00, 0x52, 0x0c, 0x6d, 0x75, 0x6c, 0x74, 0x69, + 0x6d, 0x61, 0x70, 0x53, 0x70, 0x65, 0x63, 0x12, 0x4b, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, + 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x42, 0x06, 0x0a, 0x04, 0x73, 0x70, 0x65, 0x63, 0x22, 0x35, 0x0a, 0x18, + 0x52, 0x65, 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x79, 0x57, 0x72, 0x69, 0x74, 0x65, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x6f, 0x64, 0x65, + 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x64, 0x65, + 0x72, 0x49, 0x64, 0x22, 0x38, 0x0a, 0x0c, 0x42, 0x61, 0x67, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, - 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x22, 0x99, 0x01, - 0x0a, 0x0f, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x53, 0x70, 0x65, - 0x63, 0x12, 0x53, 0x0a, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x32, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, - 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, - 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x44, - 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x0a, 0x74, 0x69, 0x6d, 0x65, - 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x31, 0x0a, 0x15, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x5f, - 0x66, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x46, 0x61, 0x6d, 0x69, - 0x6c, 0x79, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x22, 0x40, 0x0a, 0x09, 0x49, 0x73, 0x42, - 0x6f, 0x75, 0x6e, 0x64, 0x65, 0x64, 0x22, 0x33, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x0f, - 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, - 0x0d, 0x0a, 0x09, 0x55, 0x4e, 0x42, 0x4f, 0x55, 0x4e, 0x44, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0b, - 0x0a, 0x07, 0x42, 0x4f, 0x55, 0x4e, 0x44, 0x45, 0x44, 0x10, 0x02, 0x22, 0xa8, 0x01, 0x0a, 0x0b, - 0x52, 0x65, 0x61, 0x64, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x47, 0x0a, 0x06, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, - 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, - 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x06, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x12, 0x50, 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x62, 0x6f, 0x75, 0x6e, 0x64, - 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x31, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, - 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, - 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x73, 0x42, - 0x6f, 0x75, 0x6e, 0x64, 0x65, 0x64, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x09, 0x69, 0x73, 0x42, - 0x6f, 0x75, 0x6e, 0x64, 0x65, 0x64, 0x22, 0x61, 0x0a, 0x11, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, - 0x49, 0x6e, 0x74, 0x6f, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x4c, 0x0a, 0x09, 0x77, - 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x5f, 0x66, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, - 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, - 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, - 0x08, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x46, 0x6e, 0x22, 0x92, 0x01, 0x0a, 0x0e, 0x43, 0x6f, - 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x4e, 0x0a, 0x0a, - 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x5f, 0x66, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, + 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x22, 0x40, 0x0a, + 0x14, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x65, 0x64, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0e, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x22, + 0x96, 0x01, 0x0a, 0x12, 0x43, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x30, 0x0a, 0x14, 0x61, 0x63, 0x63, 0x75, 0x6d, 0x75, + 0x6c, 0x61, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x61, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x6f, + 0x72, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x12, 0x4e, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x62, + 0x69, 0x6e, 0x65, 0x5f, 0x66, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, + 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, + 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x09, 0x63, + 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x46, 0x6e, 0x22, 0x56, 0x0a, 0x0c, 0x4d, 0x61, 0x70, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x20, 0x0a, 0x0c, 0x6b, 0x65, 0x79, 0x5f, + 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x6b, 0x65, 0x79, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x12, 0x24, 0x0a, 0x0e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0c, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, + 0x22, 0x5b, 0x0a, 0x11, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x20, 0x0a, 0x0c, 0x6b, 0x65, 0x79, 0x5f, 0x63, 0x6f, 0x64, + 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6b, 0x65, 0x79, + 0x43, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x12, 0x24, 0x0a, 0x0e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0c, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x22, 0x38, 0x0a, + 0x0c, 0x53, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x28, 0x0a, + 0x10, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x43, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x22, 0x99, 0x01, 0x0a, 0x0f, 0x54, 0x69, 0x6d, 0x65, + 0x72, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x53, 0x70, 0x65, 0x63, 0x12, 0x53, 0x0a, 0x0b, 0x74, + 0x69, 0x6d, 0x65, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x32, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, - 0x63, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x46, 0x6e, 0x12, 0x30, 0x0a, 0x14, - 0x61, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x64, 0x65, - 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x61, 0x63, 0x63, 0x75, - 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x22, 0xcd, - 0x07, 0x0a, 0x11, 0x54, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x50, 0x61, 0x79, - 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x12, - 0x52, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x3a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, - 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x50, 0x61, - 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, - 0x6e, 0x74, 0x73, 0x12, 0x53, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, + 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, + 0x12, 0x31, 0x0a, 0x15, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x5f, 0x66, 0x61, 0x6d, 0x69, 0x6c, 0x79, + 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x12, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x43, 0x6f, 0x64, 0x65, + 0x72, 0x49, 0x64, 0x22, 0x40, 0x0a, 0x09, 0x49, 0x73, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x65, 0x64, + 0x22, 0x33, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, + 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x4e, 0x42, + 0x4f, 0x55, 0x4e, 0x44, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x42, 0x4f, 0x55, 0x4e, + 0x44, 0x45, 0x44, 0x10, 0x02, 0x22, 0xa8, 0x01, 0x0a, 0x0b, 0x52, 0x65, 0x61, 0x64, 0x50, 0x61, + 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x47, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, - 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x70, 0x69, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x08, - 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x1a, 0x96, 0x05, 0x0a, 0x05, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x12, 0x76, 0x0a, 0x0f, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x5f, - 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4b, 0x2e, 0x6f, 0x72, - 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, - 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x54, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x64, 0x76, 0x61, 0x6e, 0x63, 0x65, 0x57, - 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x48, 0x00, 0x52, 0x0e, 0x77, 0x61, 0x74, 0x65, - 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x86, 0x01, 0x0a, 0x15, 0x70, - 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x65, - 0x76, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x50, 0x2e, 0x6f, 0x72, 0x67, - 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, - 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, - 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, - 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x64, 0x76, 0x61, 0x6e, 0x63, 0x65, 0x50, 0x72, - 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x48, 0x00, 0x52, 0x13, - 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x45, 0x76, - 0x65, 0x6e, 0x74, 0x12, 0x6d, 0x0a, 0x0d, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x65, - 0x76, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x46, 0x2e, 0x6f, 0x72, 0x67, - 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, - 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, - 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, - 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x64, 0x64, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x73, 0x48, 0x00, 0x52, 0x0c, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x1a, 0x49, 0x0a, 0x10, 0x41, 0x64, 0x76, 0x61, 0x6e, 0x63, 0x65, 0x57, 0x61, 0x74, - 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x12, 0x23, 0x0a, 0x0d, 0x6e, 0x65, 0x77, 0x5f, 0x77, 0x61, - 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x6e, - 0x65, 0x77, 0x57, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x12, 0x10, 0x0a, 0x03, 0x74, - 0x61, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x61, 0x67, 0x1a, 0x42, 0x0a, - 0x15, 0x41, 0x64, 0x76, 0x61, 0x6e, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, - 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x61, 0x64, 0x76, 0x61, 0x6e, 0x63, - 0x65, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x0f, 0x61, 0x64, 0x76, 0x61, 0x6e, 0x63, 0x65, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x1a, 0x84, 0x01, 0x0a, 0x0b, 0x41, 0x64, 0x64, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x73, 0x12, 0x63, 0x0a, 0x08, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x47, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, - 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, - 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x65, 0x64, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x08, 0x65, 0x6c, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x61, 0x67, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x61, 0x67, 0x42, 0x07, 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, - 0x74, 0x1a, 0x5b, 0x0a, 0x12, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x65, 0x64, - 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x6e, 0x63, 0x6f, 0x64, - 0x65, 0x64, 0x5f, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x0e, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x2e, - 0x0a, 0x0d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x1d, 0x0a, 0x0a, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x49, 0x64, 0x73, 0x22, 0xed, - 0x03, 0x0a, 0x11, 0x57, 0x72, 0x69, 0x74, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x50, 0x61, 0x79, - 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x43, 0x0a, 0x04, 0x73, 0x69, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, - 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, - 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x70, 0x65, 0x63, 0x52, 0x04, 0x73, 0x69, 0x6e, 0x6b, 0x12, 0x58, 0x0a, 0x0f, 0x66, 0x6f, 0x72, - 0x6d, 0x61, 0x74, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, + 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x50, + 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x31, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, - 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x70, 0x65, 0x63, 0x52, 0x0e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x46, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x27, 0x0a, 0x0f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x65, 0x64, 0x5f, - 0x77, 0x72, 0x69, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x77, 0x69, - 0x6e, 0x64, 0x6f, 0x77, 0x65, 0x64, 0x57, 0x72, 0x69, 0x74, 0x65, 0x73, 0x12, 0x3c, 0x0a, 0x1a, - 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x65, - 0x64, 0x5f, 0x73, 0x68, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x18, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x44, 0x65, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, - 0x65, 0x64, 0x53, 0x68, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x65, 0x0a, 0x0b, 0x73, 0x69, - 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x44, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, + 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x73, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x65, 0x64, + 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x09, 0x69, 0x73, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x65, 0x64, + 0x22, 0x61, 0x0a, 0x11, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x49, 0x6e, 0x74, 0x6f, 0x50, 0x61, + 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x4c, 0x0a, 0x09, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x5f, + 0x66, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, + 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, + 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x08, 0x77, 0x69, 0x6e, 0x64, 0x6f, + 0x77, 0x46, 0x6e, 0x22, 0x92, 0x01, 0x0a, 0x0e, 0x43, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x50, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x4e, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, + 0x65, 0x5f, 0x66, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, + 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x09, 0x63, 0x6f, 0x6d, + 0x62, 0x69, 0x6e, 0x65, 0x46, 0x6e, 0x12, 0x30, 0x0a, 0x14, 0x61, 0x63, 0x63, 0x75, 0x6d, 0x75, + 0x6c, 0x61, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x61, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x6f, + 0x72, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x22, 0xcd, 0x07, 0x0a, 0x11, 0x54, 0x65, 0x73, + 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x19, + 0x0a, 0x08, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x12, 0x52, 0x0a, 0x06, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, + 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, + 0x73, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x53, 0x0a, + 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x50, 0x61, - 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x73, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, - 0x73, 0x1a, 0x6b, 0x0a, 0x0f, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x42, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, + 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, + 0x6e, 0x74, 0x1a, 0x96, 0x05, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x76, 0x0a, 0x0f, + 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4b, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, - 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, - 0x70, 0x75, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xcc, - 0x02, 0x0a, 0x11, 0x50, 0x75, 0x62, 0x53, 0x75, 0x62, 0x52, 0x65, 0x61, 0x64, 0x50, 0x61, 0x79, - 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x22, 0x0a, 0x0c, 0x73, 0x75, - 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0c, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2f, - 0x0a, 0x13, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x61, 0x74, 0x74, 0x72, - 0x69, 0x62, 0x75, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x74, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x12, - 0x21, 0x0a, 0x0c, 0x69, 0x64, 0x5f, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x64, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, - 0x74, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x77, 0x69, 0x74, 0x68, 0x5f, 0x61, 0x74, 0x74, 0x72, 0x69, - 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x77, 0x69, 0x74, - 0x68, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x18, 0x74, - 0x6f, 0x70, 0x69, 0x63, 0x5f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x64, 0x65, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x74, - 0x6f, 0x70, 0x69, 0x63, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x64, 0x65, 0x6e, 0x12, 0x46, 0x0a, 0x1f, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x64, 0x65, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1d, - 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6e, 0x74, - 0x69, 0x6d, 0x65, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x64, 0x65, 0x6e, 0x22, 0xb8, 0x01, - 0x0a, 0x12, 0x50, 0x75, 0x62, 0x53, 0x75, 0x62, 0x57, 0x72, 0x69, 0x74, 0x65, 0x50, 0x61, 0x79, - 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x2f, 0x0a, 0x13, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x69, - 0x64, 0x5f, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0b, 0x69, 0x64, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x12, 0x38, - 0x0a, 0x18, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x5f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5f, - 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x64, 0x65, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x16, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x64, 0x65, 0x6e, 0x22, 0xa5, 0x01, 0x0a, 0x17, 0x47, 0x72, 0x6f, - 0x75, 0x70, 0x49, 0x6e, 0x74, 0x6f, 0x42, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x50, 0x61, 0x79, - 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x73, 0x69, - 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x62, 0x61, 0x74, 0x63, 0x68, 0x53, - 0x69, 0x7a, 0x65, 0x12, 0x28, 0x0a, 0x10, 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x73, 0x69, 0x7a, - 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x62, - 0x61, 0x74, 0x63, 0x68, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x41, 0x0a, - 0x1d, 0x6d, 0x61, 0x78, 0x5f, 0x62, 0x75, 0x66, 0x66, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x64, - 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x1a, 0x6d, 0x61, 0x78, 0x42, 0x75, 0x66, 0x66, 0x65, 0x72, 0x69, - 0x6e, 0x67, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, - 0x22, 0x7c, 0x0a, 0x05, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x12, 0x43, 0x0a, 0x04, 0x73, 0x70, 0x65, - 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, - 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, - 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, 0x12, 0x2e, - 0x0a, 0x13, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x6f, 0x64, 0x65, - 0x72, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x63, 0x6f, 0x6d, - 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x73, 0x22, 0xf5, - 0x06, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x43, 0x6f, 0x64, 0x65, 0x72, - 0x73, 0x22, 0xe2, 0x06, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x24, 0x0a, 0x05, 0x42, 0x59, - 0x54, 0x45, 0x53, 0x10, 0x00, 0x1a, 0x19, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x13, 0x62, 0x65, 0x61, - 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x62, 0x79, 0x74, 0x65, 0x73, 0x3a, 0x76, 0x31, - 0x12, 0x30, 0x0a, 0x0b, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x5f, 0x55, 0x54, 0x46, 0x38, 0x10, - 0x0a, 0x1a, 0x1f, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x19, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, - 0x64, 0x65, 0x72, 0x3a, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x75, 0x74, 0x66, 0x38, 0x3a, - 0x76, 0x31, 0x12, 0x1e, 0x0a, 0x02, 0x4b, 0x56, 0x10, 0x01, 0x1a, 0x16, 0xa2, 0xb4, 0xfa, 0xc2, - 0x05, 0x10, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x6b, 0x76, 0x3a, - 0x76, 0x31, 0x12, 0x22, 0x0a, 0x04, 0x42, 0x4f, 0x4f, 0x4c, 0x10, 0x0c, 0x1a, 0x18, 0xa2, 0xb4, - 0xfa, 0xc2, 0x05, 0x12, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x62, - 0x6f, 0x6f, 0x6c, 0x3a, 0x76, 0x31, 0x12, 0x26, 0x0a, 0x06, 0x56, 0x41, 0x52, 0x49, 0x4e, 0x54, - 0x10, 0x02, 0x1a, 0x1a, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x14, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, - 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x76, 0x61, 0x72, 0x69, 0x6e, 0x74, 0x3a, 0x76, 0x31, 0x12, 0x26, - 0x0a, 0x06, 0x44, 0x4f, 0x55, 0x42, 0x4c, 0x45, 0x10, 0x0b, 0x1a, 0x1a, 0xa2, 0xb4, 0xfa, 0xc2, - 0x05, 0x14, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x64, 0x6f, 0x75, - 0x62, 0x6c, 0x65, 0x3a, 0x76, 0x31, 0x12, 0x2a, 0x0a, 0x08, 0x49, 0x54, 0x45, 0x52, 0x41, 0x42, - 0x4c, 0x45, 0x10, 0x03, 0x1a, 0x1c, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x16, 0x62, 0x65, 0x61, 0x6d, - 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x69, 0x74, 0x65, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x3a, - 0x76, 0x31, 0x12, 0x24, 0x0a, 0x05, 0x54, 0x49, 0x4d, 0x45, 0x52, 0x10, 0x04, 0x1a, 0x19, 0xa2, - 0xb4, 0xfa, 0xc2, 0x05, 0x13, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, - 0x74, 0x69, 0x6d, 0x65, 0x72, 0x3a, 0x76, 0x31, 0x12, 0x38, 0x0a, 0x0f, 0x49, 0x4e, 0x54, 0x45, - 0x52, 0x56, 0x41, 0x4c, 0x5f, 0x57, 0x49, 0x4e, 0x44, 0x4f, 0x57, 0x10, 0x05, 0x1a, 0x23, 0xa2, - 0xb4, 0xfa, 0xc2, 0x05, 0x1d, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x3a, - 0x76, 0x31, 0x12, 0x34, 0x0a, 0x0d, 0x4c, 0x45, 0x4e, 0x47, 0x54, 0x48, 0x5f, 0x50, 0x52, 0x45, - 0x46, 0x49, 0x58, 0x10, 0x06, 0x1a, 0x21, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1b, 0x62, 0x65, 0x61, - 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x5f, 0x70, - 0x72, 0x65, 0x66, 0x69, 0x78, 0x3a, 0x76, 0x31, 0x12, 0x34, 0x0a, 0x0d, 0x47, 0x4c, 0x4f, 0x42, - 0x41, 0x4c, 0x5f, 0x57, 0x49, 0x4e, 0x44, 0x4f, 0x57, 0x10, 0x07, 0x1a, 0x21, 0xa2, 0xb4, 0xfa, - 0xc2, 0x05, 0x1b, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x67, 0x6c, - 0x6f, 0x62, 0x61, 0x6c, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x3a, 0x76, 0x31, 0x12, 0x36, - 0x0a, 0x0e, 0x57, 0x49, 0x4e, 0x44, 0x4f, 0x57, 0x45, 0x44, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, - 0x10, 0x08, 0x1a, 0x22, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1c, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, - 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x65, 0x64, 0x5f, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3a, 0x76, 0x31, 0x12, 0x42, 0x0a, 0x14, 0x50, 0x41, 0x52, 0x41, 0x4d, 0x5f, - 0x57, 0x49, 0x4e, 0x44, 0x4f, 0x57, 0x45, 0x44, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x10, 0x0e, - 0x1a, 0x28, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x22, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, - 0x65, 0x72, 0x3a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x65, - 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x76, 0x31, 0x12, 0x44, 0x0a, 0x15, 0x53, 0x54, - 0x41, 0x54, 0x45, 0x5f, 0x42, 0x41, 0x43, 0x4b, 0x45, 0x44, 0x5f, 0x49, 0x54, 0x45, 0x52, 0x41, - 0x42, 0x4c, 0x45, 0x10, 0x09, 0x1a, 0x29, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x23, 0x62, 0x65, 0x61, - 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x62, 0x61, - 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x69, 0x74, 0x65, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x3a, 0x76, 0x31, - 0x12, 0x34, 0x0a, 0x0d, 0x43, 0x55, 0x53, 0x54, 0x4f, 0x4d, 0x5f, 0x57, 0x49, 0x4e, 0x44, 0x4f, - 0x57, 0x10, 0x10, 0x1a, 0x21, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1b, 0x62, 0x65, 0x61, 0x6d, 0x3a, - 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x77, 0x69, 0x6e, - 0x64, 0x6f, 0x77, 0x3a, 0x76, 0x31, 0x12, 0x20, 0x0a, 0x03, 0x52, 0x4f, 0x57, 0x10, 0x0d, 0x1a, - 0x17, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x11, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, - 0x72, 0x3a, 0x72, 0x6f, 0x77, 0x3a, 0x76, 0x31, 0x12, 0x30, 0x0a, 0x0b, 0x53, 0x48, 0x41, 0x52, - 0x44, 0x45, 0x44, 0x5f, 0x4b, 0x45, 0x59, 0x10, 0x0f, 0x1a, 0x1f, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, - 0x19, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x73, 0x68, 0x61, 0x72, - 0x64, 0x65, 0x64, 0x5f, 0x6b, 0x65, 0x79, 0x3a, 0x76, 0x31, 0x12, 0x2a, 0x0a, 0x08, 0x4e, 0x55, - 0x4c, 0x4c, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x11, 0x1a, 0x1c, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x16, - 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x61, - 0x62, 0x6c, 0x65, 0x3a, 0x76, 0x31, 0x22, 0xae, 0x06, 0x0a, 0x11, 0x57, 0x69, 0x6e, 0x64, 0x6f, - 0x77, 0x69, 0x6e, 0x67, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x12, 0x4c, 0x0a, 0x09, - 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x5f, 0x66, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, - 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, - 0x52, 0x08, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x46, 0x6e, 0x12, 0x56, 0x0a, 0x0c, 0x6d, 0x65, - 0x72, 0x67, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x33, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, - 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x72, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x0b, 0x6d, 0x65, 0x72, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x5f, 0x63, 0x6f, 0x64, - 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x77, 0x69, 0x6e, - 0x64, 0x6f, 0x77, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x12, 0x44, 0x0a, 0x07, 0x74, 0x72, - 0x69, 0x67, 0x67, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6f, 0x72, - 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, - 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, 0x07, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, - 0x12, 0x65, 0x0a, 0x11, 0x61, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x38, 0x2e, 0x6f, 0x72, - 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, - 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, - 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x10, 0x61, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x53, 0x0a, 0x0b, 0x6f, 0x75, 0x74, 0x70, 0x75, - 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x32, 0x2e, 0x6f, + 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x53, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x2e, 0x41, 0x64, 0x76, 0x61, 0x6e, 0x63, 0x65, 0x57, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, + 0x72, 0x6b, 0x48, 0x00, 0x52, 0x0e, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x45, + 0x76, 0x65, 0x6e, 0x74, 0x12, 0x86, 0x01, 0x0a, 0x15, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, + 0x69, 0x6e, 0x67, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x50, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, + 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, + 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x2e, 0x41, 0x64, 0x76, 0x61, 0x6e, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, + 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x48, 0x00, 0x52, 0x13, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, + 0x73, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x6d, 0x0a, + 0x0d, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x46, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, + 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, + 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x2e, 0x41, 0x64, 0x64, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x48, 0x00, 0x52, 0x0c, + 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x1a, 0x49, 0x0a, 0x10, + 0x41, 0x64, 0x76, 0x61, 0x6e, 0x63, 0x65, 0x57, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, + 0x12, 0x23, 0x0a, 0x0d, 0x6e, 0x65, 0x77, 0x5f, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, + 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x6e, 0x65, 0x77, 0x57, 0x61, 0x74, 0x65, + 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x61, 0x67, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x74, 0x61, 0x67, 0x1a, 0x42, 0x0a, 0x15, 0x41, 0x64, 0x76, 0x61, 0x6e, + 0x63, 0x65, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, + 0x12, 0x29, 0x0a, 0x10, 0x61, 0x64, 0x76, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x64, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x61, 0x64, 0x76, 0x61, + 0x6e, 0x63, 0x65, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x84, 0x01, 0x0a, 0x0b, + 0x41, 0x64, 0x64, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x63, 0x0a, 0x08, 0x65, + 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x47, 0x2e, + 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, + 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x50, 0x61, 0x79, 0x6c, + 0x6f, 0x61, 0x64, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x65, 0x64, 0x45, + 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x08, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, + 0x12, 0x10, 0x0a, 0x03, 0x74, 0x61, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, + 0x61, 0x67, 0x42, 0x07, 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x1a, 0x5b, 0x0a, 0x12, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x65, 0x64, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x5f, 0x65, 0x6c, 0x65, + 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0e, 0x65, 0x6e, 0x63, 0x6f, + 0x64, 0x65, 0x64, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x2e, 0x0a, 0x0d, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x6f, 0x75, 0x74, + 0x70, 0x75, 0x74, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x6f, + 0x75, 0x74, 0x70, 0x75, 0x74, 0x49, 0x64, 0x73, 0x22, 0xed, 0x03, 0x0a, 0x11, 0x57, 0x72, 0x69, + 0x74, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x43, + 0x0a, 0x04, 0x73, 0x69, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x2e, 0x45, 0x6e, 0x75, 0x6d, - 0x52, 0x0a, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x62, 0x0a, 0x10, - 0x63, 0x6c, 0x6f, 0x73, 0x69, 0x6e, 0x67, 0x5f, 0x62, 0x65, 0x68, 0x61, 0x76, 0x69, 0x6f, 0x72, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, - 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, - 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x69, - 0x6e, 0x67, 0x42, 0x65, 0x68, 0x61, 0x76, 0x69, 0x6f, 0x72, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, - 0x0f, 0x63, 0x6c, 0x6f, 0x73, 0x69, 0x6e, 0x67, 0x42, 0x65, 0x68, 0x61, 0x76, 0x69, 0x6f, 0x72, - 0x12, 0x29, 0x0a, 0x10, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x5f, 0x6c, 0x61, 0x74, 0x65, - 0x6e, 0x65, 0x73, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x61, 0x6c, 0x6c, 0x6f, - 0x77, 0x65, 0x64, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x65, 0x73, 0x73, 0x12, 0x60, 0x0a, 0x10, 0x6f, - 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x62, 0x65, 0x68, 0x61, 0x76, 0x69, 0x6f, 0x72, 0x18, - 0x09, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, - 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, - 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x6e, 0x54, 0x69, 0x6d, 0x65, - 0x42, 0x65, 0x68, 0x61, 0x76, 0x69, 0x6f, 0x72, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x0e, 0x6f, - 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x42, 0x65, 0x68, 0x61, 0x76, 0x69, 0x6f, 0x72, 0x12, 0x31, 0x0a, - 0x15, 0x61, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x73, 0x5f, 0x74, 0x6f, 0x5f, 0x6f, 0x6e, 0x65, 0x5f, - 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x61, 0x73, - 0x73, 0x69, 0x67, 0x6e, 0x73, 0x54, 0x6f, 0x4f, 0x6e, 0x65, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, - 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x5f, - 0x69, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, - 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x22, 0x5c, 0x0a, 0x0b, 0x4d, 0x65, 0x72, 0x67, 0x65, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x4d, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x0f, - 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, - 0x0f, 0x0a, 0x0b, 0x4e, 0x4f, 0x4e, 0x5f, 0x4d, 0x45, 0x52, 0x47, 0x49, 0x4e, 0x47, 0x10, 0x01, - 0x12, 0x0f, 0x0a, 0x0b, 0x4e, 0x45, 0x45, 0x44, 0x53, 0x5f, 0x4d, 0x45, 0x52, 0x47, 0x45, 0x10, - 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x41, 0x4c, 0x52, 0x45, 0x41, 0x44, 0x59, 0x5f, 0x4d, 0x45, 0x52, - 0x47, 0x45, 0x44, 0x10, 0x03, 0x22, 0x5d, 0x0a, 0x10, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x22, 0x49, 0x0a, 0x04, 0x45, 0x6e, 0x75, - 0x6d, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, - 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x44, 0x49, 0x53, 0x43, 0x41, 0x52, 0x44, 0x49, 0x4e, 0x47, - 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x41, 0x43, 0x43, 0x55, 0x4d, 0x55, 0x4c, 0x41, 0x54, 0x49, - 0x4e, 0x47, 0x10, 0x02, 0x12, 0x0e, 0x0a, 0x0a, 0x52, 0x45, 0x54, 0x52, 0x41, 0x43, 0x54, 0x49, - 0x4e, 0x47, 0x10, 0x03, 0x22, 0x51, 0x0a, 0x0f, 0x43, 0x6c, 0x6f, 0x73, 0x69, 0x6e, 0x67, 0x42, - 0x65, 0x68, 0x61, 0x76, 0x69, 0x6f, 0x72, 0x22, 0x3e, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, - 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, - 0x12, 0x0f, 0x0a, 0x0b, 0x45, 0x4d, 0x49, 0x54, 0x5f, 0x41, 0x4c, 0x57, 0x41, 0x59, 0x53, 0x10, - 0x01, 0x12, 0x14, 0x0a, 0x10, 0x45, 0x4d, 0x49, 0x54, 0x5f, 0x49, 0x46, 0x5f, 0x4e, 0x4f, 0x4e, - 0x45, 0x4d, 0x50, 0x54, 0x59, 0x10, 0x02, 0x22, 0x50, 0x0a, 0x0e, 0x4f, 0x6e, 0x54, 0x69, 0x6d, - 0x65, 0x42, 0x65, 0x68, 0x61, 0x76, 0x69, 0x6f, 0x72, 0x22, 0x3e, 0x0a, 0x04, 0x45, 0x6e, 0x75, - 0x6d, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, - 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x46, 0x49, 0x52, 0x45, 0x5f, 0x41, 0x4c, 0x57, 0x41, 0x59, - 0x53, 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10, 0x46, 0x49, 0x52, 0x45, 0x5f, 0x49, 0x46, 0x5f, 0x4e, - 0x4f, 0x4e, 0x45, 0x4d, 0x50, 0x54, 0x59, 0x10, 0x02, 0x22, 0x62, 0x0a, 0x0a, 0x4f, 0x75, 0x74, - 0x70, 0x75, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x54, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, - 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, - 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x4e, 0x44, 0x5f, 0x4f, 0x46, 0x5f, 0x57, 0x49, 0x4e, 0x44, 0x4f, - 0x57, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x4c, 0x41, 0x54, 0x45, 0x53, 0x54, 0x5f, 0x49, 0x4e, - 0x5f, 0x50, 0x41, 0x4e, 0x45, 0x10, 0x02, 0x12, 0x14, 0x0a, 0x10, 0x45, 0x41, 0x52, 0x4c, 0x49, - 0x45, 0x53, 0x54, 0x5f, 0x49, 0x4e, 0x5f, 0x50, 0x41, 0x4e, 0x45, 0x10, 0x03, 0x22, 0x6e, 0x0a, - 0x0a, 0x54, 0x69, 0x6d, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x22, 0x60, 0x0a, 0x04, 0x45, - 0x6e, 0x75, 0x6d, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, - 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x49, - 0x4d, 0x45, 0x10, 0x01, 0x12, 0x13, 0x0a, 0x0f, 0x50, 0x52, 0x4f, 0x43, 0x45, 0x53, 0x53, 0x49, - 0x4e, 0x47, 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x02, 0x22, 0x04, 0x08, 0x03, 0x10, 0x03, 0x2a, - 0x1c, 0x53, 0x59, 0x4e, 0x43, 0x48, 0x52, 0x4f, 0x4e, 0x49, 0x5a, 0x45, 0x44, 0x5f, 0x50, 0x52, - 0x4f, 0x43, 0x45, 0x53, 0x53, 0x49, 0x4e, 0x47, 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x22, 0xa3, 0x10, - 0x0a, 0x07, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x12, 0x52, 0x0a, 0x09, 0x61, 0x66, 0x74, - 0x65, 0x72, 0x5f, 0x61, 0x6c, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x6f, + 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x04, 0x73, + 0x69, 0x6e, 0x6b, 0x12, 0x58, 0x0a, 0x0f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x5f, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x66, 0x74, 0x65, 0x72, 0x41, 0x6c, - 0x6c, 0x48, 0x00, 0x52, 0x08, 0x61, 0x66, 0x74, 0x65, 0x72, 0x41, 0x6c, 0x6c, 0x12, 0x52, 0x0a, - 0x09, 0x61, 0x66, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x6e, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x33, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, - 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x66, 0x74, - 0x65, 0x72, 0x41, 0x6e, 0x79, 0x48, 0x00, 0x52, 0x08, 0x61, 0x66, 0x74, 0x65, 0x72, 0x41, 0x6e, - 0x79, 0x12, 0x55, 0x0a, 0x0a, 0x61, 0x66, 0x74, 0x65, 0x72, 0x5f, 0x65, 0x61, 0x63, 0x68, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, - 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, - 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, - 0x72, 0x2e, 0x41, 0x66, 0x74, 0x65, 0x72, 0x45, 0x61, 0x63, 0x68, 0x48, 0x00, 0x52, 0x09, 0x61, - 0x66, 0x74, 0x65, 0x72, 0x45, 0x61, 0x63, 0x68, 0x12, 0x6c, 0x0a, 0x13, 0x61, 0x66, 0x74, 0x65, - 0x72, 0x5f, 0x65, 0x6e, 0x64, 0x5f, 0x6f, 0x66, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, + 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0e, 0x66, + 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x27, 0x0a, + 0x0f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x65, 0x64, 0x5f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x73, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x65, 0x64, + 0x57, 0x72, 0x69, 0x74, 0x65, 0x73, 0x12, 0x3c, 0x0a, 0x1a, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, + 0x5f, 0x64, 0x65, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x65, 0x64, 0x5f, 0x73, 0x68, 0x61, 0x72, + 0x64, 0x69, 0x6e, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x18, 0x72, 0x75, 0x6e, 0x6e, + 0x65, 0x72, 0x44, 0x65, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x65, 0x64, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x69, 0x6e, 0x67, 0x12, 0x65, 0x0a, 0x0b, 0x73, 0x69, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x70, + 0x75, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x6f, 0x72, 0x67, 0x2e, + 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x72, + 0x69, 0x74, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, + 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x0a, 0x73, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x1a, 0x6b, 0x0a, 0x0f, 0x53, + 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x42, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x2c, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, + 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xcc, 0x02, 0x0a, 0x11, 0x50, 0x75, 0x62, + 0x53, 0x75, 0x62, 0x52, 0x65, 0x61, 0x64, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x14, + 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, + 0x6f, 0x70, 0x69, 0x63, 0x12, 0x22, 0x0a, 0x0c, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x75, 0x62, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2f, 0x0a, 0x13, 0x74, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x64, 0x5f, + 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x69, 0x64, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x12, 0x27, 0x0a, 0x0f, + 0x77, 0x69, 0x74, 0x68, 0x5f, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x77, 0x69, 0x74, 0x68, 0x41, 0x74, 0x74, 0x72, 0x69, + 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x18, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x5f, 0x72, + 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x64, 0x65, + 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x52, 0x75, + 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x64, 0x65, 0x6e, 0x12, + 0x46, 0x0a, 0x1f, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x64, + 0x65, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1d, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x64, 0x65, 0x6e, 0x22, 0xb8, 0x01, 0x0a, 0x12, 0x50, 0x75, 0x62, 0x53, + 0x75, 0x62, 0x57, 0x72, 0x69, 0x74, 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x14, + 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, + 0x6f, 0x70, 0x69, 0x63, 0x12, 0x2f, 0x0a, 0x13, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x5f, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x12, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x41, 0x74, 0x74, 0x72, + 0x69, 0x62, 0x75, 0x74, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x64, 0x5f, 0x61, 0x74, 0x74, 0x72, + 0x69, 0x62, 0x75, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x64, 0x41, + 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x12, 0x38, 0x0a, 0x18, 0x74, 0x6f, 0x70, 0x69, + 0x63, 0x5f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, + 0x64, 0x64, 0x65, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x74, 0x6f, 0x70, 0x69, + 0x63, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x64, + 0x65, 0x6e, 0x22, 0xa5, 0x01, 0x0a, 0x17, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x6e, 0x74, 0x6f, + 0x42, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x1d, + 0x0a, 0x0a, 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x09, 0x62, 0x61, 0x74, 0x63, 0x68, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x28, 0x0a, + 0x10, 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, + 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x62, 0x61, 0x74, 0x63, 0x68, 0x53, 0x69, + 0x7a, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x41, 0x0a, 0x1d, 0x6d, 0x61, 0x78, 0x5f, 0x62, + 0x75, 0x66, 0x66, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x1a, + 0x6d, 0x61, 0x78, 0x42, 0x75, 0x66, 0x66, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x22, 0x7c, 0x0a, 0x05, 0x43, 0x6f, + 0x64, 0x65, 0x72, 0x12, 0x43, 0x0a, 0x04, 0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, + 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, + 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, + 0x65, 0x63, 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, 0x12, 0x2e, 0x0a, 0x13, 0x63, 0x6f, 0x6d, 0x70, + 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x73, 0x18, + 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, + 0x43, 0x6f, 0x64, 0x65, 0x72, 0x49, 0x64, 0x73, 0x22, 0xf5, 0x06, 0x0a, 0x0e, 0x53, 0x74, 0x61, + 0x6e, 0x64, 0x61, 0x72, 0x64, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x73, 0x22, 0xe2, 0x06, 0x0a, 0x04, + 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x24, 0x0a, 0x05, 0x42, 0x59, 0x54, 0x45, 0x53, 0x10, 0x00, 0x1a, + 0x19, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x13, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, + 0x72, 0x3a, 0x62, 0x79, 0x74, 0x65, 0x73, 0x3a, 0x76, 0x31, 0x12, 0x30, 0x0a, 0x0b, 0x53, 0x54, + 0x52, 0x49, 0x4e, 0x47, 0x5f, 0x55, 0x54, 0x46, 0x38, 0x10, 0x0a, 0x1a, 0x1f, 0xa2, 0xb4, 0xfa, + 0xc2, 0x05, 0x19, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x73, 0x74, + 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x75, 0x74, 0x66, 0x38, 0x3a, 0x76, 0x31, 0x12, 0x1e, 0x0a, 0x02, + 0x4b, 0x56, 0x10, 0x01, 0x1a, 0x16, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x10, 0x62, 0x65, 0x61, 0x6d, + 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x6b, 0x76, 0x3a, 0x76, 0x31, 0x12, 0x22, 0x0a, 0x04, + 0x42, 0x4f, 0x4f, 0x4c, 0x10, 0x0c, 0x1a, 0x18, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x12, 0x62, 0x65, + 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x62, 0x6f, 0x6f, 0x6c, 0x3a, 0x76, 0x31, + 0x12, 0x26, 0x0a, 0x06, 0x56, 0x41, 0x52, 0x49, 0x4e, 0x54, 0x10, 0x02, 0x1a, 0x1a, 0xa2, 0xb4, + 0xfa, 0xc2, 0x05, 0x14, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x76, + 0x61, 0x72, 0x69, 0x6e, 0x74, 0x3a, 0x76, 0x31, 0x12, 0x26, 0x0a, 0x06, 0x44, 0x4f, 0x55, 0x42, + 0x4c, 0x45, 0x10, 0x0b, 0x1a, 0x1a, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x14, 0x62, 0x65, 0x61, 0x6d, + 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x3a, 0x76, 0x31, + 0x12, 0x2a, 0x0a, 0x08, 0x49, 0x54, 0x45, 0x52, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x03, 0x1a, 0x1c, + 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x16, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, + 0x3a, 0x69, 0x74, 0x65, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x3a, 0x76, 0x31, 0x12, 0x24, 0x0a, 0x05, + 0x54, 0x49, 0x4d, 0x45, 0x52, 0x10, 0x04, 0x1a, 0x19, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x13, 0x62, + 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x3a, + 0x76, 0x31, 0x12, 0x38, 0x0a, 0x0f, 0x49, 0x4e, 0x54, 0x45, 0x52, 0x56, 0x41, 0x4c, 0x5f, 0x57, + 0x49, 0x4e, 0x44, 0x4f, 0x57, 0x10, 0x05, 0x1a, 0x23, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1d, 0x62, + 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, + 0x61, 0x6c, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x3a, 0x76, 0x31, 0x12, 0x34, 0x0a, 0x0d, + 0x4c, 0x45, 0x4e, 0x47, 0x54, 0x48, 0x5f, 0x50, 0x52, 0x45, 0x46, 0x49, 0x58, 0x10, 0x06, 0x1a, + 0x21, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1b, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, + 0x72, 0x3a, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x3a, + 0x76, 0x31, 0x12, 0x34, 0x0a, 0x0d, 0x47, 0x4c, 0x4f, 0x42, 0x41, 0x4c, 0x5f, 0x57, 0x49, 0x4e, + 0x44, 0x4f, 0x57, 0x10, 0x07, 0x1a, 0x21, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1b, 0x62, 0x65, 0x61, + 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x5f, 0x77, + 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x3a, 0x76, 0x31, 0x12, 0x36, 0x0a, 0x0e, 0x57, 0x49, 0x4e, 0x44, + 0x4f, 0x57, 0x45, 0x44, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x10, 0x08, 0x1a, 0x22, 0xa2, 0xb4, + 0xfa, 0xc2, 0x05, 0x1c, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x77, + 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x65, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x76, 0x31, + 0x12, 0x42, 0x0a, 0x14, 0x50, 0x41, 0x52, 0x41, 0x4d, 0x5f, 0x57, 0x49, 0x4e, 0x44, 0x4f, 0x57, + 0x45, 0x44, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x10, 0x0e, 0x1a, 0x28, 0xa2, 0xb4, 0xfa, 0xc2, + 0x05, 0x22, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x70, 0x61, 0x72, + 0x61, 0x6d, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x65, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x76, 0x31, 0x12, 0x44, 0x0a, 0x15, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x42, 0x41, + 0x43, 0x4b, 0x45, 0x44, 0x5f, 0x49, 0x54, 0x45, 0x52, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x09, 0x1a, + 0x29, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x23, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, + 0x72, 0x3a, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x69, + 0x74, 0x65, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x3a, 0x76, 0x31, 0x12, 0x34, 0x0a, 0x0d, 0x43, 0x55, + 0x53, 0x54, 0x4f, 0x4d, 0x5f, 0x57, 0x49, 0x4e, 0x44, 0x4f, 0x57, 0x10, 0x10, 0x1a, 0x21, 0xa2, + 0xb4, 0xfa, 0xc2, 0x05, 0x1b, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, + 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x3a, 0x76, 0x31, + 0x12, 0x20, 0x0a, 0x03, 0x52, 0x4f, 0x57, 0x10, 0x0d, 0x1a, 0x17, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, + 0x11, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x72, 0x6f, 0x77, 0x3a, + 0x76, 0x31, 0x12, 0x30, 0x0a, 0x0b, 0x53, 0x48, 0x41, 0x52, 0x44, 0x45, 0x44, 0x5f, 0x4b, 0x45, + 0x59, 0x10, 0x0f, 0x1a, 0x1f, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x19, 0x62, 0x65, 0x61, 0x6d, 0x3a, + 0x63, 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x73, 0x68, 0x61, 0x72, 0x64, 0x65, 0x64, 0x5f, 0x6b, 0x65, + 0x79, 0x3a, 0x76, 0x31, 0x12, 0x2a, 0x0a, 0x08, 0x4e, 0x55, 0x4c, 0x4c, 0x41, 0x42, 0x4c, 0x45, + 0x10, 0x11, 0x1a, 0x1c, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x16, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x63, + 0x6f, 0x64, 0x65, 0x72, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x3a, 0x76, 0x31, + 0x22, 0xae, 0x06, 0x0a, 0x11, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x69, 0x6e, 0x67, 0x53, 0x74, + 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x12, 0x4c, 0x0a, 0x09, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, + 0x5f, 0x66, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, + 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x08, 0x77, 0x69, 0x6e, 0x64, + 0x6f, 0x77, 0x46, 0x6e, 0x12, 0x56, 0x0a, 0x0c, 0x6d, 0x65, 0x72, 0x67, 0x65, 0x5f, 0x73, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x33, 0x2e, 0x6f, 0x72, 0x67, + 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, + 0x65, 0x72, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, + 0x0b, 0x6d, 0x65, 0x72, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x26, 0x0a, 0x0f, + 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x43, 0x6f, 0x64, + 0x65, 0x72, 0x49, 0x64, 0x12, 0x44, 0x0a, 0x07, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, - 0x72, 0x2e, 0x41, 0x66, 0x74, 0x65, 0x72, 0x45, 0x6e, 0x64, 0x4f, 0x66, 0x57, 0x69, 0x6e, 0x64, - 0x6f, 0x77, 0x48, 0x00, 0x52, 0x10, 0x61, 0x66, 0x74, 0x65, 0x72, 0x45, 0x6e, 0x64, 0x4f, 0x66, - 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x74, 0x0a, 0x15, 0x61, 0x66, 0x74, 0x65, 0x72, 0x5f, - 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, + 0x72, 0x52, 0x07, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x12, 0x65, 0x0a, 0x11, 0x61, 0x63, + 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x38, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, - 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, - 0x72, 0x2e, 0x41, 0x66, 0x74, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, - 0x67, 0x54, 0x69, 0x6d, 0x65, 0x48, 0x00, 0x52, 0x13, 0x61, 0x66, 0x74, 0x65, 0x72, 0x50, 0x72, - 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x99, 0x01, 0x0a, - 0x22, 0x61, 0x66, 0x74, 0x65, 0x72, 0x5f, 0x73, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, - 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x5f, 0x74, - 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, - 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, - 0x69, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x66, 0x74, 0x65, 0x72, 0x53, 0x79, 0x6e, 0x63, 0x68, - 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x64, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, - 0x67, 0x54, 0x69, 0x6d, 0x65, 0x48, 0x00, 0x52, 0x1f, 0x61, 0x66, 0x74, 0x65, 0x72, 0x53, 0x79, - 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x64, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, - 0x73, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x4b, 0x0a, 0x06, 0x61, 0x6c, 0x77, 0x61, - 0x79, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, - 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, - 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, - 0x67, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x6c, 0x77, 0x61, 0x79, 0x73, 0x48, 0x00, 0x52, 0x06, 0x61, - 0x6c, 0x77, 0x61, 0x79, 0x73, 0x12, 0x4e, 0x0a, 0x07, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, + 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, + 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, + 0x10, 0x61, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, + 0x65, 0x12, 0x53, 0x0a, 0x0b, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x32, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, - 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, - 0x65, 0x72, 0x2e, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x48, 0x00, 0x52, 0x07, 0x64, 0x65, - 0x66, 0x61, 0x75, 0x6c, 0x74, 0x12, 0x5e, 0x0a, 0x0d, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, - 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, - 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x48, 0x0a, 0x05, 0x6e, 0x65, 0x76, 0x65, 0x72, 0x18, 0x09, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, - 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, - 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, - 0x2e, 0x4e, 0x65, 0x76, 0x65, 0x72, 0x48, 0x00, 0x52, 0x05, 0x6e, 0x65, 0x76, 0x65, 0x72, 0x12, - 0x55, 0x0a, 0x0a, 0x6f, 0x72, 0x5f, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x18, 0x0a, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, - 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, - 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x2e, - 0x4f, 0x72, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x48, 0x00, 0x52, 0x09, 0x6f, 0x72, 0x46, - 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x12, 0x4b, 0x0a, 0x06, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, - 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, + 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75, + 0x74, 0x54, 0x69, 0x6d, 0x65, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x0a, 0x6f, 0x75, 0x74, 0x70, + 0x75, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x62, 0x0a, 0x10, 0x63, 0x6c, 0x6f, 0x73, 0x69, 0x6e, + 0x67, 0x5f, 0x62, 0x65, 0x68, 0x61, 0x76, 0x69, 0x6f, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, + 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x69, 0x6e, 0x67, 0x42, 0x65, 0x68, 0x61, + 0x76, 0x69, 0x6f, 0x72, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x0f, 0x63, 0x6c, 0x6f, 0x73, 0x69, + 0x6e, 0x67, 0x42, 0x65, 0x68, 0x61, 0x76, 0x69, 0x6f, 0x72, 0x12, 0x29, 0x0a, 0x10, 0x61, 0x6c, + 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x5f, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x65, 0x73, 0x73, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x4c, 0x61, 0x74, + 0x65, 0x6e, 0x65, 0x73, 0x73, 0x12, 0x60, 0x0a, 0x10, 0x6f, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, + 0x5f, 0x62, 0x65, 0x68, 0x61, 0x76, 0x69, 0x6f, 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x36, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, + 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x42, 0x65, 0x68, 0x61, 0x76, 0x69, + 0x6f, 0x72, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x0e, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x42, + 0x65, 0x68, 0x61, 0x76, 0x69, 0x6f, 0x72, 0x12, 0x31, 0x0a, 0x15, 0x61, 0x73, 0x73, 0x69, 0x67, + 0x6e, 0x73, 0x5f, 0x74, 0x6f, 0x5f, 0x6f, 0x6e, 0x65, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x61, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x73, 0x54, + 0x6f, 0x4f, 0x6e, 0x65, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x0b, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0d, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x49, + 0x64, 0x22, 0x5c, 0x0a, 0x0b, 0x4d, 0x65, 0x72, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x22, 0x4d, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, + 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x4e, 0x4f, 0x4e, + 0x5f, 0x4d, 0x45, 0x52, 0x47, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x4e, 0x45, + 0x45, 0x44, 0x53, 0x5f, 0x4d, 0x45, 0x52, 0x47, 0x45, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x41, + 0x4c, 0x52, 0x45, 0x41, 0x44, 0x59, 0x5f, 0x4d, 0x45, 0x52, 0x47, 0x45, 0x44, 0x10, 0x03, 0x22, + 0x5d, 0x0a, 0x10, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, + 0x6f, 0x64, 0x65, 0x22, 0x49, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x0f, 0x0a, 0x0b, 0x55, + 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, + 0x44, 0x49, 0x53, 0x43, 0x41, 0x52, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, + 0x41, 0x43, 0x43, 0x55, 0x4d, 0x55, 0x4c, 0x41, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x0e, + 0x0a, 0x0a, 0x52, 0x45, 0x54, 0x52, 0x41, 0x43, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x22, 0x51, + 0x0a, 0x0f, 0x43, 0x6c, 0x6f, 0x73, 0x69, 0x6e, 0x67, 0x42, 0x65, 0x68, 0x61, 0x76, 0x69, 0x6f, + 0x72, 0x22, 0x3e, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, + 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x45, 0x4d, + 0x49, 0x54, 0x5f, 0x41, 0x4c, 0x57, 0x41, 0x59, 0x53, 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10, 0x45, + 0x4d, 0x49, 0x54, 0x5f, 0x49, 0x46, 0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x4d, 0x50, 0x54, 0x59, 0x10, + 0x02, 0x22, 0x50, 0x0a, 0x0e, 0x4f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x42, 0x65, 0x68, 0x61, 0x76, + 0x69, 0x6f, 0x72, 0x22, 0x3e, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x0f, 0x0a, 0x0b, 0x55, + 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, + 0x46, 0x49, 0x52, 0x45, 0x5f, 0x41, 0x4c, 0x57, 0x41, 0x59, 0x53, 0x10, 0x01, 0x12, 0x14, 0x0a, + 0x10, 0x46, 0x49, 0x52, 0x45, 0x5f, 0x49, 0x46, 0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x4d, 0x50, 0x54, + 0x59, 0x10, 0x02, 0x22, 0x62, 0x0a, 0x0a, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x54, 0x69, 0x6d, + 0x65, 0x22, 0x54, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, + 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x4e, + 0x44, 0x5f, 0x4f, 0x46, 0x5f, 0x57, 0x49, 0x4e, 0x44, 0x4f, 0x57, 0x10, 0x01, 0x12, 0x12, 0x0a, + 0x0e, 0x4c, 0x41, 0x54, 0x45, 0x53, 0x54, 0x5f, 0x49, 0x4e, 0x5f, 0x50, 0x41, 0x4e, 0x45, 0x10, + 0x02, 0x12, 0x14, 0x0a, 0x10, 0x45, 0x41, 0x52, 0x4c, 0x49, 0x45, 0x53, 0x54, 0x5f, 0x49, 0x4e, + 0x5f, 0x50, 0x41, 0x4e, 0x45, 0x10, 0x03, 0x22, 0x6e, 0x0a, 0x0a, 0x54, 0x69, 0x6d, 0x65, 0x44, + 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x22, 0x60, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x0f, 0x0a, + 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, + 0x0a, 0x0a, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x01, 0x12, 0x13, + 0x0a, 0x0f, 0x50, 0x52, 0x4f, 0x43, 0x45, 0x53, 0x53, 0x49, 0x4e, 0x47, 0x5f, 0x54, 0x49, 0x4d, + 0x45, 0x10, 0x02, 0x22, 0x04, 0x08, 0x03, 0x10, 0x03, 0x2a, 0x1c, 0x53, 0x59, 0x4e, 0x43, 0x48, + 0x52, 0x4f, 0x4e, 0x49, 0x5a, 0x45, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x43, 0x45, 0x53, 0x53, 0x49, + 0x4e, 0x47, 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x22, 0xa3, 0x10, 0x0a, 0x07, 0x54, 0x72, 0x69, 0x67, + 0x67, 0x65, 0x72, 0x12, 0x52, 0x0a, 0x09, 0x61, 0x66, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x6c, 0x6c, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, - 0x65, 0x72, 0x2e, 0x52, 0x65, 0x70, 0x65, 0x61, 0x74, 0x48, 0x00, 0x52, 0x06, 0x72, 0x65, 0x70, - 0x65, 0x61, 0x74, 0x1a, 0x58, 0x0a, 0x08, 0x41, 0x66, 0x74, 0x65, 0x72, 0x41, 0x6c, 0x6c, 0x12, - 0x4c, 0x0a, 0x0b, 0x73, 0x75, 0x62, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, - 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, - 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, - 0x52, 0x0b, 0x73, 0x75, 0x62, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x73, 0x1a, 0x58, 0x0a, - 0x08, 0x41, 0x66, 0x74, 0x65, 0x72, 0x41, 0x6e, 0x79, 0x12, 0x4c, 0x0a, 0x0b, 0x73, 0x75, 0x62, + 0x65, 0x72, 0x2e, 0x41, 0x66, 0x74, 0x65, 0x72, 0x41, 0x6c, 0x6c, 0x48, 0x00, 0x52, 0x08, 0x61, + 0x66, 0x74, 0x65, 0x72, 0x41, 0x6c, 0x6c, 0x12, 0x52, 0x0a, 0x09, 0x61, 0x66, 0x74, 0x65, 0x72, + 0x5f, 0x61, 0x6e, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x6f, 0x72, 0x67, + 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, + 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x66, 0x74, 0x65, 0x72, 0x41, 0x6e, 0x79, 0x48, + 0x00, 0x52, 0x08, 0x61, 0x66, 0x74, 0x65, 0x72, 0x41, 0x6e, 0x79, 0x12, 0x55, 0x0a, 0x0a, 0x61, + 0x66, 0x74, 0x65, 0x72, 0x5f, 0x65, 0x61, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x34, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, + 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x66, 0x74, 0x65, + 0x72, 0x45, 0x61, 0x63, 0x68, 0x48, 0x00, 0x52, 0x09, 0x61, 0x66, 0x74, 0x65, 0x72, 0x45, 0x61, + 0x63, 0x68, 0x12, 0x6c, 0x0a, 0x13, 0x61, 0x66, 0x74, 0x65, 0x72, 0x5f, 0x65, 0x6e, 0x64, 0x5f, + 0x6f, 0x66, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x3b, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, + 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x66, 0x74, 0x65, + 0x72, 0x45, 0x6e, 0x64, 0x4f, 0x66, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x48, 0x00, 0x52, 0x10, + 0x61, 0x66, 0x74, 0x65, 0x72, 0x45, 0x6e, 0x64, 0x4f, 0x66, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, + 0x12, 0x74, 0x0a, 0x15, 0x61, 0x66, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, + 0x73, 0x69, 0x6e, 0x67, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x3e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, + 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x66, 0x74, 0x65, + 0x72, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x48, + 0x00, 0x52, 0x13, 0x61, 0x66, 0x74, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, + 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x99, 0x01, 0x0a, 0x22, 0x61, 0x66, 0x74, 0x65, 0x72, + 0x5f, 0x73, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x72, + 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x4a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, + 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, + 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x2e, + 0x41, 0x66, 0x74, 0x65, 0x72, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, + 0x64, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x48, + 0x00, 0x52, 0x1f, 0x61, 0x66, 0x74, 0x65, 0x72, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, + 0x69, 0x7a, 0x65, 0x64, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x54, 0x69, + 0x6d, 0x65, 0x12, 0x4b, 0x0a, 0x06, 0x61, 0x6c, 0x77, 0x61, 0x79, 0x73, 0x18, 0x0c, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, + 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, + 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x41, + 0x6c, 0x77, 0x61, 0x79, 0x73, 0x48, 0x00, 0x52, 0x06, 0x61, 0x6c, 0x77, 0x61, 0x79, 0x73, 0x12, + 0x4e, 0x0a, 0x07, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x32, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, + 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x65, 0x66, + 0x61, 0x75, 0x6c, 0x74, 0x48, 0x00, 0x52, 0x07, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x12, + 0x5e, 0x0a, 0x0d, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, + 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, + 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, + 0x65, 0x72, 0x2e, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x48, + 0x00, 0x52, 0x0c, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, + 0x48, 0x0a, 0x05, 0x6e, 0x65, 0x76, 0x65, 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, + 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, + 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, + 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x4e, 0x65, 0x76, 0x65, 0x72, + 0x48, 0x00, 0x52, 0x05, 0x6e, 0x65, 0x76, 0x65, 0x72, 0x12, 0x55, 0x0a, 0x0a, 0x6f, 0x72, 0x5f, + 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, + 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, + 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x4f, 0x72, 0x46, 0x69, 0x6e, 0x61, + 0x6c, 0x6c, 0x79, 0x48, 0x00, 0x52, 0x09, 0x6f, 0x72, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, + 0x12, 0x4b, 0x0a, 0x06, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x31, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, + 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x70, + 0x65, 0x61, 0x74, 0x48, 0x00, 0x52, 0x06, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x1a, 0x58, 0x0a, + 0x08, 0x41, 0x66, 0x74, 0x65, 0x72, 0x41, 0x6c, 0x6c, 0x12, 0x4c, 0x0a, 0x0b, 0x73, 0x75, 0x62, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, 0x0b, 0x73, 0x75, 0x62, 0x74, - 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x73, 0x1a, 0x59, 0x0a, 0x09, 0x41, 0x66, 0x74, 0x65, 0x72, - 0x45, 0x61, 0x63, 0x68, 0x12, 0x4c, 0x0a, 0x0b, 0x73, 0x75, 0x62, 0x74, 0x72, 0x69, 0x67, 0x67, - 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, + 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x73, 0x1a, 0x58, 0x0a, 0x08, 0x41, 0x66, 0x74, 0x65, 0x72, + 0x41, 0x6e, 0x79, 0x12, 0x4c, 0x0a, 0x0b, 0x73, 0x75, 0x62, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, + 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, + 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, + 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, + 0x67, 0x67, 0x65, 0x72, 0x52, 0x0b, 0x73, 0x75, 0x62, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, + 0x73, 0x1a, 0x59, 0x0a, 0x09, 0x41, 0x66, 0x74, 0x65, 0x72, 0x45, 0x61, 0x63, 0x68, 0x12, 0x4c, + 0x0a, 0x0b, 0x73, 0x75, 0x62, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, + 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, + 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, + 0x0b, 0x73, 0x75, 0x62, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x73, 0x1a, 0xb2, 0x01, 0x0a, + 0x10, 0x41, 0x66, 0x74, 0x65, 0x72, 0x45, 0x6e, 0x64, 0x4f, 0x66, 0x57, 0x69, 0x6e, 0x64, 0x6f, + 0x77, 0x12, 0x4f, 0x0a, 0x0d, 0x65, 0x61, 0x72, 0x6c, 0x79, 0x5f, 0x66, 0x69, 0x72, 0x69, 0x6e, + 0x67, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, + 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, + 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, + 0x67, 0x67, 0x65, 0x72, 0x52, 0x0c, 0x65, 0x61, 0x72, 0x6c, 0x79, 0x46, 0x69, 0x72, 0x69, 0x6e, + 0x67, 0x73, 0x12, 0x4d, 0x0a, 0x0c, 0x6c, 0x61, 0x74, 0x65, 0x5f, 0x66, 0x69, 0x72, 0x69, 0x6e, + 0x67, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, + 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, + 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, + 0x67, 0x67, 0x65, 0x72, 0x52, 0x0b, 0x6c, 0x61, 0x74, 0x65, 0x46, 0x69, 0x72, 0x69, 0x6e, 0x67, + 0x73, 0x1a, 0x7f, 0x0a, 0x13, 0x41, 0x66, 0x74, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, + 0x73, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x68, 0x0a, 0x14, 0x74, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, + 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, + 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x13, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, + 0x6d, 0x73, 0x1a, 0x21, 0x0a, 0x1f, 0x41, 0x66, 0x74, 0x65, 0x72, 0x53, 0x79, 0x6e, 0x63, 0x68, + 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x64, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, + 0x67, 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x09, 0x0a, 0x07, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, + 0x1a, 0x33, 0x0a, 0x0c, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, + 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x6f, 0x75, 0x6e, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x1a, 0x07, 0x0a, 0x05, 0x4e, 0x65, 0x76, 0x65, 0x72, 0x1a, 0x08, + 0x0a, 0x06, 0x41, 0x6c, 0x77, 0x61, 0x79, 0x73, 0x1a, 0x91, 0x01, 0x0a, 0x09, 0x4f, 0x72, 0x46, + 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x12, 0x3e, 0x0a, 0x04, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, + 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, + 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, + 0x52, 0x04, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x44, 0x0a, 0x07, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x6c, + 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, + 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, + 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, + 0x67, 0x65, 0x72, 0x52, 0x07, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x1a, 0x54, 0x0a, 0x06, + 0x52, 0x65, 0x70, 0x65, 0x61, 0x74, 0x12, 0x4a, 0x0a, 0x0a, 0x73, 0x75, 0x62, 0x74, 0x72, 0x69, + 0x67, 0x67, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, + 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, + 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, 0x0a, 0x73, 0x75, 0x62, 0x74, 0x72, 0x69, 0x67, 0x67, + 0x65, 0x72, 0x42, 0x09, 0x0a, 0x07, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x22, 0xc3, 0x02, + 0x0a, 0x12, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x54, 0x72, 0x61, 0x6e, 0x73, + 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x53, 0x0a, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, + 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, + 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x2e, 0x44, 0x65, 0x6c, 0x61, 0x79, + 0x48, 0x00, 0x52, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x12, 0x5a, 0x0a, 0x08, 0x61, 0x6c, 0x69, + 0x67, 0x6e, 0x5f, 0x74, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x6f, 0x72, + 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, + 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, + 0x72, 0x6d, 0x2e, 0x41, 0x6c, 0x69, 0x67, 0x6e, 0x54, 0x6f, 0x48, 0x00, 0x52, 0x07, 0x61, 0x6c, + 0x69, 0x67, 0x6e, 0x54, 0x6f, 0x1a, 0x2a, 0x0a, 0x05, 0x44, 0x65, 0x6c, 0x61, 0x79, 0x12, 0x21, + 0x0a, 0x0c, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x4d, 0x69, 0x6c, 0x6c, 0x69, + 0x73, 0x1a, 0x39, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x67, 0x6e, 0x54, 0x6f, 0x12, 0x16, 0x0a, 0x06, + 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x70, 0x65, + 0x72, 0x69, 0x6f, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x42, 0x15, 0x0a, 0x13, + 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, + 0x6f, 0x72, 0x6d, 0x22, 0x8a, 0x02, 0x0a, 0x09, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, + 0x74, 0x12, 0x56, 0x0a, 0x0e, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x70, 0x61, 0x74, 0x74, + 0x65, 0x72, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, - 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, - 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, 0x0b, 0x73, 0x75, 0x62, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, - 0x72, 0x73, 0x1a, 0xb2, 0x01, 0x0a, 0x10, 0x41, 0x66, 0x74, 0x65, 0x72, 0x45, 0x6e, 0x64, 0x4f, - 0x66, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x4f, 0x0a, 0x0d, 0x65, 0x61, 0x72, 0x6c, 0x79, - 0x5f, 0x66, 0x69, 0x72, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, - 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, - 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, 0x0c, 0x65, 0x61, 0x72, 0x6c, - 0x79, 0x46, 0x69, 0x72, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x4d, 0x0a, 0x0c, 0x6c, 0x61, 0x74, 0x65, - 0x5f, 0x66, 0x69, 0x72, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, + 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0d, 0x61, 0x63, 0x63, 0x65, + 0x73, 0x73, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x12, 0x48, 0x0a, 0x07, 0x76, 0x69, 0x65, + 0x77, 0x5f, 0x66, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, + 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x06, 0x76, 0x69, 0x65, + 0x77, 0x46, 0x6e, 0x12, 0x5b, 0x0a, 0x11, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x5f, 0x6d, 0x61, + 0x70, 0x70, 0x69, 0x6e, 0x67, 0x5f, 0x66, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, 0x0b, 0x6c, 0x61, 0x74, 0x65, - 0x46, 0x69, 0x72, 0x69, 0x6e, 0x67, 0x73, 0x1a, 0x7f, 0x0a, 0x13, 0x41, 0x66, 0x74, 0x65, 0x72, - 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x68, - 0x0a, 0x14, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x74, 0x72, 0x61, 0x6e, - 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x6f, + 0x76, 0x31, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, + 0x0f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x46, 0x6e, + 0x22, 0x89, 0x04, 0x0a, 0x11, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x41, 0x72, 0x74, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x22, 0x9f, 0x02, 0x0a, 0x05, 0x54, 0x79, 0x70, 0x65, 0x73, + 0x12, 0x2a, 0x0a, 0x04, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x00, 0x1a, 0x20, 0xa2, 0xb4, 0xfa, 0xc2, + 0x05, 0x1a, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x3a, + 0x74, 0x79, 0x70, 0x65, 0x3a, 0x66, 0x69, 0x6c, 0x65, 0x3a, 0x76, 0x31, 0x12, 0x28, 0x0a, 0x03, + 0x55, 0x52, 0x4c, 0x10, 0x01, 0x1a, 0x1f, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x19, 0x62, 0x65, 0x61, + 0x6d, 0x3a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x3a, 0x74, 0x79, 0x70, 0x65, 0x3a, + 0x75, 0x72, 0x6c, 0x3a, 0x76, 0x31, 0x12, 0x32, 0x0a, 0x08, 0x45, 0x4d, 0x42, 0x45, 0x44, 0x44, + 0x45, 0x44, 0x10, 0x02, 0x1a, 0x24, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1e, 0x62, 0x65, 0x61, 0x6d, + 0x3a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x3a, 0x74, 0x79, 0x70, 0x65, 0x3a, 0x65, + 0x6d, 0x62, 0x65, 0x64, 0x64, 0x65, 0x64, 0x3a, 0x76, 0x31, 0x12, 0x2a, 0x0a, 0x04, 0x50, 0x59, + 0x50, 0x49, 0x10, 0x03, 0x1a, 0x20, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1a, 0x62, 0x65, 0x61, 0x6d, + 0x3a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x3a, 0x74, 0x79, 0x70, 0x65, 0x3a, 0x70, + 0x79, 0x70, 0x69, 0x3a, 0x76, 0x31, 0x12, 0x2c, 0x0a, 0x05, 0x4d, 0x41, 0x56, 0x45, 0x4e, 0x10, + 0x04, 0x1a, 0x21, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1b, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x61, 0x72, + 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x3a, 0x74, 0x79, 0x70, 0x65, 0x3a, 0x6d, 0x61, 0x76, 0x65, + 0x6e, 0x3a, 0x76, 0x31, 0x12, 0x32, 0x0a, 0x08, 0x44, 0x45, 0x46, 0x45, 0x52, 0x52, 0x45, 0x44, + 0x10, 0x05, 0x1a, 0x24, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1e, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x61, + 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x3a, 0x74, 0x79, 0x70, 0x65, 0x3a, 0x64, 0x65, 0x66, + 0x65, 0x72, 0x72, 0x65, 0x64, 0x3a, 0x76, 0x31, 0x22, 0xd1, 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x6c, + 0x65, 0x73, 0x12, 0x36, 0x0a, 0x0a, 0x53, 0x54, 0x41, 0x47, 0x49, 0x4e, 0x47, 0x5f, 0x54, 0x4f, + 0x10, 0x00, 0x1a, 0x26, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x20, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x61, + 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x3a, 0x72, 0x6f, 0x6c, 0x65, 0x3a, 0x73, 0x74, 0x61, + 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x74, 0x6f, 0x3a, 0x76, 0x31, 0x12, 0x4c, 0x0a, 0x15, 0x50, 0x49, + 0x50, 0x5f, 0x52, 0x45, 0x51, 0x55, 0x49, 0x52, 0x45, 0x4d, 0x45, 0x4e, 0x54, 0x53, 0x5f, 0x46, + 0x49, 0x4c, 0x45, 0x10, 0x01, 0x1a, 0x31, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x2b, 0x62, 0x65, 0x61, + 0x6d, 0x3a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x3a, 0x72, 0x6f, 0x6c, 0x65, 0x3a, + 0x70, 0x69, 0x70, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, + 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x3a, 0x76, 0x31, 0x12, 0x42, 0x0a, 0x10, 0x47, 0x4f, 0x5f, 0x57, + 0x4f, 0x52, 0x4b, 0x45, 0x52, 0x5f, 0x42, 0x49, 0x4e, 0x41, 0x52, 0x59, 0x10, 0x02, 0x1a, 0x2c, + 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x26, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x61, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x3a, 0x72, 0x6f, 0x6c, 0x65, 0x3a, 0x67, 0x6f, 0x5f, 0x77, 0x6f, 0x72, 0x6b, + 0x65, 0x72, 0x5f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x3a, 0x76, 0x31, 0x22, 0x41, 0x0a, 0x13, + 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x79, 0x6c, + 0x6f, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x32, 0x35, + 0x36, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, 0x22, + 0x3e, 0x0a, 0x12, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x55, 0x72, 0x6c, 0x50, 0x61, + 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x32, 0x35, + 0x36, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, 0x22, + 0x29, 0x0a, 0x13, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x50, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x48, 0x0a, 0x0b, 0x50, 0x79, + 0x50, 0x49, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x72, 0x74, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x51, 0x0a, 0x0c, 0x4d, 0x61, 0x76, 0x65, 0x6e, 0x50, 0x61, 0x79, + 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x5f, 0x75, + 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, + 0x74, 0x6f, 0x72, 0x79, 0x55, 0x72, 0x6c, 0x22, 0x3f, 0x0a, 0x17, 0x44, 0x65, 0x66, 0x65, 0x72, + 0x72, 0x65, 0x64, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, + 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x3f, 0x0a, 0x1c, 0x41, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x54, 0x6f, 0x52, 0x6f, 0x6c, + 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x61, 0x67, + 0x65, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, + 0x74, 0x61, 0x67, 0x65, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x91, 0x01, 0x0a, 0x13, 0x41, 0x72, + 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x75, 0x72, 0x6e, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x79, 0x70, 0x65, 0x55, 0x72, 0x6e, 0x12, 0x21, 0x0a, 0x0c, + 0x74, 0x79, 0x70, 0x65, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, + 0x19, 0x0a, 0x08, 0x72, 0x6f, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x72, 0x6f, 0x6c, 0x65, 0x55, 0x72, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x6f, + 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x0b, 0x72, 0x6f, 0x6c, 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0xbe, 0x03, + 0x0a, 0x0b, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x10, 0x0a, + 0x03, 0x75, 0x72, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6e, 0x12, + 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x51, 0x0a, 0x0c, 0x64, 0x69, 0x73, + 0x70, 0x6c, 0x61, 0x79, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x2e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, + 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x44, 0x61, 0x74, 0x61, 0x52, + 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x44, 0x61, 0x74, 0x61, 0x12, 0x22, 0x0a, 0x0c, + 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, + 0x12, 0x5a, 0x0a, 0x0c, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, + 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, + 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, + 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, + 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x12, 0x68, 0x0a, 0x0e, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x68, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x07, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, + 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, + 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x48, 0x69, 0x6e, + 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x48, 0x69, 0x6e, 0x74, 0x73, 0x1a, 0x40, 0x0a, 0x12, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x48, 0x69, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x22, 0xc7, + 0x01, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xae, 0x01, 0x0a, 0x0c, 0x45, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x24, 0x0a, 0x06, 0x44, 0x4f, 0x43, 0x4b, + 0x45, 0x52, 0x10, 0x00, 0x1a, 0x18, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x12, 0x62, 0x65, 0x61, 0x6d, + 0x3a, 0x65, 0x6e, 0x76, 0x3a, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x3a, 0x76, 0x31, 0x12, 0x26, + 0x0a, 0x07, 0x50, 0x52, 0x4f, 0x43, 0x45, 0x53, 0x53, 0x10, 0x01, 0x1a, 0x19, 0xa2, 0xb4, 0xfa, + 0xc2, 0x05, 0x13, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x65, 0x6e, 0x76, 0x3a, 0x70, 0x72, 0x6f, 0x63, + 0x65, 0x73, 0x73, 0x3a, 0x76, 0x31, 0x12, 0x28, 0x0a, 0x08, 0x45, 0x58, 0x54, 0x45, 0x52, 0x4e, + 0x41, 0x4c, 0x10, 0x02, 0x1a, 0x1a, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x14, 0x62, 0x65, 0x61, 0x6d, + 0x3a, 0x65, 0x6e, 0x76, 0x3a, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x3a, 0x76, 0x31, + 0x12, 0x26, 0x0a, 0x07, 0x44, 0x45, 0x46, 0x41, 0x55, 0x4c, 0x54, 0x10, 0x03, 0x1a, 0x19, 0xa2, + 0xb4, 0xfa, 0xc2, 0x05, 0x13, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x65, 0x6e, 0x76, 0x3a, 0x64, 0x65, + 0x66, 0x61, 0x75, 0x6c, 0x74, 0x3a, 0x76, 0x31, 0x22, 0x38, 0x0a, 0x0d, 0x44, 0x6f, 0x63, 0x6b, + 0x65, 0x72, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x6d, 0x61, + 0x67, 0x65, 0x22, 0xd4, 0x01, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x50, 0x61, + 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x02, 0x6f, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x61, 0x72, 0x63, 0x68, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x61, 0x72, 0x63, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6d, + 0x6d, 0x61, 0x6e, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x12, 0x4c, 0x0a, 0x03, 0x65, 0x6e, 0x76, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x3a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, + 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x50, 0x61, 0x79, 0x6c, + 0x6f, 0x61, 0x64, 0x2e, 0x45, 0x6e, 0x76, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, 0x65, 0x6e, + 0x76, 0x1a, 0x36, 0x0a, 0x08, 0x45, 0x6e, 0x76, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xf9, 0x01, 0x0a, 0x0f, 0x45, 0x78, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x53, 0x0a, + 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, + 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, + 0x6e, 0x74, 0x12, 0x56, 0x0a, 0x06, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, + 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, + 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x50, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x06, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x50, 0x61, + 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x9a, 0x05, 0x0a, 0x11, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x61, + 0x72, 0x64, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x22, 0x84, 0x05, 0x0a, 0x04, + 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x48, 0x0a, 0x19, 0x4c, 0x45, 0x47, 0x41, 0x43, 0x59, 0x5f, 0x50, + 0x52, 0x4f, 0x47, 0x52, 0x45, 0x53, 0x53, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x52, 0x54, 0x49, 0x4e, + 0x47, 0x10, 0x00, 0x1a, 0x29, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x23, 0x62, 0x65, 0x61, 0x6d, 0x3a, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x3a, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, + 0x73, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x3a, 0x76, 0x30, 0x12, 0x41, + 0x0a, 0x12, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, 0x53, 0x53, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x52, + 0x54, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x1a, 0x29, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x23, 0x62, 0x65, + 0x61, 0x6d, 0x3a, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x3a, 0x70, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x3a, 0x76, + 0x31, 0x12, 0x37, 0x0a, 0x0d, 0x57, 0x4f, 0x52, 0x4b, 0x45, 0x52, 0x5f, 0x53, 0x54, 0x41, 0x54, + 0x55, 0x53, 0x10, 0x02, 0x1a, 0x24, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1e, 0x62, 0x65, 0x61, 0x6d, + 0x3a, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x3a, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, + 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x3a, 0x76, 0x31, 0x12, 0x55, 0x0a, 0x1c, 0x4d, 0x55, + 0x4c, 0x54, 0x49, 0x5f, 0x43, 0x4f, 0x52, 0x45, 0x5f, 0x42, 0x55, 0x4e, 0x44, 0x4c, 0x45, 0x5f, + 0x50, 0x52, 0x4f, 0x43, 0x45, 0x53, 0x53, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x1a, 0x33, 0xa2, 0xb4, + 0xfa, 0xc2, 0x05, 0x2d, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6c, 0x3a, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x5f, 0x63, 0x6f, 0x72, 0x65, 0x5f, 0x62, 0x75, 0x6e, + 0x64, 0x6c, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x3a, 0x76, + 0x31, 0x12, 0x3b, 0x0a, 0x0f, 0x53, 0x49, 0x42, 0x4c, 0x49, 0x4e, 0x47, 0x5f, 0x57, 0x4f, 0x52, + 0x4b, 0x45, 0x52, 0x53, 0x10, 0x05, 0x1a, 0x26, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x20, 0x62, 0x65, + 0x61, 0x6d, 0x3a, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x3a, 0x73, 0x69, 0x62, 0x6c, + 0x69, 0x6e, 0x67, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x3a, 0x76, 0x31, 0x12, 0x4d, + 0x0a, 0x18, 0x48, 0x41, 0x52, 0x4e, 0x45, 0x53, 0x53, 0x5f, 0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, + 0x52, 0x49, 0x4e, 0x47, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x53, 0x10, 0x04, 0x1a, 0x2f, 0xa2, 0xb4, + 0xfa, 0xc2, 0x05, 0x29, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6c, 0x3a, 0x68, 0x61, 0x72, 0x6e, 0x65, 0x73, 0x73, 0x5f, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, + 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x3a, 0x76, 0x31, 0x12, 0x61, 0x0a, + 0x22, 0x43, 0x4f, 0x4e, 0x54, 0x52, 0x4f, 0x4c, 0x5f, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, + 0x5f, 0x45, 0x4c, 0x45, 0x4d, 0x45, 0x4e, 0x54, 0x53, 0x5f, 0x45, 0x4d, 0x42, 0x45, 0x44, 0x44, + 0x49, 0x4e, 0x47, 0x10, 0x06, 0x1a, 0x39, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x33, 0x62, 0x65, 0x61, + 0x6d, 0x3a, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x3a, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x65, 0x6c, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x5f, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x3a, 0x76, 0x31, + 0x12, 0x37, 0x0a, 0x0d, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x43, 0x41, 0x43, 0x48, 0x49, 0x4e, + 0x47, 0x10, 0x07, 0x1a, 0x24, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1e, 0x62, 0x65, 0x61, 0x6d, 0x3a, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x3a, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x63, + 0x61, 0x63, 0x68, 0x69, 0x6e, 0x67, 0x3a, 0x76, 0x31, 0x12, 0x37, 0x0a, 0x0d, 0x44, 0x41, 0x54, + 0x41, 0x5f, 0x53, 0x41, 0x4d, 0x50, 0x4c, 0x49, 0x4e, 0x47, 0x10, 0x08, 0x1a, 0x24, 0xa2, 0xb4, + 0xfa, 0xc2, 0x05, 0x1e, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6c, 0x3a, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x69, 0x6e, 0x67, 0x3a, + 0x76, 0x31, 0x22, 0xd8, 0x01, 0x0a, 0x17, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x52, + 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x22, 0xbc, + 0x01, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x4f, 0x0a, 0x19, 0x4d, 0x4f, 0x4e, 0x49, 0x54, + 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x53, 0x48, 0x4f, 0x52, 0x54, + 0x5f, 0x49, 0x44, 0x53, 0x10, 0x00, 0x1a, 0x30, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x2a, 0x62, 0x65, + 0x61, 0x6d, 0x3a, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x3a, 0x6d, 0x6f, 0x6e, 0x69, + 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x5f, 0x73, 0x68, 0x6f, 0x72, + 0x74, 0x5f, 0x69, 0x64, 0x73, 0x3a, 0x76, 0x31, 0x12, 0x63, 0x0a, 0x23, 0x43, 0x4f, 0x4e, 0x54, + 0x52, 0x4f, 0x4c, 0x5f, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45, 0x5f, 0x45, 0x4c, 0x45, + 0x4d, 0x45, 0x4e, 0x54, 0x53, 0x5f, 0x45, 0x4d, 0x42, 0x45, 0x44, 0x44, 0x49, 0x4e, 0x47, 0x10, + 0x06, 0x1a, 0x3a, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x34, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x3a, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x72, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, + 0x5f, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x3a, 0x76, 0x31, 0x22, 0xff, 0x03, + 0x0a, 0x14, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x69, 0x72, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xe6, 0x03, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, + 0x4a, 0x0a, 0x1c, 0x52, 0x45, 0x51, 0x55, 0x49, 0x52, 0x45, 0x53, 0x5f, 0x53, 0x54, 0x41, 0x54, + 0x45, 0x46, 0x55, 0x4c, 0x5f, 0x50, 0x52, 0x4f, 0x43, 0x45, 0x53, 0x53, 0x49, 0x4e, 0x47, 0x10, + 0x00, 0x1a, 0x28, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x22, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x72, 0x65, + 0x71, 0x75, 0x69, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x3a, 0x70, 0x61, 0x72, 0x64, 0x6f, 0x3a, + 0x73, 0x74, 0x61, 0x74, 0x65, 0x66, 0x75, 0x6c, 0x3a, 0x76, 0x31, 0x12, 0x4e, 0x0a, 0x1c, 0x52, + 0x45, 0x51, 0x55, 0x49, 0x52, 0x45, 0x53, 0x5f, 0x42, 0x55, 0x4e, 0x44, 0x4c, 0x45, 0x5f, 0x46, + 0x49, 0x4e, 0x41, 0x4c, 0x49, 0x5a, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x01, 0x1a, 0x2c, 0xa2, + 0xb4, 0xfa, 0xc2, 0x05, 0x26, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x3a, 0x70, 0x61, 0x72, 0x64, 0x6f, 0x3a, 0x66, 0x69, 0x6e, 0x61, + 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x76, 0x31, 0x12, 0x47, 0x0a, 0x15, 0x52, + 0x45, 0x51, 0x55, 0x49, 0x52, 0x45, 0x53, 0x5f, 0x53, 0x54, 0x41, 0x42, 0x4c, 0x45, 0x5f, 0x49, + 0x4e, 0x50, 0x55, 0x54, 0x10, 0x02, 0x1a, 0x2c, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x26, 0x62, 0x65, + 0x61, 0x6d, 0x3a, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x3a, 0x70, + 0x61, 0x72, 0x64, 0x6f, 0x3a, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, + 0x74, 0x3a, 0x76, 0x31, 0x12, 0x51, 0x0a, 0x1a, 0x52, 0x45, 0x51, 0x55, 0x49, 0x52, 0x45, 0x53, + 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x5f, 0x53, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x5f, 0x49, 0x4e, 0x50, + 0x55, 0x54, 0x10, 0x03, 0x1a, 0x31, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x2b, 0x62, 0x65, 0x61, 0x6d, + 0x3a, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x3a, 0x70, 0x61, 0x72, + 0x64, 0x6f, 0x3a, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x69, + 0x6e, 0x70, 0x75, 0x74, 0x3a, 0x76, 0x31, 0x12, 0x4d, 0x0a, 0x18, 0x52, 0x45, 0x51, 0x55, 0x49, + 0x52, 0x45, 0x53, 0x5f, 0x53, 0x50, 0x4c, 0x49, 0x54, 0x54, 0x41, 0x42, 0x4c, 0x45, 0x5f, 0x44, + 0x4f, 0x46, 0x4e, 0x10, 0x04, 0x1a, 0x2f, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x29, 0x62, 0x65, 0x61, + 0x6d, 0x3a, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x3a, 0x70, 0x61, + 0x72, 0x64, 0x6f, 0x3a, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x64, + 0x6f, 0x66, 0x6e, 0x3a, 0x76, 0x31, 0x12, 0x57, 0x0a, 0x1d, 0x52, 0x45, 0x51, 0x55, 0x49, 0x52, + 0x45, 0x53, 0x5f, 0x4f, 0x4e, 0x5f, 0x57, 0x49, 0x4e, 0x44, 0x4f, 0x57, 0x5f, 0x45, 0x58, 0x50, + 0x49, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x05, 0x1a, 0x34, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, + 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x3a, 0x70, 0x61, 0x72, 0x64, 0x6f, 0x3a, 0x6f, 0x6e, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, + 0x77, 0x5f, 0x65, 0x78, 0x70, 0x69, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x76, 0x31, 0x22, + 0x3a, 0x0a, 0x0c, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x12, + 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, + 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x57, 0x0a, 0x13, 0x53, + 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x44, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x44, 0x61, + 0x74, 0x61, 0x22, 0x40, 0x0a, 0x0b, 0x44, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x44, 0x61, 0x74, + 0x61, 0x12, 0x31, 0x0a, 0x08, 0x4c, 0x41, 0x42, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x00, 0x1a, + 0x23, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1d, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x64, 0x69, 0x73, 0x70, + 0x6c, 0x61, 0x79, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x6c, 0x65, + 0x64, 0x3a, 0x76, 0x31, 0x22, 0xea, 0x01, 0x0a, 0x0f, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x6c, 0x65, + 0x64, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x61, 0x62, 0x65, + 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x23, + 0x0a, 0x0c, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, + 0x6c, 0x75, 0x65, 0x12, 0x1f, 0x0a, 0x0a, 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x09, 0x62, 0x6f, 0x6f, 0x6c, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x5f, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x0b, 0x64, 0x6f, + 0x75, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1d, 0x0a, 0x09, 0x69, 0x6e, 0x74, + 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x08, + 0x69, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, + 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, + 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x22, 0x39, 0x0a, 0x0b, 0x44, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x44, 0x61, 0x74, 0x61, + 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, + 0x72, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0xd7, 0x07, 0x0a, + 0x15, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x57, 0x69, 0x74, 0x68, 0x43, 0x6f, 0x6d, 0x70, + 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x4d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, + 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x6f, 0x72, 0x67, + 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, + 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x6f, + 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x40, 0x0a, 0x05, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, + 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, + 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x48, 0x00, + 0x52, 0x05, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x12, 0x5c, 0x0a, 0x0f, 0x63, 0x6f, 0x6d, 0x62, 0x69, + 0x6e, 0x65, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x31, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, + 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x50, 0x61, 0x79, 0x6c, + 0x6f, 0x61, 0x64, 0x48, 0x00, 0x52, 0x0e, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x50, 0x61, + 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x56, 0x0a, 0x0d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, - 0x6f, 0x72, 0x6d, 0x52, 0x13, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x54, 0x72, - 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x1a, 0x21, 0x0a, 0x1f, 0x41, 0x66, 0x74, 0x65, - 0x72, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x64, 0x50, 0x72, 0x6f, - 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x09, 0x0a, 0x07, 0x44, - 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x1a, 0x33, 0x0a, 0x0c, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x65, - 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x1a, 0x07, 0x0a, 0x05, 0x4e, - 0x65, 0x76, 0x65, 0x72, 0x1a, 0x08, 0x0a, 0x06, 0x41, 0x6c, 0x77, 0x61, 0x79, 0x73, 0x1a, 0x91, - 0x01, 0x0a, 0x09, 0x4f, 0x72, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x12, 0x3e, 0x0a, 0x04, - 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, + 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x48, 0x00, 0x52, + 0x0c, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x12, 0x57, 0x0a, + 0x0e, 0x70, 0x61, 0x72, 0x5f, 0x64, 0x6f, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, + 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, + 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x44, 0x6f, 0x50, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x48, 0x00, 0x52, 0x0c, 0x70, 0x61, 0x72, 0x44, 0x6f, 0x50, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x4f, 0x0a, 0x0a, 0x70, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, - 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, - 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, 0x04, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x44, 0x0a, 0x07, - 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, - 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, - 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, 0x07, 0x66, 0x69, 0x6e, 0x61, 0x6c, - 0x6c, 0x79, 0x1a, 0x54, 0x0a, 0x06, 0x52, 0x65, 0x70, 0x65, 0x61, 0x74, 0x12, 0x4a, 0x0a, 0x0a, - 0x73, 0x75, 0x62, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, - 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, 0x0a, 0x73, 0x75, - 0x62, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x42, 0x09, 0x0a, 0x07, 0x74, 0x72, 0x69, 0x67, - 0x67, 0x65, 0x72, 0x22, 0xc3, 0x02, 0x0a, 0x12, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x53, 0x0a, 0x05, 0x64, 0x65, - 0x6c, 0x61, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, - 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, - 0x2e, 0x44, 0x65, 0x6c, 0x61, 0x79, 0x48, 0x00, 0x52, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x12, - 0x5a, 0x0a, 0x08, 0x61, 0x6c, 0x69, 0x67, 0x6e, 0x5f, 0x74, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x3d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, + 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, + 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x48, 0x00, 0x52, 0x0a, 0x70, 0x74, 0x72, + 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x52, 0x0a, 0x0b, 0x70, 0x63, 0x6f, 0x6c, 0x6c, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6f, + 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, + 0x2e, 0x50, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x0b, + 0x70, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x53, 0x0a, 0x0c, 0x72, + 0x65, 0x61, 0x64, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x2e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, - 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x54, - 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x2e, 0x41, 0x6c, 0x69, 0x67, 0x6e, 0x54, 0x6f, - 0x48, 0x00, 0x52, 0x07, 0x61, 0x6c, 0x69, 0x67, 0x6e, 0x54, 0x6f, 0x1a, 0x2a, 0x0a, 0x05, 0x44, - 0x65, 0x6c, 0x61, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x5f, 0x6d, 0x69, - 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x64, 0x65, 0x6c, 0x61, - 0x79, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x1a, 0x39, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x67, 0x6e, - 0x54, 0x6f, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x06, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, - 0x66, 0x73, 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, - 0x65, 0x74, 0x42, 0x15, 0x0a, 0x13, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, - 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x22, 0x8a, 0x02, 0x0a, 0x09, 0x53, 0x69, - 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x56, 0x0a, 0x0e, 0x61, 0x63, 0x63, 0x65, 0x73, - 0x73, 0x5f, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, - 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, - 0x52, 0x0d, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x12, - 0x48, 0x0a, 0x07, 0x76, 0x69, 0x65, 0x77, 0x5f, 0x66, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, - 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, - 0x63, 0x52, 0x06, 0x76, 0x69, 0x65, 0x77, 0x46, 0x6e, 0x12, 0x5b, 0x0a, 0x11, 0x77, 0x69, 0x6e, - 0x64, 0x6f, 0x77, 0x5f, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x5f, 0x66, 0x6e, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, + 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, + 0x64, 0x48, 0x00, 0x52, 0x0b, 0x72, 0x65, 0x61, 0x64, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, + 0x12, 0x4d, 0x0a, 0x0a, 0x73, 0x69, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x0b, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, - 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x4d, 0x61, 0x70, - 0x70, 0x69, 0x6e, 0x67, 0x46, 0x6e, 0x22, 0x89, 0x04, 0x0a, 0x11, 0x53, 0x74, 0x61, 0x6e, 0x64, - 0x61, 0x72, 0x64, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x22, 0x9f, 0x02, 0x0a, - 0x05, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x2a, 0x0a, 0x04, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x00, - 0x1a, 0x20, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1a, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x61, 0x72, 0x74, - 0x69, 0x66, 0x61, 0x63, 0x74, 0x3a, 0x74, 0x79, 0x70, 0x65, 0x3a, 0x66, 0x69, 0x6c, 0x65, 0x3a, - 0x76, 0x31, 0x12, 0x28, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x10, 0x01, 0x1a, 0x1f, 0xa2, 0xb4, 0xfa, - 0xc2, 0x05, 0x19, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x3a, 0x74, 0x79, 0x70, 0x65, 0x3a, 0x75, 0x72, 0x6c, 0x3a, 0x76, 0x31, 0x12, 0x32, 0x0a, 0x08, - 0x45, 0x4d, 0x42, 0x45, 0x44, 0x44, 0x45, 0x44, 0x10, 0x02, 0x1a, 0x24, 0xa2, 0xb4, 0xfa, 0xc2, - 0x05, 0x1e, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x3a, - 0x74, 0x79, 0x70, 0x65, 0x3a, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x65, 0x64, 0x3a, 0x76, 0x31, - 0x12, 0x2a, 0x0a, 0x04, 0x50, 0x59, 0x50, 0x49, 0x10, 0x03, 0x1a, 0x20, 0xa2, 0xb4, 0xfa, 0xc2, - 0x05, 0x1a, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x3a, - 0x74, 0x79, 0x70, 0x65, 0x3a, 0x70, 0x79, 0x70, 0x69, 0x3a, 0x76, 0x31, 0x12, 0x2c, 0x0a, 0x05, - 0x4d, 0x41, 0x56, 0x45, 0x4e, 0x10, 0x04, 0x1a, 0x21, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1b, 0x62, - 0x65, 0x61, 0x6d, 0x3a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x3a, 0x74, 0x79, 0x70, - 0x65, 0x3a, 0x6d, 0x61, 0x76, 0x65, 0x6e, 0x3a, 0x76, 0x31, 0x12, 0x32, 0x0a, 0x08, 0x44, 0x45, - 0x46, 0x45, 0x52, 0x52, 0x45, 0x44, 0x10, 0x05, 0x1a, 0x24, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1e, - 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x3a, 0x74, 0x79, - 0x70, 0x65, 0x3a, 0x64, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x3a, 0x76, 0x31, 0x22, 0xd1, - 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x6c, 0x65, 0x73, 0x12, 0x36, 0x0a, 0x0a, 0x53, 0x54, 0x41, 0x47, - 0x49, 0x4e, 0x47, 0x5f, 0x54, 0x4f, 0x10, 0x00, 0x1a, 0x26, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x20, - 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x3a, 0x72, 0x6f, - 0x6c, 0x65, 0x3a, 0x73, 0x74, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x74, 0x6f, 0x3a, 0x76, 0x31, - 0x12, 0x4c, 0x0a, 0x15, 0x50, 0x49, 0x50, 0x5f, 0x52, 0x45, 0x51, 0x55, 0x49, 0x52, 0x45, 0x4d, - 0x45, 0x4e, 0x54, 0x53, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x01, 0x1a, 0x31, 0xa2, 0xb4, 0xfa, - 0xc2, 0x05, 0x2b, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x3a, 0x72, 0x6f, 0x6c, 0x65, 0x3a, 0x70, 0x69, 0x70, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x3a, 0x76, 0x31, 0x12, 0x42, - 0x0a, 0x10, 0x47, 0x4f, 0x5f, 0x57, 0x4f, 0x52, 0x4b, 0x45, 0x52, 0x5f, 0x42, 0x49, 0x4e, 0x41, - 0x52, 0x59, 0x10, 0x02, 0x1a, 0x2c, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x26, 0x62, 0x65, 0x61, 0x6d, - 0x3a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x3a, 0x72, 0x6f, 0x6c, 0x65, 0x3a, 0x67, - 0x6f, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x3a, - 0x76, 0x31, 0x22, 0x41, 0x0a, 0x13, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x46, 0x69, - 0x6c, 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, - 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x16, 0x0a, - 0x06, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, - 0x68, 0x61, 0x32, 0x35, 0x36, 0x22, 0x3e, 0x0a, 0x12, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x55, 0x72, 0x6c, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x75, - 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x16, 0x0a, - 0x06, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, - 0x68, 0x61, 0x32, 0x35, 0x36, 0x22, 0x29, 0x0a, 0x13, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x65, - 0x64, 0x46, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, - 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, - 0x22, 0x48, 0x0a, 0x0b, 0x50, 0x79, 0x50, 0x49, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, - 0x1f, 0x0a, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x64, - 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x51, 0x0a, 0x0c, 0x4d, 0x61, - 0x76, 0x65, 0x6e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x61, 0x72, - 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x61, 0x72, - 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, - 0x74, 0x6f, 0x72, 0x79, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, - 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x55, 0x72, 0x6c, 0x22, 0x3f, 0x0a, - 0x17, 0x44, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, - 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x3f, - 0x0a, 0x1c, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x61, 0x67, 0x69, 0x6e, - 0x67, 0x54, 0x6f, 0x52, 0x6f, 0x6c, 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x1f, - 0x0a, 0x0b, 0x73, 0x74, 0x61, 0x67, 0x65, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x74, 0x61, 0x67, 0x65, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x22, - 0x91, 0x01, 0x0a, 0x13, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x6e, 0x66, 0x6f, - 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x79, 0x70, 0x65, 0x5f, - 0x75, 0x72, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x79, 0x70, 0x65, 0x55, - 0x72, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, - 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x50, 0x61, - 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x72, 0x6f, 0x6c, 0x65, 0x5f, 0x75, 0x72, - 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x72, 0x6f, 0x6c, 0x65, 0x55, 0x72, 0x6e, - 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x6f, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x72, 0x6f, 0x6c, 0x65, 0x50, 0x61, 0x79, 0x6c, - 0x6f, 0x61, 0x64, 0x22, 0xbe, 0x03, 0x0a, 0x0b, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, - 0x65, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x75, 0x72, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, - 0x51, 0x0a, 0x0c, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, - 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, - 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x73, 0x70, 0x6c, 0x61, - 0x79, 0x44, 0x61, 0x74, 0x61, 0x52, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x44, 0x61, - 0x74, 0x61, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, - 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, - 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x5a, 0x0a, 0x0c, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, - 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x6f, + 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, + 0x75, 0x74, 0x48, 0x00, 0x52, 0x09, 0x73, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, + 0x66, 0x0a, 0x13, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x5f, 0x69, 0x6e, 0x74, 0x6f, 0x5f, 0x70, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, - 0x65, 0x73, 0x12, 0x68, 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x68, - 0x69, 0x6e, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x6f, 0x72, 0x67, - 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, - 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, - 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x48, 0x69, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x72, - 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x48, 0x69, 0x6e, 0x74, 0x73, 0x1a, 0x40, 0x0a, 0x12, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x48, 0x69, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a, 0x04, - 0x08, 0x01, 0x10, 0x02, 0x22, 0xc7, 0x01, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, - 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xae, 0x01, - 0x0a, 0x0c, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x24, - 0x0a, 0x06, 0x44, 0x4f, 0x43, 0x4b, 0x45, 0x52, 0x10, 0x00, 0x1a, 0x18, 0xa2, 0xb4, 0xfa, 0xc2, - 0x05, 0x12, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x65, 0x6e, 0x76, 0x3a, 0x64, 0x6f, 0x63, 0x6b, 0x65, - 0x72, 0x3a, 0x76, 0x31, 0x12, 0x26, 0x0a, 0x07, 0x50, 0x52, 0x4f, 0x43, 0x45, 0x53, 0x53, 0x10, - 0x01, 0x1a, 0x19, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x13, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x65, 0x6e, - 0x76, 0x3a, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x3a, 0x76, 0x31, 0x12, 0x28, 0x0a, 0x08, - 0x45, 0x58, 0x54, 0x45, 0x52, 0x4e, 0x41, 0x4c, 0x10, 0x02, 0x1a, 0x1a, 0xa2, 0xb4, 0xfa, 0xc2, - 0x05, 0x14, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x65, 0x6e, 0x76, 0x3a, 0x65, 0x78, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x3a, 0x76, 0x31, 0x12, 0x26, 0x0a, 0x07, 0x44, 0x45, 0x46, 0x41, 0x55, 0x4c, - 0x54, 0x10, 0x03, 0x1a, 0x19, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x13, 0x62, 0x65, 0x61, 0x6d, 0x3a, - 0x65, 0x6e, 0x76, 0x3a, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x3a, 0x76, 0x31, 0x22, 0x38, - 0x0a, 0x0d, 0x44, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, - 0x27, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x6d, 0x61, - 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x22, 0xd4, 0x01, 0x0a, 0x0e, 0x50, 0x72, 0x6f, - 0x63, 0x65, 0x73, 0x73, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x0e, 0x0a, 0x02, 0x6f, - 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x6f, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x61, - 0x72, 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x61, 0x72, 0x63, 0x68, 0x12, - 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x4c, 0x0a, 0x03, 0x65, 0x6e, 0x76, - 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, + 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x49, 0x6e, 0x74, 0x6f, 0x50, 0x61, 0x79, 0x6c, 0x6f, + 0x61, 0x64, 0x48, 0x00, 0x52, 0x11, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x49, 0x6e, 0x74, 0x6f, + 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x65, 0x0a, 0x12, 0x77, 0x69, 0x6e, 0x64, 0x6f, + 0x77, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x18, 0x0d, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, + 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, + 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x69, 0x6e, + 0x67, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x48, 0x00, 0x52, 0x11, 0x77, 0x69, 0x6e, + 0x64, 0x6f, 0x77, 0x69, 0x6e, 0x67, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x42, 0x06, + 0x0a, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x22, 0xb6, 0x0a, 0x0a, 0x16, 0x45, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x67, 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, + 0x64, 0x12, 0x50, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, - 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, - 0x73, 0x73, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x45, 0x6e, 0x76, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x03, 0x65, 0x6e, 0x76, 0x1a, 0x36, 0x0a, 0x08, 0x45, 0x6e, 0x76, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, - 0xf9, 0x01, 0x0a, 0x0f, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x50, 0x61, 0x79, 0x6c, - 0x6f, 0x61, 0x64, 0x12, 0x53, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, - 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, - 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x70, 0x69, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x08, - 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x56, 0x0a, 0x06, 0x70, 0x61, 0x72, 0x61, - 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, - 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, - 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x50, 0x61, 0x72, - 0x61, 0x6d, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, - 0x1a, 0x39, 0x0a, 0x0b, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xe1, 0x04, 0x0a, 0x11, - 0x53, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, - 0x73, 0x22, 0xcb, 0x04, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x48, 0x0a, 0x19, 0x4c, 0x45, - 0x47, 0x41, 0x43, 0x59, 0x5f, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, 0x53, 0x53, 0x5f, 0x52, 0x45, - 0x50, 0x4f, 0x52, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x00, 0x1a, 0x29, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, - 0x23, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x3a, 0x70, - 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x69, 0x6e, - 0x67, 0x3a, 0x76, 0x30, 0x12, 0x41, 0x0a, 0x12, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, 0x53, 0x53, - 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x52, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x1a, 0x29, 0xa2, 0xb4, - 0xfa, 0xc2, 0x05, 0x23, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6c, 0x3a, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x72, - 0x74, 0x69, 0x6e, 0x67, 0x3a, 0x76, 0x31, 0x12, 0x37, 0x0a, 0x0d, 0x57, 0x4f, 0x52, 0x4b, 0x45, - 0x52, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x10, 0x02, 0x1a, 0x24, 0xa2, 0xb4, 0xfa, 0xc2, - 0x05, 0x1e, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x3a, - 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x3a, 0x76, 0x31, - 0x12, 0x55, 0x0a, 0x1c, 0x4d, 0x55, 0x4c, 0x54, 0x49, 0x5f, 0x43, 0x4f, 0x52, 0x45, 0x5f, 0x42, - 0x55, 0x4e, 0x44, 0x4c, 0x45, 0x5f, 0x50, 0x52, 0x4f, 0x43, 0x45, 0x53, 0x53, 0x49, 0x4e, 0x47, - 0x10, 0x03, 0x1a, 0x33, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x2d, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x3a, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x5f, 0x63, 0x6f, - 0x72, 0x65, 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, - 0x73, 0x69, 0x6e, 0x67, 0x3a, 0x76, 0x31, 0x12, 0x3b, 0x0a, 0x0f, 0x53, 0x49, 0x42, 0x4c, 0x49, - 0x4e, 0x47, 0x5f, 0x57, 0x4f, 0x52, 0x4b, 0x45, 0x52, 0x53, 0x10, 0x05, 0x1a, 0x26, 0xa2, 0xb4, - 0xfa, 0xc2, 0x05, 0x20, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6c, 0x3a, 0x73, 0x69, 0x62, 0x6c, 0x69, 0x6e, 0x67, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, - 0x73, 0x3a, 0x76, 0x31, 0x12, 0x4d, 0x0a, 0x18, 0x48, 0x41, 0x52, 0x4e, 0x45, 0x53, 0x53, 0x5f, - 0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x53, - 0x10, 0x04, 0x1a, 0x2f, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x29, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x3a, 0x68, 0x61, 0x72, 0x6e, 0x65, 0x73, 0x73, 0x5f, - 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, - 0x3a, 0x76, 0x31, 0x12, 0x61, 0x0a, 0x22, 0x43, 0x4f, 0x4e, 0x54, 0x52, 0x4f, 0x4c, 0x5f, 0x52, - 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x45, 0x4c, 0x45, 0x4d, 0x45, 0x4e, 0x54, 0x53, 0x5f, - 0x45, 0x4d, 0x42, 0x45, 0x44, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x06, 0x1a, 0x39, 0xa2, 0xb4, 0xfa, - 0xc2, 0x05, 0x33, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, - 0x3a, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x5f, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x5f, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, - 0x69, 0x6e, 0x67, 0x3a, 0x76, 0x31, 0x12, 0x37, 0x0a, 0x0d, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, - 0x43, 0x41, 0x43, 0x48, 0x49, 0x4e, 0x47, 0x10, 0x07, 0x1a, 0x24, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, - 0x1e, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x3a, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x67, 0x3a, 0x76, 0x31, 0x22, - 0xd8, 0x01, 0x0a, 0x17, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x52, 0x75, 0x6e, 0x6e, - 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x22, 0xbc, 0x01, 0x0a, 0x04, - 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x4f, 0x0a, 0x19, 0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, - 0x4e, 0x47, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x53, 0x48, 0x4f, 0x52, 0x54, 0x5f, 0x49, 0x44, - 0x53, 0x10, 0x00, 0x1a, 0x30, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x2a, 0x62, 0x65, 0x61, 0x6d, 0x3a, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x3a, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, - 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x5f, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x5f, 0x69, - 0x64, 0x73, 0x3a, 0x76, 0x31, 0x12, 0x63, 0x0a, 0x23, 0x43, 0x4f, 0x4e, 0x54, 0x52, 0x4f, 0x4c, - 0x5f, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45, 0x5f, 0x45, 0x4c, 0x45, 0x4d, 0x45, 0x4e, - 0x54, 0x53, 0x5f, 0x45, 0x4d, 0x42, 0x45, 0x44, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x06, 0x1a, 0x3a, - 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x34, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6c, 0x3a, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x72, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x5f, 0x65, 0x6d, - 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x3a, 0x76, 0x31, 0x22, 0xff, 0x03, 0x0a, 0x14, 0x53, - 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x73, 0x22, 0xe6, 0x03, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x4a, 0x0a, 0x1c, - 0x52, 0x45, 0x51, 0x55, 0x49, 0x52, 0x45, 0x53, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x46, 0x55, - 0x4c, 0x5f, 0x50, 0x52, 0x4f, 0x43, 0x45, 0x53, 0x53, 0x49, 0x4e, 0x47, 0x10, 0x00, 0x1a, 0x28, - 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x22, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x72, 0x65, 0x71, 0x75, 0x69, - 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x3a, 0x70, 0x61, 0x72, 0x64, 0x6f, 0x3a, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x66, 0x75, 0x6c, 0x3a, 0x76, 0x31, 0x12, 0x4e, 0x0a, 0x1c, 0x52, 0x45, 0x51, 0x55, - 0x49, 0x52, 0x45, 0x53, 0x5f, 0x42, 0x55, 0x4e, 0x44, 0x4c, 0x45, 0x5f, 0x46, 0x49, 0x4e, 0x41, - 0x4c, 0x49, 0x5a, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x01, 0x1a, 0x2c, 0xa2, 0xb4, 0xfa, 0xc2, - 0x05, 0x26, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x3a, 0x70, 0x61, 0x72, 0x64, 0x6f, 0x3a, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x76, 0x31, 0x12, 0x47, 0x0a, 0x15, 0x52, 0x45, 0x51, 0x55, - 0x49, 0x52, 0x45, 0x53, 0x5f, 0x53, 0x54, 0x41, 0x42, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x50, 0x55, - 0x54, 0x10, 0x02, 0x1a, 0x2c, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x26, 0x62, 0x65, 0x61, 0x6d, 0x3a, - 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x3a, 0x70, 0x61, 0x72, 0x64, - 0x6f, 0x3a, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3a, 0x76, - 0x31, 0x12, 0x51, 0x0a, 0x1a, 0x52, 0x45, 0x51, 0x55, 0x49, 0x52, 0x45, 0x53, 0x5f, 0x54, 0x49, - 0x4d, 0x45, 0x5f, 0x53, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x5f, 0x49, 0x4e, 0x50, 0x55, 0x54, 0x10, - 0x03, 0x1a, 0x31, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x2b, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x72, 0x65, - 0x71, 0x75, 0x69, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x3a, 0x70, 0x61, 0x72, 0x64, 0x6f, 0x3a, - 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, - 0x74, 0x3a, 0x76, 0x31, 0x12, 0x4d, 0x0a, 0x18, 0x52, 0x45, 0x51, 0x55, 0x49, 0x52, 0x45, 0x53, - 0x5f, 0x53, 0x50, 0x4c, 0x49, 0x54, 0x54, 0x41, 0x42, 0x4c, 0x45, 0x5f, 0x44, 0x4f, 0x46, 0x4e, - 0x10, 0x04, 0x1a, 0x2f, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x29, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x72, - 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x3a, 0x70, 0x61, 0x72, 0x64, 0x6f, - 0x3a, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x64, 0x6f, 0x66, 0x6e, - 0x3a, 0x76, 0x31, 0x12, 0x57, 0x0a, 0x1d, 0x52, 0x45, 0x51, 0x55, 0x49, 0x52, 0x45, 0x53, 0x5f, - 0x4f, 0x4e, 0x5f, 0x57, 0x49, 0x4e, 0x44, 0x4f, 0x57, 0x5f, 0x45, 0x58, 0x50, 0x49, 0x52, 0x41, - 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x05, 0x1a, 0x34, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x2e, 0x62, 0x65, - 0x61, 0x6d, 0x3a, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x3a, 0x70, - 0x61, 0x72, 0x64, 0x6f, 0x3a, 0x6f, 0x6e, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x5f, 0x65, - 0x78, 0x70, 0x69, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x76, 0x31, 0x22, 0x3a, 0x0a, 0x0c, - 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x12, 0x10, 0x0a, 0x03, - 0x75, 0x72, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6e, 0x12, 0x18, - 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x57, 0x0a, 0x13, 0x53, 0x74, 0x61, 0x6e, - 0x64, 0x61, 0x72, 0x64, 0x44, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x44, 0x61, 0x74, 0x61, 0x22, - 0x40, 0x0a, 0x0b, 0x44, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x44, 0x61, 0x74, 0x61, 0x12, 0x31, - 0x0a, 0x08, 0x4c, 0x41, 0x42, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x00, 0x1a, 0x23, 0xa2, 0xb4, - 0xfa, 0xc2, 0x05, 0x1d, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, - 0x5f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x6c, 0x65, 0x64, 0x3a, 0x76, - 0x31, 0x22, 0xea, 0x01, 0x0a, 0x0f, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x6c, 0x65, 0x64, 0x50, 0x61, - 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x23, 0x0a, 0x0c, 0x73, - 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x48, 0x00, 0x52, 0x0b, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x12, 0x1f, 0x0a, 0x0a, 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x09, 0x62, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, - 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x0b, 0x64, 0x6f, 0x75, 0x62, 0x6c, - 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1d, 0x0a, 0x09, 0x69, 0x6e, 0x74, 0x5f, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x08, 0x69, 0x6e, 0x74, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, - 0x70, 0x61, 0x63, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x39, - 0x0a, 0x0b, 0x44, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x44, 0x61, 0x74, 0x61, 0x12, 0x10, 0x0a, - 0x03, 0x75, 0x72, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6e, 0x12, - 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0xd7, 0x07, 0x0a, 0x15, 0x4d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x57, 0x69, 0x74, 0x68, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, - 0x6e, 0x74, 0x73, 0x12, 0x4d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, - 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, - 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, - 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x70, - 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, - 0x74, 0x73, 0x12, 0x40, 0x0a, 0x05, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, - 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, - 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, 0x05, 0x63, - 0x6f, 0x64, 0x65, 0x72, 0x12, 0x5c, 0x0a, 0x0f, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x5f, - 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x31, 0x2e, - 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, - 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, - 0x48, 0x00, 0x52, 0x0e, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, - 0x61, 0x64, 0x12, 0x56, 0x0a, 0x0d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, - 0x70, 0x65, 0x63, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, - 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x48, 0x00, 0x52, 0x0c, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x12, 0x57, 0x0a, 0x0e, 0x70, 0x61, - 0x72, 0x5f, 0x64, 0x6f, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, - 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, - 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x44, 0x6f, 0x50, 0x61, 0x79, 0x6c, - 0x6f, 0x61, 0x64, 0x48, 0x00, 0x52, 0x0c, 0x70, 0x61, 0x72, 0x44, 0x6f, 0x50, 0x61, 0x79, 0x6c, - 0x6f, 0x61, 0x64, 0x12, 0x4f, 0x0a, 0x0a, 0x70, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, - 0x6d, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, - 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, - 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x54, 0x72, 0x61, - 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x48, 0x00, 0x52, 0x0a, 0x70, 0x74, 0x72, 0x61, 0x6e, 0x73, - 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x52, 0x0a, 0x0b, 0x70, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, - 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x43, - 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x0b, 0x70, 0x63, 0x6f, - 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x53, 0x0a, 0x0c, 0x72, 0x65, 0x61, 0x64, - 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, - 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, - 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x48, 0x00, - 0x52, 0x0b, 0x72, 0x65, 0x61, 0x64, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x4d, 0x0a, - 0x0a, 0x73, 0x69, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x2c, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, - 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, - 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x48, - 0x00, 0x52, 0x09, 0x73, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x66, 0x0a, 0x13, - 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x5f, 0x69, 0x6e, 0x74, 0x6f, 0x5f, 0x70, 0x61, 0x79, 0x6c, - 0x6f, 0x61, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, - 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x69, - 0x6e, 0x64, 0x6f, 0x77, 0x49, 0x6e, 0x74, 0x6f, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x48, - 0x00, 0x52, 0x11, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x49, 0x6e, 0x74, 0x6f, 0x50, 0x61, 0x79, - 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x65, 0x0a, 0x12, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x69, 0x6e, - 0x67, 0x5f, 0x73, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x34, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, + 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x12, 0x7a, 0x0a, 0x13, 0x77, 0x69, 0x72, 0x65, 0x5f, 0x63, 0x6f, 0x64, 0x65, + 0x72, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x4a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x69, 0x6e, 0x67, 0x53, 0x74, - 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x48, 0x00, 0x52, 0x11, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, - 0x69, 0x6e, 0x67, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x42, 0x06, 0x0a, 0x04, 0x72, - 0x6f, 0x6f, 0x74, 0x22, 0xb6, 0x0a, 0x0a, 0x16, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, - 0x6c, 0x65, 0x53, 0x74, 0x61, 0x67, 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x50, - 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, - 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, - 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, - 0x65, 0x6e, 0x74, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, - 0x12, 0x7a, 0x0a, 0x13, 0x77, 0x69, 0x72, 0x65, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x73, - 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4a, 0x2e, - 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, - 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x67, - 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x57, 0x69, 0x72, 0x65, 0x43, 0x6f, 0x64, - 0x65, 0x72, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x11, 0x77, 0x69, 0x72, 0x65, 0x43, - 0x6f, 0x64, 0x65, 0x72, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x14, 0x0a, 0x05, - 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x70, - 0x75, 0x74, 0x12, 0x66, 0x0a, 0x0b, 0x73, 0x69, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, - 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, - 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x67, 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, - 0x61, 0x64, 0x2e, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x49, 0x64, 0x52, 0x0a, - 0x73, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x74, 0x72, - 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, - 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x6f, 0x75, - 0x74, 0x70, 0x75, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x6f, 0x75, 0x74, - 0x70, 0x75, 0x74, 0x73, 0x12, 0x4d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, - 0x74, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, - 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, - 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, - 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, - 0x6e, 0x74, 0x73, 0x12, 0x66, 0x0a, 0x0b, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, - 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, - 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, - 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x67, 0x65, 0x50, 0x61, 0x79, 0x6c, - 0x6f, 0x61, 0x64, 0x2e, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x49, 0x64, 0x52, - 0x0a, 0x75, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x12, 0x59, 0x0a, 0x06, 0x74, - 0x69, 0x6d, 0x65, 0x72, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x6f, 0x72, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, + 0x74, 0x61, 0x67, 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x57, 0x69, 0x72, 0x65, + 0x43, 0x6f, 0x64, 0x65, 0x72, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x11, 0x77, 0x69, + 0x72, 0x65, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, + 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x69, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x66, 0x0a, 0x0b, 0x73, 0x69, 0x64, 0x65, 0x5f, 0x69, 0x6e, + 0x70, 0x75, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x6f, 0x72, 0x67, + 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x67, 0x65, 0x50, 0x61, + 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x49, + 0x64, 0x52, 0x0a, 0x73, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x1e, 0x0a, + 0x0a, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x0a, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x12, 0x18, 0x0a, + 0x07, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, + 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, 0x12, 0x4d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x6f, + 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x6f, 0x72, + 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, + 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, + 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x0a, 0x63, 0x6f, 0x6d, 0x70, + 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x66, 0x0a, 0x0b, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x67, 0x65, 0x50, - 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x49, 0x64, 0x52, 0x06, - 0x74, 0x69, 0x6d, 0x65, 0x72, 0x73, 0x12, 0x6d, 0x0a, 0x0d, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x46, - 0x61, 0x6d, 0x69, 0x6c, 0x69, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x47, 0x2e, - 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, - 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x67, - 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x46, 0x61, - 0x6d, 0x69, 0x6c, 0x79, 0x49, 0x64, 0x52, 0x0d, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x46, 0x61, 0x6d, - 0x69, 0x6c, 0x69, 0x65, 0x73, 0x1a, 0x4f, 0x0a, 0x0b, 0x53, 0x69, 0x64, 0x65, 0x49, 0x6e, 0x70, - 0x75, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, - 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, - 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x6f, 0x63, 0x61, 0x6c, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6c, 0x6f, 0x63, - 0x61, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x1a, 0x4f, 0x0a, 0x0b, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, - 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, - 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x6f, 0x63, 0x61, - 0x6c, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6c, 0x6f, - 0x63, 0x61, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x1a, 0x4b, 0x0a, 0x07, 0x54, 0x69, 0x6d, 0x65, 0x72, - 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, - 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x6c, - 0x4e, 0x61, 0x6d, 0x65, 0x1a, 0x51, 0x0a, 0x0d, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x46, 0x61, 0x6d, - 0x69, 0x6c, 0x79, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x49, 0x64, 0x52, 0x0a, 0x75, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x12, 0x59, + 0x0a, 0x06, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x41, + 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, + 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, + 0x76, 0x31, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x74, 0x61, + 0x67, 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x49, + 0x64, 0x52, 0x06, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x73, 0x12, 0x6d, 0x0a, 0x0d, 0x74, 0x69, 0x6d, + 0x65, 0x72, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x69, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x47, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, + 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, + 0x74, 0x61, 0x67, 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x72, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x49, 0x64, 0x52, 0x0d, 0x74, 0x69, 0x6d, 0x65, 0x72, + 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x69, 0x65, 0x73, 0x1a, 0x4f, 0x0a, 0x0b, 0x53, 0x69, 0x64, 0x65, + 0x49, 0x6e, 0x70, 0x75, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, + 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x1a, 0x4f, 0x0a, 0x0b, 0x55, 0x73, 0x65, + 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, + 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, + 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x1a, 0x4b, 0x0a, 0x07, 0x54, 0x69, + 0x6d, 0x65, 0x72, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6c, 0x6f, - 0x63, 0x61, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x1a, 0xd2, 0x01, 0x0a, 0x10, 0x57, 0x69, 0x72, 0x65, - 0x43, 0x6f, 0x64, 0x65, 0x72, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x12, 0x10, 0x0a, 0x03, - 0x75, 0x72, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6e, 0x12, 0x18, - 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x2d, 0x0a, 0x12, 0x69, 0x6e, 0x70, 0x75, - 0x74, 0x5f, 0x6f, 0x72, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x4f, 0x72, 0x4f, - 0x75, 0x74, 0x70, 0x75, 0x74, 0x49, 0x64, 0x12, 0x59, 0x0a, 0x05, 0x74, 0x69, 0x6d, 0x65, 0x72, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, + 0x63, 0x61, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x1a, 0x51, 0x0a, 0x0d, 0x54, 0x69, 0x6d, 0x65, 0x72, + 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x61, 0x6e, + 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, + 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x1a, 0xd2, 0x01, 0x0a, 0x10, 0x57, + 0x69, 0x72, 0x65, 0x43, 0x6f, 0x64, 0x65, 0x72, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x12, + 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, + 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x2d, 0x0a, 0x12, 0x69, + 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x6f, 0x72, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x69, + 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0f, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x4f, 0x72, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x49, 0x64, 0x12, 0x59, 0x0a, 0x05, 0x74, 0x69, + 0x6d, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x6f, 0x72, 0x67, 0x2e, + 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x67, 0x65, 0x50, 0x61, 0x79, + 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x49, 0x64, 0x48, 0x00, 0x52, 0x05, + 0x74, 0x69, 0x6d, 0x65, 0x72, 0x42, 0x08, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, + 0xc2, 0x01, 0x0a, 0x15, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x48, 0x69, 0x6e, 0x74, 0x73, 0x22, 0xa8, 0x01, 0x0a, 0x04, 0x45, 0x6e, + 0x75, 0x6d, 0x12, 0x34, 0x0a, 0x0b, 0x41, 0x43, 0x43, 0x45, 0x4c, 0x45, 0x52, 0x41, 0x54, 0x4f, + 0x52, 0x10, 0x00, 0x1a, 0x23, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1d, 0x62, 0x65, 0x61, 0x6d, 0x3a, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x3a, 0x61, 0x63, 0x63, 0x65, 0x6c, 0x65, + 0x72, 0x61, 0x74, 0x6f, 0x72, 0x3a, 0x76, 0x31, 0x12, 0x38, 0x0a, 0x0d, 0x4d, 0x49, 0x4e, 0x5f, + 0x52, 0x41, 0x4d, 0x5f, 0x42, 0x59, 0x54, 0x45, 0x53, 0x10, 0x01, 0x1a, 0x25, 0xa2, 0xb4, 0xfa, + 0xc2, 0x05, 0x1f, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x73, 0x3a, 0x6d, 0x69, 0x6e, 0x5f, 0x72, 0x61, 0x6d, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x3a, + 0x76, 0x31, 0x12, 0x30, 0x0a, 0x09, 0x43, 0x50, 0x55, 0x5f, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x10, + 0x02, 0x1a, 0x21, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1b, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x72, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x3a, 0x63, 0x70, 0x75, 0x5f, 0x63, 0x6f, 0x75, 0x6e, + 0x74, 0x3a, 0x76, 0x31, 0x32, 0x8f, 0x01, 0x0a, 0x11, 0x54, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x7a, 0x0a, 0x06, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x73, 0x12, 0x30, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, + 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, + 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, - 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x67, 0x65, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x49, 0x64, 0x48, 0x00, 0x52, 0x05, 0x74, 0x69, 0x6d, - 0x65, 0x72, 0x42, 0x08, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0x8f, 0x01, 0x0a, - 0x15, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x48, 0x69, 0x6e, 0x74, 0x73, 0x22, 0x76, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x34, - 0x0a, 0x0b, 0x41, 0x43, 0x43, 0x45, 0x4c, 0x45, 0x52, 0x41, 0x54, 0x4f, 0x52, 0x10, 0x00, 0x1a, - 0x23, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1d, 0x62, 0x65, 0x61, 0x6d, 0x3a, 0x72, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x73, 0x3a, 0x61, 0x63, 0x63, 0x65, 0x6c, 0x65, 0x72, 0x61, 0x74, 0x6f, - 0x72, 0x3a, 0x76, 0x31, 0x12, 0x38, 0x0a, 0x0d, 0x4d, 0x49, 0x4e, 0x5f, 0x52, 0x41, 0x4d, 0x5f, - 0x42, 0x59, 0x54, 0x45, 0x53, 0x10, 0x01, 0x1a, 0x25, 0xa2, 0xb4, 0xfa, 0xc2, 0x05, 0x1f, 0x62, - 0x65, 0x61, 0x6d, 0x3a, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x3a, 0x6d, 0x69, - 0x6e, 0x5f, 0x72, 0x61, 0x6d, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x3a, 0x76, 0x31, 0x32, 0x8f, - 0x01, 0x0a, 0x11, 0x54, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x12, 0x7a, 0x0a, 0x06, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x30, - 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, - 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x3a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, - 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x50, - 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x22, 0x00, 0x30, 0x01, - 0x3a, 0x3f, 0x0a, 0x08, 0x62, 0x65, 0x61, 0x6d, 0x5f, 0x75, 0x72, 0x6e, 0x12, 0x21, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6e, 0x75, 0x6d, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, - 0xc4, 0xa6, 0xaf, 0x58, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x65, 0x61, 0x6d, 0x55, 0x72, - 0x6e, 0x3a, 0x49, 0x0a, 0x0d, 0x62, 0x65, 0x61, 0x6d, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x61, - 0x6e, 0x74, 0x12, 0x21, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x53, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x22, 0x00, 0x30, 0x01, 0x3a, 0x3f, 0x0a, 0x08, 0x62, 0x65, 0x61, 0x6d, 0x5f, 0x75, + 0x72, 0x6e, 0x12, 0x21, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0xc5, 0xa6, 0xaf, 0x58, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, - 0x62, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x42, 0x78, 0x0a, 0x21, - 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, - 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, - 0x31, 0x42, 0x09, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x41, 0x70, 0x69, 0x5a, 0x48, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2f, - 0x62, 0x65, 0x61, 0x6d, 0x2f, 0x73, 0x64, 0x6b, 0x73, 0x2f, 0x76, 0x32, 0x2f, 0x67, 0x6f, 0x2f, - 0x70, 0x6b, 0x67, 0x2f, 0x62, 0x65, 0x61, 0x6d, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2f, 0x70, - 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x76, 0x31, 0x3b, 0x70, 0x69, 0x70, 0x65, 0x6c, - 0x69, 0x6e, 0x65, 0x5f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0xc4, 0xa6, 0xaf, 0x58, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x62, 0x65, 0x61, 0x6d, 0x55, 0x72, 0x6e, 0x3a, 0x49, 0x0a, 0x0d, 0x62, 0x65, 0x61, 0x6d, 0x5f, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x12, 0x21, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0xc5, 0xa6, 0xaf, 0x58, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x62, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x61, + 0x6e, 0x74, 0x42, 0x78, 0x0a, 0x21, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, + 0x2e, 0x62, 0x65, 0x61, 0x6d, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x69, 0x70, 0x65, + 0x6c, 0x69, 0x6e, 0x65, 0x2e, 0x76, 0x31, 0x42, 0x09, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x41, + 0x70, 0x69, 0x5a, 0x48, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, + 0x70, 0x61, 0x63, 0x68, 0x65, 0x2f, 0x62, 0x65, 0x61, 0x6d, 0x2f, 0x73, 0x64, 0x6b, 0x73, 0x2f, + 0x76, 0x32, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x62, 0x65, 0x61, 0x6d, 0x2f, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0x2f, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x76, 0x31, + 0x3b, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -9184,7 +9283,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDescGZIP() } var file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_enumTypes = make([]protoimpl.EnumInfo, 25) -var file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes = make([]protoimpl.MessageInfo, 102) +var file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes = make([]protoimpl.MessageInfo, 103) var file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_goTypes = []interface{}{ (BeamConstants_Constants)(0), // 0: org.apache.beam.model.pipeline.v1.BeamConstants.Constants (StandardPTransforms_Primitives)(0), // 1: org.apache.beam.model.pipeline.v1.StandardPTransforms.Primitives @@ -9226,214 +9325,216 @@ var file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_goTypes = []int (*OrderedListStateSpec)(nil), // 37: org.apache.beam.model.pipeline.v1.OrderedListStateSpec (*CombiningStateSpec)(nil), // 38: org.apache.beam.model.pipeline.v1.CombiningStateSpec (*MapStateSpec)(nil), // 39: org.apache.beam.model.pipeline.v1.MapStateSpec - (*SetStateSpec)(nil), // 40: org.apache.beam.model.pipeline.v1.SetStateSpec - (*TimerFamilySpec)(nil), // 41: org.apache.beam.model.pipeline.v1.TimerFamilySpec - (*IsBounded)(nil), // 42: org.apache.beam.model.pipeline.v1.IsBounded - (*ReadPayload)(nil), // 43: org.apache.beam.model.pipeline.v1.ReadPayload - (*WindowIntoPayload)(nil), // 44: org.apache.beam.model.pipeline.v1.WindowIntoPayload - (*CombinePayload)(nil), // 45: org.apache.beam.model.pipeline.v1.CombinePayload - (*TestStreamPayload)(nil), // 46: org.apache.beam.model.pipeline.v1.TestStreamPayload - (*EventsRequest)(nil), // 47: org.apache.beam.model.pipeline.v1.EventsRequest - (*WriteFilesPayload)(nil), // 48: org.apache.beam.model.pipeline.v1.WriteFilesPayload - (*PubSubReadPayload)(nil), // 49: org.apache.beam.model.pipeline.v1.PubSubReadPayload - (*PubSubWritePayload)(nil), // 50: org.apache.beam.model.pipeline.v1.PubSubWritePayload - (*GroupIntoBatchesPayload)(nil), // 51: org.apache.beam.model.pipeline.v1.GroupIntoBatchesPayload - (*Coder)(nil), // 52: org.apache.beam.model.pipeline.v1.Coder - (*StandardCoders)(nil), // 53: org.apache.beam.model.pipeline.v1.StandardCoders - (*WindowingStrategy)(nil), // 54: org.apache.beam.model.pipeline.v1.WindowingStrategy - (*MergeStatus)(nil), // 55: org.apache.beam.model.pipeline.v1.MergeStatus - (*AccumulationMode)(nil), // 56: org.apache.beam.model.pipeline.v1.AccumulationMode - (*ClosingBehavior)(nil), // 57: org.apache.beam.model.pipeline.v1.ClosingBehavior - (*OnTimeBehavior)(nil), // 58: org.apache.beam.model.pipeline.v1.OnTimeBehavior - (*OutputTime)(nil), // 59: org.apache.beam.model.pipeline.v1.OutputTime - (*TimeDomain)(nil), // 60: org.apache.beam.model.pipeline.v1.TimeDomain - (*Trigger)(nil), // 61: org.apache.beam.model.pipeline.v1.Trigger - (*TimestampTransform)(nil), // 62: org.apache.beam.model.pipeline.v1.TimestampTransform - (*SideInput)(nil), // 63: org.apache.beam.model.pipeline.v1.SideInput - (*StandardArtifacts)(nil), // 64: org.apache.beam.model.pipeline.v1.StandardArtifacts - (*ArtifactFilePayload)(nil), // 65: org.apache.beam.model.pipeline.v1.ArtifactFilePayload - (*ArtifactUrlPayload)(nil), // 66: org.apache.beam.model.pipeline.v1.ArtifactUrlPayload - (*EmbeddedFilePayload)(nil), // 67: org.apache.beam.model.pipeline.v1.EmbeddedFilePayload - (*PyPIPayload)(nil), // 68: org.apache.beam.model.pipeline.v1.PyPIPayload - (*MavenPayload)(nil), // 69: org.apache.beam.model.pipeline.v1.MavenPayload - (*DeferredArtifactPayload)(nil), // 70: org.apache.beam.model.pipeline.v1.DeferredArtifactPayload - (*ArtifactStagingToRolePayload)(nil), // 71: org.apache.beam.model.pipeline.v1.ArtifactStagingToRolePayload - (*ArtifactInformation)(nil), // 72: org.apache.beam.model.pipeline.v1.ArtifactInformation - (*Environment)(nil), // 73: org.apache.beam.model.pipeline.v1.Environment - (*StandardEnvironments)(nil), // 74: org.apache.beam.model.pipeline.v1.StandardEnvironments - (*DockerPayload)(nil), // 75: org.apache.beam.model.pipeline.v1.DockerPayload - (*ProcessPayload)(nil), // 76: org.apache.beam.model.pipeline.v1.ProcessPayload - (*ExternalPayload)(nil), // 77: org.apache.beam.model.pipeline.v1.ExternalPayload - (*StandardProtocols)(nil), // 78: org.apache.beam.model.pipeline.v1.StandardProtocols - (*StandardRunnerProtocols)(nil), // 79: org.apache.beam.model.pipeline.v1.StandardRunnerProtocols - (*StandardRequirements)(nil), // 80: org.apache.beam.model.pipeline.v1.StandardRequirements - (*FunctionSpec)(nil), // 81: org.apache.beam.model.pipeline.v1.FunctionSpec - (*StandardDisplayData)(nil), // 82: org.apache.beam.model.pipeline.v1.StandardDisplayData - (*LabelledPayload)(nil), // 83: org.apache.beam.model.pipeline.v1.LabelledPayload - (*DisplayData)(nil), // 84: org.apache.beam.model.pipeline.v1.DisplayData - (*MessageWithComponents)(nil), // 85: org.apache.beam.model.pipeline.v1.MessageWithComponents - (*ExecutableStagePayload)(nil), // 86: org.apache.beam.model.pipeline.v1.ExecutableStagePayload - (*StandardResourceHints)(nil), // 87: org.apache.beam.model.pipeline.v1.StandardResourceHints - nil, // 88: org.apache.beam.model.pipeline.v1.Components.TransformsEntry - nil, // 89: org.apache.beam.model.pipeline.v1.Components.PcollectionsEntry - nil, // 90: org.apache.beam.model.pipeline.v1.Components.WindowingStrategiesEntry - nil, // 91: org.apache.beam.model.pipeline.v1.Components.CodersEntry - nil, // 92: org.apache.beam.model.pipeline.v1.Components.EnvironmentsEntry - nil, // 93: org.apache.beam.model.pipeline.v1.PTransform.InputsEntry - nil, // 94: org.apache.beam.model.pipeline.v1.PTransform.OutputsEntry - nil, // 95: org.apache.beam.model.pipeline.v1.PTransform.AnnotationsEntry - nil, // 96: org.apache.beam.model.pipeline.v1.ParDoPayload.SideInputsEntry - nil, // 97: org.apache.beam.model.pipeline.v1.ParDoPayload.StateSpecsEntry - nil, // 98: org.apache.beam.model.pipeline.v1.ParDoPayload.TimerFamilySpecsEntry - (*TestStreamPayload_Event)(nil), // 99: org.apache.beam.model.pipeline.v1.TestStreamPayload.Event - (*TestStreamPayload_TimestampedElement)(nil), // 100: org.apache.beam.model.pipeline.v1.TestStreamPayload.TimestampedElement - (*TestStreamPayload_Event_AdvanceWatermark)(nil), // 101: org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.AdvanceWatermark - (*TestStreamPayload_Event_AdvanceProcessingTime)(nil), // 102: org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.AdvanceProcessingTime - (*TestStreamPayload_Event_AddElements)(nil), // 103: org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.AddElements - nil, // 104: org.apache.beam.model.pipeline.v1.WriteFilesPayload.SideInputsEntry - (*Trigger_AfterAll)(nil), // 105: org.apache.beam.model.pipeline.v1.Trigger.AfterAll - (*Trigger_AfterAny)(nil), // 106: org.apache.beam.model.pipeline.v1.Trigger.AfterAny - (*Trigger_AfterEach)(nil), // 107: org.apache.beam.model.pipeline.v1.Trigger.AfterEach - (*Trigger_AfterEndOfWindow)(nil), // 108: org.apache.beam.model.pipeline.v1.Trigger.AfterEndOfWindow - (*Trigger_AfterProcessingTime)(nil), // 109: org.apache.beam.model.pipeline.v1.Trigger.AfterProcessingTime - (*Trigger_AfterSynchronizedProcessingTime)(nil), // 110: org.apache.beam.model.pipeline.v1.Trigger.AfterSynchronizedProcessingTime - (*Trigger_Default)(nil), // 111: org.apache.beam.model.pipeline.v1.Trigger.Default - (*Trigger_ElementCount)(nil), // 112: org.apache.beam.model.pipeline.v1.Trigger.ElementCount - (*Trigger_Never)(nil), // 113: org.apache.beam.model.pipeline.v1.Trigger.Never - (*Trigger_Always)(nil), // 114: org.apache.beam.model.pipeline.v1.Trigger.Always - (*Trigger_OrFinally)(nil), // 115: org.apache.beam.model.pipeline.v1.Trigger.OrFinally - (*Trigger_Repeat)(nil), // 116: org.apache.beam.model.pipeline.v1.Trigger.Repeat - (*TimestampTransform_Delay)(nil), // 117: org.apache.beam.model.pipeline.v1.TimestampTransform.Delay - (*TimestampTransform_AlignTo)(nil), // 118: org.apache.beam.model.pipeline.v1.TimestampTransform.AlignTo - nil, // 119: org.apache.beam.model.pipeline.v1.Environment.ResourceHintsEntry - nil, // 120: org.apache.beam.model.pipeline.v1.ProcessPayload.EnvEntry - nil, // 121: org.apache.beam.model.pipeline.v1.ExternalPayload.ParamsEntry - (*ExecutableStagePayload_SideInputId)(nil), // 122: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.SideInputId - (*ExecutableStagePayload_UserStateId)(nil), // 123: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.UserStateId - (*ExecutableStagePayload_TimerId)(nil), // 124: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.TimerId - (*ExecutableStagePayload_TimerFamilyId)(nil), // 125: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.TimerFamilyId - (*ExecutableStagePayload_WireCoderSetting)(nil), // 126: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.WireCoderSetting - (*ApiServiceDescriptor)(nil), // 127: org.apache.beam.model.pipeline.v1.ApiServiceDescriptor - (*descriptorpb.EnumValueOptions)(nil), // 128: google.protobuf.EnumValueOptions + (*MultimapStateSpec)(nil), // 40: org.apache.beam.model.pipeline.v1.MultimapStateSpec + (*SetStateSpec)(nil), // 41: org.apache.beam.model.pipeline.v1.SetStateSpec + (*TimerFamilySpec)(nil), // 42: org.apache.beam.model.pipeline.v1.TimerFamilySpec + (*IsBounded)(nil), // 43: org.apache.beam.model.pipeline.v1.IsBounded + (*ReadPayload)(nil), // 44: org.apache.beam.model.pipeline.v1.ReadPayload + (*WindowIntoPayload)(nil), // 45: org.apache.beam.model.pipeline.v1.WindowIntoPayload + (*CombinePayload)(nil), // 46: org.apache.beam.model.pipeline.v1.CombinePayload + (*TestStreamPayload)(nil), // 47: org.apache.beam.model.pipeline.v1.TestStreamPayload + (*EventsRequest)(nil), // 48: org.apache.beam.model.pipeline.v1.EventsRequest + (*WriteFilesPayload)(nil), // 49: org.apache.beam.model.pipeline.v1.WriteFilesPayload + (*PubSubReadPayload)(nil), // 50: org.apache.beam.model.pipeline.v1.PubSubReadPayload + (*PubSubWritePayload)(nil), // 51: org.apache.beam.model.pipeline.v1.PubSubWritePayload + (*GroupIntoBatchesPayload)(nil), // 52: org.apache.beam.model.pipeline.v1.GroupIntoBatchesPayload + (*Coder)(nil), // 53: org.apache.beam.model.pipeline.v1.Coder + (*StandardCoders)(nil), // 54: org.apache.beam.model.pipeline.v1.StandardCoders + (*WindowingStrategy)(nil), // 55: org.apache.beam.model.pipeline.v1.WindowingStrategy + (*MergeStatus)(nil), // 56: org.apache.beam.model.pipeline.v1.MergeStatus + (*AccumulationMode)(nil), // 57: org.apache.beam.model.pipeline.v1.AccumulationMode + (*ClosingBehavior)(nil), // 58: org.apache.beam.model.pipeline.v1.ClosingBehavior + (*OnTimeBehavior)(nil), // 59: org.apache.beam.model.pipeline.v1.OnTimeBehavior + (*OutputTime)(nil), // 60: org.apache.beam.model.pipeline.v1.OutputTime + (*TimeDomain)(nil), // 61: org.apache.beam.model.pipeline.v1.TimeDomain + (*Trigger)(nil), // 62: org.apache.beam.model.pipeline.v1.Trigger + (*TimestampTransform)(nil), // 63: org.apache.beam.model.pipeline.v1.TimestampTransform + (*SideInput)(nil), // 64: org.apache.beam.model.pipeline.v1.SideInput + (*StandardArtifacts)(nil), // 65: org.apache.beam.model.pipeline.v1.StandardArtifacts + (*ArtifactFilePayload)(nil), // 66: org.apache.beam.model.pipeline.v1.ArtifactFilePayload + (*ArtifactUrlPayload)(nil), // 67: org.apache.beam.model.pipeline.v1.ArtifactUrlPayload + (*EmbeddedFilePayload)(nil), // 68: org.apache.beam.model.pipeline.v1.EmbeddedFilePayload + (*PyPIPayload)(nil), // 69: org.apache.beam.model.pipeline.v1.PyPIPayload + (*MavenPayload)(nil), // 70: org.apache.beam.model.pipeline.v1.MavenPayload + (*DeferredArtifactPayload)(nil), // 71: org.apache.beam.model.pipeline.v1.DeferredArtifactPayload + (*ArtifactStagingToRolePayload)(nil), // 72: org.apache.beam.model.pipeline.v1.ArtifactStagingToRolePayload + (*ArtifactInformation)(nil), // 73: org.apache.beam.model.pipeline.v1.ArtifactInformation + (*Environment)(nil), // 74: org.apache.beam.model.pipeline.v1.Environment + (*StandardEnvironments)(nil), // 75: org.apache.beam.model.pipeline.v1.StandardEnvironments + (*DockerPayload)(nil), // 76: org.apache.beam.model.pipeline.v1.DockerPayload + (*ProcessPayload)(nil), // 77: org.apache.beam.model.pipeline.v1.ProcessPayload + (*ExternalPayload)(nil), // 78: org.apache.beam.model.pipeline.v1.ExternalPayload + (*StandardProtocols)(nil), // 79: org.apache.beam.model.pipeline.v1.StandardProtocols + (*StandardRunnerProtocols)(nil), // 80: org.apache.beam.model.pipeline.v1.StandardRunnerProtocols + (*StandardRequirements)(nil), // 81: org.apache.beam.model.pipeline.v1.StandardRequirements + (*FunctionSpec)(nil), // 82: org.apache.beam.model.pipeline.v1.FunctionSpec + (*StandardDisplayData)(nil), // 83: org.apache.beam.model.pipeline.v1.StandardDisplayData + (*LabelledPayload)(nil), // 84: org.apache.beam.model.pipeline.v1.LabelledPayload + (*DisplayData)(nil), // 85: org.apache.beam.model.pipeline.v1.DisplayData + (*MessageWithComponents)(nil), // 86: org.apache.beam.model.pipeline.v1.MessageWithComponents + (*ExecutableStagePayload)(nil), // 87: org.apache.beam.model.pipeline.v1.ExecutableStagePayload + (*StandardResourceHints)(nil), // 88: org.apache.beam.model.pipeline.v1.StandardResourceHints + nil, // 89: org.apache.beam.model.pipeline.v1.Components.TransformsEntry + nil, // 90: org.apache.beam.model.pipeline.v1.Components.PcollectionsEntry + nil, // 91: org.apache.beam.model.pipeline.v1.Components.WindowingStrategiesEntry + nil, // 92: org.apache.beam.model.pipeline.v1.Components.CodersEntry + nil, // 93: org.apache.beam.model.pipeline.v1.Components.EnvironmentsEntry + nil, // 94: org.apache.beam.model.pipeline.v1.PTransform.InputsEntry + nil, // 95: org.apache.beam.model.pipeline.v1.PTransform.OutputsEntry + nil, // 96: org.apache.beam.model.pipeline.v1.PTransform.AnnotationsEntry + nil, // 97: org.apache.beam.model.pipeline.v1.ParDoPayload.SideInputsEntry + nil, // 98: org.apache.beam.model.pipeline.v1.ParDoPayload.StateSpecsEntry + nil, // 99: org.apache.beam.model.pipeline.v1.ParDoPayload.TimerFamilySpecsEntry + (*TestStreamPayload_Event)(nil), // 100: org.apache.beam.model.pipeline.v1.TestStreamPayload.Event + (*TestStreamPayload_TimestampedElement)(nil), // 101: org.apache.beam.model.pipeline.v1.TestStreamPayload.TimestampedElement + (*TestStreamPayload_Event_AdvanceWatermark)(nil), // 102: org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.AdvanceWatermark + (*TestStreamPayload_Event_AdvanceProcessingTime)(nil), // 103: org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.AdvanceProcessingTime + (*TestStreamPayload_Event_AddElements)(nil), // 104: org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.AddElements + nil, // 105: org.apache.beam.model.pipeline.v1.WriteFilesPayload.SideInputsEntry + (*Trigger_AfterAll)(nil), // 106: org.apache.beam.model.pipeline.v1.Trigger.AfterAll + (*Trigger_AfterAny)(nil), // 107: org.apache.beam.model.pipeline.v1.Trigger.AfterAny + (*Trigger_AfterEach)(nil), // 108: org.apache.beam.model.pipeline.v1.Trigger.AfterEach + (*Trigger_AfterEndOfWindow)(nil), // 109: org.apache.beam.model.pipeline.v1.Trigger.AfterEndOfWindow + (*Trigger_AfterProcessingTime)(nil), // 110: org.apache.beam.model.pipeline.v1.Trigger.AfterProcessingTime + (*Trigger_AfterSynchronizedProcessingTime)(nil), // 111: org.apache.beam.model.pipeline.v1.Trigger.AfterSynchronizedProcessingTime + (*Trigger_Default)(nil), // 112: org.apache.beam.model.pipeline.v1.Trigger.Default + (*Trigger_ElementCount)(nil), // 113: org.apache.beam.model.pipeline.v1.Trigger.ElementCount + (*Trigger_Never)(nil), // 114: org.apache.beam.model.pipeline.v1.Trigger.Never + (*Trigger_Always)(nil), // 115: org.apache.beam.model.pipeline.v1.Trigger.Always + (*Trigger_OrFinally)(nil), // 116: org.apache.beam.model.pipeline.v1.Trigger.OrFinally + (*Trigger_Repeat)(nil), // 117: org.apache.beam.model.pipeline.v1.Trigger.Repeat + (*TimestampTransform_Delay)(nil), // 118: org.apache.beam.model.pipeline.v1.TimestampTransform.Delay + (*TimestampTransform_AlignTo)(nil), // 119: org.apache.beam.model.pipeline.v1.TimestampTransform.AlignTo + nil, // 120: org.apache.beam.model.pipeline.v1.Environment.ResourceHintsEntry + nil, // 121: org.apache.beam.model.pipeline.v1.ProcessPayload.EnvEntry + nil, // 122: org.apache.beam.model.pipeline.v1.ExternalPayload.ParamsEntry + (*ExecutableStagePayload_SideInputId)(nil), // 123: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.SideInputId + (*ExecutableStagePayload_UserStateId)(nil), // 124: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.UserStateId + (*ExecutableStagePayload_TimerId)(nil), // 125: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.TimerId + (*ExecutableStagePayload_TimerFamilyId)(nil), // 126: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.TimerFamilyId + (*ExecutableStagePayload_WireCoderSetting)(nil), // 127: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.WireCoderSetting + (*ApiServiceDescriptor)(nil), // 128: org.apache.beam.model.pipeline.v1.ApiServiceDescriptor + (*descriptorpb.EnumValueOptions)(nil), // 129: google.protobuf.EnumValueOptions } var file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_depIdxs = []int32{ - 88, // 0: org.apache.beam.model.pipeline.v1.Components.transforms:type_name -> org.apache.beam.model.pipeline.v1.Components.TransformsEntry - 89, // 1: org.apache.beam.model.pipeline.v1.Components.pcollections:type_name -> org.apache.beam.model.pipeline.v1.Components.PcollectionsEntry - 90, // 2: org.apache.beam.model.pipeline.v1.Components.windowing_strategies:type_name -> org.apache.beam.model.pipeline.v1.Components.WindowingStrategiesEntry - 91, // 3: org.apache.beam.model.pipeline.v1.Components.coders:type_name -> org.apache.beam.model.pipeline.v1.Components.CodersEntry - 92, // 4: org.apache.beam.model.pipeline.v1.Components.environments:type_name -> org.apache.beam.model.pipeline.v1.Components.EnvironmentsEntry + 89, // 0: org.apache.beam.model.pipeline.v1.Components.transforms:type_name -> org.apache.beam.model.pipeline.v1.Components.TransformsEntry + 90, // 1: org.apache.beam.model.pipeline.v1.Components.pcollections:type_name -> org.apache.beam.model.pipeline.v1.Components.PcollectionsEntry + 91, // 2: org.apache.beam.model.pipeline.v1.Components.windowing_strategies:type_name -> org.apache.beam.model.pipeline.v1.Components.WindowingStrategiesEntry + 92, // 3: org.apache.beam.model.pipeline.v1.Components.coders:type_name -> org.apache.beam.model.pipeline.v1.Components.CodersEntry + 93, // 4: org.apache.beam.model.pipeline.v1.Components.environments:type_name -> org.apache.beam.model.pipeline.v1.Components.EnvironmentsEntry 26, // 5: org.apache.beam.model.pipeline.v1.Pipeline.components:type_name -> org.apache.beam.model.pipeline.v1.Components - 84, // 6: org.apache.beam.model.pipeline.v1.Pipeline.display_data:type_name -> org.apache.beam.model.pipeline.v1.DisplayData - 81, // 7: org.apache.beam.model.pipeline.v1.PTransform.spec:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec - 93, // 8: org.apache.beam.model.pipeline.v1.PTransform.inputs:type_name -> org.apache.beam.model.pipeline.v1.PTransform.InputsEntry - 94, // 9: org.apache.beam.model.pipeline.v1.PTransform.outputs:type_name -> org.apache.beam.model.pipeline.v1.PTransform.OutputsEntry - 84, // 10: org.apache.beam.model.pipeline.v1.PTransform.display_data:type_name -> org.apache.beam.model.pipeline.v1.DisplayData - 95, // 11: org.apache.beam.model.pipeline.v1.PTransform.annotations:type_name -> org.apache.beam.model.pipeline.v1.PTransform.AnnotationsEntry + 85, // 6: org.apache.beam.model.pipeline.v1.Pipeline.display_data:type_name -> org.apache.beam.model.pipeline.v1.DisplayData + 82, // 7: org.apache.beam.model.pipeline.v1.PTransform.spec:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec + 94, // 8: org.apache.beam.model.pipeline.v1.PTransform.inputs:type_name -> org.apache.beam.model.pipeline.v1.PTransform.InputsEntry + 95, // 9: org.apache.beam.model.pipeline.v1.PTransform.outputs:type_name -> org.apache.beam.model.pipeline.v1.PTransform.OutputsEntry + 85, // 10: org.apache.beam.model.pipeline.v1.PTransform.display_data:type_name -> org.apache.beam.model.pipeline.v1.DisplayData + 96, // 11: org.apache.beam.model.pipeline.v1.PTransform.annotations:type_name -> org.apache.beam.model.pipeline.v1.PTransform.AnnotationsEntry 9, // 12: org.apache.beam.model.pipeline.v1.PCollection.is_bounded:type_name -> org.apache.beam.model.pipeline.v1.IsBounded.Enum - 84, // 13: org.apache.beam.model.pipeline.v1.PCollection.display_data:type_name -> org.apache.beam.model.pipeline.v1.DisplayData - 81, // 14: org.apache.beam.model.pipeline.v1.ParDoPayload.do_fn:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec - 96, // 15: org.apache.beam.model.pipeline.v1.ParDoPayload.side_inputs:type_name -> org.apache.beam.model.pipeline.v1.ParDoPayload.SideInputsEntry - 97, // 16: org.apache.beam.model.pipeline.v1.ParDoPayload.state_specs:type_name -> org.apache.beam.model.pipeline.v1.ParDoPayload.StateSpecsEntry - 98, // 17: org.apache.beam.model.pipeline.v1.ParDoPayload.timer_family_specs:type_name -> org.apache.beam.model.pipeline.v1.ParDoPayload.TimerFamilySpecsEntry + 85, // 13: org.apache.beam.model.pipeline.v1.PCollection.display_data:type_name -> org.apache.beam.model.pipeline.v1.DisplayData + 82, // 14: org.apache.beam.model.pipeline.v1.ParDoPayload.do_fn:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec + 97, // 15: org.apache.beam.model.pipeline.v1.ParDoPayload.side_inputs:type_name -> org.apache.beam.model.pipeline.v1.ParDoPayload.SideInputsEntry + 98, // 16: org.apache.beam.model.pipeline.v1.ParDoPayload.state_specs:type_name -> org.apache.beam.model.pipeline.v1.ParDoPayload.StateSpecsEntry + 99, // 17: org.apache.beam.model.pipeline.v1.ParDoPayload.timer_family_specs:type_name -> org.apache.beam.model.pipeline.v1.ParDoPayload.TimerFamilySpecsEntry 35, // 18: org.apache.beam.model.pipeline.v1.StateSpec.read_modify_write_spec:type_name -> org.apache.beam.model.pipeline.v1.ReadModifyWriteStateSpec 36, // 19: org.apache.beam.model.pipeline.v1.StateSpec.bag_spec:type_name -> org.apache.beam.model.pipeline.v1.BagStateSpec 38, // 20: org.apache.beam.model.pipeline.v1.StateSpec.combining_spec:type_name -> org.apache.beam.model.pipeline.v1.CombiningStateSpec 39, // 21: org.apache.beam.model.pipeline.v1.StateSpec.map_spec:type_name -> org.apache.beam.model.pipeline.v1.MapStateSpec - 40, // 22: org.apache.beam.model.pipeline.v1.StateSpec.set_spec:type_name -> org.apache.beam.model.pipeline.v1.SetStateSpec + 41, // 22: org.apache.beam.model.pipeline.v1.StateSpec.set_spec:type_name -> org.apache.beam.model.pipeline.v1.SetStateSpec 37, // 23: org.apache.beam.model.pipeline.v1.StateSpec.ordered_list_spec:type_name -> org.apache.beam.model.pipeline.v1.OrderedListStateSpec - 81, // 24: org.apache.beam.model.pipeline.v1.StateSpec.protocol:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec - 81, // 25: org.apache.beam.model.pipeline.v1.CombiningStateSpec.combine_fn:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec - 16, // 26: org.apache.beam.model.pipeline.v1.TimerFamilySpec.time_domain:type_name -> org.apache.beam.model.pipeline.v1.TimeDomain.Enum - 81, // 27: org.apache.beam.model.pipeline.v1.ReadPayload.source:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec - 9, // 28: org.apache.beam.model.pipeline.v1.ReadPayload.is_bounded:type_name -> org.apache.beam.model.pipeline.v1.IsBounded.Enum - 81, // 29: org.apache.beam.model.pipeline.v1.WindowIntoPayload.window_fn:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec - 81, // 30: org.apache.beam.model.pipeline.v1.CombinePayload.combine_fn:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec - 99, // 31: org.apache.beam.model.pipeline.v1.TestStreamPayload.events:type_name -> org.apache.beam.model.pipeline.v1.TestStreamPayload.Event - 127, // 32: org.apache.beam.model.pipeline.v1.TestStreamPayload.endpoint:type_name -> org.apache.beam.model.pipeline.v1.ApiServiceDescriptor - 81, // 33: org.apache.beam.model.pipeline.v1.WriteFilesPayload.sink:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec - 81, // 34: org.apache.beam.model.pipeline.v1.WriteFilesPayload.format_function:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec - 104, // 35: org.apache.beam.model.pipeline.v1.WriteFilesPayload.side_inputs:type_name -> org.apache.beam.model.pipeline.v1.WriteFilesPayload.SideInputsEntry - 81, // 36: org.apache.beam.model.pipeline.v1.Coder.spec:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec - 81, // 37: org.apache.beam.model.pipeline.v1.WindowingStrategy.window_fn:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec - 11, // 38: org.apache.beam.model.pipeline.v1.WindowingStrategy.merge_status:type_name -> org.apache.beam.model.pipeline.v1.MergeStatus.Enum - 61, // 39: org.apache.beam.model.pipeline.v1.WindowingStrategy.trigger:type_name -> org.apache.beam.model.pipeline.v1.Trigger - 12, // 40: org.apache.beam.model.pipeline.v1.WindowingStrategy.accumulation_mode:type_name -> org.apache.beam.model.pipeline.v1.AccumulationMode.Enum - 15, // 41: org.apache.beam.model.pipeline.v1.WindowingStrategy.output_time:type_name -> org.apache.beam.model.pipeline.v1.OutputTime.Enum - 13, // 42: org.apache.beam.model.pipeline.v1.WindowingStrategy.closing_behavior:type_name -> org.apache.beam.model.pipeline.v1.ClosingBehavior.Enum - 14, // 43: org.apache.beam.model.pipeline.v1.WindowingStrategy.on_time_behavior:type_name -> org.apache.beam.model.pipeline.v1.OnTimeBehavior.Enum - 105, // 44: org.apache.beam.model.pipeline.v1.Trigger.after_all:type_name -> org.apache.beam.model.pipeline.v1.Trigger.AfterAll - 106, // 45: org.apache.beam.model.pipeline.v1.Trigger.after_any:type_name -> org.apache.beam.model.pipeline.v1.Trigger.AfterAny - 107, // 46: org.apache.beam.model.pipeline.v1.Trigger.after_each:type_name -> org.apache.beam.model.pipeline.v1.Trigger.AfterEach - 108, // 47: org.apache.beam.model.pipeline.v1.Trigger.after_end_of_window:type_name -> org.apache.beam.model.pipeline.v1.Trigger.AfterEndOfWindow - 109, // 48: org.apache.beam.model.pipeline.v1.Trigger.after_processing_time:type_name -> org.apache.beam.model.pipeline.v1.Trigger.AfterProcessingTime - 110, // 49: org.apache.beam.model.pipeline.v1.Trigger.after_synchronized_processing_time:type_name -> org.apache.beam.model.pipeline.v1.Trigger.AfterSynchronizedProcessingTime - 114, // 50: org.apache.beam.model.pipeline.v1.Trigger.always:type_name -> org.apache.beam.model.pipeline.v1.Trigger.Always - 111, // 51: org.apache.beam.model.pipeline.v1.Trigger.default:type_name -> org.apache.beam.model.pipeline.v1.Trigger.Default - 112, // 52: org.apache.beam.model.pipeline.v1.Trigger.element_count:type_name -> org.apache.beam.model.pipeline.v1.Trigger.ElementCount - 113, // 53: org.apache.beam.model.pipeline.v1.Trigger.never:type_name -> org.apache.beam.model.pipeline.v1.Trigger.Never - 115, // 54: org.apache.beam.model.pipeline.v1.Trigger.or_finally:type_name -> org.apache.beam.model.pipeline.v1.Trigger.OrFinally - 116, // 55: org.apache.beam.model.pipeline.v1.Trigger.repeat:type_name -> org.apache.beam.model.pipeline.v1.Trigger.Repeat - 117, // 56: org.apache.beam.model.pipeline.v1.TimestampTransform.delay:type_name -> org.apache.beam.model.pipeline.v1.TimestampTransform.Delay - 118, // 57: org.apache.beam.model.pipeline.v1.TimestampTransform.align_to:type_name -> org.apache.beam.model.pipeline.v1.TimestampTransform.AlignTo - 81, // 58: org.apache.beam.model.pipeline.v1.SideInput.access_pattern:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec - 81, // 59: org.apache.beam.model.pipeline.v1.SideInput.view_fn:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec - 81, // 60: org.apache.beam.model.pipeline.v1.SideInput.window_mapping_fn:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec - 84, // 61: org.apache.beam.model.pipeline.v1.Environment.display_data:type_name -> org.apache.beam.model.pipeline.v1.DisplayData - 72, // 62: org.apache.beam.model.pipeline.v1.Environment.dependencies:type_name -> org.apache.beam.model.pipeline.v1.ArtifactInformation - 119, // 63: org.apache.beam.model.pipeline.v1.Environment.resource_hints:type_name -> org.apache.beam.model.pipeline.v1.Environment.ResourceHintsEntry - 120, // 64: org.apache.beam.model.pipeline.v1.ProcessPayload.env:type_name -> org.apache.beam.model.pipeline.v1.ProcessPayload.EnvEntry - 127, // 65: org.apache.beam.model.pipeline.v1.ExternalPayload.endpoint:type_name -> org.apache.beam.model.pipeline.v1.ApiServiceDescriptor - 121, // 66: org.apache.beam.model.pipeline.v1.ExternalPayload.params:type_name -> org.apache.beam.model.pipeline.v1.ExternalPayload.ParamsEntry - 26, // 67: org.apache.beam.model.pipeline.v1.MessageWithComponents.components:type_name -> org.apache.beam.model.pipeline.v1.Components - 52, // 68: org.apache.beam.model.pipeline.v1.MessageWithComponents.coder:type_name -> org.apache.beam.model.pipeline.v1.Coder - 45, // 69: org.apache.beam.model.pipeline.v1.MessageWithComponents.combine_payload:type_name -> org.apache.beam.model.pipeline.v1.CombinePayload - 81, // 70: org.apache.beam.model.pipeline.v1.MessageWithComponents.function_spec:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec - 33, // 71: org.apache.beam.model.pipeline.v1.MessageWithComponents.par_do_payload:type_name -> org.apache.beam.model.pipeline.v1.ParDoPayload - 28, // 72: org.apache.beam.model.pipeline.v1.MessageWithComponents.ptransform:type_name -> org.apache.beam.model.pipeline.v1.PTransform - 32, // 73: org.apache.beam.model.pipeline.v1.MessageWithComponents.pcollection:type_name -> org.apache.beam.model.pipeline.v1.PCollection - 43, // 74: org.apache.beam.model.pipeline.v1.MessageWithComponents.read_payload:type_name -> org.apache.beam.model.pipeline.v1.ReadPayload - 63, // 75: org.apache.beam.model.pipeline.v1.MessageWithComponents.side_input:type_name -> org.apache.beam.model.pipeline.v1.SideInput - 44, // 76: org.apache.beam.model.pipeline.v1.MessageWithComponents.window_into_payload:type_name -> org.apache.beam.model.pipeline.v1.WindowIntoPayload - 54, // 77: org.apache.beam.model.pipeline.v1.MessageWithComponents.windowing_strategy:type_name -> org.apache.beam.model.pipeline.v1.WindowingStrategy - 73, // 78: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.environment:type_name -> org.apache.beam.model.pipeline.v1.Environment - 126, // 79: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.wire_coder_settings:type_name -> org.apache.beam.model.pipeline.v1.ExecutableStagePayload.WireCoderSetting - 122, // 80: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.side_inputs:type_name -> org.apache.beam.model.pipeline.v1.ExecutableStagePayload.SideInputId - 26, // 81: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.components:type_name -> org.apache.beam.model.pipeline.v1.Components - 123, // 82: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.user_states:type_name -> org.apache.beam.model.pipeline.v1.ExecutableStagePayload.UserStateId - 124, // 83: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.timers:type_name -> org.apache.beam.model.pipeline.v1.ExecutableStagePayload.TimerId - 125, // 84: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.timerFamilies:type_name -> org.apache.beam.model.pipeline.v1.ExecutableStagePayload.TimerFamilyId - 28, // 85: org.apache.beam.model.pipeline.v1.Components.TransformsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.PTransform - 32, // 86: org.apache.beam.model.pipeline.v1.Components.PcollectionsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.PCollection - 54, // 87: org.apache.beam.model.pipeline.v1.Components.WindowingStrategiesEntry.value:type_name -> org.apache.beam.model.pipeline.v1.WindowingStrategy - 52, // 88: org.apache.beam.model.pipeline.v1.Components.CodersEntry.value:type_name -> org.apache.beam.model.pipeline.v1.Coder - 73, // 89: org.apache.beam.model.pipeline.v1.Components.EnvironmentsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.Environment - 63, // 90: org.apache.beam.model.pipeline.v1.ParDoPayload.SideInputsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.SideInput - 34, // 91: org.apache.beam.model.pipeline.v1.ParDoPayload.StateSpecsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.StateSpec - 41, // 92: org.apache.beam.model.pipeline.v1.ParDoPayload.TimerFamilySpecsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.TimerFamilySpec - 101, // 93: org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.watermark_event:type_name -> org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.AdvanceWatermark - 102, // 94: org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.processing_time_event:type_name -> org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.AdvanceProcessingTime - 103, // 95: org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.element_event:type_name -> org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.AddElements - 100, // 96: org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.AddElements.elements:type_name -> org.apache.beam.model.pipeline.v1.TestStreamPayload.TimestampedElement - 63, // 97: org.apache.beam.model.pipeline.v1.WriteFilesPayload.SideInputsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.SideInput - 61, // 98: org.apache.beam.model.pipeline.v1.Trigger.AfterAll.subtriggers:type_name -> org.apache.beam.model.pipeline.v1.Trigger - 61, // 99: org.apache.beam.model.pipeline.v1.Trigger.AfterAny.subtriggers:type_name -> org.apache.beam.model.pipeline.v1.Trigger - 61, // 100: org.apache.beam.model.pipeline.v1.Trigger.AfterEach.subtriggers:type_name -> org.apache.beam.model.pipeline.v1.Trigger - 61, // 101: org.apache.beam.model.pipeline.v1.Trigger.AfterEndOfWindow.early_firings:type_name -> org.apache.beam.model.pipeline.v1.Trigger - 61, // 102: org.apache.beam.model.pipeline.v1.Trigger.AfterEndOfWindow.late_firings:type_name -> org.apache.beam.model.pipeline.v1.Trigger - 62, // 103: org.apache.beam.model.pipeline.v1.Trigger.AfterProcessingTime.timestamp_transforms:type_name -> org.apache.beam.model.pipeline.v1.TimestampTransform - 61, // 104: org.apache.beam.model.pipeline.v1.Trigger.OrFinally.main:type_name -> org.apache.beam.model.pipeline.v1.Trigger - 61, // 105: org.apache.beam.model.pipeline.v1.Trigger.OrFinally.finally:type_name -> org.apache.beam.model.pipeline.v1.Trigger - 61, // 106: org.apache.beam.model.pipeline.v1.Trigger.Repeat.subtrigger:type_name -> org.apache.beam.model.pipeline.v1.Trigger - 124, // 107: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.WireCoderSetting.timer:type_name -> org.apache.beam.model.pipeline.v1.ExecutableStagePayload.TimerId - 128, // 108: org.apache.beam.model.pipeline.v1.beam_urn:extendee -> google.protobuf.EnumValueOptions - 128, // 109: org.apache.beam.model.pipeline.v1.beam_constant:extendee -> google.protobuf.EnumValueOptions - 47, // 110: org.apache.beam.model.pipeline.v1.TestStreamService.Events:input_type -> org.apache.beam.model.pipeline.v1.EventsRequest - 99, // 111: org.apache.beam.model.pipeline.v1.TestStreamService.Events:output_type -> org.apache.beam.model.pipeline.v1.TestStreamPayload.Event - 111, // [111:112] is the sub-list for method output_type - 110, // [110:111] is the sub-list for method input_type - 110, // [110:110] is the sub-list for extension type_name - 108, // [108:110] is the sub-list for extension extendee - 0, // [0:108] is the sub-list for field type_name + 40, // 24: org.apache.beam.model.pipeline.v1.StateSpec.multimap_spec:type_name -> org.apache.beam.model.pipeline.v1.MultimapStateSpec + 82, // 25: org.apache.beam.model.pipeline.v1.StateSpec.protocol:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec + 82, // 26: org.apache.beam.model.pipeline.v1.CombiningStateSpec.combine_fn:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec + 16, // 27: org.apache.beam.model.pipeline.v1.TimerFamilySpec.time_domain:type_name -> org.apache.beam.model.pipeline.v1.TimeDomain.Enum + 82, // 28: org.apache.beam.model.pipeline.v1.ReadPayload.source:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec + 9, // 29: org.apache.beam.model.pipeline.v1.ReadPayload.is_bounded:type_name -> org.apache.beam.model.pipeline.v1.IsBounded.Enum + 82, // 30: org.apache.beam.model.pipeline.v1.WindowIntoPayload.window_fn:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec + 82, // 31: org.apache.beam.model.pipeline.v1.CombinePayload.combine_fn:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec + 100, // 32: org.apache.beam.model.pipeline.v1.TestStreamPayload.events:type_name -> org.apache.beam.model.pipeline.v1.TestStreamPayload.Event + 128, // 33: org.apache.beam.model.pipeline.v1.TestStreamPayload.endpoint:type_name -> org.apache.beam.model.pipeline.v1.ApiServiceDescriptor + 82, // 34: org.apache.beam.model.pipeline.v1.WriteFilesPayload.sink:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec + 82, // 35: org.apache.beam.model.pipeline.v1.WriteFilesPayload.format_function:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec + 105, // 36: org.apache.beam.model.pipeline.v1.WriteFilesPayload.side_inputs:type_name -> org.apache.beam.model.pipeline.v1.WriteFilesPayload.SideInputsEntry + 82, // 37: org.apache.beam.model.pipeline.v1.Coder.spec:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec + 82, // 38: org.apache.beam.model.pipeline.v1.WindowingStrategy.window_fn:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec + 11, // 39: org.apache.beam.model.pipeline.v1.WindowingStrategy.merge_status:type_name -> org.apache.beam.model.pipeline.v1.MergeStatus.Enum + 62, // 40: org.apache.beam.model.pipeline.v1.WindowingStrategy.trigger:type_name -> org.apache.beam.model.pipeline.v1.Trigger + 12, // 41: org.apache.beam.model.pipeline.v1.WindowingStrategy.accumulation_mode:type_name -> org.apache.beam.model.pipeline.v1.AccumulationMode.Enum + 15, // 42: org.apache.beam.model.pipeline.v1.WindowingStrategy.output_time:type_name -> org.apache.beam.model.pipeline.v1.OutputTime.Enum + 13, // 43: org.apache.beam.model.pipeline.v1.WindowingStrategy.closing_behavior:type_name -> org.apache.beam.model.pipeline.v1.ClosingBehavior.Enum + 14, // 44: org.apache.beam.model.pipeline.v1.WindowingStrategy.on_time_behavior:type_name -> org.apache.beam.model.pipeline.v1.OnTimeBehavior.Enum + 106, // 45: org.apache.beam.model.pipeline.v1.Trigger.after_all:type_name -> org.apache.beam.model.pipeline.v1.Trigger.AfterAll + 107, // 46: org.apache.beam.model.pipeline.v1.Trigger.after_any:type_name -> org.apache.beam.model.pipeline.v1.Trigger.AfterAny + 108, // 47: org.apache.beam.model.pipeline.v1.Trigger.after_each:type_name -> org.apache.beam.model.pipeline.v1.Trigger.AfterEach + 109, // 48: org.apache.beam.model.pipeline.v1.Trigger.after_end_of_window:type_name -> org.apache.beam.model.pipeline.v1.Trigger.AfterEndOfWindow + 110, // 49: org.apache.beam.model.pipeline.v1.Trigger.after_processing_time:type_name -> org.apache.beam.model.pipeline.v1.Trigger.AfterProcessingTime + 111, // 50: org.apache.beam.model.pipeline.v1.Trigger.after_synchronized_processing_time:type_name -> org.apache.beam.model.pipeline.v1.Trigger.AfterSynchronizedProcessingTime + 115, // 51: org.apache.beam.model.pipeline.v1.Trigger.always:type_name -> org.apache.beam.model.pipeline.v1.Trigger.Always + 112, // 52: org.apache.beam.model.pipeline.v1.Trigger.default:type_name -> org.apache.beam.model.pipeline.v1.Trigger.Default + 113, // 53: org.apache.beam.model.pipeline.v1.Trigger.element_count:type_name -> org.apache.beam.model.pipeline.v1.Trigger.ElementCount + 114, // 54: org.apache.beam.model.pipeline.v1.Trigger.never:type_name -> org.apache.beam.model.pipeline.v1.Trigger.Never + 116, // 55: org.apache.beam.model.pipeline.v1.Trigger.or_finally:type_name -> org.apache.beam.model.pipeline.v1.Trigger.OrFinally + 117, // 56: org.apache.beam.model.pipeline.v1.Trigger.repeat:type_name -> org.apache.beam.model.pipeline.v1.Trigger.Repeat + 118, // 57: org.apache.beam.model.pipeline.v1.TimestampTransform.delay:type_name -> org.apache.beam.model.pipeline.v1.TimestampTransform.Delay + 119, // 58: org.apache.beam.model.pipeline.v1.TimestampTransform.align_to:type_name -> org.apache.beam.model.pipeline.v1.TimestampTransform.AlignTo + 82, // 59: org.apache.beam.model.pipeline.v1.SideInput.access_pattern:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec + 82, // 60: org.apache.beam.model.pipeline.v1.SideInput.view_fn:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec + 82, // 61: org.apache.beam.model.pipeline.v1.SideInput.window_mapping_fn:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec + 85, // 62: org.apache.beam.model.pipeline.v1.Environment.display_data:type_name -> org.apache.beam.model.pipeline.v1.DisplayData + 73, // 63: org.apache.beam.model.pipeline.v1.Environment.dependencies:type_name -> org.apache.beam.model.pipeline.v1.ArtifactInformation + 120, // 64: org.apache.beam.model.pipeline.v1.Environment.resource_hints:type_name -> org.apache.beam.model.pipeline.v1.Environment.ResourceHintsEntry + 121, // 65: org.apache.beam.model.pipeline.v1.ProcessPayload.env:type_name -> org.apache.beam.model.pipeline.v1.ProcessPayload.EnvEntry + 128, // 66: org.apache.beam.model.pipeline.v1.ExternalPayload.endpoint:type_name -> org.apache.beam.model.pipeline.v1.ApiServiceDescriptor + 122, // 67: org.apache.beam.model.pipeline.v1.ExternalPayload.params:type_name -> org.apache.beam.model.pipeline.v1.ExternalPayload.ParamsEntry + 26, // 68: org.apache.beam.model.pipeline.v1.MessageWithComponents.components:type_name -> org.apache.beam.model.pipeline.v1.Components + 53, // 69: org.apache.beam.model.pipeline.v1.MessageWithComponents.coder:type_name -> org.apache.beam.model.pipeline.v1.Coder + 46, // 70: org.apache.beam.model.pipeline.v1.MessageWithComponents.combine_payload:type_name -> org.apache.beam.model.pipeline.v1.CombinePayload + 82, // 71: org.apache.beam.model.pipeline.v1.MessageWithComponents.function_spec:type_name -> org.apache.beam.model.pipeline.v1.FunctionSpec + 33, // 72: org.apache.beam.model.pipeline.v1.MessageWithComponents.par_do_payload:type_name -> org.apache.beam.model.pipeline.v1.ParDoPayload + 28, // 73: org.apache.beam.model.pipeline.v1.MessageWithComponents.ptransform:type_name -> org.apache.beam.model.pipeline.v1.PTransform + 32, // 74: org.apache.beam.model.pipeline.v1.MessageWithComponents.pcollection:type_name -> org.apache.beam.model.pipeline.v1.PCollection + 44, // 75: org.apache.beam.model.pipeline.v1.MessageWithComponents.read_payload:type_name -> org.apache.beam.model.pipeline.v1.ReadPayload + 64, // 76: org.apache.beam.model.pipeline.v1.MessageWithComponents.side_input:type_name -> org.apache.beam.model.pipeline.v1.SideInput + 45, // 77: org.apache.beam.model.pipeline.v1.MessageWithComponents.window_into_payload:type_name -> org.apache.beam.model.pipeline.v1.WindowIntoPayload + 55, // 78: org.apache.beam.model.pipeline.v1.MessageWithComponents.windowing_strategy:type_name -> org.apache.beam.model.pipeline.v1.WindowingStrategy + 74, // 79: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.environment:type_name -> org.apache.beam.model.pipeline.v1.Environment + 127, // 80: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.wire_coder_settings:type_name -> org.apache.beam.model.pipeline.v1.ExecutableStagePayload.WireCoderSetting + 123, // 81: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.side_inputs:type_name -> org.apache.beam.model.pipeline.v1.ExecutableStagePayload.SideInputId + 26, // 82: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.components:type_name -> org.apache.beam.model.pipeline.v1.Components + 124, // 83: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.user_states:type_name -> org.apache.beam.model.pipeline.v1.ExecutableStagePayload.UserStateId + 125, // 84: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.timers:type_name -> org.apache.beam.model.pipeline.v1.ExecutableStagePayload.TimerId + 126, // 85: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.timerFamilies:type_name -> org.apache.beam.model.pipeline.v1.ExecutableStagePayload.TimerFamilyId + 28, // 86: org.apache.beam.model.pipeline.v1.Components.TransformsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.PTransform + 32, // 87: org.apache.beam.model.pipeline.v1.Components.PcollectionsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.PCollection + 55, // 88: org.apache.beam.model.pipeline.v1.Components.WindowingStrategiesEntry.value:type_name -> org.apache.beam.model.pipeline.v1.WindowingStrategy + 53, // 89: org.apache.beam.model.pipeline.v1.Components.CodersEntry.value:type_name -> org.apache.beam.model.pipeline.v1.Coder + 74, // 90: org.apache.beam.model.pipeline.v1.Components.EnvironmentsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.Environment + 64, // 91: org.apache.beam.model.pipeline.v1.ParDoPayload.SideInputsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.SideInput + 34, // 92: org.apache.beam.model.pipeline.v1.ParDoPayload.StateSpecsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.StateSpec + 42, // 93: org.apache.beam.model.pipeline.v1.ParDoPayload.TimerFamilySpecsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.TimerFamilySpec + 102, // 94: org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.watermark_event:type_name -> org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.AdvanceWatermark + 103, // 95: org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.processing_time_event:type_name -> org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.AdvanceProcessingTime + 104, // 96: org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.element_event:type_name -> org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.AddElements + 101, // 97: org.apache.beam.model.pipeline.v1.TestStreamPayload.Event.AddElements.elements:type_name -> org.apache.beam.model.pipeline.v1.TestStreamPayload.TimestampedElement + 64, // 98: org.apache.beam.model.pipeline.v1.WriteFilesPayload.SideInputsEntry.value:type_name -> org.apache.beam.model.pipeline.v1.SideInput + 62, // 99: org.apache.beam.model.pipeline.v1.Trigger.AfterAll.subtriggers:type_name -> org.apache.beam.model.pipeline.v1.Trigger + 62, // 100: org.apache.beam.model.pipeline.v1.Trigger.AfterAny.subtriggers:type_name -> org.apache.beam.model.pipeline.v1.Trigger + 62, // 101: org.apache.beam.model.pipeline.v1.Trigger.AfterEach.subtriggers:type_name -> org.apache.beam.model.pipeline.v1.Trigger + 62, // 102: org.apache.beam.model.pipeline.v1.Trigger.AfterEndOfWindow.early_firings:type_name -> org.apache.beam.model.pipeline.v1.Trigger + 62, // 103: org.apache.beam.model.pipeline.v1.Trigger.AfterEndOfWindow.late_firings:type_name -> org.apache.beam.model.pipeline.v1.Trigger + 63, // 104: org.apache.beam.model.pipeline.v1.Trigger.AfterProcessingTime.timestamp_transforms:type_name -> org.apache.beam.model.pipeline.v1.TimestampTransform + 62, // 105: org.apache.beam.model.pipeline.v1.Trigger.OrFinally.main:type_name -> org.apache.beam.model.pipeline.v1.Trigger + 62, // 106: org.apache.beam.model.pipeline.v1.Trigger.OrFinally.finally:type_name -> org.apache.beam.model.pipeline.v1.Trigger + 62, // 107: org.apache.beam.model.pipeline.v1.Trigger.Repeat.subtrigger:type_name -> org.apache.beam.model.pipeline.v1.Trigger + 125, // 108: org.apache.beam.model.pipeline.v1.ExecutableStagePayload.WireCoderSetting.timer:type_name -> org.apache.beam.model.pipeline.v1.ExecutableStagePayload.TimerId + 129, // 109: org.apache.beam.model.pipeline.v1.beam_urn:extendee -> google.protobuf.EnumValueOptions + 129, // 110: org.apache.beam.model.pipeline.v1.beam_constant:extendee -> google.protobuf.EnumValueOptions + 48, // 111: org.apache.beam.model.pipeline.v1.TestStreamService.Events:input_type -> org.apache.beam.model.pipeline.v1.EventsRequest + 100, // 112: org.apache.beam.model.pipeline.v1.TestStreamService.Events:output_type -> org.apache.beam.model.pipeline.v1.TestStreamPayload.Event + 112, // [112:113] is the sub-list for method output_type + 111, // [111:112] is the sub-list for method input_type + 111, // [111:111] is the sub-list for extension type_name + 109, // [109:111] is the sub-list for extension extendee + 0, // [0:109] is the sub-list for field type_name } func init() { file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() } @@ -9624,7 +9725,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetStateSpec); i { + switch v := v.(*MultimapStateSpec); i { case 0: return &v.state case 1: @@ -9636,7 +9737,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TimerFamilySpec); i { + switch v := v.(*SetStateSpec); i { case 0: return &v.state case 1: @@ -9648,7 +9749,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*IsBounded); i { + switch v := v.(*TimerFamilySpec); i { case 0: return &v.state case 1: @@ -9660,7 +9761,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ReadPayload); i { + switch v := v.(*IsBounded); i { case 0: return &v.state case 1: @@ -9672,7 +9773,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*WindowIntoPayload); i { + switch v := v.(*ReadPayload); i { case 0: return &v.state case 1: @@ -9684,7 +9785,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CombinePayload); i { + switch v := v.(*WindowIntoPayload); i { case 0: return &v.state case 1: @@ -9696,7 +9797,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TestStreamPayload); i { + switch v := v.(*CombinePayload); i { case 0: return &v.state case 1: @@ -9708,7 +9809,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EventsRequest); i { + switch v := v.(*TestStreamPayload); i { case 0: return &v.state case 1: @@ -9720,7 +9821,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*WriteFilesPayload); i { + switch v := v.(*EventsRequest); i { case 0: return &v.state case 1: @@ -9732,7 +9833,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PubSubReadPayload); i { + switch v := v.(*WriteFilesPayload); i { case 0: return &v.state case 1: @@ -9744,7 +9845,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PubSubWritePayload); i { + switch v := v.(*PubSubReadPayload); i { case 0: return &v.state case 1: @@ -9756,7 +9857,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GroupIntoBatchesPayload); i { + switch v := v.(*PubSubWritePayload); i { case 0: return &v.state case 1: @@ -9768,7 +9869,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Coder); i { + switch v := v.(*GroupIntoBatchesPayload); i { case 0: return &v.state case 1: @@ -9780,7 +9881,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StandardCoders); i { + switch v := v.(*Coder); i { case 0: return &v.state case 1: @@ -9792,7 +9893,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*WindowingStrategy); i { + switch v := v.(*StandardCoders); i { case 0: return &v.state case 1: @@ -9804,7 +9905,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MergeStatus); i { + switch v := v.(*WindowingStrategy); i { case 0: return &v.state case 1: @@ -9816,7 +9917,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AccumulationMode); i { + switch v := v.(*MergeStatus); i { case 0: return &v.state case 1: @@ -9828,7 +9929,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ClosingBehavior); i { + switch v := v.(*AccumulationMode); i { case 0: return &v.state case 1: @@ -9840,7 +9941,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OnTimeBehavior); i { + switch v := v.(*ClosingBehavior); i { case 0: return &v.state case 1: @@ -9852,7 +9953,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OutputTime); i { + switch v := v.(*OnTimeBehavior); i { case 0: return &v.state case 1: @@ -9864,7 +9965,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TimeDomain); i { + switch v := v.(*OutputTime); i { case 0: return &v.state case 1: @@ -9876,7 +9977,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Trigger); i { + switch v := v.(*TimeDomain); i { case 0: return &v.state case 1: @@ -9888,7 +9989,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TimestampTransform); i { + switch v := v.(*Trigger); i { case 0: return &v.state case 1: @@ -9900,7 +10001,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SideInput); i { + switch v := v.(*TimestampTransform); i { case 0: return &v.state case 1: @@ -9912,7 +10013,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StandardArtifacts); i { + switch v := v.(*SideInput); i { case 0: return &v.state case 1: @@ -9924,7 +10025,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ArtifactFilePayload); i { + switch v := v.(*StandardArtifacts); i { case 0: return &v.state case 1: @@ -9936,7 +10037,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ArtifactUrlPayload); i { + switch v := v.(*ArtifactFilePayload); i { case 0: return &v.state case 1: @@ -9948,7 +10049,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EmbeddedFilePayload); i { + switch v := v.(*ArtifactUrlPayload); i { case 0: return &v.state case 1: @@ -9960,7 +10061,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PyPIPayload); i { + switch v := v.(*EmbeddedFilePayload); i { case 0: return &v.state case 1: @@ -9972,7 +10073,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MavenPayload); i { + switch v := v.(*PyPIPayload); i { case 0: return &v.state case 1: @@ -9984,7 +10085,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeferredArtifactPayload); i { + switch v := v.(*MavenPayload); i { case 0: return &v.state case 1: @@ -9996,7 +10097,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ArtifactStagingToRolePayload); i { + switch v := v.(*DeferredArtifactPayload); i { case 0: return &v.state case 1: @@ -10008,7 +10109,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ArtifactInformation); i { + switch v := v.(*ArtifactStagingToRolePayload); i { case 0: return &v.state case 1: @@ -10020,7 +10121,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Environment); i { + switch v := v.(*ArtifactInformation); i { case 0: return &v.state case 1: @@ -10032,7 +10133,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StandardEnvironments); i { + switch v := v.(*Environment); i { case 0: return &v.state case 1: @@ -10044,7 +10145,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DockerPayload); i { + switch v := v.(*StandardEnvironments); i { case 0: return &v.state case 1: @@ -10056,7 +10157,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProcessPayload); i { + switch v := v.(*DockerPayload); i { case 0: return &v.state case 1: @@ -10068,7 +10169,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ExternalPayload); i { + switch v := v.(*ProcessPayload); i { case 0: return &v.state case 1: @@ -10080,7 +10181,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StandardProtocols); i { + switch v := v.(*ExternalPayload); i { case 0: return &v.state case 1: @@ -10092,7 +10193,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StandardRunnerProtocols); i { + switch v := v.(*StandardProtocols); i { case 0: return &v.state case 1: @@ -10104,7 +10205,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StandardRequirements); i { + switch v := v.(*StandardRunnerProtocols); i { case 0: return &v.state case 1: @@ -10116,7 +10217,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FunctionSpec); i { + switch v := v.(*StandardRequirements); i { case 0: return &v.state case 1: @@ -10128,7 +10229,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StandardDisplayData); i { + switch v := v.(*FunctionSpec); i { case 0: return &v.state case 1: @@ -10140,7 +10241,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LabelledPayload); i { + switch v := v.(*StandardDisplayData); i { case 0: return &v.state case 1: @@ -10152,7 +10253,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DisplayData); i { + switch v := v.(*LabelledPayload); i { case 0: return &v.state case 1: @@ -10164,7 +10265,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MessageWithComponents); i { + switch v := v.(*DisplayData); i { case 0: return &v.state case 1: @@ -10176,7 +10277,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ExecutableStagePayload); i { + switch v := v.(*MessageWithComponents); i { case 0: return &v.state case 1: @@ -10188,6 +10289,18 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { } } file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ExecutableStagePayload); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*StandardResourceHints); i { case 0: return &v.state @@ -10199,7 +10312,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[74].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[75].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*TestStreamPayload_Event); i { case 0: return &v.state @@ -10211,7 +10324,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[75].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[76].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*TestStreamPayload_TimestampedElement); i { case 0: return &v.state @@ -10223,7 +10336,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[76].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[77].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*TestStreamPayload_Event_AdvanceWatermark); i { case 0: return &v.state @@ -10235,7 +10348,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[77].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[78].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*TestStreamPayload_Event_AdvanceProcessingTime); i { case 0: return &v.state @@ -10247,7 +10360,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[78].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[79].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*TestStreamPayload_Event_AddElements); i { case 0: return &v.state @@ -10259,7 +10372,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[80].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[81].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Trigger_AfterAll); i { case 0: return &v.state @@ -10271,7 +10384,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[81].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[82].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Trigger_AfterAny); i { case 0: return &v.state @@ -10283,7 +10396,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[82].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[83].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Trigger_AfterEach); i { case 0: return &v.state @@ -10295,7 +10408,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[83].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[84].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Trigger_AfterEndOfWindow); i { case 0: return &v.state @@ -10307,7 +10420,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[84].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[85].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Trigger_AfterProcessingTime); i { case 0: return &v.state @@ -10319,7 +10432,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[85].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[86].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Trigger_AfterSynchronizedProcessingTime); i { case 0: return &v.state @@ -10331,7 +10444,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[86].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[87].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Trigger_Default); i { case 0: return &v.state @@ -10343,7 +10456,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[87].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[88].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Trigger_ElementCount); i { case 0: return &v.state @@ -10355,7 +10468,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[88].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[89].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Trigger_Never); i { case 0: return &v.state @@ -10367,7 +10480,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[89].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[90].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Trigger_Always); i { case 0: return &v.state @@ -10379,7 +10492,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[90].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[91].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Trigger_OrFinally); i { case 0: return &v.state @@ -10391,7 +10504,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[91].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[92].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Trigger_Repeat); i { case 0: return &v.state @@ -10403,7 +10516,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[92].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[93].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*TimestampTransform_Delay); i { case 0: return &v.state @@ -10415,7 +10528,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[93].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[94].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*TimestampTransform_AlignTo); i { case 0: return &v.state @@ -10427,7 +10540,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[97].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[98].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ExecutableStagePayload_SideInputId); i { case 0: return &v.state @@ -10439,7 +10552,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[98].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[99].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ExecutableStagePayload_UserStateId); i { case 0: return &v.state @@ -10451,7 +10564,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[99].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[100].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ExecutableStagePayload_TimerId); i { case 0: return &v.state @@ -10463,7 +10576,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[100].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[101].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ExecutableStagePayload_TimerFamilyId); i { case 0: return &v.state @@ -10475,7 +10588,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { return nil } } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[101].Exporter = func(v interface{}, i int) interface{} { + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[102].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ExecutableStagePayload_WireCoderSetting); i { case 0: return &v.state @@ -10495,8 +10608,9 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { (*StateSpec_MapSpec)(nil), (*StateSpec_SetSpec)(nil), (*StateSpec_OrderedListSpec)(nil), + (*StateSpec_MultimapSpec)(nil), } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[36].OneofWrappers = []interface{}{ + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[37].OneofWrappers = []interface{}{ (*Trigger_AfterAll_)(nil), (*Trigger_AfterAny_)(nil), (*Trigger_AfterEach_)(nil), @@ -10510,17 +10624,17 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { (*Trigger_OrFinally_)(nil), (*Trigger_Repeat_)(nil), } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[37].OneofWrappers = []interface{}{ + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[38].OneofWrappers = []interface{}{ (*TimestampTransform_Delay_)(nil), (*TimestampTransform_AlignTo_)(nil), } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[58].OneofWrappers = []interface{}{ + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[59].OneofWrappers = []interface{}{ (*LabelledPayload_StringValue)(nil), (*LabelledPayload_BoolValue)(nil), (*LabelledPayload_DoubleValue)(nil), (*LabelledPayload_IntValue)(nil), } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[60].OneofWrappers = []interface{}{ + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[61].OneofWrappers = []interface{}{ (*MessageWithComponents_Coder)(nil), (*MessageWithComponents_CombinePayload)(nil), (*MessageWithComponents_FunctionSpec)(nil), @@ -10532,12 +10646,12 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { (*MessageWithComponents_WindowIntoPayload)(nil), (*MessageWithComponents_WindowingStrategy)(nil), } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[74].OneofWrappers = []interface{}{ + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[75].OneofWrappers = []interface{}{ (*TestStreamPayload_Event_WatermarkEvent)(nil), (*TestStreamPayload_Event_ProcessingTimeEvent)(nil), (*TestStreamPayload_Event_ElementEvent)(nil), } - file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[101].OneofWrappers = []interface{}{ + file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_msgTypes[102].OneofWrappers = []interface{}{ (*ExecutableStagePayload_WireCoderSetting_InputOrOutputId)(nil), (*ExecutableStagePayload_WireCoderSetting_Timer)(nil), } @@ -10547,7 +10661,7 @@ func file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_org_apache_beam_model_pipeline_v1_beam_runner_api_proto_rawDesc, NumEnums: 25, - NumMessages: 102, + NumMessages: 103, NumExtensions: 2, NumServices: 1, }, diff --git a/sdks/go/pkg/beam/model/pipeline_v1/beam_runner_api_grpc.pb.go b/sdks/go/pkg/beam/model/pipeline_v1/beam_runner_api_grpc.pb.go index 79cc55b88493b..20a30cf4dd011 100644 --- a/sdks/go/pkg/beam/model/pipeline_v1/beam_runner_api_grpc.pb.go +++ b/sdks/go/pkg/beam/model/pipeline_v1/beam_runner_api_grpc.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.1.0 -// - protoc v3.21.9 +// - protoc v4.24.4 // source: org/apache/beam/model/pipeline/v1/beam_runner_api.proto package pipeline_v1 diff --git a/sdks/go/pkg/beam/model/pipeline_v1/endpoints.pb.go b/sdks/go/pkg/beam/model/pipeline_v1/endpoints.pb.go index 7fef570b58e54..2dfaffa2bff00 100644 --- a/sdks/go/pkg/beam/model/pipeline_v1/endpoints.pb.go +++ b/sdks/go/pkg/beam/model/pipeline_v1/endpoints.pb.go @@ -21,7 +21,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.27.1 -// protoc v3.21.9 +// protoc v4.24.4 // source: org/apache/beam/model/pipeline/v1/endpoints.proto package pipeline_v1 diff --git a/sdks/go/pkg/beam/model/pipeline_v1/external_transforms.pb.go b/sdks/go/pkg/beam/model/pipeline_v1/external_transforms.pb.go index 0e65a09d08bf4..edbe82264f5ea 100644 --- a/sdks/go/pkg/beam/model/pipeline_v1/external_transforms.pb.go +++ b/sdks/go/pkg/beam/model/pipeline_v1/external_transforms.pb.go @@ -21,7 +21,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.27.1 -// protoc v3.21.9 +// protoc v4.24.4 // source: org/apache/beam/model/pipeline/v1/external_transforms.proto package pipeline_v1 diff --git a/sdks/go/pkg/beam/model/pipeline_v1/metrics.pb.go b/sdks/go/pkg/beam/model/pipeline_v1/metrics.pb.go index b49f461a5a18f..60edad2363be7 100644 --- a/sdks/go/pkg/beam/model/pipeline_v1/metrics.pb.go +++ b/sdks/go/pkg/beam/model/pipeline_v1/metrics.pb.go @@ -21,7 +21,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.27.1 -// protoc v3.21.9 +// protoc v4.24.4 // source: org/apache/beam/model/pipeline/v1/metrics.proto package pipeline_v1 diff --git a/sdks/go/pkg/beam/model/pipeline_v1/schema.pb.go b/sdks/go/pkg/beam/model/pipeline_v1/schema.pb.go index a6c7558282cc2..4bc6a57044cd0 100644 --- a/sdks/go/pkg/beam/model/pipeline_v1/schema.pb.go +++ b/sdks/go/pkg/beam/model/pipeline_v1/schema.pb.go @@ -15,7 +15,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// ** Experimental ** // Protocol Buffers describing Beam Schemas, a portable representation for // complex types. // @@ -25,7 +24,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.27.1 -// protoc v3.21.9 +// protoc v4.24.4 // source: org/apache/beam/model/pipeline/v1/schema.proto package pipeline_v1 diff --git a/sdks/go/pkg/beam/model/pipeline_v1/standard_window_fns.pb.go b/sdks/go/pkg/beam/model/pipeline_v1/standard_window_fns.pb.go index c010e3ba58701..e0522806df73a 100644 --- a/sdks/go/pkg/beam/model/pipeline_v1/standard_window_fns.pb.go +++ b/sdks/go/pkg/beam/model/pipeline_v1/standard_window_fns.pb.go @@ -22,7 +22,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.27.1 -// protoc v3.21.9 +// protoc v4.24.4 // source: org/apache/beam/model/pipeline/v1/standard_window_fns.proto package pipeline_v1 diff --git a/sdks/go/pkg/beam/options/resource/hint.go b/sdks/go/pkg/beam/options/resource/hint.go index 1538fe65def28..d823f4feafa9c 100644 --- a/sdks/go/pkg/beam/options/resource/hint.go +++ b/sdks/go/pkg/beam/options/resource/hint.go @@ -196,3 +196,40 @@ func (h acceleratorHint) MergeWithOuter(outer Hint) Hint { func (h acceleratorHint) String() string { return fmt.Sprintf("accelerator=%v", h.value) } + +// CPUCount hints that this scope should be put in a machine with at least this many CPUs or vCPUs. +// +// Hints are advisory only and runners may not respect them. +// +// See https://beam.apache.org/documentation/runtime/resource-hints/ for more information about +// resource hints. +func CPUCount(v uint64) Hint { + return CPUCountHint{value: uint64(v)} +} + +type CPUCountHint struct { + value uint64 +} + +func (CPUCountHint) URN() string { + return "beam:resources:cpu_count:v1" +} + +func (h CPUCountHint) Payload() []byte { + // Go strings are utf8, and if the string is ascii, + // byte conversion handles that directly. + return []byte(strconv.FormatUint(h.value, 10)) +} + +// MergeWithOuter by keeping the maximum of the two cpu counts. +func (h CPUCountHint) MergeWithOuter(outer Hint) Hint { + // Intentional runtime panic from type assertion to catch hint merge errors. + if outer.(CPUCountHint).value > h.value { + return outer + } + return h +} + +func (h CPUCountHint) String() string { + return fmt.Sprintf("cpu_count=%v", humanize.Bytes(uint64(h.value))) +} diff --git a/sdks/go/pkg/beam/options/resource/hint_test.go b/sdks/go/pkg/beam/options/resource/hint_test.go index cf24b47b6c916..7c2a1df792941 100644 --- a/sdks/go/pkg/beam/options/resource/hint_test.go +++ b/sdks/go/pkg/beam/options/resource/hint_test.go @@ -111,6 +111,38 @@ func TestParseMinRAMHint_panic(t *testing.T) { ParseMinRAM("a bad byte string") } +func TestCPUCountHint_MergeWith(t *testing.T) { + low := CPUCountHint{value: 2} + high := CPUCountHint{value: 128} + + if got, want := low.MergeWithOuter(high), high; got != want { + t.Errorf("%v.MergeWith(%v) = %v, want %v", low, high, got, want) + } + if got, want := high.MergeWithOuter(low), high; got != want { + t.Errorf("%v.MergeWith(%v) = %v, want %v", high, low, got, want) + } +} + +func TestCPUCountHint_Payload(t *testing.T) { + tests := []struct { + value uint64 + payload string + }{ + {0, "0"}, + {2, "2"}, + {11, "11"}, + {2003, "2003"}, + {1.2e7, "12000000"}, + } + + for _, test := range tests { + h := CPUCountHint{value: test.value} + if got, want := h.Payload(), []byte(test.payload); !bytes.Equal(got, want) { + t.Errorf("%v.Payload() = %v, want %v", h, got, want) + } + } +} + // We copy the URN from the proto for use as a constant rather than perform a direct look up // each time, or increase initialization time. However we do need to validate that they are // correct, and match the standard hint urns, so that's done here. @@ -130,7 +162,11 @@ func TestStandardHintUrns(t *testing.T) { }, { h: MinRAMBytes(2e9), urn: getStandardURN(pipepb.StandardResourceHints_MIN_RAM_BYTES), + }, { + h: CPUCount(4), + urn: getStandardURN(pipepb.StandardResourceHints_CPU_COUNT), }} + for _, test := range tests { if got, want := test.h.URN(), test.urn; got != want { t.Errorf("Checked urn for %T, got %q, want %q", test.h, got, want) @@ -154,12 +190,12 @@ func (h customHint) MergeWithOuter(outer Hint) Hint { } func TestHints_Equal(t *testing.T) { - hs := NewHints(MinRAMBytes(2e9), Accelerator("type:pants;count1;install-pajamas")) + hs := NewHints(MinRAMBytes(2e9), Accelerator("type:pants;count1;install-pajamas"), CPUCount(4)) if got, want := hs.Equal(hs), true; got != want { t.Errorf("Self equal test: hs.Equal(hs) = %v, want %v", got, want) } - eq := NewHints(MinRAMBytes(2e9), Accelerator("type:pants;count1;install-pajamas")) + eq := NewHints(MinRAMBytes(2e9), Accelerator("type:pants;count1;install-pajamas"), CPUCount(4)) if got, want := hs.Equal(eq), true; got != want { t.Errorf("identical equal test: hs.Equal(eq) = %v, want %v", got, want) } @@ -223,12 +259,13 @@ func TestHints_MergeWithOuter(t *testing.T) { func TestHints_Payloads(t *testing.T) { { - hs := NewHints(MinRAMBytes(2e9), Accelerator("type:jeans;count1;")) + hs := NewHints(MinRAMBytes(2e9), Accelerator("type:jeans;count1;"), CPUCount(4)) got := hs.Payloads() want := map[string][]byte{ "beam:resources:min_ram_bytes:v1": []byte("2000000000"), "beam:resources:accelerator:v1": []byte("type:jeans;count1;"), + "beam:resources:cpu_count:v1": []byte("4"), } if !reflect.DeepEqual(got, want) { t.Errorf("hs.Payloads() = %v, want %v", got, want) @@ -248,7 +285,7 @@ func TestHints_Payloads(t *testing.T) { func TestHints_NilHints(t *testing.T) { var hs1, hs2 Hints - hs := NewHints(MinRAMBytes(2e9), Accelerator("type:pants;count1;install-pajamas")) + hs := NewHints(MinRAMBytes(2e9), Accelerator("type:pants;count1;install-pajamas"), CPUCount(4)) if got, want := hs1.Equal(hs2), true; got != want { t.Errorf("nils equal test: (nil).Equal(nil) = %v, want %v", got, want) diff --git a/sdks/go/pkg/beam/pardo_test.go b/sdks/go/pkg/beam/pardo_test.go index b88a6d642ea93..56ed7e3e9fa66 100644 --- a/sdks/go/pkg/beam/pardo_test.go +++ b/sdks/go/pkg/beam/pardo_test.go @@ -72,9 +72,9 @@ func testFunction() int64 { func TestFormatParDoError(t *testing.T) { got := formatParDoError(testFunction, 2, 1) - want := "beam.testFunction has 2 outputs, but ParDo requires 1 outputs, use ParDo2 instead." + want := "has 2 outputs, but ParDo requires 1 outputs, use ParDo2 instead." if !strings.Contains(got, want) { - t.Errorf("formatParDoError(testFunction,2,1) = %v, want = %v", got, want) + t.Errorf("formatParDoError(testFunction,2,1) = \n%q want =\n%q", got, want) } } diff --git a/sdks/go/pkg/beam/register/iter.go b/sdks/go/pkg/beam/register/iter.go index df049c96fa965..b93ab4576b0f1 100644 --- a/sdks/go/pkg/beam/register/iter.go +++ b/sdks/go/pkg/beam/register/iter.go @@ -131,7 +131,7 @@ func Iter1[T any]() { exec.RegisterInput(itT, registerFunc) } -// Iter1 registers parameters from your DoFn with a +// Iter2 registers parameters from your DoFn with a // signature func(*T1, *T2) bool and optimizes their execution. // This must be done by passing in type parameters of all inputs as constraints, // aka: register.Iter2[T1, T2]() diff --git a/sdks/go/pkg/beam/runner.go b/sdks/go/pkg/beam/runner.go index 43f6ccce5cd07..c9747da602e1e 100644 --- a/sdks/go/pkg/beam/runner.go +++ b/sdks/go/pkg/beam/runner.go @@ -22,10 +22,6 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam/log" ) -// TODO(herohde) 7/6/2017: do we want to make the selected runner visible to -// transformations? That would allow runner-dependent operations or -// verification, but require that it is stored in Init and used for Run. - var ( runners = make(map[string]func(ctx context.Context, p *Pipeline) (PipelineResult, error)) ) diff --git a/sdks/go/pkg/beam/runners/dataflow/dataflow.go b/sdks/go/pkg/beam/runners/dataflow/dataflow.go index 36e418fb5231e..7b43ba78f0543 100644 --- a/sdks/go/pkg/beam/runners/dataflow/dataflow.go +++ b/sdks/go/pkg/beam/runners/dataflow/dataflow.go @@ -69,6 +69,7 @@ var ( network = flag.String("network", "", "GCP network (optional)") subnetwork = flag.String("subnetwork", "", "GCP subnetwork (optional)") noUsePublicIPs = flag.Bool("no_use_public_ips", false, "Workers must not use public IP addresses (optional)") + usePublicIPs = flag.Bool("use_public_ips", true, "Workers must use public IP addresses (optional)") tempLocation = flag.String("temp_location", "", "Temp location (optional)") workerMachineType = flag.String("worker_machine_type", "", "GCE machine type (optional)") machineType = flag.String("machine_type", "", "alias of worker_machine_type (optional)") @@ -245,6 +246,16 @@ func Execute(ctx context.Context, p *beam.Pipeline) (beam.PipelineResult, error) return dataflowlib.Execute(ctx, model, opts, workerURL, modelURL, *endpoint, *jobopts.Async) } +func isFlagPassed(name string) bool { + found := false + flag.Visit(func(f *flag.Flag) { + if f.Name == name { + found = true + } + }) + return found +} + func getJobOptions(ctx context.Context, streaming bool) (*dataflowlib.JobOptions, error) { project := gcpopts.GetProjectFromFlagOrEnvironment(ctx) if project == "" { @@ -294,6 +305,17 @@ func getJobOptions(ctx context.Context, streaming bool) (*dataflowlib.JobOptions return nil, errors.Wrapf(err, "error reading --transform_name_mapping flag as JSON") } } + if *usePublicIPs == *noUsePublicIPs { + useSet := isFlagPassed("use_public_ips") + noUseSet := isFlagPassed("no_use_public_ips") + // If use_public_ips was explicitly set but no_use_public_ips was not, use that value + // We take the explicit value of no_use_public_ips if it was set but use_public_ips was not. + if useSet && !noUseSet { + *noUsePublicIPs = !*usePublicIPs + } else if useSet && noUseSet { + return nil, errors.New("exactly one of usePublicIPs and noUsePublicIPs must be true, please check that only one is true") + } + } hooks.SerializeHooksToOptions() diff --git a/sdks/go/pkg/beam/runners/dataflow/dataflow_test.go b/sdks/go/pkg/beam/runners/dataflow/dataflow_test.go index d3518964da8a3..663695f00c8e1 100644 --- a/sdks/go/pkg/beam/runners/dataflow/dataflow_test.go +++ b/sdks/go/pkg/beam/runners/dataflow/dataflow_test.go @@ -427,6 +427,81 @@ func TestGetJobOptions_AliasAreEffective(t *testing.T) { } } +func TestGetJobOptions_BadTruePublicIPs(t *testing.T) { + resetGlobals() + *usePublicIPs = true + *noUsePublicIPs = true + + opts, err := getJobOptions(context.Background(), false) + if err == nil { + t.Error("getJobOptions() returned error nil, want an error") + } + if opts != nil { + t.Errorf("getJobOptions() returned JobOptions when it should not have, got %#v, want nil", opts) + } +} + +func TestGetJobOptions_BadFalsePublicIPs(t *testing.T) { + resetGlobals() + *usePublicIPs = false + *noUsePublicIPs = false + + opts, err := getJobOptions(context.Background(), false) + if err == nil { + t.Error("getJobOptions() returned error nil, want an error") + } + if opts != nil { + t.Errorf("getJobOptions() returned JobOptions when it should not have, got %#v, want nil", opts) + } +} + +func TestGetJobOptions_DefaultPublicIPs(t *testing.T) { + resetGlobals() + *labels = `{"label1": "val1", "label2": "val2"}` + *stagingLocation = "gs://testStagingLocation" + *minCPUPlatform = "testPlatform" + *flexRSGoal = "FLEXRS_SPEED_OPTIMIZED" + *dataflowServiceOptions = "opt1,opt2" + + *gcpopts.Project = "testProject" + *gcpopts.Region = "testRegion" + + *jobopts.Experiments = "use_runner_v2,use_portable_job_submission" + *jobopts.JobName = "testJob" + + opts, err := getJobOptions(context.Background(), false) + if err != nil { + t.Fatalf("getJobOptions() returned error %q, want %q", err, "nil") + } + if got, want := opts.NoUsePublicIPs, false; got != want { + t.Errorf("getJobOptions().NoUsePublicIPs = %t, want %t", got, want) + } +} + +func TestGetJobOptions_NoUsePublicIPs(t *testing.T) { + resetGlobals() + *labels = `{"label1": "val1", "label2": "val2"}` + *stagingLocation = "gs://testStagingLocation" + *minCPUPlatform = "testPlatform" + *flexRSGoal = "FLEXRS_SPEED_OPTIMIZED" + *dataflowServiceOptions = "opt1,opt2" + *noUsePublicIPs = true + + *gcpopts.Project = "testProject" + *gcpopts.Region = "testRegion" + + *jobopts.Experiments = "use_runner_v2,use_portable_job_submission" + *jobopts.JobName = "testJob" + + opts, err := getJobOptions(context.Background(), false) + if err != nil { + t.Fatalf("getJobOptions() returned error %q, want %q", err, "nil") + } + if got, want := opts.NoUsePublicIPs, true; got != want { + t.Errorf("getJobOptions().NoUsePublicIPs = %t, want %t", got, want) + } +} + func getFieldFromOpt(fieldName string, opts *dataflowlib.JobOptions) string { return reflect.ValueOf(opts).Elem().FieldByName(fieldName).String() } @@ -447,6 +522,8 @@ func resetGlobals() { *stagingLocation = "" *transformMapping = "" *update = false + *usePublicIPs = true + *noUsePublicIPs = false *workerHarnessImage = "" *workerMachineType = "" *machineType = "" diff --git a/sdks/go/pkg/beam/runners/dataflow/dataflowlib/execute.go b/sdks/go/pkg/beam/runners/dataflow/dataflowlib/execute.go index 67ed337bed575..9a1641e314d12 100644 --- a/sdks/go/pkg/beam/runners/dataflow/dataflowlib/execute.go +++ b/sdks/go/pkg/beam/runners/dataflow/dataflowlib/execute.go @@ -21,6 +21,7 @@ import ( "context" "encoding/json" "os" + "strings" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/metrics" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/graphx" @@ -47,7 +48,12 @@ func Execute(ctx context.Context, raw *pipepb.Pipeline, opts *JobOptions, worker } else { // Cross-compile as last resort. - worker, err := runnerlib.BuildTempWorkerBinary(ctx) + var copts runnerlib.CompileOpts + if strings.HasPrefix(opts.MachineType, "t2a") { + copts.Arch = "arm64" + } + + worker, err := runnerlib.BuildTempWorkerBinary(ctx, copts) if err != nil { return presult, err } diff --git a/sdks/go/pkg/beam/runners/direct/direct.go b/sdks/go/pkg/beam/runners/direct/direct.go index 21cbb1155ea20..13288306066f6 100644 --- a/sdks/go/pkg/beam/runners/direct/direct.go +++ b/sdks/go/pkg/beam/runners/direct/direct.go @@ -15,6 +15,9 @@ // Package direct contains the direct runner for running single-bundle // pipelines in the current process. Useful for testing. +// +// Deprecated: Use prism as a local runner instead. +// Reliance on the direct runner leads to non-portable pipelines. package direct import ( diff --git a/sdks/go/pkg/beam/runners/prism/README.md b/sdks/go/pkg/beam/runners/prism/README.md index 0fc6e6e684166..7ad9dc1d45796 100644 --- a/sdks/go/pkg/beam/runners/prism/README.md +++ b/sdks/go/pkg/beam/runners/prism/README.md @@ -30,8 +30,8 @@ single machine use. For Go SDK users: - `import "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism"` - - Short term: set runner to "prism" to use it, or invoke directly. - - Medium term: switch the default from "direct" to "prism". + - Short term: set runner to "prism" to use it, or invoke directly. ☑ + - Medium term: switch the default from "direct" to "prism". ☑ - Long term: alias "direct" to "prism", and delete legacy Go direct runner. Prisms allow breaking apart and separating a beam of light into @@ -118,7 +118,7 @@ can have features selectively disabled to ensure ## Current Limitations -* Experimental and testing use only. +* Testing use only. * Executing docker containers isn't yet implemented. * This precludes running the Java and Python SDKs, or their transforms for Cross Language. * Loopback execution only. @@ -127,7 +127,6 @@ can have features selectively disabled to ensure * Not yet suitable for larger jobs, which may have intermediate data that exceeds memory bounds. * Doesn't yet support sufficient intermediate data garbage collection for indefinite stream processing. * Doesn't yet execute all beam pipeline features. -* No UI for job status inspection. ## Implemented so far. @@ -140,18 +139,24 @@ can have features selectively disabled to ensure * Global Window * Interval Windowing * Session Windows. +* CoGBKs * Combines lifted and unlifted. * Expands Splittable DoFns +* Process Continuations (AKA Streaming transform support) * Limited support for Process Continuations * Residuals are rescheduled for execution immeadiately. * The transform must be finite (and eventually return a stop process continuation) * Basic Metrics support +* Stand alone execution support + * Web UI available when run as a standalone command. +* Progess tracking + * Channel Splitting + * Dynamic Splitting ## Next feature short list (unordered) See https://github.com/apache/beam/issues/24789 for current status. -* Resolve watermark advancement for Process Continuations * Test Stream * Triggers & Complex Windowing Strategy execution. * State @@ -162,11 +167,6 @@ See https://github.com/apache/beam/issues/24789 for current status. * FnAPI Optimizations * Fusion * Data with ProcessBundleRequest & Response -* Progess tracking - * Channel Splitting - * Dynamic Splitting -* Stand alone execution support -* UI reporting of in progress jobs This is not a comprehensive feature set, but a set of goals to best support users of the Go SDK in testing their pipelines. diff --git a/sdks/go/pkg/beam/runners/prism/internal/coders.go b/sdks/go/pkg/beam/runners/prism/internal/coders.go index a141440400ece..64005177b94b7 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/coders.go +++ b/sdks/go/pkg/beam/runners/prism/internal/coders.go @@ -53,9 +53,12 @@ func isLeafCoder(c *pipepb.Coder) bool { // // PCollection coders are not inherently WindowValueCoder wrapped, and they are added by the runner // for crossing the FnAPI boundary at data sources and data sinks. -func makeWindowedValueCoder(pID string, comps *pipepb.Components, coders map[string]*pipepb.Coder) string { +func makeWindowedValueCoder(pID string, comps *pipepb.Components, coders map[string]*pipepb.Coder) (string, error) { col := comps.GetPcollections()[pID] - cID := lpUnknownCoders(col.GetCoderId(), coders, comps.GetCoders()) + cID, err := lpUnknownCoders(col.GetCoderId(), coders, comps.GetCoders()) + if err != nil { + return "", fmt.Errorf("makeWindowedValueCoder: couldn't process coder for pcollection %q %v: %w", pID, prototext.Format(col), err) + } wcID := comps.GetWindowingStrategies()[col.GetWindowingStrategyId()].GetWindowCoderId() // The runner needs to be defensive, and tell the SDK to Length Prefix @@ -73,7 +76,7 @@ func makeWindowedValueCoder(pID string, comps *pipepb.Components, coders map[str } // Populate the coders to send with the new windowed value coder. coders[wvcID] = wInC - return wvcID + return wvcID, nil } // makeWindowCoders makes the coder pair but behavior is ultimately determined by the strategy's windowFn. @@ -94,22 +97,22 @@ func makeWindowCoders(wc *pipepb.Coder) (exec.WindowDecoder, exec.WindowEncoder) // lpUnknownCoders takes a coder, and populates coders with any new coders // coders that the runner needs to be safe, and speedy. // It returns either the passed in coder id, or the new safe coder id. -func lpUnknownCoders(cID string, bundle, base map[string]*pipepb.Coder) string { +func lpUnknownCoders(cID string, bundle, base map[string]*pipepb.Coder) (string, error) { // First check if we've already added the LP version of this coder to coders already. lpcID := cID + "_lp" // Check if we've done this one before. if _, ok := bundle[lpcID]; ok { - return lpcID + return lpcID, nil } // All coders in the coders map have been processed. if _, ok := bundle[cID]; ok { - return cID + return cID, nil } // Look up the canonical location. c, ok := base[cID] if !ok { // We messed up somewhere. - panic(fmt.Sprint("unknown coder id:", cID)) + return "", fmt.Errorf("lpUnknownCoders: coder %q not present in base map", cID) } // Add the original coder to the coders map. bundle[cID] = c @@ -124,7 +127,7 @@ func lpUnknownCoders(cID string, bundle, base map[string]*pipepb.Coder) string { ComponentCoderIds: []string{cID}, } bundle[lpcID] = lpc - return lpcID + return lpcID, nil } // We know we have a composite, so if we count this as a leaf, move everything to // the coders map. @@ -133,12 +136,15 @@ func lpUnknownCoders(cID string, bundle, base map[string]*pipepb.Coder) string { for _, cc := range c.GetComponentCoderIds() { bundle[cc] = base[cc] } - return cID + return cID, nil } var needNewComposite bool var comps []string - for _, cc := range c.GetComponentCoderIds() { - rcc := lpUnknownCoders(cc, bundle, base) + for i, cc := range c.GetComponentCoderIds() { + rcc, err := lpUnknownCoders(cc, bundle, base) + if err != nil { + return "", fmt.Errorf("lpUnknownCoders: couldn't handle component %d %q of %q %v:\n%w", i, cc, cID, prototext.Format(c), err) + } if cc != rcc { needNewComposite = true } @@ -150,9 +156,9 @@ func lpUnknownCoders(cID string, bundle, base map[string]*pipepb.Coder) string { ComponentCoderIds: comps, } bundle[lpcID] = lpc - return lpcID + return lpcID, nil } - return cID + return cID, nil } // reconcileCoders ensures that the bundle coders are primed with initial coders from diff --git a/sdks/go/pkg/beam/runners/prism/internal/coders_test.go b/sdks/go/pkg/beam/runners/prism/internal/coders_test.go index c6e32c895fe6f..3f9557ff83613 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/coders_test.go +++ b/sdks/go/pkg/beam/runners/prism/internal/coders_test.go @@ -62,7 +62,7 @@ func Test_isLeafCoder(t *testing.T) { func Test_makeWindowedValueCoder(t *testing.T) { coders := map[string]*pipepb.Coder{} - gotID := makeWindowedValueCoder("testPID", &pipepb.Components{ + gotID, err := makeWindowedValueCoder("testPID", &pipepb.Components{ Pcollections: map[string]*pipepb.PCollection{ "testPID": {CoderId: "testCoderID"}, }, @@ -74,7 +74,9 @@ func Test_makeWindowedValueCoder(t *testing.T) { }, }, }, coders) - + if err != nil { + t.Errorf("makeWindowedValueCoder(...) = error %v, want nil", err) + } if gotID == "" { t.Errorf("makeWindowedValueCoder(...) = %v, want non-empty", gotID) } diff --git a/sdks/go/pkg/beam/runners/prism/internal/config/config.go b/sdks/go/pkg/beam/runners/prism/internal/config/config.go index 9c3bdd012bcb0..d188326ff6591 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/config/config.go +++ b/sdks/go/pkg/beam/runners/prism/internal/config/config.go @@ -231,7 +231,7 @@ func (r *HandlerRegistry) Variants() []string { return r.variantIDs } -// Handlers returns the IDs of all handlers used in variations. +// UsedHandlers returns the IDs of all handlers used in variations. func (r *HandlerRegistry) UsedHandlers() []string { return r.handerIDs } diff --git a/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go b/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go index 89fececea1087..df53bce8ac579 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go +++ b/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go @@ -209,11 +209,11 @@ func (rb RunBundle) LogValue() slog.Value { // remaining. func (em *ElementManager) Bundles(ctx context.Context, nextBundID func() string) <-chan RunBundle { runStageCh := make(chan RunBundle) - ctx, cancelFn := context.WithCancel(ctx) + ctx, cancelFn := context.WithCancelCause(ctx) go func() { em.pendingElements.Wait() - slog.Info("no more pending elements: terminating pipeline") - cancelFn() + slog.Debug("no more pending elements: terminating pipeline") + cancelFn(fmt.Errorf("elementManager out of elements, cleaning up")) // Ensure the watermark evaluation goroutine exits. em.refreshCond.Broadcast() }() @@ -287,8 +287,12 @@ func reElementResiduals(residuals [][]byte, inputInfo PColInfo, rb RunBundle) [] if err == io.EOF { break } - slog.Error("reElementResiduals: error decoding residual header", err, "bundle", rb) - panic("error decoding residual header") + slog.Error("reElementResiduals: error decoding residual header", "error", err, "bundle", rb) + panic("error decoding residual header:" + err.Error()) + } + if len(ws) == 0 { + slog.Error("reElementResiduals: sdk provided a windowed value header 0 windows", "bundle", rb) + panic("error decoding residual header: sdk provided a windowed value header 0 windows") } for _, w := range ws { @@ -332,9 +336,13 @@ func (em *ElementManager) PersistBundle(rb RunBundle, col2Coders map[string]PCol if err == io.EOF { break } - slog.Error("PersistBundle: error decoding watermarks", err, "bundle", rb, slog.String("output", output)) + slog.Error("PersistBundle: error decoding watermarks", "error", err, "bundle", rb, slog.String("output", output)) panic("error decoding watermarks") } + if len(ws) == 0 { + slog.Error("PersistBundle: sdk provided a windowed value header 0 windows", "bundle", rb) + panic("error decoding residual header: sdk provided a windowed value header 0 windows") + } // TODO: Optimize unnecessary copies. This is doubleteeing. elmBytes := info.EDec(tee) for _, w := range ws { @@ -386,6 +394,17 @@ func (em *ElementManager) PersistBundle(rb RunBundle, col2Coders map[string]PCol em.addRefreshAndClearBundle(stage.ID, rb.BundleID) } +// FailBundle clears the extant data allowing the execution to shut down. +func (em *ElementManager) FailBundle(rb RunBundle) { + stage := em.stages[rb.StageID] + stage.mu.Lock() + completed := stage.inprogress[rb.BundleID] + em.pendingElements.Add(-len(completed.es)) + delete(stage.inprogress, rb.BundleID) + stage.mu.Unlock() + em.addRefreshAndClearBundle(rb.StageID, rb.BundleID) +} + // ReturnResiduals is called after a successful split, so the remaining work // can be re-assigned to a new bundle. func (em *ElementManager) ReturnResiduals(rb RunBundle, firstRsIndex int, inputInfo PColInfo, residuals [][]byte) { @@ -570,7 +589,7 @@ func (ss *stageState) startBundle(watermark mtime.Time, genBundID func() string) var toProcess, notYet []element for _, e := range ss.pending { - if !ss.aggregate || ss.aggregate && ss.strat.EarliestCompletion(e.window) <= watermark { + if !ss.aggregate || ss.aggregate && ss.strat.EarliestCompletion(e.window) < watermark { toProcess = append(toProcess, e) } else { notYet = append(notYet, e) @@ -706,8 +725,14 @@ func (ss *stageState) bundleReady(em *ElementManager) (mtime.Time, bool) { } ready := true for _, side := range ss.sides { - pID := em.pcolParents[side] - parent := em.stages[pID] + pID, ok := em.pcolParents[side] + if !ok { + panic(fmt.Sprintf("stage[%v] no parent ID for side input %v", ss.ID, side)) + } + parent, ok := em.stages[pID] + if !ok { + panic(fmt.Sprintf("stage[%v] no parent for side input %v, with parent ID %v", ss.ID, side, pID)) + } ow := parent.OutputWatermark() if upstreamW > ow { ready = false diff --git a/sdks/go/pkg/beam/runners/prism/internal/environments.go b/sdks/go/pkg/beam/runners/prism/internal/environments.go new file mode 100644 index 0000000000000..3a429920fb289 --- /dev/null +++ b/sdks/go/pkg/beam/runners/prism/internal/environments.go @@ -0,0 +1,204 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + + fnpb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/fnexecution_v1" + pipepb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/pipeline_v1" + "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/jobservices" + "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/urns" + "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/worker" + "golang.org/x/exp/slog" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/proto" + + dtyp "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" + dcli "github.com/docker/docker/client" + "github.com/docker/docker/pkg/stdcopy" +) + +// TODO move environment handling to the worker package. + +func runEnvironment(ctx context.Context, j *jobservices.Job, env string, wk *worker.W) error { + logger := slog.With(slog.String("envID", wk.Env)) + // TODO fix broken abstraction. + // We're starting a worker pool here, because that's the loopback environment. + // It's sort of a mess, largely because of loopback, which has + // a different flow from a provisioned docker container. + e := j.Pipeline.GetComponents().GetEnvironments()[env] + switch e.GetUrn() { + case urns.EnvExternal: + ep := &pipepb.ExternalPayload{} + if err := (proto.UnmarshalOptions{}).Unmarshal(e.GetPayload(), ep); err != nil { + logger.Error("unmarshing external environment payload", "error", err) + } + go func() { + externalEnvironment(ctx, ep, wk) + slog.Debug("environment stopped", slog.String("job", j.String())) + }() + return nil + case urns.EnvDocker: + dp := &pipepb.DockerPayload{} + if err := (proto.UnmarshalOptions{}).Unmarshal(e.GetPayload(), dp); err != nil { + logger.Error("unmarshing docker environment payload", "error", err) + } + return dockerEnvironment(ctx, logger, dp, wk, j.ArtifactEndpoint()) + default: + return fmt.Errorf("environment %v with urn %v unimplemented", env, e.GetUrn()) + } +} + +func externalEnvironment(ctx context.Context, ep *pipepb.ExternalPayload, wk *worker.W) { + conn, err := grpc.Dial(ep.GetEndpoint().GetUrl(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + panic(fmt.Sprintf("unable to dial sdk worker %v: %v", ep.GetEndpoint().GetUrl(), err)) + } + defer conn.Close() + pool := fnpb.NewBeamFnExternalWorkerPoolClient(conn) + + endpoint := &pipepb.ApiServiceDescriptor{ + Url: wk.Endpoint(), + } + pool.StartWorker(ctx, &fnpb.StartWorkerRequest{ + WorkerId: wk.ID, + ControlEndpoint: endpoint, + LoggingEndpoint: endpoint, + ArtifactEndpoint: endpoint, + ProvisionEndpoint: endpoint, + Params: ep.GetParams(), + }) + // Job processing happens here, but orchestrated by other goroutines + // This goroutine blocks until the context is cancelled, signalling + // that the pool runner should stop the worker. + <-ctx.Done() + + // Previous context cancelled so we need a new one + // for this request. + pool.StopWorker(context.Background(), &fnpb.StopWorkerRequest{ + WorkerId: wk.ID, + }) + wk.Stop() +} + +func dockerEnvironment(ctx context.Context, logger *slog.Logger, dp *pipepb.DockerPayload, wk *worker.W, artifactEndpoint string) error { + logger = logger.With("worker_id", wk.ID, "image", dp.GetContainerImage()) + + // TODO consider preserving client? + cli, err := dcli.NewClientWithOpts(dcli.FromEnv, dcli.WithAPIVersionNegotiation()) + if err != nil { + return fmt.Errorf("couldn't connect to docker:%w", err) + } + + // TODO abstract mounting cloud specific auths better. + const gcloudCredsEnv = "GOOGLE_APPLICATION_CREDENTIALS" + gcloudCredsFile, ok := os.LookupEnv(gcloudCredsEnv) + var mounts []mount.Mount + var envs []string + if ok { + _, err := os.Stat(gcloudCredsFile) + // File exists + if err == nil { + dockerGcloudCredsFile := "/docker_cred_file.json" + mounts = append(mounts, mount.Mount{ + Type: "bind", + Source: gcloudCredsFile, + Target: dockerGcloudCredsFile, + }) + credEnv := fmt.Sprintf("%v=%v", gcloudCredsEnv, dockerGcloudCredsFile) + envs = append(envs, credEnv) + } + } + if _, _, err := cli.ImageInspectWithRaw(ctx, dp.GetContainerImage()); err != nil { + // We don't have a local image, so we should pull it. + if rc, err := cli.ImagePull(ctx, dp.GetContainerImage(), dtyp.ImagePullOptions{}); err == nil { + // Copy the output, but discard it so we can wait until the image pull is finished. + io.Copy(io.Discard, rc) + rc.Close() + } else { + logger.Warn("unable to pull image and it's not local", "error", err) + } + } + + ccr, err := cli.ContainerCreate(ctx, &container.Config{ + Image: dp.GetContainerImage(), + Cmd: []string{ + fmt.Sprintf("--id=%v-%v", wk.JobKey, wk.Env), + fmt.Sprintf("--control_endpoint=%v", wk.Endpoint()), + fmt.Sprintf("--artifact_endpoint=%v", artifactEndpoint), + fmt.Sprintf("--provision_endpoint=%v", wk.Endpoint()), + fmt.Sprintf("--logging_endpoint=%v", wk.Endpoint()), + }, + Env: envs, + Tty: false, + }, &container.HostConfig{ + NetworkMode: "host", + Mounts: mounts, + AutoRemove: true, + }, nil, nil, "") + if err != nil { + cli.Close() + return fmt.Errorf("unable to create container image %v with docker for env %v, err: %w", dp.GetContainerImage(), wk.Env, err) + } + containerID := ccr.ID + logger = logger.With("container", containerID) + + if err := cli.ContainerStart(ctx, containerID, dtyp.ContainerStartOptions{}); err != nil { + cli.Close() + return fmt.Errorf("unable to start container image %v with docker for env %v, err: %w", dp.GetContainerImage(), wk.Env, err) + } + + // Start goroutine to wait on container state. + go func() { + defer cli.Close() + defer wk.Stop() + + statusCh, errCh := cli.ContainerWait(ctx, containerID, container.WaitConditionNotRunning) + select { + case <-ctx.Done(): + // Can't use command context, since it's already canceled here. + err := cli.ContainerKill(context.Background(), containerID, "") + if err != nil { + logger.Error("docker container kill error", "error", err) + } + case err := <-errCh: + if err != nil { + logger.Error("docker container wait error", "error", err) + } + case resp := <-statusCh: + logger.Info("docker container has self terminated", "status_code", resp.StatusCode) + + rc, err := cli.ContainerLogs(ctx, containerID, dtyp.ContainerLogsOptions{Details: true, ShowStdout: true, ShowStderr: true}) + if err != nil { + logger.Error("docker container logs error", "error", err) + } + defer rc.Close() + var buf bytes.Buffer + stdcopy.StdCopy(&buf, &buf, rc) + logger.Error("container self terminated", "log", buf.String()) + } + }() + + return nil +} diff --git a/sdks/go/pkg/beam/runners/prism/internal/execute.go b/sdks/go/pkg/beam/runners/prism/internal/execute.go index 13c8b2b127ccc..c1ac6ea4488c2 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/execute.go +++ b/sdks/go/pkg/beam/runners/prism/internal/execute.go @@ -20,10 +20,11 @@ import ( "fmt" "io" "sort" + "sync/atomic" + "time" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/mtime" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/exec" - fnpb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/fnexecution_v1" pipepb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/pipeline_v1" "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/engine" "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/jobservices" @@ -31,8 +32,6 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/worker" "golang.org/x/exp/maps" "golang.org/x/exp/slog" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" "google.golang.org/protobuf/proto" ) @@ -47,93 +46,73 @@ func RunPipeline(j *jobservices.Job) { // environments, and start up docker containers, but // here, we only want and need the go one, operating // in loopback mode. - env := "go" - wk := worker.New(env) // Cheating by having the worker id match the environment id. - go wk.Serve() - - // When this function exits, we + envs := j.Pipeline.GetComponents().GetEnvironments() + wks := map[string]*worker.W{} + for envID := range envs { + wk, err := makeWorker(envID, j) + if err != nil { + j.Failed(err) + return + } + wks[envID] = wk + } + // When this function exits, we cancel the context to clear + // any related job resources. defer func() { - j.CancelFn() + j.CancelFn(fmt.Errorf("runPipeline returned, cleaning up")) }() - go runEnvironment(j.RootCtx, j, env, wk) j.SendMsg("running " + j.String()) j.Running() - executePipeline(j.RootCtx, wk, j) + if err := executePipeline(j.RootCtx, wks, j); err != nil { + j.Failed(err) + return + } j.SendMsg("pipeline completed " + j.String()) - // Stop the worker. - wk.Stop() - j.SendMsg("terminating " + j.String()) j.Done() } -// TODO move environment handling to the worker package. - -func runEnvironment(ctx context.Context, j *jobservices.Job, env string, wk *worker.W) { - // TODO fix broken abstraction. - // We're starting a worker pool here, because that's the loopback environment. - // It's sort of a mess, largely because of loopback, which has - // a different flow from a provisioned docker container. - e := j.Pipeline.GetComponents().GetEnvironments()[env] - switch e.GetUrn() { - case urns.EnvExternal: - ep := &pipepb.ExternalPayload{} - if err := (proto.UnmarshalOptions{}).Unmarshal(e.GetPayload(), ep); err != nil { - slog.Error("unmarshing environment payload", err, slog.String("envID", wk.ID)) - } - externalEnvironment(ctx, ep, wk) - slog.Info("environment stopped", slog.String("envID", wk.String()), slog.String("job", j.String())) - default: - panic(fmt.Sprintf("environment %v with urn %v unimplemented", env, e.GetUrn())) - } -} - -func externalEnvironment(ctx context.Context, ep *pipepb.ExternalPayload, wk *worker.W) { - conn, err := grpc.Dial(ep.GetEndpoint().GetUrl(), grpc.WithTransportCredentials(insecure.NewCredentials())) - if err != nil { - panic(fmt.Sprintf("unable to dial sdk worker %v: %v", ep.GetEndpoint().GetUrl(), err)) - } - defer conn.Close() - pool := fnpb.NewBeamFnExternalWorkerPoolClient(conn) +// makeWorker creates a worker for that environment. +func makeWorker(env string, j *jobservices.Job) (*worker.W, error) { + wk := worker.New(j.String()+"_"+env, env) - endpoint := &pipepb.ApiServiceDescriptor{ - Url: wk.Endpoint(), - } - pool.StartWorker(ctx, &fnpb.StartWorkerRequest{ - WorkerId: wk.ID, - ControlEndpoint: endpoint, - LoggingEndpoint: endpoint, - ArtifactEndpoint: endpoint, - ProvisionEndpoint: endpoint, - Params: nil, - }) + wk.EnvPb = j.Pipeline.GetComponents().GetEnvironments()[env] + wk.PipelineOptions = j.PipelineOptions() + wk.JobKey = j.JobKey() + wk.ArtifactEndpoint = j.ArtifactEndpoint() - // Job processing happens here, but orchestrated by other goroutines - // This goroutine blocks until the context is cancelled, signalling - // that the pool runner should stop the worker. - <-ctx.Done() + go wk.Serve() - // Previous context cancelled so we need a new one - // for this request. - pool.StopWorker(context.Background(), &fnpb.StopWorkerRequest{ - WorkerId: wk.ID, + if err := runEnvironment(j.RootCtx, j, env, wk); err != nil { + return nil, fmt.Errorf("failed to start environment %v for job %v: %w", env, j, err) + } + // Check for connection succeeding after we've created the environment successfully. + timeout := 1 * time.Minute + time.AfterFunc(timeout, func() { + if wk.Connected() || wk.Stopped() { + return + } + err := fmt.Errorf("prism %v didn't get control connection to %v after %v", wk, wk.Endpoint(), timeout) + j.Failed(err) + j.CancelFn(err) }) + return wk, nil } type transformExecuter interface { ExecuteUrns() []string ExecuteWith(t *pipepb.PTransform) string - ExecuteTransform(tid string, t *pipepb.PTransform, comps *pipepb.Components, watermark mtime.Time, data [][]byte) *worker.B + ExecuteTransform(stageID, tid string, t *pipepb.PTransform, comps *pipepb.Components, watermark mtime.Time, data [][]byte) *worker.B } type processor struct { transformExecuters map[string]transformExecuter } -func executePipeline(ctx context.Context, wk *worker.W, j *jobservices.Job) { +func executePipeline(ctx context.Context, wks map[string]*worker.W, j *jobservices.Job) error { pipeline := j.Pipeline comps := proto.Clone(pipeline.GetComponents()).(*pipepb.Components) @@ -145,7 +124,8 @@ func executePipeline(ctx context.Context, wk *worker.W, j *jobservices.Job) { Combine(CombineCharacteristic{EnableLifting: true}), ParDo(ParDoCharacteristic{DisableSDF: true}), Runner(RunnerCharacteristic{ - SDKFlatten: false, + SDKFlatten: false, + SDKReshuffle: false, }), } @@ -175,10 +155,12 @@ func executePipeline(ctx context.Context, wk *worker.W, j *jobservices.Job) { // TODO move this loop and code into the preprocessor instead. stages := map[string]*stage{} var impulses []string + + // Inialize the "dataservice cache" to support side inputs. + // TODO(https://github.com/apache/beam/issues/28543), remove this concept. + ds := &worker.DataService{} + for i, stage := range topo { - if len(stage.transforms) != 1 { - panic(fmt.Sprintf("unsupported stage[%d]: contains multiple transforms: %v; TODO: implement fusion", i, stage.transforms)) - } tid := stage.transforms[0] t := ts[tid] urn := t.GetSpec().GetUrn() @@ -189,11 +171,11 @@ func executePipeline(ctx context.Context, wk *worker.W, j *jobservices.Job) { if stage.exe != nil { stage.envID = stage.exe.ExecuteWith(t) } - stage.ID = wk.NextStage() + stage.ID = fmt.Sprintf("stage-%03d", i) + wk := wks[stage.envID] switch stage.envID { case "": // Runner Transforms - var onlyOut string for _, out := range t.GetOutputs() { onlyOut = out @@ -252,19 +234,19 @@ func executePipeline(ctx context.Context, wk *worker.W, j *jobservices.Job) { em.AddStage(stage.ID, inputs, nil, []string{getOnlyValue(t.GetOutputs())}) } stages[stage.ID] = stage - wk.Descriptors[stage.ID] = stage.desc - case wk.ID: - // Great! this is for this environment. // Broken abstraction. - buildStage(stage, tid, t, comps, wk) + case wk.Env: + if err := buildDescriptor(stage, comps, wk, ds); err != nil { + return fmt.Errorf("prism error building stage %v: \n%w", stage.ID, err) + } stages[stage.ID] = stage slog.Debug("pipelineBuild", slog.Group("stage", slog.String("ID", stage.ID), slog.String("transformName", t.GetUniqueName()))) outputs := maps.Keys(stage.OutputsToCoders) sort.Strings(outputs) - em.AddStage(stage.ID, []string{stage.mainInputPCol}, stage.sides, outputs) + em.AddStage(stage.ID, []string{stage.primaryInput}, stage.sides, outputs) default: err := fmt.Errorf("unknown environment[%v]", t.GetEnvironmentId()) slog.Error("Execute", err) - panic(err) + return err } } @@ -276,34 +258,67 @@ func executePipeline(ctx context.Context, wk *worker.W, j *jobservices.Job) { // Use a channel to limit max parallelism for the pipeline. maxParallelism := make(chan struct{}, 8) // Execute stages here - for rb := range em.Bundles(ctx, wk.NextInst) { - maxParallelism <- struct{}{} - go func(rb engine.RunBundle) { - defer func() { <-maxParallelism }() - s := stages[rb.StageID] - s.Execute(j, wk, comps, em, rb) - }(rb) + bundleFailed := make(chan error) + + var instID uint64 + bundles := em.Bundles(ctx, func() string { + return fmt.Sprintf("inst%03d", atomic.AddUint64(&instID, 1)) + }) + + for { + select { + case <-ctx.Done(): + return context.Cause(ctx) + case rb, ok := <-bundles: + if !ok { + slog.Debug("pipeline done!", slog.String("job", j.String())) + return nil + } + maxParallelism <- struct{}{} + go func(rb engine.RunBundle) { + defer func() { <-maxParallelism }() + s := stages[rb.StageID] + wk := wks[s.envID] + if err := s.Execute(ctx, j, wk, ds, comps, em, rb); err != nil { + // Ensure we clean up on bundle failure + em.FailBundle(rb) + bundleFailed <- err + } + }(rb) + case err := <-bundleFailed: + return err + } } - slog.Info("pipeline done!", slog.String("job", j.String())) } func collectionPullDecoder(coldCId string, coders map[string]*pipepb.Coder, comps *pipepb.Components) func(io.Reader) []byte { - cID := lpUnknownCoders(coldCId, coders, comps.GetCoders()) + cID, err := lpUnknownCoders(coldCId, coders, comps.GetCoders()) + if err != nil { + panic(err) + } return pullDecoder(coders[cID], coders) } func getWindowValueCoders(comps *pipepb.Components, col *pipepb.PCollection, coders map[string]*pipepb.Coder) (exec.WindowDecoder, exec.WindowEncoder) { ws := comps.GetWindowingStrategies()[col.GetWindowingStrategyId()] - wcID := lpUnknownCoders(ws.GetWindowCoderId(), coders, comps.GetCoders()) + wcID, err := lpUnknownCoders(ws.GetWindowCoderId(), coders, comps.GetCoders()) + if err != nil { + panic(err) + } return makeWindowCoders(coders[wcID]) } -func getOnlyValue[K comparable, V any](in map[K]V) V { +func getOnlyPair[K comparable, V any](in map[K]V) (K, V) { if len(in) != 1 { - panic(fmt.Sprintf("expected single value map, had %v", len(in))) + panic(fmt.Sprintf("expected single value map, had %v - %v", len(in), in)) } - for _, v := range in { - return v + for k, v := range in { + return k, v } panic("unreachable") } + +func getOnlyValue[K comparable, V any](in map[K]V) V { + _, v := getOnlyPair(in) + return v +} diff --git a/sdks/go/pkg/beam/runners/prism/internal/execute_test.go b/sdks/go/pkg/beam/runners/prism/internal/execute_test.go index 3f353f0626f24..1a5ae7989a061 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/execute_test.go +++ b/sdks/go/pkg/beam/runners/prism/internal/execute_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package internal_test import ( "context" @@ -27,6 +27,8 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/metrics" "github.com/apache/beam/sdks/v2/go/pkg/beam/options/jobopts" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" + "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal" "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/jobservices" "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/universal" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/passert" @@ -38,7 +40,7 @@ import ( func initRunner(t *testing.T) { t.Helper() if *jobopts.Endpoint == "" { - s := jobservices.NewServer(0, RunPipeline) + s := jobservices.NewServer(0, internal.RunPipeline) *jobopts.Endpoint = s.Endpoint() go s.Serve() t.Cleanup(func() { @@ -318,6 +320,61 @@ func TestRunner_Pipelines(t *testing.T) { Want: []int{16, 17, 18}, }, sum) }, + }, { + name: "sideinput_sameAsMainInput", + pipeline: func(s beam.Scope) { + imp := beam.Impulse(s) + col0 := beam.ParDo(s, dofn1, imp) + sum := beam.ParDo(s, dofn3x1, col0, beam.SideInput{Input: col0}, beam.SideInput{Input: col0}) + beam.ParDo(s, &int64Check{ + Name: "sum sideinput check", + Want: []int{13, 14, 15}, + }, sum) + }, + }, { + name: "sideinput_sameAsMainInput+Derived", + pipeline: func(s beam.Scope) { + imp := beam.Impulse(s) + col0 := beam.ParDo(s, dofn1, imp) + col1 := beam.ParDo(s, dofn2, col0) + // Doesn't matter which of col0 or col1 is used. + sum := beam.ParDo(s, dofn3x1, col0, beam.SideInput{Input: col0}, beam.SideInput{Input: col1}) + beam.ParDo(s, &int64Check{ + Name: "sum sideinput check", + Want: []int{16, 17, 18}, + }, sum) + }, + }, { + // Main input is getting duplicated data, since it's being executed twice... + // But that doesn't make any sense + name: "sideinput_2iterable1Data2", + pipeline: func(s beam.Scope) { + imp := beam.Impulse(s) + col0 := beam.ParDo(s, dofn1, imp) + col1 := beam.ParDo(s, dofn2, col0) + col2 := beam.ParDo(s, dofn2, col0) + // Doesn't matter which of col1 or col2 is used. + sum := beam.ParDo(s, dofn3x1, col0, beam.SideInput{Input: col2}, beam.SideInput{Input: col1}) + beam.ParDo(s, &int64Check{ + Name: "iter sideinput check", + Want: []int{19, 20, 21}, + }, sum) + }, + }, { + // Re-use the same side inputs sequentially (the two consumers should be in the same stage.) + name: "sideinput_two_2iterable1Data", + pipeline: func(s beam.Scope) { + imp := beam.Impulse(s) + col0 := beam.ParDo(s, dofn1, imp) + sideIn1 := beam.ParDo(s, dofn1, imp) + sideIn2 := beam.ParDo(s, dofn1, imp) + col1 := beam.ParDo(s, dofn3x1, col0, beam.SideInput{Input: sideIn1}, beam.SideInput{Input: sideIn2}) + sum := beam.ParDo(s, dofn3x1, col1, beam.SideInput{Input: sideIn1}, beam.SideInput{Input: sideIn2}) + beam.ParDo(s, &int64Check{ + Name: "check_sideinput_re-use", + Want: []int{25, 26, 27}, + }, sum) + }, }, { name: "combine_perkey", pipeline: func(s beam.Scope) { @@ -379,6 +436,30 @@ func TestRunner_Pipelines(t *testing.T) { }, flat) passert.NonEmpty(s, flat) }, + }, { + name: "gbk_into_gbk", + pipeline: func(s beam.Scope) { + imp := beam.Impulse(s) + col1 := beam.ParDo(s, dofnKV, imp) + gbk1 := beam.GroupByKey(s, col1) + col2 := beam.ParDo(s, dofnGBKKV, gbk1) + gbk2 := beam.GroupByKey(s, col2) + out := beam.ParDo(s, dofnGBK, gbk2) + passert.Equals(s, out, int64(9), int64(12)) + }, + }, { + name: "lperror_gbk_into_cogbk_shared_input", + pipeline: func(s beam.Scope) { + want := beam.CreateList(s, []int{0}) + fruits := beam.CreateList(s, []int64{42, 42, 42}) + fruitsKV := beam.AddFixedKey(s, fruits) + + fruitsGBK := beam.GroupByKey(s, fruitsKV) + fooKV := beam.ParDo(s, toFoo, fruitsGBK) + fruitsFooCoGBK := beam.CoGroupByKey(s, fruitsKV, fooKV) + got := beam.ParDo(s, toID, fruitsFooCoGBK) + passert.Equals(s, got, want) + }, }, } // TODO: Explicit DoFn Failure case. @@ -418,6 +499,87 @@ func TestRunner_Metrics(t *testing.T) { }) } +func TestFailure(t *testing.T) { + initRunner(t) + + p, s := beam.NewPipelineWithRoot() + imp := beam.Impulse(s) + beam.ParDo(s, doFnFail, imp) + _, err := executeWithT(context.Background(), t, p) + if err == nil { + t.Fatalf("expected pipeline failure, but got a success") + } + if want := "doFnFail: failing as intended"; !strings.Contains(err.Error(), want) { + t.Fatalf("expected pipeline failure with %q, but was %v", want, err) + } +} + +func TestRunner_Passert(t *testing.T) { + initRunner(t) + tests := []struct { + name string + pipeline func(s beam.Scope) + metrics func(t *testing.T, pr beam.PipelineResult) + }{ + { + name: "Empty", + pipeline: func(s beam.Scope) { + imp := beam.Impulse(s) + col1 := beam.ParDo(s, dofnEmpty, imp) + passert.Empty(s, col1) + }, + }, { + name: "Equals-TwoEmpty", + pipeline: func(s beam.Scope) { + imp := beam.Impulse(s) + col1 := beam.ParDo(s, dofnEmpty, imp) + col2 := beam.ParDo(s, dofnEmpty, imp) + passert.Equals(s, col1, col2) + }, + }, { + name: "Equals", + pipeline: func(s beam.Scope) { + imp := beam.Impulse(s) + col1 := beam.ParDo(s, dofn1, imp) + col2 := beam.ParDo(s, dofn1, imp) + passert.Equals(s, col1, col2) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + p, s := beam.NewPipelineWithRoot() + test.pipeline(s) + pr, err := executeWithT(context.Background(), t, p) + if err != nil { + t.Fatal(err) + } + if test.metrics != nil { + test.metrics(t, pr) + } + }) + } +} + +func toFoo(et beam.EventTime, id int, _ func(*int64) bool) (int, string) { + return id, "ooo" +} + +func toID(et beam.EventTime, id int, fruitIter func(*int64) bool, fooIter func(*string) bool) int { + var fruit int64 + for fruitIter(&fruit) { + } + var foo string + for fooIter(&foo) { + } + return id +} + +func init() { + register.Function3x2(toFoo) + register.Function4x1(toID) +} + // TODO: PCollection metrics tests, in particular for element counts, in multi transform pipelines // There's a doubling bug since we re-use the same pcollection IDs for the source & sink, and // don't do any re-writing. diff --git a/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go b/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go index e841620625e97..3f699e47e6752 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go +++ b/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go @@ -41,8 +41,9 @@ import ( // RunnerCharacteristic holds the configuration for Runner based transforms, // such as GBKs, Flattens. type RunnerCharacteristic struct { - SDKFlatten bool // Sets whether we should force an SDK side flatten. - SDKGBK bool // Sets whether the GBK should be handled by the SDK, if possible by the SDK. + SDKFlatten bool // Sets whether we should force an SDK side flatten. + SDKGBK bool // Sets whether the GBK should be handled by the SDK, if possible by the SDK. + SDKReshuffle bool // Sets whether we should use the SDK backup implementation to handle a Reshuffle. } func Runner(config any) *runner { @@ -63,13 +64,72 @@ func (*runner) ConfigCharacteristic() reflect.Type { return reflect.TypeOf((*RunnerCharacteristic)(nil)).Elem() } +var _ transformPreparer = (*runner)(nil) + +func (*runner) PrepareUrns() []string { + return []string{urns.TransformReshuffle} +} + +// PrepareTransform handles special processing with respect runner transforms, like reshuffle. +func (h *runner) PrepareTransform(tid string, t *pipepb.PTransform, comps *pipepb.Components) (*pipepb.Components, []string) { + // TODO: Implement the windowing strategy the "backup" transforms used for Reshuffle. + // TODO: Implement a fusion break for reshuffles. + + if h.config.SDKReshuffle { + panic("SDK side reshuffle not yet supported") + } + + // A Reshuffle, in principle, is a no-op on the pipeline structure, WRT correctness. + // It could however affect performance, so it exists to tell the runner that this + // point in the pipeline needs a fusion break, to enable the pipeline to change it's + // degree of parallelism. + // + // The change of parallelism goes both ways. It could allow for larger batch sizes + // enable smaller batch sizes downstream if it is infact paralleizable. + // + // But for a single transform node per stage runner, we can elide it entirely, + // since the input collection and output collection types match. + + // Get the input and output PCollections, there should only be 1 each. + if len(t.GetInputs()) != 1 { + panic("Expected single input PCollection in reshuffle: " + prototext.Format(t)) + } + if len(t.GetOutputs()) != 1 { + panic("Expected single output PCollection in reshuffle: " + prototext.Format(t)) + } + + inColID := getOnlyValue(t.GetInputs()) + outColID := getOnlyValue(t.GetOutputs()) + + // We need to find all Transforms that consume the output collection and + // replace them so they consume the input PCollection directly. + + // We need to remove the consumers of the output PCollection. + toRemove := []string{} + + for _, t := range comps.GetTransforms() { + for li, gi := range t.GetInputs() { + if gi == outColID { + // The whole s + t.GetInputs()[li] = inColID + } + } + } + + // And all the sub transforms. + toRemove = append(toRemove, t.GetSubtransforms()...) + + // Return the new components which is the transforms consumer + return nil, toRemove +} + var _ transformExecuter = (*runner)(nil) func (*runner) ExecuteUrns() []string { - return []string{urns.TransformFlatten, urns.TransformGBK} + return []string{urns.TransformFlatten, urns.TransformGBK, urns.TransformReshuffle} } -// ExecuteWith returns what environment the +// ExecuteWith returns what environment the transform should execute in. func (h *runner) ExecuteWith(t *pipepb.PTransform) string { urn := t.GetSpec().GetUrn() if urn == urns.TransformFlatten && !h.config.SDKFlatten { @@ -82,7 +142,7 @@ func (h *runner) ExecuteWith(t *pipepb.PTransform) string { } // ExecuteTransform handles special processing with respect to runner specific transforms -func (h *runner) ExecuteTransform(tid string, t *pipepb.PTransform, comps *pipepb.Components, watermark mtime.Time, inputData [][]byte) *worker.B { +func (h *runner) ExecuteTransform(stageID, tid string, t *pipepb.PTransform, comps *pipepb.Components, watermark mtime.Time, inputData [][]byte) *worker.B { urn := t.GetSpec().GetUrn() var data [][]byte var onlyOut string @@ -102,9 +162,18 @@ func (h *runner) ExecuteTransform(tid string, t *pipepb.PTransform, comps *pipep coders := map[string]*pipepb.Coder{} // TODO assert this is a KV. It's probably fine, but we should fail anyway. - wcID := lpUnknownCoders(ws.GetWindowCoderId(), coders, comps.GetCoders()) - kcID := lpUnknownCoders(kvc.GetComponentCoderIds()[0], coders, comps.GetCoders()) - ecID := lpUnknownCoders(kvc.GetComponentCoderIds()[1], coders, comps.GetCoders()) + wcID, err := lpUnknownCoders(ws.GetWindowCoderId(), coders, comps.GetCoders()) + if err != nil { + panic(fmt.Errorf("ExecuteTransform[GBK] stage %v, transform %q %v: couldn't process window coder:\n%w", stageID, tid, prototext.Format(t), err)) + } + kcID, err := lpUnknownCoders(kvc.GetComponentCoderIds()[0], coders, comps.GetCoders()) + if err != nil { + panic(fmt.Errorf("ExecuteTransform[GBK] stage %v, transform %q %v: couldn't process key coder:\n%w", stageID, tid, prototext.Format(t), err)) + } + ecID, err := lpUnknownCoders(kvc.GetComponentCoderIds()[1], coders, comps.GetCoders()) + if err != nil { + panic(fmt.Errorf("ExecuteTransform[GBK] stage %v, transform %q %v: couldn't process value coder:\n%w", stageID, tid, prototext.Format(t), err)) + } reconcileCoders(coders, comps.GetCoders()) wc := coders[wcID] diff --git a/sdks/go/pkg/beam/runners/prism/internal/jobservices/artifact.go b/sdks/go/pkg/beam/runners/prism/internal/jobservices/artifact.go index e66def5b0fe86..99b786d459802 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/jobservices/artifact.go +++ b/sdks/go/pkg/beam/runners/prism/internal/jobservices/artifact.go @@ -16,11 +16,14 @@ package jobservices import ( + "bytes" + "context" "fmt" "io" jobpb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/jobmanagement_v1" "golang.org/x/exp/slog" + "google.golang.org/protobuf/encoding/prototext" ) func (s *Server) ReverseArtifactRetrievalService(stream jobpb.ArtifactStagingService_ReverseArtifactRetrievalServiceServer) error { @@ -47,7 +50,7 @@ func (s *Server) ReverseArtifactRetrievalService(stream jobpb.ArtifactStagingSer }, }, }) - var count int + var buf bytes.Buffer for { in, err := stream.Recv() if err == io.EOF { @@ -56,26 +59,61 @@ func (s *Server) ReverseArtifactRetrievalService(stream jobpb.ArtifactStagingSer if err != nil { return err } - if in.IsLast { - slog.Debug("GetArtifact finish", + if in.GetIsLast() { + slog.Debug("GetArtifact finished", slog.Group("dep", slog.String("urn", dep.GetTypeUrn()), slog.String("payload", string(dep.GetTypePayload()))), - slog.Int("bytesReceived", count)) + slog.Int("bytesReceived", buf.Len()), + slog.String("rtype", fmt.Sprintf("%T", in.GetResponse())), + ) break } // Here's where we go through each environment's artifacts. // We do nothing with them. switch req := in.GetResponse().(type) { case *jobpb.ArtifactResponseWrapper_GetArtifactResponse: - count += len(req.GetArtifactResponse.GetData()) + buf.Write(req.GetArtifactResponse.GetData()) + case *jobpb.ArtifactResponseWrapper_ResolveArtifactResponse: err := fmt.Errorf("unexpected ResolveArtifactResponse to GetArtifact: %v", in.GetResponse()) slog.Error("GetArtifact failure", err) return err } } + if len(s.artifacts) == 0 { + s.artifacts = map[string][]byte{} + } + s.artifacts[string(dep.GetTypePayload())] = buf.Bytes() } } return nil } + +func (s *Server) ResolveArtifacts(_ context.Context, req *jobpb.ResolveArtifactsRequest) (*jobpb.ResolveArtifactsResponse, error) { + return &jobpb.ResolveArtifactsResponse{ + Replacements: req.GetArtifacts(), + }, nil +} + +func (s *Server) GetArtifact(req *jobpb.GetArtifactRequest, stream jobpb.ArtifactRetrievalService_GetArtifactServer) error { + info := req.GetArtifact() + buf, ok := s.artifacts[string(info.GetTypePayload())] + if !ok { + pt := prototext.Format(info) + slog.Warn("unable to provide artifact to worker", "artifact_info", pt) + return fmt.Errorf("unable to provide %v to worker", pt) + } + chunk := 128 * 1024 * 1024 // 128 MB + var i int + for i+chunk < len(buf) { + stream.Send(&jobpb.GetArtifactResponse{ + Data: buf[i : i+chunk], + }) + i += chunk + } + stream.Send(&jobpb.GetArtifactResponse{ + Data: buf[i:], + }) + return nil +} diff --git a/sdks/go/pkg/beam/runners/prism/internal/jobservices/job.go b/sdks/go/pkg/beam/runners/prism/internal/jobservices/job.go index 4ac37c5db5988..cd302a70fcc07 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/jobservices/job.go +++ b/sdks/go/pkg/beam/runners/prism/internal/jobservices/job.go @@ -31,6 +31,7 @@ import ( "strings" "sync" "sync/atomic" + "time" fnpb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/fnexecution_v1" jobpb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/jobmanagement_v1" @@ -67,6 +68,8 @@ type Job struct { key string jobName string + artifactEndpoint string + Pipeline *pipepb.Pipeline options *structpb.Struct @@ -77,14 +80,24 @@ type Job struct { msgs []string stateIdx int state atomic.Value // jobpb.JobState_Enum + stateTime time.Time + failureErr error // Context used to terminate this job. RootCtx context.Context - CancelFn context.CancelFunc + CancelFn context.CancelCauseFunc metrics metricsStore } +func (j *Job) ArtifactEndpoint() string { + return j.artifactEndpoint +} + +func (j *Job) PipelineOptions() *structpb.Struct { + return j.options +} + // ContributeTentativeMetrics returns the datachannel read index, and any unknown monitoring short ids. func (j *Job) ContributeTentativeMetrics(payloads *fnpb.ProcessBundleProgressResponse) (int64, []string) { return j.metrics.ContributeTentativeMetrics(payloads) @@ -110,6 +123,10 @@ func (j *Job) LogValue() slog.Value { slog.String("name", j.jobName)) } +func (j *Job) JobKey() string { + return j.key +} + func (j *Job) SendMsg(msg string) { j.streamCond.L.Lock() defer j.streamCond.L.Unlock() @@ -134,8 +151,13 @@ func (j *Job) SendMsg(msg string) { func (j *Job) sendState(state jobpb.JobState_Enum) { j.streamCond.L.Lock() defer j.streamCond.L.Unlock() - j.stateIdx++ - j.state.Store(state) + old := j.state.Load() + // Never overwrite a failed state with another one. + if old != jobpb.JobState_FAILED { + j.state.Store(state) + j.stateTime = time.Now() + j.stateIdx++ + } j.streamCond.Broadcast() } @@ -155,6 +177,9 @@ func (j *Job) Done() { } // Failed indicates that the job completed unsuccessfully. -func (j *Job) Failed() { +func (j *Job) Failed(err error) { + slog.Error("job failed", slog.Any("job", j), slog.Any("error", err)) + j.failureErr = err j.sendState(jobpb.JobState_FAILED) + j.CancelFn(fmt.Errorf("jobFailed %v: %w", j, err)) } diff --git a/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go b/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go index 2b077c3a661e0..0fd7381e17f4b 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go +++ b/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go @@ -24,7 +24,9 @@ import ( jobpb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/jobmanagement_v1" pipepb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/pipeline_v1" "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/urns" + "golang.org/x/exp/maps" "golang.org/x/exp/slog" + "google.golang.org/protobuf/types/known/timestamppb" ) func (s *Server) nextId() string { @@ -68,7 +70,7 @@ func (s *Server) Prepare(ctx context.Context, req *jobpb.PrepareJobRequest) (*jo defer s.mu.Unlock() // Since jobs execute in the background, they should not be tied to a request's context. - rootCtx, cancelFn := context.WithCancel(context.Background()) + rootCtx, cancelFn := context.WithCancelCause(context.Background()) job := &Job{ key: s.nextId(), Pipeline: req.GetPipeline(), @@ -77,28 +79,38 @@ func (s *Server) Prepare(ctx context.Context, req *jobpb.PrepareJobRequest) (*jo streamCond: sync.NewCond(&sync.Mutex{}), RootCtx: rootCtx, CancelFn: cancelFn, + + artifactEndpoint: s.Endpoint(), } // Queue initial state of the job. job.state.Store(jobpb.JobState_STOPPED) + s.jobs[job.key] = job if err := isSupported(job.Pipeline.GetRequirements()); err != nil { + job.Failed(err) slog.Error("unable to run job", slog.String("error", err.Error()), slog.String("jobname", req.GetJobName())) return nil, err } var errs []error - check := func(feature string, got, want any) { - if got != want { - err := unimplementedError{ - feature: feature, - value: got, + check := func(feature string, got any, wants ...any) { + for _, want := range wants { + if got == want { + return } - errs = append(errs, err) } + + err := unimplementedError{ + feature: feature, + value: got, + } + errs = append(errs, err) } // Inspect Transforms for unsupported features. - for _, t := range job.Pipeline.GetComponents().GetTransforms() { + bypassedWindowingStrategies := map[string]bool{} + ts := job.Pipeline.GetComponents().GetTransforms() + for _, t := range ts { urn := t.GetSpec().GetUrn() switch urn { case urns.TransformImpulse, @@ -106,9 +118,28 @@ func (s *Server) Prepare(ctx context.Context, req *jobpb.PrepareJobRequest) (*jo urns.TransformGBK, urns.TransformFlatten, urns.TransformCombinePerKey, + urns.TransformCombineGlobally, // Used by Java SDK + urns.TransformCombineGroupedValues, // Used by Java SDK urns.TransformAssignWindows: // Very few expected transforms types for submitted pipelines. // Most URNs are for the runner to communicate back to the SDK for execution. + case urns.TransformReshuffle: + // Reshuffles use features we don't yet support, but we would like to + // support them by making them the no-op they are, and be precise about + // what we're ignoring. + var cols []string + for _, stID := range t.GetSubtransforms() { + st := ts[stID] + // Only check the outputs, since reshuffle re-instates any previous WindowingStrategy + // so we still validate the strategy used by the input, avoiding skips. + cols = append(cols, maps.Values(st.GetOutputs())...) + } + + pcs := job.Pipeline.GetComponents().GetPcollections() + for _, col := range cols { + wsID := pcs[col].GetWindowingStrategyId() + bypassedWindowingStrategies[wsID] = true + } case "": // Composites can often have no spec if len(t.GetSubtransforms()) > 0 { @@ -121,26 +152,29 @@ func (s *Server) Prepare(ctx context.Context, req *jobpb.PrepareJobRequest) (*jo } // Inspect Windowing strategies for unsupported features. - for _, ws := range job.Pipeline.GetComponents().GetWindowingStrategies() { + for wsID, ws := range job.Pipeline.GetComponents().GetWindowingStrategies() { check("WindowingStrategy.AllowedLateness", ws.GetAllowedLateness(), int64(0)) check("WindowingStrategy.ClosingBehaviour", ws.GetClosingBehavior(), pipepb.ClosingBehavior_EMIT_IF_NONEMPTY) check("WindowingStrategy.AccumulationMode", ws.GetAccumulationMode(), pipepb.AccumulationMode_DISCARDING) if ws.GetWindowFn().GetUrn() != urns.WindowFnSession { check("WindowingStrategy.MergeStatus", ws.GetMergeStatus(), pipepb.MergeStatus_NON_MERGING) } - check("WindowingStrategy.OnTimerBehavior", ws.GetOnTimeBehavior(), pipepb.OnTimeBehavior_FIRE_IF_NONEMPTY) - check("WindowingStrategy.OutputTime", ws.GetOutputTime(), pipepb.OutputTime_END_OF_WINDOW) - // Non nil triggers should fail. - if ws.GetTrigger().GetDefault() == nil { - check("WindowingStrategy.Trigger", ws.GetTrigger(), &pipepb.Trigger_Default{}) + if !bypassedWindowingStrategies[wsID] { + check("WindowingStrategy.OnTimeBehavior", ws.GetOnTimeBehavior(), pipepb.OnTimeBehavior_FIRE_IF_NONEMPTY, pipepb.OnTimeBehavior_FIRE_ALWAYS) + check("WindowingStrategy.OutputTime", ws.GetOutputTime(), pipepb.OutputTime_END_OF_WINDOW) + // Non nil triggers should fail. + if ws.GetTrigger().GetDefault() == nil { + check("WindowingStrategy.Trigger", ws.GetTrigger(), &pipepb.Trigger_Default{}) + } } } if len(errs) > 0 { - err := &joinError{errs: errs} - slog.Error("unable to run job", slog.String("cause", "unimplemented features"), slog.String("jobname", req.GetJobName()), slog.String("errors", err.Error())) - return nil, fmt.Errorf("found %v uses of features unimplemented in prism in job %v: %v", len(errs), req.GetJobName(), err) + jErr := &joinError{errs: errs} + slog.Error("unable to run job", slog.String("cause", "unimplemented features"), slog.String("jobname", req.GetJobName()), slog.String("errors", jErr.Error())) + err := fmt.Errorf("found %v uses of features unimplemented in prism in job %v:\n%v", len(errs), req.GetJobName(), jErr) + job.Failed(err) + return nil, err } - s.jobs[job.key] = job return &jobpb.PrepareJobResponse{ PreparationId: job.key, StagingSessionToken: job.key, @@ -178,20 +212,29 @@ func (s *Server) GetMessageStream(req *jobpb.JobMessagesRequest, stream jobpb.Jo curMsg := job.minMsg curState := job.stateIdx - stream.Context() - state := job.state.Load().(jobpb.JobState_Enum) for { for (curMsg >= job.maxMsg || len(job.msgs) == 0) && curState > job.stateIdx { switch state { - case jobpb.JobState_CANCELLED, jobpb.JobState_DONE, jobpb.JobState_DRAINED, jobpb.JobState_FAILED, jobpb.JobState_UPDATED: + case jobpb.JobState_CANCELLED, jobpb.JobState_DONE, jobpb.JobState_DRAINED, jobpb.JobState_UPDATED: // Reached terminal state. return nil + case jobpb.JobState_FAILED: + // Ensure we send an error message with the cause of the job failure. + stream.Send(&jobpb.JobMessagesResponse{ + Response: &jobpb.JobMessagesResponse_MessageResponse{ + MessageResponse: &jobpb.JobMessage{ + MessageText: job.failureErr.Error(), + Importance: jobpb.JobMessage_JOB_MESSAGE_ERROR, + }, + }, + }) + return nil } job.streamCond.Wait() select { // Quit out if the external connection is done. case <-stream.Context().Done(): - return stream.Context().Err() + return context.Cause(stream.Context()) default: } } @@ -283,6 +326,7 @@ func (s *Server) GetState(_ context.Context, req *jobpb.GetJobStateRequest) (*jo return nil, fmt.Errorf("job with id %v not found", req.GetJobId()) } return &jobpb.JobStateEvent{ - State: j.state.Load().(jobpb.JobState_Enum), + State: j.state.Load().(jobpb.JobState_Enum), + Timestamp: timestamppb.New(j.stateTime), }, nil } diff --git a/sdks/go/pkg/beam/runners/prism/internal/jobservices/server.go b/sdks/go/pkg/beam/runners/prism/internal/jobservices/server.go index aaff03047f68b..bf2db814813c3 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/jobservices/server.go +++ b/sdks/go/pkg/beam/runners/prism/internal/jobservices/server.go @@ -29,6 +29,7 @@ import ( type Server struct { jobpb.UnimplementedJobServiceServer jobpb.UnimplementedArtifactStagingServiceServer + jobpb.UnimplementedArtifactRetrievalServiceServer fnpb.UnimplementedProvisionServiceServer // Server management @@ -42,6 +43,9 @@ type Server struct { // execute defines how a job is executed. execute func(*Job) + + // Artifact hack + artifacts map[string][]byte } // NewServer acquires the indicated port. @@ -60,6 +64,7 @@ func NewServer(port int, execute func(*Job)) *Server { s.server = grpc.NewServer(opts...) jobpb.RegisterJobServiceServer(s.server, s) jobpb.RegisterArtifactStagingServiceServer(s.server, s) + jobpb.RegisterArtifactRetrievalServiceServer(s.server, s) return s } @@ -70,7 +75,8 @@ func (s *Server) getJob(id string) *Job { } func (s *Server) Endpoint() string { - return s.lis.Addr().String() + _, port, _ := net.SplitHostPort(s.lis.Addr().String()) + return fmt.Sprintf("localhost:%v", port) } // Serve serves on the started listener. Blocks. diff --git a/sdks/go/pkg/beam/runners/prism/internal/preprocess.go b/sdks/go/pkg/beam/runners/prism/internal/preprocess.go index 8769a05d38f47..bca40709626d9 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/preprocess.go +++ b/sdks/go/pkg/beam/runners/prism/internal/preprocess.go @@ -16,12 +16,15 @@ package internal import ( + "fmt" "sort" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/pipelinex" pipepb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/pipeline_v1" + "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/urns" "golang.org/x/exp/maps" "golang.org/x/exp/slog" + "google.golang.org/protobuf/encoding/prototext" ) // transformPreparer is an interface for handling different urns in the preprocessor @@ -136,13 +139,255 @@ func (p *preprocessor) preProcessGraph(comps *pipepb.Components) []*stage { keptLeaves := maps.Keys(leaves) sort.Strings(keptLeaves) topological := pipelinex.TopologicalSort(ts, keptLeaves) - slog.Debug("topological transform ordering", topological) + slog.Debug("topological transform ordering", slog.Any("topological", topological)) + // Basic Fusion Behavior + // + // Fusion is the practice of executing associated DoFns in the same stage. + // This often leads to more efficient processing, since costly encode/decode or + // serialize/deserialize operations can be elided. In Beam, any PCollection can + // in principle serve as a place for serializing and deserializing elements. + // + // In particular, Fusion is a stage for optimizing pipeline execution, and was + // described in the FlumeJava paper, in section 4. + // https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/35650.pdf + // + // Per the FlumeJava paper, there are two primary opportunities for Fusion, + // Producer+Consumer fusion and Sibling fusion. + // + // Producer+Consumer fusion is when the producer of a PCollection and the consumers of + // that PCollection are combined into a single stage. Sibling fusion is when two consumers + // of the same pcollection are fused into the same step. These processes can continue until + // graph structure or specific transforms dictate that fusion may not proceed futher. + // + // Examples of fusion breaks include GroupByKeys, or requiring side inputs to complete + // processing for downstream processing, since the producer and consumer of side inputs + // cannot be in the same fused stage. + // + // Additionally, at this phase, we can consider different optimizations for execution. + // For example "Flatten unzipping". In practice, there's no requirement for any stages + // to have an explicit "Flatten" present in the graph. A flatten can be "unzipped", + // duplicating the consumming transforms after the flatten, until a subsequent fusion break. + // This enables additional parallelism by allowing sources to operate in their own independant + // stages. Beam supports this naturally with the separation of work into independant + // bundles for execution. + + return defaultFusion(topological, comps) +} + +// defaultFusion is the base strategy for prism, that doesn't seek to optimize execution +// with fused stages. Input is the set of leaf nodes we're going to execute, topologically +// sorted, and the pipeline components. +// +// Default fusion behavior: Don't. Prism is intended to test all of Beam, which often +// means for testing purposes, to execute pipelines without optimization. +// +// Special Exception to unfused Go SDK pipelines. +// +// If a transform, after a GBK step, has a single input with a KV> coder +// and a single output O with a KV> coder, and if then it must be fused with +// the consumers of O. +func defaultFusion(topological []string, comps *pipepb.Components) []*stage { var stages []*stage + + // TODO figure out a better place to source the PCol Parents/Consumers analysis + // so we don't keep repeating it. + + pcolParents, pcolConsumers := computPColFacts(topological, comps) + + // Explicitly list the pcollectionID we want to fuse along. + fuseWithConsumers := map[string]string{} + for _, tid := range topological { + t := comps.GetTransforms()[tid] + + // See if this transform has a single input and output + if len(t.GetInputs()) != 1 || len(t.GetOutputs()) != 1 { + continue + } + inputID := getOnlyValue(t.GetInputs()) + outputID := getOnlyValue(t.GetOutputs()) + + parentLink := pcolParents[inputID] + + parent := comps.GetTransforms()[parentLink.transform] + + // Check if the input source is a GBK + if parent.GetSpec().GetUrn() != urns.TransformGBK { + continue + } + + // Check if the coder is a KV> + iCID := comps.GetPcollections()[inputID].GetCoderId() + oCID := comps.GetPcollections()[outputID].GetCoderId() + + if checkForExpandCoderPattern(iCID, oCID, comps) { + fuseWithConsumers[tid] = outputID + } + } + + // Since we iterate in topological order, we're guaranteed to process producers before consumers. + consumed := map[string]bool{} // Checks if we've already handled a transform already due to fusion. for _, tid := range topological { - stages = append(stages, &stage{ + if consumed[tid] { + continue + } + stg := &stage{ transforms: []string{tid}, - }) + } + // TODO validate that fused stages have the same environment. + stg.envID = comps.GetTransforms()[tid].EnvironmentId + + stages = append(stages, stg) + + pcolID, ok := fuseWithConsumers[tid] + if !ok { + continue + } + cs := pcolConsumers[pcolID] + + for _, c := range cs { + stg.transforms = append(stg.transforms, c.transform) + consumed[c.transform] = true + } + } + + for _, stg := range stages { + prepareStage(stg, comps, pcolConsumers) } return stages } + +// computPColFacts computes a map of PCollectionIDs to their parent transforms, and a map of +// PCollectionIDs to their consuming transforms. +func computPColFacts(topological []string, comps *pipepb.Components) (map[string]link, map[string][]link) { + pcolParents := map[string]link{} + pcolConsumers := map[string][]link{} + + // Use the topological ids so each PCollection only has a single + // parent. We've already pruned out composites at this stage. + for _, tID := range topological { + t := comps.GetTransforms()[tID] + for local, global := range t.GetOutputs() { + pcolParents[global] = link{transform: tID, local: local, global: global} + } + for local, global := range t.GetInputs() { + pcolConsumers[global] = append(pcolConsumers[global], link{transform: tID, local: local, global: global}) + } + } + + return pcolParents, pcolConsumers +} + +// We need to see that both coders have this pattern: KV> +func checkForExpandCoderPattern(in, out string, comps *pipepb.Components) bool { + isKV := func(id string) bool { + return comps.GetCoders()[id].GetSpec().GetUrn() == urns.CoderKV + } + getComp := func(id string, i int) string { + return comps.GetCoders()[id].GetComponentCoderIds()[i] + } + isIter := func(id string) bool { + return comps.GetCoders()[id].GetSpec().GetUrn() == urns.CoderIterable + } + if !isKV(in) || !isKV(out) { + return false + } + // Are the keys identical? + if getComp(in, 0) != getComp(out, 0) { + return false + } + // Are both values iterables? + if isIter(getComp(in, 1)) && isIter(getComp(out, 1)) { + // If so we have the ExpandCoderPattern from the Go SDK. Hurray! + return true + } + return false +} + +// prepareStage does the final pre-processing step for stages: +// +// 1. Determining the single parallel input (may be 0 for impulse stages). +// 2. Determining all outputs to the stages. +// 3. Determining all side inputs. +// 4 validating that no side input is fed by an internal PCollection. +// 4. Check that all transforms are in the same environment or are environment agnostic. (TODO for xlang) +// 5. Validate that only the primary input consuming transform are stateful. (Might be able to relax this) +// +// Those final steps are necessary to validate that the stage doesn't have any issues, WRT retries or similar. +// +// A PCollection produced by a transform in this stage is in the output set if it's consumed by a transform outside of the stage. +// +// Finally, it takes this information and caches it in the stage for simpler descriptor construction downstream. +// +// Note, this is very similar to the work done WRT composites in pipelinex.Normalize. +func prepareStage(stg *stage, comps *pipepb.Components, pipelineConsumers map[string][]link) { + // Collect all PCollections involved in this stage. + pcolParents, pcolConsumers := computPColFacts(stg.transforms, comps) + + transformSet := map[string]bool{} + for _, tid := range stg.transforms { + transformSet[tid] = true + } + + // Now we can see which consumers (inputs) aren't covered by the parents (outputs). + mainInputs := map[string]string{} + var sideInputs []link + inputs := map[string]bool{} + for pid, plinks := range pcolConsumers { + // Check if this PCollection is generated in this bundle. + if _, ok := pcolParents[pid]; ok { + // It is, so we will ignore for now. + continue + } + // Add this collection to our input set. + inputs[pid] = true + for _, link := range plinks { + t := comps.GetTransforms()[link.transform] + sis, _ := getSideInputs(t) + if _, ok := sis[link.local]; ok { + sideInputs = append(sideInputs, link) + } else { + mainInputs[link.global] = link.global + } + } + } + outputs := map[string]link{} + var internal []string + // Look at all PCollections produced in this stage. + for pid, link := range pcolParents { + // Look at all consumers of this PCollection in the pipeline + isInternal := true + for _, l := range pipelineConsumers[pid] { + // If the consuming transform isn't in the stage, it's an output. + if !transformSet[l.transform] { + isInternal = false + outputs[pid] = link + } + } + // It's consumed as an output, we already ensure the coder's in the set. + if isInternal { + internal = append(internal, pid) + } + } + + stg.internalCols = internal + stg.outputs = maps.Values(outputs) + stg.sideInputs = sideInputs + + defer func() { + if e := recover(); e != nil { + panic(fmt.Sprintf("stage %+v:\n%v\n\n%v", stg, e, prototext.Format(comps))) + } + }() + + // Impulses won't have any inputs. + if l := len(mainInputs); l == 1 { + stg.primaryInput = getOnlyValue(mainInputs) + } else if l > 1 { + // Quick check that this is a lone flatten node, which is handled runner side anyway + // and only sent SDK side as part of a fused stage. + if !(len(stg.transforms) == 1 && comps.GetTransforms()[stg.transforms[0]].GetSpec().GetUrn() == urns.TransformFlatten) { + panic("expected flatten node, but wasn't") + } + } +} diff --git a/sdks/go/pkg/beam/runners/prism/internal/preprocess_test.go b/sdks/go/pkg/beam/runners/prism/internal/preprocess_test.go index add69a7c76792..ba39d024e7160 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/preprocess_test.go +++ b/sdks/go/pkg/beam/runners/prism/internal/preprocess_test.go @@ -20,6 +20,7 @@ import ( pipepb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/pipeline_v1" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "google.golang.org/protobuf/testing/protocmp" ) @@ -73,7 +74,10 @@ func Test_preprocessor_preProcessGraph(t *testing.T) { Environments: map[string]*pipepb.Environment{}, }, - wantStages: []*stage{{transforms: []string{"e1_early"}}, {transforms: []string{"e1_late"}}}, + wantStages: []*stage{ + {transforms: []string{"e1_early"}, envID: "env1", + outputs: []link{{transform: "e1_early", local: "i0", global: "pcol1"}}}, + {transforms: []string{"e1_late"}, envID: "env1", primaryInput: "pcol1"}}, wantComponents: &pipepb.Components{ Transforms: map[string]*pipepb.PTransform{ // Original is always kept @@ -124,11 +128,11 @@ func Test_preprocessor_preProcessGraph(t *testing.T) { pre := newPreprocessor([]transformPreparer{&testPreparer{}}) gotStages := pre.preProcessGraph(test.input) - if diff := cmp.Diff(test.wantStages, gotStages, cmp.AllowUnexported(stage{})); diff != "" { + if diff := cmp.Diff(test.wantStages, gotStages, cmp.AllowUnexported(stage{}, link{}), cmpopts.EquateEmpty()); diff != "" { t.Errorf("preProcessGraph(%q) stages diff (-want,+got)\n%v", test.name, diff) } - if diff := cmp.Diff(test.input, test.wantComponents, protocmp.Transform()); diff != "" { + if diff := cmp.Diff(test.wantComponents, test.input, protocmp.Transform()); diff != "" { t.Errorf("preProcessGraph(%q) components diff (-want,+got)\n%v", test.name, diff) } }) diff --git a/sdks/go/pkg/beam/runners/prism/internal/separate_test.go b/sdks/go/pkg/beam/runners/prism/internal/separate_test.go index a234d6470a43f..97ae494e4abb7 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/separate_test.go +++ b/sdks/go/pkg/beam/runners/prism/internal/separate_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package internal_test import ( "context" diff --git a/sdks/go/pkg/beam/runners/prism/internal/stage.go b/sdks/go/pkg/beam/runners/prism/internal/stage.go index c5a10cae7b411..4ce3ce7ffeb6e 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/stage.go +++ b/sdks/go/pkg/beam/runners/prism/internal/stage.go @@ -33,23 +33,39 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/worker" "golang.org/x/exp/maps" "golang.org/x/exp/slog" + "google.golang.org/protobuf/encoding/prototext" "google.golang.org/protobuf/proto" ) -// stage represents a fused subgraph. +// link represents the tuple of a transform, the local id, and the global id for +// that transform's respective input or output. Which it is, is context dependant, +// and not knowable from just the link itself, but can be verified against the transform proto. +type link struct { + transform, local, global string +} + +// stage represents a fused subgraph executed in a single environment. +// +// TODO: Consider ignoring environment boundaries and making fusion +// only consider necessary materialization breaks. The data protocol +// should in principle be able to connect two SDK environments directly +// instead of going through the runner at all, which would be a small +// efficiency gain, in runner memory use. // -// TODO: do we guarantee that they are all -// the same environment at this point, or -// should that be handled later? +// That would also warrant an execution mode where fusion is taken into +// account, but all serialization boundaries remain since the pcollections +// would continue to get serialized. type stage struct { - ID string - transforms []string + ID string + transforms []string + primaryInput string // PCollection used as the parallel input. + outputs []link // PCollections that must escape this stage. + sideInputs []link // Non-parallel input PCollections and their consumers + internalCols []string // PCollections that escape. Used for precise coder sending. + envID string - envID string exe transformExecuter - outputCount int inputTransformID string - mainInputPCol string inputInfo engine.PColInfo desc *fnpb.ProcessBundleDescriptor sides []string @@ -59,27 +75,30 @@ type stage struct { OutputsToCoders map[string]engine.PColInfo } -func (s *stage) Execute(j *jobservices.Job, wk *worker.W, comps *pipepb.Components, em *engine.ElementManager, rb engine.RunBundle) { - tid := s.transforms[0] - slog.Debug("Execute: starting bundle", "bundle", rb, slog.String("tid", tid)) +func (s *stage) Execute(ctx context.Context, j *jobservices.Job, wk *worker.W, ds *worker.DataService, comps *pipepb.Components, em *engine.ElementManager, rb engine.RunBundle) error { + slog.Debug("Execute: starting bundle", "bundle", rb) var b *worker.B inputData := em.InputForBundle(rb, s.inputInfo) var dataReady <-chan struct{} switch s.envID { case "": // Runner Transforms + if len(s.transforms) != 1 { + panic(fmt.Sprintf("unexpected number of runner transforms, want 1: %+v", s)) + } + tid := s.transforms[0] // Runner transforms are processed immeadiately. - b = s.exe.ExecuteTransform(tid, comps.GetTransforms()[tid], comps, rb.Watermark, inputData) + b = s.exe.ExecuteTransform(s.ID, tid, comps.GetTransforms()[tid], comps, rb.Watermark, inputData) b.InstID = rb.BundleID slog.Debug("Execute: runner transform", "bundle", rb, slog.String("tid", tid)) // Do some accounting for the fake bundle. b.Resp = make(chan *fnpb.ProcessBundleResponse, 1) - close(b.Resp) // To + close(b.Resp) // To avoid blocking downstream, since we don't send on this. closed := make(chan struct{}) close(closed) dataReady = closed - case wk.ID: + case wk.Env: b = &worker.B{ PBDID: s.ID, InstID: rb.BundleID, @@ -90,7 +109,7 @@ func (s *stage) Execute(j *jobservices.Job, wk *worker.W, comps *pipepb.Componen InputData: inputData, SinkToPCollection: s.SinkToPCollection, - OutputCount: s.outputCount, + OutputCount: len(s.outputs), } b.Init() @@ -98,10 +117,10 @@ func (s *stage) Execute(j *jobservices.Job, wk *worker.W, comps *pipepb.Componen slog.Debug("Execute: processing", "bundle", rb) defer b.Cleanup(wk) - dataReady = b.ProcessOn(wk) + dataReady = b.ProcessOn(ctx, wk) default: err := fmt.Errorf("unknown environment[%v]", s.envID) - slog.Error("Execute", err) + slog.Error("Execute", "error", err) panic(err) } @@ -116,18 +135,26 @@ progress: progTick.Stop() break progress // exit progress loop on close. case <-progTick.C: - resp := b.Progress(wk) + resp, err := b.Progress(ctx, wk) + if err != nil { + slog.Debug("SDK Error from progress, aborting progress", "bundle", rb, "error", err.Error()) + break progress + } index, unknownIDs := j.ContributeTentativeMetrics(resp) if len(unknownIDs) > 0 { - md := wk.MonitoringMetadata(unknownIDs) + md := wk.MonitoringMetadata(ctx, unknownIDs) j.AddMetricShortIDs(md) } slog.Debug("progress report", "bundle", rb, "index", index) // Progress for the bundle hasn't advanced. Try splitting. if previousIndex == index && !splitsDone { - sr := b.Split(wk, 0.5 /* fraction of remainder */, nil /* allowed splits */) + sr, err := b.Split(ctx, wk, 0.5 /* fraction of remainder */, nil /* allowed splits */) + if err != nil { + slog.Warn("SDK Error from split, aborting splits", "bundle", rb, "error", err.Error()) + break progress + } if sr.GetChannelSplits() == nil { - slog.Warn("split failed", "bundle", rb) + slog.Debug("SDK returned no splits", "bundle", rb) splitsDone = true continue progress } @@ -160,16 +187,25 @@ progress: // Tentative Data is ready, commit it to the main datastore. slog.Debug("Execute: commiting data", "bundle", rb, slog.Any("outputsWithData", maps.Keys(b.OutputData.Raw)), slog.Any("outputs", maps.Keys(s.OutputsToCoders))) - resp := <-b.Resp + var resp *fnpb.ProcessBundleResponse + select { + case resp = <-b.Resp: + if b.BundleErr != nil { + return b.BundleErr + } + case <-ctx.Done(): + return context.Cause(ctx) + } + // Tally metrics immeadiately so they're available before // pipeline termination. unknownIDs := j.ContributeFinalMetrics(resp) if len(unknownIDs) > 0 { - md := wk.MonitoringMetadata(unknownIDs) + md := wk.MonitoringMetadata(ctx, unknownIDs) j.AddMetricShortIDs(md) } - // TODO handle side input data properly. - wk.D.Commit(b.OutputData) + // TODO(https://github.com/apache/beam/issues/28543) handle side input data properly. + ds.Commit(b.OutputData) var residualData [][]byte var minOutputWatermark map[string]mtime.Time for _, rr := range resp.GetResidualRoots() { @@ -191,16 +227,18 @@ progress: } } if l := len(residualData); l > 0 { - slog.Debug("returned empty residual application", "bundle", rb, slog.Int("numResiduals", l), slog.String("pcollection", s.mainInputPCol)) + slog.Debug("returned empty residual application", "bundle", rb, slog.Int("numResiduals", l), slog.String("pcollection", s.primaryInput)) } em.PersistBundle(rb, s.OutputsToCoders, b.OutputData, s.inputInfo, residualData, minOutputWatermark) b.OutputData = engine.TentativeData{} // Clear the data. + return nil } func getSideInputs(t *pipepb.PTransform) (map[string]*pipepb.SideInput, error) { if t.GetSpec().GetUrn() != urns.TransformParDo { return nil, nil } + // TODO, memoize this, so we don't need to repeatedly unmarshal. pardo := &pipepb.ParDoPayload{} if err := (proto.UnmarshalOptions{}).Unmarshal(t.GetSpec().GetPayload(), pardo); err != nil { return nil, fmt.Errorf("unable to decode ParDoPayload") @@ -222,76 +260,113 @@ func portFor(wInCid string, wk *worker.W) []byte { return sourcePortBytes } -func buildStage(s *stage, tid string, t *pipepb.PTransform, comps *pipepb.Components, wk *worker.W) { - s.inputTransformID = tid + "_source" +// buildDescriptor constructs a ProcessBundleDescriptor for bundles of this stage. +// +// Requirements: +// * The set of inputs to the stage only include one parallel input. +// * The side input pcollections are fully qualified with global pcollection ID, ingesting transform, and local inputID. +// * The outputs are fully qualified with global PCollectionID, producing transform, and local outputID. +// +// It assumes that the side inputs are not sourced from PCollections generated by any transform in this stage. +// +// Because we need the local ids for routing the sources/sinks information. +func buildDescriptor(stg *stage, comps *pipepb.Components, wk *worker.W, ds *worker.DataService) error { + // Assume stage has an indicated primary input coders := map[string]*pipepb.Coder{} - transforms := map[string]*pipepb.PTransform{ - tid: t, // The Transform to Execute! - } + transforms := map[string]*pipepb.PTransform{} - sis, err := getSideInputs(t) - if err != nil { - slog.Error("buildStage: getSide Inputs", err, slog.String("transformID", tid)) - panic(err) - } - var inputInfo engine.PColInfo - var sides []string - for local, global := range t.GetInputs() { - // This id is directly used for the source, but this also copies - // coders used by side inputs to the coders map for the bundle, so - // needs to be run for every ID. - wInCid := makeWindowedValueCoder(global, comps, coders) - _, ok := sis[local] - if ok { - sides = append(sides, global) - } else { - // this is the main input - transforms[s.inputTransformID] = sourceTransform(s.inputTransformID, portFor(wInCid, wk), global) - col := comps.GetPcollections()[global] - ed := collectionPullDecoder(col.GetCoderId(), coders, comps) - wDec, wEnc := getWindowValueCoders(comps, col, coders) - inputInfo = engine.PColInfo{ - GlobalID: global, - WDec: wDec, - WEnc: wEnc, - EDec: ed, - } - } - // We need to process all inputs to ensure we have all input coders, so we must continue. + for _, tid := range stg.transforms { + transforms[tid] = comps.GetTransforms()[tid] } - prepareSides, err := handleSideInputs(t, comps, coders, wk) - if err != nil { - slog.Error("buildStage: handleSideInputs", err, slog.String("transformID", tid)) - panic(err) - } - - // TODO: We need a new logical PCollection to represent the source - // so we can avoid double counting PCollection metrics later. - // But this also means replacing the ID for the input in the bundle. + // Start with outputs, since they're simple and uniform. sink2Col := map[string]string{} col2Coders := map[string]engine.PColInfo{} - for local, global := range t.GetOutputs() { - wOutCid := makeWindowedValueCoder(global, comps, coders) - sinkID := tid + "_" + local - col := comps.GetPcollections()[global] + for _, o := range stg.outputs { + col := comps.GetPcollections()[o.global] + wOutCid, err := makeWindowedValueCoder(o.global, comps, coders) + if err != nil { + return fmt.Errorf("buildDescriptor: failed to handle coder on stage %v for output %+v, pcol %q %v:\n%w", stg.ID, o, o.global, prototext.Format(col), err) + } + sinkID := o.transform + "_" + o.local ed := collectionPullDecoder(col.GetCoderId(), coders, comps) wDec, wEnc := getWindowValueCoders(comps, col, coders) - sink2Col[sinkID] = global - col2Coders[global] = engine.PColInfo{ - GlobalID: global, + sink2Col[sinkID] = o.global + col2Coders[o.global] = engine.PColInfo{ + GlobalID: o.global, WDec: wDec, WEnc: wEnc, EDec: ed, } - transforms[sinkID] = sinkTransform(sinkID, portFor(wOutCid, wk), global) + transforms[sinkID] = sinkTransform(sinkID, portFor(wOutCid, wk), o.global) + } + + // Then lets do Side Inputs, since they are also uniform. + var sides []string + var prepareSides []func(b *worker.B, watermark mtime.Time) + for _, si := range stg.sideInputs { + col := comps.GetPcollections()[si.global] + oCID := col.GetCoderId() + nCID, err := lpUnknownCoders(oCID, coders, comps.GetCoders()) + if err != nil { + return fmt.Errorf("buildDescriptor: failed to handle coder on stage %v for side input %+v, pcol %q %v:\n%w", stg.ID, si, si.global, prototext.Format(col), err) + } + + sides = append(sides, si.global) + if oCID != nCID { + // Add a synthetic PCollection set with the new coder. + newGlobal := si.global + "_prismside" + comps.GetPcollections()[newGlobal] = &pipepb.PCollection{ + DisplayData: col.GetDisplayData(), + UniqueName: col.GetUniqueName(), + CoderId: nCID, + IsBounded: col.GetIsBounded(), + WindowingStrategyId: col.WindowingStrategyId, + } + // Update side inputs to point to new PCollection with any replaced coders. + transforms[si.transform].GetInputs()[si.local] = newGlobal + } + prepSide, err := handleSideInput(si.transform, si.local, si.global, comps, coders, ds) + if err != nil { + slog.Error("buildDescriptor: handleSideInputs", err, slog.String("transformID", si.transform)) + return err + } + prepareSides = append(prepareSides, prepSide) + } + + // Finally, the parallel input, which is it's own special snowflake, that needs a datasource. + // This id is directly used for the source, but this also copies + // coders used by side inputs to the coders map for the bundle, so + // needs to be run for every ID. + + col := comps.GetPcollections()[stg.primaryInput] + wInCid, err := makeWindowedValueCoder(stg.primaryInput, comps, coders) + if err != nil { + return fmt.Errorf("buildDescriptor: failed to handle coder on stage %v for primary input, pcol %q %v:\n%w", stg.ID, stg.primaryInput, prototext.Format(col), err) + } + + ed := collectionPullDecoder(col.GetCoderId(), coders, comps) + wDec, wEnc := getWindowValueCoders(comps, col, coders) + inputInfo := engine.PColInfo{ + GlobalID: stg.primaryInput, + WDec: wDec, + WEnc: wEnc, + EDec: ed, + } + + stg.inputTransformID = stg.ID + "_source" + transforms[stg.inputTransformID] = sourceTransform(stg.inputTransformID, portFor(wInCid, wk), stg.primaryInput) + + // Add coders for internal collections. + for _, pid := range stg.internalCols { + lpUnknownCoders(comps.GetPcollections()[pid].GetCoderId(), coders, comps.GetCoders()) } reconcileCoders(coders, comps.GetCoders()) desc := &fnpb.ProcessBundleDescriptor{ - Id: s.ID, + Id: stg.ID, Transforms: transforms, WindowingStrategies: comps.GetWindowingStrategies(), Pcollections: comps.GetPcollections(), @@ -301,114 +376,103 @@ func buildStage(s *stage, tid string, t *pipepb.PTransform, comps *pipepb.Compon }, } - s.desc = desc - s.outputCount = len(t.Outputs) - s.prepareSides = prepareSides - s.sides = sides - s.SinkToPCollection = sink2Col - s.OutputsToCoders = col2Coders - s.mainInputPCol = inputInfo.GlobalID - s.inputInfo = inputInfo + stg.desc = desc + stg.prepareSides = func(b *worker.B, _ string, watermark mtime.Time) { + for _, prep := range prepareSides { + prep(b, watermark) + } + } + stg.sides = sides // List of the global pcollection IDs this stage needs to wait on for side inputs. + stg.SinkToPCollection = sink2Col + stg.OutputsToCoders = col2Coders + stg.inputInfo = inputInfo - wk.Descriptors[s.ID] = s.desc + wk.Descriptors[stg.ID] = stg.desc + return nil } -// handleSideInputs ensures appropriate coders are available to the bundle, and prepares a function to stage the data. -func handleSideInputs(t *pipepb.PTransform, comps *pipepb.Components, coders map[string]*pipepb.Coder, wk *worker.W) (func(b *worker.B, tid string, watermark mtime.Time), error) { +// handleSideInput returns a closure that will look up the data for a side input appropriate for the given watermark. +func handleSideInput(tid, local, global string, comps *pipepb.Components, coders map[string]*pipepb.Coder, ds *worker.DataService) (func(b *worker.B, watermark mtime.Time), error) { + t := comps.GetTransforms()[tid] sis, err := getSideInputs(t) if err != nil { return nil, err } - var prepSides []func(b *worker.B, tid string, watermark mtime.Time) - - // Get WindowedValue Coders for the transform's input and output PCollections. - for local, global := range t.GetInputs() { - si, ok := sis[local] - if !ok { - continue // This is the main input. - } - // this is a side input - switch si.GetAccessPattern().GetUrn() { - case urns.SideInputIterable: - slog.Debug("urnSideInputIterable", - slog.String("sourceTransform", t.GetUniqueName()), - slog.String("local", local), - slog.String("global", global)) - col := comps.GetPcollections()[global] - ed := collectionPullDecoder(col.GetCoderId(), coders, comps) - wDec, wEnc := getWindowValueCoders(comps, col, coders) - // May be of zero length, but that's OK. Side inputs can be empty. + switch si := sis[local]; si.GetAccessPattern().GetUrn() { + case urns.SideInputIterable: + slog.Debug("urnSideInputIterable", + slog.String("sourceTransform", t.GetUniqueName()), + slog.String("local", local), + slog.String("global", global)) + col := comps.GetPcollections()[global] + ed := collectionPullDecoder(col.GetCoderId(), coders, comps) + wDec, wEnc := getWindowValueCoders(comps, col, coders) + // May be of zero length, but that's OK. Side inputs can be empty. - global, local := global, local - prepSides = append(prepSides, func(b *worker.B, tid string, watermark mtime.Time) { - data := wk.D.GetAllData(global) + global, local := global, local + return func(b *worker.B, watermark mtime.Time) { + data := ds.GetAllData(global) - if b.IterableSideInputData == nil { - b.IterableSideInputData = map[string]map[string]map[typex.Window][][]byte{} - } - if _, ok := b.IterableSideInputData[tid]; !ok { - b.IterableSideInputData[tid] = map[string]map[typex.Window][][]byte{} - } - b.IterableSideInputData[tid][local] = collateByWindows(data, watermark, wDec, wEnc, - func(r io.Reader) [][]byte { - return [][]byte{ed(r)} - }, func(a, b [][]byte) [][]byte { - return append(a, b...) - }) - }) - - case urns.SideInputMultiMap: - slog.Debug("urnSideInputMultiMap", - slog.String("sourceTransform", t.GetUniqueName()), - slog.String("local", local), - slog.String("global", global)) - col := comps.GetPcollections()[global] - - kvc := comps.GetCoders()[col.GetCoderId()] - if kvc.GetSpec().GetUrn() != urns.CoderKV { - return nil, fmt.Errorf("multimap side inputs needs KV coder, got %v", kvc.GetSpec().GetUrn()) + if b.IterableSideInputData == nil { + b.IterableSideInputData = map[string]map[string]map[typex.Window][][]byte{} } + if _, ok := b.IterableSideInputData[tid]; !ok { + b.IterableSideInputData[tid] = map[string]map[typex.Window][][]byte{} + } + b.IterableSideInputData[tid][local] = collateByWindows(data, watermark, wDec, wEnc, + func(r io.Reader) [][]byte { + return [][]byte{ed(r)} + }, func(a, b [][]byte) [][]byte { + return append(a, b...) + }) + }, nil + + case urns.SideInputMultiMap: + slog.Debug("urnSideInputMultiMap", + slog.String("sourceTransform", t.GetUniqueName()), + slog.String("local", local), + slog.String("global", global)) + col := comps.GetPcollections()[global] - kd := collectionPullDecoder(kvc.GetComponentCoderIds()[0], coders, comps) - vd := collectionPullDecoder(kvc.GetComponentCoderIds()[1], coders, comps) - wDec, wEnc := getWindowValueCoders(comps, col, coders) - - global, local := global, local - prepSides = append(prepSides, func(b *worker.B, tid string, watermark mtime.Time) { - // May be of zero length, but that's OK. Side inputs can be empty. - data := wk.D.GetAllData(global) - if b.MultiMapSideInputData == nil { - b.MultiMapSideInputData = map[string]map[string]map[typex.Window]map[string][][]byte{} - } - if _, ok := b.MultiMapSideInputData[tid]; !ok { - b.MultiMapSideInputData[tid] = map[string]map[typex.Window]map[string][][]byte{} - } - b.MultiMapSideInputData[tid][local] = collateByWindows(data, watermark, wDec, wEnc, - func(r io.Reader) map[string][][]byte { - kb := kd(r) - return map[string][][]byte{ - string(kb): {vd(r)}, - } - }, func(a, b map[string][][]byte) map[string][][]byte { - if len(a) == 0 { - return b - } - for k, vs := range b { - a[k] = append(a[k], vs...) - } - return a - }) - }) - default: - return nil, fmt.Errorf("local input %v (global %v) uses accesspattern %v", local, global, si.GetAccessPattern().GetUrn()) + kvc := comps.GetCoders()[col.GetCoderId()] + if kvc.GetSpec().GetUrn() != urns.CoderKV { + return nil, fmt.Errorf("multimap side inputs needs KV coder, got %v", kvc.GetSpec().GetUrn()) } + + kd := collectionPullDecoder(kvc.GetComponentCoderIds()[0], coders, comps) + vd := collectionPullDecoder(kvc.GetComponentCoderIds()[1], coders, comps) + wDec, wEnc := getWindowValueCoders(comps, col, coders) + + global, local := global, local + return func(b *worker.B, watermark mtime.Time) { + // May be of zero length, but that's OK. Side inputs can be empty. + data := ds.GetAllData(global) + if b.MultiMapSideInputData == nil { + b.MultiMapSideInputData = map[string]map[string]map[typex.Window]map[string][][]byte{} + } + if _, ok := b.MultiMapSideInputData[tid]; !ok { + b.MultiMapSideInputData[tid] = map[string]map[typex.Window]map[string][][]byte{} + } + b.MultiMapSideInputData[tid][local] = collateByWindows(data, watermark, wDec, wEnc, + func(r io.Reader) map[string][][]byte { + kb := kd(r) + return map[string][][]byte{ + string(kb): {vd(r)}, + } + }, func(a, b map[string][][]byte) map[string][][]byte { + if len(a) == 0 { + return b + } + for k, vs := range b { + a[k] = append(a[k], vs...) + } + return a + }) + }, nil + default: + return nil, fmt.Errorf("local input %v (global %v) uses accesspattern %v", local, global, si.GetAccessPattern().GetUrn()) } - return func(b *worker.B, tid string, watermark mtime.Time) { - for _, prep := range prepSides { - prep(b, tid, watermark) - } - }, nil } func sourceTransform(parentID string, sourcePortBytes []byte, outPID string) *pipepb.PTransform { diff --git a/sdks/go/pkg/beam/runners/prism/internal/testdofns.go b/sdks/go/pkg/beam/runners/prism/internal/testdofns.go deleted file mode 100644 index 6e7a9d27aee52..0000000000000 --- a/sdks/go/pkg/beam/runners/prism/internal/testdofns.go +++ /dev/null @@ -1,357 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -import ( - "context" - "fmt" - "sort" - "time" - - "github.com/apache/beam/sdks/v2/go/pkg/beam" - "github.com/apache/beam/sdks/v2/go/pkg/beam/core/sdf" - "github.com/apache/beam/sdks/v2/go/pkg/beam/io/rtrackers/offsetrange" - "github.com/apache/beam/sdks/v2/go/pkg/beam/log" - "github.com/apache/beam/sdks/v2/go/pkg/beam/register" - "github.com/google/go-cmp/cmp" -) - -// The Test DoFns live outside of the test files to get coverage information on DoFn -// Lifecycle method execution. This inflates binary size, but ensures the runner is -// exercising the expected feature set. -// -// Once there's enough confidence in the runner, we can move these into a dedicated testing -// package along with the pipelines that use them. - -// Registrations should happen in the test files, so the compiler can prune these -// when they are not in use. - -func dofn1(imp []byte, emit func(int64)) { - emit(1) - emit(2) - emit(3) -} - -func dofn1kv(imp []byte, emit func(int64, int64)) { - emit(0, 1) - emit(0, 2) - emit(0, 3) -} - -func dofn1x2(imp []byte, emitA func(int64), emitB func(int64)) { - emitA(1) - emitA(2) - emitA(3) - emitB(4) - emitB(5) - emitB(6) -} - -func dofn1x5(imp []byte, emitA, emitB, emitC, emitD, emitE func(int64)) { - emitA(1) - emitB(2) - emitC(3) - emitD(4) - emitE(5) - emitA(6) - emitB(7) - emitC(8) - emitD(9) - emitE(10) -} - -func dofn2x1(imp []byte, iter func(*int64) bool, emit func(int64)) { - var v, sum, c int64 - for iter(&v) { - fmt.Println("dofn2x1 v", v, " c ", c) - sum += v - c++ - } - fmt.Println("dofn2x1 sum", sum, "count", c) - emit(sum) -} - -func dofn2x2KV(imp []byte, iter func(*string, *int64) bool, emitK func(string), emitV func(int64)) { - var k string - var v, sum int64 - for iter(&k, &v) { - sum += v - emitK(k) - } - emitV(sum) -} - -func dofnMultiMap(key string, lookup func(string) func(*int64) bool, emitK func(string), emitV func(int64)) { - var v, sum int64 - iter := lookup(key) - for iter(&v) { - sum += v - } - emitK(key) - emitV(sum) -} - -func dofn3x1(sum int64, iter1, iter2 func(*int64) bool, emit func(int64)) { - var v int64 - for iter1(&v) { - sum += v - } - for iter2(&v) { - sum += v - } - emit(sum) -} - -// int64Check validates that within a single bundle, for each window, -// we received the expected int64 values & sends them downstream. -// -// Invalid pattern for general testing, as it will fail -// on other valid execution patterns, like single element bundles. -type int64Check struct { - Name string - Want []int - got map[beam.Window][]int -} - -func (fn *int64Check) StartBundle(_ func(int64)) error { - fn.got = map[beam.Window][]int{} - return nil -} - -func (fn *int64Check) ProcessElement(w beam.Window, v int64, _ func(int64)) { - fn.got[w] = append(fn.got[w], int(v)) -} - -func (fn *int64Check) FinishBundle(_ func(int64)) error { - sort.Ints(fn.Want) - // Check for each window individually. - for _, vs := range fn.got { - sort.Ints(vs) - if d := cmp.Diff(fn.Want, vs); d != "" { - return fmt.Errorf("int64Check[%v] (-want, +got): %v", fn.Name, d) - } - // Clear for subsequent calls. - } - fn.got = nil - return nil -} - -// stringCheck validates that within a single bundle, -// we received the expected string values. -// Re-emits them downstream. -// -// Invalid pattern for general testing, as it will fail -// on other valid execution patterns, like single element bundles. -type stringCheck struct { - Name string - Want []string - got []string -} - -func (fn *stringCheck) ProcessElement(v string, _ func(string)) { - fn.got = append(fn.got, v) -} - -func (fn *stringCheck) FinishBundle(_ func(string)) error { - sort.Strings(fn.got) - sort.Strings(fn.Want) - if d := cmp.Diff(fn.Want, fn.got); d != "" { - return fmt.Errorf("stringCheck[%v] (-want, +got): %v", fn.Name, d) - } - return nil -} - -func dofn2(v int64, emit func(int64)) { - emit(v + 1) -} - -func dofnKV(imp []byte, emit func(string, int64)) { - emit("a", 1) - emit("b", 2) - emit("a", 3) - emit("b", 4) - emit("a", 5) - emit("b", 6) -} - -func dofnKV2(imp []byte, emit func(int64, string)) { - emit(1, "a") - emit(2, "b") - emit(1, "a") - emit(2, "b") - emit(1, "a") - emit(2, "b") -} - -func dofnGBK(k string, vs func(*int64) bool, emit func(int64)) { - var v, sum int64 - for vs(&v) { - sum += v - } - emit(sum) -} - -func dofnGBK2(k int64, vs func(*string) bool, emit func(string)) { - var v, sum string - for vs(&v) { - sum += v - } - emit(sum) -} - -type testRow struct { - A string - B int64 -} - -func dofnKV3(imp []byte, emit func(testRow, testRow)) { - emit(testRow{"a", 1}, testRow{"a", 1}) -} - -func dofnGBK3(k testRow, vs func(*testRow) bool, emit func(string)) { - var v testRow - vs(&v) - emit(fmt.Sprintf("%v: %v", k, v)) -} - -const ( - ns = "localtest" -) - -func dofnSink(ctx context.Context, _ []byte) { - beam.NewCounter(ns, "sunk").Inc(ctx, 73) -} - -func dofn1Counter(ctx context.Context, _ []byte, emit func(int64)) { - beam.NewCounter(ns, "count").Inc(ctx, 1) -} - -func combineIntSum(a, b int64) int64 { - return a + b -} - -// SourceConfig is a struct containing all the configuration options for a -// synthetic source. It should be created via a SourceConfigBuilder, not by -// directly initializing it (the fields are public to allow encoding). -type SourceConfig struct { - NumElements int64 `json:"num_records" beam:"num_records"` - InitialSplits int64 `json:"initial_splits" beam:"initial_splits"` -} - -// intRangeFn is a splittable DoFn for counting from 1 to N. -type intRangeFn struct{} - -// CreateInitialRestriction creates an offset range restriction representing -// the number of elements to emit. -func (fn *intRangeFn) CreateInitialRestriction(config SourceConfig) offsetrange.Restriction { - return offsetrange.Restriction{ - Start: 0, - End: int64(config.NumElements), - } -} - -// SplitRestriction splits restrictions equally according to the number of -// initial splits specified in SourceConfig. Each restriction output by this -// method will contain at least one element, so the number of splits will not -// exceed the number of elements. -func (fn *intRangeFn) SplitRestriction(config SourceConfig, rest offsetrange.Restriction) (splits []offsetrange.Restriction) { - return rest.EvenSplits(int64(config.InitialSplits)) -} - -// RestrictionSize outputs the size of the restriction as the number of elements -// that restriction will output. -func (fn *intRangeFn) RestrictionSize(_ SourceConfig, rest offsetrange.Restriction) float64 { - return rest.Size() -} - -// CreateTracker just creates an offset range restriction tracker for the -// restriction. -func (fn *intRangeFn) CreateTracker(rest offsetrange.Restriction) *sdf.LockRTracker { - return sdf.NewLockRTracker(offsetrange.NewTracker(rest)) -} - -// ProcessElement creates a number of random elements based on the restriction -// tracker received. Each element is a random byte slice key and value, in the -// form of KV<[]byte, []byte>. -func (fn *intRangeFn) ProcessElement(rt *sdf.LockRTracker, config SourceConfig, emit func(int64)) error { - for i := rt.GetRestriction().(offsetrange.Restriction).Start; rt.TryClaim(i); i++ { - // Add 1 since the restrictions are from [0 ,N), but we want [1, N] - emit(i + 1) - } - return nil -} - -func init() { - register.DoFn3x1[*sdf.LockRTracker, []byte, func(int64), sdf.ProcessContinuation](&selfCheckpointingDoFn{}) - register.Emitter1[int64]() -} - -type selfCheckpointingDoFn struct{} - -// CreateInitialRestriction creates the restriction being used by the SDF. In this case, the range -// of values produced by the restriction is [Start, End). -func (fn *selfCheckpointingDoFn) CreateInitialRestriction(_ []byte) offsetrange.Restriction { - return offsetrange.Restriction{ - Start: int64(0), - End: int64(10), - } -} - -// CreateTracker wraps the given restriction into a LockRTracker type. -func (fn *selfCheckpointingDoFn) CreateTracker(rest offsetrange.Restriction) *sdf.LockRTracker { - return sdf.NewLockRTracker(offsetrange.NewTracker(rest)) -} - -// RestrictionSize returns the size of the current restriction -func (fn *selfCheckpointingDoFn) RestrictionSize(_ []byte, rest offsetrange.Restriction) float64 { - return rest.Size() -} - -// SplitRestriction modifies the offsetrange.Restriction's sized restriction function to produce a size-zero restriction -// at the end of execution. -func (fn *selfCheckpointingDoFn) SplitRestriction(_ []byte, rest offsetrange.Restriction) []offsetrange.Restriction { - size := int64(3) - s := rest.Start - var splits []offsetrange.Restriction - for e := s + size; e <= rest.End; s, e = e, e+size { - splits = append(splits, offsetrange.Restriction{Start: s, End: e}) - } - splits = append(splits, offsetrange.Restriction{Start: s, End: rest.End}) - return splits -} - -// ProcessElement continually gets the start position of the restriction and emits it as an int64 value before checkpointing. -// This causes the restriction to be split after the claimed work and produce no primary roots. -func (fn *selfCheckpointingDoFn) ProcessElement(rt *sdf.LockRTracker, _ []byte, emit func(int64)) sdf.ProcessContinuation { - position := rt.GetRestriction().(offsetrange.Restriction).Start - - for { - if rt.TryClaim(position) { - // Successful claim, emit the value and move on. - emit(position) - position++ - } else if rt.GetError() != nil || rt.IsDone() { - // Stop processing on error or completion - if err := rt.GetError(); err != nil { - log.Errorf(context.Background(), "error in restriction tracker, got %v", err) - } - return sdf.StopProcessing() - } else { - // Resume later. - return sdf.ResumeProcessingIn(5 * time.Second) - } - } -} diff --git a/sdks/go/pkg/beam/runners/prism/internal/testdofns_test.go b/sdks/go/pkg/beam/runners/prism/internal/testdofns_test.go index 5ca4eb9fd4c5a..334d74fcae1d2 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/testdofns_test.go +++ b/sdks/go/pkg/beam/runners/prism/internal/testdofns_test.go @@ -13,17 +13,26 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package internal_test import ( + "context" + "fmt" + "sort" + "time" + "github.com/apache/beam/sdks/v2/go/pkg/beam" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/sdf" + "github.com/apache/beam/sdks/v2/go/pkg/beam/io/rtrackers/offsetrange" + "github.com/apache/beam/sdks/v2/go/pkg/beam/log" "github.com/apache/beam/sdks/v2/go/pkg/beam/register" + "github.com/google/go-cmp/cmp" ) // Test DoFns are registered in the test file, to allow them to be pruned // by the compiler outside of test use. func init() { + register.Function2x0(dofnEmpty) register.Function2x0(dofn1) register.Function2x0(dofn1kv) register.Function3x0(dofn1x2) @@ -41,12 +50,15 @@ func init() { register.Function2x0(dofnKV2) register.Function3x0(dofnGBK) register.Function3x0(dofnGBK2) + register.Function3x0(dofnGBKKV) + register.Emitter2[string, int64]() register.DoFn3x0[beam.Window, int64, func(int64)]((*int64Check)(nil)) register.DoFn2x0[string, func(string)]((*stringCheck)(nil)) register.Function2x0(dofnKV3) register.Function3x0(dofnGBK3) register.Function3x0(dofn1Counter) register.Function2x0(dofnSink) + register.Function3x1(doFnFail) register.Function2x1(combineIntSum) @@ -54,3 +66,336 @@ func init() { register.Emitter1[int64]() register.Emitter2[int64, int64]() } + +func dofnEmpty(imp []byte, emit func(int64)) { +} + +func dofn1(imp []byte, emit func(int64)) { + emit(1) + emit(2) + emit(3) +} + +func dofn1kv(imp []byte, emit func(int64, int64)) { + emit(0, 1) + emit(0, 2) + emit(0, 3) +} + +func dofn1x2(imp []byte, emitA func(int64), emitB func(int64)) { + emitA(1) + emitA(2) + emitA(3) + emitB(4) + emitB(5) + emitB(6) +} + +func dofn1x5(imp []byte, emitA, emitB, emitC, emitD, emitE func(int64)) { + emitA(1) + emitB(2) + emitC(3) + emitD(4) + emitE(5) + emitA(6) + emitB(7) + emitC(8) + emitD(9) + emitE(10) +} + +func dofn2x1(imp []byte, iter func(*int64) bool, emit func(int64)) { + var v, sum, c int64 + for iter(&v) { + fmt.Println("dofn2x1 v", v, " c ", c) + sum += v + c++ + } + fmt.Println("dofn2x1 sum", sum, "count", c) + emit(sum) +} + +func dofn2x2KV(imp []byte, iter func(*string, *int64) bool, emitK func(string), emitV func(int64)) { + var k string + var v, sum int64 + for iter(&k, &v) { + sum += v + emitK(k) + } + emitV(sum) +} + +func dofnMultiMap(key string, lookup func(string) func(*int64) bool, emitK func(string), emitV func(int64)) { + var v, sum int64 + iter := lookup(key) + for iter(&v) { + sum += v + } + emitK(key) + emitV(sum) +} + +func dofn3x1(sum int64, iter1, iter2 func(*int64) bool, emit func(int64)) { + var v int64 + for iter1(&v) { + sum += v + } + for iter2(&v) { + sum += v + } + emit(sum) +} + +// int64Check validates that within a single bundle, for each window, +// we received the expected int64 values & sends them downstream. +// +// Invalid pattern for general testing, as it will fail +// on other valid execution patterns, like single element bundles. +type int64Check struct { + Name string + Want []int + got map[beam.Window][]int +} + +func (fn *int64Check) StartBundle(_ func(int64)) error { + fn.got = map[beam.Window][]int{} + return nil +} + +func (fn *int64Check) ProcessElement(w beam.Window, v int64, _ func(int64)) { + fn.got[w] = append(fn.got[w], int(v)) +} + +func (fn *int64Check) FinishBundle(_ func(int64)) error { + sort.Ints(fn.Want) + // Check for each window individually. + for _, vs := range fn.got { + sort.Ints(vs) + if d := cmp.Diff(fn.Want, vs); d != "" { + return fmt.Errorf("int64Check[%v] (-want, +got): %v", fn.Name, d) + } + // Clear for subsequent calls. + } + fn.got = nil + return nil +} + +// stringCheck validates that within a single bundle, +// we received the expected string values. +// Re-emits them downstream. +// +// Invalid pattern for general testing, as it will fail +// on other valid execution patterns, like single element bundles. +type stringCheck struct { + Name string + Want []string + got []string +} + +func (fn *stringCheck) ProcessElement(v string, _ func(string)) { + fn.got = append(fn.got, v) +} + +func (fn *stringCheck) FinishBundle(_ func(string)) error { + sort.Strings(fn.got) + sort.Strings(fn.Want) + if d := cmp.Diff(fn.Want, fn.got); d != "" { + return fmt.Errorf("stringCheck[%v] (-want, +got): %v", fn.Name, d) + } + return nil +} + +func dofn2(v int64, emit func(int64)) { + emit(v + 1) +} + +func dofnKV(imp []byte, emit func(string, int64)) { + emit("a", 1) + emit("b", 2) + emit("a", 3) + emit("b", 4) + emit("a", 5) + emit("b", 6) +} + +func dofnKV2(imp []byte, emit func(int64, string)) { + emit(1, "a") + emit(2, "b") + emit(1, "a") + emit(2, "b") + emit(1, "a") + emit(2, "b") +} + +func dofnGBK(k string, vs func(*int64) bool, emit func(int64)) { + var v, sum int64 + for vs(&v) { + sum += v + } + emit(sum) +} + +func dofnGBK2(k int64, vs func(*string) bool, emit func(string)) { + var v, sum string + for vs(&v) { + sum += v + } + emit(sum) +} + +func dofnGBKKV(k string, vs func(*int64) bool, emit func(string, int64)) { + var v, sum int64 + for vs(&v) { + sum += v + } + emit(k, sum) +} + +type testRow struct { + A string + B int64 +} + +func dofnKV3(imp []byte, emit func(testRow, testRow)) { + emit(testRow{"a", 1}, testRow{"a", 1}) +} + +func dofnGBK3(k testRow, vs func(*testRow) bool, emit func(string)) { + var v testRow + vs(&v) + emit(fmt.Sprintf("%v: %v", k, v)) +} + +const ( + ns = "localtest" +) + +func dofnSink(ctx context.Context, _ []byte) { + beam.NewCounter(ns, "sunk").Inc(ctx, 73) +} + +func dofn1Counter(ctx context.Context, _ []byte, emit func(int64)) { + beam.NewCounter(ns, "count").Inc(ctx, 1) +} + +func doFnFail(ctx context.Context, _ []byte, emit func(int64)) error { + beam.NewCounter(ns, "count").Inc(ctx, 1) + return fmt.Errorf("doFnFail: failing as intended") +} + +func combineIntSum(a, b int64) int64 { + return a + b +} + +// SourceConfig is a struct containing all the configuration options for a +// synthetic source. It should be created via a SourceConfigBuilder, not by +// directly initializing it (the fields are public to allow encoding). +type SourceConfig struct { + NumElements int64 `json:"num_records" beam:"num_records"` + InitialSplits int64 `json:"initial_splits" beam:"initial_splits"` +} + +// intRangeFn is a splittable DoFn for counting from 1 to N. +type intRangeFn struct{} + +// CreateInitialRestriction creates an offset range restriction representing +// the number of elements to emit. +func (fn *intRangeFn) CreateInitialRestriction(config SourceConfig) offsetrange.Restriction { + return offsetrange.Restriction{ + Start: 0, + End: int64(config.NumElements), + } +} + +// SplitRestriction splits restrictions equally according to the number of +// initial splits specified in SourceConfig. Each restriction output by this +// method will contain at least one element, so the number of splits will not +// exceed the number of elements. +func (fn *intRangeFn) SplitRestriction(config SourceConfig, rest offsetrange.Restriction) (splits []offsetrange.Restriction) { + return rest.EvenSplits(int64(config.InitialSplits)) +} + +// RestrictionSize outputs the size of the restriction as the number of elements +// that restriction will output. +func (fn *intRangeFn) RestrictionSize(_ SourceConfig, rest offsetrange.Restriction) float64 { + return rest.Size() +} + +// CreateTracker just creates an offset range restriction tracker for the +// restriction. +func (fn *intRangeFn) CreateTracker(rest offsetrange.Restriction) *sdf.LockRTracker { + return sdf.NewLockRTracker(offsetrange.NewTracker(rest)) +} + +// ProcessElement creates a number of random elements based on the restriction +// tracker received. Each element is a random byte slice key and value, in the +// form of KV<[]byte, []byte>. +func (fn *intRangeFn) ProcessElement(rt *sdf.LockRTracker, config SourceConfig, emit func(int64)) error { + for i := rt.GetRestriction().(offsetrange.Restriction).Start; rt.TryClaim(i); i++ { + // Add 1 since the restrictions are from [0 ,N), but we want [1, N] + emit(i + 1) + } + return nil +} + +func init() { + register.DoFn3x1[*sdf.LockRTracker, []byte, func(int64), sdf.ProcessContinuation](&selfCheckpointingDoFn{}) + register.Emitter1[int64]() +} + +type selfCheckpointingDoFn struct{} + +// CreateInitialRestriction creates the restriction being used by the SDF. In this case, the range +// of values produced by the restriction is [Start, End). +func (fn *selfCheckpointingDoFn) CreateInitialRestriction(_ []byte) offsetrange.Restriction { + return offsetrange.Restriction{ + Start: int64(0), + End: int64(10), + } +} + +// CreateTracker wraps the given restriction into a LockRTracker type. +func (fn *selfCheckpointingDoFn) CreateTracker(rest offsetrange.Restriction) *sdf.LockRTracker { + return sdf.NewLockRTracker(offsetrange.NewTracker(rest)) +} + +// RestrictionSize returns the size of the current restriction +func (fn *selfCheckpointingDoFn) RestrictionSize(_ []byte, rest offsetrange.Restriction) float64 { + return rest.Size() +} + +// SplitRestriction modifies the offsetrange.Restriction's sized restriction function to produce a size-zero restriction +// at the end of execution. +func (fn *selfCheckpointingDoFn) SplitRestriction(_ []byte, rest offsetrange.Restriction) []offsetrange.Restriction { + size := int64(3) + s := rest.Start + var splits []offsetrange.Restriction + for e := s + size; e <= rest.End; s, e = e, e+size { + splits = append(splits, offsetrange.Restriction{Start: s, End: e}) + } + splits = append(splits, offsetrange.Restriction{Start: s, End: rest.End}) + return splits +} + +// ProcessElement continually gets the start position of the restriction and emits it as an int64 value before checkpointing. +// This causes the restriction to be split after the claimed work and produce no primary roots. +func (fn *selfCheckpointingDoFn) ProcessElement(rt *sdf.LockRTracker, _ []byte, emit func(int64)) sdf.ProcessContinuation { + position := rt.GetRestriction().(offsetrange.Restriction).Start + + for { + if rt.TryClaim(position) { + // Successful claim, emit the value and move on. + emit(position) + position++ + } else if rt.GetError() != nil || rt.IsDone() { + // Stop processing on error or completion + if err := rt.GetError(); err != nil { + log.Errorf(context.Background(), "error in restriction tracker, got %v", err) + } + return sdf.StopProcessing() + } else { + // Resume later. + return sdf.ResumeProcessingIn(5 * time.Second) + } + } +} diff --git a/sdks/go/pkg/beam/runners/prism/internal/unimplemented_test.go b/sdks/go/pkg/beam/runners/prism/internal/unimplemented_test.go index 3ff8eb842077e..5f8d387599982 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/unimplemented_test.go +++ b/sdks/go/pkg/beam/runners/prism/internal/unimplemented_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package internal_test import ( "context" @@ -41,11 +41,7 @@ func TestUnimplemented(t *testing.T) { tests := []struct { pipeline func(s beam.Scope) }{ - // These tests don't terminate, so can't be run. // {pipeline: primitives.Drain}, // Can't test drain automatically yet. - // {pipeline: primitives.Checkpoints}, // Doesn't self terminate? - // {pipeline: primitives.Flatten}, // Times out, should be quick. - // {pipeline: primitives.FlattenDup}, // Times out, should be quick. {pipeline: primitives.TestStreamBoolSequence}, {pipeline: primitives.TestStreamByteSliceSequence}, @@ -72,10 +68,6 @@ func TestUnimplemented(t *testing.T) { {pipeline: primitives.TriggerOrFinally}, {pipeline: primitives.TriggerRepeat}, - // Reshuffle (Due to missing windowing strategy features) - {pipeline: primitives.Reshuffle}, - {pipeline: primitives.ReshuffleKV}, - // State API {pipeline: primitives.BagStateParDo}, {pipeline: primitives.BagStateParDoClear}, @@ -102,3 +94,33 @@ func TestUnimplemented(t *testing.T) { }) } } + +// TODO move these to a more appropriate location. +// Mostly placed here to have structural parity with the above test +// and make it easy to move them to a "it works" expectation. +func TestImplemented(t *testing.T) { + initRunner(t) + + tests := []struct { + pipeline func(s beam.Scope) + }{ + {pipeline: primitives.Reshuffle}, + {pipeline: primitives.Flatten}, + {pipeline: primitives.FlattenDup}, + {pipeline: primitives.Checkpoints}, + + {pipeline: primitives.CoGBK}, + {pipeline: primitives.ReshuffleKV}, + } + + for _, test := range tests { + t.Run(intTestName(test.pipeline), func(t *testing.T) { + p, s := beam.NewPipelineWithRoot() + test.pipeline(s) + _, err := executeWithT(context.Background(), t, p) + if err != nil { + t.Fatalf("pipeline failed, but feature should be implemented in Prism: %v", err) + } + }) + } +} diff --git a/sdks/go/pkg/beam/runners/prism/internal/urns/urns.go b/sdks/go/pkg/beam/runners/prism/internal/urns/urns.go index 7a5fee21fc7b9..bf1e36656661b 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/urns/urns.go +++ b/sdks/go/pkg/beam/runners/prism/internal/urns/urns.go @@ -57,6 +57,9 @@ var ( // SDK transforms. TransformParDo = ptUrn(pipepb.StandardPTransforms_PAR_DO) TransformCombinePerKey = ctUrn(pipepb.StandardPTransforms_COMBINE_PER_KEY) + TransformCombineGlobally = ctUrn(pipepb.StandardPTransforms_COMBINE_GLOBALLY) + TransformReshuffle = ctUrn(pipepb.StandardPTransforms_RESHUFFLE) + TransformCombineGroupedValues = cmbtUrn(pipepb.StandardPTransforms_COMBINE_GROUPED_VALUES) TransformPreCombine = cmbtUrn(pipepb.StandardPTransforms_COMBINE_PER_KEY_PRECOMBINE) TransformMerge = cmbtUrn(pipepb.StandardPTransforms_COMBINE_PER_KEY_MERGE_ACCUMULATORS) TransformExtract = cmbtUrn(pipepb.StandardPTransforms_COMBINE_PER_KEY_EXTRACT_OUTPUTS) diff --git a/sdks/go/pkg/beam/runners/prism/internal/web/assets/style.css b/sdks/go/pkg/beam/runners/prism/internal/web/assets/style.css index 9176f5fd0dc93..d252dc020e639 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/web/assets/style.css +++ b/sdks/go/pkg/beam/runners/prism/internal/web/assets/style.css @@ -83,14 +83,37 @@ header { align-items: center; } +footer { + width: 100%; + height: 50px; + position: absolute; + bottom: 0; + left: 0; + right: 0; + display: flex; + justify-content: space-between; + background-color: var(--beam-light-orange); + padding: 5px 10px; + align-items: center; +} + .logo { color: var(--beam-white); } +#page-container { + position: relative; + min-height: 100vh; +} + +#content-wrap { + padding-bottom: 2.5rem; /* Footer height */ +} + .container { width: 100%; margin: 0 auto; - padding: 80px 20px 40px; + padding: 40px 20px 0px; } .child { @@ -118,6 +141,53 @@ header { padding: 12px 15px; } +/* Tooltip container */ +.tooltip { + display: inline-block; + border-bottom: 1px dotted var(--beam-black); +} + +/* Tooltip text */ +.tooltip .tooltiptext { + visibility: hidden; + width: max-content; + max-width: 400px; + background-color: var(--dark-grey); + color: var(--beam-white); + text-align: left; + padding: 5px 10px; + border-radius: 6px; + + /* Position the tooltip text */ + position: absolute; + z-index: 1; + bottom: 125%; + left: 50%; + margin-left: -60px; + + /* Fade in tooltip */ + opacity: 0; + transition: opacity 0.3s; +} + +/* Tooltip arrow */ +.tooltip .tooltiptext::after { + content: ""; + position: absolute; + top: 100%; + left: 18%; + margin-left: -5px; + border-width: 5px; + border-style: solid; + border-color: var(--dark-grey) transparent transparent transparent; +} + +/* Show the tooltip text when you mouse over the tooltip container */ +.tooltip:hover .tooltiptext { + visibility: visible; + opacity: 1; +} + @media screen and (max-width: 550px) { header { flex-direction: column; @@ -125,6 +195,12 @@ header { padding-bottom: 10px; } + footer { + flex-direction: column; + height: auto; + padding-top: 10px; + } + .logo { display: inline-block; margin-bottom: 10px; diff --git a/sdks/go/pkg/beam/runners/prism/internal/web/debugz.go b/sdks/go/pkg/beam/runners/prism/internal/web/debugz.go new file mode 100644 index 0000000000000..015a9103134aa --- /dev/null +++ b/sdks/go/pkg/beam/runners/prism/internal/web/debugz.go @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package web + +import ( + "fmt" + "runtime/debug" + "runtime/metrics" + "runtime/pprof" + "strings" + "time" + + "github.com/dustin/go-humanize" +) + +type debugzData struct { + Metrics []goRuntimeMetric + + errorHolder +} + +type goRuntimeMetric struct { + Name, Value, Description string +} + +func dumpMetrics() debugzData { + // Get descriptions for all supported metrics. + descs := metrics.All() + + // Create a sample for each metric. + samples := make([]metrics.Sample, len(descs)) + for i := range samples { + samples[i].Name = descs[i].Name + } + + // Sample the metrics. Re-use the samples slice if you can! + metrics.Read(samples) + + var data debugzData + + // Iterate over all results. + for i, sample := range samples { + // Pull out the name and value. + name, value := sample.Name, sample.Value + + m := goRuntimeMetric{ + Name: strings.TrimSpace(name), + Description: descs[i].Description, + } + + // Handle each sample. + switch value.Kind() { + case metrics.KindUint64: + if strings.HasSuffix(name, "bytes") { + m.Value = humanize.Bytes(value.Uint64()) + } else { + m.Value = humanize.FormatInteger("", int(value.Uint64())) + } + case metrics.KindFloat64: + if strings.HasSuffix(name, "seconds") { + m.Value = time.Duration(float64(time.Second) * value.Float64()).String() + } else { + m.Value = humanize.FormatFloat("", value.Float64()) + } + case metrics.KindFloat64Histogram: + m.Value = fmt.Sprintf("%f", medianBucket(value.Float64Histogram())) + // The histogram may be quite large, so let's just pull out + // a crude estimate for the median for the sake of this example. + case metrics.KindBad: + // This should never happen because all metrics are supported + // by construction. + m.Value = "bug in runtime/metrics package: KindBad" + default: + // This may happen as new metrics get added. + // + // The safest thing to do here is to simply log it somewhere + // as something to look into, but ignore it for now. + // In the worst case, you might temporarily miss out on a new metric. + m.Value = fmt.Sprintf("%s: unexpected metric Kind: %v - %s\n", name, value.Kind(), descs[i].Description) + } + data.Metrics = append(data.Metrics, m) + } + + var b strings.Builder + buildInfo(&b) + + data.Metrics = append(data.Metrics, goRuntimeMetric{ + Name: "BUILD INFO", + Value: b.String(), + Description: "result from runtime/debug.ReadBuildInfo()", + }) + + b.Reset() + goroutineDump(&b) + data.Metrics = append(data.Metrics, goRuntimeMetric{ + Name: "GOROUTINES", + Value: b.String(), + Description: "consolidated active goroutines", + }) + + b.Reset() + profiles(&b) + + data.Metrics = append(data.Metrics, goRuntimeMetric{ + Name: "PROFILES", + Value: b.String(), + Description: "List of available runtime/pprof profiles.", + }) + return data +} + +func goroutineDump(statusInfo *strings.Builder) { + profile := pprof.Lookup("goroutine") + if profile != nil { + profile.WriteTo(statusInfo, 1) + } +} + +func buildInfo(statusInfo *strings.Builder) { + if info, ok := debug.ReadBuildInfo(); ok { + statusInfo.WriteString(info.String()) + } +} + +func profiles(statusInfo *strings.Builder) { + ps := pprof.Profiles() + statusInfo.WriteString(fmt.Sprintf("profiles %v\n", len(ps))) + for _, p := range ps { + statusInfo.WriteString(p.Name()) + statusInfo.WriteRune('\n') + } +} + +func medianBucket(h *metrics.Float64Histogram) float64 { + total := uint64(0) + for _, count := range h.Counts { + total += count + } + thresh := total / 2 + total = 0 + for i, count := range h.Counts { + total += count + if total >= thresh { + return h.Buckets[i] + } + } + panic("should not happen") +} diff --git a/sdks/go/pkg/beam/runners/prism/internal/web/debugz.html b/sdks/go/pkg/beam/runners/prism/internal/web/debugz.html new file mode 100644 index 0000000000000..175f44da7447d --- /dev/null +++ b/sdks/go/pkg/beam/runners/prism/internal/web/debugz.html @@ -0,0 +1,52 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. See accompanying LICENSE file. +*/}} + + + + + + + + Debug Metrics - Beam Prism + + + + +
    +
    + +
    +
    + {{ if .Error}}{{.Error}}{{end}} + + + + + + {{ range .Metrics }} + + + + + {{ else }} + + + + {{ end }} +
    IDValue
    +
    {{ .Name }} + {{ .Description }} +
    +
    {{ .Value }}
    No metrics have been returned.
    +
    +
    + \ No newline at end of file diff --git a/sdks/go/pkg/beam/runners/prism/internal/web/index.html b/sdks/go/pkg/beam/runners/prism/internal/web/index.html index 1dc6282cfa9e7..1aa0ed719d87b 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/web/index.html +++ b/sdks/go/pkg/beam/runners/prism/internal/web/index.html @@ -22,30 +22,35 @@ -
    -
    - -
    -
    - {{ if .Error}}{{.Error}}{{end}} - - - - - - - {{ range .Jobs }} - - - - - - {{ else }} - - - - {{ end }} -
    IDNameState
    {{ .JobId }}{{ .JobName }}{{ .State }}
    No jobs have been run.
    -
    +
    +
    +
    + +
    +
    + {{ if .Error}}{{.Error}}{{end}} + + + + + + + {{ range .Jobs }} + + + + + + {{ else }} + + + + {{ end }} +
    IDNameState
    {{ .JobId }}{{ .JobName }}{{ .State }}
    No jobs have been run.
    +
    +
    +
    \ No newline at end of file diff --git a/sdks/go/pkg/beam/runners/prism/internal/web/web.go b/sdks/go/pkg/beam/runners/prism/internal/web/web.go index a6a8713bce733..b7afad35aeeee 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/web/web.go +++ b/sdks/go/pkg/beam/runners/prism/internal/web/web.go @@ -46,12 +46,16 @@ var indexTemplate string //go:embed jobdetails.html var jobTemplate string +//go:embed debugz.html +var debugzTemplate string + //go:embed assets/* var assets embed.FS var ( - indexPage = template.Must(template.New("index").Parse(indexTemplate)) - jobPage = template.Must(template.New("job").Parse(jobTemplate)) + indexPage = template.Must(template.New("index").Parse(indexTemplate)) + jobPage = template.Must(template.New("job").Parse(jobTemplate)) + debugzPage = template.Must(template.New("debugz").Parse(debugzTemplate)) ) type pTransform struct { @@ -184,6 +188,10 @@ func (h *jobDetailsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { trs := pipeResp.GetPipeline().GetComponents().GetTransforms() col2T, topo := preprocessTransforms(trs) + counters := toTransformMap(results.AllMetrics().Counters()) + distributions := toTransformMap(results.AllMetrics().Distributions()) + msecs := toTransformMap(results.AllMetrics().Msecs()) + data.Transforms = make([]pTransform, 0, len(trs)) for _, id := range topo { pt := trs[id] @@ -220,6 +228,29 @@ func (h *jobDetailsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { strMets = append(strMets, outMets...) } + var msecMets []string + // TODO: Figure out where uniquename or id is being used in prism. It should be all global transform ID to faciliate lookups. + for _, msec := range msecs[pt.GetUniqueName()] { + msecMets = append(msecMets, fmt.Sprintf("\n- %+v", msec.Result())) + } + if len(msecMets) > 0 { + strMets = append(strMets, "Profiling metrics") + strMets = append(strMets, msecMets...) + } + + var userMetrics []string + for _, ctr := range counters[pt.GetUniqueName()] { + userMetrics = append(userMetrics, fmt.Sprintf("\n- %s.%s: %v", ctr.Namespace(), ctr.Name(), ctr.Result())) + } + for _, dist := range distributions[pt.GetUniqueName()] { + userMetrics = append(userMetrics, fmt.Sprintf("\n- %s.%s: %+v", dist.Namespace(), dist.Name(), dist.Result())) + } + + if len(userMetrics) > 0 { + strMets = append(strMets, "User metrics") + strMets = append(strMets, userMetrics...) + } + data.Transforms = append(data.Transforms, pTransform{ ID: id, Transform: pt, @@ -230,6 +261,14 @@ func (h *jobDetailsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { renderPage(jobPage, &data, w) } +func toTransformMap[E interface{ Transform() string }](mets []E) map[string][]E { + ret := map[string][]E{} + for _, met := range mets { + ret[met.Transform()] = append(ret[met.Transform()], met) + } + return ret +} + type pcolParent struct { L string T *pipepb.PTransform @@ -240,7 +279,10 @@ type pcolParent struct { func preprocessTransforms(trs map[string]*pipepb.PTransform) (map[string]pcolParent, []string) { ret := map[string]pcolParent{} var leaves []string - for id, t := range trs { + keys := maps.Keys(trs) + sort.Strings(keys) + for _, id := range keys { + t := trs[id] // Skip composites at this time. if len(t.GetSubtransforms()) > 0 { continue @@ -284,6 +326,14 @@ func (h *jobsConsoleHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { renderPage(indexPage, data, w) } +type debugzHandler struct { +} + +func (h *debugzHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + data := dumpMetrics() + renderPage(debugzPage, &data, w) +} + // Initialize the web client to talk to the given Job Management Client. func Initialize(ctx context.Context, port int, jobcli jobpb.JobServiceClient) error { assetsFs := http.FileServer(http.FS(assets)) @@ -291,6 +341,7 @@ func Initialize(ctx context.Context, port int, jobcli jobpb.JobServiceClient) er mux.Handle("/assets/", assetsFs) mux.Handle("/job/", &jobDetailsHandler{Jobcli: jobcli}) + mux.Handle("/debugz", &debugzHandler{}) mux.Handle("/", &jobsConsoleHandler{Jobcli: jobcli}) endpoint := fmt.Sprintf("localhost:%d", port) diff --git a/sdks/go/pkg/beam/runners/prism/internal/worker/bundle.go b/sdks/go/pkg/beam/runners/prism/internal/worker/bundle.go index 77f7094a97e97..573bdf4aeb9db 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/worker/bundle.go +++ b/sdks/go/pkg/beam/runners/prism/internal/worker/bundle.go @@ -16,6 +16,8 @@ package worker import ( + "context" + "fmt" "sync/atomic" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex" @@ -50,10 +52,9 @@ type B struct { dataSema atomic.Int32 OutputData engine.TentativeData - // TODO move response channel to an atomic and an additional - // block on the DataWait channel, to allow progress & splits for - // no output DoFns. - Resp chan *fnpb.ProcessBundleResponse + Resp chan *fnpb.ProcessBundleResponse + BundleErr error + responded bool SinkToPCollection map[string]string } @@ -86,6 +87,16 @@ func (b *B) LogValue() slog.Value { } func (b *B) Respond(resp *fnpb.InstructionResponse) { + if b.responded { + slog.Warn("additional bundle response", "bundle", b, "resp", resp) + return + } + b.responded = true + if resp.GetError() != "" { + b.BundleErr = fmt.Errorf("bundle %v %v failed:%v", resp.GetInstructionId(), b.PBDID, resp.GetError()) + close(b.Resp) + return + } b.Resp <- resp.GetProcessBundle() } @@ -97,7 +108,7 @@ func (b *B) Respond(resp *fnpb.InstructionResponse) { // // While this method mostly manipulates a W, putting it on a B avoids mixing the workers // public GRPC APIs up with local calls. -func (b *B) ProcessOn(wk *W) <-chan struct{} { +func (b *B) ProcessOn(ctx context.Context, wk *W) <-chan struct{} { wk.mu.Lock() wk.activeInstructions[b.InstID] = b wk.mu.Unlock() @@ -116,7 +127,8 @@ func (b *B) ProcessOn(wk *W) <-chan struct{} { // TODO: make batching decisions. for i, d := range b.InputData { - wk.DataReqs <- &fnpb.Elements{ + select { + case wk.DataReqs <- &fnpb.Elements{ Data: []*fnpb.Elements_Data{ { InstructionId: b.InstID, @@ -125,6 +137,10 @@ func (b *B) ProcessOn(wk *W) <-chan struct{} { IsLast: i+1 == len(b.InputData), }, }, + }: + case <-ctx.Done(): + b.DataDone() + return b.DataWait } } return b.DataWait @@ -137,18 +153,24 @@ func (b *B) Cleanup(wk *W) { wk.mu.Unlock() } -func (b *B) Progress(wk *W) *fnpb.ProcessBundleProgressResponse { - return wk.sendInstruction(&fnpb.InstructionRequest{ +// Progress sends a progress request for the given bundle to the passed in worker, blocking on the response. +func (b *B) Progress(ctx context.Context, wk *W) (*fnpb.ProcessBundleProgressResponse, error) { + resp := wk.sendInstruction(ctx, &fnpb.InstructionRequest{ Request: &fnpb.InstructionRequest_ProcessBundleProgress{ ProcessBundleProgress: &fnpb.ProcessBundleProgressRequest{ InstructionId: b.InstID, }, }, - }).GetProcessBundleProgress() + }) + if resp.GetError() != "" { + return nil, fmt.Errorf("progress[%v] error from SDK: %v", b.InstID, resp.GetError()) + } + return resp.GetProcessBundleProgress(), nil } -func (b *B) Split(wk *W, fraction float64, allowedSplits []int64) *fnpb.ProcessBundleSplitResponse { - return wk.sendInstruction(&fnpb.InstructionRequest{ +// Split sends a split request for the given bundle to the passed in worker, blocking on the response. +func (b *B) Split(ctx context.Context, wk *W, fraction float64, allowedSplits []int64) (*fnpb.ProcessBundleSplitResponse, error) { + resp := wk.sendInstruction(ctx, &fnpb.InstructionRequest{ Request: &fnpb.InstructionRequest_ProcessBundleSplit{ ProcessBundleSplit: &fnpb.ProcessBundleSplitRequest{ InstructionId: b.InstID, @@ -161,5 +183,9 @@ func (b *B) Split(wk *W, fraction float64, allowedSplits []int64) *fnpb.ProcessB }, }, }, - }).GetProcessBundleSplit() + }) + if resp.GetError() != "" { + return nil, fmt.Errorf("split[%v] error from SDK: %v", b.InstID, resp.GetError()) + } + return resp.GetProcessBundleSplit(), nil } diff --git a/sdks/go/pkg/beam/runners/prism/internal/worker/bundle_test.go b/sdks/go/pkg/beam/runners/prism/internal/worker/bundle_test.go index bfef68734260b..ba5b10f5fd39a 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/worker/bundle_test.go +++ b/sdks/go/pkg/beam/runners/prism/internal/worker/bundle_test.go @@ -17,12 +17,13 @@ package worker import ( "bytes" + "context" "sync" "testing" ) func TestBundle_ProcessOn(t *testing.T) { - wk := New("test") + wk := New("test", "testEnv") b := &B{ InstID: "testInst", PBDID: "testPBDID", @@ -33,7 +34,7 @@ func TestBundle_ProcessOn(t *testing.T) { var completed sync.WaitGroup completed.Add(1) go func() { - b.ProcessOn(wk) + b.ProcessOn(context.Background(), wk) completed.Done() }() b.DataDone() diff --git a/sdks/go/pkg/beam/runners/prism/internal/worker/worker.go b/sdks/go/pkg/beam/runners/prism/internal/worker/worker.go index 3bd82807f2127..4968c9eb433e3 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/worker/worker.go +++ b/sdks/go/pkg/beam/runners/prism/internal/worker/worker.go @@ -22,7 +22,10 @@ import ( "context" "fmt" "io" + "math" "net" + "strconv" + "strings" "sync" "sync/atomic" @@ -40,6 +43,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/protobuf/types/known/structpb" ) // A W manages worker environments, sending them work @@ -52,14 +56,19 @@ type W struct { fnpb.UnimplementedBeamFnLoggingServer fnpb.UnimplementedProvisionServiceServer - ID string + ID, Env string + + JobKey, ArtifactEndpoint string + EnvPb *pipepb.Environment + PipelineOptions *structpb.Struct // Server management lis net.Listener server *grpc.Server // These are the ID sources - inst, bund uint64 + inst uint64 + connected, stopped atomic.Bool InstReqs chan *fnpb.InstructionRequest DataReqs chan *fnpb.Elements @@ -67,8 +76,6 @@ type W struct { mu sync.Mutex activeInstructions map[string]controlResponder // Active instructions keyed by InstructionID Descriptors map[string]*fnpb.ProcessBundleDescriptor // Stages keyed by PBDID - - D *DataService } type controlResponder interface { @@ -76,14 +83,17 @@ type controlResponder interface { } // New starts the worker server components of FnAPI Execution. -func New(id string) *W { +func New(id, env string) *W { lis, err := net.Listen("tcp", ":0") if err != nil { panic(fmt.Sprintf("failed to listen: %v", err)) } - var opts []grpc.ServerOption + opts := []grpc.ServerOption{ + grpc.MaxRecvMsgSize(math.MaxInt32), + } wk := &W{ ID: id, + Env: env, lis: lis, server: grpc.NewServer(opts...), @@ -92,19 +102,19 @@ func New(id string) *W { activeInstructions: make(map[string]controlResponder), Descriptors: make(map[string]*fnpb.ProcessBundleDescriptor), - - D: &DataService{}, } - slog.Info("Serving Worker components", slog.String("endpoint", wk.Endpoint())) + slog.Debug("Serving Worker components", slog.String("endpoint", wk.Endpoint())) fnpb.RegisterBeamFnControlServer(wk.server, wk) fnpb.RegisterBeamFnDataServer(wk.server, wk) fnpb.RegisterBeamFnLoggingServer(wk.server, wk) fnpb.RegisterBeamFnStateServer(wk.server, wk) + fnpb.RegisterProvisionServiceServer(wk.server, wk) return wk } func (wk *W) Endpoint() string { - return wk.lis.Addr().String() + _, port, _ := net.SplitHostPort(wk.lis.Addr().String()) + return fmt.Sprintf("localhost:%v", port) } // Serve serves on the started listener. Blocks. @@ -126,6 +136,7 @@ func (wk *W) LogValue() slog.Value { // Stop the GRPC server. func (wk *W) Stop() { slog.Debug("stopping", "worker", wk) + wk.stopped.Store(true) close(wk.InstReqs) close(wk.DataReqs) wk.server.Stop() @@ -134,11 +145,7 @@ func (wk *W) Stop() { } func (wk *W) NextInst() string { - return fmt.Sprintf("inst%03d", atomic.AddUint64(&wk.inst, 1)) -} - -func (wk *W) NextStage() string { - return fmt.Sprintf("stage%03d", atomic.AddUint64(&wk.bund, 1)) + return fmt.Sprintf("inst-%v-%03d", wk.Env, atomic.AddUint64(&wk.inst, 1)) } // TODO set logging level. @@ -150,20 +157,24 @@ func (wk *W) GetProvisionInfo(_ context.Context, _ *fnpb.GetProvisionInfoRequest } resp := &fnpb.GetProvisionInfoResponse{ Info: &fnpb.ProvisionInfo{ - // TODO: Add the job's Pipeline options // TODO: Include runner capabilities with the per job configuration. RunnerCapabilities: []string{ urns.CapabilityMonitoringInfoShortIDs, }, - LoggingEndpoint: endpoint, - ControlEndpoint: endpoint, - ArtifactEndpoint: endpoint, - // TODO add this job's RetrievalToken - // TODO add this job's artifact Dependencies + LoggingEndpoint: endpoint, + ControlEndpoint: endpoint, + ArtifactEndpoint: &pipepb.ApiServiceDescriptor{ + Url: wk.ArtifactEndpoint, + }, + + RetrievalToken: wk.JobKey, + Dependencies: wk.EnvPb.GetDependencies(), + PipelineOptions: wk.PipelineOptions, Metadata: map[string]string{ "runner": "prism", "runner_version": core.SdkVersion, + "variant": "test", }, }, } @@ -191,8 +202,18 @@ func (wk *W) Logging(stream fnpb.BeamFnLogging_LoggingServer) error { if l.Severity >= minsev { // TODO: Connect to the associated Job for this worker instead of // logging locally for SDK side logging. - slog.LogAttrs(context.TODO(), toSlogSev(l.GetSeverity()), l.GetMessage(), - slog.String(slog.SourceKey, l.GetLogLocation()), + file := l.GetLogLocation() + i := strings.LastIndex(file, ":") + line, _ := strconv.Atoi(file[i+1:]) + if i > 0 { + file = file[:i] + } + + slog.LogAttrs(stream.Context(), toSlogSev(l.GetSeverity()), l.GetMessage(), + slog.Any(slog.SourceKey, &slog.Source{ + File: file, + Line: line, + }), slog.Time(slog.TimeKey, l.GetTimestamp().AsTime()), slog.Any("worker", wk), ) @@ -229,39 +250,43 @@ func (wk *W) GetProcessBundleDescriptor(ctx context.Context, req *fnpb.GetProces return desc, nil } +// Connected indicates whether the worker has connected to the control RPC. +func (wk *W) Connected() bool { + return wk.connected.Load() +} + +// Stopped indicates that the worker has stopped. +func (wk *W) Stopped() bool { + return wk.stopped.Load() +} + // Control relays instructions to SDKs and back again, coordinated via unique instructionIDs. // // Requests come from the runner, and are sent to the client in the SDK. func (wk *W) Control(ctrl fnpb.BeamFnControl_ControlServer) error { - done := make(chan bool) + wk.connected.Store(true) + done := make(chan error, 1) go func() { for { resp, err := ctrl.Recv() if err == io.EOF { slog.Debug("ctrl.Recv finished; marking done", "worker", wk) - done <- true // means stream is finished + done <- nil // means stream is finished return } if err != nil { switch status.Code(err) { case codes.Canceled: - done <- true // means stream is finished + done <- err // means stream is finished return default: - slog.Error("ctrl.Recv failed", err, "worker", wk) + slog.Error("ctrl.Recv failed", "error", err, "worker", wk) panic(err) } } - // TODO: Do more than assume these are ProcessBundleResponses. wk.mu.Lock() if b, ok := wk.activeInstructions[resp.GetInstructionId()]; ok { - // TODO. Better pipeline error handling. - if resp.Error != "" { - slog.LogAttrs(context.TODO(), slog.LevelError, "ctrl.Recv pipeline error", - slog.String("error", resp.GetError())) - panic(resp.GetError()) - } b.Respond(resp) } else { slog.Debug("ctrl.Recv: %v", resp) @@ -270,13 +295,39 @@ func (wk *W) Control(ctrl fnpb.BeamFnControl_ControlServer) error { } }() - for req := range wk.InstReqs { - ctrl.Send(req) + for { + select { + case req, ok := <-wk.InstReqs: + if !ok { + slog.Debug("Worker shutting down.", "worker", wk) + return nil + } + if err := ctrl.Send(req); err != nil { + return err + } + case <-ctrl.Context().Done(): + wk.mu.Lock() + // Fail extant instructions + slog.Debug("SDK Disconnected", "worker", wk, "ctx_error", ctrl.Context().Err(), "outstanding_instructions", len(wk.activeInstructions)) + + msg := fmt.Sprintf("SDK worker disconnected: %v, %v active instructions", wk.String(), len(wk.activeInstructions)) + for instID, b := range wk.activeInstructions { + b.Respond(&fnpb.InstructionResponse{ + InstructionId: instID, + Error: msg, + }) + } + wk.mu.Unlock() + return context.Cause(ctrl.Context()) + case err := <-done: + if err != nil { + slog.Warn("Control done", "error", err, "worker", wk) + } else { + slog.Debug("Control done", "worker", wk) + } + return err + } } - slog.Debug("ctrl.Send finished waiting on done") - <-done - slog.Debug("Control done") - return nil } // Data relays elements and timer bytes to SDKs and back again, coordinated via @@ -322,13 +373,20 @@ func (wk *W) Data(data fnpb.BeamFnData_DataServer) error { wk.mu.Unlock() } }() - - for req := range wk.DataReqs { - if err := data.Send(req); err != nil { - slog.LogAttrs(context.TODO(), slog.LevelDebug, "data.Send error", slog.Any("error", err)) + for { + select { + case req, ok := <-wk.DataReqs: + if !ok { + return nil + } + if err := data.Send(req); err != nil { + slog.LogAttrs(context.TODO(), slog.LevelDebug, "data.Send error", slog.Any("error", err)) + } + case <-data.Context().Done(): + slog.Debug("Data context canceled") + return context.Cause(data.Context()) } } - return nil } // State relays elements and timer bytes to SDKs and back again, coordinated via @@ -361,8 +419,12 @@ func (wk *W) State(state fnpb.BeamFnState_StateServer) error { // State requests are always for an active ProcessBundle instruction wk.mu.Lock() - b := wk.activeInstructions[req.GetInstructionId()].(*B) + b, ok := wk.activeInstructions[req.GetInstructionId()].(*B) wk.mu.Unlock() + if !ok { + slog.Warn("state request after bundle inactive", "instruction", req.GetInstructionId(), "worker", wk) + continue + } key := req.GetStateKey() slog.Debug("StateRequest_Get", prototext.Format(req), "bundle", b) @@ -457,7 +519,7 @@ func (cr *chanResponder) Respond(resp *fnpb.InstructionResponse) { // sendInstruction is a helper for creating and sending worker single RPCs, blocking // until the response returns. -func (wk *W) sendInstruction(req *fnpb.InstructionRequest) *fnpb.InstructionResponse { +func (wk *W) sendInstruction(ctx context.Context, req *fnpb.InstructionRequest) *fnpb.InstructionResponse { cr := chanResponderPool.Get().(*chanResponder) progInst := wk.NextInst() wk.mu.Lock() @@ -473,15 +535,26 @@ func (wk *W) sendInstruction(req *fnpb.InstructionRequest) *fnpb.InstructionResp req.InstructionId = progInst - // Tell the SDK to start processing the bundle. + if wk.Stopped() { + return nil + } wk.InstReqs <- req - // Protos are safe as nil, so just return directly. - return <-cr.Resp + + select { + case <-ctx.Done(): + return &fnpb.InstructionResponse{ + InstructionId: progInst, + Error: "context canceled before receive", + } + case resp := <-cr.Resp: + // Protos are safe as nil, so just return directly. + return resp + } } // MonitoringMetadata is a convenience method to request the metadata for monitoring shortIDs. -func (wk *W) MonitoringMetadata(unknownIDs []string) *fnpb.MonitoringInfosMetadataResponse { - return wk.sendInstruction(&fnpb.InstructionRequest{ +func (wk *W) MonitoringMetadata(ctx context.Context, unknownIDs []string) *fnpb.MonitoringInfosMetadataResponse { + return wk.sendInstruction(ctx, &fnpb.InstructionRequest{ Request: &fnpb.InstructionRequest_MonitoringInfos{ MonitoringInfos: &fnpb.MonitoringInfosMetadataRequest{ MonitoringInfoId: unknownIDs, @@ -492,6 +565,7 @@ func (wk *W) MonitoringMetadata(unknownIDs []string) *fnpb.MonitoringInfosMetada // DataService is slated to be deleted in favour of stage based state // management for side inputs. +// TODO(https://github.com/apache/beam/issues/28543), remove this concept. type DataService struct { mu sync.Mutex // TODO actually quick process the data to windows here as well. diff --git a/sdks/go/pkg/beam/runners/prism/internal/worker/worker_test.go b/sdks/go/pkg/beam/runners/prism/internal/worker/worker_test.go index 14b879fe242e9..6a90b463c45d8 100644 --- a/sdks/go/pkg/beam/runners/prism/internal/worker/worker_test.go +++ b/sdks/go/pkg/beam/runners/prism/internal/worker/worker_test.go @@ -32,14 +32,14 @@ import ( ) func TestWorker_New(t *testing.T) { - w := New("test") + w := New("test", "testEnv") if got, want := w.ID, "test"; got != want { t.Errorf("New(%q) = %v, want %v", want, got, want) } } func TestWorker_NextInst(t *testing.T) { - w := New("test") + w := New("test", "testEnv") instIDs := map[string]struct{}{} for i := 0; i < 100; i++ { @@ -50,20 +50,8 @@ func TestWorker_NextInst(t *testing.T) { } } -func TestWorker_NextStage(t *testing.T) { - w := New("test") - - stageIDs := map[string]struct{}{} - for i := 0; i < 100; i++ { - stageIDs[w.NextStage()] = struct{}{} - } - if got, want := len(stageIDs), 100; got != want { - t.Errorf("calling w.NextStage() got %v unique ids, want %v", got, want) - } -} - func TestWorker_GetProcessBundleDescriptor(t *testing.T) { - w := New("test") + w := New("test", "testEnv") id := "available" w.Descriptors[id] = &fnpb.ProcessBundleDescriptor{ @@ -93,7 +81,7 @@ func serveTestWorker(t *testing.T) (context.Context, *W, *grpc.ClientConn) { ctx, cancelFn := context.WithCancel(context.Background()) t.Cleanup(cancelFn) - w := New("test") + w := New("test", "testEnv") lis := bufconn.Listen(2048) w.lis = lis t.Cleanup(func() { w.Stop() }) @@ -119,8 +107,17 @@ func TestWorker_Logging(t *testing.T) { logStream.Send(&fnpb.LogEntry_List{ LogEntries: []*fnpb.LogEntry{{ - Severity: fnpb.LogEntry_Severity_INFO, - Message: "squeamish ossiphrage", + Severity: fnpb.LogEntry_Severity_INFO, + Message: "squeamish ossiphrage", + LogLocation: "intentionally.go:124", + }}, + }) + + logStream.Send(&fnpb.LogEntry_List{ + LogEntries: []*fnpb.LogEntry{{ + Severity: fnpb.LogEntry_Severity_INFO, + Message: "squeamish ossiphrage the second", + LogLocation: "intentionally bad log location", }}, }) @@ -146,7 +143,7 @@ func TestWorker_Control_HappyPath(t *testing.T) { b := &B{} b.Init() wk.activeInstructions[instID] = b - b.ProcessOn(wk) + b.ProcessOn(ctx, wk) ctrlStream.Send(&fnpb.InstructionResponse{ InstructionId: instID, @@ -180,7 +177,7 @@ func TestWorker_Data_HappyPath(t *testing.T) { b := &B{ InstID: instID, - PBDID: wk.NextStage(), + PBDID: "teststageID", InputData: [][]byte{ {1, 1, 1, 1, 1, 1}, }, @@ -193,7 +190,7 @@ func TestWorker_Data_HappyPath(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - b.ProcessOn(wk) + b.ProcessOn(ctx, wk) }() wk.InstReqs <- &fnpb.InstructionRequest{ diff --git a/sdks/go/pkg/beam/runners/prism/prism.go b/sdks/go/pkg/beam/runners/prism/prism.go index 0be35ad5cc332..bcb7a3fb689f8 100644 --- a/sdks/go/pkg/beam/runners/prism/prism.go +++ b/sdks/go/pkg/beam/runners/prism/prism.go @@ -49,9 +49,9 @@ func Execute(ctx context.Context, p *beam.Pipeline) (beam.PipelineResult, error) s := jobservices.NewServer(0, internal.RunPipeline) *jobopts.Endpoint = s.Endpoint() go s.Serve() - } - if !jobopts.IsLoopback() { - *jobopts.EnvironmentType = "loopback" + if !jobopts.IsLoopback() { + *jobopts.EnvironmentType = "loopback" + } } return universal.Execute(ctx, p) } diff --git a/sdks/go/pkg/beam/runners/universal/extworker/extworker.go b/sdks/go/pkg/beam/runners/universal/extworker/extworker.go index 6dab9ebbfb0cd..a7fc308d2193b 100644 --- a/sdks/go/pkg/beam/runners/universal/extworker/extworker.go +++ b/sdks/go/pkg/beam/runners/universal/extworker/extworker.go @@ -63,7 +63,7 @@ type Loopback struct { // StartWorker initializes a new worker harness, implementing BeamFnExternalWorkerPoolServer.StartWorker. func (s *Loopback) StartWorker(ctx context.Context, req *fnpb.StartWorkerRequest) (*fnpb.StartWorkerResponse, error) { - log.Infof(ctx, "starting worker %v", req.GetWorkerId()) + log.Debugf(ctx, "starting worker %v", req.GetWorkerId()) s.mu.Lock() defer s.mu.Unlock() if s.workers == nil { @@ -136,7 +136,7 @@ func (s *Loopback) StopWorker(ctx context.Context, req *fnpb.StopWorkerRequest) func (s *Loopback) Stop(ctx context.Context) error { s.mu.Lock() - log.Infof(ctx, "stopping Loopback, and %d workers", len(s.workers)) + log.Debugf(ctx, "stopping Loopback, and %d workers", len(s.workers)) s.workers = nil s.rootCancel() diff --git a/sdks/go/pkg/beam/runners/universal/runnerlib/compile.go b/sdks/go/pkg/beam/runners/universal/runnerlib/compile.go index 8fbbc08460075..b21362ba9fdea 100644 --- a/sdks/go/pkg/beam/runners/universal/runnerlib/compile.go +++ b/sdks/go/pkg/beam/runners/universal/runnerlib/compile.go @@ -41,12 +41,20 @@ func IsWorkerCompatibleBinary() (string, bool) { var unique int32 +// CompileOpts are additional options for dynamic compiles of the local code +// for development purposes. Production runs should build the worker binary +// separately for the target environment. +// See https://beam.apache.org/documentation/sdks/go-cross-compilation/ for details. +type CompileOpts struct { + OS, Arch string +} + // BuildTempWorkerBinary creates a local worker binary in the tmp directory // for linux/amd64. Caller responsible for deleting the binary. -func BuildTempWorkerBinary(ctx context.Context) (string, error) { +func BuildTempWorkerBinary(ctx context.Context, opts CompileOpts) (string, error) { id := atomic.AddInt32(&unique, 1) filename := filepath.Join(os.TempDir(), fmt.Sprintf("worker-%v-%v", id, time.Now().UnixNano())) - if err := buildWorkerBinary(ctx, filename); err != nil { + if err := buildWorkerBinary(ctx, filename, opts); err != nil { return "", err } return filename, nil @@ -59,7 +67,7 @@ func BuildTempWorkerBinary(ctx context.Context) (string, error) { // * /Users/herohde/go/src/github.com/apache/beam/sdks/go/examples/wordcount/wordcount.go (skip: 3) // /usr/local/go/src/runtime/proc.go (skip: 4) // not always present // /usr/local/go/src/runtime/asm_amd64.s (skip: 4 or 5) -func buildWorkerBinary(ctx context.Context, filename string) error { +func buildWorkerBinary(ctx context.Context, filename string, opts CompileOpts) error { program := "" var isTest bool for i := 3; ; i++ { @@ -77,9 +85,17 @@ func buildWorkerBinary(ctx context.Context, filename string) error { } goos := "linux" goarch := "amd64" + + if opts.OS != "" { + goos = opts.OS + } + if opts.Arch != "" { + goarch = opts.Arch + } + cgo := "0" - log.Infof(ctx, "Cross-compiling %v with GOOS=%s GOARCH=%s CGO_ENABLED=%s as %v", goos, goarch, cgo, program, filename) + log.Infof(ctx, "Cross-compiling %v with GOOS=%s GOARCH=%s CGO_ENABLED=%s as %v", program, goos, goarch, cgo, filename) // Cross-compile given go program. Not awesome. program = program[:strings.LastIndex(program, "/")+1] diff --git a/sdks/go/pkg/beam/runners/universal/runnerlib/execute.go b/sdks/go/pkg/beam/runners/universal/runnerlib/execute.go index 68db9b0ee76a5..eb854dbfcdba7 100644 --- a/sdks/go/pkg/beam/runners/universal/runnerlib/execute.go +++ b/sdks/go/pkg/beam/runners/universal/runnerlib/execute.go @@ -41,14 +41,14 @@ func Execute(ctx context.Context, p *pipepb.Pipeline, endpoint string, opt *JobO presult := &universalPipelineResult{} bin := opt.Worker - if bin == "" { + if bin == "" && !opt.Loopback { if self, ok := IsWorkerCompatibleBinary(); ok { bin = self log.Infof(ctx, "Using running binary as worker binary: '%v'", bin) } else { // Cross-compile as last resort. - worker, err := BuildTempWorkerBinary(ctx) + worker, err := BuildTempWorkerBinary(ctx, CompileOpts{}) if err != nil { return presult, err } @@ -56,6 +56,11 @@ func Execute(ctx context.Context, p *pipepb.Pipeline, endpoint string, opt *JobO bin = worker } + } else if opt.Loopback { + // TODO(https://github.com/apache/beam/issues/27569: determine the canonical location for Beam temp files. + // In loopback mode, the binary is unused, so we can avoid an unnecessary compile step. + f, _ := os.CreateTemp(os.TempDir(), "beamloopbackworker-*") + bin = f.Name() } else { log.Infof(ctx, "Using specified worker binary: '%v'", bin) } diff --git a/sdks/go/pkg/beam/runners/universal/runnerlib/job.go b/sdks/go/pkg/beam/runners/universal/runnerlib/job.go index daa6896da406a..4e50661b3db8e 100644 --- a/sdks/go/pkg/beam/runners/universal/runnerlib/job.go +++ b/sdks/go/pkg/beam/runners/universal/runnerlib/job.go @@ -39,15 +39,17 @@ type JobOptions struct { // Experiments are additional experiments. Experiments []string - // TODO(herohde) 3/17/2018: add further parametrization as needed - // Worker is the worker binary override. Worker string - // RetainDocker is an option to pass to the runner. + // RetainDocker is an option to pass to the runner indicating the docker containers should be cached. RetainDocker bool + // Indicates a limit on parallelism the runner should impose. Parallelism int + + // Loopback indicates this job is running in loopback mode and will reconnect to the local process. + Loopback bool } // Prepare prepares a job to the given job service. It returns the preparation id @@ -74,7 +76,7 @@ func Prepare(ctx context.Context, client jobpb.JobServiceClient, p *pipepb.Pipel } resp, err := client.Prepare(ctx, req) if err != nil { - return "", "", "", errors.Wrap(err, "failed to connect to job service") + return "", "", "", errors.Wrap(err, "job failed to prepare") } return resp.GetPreparationId(), resp.GetArtifactStagingEndpoint().GetUrl(), resp.GetStagingSessionToken(), nil } @@ -101,10 +103,17 @@ func WaitForCompletion(ctx context.Context, client jobpb.JobServiceClient, jobID return errors.Wrap(err, "failed to get job stream") } + mostRecentError := "" + var errReceived, jobFailed bool + for { msg, err := stream.Recv() if err != nil { if err == io.EOF { + if jobFailed { + // Connection finished, so time to exit, produce what we have. + return errors.Errorf("job %v failed:\n%v", jobID, mostRecentError) + } return nil } return err @@ -114,13 +123,17 @@ func WaitForCompletion(ctx context.Context, client jobpb.JobServiceClient, jobID case msg.GetStateResponse() != nil: resp := msg.GetStateResponse() - log.Infof(ctx, "Job state: %v", resp.GetState().String()) + log.Infof(ctx, "Job[%v] state: %v", jobID, resp.GetState().String()) switch resp.State { case jobpb.JobState_DONE, jobpb.JobState_CANCELLED: return nil case jobpb.JobState_FAILED: - return errors.Errorf("job %v failed", jobID) + jobFailed = true + if errReceived { + return errors.Errorf("job %v failed:\n%v", jobID, mostRecentError) + } + // Otherwise we should wait for at least one error log from the runner. } case msg.GetMessageResponse() != nil: @@ -129,6 +142,15 @@ func WaitForCompletion(ctx context.Context, client jobpb.JobServiceClient, jobID text := fmt.Sprintf("%v (%v): %v", resp.GetTime(), resp.GetMessageId(), resp.GetMessageText()) log.Output(ctx, messageSeverity(resp.GetImportance()), 1, text) + if resp.GetImportance() >= jobpb.JobMessage_JOB_MESSAGE_ERROR { + errReceived = true + mostRecentError = resp.GetMessageText() + + if jobFailed { + return errors.Errorf("job %v failed:\n%w", jobID, errors.New(mostRecentError)) + } + } + default: return errors.Errorf("unexpected job update: %v", proto.MarshalTextString(msg)) } diff --git a/sdks/go/pkg/beam/runners/universal/runnerlib/stage.go b/sdks/go/pkg/beam/runners/universal/runnerlib/stage.go index 1ab82fd6be0ca..d5cc6aa7327a7 100644 --- a/sdks/go/pkg/beam/runners/universal/runnerlib/stage.go +++ b/sdks/go/pkg/beam/runners/universal/runnerlib/stage.go @@ -44,14 +44,14 @@ func Stage(ctx context.Context, id, endpoint, binary, st string) (retrievalToken defer cc.Close() if err := StageViaPortableAPI(ctx, cc, binary, st); err == nil { - return "", nil + return st, nil } log.Warnf(ctx, "unable to stage with PortableAPI: %v; falling back to legacy", err) return StageViaLegacyAPI(ctx, cc, binary, st) } -// StageViaPortableApi is a beam internal function for uploading artifacts to the staging service +// StageViaPortableAPI is a beam internal function for uploading artifacts to the staging service // via the portable API. // // It will be unexported at a later time. @@ -181,7 +181,7 @@ func stageFile(filename string, stream jobpb.ArtifactStagingService_ReverseArtif } } -// StageViaLegacyApi is a beam internal function for uploading artifacts to the staging service +// StageViaLegacyAPI is a beam internal function for uploading artifacts to the staging service // via the legacy API. // // It will be unexported at a later time. diff --git a/sdks/go/pkg/beam/runners/universal/universal.go b/sdks/go/pkg/beam/runners/universal/universal.go index 299a64acdd691..8af9e91e1e15e 100644 --- a/sdks/go/pkg/beam/runners/universal/universal.go +++ b/sdks/go/pkg/beam/runners/universal/universal.go @@ -101,6 +101,7 @@ func Execute(ctx context.Context, p *beam.Pipeline) (beam.PipelineResult, error) Worker: *jobopts.WorkerBinary, RetainDocker: *jobopts.RetainDockerContainers, Parallelism: *jobopts.Parallelism, + Loopback: jobopts.IsLoopback(), } return runnerlib.Execute(ctx, pipeline, endpoint, opt, *jobopts.Async) } diff --git a/sdks/go/pkg/beam/runners/vet/vet.go b/sdks/go/pkg/beam/runners/vet/vet.go index 131fa0b1ec125..739f5db61c5bc 100644 --- a/sdks/go/pkg/beam/runners/vet/vet.go +++ b/sdks/go/pkg/beam/runners/vet/vet.go @@ -54,7 +54,7 @@ func init() { type disabledResolver bool func (p disabledResolver) Sym2Addr(name string) (uintptr, error) { - return 0, errors.Errorf("%v not found. Use runtime.RegisterFunction in unit tests", name) + return 0, errors.Errorf("%v not found. Register DoFns and functions with the beam/register package.", name) } // Execute evaluates the pipeline on whether it can run without reflection. diff --git a/sdks/go/pkg/beam/staticcheck.conf b/sdks/go/pkg/beam/staticcheck.conf index 5da9745a7a0c4..d99b5ad0af4d2 100644 --- a/sdks/go/pkg/beam/staticcheck.conf +++ b/sdks/go/pkg/beam/staticcheck.conf @@ -17,4 +17,4 @@ # analysis tool. See https://staticcheck.io/docs/checks/ for descriptions of # each check. -checks = ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1022", "-ST1023", "-SA1019", "-S1021"] \ No newline at end of file +checks = ["all", "-ST1000", "-ST1003", "-SA1019"] \ No newline at end of file diff --git a/sdks/go/pkg/beam/testing/passert/equals_test.go b/sdks/go/pkg/beam/testing/passert/equals_test.go index b0ddeae8d6f75..6e6578dd3e415 100644 --- a/sdks/go/pkg/beam/testing/passert/equals_test.go +++ b/sdks/go/pkg/beam/testing/passert/equals_test.go @@ -182,10 +182,12 @@ func ExampleEqualsList_mismatch() { EqualsList(s, col, list) err := ptest.Run(p) err = unwrapError(err) - fmt.Println(err) + + // Process error for cleaner example output, demonstrating the diff. + processedErr := strings.SplitAfter(err.Error(), ".failIfBadEntries] failed:") + fmt.Println(processedErr[1]) // Output: - // DoFn[UID:1, PID:passert.failIfBadEntries, Name: github.com/apache/beam/sdks/v2/go/pkg/beam/testing/passert.failIfBadEntries] failed: // actual PCollection does not match expected values // ========= // 2 correct entries (present in both) diff --git a/sdks/go/pkg/beam/testing/passert/floats.go b/sdks/go/pkg/beam/testing/passert/floats.go index 727c313820b79..f71e550908386 100644 --- a/sdks/go/pkg/beam/testing/passert/floats.go +++ b/sdks/go/pkg/beam/testing/passert/floats.go @@ -24,8 +24,16 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/util/reflectx" "github.com/apache/beam/sdks/v2/go/pkg/beam/internal/errors" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" ) +func init() { + register.DoFn2x1[[]byte, func(*beam.T) bool, error]((*boundsFn)(nil)) + register.DoFn3x1[[]byte, func(*beam.T) bool, func(*beam.T) bool, error]((*thresholdFn)(nil)) + register.Emitter1[beam.T]() + register.Iter1[beam.T]() +} + // EqualsFloat calls into TryEqualsFloat, checkong that two PCollections of non-complex // numeric types are equal, with each element being within a provided threshold of an // expected value. Panics if TryEqualsFloat returns an error. @@ -110,11 +118,11 @@ func AllWithinBounds(s beam.Scope, col beam.PCollection, lo, hi float64) { lo, hi = hi, lo } s = s.Scope(fmt.Sprintf("passert.AllWithinBounds([%v, %v])", lo, hi)) - beam.ParDo0(s, &boundsFn{lo: lo, hi: hi}, beam.Impulse(s), beam.SideInput{Input: col}) + beam.ParDo0(s, &boundsFn{Lo: lo, Hi: hi}, beam.Impulse(s), beam.SideInput{Input: col}) } type boundsFn struct { - lo, hi float64 + Lo, Hi float64 } func (f *boundsFn) ProcessElement(_ []byte, col func(*beam.T) bool) error { @@ -122,9 +130,9 @@ func (f *boundsFn) ProcessElement(_ []byte, col func(*beam.T) bool) error { var input beam.T for col(&input) { val := toFloat(input) - if val < f.lo { + if val < f.Lo { tooLow = append(tooLow, val) - } else if val > f.hi { + } else if val > f.Hi { tooHigh = append(tooHigh, val) } } @@ -134,11 +142,11 @@ func (f *boundsFn) ProcessElement(_ []byte, col func(*beam.T) bool) error { errorStrings := []string{} if len(tooLow) != 0 { sort.Float64s(tooLow) - errorStrings = append(errorStrings, fmt.Sprintf("values below minimum value %v: %v", f.lo, tooLow)) + errorStrings = append(errorStrings, fmt.Sprintf("values below minimum value %v: %v", f.Lo, tooLow)) } if len(tooHigh) != 0 { sort.Float64s(tooHigh) - errorStrings = append(errorStrings, fmt.Sprintf("values above maximum value %v: %v", f.hi, tooHigh)) + errorStrings = append(errorStrings, fmt.Sprintf("values above maximum value %v: %v", f.Hi, tooHigh)) } return errors.New(strings.Join(errorStrings, "\n")) } diff --git a/sdks/go/pkg/beam/testing/passert/passert.go b/sdks/go/pkg/beam/testing/passert/passert.go index 990d3c8c4d47b..c4b0f490dafd2 100644 --- a/sdks/go/pkg/beam/testing/passert/passert.go +++ b/sdks/go/pkg/beam/testing/passert/passert.go @@ -39,9 +39,13 @@ import ( func Diff(s beam.Scope, a, b beam.PCollection) (left, both, right beam.PCollection) { imp := beam.Impulse(s) - t := beam.ValidateNonCompositeType(a) - beam.ValidateNonCompositeType(b) - return beam.ParDo3(s, &diffFn{Type: beam.EncodedType{T: t.Type()}}, imp, beam.SideInput{Input: a}, beam.SideInput{Input: b}) + ta := beam.ValidateNonCompositeType(a) + tb := beam.ValidateNonCompositeType(b) + + if !typex.IsEqual(ta, tb) { + panic(fmt.Sprintf("passert.Diff input PColections don't have matching types: %v != %v", ta, tb)) + } + return beam.ParDo3(s, &diffFn{Type: beam.EncodedType{T: ta.Type()}}, imp, beam.SideInput{Input: a}, beam.SideInput{Input: b}) } // diffFn computes the symmetrical multi-set difference of 2 collections, under diff --git a/sdks/go/pkg/beam/testing/passert/passert_test.go b/sdks/go/pkg/beam/testing/passert/passert_test.go index 9524bc868ebb9..d472f68839393 100644 --- a/sdks/go/pkg/beam/testing/passert/passert_test.go +++ b/sdks/go/pkg/beam/testing/passert/passert_test.go @@ -20,15 +20,30 @@ import ( "testing" "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest" ) +func TestMain(m *testing.M) { + ptest.Main(m) +} + +func isA(input string) bool { return input == "a" } +func isB(input string) bool { return input == "b" } +func lessThan13(input int) bool { return input < 13 } +func greaterThan13(input int) bool { return input > 13 } + +func init() { + register.Function1x1(isA) + register.Function1x1(isB) + register.Function1x1(lessThan13) + register.Function1x1(greaterThan13) +} + func TestTrue_string(t *testing.T) { p, s := beam.NewPipelineWithRoot() col := beam.Create(s, "a", "a", "a") - True(s, col, func(input string) bool { - return input == "a" - }) + True(s, col, isA) if err := ptest.Run(p); err != nil { t.Errorf("Pipeline failed: %v", err) } @@ -37,9 +52,7 @@ func TestTrue_string(t *testing.T) { func TestTrue_numeric(t *testing.T) { p, s := beam.NewPipelineWithRoot() col := beam.Create(s, 3, 3, 6) - True(s, col, func(input int) bool { - return input < 13 - }) + True(s, col, lessThan13) if err := ptest.Run(p); err != nil { t.Errorf("Pipeline failed: %v", err) } @@ -48,9 +61,7 @@ func TestTrue_numeric(t *testing.T) { func TestTrue_bad(t *testing.T) { p, s := beam.NewPipelineWithRoot() col := beam.Create(s, "a", "a", "b") - True(s, col, func(input string) bool { - return input == "a" - }) + True(s, col, isA) err := ptest.Run(p) if err == nil { t.Fatalf("Pipeline succeeded when it should haved failed, got %v", err) @@ -63,9 +74,7 @@ func TestTrue_bad(t *testing.T) { func TestFalse_string(t *testing.T) { p, s := beam.NewPipelineWithRoot() col := beam.Create(s, "a", "a", "a") - False(s, col, func(input string) bool { - return input == "b" - }) + False(s, col, isB) if err := ptest.Run(p); err != nil { t.Errorf("Pipeline failed: %v", err) } @@ -74,9 +83,7 @@ func TestFalse_string(t *testing.T) { func TestFalse_numeric(t *testing.T) { p, s := beam.NewPipelineWithRoot() col := beam.Create(s, 3, 3, 6) - False(s, col, func(input int) bool { - return input > 13 - }) + False(s, col, greaterThan13) if err := ptest.Run(p); err != nil { t.Errorf("Pipeline failed: %v", err) } @@ -85,9 +92,7 @@ func TestFalse_numeric(t *testing.T) { func TestFalse_bad(t *testing.T) { p, s := beam.NewPipelineWithRoot() col := beam.Create(s, "a", "a", "b") - False(s, col, func(input string) bool { - return input == "b" - }) + False(s, col, isB) err := ptest.Run(p) if err == nil { t.Fatalf("Pipeline succeeded when it should haved failed, got %v", err) diff --git a/sdks/go/pkg/beam/testing/ptest/ptest.go b/sdks/go/pkg/beam/testing/ptest/ptest.go index d2b8f01f72dd1..e025ed7aade81 100644 --- a/sdks/go/pkg/beam/testing/ptest/ptest.go +++ b/sdks/go/pkg/beam/testing/ptest/ptest.go @@ -19,18 +19,20 @@ package ptest import ( "context" "flag" + "fmt" "os" "testing" "github.com/apache/beam/sdks/v2/go/pkg/beam" "github.com/apache/beam/sdks/v2/go/pkg/beam/runners" // common runner flag. - // ptest uses the direct runner to execute pipelines by default. + // ptest uses the prism runner to execute pipelines by default. + // but includes the direct runner for legacy fallback reasons to + // support users overriding the default back to the direct runner. _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/direct" + _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism" ) -// TODO(herohde) 7/10/2017: add hooks to verify counters, logs, etc. - // Create creates a pipeline and a PCollection with the given values. func Create(values []any) (*beam.Pipeline, beam.Scope, beam.PCollection) { p := beam.NewPipeline() @@ -59,27 +61,31 @@ func CreateList2(a, b any) (*beam.Pipeline, beam.Scope, beam.PCollection, beam.P return p, s, beam.CreateList(s, a), beam.CreateList(s, b) } +const ( + defaultRunner = "prism" +) + // Runner is a flag that sets which runner pipelines under test will use. // // The test file must have a TestMain that calls Main or MainWithDefault // to function. var ( - Runner = runners.Runner - defaultRunner = "direct" - mainCalled = false + Runner = runners.Runner + defaultRunnerOverride = defaultRunner + mainCalled = false ) func getRunner() string { r := *Runner if r == "" { - r = defaultRunner + r = defaultRunnerOverride } return r } // DefaultRunner returns the default runner name for the test file. func DefaultRunner() string { - return defaultRunner + return defaultRunnerOverride } // MainCalled returns true iff Main or MainRet has been called. @@ -90,7 +96,17 @@ func MainCalled() bool { // Run runs a pipeline for testing. The semantics of the pipeline is expected // to be verified through passert. func Run(p *beam.Pipeline) error { - _, err := beam.Run(context.Background(), getRunner(), p) + r := getRunner() + _, err := beam.Run(context.Background(), r, p) + // Until a few versions from now (say, v2.56), + // if there's an error, and + // the runner is prism, and it was the set default runner, but not a manually specificed runner via the flag + // augment the error with instructions to switch back to the direct runner. + if err != nil && r == "prism" && r == defaultRunnerOverride && r != *Runner { + err = fmt.Errorf("%w\nerror may be due to Apache Beam Go's migration from the direct runner to the prism runner."+ + " While the failure(s) should be fixed, you can continue to use the direct runner with this TestMain override:"+ + " `func TestMain(m *testing.M) { ptest.MainWithDefault(m, \"direct\") }`", err) + } return err } @@ -132,7 +148,7 @@ func BuildAndRun(t *testing.T, build func(s beam.Scope)) beam.PipelineResult { // ptest.Main(m) // } func Main(m *testing.M) { - MainWithDefault(m, "direct") + MainWithDefault(m, defaultRunner) } // MainWithDefault is an implementation of testing's TestMain to permit testing @@ -140,7 +156,7 @@ func Main(m *testing.M) { // runner to use. func MainWithDefault(m *testing.M, runner string) { mainCalled = true - defaultRunner = runner + defaultRunnerOverride = runner if !flag.Parsed() { flag.Parse() } @@ -148,7 +164,7 @@ func MainWithDefault(m *testing.M, runner string) { os.Exit(m.Run()) } -// MainRet is equivelant to Main, but returns an exit code to pass to os.Exit(). +// MainRet is equivalent to Main, but returns an exit code to pass to os.Exit(). // // Example: // @@ -156,14 +172,14 @@ func MainWithDefault(m *testing.M, runner string) { // os.Exit(ptest.Main(m)) // } func MainRet(m *testing.M) int { - return MainRetWithDefault(m, "direct") + return MainRetWithDefault(m, defaultRunner) } -// MainRetWithDefault is equivelant to MainWithDefault but returns an exit code +// MainRetWithDefault is equivalent to MainWithDefault but returns an exit code // to pass to os.Exit(). func MainRetWithDefault(m *testing.M, runner string) int { mainCalled = true - defaultRunner = runner + defaultRunnerOverride = runner if !flag.Parsed() { flag.Parse() } diff --git a/sdks/go/pkg/beam/testing/ptest/ptest_test.go b/sdks/go/pkg/beam/testing/ptest/ptest_test.go index cbedd6b406fc4..844737352a5aa 100644 --- a/sdks/go/pkg/beam/testing/ptest/ptest_test.go +++ b/sdks/go/pkg/beam/testing/ptest/ptest_test.go @@ -21,6 +21,10 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/passert" ) +func TestMain(m *testing.M) { + Main(m) +} + func TestCreate(t *testing.T) { inputs := []any{"a", "b", "c"} p, s, col := Create(inputs) diff --git a/sdks/go/pkg/beam/transforms/filter/filter.go b/sdks/go/pkg/beam/transforms/filter/filter.go index 913e7355c30dd..997eec5eb4ef3 100644 --- a/sdks/go/pkg/beam/transforms/filter/filter.go +++ b/sdks/go/pkg/beam/transforms/filter/filter.go @@ -21,11 +21,15 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/funcx" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/util/reflectx" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" ) -//go:generate go install github.com/apache/beam/sdks/v2/go/cmd/starcgen -//go:generate starcgen --package=filter --identifiers=filterFn,mapFn,mergeFn -//go:generate go fmt +func init() { + register.DoFn2x0[beam.T, func(beam.T)]((*filterFn)(nil)) + register.Function1x2(mapFn) + register.Function2x1(mergeFn) + register.Emitter1[beam.T]() +} var ( sig = funcx.MakePredicate(beam.TType) // T -> bool diff --git a/sdks/go/pkg/beam/transforms/filter/filter.shims.go b/sdks/go/pkg/beam/transforms/filter/filter.shims.go deleted file mode 100644 index b0d18233ab189..0000000000000 --- a/sdks/go/pkg/beam/transforms/filter/filter.shims.go +++ /dev/null @@ -1,201 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Code generated by starcgen. DO NOT EDIT. -// File: filter.shims.go - -package filter - -import ( - "context" - "reflect" - - // Library imports - "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime" - "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/exec" - "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/graphx/schema" - "github.com/apache/beam/sdks/v2/go/pkg/beam/core/sdf" - "github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex" - "github.com/apache/beam/sdks/v2/go/pkg/beam/core/util/reflectx" -) - -func init() { - runtime.RegisterFunction(mapFn) - runtime.RegisterFunction(mergeFn) - runtime.RegisterType(reflect.TypeOf((*filterFn)(nil)).Elem()) - schema.RegisterType(reflect.TypeOf((*filterFn)(nil)).Elem()) - reflectx.RegisterStructWrapper(reflect.TypeOf((*filterFn)(nil)).Elem(), wrapMakerFilterFn) - reflectx.RegisterFunc(reflect.TypeOf((*func(int, int) int)(nil)).Elem(), funcMakerIntIntГInt) - reflectx.RegisterFunc(reflect.TypeOf((*func(typex.T, func(typex.T)))(nil)).Elem(), funcMakerTypex۰TEmitTypex۰TГ) - reflectx.RegisterFunc(reflect.TypeOf((*func(typex.T) (typex.T, int))(nil)).Elem(), funcMakerTypex۰TГTypex۰TInt) - reflectx.RegisterFunc(reflect.TypeOf((*func())(nil)).Elem(), funcMakerГ) - exec.RegisterEmitter(reflect.TypeOf((*func(typex.T))(nil)).Elem(), emitMakerTypex۰T) -} - -func wrapMakerFilterFn(fn any) map[string]reflectx.Func { - dfn := fn.(*filterFn) - return map[string]reflectx.Func{ - "ProcessElement": reflectx.MakeFunc(func(a0 typex.T, a1 func(typex.T)) { dfn.ProcessElement(a0, a1) }), - "Setup": reflectx.MakeFunc(func() { dfn.Setup() }), - } -} - -type callerIntIntГInt struct { - fn func(int, int) int -} - -func funcMakerIntIntГInt(fn any) reflectx.Func { - f := fn.(func(int, int) int) - return &callerIntIntГInt{fn: f} -} - -func (c *callerIntIntГInt) Name() string { - return reflectx.FunctionName(c.fn) -} - -func (c *callerIntIntГInt) Type() reflect.Type { - return reflect.TypeOf(c.fn) -} - -func (c *callerIntIntГInt) Call(args []any) []any { - out0 := c.fn(args[0].(int), args[1].(int)) - return []any{out0} -} - -func (c *callerIntIntГInt) Call2x1(arg0, arg1 any) any { - return c.fn(arg0.(int), arg1.(int)) -} - -type callerTypex۰TEmitTypex۰TГ struct { - fn func(typex.T, func(typex.T)) -} - -func funcMakerTypex۰TEmitTypex۰TГ(fn any) reflectx.Func { - f := fn.(func(typex.T, func(typex.T))) - return &callerTypex۰TEmitTypex۰TГ{fn: f} -} - -func (c *callerTypex۰TEmitTypex۰TГ) Name() string { - return reflectx.FunctionName(c.fn) -} - -func (c *callerTypex۰TEmitTypex۰TГ) Type() reflect.Type { - return reflect.TypeOf(c.fn) -} - -func (c *callerTypex۰TEmitTypex۰TГ) Call(args []any) []any { - c.fn(args[0].(typex.T), args[1].(func(typex.T))) - return []any{} -} - -func (c *callerTypex۰TEmitTypex۰TГ) Call2x0(arg0, arg1 any) { - c.fn(arg0.(typex.T), arg1.(func(typex.T))) -} - -type callerTypex۰TГTypex۰TInt struct { - fn func(typex.T) (typex.T, int) -} - -func funcMakerTypex۰TГTypex۰TInt(fn any) reflectx.Func { - f := fn.(func(typex.T) (typex.T, int)) - return &callerTypex۰TГTypex۰TInt{fn: f} -} - -func (c *callerTypex۰TГTypex۰TInt) Name() string { - return reflectx.FunctionName(c.fn) -} - -func (c *callerTypex۰TГTypex۰TInt) Type() reflect.Type { - return reflect.TypeOf(c.fn) -} - -func (c *callerTypex۰TГTypex۰TInt) Call(args []any) []any { - out0, out1 := c.fn(args[0].(typex.T)) - return []any{out0, out1} -} - -func (c *callerTypex۰TГTypex۰TInt) Call1x2(arg0 any) (any, any) { - return c.fn(arg0.(typex.T)) -} - -type callerГ struct { - fn func() -} - -func funcMakerГ(fn any) reflectx.Func { - f := fn.(func()) - return &callerГ{fn: f} -} - -func (c *callerГ) Name() string { - return reflectx.FunctionName(c.fn) -} - -func (c *callerГ) Type() reflect.Type { - return reflect.TypeOf(c.fn) -} - -func (c *callerГ) Call(args []any) []any { - c.fn() - return []any{} -} - -func (c *callerГ) Call0x0() { - c.fn() -} - -type emitNative struct { - n exec.ElementProcessor - fn any - est *sdf.WatermarkEstimator - - ctx context.Context - ws []typex.Window - et typex.EventTime - value exec.FullValue -} - -func (e *emitNative) Init(ctx context.Context, ws []typex.Window, et typex.EventTime) error { - e.ctx = ctx - e.ws = ws - e.et = et - return nil -} - -func (e *emitNative) Value() any { - return e.fn -} - -func (e *emitNative) AttachEstimator(est *sdf.WatermarkEstimator) { - e.est = est -} - -func emitMakerTypex۰T(n exec.ElementProcessor) exec.ReusableEmitter { - ret := &emitNative{n: n} - ret.fn = ret.invokeTypex۰T - return ret -} - -func (e *emitNative) invokeTypex۰T(val typex.T) { - e.value = exec.FullValue{Windows: e.ws, Timestamp: e.et, Elm: val} - if e.est != nil { - (*e.est).(sdf.TimestampObservingEstimator).ObserveTimestamp(e.et.ToTime()) - } - if err := e.n.ProcessElement(e.ctx, &e.value); err != nil { - panic(err) - } -} - -// DO NOT MODIFY: GENERATED CODE diff --git a/sdks/go/pkg/beam/transforms/filter/filter_test.go b/sdks/go/pkg/beam/transforms/filter/filter_test.go index 9cc5a526af9cf..ffa138e099a65 100644 --- a/sdks/go/pkg/beam/transforms/filter/filter_test.go +++ b/sdks/go/pkg/beam/transforms/filter/filter_test.go @@ -18,11 +18,28 @@ package filter_test import ( "testing" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/passert" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest" "github.com/apache/beam/sdks/v2/go/pkg/beam/transforms/filter" ) +func TestMain(m *testing.M) { + ptest.Main(m) +} + +func init() { + register.Function1x1(alwaysTrue) + register.Function1x1(alwaysFalse) + register.Function1x1(isOne) + register.Function1x1(greaterThanOne) +} + +func alwaysTrue(a int) bool { return true } +func alwaysFalse(a int) bool { return false } +func isOne(a int) bool { return a == 1 } +func greaterThanOne(a int) bool { return a > 1 } + func TestInclude(t *testing.T) { tests := []struct { in []int @@ -31,17 +48,17 @@ func TestInclude(t *testing.T) { }{ { []int{1, 2, 3}, - func(a int) bool { return true }, + alwaysTrue, []int{1, 2, 3}, }, { []int{1, 2, 3}, - func(a int) bool { return a == 1 }, + isOne, []int{1}, }, { []int{1, 2, 3}, - func(a int) bool { return a > 1 }, + greaterThanOne, []int{2, 3}, }, } @@ -64,17 +81,17 @@ func TestExclude(t *testing.T) { }{ { []int{1, 2, 3}, - func(a int) bool { return false }, + alwaysFalse, []int{1, 2, 3}, }, { []int{1, 2, 3}, - func(a int) bool { return a == 1 }, + isOne, []int{2, 3}, }, { []int{1, 2, 3}, - func(a int) bool { return a > 1 }, + greaterThanOne, []int{1}, }, } diff --git a/sdks/go/pkg/beam/transforms/stats/count_test.go b/sdks/go/pkg/beam/transforms/stats/count_test.go index 23627a92f7991..be6ce950e20a6 100644 --- a/sdks/go/pkg/beam/transforms/stats/count_test.go +++ b/sdks/go/pkg/beam/transforms/stats/count_test.go @@ -20,10 +20,19 @@ import ( "testing" "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/passert" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest" ) +func TestMain(m *testing.M) { + ptest.Main(m) +} + +func init() { + register.Function2x1(kvToCount) +} + type count struct { Elm int Count int diff --git a/sdks/go/pkg/beam/transforms/stats/max_test.go b/sdks/go/pkg/beam/transforms/stats/max_test.go index af817527dc91b..531792e70f58d 100644 --- a/sdks/go/pkg/beam/transforms/stats/max_test.go +++ b/sdks/go/pkg/beam/transforms/stats/max_test.go @@ -19,10 +19,16 @@ import ( "testing" "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/passert" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest" ) +func init() { + register.Function2x1(kvToStudent) + register.Function1x2(studentToKV) +} + type student struct { Name string Grade float64 diff --git a/sdks/go/pkg/beam/transforms/stats/quantiles.go b/sdks/go/pkg/beam/transforms/stats/quantiles.go index 79a66b58e1f02..6d2baa8b5e991 100644 --- a/sdks/go/pkg/beam/transforms/stats/quantiles.go +++ b/sdks/go/pkg/beam/transforms/stats/quantiles.go @@ -31,6 +31,7 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/util/reflectx" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" ) func init() { @@ -44,6 +45,9 @@ func init() { beam.RegisterType(reflect.TypeOf((*shardElementsFn)(nil)).Elem()) beam.RegisterCoder(compactorsType, encodeCompactors, decodeCompactors) beam.RegisterCoder(weightedElementType, encodeWeightedElement, decodeWeightedElement) + + register.Function1x2(fixedKey) + register.Function2x1(makeWeightedElement) } // Opts contains settings used to configure how approximate quantiles are computed. @@ -663,12 +667,14 @@ func makeWeightedElement(weight int, element beam.T) weightedElement { return weightedElement{weight: weight, element: element} } +func fixedKey(e beam.T) (int, beam.T) { return 1, e } + // ApproximateQuantiles computes approximate quantiles for the input PCollection. // // The output PCollection contains a single element: a list of numQuantiles - 1 elements approximately splitting up the input collection into numQuantiles separate quantiles. // For example, if numQuantiles = 2, the returned list would contain a single element such that approximately half of the input would be less than that element and half would be greater. func ApproximateQuantiles(s beam.Scope, pc beam.PCollection, less any, opts Opts) beam.PCollection { - return ApproximateWeightedQuantiles(s, beam.ParDo(s, func(e beam.T) (int, beam.T) { return 1, e }, pc), less, opts) + return ApproximateWeightedQuantiles(s, beam.ParDo(s, fixedKey, pc), less, opts) } // reduce takes a PCollection and returns a PCollection<*compactors>. The output PCollection may have at most shardSizes[len(shardSizes) - 1] compactors. diff --git a/sdks/go/pkg/beam/transforms/stats/quantiles_test.go b/sdks/go/pkg/beam/transforms/stats/quantiles_test.go index c03620d0b9b7a..1e389eed128b1 100644 --- a/sdks/go/pkg/beam/transforms/stats/quantiles_test.go +++ b/sdks/go/pkg/beam/transforms/stats/quantiles_test.go @@ -16,46 +16,19 @@ package stats import ( - "reflect" "testing" "github.com/apache/beam/sdks/v2/go/pkg/beam" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/util/reflectx" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/passert" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest" "github.com/google/go-cmp/cmp" ) func init() { - beam.RegisterFunction(weightedElementToKv) - - // In practice, this runs faster than plain reflection. - // TODO(https://github.com/apache/beam/issues/20271): Remove once collisions don't occur for starcgen over test code and an equivalent is generated for us. - reflectx.RegisterFunc(reflect.ValueOf(less).Type(), func(_ any) reflectx.Func { - return newIntLess() - }) -} - -type intLess struct { - name string - t reflect.Type -} - -func newIntLess() *intLess { - return &intLess{ - name: reflectx.FunctionName(reflect.ValueOf(less).Interface()), - t: reflect.ValueOf(less).Type(), - } -} - -func (i *intLess) Name() string { - return i.name -} -func (i *intLess) Type() reflect.Type { - return i.t -} -func (i *intLess) Call(args []any) []any { - return []any{args[0].(int) < args[1].(int)} + register.Function1x2(weightedElementToKv) + register.Function2x1(less) } func less(a, b int) bool { @@ -68,7 +41,7 @@ func TestLargeQuantiles(t *testing.T) { for i := 0; i < numElements; i++ { inputSlice = append(inputSlice, i) } - p, s, input, expected := ptest.CreateList2(inputSlice, [][]int{[]int{10006, 19973}}) + p, s, input, expected := ptest.CreateList2(inputSlice, [][]int{{10006, 19973}}) quantiles := ApproximateQuantiles(s, input, less, Opts{ K: 200, NumQuantiles: 3, @@ -85,7 +58,7 @@ func TestLargeQuantilesReversed(t *testing.T) { for i := numElements - 1; i >= 0; i-- { inputSlice = append(inputSlice, i) } - p, s, input, expected := ptest.CreateList2(inputSlice, [][]int{[]int{9985, 19959}}) + p, s, input, expected := ptest.CreateList2(inputSlice, [][]int{{9985, 19959}}) quantiles := ApproximateQuantiles(s, input, less, Opts{ K: 200, NumQuantiles: 3, @@ -103,8 +76,8 @@ func TestBasicQuantiles(t *testing.T) { Expected [][]int }{ {[]int{}, [][]int{}}, - {[]int{1}, [][]int{[]int{1}}}, - {[]int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, [][]int{[]int{6, 13}}}, + {[]int{1}, [][]int{{1}}}, + {[]int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, [][]int{{6, 13}}}, } for _, test := range tests { @@ -180,7 +153,7 @@ func TestMerging(t *testing.T) { K: 3, NumberOfCompactions: 1, Compactors: []compactor{{ - sorted: [][]beam.T{[]beam.T{1}, []beam.T{2}, []beam.T{3}}, + sorted: [][]beam.T{{1}, {2}, {3}}, unsorted: []beam.T{6, 5, 4}, capacity: 4, }}, @@ -191,7 +164,7 @@ func TestMerging(t *testing.T) { NumberOfCompactions: 1, Compactors: []compactor{ { - sorted: [][]beam.T{[]beam.T{7}, []beam.T{8}, []beam.T{9}}, + sorted: [][]beam.T{{7}, {8}, {9}}, unsorted: []beam.T{12, 11, 10}, capacity: 4}, }, @@ -205,7 +178,7 @@ func TestMerging(t *testing.T) { Compactors: []compactor{ {capacity: 4}, { - sorted: [][]beam.T{[]beam.T{1, 3, 5, 7, 9, 11}}, + sorted: [][]beam.T{{1, 3, 5, 7, 9, 11}}, capacity: 4, }, }, @@ -222,12 +195,12 @@ func TestCompactorsEncoding(t *testing.T) { Compactors: []compactor{ { capacity: 4, - sorted: [][]beam.T{[]beam.T{1, 2}}, + sorted: [][]beam.T{{1, 2}}, unsorted: []beam.T{3, 4}, }, { capacity: 4, - sorted: [][]beam.T{[]beam.T{5, 6}}, + sorted: [][]beam.T{{5, 6}}, unsorted: []beam.T{7, 8}, }, }, diff --git a/sdks/go/pkg/beam/transforms/top/top.go b/sdks/go/pkg/beam/transforms/top/top.go index f93786cd2293e..f82dc920aa865 100644 --- a/sdks/go/pkg/beam/transforms/top/top.go +++ b/sdks/go/pkg/beam/transforms/top/top.go @@ -29,14 +29,11 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/util/reflectx" "github.com/apache/beam/sdks/v2/go/pkg/beam/internal/errors" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" ) -//go:generate go install github.com/apache/beam/sdks/v2/go/cmd/starcgen -//go:generate starcgen --package=top -//go:generate go fmt - func init() { - beam.RegisterDoFn(reflect.TypeOf((*combineFn)(nil))) + register.Combiner3[accum, beam.T, []beam.T]((*combineFn)(nil)) } var ( @@ -157,10 +154,13 @@ func accumEnc() func(accum) ([]byte, error) { panic(err) } return func(a accum) ([]byte, error) { - if a.enc == nil { - return nil, errors.Errorf("top.accum: element encoder unspecified") + if len(a.list) > 0 && a.enc == nil { + return nil, errors.Errorf("top.accum: element encoder unspecified with non-zero elements: %v data available", len(a.data)) } var values [][]byte + if len(a.list) == 0 && len(a.data) > 0 { + values = a.data + } for _, value := range a.list { var buf bytes.Buffer if err := a.enc.Encode(value, &buf); err != nil { @@ -168,7 +168,6 @@ func accumEnc() func(accum) ([]byte, error) { } values = append(values, buf.Bytes()) } - a.list = nil var buf bytes.Buffer if err := coder.WriteSimpleRowHeader(1, &buf); err != nil { diff --git a/sdks/go/pkg/beam/transforms/top/top.shims.go b/sdks/go/pkg/beam/transforms/top/top.shims.go deleted file mode 100644 index 687046dfc86fd..0000000000000 --- a/sdks/go/pkg/beam/transforms/top/top.shims.go +++ /dev/null @@ -1,185 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Code generated by starcgen. DO NOT EDIT. -// File: top.shims.go - -package top - -import ( - "reflect" - - // Library imports - "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime" - "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/graphx/schema" - "github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex" - "github.com/apache/beam/sdks/v2/go/pkg/beam/core/util/reflectx" -) - -func init() { - runtime.RegisterType(reflect.TypeOf((*accum)(nil)).Elem()) - schema.RegisterType(reflect.TypeOf((*accum)(nil)).Elem()) - runtime.RegisterType(reflect.TypeOf((*combineFn)(nil)).Elem()) - schema.RegisterType(reflect.TypeOf((*combineFn)(nil)).Elem()) - reflectx.RegisterStructWrapper(reflect.TypeOf((*combineFn)(nil)).Elem(), wrapMakerCombineFn) - reflectx.RegisterFunc(reflect.TypeOf((*func(accum, accum) accum)(nil)).Elem(), funcMakerAccumAccumГAccum) - reflectx.RegisterFunc(reflect.TypeOf((*func(accum, typex.T) accum)(nil)).Elem(), funcMakerAccumTypex۰TГAccum) - reflectx.RegisterFunc(reflect.TypeOf((*func(accum) []typex.T)(nil)).Elem(), funcMakerAccumГSliceOfTypex۰T) - reflectx.RegisterFunc(reflect.TypeOf((*func())(nil)).Elem(), funcMakerГ) - reflectx.RegisterFunc(reflect.TypeOf((*func() accum)(nil)).Elem(), funcMakerГAccum) -} - -func wrapMakerCombineFn(fn any) map[string]reflectx.Func { - dfn := fn.(*combineFn) - return map[string]reflectx.Func{ - "AddInput": reflectx.MakeFunc(func(a0 accum, a1 typex.T) accum { return dfn.AddInput(a0, a1) }), - "CreateAccumulator": reflectx.MakeFunc(func() accum { return dfn.CreateAccumulator() }), - "ExtractOutput": reflectx.MakeFunc(func(a0 accum) []typex.T { return dfn.ExtractOutput(a0) }), - "MergeAccumulators": reflectx.MakeFunc(func(a0 accum, a1 accum) accum { return dfn.MergeAccumulators(a0, a1) }), - "Setup": reflectx.MakeFunc(func() { dfn.Setup() }), - } -} - -type callerAccumAccumГAccum struct { - fn func(accum, accum) accum -} - -func funcMakerAccumAccumГAccum(fn any) reflectx.Func { - f := fn.(func(accum, accum) accum) - return &callerAccumAccumГAccum{fn: f} -} - -func (c *callerAccumAccumГAccum) Name() string { - return reflectx.FunctionName(c.fn) -} - -func (c *callerAccumAccumГAccum) Type() reflect.Type { - return reflect.TypeOf(c.fn) -} - -func (c *callerAccumAccumГAccum) Call(args []any) []any { - out0 := c.fn(args[0].(accum), args[1].(accum)) - return []any{out0} -} - -func (c *callerAccumAccumГAccum) Call2x1(arg0, arg1 any) any { - return c.fn(arg0.(accum), arg1.(accum)) -} - -type callerAccumTypex۰TГAccum struct { - fn func(accum, typex.T) accum -} - -func funcMakerAccumTypex۰TГAccum(fn any) reflectx.Func { - f := fn.(func(accum, typex.T) accum) - return &callerAccumTypex۰TГAccum{fn: f} -} - -func (c *callerAccumTypex۰TГAccum) Name() string { - return reflectx.FunctionName(c.fn) -} - -func (c *callerAccumTypex۰TГAccum) Type() reflect.Type { - return reflect.TypeOf(c.fn) -} - -func (c *callerAccumTypex۰TГAccum) Call(args []any) []any { - out0 := c.fn(args[0].(accum), args[1].(typex.T)) - return []any{out0} -} - -func (c *callerAccumTypex۰TГAccum) Call2x1(arg0, arg1 any) any { - return c.fn(arg0.(accum), arg1.(typex.T)) -} - -type callerAccumГSliceOfTypex۰T struct { - fn func(accum) []typex.T -} - -func funcMakerAccumГSliceOfTypex۰T(fn any) reflectx.Func { - f := fn.(func(accum) []typex.T) - return &callerAccumГSliceOfTypex۰T{fn: f} -} - -func (c *callerAccumГSliceOfTypex۰T) Name() string { - return reflectx.FunctionName(c.fn) -} - -func (c *callerAccumГSliceOfTypex۰T) Type() reflect.Type { - return reflect.TypeOf(c.fn) -} - -func (c *callerAccumГSliceOfTypex۰T) Call(args []any) []any { - out0 := c.fn(args[0].(accum)) - return []any{out0} -} - -func (c *callerAccumГSliceOfTypex۰T) Call1x1(arg0 any) any { - return c.fn(arg0.(accum)) -} - -type callerГ struct { - fn func() -} - -func funcMakerГ(fn any) reflectx.Func { - f := fn.(func()) - return &callerГ{fn: f} -} - -func (c *callerГ) Name() string { - return reflectx.FunctionName(c.fn) -} - -func (c *callerГ) Type() reflect.Type { - return reflect.TypeOf(c.fn) -} - -func (c *callerГ) Call(args []any) []any { - c.fn() - return []any{} -} - -func (c *callerГ) Call0x0() { - c.fn() -} - -type callerГAccum struct { - fn func() accum -} - -func funcMakerГAccum(fn any) reflectx.Func { - f := fn.(func() accum) - return &callerГAccum{fn: f} -} - -func (c *callerГAccum) Name() string { - return reflectx.FunctionName(c.fn) -} - -func (c *callerГAccum) Type() reflect.Type { - return reflect.TypeOf(c.fn) -} - -func (c *callerГAccum) Call(args []any) []any { - out0 := c.fn() - return []any{out0} -} - -func (c *callerГAccum) Call0x1() any { - return c.fn() -} - -// DO NOT MODIFY: GENERATED CODE diff --git a/sdks/go/pkg/beam/transforms/top/top_test.go b/sdks/go/pkg/beam/transforms/top/top_test.go index bf641e6ec3730..39d774a663008 100644 --- a/sdks/go/pkg/beam/transforms/top/top_test.go +++ b/sdks/go/pkg/beam/transforms/top/top_test.go @@ -22,17 +22,33 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/util/reflectx" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/passert" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest" ) +func TestMain(m *testing.M) { + ptest.Main(m) +} + +func init() { + register.Function2x2(addKeyFn) + register.Function2x1(lessInt) + register.Function2x1(shorterString) +} + +func lessInt(a, b int) bool { + return a < b +} + +func shorterString(a, b string) bool { + return len(a) < len(b) +} + // TestCombineFn3String verifies that the accumulator correctly // maintains the top 3 longest strings. func TestCombineFn3String(t *testing.T) { - less := func(a, b string) bool { - return len(a) < len(b) - } - fn := newCombineFn(less, 3, reflectx.String, false) + fn := newCombineFn(shorterString, 3, reflectx.String, false) tests := []struct { Elms []string @@ -57,10 +73,7 @@ func TestCombineFn3String(t *testing.T) { // TestCombineFn3RevString verifies that the accumulator correctly // maintains the top 3 shortest strings. func TestCombineFn3RevString(t *testing.T) { - less := func(a, b string) bool { - return len(a) < len(b) - } - fn := newCombineFn(less, 3, reflectx.String, true) + fn := newCombineFn(shorterString, 3, reflectx.String, true) tests := []struct { Elms []string @@ -86,10 +99,7 @@ func TestCombineFn3RevString(t *testing.T) { // extractOutput still works on the marshalled accumulators it receives after // merging. func TestCombineFnMerge(t *testing.T) { - less := func(a, b string) bool { - return len(a) < len(b) - } - fn := newCombineFn(less, 3, reflectx.String, false) + fn := newCombineFn(shorterString, 3, reflectx.String, false) tests := []struct { Elms [][]string Expected []string @@ -170,12 +180,9 @@ func output(fn *combineFn, a accum) []string { // TestLargest checks that the Largest transform outputs the correct elements // for a given PCollection of ints and a comparator function. func TestLargest(t *testing.T) { - less := func(a, b int) bool { - return a < b - } p, s := beam.NewPipelineWithRoot() col := beam.Create(s, 1, 11, 7, 5, 10) - topTwo := Largest(s, col, 2, less) + topTwo := Largest(s, col, 2, lessInt) passert.Equals(s, topTwo, []int{11, 10}) if err := ptest.Run(p); err != nil { t.Errorf("pipeline failed but should have succeeded, got %v", err) @@ -185,12 +192,9 @@ func TestLargest(t *testing.T) { // TestSmallest checks that the Smallest transform outputs the correct elements // for a given PCollection of ints and a comparator function. func TestSmallest(t *testing.T) { - less := func(a, b int) bool { - return a < b - } p, s := beam.NewPipelineWithRoot() col := beam.Create(s, 1, 11, 7, 5, 10) - botTwo := Smallest(s, col, 2, less) + botTwo := Smallest(s, col, 2, lessInt) passert.Equals(s, botTwo, []int{1, 5}) if err := ptest.Run(p); err != nil { t.Errorf("pipeline failed but should have succeeded, got %v", err) @@ -209,9 +213,6 @@ func addKeyFn(elm beam.T, newKey int) (int, beam.T) { // TestLargestPerKey ensures that the LargestPerKey transform outputs the proper // collection for a PCollection of type . func TestLargestPerKey(t *testing.T) { - less := func(a, b int) bool { - return a < b - } p, s := beam.NewPipelineWithRoot() colZero := beam.Create(s, 1, 11, 7, 5, 10) keyedZero := addKey(s, colZero, 0) @@ -220,7 +221,7 @@ func TestLargestPerKey(t *testing.T) { keyedOne := addKey(s, colOne, 1) col := beam.Flatten(s, keyedZero, keyedOne) - top := LargestPerKey(s, col, 2, less) + top := LargestPerKey(s, col, 2, lessInt) out := beam.DropKey(s, top) passert.Equals(s, out, []int{11, 10}, []int{12, 11}) if err := ptest.Run(p); err != nil { @@ -231,9 +232,6 @@ func TestLargestPerKey(t *testing.T) { // TestSmallestPerKey ensures that the SmallestPerKey transform outputs the proper // collection for a PCollection of type . func TestSmallestPerKey(t *testing.T) { - less := func(a, b int) bool { - return a < b - } p, s := beam.NewPipelineWithRoot() colZero := beam.Create(s, 1, 11, 7, 5, 10) keyedZero := addKey(s, colZero, 0) @@ -242,7 +240,7 @@ func TestSmallestPerKey(t *testing.T) { keyedOne := addKey(s, colOne, 1) col := beam.Flatten(s, keyedZero, keyedOne) - bot := SmallestPerKey(s, col, 2, less) + bot := SmallestPerKey(s, col, 2, lessInt) out := beam.DropKey(s, bot) passert.Equals(s, out, []int{1, 5}, []int{2, 6}) if err := ptest.Run(p); err != nil { diff --git a/sdks/go/pkg/beam/transforms/xlang/inference/inference.go b/sdks/go/pkg/beam/transforms/xlang/inference/inference.go index b55ccb276c4b5..35e0a0b234c6d 100644 --- a/sdks/go/pkg/beam/transforms/xlang/inference/inference.go +++ b/sdks/go/pkg/beam/transforms/xlang/inference/inference.go @@ -14,7 +14,7 @@ // limitations under the License. // Package inference has the cross language implementation of RunInference API implemented in Python SDK. -// An exapnsion service for python external transforms can be started by running +// An expansion service for python external transforms can be started by running // // $ python -m apache_beam.runners.portability.expansion_service_main -p $PORT_FOR_EXPANSION_SERVICE package inference diff --git a/sdks/go/pkg/beam/transforms/xlang/schema/external.go b/sdks/go/pkg/beam/transforms/xlang/schema/external.go new file mode 100644 index 0000000000000..75be90cbe7b3a --- /dev/null +++ b/sdks/go/pkg/beam/transforms/xlang/schema/external.go @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package schema has the cross language implementation for calling schema transforms in other language SDKs. +package schema + +import ( + "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/xlangx" + pipepb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/pipeline_v1" + "github.com/golang/protobuf/proto" +) + +const schemaTransformURN = "beam:expansion:payload:schematransform:v1" + +type options struct { + inputs map[string]beam.PCollection + outputTypes map[string]beam.FullType + expansionAddr string +} + +// Option is the base type of all the schema transform options. +type Option func(*options) + +// Input adds a named PCollection input to the transform. +func Input(name string, in beam.PCollection) Option { + return func(opts *options) { + if opts.inputs == nil { + opts.inputs = map[string]beam.PCollection{} + } + opts.inputs[name] = in + } +} + +// OutputType specifies an output PCollection type of the transform. +// It must match the external transform's output schema. +func OutputType(name string, tpe beam.FullType) Option { + return func(opts *options) { + if opts.outputTypes == nil { + opts.outputTypes = map[string]beam.FullType{} + } + opts.outputTypes[name] = tpe + } +} + +// UnnamedOutputType specifies an output PCollection type of the transform. +// It must match the external transform's output schema. This is simply +// syntactic sugar for OutputType(beam.UnnamedOutputTag(), tpe). +func UnnamedOutputType(tpe beam.FullType) Option { + return OutputType(beam.UnnamedOutputTag(), tpe) +} + +// ExpansionAddr is the URL of the expansion service to use. +func ExpansionAddr(addr string) Option { + return func(opts *options) { + opts.expansionAddr = addr + } +} + +// Transform configures a new cross language transform to call a "schema transform" in an external SDK. +func Transform(scope beam.Scope, config any, transformIdentifier string, opts ...Option) map[string]beam.PCollection { + ecp, err := xlangx.CreateExternalConfigurationPayload(config) + if err != nil { + panic(err) + } + + pl, err := proto.Marshal(&pipepb.SchemaTransformPayload{ + Identifier: transformIdentifier, + ConfigurationSchema: ecp.GetSchema(), + ConfigurationRow: ecp.GetPayload(), + }) + if err != nil { + panic(err) + } + + allOpts := &options{} + for _, o := range opts { + o(allOpts) + } + + return beam.CrossLanguage(scope, schemaTransformURN, pl, allOpts.expansionAddr, allOpts.inputs, allOpts.outputTypes) +} diff --git a/sdks/go/pkg/beam/util.go b/sdks/go/pkg/beam/util.go index d591dedd7624a..4b24af8311349 100644 --- a/sdks/go/pkg/beam/util.go +++ b/sdks/go/pkg/beam/util.go @@ -16,7 +16,7 @@ package beam //go:generate go install github.com/apache/beam/sdks/v2/go/cmd/starcgen -//go:generate starcgen --package=beam --identifiers=addFixedKeyFn,dropKeyFn,dropValueFn,swapKVFn,explodeFn,jsonDec,jsonEnc,protoEnc,protoDec,schemaEnc,schemaDec,makePartitionFn,createFn +//go:generate starcgen --package=beam --identifiers=addFixedKeyFn,dropKeyFn,dropValueFn,swapKVFn,explodeFn,jsonDec,jsonEnc,protoEnc,protoDec,schemaEnc,schemaDec,makePartitionFn //go:generate go fmt // We have some freedom to create various utilities, users can use depending on diff --git a/sdks/go/pkg/beam/util/execx/exec.go b/sdks/go/pkg/beam/util/execx/exec.go index 6558df3a2cb4a..aaaf9355e7c1e 100644 --- a/sdks/go/pkg/beam/util/execx/exec.go +++ b/sdks/go/pkg/beam/util/execx/exec.go @@ -17,6 +17,7 @@ package execx import ( + "io" "os" "os/exec" ) @@ -24,16 +25,22 @@ import ( // Execute runs the program with the given arguments. It attaches stdio to the // child process. func Execute(prog string, args ...string) error { - return ExecuteEnv(nil, prog, args...) + return ExecuteEnvWithIO(nil, os.Stdin, os.Stdout, os.Stderr, prog, args...) } -// Execute runs the program with the given arguments with additional environment +// ExecuteEnv runs the program with the given arguments with additional environment // variables. It attaches stdio to the child process. func ExecuteEnv(env map[string]string, prog string, args ...string) error { + return ExecuteEnvWithIO(env, os.Stdin, os.Stdout, os.Stderr, prog, args...) +} + +// ExecuteEnvWithIO runs the program with the given arguments with additional environment +// variables. It attaches custom IO to the child process. +func ExecuteEnvWithIO(env map[string]string, stdin io.Reader, stdout, stderr io.Writer, prog string, args ...string) error { cmd := exec.Command(prog, args...) - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr + cmd.Stdin = stdin + cmd.Stdout = stdout + cmd.Stderr = stderr if env != nil { cmd.Env = os.Environ() for k, v := range env { diff --git a/sdks/go/pkg/beam/x/beamx/run.go b/sdks/go/pkg/beam/x/beamx/run.go index 0355e453995e6..0be42561b658d 100644 --- a/sdks/go/pkg/beam/x/beamx/run.go +++ b/sdks/go/pkg/beam/x/beamx/run.go @@ -32,6 +32,7 @@ import ( _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/direct" _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/dot" _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/flink" + _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism" _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/samza" _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/spark" _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/universal" @@ -39,7 +40,7 @@ import ( var ( runner = runners.Runner - defaultRunner = "direct" + defaultRunner = "prism" ) func getRunner() string { @@ -51,7 +52,7 @@ func getRunner() string { } // Run invokes beam.Run with the runner supplied by the flag "runner". It -// defaults to the direct runner, but all beam-distributed runners and textio +// defaults to the prism runner, but all beam-distributed runners and textio // filesystems are implicitly registered. func Run(ctx context.Context, p *beam.Pipeline) error { _, err := beam.Run(ctx, getRunner(), p) diff --git a/sdks/go/pkg/beam/x/debug/head_test.go b/sdks/go/pkg/beam/x/debug/head_test.go index 8aa5b41545daa..4e3db1e1e3933 100644 --- a/sdks/go/pkg/beam/x/debug/head_test.go +++ b/sdks/go/pkg/beam/x/debug/head_test.go @@ -23,6 +23,12 @@ import ( "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest" ) +func TestMain(m *testing.M) { + // Override with direct runner to avoid + // flaky race conditions over the logging services. + ptest.MainWithDefault(m, "direct") +} + func TestHead(t *testing.T) { p, s, sequence := ptest.CreateList([]int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) headSequence := Head(s, sequence, 5) diff --git a/sdks/go/run_with_go_version.sh b/sdks/go/run_with_go_version.sh index b31c44771749f..aa31399193a6a 100755 --- a/sdks/go/run_with_go_version.sh +++ b/sdks/go/run_with_go_version.sh @@ -37,7 +37,7 @@ set -e # # This variable is also used as the execution command downscript. # The list of downloadable versions are at https://go.dev/dl/ -GOVERS=go1.20.4 +GOVERS=go1.21.0 if ! command -v go &> /dev/null then diff --git a/sdks/go/test/build.gradle b/sdks/go/test/build.gradle index d1048b49c0bde..5b39cf81400f8 100644 --- a/sdks/go/test/build.gradle +++ b/sdks/go/test/build.gradle @@ -28,11 +28,28 @@ task dataflowValidatesRunner() { group = "Verification" dependsOn ":sdks:go:test:goBuild" - dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar" + + doLast { + def options = [ + "--runner dataflow", + ] + exec { + executable "sh" + args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" + } + } +} + +// ValidatesRunner tests for Dataflow. Runs tests in the integration directory +// with Dataflow to validate that the runner behaves as expected, on arm64 machines. +task dataflowValidatesRunnerARM64() { + group = "Verification" + + dependsOn ":sdks:go:test:goBuild" doLast { def pipelineOptions = [ // Pipeline options piped directly to Go SDK flags. - "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").buildTestExpansionServiceJar.archivePath}", + "--machine_type=t2a-standard-1", ] def options = [ "--runner dataflow", @@ -156,6 +173,30 @@ tasks.register("ulrValidatesRunner") { } } +// ValidatesRunner tests for Prism. Runs tests in the integration directory +// with prism in docker mod to validate that the runner behaves as expected. +task prismValidatesRunner { + group = "Verification" + + dependsOn ":sdks:go:test:goBuild" + dependsOn ":sdks:go:container:docker" + dependsOn ":sdks:java:container:java8:docker" + dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar" + doLast { + def pipelineOptions = [ // Pipeline options piped directly to Go SDK flags. + "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").buildTestExpansionServiceJar.archivePath}", + ] + def options = [ + "--runner prism", + "--pipeline_opts \"${pipelineOptions.join(' ')}\"", + ] + exec { + executable "sh" + args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" + } + } +} + // A method for configuring a cross-language validates runner test task, // intended to be used in calls to createCrossLanguageValidatesRunnerTask. ext.goIoValidatesRunnerTask = { proj, name, scriptOpts, pipelineOpts -> @@ -183,6 +224,7 @@ ext.goIoValidatesRunnerTask = { proj, name, scriptOpts, pipelineOpts -> "--expansion_jar=debeziumio:${debeziumIoExpJar}", "--expansion_jar=gcpio:${gcpIoExpJar}", "--bq_dataset=apache-beam-testing.beam_bigquery_io_test_temp", + "--bt_instance=projects/apache-beam-testing/instances/beam-test" ] pipelineOptions.addAll(pipelineOpts) def options = [ diff --git a/sdks/go/test/integration/flags.go b/sdks/go/test/integration/flags.go index 75e2fdcb10c68..4e208f6939cc7 100644 --- a/sdks/go/test/integration/flags.go +++ b/sdks/go/test/integration/flags.go @@ -47,6 +47,11 @@ var ( BigQueryDataset = flag.String("bq_dataset", "", "Name of the dataset to create tables in for BigQuery tests.") + // BigtableInstance is the name of the Bigtable instance to create tables in + // for Bigtable integration tests. + BigtableInstance = flag.String("bt_instance", "", + "Name of the Bigtable instance to create tables in for Bigtable tests.") + // ExpansionJars contains elements in the form "label:jar" describing jar // filepaths for expansion services to use in integration tests, and the // corresponding labels. Once provided through this flag, those jars can diff --git a/sdks/go/test/integration/integration.go b/sdks/go/test/integration/integration.go index 1007d9e64dc60..5b7473fb561a4 100644 --- a/sdks/go/test/integration/integration.go +++ b/sdks/go/test/integration/integration.go @@ -68,6 +68,7 @@ var directFilters = []string{ "TestXLang.*", "TestKafkaIO.*", "TestBigQueryIO.*", + "TestBigtableIO.*", "TestSpannerIO.*", "TestDebeziumIO_BasicRead", "TestJDBCIO_BasicReadWrite", @@ -114,6 +115,7 @@ var portableFilters = []string{ "TestKafkaIO.*", // TODO(BEAM-13215): GCP IOs currently do not work in non-Dataflow portable runners. "TestBigQueryIO.*", + "TestBigtableIO.*", "TestSpannerIO.*", // The portable runner does not support self-checkpointing "TestCheckpointing", @@ -136,12 +138,46 @@ var portableFilters = []string{ "TestSetStateClear", } +var prismFilters = []string{ + // The prism runner does not yet support Java's CoGBK. + "TestXLang_CoGroupBy", + // The prism runner does not support the TestStream primitive + "TestTestStream.*", + // The trigger and pane tests uses TestStream + "TestTrigger.*", + "TestPanes", + + // TODO(https://github.com/apache/beam/issues/21058): Xlang ios don't yet work on prism. + "TestKafkaIO.*", + // TODO(BEAM-13215): GCP IOs currently do not work in non-Dataflow portable runners. + "TestBigQueryIO.*", + "TestSpannerIO.*", + // The prism runner does not support pipeline drain for SDF. + "TestDrain", + // FhirIO currently only supports Dataflow runner + "TestFhirIO.*", + // OOMs currently only lead to heap dumps on Dataflow runner + "TestOomParDo", + // The prism runner does not support user state. + "TestValueState", + "TestValueStateWindowed", + "TestValueStateClear", + "TestBagState", + "TestBagStateClear", + "TestCombiningState", + "TestMapState", + "TestMapStateClear", + "TestSetState", + "TestSetStateClear", +} + var flinkFilters = []string{ // TODO(https://github.com/apache/beam/issues/20723): Flink tests timing out on reads. "TestXLang_Combine.*", "TestDebeziumIO_BasicRead", // TODO(BEAM-13215): GCP IOs currently do not work in non-Dataflow portable runners. "TestBigQueryIO.*", + "TestBigtableIO.*", "TestSpannerIO.*", // The number of produced outputs in AfterSynchronizedProcessingTime varies in different runs. "TestTriggerAfterSynchronizedProcessingTime", @@ -171,6 +207,7 @@ var samzaFilters = []string{ "TestWordCount.*", // TODO(BEAM-13215): GCP IOs currently do not work in non-Dataflow portable runners. "TestBigQueryIO.*", + "TestBigtableIO.*", "TestSpannerIO.*", // The Samza runner does not support self-checkpointing "TestCheckpointing", @@ -209,6 +246,7 @@ var sparkFilters = []string{ "TestDebeziumIO_BasicRead", // TODO(BEAM-13215): GCP IOs currently do not work in non-Dataflow portable runners. "TestBigQueryIO.*", + "TestBigtableIO.*", "TestSpannerIO.*", // The spark runner does not support self-checkpointing "TestCheckpointing", @@ -249,7 +287,7 @@ var dataflowFilters = []string{ "TestCheckpointing", // TODO(21761): This test needs to provide GCP project to expansion service. "TestBigQueryIO_BasicWriteQueryRead", - // Can't handle the test spanner container or access a local spanner. + // Can't handle the test spanner container or access a local spanner. "TestSpannerIO.*", // Dataflow does not drain jobs by itself. "TestDrain", @@ -282,7 +320,6 @@ func CheckFilters(t *testing.T) { s1 := rand.NewSource(time.Now().UnixNano()) r1 := rand.New(s1) *jobopts.JobName = fmt.Sprintf("go-%v-%v", strings.ToLower(n), r1.Intn(1000)) - // Test for runner-specific skipping second. var filters []string runner := *ptest.Runner @@ -292,6 +329,8 @@ func CheckFilters(t *testing.T) { switch runner { case "direct", "DirectRunner": filters = directFilters + case "prism", "PrismRunner": + filters = prismFilters case "portable", "PortableRunner": filters = portableFilters case "flink", "FlinkRunner": diff --git a/sdks/go/test/integration/io/fhirio/fhirio_test.go b/sdks/go/test/integration/io/fhirio/fhirio_test.go index 03e3654d5c49b..01f6db9324f31 100644 --- a/sdks/go/test/integration/io/fhirio/fhirio_test.go +++ b/sdks/go/test/integration/io/fhirio/fhirio_test.go @@ -96,9 +96,9 @@ func setupFhirStore(t *testing.T, shouldPopulateStore bool) (fhirStoreInfo, func var resourcePaths [][]byte if shouldPopulateStore { - resourcePaths = populateStore(createdFhirStorePath) - if len(resourcePaths) == 0 { - t.Fatal("No data got populated to test") + resourcePaths, err = populateStore(createdFhirStorePath) + if err != nil { + t.Fatal(err) } } @@ -127,11 +127,13 @@ func deleteStore(storePath string) (*healthcare.Empty, error) { // Populates fhir store with data. Note that failure to populate some data is not // detrimental to the tests, so it is fine to ignore. -func populateStore(storePath string) [][]byte { +func populateStore(storePath string) ([][]byte, error) { resourcePaths := make([][]byte, 0) + bufferedErrors := make([]string, 0) for _, bundle := range readPrettyBundles() { response, err := storeService.ExecuteBundle(storePath, strings.NewReader(bundle)).Do() if err != nil { + bufferedErrors = append(bufferedErrors, err.Error()) continue } @@ -145,23 +147,30 @@ func populateStore(storePath string) [][]byte { } err = json.NewDecoder(response.Body).Decode(&body) if err != nil { + bufferedErrors = append(bufferedErrors, err.Error()) continue } for _, entry := range body.Entry { bundleFailedToBeCreated := !strings.Contains(entry.Response.Status, "201") if bundleFailedToBeCreated { + bufferedErrors = append(bufferedErrors, fmt.Sprintf("Bundle creation failed with: %v", entry.Response)) continue } resourcePath, err := extractResourcePathFrom(entry.Response.Location) if err != nil { + bufferedErrors = append(bufferedErrors, err.Error()) continue } resourcePaths = append(resourcePaths, resourcePath) } } - return resourcePaths + if len(resourcePaths) == 0 { + return nil, fmt.Errorf("failed to populate fhir store with any data. Errors with requests: %s", bufferedErrors) + } + + return resourcePaths, nil } func readPrettyBundles() []string { diff --git a/sdks/go/test/integration/io/xlang/bigtable/bigtable.go b/sdks/go/test/integration/io/xlang/bigtable/bigtable.go new file mode 100644 index 0000000000000..a578ed380288e --- /dev/null +++ b/sdks/go/test/integration/io/xlang/bigtable/bigtable.go @@ -0,0 +1,17 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package bigtable contains integration tests for cross-language Bigtable IO transforms. +package bigtable diff --git a/sdks/go/test/integration/io/xlang/bigtable/bigtable_test.go b/sdks/go/test/integration/io/xlang/bigtable/bigtable_test.go new file mode 100644 index 0000000000000..2b8eece9d69c9 --- /dev/null +++ b/sdks/go/test/integration/io/xlang/bigtable/bigtable_test.go @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigtable + +import ( + "context" + "flag" + "fmt" + "log" + "regexp" + "strconv" + "testing" + "time" + + bt "cloud.google.com/go/bigtable" + "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/io/bigtableio" + xlangbt "github.com/apache/beam/sdks/v2/go/pkg/beam/io/xlang/bigtableio" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" + _ "github.com/apache/beam/sdks/v2/go/pkg/beam/runners/dataflow" + "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/passert" + "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest" + "github.com/apache/beam/sdks/v2/go/test/integration" +) + +var expansionAddr string + +func init() { + beam.RegisterDoFn(&CreateTestRowsFn{}) + beam.RegisterDoFn(&RowToMutationFn{}) + + register.DoFn2x0[[]byte, func(xlangbt.Row)](&CreateTestRowsFn{}) + register.DoFn2x0[xlangbt.Row, func(mutation bigtableio.Mutation)](&RowToMutationFn{}) + register.Emitter1[bigtableio.Mutation]() + register.Emitter1[xlangbt.Row]() +} + +type CreateTestRowsFn struct { +} + +type RowToMutationFn struct { +} + +func (fn *RowToMutationFn) ProcessElement(r xlangbt.Row, emit func(mutation bigtableio.Mutation)) { + mut := bigtableio.NewMutation(string(r.Key)) + for cf, cols := range r.Column_families { + for col, vals := range cols { + for _, val := range vals { + mut.Set(cf, col, bt.Timestamp(val.Timestamp_micros), val.Value) + } + } + } + emit(*mut) +} + +func createTestRows() []xlangbt.Row { + rows := make([]xlangbt.Row, 100) + + for i := 0; i < 100; i++ { + key := strconv.FormatInt(int64(i), 10) + row := xlangbt.Row{ + Key: []byte(key), + } + row.AddCell("cf1", "q1", []byte(strconv.FormatInt(int64(i)*1000, 10)), int64(i)*1000) + + rows[i] = row + } + return rows +} + +func (fn *CreateTestRowsFn) ProcessElement(_ []byte, emit func(row xlangbt.Row)) { + for _, r := range createTestRows() { + emit(r) + } +} + +func checkFlags(t *testing.T) { + if *integration.BigtableInstance == "" { + t.Skip("No Bigtable instance provided.") + } +} + +func WritePipeline(project, instance, table string) *beam.Pipeline { + p := beam.NewPipeline() + s := p.Root() + + rows := beam.ParDo(s, &CreateTestRowsFn{}, beam.Impulse(s)) + muts := beam.ParDo(s, &RowToMutationFn{}, rows) + bigtableio.Write(s, project, instance, table, muts) + return p +} + +func ReadPipeline(project, instance, table, expansionAddr string) *beam.Pipeline { + p := beam.NewPipeline() + s := p.Root() + + rowsFromBt := xlangbt.Read(s, project, instance, table, xlangbt.ReadExpansionAddr(expansionAddr)) + + rows := beam.ParDo(s, &CreateTestRowsFn{}, beam.Impulse(s)) + + passert.Equals(s, rowsFromBt, rows) + return p +} + +var instanceRegex = regexp.MustCompile("projects/(?P[^/]+)/instances/(?P.+)") + +func createTempTable(admin *bt.AdminClient) string { + tableName := fmt.Sprintf("go_btio_it_temp_%v", time.Now().UnixNano()) + err := admin.CreateTableFromConf(context.Background(), &bt.TableConf{ + TableID: tableName, + Families: map[string]bt.GCPolicy{ + "cf1": bt.NoGcPolicy(), + }, + }) + if err != nil { + panic(err) + } + + return tableName +} + +func deleteTempTable(admin *bt.AdminClient, table string) { + _ = admin.DeleteTable(context.Background(), table) +} + +func TestBigtableIO_BasicWriteRead(t *testing.T) { + integration.CheckFilters(t) + checkFlags(t) + + instancePath := *integration.BigtableInstance + matches := instanceRegex.FindStringSubmatch(instancePath) + project := matches[1] + instance := matches[2] + + admin, err := bt.NewAdminClient(context.Background(), project, instance) + if err != nil { + panic(err) + } + + table := createTempTable(admin) + t.Cleanup(func() { + deleteTempTable(admin, table) + }) + + write := WritePipeline(project, instance, table) + ptest.RunAndValidate(t, write) + + read := ReadPipeline(project, instance, table, expansionAddr) + ptest.RunAndValidate(t, read) +} + +func TestMain(m *testing.M) { + flag.Parse() + beam.Init() + + services := integration.NewExpansionServices() + defer func() { services.Shutdown() }() + addr, err := services.GetAddr("gcpio") + if err != nil { + log.Printf("skipping missing expansion service: %v", err) + } else { + expansionAddr = addr + } + + ptest.MainRet(m) +} diff --git a/sdks/go/test/integration/primitives/cogbk.go b/sdks/go/test/integration/primitives/cogbk.go index a624efc0b81bb..4a3a39b819e82 100644 --- a/sdks/go/test/integration/primitives/cogbk.go +++ b/sdks/go/test/integration/primitives/cogbk.go @@ -20,9 +20,23 @@ import ( "fmt" "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/passert" ) +func init() { + register.Function2x0(genA) + register.Function2x0(genB) + register.Function2x0(genC) + register.Function2x0(genD) + register.Function3x0(shortFn) + register.Function5x0(joinFn) + register.Function6x0(splitFn) + register.Emitter2[string, int]() + register.Emitter2[string, string]() + register.Iter1[int]() +} + func genA(_ []byte, emit func(string, int)) { emit("a", 1) emit("a", 2) diff --git a/sdks/go/test/integration/primitives/pardo.go b/sdks/go/test/integration/primitives/pardo.go index c444dedfe9d31..2c2383ea90ba9 100644 --- a/sdks/go/test/integration/primitives/pardo.go +++ b/sdks/go/test/integration/primitives/pardo.go @@ -31,6 +31,7 @@ func init() { register.Function1x2(splitStringPair) register.Function3x2(asymJoinFn) register.Function5x0(splitByName) + register.Function2x0(emitPipelineOptions) register.Iter1[int]() register.Iter2[int, int]() diff --git a/sdks/go/test/integration/xlang/expansion_test.go b/sdks/go/test/integration/xlang/expansion_test.go index 2cd032384f51d..6b87f493d9c47 100644 --- a/sdks/go/test/integration/xlang/expansion_test.go +++ b/sdks/go/test/integration/xlang/expansion_test.go @@ -17,21 +17,56 @@ package xlang import ( "os" + "strconv" + "strings" "testing" + "github.com/apache/beam/sdks/v2/go/pkg/beam/core" "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/xlangx/expansionx" "github.com/apache/beam/sdks/v2/go/test/integration" ) const ( - // TODO(https://github.com/apache/beam/issues/21279): Select the most recent Beam release instead of a hard-coded - // string. - beamVersion = "2.34.0" gradleTarget = ":sdks:java:io:expansion-service:runExpansionService" ) +func getTargetVersion(currentVersion string) string { + curVersion := strings.Split(currentVersion, ".") + minorVersion, _ := strconv.Atoi(curVersion[1]) + targetMinor := minorVersion - 2 + curVersion[1] = strconv.Itoa(targetMinor) + return strings.Join(curVersion[0:3], ".") +} + +func TestGetTargetVersion(t *testing.T) { + var tests = []struct { + name string + input string + expOutput string + }{ + { + "normal version", + "2.50.0", + "2.48.0", + }, + { + "dev version", + "2.10.0.dev", + "2.8.0", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got, want := getTargetVersion(test.input), test.expOutput; got != want { + t.Errorf("got %v, want %v", got, want) + } + }) + } +} + func TestAutomatedExpansionService(t *testing.T) { integration.CheckFilters(t) + beamVersion := getTargetVersion(core.SdkVersion) jarPath, err := expansionx.GetBeamJar(gradleTarget, beamVersion) if err != nil { t.Fatalf("failed to get JAR path, got %v", err) diff --git a/sdks/go/test/regression/lperror.go b/sdks/go/test/regression/lperror.go index 088f81d7a7cbc..db327e588a589 100644 --- a/sdks/go/test/regression/lperror.go +++ b/sdks/go/test/regression/lperror.go @@ -22,8 +22,15 @@ import ( "sort" "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" ) +func init() { + register.Function2x2(toFoo) + register.Iter1[*fruit]() + register.Function3x1(toID) +} + // REPRO found by https://github.com/zelliott type fruit struct { diff --git a/sdks/go/test/regression/pardo.go b/sdks/go/test/regression/pardo.go index 4b8fba7f9dd61..7dc28bff2db0b 100644 --- a/sdks/go/test/regression/pardo.go +++ b/sdks/go/test/regression/pardo.go @@ -18,10 +18,22 @@ package regression import ( "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/register" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/passert" "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest" ) +func init() { + register.Function1x1(directFn) + register.Function2x0(emitFn) + register.Function3x0(emit2Fn) + register.Function2x1(mixedFn) + register.Function2x2(directCountFn) + register.Function3x1(emitCountFn) + register.Emitter1[int]() + register.Iter1[int]() +} + func directFn(elm int) int { return elm + 1 } diff --git a/sdks/go/test/run_validatesrunner_tests.sh b/sdks/go/test/run_validatesrunner_tests.sh index 444dc1ae39a71..60dd0cd97f117 100755 --- a/sdks/go/test/run_validatesrunner_tests.sh +++ b/sdks/go/test/run_validatesrunner_tests.sh @@ -257,8 +257,10 @@ print(s.getsockname()[1]) s.close() " +TMPDIR=$(mktemp -d) + # Set up environment based on runner. -if [[ "$RUNNER" == "flink" || "$RUNNER" == "spark" || "$RUNNER" == "samza" || "$RUNNER" == "portable" ]]; then +if [[ "$RUNNER" == "flink" || "$RUNNER" == "spark" || "$RUNNER" == "samza" || "$RUNNER" == "portable" || "$RUNNER" == "prism" ]]; then if [[ -z "$ENDPOINT" ]]; then JOB_PORT=$(python3 -c "$SOCKET_SCRIPT") ENDPOINT="localhost:$JOB_PORT" @@ -288,6 +290,14 @@ if [[ "$RUNNER" == "flink" || "$RUNNER" == "spark" || "$RUNNER" == "samza" || "$ python3 \ -m apache_beam.runners.portability.local_job_service_main \ --port $JOB_PORT & + elif [[ "$RUNNER" == "prism" ]]; then + PRISMBIN=$TMPDIR/prismbin + cd sdks + ./go/run_with_go_version.sh build -o $PRISMBIN go/cmd/prism/*.go + $PRISMBIN \ + --serve_http=false \ + --job_port $JOB_PORT & + cd .. else echo "Unknown runner: $RUNNER" exit 1; @@ -340,7 +350,6 @@ if [[ "$RUNNER" == "dataflow" ]]; then gcloud --version # ensure gcloud is version 186 or above - TMPDIR=$(mktemp -d) gcloud_ver=$(gcloud -v | head -1 | awk '{print $4}') if [[ "$gcloud_ver" < "186" ]] then @@ -358,13 +367,19 @@ if [[ "$RUNNER" == "dataflow" ]]; then TAG=$(date +%Y%m%d-%H%M%S) CONTAINER=us.gcr.io/$PROJECT/$USER/beam_go_sdk echo "Using container $CONTAINER" - ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG - # Verify it exists - docker images | grep $TAG + # TODO(https://github.com/apache/beam/issues/27674): remove this branch once the jenkins VM can build multiarch, or jenkins is deprecated. + if [[ "$USER" == "jenkins" ]]; then + ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG + + # Verify it exists + docker images | grep $TAG - # Push the container - gcloud docker -- push $CONTAINER:$TAG + # Push the container + gcloud docker -- push $CONTAINER:$TAG + else + ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG -Pcontainer-architecture-list=arm64,amd64 -Ppush-containers + fi if [[ -n "$TEST_EXPANSION_ADDR" || -n "$IO_EXPANSION_ADDR" || -n "$SCHEMAIO_EXPANSION_ADDR" || -n "$DEBEZIUMIO_EXPANSION_ADDR" ]]; then ARGS="$ARGS --experiments=use_portable_job_submission" @@ -396,6 +411,7 @@ fi ARGS="$ARGS -p $SIMULTANEOUS" # Assemble test arguments and pipeline options. +ARGS="$ARGS -v" ARGS="$ARGS -timeout $TIMEOUT" ARGS="$ARGS --runner=$RUNNER" ARGS="$ARGS --project=$DATAFLOW_PROJECT" @@ -431,17 +447,21 @@ cd ../.. if [[ "$RUNNER" == "dataflow" ]]; then # Delete the container locally and remotely - docker rmi $CONTAINER:$TAG || echo "Failed to remove container" - gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container" - + docker rmi $CONTAINER:$TAG || echo "Built container image was not removed. Possibly, it was not not saved locally." + # Note: we don't delete the multi-arch containers here because this command only deletes the manifest list with the tag, + # the associated container images can't be deleted because they are not tagged. However, multi-arch containers that are + # older than 6 weeks old are deleted by stale_dataflow_prebuilt_image_cleaner.sh that runs daily. + if [[ "$USER" == "jenkins" ]]; then + gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container" + fi if [[ -n "$TEST_EXPANSION_ADDR" || -n "$IO_EXPANSION_ADDR" || -n "$SCHEMAIO_EXPANSION_ADDR" || -n "$DEBEZIUMIO_EXPANSION_ADDR" ]]; then # Delete the java cross-language container locally and remotely docker rmi $JAVA_CONTAINER:$JAVA_TAG || echo "Failed to remove container" gcloud --quiet container images delete $JAVA_CONTAINER:$JAVA_TAG || echo "Failed to delete container" fi - - # Clean up tempdir - rm -rf $TMPDIR fi +# Clean up tempdir +rm -rf $TMPDIR + exit $TEST_EXIT_CODE diff --git a/sdks/java/OWNERS b/sdks/java/OWNERS deleted file mode 100644 index 5153af64bb4ec..0000000000000 --- a/sdks/java/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - aaltay - - kennknowles diff --git a/sdks/java/build-tools/OWNERS b/sdks/java/build-tools/OWNERS deleted file mode 100644 index c8a0cb2bf9ed3..0000000000000 --- a/sdks/java/build-tools/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - jasonkuster diff --git a/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml b/sdks/java/build-tools/src/main/resources/beam/checkstyle/checkstyle.xml similarity index 99% rename from sdks/java/build-tools/src/main/resources/beam/checkstyle.xml rename to sdks/java/build-tools/src/main/resources/beam/checkstyle/checkstyle.xml index df37ec4e43ebc..3c4cfdfbc6f58 100644 --- a/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml +++ b/sdks/java/build-tools/src/main/resources/beam/checkstyle/checkstyle.xml @@ -53,7 +53,7 @@ page at http://checkstyle.sourceforge.net/config.html --> - + @@ -114,7 +114,7 @@ page at http://checkstyle.sourceforge.net/config.html --> --> - + @@ -270,7 +270,7 @@ page at http://checkstyle.sourceforge.net/config.html --> - + diff --git a/sdks/java/build-tools/src/main/resources/beam/suppressions.xml b/sdks/java/build-tools/src/main/resources/beam/checkstyle/suppressions.xml similarity index 82% rename from sdks/java/build-tools/src/main/resources/beam/suppressions.xml rename to sdks/java/build-tools/src/main/resources/beam/checkstyle/suppressions.xml index 8f0249a8abb75..7037f0543f4fa 100644 --- a/sdks/java/build-tools/src/main/resources/beam/suppressions.xml +++ b/sdks/java/build-tools/src/main/resources/beam/checkstyle/suppressions.xml @@ -20,6 +20,7 @@ + @@ -30,6 +31,7 @@ + @@ -48,12 +50,20 @@ + + + + + + + + @@ -70,6 +80,13 @@ + + + + + + + diff --git a/sdks/java/container/OWNERS b/sdks/java/container/OWNERS deleted file mode 100644 index d37599eb30cdd..0000000000000 --- a/sdks/java/container/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - herohde - - aaltay diff --git a/sdks/java/container/boot.go b/sdks/java/container/boot.go index 61d5a6a3faa63..f7fd7437c88a8 100644 --- a/sdks/java/container/boot.go +++ b/sdks/java/container/boot.go @@ -22,7 +22,6 @@ import ( "encoding/json" "flag" "fmt" - "io/ioutil" "log" "os" "path/filepath" @@ -125,7 +124,9 @@ func main() { // (3) Invoke the Java harness, preserving artifact ordering in classpath. os.Setenv("HARNESS_ID", *id) - os.Setenv("PIPELINE_OPTIONS", options) + if err := makePipelineOptionsFile(options); err != nil { + logger.Fatalf(ctx, "Failed to load pipeline options to worker: %v", err) + } os.Setenv("LOGGING_API_SERVICE_DESCRIPTOR", proto.MarshalTextString(&pipepb.ApiServiceDescriptor{Url: *loggingEndpoint})) os.Setenv("CONTROL_API_SERVICE_DESCRIPTOR", proto.MarshalTextString(&pipepb.ApiServiceDescriptor{Url: *controlEndpoint})) os.Setenv("RUNNER_CAPABILITIES", strings.Join(info.GetRunnerCapabilities(), " ")) @@ -158,8 +159,9 @@ func main() { cp = append(cp, filepath.Join(dir, filepath.FromSlash(name))) } + var setRecommendedMaxXmx = strings.Contains(options, "set_recommended_max_xmx") args := []string{ - "-Xmx" + strconv.FormatUint(heapSizeLimit(info), 10), + "-Xmx" + strconv.FormatUint(heapSizeLimit(info, setRecommendedMaxXmx), 10), // ParallelGC the most adequate for high throughput and lower CPU utilization // It is the default GC in Java 8, but not on newer versions "-XX:+UseParallelGC", @@ -245,13 +247,34 @@ func main() { logger.Fatalf(ctx, "Java exited: %v", execx.Execute("java", args...)) } +// makePipelineOptionsFile writes the pipeline options to a file. +// Assumes the options string is JSON formatted. +func makePipelineOptionsFile(options string) error { + fn := "pipeline_options.json" + f, err := os.Create(fn) + if err != nil { + return fmt.Errorf("unable to create %v: %w", fn, err) + } + defer f.Close() + if _, err := f.WriteString(options); err != nil { + return fmt.Errorf("error writing %v: %w", f.Name(), err) + } + os.Setenv("PIPELINE_OPTIONS_FILE", f.Name()) + return nil +} + // heapSizeLimit returns 80% of the runner limit, if provided. If not provided, // it returns 70% of the physical memory on the machine. If it cannot determine // that value, it returns 1GB. This is an imperfect heuristic. It aims to // ensure there is memory for non-heap use and other overhead, while also not -// underutilizing the machine. -func heapSizeLimit(info *fnpb.ProvisionInfo) uint64 { - if size, err := syscallx.PhysicalMemorySize(); err == nil { +// underutilizing the machine. if set_recommended_max_xmx experiment is enabled, +// sets xmx to 32G. Under 32G JVM enables CompressedOops. CompressedOops +// utilizes memory more efficiently, and has positive impact on GC performance +// and cache hit rate. +func heapSizeLimit(info *fnpb.ProvisionInfo, setRecommendedMaxXmx bool) uint64 { + if setRecommendedMaxXmx { + return 32 << 30 + } else if size, err := syscallx.PhysicalMemorySize(); err == nil { return (size * 70) / 100 } return 1 << 30 @@ -327,7 +350,7 @@ func LoadMetaOptions(ctx context.Context, logger *tools.Logger, dir string) ([]* return nil } - content, err := ioutil.ReadFile(path) + content, err := os.ReadFile(path) if err != nil { return err } diff --git a/sdks/java/container/build.gradle b/sdks/java/container/build.gradle index b44addb510707..4c4b6aaa31fd7 100644 --- a/sdks/java/container/build.gradle +++ b/sdks/java/container/build.gradle @@ -80,7 +80,7 @@ artifacts { } task pushAll { - dependsOn ":sdks:java:container:java8:dockerPush" - dependsOn ":sdks:java:container:java11:dockerPush" - dependsOn ":sdks:java:container:java17:dockerPush" + dependsOn ":sdks:java:container:java8:docker" + dependsOn ":sdks:java:container:java11:docker" + dependsOn ":sdks:java:container:java17:docker" } diff --git a/sdks/java/container/common.gradle b/sdks/java/container/common.gradle index 5b2130f278149..cc427494ed6e3 100644 --- a/sdks/java/container/common.gradle +++ b/sdks/java/container/common.gradle @@ -63,6 +63,8 @@ task copyDockerfileDependencies(type: Copy) { task copySdkHarnessLauncher(type: Copy) { dependsOn ":sdks:java:container:downloadCloudProfilerAgent" + // if licenses are required, they should be present before this task run. + mustRunAfter ":sdks:java:container:pullLicenses" from configurations.sdkHarnessLauncher into "build/target" @@ -104,6 +106,9 @@ task validateJavaHome { } } +def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] +def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers") + docker { name containerImageName( name: "${project.docker_image_default_repo_prefix}java${imageJavaVersion}_sdk", @@ -121,8 +126,10 @@ docker { project.rootProject.hasProperty(["isRelease"]), 'java_version': imageJavaVersion, ]) - buildx project.containerPlatforms() != [project.nativeArchitecture()] + buildx useBuildx platform(*project.containerPlatforms()) + load useBuildx && !pushContainers + push pushContainers } if (project.rootProject.hasProperty(["docker-pull-licenses"]) || diff --git a/sdks/java/container/java21/java21-security.properties b/sdks/java/container/java21/java21-security.properties new file mode 100644 index 0000000000000..390cba510187e --- /dev/null +++ b/sdks/java/container/java21/java21-security.properties @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Java 21 java.security properties file override for JVM +# base properties derived from: +# openjdk version "21-ea" 2023-09-19 +# OpenJDK Runtime Environment (build 21-ea+23-1988) +# OpenJDK 64-Bit Server VM (build 21-ea+23-1988, mixed mode, sharing) + +# Java has now disabled TLSv1 and TLSv1.1. We specifically put it in the +# legacy algorithms list to allow it to be used if something better is not +# available (e.g. TLSv1.2). This will prevent breakages for existing users +# (for example JDBC with MySQL). See +# https://bugs.java.com/bugdatabase/view_bug.do?bug_id=JDK-8202343 +# for additional details. +jdk.tls.disabledAlgorithms=SSLv3, DTLSv1.0, RC4, DES, \ + MD5withRSA, DH keySize < 1024, EC keySize < 224, 3DES_EDE_CBC, anon, NULL, \ + ECDH + +# The raw value from 21-ea for legacyAlgorithms is +# NULL, anon, RC4, DES, 3DES_EDE_CBC +# Because these values are in disabledAlgorithms, it is erroneous to include +# them in legacy (they are disabled in Java 8, 11, and 17 as well). Here we +# only include TLSv1 and TLSv1.1 which were removed from disabledAlgorithms +jdk.tls.legacyAlgorithms=TLSv1, TLSv1.1 + +# /dev/random blocks in virtualized environments due to lack of +# good entropy sources, which makes SecureRandom use impractical. +# In particular, that affects the performance of HTTPS that relies +# on SecureRandom. +# +# Due to that, /dev/urandom is used as the default. +# +# See http://www.2uo.de/myths-about-urandom/ for some background +# on security of /dev/urandom on Linux. +securerandom.source=file:/dev/./urandom \ No newline at end of file diff --git a/sdks/java/container/java21/option-java21-security.json b/sdks/java/container/java21/option-java21-security.json new file mode 100644 index 0000000000000..2844fa7e0a5be --- /dev/null +++ b/sdks/java/container/java21/option-java21-security.json @@ -0,0 +1,9 @@ +{ + "name": "java-security", + "enabled": true, + "options": { + "properties": { + "java.security.properties": "/opt/apache/beam/options/java21-security.properties" + } + } +} diff --git a/sdks/java/container/license_scripts/dep_urls_java.yaml b/sdks/java/container/license_scripts/dep_urls_java.yaml index 3496d1e567cb4..8a028f459727c 100644 --- a/sdks/java/container/license_scripts/dep_urls_java.yaml +++ b/sdks/java/container/license_scripts/dep_urls_java.yaml @@ -38,11 +38,15 @@ jcip-annotations: '1.0': license: "https://raw.githubusercontent.com/stephenc/jcip-annotations/master/LICENSE.txt" type: "Apache License 2.0" +guava-parent: + '32.1.2-jre': + license: "https://raw.githubusercontent.com/google/guava/master/LICENSE" + type: "Apache License 2.0" jaxen: '1.1.6': type: "3-Clause BSD" libraries-bom: - '26.16.0': + '26.23.0': license: "https://raw.githubusercontent.com/GoogleCloudPlatform/cloud-opensource-java/master/LICENSE" type: "Apache License 2.0" paranamer: diff --git a/sdks/java/container/license_scripts/requirement.txt b/sdks/java/container/license_scripts/requirement.txt index d978cb27ef83c..244daf5a4433c 100644 --- a/sdks/java/container/license_scripts/requirement.txt +++ b/sdks/java/container/license_scripts/requirement.txt @@ -17,5 +17,5 @@ ############################################################################### beautifulsoup4>=4.9.0,<5.0 -pyyaml>=3.12,<6.0.0 +pyyaml>=6.0.1,<7 tenacity>=6.1.0,<9.0 diff --git a/sdks/java/core/OWNERS b/sdks/java/core/OWNERS deleted file mode 100644 index af32c311923a2..0000000000000 --- a/sdks/java/core/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lukecwik diff --git a/sdks/java/core/build.gradle b/sdks/java/core/build.gradle index 8014e9fde055f..7c788eaba49dd 100644 --- a/sdks/java/core/build.gradle +++ b/sdks/java/core/build.gradle @@ -36,7 +36,6 @@ applyJavaNature( relocate "org.antlr.v4", getJavaRelocatedPath("org.antlr.v4") }, ) -applyAvroNature() applyAntlrNature() generateGrammarSource { @@ -77,7 +76,7 @@ dependencies { shadow project(path: ":model:pipeline", configuration: "shadow") shadow project(path: ":model:job-management", configuration: "shadow") shadow library.java.vendored_grpc_1_54_0 - shadow library.java.vendored_guava_26_0_jre + shadow library.java.vendored_guava_32_1_2_jre shadow library.java.byte_buddy implementation library.java.antlr_runtime implementation library.java.commons_compress @@ -89,7 +88,6 @@ dependencies { shadow library.java.jackson_annotations shadow library.java.jackson_databind shadow library.java.slf4j_api - shadow library.java.avro shadow library.java.snappy_java shadow library.java.joda_time implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) @@ -114,7 +112,6 @@ dependencies { shadowTest "com.esotericsoftware.kryo:kryo:2.21" shadowTest library.java.quickcheck_core shadowTest library.java.quickcheck_generators - shadowTest library.java.avro_tests shadowTest library.java.zstd_jni shadowTest library.java.commons_logging shadowTest library.java.log4j @@ -124,5 +121,6 @@ dependencies { } project.tasks.compileTestJava { - options.compilerArgs += ['-Xlint:-rawtypes'] // generated avro uses rawtypes without suppression + // TODO: fix other places with warnings in tests and delete this option + options.compilerArgs += ['-Xlint:-rawtypes'] } diff --git a/sdks/java/core/jmh/build.gradle b/sdks/java/core/jmh/build.gradle index 56711f169110f..67e23e704530c 100644 --- a/sdks/java/core/jmh/build.gradle +++ b/sdks/java/core/jmh/build.gradle @@ -31,7 +31,7 @@ dependencies { implementation project(path: ":sdks:java:core", configuration: "shadowTest") implementation library.java.joda_time implementation library.java.vendored_grpc_1_54_0 - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.hadoop_common runtimeOnly library.java.slf4j_jdk14 testImplementation library.java.junit diff --git a/sdks/java/core/jmh/src/main/java/org/apache/beam/sdk/jmh/schemas/RowBundle.java b/sdks/java/core/jmh/src/main/java/org/apache/beam/sdk/jmh/schemas/RowBundle.java index 4368fd709b236..432424ea9ac1c 100644 --- a/sdks/java/core/jmh/src/main/java/org/apache/beam/sdk/jmh/schemas/RowBundle.java +++ b/sdks/java/core/jmh/src/main/java/org/apache/beam/sdk/jmh/schemas/RowBundle.java @@ -30,8 +30,8 @@ import org.apache.beam.sdk.values.RowWithGetters; import org.apache.beam.sdk.values.RowWithStorage; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.DateTime; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/core/jmh/src/main/java/org/apache/beam/sdk/jmh/util/ByteStringOutputStreamBenchmark.java b/sdks/java/core/jmh/src/main/java/org/apache/beam/sdk/jmh/util/ByteStringOutputStreamBenchmark.java index 729c09ccb1871..21fb96d85191b 100644 --- a/sdks/java/core/jmh/src/main/java/org/apache/beam/sdk/jmh/util/ByteStringOutputStreamBenchmark.java +++ b/sdks/java/core/jmh/src/main/java/org/apache/beam/sdk/jmh/util/ByteStringOutputStreamBenchmark.java @@ -21,7 +21,7 @@ import org.apache.beam.sdk.util.ByteStringOutputStream; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.UnsafeByteOperations; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java index eb6ffdacc7704..bd0215e1326ea 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java @@ -17,9 +17,9 @@ */ package org.apache.beam.sdk; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.transform; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.transform; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.Collection; @@ -51,17 +51,17 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicate; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Collections2; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.SetMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicate; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Collections2; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.SetMultimap; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineRunner.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineRunner.java index 6a5eb054c9ff9..ff56f1e9b50e4 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineRunner.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineRunner.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import org.apache.beam.sdk.io.FileSystems; import org.apache.beam.sdk.options.PipelineOptions; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java deleted file mode 100644 index 8fa162ecf8e4a..0000000000000 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java +++ /dev/null @@ -1,820 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.coders; - -import com.google.errorprone.annotations.FormatMethod; -import com.google.errorprone.annotations.FormatString; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.Serializable; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; -import org.apache.avro.AvroRuntimeException; -import org.apache.avro.Conversion; -import org.apache.avro.LogicalType; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.avro.io.BinaryDecoder; -import org.apache.avro.io.BinaryEncoder; -import org.apache.avro.io.DatumReader; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.reflect.AvroEncode; -import org.apache.avro.reflect.AvroName; -import org.apache.avro.reflect.AvroSchema; -import org.apache.avro.reflect.ReflectData; -import org.apache.avro.reflect.ReflectDatumReader; -import org.apache.avro.reflect.ReflectDatumWriter; -import org.apache.avro.reflect.Union; -import org.apache.avro.specific.SpecificData; -import org.apache.avro.specific.SpecificDatumReader; -import org.apache.avro.specific.SpecificDatumWriter; -import org.apache.avro.specific.SpecificRecord; -import org.apache.avro.util.ClassUtils; -import org.apache.avro.util.Utf8; -import org.apache.beam.sdk.util.EmptyOnDeserializationThreadLocal; -import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Suppliers; -import org.checkerframework.checker.nullness.qual.Nullable; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; - -/** - * A {@link Coder} using Avro binary format. - * - *

    Each instance of {@code AvroCoder} encapsulates an Avro schema for objects of type {@code - * T}. - * - *

    The Avro schema may be provided explicitly via {@link AvroCoder#of(Class, Schema)} or omitted - * via {@link AvroCoder#of(Class)}, in which case it will be inferred using Avro's {@link - * org.apache.avro.reflect.ReflectData}. - * - *

    For complete details about schema generation and how it can be controlled please see the - * {@link org.apache.avro.reflect} package. Only concrete classes with a no-argument constructor can - * be mapped to Avro records. All inherited fields that are not static or transient are included. - * Fields are not permitted to be null unless annotated by {@link Nullable} or a {@link Union} - * schema containing {@code "null"}. - * - *

    To use, specify the {@code Coder} type on a PCollection: - * - *

    {@code
    - * PCollection records =
    - *     input.apply(...)
    - *          .setCoder(AvroCoder.of(MyCustomElement.class));
    - * }
    - * - *

    or annotate the element class using {@code @DefaultCoder}. - * - *

    {@code @DefaultCoder(AvroCoder.class)
    - * public class MyCustomElement {
    - *     ...
    - * }
    - * }
    - * - *

    The implementation attempts to determine if the Avro encoding of the given type will satisfy - * the criteria of {@link Coder#verifyDeterministic} by inspecting both the type and the Schema - * provided or generated by Avro. Only coders that are deterministic can be used in {@link - * org.apache.beam.sdk.transforms.GroupByKey} operations. - * - * @param the type of elements handled by this coder - * @deprecated Avro related classes are deprecated in module beam-sdks-java-core and - * will be eventually removed. Please, migrate to a new module - * beam-sdks-java-extensions-avro by importing - * org.apache.beam.sdk.extensions.avro.coders.AvroCoder instead of this one. - */ -@SuppressWarnings({ - "nullness" // TODO(https://github.com/apache/beam/issues/20497) -}) -@Deprecated -public class AvroCoder extends CustomCoder { - - /** - * Returns an {@code AvroCoder} instance for the provided element type. - * - * @param the element type - */ - public static AvroCoder of(TypeDescriptor type) { - return of(type, true); - } - - /** - * Returns an {@code AvroCoder} instance for the provided element type, respecting whether to use - * Avro's Reflect* or Specific* suite for encoding and decoding. - * - * @param the element type - */ - public static AvroCoder of(TypeDescriptor type, boolean useReflectApi) { - @SuppressWarnings("unchecked") - Class clazz = (Class) type.getRawType(); - return of(clazz, useReflectApi); - } - - /** - * Returns an {@code AvroCoder} instance for the provided element class. - * - * @param the element type - */ - public static AvroCoder of(Class clazz) { - return of(clazz, true); - } - - /** - * Returns an {@code AvroGenericCoder} instance for the Avro schema. The implicit type is - * GenericRecord. - */ - public static AvroGenericCoder of(Schema schema) { - return AvroGenericCoder.of(schema); - } - - /** - * Returns an {@code AvroCoder} instance for the given class, respecting whether to use Avro's - * Reflect* or Specific* suite for encoding and decoding. - * - * @param the element type - */ - public static AvroCoder of(Class type, boolean useReflectApi) { - ClassLoader cl = type.getClassLoader(); - SpecificData data = useReflectApi ? new ReflectData(cl) : new SpecificData(cl); - return of(type, data.getSchema(type), useReflectApi); - } - - /** - * Returns an {@code AvroCoder} instance for the provided element type using the provided Avro - * schema. - * - *

    The schema must correspond to the type provided. - * - * @param the element type - */ - public static AvroCoder of(Class type, Schema schema) { - return of(type, schema, true); - } - - /** - * Returns an {@code AvroCoder} instance for the given class and schema, respecting whether to use - * Avro's Reflect* or Specific* suite for encoding and decoding. - * - * @param the element type - */ - public static AvroCoder of(Class type, Schema schema, boolean useReflectApi) { - return new AvroCoder<>(type, schema, useReflectApi); - } - - /** - * Returns a {@link CoderProvider} which uses the {@link AvroCoder} if possible for all types. - * - *

    It is unsafe to register this as a {@link CoderProvider} because Avro will reflectively - * accept dangerous types such as {@link Object}. - * - *

    This method is invoked reflectively from {@link DefaultCoder}. - */ - @SuppressWarnings("unused") - public static CoderProvider getCoderProvider() { - return new AvroCoderProvider(); - } - - /** - * A {@link CoderProvider} that constructs an {@link AvroCoder} for Avro compatible classes. - * - *

    It is unsafe to register this as a {@link CoderProvider} because Avro will reflectively - * accept dangerous types such as {@link Object}. - */ - static class AvroCoderProvider extends CoderProvider { - @Override - public Coder coderFor( - TypeDescriptor typeDescriptor, List> componentCoders) - throws CannotProvideCoderException { - try { - return AvroCoder.of(typeDescriptor); - } catch (AvroRuntimeException e) { - throw new CannotProvideCoderException( - String.format("%s is not compatible with Avro", typeDescriptor), e); - } - } - } - - private final Class type; - private final boolean useReflectApi; - private final SerializableSchemaSupplier schemaSupplier; - private final TypeDescriptor typeDescriptor; - - private final List nonDeterministicReasons; - - // Factories allocated by .get() are thread-safe and immutable. - private static final EncoderFactory ENCODER_FACTORY = EncoderFactory.get(); - private static final DecoderFactory DECODER_FACTORY = DecoderFactory.get(); - - /** - * A {@link Serializable} object that holds the {@link String} version of a {@link Schema}. This - * is paired with the {@link SerializableSchemaSupplier} via {@link Serializable}'s usage of the - * {@link #readResolve} method. - */ - private static class SerializableSchemaString implements Serializable { - private final String schema; - - private SerializableSchemaString(String schema) { - this.schema = schema; - } - - private Object readResolve() throws IOException, ClassNotFoundException { - return new SerializableSchemaSupplier(new Schema.Parser().parse(schema)); - } - } - - /** - * A {@link Serializable} object that delegates to the {@link SerializableSchemaString} via {@link - * Serializable}'s usage of the {@link #writeReplace} method. Kryo doesn't utilize Java's - * serialization and hence is able to encode the {@link Schema} object directly. - */ - private static class SerializableSchemaSupplier implements Serializable, Supplier { - // writeReplace makes this object serializable. This is a limitation of FindBugs as discussed - // here: - // http://stackoverflow.com/questions/26156523/is-writeobject-not-neccesary-using-the-serialization-proxy-pattern - @SuppressFBWarnings("SE_BAD_FIELD") - private final Schema schema; - - private SerializableSchemaSupplier(Schema schema) { - this.schema = schema; - } - - private Object writeReplace() { - return new SerializableSchemaString(schema.toString()); - } - - @Override - public Schema get() { - return schema; - } - } - - /** - * A {@link Serializable} object that lazily supplies a {@link ReflectData} built from the - * appropriate {@link ClassLoader} for the type encoded by this {@link AvroCoder}. - */ - private static class SerializableReflectDataSupplier - implements Serializable, Supplier { - - private final Class clazz; - - private SerializableReflectDataSupplier(Class clazz) { - this.clazz = clazz; - } - - @Override - public ReflectData get() { - ReflectData reflectData = new ReflectData(clazz.getClassLoader()); - reflectData.addLogicalTypeConversion(new JodaTimestampConversion()); - return reflectData; - } - } - - // Cache the old encoder/decoder and let the factories reuse them when possible. To be threadsafe, - // these are ThreadLocal. This code does not need to be re-entrant as AvroCoder does not use - // an inner coder. - private final EmptyOnDeserializationThreadLocal decoder; - private final EmptyOnDeserializationThreadLocal encoder; - private final EmptyOnDeserializationThreadLocal> writer; - private final EmptyOnDeserializationThreadLocal> reader; - - // Lazily re-instantiated after deserialization - private final Supplier reflectData; - - protected AvroCoder(Class type, Schema schema) { - this(type, schema, false); - } - - protected AvroCoder(Class type, Schema schema, boolean useReflectApi) { - this.type = type; - this.useReflectApi = useReflectApi; - this.schemaSupplier = new SerializableSchemaSupplier(schema); - typeDescriptor = TypeDescriptor.of(type); - nonDeterministicReasons = new AvroDeterminismChecker().check(TypeDescriptor.of(type), schema); - - // Decoder and Encoder start off null for each thread. They are allocated and potentially - // reused inside encode/decode. - this.decoder = new EmptyOnDeserializationThreadLocal<>(); - this.encoder = new EmptyOnDeserializationThreadLocal<>(); - - this.reflectData = Suppliers.memoize(new SerializableReflectDataSupplier(getType())); - - // Reader and writer are allocated once per thread per Coder - this.reader = - new EmptyOnDeserializationThreadLocal>() { - private final AvroCoder myCoder = AvroCoder.this; - - @Override - public DatumReader initialValue() { - if (myCoder.getType().equals(GenericRecord.class)) { - return new GenericDatumReader<>(myCoder.getSchema()); - } else if (SpecificRecord.class.isAssignableFrom(myCoder.getType()) && !useReflectApi) { - return new SpecificDatumReader<>(myCoder.getType()); - } - return new ReflectDatumReader<>( - myCoder.getSchema(), myCoder.getSchema(), myCoder.reflectData.get()); - } - }; - - this.writer = - new EmptyOnDeserializationThreadLocal>() { - private final AvroCoder myCoder = AvroCoder.this; - - @Override - public DatumWriter initialValue() { - if (myCoder.getType().equals(GenericRecord.class)) { - return new GenericDatumWriter<>(myCoder.getSchema()); - } else if (SpecificRecord.class.isAssignableFrom(myCoder.getType()) && !useReflectApi) { - return new SpecificDatumWriter<>(myCoder.getType()); - } - return new ReflectDatumWriter<>(myCoder.getSchema(), myCoder.reflectData.get()); - } - }; - } - - /** Returns the type this coder encodes/decodes. */ - public Class getType() { - return type; - } - - public boolean useReflectApi() { - return useReflectApi; - } - - @Override - public void encode(T value, OutputStream outStream) throws IOException { - // Get a BinaryEncoder instance from the ThreadLocal cache and attempt to reuse it. - BinaryEncoder encoderInstance = ENCODER_FACTORY.directBinaryEncoder(outStream, encoder.get()); - // Save the potentially-new instance for reuse later. - encoder.set(encoderInstance); - writer.get().write(value, encoderInstance); - // Direct binary encoder does not buffer any data and need not be flushed. - } - - @Override - public T decode(InputStream inStream) throws IOException { - // Get a BinaryDecoder instance from the ThreadLocal cache and attempt to reuse it. - BinaryDecoder decoderInstance = DECODER_FACTORY.directBinaryDecoder(inStream, decoder.get()); - // Save the potentially-new instance for later. - decoder.set(decoderInstance); - return reader.get().read(null, decoderInstance); - } - - /** - * @throws NonDeterministicException when the type may not be deterministically encoded using the - * given {@link Schema}, the {@code directBinaryEncoder}, and the {@link ReflectDatumWriter} - * or {@link GenericDatumWriter}. - */ - @Override - public void verifyDeterministic() throws NonDeterministicException { - if (!nonDeterministicReasons.isEmpty()) { - throw new NonDeterministicException(this, nonDeterministicReasons); - } - } - - /** Returns the schema used by this coder. */ - public Schema getSchema() { - return schemaSupplier.get(); - } - - @Override - public TypeDescriptor getEncodedTypeDescriptor() { - return typeDescriptor; - } - - /** - * Helper class encapsulating the various pieces of state maintained by the recursive walk used - * for checking if the encoding will be deterministic. - */ - private static class AvroDeterminismChecker { - - // Reasons that the original type are not deterministic. This accumulates - // the actual output. - private List reasons = new ArrayList<>(); - - // Types that are currently "open". Used to make sure we don't have any - // recursive types. Note that we assume that all occurrences of a given type - // are equal, rather than tracking pairs of type + schema. - private Set> activeTypes = new HashSet<>(); - - // Similarly to how we record active types, we record the schemas we visit - // to make sure we don't encounter recursive fields. - private Set activeSchemas = new HashSet<>(); - - /** Report an error in the current context. */ - @FormatMethod - private void reportError(String context, @FormatString String fmt, Object... args) { - String message = String.format(fmt, args); - reasons.add(context + ": " + message); - } - - /** - * Classes that are serialized by Avro as a String include - * - *

      - *
    • Subtypes of CharSequence (including String, Avro's mutable Utf8, etc.) - *
    • Several predefined classes (BigDecimal, BigInteger, URI, URL) - *
    • Classes annotated with @Stringable (uses their #toString() and a String constructor) - *
    - * - *

    Rather than determine which of these cases are deterministic, we list some classes that - * definitely are, and treat any others as non-deterministic. - */ - private static final Set> DETERMINISTIC_STRINGABLE_CLASSES = new HashSet<>(); - - static { - // CharSequences: - DETERMINISTIC_STRINGABLE_CLASSES.add(String.class); - DETERMINISTIC_STRINGABLE_CLASSES.add(Utf8.class); - - // Explicitly Stringable: - DETERMINISTIC_STRINGABLE_CLASSES.add(java.math.BigDecimal.class); - DETERMINISTIC_STRINGABLE_CLASSES.add(java.math.BigInteger.class); - DETERMINISTIC_STRINGABLE_CLASSES.add(java.net.URI.class); - DETERMINISTIC_STRINGABLE_CLASSES.add(java.net.URL.class); - - // Classes annotated with @Stringable: - } - - /** Return true if the given type token is a subtype of *any* of the listed parents. */ - private static boolean isSubtypeOf(TypeDescriptor type, Class... parents) { - for (Class parent : parents) { - if (type.isSubtypeOf(TypeDescriptor.of(parent))) { - return true; - } - } - return false; - } - - protected AvroDeterminismChecker() {} - - // The entry point for the check. Should not be recursively called. - public List check(TypeDescriptor type, Schema schema) { - recurse(type.getRawType().getName(), type, schema); - return reasons; - } - - // This is the method that should be recursively called. It sets up the path - // and visited types correctly. - private void recurse(String context, TypeDescriptor type, Schema schema) { - if (type.getRawType().isAnnotationPresent(AvroSchema.class)) { - reportError(context, "Custom schemas are not supported -- remove @AvroSchema."); - return; - } - - if (!activeTypes.add(type)) { - reportError(context, "%s appears recursively", type); - return; - } - - // If the record isn't a true class, but rather a GenericRecord, SpecificRecord, etc. - // with a specified schema, then we need to make the decision based on the generated - // implementations. - if (isSubtypeOf(type, IndexedRecord.class)) { - checkIndexedRecord(context, schema, null); - } else { - doCheck(context, type, schema); - } - - activeTypes.remove(type); - } - - private void doCheck(String context, TypeDescriptor type, Schema schema) { - switch (schema.getType()) { - case ARRAY: - checkArray(context, type, schema); - break; - case ENUM: - // Enums should be deterministic, since they depend only on the ordinal. - break; - case FIXED: - // Depending on the implementation of GenericFixed, we don't know how - // the given field will be encoded. So, we assume that it isn't - // deterministic. - reportError(context, "FIXED encodings are not guaranteed to be deterministic"); - break; - case MAP: - checkMap(context, type, schema); - break; - case RECORD: - if (!(type.getType() instanceof Class)) { - reportError(context, "Cannot determine type from generic %s due to erasure", type); - return; - } - checkRecord(type, schema); - break; - case UNION: - checkUnion(context, type, schema); - break; - case STRING: - checkString(context, type); - break; - case BOOLEAN: - case BYTES: - case DOUBLE: - case INT: - case FLOAT: - case LONG: - case NULL: - // For types that Avro encodes using one of the above primitives, we assume they are - // deterministic. - break; - default: - // In any other case (eg., new types added to Avro) we cautiously return - // false. - reportError(context, "Unknown schema type %s may be non-deterministic", schema.getType()); - break; - } - } - - private void checkString(String context, TypeDescriptor type) { - // For types that are encoded as strings, we need to make sure they're in an approved - // list. For other types that are annotated @Stringable, Avro will just use the - // #toString() methods, which has no guarantees of determinism. - if (!DETERMINISTIC_STRINGABLE_CLASSES.contains(type.getRawType())) { - reportError(context, "%s may not have deterministic #toString()", type); - } - } - - private static final Schema AVRO_NULL_SCHEMA = Schema.create(Schema.Type.NULL); - - private void checkUnion(String context, TypeDescriptor type, Schema schema) { - final List unionTypes = schema.getTypes(); - - if (!type.getRawType().isAnnotationPresent(Union.class)) { - // First check for @Nullable field, which shows up as a union of field type and null. - if (unionTypes.size() == 2 && unionTypes.contains(AVRO_NULL_SCHEMA)) { - // Find the Schema that is not NULL and recursively check that it is deterministic. - Schema nullableFieldSchema = - unionTypes.get(0).equals(AVRO_NULL_SCHEMA) ? unionTypes.get(1) : unionTypes.get(0); - doCheck(context, type, nullableFieldSchema); - return; - } - - // Otherwise report a schema error. - reportError(context, "Expected type %s to have @Union annotation", type); - return; - } - - // Errors associated with this union will use the base class as their context. - String baseClassContext = type.getRawType().getName(); - - // For a union, we need to make sure that each possible instantiation is deterministic. - for (Schema concrete : unionTypes) { - @SuppressWarnings("unchecked") - TypeDescriptor unionType = TypeDescriptor.of(ReflectData.get().getClass(concrete)); - - recurse(baseClassContext, unionType, concrete); - } - } - - private void checkRecord(TypeDescriptor type, Schema schema) { - // For a record, we want to make sure that all the fields are deterministic. - Class clazz = type.getRawType(); - for (Schema.Field fieldSchema : schema.getFields()) { - Field field = getField(clazz, fieldSchema.name()); - String fieldContext = field.getDeclaringClass().getName() + "#" + field.getName(); - - if (field.isAnnotationPresent(AvroEncode.class)) { - reportError( - fieldContext, "Custom encoders may be non-deterministic -- remove @AvroEncode"); - continue; - } - - if (!IndexedRecord.class.isAssignableFrom(field.getType()) - && field.isAnnotationPresent(AvroSchema.class)) { - // TODO: We should be able to support custom schemas on POJO fields, but we shouldn't - // need to, so we just allow it in the case of IndexedRecords. - reportError( - fieldContext, "Custom schemas are only supported for subtypes of IndexedRecord."); - continue; - } - - TypeDescriptor fieldType = type.resolveType(field.getGenericType()); - recurse(fieldContext, fieldType, fieldSchema.schema()); - } - } - - private void checkIndexedRecord( - String context, Schema schema, @Nullable String specificClassStr) { - - if (!activeSchemas.add(schema)) { - reportError(context, "%s appears recursively", schema.getName()); - return; - } - - switch (schema.getType()) { - case ARRAY: - // Generic Records use GenericData.Array to implement arrays, which is - // essentially an ArrayList, and therefore ordering is deterministic. - // The array is thus deterministic if the elements are deterministic. - checkIndexedRecord(context, schema.getElementType(), null); - break; - case ENUM: - // Enums are deterministic because they encode as a single integer. - break; - case FIXED: - // In the case of GenericRecords, FIXED is deterministic because it - // encodes/decodes as a Byte[]. - break; - case MAP: - reportError( - context, - "GenericRecord and SpecificRecords use a HashMap to represent MAPs," - + " so it is non-deterministic"); - break; - case RECORD: - for (Schema.Field field : schema.getFields()) { - checkIndexedRecord( - schema.getName() + "." + field.name(), - field.schema(), - field.getProp(SpecificData.CLASS_PROP)); - } - break; - case STRING: - // GenericDatumWriter#findStringClass will use a CharSequence or a String - // for each string, so it is deterministic. - - // SpecificCompiler#getStringType will use java.lang.String, org.apache.avro.util.Utf8, - // or java.lang.CharSequence, unless SpecificData.CLASS_PROP overrides that. - if (specificClassStr != null) { - Class specificClass; - try { - specificClass = ClassUtils.forName(specificClassStr); - if (!DETERMINISTIC_STRINGABLE_CLASSES.contains(specificClass)) { - reportError( - context, - "Specific class %s is not known to be deterministic", - specificClassStr); - } - } catch (ClassNotFoundException e) { - reportError( - context, "Specific class %s is not known to be deterministic", specificClassStr); - } - } - break; - case UNION: - for (Schema subschema : schema.getTypes()) { - checkIndexedRecord(subschema.getName(), subschema, null); - } - break; - case BOOLEAN: - case BYTES: - case DOUBLE: - case INT: - case FLOAT: - case LONG: - case NULL: - // For types that Avro encodes using one of the above primitives, we assume they are - // deterministic. - break; - default: - reportError(context, "Unknown schema type %s may be non-deterministic", schema.getType()); - break; - } - - activeSchemas.remove(schema); - } - - private void checkMap(String context, TypeDescriptor type, Schema schema) { - if (!isSubtypeOf(type, SortedMap.class)) { - reportError(context, "%s may not be deterministically ordered", type); - } - - // Avro (currently) asserts that all keys are strings. - // In case that changes, we double check that the key was a string: - Class keyType = type.resolveType(Map.class.getTypeParameters()[0]).getRawType(); - if (!String.class.equals(keyType)) { - reportError(context, "map keys should be Strings, but was %s", keyType); - } - - recurse(context, type.resolveType(Map.class.getTypeParameters()[1]), schema.getValueType()); - } - - private void checkArray(String context, TypeDescriptor type, Schema schema) { - TypeDescriptor elementType = null; - if (type.isArray()) { - // The type is an array (with ordering)-> deterministic iff the element is deterministic. - elementType = type.getComponentType(); - } else if (isSubtypeOf(type, Collection.class)) { - if (isSubtypeOf(type, List.class, SortedSet.class)) { - // Ordered collection -> deterministic iff the element is deterministic - elementType = type.resolveType(Collection.class.getTypeParameters()[0]); - } else { - // Not an ordered collection -> not deterministic - reportError(context, "%s may not be deterministically ordered", type); - return; - } - } else { - // If it was an unknown type encoded as an array, be conservative and assume - // that we don't know anything about the order. - reportError(context, "encoding %s as an ARRAY was unexpected", type); - return; - } - - // If we get here, it's either a deterministically-ordered Collection, or - // an array. Either way, the type is deterministic iff the element type is - // deterministic. - recurse(context, elementType, schema.getElementType()); - } - - /** - * Extract a field from a class. We need to look at the declared fields so that we can see - * private fields. We may need to walk up to the parent to get classes from the parent. - */ - private static Field getField(Class originalClazz, String name) { - Class clazz = originalClazz; - while (clazz != null) { - for (Field field : clazz.getDeclaredFields()) { - AvroName avroName = field.getAnnotation(AvroName.class); - if (avroName != null && name.equals(avroName.value())) { - return field; - } else if (avroName == null && name.equals(field.getName())) { - return field; - } - } - clazz = clazz.getSuperclass(); - } - - throw new IllegalArgumentException("Unable to get field " + name + " from " + originalClazz); - } - } - - @Override - public boolean equals(@Nullable Object other) { - if (other == this) { - return true; - } - if (!(other instanceof AvroCoder)) { - return false; - } - AvroCoder that = (AvroCoder) other; - return Objects.equals(this.schemaSupplier.get(), that.schemaSupplier.get()) - && Objects.equals(this.typeDescriptor, that.typeDescriptor) - && this.useReflectApi == that.useReflectApi; - } - - @Override - public int hashCode() { - return Objects.hash(schemaSupplier.get(), typeDescriptor, useReflectApi); - } - - /** - * Conversion for DateTime. - * - *

    This is a copy from Avro 1.8's TimestampConversion, which is renamed in Avro 1.9. Defining - * own copy gives flexibility for Beam Java SDK to work with Avro 1.8 and 1.9 at runtime. - * - * @see BEAM-9144: Beam's own Avro - * TimeConversion class in beam-sdk-java-core - */ - public static class JodaTimestampConversion extends Conversion { - @Override - public Class getConvertedType() { - return DateTime.class; - } - - @Override - public String getLogicalTypeName() { - return "timestamp-millis"; - } - - @Override - public DateTime fromLong(Long millisFromEpoch, Schema schema, LogicalType type) { - return new DateTime(millisFromEpoch, DateTimeZone.UTC); - } - - @Override - public Long toLong(DateTime timestamp, Schema schema, LogicalType type) { - return timestamp.getMillis(); - } - } -} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroGenericCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroGenericCoder.java deleted file mode 100644 index 7d90206ce4c5a..0000000000000 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroGenericCoder.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.coders; - -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; - -/** - * AvroCoder specialisation for GenericRecord. - * - * @deprecated Avro related classes are deprecated in module beam-sdks-java-core and - * will be eventually removed. Please, migrate to a new module - * beam-sdks-java-extensions-avro by importing - * org.apache.beam.sdk.extensions.avro.coders.AvroGenericCoder instead of this one. - */ -@Deprecated -public class AvroGenericCoder extends AvroCoder { - AvroGenericCoder(Schema schema) { - super(GenericRecord.class, schema); - } - - public static AvroGenericCoder of(Schema schema) { - return new AvroGenericCoder(schema); - } -} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java index bc9ce55da1a77..6cfd6a435dd7e 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigDecimalCoder.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.coders; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.io.InputStream; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigIntegerCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigIntegerCoder.java index 0c497e19029e6..953038d68eaa6 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigIntegerCoder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BigIntegerCoder.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.coders; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.io.InputStream; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BitConverters.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BitConverters.java index a5352ebd1336b..e5eae8ab51aa0 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BitConverters.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BitConverters.java @@ -21,8 +21,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Longs; class BitConverters { private BitConverters() {} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java index fd555bcf03d68..b208bd30c07c9 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ByteArrayCoder.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.util.StreamUtils; import org.apache.beam.sdk.util.VarInt; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; /** * A {@link Coder} for {@code byte[]}. diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java index 3461cb5552475..04bf8af4d1878 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/Coder.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.coders; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -29,11 +29,11 @@ import org.apache.beam.sdk.PipelineRunner; import org.apache.beam.sdk.util.common.ElementByteSizeObserver; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CountingOutputStream; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CountingOutputStream; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderProviders.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderProviders.java index 319bc6aba6d09..8e47f4f2bc9cf 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderProviders.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderProviders.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.coders; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -26,7 +26,7 @@ import java.util.List; import org.apache.beam.sdk.util.Preconditions; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; /** Static utility methods for creating and working with {@link CoderProvider}s. */ public final class CoderProviders { diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java index 09c395d255138..df64789ac3d27 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.coders; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; @@ -51,15 +51,15 @@ import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSetMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.SetMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSetMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.SetMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; import org.slf4j.Logger; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DefaultCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DefaultCoder.java index f830726265547..52718fcde2afe 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DefaultCoder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DefaultCoder.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.util.Preconditions; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DelegateCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DelegateCoder.java index 02d2fc5c064dc..03550cc4e06cc 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DelegateCoder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/DelegateCoder.java @@ -22,8 +22,8 @@ import java.io.OutputStream; import java.io.Serializable; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java index 13c892bb0548d..a7acae46103d0 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/IterableLikeCoder.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.coders; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.io.InputStream; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/LengthPrefixCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/LengthPrefixCoder.java index a86524d257239..84711e70f1ccc 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/LengthPrefixCoder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/LengthPrefixCoder.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.coders; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -25,8 +25,8 @@ import java.io.OutputStream; import java.util.List; import org.apache.beam.sdk.util.VarInt; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; /** * A {@link Coder} which is able to take any existing coder and wrap it such that it is only invoked diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java index 794de3647d4cc..8e75c1f8fc4c2 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/MapCoder.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.util.common.ElementByteSizeObserver; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeParameter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; /** * A {@link Coder} for {@link Map Maps} that encodes them according to provided coders for keys and diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java index 48efb66663581..0fab0d2acece6 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/NullableCoder.java @@ -23,8 +23,8 @@ import java.util.List; import org.apache.beam.sdk.util.common.ElementByteSizeObserver; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/RowCoderGenerator.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/RowCoderGenerator.java index 1a16516a413ed..67aad1baaa0bf 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/RowCoderGenerator.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/RowCoderGenerator.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.coders; import static org.apache.beam.sdk.util.ByteBuddyUtils.getClassLoadingStrategy; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.io.InputStream; @@ -55,8 +55,8 @@ import org.apache.beam.sdk.schemas.SchemaCoder; import org.apache.beam.sdk.util.common.ReflectHelpers; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; /** * A utility for automatically generating a {@link Coder} for {@link Row} objects corresponding to a diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java index 7059e0c57f848..08c223e3de8a7 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SerializableCoder.java @@ -31,7 +31,7 @@ import java.util.WeakHashMap; import org.apache.beam.sdk.PipelineRunner; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ShardedKeyCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ShardedKeyCoder.java index 2487ed297e19e..fef0b03dbc927 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ShardedKeyCoder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ShardedKeyCoder.java @@ -23,7 +23,7 @@ import java.util.Arrays; import java.util.List; import org.apache.beam.sdk.values.ShardedKey; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** A {@link Coder} for {@link ShardedKey}, using a wrapped key {@link Coder}. */ @VisibleForTesting diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SnappyCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SnappyCoder.java index cf410ac12d7d5..d14f80ba38134 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SnappyCoder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SnappyCoder.java @@ -22,7 +22,7 @@ import java.io.OutputStream; import java.util.List; import org.apache.beam.sdk.util.CoderUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.xerial.snappy.Snappy; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SortedMapCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SortedMapCoder.java index 1520abd1f40c9..eea22064cd601 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SortedMapCoder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/SortedMapCoder.java @@ -32,7 +32,7 @@ import org.apache.beam.sdk.util.common.ElementByteSizeObserver; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeParameter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; /** * A {@link Coder} for {@link SortedMap Maps} that encodes them according to provided coders for diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringUtf8Coder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringUtf8Coder.java index feb11c5cbe6c4..5b33a618e9efd 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringUtf8Coder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StringUtf8Coder.java @@ -27,8 +27,8 @@ import org.apache.beam.sdk.util.StreamUtils; import org.apache.beam.sdk.util.VarInt; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Utf8; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Utf8; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; /** * A {@link Coder} that encodes {@link String Strings} in UTF-8 encoding. If in a nested context, diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuralByteArray.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuralByteArray.java index 6b4c542c272b3..67b1c01b92d1a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuralByteArray.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StructuralByteArray.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.coders; import java.util.Arrays; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/TimestampPrefixingWindowCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/TimestampPrefixingWindowCoder.java index ea7d56eb0c8bb..88d1499d9873b 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/TimestampPrefixingWindowCoder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/TimestampPrefixingWindowCoder.java @@ -23,7 +23,7 @@ import java.util.List; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.common.ElementByteSizeObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; /** * A {@link TimestampPrefixingWindowCoder} wraps arbitrary user custom window coder. While encoding diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ZstdCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ZstdCoder.java index a5604d927c9d8..2abd904979436 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ZstdCoder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ZstdCoder.java @@ -28,9 +28,9 @@ import java.util.Objects; import javax.annotation.Nullable; import org.apache.beam.sdk.util.CoderUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; /** * Wraps an existing coder with Zstandard compression. It makes sense to use this coder when it's diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/expansion/ExternalTransformRegistrar.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/expansion/ExternalTransformRegistrar.java index 898e17dbd6051..657026322b2ae 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/expansion/ExternalTransformRegistrar.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/expansion/ExternalTransformRegistrar.java @@ -21,8 +21,8 @@ import java.util.Map; import java.util.Map.Entry; import org.apache.beam.sdk.transforms.ExternalTransformBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * A registrar which contains a mapping from URNs to available {@link ExternalTransformBuilder}s. diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java deleted file mode 100644 index 5660921fa4022..0000000000000 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java +++ /dev/null @@ -1,2031 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io; - -import static org.apache.beam.sdk.io.FileIO.ReadMatches.DirectoryTreatment; -import static org.apache.beam.sdk.io.ReadAllViaFileBasedSource.ReadFileRangesFnExceptionHandler; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; - -import com.google.auto.value.AutoValue; -import java.io.IOException; -import java.io.Serializable; -import java.nio.channels.Channels; -import java.nio.channels.WritableByteChannel; -import java.util.Map; -import org.apache.avro.Schema; -import org.apache.avro.file.CodecFactory; -import org.apache.avro.file.DataFileConstants; -import org.apache.avro.file.DataFileWriter; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.avro.reflect.ReflectData; -import org.apache.avro.reflect.ReflectDatumWriter; -import org.apache.beam.sdk.coders.AvroCoder; -import org.apache.beam.sdk.coders.CannotProvideCoderException; -import org.apache.beam.sdk.coders.Coder; -import org.apache.beam.sdk.coders.CoderRegistry; -import org.apache.beam.sdk.coders.StringUtf8Coder; -import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy; -import org.apache.beam.sdk.io.FileIO.MatchConfiguration; -import org.apache.beam.sdk.io.FileIO.ReadableFile; -import org.apache.beam.sdk.io.fs.EmptyMatchTreatment; -import org.apache.beam.sdk.io.fs.ResourceId; -import org.apache.beam.sdk.options.ValueProvider; -import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider; -import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider; -import org.apache.beam.sdk.schemas.utils.AvroUtils; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.sdk.transforms.SerializableFunctions; -import org.apache.beam.sdk.transforms.Watch.Growth.TerminationCondition; -import org.apache.beam.sdk.transforms.display.DisplayData; -import org.apache.beam.sdk.values.PBegin; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PDone; -import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Suppliers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.checkerframework.checker.nullness.qual.Nullable; -import org.joda.time.Duration; - -/** - * {@link PTransform}s for reading and writing Avro files. - * - *

    Reading Avro files

    - * - *

    To read a {@link PCollection} from one or more Avro files with the same schema known at - * pipeline construction time, use {@link #read}, using {@link AvroIO.Read#from} to specify the - * filename or filepattern to read from. If the filepatterns to be read are themselves in a {@link - * PCollection} you can use {@link FileIO} to match them and {@link AvroIO#readFiles} to read them. - * If the schema is unknown at pipeline construction time, use {@link #parseGenericRecords} or - * {@link #parseFilesGenericRecords}. - * - *

    Many configuration options below apply to several or all of these transforms. - * - *

    See {@link FileSystems} for information on supported file systems and filepatterns. - * - *

    Filepattern expansion and watching

    - * - *

    By default, the filepatterns are expanded only once. {@link Read#watchForNewFiles} or the - * combination of {@link FileIO.Match#continuously(Duration, TerminationCondition)} and {@link - * AvroIO#readFiles(Class)} allow streaming of new files matching the filepattern(s). - * - *

    By default, {@link #read} prohibits filepatterns that match no files, and {@link - * AvroIO#readFiles(Class)} allows them in case the filepattern contains a glob wildcard character. - * Use {@link Read#withEmptyMatchTreatment} or {@link - * FileIO.Match#withEmptyMatchTreatment(EmptyMatchTreatment)} plus {@link AvroIO#readFiles(Class)} - * to configure this behavior. - * - *

    Reading records of a known schema

    - * - *

    To read specific records, such as Avro-generated classes, use {@link #read(Class)}. To read - * {@link GenericRecord GenericRecords}, use {@link #readGenericRecords(Schema)} which takes a - * {@link Schema} object, or {@link #readGenericRecords(String)} which takes an Avro schema in a - * JSON-encoded string form. An exception will be thrown if a record doesn't match the specified - * schema. Likewise, to read a {@link PCollection} of filepatterns, apply {@link FileIO} matching - * plus {@link #readFilesGenericRecords}. - * - *

    For example: - * - *

    {@code
    - * Pipeline p = ...;
    - *
    - * // Read Avro-generated classes from files on GCS
    - * PCollection records =
    - *     p.apply(AvroIO.read(AvroAutoGenClass.class).from("gs://my_bucket/path/to/records-*.avro"));
    - *
    - * // Read GenericRecord's of the given schema from files on GCS
    - * Schema schema = new Schema.Parser().parse(new File("schema.avsc"));
    - * PCollection records =
    - *     p.apply(AvroIO.readGenericRecords(schema)
    - *                .from("gs://my_bucket/path/to/records-*.avro"));
    - * }
    - * - *

    Reading records of an unknown schema

    - * - *

    To read records from files whose schema is unknown at pipeline construction time or differs - * between files, use {@link #parseGenericRecords} - in this case, you will need to specify a - * parsing function for converting each {@link GenericRecord} into a value of your custom type. - * Likewise, to read a {@link PCollection} of filepatterns with unknown schema, use {@link FileIO} - * matching plus {@link #parseFilesGenericRecords(SerializableFunction)}. - * - *

    For example: - * - *

    {@code
    - * Pipeline p = ...;
    - *
    - * PCollection records =
    - *     p.apply(AvroIO.parseGenericRecords(new SerializableFunction() {
    - *       public Foo apply(GenericRecord record) {
    - *         // If needed, access the schema of the record using record.getSchema()
    - *         return ...;
    - *       }
    - *     }));
    - * }
    - * - *

    Reading from a {@link PCollection} of filepatterns

    - * - *
    {@code
    - * Pipeline p = ...;
    - *
    - * PCollection filepatterns = p.apply(...);
    - * PCollection records =
    - *     filepatterns.apply(AvroIO.readAll(AvroAutoGenClass.class));
    - * PCollection records =
    - *     filepatterns
    - *         .apply(FileIO.matchAll())
    - *         .apply(FileIO.readMatches())
    - *         .apply(AvroIO.readFiles(AvroAutoGenClass.class));
    - * PCollection genericRecords =
    - *     filepatterns.apply(AvroIO.readGenericRecords(schema));
    - * PCollection records =
    - *     filepatterns
    - *         .apply(FileIO.matchAll())
    - *         .apply(FileIO.readMatches())
    - *         .apply(AvroIO.parseFilesGenericRecords(new SerializableFunction...);
    - * }
    - * - *

    Streaming new files matching a filepattern

    - * - *
    {@code
    - * Pipeline p = ...;
    - *
    - * PCollection lines = p.apply(AvroIO
    - *     .read(AvroAutoGenClass.class)
    - *     .from("gs://my_bucket/path/to/records-*.avro")
    - *     .watchForNewFiles(
    - *       // Check for new files every minute
    - *       Duration.standardMinutes(1),
    - *       // Stop watching the filepattern if no new files appear within an hour
    - *       afterTimeSinceNewOutput(Duration.standardHours(1))));
    - * }
    - * - *

    Reading a very large number of files

    - * - *

    If it is known that the filepattern will match a very large number of files (e.g. tens of - * thousands or more), use {@link Read#withHintMatchesManyFiles} for better performance and - * scalability. Note that it may decrease performance if the filepattern matches only a small number - * of files. - * - *

    Inferring Beam schemas from Avro files

    - * - *

    If you want to use SQL or schema based operations on an Avro-based PCollection, you must - * configure the read transform to infer the Beam schema and automatically setup the Beam related - * coders by doing: - * - *

    {@code
    - * PCollection records =
    - *     p.apply(AvroIO.read(...).from(...).withBeamSchemas(true));
    - * }
    - * - *

    Inferring Beam schemas from Avro PCollections

    - * - *

    If you created an Avro-based PCollection by other means e.g. reading records from Kafka or as - * the output of another PTransform, you may be interested on making your PCollection schema-aware - * so you can use the Schema-based APIs or Beam's SqlTransform. - * - *

    If you are using Avro specific records (generated classes from an Avro schema), you can - * register a schema provider for the specific Avro class to make any PCollection of these objects - * schema-aware. - * - *

    {@code
    - * pipeline.getSchemaRegistry().registerSchemaProvider(AvroAutoGenClass.class, AvroAutoGenClass.getClassSchema());
    - * }
    - * - * You can also manually set an Avro-backed Schema coder for a PCollection using {@link - * org.apache.beam.sdk.schemas.utils.AvroUtils#schemaCoder(Class, Schema)} to make it schema-aware. - * - *
    {@code
    - * PCollection records = ...
    - * AvroCoder coder = (AvroCoder) users.getCoder();
    - * records.setCoder(AvroUtils.schemaCoder(coder.getType(), coder.getSchema()));
    - * }
    - * - *

    If you are using GenericRecords you may need to set a specific Beam schema coder for each - * PCollection to match their internal Avro schema. - * - *

    {@code
    - * org.apache.avro.Schema avroSchema = ...
    - * PCollection records = ...
    - * records.setCoder(AvroUtils.schemaCoder(avroSchema));
    - * }
    - * - *

    Writing Avro files

    - * - *

    To write a {@link PCollection} to one or more Avro files, use {@link AvroIO.Write}, using - * {@code AvroIO.write().to(String)} to specify the output filename prefix. The default {@link - * DefaultFilenamePolicy} will use this prefix, in conjunction with a {@link ShardNameTemplate} (set - * via {@link Write#withShardNameTemplate(String)}) and optional filename suffix (set via {@link - * Write#withSuffix(String)}, to generate output filenames in a sharded way. You can override this - * default write filename policy using {@link Write#to(FileBasedSink.FilenamePolicy)} to specify a - * custom file naming policy. - * - *

    By default, {@link AvroIO.Write} produces output files that are compressed using the {@link - * org.apache.avro.file.Codec CodecFactory.snappyCodec()}. This default can be changed or overridden - * using {@link AvroIO.Write#withCodec}. - * - *

    Writing specific or generic records

    - * - *

    To write specific records, such as Avro-generated classes, use {@link #write(Class)}. To write - * {@link GenericRecord GenericRecords}, use either {@link #writeGenericRecords(Schema)} which takes - * a {@link Schema} object, or {@link #writeGenericRecords(String)} which takes a schema in a - * JSON-encoded string form. An exception will be thrown if a record doesn't match the specified - * schema. - * - *

    For example: - * - *

    {@code
    - * // A simple Write to a local file (only runs locally):
    - * PCollection records = ...;
    - * records.apply(AvroIO.write(AvroAutoGenClass.class).to("/path/to/file.avro"));
    - *
    - * // A Write to a sharded GCS file (runs locally and using remote execution):
    - * Schema schema = new Schema.Parser().parse(new File("schema.avsc"));
    - * PCollection records = ...;
    - * records.apply("WriteToAvro", AvroIO.writeGenericRecords(schema)
    - *     .to("gs://my_bucket/path/to/numbers")
    - *     .withSuffix(".avro"));
    - * }
    - * - *

    Writing windowed or unbounded data

    - * - *

    By default, all input is put into the global window before writing. If per-window writes are - * desired - for example, when using a streaming runner - {@link AvroIO.Write#withWindowedWrites()} - * will cause windowing and triggering to be preserved. When producing windowed writes with a - * streaming runner that supports triggers, the number of output shards must be set explicitly using - * {@link AvroIO.Write#withNumShards(int)}; some runners may set this for you to a runner-chosen - * value, so you may need not set it yourself. A {@link FileBasedSink.FilenamePolicy} must be set, - * and unique windows and triggers must produce unique filenames. - * - *

    Writing data to multiple destinations

    - * - *

    The following shows a more-complex example of AvroIO.Write usage, generating dynamic file - * destinations as well as a dynamic Avro schema per file. In this example, a PCollection of user - * events (e.g. actions on a website) is written out to Avro files. Each event contains the user id - * as an integer field. We want events for each user to go into a specific directory for that user, - * and each user's data should be written with a specific schema for that user; a side input is - * used, so the schema can be calculated in a different stage. - * - *

    {@code
    - * // This is the user class that controls dynamic destinations for this avro write. The input to
    - * // AvroIO.Write will be UserEvent, and we will be writing GenericRecords to the file (in order
    - * // to have dynamic schemas). Everything is per userid, so we define a dynamic destination type
    - * // of Integer.
    - * class UserDynamicAvroDestinations
    - *     extends DynamicAvroDestinations {
    - *   private final PCollectionView> userToSchemaMap;
    - *   public UserDynamicAvroDestinations( PCollectionView> userToSchemaMap) {
    - *     this.userToSchemaMap = userToSchemaMap;
    - *   }
    - *   public GenericRecord formatRecord(UserEvent record) {
    - *     return formatUserRecord(record, getSchema(record.getUserId()));
    - *   }
    - *   public Schema getSchema(Integer userId) {
    - *     return new Schema.Parser().parse(sideInput(userToSchemaMap).get(userId));
    - *   }
    - *   public Integer getDestination(UserEvent record) {
    - *     return record.getUserId();
    - *   }
    - *   public Integer getDefaultDestination() {
    - *     return 0;
    - *   }
    - *   public FilenamePolicy getFilenamePolicy(Integer userId) {
    - *     return DefaultFilenamePolicy.fromParams(new Params().withBaseFilename(baseDir + "/user-"
    - *     + userId + "/events"));
    - *   }
    - *   public List> getSideInputs() {
    - *     return ImmutableList.>of(userToSchemaMap);
    - *   }
    - * }
    - * PCollection events = ...;
    - * PCollectionView> userToSchemaMap = events.apply(
    - *     "ComputePerUserSchemas", new ComputePerUserSchemas());
    - * events.apply("WriteAvros", AvroIO.writeCustomTypeToGenericRecords()
    - *     .to(new UserDynamicAvroDestinations(userToSchemaMap)));
    - * }
    - * - * @deprecated Avro related classes are deprecated in module beam-sdks-java-core and - * will be eventually removed. Please, migrate to a new module - * beam-sdks-java-extensions-avro by importing - * org.apache.beam.sdk.extensions.avro.io.AvroIO instead of this one. - */ -@SuppressWarnings({ - "nullness" // TODO(https://github.com/apache/beam/issues/20497) -}) -@Deprecated -public class AvroIO { - /** - * Reads records of the given type from an Avro file (or multiple Avro files matching a pattern). - * - *

    The schema must be specified using one of the {@code withSchema} functions. - */ - public static Read read(Class recordClass) { - return new AutoValue_AvroIO_Read.Builder() - .setMatchConfiguration(MatchConfiguration.create(EmptyMatchTreatment.DISALLOW)) - .setRecordClass(recordClass) - .setSchema(ReflectData.get().getSchema(recordClass)) - .setInferBeamSchema(false) - .setHintMatchesManyFiles(false) - .build(); - } - - /** - * Like {@link #read}, but reads each file in a {@link PCollection} of {@link ReadableFile}, - * returned by {@link FileIO#readMatches}. - * - *

    You can read {@link GenericRecord} by using {@code #readFiles(GenericRecord.class)} or - * {@code #readFiles(new Schema.Parser().parse(schema))} if the schema is a String. - */ - public static ReadFiles readFiles(Class recordClass) { - return new AutoValue_AvroIO_ReadFiles.Builder() - .setRecordClass(recordClass) - .setSchema(ReflectData.get().getSchema(recordClass)) - .setInferBeamSchema(false) - .setDesiredBundleSizeBytes(DEFAULT_BUNDLE_SIZE_BYTES) - .setUsesReshuffle(ReadAllViaFileBasedSource.DEFAULT_USES_RESHUFFLE) - .setFileExceptionHandler(new ReadFileRangesFnExceptionHandler()) - .build(); - } - - /** - * Like {@link #read}, but reads each filepattern in the input {@link PCollection}. - * - * @deprecated You can achieve The functionality of {@link #readAll} using {@link FileIO} matching - * plus {@link #readFiles(Class)}. This is the preferred method to make composition explicit. - * {@link ReadAll} will not receive upgrades and will be removed in a future version of Beam. - */ - @Deprecated - public static ReadAll readAll(Class recordClass) { - return new AutoValue_AvroIO_ReadAll.Builder() - .setMatchConfiguration(MatchConfiguration.create(EmptyMatchTreatment.ALLOW_IF_WILDCARD)) - .setRecordClass(recordClass) - .setSchema(ReflectData.get().getSchema(recordClass)) - .setInferBeamSchema(false) - .setDesiredBundleSizeBytes(DEFAULT_BUNDLE_SIZE_BYTES) - .build(); - } - - /** Reads Avro file(s) containing records of the specified schema. */ - public static Read readGenericRecords(Schema schema) { - return new AutoValue_AvroIO_Read.Builder() - .setMatchConfiguration(MatchConfiguration.create(EmptyMatchTreatment.DISALLOW)) - .setRecordClass(GenericRecord.class) - .setSchema(schema) - .setInferBeamSchema(false) - .setHintMatchesManyFiles(false) - .build(); - } - - /** - * Like {@link #readGenericRecords(Schema)}, but for a {@link PCollection} of {@link - * ReadableFile}, for example, returned by {@link FileIO#readMatches}. - */ - public static ReadFiles readFilesGenericRecords(Schema schema) { - return new AutoValue_AvroIO_ReadFiles.Builder() - .setRecordClass(GenericRecord.class) - .setSchema(schema) - .setInferBeamSchema(false) - .setDesiredBundleSizeBytes(DEFAULT_BUNDLE_SIZE_BYTES) - .setUsesReshuffle(ReadAllViaFileBasedSource.DEFAULT_USES_RESHUFFLE) - .setFileExceptionHandler(new ReadFileRangesFnExceptionHandler()) - .build(); - } - - /** - * Like {@link #readGenericRecords(Schema)}, but for a {@link PCollection} of {@link - * ReadableFile}, for example, returned by {@link FileIO#readMatches}. - * - * @deprecated You can achieve The functionality of {@link #readAllGenericRecords(Schema)} using - * {@link FileIO} matching plus {@link #readFilesGenericRecords(Schema)}. This is the - * preferred method to make composition explicit. {@link ReadAll} will not receive upgrades - * and will be removed in a future version of Beam. - */ - @Deprecated - public static ReadAll readAllGenericRecords(Schema schema) { - return new AutoValue_AvroIO_ReadAll.Builder() - .setMatchConfiguration(MatchConfiguration.create(EmptyMatchTreatment.ALLOW_IF_WILDCARD)) - .setRecordClass(GenericRecord.class) - .setSchema(schema) - .setInferBeamSchema(false) - .setDesiredBundleSizeBytes(DEFAULT_BUNDLE_SIZE_BYTES) - .build(); - } - - /** - * Reads Avro file(s) containing records of the specified schema. The schema is specified as a - * JSON-encoded string. - */ - public static Read readGenericRecords(String schema) { - return readGenericRecords(new Schema.Parser().parse(schema)); - } - - /** Like {@link #readGenericRecords(String)}, but for {@link ReadableFile} collections. */ - public static ReadFiles readFilesGenericRecords(String schema) { - return readFilesGenericRecords(new Schema.Parser().parse(schema)); - } - - /** - * Like {@link #readGenericRecords(String)}, but reads each filepattern in the input {@link - * PCollection}. - * - * @deprecated You can achieve The functionality of {@link #readAllGenericRecords(String)} using - * {@link FileIO} matching plus {@link #readFilesGenericRecords(String)}. This is the - * preferred method to make composition explicit. {@link ReadAll} will not receive upgrades - * and will be removed in a future version of Beam. - */ - @Deprecated - public static ReadAll readAllGenericRecords(String schema) { - return readAllGenericRecords(new Schema.Parser().parse(schema)); - } - - /** - * Reads Avro file(s) containing records of an unspecified schema and converting each record to a - * custom type. - */ - public static Parse parseGenericRecords(SerializableFunction parseFn) { - return new AutoValue_AvroIO_Parse.Builder() - .setMatchConfiguration(MatchConfiguration.create(EmptyMatchTreatment.DISALLOW)) - .setParseFn(parseFn) - .setHintMatchesManyFiles(false) - .build(); - } - - /** - * Like {@link #parseGenericRecords(SerializableFunction)}, but reads each {@link ReadableFile} in - * the input {@link PCollection}. - */ - public static ParseFiles parseFilesGenericRecords( - SerializableFunction parseFn) { - return new AutoValue_AvroIO_ParseFiles.Builder() - .setParseFn(parseFn) - .setDesiredBundleSizeBytes(DEFAULT_BUNDLE_SIZE_BYTES) - .setUsesReshuffle(ReadAllViaFileBasedSource.DEFAULT_USES_RESHUFFLE) - .setFileExceptionHandler(new ReadFileRangesFnExceptionHandler()) - .build(); - } - - /** - * Like {@link #parseGenericRecords(SerializableFunction)}, but reads each filepattern in the - * input {@link PCollection}. - * - * @deprecated You can achieve The functionality of {@link - * #parseAllGenericRecords(SerializableFunction)} using {@link FileIO} matching plus {@link - * #parseFilesGenericRecords(SerializableFunction)} ()}. This is the preferred method to make - * composition explicit. {@link ParseAll} will not receive upgrades and will be removed in a - * future version of Beam. - */ - @Deprecated - public static ParseAll parseAllGenericRecords( - SerializableFunction parseFn) { - return new AutoValue_AvroIO_ParseAll.Builder() - .setMatchConfiguration(MatchConfiguration.create(EmptyMatchTreatment.ALLOW_IF_WILDCARD)) - .setParseFn(parseFn) - .setDesiredBundleSizeBytes(DEFAULT_BUNDLE_SIZE_BYTES) - .build(); - } - - /** - * Writes a {@link PCollection} to an Avro file (or multiple Avro files matching a sharding - * pattern). - */ - public static Write write(Class recordClass) { - return new Write<>( - AvroIO.defaultWriteBuilder() - .setGenericRecords(false) - .setSchema(ReflectData.get().getSchema(recordClass)) - .build()); - } - - /** Writes Avro records of the specified schema. */ - public static Write writeGenericRecords(Schema schema) { - return new Write<>( - AvroIO.defaultWriteBuilder() - .setGenericRecords(true) - .setSchema(schema) - .build()); - } - - /** - * A {@link PTransform} that writes a {@link PCollection} to an avro file (or multiple avro files - * matching a sharding pattern), with each element of the input collection encoded into its own - * record of type OutputT. - * - *

    This version allows you to apply {@link AvroIO} writes to a PCollection of a custom type - * {@link UserT}. A format mechanism that converts the input type {@link UserT} to the output type - * that will be written to the file must be specified. If using a custom {@link - * DynamicAvroDestinations} object this is done using {@link - * DynamicAvroDestinations#formatRecord}, otherwise the {@link - * AvroIO.TypedWrite#withFormatFunction} can be used to specify a format function. - * - *

    The advantage of using a custom type is that is it allows a user-provided {@link - * DynamicAvroDestinations} object, set via {@link AvroIO.Write#to(DynamicAvroDestinations)} to - * examine the custom type when choosing a destination. - * - *

    If the output type is {@link GenericRecord} use {@link #writeCustomTypeToGenericRecords()} - * instead. - */ - public static TypedWrite writeCustomType() { - return AvroIO.defaultWriteBuilder().setGenericRecords(false).build(); - } - - /** - * Similar to {@link #writeCustomType()}, but specialized for the case where the output type is - * {@link GenericRecord}. A schema must be specified either in {@link - * DynamicAvroDestinations#getSchema} or if not using dynamic destinations, by using {@link - * TypedWrite#withSchema(Schema)}. - */ - public static TypedWrite writeCustomTypeToGenericRecords() { - return AvroIO.defaultWriteBuilder().setGenericRecords(true).build(); - } - - /** - * Writes Avro records of the specified schema. The schema is specified as a JSON-encoded string. - */ - public static Write writeGenericRecords(String schema) { - return writeGenericRecords(new Schema.Parser().parse(schema)); - } - - private static TypedWrite.Builder defaultWriteBuilder() { - return new AutoValue_AvroIO_TypedWrite.Builder() - .setFilenameSuffix(null) - .setShardTemplate(null) - .setNumShards(0) - .setCodec(TypedWrite.DEFAULT_SERIALIZABLE_CODEC) - .setMetadata(ImmutableMap.of()) - .setWindowedWrites(false) - .setNoSpilling(false) - .setSyncInterval(DataFileConstants.DEFAULT_SYNC_INTERVAL); - } - - private static PCollection setBeamSchema( - PCollection pc, Class clazz, @Nullable Schema schema) { - return pc.setCoder(AvroUtils.schemaCoder(clazz, schema)); - } - - /** - * 64MB is a reasonable value that allows to amortize the cost of opening files, but is not so - * large as to exhaust a typical runner's maximum amount of output per ProcessElement call. - */ - private static final long DEFAULT_BUNDLE_SIZE_BYTES = 64 * 1024 * 1024L; - - /** Implementation of {@link #read} and {@link #readGenericRecords}. */ - @AutoValue - public abstract static class Read extends PTransform> { - - abstract @Nullable ValueProvider getFilepattern(); - - abstract MatchConfiguration getMatchConfiguration(); - - abstract @Nullable Class getRecordClass(); - - abstract @Nullable Schema getSchema(); - - abstract boolean getInferBeamSchema(); - - abstract boolean getHintMatchesManyFiles(); - - abstract Builder toBuilder(); - - @AutoValue.Builder - abstract static class Builder { - abstract Builder setFilepattern(ValueProvider filepattern); - - abstract Builder setMatchConfiguration(MatchConfiguration matchConfiguration); - - abstract Builder setRecordClass(Class recordClass); - - abstract Builder setSchema(Schema schema); - - abstract Builder setInferBeamSchema(boolean infer); - - abstract Builder setHintMatchesManyFiles(boolean hintManyFiles); - - abstract Read build(); - } - - /** - * Reads from the given filename or filepattern. - * - *

    If it is known that the filepattern will match a very large number of files (at least tens - * of thousands), use {@link #withHintMatchesManyFiles} for better performance and scalability. - */ - public Read from(ValueProvider filepattern) { - return toBuilder().setFilepattern(filepattern).build(); - } - - /** Like {@link #from(ValueProvider)}. */ - public Read from(String filepattern) { - return from(StaticValueProvider.of(filepattern)); - } - - /** Sets the {@link MatchConfiguration}. */ - public Read withMatchConfiguration(MatchConfiguration matchConfiguration) { - return toBuilder().setMatchConfiguration(matchConfiguration).build(); - } - - /** Configures whether or not a filepattern matching no files is allowed. */ - public Read withEmptyMatchTreatment(EmptyMatchTreatment treatment) { - return withMatchConfiguration(getMatchConfiguration().withEmptyMatchTreatment(treatment)); - } - - /** - * Continuously watches for new files matching the filepattern, polling it at the given - * interval, until the given termination condition is reached. The returned {@link PCollection} - * is unbounded. If {@code matchUpdatedFiles} is set, also watches for files with timestamp - * change. - * - *

    This works only in runners supporting splittable {@link - * org.apache.beam.sdk.transforms.DoFn}. - */ - public Read watchForNewFiles( - Duration pollInterval, - TerminationCondition terminationCondition, - boolean matchUpdatedFiles) { - return withMatchConfiguration( - getMatchConfiguration() - .continuously(pollInterval, terminationCondition, matchUpdatedFiles)); - } - - /** - * Same as {@link Read#watchForNewFiles(Duration, TerminationCondition, boolean)} with {@code - * matchUpdatedFiles=false}. - */ - public Read watchForNewFiles( - Duration pollInterval, TerminationCondition terminationCondition) { - return watchForNewFiles(pollInterval, terminationCondition, false); - } - - /** - * Hints that the filepattern specified in {@link #from(String)} matches a very large number of - * files. - * - *

    This hint may cause a runner to execute the transform differently, in a way that improves - * performance for this case, but it may worsen performance if the filepattern matches only a - * small number of files (e.g., in a runner that supports dynamic work rebalancing, it will - * happen less efficiently within individual files). - */ - public Read withHintMatchesManyFiles() { - return toBuilder().setHintMatchesManyFiles(true).build(); - } - - /** - * If set to true, a Beam schema will be inferred from the AVRO schema. This allows the output - * to be used by SQL and by the schema-transform library. - */ - public Read withBeamSchemas(boolean withBeamSchemas) { - return toBuilder().setInferBeamSchema(withBeamSchemas).build(); - } - - @Override - @SuppressWarnings("unchecked") - public PCollection expand(PBegin input) { - checkNotNull(getFilepattern(), "filepattern"); - checkNotNull(getSchema(), "schema"); - - if (getMatchConfiguration().getWatchInterval() == null && !getHintMatchesManyFiles()) { - PCollection read = - input.apply( - "Read", - org.apache.beam.sdk.io.Read.from( - createSource( - getFilepattern(), - getMatchConfiguration().getEmptyMatchTreatment(), - getRecordClass(), - getSchema(), - null))); - return getInferBeamSchema() ? setBeamSchema(read, getRecordClass(), getSchema()) : read; - } - - // All other cases go through FileIO + ReadFiles - ReadFiles readFiles = - (getRecordClass() == GenericRecord.class) - ? (ReadFiles) readFilesGenericRecords(getSchema()) - : readFiles(getRecordClass()); - return input - .apply("Create filepattern", Create.ofProvider(getFilepattern(), StringUtf8Coder.of())) - .apply("Match All", FileIO.matchAll().withConfiguration(getMatchConfiguration())) - .apply( - "Read Matches", - FileIO.readMatches().withDirectoryTreatment(DirectoryTreatment.PROHIBIT)) - .apply("Via ReadFiles", readFiles); - } - - @Override - public void populateDisplayData(DisplayData.Builder builder) { - super.populateDisplayData(builder); - builder - .add( - DisplayData.item("inferBeamSchema", getInferBeamSchema()) - .withLabel("Infer Beam Schema")) - .addIfNotNull(DisplayData.item("schema", String.valueOf(getSchema()))) - .addIfNotNull(DisplayData.item("recordClass", getRecordClass()).withLabel("Record Class")) - .addIfNotNull( - DisplayData.item("filePattern", getFilepattern()).withLabel("Input File Pattern")) - .include("matchConfiguration", getMatchConfiguration()); - } - - @SuppressWarnings("unchecked") - private static AvroSource createSource( - ValueProvider filepattern, - EmptyMatchTreatment emptyMatchTreatment, - Class recordClass, - Schema schema, - AvroSource.@Nullable DatumReaderFactory readerFactory) { - AvroSource source = - AvroSource.from(filepattern).withEmptyMatchTreatment(emptyMatchTreatment); - - if (readerFactory != null) { - source = source.withDatumReaderFactory(readerFactory); - } - return recordClass == GenericRecord.class - ? (AvroSource) source.withSchema(schema) - : source.withSchema(recordClass); - } - } - - ///////////////////////////////////////////////////////////////////////////// - - /** Implementation of {@link #readFiles}. */ - @AutoValue - public abstract static class ReadFiles - extends PTransform, PCollection> { - - abstract @Nullable Class getRecordClass(); - - abstract @Nullable Schema getSchema(); - - abstract boolean getUsesReshuffle(); - - abstract ReadFileRangesFnExceptionHandler getFileExceptionHandler(); - - abstract long getDesiredBundleSizeBytes(); - - abstract boolean getInferBeamSchema(); - - abstract AvroSource.@Nullable DatumReaderFactory getDatumReaderFactory(); - - abstract Builder toBuilder(); - - @AutoValue.Builder - abstract static class Builder { - abstract Builder setRecordClass(Class recordClass); - - abstract Builder setSchema(Schema schema); - - abstract Builder setUsesReshuffle(boolean usesReshuffle); - - abstract Builder setFileExceptionHandler( - ReadFileRangesFnExceptionHandler exceptionHandler); - - abstract Builder setDesiredBundleSizeBytes(long desiredBundleSizeBytes); - - abstract Builder setInferBeamSchema(boolean infer); - - abstract Builder setDatumReaderFactory(AvroSource.DatumReaderFactory factory); - - abstract ReadFiles build(); - } - - /** - * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a - * lower value (e.g. 1 MB) for streaming applications. - */ - public ReadFiles withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { - return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build(); - } - - /** Specifies if a Reshuffle should run before file reads occur. */ - public ReadFiles withUsesReshuffle(boolean usesReshuffle) { - return toBuilder().setUsesReshuffle(usesReshuffle).build(); - } - - /** Specifies if exceptions should be logged only for streaming pipelines. */ - public ReadFiles withFileExceptionHandler( - ReadFileRangesFnExceptionHandler exceptionHandler) { - return toBuilder().setFileExceptionHandler(exceptionHandler).build(); - } - - /** - * If set to true, a Beam schema will be inferred from the AVRO schema. This allows the output - * to be used by SQL and by the schema-transform library. - */ - public ReadFiles withBeamSchemas(boolean withBeamSchemas) { - return toBuilder().setInferBeamSchema(withBeamSchemas).build(); - } - - public ReadFiles withDatumReaderFactory(AvroSource.DatumReaderFactory factory) { - return toBuilder().setDatumReaderFactory(factory).build(); - } - - @Override - public PCollection expand(PCollection input) { - checkNotNull(getSchema(), "schema"); - PCollection read = - input.apply( - "Read all via FileBasedSource", - new ReadAllViaFileBasedSource<>( - getDesiredBundleSizeBytes(), - new CreateSourceFn<>( - getRecordClass(), getSchema().toString(), getDatumReaderFactory()), - AvroCoder.of(getRecordClass(), getSchema()), - getUsesReshuffle(), - getFileExceptionHandler())); - return getInferBeamSchema() ? setBeamSchema(read, getRecordClass(), getSchema()) : read; - } - - @Override - public void populateDisplayData(DisplayData.Builder builder) { - super.populateDisplayData(builder); - builder - .add( - DisplayData.item("inferBeamSchema", getInferBeamSchema()) - .withLabel("Infer Beam Schema")) - .addIfNotNull(DisplayData.item("schema", String.valueOf(getSchema()))) - .addIfNotNull( - DisplayData.item("recordClass", getRecordClass()).withLabel("Record Class")); - } - } - - ///////////////////////////////////////////////////////////////////////////// - - /** - * Implementation of {@link #readAll}. - * - * @deprecated See {@link #readAll(Class)} for details. - */ - @Deprecated - @AutoValue - public abstract static class ReadAll extends PTransform, PCollection> { - abstract MatchConfiguration getMatchConfiguration(); - - abstract @Nullable Class getRecordClass(); - - abstract @Nullable Schema getSchema(); - - abstract long getDesiredBundleSizeBytes(); - - abstract boolean getInferBeamSchema(); - - abstract Builder toBuilder(); - - @AutoValue.Builder - abstract static class Builder { - abstract Builder setMatchConfiguration(MatchConfiguration matchConfiguration); - - abstract Builder setRecordClass(Class recordClass); - - abstract Builder setSchema(Schema schema); - - abstract Builder setDesiredBundleSizeBytes(long desiredBundleSizeBytes); - - abstract Builder setInferBeamSchema(boolean infer); - - abstract ReadAll build(); - } - - /** Sets the {@link MatchConfiguration}. */ - public ReadAll withMatchConfiguration(MatchConfiguration configuration) { - return toBuilder().setMatchConfiguration(configuration).build(); - } - - /** Like {@link Read#withEmptyMatchTreatment}. */ - public ReadAll withEmptyMatchTreatment(EmptyMatchTreatment treatment) { - return withMatchConfiguration(getMatchConfiguration().withEmptyMatchTreatment(treatment)); - } - - /** Like {@link Read#watchForNewFiles}. */ - public ReadAll watchForNewFiles( - Duration pollInterval, TerminationCondition terminationCondition) { - return withMatchConfiguration( - getMatchConfiguration().continuously(pollInterval, terminationCondition)); - } - - /** - * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a - * lower value (e.g. 1 MB) for streaming applications. - */ - public ReadAll withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { - return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build(); - } - - /** - * If set to true, a Beam schema will be inferred from the AVRO schema. This allows the output - * to be used by SQL and by the schema-transform library. - */ - public ReadAll withBeamSchemas(boolean withBeamSchemas) { - return toBuilder().setInferBeamSchema(withBeamSchemas).build(); - } - - @Override - public PCollection expand(PCollection input) { - checkNotNull(getSchema(), "schema"); - PCollection read = - input - .apply(FileIO.matchAll().withConfiguration(getMatchConfiguration())) - .apply(FileIO.readMatches().withDirectoryTreatment(DirectoryTreatment.PROHIBIT)) - .apply(readFiles(getRecordClass())); - return getInferBeamSchema() ? setBeamSchema(read, getRecordClass(), getSchema()) : read; - } - - @Override - public void populateDisplayData(DisplayData.Builder builder) { - super.populateDisplayData(builder); - builder - .add( - DisplayData.item("inferBeamSchema", getInferBeamSchema()) - .withLabel("Infer Beam Schema")) - .addIfNotNull(DisplayData.item("schema", String.valueOf(getSchema()))) - .addIfNotNull(DisplayData.item("recordClass", getRecordClass()).withLabel("Record Class")) - .include("matchConfiguration", getMatchConfiguration()); - } - } - - private static class CreateSourceFn - implements SerializableFunction> { - private final Class recordClass; - private final Supplier schemaSupplier; - private final AvroSource.DatumReaderFactory readerFactory; - - CreateSourceFn( - Class recordClass, String jsonSchema, AvroSource.DatumReaderFactory readerFactory) { - this.recordClass = recordClass; - this.schemaSupplier = - Suppliers.memoize( - Suppliers.compose(new JsonToSchema(), Suppliers.ofInstance(jsonSchema))); - this.readerFactory = readerFactory; - } - - @Override - public FileBasedSource apply(String input) { - return Read.createSource( - StaticValueProvider.of(input), - EmptyMatchTreatment.DISALLOW, - recordClass, - schemaSupplier.get(), - readerFactory); - } - - private static class JsonToSchema implements Function, Serializable { - @Override - public Schema apply(String input) { - return new Schema.Parser().parse(input); - } - } - } - - ///////////////////////////////////////////////////////////////////////////// - - /** Implementation of {@link #parseGenericRecords}. */ - @AutoValue - public abstract static class Parse extends PTransform> { - - abstract @Nullable ValueProvider getFilepattern(); - - abstract MatchConfiguration getMatchConfiguration(); - - abstract SerializableFunction getParseFn(); - - abstract @Nullable Coder getCoder(); - - abstract boolean getHintMatchesManyFiles(); - - abstract Builder toBuilder(); - - @AutoValue.Builder - abstract static class Builder { - abstract Builder setFilepattern(ValueProvider filepattern); - - abstract Builder setMatchConfiguration(MatchConfiguration matchConfiguration); - - abstract Builder setParseFn(SerializableFunction parseFn); - - abstract Builder setCoder(Coder coder); - - abstract Builder setHintMatchesManyFiles(boolean hintMatchesManyFiles); - - abstract Parse build(); - } - - /** Reads from the given filename or filepattern. */ - public Parse from(String filepattern) { - return from(StaticValueProvider.of(filepattern)); - } - - /** Like {@link #from(String)}. */ - public Parse from(ValueProvider filepattern) { - return toBuilder().setFilepattern(filepattern).build(); - } - - /** Sets the {@link MatchConfiguration}. */ - public Parse withMatchConfiguration(MatchConfiguration configuration) { - return toBuilder().setMatchConfiguration(configuration).build(); - } - - /** Like {@link Read#withEmptyMatchTreatment}. */ - public Parse withEmptyMatchTreatment(EmptyMatchTreatment treatment) { - return withMatchConfiguration(getMatchConfiguration().withEmptyMatchTreatment(treatment)); - } - - /** Like {@link Read#watchForNewFiles}. */ - public Parse watchForNewFiles( - Duration pollInterval, TerminationCondition terminationCondition) { - return withMatchConfiguration( - getMatchConfiguration().continuously(pollInterval, terminationCondition)); - } - - /** Sets a coder for the result of the parse function. */ - public Parse withCoder(Coder coder) { - return toBuilder().setCoder(coder).build(); - } - - /** Like {@link Read#withHintMatchesManyFiles()}. */ - public Parse withHintMatchesManyFiles() { - return toBuilder().setHintMatchesManyFiles(true).build(); - } - - @Override - public PCollection expand(PBegin input) { - checkNotNull(getFilepattern(), "filepattern"); - Coder coder = inferCoder(getCoder(), getParseFn(), input.getPipeline().getCoderRegistry()); - - if (getMatchConfiguration().getWatchInterval() == null && !getHintMatchesManyFiles()) { - return input.apply( - org.apache.beam.sdk.io.Read.from( - AvroSource.from(getFilepattern()).withParseFn(getParseFn(), coder))); - } - - // All other cases go through FileIO + ParseFilesGenericRecords. - return input - .apply("Create filepattern", Create.ofProvider(getFilepattern(), StringUtf8Coder.of())) - .apply("Match All", FileIO.matchAll().withConfiguration(getMatchConfiguration())) - .apply( - "Read Matches", - FileIO.readMatches().withDirectoryTreatment(DirectoryTreatment.PROHIBIT)) - .apply("Via ParseFiles", parseFilesGenericRecords(getParseFn()).withCoder(coder)); - } - - private static Coder inferCoder( - @Nullable Coder explicitCoder, - SerializableFunction parseFn, - CoderRegistry coderRegistry) { - if (explicitCoder != null) { - return explicitCoder; - } - // If a coder was not specified explicitly, infer it from parse fn. - try { - return coderRegistry.getCoder(TypeDescriptors.outputOf(parseFn)); - } catch (CannotProvideCoderException e) { - throw new IllegalArgumentException( - "Unable to infer coder for output of parseFn. Specify it explicitly using withCoder().", - e); - } - } - - @Override - public void populateDisplayData(DisplayData.Builder builder) { - super.populateDisplayData(builder); - builder - .addIfNotNull( - DisplayData.item("filePattern", getFilepattern()).withLabel("Input File Pattern")) - .add(DisplayData.item("parseFn", getParseFn().getClass()).withLabel("Parse function")) - .include("matchConfiguration", getMatchConfiguration()); - } - } - - ///////////////////////////////////////////////////////////////////////////// - - /** Implementation of {@link #parseFilesGenericRecords}. */ - @AutoValue - public abstract static class ParseFiles - extends PTransform, PCollection> { - abstract SerializableFunction getParseFn(); - - abstract @Nullable Coder getCoder(); - - abstract boolean getUsesReshuffle(); - - abstract ReadFileRangesFnExceptionHandler getFileExceptionHandler(); - - abstract long getDesiredBundleSizeBytes(); - - abstract Builder toBuilder(); - - @AutoValue.Builder - abstract static class Builder { - abstract Builder setParseFn(SerializableFunction parseFn); - - abstract Builder setCoder(Coder coder); - - abstract Builder setUsesReshuffle(boolean usesReshuffle); - - abstract Builder setFileExceptionHandler( - ReadFileRangesFnExceptionHandler exceptionHandler); - - abstract Builder setDesiredBundleSizeBytes(long desiredBundleSizeBytes); - - abstract ParseFiles build(); - } - - /** Specifies the coder for the result of the {@code parseFn}. */ - public ParseFiles withCoder(Coder coder) { - return toBuilder().setCoder(coder).build(); - } - - /** Specifies if a Reshuffle should run before file reads occur. */ - public ParseFiles withUsesReshuffle(boolean usesReshuffle) { - return toBuilder().setUsesReshuffle(usesReshuffle).build(); - } - - /** Specifies if exceptions should be logged only for streaming pipelines. */ - public ParseFiles withFileExceptionHandler( - ReadFileRangesFnExceptionHandler exceptionHandler) { - return toBuilder().setFileExceptionHandler(exceptionHandler).build(); - } - - /** - * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a - * lower value (e.g. 1 MB) for streaming applications. - */ - public ParseFiles withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { - return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build(); - } - - @Override - public PCollection expand(PCollection input) { - final Coder coder = - Parse.inferCoder(getCoder(), getParseFn(), input.getPipeline().getCoderRegistry()); - final SerializableFunction parseFn = getParseFn(); - final SerializableFunction> createSource = - new CreateParseSourceFn<>(parseFn, coder); - return input.apply( - "Parse Files via FileBasedSource", - new ReadAllViaFileBasedSource<>( - getDesiredBundleSizeBytes(), - createSource, - coder, - getUsesReshuffle(), - getFileExceptionHandler())); - } - - @Override - public void populateDisplayData(DisplayData.Builder builder) { - super.populateDisplayData(builder); - builder.add(DisplayData.item("parseFn", getParseFn().getClass()).withLabel("Parse function")); - } - - private static class CreateParseSourceFn - implements SerializableFunction> { - private final SerializableFunction parseFn; - private final Coder coder; - - CreateParseSourceFn(SerializableFunction parseFn, Coder coder) { - this.parseFn = parseFn; - this.coder = coder; - } - - @Override - public FileBasedSource apply(String input) { - return AvroSource.from(input).withParseFn(parseFn, coder); - } - } - } - - ///////////////////////////////////////////////////////////////////////////// - - /** - * Implementation of {@link #parseAllGenericRecords}. - * - * @deprecated See {@link #parseAllGenericRecords(SerializableFunction)} for details. - */ - @Deprecated - @AutoValue - public abstract static class ParseAll extends PTransform, PCollection> { - abstract MatchConfiguration getMatchConfiguration(); - - abstract SerializableFunction getParseFn(); - - abstract @Nullable Coder getCoder(); - - abstract long getDesiredBundleSizeBytes(); - - abstract Builder toBuilder(); - - @AutoValue.Builder - abstract static class Builder { - abstract Builder setMatchConfiguration(MatchConfiguration matchConfiguration); - - abstract Builder setParseFn(SerializableFunction parseFn); - - abstract Builder setCoder(Coder coder); - - abstract Builder setDesiredBundleSizeBytes(long desiredBundleSizeBytes); - - abstract ParseAll build(); - } - - /** Sets the {@link MatchConfiguration}. */ - public ParseAll withMatchConfiguration(MatchConfiguration configuration) { - return toBuilder().setMatchConfiguration(configuration).build(); - } - - /** Like {@link Read#withEmptyMatchTreatment}. */ - public ParseAll withEmptyMatchTreatment(EmptyMatchTreatment treatment) { - return withMatchConfiguration(getMatchConfiguration().withEmptyMatchTreatment(treatment)); - } - - /** Like {@link Read#watchForNewFiles(Duration, TerminationCondition, boolean)}. */ - public ParseAll watchForNewFiles( - Duration pollInterval, - TerminationCondition terminationCondition, - boolean matchUpdatedFiles) { - return withMatchConfiguration( - getMatchConfiguration() - .continuously(pollInterval, terminationCondition, matchUpdatedFiles)); - } - - /** Like {@link Read#watchForNewFiles(Duration, TerminationCondition)}. */ - public ParseAll watchForNewFiles( - Duration pollInterval, TerminationCondition terminationCondition) { - return watchForNewFiles(pollInterval, terminationCondition, false); - } - - /** Specifies the coder for the result of the {@code parseFn}. */ - public ParseAll withCoder(Coder coder) { - return toBuilder().setCoder(coder).build(); - } - - /** - * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a - * lower value (e.g. 1 MB) for streaming applications. - */ - public ParseAll withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { - return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build(); - } - - @Override - public PCollection expand(PCollection input) { - return input - .apply(FileIO.matchAll().withConfiguration(getMatchConfiguration())) - .apply(FileIO.readMatches().withDirectoryTreatment(DirectoryTreatment.PROHIBIT)) - .apply( - "Parse all via FileBasedSource", - parseFilesGenericRecords(getParseFn()).withCoder(getCoder())); - } - - @Override - public void populateDisplayData(DisplayData.Builder builder) { - super.populateDisplayData(builder); - builder - .add(DisplayData.item("parseFn", getParseFn().getClass()).withLabel("Parse function")) - .include("matchConfiguration", getMatchConfiguration()); - } - } - - ///////////////////////////////////////////////////////////////////////////// - - /** Implementation of {@link #write}. */ - @AutoValue - public abstract static class TypedWrite - extends PTransform, WriteFilesResult> { - static final CodecFactory DEFAULT_CODEC = CodecFactory.snappyCodec(); - static final SerializableAvroCodecFactory DEFAULT_SERIALIZABLE_CODEC = - new SerializableAvroCodecFactory(DEFAULT_CODEC); - - abstract @Nullable SerializableFunction getFormatFunction(); - - abstract @Nullable ValueProvider getFilenamePrefix(); - - abstract @Nullable String getShardTemplate(); - - abstract @Nullable String getFilenameSuffix(); - - abstract @Nullable ValueProvider getTempDirectory(); - - abstract int getNumShards(); - - abstract boolean getGenericRecords(); - - abstract int getSyncInterval(); - - abstract @Nullable Schema getSchema(); - - abstract boolean getWindowedWrites(); - - abstract boolean getNoSpilling(); - - abstract @Nullable FilenamePolicy getFilenamePolicy(); - - abstract @Nullable DynamicAvroDestinations - getDynamicDestinations(); - - abstract AvroSink.@Nullable DatumWriterFactory getDatumWriterFactory(); - - /** - * The codec used to encode the blocks in the Avro file. String value drawn from those in - * https://avro.apache.org/docs/1.7.7/api/java/org/apache/avro/file/CodecFactory.html - */ - abstract SerializableAvroCodecFactory getCodec(); - /** Avro file metadata. */ - abstract ImmutableMap getMetadata(); - - abstract Builder toBuilder(); - - @AutoValue.Builder - abstract static class Builder { - abstract Builder setFormatFunction( - @Nullable SerializableFunction formatFunction); - - abstract Builder setFilenamePrefix( - ValueProvider filenamePrefix); - - abstract Builder setFilenameSuffix( - @Nullable String filenameSuffix); - - abstract Builder setTempDirectory( - ValueProvider tempDirectory); - - abstract Builder setNumShards(int numShards); - - abstract Builder setShardTemplate( - @Nullable String shardTemplate); - - abstract Builder setGenericRecords(boolean genericRecords); - - abstract Builder setSyncInterval(int syncInterval); - - abstract Builder setSchema(Schema schema); - - abstract Builder setWindowedWrites(boolean windowedWrites); - - abstract Builder setNoSpilling(boolean noSpilling); - - abstract Builder setFilenamePolicy( - FilenamePolicy filenamePolicy); - - abstract Builder setCodec(SerializableAvroCodecFactory codec); - - abstract Builder setMetadata( - ImmutableMap metadata); - - abstract Builder setDynamicDestinations( - DynamicAvroDestinations dynamicDestinations); - - abstract Builder setDatumWriterFactory( - AvroSink.DatumWriterFactory datumWriterFactory); - - abstract TypedWrite build(); - } - - /** - * Writes to file(s) with the given output prefix. See {@link FileSystems} for information on - * supported file systems. - * - *

    The name of the output files will be determined by the {@link FilenamePolicy} used. - * - *

    By default, a {@link DefaultFilenamePolicy} will build output filenames using the - * specified prefix, a shard name template (see {@link #withShardNameTemplate(String)}, and a - * common suffix (if supplied using {@link #withSuffix(String)}). This default can be overridden - * using {@link #to(FilenamePolicy)}. - */ - public TypedWrite to(String outputPrefix) { - return to(FileBasedSink.convertToFileResourceIfPossible(outputPrefix)); - } - - /** - * Writes to file(s) with the given output prefix. See {@link FileSystems} for information on - * supported file systems. This prefix is used by the {@link DefaultFilenamePolicy} to generate - * filenames. - * - *

    By default, a {@link DefaultFilenamePolicy} will build output filenames using the - * specified prefix, a shard name template (see {@link #withShardNameTemplate(String)}, and a - * common suffix (if supplied using {@link #withSuffix(String)}). This default can be overridden - * using {@link #to(FilenamePolicy)}. - * - *

    This default policy can be overridden using {@link #to(FilenamePolicy)}, in which case - * {@link #withShardNameTemplate(String)} and {@link #withSuffix(String)} should not be set. - * Custom filename policies do not automatically see this prefix - you should explicitly pass - * the prefix into your {@link FilenamePolicy} object if you need this. - * - *

    If {@link #withTempDirectory} has not been called, this filename prefix will be used to - * infer a directory for temporary files. - */ - public TypedWrite to(ResourceId outputPrefix) { - return toResource(StaticValueProvider.of(outputPrefix)); - } - - private static class OutputPrefixToResourceId - implements SerializableFunction { - @Override - public ResourceId apply(String input) { - return FileBasedSink.convertToFileResourceIfPossible(input); - } - } - - /** Like {@link #to(String)}. */ - public TypedWrite to(ValueProvider outputPrefix) { - return toResource( - NestedValueProvider.of( - outputPrefix, - // The function cannot be created as an anonymous class here since the enclosed class - // may contain unserializable members. - new OutputPrefixToResourceId())); - } - - /** Like {@link #to(ResourceId)}. */ - public TypedWrite toResource( - ValueProvider outputPrefix) { - return toBuilder().setFilenamePrefix(outputPrefix).build(); - } - - /** - * Writes to files named according to the given {@link FileBasedSink.FilenamePolicy}. A - * directory for temporary files must be specified using {@link #withTempDirectory}. - */ - public TypedWrite to(FilenamePolicy filenamePolicy) { - return toBuilder().setFilenamePolicy(filenamePolicy).build(); - } - - /** - * Use a {@link DynamicAvroDestinations} object to vend {@link FilenamePolicy} objects. These - * objects can examine the input record when creating a {@link FilenamePolicy}. A directory for - * temporary files must be specified using {@link #withTempDirectory}. - * - * @deprecated Use {@link FileIO#write()} or {@link FileIO#writeDynamic()} instead. - */ - @Deprecated - public TypedWrite to( - DynamicAvroDestinations dynamicDestinations) { - return toBuilder() - .setDynamicDestinations((DynamicAvroDestinations) dynamicDestinations) - .build(); - } - - /** - * Sets the approximate number of uncompressed bytes to write in each block for the AVRO - * container format. - */ - public TypedWrite withSyncInterval(int syncInterval) { - return toBuilder().setSyncInterval(syncInterval).build(); - } - - /** - * Sets the output schema. Can only be used when the output type is {@link GenericRecord} and - * when not using {@link #to(DynamicAvroDestinations)}. - */ - public TypedWrite withSchema(Schema schema) { - return toBuilder().setSchema(schema).build(); - } - - /** - * Specifies a format function to convert {@link UserT} to the output type. If {@link - * #to(DynamicAvroDestinations)} is used, {@link DynamicAvroDestinations#formatRecord} must be - * used instead. - */ - public TypedWrite withFormatFunction( - @Nullable SerializableFunction formatFunction) { - return toBuilder().setFormatFunction(formatFunction).build(); - } - - /** Set the base directory used to generate temporary files. */ - public TypedWrite withTempDirectory( - ValueProvider tempDirectory) { - return toBuilder().setTempDirectory(tempDirectory).build(); - } - - /** Set the base directory used to generate temporary files. */ - public TypedWrite withTempDirectory(ResourceId tempDirectory) { - return withTempDirectory(StaticValueProvider.of(tempDirectory)); - } - - /** - * Uses the given {@link ShardNameTemplate} for naming output files. This option may only be - * used when using one of the default filename-prefix to() overrides. - * - *

    See {@link DefaultFilenamePolicy} for how the prefix, shard name template, and suffix are - * used. - */ - public TypedWrite withShardNameTemplate(String shardTemplate) { - return toBuilder().setShardTemplate(shardTemplate).build(); - } - - /** - * Configures the filename suffix for written files. This option may only be used when using one - * of the default filename-prefix to() overrides. - * - *

    See {@link DefaultFilenamePolicy} for how the prefix, shard name template, and suffix are - * used. - */ - public TypedWrite withSuffix(String filenameSuffix) { - return toBuilder().setFilenameSuffix(filenameSuffix).build(); - } - - /** - * Configures the number of output shards produced overall (when using unwindowed writes) or - * per-window (when using windowed writes). - * - *

    For unwindowed writes, constraining the number of shards is likely to reduce the - * performance of a pipeline. Setting this value is not recommended unless you require a - * specific number of output files. - * - * @param numShards the number of shards to use, or 0 to let the system decide. - */ - public TypedWrite withNumShards(int numShards) { - checkArgument(numShards >= 0); - return toBuilder().setNumShards(numShards).build(); - } - - /** - * Forces a single file as output and empty shard name template. This option is only compatible - * with unwindowed writes. - * - *

    For unwindowed writes, constraining the number of shards is likely to reduce the - * performance of a pipeline. Setting this value is not recommended unless you require a - * specific number of output files. - * - *

    This is equivalent to {@code .withNumShards(1).withShardNameTemplate("")} - */ - public TypedWrite withoutSharding() { - return withNumShards(1).withShardNameTemplate(""); - } - - /** - * Preserves windowing of input elements and writes them to files based on the element's window. - * - *

    If using {@link #to(FileBasedSink.FilenamePolicy)}. Filenames will be generated using - * {@link FilenamePolicy#windowedFilename}. See also {@link WriteFiles#withWindowedWrites()}. - */ - public TypedWrite withWindowedWrites() { - return toBuilder().setWindowedWrites(true).build(); - } - - /** See {@link WriteFiles#withNoSpilling()}. */ - public TypedWrite withNoSpilling() { - return toBuilder().setNoSpilling(true).build(); - } - - /** Writes to Avro file(s) compressed using specified codec. */ - public TypedWrite withCodec(CodecFactory codec) { - return toBuilder().setCodec(new SerializableAvroCodecFactory(codec)).build(); - } - - /** - * Specifies a {@link AvroSink.DatumWriterFactory} to use for creating {@link - * org.apache.avro.io.DatumWriter} instances. - */ - public TypedWrite withDatumWriterFactory( - AvroSink.DatumWriterFactory datumWriterFactory) { - return toBuilder().setDatumWriterFactory(datumWriterFactory).build(); - } - - /** - * Writes to Avro file(s) with the specified metadata. - * - *

    Supported value types are String, Long, and byte[]. - */ - public TypedWrite withMetadata(Map metadata) { - Map badKeys = Maps.newLinkedHashMap(); - for (Map.Entry entry : metadata.entrySet()) { - Object v = entry.getValue(); - if (!(v instanceof String || v instanceof Long || v instanceof byte[])) { - badKeys.put(entry.getKey(), v.getClass().getSimpleName()); - } - } - checkArgument( - badKeys.isEmpty(), - "Metadata value type must be one of String, Long, or byte[]. Found %s", - badKeys); - return toBuilder().setMetadata(ImmutableMap.copyOf(metadata)).build(); - } - - DynamicAvroDestinations resolveDynamicDestinations() { - DynamicAvroDestinations dynamicDestinations = - getDynamicDestinations(); - if (dynamicDestinations == null) { - // In this case DestinationT is Void. - FilenamePolicy usedFilenamePolicy = getFilenamePolicy(); - if (usedFilenamePolicy == null) { - usedFilenamePolicy = - DefaultFilenamePolicy.fromStandardParameters( - getFilenamePrefix(), - getShardTemplate(), - getFilenameSuffix(), - getWindowedWrites()); - } - dynamicDestinations = - (DynamicAvroDestinations) - constantDestinations( - usedFilenamePolicy, - getSchema(), - getMetadata(), - getCodec().getCodec(), - getFormatFunction(), - getDatumWriterFactory()); - } - return dynamicDestinations; - } - - @Override - public WriteFilesResult expand(PCollection input) { - checkArgument( - getFilenamePrefix() != null || getTempDirectory() != null, - "Need to set either the filename prefix or the tempDirectory of a AvroIO.Write " - + "transform."); - if (getFilenamePolicy() != null) { - checkArgument( - getShardTemplate() == null && getFilenameSuffix() == null, - "shardTemplate and filenameSuffix should only be used with the default " - + "filename policy"); - } - if (getDynamicDestinations() != null) { - checkArgument( - getFormatFunction() == null, - "A format function should not be specified " - + "with DynamicDestinations. Use DynamicDestinations.formatRecord instead"); - } else { - checkArgument( - getSchema() != null, "Unless using DynamicDestinations, .withSchema() is required."); - } - - ValueProvider tempDirectory = getTempDirectory(); - if (tempDirectory == null) { - tempDirectory = getFilenamePrefix(); - } - WriteFiles write = - WriteFiles.to( - new AvroSink<>( - tempDirectory, - resolveDynamicDestinations(), - getGenericRecords(), - getSyncInterval())); - if (getNumShards() > 0) { - write = write.withNumShards(getNumShards()); - } - if (getWindowedWrites()) { - write = write.withWindowedWrites(); - } - if (getNoSpilling()) { - write = write.withNoSpilling(); - } - return input.apply("Write", write); - } - - @Override - public void populateDisplayData(DisplayData.Builder builder) { - super.populateDisplayData(builder); - resolveDynamicDestinations().populateDisplayData(builder); - builder - .addIfNotDefault( - DisplayData.item("numShards", getNumShards()).withLabel("Maximum Output Shards"), 0) - .addIfNotNull( - DisplayData.item("tempDirectory", getTempDirectory()) - .withLabel("Directory for temporary files")); - } - } - - /** - * This class is used as the default return value of {@link AvroIO#write} - * - *

    All methods in this class delegate to the appropriate method of {@link AvroIO.TypedWrite}. - * This class exists for backwards compatibility, and will be removed in Beam 3.0. - */ - public static class Write extends PTransform, PDone> { - @VisibleForTesting final TypedWrite inner; - - Write(TypedWrite inner) { - this.inner = inner; - } - - /** See {@link TypedWrite#to(String)}. */ - public Write to(String outputPrefix) { - return new Write<>( - inner - .to(FileBasedSink.convertToFileResourceIfPossible(outputPrefix)) - .withFormatFunction(SerializableFunctions.identity())); - } - - /** See {@link TypedWrite#to(ResourceId)} . */ - public Write to(ResourceId outputPrefix) { - return new Write<>( - inner.to(outputPrefix).withFormatFunction(SerializableFunctions.identity())); - } - - /** See {@link TypedWrite#to(ValueProvider)}. */ - public Write to(ValueProvider outputPrefix) { - return new Write<>( - inner.to(outputPrefix).withFormatFunction(SerializableFunctions.identity())); - } - - /** See {@link TypedWrite#to(ResourceId)}. */ - public Write toResource(ValueProvider outputPrefix) { - return new Write<>( - inner.toResource(outputPrefix).withFormatFunction(SerializableFunctions.identity())); - } - - /** See {@link TypedWrite#to(FilenamePolicy)}. */ - public Write to(FilenamePolicy filenamePolicy) { - return new Write<>( - inner.to(filenamePolicy).withFormatFunction(SerializableFunctions.identity())); - } - - /** - * See {@link TypedWrite#to(DynamicAvroDestinations)}. - * - * @deprecated Use {@link FileIO#write()} or {@link FileIO#writeDynamic()} instead. - */ - @Deprecated - public Write to(DynamicAvroDestinations dynamicDestinations) { - return new Write<>(inner.to(dynamicDestinations).withFormatFunction(null)); - } - - /** See {@link TypedWrite#withSyncInterval}. */ - public Write withSyncInterval(int syncInterval) { - return new Write<>(inner.withSyncInterval(syncInterval)); - } - - /** See {@link TypedWrite#withSchema}. */ - public Write withSchema(Schema schema) { - return new Write<>(inner.withSchema(schema)); - } - - /** See {@link TypedWrite#withTempDirectory(ValueProvider)}. */ - public Write withTempDirectory(ValueProvider tempDirectory) { - return new Write<>(inner.withTempDirectory(tempDirectory)); - } - - /** See {@link TypedWrite#withTempDirectory(ResourceId)}. */ - public Write withTempDirectory(ResourceId tempDirectory) { - return new Write<>(inner.withTempDirectory(tempDirectory)); - } - - /** See {@link TypedWrite#withShardNameTemplate}. */ - public Write withShardNameTemplate(String shardTemplate) { - return new Write<>(inner.withShardNameTemplate(shardTemplate)); - } - - /** See {@link TypedWrite#withSuffix}. */ - public Write withSuffix(String filenameSuffix) { - return new Write<>(inner.withSuffix(filenameSuffix)); - } - - /** See {@link TypedWrite#withNumShards}. */ - public Write withNumShards(int numShards) { - return new Write<>(inner.withNumShards(numShards)); - } - - /** See {@link TypedWrite#withoutSharding}. */ - public Write withoutSharding() { - return new Write<>(inner.withoutSharding()); - } - - /** See {@link TypedWrite#withWindowedWrites}. */ - public Write withWindowedWrites() { - return new Write<>(inner.withWindowedWrites()); - } - - /** See {@link TypedWrite#withCodec}. */ - public Write withCodec(CodecFactory codec) { - return new Write<>(inner.withCodec(codec)); - } - - /** See {@link TypedWrite#withDatumWriterFactory}. */ - public Write withDatumWriterFactory(AvroSink.DatumWriterFactory datumWriterFactory) { - return new Write<>(inner.withDatumWriterFactory(datumWriterFactory)); - } - - /** - * Specify that output filenames are wanted. - * - *

    The nested {@link TypedWrite}transform always has access to output filenames, however due - * to backwards-compatibility concerns, {@link Write} cannot return them. This method simply - * returns the inner {@link TypedWrite} transform which has {@link WriteFilesResult} as its - * output type, allowing access to output files. - * - *

    The supplied {@code DestinationT} type must be: the same as that supplied in {@link - * #to(DynamicAvroDestinations)} if that method was used, or {@code Void} otherwise. - */ - public TypedWrite withOutputFilenames() { - return (TypedWrite) inner; - } - - /** See {@link TypedWrite#withMetadata} . */ - public Write withMetadata(Map metadata) { - return new Write<>(inner.withMetadata(metadata)); - } - - @Override - public PDone expand(PCollection input) { - input.apply(inner); - return PDone.in(input.getPipeline()); - } - - @Override - public void populateDisplayData(DisplayData.Builder builder) { - inner.populateDisplayData(builder); - } - } - - /** - * Returns a {@link DynamicAvroDestinations} that always returns the same {@link FilenamePolicy}, - * schema, metadata, and codec. - */ - public static DynamicAvroDestinations constantDestinations( - FilenamePolicy filenamePolicy, - Schema schema, - Map metadata, - CodecFactory codec, - SerializableFunction formatFunction) { - return constantDestinations(filenamePolicy, schema, metadata, codec, formatFunction, null); - } - - /** - * Returns a {@link DynamicAvroDestinations} that always returns the same {@link FilenamePolicy}, - * schema, metadata, and codec. - */ - public static DynamicAvroDestinations constantDestinations( - FilenamePolicy filenamePolicy, - Schema schema, - Map metadata, - CodecFactory codec, - SerializableFunction formatFunction, - AvroSink.@Nullable DatumWriterFactory datumWriterFactory) { - return new ConstantAvroDestination<>( - filenamePolicy, schema, metadata, codec, formatFunction, datumWriterFactory); - } - ///////////////////////////////////////////////////////////////////////////// - - /** - * Formats an element of a user type into a record with the given schema. - * - * @deprecated Users can achieve the same by providing this transform in a {@link - * org.apache.beam.sdk.transforms.ParDo} before using write in AvroIO {@link #write(Class)}. - */ - @Deprecated - public interface RecordFormatter extends Serializable { - GenericRecord formatRecord(ElementT element, Schema schema); - } - - /** - * A {@link Sink} for use with {@link FileIO#write} and {@link FileIO#writeDynamic}, writing - * elements of the given generated class, like {@link #write(Class)}. - */ - public static Sink sink(final Class clazz) { - return new AutoValue_AvroIO_Sink.Builder() - .setJsonSchema(ReflectData.get().getSchema(clazz).toString()) - .setMetadata(ImmutableMap.of()) - .setCodec(TypedWrite.DEFAULT_SERIALIZABLE_CODEC) - .build(); - } - - /** - * A {@link Sink} for use with {@link FileIO#write} and {@link FileIO#writeDynamic}, writing - * elements with a given (common) schema, like {@link #writeGenericRecords(Schema)}. - */ - public static Sink sink(Schema schema) { - return sink(schema.toString()); - } - - /** - * A {@link Sink} for use with {@link FileIO#write} and {@link FileIO#writeDynamic}, writing - * elements with a given (common) schema, like {@link #writeGenericRecords(String)}. - */ - public static Sink sink(String jsonSchema) { - return new AutoValue_AvroIO_Sink.Builder() - .setJsonSchema(jsonSchema) - .setMetadata(ImmutableMap.of()) - .setCodec(TypedWrite.DEFAULT_SERIALIZABLE_CODEC) - .build(); - } - - /** - * A {@link Sink} for use with {@link FileIO#write} and {@link FileIO#writeDynamic}, writing - * elements by converting each one to a {@link GenericRecord} with a given (common) schema, like - * {@link #writeCustomTypeToGenericRecords()}. - * - * @deprecated RecordFormatter will be removed in future versions. - */ - @Deprecated - public static Sink sinkViaGenericRecords( - Schema schema, RecordFormatter formatter) { - return new AutoValue_AvroIO_Sink.Builder() - .setRecordFormatter(formatter) - .setJsonSchema(schema.toString()) - .setMetadata(ImmutableMap.of()) - .setCodec(TypedWrite.DEFAULT_SERIALIZABLE_CODEC) - .build(); - } - - /** Implementation of {@link #sink} and {@link #sinkViaGenericRecords}. */ - @AutoValue - public abstract static class Sink implements FileIO.Sink { - /** @deprecated RecordFormatter will be removed in future versions. */ - @Deprecated - abstract @Nullable RecordFormatter getRecordFormatter(); - - abstract @Nullable String getJsonSchema(); - - abstract Map getMetadata(); - - abstract SerializableAvroCodecFactory getCodec(); - - abstract Builder toBuilder(); - - @AutoValue.Builder - abstract static class Builder { - /** @deprecated RecordFormatter will be removed in future versions. */ - @Deprecated - abstract Builder setRecordFormatter(RecordFormatter formatter); - - abstract Builder setJsonSchema(String jsonSchema); - - abstract Builder setMetadata(Map metadata); - - abstract Builder setCodec(SerializableAvroCodecFactory codec); - - abstract Sink build(); - } - - /** Specifies to put the given metadata into each generated file. By default, empty. */ - public Sink withMetadata(Map metadata) { - return toBuilder().setMetadata(metadata).build(); - } - - /** - * Specifies to use the given {@link CodecFactory} for each generated file. By default, {@code - * CodecFactory.snappyCodec()}. - */ - public Sink withCodec(CodecFactory codec) { - return toBuilder().setCodec(new SerializableAvroCodecFactory(codec)).build(); - } - - private transient @Nullable Schema schema; - private transient @Nullable DataFileWriter reflectWriter; - private transient @Nullable DataFileWriter genericWriter; - - @Override - public void open(WritableByteChannel channel) throws IOException { - this.schema = new Schema.Parser().parse(getJsonSchema()); - DataFileWriter writer; - if (getRecordFormatter() == null) { - writer = reflectWriter = new DataFileWriter<>(new ReflectDatumWriter<>(schema)); - } else { - writer = genericWriter = new DataFileWriter<>(new GenericDatumWriter<>(schema)); - } - writer.setCodec(getCodec().getCodec()); - for (Map.Entry entry : getMetadata().entrySet()) { - Object v = entry.getValue(); - if (v instanceof String) { - writer.setMeta(entry.getKey(), (String) v); - } else if (v instanceof Long) { - writer.setMeta(entry.getKey(), (Long) v); - } else if (v instanceof byte[]) { - writer.setMeta(entry.getKey(), (byte[]) v); - } else { - throw new IllegalStateException( - "Metadata value type must be one of String, Long, or byte[]. Found " - + v.getClass().getSimpleName()); - } - } - writer.create(schema, Channels.newOutputStream(channel)); - } - - @Override - public void write(ElementT element) throws IOException { - if (getRecordFormatter() == null) { - reflectWriter.append(element); - } else { - genericWriter.append(getRecordFormatter().formatRecord(element, schema)); - } - } - - @Override - public void flush() throws IOException { - MoreObjects.firstNonNull(reflectWriter, genericWriter).flush(); - } - } - - /** Disallow construction of utility class. */ - private AvroIO() {} -} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSchemaIOProvider.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSchemaIOProvider.java deleted file mode 100644 index 43498235992a7..0000000000000 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSchemaIOProvider.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io; - -import com.google.auto.service.AutoService; -import java.io.Serializable; -import org.apache.avro.generic.GenericRecord; -import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.sdk.io.AvroIO.Write; -import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.sdk.schemas.Schema.FieldType; -import org.apache.beam.sdk.schemas.io.SchemaIO; -import org.apache.beam.sdk.schemas.io.SchemaIOProvider; -import org.apache.beam.sdk.schemas.transforms.Convert; -import org.apache.beam.sdk.schemas.utils.AvroUtils; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.transforms.windowing.FixedWindows; -import org.apache.beam.sdk.transforms.windowing.Window; -import org.apache.beam.sdk.values.PBegin; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PCollection.IsBounded; -import org.apache.beam.sdk.values.PDone; -import org.apache.beam.sdk.values.POutput; -import org.apache.beam.sdk.values.Row; -import org.checkerframework.checker.nullness.qual.Nullable; -import org.joda.time.Duration; - -/** - * An implementation of {@link SchemaIOProvider} for reading and writing Avro files with {@link - * AvroIO}. - * - * @deprecated Avro related classes are deprecated in module beam-sdks-java-core and - * will be eventually removed. Please, migrate to a new module - * beam-sdks-java-extensions-avro by importing - * org.apache.beam.sdk.extensions.avro.io.AvroSchemaIOProvider instead of this one. - */ -@Internal -@AutoService(SchemaIOProvider.class) -@SuppressWarnings({ - "nullness" // TODO(https://github.com/apache/beam/issues/20497) -}) -@Deprecated -public class AvroSchemaIOProvider implements SchemaIOProvider { - /** Returns an id that uniquely represents this IO. */ - @Override - public String identifier() { - return "avro"; - } - - /** - * Returns the expected schema of the configuration object. Note this is distinct from the schema - * of the data source itself. No configuration expected for Avro. - */ - @Override - public Schema configurationSchema() { - return Schema.builder().addNullableField("writeWindowSizeSeconds", FieldType.INT64).build(); - } - - /** - * Produce a SchemaIO given a String representing the data's location, the schema of the data that - * resides there, and some IO-specific configuration object. - */ - @Override - public AvroSchemaIO from(String location, Row configuration, Schema dataSchema) { - return new AvroSchemaIO(location, dataSchema, configuration); - } - - @Override - public boolean requiresDataSchema() { - return true; - } - - @Override - public PCollection.IsBounded isBounded() { - // This supports streaming now as well but there's no option for this. The move to - // SchemaTransform will remove the need to provide this. - return PCollection.IsBounded.BOUNDED; - } - - /** An abstraction to create schema aware IOs. */ - private static class AvroSchemaIO implements SchemaIO, Serializable { - protected final Schema dataSchema; - protected final String location; - protected final @Nullable Duration windowSize; - - private AvroSchemaIO(String location, Schema dataSchema, Row configuration) { - this.dataSchema = dataSchema; - this.location = location; - if (configuration.getInt64("writeWindowSizeSeconds") != null) { - windowSize = Duration.standardSeconds(configuration.getInt64("writeWindowSizeSeconds")); - } else { - windowSize = null; - } - } - - @Override - public Schema schema() { - return dataSchema; - } - - @Override - public PTransform> buildReader() { - return new PTransform>() { - @Override - public PCollection expand(PBegin begin) { - return begin - .apply( - "AvroIORead", - AvroIO.readGenericRecords(AvroUtils.toAvroSchema(dataSchema, null, null)) - .withBeamSchemas(true) - .from(location)) - .apply("ToRows", Convert.toRows()); - } - }; - } - - @Override - public PTransform, POutput> buildWriter() { - return new PTransform, POutput>() { - @Override - public PDone expand(PCollection input) { - PCollection asRecords = - input.apply("ToGenericRecords", Convert.to(GenericRecord.class)); - Write avroWrite = - AvroIO.writeGenericRecords(AvroUtils.toAvroSchema(dataSchema, null, null)) - .to(location); - if (input.isBounded() == IsBounded.UNBOUNDED || windowSize != null) { - asRecords = - asRecords.apply( - Window.into( - FixedWindows.of( - windowSize == null ? Duration.standardMinutes(1) : windowSize))); - avroWrite = avroWrite.withWindowedWrites().withNumShards(1); - } else { - avroWrite = avroWrite.withoutSharding(); - } - return asRecords.apply("AvroIOWrite", avroWrite); - } - }; - } - } -} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSink.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSink.java deleted file mode 100644 index bc92113925cd7..0000000000000 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSink.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io; - -import java.io.Serializable; -import java.nio.channels.Channels; -import java.nio.channels.WritableByteChannel; -import java.util.Map; -import org.apache.avro.Schema; -import org.apache.avro.file.CodecFactory; -import org.apache.avro.file.DataFileWriter; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.reflect.ReflectDatumWriter; -import org.apache.beam.sdk.io.fs.ResourceId; -import org.apache.beam.sdk.options.ValueProvider; -import org.apache.beam.sdk.util.MimeTypes; -import org.checkerframework.checker.nullness.qual.Nullable; - -/** - * A {@link FileBasedSink} for Avro files. - * - * @deprecated Avro related classes are deprecated in module beam-sdks-java-core and - * will be eventually removed. Please, migrate to a new module - * beam-sdks-java-extensions-avro by importing - * org.apache.beam.sdk.extensions.avro.io.AvroSink instead of this one. - */ -@SuppressWarnings({ - "nullness" // TODO(https://github.com/apache/beam/issues/20497) -}) -@Deprecated -public class AvroSink - extends FileBasedSink { - private final boolean genericRecords; - private final int syncInterval; - - @FunctionalInterface - public interface DatumWriterFactory extends Serializable { - DatumWriter apply(Schema writer); - } - - AvroSink( - ValueProvider outputPrefix, - DynamicAvroDestinations dynamicDestinations, - boolean genericRecords, - int syncInterval) { - // Avro handles compression internally using the codec. - super(outputPrefix, dynamicDestinations, Compression.UNCOMPRESSED); - this.genericRecords = genericRecords; - this.syncInterval = syncInterval; - } - - @Override - public DynamicAvroDestinations getDynamicDestinations() { - return (DynamicAvroDestinations) super.getDynamicDestinations(); - } - - @Override - public WriteOperation createWriteOperation() { - return new AvroWriteOperation<>(this, genericRecords, syncInterval); - } - - /** A {@link WriteOperation WriteOperation} for Avro files. */ - private static class AvroWriteOperation - extends WriteOperation { - private final DynamicAvroDestinations dynamicDestinations; - private final boolean genericRecords; - private final int syncInterval; - - private AvroWriteOperation( - AvroSink sink, boolean genericRecords, int syncInterval) { - super(sink); - this.dynamicDestinations = sink.getDynamicDestinations(); - this.genericRecords = genericRecords; - this.syncInterval = syncInterval; - } - - @Override - public Writer createWriter() throws Exception { - return new AvroWriter<>(this, dynamicDestinations, genericRecords, syncInterval); - } - } - - /** A {@link Writer Writer} for Avro files. */ - private static class AvroWriter extends Writer { - - // Initialized in prepareWrite - private @Nullable DataFileWriter dataFileWriter; - - private final DynamicAvroDestinations dynamicDestinations; - private final boolean genericRecords; - private final int syncInterval; - - public AvroWriter( - WriteOperation writeOperation, - DynamicAvroDestinations dynamicDestinations, - boolean genericRecords, - int syncInterval) { - super(writeOperation, MimeTypes.BINARY); - this.dynamicDestinations = dynamicDestinations; - this.genericRecords = genericRecords; - this.syncInterval = syncInterval; - } - - @SuppressWarnings("deprecation") // uses internal test functionality. - @Override - protected void prepareWrite(WritableByteChannel channel) throws Exception { - DestinationT destination = getDestination(); - CodecFactory codec = dynamicDestinations.getCodec(destination); - Schema schema = dynamicDestinations.getSchema(destination); - Map metadata = dynamicDestinations.getMetadata(destination); - DatumWriter datumWriter; - DatumWriterFactory datumWriterFactory = - dynamicDestinations.getDatumWriterFactory(destination); - - if (datumWriterFactory == null) { - datumWriter = - genericRecords ? new GenericDatumWriter<>(schema) : new ReflectDatumWriter<>(schema); - } else { - datumWriter = datumWriterFactory.apply(schema); - } - - dataFileWriter = new DataFileWriter<>(datumWriter).setCodec(codec); - for (Map.Entry entry : metadata.entrySet()) { - Object v = entry.getValue(); - if (v instanceof String) { - dataFileWriter.setMeta(entry.getKey(), (String) v); - } else if (v instanceof Long) { - dataFileWriter.setMeta(entry.getKey(), (Long) v); - } else if (v instanceof byte[]) { - dataFileWriter.setMeta(entry.getKey(), (byte[]) v); - } else { - throw new IllegalStateException( - "Metadata value type must be one of String, Long, or byte[]. Found " - + v.getClass().getSimpleName()); - } - } - dataFileWriter.setSyncInterval(syncInterval); - dataFileWriter.create(schema, Channels.newOutputStream(channel)); - } - - @Override - public void write(OutputT value) throws Exception { - dataFileWriter.append(value); - } - - @Override - protected void finishWrite() throws Exception { - dataFileWriter.flush(); - } - } -} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java deleted file mode 100644 index 1bf781869aa6f..0000000000000 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java +++ /dev/null @@ -1,773 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io; - -import static org.apache.beam.sdk.io.FileBasedSource.Mode.SINGLE_FILE_OR_SUBRANGE; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InvalidObjectException; -import java.io.ObjectInputStream; -import java.io.ObjectStreamException; -import java.io.Serializable; -import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.ReadableByteChannel; -import java.nio.channels.SeekableByteChannel; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.Iterator; -import java.util.Map; -import java.util.WeakHashMap; -import javax.annotation.concurrent.GuardedBy; -import org.apache.avro.Schema; -import org.apache.avro.file.DataFileConstants; -import org.apache.avro.file.DataFileReader; -import org.apache.avro.file.SeekableInput; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.BinaryDecoder; -import org.apache.avro.io.DatumReader; -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.reflect.ReflectData; -import org.apache.avro.reflect.ReflectDatumReader; -import org.apache.beam.sdk.PipelineRunner; -import org.apache.beam.sdk.coders.AvroCoder; -import org.apache.beam.sdk.coders.Coder; -import org.apache.beam.sdk.io.fs.EmptyMatchTreatment; -import org.apache.beam.sdk.io.fs.MatchResult.Metadata; -import org.apache.beam.sdk.io.fs.ResourceId; -import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.sdk.options.ValueProvider; -import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.sdk.util.VarInt; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.checkerframework.checker.nullness.qual.Nullable; - -// CHECKSTYLE.OFF: JavadocStyle -/** - * Do not use in pipelines directly: most users should use {@link AvroIO.Read}. - * - *

    A {@link FileBasedSource} for reading Avro files. - * - *

    To read a {@link PCollection} of objects from one or more Avro files, use {@link - * AvroSource#from} to specify the path(s) of the files to read. The {@link AvroSource} that is - * returned will read objects of type {@link GenericRecord} with the schema(s) that were written at - * file creation. To further configure the {@link AvroSource} to read with a user-defined schema, or - * to return records of a type other than {@link GenericRecord}, use {@link - * AvroSource#withSchema(Schema)} (using an Avro {@link Schema}), {@link - * AvroSource#withSchema(String)} (using a JSON schema), or {@link AvroSource#withSchema(Class)} (to - * return objects of the Avro-generated class specified). - * - *

    An {@link AvroSource} can be read from using the {@link Read} transform. For example: - * - *

    {@code
    - * AvroSource source = AvroSource.from(file.toPath()).withSchema(MyType.class);
    - * PCollection records = Read.from(mySource);
    - * }
    - * - *

    This class's implementation is based on the Avro 1.7.7 specification and implements - * parsing of some parts of Avro Object Container Files. The rationale for doing so is that the Avro - * API does not provide efficient ways of computing the precise offsets of blocks within a file, - * which is necessary to support dynamic work rebalancing. However, whenever it is possible to use - * the Avro API in a way that supports maintaining precise offsets, this class uses the Avro API. - * - *

    Avro Object Container files store records in blocks. Each block contains a collection of - * records. Blocks may be encoded (e.g., with bzip2, deflate, snappy, etc.). Blocks are delineated - * from one another by a 16-byte sync marker. - * - *

    An {@link AvroSource} for a subrange of a single file contains records in the blocks such that - * the start offset of the block is greater than or equal to the start offset of the source and less - * than the end offset of the source. - * - *

    To use XZ-encoded Avro files, please include an explicit dependency on {@code xz-1.8.jar}, - * which has been marked as optional in the Maven {@code sdk/pom.xml}. - * - *

    {@code
    - * 
    - *   org.tukaani
    - *   xz
    - *   1.8
    - * 
    - * }
    - * - *

    Permissions

    - * - *

    Permission requirements depend on the {@link PipelineRunner} that is used to execute the - * pipeline. Please refer to the documentation of corresponding {@link PipelineRunner}s for more - * details. - * - * @param The type of records to be read from the source. - * @deprecated Avro related classes are deprecated in module beam-sdks-java-core and - * will be eventually removed. Please, migrate to a new module - * beam-sdks-java-extensions-avro by importing - * org.apache.beam.sdk.extensions.avro.io.AvroSource instead of this one. - */ -// CHECKSTYLE.ON: JavadocStyle - -@SuppressWarnings({ - "nullness" // TODO(https://github.com/apache/beam/issues/20497) -}) -@Deprecated -public class AvroSource extends BlockBasedSource { - // Default minimum bundle size (chosen as two default-size Avro blocks to attempt to - // ensure that every source has at least one block of records). - // The default sync interval is 64k. - private static final long DEFAULT_MIN_BUNDLE_SIZE = 2L * DataFileConstants.DEFAULT_SYNC_INTERVAL; - - @FunctionalInterface - public interface DatumReaderFactory extends Serializable { - DatumReader apply(Schema writer, Schema reader); - } - - private static final DatumReaderFactory GENERIC_DATUM_READER_FACTORY = GenericDatumReader::new; - - private static final DatumReaderFactory REFLECT_DATUM_READER_FACTORY = ReflectDatumReader::new; - - // Use cases of AvroSource are: - // 1) AvroSource Reading GenericRecord records with a specified schema. - // 2) AvroSource Reading records of a generated Avro class Foo. - // 3) AvroSource Reading GenericRecord records with an unspecified schema - // and converting them to type T. - // | Case 1 | Case 2 | Case 3 | - // type | GenericRecord | Foo | GenericRecord | - // readerSchemaString | non-null | non-null | null | - // parseFn | null | null | non-null | - // outputCoder | null | null | non-null | - // readerFactory | either | either | either | - private static class Mode implements Serializable { - private final Class type; - - // The JSON schema used to decode records. - private @Nullable String readerSchemaString; - - private final @Nullable SerializableFunction parseFn; - - private final @Nullable Coder outputCoder; - - private final @Nullable DatumReaderFactory readerFactory; - - private Mode( - Class type, - @Nullable String readerSchemaString, - @Nullable SerializableFunction parseFn, - @Nullable Coder outputCoder, - @Nullable DatumReaderFactory readerFactory) { - this.type = type; - this.readerSchemaString = internSchemaString(readerSchemaString); - this.parseFn = parseFn; - this.outputCoder = outputCoder; - this.readerFactory = readerFactory; - } - - private void readObject(ObjectInputStream is) throws IOException, ClassNotFoundException { - is.defaultReadObject(); - readerSchemaString = internSchemaString(readerSchemaString); - } - - private Coder getOutputCoder() { - if (parseFn == null) { - return AvroCoder.of((Class) type, internOrParseSchemaString(readerSchemaString)); - } else { - return outputCoder; - } - } - - private void validate() { - if (parseFn == null) { - checkArgument( - readerSchemaString != null, - "schema must be specified using withSchema() when not using a parse fn"); - } - } - - private Mode withReaderFactory(DatumReaderFactory factory) { - return new Mode<>(type, readerSchemaString, parseFn, outputCoder, factory); - } - - private DatumReader createReader(Schema writerSchema, Schema readerSchema) { - DatumReaderFactory factory = this.readerFactory; - if (factory == null) { - factory = - (type == GenericRecord.class) - ? GENERIC_DATUM_READER_FACTORY - : REFLECT_DATUM_READER_FACTORY; - } - return factory.apply(writerSchema, readerSchema); - } - } - - private static Mode readGenericRecordsWithSchema( - String schema, @Nullable DatumReaderFactory factory) { - return new Mode<>(GenericRecord.class, schema, null, null, factory); - } - - private static Mode readGeneratedClasses( - Class clazz, @Nullable DatumReaderFactory factory) { - return new Mode<>(clazz, ReflectData.get().getSchema(clazz).toString(), null, null, factory); - } - - private static Mode parseGenericRecords( - SerializableFunction parseFn, - Coder outputCoder, - @Nullable DatumReaderFactory factory) { - return new Mode<>(GenericRecord.class, null, parseFn, outputCoder, factory); - } - - private final Mode mode; - - /** - * Reads from the given file name or pattern ("glob"). The returned source needs to be further - * configured by calling {@link #withSchema} to return a type other than {@link GenericRecord}. - */ - public static AvroSource from(ValueProvider fileNameOrPattern) { - return new AvroSource<>( - fileNameOrPattern, - EmptyMatchTreatment.DISALLOW, - DEFAULT_MIN_BUNDLE_SIZE, - readGenericRecordsWithSchema(null /* will need to be specified in withSchema */, null)); - } - - public static AvroSource from(Metadata metadata) { - return new AvroSource<>( - metadata, - DEFAULT_MIN_BUNDLE_SIZE, - 0, - metadata.sizeBytes(), - readGenericRecordsWithSchema(null /* will need to be specified in withSchema */, null)); - } - - /** Like {@link #from(ValueProvider)}. */ - public static AvroSource from(String fileNameOrPattern) { - return from(ValueProvider.StaticValueProvider.of(fileNameOrPattern)); - } - - public AvroSource withEmptyMatchTreatment(EmptyMatchTreatment emptyMatchTreatment) { - return new AvroSource<>( - getFileOrPatternSpecProvider(), emptyMatchTreatment, getMinBundleSize(), mode); - } - - /** Reads files containing records that conform to the given schema. */ - public AvroSource withSchema(String schema) { - checkArgument(schema != null, "schema can not be null"); - return new AvroSource<>( - getFileOrPatternSpecProvider(), - getEmptyMatchTreatment(), - getMinBundleSize(), - readGenericRecordsWithSchema(schema, mode.readerFactory)); - } - - /** Like {@link #withSchema(String)}. */ - public AvroSource withSchema(Schema schema) { - checkArgument(schema != null, "schema can not be null"); - return withSchema(schema.toString()); - } - - /** Reads files containing records of the given class. */ - public AvroSource withSchema(Class clazz) { - checkArgument(clazz != null, "clazz can not be null"); - if (getMode() == SINGLE_FILE_OR_SUBRANGE) { - return new AvroSource<>( - getSingleFileMetadata(), - getMinBundleSize(), - getStartOffset(), - getEndOffset(), - readGeneratedClasses(clazz, mode.readerFactory)); - } - return new AvroSource<>( - getFileOrPatternSpecProvider(), - getEmptyMatchTreatment(), - getMinBundleSize(), - readGeneratedClasses(clazz, mode.readerFactory)); - } - - /** - * Reads {@link GenericRecord} of unspecified schema and maps them to instances of a custom type - * using the given {@code parseFn} and encoded using the given coder. - */ - public AvroSource withParseFn( - SerializableFunction parseFn, Coder coder) { - checkArgument(parseFn != null, "parseFn can not be null"); - checkArgument(coder != null, "coder can not be null"); - if (getMode() == SINGLE_FILE_OR_SUBRANGE) { - return new AvroSource<>( - getSingleFileMetadata(), - getMinBundleSize(), - getStartOffset(), - getEndOffset(), - parseGenericRecords(parseFn, coder, mode.readerFactory)); - } - return new AvroSource<>( - getFileOrPatternSpecProvider(), - getEmptyMatchTreatment(), - getMinBundleSize(), - parseGenericRecords(parseFn, coder, mode.readerFactory)); - } - - /** - * Sets the minimum bundle size. Refer to {@link OffsetBasedSource} for a description of {@code - * minBundleSize} and its use. - */ - public AvroSource withMinBundleSize(long minBundleSize) { - if (getMode() == SINGLE_FILE_OR_SUBRANGE) { - return new AvroSource<>( - getSingleFileMetadata(), minBundleSize, getStartOffset(), getEndOffset(), mode); - } - return new AvroSource<>( - getFileOrPatternSpecProvider(), getEmptyMatchTreatment(), minBundleSize, mode); - } - - public AvroSource withDatumReaderFactory(DatumReaderFactory factory) { - Mode newMode = mode.withReaderFactory(factory); - if (getMode() == SINGLE_FILE_OR_SUBRANGE) { - return new AvroSource<>( - getSingleFileMetadata(), getMinBundleSize(), getStartOffset(), getEndOffset(), newMode); - } - return new AvroSource<>( - getFileOrPatternSpecProvider(), getEmptyMatchTreatment(), getMinBundleSize(), newMode); - } - - /** Constructor for FILEPATTERN mode. */ - private AvroSource( - ValueProvider fileNameOrPattern, - EmptyMatchTreatment emptyMatchTreatment, - long minBundleSize, - Mode mode) { - super(fileNameOrPattern, emptyMatchTreatment, minBundleSize); - this.mode = mode; - } - - /** Constructor for SINGLE_FILE_OR_SUBRANGE mode. */ - private AvroSource( - Metadata metadata, long minBundleSize, long startOffset, long endOffset, Mode mode) { - super(metadata, minBundleSize, startOffset, endOffset); - this.mode = mode; - } - - @Override - public void validate() { - super.validate(); - mode.validate(); - } - - /** - * Used by the Dataflow worker. Do not introduce new usages. Do not delete without confirming that - * Dataflow ValidatesRunner tests pass. - * - * @deprecated Used by Dataflow worker - */ - @Deprecated - public BlockBasedSource createForSubrangeOfFile(String fileName, long start, long end) - throws IOException { - return createForSubrangeOfFile(FileSystems.matchSingleFileSpec(fileName), start, end); - } - - @Override - public BlockBasedSource createForSubrangeOfFile(Metadata fileMetadata, long start, long end) { - return new AvroSource<>(fileMetadata, getMinBundleSize(), start, end, mode); - } - - @Override - protected BlockBasedReader createSingleFileReader(PipelineOptions options) { - return new AvroReader<>(this); - } - - @Override - public Coder getOutputCoder() { - return mode.getOutputCoder(); - } - - @VisibleForTesting - @Nullable - String getReaderSchemaString() { - return mode.readerSchemaString; - } - - /** Avro file metadata. */ - @VisibleForTesting - static class AvroMetadata { - private final byte[] syncMarker; - private final String codec; - private final String schemaString; - - AvroMetadata(byte[] syncMarker, String codec, String schemaString) { - this.syncMarker = checkNotNull(syncMarker, "syncMarker"); - this.codec = checkNotNull(codec, "codec"); - this.schemaString = internSchemaString(checkNotNull(schemaString, "schemaString")); - } - - /** - * The JSON-encoded schema - * string for the file. - */ - public String getSchemaString() { - return schemaString; - } - - /** - * The codec of the - * file. - */ - public String getCodec() { - return codec; - } - - /** - * The 16-byte sync marker for the file. See the documentation for Object Container - * File for more information. - */ - public byte[] getSyncMarker() { - return syncMarker; - } - } - - /** - * Reads the {@link AvroMetadata} from the header of an Avro file. - * - *

    This method parses the header of an Avro Object Container - * File. - * - * @throws IOException if the file is an invalid format. - */ - @VisibleForTesting - static AvroMetadata readMetadataFromFile(ResourceId fileResource) throws IOException { - String codec = null; - String schemaString = null; - byte[] syncMarker; - try (InputStream stream = Channels.newInputStream(FileSystems.open(fileResource))) { - BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(stream, null); - - // The header of an object container file begins with a four-byte magic number, followed - // by the file metadata (including the schema and codec), encoded as a map. Finally, the - // header ends with the file's 16-byte sync marker. - // See https://avro.apache.org/docs/1.7.7/spec.html#Object+Container+Files for details on - // the encoding of container files. - - // Read the magic number. - byte[] magic = new byte[DataFileConstants.MAGIC.length]; - decoder.readFixed(magic); - if (!Arrays.equals(magic, DataFileConstants.MAGIC)) { - throw new IOException("Missing Avro file signature: " + fileResource); - } - - // Read the metadata to find the codec and schema. - ByteBuffer valueBuffer = ByteBuffer.allocate(512); - long numRecords = decoder.readMapStart(); - while (numRecords > 0) { - for (long recordIndex = 0; recordIndex < numRecords; recordIndex++) { - String key = decoder.readString(); - // readBytes() clears the buffer and returns a buffer where: - // - position is the start of the bytes read - // - limit is the end of the bytes read - valueBuffer = decoder.readBytes(valueBuffer); - byte[] bytes = new byte[valueBuffer.remaining()]; - valueBuffer.get(bytes); - if (key.equals(DataFileConstants.CODEC)) { - codec = new String(bytes, StandardCharsets.UTF_8); - } else if (key.equals(DataFileConstants.SCHEMA)) { - schemaString = new String(bytes, StandardCharsets.UTF_8); - } - } - numRecords = decoder.mapNext(); - } - if (codec == null) { - codec = DataFileConstants.NULL_CODEC; - } - - // Finally, read the sync marker. - syncMarker = new byte[DataFileConstants.SYNC_SIZE]; - decoder.readFixed(syncMarker); - } - checkState(schemaString != null, "No schema present in Avro file metadata %s", fileResource); - return new AvroMetadata(syncMarker, codec, schemaString); - } - - // A logical reference cache used to store schemas and schema strings to allow us to - // "intern" values and reduce the number of copies of equivalent objects. - private static final Map schemaLogicalReferenceCache = new WeakHashMap<>(); - private static final Map schemaStringLogicalReferenceCache = new WeakHashMap<>(); - - // We avoid String.intern() because depending on the JVM, these may be added to the PermGenSpace - // which we want to avoid otherwise we could run out of PermGenSpace. - private static synchronized String internSchemaString(String schema) { - String internSchema = schemaStringLogicalReferenceCache.get(schema); - if (internSchema != null) { - return internSchema; - } - schemaStringLogicalReferenceCache.put(schema, schema); - return schema; - } - - static synchronized Schema internOrParseSchemaString(String schemaString) { - Schema schema = schemaLogicalReferenceCache.get(schemaString); - if (schema != null) { - return schema; - } - Schema.Parser parser = new Schema.Parser(); - schema = parser.parse(schemaString); - schemaLogicalReferenceCache.put(schemaString, schema); - return schema; - } - - // Reading the object from Java serialization typically does not go through the constructor, - // we use readResolve to replace the constructed instance with one which uses the constructor - // allowing us to intern any schemas. - @SuppressWarnings("unused") - private Object readResolve() throws ObjectStreamException { - switch (getMode()) { - case SINGLE_FILE_OR_SUBRANGE: - return new AvroSource<>( - getSingleFileMetadata(), getMinBundleSize(), getStartOffset(), getEndOffset(), mode); - case FILEPATTERN: - return new AvroSource<>( - getFileOrPatternSpecProvider(), getEmptyMatchTreatment(), getMinBundleSize(), mode); - default: - throw new InvalidObjectException( - String.format("Unknown mode %s for AvroSource %s", getMode(), this)); - } - } - - /** - * A {@link BlockBasedSource.Block} of Avro records. - * - * @param The type of records stored in the block. - */ - static class AvroBlock extends Block { - - // The current record in the block. Initialized in readNextRecord. - private @Nullable T currentRecord; - - // The index of the current record in the block. - private long currentRecordIndex = 0; - - private final Iterator iterator; - - private final SerializableFunction parseFn; - - private final long numRecordsInBlock; - - AvroBlock( - Iterator iter, SerializableFunction parseFn, long numRecordsInBlock) { - this.iterator = iter; - this.parseFn = parseFn; - this.numRecordsInBlock = numRecordsInBlock; - } - - @Override - public T getCurrentRecord() { - return currentRecord; - } - - @Override - public boolean readNextRecord() { - if (currentRecordIndex >= numRecordsInBlock) { - return false; - } - - Object record = iterator.next(); - currentRecord = (parseFn == null) ? ((T) record) : parseFn.apply((GenericRecord) record); - currentRecordIndex++; - return true; - } - - @Override - public double getFractionOfBlockConsumed() { - return ((double) currentRecordIndex) / numRecordsInBlock; - } - } - - /** - * A {@link BlockBasedSource.BlockBasedReader} for reading blocks from Avro files. - * - *

    An Avro Object Container File consists of a header followed by a 16-bit sync marker and then - * a sequence of blocks, where each block begins with two encoded longs representing the total - * number of records in the block and the block's size in bytes, followed by the block's - * (optionally-encoded) records. Each block is terminated by a 16-bit sync marker. - * - * @param The type of records contained in the block. - */ - public static class AvroReader extends BlockBasedReader { - - private static class SeekableChannelInput implements SeekableInput { - - private final SeekableByteChannel channel; - private final InputStream input; - - SeekableChannelInput(SeekableByteChannel channel) { - this.channel = channel; - this.input = Channels.newInputStream(channel); - } - - @Override - public void seek(long p) throws IOException { - channel.position(p); - } - - @Override - public long tell() throws IOException { - return channel.position(); - } - - @Override - public long length() throws IOException { - return channel.size(); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - return input.read(b, off, len); - } - - @Override - public void close() throws IOException { - channel.close(); - } - } - - // The current block. - // Initialized in readNextRecord. - private @Nullable AvroBlock currentBlock; - - private @Nullable DataFileReader dataFileReader; - - // A lock used to synchronize block offsets for getRemainingParallelism - private final Object progressLock = new Object(); - - // Offset of the current block. - @GuardedBy("progressLock") - private long currentBlockOffset = 0; - - // Size of the current block. - @GuardedBy("progressLock") - private long currentBlockSizeBytes = 0; - - /** Reads Avro records of type {@code T} from the specified source. */ - public AvroReader(AvroSource source) { - super(source); - } - - @Override - public synchronized AvroSource getCurrentSource() { - return (AvroSource) super.getCurrentSource(); - } - - // Precondition: the stream is positioned after the sync marker in the current (about to be - // previous) block. currentBlockSize equals the size of the current block, or zero if this - // reader was just started. - // - // Postcondition: same as above, but for the new current (formerly next) block. - @Override - public boolean readNextBlock() { - if (!dataFileReader.hasNext()) { - return false; - } - - long headerLength = - (long) VarInt.getLength(dataFileReader.getBlockCount()) - + VarInt.getLength(dataFileReader.getBlockSize()) - + DataFileConstants.SYNC_SIZE; - - currentBlock = - new AvroBlock<>( - dataFileReader, getCurrentSource().mode.parseFn, dataFileReader.getBlockCount()); - - // Atomically update both the position and offset of the new block. - synchronized (progressLock) { - currentBlockOffset = dataFileReader.previousSync(); - // Total block size includes the header, block content, and trailing sync marker. - currentBlockSizeBytes = dataFileReader.getBlockSize() + headerLength; - } - - return true; - } - - @Override - public AvroBlock getCurrentBlock() { - return currentBlock; - } - - @Override - public long getCurrentBlockOffset() { - synchronized (progressLock) { - return currentBlockOffset; - } - } - - @Override - public long getCurrentBlockSize() { - synchronized (progressLock) { - return currentBlockSizeBytes; - } - } - - @Override - public long getSplitPointsRemaining() { - if (isDone()) { - return 0; - } - synchronized (progressLock) { - if (currentBlockOffset + currentBlockSizeBytes >= getCurrentSource().getEndOffset()) { - // This block is known to be the last block in the range. - return 1; - } - } - return super.getSplitPointsRemaining(); - } - - // Postcondition: the stream is positioned at the beginning of the first block after the start - // of the current source, and currentBlockOffset is that position. Additionally, - // currentBlockSizeBytes will be set to 0 indicating that the previous block was empty. - @Override - protected void startReading(ReadableByteChannel channel) throws IOException { - SeekableChannelInput seekableChannelInput = - new SeekableChannelInput((SeekableByteChannel) channel); - // the channel needs to be at the beginning of the file in order for the DataFileReader to - // read the header, etc, we'll seek it back to where it should be after creating the DFR. - seekableChannelInput.seek(0); - - Schema readerSchema = null; - String readerSchemaString = this.getCurrentSource().getReaderSchemaString(); - if (readerSchemaString != null) { - readerSchema = AvroSource.internOrParseSchemaString(readerSchemaString); - } - // the DataFileReader will call setSchema with the writer schema when created. - DatumReader reader = this.getCurrentSource().mode.createReader(readerSchema, readerSchema); - - dataFileReader = new DataFileReader<>(seekableChannelInput, reader); - - long startOffset = getCurrentSource().getStartOffset(); - if (startOffset != 0) { - // the start offset may be in the middle of a sync marker, by rewinding SYNC_SIZE bytes we - // ensure that we won't miss the block if so. - dataFileReader.sync(Math.max(0, startOffset - DataFileConstants.SYNC_SIZE)); - } - - synchronized (progressLock) { - currentBlockOffset = dataFileReader.previousSync(); - currentBlockSizeBytes = 0; - } - } - } -} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSource.java index ecd114950c220..704843f16efca 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSource.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedReadFromUnboundedSource.java @@ -42,7 +42,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.ValueWithRecordId; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/ClassLoaderFileSystem.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/ClassLoaderFileSystem.java index 5167abb24c55d..f2eb1e1c079c1 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/ClassLoaderFileSystem.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/ClassLoaderFileSystem.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; import java.io.IOException; @@ -33,7 +33,7 @@ import org.apache.beam.sdk.io.fs.ResolveOptions; import org.apache.beam.sdk.io.fs.ResourceId; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; /** A read-only {@link FileSystem} implementation looking up resources using a ClassLoader. */ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java index 836b4bc1d6a90..2950702bef487 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.io.Serializable; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Compression.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Compression.java index 9e81aa96aa5c8..d9e7757547f53 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Compression.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Compression.java @@ -28,9 +28,9 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import org.apache.beam.sdk.util.LzoCompression; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Ints; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Ints; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/ConstantAvroDestination.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/ConstantAvroDestination.java deleted file mode 100644 index cc9957474b9bd..0000000000000 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/ConstantAvroDestination.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io; - -import java.io.Serializable; -import java.util.Map; -import org.apache.avro.Schema; -import org.apache.avro.file.CodecFactory; -import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy; -import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.sdk.transforms.display.DisplayData; -import org.apache.beam.sdk.transforms.display.HasDisplayData; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Suppliers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; -import org.checkerframework.checker.nullness.qual.Nullable; - -/** - * Always returns a constant {@link FilenamePolicy}, {@link Schema}, metadata, and codec. - * - * @deprecated Avro related classes are deprecated in module beam-sdks-java-core and - * will be eventually removed. Please, migrate to a new module - * beam-sdks-java-extensions-avro by importing - * org.apache.beam.sdk.extensions.avro.io.ConstantAvroDestination instead of this one. - */ -@SuppressWarnings({ - "nullness" // TODO(https://github.com/apache/beam/issues/20497) -}) -@Deprecated -class ConstantAvroDestination - extends DynamicAvroDestinations { - private static class SchemaFunction implements Serializable, Function { - @Override - public Schema apply(String input) { - return new Schema.Parser().parse(input); - } - } - - // This should be a multiple of 4 to not get a partial encoded byte. - private static final int METADATA_BYTES_MAX_LENGTH = 40; - private final FilenamePolicy filenamePolicy; - private final Supplier schema; - private final Map metadata; - private final SerializableAvroCodecFactory codec; - private final SerializableFunction formatFunction; - private final AvroSink.DatumWriterFactory datumWriterFactory; - - private class Metadata implements HasDisplayData { - @Override - public void populateDisplayData(DisplayData.Builder builder) { - for (Map.Entry entry : metadata.entrySet()) { - DisplayData.Type type = DisplayData.inferType(entry.getValue()); - if (type != null) { - builder.add(DisplayData.item(entry.getKey(), type, entry.getValue())); - } else { - String base64 = BaseEncoding.base64().encode((byte[]) entry.getValue()); - String repr = - base64.length() <= METADATA_BYTES_MAX_LENGTH - ? base64 - : base64.substring(0, METADATA_BYTES_MAX_LENGTH) + "..."; - builder.add(DisplayData.item(entry.getKey(), repr)); - } - } - } - } - - public ConstantAvroDestination( - FilenamePolicy filenamePolicy, - Schema schema, - Map metadata, - CodecFactory codec, - SerializableFunction formatFunction) { - this(filenamePolicy, schema, metadata, codec, formatFunction, null); - } - - public ConstantAvroDestination( - FilenamePolicy filenamePolicy, - Schema schema, - Map metadata, - CodecFactory codec, - SerializableFunction formatFunction, - AvroSink.@Nullable DatumWriterFactory datumWriterFactory) { - this.filenamePolicy = filenamePolicy; - this.schema = Suppliers.compose(new SchemaFunction(), Suppliers.ofInstance(schema.toString())); - this.metadata = metadata; - this.codec = new SerializableAvroCodecFactory(codec); - this.formatFunction = formatFunction; - this.datumWriterFactory = datumWriterFactory; - } - - @Override - public OutputT formatRecord(UserT record) { - return formatFunction.apply(record); - } - - @Override - public @Nullable Void getDestination(UserT element) { - return (Void) null; - } - - @Override - public @Nullable Void getDefaultDestination() { - return (Void) null; - } - - @Override - public FilenamePolicy getFilenamePolicy(Void destination) { - return filenamePolicy; - } - - @Override - public Schema getSchema(Void destination) { - return schema.get(); - } - - @Override - public Map getMetadata(Void destination) { - return metadata; - } - - @Override - public CodecFactory getCodec(Void destination) { - return codec.getCodec(); - } - - @Override - public AvroSink.@Nullable DatumWriterFactory getDatumWriterFactory(Void destination) { - return datumWriterFactory; - } - - @Override - public void populateDisplayData(DisplayData.Builder builder) { - filenamePolicy.populateDisplayData(builder); - builder.add(DisplayData.item("schema", schema.get().toString()).withLabel("Record Schema")); - builder.addIfNotDefault( - DisplayData.item("codec", codec.getCodec().toString()).withLabel("Avro Compression Codec"), - AvroIO.TypedWrite.DEFAULT_SERIALIZABLE_CODEC.toString()); - builder.include("Metadata", new Metadata()); - } -} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CountingSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CountingSource.java index 3e973cc2e1ada..9d30efb2f1136 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CountingSource.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CountingSource.java @@ -17,16 +17,22 @@ */ package org.apache.beam.sdk.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.util.List; import java.util.NoSuchElementException; import java.util.Objects; -import org.apache.beam.sdk.coders.AvroCoder; +import org.apache.beam.sdk.coders.BigEndianLongCoder; import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.coders.CustomCoder; import org.apache.beam.sdk.coders.DefaultCoder; +import org.apache.beam.sdk.coders.InstantCoder; import org.apache.beam.sdk.coders.VarLongCoder; import org.apache.beam.sdk.io.UnboundedSource.UnboundedReader; import org.apache.beam.sdk.metrics.Counter; @@ -34,7 +40,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; @@ -354,7 +360,7 @@ public UnboundedReader createReader(PipelineOptions options, CounterMark c @Override public Coder getCheckpointMarkCoder() { - return AvroCoder.of(CountingSource.CounterMark.class); + return new CounterMarkCoder(); } @Override @@ -485,7 +491,7 @@ public long getSplitBacklogBytes() { * The checkpoint for an unbounded {@link CountingSource} is simply the last value produced. The * associated source object encapsulates the information needed to produce the next value. */ - @DefaultCoder(AvroCoder.class) + @DefaultCoder(CounterMarkCoder.class) public static class CounterMark implements UnboundedSource.CheckpointMark { /** The last value emitted. */ private final long lastEmitted; @@ -519,4 +525,22 @@ private CounterMark() { @Override public void finalizeCheckpoint() throws IOException {} } + + /** A custom coder for {@code CounterMark}. */ + public static class CounterMarkCoder extends CustomCoder { + @Override + public void encode(CounterMark value, OutputStream outStream) throws IOException { + DataOutputStream stream = new DataOutputStream(outStream); + BigEndianLongCoder.of().encode(value.lastEmitted, stream); + InstantCoder.of().encode(value.startTime, stream); + } + + @Override + public CounterMark decode(InputStream inStream) throws IOException { + DataInputStream stream = new DataInputStream(inStream); + long lastEmitted = BigEndianLongCoder.of().decode(stream); + Instant startTime = InstantCoder.of().decode(stream); + return new CounterMark(lastEmitted, startTime); + } + } } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DefaultFilenamePolicy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DefaultFilenamePolicy.java index 5803f450aeaa4..77e667e88ee7a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DefaultFilenamePolicy.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DefaultFilenamePolicy.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; import java.io.IOException; import java.io.InputStream; @@ -43,9 +43,9 @@ import org.apache.beam.sdk.transforms.windowing.IntervalWindow; import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicAvroDestinations.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicAvroDestinations.java deleted file mode 100644 index 40c8657a01aee..0000000000000 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicAvroDestinations.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io; - -import java.util.Map; -import org.apache.avro.Schema; -import org.apache.avro.file.CodecFactory; -import org.apache.beam.sdk.io.FileBasedSink.DynamicDestinations; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; - -/** - * A specialization of {@link DynamicDestinations} for {@link AvroIO}. In addition to dynamic file - * destinations, this allows specifying other AVRO properties (schema, metadata, codec, datum - * writer) per destination. - * - * @deprecated Avro related classes are deprecated in module beam-sdks-java-core and - * will be eventually removed. Please, migrate to a new module - * beam-sdks-java-extensions-avro by importing - * org.apache.beam.sdk.extensions.avro.io.DynamicAvroDestinations instead of this one. - */ -@Deprecated -public abstract class DynamicAvroDestinations - extends DynamicDestinations { - /** Return an AVRO schema for a given destination. */ - public abstract Schema getSchema(DestinationT destination); - - /** Return AVRO file metadata for a given destination. */ - public Map getMetadata(DestinationT destination) { - return ImmutableMap.of(); - } - - /** Return an AVRO codec for a given destination. */ - public CodecFactory getCodec(DestinationT destination) { - return AvroIO.TypedWrite.DEFAULT_CODEC; - } - - /** - * Return a {@link AvroSink.DatumWriterFactory} for a given destination. If provided, it will be - * used to created {@link org.apache.avro.io.DatumWriter} instances as required. - */ - public AvroSink.@Nullable DatumWriterFactory getDatumWriterFactory( - DestinationT destinationT) { - return null; - } -} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicFileDestinations.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicFileDestinations.java index ab33c1e93c90c..5dee85b8dcf22 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicFileDestinations.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/DynamicFileDestinations.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.io.DefaultFilenamePolicy.Params; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java index 066a46a3c7ed9..4567187c9d83c 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java @@ -19,11 +19,11 @@ import static org.apache.beam.sdk.io.WriteFiles.UNKNOWN_SHARDNUM; import static org.apache.beam.sdk.values.TypeDescriptors.extractFromTypeParameters; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Verify.verifyNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Verify.verifyNotNull; import java.io.IOException; import java.io.InputStream; @@ -72,14 +72,14 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors.TypeVariableExtractor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java index 9d1c1bd80fa4a..43ab473dcc49b 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSource.java @@ -17,10 +17,10 @@ */ package org.apache.beam.sdk.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Verify.verify; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Verify.verify; import java.io.IOException; import java.nio.channels.ReadableByteChannel; @@ -37,7 +37,7 @@ import org.apache.beam.sdk.options.ValueProvider; import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider; import org.apache.beam.sdk.transforms.display.DisplayData; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; import org.slf4j.Logger; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileIO.java index 76db2a5592dcf..2d28279f90b64 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileIO.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileIO.java @@ -19,8 +19,8 @@ import static org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions.RESOLVE_FILE; import static org.apache.beam.sdk.transforms.Contextful.fn; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.io.IOException; @@ -72,10 +72,10 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; @@ -512,9 +512,18 @@ public MatchConfiguration withEmptyMatchTreatment(EmptyMatchTreatment treatment) * the watching frequency given by the {@code interval}. The pipeline will throw a {@code * RuntimeError} if timestamp extraction for the matched file has failed, suggesting the * timestamp metadata is not available with the IO connector. + * + *

    Matching continuously scales poorly, as it is stateful, and requires storing file ids in + * memory. In addition, because it is memory-only, if a pipeline is restarted, already processed + * files will be reprocessed. Consider an alternate technique, such as Pub/Sub Notifications + * when using GCS if possible. */ public MatchConfiguration continuously( Duration interval, TerminationCondition condition, boolean matchUpdatedFiles) { + LOG.warn( + "Matching Continuously is stateful, and can scale poorly. Consider using Pub/Sub " + + "Notifications (https://cloud.google.com/storage/docs/pubsub-notifications) if possible"); return toBuilder() .setWatchInterval(interval) .setWatchTerminationCondition(condition) diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystemUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystemUtils.java index f0c4724bf9dd8..1f108ca510e2a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystemUtils.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystemUtils.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; public class FileSystemUtils { diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java index cb899ce92625b..eb78bd0e27d23 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java @@ -17,9 +17,9 @@ */ package org.apache.beam.sdk.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Verify.verify; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Verify.verify; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.FileNotFoundException; @@ -50,17 +50,18 @@ import org.apache.beam.sdk.io.fs.ResourceId; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.TreeMultimap; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.TreeMultimap; /** Clients facing {@link FileSystem} utility. */ @SuppressWarnings({ @@ -74,6 +75,9 @@ public class FileSystems { Pattern.compile("(?[a-zA-Z][-a-zA-Z0-9+.]*):/.*"); private static final Pattern GLOB_PATTERN = Pattern.compile("[*?{}]"); + private static final AtomicReference> FILESYSTEM_REVISION = + new AtomicReference<>(); + private static final AtomicReference> SCHEME_TO_FILESYSTEM = new AtomicReference<>(ImmutableMap.of(DEFAULT_SCHEME, new LocalFileSystem())); @@ -529,13 +533,27 @@ static FileSystem getFileSystemInternal(String scheme) { @Internal public static void setDefaultPipelineOptions(PipelineOptions options) { checkNotNull(options, "options"); - Set registrars = - Sets.newTreeSet(ReflectHelpers.ObjectsClassComparator.INSTANCE); - registrars.addAll( - Lists.newArrayList( - ServiceLoader.load(FileSystemRegistrar.class, ReflectHelpers.findClassLoader()))); + long id = options.getOptionsId(); + int nextRevision = options.revision(); + + while (true) { + KV revision = FILESYSTEM_REVISION.get(); + // only update file systems if the pipeline changed or the options revision increased + if (revision != null && revision.getKey().equals(id) && revision.getValue() >= nextRevision) { + return; + } + + if (FILESYSTEM_REVISION.compareAndSet(revision, KV.of(id, nextRevision))) { + Set registrars = + Sets.newTreeSet(ReflectHelpers.ObjectsClassComparator.INSTANCE); + registrars.addAll( + Lists.newArrayList( + ServiceLoader.load(FileSystemRegistrar.class, ReflectHelpers.findClassLoader()))); - SCHEME_TO_FILESYSTEM.set(verifySchemesAreUnique(options, registrars)); + SCHEME_TO_FILESYSTEM.set(verifySchemesAreUnique(options, registrars)); + return; + } + } } @VisibleForTesting diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/GenerateSequence.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/GenerateSequence.java index 4a1425fc8e1bb..3cecb2ecfd076 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/GenerateSequence.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/GenerateSequence.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io; import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; import com.google.auto.value.AutoValue; @@ -31,8 +31,8 @@ import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.dataflow.qual.Pure; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystem.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystem.java index b13d4797c9dc7..1905becc8b854 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystem.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystem.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files.fileTraverser; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files.fileTraverser; import java.io.BufferedOutputStream; import java.io.File; @@ -48,10 +48,10 @@ import org.apache.beam.sdk.io.fs.MatchResult.Metadata; import org.apache.beam.sdk.io.fs.MatchResult.Status; import org.apache.beam.sdk.io.fs.MoveOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.commons.lang3.SystemUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -290,7 +290,8 @@ private MatchResult matchOne(String baseDir, String spec) { StreamSupport.stream(files.spliterator(), false) .filter( Predicates.and( - org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files.isFile(), + org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files + .isFile(), input -> matcher.matches(input.toPath())) ::apply) .collect(Collectors.toList()); diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystemRegistrar.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystemRegistrar.java index e7caec878e717..1ba666d7183a2 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystemRegistrar.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalFileSystemRegistrar.java @@ -19,7 +19,7 @@ import com.google.auto.service.AutoService; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; /** {@link AutoService} registrar for the {@link LocalFileSystem}. */ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalResourceId.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalResourceId.java index 3a109267577ed..76982cc9b9376 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalResourceId.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalResourceId.java @@ -19,8 +19,8 @@ import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.File; import java.nio.file.Path; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java index 84ba01a63a851..a05487731a575 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.util.ArrayList; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java index a66fa1b85d0a8..715e81c0f9301 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Read.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io; import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.io.IOException; @@ -64,10 +64,11 @@ import org.apache.beam.sdk.values.ValueWithRecordId; import org.apache.beam.sdk.values.ValueWithRecordId.StripIdsDoFn; import org.apache.beam.sdk.values.ValueWithRecordId.ValueWithRecordIdCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.RemovalListener; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.RemovalListener; import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.common.value.qual.ArrayLen; import org.checkerframework.dataflow.qual.Pure; @@ -500,7 +501,7 @@ public void setUp() throws Exception { removalNotification -> { if (removalNotification.wasEvicted()) { try { - removalNotification.getValue().close(); + Preconditions.checkNotNull(removalNotification.getValue()).close(); } catch (IOException e) { LOG.warn("Failed to close UnboundedReader.", e); } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/SerializableAvroCodecFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/SerializableAvroCodecFactory.java deleted file mode 100644 index b7b5da72ba480..0000000000000 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/SerializableAvroCodecFactory.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io; - -import static org.apache.avro.file.DataFileConstants.BZIP2_CODEC; -import static org.apache.avro.file.DataFileConstants.DEFLATE_CODEC; -import static org.apache.avro.file.DataFileConstants.NULL_CODEC; -import static org.apache.avro.file.DataFileConstants.SNAPPY_CODEC; -import static org.apache.avro.file.DataFileConstants.XZ_CODEC; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; - -import java.io.Externalizable; -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; -import java.util.Arrays; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import org.apache.avro.file.CodecFactory; -import org.checkerframework.checker.nullness.qual.Nullable; - -/** - * A wrapper that allows {@link org.apache.avro.file.CodecFactory}s to be serialized using Java's - * standard serialization mechanisms. - * - * @deprecated Avro related classes are deprecated in module beam-sdks-java-core and - * will be eventually removed. Please, migrate to a new module - * beam-sdks-java-extensions-avro by importing - * org.apache.beam.sdk.extensions.avro.io.SerializableAvroCodecFactory instead of this - * one. - */ -@SuppressWarnings({ - "nullness" // TODO(https://github.com/apache/beam/issues/20497) -}) -@Deprecated -class SerializableAvroCodecFactory implements Externalizable { - private static final long serialVersionUID = 7445324844109564303L; - private static final List noOptAvroCodecs = - Arrays.asList(NULL_CODEC, SNAPPY_CODEC, BZIP2_CODEC); - private static final Pattern deflatePattern = Pattern.compile(DEFLATE_CODEC + "-(?-?\\d)"); - private static final Pattern xzPattern = Pattern.compile(XZ_CODEC + "-(?\\d)"); - - private @Nullable CodecFactory codecFactory; - - // For java.io.Externalizable - public SerializableAvroCodecFactory() {} - - public SerializableAvroCodecFactory(CodecFactory codecFactory) { - checkNotNull(codecFactory, "Codec can't be null"); - checkState(checkIsSupportedCodec(codecFactory), "%s is not supported", codecFactory); - this.codecFactory = codecFactory; - } - - private boolean checkIsSupportedCodec(CodecFactory codecFactory) { - final String codecStr = codecFactory.toString(); - return noOptAvroCodecs.contains(codecStr) - || deflatePattern.matcher(codecStr).matches() - || xzPattern.matcher(codecStr).matches(); - } - - @Override - public void writeExternal(ObjectOutput out) throws IOException { - out.writeUTF(codecFactory.toString()); - } - - @Override - public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { - final String codecStr = in.readUTF(); - - switch (codecStr) { - case NULL_CODEC: - case SNAPPY_CODEC: - case BZIP2_CODEC: - codecFactory = CodecFactory.fromString(codecStr); - return; - } - - Matcher deflateMatcher = deflatePattern.matcher(codecStr); - if (deflateMatcher.find()) { - codecFactory = CodecFactory.deflateCodec(Integer.parseInt(deflateMatcher.group("level"))); - return; - } - - Matcher xzMatcher = xzPattern.matcher(codecStr); - if (xzMatcher.find()) { - codecFactory = CodecFactory.xzCodec(Integer.parseInt(xzMatcher.group("level"))); - return; - } - - throw new IllegalStateException(codecStr + " is not supported"); - } - - public CodecFactory getCodec() { - return codecFactory; - } - - @Override - public String toString() { - checkNotNull(codecFactory, "Inner CodecFactory is null, please use non default constructor"); - return codecFactory.toString(); - } -} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Source.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Source.java index 2a9c997b0948c..51a65e2db8306 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Source.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Source.java @@ -23,7 +23,7 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.transforms.display.HasDisplayData; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.joda.time.Instant; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java index b1c770b0f5fbb..dc76d90165773 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.io.IOException; @@ -42,9 +42,9 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.HashFunction; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java index 1890f3fb8d46d..33beff23b311e 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java @@ -18,9 +18,9 @@ package org.apache.beam.sdk.io; import static org.apache.beam.sdk.io.FileIO.ReadMatches.DirectoryTreatment; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.apache.commons.compress.utils.CharsetNames.UTF_8; import com.google.auto.value.AutoValue; @@ -54,10 +54,10 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextSource.java index e287b5f338f13..bef30dffa8ac7 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextSource.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextSource.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -33,7 +33,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.ValueProvider; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java index d5a8d1b14c5e1..91d6082eede44 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import java.io.IOException; @@ -82,13 +82,13 @@ import org.apache.beam.sdk.values.ShardedKey; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFilesResult.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFilesResult.java index 7e5e6be1250b3..cf11a8112a0b4 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFilesResult.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFilesResult.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** The result of a {@link WriteFiles} transform. */ public class WriteFilesResult implements POutput { diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceIdTester.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceIdTester.java index 837987b018446..b80b5640856e5 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceIdTester.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceIdTester.java @@ -19,7 +19,7 @@ import static org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions.RESOLVE_DIRECTORY; import static org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions.RESOLVE_FILE; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/package-info.java index df4efa1b603be..3f2433c1f264b 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/package-info.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/package-info.java @@ -24,7 +24,7 @@ * *

    {@code
      * PCollection inputData = pipeline.apply(
    - *     BigQueryIO.readTableRows().from("clouddataflow-readonly:samples.weather_stations"));
    + *     BigQueryIO.readTableRows().from("apache-beam-testing.samples.weather_stations"));
      * }
    * * and {@code Write} transforms that persist PCollections to external storage: diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKey.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKey.java index 9e5e3df0157a4..5c3473c39001a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKey.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKey.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.range; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.Serializable; import java.nio.ByteBuffer; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRange.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRange.java index 217cd0af99d46..be808dfea9a5a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRange.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRange.java @@ -17,10 +17,10 @@ */ package org.apache.beam.sdk.io.range; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Verify.verify; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Verify.verify; import java.io.Serializable; import java.math.BigDecimal; @@ -29,8 +29,8 @@ import java.util.List; import java.util.Objects; import org.apache.beam.sdk.transforms.splittabledofn.HasDefaultTracker; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java index ddda1b2fa7528..4cf28fee27ee8 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.range; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.toStringHelper; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.toStringHelper; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import org.apache.beam.sdk.io.BoundedSource.BoundedReader; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/OffsetRange.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/OffsetRange.java index a91dfed46ee4e..f8437d743c215 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/OffsetRange.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/OffsetRange.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.range; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.io.InputStream; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/OffsetRangeTracker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/OffsetRangeTracker.java index 22dbbd43e6558..21e325276eada 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/OffsetRangeTracker.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/OffsetRangeTracker.java @@ -17,10 +17,10 @@ */ package org.apache.beam.sdk.io.range; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import org.apache.beam.sdk.io.BoundedSource.BoundedReader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricFiltering.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricFiltering.java index d88e694576f01..f40baf2295906 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricFiltering.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricFiltering.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.metrics; import java.util.Set; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricName.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricName.java index f00869d5f2115..f452ebeefc4c9 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricName.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricName.java @@ -17,11 +17,11 @@ */ package org.apache.beam.sdk.metrics; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.io.Serializable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; /** * The name of a metric consists of a {@link #getNamespace} and a {@link #getName}. The {@link diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricNameFilter.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricNameFilter.java index 9e09c29024d64..704df17fc701e 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricNameFilter.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricNameFilter.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.metrics; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricQueryResults.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricQueryResults.java index 7156ad7b53e1f..86b1c1092824f 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricQueryResults.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricQueryResults.java @@ -19,7 +19,7 @@ import com.google.auto.value.AutoValue; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * The results of a query for metrics. Allows accessing all of the metrics that matched the filter. diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java index e93f8677b814d..f48b9195c37cb 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsContainer.java @@ -33,6 +33,14 @@ public interface MetricsContainer extends Serializable { */ Counter getCounter(MetricName metricName); + /** + * Return the {@link Counter} that should be used for implementing the given per-worker {@code metricName) + * in this container. + */ + default Counter getPerWorkerCounter(MetricName metricName) { + return NoOpCounter.getInstance(); + } + /** * Return the {@link Distribution} that should be used for implementing the given {@code * metricName} in this container. @@ -52,6 +60,14 @@ public interface MetricsContainer extends Serializable { default Histogram getHistogram(MetricName metricName, HistogramData.BucketType bucketType) { throw new RuntimeException("Histogram metric is not supported yet."); } + /** + * Return the {@link Histogram} that should be used for implementing the given per-worker {@code + * metricName} in this container. + */ + default Histogram getPerWorkerHistogram( + MetricName metricName, HistogramData.BucketType bucketType) { + return NoOpHistogram.getInstance(); + } /** Return the cumulative values for any metrics in this container as MonitoringInfos. */ default Iterable getMonitoringInfos() { diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsFilter.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsFilter.java index 1de85f8f3fea9..06616154689b6 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsFilter.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/MetricsFilter.java @@ -19,7 +19,7 @@ import com.google.auto.value.AutoValue; import java.util.Set; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; /** Simple POJO representing a filter for querying metrics. */ @AutoValue diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/NoOpCounter.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/NoOpCounter.java new file mode 100644 index 0000000000000..ab4fa685f9c20 --- /dev/null +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/NoOpCounter.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.metrics; + +/** + * A no-op implementation of Counter. This class exists to provide a default if an implementation of + * MetricsContainer does not override a Counter getter. + */ +public class NoOpCounter implements Counter { + + private static final NoOpCounter singleton = new NoOpCounter(); + private static final MetricName name = MetricName.named(NoOpCounter.class, "singleton"); + + private NoOpCounter() {} + + @Override + public void inc() {} + + @Override + public void inc(long n) {} + + @Override + public void dec() {} + + @Override + public void dec(long n) {} + + @Override + public MetricName getName() { + return name; + } + + public static NoOpCounter getInstance() { + return singleton; + } +} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/NoOpHistogram.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/NoOpHistogram.java new file mode 100644 index 0000000000000..a088223ffe2b8 --- /dev/null +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/NoOpHistogram.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.metrics; + +/** + * A no-op implementation of Histogram. This class exists to provide a default if an implementation + * of MetricsContainer does not override a Histogram getter. + */ +public class NoOpHistogram implements Histogram { + + private static final NoOpHistogram singleton = new NoOpHistogram(); + private static final MetricName name = MetricName.named(NoOpHistogram.class, "singleton"); + + private NoOpHistogram() {} + + @Override + public void update(double value) {} + + @Override + public MetricName getName() { + return name; + } + + public static NoOpHistogram getInstance() { + return singleton; + } +} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/SourceMetrics.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/SourceMetrics.java index 4e6ee0d4fd3bb..e9677626f88be 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/SourceMetrics.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/SourceMetrics.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.metrics; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; /** Standard {@link org.apache.beam.sdk.io.Source} Metrics. */ public class SourceMetrics { diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/DefaultPipelineOptionsRegistrar.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/DefaultPipelineOptionsRegistrar.java index 62b54bab251ac..7fc8a829482da 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/DefaultPipelineOptionsRegistrar.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/DefaultPipelineOptionsRegistrar.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.options; import com.google.auto.service.AutoService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * A {@link PipelineOptionsRegistrar} containing the {@link PipelineOptions} subclasses available by diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ExperimentalOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ExperimentalOptions.java index 32f9e84c8286d..c0e2e1dcb48f6 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ExperimentalOptions.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ExperimentalOptions.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.options; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ManualDockerEnvironmentOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ManualDockerEnvironmentOptions.java index 94edc83c5700b..58350b0a97420 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ManualDockerEnvironmentOptions.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ManualDockerEnvironmentOptions.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.options; import com.google.auto.service.AutoService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Pipeline options to tune DockerEnvironment. */ @Hidden diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/MemoryMonitorOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/MemoryMonitorOptions.java new file mode 100644 index 0000000000000..fc9bd3bc80039 --- /dev/null +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/MemoryMonitorOptions.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.options; + +/** Options that are used to control the Memory Monitor. */ +@Description("Options that are used to control the Memory Monitor.") +public interface MemoryMonitorOptions extends PipelineOptions { + + /** + * The GC thrashing threshold percentage. A given period of time is considered "thrashing" if this + * percentage of CPU time is spent in garbage collection. Harness will force fail tasks after + * sustained periods of thrashing. + * + *

    If {@literal 100} is given as the value, MemoryMonitor will be disabled. + */ + @Description( + "The GC thrashing threshold percentage. A given period of time is considered \"thrashing\" if this " + + "percentage of CPU time is spent in garbage collection. Dataflow will force fail tasks after " + + "sustained periods of thrashing.") + @Default.Double(50.0) + Double getGCThrashingPercentagePerPeriod(); + + void setGCThrashingPercentagePerPeriod(Double value); +} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java index 0cefaba81e11f..2eba8c6ef68dd 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java @@ -36,7 +36,7 @@ import org.apache.beam.sdk.transforms.display.HasDisplayData; import org.apache.beam.sdk.util.ReleaseInfo; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.joda.time.DateTimeUtils; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; @@ -357,6 +357,12 @@ public String create(PipelineOptions options) { */ Map> outputRuntimeOptions(); + /** + * A monotonically increasing revision number of this {@link PipelineOptions} object that can be + * used to detect changes. + */ + int revision(); + /** * Provides a process wide unique ID for this {@link PipelineOptions} object, assigned at graph * construction time. diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java index 4d3f993257aad..555f50bc823b4 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsFactory.java @@ -18,8 +18,8 @@ package org.apache.beam.sdk.options; import static java.util.Locale.ROOT; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.core.JsonParseException; @@ -90,28 +90,28 @@ import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.util.StringUtils; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.CaseFormat; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicate; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSortedSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.RowSortedTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.SortedSetMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.TreeBasedTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.TreeMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CaseFormat; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicate; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSortedSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.RowSortedTable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.SortedSetMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.TreeBasedTable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.TreeMultimap; import org.checkerframework.checker.nullness.qual.KeyFor; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; @@ -1269,6 +1269,7 @@ private static void validateMethodsAreEitherBeanMethodOrKnownMethod( try { knownMethods.add(iface.getMethod("as", Class.class)); knownMethods.add(iface.getMethod("outputRuntimeOptions")); + knownMethods.add(iface.getMethod("revision")); knownMethods.add(iface.getMethod("populateDisplayData", DisplayData.Builder.class)); } catch (NoSuchMethodException | SecurityException e) { throw new RuntimeException(e); diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsReflector.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsReflector.java index 5a9e9db233a67..64722475402ff 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsReflector.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsReflector.java @@ -22,9 +22,9 @@ import java.util.Map; import java.util.Set; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; /** Utilities to reflect over {@link PipelineOptions}. */ class PipelineOptionsReflector { diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsValidator.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsValidator.java index f414b48316c5a..7635c35da1cff 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsValidator.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionsValidator.java @@ -17,18 +17,18 @@ */ package org.apache.beam.sdk.options; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.util.Collection; import org.apache.beam.sdk.options.Validation.Required; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Collections2; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.SortedSetMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.TreeMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Collections2; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.SortedSetMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.TreeMultimap; /** Validates that the {@link PipelineOptions} conforms to all the {@link Validation} criteria. */ @SuppressWarnings({ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PortablePipelineOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PortablePipelineOptions.java index 6bfab14ed0e43..ec82aebbef665 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PortablePipelineOptions.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PortablePipelineOptions.java @@ -123,4 +123,21 @@ static String getEnvironmentOption( return ""; } + + /** + * If {@literal true} and PipelineOption tempLocation is set, save a heap dump before shutting + * down the JVM due to GC thrashing or out of memory. The heap will be dumped to local disk and + * then uploaded to the tempLocation. + * + *

    CAUTION: Heap dumps can take up more disk than the JVM memory. Ensure the local disk is + * configured to have sufficient free space before enabling this option. + */ + @Description( + "If {@literal true} and PipelineOption tempLocation is set, save a heap dump before shutting" + + " down the JVM due to GC thrashing or out of memory. The heap will be dumped to local" + + " disk and then uploaded to the tempLocation.") + @Default.Boolean(false) + boolean getEnableHeapDumps(); + + void setEnableHeapDumps(boolean enableHeapDumps); } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java index 77324eb7f18f1..7a9cc8568ee04 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.options; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.core.JsonGenerator; @@ -49,11 +49,13 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.concurrent.ThreadSafe; import org.apache.beam.sdk.options.PipelineOptionsFactory.AnnotationPredicates; import org.apache.beam.sdk.options.PipelineOptionsFactory.Registration; @@ -63,16 +65,16 @@ import org.apache.beam.sdk.transforms.display.HasDisplayData; import org.apache.beam.sdk.util.InstanceBuilder; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Defaults; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableClassToInstanceMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Defaults; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableClassToInstanceMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; /** @@ -98,6 +100,8 @@ class ProxyInvocationHandler implements InvocationHandler, Serializable { */ private final int hashCode = ThreadLocalRandom.current().nextInt(); + private final AtomicInteger revision; + private static final class ComputedProperties { final ImmutableClassToInstanceMap interfaceToProxyCache; final ImmutableMap gettersToPropertyNames; @@ -158,7 +162,7 @@ ComputedProperties updated( private final ImmutableMap jsonOptions; ProxyInvocationHandler(Map options) { - this(bindOptions(options), Maps.newHashMap()); + this(bindOptions(options), Maps.newHashMap(), 0); } private static Map bindOptions(Map inputOptions) { @@ -171,9 +175,10 @@ private static Map bindOptions(Map inputOpti } private ProxyInvocationHandler( - Map options, Map jsonOptions) { + Map options, Map jsonOptions, int revision) { this.options = new ConcurrentHashMap<>(options); this.jsonOptions = ImmutableMap.copyOf(jsonOptions); + this.revision = new AtomicInteger(revision); this.computedProperties = new ComputedProperties( ImmutableClassToInstanceMap.of(), @@ -193,6 +198,8 @@ public Object invoke(Object proxy, Method method, Object[] args) { return hashCode(); } else if (args == null && "outputRuntimeOptions".equals(method.getName())) { return outputRuntimeOptions((PipelineOptions) proxy); + } else if (args == null && "revision".equals(method.getName())) { + return revision.get(); } else if (args != null && "as".equals(method.getName()) && args[0] instanceof Class) { @SuppressWarnings("unchecked") Class clazz = (Class) args[0]; @@ -222,9 +229,13 @@ public Object invoke(Object proxy, Method method, Object[] args) { } return options.get(propertyName).getValue(); } else if (properties.settersToPropertyNames.containsKey(methodName)) { - options.put( - properties.settersToPropertyNames.get(methodName), - BoundValue.fromExplicitOption(args[0])); + BoundValue prev = + options.put( + properties.settersToPropertyNames.get(methodName), + BoundValue.fromExplicitOption(args[0])); + if (prev == null ? args[0] != null : !Objects.equals(args[0], prev.getValue())) { + revision.incrementAndGet(); + } return Void.TYPE; } throw new RuntimeException( @@ -781,6 +792,8 @@ public void serialize(PipelineOptions value, JsonGenerator jgen, SerializerProvi jgen.writeFieldName("display_data"); jgen.writeObject(serializedDisplayData); + + jgen.writeNumberField("revision", handler.revision.get()); jgen.writeEndObject(); } @@ -879,9 +892,9 @@ public PipelineOptions deserialize(JsonParser jp, DeserializationContext ctxt) fields.put(field.getKey(), field.getValue()); } } - + int revision = objectNode.hasNonNull("revision") ? objectNode.get("revision").asInt() : 0; PipelineOptions options = - new ProxyInvocationHandler(Maps.newHashMap(), fields).as(PipelineOptions.class); + new ProxyInvocationHandler(Maps.newHashMap(), fields, revision).as(PipelineOptions.class); ValueProvider.RuntimeValueProvider.setRuntimeOptions(options); return options; } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/RemoteEnvironmentOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/RemoteEnvironmentOptions.java index a1c2f7aa72e7f..cf53d7d94b4d1 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/RemoteEnvironmentOptions.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/RemoteEnvironmentOptions.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.options; import com.google.auto.service.AutoService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Options that are used to control configuration of the remote environment. */ @Hidden diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/SdkHarnessOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/SdkHarnessOptions.java index 70ba18e614c01..6e5843f533dbb 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/SdkHarnessOptions.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/SdkHarnessOptions.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.options; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.fasterxml.jackson.annotation.JsonCreator; import java.util.ArrayList; @@ -29,13 +29,13 @@ import java.util.logging.LogManager; import java.util.logging.Logger; import org.apache.beam.sdk.util.InstanceBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.index.qual.NonNegative; /** Options that are used to control configuration of the SDK harness. */ @Description("Options that are used to control configuration of the SDK harness.") -public interface SdkHarnessOptions extends PipelineOptions { +public interface SdkHarnessOptions extends PipelineOptions, MemoryMonitorOptions { /** The set of log levels that can be used in the SDK harness. */ enum LogLevel { /** Special level used to turn off logging. */ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProvider.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProvider.java index 5da4dced84773..54b31a252b99c 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProvider.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProvider.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.options; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; @@ -43,7 +43,7 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java index 16178607ce2ad..e88dab7eb2a76 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProviders.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.options; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java index 16eb06909dcb7..77b22d25d23d8 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java @@ -17,9 +17,9 @@ */ package org.apache.beam.sdk.runners; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.ArrayList; import java.util.Collections; @@ -42,9 +42,9 @@ import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.PValues; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/AutoValueSchema.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/AutoValueSchema.java index 2c4ea2f945974..abd9bc46bd466 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/AutoValueSchema.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/AutoValueSchema.java @@ -29,9 +29,9 @@ import org.apache.beam.sdk.schemas.utils.JavaBeanUtils; import org.apache.beam.sdk.schemas.utils.ReflectUtils; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; /** A {@link SchemaProvider} for AutoValue classes. */ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/AvroRecordSchema.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/AvroRecordSchema.java deleted file mode 100644 index 19027cd4527f1..0000000000000 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/AvroRecordSchema.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.schemas; - -import static org.apache.beam.sdk.schemas.utils.AvroUtils.toBeamSchema; - -import java.util.List; -import org.apache.avro.reflect.ReflectData; -import org.apache.beam.sdk.schemas.utils.AvroUtils; -import org.apache.beam.sdk.values.TypeDescriptor; - -/** - * A {@link SchemaProvider} for AVRO generated SpecificRecords and POJOs. - * - *

    This provider infers a schema from generated SpecificRecord objects, and creates schemas and - * rows that bind to the appropriate fields. This provider also infers schemas from Java POJO - * objects, creating a schema that matches that inferred by the AVRO libraries. - * - * @deprecated Avro related classes are deprecated in module beam-sdks-java-core and - * will be eventually removed. Please, migrate to a new module - * beam-sdks-java-extensions-avro by importing - * org.apache.beam.sdk.extensions.avro.schemas.AvroRecordSchema instead of this one. - */ -@SuppressWarnings({ - "rawtypes" // TODO(https://github.com/apache/beam/issues/20447) -}) -@Deprecated -public class AvroRecordSchema extends GetterBasedSchemaProvider { - @Override - public Schema schemaFor(TypeDescriptor typeDescriptor) { - return toBeamSchema(ReflectData.get().getSchema(typeDescriptor.getRawType())); - } - - @Override - public List fieldValueGetters(Class targetClass, Schema schema) { - return AvroUtils.getGetters(targetClass, schema); - } - - @Override - public List fieldValueTypeInformations( - Class targetClass, Schema schema) { - return AvroUtils.getFieldTypes(targetClass, schema); - } - - @Override - public SchemaUserTypeCreator schemaTypeCreator(Class targetClass, Schema schema) { - return AvroUtils.getCreator(targetClass, schema); - } -} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/FieldAccessDescriptor.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/FieldAccessDescriptor.java index 8405dc8d600f2..b465212c61de1 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/FieldAccessDescriptor.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/FieldAccessDescriptor.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.schemas; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoOneOf; import com.google.auto.value.AutoValue; @@ -40,14 +40,14 @@ import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.Schema.TypeName; import org.apache.beam.sdk.schemas.parser.FieldAccessDescriptorParser; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.LinkedListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.LinkedListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/FieldTypeDescriptors.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/FieldTypeDescriptors.java index 1238cb0c304d9..61a235ea7aa6e 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/FieldTypeDescriptors.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/FieldTypeDescriptors.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.schemas; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.lang.reflect.ParameterizedType; import java.util.Collection; @@ -27,8 +27,8 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableBiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableBiMap; import org.joda.time.Instant; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/FieldValueTypeInformation.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/FieldValueTypeInformation.java index b11fd8cbc52ca..750709192c08b 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/FieldValueTypeInformation.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/FieldValueTypeInformation.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.schemas.logicaltypes.OneOfType; import org.apache.beam.sdk.schemas.utils.ReflectUtils; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.CaseFormat; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CaseFormat; import org.checkerframework.checker.nullness.qual.Nullable; /** Represents type information for a Java type that will be used to infer a Schema type. */ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/FromRowUsingCreator.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/FromRowUsingCreator.java index 06bb20897a179..53c098599c367 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/FromRowUsingCreator.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/FromRowUsingCreator.java @@ -17,10 +17,11 @@ */ package org.apache.beam.sdk.schemas; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; -import java.lang.reflect.Type; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.Collection; import java.util.List; import java.util.Map; @@ -32,11 +33,12 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.RowWithGetters; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Collections2; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Collections2; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.Nullable; /** Function to convert a {@link Row} to a user type using a creator factory. */ @@ -44,188 +46,220 @@ "nullness", // TODO(https://github.com/apache/beam/issues/20497) "rawtypes" }) -class FromRowUsingCreator implements SerializableFunction { +class FromRowUsingCreator implements SerializableFunction, Function { private final Class clazz; private final GetterBasedSchemaProvider schemaProvider; private final Factory schemaTypeCreatorFactory; - private final Factory> fieldValueTypeInformationFactory; + + @SuppressFBWarnings("SE_TRANSIENT_FIELD_NOT_RESTORED") + private transient @MonotonicNonNull Function[] fieldConverters; public FromRowUsingCreator(Class clazz, GetterBasedSchemaProvider schemaProvider) { + this(clazz, schemaProvider, new CachingFactory<>(schemaProvider::schemaTypeCreator), null); + } + + private FromRowUsingCreator( + Class clazz, + GetterBasedSchemaProvider schemaProvider, + Factory schemaTypeCreatorFactory, + @Nullable Function[] fieldConverters) { this.clazz = clazz; this.schemaProvider = schemaProvider; - this.schemaTypeCreatorFactory = new CachingFactory<>(schemaProvider::schemaTypeCreator); - this.fieldValueTypeInformationFactory = - new CachingFactory<>(schemaProvider::fieldValueTypeInformations); + this.schemaTypeCreatorFactory = schemaTypeCreatorFactory; + this.fieldConverters = fieldConverters; } @Override - public T apply(Row row) { - return fromRow(row, clazz, fieldValueTypeInformationFactory); - } - @SuppressWarnings("unchecked") - public ValueT fromRow( - Row row, Class clazz, Factory> typeFactory) { + public T apply(Row row) { + if (row == null) { + return null; + } if (row instanceof RowWithGetters) { Object target = ((RowWithGetters) row).getGetterTarget(); if (target.getClass().equals(clazz)) { // Efficient path: simply extract the underlying object instead of creating a new one. - return (ValueT) target; + return (T) target; } } + if (fieldConverters == null) { + initFieldConverters(row.getSchema()); + } + checkState(fieldConverters.length == row.getFieldCount(), "Unexpected field count"); Object[] params = new Object[row.getFieldCount()]; - Schema schema = row.getSchema(); - List typeInformations = typeFactory.create(clazz, schema); + for (int i = 0; i < row.getFieldCount(); ++i) { + params[i] = fieldConverters[i].apply(row.getValue(i)); + } + SchemaUserTypeCreator creator = schemaTypeCreatorFactory.create(clazz, row.getSchema()); + return (T) creator.create(params); + } + + private synchronized void initFieldConverters(Schema schema) { + if (fieldConverters == null) { + CachingFactory> typeFactory = + new CachingFactory<>(schemaProvider::fieldValueTypeInformations); + fieldConverters = fieldConverters(clazz, schema, typeFactory); + } + } + + private Function[] fieldConverters( + Class clazz, Schema schema, Factory> typeFactory) { + List typeInfos = typeFactory.create(clazz, schema); checkState( - typeInformations.size() == row.getFieldCount(), + typeInfos.size() == schema.getFieldCount(), "Did not have a matching number of type informations and fields."); - - for (int i = 0; i < row.getFieldCount(); ++i) { - FieldType type = schema.getField(i).getType(); - FieldValueTypeInformation typeInformation = checkNotNull(typeInformations.get(i)); - params[i] = - fromValue( - type, row.getValue(i), typeInformation.getRawType(), typeInformation, typeFactory); + Function[] converters = new Function[schema.getFieldCount()]; + for (int i = 0; i < converters.length; i++) { + converters[i] = fieldConverter(schema.getField(i).getType(), typeInfos.get(i), typeFactory); } + return converters; + } - SchemaUserTypeCreator creator = schemaTypeCreatorFactory.create(clazz, schema); - return (ValueT) creator.create(params); + private static boolean needsConversion(FieldType type) { + TypeName typeName = type.getTypeName(); + return typeName.equals(TypeName.ROW) + || typeName.isLogicalType() + || ((typeName.equals(TypeName.ARRAY) || typeName.equals(TypeName.ITERABLE)) + && needsConversion(type.getCollectionElementType())) + || (typeName.equals(TypeName.MAP) + && (needsConversion(type.getMapKeyType()) || needsConversion(type.getMapValueType()))); } - @SuppressWarnings("unchecked") - private @Nullable ValueT fromValue( + private Function fieldConverter( FieldType type, - ValueT value, - Type fieldType, - FieldValueTypeInformation fieldValueTypeInformation, + FieldValueTypeInformation typeInfo, Factory> typeFactory) { - FieldValueTypeInformation elementType = fieldValueTypeInformation.getElementType(); - FieldValueTypeInformation keyType = fieldValueTypeInformation.getMapKeyType(); - FieldValueTypeInformation valueType = fieldValueTypeInformation.getMapValueType(); - if (value == null) { - return null; - } - if (TypeName.ROW.equals(type.getTypeName())) { - return (ValueT) fromRow((Row) value, (Class) fieldType, typeFactory); + if (!needsConversion(type)) { + return FieldConverter.IDENTITY; + } else if (TypeName.ROW.equals(type.getTypeName())) { + Function[] converters = + fieldConverters(typeInfo.getRawType(), type.getRowSchema(), typeFactory); + return new FromRowUsingCreator( + typeInfo.getRawType(), schemaProvider, schemaTypeCreatorFactory, converters); } else if (TypeName.ARRAY.equals(type.getTypeName())) { - return (ValueT) - fromCollectionValue( - type.getCollectionElementType(), (Collection) value, elementType, typeFactory); + return new ConvertCollection( + fieldConverter(type.getCollectionElementType(), typeInfo.getElementType(), typeFactory)); } else if (TypeName.ITERABLE.equals(type.getTypeName())) { - return (ValueT) - fromIterableValue( - type.getCollectionElementType(), (Iterable) value, elementType, typeFactory); - } - if (TypeName.MAP.equals(type.getTypeName())) { - return (ValueT) - fromMapValue( - type.getMapKeyType(), - type.getMapValueType(), - (Map) value, - keyType, - valueType, - typeFactory); - } else { - if (type.isLogicalType(OneOfType.IDENTIFIER)) { - OneOfType oneOfType = type.getLogicalType(OneOfType.class); - EnumerationType oneOfEnum = oneOfType.getCaseEnumType(); - OneOfType.Value oneOfValue = (OneOfType.Value) value; - FieldValueTypeInformation oneOfFieldValueTypeInformation = - checkNotNull( - fieldValueTypeInformation - .getOneOfTypes() - .get(oneOfEnum.toString(oneOfValue.getCaseType()))); - Object fromValue = - fromValue( - oneOfType.getFieldType(oneOfValue), - oneOfValue.getValue(), - oneOfFieldValueTypeInformation.getRawType(), - oneOfFieldValueTypeInformation, - typeFactory); - return (ValueT) oneOfType.createValue(oneOfValue.getCaseType(), fromValue); - } else if (type.getTypeName().isLogicalType()) { - Schema.LogicalType logicalType = - (Schema.LogicalType) type.getLogicalType(); - return logicalType.toBaseType(value); + return new ConvertIterable( + fieldConverter(type.getCollectionElementType(), typeInfo.getElementType(), typeFactory)); + } else if (TypeName.MAP.equals(type.getTypeName())) { + return new ConvertMap( + fieldConverter(type.getMapKeyType(), typeInfo.getMapKeyType(), typeFactory), + fieldConverter(type.getMapValueType(), typeInfo.getMapValueType(), typeFactory)); + } else if (type.isLogicalType(OneOfType.IDENTIFIER)) { + OneOfType oneOfType = type.getLogicalType(OneOfType.class); + Schema schema = oneOfType.getOneOfSchema(); + Map readers = Maps.newHashMapWithExpectedSize(schema.getFieldCount()); + oneOfType + .getCaseEnumType() + .getValuesMap() + .forEach( + (name, id) -> { + FieldType caseType = schema.getField(name).getType(); + FieldValueTypeInformation caseTypeInfo = + checkNotNull(typeInfo.getOneOfTypes().get(name)); + readers.put(id, fieldConverter(caseType, caseTypeInfo, typeFactory)); + }); + return new ConvertOneOf(oneOfType, readers); + } else if (type.getTypeName().isLogicalType()) { + return new ConvertLogicalType<>(type.getLogicalType()); + } + return FieldConverter.IDENTITY; + } + + private interface FieldConverter + extends SerializableFunction, Function { + Function IDENTITY = v -> v; + + ValueT convert(FieldT field); + + @Override + default @Nullable ValueT apply(@Nullable FieldT fieldValue) { + return fieldValue == null ? null : convert(fieldValue); + } + } + + private static class ConvertCollection implements FieldConverter { + final Function converter; + + ConvertCollection(Function converter) { + this.converter = converter; + } + + @Override + public Collection convert(Collection collection) { + if (collection instanceof List) { + // For performance reasons if the input is a list, make sure that we produce a list. + // Otherwise Row unwrapping is forced to physically copy the collection into a new List + // object. + return Lists.transform((List) collection, converter); + } else { + return Collections2.transform(collection, converter); } - return value; } } - private static Collection transformCollection( - Collection collection, Function function) { - if (collection instanceof List) { - // For performance reasons if the input is a list, make sure that we produce a list. Otherwise - // Row unwrapping - // is forced to physically copy the collection into a new List object. - return Lists.transform((List) collection, function); - } else { - return Collections2.transform(collection, function); + private static class ConvertIterable implements FieldConverter { + final Function converter; + + ConvertIterable(Function converter) { + this.converter = converter; + } + + @Override + public Iterable convert(Iterable iterable) { + return Iterables.transform(iterable, converter); } } - @SuppressWarnings("unchecked") - private Collection fromCollectionValue( - FieldType elementType, - Collection rowCollection, - FieldValueTypeInformation elementTypeInformation, - Factory> typeFactory) { - return transformCollection( - rowCollection, - element -> - fromValue( - elementType, - element, - elementTypeInformation.getType().getType(), - elementTypeInformation, - typeFactory)); + private static class ConvertMap implements FieldConverter { + final Function keyConverter, valueConverter; + + ConvertMap(Function keyConverter, Function valueConverter) { + this.keyConverter = keyConverter; + this.valueConverter = valueConverter; + } + + @Override + public Map convert(Map field) { + Map result = Maps.newHashMapWithExpectedSize(field.size()); + field.forEach((k, v) -> result.put(keyConverter.apply(k), valueConverter.apply(v))); + return result; + } } - @SuppressWarnings("unchecked") - private Iterable fromIterableValue( - FieldType elementType, - Iterable rowIterable, - FieldValueTypeInformation elementTypeInformation, - Factory> typeFactory) { - return Iterables.transform( - rowIterable, - element -> - fromValue( - elementType, - element, - elementTypeInformation.getType().getType(), - elementTypeInformation, - typeFactory)); + private static class ConvertOneOf implements FieldConverter { + final OneOfType oneOfType; + final Map converters; + + ConvertOneOf(OneOfType oneOfType, Map converters) { + this.oneOfType = oneOfType; + this.converters = converters; + } + + @Override + public OneOfType.Value convert(OneOfType.Value field) { + EnumerationType.Value caseType = field.getCaseType(); + Function converter = + checkStateNotNull( + converters.get(caseType.getValue()), "Missing OneOf converter for case %s."); + return oneOfType.createValue(caseType, converter.apply(field.getValue())); + } } - @SuppressWarnings("unchecked") - private Map fromMapValue( - FieldType keyType, - FieldType valueType, - Map map, - FieldValueTypeInformation keyTypeInformation, - FieldValueTypeInformation valueTypeInformation, - Factory> typeFactory) { - Map newMap = Maps.newHashMap(); - for (Map.Entry entry : map.entrySet()) { - Object key = - fromValue( - keyType, - entry.getKey(), - keyTypeInformation.getType().getType(), - keyTypeInformation, - typeFactory); - Object value = - fromValue( - valueType, - entry.getValue(), - valueTypeInformation.getType().getType(), - valueTypeInformation, - typeFactory); - newMap.put(key, value); - } - return newMap; + private static class ConvertLogicalType + implements FieldConverter { + final Schema.LogicalType logicalType; + + ConvertLogicalType(Schema.LogicalType logicalType) { + this.logicalType = logicalType; + } + + @Override + public ValueT convert(FieldT field) { + return logicalType.toBaseType(field); + } } @Override diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/GetterBasedSchemaProvider.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/GetterBasedSchemaProvider.java index 4d28e8132a59e..2b697bebd8152 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/GetterBasedSchemaProvider.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/GetterBasedSchemaProvider.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.schemas; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.ArrayList; import java.util.Collection; @@ -32,10 +32,10 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Collections2; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Collections2; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/JavaBeanSchema.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/JavaBeanSchema.java index 4fc1d33a57431..7024e8be86cf7 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/JavaBeanSchema.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/JavaBeanSchema.java @@ -31,9 +31,9 @@ import org.apache.beam.sdk.schemas.utils.JavaBeanUtils; import org.apache.beam.sdk.schemas.utils.ReflectUtils; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/JavaFieldSchema.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/JavaFieldSchema.java index bf5dac0eca2e2..16b96f1c7ae16 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/JavaFieldSchema.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/JavaFieldSchema.java @@ -32,9 +32,9 @@ import org.apache.beam.sdk.schemas.utils.POJOUtils; import org.apache.beam.sdk.schemas.utils.ReflectUtils; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; /** * A {@link SchemaProvider} for Java POJO objects. diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/Schema.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/Schema.java index ae3ffd94af567..39bee9ebaea44 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/Schema.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/Schema.java @@ -37,14 +37,14 @@ import java.util.stream.Collectors; import javax.annotation.concurrent.Immutable; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashBiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaCoder.java index 1e0805f779c1d..323f4e98dc551 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaCoder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaCoder.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.schemas; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.io.InputStream; @@ -35,7 +35,7 @@ import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; /** {@link SchemaCoder} is used as the coder for types that have schemas registered. */ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaCoderHelpers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaCoderHelpers.java index c661a59926f29..b2e707e5607aa 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaCoderHelpers.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaCoderHelpers.java @@ -44,8 +44,8 @@ import org.apache.beam.sdk.schemas.Schema.LogicalType; import org.apache.beam.sdk.schemas.Schema.TypeName; import org.apache.beam.sdk.util.common.ElementByteSizeObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.ReadableInstant; @SuppressWarnings({ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaRegistry.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaRegistry.java index 509aad2dceda4..372e2d00b768b 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaRegistry.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaRegistry.java @@ -29,10 +29,10 @@ import org.apache.beam.sdk.util.common.ReflectHelpers.ObjectsClassComparator; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaTranslation.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaTranslation.java index cfcfdcc0945b8..c0683ef446167 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaTranslation.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/SchemaTranslation.java @@ -54,12 +54,13 @@ import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; import org.apache.commons.lang3.ClassUtils; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; @@ -74,7 +75,23 @@ public class SchemaTranslation { private static final Logger LOG = LoggerFactory.getLogger(SchemaTranslation.class); private static final String URN_BEAM_LOGICAL_DECIMAL = FixedPrecisionNumeric.BASE_IDENTIFIER; - private static final String URN_BEAM_LOGICAL_JAVASDK = "beam:logical_type:javasdk:v1"; + + private static String getLogicalTypeUrn(String identifier) { + if (identifier.startsWith("beam:logical_type:")) { + return identifier; + } else { + String filtered = identifier.replaceAll("[^0-9A-Za-z_]", "").toLowerCase(); + if (!Strings.isNullOrEmpty(filtered)) { + // urn for non-standard Java SDK logical types are assigned with javasdk_ + return String.format("beam:logical_type:javasdk_%s:v1", filtered); + } else { + // raw "javasdk" name should only be a last resort. Types defined in Beam should have their + // own URN. + return "beam:logical_type:javasdk:v1"; + } + } + } + private static final String URN_BEAM_LOGICAL_MILLIS_INSTANT = SchemaApi.LogicalTypes.Enum.MILLIS_INSTANT .getValueDescriptor() @@ -84,18 +101,18 @@ public class SchemaTranslation { // TODO(https://github.com/apache/beam/issues/19715): Populate this with a LogicalTypeRegistrar, // which includes a way to construct // the LogicalType given an argument. - private static final ImmutableMap>> - STANDARD_LOGICAL_TYPES = - ImmutableMap.>>builder() - .put(FixedPrecisionNumeric.IDENTIFIER, FixedPrecisionNumeric.class) - .put(MicrosInstant.IDENTIFIER, MicrosInstant.class) - .put(SchemaLogicalType.IDENTIFIER, SchemaLogicalType.class) - .put(PythonCallable.IDENTIFIER, PythonCallable.class) - .put(FixedBytes.IDENTIFIER, FixedBytes.class) - .put(VariableBytes.IDENTIFIER, VariableBytes.class) - .put(FixedString.IDENTIFIER, FixedString.class) - .put(VariableString.IDENTIFIER, VariableString.class) - .build(); + @VisibleForTesting + static final ImmutableMap>> STANDARD_LOGICAL_TYPES = + ImmutableMap.>>builder() + .put(FixedPrecisionNumeric.IDENTIFIER, FixedPrecisionNumeric.class) + .put(MicrosInstant.IDENTIFIER, MicrosInstant.class) + .put(SchemaLogicalType.IDENTIFIER, SchemaLogicalType.class) + .put(PythonCallable.IDENTIFIER, PythonCallable.class) + .put(FixedBytes.IDENTIFIER, FixedBytes.class) + .put(VariableBytes.IDENTIFIER, VariableBytes.class) + .put(FixedString.IDENTIFIER, FixedString.class) + .put(VariableString.IDENTIFIER, VariableString.class) + .build(); public static SchemaApi.Schema schemaToProto(Schema schema, boolean serializeLogicalType) { String uuid = schema.getUUID() != null ? schema.getUUID().toString() : ""; @@ -179,11 +196,7 @@ static SchemaApi.FieldType fieldTypeToProto(FieldType fieldType, boolean seriali fieldValueToProto(logicalType.getArgumentType(), logicalType.getArgument())); } } else { - // TODO(https://github.com/apache/beam/issues/19715): "javasdk" types should only - // be a last resort. Types defined in Beam should have their own URN, and there - // should be a mechanism for users to register their own types by URN. - String urn = - identifier.startsWith("beam:logical_type:") ? identifier : URN_BEAM_LOGICAL_JAVASDK; + String urn = getLogicalTypeUrn(identifier); logicalTypeBuilder = SchemaApi.LogicalType.newBuilder() .setRepresentation( @@ -429,15 +442,22 @@ private static FieldType fieldTypeFromProtoWithoutNullable(SchemaApi.FieldType p } else if (urn.equals(URN_BEAM_LOGICAL_DECIMAL)) { return FieldType.DECIMAL; } else if (urn.startsWith("beam:logical_type:")) { - try { - return FieldType.logicalType( - (LogicalType) - SerializableUtils.deserializeFromByteArray( - logicalType.getPayload().toByteArray(), "logicalType")); - } catch (IllegalArgumentException e) { - LOG.warn( - "Unable to deserialize the logical type {} from proto. Mark as UnknownLogicalType.", - urn); + if (!logicalType.getPayload().isEmpty()) { + // logical type has a payload, try to recover the instance by deserialization + try { + return FieldType.logicalType( + (LogicalType) + SerializableUtils.deserializeFromByteArray( + logicalType.getPayload().toByteArray(), "logicalType")); + } catch (IllegalArgumentException e) { + LOG.warn( + "Unable to deserialize the logical type {} from proto. Mark as UnknownLogicalType.", + urn); + } + } else { + // logical type does not have a payload. This happens when it is passed xlang. + // TODO(yathu) it appears this path is called heavily, consider cache the instance + LOG.debug("Constructing non-standard logical type {} as UnknownLogicalType", urn); } } // assemble an UnknownLogicalType diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/annotations/DefaultSchema.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/annotations/DefaultSchema.java index 1466993a0daeb..2ad3feb686fa9 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/annotations/DefaultSchema.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/annotations/DefaultSchema.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.schemas.annotations; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.Serializable; import java.lang.annotation.Documented; @@ -35,8 +35,8 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/annotations/SchemaCaseFormat.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/annotations/SchemaCaseFormat.java index 9f14fe5f774a1..edbdc9c187649 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/annotations/SchemaCaseFormat.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/annotations/SchemaCaseFormat.java @@ -23,7 +23,7 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import javax.annotation.Nonnull; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.CaseFormat; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CaseFormat; /** * When used on a {@link org.apache.beam.sdk.schemas.JavaFieldSchema POJO}, {@link diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/DeadLetteredTransform.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/DeadLetteredTransform.java index 6a970be6501be..4923cbe53c224 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/DeadLetteredTransform.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/DeadLetteredTransform.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.exception.ExceptionUtils; @Internal diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/GenericDlq.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/GenericDlq.java index e2eb4dbf10cdc..9b8870eadfefa 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/GenericDlq.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/GenericDlq.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.schemas.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.List; import java.util.Map; @@ -25,7 +25,7 @@ import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; /** Helper to generate a DLQ transform to write PCollection to an external system. */ @Internal diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/Providers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/Providers.java index e2e4b5df5127b..ed3abcd1ba836 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/Providers.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/Providers.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.schemas.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.HashMap; import java.util.Map; @@ -40,27 +40,12 @@ private Providers() {} public static Map loadProviders(Class klass) { Map providers = new HashMap<>(); for (T provider : ServiceLoader.load(klass)) { - // Avro provider is treated as a special case since two Avro providers may want to be loaded - - // from "core" (deprecated) and from "extensions/avro" (actual) - but only one must succeed. - // TODO: we won't need this check once all Avro providers from "core" will be - // removed - if (provider.identifier().equals("avro")) { - // Avro provider from "extensions/avro" must have a priority. - if (provider.getClass().getName().startsWith("org.apache.beam.sdk.extensions.avro")) { - // Load Avro provider from "extensions/avro" by any case. - providers.put(provider.identifier(), provider); - } else { - // Load Avro provider from "core" if it was not loaded from Avro extension before. - providers.putIfAbsent(provider.identifier(), provider); - } - } else { - checkState( - !providers.containsKey(provider.identifier()), - "Duplicate providers exist with identifier `%s` for class %s.", - provider.identifier(), - klass); - providers.put(provider.identifier(), provider); - } + checkState( + !providers.containsKey(provider.identifier()), + "Duplicate providers exist with identifier `%s` for class %s.", + provider.identifier(), + klass); + providers.put(provider.identifier(), provider); } return providers; } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/AvroPayloadSerializerProvider.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/AvroPayloadSerializerProvider.java deleted file mode 100644 index e8f99b33c0ddd..0000000000000 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/AvroPayloadSerializerProvider.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.schemas.io.payloads; - -import com.google.auto.service.AutoService; -import java.util.Map; -import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.sdk.schemas.utils.AvroUtils; - -/** - * @deprecated Avro related classes are deprecated in module beam-sdks-java-core and - * will be eventually removed. Please, migrate to a new module - * beam-sdks-java-extensions-avro by importing - * org.apache.beam.sdk.extensions.avro.schemas.io.payloads.AvroPayloadSerializerProvider - * instead of this one. - */ -@Internal -@Deprecated -@AutoService(PayloadSerializerProvider.class) -public class AvroPayloadSerializerProvider implements PayloadSerializerProvider { - @Override - public String identifier() { - return "avro"; - } - - @Override - public PayloadSerializer getSerializer(Schema schema, Map tableParams) { - return PayloadSerializer.of( - AvroUtils.getRowToAvroBytesFunction(schema), AvroUtils.getAvroBytesToRowFunction(schema)); - } -} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/PayloadSerializers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/PayloadSerializers.java index 936396771650a..f65a1276e9904 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/PayloadSerializers.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/payloads/PayloadSerializers.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.schemas.io.payloads; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Map; import org.apache.beam.sdk.annotations.Internal; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/EnumerationType.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/EnumerationType.java index 8793f87f3a336..9ec63ec8c8edd 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/EnumerationType.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/EnumerationType.java @@ -29,8 +29,8 @@ import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.Schema.LogicalType; import org.apache.beam.sdk.schemas.logicaltypes.EnumerationType.Value; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashBiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBiMap; import org.checkerframework.checker.nullness.qual.Nullable; /** This {@link LogicalType} represent an enumeration over a fixed set of values. */ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/FixedBytes.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/FixedBytes.java index 886f0851c494e..f87063283ab95 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/FixedBytes.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/FixedBytes.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.schemas.logicaltypes; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Arrays; import org.apache.beam.model.pipeline.v1.RunnerApi; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/FixedPrecisionNumeric.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/FixedPrecisionNumeric.java index 63be36ec1bef0..b089a58c5dd65 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/FixedPrecisionNumeric.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/FixedPrecisionNumeric.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.schemas.logicaltypes; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.math.BigDecimal; import java.math.MathContext; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/FixedString.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/FixedString.java index 72dd97fae8378..55e5522e8282a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/FixedString.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/FixedString.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.schemas.logicaltypes; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import org.apache.beam.model.pipeline.v1.RunnerApi; import org.apache.beam.model.pipeline.v1.SchemaApi; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/OneOfType.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/OneOfType.java index ea7c38f3fe2c1..4a7573b036e25 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/OneOfType.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/OneOfType.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.schemas.logicaltypes; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.Arrays; import java.util.List; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/VariableBytes.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/VariableBytes.java index 4c1cb87f8f9c8..d1b9cc26df7ad 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/VariableBytes.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/VariableBytes.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.schemas.logicaltypes; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import org.apache.beam.model.pipeline.v1.RunnerApi; import org.apache.beam.model.pipeline.v1.SchemaApi; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/VariableString.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/VariableString.java index c635e70a625af..bd2353301efb8 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/VariableString.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/VariableString.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.schemas.logicaltypes; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import org.apache.beam.model.pipeline.v1.RunnerApi; import org.apache.beam.model.pipeline.v1.SchemaApi; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/parser/FieldAccessDescriptorParser.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/parser/FieldAccessDescriptorParser.java index 28ad760bab0e3..367042ce04809 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/parser/FieldAccessDescriptorParser.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/parser/FieldAccessDescriptorParser.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.schemas.parser; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.List; import java.util.stream.Collectors; @@ -42,7 +42,7 @@ import org.apache.beam.sdk.schemas.parser.generated.FieldSpecifierNotationParser.QualifyComponentContext; import org.apache.beam.sdk.schemas.parser.generated.FieldSpecifierNotationParser.SimpleIdentifierContext; import org.apache.beam.sdk.schemas.parser.generated.FieldSpecifierNotationParser.WildcardContext; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; /** Parser for textual field-access selector. */ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/AddFields.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/AddFields.java index 2e0e9cd66664f..e04efb3b9c2af 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/AddFields.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/AddFields.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.schemas.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import java.io.Serializable; @@ -37,12 +37,12 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimaps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimaps; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Cast.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Cast.java index c583f11445b8c..cba5ba0d82c35 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Cast.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Cast.java @@ -37,10 +37,10 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; /** Set of utilities for casting rows between schemas. */ @AutoValue diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/CoGroup.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/CoGroup.java index 11b64c34e9193..60757a0033961 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/CoGroup.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/CoGroup.java @@ -52,11 +52,11 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/DropFields.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/DropFields.java index 1ad5d1eef8583..c3dc18f1cdd5d 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/DropFields.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/DropFields.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.schemas.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; import java.util.List; import java.util.Map; @@ -30,7 +30,7 @@ import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; /** * A transform to drop fields from a schema. diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Filter.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Filter.java index 5f5fe9f496e0b..a3db3256b5b8b 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Filter.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Filter.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Group.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Group.java index fb48ceed31167..c77c8dcd20a4e 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Group.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Group.java @@ -17,7 +17,9 @@ */ package org.apache.beam.sdk.schemas.transforms; +import com.google.auto.value.AutoOneOf; import com.google.auto.value.AutoValue; +import java.io.Serializable; import java.util.List; import org.apache.beam.sdk.coders.KvCoder; import org.apache.beam.sdk.schemas.FieldAccessDescriptor; @@ -34,6 +36,7 @@ import org.apache.beam.sdk.transforms.GroupByKey; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.Values; import org.apache.beam.sdk.transforms.WithKeys; import org.apache.beam.sdk.transforms.windowing.GlobalWindows; @@ -41,7 +44,8 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.checkerframework.checker.nullness.qual.Nullable; /** * A generic grouping transform for schema {@link PCollection}s. @@ -153,7 +157,7 @@ public static class Global */ public CombineGlobally aggregate( CombineFn combineFn) { - return new CombineGlobally<>(combineFn); + return new CombineGlobally<>(combineFn, 0); } /** @@ -169,10 +173,8 @@ public CombineFieldsGlobally agg return new CombineFieldsGlobally<>( SchemaAggregateFn.create() .aggregateFields( - FieldAccessDescriptor.withFieldNames(inputFieldName), - false, - fn, - outputFieldName)); + FieldAccessDescriptor.withFieldNames(inputFieldName), false, fn, outputFieldName), + 0); } public @@ -183,7 +185,8 @@ CombineFieldsGlobally aggregateFieldBaseValue( return new CombineFieldsGlobally<>( SchemaAggregateFn.create() .aggregateFields( - FieldAccessDescriptor.withFieldNames(inputFieldName), true, fn, outputFieldName)); + FieldAccessDescriptor.withFieldNames(inputFieldName), true, fn, outputFieldName), + 0); } /** The same as {@link #aggregateField} but using field id. */ @@ -194,7 +197,8 @@ public CombineFieldsGlobally agg return new CombineFieldsGlobally<>( SchemaAggregateFn.create() .aggregateFields( - FieldAccessDescriptor.withFieldIds(inputFieldId), false, fn, outputFieldName)); + FieldAccessDescriptor.withFieldIds(inputFieldId), false, fn, outputFieldName), + 0); } public @@ -205,7 +209,8 @@ CombineFieldsGlobally aggregateFieldBaseValue( return new CombineFieldsGlobally<>( SchemaAggregateFn.create() .aggregateFields( - FieldAccessDescriptor.withFieldIds(inputFieldId), true, fn, outputFieldName)); + FieldAccessDescriptor.withFieldIds(inputFieldId), true, fn, outputFieldName), + 0); } /** @@ -221,7 +226,8 @@ public CombineFieldsGlobally agg return new CombineFieldsGlobally<>( SchemaAggregateFn.create() .aggregateFields( - FieldAccessDescriptor.withFieldNames(inputFieldName), false, fn, outputField)); + FieldAccessDescriptor.withFieldNames(inputFieldName), false, fn, outputField), + 0); } public @@ -232,7 +238,8 @@ CombineFieldsGlobally aggregateFieldBaseValue( return new CombineFieldsGlobally<>( SchemaAggregateFn.create() .aggregateFields( - FieldAccessDescriptor.withFieldNames(inputFieldName), true, fn, outputField)); + FieldAccessDescriptor.withFieldNames(inputFieldName), true, fn, outputField), + 0); } /** The same as {@link #aggregateField} but using field id. */ @@ -241,7 +248,8 @@ public CombineFieldsGlobally agg return new CombineFieldsGlobally<>( SchemaAggregateFn.create() .aggregateFields( - FieldAccessDescriptor.withFieldIds(inputFielId), false, fn, outputField)); + FieldAccessDescriptor.withFieldIds(inputFielId), false, fn, outputField), + 0); } public @@ -252,7 +260,8 @@ CombineFieldsGlobally aggregateFieldBaseValue( return new CombineFieldsGlobally<>( SchemaAggregateFn.create() .aggregateFields( - FieldAccessDescriptor.withFieldIds(inputFielId), true, fn, outputField)); + FieldAccessDescriptor.withFieldIds(inputFielId), true, fn, outputField), + 0); } /** @@ -298,8 +307,8 @@ public CombineFieldsGlobally agg CombineFn fn, String outputFieldName) { return new CombineFieldsGlobally<>( - SchemaAggregateFn.create() - .aggregateFields(fieldsToAggregate, false, fn, outputFieldName)); + SchemaAggregateFn.create().aggregateFields(fieldsToAggregate, false, fn, outputFieldName), + 0); } /** @@ -335,7 +344,7 @@ public CombineFieldsGlobally agg CombineFn fn, Field outputField) { return new CombineFieldsGlobally<>( - SchemaAggregateFn.create().aggregateFields(fieldsToAggregate, false, fn, outputField)); + SchemaAggregateFn.create().aggregateFields(fieldsToAggregate, false, fn, outputField), 0); } @Override @@ -351,14 +360,20 @@ public PCollection> expand(PCollection input) { public static class CombineGlobally extends PTransform, PCollection> { final CombineFn combineFn; + int fanout; - CombineGlobally(CombineFn combineFn) { + CombineGlobally(CombineFn combineFn, int fanout) { this.combineFn = combineFn; + this.fanout = fanout; + } + + public CombineGlobally withFanout(int fanout) { + return new CombineGlobally<>(combineFn, fanout); } @Override public PCollection expand(PCollection input) { - return input.apply("globalCombine", Combine.globally(combineFn)); + return input.apply("globalCombine", Combine.globally(combineFn).withFanout(fanout)); } } @@ -420,9 +435,11 @@ AggregateCombiner aggregateFieldsById( */ public static class CombineFieldsGlobally extends AggregateCombiner { private final SchemaAggregateFn.Inner schemaAggregateFn; + private final int fanout; - CombineFieldsGlobally(SchemaAggregateFn.Inner schemaAggregateFn) { + CombineFieldsGlobally(SchemaAggregateFn.Inner schemaAggregateFn, int fanout) { this.schemaAggregateFn = schemaAggregateFn; + this.fanout = fanout; } /** @@ -431,7 +448,7 @@ public static class CombineFieldsGlobally extends AggregateCombiner create() { - return new CombineFieldsGlobally<>(SchemaAggregateFn.create()); + return new CombineFieldsGlobally<>(SchemaAggregateFn.create(), 0); } /** @@ -450,7 +467,8 @@ public CombineFieldsGlobally agg String outputFieldName) { return new CombineFieldsGlobally<>( schemaAggregateFn.aggregateFields( - FieldAccessDescriptor.withFieldNames(inputFieldName), false, fn, outputFieldName)); + FieldAccessDescriptor.withFieldNames(inputFieldName), false, fn, outputFieldName), + fanout); } public @@ -460,7 +478,8 @@ CombineFieldsGlobally aggregateFieldBaseValue( String outputFieldName) { return new CombineFieldsGlobally<>( schemaAggregateFn.aggregateFields( - FieldAccessDescriptor.withFieldNames(inputFieldName), true, fn, outputFieldName)); + FieldAccessDescriptor.withFieldNames(inputFieldName), true, fn, outputFieldName), + fanout); } public CombineFieldsGlobally aggregateField( @@ -469,7 +488,8 @@ public CombineFieldsGlobally agg String outputFieldName) { return new CombineFieldsGlobally<>( schemaAggregateFn.aggregateFields( - FieldAccessDescriptor.withFieldIds(inputFieldId), false, fn, outputFieldName)); + FieldAccessDescriptor.withFieldIds(inputFieldId), false, fn, outputFieldName), + fanout); } public @@ -479,7 +499,8 @@ CombineFieldsGlobally aggregateFieldBaseValue( String outputFieldName) { return new CombineFieldsGlobally<>( schemaAggregateFn.aggregateFields( - FieldAccessDescriptor.withFieldIds(inputFieldId), true, fn, outputFieldName)); + FieldAccessDescriptor.withFieldIds(inputFieldId), true, fn, outputFieldName), + fanout); } /** @@ -495,7 +516,8 @@ public CombineFieldsGlobally agg Field outputField) { return new CombineFieldsGlobally<>( schemaAggregateFn.aggregateFields( - FieldAccessDescriptor.withFieldNames(inputFieldName), false, fn, outputField)); + FieldAccessDescriptor.withFieldNames(inputFieldName), false, fn, outputField), + fanout); } public @@ -505,7 +527,8 @@ CombineFieldsGlobally aggregateFieldBaseValue( Field outputField) { return new CombineFieldsGlobally<>( schemaAggregateFn.aggregateFields( - FieldAccessDescriptor.withFieldNames(inputFieldName), true, fn, outputField)); + FieldAccessDescriptor.withFieldNames(inputFieldName), true, fn, outputField), + fanout); } @Override @@ -513,7 +536,8 @@ public CombineFieldsGlobally agg int inputFieldId, CombineFn fn, Field outputField) { return new CombineFieldsGlobally<>( schemaAggregateFn.aggregateFields( - FieldAccessDescriptor.withFieldIds(inputFieldId), false, fn, outputField)); + FieldAccessDescriptor.withFieldIds(inputFieldId), false, fn, outputField), + fanout); } public @@ -523,7 +547,8 @@ CombineFieldsGlobally aggregateFieldBaseValue( Field outputField) { return new CombineFieldsGlobally<>( schemaAggregateFn.aggregateFields( - FieldAccessDescriptor.withFieldIds(inputFieldId), true, fn, outputField)); + FieldAccessDescriptor.withFieldIds(inputFieldId), true, fn, outputField), + fanout); } /** @@ -568,7 +593,8 @@ public CombineFieldsGlobally agg CombineFn fn, String outputFieldName) { return new CombineFieldsGlobally<>( - schemaAggregateFn.aggregateFields(fieldAccessDescriptor, false, fn, outputFieldName)); + schemaAggregateFn.aggregateFields(fieldAccessDescriptor, false, fn, outputFieldName), + fanout); } /** @@ -605,13 +631,17 @@ public CombineFieldsGlobally agg CombineFn fn, Field outputField) { return new CombineFieldsGlobally<>( - schemaAggregateFn.aggregateFields(fieldAccessDescriptor, false, fn, outputField)); + schemaAggregateFn.aggregateFields(fieldAccessDescriptor, false, fn, outputField), fanout); + } + + public CombineFieldsGlobally withFanout(int fanout) { + return new CombineFieldsGlobally<>(schemaAggregateFn, fanout); } @Override public PCollection expand(PCollection input) { SchemaAggregateFn.Inner fn = schemaAggregateFn.withSchema(input.getSchema()); - Combine.Globally combineFn = Combine.globally(fn); + Combine.Globally combineFn = Combine.globally(fn).withFanout(fanout); if (!(input.getWindowingStrategy().getWindowFn() instanceof GlobalWindows)) { combineFn = combineFn.withoutDefaults(); } @@ -631,6 +661,7 @@ public PCollection expand(PCollection input) { */ @AutoValue public abstract static class ByFields extends AggregateCombiner { + abstract FieldAccessDescriptor getFieldAccessDescriptor(); abstract String getKeyField(); @@ -651,11 +682,11 @@ abstract Builder setFieldAccessDescriptor( abstract ByFields build(); } - class ToKv extends PTransform, PCollection>>> { + class ToKV extends PTransform, PCollection>> { private RowSelector rowSelector; @Override - public PCollection>> expand(PCollection input) { + public PCollection> expand(PCollection input) { Schema schema = input.getSchema(); FieldAccessDescriptor resolved = getFieldAccessDescriptor().resolve(schema); rowSelector = new RowSelectorContainer(schema, resolved, true); @@ -666,13 +697,12 @@ public PCollection>> expand(PCollection input) { .apply( "selectKeys", WithKeys.of((Row e) -> rowSelector.select(e)).withKeyType(TypeDescriptors.rows())) - .setCoder(KvCoder.of(SchemaCoder.of(keySchema), SchemaCoder.of(schema))) - .apply("GroupByKey", GroupByKey.create()); + .setCoder(KvCoder.of(SchemaCoder.of(keySchema), SchemaCoder.of(schema))); } } - public ToKv getToKvs() { - return new ToKv(); + public ToKV getToKV() { + return new ToKV(); } private static ByFields of(FieldAccessDescriptor fieldAccessDescriptor) { @@ -919,7 +949,8 @@ public PCollection expand(PCollection input) { .build(); return input - .apply("ToKvs", getToKvs()) + .apply("ToKvs", getToKV()) + .apply("GroupByKey", GroupByKey.create()) .apply( "ToRow", ParDo.of( @@ -942,6 +973,33 @@ public void process(@Element KV> e, OutputReceiver o) { */ @AutoValue public abstract static class CombineFieldsByFields extends AggregateCombiner { + + @AutoOneOf(Fanout.Kind.class) + public abstract static class Fanout implements Serializable { + public enum Kind { + NUMBER, + FUNCTION + } + + public abstract Kind getKind(); + + public abstract Integer getNumber(); + + public abstract SerializableFunction getFunction(); + + public static Fanout of(int n) { + return AutoOneOf_Group_CombineFieldsByFields_Fanout.number(n); + } + + public static Fanout of(SerializableFunction f) { + return AutoOneOf_Group_CombineFieldsByFields_Fanout.function(f); + } + } + + abstract @Nullable Fanout getFanout(); + + abstract Boolean getFewKeys(); + abstract ByFields getByFields(); abstract SchemaAggregateFn.Inner getSchemaAggregateFn(); @@ -954,6 +1012,10 @@ public abstract static class CombineFieldsByFields extends AggregateComb @AutoValue.Builder abstract static class Builder { + public abstract Builder setFanout(@Nullable Fanout value); + + public abstract Builder setFewKeys(Boolean fewKeys); + abstract Builder setByFields(ByFields byFields); abstract Builder setSchemaAggregateFn(SchemaAggregateFn.Inner schemaAggregateFn); @@ -975,6 +1037,9 @@ static CombineFieldsByFields of( .setSchemaAggregateFn(schemaAggregateFn) .setKeyField(keyField) .setValueField(valueField) + .setFewKeys( + true) // We are often selecting only certain fields, so combiner lifting usually + // helps. .build(); } @@ -988,6 +1053,25 @@ public CombineFieldsByFields witValueField(String valueField) { return toBuilder().setValueField(valueField).build(); } + /** + * Enable precombining. + * + *

    This is on by default. In certain cases (e.g. if there are many unique field values and + * the combiner's intermediate state is larger than the average row size) precombining makes + * things worse, in which case it can be turned off. + */ + public CombineFieldsByFields withPrecombining(boolean value) { + return toBuilder().setFewKeys(value).build(); + } + + public CombineFieldsByFields withHotKeyFanout(int n) { + return toBuilder().setFanout(Fanout.of(n)).build(); + } + + public CombineFieldsByFields withHotKeyFanout(SerializableFunction f) { + return toBuilder().setFanout(Fanout.of(f)).build(); + } + /** * Build up an aggregation function over the input elements. * @@ -1187,9 +1271,25 @@ public CombineFieldsByFields agg .build(); } + PTransform>, PCollection>> getCombineTransform( + Schema schema) { + SchemaAggregateFn.Inner fn = getSchemaAggregateFn().withSchema(schema); + @Nullable Fanout fanout = getFanout(); + if (fanout != null) { + switch (fanout.getKind()) { + case NUMBER: + return Combine.perKey(fn).withHotKeyFanout(fanout.getNumber()); + case FUNCTION: + return Combine.perKey(fn).withHotKeyFanout(fanout.getFunction()); + default: + throw new RuntimeException("Unexpected kind: " + fanout.getKind()); + } + } + return getFewKeys() ? Combine.fewKeys(fn) : Combine.perKey(fn); + } + @Override public PCollection expand(PCollection input) { - SchemaAggregateFn.Inner fn = getSchemaAggregateFn().withSchema(input.getSchema()); Schema keySchema = getByFields().getKeySchema(input.getSchema()); Schema outputSchema = @@ -1199,8 +1299,8 @@ public PCollection expand(PCollection input) { .build(); return input - .apply("ToKvs", getByFields().getToKvs()) - .apply("Combine", Combine.groupedValues(fn)) + .apply("ToKvs", getByFields().getToKV()) + .apply("Combine", getCombineTransform(input.getSchema())) .apply( "ToRow", ParDo.of( diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/RenameFields.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/RenameFields.java index 0ecee5f2ca3b5..a568879dd2328 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/RenameFields.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/RenameFields.java @@ -36,13 +36,13 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; /** * A transform for renaming fields inside an existing schema. Top level or nested fields can be diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaAggregateFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaAggregateFn.java index fcb6cf7a79191..f37f92ff2e439 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaAggregateFn.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaAggregateFn.java @@ -40,8 +40,8 @@ import org.apache.beam.sdk.transforms.SimpleFunction; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; /** This is the builder used by {@link Group} to build up a composed {@link CombineFn}. */ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaTransform.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaTransform.java index 1e239eba09cb2..283720e097727 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaTransform.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaTransform.java @@ -22,7 +22,7 @@ import org.apache.beam.sdk.values.PCollectionRowTuple; /** - * An abstraction to create schema capable and aware transforms. The interface is intended to be + * An abstraction representing schema capable and aware transforms. The interface is intended to be * used in conjunction with the interface {@link SchemaTransformProvider}. * *

    The interfaces can be implemented to make transforms available in other SDKs in addition to @@ -33,6 +33,5 @@ * compatibility guarantees and it should not be implemented outside of the Beam repository. */ @Internal -public interface SchemaTransform { - PTransform buildTransform(); -} +public abstract class SchemaTransform + extends PTransform {} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaTransformProvider.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaTransformProvider.java index 97a5f1830e94b..e542007c9a55a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaTransformProvider.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaTransformProvider.java @@ -43,8 +43,8 @@ public interface SchemaTransformProvider { Schema configurationSchema(); /** - * Produce a SchemaTransform some transform-specific configuration object. Can throw a {@link - * InvalidConfigurationException} or a {@link InvalidSchemaException}. + * Produce a {@link SchemaTransform} from some transform-specific configuration object. Can throw + * a {@link InvalidConfigurationException} or a {@link InvalidSchemaException}. */ SchemaTransform from(Row configuration); diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Select.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Select.java index 22adb8100da2d..84ae7c42cb649 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Select.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/Select.java @@ -38,9 +38,9 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/AutoValueUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/AutoValueUtils.java index dc5e679fe6083..769b228794186 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/AutoValueUtils.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/AutoValueUtils.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.schemas.utils; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; @@ -62,7 +62,7 @@ import org.apache.beam.sdk.schemas.utils.ByteBuddyUtils.TypeConversionsFactory; import org.apache.beam.sdk.util.common.ReflectHelpers; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; /** Utilities for managing AutoValue schemas. */ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/AvroByteBuddyUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/AvroByteBuddyUtils.java deleted file mode 100644 index 7253f5fd4dac5..0000000000000 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/AvroByteBuddyUtils.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.schemas.utils; - -import static org.apache.beam.sdk.util.ByteBuddyUtils.getClassLoadingStrategy; - -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Type; -import java.util.Map; -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.asm.AsmVisitorWrapper; -import net.bytebuddy.description.type.TypeDescription.ForLoadedType; -import net.bytebuddy.dynamic.DynamicType; -import net.bytebuddy.implementation.MethodCall; -import net.bytebuddy.implementation.bytecode.StackManipulation; -import net.bytebuddy.implementation.bytecode.assign.TypeCasting; -import net.bytebuddy.implementation.bytecode.collection.ArrayAccess; -import net.bytebuddy.implementation.bytecode.constant.IntegerConstant; -import net.bytebuddy.implementation.bytecode.member.MethodVariableAccess; -import net.bytebuddy.jar.asm.ClassWriter; -import net.bytebuddy.matcher.ElementMatchers; -import org.apache.avro.specific.SpecificRecord; -import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.sdk.schemas.SchemaUserTypeCreator; -import org.apache.beam.sdk.schemas.utils.AvroUtils.AvroTypeConversionFactory; -import org.apache.beam.sdk.schemas.utils.ByteBuddyUtils.InjectPackageStrategy; -import org.apache.beam.sdk.schemas.utils.ByteBuddyUtils.TypeConversion; -import org.apache.beam.sdk.schemas.utils.ByteBuddyUtils.TypeConversionsFactory; -import org.apache.beam.sdk.schemas.utils.ReflectUtils.ClassWithSchema; -import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; - -/** - * @deprecated Avro related classes are deprecated in module beam-sdks-java-core and - * will be eventually removed. Please, migrate to a new module - * beam-sdks-java-extensions-avro by importing - * org.apache.beam.sdk.extensions.avro.schemas.utils.AvroByteBuddyUtils instead of this - * one. - */ -@SuppressWarnings({ - "nullness", // TODO(https://github.com/apache/beam/issues/20497) - "rawtypes" -}) -@Deprecated -class AvroByteBuddyUtils { - private static final ByteBuddy BYTE_BUDDY = new ByteBuddy(); - - // Cache the generated constructors. - private static final Map CACHED_CREATORS = - Maps.newConcurrentMap(); - - static SchemaUserTypeCreator getCreator( - Class clazz, Schema schema) { - return CACHED_CREATORS.computeIfAbsent( - ClassWithSchema.create(clazz, schema), c -> createCreator(clazz, schema)); - } - - private static SchemaUserTypeCreator createCreator(Class clazz, Schema schema) { - Constructor baseConstructor = null; - Constructor[] constructors = clazz.getDeclaredConstructors(); - for (Constructor constructor : constructors) { - // TODO: This assumes that Avro only generates one constructor with this many fields. - if (constructor.getParameterCount() == schema.getFieldCount()) { - baseConstructor = constructor; - } - } - if (baseConstructor == null) { - throw new RuntimeException("No matching constructor found for class " + clazz); - } - - // Generate a method call to create and invoke the SpecificRecord's constructor. . - MethodCall construct = MethodCall.construct(baseConstructor); - for (int i = 0; i < baseConstructor.getParameterTypes().length; ++i) { - Class baseType = baseConstructor.getParameterTypes()[i]; - construct = construct.with(readAndConvertParameter(baseType, i), baseType); - } - - try { - DynamicType.Builder builder = - BYTE_BUDDY - .with(new InjectPackageStrategy(clazz)) - .subclass(SchemaUserTypeCreator.class) - .method(ElementMatchers.named("create")) - .intercept(construct); - - return builder - .visit(new AsmVisitorWrapper.ForDeclaredMethods().writerFlags(ClassWriter.COMPUTE_FRAMES)) - .make() - .load( - ReflectHelpers.findClassLoader(clazz.getClassLoader()), - getClassLoadingStrategy(clazz)) - .getLoaded() - .getDeclaredConstructor() - .newInstance(); - } catch (InstantiationException - | IllegalAccessException - | NoSuchMethodException - | InvocationTargetException e) { - throw new RuntimeException( - "Unable to generate a getter for class " + clazz + " with schema " + schema); - } - } - - private static StackManipulation readAndConvertParameter( - Class constructorParameterType, int index) { - TypeConversionsFactory typeConversionsFactory = new AvroTypeConversionFactory(); - - // The types in the AVRO-generated constructor might be the types returned by Beam's Row class, - // so we have to convert the types used by Beam's Row class. - // We know that AVRO generates constructor parameters in the same order as fields - // in the schema, so we can just add the parameters sequentially. - TypeConversion convertType = typeConversionsFactory.createTypeConversion(true); - - // Map the AVRO-generated type to the one Beam will use. - ForLoadedType convertedType = - new ForLoadedType((Class) convertType.convert(TypeDescriptor.of(constructorParameterType))); - - // This will run inside the generated creator. Read the parameter and convert it to the - // type required by the SpecificRecord constructor. - StackManipulation readParameter = - new StackManipulation.Compound( - MethodVariableAccess.REFERENCE.loadFrom(1), - IntegerConstant.forValue(index), - ArrayAccess.REFERENCE.load(), - TypeCasting.to(convertedType)); - - // Convert to the parameter accepted by the SpecificRecord constructor. - return typeConversionsFactory - .createSetterConversions(readParameter) - .convert(TypeDescriptor.of(constructorParameterType)); - } -} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/AvroUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/AvroUtils.java deleted file mode 100644 index 3805a27ef5a44..0000000000000 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/AvroUtils.java +++ /dev/null @@ -1,1396 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.schemas.utils; - -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.lang.reflect.Method; -import java.math.BigDecimal; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; -import javax.annotation.Nonnull; -import net.bytebuddy.description.type.TypeDescription.ForLoadedType; -import net.bytebuddy.implementation.bytecode.Duplication; -import net.bytebuddy.implementation.bytecode.StackManipulation; -import net.bytebuddy.implementation.bytecode.StackManipulation.Compound; -import net.bytebuddy.implementation.bytecode.TypeCreation; -import net.bytebuddy.implementation.bytecode.assign.TypeCasting; -import net.bytebuddy.implementation.bytecode.member.MethodInvocation; -import net.bytebuddy.matcher.ElementMatchers; -import org.apache.avro.AvroRuntimeException; -import org.apache.avro.Conversions; -import org.apache.avro.LogicalType; -import org.apache.avro.LogicalTypes; -import org.apache.avro.Schema.Type; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericFixed; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.GenericRecordBuilder; -import org.apache.avro.reflect.AvroIgnore; -import org.apache.avro.reflect.AvroName; -import org.apache.avro.reflect.ReflectData; -import org.apache.avro.specific.SpecificData; -import org.apache.avro.specific.SpecificRecord; -import org.apache.avro.util.Utf8; -import org.apache.beam.sdk.coders.AvroCoder; -import org.apache.beam.sdk.coders.AvroCoder.JodaTimestampConversion; -import org.apache.beam.sdk.schemas.AvroRecordSchema; -import org.apache.beam.sdk.schemas.FieldValueGetter; -import org.apache.beam.sdk.schemas.FieldValueTypeInformation; -import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.sdk.schemas.Schema.Field; -import org.apache.beam.sdk.schemas.Schema.FieldType; -import org.apache.beam.sdk.schemas.Schema.TypeName; -import org.apache.beam.sdk.schemas.SchemaCoder; -import org.apache.beam.sdk.schemas.SchemaUserTypeCreator; -import org.apache.beam.sdk.schemas.logicaltypes.EnumerationType; -import org.apache.beam.sdk.schemas.logicaltypes.FixedBytes; -import org.apache.beam.sdk.schemas.logicaltypes.FixedString; -import org.apache.beam.sdk.schemas.logicaltypes.OneOfType; -import org.apache.beam.sdk.schemas.logicaltypes.SqlTypes; -import org.apache.beam.sdk.schemas.logicaltypes.VariableBytes; -import org.apache.beam.sdk.schemas.logicaltypes.VariableString; -import org.apache.beam.sdk.schemas.utils.ByteBuddyUtils.ConvertType; -import org.apache.beam.sdk.schemas.utils.ByteBuddyUtils.ConvertValueForGetter; -import org.apache.beam.sdk.schemas.utils.ByteBuddyUtils.ConvertValueForSetter; -import org.apache.beam.sdk.schemas.utils.ByteBuddyUtils.TypeConversion; -import org.apache.beam.sdk.schemas.utils.ByteBuddyUtils.TypeConversionsFactory; -import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.sdk.transforms.SimpleFunction; -import org.apache.beam.sdk.values.Row; -import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.CaseFormat; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.checkerframework.checker.nullness.qual.Nullable; -import org.joda.time.Days; -import org.joda.time.Duration; -import org.joda.time.Instant; -import org.joda.time.ReadableInstant; - -/** - * Utils to convert AVRO records to Beam rows. Imposes a mapping between common avro types and Beam - * portable schemas (https://s.apache.org/beam-schemas): - * - *

    - *   Avro                Beam Field Type
    - *   INT         <-----> INT32
    - *   LONG        <-----> INT64
    - *   FLOAT       <-----> FLOAT
    - *   DOUBLE      <-----> DOUBLE
    - *   BOOLEAN     <-----> BOOLEAN
    - *   STRING      <-----> STRING
    - *   BYTES       <-----> BYTES
    - *               <------ LogicalType(urn="beam:logical_type:var_bytes:v1")
    - *   FIXED       <-----> LogicalType(urn="beam:logical_type:fixed_bytes:v1")
    - *   ARRAY       <-----> ARRAY
    - *   ENUM        <-----> LogicalType(EnumerationType)
    - *   MAP         <-----> MAP
    - *   RECORD      <-----> ROW
    - *   UNION       <-----> LogicalType(OneOfType)
    - *   LogicalTypes.Date              <-----> LogicalType(DATE)
    - *                                  <------ LogicalType(urn="beam:logical_type:date:v1")
    - *   LogicalTypes.TimestampMillis   <-----> DATETIME
    - *   LogicalTypes.Decimal           <-----> DECIMAL
    - * 
    - * - * For SQL CHAR/VARCHAR types, an Avro schema - * - *
    - *   LogicalType({"type":"string","logicalType":"char","maxLength":MAX_LENGTH}) or
    - *   LogicalType({"type":"string","logicalType":"varchar","maxLength":MAX_LENGTH})
    - * 
    - * - * is used. - * - * @deprecated Avro related classes are deprecated in module beam-sdks-java-core and - * will be eventually removed. Please, migrate to a new module - * beam-sdks-java-extensions-avro by importing - * org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils instead of this one. - */ -@SuppressWarnings({ - "nullness", // TODO(https://github.com/apache/beam/issues/20497) - "rawtypes" -}) -@Deprecated -public class AvroUtils { - static { - // This works around a bug in the Avro library (AVRO-1891) around SpecificRecord's handling - // of DateTime types. - SpecificData.get().addLogicalTypeConversion(new JodaTimestampConversion()); - GenericData.get().addLogicalTypeConversion(new JodaTimestampConversion()); - } - - /** Unwrap an AVRO schema into the base type an whether it is nullable. */ - public static class TypeWithNullability { - final org.apache.avro.Schema type; - final boolean nullable; - - public static TypeWithNullability create(org.apache.avro.Schema avroSchema) { - return new TypeWithNullability(avroSchema); - } - - TypeWithNullability(org.apache.avro.Schema avroSchema) { - if (avroSchema.getType() == org.apache.avro.Schema.Type.UNION) { - List types = avroSchema.getTypes(); - - // optional fields in AVRO have form of: - // {"name": "foo", "type": ["null", "something"]} - - // don't need recursion because nested unions aren't supported in AVRO - List nonNullTypes = - types.stream() - .filter(x -> x.getType() != org.apache.avro.Schema.Type.NULL) - .collect(Collectors.toList()); - - if (nonNullTypes.size() == types.size() || nonNullTypes.isEmpty()) { - // union without `null` or all 'null' union, keep as is. - type = avroSchema; - nullable = false; - } else if (nonNullTypes.size() > 1) { - type = org.apache.avro.Schema.createUnion(nonNullTypes); - nullable = true; - } else { - // One non-null type. - type = nonNullTypes.get(0); - nullable = true; - } - } else { - type = avroSchema; - nullable = false; - } - } - - public Boolean isNullable() { - return nullable; - } - - public org.apache.avro.Schema getType() { - return type; - } - } - - /** Wrapper for fixed byte fields. */ - public static class FixedBytesField { - private final int size; - - private FixedBytesField(int size) { - this.size = size; - } - - /** Create a {@link FixedBytesField} with the specified size. */ - public static FixedBytesField withSize(int size) { - return new FixedBytesField(size); - } - - /** Create a {@link FixedBytesField} from a Beam {@link FieldType}. */ - public static @Nullable FixedBytesField fromBeamFieldType(FieldType fieldType) { - if (fieldType.getTypeName().isLogicalType() - && fieldType.getLogicalType().getIdentifier().equals(FixedBytes.IDENTIFIER)) { - int length = fieldType.getLogicalType(FixedBytes.class).getLength(); - return new FixedBytesField(length); - } else { - return null; - } - } - - /** Create a {@link FixedBytesField} from an AVRO type. */ - public static @Nullable FixedBytesField fromAvroType(org.apache.avro.Schema type) { - if (type.getType().equals(Type.FIXED)) { - return new FixedBytesField(type.getFixedSize()); - } else { - return null; - } - } - - /** Get the size. */ - public int getSize() { - return size; - } - - /** Convert to a Beam type. */ - public FieldType toBeamType() { - return Schema.FieldType.logicalType(FixedBytes.of(size)); - } - - /** Convert to an AVRO type. */ - public org.apache.avro.Schema toAvroType(String name, String namespace) { - return org.apache.avro.Schema.createFixed(name, null, namespace, size); - } - } - - public static class AvroConvertType extends ConvertType { - public AvroConvertType(boolean returnRawType) { - super(returnRawType); - } - - @Override - protected java.lang.reflect.Type convertDefault(TypeDescriptor type) { - if (type.isSubtypeOf(TypeDescriptor.of(GenericFixed.class))) { - return byte[].class; - } else { - return super.convertDefault(type); - } - } - } - - public static class AvroConvertValueForGetter extends ConvertValueForGetter { - AvroConvertValueForGetter(StackManipulation readValue) { - super(readValue); - } - - @Override - protected TypeConversionsFactory getFactory() { - return new AvroTypeConversionFactory(); - } - - @Override - protected StackManipulation convertDefault(TypeDescriptor type) { - if (type.isSubtypeOf(TypeDescriptor.of(GenericFixed.class))) { - // Generate the following code: - // return value.bytes(); - return new Compound( - readValue, - MethodInvocation.invoke( - new ForLoadedType(GenericFixed.class) - .getDeclaredMethods() - .filter( - ElementMatchers.named("bytes") - .and(ElementMatchers.returns(new ForLoadedType(byte[].class)))) - .getOnly())); - } - return super.convertDefault(type); - } - } - - public static class AvroConvertValueForSetter extends ConvertValueForSetter { - AvroConvertValueForSetter(StackManipulation readValue) { - super(readValue); - } - - @Override - protected TypeConversionsFactory getFactory() { - return new AvroTypeConversionFactory(); - } - - @Override - protected StackManipulation convertDefault(TypeDescriptor type) { - final ForLoadedType byteArrayType = new ForLoadedType(byte[].class); - if (type.isSubtypeOf(TypeDescriptor.of(GenericFixed.class))) { - // Generate the following code: - // return new T((byte[]) value); - ForLoadedType loadedType = new ForLoadedType(type.getRawType()); - return new Compound( - TypeCreation.of(loadedType), - Duplication.SINGLE, - // Load the parameter and cast it to a byte[]. - readValue, - TypeCasting.to(byteArrayType), - // Create a new instance that wraps this byte[]. - MethodInvocation.invoke( - loadedType - .getDeclaredMethods() - .filter( - ElementMatchers.isConstructor() - .and(ElementMatchers.takesArguments(byteArrayType))) - .getOnly())); - } - return super.convertDefault(type); - } - } - - static class AvroTypeConversionFactory implements TypeConversionsFactory { - - @Override - public TypeConversion createTypeConversion(boolean returnRawTypes) { - return new AvroConvertType(returnRawTypes); - } - - @Override - public TypeConversion createGetterConversions(StackManipulation readValue) { - return new AvroConvertValueForGetter(readValue); - } - - @Override - public TypeConversion createSetterConversions(StackManipulation readValue) { - return new AvroConvertValueForSetter(readValue); - } - } - - /** Get Beam Field from avro Field. */ - public static Schema.Field toBeamField(org.apache.avro.Schema.Field field) { - TypeWithNullability nullableType = new TypeWithNullability(field.schema()); - FieldType beamFieldType = toFieldType(nullableType); - return Field.of(field.name(), beamFieldType); - } - - /** Get Avro Field from Beam Field. */ - public static org.apache.avro.Schema.Field toAvroField(Schema.Field field, String namespace) { - org.apache.avro.Schema fieldSchema = - getFieldSchema(field.getType(), field.getName(), namespace); - return new org.apache.avro.Schema.Field( - field.getName(), fieldSchema, field.getDescription(), (Object) null); - } - - private AvroUtils() {} - - /** - * Converts AVRO schema to Beam row schema. - * - * @param schema schema of type RECORD - */ - public static Schema toBeamSchema(org.apache.avro.Schema schema) { - Schema.Builder builder = Schema.builder(); - - for (org.apache.avro.Schema.Field field : schema.getFields()) { - Field beamField = toBeamField(field); - if (field.doc() != null) { - beamField = beamField.withDescription(field.doc()); - } - builder.addField(beamField); - } - - return builder.build(); - } - - /** Converts a Beam Schema into an AVRO schema. */ - public static org.apache.avro.Schema toAvroSchema( - Schema beamSchema, @Nullable String name, @Nullable String namespace) { - final String schemaName = Strings.isNullOrEmpty(name) ? "topLevelRecord" : name; - final String schemaNamespace = namespace == null ? "" : namespace; - String childNamespace = - !"".equals(schemaNamespace) ? schemaNamespace + "." + schemaName : schemaName; - List fields = Lists.newArrayList(); - for (Schema.Field field : beamSchema.getFields()) { - org.apache.avro.Schema.Field recordField = toAvroField(field, childNamespace); - fields.add(recordField); - } - return org.apache.avro.Schema.createRecord(schemaName, null, schemaNamespace, false, fields); - } - - public static org.apache.avro.Schema toAvroSchema(Schema beamSchema) { - return toAvroSchema(beamSchema, null, null); - } - - /** - * Strict conversion from AVRO to Beam, strict because it doesn't do widening or narrowing during - * conversion. If Schema is not provided, one is inferred from the AVRO schema. - */ - public static Row toBeamRowStrict(GenericRecord record, @Nullable Schema schema) { - if (schema == null) { - schema = toBeamSchema(record.getSchema()); - } - - Row.Builder builder = Row.withSchema(schema); - org.apache.avro.Schema avroSchema = record.getSchema(); - - for (Schema.Field field : schema.getFields()) { - Object value = record.get(field.getName()); - org.apache.avro.Schema fieldAvroSchema = avroSchema.getField(field.getName()).schema(); - builder.addValue(convertAvroFieldStrict(value, fieldAvroSchema, field.getType())); - } - - return builder.build(); - } - - /** - * Convert from a Beam Row to an AVRO GenericRecord. The Avro Schema is inferred from the Beam - * schema on the row. - */ - public static GenericRecord toGenericRecord(Row row) { - return toGenericRecord(row, null); - } - - /** - * Convert from a Beam Row to an AVRO GenericRecord. If a Schema is not provided, one is inferred - * from the Beam schema on the row. - */ - public static GenericRecord toGenericRecord( - Row row, org.apache.avro.@Nullable Schema avroSchema) { - Schema beamSchema = row.getSchema(); - // Use the provided AVRO schema if present, otherwise infer an AVRO schema from the row - // schema. - if (avroSchema != null && avroSchema.getFields().size() != beamSchema.getFieldCount()) { - throw new IllegalArgumentException( - "AVRO schema doesn't match row schema. Row schema " - + beamSchema - + ". AVRO schema + " - + avroSchema); - } - if (avroSchema == null) { - avroSchema = toAvroSchema(beamSchema); - } - - GenericRecordBuilder builder = new GenericRecordBuilder(avroSchema); - for (int i = 0; i < beamSchema.getFieldCount(); ++i) { - Schema.Field field = beamSchema.getField(i); - builder.set( - field.getName(), - genericFromBeamField( - field.getType(), avroSchema.getField(field.getName()).schema(), row.getValue(i))); - } - return builder.build(); - } - - @SuppressWarnings("unchecked") - public static SerializableFunction getToRowFunction( - Class clazz, org.apache.avro.@Nullable Schema schema) { - if (GenericRecord.class.equals(clazz)) { - Schema beamSchema = toBeamSchema(schema); - return (SerializableFunction) getGenericRecordToRowFunction(beamSchema); - } else { - return new AvroRecordSchema().toRowFunction(TypeDescriptor.of(clazz)); - } - } - - @SuppressWarnings("unchecked") - public static SerializableFunction getFromRowFunction(Class clazz) { - return GenericRecord.class.equals(clazz) - ? (SerializableFunction) getRowToGenericRecordFunction(null) - : new AvroRecordSchema().fromRowFunction(TypeDescriptor.of(clazz)); - } - - public static @Nullable Schema getSchema( - Class clazz, org.apache.avro.@Nullable Schema schema) { - if (schema != null) { - return schema.getType().equals(Type.RECORD) ? toBeamSchema(schema) : null; - } - if (GenericRecord.class.equals(clazz)) { - throw new IllegalArgumentException("No schema provided for getSchema(GenericRecord)"); - } - return new AvroRecordSchema().schemaFor(TypeDescriptor.of(clazz)); - } - - /** Returns a function mapping encoded AVRO {@link GenericRecord}s to Beam {@link Row}s. */ - public static SimpleFunction getAvroBytesToRowFunction(Schema beamSchema) { - return new AvroBytesToRowFn(beamSchema); - } - - private static class AvroBytesToRowFn extends SimpleFunction { - private final AvroCoder coder; - private final Schema beamSchema; - - AvroBytesToRowFn(Schema beamSchema) { - org.apache.avro.Schema avroSchema = toAvroSchema(beamSchema); - coder = AvroCoder.of(avroSchema); - this.beamSchema = beamSchema; - } - - @Override - public Row apply(byte[] bytes) { - try { - ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes); - GenericRecord record = coder.decode(inputStream); - return AvroUtils.toBeamRowStrict(record, beamSchema); - } catch (Exception e) { - throw new AvroRuntimeException( - "Could not decode avro record from given bytes " - + new String(bytes, StandardCharsets.UTF_8), - e); - } - } - } - - /** Returns a function mapping Beam {@link Row}s to encoded AVRO {@link GenericRecord}s. */ - public static SimpleFunction getRowToAvroBytesFunction(Schema beamSchema) { - return new RowToAvroBytesFn(beamSchema); - } - - private static class RowToAvroBytesFn extends SimpleFunction { - private final transient org.apache.avro.Schema avroSchema; - private final AvroCoder coder; - - RowToAvroBytesFn(Schema beamSchema) { - avroSchema = toAvroSchema(beamSchema); - coder = AvroCoder.of(avroSchema); - } - - @Override - public byte[] apply(Row row) { - try { - GenericRecord record = toGenericRecord(row, avroSchema); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - coder.encode(record, outputStream); - return outputStream.toByteArray(); - } catch (Exception e) { - throw new AvroRuntimeException( - String.format("Could not encode avro from given row: %s", row), e); - } - } - } - - /** - * Returns a function mapping AVRO {@link GenericRecord}s to Beam {@link Row}s for use in {@link - * org.apache.beam.sdk.values.PCollection#setSchema}. - */ - public static SerializableFunction getGenericRecordToRowFunction( - @Nullable Schema schema) { - return new GenericRecordToRowFn(schema); - } - - private static class GenericRecordToRowFn implements SerializableFunction { - private final Schema schema; - - GenericRecordToRowFn(Schema schema) { - this.schema = schema; - } - - @Override - public Row apply(GenericRecord input) { - return toBeamRowStrict(input, schema); - } - - @Override - public boolean equals(@Nullable Object other) { - if (this == other) { - return true; - } - if (other == null || getClass() != other.getClass()) { - return false; - } - GenericRecordToRowFn that = (GenericRecordToRowFn) other; - return Objects.equals(this.schema, that.schema); - } - - @Override - public int hashCode() { - return Objects.hash(schema); - } - } - - /** - * Returns a function mapping Beam {@link Row}s to AVRO {@link GenericRecord}s for use in {@link - * org.apache.beam.sdk.values.PCollection#setSchema}. - */ - public static SerializableFunction getRowToGenericRecordFunction( - org.apache.avro.@Nullable Schema avroSchema) { - return new RowToGenericRecordFn(avroSchema); - } - - private static class RowToGenericRecordFn implements SerializableFunction { - private transient org.apache.avro.Schema avroSchema; - - RowToGenericRecordFn(org.apache.avro.@Nullable Schema avroSchema) { - this.avroSchema = avroSchema; - } - - @Override - public GenericRecord apply(Row input) { - return toGenericRecord(input, avroSchema); - } - - @Override - public boolean equals(@Nullable Object other) { - if (this == other) { - return true; - } - if (other == null || getClass() != other.getClass()) { - return false; - } - RowToGenericRecordFn that = (RowToGenericRecordFn) other; - return Objects.equals(this.avroSchema, that.avroSchema); - } - - @Override - public int hashCode() { - return Objects.hash(avroSchema); - } - - private void writeObject(ObjectOutputStream out) throws IOException { - final String avroSchemaAsString = (avroSchema == null) ? null : avroSchema.toString(); - out.writeObject(avroSchemaAsString); - } - - private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { - final String avroSchemaAsString = (String) in.readObject(); - avroSchema = - (avroSchemaAsString == null) - ? null - : new org.apache.avro.Schema.Parser().parse(avroSchemaAsString); - } - } - - /** - * Returns an {@code SchemaCoder} instance for the provided element type. - * - * @param the element type - */ - public static SchemaCoder schemaCoder(TypeDescriptor type) { - @SuppressWarnings("unchecked") - Class clazz = (Class) type.getRawType(); - org.apache.avro.Schema avroSchema = new ReflectData(clazz.getClassLoader()).getSchema(clazz); - Schema beamSchema = toBeamSchema(avroSchema); - return SchemaCoder.of( - beamSchema, type, getToRowFunction(clazz, avroSchema), getFromRowFunction(clazz)); - } - - /** - * Returns an {@code SchemaCoder} instance for the provided element class. - * - * @param the element type - */ - public static SchemaCoder schemaCoder(Class clazz) { - return schemaCoder(TypeDescriptor.of(clazz)); - } - - /** - * Returns an {@code SchemaCoder} instance for the Avro schema. The implicit type is - * GenericRecord. - */ - public static SchemaCoder schemaCoder(org.apache.avro.Schema schema) { - Schema beamSchema = toBeamSchema(schema); - return SchemaCoder.of( - beamSchema, - TypeDescriptor.of(GenericRecord.class), - getGenericRecordToRowFunction(beamSchema), - getRowToGenericRecordFunction(schema)); - } - - /** - * Returns an {@code SchemaCoder} instance for the provided element type using the provided Avro - * schema. - * - *

    If the type argument is GenericRecord, the schema may be arbitrary. Otherwise, the schema - * must correspond to the type provided. - * - * @param the element type - */ - public static SchemaCoder schemaCoder(Class clazz, org.apache.avro.Schema schema) { - return SchemaCoder.of( - getSchema(clazz, schema), - TypeDescriptor.of(clazz), - getToRowFunction(clazz, schema), - getFromRowFunction(clazz)); - } - - /** - * Returns an {@code SchemaCoder} instance based on the provided AvroCoder for the element type. - * - * @param the element type - */ - public static SchemaCoder schemaCoder(AvroCoder avroCoder) { - return schemaCoder(avroCoder.getType(), avroCoder.getSchema()); - } - - private static final class AvroSpecificRecordFieldValueTypeSupplier - implements FieldValueTypeSupplier { - @Override - public List get(Class clazz) { - throw new RuntimeException("Unexpected call."); - } - - @Override - public List get(Class clazz, Schema schema) { - Map mapping = getMapping(schema); - List methods = ReflectUtils.getMethods(clazz); - List types = Lists.newArrayList(); - for (int i = 0; i < methods.size(); ++i) { - Method method = methods.get(i); - if (ReflectUtils.isGetter(method)) { - FieldValueTypeInformation fieldValueTypeInformation = - FieldValueTypeInformation.forGetter(method, i); - String name = mapping.get(fieldValueTypeInformation.getName()); - if (name != null) { - types.add(fieldValueTypeInformation.withName(name)); - } - } - } - - // Return the list ordered by the schema fields. - return StaticSchemaInference.sortBySchema(types, schema); - } - - private Map getMapping(Schema schema) { - Map mapping = Maps.newHashMap(); - for (Field field : schema.getFields()) { - String fieldName = field.getName(); - String getter; - if (fieldName.contains("_")) { - if (Character.isLowerCase(fieldName.charAt(0))) { - // field_name -> fieldName - getter = CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.LOWER_CAMEL, fieldName); - } else { - // FIELD_NAME -> fIELDNAME - // must remove underscore and then convert to match compiled Avro schema getter name - getter = CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_CAMEL, fieldName.replace("_", "")); - } - } else if (Character.isUpperCase(fieldName.charAt(0))) { - // FieldName -> fieldName - getter = CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_CAMEL, fieldName); - } else { - // If the field is in camel case already, then it's the identity mapping. - getter = fieldName; - } - mapping.put(getter, fieldName); - // The Avro compiler might add a $ at the end of a getter to disambiguate. - mapping.put(getter + "$", fieldName); - } - return mapping; - } - } - - private static final class AvroPojoFieldValueTypeSupplier implements FieldValueTypeSupplier { - @Override - public List get(Class clazz) { - List classFields = ReflectUtils.getFields(clazz); - Map types = Maps.newHashMap(); - for (int i = 0; i < classFields.size(); ++i) { - java.lang.reflect.Field f = classFields.get(i); - if (!f.isAnnotationPresent(AvroIgnore.class)) { - FieldValueTypeInformation typeInformation = FieldValueTypeInformation.forField(f, i); - AvroName avroname = f.getAnnotation(AvroName.class); - if (avroname != null) { - typeInformation = typeInformation.withName(avroname.value()); - } - types.put(typeInformation.getName(), typeInformation); - } - } - return Lists.newArrayList(types.values()); - } - } - - /** Get field types for an AVRO-generated SpecificRecord or a POJO. */ - public static List getFieldTypes(Class clazz, Schema schema) { - if (TypeDescriptor.of(clazz).isSubtypeOf(TypeDescriptor.of(SpecificRecord.class))) { - return JavaBeanUtils.getFieldTypes( - clazz, schema, new AvroSpecificRecordFieldValueTypeSupplier()); - } else { - return POJOUtils.getFieldTypes(clazz, schema, new AvroPojoFieldValueTypeSupplier()); - } - } - - /** Get generated getters for an AVRO-generated SpecificRecord or a POJO. */ - public static List getGetters(Class clazz, Schema schema) { - if (TypeDescriptor.of(clazz).isSubtypeOf(TypeDescriptor.of(SpecificRecord.class))) { - return JavaBeanUtils.getGetters( - clazz, - schema, - new AvroSpecificRecordFieldValueTypeSupplier(), - new AvroTypeConversionFactory()); - } else { - return POJOUtils.getGetters( - clazz, schema, new AvroPojoFieldValueTypeSupplier(), new AvroTypeConversionFactory()); - } - } - - /** Get an object creator for an AVRO-generated SpecificRecord. */ - public static SchemaUserTypeCreator getCreator(Class clazz, Schema schema) { - if (TypeDescriptor.of(clazz).isSubtypeOf(TypeDescriptor.of(SpecificRecord.class))) { - return AvroByteBuddyUtils.getCreator((Class) clazz, schema); - } else { - return POJOUtils.getSetFieldCreator( - clazz, schema, new AvroPojoFieldValueTypeSupplier(), new AvroTypeConversionFactory()); - } - } - - /** Converts AVRO schema to Beam field. */ - private static Schema.FieldType toFieldType(TypeWithNullability type) { - Schema.FieldType fieldType = null; - org.apache.avro.Schema avroSchema = type.type; - - LogicalType logicalType = LogicalTypes.fromSchema(avroSchema); - if (logicalType != null) { - if (logicalType instanceof LogicalTypes.Decimal) { - fieldType = FieldType.DECIMAL; - } else if (logicalType instanceof LogicalTypes.TimestampMillis) { - // TODO: There is a desire to move Beam schema DATETIME to a micros representation. When - // this is done, this logical type needs to be changed. - fieldType = FieldType.DATETIME; - } else if (logicalType instanceof LogicalTypes.Date) { - fieldType = FieldType.DATETIME; - } - } - - if (fieldType == null) { - switch (type.type.getType()) { - case RECORD: - fieldType = Schema.FieldType.row(toBeamSchema(avroSchema)); - break; - - case ENUM: - fieldType = FieldType.logicalType(EnumerationType.create(type.type.getEnumSymbols())); - break; - - case ARRAY: - Schema.FieldType elementType = - toFieldType(new TypeWithNullability(avroSchema.getElementType())); - fieldType = Schema.FieldType.array(elementType); - break; - - case MAP: - fieldType = - Schema.FieldType.map( - Schema.FieldType.STRING, - toFieldType(new TypeWithNullability(avroSchema.getValueType()))); - break; - - case FIXED: - fieldType = FixedBytesField.fromAvroType(type.type).toBeamType(); - break; - - case STRING: - fieldType = Schema.FieldType.STRING; - break; - - case BYTES: - fieldType = Schema.FieldType.BYTES; - break; - - case INT: - fieldType = Schema.FieldType.INT32; - break; - - case LONG: - fieldType = Schema.FieldType.INT64; - break; - - case FLOAT: - fieldType = Schema.FieldType.FLOAT; - break; - - case DOUBLE: - fieldType = Schema.FieldType.DOUBLE; - break; - - case BOOLEAN: - fieldType = Schema.FieldType.BOOLEAN; - break; - - case UNION: - fieldType = - FieldType.logicalType( - OneOfType.create( - avroSchema.getTypes().stream() - .map(x -> Field.of(x.getName(), toFieldType(new TypeWithNullability(x)))) - .collect(Collectors.toList()))); - break; - case NULL: - throw new IllegalArgumentException("Can't convert 'null' to FieldType"); - - default: - throw new AssertionError("Unexpected AVRO Schema.Type: " + avroSchema.getType()); - } - } - fieldType = fieldType.withNullable(type.nullable); - return fieldType; - } - - private static org.apache.avro.Schema getFieldSchema( - Schema.FieldType fieldType, String fieldName, String namespace) { - org.apache.avro.Schema baseType; - switch (fieldType.getTypeName()) { - case BYTE: - case INT16: - case INT32: - baseType = org.apache.avro.Schema.create(Type.INT); - break; - - case INT64: - baseType = org.apache.avro.Schema.create(Type.LONG); - break; - - case DECIMAL: - baseType = - LogicalTypes.decimal(Integer.MAX_VALUE) - .addToSchema(org.apache.avro.Schema.create(Type.BYTES)); - break; - - case FLOAT: - baseType = org.apache.avro.Schema.create(Type.FLOAT); - break; - - case DOUBLE: - baseType = org.apache.avro.Schema.create(Type.DOUBLE); - break; - - case STRING: - baseType = org.apache.avro.Schema.create(Type.STRING); - break; - - case DATETIME: - // TODO: There is a desire to move Beam schema DATETIME to a micros representation. When - // this is done, this logical type needs to be changed. - baseType = - LogicalTypes.timestampMillis().addToSchema(org.apache.avro.Schema.create(Type.LONG)); - break; - - case BOOLEAN: - baseType = org.apache.avro.Schema.create(Type.BOOLEAN); - break; - - case BYTES: - baseType = org.apache.avro.Schema.create(Type.BYTES); - break; - - case LOGICAL_TYPE: - String identifier = fieldType.getLogicalType().getIdentifier(); - if (FixedBytes.IDENTIFIER.equals(identifier)) { - FixedBytesField fixedBytesField = - checkNotNull(FixedBytesField.fromBeamFieldType(fieldType)); - baseType = fixedBytesField.toAvroType("fixed", namespace + "." + fieldName); - } else if (VariableBytes.IDENTIFIER.equals(identifier)) { - // treat VARBINARY as bytes as that is what avro supports - baseType = org.apache.avro.Schema.create(Type.BYTES); - } else if (FixedString.IDENTIFIER.equals(identifier) - || "CHAR".equals(identifier) - || "NCHAR".equals(identifier)) { - baseType = - buildHiveLogicalTypeSchema("char", (int) fieldType.getLogicalType().getArgument()); - } else if (VariableString.IDENTIFIER.equals(identifier) - || "NVARCHAR".equals(identifier) - || "VARCHAR".equals(identifier) - || "LONGNVARCHAR".equals(identifier) - || "LONGVARCHAR".equals(identifier)) { - baseType = - buildHiveLogicalTypeSchema("varchar", (int) fieldType.getLogicalType().getArgument()); - } else if (EnumerationType.IDENTIFIER.equals(identifier)) { - EnumerationType enumerationType = fieldType.getLogicalType(EnumerationType.class); - baseType = - org.apache.avro.Schema.createEnum(fieldName, "", "", enumerationType.getValues()); - } else if (OneOfType.IDENTIFIER.equals(identifier)) { - OneOfType oneOfType = fieldType.getLogicalType(OneOfType.class); - baseType = - org.apache.avro.Schema.createUnion( - oneOfType.getOneOfSchema().getFields().stream() - .map(x -> getFieldSchema(x.getType(), x.getName(), namespace)) - .collect(Collectors.toList())); - } else if ("DATE".equals(identifier) || SqlTypes.DATE.getIdentifier().equals(identifier)) { - baseType = LogicalTypes.date().addToSchema(org.apache.avro.Schema.create(Type.INT)); - } else if ("TIME".equals(identifier)) { - baseType = LogicalTypes.timeMillis().addToSchema(org.apache.avro.Schema.create(Type.INT)); - } else { - throw new RuntimeException( - "Unhandled logical type " + fieldType.getLogicalType().getIdentifier()); - } - break; - - case ARRAY: - case ITERABLE: - baseType = - org.apache.avro.Schema.createArray( - getFieldSchema(fieldType.getCollectionElementType(), fieldName, namespace)); - break; - - case MAP: - if (fieldType.getMapKeyType().getTypeName().isStringType()) { - // Avro only supports string keys in maps. - baseType = - org.apache.avro.Schema.createMap( - getFieldSchema(fieldType.getMapValueType(), fieldName, namespace)); - } else { - throw new IllegalArgumentException("Avro only supports maps with string keys"); - } - break; - - case ROW: - baseType = toAvroSchema(fieldType.getRowSchema(), fieldName, namespace); - break; - - default: - throw new IllegalArgumentException("Unexpected type " + fieldType); - } - return fieldType.getNullable() ? ReflectData.makeNullable(baseType) : baseType; - } - - private static @Nullable Object genericFromBeamField( - Schema.FieldType fieldType, org.apache.avro.Schema avroSchema, @Nullable Object value) { - TypeWithNullability typeWithNullability = new TypeWithNullability(avroSchema); - if (!fieldType.getNullable().equals(typeWithNullability.nullable)) { - throw new IllegalArgumentException( - "FieldType " - + fieldType - + " and AVRO schema " - + avroSchema - + " don't have matching nullability"); - } - - if (value == null) { - return value; - } - - switch (fieldType.getTypeName()) { - case BYTE: - case INT16: - case INT32: - case INT64: - case FLOAT: - case DOUBLE: - case BOOLEAN: - return value; - - case STRING: - return new Utf8((String) value); - - case DECIMAL: - BigDecimal decimal = (BigDecimal) value; - LogicalType logicalType = typeWithNullability.type.getLogicalType(); - return new Conversions.DecimalConversion().toBytes(decimal, null, logicalType); - - case DATETIME: - if (typeWithNullability.type.getType() == Type.INT) { - ReadableInstant instant = (ReadableInstant) value; - return (int) Days.daysBetween(Instant.EPOCH, instant).getDays(); - } else if (typeWithNullability.type.getType() == Type.LONG) { - ReadableInstant instant = (ReadableInstant) value; - return (long) instant.getMillis(); - } else { - throw new IllegalArgumentException( - "Can't represent " + fieldType + " as " + typeWithNullability.type.getType()); - } - - case BYTES: - return ByteBuffer.wrap((byte[]) value); - - case LOGICAL_TYPE: - String identifier = fieldType.getLogicalType().getIdentifier(); - if (FixedBytes.IDENTIFIER.equals(identifier)) { - FixedBytesField fixedBytesField = - checkNotNull(FixedBytesField.fromBeamFieldType(fieldType)); - byte[] byteArray = (byte[]) value; - if (byteArray.length != fixedBytesField.getSize()) { - throw new IllegalArgumentException("Incorrectly sized byte array."); - } - return GenericData.get().createFixed(null, (byte[]) value, typeWithNullability.type); - } else if (VariableBytes.IDENTIFIER.equals(identifier)) { - return GenericData.get().createFixed(null, (byte[]) value, typeWithNullability.type); - } else if (FixedString.IDENTIFIER.equals(identifier) - || "CHAR".equals(identifier) - || "NCHAR".equals(identifier)) { - return new Utf8((String) value); - } else if (VariableString.IDENTIFIER.equals(identifier) - || "NVARCHAR".equals(identifier) - || "VARCHAR".equals(identifier) - || "LONGNVARCHAR".equals(identifier) - || "LONGVARCHAR".equals(identifier)) { - return new Utf8((String) value); - } else if (EnumerationType.IDENTIFIER.equals(identifier)) { - EnumerationType enumerationType = fieldType.getLogicalType(EnumerationType.class); - return GenericData.get() - .createEnum( - enumerationType.toString((EnumerationType.Value) value), - typeWithNullability.type); - } else if (OneOfType.IDENTIFIER.equals(identifier)) { - OneOfType oneOfType = fieldType.getLogicalType(OneOfType.class); - OneOfType.Value oneOfValue = (OneOfType.Value) value; - FieldType innerFieldType = oneOfType.getFieldType(oneOfValue); - if (typeWithNullability.nullable && oneOfValue.getValue() == null) { - return null; - } else { - return genericFromBeamField( - innerFieldType.withNullable(false), - typeWithNullability.type.getTypes().get(oneOfValue.getCaseType().getValue()), - oneOfValue.getValue()); - } - } else if ("DATE".equals(identifier)) { - // "Date" is backed by joda.time.Instant - return Days.daysBetween(Instant.EPOCH, (Instant) value).getDays(); - } else if (SqlTypes.DATE.getIdentifier().equals(identifier)) { - // portable SqlTypes.DATE is backed by java.time.LocalDate - return ((java.time.LocalDate) value).toEpochDay(); - } else if ("TIME".equals(identifier)) { - return (int) ((Instant) value).getMillis(); - } else { - throw new RuntimeException("Unhandled logical type " + identifier); - } - - case ARRAY: - case ITERABLE: - Iterable iterable = (Iterable) value; - List translatedArray = Lists.newArrayListWithExpectedSize(Iterables.size(iterable)); - - for (Object arrayElement : iterable) { - translatedArray.add( - genericFromBeamField( - fieldType.getCollectionElementType(), - typeWithNullability.type.getElementType(), - arrayElement)); - } - return translatedArray; - - case MAP: - Map map = Maps.newHashMap(); - Map valueMap = (Map) value; - for (Map.Entry entry : valueMap.entrySet()) { - Utf8 key = new Utf8((String) entry.getKey()); - map.put( - key, - genericFromBeamField( - fieldType.getMapValueType(), - typeWithNullability.type.getValueType(), - entry.getValue())); - } - return map; - - case ROW: - return toGenericRecord((Row) value, typeWithNullability.type); - - default: - throw new IllegalArgumentException("Unsupported type " + fieldType); - } - } - - /** - * Strict conversion from AVRO to Beam, strict because it doesn't do widening or narrowing during - * conversion. - * - * @param value {@link GenericRecord} or any nested value - * @param avroSchema schema for value - * @param fieldType target beam field type - * @return value converted for {@link Row} - */ - @SuppressWarnings("unchecked") - public static @Nullable Object convertAvroFieldStrict( - @Nullable Object value, - @Nonnull org.apache.avro.Schema avroSchema, - @Nonnull Schema.FieldType fieldType) { - if (value == null) { - return null; - } - - TypeWithNullability type = new TypeWithNullability(avroSchema); - LogicalType logicalType = LogicalTypes.fromSchema(type.type); - if (logicalType != null) { - if (logicalType instanceof LogicalTypes.Decimal) { - ByteBuffer byteBuffer = (ByteBuffer) value; - BigDecimal bigDecimal = - new Conversions.DecimalConversion() - .fromBytes(byteBuffer.duplicate(), type.type, logicalType); - return convertDecimal(bigDecimal, fieldType); - } else if (logicalType instanceof LogicalTypes.TimestampMillis) { - if (value instanceof ReadableInstant) { - return convertDateTimeStrict(((ReadableInstant) value).getMillis(), fieldType); - } else { - return convertDateTimeStrict((Long) value, fieldType); - } - } else if (logicalType instanceof LogicalTypes.Date) { - if (value instanceof ReadableInstant) { - int epochDays = Days.daysBetween(Instant.EPOCH, (ReadableInstant) value).getDays(); - return convertDateStrict(epochDays, fieldType); - } else if (value instanceof java.time.LocalDate) { - return convertDateStrict((int) ((java.time.LocalDate) value).toEpochDay(), fieldType); - } else { - return convertDateStrict((Integer) value, fieldType); - } - } - } - - switch (type.type.getType()) { - case FIXED: - return convertFixedStrict((GenericFixed) value, fieldType); - - case BYTES: - return convertBytesStrict((ByteBuffer) value, fieldType); - - case STRING: - return convertStringStrict((CharSequence) value, fieldType); - - case INT: - return convertIntStrict((Integer) value, fieldType); - - case LONG: - return convertLongStrict((Long) value, fieldType); - - case FLOAT: - return convertFloatStrict((Float) value, fieldType); - - case DOUBLE: - return convertDoubleStrict((Double) value, fieldType); - - case BOOLEAN: - return convertBooleanStrict((Boolean) value, fieldType); - - case RECORD: - return convertRecordStrict((GenericRecord) value, fieldType); - - case ENUM: - // enums are either Java enums, or GenericEnumSymbol, - // they don't share common interface, but override toString() - return convertEnumStrict(value, fieldType); - - case ARRAY: - return convertArrayStrict((List) value, type.type.getElementType(), fieldType); - - case MAP: - return convertMapStrict( - (Map) value, type.type.getValueType(), fieldType); - - case UNION: - return convertUnionStrict(value, type.type, fieldType); - - case NULL: - throw new IllegalArgumentException("Can't convert 'null' to non-nullable field"); - - default: - throw new AssertionError("Unexpected AVRO Schema.Type: " + type.type.getType()); - } - } - - private static Object convertRecordStrict(GenericRecord record, Schema.FieldType fieldType) { - checkTypeName(fieldType.getTypeName(), Schema.TypeName.ROW, "record"); - return toBeamRowStrict(record, fieldType.getRowSchema()); - } - - private static Object convertBytesStrict(ByteBuffer bb, Schema.FieldType fieldType) { - checkTypeName(fieldType.getTypeName(), Schema.TypeName.BYTES, "bytes"); - - byte[] bytes = new byte[bb.remaining()]; - bb.duplicate().get(bytes); - return bytes; - } - - private static Object convertFixedStrict(GenericFixed fixed, Schema.FieldType fieldType) { - checkTypeName(fieldType.getTypeName(), TypeName.LOGICAL_TYPE, "fixed"); - checkArgument(FixedBytes.IDENTIFIER.equals(fieldType.getLogicalType().getIdentifier())); - return fixed.bytes().clone(); // clone because GenericFixed is mutable - } - - private static Object convertStringStrict(CharSequence value, Schema.FieldType fieldType) { - checkTypeName(fieldType.getTypeName(), Schema.TypeName.STRING, "string"); - return value.toString(); - } - - private static Object convertIntStrict(Integer value, Schema.FieldType fieldType) { - checkTypeName(fieldType.getTypeName(), Schema.TypeName.INT32, "int"); - return value; - } - - private static Object convertLongStrict(Long value, Schema.FieldType fieldType) { - checkTypeName(fieldType.getTypeName(), Schema.TypeName.INT64, "long"); - return value; - } - - private static Object convertDecimal(BigDecimal value, Schema.FieldType fieldType) { - checkTypeName(fieldType.getTypeName(), TypeName.DECIMAL, "decimal"); - return value; - } - - private static Object convertDateStrict(Integer epochDays, Schema.FieldType fieldType) { - checkTypeName(fieldType.getTypeName(), TypeName.DATETIME, "date"); - return Instant.EPOCH.plus(Duration.standardDays(epochDays)); - } - - private static Object convertDateTimeStrict(Long value, Schema.FieldType fieldType) { - checkTypeName(fieldType.getTypeName(), TypeName.DATETIME, "dateTime"); - return new Instant(value); - } - - private static Object convertFloatStrict(Float value, Schema.FieldType fieldType) { - checkTypeName(fieldType.getTypeName(), Schema.TypeName.FLOAT, "float"); - return value; - } - - private static Object convertDoubleStrict(Double value, Schema.FieldType fieldType) { - checkTypeName(fieldType.getTypeName(), Schema.TypeName.DOUBLE, "double"); - return value; - } - - private static Object convertBooleanStrict(Boolean value, Schema.FieldType fieldType) { - checkTypeName(fieldType.getTypeName(), Schema.TypeName.BOOLEAN, "boolean"); - return value; - } - - private static Object convertEnumStrict(Object value, Schema.FieldType fieldType) { - checkTypeName(fieldType.getTypeName(), TypeName.LOGICAL_TYPE, "enum"); - checkArgument(fieldType.getLogicalType().getIdentifier().equals(EnumerationType.IDENTIFIER)); - EnumerationType enumerationType = fieldType.getLogicalType(EnumerationType.class); - return enumerationType.valueOf(value.toString()); - } - - private static Object convertUnionStrict( - Object value, org.apache.avro.Schema unionAvroSchema, Schema.FieldType fieldType) { - checkTypeName(fieldType.getTypeName(), TypeName.LOGICAL_TYPE, "oneOfType"); - checkArgument(fieldType.getLogicalType().getIdentifier().equals(OneOfType.IDENTIFIER)); - OneOfType oneOfType = fieldType.getLogicalType(OneOfType.class); - int fieldNumber = GenericData.get().resolveUnion(unionAvroSchema, value); - FieldType baseFieldType = oneOfType.getOneOfSchema().getField(fieldNumber).getType(); - Object convertedValue = - convertAvroFieldStrict(value, unionAvroSchema.getTypes().get(fieldNumber), baseFieldType); - return oneOfType.createValue(fieldNumber, convertedValue); - } - - private static Object convertArrayStrict( - List values, org.apache.avro.Schema elemAvroSchema, Schema.FieldType fieldType) { - checkTypeName(fieldType.getTypeName(), Schema.TypeName.ARRAY, "array"); - - List ret = new ArrayList<>(values.size()); - Schema.FieldType elemFieldType = fieldType.getCollectionElementType(); - - for (Object value : values) { - ret.add(convertAvroFieldStrict(value, elemAvroSchema, elemFieldType)); - } - - return ret; - } - - private static Object convertMapStrict( - Map values, - org.apache.avro.Schema valueAvroSchema, - Schema.FieldType fieldType) { - checkTypeName(fieldType.getTypeName(), Schema.TypeName.MAP, "map"); - checkNotNull(fieldType.getMapKeyType()); - checkNotNull(fieldType.getMapValueType()); - - if (!fieldType.getMapKeyType().equals(Schema.FieldType.STRING)) { - throw new IllegalArgumentException( - "Can't convert 'string' map keys to " + fieldType.getMapKeyType()); - } - - Map ret = new HashMap<>(); - - for (Map.Entry value : values.entrySet()) { - ret.put( - convertStringStrict(value.getKey(), fieldType.getMapKeyType()), - convertAvroFieldStrict(value.getValue(), valueAvroSchema, fieldType.getMapValueType())); - } - - return ret; - } - - private static void checkTypeName(Schema.TypeName got, Schema.TypeName expected, String label) { - checkArgument( - got.equals(expected), "Can't convert '%s' to %s, expected: %s", label, got, expected); - } - - /** - * Helper factory to build Avro Logical types schemas for SQL *CHAR types. This method represents - * the logical as Hive does. - */ - private static org.apache.avro.Schema buildHiveLogicalTypeSchema( - String hiveLogicalType, int size) { - String schemaJson = - String.format( - "{\"type\": \"string\", \"logicalType\": \"%s\", \"maxLength\": %s}", - hiveLogicalType, size); - return new org.apache.avro.Schema.Parser().parse(schemaJson); - } -} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/ByteBuddyLocalVariableManager.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/ByteBuddyLocalVariableManager.java index ea3e9e1ba5ef3..e1b99f691f863 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/ByteBuddyLocalVariableManager.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/ByteBuddyLocalVariableManager.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.schemas.utils; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import net.bytebuddy.description.type.TypeDescription.ForLoadedType; import net.bytebuddy.implementation.bytecode.StackManipulation; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/ByteBuddyUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/ByteBuddyUtils.java index b34a3616d921c..c2b33c2d2315b 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/ByteBuddyUtils.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/ByteBuddyUtils.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.schemas.utils; import static org.apache.beam.sdk.util.ByteBuddyUtils.getClassLoadingStrategy; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.lang.reflect.Constructor; import java.lang.reflect.Method; @@ -76,12 +76,12 @@ import org.apache.beam.sdk.util.common.ReflectHelpers; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeParameter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Collections2; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Primitives; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Collections2; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Primitives; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.ClassUtils; import org.checkerframework.checker.nullness.qual.Nullable; @@ -464,7 +464,8 @@ public InstrumentedType prepare(InstrumentedType instrumentedType) { .make() .load( ReflectHelpers.findClassLoader(((Class) fromType).getClassLoader()), - getClassLoadingStrategy(Function.class)) + getClassLoadingStrategy( + ((Class) fromType).getClassLoader() == null ? Function.class : (Class) fromType)) .getLoaded(); } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/ConvertHelpers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/ConvertHelpers.java index 52e737bcb52ce..7f2403035d97f 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/ConvertHelpers.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/ConvertHelpers.java @@ -22,6 +22,7 @@ import java.io.Serializable; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Type; +import java.util.ServiceLoader; import net.bytebuddy.ByteBuddy; import net.bytebuddy.asm.AsmVisitorWrapper; import net.bytebuddy.description.type.TypeDescription; @@ -35,7 +36,6 @@ import net.bytebuddy.implementation.bytecode.member.MethodVariableAccess; import net.bytebuddy.jar.asm.ClassWriter; import net.bytebuddy.matcher.ElementMatchers; -import org.apache.avro.generic.GenericRecord; import org.apache.beam.sdk.schemas.JavaFieldSchema.JavaFieldTypeSupplier; import org.apache.beam.sdk.schemas.NoSuchSchemaException; import org.apache.beam.sdk.schemas.Schema; @@ -45,9 +45,8 @@ import org.apache.beam.sdk.schemas.utils.ByteBuddyUtils.TypeConversionsFactory; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Primitives; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Primitives; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,6 +57,11 @@ "rawtypes" }) public class ConvertHelpers { + private static class SchemaInformationProviders { + private static final ServiceLoader INSTANCE = + ServiceLoader.load(SchemaInformationProvider.class); + } + private static final Logger LOG = LoggerFactory.getLogger(ConvertHelpers.class); /** Return value after converting a schema. */ @@ -80,17 +84,17 @@ public ConvertedSchemaInformation( /** Get the coder used for converting from an inputSchema to a given type. */ public static ConvertedSchemaInformation getConvertedSchemaInformation( Schema inputSchema, TypeDescriptor outputType, SchemaRegistry schemaRegistry) { - ConvertedSchemaInformation convertedSchema = null; - if (outputType.equals(TypeDescriptor.of(Row.class))) { - // If the output is of type Row, then just forward the schema of the input type to the - // output. - convertedSchema = - new ConvertedSchemaInformation<>((SchemaCoder) SchemaCoder.of(inputSchema), null); - } else if (outputType.equals(TypeDescriptor.of(GenericRecord.class))) { - convertedSchema = - new ConvertedSchemaInformation( - (SchemaCoder) AvroUtils.schemaCoder(AvroUtils.toAvroSchema(inputSchema)), null); - } else { + + ConvertedSchemaInformation schemaInformation = null; + // Try to load schema information from loaded providers + for (SchemaInformationProvider provider : SchemaInformationProviders.INSTANCE) { + schemaInformation = provider.getConvertedSchemaInformation(inputSchema, outputType); + if (schemaInformation != null) { + return schemaInformation; + } + } + + if (schemaInformation == null) { // Otherwise, try to find a schema for the output type in the schema registry. Schema outputSchema = null; SchemaCoder outputSchemaCoder = null; @@ -129,9 +133,9 @@ public static ConvertedSchemaInformation getConvertedSchemaInformation( + outputSchema); } } - convertedSchema = new ConvertedSchemaInformation(outputSchemaCoder, unboxedType); + schemaInformation = new ConvertedSchemaInformation(outputSchemaCoder, unboxedType); } - return convertedSchema; + return schemaInformation; } /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/JavaBeanUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/JavaBeanUtils.java index bfb130d639569..6573f25c66e21 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/JavaBeanUtils.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/JavaBeanUtils.java @@ -53,7 +53,7 @@ import org.apache.beam.sdk.schemas.utils.ByteBuddyUtils.TypeConversionsFactory; import org.apache.beam.sdk.schemas.utils.ReflectUtils.ClassWithSchema; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; /** A set of utilities to generate getter and setter classes for JavaBean objects. */ @SuppressWarnings({ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/POJOUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/POJOUtils.java index 49805e74494b9..66b0a5910570f 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/POJOUtils.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/POJOUtils.java @@ -63,7 +63,7 @@ import org.apache.beam.sdk.schemas.utils.ReflectUtils.ClassWithSchema; import org.apache.beam.sdk.util.common.ReflectHelpers; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; /** A set of utilities to generate getter and setter classes for POJOs. */ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/ReflectUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/ReflectUtils.java index db8e6505d7876..f3888a5ed4438 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/ReflectUtils.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/ReflectUtils.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.schemas.utils; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.lang.reflect.Constructor; @@ -37,11 +37,11 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.annotations.SchemaCreate; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimaps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Primitives; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimaps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Primitives; import org.checkerframework.checker.nullness.qual.Nullable; /** A set of reflection helper methods. */ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/RowSchemaInformationProvider.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/RowSchemaInformationProvider.java new file mode 100644 index 0000000000000..b9f65c83bafd4 --- /dev/null +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/RowSchemaInformationProvider.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.schemas.utils; + +import com.google.auto.service.AutoService; +import javax.annotation.Nullable; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.schemas.SchemaCoder; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TypeDescriptor; + +@AutoService(SchemaInformationProvider.class) +public class RowSchemaInformationProvider implements SchemaInformationProvider { + @Override + @Nullable + public ConvertHelpers.ConvertedSchemaInformation getConvertedSchemaInformation( + Schema inputSchema, TypeDescriptor outputType) { + if (outputType.equals(TypeDescriptor.of(Row.class))) { + // If the output is of type Row, then just forward the schema of the input type to the + // output. + return new ConvertHelpers.ConvertedSchemaInformation<>( + (SchemaCoder) SchemaCoder.of(inputSchema), null); + } + return null; + } +} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/SchemaInformationProvider.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/SchemaInformationProvider.java new file mode 100644 index 0000000000000..10727d2a247f9 --- /dev/null +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/SchemaInformationProvider.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.schemas.utils; + +import javax.annotation.Nullable; +import org.apache.beam.sdk.annotations.Internal; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.schemas.utils.ConvertHelpers.ConvertedSchemaInformation; +import org.apache.beam.sdk.values.TypeDescriptor; + +/** Provides an instance of {@link ConvertedSchemaInformation}. Use for internal purposes. */ +@Internal +public interface SchemaInformationProvider { + @Nullable + ConvertedSchemaInformation getConvertedSchemaInformation( + Schema inputSchema, TypeDescriptor outputType); +} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/SchemaZipFold.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/SchemaZipFold.java index 8137c0f12712b..064e4767481a6 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/SchemaZipFold.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/SchemaZipFold.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.schemas.Schema.Field; import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.Schema.TypeName; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * Visitor that zips schemas, and accepts pairs of fields and their types. diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/SelectByteBuddyHelpers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/SelectByteBuddyHelpers.java index eabaf909b44e2..32a418dcbb9ae 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/SelectByteBuddyHelpers.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/SelectByteBuddyHelpers.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.schemas.utils; import static org.apache.beam.sdk.util.ByteBuddyUtils.getClassLoadingStrategy; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import java.lang.reflect.InvocationTargetException; @@ -68,9 +68,9 @@ import org.apache.beam.sdk.schemas.utils.ByteBuddyUtils.ShortCircuitReturnNull; import org.apache.beam.sdk.util.common.ReflectHelpers; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; @SuppressWarnings({ "nullness" // TODO(https://github.com/apache/beam/issues/20497) diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/SelectHelpers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/SelectHelpers.java index 7bf9ec5634a4a..c195718966b4a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/SelectHelpers.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/SelectHelpers.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.schemas.utils; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.Serializable; import java.util.Collections; @@ -35,9 +35,9 @@ import org.apache.beam.sdk.schemas.Schema.TypeName; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; /** Helper methods to select subrows out of rows. */ @SuppressWarnings({ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/StaticSchemaInference.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/StaticSchemaInference.java index 37b0400e491df..72d79adb8288b 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/StaticSchemaInference.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/StaticSchemaInference.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.schemas.utils; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.lang.reflect.ParameterizedType; import java.math.BigDecimal; @@ -34,7 +34,7 @@ import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.logicaltypes.EnumerationType; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.ReadableInstant; /** A set of utilities for inferring a Beam {@link Schema} from static Java types. */ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java index 83457f3207198..942881522cf05 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/StateSpecs.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.state; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Objects; import org.apache.beam.sdk.annotations.Internal; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/Annotations.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/Annotations.java index c119d697e4c70..38b274f94c55b 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/Annotations.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/Annotations.java @@ -19,8 +19,8 @@ import java.lang.annotation.Annotation; import java.util.Arrays; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicate; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicate; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; import org.junit.experimental.categories.Category; /** A utility class for querying annotations. */ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CoderProperties.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CoderProperties.java index a1b3a92a7568e..fb075b301a0fa 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CoderProperties.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/CoderProperties.java @@ -39,12 +39,12 @@ import org.apache.beam.sdk.util.UnownedInputStream; import org.apache.beam.sdk.util.UnownedOutputStream; import org.apache.beam.sdk.util.common.ElementByteSizeObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CountingInputStream; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CountingOutputStream; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CountingInputStream; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CountingOutputStream; /** * Properties for use in {@link Coder} tests. These are implemented with junit assertions rather diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/FileChecksumMatcher.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/FileChecksumMatcher.java index 9be5eaab6d6e6..4285aecc06a5c 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/FileChecksumMatcher.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/FileChecksumMatcher.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.testing; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -26,9 +26,9 @@ import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.ShardedFile; import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashCode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.HashCode; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; import org.hamcrest.Description; import org.hamcrest.TypeSafeMatcher; import org.joda.time.Duration; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherDeserializer.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherDeserializer.java index 5a8228d34dc9f..f2e31ec88b2c1 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherDeserializer.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherDeserializer.java @@ -24,7 +24,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import org.apache.beam.sdk.util.SerializableUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; /** MatcherDeserializer is used with Jackson to enable deserialization of SerializableMatchers. */ class MatcherDeserializer extends JsonDeserializer> { diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherSerializer.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherSerializer.java index 1295b9aa223ad..b0aae962e2cfd 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherSerializer.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/MatcherSerializer.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.databind.SerializerProvider; import java.io.IOException; import org.apache.beam.sdk.util.SerializableUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; /** MatcherSerializer is used with Jackson to enable serialization of SerializableMatchers. */ class MatcherSerializer extends JsonSerializer> { diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PAssert.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PAssert.java index 44d14acb90728..b6d7ee3267a6a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PAssert.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/PAssert.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.testing; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; @@ -73,10 +73,10 @@ import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.ValueInSingleWindow; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SerializableMatchers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SerializableMatchers.java index 8ccc32c0cefa5..749d95960263c 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SerializableMatchers.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SerializableMatchers.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.util.UserCodeException; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SourceTestUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SourceTestUtils.java index aa2afcf63864a..74b9ea047a782 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SourceTestUtils.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SourceTestUtils.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.testing; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; @@ -43,8 +43,8 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; import org.junit.Assert; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/StaticWindows.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/StaticWindows.java index a9615a2ef45d1..fe95616bec9d0 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/StaticWindows.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/StaticWindows.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.testing; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Collection; import java.util.Collections; @@ -28,8 +28,8 @@ import org.apache.beam.sdk.transforms.windowing.NonMergingWindowFn; import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.transforms.windowing.WindowMappingFn; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Duration; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SuccessOrFailure.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SuccessOrFailure.java index 82d0c69d94cda..71704552f8ad6 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SuccessOrFailure.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SuccessOrFailure.java @@ -21,8 +21,8 @@ import org.apache.beam.sdk.coders.DefaultCoder; import org.apache.beam.sdk.coders.SerializableCoder; import org.apache.beam.sdk.util.SerializableThrowable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.Nullable; /** Output of {@link PAssert}. Passed to a conclude function to act upon. */ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java index c739630f62d7c..ee4cf8b439e4e 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.testing; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; @@ -44,12 +44,12 @@ import org.apache.beam.sdk.runners.TransformHierarchy; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicate; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicate; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.experimental.categories.Category; import org.junit.rules.TestRule; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java index c4a5c36388eba..d63753fc8b340 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.testing; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import java.io.IOException; @@ -48,8 +48,8 @@ import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowFnTestUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowFnTestUtils.java index 5fe756e40fe60..c77c10c8faafb 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowFnTestUtils.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowFnTestUtils.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.testing; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThan; @@ -34,8 +34,8 @@ import org.apache.beam.sdk.transforms.windowing.TimestampCombiner; import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; import org.joda.time.ReadableInstant; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowSupplier.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowSupplier.java index d77fa8172c8cf..0f8e38de73a98 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowSupplier.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowSupplier.java @@ -24,9 +24,9 @@ import org.apache.beam.sdk.coders.CoderException; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.CoderUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java index 61b18fc047737..5cf9276c766a1 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateQuantiles.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -46,9 +46,9 @@ import org.apache.beam.sdk.util.common.ElementByteSizeObserver; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.UnmodifiableIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.UnmodifiableIterator; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateUnique.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateUnique.java index feec8ab5b6e0d..fbf134e4cd21f 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateUnique.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ApproximateUnique.java @@ -31,12 +31,12 @@ import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashingOutputStream; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.HashingOutputStream; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java index 422450c89d066..ffbfac460dcd1 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.io.InputStream; @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ThreadLocalRandom; +import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.coders.CannotProvideCoderException; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.CoderException; @@ -66,8 +67,8 @@ import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; /** @@ -202,6 +203,13 @@ private static PerKey perKey( return new PerKey<>(fn, fnDisplayData, false /*fewKeys*/); } + /** Returns a {@link PerKey Combine.PerKey}, and set fewKeys in {@link GroupByKey}. */ + @Internal + public static PerKey fewKeys( + GlobalCombineFn fn) { + return new PerKey<>(fn, displayDataForFn(fn), true /*fewKeys*/); + } + /** Returns a {@link PerKey Combine.PerKey}, and set fewKeys in {@link GroupByKey}. */ private static PerKey fewKeys( GlobalCombineFn fn, @@ -1547,7 +1555,7 @@ public PerKey withSideInputs( */ public PerKeyWithHotKeyFanout withHotKeyFanout( SerializableFunction hotKeyFanout) { - return new PerKeyWithHotKeyFanout<>(fn, fnDisplayData, hotKeyFanout); + return new PerKeyWithHotKeyFanout<>(fn, fnDisplayData, hotKeyFanout, fewKeys); } /** @@ -1569,7 +1577,8 @@ public void populateDisplayData(DisplayData.Builder builder) { public Integer apply(K unused) { return hotKeyFanout; } - }); + }, + fewKeys); } /** Returns the {@link GlobalCombineFn} used by this Combine operation. */ @@ -1616,13 +1625,17 @@ public static class PerKeyWithHotKeyFanout private final DisplayData.ItemSpec> fnDisplayData; private final SerializableFunction hotKeyFanout; + private final boolean fewKeys; + private PerKeyWithHotKeyFanout( GlobalCombineFn fn, DisplayData.ItemSpec> fnDisplayData, - SerializableFunction hotKeyFanout) { + SerializableFunction hotKeyFanout, + boolean fewKeys) { this.fn = fn; this.fnDisplayData = fnDisplayData; this.hotKeyFanout = hotKeyFanout; + this.fewKeys = fewKeys; } @Override @@ -1911,6 +1924,10 @@ public void processElement( } // Combine the hot and cold keys separately. + Combine.PerKey, InputT, AccumT> hotPreCombineTransform = + fewKeys + ? Combine.fewKeys(hotPreCombine, fnDisplayData) + : Combine.perKey(hotPreCombine, fnDisplayData); PCollection>> precombinedHot = split .get(hot) @@ -1919,7 +1936,7 @@ public void processElement( KvCoder.of(inputCoder.getKeyCoder(), VarIntCoder.of()), inputCoder.getValueCoder())) .setWindowingStrategyInternal(preCombineStrategy) - .apply("PreCombineHot", Combine.perKey(hotPreCombine, fnDisplayData)) + .apply("PreCombineHot", hotPreCombineTransform) .apply( "StripNonce", MapElements.via( @@ -1954,10 +1971,14 @@ public KV> apply(KV element) { .setCoder(KvCoder.of(inputCoder.getKeyCoder(), inputOrAccumCoder)); // Combine the union of the pre-processed hot and cold key results. + Combine.PerKey, OutputT> postCombineTransform = + fewKeys + ? Combine.fewKeys(postCombine, fnDisplayData) + : Combine.perKey(postCombine, fnDisplayData); return PCollectionList.of(precombinedHot) .and(preprocessedCold) .apply(Flatten.pCollections()) - .apply("PostCombine", Combine.perKey(postCombine, fnDisplayData)); + .apply("PostCombine", postCombineTransform); } @Override diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFnBase.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFnBase.java index 08f518a4c6b95..e8b027f10cc15 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFnBase.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFnBase.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.transforms.display.HasDisplayData; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * For internal use only; no backwards-compatibility guarantees. diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java index a537b515eed91..81a31d1335098 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.io.InputStream; @@ -41,12 +41,12 @@ import org.apache.beam.sdk.transforms.display.HasDisplayData; import org.apache.beam.sdk.util.CombineFnUtil; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; /** Static utility methods that create combine function instances. */ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Contextful.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Contextful.java index ae19993909ad8..c2f48cad088e8 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Contextful.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Contextful.java @@ -19,7 +19,7 @@ import java.io.Serializable; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; /** Pair of a bit of user code (a "closure") and the {@link Requirements} needed to run it. */ public final class Contextful implements Serializable { diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java index a112c8030d4d6..079379953cd95 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java @@ -17,13 +17,14 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Deque; import java.util.Iterator; import java.util.List; @@ -62,10 +63,10 @@ import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TimestampedValue.TimestampedValueCoder; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; @@ -122,7 +123,7 @@ public class Create { * Otherwise, use {@link Create.Values#withCoder} to set the coder explicitly. */ public static Values of(Iterable elems) { - return new Values<>(elems, Optional.absent(), Optional.absent()); + return new Values<>(elems, Optional.absent(), Optional.absent(), false); } /** @@ -154,7 +155,7 @@ public static Values of(@Nullable T elem, @Nullable T... elems) { */ public static Values empty(Schema schema) { return new Values( - new ArrayList<>(), Optional.of(SchemaCoder.of(schema)), Optional.absent()); + new ArrayList<>(), Optional.of(SchemaCoder.of(schema)), Optional.absent(), false); } /** @@ -167,7 +168,7 @@ public static Values empty(Schema schema) { * the {@code Coder} is provided via the {@code coder} argument. */ public static Values empty(Coder coder) { - return new Values<>(new ArrayList<>(), Optional.of(coder), Optional.absent()); + return new Values<>(new ArrayList<>(), Optional.of(coder), Optional.absent(), false); } /** @@ -181,7 +182,7 @@ public static Values empty(Coder coder) { * must be registered for the class described in the {@code TypeDescriptor}. */ public static Values empty(TypeDescriptor type) { - return new Values<>(new ArrayList<>(), Optional.absent(), Optional.of(type)); + return new Values<>(new ArrayList<>(), Optional.absent(), Optional.of(type), false); } /** @@ -284,7 +285,7 @@ public static class Values extends PTransform> { *

    Note that for {@link Create.Values} with no elements, the {@link VoidCoder} is used. */ public Values withCoder(Coder coder) { - return new Values<>(elems, Optional.of(coder), typeDescriptor); + return new Values<>(elems, Optional.of(coder), typeDescriptor, alwaysUseRead); } /** @@ -321,7 +322,11 @@ public Values withRowSchema(Schema schema) { *

    Note that for {@link Create.Values} with no elements, the {@link VoidCoder} is used. */ public Values withType(TypeDescriptor type) { - return new Values<>(elems, coder, Optional.of(type)); + return new Values<>(elems, coder, Optional.of(type), alwaysUseRead); + } + + public Values alwaysUseRead() { + return new AlwaysUseRead<>(elems, coder, typeDescriptor); } public Iterable getElements() { @@ -362,6 +367,41 @@ public PCollection expand(PBegin input) { e); } try { + if (!alwaysUseRead) { + int numElements = Iterables.size(elems); + if (numElements == 0) { + return input + .apply(Impulse.create()) + .apply( + FlatMapElements.via( + new SimpleFunction>() { + @Override + public Iterable apply(byte[] input) { + return Collections.emptyList(); + } + })) + .setCoder(coder); + } else if (numElements == 1) { + final byte[] encodedElement = + CoderUtils.encodeToByteArray(coder, Iterables.getOnlyElement(elems)); + final Coder capturedCoder = coder; + return input + .apply(Impulse.create()) + .apply( + MapElements.via( + new SimpleFunction() { + @Override + public T apply(byte[] input) { + try { + return CoderUtils.decodeFromByteArray(capturedCoder, encodedElement); + } catch (CoderException exn) { + throw new RuntimeException(exn); + } + } + })) + .setCoder(coder); + } + } CreateSource source = CreateSource.fromIterable(elems, coder); return input.getPipeline().apply(Read.from(source)); } catch (IOException e) { @@ -381,6 +421,9 @@ public PCollection expand(PBegin input) { /** The value type. */ private final transient Optional> typeDescriptor; + /** Whether to unconditionally implement this via reading a CreateSource. */ + private final transient boolean alwaysUseRead; + /** * Constructs a {@code Create.Values} transform that produces a {@link PCollection} containing * the specified elements. @@ -388,10 +431,14 @@ public PCollection expand(PBegin input) { *

    The arguments should not be modified after this is called. */ private Values( - Iterable elems, Optional> coder, Optional> typeDescriptor) { + Iterable elems, + Optional> coder, + Optional> typeDescriptor, + boolean alwaysUseRead) { this.elems = elems; this.coder = coder; this.typeDescriptor = typeDescriptor; + this.alwaysUseRead = alwaysUseRead; } @VisibleForTesting @@ -514,6 +561,14 @@ protected boolean advanceImpl() throws IOException { return true; } } + + /** A subclass to avoid getting re-matched. */ + private static class AlwaysUseRead extends Values { + private AlwaysUseRead( + Iterable elems, Optional> coder, Optional> typeDescriptor) { + super(elems, coder, typeDescriptor, true); + } + } } ///////////////////////////////////////////////////////////////////////////// diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnOutputReceivers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnOutputReceivers.java index 27fbb9754ecb2..4f3719f2cc0eb 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnOutputReceivers.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnOutputReceivers.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Map; import org.apache.beam.sdk.annotations.Internal; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnSchemaInformation.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnSchemaInformation.java index f34165deae247..7576eb71b3a48 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnSchemaInformation.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnSchemaInformation.java @@ -32,7 +32,7 @@ import org.apache.beam.sdk.schemas.utils.SelectHelpers.RowSelectorContainer; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Represents information about how a DoFn extracts schemas. */ @AutoValue diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java index 6932005f4ba3c..f72de2af35efc 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.ArrayList; import java.util.Arrays; @@ -52,8 +52,8 @@ import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.ValueInSingleWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; import org.slf4j.Logger; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/FlatMapElements.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/FlatMapElements.java index b89e4018ddebf..f848ea9a84ba4 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/FlatMapElements.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/FlatMapElements.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import org.apache.beam.sdk.transforms.Contextful.Fn; import org.apache.beam.sdk.transforms.WithFailures.ExceptionElement; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupByKey.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupByKey.java index 63eb914ede03c..d0b320a87654b 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupByKey.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupByKey.java @@ -37,7 +37,7 @@ import org.apache.beam.sdk.values.PCollection.IsBounded; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java index cb8e226fc1ddc..8acda013f4db3 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.io.Serializable; @@ -40,10 +40,10 @@ import org.apache.beam.sdk.util.common.ElementByteSizeObserver; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Duration; import org.joda.time.Instant; import org.slf4j.Logger; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/JsonToRow.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/JsonToRow.java index 3f6df4ab0dbed..d812d299a836d 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/JsonToRow.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/JsonToRow.java @@ -39,7 +39,7 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Latest.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Latest.java index 4b88f0e5f1c3a..cd52f05cad5e4 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Latest.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Latest.java @@ -17,9 +17,9 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Iterator; import org.apache.beam.sdk.coders.CannotProvideCoderException; @@ -30,7 +30,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** * {@link PTransform} and {@link Combine.CombineFn} for computing the latest element in a {@link diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java index a8fe47ef5f53d..6b123d3bd1062 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java @@ -17,9 +17,9 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import org.apache.beam.sdk.transforms.Contextful.Fn; import org.apache.beam.sdk.transforms.WithFailures.ExceptionElement; @@ -159,6 +159,10 @@ public void processElement( /** A DoFn implementation that handles a trivial map call. */ private abstract class MapDoFn extends DoFn { + + /** Holds {@link MapDoFn#outer instance} of enclosing class, used by runner implementations. */ + final MapElements outer = MapElements.this; + @Override public void populateDisplayData(DisplayData.Builder builder) { builder.delegate(MapElements.this); diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapKeys.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapKeys.java index 5d1e68e09d93b..cb6ee84c9aaff 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapKeys.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapKeys.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import org.apache.beam.sdk.transforms.Contextful.Fn; import org.apache.beam.sdk.transforms.WithFailures.ExceptionElement; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapValues.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapValues.java index 65fdd7e7d7020..072fd86b25270 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapValues.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapValues.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import org.apache.beam.sdk.transforms.Contextful.Fn; import org.apache.beam.sdk.transforms.WithFailures.ExceptionElement; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Materializations.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Materializations.java index a8471aecf8c1e..d9b133ee4869b 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Materializations.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Materializations.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import org.apache.beam.model.pipeline.v1.RunnerApi; import org.apache.beam.model.pipeline.v1.RunnerApi.StandardSideInputTypes; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java index 925cf89afe77c..93f233248e4ed 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Mean.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.coders.CoderRegistry; import org.apache.beam.sdk.coders.DoubleCoder; import org.apache.beam.sdk.transforms.Combine.AccumulatingCombineFn.Accumulator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java index ffae530a2da78..7779d2626da52 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.Serializable; import java.lang.reflect.ParameterizedType; @@ -65,7 +65,7 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PeriodicImpulse.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PeriodicImpulse.java index fc836bc3e4757..db4f141ee624f 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PeriodicImpulse.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PeriodicImpulse.java @@ -17,11 +17,15 @@ */ package org.apache.beam.sdk.transforms; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.FixedWindows; import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; +import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; @@ -34,27 +38,58 @@ */ public class PeriodicImpulse extends PTransform> { - Instant startTimestamp = Instant.now(); - Instant stopTimestamp = BoundedWindow.TIMESTAMP_MAX_VALUE; - Duration fireInterval = Duration.standardMinutes(1); + Instant startTimestamp; + Instant stopTimestamp; + @Nullable Duration stopDuration; + Duration fireInterval; boolean applyWindowing = false; + boolean catchUpToNow = true; - private PeriodicImpulse() {} + private PeriodicImpulse() { + this.startTimestamp = Instant.now(); + this.stopTimestamp = BoundedWindow.TIMESTAMP_MAX_VALUE; + this.fireInterval = Duration.standardMinutes(1); + } public static PeriodicImpulse create() { return new PeriodicImpulse(); } + /** + * Assign a timestamp when the pipeliene starts to produce data. + * + *

    Cannot be used along with {@link #stopAfter}. + */ public PeriodicImpulse startAt(Instant startTime) { + checkArgument(stopDuration == null, "startAt and stopAfter cannot be set at the same time"); this.startTimestamp = startTime; return this; } + /** + * Assign a timestamp when the pipeliene stops producing data. + * + *

    Cannot be used along with {@link #stopAfter}. + */ public PeriodicImpulse stopAt(Instant stopTime) { + checkArgument(stopDuration == null, "stopAt and stopAfter cannot be set at the same time"); this.stopTimestamp = stopTime; return this; } + /** + * For internal use only; no backwards-compatibility guarantees. + * + *

    Assign a time interval at which the pipeliene produces data. This is different from setting + * {@link #startAt} and {@link #stopAt}, as the first timestamp is determined at run time + * (pipeline starts processing). + */ + @Internal + public PeriodicImpulse stopAfter(Duration duration) { + this.stopDuration = duration; + return this; + } + public PeriodicImpulse withInterval(Duration interval) { this.fireInterval = interval; return this; @@ -65,22 +100,66 @@ public PeriodicImpulse applyWindowing() { return this; } + /** + * For internal use only; no backwards-compatibility guarantees. + * + *

    The default behavior is that PeriodicImpulse emits all instants until Instant.now(), then + * starts firing at the specified interval. If this is set to false, the PeriodicImpulse will + * perform the interval wait before firing each instant. + */ + @Internal + public PeriodicImpulse catchUpToNow(boolean catchUpToNow) { + this.catchUpToNow = catchUpToNow; + return this; + } + @Override public PCollection expand(PBegin input) { - PCollection result = - input - .apply( - Create.of( - new PeriodicSequence.SequenceDefinition( - startTimestamp, stopTimestamp, fireInterval))) - .apply(PeriodicSequence.create()); + PCollection seqDef; + if (stopDuration != null) { + // nonnull guaranteed + Duration d = stopDuration; + seqDef = + input + .apply(Impulse.create()) + .apply(ParDo.of(new RuntimeSequenceFn(d, fireInterval, catchUpToNow))); + } else { + seqDef = + input.apply( + Create.of( + new PeriodicSequence.SequenceDefinition( + startTimestamp, stopTimestamp, fireInterval, catchUpToNow))); + } + PCollection result = seqDef.apply(PeriodicSequence.create()); if (this.applyWindowing) { result = - result.apply( - Window.into(FixedWindows.of(Duration.millis(fireInterval.getMillis())))); + result.apply(Window.into(FixedWindows.of(Duration.millis(fireInterval.getMillis())))); } - return result; } + + /** + * A DoFn generated a SequenceDefinition at run time. This enables set first element timestamp at + * pipeline start processing data. + */ + private static class RuntimeSequenceFn extends DoFn { + Duration stopDuration; + Duration fireInterval; + boolean catchUpToNow; + + RuntimeSequenceFn(Duration stopDuration, Duration fireInterval, boolean catchUpToNow) { + this.stopDuration = stopDuration; + this.fireInterval = fireInterval; + this.catchUpToNow = catchUpToNow; + } + + @ProcessElement + public void process(ProcessContext c) { + Instant now = Instant.now(); + c.output( + new PeriodicSequence.SequenceDefinition( + now, now.plus(stopDuration), fireInterval, catchUpToNow)); + } + } } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PeriodicSequence.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PeriodicSequence.java index 491b2efa0787a..12cbecd04b02d 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PeriodicSequence.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PeriodicSequence.java @@ -17,11 +17,12 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Objects; +import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.io.range.OffsetRange; import org.apache.beam.sdk.schemas.JavaFieldSchema; import org.apache.beam.sdk.schemas.annotations.DefaultSchema; @@ -32,7 +33,7 @@ import org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimators; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; @@ -56,6 +57,7 @@ public static class SequenceDefinition { public Instant first; public Instant last; public Long durationMilliSec; + public boolean catchUpToNow; public SequenceDefinition() {} @@ -63,6 +65,17 @@ public SequenceDefinition(Instant first, Instant last, Duration duration) { this.first = first; this.last = last; this.durationMilliSec = duration.getMillis(); + this.catchUpToNow = true; + } + + /** catchUpToNow is experimental; no backwards-compatibility guarantees. */ + @Internal + public SequenceDefinition( + Instant first, Instant last, Duration duration, boolean catchUpToNow) { + this.first = first; + this.last = last; + this.durationMilliSec = duration.getMillis(); + this.catchUpToNow = catchUpToNow; } @Override @@ -223,11 +236,17 @@ public ProcessContinuation processElement( estimator.setWatermark(output); nextOutput = nextOutput + interval; } + if (!srcElement.catchUpToNow) { + break; + } } ProcessContinuation continuation = ProcessContinuation.stop(); if (claimSuccess) { - Duration offset = new Duration(Instant.now(), Instant.ofEpochMilli(nextOutput)); + Duration offset = + srcElement.catchUpToNow + ? new Duration(Instant.now(), Instant.ofEpochMilli(nextOutput)) + : new Duration(interval); continuation = ProcessContinuation.resume().withResumeDelay(offset); } return continuation; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Requirements.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Requirements.java index 892b241625c72..cee0b89d73f65 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Requirements.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Requirements.java @@ -23,7 +23,7 @@ import java.util.Collections; import java.util.Set; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; /** Describes the run-time requirements of a {@link Contextful}, such as access to side inputs. */ @SuppressWarnings({ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java index 5dfff822afe05..42f0f6accc7f0 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.UnsignedInteger; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedInteger; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; @@ -153,7 +153,7 @@ public void processElement(@Element T element, OutputReceiver> r) // http://hydronitrogen.com/poor-hash-partitioning-of-timestamps-integers-and-longs-in- // spark.html // This hashing strategy is copied from - // org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Hashing.smear(). + // org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Hashing.smear(). int hashOfShard = 0x1b873593 * Integer.rotateLeft(shard * 0xcc9e2d51, 15); if (numBuckets != null) { UnsignedInteger unsignedNumBuckets = UnsignedInteger.fromIntBits(numBuckets); diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sample.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sample.java index 2288707e99e10..d0017c88f6a49 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sample.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sample.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.ArrayList; import java.util.Iterator; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sets.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sets.java index 16a728167b242..de52c62768a08 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sets.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Sets.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.ArrayList; import java.util.List; @@ -33,7 +33,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** * The {@code PTransform}s that allow to compute different set functions across {@link diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ToString.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ToString.java index 473a59c5778e1..49bf9ad7acdc4 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ToString.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ToString.java @@ -19,7 +19,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; /** * {@link PTransform PTransforms} for converting a {@link PCollection PCollection<?>}, {@link diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java index a8640a8f833c8..e3008a665921e 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Top.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.io.InputStream; @@ -44,7 +44,7 @@ import org.apache.beam.sdk.util.common.ElementByteSizeObserver; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Wait.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Wait.java index b2a193bcb52d7..454fc4dbcd217 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Wait.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Wait.java @@ -28,8 +28,8 @@ import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Watch.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Watch.java index 0c92cf6eacd0d..3f9423610dfa9 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Watch.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Watch.java @@ -20,9 +20,9 @@ import static org.apache.beam.sdk.transforms.Contextful.Fn.Context.wrapProcessContext; import static org.apache.beam.sdk.transforms.DoFn.ProcessContinuation.resume; import static org.apache.beam.sdk.transforms.DoFn.ProcessContinuation.stop; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.io.IOException; @@ -66,16 +66,16 @@ import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; import org.apache.beam.sdk.values.TypeDescriptors.TypeVariableExtractor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Funnel; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Funnels; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashCode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Funnel; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Funnels; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.HashCode; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithFailures.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithFailures.java index 58fffa7c18ad8..849dc36ca073c 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithFailures.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithFailures.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithKeys.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithKeys.java index 7bce88ca4b3f2..a6f66f6208dfd 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithKeys.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithKeys.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import javax.annotation.CheckForNull; import org.apache.beam.sdk.coders.CannotProvideCoderException; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithTimestamps.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithTimestamps.java index 8a94925915fc0..31ff3e6eeb47a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithTimestamps.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/WithTimestamps.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import org.apache.beam.sdk.io.Source; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/display/DisplayData.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/display/DisplayData.java index 7a7abd17e6592..966145f083644 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/display/DisplayData.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/display/DisplayData.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.transforms.display; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -33,11 +33,11 @@ import java.util.Set; import org.apache.beam.sdk.options.ValueProvider; import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java index 3c5f8a5e08ede..ba5bce20d090b 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGbkResult.java @@ -33,10 +33,10 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/KeyedPCollectionTuple.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/KeyedPCollectionTuple.java index 185b71e15f219..51101046560c2 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/KeyedPCollectionTuple.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/KeyedPCollectionTuple.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java index aa10c403336d8..82be7759d8e5a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.transforms.reflect; import static org.apache.beam.sdk.util.common.ReflectHelpers.findClassLoader; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; @@ -105,8 +105,8 @@ import org.apache.beam.sdk.util.UserCodeException; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Primitives; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Primitives; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyOnTimerInvokerFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyOnTimerInvokerFactory.java index 09e6212fa661f..7b9ac7e15c2e8 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyOnTimerInvokerFactory.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyOnTimerInvokerFactory.java @@ -42,12 +42,12 @@ import org.apache.beam.sdk.transforms.DoFn.OnTimer; import org.apache.beam.sdk.transforms.DoFn.TimerId; import org.apache.beam.sdk.transforms.reflect.ByteBuddyDoFnInvokerFactory.DoFnMethodWithExtraParametersDelegation; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.CharMatcher; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LoadingCache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CharMatcher; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; /** * Dynamically generates {@link OnTimerInvoker} instances for invoking a particular {@link TimerId} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java index 02ed0f44fb71c..804ae8b15f1c5 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java @@ -42,7 +42,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.Instant; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java index 0ecf64f2164f6..2af1006a9bea4 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java @@ -51,7 +51,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java index 5b8c60312ad7a..3e41d9d287b9e 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.transforms.reflect; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import com.google.errorprone.annotations.FormatMethod; @@ -97,11 +97,11 @@ import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; import org.apache.beam.sdk.values.TypeParameter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/StableInvokerNamingStrategy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/StableInvokerNamingStrategy.java index 5edf613554032..d7e4aade8b483 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/StableInvokerNamingStrategy.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/StableInvokerNamingStrategy.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.transforms.reflect; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; import com.google.auto.value.AutoValue; import net.bytebuddy.NamingStrategy; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/resourcehints/ResourceHints.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/resourcehints/ResourceHints.java index d7d157a48d845..85cb2df9deab1 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/resourcehints/ResourceHints.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/resourcehints/ResourceHints.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.transforms.resourcehints; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.List; import java.util.Map; @@ -28,9 +28,10 @@ import org.apache.beam.model.pipeline.v1.RunnerApi.StandardResourceHints; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ProtocolMessageEnum; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,6 +49,8 @@ public class ResourceHints { private static final String MIN_RAM_URN = "beam:resources:min_ram_bytes:v1"; private static final String ACCELERATOR_URN = "beam:resources:accelerator:v1"; + private static final String CPU_COUNT_URN = "beam:resources:cpu_count:v1"; + // TODO: reference this from a common location in all packages that use this. private static String getUrn(ProtocolMessageEnum value) { return value.getValueDescriptor().getOptions().getExtension(RunnerApi.beamUrn); @@ -56,6 +59,7 @@ private static String getUrn(ProtocolMessageEnum value) { static { checkState(MIN_RAM_URN.equals(getUrn(StandardResourceHints.Enum.MIN_RAM_BYTES))); checkState(ACCELERATOR_URN.equals(getUrn(StandardResourceHints.Enum.ACCELERATOR))); + checkState(CPU_COUNT_URN.equals(getUrn(StandardResourceHints.Enum.CPU_COUNT))); } private static ImmutableMap hintNameToUrn = @@ -63,12 +67,15 @@ private static String getUrn(ProtocolMessageEnum value) { .put("minRam", MIN_RAM_URN) .put("min_ram", MIN_RAM_URN) // Courtesy alias. .put("accelerator", ACCELERATOR_URN) + .put("cpuCount", CPU_COUNT_URN) + .put("cpu_count", CPU_COUNT_URN) // Courtesy alias. .build(); private static ImmutableMap> parsers = ImmutableMap.>builder() .put(MIN_RAM_URN, s -> new BytesHint(BytesHint.parse(s))) .put(ACCELERATOR_URN, s -> new StringHint(s)) + .put(CPU_COUNT_URN, s -> new IntHint(IntHint.parse(s))) .build(); private static final ResourceHints EMPTY = new ResourceHints(ImmutableMap.of()); @@ -106,7 +113,8 @@ public static ResourceHints fromOptions(PipelineOptions options) { } else { urn = nameOrUrn; } - ResourceHint value = parsers.getOrDefault(urn, s -> new StringHint(s)).apply(stringValue); + ResourceHint value = + Preconditions.checkNotNull(parsers.getOrDefault(urn, StringHint::new)).apply(stringValue); result = result.withHint(urn, value); } return result; @@ -210,6 +218,46 @@ public int hashCode() { } } + /*package*/ static class IntHint extends ResourceHint { + private final int value; + + @Override + public boolean equals(@Nullable Object other) { + if (other == null) { + return false; + } else if (this == other) { + return true; + } else if (other instanceof IntHint) { + return ((IntHint) other).value == value; + } else { + return false; + } + } + + @Override + public int hashCode() { + return Integer.hashCode(value); + } + + public IntHint(int value) { + this.value = value; + } + + public static int parse(String s) { + return Integer.parseInt(s, 10); + } + + @Override + public ResourceHint mergeWithOuter(ResourceHint outer) { + return new IntHint(Math.max(value, ((IntHint) outer).value)); + } + + @Override + public byte[] toBytes() { + return String.valueOf(value).getBytes(Charsets.US_ASCII); + } + } + /** * Sets desired minimal available RAM size to have in transform's execution environment. * @@ -262,6 +310,23 @@ public ResourceHints withHint(String urn, ResourceHint hint) { return new ResourceHints(newHints.build()); } + /** + * Sets desired minimal CPU or vCPU count to have in transform's execution environment. + * + * @param cpuCount specifies a positive CPU count. + */ + public ResourceHints withCPUCount(int cpuCount) { + if (cpuCount <= 0) { + LOG.error( + "Encountered invalid non-positive cpu count hint value {}.\n" + + "The value is ignored. In the future, The method will require an object Long type " + + "and throw an IllegalArgumentException for invalid values.", + cpuCount); + return this; + } + return withHint(CPU_COUNT_URN, new IntHint(cpuCount)); + } + public Map hints() { return hints; } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/resourcehints/ResourceHintsOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/resourcehints/ResourceHintsOptions.java index ec04b5c4104e3..d9ad8b790779b 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/resourcehints/ResourceHintsOptions.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/resourcehints/ResourceHintsOptions.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.options.Description; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Options that are used to control configuration of the remote environment. */ public interface ResourceHintsOptions extends PipelineOptions { diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/ByteKeyRangeTracker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/ByteKeyRangeTracker.java index e35205eb972d9..d429009149059 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/ByteKeyRangeTracker.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/ByteKeyRangeTracker.java @@ -17,16 +17,16 @@ */ package org.apache.beam.sdk.transforms.splittabledofn; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import org.apache.beam.sdk.io.range.ByteKey; import org.apache.beam.sdk.io.range.ByteKeyRange; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker.HasProgress; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Bytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Bytes; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/GrowableOffsetRangeTracker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/GrowableOffsetRangeTracker.java index b4f7ff6eef451..33cd7aeb2d42a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/GrowableOffsetRangeTracker.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/GrowableOffsetRangeTracker.java @@ -17,12 +17,12 @@ */ package org.apache.beam.sdk.transforms.splittabledofn; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.math.BigDecimal; import java.math.MathContext; import org.apache.beam.sdk.io.range.OffsetRange; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Suppliers; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Suppliers; /** * An {@link OffsetRangeTracker} for tracking a growable offset range. {@code Long.MAX_VALUE} is diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTracker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTracker.java index ba6355e6f326f..88ee2d2ac132e 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTracker.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/OffsetRangeTracker.java @@ -17,15 +17,15 @@ */ package org.apache.beam.sdk.transforms.splittabledofn; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.math.BigDecimal; import java.math.MathContext; import org.apache.beam.sdk.io.range.OffsetRange; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker.HasProgress; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/WatermarkEstimators.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/WatermarkEstimators.java index 4cc034f00413e..4997af837e555 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/WatermarkEstimators.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/WatermarkEstimators.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.transforms.splittabledofn; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterAll.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterAll.java index e6b9d4fd8b6a4..b5838b775211b 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterAll.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterAll.java @@ -17,13 +17,13 @@ */ package org.apache.beam.sdk.transforms.windowing; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Arrays; import java.util.List; import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; import org.joda.time.Instant; /** A composite {@link Trigger} that fires when all of its sub-triggers are ready. */ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterEach.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterEach.java index c419715d502ef..654c7a466478a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterEach.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterEach.java @@ -17,11 +17,11 @@ */ package org.apache.beam.sdk.transforms.windowing; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Arrays; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; import org.joda.time.Instant; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterFirst.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterFirst.java index 1f250105a8004..b8118848c96e0 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterFirst.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterFirst.java @@ -17,12 +17,12 @@ */ package org.apache.beam.sdk.transforms.windowing; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Arrays; import java.util.List; import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; import org.joda.time.Instant; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTime.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTime.java index 864c3d078ba1e..c9612c0692072 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTime.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterProcessingTime.java @@ -22,7 +22,7 @@ import java.util.Locale; import java.util.Objects; import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterSynchronizedProcessingTime.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterSynchronizedProcessingTime.java index f770876220d33..b3f25c6b14c3d 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterSynchronizedProcessingTime.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterSynchronizedProcessingTime.java @@ -19,7 +19,7 @@ import java.util.List; import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterWatermark.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterWatermark.java index df8e710f76336..295b277267cef 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterWatermark.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/AfterWatermark.java @@ -17,13 +17,13 @@ */ package org.apache.beam.sdk.transforms.windowing; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.List; import java.util.Objects; import org.apache.beam.sdk.state.TimeDomain; import org.apache.beam.sdk.transforms.windowing.Trigger.OnceTrigger; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTrigger.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTrigger.java index d2ea3f12f9b18..5a16c818da74c 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTrigger.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/OrFinallyTrigger.java @@ -19,7 +19,7 @@ import java.util.Arrays; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.Instant; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java index e75a3ee39b6e3..6e4c694d48e3f 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.transforms.windowing; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.io.InputStream; @@ -30,8 +30,9 @@ import org.apache.beam.sdk.transforms.DoFn.WindowedContext; import org.apache.beam.sdk.transforms.GroupByKey; import org.apache.beam.sdk.util.VarInt; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** @@ -357,7 +358,7 @@ public void encode(PaneInfo value, final OutputStream outStream) @Override public PaneInfo decode(final InputStream inStream) throws CoderException, IOException { byte keyAndTag = (byte) inStream.read(); - PaneInfo base = BYTE_TO_PANE_INFO.get((byte) (keyAndTag & 0x0F)); + PaneInfo base = Preconditions.checkNotNull(BYTE_TO_PANE_INFO.get((byte) (keyAndTag & 0x0F))); long index, onTimeIndex; switch (Encoding.fromTag(keyAndTag)) { case FIRST: diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/TimestampCombiner.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/TimestampCombiner.java index 982fa610f9f7e..b5cedebf86aae 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/TimestampCombiner.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/TimestampCombiner.java @@ -17,14 +17,14 @@ */ package org.apache.beam.sdk.transforms.windowing; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Arrays; import java.util.Collections; import org.apache.beam.sdk.transforms.Combine; import org.apache.beam.sdk.transforms.GroupByKey; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; import org.joda.time.Instant; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Trigger.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Trigger.java index 72c9062e8cf0b..16daa453021ac 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Trigger.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Trigger.java @@ -24,8 +24,8 @@ import java.util.Objects; import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.transforms.GroupByKey; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java index 591f1fb9f8ef7..c22aa35d8f3aa 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java @@ -29,8 +29,8 @@ import org.apache.beam.sdk.values.PCollectionList; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java index 6a5107ad616d2..21342d3670394 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java @@ -36,23 +36,23 @@ import java.util.Set; import java.util.regex.Pattern; import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicate; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimaps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.reflect.ClassPath; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.reflect.Invokable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.reflect.Parameter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.reflect.TypeToken; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicate; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimaps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.reflect.ClassPath; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.reflect.Invokable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.reflect.Parameter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.reflect.TypeToken; import org.hamcrest.Description; import org.hamcrest.Matcher; import org.hamcrest.StringDescription; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/AppliedCombineFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/AppliedCombineFn.java index a09f6e331794a..38d692083c05f 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/AppliedCombineFn.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/AppliedCombineFn.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.transforms.CombineFnBase.GlobalCombineFn; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** * A {@link GlobalCombineFn} with a fixed accumulator coder. This is created from a specific diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BucketingFunction.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BucketingFunction.java index 638b3459787d4..3defd1172c0af 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BucketingFunction.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BucketingFunction.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.HashMap; import java.util.Map; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStream.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStream.java index 3470dbfd55bdb..9e33a94d2dcfd 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStream.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStream.java @@ -24,7 +24,7 @@ import javax.annotation.concurrent.NotThreadSafe; import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.coders.Coder.Context; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** * Provides an efficient encoding for {@link Iterable}s containing small values by buffering up to diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ByteStringOutputStream.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ByteStringOutputStream.java index 86f5cecce1696..2908e637c55a2 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ByteStringOutputStream.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ByteStringOutputStream.java @@ -21,7 +21,7 @@ import javax.annotation.concurrent.NotThreadSafe; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.UnsafeByteOperations; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; /** * An {@link OutputStream} that produces {@link ByteString}s. diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java index 416ca95c7e9aa..cbd395ff88073 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CoderUtils.java @@ -28,8 +28,8 @@ import org.apache.beam.sdk.coders.CoderException; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; /** Utilities for working with Coders. */ @SuppressWarnings({ diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExplicitShardedFile.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExplicitShardedFile.java index 0d4188e57699e..b31ef7b7997c3 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExplicitShardedFile.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExplicitShardedFile.java @@ -28,10 +28,10 @@ import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.io.FileSystems; import org.apache.beam.sdk.io.fs.MatchResult.Metadata; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CharStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CharStreams; import org.joda.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FilePatternMatchingShardedFile.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FilePatternMatchingShardedFile.java index 2cb31afbb026e..b00f1a84b3ce4 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FilePatternMatchingShardedFile.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FilePatternMatchingShardedFile.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.io.Reader; @@ -30,10 +30,10 @@ import org.apache.beam.sdk.io.FileSystems; import org.apache.beam.sdk.io.fs.MatchResult.Metadata; import org.apache.beam.sdk.io.fs.ResourceId; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CharStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CharStreams; import org.joda.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FluentBackoff.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FluentBackoff.java index 656c2be8b29c7..0d45ff0a57fbe 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FluentBackoff.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/FluentBackoff.java @@ -17,9 +17,9 @@ */ package org.apache.beam.sdk.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.joda.time.Duration; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/HistogramData.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/HistogramData.java index ca747b28f0a36..b28e1cfd5af2e 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/HistogramData.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/HistogramData.java @@ -22,7 +22,7 @@ import java.math.RoundingMode; import java.util.Arrays; import java.util.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.math.DoubleMath; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.math.DoubleMath; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/InstanceBuilder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/InstanceBuilder.java index 2e3a7edff7b27..467f5ab61a3d5 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/InstanceBuilder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/InstanceBuilder.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; @@ -27,7 +27,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MovingFunction.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MovingFunction.java index a7e4c452578ee..320c362cc877a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MovingFunction.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MovingFunction.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Arrays; import org.apache.beam.sdk.transforms.Combine; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NameUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NameUtils.java index a721866631076..84597372f0dbf 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NameUtils.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NameUtils.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -25,8 +25,8 @@ import org.apache.beam.sdk.transforms.Combine.CombineFn; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; /** Helpers for extracting the name of objects and classes. */ @Internal diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NoopLock.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NoopLock.java index 0fc822987a6ed..36454a125d672 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NoopLock.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NoopLock.java @@ -21,21 +21,19 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; -import javax.annotation.Nonnull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.NonNull; /** * A lock which can always be acquired. It should not be used when a proper lock is required, but it * is useful as a performance optimization when locking is not necessary but the code paths have to * be shared between the locking and the non-locking variant. */ -@SuppressWarnings({ - "nullness" // TODO(https://github.com/apache/beam/issues/20497) -}) public class NoopLock implements Lock, Serializable { - private static NoopLock instance; + private static @MonotonicNonNull NoopLock instance; - public static NoopLock get() { + public static @NonNull NoopLock get() { if (instance == null) { instance = new NoopLock(); } @@ -56,14 +54,13 @@ public boolean tryLock() { } @Override - public boolean tryLock(long time, @Nonnull TimeUnit unit) { + public boolean tryLock(long time, TimeUnit unit) { return true; } @Override public void unlock() {} - @Nonnull @Override public Condition newCondition() { throw new UnsupportedOperationException("Not implemented"); diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java index 2dbaac8030285..e9dcdb84140f9 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.io.Reader; @@ -32,11 +32,11 @@ import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.io.FileSystems; import org.apache.beam.sdk.io.fs.MatchResult.Metadata; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CharStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CharStreams; import org.joda.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Preconditions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Preconditions.java index d32509c8d1241..7bb08039c81d3 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Preconditions.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Preconditions.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings.lenientFormat; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings.lenientFormat; import com.google.errorprone.annotations.CanIgnoreReturnValue; import org.apache.beam.sdk.annotations.Internal; @@ -27,7 +27,7 @@ /** * Beam-specific variants of {@link - * org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions} that throws more + * org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions} that throws more * appropriate exception classes while being static analysis friendly. */ @Internal @@ -494,7 +494,7 @@ public class Preconditions { @Nullable String errorMessageTemplate, @Nullable Object... errorMessageArgs) { if (obj == null) { - throw new IllegalArgumentException(lenientFormat(errorMessageTemplate, errorMessageArgs)); + throw new IllegalStateException(lenientFormat(errorMessageTemplate, errorMessageArgs)); } return obj; } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java index 0a4b1f0b6dc5c..e147620c35d3c 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReleaseInfo.java @@ -24,7 +24,7 @@ import java.util.Map; import java.util.Properties; import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/RowJson.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/RowJson.java index 1da4f9edbfa58..c63f673ade214 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/RowJson.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/RowJson.java @@ -41,7 +41,7 @@ import static org.apache.beam.sdk.util.RowJsonValueExtractors.stringValueExtractor; import static org.apache.beam.sdk.util.RowJsonValueExtractors.timeValueExtractor; import static org.apache.beam.sdk.values.Row.toRow; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList.toImmutableList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList.toImmutableList; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; @@ -66,9 +66,9 @@ import org.apache.beam.sdk.schemas.logicaltypes.SqlTypes; import org.apache.beam.sdk.util.RowJsonValueExtractors.ValueExtractor; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.ReadableInstant; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableUtils.java index b1c0679d43503..54f2a6572f6f0 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableUtils.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableUtils.java @@ -19,7 +19,7 @@ import static org.apache.beam.sdk.util.CoderUtils.decodeFromByteArray; import static org.apache.beam.sdk.util.CoderUtils.encodeToByteArray; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ShardedKey.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ShardedKey.java index 049e32536a637..dbc7b2e215e2a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ShardedKey.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ShardedKey.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.io.InputStream; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/StringUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/StringUtils.java index 12bbdba315197..13105fb6c02c4 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/StringUtils.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/StringUtils.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.ArrayList; import java.util.List; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/UnboundedScheduledExecutorService.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/UnboundedScheduledExecutorService.java index 1c243ca2ada3b..6b48c08bca0c6 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/UnboundedScheduledExecutorService.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/UnboundedScheduledExecutorService.java @@ -42,11 +42,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.math.LongMath; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.math.LongMath; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Longs; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.checkerframework.checker.nullness.qual.KeyForBottom; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/UnownedInputStream.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/UnownedInputStream.java index f18ce4e3bb04d..acf70ed6b00d1 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/UnownedInputStream.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/UnownedInputStream.java @@ -22,7 +22,7 @@ import java.io.InputStream; import java.io.OutputStream; import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/UnownedOutputStream.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/UnownedOutputStream.java index f8a996cf969eb..beec695e20c86 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/UnownedOutputStream.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/UnownedOutputStream.java @@ -21,7 +21,7 @@ import java.io.IOException; import java.io.OutputStream; import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java index ad0649290f591..d11166001f053 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowedValue.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -46,8 +46,8 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.transforms.windowing.PaneInfo.PaneInfoCoder; import org.apache.beam.sdk.util.common.ElementByteSizeObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ZipFiles.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ZipFiles.java index 3902c3270a938..e8109cfefb739 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ZipFiles.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ZipFiles.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.util; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.BufferedOutputStream; import java.io.File; @@ -33,11 +33,11 @@ import java.util.zip.ZipFile; import java.util.zip.ZipOutputStream; import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteSource; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CharSource; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteSource; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CharSource; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; /** * Functions for zipping a directory (including a subdirectory) into a ZIP-file or unzipping it diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/common/ReflectHelpers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/common/ReflectHelpers.java index 99bc280adfc33..62a43a6763b09 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/common/ReflectHelpers.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/common/ReflectHelpers.java @@ -18,8 +18,8 @@ package org.apache.beam.sdk.util.common; import static java.util.Arrays.asList; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.lang.annotation.Annotation; import java.lang.reflect.AnnotatedElement; @@ -38,12 +38,12 @@ import java.util.Queue; import java.util.ServiceLoader; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSortedSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Queues; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSortedSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Queues; /** Utilities for working with with {@link Class Classes} and {@link Method Methods}. */ @SuppressWarnings({"nullness", "keyfor"}) // TODO(https://github.com/apache/beam/issues/20497) diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/FailsafeValueInSingleWindow.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/FailsafeValueInSingleWindow.java index ac9b81996fda1..558552246692a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/FailsafeValueInSingleWindow.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/FailsafeValueInSingleWindow.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.coders.StructuredCoder; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.PaneInfo; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/KV.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/KV.java index 0c095ade1555c..0744c2e008e4a 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/KV.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/KV.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.transforms.GroupByKey; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.SerializableComparator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java index f2b1219eafe02..8c26e55f940e7 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.values; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Collections; import java.util.Map; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionList.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionList.java index 6f8d7b764c723..e5b342510e6a5 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionList.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionList.java @@ -25,8 +25,8 @@ import org.apache.beam.sdk.transforms.Flatten; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.Partition; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionRowTuple.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionRowTuple.java index 4d1a332592634..0e7c52c4ae7db 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionRowTuple.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionRowTuple.java @@ -23,7 +23,7 @@ import java.util.Objects; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java index ffeff2d719f77..2b5b8eebd352f 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionTuple.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection.IsBounded; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java index a745b04976107..99ee7c57041ed 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.values; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.io.InputStream; @@ -58,18 +58,18 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.WindowMappingFn; import org.apache.beam.sdk.util.CoderUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Suppliers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSortedMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Ints; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Suppliers; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSortedMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Ints; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Longs; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java index b9d747ea390b2..b60b5a28b1bdd 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.values; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.annotations.Internal; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValues.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValues.java index 4f42c7d4b112c..94add5702c811 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValues.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValues.java @@ -21,7 +21,7 @@ import java.util.Map; import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/Row.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/Row.java index 18262164c90d8..f5c6c7fcf34a4 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/Row.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/Row.java @@ -17,9 +17,9 @@ */ package org.apache.beam.sdk.values; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.Serializable; import java.math.BigDecimal; @@ -47,7 +47,7 @@ import org.apache.beam.sdk.values.RowUtils.FieldOverrides; import org.apache.beam.sdk.values.RowUtils.RowFieldMatcher; import org.apache.beam.sdk.values.RowUtils.RowPosition; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTime; import org.joda.time.ReadableDateTime; @@ -786,12 +786,12 @@ public FieldValueBuilder withFieldValues(Map values) { // withFieldValue or // withFieldValues. - public Builder addValue(@Nullable Object values) { - this.values.add(values); + public Builder addValue(@Nullable Object value) { + this.values.add(value); return this; } - public Builder addValues(List values) { + public Builder addValues(List<@Nullable Object> values) { this.values.addAll(values); return this; } @@ -822,7 +822,7 @@ public Builder addIterable(Iterable values) { // method is largely // used internal to Beam. @Internal - public Row attachValues(List attachedValues) { + public Row attachValues(List<@Nullable Object> attachedValues) { checkState(this.values.isEmpty()); return new RowWithStorage(schema, attachedValues); } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/RowUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/RowUtils.java index 110dc03ad8baf..8a312f1ff0b81 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/RowUtils.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/RowUtils.java @@ -32,9 +32,9 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.Schema.LogicalType; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; import org.joda.time.base.AbstractInstant; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TaggedPValue.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TaggedPValue.java index 51cdb5afa7412..ded5bef31131f 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TaggedPValue.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TaggedPValue.java @@ -19,7 +19,7 @@ import com.google.auto.value.AutoValue; import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** * For internal use only; no backwards-compatibility guarantees. diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java index bc318f601dc0e..753ee4c2b9f8c 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TimestampedValue.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.values; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.io.InputStream; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TupleTag.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TupleTag.java index 9647e3d0d9f38..12d8a962a8756 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TupleTag.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TupleTag.java @@ -20,8 +20,8 @@ import java.io.Serializable; import java.util.Random; import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashMultiset; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multiset; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashMultiset; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multiset; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TupleTagList.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TupleTagList.java index 9578cc5dbc495..ac0f3518a4271 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TupleTagList.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TupleTagList.java @@ -22,8 +22,8 @@ import java.util.Collections; import java.util.List; import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * A {@link TupleTagList} is an immutable list of heterogeneously typed {@link TupleTag TupleTags}. diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TypeDescriptor.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TypeDescriptor.java index 9831c561388bd..045662d1680cf 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TypeDescriptor.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TypeDescriptor.java @@ -24,11 +24,11 @@ import java.lang.reflect.Type; import java.lang.reflect.TypeVariable; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.reflect.Invokable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.reflect.Parameter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.reflect.TypeResolver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.reflect.TypeToken; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.reflect.Invokable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.reflect.Parameter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.reflect.TypeResolver; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.reflect.TypeToken; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TypeParameter.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TypeParameter.java index bac7a37800548..8a777e816bbda 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TypeParameter.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/TypeParameter.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.values; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueInSingleWindow.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueInSingleWindow.java index aed94541fe2bd..d91bb8057a058 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueInSingleWindow.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueInSingleWindow.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.coders.StructuredCoder; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.PaneInfo; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java index a0664f446cc08..024376691b3fe 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/ValueWithRecordId.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.StructuredCoder; import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/WindowingStrategy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/WindowingStrategy.java index 7c6a249eaa8ac..75ca46078ae14 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/WindowingStrategy.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/WindowingStrategy.java @@ -29,8 +29,8 @@ import org.apache.beam.sdk.transforms.windowing.Window.ClosingBehavior; import org.apache.beam.sdk.transforms.windowing.Window.OnTimeBehavior; import org.apache.beam.sdk.transforms.windowing.WindowFn; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/PipelineTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/PipelineTest.java index 711021f49381f..6279f53bb9b13 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/PipelineTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/PipelineTest.java @@ -66,8 +66,8 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.TaggedPValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.hamcrest.Matchers; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/AvroCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/AvroCoderTest.java deleted file mode 100644 index 76e1568dfab9b..0000000000000 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/AvroCoderTest.java +++ /dev/null @@ -1,1106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.coders; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import com.esotericsoftware.kryo.Kryo; -import com.esotericsoftware.kryo.io.Input; -import com.esotericsoftware.kryo.io.Output; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; -import org.apache.avro.AvroRuntimeException; -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.reflect.AvroName; -import org.apache.avro.reflect.AvroSchema; -import org.apache.avro.reflect.ReflectData; -import org.apache.avro.reflect.Stringable; -import org.apache.avro.reflect.Union; -import org.apache.avro.specific.SpecificData; -import org.apache.avro.specific.SpecificRecord; -import org.apache.avro.util.Utf8; -import org.apache.beam.sdk.coders.Coder.Context; -import org.apache.beam.sdk.coders.Coder.NonDeterministicException; -import org.apache.beam.sdk.schemas.TestAvro; -import org.apache.beam.sdk.schemas.TestAvroNested; -import org.apache.beam.sdk.schemas.TestEnum; -import org.apache.beam.sdk.schemas.fixed4; -import org.apache.beam.sdk.testing.CoderProperties; -import org.apache.beam.sdk.testing.InterceptingUrlClassLoader; -import org.apache.beam.sdk.testing.NeedsRunner; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.util.CoderUtils; -import org.apache.beam.sdk.util.InstanceBuilder; -import org.apache.beam.sdk.util.SerializableUtils; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; -import org.hamcrest.Description; -import org.hamcrest.Matcher; -import org.hamcrest.Matchers; -import org.hamcrest.TypeSafeMatcher; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.joda.time.LocalDate; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.objenesis.strategy.StdInstantiatorStrategy; - -/** Tests for {@link AvroCoder}. */ -@RunWith(JUnit4.class) -public class AvroCoderTest { - - public static final DateTime DATETIME_A = - new DateTime().withDate(1994, 10, 31).withZone(DateTimeZone.UTC); - public static final DateTime DATETIME_B = - new DateTime().withDate(1997, 4, 25).withZone(DateTimeZone.UTC); - private static final TestAvroNested AVRO_NESTED_SPECIFIC_RECORD = new TestAvroNested(true, 42); - private static final TestAvro AVRO_SPECIFIC_RECORD = - new TestAvro( - true, - 43, - 44L, - 44.1f, - 44.2d, - "mystring", - ByteBuffer.wrap(new byte[] {1, 2, 3, 4}), - new fixed4(new byte[] {1, 2, 3, 4}), - new LocalDate(1979, 3, 14), - new DateTime().withDate(1979, 3, 14).withTime(1, 2, 3, 4), - TestEnum.abc, - AVRO_NESTED_SPECIFIC_RECORD, - ImmutableList.of(AVRO_NESTED_SPECIFIC_RECORD, AVRO_NESTED_SPECIFIC_RECORD), - ImmutableMap.of("k1", AVRO_NESTED_SPECIFIC_RECORD, "k2", AVRO_NESTED_SPECIFIC_RECORD)); - - @DefaultCoder(AvroCoder.class) - private static class Pojo { - public String text; - public int count; - - @AvroSchema("{\"type\": \"long\", \"logicalType\": \"timestamp-millis\"}") - public DateTime timestamp; - - // Empty constructor required for Avro decoding. - @SuppressWarnings("unused") - public Pojo() {} - - public Pojo(String text, int count, DateTime timestamp) { - this.text = text; - this.count = count; - this.timestamp = timestamp; - } - - // auto-generated - @Override - public boolean equals(@Nullable Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - Pojo pojo = (Pojo) o; - - if (count != pojo.count) { - return false; - } - if (text != null ? !text.equals(pojo.text) : pojo.text != null) { - return false; - } - if (timestamp != null ? !timestamp.equals(pojo.timestamp) : pojo.timestamp != null) { - return false; - } - - return true; - } - - @Override - public int hashCode() { - return 0; - } - - @Override - public String toString() { - return "Pojo{" - + "text='" - + text - + '\'' - + ", count=" - + count - + ", timestamp=" - + timestamp - + '}'; - } - } - - private static class GetTextFn extends DoFn { - @ProcessElement - public void processElement(ProcessContext c) { - c.output(c.element().text); - } - } - - @Rule public TestPipeline pipeline = TestPipeline.create(); - - @Test - public void testAvroCoderEncoding() throws Exception { - AvroCoder coder = AvroCoder.of(Pojo.class); - CoderProperties.coderSerializable(coder); - AvroCoder copy = SerializableUtils.clone(coder); - - Pojo pojo = new Pojo("foo", 3, DATETIME_A); - Pojo equalPojo = new Pojo("foo", 3, DATETIME_A); - Pojo otherPojo = new Pojo("bar", -19, DATETIME_B); - CoderProperties.coderConsistentWithEquals(coder, pojo, equalPojo); - CoderProperties.coderConsistentWithEquals(copy, pojo, equalPojo); - CoderProperties.coderConsistentWithEquals(coder, pojo, otherPojo); - CoderProperties.coderConsistentWithEquals(copy, pojo, otherPojo); - } - - /** - * Tests that {@link AvroCoder} works around issues in Avro where cache classes might be from the - * wrong ClassLoader, causing confusing "Cannot cast X to X" error messages. - */ - @SuppressWarnings("ReturnValueIgnored") - @Test - public void testTwoClassLoaders() throws Exception { - ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); - ClassLoader loader1 = - new InterceptingUrlClassLoader(contextClassLoader, AvroCoderTestPojo.class.getName()); - ClassLoader loader2 = - new InterceptingUrlClassLoader(contextClassLoader, AvroCoderTestPojo.class.getName()); - - Class pojoClass1 = loader1.loadClass(AvroCoderTestPojo.class.getName()); - Class pojoClass2 = loader2.loadClass(AvroCoderTestPojo.class.getName()); - - Object pojo1 = InstanceBuilder.ofType(pojoClass1).withArg(String.class, "hello").build(); - Object pojo2 = InstanceBuilder.ofType(pojoClass2).withArg(String.class, "goodbye").build(); - - // Confirm incompatibility - try { - pojoClass2.cast(pojo1); - fail("Expected ClassCastException; without it, this test is vacuous"); - } catch (ClassCastException e) { - // g2g - } - - // The first coder is expected to populate the Avro SpecificData cache - // The second coder is expected to be corrupted if the caching is done wrong. - AvroCoder avroCoder1 = (AvroCoder) AvroCoder.of(pojoClass1); - AvroCoder avroCoder2 = (AvroCoder) AvroCoder.of(pojoClass2); - - Object cloned1 = CoderUtils.clone(avroCoder1, pojo1); - Object cloned2 = CoderUtils.clone(avroCoder2, pojo2); - - // Confirming that the uncorrupted coder is fine - pojoClass1.cast(cloned1); - - // Confirmed to fail prior to the fix - pojoClass2.cast(cloned2); - } - - /** - * Confirm that we can serialize and deserialize an AvroCoder object and still decode after. - * (https://github.com/apache/beam/issues/18022). - * - * @throws Exception - */ - @Test - public void testTransientFieldInitialization() throws Exception { - Pojo value = new Pojo("Hello", 42, DATETIME_A); - AvroCoder coder = AvroCoder.of(Pojo.class); - - // Serialization of object - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - ObjectOutputStream out = new ObjectOutputStream(bos); - out.writeObject(coder); - - // De-serialization of object - ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); - ObjectInputStream in = new ObjectInputStream(bis); - AvroCoder copied = (AvroCoder) in.readObject(); - - CoderProperties.coderDecodeEncodeEqual(copied, value); - } - - /** - * Confirm that we can serialize and deserialize an AvroCoder object using Kryo. (BEAM-626). - * - * @throws Exception - */ - @Test - public void testKryoSerialization() throws Exception { - Pojo value = new Pojo("Hello", 42, DATETIME_A); - AvroCoder coder = AvroCoder.of(Pojo.class); - - // Kryo instantiation - Kryo kryo = new Kryo(); - kryo.setInstantiatorStrategy(new StdInstantiatorStrategy()); - - // Serialization of object without any memoization - ByteArrayOutputStream coderWithoutMemoizationBos = new ByteArrayOutputStream(); - try (Output output = new Output(coderWithoutMemoizationBos)) { - kryo.writeObject(output, coder); - } - - // Force thread local memoization to store values. - CoderProperties.coderDecodeEncodeEqual(coder, value); - - // Serialization of object with memoized fields - ByteArrayOutputStream coderWithMemoizationBos = new ByteArrayOutputStream(); - try (Output output = new Output(coderWithMemoizationBos)) { - kryo.writeObject(output, coder); - } - - // Copy empty and memoized variants of the Coder - ByteArrayInputStream bisWithoutMemoization = - new ByteArrayInputStream(coderWithoutMemoizationBos.toByteArray()); - AvroCoder copiedWithoutMemoization = - (AvroCoder) kryo.readObject(new Input(bisWithoutMemoization), AvroCoder.class); - ByteArrayInputStream bisWithMemoization = - new ByteArrayInputStream(coderWithMemoizationBos.toByteArray()); - AvroCoder copiedWithMemoization = - (AvroCoder) kryo.readObject(new Input(bisWithMemoization), AvroCoder.class); - - CoderProperties.coderDecodeEncodeEqual(copiedWithoutMemoization, value); - CoderProperties.coderDecodeEncodeEqual(copiedWithMemoization, value); - } - - @Test - public void testPojoEncoding() throws Exception { - Pojo value = new Pojo("Hello", 42, DATETIME_A); - AvroCoder coder = AvroCoder.of(Pojo.class); - - CoderProperties.coderDecodeEncodeEqual(coder, value); - } - - @Test - public void testSpecificRecordEncoding() throws Exception { - AvroCoder coder = - AvroCoder.of(TestAvro.class, AVRO_SPECIFIC_RECORD.getSchema(), false); - - assertTrue(SpecificRecord.class.isAssignableFrom(coder.getType())); - CoderProperties.coderDecodeEncodeEqual(coder, AVRO_SPECIFIC_RECORD); - } - - @Test - public void testReflectRecordEncoding() throws Exception { - AvroCoder coder = AvroCoder.of(TestAvro.class, true); - AvroCoder coderWithSchema = - AvroCoder.of(TestAvro.class, AVRO_SPECIFIC_RECORD.getSchema(), true); - - assertTrue(SpecificRecord.class.isAssignableFrom(coder.getType())); - assertTrue(SpecificRecord.class.isAssignableFrom(coderWithSchema.getType())); - - CoderProperties.coderDecodeEncodeEqual(coder, AVRO_SPECIFIC_RECORD); - CoderProperties.coderDecodeEncodeEqual(coderWithSchema, AVRO_SPECIFIC_RECORD); - } - - @Test - public void testDisableReflectionEncoding() { - try { - AvroCoder.of(Pojo.class, false); - fail("When userReclectApi is disable, schema should not be generated through reflection"); - } catch (AvroRuntimeException e) { - String message = - "avro.shaded.com.google.common.util.concurrent.UncheckedExecutionException: " - + "org.apache.avro.AvroRuntimeException: " - + "Not a Specific class: class org.apache.beam.sdk.coders.AvroCoderTest$Pojo"; - assertEquals(message, e.getMessage()); - } - } - - @Test - public void testGenericRecordEncoding() throws Exception { - String schemaString = - "{\"namespace\": \"example.avro\",\n" - + " \"type\": \"record\",\n" - + " \"name\": \"User\",\n" - + " \"fields\": [\n" - + " {\"name\": \"name\", \"type\": \"string\"},\n" - + " {\"name\": \"favorite_number\", \"type\": [\"int\", \"null\"]},\n" - + " {\"name\": \"favorite_color\", \"type\": [\"string\", \"null\"]}\n" - + " ]\n" - + "}"; - Schema schema = new Schema.Parser().parse(schemaString); - - GenericRecord before = new GenericData.Record(schema); - before.put("name", "Bob"); - before.put("favorite_number", 256); - // Leave favorite_color null - - AvroCoder coder = AvroCoder.of(GenericRecord.class, schema); - - CoderProperties.coderDecodeEncodeEqual(coder, before); - assertEquals(schema, coder.getSchema()); - } - - @Test - public void testEncodingNotBuffered() throws Exception { - // This test ensures that the coder doesn't read ahead and buffer data. - // Reading ahead causes a problem if the stream consists of records of different - // types. - Pojo before = new Pojo("Hello", 42, DATETIME_A); - - AvroCoder coder = AvroCoder.of(Pojo.class); - SerializableCoder intCoder = SerializableCoder.of(Integer.class); - - ByteArrayOutputStream outStream = new ByteArrayOutputStream(); - - Context context = Context.NESTED; - coder.encode(before, outStream, context); - intCoder.encode(10, outStream, context); - - ByteArrayInputStream inStream = new ByteArrayInputStream(outStream.toByteArray()); - - Pojo after = coder.decode(inStream, context); - assertEquals(before, after); - - Integer intAfter = intCoder.decode(inStream, context); - assertEquals(Integer.valueOf(10), intAfter); - } - - @Test - @Category(NeedsRunner.class) - public void testDefaultCoder() throws Exception { - // Use MyRecord as input and output types without explicitly specifying - // a coder (this uses the default coders, which may not be AvroCoder). - PCollection output = - pipeline - .apply(Create.of(new Pojo("hello", 1, DATETIME_A), new Pojo("world", 2, DATETIME_B))) - .apply(ParDo.of(new GetTextFn())); - - PAssert.that(output).containsInAnyOrder("hello", "world"); - pipeline.run(); - } - - @Test - public void testAvroCoderIsSerializable() throws Exception { - AvroCoder coder = AvroCoder.of(Pojo.class); - - // Check that the coder is serializable using the regular JSON approach. - SerializableUtils.ensureSerializable(coder); - } - - @Test - public void testAvroSpecificCoderIsSerializable() throws Exception { - AvroCoder coder = AvroCoder.of(TestAvro.class, false); - - // Check that the coder is serializable using the regular JSON approach. - SerializableUtils.ensureSerializable(coder); - } - - private void assertDeterministic(AvroCoder coder) { - try { - coder.verifyDeterministic(); - } catch (NonDeterministicException e) { - fail("Expected " + coder + " to be deterministic, but got:\n" + e); - } - } - - private void assertNonDeterministic(AvroCoder coder, Matcher reason1) { - try { - coder.verifyDeterministic(); - fail("Expected " + coder + " to be non-deterministic."); - } catch (NonDeterministicException e) { - assertThat(e.getReasons(), Matchers.iterableWithSize(1)); - assertThat(e.getReasons(), Matchers.contains(reason1)); - } - } - - @Test - public void testDeterministicInteger() { - assertDeterministic(AvroCoder.of(Integer.class)); - } - - @Test - public void testDeterministicInt() { - assertDeterministic(AvroCoder.of(int.class)); - } - - private static class SimpleDeterministicClass { - @SuppressWarnings("unused") - private Integer intField; - - @SuppressWarnings("unused") - private char charField; - - @SuppressWarnings("unused") - private Integer[] intArray; - - @SuppressWarnings("unused") - private Utf8 utf8field; - } - - @Test - public void testDeterministicSimple() { - assertDeterministic(AvroCoder.of(SimpleDeterministicClass.class)); - } - - private static class UnorderedMapClass { - @SuppressWarnings("unused") - private Map mapField; - } - - private Matcher reason(final String prefix, final String messagePart) { - return new TypeSafeMatcher(String.class) { - @Override - public void describeTo(Description description) { - description.appendText( - String.format("Reason starting with '%s:' containing '%s'", prefix, messagePart)); - } - - @Override - protected boolean matchesSafely(String item) { - return item.startsWith(prefix + ":") && item.contains(messagePart); - } - }; - } - - private Matcher reasonClass(Class clazz, String message) { - return reason(clazz.getName(), message); - } - - private Matcher reasonField(Class clazz, String field, String message) { - return reason(clazz.getName() + "#" + field, message); - } - - @Test - public void testDeterministicUnorderedMap() { - assertNonDeterministic( - AvroCoder.of(UnorderedMapClass.class), - reasonField( - UnorderedMapClass.class, - "mapField", - "java.util.Map " - + "may not be deterministically ordered")); - } - - private static class NonDeterministicArray { - @SuppressWarnings("unused") - private UnorderedMapClass[] arrayField; - } - - @Test - public void testDeterministicNonDeterministicArray() { - assertNonDeterministic( - AvroCoder.of(NonDeterministicArray.class), - reasonField( - UnorderedMapClass.class, - "mapField", - "java.util.Map" - + " may not be deterministically ordered")); - } - - private static class SubclassOfUnorderedMapClass extends UnorderedMapClass {} - - @Test - public void testDeterministicNonDeterministicChild() { - // Super class has non deterministic fields. - assertNonDeterministic( - AvroCoder.of(SubclassOfUnorderedMapClass.class), - reasonField(UnorderedMapClass.class, "mapField", "may not be deterministically ordered")); - } - - private static class SubclassHidingParent extends UnorderedMapClass { - @SuppressWarnings("unused") - @AvroName("mapField2") // AvroName is not enough - private int mapField; - } - - @Test - public void testAvroProhibitsShadowing() { - // This test verifies that Avro won't serialize a class with two fields of - // the same name. This is important for our error reporting, and also how - // we lookup a field. - try { - ReflectData.get().getSchema(SubclassHidingParent.class); - fail("Expected AvroTypeException"); - } catch (AvroRuntimeException e) { - assertThat(e.getMessage(), containsString("mapField")); - assertThat(e.getMessage(), containsString("two fields named")); - } - } - - private static class FieldWithAvroName { - @AvroName("name") - @SuppressWarnings("unused") - private int someField; - } - - @Test - public void testDeterministicWithAvroName() { - assertDeterministic(AvroCoder.of(FieldWithAvroName.class)); - } - - @Test - public void testDeterminismSortedMap() { - assertDeterministic(AvroCoder.of(StringSortedMapField.class)); - } - - private static class StringSortedMapField { - @SuppressWarnings("unused") - SortedMap sortedMapField; - } - - @Test - public void testDeterminismTreeMapValue() { - // The value is non-deterministic, so we should fail. - assertNonDeterministic( - AvroCoder.of(TreeMapNonDetValue.class), - reasonField( - UnorderedMapClass.class, - "mapField", - "java.util.Map " - + "may not be deterministically ordered")); - } - - private static class TreeMapNonDetValue { - @SuppressWarnings("unused") - TreeMap nonDeterministicField; - } - - @Test - public void testDeterminismUnorderedMap() { - // LinkedHashMap is not deterministically ordered, so we should fail. - assertNonDeterministic( - AvroCoder.of(LinkedHashMapField.class), - reasonField( - LinkedHashMapField.class, - "nonDeterministicMap", - "java.util.LinkedHashMap " - + "may not be deterministically ordered")); - } - - private static class LinkedHashMapField { - @SuppressWarnings("unused") - LinkedHashMap nonDeterministicMap; - } - - @Test - public void testDeterminismCollection() { - assertNonDeterministic( - AvroCoder.of(StringCollection.class), - reasonField( - StringCollection.class, - "stringCollection", - "java.util.Collection may not be deterministically ordered")); - } - - private static class StringCollection { - @SuppressWarnings("unused") - Collection stringCollection; - } - - @Test - public void testDeterminismList() { - assertDeterministic(AvroCoder.of(StringList.class)); - assertDeterministic(AvroCoder.of(StringArrayList.class)); - } - - private static class StringList { - @SuppressWarnings("unused") - List stringCollection; - } - - private static class StringArrayList { - @SuppressWarnings("unused") - ArrayList stringCollection; - } - - @Test - public void testDeterminismSet() { - assertDeterministic(AvroCoder.of(StringSortedSet.class)); - assertDeterministic(AvroCoder.of(StringTreeSet.class)); - assertNonDeterministic( - AvroCoder.of(StringHashSet.class), - reasonField( - StringHashSet.class, - "stringCollection", - "java.util.HashSet may not be deterministically ordered")); - } - - private static class StringSortedSet { - @SuppressWarnings("unused") - SortedSet stringCollection; - } - - private static class StringTreeSet { - @SuppressWarnings("unused") - TreeSet stringCollection; - } - - private static class StringHashSet { - @SuppressWarnings("unused") - HashSet stringCollection; - } - - @Test - public void testDeterminismCollectionValue() { - assertNonDeterministic( - AvroCoder.of(OrderedSetOfNonDetValues.class), - reasonField(UnorderedMapClass.class, "mapField", "may not be deterministically ordered")); - assertNonDeterministic( - AvroCoder.of(ListOfNonDetValues.class), - reasonField(UnorderedMapClass.class, "mapField", "may not be deterministically ordered")); - } - - private static class OrderedSetOfNonDetValues { - @SuppressWarnings("unused") - SortedSet set; - } - - private static class ListOfNonDetValues { - @SuppressWarnings("unused") - List set; - } - - @Test - public void testDeterminismUnion() { - assertDeterministic(AvroCoder.of(DeterministicUnionBase.class)); - assertNonDeterministic( - AvroCoder.of(NonDeterministicUnionBase.class), - reasonField(UnionCase3.class, "mapField", "may not be deterministically ordered")); - } - - @Test - public void testDeterminismStringable() { - assertDeterministic(AvroCoder.of(String.class)); - assertNonDeterministic( - AvroCoder.of(StringableClass.class), - reasonClass(StringableClass.class, "may not have deterministic #toString()")); - } - - @Stringable - private static class StringableClass {} - - @Test - public void testDeterminismCyclicClass() { - assertNonDeterministic( - AvroCoder.of(Cyclic.class), - reasonField(Cyclic.class, "cyclicField", "appears recursively")); - assertNonDeterministic( - AvroCoder.of(CyclicField.class), - reasonField(Cyclic.class, "cyclicField", Cyclic.class.getName() + " appears recursively")); - assertNonDeterministic( - AvroCoder.of(IndirectCycle1.class), - reasonField( - IndirectCycle2.class, - "field2", - IndirectCycle1.class.getName() + " appears recursively")); - } - - private static class Cyclic { - @SuppressWarnings("unused") - int intField; - - @SuppressWarnings("unused") - Cyclic cyclicField; - } - - private static class CyclicField { - @SuppressWarnings("unused") - Cyclic cyclicField2; - } - - private static class IndirectCycle1 { - @SuppressWarnings("unused") - IndirectCycle2 field1; - } - - private static class IndirectCycle2 { - @SuppressWarnings("unused") - IndirectCycle1 field2; - } - - @Test - public void testDeterminismHasGenericRecord() { - assertDeterministic(AvroCoder.of(HasGenericRecord.class)); - } - - private static class HasGenericRecord { - @AvroSchema( - "{\"name\": \"bar\", \"type\": \"record\", \"fields\": [" - + "{\"name\": \"foo\", \"type\": \"int\"}]}") - GenericRecord genericRecord; - } - - @Test - public void testDeterminismHasCustomSchema() { - assertNonDeterministic( - AvroCoder.of(HasCustomSchema.class), - reasonField( - HasCustomSchema.class, - "withCustomSchema", - "Custom schemas are only supported for subtypes of IndexedRecord.")); - } - - private static class HasCustomSchema { - @AvroSchema( - "{\"name\": \"bar\", \"type\": \"record\", \"fields\": [" - + "{\"name\": \"foo\", \"type\": \"int\"}]}") - int withCustomSchema; - } - - @Test - public void testAvroCoderTreeMapDeterminism() throws Exception, NonDeterministicException { - TreeMapField size1 = new TreeMapField(); - TreeMapField size2 = new TreeMapField(); - - // Different order for entries - size1.field.put("hello", "world"); - size1.field.put("another", "entry"); - - size2.field.put("another", "entry"); - size2.field.put("hello", "world"); - - AvroCoder coder = AvroCoder.of(TreeMapField.class); - coder.verifyDeterministic(); - - ByteArrayOutputStream outStream1 = new ByteArrayOutputStream(); - ByteArrayOutputStream outStream2 = new ByteArrayOutputStream(); - - Context context = Context.NESTED; - coder.encode(size1, outStream1, context); - coder.encode(size2, outStream2, context); - - assertArrayEquals(outStream1.toByteArray(), outStream2.toByteArray()); - } - - private static class TreeMapField { - private TreeMap field = new TreeMap<>(); - } - - @Union({UnionCase1.class, UnionCase2.class}) - private abstract static class DeterministicUnionBase {} - - @Union({UnionCase1.class, UnionCase2.class, UnionCase3.class}) - private abstract static class NonDeterministicUnionBase {} - - private static class UnionCase1 extends DeterministicUnionBase {} - - private static class UnionCase2 extends DeterministicUnionBase { - @SuppressWarnings("unused") - String field; - } - - private static class UnionCase3 extends NonDeterministicUnionBase { - @SuppressWarnings("unused") - private Map mapField; - } - - @Test - public void testAvroCoderSimpleSchemaDeterminism() { - assertDeterministic(AvroCoder.of(SchemaBuilder.record("someRecord").fields().endRecord())); - assertDeterministic( - AvroCoder.of( - SchemaBuilder.record("someRecord") - .fields() - .name("int") - .type() - .intType() - .noDefault() - .endRecord())); - assertDeterministic( - AvroCoder.of( - SchemaBuilder.record("someRecord") - .fields() - .name("string") - .type() - .stringType() - .noDefault() - .endRecord())); - - assertNonDeterministic( - AvroCoder.of( - SchemaBuilder.record("someRecord") - .fields() - .name("map") - .type() - .map() - .values() - .stringType() - .noDefault() - .endRecord()), - reason("someRecord.map", "HashMap to represent MAPs")); - - assertDeterministic( - AvroCoder.of( - SchemaBuilder.record("someRecord") - .fields() - .name("array") - .type() - .array() - .items() - .stringType() - .noDefault() - .endRecord())); - - assertDeterministic( - AvroCoder.of( - SchemaBuilder.record("someRecord") - .fields() - .name("enum") - .type() - .enumeration("anEnum") - .symbols("s1", "s2") - .enumDefault("s1") - .endRecord())); - - assertDeterministic( - AvroCoder.of( - SchemaBuilder.unionOf() - .intType() - .and() - .record("someRecord") - .fields() - .nullableString("someField", "") - .endRecord() - .endUnion())); - } - - @Test - public void testAvroCoderStrings() { - // Custom Strings in Records - assertDeterministic( - AvroCoder.of( - SchemaBuilder.record("someRecord") - .fields() - .name("string") - .prop(SpecificData.CLASS_PROP, "java.lang.String") - .type() - .stringType() - .noDefault() - .endRecord())); - assertNonDeterministic( - AvroCoder.of( - SchemaBuilder.record("someRecord") - .fields() - .name("string") - .prop(SpecificData.CLASS_PROP, "unknownString") - .type() - .stringType() - .noDefault() - .endRecord()), - reason("someRecord.string", "unknownString is not known to be deterministic")); - - // Custom Strings in Unions - assertNonDeterministic( - AvroCoder.of( - SchemaBuilder.unionOf() - .intType() - .and() - .record("someRecord") - .fields() - .name("someField") - .prop(SpecificData.CLASS_PROP, "unknownString") - .type() - .stringType() - .noDefault() - .endRecord() - .endUnion()), - reason("someRecord.someField", "unknownString is not known to be deterministic")); - } - - @Test - public void testAvroCoderNestedRecords() { - // Nested Record - assertDeterministic( - AvroCoder.of( - SchemaBuilder.record("nestedRecord") - .fields() - .name("subRecord") - .type() - .record("subRecord") - .fields() - .name("innerField") - .type() - .stringType() - .noDefault() - .endRecord() - .noDefault() - .endRecord())); - } - - @Test - public void testAvroCoderCyclicRecords() { - // Recursive record - assertNonDeterministic( - AvroCoder.of( - SchemaBuilder.record("cyclicRecord") - .fields() - .name("cycle") - .type("cyclicRecord") - .noDefault() - .endRecord()), - reason("cyclicRecord.cycle", "cyclicRecord appears recursively")); - } - - private static class NullableField { - @SuppressWarnings("unused") - private @Nullable String nullable; - } - - @Test - public void testNullableField() { - assertDeterministic(AvroCoder.of(NullableField.class)); - } - - private static class NullableNonDeterministicField { - @SuppressWarnings("unused") - private @Nullable NonDeterministicArray nullableNonDetArray; - } - - private static class NullableCyclic { - @SuppressWarnings("unused") - private @Nullable NullableCyclic nullableNullableCyclicField; - } - - private static class NullableCyclicField { - @SuppressWarnings("unused") - private @Nullable Cyclic nullableCyclicField; - } - - @Test - public void testNullableNonDeterministicField() { - assertNonDeterministic( - AvroCoder.of(NullableCyclic.class), - reasonField( - NullableCyclic.class, - "nullableNullableCyclicField", - NullableCyclic.class.getName() + " appears recursively")); - assertNonDeterministic( - AvroCoder.of(NullableCyclicField.class), - reasonField(Cyclic.class, "cyclicField", Cyclic.class.getName() + " appears recursively")); - assertNonDeterministic( - AvroCoder.of(NullableNonDeterministicField.class), - reasonField(UnorderedMapClass.class, "mapField", " may not be deterministically ordered")); - } - - /** - * Tests that a parameterized class can have an automatically generated schema if the generic - * field is annotated with a union tag. - */ - @Test - public void testGenericClassWithUnionAnnotation() throws Exception { - // Cast is safe as long as the same coder is used for encoding and decoding. - @SuppressWarnings({"unchecked", "rawtypes"}) - AvroCoder> coder = - (AvroCoder) AvroCoder.of(GenericWithAnnotation.class); - - assertThat( - coder.getSchema().getField("onlySomeTypesAllowed").schema().getType(), - equalTo(Schema.Type.UNION)); - - CoderProperties.coderDecodeEncodeEqual(coder, new GenericWithAnnotation<>("hello")); - } - - private static class GenericWithAnnotation { - @AvroSchema("[\"string\", \"int\"]") - private T onlySomeTypesAllowed; - - public GenericWithAnnotation(T value) { - onlySomeTypesAllowed = value; - } - - // For deserialization only - @SuppressWarnings("unused") - protected GenericWithAnnotation() {} - - @Override - public boolean equals(@Nullable Object other) { - return other instanceof GenericWithAnnotation - && onlySomeTypesAllowed.equals(((GenericWithAnnotation) other).onlySomeTypesAllowed); - } - - @Override - public int hashCode() { - return Objects.hash(getClass(), onlySomeTypesAllowed); - } - } - - @Test - public void testAvroCoderForGenerics() throws Exception { - Schema fooSchema = AvroCoder.of(Foo.class).getSchema(); - Schema schema = - new Schema.Parser() - .parse( - "{" - + "\"type\":\"record\"," - + "\"name\":\"SomeGeneric\"," - + "\"namespace\":\"ns\"," - + "\"fields\":[" - + " {\"name\":\"foo\", \"type\":" - + fooSchema.toString() - + "}" - + "]}"); - @SuppressWarnings("rawtypes") - AvroCoder coder = AvroCoder.of(SomeGeneric.class, schema); - - assertNonDeterministic(coder, reasonField(SomeGeneric.class, "foo", "erasure")); - } - - @Test - public void testEncodedTypeDescriptor() throws Exception { - AvroCoder coder = AvroCoder.of(Pojo.class); - assertThat(coder.getEncodedTypeDescriptor(), equalTo(TypeDescriptor.of(Pojo.class))); - } - - private static class SomeGeneric { - @SuppressWarnings("unused") - private T foo; - } - - private static class Foo { - @SuppressWarnings("unused") - String id; - } -} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/AvroCoderTestPojo.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/AvroCoderTestPojo.java deleted file mode 100644 index d48ae9091f7c0..0000000000000 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/AvroCoderTestPojo.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.coders; - -import java.util.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.checkerframework.checker.nullness.qual.Nullable; - -/** A Pojo at the top level for use in tests. */ -class AvroCoderTestPojo { - - public String text; - - // Empty constructor required for Avro decoding. - @SuppressWarnings("unused") - public AvroCoderTestPojo() {} - - public AvroCoderTestPojo(String text) { - this.text = text; - } - - @Override - public boolean equals(@Nullable Object other) { - return (other instanceof AvroCoderTestPojo) && ((AvroCoderTestPojo) other).text.equals(text); - } - - @Override - public int hashCode() { - return Objects.hash(AvroCoderTestPojo.class, text); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this).add("text", text).toString(); - } -} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/BigDecimalCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/BigDecimalCoderTest.java index cee637ecc23fe..063ca9517a23c 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/BigDecimalCoderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/BigDecimalCoderTest.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.testing.CoderProperties.TestElementByteSizeObserver; import org.apache.beam.sdk.util.CoderUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/BigIntegerCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/BigIntegerCoderTest.java index 860315c826ecb..3756053eb552e 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/BigIntegerCoderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/BigIntegerCoderTest.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.testing.CoderProperties.TestElementByteSizeObserver; import org.apache.beam.sdk.util.CoderUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java index cd50db4d344a6..36966c2d35d61 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/CoderRegistryTest.java @@ -44,7 +44,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -463,7 +463,7 @@ public void testCoderPrecedence() throws Exception { CoderRegistry registry = CoderRegistry.createDefault(); // DefaultCoder precedes CoderProviderRegistrar - assertEquals(AvroCoder.of(MyValueA.class), registry.getCoder(MyValueA.class)); + assertEquals(MockDefaultCoder.of(MyValueA.class), registry.getCoder(MyValueA.class)); // CoderProviderRegistrar precedes SerializableCoder assertEquals(MyValueBCoder.INSTANCE, registry.getCoder(MyValueB.class)); @@ -472,7 +472,7 @@ public void testCoderPrecedence() throws Exception { assertEquals(SerializableCoder.of(MyValueC.class), registry.getCoder(MyValueC.class)); } - @DefaultCoder(AvroCoder.class) + @DefaultCoder(MockDefaultCoder.class) private static class MyValueA implements Serializable {} private static class MyValueB implements Serializable {} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DefaultCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DefaultCoderTest.java index b062a356c0022..62d6d7e1d0491 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DefaultCoderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DefaultCoderTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.coders; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.instanceOf; @@ -39,7 +39,7 @@ public class DefaultCoderTest { @Rule public ExpectedException thrown = ExpectedException.none(); - @DefaultCoder(AvroCoder.class) + @DefaultCoder(MockDefaultCoder.class) private static class AvroRecord {} private static class SerializableBase implements Serializable {} @@ -111,7 +111,7 @@ public Coder coderFor( public void testCodersWithoutComponents() throws Exception { CoderRegistry registry = CoderRegistry.createDefault(); registry.registerCoderProvider(new DefaultCoderProvider()); - assertThat(registry.getCoder(AvroRecord.class), instanceOf(AvroCoder.class)); + assertThat(registry.getCoder(AvroRecord.class), instanceOf(MockDefaultCoder.class)); assertThat(registry.getCoder(SerializableRecord.class), instanceOf(SerializableCoder.class)); assertThat(registry.getCoder(CustomRecord.class), instanceOf(CustomSerializableCoder.class)); assertThat( @@ -125,7 +125,7 @@ public void testDefaultCoderInCollection() throws Exception { Coder> avroRecordCoder = registry.getCoder(new TypeDescriptor>() {}); assertThat(avroRecordCoder, instanceOf(ListCoder.class)); - assertThat(((ListCoder) avroRecordCoder).getElemCoder(), instanceOf(AvroCoder.class)); + assertThat(((ListCoder) avroRecordCoder).getElemCoder(), instanceOf(MockDefaultCoder.class)); assertThat( registry.getCoder(new TypeDescriptor>() {}), Matchers.equalTo(ListCoder.of(SerializableCoder.of(SerializableRecord.class)))); diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DelegateCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DelegateCoderTest.java index fdb3e876ef8a0..9cf332b15eb1d 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DelegateCoderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DelegateCoderTest.java @@ -30,8 +30,8 @@ import java.util.Set; import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DequeCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DequeCoderTest.java index b2c1f53783657..4c585faf18ff9 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DequeCoderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DequeCoderTest.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DurationCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DurationCoderTest.java index 688dbd32ca86a..ef94cd393740f 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DurationCoderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/DurationCoderTest.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.ReadableDuration; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/InstantCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/InstantCoderTest.java index 41f7077d33000..4daff2b5557b0 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/InstantCoderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/InstantCoderTest.java @@ -27,8 +27,8 @@ import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.UnsignedBytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedBytes; import org.joda.time.Instant; import org.junit.Assert; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/LengthPrefixCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/LengthPrefixCoderTest.java index 8f21d941dd5ad..147b8babd228b 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/LengthPrefixCoderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/LengthPrefixCoderTest.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.CoderUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/MapCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/MapCoderTest.java index 0777ae0ad6154..86348669edba7 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/MapCoderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/MapCoderTest.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/MockDefaultCoder.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/MockDefaultCoder.java new file mode 100644 index 0000000000000..4f20e86bf6bb2 --- /dev/null +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/MockDefaultCoder.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.coders; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.List; +import org.apache.beam.sdk.values.TypeDescriptor; + +/** + * Used only for tests. + * + * @param + */ +class MockDefaultCoder extends CustomCoder { + private static final MockDefaultCoder INSTANCE = new MockDefaultCoder(); + + @Override + public void encode(T value, OutputStream outStream) throws IOException {} + + @Override + public T decode(InputStream inStream) throws IOException { + return null; + } + + public static MockDefaultCoder of(Class clazz) { + return INSTANCE; + } + + public static CoderProvider getCoderProvider() { + return new MockAvroCoderProvider(); + } + + static class MockAvroCoderProvider extends CoderProvider { + @Override + public Coder coderFor( + TypeDescriptor typeDescriptor, List> componentCoders) { + return (Coder) MockDefaultCoder.INSTANCE; + } + } +} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/NullableCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/NullableCoderTest.java index d67b08e7c7585..6364caae5c713 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/NullableCoderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/NullableCoderTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.coders; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.theInstance; @@ -36,7 +36,7 @@ import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/PrintBase64Encodings.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/PrintBase64Encodings.java index 5b97ecff2a8c5..85517aec373b0 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/PrintBase64Encodings.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/PrintBase64Encodings.java @@ -21,8 +21,8 @@ import java.lang.reflect.Modifier; import java.util.List; import org.apache.beam.sdk.util.CoderUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; /** * A command-line utility for printing the base-64 encodings of test values, for generating exact diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/RowCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/RowCoderTest.java index 5bdd4440715c0..7bc67526e0153 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/RowCoderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/RowCoderTest.java @@ -34,9 +34,9 @@ import org.apache.beam.sdk.schemas.logicaltypes.EnumerationType.Value; import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.junit.Assume; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/StructuralByteArrayTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/StructuralByteArrayTest.java index f95c02d9668df..bd8fdd84fb096 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/StructuralByteArrayTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/StructuralByteArrayTest.java @@ -20,7 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/StructuredCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/StructuredCoderTest.java index b3a20a62391e5..e7ff5ec85d8d8 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/StructuredCoderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/StructuredCoderTest.java @@ -27,7 +27,7 @@ import java.util.List; import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.CoreMatchers; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/TimestampPrefixingWindowCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/TimestampPrefixingWindowCoderTest.java index a9f8123505aaf..6f71962ecc7e5 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/TimestampPrefixingWindowCoderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/TimestampPrefixingWindowCoderTest.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.transforms.windowing.IntervalWindow; import org.apache.beam.sdk.util.common.ElementByteSizeObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java deleted file mode 100644 index e26d422ad8e79..0000000000000 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java +++ /dev/null @@ -1,1627 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io; - -import static org.apache.avro.file.DataFileConstants.SNAPPY_CODEC; -import static org.apache.beam.sdk.io.Compression.AUTO; -import static org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions.RESOLVE_FILE; -import static org.apache.beam.sdk.transforms.Contextful.fn; -import static org.apache.beam.sdk.transforms.Requirements.requiresSideInputs; -import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.Serializable; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Random; -import java.util.stream.Collectors; -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; -import org.apache.avro.file.CodecFactory; -import org.apache.avro.file.DataFileReader; -import org.apache.avro.file.DataFileStream; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.GenericRecordBuilder; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.io.Encoder; -import org.apache.avro.reflect.ReflectData; -import org.apache.avro.reflect.ReflectDatumReader; -import org.apache.beam.sdk.coders.AvroCoder; -import org.apache.beam.sdk.coders.Coder; -import org.apache.beam.sdk.coders.CoderException; -import org.apache.beam.sdk.coders.DefaultCoder; -import org.apache.beam.sdk.coders.KvCoder; -import org.apache.beam.sdk.coders.StringUtf8Coder; -import org.apache.beam.sdk.io.FileBasedSink.FilenamePolicy; -import org.apache.beam.sdk.io.FileBasedSink.OutputFileHints; -import org.apache.beam.sdk.io.fs.ResourceId; -import org.apache.beam.sdk.options.ValueProvider; -import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider; -import org.apache.beam.sdk.testing.NeedsRunner; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.testing.TestStream; -import org.apache.beam.sdk.testing.UsesTestStream; -import org.apache.beam.sdk.testing.UsesUnboundedSplittableParDo; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.sdk.transforms.SimpleFunction; -import org.apache.beam.sdk.transforms.View; -import org.apache.beam.sdk.transforms.Watch; -import org.apache.beam.sdk.transforms.display.DisplayData; -import org.apache.beam.sdk.transforms.windowing.AfterPane; -import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.sdk.transforms.windowing.FixedWindows; -import org.apache.beam.sdk.transforms.windowing.IntervalWindow; -import org.apache.beam.sdk.transforms.windowing.PaneInfo; -import org.apache.beam.sdk.transforms.windowing.Repeatedly; -import org.apache.beam.sdk.transforms.windowing.Window; -import org.apache.beam.sdk.util.SerializableUtils; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; -import org.checkerframework.checker.nullness.qual.Nullable; -import org.joda.time.Duration; -import org.joda.time.Instant; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; -import org.junit.rules.Timeout; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.junit.runners.Parameterized; - -/** Tests for AvroIO Read and Write transforms. */ -@SuppressWarnings({ - "rawtypes", // TODO(https://github.com/apache/beam/issues/20447) -}) -public class AvroIOTest implements Serializable { - /** Unit tests. */ - @RunWith(JUnit4.class) - public static class SimpleTests implements Serializable { - @Test - public void testAvroIOGetName() { - assertEquals("AvroIO.Read", AvroIO.read(String.class).from("/tmp/foo*/baz").getName()); - assertEquals("AvroIO.Write", AvroIO.write(String.class).to("/tmp/foo/baz").getName()); - } - - @Test - public void testWriteWithDefaultCodec() { - AvroIO.Write write = AvroIO.write(String.class).to("/tmp/foo/baz"); - assertEquals(CodecFactory.snappyCodec().toString(), write.inner.getCodec().toString()); - } - - @Test - public void testWriteWithCustomCodec() { - AvroIO.Write write = - AvroIO.write(String.class).to("/tmp/foo/baz").withCodec(CodecFactory.snappyCodec()); - assertEquals(SNAPPY_CODEC, write.inner.getCodec().toString()); - } - - @Test - public void testWriteWithSerDeCustomDeflateCodec() { - AvroIO.Write write = - AvroIO.write(String.class).to("/tmp/foo/baz").withCodec(CodecFactory.deflateCodec(9)); - - assertEquals( - CodecFactory.deflateCodec(9).toString(), - SerializableUtils.clone(write.inner.getCodec()).getCodec().toString()); - } - - @Test - public void testWriteWithSerDeCustomXZCodec() { - AvroIO.Write write = - AvroIO.write(String.class).to("/tmp/foo/baz").withCodec(CodecFactory.xzCodec(9)); - - assertEquals( - CodecFactory.xzCodec(9).toString(), - SerializableUtils.clone(write.inner.getCodec()).getCodec().toString()); - } - - @Test - public void testReadDisplayData() { - AvroIO.Read read = AvroIO.read(String.class).from("/foo.*"); - - DisplayData displayData = DisplayData.from(read); - assertThat(displayData, hasDisplayItem("filePattern", "/foo.*")); - } - } - - /** NeedsRunner tests. */ - @RunWith(Parameterized.class) - @Category(NeedsRunner.class) - public static class NeedsRunnerTests implements Serializable { - @Rule public transient TestPipeline writePipeline = TestPipeline.create(); - - @Rule public transient TestPipeline readPipeline = TestPipeline.create(); - - @Rule public transient TestPipeline windowedAvroWritePipeline = TestPipeline.create(); - - @Rule public transient TemporaryFolder tmpFolder = new TemporaryFolder(); - - @Rule public transient ExpectedException expectedException = ExpectedException.none(); - - @Rule public transient Timeout globalTimeout = Timeout.seconds(1200); - - @Parameterized.Parameters(name = "{index}: {0}") - public static Collection params() { - return Arrays.asList(new Object[][] {{true}, {false}}); - } - - @Parameterized.Parameter public boolean withBeamSchemas; - - @DefaultCoder(AvroCoder.class) - static class GenericClass { - int intField; - String stringField; - - GenericClass() {} - - GenericClass(int intField, String stringField) { - this.intField = intField; - this.stringField = stringField; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(getClass()) - .add("intField", intField) - .add("stringField", stringField) - .toString(); - } - - @Override - public int hashCode() { - return Objects.hash(intField, stringField); - } - - @Override - public boolean equals(@Nullable Object other) { - if (other == null || !(other instanceof GenericClass)) { - return false; - } - GenericClass o = (GenericClass) other; - return intField == o.intField && Objects.equals(stringField, o.stringField); - } - } - - private static class ParseGenericClass - implements SerializableFunction { - @Override - public GenericClass apply(GenericRecord input) { - return new GenericClass((int) input.get("intField"), input.get("stringField").toString()); - } - - @Test - public void testWriteDisplayData() { - AvroIO.Write write = - AvroIO.write(GenericClass.class) - .to("/foo") - .withShardNameTemplate("-SS-of-NN-") - .withSuffix("bar") - .withNumShards(100) - .withCodec(CodecFactory.deflateCodec(6)); - - DisplayData displayData = DisplayData.from(write); - - assertThat(displayData, hasDisplayItem("filePrefix", "/foo")); - assertThat(displayData, hasDisplayItem("shardNameTemplate", "-SS-of-NN-")); - assertThat(displayData, hasDisplayItem("fileSuffix", "bar")); - assertThat( - displayData, - hasDisplayItem( - "schema", - "{\"type\":\"record\",\"name\":\"GenericClass\",\"namespace\":\"org.apache.beam.sdk.io" - + ".AvroIOTest$\",\"fields\":[{\"name\":\"intField\",\"type\":\"int\"}," - + "{\"name\":\"stringField\",\"type\":\"string\"}]}")); - assertThat(displayData, hasDisplayItem("numShards", 100)); - assertThat(displayData, hasDisplayItem("codec", CodecFactory.deflateCodec(6).toString())); - } - } - - private enum Sharding { - RUNNER_DETERMINED, - WITHOUT_SHARDING, - FIXED_3_SHARDS - } - - private enum WriteMethod { - AVROIO_WRITE, - AVROIO_SINK_WITH_CLASS, - AVROIO_SINK_WITH_SCHEMA, - /** @deprecated Test code for the deprecated {AvroIO.RecordFormatter}. */ - @Deprecated - AVROIO_SINK_WITH_FORMATTER - } - - private static final String SCHEMA_STRING = - "{\"namespace\": \"example.avro\",\n" - + " \"type\": \"record\",\n" - + " \"name\": \"AvroGeneratedUser\",\n" - + " \"fields\": [\n" - + " {\"name\": \"name\", \"type\": \"string\"},\n" - + " {\"name\": \"favorite_number\", \"type\": [\"int\", \"null\"]},\n" - + " {\"name\": \"favorite_color\", \"type\": [\"string\", \"null\"]}\n" - + " ]\n" - + "}"; - - private static final Schema SCHEMA = new Schema.Parser().parse(SCHEMA_STRING); - - @Test - @Category(NeedsRunner.class) - public void testWriteThenReadJavaClass() throws Throwable { - List values = - ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar")); - File outputFile = tmpFolder.newFile("output.avro"); - - writePipeline - .apply(Create.of(values)) - .apply( - AvroIO.write(GenericClass.class) - .to(writePipeline.newProvider(outputFile.getAbsolutePath())) - .withoutSharding()); - writePipeline.run(); - - PAssert.that( - readPipeline.apply( - "Read", - AvroIO.read(GenericClass.class) - .withBeamSchemas(withBeamSchemas) - .from(readPipeline.newProvider(outputFile.getAbsolutePath())))) - .containsInAnyOrder(values); - - readPipeline.run(); - } - - @Test - @Category(NeedsRunner.class) - public void testReadWithFilename() throws Throwable { - List values = - ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar")); - File outputFile = tmpFolder.newFile("output.avro"); - - writePipeline - .apply(Create.of(values)) - .apply( - AvroIO.write(GenericClass.class) - .to(writePipeline.newProvider(outputFile.getAbsolutePath())) - .withoutSharding()); - writePipeline.run(); - - SerializableFunction> createSource = - input -> - AvroSource.from(ValueProvider.StaticValueProvider.of(input)) - .withSchema(GenericClass.class); - - final PCollection> lines = - readPipeline - .apply(Create.of(Collections.singletonList(outputFile.getAbsolutePath()))) - .apply(FileIO.matchAll()) - .apply(FileIO.readMatches().withCompression(AUTO)) - .apply( - new ReadAllViaFileBasedSourceWithFilename<>( - 10, - createSource, - KvCoder.of(StringUtf8Coder.of(), AvroCoder.of(GenericClass.class)))); - - PAssert.that(lines) - .containsInAnyOrder( - values.stream() - .map(v -> KV.of(outputFile.getAbsolutePath(), v)) - .collect(Collectors.toList())); - readPipeline.run(); - } - - @Test - @Category(NeedsRunner.class) - public void testWriteThenReadCustomType() throws Throwable { - List values = Arrays.asList(0L, 1L, 2L); - File outputFile = tmpFolder.newFile("output.avro"); - - writePipeline - .apply(Create.of(values)) - .apply( - AvroIO.writeCustomType() - .to(writePipeline.newProvider(outputFile.getAbsolutePath())) - .withFormatFunction(new CreateGenericClass()) - .withSchema(ReflectData.get().getSchema(GenericClass.class)) - .withoutSharding()); - writePipeline.run(); - - PAssert.that( - readPipeline - .apply( - "Read", - AvroIO.read(GenericClass.class) - .withBeamSchemas(withBeamSchemas) - .from(readPipeline.newProvider(outputFile.getAbsolutePath()))) - .apply( - MapElements.via( - new SimpleFunction() { - @Override - public Long apply(GenericClass input) { - return (long) input.intField; - } - }))) - .containsInAnyOrder(values); - - readPipeline.run(); - } - - private void testWriteThenReadGeneratedClass( - AvroIO.Write writeTransform, AvroIO.Read readTransform) throws Exception { - File outputFile = tmpFolder.newFile("output.avro"); - - List values = - ImmutableList.of( - (T) new AvroGeneratedUser("Bob", 256, null), - (T) new AvroGeneratedUser("Alice", 128, null), - (T) new AvroGeneratedUser("Ted", null, "white")); - - writePipeline - .apply(Create.of(values)) - .apply( - writeTransform - .to(writePipeline.newProvider(outputFile.getAbsolutePath())) - .withoutSharding()); - writePipeline.run(); - - PAssert.that( - readPipeline.apply( - "Read", - readTransform.from(readPipeline.newProvider(outputFile.getAbsolutePath())))) - .containsInAnyOrder(values); - - readPipeline.run(); - } - - @Test - @Category(NeedsRunner.class) - public void testWriteThenReadGeneratedClassWithClass() throws Throwable { - testWriteThenReadGeneratedClass( - AvroIO.write(AvroGeneratedUser.class), - AvroIO.read(AvroGeneratedUser.class).withBeamSchemas(withBeamSchemas)); - } - - @Test - @Category(NeedsRunner.class) - public void testWriteThenReadGeneratedClassWithSchema() throws Throwable { - testWriteThenReadGeneratedClass( - AvroIO.writeGenericRecords(SCHEMA), - AvroIO.readGenericRecords(SCHEMA).withBeamSchemas(withBeamSchemas)); - } - - @Test - @Category(NeedsRunner.class) - public void testWriteThenReadGeneratedClassWithSchemaString() throws Throwable { - testWriteThenReadGeneratedClass( - AvroIO.writeGenericRecords(SCHEMA.toString()), - AvroIO.readGenericRecords(SCHEMA.toString()).withBeamSchemas(withBeamSchemas)); - } - - @Test - @Category(NeedsRunner.class) - public void testWriteSingleFileThenReadUsingAllMethods() throws Throwable { - List values = - ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar")); - File outputFile = tmpFolder.newFile("output.avro"); - - writePipeline - .apply(Create.of(values)) - .apply( - AvroIO.write(GenericClass.class).to(outputFile.getAbsolutePath()).withoutSharding()); - writePipeline.run(); - - // Test the same data using all versions of read(). - PCollection path = - readPipeline.apply("Create path", Create.of(outputFile.getAbsolutePath())); - PAssert.that( - readPipeline.apply( - "Read", - AvroIO.read(GenericClass.class) - .withBeamSchemas(withBeamSchemas) - .from(outputFile.getAbsolutePath()))) - .containsInAnyOrder(values); - PAssert.that( - readPipeline.apply( - "Read withHintMatchesManyFiles", - AvroIO.read(GenericClass.class) - .withBeamSchemas(withBeamSchemas) - .from(outputFile.getAbsolutePath()) - .withHintMatchesManyFiles())) - .containsInAnyOrder(values); - PAssert.that( - path.apply("MatchAllReadFiles", FileIO.matchAll()) - .apply("ReadMatchesReadFiles", FileIO.readMatches().withCompression(AUTO)) - .apply( - "ReadFiles", - AvroIO.readFiles(GenericClass.class) - .withBeamSchemas(withBeamSchemas) - .withDesiredBundleSizeBytes(10))) - .containsInAnyOrder(values); - PAssert.that( - path.apply( - "ReadAll", - AvroIO.readAll(GenericClass.class) - .withBeamSchemas(withBeamSchemas) - .withDesiredBundleSizeBytes(10))) - .containsInAnyOrder(values); - PAssert.that( - readPipeline.apply( - "Parse", - AvroIO.parseGenericRecords(new ParseGenericClass()) - .from(outputFile.getAbsolutePath()) - .withCoder(AvroCoder.of(GenericClass.class)))) - .containsInAnyOrder(values); - PAssert.that( - readPipeline.apply( - "Parse withHintMatchesManyFiles", - AvroIO.parseGenericRecords(new ParseGenericClass()) - .from(outputFile.getAbsolutePath()) - .withCoder(AvroCoder.of(GenericClass.class)) - .withHintMatchesManyFiles())) - .containsInAnyOrder(values); - PAssert.that( - path.apply("MatchAllParseFilesGenericRecords", FileIO.matchAll()) - .apply( - "ReadMatchesParseFilesGenericRecords", - FileIO.readMatches() - .withDirectoryTreatment(FileIO.ReadMatches.DirectoryTreatment.PROHIBIT)) - .apply( - "ParseFilesGenericRecords", - AvroIO.parseFilesGenericRecords(new ParseGenericClass()) - .withCoder(AvroCoder.of(GenericClass.class)) - .withUsesReshuffle(false) - .withDesiredBundleSizeBytes(10))) - .containsInAnyOrder(values); - PAssert.that( - path.apply("MatchAllParseFilesGenericRecordsWithShuffle", FileIO.matchAll()) - .apply( - "ReadMatchesParseFilesGenericRecordsWithShuffle", - FileIO.readMatches() - .withDirectoryTreatment(FileIO.ReadMatches.DirectoryTreatment.PROHIBIT)) - .apply( - "ParseFilesGenericRecordsWithShuffle", - AvroIO.parseFilesGenericRecords(new ParseGenericClass()) - .withCoder(AvroCoder.of(GenericClass.class)) - .withUsesReshuffle(true) - .withDesiredBundleSizeBytes(10))) - .containsInAnyOrder(values); - PAssert.that( - path.apply( - "ParseAllGenericRecords", - AvroIO.parseAllGenericRecords(new ParseGenericClass()) - .withCoder(AvroCoder.of(GenericClass.class)) - .withDesiredBundleSizeBytes(10))) - .containsInAnyOrder(values); - - readPipeline.run(); - } - - @Test - @Category(NeedsRunner.class) - public void testWriteThenReadMultipleFilepatterns() { - List firstValues = new ArrayList<>(); - List secondValues = new ArrayList<>(); - for (int i = 0; i < 10; ++i) { - firstValues.add(new GenericClass(i, "a" + i)); - secondValues.add(new GenericClass(i, "b" + i)); - } - writePipeline - .apply("Create first", Create.of(firstValues)) - .apply( - "Write first", - AvroIO.write(GenericClass.class) - .to(tmpFolder.getRoot().getAbsolutePath() + "/first") - .withNumShards(2)); - writePipeline - .apply("Create second", Create.of(secondValues)) - .apply( - "Write second", - AvroIO.write(GenericClass.class) - .to(tmpFolder.getRoot().getAbsolutePath() + "/second") - .withNumShards(3)); - writePipeline.run(); - - // Test readFiles(), readAll(), parseFilesGenericRecords() and parseAllGenericRecords(). - PCollection paths = - readPipeline.apply( - "Create paths", - Create.of( - tmpFolder.getRoot().getAbsolutePath() + "/first*", - tmpFolder.getRoot().getAbsolutePath() + "/second*")); - PAssert.that( - paths - .apply("MatchAllReadFiles", FileIO.matchAll()) - .apply("ReadMatchesReadFiles", FileIO.readMatches().withCompression(AUTO)) - .apply( - "ReadFiles", - AvroIO.readFiles(GenericClass.class) - .withBeamSchemas(withBeamSchemas) - .withDesiredBundleSizeBytes(10))) - .containsInAnyOrder(Iterables.concat(firstValues, secondValues)); - PAssert.that( - paths.apply( - "ReadAll", - AvroIO.readAll(GenericClass.class) - .withBeamSchemas(withBeamSchemas) - .withDesiredBundleSizeBytes(10))) - .containsInAnyOrder(Iterables.concat(firstValues, secondValues)); - PAssert.that( - paths - .apply("MatchAllParseFilesGenericRecords", FileIO.matchAll()) - .apply( - "ReadMatchesParseFilesGenericRecords", - FileIO.readMatches() - .withDirectoryTreatment(FileIO.ReadMatches.DirectoryTreatment.PROHIBIT)) - .apply( - "ParseFilesGenericRecords", - AvroIO.parseFilesGenericRecords(new ParseGenericClass()) - .withCoder(AvroCoder.of(GenericClass.class)) - .withDesiredBundleSizeBytes(10))) - .containsInAnyOrder(Iterables.concat(firstValues, secondValues)); - PAssert.that( - paths.apply( - "ParseAllGenericRecords", - AvroIO.parseAllGenericRecords(new ParseGenericClass()) - .withCoder(AvroCoder.of(GenericClass.class)) - .withDesiredBundleSizeBytes(10))) - .containsInAnyOrder(Iterables.concat(firstValues, secondValues)); - - readPipeline.run(); - } - - private static class CreateGenericClass extends SimpleFunction { - @Override - public GenericClass apply(Long i) { - return new GenericClass(i.intValue(), "value" + i); - } - } - - @Test - @Category({NeedsRunner.class, UsesUnboundedSplittableParDo.class}) - public void testContinuouslyWriteAndReadMultipleFilepatterns() { - SimpleFunction mapFn = new CreateGenericClass(); - List firstValues = new ArrayList<>(); - List secondValues = new ArrayList<>(); - for (int i = 0; i < 7; ++i) { - (i < 3 ? firstValues : secondValues).add(mapFn.apply((long) i)); - } - // Configure windowing of the input so that it fires every time a new element is generated, - // so that files are written continuously. - Window window = - Window.into(FixedWindows.of(Duration.millis(100))) - .withAllowedLateness(Duration.ZERO) - .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1))) - .discardingFiredPanes(); - readPipeline - .apply("Sequence first", GenerateSequence.from(0).to(3).withRate(1, Duration.millis(300))) - .apply("Window first", window) - .apply("Map first", MapElements.via(mapFn)) - .apply( - "Write first", - AvroIO.write(GenericClass.class) - .to(tmpFolder.getRoot().getAbsolutePath() + "/first") - .withNumShards(2) - .withWindowedWrites()); - readPipeline - .apply( - "Sequence second", GenerateSequence.from(3).to(7).withRate(1, Duration.millis(300))) - .apply("Window second", window) - .apply("Map second", MapElements.via(mapFn)) - .apply( - "Write second", - AvroIO.write(GenericClass.class) - .to(tmpFolder.getRoot().getAbsolutePath() + "/second") - .withNumShards(3) - .withWindowedWrites()); - - // Test read(), readFiles(), readAll(), parse(), parseFilesGenericRecords() and - // parseAllGenericRecords() with watchForNewFiles(). - PAssert.that( - readPipeline.apply( - "Read", - AvroIO.read(GenericClass.class) - .withBeamSchemas(withBeamSchemas) - .from(tmpFolder.getRoot().getAbsolutePath() + "/first*") - .watchForNewFiles( - Duration.millis(100), - Watch.Growth.afterTimeSinceNewOutput(Duration.standardSeconds(3))))) - .containsInAnyOrder(firstValues); - PAssert.that( - readPipeline.apply( - "Parse", - AvroIO.parseGenericRecords(new ParseGenericClass()) - .from(tmpFolder.getRoot().getAbsolutePath() + "/first*") - .watchForNewFiles( - Duration.millis(100), - Watch.Growth.afterTimeSinceNewOutput(Duration.standardSeconds(3))))) - .containsInAnyOrder(firstValues); - - PCollection paths = - readPipeline.apply( - "Create paths", - Create.of( - tmpFolder.getRoot().getAbsolutePath() + "/first*", - tmpFolder.getRoot().getAbsolutePath() + "/second*")); - PAssert.that( - paths - .apply( - "Match All Read files", - FileIO.matchAll() - .continuously( - Duration.millis(100), - Watch.Growth.afterTimeSinceNewOutput(Duration.standardSeconds(3)))) - .apply( - "Read Matches Read files", - FileIO.readMatches() - .withDirectoryTreatment(FileIO.ReadMatches.DirectoryTreatment.PROHIBIT)) - .apply( - "Read files", - AvroIO.readFiles(GenericClass.class) - .withBeamSchemas(withBeamSchemas) - .withDesiredBundleSizeBytes(10))) - .containsInAnyOrder(Iterables.concat(firstValues, secondValues)); - PAssert.that( - paths.apply( - "Read all", - AvroIO.readAll(GenericClass.class) - .withBeamSchemas(withBeamSchemas) - .watchForNewFiles( - Duration.millis(100), - Watch.Growth.afterTimeSinceNewOutput(Duration.standardSeconds(3))) - .withDesiredBundleSizeBytes(10))) - .containsInAnyOrder(Iterables.concat(firstValues, secondValues)); - PAssert.that( - paths - .apply( - "Match All ParseFilesGenericRecords", - FileIO.matchAll() - .continuously( - Duration.millis(100), - Watch.Growth.afterTimeSinceNewOutput(Duration.standardSeconds(3)))) - .apply( - "Match Matches ParseFilesGenericRecords", - FileIO.readMatches() - .withDirectoryTreatment(FileIO.ReadMatches.DirectoryTreatment.PROHIBIT)) - .apply( - "ParseFilesGenericRecords", - AvroIO.parseFilesGenericRecords(new ParseGenericClass()) - .withCoder(AvroCoder.of(GenericClass.class)) - .withDesiredBundleSizeBytes(10))) - .containsInAnyOrder(Iterables.concat(firstValues, secondValues)); - PAssert.that( - paths.apply( - "ParseAllGenericRecords", - AvroIO.parseAllGenericRecords(new ParseGenericClass()) - .withCoder(AvroCoder.of(GenericClass.class)) - .watchForNewFiles( - Duration.millis(100), - Watch.Growth.afterTimeSinceNewOutput(Duration.standardSeconds(3))) - .withDesiredBundleSizeBytes(10))) - .containsInAnyOrder(Iterables.concat(firstValues, secondValues)); - readPipeline.run(); - } - - @Test - @SuppressWarnings("unchecked") - @Category(NeedsRunner.class) - public void testCompressedWriteAndReadASingleFile() throws Throwable { - List values = - ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar")); - File outputFile = tmpFolder.newFile("output.avro"); - - writePipeline - .apply(Create.of(values)) - .apply( - AvroIO.write(GenericClass.class) - .to(outputFile.getAbsolutePath()) - .withoutSharding() - .withCodec(CodecFactory.deflateCodec(9))); - writePipeline.run(); - - PAssert.that( - readPipeline.apply( - AvroIO.read(GenericClass.class) - .withBeamSchemas(withBeamSchemas) - .from(outputFile.getAbsolutePath()))) - .containsInAnyOrder(values); - readPipeline.run(); - - try (DataFileStream dataFileStream = - new DataFileStream(new FileInputStream(outputFile), new GenericDatumReader())) { - assertEquals("deflate", dataFileStream.getMetaString("avro.codec")); - } - } - - @Test - @SuppressWarnings("unchecked") - @Category(NeedsRunner.class) - public void testWriteThenReadASingleFileWithNullCodec() throws Throwable { - List values = - ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar")); - File outputFile = tmpFolder.newFile("output.avro"); - - writePipeline - .apply(Create.of(values)) - .apply( - AvroIO.write(GenericClass.class) - .to(outputFile.getAbsolutePath()) - .withoutSharding() - .withCodec(CodecFactory.nullCodec())); - writePipeline.run(); - - PAssert.that( - readPipeline.apply( - AvroIO.read(GenericClass.class) - .withBeamSchemas(withBeamSchemas) - .from(outputFile.getAbsolutePath()))) - .containsInAnyOrder(values); - readPipeline.run(); - - try (DataFileStream dataFileStream = - new DataFileStream(new FileInputStream(outputFile), new GenericDatumReader())) { - assertEquals("null", dataFileStream.getMetaString("avro.codec")); - } - } - - @DefaultCoder(AvroCoder.class) - static class GenericClassV2 { - int intField; - String stringField; - @org.apache.avro.reflect.Nullable String nullableField; - - GenericClassV2() {} - - GenericClassV2(int intValue, String stringValue, String nullableValue) { - this.intField = intValue; - this.stringField = stringValue; - this.nullableField = nullableValue; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(getClass()) - .add("intField", intField) - .add("stringField", stringField) - .add("nullableField", nullableField) - .toString(); - } - - @Override - public int hashCode() { - return Objects.hash(intField, stringField, nullableField); - } - - @Override - public boolean equals(@Nullable Object other) { - if (!(other instanceof GenericClassV2)) { - return false; - } - GenericClassV2 o = (GenericClassV2) other; - return intField == o.intField - && Objects.equals(stringField, o.stringField) - && Objects.equals(nullableField, o.nullableField); - } - } - - /** - * Tests that {@code AvroIO} can read an upgraded version of an old class, as long as the schema - * resolution process succeeds. This test covers the case when a new, {@code @Nullable} field - * has been added. - * - *

    For more information, see http://avro.apache.org/docs/1.7.7/spec.html#Schema+Resolution - */ - @Test - @Category(NeedsRunner.class) - public void testWriteThenReadSchemaUpgrade() throws Throwable { - List values = - ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar")); - File outputFile = tmpFolder.newFile("output.avro"); - - writePipeline - .apply(Create.of(values)) - .apply( - AvroIO.write(GenericClass.class).to(outputFile.getAbsolutePath()).withoutSharding()); - writePipeline.run(); - - List expected = - ImmutableList.of(new GenericClassV2(3, "hi", null), new GenericClassV2(5, "bar", null)); - - PAssert.that( - readPipeline.apply( - AvroIO.read(GenericClassV2.class) - .withBeamSchemas(withBeamSchemas) - .from(outputFile.getAbsolutePath()))) - .containsInAnyOrder(expected); - readPipeline.run(); - } - - private static class WindowedFilenamePolicy extends FilenamePolicy { - final ResourceId outputFilePrefix; - - WindowedFilenamePolicy(ResourceId outputFilePrefix) { - this.outputFilePrefix = outputFilePrefix; - } - - @Override - public ResourceId windowedFilename( - int shardNumber, - int numShards, - BoundedWindow window, - PaneInfo paneInfo, - OutputFileHints outputFileHints) { - String filenamePrefix = - outputFilePrefix.isDirectory() ? "" : firstNonNull(outputFilePrefix.getFilename(), ""); - - IntervalWindow interval = (IntervalWindow) window; - String windowStr = - String.format("%s-%s", interval.start().toString(), interval.end().toString()); - String filename = - String.format( - "%s-%s-%s-of-%s-pane-%s%s%s.avro", - filenamePrefix, - windowStr, - shardNumber, - numShards, - paneInfo.getIndex(), - paneInfo.isLast() ? "-last" : "", - outputFileHints.getSuggestedFilenameSuffix()); - return outputFilePrefix.getCurrentDirectory().resolve(filename, RESOLVE_FILE); - } - - @Override - public ResourceId unwindowedFilename( - int shardNumber, int numShards, OutputFileHints outputFileHints) { - throw new UnsupportedOperationException("Expecting windowed outputs only"); - } - - @Override - public void populateDisplayData(DisplayData.Builder builder) { - builder.add( - DisplayData.item("fileNamePrefix", outputFilePrefix.toString()) - .withLabel("File Name Prefix")); - } - } - - @Test - @Category({NeedsRunner.class, UsesTestStream.class}) - public void testWriteWindowed() throws Throwable { - testWindowedAvroIOWriteUsingMethod(WriteMethod.AVROIO_WRITE); - } - - @Test - @Category({NeedsRunner.class, UsesTestStream.class}) - public void testWindowedAvroIOWriteViaSink() throws Throwable { - testWindowedAvroIOWriteUsingMethod(WriteMethod.AVROIO_SINK_WITH_CLASS); - } - - void testWindowedAvroIOWriteUsingMethod(WriteMethod method) throws IOException { - Path baseDir = Files.createTempDirectory(tmpFolder.getRoot().toPath(), "testwrite"); - final String baseFilename = baseDir.resolve("prefix").toString(); - - Instant base = new Instant(0); - ArrayList allElements = new ArrayList<>(); - ArrayList> firstWindowElements = new ArrayList<>(); - ArrayList firstWindowTimestamps = - Lists.newArrayList( - base.plus(Duration.ZERO), base.plus(Duration.standardSeconds(10)), - base.plus(Duration.standardSeconds(20)), base.plus(Duration.standardSeconds(30))); - - Random random = new Random(); - for (int i = 0; i < 100; ++i) { - GenericClass item = new GenericClass(i, String.valueOf(i)); - allElements.add(item); - firstWindowElements.add( - TimestampedValue.of( - item, firstWindowTimestamps.get(random.nextInt(firstWindowTimestamps.size())))); - } - - ArrayList> secondWindowElements = new ArrayList<>(); - ArrayList secondWindowTimestamps = - Lists.newArrayList( - base.plus(Duration.standardSeconds(60)), base.plus(Duration.standardSeconds(70)), - base.plus(Duration.standardSeconds(80)), base.plus(Duration.standardSeconds(90))); - for (int i = 100; i < 200; ++i) { - GenericClass item = new GenericClass(i, String.valueOf(i)); - allElements.add(new GenericClass(i, String.valueOf(i))); - secondWindowElements.add( - TimestampedValue.of( - item, secondWindowTimestamps.get(random.nextInt(secondWindowTimestamps.size())))); - } - - TimestampedValue[] firstWindowArray = - firstWindowElements.toArray(new TimestampedValue[100]); - TimestampedValue[] secondWindowArray = - secondWindowElements.toArray(new TimestampedValue[100]); - - TestStream values = - TestStream.create(AvroCoder.of(GenericClass.class)) - .advanceWatermarkTo(new Instant(0)) - .addElements( - firstWindowArray[0], - Arrays.copyOfRange(firstWindowArray, 1, firstWindowArray.length)) - .advanceWatermarkTo(new Instant(0).plus(Duration.standardMinutes(1))) - .addElements( - secondWindowArray[0], - Arrays.copyOfRange(secondWindowArray, 1, secondWindowArray.length)) - .advanceWatermarkToInfinity(); - - final PTransform, WriteFilesResult> write; - switch (method) { - case AVROIO_WRITE: - { - FilenamePolicy policy = - new WindowedFilenamePolicy( - FileBasedSink.convertToFileResourceIfPossible(baseFilename)); - write = - AvroIO.write(GenericClass.class) - .to(policy) - .withTempDirectory( - StaticValueProvider.of( - FileSystems.matchNewResource(baseDir.toString(), true))) - .withWindowedWrites() - .withNumShards(2) - .withOutputFilenames(); - break; - } - - case AVROIO_SINK_WITH_CLASS: - { - write = - FileIO.write() - .via(AvroIO.sink(GenericClass.class)) - .to(baseDir.toString()) - .withPrefix("prefix") - .withSuffix(".avro") - .withTempDirectory(baseDir.toString()) - .withNumShards(2); - break; - } - - default: - throw new UnsupportedOperationException(); - } - windowedAvroWritePipeline - .apply(values) - .apply(Window.into(FixedWindows.of(Duration.standardMinutes(1)))) - .apply(write); - windowedAvroWritePipeline.run(); - - // Validate that the data written matches the expected elements in the expected order - List expectedFiles = new ArrayList<>(); - for (int shard = 0; shard < 2; shard++) { - for (int window = 0; window < 2; window++) { - Instant windowStart = new Instant(0).plus(Duration.standardMinutes(window)); - IntervalWindow iw = new IntervalWindow(windowStart, Duration.standardMinutes(1)); - String baseAndWindow = baseFilename + "-" + iw.start() + "-" + iw.end(); - switch (method) { - case AVROIO_WRITE: - expectedFiles.add(new File(baseAndWindow + "-" + shard + "-of-2-pane-0-last.avro")); - break; - case AVROIO_SINK_WITH_CLASS: - expectedFiles.add(new File(baseAndWindow + "-0000" + shard + "-of-00002.avro")); - break; - default: - throw new UnsupportedOperationException("Unknown write method " + method); - } - } - } - - List actualElements = new ArrayList<>(); - for (File outputFile : expectedFiles) { - assertTrue("Expected output file " + outputFile.getAbsolutePath(), outputFile.exists()); - try (DataFileReader reader = - new DataFileReader<>( - outputFile, - new ReflectDatumReader<>(ReflectData.get().getSchema(GenericClass.class)))) { - Iterators.addAll(actualElements, reader); - } - outputFile.delete(); - } - assertThat(actualElements, containsInAnyOrder(allElements.toArray())); - } - - private static final String SCHEMA_TEMPLATE_STRING = - "{\"namespace\": \"example.avro\",\n" - + " \"type\": \"record\",\n" - + " \"name\": \"$$TestTemplateSchema\",\n" - + " \"fields\": [\n" - + " {\"name\": \"$$full\", \"type\": \"string\"},\n" - + " {\"name\": \"$$suffix\", \"type\": [\"string\", \"null\"]}\n" - + " ]\n" - + "}"; - - private static String schemaFromPrefix(String prefix) { - return SCHEMA_TEMPLATE_STRING.replace("$$", prefix); - } - - private static GenericRecord createRecord(String record, String prefix, Schema schema) { - GenericRecord genericRecord = new GenericData.Record(schema); - genericRecord.put(prefix + "full", record); - genericRecord.put(prefix + "suffix", record.substring(1)); - return genericRecord; - } - - private static class TestDynamicDestinations - extends DynamicAvroDestinations { - final ResourceId baseDir; - final PCollectionView> schemaView; - - TestDynamicDestinations(ResourceId baseDir, PCollectionView> schemaView) { - this.baseDir = baseDir; - this.schemaView = schemaView; - } - - @Override - public Schema getSchema(String destination) { - // Return a per-destination schema. - String schema = sideInput(schemaView).get(destination); - return new Schema.Parser().parse(schema); - } - - @Override - public List> getSideInputs() { - return ImmutableList.of(schemaView); - } - - @Override - public GenericRecord formatRecord(String record) { - String prefix = record.substring(0, 1); - return createRecord(record, prefix, getSchema(prefix)); - } - - @Override - public String getDestination(String element) { - // Destination is based on first character of string. - return element.substring(0, 1); - } - - @Override - public String getDefaultDestination() { - return ""; - } - - @Override - public FilenamePolicy getFilenamePolicy(String destination) { - return DefaultFilenamePolicy.fromStandardParameters( - StaticValueProvider.of(baseDir.resolve("file_" + destination, RESOLVE_FILE)), - "-SSSSS-of-NNNNN", - ".avro", - false); - } - } - - /** - * Example of a {@link Coder} for a collection of Avro records with different schemas. - * - *

    All the schemas are known at pipeline construction, and are keyed internally on the prefix - * character (lower byte only for UTF-8 data). - */ - private static class AvroMultiplexCoder extends Coder { - - /** Lookup table for the possible schemas, keyed on the prefix character. */ - private final Map> coderMap = Maps.newHashMap(); - - protected AvroMultiplexCoder(Map schemaMap) { - for (Map.Entry entry : schemaMap.entrySet()) { - coderMap.put( - entry.getKey().charAt(0), AvroCoder.of(new Schema.Parser().parse(entry.getValue()))); - } - } - - @Override - public void encode(GenericRecord value, OutputStream outStream) throws IOException { - char prefix = value.getSchema().getName().charAt(0); - outStream.write(prefix); // Only reads and writes the low byte. - coderMap.get(prefix).encode(value, outStream); - } - - @Override - public GenericRecord decode(InputStream inStream) throws CoderException, IOException { - char prefix = (char) inStream.read(); - return coderMap.get(prefix).decode(inStream); - } - - @Override - public List> getCoderArguments() { - return Collections.emptyList(); - } - - @Override - public void verifyDeterministic() throws NonDeterministicException { - for (AvroCoder internalCoder : coderMap.values()) { - internalCoder.verifyDeterministic(); - } - } - } - - private void testDynamicDestinationsUnwindowedWithSharding( - WriteMethod writeMethod, Sharding sharding) throws Exception { - final ResourceId baseDir = - FileSystems.matchNewResource( - Files.createTempDirectory(tmpFolder.getRoot().toPath(), "testDynamicDestinations") - .toString(), - true); - - List elements = Lists.newArrayList("aaaa", "aaab", "baaa", "baab", "caaa", "caab"); - Multimap expectedElements = ArrayListMultimap.create(); - Map schemaMap = Maps.newHashMap(); - for (String element : elements) { - String prefix = element.substring(0, 1); - String jsonSchema = schemaFromPrefix(prefix); - schemaMap.put(prefix, jsonSchema); - expectedElements.put( - prefix, createRecord(element, prefix, new Schema.Parser().parse(jsonSchema))); - } - final PCollectionView> schemaView = - writePipeline.apply("createSchemaView", Create.of(schemaMap)).apply(View.asMap()); - - PCollection input = - writePipeline.apply("createInput", Create.of(elements).withCoder(StringUtf8Coder.of())); - - switch (writeMethod) { - case AVROIO_WRITE: - { - AvroIO.TypedWrite write = - AvroIO.writeCustomTypeToGenericRecords() - .to(new TestDynamicDestinations(baseDir, schemaView)) - .withTempDirectory(baseDir); - - switch (sharding) { - case RUNNER_DETERMINED: - break; - case WITHOUT_SHARDING: - write = write.withoutSharding(); - break; - case FIXED_3_SHARDS: - write = write.withNumShards(3); - break; - default: - throw new IllegalArgumentException("Unknown sharding " + sharding); - } - - input.apply(write); - break; - } - - case AVROIO_SINK_WITH_SCHEMA: - { - FileIO.Write write = - FileIO.writeDynamic() - .by( - fn( - (element, c) -> { - c.sideInput(schemaView); // Ignore result - return element.getSchema().getName().substring(0, 1); - }, - requiresSideInputs(schemaView))) - .via( - fn( - (dest, c) -> { - Schema schema = - new Schema.Parser().parse(c.sideInput(schemaView).get(dest)); - return AvroIO.sink(schema); - }, - requiresSideInputs(schemaView))) - .to(baseDir.toString()) - .withNaming( - fn( - (dest, c) -> { - c.sideInput(schemaView); // Ignore result - return FileIO.Write.defaultNaming("file_" + dest, ".avro"); - }, - requiresSideInputs(schemaView))) - .withTempDirectory(baseDir.toString()) - .withDestinationCoder(StringUtf8Coder.of()) - .withIgnoreWindowing(); - switch (sharding) { - case RUNNER_DETERMINED: - break; - case WITHOUT_SHARDING: - write = write.withNumShards(1); - break; - case FIXED_3_SHARDS: - write = write.withNumShards(3); - break; - default: - throw new IllegalArgumentException("Unknown sharding " + sharding); - } - - MapElements toRecord = - MapElements.via( - new SimpleFunction() { - @Override - public GenericRecord apply(String element) { - String prefix = element.substring(0, 1); - GenericRecord record = - new GenericData.Record( - new Schema.Parser().parse(schemaFromPrefix(prefix))); - record.put(prefix + "full", element); - record.put(prefix + "suffix", element.substring(1)); - return record; - } - }); - - input.apply(toRecord).setCoder(new AvroMultiplexCoder(schemaMap)).apply(write); - break; - } - - case AVROIO_SINK_WITH_FORMATTER: - { - final AvroIO.RecordFormatter formatter = - (element, schema) -> { - String prefix = element.substring(0, 1); - GenericRecord record = new GenericData.Record(schema); - record.put(prefix + "full", element); - record.put(prefix + "suffix", element.substring(1)); - return record; - }; - FileIO.Write write = - FileIO.writeDynamic() - .by( - fn( - (element, c) -> { - c.sideInput(schemaView); // Ignore result - return element.substring(0, 1); - }, - requiresSideInputs(schemaView))) - .via( - fn( - (dest, c) -> { - Schema schema = - new Schema.Parser().parse(c.sideInput(schemaView).get(dest)); - return AvroIO.sinkViaGenericRecords(schema, formatter); - }, - requiresSideInputs(schemaView))) - .to(baseDir.toString()) - .withNaming( - fn( - (dest, c) -> { - c.sideInput(schemaView); // Ignore result - return FileIO.Write.defaultNaming("file_" + dest, ".avro"); - }, - requiresSideInputs(schemaView))) - .withTempDirectory(baseDir.toString()) - .withDestinationCoder(StringUtf8Coder.of()) - .withIgnoreWindowing(); - switch (sharding) { - case RUNNER_DETERMINED: - break; - case WITHOUT_SHARDING: - write = write.withNumShards(1); - break; - case FIXED_3_SHARDS: - write = write.withNumShards(3); - break; - default: - throw new IllegalArgumentException("Unknown sharding " + sharding); - } - - input.apply(write); - break; - } - default: - throw new UnsupportedOperationException("Unknown write method " + writeMethod); - } - - writePipeline.run(); - - // Validate that the data written matches the expected elements in the expected order. - - for (String prefix : expectedElements.keySet()) { - String shardPattern; - switch (sharding) { - case RUNNER_DETERMINED: - shardPattern = "-*"; - break; - case WITHOUT_SHARDING: - shardPattern = "-00000-of-00001"; - break; - case FIXED_3_SHARDS: - shardPattern = "-*-of-00003"; - break; - default: - throw new IllegalArgumentException("Unknown sharding " + sharding); - } - String expectedFilepattern = - baseDir.resolve("file_" + prefix + shardPattern + ".avro", RESOLVE_FILE).toString(); - - PCollection records = - readPipeline.apply( - "read_" + prefix, - AvroIO.readGenericRecords(schemaFromPrefix(prefix)) - .withBeamSchemas(withBeamSchemas) - .from(expectedFilepattern)); - PAssert.that(records).containsInAnyOrder(expectedElements.get(prefix)); - } - readPipeline.run(); - } - - @Test - @Category(NeedsRunner.class) - public void testDynamicDestinationsRunnerDeterminedSharding() throws Exception { - testDynamicDestinationsUnwindowedWithSharding( - WriteMethod.AVROIO_WRITE, Sharding.RUNNER_DETERMINED); - } - - @Test - @Category(NeedsRunner.class) - public void testDynamicDestinationsWithoutSharding() throws Exception { - testDynamicDestinationsUnwindowedWithSharding( - WriteMethod.AVROIO_WRITE, Sharding.WITHOUT_SHARDING); - } - - @Test - @Category(NeedsRunner.class) - public void testDynamicDestinationsWithNumShards() throws Exception { - testDynamicDestinationsUnwindowedWithSharding( - WriteMethod.AVROIO_WRITE, Sharding.FIXED_3_SHARDS); - } - - @Test - @Category(NeedsRunner.class) - public void testDynamicDestinationsViaSinkRunnerDeterminedSharding() throws Exception { - testDynamicDestinationsUnwindowedWithSharding( - WriteMethod.AVROIO_SINK_WITH_SCHEMA, Sharding.RUNNER_DETERMINED); - } - - @Test - @Category(NeedsRunner.class) - public void testDynamicDestinationsViaSinkWithoutSharding() throws Exception { - testDynamicDestinationsUnwindowedWithSharding( - WriteMethod.AVROIO_SINK_WITH_SCHEMA, Sharding.WITHOUT_SHARDING); - } - - @Test - @Category(NeedsRunner.class) - public void testDynamicDestinationsViaSinkWithNumShards() throws Exception { - testDynamicDestinationsUnwindowedWithSharding( - WriteMethod.AVROIO_SINK_WITH_SCHEMA, Sharding.FIXED_3_SHARDS); - } - - @Test - @Category(NeedsRunner.class) - public void testDynamicDestinationsViaSinkWithFormatterRunnerDeterminedSharding() - throws Exception { - testDynamicDestinationsUnwindowedWithSharding( - WriteMethod.AVROIO_SINK_WITH_FORMATTER, Sharding.RUNNER_DETERMINED); - } - - @Test - @Category(NeedsRunner.class) - public void testDynamicDestinationsViaSinkWithFormatterWithoutSharding() throws Exception { - testDynamicDestinationsUnwindowedWithSharding( - WriteMethod.AVROIO_SINK_WITH_FORMATTER, Sharding.WITHOUT_SHARDING); - } - - @Test - @Category(NeedsRunner.class) - public void testDynamicDestinationsViaSinkWithFormatterWithNumShards() throws Exception { - testDynamicDestinationsUnwindowedWithSharding( - WriteMethod.AVROIO_SINK_WITH_FORMATTER, Sharding.FIXED_3_SHARDS); - } - - @Test - @SuppressWarnings("unchecked") - @Category(NeedsRunner.class) - public void testMetadata() throws Exception { - List values = - ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar")); - File outputFile = tmpFolder.newFile("output.avro"); - - writePipeline - .apply(Create.of(values)) - .apply( - AvroIO.write(GenericClass.class) - .to(outputFile.getAbsolutePath()) - .withoutSharding() - .withMetadata( - ImmutableMap.of( - "stringKey", - "stringValue", - "longKey", - 100L, - "bytesKey", - "bytesValue".getBytes(Charsets.UTF_8)))); - writePipeline.run(); - - try (DataFileStream dataFileStream = - new DataFileStream(new FileInputStream(outputFile), new GenericDatumReader())) { - assertEquals("stringValue", dataFileStream.getMetaString("stringKey")); - assertEquals(100L, dataFileStream.getMetaLong("longKey")); - assertArrayEquals( - "bytesValue".getBytes(Charsets.UTF_8), dataFileStream.getMeta("bytesKey")); - } - } - - // using AvroCoder#createDatumReader for tests. - private void runTestWrite(String[] expectedElements, int numShards) throws IOException { - File baseOutputFile = new File(tmpFolder.getRoot(), "prefix"); - String outputFilePrefix = baseOutputFile.getAbsolutePath(); - - AvroIO.Write write = - AvroIO.write(String.class).to(outputFilePrefix).withSuffix(".avro"); - if (numShards > 1) { - write = write.withNumShards(numShards); - } else { - write = write.withoutSharding(); - } - writePipeline.apply(Create.of(ImmutableList.copyOf(expectedElements))).apply(write); - writePipeline.run(); - - String shardNameTemplate = - firstNonNull( - write.inner.getShardTemplate(), - DefaultFilenamePolicy.DEFAULT_UNWINDOWED_SHARD_TEMPLATE); - - assertTestOutputs(expectedElements, numShards, outputFilePrefix, shardNameTemplate); - } - - static void assertTestOutputs( - String[] expectedElements, int numShards, String outputFilePrefix, String shardNameTemplate) - throws IOException { - // Validate that the data written matches the expected elements in the expected order - List expectedFiles = new ArrayList<>(); - for (int i = 0; i < numShards; i++) { - expectedFiles.add( - new File( - DefaultFilenamePolicy.constructName( - FileBasedSink.convertToFileResourceIfPossible(outputFilePrefix), - shardNameTemplate, - ".avro", - i, - numShards, - null, - null) - .toString())); - } - - List actualElements = new ArrayList<>(); - for (File outputFile : expectedFiles) { - assertTrue("Expected output file " + outputFile.getName(), outputFile.exists()); - try (DataFileReader reader = - new DataFileReader<>( - outputFile, new ReflectDatumReader(ReflectData.get().getSchema(String.class)))) { - Iterators.addAll(actualElements, reader); - } - } - assertThat(actualElements, containsInAnyOrder(expectedElements)); - } - - @Test - @Category(NeedsRunner.class) - public void testAvroSinkWrite() throws Exception { - String[] expectedElements = new String[] {"first", "second", "third"}; - - runTestWrite(expectedElements, 1); - } - - @Test - @Category(NeedsRunner.class) - public void testAvroSinkShardedWrite() throws Exception { - String[] expectedElements = new String[] {"first", "second", "third", "fourth", "fifth"}; - - runTestWrite(expectedElements, 4); - } - - @Test - @Category(NeedsRunner.class) - public void testAvroSinkWriteWithCustomFactory() throws Exception { - Integer[] expectedElements = new Integer[] {1, 2, 3, 4, 5}; - - File baseOutputFile = new File(tmpFolder.getRoot(), "prefix"); - String outputFilePrefix = baseOutputFile.getAbsolutePath(); - - Schema recordSchema = SchemaBuilder.record("root").fields().requiredInt("i1").endRecord(); - - AvroIO.TypedWrite write = - AvroIO.writeCustomType() - .to(outputFilePrefix) - .withSchema(recordSchema) - .withFormatFunction(f -> f) - .withDatumWriterFactory( - f -> - new DatumWriter() { - private DatumWriter inner = new GenericDatumWriter<>(f); - - @Override - public void setSchema(Schema schema) { - inner.setSchema(schema); - } - - @Override - public void write(Integer datum, Encoder out) throws IOException { - GenericRecord record = - new GenericRecordBuilder(f).set("i1", datum).build(); - inner.write(record, out); - } - }) - .withSuffix(".avro"); - - write = write.withoutSharding(); - - writePipeline.apply(Create.of(ImmutableList.copyOf(expectedElements))).apply(write); - writePipeline.run(); - - File expectedFile = - new File( - DefaultFilenamePolicy.constructName( - FileBasedSink.convertToFileResourceIfPossible(outputFilePrefix), - "", - ".avro", - 1, - 1, - null, - null) - .toString()); - - assertTrue("Expected output file " + expectedFile.getName(), expectedFile.exists()); - DataFileReader dataFileReader = - new DataFileReader<>(expectedFile, new GenericDatumReader<>(recordSchema)); - - List actualRecords = new ArrayList<>(); - Iterators.addAll(actualRecords, dataFileReader); - - GenericRecord[] expectedRecords = - Arrays.stream(expectedElements) - .map(i -> new GenericRecordBuilder(recordSchema).set("i1", i).build()) - .toArray(GenericRecord[]::new); - - assertThat(actualRecords, containsInAnyOrder(expectedRecords)); - } - - // TODO: for Write only, test withSuffix, - // withShardNameTemplate and withoutSharding. - } -} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroSchemaIOProviderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroSchemaIOProviderTest.java deleted file mode 100644 index cf68633dc7ae3..0000000000000 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroSchemaIOProviderTest.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io; - -import static org.junit.Assert.assertEquals; - -import java.io.File; -import java.time.Duration; -import java.util.Arrays; -import java.util.List; -import org.apache.beam.sdk.coders.RowCoder; -import org.apache.beam.sdk.io.fs.MatchResult; -import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.sdk.schemas.io.SchemaIO; -import org.apache.beam.sdk.testing.NeedsRunner; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.testing.TestStream; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.Row; -import org.apache.beam.sdk.values.TimestampedValue; -import org.joda.time.Instant; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Test for AvroSchemaIOProvider. */ -@RunWith(JUnit4.class) -public class AvroSchemaIOProviderTest { - @Rule public TestPipeline writePipeline = TestPipeline.create(); - @Rule public TestPipeline readPipeline = TestPipeline.create(); - @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); - - private static final Schema SCHEMA = - Schema.builder().addInt64Field("age").addStringField("age_str").build(); - - private Row createRow(long l) { - return Row.withSchema(SCHEMA).addValues(l, Long.valueOf(l).toString()).build(); - } - - @Test - @Category({NeedsRunner.class}) - public void testWriteAndReadTable() { - File destinationFile = new File(tempFolder.getRoot(), "person-info.avro"); - - AvroSchemaIOProvider provider = new AvroSchemaIOProvider(); - Row configuration = Row.withSchema(provider.configurationSchema()).addValue(null).build(); - SchemaIO io = provider.from(destinationFile.getAbsolutePath(), configuration, SCHEMA); - - List rowsList = Arrays.asList(createRow(1L), createRow(3L), createRow(4L)); - PCollection rows = - writePipeline.apply("Create", Create.of(rowsList).withCoder(RowCoder.of(SCHEMA))); - rows.apply(io.buildWriter()); - writePipeline.run(); - - PCollection read = readPipeline.begin().apply(io.buildReader()); - PAssert.that(read).containsInAnyOrder(rowsList); - readPipeline.run(); - } - - @Test - @Category({NeedsRunner.class}) - public void testStreamingWriteDefault() throws Exception { - File destinationFile = new File(tempFolder.getRoot(), "person-info"); - - AvroSchemaIOProvider provider = new AvroSchemaIOProvider(); - Row config = Row.withSchema(provider.configurationSchema()).addValue(null).build(); - SchemaIO writeIO = provider.from(destinationFile.getAbsolutePath(), config, SCHEMA); - - TestStream createEvents = - TestStream.create(RowCoder.of(SCHEMA)) - .addElements(TimestampedValue.of(createRow(1L), new Instant(1L))) - .addElements(TimestampedValue.of(createRow(2L), Instant.ofEpochSecond(120L))) - .advanceWatermarkToInfinity(); - - writePipeline.apply("create", createEvents).apply("write", writeIO.buildWriter()); - writePipeline.run(); - - // Verify we wrote two files. - String wildcardPath = destinationFile.getAbsolutePath() + "*"; - MatchResult result = FileSystems.match(wildcardPath); - assertEquals(2, result.metadata().size()); - - // Verify results of the files. - SchemaIO readIO = provider.from(wildcardPath, config, SCHEMA); - PCollection read = readPipeline.begin().apply("read", readIO.buildReader()); - PAssert.that(read).containsInAnyOrder(createRow(1L), createRow(2L)); - readPipeline.run(); - } - - @Test - @Category({NeedsRunner.class}) - public void testStreamingCustomWindowSize() throws Exception { - File destinationFile = new File(tempFolder.getRoot(), "person-info"); - - AvroSchemaIOProvider provider = new AvroSchemaIOProvider(); - Row config = - Row.withSchema(provider.configurationSchema()) - .addValue(Duration.ofMinutes(4).getSeconds()) - .build(); - SchemaIO writeIO = provider.from(destinationFile.getAbsolutePath(), config, SCHEMA); - - TestStream createEvents = - TestStream.create(RowCoder.of(SCHEMA)) - .addElements(TimestampedValue.of(createRow(1L), new Instant(1L))) - .addElements(TimestampedValue.of(createRow(2L), Instant.ofEpochSecond(120L))) - .advanceWatermarkToInfinity(); - - writePipeline.apply("create", createEvents).apply("write", writeIO.buildWriter()); - writePipeline.run(); - - // Verify we wrote one file. - String wildcardPath = destinationFile.getAbsolutePath() + "*"; - MatchResult result = FileSystems.match(wildcardPath); - assertEquals(1, result.metadata().size()); - - // Verify results of the files. - SchemaIO readIO = provider.from(wildcardPath, config, SCHEMA); - PCollection read = readPipeline.begin().apply("read", readIO.buildReader()); - PAssert.that(read).containsInAnyOrder(createRow(1L), createRow(2L)); - readPipeline.run(); - } - - @Test - @Category({NeedsRunner.class}) - public void testBatchCustomWindowSize() throws Exception { - File destinationFile = new File(tempFolder.getRoot(), "person-info"); - - AvroSchemaIOProvider provider = new AvroSchemaIOProvider(); - Row config = - Row.withSchema(provider.configurationSchema()) - .addValue(Duration.ofMinutes(4).getSeconds()) - .build(); - SchemaIO writeIO = provider.from(destinationFile.getAbsolutePath(), config, SCHEMA); - - List rowsList = Arrays.asList(createRow(1L), createRow(3L), createRow(4L)); - PCollection rows = - writePipeline.apply("Create", Create.of(rowsList).withCoder(RowCoder.of(SCHEMA))); - - rows.apply("write", writeIO.buildWriter()); - writePipeline.run(); - - // Verify we wrote one file. - String wildcardPath = destinationFile.getAbsolutePath() + "*"; - MatchResult result = FileSystems.match(wildcardPath); - assertEquals(1, result.metadata().size()); - - // Verify results of the files. - SchemaIO readIO = provider.from(wildcardPath, config, SCHEMA); - PCollection read = readPipeline.begin().apply("read", readIO.buildReader()); - PAssert.that(read).containsInAnyOrder(rowsList); - readPipeline.run(); - } -} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroSourceTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroSourceTest.java deleted file mode 100644 index c9afb4713b3f5..0000000000000 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroSourceTest.java +++ /dev/null @@ -1,843 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io; - -import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.Objects; -import java.util.Random; -import java.util.stream.Collectors; -import org.apache.avro.Schema; -import org.apache.avro.file.CodecFactory; -import org.apache.avro.file.DataFileConstants; -import org.apache.avro.file.DataFileWriter; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.io.Decoder; -import org.apache.avro.reflect.AvroDefault; -import org.apache.avro.reflect.ReflectData; -import org.apache.avro.reflect.ReflectDatumWriter; -import org.apache.beam.sdk.coders.AvroCoder; -import org.apache.beam.sdk.coders.DefaultCoder; -import org.apache.beam.sdk.io.AvroSource.AvroMetadata; -import org.apache.beam.sdk.io.BlockBasedSource.BlockBasedReader; -import org.apache.beam.sdk.io.BoundedSource.BoundedReader; -import org.apache.beam.sdk.io.fs.MatchResult.Metadata; -import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.sdk.testing.SourceTestUtils; -import org.apache.beam.sdk.transforms.display.DisplayData; -import org.apache.beam.sdk.util.SerializableUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.checkerframework.checker.nullness.qual.Nullable; -import org.hamcrest.Matchers; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Tests for AvroSource. */ -@RunWith(JUnit4.class) -public class AvroSourceTest { - @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); - - @Rule public ExpectedException expectedException = ExpectedException.none(); - - private enum SyncBehavior { - SYNC_REGULAR, // Sync at regular, user defined intervals - SYNC_RANDOM, // Sync at random intervals - SYNC_DEFAULT // Sync at default intervals (i.e., no manual syncing). - } - - private static final int DEFAULT_RECORD_COUNT = 1000; - - /** - * Generates an input Avro file containing the given records in the temporary directory and - * returns the full path of the file. - */ - private String generateTestFile( - String filename, - List elems, - SyncBehavior syncBehavior, - int syncInterval, - AvroCoder coder, - String codec) - throws IOException { - Random random = new Random(0); - File tmpFile = tmpFolder.newFile(filename); - String path = tmpFile.toString(); - - FileOutputStream os = new FileOutputStream(tmpFile); - DatumWriter datumWriter = - coder.getType().equals(GenericRecord.class) - ? new GenericDatumWriter<>(coder.getSchema()) - : new ReflectDatumWriter<>(coder.getSchema()); - try (DataFileWriter writer = new DataFileWriter<>(datumWriter)) { - writer.setCodec(CodecFactory.fromString(codec)); - writer.create(coder.getSchema(), os); - - int recordIndex = 0; - int syncIndex = syncBehavior == SyncBehavior.SYNC_RANDOM ? random.nextInt(syncInterval) : 0; - - for (T elem : elems) { - writer.append(elem); - recordIndex++; - - switch (syncBehavior) { - case SYNC_REGULAR: - if (recordIndex == syncInterval) { - recordIndex = 0; - writer.sync(); - } - break; - case SYNC_RANDOM: - if (recordIndex == syncIndex) { - recordIndex = 0; - writer.sync(); - syncIndex = random.nextInt(syncInterval); - } - break; - case SYNC_DEFAULT: - default: - } - } - } - return path; - } - - @Test - public void testReadWithDifferentCodecs() throws Exception { - // Test reading files generated using all codecs. - String[] codecs = { - DataFileConstants.NULL_CODEC, - DataFileConstants.BZIP2_CODEC, - DataFileConstants.DEFLATE_CODEC, - DataFileConstants.SNAPPY_CODEC, - DataFileConstants.XZ_CODEC, - }; - // As Avro's default block size is 64KB, write 64K records to ensure at least one full block. - // We could make this smaller than 64KB assuming each record is at least B bytes, but then the - // test could silently stop testing the failure condition from BEAM-422. - List expected = createRandomRecords(1 << 16); - - for (String codec : codecs) { - String filename = - generateTestFile( - codec, expected, SyncBehavior.SYNC_DEFAULT, 0, AvroCoder.of(Bird.class), codec); - AvroSource source = AvroSource.from(filename).withSchema(Bird.class); - List actual = SourceTestUtils.readFromSource(source, null); - assertThat(expected, containsInAnyOrder(actual.toArray())); - } - } - - @Test - public void testSplitAtFraction() throws Exception { - // A reduced dataset is enough here. - List expected = createFixedRecords(DEFAULT_RECORD_COUNT); - // Create an AvroSource where each block is 1/10th of the total set of records. - String filename = - generateTestFile( - "tmp.avro", - expected, - SyncBehavior.SYNC_REGULAR, - DEFAULT_RECORD_COUNT / 10 /* max records per block */, - AvroCoder.of(FixedRecord.class), - DataFileConstants.NULL_CODEC); - File file = new File(filename); - - AvroSource source = AvroSource.from(filename).withSchema(FixedRecord.class); - List> splits = source.split(file.length() / 3, null); - for (BoundedSource subSource : splits) { - int items = SourceTestUtils.readFromSource(subSource, null).size(); - // Shouldn't split while unstarted. - SourceTestUtils.assertSplitAtFractionFails(subSource, 0, 0.0, null); - SourceTestUtils.assertSplitAtFractionFails(subSource, 0, 0.7, null); - SourceTestUtils.assertSplitAtFractionSucceedsAndConsistent(subSource, 1, 0.7, null); - SourceTestUtils.assertSplitAtFractionSucceedsAndConsistent( - subSource, DEFAULT_RECORD_COUNT / 100, 0.7, null); - SourceTestUtils.assertSplitAtFractionSucceedsAndConsistent( - subSource, DEFAULT_RECORD_COUNT / 10, 0.1, null); - SourceTestUtils.assertSplitAtFractionFails( - subSource, DEFAULT_RECORD_COUNT / 10 + 1, 0.1, null); - SourceTestUtils.assertSplitAtFractionFails(subSource, DEFAULT_RECORD_COUNT / 3, 0.3, null); - SourceTestUtils.assertSplitAtFractionFails(subSource, items, 0.9, null); - SourceTestUtils.assertSplitAtFractionFails(subSource, items, 1.0, null); - SourceTestUtils.assertSplitAtFractionSucceedsAndConsistent(subSource, items, 0.999, null); - } - } - - @Test - public void testGetProgressFromUnstartedReader() throws Exception { - List records = createFixedRecords(DEFAULT_RECORD_COUNT); - String filename = - generateTestFile( - "tmp.avro", - records, - SyncBehavior.SYNC_DEFAULT, - 1000, - AvroCoder.of(FixedRecord.class), - DataFileConstants.NULL_CODEC); - File file = new File(filename); - - AvroSource source = AvroSource.from(filename).withSchema(FixedRecord.class); - try (BoundedSource.BoundedReader reader = source.createReader(null)) { - assertEquals(Double.valueOf(0.0), reader.getFractionConsumed()); - } - - List> splits = source.split(file.length() / 3, null); - for (BoundedSource subSource : splits) { - try (BoundedSource.BoundedReader reader = subSource.createReader(null)) { - assertEquals(Double.valueOf(0.0), reader.getFractionConsumed()); - } - } - } - - @Test - public void testProgress() throws Exception { - // 5 records, 2 per block. - List records = createFixedRecords(5); - String filename = - generateTestFile( - "tmp.avro", - records, - SyncBehavior.SYNC_REGULAR, - 2, - AvroCoder.of(FixedRecord.class), - DataFileConstants.NULL_CODEC); - - AvroSource source = AvroSource.from(filename).withSchema(FixedRecord.class); - try (BoundedSource.BoundedReader readerOrig = source.createReader(null)) { - assertThat(readerOrig, Matchers.instanceOf(BlockBasedReader.class)); - BlockBasedReader reader = (BlockBasedReader) readerOrig; - - // Before starting - assertEquals(0.0, reader.getFractionConsumed(), 1e-6); - assertEquals(0, reader.getSplitPointsConsumed()); - assertEquals(BoundedReader.SPLIT_POINTS_UNKNOWN, reader.getSplitPointsRemaining()); - - // First 2 records are in the same block. - assertTrue(reader.start()); - assertTrue(reader.isAtSplitPoint()); - assertEquals(0, reader.getSplitPointsConsumed()); - assertEquals(BoundedReader.SPLIT_POINTS_UNKNOWN, reader.getSplitPointsRemaining()); - // continued - assertTrue(reader.advance()); - assertFalse(reader.isAtSplitPoint()); - assertEquals(0, reader.getSplitPointsConsumed()); - assertEquals(BoundedReader.SPLIT_POINTS_UNKNOWN, reader.getSplitPointsRemaining()); - - // Second block -> parallelism consumed becomes 1. - assertTrue(reader.advance()); - assertTrue(reader.isAtSplitPoint()); - assertEquals(1, reader.getSplitPointsConsumed()); - assertEquals(BoundedReader.SPLIT_POINTS_UNKNOWN, reader.getSplitPointsRemaining()); - // continued - assertTrue(reader.advance()); - assertFalse(reader.isAtSplitPoint()); - assertEquals(1, reader.getSplitPointsConsumed()); - assertEquals(BoundedReader.SPLIT_POINTS_UNKNOWN, reader.getSplitPointsRemaining()); - - // Third and final block -> parallelism consumed becomes 2, remaining becomes 1. - assertTrue(reader.advance()); - assertTrue(reader.isAtSplitPoint()); - assertEquals(2, reader.getSplitPointsConsumed()); - assertEquals(1, reader.getSplitPointsRemaining()); - - // Done - assertFalse(reader.advance()); - assertEquals(3, reader.getSplitPointsConsumed()); - assertEquals(0, reader.getSplitPointsRemaining()); - assertEquals(1.0, reader.getFractionConsumed(), 1e-6); - } - } - - @Test - public void testProgressEmptySource() throws Exception { - // 0 records, 20 per block. - List records = Collections.emptyList(); - String filename = - generateTestFile( - "tmp.avro", - records, - SyncBehavior.SYNC_REGULAR, - 2, - AvroCoder.of(FixedRecord.class), - DataFileConstants.NULL_CODEC); - - AvroSource source = AvroSource.from(filename).withSchema(FixedRecord.class); - try (BoundedSource.BoundedReader readerOrig = source.createReader(null)) { - assertThat(readerOrig, Matchers.instanceOf(BlockBasedReader.class)); - BlockBasedReader reader = (BlockBasedReader) readerOrig; - - // before starting - assertEquals(0.0, reader.getFractionConsumed(), 1e-6); - assertEquals(0, reader.getSplitPointsConsumed()); - assertEquals(BoundedReader.SPLIT_POINTS_UNKNOWN, reader.getSplitPointsRemaining()); - - // confirm empty - assertFalse(reader.start()); - - // after reading empty source - assertEquals(0, reader.getSplitPointsConsumed()); - assertEquals(0, reader.getSplitPointsRemaining()); - assertEquals(1.0, reader.getFractionConsumed(), 1e-6); - } - } - - @Test - public void testGetCurrentFromUnstartedReader() throws Exception { - List records = createFixedRecords(DEFAULT_RECORD_COUNT); - String filename = - generateTestFile( - "tmp.avro", - records, - SyncBehavior.SYNC_DEFAULT, - 1000, - AvroCoder.of(FixedRecord.class), - DataFileConstants.NULL_CODEC); - - AvroSource source = AvroSource.from(filename).withSchema(FixedRecord.class); - try (BlockBasedSource.BlockBasedReader reader = - (BlockBasedSource.BlockBasedReader) source.createReader(null)) { - assertEquals(null, reader.getCurrentBlock()); - - expectedException.expect(NoSuchElementException.class); - expectedException.expectMessage("No block has been successfully read from"); - reader.getCurrent(); - } - } - - @Test - public void testSplitAtFractionExhaustive() throws Exception { - // A small-sized input is sufficient, because the test verifies that splitting is non-vacuous. - List expected = createFixedRecords(20); - String filename = - generateTestFile( - "tmp.avro", - expected, - SyncBehavior.SYNC_REGULAR, - 5, - AvroCoder.of(FixedRecord.class), - DataFileConstants.NULL_CODEC); - - AvroSource source = AvroSource.from(filename).withSchema(FixedRecord.class); - SourceTestUtils.assertSplitAtFractionExhaustive(source, null); - } - - @Test - public void testSplitsWithSmallBlocks() throws Exception { - PipelineOptions options = PipelineOptionsFactory.create(); - // Test reading from an object file with many small random-sized blocks. - // The file itself doesn't have to be big; we can use a decreased record count. - List expected = createRandomRecords(DEFAULT_RECORD_COUNT); - String filename = - generateTestFile( - "tmp.avro", - expected, - SyncBehavior.SYNC_RANDOM, - DEFAULT_RECORD_COUNT / 20 /* max records/block */, - AvroCoder.of(Bird.class), - DataFileConstants.NULL_CODEC); - File file = new File(filename); - - // Small minimum bundle size - AvroSource source = - AvroSource.from(filename).withSchema(Bird.class).withMinBundleSize(100L); - - // Assert that the source produces the expected records - assertEquals(expected, SourceTestUtils.readFromSource(source, options)); - - List> splits; - int nonEmptySplits; - - // Split with the minimum bundle size - splits = source.split(100L, options); - assertTrue(splits.size() > 2); - SourceTestUtils.assertSourcesEqualReferenceSource(source, splits, options); - nonEmptySplits = 0; - for (BoundedSource subSource : splits) { - if (SourceTestUtils.readFromSource(subSource, options).size() > 0) { - nonEmptySplits += 1; - } - } - assertTrue(nonEmptySplits > 2); - - // Split with larger bundle size - splits = source.split(file.length() / 4, options); - assertTrue(splits.size() > 2); - SourceTestUtils.assertSourcesEqualReferenceSource(source, splits, options); - nonEmptySplits = 0; - for (BoundedSource subSource : splits) { - if (SourceTestUtils.readFromSource(subSource, options).size() > 0) { - nonEmptySplits += 1; - } - } - assertTrue(nonEmptySplits > 2); - - // Split with the file length - splits = source.split(file.length(), options); - assertTrue(splits.size() == 1); - SourceTestUtils.assertSourcesEqualReferenceSource(source, splits, options); - } - - @Test - public void testMultipleFiles() throws Exception { - String baseName = "tmp-"; - List expected = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - List contents = createRandomRecords(DEFAULT_RECORD_COUNT / 10); - expected.addAll(contents); - generateTestFile( - baseName + i, - contents, - SyncBehavior.SYNC_DEFAULT, - 0, - AvroCoder.of(Bird.class), - DataFileConstants.NULL_CODEC); - } - - AvroSource source = - AvroSource.from(new File(tmpFolder.getRoot().toString(), baseName + "*").toString()) - .withSchema(Bird.class); - List actual = SourceTestUtils.readFromSource(source, null); - assertThat(actual, containsInAnyOrder(expected.toArray())); - } - - @Test - public void testCreationWithSchema() throws Exception { - List expected = createRandomRecords(100); - String filename = - generateTestFile( - "tmp.avro", - expected, - SyncBehavior.SYNC_DEFAULT, - 0, - AvroCoder.of(Bird.class), - DataFileConstants.NULL_CODEC); - - // Create a source with a schema object - Schema schema = ReflectData.get().getSchema(Bird.class); - AvroSource source = AvroSource.from(filename).withSchema(schema); - List records = SourceTestUtils.readFromSource(source, null); - assertEqualsWithGeneric(expected, records); - - // Create a source with a JSON schema - String schemaString = ReflectData.get().getSchema(Bird.class).toString(); - source = AvroSource.from(filename).withSchema(schemaString); - records = SourceTestUtils.readFromSource(source, null); - assertEqualsWithGeneric(expected, records); - } - - @Test - public void testSchemaUpdate() throws Exception { - List birds = createRandomRecords(100); - String filename = - generateTestFile( - "tmp.avro", - birds, - SyncBehavior.SYNC_DEFAULT, - 0, - AvroCoder.of(Bird.class), - DataFileConstants.NULL_CODEC); - - AvroSource source = AvroSource.from(filename).withSchema(FancyBird.class); - List actual = SourceTestUtils.readFromSource(source, null); - - List expected = new ArrayList<>(); - for (Bird bird : birds) { - expected.add( - new FancyBird( - bird.number, bird.species, bird.quality, bird.quantity, null, "MAXIMUM OVERDRIVE")); - } - - assertThat(actual, containsInAnyOrder(expected.toArray())); - } - - @Test - public void testSchemaStringIsInterned() throws Exception { - List birds = createRandomRecords(100); - String filename = - generateTestFile( - "tmp.avro", - birds, - SyncBehavior.SYNC_DEFAULT, - 0, - AvroCoder.of(Bird.class), - DataFileConstants.NULL_CODEC); - Metadata fileMetadata = FileSystems.matchSingleFileSpec(filename); - String schema = AvroSource.readMetadataFromFile(fileMetadata.resourceId()).getSchemaString(); - // Add "" to the schema to make sure it is not interned. - AvroSource sourceA = AvroSource.from(filename).withSchema("" + schema); - AvroSource sourceB = AvroSource.from(filename).withSchema("" + schema); - assertSame(sourceA.getReaderSchemaString(), sourceB.getReaderSchemaString()); - - // Ensure that deserialization still goes through interning - AvroSource sourceC = SerializableUtils.clone(sourceB); - assertSame(sourceA.getReaderSchemaString(), sourceC.getReaderSchemaString()); - } - - @Test - public void testParseFn() throws Exception { - List expected = createRandomRecords(100); - String filename = - generateTestFile( - "tmp.avro", - expected, - SyncBehavior.SYNC_DEFAULT, - 0, - AvroCoder.of(Bird.class), - DataFileConstants.NULL_CODEC); - - AvroSource source = - AvroSource.from(filename) - .withParseFn( - input -> - new Bird( - (long) input.get("number"), - input.get("species").toString(), - input.get("quality").toString(), - (long) input.get("quantity")), - AvroCoder.of(Bird.class)); - List actual = SourceTestUtils.readFromSource(source, null); - assertThat(actual, containsInAnyOrder(expected.toArray())); - } - - @Test - public void testDatumReaderFactoryWithGenericRecord() throws Exception { - List inputBirds = createRandomRecords(100); - - String filename = - generateTestFile( - "tmp.avro", - inputBirds, - SyncBehavior.SYNC_DEFAULT, - 0, - AvroCoder.of(Bird.class), - DataFileConstants.NULL_CODEC); - - AvroSource.DatumReaderFactory factory = - (writer, reader) -> - new GenericDatumReader(writer, reader) { - @Override - protected Object readString(Object old, Decoder in) throws IOException { - return super.readString(old, in) + "_custom"; - } - }; - - AvroSource source = - AvroSource.from(filename) - .withParseFn( - input -> - new Bird( - (long) input.get("number"), - input.get("species").toString(), - input.get("quality").toString(), - (long) input.get("quantity")), - AvroCoder.of(Bird.class)) - .withDatumReaderFactory(factory); - List actual = SourceTestUtils.readFromSource(source, null); - List expected = - inputBirds.stream() - .map(b -> new Bird(b.number, b.species + "_custom", b.quality + "_custom", b.quantity)) - .collect(Collectors.toList()); - - assertThat(actual, containsInAnyOrder(expected.toArray())); - } - - private void assertEqualsWithGeneric(List expected, List actual) { - assertEquals(expected.size(), actual.size()); - for (int i = 0; i < expected.size(); i++) { - Bird fixed = expected.get(i); - GenericRecord generic = actual.get(i); - assertEquals(fixed.number, generic.get("number")); - assertEquals(fixed.quality, generic.get("quality").toString()); // From Avro util.Utf8 - assertEquals(fixed.quantity, generic.get("quantity")); - assertEquals(fixed.species, generic.get("species").toString()); - } - } - - @Test - public void testDisplayData() { - AvroSource source = - AvroSource.from("foobar.txt").withSchema(Bird.class).withMinBundleSize(1234); - - DisplayData displayData = DisplayData.from(source); - assertThat(displayData, hasDisplayItem("filePattern", "foobar.txt")); - assertThat(displayData, hasDisplayItem("minBundleSize", 1234)); - } - - @Test - public void testReadMetadataWithCodecs() throws Exception { - // Test reading files generated using all codecs. - String[] codecs = { - DataFileConstants.NULL_CODEC, - DataFileConstants.BZIP2_CODEC, - DataFileConstants.DEFLATE_CODEC, - DataFileConstants.SNAPPY_CODEC, - DataFileConstants.XZ_CODEC - }; - List expected = createRandomRecords(DEFAULT_RECORD_COUNT); - - for (String codec : codecs) { - String filename = - generateTestFile( - codec, expected, SyncBehavior.SYNC_DEFAULT, 0, AvroCoder.of(Bird.class), codec); - - Metadata fileMeta = FileSystems.matchSingleFileSpec(filename); - AvroMetadata metadata = AvroSource.readMetadataFromFile(fileMeta.resourceId()); - assertEquals(codec, metadata.getCodec()); - } - } - - @Test - public void testReadSchemaString() throws Exception { - List expected = createRandomRecords(DEFAULT_RECORD_COUNT); - String codec = DataFileConstants.NULL_CODEC; - String filename = - generateTestFile( - codec, expected, SyncBehavior.SYNC_DEFAULT, 0, AvroCoder.of(Bird.class), codec); - Metadata fileMeta = FileSystems.matchSingleFileSpec(filename); - AvroMetadata metadata = AvroSource.readMetadataFromFile(fileMeta.resourceId()); - // By default, parse validates the schema, which is what we want. - Schema schema = new Schema.Parser().parse(metadata.getSchemaString()); - assertEquals(4, schema.getFields().size()); - } - - @Test - public void testCreateFromMetadata() throws Exception { - List expected = createRandomRecords(DEFAULT_RECORD_COUNT); - String codec = DataFileConstants.NULL_CODEC; - String filename = - generateTestFile( - codec, expected, SyncBehavior.SYNC_DEFAULT, 0, AvroCoder.of(Bird.class), codec); - Metadata fileMeta = FileSystems.matchSingleFileSpec(filename); - - AvroSource source = AvroSource.from(fileMeta); - AvroSource sourceWithSchema = source.withSchema(Bird.class); - AvroSource sourceWithSchemaWithMinBundleSize = sourceWithSchema.withMinBundleSize(1234); - - assertEquals(FileBasedSource.Mode.SINGLE_FILE_OR_SUBRANGE, source.getMode()); - assertEquals(FileBasedSource.Mode.SINGLE_FILE_OR_SUBRANGE, sourceWithSchema.getMode()); - assertEquals( - FileBasedSource.Mode.SINGLE_FILE_OR_SUBRANGE, sourceWithSchemaWithMinBundleSize.getMode()); - } - - /** - * Class that will encode to a fixed size: 16 bytes. - * - *

    Each object has a 15-byte array. Avro encodes an object of this type as a byte array, so - * each encoded object will consist of 1 byte that encodes the length of the array, followed by 15 - * bytes. - */ - @DefaultCoder(AvroCoder.class) - public static class FixedRecord { - private byte[] value = new byte[15]; - - public FixedRecord() { - this(0); - } - - public FixedRecord(int i) { - value[0] = (byte) i; - value[1] = (byte) (i >> 8); - value[2] = (byte) (i >> 16); - value[3] = (byte) (i >> 24); - } - - public int asInt() { - return value[0] | (value[1] << 8) | (value[2] << 16) | (value[3] << 24); - } - - @Override - public boolean equals(@Nullable Object o) { - if (o instanceof FixedRecord) { - FixedRecord other = (FixedRecord) o; - return this.asInt() == other.asInt(); - } - return false; - } - - @Override - public int hashCode() { - return toString().hashCode(); - } - - @Override - public String toString() { - return Integer.toString(this.asInt()); - } - } - - /** Create a list of count 16-byte records. */ - private static List createFixedRecords(int count) { - List records = new ArrayList<>(); - for (int i = 0; i < count; i++) { - records.add(new FixedRecord(i)); - } - return records; - } - - /** Class used as the record type in tests. */ - @DefaultCoder(AvroCoder.class) - static class Bird { - long number; - String species; - String quality; - long quantity; - - public Bird() {} - - public Bird(long number, String species, String quality, long quantity) { - this.number = number; - this.species = species; - this.quality = quality; - this.quantity = quantity; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(Bird.class) - .addValue(number) - .addValue(species) - .addValue(quantity) - .addValue(quality) - .toString(); - } - - @Override - public boolean equals(@Nullable Object obj) { - if (obj instanceof Bird) { - Bird other = (Bird) obj; - return Objects.equals(species, other.species) - && Objects.equals(quality, other.quality) - && quantity == other.quantity - && number == other.number; - } - return false; - } - - @Override - public int hashCode() { - return Objects.hash(number, species, quality, quantity); - } - } - - /** - * Class used as the record type in tests. - * - *

    Contains nullable fields and fields with default values. Can be read using a file written - * with the Bird schema. - */ - @DefaultCoder(AvroCoder.class) - public static class FancyBird { - long number; - String species; - String quality; - long quantity; - - @org.apache.avro.reflect.Nullable String habitat; - - @AvroDefault("\"MAXIMUM OVERDRIVE\"") - String fancinessLevel; - - public FancyBird() {} - - public FancyBird( - long number, - String species, - String quality, - long quantity, - String habitat, - String fancinessLevel) { - this.number = number; - this.species = species; - this.quality = quality; - this.quantity = quantity; - this.habitat = habitat; - this.fancinessLevel = fancinessLevel; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(FancyBird.class) - .addValue(number) - .addValue(species) - .addValue(quality) - .addValue(quantity) - .addValue(habitat) - .addValue(fancinessLevel) - .toString(); - } - - @Override - public boolean equals(@Nullable Object obj) { - if (obj instanceof FancyBird) { - FancyBird other = (FancyBird) obj; - return Objects.equals(species, other.species) - && Objects.equals(quality, other.quality) - && quantity == other.quantity - && number == other.number - && Objects.equals(fancinessLevel, other.fancinessLevel) - && Objects.equals(habitat, other.habitat); - } - return false; - } - - @Override - public int hashCode() { - return Objects.hash(number, species, quality, quantity, habitat, fancinessLevel); - } - } - - /** Create a list of n random records. */ - private static List createRandomRecords(long n) { - String[] qualities = { - "miserable", "forelorn", "fidgity", "squirrelly", "fanciful", "chipper", "lazy" - }; - String[] species = {"pigeons", "owls", "gulls", "hawks", "robins", "jays"}; - Random random = new Random(0); - - List records = new ArrayList<>(); - for (long i = 0; i < n; i++) { - Bird bird = new Bird(); - bird.quality = qualities[random.nextInt(qualities.length)]; - bird.species = species[random.nextInt(species.length)]; - bird.number = i; - bird.quantity = random.nextLong(); - records.add(bird); - } - return records; - } -} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java index 7002b523ca756..bb2fb09577f79 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java @@ -57,11 +57,11 @@ import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.util.LzoCompression; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashMultiset; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Bytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashMultiset; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Bytes; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.apache.commons.compress.compressors.deflate.DeflateCompressorOutputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java index 372056baeaf42..7fd54039b1dda 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io; import static org.apache.beam.sdk.io.WriteFiles.UNKNOWN_SHARDNUM; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets.UTF_8; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets.UTF_8; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; @@ -60,8 +60,8 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream; import org.apache.commons.lang3.SystemUtils; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileIOTest.java index da794f5a49ba2..b87c9caa12441 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileIOTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileIOTest.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io; import static org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions.RESOLVE_FILE; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; import static org.hamcrest.Matchers.isA; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -69,7 +69,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; import org.joda.time.Duration; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java index 41b14ccff2eb2..64a7224956672 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java @@ -42,10 +42,10 @@ import org.apache.beam.sdk.io.fs.ResourceId; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.util.MimeTypes; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; import org.apache.commons.lang3.SystemUtils; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalFileSystemRegistrarTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalFileSystemRegistrarTest.java index b1f77e4c4b2a7..fb7d2a99ab2fb 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalFileSystemRegistrarTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalFileSystemRegistrarTest.java @@ -24,7 +24,7 @@ import java.util.ServiceLoader; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalFileSystemTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalFileSystemTest.java index 53ef83b08c363..aa309158d7235 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalFileSystemTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalFileSystemTest.java @@ -47,12 +47,12 @@ import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions; import org.apache.beam.sdk.testing.RestoreSystemProperties; import org.apache.beam.sdk.util.MimeTypes; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.LineReader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.LineReader; import org.apache.commons.lang3.SystemUtils; import org.hamcrest.Matchers; import org.junit.Rule; @@ -131,7 +131,7 @@ private void testCreate(Path path) throws Exception { public void testReadWithExistingFile() throws Exception { String expected = "my test string"; File existingFile = temporaryFolder.newFile(); - Files.write(expected, existingFile, StandardCharsets.UTF_8); + Files.asCharSink(existingFile, StandardCharsets.UTF_8).write(expected); String data; try (Reader reader = Channels.newReader( diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/ReadTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/ReadTest.java index 315228e776880..aa528c4f08f4f 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/ReadTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/ReadTest.java @@ -38,7 +38,6 @@ import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.LongStream; -import org.apache.beam.sdk.coders.AvroCoder; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.CoderException; import org.apache.beam.sdk.coders.CustomCoder; @@ -66,8 +65,8 @@ import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; @@ -343,7 +342,7 @@ public Coder getOutputCoder() { @Override public Coder getCheckpointMarkCoder() { - return AvroCoder.of(CountingSource.CounterMark.class); + return new CountingSource.CounterMarkCoder(); } } diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/SerializableAvroCodecFactoryTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/SerializableAvroCodecFactoryTest.java deleted file mode 100644 index 4383a16dd28ca..0000000000000 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/SerializableAvroCodecFactoryTest.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io; - -import static org.apache.avro.file.DataFileConstants.BZIP2_CODEC; -import static org.apache.avro.file.DataFileConstants.DEFLATE_CODEC; -import static org.apache.avro.file.DataFileConstants.NULL_CODEC; -import static org.apache.avro.file.DataFileConstants.SNAPPY_CODEC; -import static org.apache.avro.file.DataFileConstants.XZ_CODEC; -import static org.junit.Assert.assertEquals; - -import java.util.Arrays; -import java.util.List; -import org.apache.avro.file.CodecFactory; -import org.apache.beam.sdk.util.SerializableUtils; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Tests of SerializableAvroCodecFactory. */ -@RunWith(JUnit4.class) -public class SerializableAvroCodecFactoryTest { - private final List avroCodecs = - Arrays.asList(NULL_CODEC, SNAPPY_CODEC, DEFLATE_CODEC, XZ_CODEC, BZIP2_CODEC); - - @Test - public void testDefaultCodecsIn() throws Exception { - for (String codec : avroCodecs) { - SerializableAvroCodecFactory codecFactory = - new SerializableAvroCodecFactory(CodecFactory.fromString(codec)); - - assertEquals(CodecFactory.fromString(codec).toString(), codecFactory.getCodec().toString()); - } - } - - @Test - public void testDefaultCodecsSerDe() throws Exception { - for (String codec : avroCodecs) { - SerializableAvroCodecFactory codecFactory = - new SerializableAvroCodecFactory(CodecFactory.fromString(codec)); - - SerializableAvroCodecFactory serdeC = SerializableUtils.clone(codecFactory); - - assertEquals(CodecFactory.fromString(codec).toString(), serdeC.getCodec().toString()); - } - } - - @Test - public void testDeflateCodecSerDeWithLevels() throws Exception { - for (int i = 0; i < 10; ++i) { - SerializableAvroCodecFactory codecFactory = - new SerializableAvroCodecFactory(CodecFactory.deflateCodec(i)); - - SerializableAvroCodecFactory serdeC = SerializableUtils.clone(codecFactory); - - assertEquals(CodecFactory.deflateCodec(i).toString(), serdeC.getCodec().toString()); - } - } - - @Test - public void testXZCodecSerDeWithLevels() throws Exception { - for (int i = 0; i < 10; ++i) { - SerializableAvroCodecFactory codecFactory = - new SerializableAvroCodecFactory(CodecFactory.xzCodec(i)); - - SerializableAvroCodecFactory serdeC = SerializableUtils.clone(codecFactory); - - assertEquals(CodecFactory.xzCodec(i).toString(), serdeC.getCodec().toString()); - } - } - - @Test(expected = NullPointerException.class) - public void testNullCodecToString() throws Exception { - // use default CTR (available cause Serializable) - SerializableAvroCodecFactory codec = new SerializableAvroCodecFactory(); - assertEquals("null", codec.toString()); - } -} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TFRecordIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TFRecordIOTest.java index 0d3b3becf0cd3..acde8c91431da 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TFRecordIOTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TFRecordIOTest.java @@ -67,10 +67,10 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; import org.apache.commons.lang3.SystemUtils; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOReadTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOReadTest.java index efae0ba77fa7e..379345b1001e6 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOReadTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOReadTest.java @@ -91,10 +91,10 @@ import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.apache.commons.compress.compressors.deflate.DeflateCompressorOutputStream; import org.apache.commons.lang3.SystemUtils; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOWriteTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOWriteTest.java index 1bb5525af3299..7551db057063a 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOWriteTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOWriteTest.java @@ -22,7 +22,7 @@ import static org.apache.beam.sdk.TestUtils.NO_LINES_ARRAY; import static org.apache.beam.sdk.io.fs.MatchResult.Status.NOT_FOUND; import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertEquals; @@ -72,15 +72,15 @@ import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Functions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicate; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Functions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicate; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.commons.lang3.SystemUtils; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextRowCountEstimatorTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextRowCountEstimatorTest.java index 4c16786a5e989..17ca3ba85fd81 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextRowCountEstimatorTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextRowCountEstimatorTest.java @@ -21,8 +21,8 @@ import java.io.FileNotFoundException; import java.io.Writer; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/WriteFilesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/WriteFilesTest.java index 059abcf8a5c64..39cb612f2d895 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/WriteFilesTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/WriteFilesTest.java @@ -19,7 +19,7 @@ import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem; import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.includesDisplayDataFor; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; @@ -90,10 +90,10 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.ShardedKey; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.commons.compress.utils.Sets; import org.hamcrest.Matchers; import org.joda.time.Duration; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeEstimateFractionTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeEstimateFractionTest.java index 1320e159cc41d..bfb017dc50577 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeEstimateFractionTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeEstimateFractionTest.java @@ -20,7 +20,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeInterpolateKeyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeInterpolateKeyTest.java index e3845435a8dd4..86c8118a0d260 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeInterpolateKeyTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeInterpolateKeyTest.java @@ -22,7 +22,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeTest.java index 3de1b682f3471..1115fb6884f0d 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/range/ByteKeyRangeTest.java @@ -28,7 +28,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java index 1970b240bbfb8..2643fb556ff47 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.options; import static java.util.Locale.ROOT; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps.uniqueIndex; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps.uniqueIndex; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; @@ -74,13 +74,13 @@ import org.apache.beam.sdk.testing.InterceptingUrlClassLoader; import org.apache.beam.sdk.testing.RestoreSystemProperties; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Collections2; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Collections2; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.apache.commons.lang3.SystemUtils; import org.junit.Rule; import org.junit.Test; @@ -103,6 +103,18 @@ public class PipelineOptionsFactoryTest { @Rule public TestRule restoreSystemProperties = new RestoreSystemProperties(); @Rule public ExpectedLogs expectedLogs = ExpectedLogs.none(PipelineOptionsFactory.class); + @Test + public void testRevision() { + PipelineOptions options = PipelineOptionsFactory.create(); + assertEquals(1, options.revision()); + for (int i = 0; i < 10; i++) { + options.setJobName("other" + i); + // updates are idempotent, the 2nd call won't increment the revision + options.setJobName("other" + i); + } + assertEquals(11, options.revision()); + } + @Test public void testAutomaticRegistrationOfPipelineOptions() { assertTrue(PipelineOptionsFactory.getRegisteredOptions().contains(RegisteredTestOptions.class)); diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsReflectorTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsReflectorTest.java index d0bf5302a7390..d4a8c96954cfb 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsReflectorTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsReflectorTest.java @@ -27,7 +27,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import java.util.Set; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.hamcrest.FeatureMatcher; import org.hamcrest.Matcher; import org.hamcrest.Matchers; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsTest.java index 09ac451ee2a54..1604f4a4bc498 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsTest.java @@ -27,7 +27,7 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PortablePipelineOptionsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PortablePipelineOptionsTest.java index 47e6d11f50426..5fb265452d040 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PortablePipelineOptionsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PortablePipelineOptionsTest.java @@ -22,7 +22,7 @@ import static org.hamcrest.core.IsNull.nullValue; import static org.junit.Assert.assertEquals; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; public class PortablePipelineOptionsTest { diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java index 4f650dfa60501..780ca9adc2705 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java @@ -76,12 +76,12 @@ import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; import org.apache.commons.lang3.SystemUtils; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matchers; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/SdkHarnessOptionsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/SdkHarnessOptionsTest.java index 9c8f70a414e35..80701773c5fce 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/SdkHarnessOptionsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/SdkHarnessOptionsTest.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.options.SdkHarnessOptions.DefaultMaxCacheMemoryUsageMb; import org.apache.beam.sdk.options.SdkHarnessOptions.SdkHarnessLogLevelOverrides; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderTest.java index 6c85ffda7a862..8faeb106d7d09 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProviderTest.java @@ -32,7 +32,7 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProvidersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProvidersTest.java index 5d033a8d61aa0..c65b4e1eee961 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProvidersTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ValueProvidersTest.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java index aeb7291986322..ad775643f904c 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java @@ -61,7 +61,7 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.hamcrest.Matchers; import org.junit.Before; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/AutoValueSchemaTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/AutoValueSchemaTest.java index f506649d72de2..d0ee623dea7cf 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/AutoValueSchemaTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/AutoValueSchemaTest.java @@ -39,7 +39,7 @@ import org.apache.beam.sdk.schemas.utils.SchemaTestUtils; import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.CaseFormat; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CaseFormat; import org.joda.time.DateTime; import org.joda.time.Instant; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/AvroSchemaTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/AvroSchemaTest.java deleted file mode 100644 index 417d45b48f52b..0000000000000 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/AvroSchemaTest.java +++ /dev/null @@ -1,496 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.schemas; - -import static org.junit.Assert.assertEquals; - -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.GenericRecordBuilder; -import org.apache.avro.reflect.AvroIgnore; -import org.apache.avro.reflect.AvroName; -import org.apache.avro.reflect.AvroSchema; -import org.apache.avro.util.Utf8; -import org.apache.beam.sdk.schemas.Schema.FieldType; -import org.apache.beam.sdk.schemas.logicaltypes.EnumerationType; -import org.apache.beam.sdk.schemas.logicaltypes.FixedBytes; -import org.apache.beam.sdk.schemas.transforms.Group; -import org.apache.beam.sdk.schemas.utils.AvroUtils; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.testing.ValidatesRunner; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.sdk.util.SerializableUtils; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.Row; -import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.checkerframework.checker.nullness.qual.Nullable; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.joda.time.Days; -import org.joda.time.LocalDate; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -/** Tests for AVRO schema classes. */ -public class AvroSchemaTest { - /** A test POJO that corresponds to our AVRO schema. */ - public static class AvroSubPojo { - @AvroName("BOOL_NON_NULLABLE") - public boolean boolNonNullable; - - @AvroName("int") - @org.apache.avro.reflect.Nullable - public Integer anInt; - - public AvroSubPojo(boolean boolNonNullable, Integer anInt) { - this.boolNonNullable = boolNonNullable; - this.anInt = anInt; - } - - public AvroSubPojo() {} - - @Override - public boolean equals(@Nullable Object o) { - if (this == o) { - return true; - } - if (!(o instanceof AvroSubPojo)) { - return false; - } - AvroSubPojo that = (AvroSubPojo) o; - return boolNonNullable == that.boolNonNullable && Objects.equals(anInt, that.anInt); - } - - @Override - public int hashCode() { - return Objects.hash(boolNonNullable, anInt); - } - - @Override - public String toString() { - return "AvroSubPojo{" + "boolNonNullable=" + boolNonNullable + ", anInt=" + anInt + '}'; - } - } - - /** A test POJO that corresponds to our AVRO schema. */ - public static class AvroPojo { - public @AvroName("bool_non_nullable") boolean boolNonNullable; - - @org.apache.avro.reflect.Nullable - public @AvroName("int") Integer anInt; - - @org.apache.avro.reflect.Nullable - public @AvroName("long") Long aLong; - - @AvroName("float") - @org.apache.avro.reflect.Nullable - public Float aFloat; - - @AvroName("double") - @org.apache.avro.reflect.Nullable - public Double aDouble; - - @org.apache.avro.reflect.Nullable public String string; - @org.apache.avro.reflect.Nullable public ByteBuffer bytes; - - @AvroSchema("{\"type\": \"fixed\", \"size\": 4, \"name\": \"fixed4\"}") - public byte[] fixed; - - @AvroSchema("{\"type\": \"int\", \"logicalType\": \"date\"}") - public LocalDate date; - - @AvroSchema("{\"type\": \"long\", \"logicalType\": \"timestamp-millis\"}") - public DateTime timestampMillis; - - @AvroSchema("{\"name\": \"TestEnum\", \"type\": \"enum\", \"symbols\": [\"abc\",\"cde\"] }") - public TestEnum testEnum; - - @org.apache.avro.reflect.Nullable public AvroSubPojo row; - @org.apache.avro.reflect.Nullable public List array; - @org.apache.avro.reflect.Nullable public Map map; - @AvroIgnore String extraField; - - @Override - public boolean equals(@Nullable Object o) { - if (this == o) { - return true; - } - if (!(o instanceof AvroPojo)) { - return false; - } - AvroPojo avroPojo = (AvroPojo) o; - return boolNonNullable == avroPojo.boolNonNullable - && Objects.equals(anInt, avroPojo.anInt) - && Objects.equals(aLong, avroPojo.aLong) - && Objects.equals(aFloat, avroPojo.aFloat) - && Objects.equals(aDouble, avroPojo.aDouble) - && Objects.equals(string, avroPojo.string) - && Objects.equals(bytes, avroPojo.bytes) - && Arrays.equals(fixed, avroPojo.fixed) - && Objects.equals(date, avroPojo.date) - && Objects.equals(timestampMillis, avroPojo.timestampMillis) - && Objects.equals(testEnum, avroPojo.testEnum) - && Objects.equals(row, avroPojo.row) - && Objects.equals(array, avroPojo.array) - && Objects.equals(map, avroPojo.map); - } - - @Override - public int hashCode() { - return Objects.hash( - boolNonNullable, - anInt, - aLong, - aFloat, - aDouble, - string, - bytes, - Arrays.hashCode(fixed), - date, - timestampMillis, - testEnum, - row, - array, - map); - } - - public AvroPojo( - boolean boolNonNullable, - int anInt, - long aLong, - float aFloat, - double aDouble, - String string, - ByteBuffer bytes, - byte[] fixed, - LocalDate date, - DateTime timestampMillis, - TestEnum testEnum, - AvroSubPojo row, - List array, - Map map) { - this.boolNonNullable = boolNonNullable; - this.anInt = anInt; - this.aLong = aLong; - this.aFloat = aFloat; - this.aDouble = aDouble; - this.string = string; - this.bytes = bytes; - this.fixed = fixed; - this.date = date; - this.timestampMillis = timestampMillis; - this.testEnum = testEnum; - this.row = row; - this.array = array; - this.map = map; - this.extraField = ""; - } - - public AvroPojo() {} - - @Override - public String toString() { - return "AvroPojo{" - + "boolNonNullable=" - + boolNonNullable - + ", anInt=" - + anInt - + ", aLong=" - + aLong - + ", aFloat=" - + aFloat - + ", aDouble=" - + aDouble - + ", string='" - + string - + '\'' - + ", bytes=" - + bytes - + ", fixed=" - + Arrays.toString(fixed) - + ", date=" - + date - + ", timestampMillis=" - + timestampMillis - + ", testEnum=" - + testEnum - + ", row=" - + row - + ", array=" - + array - + ", map=" - + map - + ", extraField='" - + extraField - + '\'' - + '}'; - } - } - - private static final Schema SUBSCHEMA = - Schema.builder() - .addField("BOOL_NON_NULLABLE", FieldType.BOOLEAN) - .addNullableField("int", FieldType.INT32) - .build(); - private static final FieldType SUB_TYPE = FieldType.row(SUBSCHEMA).withNullable(true); - - private static final EnumerationType TEST_ENUM_TYPE = EnumerationType.create("abc", "cde"); - - private static final Schema SCHEMA = - Schema.builder() - .addField("bool_non_nullable", FieldType.BOOLEAN) - .addNullableField("int", FieldType.INT32) - .addNullableField("long", FieldType.INT64) - .addNullableField("float", FieldType.FLOAT) - .addNullableField("double", FieldType.DOUBLE) - .addNullableField("string", FieldType.STRING) - .addNullableField("bytes", FieldType.BYTES) - .addField("fixed", FieldType.logicalType(FixedBytes.of(4))) - .addField("date", FieldType.DATETIME) - .addField("timestampMillis", FieldType.DATETIME) - .addField("TestEnum", FieldType.logicalType(TEST_ENUM_TYPE)) - .addNullableField("row", SUB_TYPE) - .addNullableField("array", FieldType.array(SUB_TYPE)) - .addNullableField("map", FieldType.map(FieldType.STRING, SUB_TYPE)) - .build(); - - private static final Schema POJO_SCHEMA = - Schema.builder() - .addField("bool_non_nullable", FieldType.BOOLEAN) - .addNullableField("int", FieldType.INT32) - .addNullableField("long", FieldType.INT64) - .addNullableField("float", FieldType.FLOAT) - .addNullableField("double", FieldType.DOUBLE) - .addNullableField("string", FieldType.STRING) - .addNullableField("bytes", FieldType.BYTES) - .addField("fixed", FieldType.logicalType(FixedBytes.of(4))) - .addField("date", FieldType.DATETIME) - .addField("timestampMillis", FieldType.DATETIME) - .addField("testEnum", FieldType.logicalType(TEST_ENUM_TYPE)) - .addNullableField("row", SUB_TYPE) - .addNullableField("array", FieldType.array(SUB_TYPE.withNullable(false))) - .addNullableField("map", FieldType.map(FieldType.STRING, SUB_TYPE.withNullable(false))) - .build(); - - private static final byte[] BYTE_ARRAY = new byte[] {1, 2, 3, 4}; - private static final DateTime DATE_TIME = - new DateTime().withDate(1979, 3, 14).withTime(1, 2, 3, 4); - private static final LocalDate DATE = new LocalDate(1979, 3, 14); - private static final TestAvroNested AVRO_NESTED_SPECIFIC_RECORD = new TestAvroNested(true, 42); - private static final TestAvro AVRO_SPECIFIC_RECORD = - new TestAvro( - true, - 43, - 44L, - (float) 44.1, - (double) 44.2, - "mystring", - ByteBuffer.wrap(BYTE_ARRAY), - new fixed4(BYTE_ARRAY), - DATE, - DATE_TIME, - TestEnum.abc, - AVRO_NESTED_SPECIFIC_RECORD, - ImmutableList.of(AVRO_NESTED_SPECIFIC_RECORD, AVRO_NESTED_SPECIFIC_RECORD), - ImmutableMap.of("k1", AVRO_NESTED_SPECIFIC_RECORD, "k2", AVRO_NESTED_SPECIFIC_RECORD)); - private static final GenericRecord AVRO_NESTED_GENERIC_RECORD = - new GenericRecordBuilder(TestAvroNested.SCHEMA$) - .set("BOOL_NON_NULLABLE", true) - .set("int", 42) - .build(); - private static final GenericRecord AVRO_GENERIC_RECORD = - new GenericRecordBuilder(TestAvro.SCHEMA$) - .set("bool_non_nullable", true) - .set("int", 43) - .set("long", 44L) - .set("float", (float) 44.1) - .set("double", (double) 44.2) - .set("string", new Utf8("mystring")) - .set("bytes", ByteBuffer.wrap(BYTE_ARRAY)) - .set( - "fixed", - GenericData.get() - .createFixed( - null, BYTE_ARRAY, org.apache.avro.Schema.createFixed("fixed4", "", "", 4))) - .set("date", (int) Days.daysBetween(new LocalDate(1970, 1, 1), DATE).getDays()) - .set("timestampMillis", DATE_TIME.getMillis()) - .set("TestEnum", TestEnum.abc) - .set("row", AVRO_NESTED_GENERIC_RECORD) - .set("array", ImmutableList.of(AVRO_NESTED_GENERIC_RECORD, AVRO_NESTED_GENERIC_RECORD)) - .set( - "map", - ImmutableMap.of( - new Utf8("k1"), AVRO_NESTED_GENERIC_RECORD, - new Utf8("k2"), AVRO_NESTED_GENERIC_RECORD)) - .build(); - - private static final Row NESTED_ROW = Row.withSchema(SUBSCHEMA).addValues(true, 42).build(); - private static final Row ROW = - Row.withSchema(SCHEMA) - .addValues( - true, - 43, - 44L, - (float) 44.1, - (double) 44.2, - "mystring", - ByteBuffer.wrap(BYTE_ARRAY), - BYTE_ARRAY, - DATE.toDateTimeAtStartOfDay(DateTimeZone.UTC), - DATE_TIME, - TEST_ENUM_TYPE.valueOf("abc"), - NESTED_ROW, - ImmutableList.of(NESTED_ROW, NESTED_ROW), - ImmutableMap.of("k1", NESTED_ROW, "k2", NESTED_ROW)) - .build(); - - @Test - public void testSpecificRecordSchema() { - assertEquals(SCHEMA, new AvroRecordSchema().schemaFor(TypeDescriptor.of(TestAvro.class))); - } - - @Test - public void testPojoSchema() { - assertEquals(POJO_SCHEMA, new AvroRecordSchema().schemaFor(TypeDescriptor.of(AvroPojo.class))); - } - - @Test - public void testSpecificRecordToRow() { - SerializableFunction toRow = - new AvroRecordSchema().toRowFunction(TypeDescriptor.of(TestAvro.class)); - assertEquals(ROW, toRow.apply(AVRO_SPECIFIC_RECORD)); - } - - @Test - public void testRowToSpecificRecord() { - SerializableFunction fromRow = - new AvroRecordSchema().fromRowFunction(TypeDescriptor.of(TestAvro.class)); - assertEquals(AVRO_SPECIFIC_RECORD, fromRow.apply(ROW)); - } - - @Test - public void testGenericRecordToRow() { - SerializableFunction toRow = - AvroUtils.getGenericRecordToRowFunction(SCHEMA); - assertEquals(ROW, toRow.apply(AVRO_GENERIC_RECORD)); - } - - @Test - public void testRowToGenericRecord() { - SerializableFunction fromRow = - AvroUtils.getRowToGenericRecordFunction(TestAvro.SCHEMA$); - assertEquals(AVRO_GENERIC_RECORD, fromRow.apply(ROW)); - } - - private static final AvroSubPojo SUB_POJO = new AvroSubPojo(true, 42); - private static final AvroPojo AVRO_POJO = - new AvroPojo( - true, - 43, - 44L, - (float) 44.1, - (double) 44.2, - "mystring", - ByteBuffer.wrap(BYTE_ARRAY), - BYTE_ARRAY, - DATE, - DATE_TIME, - TestEnum.abc, - SUB_POJO, - ImmutableList.of(SUB_POJO, SUB_POJO), - ImmutableMap.of("k1", SUB_POJO, "k2", SUB_POJO)); - - private static final Row ROW_FOR_POJO = - Row.withSchema(POJO_SCHEMA) - .addValues( - true, - 43, - 44L, - (float) 44.1, - (double) 44.2, - "mystring", - ByteBuffer.wrap(BYTE_ARRAY), - BYTE_ARRAY, - DATE.toDateTimeAtStartOfDay(DateTimeZone.UTC), - DATE_TIME, - TEST_ENUM_TYPE.valueOf("abc"), - NESTED_ROW, - ImmutableList.of(NESTED_ROW, NESTED_ROW), - ImmutableMap.of("k1", NESTED_ROW, "k2", NESTED_ROW)) - .build(); - - @Test - public void testPojoRecordToRow() { - SerializableFunction toRow = - new AvroRecordSchema().toRowFunction(TypeDescriptor.of(AvroPojo.class)); - assertEquals(ROW_FOR_POJO, toRow.apply(AVRO_POJO)); - } - - @Test - public void testRowToPojo() { - SerializableFunction fromRow = - new AvroRecordSchema().fromRowFunction(TypeDescriptor.of(AvroPojo.class)); - assertEquals(AVRO_POJO, fromRow.apply(ROW_FOR_POJO)); - } - - @Test - public void testPojoRecordToRowSerializable() { - SerializableUtils.ensureSerializableRoundTrip( - new AvroRecordSchema().toRowFunction(TypeDescriptor.of(AvroPojo.class))); - } - - @Test - public void testPojoRecordFromRowSerializable() { - SerializableUtils.ensureSerializableRoundTrip( - new AvroRecordSchema().fromRowFunction(TypeDescriptor.of(AvroPojo.class))); - } - - @Rule public final transient TestPipeline pipeline = TestPipeline.create(); - - @Test - @Category(ValidatesRunner.class) - public void testAvroPipelineGroupBy() { - PCollection input = pipeline.apply(Create.of(ROW_FOR_POJO).withRowSchema(POJO_SCHEMA)); - - PCollection output = input.apply(Group.byFieldNames("string")); - Schema keySchema = Schema.builder().addStringField("string").build(); - Schema outputSchema = - Schema.builder() - .addRowField("key", keySchema) - .addIterableField("value", FieldType.row(POJO_SCHEMA)) - .build(); - PAssert.that(output) - .containsInAnyOrder( - Row.withSchema(outputSchema) - .addValue(Row.withSchema(keySchema).addValue("mystring").build()) - .addIterable(ImmutableList.of(ROW_FOR_POJO)) - .build()); - - pipeline.run(); - } -} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/FieldAccessDescriptorTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/FieldAccessDescriptorTest.java index 1d0c070294c1c..99cb1b70ca090 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/FieldAccessDescriptorTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/FieldAccessDescriptorTest.java @@ -23,8 +23,8 @@ import java.util.stream.Collectors; import org.apache.beam.sdk.schemas.FieldAccessDescriptor.FieldDescriptor; import org.apache.beam.sdk.schemas.Schema.FieldType; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/JavaBeanSchemaTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/JavaBeanSchemaTest.java index 9729166c7f9a2..5313feb5c6c0b 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/JavaBeanSchemaTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/JavaBeanSchemaTest.java @@ -68,10 +68,10 @@ import org.apache.beam.sdk.schemas.utils.TestJavaBeans.SimpleBeanWithAnnotations; import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Ints; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Ints; import org.joda.time.DateTime; import org.junit.Ignore; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/JavaFieldSchemaTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/JavaFieldSchemaTest.java index c3dbc685e0dcd..11bef79b26f73 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/JavaFieldSchemaTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/JavaFieldSchemaTest.java @@ -76,10 +76,10 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Ints; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Ints; import org.joda.time.DateTime; import org.joda.time.Instant; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/JsonSchemaConversionTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/JsonSchemaConversionTest.java index 404f9c750b3bc..775f18ac87f03 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/JsonSchemaConversionTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/JsonSchemaConversionTest.java @@ -28,7 +28,7 @@ import java.util.stream.Collectors; import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.utils.JsonUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; import org.everit.json.schema.ValidationException; import org.json.JSONArray; import org.json.JSONObject; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/SchemaCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/SchemaCoderTest.java index f16441cbe73b7..cc57d382af0c3 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/SchemaCoderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/SchemaCoderTest.java @@ -26,7 +26,6 @@ import java.util.Collection; import java.util.Objects; import java.util.function.Supplier; -import org.apache.avro.reflect.AvroSchema; import org.apache.beam.sdk.coders.Coder.NonDeterministicException; import org.apache.beam.sdk.coders.RowCoder; import org.apache.beam.sdk.schemas.Schema.Field; @@ -37,7 +36,7 @@ import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matchers; import org.joda.time.DateTime; @@ -196,43 +195,6 @@ public int hashCode() { } } - @DefaultSchema(AvroRecordSchema.class) - private static class SimpleAvro { - public String string; - public Integer int32; - public Long int64; - - @AvroSchema("{\"type\": \"long\", \"logicalType\": \"timestamp-millis\"}") - public DateTime datetime; - - public SimpleAvro(String string, Integer int32, Long int64, DateTime datetime) { - this.string = string; - this.int32 = int32; - this.int64 = int64; - this.datetime = datetime; - } - - @Override - public boolean equals(@Nullable Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SimpleAvro that = (SimpleAvro) o; - return string.equals(that.string) - && int32.equals(that.int32) - && int64.equals(that.int64) - && datetime.equals(that.datetime); - } - - @Override - public int hashCode() { - return Objects.hash(string, int32, int64, datetime); - } - } - private static final SchemaRegistry REGISTRY = SchemaRegistry.createDefault(); private static SchemaCoder coderFrom(TypeDescriptor typeDescriptor) throws NoSuchSchemaException { @@ -316,23 +278,6 @@ public static Collection data() throws NoSuchSchemaException { new DateTime().withDate(1989, 3, 14).withTime(10, 30, 0, 0))), true }, - new Object[] { - coderFrom(TypeDescriptor.of(SimpleAvro.class)), - ImmutableList.>of( - () -> - new SimpleAvro( - "foo", - 9001, - 0L, - new DateTime().withDate(1979, 3, 14).withTime(10, 30, 0, 0)), - () -> - new SimpleAvro( - "bar", - 9002, - 1L, - new DateTime().withDate(1989, 3, 14).withTime(10, 30, 0, 0))), - true - }, new Object[] { RowCoder.of(LOGICAL_NANOS_SCHEMA), ImmutableList.>of( diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/SchemaRegistryTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/SchemaRegistryTest.java index ddf47ed878835..1946dfd1fdb76 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/SchemaRegistryTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/SchemaRegistryTest.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/SchemaTranslationTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/SchemaTranslationTest.java index f33cb1200ee02..3020d7e42d05a 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/SchemaTranslationTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/SchemaTranslationTest.java @@ -17,6 +17,7 @@ */ package org.apache.beam.sdk.schemas; +import static org.apache.beam.sdk.schemas.SchemaTranslation.STANDARD_LOGICAL_TYPES; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -48,12 +49,13 @@ import org.apache.beam.sdk.schemas.logicaltypes.PythonCallable; import org.apache.beam.sdk.schemas.logicaltypes.SchemaLogicalType; import org.apache.beam.sdk.schemas.logicaltypes.SqlTypes; +import org.apache.beam.sdk.schemas.logicaltypes.UnknownLogicalType; import org.apache.beam.sdk.schemas.logicaltypes.VariableBytes; import org.apache.beam.sdk.schemas.logicaltypes.VariableString; import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; import org.junit.Test; import org.junit.experimental.runners.Enclosed; @@ -186,7 +188,8 @@ public static Iterable data() { .withOptions(optionsBuilder)) .add( Schema.of( - Field.of("null_argument", FieldType.logicalType(new NullArgumentLogicalType())))) + Field.of( + "null_argument", FieldType.logicalType(new PortableNullArgLogicalType())))) .add(Schema.of(Field.of("logical_argument", FieldType.logicalType(new DateTime())))) .add( Schema.of(Field.of("single_arg_argument", FieldType.logicalType(FixedBytes.of(100))))) @@ -348,14 +351,14 @@ public static Iterable data() { .add(simpleRow(FieldType.row(row.getSchema()), row)) .add(simpleRow(FieldType.DATETIME, new Instant(23L))) .add(simpleRow(FieldType.DECIMAL, BigDecimal.valueOf(100000))) - .add(simpleRow(FieldType.logicalType(new NullArgumentLogicalType()), "str")) + .add(simpleRow(FieldType.logicalType(new PortableNullArgLogicalType()), "str")) .add(simpleRow(FieldType.logicalType(new DateTime()), LocalDateTime.of(2000, 1, 3, 3, 1))) .add(simpleNullRow(FieldType.STRING)) .add(simpleNullRow(FieldType.INT32)) .add(simpleNullRow(FieldType.map(FieldType.STRING, FieldType.INT32))) .add(simpleNullRow(FieldType.array(FieldType.STRING))) .add(simpleNullRow(FieldType.row(row.getSchema()))) - .add(simpleNullRow(FieldType.logicalType(new NullArgumentLogicalType()))) + .add(simpleNullRow(FieldType.logicalType(new PortableNullArgLogicalType()))) .add(simpleNullRow(FieldType.logicalType(new DateTime()))) .add(simpleNullRow(FieldType.DECIMAL)) .add(simpleNullRow(FieldType.DATETIME)) @@ -419,6 +422,8 @@ public static Iterable data() { .add(FieldType.logicalType(FixedString.of(10))) .add(FieldType.logicalType(VariableString.of(10))) .add(FieldType.logicalType(FixedPrecisionNumeric.of(10))) + .add(FieldType.logicalType(new PortableNullArgLogicalType())) + .add(FieldType.logicalType(new NullArgumentLogicalType())) .build(); } @@ -426,7 +431,7 @@ public static Iterable data() { public Schema.FieldType fieldType; @Test - public void testPortableLogicalTypeSerializeDeserilizeCorrectly() { + public void testLogicalTypeSerializeDeserilizeCorrectly() { SchemaApi.FieldType proto = SchemaTranslation.fieldTypeToProto(fieldType, true); Schema.FieldType translated = SchemaTranslation.fieldTypeFromProto(proto); @@ -438,14 +443,64 @@ public void testPortableLogicalTypeSerializeDeserilizeCorrectly() { assertThat( translated.getLogicalType().getArgument(), equalTo(fieldType.getLogicalType().getArgument())); + assertThat( + translated.getLogicalType().getIdentifier(), + equalTo(fieldType.getLogicalType().getIdentifier())); + } + + @Test + public void testLogicalTypeFromToProtoCorrectly() { + SchemaApi.FieldType proto = SchemaTranslation.fieldTypeToProto(fieldType, false); + Schema.FieldType translated = SchemaTranslation.fieldTypeFromProto(proto); + + if (STANDARD_LOGICAL_TYPES.containsKey(translated.getLogicalType().getIdentifier())) { + // standard logical type should be able to fully recover the original type + assertThat( + translated.getLogicalType().getClass(), equalTo(fieldType.getLogicalType().getClass())); + } else { + // non-standard type will get assembled to UnknownLogicalType + assertThat(translated.getLogicalType().getClass(), equalTo(UnknownLogicalType.class)); + } + assertThat( + translated.getLogicalType().getArgumentType(), + equalTo(fieldType.getLogicalType().getArgumentType())); + assertThat( + translated.getLogicalType().getArgument(), + equalTo(fieldType.getLogicalType().getArgument())); + if (fieldType.getLogicalType().getIdentifier().startsWith("beam:logical_type:")) { + // portable logical type should fully recover the urn + assertThat( + translated.getLogicalType().getIdentifier(), + equalTo(fieldType.getLogicalType().getIdentifier())); + } else { + // non-portable logical type would have "javasdk_" urn + assertThat( + translated.getLogicalType().getIdentifier(), + equalTo( + String.format( + "beam:logical_type:javasdk_%s:v1", + fieldType + .getLogicalType() + .getIdentifier() + .toLowerCase() + .replaceAll("[^0-9A-Za-z_]", "")))); + } } } - /** A simple logical type that has no argument. */ - private static class NullArgumentLogicalType implements Schema.LogicalType { + /** A portable logical type that has no argument. */ + private static class PortableNullArgLogicalType extends NullArgumentLogicalType { public static final String IDENTIFIER = "beam:logical_type:null_argument:v1"; - public NullArgumentLogicalType() {} + @Override + public String getIdentifier() { + return IDENTIFIER; + } + } + + /** A non-portable (Java SDK) logical type that has no argument. */ + private static class NullArgumentLogicalType implements Schema.LogicalType { + public static final String IDENTIFIER = "NULL_ARGUMENT"; @Override public String toBaseType(String input) { diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/AvroPayloadSerializerProviderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/AvroPayloadSerializerProviderTest.java deleted file mode 100644 index dbe9ae06ed1d0..0000000000000 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/AvroPayloadSerializerProviderTest.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.schemas.io; - -import static org.junit.Assert.assertEquals; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.GenericRecordBuilder; -import org.apache.beam.sdk.coders.AvroCoder; -import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.sdk.schemas.io.payloads.AvroPayloadSerializerProvider; -import org.apache.beam.sdk.schemas.utils.AvroUtils; -import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class AvroPayloadSerializerProviderTest { - private static final Schema SCHEMA = - Schema.builder().addInt64Field("abc").addStringField("xyz").build(); - private static final org.apache.avro.Schema AVRO_SCHEMA = AvroUtils.toAvroSchema(SCHEMA); - private static final AvroCoder AVRO_CODER = AvroCoder.of(AVRO_SCHEMA); - private static final Row DESERIALIZED = - Row.withSchema(SCHEMA).withFieldValue("abc", 3L).withFieldValue("xyz", "qqq").build(); - private static final GenericRecord SERIALIZED = - new GenericRecordBuilder(AVRO_SCHEMA).set("abc", 3L).set("xyz", "qqq").build(); - - private final AvroPayloadSerializerProvider provider = new AvroPayloadSerializerProvider(); - - @Test - public void serialize() throws Exception { - byte[] bytes = provider.getSerializer(SCHEMA, ImmutableMap.of()).serialize(DESERIALIZED); - GenericRecord record = AVRO_CODER.decode(new ByteArrayInputStream(bytes)); - assertEquals(3L, record.get("abc")); - assertEquals("qqq", record.get("xyz").toString()); - } - - @Test - public void deserialize() throws Exception { - ByteArrayOutputStream os = new ByteArrayOutputStream(); - AVRO_CODER.encode(SERIALIZED, os); - Row row = provider.getSerializer(SCHEMA, ImmutableMap.of()).deserialize(os.toByteArray()); - assertEquals(DESERIALIZED, row); - } -} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/DeadLetteredTransformTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/DeadLetteredTransformTest.java index 600e5931268cb..17d530ef1fe93 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/DeadLetteredTransformTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/DeadLetteredTransformTest.java @@ -34,7 +34,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/JsonPayloadSerializerProviderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/JsonPayloadSerializerProviderTest.java index 51cecad384929..868157f08246a 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/JsonPayloadSerializerProviderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/JsonPayloadSerializerProviderTest.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.io.payloads.JsonPayloadSerializerProvider; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/StoringDlqProvider.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/StoringDlqProvider.java index b7a689dc26d5e..f71938e4e1d41 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/StoringDlqProvider.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/io/StoringDlqProvider.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.schemas.io; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; import java.util.ArrayList; @@ -27,7 +27,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; @AutoService(GenericDlqProvider.class) public class StoringDlqProvider implements GenericDlqProvider { diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/logicaltypes/LogicalTypesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/logicaltypes/LogicalTypesTest.java index 23ebcf1616b0e..fc264c8104c41 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/logicaltypes/LogicalTypesTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/logicaltypes/LogicalTypesTest.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.logicaltypes.OneOfType.Value; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; /** Unit tests for logical types. */ diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/AddFieldsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/AddFieldsTest.java index 6f9590d957f2c..963ae0c7dec80 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/AddFieldsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/AddFieldsTest.java @@ -29,8 +29,8 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/CastTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/CastTest.java index 0dbed0e2eed06..101885dd9de63 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/CastTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/CastTest.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/CastValidatorTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/CastValidatorTest.java index ecc0434b2999e..07f869c06f916 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/CastValidatorTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/CastValidatorTest.java @@ -32,8 +32,8 @@ import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.Schema.TypeName; import org.apache.beam.sdk.testing.UsesSchema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/CoGroupTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/CoGroupTest.java index b2e739fd5d3bd..a0b48bb7fa0ee 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/CoGroupTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/CoGroupTest.java @@ -39,8 +39,8 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.hamcrest.Matcher; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/ConvertTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/ConvertTest.java index d64848ee9ce52..32d32e8918eb2 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/ConvertTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/ConvertTest.java @@ -20,12 +20,10 @@ import java.util.Arrays; import java.util.Map; import java.util.Objects; -import org.apache.avro.generic.GenericRecord; import org.apache.beam.sdk.schemas.JavaFieldSchema; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.annotations.DefaultSchema; -import org.apache.beam.sdk.schemas.utils.AvroUtils; import org.apache.beam.sdk.testing.NeedsRunner; import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; @@ -35,8 +33,8 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Rule; import org.junit.Test; @@ -134,9 +132,6 @@ public int hashCode() { .addValue(ImmutableMap.of("first", EXPECTED_ROW1_NESTED, "second", EXPECTED_ROW1_NESTED)) .build(); - private static final GenericRecord EXPECTED_GENERICRECORD1 = - AvroUtils.toGenericRecord(EXPECTED_ROW1, AvroUtils.toAvroSchema(EXPECTED_SCHEMA1)); - /** Test outer POJO. Different but equivalent schema. * */ @DefaultSchema(JavaFieldSchema.class) public static class POJO2 { @@ -248,13 +243,4 @@ public void testFromRowsUnboxingPrimitive() { PAssert.that(longs).containsInAnyOrder((Long) EXPECTED_ROW1.getValue("field2")); pipeline.run(); } - - @Test - @Category(NeedsRunner.class) - public void testToGenericRecords() { - PCollection records = - pipeline.apply(Create.of(new POJO1())).apply(Convert.to(GenericRecord.class)); - PAssert.that(records).containsInAnyOrder(EXPECTED_GENERICRECORD1); - pipeline.run(); - } } diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/DropFieldsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/DropFieldsTest.java index 2906f8acbf1e8..eb08f168c35f8 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/DropFieldsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/DropFieldsTest.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/FilterTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/FilterTest.java index 9a5e2e48414f7..8cf95eae49056 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/FilterTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/FilterTest.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.testing.UsesSchema; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/GroupTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/GroupTest.java index b4b074e0ca02e..357ef024bea97 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/GroupTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/GroupTest.java @@ -30,6 +30,7 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; +import javax.annotation.Nullable; import org.apache.beam.sdk.schemas.AutoValueSchema; import org.apache.beam.sdk.schemas.NoSuchSchemaException; import org.apache.beam.sdk.schemas.Schema; @@ -50,12 +51,13 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.Sample; import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.beam.sdk.transforms.SerializableFunctions; import org.apache.beam.sdk.transforms.Sum; import org.apache.beam.sdk.transforms.Top; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.hamcrest.Description; import org.hamcrest.Matcher; import org.hamcrest.TypeSafeMatcher; @@ -260,17 +262,30 @@ public void testGroupGlobally() { @Test @Category(NeedsRunner.class) - public void testGlobalAggregation() { + public void testGlobalAggregationWithoutFanout() { + globalAggregationWithFanout(false); + } + + @Test + @Category(NeedsRunner.class) + public void testGlobalAggregationWithFanout() { + globalAggregationWithFanout(true); + } + + public void globalAggregationWithFanout(boolean withFanout) { Collection elements = ImmutableList.of( Basic.of("key1", 1, "value1"), Basic.of("key1", 1, "value2"), Basic.of("key2", 2, "value3"), Basic.of("key2", 2, "value4")); - PCollection count = - pipeline - .apply(Create.of(elements)) - .apply(Group.globally().aggregate(Count.combineFn())); + + Group.CombineGlobally transform = + Group.globally().aggregate(Count.combineFn()); + if (withFanout) { + transform = transform.withFanout(10); + } + PCollection count = pipeline.apply(Create.of(elements)).apply(transform); PAssert.that(count).containsInAnyOrder(4L); pipeline.run(); @@ -426,7 +441,17 @@ public Long extractOutput(long[] accumulator) { @Test @Category(NeedsRunner.class) - public void testAggregateByMultipleFields() { + public void testAggregateByMultipleFieldsWithoutFanout() { + aggregateByMultipleFieldsWithFanout(false); + } + + @Test + @Category(NeedsRunner.class) + public void testAggregateByMultipleFieldsWithFanout() { + aggregateByMultipleFieldsWithFanout(true); + } + + public void aggregateByMultipleFieldsWithFanout(boolean withFanout) { Collection elements = ImmutableList.of( Aggregate.of(1, 1, 2), @@ -435,12 +460,14 @@ public void testAggregateByMultipleFields() { Aggregate.of(4, 2, 5)); List fieldNames = Lists.newArrayList("field1", "field2"); - PCollection aggregate = - pipeline - .apply(Create.of(elements)) - .apply( - Group.globally() - .aggregateFields(fieldNames, new MultipleFieldCombineFn(), "field1+field2")); + + Group.CombineFieldsGlobally transform = + Group.globally() + .aggregateFields(fieldNames, new MultipleFieldCombineFn(), "field1+field2"); + if (withFanout) { + transform = transform.withFanout(10); + } + PCollection aggregate = pipeline.apply(Create.of(elements)).apply(transform); Schema outputSchema = Schema.builder().addInt64Field("field1+field2").build(); Row expectedRow = Row.withSchema(outputSchema).addValues(16L).build(); @@ -462,7 +489,25 @@ static OuterAggregate of(Aggregate inner) { @Test @Category(NeedsRunner.class) - public void testByKeyWithSchemaAggregateFnNestedFields() { + public void testByKeyWithSchemaAggregateFnNestedFieldsNoFanout() { + byKeyWithSchemaAggregateFnNestedFieldsWithFanout(null); + } + + @Test + @Category(NeedsRunner.class) + public void testByKeyWithSchemaAggregateFnNestedFieldsWithNumberFanout() { + byKeyWithSchemaAggregateFnNestedFieldsWithFanout(Group.CombineFieldsByFields.Fanout.of(10)); + } + + @Test + @Category(NeedsRunner.class) + public void testByKeyWithSchemaAggregateFnNestedFieldsWithFunctionFanout() { + byKeyWithSchemaAggregateFnNestedFieldsWithFanout( + Group.CombineFieldsByFields.Fanout.of(SerializableFunctions.constant(10))); + } + + public void byKeyWithSchemaAggregateFnNestedFieldsWithFanout( + @Nullable Group.CombineFieldsByFields.Fanout fanout) { Collection elements = ImmutableList.of( OuterAggregate.of(Aggregate.of(1, 1, 2)), @@ -470,14 +515,23 @@ public void testByKeyWithSchemaAggregateFnNestedFields() { OuterAggregate.of(Aggregate.of(3, 2, 4)), OuterAggregate.of(Aggregate.of(4, 2, 5))); - PCollection aggregations = - pipeline - .apply(Create.of(elements)) - .apply( - Group.byFieldNames("inner.field2") - .aggregateField("inner.field1", Sum.ofLongs(), "field1_sum") - .aggregateField("inner.field3", Sum.ofIntegers(), "field3_sum") - .aggregateField("inner.field1", Top.largestLongsFn(1), "field1_top")); + Group.CombineFieldsByFields transform = + Group.byFieldNames("inner.field2") + .aggregateField("inner.field1", Sum.ofLongs(), "field1_sum") + .aggregateField("inner.field3", Sum.ofIntegers(), "field3_sum") + .aggregateField("inner.field1", Top.largestLongsFn(1), "field1_top"); + if (fanout != null) { + switch (fanout.getKind()) { + case NUMBER: + transform = transform.withHotKeyFanout(fanout.getNumber()); + break; + case FUNCTION: + transform = transform.withHotKeyFanout(fanout.getFunction()); + break; + } + } + + PCollection aggregations = pipeline.apply(Create.of(elements)).apply(transform); Schema keySchema = Schema.builder().addInt64Field("field2").build(); Schema valueSchema = diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/JoinTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/JoinTest.java index 0650ea751b026..6563a54566dc1 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/JoinTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/JoinTest.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/JoinTestUtils.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/JoinTestUtils.java index aeff3d8616b35..6324788e30320 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/JoinTestUtils.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/JoinTestUtils.java @@ -22,7 +22,7 @@ import java.util.stream.Collectors; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; @SuppressWarnings({ "rawtypes" // TODO(https://github.com/apache/beam/issues/20447) diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/RenameFieldsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/RenameFieldsTest.java index df2af44108108..897792e949219 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/RenameFieldsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/RenameFieldsTest.java @@ -33,9 +33,9 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/SelectTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/SelectTest.java index b7bd65dfe794e..ba8ef98115c16 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/SelectTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/SelectTest.java @@ -36,8 +36,8 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProviderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProviderTest.java index 744b4f3bf0bb9..db7b1436a128a 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProviderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProviderTest.java @@ -27,7 +27,6 @@ import org.apache.beam.sdk.schemas.AutoValueSchema; import org.apache.beam.sdk.schemas.annotations.DefaultSchema; import org.apache.beam.sdk.testing.UsesSchema; -import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; import org.junit.Test; @@ -90,7 +89,7 @@ public Optional> dependencies( } } - public static class FakeSchemaTransform implements SchemaTransform { + public static class FakeSchemaTransform extends SchemaTransform { public Configuration config; @@ -99,7 +98,7 @@ public FakeSchemaTransform(Configuration config) { } @Override - public PTransform buildTransform() { + public PCollectionRowTuple expand(PCollectionRowTuple input) { return null; } } diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/AvroGenerators.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/AvroGenerators.java deleted file mode 100644 index cbeceab24c670..0000000000000 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/AvroGenerators.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.schemas.utils; - -import com.pholser.junit.quickcheck.generator.GenerationStatus; -import com.pholser.junit.quickcheck.generator.Generator; -import com.pholser.junit.quickcheck.random.SourceOfRandomness; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import org.apache.avro.Schema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ObjectArrays; - -/** QuickCheck generators for AVRO. */ -class AvroGenerators { - - /** Generates arbitrary AVRO schemas. */ - public static class SchemaGenerator extends BaseSchemaGenerator { - - public static final SchemaGenerator INSTANCE = new SchemaGenerator(); - - private static final ImmutableList PRIMITIVE_TYPES = - ImmutableList.of( - Schema.Type.STRING, - Schema.Type.BYTES, - Schema.Type.INT, - Schema.Type.LONG, - Schema.Type.FLOAT, - Schema.Type.DOUBLE, - Schema.Type.BOOLEAN); - - private static final ImmutableList ALL_TYPES = - ImmutableList.builder() - .addAll(PRIMITIVE_TYPES) - .add(Schema.Type.FIXED) - .add(Schema.Type.ENUM) - .add(Schema.Type.RECORD) - .add(Schema.Type.ARRAY) - .add(Schema.Type.MAP) - .add(Schema.Type.UNION) - .add(Schema.Type.ARRAY) - .build(); - - private static final int MAX_NESTING = 10; - - @Override - public Schema generate(SourceOfRandomness random, GenerationStatus status) { - Schema.Type type; - - if (nesting(status) >= MAX_NESTING) { - type = random.choose(PRIMITIVE_TYPES); - } else { - type = random.choose(ALL_TYPES); - } - - if (PRIMITIVE_TYPES.contains(type)) { - return Schema.create(type); - } else { - nestingInc(status); - - if (type == Schema.Type.FIXED) { - int size = random.choose(Arrays.asList(1, 5, 12)); - return Schema.createFixed("fixed_" + branch(status), "", "", size); - } else if (type == Schema.Type.UNION) { - // only nullable fields, everything else isn't supported in row conversion code - return UnionSchemaGenerator.INSTANCE.generate(random, status); - } else if (type == Schema.Type.ENUM) { - return EnumSchemaGenerator.INSTANCE.generate(random, status); - } else if (type == Schema.Type.RECORD) { - return RecordSchemaGenerator.INSTANCE.generate(random, status); - } else if (type == Schema.Type.MAP) { - return Schema.createMap(generate(random, status)); - } else if (type == Schema.Type.ARRAY) { - return Schema.createArray(generate(random, status)); - } else { - throw new AssertionError("Unexpected AVRO type: " + type); - } - } - } - } - - public static class RecordSchemaGenerator extends BaseSchemaGenerator { - - public static final RecordSchemaGenerator INSTANCE = new RecordSchemaGenerator(); - - @Override - public Schema generate(SourceOfRandomness random, GenerationStatus status) { - List fields = - IntStream.range(0, random.nextInt(0, status.size()) + 1) - .mapToObj( - i -> { - // deterministically avoid collisions in record names - branchPush(status, String.valueOf(i)); - Schema.Field field = - createField(i, SchemaGenerator.INSTANCE.generate(random, status)); - branchPop(status); - return field; - }) - .collect(Collectors.toList()); - - return Schema.createRecord("record_" + branch(status), "", "example", false, fields); - } - - private Schema.Field createField(int i, Schema schema) { - return new Schema.Field("field_" + i, schema, null, (Object) null); - } - } - - static class UnionSchemaGenerator extends BaseSchemaGenerator { - - public static final UnionSchemaGenerator INSTANCE = new UnionSchemaGenerator(); - - @Override - public Schema generate(SourceOfRandomness random, GenerationStatus status) { - Map schemaMap = - IntStream.range(0, random.nextInt(0, status.size()) + 1) - .mapToObj( - i -> { - // deterministically avoid collisions in record names - branchPush(status, String.valueOf(i)); - Schema schema = - SchemaGenerator.INSTANCE - // nested unions aren't supported in AVRO - .filter(x -> x.getType() != Schema.Type.UNION) - .generate(random, status); - branchPop(status); - return schema; - }) - // AVRO requires uniqueness by full name - .collect(Collectors.toMap(Schema::getFullName, Function.identity(), (x, y) -> x)); - - List schemas = new ArrayList<>(schemaMap.values()); - - if (random.nextBoolean()) { - org.apache.avro.Schema nullSchema = org.apache.avro.Schema.create(Schema.Type.NULL); - schemas.add(nullSchema); - Collections.shuffle(schemas, random.toJDKRandom()); - } - - return Schema.createUnion(schemas); - } - } - - static class EnumSchemaGenerator extends BaseSchemaGenerator { - - public static final EnumSchemaGenerator INSTANCE = new EnumSchemaGenerator(); - - private static final Schema FRUITS = - Schema.createEnum("Fruit", "", "example", Arrays.asList("banana", "apple", "pear")); - - private static final Schema STATUS = - Schema.createEnum("Status", "", "example", Arrays.asList("OK", "ERROR", "WARNING")); - - @Override - public Schema generate(final SourceOfRandomness random, final GenerationStatus status) { - return random.choose(Arrays.asList(FRUITS, STATUS)); - } - } - - abstract static class BaseSchemaGenerator extends Generator { - - private static final GenerationStatus.Key NESTING_KEY = - new GenerationStatus.Key<>("nesting", Integer.class); - - private static final GenerationStatus.Key BRANCH_KEY = - new GenerationStatus.Key<>("branch", String[].class); - - BaseSchemaGenerator() { - super(org.apache.avro.Schema.class); - } - - void branchPush(GenerationStatus status, String value) { - String[] current = status.valueOf(BRANCH_KEY).orElse(new String[0]); - String[] next = ObjectArrays.concat(current, value); - - status.setValue(BRANCH_KEY, next); - } - - void branchPop(GenerationStatus status) { - String[] current = status.valueOf(BRANCH_KEY).orElse(new String[0]); - String[] next = Arrays.copyOf(current, current.length - 1); - - status.setValue(BRANCH_KEY, next); - } - - String branch(GenerationStatus status) { - return Joiner.on("_").join(status.valueOf(BRANCH_KEY).orElse(new String[0])); - } - - int nesting(GenerationStatus status) { - return status.valueOf(NESTING_KEY).orElse(0); - } - - void nestingInc(GenerationStatus status) { - status.setValue(NESTING_KEY, nesting(status) + 1); - } - } -} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/AvroUtilsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/AvroUtilsTest.java deleted file mode 100644 index 5c115aa89bccc..0000000000000 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/AvroUtilsTest.java +++ /dev/null @@ -1,914 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.schemas.utils; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import com.pholser.junit.quickcheck.From; -import com.pholser.junit.quickcheck.Property; -import com.pholser.junit.quickcheck.runner.JUnitQuickcheck; -import java.math.BigDecimal; -import java.nio.ByteBuffer; -import java.sql.JDBCType; -import java.util.List; -import java.util.Map; -import org.apache.avro.Conversions; -import org.apache.avro.LogicalType; -import org.apache.avro.LogicalTypes; -import org.apache.avro.RandomData; -import org.apache.avro.Schema.Type; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.GenericRecordBuilder; -import org.apache.avro.reflect.ReflectData; -import org.apache.avro.util.Utf8; -import org.apache.beam.sdk.Pipeline; -import org.apache.beam.sdk.coders.AvroCoder; -import org.apache.beam.sdk.io.AvroGeneratedUser; -import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.sdk.schemas.Schema.Field; -import org.apache.beam.sdk.schemas.Schema.FieldType; -import org.apache.beam.sdk.schemas.logicaltypes.EnumerationType; -import org.apache.beam.sdk.schemas.logicaltypes.OneOfType; -import org.apache.beam.sdk.schemas.logicaltypes.SqlTypes; -import org.apache.beam.sdk.schemas.utils.AvroGenerators.RecordSchemaGenerator; -import org.apache.beam.sdk.schemas.utils.AvroUtils.TypeWithNullability; -import org.apache.beam.sdk.testing.CoderProperties; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.SimpleFunction; -import org.apache.beam.sdk.util.SerializableUtils; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.checkerframework.checker.nullness.qual.NonNull; -import org.checkerframework.checker.nullness.qual.Nullable; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.joda.time.Days; -import org.joda.time.Instant; -import org.joda.time.LocalTime; -import org.junit.Test; -import org.junit.runner.RunWith; - -/** Tests for conversion between AVRO records and Beam rows. */ -@RunWith(JUnitQuickcheck.class) -@SuppressWarnings({ - "rawtypes", // TODO(https://github.com/apache/beam/issues/20447) -}) -public class AvroUtilsTest { - - private static final org.apache.avro.Schema NULL_SCHEMA = - org.apache.avro.Schema.create(Type.NULL); - - @Property(trials = 1000) - @SuppressWarnings("unchecked") - public void supportsAnyAvroSchema( - @From(RecordSchemaGenerator.class) org.apache.avro.Schema avroSchema) { - - Schema schema = AvroUtils.toBeamSchema(avroSchema); - Iterable iterable = new RandomData(avroSchema, 10); - List records = Lists.newArrayList((Iterable) iterable); - - for (GenericRecord record : records) { - AvroUtils.toBeamRowStrict(record, schema); - } - } - - @Property(trials = 1000) - @SuppressWarnings("unchecked") - public void avroToBeamRoundTrip( - @From(RecordSchemaGenerator.class) org.apache.avro.Schema avroSchema) { - - Schema schema = AvroUtils.toBeamSchema(avroSchema); - Iterable iterable = new RandomData(avroSchema, 10); - List records = Lists.newArrayList((Iterable) iterable); - - for (GenericRecord record : records) { - Row row = AvroUtils.toBeamRowStrict(record, schema); - GenericRecord out = AvroUtils.toGenericRecord(row, avroSchema); - assertEquals(record, out); - } - } - - @Test - public void testUnwrapNullableSchema() { - org.apache.avro.Schema avroSchema = - org.apache.avro.Schema.createUnion( - org.apache.avro.Schema.create(Type.NULL), org.apache.avro.Schema.create(Type.STRING)); - - TypeWithNullability typeWithNullability = new TypeWithNullability(avroSchema); - assertTrue(typeWithNullability.nullable); - assertEquals(org.apache.avro.Schema.create(Type.STRING), typeWithNullability.type); - } - - @Test - public void testUnwrapNullableSchemaReordered() { - org.apache.avro.Schema avroSchema = - org.apache.avro.Schema.createUnion( - org.apache.avro.Schema.create(Type.STRING), org.apache.avro.Schema.create(Type.NULL)); - - TypeWithNullability typeWithNullability = new TypeWithNullability(avroSchema); - assertTrue(typeWithNullability.nullable); - assertEquals(org.apache.avro.Schema.create(Type.STRING), typeWithNullability.type); - } - - @Test - public void testUnwrapNullableSchemaToUnion() { - org.apache.avro.Schema avroSchema = - org.apache.avro.Schema.createUnion( - org.apache.avro.Schema.create(Type.STRING), - org.apache.avro.Schema.create(Type.LONG), - org.apache.avro.Schema.create(Type.NULL)); - - TypeWithNullability typeWithNullability = new TypeWithNullability(avroSchema); - assertTrue(typeWithNullability.nullable); - assertEquals( - org.apache.avro.Schema.createUnion( - org.apache.avro.Schema.create(Type.STRING), org.apache.avro.Schema.create(Type.LONG)), - typeWithNullability.type); - } - - @Test - public void testNullableArrayFieldToBeamArrayField() { - org.apache.avro.Schema.Field avroField = - new org.apache.avro.Schema.Field( - "arrayField", - ReflectData.makeNullable( - org.apache.avro.Schema.createArray(org.apache.avro.Schema.create(Type.INT))), - "", - null); - - Field expectedBeamField = Field.nullable("arrayField", FieldType.array(FieldType.INT32)); - - Field beamField = AvroUtils.toBeamField(avroField); - assertEquals(expectedBeamField, beamField); - } - - @Test - public void testNullableBeamArrayFieldToAvroField() { - Field beamField = Field.nullable("arrayField", FieldType.array(FieldType.INT32)); - - org.apache.avro.Schema.Field expectedAvroField = - new org.apache.avro.Schema.Field( - "arrayField", - ReflectData.makeNullable( - org.apache.avro.Schema.createArray(org.apache.avro.Schema.create(Type.INT))), - "", - null); - - org.apache.avro.Schema.Field avroField = AvroUtils.toAvroField(beamField, "ignored"); - assertEquals(expectedAvroField, avroField); - } - - private static List getAvroSubSchemaFields() { - List fields = Lists.newArrayList(); - fields.add( - new org.apache.avro.Schema.Field( - "bool", org.apache.avro.Schema.create(Type.BOOLEAN), "", null)); - fields.add( - new org.apache.avro.Schema.Field("int", org.apache.avro.Schema.create(Type.INT), "", null)); - return fields; - } - - private static org.apache.avro.Schema getAvroSubSchema(String name) { - return org.apache.avro.Schema.createRecord( - name, null, "topLevelRecord", false, getAvroSubSchemaFields()); - } - - private static org.apache.avro.Schema getAvroSchema() { - List fields = Lists.newArrayList(); - fields.add( - new org.apache.avro.Schema.Field( - "bool", org.apache.avro.Schema.create(Type.BOOLEAN), "", (Object) null)); - fields.add( - new org.apache.avro.Schema.Field( - "int", org.apache.avro.Schema.create(Type.INT), "", (Object) null)); - fields.add( - new org.apache.avro.Schema.Field( - "long", org.apache.avro.Schema.create(Type.LONG), "", (Object) null)); - fields.add( - new org.apache.avro.Schema.Field( - "float", org.apache.avro.Schema.create(Type.FLOAT), "", (Object) null)); - fields.add( - new org.apache.avro.Schema.Field( - "double", org.apache.avro.Schema.create(Type.DOUBLE), "", (Object) null)); - fields.add( - new org.apache.avro.Schema.Field( - "string", org.apache.avro.Schema.create(Type.STRING), "", (Object) null)); - fields.add( - new org.apache.avro.Schema.Field( - "bytes", org.apache.avro.Schema.create(Type.BYTES), "", (Object) null)); - fields.add( - new org.apache.avro.Schema.Field( - "decimal", - LogicalTypes.decimal(Integer.MAX_VALUE) - .addToSchema(org.apache.avro.Schema.create(Type.BYTES)), - "", - (Object) null)); - fields.add( - new org.apache.avro.Schema.Field( - "timestampMillis", - LogicalTypes.timestampMillis().addToSchema(org.apache.avro.Schema.create(Type.LONG)), - "", - (Object) null)); - fields.add(new org.apache.avro.Schema.Field("row", getAvroSubSchema("row"), "", (Object) null)); - fields.add( - new org.apache.avro.Schema.Field( - "array", - org.apache.avro.Schema.createArray(getAvroSubSchema("array")), - "", - (Object) null)); - fields.add( - new org.apache.avro.Schema.Field( - "map", org.apache.avro.Schema.createMap(getAvroSubSchema("map")), "", (Object) null)); - return org.apache.avro.Schema.createRecord("topLevelRecord", null, null, false, fields); - } - - private static Schema getBeamSubSchema() { - return new Schema.Builder() - .addField(Field.of("bool", FieldType.BOOLEAN)) - .addField(Field.of("int", FieldType.INT32)) - .build(); - } - - private Schema getBeamSchema() { - Schema subSchema = getBeamSubSchema(); - return new Schema.Builder() - .addField(Field.of("bool", FieldType.BOOLEAN)) - .addField(Field.of("int", FieldType.INT32)) - .addField(Field.of("long", FieldType.INT64)) - .addField(Field.of("float", FieldType.FLOAT)) - .addField(Field.of("double", FieldType.DOUBLE)) - .addField(Field.of("string", FieldType.STRING)) - .addField(Field.of("bytes", FieldType.BYTES)) - .addField(Field.of("decimal", FieldType.DECIMAL)) - .addField(Field.of("timestampMillis", FieldType.DATETIME)) - .addField(Field.of("row", FieldType.row(subSchema))) - .addField(Field.of("array", FieldType.array(FieldType.row(subSchema)))) - .addField(Field.of("map", FieldType.map(FieldType.STRING, FieldType.row(subSchema)))) - .build(); - } - - private static final byte[] BYTE_ARRAY = new byte[] {1, 2, 3, 4}; - private static final DateTime DATE_TIME = - new DateTime().withDate(1979, 3, 14).withTime(1, 2, 3, 4).withZone(DateTimeZone.UTC); - private static final BigDecimal BIG_DECIMAL = new BigDecimal(3600); - - private Row getBeamRow() { - Row subRow = Row.withSchema(getBeamSubSchema()).addValues(true, 42).build(); - return Row.withSchema(getBeamSchema()) - .addValue(true) - .addValue(43) - .addValue(44L) - .addValue((float) 44.1) - .addValue((double) 44.2) - .addValue("string") - .addValue(BYTE_ARRAY) - .addValue(BIG_DECIMAL) - .addValue(DATE_TIME) - .addValue(subRow) - .addValue(ImmutableList.of(subRow, subRow)) - .addValue(ImmutableMap.of("k1", subRow, "k2", subRow)) - .build(); - } - - private static GenericRecord getSubGenericRecord(String name) { - return new GenericRecordBuilder(getAvroSubSchema(name)) - .set("bool", true) - .set("int", 42) - .build(); - } - - private static GenericRecord getGenericRecord() { - - LogicalType decimalType = - LogicalTypes.decimal(Integer.MAX_VALUE) - .addToSchema(org.apache.avro.Schema.create(Type.BYTES)) - .getLogicalType(); - ByteBuffer encodedDecimal = - new Conversions.DecimalConversion().toBytes(BIG_DECIMAL, null, decimalType); - - return new GenericRecordBuilder(getAvroSchema()) - .set("bool", true) - .set("int", 43) - .set("long", 44L) - .set("float", (float) 44.1) - .set("double", (double) 44.2) - .set("string", new Utf8("string")) - .set("bytes", ByteBuffer.wrap(BYTE_ARRAY)) - .set("decimal", encodedDecimal) - .set("timestampMillis", DATE_TIME.getMillis()) - .set("row", getSubGenericRecord("row")) - .set("array", ImmutableList.of(getSubGenericRecord("array"), getSubGenericRecord("array"))) - .set( - "map", - ImmutableMap.of( - new Utf8("k1"), - getSubGenericRecord("map"), - new Utf8("k2"), - getSubGenericRecord("map"))) - .build(); - } - - @Test - public void testFromAvroSchema() { - assertEquals(getBeamSchema(), AvroUtils.toBeamSchema(getAvroSchema())); - } - - @Test - public void testFromBeamSchema() { - Schema beamSchema = getBeamSchema(); - org.apache.avro.Schema avroSchema = AvroUtils.toAvroSchema(beamSchema); - assertEquals(getAvroSchema(), avroSchema); - } - - @Test - public void testAvroSchemaFromBeamSchemaCanBeParsed() { - org.apache.avro.Schema convertedSchema = AvroUtils.toAvroSchema(getBeamSchema()); - org.apache.avro.Schema validatedSchema = - new org.apache.avro.Schema.Parser().parse(convertedSchema.toString()); - assertEquals(convertedSchema, validatedSchema); - } - - @Test - public void testAvroSchemaFromBeamSchemaWithFieldCollisionCanBeParsed() { - - // Two similar schemas, the only difference is the "street" field type in the nested record. - Schema contact = - new Schema.Builder() - .addField(Field.of("name", FieldType.STRING)) - .addField( - Field.of( - "address", - FieldType.row( - new Schema.Builder() - .addField(Field.of("street", FieldType.STRING)) - .addField(Field.of("city", FieldType.STRING)) - .build()))) - .build(); - - Schema contactMultiline = - new Schema.Builder() - .addField(Field.of("name", FieldType.STRING)) - .addField( - Field.of( - "address", - FieldType.row( - new Schema.Builder() - .addField(Field.of("street", FieldType.array(FieldType.STRING))) - .addField(Field.of("city", FieldType.STRING)) - .build()))) - .build(); - - // Ensure that no collisions happen between two sibling fields with same-named child fields - // (with different schemas, between a parent field and a sub-record field with the same name, - // and artificially with the generated field name. - Schema beamSchema = - new Schema.Builder() - .addField(Field.of("home", FieldType.row(contact))) - .addField(Field.of("work", FieldType.row(contactMultiline))) - .addField(Field.of("address", FieldType.row(contact))) - .addField(Field.of("topLevelRecord", FieldType.row(contactMultiline))) - .build(); - - org.apache.avro.Schema convertedSchema = AvroUtils.toAvroSchema(beamSchema); - org.apache.avro.Schema validatedSchema = - new org.apache.avro.Schema.Parser().parse(convertedSchema.toString()); - assertEquals(convertedSchema, validatedSchema); - } - - @Test - public void testNullableFieldInAvroSchema() { - List fields = Lists.newArrayList(); - fields.add( - new org.apache.avro.Schema.Field( - "int", ReflectData.makeNullable(org.apache.avro.Schema.create(Type.INT)), "", null)); - fields.add( - new org.apache.avro.Schema.Field( - "array", - org.apache.avro.Schema.createArray( - ReflectData.makeNullable(org.apache.avro.Schema.create(Type.BYTES))), - "", - null)); - fields.add( - new org.apache.avro.Schema.Field( - "map", - org.apache.avro.Schema.createMap( - ReflectData.makeNullable(org.apache.avro.Schema.create(Type.INT))), - "", - null)); - fields.add( - new org.apache.avro.Schema.Field( - "enum", - ReflectData.makeNullable( - org.apache.avro.Schema.createEnum( - "fruit", "", "", ImmutableList.of("banana", "apple", "pear"))), - "", - null)); - - org.apache.avro.Schema avroSchema = - org.apache.avro.Schema.createRecord("topLevelRecord", null, null, false, fields); - - Schema expectedSchema = - Schema.builder() - .addNullableField("int", FieldType.INT32) - .addArrayField("array", FieldType.BYTES.withNullable(true)) - .addMapField("map", FieldType.STRING, FieldType.INT32.withNullable(true)) - .addField( - "enum", - FieldType.logicalType(EnumerationType.create("banana", "apple", "pear")) - .withNullable(true)) - .build(); - assertEquals(expectedSchema, AvroUtils.toBeamSchema(avroSchema)); - - Map nullMap = Maps.newHashMap(); - nullMap.put("k1", null); - GenericRecord genericRecord = - new GenericRecordBuilder(avroSchema) - .set("int", null) - .set("array", Lists.newArrayList((Object) null)) - .set("map", nullMap) - .set("enum", null) - .build(); - Row expectedRow = - Row.withSchema(expectedSchema) - .addValue(null) - .addValue(Lists.newArrayList((Object) null)) - .addValue(nullMap) - .addValue(null) - .build(); - assertEquals(expectedRow, AvroUtils.toBeamRowStrict(genericRecord, expectedSchema)); - } - - @Test - public void testNullableFieldsInBeamSchema() { - Schema beamSchema = - Schema.builder() - .addNullableField("int", FieldType.INT32) - .addArrayField("array", FieldType.INT32.withNullable(true)) - .addMapField("map", FieldType.STRING, FieldType.INT32.withNullable(true)) - .build(); - - List fields = Lists.newArrayList(); - fields.add( - new org.apache.avro.Schema.Field( - "int", ReflectData.makeNullable(org.apache.avro.Schema.create(Type.INT)), "", null)); - fields.add( - new org.apache.avro.Schema.Field( - "array", - org.apache.avro.Schema.createArray( - ReflectData.makeNullable(org.apache.avro.Schema.create(Type.INT))), - "", - null)); - fields.add( - new org.apache.avro.Schema.Field( - "map", - org.apache.avro.Schema.createMap( - ReflectData.makeNullable(org.apache.avro.Schema.create(Type.INT))), - "", - null)); - org.apache.avro.Schema avroSchema = - org.apache.avro.Schema.createRecord("topLevelRecord", null, null, false, fields); - assertEquals(avroSchema, AvroUtils.toAvroSchema(beamSchema)); - - Map nullMapUtf8 = Maps.newHashMap(); - nullMapUtf8.put(new Utf8("k1"), null); - Map nullMapString = Maps.newHashMap(); - nullMapString.put("k1", null); - - GenericRecord expectedGenericRecord = - new GenericRecordBuilder(avroSchema) - .set("int", null) - .set("array", Lists.newArrayList((Object) null)) - .set("map", nullMapUtf8) - .build(); - Row row = - Row.withSchema(beamSchema) - .addValue(null) - .addValue(Lists.newArrayList((Object) null)) - .addValue(nullMapString) - .build(); - assertEquals(expectedGenericRecord, AvroUtils.toGenericRecord(row, avroSchema)); - } - - @Test - public void testUnionFieldInAvroSchema() { - - List fields = Lists.newArrayList(); - List unionFields = Lists.newArrayList(); - - unionFields.add(org.apache.avro.Schema.create(Type.INT)); - unionFields.add(org.apache.avro.Schema.create(Type.STRING)); - - fields.add( - new org.apache.avro.Schema.Field( - "union", org.apache.avro.Schema.createUnion(unionFields), "", null)); - org.apache.avro.Schema avroSchema = - org.apache.avro.Schema.createRecord("topLevelRecord", null, null, false, fields); - OneOfType oneOfType = - OneOfType.create(Field.of("int", FieldType.INT32), Field.of("string", FieldType.STRING)); - - Schema expectedSchema = Schema.builder().addLogicalTypeField("union", oneOfType).build(); - assertEquals(expectedSchema, AvroUtils.toBeamSchema(avroSchema)); - GenericRecord genericRecord = new GenericRecordBuilder(avroSchema).set("union", 23423).build(); - Row expectedRow = - Row.withSchema(expectedSchema).addValue(oneOfType.createValue(0, 23423)).build(); - assertEquals(expectedRow, AvroUtils.toBeamRowStrict(genericRecord, expectedSchema)); - } - - @Test - public void testUnionFieldInBeamSchema() { - OneOfType oneOfType = - OneOfType.create(Field.of("int", FieldType.INT32), Field.of("string", FieldType.STRING)); - - Schema beamSchema = Schema.builder().addLogicalTypeField("union", oneOfType).build(); - List fields = Lists.newArrayList(); - List unionFields = Lists.newArrayList(); - - unionFields.add(org.apache.avro.Schema.create(Type.INT)); - unionFields.add(org.apache.avro.Schema.create(Type.STRING)); - fields.add( - new org.apache.avro.Schema.Field( - "union", org.apache.avro.Schema.createUnion(unionFields), "", null)); - org.apache.avro.Schema avroSchema = - org.apache.avro.Schema.createRecord("topLevelRecord", null, null, false, fields); - GenericRecord expectedGenericRecord = - new GenericRecordBuilder(avroSchema).set("union", 23423).build(); - Row row = Row.withSchema(beamSchema).addValue(oneOfType.createValue(0, 23423)).build(); - assertEquals(expectedGenericRecord, AvroUtils.toGenericRecord(row, avroSchema)); - } - - @Test - public void testJdbcLogicalVarCharRowDataToAvroSchema() { - String expectedAvroSchemaJson = - "{ " - + " \"name\": \"topLevelRecord\", " - + " \"type\": \"record\", " - + " \"fields\": [{ " - + " \"name\": \"my_varchar_field\", " - + " \"type\": {\"type\": \"string\", \"logicalType\": \"varchar\", \"maxLength\": 10}" - + " }, " - + " { " - + " \"name\": \"my_longvarchar_field\", " - + " \"type\": {\"type\": \"string\", \"logicalType\": \"varchar\", \"maxLength\": 50}" - + " }, " - + " { " - + " \"name\": \"my_nvarchar_field\", " - + " \"type\": {\"type\": \"string\", \"logicalType\": \"varchar\", \"maxLength\": 10}" - + " }, " - + " { " - + " \"name\": \"my_longnvarchar_field\", " - + " \"type\": {\"type\": \"string\", \"logicalType\": \"varchar\", \"maxLength\": 50}" - + " }, " - + " { " - + " \"name\": \"fixed_length_char_field\", " - + " \"type\": {\"type\": \"string\", \"logicalType\": \"char\", \"maxLength\": 25}" - + " } " - + " ] " - + "}"; - - Schema beamSchema = - Schema.builder() - .addField( - Field.of( - "my_varchar_field", FieldType.logicalType(JdbcType.StringType.varchar(10)))) - .addField( - Field.of( - "my_longvarchar_field", - FieldType.logicalType(JdbcType.StringType.longvarchar(50)))) - .addField( - Field.of( - "my_nvarchar_field", FieldType.logicalType(JdbcType.StringType.nvarchar(10)))) - .addField( - Field.of( - "my_longnvarchar_field", - FieldType.logicalType(JdbcType.StringType.longnvarchar(50)))) - .addField( - Field.of( - "fixed_length_char_field", - FieldType.logicalType(JdbcType.StringType.fixedLengthChar(25)))) - .build(); - - assertEquals( - new org.apache.avro.Schema.Parser().parse(expectedAvroSchemaJson), - AvroUtils.toAvroSchema(beamSchema)); - } - - @Test - public void testJdbcLogicalVarCharRowDataToGenericRecord() { - Schema beamSchema = - Schema.builder() - .addField( - Field.of( - "my_varchar_field", FieldType.logicalType(JdbcType.StringType.varchar(10)))) - .addField( - Field.of( - "my_longvarchar_field", - FieldType.logicalType(JdbcType.StringType.longvarchar(50)))) - .addField( - Field.of( - "my_nvarchar_field", FieldType.logicalType(JdbcType.StringType.nvarchar(10)))) - .addField( - Field.of( - "my_longnvarchar_field", - FieldType.logicalType(JdbcType.StringType.longnvarchar(50)))) - .build(); - - Row rowData = - Row.withSchema(beamSchema) - .addValue("varchar_value") - .addValue("longvarchar_value") - .addValue("nvarchar_value") - .addValue("longnvarchar_value") - .build(); - - org.apache.avro.Schema avroSchema = AvroUtils.toAvroSchema(beamSchema); - GenericRecord expectedRecord = - new GenericRecordBuilder(avroSchema) - .set("my_varchar_field", "varchar_value") - .set("my_longvarchar_field", "longvarchar_value") - .set("my_nvarchar_field", "nvarchar_value") - .set("my_longnvarchar_field", "longnvarchar_value") - .build(); - - assertEquals(expectedRecord, AvroUtils.toGenericRecord(rowData, avroSchema)); - } - - @Test - public void testJdbcLogicalDateAndTimeRowDataToAvroSchema() { - String expectedAvroSchemaJson = - "{ " - + " \"name\": \"topLevelRecord\", " - + " \"type\": \"record\", " - + " \"fields\": [{ " - + " \"name\": \"my_date_field\", " - + " \"type\": { \"type\": \"int\", \"logicalType\": \"date\" }" - + " }, " - + " { " - + " \"name\": \"my_time_field\", " - + " \"type\": { \"type\": \"int\", \"logicalType\": \"time-millis\" }" - + " }" - + " ] " - + "}"; - - Schema beamSchema = - Schema.builder() - .addField(Field.of("my_date_field", FieldType.logicalType(JdbcType.DATE))) - .addField(Field.of("my_time_field", FieldType.logicalType(JdbcType.TIME))) - .build(); - - assertEquals( - new org.apache.avro.Schema.Parser().parse(expectedAvroSchemaJson), - AvroUtils.toAvroSchema(beamSchema)); - } - - @Test - public void testJdbcLogicalDateAndTimeRowDataToGenericRecord() { - // Test Fixed clock at - DateTime testDateTime = DateTime.parse("2021-05-29T11:15:16.234Z"); - - Schema beamSchema = - Schema.builder() - .addField(Field.of("my_date_field", FieldType.logicalType(JdbcType.DATE))) - .addField(Field.of("my_time_field", FieldType.logicalType(JdbcType.TIME))) - .build(); - - Row rowData = - Row.withSchema(beamSchema) - .addValue(testDateTime.toLocalDate().toDateTime(LocalTime.MIDNIGHT).toInstant()) - .addValue(Instant.ofEpochMilli(testDateTime.toLocalTime().millisOfDay().get())) - .build(); - - int daysFromEpoch = - Days.daysBetween( - Instant.EPOCH, - testDateTime.toLocalDate().toDateTime(LocalTime.MIDNIGHT).toInstant()) - .getDays(); - int timeSinceMidNight = testDateTime.toLocalTime().getMillisOfDay(); - - org.apache.avro.Schema avroSchema = AvroUtils.toAvroSchema(beamSchema); - GenericRecord expectedRecord = - new GenericRecordBuilder(avroSchema) - .set("my_date_field", daysFromEpoch) - .set("my_time_field", timeSinceMidNight) - .build(); - - assertEquals(expectedRecord, AvroUtils.toGenericRecord(rowData, avroSchema)); - } - - @Test - public void testSqlTypesToGenericRecord() { - // SqlTypes to LogicalTypes.date conversion is one direction - java.time.LocalDate localDate = java.time.LocalDate.of(1979, 3, 14); - - Schema beamSchema = - Schema.builder() - .addField(Field.of("local_date", FieldType.logicalType(SqlTypes.DATE))) - .build(); - - Row rowData = Row.withSchema(beamSchema).addValue(localDate).build(); - - org.apache.avro.Schema avroSchema = AvroUtils.toAvroSchema(beamSchema); - GenericRecord expectedRecord = - new GenericRecordBuilder(avroSchema).set("local_date", localDate.toEpochDay()).build(); - - assertEquals(expectedRecord, AvroUtils.toGenericRecord(rowData, avroSchema)); - } - - @Test - public void testBeamRowToGenericRecord() { - GenericRecord genericRecord = AvroUtils.toGenericRecord(getBeamRow(), null); - assertEquals(getAvroSchema(), genericRecord.getSchema()); - assertEquals(getGenericRecord(), genericRecord); - } - - @Test - public void testBeamRowToGenericRecordInferSchema() { - GenericRecord genericRecord = AvroUtils.toGenericRecord(getBeamRow()); - assertEquals(getAvroSchema(), genericRecord.getSchema()); - assertEquals(getGenericRecord(), genericRecord); - } - - @Test - public void testRowToGenericRecordFunction() { - SerializableUtils.ensureSerializable(AvroUtils.getRowToGenericRecordFunction(NULL_SCHEMA)); - SerializableUtils.ensureSerializable(AvroUtils.getRowToGenericRecordFunction(null)); - } - - @Test - public void testGenericRecordToBeamRow() { - GenericRecord genericRecord = getGenericRecord(); - Row row = AvroUtils.toBeamRowStrict(getGenericRecord(), null); - assertEquals(getBeamRow(), row); - - // Alternatively, a timestamp-millis logical type can have a joda datum. - genericRecord.put("timestampMillis", new DateTime(genericRecord.get("timestampMillis"))); - row = AvroUtils.toBeamRowStrict(getGenericRecord(), null); - assertEquals(getBeamRow(), row); - } - - @Test - public void testGenericRecordToRowFunction() { - SerializableUtils.ensureSerializable(AvroUtils.getGenericRecordToRowFunction(Schema.of())); - SerializableUtils.ensureSerializable(AvroUtils.getGenericRecordToRowFunction(null)); - } - - @Test - public void testAvroSchemaCoders() { - Pipeline pipeline = Pipeline.create(); - org.apache.avro.Schema schema = - org.apache.avro.Schema.createRecord( - "TestSubRecord", - "TestSubRecord doc", - "org.apache.beam.sdk.schemas.utils", - false, - getAvroSubSchemaFields()); - GenericRecord record = - new GenericRecordBuilder(getAvroSubSchema("simple")) - .set("bool", true) - .set("int", 42) - .build(); - - PCollection records = - pipeline.apply(Create.of(record).withCoder(AvroCoder.of(schema))); - assertFalse(records.hasSchema()); - records.setCoder(AvroUtils.schemaCoder(schema)); - assertTrue(records.hasSchema()); - CoderProperties.coderSerializable(records.getCoder()); - - AvroGeneratedUser user = new AvroGeneratedUser("foo", 42, "green"); - PCollection users = - pipeline.apply(Create.of(user).withCoder(AvroCoder.of(AvroGeneratedUser.class))); - assertFalse(users.hasSchema()); - users.setCoder(AvroUtils.schemaCoder((AvroCoder) users.getCoder())); - assertTrue(users.hasSchema()); - CoderProperties.coderSerializable(users.getCoder()); - } - - @Test - public void testAvroBytesToRowAndRowToAvroBytesFunctions() { - Schema schema = - Schema.builder() - .addInt32Field("f_int") - .addInt64Field("f_long") - .addDoubleField("f_double") - .addStringField("f_string") - .build(); - - SimpleFunction toBytesFn = AvroUtils.getRowToAvroBytesFunction(schema); - SimpleFunction toRowFn = AvroUtils.getAvroBytesToRowFunction(schema); - - Row row = Row.withSchema(schema).attachValues(1, 1L, 1d, "string"); - - byte[] serializedRow = toBytesFn.apply(row); - Row deserializedRow = toRowFn.apply(serializedRow); - - assertEquals(row, deserializedRow); - } - - @Test - public void testNullSchemas() { - assertEquals( - AvroUtils.getFromRowFunction(GenericRecord.class), - AvroUtils.getFromRowFunction(GenericRecord.class)); - } - - /** Helper class that simulate JDBC Logical types. */ - private static class JdbcType implements Schema.LogicalType { - - private static final JdbcType DATE = - new JdbcType<>(JDBCType.DATE, FieldType.STRING, FieldType.DATETIME, ""); - private static final JdbcType TIME = - new JdbcType<>(JDBCType.TIME, FieldType.STRING, FieldType.DATETIME, ""); - - private final String identifier; - private final FieldType argumentType; - private final FieldType baseType; - private final Object argument; - - private static class StringType extends JdbcType { - - private static StringType fixedLengthChar(int size) { - return new StringType(JDBCType.CHAR, size); - } - - private static StringType varchar(int size) { - return new StringType(JDBCType.VARCHAR, size); - } - - private static StringType longvarchar(int size) { - return new StringType(JDBCType.LONGVARCHAR, size); - } - - private static StringType nvarchar(int size) { - return new StringType(JDBCType.NVARCHAR, size); - } - - private static StringType longnvarchar(int size) { - return new StringType(JDBCType.LONGNVARCHAR, size); - } - - private StringType(JDBCType type, int size) { - super(type, FieldType.INT32, FieldType.STRING, size); - } - } - - private JdbcType( - JDBCType jdbcType, FieldType argumentType, FieldType baseType, Object argument) { - this.identifier = jdbcType.getName(); - this.argumentType = argumentType; - this.baseType = baseType; - this.argument = argument; - } - - @Override - public String getIdentifier() { - return identifier; - } - - @Override - public @Nullable FieldType getArgumentType() { - return argumentType; - } - - @Override - public FieldType getBaseType() { - return baseType; - } - - @Override - @SuppressWarnings("TypeParameterUnusedInFormals") - public @Nullable T1 getArgument() { - return (T1) argument; - } - - @Override - public @NonNull T toBaseType(@NonNull T input) { - return input; - } - - @Override - public @NonNull T toInputType(@NonNull T base) { - return base; - } - } -} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/SchemaTestUtils.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/SchemaTestUtils.java index 1f039b1989c89..a7a27d6bdad1a 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/SchemaTestUtils.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/SchemaTestUtils.java @@ -32,8 +32,8 @@ import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.Schema.TypeName; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.hamcrest.Matcher; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/SchemaZipFoldTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/SchemaZipFoldTest.java index 8228acd5d1b3a..45ba86bfa4a8e 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/SchemaZipFoldTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/SchemaZipFoldTest.java @@ -27,7 +27,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; import org.junit.Test; /** Tests for {@link SchemaZipFold} with examples. */ diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/SelectHelpersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/SelectHelpersTest.java index 7e5b02c467284..1a9d00fda15bd 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/SelectHelpersTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/SelectHelpersTest.java @@ -27,9 +27,9 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/TestJavaBeans.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/TestJavaBeans.java index 90e44ccca42ec..b5ad6f989d9e4 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/TestJavaBeans.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/TestJavaBeans.java @@ -33,7 +33,7 @@ import org.apache.beam.sdk.schemas.annotations.SchemaFieldName; import org.apache.beam.sdk.schemas.annotations.SchemaFieldNumber; import org.apache.beam.sdk.schemas.annotations.SchemaIgnore; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.CaseFormat; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CaseFormat; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTime; import org.joda.time.Instant; @@ -1362,13 +1362,13 @@ public int hashCode() { @DefaultSchema(JavaBeanSchema.class) public static class ParameterNullableBean { - @org.apache.avro.reflect.Nullable private Float value; + @Nullable private Float value; - public @org.apache.avro.reflect.Nullable Float getValue() { + public @Nullable Float getValue() { return value; } - public void setValue(@org.apache.avro.reflect.Nullable Float value) { + public void setValue(@Nullable Float value) { this.value = value; } } @@ -1379,14 +1379,14 @@ public void setValue(@org.apache.avro.reflect.Nullable Float value) { @DefaultSchema(JavaBeanSchema.class) public static class FieldWithDescriptionBean { - @org.apache.avro.reflect.Nullable private Float value; + @Nullable private Float value; @SchemaFieldDescription("This value is the value stored in the object as a float.") - public @org.apache.avro.reflect.Nullable Float getValue() { + public @Nullable Float getValue() { return value; } - public void setValue(@org.apache.avro.reflect.Nullable Float value) { + public void setValue(@Nullable Float value) { this.value = value; } } diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/TestPOJOs.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/TestPOJOs.java index 103d543996ed6..789de02adee89 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/TestPOJOs.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/TestPOJOs.java @@ -34,7 +34,7 @@ import org.apache.beam.sdk.schemas.annotations.SchemaFieldNumber; import org.apache.beam.sdk.schemas.annotations.SchemaIgnore; import org.apache.beam.sdk.schemas.logicaltypes.EnumerationType; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.CaseFormat; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CaseFormat; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTime; import org.joda.time.Instant; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CoderPropertiesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CoderPropertiesTest.java index a20482eea9b4f..3530f20e722d3 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CoderPropertiesTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CoderPropertiesTest.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.coders.CoderException; import org.apache.beam.sdk.coders.CustomCoder; import org.apache.beam.sdk.coders.StringUtf8Coder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.CoreMatchers; import org.junit.Rule; @@ -136,6 +136,7 @@ public StateChangingSerializingCoder() { changedState = 10; } + @SuppressWarnings("InlineMeInliner") // inline `Strings.repeat()` - Java 11+ API only @Override public void encode(String value, OutputStream outStream) throws CoderException, IOException { changedState += 1; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CombineFnTesterTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CombineFnTesterTest.java index f76b3e1e18903..a83770f0af163 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CombineFnTesterTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/CombineFnTesterTest.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.transforms.Combine.CombineFn; import org.apache.beam.sdk.transforms.Sum; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.hamcrest.Description; import org.hamcrest.Matcher; import org.hamcrest.Matchers; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FileChecksumMatcherTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FileChecksumMatcherTest.java index 606e034e48140..09a8f960a0921 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FileChecksumMatcherTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/FileChecksumMatcherTest.java @@ -27,7 +27,7 @@ import java.nio.charset.StandardCharsets; import java.util.regex.Pattern; import org.apache.beam.sdk.util.NumberedShardedFile; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; import org.apache.commons.lang3.SystemUtils; import org.junit.Rule; import org.junit.Test; @@ -59,7 +59,7 @@ public void testPreconditionChecksumIsEmpty() throws IOException { @Test public void testMatcherThatVerifiesSingleFile() throws IOException { File tmpFile = tmpFolder.newFile("result-000-of-001"); - Files.write("Test for file checksum verifier.", tmpFile, StandardCharsets.UTF_8); + Files.asCharSink(tmpFile, StandardCharsets.UTF_8).write("Test for file checksum verifier."); assertThat( new NumberedShardedFile(tmpFile.getPath()), @@ -73,9 +73,9 @@ public void testMatcherThatVerifiesMultipleFiles() throws IOException { File tmpFile1 = tmpFolder.newFile("result-000-of-002"); File tmpFile2 = tmpFolder.newFile("result-001-of-002"); File tmpFile3 = tmpFolder.newFile("tmp"); - Files.write("To be or not to be, ", tmpFile1, StandardCharsets.UTF_8); - Files.write("it is not a question.", tmpFile2, StandardCharsets.UTF_8); - Files.write("tmp", tmpFile3, StandardCharsets.UTF_8); + Files.asCharSink(tmpFile1, StandardCharsets.UTF_8).write("To be or not to be, "); + Files.asCharSink(tmpFile2, StandardCharsets.UTF_8).write("it is not a question."); + Files.asCharSink(tmpFile3, StandardCharsets.UTF_8).write("tmp"); assertThat( new NumberedShardedFile(tmpFolder.getRoot().toPath().resolve("result-*").toString()), @@ -87,7 +87,7 @@ public void testMatcherThatVerifiesFileWithEmptyContent() throws IOException { // TODO: Java core test failing on windows, https://github.com/apache/beam/issues/20483 assumeFalse(SystemUtils.IS_OS_WINDOWS); File emptyFile = tmpFolder.newFile("result-000-of-001"); - Files.write("", emptyFile, StandardCharsets.UTF_8); + Files.asCharSink(emptyFile, StandardCharsets.UTF_8).write(""); assertThat( new NumberedShardedFile(tmpFolder.getRoot().toPath().resolve("*").toString()), @@ -101,8 +101,8 @@ public void testMatcherThatUsesCustomizedTemplate() throws Exception { assumeFalse(SystemUtils.IS_OS_WINDOWS); File tmpFile1 = tmpFolder.newFile("result0-total2"); File tmpFile2 = tmpFolder.newFile("result1-total2"); - Files.write("To be or not to be, ", tmpFile1, StandardCharsets.UTF_8); - Files.write("it is not a question.", tmpFile2, StandardCharsets.UTF_8); + Files.asCharSink(tmpFile1, StandardCharsets.UTF_8).write("To be or not to be, "); + Files.asCharSink(tmpFile2, StandardCharsets.UTF_8).write("it is not a question."); Pattern customizedTemplate = Pattern.compile("(?x) result (?\\d+) - total (?\\d+)"); diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/GatherAllPanesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/GatherAllPanesTest.java index 6c4312675d86c..cae46b0d89c4a 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/GatherAllPanesTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/GatherAllPanesTest.java @@ -34,7 +34,7 @@ import org.apache.beam.sdk.values.PCollectionList; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.ValueInSingleWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/InterceptingUrlClassLoader.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/InterceptingUrlClassLoader.java index 39e11ef801636..2f15d005f2425 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/InterceptingUrlClassLoader.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/InterceptingUrlClassLoader.java @@ -19,9 +19,9 @@ import java.io.IOException; import java.util.function.Predicate; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; /** * A classloader that intercepts loading of specifically named classes. This classloader copies the diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java index aae85133cb9e3..dfdb6282b5496 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PAssertTest.java @@ -58,9 +58,9 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; @@ -397,7 +397,7 @@ public void testPAssertEqualsSingletonFalseDefaultReasonString() throws Exceptio String message = thrown.getMessage(); - assertThat(message, containsString("Create.Values/Read(CreateSource)")); + assertThat(message, containsString("Create.Values/")); assertThat(message, containsString("Expected: <44>")); assertThat(message, containsString("but: was <42>")); } diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java index 61b4bf8c8e9e5..0d1b73bf32e9c 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.PCollectionViews.ValueOrMetadata; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; /** Methods for testing {@link PCollectionView}s. */ public final class PCollectionViewTesting { diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PaneExtractorsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PaneExtractorsTest.java index d922710e3ae46..78abbcaca872f 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PaneExtractorsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PaneExtractorsTest.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing; import org.apache.beam.sdk.values.ValueInSingleWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SerializableMatchersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SerializableMatchersTest.java index 7f720d63abd09..d951bbb132d3c 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SerializableMatchersTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SerializableMatchersTest.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.coders.AtomicCoder; import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matchers; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SourceTestUtilsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SourceTestUtilsTest.java index 81c002b5fc467..44388f4d4d3d8 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SourceTestUtilsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/SourceTestUtilsTest.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.io.CountingSource; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/StaticWindowsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/StaticWindowsTest.java index 0189270342ff3..66aa031144007 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/StaticWindowsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/StaticWindowsTest.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.transforms.windowing.IncompatibleWindowException; import org.apache.beam.sdk.transforms.windowing.IntervalWindow; import org.apache.beam.sdk.transforms.windowing.WindowFn; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.Matchers; import org.joda.time.Instant; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/TestPipelineTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/TestPipelineTest.java index c3c2dd1e01aa7..e71b9037fc569 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/TestPipelineTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/TestPipelineTest.java @@ -42,7 +42,7 @@ import org.apache.beam.sdk.transforms.SimpleFunction; import org.apache.beam.sdk.util.common.ReflectHelpers; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/TestStreamTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/TestStreamTest.java index 3e14a3126b303..4bf865475e348 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/TestStreamTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/TestStreamTest.java @@ -65,7 +65,7 @@ import org.apache.beam.sdk.values.PCollectionList; import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/WindowSupplierTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/WindowSupplierTest.java index 8d49505b77858..8559ba540654d 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/WindowSupplierTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/WindowSupplierTest.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.IntervalWindow; import org.apache.beam.sdk.util.SerializableUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.Matchers; import org.joda.time.Instant; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ApproximateQuantilesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ApproximateQuantilesTest.java index 6d6e07f1adcaa..d5949558f2790 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ApproximateQuantilesTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ApproximateQuantilesTest.java @@ -40,7 +40,7 @@ import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.hamcrest.CoreMatchers; import org.hamcrest.Description; import org.hamcrest.Matcher; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ApproximateUniqueTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ApproximateUniqueTest.java index bfa4f8d75c6eb..297d36e8827ec 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ApproximateUniqueTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ApproximateUniqueTest.java @@ -46,8 +46,8 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.hamcrest.Matcher; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java index 4f89d8b68620b..5408dad1d1f68 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineFnsTest.java @@ -47,8 +47,8 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matchers; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java index 7aa22d89df837..024fedd177e50 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java @@ -21,8 +21,8 @@ import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem; import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasNamespace; import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.includesDisplayDataFor; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; @@ -79,10 +79,10 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java index f0dee49f08389..689e5af7055b7 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java @@ -64,8 +64,8 @@ import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matchers; import org.joda.time.Instant; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DeduplicateTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DeduplicateTest.java index 0a8dc54ff9030..40538ccefac06 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DeduplicateTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DeduplicateTest.java @@ -37,9 +37,9 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DistinctTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DistinctTest.java index 124395d001940..e57074eb30dcd 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DistinctTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DistinctTest.java @@ -49,8 +49,8 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java index 2d888499f6ade..f27b936b7110c 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlatMapElementsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlatMapElementsTest.java index 9e606b39c5a9d..aa0423e44fcc4 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlatMapElementsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlatMapElementsTest.java @@ -43,8 +43,8 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlattenTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlattenTest.java index 5b6d34ccddd8b..282a41bed0dc9 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlattenTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/FlattenTest.java @@ -65,7 +65,7 @@ import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.joda.time.Duration; import org.junit.Assert; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java index 9d6bd0131a70e..5464838ad4dba 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java @@ -84,8 +84,8 @@ import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TypeDescriptors; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Streams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Streams; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matcher; import org.joda.time.Duration; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupIntoBatchesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupIntoBatchesTest.java index 22a50d4703721..bc2aab2ba0ef7 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupIntoBatchesTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupIntoBatchesTest.java @@ -53,12 +53,12 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Streams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Streams; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; @@ -664,6 +664,7 @@ public void processElement(ProcessContext c, BoundedWindow window) { pipeline.run().waitUntilFinish(); } + @SuppressWarnings("InlineMeInliner") // inline `Strings.repeat()` - Java 11+ API only @Test @Category({ ValidatesRunner.class, diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/JsonToRowTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/JsonToRowTest.java index 80c2a5b87a20b..490cb68ab9e44 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/JsonToRowTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/JsonToRowTest.java @@ -33,7 +33,7 @@ import org.apache.beam.sdk.util.RowJson.RowJsonDeserializer.NullBehavior; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/LatestFnTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/LatestFnTest.java index 6e8e410db3083..3eafbaa7c8622 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/LatestFnTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/LatestFnTest.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.coders.NullableCoder; import org.apache.beam.sdk.coders.VarLongCoder; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapKeysTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapKeysTest.java index 659388b133ea5..da783b3ddc369 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapKeysTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapKeysTest.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapValuesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapValuesTest.java index e32d6e9ca6009..264291cc6d1e7 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapValuesTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapValuesTest.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MaxTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MaxTest.java index f41de3c763d61..506993d3e78b7 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MaxTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MaxTest.java @@ -23,7 +23,7 @@ import static org.junit.Assert.assertEquals; import org.apache.beam.sdk.transforms.display.DisplayData; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MeanTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MeanTest.java index 66934471ee769..07322c523bf1f 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MeanTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MeanTest.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.transforms.Mean.CountSum; import org.apache.beam.sdk.transforms.Mean.CountSumCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MinTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MinTest.java index 5d6ea0390f997..665f136e040b7 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MinTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MinTest.java @@ -23,7 +23,7 @@ import static org.junit.Assert.assertEquals; import org.apache.beam.sdk.transforms.display.DisplayData; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java index dee441aefe80e..02d67f5261ff0 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java @@ -52,7 +52,7 @@ import org.apache.beam.sdk.values.PCollectionList; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoSchemaTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoSchemaTest.java index 4e212f41a1d74..ea1999ef6b37e 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoSchemaTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoSchemaTest.java @@ -58,8 +58,8 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java index f0c4cfb604187..e478fdad0966d 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java @@ -27,7 +27,7 @@ import static org.apache.beam.sdk.util.SerializableUtils.serializeToByteArray; import static org.apache.beam.sdk.util.StringUtils.byteArrayToJsonString; import static org.apache.beam.sdk.util.StringUtils.jsonStringToByteArray; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.anyOf; @@ -154,13 +154,13 @@ import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matchers; import org.joda.time.DateTimeUtils; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/PerKeyOrderingTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/PerKeyOrderingTest.java index f15c8b2e941cc..b023127cc522f 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/PerKeyOrderingTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/PerKeyOrderingTest.java @@ -37,7 +37,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/PeriodicSequenceTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/PeriodicSequenceTest.java index 33ef3cd9cf096..3ace145eba880 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/PeriodicSequenceTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/PeriodicSequenceTest.java @@ -38,8 +38,8 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Streams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Streams; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReshuffleTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReshuffleTest.java index 0a1dbff06f5f4..716bd8781831b 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReshuffleTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReshuffleTest.java @@ -47,8 +47,8 @@ import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TypeDescriptors; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SampleTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SampleTest.java index 9f7e89f7ae446..6157429349215 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SampleTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SampleTest.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.transforms; import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.everyItem; @@ -49,8 +49,8 @@ import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.Matchers; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SetsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SetsTest.java index ed07a65b0219f..43b76c0959e46 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SetsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SetsTest.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SplittableDoFnTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SplittableDoFnTest.java index 0012f0d4a19d1..8ce9330b9ab4f 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SplittableDoFnTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SplittableDoFnTest.java @@ -20,7 +20,7 @@ import static java.lang.Thread.sleep; import static org.apache.beam.sdk.transforms.DoFn.ProcessContinuation.resume; import static org.apache.beam.sdk.transforms.DoFn.ProcessContinuation.stop; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasEntry; import static org.junit.Assert.assertEquals; @@ -74,8 +74,8 @@ import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; import org.joda.time.Duration; import org.joda.time.Instant; import org.joda.time.MutableDateTime; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SumTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SumTest.java index 748a3b17666a1..c3dfb10173d0f 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SumTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SumTest.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.coders.DoubleCoder; import org.apache.beam.sdk.coders.VarIntCoder; import org.apache.beam.sdk.coders.VarLongCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java index a393fe4e0b1bf..38fe8740c684d 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ViewTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.transforms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -64,7 +64,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.Matchers; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/WaitTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/WaitTest.java index 2f9c832d81989..1e1c780a0e06f 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/WaitTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/WaitTest.java @@ -39,8 +39,8 @@ import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/WatchTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/WatchTest.java index b949bf10c313e..277d49a7240d8 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/WatchTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/WatchTest.java @@ -62,13 +62,13 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Funnel; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Funnels; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashCode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Funnel; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Funnels; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.HashCode; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; import org.joda.time.Duration; import org.joda.time.Instant; import org.joda.time.ReadableDuration; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluator.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluator.java index 5cdfc41e5e06e..986e7371e429f 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluator.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluator.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.POutput; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; /** * Test utilities to evaluate the {@link DisplayData} in the context of a {@link PipelineRunner}. diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataMatchers.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataMatchers.java index de5d0ab1d338a..0044d2a84a781 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataMatchers.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataMatchers.java @@ -23,7 +23,7 @@ import java.util.Arrays; import java.util.Collection; import org.apache.beam.sdk.transforms.display.DisplayData.Item; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.hamcrest.CustomTypeSafeMatcher; import org.hamcrest.Description; import org.hamcrest.FeatureMatcher; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataTest.java index 197408a4f6c0f..c80e92a897f9a 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataTest.java @@ -57,10 +57,10 @@ import org.apache.beam.sdk.transforms.display.DisplayData.Item; import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.CustomTypeSafeMatcher; import org.hamcrest.FeatureMatcher; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/join/CoGbkResultCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/join/CoGbkResultCoderTest.java index b52709ba12d6f..e88abd89c597a 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/join/CoGbkResultCoderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/join/CoGbkResultCoderTest.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/join/CoGroupByKeyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/join/CoGroupByKeyTest.java index 300e5d1eb831a..e89210736755d 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/join/CoGroupByKeyTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/join/CoGroupByKeyTest.java @@ -45,8 +45,8 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Duration; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/join/UnionCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/join/UnionCoderTest.java index 681f41899bc09..1d01e5bb7f8a5 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/join/UnionCoderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/join/UnionCoderTest.java @@ -22,7 +22,7 @@ import org.apache.beam.sdk.coders.DoubleCoder; import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.testing.CoderProperties; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.Matchers; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesSplittableDoFnTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesSplittableDoFnTest.java index 4bb957588a363..5a22203a8a766 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesSplittableDoFnTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesSplittableDoFnTest.java @@ -19,7 +19,7 @@ import static org.apache.beam.sdk.transforms.reflect.DoFnSignaturesTestUtils.analyzeProcessElementMethod; import static org.apache.beam.sdk.transforms.reflect.DoFnSignaturesTestUtils.errors; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -55,8 +55,8 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Instant; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java index 61f2971bde9e9..de4a622e03d7d 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java @@ -82,7 +82,7 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.hamcrest.Matcher; import org.hamcrest.Matchers; import org.joda.time.Instant; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/resourcehints/ResourceHintsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/resourcehints/ResourceHintsTest.java index 3cc5221763742..c7643f718aa54 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/resourcehints/ResourceHintsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/resourcehints/ResourceHintsTest.java @@ -92,10 +92,13 @@ public void testFromOptions() { .withHint("beam:resources:bar", new ResourceHints.StringHint("foo"))); options = PipelineOptionsFactory.fromArgs( - "--resourceHints=min_ram=1KB", "--resourceHints=accelerator=foo") + "--resourceHints=min_ram=1KB", + "--resourceHints=accelerator=foo", + "--resourceHints=cpu_count=4") .as(ResourceHintsOptions.class); - assertEquals( - ResourceHints.fromOptions(options), - ResourceHints.create().withMinRam(1000).withAccelerator("foo")); + ResourceHints fromOptions = ResourceHints.fromOptions(options); + ResourceHints expect = + ResourceHints.create().withMinRam(1000).withAccelerator("foo").withCPUCount(4); + assertEquals(fromOptions, expect); } } diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/GlobalWindowTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/GlobalWindowTest.java index 6bd2d86ed25b8..f2f9644fdec8f 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/GlobalWindowTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/GlobalWindowTest.java @@ -21,8 +21,8 @@ import org.apache.beam.sdk.coders.Coder.Context; import org.apache.beam.sdk.testing.CoderProperties; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CountingOutputStream; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CountingOutputStream; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/IntervalWindowTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/IntervalWindowTest.java index a79ff6fef6dcd..3c1a4b4004d0a 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/IntervalWindowTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/IntervalWindowTest.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.testing.CoderProperties.TestElementByteSizeObserver; import org.apache.beam.sdk.util.CoderUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SessionsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SessionsTest.java index 03750e7dc6ad7..783fc3b37265c 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SessionsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/SessionsTest.java @@ -33,7 +33,7 @@ import java.util.Set; import org.apache.beam.sdk.testing.WindowFnTestUtils; import org.apache.beam.sdk.transforms.display.DisplayData; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/StubTrigger.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/StubTrigger.java index d4ee23b92b2f4..bdcb82a386316 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/StubTrigger.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/StubTrigger.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.transforms.windowing; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Instant; /** No-op {@link OnceTrigger} implementation for testing. */ diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java index fffafc1cd5989..f61d6736a6ef7 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java @@ -71,8 +71,8 @@ import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matchers; import org.joda.time.Duration; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowingTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowingTest.java index 6a9ccc4adcf73..c1c3e86f57112 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowingTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowingTest.java @@ -38,7 +38,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ApiSurfaceTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ApiSurfaceTest.java index 2b63ec33713b6..eaaa088db0739 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ApiSurfaceTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ApiSurfaceTest.java @@ -23,10 +23,10 @@ import static org.hamcrest.Matchers.emptyIterable; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.hamcrest.Matcher; import org.hamcrest.Matchers; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStreamTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStreamTest.java index 9ad70a7abfa10..5298d29dad101 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStreamTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStreamTest.java @@ -34,8 +34,8 @@ import java.util.List; import java.util.Random; import org.apache.beam.sdk.coders.ByteArrayCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.collection.IsIterableContainingInOrder; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/CombineFnUtilTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/CombineFnUtilTest.java index 084b067d33f2e..42f201c5bfbf3 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/CombineFnUtilTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/CombineFnUtilTest.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext; import org.apache.beam.sdk.transforms.CombineWithContext.Context; import org.apache.beam.sdk.transforms.Sum; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ExposedByteArrayInputStreamTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ExposedByteArrayInputStreamTest.java index 03fe8ed362d47..e87f6a2b0d0a6 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ExposedByteArrayInputStreamTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ExposedByteArrayInputStreamTest.java @@ -24,7 +24,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ExposedByteArrayOutputStreamTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ExposedByteArrayOutputStreamTest.java index 55695b7916ffe..7e1b213c85b25 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ExposedByteArrayOutputStreamTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ExposedByteArrayOutputStreamTest.java @@ -25,7 +25,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.UnsupportedEncodingException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FilePatternMatchingShardedFileTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FilePatternMatchingShardedFileTest.java index 8ff0c3336cbc7..548376844d8fe 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FilePatternMatchingShardedFileTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/FilePatternMatchingShardedFileTest.java @@ -31,7 +31,7 @@ import java.nio.charset.StandardCharsets; import org.apache.beam.sdk.io.LocalResources; import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; import org.apache.commons.lang3.SystemUtils; import org.junit.Before; import org.junit.Rule; @@ -86,9 +86,9 @@ public void testReadMultipleShards() throws Exception { File tmpFile1 = tmpFolder.newFile("result-000-of-002"); File tmpFile2 = tmpFolder.newFile("result-001-of-002"); File tmpFile3 = tmpFolder.newFile("tmp"); - Files.write(contents1, tmpFile1, StandardCharsets.UTF_8); - Files.write(contents2, tmpFile2, StandardCharsets.UTF_8); - Files.write(contents3, tmpFile3, StandardCharsets.UTF_8); + Files.asCharSink(tmpFile1, StandardCharsets.UTF_8).write(contents1); + Files.asCharSink(tmpFile2, StandardCharsets.UTF_8).write(contents2); + Files.asCharSink(tmpFile3, StandardCharsets.UTF_8).write(contents3); filePattern = LocalResources.fromFile(tmpFolder.getRoot(), true) @@ -106,8 +106,8 @@ public void testReadMultipleShardsWithoutShardNumber() throws Exception { File tmpFile1 = tmpFolder.newFile("result"); File tmpFile2 = tmpFolder.newFile("tmp"); - Files.write(contents1, tmpFile1, StandardCharsets.UTF_8); - Files.write(contents2, tmpFile2, StandardCharsets.UTF_8); + Files.asCharSink(tmpFile1, StandardCharsets.UTF_8).write(contents1); + Files.asCharSink(tmpFile2, StandardCharsets.UTF_8).write(contents2); FilePatternMatchingShardedFile shardedFile = new FilePatternMatchingShardedFile(filePattern); @@ -117,7 +117,7 @@ public void testReadMultipleShardsWithoutShardNumber() throws Exception { @Test public void testReadEmpty() throws Exception { File emptyFile = tmpFolder.newFile("result-000-of-001"); - Files.write("", emptyFile, StandardCharsets.UTF_8); + Files.asCharSink(emptyFile, StandardCharsets.UTF_8).write(""); FilePatternMatchingShardedFile shardedFile = new FilePatternMatchingShardedFile(filePattern); assertThat(shardedFile.readFilesWithRetries(), empty()); @@ -126,7 +126,7 @@ public void testReadEmpty() throws Exception { @Test public void testReadWithRetriesFailsSinceFilesystemError() throws Exception { File tmpFile = tmpFolder.newFile(); - Files.write("Test for file checksum verifier.", tmpFile, StandardCharsets.UTF_8); + Files.asCharSink(tmpFile, StandardCharsets.UTF_8).write("Test for file checksum verifier."); FilePatternMatchingShardedFile shardedFile = spy(new FilePatternMatchingShardedFile(filePattern)); doThrow(IOException.class).when(shardedFile).readLines(anyCollection()); diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/MutationDetectorsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/MutationDetectorsTest.java index 55c309c4326fa..99ea9db973e2a 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/MutationDetectorsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/MutationDetectorsTest.java @@ -33,9 +33,9 @@ import org.apache.beam.sdk.coders.IterableCoder; import org.apache.beam.sdk.coders.ListCoder; import org.apache.beam.sdk.coders.VarIntCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/NumberedShardedFileTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/NumberedShardedFileTest.java index 08c8832b1bed0..c2372bd55af72 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/NumberedShardedFileTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/NumberedShardedFileTest.java @@ -32,7 +32,7 @@ import java.util.regex.Pattern; import org.apache.beam.sdk.io.LocalResources; import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; import org.apache.commons.lang3.SystemUtils; import org.junit.Before; import org.junit.Rule; @@ -88,9 +88,9 @@ public void testReadMultipleShards() throws Exception { File tmpFile1 = tmpFolder.newFile("result-000-of-002"); File tmpFile2 = tmpFolder.newFile("result-001-of-002"); File tmpFile3 = tmpFolder.newFile("tmp"); - Files.write(contents1, tmpFile1, StandardCharsets.UTF_8); - Files.write(contents2, tmpFile2, StandardCharsets.UTF_8); - Files.write(contents3, tmpFile3, StandardCharsets.UTF_8); + Files.asCharSink(tmpFile1, StandardCharsets.UTF_8).write(contents1); + Files.asCharSink(tmpFile2, StandardCharsets.UTF_8).write(contents2); + Files.asCharSink(tmpFile3, StandardCharsets.UTF_8).write(contents3); filePattern = LocalResources.fromFile(tmpFolder.getRoot(), true) @@ -104,7 +104,7 @@ public void testReadMultipleShards() throws Exception { @Test public void testReadEmpty() throws Exception { File emptyFile = tmpFolder.newFile("result-000-of-001"); - Files.write("", emptyFile, StandardCharsets.UTF_8); + Files.asCharSink(emptyFile, StandardCharsets.UTF_8).write(""); NumberedShardedFile shardedFile = new NumberedShardedFile(filePattern); assertThat(shardedFile.readFilesWithRetries(), empty()); @@ -117,8 +117,8 @@ public void testReadCustomTemplate() throws Exception { // Customized template: resultSSS-totalNNN File tmpFile1 = tmpFolder.newFile("result0-total2"); File tmpFile2 = tmpFolder.newFile("result1-total2"); - Files.write(contents1, tmpFile1, StandardCharsets.UTF_8); - Files.write(contents2, tmpFile2, StandardCharsets.UTF_8); + Files.asCharSink(tmpFile1, StandardCharsets.UTF_8).write(contents1); + Files.asCharSink(tmpFile2, StandardCharsets.UTF_8).write(contents2); Pattern customizedTemplate = Pattern.compile("(?x) result (?\\d+) - total (?\\d+)"); @@ -130,7 +130,7 @@ public void testReadCustomTemplate() throws Exception { @Test public void testReadWithRetriesFailsWhenTemplateIncorrect() throws Exception { File tmpFile = tmpFolder.newFile(); - Files.write("Test for file checksum verifier.", tmpFile, StandardCharsets.UTF_8); + Files.asCharSink(tmpFile, StandardCharsets.UTF_8).write("Test for file checksum verifier."); NumberedShardedFile shardedFile = new NumberedShardedFile(filePattern, Pattern.compile("incorrect-template")); @@ -145,7 +145,7 @@ public void testReadWithRetriesFailsWhenTemplateIncorrect() throws Exception { @Test public void testReadWithRetriesFailsSinceFilesystemError() throws Exception { File tmpFile = tmpFolder.newFile(); - Files.write("Test for file checksum verifier.", tmpFile, StandardCharsets.UTF_8); + Files.asCharSink(tmpFile, StandardCharsets.UTF_8).write("Test for file checksum verifier."); NumberedShardedFile shardedFile = spy(new NumberedShardedFile(filePattern)); doThrow(IOException.class).when(shardedFile).readLines(anyCollection()); diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/RowJsonTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/RowJsonTest.java index b8b8efa457d1e..328765bf7f150 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/RowJsonTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/RowJsonTest.java @@ -38,7 +38,7 @@ import org.apache.beam.sdk.util.RowJson.RowJsonSerializer; import org.apache.beam.sdk.util.RowJson.UnsupportedRowJsonException; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.Matcher; import org.hamcrest.Matchers; import org.joda.time.DateTime; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/SerializableUtilsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/SerializableUtilsTest.java index 9f96bce0e4c34..e15bd42dc3ce3 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/SerializableUtilsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/SerializableUtilsTest.java @@ -29,8 +29,8 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.CoderException; import org.apache.beam.sdk.testing.InterceptingUrlClassLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/WindowedValueTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/WindowedValueTest.java index 7e80cce41c04c..0a18f076762ac 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/WindowedValueTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/WindowedValueTest.java @@ -31,8 +31,8 @@ import org.apache.beam.sdk.transforms.windowing.IntervalWindow; import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Assert; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ZipFilesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ZipFilesTest.java index 2cf38be6b8c10..6855b9873ad7a 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ZipFilesTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/ZipFilesTest.java @@ -35,9 +35,9 @@ import java.util.Enumeration; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteSource; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CharSource; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteSource; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CharSource; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/KVTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/KVTest.java index c5aaf26394d6b..7cbbcbd1e7ad4 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/KVTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/KVTest.java @@ -23,7 +23,7 @@ import static org.junit.Assert.assertEquals; import java.util.Comparator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionListTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionListTest.java index 8e9812b062f24..3bdb9933da0dd 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionListTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionListTest.java @@ -32,7 +32,7 @@ import org.apache.beam.sdk.io.GenerateSequence; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionRowTupleTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionRowTupleTest.java index 1563ee7d87020..60b5ef0267b1e 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionRowTupleTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionRowTupleTest.java @@ -41,7 +41,7 @@ import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.transforms.SimpleFunction; import org.apache.beam.sdk.values.PCollection.IsBounded; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionTupleTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionTupleTest.java index b3f0d87aea706..1a2b56749a91b 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionTupleTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionTupleTest.java @@ -41,7 +41,7 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.SimpleFunction; import org.apache.beam.sdk.values.PCollection.IsBounded; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionViewsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionViewsTest.java index d2c1929e2fe04..457c83f9bed4f 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionViewsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/PCollectionViewsTest.java @@ -33,10 +33,10 @@ import java.util.TreeSet; import java.util.stream.IntStream; import org.apache.beam.sdk.io.range.OffsetRange; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/RowTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/RowTest.java index 574d672d45011..6182a6791577a 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/RowTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/RowTest.java @@ -36,9 +36,9 @@ import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.logicaltypes.EnumerationType; import org.apache.beam.sdk.schemas.logicaltypes.FixedBytes; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.junit.Rule; @@ -405,7 +405,7 @@ public void testCreateAndCompareNullMap() { } @Test - public void testCreateMapWithNullValue() { + public void testCreateAndCompareMapWithNullValue() { Map data = new HashMap(); data.put(1, "value1"); data.put(2, "value2"); @@ -416,6 +416,25 @@ public void testCreateMapWithNullValue() { .collect(toSchema()); Row row = Row.withSchema(type).addValue(data).build(); assertEquals(data, row.getMap("map")); + + Map onlyNullValueData = new HashMap(); + onlyNullValueData.put(1, null); + onlyNullValueData.put(2, null); + + Map otherOnlyNullValueData = new HashMap(); + otherOnlyNullValueData.put(3, null); + otherOnlyNullValueData.put(4, null); + + Row otherNonNullValue = + Row.withSchema(type) + .addValue(ImmutableMap.of(1, "value1", 2, "value2", 3, "value3", 4, "value4")) + .build(); + Row otherNullValue = Row.withSchema(type).addValue(data).build(); + Row onlyNullValue = Row.withSchema(type).addValue(onlyNullValueData).build(); + Row otherOnlyNullValue = Row.withSchema(type).addValue(otherOnlyNullValueData).build(); + assertNotEquals(otherNonNullValue, row); + assertEquals(otherNullValue, row); + assertNotEquals(onlyNullValue, otherOnlyNullValue); } @Test diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/TupleTagTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/TupleTagTest.java index addd92123aa28..a3f42bbc392c6 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/TupleTagTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/TupleTagTest.java @@ -23,8 +23,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/TypeDescriptorTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/TypeDescriptorTest.java index dbad49c284a39..3dc5384f502c9 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/values/TypeDescriptorTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/values/TypeDescriptorTest.java @@ -23,7 +23,7 @@ import java.lang.reflect.TypeVariable; import java.util.List; import java.util.Set; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.reflect.TypeToken; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.reflect.TypeToken; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/expansion-service/build.gradle b/sdks/java/expansion-service/build.gradle index 8d350005f7c0e..7bc77e3aea803 100644 --- a/sdks/java/expansion-service/build.gradle +++ b/sdks/java/expansion-service/build.gradle @@ -46,7 +46,7 @@ dependencies { implementation library.java.jackson_databind implementation library.java.jackson_dataformat_yaml implementation library.java.vendored_grpc_1_54_0 - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.slf4j_api runtimeOnly library.java.slf4j_jdk14 testImplementation library.java.junit diff --git a/sdks/java/expansion-service/container/build.gradle b/sdks/java/expansion-service/container/build.gradle index 2897049ca2960..d5488df3d146c 100644 --- a/sdks/java/expansion-service/container/build.gradle +++ b/sdks/java/expansion-service/container/build.gradle @@ -60,6 +60,9 @@ task copyConfigFile(type: Copy) { into "build/target" } +def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] +def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers") + docker { name containerImageName( name: project.docker_image_default_repo_prefix + "java_expansion_service", @@ -71,8 +74,10 @@ docker { // tags used by dockerTag task tags containerImageTags() files "./build" - buildx project.containerPlatforms() != [project.nativeArchitecture()] + buildx useBuildx platform(*project.containerPlatforms()) + load useBuildx && !pushContainers + push pushContainers } dockerPrepare.dependsOn goBuild @@ -96,5 +101,5 @@ if (project.rootProject.hasProperty(["docker-pull-licenses"])) { } task pushAll { - dependsOn dockerPush + dependsOn docker } diff --git a/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionServer.java b/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionServer.java index 489e2727e69cd..1b3330b490c7a 100644 --- a/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionServer.java +++ b/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionServer.java @@ -23,7 +23,7 @@ import org.apache.beam.runners.fnexecution.artifact.ArtifactRetrievalService; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Server; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.netty.NettyServerBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; /** A {@link Server gRPC Server} for an ExpansionService. */ public class ExpansionServer implements AutoCloseable { diff --git a/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java b/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java index 42ce02a5b6336..ec53e3f11e43d 100644 --- a/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java +++ b/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java @@ -18,10 +18,12 @@ package org.apache.beam.sdk.expansion.service; import static org.apache.beam.runners.core.construction.BeamUrns.getUrn; +import static org.apache.beam.runners.core.construction.PTransformTranslation.READ_TRANSFORM_URN; import static org.apache.beam.runners.core.construction.resources.PipelineResources.detectClassPathResourcesToStage; import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; import com.google.auto.service.AutoService; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; @@ -45,10 +47,12 @@ import org.apache.beam.model.pipeline.v1.RunnerApi; import org.apache.beam.model.pipeline.v1.SchemaApi; import org.apache.beam.runners.core.construction.Environments; +import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator; import org.apache.beam.runners.core.construction.PipelineTranslation; import org.apache.beam.runners.core.construction.RehydratedComponents; import org.apache.beam.runners.core.construction.SdkComponents; import org.apache.beam.runners.core.construction.SplittableParDo; +import org.apache.beam.runners.core.construction.TransformPayloadTranslatorRegistrar; import org.apache.beam.runners.fnexecution.artifact.ArtifactRetrievalService; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.PipelineResult; @@ -84,13 +88,15 @@ import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Server; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ServerBuilder; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.CaseFormat; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Converter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CaseFormat; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Converter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; @@ -178,6 +184,65 @@ public List getDependencies( } } + List deprecatedTransformURNs = ImmutableList.of(READ_TRANSFORM_URN); + for (TransformPayloadTranslatorRegistrar registrar : + ServiceLoader.load(TransformPayloadTranslatorRegistrar.class)) { + for (Map.Entry, ? extends TransformPayloadTranslator> + entry : registrar.getTransformPayloadTranslators().entrySet()) { + @Initialized TransformPayloadTranslator translator = entry.getValue(); + if (translator == null) { + continue; + } + + String urn; + try { + urn = translator.getUrn(); + if (urn == null) { + LOG.debug( + "Could not load the TransformPayloadTranslator " + + translator + + " to the Expansion Service since it did not produce a unique URN."); + continue; + } + } catch (Exception e) { + LOG.info( + "Could not load the TransformPayloadTranslator " + + translator + + " to the Expansion Service."); + continue; + } + + if (deprecatedTransformURNs.contains(urn)) { + continue; + } + final String finalUrn = urn; + TransformProvider transformProvider = + spec -> { + try { + ExternalConfigurationPayload payload = + ExternalConfigurationPayload.parseFrom(spec.getPayload()); + Row configRow = + RowCoder.of(SchemaTranslation.schemaFromProto(payload.getSchema())) + .decode(new ByteArrayInputStream(payload.getPayload().toByteArray())); + PTransform transformFromRow = translator.fromConfigRow(configRow); + if (transformFromRow != null) { + return transformFromRow; + } else { + throw new RuntimeException( + String.format( + "A transform cannot be initiated using the provided config row %s and the TransformPayloadTranslator %s", + configRow, translator)); + } + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to build transform %s from spec %s", finalUrn, spec), + e); + } + }; + builder.put(finalUrn, transformProvider); + } + } + return builder.build(); } diff --git a/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionServiceSchemaTransformProvider.java b/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionServiceSchemaTransformProvider.java index f730066021e5a..ead1fa67dc98d 100644 --- a/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionServiceSchemaTransformProvider.java +++ b/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionServiceSchemaTransformProvider.java @@ -36,7 +36,7 @@ import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; @SuppressWarnings({"rawtypes"}) @@ -129,13 +129,12 @@ public PTransform getTransform(FunctionSpec spec) { Row configRow; try { configRow = - RowCoder.of(provider.configurationSchema()) - .decode(payload.getConfigurationRow().newInput()); + RowCoder.of(configSchemaFromRequest).decode(payload.getConfigurationRow().newInput()); } catch (IOException e) { throw new RuntimeException("Error decoding payload", e); } - return provider.from(configRow).buildTransform(); + return provider.from(configRow); } Iterable getAllProviders() { diff --git a/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExpansionServiceSchemaTransformProviderTest.java b/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExpansionServiceSchemaTransformProviderTest.java index 18342fb19a98c..d7a665eabe0f9 100644 --- a/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExpansionServiceSchemaTransformProviderTest.java +++ b/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExpansionServiceSchemaTransformProviderTest.java @@ -19,9 +19,9 @@ import static org.apache.beam.runners.core.construction.BeamUrns.getUrn; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import com.google.auto.service.AutoService; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import org.apache.beam.model.expansion.v1.ExpansionApi; @@ -32,6 +32,7 @@ import org.apache.beam.runners.core.construction.ParDoTranslation; import org.apache.beam.runners.core.construction.PipelineTranslation; import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.coders.CoderException; import org.apache.beam.sdk.schemas.JavaFieldSchema; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.Schema.Field; @@ -47,15 +48,15 @@ import org.apache.beam.sdk.transforms.Impulse; import org.apache.beam.sdk.transforms.InferableFunction; import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.util.ByteStringOutputStream; +import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; +import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Test; /** Tests for {@link ExpansionServiceSchemaTransformProvider}. */ @@ -75,6 +76,13 @@ public class ExpansionServiceSchemaTransformProviderTest { Field.of("str1", FieldType.STRING), Field.of("str2", FieldType.STRING)); + private static final Schema TEST_SCHEMATRANSFORM_EQUIVALENT_CONFIG_SCHEMA = + Schema.of( + Field.of("str2", FieldType.STRING), + Field.of("str1", FieldType.STRING), + Field.of("int2", FieldType.INT32), + Field.of("int1", FieldType.INT32)); + private ExpansionService expansionService = new ExpansionService(); @DefaultSchema(JavaFieldSchema.class) @@ -126,26 +134,6 @@ public List outputCollectionNames() { } } - public static class TestSchemaTransform implements SchemaTransform { - - private String str1; - private String str2; - private Integer int1; - private Integer int2; - - public TestSchemaTransform(String str1, String str2, Integer int1, Integer int2) { - this.str1 = str1; - this.str2 = str2; - this.int1 = int1; - this.int2 = int2; - } - - @Override - public PTransform buildTransform() { - return new TestTransform(str1, str2, int1, int2); - } - } - public static class TestDoFn extends DoFn { public String str1; @@ -166,14 +154,14 @@ public void processElement(@Element String element, OutputReceiver recei } } - public static class TestTransform extends PTransform { + public static class TestSchemaTransform extends SchemaTransform { private String str1; private String str2; private Integer int1; private Integer int2; - public TestTransform(String str1, String str2, Integer int1, Integer int2) { + public TestSchemaTransform(String str1, String str2, Integer int1, Integer int2) { this.str1 = str1; this.str2 = str2; this.int1 = int1; @@ -244,7 +232,7 @@ public List outputCollectionNames() { } } - public static class TestSchemaTransformMultiInputOutput implements SchemaTransform { + public static class TestSchemaTransformMultiInputOutput extends SchemaTransform { private String str1; private String str2; @@ -259,28 +247,6 @@ public TestSchemaTransformMultiInputOutput( this.int2 = int2; } - @Override - public PTransform buildTransform() { - return new TestTransformMultiInputMultiOutput(str1, str2, int1, int2); - } - } - - public static class TestTransformMultiInputMultiOutput - extends PTransform { - - private String str1; - private String str2; - private Integer int1; - private Integer int2; - - public TestTransformMultiInputMultiOutput( - String str1, String str2, Integer int1, Integer int2) { - this.str1 = str1; - this.str2 = str2; - this.int1 = int1; - this.int2 = int2; - } - @Override public PCollectionRowTuple expand(PCollectionRowTuple input) { PCollection outputPC1 = @@ -387,31 +353,13 @@ public void testSchemaTransformExpansion() { .withFieldValue("str2", "bbb") .build(); - ByteStringOutputStream outputStream = new ByteStringOutputStream(); - try { - SchemaCoder.of(configRow.getSchema()).encode(configRow, outputStream); - } catch (IOException e) { - throw new RuntimeException(e); - } - - ExternalTransforms.SchemaTransformPayload payload = - ExternalTransforms.SchemaTransformPayload.newBuilder() - .setIdentifier("dummy_id") - .setConfigurationRow(outputStream.toByteString()) - .setConfigurationSchema( - SchemaTranslation.schemaToProto(TEST_SCHEMATRANSFORM_CONFIG_SCHEMA, true)) - .build(); - ExpansionApi.ExpansionRequest request = ExpansionApi.ExpansionRequest.newBuilder() .setComponents(pipelineProto.getComponents()) .setTransform( RunnerApi.PTransform.newBuilder() .setUniqueName(TEST_NAME) - .setSpec( - RunnerApi.FunctionSpec.newBuilder() - .setUrn(getUrn(ExpansionMethods.Enum.SCHEMA_TRANSFORM)) - .setPayload(payload.toByteString())) + .setSpec(createSpec("dummy_id", configRow)) .putInputs("input1", inputPcollId)) .setNamespace(TEST_NAMESPACE) .build(); @@ -446,35 +394,18 @@ public void testSchemaTransformExpansionMultiInputMultiOutput() { .withFieldValue("str2", "bbb") .build(); - ByteStringOutputStream outputStream = new ByteStringOutputStream(); - try { - SchemaCoder.of(configRow.getSchema()).encode(configRow, outputStream); - } catch (IOException e) { - throw new RuntimeException(e); - } - - ExternalTransforms.SchemaTransformPayload payload = - ExternalTransforms.SchemaTransformPayload.newBuilder() - .setIdentifier("dummy_id_multi_input_multi_output") - .setConfigurationRow(outputStream.toByteString()) - .setConfigurationSchema( - SchemaTranslation.schemaToProto(TEST_SCHEMATRANSFORM_CONFIG_SCHEMA, true)) - .build(); - ExpansionApi.ExpansionRequest request = ExpansionApi.ExpansionRequest.newBuilder() .setComponents(pipelineProto.getComponents()) .setTransform( RunnerApi.PTransform.newBuilder() .setUniqueName(TEST_NAME) - .setSpec( - RunnerApi.FunctionSpec.newBuilder() - .setUrn(getUrn(ExpansionMethods.Enum.SCHEMA_TRANSFORM)) - .setPayload(payload.toByteString())) + .setSpec(createSpec("dummy_id_multi_input_multi_output", configRow)) .putInputs("input1", inputPcollIds.get(0)) .putInputs("input2", inputPcollIds.get(1))) .setNamespace(TEST_NAMESPACE) .build(); + ExpansionApi.ExpansionResponse response = expansionService.expand(request); RunnerApi.PTransform expandedTransform = response.getTransform(); @@ -483,4 +414,61 @@ public void testSchemaTransformExpansionMultiInputMultiOutput() { assertEquals(2, expandedTransform.getOutputsCount()); verifyLeafTransforms(response, 2); } + + @Test + public void testSchematransformEquivalentConfigSchema() throws CoderException { + Row configRow = + Row.withSchema(TEST_SCHEMATRANSFORM_CONFIG_SCHEMA) + .withFieldValue("int1", 111) + .withFieldValue("int2", 222) + .withFieldValue("str1", "aaa") + .withFieldValue("str2", "bbb") + .build(); + + RunnerApi.FunctionSpec spec = createSpec("dummy_id", configRow); + + Row equivalentConfigRow = + Row.withSchema(TEST_SCHEMATRANSFORM_EQUIVALENT_CONFIG_SCHEMA) + .withFieldValue("int1", 111) + .withFieldValue("int2", 222) + .withFieldValue("str1", "aaa") + .withFieldValue("str2", "bbb") + .build(); + + RunnerApi.FunctionSpec equivalentSpec = createSpec("dummy_id", equivalentConfigRow); + + assertNotEquals(spec.getPayload(), equivalentSpec.getPayload()); + + TestSchemaTransform transform = + (TestSchemaTransform) ExpansionServiceSchemaTransformProvider.of().getTransform(spec); + TestSchemaTransform equivalentTransform = + (TestSchemaTransform) + ExpansionServiceSchemaTransformProvider.of().getTransform(equivalentSpec); + + assertEquals(transform.int1, equivalentTransform.int1); + assertEquals(transform.int2, equivalentTransform.int2); + assertEquals(transform.str1, equivalentTransform.str1); + assertEquals(transform.str2, equivalentTransform.str2); + } + + private RunnerApi.FunctionSpec createSpec(String identifier, Row configRow) { + byte[] encodedRow; + try { + encodedRow = CoderUtils.encodeToByteArray(SchemaCoder.of(configRow.getSchema()), configRow); + } catch (CoderException e) { + throw new RuntimeException(e); + } + + ExternalTransforms.SchemaTransformPayload payload = + ExternalTransforms.SchemaTransformPayload.newBuilder() + .setIdentifier(identifier) + .setConfigurationRow(ByteString.copyFrom(encodedRow)) + .setConfigurationSchema(SchemaTranslation.schemaToProto(configRow.getSchema(), true)) + .build(); + + return RunnerApi.FunctionSpec.newBuilder() + .setUrn(getUrn(ExpansionMethods.Enum.SCHEMA_TRANSFORM)) + .setPayload(payload.toByteString()) + .build(); + } } diff --git a/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExpansionServiceTest.java b/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExpansionServiceTest.java index 39076d0226891..618fa3333092d 100644 --- a/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExpansionServiceTest.java +++ b/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExpansionServiceTest.java @@ -55,10 +55,10 @@ import org.apache.beam.sdk.transforms.Impulse; import org.apache.beam.sdk.util.ByteStringOutputStream; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matchers; import org.junit.Test; diff --git a/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExternalTest.java b/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExternalTest.java index 944746035c985..4b949a597f0a3 100644 --- a/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExternalTest.java +++ b/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExternalTest.java @@ -45,7 +45,7 @@ import org.apache.beam.sdk.values.TypeDescriptors; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Server; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ServerBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.junit.After; diff --git a/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/JavaClassLookupTransformProviderTest.java b/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/JavaClassLookupTransformProviderTest.java index 89ad360010b66..9179f1144a2fa 100644 --- a/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/JavaClassLookupTransformProviderTest.java +++ b/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/JavaClassLookupTransformProviderTest.java @@ -65,9 +65,9 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Resources; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Resources; import org.hamcrest.Matchers; import org.junit.BeforeClass; import org.junit.Test; diff --git a/sdks/java/extensions/arrow/build.gradle b/sdks/java/extensions/arrow/build.gradle index 8e31fe2b57d14..6ce5bff965633 100644 --- a/sdks/java/extensions/arrow/build.gradle +++ b/sdks/java/extensions/arrow/build.gradle @@ -22,7 +22,7 @@ applyJavaNature( automaticModuleName: 'org.apache.beam.sdk.extensions.arrow') description = "Apache Beam :: SDKs :: Java :: Extensions :: Arrow" dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.arrow_vector implementation library.java.arrow_memory_core diff --git a/sdks/java/extensions/arrow/src/main/java/org/apache/beam/sdk/extensions/arrow/ArrowConversion.java b/sdks/java/extensions/arrow/src/main/java/org/apache/beam/sdk/extensions/arrow/ArrowConversion.java index 19583ae44b65e..88e6fefcf9d3f 100644 --- a/sdks/java/extensions/arrow/src/main/java/org/apache/beam/sdk/extensions/arrow/ArrowConversion.java +++ b/sdks/java/extensions/arrow/src/main/java/org/apache/beam/sdk/extensions/arrow/ArrowConversion.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.arrow; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.io.InputStream; diff --git a/sdks/java/extensions/arrow/src/test/java/org/apache/beam/sdk/extensions/arrow/ArrowConversionTest.java b/sdks/java/extensions/arrow/src/test/java/org/apache/beam/sdk/extensions/arrow/ArrowConversionTest.java index c5cc71f69cfaa..3af72ef5579c6 100644 --- a/sdks/java/extensions/arrow/src/test/java/org/apache/beam/sdk/extensions/arrow/ArrowConversionTest.java +++ b/sdks/java/extensions/arrow/src/test/java/org/apache/beam/sdk/extensions/arrow/ArrowConversionTest.java @@ -41,7 +41,7 @@ import org.apache.beam.sdk.schemas.Schema.Field; import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.collection.IsIterableContainingInOrder; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; diff --git a/sdks/java/extensions/avro/build.gradle b/sdks/java/extensions/avro/build.gradle index e839798309002..c5e9c285996dc 100644 --- a/sdks/java/extensions/avro/build.gradle +++ b/sdks/java/extensions/avro/build.gradle @@ -58,7 +58,7 @@ test { dependencies { implementation library.java.byte_buddy - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation(project(path: ":sdks:java:core", configuration: "shadow")) { // Exclude Avro dependencies from "core" since Avro support moved to this extension exclude group: "org.apache.avro", module: "avro" diff --git a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/coders/AvroCoder.java b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/coders/AvroCoder.java index b48446477e782..621076e0e1494 100644 --- a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/coders/AvroCoder.java +++ b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/coders/AvroCoder.java @@ -63,7 +63,7 @@ import org.apache.beam.sdk.extensions.avro.io.AvroDatumFactory; import org.apache.beam.sdk.util.EmptyOnDeserializationThreadLocal; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroDatumFactory.java b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroDatumFactory.java index 55c6e266e2797..67125a6ad24da 100644 --- a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroDatumFactory.java +++ b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroDatumFactory.java @@ -24,6 +24,7 @@ import org.apache.avro.generic.GenericRecord; import org.apache.avro.io.DatumReader; import org.apache.avro.io.DatumWriter; +import org.apache.avro.reflect.ReflectData; import org.apache.avro.reflect.ReflectDatumReader; import org.apache.avro.reflect.ReflectDatumWriter; import org.apache.avro.specific.SpecificDatumReader; @@ -33,6 +34,9 @@ import org.checkerframework.checker.nullness.qual.Nullable; /** Create {@link DatumReader} and {@link DatumWriter} for given schemas. */ +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) public abstract class AvroDatumFactory implements AvroSource.DatumReaderFactory, AvroSink.DatumWriterFactory { @@ -168,25 +172,16 @@ public ReflectDatumFactory(Class type) { @Override public DatumReader apply(Schema writer, Schema reader) { - // create the datum writer using the Class api. - // avro will load the proper class loader - ReflectDatumReader datumReader = new ReflectDatumReader<>(type); - datumReader.setExpected(reader); - datumReader.setSchema(writer); - // for backward compat, add logical type support by default - AvroUtils.addLogicalTypeConversions(datumReader.getData()); - return datumReader; + ReflectData data = new ReflectData(type.getClassLoader()); + AvroUtils.addLogicalTypeConversions(data); + return new ReflectDatumReader<>(writer, reader, data); } @Override public DatumWriter apply(Schema writer) { - // create the datum writer using the Class api. - // avro will load the proper class loader - ReflectDatumWriter datumWriter = new ReflectDatumWriter<>(type); - datumWriter.setSchema(writer); - // for backward compat, add logical type support by default - AvroUtils.addLogicalTypeConversions(datumWriter.getData()); - return datumWriter; + ReflectData data = new ReflectData(type.getClassLoader()); + AvroUtils.addLogicalTypeConversions(data); + return new ReflectDatumWriter<>(writer, data); } public static ReflectDatumFactory of(Class type) { diff --git a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java index f0997c57f38de..a65db5a90bad1 100644 --- a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java +++ b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java @@ -19,8 +19,8 @@ import static org.apache.beam.sdk.io.FileIO.ReadMatches.DirectoryTreatment; import static org.apache.beam.sdk.io.ReadAllViaFileBasedSource.ReadFileRangesFnExceptionHandler; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import java.io.IOException; @@ -73,12 +73,12 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Suppliers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Suppliers; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; diff --git a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroSource.java b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroSource.java index 90b40478fc98e..c6c7fa426dbf0 100644 --- a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroSource.java +++ b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroSource.java @@ -18,9 +18,9 @@ package org.apache.beam.sdk.extensions.avro.io; import static org.apache.beam.sdk.io.FileBasedSource.Mode.SINGLE_FILE_OR_SUBRANGE; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.io.InputStream; @@ -64,7 +64,7 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.util.VarInt; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; // CHECKSTYLE.OFF: JavadocStyle diff --git a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/ConstantAvroDestination.java b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/ConstantAvroDestination.java index 601c65935bec4..5b3683c349c97 100644 --- a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/ConstantAvroDestination.java +++ b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/ConstantAvroDestination.java @@ -25,10 +25,10 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.transforms.display.HasDisplayData; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Suppliers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Suppliers; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; import org.checkerframework.checker.nullness.qual.Nullable; /** Always returns a constant {@link FilenamePolicy}, {@link Schema}, metadata, and codec. */ diff --git a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/DynamicAvroDestinations.java b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/DynamicAvroDestinations.java index c74c98ed72712..6da2c49157d9b 100644 --- a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/DynamicAvroDestinations.java +++ b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/DynamicAvroDestinations.java @@ -21,7 +21,7 @@ import org.apache.avro.Schema; import org.apache.avro.file.CodecFactory; import org.apache.beam.sdk.io.FileBasedSink.DynamicDestinations; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/SerializableAvroCodecFactory.java b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/SerializableAvroCodecFactory.java index 8a82ffcbcd422..2dc247969da62 100644 --- a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/SerializableAvroCodecFactory.java +++ b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/SerializableAvroCodecFactory.java @@ -22,8 +22,8 @@ import static org.apache.avro.file.DataFileConstants.NULL_CODEC; import static org.apache.avro.file.DataFileConstants.SNAPPY_CODEC; import static org.apache.avro.file.DataFileConstants.XZ_CODEC; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.Externalizable; import java.io.IOException; diff --git a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroByteBuddyUtils.java b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroByteBuddyUtils.java index 6ac6dd6010d59..e07f6ffb46812 100644 --- a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroByteBuddyUtils.java +++ b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroByteBuddyUtils.java @@ -43,7 +43,7 @@ import org.apache.beam.sdk.schemas.utils.ReflectUtils.ClassWithSchema; import org.apache.beam.sdk.util.common.ReflectHelpers; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; @SuppressWarnings({ "nullness", // TODO(https://github.com/apache/beam/issues/20497) diff --git a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroSchemaInformationProvider.java b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroSchemaInformationProvider.java new file mode 100644 index 0000000000000..ce426abe69c0d --- /dev/null +++ b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroSchemaInformationProvider.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.avro.schemas.utils; + +import com.google.auto.service.AutoService; +import javax.annotation.Nullable; +import org.apache.avro.generic.GenericRecord; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.schemas.SchemaCoder; +import org.apache.beam.sdk.schemas.utils.ConvertHelpers; +import org.apache.beam.sdk.schemas.utils.SchemaInformationProvider; +import org.apache.beam.sdk.values.TypeDescriptor; + +@AutoService(SchemaInformationProvider.class) +public class AvroSchemaInformationProvider implements SchemaInformationProvider { + + @Override + @Nullable + public ConvertHelpers.ConvertedSchemaInformation getConvertedSchemaInformation( + Schema inputSchema, TypeDescriptor outputType) { + if (outputType.equals(TypeDescriptor.of(GenericRecord.class))) { + return new ConvertHelpers.ConvertedSchemaInformation( + (SchemaCoder) AvroUtils.schemaCoder(AvroUtils.toAvroSchema(inputSchema)), null); + } + return null; + } +} diff --git a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtils.java b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtils.java index 1f4fca5e831cb..50be0e6b3414b 100644 --- a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtils.java +++ b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtils.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.extensions.avro.schemas.utils; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -93,11 +93,11 @@ import org.apache.beam.sdk.transforms.SimpleFunction; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.CaseFormat; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CaseFormat; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Days; import org.joda.time.Duration; diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/coders/AvroCoderTest.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/coders/AvroCoderTest.java index d69983974f91e..a76fb59c7d10c 100644 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/coders/AvroCoderTest.java +++ b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/coders/AvroCoderTest.java @@ -85,8 +85,8 @@ import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Description; import org.hamcrest.Matcher; diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/coders/AvroCoderTestPojo.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/coders/AvroCoderTestPojo.java index 9d1700313dfaa..b981dadd0aaa7 100644 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/coders/AvroCoderTestPojo.java +++ b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/coders/AvroCoderTestPojo.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.extensions.avro.coders; import java.util.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; /** A Pojo at the top level for use in tests. */ diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/coders/DefaultCoderTest.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/coders/DefaultCoderTest.java deleted file mode 100644 index 82991f191887f..0000000000000 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/coders/DefaultCoderTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.extensions.avro.coders; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.instanceOf; - -import java.util.List; -import org.apache.beam.sdk.coders.Coder; -import org.apache.beam.sdk.coders.CoderRegistry; -import org.apache.beam.sdk.coders.DefaultCoder; -import org.apache.beam.sdk.coders.DefaultCoder.DefaultCoderProviderRegistrar.DefaultCoderProvider; -import org.apache.beam.sdk.coders.ListCoder; -import org.apache.beam.sdk.values.TypeDescriptor; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Tests for {@link DefaultCoder}. */ -@RunWith(JUnit4.class) -public class DefaultCoderTest { - - @Rule public ExpectedException thrown = ExpectedException.none(); - - @DefaultCoder(AvroCoder.class) - private static class AvroRecord {} - - @Test - public void testCodersWithoutComponents() throws Exception { - CoderRegistry registry = CoderRegistry.createDefault(); - registry.registerCoderProvider(new DefaultCoderProvider()); - assertThat(registry.getCoder(AvroRecord.class), instanceOf(AvroCoder.class)); - } - - @Test - public void testDefaultCoderInCollection() throws Exception { - CoderRegistry registry = CoderRegistry.createDefault(); - registry.registerCoderProvider(new DefaultCoderProvider()); - Coder> avroRecordCoder = - registry.getCoder(new TypeDescriptor>() {}); - assertThat(avroRecordCoder, instanceOf(ListCoder.class)); - assertThat(((ListCoder) avroRecordCoder).getElemCoder(), instanceOf(AvroCoder.class)); - } -} diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/AvroIOTest.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/AvroIOTest.java index 243386bfd3ed7..30a1a77872520 100644 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/AvroIOTest.java +++ b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/AvroIOTest.java @@ -23,7 +23,7 @@ import static org.apache.beam.sdk.transforms.Contextful.fn; import static org.apache.beam.sdk.transforms.Requirements.requiresSideInputs; import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertArrayEquals; @@ -100,16 +100,16 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/AvroSourceTest.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/AvroSourceTest.java index 34d3bedae4b0d..f4841f1cdc02f 100644 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/AvroSourceTest.java +++ b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/io/AvroSourceTest.java @@ -61,7 +61,7 @@ import org.apache.beam.sdk.testing.SourceTestUtils; import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.util.SerializableUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matchers; import org.junit.Rule; diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/AvroSchemaTest.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/AvroSchemaTest.java index 9dfdbd499e1ed..1acf77e415492 100644 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/AvroSchemaTest.java +++ b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/AvroSchemaTest.java @@ -48,8 +48,8 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/SchemaCoderTest.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/SchemaCoderTest.java index aa026d4418012..0cda00a6e86a5 100644 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/SchemaCoderTest.java +++ b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/SchemaCoderTest.java @@ -38,7 +38,7 @@ import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matchers; import org.joda.time.DateTime; diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/io/AvroPayloadSerializerProviderTest.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/io/AvroPayloadSerializerProviderTest.java index 9c56ffcdc0842..28e68afe47e87 100644 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/io/AvroPayloadSerializerProviderTest.java +++ b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/io/AvroPayloadSerializerProviderTest.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/transforms/ConvertTest.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/transforms/ConvertTest.java index a33fa013be17a..1cc746258a1c0 100644 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/transforms/ConvertTest.java +++ b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/transforms/ConvertTest.java @@ -34,8 +34,8 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroGenerators.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroGenerators.java index fa7d7cceecce1..3fc6d66829db9 100644 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroGenerators.java +++ b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroGenerators.java @@ -29,9 +29,9 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.avro.Schema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ObjectArrays; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ObjectArrays; /** QuickCheck generators for AVRO. */ class AvroGenerators { diff --git a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtilsTest.java b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtilsTest.java index 77577d54fb784..85781c4b8d0e7 100644 --- a/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtilsTest.java +++ b/sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtilsTest.java @@ -53,10 +53,10 @@ import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTime; diff --git a/sdks/java/extensions/euphoria/build.gradle b/sdks/java/extensions/euphoria/build.gradle index 3415715836362..3f41963638ba2 100644 --- a/sdks/java/extensions/euphoria/build.gradle +++ b/sdks/java/extensions/euphoria/build.gradle @@ -29,7 +29,7 @@ dependencies { implementation library.java.jackson_annotations implementation library.java.joda_time implementation library.java.slf4j_api - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre testImplementation project(":sdks:java:extensions:kryo") testImplementation library.java.slf4j_api testImplementation library.java.hamcrest diff --git a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/Distinct.java b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/Distinct.java index 3de5479cd388e..ca42a7e8330ca 100644 --- a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/Distinct.java +++ b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/Distinct.java @@ -44,7 +44,7 @@ import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; diff --git a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/FlatMap.java b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/FlatMap.java index 452a991e246b7..dd92854d616c8 100644 --- a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/FlatMap.java +++ b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/FlatMap.java @@ -34,7 +34,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; diff --git a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/ReduceWindow.java b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/ReduceWindow.java index a38d7f3f95129..0dfc4d9c9db3f 100644 --- a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/ReduceWindow.java +++ b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/ReduceWindow.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.extensions.euphoria.core.client.operator; import static java.util.Objects.requireNonNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Optional; import java.util.stream.Stream; diff --git a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/Union.java b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/Union.java index a077d0247327c..def5d422ff512 100644 --- a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/Union.java +++ b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/Union.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.euphoria.core.client.operator; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Arrays; import java.util.List; diff --git a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/WindowBuilder.java b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/WindowBuilder.java index 93c49fc3fc7e2..fe8e10018776a 100644 --- a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/WindowBuilder.java +++ b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/WindowBuilder.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.extensions.euphoria.core.client.operator; import static java.util.Objects.requireNonNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Optional; import org.apache.beam.sdk.extensions.euphoria.core.client.operator.base.Builders; diff --git a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/util/PCollectionLists.java b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/util/PCollectionLists.java index 8666f0160b04c..fb28a89bfc758 100644 --- a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/util/PCollectionLists.java +++ b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/client/util/PCollectionLists.java @@ -19,7 +19,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** Utilities related to {@link PCollection}s. */ public class PCollectionLists { diff --git a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/AbstractJoinTranslator.java b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/AbstractJoinTranslator.java index b1c04f1a035e9..607cad2330304 100644 --- a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/AbstractJoinTranslator.java +++ b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/AbstractJoinTranslator.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.euphoria.core.translate; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import org.apache.beam.sdk.extensions.euphoria.core.client.operator.Join; import org.apache.beam.sdk.extensions.euphoria.core.client.type.TypeAwareness; diff --git a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/BroadcastHashJoinTranslator.java b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/BroadcastHashJoinTranslator.java index 66dfab4e1bdf1..724f28d692346 100644 --- a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/BroadcastHashJoinTranslator.java +++ b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/BroadcastHashJoinTranslator.java @@ -30,9 +30,9 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashBasedTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Table; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBasedTable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Table; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/CompositeOperatorTranslator.java b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/CompositeOperatorTranslator.java index 53b425659642d..e1b26eb7ebbca 100644 --- a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/CompositeOperatorTranslator.java +++ b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/CompositeOperatorTranslator.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.euphoria.core.translate; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import org.apache.beam.sdk.extensions.euphoria.core.client.operator.CompositeOperator; import org.apache.beam.sdk.extensions.euphoria.core.client.operator.base.Operator; diff --git a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/OperatorTransform.java b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/OperatorTransform.java index c1199e3a48894..d2bd5ebde27d9 100644 --- a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/OperatorTransform.java +++ b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/OperatorTransform.java @@ -22,7 +22,7 @@ import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; /** * Expand operator to a beam {@link PTransform}. diff --git a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/ReduceByKeyTranslator.java b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/ReduceByKeyTranslator.java index 3359fc566832b..1ea61bad8ff91 100644 --- a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/ReduceByKeyTranslator.java +++ b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/ReduceByKeyTranslator.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.extensions.euphoria.core.translate; import static java.util.Objects.requireNonNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.stream.StreamSupport; import org.apache.beam.sdk.coders.CannotProvideCoderException; diff --git a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/provider/GenericTranslatorProvider.java b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/provider/GenericTranslatorProvider.java index a46c4f42d46bd..b30473b3c9ecf 100644 --- a/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/provider/GenericTranslatorProvider.java +++ b/sdks/java/extensions/euphoria/src/main/java/org/apache/beam/sdk/extensions/euphoria/core/translate/provider/GenericTranslatorProvider.java @@ -36,7 +36,7 @@ import org.apache.beam.sdk.extensions.euphoria.core.translate.ReduceByKeyTranslator; import org.apache.beam.sdk.extensions.euphoria.core.translate.TranslatorProvider; import org.apache.beam.sdk.extensions.euphoria.core.translate.UnionTranslator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; /** * Adjustable {@link TranslatorProvider} that selects first suitable translation for the registered diff --git a/sdks/java/extensions/euphoria/src/test/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/ReduceByKeyTest.java b/sdks/java/extensions/euphoria/src/test/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/ReduceByKeyTest.java index 86494a055bab1..2a472b2ce39c0 100644 --- a/sdks/java/extensions/euphoria/src/test/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/ReduceByKeyTest.java +++ b/sdks/java/extensions/euphoria/src/test/java/org/apache/beam/sdk/extensions/euphoria/core/client/operator/ReduceByKeyTest.java @@ -38,8 +38,8 @@ import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/extensions/euphoria/src/test/java/org/apache/beam/sdk/extensions/euphoria/core/docs/DocumentationExamplesTest.java b/sdks/java/extensions/euphoria/src/test/java/org/apache/beam/sdk/extensions/euphoria/core/docs/DocumentationExamplesTest.java index 3de9942450b10..053a3a045af32 100644 --- a/sdks/java/extensions/euphoria/src/test/java/org/apache/beam/sdk/extensions/euphoria/core/docs/DocumentationExamplesTest.java +++ b/sdks/java/extensions/euphoria/src/test/java/org/apache/beam/sdk/extensions/euphoria/core/docs/DocumentationExamplesTest.java @@ -69,7 +69,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; import org.joda.time.Duration; import org.junit.Assert; import org.junit.Before; diff --git a/sdks/java/extensions/euphoria/src/test/java/org/apache/beam/sdk/extensions/euphoria/core/testkit/ReduceByKeyTest.java b/sdks/java/extensions/euphoria/src/test/java/org/apache/beam/sdk/extensions/euphoria/core/testkit/ReduceByKeyTest.java index d9a014319b421..94b46b61fad56 100644 --- a/sdks/java/extensions/euphoria/src/test/java/org/apache/beam/sdk/extensions/euphoria/core/testkit/ReduceByKeyTest.java +++ b/sdks/java/extensions/euphoria/src/test/java/org/apache/beam/sdk/extensions/euphoria/core/testkit/ReduceByKeyTest.java @@ -50,9 +50,9 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/extensions/euphoria/src/test/java/org/apache/beam/sdk/extensions/euphoria/core/testkit/accumulators/NanosecondTimer.java b/sdks/java/extensions/euphoria/src/test/java/org/apache/beam/sdk/extensions/euphoria/core/testkit/accumulators/NanosecondTimer.java index 7d7c6661c5bd4..3d6faf49092a5 100644 --- a/sdks/java/extensions/euphoria/src/test/java/org/apache/beam/sdk/extensions/euphoria/core/testkit/accumulators/NanosecondTimer.java +++ b/sdks/java/extensions/euphoria/src/test/java/org/apache/beam/sdk/extensions/euphoria/core/testkit/accumulators/NanosecondTimer.java @@ -21,7 +21,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import org.apache.beam.sdk.extensions.euphoria.core.client.accumulators.Timer; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; final class NanosecondTimer implements Timer, Snapshotable> { diff --git a/sdks/java/extensions/google-cloud-platform-core/OWNERS b/sdks/java/extensions/google-cloud-platform-core/OWNERS deleted file mode 100644 index 7f25f229e967d..0000000000000 --- a/sdks/java/extensions/google-cloud-platform-core/OWNERS +++ /dev/null @@ -1,8 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lukecwik - - chamikaramj - - pabloem - - ihji - - johnjcasey diff --git a/sdks/java/extensions/google-cloud-platform-core/build.gradle b/sdks/java/extensions/google-cloud-platform-core/build.gradle index be269cc01a985..542c641c855a1 100644 --- a/sdks/java/extensions/google-cloud-platform-core/build.gradle +++ b/sdks/java/extensions/google-cloud-platform-core/build.gradle @@ -36,7 +36,7 @@ test { dependencies { implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(path: ":runners:core-java") implementation library.java.google_http_client_jackson2 diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/auth/GcpCredentialFactory.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/auth/GcpCredentialFactory.java index 56db598c8ca28..22e1f874367cf 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/auth/GcpCredentialFactory.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/auth/GcpCredentialFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.gcp.auth; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auth.Credentials; import com.google.auth.oauth2.GoogleCredentials; diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java index 7bb929ecd9558..9137b5594ec57 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.extensions.gcp.options; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings.isNullOrEmpty; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings.isNullOrEmpty; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.api.client.http.HttpRequestInitializer; @@ -58,9 +58,9 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.InstanceBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.slf4j.Logger; diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpPipelineOptionsRegistrar.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpPipelineOptionsRegistrar.java index 17d2d786bd9f7..9ae46d3d5ad37 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpPipelineOptionsRegistrar.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpPipelineOptionsRegistrar.java @@ -20,7 +20,7 @@ import com.google.auto.service.AutoService; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** A registrar containing the default GCP options. */ @AutoService(PipelineOptionsRegistrar.class) diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystem.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystem.java index ff42f7f2d17b7..b49c434f81c60 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystem.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystem.java @@ -18,10 +18,10 @@ package org.apache.beam.sdk.extensions.gcp.storage; import static org.apache.beam.sdk.io.FileSystemUtils.wildcardToRegexp; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.client.util.DateTime; import com.google.api.services.storage.model.Objects; @@ -49,11 +49,11 @@ import org.apache.beam.sdk.io.fs.MoveOptions; import org.apache.beam.sdk.metrics.Counter; import org.apache.beam.sdk.metrics.Metrics; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Stopwatch; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Stopwatch; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrar.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrar.java index 3ebc6a3e714c1..541998b7a6cc2 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrar.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrar.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.gcp.storage; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.service.AutoService; import javax.annotation.Nonnull; @@ -25,7 +25,7 @@ import org.apache.beam.sdk.io.FileSystem; import org.apache.beam.sdk.io.FileSystemRegistrar; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** {@link AutoService} registrar for the {@link GcsFileSystem}. */ @AutoService(FileSystemRegistrar.class) diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsPathValidator.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsPathValidator.java index df9cc8e0a65e8..8d1939d9c0684 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsPathValidator.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsPathValidator.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.gcp.storage; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsResourceId.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsResourceId.java index 076c8f3fbf5ec..8458227746732 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsResourceId.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/storage/GcsResourceId.java @@ -17,9 +17,9 @@ */ package org.apache.beam.sdk.extensions.gcp.storage; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import org.apache.beam.sdk.extensions.gcp.util.gcsfs.GcsPath; import org.apache.beam.sdk.io.fs.ResolveOptions; diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/GceMetadataUtil.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/GceMetadataUtil.java index 7113102172445..fd49b759fd6df 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/GceMetadataUtil.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/GceMetadataUtil.java @@ -22,7 +22,7 @@ import java.io.InputStreamReader; import java.io.Reader; import java.nio.charset.StandardCharsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CharStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CharStreams; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; @@ -30,40 +30,60 @@ import org.apache.http.params.BasicHttpParams; import org.apache.http.params.HttpConnectionParams; import org.apache.http.params.HttpParams; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** */ public class GceMetadataUtil { private static final String BASE_METADATA_URL = "http://metadata/computeMetadata/v1/"; + private static final Logger LOG = LoggerFactory.getLogger(GceMetadataUtil.class); + static String fetchMetadata(String key) { + String requestUrl = BASE_METADATA_URL + key; int timeoutMillis = 5000; final HttpParams httpParams = new BasicHttpParams(); HttpConnectionParams.setConnectionTimeout(httpParams, timeoutMillis); - HttpClient client = new DefaultHttpClient(httpParams); - HttpGet request = new HttpGet(BASE_METADATA_URL + key); - request.setHeader("Metadata-Flavor", "Google"); - + String ret = ""; try { + HttpClient client = new DefaultHttpClient(httpParams); + + HttpGet request = new HttpGet(requestUrl); + request.setHeader("Metadata-Flavor", "Google"); + HttpResponse response = client.execute(request); - if (response.getStatusLine().getStatusCode() != 200) { - // May mean its running on a non DataflowRunner, in which case it's perfectly normal. - return ""; + if (response.getStatusLine().getStatusCode() == 200) { + InputStream in = response.getEntity().getContent(); + try (final Reader reader = new InputStreamReader(in, StandardCharsets.UTF_8)) { + ret = CharStreams.toString(reader); + } } - InputStream in = response.getEntity().getContent(); - try (final Reader reader = new InputStreamReader(in, StandardCharsets.UTF_8)) { - return CharStreams.toString(reader); - } - } catch (IOException e) { - // May mean its running on a non DataflowRunner, in which case it's perfectly normal. + } catch (IOException ignored) { } - return ""; + + // The return value can be an empty string, which may mean it's running on a non DataflowRunner. + LOG.debug("Fetched GCE Metadata at '{}' and got '{}'", requestUrl, ret); + + return ret; + } + + private static String fetchVmInstanceMetadata(String instanceMetadataKey) { + return GceMetadataUtil.fetchMetadata("instance/" + instanceMetadataKey); } private static String fetchCustomGceMetadata(String customMetadataKey) { - return GceMetadataUtil.fetchMetadata("instance/attributes/" + customMetadataKey); + return GceMetadataUtil.fetchVmInstanceMetadata("attributes/" + customMetadataKey); } public static String fetchDataflowJobId() { return GceMetadataUtil.fetchCustomGceMetadata("job_id"); } + + public static String fetchDataflowJobName() { + return GceMetadataUtil.fetchCustomGceMetadata("job_name"); + } + + public static String fetchDataflowWorkerId() { + return GceMetadataUtil.fetchVmInstanceMetadata("id"); + } } diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtil.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtil.java index 7228ba9c5593e..0338323bb0aa4 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtil.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtil.java @@ -19,8 +19,8 @@ import static org.apache.beam.sdk.io.FileSystemUtils.wildcardToRegexp; import static org.apache.beam.sdk.options.ExperimentalOptions.hasExperiment; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.client.googleapis.batch.BatchRequest; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; @@ -85,11 +85,11 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.MoreFutures; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.slf4j.Logger; diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/Transport.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/Transport.java index 4efaa2bac0924..a789b594f46f4 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/Transport.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/Transport.java @@ -33,7 +33,7 @@ import java.security.GeneralSecurityException; import org.apache.beam.sdk.extensions.gcp.auth.NullCredentialInitializer; import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Helpers for cloud communication. */ public class Transport { diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/gcsfs/GcsPath.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/gcsfs/GcsPath.java index edf387e951ead..29c312d7a5e78 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/gcsfs/GcsPath.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/gcsfs/GcsPath.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.extensions.gcp.util.gcsfs; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings.isNullOrEmpty; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings.isNullOrEmpty; import com.google.api.services.storage.model.StorageObject; import java.io.File; diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java index 4ce530de4c2bf..f5075a3f2c551 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java @@ -23,7 +23,7 @@ import java.util.Set; import org.apache.beam.sdk.util.ApiSurface; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.hamcrest.Matcher; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java index 4b57dedc59bfa..bf30b4c030e27 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java @@ -46,8 +46,8 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.testing.RestoreSystemProperties; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -171,7 +171,7 @@ private static void makePropertiesFileWithProject(File path, String projectId) + "[dataflow]%n" + "magic = true%n", projectId); - Files.write(properties, path, StandardCharsets.UTF_8); + Files.asCharSink(path, StandardCharsets.UTF_8).write(properties); } private static String runGetProjectTest(File path, Map environment) diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrarTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrarTest.java index 69c70a80b33e5..417366c7d8091 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrarTest.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrarTest.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.io.FileSystem; import org.apache.beam.sdk.io.FileSystemRegistrar; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemTest.java index 2b56b81d8a6c6..0b79cde1f187d 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemTest.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemTest.java @@ -39,8 +39,8 @@ import org.apache.beam.sdk.io.fs.MatchResult; import org.apache.beam.sdk.io.fs.MatchResult.Status; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Before; import org.junit.Rule; diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilIT.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilIT.java index ff96566d12e43..08b9f5456dd2e 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilIT.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilIT.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.testing.TestPipelineOptions; import org.apache.beam.sdk.testing.UsesKms; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilTest.java index 26b270d5faf74..8b311abcb515d 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilTest.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilTest.java @@ -104,8 +104,8 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.util.FastNanoClockAndSleeper; import org.apache.beam.sdk.util.FluentBackoff; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Before; import org.junit.Rule; diff --git a/sdks/java/extensions/jackson/build.gradle b/sdks/java/extensions/jackson/build.gradle index b288d11178dc0..e71afdf6d4024 100644 --- a/sdks/java/extensions/jackson/build.gradle +++ b/sdks/java/extensions/jackson/build.gradle @@ -26,7 +26,7 @@ description = "Apache Beam :: SDKs :: Java :: Extensions :: Jackson" ext.summary = "Jackson extension provides PTransforms for deserializing and generating JSON strings." dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.jackson_databind implementation library.java.jackson_core diff --git a/sdks/java/extensions/jackson/src/main/java/org/apache/beam/sdk/extensions/jackson/AsJsons.java b/sdks/java/extensions/jackson/src/main/java/org/apache/beam/sdk/extensions/jackson/AsJsons.java index b19e0e97c2908..507dfd4247243 100644 --- a/sdks/java/extensions/jackson/src/main/java/org/apache/beam/sdk/extensions/jackson/AsJsons.java +++ b/sdks/java/extensions/jackson/src/main/java/org/apache/beam/sdk/extensions/jackson/AsJsons.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/extensions/jackson/src/main/java/org/apache/beam/sdk/extensions/jackson/ParseJsons.java b/sdks/java/extensions/jackson/src/main/java/org/apache/beam/sdk/extensions/jackson/ParseJsons.java index c3e249b6d7a60..566c9a47967e5 100644 --- a/sdks/java/extensions/jackson/src/main/java/org/apache/beam/sdk/extensions/jackson/ParseJsons.java +++ b/sdks/java/extensions/jackson/src/main/java/org/apache/beam/sdk/extensions/jackson/ParseJsons.java @@ -33,7 +33,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/extensions/jackson/src/test/java/org/apache/beam/sdk/extensions/jackson/JacksonTransformsTest.java b/sdks/java/extensions/jackson/src/test/java/org/apache/beam/sdk/extensions/jackson/JacksonTransformsTest.java index 2e1855c412a95..bae86595ed019 100644 --- a/sdks/java/extensions/jackson/src/test/java/org/apache/beam/sdk/extensions/jackson/JacksonTransformsTest.java +++ b/sdks/java/extensions/jackson/src/test/java/org/apache/beam/sdk/extensions/jackson/JacksonTransformsTest.java @@ -43,7 +43,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/extensions/join-library/build.gradle b/sdks/java/extensions/join-library/build.gradle index f8af4e9ed10e9..438b2fc1cc45b 100644 --- a/sdks/java/extensions/join-library/build.gradle +++ b/sdks/java/extensions/join-library/build.gradle @@ -24,7 +24,7 @@ applyJavaNature( description = "Apache Beam :: SDKs :: Java :: Extensions :: Join library" dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") testImplementation library.java.junit testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") diff --git a/sdks/java/extensions/join-library/src/main/java/org/apache/beam/sdk/extensions/joinlibrary/Join.java b/sdks/java/extensions/join-library/src/main/java/org/apache/beam/sdk/extensions/joinlibrary/Join.java index 3861b9c66ab54..74c977ec4ee82 100644 --- a/sdks/java/extensions/join-library/src/main/java/org/apache/beam/sdk/extensions/joinlibrary/Join.java +++ b/sdks/java/extensions/join-library/src/main/java/org/apache/beam/sdk/extensions/joinlibrary/Join.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.joinlibrary; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import org.apache.beam.sdk.coders.KvCoder; import org.apache.beam.sdk.transforms.DoFn; diff --git a/sdks/java/extensions/kryo/build.gradle b/sdks/java/extensions/kryo/build.gradle index af083d8d2a37b..29dfd453cc37d 100644 --- a/sdks/java/extensions/kryo/build.gradle +++ b/sdks/java/extensions/kryo/build.gradle @@ -19,7 +19,7 @@ plugins { id 'org.apache.beam.module' } ext { - kryoVersion = '4.0.2' + kryoVersion = '5.5.0' } applyJavaNature( automaticModuleName: 'org.apache.beam.sdk.extensions.kryo', @@ -40,7 +40,7 @@ description = 'Apache Beam :: SDKs :: Java :: Extensions :: Kryo' dependencies { implementation library.java.jackson_annotations - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation "com.esotericsoftware:kryo:${kryoVersion}" implementation "org.objenesis:objenesis:3.2" shadow project(path: ':sdks:java:core', configuration: 'shadow') diff --git a/sdks/java/extensions/kryo/src/main/java/org/apache/beam/sdk/extensions/kryo/KryoCoder.java b/sdks/java/extensions/kryo/src/main/java/org/apache/beam/sdk/extensions/kryo/KryoCoder.java index 2cade0ef963d6..4bedd4447f0ef 100644 --- a/sdks/java/extensions/kryo/src/main/java/org/apache/beam/sdk/extensions/kryo/KryoCoder.java +++ b/sdks/java/extensions/kryo/src/main/java/org/apache/beam/sdk/extensions/kryo/KryoCoder.java @@ -199,10 +199,10 @@ public void encode(T value, OutputStream outStream) throws IOException { outputChunked.setOutputStream(outStream); try { kryoState.getKryo().writeClassAndObject(outputChunked, value); - outputChunked.endChunks(); + outputChunked.endChunk(); outputChunked.flush(); } catch (KryoException e) { - outputChunked.clear(); + outputChunked.reset(); if (e.getCause() instanceof EOFException) { throw (EOFException) e.getCause(); } @@ -284,7 +284,7 @@ public int hashCode() { @Override public boolean equals(@Nullable Object other) { if (other != null && getClass().equals(other.getClass())) { - return instanceId.equals(((KryoCoder) other).instanceId); + return instanceId.equals(((KryoCoder) other).instanceId); } return false; } diff --git a/sdks/java/extensions/kryo/src/main/java/org/apache/beam/sdk/extensions/kryo/KryoCoderProvider.java b/sdks/java/extensions/kryo/src/main/java/org/apache/beam/sdk/extensions/kryo/KryoCoderProvider.java index cd8265cd3c9a7..8562518260526 100644 --- a/sdks/java/extensions/kryo/src/main/java/org/apache/beam/sdk/extensions/kryo/KryoCoderProvider.java +++ b/sdks/java/extensions/kryo/src/main/java/org/apache/beam/sdk/extensions/kryo/KryoCoderProvider.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** * Implementation of {@link CoderProvider}, which provides {@link KryoCoder} for any type registered diff --git a/sdks/java/extensions/kryo/src/main/java/org/apache/beam/sdk/extensions/kryo/KryoState.java b/sdks/java/extensions/kryo/src/main/java/org/apache/beam/sdk/extensions/kryo/KryoState.java index d32336c899d6e..f47d022a03695 100644 --- a/sdks/java/extensions/kryo/src/main/java/org/apache/beam/sdk/extensions/kryo/KryoState.java +++ b/sdks/java/extensions/kryo/src/main/java/org/apache/beam/sdk/extensions/kryo/KryoState.java @@ -20,6 +20,7 @@ import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.InputChunked; import com.esotericsoftware.kryo.io.OutputChunked; +import com.esotericsoftware.kryo.util.DefaultInstantiatorStrategy; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.HashMap; import java.util.Map; @@ -56,7 +57,7 @@ KryoState getOrCreate(KryoCoder coder) { final Kryo kryo = new Kryo(); // fallback in case serialized class does not have default constructor kryo.setInstantiatorStrategy( - new Kryo.DefaultInstantiatorStrategy(new StdInstantiatorStrategy())); + new DefaultInstantiatorStrategy(new StdInstantiatorStrategy())); kryo.setReferences(coder.getOptions().getReferences()); kryo.setRegistrationRequired(coder.getOptions().getRegistrationRequired()); diff --git a/sdks/java/extensions/protobuf/build.gradle b/sdks/java/extensions/protobuf/build.gradle index 910826644a145..2696f8886ddd8 100644 --- a/sdks/java/extensions/protobuf/build.gradle +++ b/sdks/java/extensions/protobuf/build.gradle @@ -34,7 +34,7 @@ ext.summary = "Add support to Apache Beam for Google Protobuf." dependencies { implementation library.java.byte_buddy - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.protobuf_java testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") diff --git a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java index 09ea7973b545f..7fd11f47e3a83 100644 --- a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java +++ b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoder.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.coders.CoderException; import org.apache.beam.sdk.util.VarInt; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; /** * A {@link Coder} for {@link ByteString} objects based on their encoded Protocol Buffer form. diff --git a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/DynamicProtoCoder.java b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/DynamicProtoCoder.java index a0f02e00c5830..e1e2e52dfa930 100644 --- a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/DynamicProtoCoder.java +++ b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/DynamicProtoCoder.java @@ -33,8 +33,8 @@ import org.apache.beam.sdk.coders.CoderRegistry; import org.apache.beam.sdk.coders.DefaultCoder; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoByteBuddyUtils.java b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoByteBuddyUtils.java index 7077dc728a30b..bb2e267bae232 100644 --- a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoByteBuddyUtils.java +++ b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoByteBuddyUtils.java @@ -101,13 +101,13 @@ import org.apache.beam.sdk.util.common.ReflectHelpers; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; import org.checkerframework.checker.nullness.qual.Nullable; @SuppressWarnings({ diff --git a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoder.java b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoder.java index 8830f50482563..9edd938e59d2e 100644 --- a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoder.java +++ b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoder.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.protobuf; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.protobuf.DynamicMessage; import com.google.protobuf.ExtensionRegistry; @@ -42,8 +42,8 @@ import org.apache.beam.sdk.coders.DefaultCoder; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoMessageSchema.java b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoMessageSchema.java index fcebd748752ec..1b3d42e355362 100644 --- a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoMessageSchema.java +++ b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoMessageSchema.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.extensions.protobuf; import static org.apache.beam.sdk.extensions.protobuf.ProtoByteBuddyUtils.getProtoGetter; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.protobuf.DynamicMessage; import com.google.protobuf.Message; @@ -40,9 +40,9 @@ import org.apache.beam.sdk.transforms.SimpleFunction; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; import org.checkerframework.checker.nullness.qual.Nullable; @SuppressWarnings({ diff --git a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoSchemaTranslator.java b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoSchemaTranslator.java index 142c4c77a80b1..9f679e61ecf37 100644 --- a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoSchemaTranslator.java +++ b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtoSchemaTranslator.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.protobuf; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.protobuf.Descriptors; import com.google.protobuf.Descriptors.EnumValueDescriptor; @@ -43,9 +43,9 @@ import org.apache.beam.sdk.schemas.logicaltypes.NanosDuration; import org.apache.beam.sdk.schemas.logicaltypes.NanosInstant; import org.apache.beam.sdk.schemas.logicaltypes.OneOfType; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtobufCoderProviderRegistrar.java b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtobufCoderProviderRegistrar.java index 9c056ee1b5e0a..88845d8d19975 100644 --- a/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtobufCoderProviderRegistrar.java +++ b/sdks/java/extensions/protobuf/src/main/java/org/apache/beam/sdk/extensions/protobuf/ProtobufCoderProviderRegistrar.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.coders.CoderProviderRegistrar; import org.apache.beam.sdk.coders.CoderProviders; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** A {@link CoderProviderRegistrar} for standard types used with Google Protobuf. */ @AutoService(CoderProviderRegistrar.class) diff --git a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoderTest.java b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoderTest.java index 776b380b7094f..72e423913152c 100644 --- a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoderTest.java +++ b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ByteStringCoderTest.java @@ -32,7 +32,7 @@ import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoderTest.java b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoderTest.java index 38aa92bfc2236..79a348c61838a 100644 --- a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoderTest.java +++ b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoCoderTest.java @@ -34,7 +34,7 @@ import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoDynamicMessageSchemaTest.java b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoDynamicMessageSchemaTest.java index 8d40ddee30740..559d5b2ad98c9 100644 --- a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoDynamicMessageSchemaTest.java +++ b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoDynamicMessageSchemaTest.java @@ -84,7 +84,7 @@ import org.apache.beam.sdk.schemas.logicaltypes.EnumerationType; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoMessageSchemaTest.java b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoMessageSchemaTest.java index f5ce63226541b..3b4568f1fac75 100644 --- a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoMessageSchemaTest.java +++ b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoMessageSchemaTest.java @@ -88,7 +88,7 @@ import org.apache.beam.sdk.transforms.SimpleFunction; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoPayloadSerializerProviderTest.java b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoPayloadSerializerProviderTest.java index 69a31f338653e..fd253ca942d59 100644 --- a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoPayloadSerializerProviderTest.java +++ b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtoPayloadSerializerProviderTest.java @@ -23,8 +23,8 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtobufUtilTest.java b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtobufUtilTest.java index c68efa294d721..5685764d15ed8 100644 --- a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtobufUtilTest.java +++ b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/ProtobufUtilTest.java @@ -34,8 +34,8 @@ import org.apache.beam.sdk.extensions.protobuf.Proto2CoderTestMessages.MessageC; import org.apache.beam.sdk.extensions.protobuf.Proto2CoderTestMessages.MessageWithMap; import org.apache.beam.sdk.extensions.protobuf.Proto2CoderTestMessages.ReferencesMessageWithMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/TestProtoSchemas.java b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/TestProtoSchemas.java index 8ed1ba8cfa71c..a5f7416ce35a0 100644 --- a/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/TestProtoSchemas.java +++ b/sdks/java/extensions/protobuf/src/test/java/org/apache/beam/sdk/extensions/protobuf/TestProtoSchemas.java @@ -65,8 +65,8 @@ import org.apache.beam.sdk.schemas.logicaltypes.NanosInstant; import org.apache.beam.sdk.schemas.logicaltypes.OneOfType; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; class TestProtoSchemas { diff --git a/sdks/java/extensions/python/build.gradle b/sdks/java/extensions/python/build.gradle index 7a04d953ccb5f..7058d27fecbf8 100644 --- a/sdks/java/extensions/python/build.gradle +++ b/sdks/java/extensions/python/build.gradle @@ -24,7 +24,7 @@ description = "Apache Beam :: SDKs :: Java :: Extensions :: Python" evaluationDependsOn(":runners:core-construction-java") dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.vendored_grpc_1_54_0 implementation library.java.slf4j_api implementation project(path: ":model:pipeline", configuration: "shadow") diff --git a/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/PythonExternalTransform.java b/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/PythonExternalTransform.java index 7a217504d626f..5ba3484964c1e 100644 --- a/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/PythonExternalTransform.java +++ b/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/PythonExternalTransform.java @@ -19,6 +19,7 @@ import java.io.File; import java.io.FileOutputStream; +import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Writer; import java.util.ArrayList; @@ -59,15 +60,16 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** Wrapper for invoking external Python transforms. */ public class PythonExternalTransform @@ -89,6 +91,8 @@ public class PythonExternalTransform> outputCoders; + private static final Logger LOG = LoggerFactory.getLogger(PythonExternalTransform.class); + private PythonExternalTransform(String fullyQualifiedName, String expansionService) { this.fullyQualifiedName = fullyQualifiedName; this.expansionService = expansionService; @@ -439,41 +443,101 @@ ExternalTransforms.ExternalConfigurationPayload generatePayload() { } } + private boolean isPythonAvailable() { + for (String executable : ImmutableList.of("python3", "python")) { + try { + new ProcessBuilder(executable, "--version").start().waitFor(); + return true; + } catch (IOException | InterruptedException exn) { + // Ignore. + } + } + return false; + } + + private boolean isDockerAvailable() { + String executable = "docker"; + try { + new ProcessBuilder(executable, "--version").start().waitFor(); + return true; + } catch (IOException | InterruptedException exn) { + // Ignore. + } + return false; + } + @Override public OutputT expand(InputT input) { try { ExternalTransforms.ExternalConfigurationPayload payload = generatePayload(); if (!Strings.isNullOrEmpty(expansionService)) { + int portIndex = expansionService.lastIndexOf(':'); + if (portIndex <= 0) { + throw new IllegalArgumentException( + "Unexpected expansion service address. Expected to be in the " + + "format \":\""); + } PythonService.waitForPort( - Iterables.get(Splitter.on(':').split(expansionService), 0), - Integer.parseInt(Iterables.get(Splitter.on(':').split(expansionService), 1)), + expansionService.substring(0, portIndex), + Integer.parseInt(expansionService.substring(portIndex + 1, expansionService.length())), 15000); return apply(input, expansionService, payload); } else { OutputT output = null; int port = PythonService.findAvailablePort(); PipelineOptionsFactory.register(PythonExternalTransformOptions.class); - if (input - .getPipeline() - .getOptions() - .as(PythonExternalTransformOptions.class) - .getUseTransformService()) { + boolean useTransformService = + input + .getPipeline() + .getOptions() + .as(PythonExternalTransformOptions.class) + .getUseTransformService(); + boolean pythonAvailable = isPythonAvailable(); + boolean dockerAvailable = isDockerAvailable(); + + File requirementsFile = null; + if (!extraPackages.isEmpty()) { + requirementsFile = File.createTempFile("requirements", ".txt"); + requirementsFile.deleteOnExit(); + try (Writer fout = + new OutputStreamWriter( + new FileOutputStream(requirementsFile.getAbsolutePath()), Charsets.UTF_8)) { + for (String pkg : extraPackages) { + fout.write(pkg); + fout.write('\n'); + } + } + } + + // We use the transform service if either of the following is true. + // * It was explicitly requested. + // * Python executable is not available in the system but Docker is available. + if (useTransformService || (!pythonAvailable && dockerAvailable)) { // A unique project name ensures that this expansion gets a dedicated instance of the // transform service. String projectName = UUID.randomUUID().toString(); - TransformServiceLauncher service = TransformServiceLauncher.forProject(projectName, port); + + String messageAppend = + useTransformService + ? "it was explicitly requested" + : "a Python executable is not available in the system"; + LOG.info( + "Using the Docker Compose based transform service since {}. Service will have the " + + "project name {} and will be made available at the port {}", + messageAppend, + projectName, + port); + + String pythonRequirementsFile = + requirementsFile != null ? requirementsFile.getAbsolutePath() : null; + TransformServiceLauncher service = + TransformServiceLauncher.forProject(projectName, port, pythonRequirementsFile); service.setBeamVersion(ReleaseInfo.getReleaseInfo().getSdkVersion()); - // TODO(https://github.com/apache/beam/issues/26833): add support for installing extra - // packages. - if (!extraPackages.isEmpty()) { - throw new RuntimeException( - "Transform Service does not support installing extra packages yet"); - } try { // Starting the transform service. service.start(); // Waiting the service to be ready. - service.waitTillUp(15000); + service.waitTillUp(-1); // Expanding the transform. output = apply(input, String.format("localhost:%s", port), payload); } finally { @@ -486,17 +550,7 @@ public OutputT expand(InputT input) { ImmutableList.Builder args = ImmutableList.builder(); args.add( "--port=" + port, "--fully_qualified_name_glob=*", "--pickle_library=cloudpickle"); - if (!extraPackages.isEmpty()) { - File requirementsFile = File.createTempFile("requirements", ".txt"); - requirementsFile.deleteOnExit(); - try (Writer fout = - new OutputStreamWriter( - new FileOutputStream(requirementsFile.getAbsolutePath()), Charsets.UTF_8)) { - for (String pkg : extraPackages) { - fout.write(pkg); - fout.write('\n'); - } - } + if (requirementsFile != null) { args.add("--requirements_file=" + requirementsFile.getAbsolutePath()); } PythonService service = diff --git a/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/PythonExternalTransformOptionsRegistrar.java b/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/PythonExternalTransformOptionsRegistrar.java index a7d138b409d74..7af05ff1ab42b 100644 --- a/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/PythonExternalTransformOptionsRegistrar.java +++ b/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/PythonExternalTransformOptionsRegistrar.java @@ -21,7 +21,7 @@ import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** A registrar for {@link PythonExternalTransformOptions}. */ @AutoService(PipelineOptionsRegistrar.class) diff --git a/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/PythonService.java b/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/PythonService.java index 0d7a918aade3f..75fe444c931fd 100644 --- a/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/PythonService.java +++ b/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/PythonService.java @@ -29,10 +29,10 @@ import java.util.List; import java.util.concurrent.TimeoutException; import org.apache.beam.sdk.util.ReleaseInfo; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/transforms/RunInference.java b/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/transforms/RunInference.java index afbc7f6774822..9ea016166cef3 100644 --- a/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/transforms/RunInference.java +++ b/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/transforms/RunInference.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/extensions/python/src/test/java/org/apache/beam/sdk/extensions/python/PythonExternalTransformTest.java b/sdks/java/extensions/python/src/test/java/org/apache/beam/sdk/extensions/python/PythonExternalTransformTest.java index 7ea55701e91e8..a13e4d77a2e3d 100644 --- a/sdks/java/extensions/python/src/test/java/org/apache/beam/sdk/extensions/python/PythonExternalTransformTest.java +++ b/sdks/java/extensions/python/src/test/java/org/apache/beam/sdk/extensions/python/PythonExternalTransformTest.java @@ -39,8 +39,8 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; diff --git a/sdks/java/extensions/python/src/test/java/org/apache/beam/sdk/extensions/python/transforms/PythonMapTest.java b/sdks/java/extensions/python/src/test/java/org/apache/beam/sdk/extensions/python/transforms/PythonMapTest.java index d90a869216650..4b162a9d5435f 100644 --- a/sdks/java/extensions/python/src/test/java/org/apache/beam/sdk/extensions/python/transforms/PythonMapTest.java +++ b/sdks/java/extensions/python/src/test/java/org/apache/beam/sdk/extensions/python/transforms/PythonMapTest.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.testing.ValidatesRunner; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; diff --git a/sdks/java/extensions/sbe/build.gradle b/sdks/java/extensions/sbe/build.gradle index 820507a643a89..ce01f9638c35e 100644 --- a/sdks/java/extensions/sbe/build.gradle +++ b/sdks/java/extensions/sbe/build.gradle @@ -27,7 +27,7 @@ ext.summary = "Add support to Beam for FIX SBE" dependencies { implementation library.java.auto_value_annotations implementation library.java.sbe_tool - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") testImplementation library.java.junit diff --git a/sdks/java/extensions/sbe/src/main/java/org/apache/beam/sdk/extensions/sbe/IrFieldGenerator.java b/sdks/java/extensions/sbe/src/main/java/org/apache/beam/sdk/extensions/sbe/IrFieldGenerator.java index 4ae170d758228..eeaf0d9196132 100644 --- a/sdks/java/extensions/sbe/src/main/java/org/apache/beam/sdk/extensions/sbe/IrFieldGenerator.java +++ b/sdks/java/extensions/sbe/src/main/java/org/apache/beam/sdk/extensions/sbe/IrFieldGenerator.java @@ -19,12 +19,12 @@ import static java.util.stream.Collectors.collectingAndThen; import static java.util.stream.Collectors.toList; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Iterator; import java.util.List; import org.apache.beam.sdk.extensions.sbe.SbeSchema.IrOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import uk.co.real_logic.sbe.ir.Encoding.Presence; import uk.co.real_logic.sbe.ir.Ir; import uk.co.real_logic.sbe.ir.Token; diff --git a/sdks/java/extensions/sbe/src/main/java/org/apache/beam/sdk/extensions/sbe/SbeSchema.java b/sdks/java/extensions/sbe/src/main/java/org/apache/beam/sdk/extensions/sbe/SbeSchema.java index cb0f305c86f76..afdf2b937e78d 100644 --- a/sdks/java/extensions/sbe/src/main/java/org/apache/beam/sdk/extensions/sbe/SbeSchema.java +++ b/sdks/java/extensions/sbe/src/main/java/org/apache/beam/sdk/extensions/sbe/SbeSchema.java @@ -17,11 +17,11 @@ */ package org.apache.beam.sdk.extensions.sbe; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.io.Serializable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import uk.co.real_logic.sbe.ir.Ir; diff --git a/sdks/java/extensions/sbe/src/main/java/org/apache/beam/sdk/extensions/sbe/UnsignedOptions.java b/sdks/java/extensions/sbe/src/main/java/org/apache/beam/sdk/extensions/sbe/UnsignedOptions.java index 24c7458d7d3f3..e5022b3a9b629 100644 --- a/sdks/java/extensions/sbe/src/main/java/org/apache/beam/sdk/extensions/sbe/UnsignedOptions.java +++ b/sdks/java/extensions/sbe/src/main/java/org/apache/beam/sdk/extensions/sbe/UnsignedOptions.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.extensions.sbe; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.io.Serializable; diff --git a/sdks/java/extensions/sbe/src/test/java/org/apache/beam/sdk/extensions/sbe/IrFieldGeneratorTest.java b/sdks/java/extensions/sbe/src/test/java/org/apache/beam/sdk/extensions/sbe/IrFieldGeneratorTest.java index ffbedcdf0e4ee..f6d16ac0a0fb5 100644 --- a/sdks/java/extensions/sbe/src/test/java/org/apache/beam/sdk/extensions/sbe/IrFieldGeneratorTest.java +++ b/sdks/java/extensions/sbe/src/test/java/org/apache/beam/sdk/extensions/sbe/IrFieldGeneratorTest.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.extensions.sbe.TestSchemas.OnlyPrimitivesMultiMessage; import org.apache.beam.sdk.extensions.sbe.TestSchemas.OnlyPrimitivesMultiMessage.Primitives1; import org.apache.beam.sdk.extensions.sbe.TestSchemas.OnlyPrimitivesMultiMessage.Primitives2; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/extensions/sbe/src/test/java/org/apache/beam/sdk/extensions/sbe/SerializableIrTest.java b/sdks/java/extensions/sbe/src/test/java/org/apache/beam/sdk/extensions/sbe/SerializableIrTest.java index a45b932b2c034..32f37d1df0bfa 100644 --- a/sdks/java/extensions/sbe/src/test/java/org/apache/beam/sdk/extensions/sbe/SerializableIrTest.java +++ b/sdks/java/extensions/sbe/src/test/java/org/apache/beam/sdk/extensions/sbe/SerializableIrTest.java @@ -29,7 +29,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Resources; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Resources; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/extensions/sbe/src/test/java/org/apache/beam/sdk/extensions/sbe/TestSchemas.java b/sdks/java/extensions/sbe/src/test/java/org/apache/beam/sdk/extensions/sbe/TestSchemas.java index 44f48764602be..ed4e77f1dce72 100644 --- a/sdks/java/extensions/sbe/src/test/java/org/apache/beam/sdk/extensions/sbe/TestSchemas.java +++ b/sdks/java/extensions/sbe/src/test/java/org/apache/beam/sdk/extensions/sbe/TestSchemas.java @@ -21,8 +21,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Resources; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Resources; import uk.co.real_logic.sbe.PrimitiveType; import uk.co.real_logic.sbe.ir.Ir; import uk.co.real_logic.sbe.xml.IrGenerator; diff --git a/sdks/java/extensions/schemaio-expansion-service/build.gradle b/sdks/java/extensions/schemaio-expansion-service/build.gradle index 467540ebfff10..246c0c155cbdc 100644 --- a/sdks/java/extensions/schemaio-expansion-service/build.gradle +++ b/sdks/java/extensions/schemaio-expansion-service/build.gradle @@ -32,14 +32,20 @@ applyJavaNature( dependencies { implementation project(path: ":sdks:java:expansion-service") permitUnusedDeclared project(path: ":sdks:java:expansion-service") // BEAM-11761 + implementation project(":sdks:java:extensions:google-cloud-platform-core") + permitUnusedDeclared project(path: ":sdks:java:extensions:google-cloud-platform-core") // BEAM-11761 + implementation project(":sdks:java:io:csv") + permitUnusedDeclared project(path: ":sdks:java:io:csv") // BEAM-11761 implementation project(":sdks:java:io:jdbc") permitUnusedDeclared project(":sdks:java:io:jdbc") // BEAM-11761 + implementation project(":sdks:java:io:json") + permitUnusedDeclared project(path: ":sdks:java:io:json") // BEAM-11761 implementation library.java.postgres permitUnusedDeclared library.java.postgres // BEAM-11761 implementation project(path: ":model:pipeline", configuration: "shadow") implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.vendored_grpc_1_54_0 - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre testImplementation library.java.junit testImplementation library.java.mockito_core } diff --git a/sdks/java/extensions/schemaio-expansion-service/src/main/java/org/apache/beam/sdk/extensions/schemaio/expansion/ExternalSchemaIOTransformRegistrar.java b/sdks/java/extensions/schemaio-expansion-service/src/main/java/org/apache/beam/sdk/extensions/schemaio/expansion/ExternalSchemaIOTransformRegistrar.java index db8c84d36acd9..7bc14afb7832b 100644 --- a/sdks/java/extensions/schemaio-expansion-service/src/main/java/org/apache/beam/sdk/extensions/schemaio/expansion/ExternalSchemaIOTransformRegistrar.java +++ b/sdks/java/extensions/schemaio-expansion-service/src/main/java/org/apache/beam/sdk/extensions/schemaio/expansion/ExternalSchemaIOTransformRegistrar.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.schemaio.expansion; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.service.AutoService; import java.io.ByteArrayInputStream; @@ -40,8 +40,8 @@ import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; @AutoService(ExternalTransformRegistrar.class) @SuppressWarnings({ diff --git a/sdks/java/extensions/sketching/build.gradle b/sdks/java/extensions/sketching/build.gradle index b239771ad2741..824f3393cd201 100644 --- a/sdks/java/extensions/sketching/build.gradle +++ b/sdks/java/extensions/sketching/build.gradle @@ -26,7 +26,7 @@ def streamlib_version = "2.9.5" def tdigest_version = "3.2" dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation "com.clearspring.analytics:stream:$streamlib_version" implementation "com.tdunning:t-digest:$tdigest_version" diff --git a/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinct.java b/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinct.java index 53906071a4b35..067a9fdb374b6 100644 --- a/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinct.java +++ b/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/ApproximateDistinct.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sketching; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.clearspring.analytics.stream.cardinality.CardinalityMergeException; import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; diff --git a/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/SketchFrequencies.java b/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/SketchFrequencies.java index 6f28253b05665..13e7f938abfef 100644 --- a/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/SketchFrequencies.java +++ b/sdks/java/extensions/sketching/src/main/java/org/apache/beam/sdk/extensions/sketching/SketchFrequencies.java @@ -39,7 +39,7 @@ import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; /** * {@code PTransform}s to compute the estimate frequency of each element in a stream. diff --git a/sdks/java/extensions/sorter/build.gradle b/sdks/java/extensions/sorter/build.gradle index af240cb814705..dbe9bb363b651 100644 --- a/sdks/java/extensions/sorter/build.gradle +++ b/sdks/java/extensions/sorter/build.gradle @@ -34,7 +34,7 @@ hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")} dependencies { implementation project(path: ":sdks:java:core", configuration: "shadow") - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.slf4j_api provided library.java.hadoop_mapreduce_client_core provided library.java.hadoop_common diff --git a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorter.java b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorter.java index 65ac564df6002..6cec1b7c71ffc 100644 --- a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorter.java +++ b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorter.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sorter; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.io.Serializable; diff --git a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/ExternalSorter.java b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/ExternalSorter.java index d07aeeffd8df0..60363055ed748 100644 --- a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/ExternalSorter.java +++ b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/ExternalSorter.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sorter; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.Serializable; diff --git a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/HadoopExternalSorter.java b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/HadoopExternalSorter.java index 7e16673825976..07c9dc230a075 100644 --- a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/HadoopExternalSorter.java +++ b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/HadoopExternalSorter.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sorter; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; diff --git a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/InMemorySorter.java b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/InMemorySorter.java index 39b89b840dba4..481771f3f029c 100644 --- a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/InMemorySorter.java +++ b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/InMemorySorter.java @@ -17,15 +17,15 @@ */ package org.apache.beam.sdk.extensions.sorter; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.UnsignedBytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedBytes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/NativeExternalSorter.java b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/NativeExternalSorter.java index 99c012fd83831..4f9436464bb74 100644 --- a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/NativeExternalSorter.java +++ b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/NativeExternalSorter.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sorter; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.nio.file.Paths; diff --git a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/NativeFileSorter.java b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/NativeFileSorter.java index 1b2951a3676fa..368eef8329fbc 100644 --- a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/NativeFileSorter.java +++ b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/NativeFileSorter.java @@ -37,9 +37,9 @@ import java.util.NoSuchElementException; import org.apache.beam.sdk.coders.ByteArrayCoder; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.UnsignedBytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedBytes; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/SorterTestUtils.java b/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/SorterTestUtils.java index 821ae8d19b0d2..533cf6c88fa3a 100644 --- a/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/SorterTestUtils.java +++ b/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/SorterTestUtils.java @@ -25,7 +25,7 @@ import java.util.Random; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.UnsignedBytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedBytes; import org.junit.rules.ExpectedException; /** A set of basic tests for {@link Sorter}s. */ diff --git a/sdks/java/extensions/sql/OWNERS b/sdks/java/extensions/sql/OWNERS deleted file mode 100644 index 371386ae63c0e..0000000000000 --- a/sdks/java/extensions/sql/OWNERS +++ /dev/null @@ -1,10 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - akedin - - apilloud - - amaliujia - - XuMingmin - - xumingming - - kennknowles - diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle index 19ebcc465c8ea..8a22bff69b7f8 100644 --- a/sdks/java/extensions/sql/build.gradle +++ b/sdks/java/extensions/sql/build.gradle @@ -21,7 +21,7 @@ import java.util.stream.Collectors plugins { id 'org.apache.beam.module' - id 'ca.coglinc.javacc' + id 'org.javacc.javacc' } applyJavaNature( generatedClassPatterns: [ @@ -84,14 +84,13 @@ dependencies { implementation library.java.jackson_databind implementation library.java.joda_time implementation library.java.vendored_calcite_1_28_0 - implementation "com.alibaba:fastjson:1.2.69" implementation "org.codehaus.janino:janino:3.0.11" implementation "org.codehaus.janino:commons-compiler:3.0.11" implementation library.java.jackson_core implementation library.java.mongo_java_driver implementation library.java.slf4j_api implementation library.java.joda_time - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre provided project(":sdks:java:io:kafka") implementation project(":sdks:java:extensions:google-cloud-platform-core") permitUnusedDeclared project(":sdks:java:extensions:google-cloud-platform-core") @@ -112,7 +111,7 @@ dependencies { permitUnusedDeclared library.java.hadoop_client provided library.java.kafka_clients testImplementation library.java.vendored_calcite_1_28_0 - testImplementation library.java.vendored_guava_26_0_jre + testImplementation library.java.vendored_guava_32_1_2_jre testImplementation library.java.junit testImplementation library.java.quickcheck_core testImplementation library.java.testcontainers_kafka diff --git a/sdks/java/extensions/sql/datacatalog/build.gradle b/sdks/java/extensions/sql/datacatalog/build.gradle index 9b4a554a89ae3..cb557cc807763 100644 --- a/sdks/java/extensions/sql/datacatalog/build.gradle +++ b/sdks/java/extensions/sql/datacatalog/build.gradle @@ -33,8 +33,8 @@ dependencies { implementation library.java.proto_google_cloud_datacatalog_v1beta1 implementation library.java.protobuf_java implementation library.java.slf4j_api - implementation library.java.vendored_guava_26_0_jre - implementation "com.alibaba:fastjson:1.2.69" + implementation library.java.vendored_guava_32_1_2_jre + implementation library.java.jackson_databind implementation project(path: ":sdks:java:core", configuration: "shadow") implementation "org.threeten:threetenbp:1.4.5" provided project(":sdks:java:extensions:sql") diff --git a/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlDataCatalogExample.java b/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlDataCatalogExample.java index 391d926a533bc..0b2f1b04cbbaa 100644 --- a/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlDataCatalogExample.java +++ b/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlDataCatalogExample.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/BigQueryTableFactory.java b/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/BigQueryTableFactory.java index 1c68bf18c6ed5..172dad752d93e 100644 --- a/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/BigQueryTableFactory.java +++ b/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/BigQueryTableFactory.java @@ -17,14 +17,13 @@ */ package org.apache.beam.sdk.extensions.sql.meta.provider.datacatalog; -import com.alibaba.fastjson.JSONObject; import com.google.cloud.datacatalog.v1beta1.Entry; import java.net.URI; import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.meta.Table; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; /** {@link TableFactory} that understands Data Catalog BigQuery entries. */ class BigQueryTableFactory implements TableFactory { @@ -49,7 +48,7 @@ public Optional tableBuilder(Entry entry) { return Optional.of( Table.builder() .location(getLocation(entry)) - .properties(new JSONObject(ImmutableMap.of("truncateTimestamps", truncateTimestamps))) + .properties(TableUtils.emptyProperties().put("truncateTimestamps", truncateTimestamps)) .type("bigquery") .comment("")); } diff --git a/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/DataCatalogPipelineOptionsRegistrar.java b/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/DataCatalogPipelineOptionsRegistrar.java index 0e7c0ea4376b0..7f055dd245afc 100644 --- a/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/DataCatalogPipelineOptionsRegistrar.java +++ b/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/DataCatalogPipelineOptionsRegistrar.java @@ -20,7 +20,7 @@ import com.google.auto.service.AutoService; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; @AutoService(PipelineOptionsRegistrar.class) public class DataCatalogPipelineOptionsRegistrar implements PipelineOptionsRegistrar { diff --git a/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/DataCatalogTableProvider.java b/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/DataCatalogTableProvider.java index 7a513e1378564..7ac67a9fc9dc0 100644 --- a/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/DataCatalogTableProvider.java +++ b/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/DataCatalogTableProvider.java @@ -46,9 +46,9 @@ import org.apache.beam.sdk.extensions.sql.meta.provider.pubsub.PubsubTableProvider; import org.apache.beam.sdk.extensions.sql.meta.provider.text.TextTableProvider; import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/GcsTableFactory.java b/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/GcsTableFactory.java index 39d963af2a805..3a85362bbeab7 100644 --- a/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/GcsTableFactory.java +++ b/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/GcsTableFactory.java @@ -17,11 +17,11 @@ */ package org.apache.beam.sdk.extensions.sql.meta.provider.datacatalog; -import com.alibaba.fastjson.JSONObject; import com.google.cloud.datacatalog.v1beta1.Entry; import com.google.cloud.datacatalog.v1beta1.GcsFilesetSpec; import java.util.List; import java.util.Optional; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.meta.Table; /** {@link TableFactory} that understands Data Catalog GCS entries. */ @@ -55,7 +55,7 @@ public Optional tableBuilder(Entry entry) { Table.builder() .type("text") .location(filePattern) - .properties(new JSONObject()) + .properties(TableUtils.emptyProperties()) .comment("")); } } diff --git a/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/PubsubTableFactory.java b/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/PubsubTableFactory.java index 656f4dd2202fa..5688713f7ee85 100644 --- a/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/PubsubTableFactory.java +++ b/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/PubsubTableFactory.java @@ -17,12 +17,12 @@ */ package org.apache.beam.sdk.extensions.sql.meta.provider.datacatalog; -import com.alibaba.fastjson.JSONObject; import com.google.cloud.datacatalog.v1beta1.Entry; import java.net.URI; import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.meta.Table; /** {@link TableFactory} that understands Data Catalog Pubsub entries. */ @@ -42,7 +42,7 @@ public Optional tableBuilder(Entry entry) { return Optional.of( Table.builder() .location(getLocation(entry)) - .properties(new JSONObject()) + .properties(TableUtils.emptyProperties()) .type("pubsub") .comment("")); } diff --git a/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/SchemaUtils.java b/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/SchemaUtils.java index 8c38541929eb3..ef66d0e656cbb 100644 --- a/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/SchemaUtils.java +++ b/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/SchemaUtils.java @@ -26,8 +26,8 @@ import org.apache.beam.sdk.schemas.Schema.Field; import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.logicaltypes.SqlTypes; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; @SuppressWarnings({ "nullness" // TODO(https://github.com/apache/beam/issues/20497) diff --git a/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/ZetaSqlIdUtils.java b/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/ZetaSqlIdUtils.java index eac82c45dc418..e00fbb3148a75 100644 --- a/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/ZetaSqlIdUtils.java +++ b/sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/ZetaSqlIdUtils.java @@ -18,10 +18,11 @@ package org.apache.beam.sdk.extensions.sql.meta.provider.datacatalog; import static java.util.stream.Collectors.joining; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.List; import java.util.regex.Pattern; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** Utils to work with ZetaSQL-compatible IDs. */ class ZetaSqlIdUtils { @@ -70,7 +71,10 @@ private static String escapeSpecialChars(String str) { private static String replaceWhitespaces(String s) { return WHITESPACES.keySet().stream() - .reduce(s, (str, whitespace) -> str.replaceAll(whitespace, WHITESPACES.get(whitespace))); + .reduce( + s, + (str, whitespace) -> + str.replaceAll(whitespace, checkNotNull(WHITESPACES.get(whitespace)))); } private static String backtickIfNeeded(String s) { diff --git a/sdks/java/extensions/sql/expansion-service/build.gradle b/sdks/java/extensions/sql/expansion-service/build.gradle index 48f31c75128d5..b6963cf7547b8 100644 --- a/sdks/java/extensions/sql/expansion-service/build.gradle +++ b/sdks/java/extensions/sql/expansion-service/build.gradle @@ -37,7 +37,7 @@ dependencies { permitUnusedDeclared project(path: ":sdks:java:expansion-service") // BEAM-11761 implementation project(path: ":sdks:java:extensions:sql") implementation project(path: ":sdks:java:extensions:sql:zetasql") - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre } diff --git a/sdks/java/extensions/sql/expansion-service/src/main/java/org/apache/beam/sdk/extensions/sql/expansion/ExternalSqlTransformRegistrar.java b/sdks/java/extensions/sql/expansion-service/src/main/java/org/apache/beam/sdk/extensions/sql/expansion/ExternalSqlTransformRegistrar.java index f5d6def7bd91d..86f2260c9742f 100644 --- a/sdks/java/extensions/sql/expansion-service/src/main/java/org/apache/beam/sdk/extensions/sql/expansion/ExternalSqlTransformRegistrar.java +++ b/sdks/java/extensions/sql/expansion-service/src/main/java/org/apache/beam/sdk/extensions/sql/expansion/ExternalSqlTransformRegistrar.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PInput; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; @AutoService(ExternalTransformRegistrar.class) @@ -43,7 +43,7 @@ public class ExternalSqlTransformRegistrar implements ExternalTransformRegistrar @Override public Map>> knownBuilders() { - return org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap.of( + return org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap.of( URN, Builder.class); } diff --git a/sdks/java/extensions/sql/expansion-service/src/main/java/org/apache/beam/sdk/extensions/sql/expansion/SqlTransformSchemaTransformProvider.java b/sdks/java/extensions/sql/expansion-service/src/main/java/org/apache/beam/sdk/extensions/sql/expansion/SqlTransformSchemaTransformProvider.java index 0d9f4f6eac500..54415644152f9 100644 --- a/sdks/java/extensions/sql/expansion-service/src/main/java/org/apache/beam/sdk/extensions/sql/expansion/SqlTransformSchemaTransformProvider.java +++ b/sdks/java/extensions/sql/expansion-service/src/main/java/org/apache/beam/sdk/extensions/sql/expansion/SqlTransformSchemaTransformProvider.java @@ -43,8 +43,8 @@ import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; @AutoService(SchemaTransformProvider.class) public class SqlTransformSchemaTransformProvider implements SchemaTransformProvider { @@ -138,7 +138,7 @@ public PDone expand(PCollection input) { } } - static class SqlSchemaTransform implements SchemaTransform { + static class SqlSchemaTransform extends SchemaTransform { final Row config; public SqlSchemaTransform(Row config) { @@ -146,94 +146,86 @@ public SqlSchemaTransform(Row config) { } @Override - public PTransform buildTransform() { - return new PTransform() { - @Override - public PCollectionRowTuple expand(PCollectionRowTuple input) { - - // Start with the query. In theory the exception can't be thrown, but all this nullness - // stuff - // isn't actually smart enough to know that. Could just cop and suppress that warning, but - // doing it the hard way for some reason. - String queryString = config.getString("query"); - if (queryString == null) { - throw new IllegalArgumentException("Configuration must provide a query string."); - } - SqlTransform transform = SqlTransform.query(queryString); - - // Allow setting the query planner class via the dialect name. - EnumerationType.Value dialect = - config.getLogicalTypeValue("dialect", EnumerationType.Value.class); - if (dialect != null) { - Class queryPlannerClass = - QUERY_PLANNERS.get(QUERY_ENUMERATION.toString(dialect)); - if (queryPlannerClass != null) { - transform = transform.withQueryPlannerClass(queryPlannerClass); - } - } - - // Add any DDL strings - String ddl = config.getString("ddl"); - if (ddl != null) { - transform = transform.withDdlString(ddl); - } - - // Check to see if we autoload or not - Boolean autoload = config.getBoolean("autoload"); - if (autoload != null && autoload) { - transform = transform.withAutoLoading(true); - } else { - transform = transform.withAutoLoading(false); - - // Add any user specified table providers from the set of available tableproviders. - Map tableProviders = new HashMap<>(); - ServiceLoader.load(TableProvider.class) - .forEach( - (provider) -> { - tableProviders.put(provider.getTableType(), provider); - }); - Collection tableproviderList = config.getArray("tableproviders"); - if (tableproviderList != null) { - for (Object nameObj : tableproviderList) { - if (nameObj != null) { // This actually could in theory be null... - TableProvider p = tableProviders.get(nameObj); - if (p - != null) { // TODO: We ignore tableproviders that don't exist, we could change - // that. - transform = transform.withTableProvider(p.getTableType(), p); - } - } + public PCollectionRowTuple expand(PCollectionRowTuple input) { + + // Start with the query. In theory the exception can't be thrown, but all this nullness + // stuff + // isn't actually smart enough to know that. Could just cop and suppress that warning, but + // doing it the hard way for some reason. + String queryString = config.getString("query"); + if (queryString == null) { + throw new IllegalArgumentException("Configuration must provide a query string."); + } + SqlTransform transform = SqlTransform.query(queryString); + + // Allow setting the query planner class via the dialect name. + EnumerationType.Value dialect = + config.getLogicalTypeValue("dialect", EnumerationType.Value.class); + if (dialect != null) { + Class queryPlannerClass = + QUERY_PLANNERS.get(QUERY_ENUMERATION.toString(dialect)); + if (queryPlannerClass != null) { + transform = transform.withQueryPlannerClass(queryPlannerClass); + } + } + + // Add any DDL strings + String ddl = config.getString("ddl"); + if (ddl != null) { + transform = transform.withDdlString(ddl); + } + + // Check to see if we autoload or not + Boolean autoload = config.getBoolean("autoload"); + if (autoload != null && autoload) { + transform = transform.withAutoLoading(true); + } else { + transform = transform.withAutoLoading(false); + + // Add any user specified table providers from the set of available tableproviders. + Map tableProviders = new HashMap<>(); + ServiceLoader.load(TableProvider.class) + .forEach( + (provider) -> { + tableProviders.put(provider.getTableType(), provider); + }); + Collection tableproviderList = config.getArray("tableproviders"); + if (tableproviderList != null) { + for (Object nameObj : tableproviderList) { + if (nameObj != null) { // This actually could in theory be null... + TableProvider p = tableProviders.get(nameObj); + if (p != null) { // TODO: We ignore tableproviders that don't exist, we could change + // that. + transform = transform.withTableProvider(p.getTableType(), p); } } } - - // TODO: Process query parameters. This is not necessary for Syndeo GA but would be - // really nice to have. - - // TODO: See about reimplementing a correct version of SqlTransform - ErrorCapture errors = new ErrorCapture(); - PCollection output = input.apply(transform.withErrorsTransformer(errors)); - - // TODO: One possibility for capturing the required tables would be to inject a - // tableprovider - // that we control and see which tables are requested during expansion. We could then - // modify the output schema to reflect these inputs via options for better validation. - - List> errorList = errors.getInputs(); - if (errorList.size() == 0) { - PCollection emptyErrors = - input - .getPipeline() - .apply(Create.empty(BeamSqlRelUtils.getErrorRowSchema(Schema.of()))); - return PCollectionRowTuple.of("output", output, "errors", emptyErrors); - } else if (errorList.size() == 1) { - return PCollectionRowTuple.of("output", output, "errors", errorList.get(0)); - } else { - throw new UnsupportedOperationException( - "SqlTransform currently only supports a single dead letter queue collection"); - } } - }; + } + + // TODO: Process query parameters. This is not necessary for Syndeo GA but would be + // really nice to have. + + // TODO: See about reimplementing a correct version of SqlTransform + ErrorCapture errors = new ErrorCapture(); + PCollection output = input.apply(transform.withErrorsTransformer(errors)); + + // TODO: One possibility for capturing the required tables would be to inject a + // tableprovider + // that we control and see which tables are requested during expansion. We could then + // modify the output schema to reflect these inputs via options for better validation. + + List> errorList = errors.getInputs(); + if (errorList.size() == 0) { + PCollection emptyErrors = + input.getPipeline().apply(Create.empty(BeamSqlRelUtils.getErrorRowSchema(Schema.of()))); + return PCollectionRowTuple.of("output", output, "errors", emptyErrors); + } else if (errorList.size() == 1) { + return PCollectionRowTuple.of("output", output, "errors", errorList.get(0)); + } else { + throw new UnsupportedOperationException( + "SqlTransform currently only supports a single dead letter queue collection"); + } } } } diff --git a/sdks/java/extensions/sql/hcatalog/build.gradle b/sdks/java/extensions/sql/hcatalog/build.gradle index dae8f1941aadf..e8abf21b7c3e2 100644 --- a/sdks/java/extensions/sql/hcatalog/build.gradle +++ b/sdks/java/extensions/sql/hcatalog/build.gradle @@ -41,8 +41,7 @@ dependencies { implementation project(":sdks:java:extensions:sql") implementation project(":sdks:java:io:hcatalog") implementation project(":sdks:java:core") - implementation "com.alibaba:fastjson:1.2.69" - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre testImplementation project(":sdks:java:io:hcatalog").sourceSets.test.output // Needed for HCatalogTableProvider tests, diff --git a/sdks/java/extensions/sql/hcatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/DatabaseProvider.java b/sdks/java/extensions/sql/hcatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/DatabaseProvider.java index 29eb8eacb8e9f..925cca4bcabb4 100644 --- a/sdks/java/extensions/sql/hcatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/DatabaseProvider.java +++ b/sdks/java/extensions/sql/hcatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/DatabaseProvider.java @@ -17,14 +17,13 @@ */ package org.apache.beam.sdk.extensions.sql.meta.provider.hcatalog; -import com.alibaba.fastjson.JSONObject; import java.util.Map; import org.apache.beam.sdk.extensions.sql.meta.BeamSqlTable; import org.apache.beam.sdk.extensions.sql.meta.Table; import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider; import org.apache.beam.sdk.io.hcatalog.HCatalogBeamSchema; import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; import org.checkerframework.checker.nullness.qual.Nullable; /** @@ -80,7 +79,6 @@ public Map getTables() { .schema(tableSchema.get()) .name(table) .location("") - .properties(new JSONObject()) .comment("") .type("hcatalog") .build(); diff --git a/sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java b/sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java index f62973cc6c719..daf14e36d7eae 100644 --- a/sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java +++ b/sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java @@ -43,7 +43,7 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.commons.lang.SystemUtils; import org.junit.AfterClass; import org.junit.BeforeClass; diff --git a/sdks/java/extensions/sql/jdbc/build.gradle b/sdks/java/extensions/sql/jdbc/build.gradle index 151a4c3a462ab..41fddce7116ab 100644 --- a/sdks/java/extensions/sql/jdbc/build.gradle +++ b/sdks/java/extensions/sql/jdbc/build.gradle @@ -35,7 +35,7 @@ dependencies { implementation "jline:jline:2.14.6" permitUnusedDeclared "jline:jline:2.14.6" // BEAM-11761 implementation "sqlline:sqlline:1.4.0" - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.vendored_calcite_1_28_0 permitUnusedDeclared library.java.vendored_calcite_1_28_0 testImplementation project(path: ":sdks:java:io:google-cloud-platform", configuration: "testRuntimeMigration") diff --git a/sdks/java/extensions/sql/jdbc/src/main/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLine.java b/sdks/java/extensions/sql/jdbc/src/main/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLine.java index 2be276578e2dc..ac049608ebcbd 100644 --- a/sdks/java/extensions/sql/jdbc/src/main/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLine.java +++ b/sdks/java/extensions/sql/jdbc/src/main/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLine.java @@ -26,7 +26,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; import org.checkerframework.checker.nullness.qual.Nullable; import sqlline.SqlLine; import sqlline.SqlLine.Status; diff --git a/sdks/java/extensions/sql/jdbc/src/test/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLineIT.java b/sdks/java/extensions/sql/jdbc/src/test/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLineIT.java index 365d65f9fe202..e41016d87a549 100644 --- a/sdks/java/extensions/sql/jdbc/src/test/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLineIT.java +++ b/sdks/java/extensions/sql/jdbc/src/test/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLineIT.java @@ -41,8 +41,8 @@ import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage; import org.apache.beam.sdk.io.gcp.pubsub.TestPubsub; import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.hamcrest.collection.IsIn; import org.joda.time.Duration; import org.junit.After; diff --git a/sdks/java/extensions/sql/perf-tests/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryIOPushDownIT.java b/sdks/java/extensions/sql/perf-tests/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryIOPushDownIT.java index 4de32cd1fdeed..ff3e62f551bdf 100644 --- a/sdks/java/extensions/sql/perf-tests/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryIOPushDownIT.java +++ b/sdks/java/extensions/sql/perf-tests/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryIOPushDownIT.java @@ -48,7 +48,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelOptRule; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSet; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlSeekableTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlSeekableTable.java index 95f4b7f47f174..4dc9bd5777ff6 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlSeekableTable.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/BeamSqlSeekableTable.java @@ -19,6 +19,9 @@ import java.io.Serializable; import java.util.List; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.values.Row; /** @@ -26,12 +29,22 @@ * FROM FACT_TABLE JOIN LOOKUP_TABLE ON ...}. */ public interface BeamSqlSeekableTable extends Serializable { - /** prepare the instance. */ - default void setUp() {}; + /** + * prepare the instance. + * + * @param joinSubsetType joining subset schema + */ + default void setUp(Schema joinSubsetType) {} + + default void startBundle( + DoFn.StartBundleContext context, PipelineOptions pipelineOptions) {} + + default void finishBundle( + DoFn.FinishBundleContext context, PipelineOptions pipelineOptions) {} /** return a list of {@code Row} with given key set. */ List seekRow(Row lookupSubRow); /** cleanup resources of the instance. */ - default void tearDown() {}; + default void tearDown() {} } diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/SqlTransform.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/SqlTransform.java index b8d5af252a8f6..07103f2ab7145 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/SqlTransform.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/SqlTransform.java @@ -44,9 +44,9 @@ import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/TableNameExtractionUtils.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/TableNameExtractionUtils.java index 5a6ec5e26a3ff..0f50499926b6b 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/TableNameExtractionUtils.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/TableNameExtractionUtils.java @@ -30,7 +30,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlNode; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlSelect; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlSetOperator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/TableUtils.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/TableUtils.java new file mode 100644 index 0000000000000..2e52a1bbf4228 --- /dev/null +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/TableUtils.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.sql; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.json.JsonReadFeature; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.Map; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; + +public class TableUtils { + private static final ObjectMapper objectMapper = + JsonMapper.builder() + .enable( + JsonReadFeature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER, + JsonReadFeature.ALLOW_JAVA_COMMENTS, + JsonReadFeature.ALLOW_MISSING_VALUES, + JsonReadFeature.ALLOW_NON_NUMERIC_NUMBERS, + JsonReadFeature.ALLOW_LEADING_ZEROS_FOR_NUMBERS, + JsonReadFeature.ALLOW_SINGLE_QUOTES, + JsonReadFeature.ALLOW_TRAILING_COMMA, + JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS, + JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES) + .build(); + + private TableUtils() { + // nothing here + } + + @VisibleForTesting + public static ObjectMapper getObjectMapper() { + return objectMapper; + } + + public static ObjectNode emptyProperties() { + return objectMapper.createObjectNode(); + } + + public static ObjectNode parseProperties(String json) { + try { + return (ObjectNode) objectMapper.readTree(json); + } catch (JsonProcessingException e) { + throw new RuntimeException("illegal table properties: " + json); + } + } + + public static Map convertNode2Map(JsonNode jsonNode) { + return objectMapper.convertValue(jsonNode, new TypeReference>() {}); + } +} diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamCalciteSchemaFactory.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamCalciteSchemaFactory.java index 2324c5b11b7b1..2828286f4db6a 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamCalciteSchemaFactory.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamCalciteSchemaFactory.java @@ -36,7 +36,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.SchemaPlus; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.SchemaVersion; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.Table; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * Factory classes that Calcite uses to create initial schema for JDBC connection. diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamCalciteTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamCalciteTable.java index b74db1903a9e5..bd968ad2cb062 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamCalciteTable.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamCalciteTable.java @@ -41,7 +41,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.ModifiableTable; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.SchemaPlus; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.TranslatableTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** Adapter from {@link BeamSqlTable} to a calcite Table. */ @SuppressWarnings({ diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlEnv.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlEnv.java index 3cdc69f4cd68c..748f32eec3de5 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlEnv.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlEnv.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql.impl; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.sql.SQLException; import java.util.AbstractMap.SimpleEntry; @@ -47,7 +47,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.Function; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; /** * Contains the metadata of tables/UDF functions, and exposes APIs to diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlPipelineOptionsRegistrar.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlPipelineOptionsRegistrar.java index 7b4fbe40332e9..6d05b57fb3c97 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlPipelineOptionsRegistrar.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlPipelineOptionsRegistrar.java @@ -20,7 +20,7 @@ import com.google.auto.service.AutoService; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** {@link AutoService} registrar for {@link BeamSqlPipelineOptions}. */ @AutoService(PipelineOptionsRegistrar.class) diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamTableStatistics.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamTableStatistics.java index f4d9c68e0a932..7aaaedd5e3a74 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamTableStatistics.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamTableStatistics.java @@ -27,7 +27,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelReferentialConstraint; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.Statistic; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** This class stores row count statistics. */ @Internal diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/CalciteQueryPlanner.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/CalciteQueryPlanner.java index adc4695846cba..bd0e7ac008952 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/CalciteQueryPlanner.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/CalciteQueryPlanner.java @@ -66,8 +66,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSet; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.ValidationException; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.BuiltInMethod; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/JavaUdfLoader.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/JavaUdfLoader.java index a44b41ef2d227..ab4b86660a6ed 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/JavaUdfLoader.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/JavaUdfLoader.java @@ -41,11 +41,11 @@ import org.apache.beam.sdk.io.FileSystems; import org.apache.beam.sdk.io.fs.ResourceId; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.commons.codec.digest.DigestUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/JdbcConnection.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/JdbcConnection.java index e9c13dcab03cf..19a55441c8d51 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/JdbcConnection.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/JdbcConnection.java @@ -27,7 +27,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.jdbc.CalciteConnection; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.jdbc.CalciteSchema; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.SchemaPlus; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/LazyAggregateCombineFn.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/LazyAggregateCombineFn.java index 81e7d2c9b8aff..81efa759fde3d 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/LazyAggregateCombineFn.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/LazyAggregateCombineFn.java @@ -28,9 +28,9 @@ import org.apache.beam.sdk.coders.CoderRegistry; import org.apache.beam.sdk.extensions.sql.udf.AggregateFn; import org.apache.beam.sdk.transforms.Combine; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * {@link org.apache.beam.sdk.transforms.Combine.CombineFn} that wraps an {@link AggregateFn}. The diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/QueryPlanner.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/QueryPlanner.java index 609b7dc39e5db..a6e6405e1c3de 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/QueryPlanner.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/QueryPlanner.java @@ -24,8 +24,8 @@ import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlNode; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * An interface that planners should implement to convert sql statement to {@link BeamRelNode} or diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/ScalarFunctionImpl.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/ScalarFunctionImpl.java index 412826f32ce69..b87635efcdaa5 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/ScalarFunctionImpl.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/ScalarFunctionImpl.java @@ -44,8 +44,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.ImplementableFunction; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.ScalarFunction; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlOperatorBinding; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMultimap; /** * Beam-customized version from {@link diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/TableName.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/TableName.java index 759bf4124df65..f69918e2c58c3 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/TableName.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/TableName.java @@ -18,9 +18,9 @@ package org.apache.beam.sdk.extensions.sql.impl; import static java.util.stream.Collectors.toList; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.util.Collections; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/TableResolutionUtils.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/TableResolutionUtils.java index 5b1bf38aadafd..b8081d0c33122 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/TableResolutionUtils.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/TableResolutionUtils.java @@ -19,7 +19,7 @@ import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.sql.SQLException; import java.util.List; @@ -30,7 +30,7 @@ import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.jdbc.CalciteSchema; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Utils to wire up the custom table resolution into Calcite's planner. */ @SuppressWarnings({ diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/UdfImplReflectiveFunctionBase.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/UdfImplReflectiveFunctionBase.java index 7dbc1f5e6a4c7..a37a5b69eaf9c 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/UdfImplReflectiveFunctionBase.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/UdfImplReflectiveFunctionBase.java @@ -30,7 +30,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.FunctionParameter; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.impl.ReflectiveFunctionBase; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ReflectUtil; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Beam-customized version from {@link ReflectiveFunctionBase}, to address BEAM-5921. */ @SuppressWarnings({ diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/cep/CEPOperator.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/cep/CEPOperator.java index 2d84d1596dba5..14cff75875d97 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/cep/CEPOperator.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/cep/CEPOperator.java @@ -21,7 +21,7 @@ import java.util.Map; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlOperator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * The {@code CEPOperator} records the operators (i.e. functions) in the {@code DEFINE} clause of diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlColumnDeclaration.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlColumnDeclaration.java index ee88e3ccae949..bf8907b9d0997 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlColumnDeclaration.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlColumnDeclaration.java @@ -27,7 +27,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlSpecialOperator; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlWriter; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Parse tree for column. */ public class SqlColumnDeclaration extends SqlCall { diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlCreateExternalTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlCreateExternalTable.java index c1bfecd932b1e..e44a152eab04d 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlCreateExternalTable.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlCreateExternalTable.java @@ -17,13 +17,12 @@ */ package org.apache.beam.sdk.extensions.sql.impl.parser; -import static com.alibaba.fastjson.JSON.parseObject; import static org.apache.beam.sdk.schemas.Schema.toSchema; import static org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Static.RESOURCE; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; -import com.alibaba.fastjson.JSONObject; import java.util.List; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.impl.BeamCalciteSchema; import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils; import org.apache.beam.sdk.extensions.sql.meta.Table; @@ -159,8 +158,8 @@ private Table toTable() { .location(SqlDdlNodes.getString(location)) .properties( (tblProperties == null) - ? new JSONObject() - : parseObject(SqlDdlNodes.getString(tblProperties))) + ? TableUtils.emptyProperties() + : TableUtils.parseProperties(SqlDdlNodes.getString(tblProperties))) .build(); } } diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlCreateFunction.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlCreateFunction.java index 015a87e337494..0378be8b9a742 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlCreateFunction.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlCreateFunction.java @@ -43,7 +43,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlWriter; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.parser.SqlParserPos; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Pair; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Parse tree for {@code CREATE FUNCTION} statement. */ public class SqlCreateFunction extends SqlCreate implements BeamSqlParser.ExecutableStatement { diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlDropObject.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlDropObject.java index 22c2fd7d0581f..16e2e536eaa55 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlDropObject.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlDropObject.java @@ -30,7 +30,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlUtil; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlWriter; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * Base class for parse trees of {@code DROP TABLE}, {@code DROP VIEW} and {@code DROP MATERIALIZED diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamRuleSets.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamRuleSets.java index 612db782afee6..9851b2fcbf21f 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamRuleSets.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/planner/BeamRuleSets.java @@ -49,7 +49,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.rules.PruneEmptyRules; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSet; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * {@link RuleSet} used in {@code BeamQueryPlanner}. It translates a standard Calcite {@link diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamAggregationRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamAggregationRel.java index 0177fdb4d02cd..a1880f6cb8c8b 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamAggregationRel.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamAggregationRel.java @@ -19,7 +19,7 @@ import static java.util.stream.Collectors.toList; import static org.apache.beam.sdk.values.PCollection.IsBounded.BOUNDED; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.Serializable; import java.util.List; @@ -60,7 +60,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Aggregate; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.AggregateCall; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamCalcRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamCalcRel.java index fe328e191d004..d647027a9ae51 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamCalcRel.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamCalcRel.java @@ -19,7 +19,7 @@ import static org.apache.beam.sdk.schemas.Schema.Field; import static org.apache.beam.sdk.schemas.Schema.FieldType; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.lang.reflect.InvocationTargetException; @@ -103,8 +103,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.validate.SqlConformance; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.validate.SqlConformanceEnum; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.validate.SqlUserDefinedFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; import org.codehaus.commons.compiler.CompileException; import org.codehaus.janino.ScriptEvaluator; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamEnumerableConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamEnumerableConverter.java index 4c8cb6f38f2f3..7866d86971f96 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamEnumerableConverter.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamEnumerableConverter.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql.impl.rel; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.time.LocalDate; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSinkRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSinkRel.java index 0c270265d8938..7c292e7ec4ea8 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSinkRel.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSinkRel.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql.impl.rel; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.List; import java.util.Map; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSourceRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSourceRel.java index 43b63b5f29f7a..989172769992a 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSourceRel.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamIOSourceRel.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql.impl.rel; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.List; import java.util.Map; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRel.java index aef47b9e953af..d6e2e4fe27b19 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRel.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamJoinRel.java @@ -41,8 +41,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLiteral; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Pair; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * An abstract {@code BeamRelNode} to implement Join Rels. diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamMatchRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamMatchRel.java index 58da090aad777..d1b118b3f295d 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamMatchRel.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamMatchRel.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.extensions.sql.impl.rel; import static org.apache.beam.sdk.extensions.sql.impl.cep.CEPUtils.makeOrderKeysFromCollation; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.ArrayList; import java.util.List; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamPushDownIOSourceRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamPushDownIOSourceRel.java index 37c1f2515adf1..06493333b12bf 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamPushDownIOSourceRel.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamPushDownIOSourceRel.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql.impl.rel; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.List; import java.util.Map; @@ -39,7 +39,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelOptTable; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelWriter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; public class BeamPushDownIOSourceRel extends BeamIOSourceRel { private final List usedFields; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSetOperatorRelBase.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSetOperatorRelBase.java index 2c612abb390af..cad845c2b3a94 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSetOperatorRelBase.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSetOperatorRelBase.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql.impl.rel; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.Serializable; import org.apache.beam.sdk.extensions.sql.impl.transform.BeamSetOperatorsTransforms; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSideInputJoinRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSideInputJoinRel.java index 8a16d13bc0926..ca9eceb78fcf7 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSideInputJoinRel.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSideInputJoinRel.java @@ -40,7 +40,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.JoinRelType; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Pair; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; /** * A {@code BeamJoinRel} which does sideinput Join diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSortRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSortRel.java index 7376845b26965..d94f228c1f99d 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSortRel.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSortRel.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.extensions.sql.impl.rel; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.Serializable; import java.math.BigDecimal; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamTableFunctionScanRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamTableFunctionScanRel.java index 4df31f2bc7293..a975f7fdabdc7 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamTableFunctionScanRel.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamTableFunctionScanRel.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql.impl.rel; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.lang.reflect.Type; import java.util.ArrayList; @@ -63,7 +63,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexInputRef; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLiteral; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Duration; /** diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUncollectRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUncollectRel.java index 3d99a42641374..55dc9afe4e177 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUncollectRel.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUncollectRel.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql.impl.rel; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Collections; import org.apache.beam.sdk.extensions.sql.impl.planner.BeamCostModel; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUnnestRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUnnestRel.java index 69d4b182be5f7..65f08cc10ce17 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUnnestRel.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamUnnestRel.java @@ -45,7 +45,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Uncollect; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataType; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.validate.SqlValidatorUtil; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamValuesRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamValuesRel.java index ca0139c938d85..6cd1716d5c5d7 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamValuesRel.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamValuesRel.java @@ -20,7 +20,7 @@ import static java.util.stream.Collectors.toList; import static org.apache.beam.sdk.extensions.sql.impl.schema.BeamTableUtils.autoCastField; import static org.apache.beam.sdk.values.Row.toRow; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.List; import java.util.Map; @@ -42,7 +42,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Values; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataType; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLiteral; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * {@code BeamRelNode} to replace a {@code Values} node. diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamWindowRel.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamWindowRel.java index 906258b164a50..de7ef2a29f697 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamWindowRel.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamWindowRel.java @@ -35,6 +35,7 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.transforms.Combine; import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.GroupByKey; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.KV; @@ -52,7 +53,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexInputRef; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLiteral; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; /** * {@code BeamRelNode} to replace a {@code Window} node. @@ -254,7 +255,9 @@ public PCollection expand(PCollectionList input) { org.apache.beam.sdk.schemas.transforms.Group.ByFields myg = org.apache.beam.sdk.schemas.transforms.Group.byFieldIds(af.partitionKeys); PCollection>> partitionBy = - inputData.apply(prefix + "partitionBy", myg.getToKvs()); + inputData + .apply(prefix + "partitionByKV", myg.getToKV()) + .apply(prefix + "partitionByGK", GroupByKey.create()); partitioned = partitionBy .apply(prefix + "selectOnlyValues", ParDo.of(new SelectOnlyValues())) diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/CalcRelSplitter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/CalcRelSplitter.java index e254ba60568ac..4cad8f6716901 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/CalcRelSplitter.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/CalcRelSplitter.java @@ -50,8 +50,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.graph.DefaultEdge; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.graph.DirectedGraph; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.graph.TopologicalOrderIterator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Ints; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Ints; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamIOPushDownRule.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamIOPushDownRule.java index 9c06d1fe0fc8a..5abce12fa2005 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamIOPushDownRule.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamIOPushDownRule.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql.impl.rule; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.ArrayDeque; import java.util.ArrayList; @@ -57,7 +57,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RelBuilder; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RelBuilderFactory; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Pair; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; @SuppressWarnings({ "nullness" // TODO(https://github.com/apache/beam/issues/20497) diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamTableFunctionScanRule.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamTableFunctionScanRule.java index 40bcb74a6dc36..9d73f8abc4ff6 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamTableFunctionScanRule.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamTableFunctionScanRule.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql.impl.rule; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.ArrayList; import java.util.Collections; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamUnnestRule.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamUnnestRule.java index 41dd38431b4c1..e2fa2bc6f5867 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamUnnestRule.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamUnnestRule.java @@ -31,7 +31,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.logical.LogicalProject; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexFieldAccess; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * A {@code ConverterRule} to replace {@link Correlate} {@link Uncollect} with {@link diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamBuiltinAggregations.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamBuiltinAggregations.java index dda884316db15..3fc299bd5a336 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamBuiltinAggregations.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamBuiltinAggregations.java @@ -45,9 +45,9 @@ import org.apache.beam.sdk.transforms.Sample; import org.apache.beam.sdk.transforms.Sum; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; /** Built-in aggregations functions for COUNT/MAX/MIN/SUM/AVG/VAR_POP/VAR_SAMP. */ diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamBuiltinAnalyticFunctions.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamBuiltinAnalyticFunctions.java index 07ce01ebd3d55..413093d416d83 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamBuiltinAnalyticFunctions.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamBuiltinAnalyticFunctions.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.transforms.Combine; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** Built-in Analytic Functions for the aggregation analytics functionality. */ @SuppressWarnings({ diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamJoinTransforms.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamJoinTransforms.java index a30822de15199..d25f98729bd4c 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamJoinTransforms.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamJoinTransforms.java @@ -26,6 +26,7 @@ import org.apache.beam.sdk.extensions.sql.impl.utils.SerializableRexFieldAccess; import org.apache.beam.sdk.extensions.sql.impl.utils.SerializableRexInputRef; import org.apache.beam.sdk.extensions.sql.impl.utils.SerializableRexNode; +import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.schemas.FieldAccessDescriptor; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.transforms.DoFn; @@ -152,7 +153,21 @@ public PCollection expand(PCollection input) { new DoFn() { @Setup public void setup() { - seekableTable.setUp(); + seekableTable.setUp(joinSubsetType); + } + + @StartBundle + public void startBundle( + DoFn.StartBundleContext context, + PipelineOptions pipelineOptions) { + seekableTable.startBundle(context, pipelineOptions); + } + + @FinishBundle + public void finishBundle( + DoFn.FinishBundleContext context, + PipelineOptions pipelineOptions) { + seekableTable.finishBundle(context, pipelineOptions); } @ProcessElement diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSetOperatorsTransforms.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSetOperatorsTransforms.java index 3c18be0a639dc..21405c7a00346 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSetOperatorsTransforms.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamSetOperatorsTransforms.java @@ -22,7 +22,7 @@ import org.apache.beam.sdk.transforms.SimpleFunction; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** Collections of {@code PTransform} and {@code DoFn} used to perform Set operations. */ @SuppressWarnings({ diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/agg/CovarianceFn.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/agg/CovarianceFn.java index 1bd480d9edd97..4727023c83207 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/agg/CovarianceFn.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/agg/CovarianceFn.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql.impl.transform.agg; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.math.BigDecimal; import java.math.MathContext; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/BigDecimalConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/BigDecimalConverter.java index 0f2340d40f80c..287ed72110bb0 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/BigDecimalConverter.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/BigDecimalConverter.java @@ -21,7 +21,7 @@ import java.util.Map; import org.apache.beam.sdk.schemas.Schema.TypeName; import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * Provides converters from {@link BigDecimal} to other numeric types based on the input {@link diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/CalciteUtils.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/CalciteUtils.java index 35a766c0cd2b5..610dd4f6888a9 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/CalciteUtils.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/utils/CalciteUtils.java @@ -35,9 +35,9 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeField; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlTypeNameSpec; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.type.SqlTypeName; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableBiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableBiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Instant; import org.joda.time.base.AbstractInstant; import org.slf4j.Logger; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/Table.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/Table.java index 1fb5d9a510b2f..23f301dd24550 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/Table.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/Table.java @@ -17,9 +17,10 @@ */ package org.apache.beam.sdk.extensions.sql.meta; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.auto.value.AutoValue; import java.io.Serializable; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.schemas.Schema; import org.checkerframework.checker.nullness.qual.Nullable; @@ -37,12 +38,12 @@ public abstract class Table implements Serializable { public abstract @Nullable String getLocation(); - public abstract JSONObject getProperties(); + public abstract ObjectNode getProperties(); public abstract Builder toBuilder(); public static Builder builder() { - return new AutoValue_Table.Builder().properties(new JSONObject()); + return new AutoValue_Table.Builder().properties(TableUtils.emptyProperties()); } /** Builder class for {@link Table}. */ @@ -58,7 +59,7 @@ public abstract static class Builder { public abstract Builder location(String location); - public abstract Builder properties(JSONObject properties); + public abstract Builder properties(ObjectNode properties); public abstract Table build(); } diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/ReadOnlyTableProvider.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/ReadOnlyTableProvider.java index 78525ccd5f350..ad8ba3ead5ce1 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/ReadOnlyTableProvider.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/ReadOnlyTableProvider.java @@ -21,7 +21,7 @@ import org.apache.beam.sdk.extensions.sql.meta.BeamSqlTable; import org.apache.beam.sdk.extensions.sql.meta.Table; import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * A {@code ReadOnlyTableProvider} provides in-memory read only set of {@code BeamSqlTable diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/SchemaIOTableProviderWrapper.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/SchemaIOTableProviderWrapper.java index 5089753641c5a..1e2629353bc85 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/SchemaIOTableProviderWrapper.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/SchemaIOTableProviderWrapper.java @@ -19,8 +19,8 @@ import static org.apache.beam.sdk.util.RowJsonUtils.newObjectMapperWith; -import com.alibaba.fastjson.JSONObject; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.Serializable; import java.util.List; import org.apache.beam.sdk.annotations.Internal; @@ -46,7 +46,7 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** A general {@link TableProvider} for IOs for consumption by Beam SQL. */ @Internal @@ -66,7 +66,7 @@ public String getTableType() { @Override public BeamSqlTable buildBeamSqlTable(Table tableDefinition) { - JSONObject tableProperties = tableDefinition.getProperties(); + ObjectNode tableProperties = tableDefinition.getProperties(); try { RowJson.RowJsonDeserializer deserializer = @@ -84,8 +84,7 @@ public BeamSqlTable buildBeamSqlTable(Table tableDefinition) { } catch (InvalidConfigurationException | InvalidSchemaException e) { throw new InvalidTableException(e.getMessage()); } catch (JsonProcessingException e) { - throw new AssertionError( - "Failed to re-parse TBLPROPERTIES JSON " + tableProperties.toString()); + throw new AssertionError("Failed to re-parse TBLPROPERTIES JSON " + tableProperties); } } diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BeamBigQuerySqlDialect.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BeamBigQuerySqlDialect.java index 7d3630bd579ef..91ba2c708d288 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BeamBigQuerySqlDialect.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BeamBigQuerySqlDialect.java @@ -30,8 +30,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlTimestampLiteral; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlWriter; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.dialect.BigQuerySqlDialect; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; // TODO(CALCITE-3381): some methods below can be deleted after updating vendor Calcite version. // Calcite v1_20_0 does not have type translation implemented, but later (unreleased) versions do. diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BeamSqlUnparseContext.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BeamSqlUnparseContext.java index ff9b4e694054c..5ab6bbcacec3c 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BeamSqlUnparseContext.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BeamSqlUnparseContext.java @@ -48,7 +48,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.type.SqlTypeName; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.BitString; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.TimestampString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; @SuppressWarnings({ diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryFilter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryFilter.java index 0b06a0a7dab9d..36450e3914ccd 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryFilter.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryFilter.java @@ -40,7 +40,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.type.SqlTypeName; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; @SuppressWarnings({ "nullness" // TODO(https://github.com/apache/beam/issues/20497) diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTable.java index 7f51846b24761..1898c28f670c3 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTable.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTable.java @@ -53,8 +53,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlNode; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,11 +81,11 @@ class BigQueryTable extends SchemaBaseBeamTable implements Serializable { this.conversionOptions = options; this.bqLocation = table.getLocation(); - if (table.getProperties().containsKey(METHOD_PROPERTY)) { + if (table.getProperties().has(METHOD_PROPERTY)) { List validMethods = Arrays.stream(Method.values()).map(Enum::toString).collect(Collectors.toList()); // toUpperCase should make it case-insensitive - String selectedMethod = table.getProperties().getString(METHOD_PROPERTY).toUpperCase(); + String selectedMethod = table.getProperties().get(METHOD_PROPERTY).asText().toUpperCase(); if (validMethods.contains(selectedMethod)) { method = Method.valueOf(selectedMethod); @@ -105,12 +105,12 @@ class BigQueryTable extends SchemaBaseBeamTable implements Serializable { LOG.info("BigQuery method is set to: {}", method); - if (table.getProperties().containsKey(WRITE_DISPOSITION_PROPERTY)) { + if (table.getProperties().has(WRITE_DISPOSITION_PROPERTY)) { List validWriteDispositions = Arrays.stream(WriteDisposition.values()).map(Enum::toString).collect(Collectors.toList()); // toUpperCase should make it case-insensitive String selectedWriteDisposition = - table.getProperties().getString(WRITE_DISPOSITION_PROPERTY).toUpperCase(); + table.getProperties().get(WRITE_DISPOSITION_PROPERTY).asText().toUpperCase(); if (validWriteDispositions.contains(selectedWriteDisposition)) { writeDisposition = WriteDisposition.valueOf(selectedWriteDisposition); diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProvider.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProvider.java index 84a446a128aec..2f6e6d1d88648 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProvider.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProvider.java @@ -17,9 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql.meta.provider.bigquery; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; - -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.auto.service.AutoService; import org.apache.beam.sdk.extensions.sql.meta.BeamSqlTable; import org.apache.beam.sdk.extensions.sql.meta.Table; @@ -56,10 +54,10 @@ public BeamSqlTable buildBeamSqlTable(Table table) { return new BigQueryTable(table, getConversionOptions(table.getProperties())); } - protected static ConversionOptions getConversionOptions(JSONObject properties) { + protected static ConversionOptions getConversionOptions(ObjectNode properties) { return ConversionOptions.builder() .setTruncateTimestamps( - firstNonNull(properties.getBoolean("truncateTimestamps"), false) + properties.path("truncateTimestamps").asBoolean(false) ? TruncateTimestamps.TRUNCATE : TruncateTimestamps.REJECT) .build(); diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableFilter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableFilter.java index 6920c9a3f8157..f419896f2ed55 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableFilter.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableFilter.java @@ -21,7 +21,7 @@ import static org.apache.beam.sdk.io.gcp.bigtable.RowUtils.KEY; import static org.apache.beam.sdk.io.gcp.bigtable.RowUtils.byteStringUtf8; import static org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind.LIKE; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.bigtable.v2.RowFilter; import java.util.List; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTable.java index 5dd84244ebdc7..60c722d32d2da 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTable.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTable.java @@ -20,10 +20,10 @@ import static java.util.stream.Collectors.toSet; import static org.apache.beam.sdk.io.gcp.bigtable.RowUtils.COLUMNS_MAPPING; import static org.apache.beam.sdk.io.gcp.bigtable.RowUtils.KEY; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps.newHashMap; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets.newHashSet; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps.newHashMap; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets.newHashSet; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.Serializable; import java.util.Collection; import java.util.HashMap; @@ -47,7 +47,7 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; public class BigtableTable extends SchemaBaseBeamTable implements Serializable { // Should match: @@ -84,9 +84,9 @@ public class BigtableTable extends SchemaBaseBeamTable implements Serializable { this.emulatorHost = host; } - JSONObject properties = table.getProperties(); - if (properties.containsKey(COLUMNS_MAPPING)) { - columnsMapping = parseColumnsMapping(properties.getString(COLUMNS_MAPPING)); + ObjectNode properties = table.getProperties(); + if (properties.has(COLUMNS_MAPPING)) { + columnsMapping = parseColumnsMapping(properties.get(COLUMNS_MAPPING).asText()); validateColumnsMapping(columnsMapping, schema); useFlatSchema = true; } diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaCSVTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaCSVTable.java index 51f8719e89e67..87ba7b8c07ea0 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaCSVTable.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaCSVTable.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.commons.csv.CSVFormat; import org.apache.kafka.clients.producer.ProducerRecord; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTable.java index 098995a1d2f2b..f1ec20831a4cd 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTable.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTable.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql.meta.provider.kafka; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Collection; import java.util.HashMap; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProvider.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProvider.java index 74f4dbad5a7cb..06cc4ea0beff6 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProvider.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProvider.java @@ -19,12 +19,14 @@ import static org.apache.beam.sdk.extensions.sql.meta.provider.kafka.Schemas.PAYLOAD_FIELD; import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.auto.service.AutoService; import java.util.List; import java.util.Optional; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.meta.BeamSqlTable; import org.apache.beam.sdk.extensions.sql.meta.Table; import org.apache.beam.sdk.extensions.sql.meta.provider.InMemoryMetaTableProvider; @@ -32,10 +34,10 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer; import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; /** @@ -80,11 +82,11 @@ private static ParsedLocation parseLocation(String location) { return parsed; } - private static List mergeParam(Optional initial, @Nullable List toMerge) { + private static List mergeParam(Optional initial, @Nullable ArrayNode toMerge) { ImmutableList.Builder merged = ImmutableList.builder(); initial.ifPresent(merged::add); if (toMerge != null) { - toMerge.forEach(o -> merged.add(o.toString())); + toMerge.forEach(o -> merged.add(o.asText())); } return merged.build(); } @@ -92,23 +94,23 @@ private static List mergeParam(Optional initial, @Nullable List< @Override public BeamSqlTable buildBeamSqlTable(Table table) { Schema schema = table.getSchema(); - JSONObject properties = table.getProperties(); + ObjectNode properties = table.getProperties(); Optional parsedLocation = Optional.empty(); if (!Strings.isNullOrEmpty(table.getLocation())) { parsedLocation = Optional.of(parseLocation(checkArgumentNotNull(table.getLocation()))); } List topics = - mergeParam(parsedLocation.map(loc -> loc.topic), properties.getJSONArray("topics")); + mergeParam(parsedLocation.map(loc -> loc.topic), (ArrayNode) properties.get("topics")); List allBootstrapServers = mergeParam( parsedLocation.map(loc -> loc.brokerLocation), - properties.getJSONArray("bootstrap_servers")); + (ArrayNode) properties.get("bootstrap_servers")); String bootstrapServers = String.join(",", allBootstrapServers); Optional payloadFormat = - properties.containsKey("format") - ? Optional.of(properties.getString("format")) + properties.has("format") + ? Optional.of(properties.get("format").asText()) : Optional.empty(); if (Schemas.isNestedSchema(schema)) { Optional serializer = @@ -117,7 +119,7 @@ public BeamSqlTable buildBeamSqlTable(Table table) { PayloadSerializers.getSerializer( format, checkArgumentNotNull(schema.getField(PAYLOAD_FIELD).getType().getRowSchema()), - properties.getInnerMap())); + TableUtils.convertNode2Map(properties))); return new NestedPayloadKafkaTable(schema, bootstrapServers, topics, serializer); } else { /* @@ -130,7 +132,8 @@ public BeamSqlTable buildBeamSqlTable(Table table) { return new BeamKafkaCSVTable(schema, bootstrapServers, topics); } PayloadSerializer serializer = - PayloadSerializers.getSerializer(payloadFormat.get(), schema, properties.getInnerMap()); + PayloadSerializers.getSerializer( + payloadFormat.get(), schema, TableUtils.convertNode2Map(properties)); return new PayloadSerializerKafkaTable(schema, bootstrapServers, topics, serializer); } } diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/NestedPayloadKafkaTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/NestedPayloadKafkaTable.java index f6563cb94584b..c2f1ed217763e 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/NestedPayloadKafkaTable.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/NestedPayloadKafkaTable.java @@ -19,7 +19,7 @@ import static org.apache.beam.sdk.schemas.transforms.Cast.castRow; import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Collection; import java.util.List; @@ -35,11 +35,11 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.Headers; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/PayloadSerializerKafkaTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/PayloadSerializerKafkaTable.java index c7a0ae71f40e6..63be5e40a9fe4 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/PayloadSerializerKafkaTable.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/PayloadSerializerKafkaTable.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.kafka.clients.producer.ProducerRecord; public class PayloadSerializerKafkaTable extends BeamKafkaTable { diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/Schemas.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/Schemas.java index f3a35e74ae604..7d474489df8fb 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/Schemas.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/Schemas.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql.meta.provider.kafka; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.Schema.EquivalenceNullablePolicy; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/mongodb/MongoDbTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/mongodb/MongoDbTable.java index 51a6a811d187c..576b623b28ab0 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/mongodb/MongoDbTable.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/mongodb/MongoDbTable.java @@ -62,8 +62,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.type.SqlTypeName; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.bson.Document; import org.bson.conversions.Bson; import org.bson.json.JsonMode; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/PubsubLiteTableProvider.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/PubsubLiteTableProvider.java index a11ab8b421ccc..cdc6a4c05dffb 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/PubsubLiteTableProvider.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/PubsubLiteTableProvider.java @@ -18,15 +18,16 @@ package org.apache.beam.sdk.extensions.sql.meta.provider.pubsublite; import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.auto.service.AutoService; import com.google.auto.value.AutoOneOf; import com.google.cloud.pubsublite.SubscriptionPath; import com.google.cloud.pubsublite.TopicPath; import com.google.cloud.pubsublite.proto.PubSubMessage; import java.util.Optional; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.meta.BeamSqlTable; import org.apache.beam.sdk.extensions.sql.meta.Table; import org.apache.beam.sdk.extensions.sql.meta.provider.InMemoryMetaTableProvider; @@ -78,15 +79,16 @@ public String getTableType() { return "pubsublite"; } - private static Optional getSerializer(Schema schema, JSONObject properties) { + private static Optional getSerializer(Schema schema, ObjectNode properties) { if (schema.getField("payload").getType().equals(FieldType.BYTES)) { checkArgument( - !properties.containsKey("format"), + !properties.has("format"), "Must not set the 'format' property if not unpacking payload."); return Optional.empty(); } - String format = properties.containsKey("format") ? properties.getString("format") : "json"; - return Optional.of(PayloadSerializers.getSerializer(format, schema, properties.getInnerMap())); + String format = properties.path("format").asText("json"); + return Optional.of( + PayloadSerializers.getSerializer(format, schema, TableUtils.convertNode2Map(properties))); } private static void checkFieldHasType(Field field, FieldType type) { @@ -165,9 +167,9 @@ private static RowHandler getRowHandler( private static PTransform, PCollection> addDlqIfPresent( - SimpleFunction transform, JSONObject properties) { - if (properties.containsKey("deadLetterQueue")) { - return new DeadLetteredTransform<>(transform, properties.getString("deadLetterQueue")); + SimpleFunction transform, ObjectNode properties) { + if (properties.has("deadLetterQueue")) { + return new DeadLetteredTransform<>(transform, properties.get("deadLetterQueue").asText()); } return MapElements.via(transform); } diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/RowHandler.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/RowHandler.java index 2a4250bc805ab..15a0b47326573 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/RowHandler.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/RowHandler.java @@ -19,7 +19,7 @@ import static org.apache.beam.sdk.schemas.transforms.Cast.castRow; import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.cloud.pubsublite.proto.AttributeValues; import com.google.cloud.pubsublite.proto.PubSubMessage; @@ -36,8 +36,8 @@ import org.apache.beam.sdk.schemas.Schema.TypeName; import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Instant; import org.joda.time.ReadableDateTime; diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/seqgen/GenerateSequenceTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/seqgen/GenerateSequenceTable.java index eea7bc474681f..e1924147b9821 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/seqgen/GenerateSequenceTable.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/seqgen/GenerateSequenceTable.java @@ -44,8 +44,8 @@ class GenerateSequenceTable extends SchemaBaseBeamTable implements Serializable GenerateSequenceTable(Table table) { super(TABLE_SCHEMA); - if (table.getProperties().containsKey("elementsPerSecond")) { - elementsPerSecond = table.getProperties().getInteger("elementsPerSecond"); + if (table.getProperties().has("elementsPerSecond")) { + elementsPerSecond = table.getProperties().get("elementsPerSecond").asInt(); } } diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProvider.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProvider.java index a2a0893bc08db..5e0851a3685e6 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProvider.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProvider.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql.meta.provider.test; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; import java.io.Serializable; @@ -152,10 +152,10 @@ public InMemoryTable(TableWithRows tableWithRows) { // The reason for introducing a property here is to simplify writing unit tests, testing // project and predicate push-down behavior when run separate and together. - if (tableWithRows.table.getProperties().containsKey(PUSH_DOWN_OPTION)) { + if (tableWithRows.table.getProperties().has(PUSH_DOWN_OPTION)) { options = PushDownOptions.valueOf( - tableWithRows.table.getProperties().getString(PUSH_DOWN_OPTION).toUpperCase()); + tableWithRows.table.getProperties().get(PUSH_DOWN_OPTION).asText().toUpperCase()); } else { options = PushDownOptions.NONE; } diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableUtils.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableUtils.java index 3342db35d2a3c..9bf7aebf071e6 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableUtils.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableUtils.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; /** Utility functions for mock classes. */ @SuppressWarnings({"keyfor", "nullness"}) // TODO(https://github.com/apache/beam/issues/20497) diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/TextTableProvider.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/TextTableProvider.java index 708de1a597d3e..3ddd78ab232b4 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/TextTableProvider.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/TextTableProvider.java @@ -22,8 +22,8 @@ import static org.apache.beam.sdk.util.RowJsonUtils.jsonToRow; import static org.apache.beam.sdk.util.RowJsonUtils.newObjectMapperWith; -import com.alibaba.fastjson.JSONObject; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.auto.service.AutoService; import com.google.auto.value.AutoValue; import java.io.Serializable; @@ -51,9 +51,8 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.apache.commons.csv.CSVFormat; import org.checkerframework.checker.nullness.qual.Nullable; @@ -89,9 +88,9 @@ public BeamSqlTable buildBeamSqlTable(Table table) { Schema schema = table.getSchema(); String filePattern = table.getLocation(); - JSONObject properties = table.getProperties(); - String format = MoreObjects.firstNonNull(properties.getString("format"), "csv"); - String deadLetterFile = properties.getString("deadLetterFile"); + ObjectNode properties = table.getProperties(); + String format = properties.path("format").asText("csv"); + String deadLetterFile = properties.path("deadLetterFile").asText(null); // Backwards compatibility: previously "type": "text" meant CSV and "format" was where the // CSV format went. So assume that any other format is the CSV format. @@ -103,7 +102,7 @@ public BeamSqlTable buildBeamSqlTable(Table table) { switch (format) { case "csv": - String specifiedCsvFormat = properties.getString("csvformat"); + String specifiedCsvFormat = properties.path("csvformat").asText(null); CSVFormat csvFormat = specifiedCsvFormat != null ? CSVFormat.valueOf(specifiedCsvFormat) diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/store/InMemoryMetaStore.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/store/InMemoryMetaStore.java index e9f69c8fc7ada..8a892cc2eb731 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/store/InMemoryMetaStore.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/store/InMemoryMetaStore.java @@ -22,7 +22,7 @@ import org.apache.beam.sdk.extensions.sql.meta.BeamSqlTable; import org.apache.beam.sdk.extensions.sql.meta.Table; import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * A {@link MetaStore} which stores the meta info in memory. diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamComplexTypeTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamComplexTypeTest.java index 21f70187c58f7..a98732cba282c 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamComplexTypeTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamComplexTypeTest.java @@ -41,8 +41,8 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Ignore; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslSqlStdOperatorsTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslSqlStdOperatorsTest.java index b5d46bcb9e81f..cfa2df719679d 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslSqlStdOperatorsTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslSqlStdOperatorsTest.java @@ -48,10 +48,10 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlOperator; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslUdfUdafTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslUdfUdafTest.java index 97d8513ca7afe..991d4d260fd96 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslUdfUdafTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslUdfUdafTest.java @@ -50,7 +50,7 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.linq4j.function.Parameter; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.TranslatableTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Instant; import org.junit.Test; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlMapTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlMapTest.java index 76ed40e202a36..4a2ad664b4b3c 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlMapTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlMapTest.java @@ -23,7 +23,7 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlMultipleSchemasTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlMultipleSchemasTest.java index 90cdef1453bbf..cc8d47be3481d 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlMultipleSchemasTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlMultipleSchemasTest.java @@ -32,7 +32,7 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/PubsubToBigqueryIT.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/PubsubToBigqueryIT.java index 97a2fb4536230..35a0f26240090 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/PubsubToBigqueryIT.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/PubsubToBigqueryIT.java @@ -31,8 +31,8 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/TestUtils.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/TestUtils.java index 9bc083b8e7c70..ed717b7f6e941 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/TestUtils.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/TestUtils.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.ArrayList; import java.util.Arrays; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/JdbcDriverTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/JdbcDriverTest.java index 2af4398d20b11..16d5bc253c956 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/JdbcDriverTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/JdbcDriverTest.java @@ -53,7 +53,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.jdbc.CalciteConnection; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.jdbc.CalciteSchema; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.SchemaPlus; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.DateTime; import org.joda.time.Duration; import org.joda.time.ReadableInstant; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/LazyAggregateCombineFnTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/LazyAggregateCombineFnTest.java index bea445a4ef9b3..fda230756c248 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/LazyAggregateCombineFnTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/LazyAggregateCombineFnTest.java @@ -34,7 +34,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.AggregateFunction; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.FunctionParameter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/parser/BeamDDLTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/parser/BeamDDLTest.java index c43caf0f7fd87..704a9d4586e1c 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/parser/BeamDDLTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/parser/BeamDDLTest.java @@ -22,9 +22,10 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.util.stream.Stream; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv; import org.apache.beam.sdk.extensions.sql.impl.ParseException; import org.apache.beam.sdk.extensions.sql.impl.parser.impl.BeamSqlParserImpl; @@ -50,8 +51,8 @@ public void testParseCreateExternalTable_full() throws Exception { TestTableProvider tableProvider = new TestTableProvider(); BeamSqlEnv env = BeamSqlEnv.withTableProvider(tableProvider); - JSONObject properties = new JSONObject(); - JSONArray hello = new JSONArray(); + ObjectNode properties = TableUtils.emptyProperties(); + ArrayNode hello = TableUtils.getObjectMapper().createArrayNode(); hello.add("james"); hello.add("bond"); properties.put("hello", hello); @@ -115,8 +116,8 @@ public void testParseCreateExternalTable_withoutTableComment() throws Exception TestTableProvider tableProvider = new TestTableProvider(); BeamSqlEnv env = BeamSqlEnv.withTableProvider(tableProvider); - JSONObject properties = new JSONObject(); - JSONArray hello = new JSONArray(); + ObjectNode properties = TableUtils.emptyProperties(); + ArrayNode hello = TableUtils.getObjectMapper().createArrayNode(); hello.add("james"); hello.add("bond"); properties.put("hello", hello); @@ -145,7 +146,7 @@ public void testParseCreateExternalTable_withoutTblProperties() throws Exception + "COMMENT 'person table' \n" + "LOCATION '/home/admin/person'\n"); assertEquals( - mockTable("person", "text", "person table", new JSONObject()), + mockTable("person", "text", "person table", TableUtils.emptyProperties()), tableProvider.getTables().get("person")); } @@ -162,7 +163,7 @@ public void testParseCreateExternalTable_withoutLocation() throws Exception { + "COMMENT 'person table' \n"); assertEquals( - mockTable("person", "text", "person table", new JSONObject(), null), + mockTable("person", "text", "person table", TableUtils.emptyProperties(), null), tableProvider.getTables().get("person")); } @@ -180,7 +181,7 @@ public void testParseCreateExternalTable_minimal() throws Exception { .schema( Stream.of(Schema.Field.of("id", CalciteUtils.INTEGER).withNullable(true)) .collect(toSchema())) - .properties(new JSONObject()) + .properties(TableUtils.emptyProperties()) .build(), tableProvider.getTables().get("person")); } @@ -249,12 +250,12 @@ public void unparseAggregateFunction() { sqlWriter.toSqlString().getSql()); } - private static Table mockTable(String name, String type, String comment, JSONObject properties) { + private static Table mockTable(String name, String type, String comment, ObjectNode properties) { return mockTable(name, type, comment, properties, "/home/admin/" + name); } private static Table mockTable( - String name, String type, String comment, JSONObject properties, String location) { + String name, String type, String comment, ObjectNode properties, String location) { return Table.builder() .name(name) diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamCalcRelTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamCalcRelTest.java index 702dd9fa1ddff..5d53ca3640756 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamCalcRelTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamCalcRelTest.java @@ -34,7 +34,7 @@ import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.DateTime; import org.joda.time.Duration; import org.junit.Assert; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSideInputLookupJoinRelTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSideInputLookupJoinRelTest.java index 2e2971ebd6e95..b5fd03045cbc1 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSideInputLookupJoinRelTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamSideInputLookupJoinRelTest.java @@ -34,6 +34,7 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.Row; import org.hamcrest.core.StringContains; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; @@ -47,11 +48,18 @@ public class BeamSideInputLookupJoinRelTest extends BaseRelTest { /** Test table for JOIN-AS-LOOKUP. */ public static class SiteLookupTable extends SchemaBaseBeamTable implements BeamSqlSeekableTable { + private Schema joinSubsetType; public SiteLookupTable(Schema schema) { super(schema); } + @Override + public void setUp(Schema joinSubsetType) { + this.joinSubsetType = joinSubsetType; + Assert.assertNotNull(joinSubsetType); + } + @Override public PCollection.IsBounded isBounded() { return PCollection.IsBounded.BOUNDED; @@ -69,6 +77,7 @@ public POutput buildIOWriter(PCollection input) { @Override public List seekRow(Row lookupSubRow) { + Assert.assertEquals(joinSubsetType, lookupSubRow.getSchema()); if (lookupSubRow.getInt32("site_id") == 2) { return Arrays.asList(Row.withSchema(getSchema()).addValues(2, "SITE1").build()); } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamAggregateProjectMergeRuleTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamAggregateProjectMergeRuleTest.java index 5d13af947777c..593febb9f190f 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamAggregateProjectMergeRuleTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamAggregateProjectMergeRuleTest.java @@ -22,7 +22,7 @@ import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder; import static org.hamcrest.core.IsInstanceOf.instanceOf; -import com.alibaba.fastjson.JSON; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv; import org.apache.beam.sdk.extensions.sql.impl.rel.BeamAggregationRel; import org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel; @@ -150,7 +150,8 @@ private static Table getTable(String name, PushDownOptions options) { .comment(name + " table") .schema(BASIC_SCHEMA) .properties( - JSON.parseObject("{ " + PUSH_DOWN_OPTION + ": " + "\"" + options.toString() + "\" }")) + TableUtils.parseProperties( + "{ " + PUSH_DOWN_OPTION + ": " + "\"" + options.toString() + "\" }")) .type("test") .build(); } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rule/IOPushDownRuleTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rule/IOPushDownRuleTest.java index d8af7fbdea2e3..32f59ddad79b6 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rule/IOPushDownRuleTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rule/IOPushDownRuleTest.java @@ -23,10 +23,10 @@ import static org.hamcrest.core.IsEqual.equalTo; import static org.hamcrest.core.IsInstanceOf.instanceOf; -import com.alibaba.fastjson.JSON; import java.util.HashSet; import java.util.List; import java.util.Set; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv; import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode; import org.apache.beam.sdk.extensions.sql.meta.Table; @@ -170,7 +170,8 @@ private static Table getTable(String name, PushDownOptions options) { .comment(name + " table") .schema(BASIC_SCHEMA) .properties( - JSON.parseObject("{ " + PUSH_DOWN_OPTION + ": " + "\"" + options.toString() + "\" }")) + TableUtils.parseProperties( + "{ " + PUSH_DOWN_OPTION + ": " + "\"" + options.toString() + "\" }")) .type("test") .build(); } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rule/JoinReorderingTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rule/JoinReorderingTest.java index 3e5f37b809751..77de4cdec0f97 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rule/JoinReorderingTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/rule/JoinReorderingTest.java @@ -65,8 +65,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSet; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSets; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Assert; import org.junit.Test; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlBuiltinFunctionsIntegrationTestBase.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlBuiltinFunctionsIntegrationTestBase.java index eba1768328072..7bdf56497e2b4 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlBuiltinFunctionsIntegrationTestBase.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/integrationtest/BeamSqlBuiltinFunctionsIntegrationTestBase.java @@ -19,7 +19,7 @@ import static org.apache.beam.sdk.extensions.sql.utils.DateTimeUtils.parseTimestampWithUTCTimeZone; import static org.apache.beam.sdk.extensions.sql.utils.RowAsserts.matchesScalar; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.junit.Assert.assertTrue; import com.google.auto.value.AutoValue; @@ -49,8 +49,8 @@ import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTime; import org.junit.Rule; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/CustomTableResolverTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/CustomTableResolverTest.java index 4a8c4e4fbccbc..7f9d52f029060 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/CustomTableResolverTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/CustomTableResolverTest.java @@ -33,7 +33,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/SchemaIOTableProviderWrapperTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/SchemaIOTableProviderWrapperTest.java index 4b45e5a1cb1d0..0bab4a109d610 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/SchemaIOTableProviderWrapperTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/SchemaIOTableProviderWrapperTest.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.extensions.sql.meta.provider; -import com.alibaba.fastjson.JSON; import java.util.List; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.meta.BeamSqlTable; import org.apache.beam.sdk.extensions.sql.meta.DefaultTableFilter; import org.apache.beam.sdk.extensions.sql.meta.Table; @@ -27,7 +27,7 @@ import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; @@ -57,7 +57,7 @@ public class SchemaIOTableProviderWrapperTest { .name("table") .comment("table") .schema(inputSchema) - .properties(JSON.parseObject("{}")) + .properties(TableUtils.parseProperties("{}")) .type("test") .build(); diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/TestSchemaIOTableProviderWrapper.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/TestSchemaIOTableProviderWrapper.java index 6cbfca504d7bd..f005f98d7bdcf 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/TestSchemaIOTableProviderWrapper.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/TestSchemaIOTableProviderWrapper.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryFilterTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryFilterTest.java index fa84883448cce..9ef2c5ebbd89a 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryFilterTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryFilterTest.java @@ -21,8 +21,8 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.instanceOf; -import com.alibaba.fastjson.JSON; import org.apache.beam.repackaged.core.org.apache.commons.lang3.tuple.Pair; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv; import org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel; import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode; @@ -33,7 +33,7 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -110,7 +110,8 @@ private static Table getTable(String name, PushDownOptions options) { .comment(name + " table") .schema(BASIC_SCHEMA) .properties( - JSON.parseObject("{ " + PUSH_DOWN_OPTION + ": " + "\"" + options.toString() + "\" }")) + TableUtils.parseProperties( + "{ " + PUSH_DOWN_OPTION + ": " + "\"" + options.toString() + "\" }")) .type("test") .build(); } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryReadWriteIT.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryReadWriteIT.java index 911cc9cea9395..176b396a14866 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryReadWriteIT.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryReadWriteIT.java @@ -58,7 +58,7 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Duration; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryRowCountIT.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryRowCountIT.java index 9469bae8aad37..91d5c5a414669 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryRowCountIT.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryRowCountIT.java @@ -38,7 +38,7 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProviderTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProviderTest.java index d113b10060eef..4486d00885be9 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProviderTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTableProviderTest.java @@ -25,8 +25,8 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; -import com.alibaba.fastjson.JSON; import java.util.stream.Stream; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.meta.BeamSqlTable; import org.apache.beam.sdk.extensions.sql.meta.Table; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead.Method; @@ -67,7 +67,7 @@ public void testDefaultMethod_whenPropertiesAreNotSet() { public void testSelectDefaultMethodExplicitly() { Table table = fakeTableWithProperties( - "hello", "{ " + METHOD_PROPERTY + ": " + "\"" + Method.DEFAULT.toString() + "\" }"); + "hello", "{" + METHOD_PROPERTY + ": " + "\"" + Method.DEFAULT.toString() + "\" }"); BigQueryTable sqlTable = (BigQueryTable) provider.buildBeamSqlTable(table); assertEquals(Method.DEFAULT, sqlTable.method); @@ -77,7 +77,7 @@ public void testSelectDefaultMethodExplicitly() { public void testSelectDirectReadMethod() { Table table = fakeTableWithProperties( - "hello", "{ " + METHOD_PROPERTY + ": " + "\"" + Method.DIRECT_READ.toString() + "\" }"); + "hello", "{" + METHOD_PROPERTY + ": " + "\"" + Method.DIRECT_READ.toString() + "\" }"); BigQueryTable sqlTable = (BigQueryTable) provider.buildBeamSqlTable(table); assertEquals(Method.DIRECT_READ, sqlTable.method); @@ -87,7 +87,7 @@ public void testSelectDirectReadMethod() { public void testSelectExportMethod() { Table table = fakeTableWithProperties( - "hello", "{ " + METHOD_PROPERTY + ": " + "\"" + Method.EXPORT.toString() + "\" }"); + "hello", "{" + METHOD_PROPERTY + ": " + "\"" + Method.EXPORT.toString() + "\" }"); BigQueryTable sqlTable = (BigQueryTable) provider.buildBeamSqlTable(table); assertEquals(Method.EXPORT, sqlTable.method); @@ -143,7 +143,7 @@ public void testSelectWriteDispositionMethodEmpty() { @Test public void testRuntimeExceptionThrown_whenAnInvalidPropertyIsSpecified() { - Table table = fakeTableWithProperties("hello", "{ " + METHOD_PROPERTY + ": \"blahblah\" }"); + Table table = fakeTableWithProperties("hello", "{" + METHOD_PROPERTY + ": \"blahblah\" }"); assertThrows( RuntimeException.class, @@ -154,7 +154,7 @@ public void testRuntimeExceptionThrown_whenAnInvalidPropertyIsSpecified() { @Test public void testRuntimeExceptionThrown_whenAPropertyOfInvalidTypeIsSpecified() { - Table table = fakeTableWithProperties("hello", "{ " + METHOD_PROPERTY + ": 1337 }"); + Table table = fakeTableWithProperties("hello", "{" + METHOD_PROPERTY + ": 1337 }"); assertThrows( RuntimeException.class, @@ -188,7 +188,7 @@ private static Table fakeTableWithProperties(String name, String properties) { Schema.Field.nullable("name", Schema.FieldType.STRING)) .collect(toSchema())) .type("bigquery") - .properties(JSON.parseObject(properties)) + .properties(TableUtils.parseProperties(properties)) .build(); } } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTestTableProvider.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTestTableProvider.java index f5c63793cf644..d6ab009e0e33a 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTestTableProvider.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTestTableProvider.java @@ -17,8 +17,6 @@ */ package org.apache.beam.sdk.extensions.sql.meta.provider.bigquery; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; - import java.util.HashMap; import java.util.Map; import org.apache.beam.sdk.extensions.sql.meta.BeamSqlTable; @@ -59,7 +57,7 @@ public BeamSqlTable buildBeamSqlTable(Table table) { table, BigQueryUtils.ConversionOptions.builder() .setTruncateTimestamps( - firstNonNull(table.getProperties().getBoolean("truncateTimestamps"), false) + table.getProperties().path("truncateTimestamps").asBoolean(false) ? BigQueryUtils.ConversionOptions.TruncateTimestamps.TRUNCATE : BigQueryUtils.ConversionOptions.TruncateTimestamps.REJECT) .build()); diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableFilterTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableFilterTest.java index dade31390f6ac..6348edcc97307 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableFilterTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableFilterTest.java @@ -22,9 +22,9 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.instanceOf; -import com.alibaba.fastjson.JSON; import java.util.Arrays; import java.util.Collection; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv; import org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel; import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode; @@ -100,7 +100,8 @@ private static Table getTable(String name, PushDownOptions options) { .comment(name + " table") .schema(BASIC_SCHEMA) .properties( - JSON.parseObject("{ " + PUSH_DOWN_OPTION + ": " + "\"" + options.toString() + "\" }")) + TableUtils.parseProperties( + "{ " + PUSH_DOWN_OPTION + ": " + "\"" + options.toString() + "\" }")) .type("test") .build(); } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableFlatTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableFlatTest.java index 08082e85e0e10..6a038842becb9 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableFlatTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableFlatTest.java @@ -36,7 +36,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.cloud.bigtable.emulator.v2.BigtableEmulatorRule; import java.io.IOException; import org.apache.beam.sdk.extensions.sql.BeamSqlCli; @@ -95,9 +95,9 @@ public void testCreatesFlatSchemaCorrectly() { assertNotNull(table); assertEquals(TEST_FLAT_SCHEMA, table.getSchema()); - JSONObject properties = table.getProperties(); - assertTrue(properties.containsKey(COLUMNS_MAPPING)); - assertEquals(columnsMappingString(), properties.getString(COLUMNS_MAPPING)); + ObjectNode properties = table.getProperties(); + assertTrue(properties.has(COLUMNS_MAPPING)); + assertEquals(columnsMappingString(), properties.get(COLUMNS_MAPPING).asText()); } @Test diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableIT.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableIT.java index d62292a3f6aba..f8e130709968f 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableIT.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableIT.java @@ -42,7 +42,7 @@ import org.apache.beam.sdk.transforms.SimpleFunction; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableTestUtils.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableTestUtils.java index ef66fb89affaf..ca70a6d1d36f2 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableTestUtils.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableTestUtils.java @@ -38,8 +38,8 @@ import java.util.List; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Longs; import org.checkerframework.checker.nullness.qual.Nullable; class BigtableTableTestUtils { diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableWithRowsTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableWithRowsTest.java index 4b60eb13c8946..203cbf708e87a 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableWithRowsTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableWithRowsTest.java @@ -44,7 +44,7 @@ import org.apache.beam.sdk.transforms.SimpleFunction; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableAvroTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableAvroTest.java index e468d5b9eef3c..1a1c06c76edf3 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableAvroTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableAvroTest.java @@ -17,7 +17,6 @@ */ package org.apache.beam.sdk.extensions.sql.meta.provider.kafka; -import com.alibaba.fastjson.JSON; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.List; @@ -25,10 +24,11 @@ import org.apache.avro.generic.GenericRecordBuilder; import org.apache.beam.sdk.extensions.avro.coders.AvroCoder; import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.meta.Table; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; public class BeamKafkaTableAvroTest extends BeamKafkaTableTest { private static final Schema EMPTY_SCHEMA = Schema.builder().build(); @@ -94,7 +94,7 @@ protected BeamKafkaTable getBeamKafkaTable() { .type("kafka") .schema(TEST_SCHEMA) .location("localhost/mytopic") - .properties(JSON.parseObject("{ \"format\": \"avro\" }")) + .properties(TableUtils.parseProperties("{ \"format\": \"avro\" }")) .build()); } } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableCSVTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableCSVTest.java index 5624f0051c021..5988b2215e187 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableCSVTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableCSVTest.java @@ -22,7 +22,7 @@ import java.util.List; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; public class BeamKafkaTableCSVTest extends BeamKafkaTableTest { private static final Schema TEST_SCHEMA = diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableJsonTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableJsonTest.java index 8a485e217dcbe..a3611b00c142e 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableJsonTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableJsonTest.java @@ -19,12 +19,12 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import com.alibaba.fastjson.JSON; import java.util.List; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.meta.Table; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; public class BeamKafkaTableJsonTest extends BeamKafkaTableTest { @@ -67,7 +67,7 @@ protected BeamKafkaTable getBeamKafkaTable() { .type("kafka") .schema(TEST_SCHEMA) .location("localhost/mytopic") - .properties(JSON.parseObject("{ \"format\": \"json\" }")) + .properties(TableUtils.parseProperties("{ \"format\": \"json\" }")) .build()); } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableProtoTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableProtoTest.java index c46e27836c8f6..7fa94aa9daa3e 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableProtoTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableProtoTest.java @@ -21,10 +21,10 @@ import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertThrows; -import com.alibaba.fastjson.JSON; import java.util.List; import org.apache.beam.sdk.coders.ByteArrayCoder; import org.apache.beam.sdk.extensions.protobuf.PayloadMessages; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.meta.Table; import org.apache.beam.sdk.io.kafka.KafkaRecordCoder; import org.apache.beam.sdk.io.kafka.ProducerRecordCoder; @@ -34,7 +34,7 @@ import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; public class BeamKafkaTableProtoTest extends BeamKafkaTableTest { @@ -94,7 +94,7 @@ private static BeamKafkaTable getBeamKafkaTable(Schema schema) { .schema(schema) .location("localhost/mytopic") .properties( - JSON.parseObject( + TableUtils.parseProperties( "{ \"format\": \"proto\", \"protoClass\": \"" + PayloadMessages.TestMessage.class.getName() + "\" }")) diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableStatisticsTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableStatisticsTest.java index eb291dddc1020..e3b4bbb69d1fb 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableStatisticsTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableStatisticsTest.java @@ -22,7 +22,7 @@ import java.util.List; import org.apache.beam.sdk.extensions.sql.impl.BeamTableStatistics; import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Assert; import org.junit.Test; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableThriftTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableThriftTest.java index a7b5e9a59b9e9..0463e4c65109a 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableThriftTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableThriftTest.java @@ -21,9 +21,9 @@ import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertThrows; -import com.alibaba.fastjson.JSON; import java.util.List; import org.apache.beam.sdk.coders.ByteArrayCoder; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.meta.Table; import org.apache.beam.sdk.io.kafka.KafkaRecordCoder; import org.apache.beam.sdk.io.kafka.ProducerRecordCoder; @@ -34,7 +34,7 @@ import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.thrift.TException; import org.apache.thrift.TSerializer; import org.apache.thrift.protocol.TCompactProtocol; @@ -100,7 +100,7 @@ private static BeamKafkaTable getBeamKafkaTable(Schema schema) { .schema(schema) .location("localhost/mytopic") .properties( - JSON.parseObject( + TableUtils.parseProperties( "{ \"format\": \"thrift\", \"thriftClass\": \"" + TestThriftMessage.class.getName() + "\", \"thriftProtocolFactoryClass\": \"" diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderIT.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderIT.java index 34b7efabb8803..e9794f2f5b3d2 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderIT.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderIT.java @@ -19,9 +19,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.beam.sdk.extensions.sql.impl.schema.BeamTableUtils.beamRow2CsvLine; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; -import com.alibaba.fastjson.JSON; import java.io.Serializable; import java.util.Arrays; import java.util.Collection; @@ -40,6 +39,7 @@ import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils; import org.apache.beam.sdk.extensions.protobuf.PayloadMessages; import org.apache.beam.sdk.extensions.protobuf.ProtoMessageSchema; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv; import org.apache.beam.sdk.extensions.sql.impl.rel.BeamSqlRelUtils; import org.apache.beam.sdk.extensions.sql.meta.Table; @@ -66,8 +66,8 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.apache.commons.csv.CSVFormat; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; @@ -153,7 +153,7 @@ public void testFake2() throws BeamKafkaTable.NoEstimationException { .location(buildLocation()) .schema(TEST_TABLE_SCHEMA) .type("kafka") - .properties(JSON.parseObject(objectsProvider.getKafkaPropertiesString())) + .properties(TableUtils.parseProperties(objectsProvider.getKafkaPropertiesString())) .build(); BeamKafkaTable kafkaTable = (BeamKafkaTable) new KafkaTableProvider().buildBeamSqlTable(table); produceSomeRecordsWithDelay(100, 20); diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderTest.java index a2a663ad6ab14..09211ba06ba0e 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderTest.java @@ -21,15 +21,16 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.util.List; import org.apache.beam.sdk.extensions.protobuf.PayloadMessages; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.meta.BeamSqlTable; import org.apache.beam.sdk.extensions.sql.meta.Table; import org.apache.beam.sdk.io.thrift.payloads.SimpleThriftMessage; import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.thrift.TBase; import org.apache.thrift.protocol.TCompactProtocol; import org.apache.thrift.protocol.TProtocolFactory; @@ -204,16 +205,20 @@ private static Table mockTable( @Nullable Class protoClass, @Nullable Class> thriftClass, @Nullable Class thriftProtocolFactoryClass) { - JSONObject properties = new JSONObject(); + ObjectNode properties = TableUtils.emptyProperties(); if (extraBootstrapServers != null) { - JSONArray bootstrapServers = new JSONArray(); - bootstrapServers.addAll(extraBootstrapServers); + ArrayNode bootstrapServers = TableUtils.getObjectMapper().createArrayNode(); + for (String server : extraBootstrapServers) { + bootstrapServers.add(server); + } properties.put("bootstrap_servers", bootstrapServers); } if (extraTopics != null) { - JSONArray topics = new JSONArray(); - topics.addAll(extraTopics); + ArrayNode topics = TableUtils.getObjectMapper().createArrayNode(); + for (String topic : extraTopics) { + topics.add(topic); + } properties.put("topics", topics); } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTestTable.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTestTable.java index f9cdf0ba9b40b..44b4dbe21acaf 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTestTable.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTestTable.java @@ -35,9 +35,9 @@ import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.MockConsumer; import org.apache.kafka.clients.consumer.OffsetAndTimestamp; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/NestedPayloadKafkaTableTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/NestedPayloadKafkaTableTest.java index beb973cd7dbac..f83366b58704f 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/NestedPayloadKafkaTableTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/NestedPayloadKafkaTableTest.java @@ -32,9 +32,9 @@ import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ListMultimap; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.Headers; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/mongodb/MongoDbFilterTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/mongodb/MongoDbFilterTest.java index aafaf9b9396d6..648f033630a91 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/mongodb/MongoDbFilterTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/mongodb/MongoDbFilterTest.java @@ -21,9 +21,9 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.instanceOf; -import com.alibaba.fastjson.JSON; import java.util.Arrays; import java.util.Collection; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv; import org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel; import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode; @@ -115,7 +115,8 @@ private static Table getTable(String name, PushDownOptions options) { .comment(name + " table") .schema(BASIC_SCHEMA) .properties( - JSON.parseObject("{ " + PUSH_DOWN_OPTION + ": " + "\"" + options.toString() + "\" }")) + TableUtils.parseProperties( + "{ " + PUSH_DOWN_OPTION + ": " + "\"" + options.toString() + "\" }")) .type("test") .build(); } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/mongodb/MongoDbReadWriteIT.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/mongodb/MongoDbReadWriteIT.java index ee2498d6b3673..76be08fe9a6e4 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/mongodb/MongoDbReadWriteIT.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/mongodb/MongoDbReadWriteIT.java @@ -56,7 +56,7 @@ import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.bson.Document; import org.junit.After; import org.junit.AfterClass; @@ -302,10 +302,9 @@ public void testPredicatePushDown() { .append( "$or", ImmutableList.of( - new Document("c_varchar", "varchar"), - new Document( - "c_varchar", new Document("$not", new Document("$eq", "fakeString")))) - .asList()) + new Document("c_varchar", "varchar"), + new Document( + "c_varchar", new Document("$not", new Document("$eq", "fakeString"))))) .append("c_boolean", true) .append("c_integer", 2147483647); final Schema expectedSchema = diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/mongodb/MongoDbTableProviderTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/mongodb/MongoDbTableProviderTest.java index a5881df01be7e..e063980977207 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/mongodb/MongoDbTableProviderTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/mongodb/MongoDbTableProviderTest.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.extensions.sql.meta.BeamSqlTable; import org.apache.beam.sdk.extensions.sql.meta.Table; import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsub/PubsubTableProviderIT.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsub/PubsubTableProviderIT.java index 721d7b365af8d..6153e6e33c2ce 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsub/PubsubTableProviderIT.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsub/PubsubTableProviderIT.java @@ -72,9 +72,9 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.jdbc.CalciteConnection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.hamcrest.Matcher; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsub/PubsubTableProviderTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsub/PubsubTableProviderTest.java index e778e2f46e1ad..95af75ed312cf 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsub/PubsubTableProviderTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsub/PubsubTableProviderTest.java @@ -21,7 +21,7 @@ import static org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils.VARCHAR; import static org.junit.Assert.assertEquals; -import com.alibaba.fastjson.JSON; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.meta.BeamSqlTable; import org.apache.beam.sdk.extensions.sql.meta.Table; import org.apache.beam.sdk.schemas.Schema; @@ -108,6 +108,6 @@ private static Table.Builder tableDefinition() { .location("projects/project/topics/topic") .schema(Schema.builder().build()) .type("pubsub") - .properties(JSON.parseObject("{ \"timestampAttributeKey\" : \"ts_field\" }")); + .properties(TableUtils.parseProperties("{ \"timestampAttributeKey\" : \"ts_field\" }")); } } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/PubsubLiteTableProviderTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/PubsubLiteTableProviderTest.java index a07eb1fe1a71a..4cdad7180b8c2 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/PubsubLiteTableProviderTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/PubsubLiteTableProviderTest.java @@ -21,17 +21,17 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; -import com.alibaba.fastjson.JSONObject; import com.google.api.gax.rpc.ApiException; import com.google.cloud.pubsublite.SubscriptionPath; import com.google.cloud.pubsublite.TopicPath; import java.util.Map; import java.util.function.Function; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.meta.BeamSqlTable; import org.apache.beam.sdk.extensions.sql.meta.Table; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.Schema.FieldType; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -65,7 +65,7 @@ private static BeamSqlTable makeTable( .name("testTable") .schema(schema) .location(location) - .properties(new JSONObject().fluentPutAll(properties)) + .properties(TableUtils.getObjectMapper().valueToTree(properties)) .build(); return PROVIDER.buildBeamSqlTable(table); } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/RowHandlerTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/RowHandlerTest.java index 91bba72808cbf..8e7fecd548006 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/RowHandlerTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/RowHandlerTest.java @@ -33,7 +33,7 @@ import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; import org.junit.Before; import org.junit.Test; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProviderTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProviderTest.java index b86f6077c2807..a91b28210df68 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProviderTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProviderTest.java @@ -19,7 +19,7 @@ import static org.apache.beam.sdk.extensions.sql.meta.provider.test.TestTableProvider.PUSH_DOWN_OPTION; -import com.alibaba.fastjson.JSON; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.meta.BeamSqlTable; import org.apache.beam.sdk.extensions.sql.meta.Table; import org.apache.beam.sdk.extensions.sql.meta.provider.test.TestTableProvider.PushDownOptions; @@ -28,7 +28,7 @@ import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.junit.Before; import org.junit.Rule; @@ -120,8 +120,13 @@ private static Table getTable(String name) { .comment(name + " table") .schema(BASIC_SCHEMA) .properties( - JSON.parseObject( - "{ " + PUSH_DOWN_OPTION + ": " + "\"" + PushDownOptions.BOTH.toString() + "\" }")) + TableUtils.parseProperties( + "{ \"" + + PUSH_DOWN_OPTION + + "\": " + + "\"" + + PushDownOptions.BOTH.toString() + + "\" }")) .type("test") .build(); } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProviderWithFilterAndProjectPushDown.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProviderWithFilterAndProjectPushDown.java index 4c178ceeb0f52..ce9f52ffea944 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProviderWithFilterAndProjectPushDown.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProviderWithFilterAndProjectPushDown.java @@ -24,8 +24,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -import com.alibaba.fastjson.JSON; import java.util.List; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv; import org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel; import org.apache.beam.sdk.extensions.sql.impl.rel.BeamIOSourceRel; @@ -44,7 +44,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelOptRule; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.rules.CoreRules; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.junit.Before; import org.junit.Rule; @@ -409,7 +409,8 @@ private static Table getTable(String name, PushDownOptions options) { .comment(name + " table") .schema(BASIC_SCHEMA) .properties( - JSON.parseObject("{ " + PUSH_DOWN_OPTION + ": " + "\"" + options.toString() + "\" }")) + TableUtils.parseProperties( + "{ " + PUSH_DOWN_OPTION + ": " + "\"" + options.toString() + "\" }")) .type("test") .build(); } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProviderWithFilterPushDown.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProviderWithFilterPushDown.java index c06bb18d7786a..eae1dfb49ee87 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProviderWithFilterPushDown.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProviderWithFilterPushDown.java @@ -25,8 +25,8 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import com.alibaba.fastjson.JSON; import java.util.List; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv; import org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel; import org.apache.beam.sdk.extensions.sql.impl.rel.BeamIOSourceRel; @@ -46,7 +46,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Calc; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.rules.CoreRules; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.collection.IsIterableContainingInAnyOrder; import org.joda.time.Duration; import org.junit.Before; @@ -290,7 +290,8 @@ private static Table getTable(String name, PushDownOptions options) { .comment(name + " table") .schema(BASIC_SCHEMA) .properties( - JSON.parseObject("{ " + PUSH_DOWN_OPTION + ": " + "\"" + options.toString() + "\" }")) + TableUtils.parseProperties( + "{ " + PUSH_DOWN_OPTION + ": " + "\"" + options.toString() + "\" }")) .type("test") .build(); } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProviderWithProjectPushDown.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProviderWithProjectPushDown.java index 8ae756497bfe3..4fa45cbab9ec3 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProviderWithProjectPushDown.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/TestTableProviderWithProjectPushDown.java @@ -24,8 +24,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import com.alibaba.fastjson.JSON; import java.util.List; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv; import org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel; import org.apache.beam.sdk.extensions.sql.impl.rel.BeamIOSourceRel; @@ -44,7 +44,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelOptRule; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.rules.CoreRules; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.junit.Before; import org.junit.Rule; @@ -255,7 +255,8 @@ private static Table getTable(String name, PushDownOptions options) { .comment(name + " table") .schema(BASIC_SCHEMA) .properties( - JSON.parseObject("{ " + PUSH_DOWN_OPTION + ": " + "\"" + options.toString() + "\" }")) + TableUtils.parseProperties( + "{ " + PUSH_DOWN_OPTION + ": " + "\"" + options.toString() + "\" }")) .type("test") .build(); } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/TextTableProviderTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/TextTableProviderTest.java index f6c4db569c6b6..e5a46f877001a 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/TextTableProviderTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/text/TextTableProviderTest.java @@ -33,7 +33,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/store/InMemoryMetaStoreTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/store/InMemoryMetaStoreTest.java index 2cf835c38f646..825f3ed06485a 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/store/InMemoryMetaStoreTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/store/InMemoryMetaStoreTest.java @@ -22,10 +22,10 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import com.alibaba.fastjson.JSONObject; import java.util.HashMap; import java.util.Map; import java.util.stream.Stream; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.meta.BeamSqlTable; import org.apache.beam.sdk.extensions.sql.meta.Table; import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider; @@ -125,7 +125,7 @@ private static Table mockTable(String name, String type) { Schema.Field.nullable("name", Schema.FieldType.STRING)) .collect(toSchema())) .type(type) - .properties(new JSONObject()) + .properties(TableUtils.emptyProperties()) .build(); } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/utils/RowAsserts.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/utils/RowAsserts.java index 779b8d114a60a..c198d39da9b76 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/utils/RowAsserts.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/utils/RowAsserts.java @@ -23,7 +23,7 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** Contain helpers to assert {@link Row}s. */ public class RowAsserts { diff --git a/sdks/java/extensions/sql/udf-test-provider/build.gradle b/sdks/java/extensions/sql/udf-test-provider/build.gradle index b8fc1b6f64a22..7531e48d34b5a 100644 --- a/sdks/java/extensions/sql/udf-test-provider/build.gradle +++ b/sdks/java/extensions/sql/udf-test-provider/build.gradle @@ -33,5 +33,5 @@ project.ext.jarPath = jar.archivePath dependencies { // No dependency (direct or transitive) on :sdks:java:core. implementation project(":sdks:java:extensions:sql:udf") - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre } diff --git a/sdks/java/extensions/sql/udf-test-provider/src/main/java/org/apache/beam/sdk/extensions/sql/provider/UdfTestProvider.java b/sdks/java/extensions/sql/udf-test-provider/src/main/java/org/apache/beam/sdk/extensions/sql/provider/UdfTestProvider.java index c0da4aa1613bb..5ba7326cf89b8 100644 --- a/sdks/java/extensions/sql/udf-test-provider/src/main/java/org/apache/beam/sdk/extensions/sql/provider/UdfTestProvider.java +++ b/sdks/java/extensions/sql/udf-test-provider/src/main/java/org/apache/beam/sdk/extensions/sql/provider/UdfTestProvider.java @@ -23,7 +23,7 @@ import org.apache.beam.sdk.extensions.sql.udf.AggregateFn; import org.apache.beam.sdk.extensions.sql.udf.ScalarFn; import org.apache.beam.sdk.extensions.sql.udf.UdfProvider; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** Defines Java UDFs for use in tests. */ @AutoService(UdfProvider.class) diff --git a/sdks/java/extensions/sql/zetasql/build.gradle b/sdks/java/extensions/sql/zetasql/build.gradle index bb8ee55330fc8..8d6e2aac0bf40 100644 --- a/sdks/java/extensions/sql/zetasql/build.gradle +++ b/sdks/java/extensions/sql/zetasql/build.gradle @@ -46,7 +46,7 @@ dependencies { implementation library.java.protobuf_java_util permitUnusedDeclared library.java.protobuf_java_util // BEAM-11761 implementation library.java.slf4j_api - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.proto_google_common_protos // Interfaces with ZetaSQL use this permitUnusedDeclared library.java.proto_google_common_protos // BEAM-11761 implementation library.java.grpc_google_common_protos // Interfaces with ZetaSQL use this @@ -56,12 +56,12 @@ dependencies { implementation "com.google.zetasql:zetasql-jni-channel:$zetasql_version" permitUnusedDeclared "com.google.zetasql:zetasql-jni-channel:$zetasql_version" // BEAM-11761 testImplementation library.java.vendored_calcite_1_28_0 - testImplementation library.java.vendored_guava_26_0_jre + testImplementation library.java.vendored_guava_32_1_2_jre testImplementation library.java.junit testImplementation library.java.hamcrest testImplementation library.java.mockito_core testImplementation library.java.quickcheck_core - testImplementation "com.alibaba:fastjson:1.2.69" + testImplementation library.java.jackson_databind testImplementation "org.codehaus.janino:janino:3.0.11" testCompileOnly project(":sdks:java:extensions:sql:udf-test-provider") testRuntimeOnly library.java.slf4j_jdk14 diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCalcRel.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCalcRel.java index dee87d370e60a..d60ebe46b370a 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCalcRel.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCalcRel.java @@ -68,8 +68,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlNode; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCatalog.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCatalog.java index 985561b5a3a52..719abe4041bc9 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCatalog.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCatalog.java @@ -56,7 +56,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeField; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.FunctionParameter; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.SchemaPlus; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * Catalog for registering tables and functions. Populates a {@link SimpleCatalog} based on a {@link diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/DateTimeUtils.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/DateTimeUtils.java index 8f313f9720107..a60cd395b581d 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/DateTimeUtils.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/DateTimeUtils.java @@ -17,15 +17,17 @@ */ package org.apache.beam.sdk.extensions.sql.zetasql; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; + import com.google.zetasql.Value; import io.grpc.Status; import java.time.LocalTime; import java.util.List; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.avatica.util.TimeUnit; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.math.LongMath; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.math.LongMath; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -79,15 +81,15 @@ public static DateTimeFormatter findDateTimePattern( String str, ImmutableMap patternMap) { if (str.indexOf('.') == -1) { if (str.indexOf('T') == -1) { - return patternMap.get(TimestampPatterns.TIMESTAMP_PATTERN); + return checkNotNull(patternMap.get(TimestampPatterns.TIMESTAMP_PATTERN)); } else { - return patternMap.get(TimestampPatterns.TIMESTAMP_PATTERN_T); + return checkNotNull(patternMap.get(TimestampPatterns.TIMESTAMP_PATTERN_T)); } } else { if (str.indexOf('T') == -1) { - return patternMap.get(TimestampPatterns.TIMESTAMP_PATTERN_SUBSECOND); + return checkNotNull(patternMap.get(TimestampPatterns.TIMESTAMP_PATTERN_SUBSECOND)); } else { - return patternMap.get(TimestampPatterns.TIMESTAMP_PATTERN_SUBSECOND_T); + return checkNotNull(patternMap.get(TimestampPatterns.TIMESTAMP_PATTERN_SUBSECOND_T)); } } } diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/QueryTrait.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/QueryTrait.java index 830b70fbc5dfd..ac1cd08203877 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/QueryTrait.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/QueryTrait.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql.zetasql; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.zetasql.Table; import com.google.zetasql.resolvedast.ResolvedColumn; diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlAnalyzer.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlAnalyzer.java index 75db85244cb9a..0b5d09515b0e6 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlAnalyzer.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlAnalyzer.java @@ -41,8 +41,8 @@ import java.util.Map; import org.apache.beam.sdk.extensions.sql.impl.QueryPlanner.QueryParameters; import org.apache.beam.sdk.extensions.sql.impl.QueryPlanner.QueryParameters.Kind; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; /** Adapter for {@link Analyzer} to simplify the API for parsing the query and resolving the AST. */ @SuppressWarnings({ diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SupportedZetaSqlBuiltinFunctions.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SupportedZetaSqlBuiltinFunctions.java index 601aee5667c23..b3f71cea9e34d 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SupportedZetaSqlBuiltinFunctions.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SupportedZetaSqlBuiltinFunctions.java @@ -19,7 +19,7 @@ import com.google.zetasql.ZetaSQLFunction.FunctionSignatureId; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * List of ZetaSQL builtin functions supported by Beam ZetaSQL. Keep this list in sync with diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolution.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolution.java index c8fb3562044f9..d077ef40ee64d 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolution.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolution.java @@ -27,7 +27,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.Schema; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.SchemaPlus; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.Table; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** Utility methods to resolve a table, given a top-level Calcite schema and a table path. */ @SuppressWarnings({ diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLQueryPlanner.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLQueryPlanner.java index 3351e4fc8067c..df14cfa73fb48 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLQueryPlanner.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLQueryPlanner.java @@ -64,7 +64,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.Frameworks; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSet; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlBeamTranslationUtils.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlBeamTranslationUtils.java index 211775908332b..318da1c414423 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlBeamTranslationUtils.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlBeamTranslationUtils.java @@ -39,7 +39,7 @@ import org.apache.beam.sdk.schemas.logicaltypes.DateTime; import org.apache.beam.sdk.schemas.logicaltypes.SqlTypes; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.math.LongMath; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.math.LongMath; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlCalciteTranslationUtils.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlCalciteTranslationUtils.java index 5c4b1c266e092..965426db287f4 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlCalciteTranslationUtils.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlCalciteTranslationUtils.java @@ -45,8 +45,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.DateString; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.TimeString; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.TimestampString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * Utility methods for ZetaSQL <=> Calcite translation. diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/AggregateScanConverter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/AggregateScanConverter.java index b10da9b592c7e..2dfc7fe372f5e 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/AggregateScanConverter.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/AggregateScanConverter.java @@ -48,8 +48,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlAggFunction; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** Converts aggregate calls. */ @SuppressWarnings({ diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ArrayScanColumnRefToUncollect.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ArrayScanColumnRefToUncollect.java index 3a16deb3706dd..ac3de648b52d8 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ArrayScanColumnRefToUncollect.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ArrayScanColumnRefToUncollect.java @@ -31,8 +31,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexInputRef; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * Converts array scan that represents a reference to an array column, or an (possibly nested) array diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ArrayScanLiteralToUncollectConverter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ArrayScanLiteralToUncollectConverter.java index de2ffb7ab9854..902b7a762fe54 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ArrayScanLiteralToUncollectConverter.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ArrayScanLiteralToUncollectConverter.java @@ -24,7 +24,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelNode; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.logical.LogicalProject; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Converts array scan that represents an array literal to uncollect. */ class ArrayScanLiteralToUncollectConverter extends RelConverter { diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ArrayScanToJoinConverter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ArrayScanToJoinConverter.java index 4a455bf7e56a3..79984ea877e00 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ArrayScanToJoinConverter.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ArrayScanToJoinConverter.java @@ -32,7 +32,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.logical.LogicalProject; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexInputRef; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; /** Converts array scan that represents join of an uncollect(array_field) to uncollect. */ class ArrayScanToJoinConverter extends RelConverter { diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ExpressionConverter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ExpressionConverter.java index c8ff4e81da283..14db554d6f0bf 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ExpressionConverter.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ExpressionConverter.java @@ -29,7 +29,7 @@ import static org.apache.beam.sdk.extensions.sql.zetasql.BeamZetaSqlCatalog.USER_DEFINED_JAVA_SCALAR_FUNCTIONS; import static org.apache.beam.sdk.extensions.sql.zetasql.BeamZetaSqlCatalog.USER_DEFINED_SQL_FUNCTIONS; import static org.apache.beam.sdk.extensions.sql.zetasql.BeamZetaSqlCatalog.ZETASQL_FUNCTION_GROUP_NAME; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.common.base.Ascii; import com.google.common.base.Preconditions; diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/JoinScanConverter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/JoinScanConverter.java index 6f841a76ea2d3..085395725fbb6 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/JoinScanConverter.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/JoinScanConverter.java @@ -27,9 +27,9 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.logical.LogicalJoin; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeField; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; /** Converts joins if neither side of the join is a WithRefScan. */ class JoinScanConverter extends RelConverter { diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/LimitOffsetScanToLimitConverter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/LimitOffsetScanToLimitConverter.java index fd97cabe060ff..5cb9569ba0743 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/LimitOffsetScanToLimitConverter.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/LimitOffsetScanToLimitConverter.java @@ -29,8 +29,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexDynamicParam; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLiteral; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** Converts LIMIT without ORDER BY. */ @SuppressWarnings({ diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/LimitOffsetScanToOrderByLimitConverter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/LimitOffsetScanToOrderByLimitConverter.java index f9e1e86c75f79..10ff26a9bb4c6 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/LimitOffsetScanToOrderByLimitConverter.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/LimitOffsetScanToOrderByLimitConverter.java @@ -36,8 +36,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.logical.LogicalSort; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLiteral; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** Converts ORDER BY LIMIT. */ @SuppressWarnings({ diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ProjectScanConverter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ProjectScanConverter.java index 51f657b5d9f90..49a1f2dbd4d97 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ProjectScanConverter.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ProjectScanConverter.java @@ -24,7 +24,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelNode; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.logical.LogicalProject; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Converts projection. */ class ProjectScanConverter extends RelConverter { diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/QueryStatementConverter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/QueryStatementConverter.java index 91f22010be4dd..e3d9042dcfe1f 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/QueryStatementConverter.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/QueryStatementConverter.java @@ -38,7 +38,7 @@ import java.util.Collections; import java.util.List; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMultimap; /** * Converts a resolved Zeta SQL query represented by a tree to corresponding Calcite representation. diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SetOperationScanConverter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SetOperationScanConverter.java index 304919b490879..f8ac7ccdd299f 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SetOperationScanConverter.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SetOperationScanConverter.java @@ -36,8 +36,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.logical.LogicalIntersect; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.logical.LogicalMinus; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.logical.LogicalUnion; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** Converts set operations. */ class SetOperationScanConverter extends RelConverter { diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlCaseWithValueOperatorRewriter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlCaseWithValueOperatorRewriter.java index 1c7e639a43b62..02ce2d57b9c43 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlCaseWithValueOperatorRewriter.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlCaseWithValueOperatorRewriter.java @@ -23,9 +23,9 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlOperator; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** * Rewrites $case_with_value calls as $case_no_value calls. diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlCoalesceOperatorRewriter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlCoalesceOperatorRewriter.java index 5cecb89a35784..df1217fa15bd3 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlCoalesceOperatorRewriter.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlCoalesceOperatorRewriter.java @@ -24,8 +24,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlOperator; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Util; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * Rewrites COALESCE calls as CASE ($case_no_value) calls. diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlIfNullOperatorRewriter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlIfNullOperatorRewriter.java index 2a996ea1f49db..ecab784a5e570 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlIfNullOperatorRewriter.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlIfNullOperatorRewriter.java @@ -22,8 +22,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlOperator; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * Rewrites IFNULL calls as CASE ($case_no_value) calls. diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlInOperatorRewriter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlInOperatorRewriter.java index 3775af7ffd9e5..a73d0abde14d5 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlInOperatorRewriter.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlInOperatorRewriter.java @@ -21,8 +21,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexBuilder; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLiteral; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Rewrites $in calls as SEARCH calls. */ class SqlInOperatorRewriter implements SqlOperatorRewriter { diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlNullIfOperatorRewriter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlNullIfOperatorRewriter.java index 17a7bade23f73..99dd23fff82d8 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlNullIfOperatorRewriter.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlNullIfOperatorRewriter.java @@ -22,8 +22,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlOperator; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * Rewrites NULLIF calls as CASE ($case_no_value) calls. diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperatorMappingTable.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperatorMappingTable.java index 75ac290291002..c3b0c63768714 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperatorMappingTable.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperatorMappingTable.java @@ -22,7 +22,7 @@ import java.util.function.Function; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlOperator; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** SqlOperatorMappingTable. */ diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperators.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperators.java index 59d54063d967f..0f6dcea2f6924 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperators.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperators.java @@ -66,8 +66,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.validate.SqlUserDefinedFunction; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Optionality; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Util; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; /** * A separate SqlOperators table for those functions that do not exist or not compatible with diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/TableScanConverter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/TableScanConverter.java index 3b726212e3ba7..8b7af956ec9aa 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/TableScanConverter.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/TableScanConverter.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql.zetasql.translation; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedTableScan; import java.util.List; diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/UserFunctionDefinitions.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/UserFunctionDefinitions.java index b66ace777eed4..b1891337a5508 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/UserFunctionDefinitions.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/UserFunctionDefinitions.java @@ -23,7 +23,7 @@ import java.lang.reflect.Method; import java.util.List; import org.apache.beam.sdk.transforms.Combine; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** Holds user defined function definitions. */ @AutoValue diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/unnest/BeamZetaSqlUncollectRel.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/unnest/BeamZetaSqlUncollectRel.java index fd41e7bb80824..4cce106c6279a 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/unnest/BeamZetaSqlUncollectRel.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/unnest/BeamZetaSqlUncollectRel.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.sql.zetasql.unnest; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import org.apache.beam.sdk.extensions.sql.impl.planner.BeamCostModel; import org.apache.beam.sdk.extensions.sql.impl.planner.BeamRelMetadataQuery; diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/unnest/BeamZetaSqlUnnestRel.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/unnest/BeamZetaSqlUnnestRel.java index 4e81339d9b1ea..54a0cc5ad919d 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/unnest/BeamZetaSqlUnnestRel.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/unnest/BeamZetaSqlUnnestRel.java @@ -41,7 +41,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.JoinRelType; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataType; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.validate.SqlValidatorUtil; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/unnest/BeamZetaSqlUnnestRule.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/unnest/BeamZetaSqlUnnestRule.java index d1f1c8bfb8b90..d9fed7f6ae6b1 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/unnest/BeamZetaSqlUnnestRule.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/unnest/BeamZetaSqlUnnestRule.java @@ -29,7 +29,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.logical.LogicalProject; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexFieldAccess; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * A {@code ConverterRule} to replace {@link Correlate} {@link ZetaSqlUnnest} with {@link diff --git a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamJavaUdfCalcRuleTest.java b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamJavaUdfCalcRuleTest.java index e0307f84ee705..a6ca07b307ead 100644 --- a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamJavaUdfCalcRuleTest.java +++ b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamJavaUdfCalcRuleTest.java @@ -30,7 +30,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelOptPlanner; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.Frameworks; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.junit.Before; import org.junit.Rule; diff --git a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCalcRelTest.java b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCalcRelTest.java index b490458df3385..2cb8501eb276d 100644 --- a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCalcRelTest.java +++ b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCalcRelTest.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; diff --git a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCatalogTest.java b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCatalogTest.java index b0741b1f8526b..733de268c88bd 100644 --- a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCatalogTest.java +++ b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCatalogTest.java @@ -35,8 +35,8 @@ import org.apache.beam.sdk.extensions.sql.zetasql.translation.UserFunctionDefinitions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.SchemaPlus; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolutionTest.java b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolutionTest.java index bf82241773eed..fdb3db8832ed6 100644 --- a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolutionTest.java +++ b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolutionTest.java @@ -22,7 +22,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.SchemaPlus; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.Table; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.Matchers; import org.junit.Assert; import org.junit.Before; diff --git a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/TestInput.java b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/TestInput.java index 7aa76d8c7b74e..802299fc73ee5 100644 --- a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/TestInput.java +++ b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/TestInput.java @@ -29,8 +29,8 @@ import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.logicaltypes.SqlTypes; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** TestInput. */ class TestInput { diff --git a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLPushDownTest.java b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLPushDownTest.java index a9345110218b2..fd2b7dc7ca5e6 100644 --- a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLPushDownTest.java +++ b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLPushDownTest.java @@ -22,7 +22,7 @@ import static org.hamcrest.Matchers.instanceOf; import static org.junit.Assert.assertEquals; -import com.alibaba.fastjson.JSON; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv; import org.apache.beam.sdk.extensions.sql.impl.JdbcConnection; import org.apache.beam.sdk.extensions.sql.impl.JdbcDriver; @@ -44,7 +44,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.FrameworkConfig; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.Frameworks; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.junit.BeforeClass; import org.junit.Rule; @@ -220,7 +220,8 @@ private static Table getTable(String name, PushDownOptions options) { .comment(name + " table") .schema(BASIC_SCHEMA) .properties( - JSON.parseObject("{ " + PUSH_DOWN_OPTION + ": " + "\"" + options.toString() + "\" }")) + TableUtils.parseProperties( + "{ " + PUSH_DOWN_OPTION + ": " + "\"" + options.toString() + "\" }")) .type("test") .build(); } diff --git a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlDialectSpecTest.java b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlDialectSpecTest.java index ec5ea89f97a9b..a62ce2dc9a52e 100644 --- a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlDialectSpecTest.java +++ b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlDialectSpecTest.java @@ -48,8 +48,8 @@ import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.DateTime; import org.joda.time.Duration; import org.joda.time.chrono.ISOChronology; diff --git a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlJavaUdfTest.java b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlJavaUdfTest.java index 7f64acbeeeca8..8b21bec08be6e 100644 --- a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlJavaUdfTest.java +++ b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlJavaUdfTest.java @@ -44,7 +44,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.Frameworks; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.codehaus.commons.compiler.CompileException; import org.joda.time.Duration; diff --git a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlJavaUdfTypeTest.java b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlJavaUdfTypeTest.java index 4e0cb3c135b1a..74569afd308b7 100644 --- a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlJavaUdfTypeTest.java +++ b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlJavaUdfTypeTest.java @@ -39,8 +39,8 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.SchemaPlus; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.Frameworks; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.Duration; diff --git a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlNumberTypesTest.java b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlNumberTypesTest.java index 121c4979c6f39..a3178c3031c2f 100644 --- a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlNumberTypesTest.java +++ b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlNumberTypesTest.java @@ -19,7 +19,7 @@ import com.google.zetasql.Value; import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlTestBase.java b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlTestBase.java index 8f4f5d5bb0c2d..5b67ad76fd9d5 100644 --- a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlTestBase.java +++ b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlTestBase.java @@ -31,7 +31,7 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.FrameworkConfig; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.Frameworks; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Common setup for ZetaSQL tests. */ public abstract class ZetaSqlTestBase { diff --git a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlTimeFunctionsTest.java b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlTimeFunctionsTest.java index dcd60ea76dd6d..cfc0fce737bc5 100644 --- a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlTimeFunctionsTest.java +++ b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlTimeFunctionsTest.java @@ -38,7 +38,7 @@ import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Duration; import org.junit.Before; import org.junit.Rule; diff --git a/sdks/java/extensions/timeseries/build.gradle b/sdks/java/extensions/timeseries/build.gradle index fd8b961b0add0..86bf89d729202 100644 --- a/sdks/java/extensions/timeseries/build.gradle +++ b/sdks/java/extensions/timeseries/build.gradle @@ -24,9 +24,10 @@ applyJavaNature( description = "Apache Beam :: SDKs :: Java :: Extensions :: Timeseries" dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.joda_time implementation project(path: ":sdks:java:core", configuration: "shadow") testImplementation library.java.junit testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") + testImplementation project(path: ":sdks:java:extensions:avro", configuration: "testRuntimeMigration") } diff --git a/sdks/java/extensions/timeseries/src/main/java/org/apache/beam/sdk/extensions/timeseries/FillGaps.java b/sdks/java/extensions/timeseries/src/main/java/org/apache/beam/sdk/extensions/timeseries/FillGaps.java index 050c627f64663..97dabfe93cabd 100644 --- a/sdks/java/extensions/timeseries/src/main/java/org/apache/beam/sdk/extensions/timeseries/FillGaps.java +++ b/sdks/java/extensions/timeseries/src/main/java/org/apache/beam/sdk/extensions/timeseries/FillGaps.java @@ -50,8 +50,8 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TimestampedValue.TimestampedValueCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/extensions/timeseries/src/test/java/org/apache/beam/sdk/extensions/timeseries/FillGapsTest.java b/sdks/java/extensions/timeseries/src/test/java/org/apache/beam/sdk/extensions/timeseries/FillGapsTest.java index da419ec182042..6c3cabbb05982 100644 --- a/sdks/java/extensions/timeseries/src/test/java/org/apache/beam/sdk/extensions/timeseries/FillGapsTest.java +++ b/sdks/java/extensions/timeseries/src/test/java/org/apache/beam/sdk/extensions/timeseries/FillGapsTest.java @@ -35,9 +35,9 @@ import org.apache.beam.sdk.transforms.windowing.FixedWindows; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/sdks/java/extensions/zetasketch/build.gradle b/sdks/java/extensions/zetasketch/build.gradle index 3f3575b1a48b5..bb532ad08aa15 100644 --- a/sdks/java/extensions/zetasketch/build.gradle +++ b/sdks/java/extensions/zetasketch/build.gradle @@ -33,7 +33,7 @@ dependencies { implementation library.java.auto_value_annotations implementation library.java.slf4j_api - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation "com.google.zetasketch:zetasketch:$zetasketch_version" testImplementation library.java.junit @@ -42,6 +42,7 @@ dependencies { testImplementation project(":sdks:java:extensions:google-cloud-platform-core") testImplementation library.java.google_api_services_bigquery testImplementation library.java.proto_google_cloud_bigquery_storage_v1 + testImplementation project(path: ":sdks:java:extensions:avro", configuration: "testRuntimeMigration") testRuntimeOnly library.java.slf4j_simple testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") testRuntimeOnly project(":runners:google-cloud-dataflow-java") diff --git a/sdks/java/extensions/zetasketch/src/main/java/org/apache/beam/sdk/extensions/zetasketch/ApproximateCountDistinct.java b/sdks/java/extensions/zetasketch/src/main/java/org/apache/beam/sdk/extensions/zetasketch/ApproximateCountDistinct.java index ff45428e38351..b1272608fd989 100644 --- a/sdks/java/extensions/zetasketch/src/main/java/org/apache/beam/sdk/extensions/zetasketch/ApproximateCountDistinct.java +++ b/sdks/java/extensions/zetasketch/src/main/java/org/apache/beam/sdk/extensions/zetasketch/ApproximateCountDistinct.java @@ -34,7 +34,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * {@code PTransform}s for estimating the number of distinct elements in a {@code PCollection}, or diff --git a/sdks/java/extensions/zetasketch/src/main/java/org/apache/beam/sdk/extensions/zetasketch/HllCountInitFn.java b/sdks/java/extensions/zetasketch/src/main/java/org/apache/beam/sdk/extensions/zetasketch/HllCountInitFn.java index 32f76be1866f8..b4840fad20e85 100644 --- a/sdks/java/extensions/zetasketch/src/main/java/org/apache/beam/sdk/extensions/zetasketch/HllCountInitFn.java +++ b/sdks/java/extensions/zetasketch/src/main/java/org/apache/beam/sdk/extensions/zetasketch/HllCountInitFn.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.zetasketch; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.zetasketch.HyperLogLogPlusPlus; import com.google.zetasketch.shaded.com.google.protobuf.ByteString; diff --git a/sdks/java/extensions/zetasketch/src/main/java/org/apache/beam/sdk/extensions/zetasketch/HyperLogLogPlusPlusCoder.java b/sdks/java/extensions/zetasketch/src/main/java/org/apache/beam/sdk/extensions/zetasketch/HyperLogLogPlusPlusCoder.java index ad1a897473c35..3e702238d7281 100644 --- a/sdks/java/extensions/zetasketch/src/main/java/org/apache/beam/sdk/extensions/zetasketch/HyperLogLogPlusPlusCoder.java +++ b/sdks/java/extensions/zetasketch/src/main/java/org/apache/beam/sdk/extensions/zetasketch/HyperLogLogPlusPlusCoder.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.extensions.zetasketch; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.zetasketch.HyperLogLogPlusPlus; import java.io.IOException; diff --git a/sdks/java/extensions/zetasketch/src/test/java/org/apache/beam/sdk/extensions/zetasketch/ApproximateCountDistinctTest.java b/sdks/java/extensions/zetasketch/src/test/java/org/apache/beam/sdk/extensions/zetasketch/ApproximateCountDistinctTest.java index 15dbbfc8524ef..559ad5df99a7e 100644 --- a/sdks/java/extensions/zetasketch/src/test/java/org/apache/beam/sdk/extensions/zetasketch/ApproximateCountDistinctTest.java +++ b/sdks/java/extensions/zetasketch/src/test/java/org/apache/beam/sdk/extensions/zetasketch/ApproximateCountDistinctTest.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/sdks/java/fn-execution/OWNERS b/sdks/java/fn-execution/OWNERS deleted file mode 100644 index 201e7cb37505c..0000000000000 --- a/sdks/java/fn-execution/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lukecwik - - angoenka diff --git a/sdks/java/fn-execution/build.gradle b/sdks/java/fn-execution/build.gradle index 90821bdf6eedc..b04243a315ba1 100644 --- a/sdks/java/fn-execution/build.gradle +++ b/sdks/java/fn-execution/build.gradle @@ -30,7 +30,7 @@ dependencies { implementation project(path: ":model:fn-execution", configuration: "shadow") implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.vendored_grpc_1_54_0 - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.slf4j_api implementation library.java.joda_time provided library.java.junit diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/CancellableQueue.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/CancellableQueue.java index 96b0f8b9a0da7..e67423a8a044d 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/CancellableQueue.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/CancellableQueue.java @@ -17,14 +17,14 @@ */ package org.apache.beam.sdk.fn; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; /** * A simplified {@link ThreadSafe} blocking queue that can be cancelled freeing any blocked {@link @@ -36,7 +36,7 @@ public class CancellableQueue { private final int capacity; - private final Object[] elements; + private final @Nullable Object[] elements; private final Lock lock; private final Condition notFull; private final Condition notEmpty; @@ -86,8 +86,9 @@ public void put(T t) throws Exception, InterruptedException { * must invoke {@link #cancel} if the interrupt is unrecoverable. * @throws Exception if the queue is cancelled. */ + @SuppressWarnings({"cast"}) public T take() throws Exception, InterruptedException { - Object rval; + T rval; try { lock.lockInterruptibly(); while (count == 0 && cancellationException == null) { @@ -97,14 +98,15 @@ public T take() throws Exception, InterruptedException { throw cancellationException; } - rval = elements[takeIndex]; + rval = (T) elements[takeIndex]; + elements[takeIndex] = null; takeIndex = (takeIndex + 1) % elements.length; count -= 1; notFull.signal(); } finally { lock.unlock(); } - return (T) rval; + return rval; } /** @@ -119,6 +121,7 @@ public void cancel(Exception exception) { try { if (cancellationException == null) { cancellationException = exception; + clearElementsLocked(); } notEmpty.signalAll(); notFull.signalAll(); @@ -127,14 +130,21 @@ public void cancel(Exception exception) { } } + private void clearElementsLocked() { + for (int i = takeIndex; count > 0; i = (i + 1) % elements.length) { + elements[i] = null; + --count; + } + addIndex = 0; + takeIndex = 0; + } + /** Enables the queue to be re-used after it has been cancelled. */ public void reset() { lock.lock(); try { cancellationException = null; - addIndex = 0; - takeIndex = 0; - count = 0; + clearElementsLocked(); } finally { lock.unlock(); } diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/channel/AddHarnessIdInterceptor.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/channel/AddHarnessIdInterceptor.java index d3a5fd18f3afc..647e1b4a3d560 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/channel/AddHarnessIdInterceptor.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/channel/AddHarnessIdInterceptor.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.fn.channel; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ClientInterceptor; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Metadata; diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/channel/ManagedChannelFactory.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/channel/ManagedChannelFactory.java index 7c5b910b61b69..c2914c3dd8ad5 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/channel/ManagedChannelFactory.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/channel/ManagedChannelFactory.java @@ -30,7 +30,7 @@ import org.apache.beam.vendor.grpc.v1p54p0.io.netty.channel.epoll.EpollEventLoopGroup; import org.apache.beam.vendor.grpc.v1p54p0.io.netty.channel.epoll.EpollSocketChannel; import org.apache.beam.vendor.grpc.v1p54p0.io.netty.channel.unix.DomainSocketAddress; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** A Factory which creates {@link ManagedChannel} instances. */ public class ManagedChannelFactory { diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/channel/SocketAddressFactory.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/channel/SocketAddressFactory.java index 6bbf8b987dd05..c835ea8312fa8 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/channel/SocketAddressFactory.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/channel/SocketAddressFactory.java @@ -17,14 +17,14 @@ */ package org.apache.beam.sdk.fn.channel; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketAddress; import org.apache.beam.vendor.grpc.v1p54p0.io.netty.channel.unix.DomainSocketAddress; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.net.HostAndPort; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.net.HostAndPort; /** Creates a {@link SocketAddress} based upon a supplied string. */ public class SocketAddressFactory { diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/BeamFnDataGrpcMultiplexer.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/BeamFnDataGrpcMultiplexer.java index e1f5538f12117..c05a86fdb1b7b 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/BeamFnDataGrpcMultiplexer.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/BeamFnDataGrpcMultiplexer.java @@ -28,9 +28,9 @@ import org.apache.beam.sdk.fn.stream.OutboundObserverFactory; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Status; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/BeamFnDataOutboundAggregator.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/BeamFnDataOutboundAggregator.java index 3b97643dbc34e..ac2069fe264b7 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/BeamFnDataOutboundAggregator.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/BeamFnDataOutboundAggregator.java @@ -38,8 +38,8 @@ import org.apache.beam.sdk.util.ByteStringOutputStream; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/RemoteGrpcPortRead.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/RemoteGrpcPortRead.java index 5cffec4f27914..ba3ef217d7cf5 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/RemoteGrpcPortRead.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/RemoteGrpcPortRead.java @@ -17,14 +17,14 @@ */ package org.apache.beam.sdk.fn.data; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import org.apache.beam.model.fnexecution.v1.BeamFnApi.RemoteGrpcPort; import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec; import org.apache.beam.model.pipeline.v1.RunnerApi.PTransform; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** * An execution-time only {@link PTransform} which represents an SDK harness reading from a {@link diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/RemoteGrpcPortWrite.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/RemoteGrpcPortWrite.java index 19906dc73c43b..3bbd31d8bf250 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/RemoteGrpcPortWrite.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/RemoteGrpcPortWrite.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.fn.data; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import org.apache.beam.model.fnexecution.v1.BeamFnApi.RemoteGrpcPort; @@ -25,7 +25,7 @@ import org.apache.beam.model.pipeline.v1.RunnerApi.PCollection; import org.apache.beam.model.pipeline.v1.RunnerApi.PTransform; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** * An execution-time only {@link PTransform} which represents a write from within an SDK harness to diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/WeightedList.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/WeightedList.java new file mode 100644 index 0000000000000..4579a6903a246 --- /dev/null +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/data/WeightedList.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.fn.data; + +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +/** Facade for a {@link List} that keeps track of weight, for cache limit reasons. */ +public class WeightedList { + + /** Original list that is being wrapped. */ + private final List backing; + + /** Weight of all the elements being tracked. */ + private final AtomicLong weight; + + public WeightedList(List backing, long weight) { + this.backing = backing; + this.weight = new AtomicLong(weight); + } + + public List getBacking() { + return this.backing; + } + + public int size() { + return this.backing.size(); + } + + public boolean isEmpty() { + return this.backing.isEmpty(); + } + + public long getWeight() { + return weight.longValue(); + } + + public void add(T element, long weight) { + this.backing.add(element); + accumulateWeight(weight); + } + + public void addAll(WeightedList values) { + this.addAll(values.getBacking(), values.getWeight()); + } + + public void addAll(List values, long weight) { + this.backing.addAll(values); + accumulateWeight(weight); + } + + public void accumulateWeight(long weight) { + this.weight.accumulateAndGet( + weight, + (first, second) -> { + try { + return Math.addExact(first, second); + } catch (ArithmeticException e) { + return Long.MAX_VALUE; + } + }); + } +} diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/server/GrpcContextHeaderAccessorProvider.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/server/GrpcContextHeaderAccessorProvider.java index 26d319108563b..2ccae386d1bf8 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/server/GrpcContextHeaderAccessorProvider.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/server/GrpcContextHeaderAccessorProvider.java @@ -25,7 +25,7 @@ import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ServerCall.Listener; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ServerCallHandler; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ServerInterceptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; /** * A HeaderAccessorProvider which intercept the header in a GRPC request and expose the relevant diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/server/GrpcFnServer.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/server/GrpcFnServer.java index 7e2e5040427e0..fdf4f9e830683 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/server/GrpcFnServer.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/server/GrpcFnServer.java @@ -24,8 +24,8 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Server; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; /** * A {@link Server gRPC Server} which manages a single {@link FnService}. The lifetime of the diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/server/ServerFactory.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/server/ServerFactory.java index 99393a09f0659..a50e5cbef5bca 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/server/ServerFactory.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/server/ServerFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.fn.server; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.File; import java.io.IOException; @@ -39,7 +39,7 @@ import org.apache.beam.vendor.grpc.v1p54p0.io.netty.channel.epoll.EpollServerSocketChannel; import org.apache.beam.vendor.grpc.v1p54p0.io.netty.channel.unix.DomainSocketAddress; import org.apache.beam.vendor.grpc.v1p54p0.io.netty.util.internal.ThreadLocalRandom; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.net.HostAndPort; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.net.HostAndPort; /** A {@link Server gRPC server} factory. */ @SuppressWarnings({ diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/BufferingStreamObserver.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/BufferingStreamObserver.java index d3937b2820e9a..4876c6a42011d 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/BufferingStreamObserver.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/BufferingStreamObserver.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.fn.CancellableQueue; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.CallStreamObserver; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.NonNull; /** diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/DataStreams.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/DataStreams.java index 95f41291f6c74..3a541b44fa308 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/DataStreams.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/DataStreams.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.fn.stream; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.io.InputStream; @@ -27,6 +27,7 @@ import java.util.List; import java.util.NoSuchElementException; import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.fn.data.WeightedList; import org.apache.beam.sdk.util.ByteStringOutputStream; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; @@ -42,6 +43,9 @@ public class DataStreams { public static final int DEFAULT_OUTBOUND_BUFFER_LIMIT_BYTES = 1_000_000; + /** The number of bytes to add to cache estimates for each element in the weighted list. */ + private static final long BYTES_LIST_ELEMENT_OVERHEAD = 8L; + /** * Converts a single element delimited {@link OutputStream} into multiple {@link ByteString * ByteStrings}. @@ -186,16 +190,25 @@ public DataStreamDecoder(Coder coder, PrefetchableIterator inputS * ByteString} in the underlying {@link ByteString} {@link Iterator iterator} and decoding * elements till at the next boundary. */ - public List decodeFromChunkBoundaryToChunkBoundary() { - inbound.currentStream = inputByteStrings.next().newInput(); + public WeightedList decodeFromChunkBoundaryToChunkBoundary() { + ByteString byteString = inputByteStrings.next(); + inbound.currentStream = byteString.newInput(); inbound.position = 0; + try { InputStream previousStream = inbound.currentStream; List rvals = new ArrayList<>(); while (previousStream == inbound.currentStream && inbound.currentStream.available() != 0) { - rvals.add(next()); + T next = next(); + rvals.add(next); } - return rvals; + + // Uses the size of the ByteString as an approximation for the heap size occupied by the + // page, considering an overhead of {@link BYTES_LIST_ELEMENT_OVERHEAD} for each element. + long elementOverhead = rvals.size() * BYTES_LIST_ELEMENT_OVERHEAD; + long totalWeight = byteString.size() + elementOverhead; + + return new WeightedList<>(rvals, totalWeight); } catch (IOException e) { throw new IllegalStateException(e); } diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/DirectStreamObserver.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/DirectStreamObserver.java index 99a831c20ee6c..3bc72ac16c7d1 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/DirectStreamObserver.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/DirectStreamObserver.java @@ -96,9 +96,13 @@ public void onNext(T value) { // been invoked. if (initialPhase == phase) { LOG.info( - "Output channel stalled for {}s, outbound thread {}. See: " - + "https://issues.apache.org/jira/browse/BEAM-4280 for the history for " - + "this issue.", + "Output channel stalled for {}s, outbound thread {}. OnReady notification was " + + "not invoked, ensure the inbound gRPC thread is not used for output.", + totalSecondsWaited, + Thread.currentThread().getName()); + } else if (totalSecondsWaited > 60) { + LOG.warn( + "Output channel stalled for {}s, outbound thread {}.", totalSecondsWaited, Thread.currentThread().getName()); } else { diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/ForwardingClientResponseObserver.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/ForwardingClientResponseObserver.java index e3630c52db824..73ac226aaaca6 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/ForwardingClientResponseObserver.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/ForwardingClientResponseObserver.java @@ -25,7 +25,8 @@ * A {@link ClientResponseObserver} which delegates all {@link StreamObserver} calls. * *

    Used to wrap existing {@link StreamObserver}s to be able to install an {@link - * ClientCallStreamObserver#setOnReadyHandler(Runnable) onReadyHandler}. + * ClientCallStreamObserver#setOnReadyHandler(Runnable) onReadyHandler} and a handler invoked when + * the stream terminates. * *

    This is as thread-safe as the underlying stream observer that is being wrapped. */ @@ -33,15 +34,23 @@ public final class ForwardingClientResponseObserver implements ClientResponseObserver { public static ForwardingClientResponseObserver create( StreamObserver inbound, Runnable onReadyHandler) { - return new ForwardingClientResponseObserver<>(inbound, onReadyHandler); + return new ForwardingClientResponseObserver<>(inbound, onReadyHandler, () -> {}); + } + + public static ForwardingClientResponseObserver create( + StreamObserver inbound, Runnable onReadyHandler, Runnable onDoneHandler) { + return new ForwardingClientResponseObserver<>(inbound, onReadyHandler, onDoneHandler); } private final Runnable onReadyHandler; + private final Runnable onDoneHandler; private final StreamObserver inboundObserver; - ForwardingClientResponseObserver(StreamObserver inboundObserver, Runnable onReadyHandler) { + private ForwardingClientResponseObserver( + StreamObserver inboundObserver, Runnable onReadyHandler, Runnable onDoneHandler) { this.inboundObserver = inboundObserver; this.onReadyHandler = onReadyHandler; + this.onDoneHandler = onDoneHandler; } @Override @@ -51,11 +60,13 @@ public void onNext(ReqT value) { @Override public void onError(Throwable t) { + onDoneHandler.run(); inboundObserver.onError(t); } @Override public void onCompleted() { + onDoneHandler.run(); inboundObserver.onCompleted(); } diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/OutboundObserverFactory.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/OutboundObserverFactory.java index dfcc4ff475749..e2eeb1a0568df 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/OutboundObserverFactory.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/OutboundObserverFactory.java @@ -89,7 +89,9 @@ public StreamObserver outboundObserverFor( BasicFactory baseOutboundObserverFactory, StreamObserver inboundObserver) { AdvancingPhaser phaser = new AdvancingPhaser(1); - inboundObserver = ForwardingClientResponseObserver.create(inboundObserver, phaser::arrive); + inboundObserver = + ForwardingClientResponseObserver.create( + inboundObserver, phaser::arrive, phaser::forceTermination); CallStreamObserver outboundObserver = (CallStreamObserver) baseOutboundObserverFactory.outboundObserverFor(inboundObserver); @@ -126,7 +128,9 @@ public StreamObserver outboundObserverFor( BasicFactory baseOutboundObserverFactory, StreamObserver inboundObserver) { AdvancingPhaser phaser = new AdvancingPhaser(1); - inboundObserver = ForwardingClientResponseObserver.create(inboundObserver, phaser::arrive); + inboundObserver = + ForwardingClientResponseObserver.create( + inboundObserver, phaser::arrive, phaser::forceTermination); CallStreamObserver outboundObserver = (CallStreamObserver) baseOutboundObserverFactory.outboundObserverFor(inboundObserver); diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/PrefetchableIterables.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/PrefetchableIterables.java index 2e3f13c2f0bc4..dd7ec6b0f65a7 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/PrefetchableIterables.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/stream/PrefetchableIterables.java @@ -19,7 +19,7 @@ import java.util.NoSuchElementException; import javax.annotation.Nullable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; /** * This class contains static utility functions that operate on or return objects of type {@link diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/test/TestExecutors.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/test/TestExecutors.java index 81f582eeb3869..c7172b0b0fa88 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/test/TestExecutors.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/test/TestExecutors.java @@ -20,7 +20,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ForwardingExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ForwardingExecutorService; import org.junit.rules.TestRule; import org.junit.runner.Description; import org.junit.runners.model.Statement; diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/windowing/EncodedBoundedWindow.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/windowing/EncodedBoundedWindow.java index cb363a6f3ced9..f4270f40fc12f 100644 --- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/windowing/EncodedBoundedWindow.java +++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/windowing/EncodedBoundedWindow.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.VarInt; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; import org.joda.time.Instant; /** diff --git a/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/CancellableQueueTest.java b/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/CancellableQueueTest.java index b8b577b1805db..aa85e7a2cec66 100644 --- a/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/CancellableQueueTest.java +++ b/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/CancellableQueueTest.java @@ -21,7 +21,9 @@ import static org.hamcrest.Matchers.empty; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import java.lang.ref.WeakReference; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -31,7 +33,7 @@ import java.util.concurrent.Future; import org.apache.beam.sdk.fn.test.TestExecutors; import org.apache.beam.sdk.fn.test.TestExecutors.TestExecutorService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -219,4 +221,30 @@ public void testFirstCancellationError() throws Exception { queue.cancel(new RuntimeException("Second cancel exception")); assertThrows("First cancel exception", RuntimeException.class, () -> queue.take()); } + + @Test + public void testMemoryReferenceOnTake() throws Exception { + String s1 = new String("test1"); + String s2 = new String("test2"); + WeakReference weakReference1 = new WeakReference<>(s1); + WeakReference weakReference2 = new WeakReference<>(s2); + CancellableQueue queue = new CancellableQueue<>(100); + queue.put(s1); + queue.put(s2); + s1 = null; + s2 = null; + System.gc(); + assertTrue(weakReference1.get() != null); + assertTrue(weakReference2.get() != null); + + assertEquals("test1", queue.take()); + System.gc(); + assertTrue(weakReference1.get() == null); + assertTrue(weakReference2.get() != null); + + queue.reset(); + System.gc(); + assertTrue(weakReference1.get() == null); + assertTrue(weakReference2.get() == null); + } } diff --git a/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/data/BeamFnDataGrpcMultiplexerTest.java b/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/data/BeamFnDataGrpcMultiplexerTest.java index b1cb517507f0b..ee388885a632c 100644 --- a/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/data/BeamFnDataGrpcMultiplexerTest.java +++ b/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/data/BeamFnDataGrpcMultiplexerTest.java @@ -37,7 +37,7 @@ import org.apache.beam.sdk.fn.test.TestExecutors.TestExecutorService; import org.apache.beam.sdk.fn.test.TestStreams; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/data/BeamFnDataOutboundAggregatorTest.java b/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/data/BeamFnDataOutboundAggregatorTest.java index 8b2adc6f2e277..6f1dd8271fa50 100644 --- a/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/data/BeamFnDataOutboundAggregatorTest.java +++ b/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/data/BeamFnDataOutboundAggregatorTest.java @@ -41,7 +41,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.util.ByteStringOutputStream; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.hamcrest.Matchers; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/BufferingStreamObserverTest.java b/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/BufferingStreamObserverTest.java index 4fe4ebf31d26b..a294ba0bedc01 100644 --- a/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/BufferingStreamObserverTest.java +++ b/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/BufferingStreamObserverTest.java @@ -31,8 +31,8 @@ import org.apache.beam.sdk.fn.test.TestExecutors; import org.apache.beam.sdk.fn.test.TestExecutors.TestExecutorService; import org.apache.beam.sdk.fn.test.TestStreams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/DataStreamsTest.java b/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/DataStreamsTest.java index 7116cc9bb9177..2c0b99a7ff8ee 100644 --- a/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/DataStreamsTest.java +++ b/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/DataStreamsTest.java @@ -20,6 +20,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.collection.IsCollectionWithSize.hasSize; import static org.junit.Assert.assertArrayEquals; @@ -35,14 +36,15 @@ import java.util.NoSuchElementException; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.StringUtf8Coder; +import org.apache.beam.sdk.fn.data.WeightedList; import org.apache.beam.sdk.fn.stream.DataStreams.DataStreamDecoder; import org.apache.beam.sdk.fn.stream.DataStreams.ElementDelimitedOutputStream; import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.ByteStringOutputStream; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CountingOutputStream; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CountingOutputStream; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.runners.Enclosed; @@ -57,6 +59,7 @@ public class DataStreamsTest { /** Tests for {@link DataStreams.DataStreamDecoder}. */ @RunWith(JUnit4.class) public static class DataStreamDecoderTest { + @Rule public ExpectedException thrown = ExpectedException.none(); @Test @@ -136,12 +139,19 @@ public void testDecodeFromChunkBoundaryToChunkBoundary() throws Exception { singleElementToSplit.substring(0, singleElementToSplit.size() - 1), singleElementToSplit.substring(singleElementToSplit.size() - 1)))); - assertThat(decoder.decodeFromChunkBoundaryToChunkBoundary(), contains("A")); - assertThat(decoder.decodeFromChunkBoundaryToChunkBoundary(), contains("B", "BigElementC")); - assertThat(decoder.decodeFromChunkBoundaryToChunkBoundary(), contains("D")); - assertThat(decoder.decodeFromChunkBoundaryToChunkBoundary(), is(empty())); - assertThat(decoder.decodeFromChunkBoundaryToChunkBoundary(), contains("E", "F")); - assertThat(decoder.decodeFromChunkBoundaryToChunkBoundary(), contains("BigElementG")); + WeightedList weightedListA = decoder.decodeFromChunkBoundaryToChunkBoundary(); + assertThat(weightedListA.getBacking(), contains("A")); + assertThat(weightedListA.getWeight(), equalTo(10L)); // 2 + 8 + + WeightedList weightedListBC = decoder.decodeFromChunkBoundaryToChunkBoundary(); + assertThat(weightedListBC.getBacking(), contains("B", "BigElementC")); + assertThat(weightedListBC.getWeight(), equalTo(29L)); // 2 + 8 + 11 + 8 + + assertThat(decoder.decodeFromChunkBoundaryToChunkBoundary().getBacking(), contains("D")); + assertThat(decoder.decodeFromChunkBoundaryToChunkBoundary().getBacking(), is(empty())); + assertThat(decoder.decodeFromChunkBoundaryToChunkBoundary().getBacking(), contains("E", "F")); + assertThat( + decoder.decodeFromChunkBoundaryToChunkBoundary().getBacking(), contains("BigElementG")); assertFalse(decoder.hasNext()); } @@ -188,6 +198,7 @@ private void testDecoderWith(Coder coder, T[] expected, List /** Tests for {@link ElementDelimitedOutputStream delimited streams}. */ @RunWith(JUnit4.class) public static class ElementDelimitedOutputStreamTest { + @Test public void testNothingWritten() throws Exception { List output = new ArrayList<>(); diff --git a/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/DirectStreamObserverTest.java b/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/DirectStreamObserverTest.java index 6043277216c16..1d442c5f84e0c 100644 --- a/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/DirectStreamObserverTest.java +++ b/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/DirectStreamObserverTest.java @@ -27,6 +27,7 @@ import java.util.Collection; import java.util.List; import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -35,9 +36,10 @@ import org.apache.beam.sdk.fn.test.TestExecutors; import org.apache.beam.sdk.fn.test.TestExecutors.TestExecutorService; import org.apache.beam.sdk.fn.test.TestStreams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -129,6 +131,50 @@ public void testIsReadyIsHonored() throws Exception { streamObserver.onCompleted(); } + @Test + public void testIsReadyIsHonoredTermination() throws Exception { + AdvancingPhaser phaser = new AdvancingPhaser(1); + final AtomicBoolean elementsAllowed = new AtomicBoolean(); + final DirectStreamObserver streamObserver = + new DirectStreamObserver<>( + phaser, + TestStreams.withOnNext( + (String t) -> { + if (phaser.isTerminated()) { + throw new RuntimeException("Test stream terminated."); + } + assertTrue(elementsAllowed.get()); + }) + .withIsReady(elementsAllowed::get) + .build(), + 0); + + // Start all the tasks + List> results = new ArrayList<>(); + for (final String prefix : ImmutableList.of("0", "1", "2", "3", "4")) { + results.add( + executor.submit( + () -> { + for (int i = 0; i < 10; i++) { + streamObserver.onNext(prefix + i); + } + return prefix; + })); + } + + // Have them wait and then terminate the phaser and ensure sends occur. + Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS); + phaser.forceTermination(); + + Assert.assertThrows( + "Test stream terminated.", RuntimeException.class, () -> streamObserver.onNext("100")); + + for (Future result : results) { + Assert.assertThrows(ExecutionException.class, () -> result.get()); + } + streamObserver.onCompleted(); + } + /** * This test specifically covers the case if the outbound observer is being invoked on the same * thread that the inbound observer is. gRPC documentation states: @@ -219,4 +265,50 @@ public void testMessageCheckInterval() throws Exception { prefixesIndex[prefix] += 1; } } + + @Test + public void testPhaserTermination() throws Exception { + final AtomicInteger index = new AtomicInteger(); + ArrayListMultimap values = ArrayListMultimap.create(); + final DirectStreamObserver streamObserver = + new DirectStreamObserver<>( + new AdvancingPhaser(1), + TestStreams.withOnNext((String t) -> assertTrue(values.put(index.get(), t))) + .withIsReady( + () -> { + index.incrementAndGet(); + return true; + }) + .build(), + 10); + + List prefixes = ImmutableList.of("0", "1", "2", "3", "4"); + List> results = new ArrayList<>(); + for (final String prefix : prefixes) { + results.add( + executor.submit( + () -> { + for (int i = 0; i < 10; i++) { + streamObserver.onNext(prefix + i); + } + return prefix; + })); + } + for (Future result : results) { + result.get(); + } + assertEquals(50, values.size()); + for (Collection valuesPerMessageCheck : values.asMap().values()) { + assertThat(valuesPerMessageCheck, hasSize(10)); + } + + // Check that order was maintained per writer. + int[] prefixesIndex = new int[prefixes.size()]; + for (String onNextValue : values.values()) { + int prefix = Integer.parseInt(onNextValue.substring(0, 1)); + int suffix = Integer.parseInt(onNextValue.substring(1, 2)); + assertEquals(prefixesIndex[prefix], suffix); + prefixesIndex[prefix] += 1; + } + } } diff --git a/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/ForwardingClientResponseObserverTest.java b/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/ForwardingClientResponseObserverTest.java index 114e1c68ac4d0..8d0e462bd4e89 100644 --- a/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/ForwardingClientResponseObserverTest.java +++ b/sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/stream/ForwardingClientResponseObserverTest.java @@ -32,23 +32,61 @@ @RunWith(JUnit4.class) public class ForwardingClientResponseObserverTest { @Test - public void testCallsAreForwardedAndOnReadyHandlerBound() { + public void testCallsAreForwardedAndOnReadyHandlerBoundSuccess() { @SuppressWarnings("unchecked") StreamObserver delegateObserver = mock(StreamObserver.class); @SuppressWarnings("unchecked") ClientCallStreamObserver callStreamObserver = mock(ClientCallStreamObserver.class); Runnable onReadyHandler = () -> {}; ClientResponseObserver observer = - new ForwardingClientResponseObserver<>(delegateObserver, onReadyHandler); + ForwardingClientResponseObserver.create(delegateObserver, onReadyHandler); + observer.onNext("A"); + verify(delegateObserver).onNext("A"); + observer.onCompleted(); + verify(delegateObserver).onCompleted(); + observer.beforeStart(callStreamObserver); + verify(callStreamObserver).setOnReadyHandler(onReadyHandler); + verifyNoMoreInteractions(delegateObserver, callStreamObserver); + } + + @Test + public void testCallsAreForwardedAndOnReadyHandlerBoundSuccessWithDoneHandler() { + @SuppressWarnings("unchecked") + StreamObserver delegateObserver = mock(StreamObserver.class); + @SuppressWarnings("unchecked") + ClientCallStreamObserver callStreamObserver = mock(ClientCallStreamObserver.class); + Runnable onReadyHandler = () -> {}; + Runnable onDoneHandler = mock(Runnable.class); + ClientResponseObserver observer = + ForwardingClientResponseObserver.create(delegateObserver, onReadyHandler, onDoneHandler); + observer.onNext("A"); + verify(delegateObserver).onNext("A"); + observer.onCompleted(); + verify(delegateObserver).onCompleted(); + observer.beforeStart(callStreamObserver); + verify(callStreamObserver).setOnReadyHandler(onReadyHandler); + verifyNoMoreInteractions(delegateObserver, callStreamObserver); + verify(onDoneHandler).run(); + } + + @Test + public void testCallsAreForwardedAndOnReadyHandlerBoundError() { + @SuppressWarnings("unchecked") + StreamObserver delegateObserver = mock(StreamObserver.class); + @SuppressWarnings("unchecked") + ClientCallStreamObserver callStreamObserver = mock(ClientCallStreamObserver.class); + Runnable onReadyHandler = () -> {}; + Runnable onDoneHandler = mock(Runnable.class); + ClientResponseObserver observer = + ForwardingClientResponseObserver.create(delegateObserver, onReadyHandler, onDoneHandler); observer.onNext("A"); verify(delegateObserver).onNext("A"); Throwable t = new RuntimeException(); observer.onError(t); verify(delegateObserver).onError(t); - observer.onCompleted(); - verify(delegateObserver).onCompleted(); observer.beforeStart(callStreamObserver); verify(callStreamObserver).setOnReadyHandler(onReadyHandler); verifyNoMoreInteractions(delegateObserver, callStreamObserver); + verify(onDoneHandler).run(); } } diff --git a/sdks/java/harness/OWNERS b/sdks/java/harness/OWNERS deleted file mode 100644 index af32c311923a2..0000000000000 --- a/sdks/java/harness/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lukecwik diff --git a/sdks/java/harness/build.gradle b/sdks/java/harness/build.gradle index 816fee1e52d5c..50eed2f2573b2 100644 --- a/sdks/java/harness/build.gradle +++ b/sdks/java/harness/build.gradle @@ -29,10 +29,11 @@ dependencies { // :sdks:java:core and transitive dependencies provided project(path: ":model:pipeline", configuration: "shadow") provided project(path: ":sdks:java:core", configuration: "shadow") + provided project(path: ":sdks:java:transform-service:launcher", configuration: "shadow") provided library.java.joda_time provided library.java.slf4j_api provided library.java.vendored_grpc_1_54_0 - provided library.java.vendored_guava_26_0_jre + provided library.java.vendored_guava_32_1_2_jre provided library.java.jamm } @@ -46,18 +47,24 @@ applyJavaNature( automaticModuleName: 'org.apache.beam.fn.harness', testShadowJar: true, shadowJarValidationExcludes: [ + "avro/shaded/com/google/**", + "com/thoughtworks/paranamer/**", "junit/**", "io/github/classgraph/**", "nonapi/io/github/classgraph/**", + "org/apache/avro/**", "org/apache/beam/fn/harness/**", "org/apache/beam/model/fnexecution/**", "org/apache/beam/runners/core/**", "org/apache/beam/runners/core/construction/**", "org/apache/beam/sdk/extensions/avro/**", "org/apache/beam/sdk/fn/**", + "org/apache/commons/**", "org/checkerframework/**", + "org/codehaus/jackson/**", "org/hamcrest/**", "org/junit/**", + "org/tukaani/xz/**", ], shadowClosure: { @@ -80,6 +87,7 @@ dependencies { implementation project(":runners:core-construction-java") implementation project(":runners:core-java") implementation project(":sdks:java:fn-execution") + permitUnusedDeclared project(path: ":sdks:java:transform-service:launcher") testImplementation library.java.junit testImplementation library.java.mockito_core shadowTestRuntimeClasspath project(path: ":sdks:java:core", configuration: "shadowTest") diff --git a/sdks/java/harness/jmh/build.gradle b/sdks/java/harness/jmh/build.gradle index fdb67afb6047a..9dfb79bfebb9b 100644 --- a/sdks/java/harness/jmh/build.gradle +++ b/sdks/java/harness/jmh/build.gradle @@ -36,7 +36,7 @@ dependencies { implementation project(path: ":runners:java-fn-execution") implementation project(path: ":model:pipeline", configuration: "shadow") implementation library.java.vendored_grpc_1_54_0 - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.slf4j_api implementation library.java.joda_time runtimeOnly library.java.slf4j_jdk14 diff --git a/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/ProcessBundleBenchmark.java b/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/ProcessBundleBenchmark.java index a5d9b2d813e32..7dc6a10b38f87 100644 --- a/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/ProcessBundleBenchmark.java +++ b/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/ProcessBundleBenchmark.java @@ -18,7 +18,7 @@ package org.apache.beam.fn.harness.jmh; import static org.apache.beam.sdk.util.WindowedValue.valueInGlobalWindow; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.junit.Assert.assertEquals; import java.util.ArrayList; @@ -91,9 +91,9 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; diff --git a/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/control/MetricsBenchmark.java b/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/control/MetricsBenchmark.java index 95892d5490bcd..d3276e471a486 100644 --- a/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/control/MetricsBenchmark.java +++ b/sdks/java/harness/jmh/src/main/java/org/apache/beam/fn/harness/jmh/control/MetricsBenchmark.java @@ -17,7 +17,7 @@ */ package org.apache.beam.fn.harness.jmh.control; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.HashMap; import java.util.Map; diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/AssignWindowsRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/AssignWindowsRunner.java index aa320099ffd18..c9a34b5ce4ed9 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/AssignWindowsRunner.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/AssignWindowsRunner.java @@ -17,7 +17,7 @@ */ package org.apache.beam.fn.harness; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; import java.io.IOException; @@ -33,9 +33,9 @@ import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Instant; /** The Java SDK Harness implementation of the {@link Window.Assign} primitive. */ diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java index 836ea08174d6f..295818837b77c 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataReadRunner.java @@ -17,8 +17,8 @@ */ package org.apache.beam.fn.harness; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.getOnlyElement; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.getOnlyElement; import com.google.auto.service.AutoService; import java.io.IOException; @@ -51,7 +51,7 @@ import org.apache.beam.sdk.fn.data.RemoteGrpcPortRead; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java index 5ddf78bf3b001..c47d86cd645e2 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BeamFnDataWriteRunner.java @@ -17,7 +17,7 @@ */ package org.apache.beam.fn.harness; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.getOnlyElement; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.getOnlyElement; import com.google.auto.service.AutoService; import java.io.IOException; @@ -34,7 +34,7 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.fn.data.RemoteGrpcPortWrite; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * Registers as a consumer with the Beam Fn Data Api. Consumes elements and encodes them for diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/Caches.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/Caches.java index 5b2330b72f6d6..1d291c65510ac 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/Caches.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/Caches.java @@ -30,12 +30,12 @@ import org.apache.beam.sdk.options.SdkHarnessOptions; import org.apache.beam.sdk.util.Weighted; import org.apache.beam.sdk.util.WeightedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheStats; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.RemovalListener; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.RemovalNotification; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Weigher; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheStats; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.RemovalListener; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.RemovalNotification; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Weigher; import org.github.jamm.MemoryMeter; import org.github.jamm.MemoryMeter.Guess; import org.slf4j.Logger; @@ -109,7 +109,7 @@ public static boolean shouldUpdateOnSizeChange(long oldSize, long newSize) { @VisibleForTesting static class ShrinkOnEviction implements RemovalListener> { - private final org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache< + private final org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache< CompositeKey, WeightedValue> cache; private final LongAdder weightInBytes; @@ -120,7 +120,7 @@ static class ShrinkOnEviction implements RemovalListener> getCache() { return cache; @@ -265,7 +265,7 @@ private static WeightedValue addWeightedValue( * specified prefixes. */ private static class SubCache implements Cache { - private final org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache< + private final org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache< CompositeKey, WeightedValue> cache; private final CompositeKeyPrefix keyPrefix; @@ -273,7 +273,7 @@ private static class SubCache implements Cache { private final LongAdder weightInBytes; SubCache( - org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache< + org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache< CompositeKey, WeightedValue> cache, CompositeKeyPrefix keyPrefix, diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/CombineRunners.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/CombineRunners.java index b64f15be01da5..42810693bbe68 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/CombineRunners.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/CombineRunners.java @@ -37,9 +37,9 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** Executes different components of Combine PTransforms. */ @SuppressWarnings({ diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FlattenRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FlattenRunner.java index 7d278c28063cc..1a223ff7de6c7 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FlattenRunner.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FlattenRunner.java @@ -17,7 +17,7 @@ */ package org.apache.beam.fn.harness; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.getOnlyElement; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.getOnlyElement; import com.google.auto.service.AutoService; import java.io.IOException; @@ -25,7 +25,7 @@ import org.apache.beam.runners.core.construction.PTransformTranslation; import org.apache.beam.sdk.fn.data.FnDataReceiver; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** Executes flatten PTransforms. */ @SuppressWarnings({ diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java index eb941bc48b5b3..1800e997b2d83 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java @@ -17,9 +17,9 @@ */ package org.apache.beam.fn.harness; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.service.AutoService; import com.google.auto.value.AutoValue; @@ -116,13 +116,13 @@ import org.apache.beam.sdk.values.WindowingStrategy; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.util.Durations; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Table; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Table; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTimeUtils; import org.joda.time.Duration; diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java index 0d25137beefb7..e103da4d6007d 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnHarness.java @@ -17,6 +17,10 @@ */ package org.apache.beam.fn.harness; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Collections; import java.util.EnumMap; import java.util.Set; @@ -57,10 +61,10 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.TextFormat; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ManagedChannel; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,9 +93,10 @@ public class FnHarness { private static final String CONTROL_API_SERVICE_DESCRIPTOR = "CONTROL_API_SERVICE_DESCRIPTOR"; private static final String LOGGING_API_SERVICE_DESCRIPTOR = "LOGGING_API_SERVICE_DESCRIPTOR"; private static final String STATUS_API_SERVICE_DESCRIPTOR = "STATUS_API_SERVICE_DESCRIPTOR"; + + private static final String PIPELINE_OPTIONS_FILE = "PIPELINE_OPTIONS_FILE"; private static final String PIPELINE_OPTIONS = "PIPELINE_OPTIONS"; private static final String RUNNER_CAPABILITIES = "RUNNER_CAPABILITIES"; - private static final String ENABLE_DATA_SAMPLING_EXPERIMENT = "enable_data_sampling"; private static final Logger LOG = LoggerFactory.getLogger(FnHarness.class); private static Endpoints.ApiServiceDescriptor getApiServiceDescriptor(String descriptor) @@ -117,11 +122,29 @@ public static void main(Function environmentVarGetter) throws Ex "Control location %s%n", environmentVarGetter.apply(CONTROL_API_SERVICE_DESCRIPTOR)); System.out.format( "Status location %s%n", environmentVarGetter.apply(STATUS_API_SERVICE_DESCRIPTOR)); - System.out.format("Pipeline options %s%n", environmentVarGetter.apply(PIPELINE_OPTIONS)); - String id = environmentVarGetter.apply(HARNESS_ID); - PipelineOptions options = - PipelineOptionsTranslation.fromJson(environmentVarGetter.apply(PIPELINE_OPTIONS)); + + String pipelineOptionsJson = environmentVarGetter.apply(PIPELINE_OPTIONS); + // Try looking for a file first. If that exists it should override PIPELINE_OPTIONS to avoid + // maxing out the kernel's environment space + try { + String pipelineOptionsPath = environmentVarGetter.apply(PIPELINE_OPTIONS_FILE); + System.out.format("Pipeline Options File %s%n", pipelineOptionsPath); + if (pipelineOptionsPath != null) { + Path filePath = Paths.get(pipelineOptionsPath); + if (Files.exists(filePath)) { + System.out.format( + "Pipeline Options File %s exists. Overriding existing options.%n", + pipelineOptionsPath); + pipelineOptionsJson = new String(Files.readAllBytes(filePath), StandardCharsets.UTF_8); + } + } + } catch (Exception e) { + System.out.format("Problem loading pipeline options from file: %s%n", e.getMessage()); + } + + System.out.format("Pipeline options %s%n", pipelineOptionsJson); + PipelineOptions options = PipelineOptionsTranslation.fromJson(pipelineOptionsJson); Endpoints.ApiServiceDescriptor loggingApiServiceDescriptor = getApiServiceDescriptor(environmentVarGetter.apply(LOGGING_API_SERVICE_DESCRIPTOR)); @@ -154,7 +177,7 @@ public static void main(Function environmentVarGetter) throws Ex * * @param id Harness ID * @param options The options for this pipeline - * @param runnerCapabilites + * @param runnerCapabilities * @param loggingApiServiceDescriptor * @param controlApiServiceDescriptor * @param statusApiServiceDescriptor @@ -163,7 +186,7 @@ public static void main(Function environmentVarGetter) throws Ex public static void main( String id, PipelineOptions options, - Set runnerCapabilites, + Set runnerCapabilities, Endpoints.ApiServiceDescriptor loggingApiServiceDescriptor, Endpoints.ApiServiceDescriptor controlApiServiceDescriptor, @Nullable Endpoints.ApiServiceDescriptor statusApiServiceDescriptor) @@ -180,7 +203,7 @@ public static void main( main( id, options, - runnerCapabilites, + runnerCapabilities, loggingApiServiceDescriptor, controlApiServiceDescriptor, statusApiServiceDescriptor, @@ -224,7 +247,8 @@ public static void main( options.as(ExecutorOptions.class).getScheduledExecutorService(); ExecutionStateSampler executionStateSampler = new ExecutionStateSampler(options, System::currentTimeMillis); - final DataSampler dataSampler = new DataSampler(); + + final @Nullable DataSampler dataSampler = DataSampler.create(options); // The logging client variable is not used per se, but during its lifetime (until close()) it // intercepts logging and sends it to the logging service. @@ -252,10 +276,6 @@ public static void main( FinalizeBundleHandler finalizeBundleHandler = new FinalizeBundleHandler(executorService); - // Create the sampler, if the experiment is enabled. - boolean shouldSample = - ExperimentalOptions.hasExperiment(options, ENABLE_DATA_SAMPLING_EXPERIMENT); - // Retrieves the ProcessBundleDescriptor from cache. Requests the PBD from the Runner if it // doesn't exist. Additionally, runs any graph modifications. Function getProcessBundleDescriptor = @@ -290,8 +310,7 @@ private BeamFnApi.ProcessBundleDescriptor loadDescriptor(String id) { metricsShortIds, executionStateSampler, processWideCache, - shouldSample ? dataSampler : null); - logging.setProcessBundleHandler(processBundleHandler); + dataSampler); BeamFnStatusClient beamFnStatusClient = null; if (statusApiServiceDescriptor != null) { @@ -339,7 +358,12 @@ private BeamFnApi.ProcessBundleDescriptor loadDescriptor(String id) { InstructionRequest.RequestCase.HARNESS_MONITORING_INFOS, processWideHandler::harnessMonitoringInfos); handlers.put( - InstructionRequest.RequestCase.SAMPLE_DATA, dataSampler::handleDataSampleRequest); + InstructionRequest.RequestCase.SAMPLE_DATA, + request -> + dataSampler == null + ? BeamFnApi.InstructionResponse.newBuilder() + .setSampleData(BeamFnApi.SampleDataResponse.newBuilder()) + : dataSampler.handleDataSampleRequest(request)); JvmInitializers.runBeforeProcessing(options); diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/MapFnRunners.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/MapFnRunners.java index dd7602eff6e9f..ea4d4aa299fc7 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/MapFnRunners.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/MapFnRunners.java @@ -17,14 +17,14 @@ */ package org.apache.beam.fn.harness; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.getOnlyElement; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.getOnlyElement; import java.io.IOException; import org.apache.beam.model.pipeline.v1.RunnerApi.PTransform; import org.apache.beam.sdk.fn.data.FnDataReceiver; import org.apache.beam.sdk.function.ThrowingFunction; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** * Utilities to create {@code PTransformRunners} which execute simple map functions. diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/PrecombineGroupingTable.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/PrecombineGroupingTable.java index f250934e5f76c..06cc17062bdb6 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/PrecombineGroupingTable.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/PrecombineGroupingTable.java @@ -39,7 +39,7 @@ import org.apache.beam.sdk.util.Weighted; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.Instant; /** diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/ToStringFnRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/ToStringFnRunner.java index 28aec1f955f76..5642489a18848 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/ToStringFnRunner.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/ToStringFnRunner.java @@ -24,7 +24,7 @@ import org.apache.beam.runners.core.construction.PTransformTranslation; import org.apache.beam.sdk.function.ThrowingFunction; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * Translates from elements to human-readable string. diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/WindowMappingFnRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/WindowMappingFnRunner.java index 7778f7eccb222..53f1a847364c8 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/WindowMappingFnRunner.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/WindowMappingFnRunner.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.WindowMappingFn; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * Represents mapping of main input window onto side input window. diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/WindowMergingFnRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/WindowMergingFnRunner.java index b110862300cd6..156add3df7c67 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/WindowMergingFnRunner.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/WindowMergingFnRunner.java @@ -33,9 +33,9 @@ import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.transforms.windowing.WindowFn.MergeContext; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; /** * Merges windows using a {@link org.apache.beam.sdk.transforms.windowing.WindowFn}. diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/BeamFnControlClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/BeamFnControlClient.java index ffac6d7aa98e9..876a838f16625 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/BeamFnControlClient.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/BeamFnControlClient.java @@ -17,7 +17,7 @@ */ package org.apache.beam.fn.harness.control; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables.getStackTraceAsString; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables.getStackTraceAsString; import java.util.EnumMap; import java.util.concurrent.CompletableFuture; @@ -118,11 +118,11 @@ public void onNext(BeamFnApi.InstructionRequest request) { sendErrorResponse(e); throw e; } finally { - BeamFnLoggingMDC.setInstructionId(null); + BeamFnLoggingMDC.reset(); } }); } finally { - BeamFnLoggingMDC.setInstructionId(null); + BeamFnLoggingMDC.reset(); } } diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ExecutionStateSampler.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ExecutionStateSampler.java index 2c2485dd842c9..a82ce92768201 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ExecutionStateSampler.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ExecutionStateSampler.java @@ -32,6 +32,7 @@ import java.util.concurrent.atomic.AtomicReference; import javax.annotation.concurrent.GuardedBy; import org.apache.beam.fn.harness.control.ProcessBundleHandler.BundleProcessor; +import org.apache.beam.fn.harness.logging.BeamFnLoggingMDC; import org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo; import org.apache.beam.runners.core.metrics.MetricsContainerStepMap; import org.apache.beam.runners.core.metrics.MonitoringInfoEncodings; @@ -46,7 +47,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.HistogramData; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTimeUtils.MillisProvider; import org.joda.time.Duration; @@ -120,6 +121,14 @@ public interface ExecutionState { *

    Must only be invoked by the bundle processing thread. */ void deactivate(); + + /** + * Sets the error state to the currently executing state. Returns true if this was the first + * time the error was set. Returns false otherwise. + * + *

    This can only be set once. + */ + boolean error(); } /** Stops the execution of the state sampler. */ @@ -250,6 +259,8 @@ public class ExecutionStateTracker implements BundleProgressReporter { private @Nullable ExecutionStateImpl currentState; // Read by multiple threads, written by the bundle processing thread lazily. private final AtomicReference<@Nullable ExecutionStateImpl> currentStateLazy; + // If an exception occurs, this will be to state at the time of exception. + private boolean inErrorState = false; // Read and written by the ExecutionStateSampler thread private long transitionsAtLastSample; @@ -363,9 +374,14 @@ private void takeSample(long currentTimeMillis, long millisSinceLastSample) { ExecutionStateImpl current = currentStateLazy.get(); if (current != null) { return ExecutionStateTrackerStatus.create( - current.ptransformId, current.ptransformUniqueName, thread, lastTransitionTimeMs); + current.ptransformId, + current.ptransformUniqueName, + thread, + lastTransitionTimeMs, + processBundleId.get()); } else { - return ExecutionStateTrackerStatus.create(null, null, thread, lastTransitionTimeMs); + return ExecutionStateTrackerStatus.create( + null, null, thread, lastTransitionTimeMs, processBundleId.get()); } } @@ -460,6 +476,15 @@ public void deactivate() { numTransitions += 1; numTransitionsLazy.lazySet(numTransitions); } + + @Override + public boolean error() { + if (!inErrorState) { + inErrorState = true; + return true; + } + return false; + } } /** @@ -468,6 +493,7 @@ public void deactivate() { *

    Only invoked by the bundle processing thread. */ public void start(String processBundleId) { + BeamFnLoggingMDC.setStateTracker(this); this.processBundleId.lazySet(processBundleId); this.lastTransitionTime.lazySet(clock.getMillis()); this.trackedThread.lazySet(Thread.currentThread()); @@ -509,6 +535,8 @@ public void reset() { this.numTransitionsLazy.lazySet(0); this.lastTransitionTime.lazySet(0); this.metricsContainerRegistry.reset(); + this.inErrorState = false; + BeamFnLoggingMDC.setStateTracker(null); } } @@ -518,9 +546,10 @@ public static ExecutionStateTrackerStatus create( @Nullable String ptransformId, @Nullable String ptransformUniqueName, Thread trackedThread, - long lastTransitionTimeMs) { + long lastTransitionTimeMs, + @Nullable String processBundleId) { return new AutoValue_ExecutionStateSampler_ExecutionStateTrackerStatus( - ptransformId, ptransformUniqueName, trackedThread, lastTransitionTimeMs); + ptransformId, ptransformUniqueName, trackedThread, lastTransitionTimeMs, processBundleId); } public abstract @Nullable String getPTransformId(); @@ -530,5 +559,7 @@ public static ExecutionStateTrackerStatus create( public abstract Thread getTrackedThread(); public abstract long getLastTransitionTimeMillis(); + + public abstract @Nullable String getProcessBundleId(); } } diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/FinalizeBundleHandler.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/FinalizeBundleHandler.java index 5912949883165..d3e9eddf75aa0 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/FinalizeBundleHandler.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/FinalizeBundleHandler.java @@ -17,7 +17,7 @@ */ package org.apache.beam.fn.harness.control; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.util.ArrayList; diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java index 1df4c85b12638..ecec509bf950b 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java @@ -92,16 +92,16 @@ import org.apache.beam.sdk.util.common.ReflectHelpers; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.TextFormat; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LoadingCache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.SetMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.SetMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; import org.slf4j.Logger; diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/PCollectionConsumerRegistry.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/PCollectionConsumerRegistry.java index 150580c7f64fb..e27df577779cc 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/PCollectionConsumerRegistry.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/data/PCollectionConsumerRegistry.java @@ -27,6 +27,7 @@ import javax.annotation.Nullable; import org.apache.beam.fn.harness.HandlesSplits; import org.apache.beam.fn.harness.control.BundleProgressReporter; +import org.apache.beam.fn.harness.control.ExecutionStateSampler; import org.apache.beam.fn.harness.control.ExecutionStateSampler.ExecutionState; import org.apache.beam.fn.harness.control.ExecutionStateSampler.ExecutionStateTracker; import org.apache.beam.fn.harness.control.Metrics; @@ -52,6 +53,8 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder; import org.apache.beam.sdk.util.common.ElementByteSizeObserver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * The {@code PCollectionConsumerRegistry} is used to maintain a collection of consuming @@ -71,9 +74,12 @@ public class PCollectionConsumerRegistry { @SuppressWarnings({"rawtypes"}) abstract static class ConsumerAndMetadata { public static ConsumerAndMetadata forConsumer( - FnDataReceiver consumer, String pTransformId, ExecutionState state) { + FnDataReceiver consumer, + String pTransformId, + ExecutionState state, + ExecutionStateTracker stateTracker) { return new AutoValue_PCollectionConsumerRegistry_ConsumerAndMetadata( - consumer, pTransformId, state); + consumer, pTransformId, state, stateTracker); } public abstract FnDataReceiver getConsumer(); @@ -81,6 +87,8 @@ public static ConsumerAndMetadata forConsumer( public abstract String getPTransformId(); public abstract ExecutionState getExecutionState(); + + public abstract ExecutionStateTracker getExecutionStateTracker(); } private final ExecutionStateTracker stateTracker; @@ -91,6 +99,7 @@ public static ConsumerAndMetadata forConsumer( private final ProcessBundleDescriptor processBundleDescriptor; private final RehydratedComponents rehydratedComponents; private final @Nullable DataSampler dataSampler; + private static final Logger LOG = LoggerFactory.getLogger(PCollectionConsumerRegistry.class); public PCollectionConsumerRegistry( ExecutionStateTracker stateTracker, @@ -176,7 +185,7 @@ public void register( List consumerAndMetadatas = pCollectionIdsToConsumers.computeIfAbsent(pCollectionId, (unused) -> new ArrayList<>()); consumerAndMetadatas.add( - ConsumerAndMetadata.forConsumer(consumer, pTransformId, executionState)); + ConsumerAndMetadata.forConsumer(consumer, pTransformId, executionState, stateTracker)); } /** @@ -236,6 +245,26 @@ public FnDataReceiver> getMultiplexingConsumer(String pCollecti }); } + private static void logAndRethrow( + Exception e, + ExecutionState executionState, + ExecutionStateTracker executionStateTracker, + String ptransformId, + @Nullable OutputSampler outputSampler, + @Nullable ElementSample elementSample) + throws Exception { + ExecutionStateSampler.ExecutionStateTrackerStatus status = executionStateTracker.getStatus(); + String processBundleId = status == null ? null : status.getProcessBundleId(); + if (outputSampler != null) { + outputSampler.exception(elementSample, e, ptransformId, processBundleId); + } + + if (executionState.error()) { + LOG.error("Failed to process element for bundle \"{}\"", processBundleId, e); + } + throw e; + } + /** * A wrapping {@code FnDataReceiver>} which counts the number of elements * consumed by the original {@code FnDataReceiver> consumer} and sets up metrics @@ -250,6 +279,8 @@ private class MetricTrackingFnDataReceiver implements FnDataReceiver sampledByteSizeDistribution; private final Coder coder; private final @Nullable OutputSampler outputSampler; + private final String ptransformId; + private final ExecutionStateTracker executionStateTracker; public MetricTrackingFnDataReceiver( String pCollectionId, @@ -258,6 +289,8 @@ public MetricTrackingFnDataReceiver( @Nullable OutputSampler outputSampler) { this.delegate = consumerAndMetadata.getConsumer(); this.executionState = consumerAndMetadata.getExecutionState(); + this.executionStateTracker = consumerAndMetadata.getExecutionStateTracker(); + this.ptransformId = consumerAndMetadata.getPTransformId(); HashMap labels = new HashMap<>(); labels.put(Labels.PCOLLECTION, pCollectionId); @@ -314,10 +347,8 @@ public void accept(WindowedValue input) throws Exception { try { this.delegate.accept(input); } catch (Exception e) { - if (outputSampler != null) { - outputSampler.exception(elementSample, e); - } - throw e; + logAndRethrow( + e, executionState, executionStateTracker, ptransformId, outputSampler, elementSample); } finally { executionState.deactivate(); } @@ -406,10 +437,13 @@ public void accept(WindowedValue input) throws Exception { try { consumerAndMetadata.getConsumer().accept(input); } catch (Exception e) { - if (outputSampler != null) { - outputSampler.exception(elementSample, e); - } - throw e; + logAndRethrow( + e, + state, + consumerAndMetadata.getExecutionStateTracker(), + consumerAndMetadata.getPTransformId(), + outputSampler, + elementSample); } finally { state.deactivate(); } diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/debug/DataSampler.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/debug/DataSampler.java index 2a13b5dac3d3a..29011b82a4dce 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/debug/DataSampler.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/debug/DataSampler.java @@ -17,15 +17,18 @@ */ package org.apache.beam.fn.harness.debug; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.Nullable; import org.apache.beam.model.fnexecution.v1.BeamFnApi; import org.apache.beam.model.fnexecution.v1.BeamFnApi.SampleDataResponse.ElementList; import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.options.ExperimentalOptions; +import org.apache.beam.sdk.options.PipelineOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,19 +40,66 @@ */ public class DataSampler { private static final Logger LOG = LoggerFactory.getLogger(DataSampler.class); + private static final String ENABLE_DATA_SAMPLING_EXPERIMENT = "enable_data_sampling"; + private static final String ENABLE_ALWAYS_ON_EXCEPTION_SAMPLING_EXPERIMENT = + "enable_always_on_exception_sampling"; + private static final String DISABLE_ALWAYS_ON_EXCEPTION_SAMPLING_EXPERIMENT = + "disable_always_on_exception_sampling"; + + /** + * Optionally returns a DataSampler if the experiment "enable_data_sampling" is present or + * "enable_always_on_exception_sampling" is present. Returns null is data sampling is not enabled + * or "disable_always_on_exception_sampling" experiment is given. + * + * @param options the pipeline options given to this SDK Harness. + * @return the DataSampler if enabled or null, otherwise. + */ + public static @Nullable DataSampler create(PipelineOptions options) { + boolean disableAlwaysOnExceptionSampling = + ExperimentalOptions.hasExperiment(options, DISABLE_ALWAYS_ON_EXCEPTION_SAMPLING_EXPERIMENT); + boolean enableAlwaysOnExceptionSampling = + ExperimentalOptions.hasExperiment(options, ENABLE_ALWAYS_ON_EXCEPTION_SAMPLING_EXPERIMENT); + boolean enableDataSampling = + ExperimentalOptions.hasExperiment(options, ENABLE_DATA_SAMPLING_EXPERIMENT); + // Enable exception sampling, unless the user specifies for it to be disabled. + enableAlwaysOnExceptionSampling = + enableAlwaysOnExceptionSampling && !disableAlwaysOnExceptionSampling; + + // If no sampling is enabled, don't create the DataSampler. + if (enableDataSampling || enableAlwaysOnExceptionSampling) { + // For performance reasons, sampling all elements should only be done when the user requests + // it. + // But, exception sampling doesn't need to worry about performance implications, since the SDK + // is already in a bad state. Thus, enable only exception sampling when the user does not + // request for the sampling of all elements. + boolean onlySampleExceptions = enableAlwaysOnExceptionSampling && !enableDataSampling; + return new DataSampler(onlySampleExceptions); + } else { + return null; + } + } /** * Creates a DataSampler to sample every 1000 elements while keeping a maximum of 10 in memory. */ public DataSampler() { - this(10, 1000); + this(10, 1000, false); + } + + /** + * Creates a DataSampler to sample every 1000 elements while keeping a maximum of 10 in memory. + * + * @param onlySampleExceptions If true, only samples elements from exceptions. + */ + public DataSampler(Boolean onlySampleExceptions) { + this(10, 1000, onlySampleExceptions); } /** * @param maxSamples Sets the maximum number of samples held in memory at once. * @param sampleEveryN Sets how often to sample. */ - public DataSampler(int maxSamples, int sampleEveryN) { + public DataSampler(int maxSamples, int sampleEveryN, Boolean onlySampleExceptions) { checkArgument( maxSamples > 0, "Expected positive number of samples, did you mean to disable data sampling?"); @@ -58,6 +108,7 @@ public DataSampler(int maxSamples, int sampleEveryN) { "Expected positive number for sampling period, did you mean to disable data sampling?"); this.maxSamples = maxSamples; this.sampleEveryN = sampleEveryN; + this.onlySampleExceptions = onlySampleExceptions; } // Maximum number of elements in buffer. @@ -66,6 +117,9 @@ public DataSampler(int maxSamples, int sampleEveryN) { // Sampling rate. private final int sampleEveryN; + // If true, only takes samples when exceptions in UDFs occur. + private final Boolean onlySampleExceptions; + // The fully-qualified type is: Map[PCollectionId, OutputSampler]. In order to sample // on a PCollection-basis and not per-bundle, this keeps track of shared samples between states. private final Map> outputSamplers = new ConcurrentHashMap<>(); @@ -86,7 +140,10 @@ public DataSampler(int maxSamples, int sampleEveryN) { public OutputSampler sampleOutput(String pcollectionId, Coder coder) { return (OutputSampler) outputSamplers.computeIfAbsent( - pcollectionId, k -> new OutputSampler<>(coder, this.maxSamples, this.sampleEveryN)); + pcollectionId, + k -> + new OutputSampler<>( + coder, this.maxSamples, this.sampleEveryN, this.onlySampleExceptions)); } /** diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/debug/ElementSample.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/debug/ElementSample.java index 85abd02e1d96f..4ef1674c9ec66 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/debug/ElementSample.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/debug/ElementSample.java @@ -34,8 +34,21 @@ public class ElementSample { // The element sample to be serialized and later queried. public final WindowedValue sample; + public static class ExceptionMetadata { + ExceptionMetadata(String message, String ptransformId) { + this.message = message; + this.ptransformId = ptransformId; + } + + // The stringified exception that caused the bundle to fail. + public final String message; + + // The PTransform of where the exception occurred first. + public final String ptransformId; + } + // An optional exception to be given as metadata on the FnApi for the given sample. - @Nullable public Exception exception = null; + @Nullable public ExceptionMetadata exception = null; ElementSample(long id, WindowedValue sample) { this.id = id; diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/debug/OutputSampler.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/debug/OutputSampler.java index a81796a239ae1..f7fabae0cc213 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/debug/OutputSampler.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/debug/OutputSampler.java @@ -19,8 +19,10 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicLong; import javax.annotation.Nullable; @@ -44,7 +46,7 @@ public class OutputSampler { // Temporarily holds exceptional elements. These elements can also be duplicated in the main // buffer. This is in order to always track exceptional elements even if the number of samples in // the main buffer drops it. - private final List> exceptions = new ArrayList<>(); + private Map> exceptions = new HashMap<>(); // Maximum number of elements in buffer. private final int maxElements; @@ -58,14 +60,19 @@ public class OutputSampler { // Index into the buffer of where to overwrite samples. private int resampleIndex = 0; + // If true, only takes samples when exceptions in UDFs occur. + private final Boolean onlySampleExceptions; + @Nullable private final Coder valueCoder; @Nullable private final Coder> windowedValueCoder; - public OutputSampler(Coder coder, int maxElements, int sampleEveryN) { + public OutputSampler( + Coder coder, int maxElements, int sampleEveryN, boolean onlySampleExceptions) { this.maxElements = maxElements; this.sampleEveryN = sampleEveryN; this.buffer = new ArrayList<>(this.maxElements); + this.onlySampleExceptions = onlySampleExceptions; // The samples taken and encoded should match exactly to the specification from the // ProcessBundleDescriptor. The coder given can either be a WindowedValueCoder, in which the @@ -101,7 +108,7 @@ public ElementSample sample(WindowedValue element) { ElementSample elementSample = new ElementSample<>(ThreadLocalRandom.current().nextInt(), element); - if (samples > 10 && samples % sampleEveryN != 0) { + if (onlySampleExceptions || (samples > 10 && samples % sampleEveryN != 0)) { return elementSample; } @@ -120,22 +127,68 @@ public ElementSample sample(WindowedValue element) { } /** - * Samples an exceptional element to be later queried. + * Samples an exceptional element to be later queried. The enforces that only one exception occurs + * per bundle. * * @param elementSample the sampled element to add an exception to. * @param e the exception. + * @param ptransformId the source of the exception. + * @param processBundleId the failing bundle. */ - public void exception(ElementSample elementSample, Exception e) { - if (elementSample == null) { + public void exception( + ElementSample elementSample, Exception e, String ptransformId, String processBundleId) { + if (elementSample == null || processBundleId == null) { return; } synchronized (this) { - elementSample.exception = e; - exceptions.add(elementSample); + exceptions.computeIfAbsent( + processBundleId, + pbId -> { + elementSample.exception = + new ElementSample.ExceptionMetadata(e.toString(), ptransformId); + return elementSample; + }); } } + /** + * Fills and returns the BeamFnApi proto. + * + * @param sample the sampled element. + * @param stream the stream to use to serialize the element. + * @param processBundleId the bundle the element belongs to. Currently only set when there is an + * exception. + */ + private BeamFnApi.SampledElement sampleToProto( + ElementSample sample, ByteStringOutputStream stream, @Nullable String processBundleId) + throws IOException { + if (valueCoder != null) { + this.valueCoder.encode(sample.sample.getValue(), stream, Coder.Context.NESTED); + } else if (windowedValueCoder != null) { + this.windowedValueCoder.encode(sample.sample, stream, Coder.Context.NESTED); + } + + BeamFnApi.SampledElement.Builder elementBuilder = + BeamFnApi.SampledElement.newBuilder().setElement(stream.toByteStringAndReset()); + + ElementSample.ExceptionMetadata exception = sample.exception; + if (exception != null) { + BeamFnApi.SampledElement.Exception.Builder exceptionBuilder = + BeamFnApi.SampledElement.Exception.newBuilder() + .setTransformId(exception.ptransformId) + .setError(exception.message); + + if (processBundleId != null) { + exceptionBuilder.setInstructionId(processBundleId); + } + + elementBuilder.setException(exceptionBuilder); + } + + return elementBuilder.build(); + } + /** * Clears samples at end of call. This is to help mitigate memory use. * @@ -150,11 +203,16 @@ public List samples() throws IOException { // Serializing can take a lot of CPU time for larger or complex elements. Copy the array here // so as to not slow down the main processing hot path. List> bufferToSend; + Map> exceptionsToSend; int sampleIndex = 0; synchronized (this) { bufferToSend = buffer; - sampleIndex = resampleIndex; buffer = new ArrayList<>(maxElements); + + exceptionsToSend = exceptions; + exceptions = new HashMap<>(exceptions.size()); + + sampleIndex = resampleIndex; resampleIndex = 0; } @@ -162,39 +220,25 @@ public List samples() throws IOException { // to deduplicate samples. HashSet seen = new HashSet<>(); ByteStringOutputStream stream = new ByteStringOutputStream(); - for (int i = 0; i < bufferToSend.size(); i++) { - int index = (sampleIndex + i) % bufferToSend.size(); - ElementSample sample = bufferToSend.get(index); + for (Map.Entry> pair : exceptionsToSend.entrySet()) { + String processBundleId = pair.getKey(); + ElementSample sample = pair.getValue(); seen.add(sample.id); - if (valueCoder != null) { - this.valueCoder.encode(sample.sample.getValue(), stream, Coder.Context.NESTED); - } else if (windowedValueCoder != null) { - this.windowedValueCoder.encode(sample.sample, stream, Coder.Context.NESTED); - } - - ret.add( - BeamFnApi.SampledElement.newBuilder().setElement(stream.toByteStringAndReset()).build()); + ret.add(sampleToProto(sample, stream, processBundleId)); } - // TODO: set the exception metadata on the proto once that PR is merged. - for (ElementSample sample : exceptions) { + for (int i = 0; i < bufferToSend.size(); i++) { + int index = (sampleIndex + i) % bufferToSend.size(); + + ElementSample sample = bufferToSend.get(index); if (seen.contains(sample.id)) { continue; } - if (valueCoder != null) { - this.valueCoder.encode(sample.sample.getValue(), stream, Coder.Context.NESTED); - } else if (windowedValueCoder != null) { - this.windowedValueCoder.encode(sample.sample, stream, Coder.Context.NESTED); - } - - ret.add( - BeamFnApi.SampledElement.newBuilder().setElement(stream.toByteStringAndReset()).build()); + ret.add(sampleToProto(sample, stream, null)); } - exceptions.clear(); - return ret; } } diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java index 3b76f7fc4d01f..8fa074b047683 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClient.java @@ -18,8 +18,8 @@ package org.apache.beam.fn.harness.logging; import static java.util.concurrent.TimeUnit.SECONDS; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables.getStackTraceAsString; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables.getStackTraceAsString; import java.io.IOException; import java.util.ArrayList; @@ -40,8 +40,7 @@ import java.util.logging.LogRecord; import java.util.logging.Logger; import java.util.logging.SimpleFormatter; -import org.apache.beam.fn.harness.control.ProcessBundleHandler; -import org.apache.beam.fn.harness.control.ProcessBundleHandler.BundleProcessor; +import org.apache.beam.fn.harness.control.ExecutionStateSampler; import org.apache.beam.model.fnexecution.v1.BeamFnApi; import org.apache.beam.model.fnexecution.v1.BeamFnApi.LogEntry; import org.apache.beam.model.fnexecution.v1.BeamFnLoggingGrpc; @@ -59,8 +58,8 @@ import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.ClientCallStreamObserver; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.ClientResponseObserver; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.initialization.qual.UnderInitialization; import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.RequiresNonNull; @@ -105,8 +104,6 @@ public class BeamFnLoggingClient implements AutoCloseable { * so if they are garbage collected, our hierarchical configuration will be lost. */ private final Collection configuredLoggers = new ArrayList<>(); - private @Nullable ProcessBundleHandler processBundleHandler; - private final BlockingQueue bufferedLogEntries = new ArrayBlockingQueue<>(MAX_BUFFERED_LOG_ENTRY_COUNT); @@ -347,10 +344,6 @@ public void close() throws Exception { } } - public void setProcessBundleHandler(ProcessBundleHandler processBundleHandler) { - this.processBundleHandler = processBundleHandler; - } - // Reset the logging configuration to what it is at startup. @RequiresNonNull("configuredLoggers") @RequiresNonNull("logRecordHandler") @@ -375,7 +368,9 @@ void flushFinalLogs(@UnderInitialization BeamFnLoggingClient this) { bufferedLogEntries.drainTo(finalLogEntries); for (BeamFnApi.LogEntry logEntry : finalLogEntries) { LogRecord logRecord = - new LogRecord(REVERSE_LOG_LEVEL_MAP.get(logEntry.getSeverity()), logEntry.getMessage()); + new LogRecord( + checkNotNull(REVERSE_LOG_LEVEL_MAP.get(logEntry.getSeverity())), + logEntry.getMessage()); logRecord.setLoggerName(logEntry.getLogLocation()); logRecord.setMillis( logEntry.getTimestamp().getSeconds() * 1000 @@ -438,14 +433,12 @@ public void publish(LogRecord record) { if (loggerName != null) { builder.setLogLocation(loggerName); } - if (instructionId != null && processBundleHandler != null) { - BundleProcessor bundleProcessor = - processBundleHandler.getBundleProcessorCache().find(instructionId); - if (bundleProcessor != null) { - String transformId = bundleProcessor.getStateTracker().getCurrentThreadsPTransformId(); - if (transformId != null) { - builder.setTransformId(transformId); - } + + ExecutionStateSampler.ExecutionStateTracker stateTracker = BeamFnLoggingMDC.getStateTracker(); + if (stateTracker != null) { + String transformId = stateTracker.getCurrentThreadsPTransformId(); + if (transformId != null) { + builder.setTransformId(transformId); } } diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingMDC.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingMDC.java index bcfcd4b34ea5e..68b03a484904e 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingMDC.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/logging/BeamFnLoggingMDC.java @@ -17,12 +17,16 @@ */ package org.apache.beam.fn.harness.logging; +import org.apache.beam.fn.harness.control.ExecutionStateSampler.ExecutionStateTracker; import org.checkerframework.checker.nullness.qual.Nullable; /** Mapped diagnostic context to be consumed and set on LogEntry protos in BeamFnLoggingClient. */ public class BeamFnLoggingMDC { private static final ThreadLocal<@Nullable String> instructionId = new ThreadLocal<>(); + private static final ThreadLocal<@Nullable ExecutionStateTracker> stateTracker = + new ThreadLocal<>(); + /** Sets the Instruction ID of the current thread, which will be inherited by child threads. */ public static void setInstructionId(@Nullable String newInstructionId) { instructionId.set(newInstructionId); @@ -32,4 +36,20 @@ public static void setInstructionId(@Nullable String newInstructionId) { public static @Nullable String getInstructionId() { return instructionId.get(); } + + /** Sets the State Tracker of the current thread, which will be inherited by child threads. */ + public static void setStateTracker(@Nullable ExecutionStateTracker newStateTracker) { + stateTracker.set(newStateTracker); + } + + /** Gets the State Tracker of the current thread. */ + public static @Nullable ExecutionStateTracker getStateTracker() { + return stateTracker.get(); + } + + /** Resets to a default state. */ + public static void reset() { + instructionId.set(null); + stateTracker.set(null); + } } diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BagUserState.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BagUserState.java index 57bd1a0e301ec..64094a9b62b26 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BagUserState.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/BagUserState.java @@ -17,8 +17,8 @@ */ package org.apache.beam.fn.harness.state; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.ArrayList; import java.util.Collections; @@ -33,7 +33,7 @@ import org.apache.beam.sdk.fn.stream.PrefetchableIterable; import org.apache.beam.sdk.fn.stream.PrefetchableIterables; import org.apache.beam.sdk.util.ByteStringOutputStream; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** * An implementation of a bag user state that utilizes the Beam Fn State API to fetch, clear and diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiStateAccessor.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiStateAccessor.java index f7cd58be18594..69d9a1ff6c8ea 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiStateAccessor.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiStateAccessor.java @@ -17,8 +17,8 @@ */ package org.apache.beam.fn.harness.state; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.util.ArrayList; @@ -62,9 +62,9 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; /** Provides access to side inputs and state via a {@link BeamFnStateClient}. */ diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiTimerBundleTracker.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiTimerBundleTracker.java index c8f9b2930fef9..01b67c67cf3ee 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiTimerBundleTracker.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiTimerBundleTracker.java @@ -17,7 +17,7 @@ */ package org.apache.beam.fn.harness.state; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.io.IOException; @@ -34,11 +34,11 @@ import org.apache.beam.sdk.util.ByteStringOutputStream; import org.apache.beam.sdk.util.UserCodeException; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ComparisonChain; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashBasedTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Table; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Table.Cell; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ComparisonChain; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBasedTable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Table; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Table.Cell; public class FnApiTimerBundleTracker { private final Supplier encodedCurrentKeySupplier; diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/IterableSideInput.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/IterableSideInput.java index a895b88abe66c..7f87cb4d4e41e 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/IterableSideInput.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/IterableSideInput.java @@ -17,7 +17,7 @@ */ package org.apache.beam.fn.harness.state; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import org.apache.beam.fn.harness.Cache; import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateKey; diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/LazyCachingIteratorToIterable.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/LazyCachingIteratorToIterable.java index d7e76fa9eb090..385ee8c606da4 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/LazyCachingIteratorToIterable.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/LazyCachingIteratorToIterable.java @@ -24,7 +24,7 @@ import java.util.Objects; import org.apache.beam.sdk.fn.stream.PrefetchableIterables; import org.apache.beam.sdk.fn.stream.PrefetchableIterator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/MultimapSideInput.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/MultimapSideInput.java index cd9b7eda60232..4bba038757748 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/MultimapSideInput.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/MultimapSideInput.java @@ -17,7 +17,7 @@ */ package org.apache.beam.fn.harness.state; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import org.apache.beam.fn.harness.Cache; diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/MultimapUserState.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/MultimapUserState.java index d1a2fa5129682..112ded3161c1e 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/MultimapUserState.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/MultimapUserState.java @@ -17,8 +17,8 @@ */ package org.apache.beam.fn.harness.state; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.util.ArrayList; @@ -44,7 +44,7 @@ import org.apache.beam.sdk.util.ByteStringOutputStream; import org.apache.beam.sdk.values.KV; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; /** * An implementation of a multimap user state that utilizes the Beam Fn State API to fetch, clear diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateBackedIterable.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateBackedIterable.java index d5b4af8d334d3..9c95e9ad90ee3 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateBackedIterable.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateBackedIterable.java @@ -44,11 +44,11 @@ import org.apache.beam.sdk.util.BufferedElementCountingOutputStream; import org.apache.beam.sdk.util.VarInt; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; /** * A {@link BeamFnStateClient state} backed iterable which allows for fetching elements over the diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java index 190b9463e16aa..00abd835e3af0 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/StateFetchingIterators.java @@ -17,7 +17,7 @@ */ package org.apache.beam.fn.harness.state; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.util.ArrayList; @@ -38,13 +38,14 @@ import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest; import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse; import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.fn.data.WeightedList; import org.apache.beam.sdk.fn.stream.DataStreams.DataStreamDecoder; import org.apache.beam.sdk.fn.stream.PrefetchableIterables; import org.apache.beam.sdk.fn.stream.PrefetchableIterator; import org.apache.beam.sdk.util.Weighted; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; /** * Adapters which convert a logical series of chunks using continuation tokens over the Beam Fn @@ -92,6 +93,7 @@ public static CachingStateIterable readAllAndDecodeStartingFrom( @VisibleForTesting static class IterableCacheKey implements Weighted { + private IterableCacheKey() {} static final IterableCacheKey INSTANCE = new IterableCacheKey(); @@ -110,10 +112,12 @@ static class CachingStateIterable extends PrefetchableIterables.Default { /** Represents a set of elements. */ abstract static class Blocks implements Weighted { + public abstract List> getBlocks(); } static class MutatedBlocks extends Blocks { + private final Block wholeBlock; MutatedBlocks(Block wholeBlock) { @@ -143,19 +147,12 @@ private static long sumWeight(List> blocks) { } } - private static long addBoundByMax(long first, long second) { - try { - return Math.addExact(first, second); - } catch (ArithmeticException e) { - return Long.MAX_VALUE; - } - } - /** * Represents a logical prefix of elements for the logical stream over the state API. This * prefix cannot represent a mutated in memory representation. */ static class BlocksPrefix extends Blocks implements Shrinkable> { + private final List> blocks; @Override @@ -185,14 +182,24 @@ public List> getBlocks() { @AutoValue abstract static class Block implements Weighted { + public static Block mutatedBlock(List values, long weight) { + return mutatedBlock(new WeightedList<>(values, weight)); + } + + public static Block mutatedBlock(WeightedList weightedList) { return new AutoValue_StateFetchingIterators_CachingStateIterable_Block<>( - values, null, weight); + weightedList.getBacking(), null, weightedList.getWeight()); } public static Block fromValues(List values, @Nullable ByteString nextToken) { + return fromValues(new WeightedList<>(values, Caches.weigh(values)), nextToken); + } + + public static Block fromValues( + WeightedList values, @Nullable ByteString nextToken) { return new AutoValue_StateFetchingIterators_CachingStateIterable_Block<>( - values, nextToken, Caches.weigh(values) + Caches.weigh(nextToken)); + values.getBacking(), nextToken, values.getWeight() + Caches.weigh(nextToken)); } abstract List getValues(); @@ -243,37 +250,36 @@ public void remove(Set toRemoveStructuralValues) { } // Combine all the individual blocks into one block containing all the values since - // they were mutated and we must evict all or none of the blocks. When consuming the blocks, + // they were mutated, and we must evict all or none of the blocks. When consuming the blocks, // we must have a reference to all or none of the blocks (which forces a load). List> blocks = existing.getBlocks(); - long totalWeight = 0; int totalSize = 0; - for (int i = 0; i < blocks.size(); ++i) { - totalSize += blocks.get(i).getValues().size(); + for (Block tBlock : blocks) { + totalSize += tBlock.getValues().size(); } - List allValues = new ArrayList<>(totalSize); - for (int i = 0; i < blocks.size(); ++i) { - int startIndex = allValues.size(); - for (T value : blocks.get(i).getValues()) { + + WeightedList allValues = new WeightedList<>(new ArrayList<>(totalSize), 0L); + for (Block block : blocks) { + boolean valueRemovedFromBlock = false; + List blockValuesToKeep = new ArrayList<>(); + for (T value : block.getValues()) { if (!toRemoveStructuralValues.contains(valueCoder.structuralValue(value))) { - allValues.add(value); + blockValuesToKeep.add(value); + } else { + valueRemovedFromBlock = true; } } - // If we didn't remove a value then use the weight of the entire block that has - // already been calculated as a shortcut. - if (startIndex + blocks.get(i).getValues().size() == allValues.size()) { - totalWeight = addBoundByMax(totalWeight, blocks.get(i).getWeight()); - continue; - } - // Fallback to getting the weight of each element. - for (int j = startIndex; j < allValues.size(); ++j) { - totalWeight = addBoundByMax(totalWeight, Caches.weigh(allValues.get(j))); + + // If any value was removed from this block, need to estimate the weight again. + // Otherwise, just reuse the block's weight. + if (valueRemovedFromBlock) { + allValues.addAll(blockValuesToKeep, Caches.weigh(block.getValues())); + } else { + allValues.addAll(block.getValues(), block.getWeight()); } } - cache.put( - IterableCacheKey.INSTANCE, - new MutatedBlocks<>(Block.mutatedBlock(allValues, totalWeight))); + cache.put(IterableCacheKey.INSTANCE, new MutatedBlocks<>(Block.mutatedBlock(allValues))); } /** @@ -285,9 +291,20 @@ public void remove(Set toRemoveStructuralValues) { * requesting data from the state cache. */ public void clearAndAppend(List values) { - cache.put( - IterableCacheKey.INSTANCE, - new MutatedBlocks<>(Block.mutatedBlock(values, Caches.weigh(values)))); + clearAndAppend(new WeightedList<>(values, Caches.weigh(values))); + } + + /** + * Clears the cached iterable and appends the set of values wrapped as a {@link + * WeightedList}, taking ownership of the list. + * + *

    Mutations over the Beam Fn State API must have been performed before any future lookups. + * + *

    Ensures that a cache entry exists for the entire iterable enabling future lookups to miss + * requesting data from the state cache. + */ + public void clearAndAppend(WeightedList values) { + cache.put(IterableCacheKey.INSTANCE, new MutatedBlocks<>(Block.mutatedBlock(values))); } @Override @@ -304,6 +321,18 @@ public PrefetchableIterator createIterator() { * cache. */ public void append(List values) { + append(new WeightedList<>(values, Caches.weigh(values))); + } + + /** + * Appends the values, wrapped as a {@link WeightedList}, to the cached iterable. + * + *

    Mutations over the Beam Fn State API must have been performed before any future lookups. + * + *

    A cache entry will only continue to exist if the entire iterable has been loaded into the + * cache. + */ + public void append(WeightedList values) { if (values.isEmpty()) { return; } @@ -318,23 +347,20 @@ public void append(List values) { } // Combine all the individual blocks into one block containing all the values since - // they were mutated and we must evict all or none of the blocks. When consuming the blocks, + // they were mutated, and we must evict all or none of the blocks. When consuming the blocks, // we must have a reference to all or none of the blocks (which forces a load). List> blocks = existing.getBlocks(); - long totalWeight = addBoundByMax(Caches.weigh(values), sumWeight(blocks)); int totalSize = values.size(); - for (int i = 0; i < blocks.size(); ++i) { - totalSize += blocks.get(i).getValues().size(); + for (Block block : blocks) { + totalSize += block.getValues().size(); } - List allValues = new ArrayList<>(totalSize); - for (int i = 0; i < blocks.size(); ++i) { - allValues.addAll(blocks.get(i).getValues()); + WeightedList allValues = new WeightedList<>(new ArrayList<>(totalSize), 0L); + for (Block block : blocks) { + allValues.addAll(block.getValues(), block.getWeight()); } allValues.addAll(values); - cache.put( - IterableCacheKey.INSTANCE, - new MutatedBlocks<>(Block.mutatedBlock(allValues, totalWeight))); + cache.put(IterableCacheKey.INSTANCE, new MutatedBlocks<>(Block.mutatedBlock(allValues))); } class CachingStateIterator implements PrefetchableIterator { @@ -351,7 +377,8 @@ public CachingStateIterator() { new DataStreamDecoder<>(valueCoder, underlyingStateFetchingIterator); this.currentBlock = Block.fromValues( - Collections.emptyList(), stateRequestForFirstChunk.getGet().getContinuationToken()); + new WeightedList<>(Collections.emptyList(), 0L), + stateRequestForFirstChunk.getGet().getContinuationToken()); this.currentCachedBlockValueIndex = 0; } @@ -464,7 +491,7 @@ public boolean hasNext() { @VisibleForTesting Block loadNextBlock(ByteString continuationToken) { underlyingStateFetchingIterator.seekToContinuationToken(continuationToken); - List values = dataStreamDecoder.decodeFromChunkBoundaryToChunkBoundary(); + WeightedList values = dataStreamDecoder.decodeFromChunkBoundaryToChunkBoundary(); ByteString nextToken = underlyingStateFetchingIterator.getContinuationToken(); if (ByteString.EMPTY.equals(nextToken)) { nextToken = null; diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/status/BeamFnStatusClient.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/status/BeamFnStatusClient.java index f4362d962f17a..ef8ec626322c4 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/status/BeamFnStatusClient.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/status/BeamFnStatusClient.java @@ -40,7 +40,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ManagedChannel; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTimeUtils; import org.slf4j.Logger; diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/status/MemoryMonitor.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/status/MemoryMonitor.java index f9e91eda7f63f..b1cc3a56ee9ba 100644 --- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/status/MemoryMonitor.java +++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/status/MemoryMonitor.java @@ -43,11 +43,13 @@ import org.apache.beam.sdk.io.fs.CreateOptions.StandardCreateOptions; import org.apache.beam.sdk.io.fs.ResourceId; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.AtomicDouble; +import org.apache.beam.sdk.options.PortablePipelineOptions; +import org.apache.beam.sdk.options.SdkHarnessOptions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.AtomicDouble; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -95,14 +97,6 @@ public class MemoryMonitor implements Runnable { */ private static final double GC_THRASHING_PERCENTAGE_PER_SERVER = 60.0; - /** - * The GC thrashing threshold percentage. A given period of time is considered "thrashing" if this - * percentage of CPU time is spent in garbage collection. - * - *

    If {@literal 100} is given as the value, MemoryMonitor will be disabled. - */ - private static final double GC_THRASHING_PERCENTAGE_PER_PERIOD = 50.0; - /** * The amount of memory (in bytes) we should pre-allocate, in order to be able to dump the heap. * @@ -201,15 +195,18 @@ public long totalGCTimeMilliseconds() { private final File localDumpFolder; public static MemoryMonitor fromOptions(PipelineOptions options) { + SdkHarnessOptions sdkHarnessOptions = options.as(SdkHarnessOptions.class); String uploadFilePath = options.getTempLocation(); - boolean canDumpHeap = uploadFilePath != null; + PortablePipelineOptions portableOptions = options.as(PortablePipelineOptions.class); + boolean canDumpHeap = uploadFilePath != null && portableOptions.getEnableHeapDumps(); + double gcThrashingPercentagePerPeriod = sdkHarnessOptions.getGCThrashingPercentagePerPeriod(); return new MemoryMonitor( new SystemGCStatsProvider(), DEFAULT_SLEEP_TIME_MILLIS, DEFAULT_SHUT_DOWN_AFTER_NUM_GCTHRASHING, canDumpHeap, - GC_THRASHING_PERCENTAGE_PER_PERIOD, + gcThrashingPercentagePerPeriod, uploadFilePath, getLoggingDir()); } diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/AssignWindowsRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/AssignWindowsRunnerTest.java index 57340148653fb..659afff2a8855 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/AssignWindowsRunnerTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/AssignWindowsRunnerTest.java @@ -50,8 +50,8 @@ import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.transforms.windowing.WindowMappingFn; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java index 08843a89ced86..e8ad49eda1254 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataReadRunnerTest.java @@ -68,9 +68,9 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.hamcrest.collection.IsMapContaining; import org.junit.Before; import org.junit.Rule; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java index 408705312b705..13b6d0a7a29b6 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BeamFnDataWriteRunnerTest.java @@ -54,7 +54,7 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.hamcrest.collection.IsMapContaining; import org.junit.Before; import org.junit.Test; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/CombineRunnersTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/CombineRunnersTest.java index 2f20ee4e1c0d6..b43f480977a63 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/CombineRunnersTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/CombineRunnersTest.java @@ -44,7 +44,7 @@ import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java index 4017fac1e34c4..3bc9ec38e3911 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnApiDoFnRunnerTest.java @@ -21,7 +21,7 @@ import static org.apache.beam.sdk.options.ExperimentalOptions.addExperiment; import static org.apache.beam.sdk.util.WindowedValue.timestampedValueInGlobalWindow; import static org.apache.beam.sdk.util.WindowedValue.valueInGlobalWindow; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.contains; @@ -144,9 +144,9 @@ import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.util.Durations; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.hamcrest.collection.IsMapContaining; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java index c02c288f09334..4ee781f69a335 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/FnHarnessTest.java @@ -47,7 +47,7 @@ import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Server; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ServerBuilder; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/MapFnRunnersTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/MapFnRunnersTest.java index 4d00c067b9784..4a9f05b65ab51 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/MapFnRunnersTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/MapFnRunnersTest.java @@ -38,7 +38,7 @@ import org.apache.beam.sdk.transforms.windowing.IntervalWindow; import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Test; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/PrecombineGroupingTableTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/PrecombineGroupingTableTest.java index d9fd19c651a92..64f160d183d25 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/PrecombineGroupingTableTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/PrecombineGroupingTableTest.java @@ -50,7 +50,7 @@ import org.apache.beam.sdk.transforms.Combine.CombineFn; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; import org.hamcrest.Description; import org.hamcrest.TypeSafeDiagnosingMatcher; import org.joda.time.Instant; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/WindowMergingFnRunnerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/WindowMergingFnRunnerTest.java index 359ea98546b95..189d83d87b9cd 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/WindowMergingFnRunnerTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/WindowMergingFnRunnerTest.java @@ -33,9 +33,9 @@ import org.apache.beam.sdk.transforms.windowing.Sessions; import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Test; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/BeamFnControlClientTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/BeamFnControlClientTest.java index 5157e124b3190..27799ce2a58af 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/BeamFnControlClientTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/BeamFnControlClientTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.fn.harness.control; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables.getStackTraceAsString; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables.getStackTraceAsString; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.not; @@ -48,7 +48,7 @@ import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessServerBuilder; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.CallStreamObserver; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestRule; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ExecutionStateSamplerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ExecutionStateSamplerTest.java index 9d79de0fa1530..47866adc892bd 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ExecutionStateSamplerTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ExecutionStateSamplerTest.java @@ -22,6 +22,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -648,4 +649,28 @@ public Long answer(InvocationOnMock invocation) throws Throwable { sampler.stop(); expectedLogs.verifyWarn("Operation ongoing in bundle bundleId for PTransform"); } + + @Test + public void testErrorState() throws Exception { + MillisProvider clock = mock(MillisProvider.class); + ExecutionStateSampler sampler = + new ExecutionStateSampler( + PipelineOptionsFactory.fromArgs("--experiments=state_sampling_period_millis=10") + .create(), + clock); + ExecutionStateTracker tracker = sampler.create(); + ExecutionState state1 = + tracker.create("shortId1", "ptransformId1", "ptransformIdName1", "process"); + ExecutionState state2 = + tracker.create("shortId2", "ptransformId2", "ptransformIdName2", "process"); + + state1.activate(); + state2.activate(); + assertTrue(state2.error()); + assertFalse(state2.error()); + state2.deactivate(); + assertFalse(state2.error()); + tracker.reset(); + assertTrue(state1.error()); + } } diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java index 0a306443a3d16..ae3adf5ee56fc 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/control/ProcessBundleHandlerTest.java @@ -19,7 +19,7 @@ import static java.util.Arrays.asList; import static org.apache.beam.fn.harness.control.ProcessBundleHandler.REGISTERED_RUNNER_FACTORIES; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -145,10 +145,10 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.joda.time.Instant; import org.junit.After; import org.junit.Before; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/PCollectionConsumerRegistryTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/PCollectionConsumerRegistryTest.java index 03c7fbbd13a36..f75b84e76ad5e 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/PCollectionConsumerRegistryTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/data/PCollectionConsumerRegistryTest.java @@ -33,17 +33,26 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import org.apache.beam.fn.harness.HandlesSplits; import org.apache.beam.fn.harness.control.BundleProgressReporter; import org.apache.beam.fn.harness.control.ExecutionStateSampler; import org.apache.beam.fn.harness.control.ExecutionStateSampler.ExecutionStateTracker; import org.apache.beam.fn.harness.debug.DataSampler; +import org.apache.beam.fn.harness.logging.BeamFnLoggingClient; +import org.apache.beam.fn.harness.logging.BeamFnLoggingMDC; import org.apache.beam.model.fnexecution.v1.BeamFnApi; import org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor; +import org.apache.beam.model.fnexecution.v1.BeamFnLoggingGrpc; +import org.apache.beam.model.pipeline.v1.Endpoints; import org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo; import org.apache.beam.model.pipeline.v1.RunnerApi.PCollection; import org.apache.beam.runners.core.construction.SdkComponents; @@ -56,6 +65,7 @@ import org.apache.beam.sdk.coders.IterableCoder; import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.fn.data.FnDataReceiver; +import org.apache.beam.sdk.fn.test.TestStreams; import org.apache.beam.sdk.metrics.Counter; import org.apache.beam.sdk.metrics.Metrics; import org.apache.beam.sdk.metrics.MetricsEnvironment; @@ -65,7 +75,13 @@ import org.apache.beam.sdk.util.common.ElementByteSizeObservableIterable; import org.apache.beam.sdk.util.common.ElementByteSizeObservableIterator; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ManagedChannel; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Server; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessChannelBuilder; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessServerBuilder; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.CallStreamObserver; +import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -567,6 +583,124 @@ public void dataSampling() throws Exception { assertTrue(elementList.getElementsList().containsAll(expectedSamples)); } + @Test + public void logsExceptionWithTransformId() throws Exception { + final String pTransformId = "pTransformId"; + final String message = "testException"; + final String instructionId = "instruction"; + final Exception thrownException = new Exception(message); + + // The following is a bunch of boiler-plate to set up a local FnApiLoggingService to catch any + // logs for later test + // expectations. + AtomicBoolean clientClosedStream = new AtomicBoolean(); + Collection values = new ConcurrentLinkedQueue<>(); + AtomicReference> outboundServerObserver = + new AtomicReference<>(); + CallStreamObserver inboundServerObserver = + TestStreams.withOnNext( + (BeamFnApi.LogEntry.List logEntries) -> + values.addAll(logEntries.getLogEntriesList())) + .withOnCompleted( + () -> { + // Remember that the client told us that this stream completed + clientClosedStream.set(true); + outboundServerObserver.get().onCompleted(); + }) + .build(); + + Endpoints.ApiServiceDescriptor apiServiceDescriptor = + Endpoints.ApiServiceDescriptor.newBuilder() + .setUrl(this.getClass().getName() + "-" + UUID.randomUUID().toString()) + .build(); + Server server = + InProcessServerBuilder.forName(apiServiceDescriptor.getUrl()) + .addService( + new BeamFnLoggingGrpc.BeamFnLoggingImplBase() { + @Override + public StreamObserver logging( + StreamObserver outboundObserver) { + outboundServerObserver.set(outboundObserver); + return inboundServerObserver; + } + }) + .build(); + server.start(); + ManagedChannel channel = InProcessChannelBuilder.forName(apiServiceDescriptor.getUrl()).build(); + // End logging boiler-plate... + + // This section is to set up the StateSampler with the expected metadata. + ExecutionStateSampler sampler = + new ExecutionStateSampler(PipelineOptionsFactory.create(), System::currentTimeMillis); + ExecutionStateSampler.ExecutionStateTracker stateTracker = sampler.create(); + stateTracker.start("process-bundle"); + ExecutionStateSampler.ExecutionState state = + stateTracker.create("shortId", pTransformId, pTransformId, "process"); + state.activate(); + + // Track the instruction and state in the logging system. In a real run, this is set when a + // ProcessBundlehandler + // starts processing. + BeamFnLoggingMDC.setInstructionId(instructionId); + BeamFnLoggingMDC.setStateTracker(stateTracker); + + // Start the test within the logging context. This reroutes logging through to the boiler-plate + // that was set up + // earlier. + try (BeamFnLoggingClient ignored = + BeamFnLoggingClient.createAndStart( + PipelineOptionsFactory.create(), + apiServiceDescriptor, + (Endpoints.ApiServiceDescriptor descriptor) -> channel)) { + + // Set up the component under test, the FnDataReceiver, to emit an exception when it starts. + ShortIdMap shortIds = new ShortIdMap(); + BundleProgressReporter.InMemory reporterAndRegistrar = new BundleProgressReporter.InMemory(); + PCollectionConsumerRegistry consumers = + new PCollectionConsumerRegistry( + stateTracker, shortIds, reporterAndRegistrar, TEST_DESCRIPTOR); + FnDataReceiver> consumer = mock(FnDataReceiver.class); + + consumers.register(P_COLLECTION_A, pTransformId, pTransformId + "Name", consumer); + + FnDataReceiver> wrapperConsumer = + (FnDataReceiver>) + (FnDataReceiver) consumers.getMultiplexingConsumer(P_COLLECTION_A); + + doThrow(thrownException).when(consumer).accept(any()); + expectedException.expectMessage(message); + expectedException.expect(Exception.class); + + // Run the test. + wrapperConsumer.accept(valueInGlobalWindow("elem")); + + } finally { + // The actual log entry has a lot of metadata that can't easily be controlled. So set the + // entries that are needed + // for this test and cull everything else. + final BeamFnApi.LogEntry expectedEntry = + BeamFnApi.LogEntry.newBuilder() + .setInstructionId(instructionId) + .setTransformId(pTransformId) + .setMessage("Failed to process element for bundle \"process-bundle\"") + .build(); + + List entries = new ArrayList<>(values); + assertEquals(1, entries.size()); + BeamFnApi.LogEntry actualEntry = entries.get(0); + BeamFnApi.LogEntry actualEntryCulled = + BeamFnApi.LogEntry.newBuilder() + .setInstructionId(actualEntry.getInstructionId()) + .setTransformId(actualEntry.getTransformId()) + .setMessage(actualEntry.getMessage()) + .build(); + + assertEquals(expectedEntry, actualEntryCulled); + + server.shutdownNow(); + } + } + private static class TestElementByteSizeObservableIterable extends ElementByteSizeObservableIterable> { private List elements; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/debug/DataSamplerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/debug/DataSamplerTest.java index c71ddff349ab5..1cad5210380b5 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/debug/DataSamplerTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/debug/DataSamplerTest.java @@ -21,6 +21,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.io.ByteArrayOutputStream; @@ -32,13 +33,16 @@ import java.util.Map; import java.util.concurrent.CountDownLatch; import org.apache.beam.model.fnexecution.v1.BeamFnApi; +import org.apache.beam.model.fnexecution.v1.BeamFnApi.SampledElement; import org.apache.beam.sdk.coders.ByteArrayCoder; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.coders.VarIntCoder; +import org.apache.beam.sdk.options.ExperimentalOptions; +import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -117,6 +121,21 @@ void assertHasSamples( assertTrue(elementList.getElementsList().containsAll(expectedSamples)); } + void assertHasSamples( + BeamFnApi.InstructionResponse response, + String pcollection, + List elements) { + Map elementSamplesMap = + response.getSampleData().getElementSamplesMap(); + + assertFalse(elementSamplesMap.isEmpty()); + + BeamFnApi.SampleDataResponse.ElementList elementList = elementSamplesMap.get(pcollection); + assertNotNull(elementList); + + assertTrue(elementList.getElementsList().containsAll(elements)); + } + /** * Smoke test that a samples show in the output map. * @@ -203,7 +222,7 @@ void generateStringSamples(DataSampler sampler) { */ @Test public void testFiltersSinglePCollectionId() throws Exception { - DataSampler sampler = new DataSampler(10, 10); + DataSampler sampler = new DataSampler(10, 10, false); generateStringSamples(sampler); BeamFnApi.InstructionResponse samples = getSamplesForPCollection(sampler, "a"); @@ -219,7 +238,7 @@ public void testFiltersSinglePCollectionId() throws Exception { public void testFiltersMultiplePCollectionIds() throws Exception { List pcollectionIds = ImmutableList.of("a", "c"); - DataSampler sampler = new DataSampler(10, 10); + DataSampler sampler = new DataSampler(10, 10, false); generateStringSamples(sampler); BeamFnApi.InstructionResponse samples = getSamplesForPCollections(sampler, pcollectionIds); @@ -275,4 +294,87 @@ public void testConcurrentNewSampler() throws Exception { sampleThread.join(); } } + + /** + * Tests that including the "enable_always_on_exception_sampling" can sample. + * + * @throws Exception + */ + @Test + public void testEnableAlwaysOnExceptionSampling() throws Exception { + ExperimentalOptions experimentalOptions = PipelineOptionsFactory.as(ExperimentalOptions.class); + experimentalOptions.setExperiments( + Collections.singletonList("enable_always_on_exception_sampling")); + DataSampler sampler = DataSampler.create(experimentalOptions); + assertNotNull(sampler); + + VarIntCoder coder = VarIntCoder.of(); + OutputSampler outputSampler = sampler.sampleOutput("pcollection-id", coder); + ElementSample elementSample = outputSampler.sample(globalWindowedValue(1)); + outputSampler.exception(elementSample, new RuntimeException(), "", ""); + + outputSampler.sample(globalWindowedValue(2)); + + BeamFnApi.InstructionResponse samples = getAllSamples(sampler); + List expectedSamples = + ImmutableList.of( + SampledElement.newBuilder() + .setElement(ByteString.copyFrom(encodeInt(1))) + .setException( + SampledElement.Exception.newBuilder() + .setError(new RuntimeException().toString())) + .build()); + assertHasSamples(samples, "pcollection-id", expectedSamples); + } + + /** + * Tests that "disable_always_on_exception_sampling" overrides the always on experiment. + * + * @throws Exception + */ + @Test + public void testDisableAlwaysOnExceptionSampling() throws Exception { + ExperimentalOptions experimentalOptions = PipelineOptionsFactory.as(ExperimentalOptions.class); + experimentalOptions.setExperiments( + ImmutableList.of( + "enable_always_on_exception_sampling", "disable_always_on_exception_sampling")); + DataSampler sampler = DataSampler.create(experimentalOptions); + assertNull(sampler); + } + + /** + * Tests that the "enable_data_sampling" experiment overrides + * "disable_always_on_exception_sampling". + * + * @throws Exception + */ + @Test + public void testDisableAlwaysOnExceptionSamplingWithEnableDataSampling() throws Exception { + ExperimentalOptions experimentalOptions = PipelineOptionsFactory.as(ExperimentalOptions.class); + experimentalOptions.setExperiments( + ImmutableList.of( + "enable_data_sampling", + "enable_always_on_exception_sampling", + "disable_always_on_exception_sampling")); + DataSampler sampler = DataSampler.create(experimentalOptions); + assertNotNull(sampler); + + VarIntCoder coder = VarIntCoder.of(); + OutputSampler outputSampler = sampler.sampleOutput("pcollection-id", coder); + ElementSample elementSample = outputSampler.sample(globalWindowedValue(1)); + outputSampler.exception(elementSample, new RuntimeException(), "", ""); + + outputSampler.sample(globalWindowedValue(2)); + + BeamFnApi.InstructionResponse samples = getAllSamples(sampler); + List expectedSamples = + ImmutableList.of( + SampledElement.newBuilder() + .setElement(ByteString.copyFrom(encodeInt(1))) + .setException( + SampledElement.Exception.newBuilder() + .setError(new RuntimeException().toString())) + .build()); + assertHasSamples(samples, "pcollection-id", expectedSamples); + } } diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/debug/OutputSamplerTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/debug/OutputSamplerTest.java index 7946251b30f9c..26285205bd34c 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/debug/OutputSamplerTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/debug/OutputSamplerTest.java @@ -27,12 +27,13 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; +import javax.annotation.Nullable; import org.apache.beam.model.fnexecution.v1.BeamFnApi; import org.apache.beam.sdk.coders.VarIntCoder; import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -59,6 +60,28 @@ public BeamFnApi.SampledElement encodeGlobalWindowedInt(Integer i) throws IOExce .build(); } + public BeamFnApi.SampledElement encodeException( + Integer i, String error, String ptransformId, @Nullable String processBundleId) + throws IOException { + VarIntCoder coder = VarIntCoder.of(); + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + coder.encode(i, stream); + + BeamFnApi.SampledElement.Exception.Builder builder = + BeamFnApi.SampledElement.Exception.newBuilder() + .setTransformId(ptransformId) + .setError(error); + + if (processBundleId != null) { + builder.setInstructionId(processBundleId); + } + + return BeamFnApi.SampledElement.newBuilder() + .setElement(ByteString.copyFrom(stream.toByteArray())) + .setException(builder) + .build(); + } + /** * Test that the first N are always sampled. * @@ -67,7 +90,7 @@ public BeamFnApi.SampledElement encodeGlobalWindowedInt(Integer i) throws IOExce @Test public void testSamplesFirstN() throws IOException { VarIntCoder coder = VarIntCoder.of(); - OutputSampler outputSampler = new OutputSampler<>(coder, 10, 10); + OutputSampler outputSampler = new OutputSampler<>(coder, 10, 10, false); // Purposely go over maxSamples and sampleEveryN. This helps to increase confidence. for (int i = 0; i < 15; ++i) { @@ -89,7 +112,7 @@ public void testWindowedValueSample() throws IOException { WindowedValue.WindowedValueCoder coder = WindowedValue.FullWindowedValueCoder.of(VarIntCoder.of(), GlobalWindow.Coder.INSTANCE); - OutputSampler outputSampler = new OutputSampler<>(coder, 10, 10); + OutputSampler outputSampler = new OutputSampler<>(coder, 10, 10, false); outputSampler.sample(WindowedValue.valueInGlobalWindow(0)); // The expected list is only 0..9 inclusive. @@ -102,7 +125,7 @@ public void testWindowedValueSample() throws IOException { public void testNonWindowedValueSample() throws IOException { VarIntCoder coder = VarIntCoder.of(); - OutputSampler outputSampler = new OutputSampler<>(coder, 10, 10); + OutputSampler outputSampler = new OutputSampler<>(coder, 10, 10, false); outputSampler.sample(WindowedValue.valueInGlobalWindow(0)); // The expected list is only 0..9 inclusive. @@ -119,7 +142,7 @@ public void testNonWindowedValueSample() throws IOException { @Test public void testActsLikeCircularBuffer() throws IOException { VarIntCoder coder = VarIntCoder.of(); - OutputSampler outputSampler = new OutputSampler<>(coder, 5, 20); + OutputSampler outputSampler = new OutputSampler<>(coder, 5, 20, false); for (int i = 0; i < 100; ++i) { outputSampler.sample(WindowedValue.valueInGlobalWindow(i)); @@ -148,17 +171,71 @@ public void testActsLikeCircularBuffer() throws IOException { @Test public void testCanSampleExceptions() throws IOException { VarIntCoder coder = VarIntCoder.of(); - OutputSampler outputSampler = new OutputSampler<>(coder, 5, 20); + OutputSampler outputSampler = new OutputSampler<>(coder, 5, 20, false); WindowedValue windowedValue = WindowedValue.valueInGlobalWindow(1); ElementSample elementSample = outputSampler.sample(windowedValue); Exception exception = new RuntimeException("Test exception"); - outputSampler.exception(elementSample, exception); + String ptransformId = "ptransform"; + String processBundleId = "processBundle"; + outputSampler.exception(elementSample, exception, ptransformId, processBundleId); + + List expected = new ArrayList<>(); + expected.add(encodeException(1, exception.toString(), ptransformId, processBundleId)); + + List samples = outputSampler.samples(); + assertThat(samples, containsInAnyOrder(expected.toArray())); + } + + /** + * Test that in the event that an exception happens multiple times in a bundle, it's only recorded + * at the source. + * + * @throws IOException when encoding fails (shouldn't happen). + */ + @Test + public void testNoDuplicateExceptions() throws IOException { + VarIntCoder coder = VarIntCoder.of(); + OutputSampler outputSampler = new OutputSampler<>(coder, 5, 20, false); + + ElementSample elementSampleA = + outputSampler.sample(WindowedValue.valueInGlobalWindow(1)); + ElementSample elementSampleB = + outputSampler.sample(WindowedValue.valueInGlobalWindow(2)); + + Exception exception = new RuntimeException("Test exception"); + String ptransformIdA = "ptransformA"; + String ptransformIdB = "ptransformB"; + String processBundleId = "processBundle"; + outputSampler.exception(elementSampleA, exception, ptransformIdA, processBundleId); + outputSampler.exception(elementSampleB, exception, ptransformIdB, processBundleId); + + List expected = new ArrayList<>(); + expected.add(encodeException(1, exception.toString(), ptransformIdA, processBundleId)); + expected.add(encodeInt(2)); + + List samples = outputSampler.samples(); + assertThat(samples, containsInAnyOrder(expected.toArray())); + } + + /** + * Test that exception metadata is only set if there is a process bundle. + * + * @throws IOException when encoding fails (shouldn't happen). + */ + @Test + public void testExceptionOnlySampledIfNonNullProcessBundle() throws IOException { + VarIntCoder coder = VarIntCoder.of(); + OutputSampler outputSampler = new OutputSampler<>(coder, 5, 20, false); + + WindowedValue windowedValue = WindowedValue.valueInGlobalWindow(1); + ElementSample elementSample = outputSampler.sample(windowedValue); + + Exception exception = new RuntimeException("Test exception"); + String ptransformId = "ptransform"; + outputSampler.exception(elementSample, exception, ptransformId, null); - // The first 10 are always sampled, but with maxSamples = 5, the first ten are downsampled to - // 4..9 inclusive. Then, - // the 20th element is sampled (19) and every 20 after. List expected = new ArrayList<>(); expected.add(encodeInt(1)); @@ -167,15 +244,14 @@ public void testCanSampleExceptions() throws IOException { } /** - * Tests that multiple samples don't push out exception samples. TODO: test that the exception - * metadata is set. + * Tests that multiple samples don't push out exception samples. * * @throws IOException when encoding fails (shouldn't happen). */ @Test public void testExceptionSamplesAreNotRemoved() throws IOException { VarIntCoder coder = VarIntCoder.of(); - OutputSampler outputSampler = new OutputSampler<>(coder, 5, 20); + OutputSampler outputSampler = new OutputSampler<>(coder, 5, 20, false); WindowedValue windowedValue = WindowedValue.valueInGlobalWindow(0); ElementSample elementSample = outputSampler.sample(windowedValue); @@ -185,7 +261,9 @@ public void testExceptionSamplesAreNotRemoved() throws IOException { } Exception exception = new RuntimeException("Test exception"); - outputSampler.exception(elementSample, exception); + String ptransformId = "ptransform"; + String processBundleId = "processBundle"; + outputSampler.exception(elementSample, exception, ptransformId, processBundleId); // The first 10 are always sampled, but with maxSamples = 5, the first ten are downsampled to // 4..9 inclusive. Then, the 20th element is sampled (19) and every 20 after. Finally, @@ -196,7 +274,33 @@ public void testExceptionSamplesAreNotRemoved() throws IOException { expected.add(encodeInt(59)); expected.add(encodeInt(79)); expected.add(encodeInt(99)); - expected.add(encodeInt(0)); + expected.add(encodeException(0, exception.toString(), ptransformId, processBundleId)); + + List samples = outputSampler.samples(); + assertThat(samples, containsInAnyOrder(expected.toArray())); + } + + /** + * Test that elements the onlySampleExceptions flag works. + * + * @throws IOException when encoding fails (shouldn't happen). + */ + @Test + public void testOnlySampleExceptions() throws IOException { + VarIntCoder coder = VarIntCoder.of(); + OutputSampler outputSampler = new OutputSampler<>(coder, 5, 20, true); + + WindowedValue windowedValue = WindowedValue.valueInGlobalWindow(1); + outputSampler.sample(WindowedValue.valueInGlobalWindow(2)); + ElementSample elementSample = outputSampler.sample(windowedValue); + + Exception exception = new RuntimeException("Test exception"); + String ptransformId = "ptransform"; + String processBundleId = "processBundle"; + outputSampler.exception(elementSample, exception, ptransformId, processBundleId); + + List expected = new ArrayList<>(); + expected.add(encodeException(1, exception.toString(), ptransformId, processBundleId)); List samples = outputSampler.samples(); assertThat(samples, containsInAnyOrder(expected.toArray())); @@ -210,7 +314,7 @@ public void testExceptionSamplesAreNotRemoved() throws IOException { @Test public void testConcurrentSamples() throws IOException, InterruptedException { VarIntCoder coder = VarIntCoder.of(); - OutputSampler outputSampler = new OutputSampler<>(coder, 10, 2); + OutputSampler outputSampler = new OutputSampler<>(coder, 10, 2, false); CountDownLatch startSignal = new CountDownLatch(1); CountDownLatch doneSignal = new CountDownLatch(2); @@ -228,7 +332,9 @@ public void testConcurrentSamples() throws IOException, InterruptedException { } for (int i = 0; i < 1000000; i++) { - outputSampler.sample(WindowedValue.valueInGlobalWindow(i)); + ElementSample sample = + outputSampler.sample(WindowedValue.valueInGlobalWindow(i)); + outputSampler.exception(sample, new RuntimeException(""), "ptransformId", "pbId"); } doneSignal.countDown(); @@ -245,7 +351,9 @@ public void testConcurrentSamples() throws IOException, InterruptedException { } for (int i = -1000000; i < 0; i++) { - outputSampler.sample(WindowedValue.valueInGlobalWindow(i)); + ElementSample sample = + outputSampler.sample(WindowedValue.valueInGlobalWindow(i)); + outputSampler.exception(sample, new RuntimeException(""), "ptransformId", "pbId"); } doneSignal.countDown(); diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClientTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClientTest.java index 1fd8e249dd05b..8c7a40f8db904 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClientTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/logging/BeamFnLoggingClientTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.fn.harness.logging; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables.getStackTraceAsString; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables.getStackTraceAsString; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.junit.Assert.assertEquals; @@ -27,9 +27,10 @@ import java.util.Collection; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.Phaser; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Handler; import java.util.logging.Level; @@ -37,6 +38,7 @@ import java.util.logging.LogRecord; import java.util.logging.Logger; import java.util.logging.SimpleFormatter; +import org.apache.beam.fn.harness.control.ExecutionStateSampler; import org.apache.beam.model.fnexecution.v1.BeamFnApi; import org.apache.beam.model.fnexecution.v1.BeamFnLoggingGrpc; import org.apache.beam.model.pipeline.v1.Endpoints; @@ -94,6 +96,7 @@ public class BeamFnLoggingClientTest { .setInstructionId("instruction-1") .setSeverity(BeamFnApi.LogEntry.Severity.Enum.DEBUG) .setMessage("Message") + .setTransformId("ptransformId") .setThread("12345") .setTimestamp(Timestamp.newBuilder().setSeconds(1234567).setNanos(890000000).build()) .setLogLocation("LoggerName") @@ -103,6 +106,7 @@ public class BeamFnLoggingClientTest { .setInstructionId("instruction-1") .setSeverity(BeamFnApi.LogEntry.Severity.Enum.DEBUG) .setMessage("testMdcValue:Message") + .setTransformId("ptransformId") .setCustomData( Struct.newBuilder() .putFields( @@ -116,6 +120,7 @@ public class BeamFnLoggingClientTest { .setInstructionId("instruction-1") .setSeverity(BeamFnApi.LogEntry.Severity.Enum.WARN) .setMessage("MessageWithException") + .setTransformId("errorPtransformId") .setTrace(getStackTraceAsString(TEST_RECORD_WITH_EXCEPTION.getThrown())) .setThread("12345") .setTimestamp(Timestamp.newBuilder().setSeconds(1234567).setNanos(890000000).build()) @@ -125,7 +130,16 @@ public class BeamFnLoggingClientTest { @Test public void testLogging() throws Exception { + ExecutionStateSampler sampler = + new ExecutionStateSampler(PipelineOptionsFactory.create(), null); + ExecutionStateSampler.ExecutionStateTracker stateTracker = sampler.create(); + ExecutionStateSampler.ExecutionState state = + stateTracker.create("shortId", "ptransformId", "ptransformIdName", "process"); + state.activate(); + BeamFnLoggingMDC.setInstructionId("instruction-1"); + BeamFnLoggingMDC.setStateTracker(stateTracker); + AtomicBoolean clientClosedStream = new AtomicBoolean(); Collection values = new ConcurrentLinkedQueue<>(); AtomicReference> outboundServerObserver = @@ -187,7 +201,14 @@ public StreamObserver logging( rootLogger.log(FILTERED_RECORD); // Should not be filtered because the default log level override for ConfiguredLogger is DEBUG configuredLogger.log(TEST_RECORD); + + // Simulate an exception. This sets an internal error state where the PTransform should come + // from. + ExecutionStateSampler.ExecutionState errorState = + stateTracker.create("shortId", "errorPtransformId", "errorPtransformIdName", "process"); + errorState.activate(); configuredLogger.log(TEST_RECORD_WITH_EXCEPTION); + errorState.deactivate(); // Ensure that configuring a custom formatter on the logging handler will be honored. for (Handler handler : rootLogger.getHandlers()) { @@ -234,7 +255,7 @@ public void testWhenServerFailsThatClientIsAbleToCleanup() throws Exception { // removes the only reference and the logger may get GC'd before the assertions (BEAM-4136). Logger rootLogger = null; Logger configuredLogger = null; - Phaser streamBlocker = new Phaser(1); + CompletableFuture streamBlocker = new CompletableFuture(); Endpoints.ApiServiceDescriptor apiServiceDescriptor = Endpoints.ApiServiceDescriptor.newBuilder() @@ -249,7 +270,7 @@ public StreamObserver logging( StreamObserver outboundObserver) { // Block before returning an error on the stream so that we can observe the // loggers before they are reset. - streamBlocker.awaitAdvance(1); + streamBlocker.join(); outboundServerObserver.set(outboundObserver); outboundObserver.onError( Status.INTERNAL.withDescription("TEST ERROR").asException()); @@ -275,7 +296,7 @@ public StreamObserver logging( rootLogger = LogManager.getLogManager().getLogger(""); configuredLogger = LogManager.getLogManager().getLogger("ConfiguredLogger"); // Allow the stream to return with an error. - assertEquals(0, streamBlocker.arrive()); + streamBlocker.complete(new Object()); thrown.expectMessage("TEST ERROR"); client.close(); } finally { @@ -356,15 +377,22 @@ public StreamObserver logging( @Test public void testClosableWhenBlockingForOnReady() throws Exception { BeamFnLoggingMDC.setInstructionId("instruction-1"); - Collection values = new ConcurrentLinkedQueue<>(); + AtomicInteger testEntriesObserved = new AtomicInteger(); + AtomicBoolean onReadyBlocking = new AtomicBoolean(); AtomicReference> outboundServerObserver = new AtomicReference<>(); final AtomicBoolean elementsAllowed = new AtomicBoolean(true); CallStreamObserver inboundServerObserver = TestStreams.withOnNext( - (BeamFnApi.LogEntry.List logEntries) -> - values.addAll(logEntries.getLogEntriesList())) + (BeamFnApi.LogEntry.List logEntries) -> { + for (BeamFnApi.LogEntry entry : logEntries.getLogEntriesList()) { + if (entry.toBuilder().clearCustomData().build().equals(TEST_ENTRY)) { + testEntriesObserved.addAndGet(1); + } + } + }) + .withOnCompleted(() -> outboundServerObserver.get().onCompleted()) .build(); Endpoints.ApiServiceDescriptor apiServiceDescriptor = @@ -397,6 +425,10 @@ public ClientCall interceptCall( delegate) { @Override public boolean isReady() { + if (elementsAllowed.get()) { + return true; + } + onReadyBlocking.set(true); return elementsAllowed.get(); } }; @@ -430,20 +462,28 @@ public boolean isReady() { } // Measure how long it takes all the logs to appear. int sleepTime = 0; - while (values.size() < numEntries) { + while (testEntriesObserved.get() < numEntries) { ++sleepTime; Thread.sleep(1); } // Attempt to enter the blocking state by pushing back on the stream, publishing records and // then giving them time for it to block. elementsAllowed.set(false); - for (int i = 0; i < numEntries; ++i) { + int postAllowedLogs = 0; + while (!onReadyBlocking.get()) { + ++postAllowedLogs; configuredLogger.log(TEST_RECORD); + Thread.sleep(1); } + + // Even with sleeping to give some additional time for the logs that were sent by the client + // to be observed by the server we should not observe all the client logs, indicating we're + // blocking as intended. Thread.sleep(sleepTime * 3); - // At this point, the background thread is either blocking as intended or the background - // thread hasn't yet observed all the input. In either case the test should pass. - assertTrue(values.size() < numEntries * 2); + assertTrue(testEntriesObserved.get() < numEntries + postAllowedLogs); + + // Allow entries to drain to speed up close. + elementsAllowed.set(true); client.close(); @@ -463,13 +503,11 @@ public boolean isReady() { @Test public void testServerCloseNotifiesTermination() throws Exception { BeamFnLoggingMDC.setInstructionId("instruction-1"); - Collection values = new ConcurrentLinkedQueue<>(); AtomicReference> outboundServerObserver = new AtomicReference<>(); CallStreamObserver inboundServerObserver = - TestStreams.withOnNext( - (BeamFnApi.LogEntry.List logEntries) -> - values.addAll(logEntries.getLogEntriesList())) + TestStreams.withOnNext((BeamFnApi.LogEntry.List logEntries) -> {}) + .withOnCompleted(() -> outboundServerObserver.get().onCompleted()) .build(); Endpoints.ApiServiceDescriptor apiServiceDescriptor = diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BagUserStateTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BagUserStateTest.java index 2ddece320fd1b..b4d6f5ae40e24 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BagUserStateTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BagUserStateTest.java @@ -32,9 +32,9 @@ import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.util.ByteStringOutputStream; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -110,6 +110,7 @@ public void testAppend() throws Exception { assertThrows(IllegalStateException.class, () -> userState.append("A4")); } + @SuppressWarnings("InlineMeInliner") @Test public void testAppendBatchingLimit() throws Exception { String a1 = "A1"; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCacheTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCacheTest.java index 85857ae84eb5f..dc48a7a3e5bd0 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCacheTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/BeamFnStateGrpcClientCacheTest.java @@ -48,7 +48,7 @@ import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessServerBuilder; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.CallStreamObserver; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.junit.After; import org.junit.Before; import org.junit.Rule; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/FakeBeamFnStateClient.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/FakeBeamFnStateClient.java index 2e148515a62f1..4aaaa3d945d9d 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/FakeBeamFnStateClient.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/FakeBeamFnStateClient.java @@ -40,7 +40,7 @@ import org.apache.beam.sdk.util.ByteStringOutputStream; import org.apache.beam.sdk.values.KV; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; /** A fake implementation of a {@link BeamFnStateClient} to aid with testing. */ public class FakeBeamFnStateClient implements BeamFnStateClient { diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/IterableSideInputTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/IterableSideInputTest.java index 02e5d596b6c47..2c1f73fcaf0d6 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/IterableSideInputTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/IterableSideInputTest.java @@ -26,8 +26,8 @@ import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateKey; import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/LazyCachingIteratorToIterableTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/LazyCachingIteratorToIterableTest.java index 0914b017e703b..fd701099a778b 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/LazyCachingIteratorToIterableTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/LazyCachingIteratorToIterableTest.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.fn.stream.PrefetchableIterator; import org.apache.beam.sdk.fn.stream.PrefetchableIterators; import org.apache.beam.sdk.fn.stream.PrefetchableIteratorsTest.ReadyAfterPrefetch; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/MultimapSideInputTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/MultimapSideInputTest.java index 77dead8fc7057..a4c20e3593ace 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/MultimapSideInputTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/MultimapSideInputTest.java @@ -31,8 +31,8 @@ import org.apache.beam.sdk.util.ByteStringOutputStream; import org.apache.beam.sdk.values.KV; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/MultimapUserStateTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/MultimapUserStateTest.java index 388e69b58e74d..d43056750ad9a 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/MultimapUserStateTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/MultimapUserStateTest.java @@ -44,8 +44,8 @@ import org.apache.beam.sdk.util.ByteStringOutputStream; import org.apache.beam.sdk.values.KV; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateBackedIterableTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateBackedIterableTest.java index f8f7d29b2d0b3..4d53bcaef117e 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateBackedIterableTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateBackedIterableTest.java @@ -37,10 +37,10 @@ import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.util.ByteStringOutputStream; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Test; import org.junit.experimental.runners.Enclosed; import org.junit.runner.RunWith; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateFetchingIteratorsTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateFetchingIteratorsTest.java index 291d36f91ed8c..bb70b585d4c3e 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateFetchingIteratorsTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/StateFetchingIteratorsTest.java @@ -49,8 +49,8 @@ import org.apache.beam.sdk.coders.BigEndianIntegerCoder; import org.apache.beam.sdk.fn.stream.PrefetchableIterator; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Ints; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Ints; import org.junit.Test; import org.junit.experimental.runners.Enclosed; import org.junit.runner.RunWith; diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/status/BeamFnStatusClientTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/status/BeamFnStatusClientTest.java index f4265b1583485..19cd568f64290 100644 --- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/status/BeamFnStatusClientTest.java +++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/status/BeamFnStatusClientTest.java @@ -50,7 +50,7 @@ import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.Server; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessServerBuilder; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -75,7 +75,7 @@ public void testActiveBundleState() { when(executionStateTracker.getStatus()) .thenReturn( ExecutionStateTrackerStatus.create( - "ptransformId", "ptransformIdName", Thread.currentThread(), i * 1000)); + "ptransformId", "ptransformIdName", Thread.currentThread(), i * 1000, null)); String instruction = Integer.toString(i); when(processorCache.find(instruction)).thenReturn(processor); bundleProcessorMap.put(instruction, processor); diff --git a/sdks/java/io/OWNERS b/sdks/java/io/OWNERS deleted file mode 100644 index a9ed5d3327f0a..0000000000000 --- a/sdks/java/io/OWNERS +++ /dev/null @@ -1,7 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - jbonofre - - lukecwik - - chamikaramj - - timrobertson100 diff --git a/sdks/java/io/amazon-web-services/OWNERS b/sdks/java/io/amazon-web-services/OWNERS deleted file mode 100644 index 377d5939afc6c..0000000000000 --- a/sdks/java/io/amazon-web-services/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - aromanenko-dev - - mosche diff --git a/sdks/java/io/amazon-web-services/build.gradle b/sdks/java/io/amazon-web-services/build.gradle index 9cc453572e1ad..35aabe1507a62 100644 --- a/sdks/java/io/amazon-web-services/build.gradle +++ b/sdks/java/io/amazon-web-services/build.gradle @@ -33,7 +33,7 @@ description = "Apache Beam :: SDKs :: Java :: IO :: Amazon Web Services" ext.summary = "IO library to read and write Amazon Web Services services from Beam." dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.aws_java_sdk_cloudwatch implementation library.java.aws_java_sdk_core diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/coders/AwsCoders.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/coders/AwsCoders.java index 22f5f9e7cc938..501bfc0158607 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/coders/AwsCoders.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/coders/AwsCoders.java @@ -33,7 +33,7 @@ import org.apache.beam.sdk.coders.NullableCoder; import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.coders.VarIntCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** {@link Coder}s for common AWS SDK objects. */ @SuppressWarnings({ diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/dynamodb/AttributeValueCoderProviderRegistrar.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/dynamodb/AttributeValueCoderProviderRegistrar.java index ff3f633696af7..5a187e734d661 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/dynamodb/AttributeValueCoderProviderRegistrar.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/dynamodb/AttributeValueCoderProviderRegistrar.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.coders.CoderProviderRegistrar; import org.apache.beam.sdk.coders.CoderProviders; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** A {@link CoderProviderRegistrar} for standard types used with {@link DynamoDBIO}. */ @AutoService(CoderProviderRegistrar.class) diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/dynamodb/BasicDynamoDBProvider.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/dynamodb/BasicDynamoDBProvider.java index b868b9e43cef1..b4ee1be74abe2 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/dynamodb/BasicDynamoDBProvider.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/dynamodb/BasicDynamoDBProvider.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.aws.dynamodb; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider; diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIO.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIO.java index cc4e9c63a1b7a..e2c04c58b45dd 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIO.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIO.java @@ -20,7 +20,7 @@ import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.mapping; import static java.util.stream.Collectors.toList; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.amazonaws.regions.Regions; import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; @@ -62,8 +62,8 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.apache.http.HttpStatus; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsModule.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsModule.java index 4155268cd42d4..326758f1d1bb8 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsModule.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsModule.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.aws.options; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentials; @@ -53,7 +53,7 @@ import java.io.IOException; import java.util.Map; import org.apache.beam.repackaged.core.org.apache.commons.lang3.reflect.FieldUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; /** * A Jackson {@link Module} that registers a {@link JsonSerializer} and {@link JsonDeserializer} for diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsPipelineOptionsRegistrar.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsPipelineOptionsRegistrar.java index 3cee2196dc20d..3dad9fd611cb0 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsPipelineOptionsRegistrar.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsPipelineOptionsRegistrar.java @@ -20,7 +20,7 @@ import com.google.auto.service.AutoService; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** A registrar containing the default AWS options. */ @AutoService(PipelineOptionsRegistrar.class) diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/DefaultS3ClientBuilderFactory.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/DefaultS3ClientBuilderFactory.java index a00cfc5fea019..fa96d79b63a72 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/DefaultS3ClientBuilderFactory.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/DefaultS3ClientBuilderFactory.java @@ -21,7 +21,7 @@ import com.amazonaws.services.s3.AmazonS3ClientBuilder; import org.apache.beam.sdk.io.aws.options.S3ClientBuilderFactory; import org.apache.beam.sdk.io.aws.options.S3Options; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; /** * Construct AmazonS3ClientBuilder with default values of S3 client properties like path style diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/DefaultS3FileSystemSchemeRegistrar.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/DefaultS3FileSystemSchemeRegistrar.java index fc34ef30533e1..0988309cb0e2e 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/DefaultS3FileSystemSchemeRegistrar.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/DefaultS3FileSystemSchemeRegistrar.java @@ -17,13 +17,13 @@ */ package org.apache.beam.sdk.io.aws.s3; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.service.AutoService; import javax.annotation.Nonnull; import org.apache.beam.sdk.io.aws.options.S3Options; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Registers the "s3" uri schema to be handled by {@link S3FileSystem}. */ @AutoService(S3FileSystemSchemeRegistrar.class) diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java index 63eb83f30e4cf..fa442374e3cbd 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java @@ -18,9 +18,9 @@ package org.apache.beam.sdk.io.aws.s3; import static org.apache.beam.sdk.io.FileSystemUtils.wildcardToRegexp; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.amazonaws.AmazonClientException; import com.amazonaws.services.s3.AmazonS3; @@ -66,18 +66,18 @@ import org.apache.beam.sdk.io.fs.MatchResult; import org.apache.beam.sdk.io.fs.MoveOptions; import org.apache.beam.sdk.util.MoreFutures; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Suppliers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Suppliers; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystemRegistrar.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystemRegistrar.java index 47cb49a469509..af153de426227 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystemRegistrar.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystemRegistrar.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.aws.s3; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.service.AutoService; import java.util.Map; @@ -28,7 +28,7 @@ import org.apache.beam.sdk.io.FileSystemRegistrar; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Streams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Streams; /** * {@link AutoService} registrar for the {@link S3FileSystem}. diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3ReadableSeekableByteChannel.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3ReadableSeekableByteChannel.java index 80c4f8d0367b0..bef1fc340888e 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3ReadableSeekableByteChannel.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3ReadableSeekableByteChannel.java @@ -18,8 +18,8 @@ package org.apache.beam.sdk.io.aws.s3; import static com.amazonaws.util.IOUtils.drainInputStream; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.amazonaws.AmazonClientException; import com.amazonaws.services.s3.AmazonS3; diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3ResourceId.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3ResourceId.java index 646242ee8a4ad..2751f98d7f6ed 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3ResourceId.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3ResourceId.java @@ -17,9 +17,9 @@ */ package org.apache.beam.sdk.io.aws.s3; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.ObjectStreamException; import java.util.Date; @@ -29,8 +29,8 @@ import org.apache.beam.sdk.io.fs.ResolveOptions; import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions; import org.apache.beam.sdk.io.fs.ResourceId; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.checkerframework.checker.nullness.qual.Nullable; @SuppressWarnings({ diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3WritableByteChannel.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3WritableByteChannel.java index a3beaa48832b6..3594ca5b0aaa8 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3WritableByteChannel.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3WritableByteChannel.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.aws.s3; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.amazonaws.AmazonClientException; import com.amazonaws.services.s3.AmazonS3; @@ -39,7 +39,7 @@ import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** A writable S3 object, as a {@link WritableByteChannel}. */ @SuppressWarnings({ diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sns/BasicSnsProvider.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sns/BasicSnsProvider.java index 10599ddcadb94..aba3a74ccb2a8 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sns/BasicSnsProvider.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sns/BasicSnsProvider.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.aws.sns; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider; diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sns/SnsCoderProviderRegistrar.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sns/SnsCoderProviderRegistrar.java index 9893a027c03a7..315435861419b 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sns/SnsCoderProviderRegistrar.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sns/SnsCoderProviderRegistrar.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.coders.CoderProviderRegistrar; import org.apache.beam.sdk.coders.CoderProviders; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** A {@link CoderProviderRegistrar} for standard types used with {@link SnsIO}. */ @AutoService(CoderProviderRegistrar.class) diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sns/SnsIO.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sns/SnsIO.java index e3b8493f2a93c..291026f82f7ea 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sns/SnsIO.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sns/SnsIO.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.aws.sns; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.amazonaws.regions.Regions; import com.amazonaws.services.sns.AmazonSNS; @@ -43,8 +43,8 @@ import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.apache.http.HttpStatus; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sqs/SqsCheckpointMark.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sqs/SqsCheckpointMark.java index 9e6a01869c37e..b3e23bff5554e 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sqs/SqsCheckpointMark.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sqs/SqsCheckpointMark.java @@ -17,15 +17,15 @@ */ package org.apache.beam.sdk.io.aws.sqs; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.io.Serializable; import java.util.List; import org.apache.beam.sdk.io.UnboundedSource; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; @SuppressWarnings({ diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sqs/SqsIO.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sqs/SqsIO.java index 24e476d5cd480..26ca03c95c33d 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sqs/SqsIO.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sqs/SqsIO.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.aws.sqs; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.amazonaws.services.sqs.AmazonSQS; import com.amazonaws.services.sqs.AmazonSQSClientBuilder; diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sqs/SqsUnboundedReader.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sqs/SqsUnboundedReader.java index ab6e1ec1fd677..1fd5e38f54643 100644 --- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sqs/SqsUnboundedReader.java +++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sqs/SqsUnboundedReader.java @@ -20,7 +20,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toMap; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.amazonaws.services.sqs.AmazonSQS; import com.amazonaws.services.sqs.AmazonSQSClientBuilder; @@ -63,10 +63,10 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.BucketingFunction; import org.apache.beam.sdk.util.MovingFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.EvictingQueue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.EvictingQueue; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.Instant; import org.slf4j.Logger; diff --git a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/ITEnvironment.java b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/ITEnvironment.java index e05e7244db6d5..ee882a71a5cd7 100644 --- a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/ITEnvironment.java +++ b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/ITEnvironment.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.testing.TestPipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.commons.lang3.StringUtils; import org.junit.rules.ExternalResource; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/coders/AwsCodersTest.java b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/coders/AwsCodersTest.java index 46ce02de79932..1ee20a6fa7ea8 100644 --- a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/coders/AwsCodersTest.java +++ b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/coders/AwsCodersTest.java @@ -25,7 +25,7 @@ import com.amazonaws.http.SdkHttpMetadata; import java.util.UUID; import org.apache.beam.sdk.util.CoderUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; /** Tests for AWS coders. */ diff --git a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/dynamodb/AttributeValueCoderTest.java b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/dynamodb/AttributeValueCoderTest.java index ade41c4736eb1..489feb7a87c9d 100644 --- a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/dynamodb/AttributeValueCoderTest.java +++ b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/dynamodb/AttributeValueCoderTest.java @@ -27,7 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Assert; import org.junit.Test; diff --git a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIOIT.java b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIOIT.java index 860e04236216a..e3aa62450ce5d 100644 --- a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIOIT.java +++ b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIOIT.java @@ -53,7 +53,7 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIOReadTest.java b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIOReadTest.java index bf057b96c7856..27e2a84076b7a 100644 --- a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIOReadTest.java +++ b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIOReadTest.java @@ -20,9 +20,9 @@ import static java.lang.Math.min; import static java.util.stream.Collectors.toList; import static java.util.stream.IntStream.range; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.getLast; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.transform; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.getLast; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.transform; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -41,7 +41,7 @@ import org.apache.beam.sdk.transforms.Count; import org.apache.beam.sdk.transforms.Flatten; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIOWriteTest.java b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIOWriteTest.java index 92f31977d8b2a..e49276ed4c407 100644 --- a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIOWriteTest.java +++ b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIOWriteTest.java @@ -21,8 +21,8 @@ import static java.util.stream.IntStream.range; import static java.util.stream.IntStream.rangeClosed; import static org.apache.beam.sdk.io.aws.dynamodb.DynamoDBIO.Write.WriteFn.RETRY_ERROR_LOG; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps.filterKeys; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps.transformValues; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps.filterKeys; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps.transformValues; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; @@ -64,8 +64,8 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Duration; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/s3/MatchResultMatcher.java b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/s3/MatchResultMatcher.java index 2766a64b8be07..e6b127947df09 100644 --- a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/s3/MatchResultMatcher.java +++ b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/s3/MatchResultMatcher.java @@ -17,14 +17,14 @@ */ package org.apache.beam.sdk.io.aws.s3; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.util.List; import org.apache.beam.sdk.io.fs.MatchResult; import org.apache.beam.sdk.io.fs.ResourceId; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.hamcrest.Matcher; diff --git a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/s3/S3FileSystemTest.java b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/s3/S3FileSystemTest.java index 1aaac2fcee998..fbef40f4b5c04 100644 --- a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/s3/S3FileSystemTest.java +++ b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/s3/S3FileSystemTest.java @@ -25,7 +25,7 @@ import static org.apache.beam.sdk.io.aws.s3.S3TestUtils.s3OptionsWithCustomEndpointAndPathStyleAccessEnabled; import static org.apache.beam.sdk.io.aws.s3.S3TestUtils.s3OptionsWithSSECustomerKey; import static org.apache.beam.sdk.io.aws.s3.S3TestUtils.toMd5; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.junit.Assert.assertArrayEquals; @@ -74,7 +74,7 @@ import org.apache.beam.sdk.io.aws.options.S3Options; import org.apache.beam.sdk.io.fs.CreateOptions; import org.apache.beam.sdk.io.fs.MatchResult; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; diff --git a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/sns/PublishResultCodersTest.java b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/sns/PublishResultCodersTest.java index 8e172a61857b7..e8f8643cbbd4e 100644 --- a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/sns/PublishResultCodersTest.java +++ b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/sns/PublishResultCodersTest.java @@ -27,7 +27,7 @@ import java.util.UUID; import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.util.CoderUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; /** Tests for PublishResult coders. */ diff --git a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/sns/SnsIOTest.java b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/sns/SnsIOTest.java index 51c56c29effa9..f86c0851a01cd 100644 --- a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/sns/SnsIOTest.java +++ b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/sns/SnsIOTest.java @@ -46,7 +46,7 @@ import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/sqs/SqsMessageCoderTest.java b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/sqs/SqsMessageCoderTest.java index 3790dcb218284..933028306d8ba 100644 --- a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/sqs/SqsMessageCoderTest.java +++ b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/sqs/SqsMessageCoderTest.java @@ -25,7 +25,7 @@ import com.amazonaws.services.sqs.model.MessageAttributeValue; import java.util.Random; import org.apache.beam.sdk.util.CoderUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; public class SqsMessageCoderTest { diff --git a/sdks/java/io/amazon-web-services2/OWNERS b/sdks/java/io/amazon-web-services2/OWNERS deleted file mode 100644 index 377d5939afc6c..0000000000000 --- a/sdks/java/io/amazon-web-services2/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - aromanenko-dev - - mosche diff --git a/sdks/java/io/amazon-web-services2/build.gradle b/sdks/java/io/amazon-web-services2/build.gradle index cfd98fcb9877e..b681108dc6184 100644 --- a/sdks/java/io/amazon-web-services2/build.gradle +++ b/sdks/java/io/amazon-web-services2/build.gradle @@ -33,7 +33,7 @@ def excludeNetty = { } dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.error_prone_annotations implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.aws_java_sdk2_cloudwatch, excludeNetty diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/AsyncBatchWriteHandler.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/AsyncBatchWriteHandler.java new file mode 100644 index 0000000000000..95a65fe116586 --- /dev/null +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/AsyncBatchWriteHandler.java @@ -0,0 +1,403 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.aws2.common; + +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.counting; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.joining; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates.notNull; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Function; +import javax.annotation.Nullable; +import javax.annotation.concurrent.NotThreadSafe; +import org.apache.beam.sdk.annotations.Internal; +import org.apache.beam.sdk.util.BackOff; +import org.apache.beam.sdk.util.BackOffUtils; +import org.apache.beam.sdk.util.FluentBackoff; +import org.apache.beam.sdk.util.Sleeper; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.joda.time.DateTimeUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Async handler that automatically retries unprocessed records in case of a partial success. + * + *

    The handler enforces the provided upper limit of concurrent requests. Once that limit is + * reached any further call to {@link #batchWrite(String, List)} will block until another request + * completed. + * + *

    The handler is fail fast and won't submit any further request after a failure. Async failures + * can be polled using {@link #checkForAsyncFailure()}. + * + * @param Record type in batch + * @param Potentially erroneous result that needs to be correlated to a record using {@link + * #failedRecords(List, List)} + */ +@NotThreadSafe +@Internal +public abstract class AsyncBatchWriteHandler { + private static final Logger LOG = LoggerFactory.getLogger(AsyncBatchWriteHandler.class); + private final FluentBackoff backoff; + private final int concurrentRequests; + private final Stats stats; + protected final BiFunction, CompletableFuture>> submitFn; + protected final Function errorCodeFn; + + private AtomicBoolean hasErrored; + private AtomicReference asyncFailure; + private Semaphore requestPermits; + + protected AsyncBatchWriteHandler( + int concurrency, + FluentBackoff backoff, + Stats stats, + Function errorCodeFn, + BiFunction, CompletableFuture>> submitFn) { + this.backoff = backoff; + this.concurrentRequests = concurrency; + this.errorCodeFn = errorCodeFn; + this.submitFn = submitFn; + this.hasErrored = new AtomicBoolean(false); + this.asyncFailure = new AtomicReference<>(); + this.requestPermits = new Semaphore(concurrentRequests); + this.stats = stats; + } + + public final int requestsInProgress() { + return concurrentRequests - requestPermits.availablePermits(); + } + + public final void reset() { + hasErrored = new AtomicBoolean(false); + asyncFailure = new AtomicReference<>(); + requestPermits = new Semaphore(concurrentRequests); + } + + /** If this handler has errored since it was last reset. */ + public final boolean hasErrored() { + return hasErrored.get(); + } + + /** + * Check if any failure happened async. + * + * @throws Throwable The last async failure, afterwards reset it. + */ + public final void checkForAsyncFailure() throws Throwable { + @SuppressWarnings("nullness") + Throwable failure = asyncFailure.getAndSet(null); + if (failure != null) { + throw failure; + } + } + + /** + * Wait for all pending requests to complete and check for failures. + * + * @throws Throwable The last async failure if present using {@link #checkForAsyncFailure()} + */ + public final void waitForCompletion() throws Throwable { + requestPermits.acquireUninterruptibly(concurrentRequests); + checkForAsyncFailure(); + } + + /** + * Asynchronously trigger a batch write request (unless already in error state). + * + *

    This will respect the concurrency limit of the handler and first wait for a permit. + * + * @throws Throwable The last async failure if present using {@link #checkForAsyncFailure()} + */ + public final void batchWrite(String destination, List records) throws Throwable { + batchWrite(destination, records, true); + } + + /** + * Asynchronously trigger a batch write request (unless already in error state). + * + *

    This will respect the concurrency limit of the handler and first wait for a permit. + * + * @param throwAsyncFailures If to check and throw pending async failures + * @throws Throwable The last async failure if present using {@link #checkForAsyncFailure()} + */ + public final void batchWrite(String destination, List records, boolean throwAsyncFailures) + throws Throwable { + if (!hasErrored()) { + requestPermits.acquireUninterruptibly(); + new RetryHandler(destination, records).run(); + } + if (throwAsyncFailures) { + checkForAsyncFailure(); + } + } + + protected abstract List failedRecords(List records, List results); + + protected abstract boolean hasFailedRecords(List results); + + /** Statistics on the batch request. */ + public interface Stats { + Stats NONE = new Stats() {}; + + default void addBatchWriteRequest(long latencyMillis, boolean isPartialRetry) {} + } + + /** + * AsyncBatchWriteHandler that correlates records and results by position in the respective list. + */ + public static AsyncBatchWriteHandler byPosition( + int concurrency, + int partialRetries, + @Nullable RetryConfiguration retry, + Stats stats, + BiFunction, CompletableFuture>> submitFn, + Function errorCodeFn) { + FluentBackoff backoff = retryBackoff(partialRetries, retry); + return byPosition(concurrency, backoff, stats, submitFn, errorCodeFn); + } + + /** + * AsyncBatchWriteHandler that correlates records and results by position in the respective list. + */ + public static AsyncBatchWriteHandler byPosition( + int concurrency, + FluentBackoff backoff, + Stats stats, + BiFunction, CompletableFuture>> submitFn, + Function errorCodeFn) { + return new AsyncBatchWriteHandler( + concurrency, backoff, stats, errorCodeFn, submitFn) { + + @Override + protected boolean hasFailedRecords(List results) { + for (int i = 0; i < results.size(); i++) { + if (errorCodeFn.apply(results.get(i)) != null) { + return true; + } + } + return false; + } + + @Override + protected List failedRecords(List records, List results) { + int size = Math.min(records.size(), results.size()); + List filtered = new ArrayList<>(); + for (int i = 0; i < size; i++) { + if (errorCodeFn.apply(results.get(i)) != null) { + filtered.add(records.get(i)); + } + } + return filtered; + } + }; + } + + /** + * AsyncBatchWriteHandler that correlates records and results by id, all results are erroneous. + */ + public static AsyncBatchWriteHandler byId( + int concurrency, + int partialRetries, + @Nullable RetryConfiguration retry, + Stats stats, + BiFunction, CompletableFuture>> submitFn, + Function errorCodeFn, + Function recordIdFn, + Function errorIdFn) { + FluentBackoff backoff = retryBackoff(partialRetries, retry); + return byId(concurrency, backoff, stats, submitFn, errorCodeFn, recordIdFn, errorIdFn); + } + + /** + * AsyncBatchWriteHandler that correlates records and results by id, all results are erroneous. + */ + public static AsyncBatchWriteHandler byId( + int concurrency, + FluentBackoff backoff, + Stats stats, + BiFunction, CompletableFuture>> submitFn, + Function errorCodeFn, + Function recordIdFn, + Function errorIdFn) { + return new AsyncBatchWriteHandler( + concurrency, backoff, stats, errorCodeFn, submitFn) { + @Override + protected boolean hasFailedRecords(List errors) { + return !errors.isEmpty(); + } + + @Override + protected List failedRecords(List records, List errors) { + Set ids = Sets.newHashSetWithExpectedSize(errors.size()); + errors.forEach(e -> ids.add(errorIdFn.apply(e))); + + List filtered = new ArrayList<>(errors.size()); + for (int i = 0; i < records.size(); i++) { + RecT rec = records.get(i); + if (ids.contains(recordIdFn.apply(rec))) { + filtered.add(rec); + if (filtered.size() == errors.size()) { + return filtered; + } + } + } + return filtered; + } + }; + } + + /** + * This handler coordinates retries in case of a partial success. + * + *

      + *
    • Release permit if all (remaining) records are successful to allow for a new request to + * start. + *
    • Attempt retry in case of partial success for all erroneous records using backoff. Set + * async failure once retries are exceeded. + *
    • Set async failure if the entire request fails. Retries, if configured & applicable, have + * already been attempted by the AWS SDK in that case. + *
    + * + * The next call of {@link #checkForAsyncFailure()}, {@link #batchWrite(String, List< RecT >)}} or + * {@link #waitForCompletion()} will check for the last async failure and throw it. Afterwards the + * failure state is reset. + */ + private class RetryHandler implements BiConsumer, Throwable> { + private final String destination; + private final int totalRecords; + private final BackOff backoff; // backoff in case of throttling + + private final long handlerStartTime; + private long requestStartTime; + private int requests; + + private List records; + + RetryHandler(String destination, List records) { + this.destination = destination; + this.totalRecords = records.size(); + this.records = records; + this.backoff = AsyncBatchWriteHandler.this.backoff.backoff(); + this.handlerStartTime = DateTimeUtils.currentTimeMillis(); + this.requestStartTime = 0; + this.requests = 0; + } + + @SuppressWarnings({"FutureReturnValueIgnored"}) + void run() { + if (!hasErrored.get()) { + try { + requests++; + requestStartTime = DateTimeUtils.currentTimeMillis(); + submitFn.apply(destination, records).whenComplete(this); + } catch (Throwable e) { + setAsyncFailure(e); + } + } + } + + @Override + public void accept(List results, Throwable throwable) { + try { + long now = DateTimeUtils.currentTimeMillis(); + long latencyMillis = now - requestStartTime; + synchronized (stats) { + stats.addBatchWriteRequest(latencyMillis, requests > 1); + } + if (results != null && !hasErrored.get()) { + if (!hasFailedRecords(results)) { + // Request succeeded, release one permit + requestPermits.release(); + LOG.debug( + "Done writing {} records [{} ms, {} request(s)]", + totalRecords, + now - handlerStartTime, + requests); + } else { + try { + if (BackOffUtils.next(Sleeper.DEFAULT, backoff)) { + LOG.info(summarizeErrors("Attempting retry", results)); + records = failedRecords(records, results); + run(); + } else { + throwable = new IOException(summarizeErrors("Exceeded retries", results)); + } + } catch (Throwable e) { + throwable = new IOException(summarizeErrors("Aborted retries", results), e); + } + } + } + } catch (Throwable e) { + throwable = e; + } + if (throwable != null) { + setAsyncFailure(throwable); + } + } + + private void setAsyncFailure(Throwable throwable) { + LOG.warn("Error when writing batch.", throwable); + hasErrored.set(true); + asyncFailure.updateAndGet( + ex -> { + if (ex != null) { + throwable.addSuppressed(ex); + } + return throwable; + }); + requestPermits.release(concurrentRequests); // unblock everything to fail fast + } + + private String summarizeErrors(String prefix, List results) { + Map countsPerError = + results.stream() + .map(errorCodeFn) + .filter(notNull()) + .collect(groupingBy(identity(), counting())); + return countsPerError.entrySet().stream() + .map(kv -> String.format("code %s for %d record(s)", kv.getKey(), kv.getValue())) + .collect(joining(", ", prefix + " after partial failure: ", ".")); + } + } + + private static FluentBackoff retryBackoff(int retries, @Nullable RetryConfiguration retry) { + FluentBackoff backoff = FluentBackoff.DEFAULT.withMaxRetries(retries); + if (retry != null) { + if (retry.throttledBaseBackoff() != null) { + backoff = backoff.withInitialBackoff(retry.throttledBaseBackoff()); + } + if (retry.maxBackoff() != null) { + backoff = backoff.withMaxBackoff(retry.maxBackoff()); + } + } + return backoff; + } +} diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ClientBuilderFactory.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ClientBuilderFactory.java index aa5e41e68a531..6398de57b5c3e 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ClientBuilderFactory.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ClientBuilderFactory.java @@ -18,9 +18,9 @@ package org.apache.beam.sdk.io.aws2.common; import static java.time.Duration.ofMillis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.Serializable; import java.net.URI; @@ -30,7 +30,7 @@ import javax.annotation.Nullable; import org.apache.beam.sdk.io.aws2.options.AwsOptions; import org.apache.beam.sdk.util.InstanceBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.awscore.client.builder.AwsClientBuilder; import software.amazon.awssdk.core.client.builder.SdkAsyncClientBuilder; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ClientConfiguration.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ClientConfiguration.java index 9ee8eb277ddce..08fb595bd037d 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ClientConfiguration.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ClientConfiguration.java @@ -52,6 +52,7 @@ @JsonInclude(value = JsonInclude.Include.NON_EMPTY) @JsonDeserialize(builder = ClientConfiguration.Builder.class) public abstract class ClientConfiguration implements Serializable { + public static final ClientConfiguration EMPTY = ClientConfiguration.builder().build(); /** * Optional {@link AwsCredentialsProvider}. If set, this overwrites the default in {@link diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ObjectPool.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ObjectPool.java index d1b109260e92e..ea3c95b2bbd56 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ObjectPool.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ObjectPool.java @@ -23,8 +23,8 @@ import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.function.ThrowingConsumer; import org.apache.beam.sdk.io.aws2.options.AwsOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashBiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBiMap; import org.apache.commons.lang3.tuple.Pair; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/RetryConfiguration.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/RetryConfiguration.java index 4a816cb3d91ea..8a27f3d65dd0c 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/RetryConfiguration.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/RetryConfiguration.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.aws2.common; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.joda.time.Duration.ZERO; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/dynamodb/DynamoDBIO.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/dynamodb/DynamoDBIO.java index dc97371df0a43..b7cf43f93eb21 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/dynamodb/DynamoDBIO.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/dynamodb/DynamoDBIO.java @@ -20,8 +20,8 @@ import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.mapping; import static java.util.stream.Collectors.toList; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import java.io.IOException; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/AsyncPutRecordsHandler.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/AsyncPutRecordsHandler.java deleted file mode 100644 index 78439cb1d3497..0000000000000 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/AsyncPutRecordsHandler.java +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io.aws2.kinesis; - -import static java.util.function.Function.identity; -import static java.util.stream.Collectors.counting; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.joining; -import static java.util.stream.Collectors.toList; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Semaphore; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.BiConsumer; -import java.util.function.Supplier; -import javax.annotation.concurrent.NotThreadSafe; -import org.apache.beam.repackaged.core.org.apache.commons.lang3.tuple.Pair; -import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.sdk.util.BackOff; -import org.apache.beam.sdk.util.BackOffUtils; -import org.apache.beam.sdk.util.FluentBackoff; -import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Streams; -import org.joda.time.DateTimeUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.services.kinesis.KinesisAsyncClient; -import software.amazon.awssdk.services.kinesis.model.PutRecordsRequest; -import software.amazon.awssdk.services.kinesis.model.PutRecordsRequestEntry; -import software.amazon.awssdk.services.kinesis.model.PutRecordsResponse; - -/** - * Async handler for {@link KinesisAsyncClient#putRecords(PutRecordsRequest)} that automatically - * retries unprocessed records in case of a partial success. - * - *

    The handler enforces the provided upper limit of concurrent requests. Once that limit is - * reached any further call to {@link #putRecords(String, List)} will block until another request - * completed. - * - *

    The handler is fail fast and won't submit any further request after a failure. Async failures - * can be polled using {@link #checkForAsyncFailure()}. - */ -@NotThreadSafe -@Internal -class AsyncPutRecordsHandler { - private static final Logger LOG = LoggerFactory.getLogger(AsyncPutRecordsHandler.class); - private final KinesisAsyncClient kinesis; - private final Supplier backoff; - private final int concurrentRequests; - private final Stats stats; - - private AtomicBoolean hasErrored; - private AtomicReference asyncFailure; - private Semaphore pendingRequests; - - AsyncPutRecordsHandler( - KinesisAsyncClient kinesis, int concurrency, Supplier backoff, Stats stats) { - this.kinesis = kinesis; - this.backoff = backoff; - this.concurrentRequests = concurrency; - this.hasErrored = new AtomicBoolean(false); - this.asyncFailure = new AtomicReference<>(); - this.pendingRequests = new Semaphore(concurrentRequests); - this.stats = stats; - } - - AsyncPutRecordsHandler( - KinesisAsyncClient kinesis, int concurrency, FluentBackoff backoff, Stats stats) { - this(kinesis, concurrency, () -> backoff.backoff(), stats); - } - - protected int pendingRequests() { - return concurrentRequests - pendingRequests.availablePermits(); - } - - void reset() { - hasErrored = new AtomicBoolean(false); - asyncFailure = new AtomicReference<>(); - pendingRequests = new Semaphore(concurrentRequests); - } - - /** If this handler has errored since it was last reset. */ - boolean hasErrored() { - return hasErrored.get(); - } - - /** - * Check if any failure happened async. - * - * @throws Throwable The last async failure, afterwards reset it. - */ - void checkForAsyncFailure() throws Throwable { - @SuppressWarnings("nullness") - Throwable failure = asyncFailure.getAndSet(null); - if (failure != null) { - throw failure; - } - } - - /** - * Wait for all pending requests to complete and check for failures. - * - * @throws Throwable The last async failure if present using {@link #checkForAsyncFailure()} - */ - void waitForCompletion() throws Throwable { - pendingRequests.acquireUninterruptibly(concurrentRequests); - checkForAsyncFailure(); - } - - /** - * Asynchronously trigger a put records request. - * - *

    This will respect the concurrency limit of the handler and first wait for a permit. - * - * @throws Throwable The last async failure if present using {@link #checkForAsyncFailure()} - */ - void putRecords(String stream, List records) throws Throwable { - pendingRequests.acquireUninterruptibly(); - new RetryHandler(stream, records).run(); - checkForAsyncFailure(); - } - - interface Stats { - void addPutRecordsRequest(long latencyMillis, boolean isPartialRetry); - } - - /** - * This handler coordinates retries in case of a partial success. - * - *

      - *
    • Release permit if all (remaining) records are successful to allow for a new request to - * start. - *
    • Attempt retry in case of partial success for all erroneous records using backoff. Set - * async failure once retries are exceeded. - *
    • Set async failure if the entire request fails. Retries, if configured & applicable, have - * already been attempted by the AWS SDK in that case. - *
    - * - * The next call of {@link #checkForAsyncFailure()}, {@link #putRecords(String, List)} or {@link - * #waitForCompletion()} will check for the last async failure and throw it. Afterwards the - * failure state is reset. - */ - private class RetryHandler implements BiConsumer { - private final int totalRecords; - private final String stream; - private final BackOff backoff; // backoff in case of throttling - - private final long handlerStartTime; - private long requestStartTime; - private int requests; - - private List records; - - RetryHandler(String stream, List records) { - this.stream = stream; - this.totalRecords = records.size(); - this.records = records; - this.backoff = AsyncPutRecordsHandler.this.backoff.get(); - this.handlerStartTime = DateTimeUtils.currentTimeMillis(); - this.requestStartTime = 0; - this.requests = 0; - } - - @SuppressWarnings({"FutureReturnValueIgnored"}) - void run() { - if (!hasErrored.get()) { - try { - requests++; - requestStartTime = DateTimeUtils.currentTimeMillis(); - PutRecordsRequest request = - PutRecordsRequest.builder().streamName(stream).records(records).build(); - kinesis.putRecords(request).whenComplete(this); - } catch (Throwable e) { - setAsyncFailure(e); - } - } - } - - @Override - public void accept(PutRecordsResponse response, Throwable throwable) { - try { - long now = DateTimeUtils.currentTimeMillis(); - long latencyMillis = now - requestStartTime; - synchronized (stats) { - stats.addPutRecordsRequest(latencyMillis, requests > 1); - } - if (response != null && !hasErrored.get()) { - if (!hasErrors(response)) { - // Request succeeded, release one permit - pendingRequests.release(); - LOG.debug( - "Done writing {} records [{} ms, {} request(s)]", - totalRecords, - now - handlerStartTime, - requests); - } else { - try { - if (BackOffUtils.next(Sleeper.DEFAULT, backoff)) { - LOG.info(summarizeErrors("Attempting retry", response)); - records = failedRecords(response); - run(); - } else { - throwable = new IOException(summarizeErrors("Exceeded retries", response)); - } - } catch (Throwable e) { - throwable = new IOException(summarizeErrors("Aborted retries", response), e); - } - } - } - } catch (Throwable e) { - throwable = e; - } - if (throwable != null) { - setAsyncFailure(throwable); - } - } - - private void setAsyncFailure(Throwable throwable) { - LOG.warn("Error when writing to Kinesis.", throwable); - hasErrored.set(true); - asyncFailure.updateAndGet( - ex -> { - if (ex != null) { - throwable.addSuppressed(ex); - } - return throwable; - }); - pendingRequests.release(concurrentRequests); // unblock everything to fail fast - } - - private boolean hasErrors(PutRecordsResponse response) { - return response.records().stream().anyMatch(e -> e.errorCode() != null); - } - - private List failedRecords(PutRecordsResponse response) { - return Streams.zip(records.stream(), response.records().stream(), Pair::of) - .filter(p -> p.getRight().errorCode() != null) - .map(p -> p.getLeft()) - .collect(toList()); - } - - private String summarizeErrors(String prefix, PutRecordsResponse response) { - Map countPerError = - response.records().stream() - .filter(e -> e.errorCode() != null) - .map(e -> e.errorCode()) - .collect(groupingBy(identity(), counting())); - return countPerError.entrySet().stream() - .map(kv -> String.format("%s for %d record(s)", kv.getKey(), kv.getValue())) - .collect(joining(", ", prefix + " after failure when writing to Kinesis: ", ".")); - } - } -} diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/EFOShardSubscriber.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/EFOShardSubscriber.java index be08b18780bca..67bc8472a7c52 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/EFOShardSubscriber.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/EFOShardSubscriber.java @@ -22,7 +22,7 @@ import static org.apache.beam.sdk.io.aws2.kinesis.EFOShardSubscriber.State.RUNNING; import static org.apache.beam.sdk.io.aws2.kinesis.EFOShardSubscriber.State.STOPPED; import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import io.netty.channel.ChannelException; import java.nio.channels.ClosedChannelException; @@ -31,7 +31,7 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.reactivestreams.Subscriber; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/EFOShardSubscribersPool.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/EFOShardSubscribersPool.java index b97ed6b08dd80..2786f680444f6 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/EFOShardSubscribersPool.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/EFOShardSubscribersPool.java @@ -19,8 +19,8 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.beam.sdk.io.aws2.kinesis.TimeUtil.minTimestamp; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.util.ArrayList; @@ -38,8 +38,8 @@ import java.util.function.Supplier; import org.apache.beam.repackaged.core.org.apache.commons.lang3.RandomStringUtils; import org.apache.beam.sdk.util.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ForwardingIterator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ForwardingIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisIO.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisIO.java index cf6796198bb45..6f06f4f731697 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisIO.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisIO.java @@ -18,8 +18,8 @@ package org.apache.beam.sdk.io.aws2.kinesis; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.apache.commons.lang3.ArrayUtils.EMPTY_BYTE_ARRAY; import static org.apache.commons.lang3.StringUtils.isEmpty; import static software.amazon.awssdk.services.kinesis.model.ShardFilterType.AT_LATEST; @@ -35,6 +35,7 @@ import java.util.Map; import java.util.NavigableSet; import java.util.TreeSet; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.function.Supplier; @@ -42,11 +43,11 @@ import javax.annotation.concurrent.ThreadSafe; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.io.Read.Unbounded; +import org.apache.beam.sdk.io.aws2.common.AsyncBatchWriteHandler; import org.apache.beam.sdk.io.aws2.common.ClientBuilderFactory; import org.apache.beam.sdk.io.aws2.common.ClientConfiguration; import org.apache.beam.sdk.io.aws2.common.ObjectPool; import org.apache.beam.sdk.io.aws2.common.ObjectPool.ClientPool; -import org.apache.beam.sdk.io.aws2.common.RetryConfiguration; import org.apache.beam.sdk.io.aws2.kinesis.KinesisPartitioner.ExplicitPartitioner; import org.apache.beam.sdk.io.aws2.options.AwsOptions; import org.apache.beam.sdk.metrics.Counter; @@ -61,7 +62,6 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.Sum; -import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.MovingFunction; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; @@ -69,9 +69,9 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSortedSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSortedSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; import org.joda.time.DateTimeUtils; @@ -84,7 +84,9 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.kinesis.KinesisAsyncClient; import software.amazon.awssdk.services.kinesis.model.ListShardsRequest; +import software.amazon.awssdk.services.kinesis.model.PutRecordsRequest; import software.amazon.awssdk.services.kinesis.model.PutRecordsRequestEntry; +import software.amazon.awssdk.services.kinesis.model.PutRecordsResultEntry; import software.amazon.awssdk.services.kinesis.model.Shard; import software.amazon.awssdk.services.kinesis.model.SubscribeToShardRequest; import software.amazon.awssdk.services.kinesis.model.SubscribeToShardResponseHandler; @@ -559,7 +561,7 @@ public Read withCustomRateLimitPolicy(RateLimitPolicyFactory rateLimitPolicyFact * corresponds to the number of in-flight shard events which itself can contain multiple, * potentially even aggregated records. * - * @see {@link #withConsumerArn(String)} + * @see #withConsumerArn(String) */ public Read withMaxCapacityPerShard(Integer maxCapacity) { checkArgument(maxCapacity > 0, "maxCapacity must be positive, but was: %s", maxCapacity); @@ -901,30 +903,32 @@ private static class Writer implements AutoCloseable { protected final Write spec; protected final Stats stats; - protected final AsyncPutRecordsHandler handler; + protected final AsyncBatchWriteHandler handler; protected final KinesisAsyncClient kinesis; - private List requestEntries; private int requestBytes = 0; Writer(PipelineOptions options, Write spec) { ClientConfiguration clientConfig = spec.clientConfiguration(); - RetryConfiguration retryConfig = clientConfig.retry(); - FluentBackoff backoff = FluentBackoff.DEFAULT.withMaxRetries(PARTIAL_RETRIES); - if (retryConfig != null) { - if (retryConfig.throttledBaseBackoff() != null) { - backoff = backoff.withInitialBackoff(retryConfig.throttledBaseBackoff()); - } - if (retryConfig.maxBackoff() != null) { - backoff = backoff.withMaxBackoff(retryConfig.maxBackoff()); - } - } this.spec = spec; this.stats = new Stats(); this.kinesis = CLIENTS.retain(options.as(AwsOptions.class), clientConfig); - this.handler = - new AsyncPutRecordsHandler(kinesis, spec.concurrentRequests(), backoff, stats); this.requestEntries = new ArrayList<>(); + this.handler = + AsyncBatchWriteHandler.byPosition( + spec.concurrentRequests(), + PARTIAL_RETRIES, + clientConfig.retry(), + stats, + (stream, records) -> putRecords(kinesis, stream, records), + r -> r.errorCode()); + } + + private static CompletableFuture> putRecords( + KinesisAsyncClient kinesis, String stream, List records) { + PutRecordsRequest req = + PutRecordsRequest.builder().streamName(stream).records(records).build(); + return kinesis.putRecords(req).thenApply(resp -> resp.records()); } public void startBundle() { @@ -998,7 +1002,7 @@ protected final void asyncFlushEntries() throws Throwable { List recordsToWrite = requestEntries; requestEntries = new ArrayList<>(); requestBytes = 0; - handler.putRecords(spec.streamName(), recordsToWrite); + handler.batchWrite(spec.streamName(), recordsToWrite); } } @@ -1115,7 +1119,7 @@ protected void write(String partitionKey, @Nullable String explicitHashKey, byte } // only check timeouts sporadically if concurrency is already maxed out - if (handler.pendingRequests() < spec.concurrentRequests() || Math.random() < 0.05) { + if (handler.requestsInProgress() < spec.concurrentRequests() || Math.random() < 0.05) { checkAggregationTimeouts(); } } @@ -1275,7 +1279,7 @@ private BigInteger lowerHashKey(Shard shard) { } } - private static class Stats implements AsyncPutRecordsHandler.Stats { + private static class Stats implements AsyncBatchWriteHandler.Stats { private static final Logger LOG = LoggerFactory.getLogger(Stats.class); private static final Duration LOG_STATS_PERIOD = Duration.standardSeconds(10); @@ -1328,7 +1332,7 @@ void addClientRecord(int recordBytes) { } @Override - public void addPutRecordsRequest(long latencyMillis, boolean isPartialRetry) { + public void addBatchWriteRequest(long latencyMillis, boolean isPartialRetry) { long timeMillis = DateTimeUtils.currentTimeMillis(); numPutRequests.add(timeMillis, 1); if (isPartialRetry) { diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisIOOptions.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisIOOptions.java index 90bb4493dfd45..ca1b1c4f5a0e8 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisIOOptions.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisIOOptions.java @@ -43,8 +43,8 @@ import org.apache.beam.sdk.options.Description; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * PipelineOptions for {@link KinesisIO}. diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisReader.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisReader.java index 1ca37d23cd8cf..2cf9c9669caaf 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisReader.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisReader.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.aws2.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.util.NoSuchElementException; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisReaderCheckpoint.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisReaderCheckpoint.java index dd20699a40775..fcba988d3b8d7 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisReaderCheckpoint.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisReaderCheckpoint.java @@ -17,15 +17,15 @@ */ package org.apache.beam.sdk.io.aws2.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.partition; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.partition; import java.io.IOException; import java.io.Serializable; import java.util.Iterator; import java.util.List; import org.apache.beam.sdk.io.UnboundedSource; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * Checkpoint representing a total progress in a set of shards in single stream. The set of shards diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisSource.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisSource.java index 8e1d0b091f84f..f2d4030ca5550 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisSource.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisSource.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.aws2.kinesis; import static org.apache.beam.sdk.io.aws2.common.ClientBuilderFactory.buildClient; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.util.ArrayList; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/RateLimitPolicyFactory.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/RateLimitPolicyFactory.java index 92eb170e40ba0..34d9562187ee9 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/RateLimitPolicyFactory.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/RateLimitPolicyFactory.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.util.BackOffUtils; import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/RecordFilter.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/RecordFilter.java index b6f250d27345b..6f94b7a5a3639 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/RecordFilter.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/RecordFilter.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.aws2.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; import java.util.List; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/RecordsAggregator.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/RecordsAggregator.java index bcf546e792e80..e365e30425fb2 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/RecordsAggregator.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/RecordsAggregator.java @@ -30,7 +30,7 @@ import javax.annotation.concurrent.NotThreadSafe; import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.util.VarInt; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; import software.amazon.awssdk.services.kinesis.model.PutRecordsRequestEntry; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/ShardCheckpoint.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/ShardCheckpoint.java index 312930f99549d..4a03ab5e85a29 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/ShardCheckpoint.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/ShardCheckpoint.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.aws2.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import static software.amazon.awssdk.services.kinesis.model.ShardIteratorType.AFTER_SEQUENCE_NUMBER; import static software.amazon.awssdk.services.kinesis.model.ShardIteratorType.AT_SEQUENCE_NUMBER; import static software.amazon.awssdk.services.kinesis.model.ShardIteratorType.AT_TIMESTAMP; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/ShardListingUtils.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/ShardListingUtils.java index 370c67f161850..cb5f609b9ec7c 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/ShardListingUtils.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/ShardListingUtils.java @@ -23,7 +23,7 @@ import org.apache.beam.sdk.util.BackOffUtils; import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.joda.time.Instant; import org.slf4j.Logger; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/ShardReadersPool.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/ShardReadersPool.java index ddaf65eee9670..318b7b31c4a8e 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/ShardReadersPool.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/ShardReadersPool.java @@ -19,7 +19,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.beam.sdk.io.aws2.kinesis.TimeUtil.minTimestamp; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Collection; import java.util.List; @@ -37,9 +37,9 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.apache.beam.sdk.io.aws2.kinesis.KinesisIO.Read; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Instant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/ShardRecordsIterator.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/ShardRecordsIterator.java index da1e8dc0aaddb..0a2964424a3bb 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/ShardRecordsIterator.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/ShardRecordsIterator.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.aws2.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.ArrayList; import java.util.List; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/SimplifiedKinesisClient.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/SimplifiedKinesisClient.java index 1402ef7216d43..d93252885171e 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/SimplifiedKinesisClient.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/SimplifiedKinesisClient.java @@ -17,13 +17,13 @@ */ package org.apache.beam.sdk.io.aws2.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.List; import java.util.concurrent.Callable; import java.util.function.Supplier; import java.util.stream.Collectors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; import org.joda.time.Minutes; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/StartingPoint.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/StartingPoint.java index 57c02db113d51..b094a7011e061 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/StartingPoint.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/StartingPoint.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.aws2.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.Serializable; import java.util.Objects; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/WatermarkParameters.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/WatermarkParameters.java index 55c36d1128b91..02ece22d7d121 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/WatermarkParameters.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/WatermarkParameters.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.aws2.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.io.Serializable; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/WatermarkPolicyFactory.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/WatermarkPolicyFactory.java index 9d67bd516577b..2d8759640cc62 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/WatermarkPolicyFactory.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/kinesis/WatermarkPolicyFactory.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.aws2.kinesis; import java.io.Serializable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsModule.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsModule.java index 848a45dc693bb..12ae1abfe446d 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsModule.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsModule.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.aws2.options; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -45,7 +45,7 @@ import java.io.IOException; import java.util.function.Supplier; import org.apache.beam.repackaged.core.org.apache.commons.lang3.reflect.FieldUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.NonNull; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsPipelineOptionsRegistrar.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsPipelineOptionsRegistrar.java index 4384b4c230402..cb7331cd74e12 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsPipelineOptionsRegistrar.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsPipelineOptionsRegistrar.java @@ -20,7 +20,7 @@ import com.google.auto.service.AutoService; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** A registrar containing the default AWS options. */ @AutoService(PipelineOptionsRegistrar.class) diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/DefaultS3ClientBuilderFactory.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/DefaultS3ClientBuilderFactory.java index 4d4209a22bb31..0cc8a143812ad 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/DefaultS3ClientBuilderFactory.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/DefaultS3ClientBuilderFactory.java @@ -20,7 +20,7 @@ import org.apache.beam.sdk.io.aws2.common.ClientBuilderFactory; import org.apache.beam.sdk.io.aws2.options.S3ClientBuilderFactory; import org.apache.beam.sdk.io.aws2.options.S3Options; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3ClientBuilder; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/DefaultS3FileSystemSchemeRegistrar.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/DefaultS3FileSystemSchemeRegistrar.java index cb3e5a8298490..6b6921a409238 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/DefaultS3FileSystemSchemeRegistrar.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/DefaultS3FileSystemSchemeRegistrar.java @@ -17,13 +17,13 @@ */ package org.apache.beam.sdk.io.aws2.s3; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.service.AutoService; import javax.annotation.Nonnull; import org.apache.beam.sdk.io.aws2.options.S3Options; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** Registers the "s3" uri schema to be handled by {@link S3FileSystem}. */ @AutoService(S3FileSystemSchemeRegistrar.class) diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3FileSystem.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3FileSystem.java index 41b3a409215b2..5f08600758f6c 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3FileSystem.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3FileSystem.java @@ -18,9 +18,9 @@ package org.apache.beam.sdk.io.aws2.s3; import static org.apache.beam.sdk.io.FileSystemUtils.wildcardToRegexp; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.io.FileNotFoundException; @@ -47,18 +47,18 @@ import org.apache.beam.sdk.io.fs.MatchResult; import org.apache.beam.sdk.io.fs.MoveOptions; import org.apache.beam.sdk.util.MoreFutures; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Suppliers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Suppliers; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3FileSystemRegistrar.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3FileSystemRegistrar.java index 279ca541b304d..e4af28e6bbd20 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3FileSystemRegistrar.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3FileSystemRegistrar.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.aws2.s3; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.service.AutoService; import java.util.Map; @@ -28,7 +28,7 @@ import org.apache.beam.sdk.io.FileSystemRegistrar; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.common.ReflectHelpers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Streams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Streams; /** * {@link AutoService} registrar for the {@link S3FileSystem}. diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3ReadableSeekableByteChannel.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3ReadableSeekableByteChannel.java index 02982b0581480..bbfd9eb8755b6 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3ReadableSeekableByteChannel.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3ReadableSeekableByteChannel.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.aws2.s3; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import static software.amazon.awssdk.utils.IoUtils.drainInputStream; import java.io.BufferedInputStream; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3ResourceId.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3ResourceId.java index cb9470cab51ae..eb75817df8e02 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3ResourceId.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3ResourceId.java @@ -17,9 +17,9 @@ */ package org.apache.beam.sdk.io.aws2.s3; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.ObjectStreamException; import java.util.Date; @@ -28,8 +28,8 @@ import java.util.regex.Pattern; import org.apache.beam.sdk.io.fs.ResolveOptions; import org.apache.beam.sdk.io.fs.ResourceId; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.checkerframework.checker.nullness.qual.Nullable; /** An identifier which represents a S3Object resource. */ diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3WritableByteChannel.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3WritableByteChannel.java index 5906f680a7904..d0b331366aad3 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3WritableByteChannel.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3WritableByteChannel.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.aws2.s3; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -30,7 +30,7 @@ import java.util.ArrayList; import java.util.Base64; import java.util.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKey.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKey.java index 8e0ea423378f6..f9a3229372580 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKey.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKey.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.aws2.s3; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/schemas/AwsSchemaProvider.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/schemas/AwsSchemaProvider.java index 7f8182e6136a8..65812d72df1db 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/schemas/AwsSchemaProvider.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/schemas/AwsSchemaProvider.java @@ -21,9 +21,9 @@ import static java.util.stream.Collectors.toMap; import static org.apache.beam.sdk.io.aws2.schemas.AwsSchemaUtils.getter; import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets.difference; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets.newHashSet; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets.difference; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets.newHashSet; import java.util.ArrayList; import java.util.List; @@ -45,7 +45,7 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.RowWithGetters; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; import software.amazon.awssdk.core.SdkField; import software.amazon.awssdk.core.SdkPojo; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/schemas/AwsSchemaRegistrar.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/schemas/AwsSchemaRegistrar.java index 131a62eac46cf..57599d4a2c01e 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/schemas/AwsSchemaRegistrar.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/schemas/AwsSchemaRegistrar.java @@ -21,7 +21,7 @@ import java.util.List; import org.apache.beam.sdk.schemas.SchemaProvider; import org.apache.beam.sdk.schemas.SchemaProviderRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; @AutoService(SchemaProviderRegistrar.class) public class AwsSchemaRegistrar implements SchemaProviderRegistrar { diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/schemas/AwsTypes.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/schemas/AwsTypes.java index 229ad43854d59..b941079b121f4 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/schemas/AwsTypes.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/schemas/AwsTypes.java @@ -19,7 +19,7 @@ import static java.util.Collections.singleton; import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static software.amazon.awssdk.core.protocol.MarshallingType.INSTANT; import static software.amazon.awssdk.core.protocol.MarshallingType.LIST; import static software.amazon.awssdk.core.protocol.MarshallingType.MAP; @@ -38,11 +38,11 @@ import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Ascii; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Ascii; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.joda.time.Instant; import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.core.SdkField; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/sns/SnsIO.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/sns/SnsIO.java index 182b7dbb3881a..ecbe87a5da44d 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/sns/SnsIO.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/sns/SnsIO.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.aws2.sns; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.util.function.Consumer; diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/sqs/SqsCheckpointMark.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/sqs/SqsCheckpointMark.java index dc535efa8907a..e18770c8c6a86 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/sqs/SqsCheckpointMark.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/sqs/SqsCheckpointMark.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.aws2.sqs; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.io.Serializable; @@ -25,9 +25,9 @@ import java.util.List; import java.util.Optional; import org.apache.beam.sdk.io.UnboundedSource; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; @SuppressWarnings({ diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/sqs/SqsIO.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/sqs/SqsIO.java index 72befc6003fe0..f7f767ab85eea 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/sqs/SqsIO.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/sqs/SqsIO.java @@ -17,25 +17,74 @@ */ package org.apache.beam.sdk.io.aws2.sqs; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static java.util.Collections.EMPTY_LIST; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.beam.sdk.io.aws2.common.ClientBuilderFactory.buildClient; +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.ConcurrentModificationException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; import java.util.function.Consumer; +import javax.annotation.concurrent.NotThreadSafe; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.io.aws2.common.AsyncBatchWriteHandler; +import org.apache.beam.sdk.io.aws2.common.AsyncBatchWriteHandler.Stats; import org.apache.beam.sdk.io.aws2.common.ClientBuilderFactory; import org.apache.beam.sdk.io.aws2.common.ClientConfiguration; import org.apache.beam.sdk.io.aws2.options.AwsOptions; import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.schemas.NoSuchSchemaException; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.schemas.Schema.Field; +import org.apache.beam.sdk.schemas.SchemaRegistry; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; +import org.apache.beam.sdk.values.PInput; +import org.apache.beam.sdk.values.POutput; +import org.apache.beam.sdk.values.PValue; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; +import org.checkerframework.dataflow.qual.Pure; import org.joda.time.Duration; +import org.joda.time.Instant; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sqs.SqsAsyncClient; +import software.amazon.awssdk.services.sqs.model.BatchResultErrorEntry; +import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequest; +import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequestEntry; import software.amazon.awssdk.services.sqs.model.SendMessageRequest; /** @@ -91,21 +140,30 @@ * then opt to retry the current partition in entirety or abort if the max number of retries of the * runner is reached. */ -@SuppressWarnings({ - "nullness" // TODO(https://github.com/apache/beam/issues/20497) -}) public class SqsIO { public static Read read() { return new AutoValue_SqsIO_Read.Builder() - .setClientConfiguration(ClientConfiguration.builder().build()) + .setClientConfiguration(ClientConfiguration.EMPTY) .setMaxNumRecords(Long.MAX_VALUE) .build(); } + /** @deprecated Use {@link #writeBatches()} for more configuration options. */ + @Deprecated public static Write write() { return new AutoValue_SqsIO_Write.Builder() - .setClientConfiguration(ClientConfiguration.builder().build()) + .setClientConfiguration(ClientConfiguration.EMPTY) + .build(); + } + + public static WriteBatches writeBatches() { + return new AutoValue_SqsIO_WriteBatches.Builder() + .clientConfiguration(ClientConfiguration.EMPTY) + .concurrentRequests(WriteBatches.DEFAULT_CONCURRENCY) + .batchSize(WriteBatches.MAX_BATCH_SIZE) + .batchTimeout(WriteBatches.DEFAULT_BATCH_TIMEOUT) + .strictTimeouts(false) .build(); } @@ -124,7 +182,7 @@ public abstract static class Read extends PTransform expand(PBegin input) { AwsOptions awsOptions = input.getPipeline().getOptions().as(AwsOptions.class); ClientBuilderFactory.validate(awsOptions, clientConfiguration()); @@ -188,15 +247,18 @@ public PCollection expand(PBegin input) { return input.getPipeline().apply(transform); } } - // TODO: Add write batch api to improve performance + /** * A {@link PTransform} to send messages to SQS. See {@link SqsIO} for more information on usage * and configuration. + * + * @deprecated superseded by {@link WriteBatches} */ @AutoValue + @Deprecated public abstract static class Write extends PTransform, PDone> { - abstract ClientConfiguration getClientConfiguration(); + abstract @Pure ClientConfiguration getClientConfiguration(); abstract Builder builder(); @@ -215,33 +277,665 @@ public Write withClientConfiguration(ClientConfiguration config) { @Override public PDone expand(PCollection input) { - AwsOptions awsOptions = input.getPipeline().getOptions().as(AwsOptions.class); - ClientBuilderFactory.validate(awsOptions, getClientConfiguration()); - - input.apply(ParDo.of(new SqsWriteFn(this))); + input.apply( + SqsIO.writeBatches() + .withBatchSize(1) + .to(SendMessageRequest::queueUrl)); return PDone.in(input.getPipeline()); } } - private static class SqsWriteFn extends DoFn { - private final Write spec; - private transient SqsClient sqs; + /** + * A {@link PTransform} to send messages to SQS. See {@link SqsIO} for more information on usage + * and configuration. + */ + @AutoValue + public abstract static class WriteBatches + extends PTransform, WriteBatches.Result> { + private static final Logger LOG = LoggerFactory.getLogger(WriteBatches.class); + private static final int DEFAULT_CONCURRENCY = 5; + private static final int MAX_BATCH_SIZE = 10; + private static final Duration DEFAULT_BATCH_TIMEOUT = Duration.standardSeconds(3); + + abstract @Pure int concurrentRequests(); + + abstract @Pure Duration batchTimeout(); + + abstract @Pure boolean strictTimeouts(); + + abstract @Pure int batchSize(); + + abstract @Pure ClientConfiguration clientConfiguration(); + + abstract @Pure @Nullable EntryMapperFn entryMapper(); + + abstract @Pure @Nullable DynamicDestination dynamicDestination(); + + abstract @Pure @Nullable String queueUrl(); + + abstract Builder builder(); + + public interface DynamicDestination extends Serializable { + String queueUrl(T message); + } + + @AutoValue.Builder + abstract static class Builder { + abstract Builder concurrentRequests(int concurrentRequests); + + abstract Builder batchTimeout(Duration duration); + + abstract Builder strictTimeouts(boolean strict); + + abstract Builder batchSize(int batchSize); + + abstract Builder clientConfiguration(ClientConfiguration config); + + abstract Builder entryMapper(@Nullable EntryMapperFn entryMapper); + + abstract Builder dynamicDestination(@Nullable DynamicDestination destination); + + abstract Builder queueUrl(@Nullable String queueUrl); + + abstract WriteBatches build(); + } + + /** Configuration of SQS client. */ + public WriteBatches withClientConfiguration(ClientConfiguration config) { + checkArgument(config != null, "ClientConfiguration cannot be null"); + return builder().clientConfiguration(config).build(); + } + + /** Max number of concurrent batch write requests per bundle, default is {@code 5}. */ + public WriteBatches withConcurrentRequests(int concurrentRequests) { + checkArgument(concurrentRequests > 0, "concurrentRequests must be > 0"); + return builder().concurrentRequests(concurrentRequests).build(); + } + + /** + * Optional mapper to create a batch entry from a unique entry id and the input {@code T}, + * otherwise inferred from the schema. + */ + public WriteBatches withEntryMapper(EntryMapperFn mapper) { + return builder().entryMapper(mapper).build(); + } + + /** + * Optional mapper to create a batch entry from the input {@code T} using a builder, otherwise + * inferred from the schema. + */ + public WriteBatches withEntryMapper(EntryMapperFn.Builder mapper) { + return builder().entryMapper(mapper).build(); + } + + /** The batch size to use, default (and AWS limit) is {@code 10}. */ + public WriteBatches withBatchSize(int batchSize) { + checkArgument( + batchSize > 0 && batchSize <= MAX_BATCH_SIZE, + "Maximum allowed batch size is " + MAX_BATCH_SIZE); + return builder().batchSize(batchSize).build(); + } + + /** + * The duration to accumulate records before timing out, default is 3 secs. + * + *

    By default timeouts will be checked upon arrival of records. + */ + public WriteBatches withBatchTimeout(Duration timeout) { + return withBatchTimeout(timeout, false); + } + + /** + * The duration to accumulate records before timing out, default is 3 secs. + * + *

    By default timeouts will be checked upon arrival of records. If using {@code strict} + * enforcement, timeouts will be check by a separate thread. + */ + public WriteBatches withBatchTimeout(Duration timeout, boolean strict) { + return builder().batchTimeout(timeout).strictTimeouts(strict).build(); + } + + /** Dynamic record based destination to write to. */ + public WriteBatches to(DynamicDestination destination) { + checkArgument(destination != null, "DynamicDestination cannot be null"); + return builder().queueUrl(null).dynamicDestination(destination).build(); + } - SqsWriteFn(Write write) { - this.spec = write; + /** Queue url to write to. */ + public WriteBatches to(String queueUrl) { + checkArgument(queueUrl != null, "queueUrl cannot be null"); + return builder().dynamicDestination(null).queueUrl(queueUrl).build(); } - @Setup - public void setup(PipelineOptions options) throws Exception { - AwsOptions awsOpts = options.as(AwsOptions.class); - sqs = - ClientBuilderFactory.buildClient( - awsOpts, SqsClient.builder(), spec.getClientConfiguration()); + private EntryMapperFn schemaEntryMapper(PCollection input) { + checkState(input.hasSchema(), "withEntryMapper is required if schema is not available"); + SchemaRegistry registry = input.getPipeline().getSchemaRegistry(); + try { + return new SchemaEntryMapper<>( + input.getSchema(), + registry.getSchema(SendMessageBatchRequestEntry.class), + input.getToRowFunction(), + registry.getFromRowFunction(SendMessageBatchRequestEntry.class)); + } catch (NoSuchSchemaException e) { + throw new RuntimeException(e); + } } - @ProcessElement - public void processElement(ProcessContext processContext) throws Exception { - sqs.sendMessage(processContext.element()); + @Override + public Result expand(PCollection input) { + AwsOptions awsOptions = input.getPipeline().getOptions().as(AwsOptions.class); + ClientBuilderFactory.validate(awsOptions, clientConfiguration()); + EntryMapperFn mapper = entryMapper() != null ? entryMapper() : schemaEntryMapper(input); + input.apply( + ParDo.of( + new DoFn() { + private @Nullable BatchHandler handler = null; + + @Setup + public void setup(PipelineOptions options) { + handler = + new BatchHandler<>(WriteBatches.this, mapper, options.as(AwsOptions.class)); + } + + @StartBundle + public void startBundle() { + handler().startBundle(); + } + + @ProcessElement + public void processElement(ProcessContext cxt) throws Throwable { + handler().process(cxt.element()); + } + + @FinishBundle + public void finishBundle() throws Throwable { + handler().finishBundle(); + } + + @Teardown + public void teardown() throws Exception { + if (handler != null) { + handler.close(); + handler = null; + } + } + + private BatchHandler handler() { + return checkStateNotNull(handler, "SQS handler is null"); + } + })); + return new Result(input.getPipeline()); + } + + /** + * Mapper to create a {@link SendMessageBatchRequestEntry} from a unique batch entry id and the + * input {@code T}. + */ + public interface EntryMapperFn + extends BiFunction, Serializable { + + /** A more convenient {@link EntryMapperFn} variant that already sets the entry id. */ + interface Builder + extends BiConsumer, EntryMapperFn { + @Override + default SendMessageBatchRequestEntry apply(String entryId, T msg) { + SendMessageBatchRequestEntry.Builder builder = SendMessageBatchRequestEntry.builder(); + accept(builder, msg); + return builder.id(entryId).build(); + } + } + } + + @VisibleForTesting + static class SchemaEntryMapper implements EntryMapperFn { + private final SerializableFunction toRow; + private final SerializableFunction fromRow; + private final Schema schema; + private final int[] fieldMapping; + + SchemaEntryMapper( + Schema sourceSchema, + Schema targetSchema, + SerializableFunction toRow, + SerializableFunction fromRow) { + this.toRow = toRow; + this.fromRow = fromRow; + this.schema = targetSchema; + this.fieldMapping = new int[targetSchema.getFieldCount()]; + + Arrays.fill(fieldMapping, -1); + + List ignored = Lists.newLinkedList(); + List invalid = Lists.newLinkedList(); + + for (int i = 0; i < sourceSchema.getFieldCount(); i++) { + Field sourceField = sourceSchema.getField(i); + if (targetSchema.hasField(sourceField.getName())) { + int targetIdx = targetSchema.indexOf(sourceField.getName()); + // make sure field types match + if (!sourceField.typesEqual(targetSchema.getField(targetIdx))) { + invalid.add(sourceField.getName()); + } + fieldMapping[targetIdx] = i; + } else { + ignored.add(sourceField.getName()); + } + } + checkState( + ignored.size() < sourceSchema.getFieldCount(), + "No fields matched, expected %s but got %s", + schema.getFieldNames(), + ignored); + + checkState(invalid.isEmpty(), "Detected incompatible types for input fields: {}", invalid); + + if (!ignored.isEmpty()) { + LOG.warn("Ignoring unmatched input fields: {}", ignored); + } + } + + @Override + public SendMessageBatchRequestEntry apply(String entryId, T input) { + Row row = toRow.apply(input); + Object[] values = new Object[fieldMapping.length]; + values[0] = entryId; + for (int i = 0; i < values.length; i++) { + if (fieldMapping[i] >= 0) { + values[i] = row.getValue(fieldMapping[i]); + } + } + return fromRow.apply(Row.withSchema(schema).attachValues(values)); + } + } + + /** Result of {@link #writeBatches}. */ + public static class Result implements POutput { + private final Pipeline pipeline; + + private Result(Pipeline pipeline) { + this.pipeline = pipeline; + } + + @Override + public Pipeline getPipeline() { + return pipeline; + } + + @Override + public Map, PValue> expand() { + return ImmutableMap.of(); + } + + @Override + public void finishSpecifyingOutput( + String transformName, PInput input, PTransform transform) {} + } + + private static class BatchHandler implements AutoCloseable { + private static final int CHECKS_PER_TIMEOUT_PERIOD = 5; + public static final int EXPIRATION_CHECK_TIMEOUT_SECS = 3; + + private final WriteBatches spec; + private final SqsAsyncClient sqs; + private final Batches batches; + private final EntryMapperFn entryMapper; + private final AsyncBatchWriteHandler + handler; + private final @Nullable ScheduledExecutorService scheduler; + + private @MonotonicNonNull ScheduledFuture expirationCheck = null; + + BatchHandler(WriteBatches spec, EntryMapperFn entryMapper, AwsOptions options) { + this.spec = spec; + this.sqs = buildClient(options, SqsAsyncClient.builder(), spec.clientConfiguration()); + this.entryMapper = entryMapper; + this.handler = + AsyncBatchWriteHandler.byId( + spec.concurrentRequests(), + spec.batchSize(), + spec.clientConfiguration().retry(), + Stats.NONE, + (queue, records) -> sendMessageBatch(sqs, queue, records), + error -> error.code(), + record -> record.id(), + error -> error.id()); + this.scheduler = + spec.strictTimeouts() ? Executors.newSingleThreadScheduledExecutor() : null; + if (spec.queueUrl() != null) { + this.batches = new Single(); + } else if (spec.dynamicDestination() != null) { + this.batches = new Dynamic(spec.dynamicDestination()); + } else { + throw new IllegalStateException("to(queueUrl) or to(dynamicDestination) is required"); + } + } + + private static CompletableFuture> sendMessageBatch( + SqsAsyncClient sqs, String queue, List records) { + SendMessageBatchRequest request = + SendMessageBatchRequest.builder().queueUrl(queue).entries(records).build(); + return sqs.sendMessageBatch(request).thenApply(resp -> resp.failed()); + } + + public void startBundle() { + handler.reset(); + if (scheduler != null && spec.strictTimeouts()) { + long timeout = spec.batchTimeout().getMillis(); + long period = timeout / CHECKS_PER_TIMEOUT_PERIOD; + expirationCheck = + scheduler.scheduleWithFixedDelay( + () -> batches.submitExpired(false), timeout, period, MILLISECONDS); + } + } + + public void process(T msg) { + SendMessageBatchRequestEntry entry = entryMapper.apply(batches.nextId(), msg); + Batch batch = batches.getLocked(msg); + batch.add(entry); + if (batch.size() >= spec.batchSize() || batch.isExpired()) { + submitEntries(batch, true); + } else { + checkState(batch.lock(false)); // unlock to continue writing to batch + } + + if (scheduler == null) { + // check for expired batches synchronously + batches.submitExpired(true); + } + } + + /** Submit entries of a {@link Batch} to the async write handler. */ + private void submitEntries(Batch batch, boolean throwFailures) { + try { + handler.batchWrite(batch.queue, batch.getAndClose(), throwFailures); + } catch (RuntimeException e) { + throw e; + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + public void finishBundle() throws Throwable { + if (expirationCheck != null) { + expirationCheck.cancel(false); + while (true) { + try { + expirationCheck.get(EXPIRATION_CHECK_TIMEOUT_SECS, TimeUnit.SECONDS); + } catch (TimeoutException e) { + LOG.warn("Waiting for timeout check to complete"); + } catch (CancellationException e) { + break; // scheduled checks completed after cancellation + } + } + } + // safe to write remaining batches without risking to encounter locked ones + checkState(batches.submitAll()); + handler.waitForCompletion(); + } + + @Override + public void close() throws Exception { + sqs.close(); + if (scheduler != null) { + scheduler.shutdown(); + } + } + + /** + * Batch(es) of a single fixed or several dynamic queues. + * + *

    A {@link Batch} can only ever be modified from the single runner thread. + * + *

    In case of strict timeouts, a batch may be submitted to the write handler by periodic + * expiration checks using a scheduler. Otherwise, and by default, this is done after + * appending to a batch. {@link Batch#lock(boolean)} prevents concurrent access to a batch + * between threads. Once a batch was locked by an expiration check, it must always be + * submitted to the write handler. + */ + @NotThreadSafe + private abstract class Batches { + private int nextId = 0; // only ever used from one "runner" thread + + abstract int maxBatches(); + + /** + * Next batch entry id is guaranteed to be unique for all open batches. + * + *

    This method is not thread-safe and may only ever be called from the single runner + * thread. + */ + String nextId() { + if (nextId >= (spec.batchSize() * maxBatches())) { + nextId = 0; + } + return Integer.toString(nextId++); + } + + /** + * Get an existing or new locked batch to append new messages. + * + *

    This method is not thread-safe and may only ever be called from a single runner + * thread. If this encounters a locked batch, it assumes the {@link Batch} is currently + * written to SQS and creates a new one. + */ + abstract Batch getLocked(T record); + + /** + * Submit all remaining batches (that can be locked) to the write handler. + * + * @return {@code true} if successful for all batches. + */ + abstract boolean submitAll(); + + /** + * Submit all expired batches (that can be locked) to the write handler. + * + *

    This is the only method that may be invoked from a thread other than the runner + * thread. + */ + abstract void submitExpired(boolean throwFailures); + + /** + * Submit a batch to the write handler if it can be locked. + * + * @return {@code true} if successful (or closed). + */ + protected boolean lockAndSubmit(Batch batch, boolean throwFailures) { + if (batch.isClosed()) { + return true; // nothing to submit + } else if (batch.lock(true)) { + submitEntries(batch, throwFailures); + return true; + } + return false; + } + } + + /** Batch of a single, fixed queue. */ + @NotThreadSafe + private class Single extends Batches { + private @Nullable Batch batch; + + @Override + int maxBatches() { + return 1; + } + + @Override + Batch getLocked(T record) { + if (batch == null || !batch.lock(true)) { + batch = Batch.createLocked(checkStateNotNull(spec.queueUrl()), spec); + } + return batch; + } + + @Override + boolean submitAll() { + return batch == null || lockAndSubmit(batch, true); + } + + @Override + void submitExpired(boolean throwFailures) { + if (batch != null && batch.isExpired()) { + lockAndSubmit(batch, throwFailures); + } + } + } + + /** Batches of one or several dynamic queues. */ + @NotThreadSafe + private class Dynamic extends Batches { + @SuppressWarnings("method.invocation") // necessary dependencies are initialized + private final BiFunction<@NonNull String, @Nullable Batch, Batch> getLocked = + (queue, batch) -> batch != null && batch.lock(true) ? batch : createLocked(queue); + + private final Map<@NonNull String, Batch> batches = new HashMap<>(); + private final AtomicBoolean submitExpiredRunning = new AtomicBoolean(false); + private final AtomicReference nextTimeout = new AtomicReference<>(Batch.NEVER); + private final DynamicDestination destination; + + Dynamic(DynamicDestination destination) { + this.destination = destination; + } + + @Override + int maxBatches() { + return batches.size() + 1; // next record and id might belong to new batch + } + + @Override + Batch getLocked(T record) { + return batches.compute(destination.queueUrl(record), getLocked); + } + + @Override + boolean submitAll() { + AtomicBoolean res = new AtomicBoolean(true); + batches.values().forEach(batch -> res.compareAndSet(true, lockAndSubmit(batch, true))); + batches.clear(); + nextTimeout.set(Batch.NEVER); + return res.get(); + } + + private void updateNextTimeout(Batch batch) { + Instant prev; + do { + prev = nextTimeout.get(); + } while (batch.expirationTime.isBefore(prev) + && !nextTimeout.compareAndSet(prev, batch.expirationTime)); + } + + private void submitExpired(Batch batch, boolean throwFailures) { + if (!batch.isClosed() && (!batch.isExpired() || !lockAndSubmit(batch, throwFailures))) { + updateNextTimeout(batch); + } + } + + @Override + void submitExpired(boolean throwFailures) { + Instant timeout = nextTimeout.get(); + if (timeout.isBeforeNow()) { + // prevent concurrent checks for expired batches + if (submitExpiredRunning.compareAndSet(false, true)) { + try { + nextTimeout.set(Batch.NEVER); + batches.values().forEach(b -> submitExpired(b, throwFailures)); + } catch (ConcurrentModificationException e) { + // Can happen rarely when adding a new dynamic destination and is expected. + // Reset old timeout to repeat check asap. + nextTimeout.set(timeout); + } finally { + submitExpiredRunning.set(false); + } + } + } + } + + Batch createLocked(String queue) { + Batch batch = Batch.createLocked(queue, spec); + updateNextTimeout(batch); + return batch; + } + } + } + + /** Batch of entries of a queue. */ + @NotThreadSafe + private abstract static class Batch { + private static final Instant NEVER = Instant.ofEpochMilli(Long.MAX_VALUE); + + private final String queue; + private final Instant expirationTime; + private List entries; + + static Batch createLocked(String queue, SqsIO.WriteBatches spec) { + return spec.strictTimeouts() + ? new BatchWithAtomicLock(queue, spec.batchSize(), spec.batchTimeout()) + : new BatchWithNoopLock(queue, spec.batchSize(), spec.batchTimeout()); + } + + /** A {@link Batch} with a noop lock that just rejects un/locking if closed. */ + private static class BatchWithNoopLock extends Batch { + BatchWithNoopLock(String queue, int size, Duration timeout) { + super(queue, size, timeout); + } + + @Override + boolean lock(boolean lock) { + return !isClosed(); // always un/lock unless closed + } + } + + /** A {@link Batch} supporting atomic locking for concurrent usage. */ + private static class BatchWithAtomicLock extends Batch { + private final AtomicBoolean locked = new AtomicBoolean(true); // always lock on creation + + BatchWithAtomicLock(String queue, int size, Duration timeout) { + super(queue, size, timeout); + } + + @Override + boolean lock(boolean lock) { + return !isClosed() && locked.compareAndSet(!lock, lock); + } + } + + private Batch(String queue, int size, Duration timeout) { + this.queue = queue; + this.entries = new ArrayList<>(size); + this.expirationTime = Instant.now().plus(timeout); + } + + /** Attempt to un/lock this batch, if closed this always fails. */ + abstract boolean lock(boolean lock); + + /** + * Get and clear entries for submission to the write handler. + * + *

    The batch must be locked and kept locked, it can't be modified anymore. + */ + List getAndClose() { + List res = entries; + entries = EMPTY_LIST; + return res; + } + + /** Append entry (only use if locked!). */ + void add(SendMessageBatchRequestEntry entry) { + entries.add(entry); + } + + int size() { + return entries.size(); + } + + boolean isExpired() { + return expirationTime.isBeforeNow(); + } + + boolean isClosed() { + return entries == EMPTY_LIST; + } } } } diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/sqs/SqsUnboundedReader.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/sqs/SqsUnboundedReader.java index 56a1be24115b7..595d71c074dd4 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/sqs/SqsUnboundedReader.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/sqs/SqsUnboundedReader.java @@ -26,9 +26,9 @@ import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toSet; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Collections2.transform; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Streams.mapWithIndex; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Collections2.transform; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Streams.mapWithIndex; import static software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName.SENT_TIMESTAMP; import static software.amazon.awssdk.services.sqs.model.QueueAttributeName.VISIBILITY_TIMEOUT; @@ -60,10 +60,10 @@ import org.apache.beam.sdk.util.BucketingFunction; import org.apache.beam.sdk.util.MovingFunction; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Streams.FunctionWithIndex; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Streams.FunctionWithIndex; import org.joda.time.Duration; import org.joda.time.Instant; import org.slf4j.Logger; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/ITEnvironment.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/ITEnvironment.java index 633aabad1678d..4497ededb3c9c 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/ITEnvironment.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/ITEnvironment.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.testing.TestPipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.commons.lang3.StringUtils; import org.junit.rules.ExternalResource; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/AsyncBatchWriteHandlerTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/AsyncBatchWriteHandlerTest.java new file mode 100644 index 0000000000000..7bf28c4f394a6 --- /dev/null +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/AsyncBatchWriteHandlerTest.java @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.aws2.common; + +import static java.util.Collections.emptyList; +import static java.util.concurrent.ForkJoinPool.commonPool; +import static java.util.function.Function.identity; +import static org.apache.beam.sdk.io.aws2.common.AsyncBatchWriteHandler.byId; +import static org.apache.beam.sdk.io.aws2.common.AsyncBatchWriteHandler.byPosition; +import static org.apache.beam.sdk.util.FluentBackoff.DEFAULT; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.joda.time.Duration.millis; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.Supplier; +import org.apache.beam.sdk.io.aws2.common.AsyncBatchWriteHandler.Stats; +import org.apache.beam.sdk.util.FluentBackoff; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.junit.Test; +import org.junit.function.ThrowingRunnable; +import org.mockito.Mockito; + +public class AsyncBatchWriteHandlerTest { + private static final int CONCURRENCY = 10; + + private CompletableFuture> resultsByPos = new CompletableFuture<>(); + private CompletableFuture> errorsById = new CompletableFuture<>(); + + private AsyncBatchWriteHandler byPositionHandler(FluentBackoff backoff) { + SubmitFn submitFn = Mockito.spy(new SubmitFn<>(() -> resultsByPos)); + Function errorFn = success -> success ? null : "REASON"; + return byPosition(CONCURRENCY, backoff, Stats.NONE, submitFn, errorFn); + } + + private AsyncBatchWriteHandler byIdHandler(FluentBackoff backoff) { + SubmitFn submitFn = Mockito.spy(new SubmitFn<>(() -> errorsById)); + Function errorFn = err -> "REASON"; + return byId(CONCURRENCY, backoff, Stats.NONE, submitFn, errorFn, identity(), identity()); + } + + @Test + public void retryOnPartialSuccessByPosition() throws Throwable { + AsyncBatchWriteHandler handler = + byPositionHandler(DEFAULT.withMaxBackoff(millis(1))); + CompletableFuture> pendingResponse1 = new CompletableFuture<>(); + CompletableFuture> pendingResponse2 = new CompletableFuture<>(); + CompletableFuture> pendingResponse3 = new CompletableFuture<>(); + + resultsByPos = pendingResponse1; + handler.batchWrite("destination", ImmutableList.of(1, 2, 3, 4)); + + // 1st attempt + eventually(5, () -> verify(handler.submitFn, times(1)).apply(anyString(), anyList())); + verify(handler.submitFn).apply("destination", ImmutableList.of(1, 2, 3, 4)); + assertThat(handler.requestsInProgress()).isEqualTo(1); + + resultsByPos = pendingResponse2; + pendingResponse1.complete(ImmutableList.of(true, true, false, false)); + + // 2nd attempt + eventually(5, () -> verify(handler.submitFn, times(2)).apply(anyString(), anyList())); + verify(handler.submitFn).apply("destination", ImmutableList.of(3, 4)); + assertThat(handler.requestsInProgress()).isEqualTo(1); + + // 3rd attempt + resultsByPos = pendingResponse3; + pendingResponse2.complete(ImmutableList.of(true, false)); + + eventually(5, () -> verify(handler.submitFn, times(3)).apply(anyString(), anyList())); + verify(handler.submitFn).apply("destination", ImmutableList.of(4)); + + assertThat(handler.requestsInProgress()).isEqualTo(1); + + // 4th attempt + pendingResponse3.complete(ImmutableList.of(true)); // success + + eventually(5, () -> assertThat(handler.requestsInProgress()).isEqualTo(0)); + verify(handler.submitFn, times(3)).apply(anyString(), anyList()); + } + + @Test + public void retryOnPartialSuccessById() throws Throwable { + AsyncBatchWriteHandler handler = byIdHandler(DEFAULT.withMaxBackoff(millis(1))); + CompletableFuture> pendingResponse1 = new CompletableFuture<>(); + CompletableFuture> pendingResponse2 = new CompletableFuture<>(); + CompletableFuture> pendingResponse3 = new CompletableFuture<>(); + + errorsById = pendingResponse1; + handler.batchWrite("destination", ImmutableList.of("1", "2", "3", "4")); + + // 1st attempt + eventually(5, () -> verify(handler.submitFn, times(1)).apply(anyString(), anyList())); + verify(handler.submitFn).apply("destination", ImmutableList.of("1", "2", "3", "4")); + assertThat(handler.requestsInProgress()).isEqualTo(1); + + errorsById = pendingResponse2; + pendingResponse1.complete(ImmutableList.of("3", "4")); + + // 2nd attempt + eventually(5, () -> verify(handler.submitFn, times(2)).apply(anyString(), anyList())); + verify(handler.submitFn).apply("destination", ImmutableList.of("3", "4")); + assertThat(handler.requestsInProgress()).isEqualTo(1); + + // 3rd attempt + errorsById = pendingResponse3; + pendingResponse2.complete(ImmutableList.of("4")); + + eventually(5, () -> verify(handler.submitFn, times(3)).apply(anyString(), anyList())); + verify(handler.submitFn).apply("destination", ImmutableList.of("4")); + + assertThat(handler.requestsInProgress()).isEqualTo(1); + + // 4th attempt + pendingResponse3.complete(ImmutableList.of()); // success + + eventually(5, () -> assertThat(handler.requestsInProgress()).isEqualTo(0)); + verify(handler.submitFn, times(3)).apply(anyString(), anyList()); + } + + @Test + public void retryLimitOnPartialSuccessByPosition() throws Throwable { + AsyncBatchWriteHandler handler = byPositionHandler(DEFAULT.withMaxRetries(0)); + + handler.batchWrite("destination", ImmutableList.of(1, 2, 3, 4)); + + resultsByPos.complete(ImmutableList.of(true, true, false, false)); + + assertThatThrownBy(() -> handler.waitForCompletion()) + .hasMessageContaining("Exceeded retries") + .hasMessageEndingWith("REASON for 2 record(s).") + .isInstanceOf(IOException.class); + verify(handler.submitFn).apply("destination", ImmutableList.of(1, 2, 3, 4)); + } + + @Test + public void retryLimitOnPartialSuccessById() throws Throwable { + AsyncBatchWriteHandler handler = byIdHandler(DEFAULT.withMaxRetries(0)); + + handler.batchWrite("destination", ImmutableList.of("1", "2", "3", "4")); + + errorsById.complete(ImmutableList.of("3", "4")); + + assertThatThrownBy(() -> handler.waitForCompletion()) + .hasMessageContaining("Exceeded retries") + .hasMessageEndingWith("REASON for 2 record(s).") + .isInstanceOf(IOException.class); + verify(handler.submitFn).apply("destination", ImmutableList.of("1", "2", "3", "4")); + } + + @Test + public void propagateErrorOnPutRecords() throws Throwable { + AsyncBatchWriteHandler handler = byPositionHandler(DEFAULT); + handler.batchWrite("destination", emptyList()); + resultsByPos.completeExceptionally(new RuntimeException("Request failed")); + + assertThatThrownBy(() -> handler.batchWrite("destination", emptyList())) + .hasMessage("Request failed"); + assertThat(handler.hasErrored()).isTrue(); + verify(handler.submitFn).apply("destination", emptyList()); + } + + @Test + public void propagateErrorWhenPolling() throws Throwable { + AsyncBatchWriteHandler handler = byPositionHandler(DEFAULT); + handler.batchWrite("destination", emptyList()); + handler.checkForAsyncFailure(); // none yet + resultsByPos.completeExceptionally(new RuntimeException("Request failed")); + + assertThatThrownBy(() -> handler.checkForAsyncFailure()).hasMessage("Request failed"); + assertThat(handler.hasErrored()).isTrue(); + handler.checkForAsyncFailure(); // already reset + } + + @Test + public void propagateErrorOnWaitForCompletion() throws Throwable { + AsyncBatchWriteHandler handler = byPositionHandler(DEFAULT); + handler.batchWrite("destination", emptyList()); + resultsByPos.completeExceptionally(new RuntimeException("Request failed")); + + assertThatThrownBy(() -> handler.waitForCompletion()).hasMessage("Request failed"); + } + + @Test + public void correctlyLimitConcurrency() throws Throwable { + AsyncBatchWriteHandler handler = byPositionHandler(DEFAULT); + + // exhaust concurrency limit so that batchWrite blocks + Runnable task = repeat(CONCURRENCY + 1, () -> handler.batchWrite("destination", emptyList())); + Future future = commonPool().submit(task); + + eventually(5, () -> assertThat(handler.requestsInProgress()).isEqualTo(CONCURRENCY)); + eventually( + 5, () -> verify(handler.submitFn, times(CONCURRENCY)).apply("destination", emptyList())); + assertThat(future).isNotDone(); + + // complete responses and unblock last request + resultsByPos.complete(emptyList()); + + eventually( + 5, + () -> verify(handler.submitFn, times(CONCURRENCY + 1)).apply("destination", emptyList())); + handler.waitForCompletion(); + assertThat(future).isDone(); + } + + static class SubmitFn implements BiFunction, CompletableFuture>> { + private final Supplier>> resp; + + SubmitFn(Supplier>> resp) { + this.resp = resp; + } + + @Override + public CompletableFuture> apply(String destination, List input) { + return resp.get(); + } + } + + private void eventually(int attempts, Runnable fun) { + for (int i = 0; i < attempts - 1; i++) { + try { + Thread.sleep(i * 100); + fun.run(); + return; + } catch (AssertionError | InterruptedException t) { + } + } + fun.run(); + } + + private Runnable repeat(int times, ThrowingRunnable fun) { + return () -> { + for (int i = 0; i < times; i++) { + try { + fun.run(); + } catch (Throwable t) { + throw new RuntimeException(t); + } + } + }; + } +} diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/dynamodb/AttributeValueCoderTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/dynamodb/AttributeValueCoderTest.java index c0bc4e52d9c4d..89a76baf44005 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/dynamodb/AttributeValueCoderTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/dynamodb/AttributeValueCoderTest.java @@ -25,7 +25,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Assert; import org.junit.Test; import software.amazon.awssdk.core.SdkBytes; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/dynamodb/DynamoDBIOReadTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/dynamodb/DynamoDBIOReadTest.java index 2abbca2cedb13..6a7fb85e866ba 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/dynamodb/DynamoDBIOReadTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/dynamodb/DynamoDBIOReadTest.java @@ -20,9 +20,9 @@ import static java.lang.Math.min; import static java.util.stream.Collectors.toList; import static java.util.stream.IntStream.range; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.getLast; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.transform; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.getLast; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.transform; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -39,7 +39,7 @@ import org.apache.beam.sdk.transforms.Count; import org.apache.beam.sdk.transforms.Flatten; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/dynamodb/DynamoDBIOWriteTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/dynamodb/DynamoDBIOWriteTest.java index 10959341499ee..041615685c69f 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/dynamodb/DynamoDBIOWriteTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/dynamodb/DynamoDBIOWriteTest.java @@ -20,8 +20,8 @@ import static java.util.stream.Collectors.toList; import static java.util.stream.IntStream.range; import static java.util.stream.IntStream.rangeClosed; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps.filterKeys; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps.transformValues; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps.filterKeys; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps.transformValues; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; @@ -52,8 +52,8 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/dynamodb/testing/DynamoDBIOIT.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/dynamodb/testing/DynamoDBIOIT.java index 518086fc4cff8..9ba35044c3f25 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/dynamodb/testing/DynamoDBIOIT.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/dynamodb/testing/DynamoDBIOIT.java @@ -39,7 +39,7 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/AsyncPutRecordsHandlerTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/AsyncPutRecordsHandlerTest.java deleted file mode 100644 index 6f0b92654de4f..0000000000000 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/AsyncPutRecordsHandlerTest.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io.aws2.kinesis; - -import static java.util.Collections.emptyList; -import static java.util.concurrent.ForkJoinPool.commonPool; -import static org.apache.beam.sdk.io.common.TestRow.getExpectedValues; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Future; -import java.util.function.Supplier; -import org.apache.beam.sdk.util.BackOff; -import org.junit.Before; -import org.junit.Test; -import org.junit.function.ThrowingRunnable; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; -import software.amazon.awssdk.services.kinesis.KinesisAsyncClient; -import software.amazon.awssdk.services.kinesis.model.PutRecordsRequest; -import software.amazon.awssdk.services.kinesis.model.PutRecordsRequestEntry; -import software.amazon.awssdk.services.kinesis.model.PutRecordsResponse; - -@RunWith(MockitoJUnitRunner.StrictStubs.class) -public class AsyncPutRecordsHandlerTest extends PutRecordsHelpers { - private static final String STREAM = "streamName"; - private static final int CONCURRENCY = 10; - - private CompletableFuture pendingResponse = new CompletableFuture<>(); - - @Mock private KinesisAsyncClient client; - @Mock private Supplier backoff; - private AsyncPutRecordsHandler handler; - - @Before - public void init() { - handler = - new AsyncPutRecordsHandler( - client, CONCURRENCY, backoff, mock(AsyncPutRecordsHandler.Stats.class)); - when(client.putRecords(anyRequest())).thenReturn(pendingResponse); - } - - @Test - public void retryOnPartialSuccess() throws Throwable { - when(backoff.get()).thenReturn(BackOff.ZERO_BACKOFF); - CompletableFuture pendingResponse2 = new CompletableFuture<>(); - CompletableFuture pendingResponse3 = new CompletableFuture<>(); - when(client.putRecords(anyRequest())) - .thenReturn(pendingResponse, pendingResponse2, pendingResponse3); - - List records = fromTestRows(getExpectedValues(0, 100)); - handler.putRecords(STREAM, records); - - // 1st attempt - eventually(5, () -> verify(client, times(1)).putRecords(anyRequest())); - verify(client).putRecords(request(records)); - assertThat(handler.pendingRequests()).isEqualTo(1); - - // 2nd attempt - pendingResponse.complete(partialSuccessResponse(50, 50)); - eventually(5, () -> verify(client, times(2)).putRecords(anyRequest())); - verify(client).putRecords(request(records.subList(50, 100))); - assertThat(handler.pendingRequests()).isEqualTo(1); - - // 3rd attempt - pendingResponse2.complete(partialSuccessResponse(25, 25)); - eventually(5, () -> verify(client, times(3)).putRecords(anyRequest())); - verify(client).putRecords(request(records.subList(75, 100))); - assertThat(handler.pendingRequests()).isEqualTo(1); - - // 4th attempt - pendingResponse3.complete(PutRecordsResponse.builder().build()); // success - verifyNoMoreInteractions(client); - - eventually(5, () -> assertThat(handler.pendingRequests()).isEqualTo(0)); - } - - @Test - public void retryLimitOnPartialSuccess() throws Throwable { - when(backoff.get()).thenReturn(BackOff.STOP_BACKOFF); - - List records = fromTestRows(getExpectedValues(0, 100)); - handler.putRecords(STREAM, records); - - pendingResponse.complete(partialSuccessResponse(98, 2)); - - assertThatThrownBy(() -> handler.waitForCompletion()) - .hasMessageContaining("Exceeded retries") - .hasMessageEndingWith(ERROR_CODE + " for 2 record(s).") - .isInstanceOf(IOException.class); - verify(client).putRecords(anyRequest()); - } - - @Test - public void propagateErrorOnPutRecords() throws Throwable { - handler.putRecords(STREAM, emptyList()); - pendingResponse.completeExceptionally(new RuntimeException("Request failed")); - - assertThatThrownBy(() -> handler.putRecords(STREAM, emptyList())).hasMessage("Request failed"); - assertThat(handler.hasErrored()).isTrue(); - verify(client).putRecords(anyRequest()); - } - - @Test - public void propagateErrorWhenPolling() throws Throwable { - handler.putRecords(STREAM, emptyList()); - handler.checkForAsyncFailure(); // none yet - pendingResponse.completeExceptionally(new RuntimeException("Request failed")); - - assertThatThrownBy(() -> handler.checkForAsyncFailure()).hasMessage("Request failed"); - assertThat(handler.hasErrored()).isTrue(); - handler.checkForAsyncFailure(); // already reset - } - - @Test - public void propagateErrorOnWaitForCompletion() throws Throwable { - handler.putRecords(STREAM, emptyList()); - pendingResponse.completeExceptionally(new RuntimeException("Request failed")); - - assertThatThrownBy(() -> handler.waitForCompletion()).hasMessage("Request failed"); - } - - @Test - public void correctlyLimitConcurrency() throws Throwable { - // exhaust concurrency limit so that putRecords blocks - Runnable task = repeat(CONCURRENCY + 1, () -> handler.putRecords(STREAM, emptyList())); - Future future = commonPool().submit(task); - - eventually(5, () -> assertThat(handler.pendingRequests()).isEqualTo(CONCURRENCY)); - eventually(5, () -> verify(client, times(CONCURRENCY)).putRecords(anyRequest())); - assertThat(future).isNotDone(); - - // complete responses and unblock last request - pendingResponse.complete(PutRecordsResponse.builder().build()); - - eventually(5, () -> verify(client, times(CONCURRENCY + 1)).putRecords(anyRequest())); - handler.waitForCompletion(); - assertThat(future).isDone(); - } - - private PutRecordsRequest request(List records) { - return PutRecordsRequest.builder().streamName(STREAM).records(records).build(); - } - - private void eventually(int attempts, Runnable fun) { - for (int i = 0; i < attempts - 1; i++) { - try { - Thread.sleep(i * 100); - fun.run(); - return; - } catch (AssertionError | InterruptedException t) { - } - } - fun.run(); - } - - private Runnable repeat(int times, ThrowingRunnable fun) { - return () -> { - for (int i = 0; i < times; i++) { - try { - fun.run(); - } catch (Throwable t) { - throw new RuntimeException(t); - } - } - }; - } -} diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/EFOShardSubscribersPoolTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/EFOShardSubscribersPoolTest.java index 6f290fab9fbb2..d99a35a7dda3b 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/EFOShardSubscribersPoolTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/EFOShardSubscribersPoolTest.java @@ -41,7 +41,7 @@ import java.util.Objects; import java.util.concurrent.CompletionException; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.After; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/EFOStubbedKinesisAsyncClient.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/EFOStubbedKinesisAsyncClient.java index 55c63d4bf1088..f49d9a1203748 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/EFOStubbedKinesisAsyncClient.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/EFOStubbedKinesisAsyncClient.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.aws2.kinesis; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.ArrayDeque; import java.util.ArrayList; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisIOReadTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisIOReadTest.java index 02d28eaad586d..1b740fa52f06b 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisIOReadTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisIOReadTest.java @@ -25,7 +25,7 @@ import static org.apache.beam.sdk.io.aws2.kinesis.TestHelpers.mockShardIterators; import static org.apache.beam.sdk.io.aws2.kinesis.TestHelpers.mockShards; import static org.apache.beam.sdk.io.aws2.kinesis.TestHelpers.record; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.concat; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.concat; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; @@ -43,7 +43,7 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Before; @@ -111,12 +111,13 @@ public void testReadFromShards() { @Test public void testReadWithEFOFromShards() { SubscribeToShardEvent shard0event = eventWithRecords(3); - SubscribeToShardEvent shard1event = eventWithRecords(3); - SubscribeToShardEvent shard2event = eventWithRecords(3); + SubscribeToShardEvent shard1event = eventWithRecords(4); + SubscribeToShardEvent shard2event = eventWithRecords(5); EFOStubbedKinesisAsyncClient asyncClientStub = new EFOStubbedKinesisAsyncClient(10); asyncClientStub.stubSubscribeToShard("0", shard0event); asyncClientStub.stubSubscribeToShard("1", shard1event); - asyncClientStub.stubSubscribeToShard("2", shard1event); + asyncClientStub.stubSubscribeToShard("2", shard2event); + MockClientBuilderFactory.set(p, KinesisAsyncClientBuilder.class, asyncClientStub); Iterable expectedRecords = concat(shard0event.records(), shard1event.records(), shard2event.records()); @@ -128,7 +129,7 @@ public void testReadWithEFOFromShards() { .withConsumerArn("consumer") .withInitialPositionInStream(TRIM_HORIZON) .withArrivalTimeWatermarkPolicy() - .withMaxNumRecords(9); + .withMaxNumRecords(12); PCollection result = p.apply(read).apply(ParDo.of(new KinesisIOReadTest.ToRecord())); PAssert.that(result).containsInAnyOrder(expectedRecords); diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisIOWriteTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisIOWriteTest.java index 08ccd724ab662..61020ff571c36 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisIOWriteTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisIOWriteTest.java @@ -30,9 +30,9 @@ import static org.apache.beam.sdk.io.aws2.kinesis.KinesisPartitioner.MIN_HASH_KEY; import static org.apache.beam.sdk.io.aws2.kinesis.KinesisPartitioner.explicitRandomPartitioner; import static org.apache.beam.sdk.io.common.TestRow.getExpectedValues; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.concat; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.transform; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.concat; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.transform; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.joda.time.Duration.ZERO; @@ -69,7 +69,7 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.assertj.core.api.ThrowableAssert; import org.joda.time.DateTimeUtils; import org.joda.time.Instant; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisReaderCheckpointTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisReaderCheckpointTest.java index 309141c1d510c..7e01cc6cd7c6c 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisReaderCheckpointTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisReaderCheckpointTest.java @@ -29,18 +29,16 @@ import java.util.Base64; import java.util.Iterator; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Test; import software.amazon.awssdk.services.kinesis.model.ShardIteratorType; public class KinesisReaderCheckpointTest { - /** - * This was generated with Beam master branch. - * https://github.com/apache/beam/commit/ca0787642a6b3804a742326147281c99ae8d08d2 - */ + + /** This was generated manually. */ private static final String OLDER_VERSION_SERIALIZED_CHECKPOINT = - "rO0ABXNyADtvcmcuYXBhY2hlLmJlYW0uc2RrLmlvLmF3czIua2luZXNpcy5LaW5lc2lzUmVhZGVyQ2hlY2twb2ludKHLb3bO/6XJAgABTAAQc2hhcmRDaGVja3BvaW50c3QAEExqYXZhL3V0aWwvTGlzdDt4cHNyAF1vcmcuYXBhY2hlLmJlYW0udmVuZG9yLmd1YXZhLnYyNl8wX2pyZS5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZUxpc3QkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAVsACGVsZW1lbnRzdAATW0xqYXZhL2xhbmcvT2JqZWN0O3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNyADNvcmcuYXBhY2hlLmJlYW0uc2RrLmlvLmF3czIua2luZXNpcy5TaGFyZENoZWNrcG9pbnQBb9XvUdq1BwIABkwADnNlcXVlbmNlTnVtYmVydAASTGphdmEvbGFuZy9TdHJpbmc7TAAHc2hhcmRJZHEAfgAJTAARc2hhcmRJdGVyYXRvclR5cGV0AEFMc29mdHdhcmUvYW1hem9uL2F3c3Nkay9zZXJ2aWNlcy9raW5lc2lzL21vZGVsL1NoYXJkSXRlcmF0b3JUeXBlO0wACnN0cmVhbU5hbWVxAH4ACUwAEXN1YlNlcXVlbmNlTnVtYmVydAAQTGphdmEvbGFuZy9Mb25nO0wACXRpbWVzdGFtcHQAF0xvcmcvam9kYS90aW1lL0luc3RhbnQ7eHB0AAI0MnQACXNoYXJkLTAwMH5yAD9zb2Z0d2FyZS5hbWF6b24uYXdzc2RrLnNlcnZpY2VzLmtpbmVzaXMubW9kZWwuU2hhcmRJdGVyYXRvclR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0ABVBRlRFUl9TRVFVRU5DRV9OVU1CRVJ0AAlzdHJlYW0tMDFzcgAOamF2YS5sYW5nLkxvbmc7i+SQzI8j3wIAAUoABXZhbHVleHIAEGphdmEubGFuZy5OdW1iZXKGrJUdC5TgiwIAAHhwAAAAAAAAAAxw"; + "rO0ABXNyADtvcmcuYXBhY2hlLmJlYW0uc2RrLmlvLmF3czIua2luZXNpcy5LaW5lc2lzUmVhZGVyQ2hlY2twb2ludKHLb3bO/6XJAgABTAAQc2hhcmRDaGVja3BvaW50c3QAEExqYXZhL3V0aWwvTGlzdDt4cHNyAF9vcmcuYXBhY2hlLmJlYW0udmVuZG9yLmd1YXZhLnYzMl8xXzJfanJlLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTGlzdCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgABWwAIZWxlbWVudHN0ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAABc3IAM29yZy5hcGFjaGUuYmVhbS5zZGsuaW8uYXdzMi5raW5lc2lzLlNoYXJkQ2hlY2twb2ludAFv1e9R2rUHAgAGTAAOc2VxdWVuY2VOdW1iZXJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAdzaGFyZElkcQB+AAlMABFzaGFyZEl0ZXJhdG9yVHlwZXQAQUxzb2Z0d2FyZS9hbWF6b24vYXdzc2RrL3NlcnZpY2VzL2tpbmVzaXMvbW9kZWwvU2hhcmRJdGVyYXRvclR5cGU7TAAKc3RyZWFtTmFtZXEAfgAJTAARc3ViU2VxdWVuY2VOdW1iZXJ0ABBMamF2YS9sYW5nL0xvbmc7TAAJdGltZXN0YW1wdAAXTG9yZy9qb2RhL3RpbWUvSW5zdGFudDt4cHQAAjQydAAJc2hhcmQtMDAwfnIAP3NvZnR3YXJlLmFtYXpvbi5hd3NzZGsuc2VydmljZXMua2luZXNpcy5tb2RlbC5TaGFyZEl0ZXJhdG9yVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAFUFGVEVSX1NFUVVFTkNFX05VTUJFUnQACXN0cmVhbS0wMXNyAA5qYXZhLmxhbmcuTG9uZzuL5JDMjyPfAgABSgAFdmFsdWV4cgAQamF2YS5sYW5nLk51bWJlcoaslR0LlOCLAgAAeHAAAAAAAAAADHA="; private ShardCheckpoint a = new ShardCheckpoint( @@ -79,13 +77,21 @@ public void testJavaSerialization() throws IOException, ClassNotFoundException { String serializedCheckpoint = serializeObjectToString(checkpoint); KinesisReaderCheckpoint deserializedCheckpoint = (KinesisReaderCheckpoint) deSerializeObjectFromString(serializedCheckpoint); - - KinesisReaderCheckpoint olderVersionDeserializedCheckpoint = - (KinesisReaderCheckpoint) deSerializeObjectFromString(OLDER_VERSION_SERIALIZED_CHECKPOINT); - assertThat(checkpoint).containsExactlyInAnyOrder(shardCheckpoint); assertThat(deserializedCheckpoint).containsExactlyInAnyOrder(shardCheckpoint); - assertThat(olderVersionDeserializedCheckpoint).containsExactlyInAnyOrder(shardCheckpoint); + + try { + KinesisReaderCheckpoint olderVersionDeserializedCheckpoint = + (KinesisReaderCheckpoint) + deSerializeObjectFromString(OLDER_VERSION_SERIALIZED_CHECKPOINT); + assertThat(olderVersionDeserializedCheckpoint).containsExactlyInAnyOrder(shardCheckpoint); + } catch (ClassNotFoundException e) { + String errorMessage = + String.format( + "KinesisReaderCheckpoint may have changed. Consider update OLDER_VERSION_SERIALIZED_CHECKPOINT: \"%s\"", + serializedCheckpoint); + throw new RuntimeException(errorMessage, e); + } } private void verifySplitInto(KinesisReaderCheckpoint checkpoint, int size) { diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisSourceTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisSourceTest.java index 7eed152104547..1a50712d7631e 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisSourceTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisSourceTest.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.io.aws2.options.AwsOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/PutRecordsHelpers.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/PutRecordsHelpers.java index ae178118d51bf..59f8c3a7edb43 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/PutRecordsHelpers.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/PutRecordsHelpers.java @@ -19,12 +19,12 @@ import static java.util.Arrays.asList; import static java.util.stream.Collectors.toList; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.concat; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.cycle; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.limit; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.transform; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Streams.stream; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.concat; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.cycle; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.limit; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.transform; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Streams.stream; import static org.mockito.ArgumentMatchers.any; import java.nio.ByteBuffer; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/RateLimitPolicyFactoryTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/RateLimitPolicyFactoryTest.java index 3aefa85602083..98981e14160ec 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/RateLimitPolicyFactoryTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/RateLimitPolicyFactoryTest.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.io.aws2.kinesis.RateLimitPolicyFactory.DefaultRateLimiter; import org.apache.beam.sdk.util.BackOff; import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; public class RateLimitPolicyFactoryTest { diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/RecordFilterTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/RecordFilterTest.java index 05304bc476d24..faa31323b53e6 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/RecordFilterTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/RecordFilterTest.java @@ -21,7 +21,7 @@ import java.util.Collections; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.assertj.core.api.Assertions; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/RecordsAggregatorTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/RecordsAggregatorTest.java index 52da1036ed2c7..c9a317f31112a 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/RecordsAggregatorTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/RecordsAggregatorTest.java @@ -31,7 +31,7 @@ import java.util.Random; import java.util.stream.LongStream; import java.util.stream.Stream; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; import org.junit.Test; import software.amazon.awssdk.services.kinesis.model.PutRecordsRequestEntry; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/ShardReadersPoolExtendedTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/ShardReadersPoolExtendedTest.java index cd30092e7c011..78ca0cc9aa547 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/ShardReadersPoolExtendedTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/ShardReadersPoolExtendedTest.java @@ -28,7 +28,7 @@ import java.util.List; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; import org.junit.After; import org.junit.Before; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/ShardReadersPoolTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/ShardReadersPoolTest.java index ef083eb56d02e..3d2a7880b68de 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/ShardReadersPoolTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/ShardReadersPoolTest.java @@ -33,8 +33,8 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Stopwatch; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Stopwatch; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.After; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/TestHelpers.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/TestHelpers.java index 97565f16e5054..485f2def20e21 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/TestHelpers.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/TestHelpers.java @@ -32,7 +32,7 @@ import java.util.function.IntFunction; import java.util.stream.Collectors; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.DateTime; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/TimeUtilTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/TimeUtilTest.java index 43a6f6ffe98b9..e6bcca1aff627 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/TimeUtilTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/TimeUtilTest.java @@ -21,7 +21,7 @@ import static org.assertj.core.api.Assertions.assertThat; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/s3/MatchResultMatcher.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/s3/MatchResultMatcher.java index d85a0289f0bb7..bdc21fcb4c199 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/s3/MatchResultMatcher.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/s3/MatchResultMatcher.java @@ -17,14 +17,14 @@ */ package org.apache.beam.sdk.io.aws2.s3; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.util.List; import org.apache.beam.sdk.io.fs.MatchResult; import org.apache.beam.sdk.io.fs.ResourceId; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.hamcrest.Matcher; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/s3/S3FileSystemTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/s3/S3FileSystemTest.java index 309823ec98460..423176e52a75f 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/s3/S3FileSystemTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/s3/S3FileSystemTest.java @@ -25,7 +25,7 @@ import static org.apache.beam.sdk.io.aws2.s3.S3TestUtils.s3OptionsWithPathStyleAccessEnabled; import static org.apache.beam.sdk.io.aws2.s3.S3TestUtils.s3OptionsWithSSECustomerKey; import static org.apache.beam.sdk.io.aws2.s3.S3TestUtils.toMd5; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.junit.Assert.assertArrayEquals; @@ -55,7 +55,7 @@ import org.apache.beam.sdk.io.aws2.options.S3Options; import org.apache.beam.sdk.io.fs.CreateOptions; import org.apache.beam.sdk.io.fs.MatchResult; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/schemas/AwsSchemaProviderTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/schemas/AwsSchemaProviderTest.java index 71767c915fe79..98a587c7b24e4 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/schemas/AwsSchemaProviderTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/schemas/AwsSchemaProviderTest.java @@ -39,8 +39,8 @@ import org.apache.beam.sdk.schemas.utils.SchemaTestUtils; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.assertj.core.api.Condition; import org.assertj.core.api.SoftAssertions; import org.junit.Test; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/schemas/Sample.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/schemas/Sample.java index cb8cf16f63cd3..0a371a0f6cf08 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/schemas/Sample.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/schemas/Sample.java @@ -40,7 +40,7 @@ import java.util.function.Function; import org.apache.beam.repackaged.core.org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.beam.repackaged.core.org.apache.commons.lang3.builder.HashCodeBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.commons.lang3.builder.ReflectionToStringBuilder; import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.core.SdkField; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sns/SnsIOTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sns/SnsIOTest.java index 59fde7f532bcf..fc6aaee4a06aa 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sns/SnsIOTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sns/SnsIOTest.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/SqsIOReadTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/SqsIOReadTest.java index 1983553c8e5b2..9c033f24767a1 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/SqsIOReadTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/SqsIOReadTest.java @@ -34,8 +34,8 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/SqsIOWriteBatchesTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/SqsIOWriteBatchesTest.java new file mode 100644 index 0000000000000..e92720bfb5a5c --- /dev/null +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/SqsIOWriteBatchesTest.java @@ -0,0 +1,460 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.aws2.sqs; + +import static java.lang.Math.sqrt; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.CompletableFuture.completedFuture; +import static java.util.concurrent.CompletableFuture.supplyAsync; +import static java.util.stream.Collectors.toList; +import static java.util.stream.IntStream.range; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.joda.time.Duration.millis; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.io.aws2.MockClientBuilderFactory; +import org.apache.beam.sdk.io.aws2.common.AsyncBatchWriteHandler; +import org.apache.beam.sdk.io.aws2.common.ClientConfiguration; +import org.apache.beam.sdk.io.aws2.common.RetryConfiguration; +import org.apache.beam.sdk.io.aws2.sqs.SqsIO.WriteBatches; +import org.apache.beam.sdk.io.aws2.sqs.SqsIO.WriteBatches.DynamicDestination; +import org.apache.beam.sdk.io.aws2.sqs.SqsIO.WriteBatches.EntryMapperFn; +import org.apache.beam.sdk.schemas.SchemaRegistry; +import org.apache.beam.sdk.testing.ExpectedLogs; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Streams; +import org.apache.commons.lang3.RandomUtils; +import org.joda.time.Duration; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.services.sqs.SqsAsyncClient; +import software.amazon.awssdk.services.sqs.SqsAsyncClientBuilder; +import software.amazon.awssdk.services.sqs.model.BatchResultErrorEntry; +import software.amazon.awssdk.services.sqs.model.MessageAttributeValue; +import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeValue; +import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequest; +import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequestEntry; +import software.amazon.awssdk.services.sqs.model.SendMessageBatchResponse; +import software.amazon.awssdk.services.sqs.model.SendMessageRequest; + +/** Tests for {@link WriteBatches}. */ +@RunWith(MockitoJUnitRunner.class) +public class SqsIOWriteBatchesTest { + private static final EntryMapperFn.Builder SET_MESSAGE_BODY = + SendMessageBatchRequestEntry.Builder::messageBody; + private static final SendMessageBatchResponse SUCCESS = + SendMessageBatchResponse.builder().build(); + + @Rule public TestPipeline p = TestPipeline.create(); + @Mock public SqsAsyncClient sqs; + @Rule public ExpectedLogs logs = ExpectedLogs.none(AsyncBatchWriteHandler.class); + + @Before + public void configureClientBuilderFactory() { + MockClientBuilderFactory.set(p, SqsAsyncClientBuilder.class, sqs); + } + + @Test + public void testSchemaEntryMapper() throws Exception { + SchemaRegistry registry = p.getSchemaRegistry(); + + Map attributes = + ImmutableMap.of("key", MessageAttributeValue.builder().stringValue("value").build()); + Map systemAttributes = + ImmutableMap.of( + "key", + MessageSystemAttributeValue.builder() + .binaryValue(SdkBytes.fromString("bytes", UTF_8)) + .build()); + + SendMessageRequest input = + SendMessageRequest.builder() + .messageBody("body") + .delaySeconds(3) + .messageAttributes(attributes) + .messageSystemAttributesWithStrings(systemAttributes) + .build(); + + SqsIO.WriteBatches.EntryMapperFn mapper = + new SqsIO.WriteBatches.SchemaEntryMapper<>( + registry.getSchema(SendMessageRequest.class), + registry.getSchema(SendMessageBatchRequestEntry.class), + registry.getToRowFunction(SendMessageRequest.class), + registry.getFromRowFunction(SendMessageBatchRequestEntry.class)); + + assertThat(mapper.apply("1", input)) + .isEqualTo( + SendMessageBatchRequestEntry.builder() + .id("1") + .messageBody("body") + .delaySeconds(3) + .messageAttributes(attributes) + .messageSystemAttributesWithStrings(systemAttributes) + .build()); + } + + @Test + public void testWrite() { + // write uses writeBatches with batch size 1 + when(sqs.sendMessageBatch(anyRequest())).thenReturn(completedFuture(SUCCESS)); + + SendMessageRequest.Builder msgBuilder = SendMessageRequest.builder().queueUrl("queue"); + Set messages = + range(0, 100) + .mapToObj(i -> msgBuilder.messageBody("test" + i).build()) + .collect(Collectors.toSet()); + + p.apply(Create.of(messages)).apply(SqsIO.write()); + p.run().waitUntilFinish(); + + ArgumentCaptor captor = + ArgumentCaptor.forClass(SendMessageBatchRequest.class); + verify(sqs, times(100)).sendMessageBatch(captor.capture()); + + for (SendMessageBatchRequest req : captor.getAllValues()) { + assertThat(req.queueUrl()).isEqualTo("queue"); + assertThat(req.entries()).hasSize(1); + for (SendMessageBatchRequestEntry entry : req.entries()) { + assertTrue(messages.remove(msgBuilder.messageBody(entry.messageBody()).build())); + } + } + assertTrue(messages.isEmpty()); + } + + @Test + public void testWriteBatches() { + when(sqs.sendMessageBatch(anyRequest())).thenReturn(completedFuture(SUCCESS)); + + p.apply(Create.of(23)) + .apply(ParDo.of(new CreateMessages())) + .apply(SqsIO.writeBatches().withEntryMapper(SET_MESSAGE_BODY).to("queue")); + + p.run().waitUntilFinish(); + + verify(sqs).sendMessageBatch(request("queue", range(0, 10))); + verify(sqs).sendMessageBatch(request("queue", range(10, 20))); + verify(sqs).sendMessageBatch(request("queue", range(20, 23))); + + verify(sqs).close(); + verifyNoMoreInteractions(sqs); + } + + @Test + public void testWriteBatchesFailure() { + when(sqs.sendMessageBatch(anyRequest())) + .thenReturn( + completedFuture(SUCCESS), + supplyAsync(() -> checkNotNull(null, "sendMessageBatch failed")), + completedFuture(SUCCESS)); + + p.apply(Create.of(23)) + .apply(ParDo.of(new CreateMessages())) + .apply(SqsIO.writeBatches().withEntryMapper(SET_MESSAGE_BODY).to("queue")); + + assertThatThrownBy(() -> p.run().waitUntilFinish()) + .isInstanceOf(Pipeline.PipelineExecutionException.class) + .hasMessageContaining("sendMessageBatch failed"); + } + + @Test + public void testWriteBatchesPartialSuccess() { + SendMessageBatchRequestEntry[] entries = entries(range(0, 10)); + when(sqs.sendMessageBatch(anyRequest())) + .thenReturn( + completedFuture(partialSuccessResponse(entries[2].id(), entries[3].id())), + completedFuture(partialSuccessResponse(entries[3].id())), + completedFuture(SUCCESS)); + + p.apply(Create.of(23)) + .apply(ParDo.of(new CreateMessages())) + .apply(SqsIO.writeBatches().withEntryMapper(SET_MESSAGE_BODY).to("queue")); + + p.run().waitUntilFinish(); + + verify(sqs).sendMessageBatch(request("queue", entries)); + verify(sqs).sendMessageBatch(request("queue", entries[2], entries[3])); + verify(sqs).sendMessageBatch(request("queue", entries[3])); + verify(sqs).sendMessageBatch(request("queue", range(10, 20))); + verify(sqs).sendMessageBatch(request("queue", range(20, 23))); + + verify(sqs).close(); + verifyNoMoreInteractions(sqs); + + logs.verifyInfo("retry after partial failure: code REASON for 2 record(s)"); + logs.verifyInfo("retry after partial failure: code REASON for 1 record(s)"); + } + + @Test + public void testWriteCustomBatches() { + when(sqs.sendMessageBatch(anyRequest())).thenReturn(completedFuture(SUCCESS)); + + p.apply(Create.of(8)) + .apply(ParDo.of(new CreateMessages())) + .apply( + SqsIO.writeBatches() + .withEntryMapper(SET_MESSAGE_BODY) + .withBatchSize(3) + .to("queue")); + + p.run().waitUntilFinish(); + + verify(sqs).sendMessageBatch(request("queue", range(0, 3))); + verify(sqs).sendMessageBatch(request("queue", range(3, 6))); + verify(sqs).sendMessageBatch(request("queue", range(6, 8))); + + verify(sqs).close(); + verifyNoMoreInteractions(sqs); + } + + @Test + public void testWriteBatchesWithTimeout() { + when(sqs.sendMessageBatch(anyRequest())).thenReturn(completedFuture(SUCCESS)); + + p.apply(Create.of(5)) + .apply(ParDo.of(new CreateMessages())) + .apply( + // simulate delay between messages > batch timeout + SqsIO.writeBatches() + .withEntryMapper(withDelay(millis(100), SET_MESSAGE_BODY)) + .withBatchTimeout(millis(150)) + .to("queue")); + + p.run().waitUntilFinish(); + + SendMessageBatchRequestEntry[] entries = entries(range(0, 5)); + // due to added delay, batches are timed out on arrival of every 3rd msg + verify(sqs).sendMessageBatch(request("queue", entries[0], entries[1], entries[2])); + verify(sqs).sendMessageBatch(request("queue", entries[3], entries[4])); + } + + @Test + public void testWriteBatchesWithStrictTimeout() { + when(sqs.sendMessageBatch(any(SendMessageBatchRequest.class))) + .thenReturn(completedFuture(SendMessageBatchResponse.builder().build())); + + p.apply(Create.of(5)) + .apply(ParDo.of(new CreateMessages())) + .apply( + // simulate delay between messages > batch timeout + SqsIO.writeBatches() + .withEntryMapper(withDelay(millis(100), SET_MESSAGE_BODY)) + .withBatchTimeout(millis(150), true) + .to("queue")); + + p.run().waitUntilFinish(); + + SendMessageBatchRequestEntry[] entries = entries(range(0, 5)); + // using strict timeouts batches, batches are timed out by a separate thread + verify(sqs).sendMessageBatch(request("queue", entries[0], entries[1])); + verify(sqs).sendMessageBatch(request("queue", entries[2], entries[3])); + verify(sqs).sendMessageBatch(request("queue", entries[4])); + } + + @Test + public void testWriteBatchesToDynamic() { + when(sqs.sendMessageBatch(anyRequest())).thenReturn(completedFuture(SUCCESS)); + + // minimize delay due to retries + RetryConfiguration retry = RetryConfiguration.builder().maxBackoff(millis(1)).build(); + + p.apply(Create.of(10)) + .apply(ParDo.of(new CreateMessages())) + .apply( + SqsIO.writeBatches() + .withEntryMapper(SET_MESSAGE_BODY) + .withClientConfiguration(ClientConfiguration.builder().retry(retry).build()) + .withBatchSize(3) + .to(msg -> Integer.valueOf(msg) % 2 == 0 ? "even" : "uneven")); + + p.run().waitUntilFinish(); + + // id generator creates ids in range of [0, batch size * (queues + 1)) + SendMessageBatchRequestEntry[] entries = entries(range(0, 9), range(9, 10)); + + verify(sqs).sendMessageBatch(request("even", entries[0], entries[2], entries[4])); + verify(sqs).sendMessageBatch(request("uneven", entries[1], entries[3], entries[5])); + verify(sqs).sendMessageBatch(request("even", entries[6], entries[8])); + verify(sqs).sendMessageBatch(request("uneven", entries[7], entries[9])); + + verify(sqs).close(); + verifyNoMoreInteractions(sqs); + } + + @Test + public void testWriteBatchesToDynamicWithTimeout() { + when(sqs.sendMessageBatch(anyRequest())).thenReturn(completedFuture(SUCCESS)); + + p.apply(Create.of(5)) + .apply(ParDo.of(new CreateMessages())) + .apply( + // simulate delay between messages > batch timeout + SqsIO.writeBatches() + .withEntryMapper(withDelay(millis(100), SET_MESSAGE_BODY)) + .withBatchTimeout(millis(150)) + .to(msg -> Integer.valueOf(msg) % 2 == 0 ? "even" : "uneven")); + + p.run().waitUntilFinish(); + + SendMessageBatchRequestEntry[] entries = entries(range(0, 5)); + // due to added delay, dynamic batches are timed out on arrival of every 2nd msg (per batch) + verify(sqs).sendMessageBatch(request("even", entries[0], entries[2])); + verify(sqs).sendMessageBatch(request("uneven", entries[1], entries[3])); + verify(sqs).sendMessageBatch(request("even", entries[4])); + } + + @Test + public void testWriteBatchesToDynamicWithStrictTimeout() { + when(sqs.sendMessageBatch(any(SendMessageBatchRequest.class))) + .thenReturn(completedFuture(SendMessageBatchResponse.builder().build())); + + p.apply(Create.of(5)) + .apply(ParDo.of(new CreateMessages())) + .apply( + // simulate delay between messages > batch timeout + SqsIO.writeBatches() + .withEntryMapper(withDelay(millis(100), SET_MESSAGE_BODY)) + .withBatchTimeout(millis(150), true) + .to(msg -> Integer.valueOf(msg) % 2 == 0 ? "even" : "uneven")); + + p.run().waitUntilFinish(); + + SendMessageBatchRequestEntry[] entries = entries(range(0, 5)); + // using strict timeouts batches, batches are timed out by a separate thread before any 2nd + // entry + verify(sqs).sendMessageBatch(request("even", entries[0])); + verify(sqs).sendMessageBatch(request("uneven", entries[1])); + verify(sqs).sendMessageBatch(request("even", entries[2])); + verify(sqs).sendMessageBatch(request("uneven", entries[3])); + verify(sqs).sendMessageBatch(request("even", entries[4])); + } + + @Test + public void testWriteBatchesToDynamicWithStrictTimeoutAtHighVolume() { + when(sqs.sendMessageBatch(any(SendMessageBatchRequest.class))) + .thenReturn(completedFuture(SendMessageBatchResponse.builder().build())); + + // Use sqrt to change the rate of newly created dynamic destinations over time + DynamicDestination dynamicDestination = + msg -> String.valueOf(RandomUtils.nextInt(0, (int) (1 + sqrt(Integer.valueOf(msg))))); + + p.apply(Create.of(100000)) + .apply(ParDo.of(new CreateMessages())) + .apply( + SqsIO.writeBatches() + .withEntryMapper(SET_MESSAGE_BODY) + .withBatchTimeout(millis(10), true) + .to(dynamicDestination)); + + p.run().waitUntilFinish(); + + ArgumentCaptor reqCaptor = + ArgumentCaptor.forClass(SendMessageBatchRequest.class); + verify(sqs, atLeastOnce()).sendMessageBatch(reqCaptor.capture()); + + Set capturedMessages = new HashSet<>(); + for (SendMessageBatchRequest req : reqCaptor.getAllValues()) { + for (SendMessageBatchRequestEntry entry : req.entries()) { + assertTrue("duplicate message", capturedMessages.add(entry.messageBody())); + } + } + assertEquals("Invalid message count", 100000, capturedMessages.size()); + } + + private SendMessageBatchRequest anyRequest() { + return any(); + } + + private SendMessageBatchRequest request(String queue, SendMessageBatchRequestEntry... entries) { + return SendMessageBatchRequest.builder() + .queueUrl(queue) + .entries(Arrays.asList(entries)) + .build(); + } + + private SendMessageBatchRequest request(String queue, IntStream msgs) { + return request(queue, entries(msgs)); + } + + private SendMessageBatchRequestEntry[] entries(IntStream... msgStreams) { + return Arrays.stream(msgStreams) + .flatMap(msgs -> Streams.mapWithIndex(msgs, this::entry)) + .toArray(SendMessageBatchRequestEntry[]::new); + } + + private SendMessageBatchRequestEntry entry(int msg, long id) { + return SendMessageBatchRequestEntry.builder() + .id(Long.toString(id)) + .messageBody(Integer.toString(msg)) + .build(); + } + + private SendMessageBatchResponse partialSuccessResponse(String... failedIds) { + Stream errors = + Arrays.stream(failedIds) + .map(BatchResultErrorEntry.builder()::id) + .map(b -> b.code("REASON").build()); + return SendMessageBatchResponse.builder().failed(errors.collect(toList())).build(); + } + + private static class CreateMessages extends DoFn { + @ProcessElement + public void processElement(@Element Integer count, OutputReceiver out) { + for (int i = 0; i < count; i++) { + out.output(Integer.toString(i)); + } + } + } + + private static EntryMapperFn.Builder withDelay( + Duration delay, EntryMapperFn.Builder builder) { + return (t1, t2) -> { + builder.accept(t1, t2); + try { + Thread.sleep(delay.getMillis()); + } catch (InterruptedException e) { + } + }; + } +} diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/SqsIOWriteTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/SqsIOWriteTest.java deleted file mode 100644 index 738cf282adf68..0000000000000 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/SqsIOWriteTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io.aws2.sqs; - -import static java.util.stream.IntStream.range; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.util.List; -import java.util.stream.Collectors; -import org.apache.beam.sdk.io.aws2.MockClientBuilderFactory; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.transforms.Create; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; -import software.amazon.awssdk.services.sqs.SqsClient; -import software.amazon.awssdk.services.sqs.SqsClientBuilder; -import software.amazon.awssdk.services.sqs.model.SendMessageRequest; -import software.amazon.awssdk.services.sqs.model.SendMessageResponse; - -/** Tests for {@link SqsIO.Write}. */ -@RunWith(MockitoJUnitRunner.class) -public class SqsIOWriteTest { - @Rule public TestPipeline p = TestPipeline.create(); - @Mock public SqsClient sqs; - - @Before - public void configureClientBuilderFactory() { - MockClientBuilderFactory.set(p, SqsClientBuilder.class, sqs); - } - - @Test - public void testWrite() { - when(sqs.sendMessage(any(SendMessageRequest.class))) - .thenReturn(SendMessageResponse.builder().build()); - - SendMessageRequest.Builder builder = SendMessageRequest.builder().queueUrl("url"); - List messages = - range(0, 100) - .mapToObj(i -> builder.messageBody("test" + i).build()) - .collect(Collectors.toList()); - - p.apply(Create.of(messages)).apply(SqsIO.write()); - p.run().waitUntilFinish(); - - messages.forEach(msg -> verify(sqs).sendMessage(msg)); - } -} diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/testing/SqsIOIT.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/testing/SqsIOIT.java index f1e176f572d60..9e78f7a15eff0 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/testing/SqsIOIT.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/testing/SqsIOIT.java @@ -19,6 +19,7 @@ import static org.apache.beam.sdk.io.common.TestRow.getExpectedHashForRowCount; import static org.apache.beam.sdk.values.TypeDescriptors.strings; +import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; import static org.testcontainers.containers.localstack.LocalStackContainer.Service.SQS; import java.io.Serializable; @@ -103,6 +104,35 @@ public void testWriteThenRead() { pipelineRead.run(); } + @Test + public void testWriteBatchesThenRead() { + int rows = env.options().getNumberOfRows(); + + // Write test dataset to SQS. + pipelineWrite + .apply("Generate Sequence", GenerateSequence.from(0).to(rows)) + .apply("Prepare TestRows", ParDo.of(new DeterministicallyConstructTestRowFn())) + .apply( + "Write to SQS", + SqsIO.writeBatches() + .withEntryMapper((b, row) -> b.messageBody(row.name())) + .to(sqsQueue.url)); + + // Read test dataset from SQS. + PCollection output = + pipelineRead + .apply("Read from SQS", SqsIO.read().withQueueUrl(sqsQueue.url).withMaxNumRecords(rows)) + .apply("Extract body", MapElements.into(strings()).via(SqsMessage::getBody)); + + PAssert.thatSingleton(output.apply("Count All", Count.globally())).isEqualTo((long) rows); + + PAssert.that(output.apply(Combine.globally(new HashingFn()).withoutDefaults())) + .containsInAnyOrder(getExpectedHashForRowCount(rows)); + + pipelineWrite.run(); + pipelineRead.run(); + } + private static class SqsQueue extends ExternalResource implements Serializable { private transient SqsClient client = env.buildClient(SqsClient.builder()); private String url; @@ -113,7 +143,8 @@ SendMessageRequest messageRequest(TestRow r) { @Override protected void before() throws Throwable { - url = client.createQueue(b -> b.queueName("beam-sqsio-it")).queueUrl(); + url = + client.createQueue(b -> b.queueName("beam-sqsio-it-" + randomAlphanumeric(4))).queueUrl(); } @Override diff --git a/sdks/java/io/amqp/OWNERS b/sdks/java/io/amqp/OWNERS deleted file mode 100644 index 8b0b05531cd10..0000000000000 --- a/sdks/java/io/amqp/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - jbonofre diff --git a/sdks/java/io/amqp/build.gradle b/sdks/java/io/amqp/build.gradle index 124b1ad7f9c51..628cc5a9a387b 100644 --- a/sdks/java/io/amqp/build.gradle +++ b/sdks/java/io/amqp/build.gradle @@ -23,7 +23,7 @@ description = "Apache Beam :: SDKs :: Java :: IO :: AMQP" ext.summary = "IO to read and write using AMQP 1.0 protocol (http://www.amqp.org)." dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.joda_time implementation "org.apache.qpid:proton-j:0.16.0" diff --git a/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpIO.java b/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpIO.java index d511b4b56a2b2..696332abf05e1 100644 --- a/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpIO.java +++ b/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpIO.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.amqp; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.io.IOException; @@ -36,7 +36,7 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; import org.apache.qpid.proton.message.Message; import org.apache.qpid.proton.messenger.Messenger; import org.apache.qpid.proton.messenger.Tracker; diff --git a/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoder.java b/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoder.java index 9377f1ec1fd9e..dff6ce28d4087 100644 --- a/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoder.java +++ b/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoder.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.coders.CoderException; import org.apache.beam.sdk.coders.CustomCoder; import org.apache.beam.sdk.util.VarInt; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; import org.apache.qpid.proton.message.Message; /** A coder for AMQP message. */ diff --git a/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoderProviderRegistrar.java b/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoderProviderRegistrar.java index eb2fbbe421810..08dd16ffe33df 100644 --- a/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoderProviderRegistrar.java +++ b/sdks/java/io/amqp/src/main/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoderProviderRegistrar.java @@ -23,7 +23,7 @@ import org.apache.beam.sdk.coders.CoderProviderRegistrar; import org.apache.beam.sdk.coders.CoderProviders; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.qpid.proton.message.Message; /** A {@link CoderProviderRegistrar} for standard types used with {@link AmqpIO}. */ diff --git a/sdks/java/io/amqp/src/test/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoderTest.java b/sdks/java/io/amqp/src/test/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoderTest.java index 96d65efda4979..ed67fdf9e0f9f 100644 --- a/sdks/java/io/amqp/src/test/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoderTest.java +++ b/sdks/java/io/amqp/src/test/java/org/apache/beam/sdk/io/amqp/AmqpMessageCoderTest.java @@ -22,7 +22,7 @@ import java.util.Collections; import org.apache.beam.sdk.coders.CoderException; import org.apache.beam.sdk.util.CoderUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; import org.apache.qpid.proton.amqp.messaging.AmqpValue; import org.apache.qpid.proton.message.Message; import org.junit.Rule; diff --git a/sdks/java/io/azure-cosmos/build.gradle b/sdks/java/io/azure-cosmos/build.gradle new file mode 100644 index 0000000000000..3f8a4914fcb6e --- /dev/null +++ b/sdks/java/io/azure-cosmos/build.gradle @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("org.apache.beam.module") +} + +applyJavaNature(automaticModuleName: "org.apache.beam.sdk.io.azure.cosmos") + +description = "Apache Beam :: SDKs :: Java :: IO :: Azure Cosmos DB" +ext.summary = "IO library to read and write Azure Cosmos DB" + +def excludeNetty = { + exclude group: "io.netty", module: "*" // exclude more recent Netty version +} + +dependencies { + implementation platform(library.java.azure_sdk_bom) + implementation "com.azure:azure-cosmos", excludeNetty + implementation "com.azure:azure-core", excludeNetty + permitUnusedDeclared library.java.commons_io // BEAM-11761 + implementation library.java.jackson_annotations + implementation library.java.jackson_databind + implementation library.java.vendored_guava_32_1_2_jre + implementation "io.projectreactor:reactor-core:3.4.29" + implementation project(path: ":sdks:java:core", configuration: "shadow") + implementation project(path: ":sdks:java:io:azure") + runtimeOnly library.java.netty_all // force version of netty used by Beam + + testImplementation library.java.junit + testImplementation library.java.testcontainers_azure + testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") + testRuntimeOnly library.java.slf4j_jdk14 + testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") +} diff --git a/sdks/java/io/azure-cosmos/src/main/java/org/apache/beam/sdk/io/azure/cosmos/CosmosIO.java b/sdks/java/io/azure-cosmos/src/main/java/org/apache/beam/sdk/io/azure/cosmos/CosmosIO.java new file mode 100644 index 0000000000000..851d18d0eab73 --- /dev/null +++ b/sdks/java/io/azure-cosmos/src/main/java/org/apache/beam/sdk/io/azure/cosmos/CosmosIO.java @@ -0,0 +1,307 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.azure.cosmos; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; + +import com.azure.cosmos.CosmosAsyncClient; +import com.azure.cosmos.CosmosAsyncContainer; +import com.azure.cosmos.CosmosAsyncDatabase; +import com.azure.cosmos.CosmosBridgeInternal; +import com.azure.cosmos.CosmosClientBuilder; +import com.azure.cosmos.implementation.AsyncDocumentClient; +import com.azure.cosmos.implementation.DocumentCollection; +import com.azure.cosmos.implementation.ImplementationBridgeHelpers; +import com.azure.cosmos.implementation.Paths; +import com.azure.cosmos.implementation.Utils; +import com.azure.cosmos.implementation.feedranges.FeedRangeInternal; +import com.azure.cosmos.models.CosmosChangeFeedRequestOptions; +import com.azure.cosmos.models.CosmosQueryRequestOptions; +import com.azure.cosmos.models.FeedResponse; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.auto.value.AutoValue; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.stream.Collectors; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.io.BoundedSource; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.PBegin; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.checkerframework.checker.nullness.qual.Nullable; +import reactor.core.publisher.Mono; + +@SuppressWarnings({ + "nullness" // CosmosAsyncClient does not use nullable annotations +}) +public class CosmosIO { + + private CosmosIO() {} + + private static final String DEFAULT_QUERY = "SELECT * FROM root"; + + /** Provide a {@link Read} {@link PTransform} to read data from a Cosmos DB. */ + public static Read read(Class classType) { + return new AutoValue_CosmosIO_Read.Builder().setClassType(classType).build(); + } + + @AutoValue + @AutoValue.CopyAnnotations + @SuppressWarnings({"rawtypes"}) + public abstract static class Read extends PTransform> { + + abstract @Nullable Class getClassType(); + + abstract @Nullable String getDatabase(); + + abstract @Nullable String getContainer(); + + abstract @Nullable String getQuery(); + + abstract @Nullable Coder getCoder(); + + abstract Builder builder(); + + @AutoValue.Builder + abstract static class Builder { + + abstract Builder setClassType(Class classType); + + abstract Builder setDatabase(String database); + + abstract Builder setContainer(String container); + + abstract Builder setQuery(String query); + + abstract Builder setCoder(Coder coder); + + abstract Read build(); + } + + /** Specify the Cosmos database to read from. */ + public Read withDatabase(String database) { + checkArgument(database != null, "database can not be null"); + checkArgument(!database.isEmpty(), "database can not be empty"); + return builder().setDatabase(database).build(); + } + + /** Specify the Cosmos container to read from. */ + public Read withContainer(String container) { + checkArgument(container != null, "container can not be null"); + checkArgument(!container.isEmpty(), "container can not be empty"); + return builder().setContainer(container).build(); + } + + /** Specify the query to read data. */ + public Read withQuery(String query) { + return builder().setQuery(query).build(); + } + + /** Specify the {@link Coder} used to serialize the document in the {@link PCollection}. */ + public Read withCoder(Coder coder) { + checkArgument(coder != null, "coder can not be null"); + return builder().setCoder(coder).build(); + } + + @Override + public PCollection expand(PBegin input) { + checkState(getDatabase() != null, "withDatabase() is required"); + checkState(getContainer() != null, "withContainer() is required"); + checkState(getCoder() != null, "withCoder() is required"); + return input.apply(org.apache.beam.sdk.io.Read.from(new BoundedCosmosBDSource<>(this))); + } + } + + /** A {@link BoundedSource} reading from Comos. */ + @VisibleForTesting + public static class BoundedCosmosBDSource extends BoundedSource { + + private final Read spec; + private final NormalizedRange range; + + private @Nullable Long estimatedByteSize; + + BoundedCosmosBDSource(Read spec) { + this(spec, NormalizedRange.FULL_RANGE, null); + } + + BoundedCosmosBDSource(Read spec, NormalizedRange range, @Nullable Long estimatedSize) { + this.spec = spec; + this.range = range; + this.estimatedByteSize = estimatedSize; + } + + @Override + public List> split( + long desiredBundleSizeBytes, PipelineOptions options) throws Exception { + CosmosClientBuilder builder = options.as(CosmosOptions.class).getCosmosClientBuilder(); + try (CosmosAsyncClient client = builder.buildAsyncClient()) { + CosmosAsyncDatabase database = client.getDatabase(spec.getDatabase()); + CosmosAsyncContainer container = database.getContainer(spec.getContainer()); + AsyncDocumentClient document = CosmosBridgeInternal.getAsyncDocumentClient(client); + + List> sources = new ArrayList<>(); + long rangeSize = getEstimatedSizeBytes(options); + float splitsFloat = (float) rangeSize / desiredBundleSizeBytes; + int splits = (int) Math.ceil(splitsFloat); + + // match internal impl of CosmosAsyncContainer trySplitFeedRange + String databaseLink = + ImplementationBridgeHelpers.CosmosAsyncDatabaseHelper.getCosmosAsyncDatabaseAccessor() + .getLink(database); + String containerLink = + databaseLink + "/" + Paths.COLLECTIONS_PATH_SEGMENT + "/" + container.getId(); + Mono> getCollectionObservable = + document + .getCollectionCache() + .resolveByNameAsync(null, containerLink, null) + .map(Utils.ValueHolder::initialize); + + List subRanges = + FeedRangeInternal.convert(range.toFeedRange()) + .trySplit( + document.getPartitionKeyRangeCache(), null, getCollectionObservable, splits) + .block().stream() + .map(NormalizedRange::fromFeedRange) + .collect(Collectors.toList()); + + long estimatedSubRangeSize = rangeSize / subRanges.size(); + for (NormalizedRange subrange : subRanges) { + sources.add(new BoundedCosmosBDSource<>(spec, subrange, estimatedSubRangeSize)); + } + + return sources; + } + } + + @Override + public long getEstimatedSizeBytes(PipelineOptions options) throws Exception { + if (estimatedByteSize != null) { + return estimatedByteSize; + } + CosmosClientBuilder builder = options.as(CosmosOptions.class).getCosmosClientBuilder(); + try (CosmosAsyncClient client = builder.buildAsyncClient()) { + CosmosAsyncContainer container = + client.getDatabase(spec.getDatabase()).getContainer(spec.getContainer()); + + CosmosChangeFeedRequestOptions requestOptions = + CosmosChangeFeedRequestOptions.createForProcessingFromNow(range.toFeedRange()); + requestOptions.setMaxItemCount(1); + requestOptions.setMaxPrefetchPageCount(1); + requestOptions.setQuotaInfoEnabled(true); + + estimatedByteSize = + container + .queryChangeFeed(requestOptions, ObjectNode.class) + .byPage() + .take(1) + .map(FeedResponse::getDocumentUsage) + .map(kiloByteSize -> kiloByteSize * 1024) // conversion from KB to bytes + .single() + .block(); + + return estimatedByteSize == null ? 0 : estimatedByteSize; + } + } + + @Override + public Coder getOutputCoder() { + return spec.getCoder(); + } + + @Override + public BoundedReader createReader(PipelineOptions options) throws IOException { + return new BoundedCosmosReader<>(this, options.as(CosmosOptions.class)); + } + } + + private static class BoundedCosmosReader extends BoundedSource.BoundedReader { + + private final BoundedCosmosBDSource source; + + private final CosmosAsyncClient client; + + private T current; + private Iterator iterator; + + private BoundedCosmosReader(BoundedCosmosBDSource source, CosmosOptions options) { + this.source = source; + this.client = options.as(CosmosOptions.class).getCosmosClientBuilder().buildAsyncClient(); + } + + @Override + public boolean start() throws IOException { + String database = source.spec.getDatabase(); + String container = source.spec.getContainer(); + String query = source.spec.getQuery(); + Class classType = source.spec.getClassType(); + CosmosAsyncContainer c = client.getDatabase(database).getContainer(container); + + // custom options with query plan disabled + CosmosQueryRequestOptions queryOptions = + ImplementationBridgeHelpers.CosmosQueryRequestOptionsHelper + .getCosmosQueryRequestOptionsAccessor() + .disallowQueryPlanRetrieval(new CosmosQueryRequestOptions()) + .setFeedRange(source.range.toFeedRange()); + + iterator = + c.queryItems(query == null ? DEFAULT_QUERY : query, queryOptions, classType) + .toIterable() + .iterator(); + + return readNext(); + } + + @Override + public boolean advance() throws IOException { + return readNext(); + } + + private boolean readNext() { + boolean nonEmpty = iterator.hasNext(); + if (nonEmpty) { + current = iterator.next(); + } + return nonEmpty; + } + + @Override + public T getCurrent() throws NoSuchElementException { + if (current == null) { + throw new NoSuchElementException(); + } + return current; + } + + @Override + public void close() throws IOException { + client.close(); + } + + @Override + public BoundedSource getCurrentSource() { + return source; + } + } +} diff --git a/sdks/java/io/azure-cosmos/src/main/java/org/apache/beam/sdk/io/azure/cosmos/CosmosOptions.java b/sdks/java/io/azure-cosmos/src/main/java/org/apache/beam/sdk/io/azure/cosmos/CosmosOptions.java new file mode 100644 index 0000000000000..0c4b65b9bd8f4 --- /dev/null +++ b/sdks/java/io/azure-cosmos/src/main/java/org/apache/beam/sdk/io/azure/cosmos/CosmosOptions.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.azure.cosmos; + +import com.azure.core.credential.TokenCredential; +import com.azure.cosmos.CosmosClientBuilder; +import com.fasterxml.jackson.annotation.JsonIgnore; +import org.apache.beam.sdk.io.azure.options.AzureOptions; +import org.apache.beam.sdk.options.Default; +import org.apache.beam.sdk.options.DefaultValueFactory; +import org.apache.beam.sdk.options.Description; +import org.apache.beam.sdk.options.PipelineOptions; +import org.checkerframework.checker.nullness.qual.Nullable; + +public interface CosmosOptions extends AzureOptions { + + @JsonIgnore + @Description( + "The Azure Cosmos client builder. " + + "If no client has been set explicitly, the default is to use the instance factory.") + @Default.InstanceFactory(CosmosClientBuilderFactory.class) + CosmosClientBuilder getCosmosClientBuilder(); + + void setCosmosClientBuilder(CosmosClientBuilder builder); + + /** The Azure Cosmos service endpoint used by the Cosmos client. */ + @Description("Sets the cosmos service endpoint") + @Nullable + String getCosmosServiceEndpoint(); + + void setCosmosServiceEndpoint(String endpoint); + + /** The Azure Cosmos key used to perform authentication for accessing resource. */ + @Description("Sets the cosmos service endpoint") + @Nullable + String getCosmosKey(); + + void setCosmosKey(String key); + + /** Create a cosmos client from the pipeline options. */ + class CosmosClientBuilderFactory implements DefaultValueFactory { + + @Override + public CosmosClientBuilder create(PipelineOptions options) { + CosmosOptions cosmosOptions = options.as(CosmosOptions.class); + CosmosClientBuilder builder = new CosmosClientBuilder(); + + TokenCredential credential = cosmosOptions.getAzureCredentialsProvider(); + if (credential != null) { + builder = builder.credential(credential); + } + + String endpoint = cosmosOptions.getCosmosServiceEndpoint(); + if (endpoint != null && !endpoint.isEmpty()) { + builder = builder.endpoint(endpoint); + } + + String key = cosmosOptions.getCosmosKey(); + if (key != null && !key.isEmpty()) { + builder = builder.key(key); + } + + return builder; + } + } +} diff --git a/sdks/java/io/azure-cosmos/src/main/java/org/apache/beam/sdk/io/azure/cosmos/NormalizedRange.java b/sdks/java/io/azure-cosmos/src/main/java/org/apache/beam/sdk/io/azure/cosmos/NormalizedRange.java new file mode 100644 index 0000000000000..670da3ab11168 --- /dev/null +++ b/sdks/java/io/azure-cosmos/src/main/java/org/apache/beam/sdk/io/azure/cosmos/NormalizedRange.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.azure.cosmos; + +import com.azure.cosmos.implementation.feedranges.FeedRangeEpkImpl; +import com.azure.cosmos.implementation.routing.PartitionKeyInternalHelper; +import com.azure.cosmos.implementation.routing.Range; +import com.azure.cosmos.models.FeedRange; +import java.io.Serializable; +import java.math.BigInteger; + +public class NormalizedRange implements Serializable { + + public static final NormalizedRange FULL_RANGE = + new NormalizedRange( + PartitionKeyInternalHelper.MinimumInclusiveEffectivePartitionKey, + PartitionKeyInternalHelper.MaximumExclusiveEffectivePartitionKey); + + private final String minInclusive; + private final String maxExclusive; + + public static NormalizedRange fromFeedRange(FeedRange feedRange) { + if (feedRange instanceof FeedRangeEpkImpl) { + FeedRangeEpkImpl ekp = (FeedRangeEpkImpl) feedRange; + Range range = ekp.getRange(); + String min = range.getMin(); + String max = range.getMax(); + + if (!range.isMinInclusive()) { + min = new BigInteger(min, 16).add(BigInteger.ONE).toString(16); + } + + if (range.isMaxInclusive()) { + max = new BigInteger(max, 16).subtract(BigInteger.ONE).toString(16); + } + + return new NormalizedRange(min, max); + } else { + throw new IllegalArgumentException("Only FeedRangeEpkImpl are supported. got: " + feedRange); + } + } + + public NormalizedRange(String minInclusive, String maxExclusive) { + this.minInclusive = minInclusive; + this.maxExclusive = maxExclusive; + } + + public FeedRange toFeedRange() { + return new FeedRangeEpkImpl(new Range<>(minInclusive, maxExclusive, true, false)); + } +} diff --git a/sdks/java/io/azure-cosmos/src/main/java/org/apache/beam/sdk/io/azure/cosmos/package-info.java b/sdks/java/io/azure-cosmos/src/main/java/org/apache/beam/sdk/io/azure/cosmos/package-info.java new file mode 100644 index 0000000000000..c29335b53cece --- /dev/null +++ b/sdks/java/io/azure-cosmos/src/main/java/org/apache/beam/sdk/io/azure/cosmos/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Defines IO connectors for Azure Cosmos DB. */ +package org.apache.beam.sdk.io.azure.cosmos; diff --git a/sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Address.java b/sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Address.java new file mode 100644 index 0000000000000..84ed1328e93e1 --- /dev/null +++ b/sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Address.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.azure.cosmos.examples.common; + +import java.io.Serializable; + +public class Address implements Serializable { + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public String getCounty() { + return county; + } + + public void setCounty(String county) { + this.county = county; + } + + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + private String state = ""; + private String county = ""; + private String city = ""; +} diff --git a/sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Child.java b/sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Child.java new file mode 100644 index 0000000000000..d3c1d9dbdcbd5 --- /dev/null +++ b/sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Child.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.azure.cosmos.examples.common; + +import java.io.Serializable; + +public class Child implements Serializable { + public String getFamilyName() { + return familyName; + } + + public void setFamilyName(String familyName) { + this.familyName = familyName; + } + + public String getFirstName() { + return firstName; + } + + public void setFirstName(String firstName) { + this.firstName = firstName; + } + + public String getGender() { + return gender; + } + + public void setGender(String gender) { + this.gender = gender; + } + + public int getGrade() { + return grade; + } + + public void setGrade(int grade) { + this.grade = grade; + } + + public Pet[] getPets() { + return pets; + } + + public void setPets(Pet[] pets) { + this.pets = pets; + } + + private String familyName; + private String firstName; + private String gender; + private int grade; + private Pet[] pets; +} diff --git a/sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Families.java b/sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Families.java new file mode 100644 index 0000000000000..69e2a687e4d8f --- /dev/null +++ b/sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Families.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.azure.cosmos.examples.common; + +import java.io.Serializable; + +public class Families implements Serializable { + + public static Family getAndersenFamilyItem() { + Family andersenFamily = new Family(); + andersenFamily.setId("Andersen-" + System.currentTimeMillis()); + andersenFamily.setLastName("Andersen"); + + Parent parent1 = new Parent(); + parent1.setFirstName("Thomas"); + + Parent parent2 = new Parent(); + parent2.setFirstName("Mary Kay"); + + andersenFamily.setParents(new Parent[] {parent1, parent2}); + + Child child1 = new Child(); + child1.setFirstName("Henriette Thaulow"); + child1.setGender("female"); + child1.setGrade(5); + + Pet pet1 = new Pet(); + pet1.setGivenName("Fluffy"); + + child1.setPets(new Pet[] {pet1}); + + andersenFamily.setDistrict("WA5"); + Address address = new Address(); + address.setCity("Seattle"); + address.setCounty("King"); + address.setState("WA"); + + andersenFamily.setAddress(address); + andersenFamily.setRegistered(true); + + return andersenFamily; + } + + public static Family getWakefieldFamilyItem() { + Family wakefieldFamily = new Family(); + wakefieldFamily.setId("Wakefield-" + System.currentTimeMillis()); + wakefieldFamily.setLastName("Wakefield"); + + Parent parent1 = new Parent(); + parent1.setFamilyName("Wakefield"); + parent1.setFirstName("Robin"); + + Parent parent2 = new Parent(); + parent2.setFamilyName("Miller"); + parent2.setFirstName("Ben"); + + wakefieldFamily.setParents(new Parent[] {parent1, parent2}); + + Child child1 = new Child(); + child1.setFirstName("Jesse"); + child1.setFamilyName("Merriam"); + child1.setGrade(8); + + Pet pet1 = new Pet(); + pet1.setGivenName("Goofy"); + + Pet pet2 = new Pet(); + pet2.setGivenName("Shadow"); + + child1.setPets(new Pet[] {pet1, pet2}); + + Child child2 = new Child(); + child2.setFirstName("Lisa"); + child2.setFamilyName("Miller"); + child2.setGrade(1); + child2.setGender("female"); + + wakefieldFamily.setChildren(new Child[] {child1, child2}); + + Address address = new Address(); + address.setCity("NY"); + address.setCounty("Manhattan"); + address.setState("NY"); + + wakefieldFamily.setAddress(address); + wakefieldFamily.setDistrict("NY23"); + wakefieldFamily.setRegistered(true); + return wakefieldFamily; + } + + public static Family getJohnsonFamilyItem() { + Family andersenFamily = new Family(); + andersenFamily.setId("Johnson-" + System.currentTimeMillis()); + andersenFamily.setLastName("Johnson"); + + Parent parent1 = new Parent(); + parent1.setFirstName("John"); + + Parent parent2 = new Parent(); + parent2.setFirstName("Lili"); + + return andersenFamily; + } + + public static Family getSmithFamilyItem() { + Family andersenFamily = new Family(); + andersenFamily.setId("Smith-" + System.currentTimeMillis()); + andersenFamily.setLastName("Smith"); + + Parent parent1 = new Parent(); + parent1.setFirstName("John"); + + Parent parent2 = new Parent(); + parent2.setFirstName("Cynthia"); + + return andersenFamily; + } +} diff --git a/sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Family.java b/sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Family.java new file mode 100644 index 0000000000000..c76d029e8dee5 --- /dev/null +++ b/sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Family.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.azure.cosmos.examples.common; + +import java.io.Serializable; + +public class Family implements Serializable { + public Family() {} + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getLastName() { + return lastName; + } + + public void setLastName(String lastName) { + this.lastName = lastName; + } + + public String getDistrict() { + return district; + } + + public void setDistrict(String district) { + this.district = district; + } + + public Parent[] getParents() { + return parents; + } + + public void setParents(Parent[] parents) { + this.parents = parents; + } + + public Child[] getChildren() { + return children; + } + + public void setChildren(Child[] children) { + this.children = children; + } + + public Address getAddress() { + return address; + } + + public void setAddress(Address address) { + this.address = address; + } + + public boolean isRegistered() { + return isRegistered; + } + + public void setRegistered(boolean isRegistered) { + this.isRegistered = isRegistered; + } + + private String id = ""; + private String lastName = ""; + private String district = ""; + private Parent[] parents = {}; + private Child[] children = {}; + private Address address = new Address(); + private boolean isRegistered = false; +} diff --git a/sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Parent.java b/sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Parent.java new file mode 100644 index 0000000000000..a8d02f2b51763 --- /dev/null +++ b/sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Parent.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.azure.cosmos.examples.common; + +import java.io.Serializable; + +public class Parent implements Serializable { + + public Parent() {} + + public Parent(String firstName) { + this.firstName = firstName; + } + + public String getFamilyName() { + return familyName; + } + + public void setFamilyName(String familyName) { + this.familyName = familyName; + } + + public String getFirstName() { + return firstName; + } + + public void setFirstName(String firstName) { + this.firstName = firstName; + } + + private String familyName; + private String firstName; +} diff --git a/sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Pet.java b/sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Pet.java new file mode 100644 index 0000000000000..bca68405fc18f --- /dev/null +++ b/sdks/java/io/azure-cosmos/src/test/java/com/azure/cosmos/examples/common/Pet.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.azure.cosmos.examples.common; + +import java.io.Serializable; + +public class Pet implements Serializable { + public String getGivenName() { + return givenName; + } + + public void setGivenName(String givenName) { + this.givenName = givenName; + } + + private String givenName; +} diff --git a/sdks/java/io/azure-cosmos/src/test/java/org/apache/beam/sdk/io/azure/cosmos/CosmosIOTest.java b/sdks/java/io/azure-cosmos/src/test/java/org/apache/beam/sdk/io/azure/cosmos/CosmosIOTest.java new file mode 100644 index 0000000000000..051153cbb31c4 --- /dev/null +++ b/sdks/java/io/azure-cosmos/src/test/java/org/apache/beam/sdk/io/azure/cosmos/CosmosIOTest.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.azure.cosmos; + +import static org.junit.Assert.assertEquals; + +import com.azure.cosmos.CosmosClient; +import com.azure.cosmos.CosmosClientBuilder; +import com.azure.cosmos.CosmosContainer; +import com.azure.cosmos.CosmosDatabase; +import com.azure.cosmos.examples.common.Families; +import com.azure.cosmos.examples.common.Family; +import com.azure.cosmos.models.CosmosItemRequestOptions; +import com.azure.cosmos.models.PartitionKey; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyStore; +import java.util.ArrayList; +import java.util.List; +import org.apache.beam.sdk.coders.SerializableCoder; +import org.apache.beam.sdk.io.BoundedSource; +import org.apache.beam.sdk.io.azure.cosmos.CosmosIO.BoundedCosmosBDSource; +import org.apache.beam.sdk.io.azure.cosmos.CosmosIO.Read; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Count; +import org.apache.beam.sdk.values.PCollection; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.testcontainers.containers.CosmosDBEmulatorContainer; +import org.testcontainers.utility.DockerImageName; + +// See https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/quickstart-java +public class CosmosIOTest { + + private static final String DOCKER_IMAGE_NAME = + "mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:latest"; + private static final String DATABASE = "AzureSampleFamilyDB"; + private static final String CONTAINER = "FamilyContainer"; + private static final String PARTITION_KEY_PATH = "/lastName"; + + private static PipelineOptions pipelineOptions = PipelineOptionsFactory.create(); + private static CosmosDBEmulatorContainer container; + private static CosmosClient client; + + @BeforeClass + public static void beforeClass() throws Exception { + // Create the azure cosmos container and pull image as we use latest + container = new CosmosDBEmulatorContainer(DockerImageName.parse(DOCKER_IMAGE_NAME)); + container.withImagePullPolicy(imageName -> true); + + // Start the container. This step might take some time... + container.start(); + TemporaryFolder tempFolder = new TemporaryFolder(); + tempFolder.create(); + Path keyStoreFile = tempFolder.newFile("azure-cosmos-emulator.keystore").toPath(); + KeyStore keyStore = container.buildNewKeyStore(); + keyStore.store(Files.newOutputStream(keyStoreFile), container.getEmulatorKey().toCharArray()); + System.setProperty("javax.net.ssl.trustStore", keyStoreFile.toString()); + System.setProperty("javax.net.ssl.trustStorePassword", container.getEmulatorKey()); + System.setProperty("javax.net.ssl.trustStoreType", "PKCS12"); + + CosmosOptions cosmosOptions = pipelineOptions.as(CosmosOptions.class); + cosmosOptions.setCosmosServiceEndpoint(container.getEmulatorEndpoint()); + cosmosOptions.setCosmosKey(container.getEmulatorKey()); + + client = + new CosmosClientBuilder() + .gatewayMode() + .endpointDiscoveryEnabled(false) + .endpoint(container.getEmulatorEndpoint()) + .key(container.getEmulatorKey()) + .buildClient(); + + client.createDatabase(DATABASE); + CosmosDatabase db = client.getDatabase(DATABASE); + db.createContainer(CONTAINER, PARTITION_KEY_PATH); + CosmosContainer container = db.getContainer(CONTAINER); + List families = new ArrayList<>(); + families.add(Families.getAndersenFamilyItem()); + families.add(Families.getJohnsonFamilyItem()); + families.add(Families.getSmithFamilyItem()); + families.add(Families.getWakefieldFamilyItem()); + + CosmosItemRequestOptions cosmosItemRequestOptions = new CosmosItemRequestOptions(); + for (Family f : families) { + container.createItem(f, new PartitionKey(f.getLastName()), cosmosItemRequestOptions); + } + } + + @AfterClass + public static void afterClass() { + container.stop(); + if (client != null) { + client.close(); + } + } + + @Rule public TestPipeline pipeline = TestPipeline.fromOptions(pipelineOptions); + + @Test + public void testEstimatedSizeBytes() throws Exception { + Read read = + CosmosIO.read(Family.class) + .withContainer(CONTAINER) + .withDatabase(DATABASE) + .withCoder(SerializableCoder.of(Family.class)); + + BoundedCosmosBDSource initialSource = new BoundedCosmosBDSource<>(read); + // Cosmos DB precision is in KB. Inserted test data is ~3KB + long estimatedSize = initialSource.getEstimatedSizeBytes(pipelineOptions); + assertEquals("Wrong estimated size", 3072, estimatedSize); + } + + @Test + public void testSplit() throws Exception { + Read read = + CosmosIO.read(Family.class) + .withContainer(CONTAINER) + .withDatabase(DATABASE) + .withCoder(SerializableCoder.of(Family.class)); + + BoundedCosmosBDSource initialSource = new BoundedCosmosBDSource<>(read); + // Cosmos DB precision is in KB. Inserted test data is ~3KB + List> splits = initialSource.split(1024, pipelineOptions); + assertEquals("Wrong split", 3, splits.size()); + } + + @Test + public void testRead() { + PCollection output = + pipeline.apply( + CosmosIO.read(Family.class) + .withContainer(CONTAINER) + .withDatabase(DATABASE) + .withCoder(SerializableCoder.of(Family.class))); + + PAssert.thatSingleton(output.apply("Count", Count.globally())).isEqualTo(4L); + pipeline.run(); + } + + @Test + public void testReadWithQuery() { + PCollection output = + pipeline.apply( + CosmosIO.read(Family.class) + .withContainer(CONTAINER) + .withDatabase(DATABASE) + .withCoder(SerializableCoder.of(Family.class)) + .withQuery( + "SELECT * FROM Family WHERE Family.lastName IN ('Andersen', 'Wakefield', 'Johnson')")); + + PAssert.thatSingleton(output.apply("Count", Count.globally())).isEqualTo(3L); + pipeline.run(); + } +} diff --git a/sdks/java/io/azure/README.md b/sdks/java/io/azure/README.md deleted file mode 100644 index b589baa57eac9..0000000000000 --- a/sdks/java/io/azure/README.md +++ /dev/null @@ -1,19 +0,0 @@ - - -This directory contains an Azure IO module to read and write from Azure Blob Storage with Beam. diff --git a/sdks/java/io/azure/build.gradle b/sdks/java/io/azure/build.gradle index 2b9163b3a3402..8504ef6fcef29 100644 --- a/sdks/java/io/azure/build.gradle +++ b/sdks/java/io/azure/build.gradle @@ -26,20 +26,27 @@ applyJavaNature( description = "Apache Beam :: SDKs :: Java :: IO :: Azure" ext.summary = "IO library to read and write Azure services from Beam." +def excludeNetty = { + exclude group: "io.netty", module: "*" // exclude more recent Netty version +} + dependencies { + implementation platform(library.java.azure_sdk_bom) implementation library.java.commons_io permitUnusedDeclared library.java.commons_io // BEAM-11761 - implementation library.java.slf4j_api - implementation library.java.vendored_guava_26_0_jre - implementation project(path: ":sdks:java:core", configuration: "shadow") - implementation "com.azure:azure-storage-blob:12.10.0" - implementation "com.azure:azure-identity:1.0.8" - implementation "com.microsoft.azure:azure-storage:8.6.5" - implementation "com.azure:azure-core:1.9.0" - implementation "com.azure:azure-storage-common:12.10.0" implementation library.java.jackson_annotations implementation library.java.jackson_core implementation library.java.jackson_databind + implementation library.java.slf4j_api + implementation library.java.vendored_guava_32_1_2_jre + implementation "com.azure:azure-core", excludeNetty + implementation "com.azure:azure-identity", excludeNetty + implementation "com.azure:azure-storage-blob", excludeNetty + implementation "com.azure:azure-storage-common", excludeNetty + implementation "com.microsoft.azure:azure-storage:8.6.5" + implementation project(path: ":sdks:java:core", configuration: "shadow") + runtimeOnly library.java.netty_all // force version of netty used by Beam + testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") testImplementation library.java.mockito_core testImplementation library.java.junit diff --git a/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/AzfsResourceId.java b/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/AzfsResourceId.java index 1723b9d4cea06..592b09c397185 100644 --- a/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/AzfsResourceId.java +++ b/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/AzfsResourceId.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.azure.blobstore; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Date; import java.util.Objects; @@ -26,8 +26,8 @@ import java.util.regex.Pattern; import org.apache.beam.sdk.io.fs.ResolveOptions; import org.apache.beam.sdk.io.fs.ResourceId; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.checkerframework.checker.nullness.qual.Nullable; @SuppressWarnings({ diff --git a/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/AzureBlobStoreFileSystem.java b/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/AzureBlobStoreFileSystem.java index 803c481f1a6d1..3b6e79b4ef7cc 100644 --- a/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/AzureBlobStoreFileSystem.java +++ b/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/AzureBlobStoreFileSystem.java @@ -19,9 +19,9 @@ import static java.nio.channels.Channels.newChannel; import static org.apache.beam.sdk.io.FileSystemUtils.wildcardToRegexp; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.azure.core.http.rest.PagedIterable; import com.azure.storage.blob.BlobClient; @@ -54,13 +54,13 @@ import org.apache.beam.sdk.io.fs.MatchResult; import org.apache.beam.sdk.io.fs.MoveOptions; import org.apache.beam.sdk.util.InstanceBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Suppliers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Suppliers; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/AzureBlobStoreFileSystemRegistrar.java b/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/AzureBlobStoreFileSystemRegistrar.java index 0373cdfe79609..6cd4394a57bdd 100644 --- a/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/AzureBlobStoreFileSystemRegistrar.java +++ b/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/AzureBlobStoreFileSystemRegistrar.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.azure.blobstore; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.service.AutoService; import javax.annotation.Nonnull; @@ -25,7 +25,7 @@ import org.apache.beam.sdk.io.FileSystemRegistrar; import org.apache.beam.sdk.io.azure.options.BlobstoreOptions; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** {@link AutoService} registrar for the {@link AzureBlobStoreFileSystem}. */ @AutoService(FileSystemRegistrar.class) diff --git a/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/AzureReadableSeekableByteChannel.java b/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/AzureReadableSeekableByteChannel.java index 458198de80801..9123fb9d15b12 100644 --- a/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/AzureReadableSeekableByteChannel.java +++ b/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/AzureReadableSeekableByteChannel.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.azure.blobstore; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.specialized.BlobInputStream; diff --git a/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/DefaultBlobstoreClientBuilderFactory.java b/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/DefaultBlobstoreClientBuilderFactory.java index 03bab36af1489..775c721fe037e 100644 --- a/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/DefaultBlobstoreClientBuilderFactory.java +++ b/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/DefaultBlobstoreClientBuilderFactory.java @@ -21,7 +21,7 @@ import com.azure.storage.common.StorageSharedKeyCredential; import org.apache.beam.sdk.io.azure.options.BlobstoreClientBuilderFactory; import org.apache.beam.sdk.io.azure.options.BlobstoreOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; /** Construct BlobServiceClientBuilder with given values of Azure client properties. */ @SuppressWarnings({ diff --git a/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/options/AzureOptions.java b/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/options/AzureOptions.java new file mode 100644 index 0000000000000..c4793300fe48b --- /dev/null +++ b/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/options/AzureOptions.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.azure.options; + +import com.azure.core.credential.TokenCredential; +import com.azure.identity.DefaultAzureCredentialBuilder; +import org.apache.beam.sdk.options.Default; +import org.apache.beam.sdk.options.DefaultValueFactory; +import org.apache.beam.sdk.options.Description; +import org.apache.beam.sdk.options.PipelineOptions; + +public interface AzureOptions extends PipelineOptions { + + /* Refer to {@link DefaultAWSCredentialsProviderChain} Javadoc for usage help. */ + + /** + * The credential instance that should be used to authenticate against Azure services. The option + * value must contain a "@type" field and an Azure credentials provider class as the field value. + * + *

    For example, to specify the Azure client id, tenant id, and client secret, specify the + * following: + * {"@type" : "ClientSecretCredential", "azureClientId": "client_id_value", + * "azureTenantId": "tenant_id_value", "azureClientSecret": "client_secret_value"} + * + */ + @Description( + "The credential instance that should be used to authenticate " + + "against Azure services. The option value must contain \"@type\" field " + + "and an Azure credentials provider class name as the field value. " + + " For example, to specify the Azure client id, tenant id, and client secret, specify the following: " + + "{\"@type\" : \"ClientSecretCredential\", \"azureClientId\": \"client_id_value\", " + + "\"azureTenantId\": \"tenant_id_value\", \"azureClientSecret\": \"client_secret_value\"}") + @Default.InstanceFactory(AzureUserCredentialsFactory.class) + TokenCredential getAzureCredentialsProvider(); + + void setAzureCredentialsProvider(TokenCredential value); + + /** Attempts to load Azure credentials. */ + class AzureUserCredentialsFactory implements DefaultValueFactory { + + @Override + public TokenCredential create(PipelineOptions options) { + return new DefaultAzureCredentialBuilder().build(); + } + } +} diff --git a/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/options/AzurePipelineOptionsRegistrar.java b/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/options/AzurePipelineOptionsRegistrar.java index 353792d52da05..ca475a3659406 100644 --- a/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/options/AzurePipelineOptionsRegistrar.java +++ b/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/options/AzurePipelineOptionsRegistrar.java @@ -20,7 +20,7 @@ import com.google.auto.service.AutoService; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** A registrar containing the default Azure options. */ @AutoService(PipelineOptionsRegistrar.class) diff --git a/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/options/BlobstoreOptions.java b/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/options/BlobstoreOptions.java index b5924ebd3beea..7471452dfce01 100644 --- a/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/options/BlobstoreOptions.java +++ b/sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/options/BlobstoreOptions.java @@ -17,23 +17,19 @@ */ package org.apache.beam.sdk.io.azure.options; -import com.azure.core.credential.TokenCredential; import com.azure.core.http.HttpClient; import com.azure.core.http.HttpPipeline; import com.azure.core.http.policy.HttpPipelinePolicy; -import com.azure.identity.DefaultAzureCredentialBuilder; import com.azure.storage.blob.models.CustomerProvidedKey; import org.apache.beam.sdk.io.azure.blobstore.DefaultBlobstoreClientBuilderFactory; import org.apache.beam.sdk.options.Default; -import org.apache.beam.sdk.options.DefaultValueFactory; import org.apache.beam.sdk.options.Description; -import org.apache.beam.sdk.options.PipelineOptions; import org.checkerframework.checker.nullness.qual.Nullable; // TODO: Tag each option with @Default or @Nullable /** Options used to configure Microsoft Azure Blob Storage. */ -public interface BlobstoreOptions extends PipelineOptions { +public interface BlobstoreOptions extends AzureOptions { @Description( "Factory class that should be created and used to create a builder of Azure Blobstore client." @@ -95,37 +91,4 @@ void setBlobstoreClientFactoryClass( HttpPipeline getHttpPipeline(); void setHttpPipeline(HttpPipeline httpPipeline); - - /* Refer to {@link DefaultAWSCredentialsProviderChain} Javadoc for usage help. */ - - /** - * The credential instance that should be used to authenticate against Azure services. The option - * value must contain a "@type" field and an Azure credentials provider class as the field value. - * - *

    For example, to specify the Azure client id, tenant id, and client secret, specify the - * following: - * {"@type" : "ClientSecretCredential", "azureClientId": "client_id_value", - * "azureTenantId": "tenant_id_value", "azureClientSecret": "client_secret_value"} - * - */ - @Description( - "The credential instance that should be used to authenticate " - + "against Azure services. The option value must contain \"@type\" field " - + "and an Azure credentials provider class name as the field value. " - + " For example, to specify the Azure client id, tenant id, and client secret, specify the following: " - + "{\"@type\" : \"ClientSecretCredential\", \"azureClientId\": \"client_id_value\", " - + "\"azureTenantId\": \"tenant_id_value\", \"azureClientSecret\": \"client_secret_value\"}") - @Default.InstanceFactory(AzureUserCredentialsFactory.class) - TokenCredential getAzureCredentialsProvider(); - - void setAzureCredentialsProvider(TokenCredential value); - - /** Attempts to load Azure credentials. */ - class AzureUserCredentialsFactory implements DefaultValueFactory { - - @Override - public TokenCredential create(PipelineOptions options) { - return new DefaultAzureCredentialBuilder().build(); - } - } } diff --git a/sdks/java/io/azure/src/test/java/org/apache/beam/sdk/io/azure/blobstore/AzureBlobStoreFileSystemTest.java b/sdks/java/io/azure/src/test/java/org/apache/beam/sdk/io/azure/blobstore/AzureBlobStoreFileSystemTest.java index d418416047843..545f314688c3c 100644 --- a/sdks/java/io/azure/src/test/java/org/apache/beam/sdk/io/azure/blobstore/AzureBlobStoreFileSystemTest.java +++ b/sdks/java/io/azure/src/test/java/org/apache/beam/sdk/io/azure/blobstore/AzureBlobStoreFileSystemTest.java @@ -52,8 +52,8 @@ import org.apache.beam.sdk.io.fs.CreateOptions; import org.apache.beam.sdk.io.fs.MatchResult; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; diff --git a/sdks/java/io/cassandra/OWNERS b/sdks/java/io/cassandra/OWNERS deleted file mode 100644 index 41bc716749345..0000000000000 --- a/sdks/java/io/cassandra/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - jbonofre - - echauchot diff --git a/sdks/java/io/cassandra/build.gradle b/sdks/java/io/cassandra/build.gradle index 7fb97c75a9016..0a1f2f4accdad 100644 --- a/sdks/java/io/cassandra/build.gradle +++ b/sdks/java/io/cassandra/build.gradle @@ -32,7 +32,7 @@ ext.summary = "IO to read and write with Apache Cassandra database" def achilles_version = "6.1.0" dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.slf4j_api implementation library.java.cassandra_driver_core diff --git a/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/CassandraIO.java b/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/CassandraIO.java index 46d7848f48aeb..d33642b9c3ab7 100644 --- a/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/CassandraIO.java +++ b/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/CassandraIO.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.cassandra; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.datastax.driver.core.Cluster; import com.datastax.driver.core.ConsistencyLevel; @@ -54,8 +54,8 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/ReadFn.java b/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/ReadFn.java index 3bb5360291879..b8fff22971219 100644 --- a/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/ReadFn.java +++ b/sdks/java/io/cassandra/src/main/java/org/apache/beam/sdk/io/cassandra/ReadFn.java @@ -29,7 +29,7 @@ import java.util.stream.Collectors; import org.apache.beam.sdk.io.cassandra.CassandraIO.Read; import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraIOTest.java b/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraIOTest.java index a472b9ee1c3a2..a7090d5c7bcc1 100644 --- a/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraIOTest.java +++ b/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraIOTest.java @@ -70,9 +70,9 @@ import org.apache.beam.sdk.transforms.SimpleFunction; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.apache.cassandra.service.StorageServiceMBean; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.AfterClass; diff --git a/sdks/java/io/cdap/OWNERS b/sdks/java/io/cdap/OWNERS deleted file mode 100644 index e60eec69932b0..0000000000000 --- a/sdks/java/io/cdap/OWNERS +++ /dev/null @@ -1 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners diff --git a/sdks/java/io/cdap/build.gradle b/sdks/java/io/cdap/build.gradle index a9122d1c8f6d1..cda4896a37cab 100644 --- a/sdks/java/io/cdap/build.gradle +++ b/sdks/java/io/cdap/build.gradle @@ -58,14 +58,14 @@ dependencies { implementation library.java.slf4j_api implementation library.java.spark_streaming implementation library.java.tephra - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(":sdks:java:io:sparkreceiver:2") implementation project(":sdks:java:io:hadoop-format") testImplementation library.java.cdap_plugin_service_now testImplementation library.java.cdap_etl_api testImplementation library.java.hadoop_common - testImplementation library.java.vendored_guava_26_0_jre + testImplementation library.java.vendored_guava_32_1_2_jre testImplementation library.java.junit testImplementation library.java.mockito_core testImplementation library.java.testcontainers_postgresql diff --git a/sdks/java/io/cdap/src/main/java/org/apache/beam/sdk/io/cdap/CdapIO.java b/sdks/java/io/cdap/src/main/java/org/apache/beam/sdk/io/cdap/CdapIO.java index b0ed62f958f15..2d4c2156d61b9 100644 --- a/sdks/java/io/cdap/src/main/java/org/apache/beam/sdk/io/cdap/CdapIO.java +++ b/sdks/java/io/cdap/src/main/java/org/apache/beam/sdk/io/cdap/CdapIO.java @@ -19,7 +19,7 @@ import static org.apache.beam.sdk.io.cdap.MappingUtils.getPluginByClass; import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import io.cdap.cdap.api.plugin.PluginConfig; diff --git a/sdks/java/io/cdap/src/main/java/org/apache/beam/sdk/io/cdap/MappingUtils.java b/sdks/java/io/cdap/src/main/java/org/apache/beam/sdk/io/cdap/MappingUtils.java index 8ac28a9b6b38f..0aa7711ea2e0e 100644 --- a/sdks/java/io/cdap/src/main/java/org/apache/beam/sdk/io/cdap/MappingUtils.java +++ b/sdks/java/io/cdap/src/main/java/org/apache/beam/sdk/io/cdap/MappingUtils.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.cdap; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import io.cdap.plugin.common.SourceInputFormatProvider; import io.cdap.plugin.hubspot.sink.batch.HubspotBatchSink; diff --git a/sdks/java/io/cdap/src/main/java/org/apache/beam/sdk/io/cdap/Plugin.java b/sdks/java/io/cdap/src/main/java/org/apache/beam/sdk/io/cdap/Plugin.java index 487c2a1930d45..f74a694846321 100644 --- a/sdks/java/io/cdap/src/main/java/org/apache/beam/sdk/io/cdap/Plugin.java +++ b/sdks/java/io/cdap/src/main/java/org/apache/beam/sdk/io/cdap/Plugin.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.cdap; import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import io.cdap.cdap.api.plugin.PluginConfig; diff --git a/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/CdapIOTest.java b/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/CdapIOTest.java index a73df9bdcf8ce..5ca505a799491 100644 --- a/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/CdapIOTest.java +++ b/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/CdapIOTest.java @@ -53,7 +53,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.hadoop.mapreduce.OutputCommitter; import org.joda.time.Duration; import org.junit.Before; diff --git a/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/ConfigWrapperTest.java b/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/ConfigWrapperTest.java index 55f3ab49050f1..67cdbb74162a4 100644 --- a/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/ConfigWrapperTest.java +++ b/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/ConfigWrapperTest.java @@ -25,7 +25,7 @@ import io.cdap.plugin.servicenow.source.util.ServiceNowConstants; import java.io.File; import java.util.Map; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/PluginConfigInstantiationUtilsTest.java b/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/PluginConfigInstantiationUtilsTest.java index 90f4e49880642..d8f87a108199c 100644 --- a/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/PluginConfigInstantiationUtilsTest.java +++ b/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/PluginConfigInstantiationUtilsTest.java @@ -25,7 +25,7 @@ import io.cdap.plugin.servicenow.source.util.ServiceNowConstants; import java.util.HashMap; import java.util.Map; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/PluginTest.java b/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/PluginTest.java index 4881c4573367b..61ced50baaf2d 100644 --- a/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/PluginTest.java +++ b/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/PluginTest.java @@ -26,7 +26,7 @@ import io.cdap.plugin.servicenow.source.ServiceNowSource; import io.cdap.plugin.servicenow.source.ServiceNowSourceConfig; import io.cdap.plugin.servicenow.source.util.ServiceNowConstants; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.hadoop.io.MapWritable; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/batch/EmployeeInputFormatProvider.java b/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/batch/EmployeeInputFormatProvider.java index 6a2f87783f491..77cd91ed8c877 100644 --- a/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/batch/EmployeeInputFormatProvider.java +++ b/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/batch/EmployeeInputFormatProvider.java @@ -23,7 +23,7 @@ import java.util.Map; import org.apache.beam.sdk.io.cdap.CdapIO; import org.apache.beam.sdk.io.cdap.EmployeeConfig; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * {@link InputFormatProvider} for {@link EmployeeBatchSource} CDAP plugin. Used to test {@link diff --git a/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/batch/EmployeeOutputFormatProvider.java b/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/batch/EmployeeOutputFormatProvider.java index a42c0c89aca1e..e175259b1e43d 100644 --- a/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/batch/EmployeeOutputFormatProvider.java +++ b/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/batch/EmployeeOutputFormatProvider.java @@ -23,7 +23,7 @@ import java.util.Map; import org.apache.beam.sdk.io.cdap.CdapIO; import org.apache.beam.sdk.io.cdap.EmployeeConfig; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * {@link OutputFormatProvider} for {@link EmployeeBatchSink} CDAP plugin. Used to test {@link diff --git a/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/context/BatchSinkContextImplTest.java b/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/context/BatchSinkContextImplTest.java index 37a44af03754d..7da129086fcdf 100644 --- a/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/context/BatchSinkContextImplTest.java +++ b/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/context/BatchSinkContextImplTest.java @@ -30,7 +30,7 @@ import java.sql.Timestamp; import java.util.List; import org.apache.beam.sdk.io.cdap.ConfigWrapper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; /** Test class for {@link BatchSinkContextImpl}. */ diff --git a/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/context/BatchSourceContextImplTest.java b/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/context/BatchSourceContextImplTest.java index fb35cc51124a0..4236235307f38 100644 --- a/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/context/BatchSourceContextImplTest.java +++ b/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/context/BatchSourceContextImplTest.java @@ -29,7 +29,7 @@ import java.sql.Timestamp; import java.util.List; import org.apache.beam.sdk.io.cdap.ConfigWrapper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; /** Test class for {@link BatchSourceContextImpl}. */ diff --git a/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/streaming/EmployeeReceiver.java b/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/streaming/EmployeeReceiver.java index fcd0fa7b8d766..6ee53bca4de05 100644 --- a/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/streaming/EmployeeReceiver.java +++ b/sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/streaming/EmployeeReceiver.java @@ -23,7 +23,7 @@ import java.util.concurrent.TimeUnit; import org.apache.beam.sdk.io.cdap.EmployeeConfig; import org.apache.beam.sdk.io.sparkreceiver.HasOffset; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.spark.storage.StorageLevel; import org.apache.spark.streaming.receiver.Receiver; import org.slf4j.Logger; diff --git a/sdks/java/io/clickhouse/OWNERS b/sdks/java/io/clickhouse/OWNERS deleted file mode 100644 index 9070679c580be..0000000000000 --- a/sdks/java/io/clickhouse/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - kanterov diff --git a/sdks/java/io/clickhouse/build.gradle b/sdks/java/io/clickhouse/build.gradle index 3f0de5f62ec0b..d711fb7fa3165 100644 --- a/sdks/java/io/clickhouse/build.gradle +++ b/sdks/java/io/clickhouse/build.gradle @@ -1,3 +1,4 @@ + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -18,7 +19,7 @@ plugins { id 'org.apache.beam.module' - id 'ca.coglinc.javacc' + id 'org.javacc.javacc' } applyJavaNature( automaticModuleName: 'org.apache.beam.sdk.io.clickhouse', @@ -57,7 +58,7 @@ dependencies { implementation library.java.guava implementation library.java.joda_time implementation library.java.slf4j_api - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation "com.clickhouse:clickhouse-jdbc:$clickhouse_jdbc_version:all" testImplementation library.java.slf4j_api testImplementation library.java.junit @@ -65,6 +66,7 @@ dependencies { testImplementation library.java.testcontainers_clickhouse testRuntimeOnly library.java.slf4j_jdk14 testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") + testImplementation project(path: ":sdks:java:extensions:avro", configuration: "testRuntimeMigration") } processTestResources { diff --git a/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseIO.java b/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseIO.java index db04d36d00924..7b3a3e8974b95 100644 --- a/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseIO.java +++ b/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseIO.java @@ -49,8 +49,8 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.slf4j.Logger; diff --git a/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseWriter.java b/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseWriter.java index 525cb40a7f0d7..97eea2b825580 100644 --- a/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseWriter.java +++ b/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseWriter.java @@ -24,8 +24,8 @@ import java.util.List; import org.apache.beam.sdk.io.clickhouse.TableSchema.ColumnType; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.joda.time.Days; import org.joda.time.Instant; import org.joda.time.ReadableInstant; diff --git a/sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/TableSchemaTest.java b/sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/TableSchemaTest.java index f479871440477..bcc2c5c287ad7 100644 --- a/sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/TableSchemaTest.java +++ b/sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/TableSchemaTest.java @@ -22,7 +22,7 @@ import java.util.Map; import org.apache.beam.sdk.io.clickhouse.TableSchema.ColumnType; import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; /** Tests for {@link TableSchema}. */ diff --git a/sdks/java/io/common/OWNERS b/sdks/java/io/common/OWNERS deleted file mode 100644 index 5dc24af7d702e..0000000000000 --- a/sdks/java/io/common/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - chamikaramj - - timrobertson100 diff --git a/sdks/java/io/common/build.gradle b/sdks/java/io/common/build.gradle index f5d16d0b4ae0f..3b0ef6ffd8104 100644 --- a/sdks/java/io/common/build.gradle +++ b/sdks/java/io/common/build.gradle @@ -23,7 +23,7 @@ description = "Apache Beam :: SDKs :: Java :: IO :: Common" ext.summary = "Code used by all Beam IOs" dependencies { - testImplementation library.java.vendored_guava_26_0_jre + testImplementation library.java.vendored_guava_32_1_2_jre testImplementation library.java.junit testImplementation library.java.hamcrest testImplementation library.java.postgres diff --git a/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/DatabaseTestHelper.java b/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/DatabaseTestHelper.java index 1204fa7ada61e..25ff7727cd4ca 100644 --- a/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/DatabaseTestHelper.java +++ b/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/DatabaseTestHelper.java @@ -39,7 +39,7 @@ import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.Sleeper; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.postgresql.ds.PGSimpleDataSource; import org.testcontainers.containers.JdbcDatabaseContainer; diff --git a/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/HashingFn.java b/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/HashingFn.java index 5107f264345ed..f69225eaac24f 100644 --- a/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/HashingFn.java +++ b/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/HashingFn.java @@ -28,9 +28,9 @@ import org.apache.beam.sdk.coders.CoderRegistry; import org.apache.beam.sdk.coders.SerializableCoder; import org.apache.beam.sdk.transforms.Combine.CombineFn; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashCode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.HashCode; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; import org.checkerframework.checker.nullness.qual.Nullable; /** Custom Function for Hashing. The combiner is combineUnordered, and accumulator is a HashCode. */ diff --git a/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/TestRow.java b/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/TestRow.java index f8e86caf98038..94cea2a602b2d 100644 --- a/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/TestRow.java +++ b/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/TestRow.java @@ -24,7 +24,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** Used to pass values around within test pipelines. */ @AutoValue diff --git a/sdks/java/io/components/build.gradle b/sdks/java/io/components/build.gradle index 3bf987d82bccd..5b2ad69fb0b0e 100644 --- a/sdks/java/io/components/build.gradle +++ b/sdks/java/io/components/build.gradle @@ -29,7 +29,7 @@ dependencies { implementation library.java.protobuf_java permitUnusedDeclared library.java.protobuf_java // BEAM-11761 implementation library.java.slf4j_api - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.vendored_grpc_1_54_0 testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") diff --git a/sdks/java/io/contextualtextio/build.gradle b/sdks/java/io/contextualtextio/build.gradle index afb1ac1f4590d..0557a1dfa259b 100644 --- a/sdks/java/io/contextualtextio/build.gradle +++ b/sdks/java/io/contextualtextio/build.gradle @@ -29,10 +29,11 @@ dependencies { implementation library.java.protobuf_java permitUnusedDeclared library.java.protobuf_java // BEAM-11761 implementation library.java.slf4j_api - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.vendored_grpc_1_54_0 testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") + testImplementation library.java.commons_compress testImplementation library.java.guava_testlib testImplementation library.java.junit testImplementation library.java.hamcrest diff --git a/sdks/java/io/contextualtextio/src/main/java/org/apache/beam/sdk/io/contextualtextio/ContextualTextIO.java b/sdks/java/io/contextualtextio/src/main/java/org/apache/beam/sdk/io/contextualtextio/ContextualTextIO.java index 81b8195a4179f..7f3d72cc7c57c 100644 --- a/sdks/java/io/contextualtextio/src/main/java/org/apache/beam/sdk/io/contextualtextio/ContextualTextIO.java +++ b/sdks/java/io/contextualtextio/src/main/java/org/apache/beam/sdk/io/contextualtextio/ContextualTextIO.java @@ -18,8 +18,8 @@ package org.apache.beam.sdk.io.contextualtextio; import static org.apache.beam.sdk.io.FileIO.ReadMatches.DirectoryTreatment; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import java.io.Serializable; @@ -72,9 +72,9 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.slf4j.Logger; diff --git a/sdks/java/io/contextualtextio/src/main/java/org/apache/beam/sdk/io/contextualtextio/ContextualTextIOSource.java b/sdks/java/io/contextualtextio/src/main/java/org/apache/beam/sdk/io/contextualtextio/ContextualTextIOSource.java index dd4a8191fe38b..9be8b5cb4b9bb 100644 --- a/sdks/java/io/contextualtextio/src/main/java/org/apache/beam/sdk/io/contextualtextio/ContextualTextIOSource.java +++ b/sdks/java/io/contextualtextio/src/main/java/org/apache/beam/sdk/io/contextualtextio/ContextualTextIOSource.java @@ -31,8 +31,8 @@ import org.apache.beam.sdk.schemas.SchemaCoder; import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/io/contextualtextio/src/main/java/org/apache/beam/sdk/io/contextualtextio/RecordWithMetadata.java b/sdks/java/io/contextualtextio/src/main/java/org/apache/beam/sdk/io/contextualtextio/RecordWithMetadata.java index eb24024e68530..597cc834d2f29 100644 --- a/sdks/java/io/contextualtextio/src/main/java/org/apache/beam/sdk/io/contextualtextio/RecordWithMetadata.java +++ b/sdks/java/io/contextualtextio/src/main/java/org/apache/beam/sdk/io/contextualtextio/RecordWithMetadata.java @@ -23,7 +23,7 @@ import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.Schema.LogicalType; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/io/contextualtextio/src/test/java/org/apache/beam/sdk/io/contextualtextio/ContextualTextIOTest.java b/sdks/java/io/contextualtextio/src/test/java/org/apache/beam/sdk/io/contextualtextio/ContextualTextIOTest.java index ee57c3c11ac3b..48904cae430f1 100644 --- a/sdks/java/io/contextualtextio/src/test/java/org/apache/beam/sdk/io/contextualtextio/ContextualTextIOTest.java +++ b/sdks/java/io/contextualtextio/src/test/java/org/apache/beam/sdk/io/contextualtextio/ContextualTextIOTest.java @@ -87,10 +87,10 @@ import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.apache.commons.compress.compressors.deflate.DeflateCompressorOutputStream; import org.joda.time.Duration; diff --git a/sdks/java/io/csv/build.gradle b/sdks/java/io/csv/build.gradle index 6c4c56901bc07..2be8f59d1f399 100644 --- a/sdks/java/io/csv/build.gradle +++ b/sdks/java/io/csv/build.gradle @@ -27,7 +27,7 @@ ext.summary = "IO to read and write CSV files." dependencies { implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.commons_csv - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") testImplementation library.java.junit testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") diff --git a/sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvIO.java b/sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvIO.java index cfe51ea656ec9..5bed0186e0d67 100644 --- a/sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvIO.java +++ b/sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvIO.java @@ -20,7 +20,7 @@ import static java.util.Objects.requireNonNull; import static org.apache.beam.sdk.values.TypeDescriptors.rows; import static org.apache.beam.sdk.values.TypeDescriptors.strings; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.util.ArrayList; @@ -44,7 +44,7 @@ import org.apache.beam.sdk.transforms.display.HasDisplayData; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.apache.commons.csv.CSVFormat; /** diff --git a/sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvRowConversions.java b/sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvRowConversions.java index 6faccfe021f67..01d944c2bad15 100644 --- a/sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvRowConversions.java +++ b/sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvRowConversions.java @@ -18,8 +18,8 @@ package org.apache.beam.sdk.io.csv; import static org.apache.beam.sdk.io.csv.CsvIO.VALID_FIELD_TYPE_SET; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import java.util.ArrayList; diff --git a/sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/providers/CsvWriteTransformProvider.java b/sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/providers/CsvWriteTransformProvider.java new file mode 100644 index 0000000000000..4e07a06197f57 --- /dev/null +++ b/sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/providers/CsvWriteTransformProvider.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.csv.providers; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import com.google.auto.service.AutoService; +import com.google.auto.value.AutoValue; +import java.util.Collections; +import java.util.List; +import org.apache.beam.sdk.io.WriteFilesResult; +import org.apache.beam.sdk.io.csv.CsvIO; +import org.apache.beam.sdk.schemas.AutoValueSchema; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.schemas.Schema.Field; +import org.apache.beam.sdk.schemas.Schema.FieldType; +import org.apache.beam.sdk.schemas.annotations.DefaultSchema; +import org.apache.beam.sdk.schemas.annotations.SchemaFieldDescription; +import org.apache.beam.sdk.schemas.transforms.SchemaTransform; +import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; +import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.values.PCollectionRowTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TypeDescriptors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.commons.csv.CSVFormat; + +/** + * An implementation of {@link TypedSchemaTransformProvider} for {@link CsvIO#write}. + * + *

    Internal only: This class is actively being worked on, and it will likely change. We + * provide no backwards compatibility guarantees, and it should not be implemented outside the Beam + * repository. + */ +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) +@AutoService(SchemaTransformProvider.class) +public class CsvWriteTransformProvider + extends TypedSchemaTransformProvider { + private static final String INPUT_ROWS_TAG = "input"; + private static final String WRITE_RESULTS = "output"; + + @Override + protected Class configurationClass() { + return CsvWriteConfiguration.class; + } + + @Override + protected SchemaTransform from(CsvWriteConfiguration configuration) { + return new CsvWriteTransform(configuration); + } + + @Override + public String identifier() { + return String.format("beam:schematransform:org.apache.beam:csv_write:v1"); + } + + @Override + public List inputCollectionNames() { + return Collections.singletonList(INPUT_ROWS_TAG); + } + + @Override + public List outputCollectionNames() { + return Collections.singletonList(WRITE_RESULTS); + } + + /** Configuration for writing to BigQuery with Storage Write API. */ + @DefaultSchema(AutoValueSchema.class) + @AutoValue + public abstract static class CsvWriteConfiguration { + + public void validate() { + checkArgument( + !Strings.isNullOrEmpty(this.getPath()), "Path for a CSV Write must be specified."); + } + + public static Builder builder() { + return new AutoValue_CsvWriteTransformProvider_CsvWriteConfiguration.Builder(); + } + + @SchemaFieldDescription("The file path to write to.") + public abstract String getPath(); + + /** Builder for {@link CsvWriteConfiguration}. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setPath(String path); + + /** Builds a {@link CsvWriteConfiguration} instance. */ + public abstract CsvWriteConfiguration build(); + } + } + + /** A {@link SchemaTransform} for {@link CsvIO#write}. */ + protected static class CsvWriteTransform extends SchemaTransform { + + private final CsvWriteConfiguration configuration; + + CsvWriteTransform(CsvWriteConfiguration configuration) { + configuration.validate(); + this.configuration = configuration; + } + + @Override + public PCollectionRowTuple expand(PCollectionRowTuple input) { + WriteFilesResult result = + input + .get(INPUT_ROWS_TAG) + .apply(CsvIO.writeRows(configuration.getPath(), CSVFormat.DEFAULT).withSuffix("")); + Schema outputSchema = Schema.of(Field.of("filename", FieldType.STRING)); + return PCollectionRowTuple.of( + WRITE_RESULTS, + result + .getPerDestinationOutputFilenames() + .apply( + "Collect filenames", + MapElements.into(TypeDescriptors.rows()) + .via( + (destinationAndRow) -> + Row.withSchema(outputSchema) + .withFieldValue("filename", destinationAndRow.getValue()) + .build())) + .setRowSchema(outputSchema)); + } + } +} diff --git a/sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/providers/package-info.java b/sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/providers/package-info.java new file mode 100644 index 0000000000000..646e69b7cb8c0 --- /dev/null +++ b/sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/providers/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Transforms for reading and writing CSV files. */ +package org.apache.beam.sdk.io.csv.providers; diff --git a/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOWriteDisplayDataTest.java b/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOWriteDisplayDataTest.java index efd43e9c60c03..54cdd285f48f5 100644 --- a/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOWriteDisplayDataTest.java +++ b/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOWriteDisplayDataTest.java @@ -22,7 +22,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.beam.sdk.transforms.display.DisplayData; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.commons.csv.CSVFormat; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOWriteTest.java b/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOWriteTest.java index a35b97a00deed..1bd17b12028bb 100644 --- a/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOWriteTest.java +++ b/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOWriteTest.java @@ -21,7 +21,7 @@ import static org.apache.beam.sdk.io.common.SchemaAwareJavaBeans.ALL_PRIMITIVE_DATA_TYPES_SCHEMA; import static org.apache.beam.sdk.io.common.SchemaAwareJavaBeans.allPrimitiveDataTypes; import static org.apache.beam.sdk.io.csv.CsvIOTestData.DATA; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThrows; @@ -40,7 +40,7 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; import org.apache.commons.csv.CSVFormat; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/RowToCsvCSVFormatTest.java b/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/RowToCsvCSVFormatTest.java index c65252dc79cfd..bedc3073b7b96 100644 --- a/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/RowToCsvCSVFormatTest.java +++ b/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/RowToCsvCSVFormatTest.java @@ -21,7 +21,7 @@ import static org.apache.beam.sdk.io.common.SchemaAwareJavaBeans.NULLABLE_ALL_PRIMITIVE_DATA_TYPES_SCHEMA; import static org.apache.beam.sdk.io.common.SchemaAwareJavaBeans.TIME_CONTAINING_SCHEMA; import static org.apache.beam.sdk.io.csv.CsvIOTestData.DATA; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; diff --git a/sdks/java/io/debezium/build.gradle b/sdks/java/io/debezium/build.gradle index 6cb019d577272..121a435159265 100644 --- a/sdks/java/io/debezium/build.gradle +++ b/sdks/java/io/debezium/build.gradle @@ -31,7 +31,7 @@ description = "Apache Beam :: SDKs :: Java :: IO :: Debezium" ext.summary = "Library to work with Debezium data." dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.vendored_grpc_1_54_0 implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.slf4j_api diff --git a/sdks/java/io/debezium/src/main/java/org/apache/beam/io/debezium/DebeziumIO.java b/sdks/java/io/debezium/src/main/java/org/apache/beam/io/debezium/DebeziumIO.java index 59b72f146793c..30ad8a5f9f74e 100644 --- a/sdks/java/io/debezium/src/main/java/org/apache/beam/io/debezium/DebeziumIO.java +++ b/sdks/java/io/debezium/src/main/java/org/apache/beam/io/debezium/DebeziumIO.java @@ -17,7 +17,7 @@ */ package org.apache.beam.io.debezium; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.io.Serializable; @@ -34,9 +34,9 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.apache.kafka.connect.source.SourceConnector; import org.apache.kafka.connect.source.SourceRecord; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/sdks/java/io/debezium/src/main/java/org/apache/beam/io/debezium/DebeziumReadSchemaTransformProvider.java b/sdks/java/io/debezium/src/main/java/org/apache/beam/io/debezium/DebeziumReadSchemaTransformProvider.java index 6579a3899d95d..9f227708e5e63 100644 --- a/sdks/java/io/debezium/src/main/java/org/apache/beam/io/debezium/DebeziumReadSchemaTransformProvider.java +++ b/sdks/java/io/debezium/src/main/java/org/apache/beam/io/debezium/DebeziumReadSchemaTransformProvider.java @@ -33,11 +33,10 @@ import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; -import org.checkerframework.checker.nullness.qual.UnknownKeyFor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,86 +86,75 @@ protected DebeziumReadSchemaTransformProvider( // TODO(pabloem): Validate configuration parameters to ensure formatting is correct. return new SchemaTransform() { @Override - public @UnknownKeyFor @NonNull @Initialized PTransform< - @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple, - @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple> - buildTransform() { - return new PTransform() { - @Override - public PCollectionRowTuple expand(PCollectionRowTuple input) { - // TODO(pabloem): Test this behavior - Collection connectors = - Arrays.stream(Connectors.values()) - .map(Object::toString) - .collect(Collectors.toSet()); - if (!connectors.contains(configuration.getDatabase())) { - throw new IllegalArgumentException( - "Unsupported database " - + configuration.getDatabase() - + ". Unable to select a JDBC driver for it. Supported Databases are: " - + String.join(", ", connectors)); - } - Class connectorClass = - Objects.requireNonNull(Connectors.valueOf(configuration.getDatabase())) - .getConnector(); - DebeziumIO.ConnectorConfiguration connectorConfiguration = - DebeziumIO.ConnectorConfiguration.create() - .withUsername(configuration.getUsername()) - .withPassword(configuration.getPassword()) - .withHostName(configuration.getHost()) - .withPort(Integer.toString(configuration.getPort())) - .withConnectorClass(connectorClass); - connectorConfiguration = - connectorConfiguration - .withConnectionProperty("table.include.list", configuration.getTable()) - .withConnectionProperty("include.schema.changes", "false") - .withConnectionProperty("database.server.name", "beam-pipeline-server"); - if (configuration.getDatabase().equals("POSTGRES")) { - LOG.info( - "As Database is POSTGRES, we set the `database.dbname` property to {}.", + public PCollectionRowTuple expand(PCollectionRowTuple input) { + // TODO(pabloem): Test this behavior + Collection connectors = + Arrays.stream(Connectors.values()).map(Object::toString).collect(Collectors.toSet()); + if (!connectors.contains(configuration.getDatabase())) { + throw new IllegalArgumentException( + "Unsupported database " + + configuration.getDatabase() + + ". Unable to select a JDBC driver for it. Supported Databases are: " + + String.join(", ", connectors)); + } + Class connectorClass = + Objects.requireNonNull(Connectors.valueOf(configuration.getDatabase())).getConnector(); + DebeziumIO.ConnectorConfiguration connectorConfiguration = + DebeziumIO.ConnectorConfiguration.create() + .withUsername(configuration.getUsername()) + .withPassword(configuration.getPassword()) + .withHostName(configuration.getHost()) + .withPort(Integer.toString(configuration.getPort())) + .withConnectorClass(connectorClass); + connectorConfiguration = + connectorConfiguration + .withConnectionProperty("table.include.list", configuration.getTable()) + .withConnectionProperty("include.schema.changes", "false") + .withConnectionProperty("database.server.name", "beam-pipeline-server"); + if (configuration.getDatabase().equals("POSTGRES")) { + LOG.info( + "As Database is POSTGRES, we set the `database.dbname` property to {}.", + configuration.getTable().substring(0, configuration.getTable().indexOf("."))); + connectorConfiguration = + connectorConfiguration.withConnectionProperty( + "database.dbname", configuration.getTable().substring(0, configuration.getTable().indexOf("."))); - connectorConfiguration = - connectorConfiguration.withConnectionProperty( - "database.dbname", - configuration.getTable().substring(0, configuration.getTable().indexOf("."))); - } - - final List debeziumConnectionProperties = - configuration.getDebeziumConnectionProperties(); - if (debeziumConnectionProperties != null) { - for (String connectionProperty : debeziumConnectionProperties) { - String[] parts = connectionProperty.split("=", -1); - String key = parts[0]; - String value = parts[1]; - connectorConfiguration.withConnectionProperty(key, value); - } - } - - DebeziumIO.Read readTransform = - DebeziumIO.read().withConnectorConfiguration(connectorConfiguration); - - if (isTest) { - readTransform = - readTransform - .withMaxNumberOfRecords(testLimitRecords) - .withMaxTimeToRun(testLimitMilliseconds); - } - - // TODO(pabloem): Database connection issues can be debugged here. - Schema recordSchema = readTransform.getRecordSchema(); - LOG.info( - "Computed schema for table {} from {}: {}", - configuration.getTable(), - configuration.getDatabase(), - recordSchema); - SourceRecordMapper formatFn = - KafkaConnectUtils.beamRowFromSourceRecordFn(recordSchema); - readTransform = - readTransform.withFormatFunction(formatFn).withCoder(RowCoder.of(recordSchema)); - - return PCollectionRowTuple.of("output", input.getPipeline().apply(readTransform)); + } + + final List debeziumConnectionProperties = + configuration.getDebeziumConnectionProperties(); + if (debeziumConnectionProperties != null) { + for (String connectionProperty : debeziumConnectionProperties) { + String[] parts = connectionProperty.split("=", -1); + String key = parts[0]; + String value = parts[1]; + connectorConfiguration.withConnectionProperty(key, value); } - }; + } + + DebeziumIO.Read readTransform = + DebeziumIO.read().withConnectorConfiguration(connectorConfiguration); + + if (isTest) { + readTransform = + readTransform + .withMaxNumberOfRecords(testLimitRecords) + .withMaxTimeToRun(testLimitMilliseconds); + } + + // TODO(pabloem): Database connection issues can be debugged here. + Schema recordSchema = readTransform.getRecordSchema(); + LOG.info( + "Computed schema for table {} from {}: {}", + configuration.getTable(), + configuration.getDatabase(), + recordSchema); + SourceRecordMapper formatFn = + KafkaConnectUtils.beamRowFromSourceRecordFn(recordSchema); + readTransform = + readTransform.withFormatFunction(formatFn).withCoder(RowCoder.of(recordSchema)); + + return PCollectionRowTuple.of("output", input.getPipeline().apply(readTransform)); } }; } diff --git a/sdks/java/io/debezium/src/main/java/org/apache/beam/io/debezium/DebeziumTransformRegistrar.java b/sdks/java/io/debezium/src/main/java/org/apache/beam/io/debezium/DebeziumTransformRegistrar.java index 7b5c39fadeb9a..47fc97849ee1d 100644 --- a/sdks/java/io/debezium/src/main/java/org/apache/beam/io/debezium/DebeziumTransformRegistrar.java +++ b/sdks/java/io/debezium/src/main/java/org/apache/beam/io/debezium/DebeziumTransformRegistrar.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** Exposes {@link DebeziumIO.Read} as an external transform for cross-language usage. */ diff --git a/sdks/java/io/debezium/src/main/java/org/apache/beam/io/debezium/KafkaSourceConsumerFn.java b/sdks/java/io/debezium/src/main/java/org/apache/beam/io/debezium/KafkaSourceConsumerFn.java index 96955afdc8605..0c9632a69926e 100644 --- a/sdks/java/io/debezium/src/main/java/org/apache/beam/io/debezium/KafkaSourceConsumerFn.java +++ b/sdks/java/io/debezium/src/main/java/org/apache/beam/io/debezium/KafkaSourceConsumerFn.java @@ -44,8 +44,8 @@ import org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimator; import org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimators; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.kafka.connect.source.SourceConnector; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceTask; @@ -400,15 +400,13 @@ public boolean tryClaim(Map position) { this.restriction.fetchedRecords = fetchedRecords; LOG.debug("-------------- History: {}", this.restriction.history); - if (this.restriction.maxRecords == null && this.restriction.milisToRun == -1) { - return true; - } - // If we've reached the maximum number of records OR the maximum time, we reject // the attempt to claim. // If we've reached neither, then we continue approve the claim. return (this.restriction.maxRecords == null || fetchedRecords < this.restriction.maxRecords) - && (this.restriction.milisToRun == null || elapsedTime < this.restriction.milisToRun); + && (this.restriction.milisToRun == null + || this.restriction.milisToRun == -1 + || elapsedTime < this.restriction.milisToRun); } @Override diff --git a/sdks/java/io/debezium/src/test/java/org/apache/beam/io/debezium/DebeziumIOMySqlConnectorIT.java b/sdks/java/io/debezium/src/test/java/org/apache/beam/io/debezium/DebeziumIOMySqlConnectorIT.java index 5a5dedfe2e6ee..12ba57bad45d3 100644 --- a/sdks/java/io/debezium/src/test/java/org/apache/beam/io/debezium/DebeziumIOMySqlConnectorIT.java +++ b/sdks/java/io/debezium/src/test/java/org/apache/beam/io/debezium/DebeziumIOMySqlConnectorIT.java @@ -43,7 +43,7 @@ import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; @@ -158,8 +158,7 @@ public void testDebeziumSchemaTransformMysqlRead() throws InterruptedException { .setHost("localhost") .setTable("inventory.customers") .setPort(MY_SQL_CONTAINER.getMappedPort(3306)) - .build()) - .buildTransform()) + .build())) .get("output"); PAssert.that(result) diff --git a/sdks/java/io/debezium/src/test/java/org/apache/beam/io/debezium/DebeziumIOPostgresSqlConnectorIT.java b/sdks/java/io/debezium/src/test/java/org/apache/beam/io/debezium/DebeziumIOPostgresSqlConnectorIT.java index cc6aee97a4a42..2bfa694aebc05 100644 --- a/sdks/java/io/debezium/src/test/java/org/apache/beam/io/debezium/DebeziumIOPostgresSqlConnectorIT.java +++ b/sdks/java/io/debezium/src/test/java/org/apache/beam/io/debezium/DebeziumIOPostgresSqlConnectorIT.java @@ -36,7 +36,7 @@ import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.junit.ClassRule; import org.junit.Test; @@ -131,8 +131,7 @@ public void testDebeziumSchemaTransformPostgresRead() throws InterruptedExceptio .setHost("localhost") .setTable("inventory.customers") .setPort(POSTGRES_SQL_CONTAINER.getMappedPort(5432)) - .build()) - .buildTransform()) + .build())) .get("output"); PAssert.that(result) diff --git a/sdks/java/io/debezium/src/test/java/org/apache/beam/io/debezium/DebeziumReadSchemaTransformTest.java b/sdks/java/io/debezium/src/test/java/org/apache/beam/io/debezium/DebeziumReadSchemaTransformTest.java index 7d206f7da8989..db57974977245 100644 --- a/sdks/java/io/debezium/src/test/java/org/apache/beam/io/debezium/DebeziumReadSchemaTransformTest.java +++ b/sdks/java/io/debezium/src/test/java/org/apache/beam/io/debezium/DebeziumReadSchemaTransformTest.java @@ -104,8 +104,7 @@ private PTransform makePtransform( // is "database.table". .setTable("inventory.customers") .setPort(port) - .build()) - .buildTransform(); + .build()); } @Test diff --git a/sdks/java/io/debezium/src/test/java/org/apache/beam/io/debezium/KafkaSourceConsumerFnTest.java b/sdks/java/io/debezium/src/test/java/org/apache/beam/io/debezium/KafkaSourceConsumerFnTest.java index a5b7ace4fc310..2764b1c87d01a 100644 --- a/sdks/java/io/debezium/src/test/java/org/apache/beam/io/debezium/KafkaSourceConsumerFnTest.java +++ b/sdks/java/io/debezium/src/test/java/org/apache/beam/io/debezium/KafkaSourceConsumerFnTest.java @@ -30,8 +30,8 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.kafka.common.config.AbstractConfig; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.connect.connector.Task; diff --git a/sdks/java/io/elasticsearch-tests/OWNERS b/sdks/java/io/elasticsearch-tests/OWNERS deleted file mode 100644 index 07f4de0afef2c..0000000000000 --- a/sdks/java/io/elasticsearch-tests/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - echauchot - - timrobertson100 diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTestCommon.java b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTestCommon.java index 117e004526a7c..d2e8efe489929 100644 --- a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTestCommon.java +++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTestCommon.java @@ -94,7 +94,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.http.HttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.nio.entity.NStringEntity; diff --git a/sdks/java/io/elasticsearch/OWNERS b/sdks/java/io/elasticsearch/OWNERS deleted file mode 100644 index c4a265bcd8481..0000000000000 --- a/sdks/java/io/elasticsearch/OWNERS +++ /dev/null @@ -1,7 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - egalpin - - echauchot - - jbonofre - - timrobertson100 diff --git a/sdks/java/io/elasticsearch/build.gradle b/sdks/java/io/elasticsearch/build.gradle index 8a476a943ae49..113bdf68d1cbd 100644 --- a/sdks/java/io/elasticsearch/build.gradle +++ b/sdks/java/io/elasticsearch/build.gradle @@ -23,7 +23,7 @@ description = "Apache Beam :: SDKs :: Java :: IO :: Elasticsearch" ext.summary = "IO to read and write on Elasticsearch" dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.http_client implementation library.java.http_core diff --git a/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.java b/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.java index 9ab27a999dc02..69cefeac66bfc 100644 --- a/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.java +++ b/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.elasticsearch; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonNode; @@ -84,9 +84,9 @@ import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Streams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Streams; import org.apache.http.ConnectionClosedException; import org.apache.http.Header; import org.apache.http.HttpEntity; diff --git a/sdks/java/io/file-based-io-tests/OWNERS b/sdks/java/io/file-based-io-tests/OWNERS deleted file mode 100644 index c1a7da5149548..0000000000000 --- a/sdks/java/io/file-based-io-tests/OWNERS +++ /dev/null @@ -1,7 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lgajowy - - kkucharc - - chamikaramj - - timrobertson100 diff --git a/sdks/java/io/file-based-io-tests/src/test/java/org/apache/beam/sdk/io/common/FileBasedIOITHelper.java b/sdks/java/io/file-based-io-tests/src/test/java/org/apache/beam/sdk/io/common/FileBasedIOITHelper.java index bb5a47c971425..88cacab860d04 100644 --- a/sdks/java/io/file-based-io-tests/src/test/java/org/apache/beam/sdk/io/common/FileBasedIOITHelper.java +++ b/sdks/java/io/file-based-io-tests/src/test/java/org/apache/beam/sdk/io/common/FileBasedIOITHelper.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.io.fs.MatchResult; import org.apache.beam.sdk.io.fs.ResourceId; import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; /** Contains helper methods for file based IO Integration tests. */ public class FileBasedIOITHelper { diff --git a/sdks/java/io/file-schema-transform/OWNERS b/sdks/java/io/file-schema-transform/OWNERS deleted file mode 100644 index aed54489c35c8..0000000000000 --- a/sdks/java/io/file-schema-transform/OWNERS +++ /dev/null @@ -1,8 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners -# Reviewers below listed in alphabetic order - -reviewers: - - ahmedabu98 - - damondouglas - - johnjcasey - - pabloem diff --git a/sdks/java/io/file-schema-transform/build.gradle b/sdks/java/io/file-schema-transform/build.gradle index 9a846963e34cf..7b5f4cdf67a19 100644 --- a/sdks/java/io/file-schema-transform/build.gradle +++ b/sdks/java/io/file-schema-transform/build.gradle @@ -39,7 +39,7 @@ dependencies { implementation library.java.commons_csv implementation library.java.jaxb_api implementation library.java.joda_time - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(":sdks:java:extensions:avro") implementation project(path: ":sdks:java:io:csv") diff --git a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/AvroWriteSchemaTransformFormatProvider.java b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/AvroWriteSchemaTransformFormatProvider.java index 34b74089405c4..f36a2280e7915 100644 --- a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/AvroWriteSchemaTransformFormatProvider.java +++ b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/AvroWriteSchemaTransformFormatProvider.java @@ -19,23 +19,32 @@ import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.getNumShards; import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.getShardNameTemplate; +import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider.ERROR_SCHEMA; +import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider.ERROR_TAG; +import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider.RESULT_TAG; import com.google.auto.service.AutoService; import org.apache.avro.generic.GenericRecord; import org.apache.beam.sdk.extensions.avro.coders.AvroCoder; import org.apache.beam.sdk.extensions.avro.io.AvroIO; import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils; +import org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.BeamRowMapperWithDlq; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.Values; import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.sdk.values.TupleTagList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; /** A {@link FileWriteSchemaTransformFormatProvider} for avro format. */ @AutoService(FileWriteSchemaTransformFormatProvider.class) public class AvroWriteSchemaTransformFormatProvider implements FileWriteSchemaTransformFormatProvider { + static final TupleTag ERROR_FN_OUPUT_TAG = new TupleTag() {}; @Override public String identifier() { @@ -44,25 +53,31 @@ public String identifier() { /** * Builds a {@link PTransform} that transforms a {@link Row} {@link PCollection} into result - * {@link PCollection} file names written using {@link AvroIO.Write}. + * {@link PCollectionTuple} with two tags, one for file names written using {@link AvroIO.Write}, + * another for errored-out rows. */ @Override - public PTransform, PCollection> buildTransform( + public PTransform, PCollectionTuple> buildTransform( FileWriteSchemaTransformConfiguration configuration, Schema schema) { - return new PTransform, PCollection>() { + return new PTransform, PCollectionTuple>() { @Override - public PCollection expand(PCollection input) { + public PCollectionTuple expand(PCollection input) { org.apache.avro.Schema avroSchema = AvroUtils.toAvroSchema(schema); AvroCoder coder = AvroCoder.of(avroSchema); - PCollection avro = - input - .apply( - "Row To Avro Generic Record", - FileWriteSchemaTransformFormatProviders.mapRowsToGenericRecords(schema)) - .setCoder(coder); + PCollectionTuple tuple = + input.apply( + "Row To Avro Generic Record", + ParDo.of( + new BeamRowMapperWithDlq( + "Avro-write-error-counter", + AvroUtils.getRowToGenericRecordFunction(AvroUtils.toAvroSchema(schema)), + ERROR_FN_OUPUT_TAG)) + .withOutputTags(ERROR_FN_OUPUT_TAG, TupleTagList.of(ERROR_TAG))); + + PCollection avro = tuple.get(ERROR_FN_OUPUT_TAG).setCoder(coder); AvroIO.Write write = AvroIO.writeGenericRecords(avroSchema).to(configuration.getFilenamePrefix()); @@ -79,9 +94,13 @@ public PCollection expand(PCollection input) { write = write.withShardNameTemplate(getShardNameTemplate(configuration)); } - return avro.apply("Write Avro", write.withOutputFilenames()) - .getPerDestinationOutputFilenames() - .apply("perDestinationOutputFilenames", Values.create()); + PCollection output = + avro.apply("Write Avro", write.withOutputFilenames()) + .getPerDestinationOutputFilenames() + .apply("perDestinationOutputFilenames", Values.create()); + + return PCollectionTuple.of(RESULT_TAG, output) + .and(ERROR_TAG, tuple.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA)); } }; } diff --git a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/CsvWriteSchemaTransformFormatProvider.java b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/CsvWriteSchemaTransformFormatProvider.java index f32346bae6396..2c3639e84a46b 100644 --- a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/CsvWriteSchemaTransformFormatProvider.java +++ b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/CsvWriteSchemaTransformFormatProvider.java @@ -22,7 +22,8 @@ import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.getFilenameSuffix; import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.getNumShards; import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.getShardNameTemplate; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider.RESULT_TAG; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.service.AutoService; import java.util.Optional; @@ -32,8 +33,9 @@ import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.Values; import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.apache.commons.csv.CSVFormat; /** A {@link FileWriteSchemaTransformFormatProvider} for CSV format. */ @@ -49,12 +51,12 @@ public String identifier() { } @Override - public PTransform, PCollection> buildTransform( + public PTransform, PCollectionTuple> buildTransform( FileWriteSchemaTransformConfiguration configuration, Schema schema) { - return new PTransform, PCollection>() { + return new PTransform, PCollectionTuple>() { @Override - public PCollection expand(PCollection input) { + public PCollectionTuple expand(PCollection input) { FileWriteSchemaTransformConfiguration.CsvConfiguration csvConfiguration = getCSVConfiguration(configuration); CSVFormat csvFormat = @@ -83,9 +85,11 @@ public PCollection expand(PCollection input) { } WriteFilesResult result = input.apply("Row to CSV", write); - return result - .getPerDestinationOutputFilenames() - .apply("perDestinationOutputFilenames", Values.create()); + PCollection output = + result + .getPerDestinationOutputFilenames() + .apply("perDestinationOutputFilenames", Values.create()); + return PCollectionTuple.of(RESULT_TAG, output); } }; } diff --git a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileReadSchemaTransformConfiguration.java b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileReadSchemaTransformConfiguration.java index ed25e04ac480f..855434c7cd21d 100644 --- a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileReadSchemaTransformConfiguration.java +++ b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileReadSchemaTransformConfiguration.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.fileschematransform; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.util.Optional; @@ -28,8 +28,8 @@ import org.apache.beam.sdk.schemas.annotations.DefaultSchema; import org.apache.beam.sdk.schemas.annotations.SchemaFieldDescription; import org.apache.beam.sdk.schemas.io.Providers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; @DefaultSchema(AutoValueSchema.class) @AutoValue diff --git a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileReadSchemaTransformProvider.java b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileReadSchemaTransformProvider.java index 8188673d57d32..2ea3879adbcf3 100644 --- a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileReadSchemaTransformProvider.java +++ b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileReadSchemaTransformProvider.java @@ -18,8 +18,8 @@ package org.apache.beam.sdk.io.fileschematransform; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.service.AutoService; import java.io.IOException; @@ -49,9 +49,9 @@ import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CharStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CharStreams; import org.joda.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,8 +91,7 @@ public List outputCollectionNames() { } @VisibleForTesting - static class FileReadSchemaTransform extends PTransform - implements SchemaTransform { + static class FileReadSchemaTransform extends SchemaTransform { private FileReadSchemaTransformConfiguration configuration; private boolean useInputPCollection; @@ -239,10 +238,5 @@ private FileReadSchemaTransformFormatProvider getProvider() { checkState(provider.isPresent()); return provider.get(); } - - @Override - public PTransform buildTransform() { - return this; - } } } diff --git a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProvider.java b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProvider.java index 6eaec310cd2ac..a1eca39e16037 100644 --- a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProvider.java +++ b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProvider.java @@ -22,6 +22,7 @@ import org.apache.beam.sdk.schemas.io.Providers; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.Row; /** @@ -36,8 +37,9 @@ public interface FileWriteSchemaTransformFormatProvider extends Providers.Identi /** * Builds a {@link PTransform} that writes a {@link Row} {@link PCollection} and outputs the - * resulting {@link PCollection} of the file names. + * resulting {@link PCollectionTuple} with two tags, one for the file names, and another + * errored-out rows. */ - PTransform, PCollection> buildTransform( + PTransform, PCollectionTuple> buildTransform( FileWriteSchemaTransformConfiguration configuration, Schema schema); } diff --git a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProviders.java b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProviders.java index f1b00db896d7c..85c48ea498a3e 100644 --- a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProviders.java +++ b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProviders.java @@ -17,8 +17,11 @@ */ package org.apache.beam.sdk.io.fileschematransform; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider.ERROR_SCHEMA; +import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider.ERROR_TAG; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; +import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.Optional; import org.apache.avro.generic.GenericRecord; @@ -27,12 +30,19 @@ import org.apache.beam.sdk.io.Compression; import org.apache.beam.sdk.io.FileIO; import org.apache.beam.sdk.io.TextIO; +import org.apache.beam.sdk.metrics.Counter; +import org.apache.beam.sdk.metrics.Metrics; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.io.Providers; +import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * {@link FileWriteSchemaTransformFormatProviders} contains {@link @@ -49,6 +59,8 @@ public final class FileWriteSchemaTransformFormatProviders { static final String JSON = "json"; static final String PARQUET = "parquet"; static final String XML = "xml"; + private static final Logger LOG = + LoggerFactory.getLogger(FileWriteSchemaTransformFormatProviders.class); /** Load all {@link FileWriteSchemaTransformFormatProvider} implementations. */ public static Map loadProviders() { @@ -61,6 +73,49 @@ static MapElements mapRowsToGenericRecords(Schema beamSchema .via(AvroUtils.getRowToGenericRecordFunction(AvroUtils.toAvroSchema(beamSchema))); } + // Applies generic mapping from Beam row to other data types through the provided mapFn. + // Implemenets error handling with metrics and DLQ support. + // Arguments: + // name: the metric name to use. + // mapFn: the mapping function for mapping from Beam row to other data types. + // outputTag: TupleTag for output. Used to direct output to correct output source, or in the + // case of error, a DLQ. + static class BeamRowMapperWithDlq extends DoFn { + private SerializableFunction mapFn; + private Counter errorCounter; + private TupleTag outputTag; + private long errorsInBundle = 0L; + + public BeamRowMapperWithDlq( + String name, SerializableFunction mapFn, TupleTag outputTag) { + errorCounter = Metrics.counter(FileWriteSchemaTransformFormatProvider.class, name); + this.mapFn = mapFn; + this.outputTag = outputTag; + } + + @ProcessElement + public void process(@DoFn.Element Row row, MultiOutputReceiver receiver) { + try { + receiver.get(outputTag).output(mapFn.apply(row)); + } catch (Exception e) { + errorsInBundle += 1; + LOG.warn("Error while parsing input element", e); + receiver + .get(ERROR_TAG) + .output( + Row.withSchema(ERROR_SCHEMA) + .addValues(e.toString(), row.toString().getBytes(StandardCharsets.UTF_8)) + .build()); + } + } + + @FinishBundle + public void finish() { + errorCounter.inc(errorsInBundle); + errorsInBundle = 0L; + } + } + /** * Applies common parameters from {@link FileWriteSchemaTransformConfiguration} to {@link * FileIO.Write}. diff --git a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformProvider.java b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformProvider.java index f0070f13919b4..f3c76897c0a36 100644 --- a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformProvider.java +++ b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformProvider.java @@ -22,7 +22,7 @@ import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.PARQUET; import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.XML; import static org.apache.beam.sdk.values.TypeDescriptors.rows; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.service.AutoService; import java.util.Collections; @@ -39,8 +39,10 @@ import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionRowTuple; +import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; /** * A {@link TypedSchemaTransformProvider} implementation for writing a {@link Row} {@link @@ -52,10 +54,15 @@ public class FileWriteSchemaTransformProvider public static final Field FILE_NAME_FIELD = Field.of("fileName", FieldType.STRING); public static final Schema OUTPUT_SCHEMA = Schema.of(FILE_NAME_FIELD); + public static final Schema ERROR_SCHEMA = + Schema.builder().addStringField("error").addNullableByteArrayField("row").build(); private static final String IDENTIFIER = "beam:schematransform:org.apache.beam:file_write:v1"; static final String INPUT_TAG = "input"; static final String OUTPUT_TAG = "output"; + static final String ERROR_STRING = "error"; + static final TupleTag ERROR_TAG = new TupleTag() {}; + static final TupleTag RESULT_TAG = new TupleTag() {}; /** Provides the required {@link TypedSchemaTransformProvider#configurationClass()}. */ @Override @@ -92,8 +99,7 @@ public List outputCollectionNames() { * #inputCollectionNames()} tagged {@link Row}s into a {@link PCollectionRowTuple} of {@link * #outputCollectionNames()} tagged {@link Row}s. */ - static class FileWriteSchemaTransform extends PTransform - implements SchemaTransform { + static class FileWriteSchemaTransform extends SchemaTransform { final FileWriteSchemaTransformConfiguration configuration; @@ -113,12 +119,13 @@ public PCollectionRowTuple expand(PCollectionRowTuple input) { PCollection rowInput = input.get(INPUT_TAG); - PTransform, PCollection> transform = + PTransform, PCollectionTuple> transform = getProvider().buildTransform(configuration, rowInput.getSchema()); - PCollection files = rowInput.apply("Write Rows", transform); + PCollectionTuple files = rowInput.apply("Write Rows", transform); PCollection output = files + .get(RESULT_TAG) .apply( "Filenames to Rows", MapElements.into(rows()) @@ -129,12 +136,11 @@ public PCollectionRowTuple expand(PCollectionRowTuple input) { .build())) .setRowSchema(OUTPUT_SCHEMA); - return PCollectionRowTuple.of(OUTPUT_TAG, output); - } - - @Override - public PTransform buildTransform() { - return this; + if (files.has(ERROR_TAG)) { + return PCollectionRowTuple.of(OUTPUT_TAG, output).and(ERROR_STRING, files.get(ERROR_TAG)); + } else { + return PCollectionRowTuple.of(OUTPUT_TAG, output); + } } /** diff --git a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/JsonReadSchemaTransformFormatProvider.java b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/JsonReadSchemaTransformFormatProvider.java index 20c5ecf791dc1..7fedcc693351e 100644 --- a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/JsonReadSchemaTransformFormatProvider.java +++ b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/JsonReadSchemaTransformFormatProvider.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** A {@link FileReadSchemaTransformFormatProvider} that reads newline-delimited JSONs. */ @AutoService(FileReadSchemaTransformFormatProvider.class) diff --git a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/JsonWriteSchemaTransformFormatProvider.java b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/JsonWriteSchemaTransformFormatProvider.java index 68f12e3d26bff..d993a21cec1d3 100644 --- a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/JsonWriteSchemaTransformFormatProvider.java +++ b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/JsonWriteSchemaTransformFormatProvider.java @@ -18,21 +18,30 @@ package org.apache.beam.sdk.io.fileschematransform; import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.applyCommonTextIOWriteFeatures; +import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider.ERROR_SCHEMA; +import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider.ERROR_TAG; +import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider.RESULT_TAG; import static org.apache.beam.sdk.values.TypeDescriptors.strings; import com.google.auto.service.AutoService; import java.nio.charset.StandardCharsets; import org.apache.beam.sdk.io.TextIO; +import org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.BeamRowMapperWithDlq; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.io.payloads.JsonPayloadSerializerProvider; import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer; import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.Values; import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.sdk.values.TupleTagList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** A {@link FileWriteSchemaTransformFormatProvider} for JSON format. */ @AutoService(FileWriteSchemaTransformFormatProvider.class) @@ -40,6 +49,7 @@ public class JsonWriteSchemaTransformFormatProvider implements FileWriteSchemaTransformFormatProvider { final String suffix = String.format(".%s", FileWriteSchemaTransformFormatProviders.JSON); + static final TupleTag ERROR_FN_OUPUT_TAG = new TupleTag() {}; @Override public String identifier() { @@ -48,25 +58,37 @@ public String identifier() { /** * Builds a {@link PTransform} that transforms a {@link Row} {@link PCollection} into result - * {@link PCollection} file names written using {@link TextIO.Write}. + * {@link PCollectionTuple} with two tags, one for file names written using {@link TextIO.Write}, + * another for errored-out rows. */ @Override - public PTransform, PCollection> buildTransform( + public PTransform, PCollectionTuple> buildTransform( FileWriteSchemaTransformConfiguration configuration, Schema schema) { - return new PTransform, PCollection>() { + return new PTransform, PCollectionTuple>() { @Override - public PCollection expand(PCollection input) { + public PCollectionTuple expand(PCollection input) { - PCollection json = input.apply("Row To Json", mapRowsToJsonStrings(schema)); + PCollectionTuple json = + input.apply( + ParDo.of( + new BeamRowMapperWithDlq( + "Json-write-error-counter", + new RowToJsonFn(schema), + ERROR_FN_OUPUT_TAG)) + .withOutputTags(ERROR_FN_OUPUT_TAG, TupleTagList.of(ERROR_TAG))); TextIO.Write write = TextIO.write().to(configuration.getFilenamePrefix()).withSuffix(suffix); write = applyCommonTextIOWriteFeatures(write, configuration); - return json.apply("Write Json", write.withOutputFilenames()) - .getPerDestinationOutputFilenames() - .apply("perDestinationOutputFilenames", Values.create()); + PCollection output = + json.get(ERROR_FN_OUPUT_TAG) + .apply("Write Json", write.withOutputFilenames()) + .getPerDestinationOutputFilenames() + .apply("perDestinationOutputFilenames", Values.create()); + return PCollectionTuple.of(RESULT_TAG, output) + .and(ERROR_TAG, json.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA)); } }; } @@ -76,7 +98,8 @@ MapElements mapRowsToJsonStrings(Schema schema) { return MapElements.into(strings()).via(new RowToJsonFn(schema)); } - private static class RowToJsonFn implements SerializableFunction { + @VisibleForTesting + static class RowToJsonFn implements SerializableFunction { private final PayloadSerializer payloadSerializer; diff --git a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/ParquetWriteSchemaTransformFormatProvider.java b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/ParquetWriteSchemaTransformFormatProvider.java index cea5f6ed13993..5af4865ec0259 100644 --- a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/ParquetWriteSchemaTransformFormatProvider.java +++ b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/ParquetWriteSchemaTransformFormatProvider.java @@ -18,7 +18,10 @@ package org.apache.beam.sdk.io.fileschematransform; import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.applyCommonFileIOWriteFeatures; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider.ERROR_SCHEMA; +import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider.ERROR_TAG; +import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider.RESULT_TAG; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.service.AutoService; import java.util.Optional; @@ -27,12 +30,17 @@ import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils; import org.apache.beam.sdk.io.FileIO; import org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformConfiguration.ParquetConfiguration; +import org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.BeamRowMapperWithDlq; import org.apache.beam.sdk.io.parquet.ParquetIO; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.Values; import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.sdk.values.TupleTagList; import org.apache.parquet.hadoop.metadata.CompressionCodecName; /** A {@link FileWriteSchemaTransformFormatProvider} for Parquet format. */ @@ -43,6 +51,8 @@ public class ParquetWriteSchemaTransformFormatProvider private static final String SUFFIX = String.format(".%s", FileWriteSchemaTransformFormatProviders.PARQUET); + static final TupleTag ERROR_FN_OUPUT_TAG = new TupleTag() {}; + @Override public String identifier() { return FileWriteSchemaTransformFormatProviders.PARQUET; @@ -50,14 +60,15 @@ public String identifier() { /** * Builds a {@link PTransform} that transforms a {@link Row} {@link PCollection} into result - * {@link PCollection} file names written using {@link ParquetIO.Sink} and {@link FileIO.Write}. + * {@link PCollectionTuple} with two tags, one for file names written using {@link ParquetIO.Sink} + * and {@link FileIO.Write}, another for errored-out rows. */ @Override - public PTransform, PCollection> buildTransform( + public PTransform, PCollectionTuple> buildTransform( FileWriteSchemaTransformConfiguration configuration, Schema schema) { - return new PTransform, PCollection>() { + return new PTransform, PCollectionTuple>() { @Override - public PCollection expand(PCollection input) { + public PCollectionTuple expand(PCollection input) { org.apache.avro.Schema avroSchema = AvroUtils.toAvroSchema(schema); AvroCoder coder = AvroCoder.of(avroSchema); @@ -69,14 +80,26 @@ public PCollection expand(PCollection input) { write = applyCommonFileIOWriteFeatures(write, configuration); - return input - .apply( + PCollectionTuple parquet = + input.apply( "Row To GenericRecord", - FileWriteSchemaTransformFormatProviders.mapRowsToGenericRecords(schema)) - .setCoder(coder) - .apply("Write Parquet", write) - .getPerDestinationOutputFilenames() - .apply("perDestinationOutputFilenames", Values.create()); + ParDo.of( + new BeamRowMapperWithDlq( + "Parquet-write-error-counter", + AvroUtils.getRowToGenericRecordFunction(AvroUtils.toAvroSchema(schema)), + ERROR_FN_OUPUT_TAG)) + .withOutputTags(ERROR_FN_OUPUT_TAG, TupleTagList.of(ERROR_TAG))); + + PCollection output = + parquet + .get(ERROR_FN_OUPUT_TAG) + .setCoder(coder) + .apply("Write Parquet", write) + .getPerDestinationOutputFilenames() + .apply("perDestinationOutputFilenames", Values.create()); + + return PCollectionTuple.of(RESULT_TAG, output) + .and(ERROR_TAG, parquet.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA)); } }; } diff --git a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/XmlRowValue.java b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/XmlRowValue.java index df4392665a421..74e74bb842216 100644 --- a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/XmlRowValue.java +++ b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/XmlRowValue.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.fileschematransform; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.Serializable; import java.util.ArrayList; diff --git a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/XmlWriteSchemaTransformFormatProvider.java b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/XmlWriteSchemaTransformFormatProvider.java index 7c8f4ff0c84b8..2d24980a24665 100644 --- a/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/XmlWriteSchemaTransformFormatProvider.java +++ b/sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/XmlWriteSchemaTransformFormatProvider.java @@ -19,24 +19,30 @@ import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.XML; import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.applyCommonFileIOWriteFeatures; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider.ERROR_SCHEMA; +import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider.ERROR_TAG; +import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider.RESULT_TAG; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.service.AutoService; import java.nio.charset.Charset; import java.util.Optional; import org.apache.beam.sdk.io.FileIO; import org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformConfiguration.XmlConfiguration; +import org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.BeamRowMapperWithDlq; import org.apache.beam.sdk.io.xml.XmlIO; import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.Values; import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.Row; -import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.sdk.values.TupleTagList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; /** A {@link FileWriteSchemaTransformFormatProvider} for XML format. */ @AutoService(FileWriteSchemaTransformFormatProvider.class) @@ -44,6 +50,7 @@ public class XmlWriteSchemaTransformFormatProvider implements FileWriteSchemaTransformFormatProvider { private static final String SUFFIX = String.format(".%s", XML); + static final TupleTag ERROR_FN_OUPUT_TAG = new TupleTag() {}; @Override public String identifier() { @@ -52,19 +59,23 @@ public String identifier() { /** * Builds a {@link PTransform} that transforms a {@link Row} {@link PCollection} into result - * {@link PCollection} file names written using {@link XmlIO.Sink} and {@link FileIO.Write}. + * {@link PCollectionTuple} with two tags, one for file names written using {@link XmlIO.Sink} and + * {@link FileIO.Write}, another for errored-out rows. */ @Override - public PTransform, PCollection> buildTransform( + public PTransform, PCollectionTuple> buildTransform( FileWriteSchemaTransformConfiguration configuration, Schema schema) { - return new PTransform, PCollection>() { + return new PTransform, PCollectionTuple>() { @Override - public PCollection expand(PCollection input) { + public PCollectionTuple expand(PCollection input) { - PCollection xml = + PCollectionTuple xml = input.apply( "Row to XML", - MapElements.into(TypeDescriptor.of(XmlRowAdapter.class)).via(new RowToXmlFn())); + ParDo.of( + new BeamRowMapperWithDlq( + "Xml-write-error-counter", new RowToXmlFn(), ERROR_FN_OUPUT_TAG)) + .withOutputTags(ERROR_FN_OUPUT_TAG, TupleTagList.of(ERROR_TAG))); XmlConfiguration xmlConfig = xmlConfiguration(configuration); @@ -87,9 +98,13 @@ public PCollection expand(PCollection input) { write = applyCommonFileIOWriteFeatures(write, configuration); - return xml.apply("Write XML", write) - .getPerDestinationOutputFilenames() - .apply("perDestinationOutputFilenames", Values.create()); + PCollection output = + xml.get(ERROR_FN_OUPUT_TAG) + .apply("Write XML", write) + .getPerDestinationOutputFilenames() + .apply("perDestinationOutputFilenames", Values.create()); + return PCollectionTuple.of(RESULT_TAG, output) + .and(ERROR_TAG, xml.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA)); } }; } diff --git a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/AvroReadSchemaTransformFormatProviderTest.java b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/AvroReadSchemaTransformFormatProviderTest.java index 6a9d9def78b86..5725ceff3a126 100644 --- a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/AvroReadSchemaTransformFormatProviderTest.java +++ b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/AvroReadSchemaTransformFormatProviderTest.java @@ -50,7 +50,7 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.joda.time.Duration; import org.junit.Test; @@ -91,8 +91,7 @@ public void runWriteAndReadTest( .build(); SchemaTransform readTransform = new FileReadSchemaTransformProvider().from(config); - PCollectionRowTuple output = - PCollectionRowTuple.empty(readPipeline).apply(readTransform.buildTransform()); + PCollectionRowTuple output = PCollectionRowTuple.empty(readPipeline).apply(readTransform); PAssert.that(output.get(FileReadSchemaTransformProvider.OUTPUT_TAG)).containsInAnyOrder(rows); readPipeline.run(); @@ -133,8 +132,7 @@ public void testStreamingRead() { .build(); SchemaTransform readTransform = new FileReadSchemaTransformProvider().from(config); - PCollectionRowTuple output = - PCollectionRowTuple.empty(readPipeline).apply(readTransform.buildTransform()); + PCollectionRowTuple output = PCollectionRowTuple.empty(readPipeline).apply(readTransform); // Write to three different files (test_1..., test_2..., test_3) // All three new files should be picked up and read. @@ -210,7 +208,7 @@ public void testReadWithPCollectionOfFilepatterns() { PCollectionRowTuple output = PCollectionRowTuple.of(FileReadSchemaTransformProvider.INPUT_TAG, filepatterns) - .apply(readTransform.buildTransform()); + .apply(readTransform); // Check output matches with expected rows PAssert.that(output.get(FileReadSchemaTransformProvider.OUTPUT_TAG)).containsInAnyOrder(rows); diff --git a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/AvroWriteSchemaTransformFormatProviderTest.java b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/AvroWriteSchemaTransformFormatProviderTest.java index 7fea133b75cea..c4525e819cf3d 100644 --- a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/AvroWriteSchemaTransformFormatProviderTest.java +++ b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/AvroWriteSchemaTransformFormatProviderTest.java @@ -19,21 +19,36 @@ import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.AVRO; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.GenericRecordBuilder; import org.apache.beam.sdk.extensions.avro.coders.AvroCoder; import org.apache.beam.sdk.extensions.avro.io.AvroIO; import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils; +import org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.BeamRowMapperWithDlq; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.transforms.Count; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.sdk.values.TupleTagList; +import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -/** Tests for {@link AvroWriteSchemaTransformFormatProvider}. */ +/** + * Tests for {@link + * org.apache.beam.sdk.io.fileschematransform.AvroWriteSchemaTransformFormatProvider}. + */ @RunWith(JUnit4.class) public class AvroWriteSchemaTransformFormatProviderTest extends FileWriteSchemaTransformFormatProviderTest { @@ -101,4 +116,66 @@ protected Optional expectedErrorWhenCsvConfigurationSet() { return Optional.of( "configuration with org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformConfiguration$CsvConfiguration is not compatible with a avro format"); } + + @Test + public void testAvroErrorCounterSuccess() { + SerializableFunction mapFn = + AvroUtils.getRowToGenericRecordFunction(AvroUtils.toAvroSchema(BEAM_SCHEMA)); + + List records = + Arrays.asList( + new GenericRecordBuilder(AVRO_SCHEMA).set("name", "a").build(), + new GenericRecordBuilder(AVRO_SCHEMA).set("name", "b").build(), + new GenericRecordBuilder(AVRO_SCHEMA).set("name", "c").build()); + + PCollection input = writePipeline.apply(Create.of(ROWS)); + PCollectionTuple output = + input.apply( + ParDo.of( + new BeamRowMapperWithDlq( + "Avro-write-error-counter", mapFn, OUTPUT_TAG)) + .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); + output.get(OUTPUT_TAG).setCoder(CODER); + output.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA); + PAssert.that(output.get(OUTPUT_TAG)).containsInAnyOrder(records); + writePipeline.run().waitUntilFinish(); + } + + @Test + public void testAvroErrorCounterFailure() { + SerializableFunction mapFn = + AvroUtils.getRowToGenericRecordFunction(AvroUtils.toAvroSchema(BEAM_SCHEMA_DLQ)); + + PCollection input = writePipeline.apply(Create.of(ROWS)); + PCollectionTuple output = + input.apply( + ParDo.of( + new BeamRowMapperWithDlq( + "Avro-write-error-counter", mapFn, OUTPUT_TAG)) + .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); + output.get(OUTPUT_TAG).setCoder(CODER); + output.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA); + PCollection count = output.get(ERROR_TAG).apply(Count.globally()); + PAssert.that(count).containsInAnyOrder(Collections.singletonList(3L)); + writePipeline.run().waitUntilFinish(); + } + + private static final TupleTag OUTPUT_TAG = + AvroWriteSchemaTransformFormatProvider.ERROR_FN_OUPUT_TAG; + private static final TupleTag ERROR_TAG = FileWriteSchemaTransformProvider.ERROR_TAG; + + private static final Schema BEAM_SCHEMA = + Schema.of(Schema.Field.of("name", Schema.FieldType.STRING)); + private static final Schema BEAM_SCHEMA_DLQ = + Schema.of(Schema.Field.of("error", Schema.FieldType.INT32)); + private static final Schema ERROR_SCHEMA = FileWriteSchemaTransformProvider.ERROR_SCHEMA; + private static final org.apache.avro.Schema AVRO_SCHEMA = AvroUtils.toAvroSchema(BEAM_SCHEMA); + + private static final AvroCoder CODER = AvroCoder.of(AVRO_SCHEMA); + + private static final List ROWS = + Arrays.asList( + Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "a").build(), + Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "b").build(), + Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "c").build()); } diff --git a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/CsvWriteSchemaTransformFormatProviderTest.java b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/CsvWriteSchemaTransformFormatProviderTest.java index 384d1cca1dd03..777f468771a9c 100644 --- a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/CsvWriteSchemaTransformFormatProviderTest.java +++ b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/CsvWriteSchemaTransformFormatProviderTest.java @@ -26,6 +26,7 @@ import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformConfiguration.csvConfigurationBuilder; import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviderTestData.DATA; import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.CSV; +import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider.RESULT_TAG; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; @@ -231,7 +232,9 @@ public void timeContainingSchemaWithListRemovedShouldWriteCSV() { Schema modifiedSchema = modifiedInput.getSchema(); FileWriteSchemaTransformConfiguration configuration = buildConfiguration(prefix); PCollection result = - modifiedInput.apply(getProvider().buildTransform(configuration, modifiedSchema)); + modifiedInput + .apply(getProvider().buildTransform(configuration, modifiedSchema)) + .get(RESULT_TAG); PCollection numFiles = result.apply(Count.globally()); PAssert.thatSingleton(numFiles).isEqualTo(1L); writePipeline.run().waitUntilFinish(); @@ -259,7 +262,9 @@ public void byteTypeNonRepeated() { Schema modifiedSchema = modifiedInput.getSchema(); FileWriteSchemaTransformConfiguration configuration = buildConfiguration(prefix); PCollection result = - modifiedInput.apply(getProvider().buildTransform(configuration, modifiedSchema)); + modifiedInput + .apply(getProvider().buildTransform(configuration, modifiedSchema)) + .get(RESULT_TAG); PCollection numFiles = result.apply(Count.globally()); PAssert.thatSingleton(numFiles).isEqualTo(1L); writePipeline.run().waitUntilFinish(); diff --git a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileReadSchemaTransformFormatProviderTest.java b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileReadSchemaTransformFormatProviderTest.java index 81bb7c005be74..d01e0051c7b8e 100644 --- a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileReadSchemaTransformFormatProviderTest.java +++ b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileReadSchemaTransformFormatProviderTest.java @@ -246,7 +246,7 @@ public void testWriteAndReadWithSchemaTransforms() { PCollection inputRows = writePipeline.apply(Create.of(rows).withRowSchema(schema)); PCollection filePatterns = PCollectionRowTuple.of("input", inputRows) - .apply(writeTransform.buildTransform()) + .apply(writeTransform) .get("output") .setRowSchema(FileWriteSchemaTransformProvider.OUTPUT_SCHEMA) .apply( @@ -261,9 +261,7 @@ public void testWriteAndReadWithSchemaTransforms() { .setRowSchema(filePatternSchema); PCollection outputRows = - PCollectionRowTuple.of("input", filePatterns) - .apply(readTransform.buildTransform()) - .get("output"); + PCollectionRowTuple.of("input", filePatterns).apply(readTransform).get("output"); if (getFormat().equals("json")) { rows = diff --git a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProviderTest.java b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProviderTest.java index 738343e7fc91e..e7bae565d22a4 100644 --- a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProviderTest.java +++ b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProviderTest.java @@ -30,6 +30,7 @@ import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformConfiguration.xmlConfigurationBuilder; import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviderTestData.DATA; import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.loadProviders; +import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider.RESULT_TAG; import static org.apache.beam.sdk.values.TypeDescriptors.booleans; import static org.apache.beam.sdk.values.TypeDescriptors.strings; import static org.junit.Assert.assertEquals; @@ -56,7 +57,7 @@ import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; import org.apache.commons.csv.CSVFormat; import org.apache.parquet.hadoop.metadata.CompressionCodecName; import org.junit.Rule; @@ -455,7 +456,8 @@ private PCollection applyProviderAndAssertFilesWritten( private PCollection applyProviderAndAssertFilesWritten( List rows, Schema schema, FileWriteSchemaTransformConfiguration configuration) { PCollection input = writePipeline.apply(Create.of(rows).withRowSchema(schema)); - PCollection files = input.apply(getProvider().buildTransform(configuration, schema)); + PCollection files = + input.apply(getProvider().buildTransform(configuration, schema)).get(RESULT_TAG); PCollection count = files.apply("count number of files", Count.globally()); PAssert.thatSingleton("At least one file should be written", count).notEqualTo(0L); return files; diff --git a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProvidersTest.java b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProvidersTest.java index b145243fb149a..08eafedc64fd1 100644 --- a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProvidersTest.java +++ b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProvidersTest.java @@ -24,9 +24,26 @@ import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.XML; import static org.junit.Assert.assertEquals; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.BeamRowMapperWithDlq; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Count; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.sdk.values.TupleTagList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -34,6 +51,8 @@ /** Tests for {@link FileWriteSchemaTransformFormatProviders}. */ @RunWith(JUnit4.class) public class FileWriteSchemaTransformFormatProvidersTest { + @Rule public TestPipeline p = TestPipeline.create(); + @Test public void loadProviders() { Map formatProviderMap = @@ -41,4 +60,71 @@ public void loadProviders() { Set keys = formatProviderMap.keySet(); assertEquals(ImmutableSet.of(AVRO, CSV, JSON, PARQUET, XML), keys); } + + @Test + public void testErrorCounterSuccess() { + SerializableFunction mapFn = new SimpleMapFn(false); + List records = Arrays.asList(NO_DLQ_MSG, NO_DLQ_MSG, NO_DLQ_MSG); + + PCollection input = p.apply(Create.of(ROWS)); + PCollectionTuple output = + input.apply( + ParDo.of( + new BeamRowMapperWithDlq( + "Generic-write-error-counter", mapFn, OUTPUT_TAG)) + .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); + output.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA); + PAssert.that(output.get(OUTPUT_TAG)).containsInAnyOrder(records); + p.run().waitUntilFinish(); + } + + @Test + public void testErrorCounterDlqSuccess() { + SerializableFunction mapFn = new SimpleMapFn(true); + + PCollection input = p.apply(Create.of(ROWS)); + PCollectionTuple output = + input.apply( + ParDo.of( + new BeamRowMapperWithDlq( + "Generic-write-error-counter", mapFn, OUTPUT_TAG)) + .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); + output.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA); + PCollection count = output.get(ERROR_TAG).apply(Count.globally()); + PAssert.that(count).containsInAnyOrder(Collections.singletonList(3L)); + p.run().waitUntilFinish(); + } + + private static class SimpleMapFn implements SerializableFunction { + private boolean useDlq; + + SimpleMapFn(boolean useDlq) { + this.useDlq = useDlq; + } + + @Override + public String apply(Row row) { + if (useDlq) { + throw new IllegalArgumentException(DLQ_MSG); + } else { + return NO_DLQ_MSG; + } + } + } + + private static final TupleTag OUTPUT_TAG = new TupleTag() {}; + private static final TupleTag ERROR_TAG = FileWriteSchemaTransformProvider.ERROR_TAG; + + private static final Schema BEAM_SCHEMA = + Schema.of(Schema.Field.of("name", Schema.FieldType.STRING)); + private static final Schema ERROR_SCHEMA = FileWriteSchemaTransformProvider.ERROR_SCHEMA; + + private static final List ROWS = + Arrays.asList( + Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "a").build(), + Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "b").build(), + Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "c").build()); + + private static final String DLQ_MSG = "Testing DLQ behavior"; + private static final String NO_DLQ_MSG = "Testing without DLQ"; } diff --git a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformProviderTest.java b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformProviderTest.java index e1cc231f93415..c8494446deda1 100644 --- a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformProviderTest.java +++ b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformProviderTest.java @@ -64,7 +64,7 @@ public void receivedUnexpectedInputTagsThrowsAnError() { PROVIDER.from(rowConfiguration(defaultConfiguration().setFormat(JSON).build())); PCollectionRowTuple empty = PCollectionRowTuple.empty(errorPipeline); IllegalArgumentException emptyInputError = - assertThrows(IllegalArgumentException.class, () -> empty.apply(transform.buildTransform())); + assertThrows(IllegalArgumentException.class, () -> empty.apply(transform)); assertEquals( "org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider$FileWriteSchemaTransform expects a single input tagged PCollection input", @@ -81,8 +81,7 @@ public void receivedUnexpectedInputTagsThrowsAnError() { PCollectionRowTuple tooManyTags = PCollectionRowTuple.of(INPUT_TAG, rows1).and("another", rows2); IllegalArgumentException tooManyTagsError = - assertThrows( - IllegalArgumentException.class, () -> tooManyTags.apply(transform.buildTransform())); + assertThrows(IllegalArgumentException.class, () -> tooManyTags.apply(transform)); assertEquals( "org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformProvider$FileWriteSchemaTransform expects a single input tagged PCollection input", @@ -90,7 +89,7 @@ public void receivedUnexpectedInputTagsThrowsAnError() { // should not throw an error PCollectionRowTuple input = PCollectionRowTuple.of(INPUT_TAG, rows1); - input.apply(transform.buildTransform()); + input.apply(transform); } @Test diff --git a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/JsonReadSchemaTransformFormatProviderTest.java b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/JsonReadSchemaTransformFormatProviderTest.java index b8bc807c866e6..e4bc63cd8d8ed 100644 --- a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/JsonReadSchemaTransformFormatProviderTest.java +++ b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/JsonReadSchemaTransformFormatProviderTest.java @@ -54,8 +54,8 @@ import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Duration; import org.junit.Test; @@ -189,8 +189,7 @@ public void runWriteAndReadTest( .build(); SchemaTransform readTransform = new FileReadSchemaTransformProvider().from(config); - PCollectionRowTuple output = - PCollectionRowTuple.empty(readPipeline).apply(readTransform.buildTransform()); + PCollectionRowTuple output = PCollectionRowTuple.empty(readPipeline).apply(readTransform); List expectedRows = rows.stream().map(row -> getExpectedRow(row)).collect(Collectors.toList()); @@ -236,8 +235,7 @@ public void testStreamingRead() { .build(); SchemaTransform readTransform = new FileReadSchemaTransformProvider().from(config); - PCollectionRowTuple output = - PCollectionRowTuple.empty(readPipeline).apply(readTransform.buildTransform()); + PCollectionRowTuple output = PCollectionRowTuple.empty(readPipeline).apply(readTransform); PayloadSerializer payloadSerializer = new JsonPayloadSerializerProvider().getSerializer(schema, ImmutableMap.of()); @@ -323,7 +321,7 @@ public void testReadWithPCollectionOfFilepatterns() { PCollectionRowTuple output = PCollectionRowTuple.of(FileReadSchemaTransformProvider.INPUT_TAG, filepatterns) - .apply(readTransform.buildTransform()); + .apply(readTransform); // Check output matches with expected rows List expectedRows = diff --git a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/JsonWriteSchemaTransformFormatProviderTest.java b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/JsonWriteSchemaTransformFormatProviderTest.java index 4689ad307e9db..90c84134cee76 100644 --- a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/JsonWriteSchemaTransformFormatProviderTest.java +++ b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/JsonWriteSchemaTransformFormatProviderTest.java @@ -20,17 +20,29 @@ import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.JSON; import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import org.apache.beam.sdk.io.TextIO; +import org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.BeamRowMapperWithDlq; +import org.apache.beam.sdk.io.fileschematransform.JsonWriteSchemaTransformFormatProvider.RowToJsonFn; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.io.payloads.JsonPayloadSerializerProvider; import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer; import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.transforms.Count; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.sdk.values.TupleTagList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -101,4 +113,52 @@ protected Optional expectedErrorWhenCsvConfigurationSet() { return Optional.of( "configuration with org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformConfiguration$CsvConfiguration is not compatible with a json format"); } + + @Test + public void testJsonErrorCounterSuccess() { + SerializableFunction mapFn = new RowToJsonFn(BEAM_SCHEMA); + + PCollection input = writePipeline.apply(Create.of(ROWS)); + PCollectionTuple output = + input.apply( + ParDo.of( + new BeamRowMapperWithDlq("Json-write-error-counter", mapFn, OUTPUT_TAG)) + .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); + output.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA); + PCollection count = output.get(OUTPUT_TAG).apply(Count.globally()); + PAssert.that(count).containsInAnyOrder(Collections.singleton(3L)); + writePipeline.run().waitUntilFinish(); + } + + @Test + public void testJsonErrorCounterFailure() { + SerializableFunction mapFn = new RowToJsonFn(BEAM_SCHEMA_DLQ); + + PCollection input = writePipeline.apply(Create.of(ROWS)); + PCollectionTuple output = + input.apply( + ParDo.of( + new BeamRowMapperWithDlq("Json-write-error-counter", mapFn, OUTPUT_TAG)) + .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); + output.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA); + PCollection count = output.get(ERROR_TAG).apply(Count.globally()); + PAssert.that(count).containsInAnyOrder(Collections.singletonList(3L)); + writePipeline.run().waitUntilFinish(); + } + + private static final TupleTag OUTPUT_TAG = + JsonWriteSchemaTransformFormatProvider.ERROR_FN_OUPUT_TAG; + private static final TupleTag ERROR_TAG = FileWriteSchemaTransformProvider.ERROR_TAG; + + private static final Schema BEAM_SCHEMA = + Schema.of(Schema.Field.of("name", Schema.FieldType.STRING)); + private static final Schema BEAM_SCHEMA_DLQ = + Schema.of(Schema.Field.of("error", Schema.FieldType.INT32)); + private static final Schema ERROR_SCHEMA = FileWriteSchemaTransformProvider.ERROR_SCHEMA; + + private static final List ROWS = + Arrays.asList( + Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "a").build(), + Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "b").build(), + Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "c").build()); } diff --git a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/LineReadSchemaTransformFormatProviderTest.java b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/LineReadSchemaTransformFormatProviderTest.java index 6986a053cadd7..4c5be258651fe 100644 --- a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/LineReadSchemaTransformFormatProviderTest.java +++ b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/LineReadSchemaTransformFormatProviderTest.java @@ -100,8 +100,7 @@ public void testReadStrings() { .build(); SchemaTransform readTransform = new FileReadSchemaTransformProvider().from(config); - PCollectionRowTuple output = - PCollectionRowTuple.empty(readPipeline).apply(readTransform.buildTransform()); + PCollectionRowTuple output = PCollectionRowTuple.empty(readPipeline).apply(readTransform); PCollection outputStrings = output .get(FileReadSchemaTransformProvider.OUTPUT_TAG) @@ -133,8 +132,7 @@ public void testStreamingRead() { .build(); SchemaTransform readTransform = new FileReadSchemaTransformProvider().from(config); - PCollectionRowTuple output = - PCollectionRowTuple.empty(readPipeline).apply(readTransform.buildTransform()); + PCollectionRowTuple output = PCollectionRowTuple.empty(readPipeline).apply(readTransform); // Write to three different files (test_1..., test_2..., test_3) // All three new files should be picked up and read. @@ -216,7 +214,7 @@ public void testReadWithPCollectionOfFilepatterns() { PCollectionRowTuple output = PCollectionRowTuple.of(FileReadSchemaTransformProvider.INPUT_TAG, filepatterns) - .apply(readTransform.buildTransform()); + .apply(readTransform); PCollection outputStrings = output diff --git a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/ParquetReadSchemaTransformFormatProviderTest.java b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/ParquetReadSchemaTransformFormatProviderTest.java index f5002d0099d6a..b1d6bba06ea9d 100644 --- a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/ParquetReadSchemaTransformFormatProviderTest.java +++ b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/ParquetReadSchemaTransformFormatProviderTest.java @@ -46,7 +46,7 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.joda.time.Duration; import org.junit.Test; @@ -92,8 +92,7 @@ public void runWriteAndReadTest( .build(); SchemaTransform readTransform = new FileReadSchemaTransformProvider().from(config); - PCollectionRowTuple output = - PCollectionRowTuple.empty(readPipeline).apply(readTransform.buildTransform()); + PCollectionRowTuple output = PCollectionRowTuple.empty(readPipeline).apply(readTransform); PAssert.that(output.get(FileReadSchemaTransformProvider.OUTPUT_TAG)).containsInAnyOrder(rows); readPipeline.run(); @@ -133,8 +132,7 @@ public void testStreamingRead() { .build(); SchemaTransform readTransform = new FileReadSchemaTransformProvider().from(config); - PCollectionRowTuple output = - PCollectionRowTuple.empty(readPipeline).apply(readTransform.buildTransform()); + PCollectionRowTuple output = PCollectionRowTuple.empty(readPipeline).apply(readTransform); // Write to three different files (test_1..., test_2..., test_3) // All three new files should be picked up and read. @@ -214,7 +212,7 @@ public void testReadWithPCollectionOfFilepatterns() { PCollectionRowTuple output = PCollectionRowTuple.of(FileReadSchemaTransformProvider.INPUT_TAG, filepatterns) - .apply(readTransform.buildTransform()); + .apply(readTransform); // Check output matches with expected rows PAssert.that(output.get(FileReadSchemaTransformProvider.OUTPUT_TAG)).containsInAnyOrder(rows); diff --git a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/ParquetWriteSchemaTransformFormatProviderTest.java b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/ParquetWriteSchemaTransformFormatProviderTest.java index 41c7475a6508b..bad7811be16e6 100644 --- a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/ParquetWriteSchemaTransformFormatProviderTest.java +++ b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/ParquetWriteSchemaTransformFormatProviderTest.java @@ -20,17 +20,30 @@ import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformConfiguration.parquetConfigurationBuilder; import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.PARQUET; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.GenericRecordBuilder; +import org.apache.beam.sdk.extensions.avro.coders.AvroCoder; import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils; +import org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.BeamRowMapperWithDlq; import org.apache.beam.sdk.io.parquet.ParquetIO; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.transforms.Count; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.sdk.values.TupleTagList; import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -109,4 +122,66 @@ protected Optional expectedErrorWhenCsvConfigurationSet() { return Optional.of( "configuration with org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformConfiguration$CsvConfiguration is not compatible with a parquet format"); } + + @Test + public void testParquetErrorCounterSuccess() { + SerializableFunction mapFn = + AvroUtils.getRowToGenericRecordFunction(AvroUtils.toAvroSchema(BEAM_SCHEMA)); + + List records = + Arrays.asList( + new GenericRecordBuilder(AVRO_SCHEMA).set("name", "a").build(), + new GenericRecordBuilder(AVRO_SCHEMA).set("name", "b").build(), + new GenericRecordBuilder(AVRO_SCHEMA).set("name", "c").build()); + + PCollection input = writePipeline.apply(Create.of(ROWS)); + PCollectionTuple output = + input.apply( + ParDo.of( + new BeamRowMapperWithDlq( + "Avro-write-error-counter", mapFn, OUTPUT_TAG)) + .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); + output.get(OUTPUT_TAG).setCoder(CODER); + output.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA); + PAssert.that(output.get(OUTPUT_TAG)).containsInAnyOrder(records); + writePipeline.run().waitUntilFinish(); + } + + @Test + public void testParquetErrorCounterFailure() { + SerializableFunction mapFn = + AvroUtils.getRowToGenericRecordFunction(AvroUtils.toAvroSchema(BEAM_SCHEMA_DLQ)); + + PCollection input = writePipeline.apply(Create.of(ROWS)); + PCollectionTuple output = + input.apply( + ParDo.of( + new BeamRowMapperWithDlq( + "Avro-write-error-counter", mapFn, OUTPUT_TAG)) + .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); + output.get(OUTPUT_TAG).setCoder(CODER); + output.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA); + PCollection count = output.get(ERROR_TAG).apply(Count.globally()); + PAssert.that(count).containsInAnyOrder(Collections.singletonList(3L)); + writePipeline.run().waitUntilFinish(); + } + + private static final TupleTag OUTPUT_TAG = + ParquetWriteSchemaTransformFormatProvider.ERROR_FN_OUPUT_TAG; + private static final TupleTag ERROR_TAG = FileWriteSchemaTransformProvider.ERROR_TAG; + + private static final Schema BEAM_SCHEMA = + Schema.of(Schema.Field.of("name", Schema.FieldType.STRING)); + private static final Schema BEAM_SCHEMA_DLQ = + Schema.of(Schema.Field.of("error", Schema.FieldType.INT32)); + private static final Schema ERROR_SCHEMA = FileWriteSchemaTransformProvider.ERROR_SCHEMA; + private static final org.apache.avro.Schema AVRO_SCHEMA = AvroUtils.toAvroSchema(BEAM_SCHEMA); + + private static final AvroCoder CODER = AvroCoder.of(AVRO_SCHEMA); + + private static final List ROWS = + Arrays.asList( + Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "a").build(), + Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "b").build(), + Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "c").build()); } diff --git a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/XmlWriteSchemaTransformFormatProviderTest.java b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/XmlWriteSchemaTransformFormatProviderTest.java index a0a75fc96eac8..dbb8826e3f7be 100644 --- a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/XmlWriteSchemaTransformFormatProviderTest.java +++ b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/XmlWriteSchemaTransformFormatProviderTest.java @@ -20,14 +20,26 @@ import static org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.XML; import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviders.BeamRowMapperWithDlq; +import org.apache.beam.sdk.io.fileschematransform.XmlWriteSchemaTransformFormatProvider.RowToXmlFn; import org.apache.beam.sdk.io.xml.XmlIO; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.transforms.Count; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.sdk.values.TupleTagList; +import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -117,4 +129,35 @@ protected Optional expectedErrorWhenCsvConfigurationSet() { return Optional.of( "configuration with org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformConfiguration$CsvConfiguration is not compatible with a xml format"); } + + @Test + public void testXmlErrorCounterSuccess() { + SerializableFunction mapFn = new RowToXmlFn(); + + PCollection input = writePipeline.apply(Create.of(ROWS)); + PCollectionTuple output = + input.apply( + ParDo.of( + new BeamRowMapperWithDlq( + "Xml-write-error-counter", mapFn, OUTPUT_TAG)) + .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); + output.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA); + PCollection count = output.get(OUTPUT_TAG).apply(Count.globally()); + PAssert.that(count).containsInAnyOrder(Collections.singleton(3L)); + writePipeline.run().waitUntilFinish(); + } + + private static final TupleTag OUTPUT_TAG = + XmlWriteSchemaTransformFormatProvider.ERROR_FN_OUPUT_TAG; + private static final TupleTag ERROR_TAG = FileWriteSchemaTransformProvider.ERROR_TAG; + + private static final Schema BEAM_SCHEMA = + Schema.of(Schema.Field.of("name", Schema.FieldType.STRING)); + private static final Schema ERROR_SCHEMA = FileWriteSchemaTransformProvider.ERROR_SCHEMA; + + private static final List ROWS = + Arrays.asList( + Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "a").build(), + Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "b").build(), + Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "c").build()); } diff --git a/sdks/java/io/google-ads/build.gradle b/sdks/java/io/google-ads/build.gradle new file mode 100644 index 0000000000000..f3dbb19e04f42 --- /dev/null +++ b/sdks/java/io/google-ads/build.gradle @@ -0,0 +1,44 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* License); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an AS IS BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +plugins { id 'org.apache.beam.module' } +applyJavaNature( automaticModuleName: 'org.apache.beam.sdk.io.googleads') + +description = "Apache Beam :: SDKs :: Java :: IO :: Google Ads" +ext.summary = "IO to read from Google Ads" + +dependencies { + implementation project(path: ":sdks:java:core", configuration: "shadow") + implementation project(path: ":sdks:java:extensions:google-cloud-platform-core") + implementation library.java.jackson_annotations + implementation library.java.gax + implementation library.java.google_auth_library_credentials + implementation library.java.google_auth_library_oauth2_http + implementation library.java.protobuf_java + implementation library.java.protobuf_java_util + implementation library.java.google_ads + implementation library.java.google_ads_stubs_v14 + implementation library.java.joda_time + implementation library.java.vendored_guava_32_1_2_jre + testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") + testImplementation project(path: ":sdks:java:io:common", configuration: "testRuntimeMigration") + testImplementation library.java.mockito_core + testImplementation library.java.junit + testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") + testRuntimeOnly library.java.slf4j_jdk14 +} diff --git a/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/DefaultGoogleAdsClientFactory.java b/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/DefaultGoogleAdsClientFactory.java new file mode 100644 index 0000000000000..ace5d4d1b124f --- /dev/null +++ b/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/DefaultGoogleAdsClientFactory.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.googleads; + +import com.google.ads.googleads.lib.GoogleAdsClient; +import com.google.auth.Credentials; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** The default way to construct a {@link GoogleAdsClient}. */ +public class DefaultGoogleAdsClientFactory implements GoogleAdsClientFactory { + private static final DefaultGoogleAdsClientFactory INSTANCE = new DefaultGoogleAdsClientFactory(); + + public static DefaultGoogleAdsClientFactory getInstance() { + return INSTANCE; + } + + @Override + public GoogleAdsClient newGoogleAdsClient( + GoogleAdsOptions options, + @Nullable String developerToken, + @Nullable Long linkedCustomerId, + @Nullable Long loginCustomerId) { + + GoogleAdsClient.Builder builder = GoogleAdsClient.newBuilder(); + + Credentials credentials = options.getGoogleAdsCredential(); + if (credentials != null) { + builder.setCredentials(credentials); + } + + if (options.getGoogleAdsEndpoint() != null) { + builder.setEndpoint(options.getGoogleAdsEndpoint()); + } + + String developerTokenFromOptions = options.getGoogleAdsDeveloperToken(); + if (developerToken != null) { + builder.setDeveloperToken(developerToken); + } else if (developerTokenFromOptions != null) { + builder.setDeveloperToken(developerTokenFromOptions); + } + + if (linkedCustomerId != null) { + builder.setLinkedCustomerId(linkedCustomerId); + } + + if (loginCustomerId != null) { + builder.setLoginCustomerId(loginCustomerId); + } + + return builder.build(); + } +} diff --git a/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsClientFactory.java b/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsClientFactory.java new file mode 100644 index 0000000000000..cef32f5c4b9b3 --- /dev/null +++ b/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsClientFactory.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.googleads; + +import com.google.ads.googleads.lib.GoogleAdsClient; +import java.io.Serializable; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** Defines how to construct a {@link GoogleAdsClient}. */ +public interface GoogleAdsClientFactory extends Serializable { + GoogleAdsClient newGoogleAdsClient( + GoogleAdsOptions options, + @Nullable String developerToken, + @Nullable Long linkedCustomerId, + @Nullable Long loginCustomerId); +} diff --git a/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsIO.java b/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsIO.java new file mode 100644 index 0000000000000..de176a05e9bec --- /dev/null +++ b/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsIO.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.googleads; + +/** + * {@link GoogleAdsIO} provides an API for reading from the Google Ads API over different + * versions of the Google Ads client libraries. + * + * @see GoogleAdsV14 + */ +public class GoogleAdsIO { + private GoogleAdsIO() {} + + public static GoogleAdsV14 v14() { + return GoogleAdsV14.INSTANCE; + } +} diff --git a/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsOptions.java b/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsOptions.java new file mode 100644 index 0000000000000..738760c22eb2d --- /dev/null +++ b/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsOptions.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.googleads; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.auth.Credentials; +import java.io.IOException; +import java.security.GeneralSecurityException; +import org.apache.beam.sdk.extensions.gcp.auth.CredentialFactory; +import org.apache.beam.sdk.options.Default; +import org.apache.beam.sdk.options.DefaultValueFactory; +import org.apache.beam.sdk.options.Description; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.util.InstanceBuilder; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** Options used to configure Google Ads API specific options. */ +public interface GoogleAdsOptions extends PipelineOptions { + /** Host endpoint to use for connections to the Google Ads API. */ + @Description("Host endpoint to use for connections to the Google Ads API.") + @Default.String("googleads.googleapis.com:443") + String getGoogleAdsEndpoint(); + + void setGoogleAdsEndpoint(String endpoint); + + /** + * OAuth 2.0 Client ID identifying the application. + * + * @see https://developers.google.com/google-ads/api/docs/oauth/overview + * @see https://developers.google.com/identity/protocols/oauth2 + */ + @Description("OAuth 2.0 Client ID identifying the application.") + String getGoogleAdsClientId(); + + void setGoogleAdsClientId(String clientId); + + /** + * OAuth 2.0 Client Secret for the specified Client ID. + * + * @see https://developers.google.com/google-ads/api/docs/oauth/overview + * @see https://developers.google.com/identity/protocols/oauth2 + */ + @Description("OAuth 2.0 Client Secret for the specified Client ID.") + String getGoogleAdsClientSecret(); + + void setGoogleAdsClientSecret(String clientSecret); + + /** + * OAuth 2.0 Refresh Token for the user connecting to the Google Ads API. + * + * @see https://developers.google.com/google-ads/api/docs/oauth/overview + * @see https://developers.google.com/identity/protocols/oauth2 + */ + @Description("OAuth 2.0 Refresh Token for the user connecting to the Google Ads API.") + String getGoogleAdsRefreshToken(); + + void setGoogleAdsRefreshToken(String refreshToken); + + /** Google Ads developer token for the user connecting to the Google Ads API. */ + @Description("Google Ads developer token for the user connecting to the Google Ads API.") + @Nullable + String getGoogleAdsDeveloperToken(); + + void setGoogleAdsDeveloperToken(String developerToken); + + /** + * The class of the credential factory to create credentials if none have been explicitly set. + * + * @see #getGoogleAdsCredential() + */ + @Description( + "The class of the credential factory to create credentials if none have been explicitly set.") + @Default.Class(GoogleAdsUserCredentialFactory.class) + Class getGoogleAdsCredentialFactoryClass(); + + void setGoogleAdsCredentialFactoryClass( + Class credentialFactoryClass); + + /** + * The credential instance that should be used to authenticate against the Google Ads API. + * Defaults to a credential instance constructed by the credential factory. + * + * @see #getGoogleAdsCredential() + * @see https://github.com/googleapis/google-auth-library-java + */ + @JsonIgnore + @Description( + "The credential instance that should be used to authenticate against the Google Ads API. " + + "Defaults to a credential instance constructed by the credential factory.") + @Default.InstanceFactory(GoogleAdsCredentialsFactory.class) + @Nullable + Credentials getGoogleAdsCredential(); + + void setGoogleAdsCredential(Credentials credential); + + /** + * Attempts to load the Google Ads credentials. See {@link CredentialFactory#getCredential()} for + * more details. + */ + class GoogleAdsCredentialsFactory implements DefaultValueFactory<@Nullable Credentials> { + @Override + public @Nullable Credentials create(PipelineOptions options) { + GoogleAdsOptions googleAdsOptions = options.as(GoogleAdsOptions.class); + try { + CredentialFactory factory = + InstanceBuilder.ofType(CredentialFactory.class) + .fromClass(googleAdsOptions.getGoogleAdsCredentialFactoryClass()) + .fromFactoryMethod("fromOptions") + .withArg(PipelineOptions.class, options) + .build(); + return factory.getCredential(); + } catch (IOException | GeneralSecurityException e) { + throw new RuntimeException("Unable to obtain credential", e); + } + } + } +} diff --git a/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsUserCredentialFactory.java b/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsUserCredentialFactory.java new file mode 100644 index 0000000000000..50e05a1d51961 --- /dev/null +++ b/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsUserCredentialFactory.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.googleads; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import com.google.auth.Credentials; +import com.google.auth.oauth2.UserCredentials; +import org.apache.beam.sdk.extensions.gcp.auth.CredentialFactory; +import org.apache.beam.sdk.options.PipelineOptions; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * Constructs and returns {@link Credentials} to be used by Google Ads API calls. This factory only + * supports {@link com.google.auth.oauth2.UserCredentials}, {@link + * com.google.auth.oauth2.ServiceAccountCredentials} and domain-wide delegation are not supported. + */ +public class GoogleAdsUserCredentialFactory implements CredentialFactory { + // The OAuth client ID, client secret, and refresh token. + private String clientId; + private String clientSecret; + private String refreshToken; + + private GoogleAdsUserCredentialFactory( + String clientId, String clientSecret, String refreshToken) { + this.clientId = clientId; + this.clientSecret = clientSecret; + this.refreshToken = refreshToken; + } + + public static GoogleAdsUserCredentialFactory fromOptions(PipelineOptions options) { + GoogleAdsOptions adsOptions = options.as(GoogleAdsOptions.class); + + checkArgument( + adsOptions.getGoogleAdsClientId() != null + && adsOptions.getGoogleAdsClientSecret() != null + && adsOptions.getGoogleAdsRefreshToken() != null, + "googleAdsClientId, googleAdsClientSecret and googleAdsRefreshToken must not be null"); + + return new GoogleAdsUserCredentialFactory( + adsOptions.getGoogleAdsClientId(), + adsOptions.getGoogleAdsClientSecret(), + adsOptions.getGoogleAdsRefreshToken()); + } + + /** Returns {@link Credentials} as configured by {@link GoogleAdsOptions}. */ + @Override + public @Nullable Credentials getCredential() { + return UserCredentials.newBuilder() + .setClientId(clientId) + .setClientSecret(clientSecret) + .setRefreshToken(refreshToken) + .build(); + } +} diff --git a/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsV14.java b/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsV14.java new file mode 100644 index 0000000000000..46448e0110d90 --- /dev/null +++ b/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsV14.java @@ -0,0 +1,669 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.googleads; + +import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import com.google.ads.googleads.lib.GoogleAdsClient; +import com.google.ads.googleads.v14.errors.GoogleAdsError; +import com.google.ads.googleads.v14.errors.GoogleAdsException; +import com.google.ads.googleads.v14.errors.GoogleAdsFailure; +import com.google.ads.googleads.v14.errors.InternalErrorEnum; +import com.google.ads.googleads.v14.errors.QuotaErrorEnum; +import com.google.ads.googleads.v14.services.GoogleAdsRow; +import com.google.ads.googleads.v14.services.GoogleAdsServiceClient; +import com.google.ads.googleads.v14.services.SearchGoogleAdsStreamRequest; +import com.google.ads.googleads.v14.services.SearchGoogleAdsStreamResponse; +import com.google.auto.value.AutoValue; +import com.google.protobuf.Message; +import com.google.protobuf.util.Durations; +import java.io.IOException; +import java.io.Serializable; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.DoFn.ProcessContext; +import org.apache.beam.sdk.transforms.DoFn.ProcessElement; +import org.apache.beam.sdk.transforms.DoFn.Setup; +import org.apache.beam.sdk.transforms.DoFn.Teardown; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.display.DisplayData; +import org.apache.beam.sdk.util.BackOff; +import org.apache.beam.sdk.util.BackOffUtils; +import org.apache.beam.sdk.util.FluentBackoff; +import org.apache.beam.sdk.util.Sleeper; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.TypeDescriptor; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.RateLimiter; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.joda.time.Duration; + +/** + * {@link GoogleAdsV14} provides an API to read Google Ads API v14 reports. + * + *

    The Google Ads API does not use service account credentials in the same way as Google Cloud + * Platform APIs do. Service account credentials are typically only used to delegate (using + * domain-wide delegation) access through end user accounts. Providing credentials using the OAuth2 + * desktop flow may be preferable over domain wide delegation. Please refer to the Google Ads API + * documentation for more information on OAuth2 in the Google Ads API. + * + *

    Defaults for OAuth 2.0 credentials, refresh token and developer token can be provided using + * the following flags: + * + *

    + *   --googleAdsClientId=your-client-id
    + *   --googleAdsClientSecret=your-client-secret
    + *   --googleAdsRefreshToken=your-refresh-token
    + *   --googleAdsDeveloperToken=your-developer-token
    + * 
    + * + *

    Use {@link GoogleAdsV14#read()} to read either a bounded or unbounded {@link PCollection} of + * {@link GoogleAdsRow} from a single Google Ads Query + * Language query using {@link Read#withQuery(String)} and a {@link PCollection} of customer + * IDs. Alternatively, use {@link GoogleAdsV14#readAll()} to read either a bounded or unbounded + * {@link PCollection} of {@link GoogleAdsRow} from a {@link PCollection} of {@link + * SearchGoogleAdsStreamRequest} potentially containing many different queries. + * + *

    For example, using {@link GoogleAdsV14#read()}: + * + *

    {@code
    + * Pipeline p = Pipeline.create();
    + * PCollection customerIds =
    + *     p.apply(Create.of(Long.toString(1234567890L)));
    + * PCollection rows =
    + *     customerIds.apply(
    + *         GoogleAdsIO.v14()
    + *             .read()
    + *             .withRateLimitPolicy(MY_RATE_LIMIT_POLICY)
    + *             .withQuery(
    + *                 "SELECT"
    + *                     + "campaign.id,"
    + *                     + "campaign.name,"
    + *                     + "campaign.status"
    + *                     + "FROM campaign"));
    + * p.run();
    + * }
    + * + *

    Alternatively, using {@link GoogleAdsV14#readAll()} to execute requests from a {@link + * PCollection} of {@link SearchGoogleAdsStreamRequest}: + * + *

    {@code
    + * Pipeline p = Pipeline.create();
    + * PCollection requests =
    + *     p.apply(
    + *         Create.of(
    + *             ImmutableList.of(
    + *                 SearchGoogleAdsStreamRequest.newBuilder()
    + *                     .setCustomerId(Long.toString(1234567890L))
    + *                     .setQuery(
    + *                         "SELECT"
    + *                             + "campaign.id,"
    + *                             + "campaign.name,"
    + *                             + "campaign.status"
    + *                             + "FROM campaign")
    + *                     .build())));
    + * PCollection rows =
    + *     requests.apply(GoogleAdsIO.v14().readAll().withRateLimitPolicy(MY_RATE_LIMIT_POLICY));
    + * p.run();
    + * }
    + * + *

    Client-side rate limiting

    + * + * On construction of a {@link GoogleAdsV14#read()} or {@link GoogleAdsV14#readAll()} transform a + * rate limiting policy must be specified to stay well under the assigned quota for the Google Ads + * API. The Google Ads API enforces global rate limits from the developer token down to the customer + * ID and depending on the access level of the developer token a limit on the total number of + * executed operations per day. See Rate + * Limits and API Limits and + * Quotas in the Google Ads documentation for more details. + * + *

    It is recommended to host a shared rate limiting service to coordinate traffic to the Google + * Ads API across all applications using the same developer token. Users of these transforms are + * strongly advised to implement their own {@link RateLimitPolicy} and {@link + * RateLimitPolicyFactory} to interact with a shared rate limiting service (e.g. gubernator) for any production workloads. + * + *

    Required Minimum Functionality

    + * + * Pipelines built using these transforms may still be subject to the Required Minimum Functionality + * policy. Please review the policy carefully and have your tool reviewed by the Google Ads API + * Review Team. See Required Minimum + * Functionality and Rate + * sheet & non-compliance fees in the Google Ads API documentation for more details. + * + * @see GoogleAdsIO#v14() + * @see GoogleAdsOptions + * @see Best + * Practices in the Google Ads documentation + */ +public class GoogleAdsV14 { + static final GoogleAdsV14 INSTANCE = new GoogleAdsV14(); + + private GoogleAdsV14() {} + + public Read read() { + return new AutoValue_GoogleAdsV14_Read.Builder() + .setGoogleAdsClientFactory(DefaultGoogleAdsClientFactory.getInstance()) + .build(); + } + + public ReadAll readAll() { + return new AutoValue_GoogleAdsV14_ReadAll.Builder() + .setGoogleAdsClientFactory(DefaultGoogleAdsClientFactory.getInstance()) + .build(); + } + + /** + * A {@link PTransform} that reads the results of a Google Ads query as {@link GoogleAdsRow} + * objects. + * + * @see GoogleAdsIO#v14() + * @see #readAll() + */ + @AutoValue + public abstract static class Read + extends PTransform, PCollection> { + abstract @Nullable String getDeveloperToken(); + + abstract @Nullable Long getLoginCustomerId(); + + abstract @Nullable String getQuery(); + + abstract GoogleAdsClientFactory getGoogleAdsClientFactory(); + + abstract @Nullable RateLimitPolicyFactory getRateLimitPolicyFactory(); + + abstract Builder toBuilder(); + + @AutoValue.Builder + abstract static class Builder { + abstract Builder setDeveloperToken(@Nullable String developerToken); + + abstract Builder setLoginCustomerId(@Nullable Long loginCustomerId); + + abstract Builder setQuery(String query); + + abstract Builder setGoogleAdsClientFactory(GoogleAdsClientFactory googleAdsClientFactory); + + abstract Builder setRateLimitPolicyFactory(RateLimitPolicyFactory rateLimitPolicyFactory); + + abstract Read build(); + } + + /** + * Creates and returns a new {@link Read} transform with the specified developer token. A + * developer token is required to access the Google Ads API. + * + * @param developerToken The developer token to set. + * @return A new {@link Read} transform with the specified developer token. + * @see GoogleAdsClient + */ + public Read withDeveloperToken(@Nullable String developerToken) { + return toBuilder().setDeveloperToken(developerToken).build(); + } + + /** + * Creates and returns a new {@link Read} transform with the specified login customer ID. A + * login customer ID is only required for manager accounts. + * + * @param loginCustomerId The login customer ID to set. + * @return A new {@link Read} transform with the specified login customer ID. + * @see GoogleAdsClient + */ + public Read withLoginCustomerId(@Nullable Long loginCustomerId) { + return toBuilder().setLoginCustomerId(loginCustomerId).build(); + } + + /** + * Creates and returns a new {@link Read} transform with the specified query. The query will be + * executed for each customer ID. + * + * @param query + * @return A new {@link Read} transform with the specified query. + * @see SearchGoogleAdsStreamRequest + */ + public Read withQuery(String query) { + checkArgumentNotNull(query, "query cannot be null"); + checkArgument(!query.isEmpty(), "query cannot be empty"); + + return toBuilder().setQuery(query).build(); + } + + /** + * Creates and returns a new {@link Read} transform with the specified client factory. A {@link + * GoogleAdsClientFactory} builds the {@link GoogleAdsClient} used to construct service clients. + * The {@link DefaultGoogleAdsClientFactory} should be sufficient for most purposes unless the + * construction of {@link GoogleAdsClient} requires customization. + * + * @param googleAdsClientFactory + * @return A new {@link Read} transform with the specified client factory. + * @see GoogleAdsClient + */ + public Read withGoogleAdsClientFactory(GoogleAdsClientFactory googleAdsClientFactory) { + checkArgumentNotNull(googleAdsClientFactory, "googleAdsClientFactory cannot be null"); + + return toBuilder().setGoogleAdsClientFactory(googleAdsClientFactory).build(); + } + + /** + * Creates and returns a new {@link Read} transform with the specified rate limit policy + * factory. A {@link RateLimitPolicyFactory} builds the {@link RateLimitPolicy} used to limit + * the number of requests made by {@link ReadAll.ReadAllFn}. The Google Ads API enforces global + * limits from the developer token down to the customer ID and it is recommended to host a + * shared rate limiting service to coordinate traffic to the Google Ads API across all + * applications using the same developer token. Users of these transforms are strongly advised + * to implement their own {@link RateLimitPolicy} and {@link RateLimitPolicyFactory} to interact + * with a shared rate limiting service for any production workloads. + * + * @param rateLimitPolicyFactory + * @return A new {@link Read} transform with the specified rate limit policy factory. + * @see GoogleAdsClient + */ + public Read withRateLimitPolicy(RateLimitPolicyFactory rateLimitPolicyFactory) { + checkArgumentNotNull(rateLimitPolicyFactory, "rateLimitPolicyFactory cannot be null"); + + return toBuilder().setRateLimitPolicyFactory(rateLimitPolicyFactory).build(); + } + + @Override + public PCollection expand(PCollection input) { + String query = getQuery(); + RateLimitPolicyFactory rateLimitPolicyFactory = getRateLimitPolicyFactory(); + checkArgumentNotNull(query, "withQuery() is required"); + checkArgumentNotNull(rateLimitPolicyFactory, "withRateLimitPolicy() is required"); + + return input + .apply( + MapElements.into(TypeDescriptor.of(SearchGoogleAdsStreamRequest.class)) + .via( + customerId -> + SearchGoogleAdsStreamRequest.newBuilder() + .setCustomerId(customerId) + .setQuery(query) + .build())) + .apply( + INSTANCE + .readAll() + .withDeveloperToken(getDeveloperToken()) + .withLoginCustomerId(getLoginCustomerId()) + .withGoogleAdsClientFactory(getGoogleAdsClientFactory()) + .withRateLimitPolicy(rateLimitPolicyFactory)); + } + + @Override + public void populateDisplayData(DisplayData.Builder builder) { + super.populateDisplayData(builder); + builder.addIfNotNull( + DisplayData.item("query", String.valueOf(getQuery())).withLabel("Query")); + } + } + + /** + * A {@link PTransform} that reads the results of many {@link SearchGoogleAdsStreamRequest} + * objects as {@link GoogleAdsRow} objects. * + * + * @see GoogleAdsIO#v14() + * @see #readAll() + */ + @AutoValue + public abstract static class ReadAll + extends PTransform, PCollection> { + abstract @Nullable String getDeveloperToken(); + + abstract @Nullable Long getLoginCustomerId(); + + abstract GoogleAdsClientFactory getGoogleAdsClientFactory(); + + abstract @Nullable RateLimitPolicyFactory getRateLimitPolicyFactory(); + + abstract Builder toBuilder(); + + @AutoValue.Builder + abstract static class Builder { + abstract Builder setDeveloperToken(@Nullable String developerToken); + + abstract Builder setLoginCustomerId(@Nullable Long loginCustomerId); + + abstract Builder setGoogleAdsClientFactory(GoogleAdsClientFactory googleAdsClientFactory); + + abstract Builder setRateLimitPolicyFactory(RateLimitPolicyFactory rateLimitPolicyFactory); + + abstract ReadAll build(); + } + + /** + * Creates and returns a new {@link ReadAll} transform with the specified developer token. A + * developer token is required to access the Google Ads API. + * + * @param developerToken The developer token to set. + * @return A new {@link ReadAll} transform with the specified developer token. + * @see GoogleAdsClient + */ + public ReadAll withDeveloperToken(@Nullable String developerToken) { + return toBuilder().setDeveloperToken(developerToken).build(); + } + + /** + * Creates and returns a new {@link ReadAll} transform with the specified login customer ID. A + * login customer ID is only required for manager accounts. + * + * @param loginCustomerId The login customer ID to set. + * @return A new {@link ReadAll} transform with the specified login customer ID. + * @see GoogleAdsClient + */ + public ReadAll withLoginCustomerId(@Nullable Long loginCustomerId) { + return toBuilder().setLoginCustomerId(loginCustomerId).build(); + } + + /** + * Creates and returns a new {@link ReadAll} transform with the specified client factory. A + * {@link GoogleAdsClientFactory} builds the {@link GoogleAdsClient} used to construct service + * clients. The {@link DefaultGoogleAdsClientFactory} should be sufficient for most purposes + * unless the construction of {@link GoogleAdsClient} requires customization. + * + * @param googleAdsClientFactory + * @return A new {@link ReadAll} transform with the specified client factory. + * @see GoogleAdsClient + */ + public ReadAll withGoogleAdsClientFactory(GoogleAdsClientFactory googleAdsClientFactory) { + checkArgumentNotNull(googleAdsClientFactory, "googleAdsClientFactory cannot be null"); + + return toBuilder().setGoogleAdsClientFactory(googleAdsClientFactory).build(); + } + + /** + * Creates and returns a new {@link ReadAll} transform with the specified rate limit policy + * factory. A {@link RateLimitPolicyFactory} builds the {@link RateLimitPolicy} used to limit + * the number of requests made by {@link ReadAll.ReadAllFn}. The Google Ads API enforces global + * limits from the developer token down to the customer ID and it is recommended to host a + * shared rate limiting service to coordinate traffic to the Google Ads API across all + * applications using the same developer token. Users of these transforms are strongly advised + * to implement their own {@link RateLimitPolicy} and {@link RateLimitPolicyFactory} to interact + * with a shared rate limiting service for any production workloads. + * + * @param rateLimitPolicyFactory + * @return A new {@link ReadAll} transform with the specified rate limit policy factory. + * @see GoogleAdsClient + */ + public ReadAll withRateLimitPolicy(RateLimitPolicyFactory rateLimitPolicyFactory) { + checkArgumentNotNull(rateLimitPolicyFactory, "rateLimitPolicyFactory cannot be null"); + + return toBuilder().setRateLimitPolicyFactory(rateLimitPolicyFactory).build(); + } + + @Override + public PCollection expand(PCollection input) { + GoogleAdsOptions options = input.getPipeline().getOptions().as(GoogleAdsOptions.class); + + checkArgument( + options.getGoogleAdsDeveloperToken() != null || getDeveloperToken() != null, + "either --googleAdsDeveloperToken or .withDeveloperToken() is required"); + checkArgumentNotNull(getRateLimitPolicyFactory(), "withRateLimitPolicy() is required"); + + return input.apply(ParDo.of(new ReadAllFn(this))); + } + + /** + * A {@link DoFn} that reads reports from Google Ads for each query using the {@code + * SearchStream} method. + */ + @VisibleForTesting + static class ReadAllFn extends DoFn { + // The default retry configuration is based on that of services with a comparable + // potential volume of requests to the Google Ads API. + private static final int MAX_RETRIES = 5; + private static final FluentBackoff BACKOFF = + FluentBackoff.DEFAULT + .withExponent(2.0) + .withInitialBackoff(Duration.standardSeconds(30)) + .withMaxRetries(MAX_RETRIES); + + @VisibleForTesting static Sleeper sleeper = Sleeper.DEFAULT; + + private final GoogleAdsV14.ReadAll spec; + + private transient @Nullable GoogleAdsClient googleAdsClient; + private transient @Nullable GoogleAdsServiceClient googleAdsServiceClient; + private transient @Nullable RateLimitPolicy rateLimitPolicy; + + ReadAllFn(GoogleAdsV14.ReadAll spec) { + this.spec = spec; + } + + @Setup + @EnsuresNonNull({"googleAdsClient", "googleAdsServiceClient", "rateLimitPolicy"}) + public void setup(PipelineOptions options) { + GoogleAdsOptions adsOptions = options.as(GoogleAdsOptions.class); + + final GoogleAdsClient googleAdsClient = + spec.getGoogleAdsClientFactory() + .newGoogleAdsClient( + adsOptions, spec.getDeveloperToken(), null, spec.getLoginCustomerId()); + final GoogleAdsServiceClient googleAdsServiceClient = + googleAdsClient.getVersion14().createGoogleAdsServiceClient(); + final RateLimitPolicy rateLimitPolicy = + checkStateNotNull(spec.getRateLimitPolicyFactory()).getRateLimitPolicy(); + + this.googleAdsClient = googleAdsClient; + this.googleAdsServiceClient = googleAdsServiceClient; + this.rateLimitPolicy = rateLimitPolicy; + } + + @ProcessElement + @RequiresNonNull({"googleAdsClient", "googleAdsServiceClient", "rateLimitPolicy"}) + public void processElement(ProcessContext c) throws IOException, InterruptedException { + final GoogleAdsClient googleAdsClient = this.googleAdsClient; + final GoogleAdsServiceClient googleAdsServiceClient = this.googleAdsServiceClient; + final RateLimitPolicy rateLimitPolicy = this.rateLimitPolicy; + + BackOff backoff = BACKOFF.backoff(); + BackOff nextBackoff = backoff; + GoogleAdsException lastException = null; + + SearchGoogleAdsStreamRequest request = c.element(); + String developerToken = googleAdsClient.getDeveloperToken(); + String customerId = request.getCustomerId(); + + do { + rateLimitPolicy.onBeforeRequest(developerToken, customerId, request); + + try { + for (SearchGoogleAdsStreamResponse response : + googleAdsServiceClient.searchStreamCallable().call(request)) { + for (GoogleAdsRow row : response.getResultsList()) { + c.output(row); + } + } + rateLimitPolicy.onSuccess(developerToken, customerId, request); + return; + } catch (GoogleAdsException e) { + GoogleAdsError retryableError = + findFirstRetryableError(e.getGoogleAdsFailure()) + .orElseThrow(() -> new IOException(e)); + + rateLimitPolicy.onError(developerToken, customerId, request, retryableError); + + // If the error happens to carry a suggested retry delay, then use that instead. + // Retry these errors without incrementing the retry count or backoff interval. + // For all other retryable errors fall back to the existing backoff. + if (retryableError.getDetails().getQuotaErrorDetails().hasRetryDelay()) { + nextBackoff = + new BackOff() { + @Override + public void reset() {} + + @Override + public long nextBackOffMillis() { + return Durations.toMillis( + retryableError.getDetails().getQuotaErrorDetails().getRetryDelay()); + } + }; + } else { + nextBackoff = backoff; + } + } + } while (BackOffUtils.next(sleeper, nextBackoff)); + + throw new IOException( + String.format( + "Unable to get Google Ads response after retrying %d times using query (%s)", + MAX_RETRIES, request.getQuery()), + lastException); + } + + @Teardown + public void teardown() { + if (googleAdsServiceClient != null) { + googleAdsServiceClient.close(); + } + } + + private Optional findFirstRetryableError(GoogleAdsFailure e) { + return e.getErrorsList().stream() + .filter( + err -> + // Unexpected internal error + err.getErrorCode().getInternalError() + == InternalErrorEnum.InternalError.INTERNAL_ERROR + || + // Unexpected transient error + err.getErrorCode().getInternalError() + == InternalErrorEnum.InternalError.TRANSIENT_ERROR + || + // Too many requests + err.getErrorCode().getQuotaError() + == QuotaErrorEnum.QuotaError.RESOURCE_EXHAUSTED + || + // Too many requests in a short amount of time + err.getErrorCode().getQuotaError() + == QuotaErrorEnum.QuotaError.RESOURCE_TEMPORARILY_EXHAUSTED) + .findFirst(); + } + } + } + + /** + * Implement this interface to create a {@link RateLimitPolicy}. This should be used to limit all + * traffic sent to the Google Ads API for a pair of developer token and customer ID and any other + * relevant attributes for the specific Google Ads API service being called. + */ + public interface RateLimitPolicyFactory extends Serializable { + RateLimitPolicy getRateLimitPolicy(); + } + + /** + * This interface can be used to implement custom client-side rate limiting policies. Custom + * policies should follow best practices for interacting with the Google Ads API. + * + * @see Best + * Practices in the Google Ads documentation + */ + public interface RateLimitPolicy { + /** + * Called before a request is sent. + * + * @param developerToken The developer token used for the request. + * @param customerId The customer ID specified on the request. + * @param request Any Google Ads API request. + * @throws InterruptedException + */ + void onBeforeRequest(String developerToken, String customerId, Message request) + throws InterruptedException; + + /** + * Called after a request succeeds. + * + * @param developerToken The developer token used for the request. + * @param customerId The customer ID specified on the request. + * @param request Any Google Ads API request. + */ + void onSuccess(String developerToken, String customerId, Message request); + + /** + * Called after a request fails with a retryable error. + * + * @param developerToken The developer token used for the request. + * @param customerId The customer ID specified on the request. + * @param request Any Google Ads API request. + * @param error A retryable error. + */ + void onError(String developerToken, String customerId, Message request, GoogleAdsError error); + } + + /** + * This rate limit policy wraps a {@link RateLimiter} and can be used in low volume and + * development use cases as a client-side rate limiting policy. This policy does not enforce a + * global (per pipeline or otherwise) rate limit to requests and should not be used in deployments + * where the Google Ads API quota is shared between multiple applications. + * + *

    This policy can be used to limit requests across all {@link GoogleAdsV14.Read} or {@link + * GoogleAdsV14.ReadAll} transforms by defining and using a {@link + * GoogleAdsV14.RateLimitPolicyFactory} which holds a shared static {@link + * GoogleAdsV14.SimpleRateLimitPolicy}. Note that the desired rate must be divided by the expected + * maximum number of workers for the pipeline, otherwise the pipeline may exceed the desired rate + * after an upscaling event. + * + *

    {@code
    +   * public class SimpleRateLimitPolicyFactory implements GoogleAdsV14.RateLimitPolicyFactory {
    +   *   private static final GoogleAdsV14.RateLimitPolicy POLICY =
    +   *       new GoogleAdsV14.SimpleRateLimitPolicy(1.0 / 1000.0);
    +   *
    +   *   @Override
    +   *   public GoogleAdsV14.RateLimitPolicy getRateLimitPolicy() {
    +   *     return POLICY;
    +   *   }
    +   * }
    +   * }
    + */ + public static class SimpleRateLimitPolicy implements RateLimitPolicy { + private final RateLimiter rateLimiter; + + public SimpleRateLimitPolicy(double permitsPerSecond) { + rateLimiter = RateLimiter.create(permitsPerSecond); + } + + public SimpleRateLimitPolicy(double permitsPerSecond, long warmupPeriod, TimeUnit unit) { + rateLimiter = RateLimiter.create(permitsPerSecond, warmupPeriod, unit); + } + + @Override + public void onBeforeRequest(String developerToken, String customerId, Message request) + throws InterruptedException { + rateLimiter.acquire(); + } + + @Override + public void onSuccess(String developerToken, String customerId, Message request) {} + + @Override + public void onError( + String developerToken, String customerId, Message request, GoogleAdsError error) {} + } +} diff --git a/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/package-info.java b/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/package-info.java new file mode 100644 index 0000000000000..56aaed903e3c4 --- /dev/null +++ b/sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/package-info.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Defines transforms for reading from Google Ads. + * + * @see org.apache.beam.sdk.io.googleads.GoogleAdsIO + */ +package org.apache.beam.sdk.io.googleads; diff --git a/sdks/java/io/google-ads/src/test/java/org/apache/beam/sdk/io/googleads/DummyRateLimitPolicy.java b/sdks/java/io/google-ads/src/test/java/org/apache/beam/sdk/io/googleads/DummyRateLimitPolicy.java new file mode 100644 index 0000000000000..c2d3b230c13ce --- /dev/null +++ b/sdks/java/io/google-ads/src/test/java/org/apache/beam/sdk/io/googleads/DummyRateLimitPolicy.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.googleads; + +import com.google.ads.googleads.v14.errors.GoogleAdsError; +import com.google.protobuf.Message; + +public class DummyRateLimitPolicy implements GoogleAdsV14.RateLimitPolicy { + @Override + public void onBeforeRequest(String developerToken, String customerId, Message request) + throws InterruptedException {} + + @Override + public void onSuccess(String developerToken, String customerId, Message request) {} + + @Override + public void onError( + String developerToken, String customerId, Message request, GoogleAdsError error) {} +} diff --git a/sdks/java/io/google-ads/src/test/java/org/apache/beam/sdk/io/googleads/GoogleAdsV14Test.java b/sdks/java/io/google-ads/src/test/java/org/apache/beam/sdk/io/googleads/GoogleAdsV14Test.java new file mode 100644 index 0000000000000..efe1694691c3a --- /dev/null +++ b/sdks/java/io/google-ads/src/test/java/org/apache/beam/sdk/io/googleads/GoogleAdsV14Test.java @@ -0,0 +1,519 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.googleads; + +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.when; + +import com.google.ads.googleads.v14.errors.AuthenticationErrorEnum.AuthenticationError; +import com.google.ads.googleads.v14.errors.ErrorCode; +import com.google.ads.googleads.v14.errors.ErrorDetails; +import com.google.ads.googleads.v14.errors.GoogleAdsError; +import com.google.ads.googleads.v14.errors.GoogleAdsException; +import com.google.ads.googleads.v14.errors.GoogleAdsFailure; +import com.google.ads.googleads.v14.errors.InternalErrorEnum.InternalError; +import com.google.ads.googleads.v14.errors.QuotaErrorDetails; +import com.google.ads.googleads.v14.errors.QuotaErrorEnum.QuotaError; +import com.google.ads.googleads.v14.services.GoogleAdsRow; +import com.google.ads.googleads.v14.services.SearchGoogleAdsStreamRequest; +import com.google.ads.googleads.v14.services.SearchGoogleAdsStreamResponse; +import com.google.api.gax.grpc.GrpcStatusCode; +import com.google.api.gax.rpc.ApiException; +import com.google.protobuf.Duration; +import io.grpc.Metadata; +import io.grpc.Status.Code; +import java.io.IOException; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.Pipeline.PipelineExecutionException; +import org.apache.beam.sdk.extensions.gcp.auth.NoopCredentialFactory; +import org.apache.beam.sdk.io.googleads.GoogleAdsV14.RateLimitPolicyFactory; +import org.apache.beam.sdk.testing.NeedsRunner; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.TypeDescriptor; +import org.apache.beam.sdk.values.TypeDescriptors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(Enclosed.class) +public class GoogleAdsV14Test { + static final RateLimitPolicyFactory TEST_POLICY_FACTORY = () -> new DummyRateLimitPolicy(); + + @RunWith(JUnit4.class) + public static class ConstructionTests { + private final transient TestPipeline pipeline = TestPipeline.create(); + + @Test + public void testReadAllExpandWithDeveloperTokenFromBuilder() { + pipeline + .apply(Create.empty(new TypeDescriptor() {})) + .apply( + GoogleAdsIO.v14() + .readAll() + .withRateLimitPolicy(TEST_POLICY_FACTORY) + .withDeveloperToken("abc")); + } + + @Test + public void testReadAllExpandWithDeveloperTokenFromOptions() { + pipeline.getOptions().as(GoogleAdsOptions.class).setGoogleAdsDeveloperToken("abc"); + pipeline + .apply(Create.empty(new TypeDescriptor() {})) + .apply(GoogleAdsIO.v14().readAll().withRateLimitPolicy(TEST_POLICY_FACTORY)); + } + + @Test + public void testReadAllExpandWithDeveloperTokenFromOptionsAndBuilder() { + pipeline.getOptions().as(GoogleAdsOptions.class).setGoogleAdsDeveloperToken("abc"); + pipeline + .apply(Create.empty(new TypeDescriptor() {})) + .apply( + GoogleAdsIO.v14() + .readAll() + .withRateLimitPolicy(TEST_POLICY_FACTORY) + .withDeveloperToken(null)); + } + + @Test + public void testReadAllExpandWithoutDeveloperToken() throws Exception { + Assert.assertThrows( + "Developer token required but not provided", + IllegalArgumentException.class, + () -> + pipeline + .apply(Create.empty(new TypeDescriptor() {})) + .apply(GoogleAdsIO.v14().readAll().withRateLimitPolicy(TEST_POLICY_FACTORY))); + } + + @Test + public void testReadAllExpandWithoutRateLimitPolicy() throws Exception { + Assert.assertThrows( + "Rate limit policy required but not provided", + IllegalArgumentException.class, + () -> + pipeline + .apply(Create.empty(new TypeDescriptor() {})) + .apply(GoogleAdsIO.v14().readAll().withDeveloperToken("abc"))); + } + + @Test + public void testReadAllExpandWithoutValidGoogleAdsClientFactory() throws Exception { + Assert.assertThrows( + "Non-null googleAdsClientFactory required but not provided", + IllegalArgumentException.class, + () -> + pipeline + .apply(Create.empty(new TypeDescriptor() {})) + .apply( + GoogleAdsIO.v14() + .readAll() + .withRateLimitPolicy(TEST_POLICY_FACTORY) + .withGoogleAdsClientFactory(null))); + } + + @Test + public void testReadAllExpandWithoutValidRateLimitPolicy() throws Exception { + Assert.assertThrows( + "Non-null rateLimitPolicy required but not provided", + IllegalArgumentException.class, + () -> + pipeline + .apply(Create.empty(new TypeDescriptor() {})) + .apply(GoogleAdsIO.v14().readAll().withRateLimitPolicy(null))); + } + + @Test + public void testReadExpandWithDeveloperTokenFromBuilder() { + pipeline + .apply(Create.empty(TypeDescriptors.strings())) + .apply( + GoogleAdsIO.v14() + .read() + .withRateLimitPolicy(TEST_POLICY_FACTORY) + .withDeveloperToken("abc") + .withQuery("GAQL")); + pipeline.getOptions().as(GoogleAdsOptions.class).setGoogleAdsDeveloperToken("abc"); + pipeline + .apply(Create.empty(TypeDescriptors.strings())) + .apply( + GoogleAdsIO.v14().read().withRateLimitPolicy(TEST_POLICY_FACTORY).withQuery("GAQL")); + } + + @Test + public void testReadExpandWithDeveloperTokenFromOptions() { + pipeline.getOptions().as(GoogleAdsOptions.class).setGoogleAdsDeveloperToken("abc"); + pipeline + .apply(Create.empty(TypeDescriptors.strings())) + .apply( + GoogleAdsIO.v14().read().withRateLimitPolicy(TEST_POLICY_FACTORY).withQuery("GAQL")); + } + + @Test + public void testReadExpandWithDeveloperTokenFromOptionsAndBuilder() { + pipeline.getOptions().as(GoogleAdsOptions.class).setGoogleAdsDeveloperToken("abc"); + pipeline + .apply(Create.empty(TypeDescriptors.strings())) + .apply( + GoogleAdsIO.v14() + .read() + .withRateLimitPolicy(TEST_POLICY_FACTORY) + .withDeveloperToken(null) + .withQuery("GAQL")); + } + + @Test + public void testReadExpandWithoutDeveloperToken() throws Exception { + Assert.assertThrows( + "Developer token required but not provided", + IllegalArgumentException.class, + () -> + pipeline + .apply(Create.empty(TypeDescriptors.strings())) + .apply( + GoogleAdsIO.v14() + .read() + .withRateLimitPolicy(TEST_POLICY_FACTORY) + .withQuery("GAQL"))); + } + + @Test + public void testReadExpandWithoutQuery() throws Exception { + Assert.assertThrows( + "Query required but not provided", + IllegalArgumentException.class, + () -> + pipeline + .apply(Create.empty(TypeDescriptors.strings())) + .apply(GoogleAdsIO.v14().read().withRateLimitPolicy(TEST_POLICY_FACTORY))); + } + + @Test + public void testReadExpandWithoutRateLimitPolicy() throws Exception { + Assert.assertThrows( + "Rate limit policy required but not provided", + IllegalArgumentException.class, + () -> + pipeline + .apply(Create.empty(TypeDescriptors.strings())) + .apply(GoogleAdsIO.v14().read().withDeveloperToken("abc").withQuery("GAQL"))); + } + + @Test + public void testReadExpandWithoutValidGoogleAdsClientFactory() throws Exception { + Assert.assertThrows( + "Non-null googleAdsClientFactory required but not provided", + IllegalArgumentException.class, + () -> + pipeline + .apply(Create.empty(TypeDescriptors.strings())) + .apply( + GoogleAdsIO.v14() + .read() + .withRateLimitPolicy(TEST_POLICY_FACTORY) + .withQuery("GAQL") + .withGoogleAdsClientFactory(null))); + } + + @Test + public void testReadExpandWithoutValidQuery() throws Exception { + Assert.assertThrows( + "Non-null query required but not provided", + IllegalArgumentException.class, + () -> + pipeline + .apply(Create.empty(TypeDescriptors.strings())) + .apply( + GoogleAdsIO.v14() + .read() + .withRateLimitPolicy(TEST_POLICY_FACTORY) + .withQuery(null))); + + Assert.assertThrows( + "Non-empty query required but not provided", + IllegalArgumentException.class, + () -> + pipeline + .apply(Create.empty(TypeDescriptors.strings())) + .apply( + GoogleAdsIO.v14() + .read() + .withRateLimitPolicy(TEST_POLICY_FACTORY) + .withQuery(""))); + } + + @Test + public void testReadExpandWithoutValidRateLimitPolicy() throws Exception { + Assert.assertThrows( + "Non-null rateLimitPolicy required but not provided", + IllegalArgumentException.class, + () -> + pipeline + .apply(Create.empty(TypeDescriptors.strings())) + .apply(GoogleAdsIO.v14().read().withQuery("GAQL").withRateLimitPolicy(null))); + } + } + + @RunWith(MockitoJUnitRunner.class) + public static class ExecutionTests { + @Rule public final transient TestPipeline pipeline = TestPipeline.create(); + + @Before + public void init() { + GoogleAdsOptions options = pipeline.getOptions().as(GoogleAdsOptions.class); + options.setGoogleAdsCredentialFactoryClass(NoopCredentialFactory.class); + synchronized (GoogleAdsV14.ReadAll.ReadAllFn.class) { + GoogleAdsV14.ReadAll.ReadAllFn.sleeper = (long millis) -> {}; + } + } + + @Test + @Category(NeedsRunner.class) + public void testRead() { + when(MockGoogleAdsClientFactory.GOOGLE_ADS_SERVICE_STUB_V14 + .searchStreamCallable() + .call(any(SearchGoogleAdsStreamRequest.class)) + .iterator()) + .thenReturn( + ImmutableList.of( + SearchGoogleAdsStreamResponse.newBuilder() + .addResults(GoogleAdsRow.newBuilder()) + .build()) + .iterator()); + + PCollection rows = + pipeline + .apply(Create.of("123")) + .apply( + GoogleAdsIO.v14() + .read() + .withGoogleAdsClientFactory(new MockGoogleAdsClientFactory()) + .withRateLimitPolicy(TEST_POLICY_FACTORY) + .withDeveloperToken("abc") + .withQuery("GAQL")); + PAssert.thatSingleton(rows).isEqualTo(GoogleAdsRow.getDefaultInstance()); + + pipeline.run(); + } + + @Test + @Category(NeedsRunner.class) + public void testReadWithFailureFromMaxRetriesExceeded() throws Exception { + when(MockGoogleAdsClientFactory.GOOGLE_ADS_SERVICE_STUB_V14 + .searchStreamCallable() + .call(any(SearchGoogleAdsStreamRequest.class))) + .thenThrow( + new GoogleAdsException( + new ApiException(null, GrpcStatusCode.of(Code.UNKNOWN), false), + GoogleAdsFailure.newBuilder() + .addErrors( + GoogleAdsError.newBuilder() + .setErrorCode( + ErrorCode.newBuilder() + .setInternalError(InternalError.TRANSIENT_ERROR))) + .build(), + new Metadata())); + + pipeline + .apply(Create.of("123")) + .apply( + GoogleAdsIO.v14() + .read() + .withGoogleAdsClientFactory(new MockGoogleAdsClientFactory()) + .withRateLimitPolicy(TEST_POLICY_FACTORY) + .withDeveloperToken("abc") + .withQuery("GAQL")); + + PipelineExecutionException exception = + Assert.assertThrows( + "Last retryable error after max retries", + Pipeline.PipelineExecutionException.class, + pipeline::run); + Assert.assertEquals(IOException.class, exception.getCause().getClass()); + Assert.assertEquals( + "Unable to get Google Ads response after retrying 5 times using query (GAQL)", + exception.getCause().getMessage()); + } + + @Test + @Category(NeedsRunner.class) + public void testReadWithFailureFromNonRetryableError() throws Exception { + when(MockGoogleAdsClientFactory.GOOGLE_ADS_SERVICE_STUB_V14 + .searchStreamCallable() + .call(any(SearchGoogleAdsStreamRequest.class))) + .thenThrow( + new GoogleAdsException( + new ApiException(null, GrpcStatusCode.of(Code.UNKNOWN), false), + GoogleAdsFailure.newBuilder() + .addErrors( + GoogleAdsError.newBuilder() + .setErrorCode( + ErrorCode.newBuilder() + .setAuthenticationError( + AuthenticationError.OAUTH_TOKEN_REVOKED))) + .build(), + new Metadata())); + + pipeline + .apply(Create.of("123")) + .apply( + GoogleAdsIO.v14() + .read() + .withGoogleAdsClientFactory(new MockGoogleAdsClientFactory()) + .withRateLimitPolicy(TEST_POLICY_FACTORY) + .withDeveloperToken("abc") + .withQuery("GAQL")); + + PipelineExecutionException exception = + Assert.assertThrows( + "First non-retryable error", + Pipeline.PipelineExecutionException.class, + pipeline::run); + Assert.assertEquals(IOException.class, exception.getCause().getClass()); + Assert.assertEquals( + "com.google.ads.googleads.v14.errors.GoogleAdsException: errors {\n" + + " error_code {\n" + + " authentication_error: OAUTH_TOKEN_REVOKED\n" + + " }\n" + + "}\n", + exception.getCause().getMessage()); + } + + @Test + @Category(NeedsRunner.class) + public void testReadWithRecoveryFromInternalError() throws Exception { + when(MockGoogleAdsClientFactory.GOOGLE_ADS_SERVICE_STUB_V14 + .searchStreamCallable() + .call(any(SearchGoogleAdsStreamRequest.class)) + .iterator()) + .thenThrow( + new GoogleAdsException( + new ApiException(null, GrpcStatusCode.of(Code.UNKNOWN), false), + GoogleAdsFailure.newBuilder() + .addErrors( + GoogleAdsError.newBuilder() + .setErrorCode( + ErrorCode.newBuilder() + .setInternalError(InternalError.INTERNAL_ERROR))) + .build(), + new Metadata())) + .thenThrow( + new GoogleAdsException( + new ApiException(null, GrpcStatusCode.of(Code.UNKNOWN), false), + GoogleAdsFailure.newBuilder() + .addErrors( + GoogleAdsError.newBuilder() + .setErrorCode( + ErrorCode.newBuilder() + .setInternalError(InternalError.TRANSIENT_ERROR))) + .build(), + new Metadata())) + .thenReturn( + ImmutableList.of( + SearchGoogleAdsStreamResponse.newBuilder() + .addResults(GoogleAdsRow.newBuilder()) + .build()) + .iterator()); + + PCollection rows = + pipeline + .apply(Create.of("123")) + .apply( + GoogleAdsIO.v14() + .read() + .withGoogleAdsClientFactory(new MockGoogleAdsClientFactory()) + .withRateLimitPolicy(TEST_POLICY_FACTORY) + .withDeveloperToken("abc") + .withQuery("GAQL")); + PAssert.thatSingleton(rows).isEqualTo(GoogleAdsRow.getDefaultInstance()); + + pipeline.run(); + } + + @Test + @Category(NeedsRunner.class) + public void testReadWithRecoveryFromQuotaErrorWithRetryDelay() throws Exception { + when(MockGoogleAdsClientFactory.GOOGLE_ADS_SERVICE_STUB_V14 + .searchStreamCallable() + .call(any(SearchGoogleAdsStreamRequest.class)) + .iterator()) + .thenThrow( + new GoogleAdsException( + new ApiException(null, GrpcStatusCode.of(Code.UNKNOWN), false), + GoogleAdsFailure.newBuilder() + .addErrors( + GoogleAdsError.newBuilder() + .setErrorCode( + ErrorCode.newBuilder() + .setQuotaError(QuotaError.RESOURCE_EXHAUSTED)) + .setDetails( + ErrorDetails.newBuilder() + .setQuotaErrorDetails( + QuotaErrorDetails.newBuilder() + .setRetryDelay(Duration.newBuilder().setSeconds(0))))) + .build(), + new Metadata())) + .thenThrow( + new GoogleAdsException( + new ApiException(null, GrpcStatusCode.of(Code.UNKNOWN), false), + GoogleAdsFailure.newBuilder() + .addErrors( + GoogleAdsError.newBuilder() + .setErrorCode( + ErrorCode.newBuilder() + .setQuotaError(QuotaError.RESOURCE_EXHAUSTED)) + .setDetails( + ErrorDetails.newBuilder() + .setQuotaErrorDetails( + QuotaErrorDetails.newBuilder() + .setRetryDelay( + Duration.newBuilder().setSeconds(42))))) + .build(), + new Metadata())) + .thenReturn( + ImmutableList.of( + SearchGoogleAdsStreamResponse.newBuilder() + .addResults(GoogleAdsRow.newBuilder()) + .build()) + .iterator()); + + PCollection rows = + pipeline + .apply(Create.of("123")) + .apply( + GoogleAdsIO.v14() + .read() + .withGoogleAdsClientFactory(new MockGoogleAdsClientFactory()) + .withRateLimitPolicy(TEST_POLICY_FACTORY) + .withDeveloperToken("abc") + .withQuery("GAQL")); + PAssert.thatSingleton(rows).isEqualTo(GoogleAdsRow.getDefaultInstance()); + + pipeline.run(); + } + } +} diff --git a/sdks/java/io/google-ads/src/test/java/org/apache/beam/sdk/io/googleads/MockGoogleAdsClientFactory.java b/sdks/java/io/google-ads/src/test/java/org/apache/beam/sdk/io/googleads/MockGoogleAdsClientFactory.java new file mode 100644 index 0000000000000..258b47763a4b4 --- /dev/null +++ b/sdks/java/io/google-ads/src/test/java/org/apache/beam/sdk/io/googleads/MockGoogleAdsClientFactory.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.googleads; + +import static org.mockito.Answers.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + +import com.google.ads.googleads.lib.GoogleAdsClient; +import com.google.ads.googleads.v14.services.GoogleAdsServiceClient; +import com.google.ads.googleads.v14.services.stub.GoogleAdsServiceStub; +import org.checkerframework.checker.nullness.qual.Nullable; + +class MockGoogleAdsClientFactory implements GoogleAdsClientFactory { + static final GoogleAdsServiceStub GOOGLE_ADS_SERVICE_STUB_V14 = + mock(GoogleAdsServiceStub.class, withSettings().defaultAnswer(RETURNS_DEEP_STUBS)); + + @Override + public GoogleAdsClient newGoogleAdsClient( + GoogleAdsOptions options, + @Nullable String developerToken, + @Nullable Long linkedCustomerId, + @Nullable Long loginCustomerId) { + GoogleAdsClient mockGoogleAdsClient = + mock(GoogleAdsClient.class, withSettings().defaultAnswer(RETURNS_DEEP_STUBS)); + when(mockGoogleAdsClient.getVersion14().createGoogleAdsServiceClient()) + .thenReturn(GoogleAdsServiceClient.create(GOOGLE_ADS_SERVICE_STUB_V14)); + return mockGoogleAdsClient; + } +} diff --git a/sdks/java/io/google-cloud-platform/OWNERS b/sdks/java/io/google-cloud-platform/OWNERS deleted file mode 100644 index cfe68754c999f..0000000000000 --- a/sdks/java/io/google-cloud-platform/OWNERS +++ /dev/null @@ -1,36 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers (BigTableIO) : - - igorbernstein2 - -reviewers (SpannerIO) : - - darshan-sj - - nielm - -reviewers (SpannerIO - readChangeStream) : - - cl2012 - - nancyxu123 - - haikuo-google - -reviewers (BigQueryIO - Storage Read API) : - - kmjung - - vachan-shetty - -reviewers (FhirIO) : - - msbukal - -reviewers (DatastoreIO) : - - pcostell - -reviewers (FirestoreIO) : - - pcostell - -reviewers (PubsubLiteIO) : - - dpcollins-google - -reviewers (all GCP connectors): - - lukecwik - - chamikaramj - - pabloem - - ihji - - johnjcasey diff --git a/sdks/java/io/google-cloud-platform/build.gradle b/sdks/java/io/google-cloud-platform/build.gradle index 2d08f192beebf..c4a5086801868 100644 --- a/sdks/java/io/google-cloud-platform/build.gradle +++ b/sdks/java/io/google-cloud-platform/build.gradle @@ -143,7 +143,7 @@ dependencies { implementation library.java.protobuf_java_util implementation library.java.slf4j_api implementation library.java.vendored_grpc_1_54_0 - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.arrow_memory_core implementation library.java.arrow_vector @@ -186,11 +186,13 @@ task integrationTest(type: Test, dependsOn: processTestResources) { def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing' def gcpTempRoot = project.findProperty('gcpTempRoot') ?: 'gs://temp-storage-for-end-to-end-tests' def firestoreDb = project.findProperty('firestoreDb') ?: 'firestoredb' + def host = project.findProperty('host') ?: 'batch-firestore.googleapis.com:443' systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ "--runner=DirectRunner", "--project=${gcpProject}", "--tempRoot=${gcpTempRoot}", "--firestoreDb=${firestoreDb}", + "--host=${host}", ]) // Disable Gradle cache: these ITs interact with live service that should always be considered "out of date" @@ -200,14 +202,8 @@ task integrationTest(type: Test, dependsOn: processTestResources) { exclude '**/BigQueryIOReadIT.class' exclude '**/BigQueryIOStorageQueryIT.class' exclude '**/BigQueryIOStorageReadIT.class' - exclude '**/BigQueryIOStorageReadTableRowIT.class' exclude '**/BigQueryIOStorageWriteIT.class' exclude '**/BigQueryToTableIT.class' - exclude '**/BigQueryIOJsonTest.class' - - // Failing due to Firestore service-side changes - // https://github.com/apache/beam/issues/25851 - exclude '**/firestore/it/**/*.class' maxParallelForks 4 classpath = sourceSets.test.runtimeClasspath @@ -224,12 +220,14 @@ task integrationTestKms(type: Test) { def gcpTempRoot = project.findProperty('gcpTempRootKms') ?: 'gs://temp-storage-for-end-to-end-tests-cmek' def dataflowKmsKey = project.findProperty('dataflowKmsKey') ?: "projects/apache-beam-testing/locations/global/keyRings/beam-it/cryptoKeys/test" def firestoreDb = project.findProperty('firestoreDb') ?: 'firestoredb' + def host = project.findProperty('host') ?: 'batch-firestore.googleapis.com:443' systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ "--runner=DirectRunner", "--project=${gcpProject}", "--tempRoot=${gcpTempRoot}", "--dataflowKmsKey=${dataflowKmsKey}", "--firestoreDb=${firestoreDb}", + "--host=${host}", ]) // Disable Gradle cache: these ITs interact with live service that should always be considered "out of date" @@ -244,6 +242,48 @@ task integrationTestKms(type: Test) { } } +/* + Integration tests for BigQueryIO that run on BigQuery's early rollout region (us-east7) + with the intended purpose of catching breaking changes from new BigQuery releases. + If these tests fail here but not in `Java_GCP_IO_Direct`, there may be a new BigQuery change + that is breaking the connector. If this is the case, we should verify with the appropriate + BigQuery infrastructure API team. + + To test in a BigQuery location, we just need to create our datasets in that location. + */ +task bigQueryEarlyRolloutIntegrationTest(type: Test, dependsOn: processTestResources) { + group = "Verification" + def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing' + def gcpTempRoot = project.findProperty('gcpTempRoot') ?: 'gs://temp-storage-for-bigquery-day0-tests' + systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ + "--runner=DirectRunner", + "--project=${gcpProject}", + "--tempRoot=${gcpTempRoot}", + "--bigQueryLocation=us-east7", + ]) + + outputs.upToDateWhen { false } + + // export and direct read + include '**/BigQueryToTableIT.class' + include '**/BigQueryIOJsonIT.class' + include '**/BigQueryIOStorageReadTableRowIT.class' + // storage write api + include '**/StorageApiDirectWriteProtosIT.class' + include '**/StorageApiSinkFailedRowsIT.class' + include '**/StorageApiSinkRowUpdateIT.class' + include '**/StorageApiSinkSchemaUpdateIT.class' + include '**/TableRowToStorageApiProtoIT.class' + // file loads + include '**/BigQuerySchemaUpdateOptionsIT.class' + include '**/BigQueryTimePartitioningClusteringIT.class' + include '**/FileLoadsStreamingIT.class' + + maxParallelForks 4 + classpath = sourceSets.test.runtimeClasspath + testClassesDirs = sourceSets.test.output.classesDirs +} + // path(s) for Cloud Spanner related classes def spannerIncludes = [ '**/org/apache/beam/sdk/io/gcp/spanner/**', @@ -267,8 +307,8 @@ task spannerCodeCoverageReport(type: JacocoReport, dependsOn: test) { sourceDirectories.setFrom(files(project.sourceSets.main.allSource.srcDirs)) executionData.setFrom(file("${buildDir}/jacoco/test.exec")) reports { - html.enabled true - html.destination file("${buildDir}/reports/jacoco/spanner/") + html.getRequired().set(true) + html.getOutputLocation().set(file("${buildDir}/reports/jacoco/spanner/")) } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java index b8a7fc760bc7a..46c25d47e7a89 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java @@ -22,9 +22,9 @@ import com.google.auto.value.extension.memoized.Memoized; import com.google.cloud.bigquery.storage.v1.TableSchema; import com.google.protobuf.ByteString; +import com.google.protobuf.DescriptorProtos; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; -import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; import java.util.function.Consumer; import java.util.function.Supplier; @@ -49,7 +49,7 @@ abstract class AppendClientInfo { abstract @Nullable String getStreamName(); - abstract Descriptors.Descriptor getDescriptor(); + abstract DescriptorProtos.DescriptorProto getDescriptor(); @AutoValue.Builder abstract static class Builder { @@ -63,7 +63,7 @@ abstract static class Builder { abstract Builder setSchemaInformation(TableRowToStorageApiProto.SchemaInformation value); - abstract Builder setDescriptor(Descriptors.Descriptor value); + abstract Builder setDescriptor(DescriptorProtos.DescriptorProto value); abstract Builder setStreamName(@Nullable String name); @@ -74,8 +74,8 @@ abstract static class Builder { static AppendClientInfo of( TableSchema tableSchema, - Consumer closeAppendClient, - boolean includeCdcColumns) + DescriptorProtos.DescriptorProto descriptor, + Consumer closeAppendClient) throws Exception { return new AutoValue_AppendClientInfo.Builder() .setTableSchema(tableSchema) @@ -83,12 +83,22 @@ static AppendClientInfo of( .setJsonTableSchema(TableRowToStorageApiProto.protoSchemaToTableSchema(tableSchema)) .setSchemaInformation( TableRowToStorageApiProto.SchemaInformation.fromTableSchema(tableSchema)) - .setDescriptor( - TableRowToStorageApiProto.getDescriptorFromTableSchema( - tableSchema, true, includeCdcColumns)) + .setDescriptor(descriptor) .build(); } + static AppendClientInfo of( + TableSchema tableSchema, + Consumer closeAppendClient, + boolean includeCdcColumns) + throws Exception { + return of( + tableSchema, + TableRowToStorageApiProto.descriptorSchemaFromTableSchema( + tableSchema, true, includeCdcColumns), + closeAppendClient); + } + public AppendClientInfo withNoAppendClient() { return toBuilder().setStreamAppendClient(null).build(); } @@ -149,8 +159,10 @@ Descriptors.Descriptor getDescriptorIgnoreRequired() { public TableRow toTableRow(ByteString protoBytes) { try { return TableRowToStorageApiProto.tableRowFromMessage( - DynamicMessage.parseFrom(getDescriptor(), protoBytes), true); - } catch (InvalidProtocolBufferException e) { + DynamicMessage.parseFrom( + TableRowToStorageApiProto.wrapDescriptorProto(getDescriptor()), protoBytes), + true); + } catch (Exception e) { throw new RuntimeException(e); } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java index ca921d7bbeffa..fc153d6c97d50 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java @@ -40,10 +40,10 @@ import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils.TypeWithNullability; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Functions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Functions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Days; import org.joda.time.Instant; import org.joda.time.ReadableInstant; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java index bad44ee5b5c19..32ee29738bf85 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoads.java @@ -18,8 +18,8 @@ package org.apache.beam.sdk.io.gcp.bigquery; import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.resolveTempLocation; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.services.bigquery.model.TableReference; import com.google.api.services.bigquery.model.TableRow; @@ -34,7 +34,6 @@ import org.apache.beam.sdk.coders.KvCoder; import org.apache.beam.sdk.coders.NullableCoder; import org.apache.beam.sdk.coders.ShardedKeyCoder; -import org.apache.beam.sdk.coders.VoidCoder; import org.apache.beam.sdk.extensions.gcp.options.GcsOptions; import org.apache.beam.sdk.extensions.gcp.util.gcsfs.GcsPath; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition; @@ -75,10 +74,10 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.slf4j.Logger; @@ -399,10 +398,12 @@ private WriteResult expandTriggered(PCollection> inpu "Window Into Global Windows", Window.>into(new GlobalWindows()) .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1)))) - .apply("Add Void Key", WithKeys.of((Void) null)) - .setCoder(KvCoder.of(VoidCoder.of(), tempTables.getCoder())) - .apply("GroupByKey", GroupByKey.create()) - .apply("Extract Values", Values.create()) + // We use this and the following GBK to aggregate by final destination. + // This way, each destination has its own pane sequence + .apply("AddDestinationKeys", WithKeys.of(result -> result.getKey())) + .setCoder(KvCoder.of(destinationCoder, tempTables.getCoder())) + .apply("GroupTempTablesByFinalDestination", GroupByKey.create()) + .apply("ExtractTempTables", Values.create()) .apply( ParDo.of( new UpdateSchemaDestination( diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchedStreamingWrite.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchedStreamingWrite.java index 133fa5b4ffa19..2569f71bf0363 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchedStreamingWrite.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchedStreamingWrite.java @@ -56,9 +56,9 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.ValueInSingleWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProto.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProto.java index e6be2ddf0862e..d91ddd6843c0f 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProto.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProto.java @@ -45,11 +45,11 @@ import org.apache.beam.sdk.schemas.logicaltypes.EnumerationType; import org.apache.beam.sdk.schemas.logicaltypes.SqlTypes; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Functions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Bytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Functions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Bytes; import org.joda.time.ReadableInstant; /** diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtils.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtils.java index ed2bd168ed143..bdee2eef570d9 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtils.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtils.java @@ -21,10 +21,10 @@ import static java.time.temporal.ChronoField.MINUTE_OF_HOUR; import static java.time.temporal.ChronoField.NANO_OF_SECOND; import static java.time.temporal.ChronoField.SECOND_OF_MINUTE; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Verify.verify; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Verify.verifyNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Verify.verify; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Verify.verifyNotNull; import com.google.api.services.bigquery.model.TableFieldSchema; import com.google.api.services.bigquery.model.TableRow; @@ -48,10 +48,10 @@ import org.apache.avro.Schema.Field; import org.apache.avro.Schema.Type; import org.apache.avro.generic.GenericRecord; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableCollection; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryCoderProviderRegistrar.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryCoderProviderRegistrar.java index 183a5058e65d9..2636630d22ec0 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryCoderProviderRegistrar.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryCoderProviderRegistrar.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.coders.CoderProviderRegistrar; import org.apache.beam.sdk.coders.CoderProviders; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** A {@link CoderProviderRegistrar} for standard types used with {@link BigQueryIO}. */ @AutoService(CoderProviderRegistrar.class) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryExportReadSchemaTransformProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryExportReadSchemaTransformProvider.java index 347a0d9f73343..af6ab5c71c8b4 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryExportReadSchemaTransformProvider.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryExportReadSchemaTransformProvider.java @@ -27,13 +27,12 @@ import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; /** * An implementation of {@link TypedSchemaTransformProvider} for BigQuery read jobs configured using @@ -95,33 +94,13 @@ public List outputCollectionNames() { * An implementation of {@link SchemaTransform} for BigQuery read jobs configured using {@link * BigQueryExportReadSchemaTransformConfiguration}. */ - private static class BigQueryExportSchemaTransform implements SchemaTransform { - private final BigQueryExportReadSchemaTransformConfiguration configuration; - - BigQueryExportSchemaTransform(BigQueryExportReadSchemaTransformConfiguration configuration) { - this.configuration = configuration; - } - - /** Implements {@link SchemaTransform} buildTransform method. */ - @Override - public PTransform buildTransform() { - return new PCollectionRowTupleTransform(configuration); - } - } - - /** - * An implementation of {@link PTransform} for BigQuery read jobs configured using {@link - * BigQueryExportReadSchemaTransformConfiguration}. - */ - static class PCollectionRowTupleTransform - extends PTransform { - - private final BigQueryExportReadSchemaTransformConfiguration configuration; - + protected static class BigQueryExportSchemaTransform extends SchemaTransform { /** An instance of {@link BigQueryServices} used for testing. */ private BigQueryServices testBigQueryServices = null; - PCollectionRowTupleTransform(BigQueryExportReadSchemaTransformConfiguration configuration) { + private final BigQueryExportReadSchemaTransformConfiguration configuration; + + BigQueryExportSchemaTransform(BigQueryExportReadSchemaTransformConfiguration configuration) { this.configuration = configuration; } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryFileLoadsWriteSchemaTransformProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryFileLoadsWriteSchemaTransformProvider.java index db9f069bbb6ca..3212e2a30348b 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryFileLoadsWriteSchemaTransformProvider.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryFileLoadsWriteSchemaTransformProvider.java @@ -36,12 +36,11 @@ import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** * An implementation of {@link TypedSchemaTransformProvider} for BigQuery write jobs configured @@ -103,36 +102,13 @@ public List outputCollectionNames() { * A {@link SchemaTransform} that performs {@link BigQueryIO.Write}s based on a {@link * BigQueryFileLoadsWriteSchemaTransformConfiguration}. */ - private static class BigQueryWriteSchemaTransform implements SchemaTransform { - private final BigQueryFileLoadsWriteSchemaTransformConfiguration configuration; - - BigQueryWriteSchemaTransform(BigQueryFileLoadsWriteSchemaTransformConfiguration configuration) { - this.configuration = configuration; - } - - /** - * Overrides {@link SchemaTransform#buildTransform()} by returning a {@link - * PCollectionRowTupleTransform}. - */ - @Override - public PTransform buildTransform() { - return new PCollectionRowTupleTransform(configuration); - } - } - - /** - * An implementation of {@link PTransform} for BigQuery write jobs configured using {@link - * BigQueryFileLoadsWriteSchemaTransformConfiguration}. - */ - static class PCollectionRowTupleTransform - extends PTransform { - - private final BigQueryFileLoadsWriteSchemaTransformConfiguration configuration; - + protected static class BigQueryWriteSchemaTransform extends SchemaTransform { /** An instance of {@link BigQueryServices} used for testing. */ private BigQueryServices testBigQueryServices = null; - PCollectionRowTupleTransform(BigQueryFileLoadsWriteSchemaTransformConfiguration configuration) { + private final BigQueryFileLoadsWriteSchemaTransformConfiguration configuration; + + BigQueryWriteSchemaTransform(BigQueryFileLoadsWriteSchemaTransformConfiguration configuration) { this.configuration = configuration; } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java index 4256d70f50b2a..f4120a013ed65 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.client.util.BackOff; import com.google.api.client.util.BackOffUtils; @@ -47,8 +47,8 @@ import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.util.FluentBackoff; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.PolyNull; @@ -126,7 +126,7 @@ PendingJobManager addPendingJob( } void waitForDone() throws Exception { - LOG.info("Waiting for jobs to complete."); + LOG.debug("Waiting for BigQuery jobs to complete."); Sleeper sleeper = Sleeper.DEFAULT; while (!pendingJobs.isEmpty()) { List retryJobs = Lists.newArrayList(); diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java index 8ef39ca9cf94c..ce8ddb683d1ed 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java @@ -19,8 +19,8 @@ import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.resolveTempLocation; import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryResourceNaming.createTempTableReference; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.client.json.JsonFactory; import com.google.api.services.bigquery.model.Clustering; @@ -30,6 +30,7 @@ import com.google.api.services.bigquery.model.JobStatistics; import com.google.api.services.bigquery.model.Table; import com.google.api.services.bigquery.model.TableCell; +import com.google.api.services.bigquery.model.TableConstraints; import com.google.api.services.bigquery.model.TableFieldSchema; import com.google.api.services.bigquery.model.TableReference; import com.google.api.services.bigquery.model.TableRow; @@ -44,8 +45,12 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.Message; import java.io.IOException; import java.io.Serializable; +import java.lang.reflect.InvocationTargetException; import java.util.Collections; import java.util.List; import java.util.Map; @@ -126,17 +131,17 @@ import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; import org.apache.beam.sdk.values.ValueInSingleWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Suppliers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Function; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Suppliers; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.slf4j.Logger; @@ -264,7 +269,7 @@ * *
    {@code
      * PCollection weatherData = pipeline.apply(
    - *     BigQueryIO.readTableRows().from("clouddataflow-readonly:samples.weather_stations"));
    + *     BigQueryIO.readTableRows().from("apache-beam-testing.samples.weather_stations"));
      * }
    * * Example: Reading rows of a table and parsing them into a custom type. @@ -277,7 +282,7 @@ * return new WeatherRecord(...); * } * }) - * .from("clouddataflow-readonly:samples.weather_stations")) + * .from("apache-beam-testing.samples.weather_stations")) * .withCoder(SerializableCoder.of(WeatherRecord.class)); * } * @@ -506,7 +511,8 @@ * .apply(BigQueryIO.applyRowMutations() * .to(my_project:my_dataset.my_table) * .withSchema(schema) - * .withCreateDisposition(Write.CreateDisposition.CREATE_NEVER)); + * .withPrimaryKey(ImmutableList.of("field1", "field2")) + * .withCreateDisposition(Write.CreateDisposition.CREATE_IF_NEEDED)); * } * *

    If writing a type other than TableRow (e.g. using {@link BigQueryIO#writeGenericRecords} or @@ -519,12 +525,17 @@ * cdcEvent.apply(BigQueryIO.write() * .to("my-project:my_dataset.my_table") * .withSchema(schema) + * .withPrimaryKey(ImmutableList.of("field1", "field2")) * .withFormatFunction(CdcEvent::getTableRow) * .withRowMutationInformationFn(cdc -> RowMutationInformation.of(cdc.getChangeType(), * cdc.getSequenceNumber())) * .withMethod(Write.Method.STORAGE_API_AT_LEAST_ONCE) - * .withCreateDisposition(Write.CreateDisposition.CREATE_NEVER)); + * .withCreateDisposition(Write.CreateDisposition.CREATE_IF_NEEDED)); * } + * + *

    Note that in order to use inserts or deletes, the table must bet set up with a primary key. If + * the table is not previously created and CREATE_IF_NEEDED is used, a primary key must be + * specified. */ @SuppressWarnings({ "nullness" // TODO(https://github.com/apache/beam/issues/20506) @@ -2131,6 +2142,7 @@ public static Write write() { .setDeterministicRecordIdFn(null) .setMaxRetryJobs(1000) .setPropagateSuccessfulStorageApiWrites(false) + .setDirectWriteProtos(true) .build(); } @@ -2170,6 +2182,27 @@ public static Write writeGenericRecords() { .withAvroFormatFunction(GENERIC_RECORD_IDENTITY_FORMATTER); } + /** + * A {@link PTransform} that writes a {@link PCollection} containing protocol buffer objects to a + * BigQuery table. If using one of the storage-api write methods, these protocol buffers must + * match the schema of the table. + * + *

    If a Schema is provided using {@link Write#withSchema}, that schema will be used for + * creating the table if necessary. If no schema is provided, one will be inferred from the + * protocol buffer's descriptor. Note that inferring a schema from the protocol buffer may not + * always provide the intended schema as multiple BigQuery types can map to the same protocol + * buffer type. For example, a protocol buffer field of type INT64 may be an INT64 BigQuery type, + * but it might also represent a TIME, DATETIME, or a TIMESTAMP type. + */ + public static Write writeProtos(Class protoMessageClass) { + if (DynamicMessage.class.equals(protoMessageClass)) { + throw new IllegalArgumentException("DynamicMessage is not supported."); + } + return BigQueryIO.write() + .withFormatFunction(m -> TableRowToStorageApiProto.tableRowFromMessage(m, false)) + .withWriteProtosClass(protoMessageClass); + } + /** Implementation of {@link #write}. */ @AutoValue public abstract static class Write extends PTransform, WriteResult> { @@ -2292,6 +2325,8 @@ public enum Method { abstract @Nullable String getKmsKey(); + abstract @Nullable List getPrimaryKey(); + abstract Boolean getOptimizeWrites(); abstract Boolean getUseBeamSchema(); @@ -2302,6 +2337,10 @@ public enum Method { abstract Boolean getAutoSchemaUpdate(); + abstract @Nullable Class getWriteProtosClass(); + + abstract Boolean getDirectWriteProtos(); + abstract @Nullable SerializableFunction getDeterministicRecordIdFn(); abstract @Nullable String getWriteTempDataset(); @@ -2386,7 +2425,9 @@ abstract Builder setPropagateSuccessfulStorageApiWrites( abstract Builder setIgnoreInsertIds(Boolean ignoreInsertIds); - abstract Builder setKmsKey(String kmsKey); + abstract Builder setKmsKey(@Nullable String kmsKey); + + abstract Builder setPrimaryKey(@Nullable List primaryKey); abstract Builder setOptimizeWrites(Boolean optimizeWrites); @@ -2400,6 +2441,10 @@ abstract Builder setPropagateSuccessfulStorageApiWrites( abstract Builder setAutoSchemaUpdate(Boolean autoSchemaUpdate); + abstract Builder setWriteProtosClass(@Nullable Class clazz); + + abstract Builder setDirectWriteProtos(Boolean direct); + abstract Builder setDeterministicRecordIdFn( SerializableFunction toUniqueIdFunction); @@ -2454,6 +2499,8 @@ public enum WriteDisposition { *

    The replacement may occur in multiple steps - for instance by first removing the * existing table, then creating a replacement, then filling it in. This is not an atomic * operation, and external programs may see the table in any of these intermediate steps. + * + *

    Note: This write disposition is only supported for the FILE_LOADS write method. */ WRITE_TRUNCATE, @@ -2780,6 +2827,20 @@ public Write withRowMutationInformationFn( return toBuilder().setRowMutationInformationFn(updateFn).build(); } + Write withWriteProtosClass(Class clazz) { + return toBuilder().setWriteProtosClass(clazz).build(); + } + + /* + When using {@link Write.Method#STORAGE_API} or {@link Write.Method#STORAGE_API_AT_LEAST_ONCE} along with + {@link BigQueryIO.writeProtos}, the sink will try to write the protos directly to BigQuery without modification. + In some cases this is not supported or BigQuery cannot directly interpet the proto. In these cases, the direct + proto write + */ + public Write withDirectWriteProtos(boolean directWriteProtos) { + return toBuilder().setDirectWriteProtos(directWriteProtos).build(); + } + /** * Set the project the BigQuery load job will be initiated from. This is only applicable when * the write method is set to {@link Method#FILE_LOADS}. If omitted, the project of the @@ -2899,6 +2960,10 @@ public Write withKmsKey(String kmsKey) { return toBuilder().setKmsKey(kmsKey).build(); } + public Write withPrimaryKey(List primaryKey) { + return toBuilder().setPrimaryKey(primaryKey).build(); + } + /** * If true, enables new codepaths that are expected to use less resources while writing to * BigQuery. Not enabled by default in order to maintain backwards compatibility. @@ -2918,7 +2983,7 @@ public Write useBeamSchema() { /** * If true, enables using a dynamically determined number of shards to write to BigQuery. This - * can be used for both {@link Method#FILE_LOADS}, {@link Method#STREAMING_INSERTS} and {@link + * can be used for {@link Method#FILE_LOADS}, {@link Method#STREAMING_INSERTS} and {@link * Method#STORAGE_WRITE_API}. Only applicable to unbounded data. If using {@link * Method#FILE_LOADS}, numFileShards set via {@link #withNumFileShards} will be ignored. */ @@ -3128,39 +3193,93 @@ public WriteResult expand(PCollection input) { .collect(Collectors.toList())), "No more than one of jsonSchema, schemaFromView, or dynamicDestinations may be set"); + // Perform some argument checks + BigQueryOptions bqOptions = input.getPipeline().getOptions().as(BigQueryOptions.class); Write.Method method = resolveMethod(input); - if (input.isBounded() == IsBounded.UNBOUNDED - && (method == Write.Method.FILE_LOADS || method == Write.Method.STORAGE_WRITE_API)) { - BigQueryOptions bqOptions = input.getPipeline().getOptions().as(BigQueryOptions.class); - Duration triggeringFrequency = - (method == Write.Method.STORAGE_WRITE_API) - ? getStorageApiTriggeringFrequency(bqOptions) - : getTriggeringFrequency(); - checkArgument( - triggeringFrequency != null, - "When writing an unbounded PCollection via FILE_LOADS or STORAGE_API_WRITES, " - + "triggering frequency must be specified"); - } else { - checkArgument( - getTriggeringFrequency() == null && getNumFileShards() == 0, - "Triggering frequency or number of file shards can be specified only when writing an" - + " unbounded PCollection via FILE_LOADS or STORAGE_API_WRITES, but: the collection" - + " was %s and the method was %s", - input.isBounded(), - method); + if (input.isBounded() == IsBounded.UNBOUNDED) { + if (method == Write.Method.FILE_LOADS || method == Write.Method.STORAGE_WRITE_API) { + Duration triggeringFrequency = + (method == Write.Method.STORAGE_WRITE_API) + ? getStorageApiTriggeringFrequency(bqOptions) + : getTriggeringFrequency(); + checkArgument( + triggeringFrequency != null, + "When writing an unbounded PCollection via FILE_LOADS or STORAGE_WRITE_API, " + + "triggering frequency must be specified"); + } else { + checkArgument( + getTriggeringFrequency() == null, + "Triggering frequency can be specified only when writing via FILE_LOADS or STORAGE_WRITE_API, but the method was %s.", + method); + } + if (method != Method.FILE_LOADS) { + checkArgument( + getNumFileShards() == 0, + "Number of file shards can be specified only when writing via FILE_LOADS, but the method was %s.", + method); + } + if (method == Method.STORAGE_API_AT_LEAST_ONCE + && getStorageApiTriggeringFrequency(bqOptions) != null) { + LOG.warn( + "Storage API triggering frequency option will be ignored is it can only be specified only " + + "when writing via STORAGE_WRITE_API, but the method was {}.", + method); + } + if (getAutoSharding()) { + if (method == Method.STORAGE_WRITE_API && getStorageApiNumStreams(bqOptions) > 0) { + LOG.warn( + "Both numStorageWriteApiStreams and auto-sharding options are set. Will default to auto-sharding." + + " To set a fixed number of streams, do not enable auto-sharding."); + } else if (method == Method.FILE_LOADS && getNumFileShards() > 0) { + LOG.warn( + "Both numFileShards and auto-sharding options are set. Will default to auto-sharding." + + " To set a fixed number of file shards, do not enable auto-sharding."); + } else if (method == Method.STORAGE_API_AT_LEAST_ONCE) { + LOG.warn( + "The setting of auto-sharding is ignored. It is only supported when writing an" + + " unbounded PCollection via FILE_LOADS, STREAMING_INSERTS or" + + " STORAGE_WRITE_API, but the method was {}.", + method); + } + } + } else { // PCollection is bounded + String error = + String.format( + " is only applicable to an unbounded PCollection, but the input PCollection is %s.", + input.isBounded()); + checkArgument(getTriggeringFrequency() == null, "Triggering frequency" + error); + checkArgument(!getAutoSharding(), "Auto-sharding" + error); + checkArgument(getNumFileShards() == 0, "Number of file shards" + error); + + if (getStorageApiTriggeringFrequency(bqOptions) != null) { + LOG.warn("Setting a triggering frequency" + error); + } + if (getStorageApiNumStreams(bqOptions) != 0) { + LOG.warn("Setting the number of Storage API streams" + error); + } + } + + if (method == Method.STORAGE_API_AT_LEAST_ONCE && getStorageApiNumStreams(bqOptions) != 0) { + LOG.warn( + "Setting a number of Storage API streams is only supported when using STORAGE_WRITE_API"); } if (method != Method.STORAGE_WRITE_API && method != Method.STORAGE_API_AT_LEAST_ONCE) { checkArgument( !getAutoSchemaUpdate(), - "withAutoSchemaUpdate only supported when using storage-api writes."); + "withAutoSchemaUpdate only supported when using STORAGE_WRITE_API or STORAGE_API_AT_LEAST_ONCE."); + } else if (getWriteDisposition() == WriteDisposition.WRITE_TRUNCATE) { + LOG.error("The Storage API sink does not support the WRITE_TRUNCATE write disposition."); } if (getRowMutationInformationFn() != null) { checkArgument(getMethod() == Method.STORAGE_API_AT_LEAST_ONCE); checkArgument( - getCreateDisposition() == CreateDisposition.CREATE_NEVER, - "CREATE_IF_NEEDED is not supported when applying row updates. Tables must be precreated " - + "with a primary key specified."); + getCreateDisposition() == CreateDisposition.CREATE_NEVER || getPrimaryKey() != null, + "If specifying CREATE_IF_NEEDED along with row updates, a primary key needs to be specified"); + } + if (getPrimaryKey() != null) { + checkArgument( + getMethod() != Method.FILE_LOADS, "Primary key not supported when using FILE_LOADS"); } if (getAutoSchemaUpdate()) { @@ -3172,10 +3291,6 @@ public WriteResult expand(PCollection input) { !getUseBeamSchema(), "Auto schema update not supported when using Beam schemas."); } - if (input.isBounded() == IsBounded.BOUNDED) { - checkArgument(!getAutoSharding(), "Auto-sharding is only applicable to unbounded input."); - } - if (getJsonTimePartitioning() != null) { checkArgument( getDynamicDestinations() == null, @@ -3219,6 +3334,14 @@ public WriteResult expand(PCollection input) { getJsonTimePartitioning(), StaticValueProvider.of(BigQueryHelpers.toJsonString(getClustering()))); } + if (getPrimaryKey() != null) { + dynamicDestinations = + new DynamicDestinationsHelpers.ConstantTableConstraintsDestinations<>( + (DynamicDestinations) dynamicDestinations, + new TableConstraints() + .setPrimaryKey( + new TableConstraints.PrimaryKey().setColumns(getPrimaryKey()))); + } } return expandTyped(input, dynamicDestinations); } @@ -3237,6 +3360,7 @@ private WriteResult expandTyped( || getDynamicDestinations() != null || getSchemaFromView() != null; + Class writeProtoClass = getWriteProtosClass(); if (getUseBeamSchema()) { checkArgument(input.hasSchema(), "The input doesn't has a schema"); optimizeWrites = true; @@ -3252,11 +3376,34 @@ private WriteResult expandTyped( formatFunction = BigQueryUtils.toTableRow(input.getToRowFunction()); } // Infer the TableSchema from the input Beam schema. + // TODO: If the user provided a schema, we should use that. There are things that can be + // specified in a + // BQ schema that don't have exact matches in a Beam schema (e.g. GEOGRAPHY types). TableSchema tableSchema = BigQueryUtils.toTableSchema(input.getSchema()); dynamicDestinations = new ConstantSchemaDestinations<>( dynamicDestinations, StaticValueProvider.of(BigQueryHelpers.toJsonString(tableSchema))); + } else if (writeProtoClass != null) { + if (!hasSchema) { + try { + @SuppressWarnings({"unchecked", "nullness"}) + Descriptors.Descriptor descriptor = + (Descriptors.Descriptor) + org.apache.beam.sdk.util.Preconditions.checkStateNotNull( + writeProtoClass.getMethod("getDescriptor")) + .invoke(null); + TableSchema tableSchema = + TableRowToStorageApiProto.protoSchemaToTableSchema( + TableRowToStorageApiProto.tableSchemaFromDescriptor(descriptor)); + dynamicDestinations = + new ConstantSchemaDestinations<>( + dynamicDestinations, + StaticValueProvider.of(BigQueryHelpers.toJsonString(tableSchema))); + } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + throw new IllegalArgumentException(e); + } + } } else { // Require a schema if creating one or more tables. checkArgument( @@ -3335,6 +3482,7 @@ private WriteResult expandTyped( method); } + @SuppressWarnings("rawtypes") private WriteResult continueExpandTyped( PCollection> input, Coder elementCoder, @@ -3457,6 +3605,28 @@ private WriteResult continueExpandTyped( elementSchema, elementToRowFunction, getRowMutationInformationFn() != null); + } else if (getWriteProtosClass() != null && getDirectWriteProtos()) { + // We could support both of these by falling back to + // StorageApiDynamicDestinationsTableRow. This + // would defeat the optimization (we would be forced to create a new dynamic proto message + // and copy the data over). For now, we simply give the user a way to disable the + // optimization themselves. + checkArgument( + getRowMutationInformationFn() == null, + "Row upserts and deletes are not for direct proto writes. " + + "Try setting withDirectWriteProtos(false)"); + checkArgument( + !getAutoSchemaUpdate(), + "withAutoSchemaUpdate not supported when using writeProtos." + + " Try setting withDirectWriteProtos(false)"); + checkArgument( + !getIgnoreUnknownValues(), + "ignoreUnknownValues not supported when using writeProtos." + + " Try setting withDirectWriteProtos(false)"); + storageApiDynamicDestinations = + (StorageApiDynamicDestinations) + new StorageApiDynamicDestinationsProto( + dynamicDestinations, getWriteProtosClass()); } else if (getAvroRowWriterFactory() != null) { // we can configure the avro to storage write api proto converter for this // assuming the format function returns an Avro GenericRecord @@ -3493,6 +3663,7 @@ private WriteResult continueExpandTyped( getIgnoreUnknownValues(), getAutoSchemaUpdate()); } + int numShards = getStorageApiNumStreams(bqOptions); boolean enableAutoSharding = getAutoSharding(); if (numShards == 0) { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOMetadata.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOMetadata.java index ee64a7ab9ddba..1893418dedb34 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOMetadata.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOMetadata.java @@ -28,8 +28,15 @@ final class BigQueryIOMetadata { private @Nullable String beamJobId; - private BigQueryIOMetadata(@Nullable String beamJobId) { + private @Nullable String beamJobName; + + private @Nullable String beamWorkerId; + + private BigQueryIOMetadata( + @Nullable String beamJobId, @Nullable String beamJobName, @Nullable String beamWorkerId) { this.beamJobId = beamJobId; + this.beamJobName = beamJobName; + this.beamWorkerId = beamWorkerId; } private static final Pattern VALID_CLOUD_LABEL_PATTERN = @@ -41,17 +48,24 @@ private BigQueryIOMetadata(@Nullable String beamJobId) { */ public static BigQueryIOMetadata create() { String dataflowJobId = GceMetadataUtil.fetchDataflowJobId(); + String dataflowJobName = GceMetadataUtil.fetchDataflowJobName(); + String dataflowWorkerId = GceMetadataUtil.fetchDataflowWorkerId(); + // If a Dataflow job id is returned on GCE metadata. Then it means // this program is running on a Dataflow GCE VM. - boolean isDataflowRunner = dataflowJobId != null && !dataflowJobId.isEmpty(); + boolean isDataflowRunner = !dataflowJobId.isEmpty(); String beamJobId = null; + String beamJobName = null; + String beamWorkerId = null; if (isDataflowRunner) { if (BigQueryIOMetadata.isValidCloudLabel(dataflowJobId)) { beamJobId = dataflowJobId; + beamJobName = dataflowJobName; + beamWorkerId = dataflowWorkerId; } } - return new BigQueryIOMetadata(beamJobId); + return new BigQueryIOMetadata(beamJobId, beamJobName, beamWorkerId); } public Map addAdditionalJobLabels(Map jobLabels) { @@ -68,6 +82,20 @@ public Map addAdditionalJobLabels(Map jobLabels) return this.beamJobId; } + /* + * Returns the beam job name. Can be null if it is not running on Dataflow. + */ + public @Nullable String getBeamJobName() { + return this.beamJobName; + } + + /* + * Returns the beam worker id. Can be null if it is not running on Dataflow. + */ + public @Nullable String getBeamWorkerId() { + return this.beamWorkerId; + } + /** * Returns true if label_value is a valid cloud label string. This function can return false in * cases where the label value is valid. However, it will not return true in a case where the diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQueryHelper.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQueryHelper.java index 5b467cfc0027a..be49777417eaf 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQueryHelper.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQueryHelper.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.gcp.bigquery; import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryResourceNaming.createTempTableReference; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.services.bigquery.model.EncryptionConfiguration; import com.google.api.services.bigquery.model.Job; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java index 8a35f56941fd3..fc882b1c2a4f1 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.display.DisplayData; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** A {@link BigQuerySourceBase} for querying BigQuery tables. */ @VisibleForTesting diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySourceDef.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySourceDef.java index 9c8bffeaf9da1..585b58aa3669f 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySourceDef.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySourceDef.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.gcp.bigquery; import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryResourceNaming.createTempTableReference; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.bigquery.model.JobStatistics; import com.google.api.services.bigquery.model.TableReference; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryResourceNaming.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryResourceNaming.java index e97fc8c8aa09c..df5e641847b69 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryResourceNaming.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryResourceNaming.java @@ -21,7 +21,7 @@ import com.google.api.services.bigquery.model.TableReference; import java.util.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryRowWriter.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryRowWriter.java index f96f05d62f753..a442144e16103 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryRowWriter.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryRowWriter.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.services.bigquery.model.TableRow; import java.io.IOException; @@ -27,7 +27,7 @@ import java.util.UUID; import org.apache.beam.sdk.io.FileSystems; import org.apache.beam.sdk.io.fs.ResourceId; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.CountingOutputStream; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.CountingOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java index 6f178bd615040..1cc9049a542d9 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java @@ -43,7 +43,7 @@ import com.google.cloud.bigquery.storage.v1.SplitReadStreamResponse; import com.google.cloud.bigquery.storage.v1.TableSchema; import com.google.cloud.bigquery.storage.v1.WriteStream; -import com.google.protobuf.Descriptors.Descriptor; +import com.google.protobuf.DescriptorProtos; import java.io.IOException; import java.io.Serializable; import java.util.List; @@ -213,7 +213,8 @@ WriteStream createWriteStream(String tableUrn, WriteStream.Type type) * first. */ StreamAppendClient getStreamAppendClient( - String streamName, Descriptor descriptor, boolean useConnectionPool) throws Exception; + String streamName, DescriptorProtos.DescriptorProto descriptor, boolean useConnectionPool) + throws Exception; /** Flush a given stream up to the given offset. The stream must have type BUFFERED. */ ApiFuture flush(String streamName, long flushOffset) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java index faaa1b9aa8e6f..1b6cc555511dd 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; @@ -86,7 +86,7 @@ import com.google.cloud.bigquery.storage.v1.WriteStream; import com.google.cloud.hadoop.util.ApiErrorExtractor; import com.google.cloud.hadoop.util.ChainingHttpRequestInitializer; -import com.google.protobuf.Descriptors.Descriptor; +import com.google.protobuf.DescriptorProtos; import com.google.protobuf.Int64Value; import com.google.rpc.RetryInfo; import io.grpc.Metadata; @@ -131,16 +131,16 @@ import org.apache.beam.sdk.util.ReleaseInfo; import org.apache.beam.sdk.values.FailsafeValueInSingleWindow; import org.apache.beam.sdk.values.ValueInSingleWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Futures; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListenableFuture; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Futures; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListenableFuture; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.slf4j.Logger; @@ -154,7 +154,7 @@ "nullness", // TODO(https://github.com/apache/beam/issues/20506) "keyfor" }) -class BigQueryServicesImpl implements BigQueryServices { +public class BigQueryServicesImpl implements BigQueryServices { private static final Logger LOG = LoggerFactory.getLogger(BigQueryServicesImpl.class); // The maximum number of retries to execute a BigQuery RPC. @@ -549,7 +549,7 @@ public void close() throws Exception {} } @VisibleForTesting - static class DatasetServiceImpl implements DatasetService { + public static class DatasetServiceImpl implements DatasetService { // Backoff: 200ms * 1.5 ^ n, n=[1,5] private static final FluentBackoff INSERT_BACKOFF_FACTORY = FluentBackoff.DEFAULT.withInitialBackoff(Duration.millis(200)).withMaxRetries(5); @@ -610,7 +610,7 @@ static class DatasetServiceImpl implements DatasetService { this.executor = null; } - private DatasetServiceImpl(BigQueryOptions bqOptions) { + public DatasetServiceImpl(BigQueryOptions bqOptions) { this.errorExtractor = new ApiErrorExtractor(); this.client = newBigQueryClient(bqOptions).build(); this.newWriteClient = newBigQueryWriteClient(bqOptions); @@ -1352,9 +1352,9 @@ public WriteStream createWriteStream(String tableUrn, WriteStream.Type type) @Override public StreamAppendClient getStreamAppendClient( - String streamName, Descriptor descriptor, boolean useConnectionPool) throws Exception { - ProtoSchema protoSchema = - ProtoSchema.newBuilder().setProtoDescriptor(descriptor.toProto()).build(); + String streamName, DescriptorProtos.DescriptorProto descriptor, boolean useConnectionPool) + throws Exception { + ProtoSchema protoSchema = ProtoSchema.newBuilder().setProtoDescriptor(descriptor).build(); TransportChannelProvider transportChannelProvider = BigQueryWriteSettings.defaultGrpcTransportProviderBuilder() @@ -1364,6 +1364,15 @@ public StreamAppendClient getStreamAppendClient( .setChannelsPerCpu(2) .build(); + String traceId = + String.format( + "Dataflow:%s:%s:%s", + bqIOMetadata.getBeamJobName() == null + ? options.getJobName() + : bqIOMetadata.getBeamJobName(), + bqIOMetadata.getBeamJobId() == null ? "" : bqIOMetadata.getBeamJobId(), + bqIOMetadata.getBeamWorkerId() == null ? "" : bqIOMetadata.getBeamWorkerId()); + StreamWriter streamWriter = StreamWriter.newBuilder(streamName, newWriteClient) .setExecutorProvider( @@ -1374,11 +1383,7 @@ public StreamAppendClient getStreamAppendClient( .setEnableConnectionPool(useConnectionPool) .setMaxInflightRequests(storageWriteMaxInflightRequests) .setMaxInflightBytes(storageWriteMaxInflightBytes) - .setTraceId( - "Dataflow:" - + (bqIOMetadata.getBeamJobId() != null - ? bqIOMetadata.getBeamJobId() - : options.getJobName())) + .setTraceId(traceId) .build(); return new StreamAppendClient() { private int pins = 0; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java index c1db05b200c3a..96abde5dc3574 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java @@ -17,6 +17,8 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; +import static java.util.stream.Collectors.collectingAndThen; +import static java.util.stream.Collectors.toList; import static org.apache.beam.sdk.io.FileSystems.match; import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.resolveTempLocation; import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; @@ -29,6 +31,8 @@ import com.google.api.services.bigquery.model.TableSchema; import java.io.IOException; import java.util.List; +import java.util.stream.Stream; +import org.apache.avro.generic.GenericRecord; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.extensions.avro.io.AvroSource; import org.apache.beam.sdk.io.BoundedSource; @@ -39,8 +43,7 @@ import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -242,24 +245,19 @@ List> createSources( AvroSource.DatumReaderFactory factory = readerFactory.apply(schema); - List> avroSources = Lists.newArrayList(); + Stream> avroSources; // If metadata is available, create AvroSources with said metadata in SINGLE_FILE_OR_SUBRANGE // mode. if (metadata != null) { - for (MatchResult.Metadata file : metadata) { - avroSources.add( - (AvroSource) - AvroSource.from(file).withSchema(avroSchema).withDatumReaderFactory(factory)); - } + avroSources = metadata.stream().map(AvroSource::from); } else { - for (ResourceId file : files) { - avroSources.add( - (AvroSource) - AvroSource.from(file.toString()) - .withSchema(avroSchema) - .withDatumReaderFactory(factory)); - } + avroSources = files.stream().map(ResourceId::toString).map(AvroSource::from); } - return ImmutableList.copyOf(avroSources); + + return avroSources + .map(s -> s.withSchema(avroSchema)) + .map(s -> (AvroSource) s.withDatumReaderFactory(factory)) + .map(s -> s.withCoder(coder)) + .collect(collectingAndThen(toList(), ImmutableList::copyOf)); } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageQuerySource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageQuerySource.java index 906c66c79cec8..2d87c7c1c7db0 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageQuerySource.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageQuerySource.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.bigquery.model.JobStatistics; import com.google.api.services.bigquery.model.Table; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageSourceBase.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageSourceBase.java index 834409062ccdc..e02e749e38dfd 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageSourceBase.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageSourceBase.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.bigquery.model.Table; import com.google.api.services.bigquery.model.TableSchema; @@ -37,8 +37,8 @@ import org.apache.beam.sdk.options.ValueProvider; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.util.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageStreamBundleSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageStreamBundleSource.java index 42e99b6aae381..a2df86af1ee64 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageStreamBundleSource.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageStreamBundleSource.java @@ -19,7 +19,7 @@ import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.fromJsonString; import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.toJsonString; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.gax.rpc.ApiException; import com.google.api.services.bigquery.model.TableReference; @@ -41,7 +41,7 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.util.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.slf4j.Logger; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageStreamSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageStreamSource.java index 812524615ce22..a4336cd48f944 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageStreamSource.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageStreamSource.java @@ -19,7 +19,7 @@ import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.fromJsonString; import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.toJsonString; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.gax.rpc.ApiException; import com.google.api.gax.rpc.FailedPreconditionException; @@ -46,7 +46,7 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.util.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.slf4j.Logger; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageTableSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageTableSource.java index 26a9bed20c726..909a2551b299c 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageTableSource.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageTableSource.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.services.bigquery.model.Table; import com.google.api.services.bigquery.model.TableReference; @@ -33,7 +33,7 @@ import org.apache.beam.sdk.options.ValueProvider; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.display.DisplayData; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java index 9d24246405ac0..1b6aedf8cb176 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.display.DisplayData; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; /** A {@link BigQuerySourceBase} for reading BigQuery tables. */ diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSourceDef.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSourceDef.java index b71cc7a2a8f4b..b399900f9a248 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSourceDef.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSourceDef.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.services.bigquery.model.Table; import com.google.api.services.bigquery.model.TableReference; @@ -31,7 +31,7 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.util.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java index 05e65301564ce..fa5ffae0909d4 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java @@ -64,11 +64,12 @@ import org.apache.beam.sdk.transforms.SerializableFunctions; import org.apache.beam.sdk.util.Preconditions; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -94,8 +95,15 @@ public class BigQueryUtils { // For parsing the format used to refer to tables parameters in BigQueryIO. // "{project_id}:{dataset_id}.{table_id}" or // "{project_id}.{dataset_id}.{table_id}" + // following documentation in + // https://cloud.google.com/resource-manager/docs/creating-managing-projects#before_you_begin, + // https://cloud.google.com/bigquery/docs/datasets#dataset-naming, and + // https://cloud.google.com/bigquery/docs/tables#table_naming private static final Pattern SIMPLE_TABLE_PATTERN = - Pattern.compile("^(?[^\\.:]+)[\\.:](?[^\\.:]+)[\\.](?[^\\.:]+)$"); + Pattern.compile( + "^(?[a-z][a-z0-9.\\-:]{4,28}[a-z0-9])[\\:.]" + + "(?[a-zA-Z0-9_]{1,1024})[\\.]" + + "(?
    [\\p{L}\\p{M}\\p{N}\\p{Pc}\\p{Pd}\\p{Zs}$]{1,1024})$"); /** Options for how to convert BigQuery data to Beam data. */ @AutoValue @@ -626,23 +634,10 @@ public static Row toBeamRow(Schema rowSchema, TableRow jsonBqRow) { // 2. TableSchema objects are not serializable and are therefore harder to propagate through a // pipeline. return rowSchema.getFields().stream() - .map(field -> toBeamRowFieldValue(field, jsonBqRow.get(field.getName()))) + .map(field -> toBeamValue(field, jsonBqRow.get(field.getName()))) .collect(toRow(rowSchema)); } - private static Object toBeamRowFieldValue(Field field, Object bqValue) { - if (bqValue == null) { - if (field.getType().getNullable()) { - return null; - } else { - throw new IllegalArgumentException( - "Received null value for non-nullable field \"" + field.getName() + "\""); - } - } - - return toBeamValue(field.getType(), bqValue); - } - /** * Tries to parse the JSON {@link TableRow} from BigQuery. * @@ -664,11 +659,22 @@ public static Row toBeamRow(Schema rowSchema, TableSchema bqSchema, TableRow jso return IntStream.range(0, rowSchema.getFieldCount()) .boxed() - .map(index -> toBeamValue(rowSchema.getField(index).getType(), rawJsonValues.get(index))) + .map(index -> toBeamValue(rowSchema.getField(index), rawJsonValues.get(index))) .collect(toRow(rowSchema)); } - private static @Nullable Object toBeamValue(FieldType fieldType, Object jsonBQValue) { + private static @Nullable Object toBeamValue(Field field, Object jsonBQValue) { + FieldType fieldType = field.getType(); + + if (jsonBQValue == null) { + if (fieldType.getNullable()) { + return null; + } else { + throw new IllegalArgumentException( + "Received null value for non-nullable field \"" + field.getName() + "\""); + } + } + if (jsonBQValue instanceof String || jsonBQValue instanceof Number || jsonBQValue instanceof Boolean) { @@ -691,6 +697,10 @@ public static Row toBeamRow(Schema rowSchema, TableSchema bqSchema, TableRow jso } } + if (jsonBQValue instanceof byte[] && fieldType.getTypeName() == TypeName.BYTES) { + return jsonBQValue; + } + if (jsonBQValue instanceof List) { if (fieldType.getCollectionElementType() == null) { throw new IllegalArgumentException( @@ -710,7 +720,7 @@ public static Row toBeamRow(Schema rowSchema, TableSchema bqSchema, TableRow jso (!innerTypeIsMap && v instanceof Map) ? ((Map) v).get("v") : v) - .map(v -> toBeamValue(fieldType.getCollectionElementType(), v)) + .map(v -> toBeamValue(field.withType(fieldType.getCollectionElementType()), v)) .collect(toList()); } @@ -998,6 +1008,25 @@ private static Object convertAvroNumeric(Object value) { return null; } + /** + * @param tableReference - a BigQueryTableIdentifier that may or may not include the project. + * @return a String representation of the table destination in the form: + * `myproject.mydataset.mytable` + */ + public static @Nullable String toTableSpec(TableReference tableReference) { + if (tableReference.getDatasetId() == null || tableReference.getTableId() == null) { + throw new IllegalArgumentException( + String.format( + "Table reference [%s] must include at least a dataset and a table.", tableReference)); + } + String tableSpec = + String.format("%s.%s", tableReference.getDatasetId(), tableReference.getTableId()); + if (!Strings.isNullOrEmpty(tableReference.getProjectId())) { + tableSpec = String.format("%s.%s", tableReference.getProjectId(), tableSpec); + } + return tableSpec; + } + private static @Nullable ServiceCallMetric callMetricForMethod( @Nullable TableReference tableReference, String method) { if (tableReference != null) { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CivilTimeEncoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CivilTimeEncoder.java index bf8fe19f21fcd..b7c37dca39512 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CivilTimeEncoder.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CivilTimeEncoder.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.time.temporal.ChronoUnit; import org.joda.time.LocalDateTime; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTableHelpers.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTableHelpers.java index ae7e82c0ad12c..6edd3f71cc71c 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTableHelpers.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTableHelpers.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.client.util.BackOff; import com.google.api.client.util.BackOffUtils; @@ -25,6 +25,7 @@ import com.google.api.services.bigquery.model.Clustering; import com.google.api.services.bigquery.model.EncryptionConfiguration; import com.google.api.services.bigquery.model.Table; +import com.google.api.services.bigquery.model.TableConstraints; import com.google.api.services.bigquery.model.TableReference; import com.google.api.services.bigquery.model.TableSchema; import com.google.api.services.bigquery.model.TimePartitioning; @@ -39,9 +40,9 @@ import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService; import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; @@ -86,6 +87,7 @@ static TableDestination possiblyCreateTable( BigQueryOptions bigQueryOptions, TableDestination tableDestination, Supplier<@Nullable TableSchema> schemaSupplier, + Supplier<@Nullable TableConstraints> tableConstraintsSupplier, CreateDisposition createDisposition, @Nullable Coder tableDestinationCoder, @Nullable String kmsKey, @@ -125,6 +127,7 @@ static TableDestination possiblyCreateTable( tryCreateTable( bigQueryOptions, schemaSupplier, + tableConstraintsSupplier, tableDestination, createDisposition, tableSpec, @@ -139,6 +142,7 @@ static TableDestination possiblyCreateTable( private static void tryCreateTable( BigQueryOptions options, Supplier<@Nullable TableSchema> schemaSupplier, + Supplier<@Nullable TableConstraints> tableConstraintsSupplier, TableDestination tableDestination, CreateDisposition createDisposition, String tableSpec, @@ -151,6 +155,7 @@ private static void tryCreateTable( tableReference, Collections.emptyList(), DatasetService.TableMetadataView.BASIC) == null) { TableSchema tableSchema = schemaSupplier.get(); + @Nullable TableConstraints tableConstraints = tableConstraintsSupplier.get(); Preconditions.checkArgumentNotNull( tableSchema, "Unless create disposition is %s, a schema must be specified, i.e. " @@ -162,6 +167,10 @@ private static void tryCreateTable( tableDestination); Table table = new Table().setTableReference(tableReference).setSchema(tableSchema); + if (tableConstraints != null) { + table = table.setTableConstraints(tableConstraints); + } + String tableDescription = tableDestination.getTableDescription(); if (tableDescription != null) { table = table.setDescription(tableDescription); diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java index e1e1f22df6352..7e5299b7e674c 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java @@ -17,8 +17,9 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import com.google.api.services.bigquery.model.TableConstraints; import com.google.api.services.bigquery.model.TableSchema; import java.util.List; import java.util.Map; @@ -30,10 +31,10 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; /** @@ -113,10 +114,14 @@ public void processElement(ProcessContext context) { dest); Supplier<@Nullable TableSchema> schemaSupplier = () -> dynamicDestinations.getSchema(dest); + Supplier<@Nullable TableConstraints> tableConstraintsSupplier = + () -> dynamicDestinations.getTableConstraints(dest); + return CreateTableHelpers.possiblyCreateTable( context.getPipelineOptions().as(BigQueryOptions.class), tableDestination1, schemaSupplier, + tableConstraintsSupplier, createDisposition, dynamicDestinations.getDestinationCoder(), kmsKey, diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinations.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinations.java index f10cfc0e24c53..e5cf82d7c2e81 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinations.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinations.java @@ -18,8 +18,9 @@ package org.apache.beam.sdk.io.gcp.bigquery; import static org.apache.beam.sdk.values.TypeDescriptors.extractFromTypeParameters; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; +import com.google.api.services.bigquery.model.TableConstraints; import com.google.api.services.bigquery.model.TableSchema; import java.io.Serializable; import java.util.List; @@ -32,7 +33,7 @@ import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; import org.apache.beam.sdk.values.ValueInSingleWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; /** @@ -154,6 +155,14 @@ void setSideInputAccessorFromProcessContext(DoFn.ProcessContext context) { /** Returns the table schema for the destination. */ public abstract @Nullable TableSchema getSchema(DestinationT destination); + /** + * Returns TableConstraints (including primary and foreign key) to be used when creating the + * table. Note: this is not currently supported when using FILE_LOADS!. + */ + public @Nullable TableConstraints getTableConstraints(DestinationT destination) { + return null; + } + // Gets the destination coder. If the user does not provide one, try to find one in the coder // registry. If no coder can be found, throws CannotProvideCoderException. Coder getDestinationCoderWithDefault(CoderRegistry registry) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinationsHelpers.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinationsHelpers.java index 130a81f7aa28b..62355fd9417de 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinationsHelpers.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinationsHelpers.java @@ -17,12 +17,13 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.client.util.BackOff; import com.google.api.client.util.BackOffUtils; import com.google.api.client.util.Sleeper; import com.google.api.services.bigquery.model.Table; +import com.google.api.services.bigquery.model.TableConstraints; import com.google.api.services.bigquery.model.TableReference; import com.google.api.services.bigquery.model.TableSchema; import java.io.IOException; @@ -44,8 +45,8 @@ import org.apache.beam.sdk.util.Preconditions; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.ValueInSingleWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.slf4j.Logger; @@ -177,6 +178,11 @@ public DestinationT getDestination(@Nullable ValueInSingleWindow element) { return inner.getSchema(destination); } + @Override + public @Nullable TableConstraints getTableConstraints(DestinationT destination) { + return inner.getTableConstraints(destination); + } + @Override public TableDestination getTable(DestinationT destination) { return inner.getTable(destination); @@ -214,6 +220,30 @@ public String toString() { } } + static class ConstantTableConstraintsDestinations + extends DelegatingDynamicDestinations { + private final String jsonTableConstraints; + + ConstantTableConstraintsDestinations( + DynamicDestinations inner, TableConstraints tableConstraints) { + super(inner); + this.jsonTableConstraints = BigQueryHelpers.toJsonString(tableConstraints); + } + + @Override + public TableConstraints getTableConstraints(DestinationT destination) { + return BigQueryHelpers.fromJsonString(jsonTableConstraints, TableConstraints.class); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("inner", inner) + .add("tableConstraints", jsonTableConstraints) + .toString(); + } + } + /** Returns the same schema for every table. */ static class ConstantSchemaDestinations extends DelegatingDynamicDestinations { @@ -421,12 +451,17 @@ public TableDestination getTable(DestinationT destination) { } } - /** Returns the table schema for the destination. */ + /** + * Returns the table schema for the destination. If possible, will return the existing table + * schema. + */ @Override public @Nullable TableSchema getSchema(DestinationT destination) { TableDestination wrappedDestination = super.getTable(destination); @Nullable Table existingTable = getBigQueryTable(wrappedDestination.getTableReference()); - if (existingTable == null || existingTable.getSchema() == null) { + if (existingTable == null + || existingTable.getSchema() == null + || existingTable.getSchema().isEmpty()) { return super.getSchema(destination); } else { return existingTable.getSchema(); diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/InsertRetryPolicy.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/InsertRetryPolicy.java index 33f846870fcec..edaa88d87b7ca 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/InsertRetryPolicy.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/InsertRetryPolicy.java @@ -21,7 +21,7 @@ import com.google.api.services.bigquery.model.TableDataInsertAllResponse; import java.io.Serializable; import java.util.Set; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; /** A retry policy for streaming BigQuery inserts. */ public abstract class InsertRetryPolicy implements Serializable { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java index c1323d7f86291..3209af383618d 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PrepareWrite.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PrepareWrite.java index 1e00e877b24a0..5c44497fdbd9b 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PrepareWrite.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PrepareWrite.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.services.bigquery.model.TableRow; import java.io.IOException; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/RetryManager.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/RetryManager.java index 34a64c1baeb94..245d05284236b 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/RetryManager.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/RetryManager.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutureCallback; @@ -38,8 +38,8 @@ import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.Preconditions; import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Queues; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Queues; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.joda.time.Duration; /** diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/SplittingIterable.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/SplittingIterable.java index a7de876b98fab..b8eeb2522cf2b 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/SplittingIterable.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/SplittingIterable.java @@ -27,7 +27,7 @@ import java.util.function.BiConsumer; import java.util.function.Function; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; @@ -104,7 +104,7 @@ public Value next() { if (autoUpdateSchema) { try { @Nullable TableRow unknownFields = payload.getUnknownFields(); - if (unknownFields != null) { + if (unknownFields != null && !unknownFields.isEmpty()) { // Protocol buffer serialization format supports concatenation. We serialize any new // "known" fields // into a proto and concatenate to the existing proto. diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiCDC.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiCDC.java index 7532f396ea3b0..48cb9bb902c9f 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiCDC.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiCDC.java @@ -36,7 +36,7 @@ */ import java.util.Set; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; /** Constants and variables for CDC support. */ public class StorageApiCDC { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinations.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinations.java index b3085b15e9510..8ec4d52e3b90f 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinations.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinations.java @@ -18,61 +18,32 @@ package org.apache.beam.sdk.io.gcp.bigquery; import com.google.api.services.bigquery.model.TableRow; -import com.google.api.services.bigquery.model.TableSchema; -import java.util.List; +import com.google.protobuf.DescriptorProtos; import javax.annotation.Nullable; -import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService; import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.sdk.values.ValueInSingleWindow; /** Base dynamicDestinations class used by the Storage API sink. */ abstract class StorageApiDynamicDestinations - extends DynamicDestinations { + extends DynamicDestinationsHelpers.DelegatingDynamicDestinations { public interface MessageConverter { com.google.cloud.bigquery.storage.v1.TableSchema getTableSchema(); + DescriptorProtos.DescriptorProto getDescriptor(boolean includeCdcColumns) throws Exception; + StorageApiWritePayload toMessage( T element, @Nullable RowMutationInformation rowMutationInformation) throws Exception; TableRow toTableRow(T element); } - private DynamicDestinations inner; - StorageApiDynamicDestinations(DynamicDestinations inner) { - this.inner = inner; + super(inner); } public abstract MessageConverter getMessageConverter( DestinationT destination, DatasetService datasetService) throws Exception; - @Override - public DestinationT getDestination(@Nullable ValueInSingleWindow element) { - return inner.getDestination(element); - } - - @Override - public @Nullable Coder getDestinationCoder() { - return inner.getDestinationCoder(); - } - - @Override - public TableDestination getTable(DestinationT destination) { - return inner.getTable(destination); - } - - @Override - public @Nullable TableSchema getSchema(DestinationT destination) { - return inner.getSchema(destination); - } - - @Override - public List> getSideInputs() { - return inner.getSideInputs(); - } - @Override void setSideInputAccessorFromProcessContext(DoFn.ProcessContext context) { super.setSideInputAccessorFromProcessContext(context); diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsBeamRow.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsBeamRow.java index e0964135be38b..7821a7d91994b 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsBeamRow.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsBeamRow.java @@ -19,6 +19,7 @@ import com.google.api.services.bigquery.model.TableRow; import com.google.cloud.bigquery.storage.v1.TableSchema; +import com.google.protobuf.DescriptorProtos; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Message; import javax.annotation.Nullable; @@ -75,6 +76,11 @@ public TableSchema getTableSchema() { return tableSchema; } + @Override + public DescriptorProtos.DescriptorProto getDescriptor(boolean includeCdcColumns) { + return cdcDescriptor != null ? cdcDescriptor.toProto() : descriptor.toProto(); + } + @Override @SuppressWarnings("nullness") public StorageApiWritePayload toMessage( diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsGenericRecord.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsGenericRecord.java index 53f3b137bf38b..79141d73a39b3 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsGenericRecord.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsGenericRecord.java @@ -19,6 +19,7 @@ import com.google.api.services.bigquery.model.TableRow; import com.google.api.services.bigquery.model.TableSchema; +import com.google.protobuf.DescriptorProtos; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Message; import org.apache.avro.Schema; @@ -109,5 +110,11 @@ public TableRow toTableRow(T element) { public com.google.cloud.bigquery.storage.v1.TableSchema getTableSchema() { return protoTableSchema; } + + @Override + public DescriptorProtos.DescriptorProto getDescriptor(boolean includeCdcColumns) + throws Exception { + return cdcDescriptor != null ? cdcDescriptor.toProto() : descriptor.toProto(); + } } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsProto.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsProto.java new file mode 100644 index 0000000000000..57dbdc9d1e770 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsProto.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.gcp.bigquery; + +import com.google.api.services.bigquery.model.TableRow; +import com.google.cloud.bigquery.storage.v1.ProtoSchemaConverter; +import com.google.cloud.bigquery.storage.v1.TableSchema; +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; +import com.google.protobuf.Message; +import java.lang.reflect.InvocationTargetException; +import javax.annotation.Nullable; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService; +import org.apache.beam.sdk.util.Preconditions; +import org.checkerframework.checker.nullness.qual.NonNull; + +/** Storage API DynamicDestinations used when the input is a compiled protocol buffer. */ +class StorageApiDynamicDestinationsProto + extends StorageApiDynamicDestinations { + DescriptorProtos.DescriptorProto descriptorProto; + + @SuppressWarnings({"unchecked", "nullness"}) + StorageApiDynamicDestinationsProto( + DynamicDestinations inner, Class protoClass) { + super(inner); + try { + this.descriptorProto = + fixNestedTypes( + (Descriptors.Descriptor) + Preconditions.checkStateNotNull(protoClass.getMethod("getDescriptor")) + .invoke(null)); + } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public MessageConverter getMessageConverter( + DestinationT destination, DatasetService datasetService) throws Exception { + return new Converter( + TableRowToStorageApiProto.schemaToProtoTableSchema( + Preconditions.checkStateNotNull(getSchema(destination)))); + } + + class Converter implements MessageConverter { + TableSchema tableSchema; + + Converter(TableSchema tableSchema) { + this.tableSchema = tableSchema; + } + + @Override + public TableSchema getTableSchema() { + return tableSchema; + } + + @Override + public DescriptorProtos.DescriptorProto getDescriptor(boolean includeCdcColumns) + throws Exception { + org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument( + !includeCdcColumns); + return descriptorProto; + } + + @Override + public StorageApiWritePayload toMessage( + T element, @Nullable RowMutationInformation rowMutationInformation) throws Exception { + // NB: What makes this path efficient is that the storage API directly understands protos, so + // we can forward + // the through directly. This means that we don't currently support ignoreUnknownValues or + // autoUpdateSchema. + return StorageApiWritePayload.of(element.toByteArray(), null); + } + + @Override + public TableRow toTableRow(T element) { + throw new RuntimeException("Not implemented!"); + } + }; + + private static DescriptorProtos.DescriptorProto fixNestedTypes( + Descriptors.Descriptor descriptor) { + return ProtoSchemaConverter.convert(descriptor).getProtoDescriptor(); + } +} diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsTableRow.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsTableRow.java index bafa0bd1c7ece..eb93d7c398f79 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsTableRow.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDynamicDestinationsTableRow.java @@ -20,6 +20,7 @@ import com.google.api.services.bigquery.model.TableReference; import com.google.api.services.bigquery.model.TableRow; import com.google.api.services.bigquery.model.TableSchema; +import com.google.protobuf.DescriptorProtos; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Message; import java.util.concurrent.ExecutionException; @@ -28,7 +29,7 @@ import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.util.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.NonNull; import org.joda.time.Duration; @@ -143,6 +144,12 @@ public com.google.cloud.bigquery.storage.v1.TableSchema getTableSchema() { return protoTableSchema; } + @Override + public DescriptorProtos.DescriptorProto getDescriptor(boolean includeCdcColumns) + throws Exception { + return cdcDescriptor != null ? cdcDescriptor.toProto() : descriptor.toProto(); + } + @Override public TableRow toTableRow(T element) { return formatFunction.apply(element); diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiFinalizeWritesDoFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiFinalizeWritesDoFn.java index 96c1e9591389c..6ce58a10ef8b6 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiFinalizeWritesDoFn.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiFinalizeWritesDoFn.java @@ -34,10 +34,10 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.util.Preconditions; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.slf4j.Logger; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiFlushAndFinalizeDoFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiFlushAndFinalizeDoFn.java index c28ef2d5eacd0..bb2bfba85fbb2 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiFlushAndFinalizeDoFn.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiFlushAndFinalizeDoFn.java @@ -40,7 +40,7 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.util.Preconditions; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWriteUnshardedRecords.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWriteUnshardedRecords.java index 3c389f7d7ffea..f54522c2ca59b 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWriteUnshardedRecords.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWriteUnshardedRecords.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutures; @@ -30,8 +30,8 @@ import com.google.cloud.bigquery.storage.v1.WriteStream; import com.google.cloud.bigquery.storage.v1.WriteStream.Type; import com.google.protobuf.ByteString; +import com.google.protobuf.DescriptorProtos; import com.google.protobuf.DynamicMessage; -import com.google.protobuf.InvalidProtocolBufferException; import io.grpc.Status; import java.io.IOException; import java.time.Instant; @@ -46,6 +46,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -71,13 +72,13 @@ import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.RemovalNotification; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.RemovalNotification; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; @@ -183,7 +184,7 @@ public StorageApiWriteUnshardedRecords( public PCollectionTuple expand(PCollection> input) { String operationName = input.getName() + "/" + getName(); BigQueryOptions options = input.getPipeline().getOptions().as(BigQueryOptions.class); - org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument( + org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument( !options.getUseStorageApiConnectionPool(), "useStorageApiConnectionPool only supported " + "when using STORAGE_API_AT_LEAST_ONCE"); TupleTagList tupleTagList = TupleTagList.of(failedRowsTag); @@ -278,6 +279,7 @@ class DestinationState { private final boolean useDefaultStream; private TableSchema initialTableSchema; + private DescriptorProtos.DescriptorProto initialDescriptor; private Instant nextCacheTickle = Instant.MAX; private final int clientNumber; private final boolean usingMultiplexing; @@ -302,6 +304,7 @@ public DestinationState( this.maybeDatasetService = datasetService; this.useDefaultStream = useDefaultStream; this.initialTableSchema = messageConverter.getTableSchema(); + this.initialDescriptor = messageConverter.getDescriptor(includeCdcColumns); this.clientNumber = new Random().nextInt(streamAppendClientCount); this.usingMultiplexing = usingMultiplexing; this.maxRequestSize = maxRequestSize; @@ -359,12 +362,13 @@ String getOrCreateStreamName() throws Exception { } AppendClientInfo generateClient(@Nullable TableSchema updatedSchema) throws Exception { - TableSchema tableSchema = - (updatedSchema != null) ? updatedSchema : getCurrentTableSchema(streamName); + SchemaAndDescriptor schemaAndDescriptor = getCurrentTableSchema(streamName, updatedSchema); + AtomicReference appendClientInfo = new AtomicReference<>( AppendClientInfo.of( - tableSchema, + schemaAndDescriptor.tableSchema, + schemaAndDescriptor.descriptor, // Make sure that the client is always closed in a different thread to avoid // blocking. client -> @@ -376,8 +380,7 @@ AppendClientInfo generateClient(@Nullable TableSchema updatedSchema) throws Exce client.unpin(); client.close(); } - }), - includeCdcColumns)); + }))); CreateTableHelpers.createTableWrapper( () -> { @@ -398,8 +401,28 @@ AppendClientInfo generateClient(@Nullable TableSchema updatedSchema) throws Exce return appendClientInfo.get(); } - TableSchema getCurrentTableSchema(String stream) throws Exception { + private class SchemaAndDescriptor { + private final TableSchema tableSchema; + private final DescriptorProtos.DescriptorProto descriptor; + + private SchemaAndDescriptor( + TableSchema tableSchema, DescriptorProtos.DescriptorProto descriptor) { + this.tableSchema = tableSchema; + this.descriptor = descriptor; + } + } + + SchemaAndDescriptor getCurrentTableSchema(String stream, @Nullable TableSchema updatedSchema) + throws Exception { + if (updatedSchema != null) { + return new SchemaAndDescriptor( + updatedSchema, + TableRowToStorageApiProto.descriptorSchemaFromTableSchema( + updatedSchema, true, includeCdcColumns)); + } + AtomicReference currentSchema = new AtomicReference<>(initialTableSchema); + AtomicBoolean updated = new AtomicBoolean(); CreateTableHelpers.createTableWrapper( () -> { if (autoUpdateSchema) { @@ -407,13 +430,27 @@ TableSchema getCurrentTableSchema(String stream) throws Exception { WriteStream writeStream = Preconditions.checkStateNotNull(maybeDatasetService).getWriteStream(streamName); if (writeStream != null && writeStream.hasTableSchema()) { - currentSchema.set(writeStream.getTableSchema()); + TableSchema updatedFromStream = writeStream.getTableSchema(); + currentSchema.set(updatedFromStream); + updated.set(true); + LOG.debug( + "Fetched updated schema for table {}:\n\t{}", tableUrn, updatedFromStream); } } return null; }, tryCreateTable); - return currentSchema.get(); + // Note: While it may appear that these two branches are the same, it's important to return + // the actual + // initial descriptor if the schema has not changed. Simply converting the schema back into + // a descriptor isn't + // the same, and would break the direct-from-proto ingestion path. + DescriptorProtos.DescriptorProto descriptor = + updated.get() + ? TableRowToStorageApiProto.descriptorSchemaFromTableSchema( + currentSchema.get(), true, includeCdcColumns) + : initialDescriptor; + return new SchemaAndDescriptor(currentSchema.get(), descriptor); } AppendClientInfo getAppendClientInfo( @@ -555,7 +592,9 @@ long flush( TableRow failedRow = TableRowToStorageApiProto.tableRowFromMessage( DynamicMessage.parseFrom( - getAppendClientInfo(true, null).getDescriptor(), rowBytes), + TableRowToStorageApiProto.wrapDescriptorProto( + getAppendClientInfo(true, null).getDescriptor()), + rowBytes), true); failedRowsReceiver.outputWithTimestamp( new BigQueryStorageApiInsertError( @@ -618,14 +657,16 @@ long flush( TableRow failedRow = TableRowToStorageApiProto.tableRowFromMessage( DynamicMessage.parseFrom( - Preconditions.checkStateNotNull(appendClientInfo).getDescriptor(), + TableRowToStorageApiProto.wrapDescriptorProto( + Preconditions.checkStateNotNull(appendClientInfo) + .getDescriptor()), protoBytes), true); failedRowsReceiver.outputWithTimestamp( new BigQueryStorageApiInsertError( failedRow, error.getRowIndexToErrorMessage().get(failedIndex)), timestamp); - } catch (InvalidProtocolBufferException e) { + } catch (Exception e) { LOG.error("Failed to insert row and could not parse the result!", e); } } @@ -716,12 +757,14 @@ long flush( TableRow row = TableRowToStorageApiProto.tableRowFromMessage( DynamicMessage.parseFrom( - Preconditions.checkStateNotNull(appendClientInfo).getDescriptor(), + TableRowToStorageApiProto.wrapDescriptorProto( + Preconditions.checkStateNotNull(appendClientInfo) + .getDescriptor()), rowBytes), true); org.joda.time.Instant timestamp = c.timestamps.get(i); successfulRowsReceiver.outputWithTimestamp(row, timestamp); - } catch (InvalidProtocolBufferException e) { + } catch (Exception e) { LOG.warn("Failure parsing TableRow", e); } } @@ -899,6 +942,7 @@ DestinationState createDestinationState( c.getPipelineOptions().as(BigQueryOptions.class), tableDestination1, () -> dynamicDestinations.getSchema(destination), + () -> dynamicDestinations.getTableConstraints(destination), createDisposition, destinationCoder, kmsKey, diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritesShardedRecords.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritesShardedRecords.java index f74a0899e7086..efcf87eac7a32 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritesShardedRecords.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritesShardedRecords.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutures; @@ -29,6 +29,7 @@ import com.google.cloud.bigquery.storage.v1.TableSchema; import com.google.cloud.bigquery.storage.v1.WriteStream.Type; import com.google.protobuf.ByteString; +import com.google.protobuf.DescriptorProtos; import io.grpc.Status; import io.grpc.Status.Code; import java.io.IOException; @@ -88,21 +89,21 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.RemovalNotification; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.RemovalNotification; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** A transform to write sharded records to BigQuery using the Storage API. */ +/** A transform to write sharded records to BigQuery using the Storage API (Streaming). */ @SuppressWarnings({ "FutureReturnValueIgnored", // TODO(https://github.com/apache/beam/issues/21230): Remove when new version of @@ -440,10 +441,12 @@ public void process( Coder destinationCoder = dynamicDestinations.getDestinationCoder(); Callable tryCreateTable = () -> { + DestinationT dest = element.getKey().getKey(); CreateTableHelpers.possiblyCreateTable( c.getPipelineOptions().as(BigQueryOptions.class), tableDestination, - () -> dynamicDestinations.getSchema(element.getKey().getKey()), + () -> dynamicDestinations.getSchema(dest), + () -> dynamicDestinations.getTableConstraints(dest), createDisposition, destinationCoder, kmsKey, @@ -458,21 +461,28 @@ public void process( Callable getAppendClientInfo = () -> { @Nullable TableSchema tableSchema; - if (autoUpdateSchema && updatedSchema.read() != null) { - // We've seen an updated schema, so we use that. - tableSchema = updatedSchema.read(); + DescriptorProtos.DescriptorProto descriptor; + TableSchema updatedSchemaValue = updatedSchema.read(); + if (autoUpdateSchema && updatedSchemaValue != null) { + // We've seen an updated schema, so we use that instead of querying the + // MessageConverter. + tableSchema = updatedSchemaValue; + descriptor = + TableRowToStorageApiProto.descriptorSchemaFromTableSchema( + tableSchema, true, false); } else { // Start off with the base schema. As we get notified of schema updates, we - // will update the - // descriptor. - tableSchema = - messageConverters - .get(element.getKey().getKey(), dynamicDestinations, datasetService) - .getTableSchema(); + // will update the descriptor. + StorageApiDynamicDestinations.MessageConverter converter = + messageConverters.get( + element.getKey().getKey(), dynamicDestinations, datasetService); + tableSchema = converter.getTableSchema(); + descriptor = converter.getDescriptor(false); } AppendClientInfo info = AppendClientInfo.of( Preconditions.checkStateNotNull(tableSchema), + descriptor, // Make sure that the client is always closed in a different thread // to // avoid blocking. @@ -483,8 +493,7 @@ public void process( // Remove the pin that is "owned" by the cache. client.unpin(); client.close(); - }), - false) + })) .withAppendClient(datasetService, getOrCreateStream, false); // This pin is "owned" by the cache. Preconditions.checkStateNotNull(info.getStreamAppendClient()).pin(); @@ -652,9 +661,25 @@ public void process( return RetryType.RETRY_ALL_OPERATIONS; } + Throwable error = Preconditions.checkStateNotNull(failedContext.getError()); + Status.Code statusCode = Status.fromThrowable(error).getCode(); + // This means that the offset we have stored does not match the current end of + // the stream in the Storage API. Usually this happens because a crash or a bundle + // failure + // happened after an append but before the worker could checkpoint it's + // state. The records that were appended in a failed bundle will be retried, + // meaning that the unflushed tail of the stream must be discarded to prevent + // duplicates. + boolean offsetMismatch = + statusCode.equals(Code.OUT_OF_RANGE) || statusCode.equals(Code.ALREADY_EXISTS); + // Invalidate the StreamWriter and force a new one to be created. - LOG.error( - "Got error " + failedContext.getError() + " closing " + failedContext.streamName); + if (!offsetMismatch) { + // Don't log errors for expected offset mismatch. These will be logged as warnings + // below. + LOG.error( + "Got error " + failedContext.getError() + " closing " + failedContext.streamName); + } // TODO: Only do this on explicit NOT_FOUND errors once BigQuery reliably produces them. try { @@ -667,17 +692,6 @@ public void process( boolean explicitStreamFinalized = failedContext.getError() instanceof StreamFinalizedException; - Throwable error = Preconditions.checkStateNotNull(failedContext.getError()); - Status.Code statusCode = Status.fromThrowable(error).getCode(); - // This means that the offset we have stored does not match the current end of - // the stream in the Storage API. Usually this happens because a crash or a bundle - // failure - // happened after an append but before the worker could checkpoint it's - // state. The records that were appended in a failed bundle will be retried, - // meaning that the unflushed tail of the stream must be discarded to prevent - // duplicates. - boolean offsetMismatch = - statusCode.equals(Code.OUT_OF_RANGE) || statusCode.equals(Code.ALREADY_EXISTS); // This implies that the stream doesn't exist or has already been finalized. In this // case we have no choice but to create a new stream. boolean streamDoesNotExist = @@ -804,6 +818,8 @@ public void process( newSchema.get(), appendClientInfo.get().getCloseAppendClient(), false)); APPEND_CLIENTS.invalidate(element.getKey()); APPEND_CLIENTS.put(element.getKey(), appendClientInfo.get()); + LOG.debug( + "Fetched updated schema for table {}:\n\t{}", tableId, updatedSchemaReturned); updatedSchema.write(newSchema.get()); } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java index 838d1e4a4701f..3f9f1b750ccd1 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java @@ -22,7 +22,7 @@ import com.google.api.services.bigquery.model.TimePartitioning; import java.io.Serializable; import java.util.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.checkerframework.checker.nullness.qual.Nullable; /** Encapsulates a BigQuery table destination. */ diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java index b56f3241b5ee7..1d9000432e33f 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.CoderException; import org.apache.beam.sdk.coders.StringUtf8Coder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** Defines a coder for {@link TableRowInfo} objects. */ @VisibleForTesting diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java index b8dbb9703c2e0..c31886da61447 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java @@ -30,6 +30,7 @@ import com.google.protobuf.DescriptorProtos.FieldDescriptorProto.Label; import com.google.protobuf.DescriptorProtos.FieldDescriptorProto.Type; import com.google.protobuf.DescriptorProtos.FileDescriptorProto; +import com.google.protobuf.Descriptors; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.DescriptorValidationException; import com.google.protobuf.Descriptors.FieldDescriptor; @@ -48,7 +49,6 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.time.format.DateTimeParseException; -import java.time.temporal.ChronoUnit; import java.util.AbstractMap; import java.util.Collections; import java.util.List; @@ -59,14 +59,14 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.apache.beam.sdk.util.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Days; @@ -217,7 +217,7 @@ public static TableFieldSchema.Mode modeToProtoMode(String mode) { return Optional.ofNullable(mode) .map(Mode::valueOf) .map(m -> MODE_MAP_JSON_PROTO.get(m)) - .orElse(TableFieldSchema.Mode.REQUIRED); + .orElse(TableFieldSchema.Mode.NULLABLE); } public static String protoModeToJsonMode(TableFieldSchema.Mode protoMode) { @@ -303,16 +303,14 @@ public static TableSchema schemaToProtoTableSchema( public static TableFieldSchema tableFieldToProtoTableField( com.google.api.services.bigquery.model.TableFieldSchema field) { TableFieldSchema.Builder builder = TableFieldSchema.newBuilder(); - builder.setName(field.getName()); + builder.setName(field.getName().toLowerCase()); if (field.getDescription() != null) { builder.setDescription(field.getDescription()); } if (field.getMaxLength() != null) { builder.setMaxLength(field.getMaxLength()); } - if (field.getMode() != null) { - builder.setMode(modeToProtoMode(field.getMode())); - } + builder.setMode(modeToProtoMode(field.getMode())); if (field.getPrecision() != null) { builder.setPrecision(field.getPrecision()); } @@ -345,7 +343,7 @@ private SchemaInformation( new SchemaInformation( field, Iterables.concat(this.parentSchemas, ImmutableList.of(this))); subFields.add(schemaInformation); - subFieldsByName.put(field.getName(), schemaInformation); + subFieldsByName.put(field.getName().toLowerCase(), schemaInformation); } } @@ -366,9 +364,9 @@ public TableFieldSchema.Type getType() { } public SchemaInformation getSchemaForField(String name) { - SchemaInformation schemaInformation = subFieldsByName.get(name); + SchemaInformation schemaInformation = subFieldsByName.get(name.toLowerCase()); if (schemaInformation == null) { - throw new RuntimeException("Schema field not found: " + name); + throw new RuntimeException("Schema field not found: " + name.toLowerCase()); } return schemaInformation; } @@ -396,7 +394,7 @@ static SchemaInformation fromTableSchema( } } - static final Map PRIMITIVE_TYPES = + static final Map PRIMITIVE_TYPES_BQ_TO_PROTO = ImmutableMap.builder() .put(TableFieldSchema.Type.INT64, Type.TYPE_INT64) .put(TableFieldSchema.Type.DOUBLE, Type.TYPE_DOUBLE) @@ -413,6 +411,26 @@ static SchemaInformation fromTableSchema( .put(TableFieldSchema.Type.JSON, Type.TYPE_STRING) .build(); + static final Map + PRIMITIVE_TYPES_PROTO_TO_BQ = + ImmutableMap.builder() + .put(Descriptors.FieldDescriptor.Type.INT32, TableFieldSchema.Type.INT64) + .put(FieldDescriptor.Type.FIXED32, TableFieldSchema.Type.INT64) + .put(FieldDescriptor.Type.UINT32, TableFieldSchema.Type.INT64) + .put(FieldDescriptor.Type.SFIXED32, TableFieldSchema.Type.INT64) + .put(FieldDescriptor.Type.SINT32, TableFieldSchema.Type.INT64) + .put(FieldDescriptor.Type.INT64, TableFieldSchema.Type.INT64) + .put(FieldDescriptor.Type.FIXED64, TableFieldSchema.Type.NUMERIC) + .put(FieldDescriptor.Type.UINT64, TableFieldSchema.Type.NUMERIC) + .put(FieldDescriptor.Type.SFIXED64, TableFieldSchema.Type.INT64) + .put(FieldDescriptor.Type.SINT64, TableFieldSchema.Type.INT64) + .put(FieldDescriptor.Type.DOUBLE, TableFieldSchema.Type.DOUBLE) + .put(FieldDescriptor.Type.FLOAT, TableFieldSchema.Type.DOUBLE) + .put(FieldDescriptor.Type.STRING, TableFieldSchema.Type.STRING) + .put(FieldDescriptor.Type.BOOL, TableFieldSchema.Type.BOOL) + .put(FieldDescriptor.Type.BYTES, TableFieldSchema.Type.BYTES) + .build(); + public static Descriptor getDescriptorFromTableSchema( com.google.api.services.bigquery.model.TableSchema jsonSchema, boolean respectRequired, @@ -429,8 +447,12 @@ public static Descriptor getDescriptorFromTableSchema( public static Descriptor getDescriptorFromTableSchema( TableSchema tableSchema, boolean respectRequired, boolean includeCdcColumns) throws DescriptorValidationException { - DescriptorProto descriptorProto = - descriptorSchemaFromTableSchema(tableSchema, respectRequired, includeCdcColumns); + return wrapDescriptorProto( + descriptorSchemaFromTableSchema(tableSchema, respectRequired, includeCdcColumns)); + } + + public static Descriptor wrapDescriptorProto(DescriptorProto descriptorProto) + throws DescriptorValidationException { FileDescriptorProto fileDescriptorProto = FileDescriptorProto.newBuilder().addMessageType(descriptorProto).build(); FileDescriptor fileDescriptor = @@ -634,6 +656,44 @@ public static DynamicMessage messageFromTableRow( } } + static TableSchema tableSchemaFromDescriptor(Descriptor descriptor) { + List tableFields = + descriptor.getFields().stream() + .map(f -> tableFieldSchemaFromDescriptorField(f)) + .collect(toList()); + return TableSchema.newBuilder().addAllFields(tableFields).build(); + } + + static TableFieldSchema tableFieldSchemaFromDescriptorField(FieldDescriptor fieldDescriptor) { + TableFieldSchema.Builder tableFieldSchemaBuilder = TableFieldSchema.newBuilder(); + tableFieldSchemaBuilder = tableFieldSchemaBuilder.setName(fieldDescriptor.getName()); + + switch (fieldDescriptor.getType()) { + case MESSAGE: + tableFieldSchemaBuilder = tableFieldSchemaBuilder.setType(TableFieldSchema.Type.STRUCT); + TableSchema nestedTableField = tableSchemaFromDescriptor(fieldDescriptor.getMessageType()); + tableFieldSchemaBuilder = + tableFieldSchemaBuilder.addAllFields(nestedTableField.getFieldsList()); + break; + default: + TableFieldSchema.Type type = PRIMITIVE_TYPES_PROTO_TO_BQ.get(fieldDescriptor.getType()); + if (type == null) { + throw new UnsupportedOperationException( + "proto type " + fieldDescriptor.getType() + " is unsupported."); + } + tableFieldSchemaBuilder = tableFieldSchemaBuilder.setType(type); + } + + if (fieldDescriptor.isRepeated()) { + tableFieldSchemaBuilder = tableFieldSchemaBuilder.setMode(TableFieldSchema.Mode.REPEATED); + } else if (fieldDescriptor.isRequired()) { + tableFieldSchemaBuilder = tableFieldSchemaBuilder.setMode(TableFieldSchema.Mode.REQUIRED); + } else { + tableFieldSchemaBuilder = tableFieldSchemaBuilder.setMode(TableFieldSchema.Mode.NULLABLE); + } + return tableFieldSchemaBuilder.build(); + } + @VisibleForTesting static DescriptorProto descriptorSchemaFromTableSchema( com.google.api.services.bigquery.model.TableSchema tableSchema, @@ -701,7 +761,7 @@ private static void fieldDescriptorFromTableField( fieldDescriptorBuilder.setType(Type.TYPE_MESSAGE).setTypeName(nested.getName()); break; default: - @Nullable Type type = PRIMITIVE_TYPES.get(fieldSchema.getType()); + @Nullable Type type = PRIMITIVE_TYPES_BQ_TO_PROTO.get(fieldSchema.getType()); if (type == null) { throw new UnsupportedOperationException( "Converting BigQuery type " + fieldSchema.getType() + " to Beam type is unsupported"); @@ -711,9 +771,9 @@ private static void fieldDescriptorFromTableField( if (fieldSchema.getMode() == TableFieldSchema.Mode.REPEATED) { fieldDescriptorBuilder = fieldDescriptorBuilder.setLabel(Label.LABEL_REPEATED); - } else if (!respectRequired || fieldSchema.getMode() == TableFieldSchema.Mode.NULLABLE) { + } else if (!respectRequired || fieldSchema.getMode() != TableFieldSchema.Mode.REQUIRED) { fieldDescriptorBuilder = fieldDescriptorBuilder.setLabel(Label.LABEL_OPTIONAL); - } else if (fieldSchema.getMode() == TableFieldSchema.Mode.REQUIRED) { + } else { fieldDescriptorBuilder = fieldDescriptorBuilder.setLabel(Label.LABEL_REQUIRED); } descriptorBuilder.addField(fieldDescriptorBuilder.build()); @@ -825,23 +885,23 @@ private static void fieldDescriptorFromTableField( try { // '2011-12-03T10:15:30Z', '2011-12-03 10:15:30+05:00' // '2011-12-03 10:15:30 UTC', '2011-12-03T10:15:30 America/New_York' - return ChronoUnit.MICROS.between( - Instant.EPOCH, Instant.from(TIMESTAMP_FORMATTER.parse((String) value))); + Instant timestamp = Instant.from(TIMESTAMP_FORMATTER.parse((String) value)); + return toEpochMicros(timestamp); } catch (DateTimeException e) { try { // for backwards compatibility, default time zone is UTC for values with no time-zone // '2011-12-03T10:15:30' - return ChronoUnit.MICROS.between( - Instant.EPOCH, - Instant.from(TIMESTAMP_FORMATTER.withZone(ZoneOffset.UTC).parse((String) value))); + Instant timestamp = + Instant.from(TIMESTAMP_FORMATTER.withZone(ZoneOffset.UTC).parse((String) value)); + return toEpochMicros(timestamp); } catch (DateTimeParseException err) { // "12345667" - return ChronoUnit.MICROS.between( - Instant.EPOCH, Instant.ofEpochMilli(Long.parseLong((String) value))); + Instant timestamp = Instant.ofEpochMilli(Long.parseLong((String) value)); + return toEpochMicros(timestamp); } } } else if (value instanceof Instant) { - return ChronoUnit.MICROS.between(Instant.EPOCH, (Instant) value); + return toEpochMicros((Instant) value); } else if (value instanceof org.joda.time.Instant) { // joda instant precision is millisecond return ((org.joda.time.Instant) value).getMillis() * 1000L; @@ -972,6 +1032,11 @@ private static void fieldDescriptorFromTableField( + schemaInformation.getType()); } + private static long toEpochMicros(Instant timestamp) { + // i.e 1970-01-01T00:01:01.000040Z: 61 * 1000_000L + 40000/1000 = 61000040 + return timestamp.getEpochSecond() * 1000_000L + timestamp.getNano() / 1000; + } + @VisibleForTesting public static TableRow tableRowFromMessage(Message message, boolean includeCdcColumns) { // TODO: Would be more correct to generate TableRows using setF. diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableSchemaCache.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableSchemaCache.java index ba6063def1c41..0013cd3d16ae1 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableSchemaCache.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableSchemaCache.java @@ -32,11 +32,11 @@ import java.util.function.Supplier; import javax.annotation.Nullable; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Monitor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Monitor.Guard; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Monitor; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Monitor.Guard; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.joda.time.Duration; import org.joda.time.Instant; import org.slf4j.Logger; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableSchemaUpdateUtils.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableSchemaUpdateUtils.java index cba394abc8904..33f47afeba0f3 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableSchemaUpdateUtils.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableSchemaUpdateUtils.java @@ -26,8 +26,8 @@ import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nullable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; /** Helper utilities for handling schema-update responses. */ public class TableSchemaUpdateUtils { @@ -78,13 +78,16 @@ private static Result getUpdatedSchema( return Result.of(newSchema, false); } + // BigQuery column names are not case-sensitive, but Map keys are. Map newSchemaMap = - newSchema.stream().collect(Collectors.toMap(TableFieldSchema::getName, x -> x)); + newSchema.stream().collect(Collectors.toMap(tr -> tr.getName().toLowerCase(), x -> x)); Set fieldNamesPopulated = Sets.newHashSet(); List updatedSchema = Lists.newArrayList(); boolean isEquivalent = oldSchema.size() == newSchema.size(); for (TableFieldSchema tableFieldSchema : oldSchema) { - @Nullable TableFieldSchema newTableFieldSchema = newSchemaMap.get(tableFieldSchema.getName()); + @Nullable + TableFieldSchema newTableFieldSchema = + newSchemaMap.get(tableFieldSchema.getName().toLowerCase()); if (newTableFieldSchema == null) { // We don't support deleting fields! return Result.empty(); diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIds.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIds.java index 7ba14d24aa542..2daac54481afe 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIds.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIds.java @@ -22,7 +22,7 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TestBigQuery.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TestBigQuery.java index 295af81cebdb9..cf5c7aa3f7a86 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TestBigQuery.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TestBigQuery.java @@ -47,7 +47,7 @@ import org.apache.beam.sdk.testing.TestPipelineOptions; import org.apache.beam.sdk.util.Preconditions; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.checkerframework.dataflow.qual.SideEffectFree; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TestBigQueryOptions.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TestBigQueryOptions.java index 3574c12ee3a99..4d8095c1879d8 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TestBigQueryOptions.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TestBigQueryOptions.java @@ -24,10 +24,17 @@ /** {@link TestPipelineOptions} for {@link TestBigQuery}. */ public interface TestBigQueryOptions extends TestPipelineOptions, BigQueryOptions, GcpOptions { + String BIGQUERY_EARLY_ROLLOUT_REGION = "us-east7"; @Description("Dataset used in the integration tests. Default is integ_test") @Default.String("integ_test") String getTargetDataset(); void setTargetDataset(String value); + + @Description("Region to perform BigQuery operations in.") + @Default.String("") + String getBigQueryLocation(); + + void setBigQueryLocation(String location); } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TwoLevelMessageConverterCache.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TwoLevelMessageConverterCache.java index cb0ea9af58ee6..5f90e1dd39505 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TwoLevelMessageConverterCache.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TwoLevelMessageConverterCache.java @@ -21,8 +21,8 @@ import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService; import org.apache.beam.sdk.io.gcp.bigquery.StorageApiDynamicDestinations.MessageConverter; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; import org.checkerframework.checker.nullness.qual.NonNull; /** diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/UpdateSchemaDestination.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/UpdateSchemaDestination.java index 4d5717388313e..51e61fe419534 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/UpdateSchemaDestination.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/UpdateSchemaDestination.java @@ -39,9 +39,9 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -271,7 +271,7 @@ private BigQueryHelpers.PendingJob startZeroLoadJob( try { destinationTable = datasetService.getTable(tableReference); if (destinationTable == null) { - return null; // no need to update schema ahead if table does not exists + return null; // no need to update schema ahead if table does not exist } } catch (IOException | InterruptedException e) { LOG.warn("Failed to get table {} with {}", tableReference, e.toString()); @@ -281,6 +281,7 @@ private BigQueryHelpers.PendingJob startZeroLoadJob( // or when destination schema is null (the write will set the schema) // or when provided schema is null (e.g. when using CREATE_NEVER disposition) if (destinationTable.getSchema() == null + || destinationTable.getSchema().isEmpty() || destinationTable.getSchema().equals(schema) || schema == null) { return null; @@ -322,7 +323,7 @@ private BigQueryHelpers.PendingJob startZeroLoadJob( jobService.startLoadJob( jobRef, loadConfig, new ByteArrayContent("text/plain", new byte[0])); } catch (IOException | InterruptedException e) { - LOG.warn("Load job {} failed with {}", jobRef, e.toString()); + LOG.warn("Schema update load job {} failed with {}", jobRef, e.toString()); throw new RuntimeException(e); } return null; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java index d3e9ebd4e9c53..894983ab664f6 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundlesToFiles.java @@ -40,8 +40,8 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.ShardedKey; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java index 03cce2631e64f..31036c58f9dcd 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java @@ -34,8 +34,8 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.ShardedKey; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; /** * Partitions temporary files based on number of files and file sizes. Output key is a pair of diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java index 0dc7e6fbcb7f2..9d798b397070c 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java @@ -39,10 +39,10 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.java index 7ea2f959ce67e..c8d46b5a0efef 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** The result of a {@link BigQueryIO.Write} transform. */ diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java index fadbca0280c33..c6a7d32e24864 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.services.bigquery.model.Clustering; import com.google.api.services.bigquery.model.EncryptionConfiguration; @@ -71,10 +71,10 @@ import org.apache.beam.sdk.values.ShardedKey; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; @@ -202,6 +202,7 @@ public void processElement( if (firstPaneCreateDisposition == CreateDisposition.CREATE_NEVER) { tableSchema = null; } else if (jsonSchemas.containsKey(destination)) { + // tableSchema for the destination stored in cache (jsonSchemas) tableSchema = BigQueryHelpers.fromJsonString(jsonSchemas.get(destination), TableSchema.class); } else { @@ -215,6 +216,7 @@ public void processElement( firstPaneCreateDisposition, dynamicDestinations, destination); + LOG.debug("Fetched TableSchema for table {}:\n\t{}", destination, tableSchema); jsonSchemas.put(destination, BigQueryHelpers.toJsonString(tableSchema)); } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryDirectReadSchemaTransformProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryDirectReadSchemaTransformProvider.java index c76108871b3d2..8b8e8179ce7db 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryDirectReadSchemaTransformProvider.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryDirectReadSchemaTransformProvider.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.gcp.bigquery.providers; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.bigquery.model.TableRow; import com.google.auto.service.AutoService; @@ -40,13 +40,12 @@ import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; /** * An implementation of {@link TypedSchemaTransformProvider} for BigQuery Storage Read API jobs @@ -162,7 +161,8 @@ public abstract static class Builder { * BigQueryDirectReadSchemaTransformConfiguration} and instantiated by {@link * BigQueryDirectReadSchemaTransformProvider}. */ - private static class BigQueryDirectReadSchemaTransform implements SchemaTransform { + protected static class BigQueryDirectReadSchemaTransform extends SchemaTransform { + private BigQueryServices testBigQueryServices = null; private final BigQueryDirectReadSchemaTransformConfiguration configuration; BigQueryDirectReadSchemaTransform( @@ -172,22 +172,6 @@ private static class BigQueryDirectReadSchemaTransform implements SchemaTransfor this.configuration = configuration; } - @Override - public PTransform buildTransform() { - return new BigQueryDirectReadPCollectionRowTupleTransform(configuration); - } - } - - static class BigQueryDirectReadPCollectionRowTupleTransform - extends PTransform { - private final BigQueryDirectReadSchemaTransformConfiguration configuration; - private BigQueryServices testBigQueryServices = null; - - BigQueryDirectReadPCollectionRowTupleTransform( - BigQueryDirectReadSchemaTransformConfiguration configuration) { - this.configuration = configuration; - } - @VisibleForTesting public void setBigQueryServices(BigQueryServices testBigQueryServices) { this.testBigQueryServices = testBigQueryServices; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProvider.java index 82d41e921fb04..1b9eb309ec46e 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProvider.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProvider.java @@ -17,11 +17,9 @@ */ package org.apache.beam.sdk.io.gcp.bigquery.providers; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; -import com.google.api.services.bigquery.model.Table; -import com.google.api.services.bigquery.model.TableReference; import com.google.auto.service.AutoService; import com.google.auto.value.AutoValue; import java.util.Arrays; @@ -34,15 +32,13 @@ import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.Method; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryOptions; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryStorageApiInsertError; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils; import org.apache.beam.sdk.io.gcp.bigquery.WriteResult; import org.apache.beam.sdk.io.gcp.bigquery.providers.BigQueryStorageWriteApiSchemaTransformProvider.BigQueryStorageWriteApiSchemaTransformConfiguration; import org.apache.beam.sdk.metrics.Counter; import org.apache.beam.sdk.metrics.Metrics; -import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.schemas.AutoValueSchema; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.Schema.Field; @@ -54,19 +50,16 @@ import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollection.IsBounded; import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Duration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * An implementation of {@link TypedSchemaTransformProvider} for BigQuery Storage Write API jobs @@ -82,8 +75,6 @@ @AutoService(SchemaTransformProvider.class) public class BigQueryStorageWriteApiSchemaTransformProvider extends TypedSchemaTransformProvider { - private static final Logger LOG = - LoggerFactory.getLogger(BigQueryStorageWriteApiSchemaTransformProvider.class); private static final Integer DEFAULT_TRIGGER_FREQUENCY_SECS = 5; private static final Duration DEFAULT_TRIGGERING_FREQUENCY = Duration.standardSeconds(DEFAULT_TRIGGER_FREQUENCY_SECS); @@ -104,7 +95,7 @@ protected SchemaTransform from( @Override public String identifier() { - return String.format("beam:schematransform:org.apache.beam:bigquery_storage_write:v1"); + return String.format("beam:schematransform:org.apache.beam:bigquery_storage_write:v2"); } @Override @@ -135,6 +126,24 @@ public abstract static class BigQueryStorageWriteApiSchemaTransformConfiguration .put(WriteDisposition.WRITE_APPEND.name(), WriteDisposition.WRITE_APPEND) .build(); + @AutoValue + public abstract static class ErrorHandling { + @SchemaFieldDescription("The name of the output PCollection containing failed writes.") + public abstract String getOutput(); + + public static Builder builder() { + return new AutoValue_BigQueryStorageWriteApiSchemaTransformProvider_BigQueryStorageWriteApiSchemaTransformConfiguration_ErrorHandling + .Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setOutput(String output); + + public abstract ErrorHandling build(); + } + } + public void validate() { String invalidConfigMessage = "Invalid BigQuery Storage Write configuration: "; @@ -161,6 +170,19 @@ public void validate() { this.getWriteDisposition(), WRITE_DISPOSITIONS.keySet()); } + + if (this.getErrorHandling() != null) { + checkArgument( + !Strings.isNullOrEmpty(this.getErrorHandling().getOutput()), + invalidConfigMessage + "Output must not be empty if error handling specified."); + } + + if (this.getAutoSharding() != null && this.getAutoSharding()) { + checkArgument( + this.getNumStreams() == 0, + invalidConfigMessage + + "Cannot set a fixed number of streams when auto-sharding is enabled. Please pick only one of the two options."); + } } /** @@ -203,11 +225,21 @@ public static Builder builder() { public abstract Boolean getUseAtLeastOnceSemantics(); @SchemaFieldDescription( - "This option enables using a dynamically determined number of shards to write to " + "This option enables using a dynamically determined number of Storage Write API streams to write to " + "BigQuery. Only applicable to unbounded data.") @Nullable public abstract Boolean getAutoSharding(); + @SchemaFieldDescription( + "Specifies the number of write streams that the Storage API sink will use. " + + "This parameter is only applicable when writing unbounded data.") + @Nullable + public abstract Integer getNumStreams(); + + @SchemaFieldDescription("This option specifies whether and where to output unwritable rows.") + @Nullable + public abstract ErrorHandling getErrorHandling(); + /** Builder for {@link BigQueryStorageWriteApiSchemaTransformConfiguration}. */ @AutoValue.Builder public abstract static class Builder { @@ -224,6 +256,10 @@ public abstract static class Builder { public abstract Builder setAutoSharding(Boolean autoSharding); + public abstract Builder setNumStreams(Integer numStreams); + + public abstract Builder setErrorHandling(ErrorHandling errorHandling); + /** Builds a {@link BigQueryStorageWriteApiSchemaTransformConfiguration} instance. */ public abstract BigQueryStorageWriteApiSchemaTransformProvider .BigQueryStorageWriteApiSchemaTransformConfiguration @@ -236,8 +272,9 @@ public abstract static class Builder { * BigQueryStorageWriteApiSchemaTransformConfiguration} and instantiated by {@link * BigQueryStorageWriteApiSchemaTransformProvider}. */ - private static class BigQueryStorageWriteApiSchemaTransform implements SchemaTransform { + protected static class BigQueryStorageWriteApiSchemaTransform extends SchemaTransform { + private BigQueryServices testBigQueryServices = null; private final BigQueryStorageWriteApiSchemaTransformConfiguration configuration; BigQueryStorageWriteApiSchemaTransform( @@ -246,23 +283,6 @@ private static class BigQueryStorageWriteApiSchemaTransform implements SchemaTra this.configuration = configuration; } - @Override - public PTransform buildTransform() { - return new BigQueryStorageWriteApiPCollectionRowTupleTransform(configuration); - } - } - - static class BigQueryStorageWriteApiPCollectionRowTupleTransform - extends PTransform { - - private final BigQueryStorageWriteApiSchemaTransformConfiguration configuration; - private BigQueryServices testBigQueryServices = null; - - BigQueryStorageWriteApiPCollectionRowTupleTransform( - BigQueryStorageWriteApiSchemaTransformConfiguration configuration) { - this.configuration = configuration; - } - @VisibleForTesting public void setBigQueryServices(BigQueryServices testBigQueryServices) { this.testBigQueryServices = testBigQueryServices; @@ -270,14 +290,14 @@ public void setBigQueryServices(BigQueryServices testBigQueryServices) { // A generic counter for PCollection of Row. Will be initialized with the given // name argument. Performs element-wise counter of the input PCollection. - private static class ElementCounterFn extends DoFn { + private static class ElementCounterFn extends DoFn { private Counter bqGenericElementCounter; private Long elementsInBundle = 0L; ElementCounterFn(String name) { this.bqGenericElementCounter = - Metrics.counter(BigQueryStorageWriteApiPCollectionRowTupleTransform.class, name); + Metrics.counter(BigQueryStorageWriteApiSchemaTransform.class, name); } @ProcessElement @@ -293,6 +313,18 @@ public void finish(FinishBundleContext c) { } } + private static class FailOnError extends DoFn { + @ProcessElement + public void process(ProcessContext c) { + throw new RuntimeException(c.element().getErrorMessage()); + } + } + + private static class NoOutputDoFn extends DoFn { + @ProcessElement + public void process(ProcessContext c) {} + } + @Override public PCollectionRowTuple expand(PCollectionRowTuple input) { // Check that the input exists @@ -304,75 +336,77 @@ public PCollectionRowTuple expand(PCollectionRowTuple input) { if (inputRows.isBounded() == IsBounded.UNBOUNDED) { Long triggeringFrequency = configuration.getTriggeringFrequencySeconds(); Boolean autoSharding = configuration.getAutoSharding(); - write = - write.withTriggeringFrequency( - (triggeringFrequency == null || triggeringFrequency <= 0) - ? DEFAULT_TRIGGERING_FREQUENCY - : Duration.standardSeconds(triggeringFrequency)); - // use default value true for autoSharding if not configured for STORAGE_WRITE_API - if (autoSharding == null || autoSharding) { + Integer numStreams = configuration.getNumStreams(); + // Triggering frequency is only applicable for exactly-once + if (!configuration.getUseAtLeastOnceSemantics()) { + write = + write.withTriggeringFrequency( + (triggeringFrequency == null || triggeringFrequency <= 0) + ? DEFAULT_TRIGGERING_FREQUENCY + : Duration.standardSeconds(triggeringFrequency)); + } + // set num streams if specified, otherwise default to autoSharding + if (numStreams > 0) { + write = write.withNumStorageWriteApiStreams(numStreams); + } else if (autoSharding == null || autoSharding) { write = write.withAutoSharding(); } } Schema inputSchema = inputRows.getSchema(); - // check if input schema is assignable to the output schema with nullability - // check disabled for field - if (write.getTable() != null) { - TableReference tableRef = write.getTable().get(); - validateSchema(input.getPipeline().getOptions(), inputSchema, tableRef); - } WriteResult result = inputRows .apply( - "element-count", ParDo.of(new ElementCounterFn("BigQuery-write-element-counter"))) + "element-count", + ParDo.of(new ElementCounterFn("BigQuery-write-element-counter"))) .setRowSchema(inputSchema) .apply(write); - Schema rowSchema = inputRows.getSchema(); - Schema errorSchema = - Schema.of( - Field.of("failed_row", FieldType.row(rowSchema)), - Field.of("error_message", FieldType.STRING)); - - // Failed rows - PCollection failedRows = + // Give something that can be followed. + PCollection postWrite = result .getFailedStorageApiInserts() - .apply( - "Construct failed rows", - MapElements.into(TypeDescriptors.rows()) - .via( - (storageError) -> - BigQueryUtils.toBeamRow(rowSchema, storageError.getRow()))) - .setRowSchema(rowSchema); - - // Failed rows with error message - PCollection failedRowsWithErrors = - result - .getFailedStorageApiInserts() - .apply( - "Construct failed rows and errors", - MapElements.into(TypeDescriptors.rows()) - .via( - (storageError) -> - Row.withSchema(errorSchema) - .withFieldValue("error_message", storageError.getErrorMessage()) - .withFieldValue( - "failed_row", - BigQueryUtils.toBeamRow(rowSchema, storageError.getRow())) - .build())) - .setRowSchema(errorSchema); - - PCollection failedRowsOutput = - failedRows - .apply("error-count", ParDo.of(new ElementCounterFn("BigQuery-write-error-counter"))) - .setRowSchema(rowSchema); - - return PCollectionRowTuple.of(FAILED_ROWS_TAG, failedRowsOutput) - .and(FAILED_ROWS_WITH_ERRORS_TAG, failedRowsWithErrors) - .and("errors", failedRowsWithErrors); + .apply("post-write", ParDo.of(new NoOutputDoFn())) + .setRowSchema(Schema.of()); + + if (configuration.getErrorHandling() == null) { + result + .getFailedStorageApiInserts() + .apply("Error on failed inserts", ParDo.of(new FailOnError())); + return PCollectionRowTuple.of("post_write", postWrite); + } else { + result + .getFailedStorageApiInserts() + .apply( + "error-count", + ParDo.of( + new ElementCounterFn( + "BigQuery-write-error-counter"))); + + // Failed rows with error message + Schema errorSchema = + Schema.of( + Field.of("failed_row", FieldType.row(inputSchema)), + Field.of("error_message", FieldType.STRING)); + PCollection failedRowsWithErrors = + result + .getFailedStorageApiInserts() + .apply( + "Construct failed rows and errors", + MapElements.into(TypeDescriptors.rows()) + .via( + (storageError) -> + Row.withSchema(errorSchema) + .withFieldValue("error_message", storageError.getErrorMessage()) + .withFieldValue( + "failed_row", + BigQueryUtils.toBeamRow(inputSchema, storageError.getRow())) + .build())) + .setRowSchema(errorSchema); + return PCollectionRowTuple.of("post_write", postWrite) + .and(configuration.getErrorHandling().getOutput(), failedRowsWithErrors); + } } BigQueryIO.Write createStorageWriteApiTransform() { @@ -409,36 +443,5 @@ BigQueryIO.Write createStorageWriteApiTransform() { return write; } - - private void validateSchema( - PipelineOptions pipelineOptions, Schema inputSchema, TableReference tableRef) { - LOG.info("Validating schema ..."); - BigQueryOptions options = pipelineOptions.as(BigQueryOptions.class); - try { - Table table = null; - if (this.testBigQueryServices != null) { - DatasetService datasetService = testBigQueryServices.getDatasetService(options); - if (datasetService != null) { - table = datasetService.getTable(tableRef); - } - } else { - table = BigQueryHelpers.getTable(options, tableRef); - } - if (table == null) { - LOG.info("Table [{}] not found, skipping schema validation.", tableRef.getTableId()); - return; - } - Schema outputSchema = BigQueryUtils.fromTableSchema(table.getSchema()); - if (!inputSchema.assignableToIgnoreNullable(outputSchema)) { - throw new IllegalArgumentException( - "Input schema is not assignable to output schema. Input schema=" - + inputSchema - + ", Output schema=" - + outputSchema); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableConfig.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableConfig.java index f1f276aa6a427..15230c8adef99 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableConfig.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableConfig.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.bigtable; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import com.google.cloud.bigtable.config.BigtableOptions; @@ -25,11 +25,12 @@ import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.extensions.gcp.auth.CredentialFactory; import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; +import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.BigtableClientOverride; import org.apache.beam.sdk.options.ValueProvider; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.display.DisplayData; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; /** Configuration for a Cloud Bigtable client. */ @@ -49,6 +50,9 @@ public abstract class BigtableConfig implements Serializable { /** Returns the app profile being read from. */ public abstract @Nullable ValueProvider getAppProfileId(); + /** Returns the Bigtable client override. */ + public abstract @Nullable BigtableClientOverride getBigtableClientOverride(); + /** * Returns the Google Cloud Bigtable instance being written to, and other parameters. * @@ -113,6 +117,8 @@ abstract Builder setBigtableOptionsConfigurator( abstract Builder setChannelCount(int count); + abstract Builder setBigtableClientOverride(BigtableClientOverride clientOverride); + abstract BigtableConfig build(); } @@ -156,6 +162,12 @@ public BigtableConfig withEmulator(String emulatorHost) { return toBuilder().setEmulatorHost(emulatorHost).build(); } + @VisibleForTesting + BigtableConfig withBigtableClientOverride(BigtableClientOverride clientOverride) { + checkArgument(clientOverride != null, "clientOverride can not be null"); + return toBuilder().setBigtableClientOverride(clientOverride).build(); + } + void validate() { checkArgument( (getProjectId() != null diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableConfigTranslator.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableConfigTranslator.java index 8585075eae03e..5100b36f23b40 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableConfigTranslator.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableConfigTranslator.java @@ -45,8 +45,8 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.options.ValueProvider; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.NonNull; import org.threeten.bp.Duration; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java index fc8b5e78e8a05..ad978e95016a7 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java @@ -19,9 +19,9 @@ import static org.apache.beam.sdk.io.gcp.bigtable.BigtableServiceFactory.BigtableServiceEntry; import static org.apache.beam.sdk.options.ValueProvider.StaticValueProvider; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import com.google.bigtable.v2.Mutation; @@ -29,6 +29,7 @@ import com.google.bigtable.v2.RowFilter; import com.google.cloud.bigtable.config.BigtableOptions; import com.google.cloud.bigtable.data.v2.models.ChangeStreamMutation; +import com.google.cloud.bigtable.data.v2.models.ChangeStreamRecord; import com.google.cloud.bigtable.data.v2.models.KeyOffset; import com.google.protobuf.ByteString; import java.io.IOException; @@ -47,13 +48,15 @@ import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.action.ActionFactory; +import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.BigtableChangeStreamAccessor; +import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.BigtableClientOverride; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.DaoFactory; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn.DetectNewPartitionsDoFn; +import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn.FilterForMutationDoFn; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn.InitializeDoFn; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn.ReadChangeStreamPartitionDoFn; -import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.BytesThroughputEstimator; -import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.SizeEstimator; +import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.CoderSizeEstimator; import org.apache.beam.sdk.io.range.ByteKey; import org.apache.beam.sdk.io.range.ByteKeyRange; import org.apache.beam.sdk.io.range.ByteKeyRangeTracker; @@ -70,12 +73,12 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.ToStringHelper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.ToStringHelper; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; @@ -309,7 +312,6 @@ public static Write write() { * *
      *
    • {@link BigtableIO.ReadChangeStream#withStartTime} which defaults to now. - *
    • {@link BigtableIO.ReadChangeStream#withHeartbeatDuration} with defaults to 1 seconds. *
    • {@link BigtableIO.ReadChangeStream#withMetadataTableProjectId} which defaults to value * from {@link BigtableIO.ReadChangeStream#withProjectId} *
    • {@link BigtableIO.ReadChangeStream#withMetadataTableInstanceId} which defaults to value @@ -455,6 +457,25 @@ public Read withTableId(String tableId) { return withTableId(StaticValueProvider.of(tableId)); } + /** + * Returns a new {@link BigtableIO.Read} that will read using the specified app profile id. + * + *

      Does not modify this object. + */ + public Read withAppProfileId(ValueProvider appProfileId) { + BigtableConfig config = getBigtableConfig(); + return toBuilder().setBigtableConfig(config.withAppProfileId(appProfileId)).build(); + } + + /** + * Returns a new {@link BigtableIO.Read} that will read using the specified app profile id. + * + *

      Does not modify this object. + */ + public Read withAppProfileId(String appProfileId) { + return withAppProfileId(StaticValueProvider.of(appProfileId)); + } + /** * WARNING: Should be used only to specify additional parameters for connection to the Cloud * Bigtable, instanceId and projectId should be provided over {@link #withInstanceId} and {@link @@ -687,14 +708,13 @@ public final String toString() { private void validateTableExists( BigtableConfig config, BigtableReadOptions readOptions, PipelineOptions options) { if (config.getValidate() && config.isDataAccessible() && readOptions.isDataAccessible()) { - String tableId = checkNotNull(readOptions.getTableId().get()); + ValueProvider tableIdProvider = checkArgumentNotNull(readOptions.getTableId()); + String tableId = checkArgumentNotNull(tableIdProvider.get()); try { - checkArgument( - getServiceFactory().checkTableExists(config, options, tableId), - "Table %s does not exist", - tableId); + boolean exists = getServiceFactory().checkTableExists(config, options, tableId); + checkArgument(exists, "Table %s does not exist", tableId); } catch (IOException e) { - LOG.warn("Error checking whether table {} exists; proceeding.", tableId, e); + throw new RuntimeException(e); } } } @@ -836,6 +856,31 @@ public Write withTableId(String tableId) { return withTableId(StaticValueProvider.of(tableId)); } + /** + * Returns a new {@link BigtableIO.Write} that will write using the specified app profile id. + * + *

      Remember that in order to use single-row transactions, this must use a single-cluster + * routing policy. + * + *

      Does not modify this object. + */ + public Write withAppProfileId(ValueProvider appProfileId) { + BigtableConfig config = getBigtableConfig(); + return toBuilder().setBigtableConfig(config.withAppProfileId(appProfileId)).build(); + } + + /** + * Returns a new {@link BigtableIO.Write} that will write using the specified app profile id. + * + *

      Remember that in order to use single-row transactions, this must use a single-cluster + * routing policy. + * + *

      Does not modify this object. + */ + public Write withAppProfileId(String appProfileId) { + return withAppProfileId(StaticValueProvider.of(appProfileId)); + } + /** * WARNING: Should be used only to specify additional parameters for connection to the Cloud * Bigtable, instanceId and projectId should be provided over {@link #withInstanceId} and {@link @@ -1120,14 +1165,13 @@ public String toString() { private void validateTableExists( BigtableConfig config, BigtableWriteOptions writeOptions, PipelineOptions options) { if (config.getValidate() && config.isDataAccessible() && writeOptions.isDataAccessible()) { - String tableId = checkNotNull(writeOptions.getTableId().get()); + ValueProvider tableIdProvider = checkArgumentNotNull(writeOptions.getTableId()); + String tableId = checkArgumentNotNull(tableIdProvider.get()); try { - checkArgument( - factory.checkTableExists(config, options, writeOptions.getTableId().get()), - "Table %s does not exist", - tableId); + boolean exists = factory.checkTableExists(config, options, tableId); + checkArgument(exists, "Table %s does not exist", tableId); } catch (IOException e) { - LOG.warn("Error checking whether table {} exists; proceeding.", tableId, e); + throw new RuntimeException(e); } } } @@ -1326,7 +1370,11 @@ public List split(long desiredBundleSizeBytes, PipelineOptions o long maximumNumberOfSplits = 4000; long sizeEstimate = getEstimatedSizeBytes(options); desiredBundleSizeBytes = - Math.max(sizeEstimate / maximumNumberOfSplits, desiredBundleSizeBytes); + Math.max( + sizeEstimate / maximumNumberOfSplits, + // BoundedReadEvaluatorFactory may provide us with a desiredBundleSizeBytes of 0 + // https://github.com/apache/beam/issues/28793 + Math.max(1, desiredBundleSizeBytes)); // Delegate to testable helper. List splits = @@ -1796,8 +1844,6 @@ public enum ExistingPipelineOptions { RESUME_OR_NEW, // Same as RESUME_OR_NEW except if previous pipeline doesn't exist, don't start. RESUME_OR_FAIL, - // Start a new pipeline. Overriding existing pipeline with the same name. - NEW, // This skips cleaning up previous pipeline metadata and starts a new pipeline. This should // only be used to skip cleanup in tests @VisibleForTesting @@ -1826,8 +1872,6 @@ static ReadChangeStream create() { abstract @Nullable Instant getEndTime(); - abstract @Nullable Duration getHeartbeatDuration(); - abstract @Nullable String getChangeStreamName(); abstract @Nullable ExistingPipelineOptions getExistingPipelineOptions(); @@ -1836,6 +1880,8 @@ static ReadChangeStream create() { abstract @Nullable String getMetadataTableId(); + abstract @Nullable Boolean getCreateOrUpdateMetadataTable(); + abstract ReadChangeStream.Builder toBuilder(); /** @@ -1908,16 +1954,6 @@ ReadChangeStream withEndTime(Instant endTime) { return toBuilder().setEndTime(endTime).build(); } - /** - * Returns a new {@link BigtableIO.ReadChangeStream} that will send heartbeat messages at - * specified interval. - * - *

      Does not modify this object. - */ - public ReadChangeStream withHeartbeatDuration(Duration interval) { - return toBuilder().setHeartbeatDuration(interval).build(); - } - /** * Returns a new {@link BigtableIO.ReadChangeStream} that uses changeStreamName as prefix for * the metadata table. @@ -1999,6 +2035,37 @@ public ReadChangeStream withMetadataTableAppProfileId(String appProfileId) { .build(); } + /** + * Returns a new {@link BigtableIO.ReadChangeStream} that overrides the config of data and/or + * admin client for streaming changes and for managing the metadata. For testing purposes only. + * Not intended for use. + * + *

      Does not modify this object. + */ + @VisibleForTesting + ReadChangeStream withBigtableClientOverride(BigtableClientOverride clientOverride) { + BigtableConfig config = getBigtableConfig(); + BigtableConfig metadataTableConfig = getMetadataTableBigtableConfig(); + return toBuilder() + .setBigtableConfig(config.withBigtableClientOverride(clientOverride)) + .setMetadataTableBigtableConfig( + metadataTableConfig.withBigtableClientOverride(clientOverride)) + .build(); + } + + /** + * Returns a new {@link BigtableIO.ReadChangeStream} that, if set to true, will create or update + * metadata table before launching pipeline. Otherwise, it is expected that a metadata table + * with correct schema exists. + * + *

      Optional: defaults to true + * + *

      Does not modify this object. + */ + public ReadChangeStream withCreateOrUpdateMetadataTable(boolean shouldCreate) { + return toBuilder().setCreateOrUpdateMetadataTable(shouldCreate).build(); + } + @Override public PCollection> expand(PBegin input) { checkArgument( @@ -2039,10 +2106,6 @@ public PCollection> expand(PBegin input) { if (startTime == null) { startTime = Instant.now(); } - Duration heartbeatDuration = getHeartbeatDuration(); - if (heartbeatDuration == null) { - heartbeatDuration = Duration.standardSeconds(1); - } String changeStreamName = getChangeStreamName(); if (changeStreamName == null || changeStreamName.isEmpty()) { changeStreamName = UniqueIdGenerator.generateRowKeyPrefix(); @@ -2052,38 +2115,70 @@ public PCollection> expand(PBegin input) { existingPipelineOptions = ExistingPipelineOptions.FAIL_IF_EXISTS; } + boolean shouldCreateOrUpdateMetadataTable = true; + if (getCreateOrUpdateMetadataTable() != null) { + shouldCreateOrUpdateMetadataTable = getCreateOrUpdateMetadataTable(); + } + ActionFactory actionFactory = new ActionFactory(); + ChangeStreamMetrics metrics = new ChangeStreamMetrics(); DaoFactory daoFactory = new DaoFactory( bigtableConfig, metadataTableConfig, getTableId(), metadataTableId, changeStreamName); - ChangeStreamMetrics metrics = new ChangeStreamMetrics(); + + try { + MetadataTableAdminDao metadataTableAdminDao = daoFactory.getMetadataTableAdminDao(); + checkArgument(metadataTableAdminDao != null); + checkArgument( + metadataTableAdminDao.isAppProfileSingleClusterAndTransactional( + metadataTableConfig.getAppProfileId().get()), + "App profile id '" + + metadataTableConfig.getAppProfileId().get() + + "' provided to access metadata table needs to use single-cluster routing policy" + + " and allow single-row transactions."); + + // Only try to create or update metadata table if option is set to true. Otherwise, just + // check if the table exists. + if (shouldCreateOrUpdateMetadataTable && metadataTableAdminDao.createMetadataTable()) { + LOG.info("Created metadata table: " + metadataTableAdminDao.getTableId()); + } + checkArgument( + metadataTableAdminDao.doesMetadataTableExist(), + "Metadata table does not exist: " + metadataTableAdminDao.getTableId()); + + try (BigtableChangeStreamAccessor bigtableChangeStreamAccessor = + BigtableChangeStreamAccessor.getOrCreate(bigtableConfig)) { + checkArgument( + bigtableChangeStreamAccessor.getTableAdminClient().exists(getTableId()), + "Change Stream table does not exist"); + } + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + daoFactory.close(); + } + InitializeDoFn initializeDoFn = - new InitializeDoFn( - daoFactory, - metadataTableConfig.getAppProfileId().get(), - startTime, - existingPipelineOptions); + new InitializeDoFn(daoFactory, startTime, existingPipelineOptions); DetectNewPartitionsDoFn detectNewPartitionsDoFn = new DetectNewPartitionsDoFn(getEndTime(), actionFactory, daoFactory, metrics); ReadChangeStreamPartitionDoFn readChangeStreamPartitionDoFn = - new ReadChangeStreamPartitionDoFn(heartbeatDuration, daoFactory, actionFactory, metrics); + new ReadChangeStreamPartitionDoFn(daoFactory, actionFactory, metrics); - PCollection> output = + PCollection> readChangeStreamOutput = input .apply(Impulse.create()) .apply("Initialize", ParDo.of(initializeDoFn)) .apply("DetectNewPartition", ParDo.of(detectNewPartitionsDoFn)) .apply("ReadChangeStreamPartition", ParDo.of(readChangeStreamPartitionDoFn)); - Coder> outputCoder = output.getCoder(); - SizeEstimator> sizeEstimator = - new SizeEstimator<>(outputCoder); - BytesThroughputEstimator> throughputEstimator = - new BytesThroughputEstimator<>( - ReadChangeStreamPartitionDoFn.THROUGHPUT_ESTIMATION_WINDOW_SECONDS, sizeEstimator); - readChangeStreamPartitionDoFn.setThroughputEstimator(throughputEstimator); + Coder> outputCoder = readChangeStreamOutput.getCoder(); + CoderSizeEstimator> sizeEstimator = + new CoderSizeEstimator<>(outputCoder); + readChangeStreamPartitionDoFn.setSizeEstimator(sizeEstimator); - return output; + return readChangeStreamOutput.apply( + "FilterForMutation", ParDo.of(new FilterForMutationDoFn())); } @AutoValue.Builder @@ -2102,14 +2197,60 @@ abstract ReadChangeStream.Builder setMetadataTableBigtableConfig( abstract ReadChangeStream.Builder setEndTime(Instant endTime); - abstract ReadChangeStream.Builder setHeartbeatDuration(Duration interval); - abstract ReadChangeStream.Builder setChangeStreamName(String changeStreamName); abstract ReadChangeStream.Builder setExistingPipelineOptions( ExistingPipelineOptions existingPipelineOptions); + abstract ReadChangeStream.Builder setCreateOrUpdateMetadataTable(boolean shouldCreate); + abstract ReadChangeStream build(); } } + + /** + * Utility method to create or update Read Change Stream metadata table. This requires Bigtable + * table create permissions. This method is useful if the pipeline isn't granted permissions to + * create Bigtable tables. Run this method with correct permissions to create the metadata table, + * which is required to read Bigtable change streams. This method only needs to be run once, and + * the metadata table can be reused for all pipelines. + * + * @param projectId project id of the metadata table, usually the same as the project of the table + * being streamed + * @param instanceId instance id of the metadata table, usually the same as the instance of the + * table being streamed + * @param tableId name of the metadata table, leave it null or empty to use default. + * @return true if the table was successfully created. Otherwise, false. + */ + public static boolean createOrUpdateReadChangeStreamMetadataTable( + String projectId, String instanceId, @Nullable String tableId) throws IOException { + BigtableConfig bigtableConfig = + BigtableConfig.builder() + .setValidate(true) + .setProjectId(StaticValueProvider.of(projectId)) + .setInstanceId(StaticValueProvider.of(instanceId)) + .setAppProfileId( + StaticValueProvider.of( + "default")) // App profile is not used. It's only required for data API. + .build(); + + if (tableId == null || tableId.isEmpty()) { + tableId = MetadataTableAdminDao.DEFAULT_METADATA_TABLE_NAME; + } + + DaoFactory daoFactory = new DaoFactory(null, bigtableConfig, null, tableId, null); + + try { + MetadataTableAdminDao metadataTableAdminDao = daoFactory.getMetadataTableAdminDao(); + + // Only try to create or update metadata table if option is set to true. Otherwise, just + // check if the table exists. + if (metadataTableAdminDao.createMetadataTable()) { + LOG.info("Created metadata table: " + metadataTableAdminDao.getTableId()); + } + return metadataTableAdminDao.doesMetadataTableExist(); + } finally { + daoFactory.close(); + } + } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadOptions.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadOptions.java index e9f2333a9695e..46834cc9756fd 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadOptions.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadOptions.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.bigtable; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import com.google.bigtable.v2.RowFilter; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadSchemaTransformProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadSchemaTransformProvider.java index 0ae02cd1c7d7f..a9f113f1ce97f 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadSchemaTransformProvider.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadSchemaTransformProvider.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.bigtable; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; import com.google.auto.value.AutoValue; @@ -38,7 +38,6 @@ import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.SimpleFunction; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionRowTuple; @@ -139,8 +138,7 @@ public abstract static class Builder { * BigtableReadSchemaTransformConfiguration} and instantiated by {@link * BigtableReadSchemaTransformProvider}. */ - private static class BigtableReadSchemaTransform - extends PTransform implements SchemaTransform { + private static class BigtableReadSchemaTransform extends SchemaTransform { private final BigtableReadSchemaTransformConfiguration configuration; BigtableReadSchemaTransform(BigtableReadSchemaTransformConfiguration configuration) { @@ -169,11 +167,6 @@ public PCollectionRowTuple expand(PCollectionRowTuple input) { return PCollectionRowTuple.of(OUTPUT_TAG, beamRows); } - - @Override - public PTransform buildTransform() { - return this; - } } public static class BigtableRowToBeamRow extends SimpleFunction { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceFactory.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceFactory.java index a9db4943c657c..635856430fb3e 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceFactory.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceFactory.java @@ -75,20 +75,19 @@ static BigtableServiceEntry create(ConfigId configId, BigtableService service) { @Override public void close() { - int refCount = - refCounts.getOrDefault(getConfigId().id(), new AtomicInteger(0)).decrementAndGet(); - if (refCount < 0) { - LOG.error( - "close() Ref count is < 0, configId=" + getConfigId().id() + " refCount=" + refCount); - } - LOG.debug("close() for config id " + getConfigId().id() + ", ref count is " + refCount); - if (refCount == 0) { - synchronized (lock) { - if (refCounts.get(getConfigId().id()).get() <= 0) { - entries.remove(getConfigId().id()); - refCounts.remove(getConfigId().id()); - getService().close(); - } + synchronized (lock) { + int refCount = + refCounts.getOrDefault(getConfigId().id(), new AtomicInteger(0)).decrementAndGet(); + if (refCount < 0) { + LOG.error( + "close() Ref count is < 0, configId=" + getConfigId().id() + " refCount=" + refCount); + } + LOG.debug( + "close() is called for config id " + getConfigId().id() + ", ref count is " + refCount); + if (refCount == 0) { + entries.remove(getConfigId().id()); + refCounts.remove(getConfigId().id()); + getService().close(); } } } @@ -105,8 +104,12 @@ BigtableServiceEntry getServiceForReading( BigtableServiceEntry entry = entries.get(configId.id()); if (entry != null) { // When entry is not null, refCount.get(configId.id()) should always exist. - // Do a getOrDefault to avoid unexpected NPEs. - refCounts.getOrDefault(configId.id(), new AtomicInteger(0)).getAndIncrement(); + // Doing a putIfAbsent to avoid NPE. + AtomicInteger count = refCounts.putIfAbsent(configId.id(), new AtomicInteger(0)); + if (count == null) { + LOG.error("entry is not null but refCount of config Id " + configId.id() + " is null."); + } + refCounts.get(configId.id()).getAndIncrement(); LOG.debug("getServiceForReading() returning an existing service entry"); return entry; } @@ -150,8 +153,12 @@ BigtableServiceEntry getServiceForWriting( LOG.debug("getServiceForWriting(), config id: " + configId.id()); if (entry != null) { // When entry is not null, refCount.get(configId.id()) should always exist. - // Do a getOrDefault to avoid unexpected NPEs. - refCounts.getOrDefault(configId.id(), new AtomicInteger(0)).getAndIncrement(); + // Doing a putIfAbsent to avoid NPE. + AtomicInteger count = refCounts.putIfAbsent(configId.id(), new AtomicInteger(0)); + if (count == null) { + LOG.error("entry is not null but refCount of config Id " + configId.id() + " is null."); + } + refCounts.get(configId.id()).getAndIncrement(); LOG.debug("getServiceForWriting() returning an existing service entry"); return entry; } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java index 540a3b5e1ac3d..31a317352beb1 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.bigtable; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors.directExecutor; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors.directExecutor; import com.google.api.gax.batching.Batcher; import com.google.api.gax.grpc.GrpcCallContext; @@ -69,14 +69,14 @@ import org.apache.beam.sdk.metrics.Distribution; import org.apache.beam.sdk.metrics.Metrics; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Stopwatch; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ComparisonChain; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.FutureCallback; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Futures; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.SettableFuture; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Stopwatch; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ComparisonChain; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.FutureCallback; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Futures; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.SettableFuture; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; @@ -337,7 +337,8 @@ static BigtableSegmentReaderImpl create( this.serviceCallMetric = serviceCallMetric; this.buffer = new ArrayDeque<>(); // Asynchronously refill buffer when there is 10% of the elements are left - this.refillSegmentWaterMark = (int) (request.getRowsLimit() * WATERMARK_PERCENTAGE); + this.refillSegmentWaterMark = + Math.max(1, (int) (request.getRowsLimit() * WATERMARK_PERCENTAGE)); this.attemptTimeout = attemptTimeout; this.operationTimeout = operationTimeout; } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteOptions.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteOptions.java index 1aae8eab25250..05e0e915f12a7 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteOptions.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteOptions.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.bigtable; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.io.Serializable; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteSchemaTransformProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteSchemaTransformProvider.java new file mode 100644 index 0000000000000..b99b69621a84d --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteSchemaTransformProvider.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.gcp.bigtable; + +import static java.util.Optional.ofNullable; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import com.google.auto.service.AutoService; +import com.google.auto.value.AutoValue; +import com.google.bigtable.v2.Mutation; +import com.google.bigtable.v2.TimestampRange; +import com.google.protobuf.ByteString; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.apache.beam.sdk.io.gcp.bigtable.BigtableWriteSchemaTransformProvider.BigtableWriteSchemaTransformConfiguration; +import org.apache.beam.sdk.schemas.AutoValueSchema; +import org.apache.beam.sdk.schemas.annotations.DefaultSchema; +import org.apache.beam.sdk.schemas.transforms.SchemaTransform; +import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; +import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.SimpleFunction; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionRowTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Longs; + +/** + * An implementation of {@link TypedSchemaTransformProvider} for Bigtable Write jobs configured via + * {@link BigtableWriteSchemaTransformConfiguration}. + * + *

      Internal only: This class is actively being worked on, and it will likely change. We + * provide no backwards compatibility guarantees, and it should not be implemented outside the Beam + * repository. + */ +@AutoService(SchemaTransformProvider.class) +public class BigtableWriteSchemaTransformProvider + extends TypedSchemaTransformProvider { + + private static final String INPUT_TAG = "input"; + + @Override + protected Class configurationClass() { + return BigtableWriteSchemaTransformConfiguration.class; + } + + @Override + protected SchemaTransform from(BigtableWriteSchemaTransformConfiguration configuration) { + return new BigtableWriteSchemaTransform(configuration); + } + + @Override + public String identifier() { + return "beam:schematransform:org.apache.beam:bigtable_write:v1"; + } + + @Override + public List inputCollectionNames() { + return Collections.singletonList(INPUT_TAG); + } + + @Override + public List outputCollectionNames() { + return Collections.emptyList(); + } + + /** Configuration for writing to Bigtable. */ + @DefaultSchema(AutoValueSchema.class) + @AutoValue + public abstract static class BigtableWriteSchemaTransformConfiguration { + /** Instantiates a {@link BigtableWriteSchemaTransformConfiguration.Builder} instance. */ + public static Builder builder() { + return new AutoValue_BigtableWriteSchemaTransformProvider_BigtableWriteSchemaTransformConfiguration + .Builder(); + } + + /** Validates the configuration object. */ + public void validate() { + String invalidConfigMessage = + "Invalid Bigtable Write configuration: %s should be a non-empty String"; + checkArgument(!this.getTableId().isEmpty(), String.format(invalidConfigMessage, "table")); + checkArgument( + !this.getInstanceId().isEmpty(), String.format(invalidConfigMessage, "instance")); + checkArgument(!this.getProjectId().isEmpty(), String.format(invalidConfigMessage, "project")); + } + + public abstract String getTableId(); + + public abstract String getInstanceId(); + + public abstract String getProjectId(); + + /** Builder for the {@link BigtableWriteSchemaTransformConfiguration}. */ + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setTableId(String tableId); + + public abstract Builder setInstanceId(String instanceId); + + public abstract Builder setProjectId(String projectId); + + /** Builds a {@link BigtableWriteSchemaTransformConfiguration} instance. */ + public abstract BigtableWriteSchemaTransformConfiguration build(); + } + } + + /** + * A {@link SchemaTransform} for Bigtable writes, configured with {@link + * BigtableWriteSchemaTransformConfiguration} and instantiated by {@link + * BigtableWriteSchemaTransformProvider}. + */ + private static class BigtableWriteSchemaTransform extends SchemaTransform { + private final BigtableWriteSchemaTransformConfiguration configuration; + + BigtableWriteSchemaTransform(BigtableWriteSchemaTransformConfiguration configuration) { + configuration.validate(); + this.configuration = configuration; + } + + @Override + public PCollectionRowTuple expand(PCollectionRowTuple input) { + checkArgument( + input.has(INPUT_TAG), + String.format( + "Could not find expected input [%s] to %s.", INPUT_TAG, getClass().getSimpleName())); + + PCollection beamRowMutations = input.get(INPUT_TAG); + PCollection>> bigtableMutations = + beamRowMutations.apply(MapElements.via(new GetMutationsFromBeamRow())); + + bigtableMutations.apply( + BigtableIO.write() + .withTableId(configuration.getTableId()) + .withInstanceId(configuration.getInstanceId()) + .withProjectId(configuration.getProjectId())); + + return PCollectionRowTuple.empty(input.getPipeline()); + } + } + + public static class GetMutationsFromBeamRow + extends SimpleFunction>> { + @Override + public KV> apply(Row row) { + ByteString key = ByteString.copyFrom(ofNullable(row.getBytes("key")).get()); + List> beamRowMutations = + (List) ofNullable(row.getArray("mutations")).get(); + + List mutations = new ArrayList<>(beamRowMutations.size()); + + for (Map mutation : beamRowMutations) { + Mutation bigtableMutation; + switch (new String(ofNullable(mutation.get("type")).get(), StandardCharsets.UTF_8)) { + case "SetCell": + Mutation.SetCell.Builder setMutation = + Mutation.SetCell.newBuilder() + .setValue(ByteString.copyFrom(ofNullable(mutation.get("value")).get())) + .setColumnQualifier( + ByteString.copyFrom(ofNullable(mutation.get("column_qualifier")).get())) + .setFamilyNameBytes( + ByteString.copyFrom(ofNullable(mutation.get("family_name")).get())) + // Use timestamp if provided, else default to -1 (current Bigtable server time) + .setTimestampMicros( + mutation.containsKey("timestamp_micros") + ? Longs.fromByteArray( + ofNullable(mutation.get("timestamp_micros")).get()) + : -1); + bigtableMutation = Mutation.newBuilder().setSetCell(setMutation.build()).build(); + break; + case "DeleteFromColumn": + Mutation.DeleteFromColumn.Builder deleteMutation = + Mutation.DeleteFromColumn.newBuilder() + .setColumnQualifier( + ByteString.copyFrom(ofNullable(mutation.get("column_qualifier")).get())) + .setFamilyNameBytes( + ByteString.copyFrom(ofNullable(mutation.get("family_name")).get())); + + // set timestamp range if applicable + if (mutation.containsKey("start_timestamp_micros") + || mutation.containsKey("end_timestamp_micros")) { + TimestampRange.Builder timeRange = TimestampRange.newBuilder(); + if (mutation.containsKey("start_timestamp_micros")) { + Long startMicros = + ByteBuffer.wrap(ofNullable(mutation.get("start_timestamp_micros")).get()) + .getLong(); + timeRange.setStartTimestampMicros(startMicros); + } + if (mutation.containsKey("end_timestamp_micros")) { + Long endMicros = + ByteBuffer.wrap(ofNullable(mutation.get("end_timestamp_micros")).get()) + .getLong(); + timeRange.setEndTimestampMicros(endMicros); + } + deleteMutation.setTimeRange(timeRange.build()); + } + bigtableMutation = + Mutation.newBuilder().setDeleteFromColumn(deleteMutation.build()).build(); + break; + case "DeleteFromFamily": + bigtableMutation = + Mutation.newBuilder() + .setDeleteFromFamily( + Mutation.DeleteFromFamily.newBuilder() + .setFamilyNameBytes( + ByteString.copyFrom(ofNullable(mutation.get("family_name")).get())) + .build()) + .build(); + break; + case "DeleteFromRow": + bigtableMutation = + Mutation.newBuilder() + .setDeleteFromRow(Mutation.DeleteFromRow.newBuilder().build()) + .build(); + break; + default: + throw new RuntimeException( + String.format( + "Unexpected mutation type [%s]: %s", + Arrays.toString(ofNullable(mutation.get("type")).get()), mutation)); + } + mutations.add(bigtableMutation); + } + return KV.of(key, mutations); + } + } +} diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/CellValueParser.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/CellValueParser.java index 3b1ff015d7e78..331a7cd311ee9 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/CellValueParser.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/CellValueParser.java @@ -19,7 +19,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.bigtable.v2.Cell; import com.google.protobuf.ByteString; @@ -27,9 +27,9 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.Schema.LogicalType; import org.apache.beam.sdk.schemas.logicaltypes.PassThroughLogicalType; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Ints; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Shorts; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Ints; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Longs; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Shorts; import org.joda.time.DateTime; class CellValueParser implements Serializable { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/VendoredListenableFutureAdapter.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/VendoredListenableFutureAdapter.java index e675438584209..46a696d49295d 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/VendoredListenableFutureAdapter.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/VendoredListenableFutureAdapter.java @@ -22,7 +22,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListenableFuture; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListenableFuture; /** Adapts {@link ListenableFuture} from bigtable-client-core to vendored guava. */ class VendoredListenableFutureAdapter implements ListenableFuture { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelper.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelper.java index 900ddae570e36..6402762338d6f 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelper.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelper.java @@ -26,7 +26,7 @@ import java.util.List; import java.util.stream.Collectors; import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** Helper functions to evaluate the completeness of collection of ByteStringRanges. */ @Internal diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ActionFactory.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ActionFactory.java index 43623b5de160c..6aabcb081ad4e 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ActionFactory.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ActionFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.bigtable.changestreams.action; -import com.google.cloud.bigtable.data.v2.models.ChangeStreamMutation; +import com.google.cloud.bigtable.data.v2.models.ChangeStreamRecord; import com.google.protobuf.ByteString; import java.io.Serializable; import javax.annotation.Nullable; @@ -25,7 +25,7 @@ import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.ChangeStreamDao; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao; -import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.ThroughputEstimator; +import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.SizeEstimator; import org.apache.beam.sdk.values.KV; import org.joda.time.Duration; import org.joda.time.Instant; @@ -56,11 +56,9 @@ public class ActionFactory implements Serializable { * * @return singleton instance of the {@link ChangeStreamAction} */ - public synchronized ChangeStreamAction changeStreamAction( - ChangeStreamMetrics metrics, - ThroughputEstimator> throughputEstimator) { + public synchronized ChangeStreamAction changeStreamAction(ChangeStreamMetrics metrics) { if (changeStreamAction == null) { - changeStreamAction = new ChangeStreamAction(metrics, throughputEstimator); + changeStreamAction = new ChangeStreamAction(metrics); } return changeStreamAction; } @@ -145,11 +143,17 @@ public synchronized ReadChangeStreamPartitionAction readChangeStreamPartitionAct ChangeStreamDao changeStreamDao, ChangeStreamMetrics metrics, ChangeStreamAction changeStreamAction, - Duration heartbeatDuration) { + Duration heartbeatDuration, + SizeEstimator> sizeEstimator) { if (readChangeStreamPartitionAction == null) { readChangeStreamPartitionAction = new ReadChangeStreamPartitionAction( - metadataTableDao, changeStreamDao, metrics, changeStreamAction, heartbeatDuration); + metadataTableDao, + changeStreamDao, + metrics, + changeStreamAction, + heartbeatDuration, + sizeEstimator); } return readChangeStreamPartitionAction; } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamAction.java index 4c3f9c5a83353..adb817a4c08a0 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamAction.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamAction.java @@ -28,10 +28,9 @@ import com.google.cloud.bigtable.data.v2.models.Range; import com.google.protobuf.ByteString; import java.util.Optional; -import java.util.stream.Collectors; import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics; -import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.ThroughputEstimator; +import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.BytesThroughputEstimator; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.StreamProgress; import org.apache.beam.sdk.transforms.DoFn; @@ -46,20 +45,15 @@ @Internal public class ChangeStreamAction { private static final Logger LOG = LoggerFactory.getLogger(ChangeStreamAction.class); - private final ChangeStreamMetrics metrics; - private final ThroughputEstimator> throughputEstimator; /** * Constructs ChangeStreamAction to process individual ChangeStreamRecord. * * @param metrics record beam metrics. */ - public ChangeStreamAction( - ChangeStreamMetrics metrics, - ThroughputEstimator> throughputEstimator) { + public ChangeStreamAction(ChangeStreamMetrics metrics) { this.metrics = metrics; - this.throughputEstimator = throughputEstimator; } /** @@ -111,65 +105,54 @@ public Optional run( PartitionRecord partitionRecord, ChangeStreamRecord record, RestrictionTracker tracker, - DoFn.OutputReceiver> receiver, + DoFn.OutputReceiver> receiver, ManualWatermarkEstimator watermarkEstimator, - boolean shouldDebug) { + BytesThroughputEstimator> throughputEstimator) { if (record instanceof Heartbeat) { Heartbeat heartbeat = (Heartbeat) record; final Instant watermark = toJodaTime(heartbeat.getEstimatedLowWatermark()); + + // These will be filtered so the key doesn't really matter but the most logical thing to + // key a heartbeat by is the partition it corresponds to. + ByteString heartbeatKey = + Range.ByteStringRange.serializeToByteString(partitionRecord.getPartition()); + KV outputRecord = KV.of(heartbeatKey, heartbeat); + throughputEstimator.update(Instant.now(), outputRecord); StreamProgress streamProgress = - new StreamProgress(heartbeat.getChangeStreamContinuationToken(), watermark); + new StreamProgress( + heartbeat.getChangeStreamContinuationToken(), + watermark, + throughputEstimator.get(), + Instant.now(), + true); watermarkEstimator.setWatermark(watermark); - if (shouldDebug) { - LOG.info( - "RCSP {}: Heartbeat partition: {} token: {} watermark: {}", - formatByteStringRange(partitionRecord.getPartition()), - formatByteStringRange(heartbeat.getChangeStreamContinuationToken().getPartition()), - heartbeat.getChangeStreamContinuationToken().getToken(), - heartbeat.getEstimatedLowWatermark()); - } // If the tracker fail to claim the streamProgress, it most likely means the runner initiated // a checkpoint. See {@link // org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.ReadChangeStreamPartitionProgressTracker} // for more information regarding runner initiated checkpoints. if (!tracker.tryClaim(streamProgress)) { - if (shouldDebug) { - LOG.info( - "RCSP {}: Checkpoint heart beat tracker", - formatByteStringRange(partitionRecord.getPartition())); - } return Optional.of(DoFn.ProcessContinuation.stop()); } metrics.incHeartbeatCount(); + // We output heartbeats so that they are factored into throughput and can be used to + // autoscale. These will be filtered in a downstream step and never returned to users. This is + // to prevent autoscaler from scaling down when we have large tables with no throughput but + // we need enough workers to keep up with heartbeats. + + // We are outputting elements with timestamp of 0 to prevent reliance on event time. This + // limits the ability to window on commit time of any data changes. It is still possible to + // window on processing time. + receiver.outputWithTimestamp(outputRecord, Instant.EPOCH); } else if (record instanceof CloseStream) { CloseStream closeStream = (CloseStream) record; StreamProgress streamProgress = new StreamProgress(closeStream); - if (shouldDebug) { - LOG.info( - "RCSP {}: CloseStream: {}", - formatByteStringRange(partitionRecord.getPartition()), - closeStream.getChangeStreamContinuationTokens().stream() - .map( - c -> - "{partition: " - + formatByteStringRange(c.getPartition()) - + " token: " - + c.getToken() - + "}") - .collect(Collectors.joining(", ", "[", "]"))); - } // If the tracker fail to claim the streamProgress, it most likely means the runner initiated // a checkpoint. See {@link // org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.ReadChangeStreamPartitionProgressTracker} // for more information regarding runner initiated checkpoints. if (!tracker.tryClaim(streamProgress)) { - if (shouldDebug) { - LOG.info( - "RCSP {}: Checkpoint close stream tracker", - formatByteStringRange(partitionRecord.getPartition())); - } return Optional.of(DoFn.ProcessContinuation.stop()); } metrics.incClosestreamCount(); @@ -185,16 +168,21 @@ public Optional run( partitionRecord.getPartition().getStart(), partitionRecord.getPartition().getEnd()), changeStreamMutation.getToken()); - StreamProgress streamProgress = new StreamProgress(changeStreamContinuationToken, watermark); + + KV outputRecord = + KV.of(changeStreamMutation.getRowKey(), changeStreamMutation); + throughputEstimator.update(Instant.now(), outputRecord); + StreamProgress streamProgress = + new StreamProgress( + changeStreamContinuationToken, + watermark, + throughputEstimator.get(), + Instant.now(), + false); // If the tracker fail to claim the streamProgress, it most likely means the runner initiated // a checkpoint. See ReadChangeStreamPartitionProgressTracker for more information regarding // runner initiated checkpoints. if (!tracker.tryClaim(streamProgress)) { - if (shouldDebug) { - LOG.info( - "RCSP {}: Checkpoint data change tracker", - formatByteStringRange(partitionRecord.getPartition())); - } return Optional.of(DoFn.ProcessContinuation.stop()); } if (changeStreamMutation.getType() == ChangeStreamMutation.MutationType.GARBAGE_COLLECTION) { @@ -206,9 +194,6 @@ public Optional run( metrics.updateProcessingDelayFromCommitTimestamp( Instant.now().getMillis() - delay.getMillis()); - KV outputRecord = - KV.of(changeStreamMutation.getRowKey(), changeStreamMutation); - throughputEstimator.update(Instant.now(), outputRecord); // We are outputting elements with timestamp of 0 to prevent reliance on event time. This // limits the ability to window on commit time of any data changes. It is still possible to // window on processing time. diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsAction.java index d444f6a2dfd9d..8354b7efc43d6 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsAction.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsAction.java @@ -45,7 +45,7 @@ import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation; import org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.Duration; import org.joda.time.Instant; import org.slf4j.Logger; @@ -314,7 +314,10 @@ public ProcessContinuation run( for (NewPartition newPartition : newPartitions) { if (processNewPartitionsAction.processNewPartition(newPartition, receiver)) { outputtedNewPartitions.add(newPartition.getPartition()); - } else { + } else if (streamPartitionsWithWatermark != null) { + // streamPartitionsWithWatermark is not null on runs that we update watermark. We only run + // reconciliation when we update watermark. Only add incompleteNewPartitions if + // reconciliation is being run partitionReconciler.addIncompleteNewPartitions(newPartition); orphanedMetadataCleaner.addIncompleteNewPartitions(newPartition); } @@ -325,26 +328,26 @@ public ProcessContinuation run( Optional maybeWatermark = getNewWatermark(streamPartitionsWithWatermark, newPartitions); maybeWatermark.ifPresent(metadataTableDao::updateDetectNewPartitionWatermark); - // Using NewPartitions and StreamPartitions, evaluate partitions that are possibly not being - // streamed. This isn't perfect because there may be partitions moving between - // StreamPartitions and NewPartitions while scanning the metadata table. Also, this does not - // include NewPartitions marked as deleted from a previous DNP run not yet processed by RCSP. - List existingPartitions = - streamPartitionsWithWatermark.stream() - .map(StreamPartitionWithWatermark::getPartition) - .collect(Collectors.toList()); - existingPartitions.addAll(outputtedNewPartitions); - List missingStreamPartitions = - getMissingPartitionsFromEntireKeySpace(existingPartitions); - orphanedMetadataCleaner.addMissingPartitions(missingStreamPartitions); - partitionReconciler.addMissingPartitions(missingStreamPartitions); - } - - // Only start reconciling after the pipeline has been running for a while. - if (tracker.currentRestriction().getFrom() > 50) { - processReconcilerPartitions( - receiver, watermarkEstimator, initialPipelineState.getStartTime()); - cleanUpOrphanedMetadata(); + // Only start reconciling after the pipeline has been running for a while. + if (tracker.currentRestriction().getFrom() > 50) { + // Using NewPartitions and StreamPartitions, evaluate partitions that are possibly not being + // streamed. This isn't perfect because there may be partitions moving between + // StreamPartitions and NewPartitions while scanning the metadata table. Also, this does not + // include NewPartitions marked as deleted from a previous DNP run not yet processed by + // RCSP. + List existingPartitions = + streamPartitionsWithWatermark.stream() + .map(StreamPartitionWithWatermark::getPartition) + .collect(Collectors.toList()); + existingPartitions.addAll(outputtedNewPartitions); + List missingStreamPartitions = + getMissingPartitionsFromEntireKeySpace(existingPartitions); + orphanedMetadataCleaner.addMissingPartitions(missingStreamPartitions); + partitionReconciler.addMissingPartitions(missingStreamPartitions); + processReconcilerPartitions( + receiver, watermarkEstimator, initialPipelineState.getStartTime()); + cleanUpOrphanedMetadata(); + } } return ProcessContinuation.resume().withResumeDelay(Duration.millis(100)); diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionAction.java index 2270cbd08ac13..8638339ac5c02 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionAction.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionAction.java @@ -25,7 +25,6 @@ import com.google.api.gax.rpc.ServerStream; import com.google.cloud.bigtable.common.Status; import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken; -import com.google.cloud.bigtable.data.v2.models.ChangeStreamMutation; import com.google.cloud.bigtable.data.v2.models.ChangeStreamRecord; import com.google.cloud.bigtable.data.v2.models.CloseStream; import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange; @@ -40,6 +39,8 @@ import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.ChangeStreamDao; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn.DetectNewPartitionsDoFn; +import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.BytesThroughputEstimator; +import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.SizeEstimator; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.StreamProgress; @@ -66,18 +67,21 @@ public class ReadChangeStreamPartitionAction { private final ChangeStreamMetrics metrics; private final ChangeStreamAction changeStreamAction; private final Duration heartbeatDuration; + private final SizeEstimator> sizeEstimator; public ReadChangeStreamPartitionAction( MetadataTableDao metadataTableDao, ChangeStreamDao changeStreamDao, ChangeStreamMetrics metrics, ChangeStreamAction changeStreamAction, - Duration heartbeatDuration) { + Duration heartbeatDuration, + SizeEstimator> sizeEstimator) { this.metadataTableDao = metadataTableDao; this.changeStreamDao = changeStreamDao; this.metrics = metrics; this.changeStreamAction = changeStreamAction; this.heartbeatDuration = heartbeatDuration; + this.sizeEstimator = sizeEstimator; } /** @@ -131,24 +135,11 @@ public ReadChangeStreamPartitionAction( public ProcessContinuation run( PartitionRecord partitionRecord, RestrictionTracker tracker, - OutputReceiver> receiver, + OutputReceiver> receiver, ManualWatermarkEstimator watermarkEstimator) throws IOException { - // Watermark being delayed beyond 5 minutes signals a possible problem. - boolean shouldDebug = - watermarkEstimator.getState().plus(Duration.standardMinutes(5)).isBeforeNow(); - - if (shouldDebug) { - LOG.info( - "RCSP {}: Partition: " - + partitionRecord - + "\n Watermark: " - + watermarkEstimator.getState() - + "\n RestrictionTracker: " - + tracker.currentRestriction(), - formatByteStringRange(partitionRecord.getPartition())); - } - + BytesThroughputEstimator> throughputEstimator = + new BytesThroughputEstimator<>(sizeEstimator, Instant.now()); // Lock the partition if (tracker.currentRestriction().isEmpty()) { boolean lockedPartition = metadataTableDao.lockAndRecordPartition(partitionRecord); @@ -260,12 +251,10 @@ public ProcessContinuation run( new NewPartition( childPartition, Collections.singletonList(token), watermarkEstimator.getState())); } - if (shouldDebug) { - LOG.info( - "RCSP {}: Split/Merge into {}", - formatByteStringRange(partitionRecord.getPartition()), - partitionsToString(childPartitions)); - } + LOG.info( + "RCSP {}: Split/Merge into {}", + formatByteStringRange(partitionRecord.getPartition()), + partitionsToString(childPartitions)); if (!coverSameKeySpace(tokenPartitions, partitionRecord.getPartition())) { LOG.warn( "RCSP {}: CloseStream has tokens {} that don't cover the entire keyspace", @@ -293,12 +282,16 @@ public ProcessContinuation run( partitionRecord, tracker.currentRestriction(), partitionRecord.getEndTime(), - heartbeatDuration, - shouldDebug); + heartbeatDuration); for (ChangeStreamRecord record : stream) { Optional result = changeStreamAction.run( - partitionRecord, record, tracker, receiver, watermarkEstimator, shouldDebug); + partitionRecord, + record, + tracker, + receiver, + watermarkEstimator, + throughputEstimator); // changeStreamAction will usually return Optional.empty() except for when a checkpoint // (either runner or pipeline initiated) is required. if (result.isPresent()) { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/BigtableChangeStreamAccessor.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/BigtableChangeStreamAccessor.java index 69c6efba1de81..cb296aef6c281 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/BigtableChangeStreamAccessor.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/BigtableChangeStreamAccessor.java @@ -47,7 +47,8 @@ * backend and the jobs on the same machine shares the same sets of connections. */ @Internal -class BigtableChangeStreamAccessor { +public class BigtableChangeStreamAccessor implements AutoCloseable { + static final Duration MUTATE_ROW_DEADLINE = Duration.ofSeconds(30); // Create one bigtable data/admin client per bigtable config (project/instance/table/app profile) private static final ConcurrentHashMap bigtableAccessors = new ConcurrentHashMap<>(); @@ -55,14 +56,31 @@ class BigtableChangeStreamAccessor { private final BigtableDataClient dataClient; private final BigtableTableAdminClient tableAdminClient; private final BigtableInstanceAdminClient instanceAdminClient; + private final BigtableConfig bigtableConfig; private BigtableChangeStreamAccessor( BigtableDataClient dataClient, BigtableTableAdminClient tableAdminClient, - BigtableInstanceAdminClient instanceAdminClient) { + BigtableInstanceAdminClient instanceAdminClient, + BigtableConfig bigtableConfig) { this.dataClient = dataClient; this.tableAdminClient = tableAdminClient; this.instanceAdminClient = instanceAdminClient; + this.bigtableConfig = bigtableConfig; + } + + @Override + public synchronized void close() { + if (dataClient != null) { + dataClient.close(); + } + if (tableAdminClient != null) { + tableAdminClient.close(); + } + if (instanceAdminClient != null) { + instanceAdminClient.close(); + } + bigtableAccessors.remove(bigtableConfig); } /** @@ -146,9 +164,10 @@ private static BigtableChangeStreamAccessor createAccessor(@NonNull BigtableConf .readRowsSettings() .setRetrySettings( readRowsRetrySettings - .setInitialRpcTimeout(Duration.ofSeconds(30)) - .setTotalTimeout(Duration.ofSeconds(30)) - .setMaxRpcTimeout(Duration.ofSeconds(30)) + // metadata table scans can get quite large, so use a higher deadline + .setInitialRpcTimeout(Duration.ofMinutes(3)) + .setTotalTimeout(Duration.ofMinutes(3)) + .setMaxRpcTimeout(Duration.ofMinutes(3)) .setMaxAttempts(10) .build()); @@ -159,9 +178,9 @@ private static BigtableChangeStreamAccessor createAccessor(@NonNull BigtableConf .mutateRowSettings() .setRetrySettings( mutateRowRetrySettings - .setInitialRpcTimeout(Duration.ofSeconds(30)) - .setTotalTimeout(Duration.ofSeconds(30)) - .setMaxRpcTimeout(Duration.ofSeconds(30)) + .setInitialRpcTimeout(MUTATE_ROW_DEADLINE) + .setTotalTimeout(MUTATE_ROW_DEADLINE) + .setMaxRpcTimeout(MUTATE_ROW_DEADLINE) .setMaxAttempts(10) .build()); @@ -185,18 +204,26 @@ private static BigtableChangeStreamAccessor createAccessor(@NonNull BigtableConf .readChangeStreamSettings() .setRetrySettings( readChangeStreamRetrySettings - .setInitialRpcTimeout(Duration.ofSeconds(60)) - .setTotalTimeout(Duration.ofSeconds(60)) - .setMaxRpcTimeout(Duration.ofSeconds(60)) + .setInitialRpcTimeout(Duration.ofSeconds(15)) + .setTotalTimeout(Duration.ofSeconds(15)) + .setMaxRpcTimeout(Duration.ofSeconds(15)) .setMaxAttempts(10) .build()); + final BigtableClientOverride clientOverride = bigtableConfig.getBigtableClientOverride(); + if (clientOverride != null) { + clientOverride.updateTableAdminClientSettings(tableAdminSettingsBuilder); + clientOverride.updateInstanceAdminClientSettings(instanceAdminSettingsBuilder); + clientOverride.updateDataClientSettings(dataSettingsBuilder); + } + BigtableDataClient dataClient = BigtableDataClient.create(dataSettingsBuilder.build()); BigtableTableAdminClient tableAdminClient = BigtableTableAdminClient.create(tableAdminSettingsBuilder.build()); BigtableInstanceAdminClient instanceAdminClient = BigtableInstanceAdminClient.create(instanceAdminSettingsBuilder.build()); - return new BigtableChangeStreamAccessor(dataClient, tableAdminClient, instanceAdminClient); + return new BigtableChangeStreamAccessor( + dataClient, tableAdminClient, instanceAdminClient, bigtableConfig); } public BigtableDataClient getDataClient() { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/BigtableClientOverride.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/BigtableClientOverride.java new file mode 100644 index 0000000000000..72b3e39871efe --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/BigtableClientOverride.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao; + +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminSettings; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import java.io.IOException; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; + +/** Override the configuration of Cloud Bigtable data and admin client. */ +@VisibleForTesting +public interface BigtableClientOverride { + /** + * Update {@link BigtableInstanceAdminSettings.Builder} with custom configurations. + * + *

      For example, to update the admin api endpoint. + * + * @param builder builds the instance admin client + * @throws IOException when dependency initialization fails + */ + void updateInstanceAdminClientSettings(BigtableInstanceAdminSettings.Builder builder) + throws IOException; + + /** + * Update {@link BigtableTableAdminSettings.Builder} with custom configurations. + * + *

      For example, to update the admin api endpoint. + * + * @param builder builds the table admin client + * @throws IOException when dependency initialization fails + */ + void updateTableAdminClientSettings(BigtableTableAdminSettings.Builder builder) + throws IOException; + + /** + * Update {@link BigtableDataSettings.Builder} with custom configurations. + * + * @param builder builds the data client + * @throws IOException when dependency initialization fails + */ + void updateDataClientSettings(BigtableDataSettings.Builder builder) throws IOException; +} diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/ChangeStreamDao.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/ChangeStreamDao.java index 1c24e09db7da9..5974af22f8632 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/ChangeStreamDao.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/ChangeStreamDao.java @@ -17,7 +17,6 @@ */ package org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao; -import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.formatByteStringRange; import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.TimestampConverter.toThreetenInstant; import com.google.api.gax.rpc.ServerStream; @@ -36,14 +35,10 @@ import org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.StreamProgress; import org.joda.time.Duration; import org.joda.time.Instant; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** Data access object to list and read stream partitions of a table. */ @Internal public class ChangeStreamDao { - private static final Logger LOG = LoggerFactory.getLogger(ChangeStreamDao.class); - private final BigtableDataClient dataClient; private final String tableId; @@ -75,8 +70,7 @@ public ServerStream readChangeStreamPartition( PartitionRecord partition, StreamProgress streamProgress, @Nullable Instant endTime, - Duration heartbeatDuration, - boolean shouldDebug) + Duration heartbeatDuration) throws IOException { ReadChangeStreamQuery query = ReadChangeStreamQuery.create(tableId).streamPartition(partition.getPartition()); @@ -98,12 +92,6 @@ public ServerStream readChangeStreamPartition( query.endTime(TimestampConverter.toThreetenInstant(endTime)); } query.heartbeatDuration(org.threeten.bp.Duration.ofMillis(heartbeatDuration.getMillis())); - if (shouldDebug) { - LOG.info( - "RCSP {} ReadChangeStreamRequest: {}", - formatByteStringRange(partition.getPartition()), - query); - } return dataClient.readChangeStream(query); } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/DaoFactory.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/DaoFactory.java index acecc22b8919a..35ac8ed29c3ed 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/DaoFactory.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/DaoFactory.java @@ -30,7 +30,7 @@ // Allows transient fields to be intialized later @SuppressWarnings("initialization.fields.uninitialized") @Internal -public class DaoFactory implements Serializable { +public class DaoFactory implements Serializable, AutoCloseable { private static final long serialVersionUID = 3732208768248394205L; private transient ChangeStreamDao changeStreamDao; @@ -58,6 +58,19 @@ public DaoFactory( this.metadataTableId = metadataTableId; } + @Override + public void close() { + try { + if (metadataTableAdminDao != null || metadataTableDao != null) { + BigtableChangeStreamAccessor.getOrCreate(metadataTableConfig).close(); + } + if (changeStreamDao != null) { + BigtableChangeStreamAccessor.getOrCreate(changeStreamConfig).close(); + } + } catch (Exception ignored) { + } + } + public String getChangeStreamName() { return changeStreamName; } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableAdminDao.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableAdminDao.java index 5194ca49d62e8..ecf56b69e000a 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableAdminDao.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableAdminDao.java @@ -29,7 +29,7 @@ import com.google.protobuf.ByteString; import java.util.List; import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * Data access object for creating and dropping the metadata table. @@ -128,6 +128,11 @@ public boolean isAppProfileSingleClusterAndTransactional(String appProfileId) { return false; } + /** @return true if metadata table exists, otherwise false. */ + public boolean doesMetadataTableExist() { + return tableAdminClient.exists(tableId); + } + /** * Create the metadata table if it does not exist yet. If the table does exist, verify all the * column families exists, if not add those column families. This table only need to be created diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDao.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDao.java index d4938e6bef26f..ea7e4f14d0570 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDao.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDao.java @@ -30,6 +30,7 @@ import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder.parseWatermarkFromRow; import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder.parseWatermarkLastUpdatedFromRow; +import com.google.api.core.ApiFuture; import com.google.api.gax.rpc.ServerStream; import com.google.cloud.bigtable.data.v2.BigtableDataClient; import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken; @@ -50,6 +51,9 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.apache.beam.repackaged.core.org.apache.commons.lang3.SerializationException; @@ -60,7 +64,8 @@ import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.StreamPartitionWithWatermark; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Longs; import org.joda.time.Duration; import org.joda.time.Instant; import org.slf4j.Logger; @@ -319,7 +324,7 @@ public void writeNewPartition(NewPartition newPartition) { .deleteCells( MetadataTableAdminDao.CF_SHOULD_DELETE, ByteStringRange.serializeToByteString(parentPartition)); - dataClient.mutateRow(rowMutation); + mutateRowWithHardTimeout(rowMutation); } /** @@ -352,7 +357,7 @@ public void markNewPartitionForDeletion(NewPartition newPartition) { ByteStringRange.serializeToByteString(token.getPartition()), 1); } - dataClient.mutateRow(rowMutation); + mutateRowWithHardTimeout(rowMutation); } /** @@ -511,7 +516,7 @@ private void writeToMdTableWatermarkHelper( MetadataTableAdminDao.QUALIFIER_DEFAULT, currentToken.getToken()); } - dataClient.mutateRow(rowMutation); + mutateRowWithHardTimeout(rowMutation); } /** @@ -705,6 +710,10 @@ tableId, convertPartitionToStreamPartitionRowKey(partitionRecord.getPartition()) boolean lockAcquired = !dataClient.checkAndMutateRow(rowMutation); if (lockAcquired) { + LOG.info( + "RCSP: {} acquired lock for uid: {}", + formatByteStringRange(partitionRecord.getPartition()), + partitionRecord.getUuid()); return true; } else { // If the lock is already held we need to check if it was acquired by a duplicate @@ -724,7 +733,7 @@ public void writeDetectNewPartitionVersion() { MetadataTableAdminDao.CF_VERSION, MetadataTableAdminDao.QUALIFIER_DEFAULT, MetadataTableAdminDao.CURRENT_METADATA_TABLE_VERSION); - dataClient.mutateRow(rowMutation); + mutateRowWithHardTimeout(rowMutation); } /** @@ -778,6 +787,34 @@ public void writeDetectNewPartitionMissingPartitions( MetadataTableAdminDao.CF_MISSING_PARTITIONS, ByteString.copyFromUtf8(MetadataTableAdminDao.QUALIFIER_DEFAULT), ByteString.copyFrom(serializedMissingPartition)); - dataClient.mutateRow(rowMutation); + mutateRowWithHardTimeout(rowMutation); + } + + /** + * This adds a hard timeout of 40 seconds to mutate row futures. These requests already have a + * 30-second deadline. This is a workaround for an extremely rare issue we see with requests not + * respecting their deadlines. This can be removed once we've pinpointed the cause. + * + * @param rowMutation Bigtable RowMutation to apply + */ + @VisibleForTesting + void mutateRowWithHardTimeout(RowMutation rowMutation) { + ApiFuture mutateRowFuture = dataClient.mutateRowAsync(rowMutation); + try { + mutateRowFuture.get( + BigtableChangeStreamAccessor.MUTATE_ROW_DEADLINE.getSeconds() + 10, TimeUnit.SECONDS); + } catch (TimeoutException timeoutException) { + mutateRowFuture.cancel(true); + throw new RuntimeException( + "Cancelled mutateRow request after exceeding deadline", timeoutException); + } catch (ExecutionException executionException) { + if (executionException.getCause() instanceof RuntimeException) { + throw (RuntimeException) executionException.getCause(); + } + throw new RuntimeException(executionException); + } catch (InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + throw new RuntimeException(interruptedException); + } } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/FilterForMutationDoFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/FilterForMutationDoFn.java new file mode 100644 index 0000000000000..968e2ecddaa65 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/FilterForMutationDoFn.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn; + +import com.google.cloud.bigtable.data.v2.models.ChangeStreamMutation; +import com.google.cloud.bigtable.data.v2.models.ChangeStreamRecord; +import com.google.protobuf.ByteString; +import java.io.Serializable; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.values.KV; + +public class FilterForMutationDoFn + extends DoFn, KV> + implements Serializable { + + @ProcessElement + public void processElement( + @Element KV changeStreamRecordKV, + OutputReceiver> receiver) { + ChangeStreamRecord inputRecord = changeStreamRecordKV.getValue(); + if (inputRecord instanceof ChangeStreamMutation) { + receiver.output(KV.of(changeStreamRecordKV.getKey(), (ChangeStreamMutation) inputRecord)); + } + } +} diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFn.java index 07826ee1db73a..c8ee12772cc0c 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFn.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFn.java @@ -36,20 +36,14 @@ @Internal public class InitializeDoFn extends DoFn implements Serializable { private static final long serialVersionUID = 1868189906451252363L; - private static final Logger LOG = LoggerFactory.getLogger(InitializeDoFn.class); private final DaoFactory daoFactory; - private final String metadataTableAppProfileId; private Instant startTime; private final ExistingPipelineOptions existingPipelineOptions; public InitializeDoFn( - DaoFactory daoFactory, - String metadataTableAppProfileId, - Instant startTime, - ExistingPipelineOptions existingPipelineOptions) { + DaoFactory daoFactory, Instant startTime, ExistingPipelineOptions existingPipelineOptions) { this.daoFactory = daoFactory; - this.metadataTableAppProfileId = metadataTableAppProfileId; this.startTime = startTime; this.existingPipelineOptions = existingPipelineOptions; } @@ -59,35 +53,12 @@ public void processElement(OutputReceiver receiver) throws LOG.info(daoFactory.getStreamTableDebugString()); LOG.info(daoFactory.getMetadataTableDebugString()); LOG.info("ChangeStreamName: " + daoFactory.getChangeStreamName()); - if (!daoFactory - .getMetadataTableAdminDao() - .isAppProfileSingleClusterAndTransactional(this.metadataTableAppProfileId)) { - LOG.error( - "App profile id '" - + metadataTableAppProfileId - + "' provided to access metadata table needs to use single-cluster routing policy" - + " and allow single-row transactions."); - // Terminate this pipeline now. - return; - } - if (daoFactory.getMetadataTableAdminDao().createMetadataTable()) { - LOG.info("Created metadata table: " + daoFactory.getMetadataTableAdminDao().getTableId()); - } else { - LOG.info( - "Reusing existing metadata table: " + daoFactory.getMetadataTableAdminDao().getTableId()); - } boolean resume = false; DetectNewPartitionsState detectNewPartitionsState = daoFactory.getMetadataTableDao().readDetectNewPartitionsState(); switch (existingPipelineOptions) { - case NEW: - // clean up table - LOG.info( - "Cleaning up an old pipeline with the same change stream name to start a new pipeline with the same name."); - daoFactory.getMetadataTableAdminDao().cleanUpPrefix(); - break; case RESUME_OR_NEW: // perform resumption. if (detectNewPartitionsState != null) { @@ -98,7 +69,6 @@ public void processElement(OutputReceiver receiver) throws LOG.info( "Attempted to resume, but previous watermark does not exist, starting at {}", startTime); - daoFactory.getMetadataTableAdminDao().cleanUpPrefix(); } break; case RESUME_OR_FAIL: @@ -118,9 +88,6 @@ public void processElement(OutputReceiver receiver) throws "A previous pipeline exists with the same change stream name and existingPipelineOption is set to FAIL_IF_EXISTS."); return; } - // We still want to clean up any existing prefixes in case there are lingering metadata that - // would interfere with the new run. - daoFactory.getMetadataTableAdminDao().cleanUpPrefix(); break; case SKIP_CLEANUP: if (detectNewPartitionsState != null) { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/ReadChangeStreamPartitionDoFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/ReadChangeStreamPartitionDoFn.java index 9fcf9e12968f5..92590a7e4b899 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/ReadChangeStreamPartitionDoFn.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/ReadChangeStreamPartitionDoFn.java @@ -17,11 +17,13 @@ */ package org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn; -import com.google.cloud.bigtable.data.v2.models.ChangeStreamMutation; +import com.google.cloud.bigtable.data.v2.models.ChangeStreamRecord; import com.google.protobuf.ByteString; import java.io.IOException; import java.math.BigDecimal; +import java.math.RoundingMode; import org.apache.beam.sdk.annotations.Internal; +import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.action.ActionFactory; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.action.ChangeStreamAction; @@ -29,9 +31,9 @@ import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.ChangeStreamDao; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.DaoFactory; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao; -import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.BytesThroughputEstimator; -import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.NullThroughputEstimator; -import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.ThroughputEstimator; +import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.CoderSizeEstimator; +import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.NullSizeEstimator; +import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.SizeEstimator; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.ReadChangeStreamPartitionProgressTracker; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.StreamProgress; @@ -41,40 +43,35 @@ import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker; import org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimators.Manual; import org.apache.beam.sdk.values.KV; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.joda.time.Duration; import org.joda.time.Instant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; // Allows for readChangeStreamPartitionAction setup -@SuppressWarnings("initialization.fields.uninitialized") +@SuppressWarnings({"initialization.fields.uninitialized", "dereference.of.nullable"}) @Internal @UnboundedPerElement public class ReadChangeStreamPartitionDoFn - extends DoFn> { + extends DoFn> { private static final long serialVersionUID = 4418739381635104479L; private static final BigDecimal MAX_DOUBLE = BigDecimal.valueOf(Double.MAX_VALUE); private static final Logger LOG = LoggerFactory.getLogger(ReadChangeStreamPartitionDoFn.class); - public static final int THROUGHPUT_ESTIMATION_WINDOW_SECONDS = 10; + private static final Duration HEARTBEAT_DURATION = Duration.standardSeconds(1); - private final Duration heartbeatDuration; private final DaoFactory daoFactory; private final ChangeStreamMetrics metrics; private final ActionFactory actionFactory; - private ThroughputEstimator> throughputEstimator; - + private SizeEstimator> sizeEstimator; private ReadChangeStreamPartitionAction readChangeStreamPartitionAction; public ReadChangeStreamPartitionDoFn( - Duration heartbeatDuration, - DaoFactory daoFactory, - ActionFactory actionFactory, - ChangeStreamMetrics metrics) { - this.heartbeatDuration = heartbeatDuration; + DaoFactory daoFactory, ActionFactory actionFactory, ChangeStreamMetrics metrics) { this.daoFactory = daoFactory; this.metrics = metrics; this.actionFactory = actionFactory; - this.throughputEstimator = new NullThroughputEstimator<>(); + this.sizeEstimator = new NullSizeEstimator<>(); } @GetInitialWatermarkEstimatorState @@ -106,30 +103,55 @@ public double getSize(@Restriction StreamProgress streamProgress) { return 0d; } Instant lowWatermark = streamProgress.getEstimatedLowWatermark(); + BigDecimal estimatedThroughput = streamProgress.getThroughputEstimate(); + Instant lastRunTimestamp = streamProgress.getLastRunTimestamp(); // This should only be null if: // 1) We've failed to lock for the partition in which case we expect 0 throughput // 2) We've received a CloseStream in which case we won't process any more data for // this partition // 3) RCSP has just started and hasn't completed a checkpoint yet, in which case we can't // estimate throughput yet - if (lowWatermark == null) { + if (lowWatermark == null || estimatedThroughput == null || lastRunTimestamp == null) { return 0; } + String partition = ""; + if (streamProgress.getCurrentToken() != null) { + partition = + ByteStringRangeHelper.formatByteStringRange( + Preconditions.checkNotNull(streamProgress.getCurrentToken()).getPartition()); + } + + // Heartbeat lowWatermark takes up to a minute to update on the server. We don't want + // this to count against the backlog and prevent scaling down, so we estimate heartbeat backlog + // using the time we most recently processed a heartbeat. Otherwise, (for mutations) we use the + // watermark. + Duration processingTimeLag = + Duration.millis( + Instant.now().getMillis() - streamProgress.getLastRunTimestamp().getMillis()); Duration watermarkLag = Duration.millis(Instant.now().getMillis() - lowWatermark.getMillis()); - BigDecimal estimatedThroughput = BigDecimal.valueOf(throughputEstimator.get()); + long lagInMillis = + (streamProgress.isHeartbeat() ? processingTimeLag : watermarkLag).getMillis(); // Return the estimated bytes per second throughput multiplied by the amount of known work // outstanding (watermark lag). Cap at max double to avoid overflow. - final double estimatedSize = + double estimatedSize = estimatedThroughput - .multiply(BigDecimal.valueOf(watermarkLag.getStandardSeconds())) + .multiply(BigDecimal.valueOf(lagInMillis)) + .divide(BigDecimal.valueOf(1000), 3, RoundingMode.DOWN) .min(MAX_DOUBLE) + // Lag can be negative from clock skew. We treat that as caught up, so + // it should return zero. + .max(BigDecimal.ZERO) .doubleValue(); + LOG.debug( - "Estimated size: throughputBytes: {} x watermarkLag {} = {}", + "Estimated size (per second): partition: {}, isHeartbeat: {}, throughputBytes: {} x watermarkLagMillis {} = {}, lastRun = {}", + partition, + streamProgress.isHeartbeat(), estimatedThroughput, - watermarkLag, - estimatedSize); + lagInMillis, + estimatedSize, + streamProgress.getLastRunTimestamp()); return estimatedSize; } @@ -137,18 +159,22 @@ public double getSize(@Restriction StreamProgress streamProgress) { public void setup() throws IOException { MetadataTableDao metadataTableDao = daoFactory.getMetadataTableDao(); ChangeStreamDao changeStreamDao = daoFactory.getChangeStreamDao(); - ChangeStreamAction changeStreamAction = - actionFactory.changeStreamAction(this.metrics, this.throughputEstimator); + ChangeStreamAction changeStreamAction = actionFactory.changeStreamAction(this.metrics); readChangeStreamPartitionAction = actionFactory.readChangeStreamPartitionAction( - metadataTableDao, changeStreamDao, metrics, changeStreamAction, heartbeatDuration); + metadataTableDao, + changeStreamDao, + metrics, + changeStreamAction, + HEARTBEAT_DURATION, + sizeEstimator); } @ProcessElement public ProcessContinuation processElement( @Element PartitionRecord partitionRecord, RestrictionTracker tracker, - OutputReceiver> receiver, + OutputReceiver> receiver, ManualWatermarkEstimator watermarkEstimator) throws InterruptedException, IOException { return readChangeStreamPartitionAction.run( @@ -158,10 +184,10 @@ public ProcessContinuation processElement( /** * Sets the estimator to track throughput for each DoFn instance. * - * @param throughputEstimator an estimator to calculate DoFn instance level throughput + * @param sizeEstimator an estimator to calculate the size of records for throughput estimates */ - public void setThroughputEstimator( - BytesThroughputEstimator> throughputEstimator) { - this.throughputEstimator = throughputEstimator; + public void setSizeEstimator( + CoderSizeEstimator> sizeEstimator) { + this.sizeEstimator = sizeEstimator; } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/encoder/MetadataTableEncoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/encoder/MetadataTableEncoder.java index 03cae41c5bd76..fc1d1af0e3cd6 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/encoder/MetadataTableEncoder.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/encoder/MetadataTableEncoder.java @@ -27,7 +27,7 @@ import java.util.List; import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Longs; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/BytesThroughputEstimator.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/BytesThroughputEstimator.java index d9fa19e87e1cb..33278a704cb5b 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/BytesThroughputEstimator.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/BytesThroughputEstimator.java @@ -17,15 +17,12 @@ */ package org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator; -import java.io.Serializable; import java.math.BigDecimal; -import java.math.MathContext; -import java.util.ArrayDeque; -import java.util.Deque; -import java.util.Random; +import java.math.RoundingMode; import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.sdk.io.gcp.bigtable.changestreams.TimestampConverter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Duration; import org.joda.time.Instant; /** @@ -42,64 +39,36 @@ * every element */ @Internal -public class BytesThroughputEstimator implements ThroughputEstimator { +public class BytesThroughputEstimator { - private static final long serialVersionUID = -1147130541208370666L; - private static final BigDecimal MAX_DOUBLE = BigDecimal.valueOf(Double.MAX_VALUE); + private static final long serialVersionUID = 6014227751984587954L; private static final int DEFAULT_SAMPLE_RATE = 50; - - /** Keeps track of how many bytes of throughput have been seen in a given timestamp. */ - private static class ThroughputEntry implements Serializable { - - private static final long serialVersionUID = 3752325891215855332L; - - private final Instant instant; - private BigDecimal bytes; - - public ThroughputEntry(Instant instant, long bytes) { - this.instant = instant; - this.bytes = BigDecimal.valueOf(bytes); - } - - public Instant getTimestamp() { - return instant; - } - - public long getSeconds() { - return TimestampConverter.toSeconds(instant); - } - - public BigDecimal getBytes() { - return bytes; - } - - public void addBytes(long bytesToAdd) { - bytes = bytes.add(BigDecimal.valueOf(bytesToAdd)); - } - } - - // The deque holds a number of windows in the past in order to calculate - // a rolling windowing throughput. - private final Deque deque; - // The number of seconds to be accounted for when calculating the throughput - private final int windowSizeSeconds; - // Estimates the size in bytes of throughput elements private final SizeEstimator sizeEstimator; private final int sampleRate; - private final Random random; + private long elementCount; + private BigDecimal currentElementSizeEstimate; + private final Instant startTimestamp; + private Instant lastElementTimestamp; + private BigDecimal totalThroughputEstimate; - public BytesThroughputEstimator(int windowSizeSeconds, SizeEstimator sizeEstimator) { - this(windowSizeSeconds, sizeEstimator, DEFAULT_SAMPLE_RATE); + public BytesThroughputEstimator( + SizeEstimator sizeEstimator, @Nullable Instant lastRunTimestamp) { + this( + sizeEstimator, + DEFAULT_SAMPLE_RATE, + lastRunTimestamp != null ? lastRunTimestamp : Instant.now()); } @VisibleForTesting public BytesThroughputEstimator( - int windowSizeSeconds, SizeEstimator sizeEstimator, int sampleRate) { - this.deque = new ArrayDeque<>(); - this.windowSizeSeconds = windowSizeSeconds; + SizeEstimator sizeEstimator, int sampleRate, Instant startTimestamp) { this.sizeEstimator = sizeEstimator; this.sampleRate = sampleRate; - this.random = new Random(); + this.startTimestamp = startTimestamp; + this.elementCount = 0; + this.currentElementSizeEstimate = BigDecimal.ZERO; + lastElementTimestamp = this.startTimestamp; + totalThroughputEstimate = BigDecimal.ZERO; } /** @@ -108,66 +77,31 @@ public BytesThroughputEstimator( * @param timeOfRecords the committed timestamp of the records * @param element the element to estimate the byte size of */ - @SuppressWarnings("nullness") // queue is never null, nor the peeked element - @Override public void update(Instant timeOfRecords, T element) { - if (random.nextInt(sampleRate) == 0) { - long bytes = sizeEstimator.sizeOf(element); - synchronized (deque) { - if (deque.isEmpty() - || TimestampConverter.toSeconds(timeOfRecords) > deque.getLast().getSeconds()) { - deque.addLast(new ThroughputEntry(timeOfRecords, bytes)); - } else { - deque.getLast().addBytes(bytes); - } - cleanQueue(deque.getLast().getTimestamp()); - } - } - } - - /** Returns the estimated throughput bytes for now. */ - @Override - public double get() { - return getFrom(Instant.now()); - } - - /** - * Returns the estimated throughput bytes for a specified time. - * - * @param time the specified timestamp to check throughput - */ - @Override - public double getFrom(Instant time) { - synchronized (deque) { - cleanQueue(time); - if (deque.size() == 0) { - return 0D; - } - BigDecimal throughput = BigDecimal.ZERO; - for (ThroughputEntry entry : deque) { - throughput = throughput.add(entry.getBytes()); - } - return throughput - // Prevents negative values - .max(BigDecimal.ZERO) - .divide(BigDecimal.valueOf(windowSizeSeconds), MathContext.DECIMAL128) - .multiply(BigDecimal.valueOf(sampleRate)) - // Cap it to Double.MAX_VALUE - .min(MAX_DOUBLE) - .doubleValue(); + // Always updates on first element re-estimates size based on sample rate. + // This is expensive so we avoid doing it too often. + if (elementCount % sampleRate == 0) { + currentElementSizeEstimate = BigDecimal.valueOf(sizeEstimator.sizeOf(element)); } + lastElementTimestamp = timeOfRecords; + elementCount += 1; + totalThroughputEstimate = totalThroughputEstimate.add(currentElementSizeEstimate); } - private void cleanQueue(Instant time) { - while (deque.size() > 0) { - final ThroughputEntry entry = deque.getFirst(); - if (entry != null - && entry.getSeconds() >= TimestampConverter.toSeconds(time) - windowSizeSeconds) { - break; - } - // Remove the element if the timestamp of the first element is beyond - // the time range to look backward. - deque.removeFirst(); + /** Returns the estimated throughput bytes for this run. */ + public BigDecimal get() { + if (elementCount == 0) { + return BigDecimal.ZERO; + } else { + BigDecimal processingTimeMillis = + BigDecimal.valueOf(new Duration(startTimestamp, lastElementTimestamp).getMillis()) + // Avoid divide by zero by rounding up to 1 ms when the difference is less + // than a full millisecond + .max(BigDecimal.ONE); + + return totalThroughputEstimate + .divide(processingTimeMillis, 3, RoundingMode.DOWN) + .multiply(BigDecimal.valueOf(1000)); } } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/CoderSizeEstimator.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/CoderSizeEstimator.java new file mode 100644 index 0000000000000..528bbd7baf88b --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/CoderSizeEstimator.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator; + +import java.io.Serializable; +import org.apache.beam.sdk.annotations.Internal; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.util.common.ElementByteSizeObserver; + +/** + * This class is used to estimate the size in bytes of a given element. It uses the given {@link + * Coder} to calculate the size of the element. + */ +@Internal +public class CoderSizeEstimator implements SizeEstimator, Serializable { + + private static final long serialVersionUID = 5564948506493524158L; + + private static class SizeEstimatorObserver extends ElementByteSizeObserver + implements Serializable { + + private static final long serialVersionUID = 4569562919962045617L; + private long observedBytes; + + @Override + protected void reportElementSize(long elementByteSize) { + observedBytes = elementByteSize; + } + } + + private final Coder coder; + private final SizeEstimatorObserver observer; + + public CoderSizeEstimator(Coder coder) { + this.coder = coder; + this.observer = new SizeEstimatorObserver(); + } + + /** + * Estimates the size in bytes of the given element with the configured {@link Coder} . + * + * @param element the element instance to be estimated + * @return the estimated size in bytes of the given element + */ + @Override + public long sizeOf(T element) { + try { + coder.registerByteSizeObserver(element, observer); + observer.advance(); + + return observer.observedBytes; + } catch (Exception e) { + throw new EncodingException(e); + } + } +} diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/NullSizeEstimator.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/NullSizeEstimator.java new file mode 100644 index 0000000000000..3af627e12ec12 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/NullSizeEstimator.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator; + +import org.apache.beam.sdk.annotations.Internal; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * NoOp implementation of a size estimator. This will always return 0 as the size and it will warn + * users that this is being used (it should not be used in production). + */ +@Internal +public class NullSizeEstimator implements SizeEstimator { + + private static final long serialVersionUID = 7088120208289907630L; + private static final Logger LOG = LoggerFactory.getLogger(NullSizeEstimator.class); + + @Override + public long sizeOf(T element) { + LOG.warn( + "Trying to estimate size using {}, this operation will always return 0", + this.getClass().getSimpleName()); + return 0; + } +} diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/NullThroughputEstimator.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/NullThroughputEstimator.java deleted file mode 100644 index 98b5761feda72..0000000000000 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/NullThroughputEstimator.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator; - -import org.apache.beam.sdk.annotations.Internal; -import org.joda.time.Instant; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * NoOp implementation of a throughput estimator. This will always return 0 as the throughput and it - * will warn users that this is being used (it should not be used in production). - */ -@Internal -public class NullThroughputEstimator implements ThroughputEstimator { - - private static final long serialVersionUID = 7088120208289907630L; - private static final Logger LOG = LoggerFactory.getLogger(NullThroughputEstimator.class); - - /** - * NoOp. - * - * @param timeOfRecords ignored - * @param element ignored - */ - @Override - public void update(Instant timeOfRecords, T element) { - LOG.warn( - "Trying to update throughput using {}, this operation will have no effect", - this.getClass().getSimpleName()); - } - - /** - * Always returns 0. - * - * @param time ignored - * @return 0 - */ - @Override - public double getFrom(Instant time) { - LOG.warn( - "Trying to retrieve throughput using {}, this operation will always return 0", - this.getClass().getSimpleName()); - return 0; - } -} diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/SizeEstimator.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/SizeEstimator.java index 30d6245059653..b79013f41af2f 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/SizeEstimator.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/SizeEstimator.java @@ -17,54 +17,6 @@ */ package org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator; -import java.io.Serializable; -import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.sdk.coders.Coder; -import org.apache.beam.sdk.util.common.ElementByteSizeObserver; - -/** - * This class is used to estimate the size in bytes of a given element. It uses the given {@link - * Coder} to calculate the size of the element. - */ -@Internal -public class SizeEstimator implements Serializable { - - private static final long serialVersionUID = 5564948506493524158L; - - private static class SizeEstimatorObserver extends ElementByteSizeObserver - implements Serializable { - - private static final long serialVersionUID = 4569562919962045617L; - private long observedBytes; - - @Override - protected void reportElementSize(long elementByteSize) { - observedBytes = elementByteSize; - } - } - - private final Coder coder; - private final SizeEstimatorObserver observer; - - public SizeEstimator(Coder coder) { - this.coder = coder; - this.observer = new SizeEstimatorObserver(); - } - - /** - * Estimates the size in bytes of the given element with the configured {@link Coder} . - * - * @param element the element instance to be estimated - * @return the estimated size in bytes of the given element - */ - public long sizeOf(T element) { - try { - coder.registerByteSizeObserver(element, observer); - observer.advance(); - - return observer.observedBytes; - } catch (Exception e) { - throw new EncodingException(e); - } - } +public interface SizeEstimator { + long sizeOf(T element); } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/ThroughputEstimator.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/ThroughputEstimator.java deleted file mode 100644 index aeedb31ed134a..0000000000000 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/ThroughputEstimator.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator; - -import java.io.Serializable; -import org.apache.beam.sdk.annotations.Internal; -import org.joda.time.Instant; - -/** An estimator to calculate the throughput of the outputted elements from a DoFn. */ -@Internal -public interface ThroughputEstimator extends Serializable { - /** - * Updates the estimator with the size of the records. - * - * @param timeOfRecords the committed timestamp of the records - * @param element the element to estimate the byte size of - */ - void update(Instant timeOfRecords, T element); - - /** Returns the estimated throughput for now. */ - default double get() { - return getFrom(Instant.now()); - } - - /** - * Returns the estimated throughput for a specified time. - * - * @param time the specified timestamp to check throughput - */ - double getFrom(Instant time); -} diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconciler.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconciler.java index 41e07081d69b4..abac10821215c 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconciler.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconciler.java @@ -72,10 +72,13 @@ public class PartitionReconciler { private final List newPartitions = new ArrayList<>(); private final MetadataTableDao metadataTableDao; private final ChangeStreamMetrics metrics; + // Ensure that we added the missing partitions before writing the missing partitions to the + // metadata table. + private boolean hasAddedMissingPartitions = false; // The amount of delay allowed before we consider a partition to be probably missing. - private static final Duration MISSING_PARTITION_SHORT_DELAY = Duration.standardMinutes(1); - private static final Duration MISSING_PARTITION_LONG_DELAY = Duration.standardMinutes(10); + private static final Duration MISSING_PARTITION_SHORT_DELAY = Duration.standardMinutes(2); + private static final Duration MISSING_PARTITION_LONG_DELAY = Duration.standardMinutes(20); public PartitionReconciler(MetadataTableDao metadataTableDao, ChangeStreamMetrics metrics) { this.metadataTableDao = metadataTableDao; @@ -100,6 +103,7 @@ public PartitionReconciler(MetadataTableDao metadataTableDao, ChangeStreamMetric * @param missingPartitions partitions not being streamed. */ public void addMissingPartitions(List missingPartitions) { + hasAddedMissingPartitions = true; HashMap alreadyMissingPartitionDurations = metadataTableDao.readDetectNewPartitionMissingPartitions(); missingPartitionDurations = new HashMap<>(); @@ -156,6 +160,8 @@ private List findOverlappingNewPartitions(ByteStringRange missingP * For missing partitions, try to organize the mismatched parent tokens in a way to fill the * missing partitions. * + *

      Must call {@link #addMissingPartitions(List)} before this. + * *

      If there are parent tokens that when combined form a missing partition, it can be outputted * as a merge of the missing partition. * @@ -169,6 +175,14 @@ private List findOverlappingNewPartitions(ByteStringRange missingP * @return reconciled PartitionRecord. */ public List getPartitionsToReconcile(Instant lowWatermark, Instant startTime) { + // We update the metadata table with the partitions that are still missing after reconciliation. + // So we must ensure that we have already added the missing partitions, otherwise, we will + // update the metadata table with an empty list of missing partitions. + if (!hasAddedMissingPartitions) { + return Collections.emptyList(); + } + // Reset to ensure we get an updated list of missing partitions before reconciling again. + hasAddedMissingPartitions = false; // This value is calculated in case that we reconcile without continuation tokens, we will use // an hour prior to low watermark because low watermark is only an estimate. By reading back 1 // hour, it should cover any changes missed. We also want to make sure that the reconcile time @@ -177,7 +191,10 @@ public List getPartitionsToReconcile(Instant lowWatermark, Inst if (reconciledTime.compareTo(startTime) < 0) { reconciledTime = startTime; } - List reconciledPartitions = new ArrayList<>(); + // PartitionRecord formed using mismatched partitions by the reconciler + List partitionsToReconcile = new ArrayList<>(); + // Partitions from the list of missing partitions that are reconciled and no longer missing + List missingPartitionsToRemove = new ArrayList<>(); for (Map.Entry partitionDuration : missingPartitionDurations.entrySet()) { // The partition hasn't been missing for even the short duration. @@ -201,11 +218,13 @@ public List getPartitionsToReconcile(Instant lowWatermark, Inst PartitionRecord record = new PartitionRecord( missingPartition, allTokens, lowWatermark, overlappingNewPartitions); - reconciledPartitions.add(record); + // Remove this partition from the list of missing partitions. + missingPartitionsToRemove.add(missingPartition); + partitionsToReconcile.add(record); continue; } // The parents are not equal to the missing partition. If the missing partition has been - // around for more than 10 minutes, it needs to be reconciled. + // around for more than 20 minutes, it needs to be reconciled. if (partitionDuration.getValue().plus(MISSING_PARTITION_LONG_DELAY).isBeforeNow()) { // Try outputting the parent partitions as their own new partition, so they can get more // recent merge targets. @@ -217,7 +236,7 @@ public List getPartitionsToReconcile(Instant lowWatermark, Inst newPartition.getChangeStreamContinuationTokens(), lowWatermark, Collections.singletonList(newPartition)); - reconciledPartitions.add(record); + partitionsToReconcile.add(record); } // Also output partition not overlapped by new partitions. // Get the missing partition from parentPartitions and missingPartition. @@ -232,11 +251,21 @@ public List getPartitionsToReconcile(Instant lowWatermark, Inst metrics.incPartitionReconciledWithoutTokenCount(); PartitionRecord record = new PartitionRecord(missing, reconciledTime, lowWatermark, Collections.emptyList()); - reconciledPartitions.add(record); + partitionsToReconcile.add(record); LOG.error("DNP: Reconciling partition because we're missing a token {}", record); } + + // Remove this partition from the list of missing partitions. + missingPartitionsToRemove.add(missingPartition); } } - return reconciledPartitions; + + // Write to the metadata table without the reconciled partitions. + for (ByteStringRange missingPartitionToRemove : missingPartitionsToRemove) { + missingPartitionDurations.remove(missingPartitionToRemove); + } + metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations); + + return partitionsToReconcile; } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTracker.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTracker.java index 923f8d3e7bbdb..c92507612d63d 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTracker.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTracker.java @@ -20,7 +20,7 @@ import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker; import org.apache.beam.sdk.transforms.splittabledofn.SplitResult; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/StreamProgress.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/StreamProgress.java index ce2cf11a842e5..fe3cdb15a2382 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/StreamProgress.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/StreamProgress.java @@ -20,6 +20,7 @@ import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken; import com.google.cloud.bigtable.data.v2.models.CloseStream; import java.io.Serializable; +import java.math.BigDecimal; import java.util.Objects; import org.apache.beam.sdk.annotations.Internal; import org.checkerframework.checker.nullness.qual.Nullable; @@ -41,15 +42,25 @@ public class StreamProgress implements Serializable { private @Nullable ChangeStreamContinuationToken currentToken; private @Nullable Instant estimatedLowWatermark; + private @Nullable BigDecimal throughputEstimate; + private @Nullable Instant lastRunTimestamp; private @Nullable CloseStream closeStream; private boolean failToLock; + private boolean isHeartbeat; public StreamProgress() {} public StreamProgress( - @Nullable ChangeStreamContinuationToken token, Instant estimatedLowWatermark) { + @Nullable ChangeStreamContinuationToken token, + Instant estimatedLowWatermark, + BigDecimal throughputEstimate, + Instant lastRunTimestamp, + boolean isHeartbeat) { this.currentToken = token; this.estimatedLowWatermark = estimatedLowWatermark; + this.throughputEstimate = throughputEstimate; + this.lastRunTimestamp = lastRunTimestamp; + this.isHeartbeat = isHeartbeat; } public StreamProgress(@Nullable CloseStream closeStream) { @@ -64,6 +75,14 @@ public StreamProgress(@Nullable CloseStream closeStream) { return estimatedLowWatermark; } + public @Nullable BigDecimal getThroughputEstimate() { + return throughputEstimate; + } + + public @Nullable Instant getLastRunTimestamp() { + return lastRunTimestamp; + } + public @Nullable CloseStream getCloseStream() { return closeStream; } @@ -76,6 +95,10 @@ public void setFailToLock(boolean failToLock) { this.failToLock = failToLock; } + public boolean isHeartbeat() { + return this.isHeartbeat; + } + public boolean isEmpty() { return closeStream == null && currentToken == null; } @@ -92,7 +115,10 @@ public boolean equals(@Nullable Object o) { return Objects.equals(getCurrentToken(), that.getCurrentToken()) && Objects.equals(getEstimatedLowWatermark(), that.getEstimatedLowWatermark()) && Objects.equals(getCloseStream(), that.getCloseStream()) - && (isFailToLock() == that.isFailToLock()); + && (isFailToLock() == that.isFailToLock()) + && Objects.equals(getThroughputEstimate(), that.getThroughputEstimate()) + && Objects.equals(getLastRunTimestamp(), that.getLastRunTimestamp()) + && (isHeartbeat() == that.isHeartbeat()); } @Override @@ -111,6 +137,12 @@ public String toString() { + closeStream + ", failToLock=" + failToLock + + ", throughputEstimate=" + + throughputEstimate + + ", lastRunTimestamp=" + + lastRunTimestamp + + ", isHeartbeat=" + + isHeartbeat + '}'; } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/common/GcpIoPipelineOptionsRegistrar.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/common/GcpIoPipelineOptionsRegistrar.java index 6d3f1b914eebb..f1ff827fc6331 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/common/GcpIoPipelineOptionsRegistrar.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/common/GcpIoPipelineOptionsRegistrar.java @@ -20,11 +20,12 @@ import com.google.auto.service.AutoService; import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryOptions; +import org.apache.beam.sdk.io.gcp.bigquery.TestBigQueryOptions; import org.apache.beam.sdk.io.gcp.firestore.FirestoreOptions; import org.apache.beam.sdk.io.gcp.pubsub.PubsubOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** A registrar containing the default GCP options. */ @AutoService(PipelineOptionsRegistrar.class) @@ -36,6 +37,7 @@ public Iterable> getPipelineOptions() { .add(BigQueryOptions.class) .add(PubsubOptions.class) .add(FirestoreOptions.class) + .add(TestBigQueryOptions.class) .build(); } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/AdaptiveThrottler.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/AdaptiveThrottler.java index c7225120c91e6..15c4e678bcb4e 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/AdaptiveThrottler.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/AdaptiveThrottler.java @@ -20,7 +20,7 @@ import java.util.Random; import org.apache.beam.sdk.transforms.Sum; import org.apache.beam.sdk.util.MovingFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** * An implementation of client-side adaptive throttling. See diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java index f5a3d243e123e..cdd003abae227 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java @@ -26,9 +26,9 @@ import static com.google.datastore.v1.client.DatastoreHelper.makeOrder; import static com.google.datastore.v1.client.DatastoreHelper.makeUpsert; import static com.google.datastore.v1.client.DatastoreHelper.makeValue; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Verify.verify; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Verify.verify; import com.google.api.client.http.HttpRequestInitializer; import com.google.auth.Credentials; @@ -99,11 +99,11 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; @@ -648,8 +648,8 @@ public DatastoreV1.Read withNamespace(ValueProvider namespace) { * chosen dynamically at runtime based on the query data size. *

    • Any value greater than {@link Read#NUM_QUERY_SPLITS_MAX} will be capped at {@code * NUM_QUERY_SPLITS_MAX}. - *
    • If the {@code query} has a user limit set, then {@code numQuerySplits} will be ignored - * and no split will be performed. + *
    • If the {@code query} has a user limit set, or contains inequality filters, then {@code + * numQuerySplits} will be ignored and no split will be performed. *
    • Under certain cases Cloud Datastore is unable to split query to the requested number of * splits. In such cases we just use whatever the Cloud Datastore returns. *
    diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/EntityToRow.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/EntityToRow.java index 90dffc8a24fc7..1c0bef6f2200d 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/EntityToRow.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/EntityToRow.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Instant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/RampupThrottlingFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/RampupThrottlingFn.java index 34922356c9520..db098c0a5166a 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/RampupThrottlingFn.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/RampupThrottlingFn.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.util.MovingFunction; import org.apache.beam.sdk.util.Sleeper; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.Duration; import org.joda.time.Instant; import org.slf4j.Logger; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreOptions.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreOptions.java index edaf99d034092..1be6568372d9c 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreOptions.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreOptions.java @@ -57,4 +57,22 @@ public interface FirestoreOptions extends PipelineOptions { /** Set the Firestore database ID to connect to. */ void setFirestoreDb(String firestoreDb); + + /** + * A host port pair to allow connecting to a Cloud Firestore instead of the default live service. + * + * @return the string representation of a host and port pair to be used when constructing Cloud + * Firestore clients. + */ + @Description("Firestore endpoint (host and port)") + @Default.String("batch-firestore.googleapis.com:443") + String getHost(); + + /** + * Define a host port pair to allow connecting to a Cloud Firestore instead of the default live + * service. + * + * @param host the host and port to connect to + */ + void setHost(String host); } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreStatefulComponentFactory.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreStatefulComponentFactory.java index d8b8ba75a1d6e..21c29c485d1e4 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreStatefulComponentFactory.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreStatefulComponentFactory.java @@ -33,7 +33,7 @@ import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.NonNull; /** @@ -66,15 +66,7 @@ private FirestoreStatefulComponentFactory() {} */ FirestoreStub getFirestoreStub(PipelineOptions options) { try { - FirestoreSettings.Builder builder = - FirestoreSettings.newBuilder() - .setHeaderProvider( - new FixedHeaderProvider() { - @Override - public Map<@NonNull String, @NonNull String> getHeaders() { - return ImmutableMap.of("User-Agent", options.getUserAgent()); - } - }); + FirestoreSettings.Builder builder = FirestoreSettings.newBuilder(); RetrySettings retrySettings = RetrySettings.newBuilder().setMaxAttempts(1).build(); @@ -86,6 +78,8 @@ FirestoreStub getFirestoreStub(PipelineOptions options) { FirestoreOptions firestoreOptions = options.as(FirestoreOptions.class); String emulatorHostPort = firestoreOptions.getEmulatorHost(); + ImmutableMap.Builder headers = ImmutableMap.builder(); + headers.put("User-Agent", options.getUserAgent()); if (emulatorHostPort != null) { builder .setCredentialsProvider(FixedCredentialsProvider.create(new EmulatorCredentials())) @@ -99,9 +93,23 @@ FirestoreStub getFirestoreStub(PipelineOptions options) { GcpOptions gcpOptions = options.as(GcpOptions.class); builder .setCredentialsProvider(FixedCredentialsProvider.create(gcpOptions.getGcpCredential())) - .setEndpoint("batch-firestore.googleapis.com:443"); + .setEndpoint(firestoreOptions.getHost()); + headers.put( + "x-goog-request-params", + "project_id=" + + gcpOptions.getProject() + + "&database_id=" + + firestoreOptions.getFirestoreDb()); } + builder.setHeaderProvider( + new FixedHeaderProvider() { + @Override + public Map<@NonNull String, @NonNull String> getHeaders() { + return headers.build(); + } + }); + ClientContext clientContext = ClientContext.create(builder.build()); return GrpcFirestoreStub.create(clientContext); } catch (Exception e) { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1ReadFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1ReadFn.java index 886a03ebc05f2..b4a334b75c999 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1ReadFn.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1ReadFn.java @@ -57,7 +57,7 @@ import org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcAttempt.Context; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.display.DisplayData; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1WriteFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1WriteFn.java index 0f327cc89401e..3e9e1890c1e3f 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1WriteFn.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1WriteFn.java @@ -52,8 +52,8 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.BackOffUtils; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/QueryUtils.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/QueryUtils.java index a1a4af07510c8..ca47ad3880814 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/QueryUtils.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/QueryUtils.java @@ -36,10 +36,10 @@ import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Ascii; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.UnsignedBytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Ascii; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedBytes; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/RpcQosImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/RpcQosImpl.java index a52bb3ba7c4be..c600ae4224b42 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/RpcQosImpl.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/RpcQosImpl.java @@ -43,10 +43,10 @@ import org.apache.beam.sdk.util.BackOff; import org.apache.beam.sdk.util.MovingFunction; import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Ints; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Ints; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/RpcQosOptions.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/RpcQosOptions.java index f228beadbfc12..8945712065f7f 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/RpcQosOptions.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/RpcQosOptions.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.gcp.firestore; import static java.util.Objects.requireNonNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.Serializable; import java.util.Comparator; @@ -28,7 +28,7 @@ import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1.BatchWriteWithSummary; import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.transforms.display.HasDisplayData; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/DicomIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/DicomIO.java index 5ecacb09a2fdc..e8c1d601912cf 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/DicomIO.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/DicomIO.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * The DicomIO connectors allows Beam pipelines to make calls to the Dicom API of the Google Cloud @@ -154,7 +154,7 @@ static class FetchStudyMetadataFn extends DoFn { FetchStudyMetadataFn() {} /** - * Instantiate the healthcare client. + * Instantiate the healthcare client (version v1). * * @throws IOException */ diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIO.java index cf06cc433877d..4fdd3513c8065 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIO.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIO.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.healthcare; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -84,9 +84,9 @@ import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -584,7 +584,7 @@ static class ReadResourceFn extends DoFn { ReadResourceFn() {} /** - * Instantiate healthcare client. + * Instantiate healthcare client (version v1). * * @throws IOException the io exception */ @@ -1460,7 +1460,7 @@ static class ExecuteBundlesFn extends DoFn, KV { HL7v2MessageGetFn() {} /** - * Instantiate healthcare client. + * Instantiate healthcare client (version v1). * * @throws IOException the io exception */ @@ -471,7 +471,7 @@ public static class HL7v2MessageGetFn extends DoFn { } /** - * Init client. + * Initialize client (v1). * * @throws IOException the io exception */ @@ -920,7 +920,7 @@ static class WriteHL7v2Fn extends DoFn filters, String pageToken) @@ -206,46 +265,101 @@ HttpBody getPatientEverything( /** * Create hl 7 v 2 store hl 7 v 2 store. * - * @param dataset the dataset - * @param name the name - * @return the hl 7 v 2 store - * @throws IOException the io exception + * @param dataset The dataset to create the HL7v2 store in. + * @param name The name of the store to be created. + * @return Empty. + * @throws IOException The IO Exception. */ Hl7V2Store createHL7v2Store(String dataset, String name) throws IOException; + /** + * Create FHIR Store with a PubSub topic listener. + * + * @param dataset The name of Dataset for the FHIR store to be created in. + * @param name The name of the FHIR store. + * @param version The version of the FHIR store (DSTU2, STU3, R4). + * @param pubsubTopic The pubsub topic listening to the FHIR store. + * @throws IOException The IO Exception. + */ FhirStore createFhirStore(String dataset, String name, String version, String pubsubTopic) throws IOException; - + /** + * Create FHIR Store. + * + * @param dataset The name of the Dataset for the FHIR store to be created in. + * @param name The name of the FHIR store. + * @param version The version of the FHIR store (DSTU2, STU3, R4). + * @throws IOException The IO Exception. + */ FhirStore createFhirStore(String dataset, String name, String version) throws IOException; /** * List all FHIR stores in a dataset. * - * @param dataset the dataset, in the format: + * @param dataset The dataset, in the format: * projects/project_id/locations/location_id/datasets/dataset_id - * @return a list of FhirStore - * @throws IOException + * @return A list of all FHIR stores in the dataset. + * @throws IOException The IO Exception. */ List listAllFhirStores(String dataset) throws IOException; /** - * Delete hl 7 v 2 store empty. + * Delete Fhir store. * - * @param store the store - * @return the empty - * @throws IOException the io exception + * @param store The FHIR store to be deleted. + * @return Empty. + * @throws IOException The IO Exception. */ - Empty deleteHL7v2Store(String store) throws IOException; - Empty deleteFhirStore(String store) throws IOException; + /** + * Retrieve DicomStudyMetadata. + * + * @param dicomWebPath The Dicom Web Path to retrieve the metadata from. + * @return The study metadata. + * @throws IOException The IO Exception. + */ String retrieveDicomStudyMetadata(String dicomWebPath) throws IOException; + /** + * Create a DicomStore. + * + * @param dataset The dataset that the Dicom Store should be in, in the format: + * projects/project_id/locations/location_id/datasets/dataset_id. + * @param name The name of the Dicom Store to be created. + * @return Empty. + * @throws IOException The IO Exception. + */ DicomStore createDicomStore(String dataset, String name) throws IOException; + /** + * Create a DicomStore with a PubSub listener. + * + * @param dataset The dataset that the Dicom Store should be in, in the format: + * projects/project_id/locations/location_id/datasets/dataset_id + * @param name The name of the Dicom Store to be created. + * @param pubsubTopic Name of PubSub topic connected with the Dicom store. + * @return Empty. + * @throws IOException The IO Exception. + */ DicomStore createDicomStore(String dataset, String name, String pubsubTopic) throws IOException; + /** + * Delete a Dicom Store. + * + * @param name The name of the Dicom Store to be deleted. + * @return Empty. + * @throws IOException The IO Exception. + */ Empty deleteDicomStore(String name) throws IOException; + /** + * Upload to a Dicom Store. + * + * @param webPath String format of webPath to upload into. + * @param filePath filePath of file to upload. + * @return Empty. + * @throws IOException The IO Exception. + */ Empty uploadToDicomStore(String webPath, String filePath) throws IOException, URISyntaxException; } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HealthcareIOError.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HealthcareIOError.java index 10e26b53cfb89..3dd6906c0715d 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HealthcareIOError.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HealthcareIOError.java @@ -20,7 +20,7 @@ import com.google.api.client.googleapis.json.GoogleJsonResponseException; import org.apache.beam.sdk.coders.DefaultCoder; import org.apache.beam.sdk.io.gcp.healthcare.HttpHealthcareApiClient.HealthcareHttpException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HttpHealthcareApiClient.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HttpHealthcareApiClient.java index 1d0e0460c5d79..9a2a16bd949c2 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HttpHealthcareApiClient.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HttpHealthcareApiClient.java @@ -76,8 +76,8 @@ import java.util.stream.Collectors; import org.apache.beam.sdk.extensions.gcp.util.RetryHttpRequestInitializer; import org.apache.beam.sdk.util.ReleaseInfo; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpUriRequest; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/ExternalRead.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/ExternalRead.java index 6a3706499a233..dcff90115c671 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/ExternalRead.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/ExternalRead.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** Exposes {@link PubsubIO.Read} as an external transform for cross-language usage. */ diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/ExternalWrite.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/ExternalWrite.java index 7c63b9023e369..658d1fc29e325 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/ExternalWrite.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/ExternalWrite.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.ValueInSingleWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** Exposes {@link PubsubIO.Write} as an external transform for cross-language usage. */ diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/NestedRowToMessage.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/NestedRowToMessage.java index b6e310d81c22a..9c56410cda6be 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/NestedRowToMessage.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/NestedRowToMessage.java @@ -22,7 +22,7 @@ import static org.apache.beam.sdk.io.gcp.pubsub.PubsubSchemaIOProvider.ATTRIBUTE_ARRAY_FIELD_TYPE; import static org.apache.beam.sdk.io.gcp.pubsub.PubsubSchemaIOProvider.ATTRIBUTE_MAP_FIELD_TYPE; import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Collection; import java.util.Map; @@ -33,7 +33,7 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.SimpleFunction; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; class NestedRowToMessage extends SimpleFunction { private static final long serialVersionUID = 65176815766314684L; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubSubPayloadTranslation.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubSubPayloadTranslation.java index 3115da55cefcc..4722a3833fa92 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubSubPayloadTranslation.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubSubPayloadTranslation.java @@ -38,7 +38,7 @@ import org.apache.beam.sdk.runners.AppliedPTransform; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.util.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; @SuppressWarnings({ "nullness" // TODO(https://github.com/apache/beam/issues/20497) @@ -52,10 +52,7 @@ static class PubSubReadPayloadTranslator implements TransformPayloadTranslator> { @Override - public String getUrn(Read.Unbounded transform) { - if (!(transform.getSource() instanceof PubsubUnboundedSource.PubsubSource)) { - return null; - } + public String getUrn() { return PTransformTranslation.PUBSUB_READ; } @@ -106,7 +103,7 @@ public RunnerApi.FunctionSpec translate( static class PubSubWritePayloadTranslator implements TransformPayloadTranslator { @Override - public String getUrn(PubsubUnboundedSink.PubsubSink transform) { + public String getUrn() { return PTransformTranslation.PUBSUB_WRITE; } @@ -140,7 +137,7 @@ public RunnerApi.FunctionSpec translate( static class PubSubDynamicWritePayloadTranslator implements TransformPayloadTranslator { @Override - public String getUrn(PubsubUnboundedSink.PubsubDynamicSink transform) { + public String getUrn() { return PTransformTranslation.PUBSUB_WRITE_DYNAMIC; } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClient.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClient.java index 06d7c344a088f..79a9bb7f07d64 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClient.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClient.java @@ -17,9 +17,9 @@ */ package org.apache.beam.sdk.io.gcp.pubsub; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.client.util.DateTime; import com.google.auto.value.AutoValue; @@ -34,10 +34,10 @@ import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** An (abstract) helper class for talking to Pubsub via an underlying transport. */ diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubCoderProviderRegistrar.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubCoderProviderRegistrar.java index dacd5b6ebe58f..7a059640149ba 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubCoderProviderRegistrar.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubCoderProviderRegistrar.java @@ -23,7 +23,7 @@ import org.apache.beam.sdk.coders.CoderProviderRegistrar; import org.apache.beam.sdk.coders.CoderProviders; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** A {@link CoderProviderRegistrar} for standard types used with {@link PubsubIO}. */ @AutoService(CoderProviderRegistrar.class) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubDlqProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubDlqProvider.java index c950f7c1f05ec..87cd735810879 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubDlqProvider.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubDlqProvider.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; @Internal @AutoService(GenericDlqProvider.class) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubGrpcClient.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubGrpcClient.java index 7d8ca3f7517f4..93fdd55240074 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubGrpcClient.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubGrpcClient.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.gcp.pubsub; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auth.Credentials; import com.google.protobuf.Timestamp; @@ -65,11 +65,11 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.checkerframework.checker.nullness.qual.Nullable; /** A helper class for talking to Pubsub via grpc. */ diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java index 36683e98b33f9..839f443a19376 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.pubsub; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.client.util.Clock; import com.google.auto.value.AutoValue; @@ -73,11 +73,11 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.ValueInSingleWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; import org.slf4j.Logger; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubJsonClient.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubJsonClient.java index 140931a9f054a..386febcf005ba 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubJsonClient.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubJsonClient.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.pubsub; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.services.pubsub.Pubsub; @@ -48,10 +48,10 @@ import org.apache.beam.sdk.extensions.gcp.util.RetryHttpRequestInitializer; import org.apache.beam.sdk.extensions.gcp.util.Transport; import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; /** A Pubsub client using JSON transport. */ diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessage.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessage.java index 3649c6c6d4609..d57ac3d802075 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessage.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessage.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.pubsub; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import java.util.Map; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessagePayloadOnlyCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessagePayloadOnlyCoder.java index 74494fda8af8b..7aa8d14ef0371 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessagePayloadOnlyCoder.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessagePayloadOnlyCoder.java @@ -23,7 +23,7 @@ import org.apache.beam.sdk.coders.ByteArrayCoder; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.CustomCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** A coder for PubsubMessage treating the raw bytes being decoded as the message's payload. */ public class PubsubMessagePayloadOnlyCoder extends CustomCoder { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageToRow.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageToRow.java index e96b64d258f38..d860bfba95ecb 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageToRow.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageToRow.java @@ -19,7 +19,7 @@ import static java.util.stream.Collectors.toList; import static org.apache.beam.sdk.io.gcp.pubsub.PubsubSchemaIOProvider.ATTRIBUTE_ARRAY_ENTRY_SCHEMA; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.io.Serializable; @@ -40,7 +40,7 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; /** Read side converter for {@link PubsubMessage} with JSON/AVRO payload. */ diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithMessageIdCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithMessageIdCoder.java index 0f03dfae997e1..95bd43c53a664 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithMessageIdCoder.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithMessageIdCoder.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.CustomCoder; import org.apache.beam.sdk.coders.StringUtf8Coder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * A coder for PubsubMessage treating the raw bytes being decoded as the message's payload, with the diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessages.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessages.java index 89879657a2b1d..9bb49d8b6688d 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessages.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessages.java @@ -21,7 +21,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import java.util.Map; import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** Common util functions for converting between PubsubMessage proto and {@link PubsubMessage}. */ public final class PubsubMessages { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadSchemaTransformConfiguration.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadSchemaTransformConfiguration.java index f663f60f09bbe..6e665baaf6b1c 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadSchemaTransformConfiguration.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadSchemaTransformConfiguration.java @@ -17,11 +17,14 @@ */ package org.apache.beam.sdk.io.gcp.pubsub; +import com.google.api.client.util.Clock; import com.google.auto.value.AutoValue; +import java.util.List; import javax.annotation.Nullable; +import org.apache.beam.sdk.io.gcp.pubsub.PubsubTestClient.PubsubTestClientFactory; import org.apache.beam.sdk.schemas.AutoValueSchema; -import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.annotations.DefaultSchema; +import org.apache.beam.sdk.schemas.annotations.SchemaFieldDescription; /** * Configuration for reading from Pub/Sub. @@ -33,137 +36,116 @@ @DefaultSchema(AutoValueSchema.class) @AutoValue public abstract class PubsubReadSchemaTransformConfiguration { + @SchemaFieldDescription( + "The name of the topic to consume data from. If a topic is specified, " + + " will create a new subscription for that topic and start consuming from that point. " + + "Either a topic or a subscription must be provided. " + + "Format: projects/${PROJECT}/topics/${TOPIC}") + public abstract @Nullable String getTopic(); + + @SchemaFieldDescription( + "The name of the subscription to consume data. " + + "Either a topic or subscription must be provided. " + + "Format: projects/${PROJECT}/subscriptions/${SUBSCRIPTION}") + public abstract @Nullable String getSubscription(); + + @SchemaFieldDescription( + "The encoding format for the data stored in Pubsub. Valid options are: " + + PubsubReadSchemaTransformProvider.VALID_FORMATS_STR) + public abstract String getFormat(); // AVRO, JSON + + @SchemaFieldDescription( + "The schema in which the data is encoded in the Pubsub topic. " + + "For AVRO data, this is a schema defined with AVRO schema syntax " + + "(https://avro.apache.org/docs/1.10.2/spec.html#schemas). " + + "For JSON data, this is a schema defined with JSON-schema syntax (https://json-schema.org/).") + public abstract String getSchema(); + + @SchemaFieldDescription( + "Any additional pubsub attributes that should be populated as String fields in the ouptut rows.") + public abstract @Nullable List getAttributes(); + + @SchemaFieldDescription( + "Any additional field that should be populated with the full set of PubSub attributes.") + public abstract @Nullable String getAttributesMap(); + + @SchemaFieldDescription( + "When reading from Cloud Pub/Sub where unique record identifiers are provided as Pub/Sub message attributes, " + + "specifies the name of the attribute containing the unique identifier. " + + "The value of the attribute can be any string that uniquely identifies this record. " + + "Pub/Sub cannot guarantee that no duplicate data will be delivered on the Pub/Sub stream. " + + "If idAttribute is not provided, Beam cannot guarantee that no duplicate data will be delivered, " + + "and deduplication of the stream will be strictly best effort.") + public abstract @Nullable String getIdAttribute(); + + @SchemaFieldDescription( + "Specifies the name of the attribute that contains the timestamp, if any. " + + "The timestamp value is expected to be represented in the attribute as either " + + "(1) a numerical value representing the number of milliseconds since the Unix epoch. " + + "For example, if using the Joda time classes, " + + "Instant.getMillis() returns the correct value for this attribute." + + " or (2) a String in RFC 3339 format. For example, 2015-10-29T23:41:41.123Z. " + + "The sub-second component of the timestamp is optional, and digits beyond the first three " + + "(i.e., time units smaller than milliseconds) will be ignored.") + public abstract @Nullable String getTimestampAttribute(); + + @SchemaFieldDescription("Specifies how to handle errors.") + public abstract @Nullable ErrorHandling getErrorHandling(); + + // Used for testing only. + public abstract @Nullable PubsubTestClientFactory getClientFactory(); + + // Used for testing only. + public abstract @Nullable Clock getClock(); + + @AutoValue + public abstract static class ErrorHandling { + @SchemaFieldDescription("The name of the output PCollection containing failed reads.") + public abstract String getOutput(); + + public static PubsubReadSchemaTransformConfiguration.ErrorHandling.Builder builder() { + return new AutoValue_PubsubReadSchemaTransformConfiguration_ErrorHandling.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract PubsubReadSchemaTransformConfiguration.ErrorHandling.Builder setOutput( + String output); + + public abstract PubsubReadSchemaTransformConfiguration.ErrorHandling build(); + } + } - /** Instantiates a {@link PubsubReadSchemaTransformConfiguration.Builder}. */ public static Builder builder() { return new AutoValue_PubsubReadSchemaTransformConfiguration.Builder(); } - /** The expected schema of the Pub/Sub message. */ - public abstract Schema getDataSchema(); - - /** - * The Pub/Sub topic path to write failures. - * - *

    See {@link PubsubIO.PubsubTopic#fromPath(String)} for more details on the format of the dead - * letter queue topic string. - */ - @Nullable - public abstract String getDeadLetterQueue(); - - /** - * The expected format of the Pub/Sub message. - * - *

    Used to retrieve the {@link org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer} from - * {@link org.apache.beam.sdk.schemas.io.payloads.PayloadSerializers}. - */ - @Nullable - public abstract String getFormat(); - - /** Used by the ProtoPayloadSerializerProvider when serializing from a Pub/Sub message. */ - @Nullable - public abstract String getProtoClass(); - - /** - * The subscription from which to read Pub/Sub messages. - * - *

    See {@link PubsubIO.PubsubSubscription#fromPath(String)} for more details on the format of - * the subscription string. - */ - @Nullable - public abstract String getSubscription(); - - /** Used by the ThriftPayloadSerializerProvider when serializing from a Pub/Sub message. */ - @Nullable - public abstract String getThriftClass(); - - /** Used by the ThriftPayloadSerializerProvider when serializing from a Pub/Sub message. */ - @Nullable - public abstract String getThriftProtocolFactoryClass(); - - /** - * When reading from Cloud Pub/Sub where record timestamps are provided as Pub/Sub message - * attributes, specifies the name of the attribute that contains the timestamp. - */ - @Nullable - public abstract String getTimestampAttribute(); - - /** - * When reading from Cloud Pub/Sub where unique record identifiers are provided as Pub/Sub message - * attributes, specifies the name of the attribute containing the unique identifier. - */ - @Nullable - public abstract String getIdAttribute(); - - /** - * The topic from which to read Pub/Sub messages. - * - *

    See {@link PubsubIO.PubsubTopic#fromPath(String)} for more details on the format of the - * topic string. - */ - @Nullable - public abstract String getTopic(); - @AutoValue.Builder public abstract static class Builder { + public abstract Builder setTopic(@Nullable String topic); + + public abstract Builder setSubscription(@Nullable String subscription); + + public abstract Builder setFormat(String format); + + public abstract Builder setSchema(String schema); + + public abstract Builder setAttributes(@Nullable List attributes); + + public abstract Builder setAttributesMap(@Nullable String attributesMap); + + public abstract Builder setIdAttribute(@Nullable String schema); + + public abstract Builder setTimestampAttribute(@Nullable String schema); + + public abstract Builder setErrorHandling(@Nullable ErrorHandling errorHandling); + + // Used for testing only. + public abstract Builder setClientFactory(@Nullable PubsubTestClientFactory clientFactory); + + // Used for testing only. + public abstract Builder setClock(@Nullable Clock clock); - /** The expected schema of the Pub/Sub message. */ - public abstract Builder setDataSchema(Schema value); - - /** - * The Pub/Sub topic path to write failures. - * - *

    See {@link PubsubIO.PubsubTopic#fromPath(String)} for more details on the format of the - * dead letter queue topic string. - */ - public abstract Builder setDeadLetterQueue(String value); - - /** - * The expected format of the Pub/Sub message. - * - *

    Used to retrieve the {@link org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer} - * from {@link org.apache.beam.sdk.schemas.io.payloads.PayloadSerializers}. - */ - public abstract Builder setFormat(String value); - - /** Used by the ProtoPayloadSerializerProvider when serializing from a Pub/Sub message. */ - public abstract Builder setProtoClass(String value); - - /** - * The subscription from which to read Pub/Sub messages. - * - *

    See {@link PubsubIO.PubsubSubscription#fromPath(String)} for more details on the format of - * the subscription string. - */ - public abstract Builder setSubscription(String value); - - /** Used by the ThriftPayloadSerializerProvider when serializing from a Pub/Sub message. */ - public abstract Builder setThriftClass(String value); - - /** Used by the ThriftPayloadSerializerProvider when serializing from a Pub/Sub message. */ - public abstract Builder setThriftProtocolFactoryClass(String value); - - /** - * When reading from Cloud Pub/Sub where record timestamps are provided as Pub/Sub message - * attributes, specifies the name of the attribute that contains the timestamp. - */ - public abstract Builder setTimestampAttribute(String value); - - /** - * When reading from Cloud Pub/Sub where unique record identifiers are provided as Pub/Sub - * message attributes, specifies the name of the attribute containing the unique identifier. - */ - public abstract Builder setIdAttribute(String value); - - /** - * The topic from which to read Pub/Sub messages. - * - *

    See {@link PubsubIO.PubsubTopic#fromPath(String)} for more details on the format of the - * topic string. - */ - public abstract Builder setTopic(String value); - - /** Builds a {@link PubsubReadSchemaTransformConfiguration} instance. */ public abstract PubsubReadSchemaTransformConfiguration build(); } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadSchemaTransformProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadSchemaTransformProvider.java index af739469a3879..c1f6b2b31754a 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadSchemaTransformProvider.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadSchemaTransformProvider.java @@ -17,24 +17,36 @@ */ package org.apache.beam.sdk.io.gcp.pubsub; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.DLQ_TAG; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.MAIN_TAG; - import com.google.api.client.util.Clock; import com.google.auto.service.AutoService; +import java.io.Serializable; +import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializers; +import java.util.Set; +import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils; +import org.apache.beam.sdk.io.gcp.pubsub.PubsubTestClient.PubsubTestClientFactory; +import org.apache.beam.sdk.metrics.Counter; +import org.apache.beam.sdk.metrics.Metrics; +import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.transforms.SchemaTransform; import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; -import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.schemas.utils.JsonUtils; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.sdk.values.TupleTagList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.checkerframework.checker.initialization.qual.Initialized; +import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; +import org.checkerframework.checker.nullness.qual.UnknownKeyFor; /** * An implementation of {@link TypedSchemaTransformProvider} for Pub/Sub reads configured using @@ -44,203 +56,276 @@ * provide no backwards compatibility guarantees, and it should not be implemented outside the Beam * repository. */ -@SuppressWarnings({ - "nullness" // TODO(https://github.com/apache/beam/issues/20497) -}) -@Internal @AutoService(SchemaTransformProvider.class) public class PubsubReadSchemaTransformProvider extends TypedSchemaTransformProvider { - static final String OUTPUT_TAG = "OUTPUT"; - /** Returns the expected class of the configuration. */ - @Override - protected Class configurationClass() { - return PubsubReadSchemaTransformConfiguration.class; - } + public static final String VALID_FORMATS_STR = "RAW,AVRO,JSON"; + public static final Set VALID_DATA_FORMATS = + Sets.newHashSet(VALID_FORMATS_STR.split(",")); - /** Returns the expected {@link SchemaTransform} of the configuration. */ - @Override - protected SchemaTransform from(PubsubReadSchemaTransformConfiguration configuration) { - PubsubMessageToRow toRowTransform = - PubsubSchemaTransformMessageToRowFactory.from(configuration).buildMessageToRow(); - return new PubsubReadSchemaTransform(configuration, toRowTransform); - } - - /** Implementation of the {@link TypedSchemaTransformProvider} identifier method. */ - @Override - public String identifier() { - return "beam:schematransform:org.apache.beam:pubsub_read:v1"; - } + public static final TupleTag OUTPUT_TAG = new TupleTag() {}; + public static final TupleTag ERROR_TAG = new TupleTag() {}; + public static final Schema ERROR_SCHEMA = + Schema.builder().addStringField("error").addNullableByteArrayField("row").build(); - /** - * Implementation of the {@link TypedSchemaTransformProvider} inputCollectionNames method. Since - * no input is expected, this returns an empty list. - */ @Override - public List inputCollectionNames() { - return Collections.emptyList(); + public Class configurationClass() { + return PubsubReadSchemaTransformConfiguration.class; } - /** - * Implementation of the {@link TypedSchemaTransformProvider} outputCollectionNames method. Since - * a single output is expected, this returns a list with a single name. - */ @Override - public List outputCollectionNames() { - return Collections.singletonList(OUTPUT_TAG); - } - - /** - * An implementation of {@link SchemaTransform} for Pub/Sub reads configured using {@link - * PubsubReadSchemaTransformConfiguration}. - */ - static class PubsubReadSchemaTransform - extends PTransform implements SchemaTransform { - - private final PubsubReadSchemaTransformConfiguration configuration; - private final PubsubMessageToRow pubsubMessageToRow; - - private PubsubClient.PubsubClientFactory clientFactory; - - private Clock clock; - - private PubsubReadSchemaTransform( - PubsubReadSchemaTransformConfiguration configuration, - PubsubMessageToRow pubsubMessageToRow) { - this.configuration = configuration; - this.pubsubMessageToRow = pubsubMessageToRow; + public SchemaTransform from(PubsubReadSchemaTransformConfiguration configuration) { + if (configuration.getSubscription() == null && configuration.getTopic() == null) { + throw new IllegalArgumentException( + "To read from Pubsub, a subscription name or a topic name must be provided"); } - /** - * Sets the {@link PubsubClient.PubsubClientFactory}. - * - *

    Used for testing. - */ - void setClientFactory(PubsubClient.PubsubClientFactory value) { - this.clientFactory = value; - } - - /** - * Sets the {@link Clock}. - * - *

    Used for testing. - */ - void setClock(Clock clock) { - this.clock = clock; + if (configuration.getSubscription() != null && configuration.getTopic() != null) { + throw new IllegalArgumentException( + "To read from Pubsub, a subscription name or a topic name must be provided. Not both."); } - /** Implements {@link SchemaTransform} buildTransform method. */ - @Override - public PTransform buildTransform() { - return this; - } - - /** Validates the {@link PubsubReadSchemaTransformConfiguration}. */ - @Override - public void validate(@Nullable PipelineOptions options) { - if (configuration.getSubscription() == null && configuration.getTopic() == null) { - throw new IllegalArgumentException( - String.format( - "%s needs to set either the topic or the subscription", - PubsubReadSchemaTransformConfiguration.class)); - } - - if (configuration.getSubscription() != null && configuration.getTopic() != null) { + if (!"RAW".equals(configuration.getFormat())) { + if ((Strings.isNullOrEmpty(configuration.getSchema()) + && !Strings.isNullOrEmpty(configuration.getFormat())) + || (!Strings.isNullOrEmpty(configuration.getSchema()) + && Strings.isNullOrEmpty(configuration.getFormat()))) { throw new IllegalArgumentException( - String.format( - "%s should not set both the topic or the subscription", - PubsubReadSchemaTransformConfiguration.class)); + "A schema was provided without a data format (or viceversa). Please provide " + + "both of these parameters to read from Pubsub, or if you would like to use the Pubsub schema service," + + " please leave both of these blank."); } + } - try { - PayloadSerializers.getSerializer( - configuration.getFormat(), configuration.getDataSchema(), new HashMap<>()); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException( - String.format( - "Invalid %s, no serializer provider exists for format `%s`", - PubsubReadSchemaTransformConfiguration.class, configuration.getFormat())); - } + Schema payloadSchema; + SerializableFunction payloadMapper; + + String format = + configuration.getFormat() == null ? null : configuration.getFormat().toUpperCase(); + if ("RAW".equals(format)) { + payloadSchema = Schema.of(Schema.Field.of("payload", Schema.FieldType.BYTES)); + payloadMapper = input -> Row.withSchema(payloadSchema).addValue(input).build(); + } else if ("JSON".equals(format)) { + payloadSchema = JsonUtils.beamSchemaFromJsonSchema(configuration.getSchema()); + payloadMapper = JsonUtils.getJsonBytesToRowFunction(payloadSchema); + } else if ("AVRO".equals(format)) { + payloadSchema = + AvroUtils.toBeamSchema( + new org.apache.avro.Schema.Parser().parse(configuration.getSchema())); + payloadMapper = AvroUtils.getAvroBytesToRowFunction(payloadSchema); + } else { + throw new IllegalArgumentException( + String.format( + "Format %s not supported. Only supported formats are %s", + configuration.getFormat(), VALID_FORMATS_STR)); } - /** Reads from Pub/Sub according to {@link PubsubReadSchemaTransformConfiguration}. */ - @Override - public PCollectionRowTuple expand(PCollectionRowTuple input) { - if (!input.getAll().isEmpty()) { - throw new IllegalArgumentException( - String.format( - "%s %s input is expected to be empty", - input.getClass().getSimpleName(), getClass().getSimpleName())); - } + PubsubReadSchemaTransform transform = + new PubsubReadSchemaTransform(configuration, payloadSchema, payloadMapper); - PCollectionTuple rowsWithDlq = - input - .getPipeline() - .apply("ReadFromPubsub", buildPubsubRead()) - .apply("PubsubMessageToRow", pubsubMessageToRow); + if (configuration.getClientFactory() != null) { + transform.setClientFactory(configuration.getClientFactory()); + } + if (configuration.getClock() != null) { + transform.setClock(configuration.getClock()); + } - writeToDeadLetterQueue(rowsWithDlq); + return transform; + } - return PCollectionRowTuple.of(OUTPUT_TAG, rowsWithDlq.get(MAIN_TAG)); - } + private static class PubsubReadSchemaTransform extends SchemaTransform implements Serializable { + final Schema beamSchema; + final SerializableFunction valueMapper; + final PubsubReadSchemaTransformConfiguration configuration; + @Nullable PubsubTestClientFactory clientFactory; + @Nullable Clock clock; - private void writeToDeadLetterQueue(PCollectionTuple rowsWithDlq) { - PubsubIO.Write deadLetterQueue = buildDeadLetterQueueWrite(); - if (deadLetterQueue == null) { - return; + PubsubReadSchemaTransform( + PubsubReadSchemaTransformConfiguration configuration, + Schema payloadSchema, + SerializableFunction valueMapper) { + this.configuration = configuration; + Schema outputSchema; + List attributes = configuration.getAttributes(); + String attributesMap = configuration.getAttributesMap(); + if (attributes == null && attributesMap == null) { + outputSchema = payloadSchema; + } else { + Schema.Builder outputSchemaBuilder = Schema.builder(); + outputSchemaBuilder.addFields(payloadSchema.getFields()); + if (attributes != null) { + for (String attribute : attributes) { + outputSchemaBuilder.addStringField(attribute); + } + } + if (attributesMap != null) { + outputSchemaBuilder.addMapField( + attributesMap, Schema.FieldType.STRING, Schema.FieldType.STRING); + } + outputSchema = outputSchemaBuilder.build(); } - rowsWithDlq.get(DLQ_TAG).apply("WriteToDeadLetterQueue", deadLetterQueue); + this.beamSchema = outputSchema; + this.valueMapper = valueMapper; } - /** - * Builds {@link PubsubIO.Write} dead letter queue from {@link - * PubsubReadSchemaTransformConfiguration}. - */ - PubsubIO.Write buildDeadLetterQueueWrite() { - if (configuration.getDeadLetterQueue() == null) { - return null; + private static class ErrorCounterFn extends DoFn { + private final Counter pubsubErrorCounter; + private Long errorsInBundle = 0L; + private final SerializableFunction valueMapper; + private final @Nullable List attributes; + private final @Nullable String attributesMap; + private final Schema outputSchema; + + final boolean useErrorOutput; + + ErrorCounterFn( + String name, + SerializableFunction valueMapper, + @Nullable List attributes, + @Nullable String attributesMap, + Schema outputSchema, + boolean useErrorOutput) { + this.pubsubErrorCounter = Metrics.counter(PubsubReadSchemaTransformProvider.class, name); + this.valueMapper = valueMapper; + this.attributes = attributes; + this.attributesMap = attributesMap; + this.outputSchema = outputSchema; + this.useErrorOutput = useErrorOutput; } - PubsubIO.Write writeDlq = - PubsubIO.writeMessages().to(configuration.getDeadLetterQueue()); + @ProcessElement + public void process(@DoFn.Element PubsubMessage message, MultiOutputReceiver receiver) + throws Exception { + + try { + Row payloadRow = valueMapper.apply(message.getPayload()); + Row outputRow; + if (attributes == null && attributesMap == null) { + outputRow = payloadRow; + } else { + Row.Builder rowBuilder = Row.withSchema(outputSchema); + List<@Nullable Object> payloadValues = payloadRow.getValues(); + if (payloadValues != null) { + rowBuilder.addValues(payloadValues); + } + if (attributes != null) { + for (String attribute : attributes) { + rowBuilder.addValue(message.getAttribute(attribute)); + } + } + if (attributesMap != null) { + rowBuilder.addValue(message.getAttributeMap()); + } + outputRow = rowBuilder.build(); + } + receiver.get(OUTPUT_TAG).output(outputRow); + } catch (Exception e) { + errorsInBundle += 1; + if (useErrorOutput) { + receiver + .get(ERROR_TAG) + .output( + Row.withSchema(ERROR_SCHEMA) + .addValues(e.toString(), message.getPayload()) + .build()); + } else { + throw e; + } + } + } - if (configuration.getTimestampAttribute() != null) { - writeDlq = writeDlq.withTimestampAttribute(configuration.getTimestampAttribute()); + @FinishBundle + public void finish(FinishBundleContext c) { + pubsubErrorCounter.inc(errorsInBundle); + errorsInBundle = 0L; } + } - return writeDlq; + void setClientFactory(@Nullable PubsubTestClientFactory factory) { + this.clientFactory = factory; } - /** Builds {@link PubsubIO.Read} from a {@link PubsubReadSchemaTransformConfiguration}. */ - PubsubIO.Read buildPubsubRead() { - PubsubIO.Read read = PubsubIO.readMessagesWithAttributes(); + void setClock(@Nullable Clock clock) { + this.clock = clock; + } - if (configuration.getSubscription() != null) { - read = read.fromSubscription(configuration.getSubscription()); + @SuppressWarnings("nullness") + PubsubIO.Read buildPubsubRead() { + PubsubIO.Read pubsubRead = + (configuration.getAttributes() == null && configuration.getAttributesMap() == null) + ? PubsubIO.readMessages() + : PubsubIO.readMessagesWithAttributes(); + if (!Strings.isNullOrEmpty(configuration.getTopic())) { + pubsubRead = pubsubRead.fromTopic(configuration.getTopic()); + } else { + pubsubRead = pubsubRead.fromSubscription(configuration.getSubscription()); } - - if (configuration.getTopic() != null) { - read = read.fromTopic(configuration.getTopic()); + if (clientFactory != null && clock != null) { + pubsubRead = pubsubRead.withClientFactory(clientFactory); + pubsubRead = clientFactory.setClock(pubsubRead, clock); + } else if (clientFactory != null || clock != null) { + throw new IllegalArgumentException( + "Both PubsubTestClientFactory and Clock need to be specified for testing, but only one is provided"); } - - if (configuration.getTimestampAttribute() != null) { - read = read.withTimestampAttribute(configuration.getTimestampAttribute()); + if (!Strings.isNullOrEmpty(configuration.getIdAttribute())) { + pubsubRead = pubsubRead.withIdAttribute(configuration.getIdAttribute()); } - - if (configuration.getIdAttribute() != null) { - read = read.withIdAttribute(configuration.getIdAttribute()); + if (!Strings.isNullOrEmpty(configuration.getTimestampAttribute())) { + pubsubRead = pubsubRead.withTimestampAttribute(configuration.getTimestampAttribute()); } + return pubsubRead; + } - if (clientFactory != null) { - read = read.withClientFactory(clientFactory); + @Override + public PCollectionRowTuple expand(PCollectionRowTuple input) { + PubsubIO.Read pubsubRead = buildPubsubRead(); + @SuppressWarnings("nullness") + String errorOutput = + configuration.getErrorHandling() == null + ? null + : configuration.getErrorHandling().getOutput(); + + PCollectionTuple outputTuple = + input + .getPipeline() + .apply(pubsubRead) + .apply( + ParDo.of( + new ErrorCounterFn( + "PubSub-read-error-counter", + valueMapper, + configuration.getAttributes(), + configuration.getAttributesMap(), + beamSchema, + errorOutput != null)) + .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); + outputTuple.get(OUTPUT_TAG).setRowSchema(beamSchema); + outputTuple.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA); + + PCollectionRowTuple result = PCollectionRowTuple.of("output", outputTuple.get(OUTPUT_TAG)); + if (errorOutput == null) { + return result; + } else { + return result.and(errorOutput, outputTuple.get(ERROR_TAG)); } + } + } - if (clock != null) { - read = read.withClock(clock); - } + @Override + public @UnknownKeyFor @NonNull @Initialized String identifier() { + return "beam:schematransform:org.apache.beam:pubsub_read:v1"; + } - return read; - } + @Override + public @UnknownKeyFor @NonNull @Initialized List<@UnknownKeyFor @NonNull @Initialized String> + inputCollectionNames() { + return Collections.emptyList(); + } + + @Override + public @UnknownKeyFor @NonNull @Initialized List<@UnknownKeyFor @NonNull @Initialized String> + outputCollectionNames() { + return Arrays.asList("output", "errors"); } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubRowToMessage.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubRowToMessage.java index b7a6e21edbb09..58849fd3f3bca 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubRowToMessage.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubRowToMessage.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.pubsub; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.util.Arrays; @@ -42,7 +42,7 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; import org.joda.time.DateTime; import org.joda.time.Instant; import org.joda.time.ReadableDateTime; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubSchemaIOProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubSchemaIOProvider.java index a31d7dfe6960c..d39cee3ddc06c 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubSchemaIOProvider.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubSchemaIOProvider.java @@ -48,7 +48,7 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubSchemaTransformMessageToRowFactory.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubSchemaTransformMessageToRowFactory.java deleted file mode 100644 index 988c593e32fa5..0000000000000 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubSchemaTransformMessageToRowFactory.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io.gcp.pubsub; - -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.ATTRIBUTES_FIELD; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.PAYLOAD_FIELD; -import static org.apache.beam.sdk.schemas.Schema.TypeName.ROW; - -import java.util.HashMap; -import java.util.Map; -import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer; -import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializers; - -/** - * Builds a {@link PubsubMessageToRow} from a {@link PubsubReadSchemaTransformConfiguration}. - * - *

    Internal only: This class is actively being worked on, and it will likely change. We - * provide no backwards compatibility guarantees, and it should not be implemented outside the Beam - * repository. - */ -@SuppressWarnings({ - "nullness" // TODO(https://github.com/apache/beam/issues/20497) -}) -@Internal -class PubsubSchemaTransformMessageToRowFactory { - private static final String DEFAULT_FORMAT = "json"; - - private static final Schema.FieldType ATTRIBUTE_MAP_FIELD_TYPE = - Schema.FieldType.map(Schema.FieldType.STRING.withNullable(false), Schema.FieldType.STRING); - private static final Schema ATTRIBUTE_ARRAY_ENTRY_SCHEMA = - Schema.builder().addStringField("key").addStringField("value").build(); - private static final Schema.FieldType ATTRIBUTE_ARRAY_FIELD_TYPE = - Schema.FieldType.array(Schema.FieldType.row(ATTRIBUTE_ARRAY_ENTRY_SCHEMA)); - - private static final String THRIFT_CLASS_KEY = "thriftClass"; - private static final String THRIFT_PROTOCOL_FACTORY_CLASS_KEY = "thriftProtocolFactoryClass"; - private static final String PROTO_CLASS_KEY = "protoClass"; - - /** - * Instantiate a {@link PubsubSchemaTransformMessageToRowFactory} from a {@link - * PubsubReadSchemaTransformConfiguration}. - */ - static PubsubSchemaTransformMessageToRowFactory from( - PubsubReadSchemaTransformConfiguration configuration) { - return new PubsubSchemaTransformMessageToRowFactory(configuration); - } - - /** Build the {@link PubsubMessageToRow}. */ - PubsubMessageToRow buildMessageToRow() { - PubsubMessageToRow.Builder builder = - PubsubMessageToRow.builder() - .messageSchema(configuration.getDataSchema()) - .useDlq( - configuration.getDeadLetterQueue() != null - && !configuration.getDeadLetterQueue().isEmpty()) - .useFlatSchema(!shouldUseNestedSchema()); - - if (needsSerializer()) { - builder = builder.serializerProvider(serializer()); - } - - return builder.build(); - } - - private final PubsubReadSchemaTransformConfiguration configuration; - - private PubsubSchemaTransformMessageToRowFactory( - PubsubReadSchemaTransformConfiguration configuration) { - this.configuration = configuration; - } - - private PayloadSerializer payloadSerializer() { - Schema schema = configuration.getDataSchema(); - String format = DEFAULT_FORMAT; - - if (configuration.getFormat() != null && !configuration.getFormat().isEmpty()) { - format = configuration.getFormat(); - } - - Map params = new HashMap<>(); - - if (configuration.getThriftClass() != null && !configuration.getThriftClass().isEmpty()) { - params.put(THRIFT_CLASS_KEY, configuration.getThriftClass()); - } - - if (configuration.getThriftProtocolFactoryClass() != null - && !configuration.getThriftProtocolFactoryClass().isEmpty()) { - params.put(THRIFT_PROTOCOL_FACTORY_CLASS_KEY, configuration.getThriftProtocolFactoryClass()); - } - - if (configuration.getProtoClass() != null && !configuration.getProtoClass().isEmpty()) { - params.put(PROTO_CLASS_KEY, configuration.getProtoClass()); - } - - return PayloadSerializers.getSerializer(format, schema, params); - } - - PubsubMessageToRow.SerializerProvider serializer() { - return input -> payloadSerializer(); - } - - /** - * Determines whether the {@link PubsubMessageToRow} needs a {@link - * PubsubMessageToRow.SerializerProvider}. - * - *

    The determination is based on {@link #shouldUseNestedSchema()} is false or if the {@link - * PubsubMessageToRow#PAYLOAD_FIELD} is not present. - */ - boolean needsSerializer() { - return !shouldUseNestedSchema() || !fieldPresent(PAYLOAD_FIELD, Schema.FieldType.BYTES); - } - - /** - * Determines whether a nested schema should be used for {@link - * PubsubReadSchemaTransformConfiguration#getDataSchema()}. - * - *

    The determination is based on {@link #schemaHasValidPayloadField()} and {@link - * #schemaHasValidAttributesField()}} - */ - boolean shouldUseNestedSchema() { - return schemaHasValidPayloadField() && schemaHasValidAttributesField(); - } - - /** - * Determines whether {@link PubsubReadSchemaTransformConfiguration#getDataSchema()} has a valid - * {@link PubsubMessageToRow#PAYLOAD_FIELD}. - */ - boolean schemaHasValidPayloadField() { - Schema schema = configuration.getDataSchema(); - if (!schema.hasField(PAYLOAD_FIELD)) { - return false; - } - if (fieldPresent(PAYLOAD_FIELD, Schema.FieldType.BYTES)) { - return true; - } - return schema.getField(PAYLOAD_FIELD).getType().getTypeName().equals(ROW); - } - - /** - * Determines whether {@link PubsubReadSchemaTransformConfiguration#getDataSchema()} has a valid - * {@link PubsubMessageToRow#ATTRIBUTES_FIELD} field. - * - *

    The determination is based on whether {@link #fieldPresent(String, Schema.FieldType)} for - * {@link PubsubMessageToRow#ATTRIBUTES_FIELD} is true for either {@link - * #ATTRIBUTE_MAP_FIELD_TYPE} or {@link #ATTRIBUTE_ARRAY_FIELD_TYPE} {@link Schema.FieldType}s. - */ - boolean schemaHasValidAttributesField() { - return fieldPresent(ATTRIBUTES_FIELD, ATTRIBUTE_MAP_FIELD_TYPE) - || fieldPresent(ATTRIBUTES_FIELD, ATTRIBUTE_ARRAY_FIELD_TYPE); - } - - /** - * Determines whether {@link PubsubReadSchemaTransformConfiguration#getDataSchema()} contains the - * field and whether that field is an expectedType {@link Schema.FieldType}. - */ - boolean fieldPresent(String field, Schema.FieldType expectedType) { - Schema schema = configuration.getDataSchema(); - return schema.hasField(field) - && expectedType.equivalent( - schema.getField(field).getType(), Schema.EquivalenceNullablePolicy.IGNORE); - } -} diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubTestClient.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubTestClient.java index 96163c1325f26..a8109d05ec380 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubTestClient.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubTestClient.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.pubsub; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.client.util.Clock; import java.io.Closeable; @@ -30,8 +30,8 @@ import java.util.Map; import java.util.Set; import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java index b53aef87bc3f0..aa8e3a4114868 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSink.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.pubsub; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.protobuf.InvalidProtocolBufferException; import java.io.IOException; @@ -67,9 +67,9 @@ import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java index 562ce824a51a4..b9a554d54ade7 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java @@ -17,9 +17,9 @@ */ package org.apache.beam.sdk.io.gcp.pubsub; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.client.util.Clock; import java.io.IOException; @@ -75,9 +75,9 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubWriteSchemaTransformConfiguration.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubWriteSchemaTransformConfiguration.java index acaf04cdfc695..f962e7185f1b9 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubWriteSchemaTransformConfiguration.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubWriteSchemaTransformConfiguration.java @@ -18,9 +18,11 @@ package org.apache.beam.sdk.io.gcp.pubsub; import com.google.auto.value.AutoValue; +import java.util.List; import javax.annotation.Nullable; import org.apache.beam.sdk.schemas.AutoValueSchema; import org.apache.beam.sdk.schemas.annotations.DefaultSchema; +import org.apache.beam.sdk.schemas.annotations.SchemaFieldDescription; /** * Configuration for writing to Pub/Sub. @@ -32,179 +34,74 @@ @DefaultSchema(AutoValueSchema.class) @AutoValue public abstract class PubsubWriteSchemaTransformConfiguration { + @SchemaFieldDescription( + "The encoding format for the data stored in Pubsub. Valid options are: " + + PubsubWriteSchemaTransformProvider.VALID_FORMATS_STR) + public abstract String getFormat(); - public static final String DEFAULT_TIMESTAMP_ATTRIBUTE = "event_timestamp"; - - public static Builder builder() { - return new AutoValue_PubsubWriteSchemaTransformConfiguration.Builder(); - } - - public static TargetConfiguration.Builder targetConfigurationBuilder() { - return new AutoValue_PubsubWriteSchemaTransformConfiguration_TargetConfiguration.Builder() - .setTimestampAttributeKey(DEFAULT_TIMESTAMP_ATTRIBUTE); - } - - public static SourceConfiguration.Builder sourceConfigurationBuilder() { - return new AutoValue_PubsubWriteSchemaTransformConfiguration_SourceConfiguration.Builder(); - } - - /** - * Configuration details of the source {@link org.apache.beam.sdk.values.Row} {@link - * org.apache.beam.sdk.schemas.Schema}. - */ - @Nullable - public abstract SourceConfiguration getSource(); - - /** Configuration details of the target {@link PubsubMessage}. */ - public abstract TargetConfiguration getTarget(); - - /** - * The topic to which to write Pub/Sub messages. - * - *

    See {@link PubsubIO.PubsubTopic#fromPath(String)} for more details on the format of the - * topic string. - */ + @SchemaFieldDescription( + "The name of the topic to write data to. " + "Format: projects/${PROJECT}/topics/${TOPIC}") public abstract String getTopic(); - /** - * The expected format of the Pub/Sub message. - * - *

    Used to retrieve the {@link org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer} from - * {@link org.apache.beam.sdk.schemas.io.payloads.PayloadSerializers}. See list of supported - * values by invoking {@link org.apache.beam.sdk.schemas.io.Providers#loadProviders(Class)}. - * - *

    {@code Providers.loadProviders(PayloadSerializer.class).keySet()}
    - */ - @Nullable - public abstract String getFormat(); + @SchemaFieldDescription( + "The set of fields to write as PubSub attributes instead of part of the payload.") + public abstract @Nullable List getAttributes(); - /** - * When writing to Cloud Pub/Sub where unique record identifiers are provided as Pub/Sub message - * attributes, specifies the name of the attribute containing the unique identifier. - */ - @Nullable - public abstract String getIdAttribute(); + @SchemaFieldDescription( + "A map field to write as PubSub attributes instead of part of the payload.") + public abstract @Nullable String getAttributesMap(); - /** Builder for {@link PubsubWriteSchemaTransformConfiguration}. */ - @AutoValue.Builder - public abstract static class Builder { + @SchemaFieldDescription( + "If set, will set an attribute for each Cloud Pub/Sub message with the given name and a unique value. " + + "This attribute can then be used in a ReadFromPubSub PTransform to deduplicate messages.") + public abstract @Nullable String getIdAttribute(); - /** - * Configuration details of the source {@link org.apache.beam.sdk.values.Row} {@link - * org.apache.beam.sdk.schemas.Schema}. - */ - public abstract Builder setSource(SourceConfiguration value); - - /** Configuration details of the target {@link PubsubMessage}. */ - public abstract Builder setTarget(TargetConfiguration value); - - /** - * The topic to which to write Pub/Sub messages. - * - *

    See {@link PubsubIO.PubsubTopic#fromPath(String)} for more details on the format of the - * topic string. - */ - public abstract Builder setTopic(String value); - - /** - * The expected format of the Pub/Sub message. - * - *

    Used to retrieve the {@link org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer} - * from {@link org.apache.beam.sdk.schemas.io.payloads.PayloadSerializers}. See list of - * supported values by invoking {@link - * org.apache.beam.sdk.schemas.io.Providers#loadProviders(Class)}. - * - *

    {@code Providers.loadProviders(PayloadSerializer.class).keySet()}
    - */ - public abstract Builder setFormat(String value); - - /** - * When reading from Cloud Pub/Sub where unique record identifiers are provided as Pub/Sub - * message attributes, specifies the name of the attribute containing the unique identifier. - */ - public abstract Builder setIdAttribute(String value); + @SchemaFieldDescription( + "If set, will set an attribute for each Cloud Pub/Sub message with the given name and the message's " + + "publish time as the value.") + public abstract @Nullable String getTimestampAttribute(); - public abstract PubsubWriteSchemaTransformConfiguration build(); - } + @SchemaFieldDescription("Specifies how to handle errors.") + public abstract @Nullable ErrorHandling getErrorHandling(); - @DefaultSchema(AutoValueSchema.class) @AutoValue - public abstract static class SourceConfiguration { - /** - * The attributes field name of the source {@link org.apache.beam.sdk.values.Row}. {@link - * org.apache.beam.sdk.schemas.Schema.FieldType} must be a Map<String, String> - * - */ - @Nullable - public abstract String getAttributesFieldName(); - - /** - * The timestamp field name of the source {@link org.apache.beam.sdk.values.Row}. {@link - * org.apache.beam.sdk.schemas.Schema.FieldType} must be a {@link - * org.apache.beam.sdk.schemas.Schema.FieldType#DATETIME}. - */ - @Nullable - public abstract String getTimestampFieldName(); - - /** - * The payload field name of the source {@link org.apache.beam.sdk.values.Row}. {@link - * org.apache.beam.sdk.schemas.Schema.FieldType} must be either {@link - * org.apache.beam.sdk.schemas.Schema.FieldType#BYTES} or a {@link - * org.apache.beam.sdk.values.Row}. If null, payload serialized from user fields other than - * attributes. Not compatible with other payload intended fields. - */ - @Nullable - public abstract String getPayloadFieldName(); + public abstract static class ErrorHandling { + @SchemaFieldDescription("The name of the output PCollection containing failed writes.") + public abstract String getOutput(); + + public static PubsubWriteSchemaTransformConfiguration.ErrorHandling.Builder builder() { + return new AutoValue_PubsubWriteSchemaTransformConfiguration_ErrorHandling.Builder(); + } @AutoValue.Builder public abstract static class Builder { - /** - * The attributes field name of the source {@link org.apache.beam.sdk.values.Row}. {@link - * org.apache.beam.sdk.schemas.Schema.FieldType} must be a Map<String, String> - * - */ - public abstract Builder setAttributesFieldName(String value); - - /** - * The timestamp field name of the source {@link org.apache.beam.sdk.values.Row}. {@link - * org.apache.beam.sdk.schemas.Schema.FieldType} must be a {@link - * org.apache.beam.sdk.schemas.Schema.FieldType#DATETIME}. - */ - public abstract Builder setTimestampFieldName(String value); - - /** - * The payload field name of the source {@link org.apache.beam.sdk.values.Row}. {@link - * org.apache.beam.sdk.schemas.Schema.FieldType} must be either {@link - * org.apache.beam.sdk.schemas.Schema.FieldType#BYTES} or a {@link - * org.apache.beam.sdk.values.Row}. If null, payload serialized from user fields other than - * attributes. Not compatible with other payload intended fields. - */ - public abstract Builder setPayloadFieldName(String value); - - public abstract SourceConfiguration build(); + public abstract PubsubWriteSchemaTransformConfiguration.ErrorHandling.Builder setOutput( + String output); + + public abstract PubsubWriteSchemaTransformConfiguration.ErrorHandling build(); } } - @DefaultSchema(AutoValueSchema.class) - @AutoValue - public abstract static class TargetConfiguration { + public static Builder builder() { + return new AutoValue_PubsubWriteSchemaTransformConfiguration.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setFormat(String format); - /** - * The attribute key to assign the {@link PubsubMessage} stringified timestamp value. {@link - * #builder()} method defaults value to {@link #DEFAULT_TIMESTAMP_ATTRIBUTE}. - */ - public abstract String getTimestampAttributeKey(); + public abstract Builder setTopic(String topic); - @AutoValue.Builder - public abstract static class Builder { + public abstract Builder setAttributes(@Nullable List attributes); - /** - * The attribute key to assign the {@link PubsubMessage} stringified timestamp value. Defaults - * to {@link #DEFAULT_TIMESTAMP_ATTRIBUTE}. - */ - public abstract Builder setTimestampAttributeKey(String value); + public abstract Builder setAttributesMap(@Nullable String attributesMap); - public abstract TargetConfiguration build(); - } + public abstract Builder setIdAttribute(@Nullable String idAttribute); + + public abstract Builder setTimestampAttribute(@Nullable String timestampAttribute); + + public abstract Builder setErrorHandling(@Nullable ErrorHandling errorHandling); + + public abstract PubsubWriteSchemaTransformConfiguration build(); } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubWriteSchemaTransformProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubWriteSchemaTransformProvider.java index bd23e3bfa07f1..6187f6f79d3e9 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubWriteSchemaTransformProvider.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubWriteSchemaTransformProvider.java @@ -17,57 +17,36 @@ */ package org.apache.beam.sdk.io.gcp.pubsub; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessage.ATTRIBUTES_FIELD_TYPE; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessage.DEFAULT_ATTRIBUTES_KEY_NAME; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessage.DEFAULT_EVENT_TIMESTAMP_KEY_NAME; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessage.DEFAULT_PAYLOAD_KEY_NAME; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessage.ERROR; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessage.EVENT_TIMESTAMP_FIELD_TYPE; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessage.OUTPUT; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessage.PAYLOAD_BYTES_TYPE_NAME; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessage.PAYLOAD_ROW_TYPE_NAME; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessage.removeFields; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; - -import com.google.api.client.util.Clock; import com.google.auto.service.AutoService; -import java.io.IOException; -import java.util.ArrayList; +import java.io.Serializable; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.stream.Stream; import javax.annotation.Nullable; -import org.apache.beam.sdk.annotations.Internal; -import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.SchemaPath; -import org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessage.FieldMatcher; -import org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessage.SchemaReflection; -import org.apache.beam.sdk.io.gcp.pubsub.PubsubWriteSchemaTransformConfiguration.SourceConfiguration; -import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils; import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.sdk.schemas.Schema.Field; -import org.apache.beam.sdk.schemas.Schema.FieldType; -import org.apache.beam.sdk.schemas.Schema.TypeName; -import org.apache.beam.sdk.schemas.io.Providers; -import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer; -import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializerProvider; -import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializers; import org.apache.beam.sdk.schemas.transforms.SchemaTransform; import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; +import org.apache.beam.sdk.schemas.utils.JsonUtils; import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.joda.time.Instant; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.sdk.values.TupleTagList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.checkerframework.checker.initialization.qual.Initialized; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.UnknownKeyFor; /** * An implementation of {@link TypedSchemaTransformProvider} for Pub/Sub reads configured using @@ -77,367 +56,211 @@ * provide no backwards compatibility guarantees, and it should not be implemented outside the Beam * repository. */ -@SuppressWarnings({ - "nullness" // TODO(https://github.com/apache/beam/issues/20497) -}) -@Internal @AutoService(SchemaTransformProvider.class) public class PubsubWriteSchemaTransformProvider extends TypedSchemaTransformProvider { - private static final String IDENTIFIER = "beam:schematransform:org.apache.beam:pubsub_write:v1"; - static final String INPUT_TAG = "input"; - static final String ERROR_TAG = "error"; - /** Returns the expected class of the configuration. */ - @Override - protected Class configurationClass() { - return PubsubWriteSchemaTransformConfiguration.class; - } + public static final TupleTag OUTPUT_TAG = new TupleTag() {}; + public static final TupleTag ERROR_TAG = new TupleTag() {}; - /** Returns the expected {@link SchemaTransform} of the configuration. */ - @Override - public SchemaTransform from(PubsubWriteSchemaTransformConfiguration configuration) { - return new PubsubWriteSchemaTransform(configuration); - } + public static final String VALID_FORMATS_STR = "RAW,AVRO,JSON"; + public static final Set VALID_DATA_FORMATS = + Sets.newHashSet(VALID_FORMATS_STR.split(",")); - /** Implementation of the {@link SchemaTransformProvider} identifier method. */ @Override - public String identifier() { - return IDENTIFIER; + public Class configurationClass() { + return PubsubWriteSchemaTransformConfiguration.class; } - /** - * Implementation of the {@link TypedSchemaTransformProvider} inputCollectionNames method. Since a - * single input is expected, this returns a list with a single name. - */ - @Override - public List inputCollectionNames() { - return Collections.singletonList(INPUT_TAG); + public static class ErrorFn extends DoFn { + private final SerializableFunction valueMapper; + private final @Nullable Set attributes; + private final @Nullable String attributesMap; + private final Schema payloadSchema; + private final Schema errorSchema; + private final boolean useErrorOutput; + + ErrorFn( + SerializableFunction valueMapper, + @Nullable List attributes, + @Nullable String attributesMap, + Schema payloadSchema, + Schema errorSchema, + boolean useErrorOutput) { + this.valueMapper = valueMapper; + this.attributes = attributes == null ? null : ImmutableSet.copyOf(attributes); + this.attributesMap = attributesMap; + this.payloadSchema = payloadSchema; + this.errorSchema = errorSchema; + this.useErrorOutput = useErrorOutput; + } + + @ProcessElement + public void processElement(@Element Row row, MultiOutputReceiver receiver) throws Exception { + try { + Row payloadRow; + Map messageAttributes = null; + if (attributes == null && attributesMap == null) { + payloadRow = row; + } else { + Row.Builder payloadRowBuilder = Row.withSchema(payloadSchema); + messageAttributes = new HashMap<>(); + List fields = row.getSchema().getFields(); + for (int ix = 0; ix < fields.size(); ix++) { + String name = fields.get(ix).getName(); + if (attributes != null && attributes.contains(name)) { + messageAttributes.put(name, row.getValue(ix)); + } else if (name.equals(attributesMap)) { + Map attrs = row.getMap(ix); + if (attrs != null) { + messageAttributes.putAll(attrs); + } + } else { + payloadRowBuilder.addValue(row.getValue(ix)); + } + } + payloadRow = payloadRowBuilder.build(); + } + receiver + .get(OUTPUT_TAG) + .output(new PubsubMessage(valueMapper.apply(payloadRow), messageAttributes)); + } catch (Exception e) { + if (useErrorOutput) { + receiver + .get(ERROR_TAG) + .output(Row.withSchema(errorSchema).addValues(e.toString(), row).build()); + } else { + throw e; + } + } + } } - /** - * Implementation of the {@link TypedSchemaTransformProvider} outputCollectionNames method. The - * only expected output is the {@link #ERROR_TAG}. - */ @Override - public List outputCollectionNames() { - return Collections.singletonList(ERROR_TAG); + public SchemaTransform from(PubsubWriteSchemaTransformConfiguration configuration) { + if (!VALID_DATA_FORMATS.contains(configuration.getFormat().toUpperCase())) { + throw new IllegalArgumentException( + String.format( + "Format %s not supported. Only supported formats are %s", + configuration.getFormat(), VALID_FORMATS_STR)); + } + return new PubsubWriteSchemaTransform(configuration); } - /** - * An implementation of {@link SchemaTransform} for Pub/Sub writes configured using {@link - * PubsubWriteSchemaTransformConfiguration}. - */ - static class PubsubWriteSchemaTransform - extends PTransform implements SchemaTransform { - - private final PubsubWriteSchemaTransformConfiguration configuration; - - private PubsubClient.PubsubClientFactory pubsubClientFactory; + private static class PubsubWriteSchemaTransform extends SchemaTransform implements Serializable { + final PubsubWriteSchemaTransformConfiguration configuration; PubsubWriteSchemaTransform(PubsubWriteSchemaTransformConfiguration configuration) { this.configuration = configuration; } - PubsubWriteSchemaTransform withPubsubClientFactory(PubsubClient.PubsubClientFactory factory) { - this.pubsubClientFactory = factory; - return this; - } - - /** Implements {@link SchemaTransform} buildTransform method. */ - @Override - public PTransform buildTransform() { - return this; - } - @Override + @SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/20497) + }) public PCollectionRowTuple expand(PCollectionRowTuple input) { - if (input.getAll().size() != 1 || !input.has(INPUT_TAG)) { - throw new IllegalArgumentException( - String.format( - "%s %s input is expected to contain a single %s tagged PCollection", - input.getClass().getSimpleName(), getClass().getSimpleName(), INPUT_TAG)); - } - - PCollection rows = input.get(INPUT_TAG); - if (rows.getSchema().getFieldCount() == 0) { - throw new IllegalArgumentException(String.format("empty Schema for %s", INPUT_TAG)); - } - - Schema targetSchema = buildTargetSchema(rows.getSchema()); - - rows = - rows.apply( - ConvertForRowToMessage.class.getSimpleName(), - convertForRowToMessage(targetSchema)) - .setRowSchema(targetSchema); - - Schema schema = rows.getSchema(); - - Schema serializableSchema = - removeFields(schema, DEFAULT_ATTRIBUTES_KEY_NAME, DEFAULT_EVENT_TIMESTAMP_KEY_NAME); - FieldMatcher payloadRowMatcher = FieldMatcher.of(DEFAULT_PAYLOAD_KEY_NAME, TypeName.ROW); - if (payloadRowMatcher.match(serializableSchema)) { - serializableSchema = - serializableSchema.getField(DEFAULT_PAYLOAD_KEY_NAME).getType().getRowSchema(); - } + String errorOutput = + configuration.getErrorHandling() == null + ? null + : configuration.getErrorHandling().getOutput(); - validateTargetSchemaAgainstPubsubSchema(serializableSchema, input.getPipeline().getOptions()); + final Schema errorSchema = + Schema.builder() + .addStringField("error") + .addNullableRowField("row", input.get("input").getSchema()) + .build(); - PCollectionTuple pct = - rows.apply( - PubsubRowToMessage.class.getSimpleName(), - buildPubsubRowToMessage(serializableSchema)); - - PCollection messages = pct.get(OUTPUT); - messages.apply(PubsubIO.Write.class.getSimpleName(), buildPubsubWrite()); - return PCollectionRowTuple.of(ERROR_TAG, pct.get(ERROR)); - } - - PayloadSerializer getPayloadSerializer(Schema schema) { - if (configuration.getFormat() == null) { - return null; - } String format = configuration.getFormat(); - Set availableFormats = - Providers.loadProviders(PayloadSerializerProvider.class).keySet(); - if (!availableFormats.contains(format)) { - String availableFormatsString = String.join(",", availableFormats); - throw new IllegalArgumentException( - String.format( - "%s is not among the valid formats: [%s]", format, availableFormatsString)); - } - return PayloadSerializers.getSerializer(configuration.getFormat(), schema, ImmutableMap.of()); - } - - PubsubRowToMessage buildPubsubRowToMessage(Schema schema) { - PubsubRowToMessage.Builder builder = - PubsubRowToMessage.builder().setPayloadSerializer(getPayloadSerializer(schema)); - - if (configuration.getTarget() != null) { - builder = - builder.setTargetTimestampAttributeName( - configuration.getTarget().getTimestampAttributeKey()); - } - - return builder.build(); - } - - PubsubIO.Write buildPubsubWrite() { - PubsubIO.Write write = PubsubIO.writeMessages().to(configuration.getTopic()); - - if (configuration.getIdAttribute() != null) { - write = write.withIdAttribute(configuration.getIdAttribute()); - } - - if (pubsubClientFactory != null) { - write = write.withClientFactory(pubsubClientFactory); - } - - return write; - } - - void validateSourceSchemaAgainstConfiguration(Schema sourceSchema) { - if (sourceSchema.getFieldCount() == 0) { - throw new IllegalArgumentException(String.format("empty Schema for %s", INPUT_TAG)); - } - - if (configuration.getSource() == null) { - return; - } - - SourceConfiguration source = configuration.getSource(); - - if (source.getAttributesFieldName() != null) { - String fieldName = source.getAttributesFieldName(); - FieldType fieldType = ATTRIBUTES_FIELD_TYPE; - FieldMatcher fieldMatcher = FieldMatcher.of(fieldName, fieldType); - checkArgument( - fieldMatcher.match(sourceSchema), - String.format("schema missing field: %s for type %s: ", fieldName, fieldType)); - } - - if (source.getTimestampFieldName() != null) { - String fieldName = source.getTimestampFieldName(); - FieldType fieldType = EVENT_TIMESTAMP_FIELD_TYPE; - FieldMatcher fieldMatcher = FieldMatcher.of(fieldName, fieldType); - checkArgument( - fieldMatcher.match(sourceSchema), - String.format("schema missing field: %s for type: %s", fieldName, fieldType)); - } - - if (source.getPayloadFieldName() == null) { - return; - } - - String fieldName = source.getPayloadFieldName(); - FieldMatcher bytesFieldMatcher = FieldMatcher.of(fieldName, PAYLOAD_BYTES_TYPE_NAME); - FieldMatcher rowFieldMatcher = FieldMatcher.of(fieldName, PAYLOAD_ROW_TYPE_NAME); - SchemaReflection schemaReflection = SchemaReflection.of(sourceSchema); - checkArgument( - schemaReflection.matchesAny(bytesFieldMatcher, rowFieldMatcher), - String.format( - "schema missing field: %s for types %s or %s", - fieldName, PAYLOAD_BYTES_TYPE_NAME, PAYLOAD_ROW_TYPE_NAME)); - - String[] fieldsToExclude = - Stream.of( - source.getAttributesFieldName(), - source.getTimestampFieldName(), - source.getPayloadFieldName()) - .filter(Objects::nonNull) - .toArray(String[]::new); - - Schema userFieldsSchema = removeFields(sourceSchema, fieldsToExclude); - - if (userFieldsSchema.getFieldCount() > 0) { - throw new IllegalArgumentException( - String.format("user fields incompatible with %s field", source.getPayloadFieldName())); - } - } - - void validateTargetSchemaAgainstPubsubSchema(Schema targetSchema, PipelineOptions options) { - checkArgument(options != null); - - try (PubsubClient pubsubClient = getPubsubClient(options.as(PubsubOptions.class))) { - PubsubClient.TopicPath topicPath = PubsubClient.topicPathFromPath(configuration.getTopic()); - PubsubClient.SchemaPath schemaPath = pubsubClient.getSchemaPath(topicPath); - if (schemaPath == null || schemaPath.equals(SchemaPath.DELETED_SCHEMA)) { - return; + Schema beamSchema = input.get("input").getSchema(); + Schema payloadSchema; + if (configuration.getAttributes() == null && configuration.getAttributesMap() == null) { + payloadSchema = beamSchema; + } else { + Schema.Builder payloadSchemaBuilder = Schema.builder(); + for (Schema.Field f : beamSchema.getFields()) { + if (!configuration.getAttributes().contains(f.getName()) + && !f.getName().equals(configuration.getAttributesMap())) { + payloadSchemaBuilder.addField(f); + } } - Schema expectedSchema = pubsubClient.getSchema(schemaPath); - checkState( - targetSchema.equals(expectedSchema), - String.format( - "input schema mismatch with expected schema at path: %s\ninput schema: %s\nPub/Sub schema: %s", - schemaPath, targetSchema, expectedSchema)); - } catch (IOException e) { - throw new IllegalStateException(e.getMessage()); + payloadSchema = payloadSchemaBuilder.build(); } - } - - Schema buildTargetSchema(Schema sourceSchema) { - validateSourceSchemaAgainstConfiguration(sourceSchema); - FieldType payloadFieldType = null; - - List fieldsToRemove = new ArrayList<>(); - - if (configuration.getSource() != null) { - SourceConfiguration source = configuration.getSource(); - - if (source.getAttributesFieldName() != null) { - fieldsToRemove.add(source.getAttributesFieldName()); + SerializableFunction fn; + if (Objects.equals(format, "RAW")) { + if (payloadSchema.getFieldCount() != 1) { + throw new IllegalArgumentException( + String.format( + "Raw output only supported for single-field schemas, got %s", payloadSchema)); } - - if (source.getTimestampFieldName() != null) { - fieldsToRemove.add(source.getTimestampFieldName()); - } - - if (source.getPayloadFieldName() != null) { - String fieldName = source.getPayloadFieldName(); - Field field = sourceSchema.getField(fieldName); - payloadFieldType = field.getType(); - fieldsToRemove.add(fieldName); + if (payloadSchema.getField(0).getType().equals(Schema.FieldType.BYTES)) { + fn = row -> row.getBytes(0); + } else if (payloadSchema.getField(0).getType().equals(Schema.FieldType.STRING)) { + fn = row -> row.getString(0).getBytes(StandardCharsets.UTF_8); + } else { + throw new IllegalArgumentException( + String.format( + "Raw output only supports bytes and string fields, got %s", + payloadSchema.getField(0))); } + } else if (Objects.equals(format, "JSON")) { + fn = JsonUtils.getRowToJsonBytesFunction(payloadSchema); + } else if (Objects.equals(format, "AVRO")) { + fn = AvroUtils.getRowToAvroBytesFunction(payloadSchema); + } else { + throw new IllegalArgumentException( + String.format( + "Format %s not supported. Only supported formats are %s", + format, VALID_FORMATS_STR)); } - Schema targetSchema = - PubsubRowToMessage.builder() - .build() - .inputSchemaFactory(payloadFieldType) - .buildSchema(sourceSchema.getFields().toArray(new Field[0])); - - return removeFields(targetSchema, fieldsToRemove.toArray(new String[0])); - } - - private PubsubClient.PubsubClientFactory getPubsubClientFactory() { - if (pubsubClientFactory != null) { - return pubsubClientFactory; + PCollectionTuple outputTuple = + input + .get("input") + .apply( + ParDo.of( + new ErrorFn( + fn, + configuration.getAttributes(), + configuration.getAttributesMap(), + payloadSchema, + errorSchema, + errorOutput != null)) + .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); + + PubsubIO.Write writeTransform = + PubsubIO.writeMessages().to(configuration.getTopic()); + if (!Strings.isNullOrEmpty(configuration.getIdAttribute())) { + writeTransform = writeTransform.withIdAttribute(configuration.getIdAttribute()); } - return PubsubGrpcClient.FACTORY; - } - - private PubsubClient getPubsubClient(PubsubOptions options) throws IOException { - return getPubsubClientFactory() - .newClient( - configuration.getTarget().getTimestampAttributeKey(), - configuration.getIdAttribute(), - options); - } - - ParDo.SingleOutput convertForRowToMessage(Schema targetSchema) { - return convertForRowToMessage(targetSchema, null); - } - - ParDo.SingleOutput convertForRowToMessage( - Schema targetSchema, @Nullable Clock clock) { - String attributesName = null; - String timestampName = null; - String payloadName = null; - SourceConfiguration source = configuration.getSource(); - if (source != null) { - attributesName = source.getAttributesFieldName(); - timestampName = source.getTimestampFieldName(); - payloadName = source.getPayloadFieldName(); + if (!Strings.isNullOrEmpty(configuration.getTimestampAttribute())) { + writeTransform = writeTransform.withIdAttribute(configuration.getTimestampAttribute()); + } + outputTuple.get(OUTPUT_TAG).apply(writeTransform); + outputTuple.get(ERROR_TAG).setRowSchema(errorSchema); + + if (errorOutput == null) { + return PCollectionRowTuple.empty(input.getPipeline()); + } else { + return PCollectionRowTuple.of( + errorOutput, outputTuple.get(ERROR_TAG).setRowSchema(errorSchema)); } - return ParDo.of( - new ConvertForRowToMessage( - targetSchema, clock, attributesName, timestampName, payloadName)); } } - private static class ConvertForRowToMessage extends DoFn { - private final Schema targetSchema; - @Nullable private final Clock clock; - @Nullable private final String attributesFieldName; - @Nullable private final String timestampFieldName; - @Nullable private final String payloadFieldName; - - ConvertForRowToMessage( - Schema targetSchema, - @Nullable Clock clock, - @Nullable String attributesFieldName, - @Nullable String timestampFieldName, - @Nullable String payloadFieldName) { - this.targetSchema = targetSchema; - this.clock = clock; - this.attributesFieldName = attributesFieldName; - this.timestampFieldName = timestampFieldName; - this.payloadFieldName = payloadFieldName; - } - - @ProcessElement - public void process(@Element Row row, OutputReceiver receiver) { - Instant now = Instant.now(); - if (clock != null) { - now = Instant.ofEpochMilli(clock.currentTimeMillis()); - } - Map values = new HashMap<>(); - - // Default attributes value - checkState(targetSchema.hasField(DEFAULT_ATTRIBUTES_KEY_NAME)); - values.put(DEFAULT_ATTRIBUTES_KEY_NAME, ImmutableMap.of()); - - // Default timestamp value - checkState(targetSchema.hasField(DEFAULT_EVENT_TIMESTAMP_KEY_NAME)); - values.put(DEFAULT_EVENT_TIMESTAMP_KEY_NAME, now); + @Override + public @UnknownKeyFor @NonNull @Initialized String identifier() { + return "beam:schematransform:org.apache.beam:pubsub_write:v1"; + } - for (String fieldName : row.getSchema().getFieldNames()) { - if (targetSchema.hasField(fieldName)) { - values.put(fieldName, row.getValue(fieldName)); - } + @Override + public @UnknownKeyFor @NonNull @Initialized List<@UnknownKeyFor @NonNull @Initialized String> + inputCollectionNames() { + return Collections.singletonList("input"); + } - if (attributesFieldName != null) { - values.put(DEFAULT_ATTRIBUTES_KEY_NAME, row.getValue(attributesFieldName)); - } - if (timestampFieldName != null) { - values.put(DEFAULT_EVENT_TIMESTAMP_KEY_NAME, row.getValue(timestampFieldName)); - } - if (payloadFieldName != null) { - values.put(DEFAULT_PAYLOAD_KEY_NAME, row.getValue(payloadFieldName)); - } - } - receiver.output(Row.withSchema(targetSchema).withFieldValues(values).build()); - } + @Override + public @UnknownKeyFor @NonNull @Initialized List<@UnknownKeyFor @NonNull @Initialized String> + outputCollectionNames() { + return Collections.singletonList("errors"); } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/TestPubsub.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/TestPubsub.java index 459f922797878..2736424e8dc02 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/TestPubsub.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/TestPubsub.java @@ -50,9 +50,9 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.testing.TestPipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Streams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Streams; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matcher; import org.joda.time.DateTime; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/TestPubsubSignal.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/TestPubsubSignal.java index f68f6c877fdc0..1d430ac4a6b60 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/TestPubsubSignal.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/TestPubsubSignal.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.gcp.pubsub; import static org.apache.beam.sdk.io.gcp.pubsub.TestPubsub.createTopicName; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.cloud.pubsub.v1.AckReplyConsumer; import com.google.cloud.pubsub.v1.MessageReceiver; @@ -54,9 +54,9 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.POutput; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Suppliers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Suppliers; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTime; import org.joda.time.Duration; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteReadSchemaTransformProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteReadSchemaTransformProvider.java index 5ea205393c5f4..f6acd081ab238 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteReadSchemaTransformProvider.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteReadSchemaTransformProvider.java @@ -44,7 +44,6 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.DoFn.FinishBundle; import org.apache.beam.sdk.transforms.DoFn.ProcessElement; -import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollectionRowTuple; @@ -52,8 +51,8 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; @@ -139,48 +138,39 @@ public void finish(FinishBundleContext c) { : AvroUtils.getAvroBytesToRowFunction(beamSchema); return new SchemaTransform() { @Override - public @UnknownKeyFor @NonNull @Initialized PTransform< - @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple, - @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple> - buildTransform() { - return new PTransform() { - @Override - public PCollectionRowTuple expand(PCollectionRowTuple input) { - String project = configuration.getProject(); - if (Strings.isNullOrEmpty(project)) { - project = input.getPipeline().getOptions().as(GcpOptions.class).getProject(); - } - if (project == null) { - throw new IllegalArgumentException( - "Unable to infer the project to read from Pubsub Lite. Please provide a project."); - } - PCollectionTuple outputTuple = - input - .getPipeline() - .apply( - PubsubLiteIO.read( - SubscriberOptions.newBuilder() - .setSubscriptionPath( - SubscriptionPath.newBuilder() - .setLocation( - CloudRegionOrZone.parse(configuration.getLocation())) - .setProject(ProjectId.of(project)) - .setName( - SubscriptionName.of( - configuration.getSubscriptionName())) - .build()) - .build())) - .apply( - ParDo.of(new ErrorFn("PubsubLite-read-error-counter", valueMapper)) - .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); - - return PCollectionRowTuple.of( - "output", - outputTuple.get(OUTPUT_TAG).setRowSchema(beamSchema), - "errors", - outputTuple.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA)); - } - }; + public PCollectionRowTuple expand(PCollectionRowTuple input) { + String project = configuration.getProject(); + if (Strings.isNullOrEmpty(project)) { + project = input.getPipeline().getOptions().as(GcpOptions.class).getProject(); + } + if (project == null) { + throw new IllegalArgumentException( + "Unable to infer the project to read from Pubsub Lite. Please provide a project."); + } + PCollectionTuple outputTuple = + input + .getPipeline() + .apply( + PubsubLiteIO.read( + SubscriberOptions.newBuilder() + .setSubscriptionPath( + SubscriptionPath.newBuilder() + .setLocation( + CloudRegionOrZone.parse(configuration.getLocation())) + .setProject(ProjectId.of(project)) + .setName( + SubscriptionName.of(configuration.getSubscriptionName())) + .build()) + .build())) + .apply( + ParDo.of(new ErrorFn("PubsubLite-read-error-counter", valueMapper)) + .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); + + return PCollectionRowTuple.of( + "output", + outputTuple.get(OUTPUT_TAG).setRowSchema(beamSchema), + "errors", + outputTuple.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA)); } }; } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteWriteSchemaTransformProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteWriteSchemaTransformProvider.java index 3785b07a3b454..1b4d2bd77fb54 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteWriteSchemaTransformProvider.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteWriteSchemaTransformProvider.java @@ -43,7 +43,6 @@ import org.apache.beam.sdk.schemas.utils.JsonUtils; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.DoFn.ProcessElement; -import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollectionRowTuple; @@ -51,7 +50,7 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.UnknownKeyFor; @@ -131,49 +130,39 @@ public void finish() { } return new SchemaTransform() { - @Override - public @UnknownKeyFor @NonNull @Initialized PTransform< - @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple, - @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple> - buildTransform() { - return new PTransform() { - @Override - public PCollectionRowTuple expand(PCollectionRowTuple input) { - Schema inputSchema = input.get("input").getSchema(); - final SerializableFunction toBytesFn = - configuration.getFormat().equals("JSON") - ? JsonUtils.getRowToJsonBytesFunction(inputSchema) - : AvroUtils.getRowToAvroBytesFunction(inputSchema); - - PCollectionTuple outputTuple = - input - .get("input") - .apply( - "Map Rows to PubSubMessages", - ParDo.of(new ErrorCounterFn("PubSubLite-write-error-counter", toBytesFn)) - .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); - - outputTuple - .get(OUTPUT_TAG) - .apply("Add UUIDs", PubsubLiteIO.addUuids()) + public PCollectionRowTuple expand(PCollectionRowTuple input) { + Schema inputSchema = input.get("input").getSchema(); + final SerializableFunction toBytesFn = + configuration.getFormat().equals("JSON") + ? JsonUtils.getRowToJsonBytesFunction(inputSchema) + : AvroUtils.getRowToAvroBytesFunction(inputSchema); + + PCollectionTuple outputTuple = + input + .get("input") .apply( - "Write to PS Lite", - PubsubLiteIO.write( - PublisherOptions.newBuilder() - .setTopicPath( - TopicPath.newBuilder() - .setProject(ProjectId.of(configuration.getProject())) - .setName(TopicName.of(configuration.getTopicName())) - .setLocation( - CloudRegionOrZone.parse(configuration.getLocation())) - .build()) - .build())); - - return PCollectionRowTuple.of( - "errors", outputTuple.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA)); - } - }; + "Map Rows to PubSubMessages", + ParDo.of(new ErrorCounterFn("PubSubLite-write-error-counter", toBytesFn)) + .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); + + outputTuple + .get(OUTPUT_TAG) + .apply("Add UUIDs", PubsubLiteIO.addUuids()) + .apply( + "Write to PS Lite", + PubsubLiteIO.write( + PublisherOptions.newBuilder() + .setTopicPath( + TopicPath.newBuilder() + .setProject(ProjectId.of(configuration.getProject())) + .setName(TopicName.of(configuration.getTopicName())) + .setLocation(CloudRegionOrZone.parse(configuration.getLocation())) + .build()) + .build())); + + return PCollectionRowTuple.of( + "errors", outputTuple.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA)); } }; } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/CheckpointMarkImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/CheckpointMarkImpl.java index b66275855b40f..0f8d5f329f2fc 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/CheckpointMarkImpl.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/CheckpointMarkImpl.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.pubsublite.internal; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.cloud.pubsublite.Offset; import java.io.IOException; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ExternalTransformRegistrarImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ExternalTransformRegistrarImpl.java index bd0110300b5c1..9b7b5b8b51ce8 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ExternalTransformRegistrarImpl.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ExternalTransformRegistrarImpl.java @@ -23,7 +23,7 @@ import org.apache.beam.sdk.io.gcp.pubsublite.internal.ExternalTransformConfig.ReadExternalBuilder; import org.apache.beam.sdk.io.gcp.pubsublite.internal.ExternalTransformConfig.WriteExternalBuilder; import org.apache.beam.sdk.transforms.ExternalTransformBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; @AutoService(ExternalTransformRegistrar.class) public class ExternalTransformRegistrarImpl implements ExternalTransformRegistrar { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/LimitingTopicBacklogReader.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/LimitingTopicBacklogReader.java index ec7c672a3dc98..43a7d1d20e4b5 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/LimitingTopicBacklogReader.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/LimitingTopicBacklogReader.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.gcp.pubsublite.internal; import static com.google.cloud.pubsublite.internal.ExtractStatus.toCanonical; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.gax.rpc.ApiException; import com.google.cloud.pubsublite.Offset; @@ -26,10 +26,10 @@ import com.google.errorprone.annotations.concurrent.GuardedBy; import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Ticker; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LoadingCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Ticker; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache; final class LimitingTopicBacklogReader implements TopicBacklogReader { private final TopicBacklogReader underlying; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/MemoryLimiterImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/MemoryLimiterImpl.java index 3e54cc57f5756..3f86e880f8de3 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/MemoryLimiterImpl.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/MemoryLimiterImpl.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.pubsublite.internal; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import javax.annotation.concurrent.GuardedBy; import org.slf4j.Logger; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/OffsetByteRangeTracker.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/OffsetByteRangeTracker.java index f4ac39da98ce2..43233294fa543 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/OffsetByteRangeTracker.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/OffsetByteRangeTracker.java @@ -18,8 +18,8 @@ package org.apache.beam.sdk.io.gcp.pubsublite.internal; import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.cloud.pubsublite.Offset; import com.google.cloud.pubsublite.proto.ComputeMessageStatsResponse; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ServiceCache.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ServiceCache.java index 5e124c04a1681..fa25933e2b2bb 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ServiceCache.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ServiceCache.java @@ -26,7 +26,7 @@ import com.google.errorprone.annotations.concurrent.GuardedBy; import java.util.HashMap; import java.util.function.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/SubscribeTransform.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/SubscribeTransform.java index f3ffbb13c2495..882294de1771e 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/SubscribeTransform.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/SubscribeTransform.java @@ -56,7 +56,7 @@ public class SubscribeTransform extends PTransform { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/BatchSpannerRead.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/BatchSpannerRead.java index 26d6a1e5eeec2..5e5c215fdebcb 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/BatchSpannerRead.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/BatchSpannerRead.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.spanner; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import com.google.cloud.spanner.BatchReadOnlyTransaction; @@ -38,10 +38,10 @@ import org.apache.beam.sdk.transforms.Reshuffle; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LoadingCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationCellCounter.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationCellCounter.java index 0ac7bc94d200d..74ed09115faa2 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationCellCounter.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationCellCounter.java @@ -22,7 +22,7 @@ import com.google.cloud.spanner.KeySet; import com.google.cloud.spanner.Mutation; import com.google.cloud.spanner.Mutation.Op; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; final class MutationCellCounter { // Prevent construction. diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationGroup.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationGroup.java index 48c6b646d2ebc..9e8b68e919c0c 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationGroup.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationGroup.java @@ -22,8 +22,8 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationKeyEncoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationKeyEncoder.java index bd00d0b2488d1..35eb922af7f71 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationKeyEncoder.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationKeyEncoder.java @@ -35,7 +35,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import org.apache.beam.sdk.io.gcp.spanner.SpannerSchema.KeyPart; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.DateTime; import org.joda.time.Days; import org.joda.time.MutableDateTime; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationUtils.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationUtils.java index 9b74e21d28525..fddbef5b256ce 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationUtils.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationUtils.java @@ -20,7 +20,7 @@ import static java.util.stream.Collectors.toList; import static org.apache.beam.sdk.io.gcp.spanner.StructUtils.beamRowToStruct; import static org.apache.beam.sdk.io.gcp.spanner.StructUtils.beamTypeToSpannerType; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.cloud.ByteArray; import com.google.cloud.Timestamp; @@ -32,7 +32,7 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.ReadableDateTime; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/NaiveSpannerRead.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/NaiveSpannerRead.java index c12b646ef954d..45cc331657f14 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/NaiveSpannerRead.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/NaiveSpannerRead.java @@ -32,7 +32,7 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; /** A naive version of Spanner read that doesn't use the Batch API. */ diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/OrderedCode.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/OrderedCode.java index fbe3dc77e7b11..34557fb38e346 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/OrderedCode.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/OrderedCode.java @@ -17,15 +17,15 @@ */ package org.apache.beam.sdk.io.gcp.spanner; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.math.RoundingMode; import java.util.ArrayList; import java.util.Arrays; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.math.LongMath; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.UnsignedInteger; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.math.LongMath; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Longs; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedInteger; /** * This module provides routines for encoding a sequence of typed entities into a byte array. The diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerConfig.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerConfig.java index d7f993694fe86..4f2b9c8bd0987 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerConfig.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerConfig.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.spanner; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.rpc.StatusCode.Code; @@ -29,8 +29,8 @@ import java.io.Serializable; import org.apache.beam.sdk.options.ValueProvider; import org.apache.beam.sdk.transforms.display.DisplayData; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java index ef584a33cec4e..786fa91f55820 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java @@ -26,8 +26,8 @@ import static org.apache.beam.sdk.io.gcp.spanner.changestreams.ChangeStreamsConstants.MAX_INCLUSIVE_END_AT; import static org.apache.beam.sdk.io.gcp.spanner.changestreams.ChangeStreamsConstants.THROUGHPUT_WINDOW_SECONDS; import static org.apache.beam.sdk.io.gcp.spanner.changestreams.NameGenerator.generatePartitionMetadataTableName; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.rpc.StatusCode.Code; @@ -121,16 +121,16 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Stopwatch; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LoadingCache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.UnsignedBytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Stopwatch; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedBytes; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; @@ -2005,10 +2005,14 @@ static class WriteToSpannerFn extends DoFn, Void> { /* Number of times an aborted write to spanner could be retried */ private static final int ABORTED_RETRY_ATTEMPTS = 5; /* Error string in Aborted exception during schema change */ - private final String errString = + private final String schemaChangeErrString = "Transaction aborted. " + "Database schema probably changed during transaction, retry may succeed."; + /* Error string in Aborted exception for concurrent transaction in Spanner Emulator */ + private final String emulatorErrorString = + "The emulator only supports one transaction at a time."; + @VisibleForTesting static Sleeper sleeper = Sleeper.DEFAULT; private final Counter mutationGroupBatchesReceived = @@ -2139,7 +2143,9 @@ private void spannerWriteWithRetryIfSchemaChange(List batch) throws Sp if (retry >= ABORTED_RETRY_ATTEMPTS) { throw e; } - if (e.isRetryable() || e.getMessage().contains(errString)) { + if (e.isRetryable() + || e.getMessage().contains(schemaChangeErrString) + || e.getMessage().contains(emulatorErrorString)) { continue; } throw e; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java index b441ed25c9064..fbeaac8ecd241 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java @@ -22,12 +22,12 @@ import com.google.cloud.spanner.Type; import java.io.Serializable; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableTable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; /** Encapsulates Cloud Spanner Schema. */ @AutoValue diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerTransformRegistrar.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerTransformRegistrar.java index 38d4bc9ca8d17..38cd97da860a9 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerTransformRegistrar.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerTransformRegistrar.java @@ -37,7 +37,7 @@ import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteResult.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteResult.java index d387aab0e9618..87f6734503651 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteResult.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteResult.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * The results of a {@link SpannerIO#write()} transform. diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteSchemaTransformProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteSchemaTransformProvider.java index 43e0de3b9032f..6c8a2541a88bd 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteSchemaTransformProvider.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteSchemaTransformProvider.java @@ -21,10 +21,13 @@ import com.google.auto.value.AutoValue; import com.google.cloud.spanner.Mutation; import java.io.Serializable; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import org.apache.beam.sdk.metrics.Counter; +import org.apache.beam.sdk.metrics.Metrics; import org.apache.beam.sdk.schemas.AutoValueSchema; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.annotations.DefaultSchema; @@ -32,15 +35,16 @@ import org.apache.beam.sdk.schemas.transforms.SchemaTransform; import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; +import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.FlatMapElements; import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.SimpleFunction; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.UnknownKeyFor; @@ -62,74 +66,90 @@ public class SpannerWriteSchemaTransformProvider return new SpannerSchemaTransformWrite(configuration); } - static class SpannerSchemaTransformWrite implements SchemaTransform, Serializable { + static class SpannerSchemaTransformWrite extends SchemaTransform implements Serializable { private final SpannerWriteSchemaTransformConfiguration configuration; SpannerSchemaTransformWrite(SpannerWriteSchemaTransformConfiguration configuration) { this.configuration = configuration; } + // A generic counter for PCollection of Row. Will be initialized with the given + // name argument. Performs element-wise counter of the input PCollection. + private static class ElementCounterFn extends DoFn { + + private Counter spannerGenericElementCounter; + private Long elementsInBundle = 0L; + + ElementCounterFn(String name) { + this.spannerGenericElementCounter = + Metrics.counter(SpannerSchemaTransformWrite.class, name); + } + + @ProcessElement + public void process(ProcessContext c) { + this.elementsInBundle += 1; + c.output(c.element()); + } + + @FinishBundle + public void finish(FinishBundleContext c) { + this.spannerGenericElementCounter.inc(this.elementsInBundle); + this.elementsInBundle = 0L; + } + } + @Override - public @UnknownKeyFor @NonNull @Initialized PTransform< - @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple, - @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple> - buildTransform() { - // TODO: For now we are allowing ourselves to fail at runtime, but we could - // perform validations here at expansion time. This TODO is to add a few - // validations (e.g. table/database/instance existence, schema match, etc). - return new PTransform<@NonNull PCollectionRowTuple, @NonNull PCollectionRowTuple>() { - @Override - public PCollectionRowTuple expand(@NonNull PCollectionRowTuple input) { - SpannerWriteResult result = - input - .get("input") - .apply( - MapElements.via( - new SimpleFunction( - row -> - MutationUtils.createMutationFromBeamRows( - Mutation.newInsertOrUpdateBuilder(configuration.getTableId()), - Objects.requireNonNull(row))) {})) - .apply( - SpannerIO.write() - .withDatabaseId(configuration.getDatabaseId()) - .withInstanceId(configuration.getInstanceId()) - .withFailureMode(SpannerIO.FailureMode.REPORT_FAILURES)); - Schema failureSchema = - Schema.builder() - .addStringField("operation") - .addStringField("instanceId") - .addStringField("databaseId") - .addStringField("tableId") - .addStringField("mutationData") - .build(); - PCollection failures = - result - .getFailedMutations() - .apply( - FlatMapElements.into(TypeDescriptors.rows()) - .via( - mtg -> - Objects.requireNonNull(mtg).attached().stream() - .map( - mutation -> - Row.withSchema(failureSchema) - .addValue(mutation.getOperation().toString()) - .addValue(configuration.getInstanceId()) - .addValue(configuration.getDatabaseId()) - .addValue(mutation.getTable()) - // TODO(pabloem): Figure out how to represent - // mutation - // contents in DLQ - .addValue( - Iterators.toString( - mutation.getValues().iterator())) - .build()) - .collect(Collectors.toList()))) - .setRowSchema(failureSchema); - return PCollectionRowTuple.of("failures", failures); - } - }; + public PCollectionRowTuple expand(@NonNull PCollectionRowTuple input) { + SpannerWriteResult result = + input + .get("input") + .apply( + MapElements.via( + new SimpleFunction( + row -> + MutationUtils.createMutationFromBeamRows( + Mutation.newInsertOrUpdateBuilder(configuration.getTableId()), + Objects.requireNonNull(row))) {})) + .apply( + SpannerIO.write() + .withDatabaseId(configuration.getDatabaseId()) + .withInstanceId(configuration.getInstanceId()) + .withFailureMode(SpannerIO.FailureMode.REPORT_FAILURES)); + Schema failureSchema = + Schema.builder() + .addStringField("operation") + .addStringField("instanceId") + .addStringField("databaseId") + .addStringField("tableId") + .addStringField("mutationData") + .build(); + PCollection failures = + result + .getFailedMutations() + .apply( + FlatMapElements.into(TypeDescriptors.rows()) + .via( + mtg -> + Objects.requireNonNull(mtg).attached().stream() + .map( + mutation -> + Row.withSchema(failureSchema) + .addValue(mutation.getOperation().toString()) + .addValue(configuration.getInstanceId()) + .addValue(configuration.getDatabaseId()) + .addValue(mutation.getTable()) + // TODO(pabloem): Figure out how to represent + // mutation + // contents in DLQ + .addValue( + Iterators.toString( + mutation.getValues().iterator())) + .build()) + .collect(Collectors.toList()))) + .setRowSchema(failureSchema) + .apply("error-count", ParDo.of(new ElementCounterFn("Spanner-write-error-counter"))) + .setRowSchema(failureSchema); + return PCollectionRowTuple.of("failures", failures).and("errors", failures); } } @@ -147,7 +167,7 @@ public PCollectionRowTuple expand(@NonNull PCollectionRowTuple input) { @Override public @UnknownKeyFor @NonNull @Initialized List<@UnknownKeyFor @NonNull @Initialized String> outputCollectionNames() { - return Collections.singletonList("failures"); + return Arrays.asList("failures", "errors"); } @AutoValue diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/StructUtils.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/StructUtils.java index 057552a6dfc69..12cb91359cd97 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/StructUtils.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/StructUtils.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.gcp.spanner; import static java.util.stream.Collectors.toList; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.cloud.ByteArray; import com.google.cloud.Timestamp; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/ChangeStreamsConstants.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/ChangeStreamsConstants.java index 7e3d17f44e532..ba8dfc8a17275 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/ChangeStreamsConstants.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/ChangeStreamsConstants.java @@ -22,7 +22,7 @@ import java.util.Collections; import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.PartitionMetadata; import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.PartitionMetadata.State; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; /** * Single place for defining the constants used in the {@code Spanner.readChangeStreams()} diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/MetadataSpannerConfigFactory.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/MetadataSpannerConfigFactory.java index 56c67f3194f46..30b5043f54e8a 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/MetadataSpannerConfigFactory.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/MetadataSpannerConfigFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.spanner.changestreams; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.rpc.StatusCode.Code; @@ -25,7 +25,7 @@ import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; import org.apache.beam.sdk.options.ValueProvider; import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.joda.time.Duration; /** diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/SpannerChangestreamsReadSchemaTransformProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/SpannerChangestreamsReadSchemaTransformProvider.java index 14001232bb52c..51e55dc7664a5 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/SpannerChangestreamsReadSchemaTransformProvider.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/SpannerChangestreamsReadSchemaTransformProvider.java @@ -55,7 +55,6 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.DoFn.FinishBundle; -import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.View; import org.apache.beam.sdk.values.PCollectionRowTuple; @@ -65,8 +64,8 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.vendor.grpc.v1p54p0.com.google.gson.Gson; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; @@ -100,54 +99,44 @@ public class SpannerChangestreamsReadSchemaTransformProvider configuration) { return new SchemaTransform() { @Override - public @UnknownKeyFor @NonNull @Initialized PTransform< - @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple, - @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple> - buildTransform() { - return new PTransform() { - @Override - public PCollectionRowTuple expand(PCollectionRowTuple input) { - Pipeline p = input.getPipeline(); - // TODO(pabloem): Does this action create/destroy a new metadata table?? - Schema tableChangesSchema = getTableSchema(configuration); - SpannerIO.ReadChangeStream readChangeStream = - SpannerIO.readChangeStream() - .withSpannerConfig( - SpannerConfig.create() - .withProjectId(configuration.getProjectId()) - .withInstanceId(configuration.getInstanceId()) - .withDatabaseId(configuration.getDatabaseId())) - .withChangeStreamName(configuration.getChangeStreamName()) - .withInclusiveStartAt( - Timestamp.parseTimestamp(configuration.getStartAtTimestamp())) - .withDatabaseId(configuration.getDatabaseId()) - .withProjectId(configuration.getProjectId()) - .withInstanceId(configuration.getInstanceId()); - - if (configuration.getEndAtTimestamp() != null) { - String endTs = - Objects.requireNonNull(Objects.requireNonNull(configuration.getEndAtTimestamp())); - readChangeStream = - readChangeStream.withInclusiveEndAt(Timestamp.parseTimestamp(endTs)); - } + public PCollectionRowTuple expand(PCollectionRowTuple input) { + Pipeline p = input.getPipeline(); + // TODO(pabloem): Does this action create/destroy a new metadata table?? + Schema tableChangesSchema = getTableSchema(configuration); + SpannerIO.ReadChangeStream readChangeStream = + SpannerIO.readChangeStream() + .withSpannerConfig( + SpannerConfig.create() + .withProjectId(configuration.getProjectId()) + .withInstanceId(configuration.getInstanceId()) + .withDatabaseId(configuration.getDatabaseId())) + .withChangeStreamName(configuration.getChangeStreamName()) + .withInclusiveStartAt(Timestamp.parseTimestamp(configuration.getStartAtTimestamp())) + .withDatabaseId(configuration.getDatabaseId()) + .withProjectId(configuration.getProjectId()) + .withInstanceId(configuration.getInstanceId()); + + if (configuration.getEndAtTimestamp() != null) { + String endTs = + Objects.requireNonNull(Objects.requireNonNull(configuration.getEndAtTimestamp())); + readChangeStream = readChangeStream.withInclusiveEndAt(Timestamp.parseTimestamp(endTs)); + } - PCollectionTuple outputTuple = - p.apply(readChangeStream) - .apply( - ParDo.of( - new DataChangeRecordToRow( - configuration.getTable(), - tableChangesSchema, - "SpannerChangestreams-read-error-counter")) - .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); - - return PCollectionRowTuple.of( - "output", - outputTuple.get(OUTPUT_TAG).setRowSchema(tableChangesSchema), - "errors", - outputTuple.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA)); - } - }; + PCollectionTuple outputTuple = + p.apply(readChangeStream) + .apply( + ParDo.of( + new DataChangeRecordToRow( + configuration.getTable(), + tableChangesSchema, + "SpannerChangestreams-read-error-counter")) + .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); + + return PCollectionRowTuple.of( + "output", + outputTuple.get(OUTPUT_TAG).setRowSchema(tableChangesSchema), + "errors", + outputTuple.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA)); } }; } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/ChildPartitionsRecordAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/ChildPartitionsRecordAction.java index 7fb69d0e7abad..ada794d20c3bf 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/ChildPartitionsRecordAction.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/ChildPartitionsRecordAction.java @@ -30,7 +30,7 @@ import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation; import org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.Instant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/DataChangeRecordAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/DataChangeRecordAction.java index e391feb6e2595..4ceda8afb3e6d 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/DataChangeRecordAction.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/DataChangeRecordAction.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation; import org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.Instant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/HeartbeatRecordAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/HeartbeatRecordAction.java index d009f774b3bcc..83a232fe2093b 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/HeartbeatRecordAction.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/HeartbeatRecordAction.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation; import org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.Instant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/QueryChangeStreamAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/QueryChangeStreamAction.java index 2a69fcb861976..92285946e56f6 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/QueryChangeStreamAction.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/QueryChangeStreamAction.java @@ -40,7 +40,7 @@ import org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker; import org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.Duration; import org.joda.time.Instant; import org.slf4j.Logger; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/dao/PartitionMetadataDao.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/dao/PartitionMetadataDao.java index bde04bb1dec51..7867932cd1adc 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/dao/PartitionMetadataDao.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/dao/PartitionMetadataDao.java @@ -51,7 +51,7 @@ import javax.annotation.Nullable; import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.PartitionMetadata; import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.PartitionMetadata.State; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/mapper/ChangeStreamRecordMapper.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/mapper/ChangeStreamRecordMapper.java index cfd6d91a65dc6..20314566dcc71 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/mapper/ChangeStreamRecordMapper.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/mapper/ChangeStreamRecordMapper.java @@ -45,7 +45,7 @@ import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.PartitionMetadata; import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.TypeCode; import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.ValueCaptureType; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; /** * This class is responsible for transforming a {@link Struct} to a {@link List} of {@link diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/mapper/PartitionMetadataMapper.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/mapper/PartitionMetadataMapper.java index cb6c3753d4751..df48d5f131266 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/mapper/PartitionMetadataMapper.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/mapper/PartitionMetadataMapper.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.io.gcp.spanner.changestreams.dao.PartitionMetadataAdminDao; import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.PartitionMetadata; import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.PartitionMetadata.State; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; /** This class is responsible for transforming a {@link Struct} to a {@link PartitionMetadata}. */ public class PartitionMetadataMapper { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/ChangeStreamRecordMetadata.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/ChangeStreamRecordMetadata.java index 22c295550b79d..8151c7202f704 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/ChangeStreamRecordMetadata.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/ChangeStreamRecordMetadata.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.coders.DefaultCoder; import org.apache.beam.sdk.extensions.avro.coders.AvroCoder; import org.apache.beam.sdk.io.gcp.spanner.changestreams.encoder.TimestampEncoding; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** Holds internal execution metrics / metadata for the processed {@link ChangeStreamRecord}. */ @SuppressWarnings({ diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/ChildPartition.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/ChildPartition.java index 84ab846712c17..53026f0bc9a12 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/ChildPartition.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/ChildPartition.java @@ -23,7 +23,7 @@ import javax.annotation.Nullable; import org.apache.beam.sdk.coders.DefaultCoder; import org.apache.beam.sdk.extensions.avro.coders.AvroCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; /** * A child partition represents a new partition that should be queried. Child partitions are emitted diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/InitialPartition.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/InitialPartition.java index 3790f577476df..25d74fde5f1fc 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/InitialPartition.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/InitialPartition.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.gcp.spanner.changestreams.model; import java.util.HashSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; /** * Utility class to determine initial partition constants and methods. diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/PartitionMetadata.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/PartitionMetadata.java index 765675d347434..f8a698271dba6 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/PartitionMetadata.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/PartitionMetadata.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.coders.DefaultCoder; import org.apache.beam.sdk.extensions.avro.coders.AvroCoder; import org.apache.beam.sdk.io.gcp.spanner.changestreams.encoder.TimestampEncoding; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; /** Model for the partition metadata database table used in the Connector. */ @SuppressWarnings("initialization.fields.uninitialized") // Avro requires the default constructor diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/restriction/TimestampRange.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/restriction/TimestampRange.java index d5d9de57c3f68..1e85f9d0b7ff6 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/restriction/TimestampRange.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/restriction/TimestampRange.java @@ -17,13 +17,13 @@ */ package org.apache.beam.sdk.io.gcp.spanner.changestreams.restriction; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.cloud.Timestamp; import java.io.Serializable; import java.util.Objects; import javax.annotation.Nullable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** A restriction represented by a range of timestamps [from, to). */ public class TimestampRange implements Serializable { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/restriction/TimestampRangeTracker.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/restriction/TimestampRangeTracker.java index 180070407f8c9..99491bd4bab34 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/restriction/TimestampRangeTracker.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/restriction/TimestampRangeTracker.java @@ -21,9 +21,9 @@ import static org.apache.beam.sdk.io.gcp.spanner.changestreams.restriction.TimestampUtils.next; import static org.apache.beam.sdk.io.gcp.spanner.changestreams.restriction.TimestampUtils.toNanos; import static org.apache.beam.sdk.io.gcp.spanner.changestreams.restriction.TimestampUtils.toTimestamp; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.cloud.Timestamp; import java.math.BigDecimal; @@ -32,7 +32,7 @@ import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker.HasProgress; import org.apache.beam.sdk.transforms.splittabledofn.SplitResult; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/BigqueryClient.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/BigqueryClient.java index 584d1c7123da4..0e9476e6a226a 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/BigqueryClient.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/BigqueryClient.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.gcp.testing; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.JsonFactory; @@ -64,8 +64,8 @@ import org.apache.beam.sdk.extensions.gcp.util.BackOffAdapter; import org.apache.beam.sdk.extensions.gcp.util.Transport; import org.apache.beam.sdk.util.FluentBackoff; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.slf4j.Logger; @@ -292,8 +292,24 @@ private QueryResponse getTypedTableRows(QueryResponse response) { public List queryUnflattened( String query, String projectId, boolean typed, boolean useStandardSql) throws IOException, InterruptedException { + return queryUnflattened(query, projectId, typed, useStandardSql, null); + } + + /** + * Performs a query without flattening results. May choose a location (GCP region) to perform this + * operation in. + */ + @Nonnull + public List queryUnflattened( + String query, + String projectId, + boolean typed, + boolean useStandardSql, + @Nullable String location) + throws IOException, InterruptedException { Random rnd = new Random(System.currentTimeMillis()); - String temporaryDatasetId = "_dataflow_temporary_dataset_" + rnd.nextInt(1000000); + String temporaryDatasetId = + String.format("_dataflow_temporary_dataset_%s_%s", System.nanoTime(), rnd.nextInt(1000000)); String temporaryTableId = "dataflow_temporary_table_" + rnd.nextInt(1000000); TableReference tempTableReference = new TableReference() @@ -301,9 +317,11 @@ public List queryUnflattened( .setDatasetId(temporaryDatasetId) .setTableId(temporaryTableId); - createNewDataset(projectId, temporaryDatasetId); + createNewDataset(projectId, temporaryDatasetId, null, location); createNewTable( - projectId, temporaryDatasetId, new Table().setTableReference(tempTableReference)); + projectId, + temporaryDatasetId, + new Table().setTableReference(tempTableReference).setLocation(location)); JobConfigurationQuery jcQuery = new JobConfigurationQuery() @@ -324,6 +342,7 @@ public List queryUnflattened( bqClient .jobs() .getQueryResults(projectId, insertedJob.getJobReference().getJobId()) + .setLocation(location) .execute(); } while (!qResponse.getJobComplete()); @@ -394,6 +413,18 @@ public void createNewDataset(String projectId, String datasetId) public void createNewDataset( String projectId, String datasetId, @Nullable Long defaultTableExpirationMs) throws IOException, InterruptedException { + createNewDataset(projectId, datasetId, defaultTableExpirationMs, null); + } + + /** + * Creates a new dataset with defaultTableExpirationMs and in a specified location (GCP region). + */ + public void createNewDataset( + String projectId, + String datasetId, + @Nullable Long defaultTableExpirationMs, + @Nullable String location) + throws IOException, InterruptedException { Sleeper sleeper = Sleeper.DEFAULT; BackOff backoff = BackOffAdapter.toGcpBackOff(BACKOFF_FACTORY.backoff()); IOException lastException = null; @@ -409,7 +440,8 @@ public void createNewDataset( projectId, new Dataset() .setDatasetReference(new DatasetReference().setDatasetId(datasetId)) - .setDefaultTableExpirationMs(defaultTableExpirationMs)) + .setDefaultTableExpirationMs(defaultTableExpirationMs) + .setLocation(location)) .execute(); if (response != null) { LOG.info("Successfully created new dataset : " + response.getId()); @@ -570,4 +602,19 @@ public Table getTableResource(String projectId, String datasetId, String tableId MAX_QUERY_RETRIES, tableId), lastException); } + + public void updateTableSchema( + String projectId, String datasetId, String tableId, TableSchema newSchema) + throws IOException { + this.bqClient + .tables() + .patch(projectId, datasetId, tableId, new Table().setSchema(newSchema)) + .execute(); + LOG.info( + "Successfully updated the schema of table {}:{}.{}. New schema:\n{}", + projectId, + datasetId, + tableId, + newSchema.toPrettyString()); + } } diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/BigqueryMatcher.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/BigqueryMatcher.java index cd23fd73d73fe..cbd7a75f7c407 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/BigqueryMatcher.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/BigqueryMatcher.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.testing; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.api.services.bigquery.model.QueryResponse; import com.google.api.services.bigquery.model.TableCell; @@ -33,10 +33,10 @@ import javax.annotation.concurrent.NotThreadSafe; import org.apache.beam.sdk.io.gcp.testing.BigqueryMatcher.TableAndQuery; import org.apache.beam.sdk.testing.SerializableMatcher; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashCode; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.HashCode; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; import org.hamcrest.Description; import org.hamcrest.TypeSafeMatcher; import org.slf4j.Logger; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/BigtableUtils.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/BigtableUtils.java index 67ed384e007f2..0d1a5e05b172a 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/BigtableUtils.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/BigtableUtils.java @@ -18,8 +18,8 @@ package org.apache.beam.sdk.io.gcp.testing; import com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Ints; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Ints; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Longs; public class BigtableUtils { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeBigQueryServices.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeBigQueryServices.java index 677cf26c3745a..5e6e3ac7ed079 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeBigQueryServices.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeBigQueryServices.java @@ -34,7 +34,7 @@ import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices; import org.apache.beam.sdk.io.gcp.bigquery.TableRowJsonCoder; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** A fake implementation of BigQuery's query service.. */ @Internal diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeDatasetService.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeDatasetService.java index 6782db77ac1b3..347a3513d8968 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeDatasetService.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeDatasetService.java @@ -43,6 +43,7 @@ import com.google.errorprone.annotations.FormatMethod; import com.google.errorprone.annotations.FormatString; import com.google.protobuf.ByteString; +import com.google.protobuf.DescriptorProtos; import com.google.protobuf.Descriptors; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.DynamicMessage; @@ -72,10 +73,10 @@ import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.values.FailsafeValueInSingleWindow; import org.apache.beam.sdk.values.ValueInSingleWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashBasedTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBasedTable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; /** A fake dataset service that can be serialized, for use in testReadFromTable. */ @Internal @@ -85,7 +86,7 @@ public class FakeDatasetService implements DatasetService, Serializable { // Table information must be static, as each ParDo will get a separate instance of // FakeDatasetServices, and they must all modify the same storage. - static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Table< + static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Table< String, String, Map> tables; static Map writeStreams; @@ -599,7 +600,8 @@ public WriteStream getWriteStream(String streamName) { @Override public StreamAppendClient getStreamAppendClient( - String streamName, Descriptor descriptor, boolean useConnectionPool) { + String streamName, DescriptorProtos.DescriptorProto descriptor, boolean useConnectionPool) + throws Exception { return new StreamAppendClient() { private Descriptor protoDescriptor; private TableSchema currentSchema; @@ -609,7 +611,8 @@ public StreamAppendClient getStreamAppendClient( private boolean usedForUpdate = false; { - this.protoDescriptor = descriptor; + this.protoDescriptor = TableRowToStorageApiProto.wrapDescriptorProto(descriptor); + synchronized (FakeDatasetService.class) { Stream stream = writeStreams.get(streamName); if (stream == null) { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeJobService.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeJobService.java index d2f6816806c6e..ac09b11638ded 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeJobService.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeJobService.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.gcp.testing; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.json.JsonFactory; @@ -82,9 +82,9 @@ import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.MimeTypes; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashBasedTable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBasedTable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; /** A fake implementation of BigQuery's job service. */ @@ -116,15 +116,15 @@ private static class JobInfo { } } - private static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Table< + private static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Table< String, String, JobInfo> allJobs; private static int numExtractJobCalls; - private static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Table< + private static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Table< String, String, List> filesForLoadJobs; - private static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Table< + private static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Table< String, String, JobStatistics> dryRunQueryResults; diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/TableContainer.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/TableContainer.java index 264d9ef1f5b6f..b50aa4d32d76c 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/TableContainer.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/TableContainer.java @@ -27,8 +27,8 @@ import java.util.stream.IntStream; import org.apache.beam.sdk.io.gcp.bigquery.TableRowJsonCoder; import org.apache.beam.sdk.util.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.checkerframework.checker.nullness.qual.Nullable; /** Encapsulates a BigQuery Table, and it's contents. */ diff --git a/sdks/java/io/google-cloud-platform/src/test/java/com/google/cloud/spanner/FakeBatchTransactionId.java b/sdks/java/io/google-cloud-platform/src/test/java/com/google/cloud/spanner/FakeBatchTransactionId.java index 759946b7b827b..4862ac9187001 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/com/google/cloud/spanner/FakeBatchTransactionId.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/com/google/cloud/spanner/FakeBatchTransactionId.java @@ -19,7 +19,7 @@ import com.google.cloud.Timestamp; import com.google.protobuf.ByteString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java index 20e39b20a5cc9..e32f45a438231 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.io.gcp.testing.BigqueryClient; import org.apache.beam.sdk.io.gcp.testing.BigqueryMatcher; import org.apache.beam.sdk.util.ApiSurface; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.hamcrest.Matcher; import org.hamcrest.Matchers; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProtoTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProtoTest.java index b584441966b19..f2f5cb692628d 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProtoTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProtoTest.java @@ -43,9 +43,9 @@ import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecordBuilder; import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Functions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Functions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Days; import org.joda.time.Instant; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProtoTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProtoTest.java index 809f980b63542..af025d9f030a3 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProtoTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProtoTest.java @@ -43,9 +43,9 @@ import org.apache.beam.sdk.schemas.logicaltypes.EnumerationType; import org.apache.beam.sdk.schemas.logicaltypes.SqlTypes; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Functions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Functions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.commons.math3.util.Pair; import org.joda.time.Instant; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java index 9b85d9adf067c..0bdecb31a3163 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java @@ -42,9 +42,9 @@ import org.apache.avro.util.Utf8; import org.apache.beam.sdk.coders.DefaultCoder; import org.apache.beam.sdk.extensions.avro.coders.AvroCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; import org.apache.commons.lang3.tuple.Pair; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryClusteringIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryClusteringIT.java index 67777b2658858..1b3c844e2a9f2 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryClusteringIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryClusteringIT.java @@ -50,7 +50,7 @@ public class BigQueryClusteringIT { private static final Long EXPECTED_BYTES = 16000L; private static final BigInteger EXPECTED_ROWS = new BigInteger("1000"); private static final String WEATHER_SAMPLES_TABLE = - "clouddataflow-readonly:samples.weather_stations"; + "apache-beam-testing.samples.weather_stations"; private static final String DATASET_NAME = "BigQueryClusteringIT"; private static final Clustering CLUSTERING = new Clustering().setFields(Arrays.asList("station_number")); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryExportReadSchemaTransformProviderTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryExportReadSchemaTransformProviderTest.java index af2f1351e1866..c732434b2bac5 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryExportReadSchemaTransformProviderTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryExportReadSchemaTransformProviderTest.java @@ -31,7 +31,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryExportReadSchemaTransformProvider.PCollectionRowTupleTransform; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryExportReadSchemaTransformProvider.BigQueryExportSchemaTransform; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead; import org.apache.beam.sdk.io.gcp.testing.FakeBigQueryServices; import org.apache.beam.sdk.io.gcp.testing.FakeDatasetService; @@ -39,7 +39,6 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.Schema.Field; import org.apache.beam.sdk.schemas.Schema.FieldType; -import org.apache.beam.sdk.schemas.transforms.SchemaTransform; import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; @@ -157,11 +156,9 @@ public void testQuery() { SchemaTransformProvider provider = new BigQueryExportReadSchemaTransformProvider(); BigQueryExportReadSchemaTransformConfiguration configuration = caze.getLeft().build(); Row configurationRow = configuration.toBeamRow(); - SchemaTransform schemaTransform = provider.from(configurationRow); - PCollectionRowTupleTransform pCollectionRowTupleTransform = - (PCollectionRowTupleTransform) schemaTransform.buildTransform(); - Map got = - DisplayData.from(pCollectionRowTupleTransform.toTypedRead()).asMap(); + BigQueryExportSchemaTransform schemaTransform = + (BigQueryExportSchemaTransform) provider.from(configurationRow); + Map got = DisplayData.from(schemaTransform.toTypedRead()).asMap(); assertEquals(want, got); } } @@ -172,14 +169,13 @@ public void testExtract() { BigQueryExportReadSchemaTransformConfiguration configuration = BigQueryExportReadSchemaTransformConfiguration.builder().setTableSpec(TABLE_SPEC).build(); Row configurationRow = configuration.toBeamRow(); - SchemaTransform schemaTransform = provider.from(configurationRow); - PCollectionRowTupleTransform pCollectionRowTupleTransform = - (PCollectionRowTupleTransform) schemaTransform.buildTransform(); + BigQueryExportSchemaTransform schemaTransform = + (BigQueryExportSchemaTransform) provider.from(configurationRow); - pCollectionRowTupleTransform.setTestBigQueryServices(fakeBigQueryServices); + schemaTransform.setTestBigQueryServices(fakeBigQueryServices); PCollectionRowTuple input = PCollectionRowTuple.empty(p); String tag = provider.outputCollectionNames().get(0); - PCollectionRowTuple output = input.apply(pCollectionRowTupleTransform); + PCollectionRowTuple output = input.apply(schemaTransform); assertTrue(output.has(tag)); PCollection got = output.get(tag); PAssert.that(got).containsInAnyOrder(ROWS); @@ -212,12 +208,11 @@ public void testInvalidConfiguration() { .setUseStandardSql(true), IllegalArgumentException.class))) { Row configurationRow = caze.getLeft().build().toBeamRow(); - SchemaTransform schemaTransform = provider.from(configurationRow); - PCollectionRowTupleTransform pCollectionRowTupleTransform = - (PCollectionRowTupleTransform) schemaTransform.buildTransform(); - pCollectionRowTupleTransform.setTestBigQueryServices(fakeBigQueryServices); + BigQueryExportSchemaTransform schemaTransform = + (BigQueryExportSchemaTransform) provider.from(configurationRow); + schemaTransform.setTestBigQueryServices(fakeBigQueryServices); PCollectionRowTuple empty = PCollectionRowTuple.empty(p); - assertThrows(caze.getRight(), () -> empty.apply(pCollectionRowTupleTransform)); + assertThrows(caze.getRight(), () -> empty.apply(schemaTransform)); } } @@ -227,13 +222,12 @@ public void testInvalidInput() { BigQueryExportReadSchemaTransformConfiguration configuration = BigQueryExportReadSchemaTransformConfiguration.builder().setTableSpec(TABLE_SPEC).build(); Row configurationRow = configuration.toBeamRow(); - SchemaTransform schemaTransform = provider.from(configurationRow); - PCollectionRowTupleTransform pCollectionRowTupleTransform = - (PCollectionRowTupleTransform) schemaTransform.buildTransform(); + BigQueryExportSchemaTransform schemaTransform = + (BigQueryExportSchemaTransform) provider.from(configurationRow); - pCollectionRowTupleTransform.setTestBigQueryServices(fakeBigQueryServices); + schemaTransform.setTestBigQueryServices(fakeBigQueryServices); PCollectionRowTuple input = PCollectionRowTuple.of("badinput", p.apply(Create.of(ROWS))); - assertThrows(IllegalArgumentException.class, () -> input.apply(pCollectionRowTupleTransform)); + assertThrows(IllegalArgumentException.class, () -> input.apply(schemaTransform)); } private void assertEquals(Map want, Map got) { diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryFileLoadsWriteSchemaTransformProviderTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryFileLoadsWriteSchemaTransformProviderTest.java index eb881801cb7d0..194746d9825ad 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryFileLoadsWriteSchemaTransformProviderTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryFileLoadsWriteSchemaTransformProviderTest.java @@ -31,7 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryFileLoadsWriteSchemaTransformProvider.PCollectionRowTupleTransform; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryFileLoadsWriteSchemaTransformProvider.BigQueryWriteSchemaTransform; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition; import org.apache.beam.sdk.io.gcp.testing.FakeBigQueryServices; @@ -41,7 +41,6 @@ import org.apache.beam.sdk.schemas.Schema.Field; import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.io.InvalidConfigurationException; -import org.apache.beam.sdk.schemas.transforms.SchemaTransform; import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; @@ -117,14 +116,13 @@ public void testLoad() throws IOException, InterruptedException { .setCreateDisposition(CreateDisposition.CREATE_IF_NEEDED.name()) .build(); Row configurationRow = configuration.toBeamRow(); - SchemaTransform schemaTransform = provider.from(configurationRow); - PCollectionRowTupleTransform pCollectionRowTupleTransform = - (PCollectionRowTupleTransform) schemaTransform.buildTransform(); - pCollectionRowTupleTransform.setTestBigQueryServices(fakeBigQueryServices); + BigQueryWriteSchemaTransform schemaTransform = + (BigQueryWriteSchemaTransform) provider.from(configurationRow); + schemaTransform.setTestBigQueryServices(fakeBigQueryServices); String tag = provider.inputCollectionNames().get(0); PCollectionRowTuple input = PCollectionRowTuple.of(tag, p.apply(Create.of(ROWS).withRowSchema(SCHEMA))); - input.apply(pCollectionRowTupleTransform); + input.apply(schemaTransform); p.run(); @@ -161,7 +159,7 @@ public void testValidatePipelineOptions() { for (Pair< BigQueryFileLoadsWriteSchemaTransformConfiguration.Builder, Class> caze : cases) { - PCollectionRowTupleTransform transform = transformFrom(caze.getLeft().build()); + BigQueryWriteSchemaTransform transform = transformFrom(caze.getLeft().build()); if (caze.getRight() != null) { assertThrows(caze.getRight(), () -> transform.validate(p.getOptions())); } else { @@ -201,7 +199,7 @@ public void testToWrite() { for (Pair< BigQueryFileLoadsWriteSchemaTransformConfiguration.Builder, BigQueryIO.Write> caze : cases) { - PCollectionRowTupleTransform transform = transformFrom(caze.getLeft().build()); + BigQueryWriteSchemaTransform transform = transformFrom(caze.getLeft().build()); Map gotDisplayData = DisplayData.from(transform.toWrite(SCHEMA)).asMap(); Map wantDisplayData = DisplayData.from(caze.getRight()).asMap(); Set keys = new HashSet<>(); @@ -237,7 +235,7 @@ public void validatePCollectionRowTupleInput() { Row.nullRow( Schema.builder().addNullableField("name", FieldType.STRING).build())))); - PCollectionRowTupleTransform transform = + BigQueryWriteSchemaTransform transform = transformFrom( BigQueryFileLoadsWriteSchemaTransformConfiguration.builder() .setTableSpec(BigQueryHelpers.toTableSpec(TABLE_REFERENCE)) @@ -254,11 +252,11 @@ public void validatePCollectionRowTupleInput() { p.run(); } - private PCollectionRowTupleTransform transformFrom( + private BigQueryWriteSchemaTransform transformFrom( BigQueryFileLoadsWriteSchemaTransformConfiguration configuration) { SchemaTransformProvider provider = new BigQueryFileLoadsWriteSchemaTransformProvider(); - PCollectionRowTupleTransform transform = - (PCollectionRowTupleTransform) provider.from(configuration.toBeamRow()).buildTransform(); + BigQueryWriteSchemaTransform transform = + (BigQueryWriteSchemaTransform) provider.from(configuration.toBeamRow()); transform.setTestBigQueryServices(fakeBigQueryServices); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpersTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpersTest.java index fbb70a9dea51b..f5f4ddb9547a0 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpersTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpersTest.java @@ -42,8 +42,8 @@ import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.WindowedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.joda.time.Duration; import org.junit.Assert; import org.junit.Rule; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOJsonIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOJsonIT.java index 6cf758c575d6c..4b7e02e4bf47b 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOJsonIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOJsonIT.java @@ -49,8 +49,8 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.json.JSONArray; import org.json.JSONObject; import org.junit.BeforeClass; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOReadIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOReadIT.java index 40d6de288008e..8a3bc0fac2b77 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOReadIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOReadIT.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.testing.TestPipelineOptions; import org.apache.beam.sdk.transforms.Count; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOReadTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOReadTest.java index 01f58880527b9..bc75ba8bd9baf 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOReadTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOReadTest.java @@ -77,9 +77,10 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.MoreCollectors; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -804,6 +805,12 @@ public void testBigQueryTableSourceInitSplit() throws Exception { // Simulate a repeated call to split(), like a Dataflow worker will sometimes do. sources = bqSource.split(200, options); assertEquals(2, sources.size()); + assertEquals( + TableRowJsonCoder.of(), + sources.stream() + .map(BoundedSource::getOutputCoder) + .distinct() + .collect(MoreCollectors.onlyElement())); // A repeated call to split() should not have caused a duplicate extract job. assertEquals(1, fakeJobService.getNumExtractJobCalls()); @@ -992,6 +999,12 @@ public void testBigQueryQuerySourceInitSplit() throws Exception { List> sources = bqSource.split(100, options); assertEquals(2, sources.size()); + assertEquals( + TableRowJsonCoder.of(), + sources.stream() + .map(BoundedSource::getOutputCoder) + .distinct() + .collect(MoreCollectors.onlyElement())); } /** @@ -1058,6 +1071,12 @@ public void testBigQueryQuerySourceInitSplit_NoReferencedTables() throws Excepti List> sources = bqSource.split(100, options); assertEquals(2, sources.size()); + assertEquals( + TableRowJsonCoder.of(), + sources.stream() + .map(BoundedSource::getOutputCoder) + .distinct() + .collect(MoreCollectors.onlyElement())); } @Test diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageQueryIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageQueryIT.java index fa29f7b8447ba..d355d6bb93366 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageQueryIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageQueryIT.java @@ -17,6 +17,8 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; +import static org.apache.beam.sdk.io.gcp.bigquery.TestBigQueryOptions.BIGQUERY_EARLY_ROLLOUT_REGION; + import java.util.Map; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; @@ -32,7 +34,7 @@ import org.apache.beam.sdk.transforms.Count; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -52,7 +54,13 @@ public class BigQueryIOStorageQueryIT { "1G", 11110839L, "1T", 11110839000L); - private static final String DATASET_ID = "big_query_storage"; + private static final String DATASET_ID = + TestPipeline.testingPipelineOptions() + .as(TestBigQueryOptions.class) + .getBigQueryLocation() + .equals(BIGQUERY_EARLY_ROLLOUT_REGION) + ? "big_query_storage_day0" + : "big_query_storage"; private static final String TABLE_PREFIX = "storage_read_"; private BigQueryIOStorageQueryOptions options; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageQueryTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageQueryTest.java index b23de6b3fc334..af6dd505b916f 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageQueryTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageQueryTest.java @@ -80,8 +80,9 @@ import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.MoreCollectors; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -426,6 +427,12 @@ private void doQuerySourceInitialSplit( List> sources = querySource.split(bundleSize, options); assertEquals(expectedStreamCount, sources.size()); + assertEquals( + TableRowJsonCoder.of(), + sources.stream() + .map(BoundedSource::getOutputCoder) + .distinct() + .collect(MoreCollectors.onlyElement())); } /** @@ -520,6 +527,12 @@ public void testQuerySourceInitialSplit_NoReferencedTables() throws Exception { List> sources = querySource.split(1024, options); assertEquals(1024, sources.size()); + assertEquals( + TableRowJsonCoder.of(), + sources.stream() + .map(BoundedSource::getOutputCoder) + .distinct() + .collect(MoreCollectors.onlyElement())); } private static final String AVRO_SCHEMA_STRING = diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadIT.java index bb576be7dd165..b4f6ddb76f720 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadIT.java @@ -17,6 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; +import static org.apache.beam.sdk.io.gcp.bigquery.TestBigQueryOptions.BIGQUERY_EARLY_ROLLOUT_REGION; import static org.junit.Assert.assertEquals; import com.google.cloud.bigquery.storage.v1.DataFormat; @@ -44,7 +45,7 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -65,7 +66,13 @@ public class BigQueryIOStorageReadIT { "1T", 11110839000L, "multi_field", 11110839L); - private static final String DATASET_ID = "big_query_storage"; + private static final String DATASET_ID = + TestPipeline.testingPipelineOptions() + .as(TestBigQueryOptions.class) + .getBigQueryLocation() + .equals(BIGQUERY_EARLY_ROLLOUT_REGION) + ? "big_query_storage_day0" + : "big_query_storage"; private static final String TABLE_PREFIX = "storage_read_"; private BigQueryIOStorageReadOptions options; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadTableRowIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadTableRowIT.java index 734c3af2c4d43..35e2676c70ef9 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadTableRowIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadTableRowIT.java @@ -17,6 +17,8 @@ */ package org.apache.beam.sdk.io.gcp.bigquery; +import static org.apache.beam.sdk.io.gcp.bigquery.TestBigQueryOptions.BIGQUERY_EARLY_ROLLOUT_REGION; + import com.google.api.services.bigquery.model.TableRow; import java.util.HashSet; import java.util.Set; @@ -52,7 +54,13 @@ @RunWith(JUnit4.class) public class BigQueryIOStorageReadTableRowIT { - private static final String DATASET_ID = "big_query_import_export"; + private static final String DATASET_ID = + TestPipeline.testingPipelineOptions() + .as(TestBigQueryOptions.class) + .getBigQueryLocation() + .equals(BIGQUERY_EARLY_ROLLOUT_REGION) + ? "big_query_import_export_day0" + : "big_query_import_export"; private static final String TABLE_PREFIX = "parallel_read_table_row_"; private BigQueryIOStorageReadTableRowOptions options; @@ -67,12 +75,11 @@ public interface BigQueryIOStorageReadTableRowOptions void setInputTable(String table); } - private static class TableRowToKVPairFn extends SimpleFunction> { + private static class TableRowToKVPairFn extends SimpleFunction> { @Override - public KV apply(TableRow input) { - CharSequence sampleString = (CharSequence) input.get("sample_string"); - String key = sampleString != null ? sampleString.toString() : "null"; - return KV.of(key, BigQueryHelpers.toJsonString(input)); + public KV apply(TableRow input) { + Integer rowId = Integer.parseInt((String) input.get("id")); + return KV.of(rowId, BigQueryHelpers.toJsonString(input)); } } @@ -87,7 +94,7 @@ private void setUpTestEnvironment(String tableName) { private static void runPipeline(BigQueryIOStorageReadTableRowOptions pipelineOptions) { Pipeline pipeline = Pipeline.create(pipelineOptions); - PCollection> jsonTableRowsFromExport = + PCollection> jsonTableRowsFromExport = pipeline .apply( "ExportTable", @@ -96,7 +103,7 @@ private static void runPipeline(BigQueryIOStorageReadTableRowOptions pipelineOpt .withMethod(Method.EXPORT)) .apply("MapExportedRows", MapElements.via(new TableRowToKVPairFn())); - PCollection> jsonTableRowsFromDirectRead = + PCollection> jsonTableRowsFromDirectRead = pipeline .apply( "DirectReadTable", @@ -108,16 +115,16 @@ private static void runPipeline(BigQueryIOStorageReadTableRowOptions pipelineOpt final TupleTag exportTag = new TupleTag<>(); final TupleTag directReadTag = new TupleTag<>(); - PCollection>> unmatchedRows = + PCollection>> unmatchedRows = KeyedPCollectionTuple.of(exportTag, jsonTableRowsFromExport) .and(directReadTag, jsonTableRowsFromDirectRead) .apply(CoGroupByKey.create()) .apply( ParDo.of( - new DoFn, KV>>() { + new DoFn, KV>>() { @ProcessElement - public void processElement(ProcessContext c) throws Exception { - KV element = c.element(); + public void processElement(ProcessContext c) { + KV element = c.element(); // Add all the exported rows for the key to a collection. Set uniqueRows = new HashSet<>(); @@ -147,20 +154,20 @@ public void processElement(ProcessContext c) throws Exception { } @Test - public void testBigQueryStorageReadTableRow1() throws Exception { - setUpTestEnvironment("1"); + public void testBigQueryStorageReadTableRow100() { + setUpTestEnvironment("100"); runPipeline(options); } @Test - public void testBigQueryStorageReadTableRow10k() throws Exception { - setUpTestEnvironment("10k"); + public void testBigQueryStorageReadTableRow1k() { + setUpTestEnvironment("1K"); runPipeline(options); } @Test - public void testBigQueryStorageReadTableRow100k() throws Exception { - setUpTestEnvironment("100k"); + public void testBigQueryStorageReadTableRow10k() { + setUpTestEnvironment("10K"); runPipeline(options); } } diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadTest.java index 491c5ada4f3a7..5a78f529e8fe0 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadTest.java @@ -115,9 +115,9 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadWithStreamBundleSourceTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadWithStreamBundleSourceTest.java index fc1ccd3c89149..5facb3c76a36c 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadWithStreamBundleSourceTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadWithStreamBundleSourceTest.java @@ -105,9 +105,9 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageWriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageWriteIT.java index db4a1008ecdd3..d061898d55c77 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageWriteIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageWriteIT.java @@ -26,16 +26,25 @@ import com.google.api.services.bigquery.model.TableRow; import com.google.api.services.bigquery.model.TableSchema; import java.io.IOException; +import java.security.SecureRandom; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; import org.apache.beam.sdk.io.GenerateSequence; import org.apache.beam.sdk.io.gcp.testing.BigqueryClient; -import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.sdk.transforms.PeriodicImpulse; +import org.apache.beam.sdk.transforms.SimpleFunction; +import org.apache.beam.sdk.values.PBegin; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; +import org.joda.time.Instant; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -53,24 +62,37 @@ private enum WriteMode { AT_LEAST_ONCE } - private String project; - private static final String DATASET_ID = "big_query_storage"; + private static String project; + private static final String DATASET_ID = + "big_query_storage_write_it_" + + System.currentTimeMillis() + + "_" + + new SecureRandom().nextInt(32); private static final String TABLE_PREFIX = "storage_write_"; - private BigQueryOptions bqOptions; + private static TestBigQueryOptions bqOptions; private static final BigqueryClient BQ_CLIENT = new BigqueryClient("BigQueryStorageIOWriteIT"); + @BeforeClass + public static void setup() throws Exception { + bqOptions = TestPipeline.testingPipelineOptions().as(TestBigQueryOptions.class); + project = bqOptions.as(GcpOptions.class).getProject(); + // Create one BQ dataset for all test cases. + BQ_CLIENT.createNewDataset(project, DATASET_ID, null, bqOptions.getBigQueryLocation()); + } + + @AfterClass + public static void cleanup() { + BQ_CLIENT.deleteDataset(project, DATASET_ID); + } + private void setUpTestEnvironment(WriteMode writeMode) { - PipelineOptionsFactory.register(BigQueryOptions.class); - bqOptions = TestPipeline.testingPipelineOptions().as(BigQueryOptions.class); - bqOptions.setProject(TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject()); bqOptions.setUseStorageWriteApi(true); if (writeMode == WriteMode.AT_LEAST_ONCE) { bqOptions.setUseStorageWriteApiAtLeastOnce(true); } bqOptions.setNumStorageWriteApiStreams(2); bqOptions.setStorageWriteApiTriggeringFrequencySec(1); - project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); } static class FillRowFn extends DoFn { @@ -81,11 +103,32 @@ public void processElement(ProcessContext c) { } } - private GenerateSequence stream(int rowCount) { - int timestampIntervalInMilliseconds = 10; - return GenerateSequence.from(0) - .to(rowCount) - .withRate(1, Duration.millis(timestampIntervalInMilliseconds)); + static class UnboundedStream extends PTransform> { + + private final int rowCount; + + public UnboundedStream(int rowCount) { + this.rowCount = rowCount; + } + + @Override + public PCollection expand(PBegin input) { + int timestampIntervalInMillis = 10; + PeriodicImpulse impulse = + PeriodicImpulse.create() + .stopAfter(Duration.millis((long) timestampIntervalInMillis * rowCount - 1)) + .withInterval(Duration.millis(timestampIntervalInMillis)); + return input + .apply(impulse) + .apply( + MapElements.via( + new SimpleFunction() { + @Override + public Long apply(Instant input) { + return input.getMillis(); + } + })); + } } private void runBigQueryIOStorageWritePipeline( @@ -102,7 +145,9 @@ private void runBigQueryIOStorageWritePipeline( new TableFieldSchema().setName("str").setType("STRING"))); Pipeline p = Pipeline.create(bqOptions); - p.apply("Input", isStreaming ? stream(rowCount) : GenerateSequence.from(0).to(rowCount)) + p.apply( + "Input", + isStreaming ? new UnboundedStream(rowCount) : GenerateSequence.from(0).to(rowCount)) .apply("GenerateMessage", ParDo.of(new FillRowFn())) .apply( "WriteToBQ", @@ -128,15 +173,15 @@ private void runBigQueryIOStorageWritePipeline( } @Test - public void testBigQueryStorageWrite30MProto() { + public void testBigQueryStorageWrite3MProto() { setUpTestEnvironment(WriteMode.EXACT_ONCE); - runBigQueryIOStorageWritePipeline(3000000, WriteMode.EXACT_ONCE, false); + runBigQueryIOStorageWritePipeline(3_000_000, WriteMode.EXACT_ONCE, false); } @Test - public void testBigQueryStorageWrite30MProtoALO() { + public void testBigQueryStorageWrite3MProtoALO() { setUpTestEnvironment(WriteMode.AT_LEAST_ONCE); - runBigQueryIOStorageWritePipeline(3000000, WriteMode.AT_LEAST_ONCE, false); + runBigQueryIOStorageWritePipeline(3_000_000, WriteMode.AT_LEAST_ONCE, false); } @Test diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java index 970580f3ef671..720419f2227a3 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java @@ -19,9 +19,9 @@ import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.toJsonString; import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -48,14 +48,17 @@ import com.google.api.services.bigquery.model.TableSchema; import com.google.api.services.bigquery.model.TimePartitioning; import com.google.auto.value.AutoValue; +import com.google.protobuf.ByteString; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Serializable; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; @@ -88,6 +91,7 @@ import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.coders.VarLongCoder; import org.apache.beam.sdk.extensions.avro.coders.AvroGenericCoder; +import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages; import org.apache.beam.sdk.io.GenerateSequence; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition; @@ -118,6 +122,7 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.DoFnTester; import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.SerializableFunctions; @@ -132,7 +137,9 @@ import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.transforms.windowing.WindowMappingFn; +import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.Row; @@ -141,13 +148,14 @@ import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; import org.apache.beam.sdk.values.ValueInSingleWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matchers; import org.joda.time.Duration; @@ -294,17 +302,18 @@ public void testWriteDynamicDestinations() throws Exception { writeDynamicDestinations(false, false); } - @Test - public void testWriteDynamicDestinationsBatchWithSchemas() throws Exception { - writeDynamicDestinations(true, false); - } - @Test public void testWriteDynamicDestinationsStreamingWithAutoSharding() throws Exception { assumeTrue(useStreaming); + assumeTrue(!useStorageApiApproximate); // STORAGE_API_AT_LEAST_ONCE ignores auto-sharding writeDynamicDestinations(true, true); } + @Test + public void testWriteDynamicDestinationsWithBeamSchemas() throws Exception { + writeDynamicDestinations(true, false); + } + public void writeDynamicDestinations(boolean schemas, boolean autoSharding) throws Exception { final Schema schema = Schema.builder().addField("name", FieldType.STRING).addField("id", FieldType.INT64).build(); @@ -599,8 +608,8 @@ public void testClusteringTableFunction() throws Exception { assertEquals(clustering, table.getClustering()); } - @Test - public void testTriggeredFileLoads() throws Exception { + public void runStreamingFileLoads(String tableRef, boolean useTempTables, boolean useTempDataset) + throws Exception { assumeTrue(!useStorageApi); assumeTrue(useStreaming); List elements = Lists.newArrayList(); @@ -620,65 +629,29 @@ public void testTriggeredFileLoads() throws Exception { elements.get(20), Iterables.toArray(elements.subList(21, 30), TableRow.class)) .advanceWatermarkToInfinity(); - BigQueryIO.Write.Method method = Method.FILE_LOADS; - p.apply(testStream) - .apply( - BigQueryIO.writeTableRows() - .to("project-id:dataset-id.table-id") - .withSchema( - new TableSchema() - .setFields( - ImmutableList.of( - new TableFieldSchema().setName("number").setType("INTEGER")))) - .withTestServices(fakeBqServices) - .withTriggeringFrequency(Duration.standardSeconds(30)) - .withNumFileShards(2) - .withMethod(method) - .withoutValidation()); - p.run(); - - assertThat( - fakeDatasetService.getAllRows("project-id", "dataset-id", "table-id"), - containsInAnyOrder(Iterables.toArray(elements, TableRow.class))); - } + BigQueryIO.Write writeTransform = + BigQueryIO.writeTableRows() + .to(tableRef) + .withSchema( + new TableSchema() + .setFields( + ImmutableList.of( + new TableFieldSchema().setName("number").setType("INTEGER")))) + .withTestServices(fakeBqServices) + .withWriteDisposition(Write.WriteDisposition.WRITE_APPEND) + .withTriggeringFrequency(Duration.standardSeconds(30)) + .withNumFileShards(2) + .withMethod(Method.FILE_LOADS) + .withoutValidation(); - @Test - public void testTriggeredFileLoadsWithTempTablesAndDataset() throws Exception { - String tableRef = "bigquery-project-id:dataset-id.table-id"; - List elements = Lists.newArrayList(); - for (int i = 0; i < 30; ++i) { - elements.add(new TableRow().set("number", i)); + if (useTempTables) { + writeTransform = writeTransform.withMaxBytesPerPartition(1).withMaxFilesPerPartition(1); + } + if (useTempDataset) { + writeTransform = writeTransform.withWriteTempDataset("temp-dataset-id"); } - TestStream testStream = - TestStream.create(TableRowJsonCoder.of()) - .addElements( - elements.get(0), Iterables.toArray(elements.subList(1, 10), TableRow.class)) - .advanceProcessingTime(Duration.standardMinutes(1)) - .addElements( - elements.get(10), Iterables.toArray(elements.subList(11, 20), TableRow.class)) - .advanceProcessingTime(Duration.standardMinutes(1)) - .addElements( - elements.get(20), Iterables.toArray(elements.subList(21, 30), TableRow.class)) - .advanceWatermarkToInfinity(); - BigQueryIO.Write.Method method = Method.FILE_LOADS; - p.apply(testStream) - .apply( - BigQueryIO.writeTableRows() - .to(tableRef) - .withSchema( - new TableSchema() - .setFields( - ImmutableList.of( - new TableFieldSchema().setName("number").setType("INTEGER")))) - .withTestServices(fakeBqServices) - .withTriggeringFrequency(Duration.standardSeconds(30)) - .withNumFileShards(2) - .withMaxBytesPerPartition(1) - .withMaxFilesPerPartition(1) - .withMethod(method) - .withoutValidation() - .withWriteTempDataset("temp-dataset-id")); + p.apply(testStream).apply(writeTransform); p.run(); final int projectIdSplitter = tableRef.indexOf(':'); @@ -690,7 +663,49 @@ public void testTriggeredFileLoadsWithTempTablesAndDataset() throws Exception { containsInAnyOrder(Iterables.toArray(elements, TableRow.class))); } - public void testTriggeredFileLoadsWithTempTables(String tableRef) throws Exception { + public void runStreamingFileLoads(String tableRef) throws Exception { + runStreamingFileLoads(tableRef, true, false); + } + + @Test + public void testStreamingFileLoads() throws Exception { + runStreamingFileLoads("project-id:dataset-id.table-id", false, false); + } + + @Test + public void testStreamingFileLoadsWithTempTables() throws Exception { + runStreamingFileLoads("project-id:dataset-id.table-id"); + } + + @Test + public void testStreamingFileLoadsWithTempTablesDefaultProject() throws Exception { + runStreamingFileLoads("dataset-id.table-id"); + } + + @Test + @ProjectOverride + public void testStreamingFileLoadsWithTempTablesBigQueryProject() throws Exception { + runStreamingFileLoads("bigquery-project-id:dataset-id.table-id"); + } + + @Test + public void testStreamingFileLoadsWithTempTablesAndDataset() throws Exception { + runStreamingFileLoads("bigquery-project-id:dataset-id.table-id", true, true); + } + + @Test + public void testStreamingFileLoadsWithTempTablesToExistingNullSchemaTable() throws Exception { + TableReference ref = + new TableReference() + .setProjectId("project-id") + .setDatasetId("dataset-id") + .setTableId("table-id"); + fakeDatasetService.createTable(new Table().setTableReference(ref).setSchema(null)); + runStreamingFileLoads("project-id:dataset-id.table-id"); + } + + @Test + public void testStreamingFileLoadsWithAutoSharding() throws Exception { assumeTrue(!useStorageApi); assumeTrue(useStreaming); List elements = Lists.newArrayList(); @@ -698,72 +713,102 @@ public void testTriggeredFileLoadsWithTempTables(String tableRef) throws Excepti elements.add(new TableRow().set("number", i)); } + Instant startInstant = new Instant(0L); TestStream testStream = TestStream.create(TableRowJsonCoder.of()) + // Initialize watermark for timer to be triggered correctly. + .advanceWatermarkTo(startInstant) .addElements( elements.get(0), Iterables.toArray(elements.subList(1, 10), TableRow.class)) .advanceProcessingTime(Duration.standardMinutes(1)) + .advanceWatermarkTo(startInstant.plus(Duration.standardSeconds(10))) .addElements( elements.get(10), Iterables.toArray(elements.subList(11, 20), TableRow.class)) .advanceProcessingTime(Duration.standardMinutes(1)) + .advanceWatermarkTo(startInstant.plus(Duration.standardSeconds(30))) .addElements( elements.get(20), Iterables.toArray(elements.subList(21, 30), TableRow.class)) + .advanceProcessingTime(Duration.standardMinutes(2)) .advanceWatermarkToInfinity(); - BigQueryIO.Write.Method method = Method.FILE_LOADS; + int numTables = 3; p.apply(testStream) .apply( BigQueryIO.writeTableRows() - .to(tableRef) + .to( + (ValueInSingleWindow vsw) -> { + String tableSpec = + "project-id:dataset-id.table-" + + ((int) vsw.getValue().get("number") % numTables); + return new TableDestination(tableSpec, null); + }) .withSchema( new TableSchema() .setFields( ImmutableList.of( new TableFieldSchema().setName("number").setType("INTEGER")))) .withTestServices(fakeBqServices) - .withTriggeringFrequency(Duration.standardSeconds(30)) - .withNumFileShards(2) - .withMaxBytesPerPartition(1) - .withMaxFilesPerPartition(1) - .withMethod(method) + // Set a triggering frequency without needing to also specify numFileShards when + // using autoSharding. + .withTriggeringFrequency(Duration.standardSeconds(100)) + .withAutoSharding() + .withMaxBytesPerPartition(1000) + .withMaxFilesPerPartition(10) + .withMethod(BigQueryIO.Write.Method.FILE_LOADS) .withoutValidation()); p.run(); - final int projectIdSplitter = tableRef.indexOf(':'); - final String projectId = - projectIdSplitter == -1 ? "project-id" : tableRef.substring(0, projectIdSplitter); - - assertThat( - fakeDatasetService.getAllRows(projectId, "dataset-id", "table-id"), - containsInAnyOrder(Iterables.toArray(elements, TableRow.class))); + Map> elementsByTableIdx = new HashMap<>(); + for (int i = 0; i < elements.size(); i++) { + elementsByTableIdx + .computeIfAbsent(i % numTables, k -> new ArrayList<>()) + .add(elements.get(i)); + } + for (Map.Entry> entry : elementsByTableIdx.entrySet()) { + assertThat( + fakeDatasetService.getAllRows("project-id", "dataset-id", "table-" + entry.getKey()), + containsInAnyOrder(Iterables.toArray(entry.getValue(), TableRow.class))); + } + // For each table destination, it's expected to create two load jobs based on the triggering + // frequency and processing time intervals. + assertEquals(2 * numTables, fakeDatasetService.getInsertCount()); } @Test - @ProjectOverride - public void testTriggeredFileLoadsWithTempTablesBigQueryProject() throws Exception { - testTriggeredFileLoadsWithTempTables("bigquery-project-id:dataset-id.table-id"); - } + public void testBatchFileLoads() throws Exception { + assumeTrue(!useStreaming); + assumeTrue(!useStorageApi); + List elements = Lists.newArrayList(); + for (int i = 0; i < 30; ++i) { + elements.add(new TableRow().set("number", i)); + } - @Test - public void testTriggeredFileLoadsWithTempTables() throws Exception { - testTriggeredFileLoadsWithTempTables("project-id:dataset-id.table-id"); - } + WriteResult result = + p.apply(Create.of(elements).withCoder(TableRowJsonCoder.of())) + .apply( + BigQueryIO.writeTableRows() + .to("dataset-id.table-id") + .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) + .withSchema( + new TableSchema() + .setFields( + ImmutableList.of( + new TableFieldSchema().setName("name").setType("STRING"), + new TableFieldSchema().setName("number").setType("INTEGER")))) + .withTestServices(fakeBqServices) + .withoutValidation()); - @Test - public void testTriggeredFileLoadsWithTempTablesToExistingNullSchemaTable() throws Exception { - Table fakeTable = new Table(); - TableReference ref = - new TableReference() - .setProjectId("project-id") - .setDatasetId("dataset-id") - .setTableId("table-id"); - fakeTable.setTableReference(ref); - fakeDatasetService.createTable(fakeTable); - testTriggeredFileLoadsWithTempTables("project-id:dataset-id.table-id"); + PAssert.that(result.getSuccessfulTableLoads()) + .containsInAnyOrder(new TableDestination("project-id:dataset-id.table-id", null)); + p.run(); + + assertThat( + fakeDatasetService.getAllRows("project-id", "dataset-id", "table-id"), + containsInAnyOrder(Iterables.toArray(elements, TableRow.class))); } @Test - public void testUntriggeredFileLoadsWithTempTables() throws Exception { + public void testBatchFileLoadsWithTempTables() throws Exception { // Test only non-streaming inserts. assumeTrue(!useStorageApi); assumeTrue(!useStreaming); @@ -771,19 +816,23 @@ public void testUntriggeredFileLoadsWithTempTables() throws Exception { for (int i = 0; i < 30; ++i) { elements.add(new TableRow().set("number", i)); } - p.apply(Create.of(elements)) - .apply( - BigQueryIO.writeTableRows() - .to("project-id:dataset-id.table-id") - .withSchema( - new TableSchema() - .setFields( - ImmutableList.of( - new TableFieldSchema().setName("number").setType("INTEGER")))) - .withTestServices(fakeBqServices) - .withMaxBytesPerPartition(1) - .withMaxFilesPerPartition(1) - .withoutValidation()); + WriteResult result = + p.apply(Create.of(elements)) + .apply( + BigQueryIO.writeTableRows() + .to("project-id:dataset-id.table-id") + .withSchema( + new TableSchema() + .setFields( + ImmutableList.of( + new TableFieldSchema().setName("number").setType("INTEGER")))) + .withTestServices(fakeBqServices) + .withMaxBytesPerPartition(1) + .withMaxFilesPerPartition(1) + .withoutValidation()); + + PAssert.that(result.getSuccessfulTableLoads()) + .containsInAnyOrder(new TableDestination("project-id:dataset-id.table-id", null)); p.run(); assertThat( @@ -792,12 +841,7 @@ public void testUntriggeredFileLoadsWithTempTables() throws Exception { } @Test - public void testTriggeredFileLoadsWithTempTablesDefaultProject() throws Exception { - testTriggeredFileLoadsWithTempTables("dataset-id.table-id"); - } - - @Test - public void testTriggeredFileLoadsWithTempTablesCreateNever() throws Exception { + public void testBatchFileLoadsWithTempTablesCreateNever() throws Exception { assumeTrue(!useStorageApi); assumeTrue(!useStreaming); @@ -841,77 +885,7 @@ public void testTriggeredFileLoadsWithTempTablesCreateNever() throws Exception { } @Test - public void testTriggeredFileLoadsWithAutoSharding() throws Exception { - assumeTrue(!useStorageApi); - assumeTrue(useStreaming); - List elements = Lists.newArrayList(); - for (int i = 0; i < 30; ++i) { - elements.add(new TableRow().set("number", i)); - } - - Instant startInstant = new Instant(0L); - TestStream testStream = - TestStream.create(TableRowJsonCoder.of()) - // Initialize watermark for timer to be triggered correctly. - .advanceWatermarkTo(startInstant) - .addElements( - elements.get(0), Iterables.toArray(elements.subList(1, 10), TableRow.class)) - .advanceProcessingTime(Duration.standardMinutes(1)) - .advanceWatermarkTo(startInstant.plus(Duration.standardSeconds(10))) - .addElements( - elements.get(10), Iterables.toArray(elements.subList(11, 20), TableRow.class)) - .advanceProcessingTime(Duration.standardMinutes(1)) - .advanceWatermarkTo(startInstant.plus(Duration.standardSeconds(30))) - .addElements( - elements.get(20), Iterables.toArray(elements.subList(21, 30), TableRow.class)) - .advanceProcessingTime(Duration.standardMinutes(2)) - .advanceWatermarkToInfinity(); - - int numTables = 3; - p.apply(testStream) - .apply( - BigQueryIO.writeTableRows() - .to( - (ValueInSingleWindow vsw) -> { - String tableSpec = - "project-id:dataset-id.table-" - + ((int) vsw.getValue().get("number") % numTables); - return new TableDestination(tableSpec, null); - }) - .withSchema( - new TableSchema() - .setFields( - ImmutableList.of( - new TableFieldSchema().setName("number").setType("INTEGER")))) - .withTestServices(fakeBqServices) - // Set a triggering frequency without needing to also specify numFileShards when - // using autoSharding. - .withTriggeringFrequency(Duration.standardSeconds(100)) - .withAutoSharding() - .withMaxBytesPerPartition(1000) - .withMaxFilesPerPartition(10) - .withMethod(BigQueryIO.Write.Method.FILE_LOADS) - .withoutValidation()); - p.run(); - - Map> elementsByTableIdx = new HashMap<>(); - for (int i = 0; i < elements.size(); i++) { - elementsByTableIdx - .computeIfAbsent(i % numTables, k -> new ArrayList<>()) - .add(elements.get(i)); - } - for (Map.Entry> entry : elementsByTableIdx.entrySet()) { - assertThat( - fakeDatasetService.getAllRows("project-id", "dataset-id", "table-" + entry.getKey()), - containsInAnyOrder(Iterables.toArray(entry.getValue(), TableRow.class))); - } - // For each table destination, it's expected to create two load jobs based on the triggering - // frequency and processing time intervals. - assertEquals(2 * numTables, fakeDatasetService.getInsertCount()); - } - - @Test - public void testFailuresNoRetryPolicy() throws Exception { + public void testStreamingInsertsFailuresNoRetryPolicy() throws Exception { assumeTrue(!useStorageApi); assumeTrue(useStreaming); TableRow row1 = new TableRow().set("name", "a").set("number", "1"); @@ -949,7 +923,7 @@ public void testFailuresNoRetryPolicy() throws Exception { } @Test - public void testRetryPolicy() throws Exception { + public void testStreamingInsertsRetryPolicy() throws Exception { assumeTrue(!useStorageApi); assumeTrue(useStreaming); TableRow row1 = new TableRow().set("name", "a").set("number", "1"); @@ -968,122 +942,58 @@ public void testRetryPolicy() throws Exception { row1, ImmutableList.of(ephemeralError, ephemeralError), row2, ImmutableList.of(ephemeralError, ephemeralError, persistentError))); - WriteResult result = - p.apply(Create.of(row1, row2, row3)) - .apply( - BigQueryIO.writeTableRows() - .to("project-id:dataset-id.table-id") - .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) - .withMethod(Method.STREAMING_INSERTS) - .withSchema( - new TableSchema() - .setFields( - ImmutableList.of( - new TableFieldSchema().setName("name").setType("STRING"), - new TableFieldSchema().setName("number").setType("INTEGER")))) - .withFailedInsertRetryPolicy(InsertRetryPolicy.retryTransientErrors()) - .withTestServices(fakeBqServices) - .withoutValidation()); - - PCollection failedRows = result.getFailedInserts(); - // row2 finally fails with a non-retryable error, so we expect to see it in the collection of - // failed rows. - PAssert.that(failedRows).containsInAnyOrder(row2); - if (useStorageApi || !useStreaming) { - PAssert.that(result.getSuccessfulInserts()).containsInAnyOrder(row1, row3); - } - p.run(); - - // Only row1 and row3 were successfully inserted. - assertThat( - fakeDatasetService.getAllRows("project-id", "dataset-id", "table-id"), - containsInAnyOrder(row1, row3)); - } - - @Test - public void testWrite() throws Exception { - p.apply( - Create.of( - new TableRow().set("name", "a").set("number", 1), - new TableRow().set("name", "b").set("number", 2), - new TableRow().set("name", "c").set("number", 3)) - .withCoder(TableRowJsonCoder.of())) - .apply( - BigQueryIO.writeTableRows() - .to("project-id:dataset-id.table-id") - .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) - .withSchema( - new TableSchema() - .setFields( - ImmutableList.of( - new TableFieldSchema().setName("name").setType("STRING"), - new TableFieldSchema().setName("number").setType("INTEGER")))) - .withTestServices(fakeBqServices) - .withoutValidation()); - p.run(); - } - - @Test - public void testWriteWithSuccessfulBatchInserts() throws Exception { - assumeTrue(!useStreaming); - assumeTrue(!useStorageApi); - - WriteResult result = - p.apply( - Create.of( - new TableRow().set("name", "a").set("number", 1), - new TableRow().set("name", "b").set("number", 2), - new TableRow().set("name", "c").set("number", 3)) - .withCoder(TableRowJsonCoder.of())) - .apply( - BigQueryIO.writeTableRows() - .to("dataset-id.table-id") - .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) - .withSchema( - new TableSchema() - .setFields( - ImmutableList.of( - new TableFieldSchema().setName("name").setType("STRING"), - new TableFieldSchema().setName("number").setType("INTEGER")))) - .withTestServices(fakeBqServices) - .withoutValidation()); - - PAssert.that(result.getSuccessfulTableLoads()) - .containsInAnyOrder(new TableDestination("project-id:dataset-id.table-id", null)); - - p.run(); - } - - @Test - public void testWriteWithSuccessfulBatchInsertsAndWriteRename() throws Exception { - assumeTrue(!useStreaming); - assumeTrue(!useStorageApi); - - WriteResult result = - p.apply( - Create.of( - new TableRow().set("name", "a").set("number", 1), - new TableRow().set("name", "b").set("number", 2), - new TableRow().set("name", "c").set("number", 3)) - .withCoder(TableRowJsonCoder.of())) + WriteResult result = + p.apply(Create.of(row1, row2, row3)) .apply( BigQueryIO.writeTableRows() - .to("dataset-id.table-id") - .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) + .to("project-id:dataset-id.table-id") + .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) + .withMethod(Method.STREAMING_INSERTS) .withSchema( new TableSchema() .setFields( ImmutableList.of( new TableFieldSchema().setName("name").setType("STRING"), new TableFieldSchema().setName("number").setType("INTEGER")))) - .withMaxFileSize(1) - .withMaxFilesPerPartition(1) + .withFailedInsertRetryPolicy(InsertRetryPolicy.retryTransientErrors()) .withTestServices(fakeBqServices) .withoutValidation()); - PAssert.that(result.getSuccessfulTableLoads()) - .containsInAnyOrder(new TableDestination("project-id:dataset-id.table-id", null)); + PCollection failedRows = result.getFailedInserts(); + // row2 finally fails with a non-retryable error, so we expect to see it in the collection of + // failed rows. + PAssert.that(failedRows).containsInAnyOrder(row2); + if (useStorageApi || !useStreaming) { + PAssert.that(result.getSuccessfulInserts()).containsInAnyOrder(row1, row3); + } + p.run(); + + // Only row1 and row3 were successfully inserted. + assertThat( + fakeDatasetService.getAllRows("project-id", "dataset-id", "table-id"), + containsInAnyOrder(row1, row3)); + } + @Test + public void testWrite() throws Exception { + p.apply( + Create.of( + new TableRow().set("name", "a").set("number", 1), + new TableRow().set("name", "b").set("number", 2), + new TableRow().set("name", "c").set("number", 3)) + .withCoder(TableRowJsonCoder.of())) + .apply( + BigQueryIO.writeTableRows() + .to("project-id:dataset-id.table-id") + .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) + .withSchema( + new TableSchema() + .setFields( + ImmutableList.of( + new TableFieldSchema().setName("name").setType("STRING"), + new TableFieldSchema().setName("number").setType("INTEGER")))) + .withTestServices(fakeBqServices) + .withoutValidation()); p.run(); } @@ -1201,7 +1111,7 @@ public void runTestWriteAvro(boolean schemaFromView) throws Exception { new TableRow() .set("strval", "test2") .set("longval", "2") - .set("doubleval", 2.0D) + .set("doubleval", 2.0) .set( "instantval", useStorageApi || useStorageApiApproximate @@ -1278,12 +1188,12 @@ protected void writeString(org.apache.avro.Schema schema, Object datum, Encoder new TableRow() .set("strVal", "test_custom") .set("longVal", "1") - .set("doubleVal", 1.0D) + .set("doubleVal", 1.0) .set("instantVal", "2019-01-01 00:00:00 UTC"), new TableRow() .set("strVal", "test2_custom") .set("longVal", "2") - .set("doubleVal", 2.0D) + .set("doubleVal", 2.0) .set("instantVal", "2019-02-01 00:00:00 UTC"))); } @@ -1298,9 +1208,13 @@ public void testStreamingWriteWithAutoSharding() throws Exception { } private void streamingWrite(boolean autoSharding) throws Exception { - if (!useStreaming) { - return; - } + assumeTrue(useStreaming); + List elements = + ImmutableList.of( + new TableRow().set("name", "a").set("number", "1"), + new TableRow().set("name", "b").set("number", "2"), + new TableRow().set("name", "c").set("number", "3"), + new TableRow().set("name", "d").set("number", "4")); BigQueryIO.Write write = BigQueryIO.writeTableRows() .to("project-id:dataset-id.table-id") @@ -1316,33 +1230,39 @@ private void streamingWrite(boolean autoSharding) throws Exception { if (autoSharding) { write = write.withAutoSharding(); } - p.apply( - Create.of( - new TableRow().set("name", "a").set("number", "1"), - new TableRow().set("name", "b").set("number", "2"), - new TableRow().set("name", "c").set("number", "3"), - new TableRow().set("name", "d").set("number", "4")) - .withCoder(TableRowJsonCoder.of())) + p.apply(Create.of(elements).withCoder(TableRowJsonCoder.of())) .setIsBoundedInternal(PCollection.IsBounded.UNBOUNDED) .apply("WriteToBQ", write); p.run(); assertThat( fakeDatasetService.getAllRows("project-id", "dataset-id", "table-id"), - containsInAnyOrder( - new TableRow().set("name", "a").set("number", "1"), - new TableRow().set("name", "b").set("number", "2"), - new TableRow().set("name", "c").set("number", "3"), - new TableRow().set("name", "d").set("number", "4"))); - } - - @Test - public void testStorageApiWriteWithAutoSharding() throws Exception { - storageWrite(true); + containsInAnyOrder(Iterables.toArray(elements, TableRow.class))); } private void storageWrite(boolean autoSharding) throws Exception { assumeTrue(useStorageApi); + if (autoSharding) { + assumeTrue(!useStorageApiApproximate); + assumeTrue(useStreaming); + } + List elements = Lists.newArrayList(); + for (int i = 0; i < 30; ++i) { + elements.add(new TableRow().set("number", String.valueOf(i))); + } + + TestStream testStream = + TestStream.create(TableRowJsonCoder.of()) + .addElements( + elements.get(0), Iterables.toArray(elements.subList(1, 10), TableRow.class)) + .advanceProcessingTime(Duration.standardMinutes(1)) + .addElements( + elements.get(10), Iterables.toArray(elements.subList(11, 20), TableRow.class)) + .advanceProcessingTime(Duration.standardMinutes(1)) + .addElements( + elements.get(20), Iterables.toArray(elements.subList(21, 30), TableRow.class)) + .advanceWatermarkToInfinity(); + BigQueryIO.Write write = BigQueryIO.writeTableRows() .to("project-id:dataset-id.table-id") @@ -1351,35 +1271,50 @@ private void storageWrite(boolean autoSharding) throws Exception { new TableSchema() .setFields( ImmutableList.of( - new TableFieldSchema().setName("name").setType("STRING"), new TableFieldSchema().setName("number").setType("INTEGER")))) .withTestServices(fakeBqServices) .withoutValidation(); - if (autoSharding) { - write = - write - .withAutoSharding() - .withTriggeringFrequency(Duration.standardSeconds(5)) - .withMethod(Method.STORAGE_WRITE_API); + + if (useStreaming) { + if (!useStorageApiApproximate) { + write = + write + .withTriggeringFrequency(Duration.standardSeconds(30)) + .withNumStorageWriteApiStreams(2); + } + if (autoSharding) { + write = write.withAutoSharding(); + } } - p.apply( - Create.of( - new TableRow().set("name", "a").set("number", "1"), - new TableRow().set("name", "b").set("number", "2"), - new TableRow().set("name", "c").set("number", "3"), - new TableRow().set("name", "d").set("number", "4")) - .withCoder(TableRowJsonCoder.of())) - .setIsBoundedInternal(PCollection.IsBounded.UNBOUNDED) - .apply("WriteToBQ", write); - p.run(); + + PTransform> source = + useStreaming ? testStream : Create.of(elements).withCoder(TableRowJsonCoder.of()); + + p.apply(source).apply("WriteToBQ", write); + p.run().waitUntilFinish(); assertThat( fakeDatasetService.getAllRows("project-id", "dataset-id", "table-id"), - containsInAnyOrder( - new TableRow().set("name", "a").set("number", "1"), - new TableRow().set("name", "b").set("number", "2"), - new TableRow().set("name", "c").set("number", "3"), - new TableRow().set("name", "d").set("number", "4"))); + containsInAnyOrder(Iterables.toArray(elements, TableRow.class))); + } + + @Test + public void testBatchStorageApiWrite() throws Exception { + assumeTrue(!useStreaming); + storageWrite(false); + } + + @Test + public void testStreamingStorageApiWrite() throws Exception { + assumeTrue(useStreaming); + storageWrite(false); + } + + @Test + public void testStreamingStorageApiWriteWithAutoSharding() throws Exception { + assumeTrue(useStreaming); + assumeTrue(!useStorageApiApproximate); + storageWrite(true); } @DefaultSchema(JavaFieldSchema.class) @@ -1395,10 +1330,14 @@ static class SchemaPojo { } @Test - public void testSchemaWriteLoads() throws Exception { - assumeTrue(!useStreaming); + public void testBatchSchemaWriteLoads() throws Exception { + // This test is actually for batch! + // We test on the useStreaming parameter however because we need it as + // true to test STORAGE_API_AT_LEAST_ONCE + assumeTrue(useStreaming); + p.getOptions().as(BigQueryOptions.class).setStorageWriteApiTriggeringFrequencySec(null); // withMethod overrides the pipeline option, so we need to explicitly request - // STORAGE_API_WRITES. + // STORAGE_WRITE_API. BigQueryIO.Write.Method method = useStorageApi ? (useStorageApiApproximate @@ -1895,7 +1834,7 @@ public void testCreateNever() throws Exception { new SimpleFunction() { @Override public TableRow apply(Long input) { - return new TableRow().set("name", "name " + input).set("number", input); + return new TableRow().set("NaMe", "name " + input).set("numBEr", input); } })) .setCoder(TableRowJsonCoder.of()); @@ -1925,6 +1864,8 @@ public void testUpdateTableSchemaNoUnknownValues() throws Exception { assumeTrue(useStreaming); assumeTrue(useStorageApi); thrown.expect(IllegalArgumentException.class); + thrown.expectMessage( + "Auto schema update currently only supported when ignoreUnknownValues also set."); p.apply("create", Create.empty(TableRowJsonCoder.of())) .apply( BigQueryIO.writeTableRows() @@ -2122,6 +2063,103 @@ TableRow filterUnknownValues(TableRow row, List tableSchemaFie return filtered; } + @Test + public void testBatchStorageWriteWithIgnoreUnknownValues() throws Exception { + batchStorageWriteWithIgnoreUnknownValues(false); + } + + @Test + public void testBatchStorageWriteWithIgnoreUnknownValuesWithInputSchema() throws Exception { + batchStorageWriteWithIgnoreUnknownValues(true); + } + + public void batchStorageWriteWithIgnoreUnknownValues(boolean withInputSchema) throws Exception { + assumeTrue(!useStreaming); + // Make sure that GroupIntoBatches does not buffer data. + p.getOptions().as(BigQueryOptions.class).setStorageApiAppendThresholdBytes(1); + + BigQueryIO.Write.Method method = + useStorageApi ? Method.STORAGE_WRITE_API : Method.STORAGE_API_AT_LEAST_ONCE; + p.enableAbandonedNodeEnforcement(false); + + TableReference tableRef = BigQueryHelpers.parseTableSpec("project-id:dataset-id.table"); + TableSchema tableSchema = + new TableSchema() + .setFields( + ImmutableList.of( + new TableFieldSchema().setName("number").setType("INTEGER"), + new TableFieldSchema().setName("name").setType("STRING"))); + + fakeDatasetService.createTable(new Table().setTableReference(tableRef).setSchema(tableSchema)); + + List rows = + Arrays.asList( + new TableRow().set("number", "1").set("name", "a"), + new TableRow().set("number", "2").set("name", "b").set("extra", "aaa"), + new TableRow() + .set("number", "3") + .set("name", "c") + .set("repeated", Arrays.asList("a", "a")), + new TableRow().set("number", "4").set("name", "d").set("req", "req_a"), + new TableRow() + .set("number", "5") + .set("name", "e") + .set("repeated", Arrays.asList("a", "a")) + .set("req", "req_a")); + + BigQueryIO.Write writeTransform = + BigQueryIO.writeTableRows() + .to(tableRef) + .withMethod(method) + .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_NEVER) + .ignoreUnknownValues() + .withTestServices(fakeBqServices) + .withoutValidation(); + if (withInputSchema) { + writeTransform = writeTransform.withSchema(tableSchema); + } + WriteResult result = p.apply(Create.of(rows)).apply(writeTransform); + // we ignore extra values instead of sending to DLQ. check that it's empty: + PAssert.that(result.getFailedStorageApiInserts()).empty(); + p.run().waitUntilFinish(); + + Iterable expectedDroppedValues = + rows.subList(1, 5).stream() + .map(tr -> filterUnknownValues(tr, tableSchema.getFields())) + .collect(Collectors.toList()); + + Iterable expectedFullValues = rows.subList(0, 1); + + assertThat( + fakeDatasetService.getAllRows( + tableRef.getProjectId(), tableRef.getDatasetId(), tableRef.getTableId()), + containsInAnyOrder( + Iterables.toArray( + Iterables.concat(expectedDroppedValues, expectedFullValues), TableRow.class))); + } + + @Test + public void testStreamingWriteValidateFailsWithoutTriggeringFrequency() { + assumeTrue(useStreaming); + assumeTrue(!useStorageApiApproximate); + p.enableAbandonedNodeEnforcement(false); + Method method = useStorageApi ? Method.STORAGE_WRITE_API : Method.FILE_LOADS; + + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("unbounded PCollection via FILE_LOADS or STORAGE_WRITE_API"); + thrown.expectMessage("triggering frequency must be specified"); + + p.getOptions().as(BigQueryOptions.class).setStorageWriteApiTriggeringFrequencySec(null); + p.apply(Create.empty(INPUT_RECORD_CODER)) + .setIsBoundedInternal(PCollection.IsBounded.UNBOUNDED) + .apply( + BigQueryIO.write() + .withAvroFormatFunction(r -> new GenericData.Record(r.getSchema())) + .to("dataset.table") + .withMethod(method) + .withCreateDisposition(CreateDisposition.CREATE_NEVER)); + } + @Test public void testBigQueryIOGetName() { assertEquals( @@ -2211,23 +2249,24 @@ public void testWriteValidateFailsWithAvroFormatAndStreamingInserts() { @Test public void testWriteValidateFailsWithBatchAutoSharding() { - assumeTrue(!useStorageApi); + assumeTrue(!useStreaming); p.enableAbandonedNodeEnforcement(false); thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("Auto-sharding is only applicable to unbounded input."); + thrown.expectMessage( + "Auto-sharding is only applicable to an unbounded PCollection, but the input PCollection is BOUNDED."); p.apply(Create.empty(INPUT_RECORD_CODER)) .apply( BigQueryIO.write() .to("dataset.table") .withSchema(new TableSchema()) - .withMethod(Method.STREAMING_INSERTS) .withAutoSharding() .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)); } @Test public void testMaxRetryJobs() { + assumeTrue(!useStorageApi); BigQueryIO.Write write = BigQueryIO.writeTableRows() .to("dataset.table") @@ -2692,7 +2731,7 @@ public void testWriteToTableDecorator() throws Exception { } @Test - public void testExtendedErrorRetrieval() throws Exception { + public void testStreamingInsertsExtendedErrorRetrieval() throws Exception { assumeTrue(!useStorageApi); TableRow row1 = new TableRow().set("name", "a").set("number", "1"); TableRow row2 = new TableRow().set("name", "b").set("number", "2"); @@ -2850,7 +2889,7 @@ public void testStorageApiErrors() throws Exception { } @Test - public void testWrongErrorConfigs() { + public void testStreamingInsertsWrongErrorConfigs() { assumeTrue(!useStorageApi); p.enableAutoRunIfMissing(true); TableRow row1 = new TableRow().set("name", "a").set("number", "1"); @@ -2914,6 +2953,7 @@ public void testWrongErrorConfigs() { void schemaUpdateOptionsTest( BigQueryIO.Write.Method insertMethod, Set schemaUpdateOptions) throws Exception { + assumeTrue(!useStorageApi); TableRow row = new TableRow().set("date", "2019-01-01").set("number", "1"); TableSchema schema = @@ -3004,38 +3044,11 @@ public void testWriteWithStorageApiWithDefaultProject() throws Exception { } @Test - public void testWriteWithStorageApiWithoutSettingShardsEnableAutoSharding() throws Exception { - assumeTrue(useStorageApi); - assumeTrue(p.getOptions().as(BigQueryOptions.class).getNumStorageWriteApiStreams() == 0); - BigQueryIO.Write write = - BigQueryIO.writeTableRows() - .to("dataset-id.table-id") - .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) - .withSchema( - new TableSchema() - .setFields( - ImmutableList.of(new TableFieldSchema().setName("name").setType("STRING")))) - .withMethod(Method.STORAGE_WRITE_API) - .withoutValidation() - .withTestServices(fakeBqServices); - - p.apply( - Create.of(new TableRow().set("name", "a"), new TableRow().set("name", "b")) - .withCoder(TableRowJsonCoder.of())) - .apply("WriteToBQ", write); - p.run(); - assertThat( - fakeDatasetService.getAllRows("project-id", "dataset-id", "table-id"), - containsInAnyOrder(new TableRow().set("name", "a"), new TableRow().set("name", "b"))); - } - - @Test - public void testBatchStorageWriteWithMultipleAppendsPerStream() throws Exception { + public void testStorageWriteWithMultipleAppendsPerStream() throws Exception { assumeTrue(useStorageApi); - assumeTrue(!useStreaming); // reduce threshold to trigger multiple stream appends - p.getOptions().as(BigQueryOptions.class).setStorageApiAppendThresholdRecordCount(0); + p.getOptions().as(BigQueryOptions.class).setStorageApiAppendThresholdBytes(0); // limit parallelism to limit the number of write streams we have open p.getOptions().as(DirectOptions.class).setTargetParallelism(1); @@ -3059,22 +3072,138 @@ public void testBatchStorageWriteWithMultipleAppendsPerStream() throws Exception for (int i = 0; i < 100; i++) { rows.add(new TableRow().set("num", String.valueOf(i)).set("name", String.valueOf(i))); } - p.apply(Create.of(rows)) - .apply( - "Save Events To BigQuery", - BigQueryIO.writeTableRows() - .to(ref) - .withMethod(Write.Method.STORAGE_WRITE_API) - .withCreateDisposition(CreateDisposition.CREATE_NEVER) - .withTestServices(fakeBqServices)); + PCollection tableRowPCollection; + if (useStreaming) { + TestStream testStream = + TestStream.create(TableRowJsonCoder.of()) + .addElements(rows.get(0), Iterables.toArray(rows.subList(1, 25), TableRow.class)) + .advanceProcessingTime(Duration.standardMinutes(1)) + .addElements(rows.get(25), Iterables.toArray(rows.subList(26, 50), TableRow.class)) + .advanceProcessingTime(Duration.standardMinutes(1)) + .addElements(rows.get(50), Iterables.toArray(rows.subList(51, 75), TableRow.class)) + .addElements(rows.get(75), Iterables.toArray(rows.subList(76, 100), TableRow.class)) + .advanceWatermarkToInfinity(); + tableRowPCollection = p.apply(testStream); + } else { + tableRowPCollection = p.apply(Create.of(rows)); + } + Method method = + useStorageApiApproximate ? Method.STORAGE_API_AT_LEAST_ONCE : Method.STORAGE_WRITE_API; + tableRowPCollection.apply( + "Save Events To BigQuery", + BigQueryIO.writeTableRows() + .to(ref) + .withMethod(method) + .withCreateDisposition(CreateDisposition.CREATE_NEVER) + .withTestServices(fakeBqServices)); - p.run(); + p.run().waitUntilFinish(); assertThat( fakeDatasetService.getAllRows("project-id", "dataset-id", "table-id"), containsInAnyOrder(Iterables.toArray(rows, TableRow.class))); } + @Test + public void testWriteProtos() throws Exception { + BigQueryIO.Write.Method method = + useStreaming + ? (useStorageApi + ? (useStorageApiApproximate + ? Method.STORAGE_API_AT_LEAST_ONCE + : Method.STORAGE_WRITE_API) + : Method.STREAMING_INSERTS) + : useStorageApi ? Method.STORAGE_WRITE_API : Method.FILE_LOADS; + Function getPrimitive = + (Integer i) -> + Proto3SchemaMessages.Primitive.newBuilder() + .setPrimitiveDouble(i) + .setPrimitiveFloat(i) + .setPrimitiveInt32(i) + .setPrimitiveInt64(i) + .setPrimitiveUint32(i) + .setPrimitiveUint64(i) + .setPrimitiveSint32(i) + .setPrimitiveSint64(i) + .setPrimitiveFixed32(i) + .setPrimitiveFixed64(i) + .setPrimitiveBool(true) + .setPrimitiveString(Integer.toString(i)) + .setPrimitiveBytes( + ByteString.copyFrom(Integer.toString(i).getBytes(StandardCharsets.UTF_8))) + .build(); + Function getPrimitiveRow = + (Integer i) -> + new TableRow() + .set("primitive_double", Double.valueOf(i)) + .set("primitive_float", Float.valueOf(i).doubleValue()) + .set("primitive_int32", i.intValue()) + .set("primitive_int64", i.toString()) + .set("primitive_uint32", i.toString()) + .set("primitive_uint64", i.toString()) + .set("primitive_sint32", i.toString()) + .set("primitive_sint64", i.toString()) + .set("primitive_fixed32", i.toString()) + .set("primitive_fixed64", i.toString()) + .set("primitive_bool", true) + .set("primitive_string", i.toString()) + .set( + "primitive_bytes", + BaseEncoding.base64() + .encode( + ByteString.copyFrom(i.toString().getBytes(StandardCharsets.UTF_8)) + .toByteArray())); + + List nestedItems = + Lists.newArrayList(getPrimitive.apply(1), getPrimitive.apply(2), getPrimitive.apply(3)); + + Iterable items = + nestedItems.stream() + .map( + p -> + Proto3SchemaMessages.Nested.newBuilder() + .setNested(p) + .addAllNestedList(Lists.newArrayList(p, p, p)) + .build()) + .collect(Collectors.toList()); + + List expectedNestedTableRows = + Lists.newArrayList( + getPrimitiveRow.apply(1), getPrimitiveRow.apply(2), getPrimitiveRow.apply(3)); + Iterable expectedItems = + expectedNestedTableRows.stream() + .map( + p -> + new TableRow().set("nested", p).set("nested_list", Lists.newArrayList(p, p, p))) + .collect(Collectors.toList()); + + BigQueryIO.Write write = + BigQueryIO.writeProtos(Proto3SchemaMessages.Nested.class) + .to("dataset-id.table-id") + .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) + .withMethod(method) + .withoutValidation() + .withTestServices(fakeBqServices); + + p.apply(Create.of(items)).apply("WriteToBQ", write); + p.run(); + + // Round trip through the coder to make sure the types match our expected types. + List allRows = + fakeDatasetService.getAllRows("project-id", "dataset-id", "table-id").stream() + .map( + tr -> { + try { + byte[] bytes = CoderUtils.encodeToByteArray(TableRowJsonCoder.of(), tr); + return CoderUtils.decodeFromByteArray(TableRowJsonCoder.of(), bytes); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + assertThat(allRows, containsInAnyOrder(Iterables.toArray(expectedItems, TableRow.class))); + } + @Test public void testUpsertAndDeleteTableRows() throws Exception { assumeTrue(useStorageApi); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryKmsKeyIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryKmsKeyIT.java index ede7db07ae8d5..85a180932f65f 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryKmsKeyIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryKmsKeyIT.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.testing.TestPipelineOptions; import org.apache.beam.sdk.testing.UsesKms; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySchemaUpdateOptionsIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySchemaUpdateOptionsIT.java index bd5a2762012fe..833a0a0829c7f 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySchemaUpdateOptionsIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySchemaUpdateOptionsIT.java @@ -46,8 +46,8 @@ import org.apache.beam.sdk.testing.TestPipelineOptions; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.ValueInSingleWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -87,7 +87,11 @@ public class BigQuerySchemaUpdateOptionsIT { @BeforeClass public static void setupTestEnvironment() throws Exception { project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); - BQ_CLIENT.createNewDataset(project, BIG_QUERY_DATASET_ID); + BQ_CLIENT.createNewDataset( + project, + BIG_QUERY_DATASET_ID, + null, + TestPipeline.testingPipelineOptions().as(TestBigQueryOptions.class).getBigQueryLocation()); } @AfterClass diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImplTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImplTest.java index 2d1067420d4ff..4b28ea481bcb0 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImplTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImplTest.java @@ -108,10 +108,10 @@ import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.values.FailsafeValueInSingleWindow; import org.apache.beam.sdk.values.ValueInSingleWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Verify; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Verify; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.junit.Before; import org.junit.Rule; @@ -920,6 +920,7 @@ public void testInsertWithinRowCountLimits() throws Exception { } /** Tests that {@link DatasetServiceImpl#insertAll} does not go over limit of rows per request. */ + @SuppressWarnings("InlineMeInliner") // inline `Strings.repeat()` - Java 11+ API only @Test public void testInsertWithinRequestByteSizeLimitsErrorsOut() throws Exception { TableReference ref = @@ -972,6 +973,7 @@ public void testInsertWithinRequestByteSizeLimitsErrorsOut() throws Exception { assertThat(e.getMessage(), containsString("exceeded BigQueryIO limit of 9MB.")); } + @SuppressWarnings("InlineMeInliner") // inline `Strings.repeat()` - Java 11+ API only @Test public void testInsertRetryTransientsAboveRequestByteSizeLimits() throws Exception { TableReference ref = diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTimePartitioningClusteringIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTimePartitioningClusteringIT.java index 7e945517cfafd..da5f396e8d893 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTimePartitioningClusteringIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTimePartitioningClusteringIT.java @@ -24,9 +24,11 @@ import com.google.api.services.bigquery.model.TableRow; import com.google.api.services.bigquery.model.TableSchema; import com.google.api.services.bigquery.model.TimePartitioning; +import java.security.SecureRandom; import java.util.Arrays; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; import org.apache.beam.sdk.io.gcp.testing.BigqueryClient; import org.apache.beam.sdk.options.Default; import org.apache.beam.sdk.options.Description; @@ -38,8 +40,10 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.ValueInSingleWindow; import org.checkerframework.checker.nullness.qual.Nullable; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -48,8 +52,16 @@ @RunWith(JUnit4.class) public class BigQueryTimePartitioningClusteringIT { private static final String WEATHER_SAMPLES_TABLE = - "clouddataflow-readonly:samples.weather_stations"; - private static final String DATASET_NAME = "BigQueryTimePartitioningIT"; + "apache-beam-testing.samples.weather_stations"; + + private static String project; + private static final BigqueryClient BQ_CLIENT = + new BigqueryClient("BigQueryTimePartitioningClusteringIT"); + private static final String DATASET_NAME = + "BigQueryTimePartitioningIT_" + + System.currentTimeMillis() + + "_" + + new SecureRandom().nextInt(32); private static final TimePartitioning TIME_PARTITIONING = new TimePartitioning().setField("date").setType("DAY"); private static final Clustering CLUSTERING = @@ -64,6 +76,16 @@ public class BigQueryTimePartitioningClusteringIT { private Bigquery bqClient; private BigQueryClusteringITOptions options; + @BeforeClass + public static void setupTestEnvironment() throws Exception { + project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); + BQ_CLIENT.createNewDataset( + project, + DATASET_NAME, + null, + TestPipeline.testingPipelineOptions().as(TestBigQueryOptions.class).getBigQueryLocation()); + } + @Before public void setUp() { PipelineOptionsFactory.register(BigQueryClusteringITOptions.class); @@ -72,6 +94,11 @@ public void setUp() { bqClient = BigqueryClient.getNewBigqueryClient(options.getAppName()); } + @AfterClass + public static void cleanup() { + BQ_CLIENT.deleteDataset(project, DATASET_NAME); + } + /** Customized PipelineOptions for BigQueryClustering Integration Test. */ public interface BigQueryClusteringITOptions extends TestPipelineOptions, ExperimentalOptions, BigQueryOptions { @@ -110,8 +137,7 @@ public ClusteredDestinations(String tableName) { @Override public TableDestination getDestination(ValueInSingleWindow element) { - return new TableDestination( - String.format("%s.%s", DATASET_NAME, tableName), null, TIME_PARTITIONING, CLUSTERING); + return new TableDestination(tableName, null, TIME_PARTITIONING, CLUSTERING); } @Override @@ -176,6 +202,7 @@ public void testE2EBigQueryClustering() throws Exception { @Test public void testE2EBigQueryClusteringTableFunction() throws Exception { String tableName = "weather_stations_clustered_table_function_" + System.currentTimeMillis(); + String destination = String.format("%s.%s", DATASET_NAME, tableName); Pipeline p = Pipeline.create(options); @@ -185,11 +212,7 @@ public void testE2EBigQueryClusteringTableFunction() throws Exception { BigQueryIO.writeTableRows() .to( (ValueInSingleWindow vsw) -> - new TableDestination( - String.format("%s.%s", DATASET_NAME, tableName), - null, - TIME_PARTITIONING, - CLUSTERING)) + new TableDestination(destination, null, TIME_PARTITIONING, CLUSTERING)) .withClustering() .withSchema(SCHEMA) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) @@ -206,6 +229,7 @@ public void testE2EBigQueryClusteringTableFunction() throws Exception { public void testE2EBigQueryClusteringDynamicDestinations() throws Exception { String tableName = "weather_stations_clustered_dynamic_destinations_" + System.currentTimeMillis(); + String destination = String.format("%s.%s", DATASET_NAME, tableName); Pipeline p = Pipeline.create(options); @@ -213,7 +237,7 @@ public void testE2EBigQueryClusteringDynamicDestinations() throws Exception { .apply(ParDo.of(new KeepStationNumberAndConvertDate())) .apply( BigQueryIO.writeTableRows() - .to(new ClusteredDestinations(tableName)) + .to(new ClusteredDestinations(destination)) .withClustering() .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE)); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryToTableIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryToTableIT.java index 4f4354fc5e731..1abe7752b2e0b 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryToTableIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryToTableIT.java @@ -46,14 +46,13 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.options.Validation; import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.testing.TestPipelineOptions; import org.apache.beam.sdk.transforms.Reshuffle; import org.apache.beam.sdk.transforms.Values; import org.apache.beam.sdk.transforms.WithKeys; import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Duration; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -214,7 +213,7 @@ private void verifyStandardQueryRes(String outputTable) throws Exception { } /** Customized PipelineOption for BigQueryToTable Pipeline. */ - public interface BigQueryToTableOptions extends TestPipelineOptions, ExperimentalOptions { + public interface BigQueryToTableOptions extends TestBigQueryOptions, ExperimentalOptions { @Description("The BigQuery query to be used for creating the source") @Validation.Required @@ -252,9 +251,11 @@ public interface BigQueryToTableOptions extends TestPipelineOptions, Experimenta @BeforeClass public static void setupTestEnvironment() throws Exception { PipelineOptionsFactory.register(BigQueryToTableOptions.class); - project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); + BigQueryToTableOptions options = + TestPipeline.testingPipelineOptions().as(BigQueryToTableOptions.class); + project = options.as(GcpOptions.class).getProject(); // Create one BQ dataset for all test cases. - BQ_CLIENT.createNewDataset(project, BIG_QUERY_DATASET_ID); + BQ_CLIENT.createNewDataset(project, BIG_QUERY_DATASET_ID, null, options.getBigQueryLocation()); // Create table and insert data for new type query test cases. BQ_CLIENT.createNewTable( diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilsTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilsTest.java index 754a67e5410d1..d73ff5e2b7124 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilsTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilsTest.java @@ -19,6 +19,7 @@ import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils.toTableRow; import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils.toTableSchema; +import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils.toTableSpec; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -55,7 +56,7 @@ import org.apache.beam.sdk.schemas.logicaltypes.EnumerationType; import org.apache.beam.sdk.schemas.logicaltypes.SqlTypes; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.DateTime; import org.joda.time.Instant; import org.joda.time.chrono.ISOChronology; @@ -87,6 +88,7 @@ public class BigQueryUtilsTest { .addNullableField("time0s_0ns", Schema.FieldType.logicalType(SqlTypes.TIME)) .addNullableField("valid", Schema.FieldType.BOOLEAN) .addNullableField("binary", Schema.FieldType.BYTES) + .addNullableField("raw_bytes", Schema.FieldType.BYTES) .addNullableField("numeric", Schema.FieldType.DECIMAL) .addNullableField("boolean", Schema.FieldType.BOOLEAN) .addNullableField("long", Schema.FieldType.INT64) @@ -105,6 +107,9 @@ public class BigQueryUtilsTest { private static final Schema MAP_TYPE = Schema.builder().addStringField("key").addDoubleField("value").build(); + private static final Schema ARRAY_TYPE_NULLS = + Schema.builder().addArrayField("ids", Schema.FieldType.INT64.withNullable(true)).build(); + private static final Schema ARRAY_TYPE = Schema.builder().addArrayField("ids", Schema.FieldType.INT64).build(); @@ -185,6 +190,9 @@ public class BigQueryUtilsTest { private static final TableFieldSchema BINARY = new TableFieldSchema().setName("binary").setType(StandardSQLTypeName.BYTES.toString()); + private static final TableFieldSchema RAW_BYTES = + new TableFieldSchema().setName("raw_bytes").setType(StandardSQLTypeName.BYTES.toString()); + private static final TableFieldSchema NUMERIC = new TableFieldSchema().setName("numeric").setType(StandardSQLTypeName.NUMERIC.toString()); @@ -243,6 +251,7 @@ public class BigQueryUtilsTest { TIME_0S_0NS, VALID, BINARY, + RAW_BYTES, NUMERIC, BOOLEAN, LONG, @@ -273,6 +282,7 @@ public class BigQueryUtilsTest { TIME_0S_0NS, VALID, BINARY, + RAW_BYTES, NUMERIC, BOOLEAN, LONG, @@ -313,6 +323,7 @@ public class BigQueryUtilsTest { LocalTime.parse("12:34"), false, Base64.getDecoder().decode("ABCD1234"), + Base64.getDecoder().decode("ABCD1234"), new BigDecimal("123.456").setScale(3, RoundingMode.HALF_UP), true, 123L, @@ -343,6 +354,7 @@ public class BigQueryUtilsTest { .set("time0s_0ns", "12:34:00") .set("valid", "false") .set("binary", "ABCD1234") + .set("raw_bytes", Base64.getDecoder().decode("ABCD1234")) .set("numeric", "123.456") .set("boolean", true) .set("long", 123L) @@ -352,7 +364,7 @@ public class BigQueryUtilsTest { Row.withSchema(FLAT_TYPE) .addValues( null, null, null, null, null, null, null, null, null, null, null, null, null, null, - null, null, null, null, null, null, null, null) + null, null, null, null, null, null, null, null, null) .build(); private static final TableRow BQ_NULL_FLAT_ROW = @@ -375,6 +387,7 @@ public class BigQueryUtilsTest { .set("time0s_0ns", null) .set("valid", null) .set("binary", null) + .set("raw_bytes", null) .set("numeric", null) .set("boolean", null) .set("long", null) @@ -388,12 +401,24 @@ public class BigQueryUtilsTest { private static final TableRow BQ_ENUM_ROW = new TableRow().set("color", "GREEN"); + private static final Row ARRAY_ROW_NULLS = + Row.withSchema(ARRAY_TYPE_NULLS).addValues((Object) Arrays.asList(123L, null, null)).build(); + private static final Row ARRAY_ROW = Row.withSchema(ARRAY_TYPE).addValues((Object) Arrays.asList(123L, 124L)).build(); private static final Row MAP_ROW = Row.withSchema(MAP_MAP_TYPE).addValues(ImmutableMap.of("test", 123.456)).build(); + private static final TableRow BQ_ARRAY_ROW_NULLS = + new TableRow() + .set( + "ids", + Arrays.asList( + Collections.singletonMap("v", "123"), + Collections.singletonMap("v", null), + Collections.singletonMap("v", null))); + private static final TableRow BQ_ARRAY_ROW = new TableRow() .set( @@ -404,6 +429,8 @@ public class BigQueryUtilsTest { // sometimes, a TableRow array will not be of format [{v: value1}, {v: value2}, ...] // it will instead be of format [value1, value2, ...] // this "inline row" covers the latter case + private static final TableRow BQ_INLINE_ARRAY_ROW_NULLS = + new TableRow().set("ids", Arrays.asList("123", null, null)); private static final TableRow BQ_INLINE_ARRAY_ROW = new TableRow().set("ids", Arrays.asList("123", "124")); @@ -440,6 +467,7 @@ public class BigQueryUtilsTest { TIME_0S_0NS, VALID, BINARY, + RAW_BYTES, NUMERIC, BOOLEAN, LONG, @@ -495,6 +523,7 @@ public void testToTableSchema_flat() { TIME_0S_0NS, VALID, BINARY, + RAW_BYTES, NUMERIC, BOOLEAN, LONG, @@ -545,6 +574,7 @@ public void testToTableSchema_row() { TIME_0S_0NS, VALID, BINARY, + RAW_BYTES, NUMERIC, BOOLEAN, LONG, @@ -581,6 +611,7 @@ public void testToTableSchema_array_row() { TIME_0S_0NS, VALID, BINARY, + RAW_BYTES, NUMERIC, BOOLEAN, LONG, @@ -603,7 +634,7 @@ public void testToTableSchema_map() { public void testToTableRow_flat() { TableRow row = toTableRow().apply(FLAT_ROW); - assertThat(row.size(), equalTo(22)); + assertThat(row.size(), equalTo(23)); assertThat(row, hasEntry("id", "123")); assertThat(row, hasEntry("value", "123.456")); assertThat(row, hasEntry("datetime", "2020-11-02T12:34:56.789876")); @@ -618,6 +649,7 @@ public void testToTableRow_flat() { assertThat(row, hasEntry("name", "test")); assertThat(row, hasEntry("valid", "false")); assertThat(row, hasEntry("binary", "ABCD1234")); + assertThat(row, hasEntry("raw_bytes", "ABCD1234")); assertThat(row, hasEntry("numeric", "123.456")); assertThat(row, hasEntry("boolean", "true")); assertThat(row, hasEntry("long", "123")); @@ -657,7 +689,7 @@ public void testToTableRow_row() { assertThat(row.size(), equalTo(1)); row = (TableRow) row.get("row"); - assertThat(row.size(), equalTo(22)); + assertThat(row.size(), equalTo(23)); assertThat(row, hasEntry("id", "123")); assertThat(row, hasEntry("value", "123.456")); assertThat(row, hasEntry("datetime", "2020-11-02T12:34:56.789876")); @@ -672,6 +704,7 @@ public void testToTableRow_row() { assertThat(row, hasEntry("name", "test")); assertThat(row, hasEntry("valid", "false")); assertThat(row, hasEntry("binary", "ABCD1234")); + assertThat(row, hasEntry("raw_bytes", "ABCD1234")); assertThat(row, hasEntry("numeric", "123.456")); assertThat(row, hasEntry("boolean", "true")); assertThat(row, hasEntry("long", "123")); @@ -684,7 +717,7 @@ public void testToTableRow_array_row() { assertThat(row.size(), equalTo(1)); row = ((List) row.get("rows")).get(0); - assertThat(row.size(), equalTo(22)); + assertThat(row.size(), equalTo(23)); assertThat(row, hasEntry("id", "123")); assertThat(row, hasEntry("value", "123.456")); assertThat(row, hasEntry("datetime", "2020-11-02T12:34:56.789876")); @@ -699,6 +732,7 @@ public void testToTableRow_array_row() { assertThat(row, hasEntry("name", "test")); assertThat(row, hasEntry("valid", "false")); assertThat(row, hasEntry("binary", "ABCD1234")); + assertThat(row, hasEntry("raw_bytes", "ABCD1234")); assertThat(row, hasEntry("numeric", "123.456")); assertThat(row, hasEntry("boolean", "true")); assertThat(row, hasEntry("long", "123")); @@ -709,7 +743,7 @@ public void testToTableRow_array_row() { public void testToTableRow_null_row() { TableRow row = toTableRow().apply(NULL_FLAT_ROW); - assertThat(row.size(), equalTo(22)); + assertThat(row.size(), equalTo(23)); assertThat(row, hasEntry("id", null)); assertThat(row, hasEntry("value", null)); assertThat(row, hasEntry("name", null)); @@ -728,6 +762,7 @@ public void testToTableRow_null_row() { assertThat(row, hasEntry("time0s_0ns", null)); assertThat(row, hasEntry("valid", null)); assertThat(row, hasEntry("binary", null)); + assertThat(row, hasEntry("raw_bytes", null)); assertThat(row, hasEntry("numeric", null)); assertThat(row, hasEntry("boolean", null)); assertThat(row, hasEntry("long", null)); @@ -878,12 +913,24 @@ public void testToBeamRow_enum() { assertEquals(ENUM_STRING_ROW, beamRow); } + @Test + public void testToBeamRow_arrayNulls() { + Row beamRow = BigQueryUtils.toBeamRow(ARRAY_TYPE_NULLS, BQ_ARRAY_ROW_NULLS); + assertEquals(ARRAY_ROW_NULLS, beamRow); + } + @Test public void testToBeamRow_array() { Row beamRow = BigQueryUtils.toBeamRow(ARRAY_TYPE, BQ_ARRAY_ROW); assertEquals(ARRAY_ROW, beamRow); } + @Test + public void testToBeamRow_inlineArrayNulls() { + Row beamRow = BigQueryUtils.toBeamRow(ARRAY_TYPE_NULLS, BQ_INLINE_ARRAY_ROW_NULLS); + assertEquals(ARRAY_ROW_NULLS, beamRow); + } + @Test public void testToBeamRow_inlineArray() { Row beamRow = BigQueryUtils.toBeamRow(ARRAY_TYPE, BQ_INLINE_ARRAY_ROW); @@ -948,6 +995,27 @@ public void testToBeamRow_avro_array_array_row() { assertEquals(expected, beamRow); } + @Test + public void testToTableSpec() { + TableReference withProject = + new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table"); + TableReference withoutProject = + new TableReference().setDatasetId("dataset").setTableId("table"); + TableReference withDatasetOnly = new TableReference().setDatasetId("dataset"); + TableReference withTableOnly = new TableReference().setTableId("table"); + + assertEquals("project.dataset.table", toTableSpec(withProject)); + assertEquals("dataset.table", toTableSpec(withoutProject)); + assertThrows( + "must include at least a dataset and a table", + IllegalArgumentException.class, + () -> toTableSpec(withDatasetOnly)); + assertThrows( + "must include at least a dataset and a table", + IllegalArgumentException.class, + () -> toTableSpec(withTableOnly)); + } + @Test public void testToTableReference() { { @@ -974,6 +1042,14 @@ public void testToTableReference() { assertEquals("mytable", tr.getTableId()); } + { + // Test project that contains a dot and colon + TableReference tr = BigQueryUtils.toTableReference("project.with:domain.mydataset.mytable"); + assertEquals("project.with:domain", tr.getProjectId()); + assertEquals("mydataset", tr.getDatasetId()); + assertEquals("mytable", tr.getTableId()); + } + // Invalid scenarios assertNull(BigQueryUtils.toTableReference("")); assertNull(BigQueryUtils.toTableReference(":.")); @@ -985,12 +1061,15 @@ public void testToTableReference() { assertNull(BigQueryUtils.toTableReference("myproject:mydataset.")); assertNull(BigQueryUtils.toTableReference("myproject:mydataset.mytable.")); assertNull(BigQueryUtils.toTableReference("myproject:mydataset:mytable:")); + assertNull(BigQueryUtils.toTableReference("myproject:my dataset:mytable:")); assertNull(BigQueryUtils.toTableReference(".invalidleadingdot.mydataset.mytable")); assertNull(BigQueryUtils.toTableReference("invalidtrailingdot.mydataset.mytable.")); assertNull(BigQueryUtils.toTableReference(":invalidleadingcolon.mydataset.mytable")); assertNull(BigQueryUtils.toTableReference("invalidtrailingcolon.mydataset.mytable:")); - assertNull(BigQueryUtils.toTableReference("myproject.mydataset.mytable.myinvalidpart")); - assertNull(BigQueryUtils.toTableReference("myproject:mydataset.mytable.myinvalidpart")); + assertNull(BigQueryUtils.toTableReference("projectendswithhyphen-.mydataset.mytable")); + assertNull( + BigQueryUtils.toTableReference( + "projectnamegoesbeyondthe30characterlimit.mydataset.mytable")); assertNull( BigQueryUtils.toTableReference("/projects/extraslash/datasets/mydataset/tables/mytable")); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FileLoadsStreamingIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FileLoadsStreamingIT.java new file mode 100644 index 0000000000000..678708062b8d3 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/FileLoadsStreamingIT.java @@ -0,0 +1,502 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.gcp.bigquery; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; + +import com.google.api.services.bigquery.model.Table; +import com.google.api.services.bigquery.model.TableFieldSchema; +import com.google.api.services.bigquery.model.TableReference; +import com.google.api.services.bigquery.model.TableRow; +import com.google.api.services.bigquery.model.TableSchema; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition; +import org.apache.beam.sdk.io.gcp.testing.BigqueryClient; +import org.apache.beam.sdk.options.ExperimentalOptions; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.testing.TestPipelineOptions; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.PeriodicImpulse; +import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TypeDescriptor; +import org.apache.beam.sdk.values.TypeDescriptors; +import org.apache.beam.sdk.values.ValueInSingleWindow; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Duration; +import org.joda.time.Instant; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@RunWith(Parameterized.class) +public class FileLoadsStreamingIT { + private static final Logger LOG = LoggerFactory.getLogger(FileLoadsStreamingIT.class); + + @Parameterized.Parameters + public static Iterable data() { + return ImmutableList.of(new Object[] {false}, new Object[] {true}); + } + + @Parameterized.Parameter(0) + public boolean useInputSchema; + + @Rule public TestName testName = new TestName(); + + private static final BigqueryClient BQ_CLIENT = new BigqueryClient("FileLoadsStreamingIT"); + private static final String PROJECT = + TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); + private static final String BIG_QUERY_DATASET_ID = "file_loads_streaming_it_" + System.nanoTime(); + + private static final String[] FIELDS = { + "BOOL", + "BOOLEAN", + "BYTES", + "INT64", + "INTEGER", + "FLOAT", + "FLOAT64", + "NUMERIC", + "STRING", + "DATE", + "TIMESTAMP" + }; + + private static final int TOTAL_N = 50; + + private final Random randomGenerator = new Random(); + + // used when test suite specifies a particular GCP location for BigQuery operations + private static String bigQueryLocation; + + @BeforeClass + public static void setUpTestEnvironment() throws IOException, InterruptedException { + // Create one BQ dataset for all test cases. + cleanUp(); + bigQueryLocation = + TestPipeline.testingPipelineOptions().as(TestBigQueryOptions.class).getBigQueryLocation(); + BQ_CLIENT.createNewDataset(PROJECT, BIG_QUERY_DATASET_ID, null, bigQueryLocation); + } + + @AfterClass + public static void cleanUp() { + BQ_CLIENT.deleteDataset(PROJECT, BIG_QUERY_DATASET_ID); + } + + static class GenerateRowFunc implements SerializableFunction { + private final List fieldNames; + + public GenerateRowFunc(List fieldNames) { + this.fieldNames = fieldNames; + } + + @Override + public TableRow apply(Long rowId) { + TableRow row = new TableRow(); + row.set("id", rowId); + + for (String name : fieldNames) { + String type = Iterables.get(Splitter.on('_').split(name), 0); + switch (type) { + case "BOOL": + case "BOOLEAN": + if (rowId % 2 == 0) { + row.set(name, false); + } else { + row.set(name, true); + } + break; + case "BYTES": + row.set(name, String.format("test_blob_%s", rowId).getBytes(StandardCharsets.UTF_8)); + break; + case "INT64": + case "INTEGER": + row.set(name, String.valueOf(rowId + 10)); + break; + case "FLOAT": + case "FLOAT64": + row.set(name, String.valueOf(0.5 + rowId)); + break; + case "NUMERIC": + row.set(name, String.valueOf(rowId + 0.12345)); + break; + case "DATE": + row.set(name, "2022-01-01"); + break; + case "TIMESTAMP": + row.set(name, "2022-01-01 10:10:10.012 UTC"); + break; + case "STRING": + row.set(name, "test_string" + rowId); + break; + default: + row.set(name, "unknown" + rowId); + break; + } + } + return row; + } + } + + private static TableSchema makeTableSchemaFromTypes(List fieldNames) { + ImmutableList.Builder builder = ImmutableList.builder(); + + // Add an id field for verification of correctness + builder.add(new TableFieldSchema().setType("INTEGER").setName("id").setMode("REQUIRED")); + + // the name is prefix with type_. + for (String name : fieldNames) { + String mode = "REQUIRED"; + builder.add(new TableFieldSchema().setType(name).setName(name).setMode(mode)); + } + + return new TableSchema().setFields(builder.build()); + } + + private String maybeCreateTable(TableSchema tableSchema, String suffix) + throws IOException, InterruptedException { + String tableId = Iterables.get(Splitter.on('[').split(testName.getMethodName()), 0); + + BQ_CLIENT.deleteTable(PROJECT, BIG_QUERY_DATASET_ID, tableId + suffix); + if (!useInputSchema) { + BQ_CLIENT.createNewTable( + PROJECT, + BIG_QUERY_DATASET_ID, + new Table() + .setSchema(tableSchema) + .setTableReference( + new TableReference() + .setTableId(tableId + suffix) + .setDatasetId(BIG_QUERY_DATASET_ID) + .setProjectId(PROJECT))); + } else { + tableId += "WithInputSchema"; + } + return String.format("%s.%s.%s", PROJECT, BIG_QUERY_DATASET_ID, tableId + suffix); + } + + private void runStreaming(int numFileShards, boolean useCopyJobs) + throws IOException, InterruptedException { + TestPipelineOptions opts = TestPipeline.testingPipelineOptions().as(TestPipelineOptions.class); + opts.setTempLocation(opts.getTempRoot()); + Pipeline p = Pipeline.create(opts); + + // Only run the most relevant test case on Dataflow. + // Testing this dimension on DirectRunner is sufficient + if (p.getOptions().getRunner().getName().contains("DataflowRunner")) { + assumeTrue("Skipping in favor of more relevant test case", useInputSchema); + // Need to manually enable streaming engine for legacy dataflow runner + ExperimentalOptions.addExperiment( + p.getOptions().as(ExperimentalOptions.class), GcpOptions.STREAMING_ENGINE_EXPERIMENT); + } + + List fieldNamesOrigin = Arrays.asList(FIELDS); + // Shuffle the fields in the write schema to do fuzz testing on field order + List fieldNamesShuffled = new ArrayList(fieldNamesOrigin); + Collections.shuffle(fieldNamesShuffled, randomGenerator); + + TableSchema bqTableSchema = makeTableSchemaFromTypes(fieldNamesOrigin); + TableSchema inputSchema = makeTableSchemaFromTypes(fieldNamesShuffled); + String tableSpec = maybeCreateTable(bqTableSchema, ""); + + // set up and build pipeline + Instant start = new Instant(0); + GenerateRowFunc generateRowFunc = new GenerateRowFunc(fieldNamesShuffled); + PCollection instants = + p.apply( + "Generate Instants", + PeriodicImpulse.create() + .startAt(start) + .stopAt(start.plus(Duration.standardSeconds(TOTAL_N - 1))) + .withInterval(Duration.standardSeconds(1)) + .catchUpToNow(false)); + PCollection rows = + instants.apply( + "Create TableRows", + MapElements.into(TypeDescriptor.of(TableRow.class)) + .via(instant -> generateRowFunc.apply(instant.getMillis() / 1000))); + // build write transform + Write write = + BigQueryIO.writeTableRows() + .to(tableSpec) + .withMethod(Write.Method.FILE_LOADS) + .withTriggeringFrequency(Duration.standardSeconds(10)); + if (useCopyJobs) { + write = write.withMaxBytesPerPartition(250); + } + if (useInputSchema) { + // we're creating the table with the input schema + write = + write + .withSchema(inputSchema) + .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) + .withWriteDisposition(WriteDisposition.WRITE_TRUNCATE); + } else { + // table already exists with a schema, no need to create it + write = + write + .withCreateDisposition(CreateDisposition.CREATE_NEVER) + .withWriteDisposition(WriteDisposition.WRITE_APPEND); + } + write = numFileShards == 0 ? write.withAutoSharding() : write.withNumFileShards(numFileShards); + + rows.apply("Stream loads to BigQuery", write); + p.run().waitUntilFinish(); + + List expectedRows = new ArrayList<>(); + for (long i = 0; i < TOTAL_N; i++) { + expectedRows.add(generateRowFunc.apply(i)); + } + + // Perform checks + checkRowCompleteness(tableSpec, inputSchema, expectedRows); + } + + // Check that the expected rows reached the table. + private static void checkRowCompleteness( + String tableSpec, TableSchema schema, List expectedRows) + throws IOException, InterruptedException { + List actualTableRows = + BQ_CLIENT.queryUnflattened( + String.format("SELECT * FROM [%s]", tableSpec), PROJECT, true, false, bigQueryLocation); + + Schema rowSchema = BigQueryUtils.fromTableSchema(schema); + List actualBeamRows = + actualTableRows.stream() + .map(tableRow -> BigQueryUtils.toBeamRow(rowSchema, tableRow)) + .collect(Collectors.toList()); + List expectedBeamRows = + expectedRows.stream() + .map(tableRow -> BigQueryUtils.toBeamRow(rowSchema, tableRow)) + .collect(Collectors.toList()); + LOG.info( + "Actual rows number: {}, expected: {}", actualBeamRows.size(), expectedBeamRows.size()); + + assertThat( + "Comparing expected rows with actual rows", + actualBeamRows, + containsInAnyOrder(expectedBeamRows.toArray())); + assertEquals( + "Checking there is no duplication", expectedBeamRows.size(), actualBeamRows.size()); + } + + @Test + public void testLoadWithFixedShards() throws IOException, InterruptedException { + runStreaming(5, false); + } + + @Test + public void testLoadWithAutoShardingAndCopyJobs() throws IOException, InterruptedException { + runStreaming(0, true); + } + + @Test + public void testDynamicDestinationsWithFixedShards() throws IOException, InterruptedException { + runStreamingToDynamicDestinations(6, false); + } + + @Test + public void testDynamicDestinationsWithAutoShardingAndCopyJobs() + throws IOException, InterruptedException { + runStreamingToDynamicDestinations(0, true); + } + + private void runStreamingToDynamicDestinations(int numFileShards, boolean useCopyJobs) + throws IOException, InterruptedException { + TestPipelineOptions opts = TestPipeline.testingPipelineOptions().as(TestPipelineOptions.class); + opts.setTempLocation(opts.getTempRoot()); + Pipeline p = Pipeline.create(opts); + // Only run the most relevant test cases on Dataflow. Testing this dimension on DirectRunner is + // sufficient + if (p.getOptions().getRunner().getName().contains("DataflowRunner")) { + assumeTrue("Skipping in favor of more relevant test case", useInputSchema); + // Need to manually enable streaming engine for legacy dataflow runner + ExperimentalOptions.addExperiment( + p.getOptions().as(ExperimentalOptions.class), GcpOptions.STREAMING_ENGINE_EXPERIMENT); + } + + List allFields = Arrays.asList(FIELDS); + List subFields0 = new ArrayList<>(allFields.subList(0, 4)); + List subFields1 = new ArrayList<>(allFields.subList(4, 8)); + List subFields2 = new ArrayList<>(allFields.subList(8, 11)); + TableSchema table0Schema = makeTableSchemaFromTypes(subFields0); + TableSchema table1Schema = makeTableSchemaFromTypes(subFields1); + TableSchema table2Schema = makeTableSchemaFromTypes(subFields2); + String table0Id = maybeCreateTable(table0Schema, "-0"); + String table1Id = maybeCreateTable(table1Schema, "-1"); + String table2Id = maybeCreateTable(table2Schema, "-2"); + GenerateRowFunc generateRowFunc0 = new GenerateRowFunc(subFields0); + GenerateRowFunc generateRowFunc1 = new GenerateRowFunc(subFields1); + GenerateRowFunc generateRowFunc2 = new GenerateRowFunc(subFields2); + + String tablePrefix = table0Id.substring(0, table0Id.length() - 2); + + // set up and build pipeline + Instant start = new Instant(0); + PCollection instants = + p.apply( + "Generate Instants", + PeriodicImpulse.create() + .startAt(start) + .stopAt(start.plus(Duration.standardSeconds(TOTAL_N - 1))) + .withInterval(Duration.standardSeconds(1)) + .catchUpToNow(false)); + PCollection longs = + instants.apply( + "Create TableRows", + MapElements.into(TypeDescriptors.longs()).via(instant -> instant.getMillis() / 1000)); + // build write transform + Write write = + BigQueryIO.write() + .to( + new TestDynamicDest( + tablePrefix, subFields0, subFields1, subFields2, useInputSchema)) + .withFormatFunction( + id -> { + long dest = id % 3; + TableRow row; + if (dest == 0) { + row = generateRowFunc0.apply(id); + } else if (dest == 1) { + row = generateRowFunc1.apply(id); + } else { + row = generateRowFunc2.apply(id); + } + return row; + }) + .withMethod(Write.Method.FILE_LOADS) + .withTriggeringFrequency(Duration.standardSeconds(10)); + if (useCopyJobs) { + write = write.withMaxBytesPerPartition(150); + } + if (useInputSchema) { + // we're creating the table with the input schema + write = + write + .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) + .withWriteDisposition(WriteDisposition.WRITE_TRUNCATE); + } else { + // table already exists with a schema, no need to create it + write = + write + .withCreateDisposition(CreateDisposition.CREATE_NEVER) + .withWriteDisposition(WriteDisposition.WRITE_APPEND); + } + write = numFileShards == 0 ? write.withAutoSharding() : write.withNumFileShards(numFileShards); + + longs.apply("Stream loads to dynamic destinations", write); + p.run().waitUntilFinish(); + + List expectedRows0 = new ArrayList<>(); + List expectedRows1 = new ArrayList<>(); + List expectedRows2 = new ArrayList<>(); + for (long i = 0; i < TOTAL_N; i++) { + long dest = i % 3; + if (dest == 0) { + expectedRows0.add(generateRowFunc0.apply(i)); + } else if (dest == 1) { + expectedRows1.add(generateRowFunc1.apply(i)); + } else { + expectedRows2.add(generateRowFunc2.apply(i)); + } + } + // Perform checks + checkRowCompleteness(table0Id, makeTableSchemaFromTypes(subFields0), expectedRows0); + checkRowCompleteness(table1Id, makeTableSchemaFromTypes(subFields1), expectedRows1); + checkRowCompleteness(table2Id, makeTableSchemaFromTypes(subFields2), expectedRows2); + } + + static class TestDynamicDest extends DynamicDestinations { + String tablePrefix; + List table0Fields; + List table1Fields; + List table2Fields; + boolean useInputSchema; + + public TestDynamicDest( + String tablePrefix, + List table0Fields, + List table1Fields, + List table2Fields, + boolean useInputSchema) { + this.tablePrefix = tablePrefix; + this.table0Fields = table0Fields; + this.table1Fields = table1Fields; + this.table2Fields = table2Fields; + this.useInputSchema = useInputSchema; + } + + @Override + public Long getDestination(@Nullable ValueInSingleWindow element) { + return element.getValue() % 3; + } + + @Override + public TableDestination getTable(Long destination) { + return new TableDestination(tablePrefix + "-" + destination, null); + } + + @Override + public @Nullable TableSchema getSchema(Long destination) { + if (!useInputSchema) { + return null; + } + List fields; + if (destination == 0) { + fields = table0Fields; + } else if (destination == 1) { + fields = table1Fields; + } else { + fields = table2Fields; + } + List tableFields = + fields.stream() + .map(name -> new TableFieldSchema().setName(name).setType(name).setMode("REQUIRED")) + .collect(Collectors.toList()); + // we attach an ID to each row in addition to the existing schema fields + tableFields.add( + 0, new TableFieldSchema().setName("id").setType("INTEGER").setMode("REQUIRED")); + return new TableSchema().setFields(tableFields); + } + } +} diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/InsertRetryPolicyTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/InsertRetryPolicyTest.java index 164d7d6fd82f7..89ceab3e5ed70 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/InsertRetryPolicyTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/InsertRetryPolicyTest.java @@ -25,7 +25,7 @@ import java.util.List; import java.util.concurrent.ThreadLocalRandom; import org.apache.beam.sdk.io.gcp.bigquery.InsertRetryPolicy.Context; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/RetryManagerTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/RetryManagerTest.java index 6e8a5746b0b29..958fd356344c4 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/RetryManagerTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/RetryManagerTest.java @@ -25,8 +25,8 @@ import org.apache.beam.sdk.io.gcp.bigquery.RetryManager.Operation; import org.apache.beam.sdk.io.gcp.bigquery.RetryManager.Operation.Context; import org.apache.beam.sdk.io.gcp.bigquery.RetryManager.RetryType; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.joda.time.Duration; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDirectWriteProtosIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDirectWriteProtosIT.java new file mode 100644 index 0000000000000..3da93c42a4800 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDirectWriteProtosIT.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.gcp.bigquery; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; + +import com.google.api.services.bigquery.model.TableRow; +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; +import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages; +import org.apache.beam.sdk.io.gcp.testing.BigqueryClient; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; +import org.joda.time.Duration; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** An example that exports nested BigQuery record to a file. */ +@RunWith(Parameterized.class) +public class StorageApiDirectWriteProtosIT { + @Parameterized.Parameters + public static Iterable data() { + return ImmutableList.of( + new Object[] {true, false, false}, + new Object[] {false, true, false}, + new Object[] {false, false, true}, + new Object[] {true, false, true}); + } + + @Parameterized.Parameter(0) + public boolean useStreamingExactlyOnce; + + @Parameterized.Parameter(1) + public boolean useAtLeastOnce; + + @Parameterized.Parameter(2) + public boolean useBatch; + + private static final BigqueryClient BQ_CLIENT = + new BigqueryClient("StorageApiDirectWriteProtosIT"); + private static final String PROJECT = + TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); + private static final String BIG_QUERY_DATASET_ID = + "storage_api_sink_direct_write_protos" + System.nanoTime(); + + private BigQueryIO.Write.Method getMethod() { + return useAtLeastOnce + ? BigQueryIO.Write.Method.STORAGE_API_AT_LEAST_ONCE + : BigQueryIO.Write.Method.STORAGE_WRITE_API; + } + + // used when test suite specifies a particular GCP location for BigQuery operations + private static String bigQueryLocation; + + @BeforeClass + public static void setUpTestEnvironment() throws IOException, InterruptedException { + // Create one BQ dataset for all test cases. + bigQueryLocation = + TestPipeline.testingPipelineOptions().as(TestBigQueryOptions.class).getBigQueryLocation(); + BQ_CLIENT.createNewDataset(PROJECT, BIG_QUERY_DATASET_ID, null, bigQueryLocation); + } + + @AfterClass + public static void cleanup() { + BQ_CLIENT.deleteDataset(PROJECT, BIG_QUERY_DATASET_ID); + } + + @Test + public void testDirectWriteProtos() throws Exception { + Function getPrimitiveProto = + (Integer i) -> + Proto3SchemaMessages.Primitive.newBuilder() + .setPrimitiveDouble(i) + .setPrimitiveFloat(i) + .setPrimitiveInt32(i) + .setPrimitiveInt64(i) + .setPrimitiveUint32(i) + .setPrimitiveUint64(i) + .setPrimitiveSint32(i) + .setPrimitiveSint64(i) + .setPrimitiveFixed32(i) + .setPrimitiveFixed64(i) + .setPrimitiveBool(true) + .setPrimitiveString(Integer.toString(i)) + .setPrimitiveBytes( + ByteString.copyFrom(Integer.toString(i).getBytes(StandardCharsets.UTF_8))) + .build(); + Function getPrimitiveRow = + (Integer i) -> + new TableRow() + .set("primitive_double", Double.valueOf(i)) + .set("primitive_float", Float.valueOf(i).doubleValue()) + .set("primitive_int32", i.toString()) + .set("primitive_int64", i.toString()) + .set("primitive_uint32", i.toString()) + .set("primitive_uint64", i.toString()) + .set("primitive_sint32", i.toString()) + .set("primitive_sint64", i.toString()) + .set("primitive_fixed32", i.toString()) + .set("primitive_fixed64", i.toString()) + .set("primitive_bool", true) + .set("primitive_string", i.toString()) + .set( + "primitive_bytes", + BaseEncoding.base64() + .encode( + ByteString.copyFrom(i.toString().getBytes(StandardCharsets.UTF_8)) + .toByteArray())); + + List nestedItems = + IntStream.range(1, 2) + .mapToObj(i -> getPrimitiveProto.apply(i)) + .collect(Collectors.toList()); + + Iterable items = + nestedItems.stream() + .map( + p -> + Proto3SchemaMessages.Nested.newBuilder() + .setNested(p) + .addAllNestedList(Lists.newArrayList(p, p, p)) + .build()) + .collect(Collectors.toList()); + + List expectedNestedItems = + IntStream.range(1, 2).mapToObj(getPrimitiveRow::apply).collect(Collectors.toList()); + + Iterable expectedItems = + expectedNestedItems.stream() + .map( + p -> + new TableRow() + .set("nested_map", Lists.newArrayList()) + .set("nested", p) + .set("nested_list", Lists.newArrayList(p, p, p))) + .collect(Collectors.toList()); + + String table = "table" + System.nanoTime(); + String tableSpec = PROJECT + "." + BIG_QUERY_DATASET_ID + "." + table; + + BigQueryIO.Write.Method method = getMethod(); + BigQueryIO.Write write = + BigQueryIO.writeProtos(Proto3SchemaMessages.Nested.class) + .to(tableSpec) + .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) + .withMethod(method); + if (method == BigQueryIO.Write.Method.STORAGE_WRITE_API) { + write = write.withNumStorageWriteApiStreams(1); + if (useStreamingExactlyOnce) { + write = write.withTriggeringFrequency(Duration.standardSeconds(1)); + } + } + + Pipeline p = Pipeline.create(); + + PCollection input = p.apply("Create test cases", Create.of(items)); + if (useStreamingExactlyOnce) { + input = input.setIsBoundedInternal(PCollection.IsBounded.UNBOUNDED); + } + input.apply("Write using Storage Write API", write); + p.run().waitUntilFinish(); + assertRowsWritten(tableSpec, expectedItems); + } + + void assertRowsWritten(String tableSpec, Iterable expectedItems) throws Exception { + List rows = + BQ_CLIENT.queryUnflattened( + String.format("SELECT * FROM %s", tableSpec), PROJECT, true, true, bigQueryLocation); + assertThat(rows, containsInAnyOrder(Iterables.toArray(expectedItems, TableRow.class))); + } +} diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiSinkCreateIfNeededIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiSinkCreateIfNeededIT.java index 3cb44c6c7b62f..18c832f0c54b3 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiSinkCreateIfNeededIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiSinkCreateIfNeededIT.java @@ -32,8 +32,8 @@ import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.hamcrest.Matchers; import org.joda.time.Duration; import org.junit.AfterClass; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiSinkFailedRowsIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiSinkFailedRowsIT.java index 465bebbf13895..f721f57147e3d 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiSinkFailedRowsIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiSinkFailedRowsIT.java @@ -35,9 +35,9 @@ import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.hamcrest.Matchers; import org.joda.time.Duration; import org.junit.AfterClass; @@ -108,10 +108,15 @@ private BigQueryIO.Write.Method getMethod() { : BigQueryIO.Write.Method.STORAGE_WRITE_API; } + // used when test suite specifies a particular GCP location for BigQuery operations + private static String bigQueryLocation; + @BeforeClass public static void setUpTestEnvironment() throws IOException, InterruptedException { // Create one BQ dataset for all test cases. - BQ_CLIENT.createNewDataset(PROJECT, BIG_QUERY_DATASET_ID); + bigQueryLocation = + TestPipeline.testingPipelineOptions().as(TestBigQueryOptions.class).getBigQueryLocation(); + BQ_CLIENT.createNewDataset(PROJECT, BIG_QUERY_DATASET_ID, null, bigQueryLocation); } @AfterClass @@ -217,7 +222,11 @@ private void assertGoodRowsWritten(String tableSpec, Iterable goodRows TableRow queryResponse = Iterables.getOnlyElement( BQ_CLIENT.queryUnflattened( - String.format("SELECT COUNT(*) FROM %s", tableSpec), PROJECT, true, true)); + String.format("SELECT COUNT(*) FROM `%s`", tableSpec), + PROJECT, + true, + true, + bigQueryLocation)); int numRowsWritten = Integer.parseInt((String) queryResponse.get("f0_")); if (useAtLeastOnce) { assertThat(numRowsWritten, Matchers.greaterThanOrEqualTo(Iterables.size(goodRows))); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiSinkRowUpdateIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiSinkRowUpdateIT.java index a7a78dd73dba0..f8cc797a87cd5 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiSinkRowUpdateIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiSinkRowUpdateIT.java @@ -20,6 +20,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; +import com.google.api.services.bigquery.model.Clustering; import com.google.api.services.bigquery.model.TableFieldSchema; import com.google.api.services.bigquery.model.TableRow; import com.google.api.services.bigquery.model.TableSchema; @@ -30,9 +31,9 @@ import org.apache.beam.sdk.io.gcp.testing.BigqueryClient; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -48,10 +49,15 @@ public class StorageApiSinkRowUpdateIT { private static final String BIG_QUERY_DATASET_ID = "storage_api_sink_rows_update" + System.nanoTime(); + // used when test suite specifies a particular GCP location for BigQuery operations + private static String bigQueryLocation; + @BeforeClass public static void setUpTestEnvironment() throws IOException, InterruptedException { // Create one BQ dataset for all test cases. - BQ_CLIENT.createNewDataset(PROJECT, BIG_QUERY_DATASET_ID); + bigQueryLocation = + TestPipeline.testingPipelineOptions().as(TestBigQueryOptions.class).getBigQueryLocation(); + BQ_CLIENT.createNewDataset(PROJECT, BIG_QUERY_DATASET_ID, null, bigQueryLocation); } @AfterClass @@ -59,38 +65,8 @@ public static void cleanup() { BQ_CLIENT.deleteDataset(PROJECT, BIG_QUERY_DATASET_ID); } - private static String createTable(TableSchema tableSchema, List primaryKey) - throws IOException, InterruptedException { - String table = "table" + System.nanoTime(); - - BQ_CLIENT.deleteTable(PROJECT, BIG_QUERY_DATASET_ID, table); - - StringBuilder ddl = - new StringBuilder("CREATE TABLE ") - .append(PROJECT) - .append(".") - .append(BIG_QUERY_DATASET_ID) - .append(".") - .append(table) - .append("("); - for (TableFieldSchema tableFieldSchema : tableSchema.getFields()) { - ddl.append(tableFieldSchema.getName()) - .append(" ") - .append(tableFieldSchema.getType()) - .append(","); - } - - String primaryKeyString = String.join(",", primaryKey); - ddl.append(" PRIMARY KEY ") - .append("(") - .append(primaryKeyString) - .append(")") - .append(" NOT ENFORCED) "); - ddl.append("CLUSTER BY ").append(primaryKeyString); - - BQ_CLIENT.queryWithRetriesUsingStandardSql(ddl.toString(), PROJECT); - - return PROJECT + "." + BIG_QUERY_DATASET_ID + "." + table; + private static String getTablespec() { + return PROJECT + "." + BIG_QUERY_DATASET_ID + "." + "table" + System.nanoTime(); } @Test @@ -130,7 +106,8 @@ public void testCdc() throws Exception { new TableRow().set("key1", "foo4").set("key2", "bar4").set("value", "1"), RowMutationInformation.of(RowMutationInformation.MutationType.DELETE, 1))); - String tableSpec = createTable(tableSchema, Lists.newArrayList("key1", "key2")); + List primaryKey = Lists.newArrayList("key1", "key2"); + String tableSpec = getTablespec(); Pipeline p = Pipeline.create(); p.apply("Create rows", Create.of(items)) .apply( @@ -138,8 +115,10 @@ public void testCdc() throws Exception { BigQueryIO.applyRowMutations() .to(tableSpec) .withSchema(tableSchema) + .withPrimaryKey(primaryKey) + .withClustering(new Clustering().setFields(primaryKey)) .withMethod(BigQueryIO.Write.Method.STORAGE_API_AT_LEAST_ONCE) - .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_NEVER)); + .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)); p.run(); @@ -155,7 +134,7 @@ private void assertRowsWritten(String tableSpec, Iterable expected) throws IOException, InterruptedException { List queryResponse = BQ_CLIENT.queryUnflattened( - String.format("SELECT * FROM %s", tableSpec), PROJECT, true, true); + String.format("SELECT * FROM %s", tableSpec), PROJECT, true, true, bigQueryLocation); assertThat(queryResponse, containsInAnyOrder(Iterables.toArray(expected, TableRow.class))); } } diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiSinkSchemaUpdateIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiSinkSchemaUpdateIT.java new file mode 100644 index 0000000000000..bc99a4f50f700 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiSinkSchemaUpdateIT.java @@ -0,0 +1,542 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.gcp.bigquery; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +import com.google.api.services.bigquery.model.Table; +import com.google.api.services.bigquery.model.TableFieldSchema; +import com.google.api.services.bigquery.model.TableReference; +import com.google.api.services.bigquery.model.TableRow; +import com.google.api.services.bigquery.model.TableSchema; +import java.io.IOException; +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.function.Function; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition; +import org.apache.beam.sdk.io.gcp.testing.BigqueryClient; +import org.apache.beam.sdk.options.ExperimentalOptions; +import org.apache.beam.sdk.state.StateSpec; +import org.apache.beam.sdk.state.StateSpecs; +import org.apache.beam.sdk.state.ValueState; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.PeriodicImpulse; +import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.beam.sdk.transforms.WithKeys; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.TypeDescriptor; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.joda.time.Duration; +import org.joda.time.Instant; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@RunWith(Parameterized.class) +public class StorageApiSinkSchemaUpdateIT { + @Parameterized.Parameters + public static Iterable data() { + return ImmutableList.of( + new Object[] {false, false}, + new Object[] {false, true}, + new Object[] {true, false}, + new Object[] {true, true}); + } + + @Parameterized.Parameter(0) + public boolean useInputSchema; + + @Parameterized.Parameter(1) + public boolean changeTableSchema; + + @Rule public TestName testName = new TestName(); + + private static final Logger LOG = LoggerFactory.getLogger(StorageApiSinkSchemaUpdateIT.class); + + private static final BigqueryClient BQ_CLIENT = + new BigqueryClient("StorageApiSinkSchemaChangeIT"); + private static final String PROJECT = + TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); + private static final String BIG_QUERY_DATASET_ID = + "storage_api_sink_schema_change_" + System.nanoTime(); + + private static final String[] FIELDS = { + "BOOL", + "BOOLEAN", + "BYTES", + "INT64", + "INTEGER", + "FLOAT", + "FLOAT64", + "NUMERIC", + "STRING", + "DATE", + "TIMESTAMP" + }; + + // ************ NOTE ************ + // The test may fail if Storage API Streams take longer than expected to recognize + // an updated schema. If that happens consistently, just increase these two numbers + // to give it more time. + // Total number of rows written to the sink + private static final int TOTAL_N = 70; + // Number of rows with the original schema + private static final int ORIGINAL_N = 60; + + private final Random randomGenerator = new Random(); + + // used when test suite specifies a particular GCP location for BigQuery operations + private static String bigQueryLocation; + + @BeforeClass + public static void setUpTestEnvironment() throws IOException, InterruptedException { + // Create one BQ dataset for all test cases. + bigQueryLocation = + TestPipeline.testingPipelineOptions().as(TestBigQueryOptions.class).getBigQueryLocation(); + BQ_CLIENT.createNewDataset(PROJECT, BIG_QUERY_DATASET_ID, null, bigQueryLocation); + } + + @AfterClass + public static void cleanUp() { + LOG.info("Cleaning up dataset {} and tables.", BIG_QUERY_DATASET_ID); + BQ_CLIENT.deleteDataset(PROJECT, BIG_QUERY_DATASET_ID); + } + + private String createTable(TableSchema tableSchema) throws IOException, InterruptedException { + String tableId = Iterables.get(Splitter.on('[').split(testName.getMethodName()), 0); + if (useInputSchema) { + tableId += "WithInputSchema"; + } + if (changeTableSchema) { + tableId += "OnSchemaChange"; + } + BQ_CLIENT.deleteTable(PROJECT, BIG_QUERY_DATASET_ID, tableId); + BQ_CLIENT.createNewTable( + PROJECT, + BIG_QUERY_DATASET_ID, + new Table() + .setSchema(tableSchema) + .setTableReference( + new TableReference() + .setTableId(tableId) + .setDatasetId(BIG_QUERY_DATASET_ID) + .setProjectId(PROJECT))); + return tableId; + } + + static class UpdateSchemaDoFn extends DoFn, TableRow> { + + private final String projectId; + private final String datasetId; + private final String tableId; + // represent as String because TableSchema is not serializable + private final String newSchema; + + private transient BigqueryClient bqClient; + + private static final String ROW_COUNTER = "rowCounter"; + + @StateId(ROW_COUNTER) + @SuppressWarnings("unused") + private final StateSpec<@org.jetbrains.annotations.NotNull ValueState> counter; + + public UpdateSchemaDoFn( + String projectId, String datasetId, String tableId, TableSchema newSchema) { + this.projectId = projectId; + this.datasetId = datasetId; + this.tableId = tableId; + this.newSchema = BigQueryHelpers.toJsonString(newSchema); + this.bqClient = null; + this.counter = StateSpecs.value(); + } + + @Setup + public void setup() { + bqClient = new BigqueryClient("StorageApiSinkSchemaChangeIT"); + } + + @ProcessElement + public void processElement(ProcessContext c, @StateId(ROW_COUNTER) ValueState counter) + throws Exception { + int current = firstNonNull(counter.read(), 0); + // We update schema early on to leave a healthy amount of time for + // StreamWriter to recognize it. + if (current == 10) { + bqClient.updateTableSchema( + projectId, + datasetId, + tableId, + BigQueryHelpers.fromJsonString(newSchema, TableSchema.class)); + } + + counter.write(++current); + c.output(c.element().getValue()); + } + } + + static class GenerateRowFunc implements SerializableFunction { + private final List fieldNames; + private final List fieldNamesWithExtra; + + public GenerateRowFunc(List fieldNames, List fieldNamesWithExtra) { + this.fieldNames = fieldNames; + this.fieldNamesWithExtra = fieldNamesWithExtra; + } + + @Override + public TableRow apply(Long rowId) { + TableRow row = new TableRow(); + row.set("id", rowId); + + List fields = rowId < ORIGINAL_N ? fieldNames : fieldNamesWithExtra; + + for (String name : fields) { + String type = Iterables.get(Splitter.on('_').split(name), 0); + switch (type) { + case "BOOL": + case "BOOLEAN": + if (rowId % 2 == 0) { + row.set(name, false); + } else { + row.set(name, true); + } + break; + case "BYTES": + row.set(name, String.format("test_blob_%s", rowId).getBytes(StandardCharsets.UTF_8)); + break; + case "INT64": + case "INTEGER": + row.set(name, rowId + 10); + break; + case "FLOAT": + case "FLOAT64": + row.set(name, 0.5 + rowId); + break; + case "NUMERIC": + row.set(name, rowId + 0.12345); + break; + case "DATE": + row.set(name, "2022-01-01"); + break; + case "TIMESTAMP": + row.set(name, "2022-01-01T10:10:10.012Z"); + break; + case "STRING": + row.set(name, "test_string" + rowId); + break; + default: + row.set(name, "unknown" + rowId); + break; + } + } + return row; + } + } + + private static TableSchema makeTableSchemaFromTypes( + List fieldNames, Set nullableFieldNames) { + ImmutableList.Builder builder = ImmutableList.builder(); + + // Add an id field for verification of correctness + builder.add(new TableFieldSchema().setType("INTEGER").setName("id").setMode("REQUIRED")); + + // the name is prefix with type_. + for (String name : fieldNames) { + String type = Iterables.get(Splitter.on('_').split(name), 0); + String mode = "REQUIRED"; + if (nullableFieldNames != null && nullableFieldNames.contains(name)) { + mode = "NULLABLE"; + } + builder.add(new TableFieldSchema().setType(type).setName(name).setMode(mode)); + } + + return new TableSchema().setFields(builder.build()); + } + + private void runStreamingPipelineWithSchemaChange( + Write.Method method, boolean useAutoSchemaUpdate, boolean useIgnoreUnknownValues) + throws Exception { + Pipeline p = Pipeline.create(TestPipeline.testingPipelineOptions()); + // Set threshold bytes to 0 so that the stream attempts to fetch an updated schema after each + // append + p.getOptions().as(BigQueryOptions.class).setStorageApiAppendThresholdBytes(0); + // Limit parallelism so that all streams recognize the new schema in an expected short amount + // of time (before we start writing rows with updated schema) + p.getOptions().as(BigQueryOptions.class).setNumStorageWriteApiStreams(3); + // Need to manually enable streaming engine for legacy dataflow runner + ExperimentalOptions.addExperiment( + p.getOptions().as(ExperimentalOptions.class), GcpOptions.STREAMING_ENGINE_EXPERIMENT); + // Only run the most relevant test case on Dataflow + if (p.getOptions().getRunner().getName().contains("DataflowRunner")) { + assumeTrue( + "Skipping in favor of more relevant test case", + changeTableSchema && useInputSchema && useAutoSchemaUpdate); + } + + List fieldNamesOrigin = new ArrayList(Arrays.asList(FIELDS)); + + // Shuffle the fields in the write schema to do fuzz testing on field order + List fieldNamesShuffled = new ArrayList(fieldNamesOrigin); + Collections.shuffle(fieldNamesShuffled, randomGenerator); + + // The updated schema includes all fields in the original schema plus a random new field + List fieldNamesWithExtra = new ArrayList(fieldNamesOrigin); + String extraField = + fieldNamesOrigin.get(randomGenerator.nextInt(fieldNamesOrigin.size())) + "_EXTRA"; + fieldNamesWithExtra.add(extraField); + + TableSchema bqTableSchema = makeTableSchemaFromTypes(fieldNamesOrigin, null); + TableSchema inputSchema = makeTableSchemaFromTypes(fieldNamesShuffled, null); + TableSchema updatedSchema = + makeTableSchemaFromTypes(fieldNamesWithExtra, ImmutableSet.of(extraField)); + + String tableId = createTable(bqTableSchema); + String tableSpec = PROJECT + ":" + BIG_QUERY_DATASET_ID + "." + tableId; + + // build write transform + Write write = + BigQueryIO.writeTableRows() + .to(tableSpec) + .withAutoSchemaUpdate(useAutoSchemaUpdate) + .withMethod(method) + .withCreateDisposition(CreateDisposition.CREATE_NEVER) + .withWriteDisposition(WriteDisposition.WRITE_APPEND); + if (method == Write.Method.STORAGE_WRITE_API) { + write = write.withTriggeringFrequency(Duration.standardSeconds(1)); + } + if (useInputSchema) { + write = write.withSchema(inputSchema); + } + if (useIgnoreUnknownValues) { + write = write.ignoreUnknownValues(); + } + + // set up and build pipeline + Instant start = new Instant(0); + // We give a healthy waiting period between each element to give Storage API streams a chance to + // recognize the new schema. Apply on relevant tests. + boolean waitLonger = changeTableSchema && (useAutoSchemaUpdate || !useInputSchema); + Duration interval = waitLonger ? Duration.standardSeconds(1) : Duration.millis(1); + Duration stop = + waitLonger ? Duration.standardSeconds(TOTAL_N - 1) : Duration.millis(TOTAL_N - 1); + Function getIdFromInstant = + waitLonger + ? (Function & Serializable) + (Instant instant) -> instant.getMillis() / 1000 + : (Function & Serializable) (Instant instant) -> instant.getMillis(); + + // Generates rows with original schema up for row IDs under ORIGINAL_N + // Then generates rows with updated schema for the rest + // Rows with updated schema should only reach the table if ignoreUnknownValues is set, + // and the extra field should be present only when autoSchemaUpdate is set + GenerateRowFunc generateRowFunc = new GenerateRowFunc(fieldNamesOrigin, fieldNamesWithExtra); + PCollection instants = + p.apply( + "Generate Instants", + PeriodicImpulse.create() + .startAt(start) + .stopAt(start.plus(stop)) + .withInterval(interval) + .catchUpToNow(false)); + PCollection rows = + instants.apply( + "Create TableRows", + MapElements.into(TypeDescriptor.of(TableRow.class)) + .via(instant -> generateRowFunc.apply(getIdFromInstant.apply(instant)))); + + if (changeTableSchema) { + rows = + rows + // UpdateSchemaDoFn uses state, so need to have a KV input + .apply("Add a dummy key", WithKeys.of(1)) + .apply( + "Update Schema", + ParDo.of( + new UpdateSchemaDoFn(PROJECT, BIG_QUERY_DATASET_ID, tableId, updatedSchema))); + } + WriteResult result = rows.apply("Stream to BigQuery", write); + if (useIgnoreUnknownValues) { + // We ignore the extra fields, so no rows should have been sent to DLQ + PAssert.that("Check DLQ is empty", result.getFailedStorageApiInserts()).empty(); + } else { + // When we don't set ignoreUnknownValues, the rows with extra fields should be sent to DLQ. + PAssert.that( + String.format("Check DLQ has %s schema errors", TOTAL_N - ORIGINAL_N), + result.getFailedStorageApiInserts()) + .satisfies(new VerifyPCollectionSize(TOTAL_N - ORIGINAL_N, extraField)); + } + p.run().waitUntilFinish(); + + // Check row completeness, non-duplication, and that schema update works as intended. + int expectedCount = useIgnoreUnknownValues ? TOTAL_N : ORIGINAL_N; + boolean checkNoDuplication = (method == Write.Method.STORAGE_WRITE_API) ? true : false; + checkRowCompleteness(tableSpec, expectedCount, checkNoDuplication); + if (useIgnoreUnknownValues) { + checkRowsWithUpdatedSchema(tableSpec, extraField, useAutoSchemaUpdate); + } + } + + private static class VerifyPCollectionSize + implements SerializableFunction, Void> { + int expectedSize; + String extraField; + + VerifyPCollectionSize(int expectedSize, String extraField) { + this.expectedSize = expectedSize; + this.extraField = extraField; + } + + @Override + public Void apply(Iterable input) { + List itemList = new ArrayList<>(); + String expectedError = "SchemaTooNarrowException"; + for (BigQueryStorageApiInsertError err : input) { + itemList.add(err); + // Check the error message is due to schema mismatch from the extra field. + assertTrue( + String.format( + "Didn't find expected [%s] error in failed message: %s", expectedError, err), + err.getErrorMessage().contains(expectedError)); + assertTrue( + String.format( + "Didn't find expected [%s] schema field in failed message: %s", expectedError, err), + err.getErrorMessage().contains(extraField)); + } + // Check we have the expected number of rows in DLQ. + // Should be equal to number of rows with extra fields. + LOG.info("Found {} failed rows in DLQ", itemList.size()); + assertEquals(expectedSize, itemList.size()); + return null; + } + } + + // Check the expected number of rows reached the table. + // If using STORAGE_WRITE_API, check no duplication happened. + private static void checkRowCompleteness( + String tableSpec, int expectedCount, boolean checkNoDuplication) + throws IOException, InterruptedException { + TableRow queryResponse = + Iterables.getOnlyElement( + BQ_CLIENT.queryUnflattened( + String.format("SELECT COUNT(DISTINCT(id)), COUNT(id) FROM [%s]", tableSpec), + PROJECT, + true, + false, + bigQueryLocation)); + + int distinctCount = Integer.parseInt((String) queryResponse.get("f0_")); + int totalCount = Integer.parseInt((String) queryResponse.get("f1_")); + + LOG.info("total distinct count = {}, total count = {}", distinctCount, totalCount); + + assertEquals(expectedCount, distinctCount); + if (checkNoDuplication) { + assertEquals(distinctCount, totalCount); + } + } + + // Performs checks on the table's rows under different conditions. + // Note: these should only be performed when ignoreUnknownValues is set. + public void checkRowsWithUpdatedSchema( + String tableSpec, String extraField, boolean useAutoSchemaUpdate) + throws IOException, InterruptedException { + List actualRows = + BQ_CLIENT.queryUnflattened( + String.format("SELECT * FROM [%s]", tableSpec), PROJECT, true, false, bigQueryLocation); + + for (TableRow row : actualRows) { + // Rows written to the table should not have the extra field if + // 1. The row has original schema + // 2. We didn't set autoSchemaUpdate (the extra field would just be dropped) + // 3. We didn't change the table's schema (again, the extra field would be dropped) + if (Integer.parseInt((String) row.get("id")) < ORIGINAL_N + || !useAutoSchemaUpdate + || !changeTableSchema) { + assertTrue( + String.format("Expected row to NOT have field %s:\n%s", extraField, row), + row.get(extraField) == null); + } else { + assertTrue( + String.format("Expected row to have field %s:\n%s", extraField, row), + row.get(extraField) != null); + } + } + } + + @Test + public void testExactlyOnce() throws Exception { + runStreamingPipelineWithSchemaChange( + Write.Method.STORAGE_WRITE_API, + /** autoSchemaUpdate */ + false, + /** ignoreUnknownvalues */ + false); + } + + @Test + public void testExactlyOnceWithIgnoreUnknownValues() throws Exception { + runStreamingPipelineWithSchemaChange(Write.Method.STORAGE_WRITE_API, false, true); + } + + @Test + public void testExactlyOnceWithAutoSchemaUpdate() throws Exception { + runStreamingPipelineWithSchemaChange(Write.Method.STORAGE_WRITE_API, true, true); + } + + @Test + public void testAtLeastOnce() throws Exception { + runStreamingPipelineWithSchemaChange(Write.Method.STORAGE_API_AT_LEAST_ONCE, false, false); + } + + @Test + public void testAtLeastOnceWithIgnoreUnknownValues() throws Exception { + runStreamingPipelineWithSchemaChange(Write.Method.STORAGE_API_AT_LEAST_ONCE, false, true); + } + + @Test + public void testAtLeastOnceWithAutoSchemaUpdate() throws Exception { + runStreamingPipelineWithSchemaChange(Write.Method.STORAGE_API_AT_LEAST_ONCE, true, true); + } +} diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoIT.java index 5f488da0210b2..f28ae588a5ecb 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoIT.java @@ -41,8 +41,8 @@ import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.SerializableFunctions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -318,10 +318,15 @@ public class TableRowToStorageApiProtoIT { .setFields(BASE_TABLE_SCHEMA.getFields())) .build()); + // used when test suite specifies a particular GCP location for BigQuery operations + private static String bigQueryLocation; + @BeforeClass public static void setUpTestEnvironment() throws IOException, InterruptedException { // Create one BQ dataset for all test cases. - BQ_CLIENT.createNewDataset(PROJECT, BIG_QUERY_DATASET_ID); + bigQueryLocation = + TestPipeline.testingPipelineOptions().as(TestBigQueryOptions.class).getBigQueryLocation(); + BQ_CLIENT.createNewDataset(PROJECT, BIG_QUERY_DATASET_ID, null, bigQueryLocation); } @AfterClass @@ -338,7 +343,7 @@ public void testBaseTableRow() throws IOException, InterruptedException { List actualTableRows = BQ_CLIENT.queryUnflattened( - String.format("SELECT * FROM %s", tableSpec), PROJECT, true, true); + String.format("SELECT * FROM %s", tableSpec), PROJECT, true, true, bigQueryLocation); assertEquals(1, actualTableRows.size()); assertEquals(BASE_TABLE_ROW_EXPECTED, actualTableRows.get(0)); @@ -364,7 +369,7 @@ public void testNestedRichTypesAndNull() throws IOException, InterruptedExceptio List actualTableRows = BQ_CLIENT.queryUnflattened( - String.format("SELECT * FROM %s", tableSpec), PROJECT, true, true); + String.format("SELECT * FROM %s", tableSpec), PROJECT, true, true, bigQueryLocation); assertEquals(1, actualTableRows.size()); assertEquals(BASE_TABLE_ROW_EXPECTED, actualTableRows.get(0).get("nestedValue1")); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java index 58f181700d7ef..aae1c2096cc52 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java @@ -45,11 +45,11 @@ import java.util.stream.Collectors; import org.apache.beam.sdk.io.gcp.bigquery.TableRowToStorageApiProto.SchemaConversionException; import org.apache.beam.sdk.io.gcp.bigquery.TableRowToStorageApiProto.SchemaInformation; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Functions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Functions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -63,7 +63,7 @@ /** Unit tests for {@link org.apache.beam.sdk.io.gcp.bigquery.TableRowToStorageApiProto}. */ public class TableRowToStorageApiProtoTest { // Schemas we test. - // The TableRow class has special semantics for fields named "f". To ensure we handel them + // The TableRow class has special semantics for fields named "f". To ensure we handle them // properly, we test schemas // both with and without a field named "f". private static final TableSchema BASE_TABLE_SCHEMA = @@ -114,6 +114,7 @@ public class TableRowToStorageApiProtoTest { .setType("TIMESTAMP") .setName("timestampValueSpaceTrailingZero")) .add(new TableFieldSchema().setType("DATETIME").setName("datetimeValueSpace")) + .add(new TableFieldSchema().setType("TIMESTAMP").setName("timestampValueMaximum")) .build()); private static final TableSchema BASE_TABLE_SCHEMA_NO_F = @@ -163,9 +164,10 @@ public class TableRowToStorageApiProtoTest { .setType("TIMESTAMP") .setName("timestampValueSpaceTrailingZero")) .add(new TableFieldSchema().setType("DATETIME").setName("datetimeValueSpace")) + .add(new TableFieldSchema().setType("TIMESTAMP").setName("timestampValueMaximum")) .build()); - private static final DescriptorProto BASE_TABLE_SCHEMA_PROTO = + private static final DescriptorProto BASE_TABLE_SCHEMA_PROTO_DESCRIPTOR = DescriptorProto.newBuilder() .addField( FieldDescriptorProto.newBuilder() @@ -356,6 +358,157 @@ public class TableRowToStorageApiProtoTest { .setType(Type.TYPE_INT64) .setLabel(Label.LABEL_OPTIONAL) .build()) + .addField( + FieldDescriptorProto.newBuilder() + .setName("timestampvaluemaximum") + .setNumber(28) + .setType(Type.TYPE_INT64) + .setLabel(Label.LABEL_OPTIONAL) + .build()) + .build(); + + private static final com.google.cloud.bigquery.storage.v1.TableSchema BASE_TABLE_PROTO_SCHEMA = + com.google.cloud.bigquery.storage.v1.TableSchema.newBuilder() + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("stringvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.STRING) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("f") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.STRING) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("bytesvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.BYTES) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("int64value") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("intvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("float64value") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.DOUBLE) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("floatvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.DOUBLE) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("boolvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.BOOL) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("booleanvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.BOOL) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timevalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("datetimevalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("datevalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("numericvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.BYTES) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("bignumericvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.BYTES) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("numericvalue2") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.BYTES) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("bignumericvalue2") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.BYTES) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("arrayvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.BYTES) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampisovalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampisovalueoffsethh") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampvaluelong") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampvaluespace") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampvaluespaceutc") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampvaluezoneregion") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampvaluespacemilli") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampvaluespacetrailingzero") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("datetimevaluespace") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampvaluemaximum") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) .build(); private static final DescriptorProto BASE_TABLE_SCHEMA_NO_F_PROTO = @@ -542,7 +695,154 @@ public class TableRowToStorageApiProtoTest { .setType(Type.TYPE_INT64) .setLabel(Label.LABEL_OPTIONAL) .build()) + .addField( + FieldDescriptorProto.newBuilder() + .setName("timestampvaluemaximum") + .setNumber(27) + .setType(Type.TYPE_INT64) + .setLabel(Label.LABEL_OPTIONAL) + .build()) .build(); + + private static final com.google.cloud.bigquery.storage.v1.TableSchema + BASE_TABLE_NO_F_PROTO_SCHEMA = + com.google.cloud.bigquery.storage.v1.TableSchema.newBuilder() + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("stringvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.STRING) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("bytesvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.BYTES) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("int64value") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("intvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("float64value") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.DOUBLE) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("floatvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.DOUBLE) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("boolvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.BOOL) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("booleanvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.BOOL) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timevalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("datetimevalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("datevalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("numericvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.BYTES) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("bignumericvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.BYTES) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("numericvalue2") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.BYTES) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("bignumericvalue2") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.BYTES) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("arrayvalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.BYTES) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampisovalue") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampisovalueoffsethh") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampvaluelong") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampvaluespace") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampvaluespaceutc") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampvaluezoneregion") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampvaluespacemilli") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampvaluespacetrailingzero") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("datetimevaluespace") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .addFields( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.newBuilder() + .setName("timestampvaluemaximum") + .setType(com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.INT64) + .build()) + .build(); private static final TableSchema NESTED_TABLE_SCHEMA = new TableSchema() .setFields( @@ -550,29 +850,33 @@ public class TableRowToStorageApiProtoTest { .add( new TableFieldSchema() .setType("STRUCT") - .setName("nestedValue1") + .setName("nestedvalue1") + .setMode("NULLABLE") .setFields(BASE_TABLE_SCHEMA.getFields())) .add( new TableFieldSchema() .setType("RECORD") - .setName("nestedValue2") + .setName("nestedvalue2") + .setMode("NULLABLE") .setFields(BASE_TABLE_SCHEMA.getFields())) .add( new TableFieldSchema() .setType("STRUCT") - .setName("nestedValueNoF1") + .setName("nestedvaluenof1") + .setMode("NULLABLE") .setFields(BASE_TABLE_SCHEMA_NO_F.getFields())) .add( new TableFieldSchema() .setType("RECORD") - .setName("nestedValueNoF2") + .setName("nestedvaluenof2") + .setMode("NULLABLE") .setFields(BASE_TABLE_SCHEMA_NO_F.getFields())) .build()); @Rule public transient ExpectedException thrown = ExpectedException.none(); @Test - public void testDescriptorFromTableSchema() { + public void testDescriptorFromTableSchema() throws Exception { DescriptorProto descriptor = TableRowToStorageApiProto.descriptorSchemaFromTableSchema(BASE_TABLE_SCHEMA, true, false); Map types = @@ -580,18 +884,37 @@ public void testDescriptorFromTableSchema() { .collect( Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType)); Map expectedTypes = - BASE_TABLE_SCHEMA_PROTO.getFieldList().stream() + BASE_TABLE_SCHEMA_PROTO_DESCRIPTOR.getFieldList().stream() .collect( Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType)); assertEquals(expectedTypes, types); + + com.google.cloud.bigquery.storage.v1.TableSchema roundtripSchema = + TableRowToStorageApiProto.tableSchemaFromDescriptor( + TableRowToStorageApiProto.wrapDescriptorProto(descriptor)); + Map roundTripTypes = + roundtripSchema.getFieldsList().stream() + .collect( + Collectors.toMap( + com.google.cloud.bigquery.storage.v1.TableFieldSchema::getName, + com.google.cloud.bigquery.storage.v1.TableFieldSchema::getType)); + + Map roundTripExpectedTypes = + BASE_TABLE_PROTO_SCHEMA.getFieldsList().stream() + .collect( + Collectors.toMap( + com.google.cloud.bigquery.storage.v1.TableFieldSchema::getName, + com.google.cloud.bigquery.storage.v1.TableFieldSchema::getType)); + + assertEquals(roundTripExpectedTypes, roundTripTypes); } @Test - public void testNestedFromTableSchema() { + public void testNestedFromTableSchema() throws Exception { DescriptorProto descriptor = TableRowToStorageApiProto.descriptorSchemaFromTableSchema(NESTED_TABLE_SCHEMA, true, false); Map expectedBaseTypes = - BASE_TABLE_SCHEMA_PROTO.getFieldList().stream() + BASE_TABLE_SCHEMA_PROTO_DESCRIPTOR.getFieldList().stream() .collect( Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType)); Map expectedBaseTypesNoF = @@ -643,6 +966,97 @@ public void testNestedFromTableSchema() { .collect( Collectors.toMap(FieldDescriptorProto::getName, FieldDescriptorProto::getType)); assertEquals(expectedBaseTypesNoF, nestedTypesNoF2); + + Map + roundTripExpectedBaseTypes = + BASE_TABLE_PROTO_SCHEMA.getFieldsList().stream() + .collect( + Collectors.toMap( + com.google.cloud.bigquery.storage.v1.TableFieldSchema::getName, + com.google.cloud.bigquery.storage.v1.TableFieldSchema::getType)); + Map + roundTripExpectedBaseTypesNoF = + BASE_TABLE_NO_F_PROTO_SCHEMA.getFieldsList().stream() + .collect( + Collectors.toMap( + com.google.cloud.bigquery.storage.v1.TableFieldSchema::getName, + com.google.cloud.bigquery.storage.v1.TableFieldSchema::getType)); + + com.google.cloud.bigquery.storage.v1.TableSchema roundtripSchema = + TableRowToStorageApiProto.tableSchemaFromDescriptor( + TableRowToStorageApiProto.wrapDescriptorProto(descriptor)); + + Map roundTripTypes = + roundtripSchema.getFieldsList().stream() + .collect( + Collectors.toMap( + com.google.cloud.bigquery.storage.v1.TableFieldSchema::getName, + com.google.cloud.bigquery.storage.v1.TableFieldSchema::getType)); + assertEquals(4, roundTripTypes.size()); + + assertEquals( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.STRUCT, + roundTripTypes.get("nestedvalue1")); + com.google.cloud.bigquery.storage.v1.TableFieldSchema nestedType = + roundtripSchema.getFieldsList().stream() + .filter(f -> f.getName().equals("nestedvalue1")) + .findFirst() + .get(); + Map nestedRoundTripTypes = + nestedType.getFieldsList().stream() + .collect( + Collectors.toMap( + com.google.cloud.bigquery.storage.v1.TableFieldSchema::getName, + com.google.cloud.bigquery.storage.v1.TableFieldSchema::getType)); + assertEquals(roundTripExpectedBaseTypes, nestedRoundTripTypes); + + assertEquals( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.STRUCT, + roundTripTypes.get("nestedvalue2")); + nestedType = + roundtripSchema.getFieldsList().stream() + .filter(f -> f.getName().equals("nestedvalue2")) + .findFirst() + .get(); + nestedRoundTripTypes = + nestedType.getFieldsList().stream() + .collect( + Collectors.toMap( + com.google.cloud.bigquery.storage.v1.TableFieldSchema::getName, + com.google.cloud.bigquery.storage.v1.TableFieldSchema::getType)); + assertEquals(roundTripExpectedBaseTypes, nestedRoundTripTypes); + + assertEquals( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.STRUCT, + roundTripTypes.get("nestedvaluenof1")); + nestedType = + roundtripSchema.getFieldsList().stream() + .filter(f -> f.getName().equals("nestedvaluenof1")) + .findFirst() + .get(); + nestedRoundTripTypes = + nestedType.getFieldsList().stream() + .collect( + Collectors.toMap( + com.google.cloud.bigquery.storage.v1.TableFieldSchema::getName, + com.google.cloud.bigquery.storage.v1.TableFieldSchema::getType)); + assertEquals(roundTripExpectedBaseTypesNoF, nestedRoundTripTypes); + + assertEquals( + com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.STRUCT, + roundTripTypes.get("nestedvaluenof2")); + nestedType = + roundtripSchema.getFieldsList().stream() + .filter(f -> f.getName().equals("nestedvaluenof2")) + .findFirst() + .get(); + nestedRoundTripTypes = + nestedType.getFieldsList().stream() + .collect( + Collectors.toMap( + com.google.cloud.bigquery.storage.v1.TableFieldSchema::getName, + com.google.cloud.bigquery.storage.v1.TableFieldSchema::getType)); + assertEquals(roundTripExpectedBaseTypesNoF, nestedRoundTripTypes); } private static final List REPEATED_BYTES = @@ -689,7 +1103,8 @@ public void testNestedFromTableSchema() { new TableCell().setV("1970-01-01 00:00:00.123456 America/New_York"), new TableCell().setV("1970-01-01 00:00:00.123"), new TableCell().setV("1970-01-01 00:00:00.1230"), - new TableCell().setV("2019-08-16 00:52:07.123456"))); + new TableCell().setV("2019-08-16 00:52:07.123456"), + new TableCell().setV("9999-12-31 23:59:59.999999Z"))); private static final TableRow BASE_TABLE_ROW_NO_F = new TableRow() @@ -721,7 +1136,8 @@ public void testNestedFromTableSchema() { .set("timestampValueZoneRegion", "1970-01-01 00:00:00.123456 America/New_York") .set("timestampValueSpaceMilli", "1970-01-01 00:00:00.123") .set("timestampValueSpaceTrailingZero", "1970-01-01 00:00:00.1230") - .set("datetimeValueSpace", "2019-08-16 00:52:07.123456"); + .set("datetimeValueSpace", "2019-08-16 00:52:07.123456") + .set("timestampValueMaximum", "9999-12-31 23:59:59.999999Z"); private static final Map BASE_ROW_EXPECTED_PROTO_VALUES = ImmutableMap.builder() @@ -761,6 +1177,7 @@ public void testNestedFromTableSchema() { .put("timestampvaluespacemilli", 123000L) .put("timestampvaluespacetrailingzero", 123000L) .put("datetimevaluespace", 142111881387172416L) + .put("timestampvaluemaximum", 253402300799999999L) .build(); private static final Map BASE_ROW_NO_F_EXPECTED_PROTO_VALUES = @@ -800,6 +1217,7 @@ public void testNestedFromTableSchema() { .put("timestampvaluespacemilli", 123000L) .put("timestampvaluespacetrailingzero", 123000L) .put("datetimevaluespace", 142111881387172416L) + .put("timestampvaluemaximum", 253402300799999999L) .build(); private void assertBaseRecord(DynamicMessage msg, boolean withF) { diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryDirectReadSchemaTransformProviderTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryDirectReadSchemaTransformProviderTest.java index a1105882173a0..2363a870bbd7e 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryDirectReadSchemaTransformProviderTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryDirectReadSchemaTransformProviderTest.java @@ -52,7 +52,7 @@ import org.apache.beam.sdk.io.gcp.bigquery.BigQueryOptions; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.StorageClient; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils; -import org.apache.beam.sdk.io.gcp.bigquery.providers.BigQueryDirectReadSchemaTransformProvider.BigQueryDirectReadPCollectionRowTupleTransform; +import org.apache.beam.sdk.io.gcp.bigquery.providers.BigQueryDirectReadSchemaTransformProvider.BigQueryDirectReadSchemaTransform; import org.apache.beam.sdk.io.gcp.bigquery.providers.BigQueryDirectReadSchemaTransformProvider.BigQueryDirectReadSchemaTransformConfiguration; import org.apache.beam.sdk.io.gcp.testing.FakeBigQueryServices; import org.apache.beam.sdk.io.gcp.testing.FakeBigQueryServices.FakeBigQueryServerStream; @@ -270,8 +270,8 @@ public void testDirectRead() throws Exception { BigQueryDirectReadSchemaTransformConfiguration.builder().setTableSpec(TABLE_SPEC).build(); BigQueryDirectReadSchemaTransformProvider provider = new BigQueryDirectReadSchemaTransformProvider(); - BigQueryDirectReadPCollectionRowTupleTransform readTransform = - (BigQueryDirectReadPCollectionRowTupleTransform) provider.from(config).buildTransform(); + BigQueryDirectReadSchemaTransform readTransform = + (BigQueryDirectReadSchemaTransform) provider.from(config); PCollectionRowTuple input = PCollectionRowTuple.empty(p); String tag = provider.outputCollectionNames().get(0); @@ -334,8 +334,8 @@ public void testDirectReadWithSelectedFieldsAndRowRestriction() throws Exception .build(); BigQueryDirectReadSchemaTransformProvider provider = new BigQueryDirectReadSchemaTransformProvider(); - BigQueryDirectReadPCollectionRowTupleTransform readTransform = - (BigQueryDirectReadPCollectionRowTupleTransform) provider.from(config).buildTransform(); + BigQueryDirectReadSchemaTransform readTransform = + (BigQueryDirectReadSchemaTransform) provider.from(config); PCollectionRowTuple input = PCollectionRowTuple.empty(p); String tag = provider.outputCollectionNames().get(0); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProviderTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProviderTest.java index fef2bb168c8f5..54c636bde5fe1 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProviderTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProviderTest.java @@ -22,8 +22,6 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; -import com.google.api.services.bigquery.model.Table; -import com.google.api.services.bigquery.model.TableReference; import com.google.api.services.bigquery.model.TableRow; import java.io.Serializable; import java.time.LocalDateTime; @@ -31,11 +29,9 @@ import java.util.Arrays; import java.util.List; import java.util.function.Function; -import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils; -import org.apache.beam.sdk.io.gcp.bigquery.providers.BigQueryStorageWriteApiSchemaTransformProvider.BigQueryStorageWriteApiPCollectionRowTupleTransform; +import org.apache.beam.sdk.io.gcp.bigquery.providers.BigQueryStorageWriteApiSchemaTransformProvider.BigQueryStorageWriteApiSchemaTransform; import org.apache.beam.sdk.io.gcp.bigquery.providers.BigQueryStorageWriteApiSchemaTransformProvider.BigQueryStorageWriteApiSchemaTransformConfiguration; import org.apache.beam.sdk.io.gcp.testing.FakeBigQueryServices; import org.apache.beam.sdk.io.gcp.testing.FakeDatasetService; @@ -45,8 +41,6 @@ import org.apache.beam.sdk.metrics.MetricResult; import org.apache.beam.sdk.metrics.MetricResults; import org.apache.beam.sdk.metrics.MetricsFilter; -import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.Schema.Field; import org.apache.beam.sdk.schemas.Schema.FieldType; @@ -54,9 +48,11 @@ import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TypeDescriptors; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -97,11 +93,6 @@ public class BigQueryStorageWriteApiSchemaTransformProviderTest { .withFieldValue("dt", LocalDateTime.parse("2000-01-03T00:00:00.123456")) .build()); - private static final Schema SCHEMA_WRONG = - Schema.of( - Field.of("name_wrong", FieldType.STRING), - Field.of("number", FieldType.INT64), - Field.of("dt", FieldType.logicalType(SqlTypes.DATETIME))); @Rule public final transient TestPipeline p = TestPipeline.create(); @Before @@ -140,17 +131,16 @@ public PCollectionRowTuple runWithConfig( BigQueryStorageWriteApiSchemaTransformProvider provider = new BigQueryStorageWriteApiSchemaTransformProvider(); - BigQueryStorageWriteApiPCollectionRowTupleTransform writeRowTupleTransform = - (BigQueryStorageWriteApiPCollectionRowTupleTransform) - provider.from(config).buildTransform(); + BigQueryStorageWriteApiSchemaTransform writeTransform = + (BigQueryStorageWriteApiSchemaTransform) provider.from(config); - writeRowTupleTransform.setBigQueryServices(fakeBigQueryServices); + writeTransform.setBigQueryServices(fakeBigQueryServices); String tag = provider.inputCollectionNames().get(0); PCollection rows = p.apply(Create.of(inputRows).withRowSchema(SCHEMA)); PCollectionRowTuple input = PCollectionRowTuple.of(tag, rows); - PCollectionRowTuple result = input.apply(writeRowTupleTransform); + PCollectionRowTuple result = input.apply(writeTransform); return result; } @@ -189,63 +179,6 @@ public void testSimpleWrite() throws Exception { rowsEquals(ROWS, fakeDatasetService.getAllRows("project", "dataset", "simple_write"))); } - @Test - public void testSchemaValidationSuccess() throws Exception { - String tableSpec = "project:dataset.schema_validation_success"; - Table table = new Table(); - TableReference tableReference = BigQueryHelpers.parseTableSpec(tableSpec); - table.setTableReference(tableReference); - table.setSchema(BigQueryUtils.toTableSchema(SCHEMA)); - fakeDatasetService.createTable(table); - BigQueryStorageWriteApiSchemaTransformConfiguration config = - BigQueryStorageWriteApiSchemaTransformConfiguration.builder() - .setTable(tableSpec) - .setCreateDisposition("CREATE_IF_NEEDED") - .build(); - - runWithConfig(config); - p.run().waitUntilFinish(); - - assertNotNull(fakeDatasetService.getTable(BigQueryHelpers.parseTableSpec(tableSpec))); - assertEquals( - 3, fakeDatasetService.getAllRows("project", "dataset", "schema_validation_success").size()); - } - - @Test(expected = RuntimeException.class) - public void testSchemaValidationFail() throws Exception { - String tableSpec = "project:dataset.schema_validation_fail"; - Table table = new Table(); - TableReference tableReference = BigQueryHelpers.parseTableSpec(tableSpec); - table.setTableReference(tableReference); - table.setSchema(BigQueryUtils.toTableSchema(SCHEMA_WRONG)); - fakeDatasetService.createTable(table); - BigQueryStorageWriteApiSchemaTransformConfiguration config = - BigQueryStorageWriteApiSchemaTransformConfiguration.builder() - .setTable(tableSpec) - .setCreateDisposition("CREATE_IF_NEEDED") - .build(); - BigQueryStorageWriteApiSchemaTransformProvider provider = - new BigQueryStorageWriteApiSchemaTransformProvider(); - - BigQueryStorageWriteApiPCollectionRowTupleTransform writeRowTupleTransform = - (BigQueryStorageWriteApiPCollectionRowTupleTransform) - provider.from(config).buildTransform(); - writeRowTupleTransform.setBigQueryServices(fakeBigQueryServices); - List testRows = - Arrays.asList( - Row.withSchema(SCHEMA) - .withFieldValue("name", "a") - .withFieldValue("number", 1L) - .withFieldValue("dt", LocalDateTime.parse("2000-01-01T00:00:00")) - .build()); - String tag = provider.inputCollectionNames().get(0); - PipelineOptions options = PipelineOptionsFactory.create(); - Pipeline pipeline = Pipeline.create(options); - PCollection rows = pipeline.apply(Create.of(testRows).withRowSchema(SCHEMA)); - PCollectionRowTuple input = PCollectionRowTuple.of(tag, rows); - writeRowTupleTransform.expand(input); - } - @Test public void testInputElementCount() throws Exception { String tableSpec = "project:dataset.input_count"; @@ -261,7 +194,7 @@ public void testInputElementCount() throws Exception { MetricsFilter.builder() .addNameFilter( MetricNameFilter.named( - BigQueryStorageWriteApiPCollectionRowTupleTransform.class, + BigQueryStorageWriteApiSchemaTransform.class, "BigQuery-write-element-counter")) .build()); @@ -280,7 +213,13 @@ public void testInputElementCount() throws Exception { public void testFailedRows() throws Exception { String tableSpec = "project:dataset.write_with_fail"; BigQueryStorageWriteApiSchemaTransformConfiguration config = - BigQueryStorageWriteApiSchemaTransformConfiguration.builder().setTable(tableSpec).build(); + BigQueryStorageWriteApiSchemaTransformConfiguration.builder() + .setTable(tableSpec) + .setErrorHandling( + BigQueryStorageWriteApiSchemaTransformConfiguration.ErrorHandling.builder() + .setOutput("FailedRows") + .build()) + .build(); String failValue = "fail_me"; @@ -303,7 +242,15 @@ public void testFailedRows() throws Exception { fakeDatasetService.setShouldFailRow(shouldFailRow); PCollectionRowTuple result = runWithConfig(config, totalRows); - PCollection failedRows = result.get("FailedRows"); + PCollection failedRows = + result + .get("FailedRows") + .apply( + "ExtractFailedRows", + MapElements.into(TypeDescriptors.rows()) + .via((rowAndError) -> rowAndError.getValue("failed_row"))) + .setRowSchema(SCHEMA); + ; PAssert.that(failedRows).containsInAnyOrder(expectedFailedRows); p.run().waitUntilFinish(); @@ -319,7 +266,13 @@ public void testFailedRows() throws Exception { public void testErrorCount() throws Exception { String tableSpec = "project:dataset.error_count"; BigQueryStorageWriteApiSchemaTransformConfiguration config = - BigQueryStorageWriteApiSchemaTransformConfiguration.builder().setTable(tableSpec).build(); + BigQueryStorageWriteApiSchemaTransformConfiguration.builder() + .setTable(tableSpec) + .setErrorHandling( + BigQueryStorageWriteApiSchemaTransformConfiguration.ErrorHandling.builder() + .setOutput("FailedRows") + .build()) + .build(); Function shouldFailRow = (Function & Serializable) tr -> tr.get("name").equals("a"); @@ -334,7 +287,7 @@ public void testErrorCount() throws Exception { MetricsFilter.builder() .addNameFilter( MetricNameFilter.named( - BigQueryStorageWriteApiPCollectionRowTupleTransform.class, + BigQueryStorageWriteApiSchemaTransform.class, "BigQuery-write-error-counter")) .build()); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BeamRowToBigtableMutationTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BeamRowToBigtableMutationTest.java index 5fa16dbffafa6..a2e0a76d3ed9f 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BeamRowToBigtableMutationTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BeamRowToBigtableMutationTest.java @@ -38,9 +38,9 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java index 1fffc0571628f..714dc9f8619d8 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java @@ -26,9 +26,9 @@ import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasKey; import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasLabel; import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasValue; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Verify.verifyNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Verify.verifyNotNull; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -107,10 +107,10 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicate; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicate; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.Matchers; import org.hamcrest.collection.IsIterableContainingInAnyOrder; @@ -219,11 +219,13 @@ public void testReadBuildsCorrectly() { .withTableId("table") .withInstanceId("instance") .withProjectId("project") + .withAppProfileId("app-profile") .withBigtableOptionsConfigurator(PORT_CONFIGURATOR); assertEquals("options_project", read.getBigtableOptions().getProjectId()); assertEquals("options_instance", read.getBigtableOptions().getInstanceId()); assertEquals("instance", read.getBigtableConfig().getInstanceId().get()); assertEquals("project", read.getBigtableConfig().getProjectId().get()); + assertEquals("app-profile", read.getBigtableConfig().getAppProfileId().get()); assertEquals("table", read.getTableId()); assertEquals(PORT_CONFIGURATOR, read.getBigtableConfig().getBigtableOptionsConfigurator()); } @@ -373,12 +375,14 @@ public void testWriteBuildsCorrectly() { .withBigtableOptions(BIGTABLE_OPTIONS) .withTableId("table") .withInstanceId("instance") - .withProjectId("project"); + .withProjectId("project") + .withAppProfileId("app-profile"); assertEquals("table", write.getBigtableWriteOptions().getTableId().get()); assertEquals("options_project", write.getBigtableOptions().getProjectId()); assertEquals("options_instance", write.getBigtableOptions().getInstanceId()); assertEquals("instance", write.getBigtableConfig().getInstanceId().get()); assertEquals("project", write.getBigtableConfig().getProjectId().get()); + assertEquals("app-profile", write.getBigtableConfig().getAppProfileId().get()); } @Test @@ -766,6 +770,39 @@ public void testReadingWithSplits() throws Exception { assertSourcesEqualReferenceSource(source, splits, null /* options */); } + /** + * Regression test for [Bug]: BigtableSource + * "Desired bundle size 0 bytes must be greater than 0" #28793. + */ + @Test + public void testSplittingWithDesiredBundleSizeZero() throws Exception { + final String table = "TEST-SPLIT-DESIRED-BUNDLE-SIZE-ZERO-TABLE"; + final int numRows = 10; + final int numSamples = 10; + final long bytesPerRow = 1L; + + // Set up test table data and sample row keys for size estimation and splitting. + makeTableData(table, numRows); + service.setupSampleRowKeys(table, numSamples, bytesPerRow); + + // Generate source and split it. + BigtableSource source = + new BigtableSource( + factory, + configId, + config, + BigtableReadOptions.builder() + .setTableId(StaticValueProvider.of(table)) + .setKeyRanges(ALL_KEY_RANGE) + .build(), + null /*size*/); + List splits = source.split(0, null /* options */); + + // Test num splits and split equality. + assertThat(splits, hasSize(numSamples)); + assertSourcesEqualReferenceSource(source, splits, null /* options */); + } + @Test public void testReadingWithSplitFailed() throws Exception { FailureBigtableService failureService = diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadSchemaTransformProviderIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadSchemaTransformProviderIT.java index c793af549a5ea..81d3103f38bf5 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadSchemaTransformProviderIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadSchemaTransformProviderIT.java @@ -32,6 +32,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -49,14 +50,19 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @RunWith(JUnit4.class) public class BigtableReadSchemaTransformProviderIT { + private static final Logger LOG = + LoggerFactory.getLogger(BigtableReadSchemaTransformProviderIT.class); + @Rule public final transient TestPipeline p = TestPipeline.create(); private static final String COLUMN_FAMILY_NAME_1 = "test_cf_1"; private static final String COLUMN_FAMILY_NAME_2 = "test_cf_2"; - private BigtableTableAdminClient adminClient; + private BigtableTableAdminClient tableAdminClient; private BigtableDataClient dataClient; private String tableId; private String projectId; @@ -106,31 +112,31 @@ public void setup() throws Exception { .setProjectId(projectId) .setInstanceId(instanceId) .build(); - adminClient = BigtableTableAdminClient.create(adminSettings); + tableAdminClient = BigtableTableAdminClient.create(adminSettings); + + tableId = String.format("BTReadSchemaTransformIT-%tF-% writeToTable(int numRows) throws Exception { - // Checks if table exists, creates table if does not exist. - if (!adminClient.exists(tableId)) { - CreateTableRequest createTableRequest = - CreateTableRequest.of(tableId) - .addFamily(COLUMN_FAMILY_NAME_1) - .addFamily(COLUMN_FAMILY_NAME_2); - adminClient.createTable(createTableRequest); - } - + public List writeToTable(int numRows) { List expectedRows = new ArrayList<>(); try { @@ -199,6 +205,7 @@ public List writeToTable(int numRows) throws Exception { expectedRows.add(expectedRow); } + LOG.info("Finished writing {} rows to table {}", numRows, tableId); } catch (NotFoundException e) { throw new RuntimeException("Failed to write to table", e); } @@ -206,8 +213,7 @@ public List writeToTable(int numRows) throws Exception { } @Test - public void testRead() throws Exception { - tableId = "BigtableReadSchemaTransformIT"; + public void testRead() { List expectedRows = writeToTable(20); BigtableReadSchemaTransformConfiguration config = @@ -218,9 +224,7 @@ public void testRead() throws Exception { .build(); SchemaTransform transform = new BigtableReadSchemaTransformProvider().from(config); - PCollection rows = - PCollectionRowTuple.empty(p).apply(transform.buildTransform()).get("output"); - + PCollection rows = PCollectionRowTuple.empty(p).apply(transform).get("output"); PAssert.that(rows).containsInAnyOrder(expectedRows); p.run().waitUntilFinish(); } diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableRowToBeamRowFlatTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableRowToBeamRowFlatTest.java index ef59c6ab7466c..a9ad5c73ecb04 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableRowToBeamRowFlatTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableRowToBeamRowFlatTest.java @@ -34,8 +34,8 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableRowToBeamRowTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableRowToBeamRowTest.java index dfcefab5728ce..bbb6d70804f70 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableRowToBeamRowTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableRowToBeamRowTest.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImplTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImplTest.java index 856d0cd22b822..f239a68462d89 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImplTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImplTest.java @@ -75,7 +75,7 @@ import org.apache.beam.sdk.metrics.MetricsEnvironment; import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.junit.Assert; import org.junit.Before; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableTestUtils.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableTestUtils.java index 5c5af10b37b76..c35b7c54c4d9e 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableTestUtils.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableTestUtils.java @@ -33,8 +33,8 @@ import java.util.List; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Longs; class BigtableTestUtils { diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteIT.java index 70df5b6c39efc..3e3f24fdd54d5 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteIT.java @@ -45,7 +45,7 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteSchemaTransformProviderIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteSchemaTransformProviderIT.java new file mode 100644 index 0000000000000..1a60fe661b52c --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteSchemaTransformProviderIT.java @@ -0,0 +1,416 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.gcp.bigtable; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; +import org.apache.beam.sdk.io.gcp.bigtable.BigtableWriteSchemaTransformProvider.BigtableWriteSchemaTransformConfiguration; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.PCollectionRowTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Longs; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class BigtableWriteSchemaTransformProviderIT { + @Rule public final transient TestPipeline p = TestPipeline.create(); + + private static final String COLUMN_FAMILY_NAME_1 = "test_cf_1"; + private static final String COLUMN_FAMILY_NAME_2 = "test_cf_2"; + private BigtableTableAdminClient tableAdminClient; + private BigtableDataClient dataClient; + private String tableId = String.format("BigtableWriteIT-%tF-% writeTransform; + private static final Schema SCHEMA = + Schema.builder() + .addByteArrayField("key") + .addArrayField( + "mutations", Schema.FieldType.map(Schema.FieldType.STRING, Schema.FieldType.BYTES)) + .build(); + + @Test + public void testInvalidConfigs() { + System.out.println(writeTransform.getName()); + // Properties cannot be empty (project, instance, and table) + List invalidConfigs = + Arrays.asList( + BigtableWriteSchemaTransformConfiguration.builder() + .setProjectId("project") + .setInstanceId("instance") + .setTableId(""), + BigtableWriteSchemaTransformConfiguration.builder() + .setProjectId("") + .setInstanceId("instance") + .setTableId("table"), + BigtableWriteSchemaTransformConfiguration.builder() + .setProjectId("project") + .setInstanceId("") + .setTableId("table")); + + for (BigtableWriteSchemaTransformConfiguration.Builder config : invalidConfigs) { + assertThrows( + IllegalArgumentException.class, + () -> { + config.build().validate(); + }); + } + } + + @Before + public void setup() throws Exception { + BigtableTestOptions options = + TestPipeline.testingPipelineOptions().as(BigtableTestOptions.class); + projectId = options.as(GcpOptions.class).getProject(); + instanceId = options.getInstanceId(); + + BigtableDataSettings settings = + BigtableDataSettings.newBuilder().setProjectId(projectId).setInstanceId(instanceId).build(); + // Creates a bigtable data client. + dataClient = BigtableDataClient.create(settings); + + BigtableTableAdminSettings adminSettings = + BigtableTableAdminSettings.newBuilder() + .setProjectId(projectId) + .setInstanceId(instanceId) + .build(); + tableAdminClient = BigtableTableAdminClient.create(adminSettings); + + // set up the table with some pre-written rows to test our mutations on. + // each test is independent of the others + if (!tableAdminClient.exists(tableId)) { + CreateTableRequest createTableRequest = + CreateTableRequest.of(tableId) + .addFamily(COLUMN_FAMILY_NAME_1) + .addFamily(COLUMN_FAMILY_NAME_2); + tableAdminClient.createTable(createTableRequest); + } + + BigtableWriteSchemaTransformConfiguration config = + BigtableWriteSchemaTransformConfiguration.builder() + .setProjectId(projectId) + .setInstanceId(instanceId) + .setTableId(tableId) + .build(); + writeTransform = new BigtableWriteSchemaTransformProvider().from(config); + } + + @After + public void tearDown() { + try { + tableAdminClient.deleteTable(tableId); + System.out.printf("Table %s deleted successfully%n", tableId); + } catch (NotFoundException e) { + System.err.println("Failed to delete a non-existent table: " + e.getMessage()); + } + dataClient.close(); + tableAdminClient.close(); + } + + @Test + public void testSetMutationsExistingColumn() { + RowMutation rowMutation = + RowMutation.create(tableId, "key-1") + .setCell(COLUMN_FAMILY_NAME_1, "col_a", 1000, "val-1-a") + .setCell(COLUMN_FAMILY_NAME_2, "col_c", 1000, "val-1-c"); + dataClient.mutateRow(rowMutation); + + List> mutations = new ArrayList<>(); + // mutation to set cell in an existing column + mutations.add( + ImmutableMap.of( + "type", "SetCell".getBytes(StandardCharsets.UTF_8), + "value", "new-val-1-a".getBytes(StandardCharsets.UTF_8), + "column_qualifier", "col_a".getBytes(StandardCharsets.UTF_8), + "family_name", COLUMN_FAMILY_NAME_1.getBytes(StandardCharsets.UTF_8), + "timestamp_micros", Longs.toByteArray(2000))); + mutations.add( + ImmutableMap.of( + "type", "SetCell".getBytes(StandardCharsets.UTF_8), + "value", "new-val-1-c".getBytes(StandardCharsets.UTF_8), + "column_qualifier", "col_c".getBytes(StandardCharsets.UTF_8), + "family_name", COLUMN_FAMILY_NAME_2.getBytes(StandardCharsets.UTF_8), + "timestamp_micros", Longs.toByteArray(2000))); + Row mutationRow = + Row.withSchema(SCHEMA) + .withFieldValue("key", "key-1".getBytes(StandardCharsets.UTF_8)) + .withFieldValue("mutations", mutations) + .build(); + + PCollectionRowTuple.of("input", p.apply(Create.of(Arrays.asList(mutationRow)))) + .apply(writeTransform); + p.run().waitUntilFinish(); + + // get rows from table + List rows = + dataClient.readRows(Query.create(tableId)).stream().collect(Collectors.toList()); + // we should still have only one row with the same key + assertEquals(1, rows.size()); + assertEquals("key-1", rows.get(0).getKey().toStringUtf8()); + + // check that we now have two cells in each column we added to and that + // the last cell in each column has the updated value + com.google.cloud.bigtable.data.v2.models.Row row = rows.get(0); + List cellsColA = + row.getCells(COLUMN_FAMILY_NAME_1, "col_a").stream() + .sorted(RowCell.compareByNative()) + .collect(Collectors.toList()); + List cellsColC = + row.getCells(COLUMN_FAMILY_NAME_2, "col_c").stream() + .sorted(RowCell.compareByNative()) + .collect(Collectors.toList()); + assertEquals(2, cellsColA.size()); + assertEquals(2, cellsColC.size()); + // Bigtable keeps cell history ordered by descending timestamp + assertEquals("new-val-1-a", cellsColA.get(0).getValue().toStringUtf8()); + assertEquals("new-val-1-c", cellsColC.get(0).getValue().toStringUtf8()); + assertEquals("val-1-a", cellsColA.get(1).getValue().toStringUtf8()); + assertEquals("val-1-c", cellsColC.get(1).getValue().toStringUtf8()); + } + + @Test + public void testSetMutationNewColumn() { + RowMutation rowMutation = + RowMutation.create(tableId, "key-1").setCell(COLUMN_FAMILY_NAME_1, "col_a", "val-1-a"); + dataClient.mutateRow(rowMutation); + + List> mutations = new ArrayList<>(); + // mutation to set cell in a new column + mutations.add( + ImmutableMap.of( + "type", "SetCell".getBytes(StandardCharsets.UTF_8), + "value", "new-val-1".getBytes(StandardCharsets.UTF_8), + "column_qualifier", "new_col".getBytes(StandardCharsets.UTF_8), + "family_name", COLUMN_FAMILY_NAME_1.getBytes(StandardCharsets.UTF_8))); + Row mutationRow = + Row.withSchema(SCHEMA) + .withFieldValue("key", "key-1".getBytes(StandardCharsets.UTF_8)) + .withFieldValue("mutations", mutations) + .build(); + + PCollectionRowTuple.of("input", p.apply(Create.of(Arrays.asList(mutationRow)))) + .apply(writeTransform); + p.run().waitUntilFinish(); + + // get rows from table + List rows = + dataClient.readRows(Query.create(tableId)).stream().collect(Collectors.toList()); + + // we should still have only one row with the same key + assertEquals(1, rows.size()); + assertEquals("key-1", rows.get(0).getKey().toStringUtf8()); + // check the new column exists with only one cell. + // also check cell value is correct + com.google.cloud.bigtable.data.v2.models.Row row = rows.get(0); + List cellsNewCol = row.getCells(COLUMN_FAMILY_NAME_1, "new_col"); + assertEquals(1, cellsNewCol.size()); + assertEquals("new-val-1", cellsNewCol.get(0).getValue().toStringUtf8()); + } + + @Test + public void testDeleteCellsFromColumn() { + RowMutation rowMutation = + RowMutation.create(tableId, "key-1") + .setCell(COLUMN_FAMILY_NAME_1, "col_a", "val-1-a") + .setCell(COLUMN_FAMILY_NAME_1, "col_b", "val-1-b"); + dataClient.mutateRow(rowMutation); + // write two cells in col_a. both should get deleted + rowMutation = + RowMutation.create(tableId, "key-1").setCell(COLUMN_FAMILY_NAME_1, "col_a", "new-val-1-a"); + dataClient.mutateRow(rowMutation); + + List> mutations = new ArrayList<>(); + // mutation to delete cells from a column + mutations.add( + ImmutableMap.of( + "type", "DeleteFromColumn".getBytes(StandardCharsets.UTF_8), + "column_qualifier", "col_a".getBytes(StandardCharsets.UTF_8), + "family_name", COLUMN_FAMILY_NAME_1.getBytes(StandardCharsets.UTF_8))); + Row mutationRow = + Row.withSchema(SCHEMA) + .withFieldValue("key", "key-1".getBytes(StandardCharsets.UTF_8)) + .withFieldValue("mutations", mutations) + .build(); + + PCollectionRowTuple.of("input", p.apply(Create.of(Arrays.asList(mutationRow)))) + .apply(writeTransform); + p.run().waitUntilFinish(); + + // get rows from table + List rows = + dataClient.readRows(Query.create(tableId)).stream().collect(Collectors.toList()); + + // we should still have one row with the same key + assertEquals(1, rows.size()); + assertEquals("key-1", rows.get(0).getKey().toStringUtf8()); + // get cells from this column family. we started with three cells and deleted two from one + // column. + // we should end up with one cell in the column we didn't touch. + // check that the remaining cell is indeed from col_b + com.google.cloud.bigtable.data.v2.models.Row row = rows.get(0); + List cells = row.getCells(COLUMN_FAMILY_NAME_1); + assertEquals(1, cells.size()); + assertEquals("col_b", cells.get(0).getQualifier().toStringUtf8()); + } + + @Test + public void testDeleteCellsFromColumnWithTimestampRange() { + // write two cells in one column with different timestamps. + RowMutation rowMutation = + RowMutation.create(tableId, "key-1") + .setCell(COLUMN_FAMILY_NAME_1, "col", 100_000_000, "val"); + dataClient.mutateRow(rowMutation); + rowMutation = + RowMutation.create(tableId, "key-1") + .setCell(COLUMN_FAMILY_NAME_1, "col", 200_000_000, "new-val"); + dataClient.mutateRow(rowMutation); + + List> mutations = new ArrayList<>(); + // mutation to delete cells from a column within a timestamp range + mutations.add( + ImmutableMap.of( + "type", "DeleteFromColumn".getBytes(StandardCharsets.UTF_8), + "column_qualifier", "col".getBytes(StandardCharsets.UTF_8), + "family_name", COLUMN_FAMILY_NAME_1.getBytes(StandardCharsets.UTF_8), + "start_timestamp_micros", Longs.toByteArray(99_999_999), + "end_timestamp_micros", Longs.toByteArray(100_000_001))); + Row mutationRow = + Row.withSchema(SCHEMA) + .withFieldValue("key", "key-1".getBytes(StandardCharsets.UTF_8)) + .withFieldValue("mutations", mutations) + .build(); + + PCollectionRowTuple.of("input", p.apply(Create.of(Arrays.asList(mutationRow)))) + .apply(writeTransform); + p.run().waitUntilFinish(); + + // get rows from table + List rows = + dataClient.readRows(Query.create(tableId)).stream().collect(Collectors.toList()); + + // we should still have one row with the same key + assertEquals(1, rows.size()); + assertEquals("key-1", rows.get(0).getKey().toStringUtf8()); + // we had two cells in col_a and deleted the older one. we should be left with the newer cell. + // check cell has correct value and timestamp + com.google.cloud.bigtable.data.v2.models.Row row = rows.get(0); + List cells = row.getCells(COLUMN_FAMILY_NAME_1, "col"); + assertEquals(1, cells.size()); + assertEquals("new-val", cells.get(0).getValue().toStringUtf8()); + assertEquals(200_000_000, cells.get(0).getTimestamp()); + } + + @Test + public void testDeleteColumnFamily() { + RowMutation rowMutation = + RowMutation.create(tableId, "key-1") + .setCell(COLUMN_FAMILY_NAME_1, "col_a", "val") + .setCell(COLUMN_FAMILY_NAME_2, "col_b", "val"); + dataClient.mutateRow(rowMutation); + + List> mutations = new ArrayList<>(); + // mutation to delete a whole column family + mutations.add( + ImmutableMap.of( + "type", "DeleteFromFamily".getBytes(StandardCharsets.UTF_8), + "family_name", COLUMN_FAMILY_NAME_1.getBytes(StandardCharsets.UTF_8))); + Row mutationRow = + Row.withSchema(SCHEMA) + .withFieldValue("key", "key-1".getBytes(StandardCharsets.UTF_8)) + .withFieldValue("mutations", mutations) + .build(); + + PCollectionRowTuple.of("input", p.apply(Create.of(Arrays.asList(mutationRow)))) + .apply(writeTransform); + p.run().waitUntilFinish(); + + // get rows from table + List rows = + dataClient.readRows(Query.create(tableId)).stream().collect(Collectors.toList()); + + // we should still have one row with the same key + assertEquals(1, rows.size()); + assertEquals("key-1", rows.get(0).getKey().toStringUtf8()); + // we had one cell in each of two column families. we deleted a column family, so should end up + // with + // one cell in the column family we didn't touch. + com.google.cloud.bigtable.data.v2.models.Row row = rows.get(0); + List cells = row.getCells(); + assertEquals(1, cells.size()); + assertEquals(COLUMN_FAMILY_NAME_2, cells.get(0).getFamily()); + } + + @Test + public void testDeleteRow() { + RowMutation rowMutation = + RowMutation.create(tableId, "key-1").setCell(COLUMN_FAMILY_NAME_1, "col", "val-1"); + dataClient.mutateRow(rowMutation); + rowMutation = + RowMutation.create(tableId, "key-2").setCell(COLUMN_FAMILY_NAME_1, "col", "val-2"); + dataClient.mutateRow(rowMutation); + + List> mutations = new ArrayList<>(); + // mutation to delete a whole row + mutations.add(ImmutableMap.of("type", "DeleteFromRow".getBytes(StandardCharsets.UTF_8))); + Row mutationRow = + Row.withSchema(SCHEMA) + .withFieldValue("key", "key-1".getBytes(StandardCharsets.UTF_8)) + .withFieldValue("mutations", mutations) + .build(); + + PCollectionRowTuple.of("input", p.apply(Create.of(Arrays.asList(mutationRow)))) + .apply(writeTransform); + p.run().waitUntilFinish(); + + // get rows from table + List rows = + dataClient.readRows(Query.create(tableId)).stream().collect(Collectors.toList()); + + // we created two rows then deleted one, so should end up with the row we didn't touch + assertEquals(1, rows.size()); + assertEquals("key-2", rows.get(0).getKey().toStringUtf8()); + } +} diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamActionTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamActionTest.java index 6fe272009ccb4..b2c9d6ac13187 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamActionTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamActionTest.java @@ -30,15 +30,17 @@ import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken; import com.google.cloud.bigtable.data.v2.models.ChangeStreamMutation; +import com.google.cloud.bigtable.data.v2.models.ChangeStreamRecord; import com.google.cloud.bigtable.data.v2.models.CloseStream; import com.google.cloud.bigtable.data.v2.models.Heartbeat; import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange; import com.google.protobuf.ByteString; import com.google.rpc.Status; +import java.math.BigDecimal; import java.util.Collections; import java.util.Optional; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics; -import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.ThroughputEstimator; +import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.BytesThroughputEstimator; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.ReadChangeStreamPartitionProgressTracker; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.StreamProgress; @@ -49,8 +51,13 @@ import org.joda.time.Instant; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +@RunWith(MockitoJUnitRunner.Silent.class) public class ChangeStreamActionTest { private ChangeStreamMetrics metrics; @@ -58,9 +65,10 @@ public class ChangeStreamActionTest { private RestrictionTracker tracker; private PartitionRecord partitionRecord; - private DoFn.OutputReceiver> receiver; + private DoFn.OutputReceiver> receiver; private ManualWatermarkEstimator watermarkEstimator; - private ThroughputEstimator> throughputEstimator; + private BytesThroughputEstimator> throughputEstimator; + @Captor private ArgumentCaptor streamProgressArgumentCaptor; @Before public void setUp() { @@ -69,10 +77,12 @@ public void setUp() { partitionRecord = mock(PartitionRecord.class); receiver = mock(DoFn.OutputReceiver.class); watermarkEstimator = mock(ManualWatermarkEstimator.class); - throughputEstimator = mock(ThroughputEstimator.class); + throughputEstimator = mock(BytesThroughputEstimator.class); - action = new ChangeStreamAction(metrics, throughputEstimator); + action = new ChangeStreamAction(metrics); when(tracker.tryClaim(any())).thenReturn(true); + when(partitionRecord.getPartition()).thenReturn(ByteStringRange.create("a", "b")); + when(throughputEstimator.get()).thenReturn(BigDecimal.valueOf(1000)); } @Test @@ -87,14 +97,40 @@ public void testHeartBeat() { .thenReturn(changeStreamContinuationToken); final Optional result = - action.run(partitionRecord, mockHeartBeat, tracker, receiver, watermarkEstimator, false); + action.run( + partitionRecord, + mockHeartBeat, + tracker, + receiver, + watermarkEstimator, + throughputEstimator); assertFalse(result.isPresent()); verify(metrics).incHeartbeatCount(); verify(watermarkEstimator).setWatermark(eq(lowWatermark)); - StreamProgress streamProgress = new StreamProgress(changeStreamContinuationToken, lowWatermark); - verify(tracker).tryClaim(eq(streamProgress)); - verify(throughputEstimator, never()).update(any(), any()); + StreamProgress streamProgress = + new StreamProgress( + changeStreamContinuationToken, + lowWatermark, + BigDecimal.valueOf(1000), + Instant.now(), + true); + verify(tracker).tryClaim(streamProgressArgumentCaptor.capture()); + assertEquals( + streamProgress.getCurrentToken(), + streamProgressArgumentCaptor.getValue().getCurrentToken()); + assertEquals( + streamProgress.getThroughputEstimate(), + streamProgressArgumentCaptor.getValue().getThroughputEstimate()); + assertEquals( + streamProgress.getEstimatedLowWatermark(), + streamProgressArgumentCaptor.getValue().getEstimatedLowWatermark()); + assertEquals( + streamProgress.isHeartbeat(), streamProgressArgumentCaptor.getValue().isHeartbeat()); + KV record = + KV.of(ByteStringRange.serializeToByteString(partitionRecord.getPartition()), mockHeartBeat); + verify(receiver).outputWithTimestamp(eq(record), eq(Instant.EPOCH)); + verify(throughputEstimator).update(any(), eq(record)); } @Test @@ -109,14 +145,19 @@ public void testCloseStreamResume() { .thenReturn(Collections.singletonList(changeStreamContinuationToken)); final Optional result = - action.run(partitionRecord, mockCloseStream, tracker, receiver, watermarkEstimator, false); + action.run( + partitionRecord, + mockCloseStream, + tracker, + receiver, + watermarkEstimator, + throughputEstimator); assertTrue(result.isPresent()); assertEquals(DoFn.ProcessContinuation.resume(), result.get()); verify(metrics).incClosestreamCount(); StreamProgress streamProgress = new StreamProgress(mockCloseStream); verify(tracker).tryClaim(eq(streamProgress)); - verify(throughputEstimator, never()).update(any(), any()); } @Test @@ -134,18 +175,40 @@ public void testChangeStreamMutationUser() { Mockito.when(changeStreamMutation.getEstimatedLowWatermark()) .thenReturn(toThreetenInstant(lowWatermark)); Mockito.when(changeStreamMutation.getType()).thenReturn(ChangeStreamMutation.MutationType.USER); - KV record = + KV record = KV.of(changeStreamMutation.getRowKey(), changeStreamMutation); final Optional result = action.run( - partitionRecord, changeStreamMutation, tracker, receiver, watermarkEstimator, false); + partitionRecord, + changeStreamMutation, + tracker, + receiver, + watermarkEstimator, + throughputEstimator); assertFalse(result.isPresent()); verify(metrics).incChangeStreamMutationUserCounter(); verify(metrics, never()).incChangeStreamMutationGcCounter(); - StreamProgress streamProgress = new StreamProgress(changeStreamContinuationToken, lowWatermark); - verify(tracker).tryClaim(eq(streamProgress)); + StreamProgress streamProgress = + new StreamProgress( + changeStreamContinuationToken, + lowWatermark, + BigDecimal.valueOf(1000), + Instant.now(), + false); + verify(tracker).tryClaim(streamProgressArgumentCaptor.capture()); + assertEquals( + streamProgress.getCurrentToken(), + streamProgressArgumentCaptor.getValue().getCurrentToken()); + assertEquals( + streamProgress.getThroughputEstimate(), + streamProgressArgumentCaptor.getValue().getThroughputEstimate()); + assertEquals( + streamProgress.getEstimatedLowWatermark(), + streamProgressArgumentCaptor.getValue().getEstimatedLowWatermark()); + assertEquals( + streamProgress.isHeartbeat(), streamProgressArgumentCaptor.getValue().isHeartbeat()); verify(receiver).outputWithTimestamp(eq(record), eq(Instant.EPOCH)); verify(watermarkEstimator).setWatermark(eq(lowWatermark)); verify(throughputEstimator).update(any(), eq(record)); @@ -167,18 +230,40 @@ public void testChangeStreamMutationGc() { .thenReturn(toThreetenInstant(lowWatermark)); Mockito.when(changeStreamMutation.getType()) .thenReturn(ChangeStreamMutation.MutationType.GARBAGE_COLLECTION); - KV record = + KV record = KV.of(changeStreamMutation.getRowKey(), changeStreamMutation); final Optional result = action.run( - partitionRecord, changeStreamMutation, tracker, receiver, watermarkEstimator, false); + partitionRecord, + changeStreamMutation, + tracker, + receiver, + watermarkEstimator, + throughputEstimator); assertFalse(result.isPresent()); verify(metrics).incChangeStreamMutationGcCounter(); verify(metrics, never()).incChangeStreamMutationUserCounter(); - StreamProgress streamProgress = new StreamProgress(changeStreamContinuationToken, lowWatermark); - verify(tracker).tryClaim(eq(streamProgress)); + StreamProgress streamProgress = + new StreamProgress( + changeStreamContinuationToken, + lowWatermark, + BigDecimal.valueOf(1000), + Instant.now(), + false); + verify(tracker).tryClaim(streamProgressArgumentCaptor.capture()); + assertEquals( + streamProgress.getCurrentToken(), + streamProgressArgumentCaptor.getValue().getCurrentToken()); + assertEquals( + streamProgress.getThroughputEstimate(), + streamProgressArgumentCaptor.getValue().getThroughputEstimate()); + assertEquals( + streamProgress.getEstimatedLowWatermark(), + streamProgressArgumentCaptor.getValue().getEstimatedLowWatermark()); + assertEquals( + streamProgress.isHeartbeat(), streamProgressArgumentCaptor.getValue().isHeartbeat()); verify(receiver).outputWithTimestamp(eq(record), eq(Instant.EPOCH)); verify(watermarkEstimator).setWatermark(eq(lowWatermark)); verify(throughputEstimator).update(any(), eq(record)); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsActionTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsActionTest.java index 9d8aee40ed3bc..b96c544160b98 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsActionTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsActionTest.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -114,6 +115,7 @@ public void setUp() throws Exception { new MetadataTableAdminDao( adminClient, null, changeStreamId, MetadataTableAdminDao.DEFAULT_METADATA_TABLE_NAME); metadataTableAdminDao.createMetadataTable(); + metadataTableAdminDao.cleanUpPrefix(); metadataTableDao = new MetadataTableDao( dataClient, @@ -687,19 +689,22 @@ public void testMissingPartitionReconciled() throws Exception { HashMap missingPartitionDurations = new HashMap<>(); ByteStringRange partitionAB = ByteStringRange.create("a", "b"); - // Partition missing for 5 minutes less 1 seconds. + // Partition missing for 10 minutes less 1 second. missingPartitionDurations.put( - partitionAB, Instant.now().minus(Duration.standardSeconds(10 * 60 - 1))); + partitionAB, Instant.now().minus(Duration.standardSeconds(20 * 60 - 1))); metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations); - // No new partitions and missing partition has not been missing for long enough. + // Since there's no NewPartition corresponding to the missing partition, we can't reconcile with + // continuation tokens. In order to reconcile without continuation tokens, the partition needs + // to have been missing for more than 10 minutes. assertEquals( DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)), action.run( tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false))); verify(receiver, never()).outputWithTimestamp(any(), any()); + assertEquals(1, metadataTableDao.readDetectNewPartitionMissingPartitions().size()); - // Sleep for 1 second, enough that the missing partition needs to be reconciled. + // Sleep for more than 1 second, enough that the missing partition needs to be reconciled. Thread.sleep(1001); // We advance the restriction tracker by 1. Because it is not a multiple of 2, we don't @@ -713,6 +718,7 @@ public void testMissingPartitionReconciled() throws Exception { action.run( tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false))); verify(receiver, never()).outputWithTimestamp(any(), any()); + assertEquals(1, metadataTableDao.readDetectNewPartitionMissingPartitions().size()); // Multiple of 2, reconciliation should happen. offsetRange = new OffsetRange(54, Long.MAX_VALUE); @@ -723,6 +729,7 @@ public void testMissingPartitionReconciled() throws Exception { DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)), action.run( tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false))); + assertEquals(0, metadataTableDao.readDetectNewPartitionMissingPartitions().size()); verify(receiver, times(1)) .outputWithTimestamp(partitionRecordArgumentCaptor.capture(), eq(Instant.EPOCH)); assertEquals(partitionAB, partitionRecordArgumentCaptor.getValue().getPartition()); @@ -735,4 +742,91 @@ public void testMissingPartitionReconciled() throws Exception { // the low watermark. assertEquals(startTime, partitionRecordArgumentCaptor.getValue().getStartTime()); } + + // Reconcile runs immediately after a previous reconcile without RCSP working on the previous + // reconciled result. This can happen if the runner is backed up and slow. + // 1. Partition in NewPartition waiting for 1 minute + // 2. Reconciler takes the partition and outputs it. Reconciler marks the partition in + // NewPartition as deleted + // 3. Reconciler runs again + @Test + public void testBackToBackReconcile() throws Exception { + // We only start reconciling after 50. + // We advance watermark on every 2 restriction tracker advancement + OffsetRange offsetRange = new OffsetRange(52, Long.MAX_VALUE); + when(tracker.currentRestriction()).thenReturn(offsetRange); + when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true); + + // Write 2 partitions to the table, missing [a, b) because [a, b) is trying to merge into [a, c) + ByteStringRange partitionEmptyA = ByteStringRange.create("", "a"); + Instant watermarkEmptyA = endTime.plus(Duration.millis(100)); + PartitionRecord partitionRecordEmptyA = + new PartitionRecord( + partitionEmptyA, + watermarkEmptyA, + UniqueIdGenerator.getNextId(), + watermarkEmptyA, + Collections.emptyList(), + null); + metadataTableDao.lockAndRecordPartition(partitionRecordEmptyA); + ByteStringRange partitionBEmpty = ByteStringRange.create("b", ""); + Instant watermarkBEmpty = endTime.plus(Duration.millis(1)); + PartitionRecord partitionRecordBEmpty = + new PartitionRecord( + partitionBEmpty, + watermarkBEmpty, + UniqueIdGenerator.getNextId(), + watermarkBEmpty, + Collections.emptyList(), + null); + metadataTableDao.lockAndRecordPartition(partitionRecordBEmpty); + + // NewPartition [a, b) trying to merge into [a, c) + ByteStringRange parentPartitionAB = ByteStringRange.create("a", "b"); + Instant watermarkAB = startTime; + ChangeStreamContinuationToken tokenAB = + ChangeStreamContinuationToken.create(parentPartitionAB, "ab"); + + ByteStringRange childPartitionAC = ByteStringRange.create("a", "c"); + + NewPartition newPartitionACFromAB = + new NewPartition(childPartitionAC, Collections.singletonList(tokenAB), watermarkAB); + + metadataTableDao.writeNewPartition(newPartitionACFromAB); + + // Artificially create that partitionAB has been missing for more than 1 minute. + HashMap missingPartitionDurations = new HashMap<>(); + missingPartitionDurations.put( + parentPartitionAB, Instant.now().minus(Duration.standardSeconds(121))); + metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations); + + assertEquals(1, metadataTableDao.readNewPartitions().size()); + + assertEquals( + DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)), + action.run( + tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false))); + // AB should be reconciled with token because it's been missing for more than 1 minute + verify(receiver, times(1)) + .outputWithTimestamp(partitionRecordArgumentCaptor.capture(), eq(Instant.EPOCH)); + + assertEquals(parentPartitionAB, partitionRecordArgumentCaptor.getValue().getPartition()); + assertEquals(watermarkAB, partitionRecordArgumentCaptor.getValue().getParentLowWatermark()); + assertEquals(endTime, partitionRecordArgumentCaptor.getValue().getEndTime()); + assertEquals( + partitionRecordArgumentCaptor.getValue().getChangeStreamContinuationTokens(), + Collections.singletonList(tokenAB)); + assertTrue(metadataTableDao.readNewPartitions().isEmpty()); + assertTrue(metadataTableDao.readDetectNewPartitionMissingPartitions().isEmpty()); + + clearInvocations(receiver); + // The reconciled partition was not processed by RCSP, so NewPartition is still marked for + // deletion and the partition is still considered missing. We run DNP again. + assertEquals( + DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)), + action.run( + tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false))); + // We don't reconcile the partition again. + verify(receiver, never()).outputWithTimestamp(any(), any()); + } } diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsActionTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsActionTest.java index 102200ab9a196..5352828cbf381 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsActionTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsActionTest.java @@ -80,6 +80,7 @@ public void setUp() throws Exception { new MetadataTableAdminDao( adminClient, null, changeStreamId, MetadataTableAdminDao.DEFAULT_METADATA_TABLE_NAME); metadataTableAdminDao.createMetadataTable(); + metadataTableAdminDao.cleanUpPrefix(); startTime = Instant.now(); endTime = startTime.plus(Duration.standardSeconds(10)); } diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionActionTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionActionTest.java index f997db07a8aea..43419b7147f6a 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionActionTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionActionTest.java @@ -20,7 +20,6 @@ import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamContinuationTokenHelper.getTokenWithCorrectPartition; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -30,7 +29,6 @@ import com.google.api.gax.rpc.ServerStream; import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken; -import com.google.cloud.bigtable.data.v2.models.ChangeStreamMutation; import com.google.cloud.bigtable.data.v2.models.ChangeStreamRecord; import com.google.cloud.bigtable.data.v2.models.CloseStream; import com.google.cloud.bigtable.data.v2.models.Heartbeat; @@ -45,6 +43,7 @@ import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.ChangeStreamDao; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao; +import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.CoderSizeEstimator; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.ReadChangeStreamPartitionProgressTracker; @@ -71,8 +70,9 @@ public class ReadChangeStreamPartitionActionTest { // private PartitionRecord partitionRecord; private StreamProgress restriction; private RestrictionTracker tracker; - private DoFn.OutputReceiver> receiver; + private DoFn.OutputReceiver> receiver; private ManualWatermarkEstimator watermarkEstimator; + private CoderSizeEstimator> sizeEstimator; private ByteStringRange partition; private String uuid; @@ -84,11 +84,17 @@ public void setUp() throws Exception { changeStreamDao = mock(ChangeStreamDao.class); metrics = mock(ChangeStreamMetrics.class); changeStreamAction = mock(ChangeStreamAction.class); + sizeEstimator = mock(CoderSizeEstimator.class); Duration heartbeatDuration = Duration.standardSeconds(1); action = new ReadChangeStreamPartitionAction( - metadataTableDao, changeStreamDao, metrics, changeStreamAction, heartbeatDuration); + metadataTableDao, + changeStreamDao, + metrics, + changeStreamAction, + heartbeatDuration, + sizeEstimator); restriction = mock(StreamProgress.class); tracker = mock(ReadChangeStreamPartitionProgressTracker.class); @@ -121,10 +127,10 @@ public void testLockingRowSucceed() throws IOException { Heartbeat mockHeartBeat = Mockito.mock(Heartbeat.class); when(responseIterator.next()).thenReturn(mockHeartBeat); when(responseIterator.hasNext()).thenReturn(true); - when(changeStreamDao.readChangeStreamPartition(any(), any(), any(), any(), anyBoolean())) + when(changeStreamDao.readChangeStreamPartition(any(), any(), any(), any())) .thenReturn(responses); - when(changeStreamAction.run(any(), any(), any(), any(), any(), anyBoolean())) + when(changeStreamAction.run(any(), any(), any(), any(), any(), any())) .thenReturn(Optional.of(DoFn.ProcessContinuation.stop())); final DoFn.ProcessContinuation result = @@ -133,7 +139,7 @@ public void testLockingRowSucceed() throws IOException { assertEquals(DoFn.ProcessContinuation.stop(), result); // Verify that on successful lock, we don't tryClaim on the tracker. verify(tracker, never()).tryClaim(any()); - verify(changeStreamAction).run(any(), any(), any(), any(), any(), anyBoolean()); + verify(changeStreamAction).run(any(), any(), any(), any(), any(), any()); } @Test @@ -150,7 +156,7 @@ public void testLockingRowFailsStops() throws IOException { StreamProgress streamProgress = new StreamProgress(); streamProgress.setFailToLock(true); verify(tracker).tryClaim(streamProgress); - verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean()); + verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), any()); } @Test @@ -167,10 +173,10 @@ public void testLockingRowNotNeededAfterFirstRun() throws IOException { Heartbeat mockHeartBeat = Mockito.mock(Heartbeat.class); when(responseIterator.next()).thenReturn(mockHeartBeat); when(responseIterator.hasNext()).thenReturn(true); - when(changeStreamDao.readChangeStreamPartition(any(), any(), any(), any(), anyBoolean())) + when(changeStreamDao.readChangeStreamPartition(any(), any(), any(), any())) .thenReturn(responses); - when(changeStreamAction.run(any(), any(), any(), any(), any(), anyBoolean())) + when(changeStreamAction.run(any(), any(), any(), any(), any(), any())) .thenReturn(Optional.of(DoFn.ProcessContinuation.stop())); final DoFn.ProcessContinuation result = @@ -179,7 +185,7 @@ public void testLockingRowNotNeededAfterFirstRun() throws IOException { assertEquals(DoFn.ProcessContinuation.stop(), result); // Verify that on successful lock, we don't tryClaim on the tracker. verify(tracker, never()).tryClaim(any()); - verify(changeStreamAction).run(any(), any(), any(), any(), any(), anyBoolean()); + verify(changeStreamAction).run(any(), any(), any(), any(), any(), any()); } @Test @@ -198,7 +204,7 @@ public void testLockingRowNotNeededAfterFirstRunNotSame() throws IOException { StreamProgress streamProgress = new StreamProgress(); streamProgress.setFailToLock(true); verify(tracker).tryClaim(streamProgress); - verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean()); + verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), any()); } @Test @@ -214,16 +220,16 @@ public void testThatChangeStreamWorkerCounterIsIncrementedOnInitialRun() throws Heartbeat mockHeartBeat = Mockito.mock(Heartbeat.class); when(responseIterator.next()).thenReturn(mockHeartBeat); when(responseIterator.hasNext()).thenReturn(true); - when(changeStreamDao.readChangeStreamPartition(any(), any(), any(), any(), anyBoolean())) + when(changeStreamDao.readChangeStreamPartition(any(), any(), any(), any())) .thenReturn(responses); - when(changeStreamAction.run(any(), any(), any(), any(), any(), anyBoolean())) + when(changeStreamAction.run(any(), any(), any(), any(), any(), any())) .thenReturn(Optional.of(DoFn.ProcessContinuation.stop())); final DoFn.ProcessContinuation result = action.run(partitionRecord, tracker, receiver, watermarkEstimator); assertEquals(DoFn.ProcessContinuation.stop(), result); - verify(changeStreamAction).run(any(), any(), any(), any(), any(), anyBoolean()); + verify(changeStreamAction).run(any(), any(), any(), any(), any(), any()); } @Test @@ -240,7 +246,7 @@ public void testCloseStreamTerminateOKStatus() throws IOException { action.run(partitionRecord, tracker, receiver, watermarkEstimator); assertEquals(DoFn.ProcessContinuation.stop(), result); // Should terminate before reaching processing stream partition responses. - verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean()); + verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), any()); // Should not try claim any restriction when processing CloseStream verify(tracker, (never())).tryClaim(any()); // Should decrement the metric on termination. @@ -266,7 +272,7 @@ public void testCloseStreamTerminateNotOutOfRangeStatus() throws IOException { action.run(partitionRecord, tracker, receiver, watermarkEstimator); assertEquals(DoFn.ProcessContinuation.stop(), result); // Should terminate before reaching processing stream partition responses. - verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean()); + verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), any()); // Should not try claim any restriction when processing CloseStream verify(tracker, (never())).tryClaim(any()); // Should decrement the metric on termination. @@ -301,7 +307,7 @@ public void testCloseStreamWritesContinuationTokens() throws IOException { action.run(partitionRecord, tracker, receiver, watermarkEstimator); assertEquals(DoFn.ProcessContinuation.stop(), result); // Should terminate before reaching processing stream partition responses. - verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean()); + verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), any()); // Should not try claim any restriction when processing CloseStream verify(tracker, (never())).tryClaim(any()); // Should decrement the metric on termination. @@ -353,7 +359,7 @@ public void testCloseStreamNewPartitionMerge() throws IOException { action.run(partitionRecord, tracker, receiver, watermarkEstimator); assertEquals(DoFn.ProcessContinuation.stop(), result); // Should terminate before reaching processing stream partition responses. - verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean()); + verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), any()); // Should not try claim any restriction when processing CloseStream verify(tracker, (never())).tryClaim(any()); // Should decrement the metric on termination. @@ -399,7 +405,7 @@ public void testCloseStreamMergeWithoutNewPartitionsField() throws IOException { action.run(partitionRecord, tracker, receiver, watermarkEstimator); assertEquals(DoFn.ProcessContinuation.stop(), result); // Should terminate before reaching processing stream partition responses. - verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean()); + verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), any()); // Should not try claim any restriction when processing CloseStream verify(tracker, (never())).tryClaim(any()); // Should decrement the metric on termination. diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ResumeFromPreviousPipelineActionTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ResumeFromPreviousPipelineActionTest.java index 3f8737c059577..86acf91f0deac 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ResumeFromPreviousPipelineActionTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ResumeFromPreviousPipelineActionTest.java @@ -99,6 +99,7 @@ public void setUp() throws Exception { new MetadataTableAdminDao( adminClient, null, changeStreamId, MetadataTableAdminDao.DEFAULT_METADATA_TABLE_NAME); metadataTableAdminDao.createMetadataTable(); + metadataTableAdminDao.cleanUpPrefix(); metadataTableDao = new MetadataTableDao( dataClient, diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDaoTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDaoTest.java index c82e3eac29990..867117b4d3927 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDaoTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDaoTest.java @@ -23,10 +23,14 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import com.google.api.core.ApiFuture; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; import com.google.cloud.bigtable.data.v2.BigtableDataClient; @@ -45,6 +49,9 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition; @@ -743,4 +750,28 @@ public void testLockPartitionRecordsMetadata() { .get(0) .getValue()); } + + @Test + public void mutateRowWithHardTimeoutErrorHandling() + throws ExecutionException, InterruptedException, TimeoutException { + BigtableDataClient mockClient = Mockito.mock(BigtableDataClient.class); + MetadataTableDao daoWithMock = + new MetadataTableDao(mockClient, "test-table", ByteString.copyFromUtf8("test")); + ApiFuture mockFuture = mock(ApiFuture.class); + when(mockClient.mutateRowAsync(any())).thenReturn(mockFuture); + + when(mockFuture.get(40, TimeUnit.SECONDS)) + .thenThrow(TimeoutException.class) + .thenThrow(InterruptedException.class) + .thenThrow(ExecutionException.class); + assertThrows( + RuntimeException.class, + () -> daoWithMock.mutateRowWithHardTimeout(RowMutation.create("test", "test").deleteRow())); + assertThrows( + RuntimeException.class, + () -> daoWithMock.mutateRowWithHardTimeout(RowMutation.create("test", "test").deleteRow())); + assertThrows( + RuntimeException.class, + () -> daoWithMock.mutateRowWithHardTimeout(RowMutation.create("test", "test").deleteRow())); + } } diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/FilterForMutationDoFnTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/FilterForMutationDoFnTest.java new file mode 100644 index 0000000000000..46fe78dd878e9 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/FilterForMutationDoFnTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import com.google.cloud.bigtable.data.v2.models.ChangeStreamMutation; +import com.google.cloud.bigtable.data.v2.models.CloseStream; +import com.google.cloud.bigtable.data.v2.models.Heartbeat; +import com.google.protobuf.ByteString; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.values.KV; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class FilterForMutationDoFnTest { + + private FilterForMutationDoFn doFn; + @Mock private DoFn.OutputReceiver> outputReceiver; + + @Before + public void setup() { + doFn = new FilterForMutationDoFn(); + } + + @Test + public void shouldNotOutputHeartbeats() { + Heartbeat heartbeat = mock(Heartbeat.class); + doFn.processElement(KV.of(ByteString.copyFromUtf8("test"), heartbeat), outputReceiver); + verify(outputReceiver, never()).output(any()); + } + + @Test + public void shouldOutputChangeStreamMutations() { + ChangeStreamMutation mutation = mock(ChangeStreamMutation.class); + doFn.processElement(KV.of(ByteString.copyFromUtf8("test"), mutation), outputReceiver); + verify(outputReceiver, times(1)).output(KV.of(ByteString.copyFromUtf8("test"), mutation)); + } + + @Test + public void shouldOutputCloseStreams() { + // This shouldn't happen but if it were to we wouldn't want the CloseStreams to be returned to + // users + CloseStream closeStream = mock(CloseStream.class); + doFn.processElement(KV.of(ByteString.copyFromUtf8("test"), closeStream), outputReceiver); + verify(outputReceiver, never()).output(any()); + } +} diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFnTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFnTest.java index a9fa54d4c493e..f0337504282fc 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFnTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFnTest.java @@ -20,7 +20,6 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; @@ -88,8 +87,6 @@ public void setUp() throws IOException { metadataTableAdminDao = spy(new MetadataTableAdminDao(adminClient, null, changeStreamName, tableId)); metadataTableAdminDao.createMetadataTable(); - doReturn(true).when(metadataTableAdminDao).isAppProfileSingleClusterAndTransactional(any()); - when(daoFactory.getMetadataTableAdminDao()).thenReturn(metadataTableAdminDao); metadataTableDao = new MetadataTableDao( dataClient, tableId, metadataTableAdminDao.getChangeStreamNamePrefix()); @@ -102,10 +99,7 @@ public void testInitializeDefault() throws IOException { Instant startTime = Instant.now(); InitializeDoFn initializeDoFn = new InitializeDoFn( - daoFactory, - "app-profile", - startTime, - BigtableIO.ExistingPipelineOptions.FAIL_IF_EXISTS); + daoFactory, startTime, BigtableIO.ExistingPipelineOptions.FAIL_IF_EXISTS); initializeDoFn.processElement(outputReceiver); verify(outputReceiver, times(1)).output(new InitialPipelineState(startTime, false)); } @@ -116,10 +110,7 @@ public void testInitializeStopWithExistingPipeline() throws IOException { Instant startTime = Instant.now(); InitializeDoFn initializeDoFn = new InitializeDoFn( - daoFactory, - "app-profile", - startTime, - BigtableIO.ExistingPipelineOptions.FAIL_IF_EXISTS); + daoFactory, startTime, BigtableIO.ExistingPipelineOptions.FAIL_IF_EXISTS); initializeDoFn.processElement(outputReceiver); verify(outputReceiver, never()).output(any()); } @@ -139,51 +130,7 @@ public void testInitializeStopWithoutDNP() throws IOException { Instant startTime = Instant.now(); InitializeDoFn initializeDoFn = new InitializeDoFn( - daoFactory, - "app-profile", - startTime, - BigtableIO.ExistingPipelineOptions.FAIL_IF_EXISTS); - initializeDoFn.processElement(outputReceiver); - verify(outputReceiver, times(1)).output(new InitialPipelineState(startTime, false)); - assertNull(dataClient.readRow(tableId, metadataTableAdminDao.getChangeStreamNamePrefix())); - } - - @Test - public void testInitializeNewWithoutDNP() throws IOException { - // DNP row doesn't exist, so we output with provided start time and we clean up any existing - // rows under the prefix. - dataClient.mutateRow( - RowMutation.create( - tableId, - metadataTableAdminDao - .getChangeStreamNamePrefix() - .concat(ByteString.copyFromUtf8("existing_row"))) - .setCell( - MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123)); - Instant startTime = Instant.now(); - InitializeDoFn initializeDoFn = - new InitializeDoFn( - daoFactory, "app-profile", startTime, BigtableIO.ExistingPipelineOptions.NEW); - initializeDoFn.processElement(outputReceiver); - verify(outputReceiver, times(1)).output(new InitialPipelineState(startTime, false)); - assertNull(dataClient.readRow(tableId, metadataTableAdminDao.getChangeStreamNamePrefix())); - } - - @Test - public void testInitializeNewWithDNP() throws IOException { - metadataTableDao.updateDetectNewPartitionWatermark(Instant.now()); - dataClient.mutateRow( - RowMutation.create( - tableId, - metadataTableAdminDao - .getChangeStreamNamePrefix() - .concat(ByteString.copyFromUtf8("existing_row"))) - .setCell( - MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123)); - Instant startTime = Instant.now(); - InitializeDoFn initializeDoFn = - new InitializeDoFn( - daoFactory, "app-profile", startTime, BigtableIO.ExistingPipelineOptions.NEW); + daoFactory, startTime, BigtableIO.ExistingPipelineOptions.FAIL_IF_EXISTS); initializeDoFn.processElement(outputReceiver); verify(outputReceiver, times(1)).output(new InitialPipelineState(startTime, false)); assertNull(dataClient.readRow(tableId, metadataTableAdminDao.getChangeStreamNamePrefix())); @@ -201,8 +148,7 @@ public void testInitializeResumeWithoutDNP() throws IOException { MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123)); Instant startTime = Instant.now(); InitializeDoFn initializeDoFn = - new InitializeDoFn( - daoFactory, "app-profile", startTime, BigtableIO.ExistingPipelineOptions.RESUME_OR_NEW); + new InitializeDoFn(daoFactory, startTime, BigtableIO.ExistingPipelineOptions.RESUME_OR_NEW); initializeDoFn.processElement(outputReceiver); // We want to resume but there's no DNP row, so we resume from the startTime provided. verify(outputReceiver, times(1)).output(new InitialPipelineState(startTime, false)); @@ -222,8 +168,7 @@ public void testInitializeResumeWithDNP() throws IOException { MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123)); Instant startTime = Instant.now(); InitializeDoFn initializeDoFn = - new InitializeDoFn( - daoFactory, "app-profile", startTime, BigtableIO.ExistingPipelineOptions.RESUME_OR_NEW); + new InitializeDoFn(daoFactory, startTime, BigtableIO.ExistingPipelineOptions.RESUME_OR_NEW); initializeDoFn.processElement(outputReceiver); verify(outputReceiver, times(1)).output(new InitialPipelineState(resumeTime, true)); assertNull(dataClient.readRow(tableId, metadataTableAdminDao.getChangeStreamNamePrefix())); @@ -241,8 +186,7 @@ public void testInitializeSkipCleanupWithoutDNP() throws IOException { MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123)); Instant startTime = Instant.now(); InitializeDoFn initializeDoFn = - new InitializeDoFn( - daoFactory, "app-profile", startTime, ExistingPipelineOptions.SKIP_CLEANUP); + new InitializeDoFn(daoFactory, startTime, ExistingPipelineOptions.SKIP_CLEANUP); initializeDoFn.processElement(outputReceiver); // Skip cleanup will always resume from startTime verify(outputReceiver, times(1)).output(new InitialPipelineState(startTime, false)); @@ -264,8 +208,7 @@ public void testInitializeSkipCleanupWithDNP() throws IOException { MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123)); Instant startTime = Instant.now(); InitializeDoFn initializeDoFn = - new InitializeDoFn( - daoFactory, "app-profile", startTime, ExistingPipelineOptions.SKIP_CLEANUP); + new InitializeDoFn(daoFactory, startTime, ExistingPipelineOptions.SKIP_CLEANUP); initializeDoFn.processElement(outputReceiver); // We don't want the pipeline to resume to avoid duplicates verify(outputReceiver, never()).output(any()); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/ReadChangeStreamPartitionDoFnTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/ReadChangeStreamPartitionDoFnTest.java index e8f59c2bd1776..b89b2bf15aa31 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/ReadChangeStreamPartitionDoFnTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/ReadChangeStreamPartitionDoFnTest.java @@ -20,7 +20,6 @@ import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.TimestampConverter.toThreetenInstant; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -33,6 +32,7 @@ import com.google.cloud.bigtable.data.v2.models.Range; import com.google.protobuf.ByteString; import java.io.IOException; +import java.math.BigDecimal; import java.util.Collections; import java.util.Iterator; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics; @@ -42,8 +42,7 @@ import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.ChangeStreamDao; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.DaoFactory; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao; -import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.BytesThroughputEstimator; -import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.SizeEstimator; +import org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator.CoderSizeEstimator; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.ReadChangeStreamPartitionProgressTracker; import org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.StreamProgress; @@ -58,7 +57,7 @@ public class ReadChangeStreamPartitionDoFnTest { private ChangeStreamDao changeStreamDao; private MetadataTableDao metadataTableDao; - private SizeEstimator> sizeEstimator; + private CoderSizeEstimator> sizeEstimator; private ReadChangeStreamPartitionDoFn doFn; @Before @@ -74,22 +73,28 @@ public void setup() throws IOException { ActionFactory actionFactory = mock(ActionFactory.class); ChangeStreamMetrics metrics = mock(ChangeStreamMetrics.class); - sizeEstimator = mock(SizeEstimator.class); - BytesThroughputEstimator> throughputEstimator = - new BytesThroughputEstimator<>( - ReadChangeStreamPartitionDoFn.THROUGHPUT_ESTIMATION_WINDOW_SECONDS, sizeEstimator, 1); - ChangeStreamAction changeStreamAction = new ChangeStreamAction(metrics, throughputEstimator); + sizeEstimator = mock(CoderSizeEstimator.class); + ChangeStreamAction changeStreamAction = new ChangeStreamAction(metrics); ReadChangeStreamPartitionAction readChangeStreamPartitionAction = new ReadChangeStreamPartitionAction( - metadataTableDao, changeStreamDao, metrics, changeStreamAction, heartbeatDuration); - when(actionFactory.changeStreamAction(metrics, throughputEstimator)) - .thenReturn(changeStreamAction); + metadataTableDao, + changeStreamDao, + metrics, + changeStreamAction, + heartbeatDuration, + sizeEstimator); + when(actionFactory.changeStreamAction(metrics)).thenReturn(changeStreamAction); when(actionFactory.readChangeStreamPartitionAction( - metadataTableDao, changeStreamDao, metrics, changeStreamAction, heartbeatDuration)) + metadataTableDao, + changeStreamDao, + metrics, + changeStreamAction, + heartbeatDuration, + sizeEstimator)) .thenReturn(readChangeStreamPartitionAction); - doFn = new ReadChangeStreamPartitionDoFn(heartbeatDuration, daoFactory, actionFactory, metrics); - doFn.setThroughputEstimator(throughputEstimator); + doFn = new ReadChangeStreamPartitionDoFn(daoFactory, actionFactory, metrics); + doFn.setSizeEstimator(sizeEstimator); } @Test @@ -112,7 +117,7 @@ public void testProcessElementAndGetSize() throws IOException, InterruptedExcept ReadChangeStreamPartitionProgressTracker restrictionTracker = mock(ReadChangeStreamPartitionProgressTracker.class); when(restrictionTracker.currentRestriction()).thenReturn(new StreamProgress()); - DoFn.OutputReceiver> receiver = + DoFn.OutputReceiver> receiver = mock(DoFn.OutputReceiver.class); ManualWatermarkEstimator watermarkEstimator = mock(ManualWatermarkEstimator.class); doFn.setup(); @@ -130,7 +135,7 @@ public void testProcessElementAndGetSize() throws IOException, InterruptedExcept when(mockResponses.hasNext()).thenReturn(true, true, true); when(mockResponses.next()).thenReturn(mockMutation, mockMutation, mockMutation); when(mockStream.iterator()).thenReturn(mockResponses); - when(changeStreamDao.readChangeStreamPartition(any(), any(), any(), any(), anyBoolean())) + when(changeStreamDao.readChangeStreamPartition(any(), any(), any(), any())) .thenReturn(mockStream); when(watermarkEstimator.getState()).thenReturn(tenSecondsAgo); @@ -138,10 +143,43 @@ public void testProcessElementAndGetSize() throws IOException, InterruptedExcept when(restrictionTracker.tryClaim(any())).thenReturn(true, true, false); doFn.processElement(partition, restrictionTracker, receiver, watermarkEstimator); - double sizeEstimate = doFn.getSize(new StreamProgress(testToken, tenSecondsAgo)); + double sizeEstimate = + doFn.getSize( + new StreamProgress( + testToken, tenSecondsAgo, BigDecimal.valueOf(20), Instant.now(), false)); // we should have output 2 100B mutations in the past 10s long bytesPerSecond = (mutationSize * 2) / 10; - assertEquals(sizeEstimate, bytesPerSecond * watermarkLag, .1d); + assertEquals(sizeEstimate, bytesPerSecond * watermarkLag, 10); verify(receiver, times(2)).outputWithTimestamp(KV.of(rowKey, mockMutation), Instant.EPOCH); } + + @Test + public void testGetSizeCantBeNegative() throws IOException { + long mutationSize = 100L; + when(sizeEstimator.sizeOf(any())).thenReturn(mutationSize); + Range.ByteStringRange partitionRange = Range.ByteStringRange.create("", ""); + ChangeStreamContinuationToken testToken = + ChangeStreamContinuationToken.create(partitionRange, "test"); + doFn.setup(); + + double mutationEstimate = + doFn.getSize( + new StreamProgress( + testToken, + Instant.now().plus(Duration.standardMinutes(10)), + BigDecimal.valueOf(1000), + Instant.now().plus(Duration.standardMinutes(10)), + false)); + assertEquals(0, mutationEstimate, 0); + + double heartbeatEstimate = + doFn.getSize( + new StreamProgress( + testToken, + Instant.now().plus(Duration.standardMinutes(10)), + BigDecimal.valueOf(1000), + Instant.now().plus(Duration.standardMinutes(10)), + true)); + assertEquals(0, heartbeatEstimate, 0); + } } diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/BytesThroughputEstimatorTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/BytesThroughputEstimatorTest.java index e10f730720668..90dd863bba106 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/BytesThroughputEstimatorTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/BytesThroughputEstimatorTest.java @@ -29,113 +29,79 @@ import java.util.Comparator; import java.util.List; import java.util.concurrent.ThreadLocalRandom; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.beam.repackaged.core.org.apache.commons.compress.utils.IOUtils; import org.apache.beam.repackaged.core.org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.beam.sdk.coders.Coder; -import org.apache.beam.sdk.io.gcp.bigtable.changestreams.TimestampConverter; import org.joda.time.Instant; -import org.junit.Before; import org.junit.Test; public class BytesThroughputEstimatorTest { private static final double DELTA = 1e-10; - private static final int WINDOW_SIZE_SECONDS = 10; - private BytesThroughputEstimator estimator; - - @Before - public void setup() { - final SizeEstimator sizeEstimator = new SizeEstimator<>(new TestCoder()); - estimator = new BytesThroughputEstimator<>(WINDOW_SIZE_SECONDS, sizeEstimator, 1); - } + private final SizeEstimator sizeEstimator = new CoderSizeEstimator<>(new TestCoder()); @Test public void testThroughputIsZeroWhenNothingsBeenRegistered() { - assertEquals(0D, estimator.get(), DELTA); - assertEquals(0D, estimator.getFrom(Instant.now()), DELTA); + BytesThroughputEstimator estimator = + new BytesThroughputEstimator<>(sizeEstimator, Instant.now()); + assertEquals(0D, estimator.get().doubleValue(), DELTA); } @Test public void testThroughputCalculation() { + BytesThroughputEstimator estimator = + new BytesThroughputEstimator<>(sizeEstimator, 1, Instant.ofEpochSecond(0)); estimator.update(Instant.ofEpochSecond(2), new byte[10]); estimator.update(Instant.ofEpochSecond(3), new byte[20]); estimator.update(Instant.ofEpochSecond(5), new byte[30]); estimator.update(Instant.ofEpochSecond(10), new byte[40]); // (10 + 20 + 30 + 40) / 10 sec window = 10 - assertEquals(10D, estimator.getFrom(Instant.ofEpochSecond(11)), DELTA); - - estimator.update(Instant.ofEpochSecond(20), new byte[10]); - estimator.update(Instant.ofEpochSecond(21), new byte[20]); - estimator.update(Instant.ofEpochSecond(21), new byte[10]); - estimator.update(Instant.ofEpochSecond(29), new byte[40]); + assertEquals(10D, estimator.get().doubleValue(), DELTA); + + BytesThroughputEstimator estimator2 = + new BytesThroughputEstimator<>(sizeEstimator, 1, Instant.ofEpochSecond(20)); + estimator2.update(Instant.ofEpochSecond(21), new byte[10]); + estimator2.update(Instant.ofEpochSecond(22), new byte[20]); + estimator2.update(Instant.ofEpochSecond(23), new byte[10]); + estimator2.update(Instant.ofEpochSecond(30), new byte[40]); // (10 + 20 + 10 + 40) / 10 sec window = 8 - assertEquals(8D, estimator.getFrom(Instant.ofEpochSecond(30)), DELTA); + assertEquals(8D, estimator2.get().doubleValue(), DELTA); - estimator.update(Instant.ofEpochSecond(31), new byte[10]); - estimator.update(Instant.ofEpochSecond(35), new byte[40]); + BytesThroughputEstimator estimator3 = + new BytesThroughputEstimator<>(sizeEstimator, 1, Instant.ofEpochSecond(30)); + estimator3.update(Instant.ofEpochSecond(31), new byte[10]); + estimator3.update(Instant.ofEpochSecond(40), new byte[40]); // (10 + 40) / 10 sec window = 5 - assertEquals(5D, estimator.getFrom(Instant.ofEpochSecond(41)), DELTA); - - // No values in the past 10 seconds - assertEquals(0D, estimator.getFrom(Instant.ofEpochSecond(50)), DELTA); + assertEquals(5D, estimator3.get().doubleValue(), DELTA); } @Test public void testThroughputIsAccumulatedWithin60SecondsWindow() { - List> pairs = generateTestData(100, 0, 10); + List> pairs = generateTestData(100, 0, 11); pairs.sort(Comparator.comparing(ImmutablePair::getLeft)); - final Instant lastUpdateTimestamp = pairs.get(pairs.size() - 1).getLeft(); BigDecimal sum = BigDecimal.valueOf(0L); for (ImmutablePair pair : pairs) { sum = sum.add(BigDecimal.valueOf(pair.getRight().length)); } - final BigDecimal want = - sum.divide(BigDecimal.valueOf(WINDOW_SIZE_SECONDS), MathContext.DECIMAL128); + final BigDecimal want = sum.divide(BigDecimal.valueOf(10), MathContext.DECIMAL128); + BytesThroughputEstimator estimator = + new BytesThroughputEstimator<>(sizeEstimator, 1, Instant.ofEpochSecond(0)); for (ImmutablePair pair : pairs) { estimator.update(pair.getLeft(), pair.getRight()); } - double actual = estimator.getFrom(Instant.ofEpochSecond(10)); - assertEquals(want.doubleValue(), actual, DELTA); - - // After window without updates the throughput should be zero - final Instant afterWindowTimestamp = - Instant.ofEpochSecond( - TimestampConverter.toSeconds(lastUpdateTimestamp) + WINDOW_SIZE_SECONDS + 1); - assertEquals(0D, estimator.getFrom(afterWindowTimestamp), DELTA); + double actual = estimator.get().doubleValue(); + assertEquals(want.doubleValue(), actual, 1); } @Test - public void testThroughputIsAccumulatedWithin50SecondsWindow() { - final List> excludedPairs = generateTestData(300, 0, 10); - final List> expectedPairs = generateTestData(50, 10, 20); - final List> pairs = - Stream.concat(excludedPairs.stream(), expectedPairs.stream()) - .sorted(Comparator.comparing(ImmutablePair::getLeft)) - .collect(Collectors.toList()); - final Instant lastUpdateTimestamp = pairs.get(pairs.size() - 1).getLeft(); - - BigDecimal sum = BigDecimal.valueOf(0L); - for (ImmutablePair pair : expectedPairs) { - sum = sum.add(BigDecimal.valueOf(pair.getRight().length)); - } - final BigDecimal want = - sum.divide(BigDecimal.valueOf(WINDOW_SIZE_SECONDS), MathContext.DECIMAL128); - for (ImmutablePair pair : pairs) { - estimator.update(pair.getLeft(), pair.getRight()); - } - - double actual = estimator.getFrom(Instant.ofEpochSecond(20)); - assertEquals(want.doubleValue(), actual, DELTA); - - // After window without updates the throughput should be zero - final Instant afterWindowTimestamp = - Instant.ofEpochSecond( - TimestampConverter.toSeconds(lastUpdateTimestamp) + WINDOW_SIZE_SECONDS + 1); - assertEquals(0D, estimator.getFrom(afterWindowTimestamp), DELTA); + public void testThroughputHandlesNoTimeDifference() { + BytesThroughputEstimator estimator = + new BytesThroughputEstimator<>(sizeEstimator, 1, Instant.ofEpochSecond(0)); + estimator.update(Instant.ofEpochSecond(0), new byte[10]); + // (10 / 1) * 1000 because we round up to one millisecond + assertEquals(10000D, estimator.get().doubleValue(), DELTA); } private List> generateTestData( diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/NullSizeEstimatorTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/NullSizeEstimatorTest.java new file mode 100644 index 0000000000000..a229c0e66bef3 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/NullSizeEstimatorTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class NullSizeEstimatorTest { + private static final double DELTA = 1e-10; + + @Test + public void alwaysReturns0AsEstimatedThroughput() { + final NullSizeEstimator estimator = new NullSizeEstimator<>(); + assertEquals(estimator.sizeOf(new byte[40]), 0D, DELTA); + assertEquals(estimator.sizeOf(new byte[20]), 0D, DELTA); + assertEquals(estimator.sizeOf(new byte[10]), 0D, DELTA); + } +} diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/NullThroughputEstimatorTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/NullThroughputEstimatorTest.java deleted file mode 100644 index 266dc1255e529..0000000000000 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/NullThroughputEstimatorTest.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io.gcp.bigtable.changestreams.estimator; - -import static org.junit.Assert.assertEquals; - -import org.joda.time.Instant; -import org.junit.Test; - -public class NullThroughputEstimatorTest { - private static final double DELTA = 1e-10; - - @Test - public void alwaysReturns0AsEstimatedThroughput() { - final NullThroughputEstimator estimator = new NullThroughputEstimator<>(); - assertEquals(estimator.get(), 0D, DELTA); - - estimator.update(Instant.ofEpochSecond(1), new byte[10]); - assertEquals(estimator.getFrom(Instant.ofEpochSecond(1)), 0D, DELTA); - estimator.update(Instant.ofEpochSecond(2), new byte[20]); - assertEquals(estimator.getFrom(Instant.ofEpochSecond(2)), 0D, DELTA); - } -} diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconcilerTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconcilerTest.java index aab2bdc8f64e8..dd6a09c9cd87d 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconcilerTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconcilerTest.java @@ -20,6 +20,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; @@ -54,8 +55,8 @@ public class PartitionReconcilerTest { @ClassRule public static final BigtableEmulatorRule BIGTABLE_EMULATOR_RULE = BigtableEmulatorRule.create(); - private static final Duration MORE_THAN_ONE_MINUTE = Duration.standardSeconds(61); - private static final Duration MORE_THAN_TEN_MINUTES = Duration.standardSeconds(10 * 60 + 1); + private static final Duration MISSING_SHORT_PERIOD = Duration.standardSeconds(2 * 60 + 1); + private static final Duration MISSING_LONG_PERIOD = Duration.standardSeconds(20 * 60 + 1); private MetadataTableDao metadataTableDao; @@ -90,6 +91,7 @@ public void setUp() throws Exception { new MetadataTableAdminDao( adminClient, null, changeStreamId, MetadataTableAdminDao.DEFAULT_METADATA_TABLE_NAME); metadataTableAdminDao.createMetadataTable(); + metadataTableAdminDao.cleanUpPrefix(); metadataTableDao = new MetadataTableDao( dataClient, @@ -135,7 +137,7 @@ public void testLongMissingMergePartitionIsReconciled() { // Artificially create that partitionAB has been missing for more than 1 minute. HashMap missingPartitionDurations = new HashMap<>(); - missingPartitionDurations.put(partitionAB, Instant.now().minus(MORE_THAN_ONE_MINUTE)); + missingPartitionDurations.put(partitionAB, Instant.now().minus(MISSING_SHORT_PERIOD)); metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations); PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao, metrics); @@ -174,7 +176,7 @@ public void testMismatchedMergePartitionIsReconciled() { ChangeStreamContinuationToken.create(ByteStringRange.create("C", "D"), "CD"); // Artificially create that partitionAD has been missing for more than 10 minutes. HashMap missingPartitionDurations = new HashMap<>(); - missingPartitionDurations.put(partitionAD, Instant.now().minus(MORE_THAN_TEN_MINUTES)); + missingPartitionDurations.put(partitionAD, Instant.now().minus(MISSING_LONG_PERIOD)); metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations); PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao, metrics); @@ -228,7 +230,7 @@ public void testMismatchedMergeSplitPartitionIsReconciled() { // Artificially create that partitionAC has been missing for more than 10 minutes. HashMap missingPartitionDurations = new HashMap<>(); - missingPartitionDurations.put(partitionAC, Instant.now().minus(MORE_THAN_TEN_MINUTES)); + missingPartitionDurations.put(partitionAC, Instant.now().minus(MISSING_LONG_PERIOD)); metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations); PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao, metrics); @@ -263,7 +265,7 @@ public void testMissingPartitionWithoutToken() { // Artificially create that partitionAB has been missing for more than 10 minutes. HashMap missingPartitionDurations = new HashMap<>(); - missingPartitionDurations.put(partitionAB, Instant.now().minus(MORE_THAN_TEN_MINUTES)); + missingPartitionDurations.put(partitionAB, Instant.now().minus(MISSING_LONG_PERIOD)); metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations); PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao, metrics); @@ -276,10 +278,11 @@ public void testMissingPartitionWithoutToken() { new PartitionRecord(partitionAB, startTime, lowWatermark, Collections.emptyList()); assertEquals(1, reconciledPartitions.size()); assertEquals(expectedRecord, reconciledPartitions.get(0)); + assertTrue(metadataTableDao.readDetectNewPartitionMissingPartitions().isEmpty()); } - // We're missing AD but we only have partition AB and CD. We should reconcile by outputting AB and - // CD and then create a new partition for BC with start_time = low watermark - 1 hour + // We're missing AD, but we only have partition AB and CD. We should reconcile by outputting AB + // and CD and then create a new partition for BC with start_time = low watermark - 1 hour @Test public void testMissingPartitionWithSomeToken() { ByteStringRange partitionAD = ByteStringRange.create("A", "D"); @@ -291,7 +294,7 @@ public void testMissingPartitionWithSomeToken() { // Artificially create that partitionAD has been missing for more than 10 minutes. HashMap missingPartitionDurations = new HashMap<>(); - missingPartitionDurations.put(partitionAD, Instant.now().minus(MORE_THAN_TEN_MINUTES)); + missingPartitionDurations.put(partitionAD, Instant.now().minus(MISSING_LONG_PERIOD)); metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations); PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao, metrics); @@ -333,4 +336,41 @@ public void testMissingPartitionWithSomeToken() { containsInAnyOrder( Arrays.asList(expectedRecordAB, expectedRecordBC, expectedRecordCD).toArray())); } + + // A partition that's missing for more than 10 minutes (DNP could have not run for more than 10 + // minutes) has continuation token should get reconciled with token. + @Test + public void testMissingPartitionWithTokenMoreThan10Minutes() { + ByteStringRange partitionAD = ByteStringRange.create("A", "D"); + ByteStringRange partitionAB = ByteStringRange.create("A", "B"); + ChangeStreamContinuationToken tokenAB = ChangeStreamContinuationToken.create(partitionAB, "AB"); + + // Artificially create that partitionAB has been missing for more than 10 minutes. + HashMap missingPartitionDurations = new HashMap<>(); + missingPartitionDurations.put(partitionAB, Instant.now().minus(MISSING_LONG_PERIOD)); + metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations); + + PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao, metrics); + partitionReconciler.addMissingPartitions(Collections.singletonList(partitionAB)); + + NewPartition newPartitionAD = + new NewPartition(partitionAD, Collections.singletonList(tokenAB), Instant.now()); + partitionReconciler.addIncompleteNewPartitions(newPartitionAD); + + List reconciledPartitions = + partitionReconciler.getPartitionsToReconcile(lowWatermark, startTime); + assertTrue(metadataTableDao.readDetectNewPartitionMissingPartitions().isEmpty()); + assertEquals(1, reconciledPartitions.size()); + + NewPartition newPartitionADWithAB = + new NewPartition( + partitionAD, Collections.singletonList(tokenAB), newPartitionAD.getLowWatermark()); + PartitionRecord expectedRecordAB = + new PartitionRecord( + partitionAB, + Collections.singletonList(tokenAB), + lowWatermark, + Collections.singletonList(newPartitionADWithAB)); + assertEquals(reconciledPartitions, Collections.singletonList(expectedRecordAB)); + } } diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTrackerTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTrackerTest.java index b0bc8de7c8114..e2f0cbb62a614 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTrackerTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTrackerTest.java @@ -26,6 +26,7 @@ import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken; import com.google.cloud.bigtable.data.v2.models.Range; +import java.math.BigDecimal; import org.apache.beam.sdk.transforms.splittabledofn.SplitResult; import org.joda.time.Instant; import org.junit.Test; @@ -41,7 +42,8 @@ public void testTryClaim() { ChangeStreamContinuationToken changeStreamContinuationToken = ChangeStreamContinuationToken.create(Range.ByteStringRange.create("a", "b"), "1234"); final StreamProgress streamProgress2 = - new StreamProgress(changeStreamContinuationToken, Instant.now()); + new StreamProgress( + changeStreamContinuationToken, Instant.now(), BigDecimal.ONE, Instant.now(), false); assertTrue(tracker.tryClaim(streamProgress2)); assertEquals(streamProgress2, tracker.currentRestriction()); assertEquals( diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/RampupThrottlingFnTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/RampupThrottlingFnTest.java index 9eb3c2b66d346..11022758543b1 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/RampupThrottlingFnTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/RampupThrottlingFnTest.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.Sleeper; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.DateTimeUtils; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/BaseFirestoreV1WriteFnTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/BaseFirestoreV1WriteFnTest.java index 16c90af36326f..d4fcf6153e470 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/BaseFirestoreV1WriteFnTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/BaseFirestoreV1WriteFnTest.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.gcp.firestore; import static org.apache.beam.sdk.io.gcp.firestore.FirestoreProtoHelpers.newWrite; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -72,7 +72,7 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Before; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnBatchGetDocumentsTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnBatchGetDocumentsTest.java index a2040cf59541c..b9c950e92fd53 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnBatchGetDocumentsTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnBatchGetDocumentsTest.java @@ -38,9 +38,9 @@ import java.util.Collection; import java.util.List; import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1ReadFn.BatchGetDocumentsFn; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.AbstractIterator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.AbstractIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Instant; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnBatchWriteWithDeadLetterQueueTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnBatchWriteWithDeadLetterQueueTest.java index 91ac22254bf38..d59b9354bd8b2 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnBatchWriteWithDeadLetterQueueTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnBatchWriteWithDeadLetterQueueTest.java @@ -50,7 +50,7 @@ import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1WriteFn.BatchWriteFnWithDeadLetterQueue; import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1WriteFn.WriteElement; import org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcWriteAttempt.Element; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.mockito.ArgumentCaptor; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnBatchWriteWithSummaryTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnBatchWriteWithSummaryTest.java index c5733dabf5523..9acc3707e3ba0 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnBatchWriteWithSummaryTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnBatchWriteWithSummaryTest.java @@ -52,7 +52,7 @@ import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1WriteFn.BatchWriteFnWithSummary; import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1WriteFn.WriteElement; import org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcWriteAttempt.Element; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Instant; import org.junit.After; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnListCollectionIdsTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnListCollectionIdsTest.java index 0227af9b6231e..eb3cd2692c8e2 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnListCollectionIdsTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnListCollectionIdsTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.firestore; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -37,8 +37,8 @@ import java.util.Iterator; import java.util.List; import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1ReadFn.ListCollectionIdsFn; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.AbstractIterator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.AbstractIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnListDocumentsTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnListDocumentsTest.java index 91fef77809e6b..2faa7c3e2f1b4 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnListDocumentsTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnListDocumentsTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.firestore; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -39,9 +39,9 @@ import java.util.Iterator; import java.util.List; import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1ReadFn.ListDocumentsFn; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.AbstractIterator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.AbstractIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Instant; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnPartitionQueryTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnPartitionQueryTest.java index d0349733765db..d6c69fbd96b21 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnPartitionQueryTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnPartitionQueryTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.firestore; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -40,8 +40,8 @@ import java.util.List; import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1ReadFn.PartitionQueryFn; import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1ReadFn.PartitionQueryPair; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.AbstractIterator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.AbstractIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnRunQueryTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnRunQueryTest.java index aead53b4aeceb..02e5f9743eaae 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnRunQueryTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/FirestoreV1FnRunQueryTest.java @@ -51,9 +51,9 @@ import java.util.List; import java.util.function.Function; import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1ReadFn.RunQueryFn; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.AbstractIterator; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.AbstractIterator; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Instant; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/PartitionQueryResponseToRunQueryRequestTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/PartitionQueryResponseToRunQueryRequestTest.java index c2ad0ea469046..c6df81f20e7e8 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/PartitionQueryResponseToRunQueryRequestTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/PartitionQueryResponseToRunQueryRequestTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.firestore; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.when; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/QueryUtilsTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/QueryUtilsTest.java index ce5ac6dd849d2..3a60381c11ddf 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/QueryUtilsTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/QueryUtilsTest.java @@ -34,8 +34,8 @@ import com.google.firestore.v1.StructuredQuery.UnaryFilter; import com.google.firestore.v1.Value; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Before; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/RpcQosOptionsTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/RpcQosOptionsTest.java index d9592fa88a3ad..997d94b3d2bf1 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/RpcQosOptionsTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/RpcQosOptionsTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.firestore; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/RpcQosTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/RpcQosTest.java index e1e1306147289..2f3724d6bae72 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/RpcQosTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/RpcQosTest.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.gcp.firestore; import static org.apache.beam.sdk.io.gcp.firestore.FirestoreProtoHelpers.newWrite; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertSame; @@ -65,7 +65,7 @@ import org.apache.beam.sdk.metrics.Distribution; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Before; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/it/BaseFirestoreIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/it/BaseFirestoreIT.java index 551d4e3d7a87f..509797892e04c 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/it/BaseFirestoreIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/it/BaseFirestoreIT.java @@ -55,11 +55,10 @@ import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.joda.time.Instant; import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; @@ -93,12 +92,10 @@ abstract class BaseFirestoreIT { .build(); protected static String project; - protected GcpOptions options; @Before public void setup() { - options = TestPipeline.testingPipelineOptions().as(GcpOptions.class); - project = options.getProject(); + project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); } private static Instant toWriteTime(WriteResult result) { @@ -167,7 +164,7 @@ public final void listCollections() throws Exception { .build()); PAssert.that(actualCollectionIds).containsInAnyOrder(allCollectionIds); - testPipeline.run(options); + testPipeline.run(TestPipeline.testingPipelineOptions()); // Reading from readTime should only get collection IDs written in the batch before readTime. PCollection actualCollectionIdsAtReadTime = @@ -182,12 +179,10 @@ public final void listCollections() throws Exception { .withRpcQosOptions(RPC_QOS_OPTIONS) .build()); PAssert.that(actualCollectionIdsAtReadTime).containsInAnyOrder(collectionIds); - testPipeline2.run(options); + testPipeline2.run(TestPipeline.testingPipelineOptions()); } @Test - @Ignore( - "https://github.com/apache/beam/issues/25851 failing due to internal Firestore breaking change") public final void listDocuments() throws Exception { DocumentGenerator documentGenerator = helper.documentGenerator(NUM_ITEMS_TO_GENERATE, "a"); Instant readTime = @@ -215,7 +210,7 @@ public final void listDocuments() throws Exception { .apply(ParDo.of(new DocumentToName())); PAssert.that(listDocumentPaths).containsInAnyOrder(allDocumentPaths); - testPipeline.run(options); + testPipeline.run(TestPipeline.testingPipelineOptions()); // Reading from readTime should only get the documents written before readTime. PCollection listDocumentPathsAtReadTime = @@ -233,7 +228,7 @@ public final void listDocuments() throws Exception { PAssert.that(listDocumentPathsAtReadTime) .containsInAnyOrder(documentGenerator.expectedDocumentPaths()); - testPipeline2.run(options); + testPipeline2.run(TestPipeline.testingPipelineOptions()); } @Test @@ -267,7 +262,7 @@ public final void runQuery() throws Exception { .apply(ParDo.of(new DocumentToName())); PAssert.that(listDocumentPaths).containsInAnyOrder(allDocumentPaths); - testPipeline.run(options); + testPipeline.run(TestPipeline.testingPipelineOptions()); // Reading from readTime should only get the documents written before readTime. PCollection listDocumentPathsAtReadTime = @@ -286,7 +281,7 @@ public final void runQuery() throws Exception { PAssert.that(listDocumentPathsAtReadTime) .containsInAnyOrder(documentGenerator.expectedDocumentPaths()); - testPipeline2.run(options); + testPipeline2.run(TestPipeline.testingPipelineOptions()); } @Test @@ -326,7 +321,7 @@ public final void partitionQuery() throws Exception { .apply(ParDo.of(new DocumentToName())); PAssert.that(listDocumentPaths).containsInAnyOrder(allDocumentPaths); - testPipeline.run(options); + testPipeline.run(TestPipeline.testingPipelineOptions()); // Reading from readTime should only get the documents written before readTime. PCollection listDocumentPathsAtReadTime = @@ -346,7 +341,7 @@ public final void partitionQuery() throws Exception { PAssert.that(listDocumentPathsAtReadTime) .containsInAnyOrder(documentGenerator.expectedDocumentPaths()); - testPipeline2.run(options); + testPipeline2.run(TestPipeline.testingPipelineOptions()); } @Test @@ -390,7 +385,7 @@ public final void batchGet() throws Exception { .apply(ParDo.of(new DocumentToName())); PAssert.that(listDocumentPaths).containsInAnyOrder(allDocumentPaths); - testPipeline.run(options); + testPipeline.run(TestPipeline.testingPipelineOptions()); // Reading from readTime should only get the documents written before readTime. PCollection listDocumentPathsAtReadTime = @@ -410,7 +405,7 @@ public final void batchGet() throws Exception { PAssert.that(listDocumentPathsAtReadTime) .containsInAnyOrder(documentGenerator.expectedDocumentPaths()); - testPipeline2.run(options); + testPipeline2.run(TestPipeline.testingPipelineOptions()); } @Test @@ -448,7 +443,7 @@ protected final void runWriteTest( .apply(createWrite) .apply(FirestoreIO.v1().write().batchWrite().withRpcQosOptions(RPC_QOS_OPTIONS).build()); - testPipeline.run(options); + testPipeline.run(TestPipeline.testingPipelineOptions()); List actualDocumentIds = helper diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/it/FirestoreTestingHelper.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/it/FirestoreTestingHelper.java index 90aba760b7f69..d8c55d44f3c8e 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/it/FirestoreTestingHelper.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/it/FirestoreTestingHelper.java @@ -65,10 +65,10 @@ import java.util.stream.StreamSupport; import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Streams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Streams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors; import org.junit.rules.TestRule; import org.junit.runner.Description; import org.junit.runners.model.Statement; @@ -104,7 +104,7 @@ enum DataLayout { DataLayout value() default DataLayout.Shallow; } - private final GcpOptions options; + private final GcpOptions gcpOptions; private final org.apache.beam.sdk.io.gcp.firestore.FirestoreOptions firestoreBeamOptions; private final FirestoreOptions firestoreOptions; @@ -125,17 +125,17 @@ enum DataLayout { "initialization.fields.uninitialized") // testClass and testName are managed via #apply public FirestoreTestingHelper(CleanupMode cleanupMode) { this.cleanupMode = cleanupMode; - options = TestPipeline.testingPipelineOptions().as(GcpOptions.class); + gcpOptions = TestPipeline.testingPipelineOptions().as(GcpOptions.class); firestoreBeamOptions = TestPipeline.testingPipelineOptions() .as(org.apache.beam.sdk.io.gcp.firestore.FirestoreOptions.class); firestoreOptions = FirestoreOptions.newBuilder() - .setCredentials(options.getGcpCredential()) - .setProjectId(options.getProject()) + .setCredentials(gcpOptions.getGcpCredential()) + .setProjectId(gcpOptions.getProject()) .setDatabaseId(firestoreBeamOptions.getFirestoreDb()) + .setHost(firestoreBeamOptions.getHost()) .build(); - fs = firestoreOptions.getService(); rpc = (FirestoreRpc) firestoreOptions.getRpc(); } diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/it/FirestoreV1IT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/it/FirestoreV1IT.java index e259021be1306..204aa67619bd6 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/it/FirestoreV1IT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/firestore/it/FirestoreV1IT.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.gcp.firestore.it; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -55,13 +55,14 @@ import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1.WriteFailure; import org.apache.beam.sdk.io.gcp.firestore.RpcQosOptions; import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; public final class FirestoreV1IT extends BaseFirestoreIT { @@ -133,7 +134,7 @@ public void batchWrite_partialFailureOutputsToDeadLetterQueue() assertFalse(iterator.hasNext()); return null; }); - testPipeline.run(this.options); + testPipeline.run(TestPipeline.testingPipelineOptions()); ApiFuture actualDocsQuery = helper.getBaseDocument().collection(collectionId).orderBy("__name__").get(); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/DicomIOReadIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/DicomIOReadIT.java index d594a13fb00e6..e550b29541e5f 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/DicomIOReadIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/DicomIOReadIT.java @@ -61,7 +61,7 @@ public void deleteDicomStore() throws IOException { client.deleteDicomStore(healthcareDataset + "/dicomStores/" + storeName); } - @Ignore("https://github.com/apache/beam/issues/20644") + @Ignore("https://github.com/apache/beam/issues/28099") @Test public void testDicomMetadataRead() throws IOException { String webPath = diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOReadIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOReadIT.java index 142b26dd2cdfe..8dedce4b6fdba 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOReadIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOReadIT.java @@ -34,7 +34,7 @@ import org.apache.beam.sdk.io.gcp.pubsub.TestPubsubSignal; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; import org.joda.time.Duration; import org.junit.After; import org.junit.Before; @@ -133,7 +133,7 @@ public void testFhirIORead() throws Exception { "waitForAnyMessage", signal.signalSuccessWhen(resources.getCoder(), anyResources -> true)); // wait for any resource - Supplier start = signal.waitForStart(Duration.standardMinutes(5)); + Supplier start = signal.waitForStart(Duration.standardMinutes(8)); pipeline.apply(signal.signalStart()); PipelineResult job = pipeline.run(); start.get(); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java index 682a7ce11f008..fc92a568ba4d6 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java @@ -40,9 +40,10 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -126,6 +127,7 @@ public void teardown() throws IOException { } } + @Ignore("https://github.com/apache/beam/issues/28505") @Test public void testFhirIOSearch() { pipeline.getOptions().as(DirectOptions.class).setBlockOnRun(false); @@ -155,6 +157,7 @@ public void testFhirIOSearch() { pipeline.run().waitUntilFinish(); } + @Ignore("https://github.com/apache/beam/issues/28505") @Test public void testFhirIOSearchWithGenericParameters() { pipeline.getOptions().as(DirectOptions.class).setBlockOnRun(false); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/NestedRowToMessageTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/NestedRowToMessageTest.java index af2cdae176c21..579841bf47e20 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/NestedRowToMessageTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/NestedRowToMessageTest.java @@ -32,8 +32,8 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PreparePubsubWriteDoFnTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PreparePubsubWriteDoFnTest.java index ae0cbb82727cb..494189d43f362 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PreparePubsubWriteDoFnTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PreparePubsubWriteDoFnTest.java @@ -26,7 +26,7 @@ import java.util.Map; import javax.naming.SizeLimitExceededException; import org.apache.beam.repackaged.core.org.apache.commons.lang3.RandomStringUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClientTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClientTest.java index 8defeec289f3f..6461af10a220b 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClientTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClientTest.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.SchemaPath; import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.SubscriptionPath; import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Instant; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubGrpcClientTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubGrpcClientTest.java index 022608c87d803..3724e169c6122 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubGrpcClientTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubGrpcClientTest.java @@ -57,9 +57,9 @@ import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath; import org.apache.beam.sdk.schemas.Schema.Field; import org.apache.beam.sdk.schemas.Schema.FieldType; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.After; import org.junit.Before; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIOExternalTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIOExternalTest.java index 620f5b228067d..020d522b5e2ed 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIOExternalTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIOExternalTest.java @@ -39,7 +39,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.hamcrest.Matchers; import org.hamcrest.text.MatchesPattern; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIOTest.java index b8b71a0080121..f7f9f5f91b74d 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIOTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIOTest.java @@ -51,10 +51,10 @@ import org.apache.beam.sdk.coders.CoderException; import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.extensions.avro.coders.AvroCoder; +import org.apache.beam.sdk.extensions.avro.io.AvroGeneratedUser; import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages.Primitive; import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; import org.apache.beam.sdk.extensions.protobuf.ProtoDomain; -import org.apache.beam.sdk.io.AvroGeneratedUser; import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.IncomingMessage; import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.OutgoingMessage; import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.SubscriptionPath; @@ -76,9 +76,9 @@ import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubJsonClientTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubJsonClientTest.java index 5de8f68aee82f..634ad42c937ae 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubJsonClientTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubJsonClientTest.java @@ -50,8 +50,8 @@ import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.ProjectPath; import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.SubscriptionPath; import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageToRowTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageToRowTest.java index bbde5f9c653bb..4da6bdefbd0a3 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageToRowTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageToRowTest.java @@ -22,7 +22,7 @@ import static org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.DLQ_TAG; import static org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.MAIN_TAG; import static org.apache.beam.sdk.io.gcp.pubsub.PubsubSchemaIOProvider.ATTRIBUTE_ARRAY_ENTRY_SCHEMA; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables.size; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.size; import static org.junit.Assert.assertEquals; import java.io.Serializable; @@ -42,9 +42,9 @@ import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.joda.time.DateTime; import org.joda.time.Instant; import org.junit.Assert; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesAndMessageIdAndOrderingKeyCoderTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesAndMessageIdAndOrderingKeyCoderTest.java index 15d4a75f1793a..769117e9a6cce 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesAndMessageIdAndOrderingKeyCoderTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesAndMessageIdAndOrderingKeyCoderTest.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesAndMessageIdCoderTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesAndMessageIdCoderTest.java index 1967bb1e4c154..3a2a1a671eeb5 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesAndMessageIdCoderTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesAndMessageIdCoderTest.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesCoderTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesCoderTest.java index 67267a9e26851..67b3675ec9043 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesCoderTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageWithAttributesCoderTest.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadIT.java index 7b370ebf7e2c1..9faa1f0847e68 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadIT.java @@ -23,8 +23,8 @@ import org.apache.beam.sdk.testing.TestPipelineOptions; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; import org.joda.time.Duration; import org.junit.Rule; import org.junit.Test; @@ -51,7 +51,7 @@ public void testReadPublicData() throws Exception { messages.apply( "waitForAnyMessage", signal.signalSuccessWhen(messages.getCoder(), anyMessages -> true)); - Supplier start = signal.waitForStart(Duration.standardMinutes(5)); + Supplier start = signal.waitForStart(Duration.standardMinutes(8)); pipeline.apply(signal.signalStart()); PipelineResult job = pipeline.run(); start.get(); @@ -79,7 +79,7 @@ public void testReadPubsubMessageId() throws Exception { "isMessageIdNonNull", signal.signalSuccessWhen(messages.getCoder(), new NonEmptyMessageIdCheck())); - Supplier start = signal.waitForStart(Duration.standardMinutes(5)); + Supplier start = signal.waitForStart(Duration.standardMinutes(8)); pipeline.apply(signal.signalStart()); PipelineResult job = pipeline.run(); start.get(); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadSchemaTransformProviderTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadSchemaTransformProviderTest.java index aaceda5342db2..dd5a9abd5ac8e 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadSchemaTransformProviderTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubReadSchemaTransformProviderTest.java @@ -18,313 +18,343 @@ package org.apache.beam.sdk.io.gcp.pubsub; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import com.google.api.client.util.Clock; -import com.google.gson.Gson; -import com.google.gson.JsonObject; -import com.google.gson.JsonPrimitive; import com.google.protobuf.ByteString; import com.google.protobuf.Timestamp; import java.io.IOException; import java.io.Serializable; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.UUID; import java.util.stream.Collectors; +import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.extensions.avro.schemas.io.payloads.AvroPayloadSerializerProvider; -import org.apache.beam.sdk.schemas.AutoValueSchema; +import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils; +import org.apache.beam.sdk.io.gcp.pubsub.PubsubTestClient.PubsubTestClientFactory; +import org.apache.beam.sdk.metrics.MetricNameFilter; +import org.apache.beam.sdk.metrics.MetricQueryResults; +import org.apache.beam.sdk.metrics.MetricResult; +import org.apache.beam.sdk.metrics.MetricResults; +import org.apache.beam.sdk.metrics.MetricsFilter; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer; import org.apache.beam.sdk.schemas.transforms.SchemaTransform; import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; -import org.apache.beam.sdk.values.TypeDescriptor; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -/** Tests for {@link PubsubReadSchemaTransformProvider}. */ +/** Tests for {@link org.apache.beam.sdk.io.gcp.pubsub.PubsubReadSchemaTransformProvider}. */ @RunWith(JUnit4.class) public class PubsubReadSchemaTransformProviderTest { - private static final Schema SCHEMA = + private static final Schema BEAM_SCHEMA = Schema.of( Schema.Field.of("name", Schema.FieldType.STRING), Schema.Field.of("number", Schema.FieldType.INT64)); - + private static final Schema BEAM_SCHEMA_WITH_ERROR = + Schema.of(Schema.Field.of("error", Schema.FieldType.STRING)); + private static final String SCHEMA = AvroUtils.toAvroSchema(BEAM_SCHEMA).toString(); private static final String SUBSCRIPTION = "projects/project/subscriptions/subscription"; private static final String TOPIC = "projects/project/topics/topic"; - private static final List cases = - Arrays.asList( - testCase( - "no configured topic or subscription", - PubsubReadSchemaTransformConfiguration.builder().setDataSchema(SCHEMA).build()) - .expectInvalidConfiguration(), - testCase( - "both topic and subscription configured", - PubsubReadSchemaTransformConfiguration.builder() - .setSubscription(SUBSCRIPTION) - .setSubscription(TOPIC) - .setDataSchema(SCHEMA) - .build()) - .expectInvalidConfiguration(), - testCase( - "invalid format configured", - PubsubReadSchemaTransformConfiguration.builder() - .setSubscription(SUBSCRIPTION) - .setDataSchema(SCHEMA) - .setFormat("invalidformat") - .build()) - .expectInvalidConfiguration(), - testCase( - "configuration with subscription", - PubsubReadSchemaTransformConfiguration.builder() - .setSubscription(SUBSCRIPTION) - .setDataSchema(SCHEMA) - .build()) - .withExpectedPubsubRead(PubsubIO.readMessages().fromSubscription(SUBSCRIPTION)), - testCase( - "configuration with topic", - PubsubReadSchemaTransformConfiguration.builder() - .setTopic(TOPIC) - .setDataSchema(SCHEMA) - .build()) - .withExpectedPubsubRead(PubsubIO.readMessages().fromTopic(TOPIC)), - testCase( - "configuration with subscription, timestamp and id attributes", - PubsubReadSchemaTransformConfiguration.builder() - .setSubscription(SUBSCRIPTION) - .setTimestampAttribute("timestampAttribute") - .setIdAttribute("idAttribute") - .setDataSchema(SCHEMA) - .build()) - .withExpectedPubsubRead( - PubsubIO.readMessages() - .fromSubscription(SUBSCRIPTION) - .withTimestampAttribute("timestampAttribute") - .withIdAttribute("idAttribute")), - testCase( - "configuration with subscription and dead letter queue", - PubsubReadSchemaTransformConfiguration.builder() - .setSubscription(SUBSCRIPTION) - .setDataSchema(SCHEMA) - .setDeadLetterQueue(TOPIC) - .build()) - .withExpectedPubsubRead(PubsubIO.readMessages().fromSubscription(SUBSCRIPTION)) - .withExpectedDeadLetterQueue(PubsubIO.writeMessages().to(TOPIC)), - testCase( - "configuration with subscription, timestamp attribute, and dead letter queue", - PubsubReadSchemaTransformConfiguration.builder() - .setSubscription(SUBSCRIPTION) - .setTimestampAttribute("timestampAttribute") - .setDataSchema(SCHEMA) - .setDeadLetterQueue(TOPIC) - .build()) - .withExpectedPubsubRead( - PubsubIO.readMessages() - .fromSubscription(SUBSCRIPTION) - .withTimestampAttribute("timestampAttribute")) - .withExpectedDeadLetterQueue( - PubsubIO.writeMessages().to(TOPIC).withTimestampAttribute("timestampAttribute"))); - - private static final AutoValueSchema AUTO_VALUE_SCHEMA = new AutoValueSchema(); - private static final TypeDescriptor TYPE_DESCRIPTOR = - TypeDescriptor.of(PubsubReadSchemaTransformConfiguration.class); - private static final SerializableFunction - ROW_SERIALIZABLE_FUNCTION = AUTO_VALUE_SCHEMA.toRowFunction(TYPE_DESCRIPTOR); - private static final List ROWS = Arrays.asList( - Row.withSchema(SCHEMA).withFieldValue("name", "a").withFieldValue("number", 100L).build(), - Row.withSchema(SCHEMA).withFieldValue("name", "b").withFieldValue("number", 200L).build(), - Row.withSchema(SCHEMA) + Row.withSchema(BEAM_SCHEMA) + .withFieldValue("name", "a") + .withFieldValue("number", 100L) + .build(), + Row.withSchema(BEAM_SCHEMA) + .withFieldValue("name", "b") + .withFieldValue("number", 200L) + .build(), + Row.withSchema(BEAM_SCHEMA) .withFieldValue("name", "c") .withFieldValue("number", 300L) .build()); - private static final Clock CLOCK = (Clock & Serializable) () -> 1656788475425L; + private static final List ROWSWITHERROR = + Arrays.asList( + Row.withSchema(BEAM_SCHEMA_WITH_ERROR).withFieldValue("error", "a").build(), + Row.withSchema(BEAM_SCHEMA_WITH_ERROR).withFieldValue("error", "b").build(), + Row.withSchema(BEAM_SCHEMA_WITH_ERROR).withFieldValue("error", "c").build()); + + private static final Clock CLOCK = (Clock & Serializable) () -> 1678988970000L; private static final AvroPayloadSerializerProvider AVRO_PAYLOAD_SERIALIZER_PROVIDER = new AvroPayloadSerializerProvider(); private static final PayloadSerializer AVRO_PAYLOAD_SERIALIZER = - AVRO_PAYLOAD_SERIALIZER_PROVIDER.getSerializer(SCHEMA, new HashMap<>()); + AVRO_PAYLOAD_SERIALIZER_PROVIDER.getSerializer(BEAM_SCHEMA, new HashMap<>()); + private static final PayloadSerializer AVRO_PAYLOAD_SERIALIZER_WITH_ERROR = + AVRO_PAYLOAD_SERIALIZER_PROVIDER.getSerializer(BEAM_SCHEMA_WITH_ERROR, new HashMap<>()); - @Rule public TestPipeline p = TestPipeline.create().enableAbandonedNodeEnforcement(false); + @Rule public transient TestPipeline p = TestPipeline.create(); @Test - public void testBuildDeadLetterQueueWrite() { - for (TestCase testCase : cases) { - PubsubIO.Write dlq = - testCase.pubsubReadSchemaTransform().buildDeadLetterQueueWrite(); - - if (testCase.expectedDeadLetterQueue == null) { - assertNull(testCase.name, dlq); - return; - } - - Map actual = DisplayData.from(dlq).asMap(); - Map expected = testCase.expectedDeadLetterQueue; - - assertEquals(testCase.name, expected, actual); - } + public void testInvalidConfigNoTopicOrSubscription() { + assertThrows( + IllegalArgumentException.class, + () -> + new PubsubReadSchemaTransformProvider() + .from( + PubsubReadSchemaTransformConfiguration.builder() + .setSchema(SCHEMA) + .setFormat("AVRO") + .build())); } @Test - public void testReadAvro() throws IOException { + public void testInvalidConfigBothTopicAndSubscription() { PCollectionRowTuple begin = PCollectionRowTuple.empty(p); - PubsubReadSchemaTransformProvider.PubsubReadSchemaTransform transform = - schemaTransformWithClock("avro"); - PubsubTestClient.PubsubTestClientFactory clientFactory = - clientFactory(incomingAvroMessagesOf(CLOCK.currentTimeMillis())); - transform.setClientFactory(clientFactory); - PCollectionRowTuple reads = begin.apply(transform.buildTransform()); - - PAssert.that(reads.get(PubsubReadSchemaTransformProvider.OUTPUT_TAG)).containsInAnyOrder(ROWS); - + assertThrows( + IllegalArgumentException.class, + () -> + begin.apply( + new PubsubReadSchemaTransformProvider() + .from( + PubsubReadSchemaTransformConfiguration.builder() + .setSchema(SCHEMA) + .setFormat("AVRO") + .setTopic(TOPIC) + .setSubscription(SUBSCRIPTION) + .build()))); p.run().waitUntilFinish(); - clientFactory.close(); } @Test - public void testReadJson() throws IOException { + public void testInvalidConfigInvalidFormat() { PCollectionRowTuple begin = PCollectionRowTuple.empty(p); - PubsubReadSchemaTransformProvider.PubsubReadSchemaTransform transform = - schemaTransformWithClock("json"); - PubsubTestClient.PubsubTestClientFactory clientFactory = - clientFactory(incomingJsonMessagesOf(CLOCK.currentTimeMillis())); - transform.setClientFactory(clientFactory); - PCollectionRowTuple reads = begin.apply(transform.buildTransform()); - - PAssert.that(reads.get(PubsubReadSchemaTransformProvider.OUTPUT_TAG)).containsInAnyOrder(ROWS); - + assertThrows( + IllegalArgumentException.class, + () -> + begin.apply( + new PubsubReadSchemaTransformProvider() + .from( + PubsubReadSchemaTransformConfiguration.builder() + .setSchema(SCHEMA) + .setFormat("BadFormat") + .setSubscription(SUBSCRIPTION) + .build()))); p.run().waitUntilFinish(); - - clientFactory.close(); } @Test - public void testBuildPubSubRead() { - for (TestCase testCase : cases) { - if (testCase.invalidConfigurationExpected) { - continue; - } - Map actual = - DisplayData.from(testCase.pubsubReadSchemaTransform().buildPubsubRead()).asMap(); + public void testNoSchema() { + PCollectionRowTuple begin = PCollectionRowTuple.empty(p); + assertThrows( + IllegalStateException.class, + () -> + begin.apply( + new PubsubReadSchemaTransformProvider() + .from( + PubsubReadSchemaTransformConfiguration.builder() + .setSubscription(SUBSCRIPTION) + .setFormat("AVRO") + .build()))); + p.run().waitUntilFinish(); + } - Map expected = testCase.expectedPubsubRead; + @Test + public void testReadRaw() throws IOException { + PCollectionRowTuple begin = PCollectionRowTuple.empty(p); - assertEquals(testCase.name, expected, actual); + Schema rawSchema = Schema.of(Schema.Field.of("payload", Schema.FieldType.BYTES)); + byte[] payload = "some payload".getBytes(Charsets.UTF_8); + + try (PubsubTestClientFactory clientFactory = + clientFactory(ImmutableList.of(incomingMessageOf(payload, CLOCK.currentTimeMillis())))) { + PubsubReadSchemaTransformConfiguration config = + PubsubReadSchemaTransformConfiguration.builder() + .setFormat("RAW") + .setSchema("") + .setSubscription(SUBSCRIPTION) + .setClientFactory(clientFactory) + .setClock(CLOCK) + .build(); + SchemaTransform transform = new PubsubReadSchemaTransformProvider().from(config); + PCollectionRowTuple reads = begin.apply(transform); + + PAssert.that(reads.get("output")) + .containsInAnyOrder( + ImmutableList.of(Row.withSchema(rawSchema).addValue(payload).build())); + + p.run().waitUntilFinish(); + } catch (Exception e) { + throw e; } } @Test - public void testInvalidConfiguration() { - for (TestCase testCase : cases) { - PCollectionRowTuple begin = PCollectionRowTuple.empty(p); - if (testCase.invalidConfigurationExpected) { - assertThrows( - testCase.name, - RuntimeException.class, - () -> begin.apply(testCase.pubsubReadSchemaTransform().buildTransform())); - } + public void testReadAttributes() throws IOException { + PCollectionRowTuple begin = PCollectionRowTuple.empty(p); + + Schema.builder() + .addByteArrayField("payload") + .addStringField("attr") + .addMapField("attrMap", Schema.FieldType.STRING, Schema.FieldType.STRING) + .build(); + + Schema rawSchema = + Schema.builder() + .addByteArrayField("payload") + .addStringField("attr") + .addMapField("attrMap", Schema.FieldType.STRING, Schema.FieldType.STRING) + .build(); + byte[] payload = "some payload".getBytes(Charsets.UTF_8); + String attr = "attr value"; + + try (PubsubTestClientFactory clientFactory = + clientFactory( + ImmutableList.of( + incomingMessageOf( + payload, CLOCK.currentTimeMillis(), ImmutableMap.of("attr", attr))))) { + PubsubReadSchemaTransformConfiguration config = + PubsubReadSchemaTransformConfiguration.builder() + .setFormat("RAW") + .setSchema("") + .setSubscription(SUBSCRIPTION) + .setAttributes(ImmutableList.of("attr")) + .setAttributesMap("attrMap") + .setClientFactory(clientFactory) + .setClock(CLOCK) + .build(); + SchemaTransform transform = new PubsubReadSchemaTransformProvider().from(config); + PCollectionRowTuple reads = begin.apply(transform); + + PAssert.that(reads.get("output")) + .containsInAnyOrder( + ImmutableList.of( + Row.withSchema(rawSchema) + .addValue(payload) + .addValue(attr) + .addValue(ImmutableMap.of("attr", attr)) + .build())); + + p.run().waitUntilFinish(); + } catch (Exception e) { + throw e; } } @Test - public void testInvalidInput() { - PCollectionRowTuple begin = PCollectionRowTuple.of("BadInput", p.apply(Create.of(ROWS))); - assertThrows( - IllegalArgumentException.class, - () -> - begin.apply( - new PubsubReadSchemaTransformProvider() - .from( - PubsubReadSchemaTransformConfiguration.builder() - .setDataSchema(SCHEMA) - .build()) - .buildTransform())); - } + public void testReadAvro() throws IOException { + PCollectionRowTuple begin = PCollectionRowTuple.empty(p); - private PubsubReadSchemaTransformProvider.PubsubReadSchemaTransform schemaTransformWithClock( - String format) { - PubsubReadSchemaTransformProvider.PubsubReadSchemaTransform transform = - (PubsubReadSchemaTransformProvider.PubsubReadSchemaTransform) - new PubsubReadSchemaTransformProvider() - .from( - PubsubReadSchemaTransformConfiguration.builder() - .setDataSchema(SCHEMA) - .setSubscription(SUBSCRIPTION) - .setFormat(format) - .build()) - .buildTransform(); + try (PubsubTestClientFactory clientFactory = clientFactory(beamRowToMessage())) { + PubsubReadSchemaTransformConfiguration config = + PubsubReadSchemaTransformConfiguration.builder() + .setFormat("AVRO") + .setSchema(SCHEMA) + .setSubscription(SUBSCRIPTION) + .setClientFactory(clientFactory) + .setClock(CLOCK) + .build(); + SchemaTransform transform = new PubsubReadSchemaTransformProvider().from(config); + PCollectionRowTuple reads = begin.apply(transform); + + PAssert.that(reads.get("output")).containsInAnyOrder(ROWS); + + p.run().waitUntilFinish(); + } catch (Exception e) { + throw e; + } + } - transform.setClock(CLOCK); + @Test + public void testReadAvroWithError() throws IOException { + PCollectionRowTuple begin = PCollectionRowTuple.empty(p); - return transform; - } + try (PubsubTestClientFactory clientFactory = clientFactory(beamRowToMessageWithError())) { + PubsubReadSchemaTransformConfiguration config = + PubsubReadSchemaTransformConfiguration.builder() + .setFormat("AVRO") + .setSchema(SCHEMA) + .setSubscription(SUBSCRIPTION) + .setErrorHandling( + PubsubReadSchemaTransformConfiguration.ErrorHandling.builder() + .setOutput("errors") + .build()) + .setClientFactory(clientFactory) + .setClock(CLOCK) + .build(); + SchemaTransform transform = new PubsubReadSchemaTransformProvider().from(config); + PCollectionRowTuple reads = begin.apply(transform); + + PAssert.that(reads.get("output")).empty(); + + PipelineResult result = p.run(); + result.waitUntilFinish(); + + MetricResults metrics = result.metrics(); + MetricQueryResults metricResults = + metrics.queryMetrics( + MetricsFilter.builder() + .addNameFilter( + MetricNameFilter.named( + PubsubReadSchemaTransformProvider.class, "PubSub-read-error-counter")) + .build()); + + Iterable> counters = metricResults.getCounters(); + if (!counters.iterator().hasNext()) { + throw new RuntimeException("no counters available "); + } - private static PubsubTestClient.PubsubTestClientFactory clientFactory( - List messages) { - return PubsubTestClient.createFactoryForPull( - CLOCK, PubsubClient.subscriptionPathFromPath(SUBSCRIPTION), 60, messages); + Long expectedCount = 3L; + for (MetricResult count : counters) { + assertEquals(expectedCount, count.getAttempted()); + } + } catch (Exception e) { + throw e; + } } - private static List incomingAvroMessagesOf(long millisSinceEpoch) { + private static List beamRowToMessage() { + long timestamp = CLOCK.currentTimeMillis(); return ROWS.stream() - .map(row -> incomingAvroMessageOf(row, millisSinceEpoch)) + .map( + row -> { + byte[] bytes = AVRO_PAYLOAD_SERIALIZER.serialize(row); + return incomingMessageOf(bytes, timestamp); + }) .collect(Collectors.toList()); } - private static PubsubClient.IncomingMessage incomingAvroMessageOf( - Row row, long millisSinceEpoch) { - byte[] bytes = AVRO_PAYLOAD_SERIALIZER.serialize(row); - return incomingMessageOf(bytes, millisSinceEpoch); - } - - private static List incomingJsonMessagesOf(long millisSinceEpoch) { - return PubsubReadSchemaTransformProviderTest.ROWS.stream() - .map(row -> incomingJsonMessageOf(row, millisSinceEpoch)) + private static List beamRowToMessageWithError() { + long timestamp = CLOCK.currentTimeMillis(); + return ROWSWITHERROR.stream() + .map( + row -> { + byte[] bytes = AVRO_PAYLOAD_SERIALIZER_WITH_ERROR.serialize(row); + return incomingMessageOf(bytes, timestamp); + }) .collect(Collectors.toList()); } - private static PubsubClient.IncomingMessage incomingJsonMessageOf( - Row row, long millisSinceEpoch) { - String name = Objects.requireNonNull(row.getString("name")); - long number = Objects.requireNonNull(row.getInt64("number")); - return incomingJsonMessageOf(name, number, millisSinceEpoch); - } - - private static PubsubClient.IncomingMessage incomingJsonMessageOf( - String name, long number, long millisSinceEpoch) { - Gson gson = new Gson(); - JsonObject obj = new JsonObject(); - obj.add("name", new JsonPrimitive(name)); - obj.add("number", new JsonPrimitive(number)); - byte[] bytes = gson.toJson(obj).getBytes(StandardCharsets.UTF_8); - return incomingMessageOf(bytes, millisSinceEpoch); + private static PubsubClient.IncomingMessage incomingMessageOf( + byte[] bytes, long millisSinceEpoch) { + return incomingMessageOf(bytes, millisSinceEpoch, ImmutableMap.of()); } private static PubsubClient.IncomingMessage incomingMessageOf( - byte[] bytes, long millisSinceEpoch) { + byte[] bytes, long millisSinceEpoch, Map attributes) { int nanos = Long.valueOf(millisSinceEpoch).intValue() * 1000; Timestamp timestamp = Timestamp.newBuilder().setNanos(nanos).build(); return PubsubClient.IncomingMessage.of( com.google.pubsub.v1.PubsubMessage.newBuilder() .setData(ByteString.copyFrom(bytes)) .setPublishTime(timestamp) + .putAllAttributes(attributes) .build(), millisSinceEpoch, 0, @@ -332,55 +362,9 @@ private static PubsubClient.IncomingMessage incomingMessageOf( UUID.randomUUID().toString()); } - static TestCase testCase(String name, PubsubReadSchemaTransformConfiguration configuration) { - return new TestCase(name, configuration); - } - - private static class TestCase { - - private final String name; - private final PubsubReadSchemaTransformConfiguration configuration; - - private Map expectedDeadLetterQueue; - - private Map expectedPubsubRead = - DisplayData.from(PubsubIO.readMessages()).asMap(); - - private boolean invalidConfigurationExpected = false; - - TestCase(String name, PubsubReadSchemaTransformConfiguration configuration) { - this.name = name; - this.configuration = configuration; - } - - SchemaTransform schemaTransform() { - PubsubReadSchemaTransformProvider provider = new PubsubReadSchemaTransformProvider(); - Row configurationRow = toBeamRow(); - return provider.from(configurationRow); - } - - PubsubReadSchemaTransformProvider.PubsubReadSchemaTransform pubsubReadSchemaTransform() { - return (PubsubReadSchemaTransformProvider.PubsubReadSchemaTransform) - schemaTransform().buildTransform(); - } - - private Row toBeamRow() { - return ROW_SERIALIZABLE_FUNCTION.apply(configuration); - } - - TestCase withExpectedDeadLetterQueue(PubsubIO.Write value) { - this.expectedDeadLetterQueue = DisplayData.from(value).asMap(); - return this; - } - - TestCase withExpectedPubsubRead(PubsubIO.Read value) { - this.expectedPubsubRead = DisplayData.from(value).asMap(); - return this; - } - - TestCase expectInvalidConfiguration() { - this.invalidConfigurationExpected = true; - return this; - } + private static PubsubTestClient.PubsubTestClientFactory clientFactory( + List messages) { + return PubsubTestClient.createFactoryForPull( + CLOCK, PubsubClient.subscriptionPathFromPath(SUBSCRIPTION), 60, messages); } } diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubRowToMessageTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubRowToMessageTest.java index 2ff0084bc0868..2489638eb7f7c 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubRowToMessageTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubRowToMessageTest.java @@ -61,7 +61,7 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.joda.time.Instant; import org.joda.time.ReadableDateTime; import org.junit.Rule; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubSchemaTransformMessageToRowFactoryTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubSchemaTransformMessageToRowFactoryTest.java deleted file mode 100644 index 709fc35e02aef..0000000000000 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubSchemaTransformMessageToRowFactoryTest.java +++ /dev/null @@ -1,337 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io.gcp.pubsub; - -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.ATTRIBUTES_FIELD; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.PAYLOAD_FIELD; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import org.apache.beam.sdk.extensions.avro.schemas.io.payloads.AvroPayloadSerializerProvider; -import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.sdk.schemas.Schema.FieldType; -import org.apache.beam.sdk.schemas.io.payloads.JsonPayloadSerializerProvider; -import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer; -import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializerProvider; -import org.apache.beam.sdk.values.Row; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Test for {@link PubsubSchemaTransformMessageToRowFactory}. */ -@RunWith(JUnit4.class) -public class PubsubSchemaTransformMessageToRowFactoryTest { - - List cases = - Arrays.asList( - testCase(PubsubReadSchemaTransformConfiguration.builder().setDataSchema(SCHEMA)) - .expectPayloadSerializerProvider(JSON_PAYLOAD_SERIALIZER_PROVIDER) - .withSerializerInput(), - testCase(PubsubReadSchemaTransformConfiguration.builder().setDataSchema(SCHEMA)) - .expectPubsubToRow( - PubsubMessageToRow.builder() - .messageSchema(SCHEMA) - .useFlatSchema(true) - .useDlq(false)), - testCase( - PubsubReadSchemaTransformConfiguration.builder() - .setDataSchema(SCHEMA) - .setDeadLetterQueue("projects/project/topics/topic")) - .expectPubsubToRow( - PubsubMessageToRow.builder() - .messageSchema(SCHEMA) - .useFlatSchema(true) - .useDlq(true)), - testCase( - PubsubReadSchemaTransformConfiguration.builder() - .setDataSchema(SCHEMA) - .setFormat("avro")) - .expectPayloadSerializerProvider(AVRO_PAYLOAD_SERIALIZER_PROVIDER) - .withSerializerInput(), - testCase( - PubsubReadSchemaTransformConfiguration.builder() - .setDataSchema(Schema.of(ATTRIBUTES_FIELD_ARRAY))) - .schemaShouldHaveValidAttributesField() - .fieldShouldBePresent( - ATTRIBUTES_FIELD_ARRAY.getName(), ATTRIBUTES_FIELD_ARRAY.getType()), - testCase( - PubsubReadSchemaTransformConfiguration.builder() - .setDataSchema(Schema.of(ATTRIBUTES_FIELD_MAP))) - .schemaShouldHaveValidAttributesField() - .fieldShouldBePresent(ATTRIBUTES_FIELD_MAP.getName(), ATTRIBUTES_FIELD_MAP.getType()), - testCase( - PubsubReadSchemaTransformConfiguration.builder() - .setDataSchema(Schema.of(ATTRIBUTES_FIELD_SHOULD_NOT_MATCH))), - testCase( - PubsubReadSchemaTransformConfiguration.builder() - .setDataSchema(Schema.of(PAYLOAD_FIELD_SHOULD_NOT_MATCH))), - testCase( - PubsubReadSchemaTransformConfiguration.builder() - .setDataSchema(Schema.of(PAYLOAD_FIELD_BYTES))) - .schemaShouldHaveValidPayloadField() - .fieldShouldBePresent(PAYLOAD_FIELD_BYTES.getName(), PAYLOAD_FIELD_BYTES.getType()), - testCase( - PubsubReadSchemaTransformConfiguration.builder() - .setDataSchema(Schema.of(PAYLOAD_FIELD_ROW))) - .schemaShouldHaveValidPayloadField() - .fieldShouldBePresent(PAYLOAD_FIELD_ROW.getName(), PAYLOAD_FIELD_ROW.getType()), - testCase( - PubsubReadSchemaTransformConfiguration.builder() - .setDataSchema(Schema.of(ATTRIBUTES_FIELD_ARRAY, PAYLOAD_FIELD_BYTES))) - .schemaShouldHaveValidAttributesField() - .schemaShouldHaveValidPayloadField() - .shouldUseNestedSchema() - .shouldNotNeedSerializer() - .expectPubsubToRow( - PubsubMessageToRow.builder() - .messageSchema(Schema.of(ATTRIBUTES_FIELD_ARRAY, PAYLOAD_FIELD_BYTES)) - .useFlatSchema(false) - .useDlq(false))); - - static final Schema.FieldType ATTRIBUTE_MAP_FIELD_TYPE = - Schema.FieldType.map(Schema.FieldType.STRING.withNullable(false), Schema.FieldType.STRING); - static final Schema ATTRIBUTE_ARRAY_ENTRY_SCHEMA = - Schema.builder().addStringField("key").addStringField("value").build(); - - static final Schema.FieldType ATTRIBUTE_ARRAY_FIELD_TYPE = - Schema.FieldType.array(Schema.FieldType.row(ATTRIBUTE_ARRAY_ENTRY_SCHEMA)); - - private static final Schema.Field ATTRIBUTES_FIELD_SHOULD_NOT_MATCH = - Schema.Field.of(ATTRIBUTES_FIELD, Schema.FieldType.STRING); - - private static final Schema.Field ATTRIBUTES_FIELD_MAP = - Schema.Field.of(ATTRIBUTES_FIELD, ATTRIBUTE_MAP_FIELD_TYPE); - - private static final Schema.Field ATTRIBUTES_FIELD_ARRAY = - Schema.Field.of(ATTRIBUTES_FIELD, ATTRIBUTE_ARRAY_FIELD_TYPE); - - private static final Schema.Field PAYLOAD_FIELD_SHOULD_NOT_MATCH = - Schema.Field.of(PAYLOAD_FIELD, Schema.FieldType.STRING); - - private static final Schema.Field PAYLOAD_FIELD_BYTES = - Schema.Field.of(PAYLOAD_FIELD, Schema.FieldType.BYTES); - - private static final Schema.Field PAYLOAD_FIELD_ROW = - Schema.Field.of(PAYLOAD_FIELD, Schema.FieldType.row(Schema.of())); - - private static final PayloadSerializerProvider JSON_PAYLOAD_SERIALIZER_PROVIDER = - new JsonPayloadSerializerProvider(); - - private static final AvroPayloadSerializerProvider AVRO_PAYLOAD_SERIALIZER_PROVIDER = - new AvroPayloadSerializerProvider(); - - private static final Schema SCHEMA = - Schema.of( - Schema.Field.of("name", Schema.FieldType.STRING), - Schema.Field.of("number", Schema.FieldType.INT64)); - - private static final Row ROW = - Row.withSchema(SCHEMA).withFieldValue("name", "a").withFieldValue("number", 1L).build(); - - @Test - public void testBuildMessageToRow() { - for (TestCase testCase : cases) { - if (testCase.expectPubsubToRow == null) { - continue; - } - - PubsubSchemaTransformMessageToRowFactory factory = testCase.factory(); - - PubsubMessageToRow expected = testCase.expectPubsubToRow; - PubsubMessageToRow actual = factory.buildMessageToRow(); - - assertEquals("messageSchema", expected.messageSchema(), actual.messageSchema()); - assertEquals("useFlatSchema", expected.useFlatSchema(), actual.useFlatSchema()); - assertEquals("useDlq", expected.useDlq(), actual.useDlq()); - } - } - - @Test - public void serializer() { - for (TestCase testCase : cases) { - PubsubSchemaTransformMessageToRowFactory factory = testCase.factory(); - - if (testCase.expectPayloadSerializerProvider == null) { - continue; - } - - Row serializerInput = testCase.serializerInput; - - byte[] expectedBytes = - testCase - .expectSerializerProvider() - .apply(testCase.dataSchema()) - .serialize(serializerInput); - - byte[] actualBytes = - factory.serializer().apply(testCase.dataSchema()).serialize(serializerInput); - - String expected = new String(expectedBytes, StandardCharsets.UTF_8); - String actual = new String(actualBytes, StandardCharsets.UTF_8); - - assertEquals(expected, actual); - } - } - - @Test - public void needsSerializer() { - for (TestCase testCase : cases) { - PubsubSchemaTransformMessageToRowFactory factory = testCase.factory(); - - boolean expected = testCase.shouldNeedSerializer; - boolean actual = factory.needsSerializer(); - - assertEquals(expected, actual); - } - } - - @Test - public void shouldUseNestedSchema() { - for (TestCase testCase : cases) { - PubsubSchemaTransformMessageToRowFactory factory = testCase.factory(); - - boolean expected = testCase.shouldUseNestedSchema; - boolean actual = factory.shouldUseNestedSchema(); - - assertEquals(expected, actual); - } - } - - @Test - public void schemaHasValidPayloadField() { - for (TestCase testCase : cases) { - PubsubSchemaTransformMessageToRowFactory factory = testCase.factory(); - - boolean expected = testCase.shouldSchemaHaveValidPayloadField; - boolean actual = factory.schemaHasValidPayloadField(); - - assertEquals(expected, actual); - } - } - - @Test - public void schemaHasValidAttributesField() { - for (TestCase testCase : cases) { - PubsubSchemaTransformMessageToRowFactory factory = testCase.factory(); - - boolean expected = testCase.shouldSchemaHaveValidAttributesField; - boolean actual = factory.schemaHasValidAttributesField(); - - assertEquals(expected, actual); - } - } - - @Test - public void fieldPresent() { - for (TestCase testCase : cases) { - PubsubSchemaTransformMessageToRowFactory factory = testCase.factory(); - for (Entry entry : testCase.shouldFieldPresent.entrySet()) { - - boolean actual = factory.fieldPresent(entry.getKey(), entry.getValue()); - - assertTrue(actual); - } - } - } - - static TestCase testCase(PubsubReadSchemaTransformConfiguration.Builder configurationBuilder) { - return new TestCase(configurationBuilder); - } - - private static class TestCase { - private final PubsubReadSchemaTransformConfiguration configuration; - - private PubsubMessageToRow expectPubsubToRow; - - private PayloadSerializerProvider expectPayloadSerializerProvider; - - private boolean shouldUseNestedSchema = false; - private boolean shouldNeedSerializer = true; - private boolean shouldSchemaHaveValidPayloadField = false; - private boolean shouldSchemaHaveValidAttributesField = false; - private final Map shouldFieldPresent = new HashMap<>(); - - private Row serializerInput; - - TestCase(PubsubReadSchemaTransformConfiguration.Builder configurationBuilder) { - this.configuration = configurationBuilder.build(); - } - - PubsubSchemaTransformMessageToRowFactory factory() { - return PubsubSchemaTransformMessageToRowFactory.from(configuration); - } - - Schema dataSchema() { - return configuration.getDataSchema(); - } - - TestCase expectPubsubToRow(PubsubMessageToRow.Builder pubsubMessageToRowBuilder) { - this.expectPubsubToRow = pubsubMessageToRowBuilder.build(); - return this; - } - - TestCase withSerializerInput() { - this.serializerInput = PubsubSchemaTransformMessageToRowFactoryTest.ROW; - return this; - } - - TestCase expectPayloadSerializerProvider(PayloadSerializerProvider value) { - this.expectPayloadSerializerProvider = value; - return this; - } - - PubsubMessageToRow.SerializerProvider expectSerializerProvider() { - Map params = new HashMap<>(); - PayloadSerializer payloadSerializer = - expectPayloadSerializerProvider.getSerializer(configuration.getDataSchema(), params); - - return (input -> payloadSerializer); - } - - TestCase shouldUseNestedSchema() { - this.shouldUseNestedSchema = true; - return this; - } - - TestCase shouldNotNeedSerializer() { - this.shouldNeedSerializer = false; - return this; - } - - TestCase schemaShouldHaveValidPayloadField() { - this.shouldSchemaHaveValidPayloadField = true; - return this; - } - - TestCase schemaShouldHaveValidAttributesField() { - this.shouldSchemaHaveValidAttributesField = true; - return this; - } - - TestCase fieldShouldBePresent(String name, Schema.FieldType expectedType) { - this.shouldFieldPresent.put(name, expectedType); - return this; - } - } -} diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubTestClientTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubTestClientTest.java index aeeca762c56ec..b94dde1b08f65 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubTestClientTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubTestClientTest.java @@ -30,9 +30,9 @@ import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.SubscriptionPath; import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath; import org.apache.beam.sdk.io.gcp.pubsub.PubsubTestClient.PubsubTestClientFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSinkTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSinkTest.java index f79732e74278a..c9b6bae45b981 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSinkTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSinkTest.java @@ -36,9 +36,9 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSourceTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSourceTest.java index 2c63cb307de5c..d3087df923867 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSourceTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSourceTest.java @@ -53,7 +53,7 @@ import org.apache.beam.sdk.testing.CoderProperties; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.util.CoderUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; import org.junit.After; import org.junit.Rule; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubWriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubWriteIT.java new file mode 100644 index 0000000000000..1898c4ae4af00 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubWriteIT.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.gcp.pubsub; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.Objects; +import org.apache.beam.sdk.io.GenerateSequence; +import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.values.TypeDescriptors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.commons.lang3.RandomStringUtils; +import org.joda.time.Instant; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Integration tests for {@link PubsubIO.Write} operations. */ +@RunWith(JUnit4.class) +public class PubsubWriteIT { + + @Rule public transient TestPipeline pipeline = TestPipeline.create(); + + private PubsubClient pubsubClient; + + private TopicPath testTopic; + + @Before + public void setup() throws IOException { + PubsubOptions options = TestPipeline.testingPipelineOptions().as(PubsubOptions.class); + String project = options.getProject(); + pubsubClient = PubsubGrpcClient.FACTORY.newClient(null, null, options); + testTopic = + PubsubClient.topicPathFromName(project, "pubsub-write-" + Instant.now().getMillis()); + pubsubClient.createTopic(testTopic); + } + + @After + public void tearDown() throws IOException { + pubsubClient.deleteTopic(testTopic); + pubsubClient.close(); + } + + @Test + public void testBoundedWriteSmallMessage() { + String smallMessage = RandomStringUtils.randomAscii(100); + pipeline.apply(Create.of(smallMessage)).apply(PubsubIO.writeStrings().to(testTopic.getPath())); + pipeline.run(); + } + + @Test + public void testBoundedWriteSequence() { + pipeline + .apply(GenerateSequence.from(0L).to(1000L)) + .apply( + MapElements.into(TypeDescriptors.strings()) + .via(sequence -> Objects.requireNonNull(sequence).toString())) + .apply(PubsubIO.writeStrings().to(testTopic.getPath())); + pipeline.run(); + } + + @Test + public void testBoundedWriteLargeMessage() { + String largeMessage = RandomStringUtils.randomAscii(1_000_000); + pipeline.apply(Create.of(largeMessage)).apply(PubsubIO.writeStrings().to(testTopic.getPath())); + pipeline.run(); + } + + @Test + public void testBoundedWriteMessageWithAttributes() { + byte[] payload = RandomStringUtils.randomAscii(1_000_000).getBytes(StandardCharsets.UTF_8); + Map attributes = + ImmutableMap.builder() + .put("id", "1") + .put("description", RandomStringUtils.randomAscii(100)) + .build(); + + pipeline + .apply(Create.of(new PubsubMessage(payload, attributes))) + .apply(PubsubIO.writeMessages().to(testTopic.getPath())); + pipeline.run(); + } +} diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubWriteSchemaTransformProviderIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubWriteSchemaTransformProviderIT.java deleted file mode 100644 index 7ada9686853a2..0000000000000 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubWriteSchemaTransformProviderIT.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io.gcp.pubsub; - -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubWriteSchemaTransformConfiguration.DEFAULT_TIMESTAMP_ATTRIBUTE; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubWriteSchemaTransformProvider.INPUT_TAG; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import org.apache.beam.sdk.PipelineResult; -import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.IncomingMessage; -import org.apache.beam.sdk.schemas.AutoValueSchema; -import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.sdk.schemas.Schema.Field; -import org.apache.beam.sdk.schemas.Schema.FieldType; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.sdk.values.PCollectionRowTuple; -import org.apache.beam.sdk.values.Row; -import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.commons.lang3.tuple.Pair; -import org.joda.time.Instant; -import org.joda.time.format.ISODateTimeFormat; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; - -/** Integration tests for {@link PubsubWriteSchemaTransformProvider}. */ -@SuppressWarnings({ - "nullness" // TODO(https://github.com/apache/beam/issues/20497) -}) -public class PubsubWriteSchemaTransformProviderIT { - - @Rule public transient TestPipeline pipeline = TestPipeline.create(); - - private static final TestPubsubOptions TEST_PUBSUB_OPTIONS = - TestPipeline.testingPipelineOptions().as(TestPubsubOptions.class); - - static { - TEST_PUBSUB_OPTIONS.setBlockOnRun(false); - } - - private static final String HAS_NO_SCHEMA = "has-no-schema"; - - private static PubsubClient pubsubClient; - - private static PubsubClient.TopicPath hasNoSchemaTopic; - - private static PubsubClient.SubscriptionPath hasNoSchemaSubscription; - - private static final Instant TIMESTAMP = Instant.now(); - - private static final String RESOURCE_NAME_POSTFIX = "-" + TIMESTAMP.getMillis(); - - private static final int ACK_DEADLINE_SECONDS = 60; - - private static final int AWAIT_TERMINATED_SECONDS = 30; - - private static final AutoValueSchema AUTO_VALUE_SCHEMA = new AutoValueSchema(); - - private static final TypeDescriptor - CONFIGURATION_TYPE_DESCRIPTOR = - TypeDescriptor.of(PubsubWriteSchemaTransformConfiguration.class); - - private static final SerializableFunction - TO_ROW_FN = AUTO_VALUE_SCHEMA.toRowFunction(CONFIGURATION_TYPE_DESCRIPTOR); - - private final Field timestampField = Field.of("timestamp", FieldType.DATETIME); - - private final Field payloadBytesField = Field.of("payload", FieldType.BYTES); - - @BeforeClass - public static void setUp() throws IOException { - String project = TEST_PUBSUB_OPTIONS.as(PubsubOptions.class).getProject(); - pubsubClient = PubsubGrpcClient.FACTORY.newClient(null, null, TEST_PUBSUB_OPTIONS); - hasNoSchemaTopic = - PubsubClient.topicPathFromName(project, HAS_NO_SCHEMA + RESOURCE_NAME_POSTFIX); - hasNoSchemaSubscription = - PubsubClient.subscriptionPathFromName(project, HAS_NO_SCHEMA + RESOURCE_NAME_POSTFIX); - - pubsubClient.createTopic(hasNoSchemaTopic); - pubsubClient.createSubscription( - hasNoSchemaTopic, hasNoSchemaSubscription, ACK_DEADLINE_SECONDS); - } - - @AfterClass - public static void tearDown() throws IOException { - pubsubClient.deleteSubscription(hasNoSchemaSubscription); - pubsubClient.deleteTopic(hasNoSchemaTopic); - - pubsubClient.close(); - } - - @Test - public void testWritePayloadBytes() throws IOException { - Instant timestamp = Instant.ofEpochMilli(100000L); - Schema schema = Schema.of(payloadBytesField, timestampField); - List input = - Collections.singletonList( - Row.withSchema(schema).attachValues("aaa".getBytes(StandardCharsets.UTF_8), timestamp)); - Row configuration = - TO_ROW_FN.apply( - PubsubWriteSchemaTransformConfiguration.builder() - .setSource( - PubsubWriteSchemaTransformConfiguration.sourceConfigurationBuilder() - .setPayloadFieldName(payloadBytesField.getName()) - .setTimestampFieldName(timestampField.getName()) - .build()) - .setTopic(hasNoSchemaTopic.getPath()) - .setTarget( - PubsubWriteSchemaTransformConfiguration.targetConfigurationBuilder().build()) - .build()); - - PCollectionRowTuple.of(INPUT_TAG, pipeline.apply(Create.of(input).withRowSchema(schema))) - .apply(new PubsubWriteSchemaTransformProvider().from(configuration).buildTransform()); - - PipelineResult job = pipeline.run(TEST_PUBSUB_OPTIONS); - Instant now = Instant.now(); - Instant stop = Instant.ofEpochMilli(now.getMillis() + AWAIT_TERMINATED_SECONDS * 1000); - List>> actualList = new ArrayList<>(); - while (now.isBefore(stop)) { - List received = pubsubClient.pull(0, hasNoSchemaSubscription, 1, true); - for (IncomingMessage incoming : received) { - actualList.add( - Pair.of( - incoming.message().getData().toStringUtf8(), - ImmutableMap.of( - DEFAULT_TIMESTAMP_ATTRIBUTE, - incoming - .message() - .getAttributesMap() - .getOrDefault(DEFAULT_TIMESTAMP_ATTRIBUTE, "")))); - } - if (actualList.size() == input.size()) { - break; - } - now = Instant.now(); - } - job.cancel(); - assertFalse( - String.format( - "messages pulled from %s should not be empty", hasNoSchemaSubscription.getPath()), - actualList.isEmpty()); - Pair> actual = actualList.get(0); - Row expected = input.get(0); - String payload = - new String( - Objects.requireNonNull(expected.getBytes(payloadBytesField.getName())), - StandardCharsets.UTF_8); - assertEquals(payload, actual.getLeft()); - assertEquals( - ISODateTimeFormat.dateTime().print(timestamp), - actual.getRight().get(DEFAULT_TIMESTAMP_ATTRIBUTE)); - } -} diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubWriteSchemaTransformProviderTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubWriteSchemaTransformProviderTest.java deleted file mode 100644 index 98939f7ddc686..0000000000000 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubWriteSchemaTransformProviderTest.java +++ /dev/null @@ -1,786 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.sdk.io.gcp.pubsub; - -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessage.ATTRIBUTES_FIELD_TYPE; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessage.DEFAULT_ATTRIBUTES_KEY_NAME; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessage.DEFAULT_EVENT_TIMESTAMP_KEY_NAME; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessage.DEFAULT_PAYLOAD_KEY_NAME; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessage.EVENT_TIMESTAMP_FIELD_TYPE; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessageTest.ALL_DATA_TYPES_SCHEMA; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessageTest.NON_USER_WITH_BYTES_PAYLOAD; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubRowToMessageTest.rowWithAllDataTypes; -import static org.apache.beam.sdk.io.gcp.pubsub.PubsubWriteSchemaTransformProvider.INPUT_TAG; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThrows; - -import com.google.api.client.util.Clock; -import java.io.IOException; -import java.io.Serializable; -import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import org.apache.avro.SchemaParseException; -import org.apache.beam.sdk.extensions.avro.schemas.io.payloads.AvroPayloadSerializerProvider; -import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.SchemaPath; -import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath; -import org.apache.beam.sdk.io.gcp.pubsub.PubsubTestClient.PubsubTestClientFactory; -import org.apache.beam.sdk.io.gcp.pubsub.PubsubWriteSchemaTransformProvider.PubsubWriteSchemaTransform; -import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.sdk.schemas.AutoValueSchema; -import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.sdk.schemas.Schema.Field; -import org.apache.beam.sdk.schemas.Schema.FieldType; -import org.apache.beam.sdk.schemas.io.payloads.JsonPayloadSerializerProvider; -import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializer; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.sdk.util.RowJson.UnsupportedRowJsonException; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PCollectionRowTuple; -import org.apache.beam.sdk.values.Row; -import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.joda.time.Instant; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Tests for {@link PubsubWriteSchemaTransformProvider}. */ -@RunWith(JUnit4.class) -public class PubsubWriteSchemaTransformProviderTest { - - private static final String ID_ATTRIBUTE = "id_attribute"; - private static final String TOPIC = "projects/project/topics/topic"; - private static final MockClock CLOCK = new MockClock(Instant.now()); - private static final AutoValueSchema AUTO_VALUE_SCHEMA = new AutoValueSchema(); - private static final TypeDescriptor TYPE_DESCRIPTOR = - TypeDescriptor.of(PubsubWriteSchemaTransformConfiguration.class); - private static final SerializableFunction TO_ROW = - AUTO_VALUE_SCHEMA.toRowFunction(TYPE_DESCRIPTOR); - - private static final PipelineOptions OPTIONS = PipelineOptionsFactory.create(); - - static { - OPTIONS.setStableUniqueNames(PipelineOptions.CheckEnabled.OFF); - } - - @Rule public transient TestPipeline pipeline = TestPipeline.create(); - - @Test - public void testBuildPubsubWrite() { - assertEquals( - "default configuration should yield a topic Pub/Sub write", - pubsubWrite(), - transform(configurationBuilder()).buildPubsubWrite()); - - assertEquals( - "idAttribute in configuration should yield a idAttribute set Pub/Sub write", - pubsubWrite().withIdAttribute(ID_ATTRIBUTE), - transform(configurationBuilder().setIdAttribute(ID_ATTRIBUTE)).buildPubsubWrite()); - } - - @Test - public void testBuildPubsubRowToMessage() { - assertEquals( - "override timestamp attribute on configuration should yield a PubsubRowToMessage with target timestamp", - rowToMessageBuilder().setTargetTimestampAttributeName("custom_timestamp_attribute").build(), - transform( - configurationBuilder() - .setTarget( - PubsubWriteSchemaTransformConfiguration.targetConfigurationBuilder() - .setTimestampAttributeKey("custom_timestamp_attribute") - .build())) - .buildPubsubRowToMessage(NON_USER_WITH_BYTES_PAYLOAD)); - - assertNull( - "failing to set format should yield a null payload serializer", - transform(configurationBuilder()) - .buildPubsubRowToMessage(ALL_DATA_TYPES_SCHEMA) - .getPayloadSerializer()); - - assertThrows( - "setting 'json' format for a unsupported field containing Schema should throw an Exception", - UnsupportedRowJsonException.class, - () -> - transform(configurationBuilder().setFormat("json")) - .buildPubsubRowToMessage( - Schema.of(Field.of(DEFAULT_ATTRIBUTES_KEY_NAME, ATTRIBUTES_FIELD_TYPE)))); - - assertThrows( - "setting 'avro' format for a unsupported field containing Schema should throw an Exception", - SchemaParseException.class, - () -> - transform(configurationBuilder().setFormat("avro")) - .buildPubsubRowToMessage( - Schema.of(Field.of(DEFAULT_ATTRIBUTES_KEY_NAME, ATTRIBUTES_FIELD_TYPE)))); - - assertNotNull( - "setting 'json' format for valid schema should yield PayloadSerializer", - transform(configurationBuilder().setFormat("json")) - .buildPubsubRowToMessage(ALL_DATA_TYPES_SCHEMA) - .getPayloadSerializer()); - - assertNotNull( - "setting 'avro' format for valid schema should yield PayloadSerializer", - transform(configurationBuilder().setFormat("avro")) - .buildPubsubRowToMessage(ALL_DATA_TYPES_SCHEMA) - .getPayloadSerializer()); - } - - @Test - public void testInvalidTaggedInput() { - Row withAllDataTypes = - rowWithAllDataTypes( - true, - (byte) 0, - Instant.now().toDateTime(), - BigDecimal.valueOf(1L), - 3.12345, - 4.1f, - (short) 5, - 2, - 7L, - "asdfjkl;"); - - PCollection rows = - pipeline.apply(Create.of(withAllDataTypes)).setRowSchema(ALL_DATA_TYPES_SCHEMA); - - assertThrows( - "empty input should not be allowed", - IllegalArgumentException.class, - () -> transform(configurationBuilder()).expand(PCollectionRowTuple.empty(pipeline))); - - assertThrows( - "input with >1 tagged rows should not be allowed", - IllegalArgumentException.class, - () -> - transform(configurationBuilder()) - .expand(PCollectionRowTuple.of(INPUT_TAG, rows).and("somethingelse", rows))); - - assertThrows( - "input missing INPUT tag should not be allowed", - IllegalArgumentException.class, - () -> - transform(configurationBuilder()) - .expand(PCollectionRowTuple.of("somethingelse", rows))); - - pipeline.run(OPTIONS); - } - - @Test - public void testValidateSourceSchemaAgainstConfiguration() { - // Only containing user fields and no configuration details should be valid - transform(configurationBuilder()) - .validateSourceSchemaAgainstConfiguration(ALL_DATA_TYPES_SCHEMA); - - // Matching attributes, timestamp, and payload (bytes) fields configured with expected types - // should be valid - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration.sourceConfigurationBuilder() - .setAttributesFieldName("attributes") - .setTimestampFieldName("timestamp") - .setPayloadFieldName("payload") - .build())) - .validateSourceSchemaAgainstConfiguration( - Schema.of( - Field.of("attributes", ATTRIBUTES_FIELD_TYPE), - Field.of("timestamp", EVENT_TIMESTAMP_FIELD_TYPE), - Field.of("payload", Schema.FieldType.BYTES))); - - // Matching attributes, timestamp, and payload (ROW) fields configured with expected types - // should be valid - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration.sourceConfigurationBuilder() - .setAttributesFieldName("attributes") - .setTimestampFieldName("timestamp") - .setPayloadFieldName("payload") - .build())) - .validateSourceSchemaAgainstConfiguration( - Schema.of( - Field.of("attributes", ATTRIBUTES_FIELD_TYPE), - Field.of("timestamp", EVENT_TIMESTAMP_FIELD_TYPE), - Field.of("payload", Schema.FieldType.row(ALL_DATA_TYPES_SCHEMA)))); - - assertThrows( - "empty Schema should be invalid", - IllegalArgumentException.class, - () -> - transform(configurationBuilder()) - .validateSourceSchemaAgainstConfiguration(Schema.of())); - - assertThrows( - "attributes field in configuration but not in schema should be invalid", - IllegalArgumentException.class, - () -> - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration.sourceConfigurationBuilder() - .setAttributesFieldName("attributes") - .build())) - .validateSourceSchemaAgainstConfiguration(ALL_DATA_TYPES_SCHEMA)); - - assertThrows( - "timestamp field in configuration but not in schema should be invalid", - IllegalArgumentException.class, - () -> - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration.sourceConfigurationBuilder() - .setTimestampFieldName("timestamp") - .build())) - .validateSourceSchemaAgainstConfiguration(ALL_DATA_TYPES_SCHEMA)); - - assertThrows( - "payload field in configuration but not in schema should be invalid", - IllegalArgumentException.class, - () -> - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration.sourceConfigurationBuilder() - .setPayloadFieldName("payload") - .build())) - .validateSourceSchemaAgainstConfiguration(ALL_DATA_TYPES_SCHEMA)); - - assertThrows( - "attributes field in configuration but mismatching attributes type should be invalid", - IllegalArgumentException.class, - () -> - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration.sourceConfigurationBuilder() - .setAttributesFieldName("attributes") - .build())) - .validateSourceSchemaAgainstConfiguration( - // should be FieldType.map(FieldType.STRING, FieldType.STRING) - Schema.of( - Field.of("attributes", FieldType.map(FieldType.BYTES, FieldType.STRING))))); - - assertThrows( - "timestamp field in configuration but mismatching timestamp type should be invalid", - IllegalArgumentException.class, - () -> - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration.sourceConfigurationBuilder() - .setAttributesFieldName("timestamp") - .build())) - .validateSourceSchemaAgainstConfiguration( - // should be FieldType.DATETIME - Schema.of(Field.of("timestamp", FieldType.STRING)))); - - assertThrows( - "payload field in configuration but mismatching payload type should be invalid", - IllegalArgumentException.class, - () -> - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration.sourceConfigurationBuilder() - .setAttributesFieldName("payload") - .build())) - .validateSourceSchemaAgainstConfiguration( - // should be FieldType.BYTES or FieldType.row(...) - Schema.of(Field.of("payload", FieldType.STRING)))); - } - - @Test - public void testValidateTargetSchemaAgainstPubsubSchema() throws IOException { - TopicPath topicPath = PubsubClient.topicPathFromPath(TOPIC); - PubsubTestClientFactory noSchemaFactory = - PubsubTestClient.createFactoryForGetSchema(topicPath, null, null); - - PubsubTestClientFactory schemaDeletedFactory = - PubsubTestClient.createFactoryForGetSchema(topicPath, SchemaPath.DELETED_SCHEMA, null); - - PubsubTestClientFactory mismatchingSchemaFactory = - PubsubTestClient.createFactoryForGetSchema( - topicPath, - PubsubClient.schemaPathFromId("testProject", "misMatch"), - Schema.of(Field.of("StringField", FieldType.STRING))); - - PubsubTestClientFactory matchingSchemaFactory = - PubsubTestClient.createFactoryForGetSchema( - topicPath, - PubsubClient.schemaPathFromId("testProject", "match"), - ALL_DATA_TYPES_SCHEMA); - - // Should pass validation exceptions if Pub/Sub topic lacks schema - transform(configurationBuilder()) - .withPubsubClientFactory(noSchemaFactory) - .validateTargetSchemaAgainstPubsubSchema(ALL_DATA_TYPES_SCHEMA, OPTIONS); - noSchemaFactory.close(); - - // Should pass validation if Pub/Sub topic schema deleted - transform(configurationBuilder()) - .withPubsubClientFactory(schemaDeletedFactory) - .validateTargetSchemaAgainstPubsubSchema(ALL_DATA_TYPES_SCHEMA, OPTIONS); - schemaDeletedFactory.close(); - - assertThrows( - "mismatched schema should be detected from Pub/Sub topic", - IllegalStateException.class, - () -> - transform(configurationBuilder()) - .withPubsubClientFactory(mismatchingSchemaFactory) - .validateTargetSchemaAgainstPubsubSchema(ALL_DATA_TYPES_SCHEMA, OPTIONS)); - mismatchingSchemaFactory.close(); - - // Should pass validation if Pub/Sub topic schema matches - transform(configurationBuilder()) - .withPubsubClientFactory(matchingSchemaFactory) - .validateTargetSchemaAgainstPubsubSchema(ALL_DATA_TYPES_SCHEMA, OPTIONS); - matchingSchemaFactory.close(); - } - - @Test - public void testBuildTargetSchema() { - - Field sourceAttributesField = Field.of("attributes", ATTRIBUTES_FIELD_TYPE); - Field sourceTimestampField = Field.of("timestamp", EVENT_TIMESTAMP_FIELD_TYPE); - Field sourcePayloadBytesField = Field.of("payload", FieldType.BYTES); - Field sourcePayloadRowField = Field.of("payload", FieldType.row(ALL_DATA_TYPES_SCHEMA)); - - Field targetAttributesField = Field.of(DEFAULT_ATTRIBUTES_KEY_NAME, ATTRIBUTES_FIELD_TYPE); - Field targetTimestampField = - Field.of(DEFAULT_EVENT_TIMESTAMP_KEY_NAME, EVENT_TIMESTAMP_FIELD_TYPE); - Field targetPayloadBytesField = Field.of(DEFAULT_PAYLOAD_KEY_NAME, FieldType.BYTES); - Field targetPayloadRowField = - Field.of(DEFAULT_PAYLOAD_KEY_NAME, FieldType.row(ALL_DATA_TYPES_SCHEMA)); - - assertEquals( - "attributes and timestamp field should append to user fields", - Schema.builder() - .addField(targetAttributesField) - .addField(targetTimestampField) - .addFields(ALL_DATA_TYPES_SCHEMA.getFields()) - .build(), - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration.sourceConfigurationBuilder() - .build())) - .buildTargetSchema(ALL_DATA_TYPES_SCHEMA)); - - assertEquals( - "timestamp field should append to user fields; attributes field name changed", - Schema.builder() - .addField(targetAttributesField) - .addField(targetTimestampField) - .addFields(ALL_DATA_TYPES_SCHEMA.getFields()) - .build(), - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration.sourceConfigurationBuilder() - .setAttributesFieldName("attributes") - .build())) - .buildTargetSchema( - Schema.builder() - .addField(sourceAttributesField) - .addFields(ALL_DATA_TYPES_SCHEMA.getFields()) - .build())); - - assertEquals( - "attributes field should append to user fields; timestamp field name changed", - Schema.builder() - .addField(targetAttributesField) - .addField(targetTimestampField) - .addFields(ALL_DATA_TYPES_SCHEMA.getFields()) - .build(), - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration.sourceConfigurationBuilder() - .setTimestampFieldName("timestamp") - .build())) - .buildTargetSchema( - Schema.builder() - .addField(sourceTimestampField) - .addFields(ALL_DATA_TYPES_SCHEMA.getFields()) - .build())); - - assertEquals( - "attributes and timestamp field appended to user payload bytes field; payload field name changed", - Schema.builder() - .addField(targetAttributesField) - .addField(targetTimestampField) - .addField(targetPayloadBytesField) - .build(), - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration.sourceConfigurationBuilder() - .setPayloadFieldName("payload") - .build())) - .buildTargetSchema(Schema.builder().addField(sourcePayloadBytesField).build())); - - assertEquals( - "attributes and timestamp field appended to user payload row field; payload field name changed", - Schema.builder() - .addField(targetAttributesField) - .addField(targetTimestampField) - .addField(targetPayloadRowField) - .build(), - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration.sourceConfigurationBuilder() - .setPayloadFieldName("payload") - .build())) - .buildTargetSchema(Schema.builder().addField(sourcePayloadRowField).build())); - - assertEquals( - "attributes and timestamp fields name changed", - Schema.builder() - .addField(targetAttributesField) - .addField(targetTimestampField) - .addFields(ALL_DATA_TYPES_SCHEMA.getFields()) - .build(), - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration.sourceConfigurationBuilder() - .setAttributesFieldName("attributes") - .setTimestampFieldName("timestamp") - .build())) - .buildTargetSchema( - Schema.builder() - .addField(sourceAttributesField) - .addField(sourceTimestampField) - .addFields(ALL_DATA_TYPES_SCHEMA.getFields()) - .build())); - - assertEquals( - "attributes, timestamp, payload bytes fields name changed", - Schema.builder() - .addField(targetAttributesField) - .addField(targetTimestampField) - .addFields(targetPayloadBytesField) - .build(), - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration.sourceConfigurationBuilder() - .setAttributesFieldName("attributes") - .setTimestampFieldName("timestamp") - .setPayloadFieldName("payload") - .build())) - .buildTargetSchema( - Schema.builder() - .addField(sourceAttributesField) - .addField(sourceTimestampField) - .addField(sourcePayloadBytesField) - .build())); - - assertEquals( - "attributes, timestamp, payload row fields name changed", - Schema.builder() - .addField(targetAttributesField) - .addField(targetTimestampField) - .addFields(targetPayloadRowField) - .build(), - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration.sourceConfigurationBuilder() - .setAttributesFieldName("attributes") - .setTimestampFieldName("timestamp") - .setPayloadFieldName("payload") - .build())) - .buildTargetSchema( - Schema.builder() - .addField(sourceAttributesField) - .addField(sourceTimestampField) - .addField(sourcePayloadRowField) - .build())); - } - - @Test - public void testConvertForRowToMessageTransform() { - Row userRow = - rowWithAllDataTypes( - false, - (byte) 0, - Instant.ofEpochMilli(CLOCK.currentTimeMillis()).toDateTime(), - BigDecimal.valueOf(1L), - 1.12345, - 1.1f, - (short) 1, - 1, - 1L, - "吃葡萄不吐葡萄皮,不吃葡萄倒吐葡萄皮"); - - Field sourceAttributes = Field.of("attributes", ATTRIBUTES_FIELD_TYPE); - Field targetAttributes = Field.of(DEFAULT_ATTRIBUTES_KEY_NAME, ATTRIBUTES_FIELD_TYPE); - - Field sourceTimestamp = Field.of("timestamp", EVENT_TIMESTAMP_FIELD_TYPE); - Field targetTimestamp = Field.of(DEFAULT_EVENT_TIMESTAMP_KEY_NAME, EVENT_TIMESTAMP_FIELD_TYPE); - - Field sourcePayloadBytes = Field.of("payload", FieldType.BYTES); - Field targetPayloadBytes = Field.of(DEFAULT_PAYLOAD_KEY_NAME, FieldType.BYTES); - - Field sourcePayloadRow = Field.of("payload", FieldType.row(ALL_DATA_TYPES_SCHEMA)); - Field targetPayloadRow = - Field.of(DEFAULT_PAYLOAD_KEY_NAME, FieldType.row(ALL_DATA_TYPES_SCHEMA)); - - Map attributes = ImmutableMap.of("a", "1"); - Instant generatedTimestamp = Instant.ofEpochMilli(CLOCK.currentTimeMillis()); - Instant timestampFromSource = Instant.ofEpochMilli(CLOCK.currentTimeMillis() + 10000L); - byte[] payloadBytes = "吃葡萄不吐葡萄皮,不吃葡萄倒吐葡萄皮".getBytes(StandardCharsets.UTF_8); - - PAssert.that( - "attributes only source yields attributes + timestamp target", - pipeline - .apply( - Create.of(Row.withSchema(Schema.of(sourceAttributes)).attachValues(attributes))) - .setRowSchema(Schema.of(sourceAttributes)) - .apply( - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration - .sourceConfigurationBuilder() - .setAttributesFieldName(sourceAttributes.getName()) - .build())) - .convertForRowToMessage( - Schema.of(targetAttributes, targetTimestamp), CLOCK)) - .setRowSchema(Schema.of(targetAttributes, targetTimestamp))) - .containsInAnyOrder( - Row.withSchema(Schema.of(targetAttributes, targetTimestamp)) - .attachValues(attributes, generatedTimestamp)); - - PAssert.that( - "timestamp only source yields attributes + timestamp target", - pipeline - .apply( - Create.of( - Row.withSchema(Schema.of(sourceTimestamp)) - .attachValues(timestampFromSource))) - .setRowSchema(Schema.of(sourceTimestamp)) - .apply( - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration - .sourceConfigurationBuilder() - .setTimestampFieldName(sourceTimestamp.getName()) - .build())) - .convertForRowToMessage( - Schema.of(targetAttributes, targetTimestamp), CLOCK)) - .setRowSchema(Schema.of(targetAttributes, targetTimestamp))) - .containsInAnyOrder( - Row.withSchema(Schema.of(targetAttributes, targetTimestamp)) - .attachValues(ImmutableMap.of(), timestampFromSource)); - - PAssert.that( - "timestamp and attributes source yields renamed fields in target", - pipeline - .apply( - Create.of( - Row.withSchema(Schema.of(sourceAttributes, sourceTimestamp)) - .attachValues(attributes, timestampFromSource))) - .setRowSchema(Schema.of(sourceAttributes, sourceTimestamp)) - .apply( - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration - .sourceConfigurationBuilder() - .setAttributesFieldName(sourceAttributes.getName()) - .setTimestampFieldName(sourceTimestamp.getName()) - .build())) - .convertForRowToMessage( - Schema.of(targetAttributes, targetTimestamp), CLOCK)) - .setRowSchema(Schema.of(targetAttributes, targetTimestamp))) - .containsInAnyOrder( - Row.withSchema(Schema.of(targetAttributes, targetTimestamp)) - .attachValues(attributes, timestampFromSource)); - - PAssert.that( - "bytes payload only source yields attributes + timestamp + renamed bytes payload target", - pipeline - .apply( - Create.of( - Row.withSchema(Schema.of(sourcePayloadBytes)) - .withFieldValue(sourcePayloadBytes.getName(), payloadBytes) - .build())) - .setRowSchema(Schema.of(sourcePayloadBytes)) - .apply( - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration - .sourceConfigurationBuilder() - .setPayloadFieldName(sourcePayloadBytes.getName()) - .build())) - .convertForRowToMessage( - Schema.of(targetAttributes, targetTimestamp, targetPayloadBytes), - CLOCK)) - .setRowSchema(Schema.of(targetAttributes, targetTimestamp, targetPayloadBytes))) - .containsInAnyOrder( - Row.withSchema(Schema.of(targetAttributes, targetTimestamp, targetPayloadBytes)) - .attachValues(ImmutableMap.of(), generatedTimestamp, payloadBytes)); - - PAssert.that( - "row payload only source yields attributes + timestamp + renamed row payload target", - pipeline - .apply(Create.of(Row.withSchema(Schema.of(sourcePayloadRow)).attachValues(userRow))) - .setRowSchema(Schema.of(sourcePayloadRow)) - .apply( - transform( - configurationBuilder() - .setSource( - PubsubWriteSchemaTransformConfiguration - .sourceConfigurationBuilder() - .setPayloadFieldName(sourcePayloadRow.getName()) - .build())) - .convertForRowToMessage( - Schema.of(targetAttributes, targetTimestamp, targetPayloadRow), CLOCK)) - .setRowSchema(Schema.of(targetAttributes, targetTimestamp, targetPayloadRow))) - .containsInAnyOrder( - Row.withSchema(Schema.of(targetAttributes, targetTimestamp, targetPayloadRow)) - .attachValues(ImmutableMap.of(), generatedTimestamp, userRow)); - - PAssert.that( - "user only fields source yields attributes + timestamp + user fields target", - pipeline - .apply(Create.of(userRow)) - .setRowSchema(ALL_DATA_TYPES_SCHEMA) - .apply( - transform(configurationBuilder()) - .convertForRowToMessage( - Schema.builder() - .addField(targetAttributes) - .addField(targetTimestamp) - .addFields(ALL_DATA_TYPES_SCHEMA.getFields()) - .build(), - CLOCK)) - .setRowSchema( - Schema.builder() - .addField(targetAttributes) - .addField(targetTimestamp) - .addFields(ALL_DATA_TYPES_SCHEMA.getFields()) - .build())) - .containsInAnyOrder( - Row.withSchema( - Schema.builder() - .addField(targetAttributes) - .addField(targetTimestamp) - .addFields(ALL_DATA_TYPES_SCHEMA.getFields()) - .build()) - .addValue(ImmutableMap.of()) - .addValue(generatedTimestamp) - .addValues(userRow.getValues()) - .build()); - - pipeline.run(OPTIONS); - } - - @Test - public void testGetPayloadSerializer() { - Row withAllDataTypes = - rowWithAllDataTypes( - false, - (byte) 0, - Instant.now().toDateTime(), - BigDecimal.valueOf(-1L), - -3.12345, - -4.1f, - (short) -5, - -2, - -7L, - "吃葡萄不吐葡萄皮,不吃葡萄倒吐葡萄皮"); - - PayloadSerializer jsonPayloadSerializer = - new JsonPayloadSerializerProvider().getSerializer(ALL_DATA_TYPES_SCHEMA, ImmutableMap.of()); - byte[] expectedJson = jsonPayloadSerializer.serialize(withAllDataTypes); - byte[] actualJson = - transform(configurationBuilder().setFormat("json")) - .getPayloadSerializer(ALL_DATA_TYPES_SCHEMA) - .serialize(withAllDataTypes); - - PayloadSerializer avroPayloadSerializer = - new AvroPayloadSerializerProvider().getSerializer(ALL_DATA_TYPES_SCHEMA, ImmutableMap.of()); - byte[] expectedAvro = avroPayloadSerializer.serialize(withAllDataTypes); - byte[] actualAvro = - transform(configurationBuilder().setFormat("avro")) - .getPayloadSerializer(ALL_DATA_TYPES_SCHEMA) - .serialize(withAllDataTypes); - - assertArrayEquals( - "configuration with json format should yield JSON PayloadSerializer", - expectedJson, - actualJson); - - assertArrayEquals( - "configuration with avro format should yield Avro PayloadSerializer", - expectedAvro, - actualAvro); - } - - private static PubsubWriteSchemaTransformConfiguration.Builder configurationBuilder() { - return PubsubWriteSchemaTransformConfiguration.builder() - .setTopic(TOPIC) - .setTarget(PubsubWriteSchemaTransformConfiguration.targetConfigurationBuilder().build()); - } - - private static PubsubRowToMessage.Builder rowToMessageBuilder() { - return PubsubRowToMessage.builder(); - } - - private static PubsubIO.Write pubsubWrite() { - return PubsubIO.writeMessages().to(TOPIC); - } - - private static PubsubWriteSchemaTransformProvider.PubsubWriteSchemaTransform transform( - PubsubWriteSchemaTransformConfiguration.Builder configurationBuilder) { - Row configurationRow = TO_ROW.apply(configurationBuilder.build()); - PubsubWriteSchemaTransformProvider provider = new PubsubWriteSchemaTransformProvider(); - return (PubsubWriteSchemaTransform) provider.from(configurationRow); - } - - private static class MockClock implements Clock, Serializable { - private final Long millis; - - private MockClock(Instant timestamp) { - this.millis = timestamp.getMillis(); - } - - @Override - public long currentTimeMillis() { - return millis; - } - } -} diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/ReadWriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/ReadWriteIT.java index ff55ca9b258db..1f2819940ff28 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/ReadWriteIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/ReadWriteIT.java @@ -63,7 +63,7 @@ import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; import org.joda.time.Duration; import org.junit.After; import org.junit.Rule; @@ -203,8 +203,7 @@ public static void writeJsonMessages(TopicPath topicPath, Pipeline pipeline) { .setLocation(ZONE.toString()) .setTopicName(topicPath.name().value()) .setProject(topicPath.project().name().value()) - .build()) - .buildTransform()); + .build())); } public static void writeMessages(TopicPath topicPath, Pipeline pipeline) { @@ -308,8 +307,7 @@ public void testPubsubLiteWriteReadWithSchemaTransform() throws Exception { + "}") .setSubscriptionName(subscription.name().value()) .setLocation(subscription.location().toString()) - .build()) - .buildTransform()) + .build())) .get("output"); PCollection ids = messages.apply( @@ -320,7 +318,7 @@ public void testPubsubLiteWriteReadWithSchemaTransform() throws Exception { return Objects.requireNonNull(row.getInt64("numberInInt")).intValue(); })); ids.apply("PubsubSignalTest", signal.signalSuccessWhen(BigEndianIntegerCoder.of(), testIds())); - Supplier start = signal.waitForStart(Duration.standardMinutes(5)); + Supplier start = signal.waitForStart(Duration.standardMinutes(8)); pipeline.apply("start signal", signal.signalStart()); PipelineResult job = pipeline.run(); start.get(); @@ -365,7 +363,7 @@ public void testReadWrite() throws Exception { PCollection messages = readMessages(subscription, pipeline); PCollection ids = messages.apply(MapElements.via(extractIds())); ids.apply("PubsubSignalTest", signal.signalSuccessWhen(BigEndianIntegerCoder.of(), testIds())); - Supplier start = signal.waitForStart(Duration.standardMinutes(5)); + Supplier start = signal.waitForStart(Duration.standardMinutes(8)); pipeline.apply(signal.signalStart()); PipelineResult job = pipeline.run(); start.get(); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/AddUuidsTransformTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/AddUuidsTransformTest.java index 8174b32202410..331613e5d7f06 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/AddUuidsTransformTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/AddUuidsTransformTest.java @@ -27,8 +27,8 @@ import org.apache.beam.sdk.testing.TestStream; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.joda.time.Duration; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/MemoryBufferedSubscriberImplTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/MemoryBufferedSubscriberImplTest.java index 456b5b4bd56e0..6026727040781 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/MemoryBufferedSubscriberImplTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/MemoryBufferedSubscriberImplTest.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.gcp.pubsublite.internal; import static com.google.cloud.pubsublite.internal.testing.UnitTestExamples.example; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThrows; @@ -41,7 +41,7 @@ import java.util.List; import java.util.function.Consumer; import java.util.function.Function; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/OffsetByteRangeTrackerTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/OffsetByteRangeTrackerTest.java index 1ba4582d3eaa6..667fd52a8b1f8 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/OffsetByteRangeTrackerTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/OffsetByteRangeTrackerTest.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.io.range.OffsetRange; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker.Progress; import org.apache.beam.sdk.transforms.splittabledofn.SplitResult; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Ticker; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Ticker; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PerSubscriptionPartitionSdfTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PerSubscriptionPartitionSdfTest.java index a6624ac2c3aa3..7047d35b21ac6 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PerSubscriptionPartitionSdfTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PerSubscriptionPartitionSdfTest.java @@ -50,7 +50,7 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker.Progress; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.math.DoubleMath; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.math.DoubleMath; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/UuidDeduplicationTransformTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/UuidDeduplicationTransformTest.java index f01f26aeb4518..d390cc5ea127b 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/UuidDeduplicationTransformTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/UuidDeduplicationTransformTest.java @@ -30,8 +30,8 @@ import org.apache.beam.sdk.testing.TestStream; import org.apache.beam.sdk.transforms.Deduplicate; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationKeyEncoderTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationKeyEncoderTest.java index b7be1f579d7d2..fb76eaec93684 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationKeyEncoderTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationKeyEncoderTest.java @@ -30,8 +30,8 @@ import java.util.Arrays; import java.util.Comparator; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.TreeMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.UnsignedBytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.TreeMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedBytes; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java index dcf80a5f18a58..f05159cbbe359 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java @@ -33,7 +33,7 @@ import com.google.cloud.spanner.Value; import java.math.BigDecimal; import java.util.Arrays; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationUtilsTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationUtilsTest.java index 137c339ac690f..434ffdc52ebe3 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationUtilsTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationUtilsTest.java @@ -31,7 +31,7 @@ import java.util.List; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.DateTime; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/OrderedCodeTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/OrderedCodeTest.java index 522ff786cbadb..8e67232ed3738 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/OrderedCodeTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/OrderedCodeTest.java @@ -25,11 +25,11 @@ import com.google.auto.value.AutoValue; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Bytes; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.UnsignedBytes; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.UnsignedInteger; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Bytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedBytes; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedInteger; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOWriteExceptionHandlingTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOWriteExceptionHandlingTest.java index 87f77a74e6bfe..5a2a5ccfa5cd5 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOWriteExceptionHandlingTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOWriteExceptionHandlingTest.java @@ -41,7 +41,7 @@ import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOWriteTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOWriteTest.java index ecc6ba31ee4e3..9cd6008a94667 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOWriteTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOWriteTest.java @@ -83,9 +83,9 @@ import org.apache.beam.sdk.util.Sleeper; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Duration; import org.junit.Before; import org.junit.Rule; @@ -758,12 +758,22 @@ public void deadlineExceededFailsAfterRetries() throws InterruptedException { @Test public void retryOnSchemaChangeException() throws InterruptedException { - List mutationList = Arrays.asList(buildUpsertMutation((long) 1)); - String errString = "Transaction aborted. " + "Database schema probably changed during transaction, retry may succeed."; + retryOnAbortedExceptionWithMessage(errString); + } + @Test + public void retryOnEmulatorRejectedConcurrentTransaction() throws InterruptedException { + String errString = + "Transaction 199 aborted due to active transaction 167. " + + "The emulator only supports one transaction at a time."; + retryOnAbortedExceptionWithMessage(errString); + } + + public void retryOnAbortedExceptionWithMessage(String errString) throws InterruptedException { + List mutationList = Arrays.asList(buildUpsertMutation((long) 1)); // mock sleeper so that it does not actually sleep. WriteToSpannerFn.sleeper = Mockito.mock(Sleeper.class); @@ -806,11 +816,22 @@ public void retryOnSchemaChangeException() throws InterruptedException { @Test public void retryMaxOnSchemaChangeException() throws InterruptedException { - List mutationList = Arrays.asList(buildUpsertMutation((long) 1)); - String errString = "Transaction aborted. " + "Database schema probably changed during transaction, retry may succeed."; + retryMaxOnAbortedExceptionWithMessage(errString); + } + + @Test + public void retryMaxOnEmulatorRejectedConcurrentTransaction() throws InterruptedException { + String errString = + "Transaction 199 aborted due to active transaction 167. " + + "The emulator only supports one transaction at a time."; + retryOnAbortedExceptionWithMessage(errString); + } + + public void retryMaxOnAbortedExceptionWithMessage(String errString) throws InterruptedException { + List mutationList = Arrays.asList(buildUpsertMutation((long) 1)); // mock sleeper so that it does not actually sleep. WriteToSpannerFn.sleeper = Mockito.mock(Sleeper.class); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java index 9594633d2dc4b..75053a6efe2db 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java @@ -54,9 +54,9 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicate; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicate; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; import org.checkerframework.checker.nullness.qual.Nullable; import org.hamcrest.TypeSafeMatcher; import org.junit.After; @@ -231,8 +231,7 @@ public void testWriteViaSchemaTransform() throws Exception { .setDatabaseId(databaseName) .setInstanceId(options.getInstanceId()) .setTableId(options.getTable()) - .build()) - .buildTransform()); + .build())); PipelineResult result = p.run(); result.waitUntilFinish(); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/StructUtilsTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/StructUtilsTest.java index 8d0ab30855525..90c132666f58b 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/StructUtilsTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/StructUtilsTest.java @@ -34,8 +34,8 @@ import java.util.List; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTime; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/SpannerChangeStreamErrorTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/SpannerChangeStreamErrorTest.java index bf2ccd454bb5b..9ffa61c930781 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/SpannerChangeStreamErrorTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/SpannerChangeStreamErrorTest.java @@ -52,7 +52,9 @@ import com.google.spanner.v1.TypeCode; import io.grpc.Status; import java.io.Serializable; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import org.apache.beam.runners.direct.DirectOptions; import org.apache.beam.runners.direct.DirectRunner; import org.apache.beam.sdk.Pipeline; @@ -68,7 +70,6 @@ import org.joda.time.Duration; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -114,16 +115,22 @@ public void tearDown() throws NoSuchFieldException, IllegalAccessException { } @Test - @Ignore("BEAM-12164 Reenable this test when databaseClient.getDialect returns the right message.") - public void testResourceExhaustedDoesNotRetry() { + // Error code UNAVAILABLE is retried repeatedly until the RPC times out. + public void testUnavailableExceptionRetries() throws InterruptedException { + DirectOptions options = PipelineOptionsFactory.as(DirectOptions.class); + options.setBlockOnRun(false); + options.setRunner(DirectRunner.class); + Pipeline nonBlockingPipeline = TestPipeline.create(options); + mockSpannerService.setExecuteStreamingSqlExecutionTime( - SimulatedExecutionTime.ofStickyException(Status.RESOURCE_EXHAUSTED.asRuntimeException())); + SimulatedExecutionTime.ofStickyException(Status.UNAVAILABLE.asRuntimeException())); final Timestamp startTimestamp = Timestamp.ofTimeSecondsAndNanos(0, 1000); final Timestamp endTimestamp = Timestamp.ofTimeSecondsAndNanos(startTimestamp.getSeconds(), startTimestamp.getNanos() + 1); + try { - pipeline.apply( + nonBlockingPipeline.apply( SpannerIO.readChangeStream() .withSpannerConfig(getSpannerConfig()) .withChangeStreamName(TEST_CHANGE_STREAM) @@ -131,33 +138,36 @@ public void testResourceExhaustedDoesNotRetry() { .withMetadataTable(TEST_TABLE) .withInclusiveStartAt(startTimestamp) .withInclusiveEndAt(endTimestamp)); - pipeline.run().waitUntilFinish(); + PipelineResult result = nonBlockingPipeline.run(); + while (result.getState() != RUNNING) { + Thread.sleep(50); + } + // The pipeline continues making requests to Spanner to retry the Unavailable errors. + assertNull(result.waitUntilFinish(Duration.millis(500))); } finally { - thrown.expect(SpannerException.class); // databaseClient.getDialect does not currently bubble up the correct message. // Instead, the error returned is: "DEADLINE_EXCEEDED: Operation did not complete " // "in the given time" - thrown.expectMessage("RESOURCE_EXHAUSTED - Statement: 'SELECT 'POSTGRESQL' AS DIALECT"); + thrown.expectMessage("DEADLINE_EXCEEDED"); assertThat( mockSpannerService.countRequestsOfType(ExecuteSqlRequest.class), Matchers.equalTo(0)); } } @Test - @Ignore("BEAM-12164 Reenable this test when databaseClient.getDialect returns the right message.") - public void testUnavailableExceptionRetries() throws InterruptedException { + // Error code ABORTED is retried repeatedly until it times out. + public void testAbortedExceptionRetries() throws InterruptedException { + mockSpannerService.setExecuteStreamingSqlExecutionTime( + SimulatedExecutionTime.ofStickyException(Status.ABORTED.asRuntimeException())); + DirectOptions options = PipelineOptionsFactory.as(DirectOptions.class); options.setBlockOnRun(false); options.setRunner(DirectRunner.class); Pipeline nonBlockingPipeline = TestPipeline.create(options); - mockSpannerService.setExecuteStreamingSqlExecutionTime( - SimulatedExecutionTime.ofStickyException(Status.UNAVAILABLE.asRuntimeException())); - final Timestamp startTimestamp = Timestamp.ofTimeSecondsAndNanos(0, 1000); final Timestamp endTimestamp = Timestamp.ofTimeSecondsAndNanos(startTimestamp.getSeconds(), startTimestamp.getNanos() + 1); - try { nonBlockingPipeline.apply( SpannerIO.readChangeStream() @@ -171,23 +181,20 @@ public void testUnavailableExceptionRetries() throws InterruptedException { while (result.getState() != RUNNING) { Thread.sleep(50); } - // The pipeline continues making requests to Spanner to retry the Unavailable errors. + // The pipeline continues making requests to Spanner to retry the Aborted errors. assertNull(result.waitUntilFinish(Duration.millis(500))); } finally { - // databaseClient.getDialect does not currently bubble up the correct message. - // Instead, the error returned is: "DEADLINE_EXCEEDED: Operation did not complete " - // "in the given time" - thrown.expectMessage("UNAVAILABLE - Statement: 'SELECT 'POSTGRESQL' AS DIALECT"); + thrown.expectMessage("DEADLINE_EXCEEDED"); assertThat( mockSpannerService.countRequestsOfType(ExecuteSqlRequest.class), Matchers.equalTo(0)); } } @Test - @Ignore("BEAM-12164 Reenable this test when databaseClient.getDialect returns the right message.") - public void testAbortedExceptionNotRetried() { + // Error code UNKNOWN is not retried. + public void testUnknownExceptionDoesNotRetry() { mockSpannerService.setExecuteStreamingSqlExecutionTime( - SimulatedExecutionTime.ofStickyException(Status.ABORTED.asRuntimeException())); + SimulatedExecutionTime.ofStickyException(Status.UNKNOWN.asRuntimeException())); final Timestamp startTimestamp = Timestamp.ofTimeSecondsAndNanos(0, 1000); final Timestamp endTimestamp = @@ -204,19 +211,43 @@ public void testAbortedExceptionNotRetried() { pipeline.run().waitUntilFinish(); } finally { thrown.expect(SpannerException.class); - // databaseClient.getDialect does not currently bubble up the correct message. - // Instead, the error returned is: "DEADLINE_EXCEEDED: Operation did not complete " - // "in the given time" - thrown.expectMessage("ABORTED - Statement: 'SELECT 'POSTGRESQL' AS DIALECT"); + thrown.expectMessage("UNKNOWN"); assertThat( mockSpannerService.countRequestsOfType(ExecuteSqlRequest.class), Matchers.equalTo(0)); } } @Test - public void testAbortedExceptionNotRetriedithDefaultsForStreamSqlRetrySettings() { + // Error code RESOURCE_EXHAUSTED is retried repeatedly. + public void testResourceExhaustedRetry() { mockSpannerService.setExecuteStreamingSqlExecutionTime( - SimulatedExecutionTime.ofStickyException(Status.ABORTED.asRuntimeException())); + SimulatedExecutionTime.ofStickyException(Status.RESOURCE_EXHAUSTED.asRuntimeException())); + + final Timestamp startTimestamp = Timestamp.ofTimeSecondsAndNanos(0, 1000); + final Timestamp endTimestamp = + Timestamp.ofTimeSecondsAndNanos(startTimestamp.getSeconds(), startTimestamp.getNanos() + 1); + + try { + pipeline.apply( + SpannerIO.readChangeStream() + .withSpannerConfig(getSpannerConfig()) + .withChangeStreamName(TEST_CHANGE_STREAM) + .withMetadataDatabase(TEST_DATABASE) + .withMetadataTable(TEST_TABLE) + .withInclusiveStartAt(startTimestamp) + .withInclusiveEndAt(endTimestamp)); + pipeline.run().waitUntilFinish(); + } finally { + thrown.expectMessage("DEADLINE_EXCEEDED"); + assertThat( + mockSpannerService.countRequestsOfType(ExecuteSqlRequest.class), Matchers.equalTo(0)); + } + } + + @Test + public void testResourceExhaustedRetryWithDefaultSettings() { + mockSpannerService.setExecuteStreamingSqlExecutionTime( + SimulatedExecutionTime.ofStickyException(Status.RESOURCE_EXHAUSTED.asRuntimeException())); final Timestamp startTimestamp = Timestamp.ofTimeSecondsAndNanos(0, 1000); final Timestamp endTimestamp = @@ -230,6 +261,7 @@ public void testAbortedExceptionNotRetriedithDefaultsForStreamSqlRetrySettings() .withProjectId(TEST_PROJECT) .withInstanceId(TEST_INSTANCE) .withDatabaseId(TEST_DATABASE); + try { pipeline.apply( SpannerIO.readChangeStream() @@ -241,24 +273,34 @@ public void testAbortedExceptionNotRetriedithDefaultsForStreamSqlRetrySettings() .withInclusiveEndAt(endTimestamp)); pipeline.run().waitUntilFinish(); } finally { - // databaseClient.getDialect does not currently bubble up the correct message. - // Instead, the error returned is: "DEADLINE_EXCEEDED: Operation did not complete " - // "in the given time" thrown.expect(SpannerException.class); - thrown.expectMessage("ABORTED - Statement: 'SELECT 'POSTGRESQL' AS DIALECT"); + thrown.expectMessage("RESOURCE_EXHAUSTED"); assertThat( mockSpannerService.countRequestsOfType(ExecuteSqlRequest.class), Matchers.equalTo(0)); } } @Test - public void testUnknownExceptionDoesNotRetry() { - mockSpannerService.setExecuteStreamingSqlExecutionTime( - SimulatedExecutionTime.ofStickyException(Status.UNKNOWN.asRuntimeException())); - + public void testInvalidRecordReceived() { final Timestamp startTimestamp = Timestamp.ofTimeSecondsAndNanos(0, 1000); final Timestamp endTimestamp = Timestamp.ofTimeSecondsAndNanos(startTimestamp.getSeconds(), startTimestamp.getNanos() + 1); + + mockGetDialect(); + mockTableExists(); + mockGetWatermark(startTimestamp); + ResultSet getPartitionResultSet = mockGetParentPartition(startTimestamp, endTimestamp); + mockGetPartitionsAfter( + Timestamp.ofTimeSecondsAndNanos(startTimestamp.getSeconds(), startTimestamp.getNanos() - 1), + getPartitionResultSet); + mockGetPartitionsAfter( + Timestamp.ofTimeSecondsAndNanos(startTimestamp.getSeconds(), startTimestamp.getNanos()), + ResultSet.newBuilder().setMetadata(PARTITION_METADATA_RESULT_SET_METADATA).build()); + mockGetPartitionsAfter( + Timestamp.ofTimeSecondsAndNanos(startTimestamp.getSeconds(), startTimestamp.getNanos() + 1), + ResultSet.newBuilder().setMetadata(PARTITION_METADATA_RESULT_SET_METADATA).build()); + mockInvalidChangeStreamRecordReceived(startTimestamp, endTimestamp); + try { pipeline.apply( SpannerIO.readChangeStream() @@ -271,15 +313,16 @@ public void testUnknownExceptionDoesNotRetry() { pipeline.run().waitUntilFinish(); } finally { thrown.expect(SpannerException.class); - thrown.expectMessage("UNKNOWN - Statement: 'SELECT 'POSTGRESQL' AS DIALECT"); + // DatabaseClient.getDialect returns "DEADLINE_EXCEEDED: Operation did not complete in the " + // given time" even though we mocked it out. + thrown.expectMessage("DEADLINE_EXCEEDED"); assertThat( mockSpannerService.countRequestsOfType(ExecuteSqlRequest.class), Matchers.equalTo(0)); } } @Test - @Ignore("BEAM-12164 Reenable this test when databaseClient.getDialect works.") - public void testInvalidRecordReceived() { + public void testInvalidRecordReceivedWithDefaultSettings() { final Timestamp startTimestamp = Timestamp.ofTimeSecondsAndNanos(0, 1000); final Timestamp endTimestamp = Timestamp.ofTimeSecondsAndNanos(startTimestamp.getSeconds(), startTimestamp.getNanos() + 1); @@ -288,6 +331,8 @@ public void testInvalidRecordReceived() { mockTableExists(); mockGetWatermark(startTimestamp); ResultSet getPartitionResultSet = mockGetParentPartition(startTimestamp, endTimestamp); + mockchangePartitionState(startTimestamp, endTimestamp, "CREATED"); + mockchangePartitionState(startTimestamp, endTimestamp, "SCHEDULED"); mockGetPartitionsAfter( Timestamp.ofTimeSecondsAndNanos(startTimestamp.getSeconds(), startTimestamp.getNanos() - 1), getPartitionResultSet); @@ -300,9 +345,26 @@ public void testInvalidRecordReceived() { mockInvalidChangeStreamRecordReceived(startTimestamp, endTimestamp); try { + RetrySettings quickRetrySettings = + RetrySettings.newBuilder() + .setInitialRetryDelay(org.threeten.bp.Duration.ofMillis(250)) + .setMaxRetryDelay(org.threeten.bp.Duration.ofSeconds(1)) + .setRetryDelayMultiplier(5) + .setTotalTimeout(org.threeten.bp.Duration.ofSeconds(1)) + .build(); + final SpannerConfig changeStreamConfig = + SpannerConfig.create() + .withEmulatorHost(StaticValueProvider.of(SPANNER_HOST)) + .withIsLocalChannelProvider(StaticValueProvider.of(true)) + .withCommitRetrySettings(quickRetrySettings) + .withExecuteStreamingSqlRetrySettings(null) + .withProjectId(TEST_PROJECT) + .withInstanceId(TEST_INSTANCE) + .withDatabaseId(TEST_DATABASE); + pipeline.apply( SpannerIO.readChangeStream() - .withSpannerConfig(getSpannerConfig()) + .withSpannerConfig(changeStreamConfig) .withChangeStreamName(TEST_CHANGE_STREAM) .withMetadataDatabase(TEST_DATABASE) .withMetadataTable(TEST_TABLE) @@ -311,11 +373,9 @@ public void testInvalidRecordReceived() { pipeline.run().waitUntilFinish(); } finally { thrown.expect(PipelineExecutionException.class); - // DatabaseClient.getDialect returns "DEADLINE_EXCEEDED: Operation did not complete in the " - // given time" even though we mocked it out. thrown.expectMessage("Field not found"); assertThat( - mockSpannerService.countRequestsOfType(ExecuteSqlRequest.class), Matchers.equalTo(0)); + mockSpannerService.countRequestsOfType(ExecuteSqlRequest.class), Matchers.greaterThan(0)); } } @@ -487,6 +547,41 @@ private void mockTableExists() { StatementResult.query(tableExistsStatement, tableExistsResultSet)); } + private ResultSet mockchangePartitionState( + Timestamp startTimestamp, Timestamp after3Seconds, String state) { + List tokens = new ArrayList<>(); + tokens.add("Parent0"); + Statement getPartitionStatement = + Statement.newBuilder( + "SELECT * FROM my-metadata-table WHERE PartitionToken IN UNNEST(@partitionTokens) AND State = @state") + .bind("partitionTokens") + .toStringArray(tokens) + .bind("state") + .to(state) + .build(); + ResultSet getPartitionResultSet = + ResultSet.newBuilder() + .addRows( + ListValue.newBuilder() + .addValues(Value.newBuilder().setStringValue("Parent0")) + .addValues(Value.newBuilder().setListValue(ListValue.newBuilder().build())) + .addValues(Value.newBuilder().setStringValue(startTimestamp.toString())) + .addValues(Value.newBuilder().setStringValue(after3Seconds.toString())) + .addValues(Value.newBuilder().setStringValue("500")) + .addValues(Value.newBuilder().setStringValue(State.CREATED.name())) + .addValues(Value.newBuilder().setStringValue(startTimestamp.toString())) + .addValues(Value.newBuilder().setStringValue(startTimestamp.toString())) + .addValues(Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build()) + .addValues(Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build()) + .addValues(Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build()) + .build()) + .setMetadata(PARTITION_METADATA_RESULT_SET_METADATA) + .build(); + mockSpannerService.putStatementResult( + StatementResult.query(getPartitionStatement, getPartitionResultSet)); + return getPartitionResultSet; + } + private void mockGetDialect() { Statement determineDialectStatement = Statement.newBuilder( diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/ChildPartitionsRecordActionTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/ChildPartitionsRecordActionTest.java index 63efaca0c82d5..03d390ea0d5d5 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/ChildPartitionsRecordActionTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/ChildPartitionsRecordActionTest.java @@ -43,7 +43,7 @@ import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation; import org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.joda.time.Instant; import org.junit.Before; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/QueryChangeStreamActionTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/QueryChangeStreamActionTest.java index 0845c054fd5ce..bf7b0adfd475b 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/QueryChangeStreamActionTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/action/QueryChangeStreamActionTest.java @@ -46,7 +46,7 @@ import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation; import org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.joda.time.Instant; import org.junit.Before; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/dao/PartitionMetadataDaoTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/dao/PartitionMetadataDaoTest.java index 907fdfb365305..83d1d8f2aa5e8 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/dao/PartitionMetadataDaoTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/dao/PartitionMetadataDaoTest.java @@ -41,8 +41,8 @@ import java.util.Map; import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.PartitionMetadata; import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.PartitionMetadata.State; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/dofn/ReadChangeStreamPartitionDoFnTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/dofn/ReadChangeStreamPartitionDoFnTest.java index d9af6cecd95cf..538bdf768664e 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/dofn/ReadChangeStreamPartitionDoFnTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/dofn/ReadChangeStreamPartitionDoFnTest.java @@ -47,7 +47,7 @@ import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation; import org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator; import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.joda.time.Instant; import org.junit.Before; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/it/SpannerChangeStreamsSchemaTransformIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/it/SpannerChangeStreamsSchemaTransformIT.java index cae1b782eed97..bac248f4861df 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/it/SpannerChangeStreamsSchemaTransformIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/it/SpannerChangeStreamsSchemaTransformIT.java @@ -113,8 +113,7 @@ public void testReadSpannerChangeStream() { .setChangeStreamName(changeStreamName) .setStartAtTimestamp(startAt.toString()) .setEndAtTimestamp(endAt.toString()) - .build()) - .buildTransform()) + .build())) .get("output") .apply( Window.into(new GlobalWindows()) diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/mapper/ChangeStreamRecordMapperTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/mapper/ChangeStreamRecordMapperTest.java index 52438ad33615d..05ed0bbae6cc0 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/mapper/ChangeStreamRecordMapperTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/mapper/ChangeStreamRecordMapperTest.java @@ -45,7 +45,7 @@ import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.PartitionMetadata.State; import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.TypeCode; import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.ValueCaptureType; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.joda.time.Duration; import org.junit.Before; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/mapper/PartitionMetadataMapperTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/mapper/PartitionMetadataMapperTest.java index bdf562a9e0389..717f436b59984 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/mapper/PartitionMetadataMapperTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/mapper/PartitionMetadataMapperTest.java @@ -35,7 +35,7 @@ import java.util.Collections; import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.PartitionMetadata; import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.PartitionMetadata.State; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.junit.Before; import org.junit.Test; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/ModelEncodingTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/ModelEncodingTest.java index ef538eb32c817..37612c7d2e936 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/ModelEncodingTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/ModelEncodingTest.java @@ -34,7 +34,7 @@ import org.apache.avro.reflect.ReflectDatumReader; import org.apache.avro.reflect.ReflectDatumWriter; import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.PartitionMetadata.State; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.junit.Test; public class ModelEncodingTest { diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/PartitionMetadataTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/PartitionMetadataTest.java index e32c9cb82e921..2ce03648746bd 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/PartitionMetadataTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/PartitionMetadataTest.java @@ -23,7 +23,7 @@ import com.google.cloud.Timestamp; import com.google.cloud.spanner.Value; import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.PartitionMetadata.State; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsKmsKeyIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsKmsKeyIT.java index 66fe8bf4ae9f8..94f632e7daa1b 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsKmsKeyIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsKmsKeyIT.java @@ -41,7 +41,7 @@ import org.apache.beam.sdk.testing.TestPipelineOptions; import org.apache.beam.sdk.testing.UsesKms; import org.apache.beam.sdk.util.NumberedShardedFile; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsMatchIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsMatchIT.java index 088fcd064952a..336bcd768f7aa 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsMatchIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsMatchIT.java @@ -49,9 +49,9 @@ import org.apache.beam.sdk.util.MimeTypes; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.FluentIterable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; import org.joda.time.Duration; import org.junit.AfterClass; import org.junit.BeforeClass; diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/testing/BigqueryMatcherTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/testing/BigqueryMatcherTest.java index 508dfecc58a44..c6f863a536cd9 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/testing/BigqueryMatcherTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/testing/BigqueryMatcherTest.java @@ -26,7 +26,7 @@ import com.google.api.services.bigquery.model.TableCell; import com.google.api.services.bigquery.model.TableRow; import java.math.BigInteger; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/hadoop-common/OWNERS b/sdks/java/io/hadoop-common/OWNERS deleted file mode 100644 index 09812f7fdedc1..0000000000000 --- a/sdks/java/io/hadoop-common/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - timrobertson100 - - aromanenko-dev diff --git a/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/SerializableConfiguration.java b/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/SerializableConfiguration.java index bb59b07cc280c..40099e6346786 100644 --- a/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/SerializableConfiguration.java +++ b/sdks/java/io/hadoop-common/src/main/java/org/apache/beam/sdk/io/hadoop/SerializableConfiguration.java @@ -17,6 +17,8 @@ */ package org.apache.beam.sdk.io.hadoop; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; import java.io.Externalizable; import java.io.IOException; import java.io.ObjectInput; @@ -39,6 +41,8 @@ public class SerializableConfiguration implements Externalizable { private transient Configuration conf; + private transient byte[] serializationCache; + public SerializableConfiguration() {} public SerializableConfiguration(Configuration conf) { @@ -49,17 +53,30 @@ public SerializableConfiguration(Configuration conf) { } public Configuration get() { + if (serializationCache != null) { + serializationCache = null; + } return conf; } @Override public void writeExternal(ObjectOutput out) throws IOException { + if (serializationCache == null) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(512); + try (DataOutputStream dos = new DataOutputStream(baos)) { + conf.write(dos); + serializationCache = baos.toByteArray(); + } + } out.writeUTF(conf.getClass().getCanonicalName()); - conf.write(out); + out.write(serializationCache); } @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + if (serializationCache != null) { + serializationCache = null; + } String className = in.readUTF(); try { conf = diff --git a/sdks/java/io/hadoop-common/src/test/java/org/apache/beam/sdk/io/hadoop/SerializableConfigurationTest.java b/sdks/java/io/hadoop-common/src/test/java/org/apache/beam/sdk/io/hadoop/SerializableConfigurationTest.java index 135b5ec349fac..702fe64412492 100644 --- a/sdks/java/io/hadoop-common/src/test/java/org/apache/beam/sdk/io/hadoop/SerializableConfigurationTest.java +++ b/sdks/java/io/hadoop-common/src/test/java/org/apache/beam/sdk/io/hadoop/SerializableConfigurationTest.java @@ -21,7 +21,7 @@ import static org.junit.Assert.assertNotNull; import org.apache.beam.repackaged.core.org.apache.commons.lang3.SerializationUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.Job; import org.junit.Rule; diff --git a/sdks/java/io/hadoop-file-system/OWNERS b/sdks/java/io/hadoop-file-system/OWNERS deleted file mode 100644 index 09812f7fdedc1..0000000000000 --- a/sdks/java/io/hadoop-file-system/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - timrobertson100 - - aromanenko-dev diff --git a/sdks/java/io/hadoop-file-system/build.gradle b/sdks/java/io/hadoop-file-system/build.gradle index 879cb0fafb15e..3fc872bb5d02c 100644 --- a/sdks/java/io/hadoop-file-system/build.gradle +++ b/sdks/java/io/hadoop-file-system/build.gradle @@ -35,7 +35,7 @@ def hadoopVersions = [ hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")} dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.jackson_core implementation library.java.jackson_databind diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java index 7fe93616b73bb..4de1d539e76a4 100644 --- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java +++ b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java @@ -38,8 +38,8 @@ import org.apache.beam.sdk.io.fs.MatchResult.Metadata; import org.apache.beam.sdk.io.fs.MatchResult.Status; import org.apache.beam.sdk.io.fs.MoveOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSInputStream; diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptions.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptions.java index 0783fa75b5207..a22dc37bafb88 100644 --- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptions.java +++ b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptions.java @@ -26,11 +26,11 @@ import org.apache.beam.sdk.options.DefaultValueFactory; import org.apache.beam.sdk.options.Description; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptionsRegistrar.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptionsRegistrar.java index 01c528ef6b712..efa62bc0f4244 100644 --- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptionsRegistrar.java +++ b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptionsRegistrar.java @@ -20,7 +20,7 @@ import com.google.auto.service.AutoService; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** {@link AutoService} registrar for {@link HadoopFileSystemOptions}. */ @AutoService(PipelineOptionsRegistrar.class) diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrar.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrar.java index 7c2ed4ce03f50..a5dc5523e13d1 100644 --- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrar.java +++ b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrar.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.hdfs; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.service.AutoService; import java.net.URI; @@ -31,15 +31,15 @@ import org.apache.beam.sdk.io.FileSystem; import org.apache.beam.sdk.io.FileSystemRegistrar; import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.hadoop.conf.Configuration; /** {@link AutoService} registrar for the {@link HadoopFileSystem}. */ @AutoService(FileSystemRegistrar.class) public class HadoopFileSystemRegistrar implements FileSystemRegistrar { - private static final List HA_SCHEMES = Arrays.asList("hdfs", "webhdfs"); + private static final List HA_SCHEMES = Arrays.asList("hdfs", "webhdfs", "viewfs"); // Using hard-coded value to avoid incompatibility between HDFS client // (org.apache.hadoop:hadoop-dfs-client) version 2.7's DFSConfigKeys and version 2.8's diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopResourceId.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopResourceId.java index 3fcea976dd2ed..8884271bbd7c6 100644 --- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopResourceId.java +++ b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopResourceId.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.hdfs; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.net.URI; import java.util.Objects; diff --git a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptionsRegistrarTest.java b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptionsRegistrarTest.java index ecfc037bb14db..6024942a994c3 100644 --- a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptionsRegistrarTest.java +++ b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptionsRegistrarTest.java @@ -22,7 +22,7 @@ import java.util.ServiceLoader; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.hamcrest.Matchers; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptionsTest.java b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptionsTest.java index 5e75b0293ebb0..c8e7d8db5fc9e 100644 --- a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptionsTest.java +++ b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemOptionsTest.java @@ -30,9 +30,9 @@ import java.util.List; import java.util.Map; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; import org.apache.hadoop.conf.Configuration; import org.hamcrest.Matchers; import org.junit.Rule; @@ -74,10 +74,10 @@ public void testDefaultUnsetEnvHdfsConfiguration() { @Test public void testDefaultJustSetHadoopConfDirConfiguration() throws IOException { - Files.write( - createPropertyData("A"), tmpFolder.newFile("core-site.xml"), StandardCharsets.UTF_8); - Files.write( - createPropertyData("B"), tmpFolder.newFile("hdfs-site.xml"), StandardCharsets.UTF_8); + Files.asCharSink(tmpFolder.newFile("core-site.xml"), StandardCharsets.UTF_8) + .write(createPropertyData("A")); + Files.asCharSink(tmpFolder.newFile("hdfs-site.xml"), StandardCharsets.UTF_8) + .write(createPropertyData("B")); HadoopFileSystemOptions.ConfigurationLocator configurationLocator = spy(new HadoopFileSystemOptions.ConfigurationLocator()); Map environment = Maps.newHashMap(); @@ -96,10 +96,10 @@ public void testDefaultJustSetHadoopConfDirMultiPathConfiguration() throws IOExc File hadoopConfDir = tmpFolder.newFolder("hadoop"); File otherConfDir = tmpFolder.newFolder("_other_"); - Files.write( - createPropertyData("A"), new File(hadoopConfDir, "core-site.xml"), StandardCharsets.UTF_8); - Files.write( - createPropertyData("B"), new File(hadoopConfDir, "hdfs-site.xml"), StandardCharsets.UTF_8); + Files.asCharSink(new File(hadoopConfDir, "core-site.xml"), StandardCharsets.UTF_8) + .write(createPropertyData("A")); + Files.asCharSink(new File(hadoopConfDir, "hdfs-site.xml"), StandardCharsets.UTF_8) + .write(createPropertyData("B")); HadoopFileSystemOptions.ConfigurationLocator configurationLocator = spy(new HadoopFileSystemOptions.ConfigurationLocator()); @@ -119,10 +119,10 @@ public void testDefaultJustSetHadoopConfDirMultiPathConfiguration() throws IOExc @Test public void testDefaultJustSetYarnConfDirConfiguration() throws IOException { - Files.write( - createPropertyData("A"), tmpFolder.newFile("core-site.xml"), StandardCharsets.UTF_8); - Files.write( - createPropertyData("B"), tmpFolder.newFile("hdfs-site.xml"), StandardCharsets.UTF_8); + Files.asCharSink(tmpFolder.newFile("core-site.xml"), StandardCharsets.UTF_8) + .write(createPropertyData("A")); + Files.asCharSink(tmpFolder.newFile("hdfs-site.xml"), StandardCharsets.UTF_8) + .write(createPropertyData("B")); HadoopFileSystemOptions.ConfigurationLocator configurationLocator = spy(new HadoopFileSystemOptions.ConfigurationLocator()); Map environment = Maps.newHashMap(); @@ -141,10 +141,10 @@ public void testDefaultJustSetYarnConfDirMultiPathConfiguration() throws IOExcep File hadoopConfDir = tmpFolder.newFolder("hadoop"); File otherConfDir = tmpFolder.newFolder("_other_"); - Files.write( - createPropertyData("A"), new File(hadoopConfDir, "core-site.xml"), StandardCharsets.UTF_8); - Files.write( - createPropertyData("B"), new File(hadoopConfDir, "hdfs-site.xml"), StandardCharsets.UTF_8); + Files.asCharSink(new File(hadoopConfDir, "core-site.xml"), StandardCharsets.UTF_8) + .write(createPropertyData("A")); + Files.asCharSink(new File(hadoopConfDir, "hdfs-site.xml"), StandardCharsets.UTF_8) + .write(createPropertyData("B")); HadoopFileSystemOptions.ConfigurationLocator configurationLocator = spy(new HadoopFileSystemOptions.ConfigurationLocator()); @@ -164,10 +164,10 @@ public void testDefaultJustSetYarnConfDirMultiPathConfiguration() throws IOExcep @Test public void testDefaultSetYarnConfDirAndHadoopConfDirAndSameConfiguration() throws IOException { - Files.write( - createPropertyData("A"), tmpFolder.newFile("core-site.xml"), StandardCharsets.UTF_8); - Files.write( - createPropertyData("B"), tmpFolder.newFile("hdfs-site.xml"), StandardCharsets.UTF_8); + Files.asCharSink(tmpFolder.newFile("core-site.xml"), StandardCharsets.UTF_8) + .write(createPropertyData("A")); + Files.asCharSink(tmpFolder.newFile("hdfs-site.xml"), StandardCharsets.UTF_8) + .write(createPropertyData("B")); HadoopFileSystemOptions.ConfigurationLocator configurationLocator = spy(new HadoopFileSystemOptions.ConfigurationLocator()); Map environment = Maps.newHashMap(); @@ -184,10 +184,10 @@ public void testDefaultSetYarnConfDirAndHadoopConfDirAndSameConfiguration() thro @Test public void testDefaultSetYarnConfDirAndHadoopConfDirAndSameDir() throws IOException { - Files.write( - createPropertyData("A"), tmpFolder.newFile("core-site.xml"), StandardCharsets.UTF_8); - Files.write( - createPropertyData("B"), tmpFolder.newFile("hdfs-site.xml"), StandardCharsets.UTF_8); + Files.asCharSink(tmpFolder.newFile("core-site.xml"), StandardCharsets.UTF_8) + .write(createPropertyData("A")); + Files.asCharSink(tmpFolder.newFile("hdfs-site.xml"), StandardCharsets.UTF_8) + .write(createPropertyData("B")); HadoopFileSystemOptions.ConfigurationLocator configurationLocator = spy(new HadoopFileSystemOptions.ConfigurationLocator()); Map environment = Maps.newHashMap(); @@ -208,10 +208,10 @@ public void testDefaultSetYarnConfDirAndHadoopConfDirMultiPathAndSameConfigurati File hadoopConfDir = tmpFolder.newFolder("hadoop"); File otherConfDir = tmpFolder.newFolder("_other_"); - Files.write( - createPropertyData("A"), new File(hadoopConfDir, "core-site.xml"), StandardCharsets.UTF_8); - Files.write( - createPropertyData("B"), new File(hadoopConfDir, "hdfs-site.xml"), StandardCharsets.UTF_8); + Files.asCharSink(new File(hadoopConfDir, "core-site.xml"), StandardCharsets.UTF_8) + .write(createPropertyData("A")); + Files.asCharSink(new File(hadoopConfDir, "hdfs-site.xml"), StandardCharsets.UTF_8) + .write(createPropertyData("B")); HadoopFileSystemOptions.ConfigurationLocator configurationLocator = spy(new HadoopFileSystemOptions.ConfigurationLocator()); @@ -234,14 +234,14 @@ public void testDefaultSetYarnConfDirAndHadoopConfDirMultiPathAndSameConfigurati public void testDefaultSetYarnConfDirAndHadoopConfDirNotSameConfiguration() throws IOException { File hadoopConfDir = tmpFolder.newFolder("hadoop"); File yarnConfDir = tmpFolder.newFolder("yarn"); - Files.write( - createPropertyData("A"), new File(hadoopConfDir, "core-site.xml"), StandardCharsets.UTF_8); - Files.write( - createPropertyData("B"), new File(hadoopConfDir, "hdfs-site.xml"), StandardCharsets.UTF_8); - Files.write( - createPropertyData("C"), new File(yarnConfDir, "core-site.xml"), StandardCharsets.UTF_8); - Files.write( - createPropertyData("D"), new File(yarnConfDir, "hdfs-site.xml"), StandardCharsets.UTF_8); + Files.asCharSink(new File(hadoopConfDir, "core-site.xml"), StandardCharsets.UTF_8) + .write(createPropertyData("A")); + Files.asCharSink(new File(hadoopConfDir, "hdfs-site.xml"), StandardCharsets.UTF_8) + .write(createPropertyData("B")); + Files.asCharSink(new File(yarnConfDir, "core-site.xml"), StandardCharsets.UTF_8) + .write(createPropertyData("C")); + Files.asCharSink(new File(yarnConfDir, "hdfs-site.xml"), StandardCharsets.UTF_8) + .write(createPropertyData("D")); HadoopFileSystemOptions.ConfigurationLocator configurationLocator = spy(new HadoopFileSystemOptions.ConfigurationLocator()); Map environment = Maps.newHashMap(); diff --git a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrarTest.java b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrarTest.java index f8aaeb6ddc5b9..3ff0341a6407a 100644 --- a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrarTest.java +++ b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrarTest.java @@ -25,9 +25,9 @@ import org.apache.beam.sdk.io.FileSystem; import org.apache.beam.sdk.io.FileSystemRegistrar; import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.junit.After; diff --git a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java index e22f552c0a568..c5918a8c9aa63 100644 --- a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java +++ b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java @@ -48,9 +48,9 @@ import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.util.MimeTypes; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; diff --git a/sdks/java/io/hadoop-format/OWNERS b/sdks/java/io/hadoop-format/OWNERS deleted file mode 100644 index 09812f7fdedc1..0000000000000 --- a/sdks/java/io/hadoop-format/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - timrobertson100 - - aromanenko-dev diff --git a/sdks/java/io/hadoop-format/build.gradle b/sdks/java/io/hadoop-format/build.gradle index 8b938bdc27b6d..986ccbe3e1189 100644 --- a/sdks/java/io/hadoop-format/build.gradle +++ b/sdks/java/io/hadoop-format/build.gradle @@ -56,7 +56,7 @@ configurations.all { dependencies { implementation project(path: ":sdks:java:core", configuration: "shadow") - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.slf4j_api implementation project(":sdks:java:io:hadoop-common") implementation library.java.joda_time diff --git a/sdks/java/io/hadoop-format/src/main/java/org/apache/beam/sdk/io/hadoop/format/HDFSSynchronization.java b/sdks/java/io/hadoop-format/src/main/java/org/apache/beam/sdk/io/hadoop/format/HDFSSynchronization.java index e3a350b436a61..5c154ee91d083 100644 --- a/sdks/java/io/hadoop-format/src/main/java/org/apache/beam/sdk/io/hadoop/format/HDFSSynchronization.java +++ b/sdks/java/io/hadoop-format/src/main/java/org/apache/beam/sdk/io/hadoop/format/HDFSSynchronization.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.io.Serializable; import java.util.Random; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; diff --git a/sdks/java/io/hadoop-format/src/main/java/org/apache/beam/sdk/io/hadoop/format/HadoopFormatIO.java b/sdks/java/io/hadoop-format/src/main/java/org/apache/beam/sdk/io/hadoop/format/HadoopFormatIO.java index d2ae22f2a01db..0de5c754c70bb 100644 --- a/sdks/java/io/hadoop-format/src/main/java/org/apache/beam/sdk/io/hadoop/format/HadoopFormatIO.java +++ b/sdks/java/io/hadoop-format/src/main/java/org/apache/beam/sdk/io/hadoop/format/HadoopFormatIO.java @@ -18,8 +18,8 @@ package org.apache.beam.sdk.io.hadoop.format; import static java.util.Objects.requireNonNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import java.io.DataInputStream; @@ -74,9 +74,9 @@ import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; import org.apache.beam.sdk.values.WindowingStrategy; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.AtomicDouble; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.AtomicDouble; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.ObjectWritable; diff --git a/sdks/java/io/hadoop-format/src/main/java/org/apache/beam/sdk/io/hadoop/format/HadoopFormats.java b/sdks/java/io/hadoop-format/src/main/java/org/apache/beam/sdk/io/hadoop/format/HadoopFormats.java index 3f81b4d9e6f2b..53c668dd11b51 100644 --- a/sdks/java/io/hadoop-format/src/main/java/org/apache/beam/sdk/io/hadoop/format/HadoopFormats.java +++ b/sdks/java/io/hadoop-format/src/main/java/org/apache/beam/sdk/io/hadoop/format/HadoopFormats.java @@ -19,7 +19,7 @@ import java.lang.reflect.InvocationTargetException; import java.util.UUID; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.MRJobConfig; diff --git a/sdks/java/io/hadoop-format/src/test/java/org/apache/beam/sdk/io/hadoop/format/EmployeeInputFormat.java b/sdks/java/io/hadoop-format/src/test/java/org/apache/beam/sdk/io/hadoop/format/EmployeeInputFormat.java index 6f491b9842b0e..27ad4bf35cac6 100644 --- a/sdks/java/io/hadoop-format/src/test/java/org/apache/beam/sdk/io/hadoop/format/EmployeeInputFormat.java +++ b/sdks/java/io/hadoop-format/src/test/java/org/apache/beam/sdk/io/hadoop/format/EmployeeInputFormat.java @@ -23,7 +23,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.InputFormat; diff --git a/sdks/java/io/hadoop-format/src/test/java/org/apache/beam/sdk/io/hadoop/format/HadoopFormatIOReadTest.java b/sdks/java/io/hadoop-format/src/test/java/org/apache/beam/sdk/io/hadoop/format/HadoopFormatIOReadTest.java index e561d9f2964e7..8e68d44f4aa91 100644 --- a/sdks/java/io/hadoop-format/src/test/java/org/apache/beam/sdk/io/hadoop/format/HadoopFormatIOReadTest.java +++ b/sdks/java/io/hadoop-format/src/test/java/org/apache/beam/sdk/io/hadoop/format/HadoopFormatIOReadTest.java @@ -53,7 +53,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; diff --git a/sdks/java/io/hadoop-format/src/test/java/org/apache/beam/sdk/io/hadoop/format/ReuseObjectsEmployeeInputFormat.java b/sdks/java/io/hadoop-format/src/test/java/org/apache/beam/sdk/io/hadoop/format/ReuseObjectsEmployeeInputFormat.java index 54cd61e7e22a2..ce6696a78a747 100644 --- a/sdks/java/io/hadoop-format/src/test/java/org/apache/beam/sdk/io/hadoop/format/ReuseObjectsEmployeeInputFormat.java +++ b/sdks/java/io/hadoop-format/src/test/java/org/apache/beam/sdk/io/hadoop/format/ReuseObjectsEmployeeInputFormat.java @@ -23,7 +23,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.InputFormat; diff --git a/sdks/java/io/hadoop-format/src/test/java/org/apache/beam/sdk/io/hadoop/format/TestEmployeeDataSet.java b/sdks/java/io/hadoop-format/src/test/java/org/apache/beam/sdk/io/hadoop/format/TestEmployeeDataSet.java index 45a9703c96b01..2ad0acb22ba39 100644 --- a/sdks/java/io/hadoop-format/src/test/java/org/apache/beam/sdk/io/hadoop/format/TestEmployeeDataSet.java +++ b/sdks/java/io/hadoop-format/src/test/java/org/apache/beam/sdk/io/hadoop/format/TestEmployeeDataSet.java @@ -21,7 +21,7 @@ import java.util.List; import java.util.stream.Collectors; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; import org.apache.hadoop.io.Text; /** diff --git a/sdks/java/io/hbase/OWNERS b/sdks/java/io/hbase/OWNERS deleted file mode 100644 index 09812f7fdedc1..0000000000000 --- a/sdks/java/io/hbase/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - timrobertson100 - - aromanenko-dev diff --git a/sdks/java/io/hbase/build.gradle b/sdks/java/io/hbase/build.gradle index 8224c804867ee..2f63ab09b6ec9 100644 --- a/sdks/java/io/hbase/build.gradle +++ b/sdks/java/io/hbase/build.gradle @@ -34,10 +34,10 @@ test { jvmArgs "-Dtest.build.data.basedirectory=build/test-data" } -def hbase_version = "1.2.6" +def hbase_version = "2.5.5" dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(":sdks:java:io:hadoop-common") implementation library.java.slf4j_api @@ -49,17 +49,7 @@ dependencies { testImplementation library.java.hadoop_minicluster testImplementation library.java.hadoop_hdfs testImplementation library.java.hadoop_common - testImplementation "org.apache.hbase:hbase-shaded-server:$hbase_version" - testImplementation("org.apache.hbase:hbase-server:$hbase_version:tests") { - // We prevent bringing in unshaded dependencies to not conflict - // with hbase-shaded-server and hbase-shaded-client - transitive = false - } - testImplementation("org.apache.hbase:hbase-common:$hbase_version:tests") { - // We prevent bringing in unshaded dependencies to not conflict - // with hbase-shaded-server and hbase-shaded-client - transitive = false - } + testImplementation("org.apache.hbase:hbase-shaded-testing-util:$hbase_version") testImplementation "org.apache.hbase:hbase-hadoop-compat:$hbase_version:tests" testImplementation "org.apache.hbase:hbase-hadoop2-compat:$hbase_version:tests" testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") diff --git a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseCoderProviderRegistrar.java b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseCoderProviderRegistrar.java index 88a0b47ef617f..be727aebdccce 100644 --- a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseCoderProviderRegistrar.java +++ b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseCoderProviderRegistrar.java @@ -23,7 +23,7 @@ import org.apache.beam.sdk.coders.CoderProviderRegistrar; import org.apache.beam.sdk.coders.CoderProviders; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.hadoop.hbase.client.Result; /** A {@link CoderProviderRegistrar} for standard types used with {@link HBaseIO}. */ diff --git a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseIO.java b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseIO.java index ec8feaf826bc7..6ca2fe1aa3bea 100644 --- a/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseIO.java +++ b/sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseIO.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.hbase; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.IOException; diff --git a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java index 6b096492437aa..b4e0de58cf37a 100644 --- a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java +++ b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java @@ -51,11 +51,12 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.StartMiniClusterOption; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.BufferedMutator; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; @@ -85,7 +86,7 @@ public class HBaseIOTest { @Rule public TemporaryHBaseTable tmpTable = new TemporaryHBaseTable(); private static HBaseTestingUtility htu; - private static HBaseAdmin admin; + private static Admin admin; private static final Configuration conf = HBaseConfiguration.create(); private static final byte[] COLUMN_FAMILY = Bytes.toBytes("info"); @@ -103,10 +104,12 @@ public static void beforeClass() throws Exception { // We don't use the full htu.startMiniCluster() to avoid starting unneeded HDFS/MR daemons htu.startMiniZKCluster(); - MiniHBaseCluster hbm = htu.startMiniHBaseCluster(1, 4); + StartMiniClusterOption option = + StartMiniClusterOption.builder().numMasters(1).numRegionServers(4).build(); + MiniHBaseCluster hbm = htu.startMiniHBaseCluster(option); hbm.waitForActiveAndReadyMaster(); - admin = htu.getHBaseAdmin(); + admin = htu.getAdmin(); } @AfterClass diff --git a/sdks/java/io/hcatalog/OWNERS b/sdks/java/io/hcatalog/OWNERS deleted file mode 100644 index 09812f7fdedc1..0000000000000 --- a/sdks/java/io/hcatalog/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - timrobertson100 - - aromanenko-dev diff --git a/sdks/java/io/hcatalog/build.gradle b/sdks/java/io/hcatalog/build.gradle index b1a100b55cc32..e66c3cf56d8b4 100644 --- a/sdks/java/io/hcatalog/build.gradle +++ b/sdks/java/io/hcatalog/build.gradle @@ -49,7 +49,7 @@ def hive_version = "3.1.3" evaluationDependsOn(":sdks:java:io:common") dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(":sdks:java:io:hadoop-common") implementation library.java.slf4j_api diff --git a/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogBeamSchema.java b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogBeamSchema.java index 7cd279eba2695..06d44aa6aa6d2 100644 --- a/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogBeamSchema.java +++ b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogBeamSchema.java @@ -20,8 +20,8 @@ import java.util.List; import java.util.Map; import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; diff --git a/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java index 94848f777ea83..ba2674653f6bb 100644 --- a/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java +++ b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.hcatalog; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.io.IOException; @@ -43,7 +43,7 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.ql.metadata.Table; diff --git a/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/PartitionReaderFn.java b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/PartitionReaderFn.java index 65c0e68217bdd..2836beb042d6a 100644 --- a/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/PartitionReaderFn.java +++ b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/PartitionReaderFn.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.hcatalog; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.ArrayList; import java.util.HashMap; diff --git a/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/SchemaUtils.java b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/SchemaUtils.java index 6da1b8dfdf9ac..7a4b0b62bbac0 100644 --- a/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/SchemaUtils.java +++ b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/SchemaUtils.java @@ -23,7 +23,7 @@ import java.util.Map; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.Schema.FieldType; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hive.hcatalog.common.HCatException; import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; diff --git a/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOIT.java b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOIT.java index 60d925888f9c5..beed57f3da1bd 100644 --- a/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOIT.java +++ b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOIT.java @@ -33,7 +33,7 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.hive.hcatalog.data.HCatRecord; import org.junit.After; import org.junit.BeforeClass; diff --git a/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java index 05ac28661193b..4bb7e1bd70441 100644 --- a/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java +++ b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java @@ -60,7 +60,7 @@ import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.util.UserCodeException; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hive.hcatalog.data.DefaultHCatRecord; import org.apache.hive.hcatalog.data.HCatRecord; diff --git a/sdks/java/io/influxdb/build.gradle b/sdks/java/io/influxdb/build.gradle index cca0d01cff054..b9de6b099adb2 100644 --- a/sdks/java/io/influxdb/build.gradle +++ b/sdks/java/io/influxdb/build.gradle @@ -28,7 +28,7 @@ dependencies { implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.influxdb_library implementation "com.squareup.okhttp3:okhttp:4.6.0" - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre testImplementation library.java.junit testImplementation library.java.powermock testImplementation library.java.powermock_mockito diff --git a/sdks/java/io/influxdb/src/main/java/org/apache/beam/sdk/io/influxdb/InfluxDbIO.java b/sdks/java/io/influxdb/src/main/java/org/apache/beam/sdk/io/influxdb/InfluxDbIO.java index 4de05ace3e86a..a2aef87669215 100644 --- a/sdks/java/io/influxdb/src/main/java/org/apache/beam/sdk/io/influxdb/InfluxDbIO.java +++ b/sdks/java/io/influxdb/src/main/java/org/apache/beam/sdk/io/influxdb/InfluxDbIO.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.influxdb; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.influxdb.BatchOptions.DEFAULT_BUFFER_LIMIT; import com.google.auto.value.AutoValue; diff --git a/sdks/java/io/jdbc/OWNERS b/sdks/java/io/jdbc/OWNERS deleted file mode 100644 index cdd510675d5a5..0000000000000 --- a/sdks/java/io/jdbc/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - jbonofre - - timrobertson100 diff --git a/sdks/java/io/jdbc/build.gradle b/sdks/java/io/jdbc/build.gradle index 07262bc793f4c..add9f75cb1576 100644 --- a/sdks/java/io/jdbc/build.gradle +++ b/sdks/java/io/jdbc/build.gradle @@ -27,7 +27,7 @@ description = "Apache Beam :: SDKs :: Java :: IO :: JDBC" ext.summary = "IO to read and write on JDBC datasource." dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.dbcp2 implementation library.java.joda_time diff --git a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcIO.java b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcIO.java index 02a21986e60e1..f8dad23d1fbd6 100644 --- a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcIO.java +++ b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcIO.java @@ -20,9 +20,9 @@ import static org.apache.beam.sdk.io.jdbc.SchemaUtil.checkNullabilityForFields; import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -360,6 +360,7 @@ public static ReadWithPartitions read return new AutoValue_JdbcIO_ReadWithPartitions.Builder() .setPartitionColumnType(partitioningColumnType) .setNumPartitions(DEFAULT_NUM_PARTITIONS) + .setFetchSize(DEFAULT_FETCH_SIZE) .setUseBeamSchema(false) .build(); } @@ -786,7 +787,7 @@ public PCollection expand(PBegin input) { // Spotbugs seems to not understand the multi-statement try-with-resources @SuppressFBWarnings("OBL_UNSATISFIED_OBLIGATION") - private static Schema inferBeamSchema(DataSource ds, String query) { + public static Schema inferBeamSchema(DataSource ds, String query) { try (Connection conn = ds.getConnection(); PreparedStatement statement = conn.prepareStatement( @@ -1195,6 +1196,9 @@ public abstract static class ReadWithPartitions @Pure abstract @Nullable String getPartitionColumn(); + @Pure + abstract int getFetchSize(); + @Pure abstract boolean getUseBeamSchema(); @@ -1233,6 +1237,8 @@ abstract Builder setDataSourceProviderFn( abstract Builder setUseBeamSchema(boolean useBeamSchema); + abstract Builder setFetchSize(int fetchSize); + abstract Builder setTable(String tableName); abstract Builder setPartitionColumnType( @@ -1357,7 +1363,8 @@ && getLowerBound() instanceof Comparable) { .withRowMapper( checkStateNotNull( JdbcUtil.JdbcReadWithPartitionsHelper.getPartitionsHelper( - getPartitionColumnType())))) + getPartitionColumnType()))) + .withFetchSize(getFetchSize())) .apply( MapElements.via( new SimpleFunction< @@ -1421,6 +1428,7 @@ public KV> apply( String.format( "select * from %1$s where %2$s >= ? and %2$s < ?", table, partitionColumn)) .withRowMapper(rowMapper) + .withFetchSize(getFetchSize()) .withParameterSetter( checkStateNotNull( JdbcUtil.JdbcReadWithPartitionsHelper.getPartitionsHelper( diff --git a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcReadSchemaTransformProvider.java b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcReadSchemaTransformProvider.java index cb2acfcac9974..dbf12f35024af 100644 --- a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcReadSchemaTransformProvider.java +++ b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcReadSchemaTransformProvider.java @@ -28,9 +28,8 @@ import org.apache.beam.sdk.schemas.transforms.SchemaTransform; import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; -import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollectionRowTuple; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.UnknownKeyFor; @@ -57,7 +56,7 @@ public class JdbcReadSchemaTransformProvider return new JdbcReadSchemaTransform(configuration); } - static class JdbcReadSchemaTransform implements SchemaTransform, Serializable { + static class JdbcReadSchemaTransform extends SchemaTransform implements Serializable { JdbcReadSchemaTransformConfiguration config; @@ -85,32 +84,22 @@ protected JdbcIO.DataSourceConfiguration dataSourceConfiguration() { } @Override - public @UnknownKeyFor @NonNull @Initialized PTransform< - @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple, - @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple> - buildTransform() { - return new PTransform() { - @Override - public PCollectionRowTuple expand(PCollectionRowTuple input) { - String query = config.getReadQuery(); - if (query == null) { - query = String.format("SELECT * FROM %s", config.getLocation()); - } - JdbcIO.ReadRows readRows = - JdbcIO.readRows() - .withDataSourceConfiguration(dataSourceConfiguration()) - .withQuery(query); - Short fetchSize = config.getFetchSize(); - if (fetchSize != null && fetchSize > 0) { - readRows = readRows.withFetchSize(fetchSize); - } - Boolean outputParallelization = config.getOutputParallelization(); - if (outputParallelization != null) { - readRows = readRows.withOutputParallelization(outputParallelization); - } - return PCollectionRowTuple.of("output", input.getPipeline().apply(readRows)); - } - }; + public PCollectionRowTuple expand(PCollectionRowTuple input) { + String query = config.getReadQuery(); + if (query == null) { + query = String.format("SELECT * FROM %s", config.getLocation()); + } + JdbcIO.ReadRows readRows = + JdbcIO.readRows().withDataSourceConfiguration(dataSourceConfiguration()).withQuery(query); + Short fetchSize = config.getFetchSize(); + if (fetchSize != null && fetchSize > 0) { + readRows = readRows.withFetchSize(fetchSize); + } + Boolean outputParallelization = config.getOutputParallelization(); + if (outputParallelization != null) { + readRows = readRows.withOutputParallelization(outputParallelization); + } + return PCollectionRowTuple.of("output", input.getPipeline().apply(readRows)); } } diff --git a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcSchemaIOProvider.java b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcSchemaIOProvider.java index c37ddf26abacd..c68b33a026077 100644 --- a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcSchemaIOProvider.java +++ b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcSchemaIOProvider.java @@ -204,9 +204,10 @@ protected JdbcIO.DataSourceConfiguration getDataSourceConfiguration() { dataSourceConfiguration = dataSourceConfiguration.withConnectionInitSqls(initSqls); } - @Nullable Integer maxConnections = config.getInt32("maxConnections"); + @Nullable Short maxConnections = config.getInt16("maxConnections"); if (maxConnections != null) { - dataSourceConfiguration = dataSourceConfiguration.withMaxConnections(maxConnections); + dataSourceConfiguration = + dataSourceConfiguration.withMaxConnections(maxConnections.intValue()); } @Nullable String driverJars = config.getString("driverJars"); diff --git a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcUtil.java b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcUtil.java index 48aa78c0e36ed..8c7ee17d5fc30 100644 --- a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcUtil.java +++ b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcUtil.java @@ -58,11 +58,11 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTime; import org.joda.time.Duration; diff --git a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcWriteSchemaTransformProvider.java b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcWriteSchemaTransformProvider.java index 7f9cc3b775609..cb9d79631ca8e 100644 --- a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcWriteSchemaTransformProvider.java +++ b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcWriteSchemaTransformProvider.java @@ -29,10 +29,9 @@ import org.apache.beam.sdk.schemas.transforms.SchemaTransform; import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; -import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.UnknownKeyFor; @@ -59,7 +58,7 @@ public class JdbcWriteSchemaTransformProvider return new JdbcWriteSchemaTransform(configuration); } - static class JdbcWriteSchemaTransform implements SchemaTransform, Serializable { + static class JdbcWriteSchemaTransform extends SchemaTransform implements Serializable { JdbcWriteSchemaTransformConfiguration config; @@ -103,26 +102,18 @@ protected String writeStatement(Schema schema) { } @Override - public @UnknownKeyFor @NonNull @Initialized PTransform< - @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple, - @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple> - buildTransform() { - return new PTransform() { - @Override - public PCollectionRowTuple expand(PCollectionRowTuple input) { - JdbcIO.Write writeRows = - JdbcIO.write() - .withDataSourceConfiguration(dataSourceConfiguration()) - .withStatement(writeStatement(input.get("input").getSchema())) - .withPreparedStatementSetter(new JdbcUtil.BeamRowPreparedStatementSetter()); - Boolean autosharding = config.getAutosharding(); - if (autosharding != null && autosharding) { - writeRows = writeRows.withAutoSharding(); - } - input.get("input").apply(writeRows); - return PCollectionRowTuple.empty(input.getPipeline()); - } - }; + public PCollectionRowTuple expand(PCollectionRowTuple input) { + JdbcIO.Write writeRows = + JdbcIO.write() + .withDataSourceConfiguration(dataSourceConfiguration()) + .withStatement(writeStatement(input.get("input").getSchema())) + .withPreparedStatementSetter(new JdbcUtil.BeamRowPreparedStatementSetter()); + Boolean autosharding = config.getAutosharding(); + if (autosharding != null && autosharding) { + writeRows = writeRows.withAutoSharding(); + } + input.get("input").apply(writeRows); + return PCollectionRowTuple.empty(input.getPipeline()); } } diff --git a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/LogicalTypes.java b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/LogicalTypes.java index a546a77c14932..6e8e46b7afa2b 100644 --- a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/LogicalTypes.java +++ b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/LogicalTypes.java @@ -19,7 +19,6 @@ import java.sql.JDBCType; import java.time.Instant; -import java.util.Objects; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.logicaltypes.FixedBytes; @@ -29,12 +28,12 @@ import org.apache.beam.sdk.schemas.logicaltypes.UuidLogicalType; import org.apache.beam.sdk.schemas.logicaltypes.VariableBytes; import org.apache.beam.sdk.schemas.logicaltypes.VariableString; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.checkerframework.checker.nullness.qual.NonNull; -import org.checkerframework.checker.nullness.qual.Nullable; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; /** Beam {@link org.apache.beam.sdk.schemas.Schema.LogicalType} implementations of JDBC types. */ class LogicalTypes { + // Logical types of the following static members are not portable and are preserved for + // compatibility reason. Consider using portable logical types when adding new ones. static final Schema.FieldType JDBC_BIT_TYPE = Schema.FieldType.logicalType( new PassThroughLogicalType( @@ -110,69 +109,4 @@ static Schema.LogicalType fixedOrVariableBytes(String name, int return FixedBytes.of(name, length); } } - - /** Base class for JDBC logical types. */ - abstract static class JdbcLogicalType - implements Schema.LogicalType { - protected final String identifier; - protected final Schema.FieldType argumentType; - protected final Schema.FieldType baseType; - protected final Object argument; - - protected JdbcLogicalType( - String identifier, - Schema.FieldType argumentType, - Schema.FieldType baseType, - Object argument) { - this.identifier = identifier; - this.argumentType = argumentType; - this.baseType = baseType; - this.argument = argument; - } - - @Override - public String getIdentifier() { - return identifier; - } - - @Override - public FieldType getArgumentType() { - return argumentType; - } - - @Override - @SuppressWarnings("TypeParameterUnusedInFormals") - public ArgumentT getArgument() { - return (ArgumentT) argument; - } - - @Override - public Schema.FieldType getBaseType() { - return baseType; - } - - @Override - public T toBaseType(T input) { - return input; - } - - @Override - public boolean equals(@Nullable Object o) { - if (this == o) { - return true; - } - if (!(o instanceof JdbcLogicalType)) { - return false; - } - JdbcLogicalType that = (JdbcLogicalType) o; - return Objects.equals(identifier, that.identifier) - && Objects.equals(baseType, that.baseType) - && Objects.equals(argument, that.argument); - } - - @Override - public int hashCode() { - return Objects.hash(identifier, baseType, argument); - } - } } diff --git a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/SchemaUtil.java b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/SchemaUtil.java index c31bd50b3ca53..65f21308ea320 100644 --- a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/SchemaUtil.java +++ b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/SchemaUtil.java @@ -27,7 +27,7 @@ import static java.sql.JDBCType.VARBINARY; import static java.sql.JDBCType.VARCHAR; import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.Serializable; import java.sql.Array; @@ -56,7 +56,7 @@ import org.apache.beam.sdk.schemas.logicaltypes.VariableBytes; import org.apache.beam.sdk.schemas.logicaltypes.VariableString; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTime; import org.joda.time.LocalDate; @@ -354,7 +354,7 @@ private static ResultSetFieldExtractor createObjectExtractor() { * A {@link org.apache.beam.sdk.io.jdbc.JdbcIO.RowMapper} implementation that converts JDBC * results into Beam {@link Row} objects. */ - static final class BeamRowMapper implements JdbcIO.RowMapper { + public static final class BeamRowMapper implements JdbcIO.RowMapper { private final Schema schema; private final List fieldExtractors; diff --git a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOAutoPartitioningIT.java b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOAutoPartitioningIT.java index 561fe9bf0ba1e..0042d93aedfc1 100644 --- a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOAutoPartitioningIT.java +++ b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOAutoPartitioningIT.java @@ -42,7 +42,7 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.DateTime; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java index 301e96496c5ec..76558fd688c1d 100644 --- a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java +++ b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java @@ -20,7 +20,7 @@ import static org.apache.beam.sdk.io.common.DatabaseTestHelper.assertRowCount; import static org.apache.beam.sdk.io.common.DatabaseTestHelper.getTestDataToWrite; import static org.apache.beam.sdk.io.common.IOITHelper.readIOTestPipelineOptions; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; import static org.junit.Assert.assertNotEquals; import com.google.cloud.Timestamp; @@ -62,7 +62,7 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.BeforeClass; diff --git a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOTest.java b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOTest.java index 0e1874a08b74f..ccee14c4691b2 100644 --- a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOTest.java +++ b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOTest.java @@ -89,7 +89,7 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.commons.dbcp2.PoolingDataSource; import org.hamcrest.Description; import org.hamcrest.TypeSafeMatcher; diff --git a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcReadSchemaTransformProviderTest.java b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcReadSchemaTransformProviderTest.java index 3e82b565fbaec..251f995dea4e3 100644 --- a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcReadSchemaTransformProviderTest.java +++ b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcReadSchemaTransformProviderTest.java @@ -111,16 +111,12 @@ public void testRead() { PCollection output = PCollectionRowTuple.empty(pipeline) .apply( - provider - .from( - JdbcReadSchemaTransformProvider.JdbcReadSchemaTransformConfiguration - .builder() - .setDriverClassName( - DATA_SOURCE_CONFIGURATION.getDriverClassName().get()) - .setJdbcUrl(DATA_SOURCE_CONFIGURATION.getUrl().get()) - .setLocation(READ_TABLE_NAME) - .build()) - .buildTransform()) + provider.from( + JdbcReadSchemaTransformProvider.JdbcReadSchemaTransformConfiguration.builder() + .setDriverClassName(DATA_SOURCE_CONFIGURATION.getDriverClassName().get()) + .setJdbcUrl(DATA_SOURCE_CONFIGURATION.getUrl().get()) + .setLocation(READ_TABLE_NAME) + .build())) .get("output"); Long expected = Long.valueOf(EXPECTED_ROW_COUNT); PAssert.that(output.apply(Count.globally())).containsInAnyOrder(expected); diff --git a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcSchemaIOProviderTest.java b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcSchemaIOProviderTest.java index d91eaaef6e627..ed380d8136258 100644 --- a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcSchemaIOProviderTest.java +++ b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcSchemaIOProviderTest.java @@ -17,9 +17,14 @@ */ package org.apache.beam.sdk.io.jdbc; +import static org.junit.Assert.assertEquals; + import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Objects; import javax.sql.DataSource; import org.apache.beam.sdk.io.common.DatabaseTestHelper; import org.apache.beam.sdk.io.common.TestRow; @@ -81,8 +86,7 @@ public void testPartitionedRead() { } // This test shouldn't work because we only support numeric and datetime columns and we are trying - // to use a string - // column as our partition source + // to use a string column as our partition source. @Test public void testPartitionedReadThatShouldntWork() throws Exception { JdbcSchemaIOProvider provider = new JdbcSchemaIOProvider(); @@ -110,6 +114,42 @@ public void testPartitionedReadThatShouldntWork() throws Exception { throw new Exception("Did not throw an exception"); } + // This test verifies we can read back existing data source configuration. It also serves as + // sanity check that the + // getDataSourceConfiguration should keep in sync with JdbcSchemaIOProvider.configurationSchema. + // Otherwise the test + // would throw an exception. + @Test + public void testAbleToReadDataSourceConfiguration() { + JdbcSchemaIOProvider provider = new JdbcSchemaIOProvider(); + + Row config = + Row.withSchema(provider.configurationSchema()) + .withFieldValue("driverClassName", "className") + .withFieldValue("jdbcUrl", "url") + .withFieldValue("username", "user") + .withFieldValue("password", "passwd") + .withFieldValue("connectionProperties", "connectionProp") + .withFieldValue("connectionInitSqls", new ArrayList<>(Collections.singleton("initSql"))) + .withFieldValue("maxConnections", (short) 3) + .withFieldValue("driverJars", "test.jar") + .build(); + JdbcSchemaIOProvider.JdbcSchemaIO schemaIO = + provider.from(READ_TABLE_NAME, config, Schema.builder().build()); + JdbcIO.DataSourceConfiguration dataSourceConf = schemaIO.getDataSourceConfiguration(); + assertEquals("className", Objects.requireNonNull(dataSourceConf.getDriverClassName()).get()); + assertEquals("url", Objects.requireNonNull(dataSourceConf.getUrl()).get()); + assertEquals("user", Objects.requireNonNull(dataSourceConf.getUsername()).get()); + assertEquals("passwd", Objects.requireNonNull(dataSourceConf.getPassword()).get()); + assertEquals( + "connectionProp", Objects.requireNonNull(dataSourceConf.getConnectionProperties()).get()); + assertEquals( + new ArrayList<>(Collections.singleton("initSql")), + Objects.requireNonNull(dataSourceConf.getConnectionInitSqls()).get()); + assertEquals(3, (int) dataSourceConf.getMaxConnections().get()); + assertEquals("test.jar", Objects.requireNonNull(dataSourceConf.getDriverJars()).get()); + } + /** Create test data that is consistent with that generated by TestRow. */ private static void addInitialData(DataSource dataSource, String tableName) throws SQLException { try (Connection connection = dataSource.getConnection()) { diff --git a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcUtilTest.java b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcUtilTest.java index 138a7b7acc340..360b579195cfa 100644 --- a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcUtilTest.java +++ b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcUtilTest.java @@ -40,7 +40,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.DateTime; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcWriteSchemaTransformProviderTest.java b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcWriteSchemaTransformProviderTest.java index 7f422affda2ce..64de7a1b56cfd 100644 --- a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcWriteSchemaTransformProviderTest.java +++ b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcWriteSchemaTransformProviderTest.java @@ -31,7 +31,7 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; @@ -115,14 +115,12 @@ public void testReadWriteToTable() throws SQLException { PCollectionRowTuple.of("input", pipeline.apply(Create.of(rows).withRowSchema(schema))) .apply( - provider - .from( - JdbcWriteSchemaTransformProvider.JdbcWriteSchemaTransformConfiguration.builder() - .setDriverClassName(DATA_SOURCE_CONFIGURATION.getDriverClassName().get()) - .setJdbcUrl(DATA_SOURCE_CONFIGURATION.getUrl().get()) - .setLocation(WRITE_TABLE_NAME) - .build()) - .buildTransform()); + provider.from( + JdbcWriteSchemaTransformProvider.JdbcWriteSchemaTransformConfiguration.builder() + .setDriverClassName(DATA_SOURCE_CONFIGURATION.getDriverClassName().get()) + .setJdbcUrl(DATA_SOURCE_CONFIGURATION.getUrl().get()) + .setLocation(WRITE_TABLE_NAME) + .build())); pipeline.run(); DatabaseTestHelper.assertRowCount(DATA_SOURCE, WRITE_TABLE_NAME, 2); } diff --git a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/OtherJdbcTypesIT.java b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/OtherJdbcTypesIT.java index f56e3d299094c..2a386a4e770bd 100644 --- a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/OtherJdbcTypesIT.java +++ b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/OtherJdbcTypesIT.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Rule; diff --git a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/SchemaUtilTest.java b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/SchemaUtilTest.java index 080a451d706fd..c14b9dc916867 100644 --- a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/SchemaUtilTest.java +++ b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/SchemaUtilTest.java @@ -40,7 +40,7 @@ import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.DateTime; import org.joda.time.LocalDate; import org.joda.time.chrono.ISOChronology; diff --git a/sdks/java/io/jms/OWNERS b/sdks/java/io/jms/OWNERS deleted file mode 100644 index 8b0b05531cd10..0000000000000 --- a/sdks/java/io/jms/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - jbonofre diff --git a/sdks/java/io/jms/build.gradle b/sdks/java/io/jms/build.gradle index 5ecc0ec19d575..1ae1229b76c25 100644 --- a/sdks/java/io/jms/build.gradle +++ b/sdks/java/io/jms/build.gradle @@ -28,7 +28,7 @@ ext.summary = """IO to read and write to JMS (Java Messaging Service) destinations (queues and topics).""" dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.slf4j_api implementation library.java.joda_time diff --git a/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsCheckpointMark.java b/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsCheckpointMark.java index 244c3cbabb207..f9f382dc3cd6e 100644 --- a/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsCheckpointMark.java +++ b/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsCheckpointMark.java @@ -25,7 +25,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import javax.jms.Message; import org.apache.beam.sdk.io.UnboundedSource; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; import org.slf4j.Logger; diff --git a/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsIO.java b/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsIO.java index 0528b576de726..2344dec449a2b 100644 --- a/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsIO.java +++ b/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/JmsIO.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.jms; import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.io.IOException; @@ -66,7 +66,7 @@ import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; diff --git a/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/RetryConfiguration.java b/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/RetryConfiguration.java index 013ac278e56e8..883dc88c4b584 100644 --- a/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/RetryConfiguration.java +++ b/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/RetryConfiguration.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.jms; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.io.Serializable; diff --git a/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/WriteJmsResult.java b/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/WriteJmsResult.java index 22034e60b0ed2..c037c14e8d67d 100644 --- a/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/WriteJmsResult.java +++ b/sdks/java/io/jms/src/main/java/org/apache/beam/sdk/io/jms/WriteJmsResult.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * Return type of {@link JmsIO.Write} transform. All messages in error are identified by: - diff --git a/sdks/java/io/jms/src/test/java/org/apache/beam/sdk/io/jms/JmsIOTest.java b/sdks/java/io/jms/src/test/java/org/apache/beam/sdk/io/jms/JmsIOTest.java index 10f3ec7317cb5..8eba8c494f0f1 100644 --- a/sdks/java/io/jms/src/test/java/org/apache/beam/sdk/io/jms/JmsIOTest.java +++ b/sdks/java/io/jms/src/test/java/org/apache/beam/sdk/io/jms/JmsIOTest.java @@ -91,7 +91,7 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.SerializableBiFunction; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; import org.apache.qpid.jms.JmsAcknowledgeCallback; import org.apache.qpid.jms.JmsConnectionFactory; import org.apache.qpid.jms.message.JmsTextMessage; diff --git a/sdks/java/io/json/build.gradle b/sdks/java/io/json/build.gradle new file mode 100644 index 0000000000000..fe1f607a3696f --- /dev/null +++ b/sdks/java/io/json/build.gradle @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.sdk.io.json' +) + +description = "Apache Beam :: SDKs :: Java :: IO :: JSON" +ext.summary = "IO to read and write JSON files." + +dependencies { + implementation project(path: ":sdks:java:core", configuration: "shadow") + implementation library.java.vendored_guava_32_1_2_jre + implementation library.java.everit_json_schema + testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") + testImplementation library.java.junit + testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") + testImplementation project(path: ":sdks:java:io:common", configuration: "testRuntimeMigration") +} \ No newline at end of file diff --git a/sdks/java/io/json/src/main/java/org/apache/beam/sdk/io/json/JsonIO.java b/sdks/java/io/json/src/main/java/org/apache/beam/sdk/io/json/JsonIO.java new file mode 100644 index 0000000000000..3abb29a804272 --- /dev/null +++ b/sdks/java/io/json/src/main/java/org/apache/beam/sdk/io/json/JsonIO.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.json; + +import static org.apache.beam.sdk.values.TypeDescriptors.rows; +import static org.apache.beam.sdk.values.TypeDescriptors.strings; + +import com.google.auto.value.AutoValue; +import org.apache.beam.sdk.coders.RowCoder; +import org.apache.beam.sdk.io.Compression; +import org.apache.beam.sdk.io.FileBasedSink; +import org.apache.beam.sdk.io.FileIO; +import org.apache.beam.sdk.io.ShardNameTemplate; +import org.apache.beam.sdk.io.TextIO; +import org.apache.beam.sdk.io.WriteFiles; +import org.apache.beam.sdk.io.WriteFilesResult; +import org.apache.beam.sdk.io.fs.ResourceId; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.schemas.utils.JsonUtils; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.Row; + +/** + * {@link PTransform}s for reading and writing JSON files. + * + *

    Reading JSON files

    + * + *

    Reading from JSON files is not yet implemented in Java. Please see https://github.com/apache/beam/issues/24552. + * + *

    Writing JSON files

    + * + *

    To write a {@link PCollection} to one or more line-delimited JSON files, use {@link + * JsonIO.Write}, using{@link JsonIO#writeRows} or {@link JsonIO#write}. {@link JsonIO.Write} + * supports writing {@link Row} or custom Java types using an inferred {@link Schema}. Examples + * below show both scenarios. See the Beam Programming Guide on inferring + * schemas for more information on how to enable Beam to infer a {@link Schema} from a custom + * Java type. + * + *

    Example usage:

    + * + *

    Suppose we have the following Transaction class annotated with + * {@code @DefaultSchema(JavaBeanSchema.class)} so that Beam can infer its {@link Schema}: + * + *

    {@code @DefaultSchema(JavaBeanSchema.class)
    + * public class Transaction {
    + *   public Transaction() { … }
    + *   public Long getTransactionId();
    + *   public void setTransactionId(Long transactionId) { … }
    + *   public String getBank() { … }
    + *   public void setBank(String bank) { … }
    + *   public double getPurchaseAmount() { … }
    + *   public void setPurchaseAmount(double purchaseAmount) { … }
    + * }
    + * }
    + * + *

    From a {@code PCollection}, {@link JsonIO.Write} can write one or many JSON + * files. + * + *

    {@code
    + * PCollection transactions = ...
    + * transactions.apply(JsonIO.write("path/to/folder/prefix"));
    + * }
    + * + *

    The resulting JSON files will look like the following where the header is repeated for every + * file, whereas by default, {@link JsonIO.Write} will write all fields in sorted order of + * the field names. + * + *

    {@code
    + * {"bank": "A", "purchaseAmount": 10.23, "transactionId": 12345}
    + * {"bank": "B", "purchaseAmount": 54.65, "transactionId": 54321}
    + * {"bank": "C", "purchaseAmount": 11,76, "transactionId": 98765}
    + * }
    + * + *

    A {@link PCollection} of {@link Row}s works just like custom Java types illustrated above, + * except we use {@link JsonIO#writeRows} as shown below for the same {@code Transaction} class. We + * derive {@code Transaction}'s {@link Schema} using a {@link + * org.apache.beam.sdk.schemas.annotations.DefaultSchema.DefaultSchemaProvider}. Note that + * hard-coding the {@link Row}s below is for illustration purposes. Developers are instead + * encouraged to take advantage of {@link + * org.apache.beam.sdk.schemas.annotations.DefaultSchema.DefaultSchemaProvider#toRowFunction}. + * + *

    {@code
    + * DefaultSchemaProvider defaultSchemaProvider = new DefaultSchemaProvider();
    + * Schema schema = defaultSchemaProvider.schemaFor(TypeDescriptor.of(Transaction.class));
    + * PCollection transactions = pipeline.apply(Create.of(
    + *  Row
    + *    .withSchema(schema)
    + *    .withFieldValue("bank", "A")
    + *    .withFieldValue("purchaseAmount", 10.23)
    + *    .withFieldValue("transactionId", "12345")
    + *    .build(),
    + *  Row
    + *    .withSchema(schema)
    + *    .withFieldValue("bank", "B")
    + *    .withFieldValue("purchaseAmount", 54.65)
    + *    .withFieldValue("transactionId", "54321")
    + *    .build(),
    + *  Row
    + *    .withSchema(schema)
    + *    .withFieldValue("bank", "C")
    + *    .withFieldValue("purchaseAmount", 11.76)
    + *    .withFieldValue("transactionId", "98765")
    + *    .build()
    + * );
    + *
    + * transactions.apply(
    + *  JsonIO
    + *    .writeRowsTo("gs://bucket/path/to/folder/prefix")
    + * );
    + * }
    + * + *

    Writing the transactions {@link PCollection} of {@link Row}s would yield the following JSON + * file content. + * + *

    {@code
    + * {"bank": "A", "purchaseAmount": 10.23, "transactionId": 12345}
    + * {"bank": "B", "purchaseAmount": 54.65, "transactionId": 54321}
    + * {"bank": "C", "purchaseAmount": 11,76, "transactionId": 98765}
    + * }
    + */ +public class JsonIO { + static final String DEFAULT_FILENAME_SUFFIX = ".json"; + + /** Instantiates a {@link Write} for writing user types in {@link JSONFormat} format. */ + public static Write write(String to) { + return new AutoValue_JsonIO_Write.Builder() + .setTextIOWrite(createDefaultTextIOWrite(to)) + .build(); + } + + /** Instantiates a {@link Write} for writing {@link Row}s in {@link JSONFormat} format. */ + public static Write writeRows(String to) { + return new AutoValue_JsonIO_Write.Builder() + .setTextIOWrite(createDefaultTextIOWrite(to)) + .build(); + } + + /** {@link PTransform} for writing JSON files. */ + @AutoValue + public abstract static class Write + extends PTransform, WriteFilesResult> { + + /** Specifies the {@link Compression} of all generated shard files. */ + public Write withCompression(Compression compression) { + return toBuilder().setTextIOWrite(getTextIOWrite().withCompression(compression)).build(); + } + + /** Whether to skip the spilling of data. See {@link WriteFiles#withNoSpilling}. */ + public Write withNoSpilling() { + return toBuilder().setTextIOWrite(getTextIOWrite().withNoSpilling()).build(); + } + + /** + * Specifies to use a given fixed number of shards per window. See {@link + * TextIO.Write#withNumShards}. + */ + public Write withNumShards(Integer numShards) { + return toBuilder().setTextIOWrite(getTextIOWrite().withNumShards(numShards)).build(); + } + + /** + * Forces a single file as output and empty shard name template. See {@link + * TextIO.Write#withoutSharding}. + */ + public Write withoutSharding() { + return toBuilder().setTextIOWrite(getTextIOWrite().withoutSharding()).build(); + } + + /** + * Uses the given {@link ShardNameTemplate} for naming output files. See {@link + * TextIO.Write#withShardNameTemplate}. + */ + public Write withShardTemplate(String shardTemplate) { + return toBuilder() + .setTextIOWrite(getTextIOWrite().withShardNameTemplate(shardTemplate)) + .build(); + } + + /** Configures the filename suffix for written files. See {@link TextIO.Write#withSuffix}. */ + public Write withSuffix(String suffix) { + return toBuilder().setTextIOWrite(getTextIOWrite().withSuffix(suffix)).build(); + } + + /** + * Set the base directory used to generate temporary files. See {@link + * TextIO.Write#withTempDirectory}. + */ + public Write withTempDirectory(ResourceId tempDirectory) { + return toBuilder().setTextIOWrite(getTextIOWrite().withTempDirectory(tempDirectory)).build(); + } + + /** + * Preserves windowing of input elements and writes them to files based on the element's window. + * See {@link TextIO.Write#withWindowedWrites}. + */ + public Write withWindowedWrites() { + return toBuilder().setTextIOWrite(getTextIOWrite().withWindowedWrites()).build(); + } + + /** + * Returns a transform for writing to text files like this one but that has the given {@link + * FileBasedSink.WritableByteChannelFactory} to be used by the {@link FileBasedSink} during + * output. See {@link TextIO.Write#withWritableByteChannelFactory}. + */ + public Write withWritableByteChannelFactory( + FileBasedSink.WritableByteChannelFactory writableByteChannelFactory) { + return toBuilder() + .setTextIOWrite( + getTextIOWrite().withWritableByteChannelFactory(writableByteChannelFactory)) + .build(); + } + + /** The underlying {@link FileIO.Write} that writes converted input to JSON formatted output. */ + abstract TextIO.Write getTextIOWrite(); + + abstract Builder toBuilder(); + + @AutoValue.Builder + abstract static class Builder { + + /** + * The underlying {@link FileIO.Write} that writes converted input to JSON formatted output. + */ + abstract Builder setTextIOWrite(TextIO.Write value); + + abstract Write autoBuild(); + + final Write build() { + return autoBuild(); + } + } + + @Override + public WriteFilesResult expand(PCollection input) { + if (!input.hasSchema()) { + throw new IllegalArgumentException( + String.format( + "%s requires an input Schema. Note that only Row or user classes are supported. Consider using TextIO or FileIO directly when writing primitive types", + Write.class.getName())); + } + + Schema schema = input.getSchema(); + + RowCoder rowCoder = RowCoder.of(schema); + + PCollection rows = + input + .apply("To Rows", MapElements.into(rows()).via(input.getToRowFunction())) + .setCoder(rowCoder); + + SerializableFunction toJsonFn = + JsonUtils.getRowToJsonStringsFunction(input.getSchema()); + + PCollection json = rows.apply("To JSON", MapElements.into(strings()).via(toJsonFn)); + + return json.apply("Write JSON", getTextIOWrite().withOutputFilenames()); + } + } + + private static TextIO.Write createDefaultTextIOWrite(String to) { + return TextIO.write().to(to).withSuffix(DEFAULT_FILENAME_SUFFIX); + } +} diff --git a/sdks/java/io/json/src/main/java/org/apache/beam/sdk/io/json/package-info.java b/sdks/java/io/json/src/main/java/org/apache/beam/sdk/io/json/package-info.java new file mode 100644 index 0000000000000..1ee1918357135 --- /dev/null +++ b/sdks/java/io/json/src/main/java/org/apache/beam/sdk/io/json/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Transforms for reading and writing JSON files. */ +package org.apache.beam.sdk.io.json; diff --git a/sdks/java/io/json/src/main/java/org/apache/beam/sdk/io/json/providers/JsonWriteTransformProvider.java b/sdks/java/io/json/src/main/java/org/apache/beam/sdk/io/json/providers/JsonWriteTransformProvider.java new file mode 100644 index 0000000000000..9e030821e5ca1 --- /dev/null +++ b/sdks/java/io/json/src/main/java/org/apache/beam/sdk/io/json/providers/JsonWriteTransformProvider.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.json.providers; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import com.google.auto.service.AutoService; +import com.google.auto.value.AutoValue; +import java.util.Collections; +import java.util.List; +import org.apache.beam.sdk.io.WriteFilesResult; +import org.apache.beam.sdk.io.json.JsonIO; +import org.apache.beam.sdk.schemas.AutoValueSchema; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.schemas.Schema.Field; +import org.apache.beam.sdk.schemas.Schema.FieldType; +import org.apache.beam.sdk.schemas.annotations.DefaultSchema; +import org.apache.beam.sdk.schemas.annotations.SchemaFieldDescription; +import org.apache.beam.sdk.schemas.transforms.SchemaTransform; +import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; +import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.values.PCollectionRowTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TypeDescriptors; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; + +/** + * An implementation of {@link TypedSchemaTransformProvider} for {@link JsonIO#write}. + * + *

    Internal only: This class is actively being worked on, and it will likely change. We + * provide no backwards compatibility guarantees, and it should not be implemented outside the Beam + * repository. + */ +@SuppressWarnings({ + "nullness" // TODO(https://github.com/apache/beam/issues/20497) +}) +@AutoService(SchemaTransformProvider.class) +public class JsonWriteTransformProvider + extends TypedSchemaTransformProvider { + private static final String INPUT_ROWS_TAG = "input"; + private static final String WRITE_RESULTS = "output"; + + @Override + protected Class configurationClass() { + return JsonWriteConfiguration.class; + } + + @Override + protected SchemaTransform from(JsonWriteConfiguration configuration) { + return new JsonWriteTransform(configuration); + } + + @Override + public String identifier() { + return String.format("beam:schematransform:org.apache.beam:json_write:v1"); + } + + @Override + public List inputCollectionNames() { + return Collections.singletonList(INPUT_ROWS_TAG); + } + + @Override + public List outputCollectionNames() { + return Collections.singletonList(WRITE_RESULTS); + } + + /** Configuration for writing to BigQuery with Storage Write API. */ + @DefaultSchema(AutoValueSchema.class) + @AutoValue + public abstract static class JsonWriteConfiguration { + + public void validate() { + checkArgument( + !Strings.isNullOrEmpty(this.getPath()), "Path for a JSON Write must be specified."); + } + + public static Builder builder() { + return new AutoValue_JsonWriteTransformProvider_JsonWriteConfiguration.Builder(); + } + + @SchemaFieldDescription("The file path to write to.") + public abstract String getPath(); + + /** Builder for {@link JsonWriteConfiguration}. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setPath(String path); + + /** Builds a {@link JsonWriteConfiguration} instance. */ + public abstract JsonWriteConfiguration build(); + } + } + + /** A {@link SchemaTransform} for {@link JsonIO#write}. */ + protected static class JsonWriteTransform extends SchemaTransform { + + private final JsonWriteConfiguration configuration; + + JsonWriteTransform(JsonWriteConfiguration configuration) { + configuration.validate(); + this.configuration = configuration; + } + + @Override + public PCollectionRowTuple expand(PCollectionRowTuple input) { + WriteFilesResult result = + input.get(INPUT_ROWS_TAG).apply(JsonIO.writeRows(configuration.getPath()).withSuffix("")); + Schema outputSchema = Schema.of(Field.of("filename", FieldType.STRING)); + return PCollectionRowTuple.of( + WRITE_RESULTS, + result + .getPerDestinationOutputFilenames() + .apply( + "Collect filenames", + MapElements.into(TypeDescriptors.rows()) + .via( + (destinationAndRow) -> + Row.withSchema(outputSchema) + .withFieldValue("filename", destinationAndRow.getValue()) + .build())) + .setRowSchema(outputSchema)); + } + } +} diff --git a/sdks/java/io/json/src/main/java/org/apache/beam/sdk/io/json/providers/package-info.java b/sdks/java/io/json/src/main/java/org/apache/beam/sdk/io/json/providers/package-info.java new file mode 100644 index 0000000000000..312454f8733b9 --- /dev/null +++ b/sdks/java/io/json/src/main/java/org/apache/beam/sdk/io/json/providers/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Transforms for reading and writing JSON files. */ +package org.apache.beam.sdk.io.json.providers; diff --git a/sdks/java/io/json/src/test/java/org/apache/beam/sdk/io/json/JsonIOWriteTest.java b/sdks/java/io/json/src/test/java/org/apache/beam/sdk/io/json/JsonIOWriteTest.java new file mode 100644 index 0000000000000..71fdcd6b3d94d --- /dev/null +++ b/sdks/java/io/json/src/test/java/org/apache/beam/sdk/io/json/JsonIOWriteTest.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.json; + +import static org.apache.beam.sdk.io.common.SchemaAwareJavaBeans.allPrimitiveDataTypes; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import java.io.File; +import java.io.IOException; +import java.math.BigDecimal; +import org.apache.beam.sdk.io.TextIO; +import org.apache.beam.sdk.io.common.SchemaAwareJavaBeans.AllPrimitiveDataTypes; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.SerializableMatcher; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.values.PCollection; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link JsonIO.Write}. */ +@RunWith(JUnit4.class) +public class JsonIOWriteTest { + @Rule public TestPipeline writePipeline = TestPipeline.create(); + + @Rule public TestPipeline readPipeline = TestPipeline.create(); + + @Rule + public TestPipeline errorPipeline = TestPipeline.create().enableAbandonedNodeEnforcement(false); + + @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); + + @Test + public void writesUserDefinedTypes() { + File folder = + createFolder(AllPrimitiveDataTypes.class.getSimpleName(), "writesUserDefinedTypes"); + + PCollection input = + writePipeline.apply( + Create.of( + allPrimitiveDataTypes(false, BigDecimal.TEN, 1.0, 1.0f, 1, 1L, "a"), + allPrimitiveDataTypes( + false, BigDecimal.TEN.add(BigDecimal.TEN), 2.0, 2.0f, 2, 2L, "b"), + allPrimitiveDataTypes( + false, + BigDecimal.TEN.add(BigDecimal.TEN).add(BigDecimal.TEN), + 3.0, + 3.0f, + 3, + 3L, + "c"))); + + input.apply(JsonIO.write(toFilenamePrefix(folder)).withNumShards(1)); + + writePipeline.run().waitUntilFinish(); + + PAssert.that(readPipeline.apply(TextIO.read().from(toFilenamePrefix(folder) + "*"))) + .containsInAnyOrder( + containsAll( + "\"aDouble\":1.0", + "\"aFloat\":1.0", + "\"aLong\":1", + "\"aString\":\"a\"", + "\"anInteger\":1", + "\"aDecimal\":10", + "\"aBoolean\":false"), + containsAll( + "\"aDouble\":2.0", + "\"aFloat\":2.0", + "\"aLong\":2", + "\"aString\":\"b\"", + "\"anInteger\":2", + "\"aDecimal\":20", + "\"aBoolean\":false"), + containsAll( + "\"aDouble\":3.0", + "\"aFloat\":3.0", + "\"aLong\":3", + "\"aString\":\"c\"", + "\"anInteger\":3", + "\"aDecimal\":30", + "\"aBoolean\":false")); + + readPipeline.run(); + } + + private static SerializableMatcher containsAll(String... needles) { + class Matcher extends BaseMatcher implements SerializableMatcher { + @Override + public boolean matches(Object item) { + if (!(item instanceof String)) { + return false; + } + + String haystack = (String) item; + for (String needle : needles) { + if (!haystack.contains(needle)) { + return false; + } + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Contains all of: "); + description.appendValueList("[", ",", "]", needles); + } + } + return new Matcher(); + } + + private static String toFilenamePrefix(File folder) { + checkArgument(folder.isDirectory()); + return folder.getAbsolutePath() + "/out"; + } + + private File createFolder(String... paths) { + try { + return tempFolder.newFolder(paths); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/sdks/java/io/kafka/OWNERS b/sdks/java/io/kafka/OWNERS deleted file mode 100644 index 6d38a74d6ccb8..0000000000000 --- a/sdks/java/io/kafka/OWNERS +++ /dev/null @@ -1,7 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - aromanenko-dev - - chamikaramj - - lukecwik - - johnjcasey diff --git a/sdks/java/io/kafka/build.gradle b/sdks/java/io/kafka/build.gradle index c2bf5b9e2a276..61209aa509284 100644 --- a/sdks/java/io/kafka/build.gradle +++ b/sdks/java/io/kafka/build.gradle @@ -46,7 +46,7 @@ def kafkaVersions = [ kafkaVersions.each{k,v -> configurations.create("kafkaVersion$k")} dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre provided library.java.jackson_dataformat_csv permitUnusedDeclared library.java.jackson_dataformat_csv implementation project(path: ":sdks:java:core", configuration: "shadow") @@ -82,7 +82,7 @@ dependencies { provided library.java.everit_json_schema testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") testImplementation project(":sdks:java:io:synthetic") - testImplementation project(":sdks:java:extensions:avro") + testImplementation project(path: ":sdks:java:extensions:avro", configuration: "testRuntimeMigration") testImplementation project(path: ":sdks:java:io:common", configuration: "testRuntimeMigration") testImplementation project(path: ":sdks:java:testing:test-utils", configuration: "testRuntimeMigration") // For testing Cross-language transforms diff --git a/sdks/java/io/kafka/kafka-integration-test.gradle b/sdks/java/io/kafka/kafka-integration-test.gradle index bfb8c7f5fd029..778f8a3c456cc 100644 --- a/sdks/java/io/kafka/kafka-integration-test.gradle +++ b/sdks/java/io/kafka/kafka-integration-test.gradle @@ -19,6 +19,7 @@ import org.apache.beam.gradle.kafka.KafkaTestUtilities apply plugin: 'org.apache.beam.module' applyJavaNature( + publish: false, automaticModuleName: 'org.apache.beam.sdk.io.kafka', mavenRepositories: [ [id: 'io.confluent', url: 'https://packages.confluent.io/maven/'] @@ -29,7 +30,9 @@ enableJavaPerformanceTesting() dependencies { implementation "org.apache.kafka:kafka-clients:$delimited" + permitUnusedDeclared "org.apache.kafka:kafka-clients:$delimited" implementation project(":sdks:java:io:kafka") + permitUnusedDeclared project(":sdks:java:io:kafka") testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1' } diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ConfluentSchemaRegistryDeserializerProvider.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ConfluentSchemaRegistryDeserializerProvider.java index 268a58a28ff80..6357a68c1a784 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ConfluentSchemaRegistryDeserializerProvider.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ConfluentSchemaRegistryDeserializerProvider.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kafka; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; import io.confluent.kafka.schemaregistry.client.SchemaMetadata; @@ -33,7 +33,7 @@ import org.apache.beam.sdk.coders.CoderRegistry; import org.apache.beam.sdk.extensions.avro.coders.AvroCoder; import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.apache.kafka.common.serialization.Deserializer; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ConsumerSpEL.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ConsumerSpEL.java index 3d38f90fdd68a..16f0b6aaa3ee5 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ConsumerSpEL.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ConsumerSpEL.java @@ -17,12 +17,12 @@ */ package org.apache.beam.sdk.io.kafka; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Collection; import java.util.Map; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.OffsetAndTimestamp; diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/CustomTimestampPolicyWithLimitedDelay.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/CustomTimestampPolicyWithLimitedDelay.java index 7b5e1f7fe9a67..bd482c1846aba 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/CustomTimestampPolicyWithLimitedDelay.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/CustomTimestampPolicyWithLimitedDelay.java @@ -20,7 +20,7 @@ import java.util.Optional; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.apache.kafka.common.TopicPartition; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaCheckpointMark.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaCheckpointMark.java index 8e34cfee3d292..966363e41f62c 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaCheckpointMark.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaCheckpointMark.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.extensions.avro.coders.AvroCoder; import org.apache.beam.sdk.io.UnboundedSource; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; /** * Checkpoint for a {@link KafkaUnboundedReader}. Consists of Kafka topic name, partition id, and diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaCommitOffset.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaCommitOffset.java index aa405b0e18050..3816ee0bb8556 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaCommitOffset.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaCommitOffset.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kafka; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Collections; import java.util.HashMap; diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaExactlyOnceSink.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaExactlyOnceSink.java index 2b22a65f7a042..ced1e24a8c299 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaExactlyOnceSink.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaExactlyOnceSink.java @@ -17,9 +17,9 @@ */ package org.apache.beam.sdk.io.kafka; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; @@ -59,16 +59,16 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; import org.apache.beam.sdk.values.TimestampedValue.TimestampedValueCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.Cache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LoadingCache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.RemovalCause; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.RemovalCause; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.OffsetAndMetadata; @@ -654,7 +654,7 @@ private static class ShardWriterCache { .>removalListener( notification -> { if (notification.getCause() != RemovalCause.EXPLICIT) { - ShardWriter writer = notification.getValue(); + ShardWriter writer = checkNotNull(notification.getValue()); LOG.info( "{} : Closing idle shard writer {} after 1 minute of idle time.", writer.shard, diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java index daa3ff239280f..7275986de8b5b 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.kafka; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.apache.kafka.clients.consumer.ConsumerConfig.AUTO_OFFSET_RESET_CONFIG; import com.google.auto.service.AutoService; @@ -37,6 +37,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.beam.runners.core.construction.PTransformMatchers; import org.apache.beam.runners.core.construction.ReplacementOutputs; @@ -92,10 +93,10 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; @@ -350,10 +351,11 @@ * href="https://beam.apache.org/blog/splittable-do-fn/">blog post and design doc. The major difference from {@link * KafkaIO.Read} is, {@link ReadSourceDescriptors} doesn't require source descriptions(e.g., {@link - * KafkaIO.Read#getTopicPartitions()}, {@link KafkaIO.Read#getTopics()}, {@link - * KafkaIO.Read#getStartReadTime()}, etc.) during the pipeline construction time. Instead, the - * pipeline can populate these source descriptions during runtime. For example, the pipeline can - * query Kafka topics from a BigQuery table and read these topics via {@link ReadSourceDescriptors}. + * KafkaIO.Read#getTopicPattern()}, {@link KafkaIO.Read#getTopicPartitions()}, {@link + * KafkaIO.Read#getTopics()}, {@link KafkaIO.Read#getStartReadTime()}, etc.) during the pipeline + * construction time. Instead, the pipeline can populate these source descriptions during runtime. + * For example, the pipeline can query Kafka topics from a BigQuery table and read these topics via + * {@link ReadSourceDescriptors}. * *

    Common Kafka Consumer Configurations

    * @@ -633,6 +635,9 @@ public abstract static class Read @Pure abstract @Nullable List getTopicPartitions(); + @Pure + abstract @Nullable Pattern getTopicPattern(); + @Pure abstract @Nullable Coder getKeyCoder(); @@ -692,6 +697,8 @@ abstract static class Builder { abstract Builder setTopicPartitions(List topicPartitions); + abstract Builder setTopicPattern(Pattern topicPattern); + abstract Builder setKeyCoder(Coder keyCoder); abstract Builder setValueCoder(Coder valueCoder); @@ -922,8 +929,9 @@ public Read withTopic(String topic) { */ public Read withTopics(List topics) { checkState( - getTopicPartitions() == null || getTopicPartitions().isEmpty(), - "Only topics or topicPartitions can be set, not both"); + (getTopicPartitions() == null || getTopicPartitions().isEmpty()) + && getTopicPattern() == null, + "Only one of topics, topicPartitions or topicPattern can be set"); return toBuilder().setTopics(ImmutableList.copyOf(topics)).build(); } @@ -936,11 +944,26 @@ public Read withTopics(List topics) { */ public Read withTopicPartitions(List topicPartitions) { checkState( - getTopics() == null || getTopics().isEmpty(), - "Only topics or topicPartitions can be set, not both"); + (getTopics() == null || getTopics().isEmpty()) && getTopicPattern() == null, + "Only one of topics, topicPartitions or topicPattern can be set"); return toBuilder().setTopicPartitions(ImmutableList.copyOf(topicPartitions)).build(); } + /** + * Internally sets a {@link java.util.regex.Pattern} of topics to read from. All the partitions + * from each of the matching topics are read. + * + *

    See {@link KafkaUnboundedSource#split(int, PipelineOptions)} for description of how the + * partitions are distributed among the splits. + */ + public Read withTopicPattern(String topicPattern) { + checkState( + (getTopics() == null || getTopics().isEmpty()) + && (getTopicPartitions() == null || getTopicPartitions().isEmpty()), + "Only one of topics, topicPartitions or topicPattern can be set"); + return toBuilder().setTopicPattern(Pattern.compile(topicPattern)).build(); + } + /** * Sets a Kafka {@link Deserializer} to interpret key bytes read from Kafka. * @@ -1274,8 +1297,9 @@ public PCollection> expand(PBegin input) { if (!isDynamicRead()) { checkArgument( (getTopics() != null && getTopics().size() > 0) - || (getTopicPartitions() != null && getTopicPartitions().size() > 0), - "Either withTopic(), withTopics() or withTopicPartitions() is required"); + || (getTopicPartitions() != null && getTopicPartitions().size() > 0) + || getTopicPattern() != null, + "Either withTopic(), withTopics(), withTopicPartitions() or withTopicPattern() is required"); } else { checkArgument( ExperimentalOptions.hasExperiment(input.getPipeline().getOptions(), "beam_fn_api"), @@ -1537,6 +1561,7 @@ public PCollection> expand(PBegin input) { kafkaRead.getConsumerConfig(), kafkaRead.getCheckStopReadingFn(), topics, + kafkaRead.getTopicPattern(), kafkaRead.getStartReadTime(), kafkaRead.getStopReadTime())); } else { @@ -1561,6 +1586,7 @@ static class GenerateKafkaSourceDescriptor extends DoFn topics; + private final @Nullable Pattern topicPattern; + @ProcessElement public void processElement(OutputReceiver receiver) { List partitions = new ArrayList<>(Preconditions.checkStateNotNull(topicPartitions)); if (partitions.isEmpty()) { try (Consumer consumer = consumerFactoryFn.apply(consumerConfig)) { - for (String topic : Preconditions.checkStateNotNull(topics)) { - for (PartitionInfo p : consumer.partitionsFor(topic)) { - partitions.add(new TopicPartition(p.topic(), p.partition())); + List topics = Preconditions.checkStateNotNull(this.topics); + if (topics.isEmpty()) { + Pattern pattern = Preconditions.checkStateNotNull(topicPattern); + for (Map.Entry> entry : + consumer.listTopics().entrySet()) { + if (pattern.matcher(entry.getKey()).matches()) { + for (PartitionInfo p : entry.getValue()) { + partitions.add(new TopicPartition(p.topic(), p.partition())); + } + } + } + } else { + for (String topic : topics) { + for (PartitionInfo p : consumer.partitionsFor(topic)) { + partitions.add(new TopicPartition(p.topic(), p.partition())); + } } } } @@ -1634,12 +1675,16 @@ public void populateDisplayData(DisplayData.Builder builder) { super.populateDisplayData(builder); List topics = Preconditions.checkStateNotNull(getTopics()); List topicPartitions = Preconditions.checkStateNotNull(getTopicPartitions()); + Pattern topicPattern = getTopicPattern(); if (topics.size() > 0) { builder.add(DisplayData.item("topics", Joiner.on(",").join(topics)).withLabel("Topic/s")); } else if (topicPartitions.size() > 0) { builder.add( DisplayData.item("topicPartitions", Joiner.on(",").join(topicPartitions)) .withLabel("Topic Partition/s")); + } else if (topicPattern != null) { + builder.add( + DisplayData.item("topicPattern", topicPattern.pattern()).withLabel("Topic Pattern")); } Set disallowedConsumerPropertiesKeys = KafkaIOUtils.DISALLOWED_CONSUMER_PROPERTIES.keySet(); diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIOReadImplementationCompatibility.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIOReadImplementationCompatibility.java index 1db28d2a3e6f0..b779de1d9cf1a 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIOReadImplementationCompatibility.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIOReadImplementationCompatibility.java @@ -28,13 +28,13 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.apache.beam.sdk.values.PBegin; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.CaseFormat; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.HashMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSortedSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CaseFormat; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSortedSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.checkerframework.checker.initialization.qual.UnderInitialization; /** @@ -79,6 +79,7 @@ enum KafkaIOReadProperties { CONSUMER_CONFIG, TOPICS, TOPIC_PARTITIONS, + TOPIC_PATTERN, KEY_CODER, VALUE_CODER, CONSUMER_FACTORY_FN, diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIOUtils.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIOUtils.java index 5c0fe5c05535b..ad17369715dd7 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIOUtils.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIOUtils.java @@ -17,13 +17,13 @@ */ package org.apache.beam.sdk.io.kafka; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.HashMap; import java.util.Map; import java.util.Random; import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformConfiguration.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformConfiguration.java index b68b7123a9353..f7e0915806803 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformConfiguration.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformConfiguration.java @@ -24,7 +24,7 @@ import org.apache.beam.sdk.schemas.AutoValueSchema; import org.apache.beam.sdk.schemas.annotations.DefaultSchema; import org.apache.beam.sdk.schemas.annotations.SchemaFieldDescription; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; /** * Configuration for reading from a Kafka topic. diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProvider.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProvider.java index 3c8472c794ec6..0c091bf9ba847 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProvider.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProvider.java @@ -47,7 +47,6 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.DoFn.FinishBundle; import org.apache.beam.sdk.transforms.DoFn.ProcessElement; -import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.Values; @@ -57,11 +56,11 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.common.serialization.ByteArrayDeserializer; @@ -100,7 +99,98 @@ protected Class configurationClass() { @Override protected SchemaTransform from(KafkaReadSchemaTransformConfiguration configuration) { - return new KafkaReadSchemaTransform(configuration, isTest, testTimeoutSecs); + final String inputSchema = configuration.getSchema(); + final Integer groupId = configuration.hashCode() % Integer.MAX_VALUE; + final String autoOffsetReset = + MoreObjects.firstNonNull(configuration.getAutoOffsetResetConfig(), "latest"); + + Map consumerConfigs = + new HashMap<>( + MoreObjects.firstNonNull(configuration.getConsumerConfigUpdates(), new HashMap<>())); + consumerConfigs.put(ConsumerConfig.GROUP_ID_CONFIG, "kafka-read-provider-" + groupId); + consumerConfigs.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true); + consumerConfigs.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, 100); + consumerConfigs.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset); + + if (inputSchema != null && !inputSchema.isEmpty()) { + assert Strings.isNullOrEmpty(configuration.getConfluentSchemaRegistryUrl()) + : "To read from Kafka, a schema must be provided directly or though Confluent " + + "Schema Registry, but not both."; + final Schema beamSchema = + Objects.equals(configuration.getFormat(), "JSON") + ? JsonUtils.beamSchemaFromJsonSchema(inputSchema) + : AvroUtils.toBeamSchema(new org.apache.avro.Schema.Parser().parse(inputSchema)); + SerializableFunction valueMapper = + Objects.equals(configuration.getFormat(), "JSON") + ? JsonUtils.getJsonBytesToRowFunction(beamSchema) + : AvroUtils.getAvroBytesToRowFunction(beamSchema); + return new SchemaTransform() { + @Override + public PCollectionRowTuple expand(PCollectionRowTuple input) { + KafkaIO.Read kafkaRead = + KafkaIO.readBytes() + .withConsumerConfigUpdates(consumerConfigs) + .withConsumerFactoryFn(new ConsumerFactoryWithGcsTrustStores()) + .withTopic(configuration.getTopic()) + .withBootstrapServers(configuration.getBootstrapServers()); + if (isTest) { + kafkaRead = kafkaRead.withMaxReadTime(Duration.standardSeconds(testTimeoutSecs)); + } + + PCollection kafkaValues = + input.getPipeline().apply(kafkaRead.withoutMetadata()).apply(Values.create()); + + PCollectionTuple outputTuple = + kafkaValues.apply( + ParDo.of(new ErrorFn("Kafka-read-error-counter", valueMapper)) + .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); + + return PCollectionRowTuple.of( + "output", + outputTuple.get(OUTPUT_TAG).setRowSchema(beamSchema), + "errors", + outputTuple.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA)); + } + }; + } else { + assert !Strings.isNullOrEmpty(configuration.getConfluentSchemaRegistryUrl()) + : "To read from Kafka, a schema must be provided directly or though Confluent " + + "Schema Registry. Neither seems to have been provided."; + return new SchemaTransform() { + @Override + public PCollectionRowTuple expand(PCollectionRowTuple input) { + final String confluentSchemaRegUrl = configuration.getConfluentSchemaRegistryUrl(); + final String confluentSchemaRegSubject = + configuration.getConfluentSchemaRegistrySubject(); + if (confluentSchemaRegUrl == null || confluentSchemaRegSubject == null) { + throw new IllegalArgumentException( + "To read from Kafka, a schema must be provided directly or though Confluent " + + "Schema Registry. Make sure you are providing one of these parameters."); + } + KafkaIO.Read kafkaRead = + KafkaIO.read() + .withTopic(configuration.getTopic()) + .withConsumerFactoryFn(new ConsumerFactoryWithGcsTrustStores()) + .withBootstrapServers(configuration.getBootstrapServers()) + .withConsumerConfigUpdates(consumerConfigs) + .withKeyDeserializer(ByteArrayDeserializer.class) + .withValueDeserializer( + ConfluentSchemaRegistryDeserializerProvider.of( + confluentSchemaRegUrl, confluentSchemaRegSubject)); + if (isTest) { + kafkaRead = kafkaRead.withMaxReadTime(Duration.standardSeconds(testTimeoutSecs)); + } + + PCollection kafkaValues = + input.getPipeline().apply(kafkaRead.withoutMetadata()).apply(Values.create()); + + assert kafkaValues.getCoder().getClass() == AvroCoder.class; + AvroCoder coder = (AvroCoder) kafkaValues.getCoder(); + kafkaValues = kafkaValues.setCoder(AvroUtils.schemaCoder(coder.getSchema())); + return PCollectionRowTuple.of("output", kafkaValues.apply(Convert.toRows())); + } + }; + } } @Override @@ -148,118 +238,6 @@ public void finish(FinishBundleContext c) { } } - private static class KafkaReadSchemaTransform implements SchemaTransform { - private final KafkaReadSchemaTransformConfiguration configuration; - private final Boolean isTest; - private final Integer testTimeoutSeconds; - - KafkaReadSchemaTransform( - KafkaReadSchemaTransformConfiguration configuration, - Boolean isTest, - Integer testTimeoutSeconds) { - configuration.validate(); - this.configuration = configuration; - this.isTest = isTest; - this.testTimeoutSeconds = testTimeoutSeconds; - } - - @Override - public PTransform buildTransform() { - final String inputSchema = configuration.getSchema(); - final Integer groupId = configuration.hashCode() % Integer.MAX_VALUE; - final String autoOffsetReset = - MoreObjects.firstNonNull(configuration.getAutoOffsetResetConfig(), "latest"); - - Map consumerConfigs = - new HashMap<>( - MoreObjects.firstNonNull(configuration.getConsumerConfigUpdates(), new HashMap<>())); - consumerConfigs.put(ConsumerConfig.GROUP_ID_CONFIG, "kafka-read-provider-" + groupId); - consumerConfigs.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true); - consumerConfigs.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, 100); - consumerConfigs.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset); - - if (inputSchema != null && !inputSchema.isEmpty()) { - assert Strings.isNullOrEmpty(configuration.getConfluentSchemaRegistryUrl()) - : "To read from Kafka, a schema must be provided directly or though Confluent " - + "Schema Registry, but not both."; - final Schema beamSchema = - Objects.equals(configuration.getFormat(), "JSON") - ? JsonUtils.beamSchemaFromJsonSchema(inputSchema) - : AvroUtils.toBeamSchema(new org.apache.avro.Schema.Parser().parse(inputSchema)); - SerializableFunction valueMapper = - Objects.equals(configuration.getFormat(), "JSON") - ? JsonUtils.getJsonBytesToRowFunction(beamSchema) - : AvroUtils.getAvroBytesToRowFunction(beamSchema); - return new PTransform() { - @Override - public PCollectionRowTuple expand(PCollectionRowTuple input) { - KafkaIO.Read kafkaRead = - KafkaIO.readBytes() - .withConsumerConfigUpdates(consumerConfigs) - .withConsumerFactoryFn(new ConsumerFactoryWithGcsTrustStores()) - .withTopic(configuration.getTopic()) - .withBootstrapServers(configuration.getBootstrapServers()); - if (isTest) { - kafkaRead = kafkaRead.withMaxReadTime(Duration.standardSeconds(testTimeoutSeconds)); - } - - PCollection kafkaValues = - input.getPipeline().apply(kafkaRead.withoutMetadata()).apply(Values.create()); - - PCollectionTuple outputTuple = - kafkaValues.apply( - ParDo.of(new ErrorFn("Kafka-read-error-counter", valueMapper)) - .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); - - return PCollectionRowTuple.of( - "output", - outputTuple.get(OUTPUT_TAG).setRowSchema(beamSchema), - "errors", - outputTuple.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA)); - } - }; - } else { - assert !Strings.isNullOrEmpty(configuration.getConfluentSchemaRegistryUrl()) - : "To read from Kafka, a schema must be provided directly or though Confluent " - + "Schema Registry. Neither seems to have been provided."; - return new PTransform() { - @Override - public PCollectionRowTuple expand(PCollectionRowTuple input) { - final String confluentSchemaRegUrl = configuration.getConfluentSchemaRegistryUrl(); - final String confluentSchemaRegSubject = - configuration.getConfluentSchemaRegistrySubject(); - if (confluentSchemaRegUrl == null || confluentSchemaRegSubject == null) { - throw new IllegalArgumentException( - "To read from Kafka, a schema must be provided directly or though Confluent " - + "Schema Registry. Make sure you are providing one of these parameters."); - } - KafkaIO.Read kafkaRead = - KafkaIO.read() - .withTopic(configuration.getTopic()) - .withConsumerFactoryFn(new ConsumerFactoryWithGcsTrustStores()) - .withBootstrapServers(configuration.getBootstrapServers()) - .withConsumerConfigUpdates(consumerConfigs) - .withKeyDeserializer(ByteArrayDeserializer.class) - .withValueDeserializer( - ConfluentSchemaRegistryDeserializerProvider.of( - confluentSchemaRegUrl, confluentSchemaRegSubject)); - if (isTest) { - kafkaRead = kafkaRead.withMaxReadTime(Duration.standardSeconds(testTimeoutSeconds)); - } - - PCollection kafkaValues = - input.getPipeline().apply(kafkaRead.withoutMetadata()).apply(Values.create()); - - assert kafkaValues.getCoder().getClass() == AvroCoder.class; - AvroCoder coder = (AvroCoder) kafkaValues.getCoder(); - kafkaValues = kafkaValues.setCoder(AvroUtils.schemaCoder(coder.getSchema())); - return PCollectionRowTuple.of("output", kafkaValues.apply(Convert.toRows())); - } - }; - } - } - }; - private static class ConsumerFactoryWithGcsTrustStores implements SerializableFunction, Consumer> { diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecord.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecord.java index 4d2c335c4174b..60aa68e0129c0 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecord.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaRecord.java @@ -19,7 +19,7 @@ import java.util.Arrays; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.apache.kafka.common.header.Headers; import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.dataflow.qual.Pure; diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaSourceDescriptor.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaSourceDescriptor.java index 6a47debf32921..d0d411c2fe276 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaSourceDescriptor.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaSourceDescriptor.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kafka; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.io.Serializable; diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaUnboundedReader.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaUnboundedReader.java index ab92370a3578b..6e6df42de8bfd 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaUnboundedReader.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaUnboundedReader.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kafka; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.util.ArrayList; @@ -50,11 +50,11 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Closeables; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Closeables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaUnboundedSource.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaUnboundedSource.java index 4af13bbf4749d..5d8a2556e4743 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaUnboundedSource.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaUnboundedSource.java @@ -17,20 +17,22 @@ */ package org.apache.beam.sdk.io.kafka; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.extensions.avro.coders.AvroCoder; import org.apache.beam.sdk.io.UnboundedSource; import org.apache.beam.sdk.io.kafka.KafkaIO.Read; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.util.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; @@ -65,22 +67,33 @@ public List> split(int desiredNumSplits, PipelineOpti if (partitions.isEmpty()) { try (Consumer consumer = spec.getConsumerFactoryFn().apply(spec.getConsumerConfig())) { - for (String topic : Preconditions.checkStateNotNull(spec.getTopics())) { - List partitionInfoList = consumer.partitionsFor(topic); - checkState( - partitionInfoList != null, - "Could not find any partitions info. Please check Kafka configuration and make sure " - + "that provided topics exist."); - for (PartitionInfo p : partitionInfoList) { - partitions.add(new TopicPartition(p.topic(), p.partition())); + List topics = Preconditions.checkStateNotNull(spec.getTopics()); + if (topics.isEmpty()) { + Pattern pattern = Preconditions.checkStateNotNull(spec.getTopicPattern()); + for (Map.Entry> entry : consumer.listTopics().entrySet()) { + if (pattern.matcher(entry.getKey()).matches()) { + for (PartitionInfo p : entry.getValue()) { + partitions.add(new TopicPartition(p.topic(), p.partition())); + } + } + } + } else { + for (String topic : topics) { + List partitionInfoList = consumer.partitionsFor(topic); + checkState( + partitionInfoList != null, + "Could not find any partitions info. Please check Kafka configuration and make sure " + + "that provided topics exist."); + for (PartitionInfo p : partitionInfoList) { + partitions.add(new TopicPartition(p.topic(), p.partition())); + } } } } } partitions.sort( - Comparator.comparing(TopicPartition::topic) - .thenComparing(Comparator.comparingInt(TopicPartition::partition))); + Comparator.comparing(TopicPartition::topic).thenComparingInt(TopicPartition::partition)); checkArgument(desiredNumSplits > 0); checkState( @@ -162,7 +175,7 @@ public Coder> getOutputCoder() { private static final Logger LOG = LoggerFactory.getLogger(KafkaUnboundedSource.class); - private final Read spec; // Contains all the relevant configuratiton of the source. + private final Read spec; // Contains all the relevant configuration of the source. private final int id; // split id, mainly for debugging public KafkaUnboundedSource(Read spec, int id) { diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaWriteSchemaTransformProvider.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaWriteSchemaTransformProvider.java index d59edbae5a678..876ef9a49e8a2 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaWriteSchemaTransformProvider.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaWriteSchemaTransformProvider.java @@ -39,7 +39,6 @@ import org.apache.beam.sdk.schemas.utils.JsonUtils; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.DoFn.ProcessElement; -import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.KV; @@ -48,7 +47,7 @@ import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.apache.kafka.common.serialization.ByteArraySerializer; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.NonNull; @@ -92,7 +91,7 @@ public class KafkaWriteSchemaTransformProvider return new KafkaWriteSchemaTransform(configuration); } - static final class KafkaWriteSchemaTransform implements SchemaTransform, Serializable { + static final class KafkaWriteSchemaTransform extends SchemaTransform implements Serializable { final KafkaWriteSchemaTransformConfiguration configuration; KafkaWriteSchemaTransform(KafkaWriteSchemaTransformConfiguration configuration) { @@ -130,45 +129,37 @@ public void finish() { } @Override - public @UnknownKeyFor @NonNull @Initialized PTransform< - @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple, - @UnknownKeyFor @NonNull @Initialized PCollectionRowTuple> - buildTransform() { - return new PTransform() { - @Override - public PCollectionRowTuple expand(PCollectionRowTuple input) { - Schema inputSchema = input.get("input").getSchema(); - final SerializableFunction toBytesFn = - configuration.getFormat().equals("JSON") - ? JsonUtils.getRowToJsonBytesFunction(inputSchema) - : AvroUtils.getRowToAvroBytesFunction(inputSchema); - - final Map configOverrides = configuration.getProducerConfigUpdates(); - PCollectionTuple outputTuple = - input - .get("input") - .apply( - "Map rows to Kafka messages", - ParDo.of(new ErrorCounterFn("Kafka-write-error-counter", toBytesFn)) - .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); - - outputTuple - .get(OUTPUT_TAG) + public PCollectionRowTuple expand(PCollectionRowTuple input) { + Schema inputSchema = input.get("input").getSchema(); + final SerializableFunction toBytesFn = + configuration.getFormat().equals("JSON") + ? JsonUtils.getRowToJsonBytesFunction(inputSchema) + : AvroUtils.getRowToAvroBytesFunction(inputSchema); + + final Map configOverrides = configuration.getProducerConfigUpdates(); + PCollectionTuple outputTuple = + input + .get("input") .apply( - KafkaIO.write() - .withTopic(configuration.getTopic()) - .withBootstrapServers(configuration.getBootstrapServers()) - .withProducerConfigUpdates( - configOverrides == null - ? new HashMap<>() - : new HashMap(configOverrides)) - .withKeySerializer(ByteArraySerializer.class) - .withValueSerializer(ByteArraySerializer.class)); - - return PCollectionRowTuple.of( - "errors", outputTuple.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA)); - } - }; + "Map rows to Kafka messages", + ParDo.of(new ErrorCounterFn("Kafka-write-error-counter", toBytesFn)) + .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG))); + + outputTuple + .get(OUTPUT_TAG) + .apply( + KafkaIO.write() + .withTopic(configuration.getTopic()) + .withBootstrapServers(configuration.getBootstrapServers()) + .withProducerConfigUpdates( + configOverrides == null + ? new HashMap<>() + : new HashMap(configOverrides)) + .withKeySerializer(ByteArraySerializer.class) + .withValueSerializer(ByteArraySerializer.class)); + + return PCollectionRowTuple.of( + "errors", outputTuple.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA)); } } diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/LocalDeserializerProvider.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/LocalDeserializerProvider.java index b9da0f61840c2..62dcf96149df8 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/LocalDeserializerProvider.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/LocalDeserializerProvider.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kafka; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.ParameterizedType; diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFn.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFn.java index 7a8acd88e7212..31620549ab222 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFn.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFn.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kafka; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.HashMap; import java.util.HashSet; @@ -43,15 +43,15 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.util.Preconditions; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Stopwatch; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Suppliers; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LoadingCache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Closeables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Stopwatch; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Suppliers; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Closeables; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -194,6 +194,7 @@ private ReadFromKafkaDoFn(ReadSourceDescriptors transform) { // Valid between bundle start and bundle finish. private transient @Nullable Deserializer keyDeserializerInstance = null; private transient @Nullable Deserializer valueDeserializerInstance = null; + private transient @Nullable Map offsetEstimatorCache; private transient @Nullable LoadingCache avgRecordSize; @@ -213,6 +214,7 @@ private static class KafkaLatestOffsetEstimator private final Consumer offsetConsumer; private final TopicPartition topicPartition; private final Supplier memoizedBacklog; + private boolean closed; KafkaLatestOffsetEstimator( Consumer offsetConsumer, TopicPartition topicPartition) { @@ -222,8 +224,10 @@ private static class KafkaLatestOffsetEstimator memoizedBacklog = Suppliers.memoizeWithExpiration( () -> { - ConsumerSpEL.evaluateSeek2End(offsetConsumer, topicPartition); - return offsetConsumer.position(topicPartition); + synchronized (offsetConsumer) { + ConsumerSpEL.evaluateSeek2End(offsetConsumer, topicPartition); + return offsetConsumer.position(topicPartition); + } }, 1, TimeUnit.SECONDS); @@ -233,6 +237,8 @@ private static class KafkaLatestOffsetEstimator protected void finalize() { try { Closeables.close(offsetConsumer, true); + closed = true; + LOG.info("Offset Estimator consumer was closed for {}", topicPartition); } catch (Exception anyException) { LOG.warn("Failed to close offset consumer for {}", topicPartition); } @@ -242,12 +248,19 @@ protected void finalize() { public long estimate() { return memoizedBacklog.get(); } + + public boolean isClosed() { + return closed; + } } @GetInitialRestriction public OffsetRange initialRestriction(@Element KafkaSourceDescriptor kafkaSourceDescriptor) { Map updatedConsumerConfig = overrideBootstrapServersConfig(consumerConfig, kafkaSourceDescriptor); + LOG.info( + "Creating Kafka consumer for initial restriction for {}", + kafkaSourceDescriptor.getTopicPartition()); try (Consumer offsetConsumer = consumerFactoryFn.apply(updatedConsumerConfig)) { ConsumerSpEL.evaluateAssign( offsetConsumer, ImmutableList.of(kafkaSourceDescriptor.getTopicPartition())); @@ -311,17 +324,30 @@ public OffsetRangeTracker restrictionTracker( if (restriction.getTo() < Long.MAX_VALUE) { return new OffsetRangeTracker(restriction); } - Map updatedConsumerConfig = - overrideBootstrapServersConfig(consumerConfig, kafkaSourceDescriptor); - KafkaLatestOffsetEstimator offsetPoller = - new KafkaLatestOffsetEstimator( - consumerFactoryFn.apply( - KafkaIOUtils.getOffsetConsumerConfig( - "tracker-" + kafkaSourceDescriptor.getTopicPartition(), - offsetConsumerConfig, - updatedConsumerConfig)), - kafkaSourceDescriptor.getTopicPartition()); - return new GrowableOffsetRangeTracker(restriction.getFrom(), offsetPoller); + + // OffsetEstimators are cached for each topic-partition because they hold a stateful connection, + // so we want to minimize the amount of connections that we start and track with Kafka. Another + // point is that it has a memoized backlog, and this should make that more reusable estimations. + final Map offsetEstimatorCacheInstance = + Preconditions.checkStateNotNull(this.offsetEstimatorCache); + + TopicPartition topicPartition = kafkaSourceDescriptor.getTopicPartition(); + KafkaLatestOffsetEstimator offsetEstimator = offsetEstimatorCacheInstance.get(topicPartition); + if (offsetEstimator == null || offsetEstimator.isClosed()) { + Map updatedConsumerConfig = + overrideBootstrapServersConfig(consumerConfig, kafkaSourceDescriptor); + + LOG.info("Creating Kafka consumer for offset estimation for {}", topicPartition); + + Consumer offsetConsumer = + consumerFactoryFn.apply( + KafkaIOUtils.getOffsetConsumerConfig( + "tracker-" + topicPartition, offsetConsumerConfig, updatedConsumerConfig)); + offsetEstimator = new KafkaLatestOffsetEstimator(offsetConsumer, topicPartition); + offsetEstimatorCacheInstance.put(topicPartition, offsetEstimator); + } + + return new GrowableOffsetRangeTracker(restriction.getFrom(), offsetEstimator); } @ProcessElement @@ -355,6 +381,10 @@ public ProcessContinuation processElement( kafkaSourceDescriptor.getTopicPartition(), Optional.ofNullable(watermarkEstimator.currentWatermark())); } + + LOG.info( + "Creating Kafka consumer for process continuation for {}", + kafkaSourceDescriptor.getTopicPartition()); try (Consumer consumer = consumerFactoryFn.apply(updatedConsumerConfig)) { // Check whether current TopicPartition is still available to read. Set existingTopicPartitions = new HashSet<>(); @@ -483,6 +513,7 @@ public AverageRecordSize load(TopicPartition topicPartition) throws Exception { }); keyDeserializerInstance = keyDeserializerProvider.getDeserializer(consumerConfig, true); valueDeserializerInstance = valueDeserializerProvider.getDeserializer(consumerConfig, false); + offsetEstimatorCache = new HashMap<>(); } @Teardown @@ -497,6 +528,10 @@ public void teardown() throws Exception { } catch (Exception anyException) { LOG.warn("Fail to close resource during finishing bundle.", anyException); } + + if (offsetEstimatorCache != null) { + offsetEstimatorCache.clear(); + } } private Map overrideBootstrapServersConfig( diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/TimestampPolicyFactory.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/TimestampPolicyFactory.java index 4e50263abe473..53a9a1c0004b7 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/TimestampPolicyFactory.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/TimestampPolicyFactory.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kafka; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.Serializable; import java.util.Optional; diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/TopicPartitionCoder.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/TopicPartitionCoder.java index 475e156e35662..be3684016bec5 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/TopicPartitionCoder.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/TopicPartitionCoder.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.coders.StructuredCoder; import org.apache.beam.sdk.coders.VarIntCoder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.kafka.common.TopicPartition; /** The {@link Coder} for encoding and decoding {@link TopicPartition} in Beam. */ diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/WatchForKafkaTopicPartitions.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/WatchForKafkaTopicPartitions.java index 21fec1bd27b1c..ed67257ca4542 100644 --- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/WatchForKafkaTopicPartitions.java +++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/WatchForKafkaTopicPartitions.java @@ -17,13 +17,14 @@ */ package org.apache.beam.sdk.io.kafka; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.firstNonNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.regex.Pattern; import org.apache.beam.sdk.metrics.Counter; import org.apache.beam.sdk.metrics.Metrics; import org.apache.beam.sdk.transforms.DoFn; @@ -36,7 +37,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; @@ -64,6 +65,7 @@ class WatchForKafkaTopicPartitions extends PTransform kafkaConsumerConfig; private final @Nullable SerializableFunction checkStopReadingFn; private final Set topics; + private final @Nullable Pattern topicPattern; private final @Nullable Instant startReadTime; private final @Nullable Instant stopReadTime; @@ -73,6 +75,7 @@ public WatchForKafkaTopicPartitions( Map kafkaConsumerConfig, @Nullable SerializableFunction checkStopReadingFn, Set topics, + @Nullable Pattern topicPattern, @Nullable Instant startReadTime, @Nullable Instant stopReadTime) { this.checkDuration = firstNonNull(checkDuration, DEFAULT_CHECK_DURATION); @@ -80,6 +83,7 @@ public WatchForKafkaTopicPartitions( this.kafkaConsumerConfig = kafkaConsumerConfig; this.checkStopReadingFn = checkStopReadingFn; this.topics = topics; + this.topicPattern = topicPattern; this.startReadTime = startReadTime; this.stopReadTime = stopReadTime; } @@ -91,7 +95,8 @@ public PCollection expand(PBegin input) { .apply( "Match new TopicPartitions", Watch.growthOf( - new WatchPartitionFn(kafkaConsumerFactoryFn, kafkaConsumerConfig, topics)) + new WatchPartitionFn( + kafkaConsumerFactoryFn, kafkaConsumerConfig, topics, topicPattern)) .withPollInterval(checkDuration)) .apply(ParDo.of(new ConvertToDescriptor(checkStopReadingFn, startReadTime, stopReadTime))); } @@ -134,14 +139,17 @@ private static class WatchPartitionFn extends PollFn { kafkaConsumerFactoryFn; private final Map kafkaConsumerConfig; private final Set topics; + private final @Nullable Pattern topicPattern; private WatchPartitionFn( SerializableFunction, Consumer> kafkaConsumerFactoryFn, Map kafkaConsumerConfig, - Set topics) { + Set topics, + @Nullable Pattern topicPattern) { this.kafkaConsumerFactoryFn = kafkaConsumerFactoryFn; this.kafkaConsumerConfig = kafkaConsumerConfig; this.topics = topics; + this.topicPattern = topicPattern; } @Override @@ -149,7 +157,9 @@ public Watch.Growth.PollResult apply(byte[] element, Context c) throws Exception { Instant now = Instant.now(); return Watch.Growth.PollResult.incomplete( - now, getAllTopicPartitions(kafkaConsumerFactoryFn, kafkaConsumerConfig, topics)) + now, + getAllTopicPartitions( + kafkaConsumerFactoryFn, kafkaConsumerConfig, topics, topicPattern)) .withWatermark(now); } } @@ -158,7 +168,8 @@ now, getAllTopicPartitions(kafkaConsumerFactoryFn, kafkaConsumerConfig, topics)) static List getAllTopicPartitions( SerializableFunction, Consumer> kafkaConsumerFactoryFn, Map kafkaConsumerConfig, - Set topics) { + Set topics, + @Nullable Pattern topicPattern) { List current = new ArrayList<>(); try (Consumer kafkaConsumer = kafkaConsumerFactoryFn.apply(kafkaConsumerConfig)) { @@ -168,12 +179,13 @@ static List getAllTopicPartitions( current.add(new TopicPartition(topic, partition.partition())); } } - } else { for (Map.Entry> topicInfo : kafkaConsumer.listTopics().entrySet()) { - for (PartitionInfo partition : topicInfo.getValue()) { - current.add(new TopicPartition(topicInfo.getKey(), partition.partition())); + if (topicPattern == null || topicPattern.matcher(topicInfo.getKey()).matches()) { + for (PartitionInfo partition : topicInfo.getValue()) { + current.add(new TopicPartition(partition.topic(), partition.partition())); + } } } } diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/CustomTimestampPolicyWithLimitedDelayTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/CustomTimestampPolicyWithLimitedDelayTest.java index 1a86070f2e618..59098fb3ec0e7 100644 --- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/CustomTimestampPolicyWithLimitedDelayTest.java +++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/CustomTimestampPolicyWithLimitedDelayTest.java @@ -26,7 +26,7 @@ import java.util.Optional; import java.util.stream.Collectors; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.kafka.common.header.internals.RecordHeaders; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOExternalTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOExternalTest.java index 889f90c2b99d7..2ccf7dcc3a934 100644 --- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOExternalTest.java +++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOExternalTest.java @@ -46,9 +46,9 @@ import org.apache.beam.sdk.util.ByteStringOutputStream; import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.header.internals.RecordHeaders; diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java index c6d07466040b8..f600e14d30f69 100644 --- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java +++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java @@ -81,9 +81,9 @@ import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.NewPartitions; import org.apache.kafka.clients.admin.NewTopic; @@ -595,8 +595,7 @@ public void runReadWriteKafkaViaSchemaTransforms( .setTopic(topicName) .setBootstrapServers(options.getKafkaBootstrapServerAddresses()) .setFormat(format) - .build()) - .buildTransform()); + .build())); PAssert.that( PCollectionRowTuple.empty(readPipeline) @@ -613,8 +612,7 @@ public void runReadWriteKafkaViaSchemaTransforms( .setSchema(schemaDefinition) .setTopic(topicName) .setBootstrapServers(options.getKafkaBootstrapServerAddresses()) - .build()) - .buildTransform()) + .build())) .get("output")) .containsInAnyOrder( LongStream.range(0L, 1000L) diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOReadImplementationCompatibilityTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOReadImplementationCompatibilityTest.java index d69c8bcd1cc7a..689780c0652d5 100644 --- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOReadImplementationCompatibilityTest.java +++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOReadImplementationCompatibilityTest.java @@ -32,8 +32,8 @@ import org.apache.beam.sdk.io.kafka.KafkaIOReadImplementationCompatibility.KafkaIOReadProperties; import org.apache.beam.sdk.io.kafka.KafkaIOTest.ValueAsTimestampFn; import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.CaseFormat; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CaseFormat; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Rule; diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java index d170b2a237b25..52ab3e20f793c 100644 --- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java +++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java @@ -68,7 +68,7 @@ import org.apache.beam.sdk.coders.BigEndianLongCoder; import org.apache.beam.sdk.coders.VarIntCoder; import org.apache.beam.sdk.coders.VarLongCoder; -import org.apache.beam.sdk.io.AvroGeneratedUser; +import org.apache.beam.sdk.extensions.avro.io.AvroGeneratedUser; import org.apache.beam.sdk.io.Read; import org.apache.beam.sdk.io.UnboundedSource; import org.apache.beam.sdk.io.UnboundedSource.UnboundedReader; @@ -106,10 +106,10 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -723,6 +723,87 @@ public void testUnboundedSourceWithExplicitPartitions() { p.run(); } + @Test + public void testUnboundedSourceWithPattern() { + int numElements = 1000; + + List topics = + ImmutableList.of( + "best", "gest", "hest", "jest", "lest", "nest", "pest", "rest", "test", "vest", "west", + "zest"); + + KafkaIO.Read reader = + KafkaIO.read() + .withBootstrapServers("none") + .withTopicPattern("[a-z]est") + .withConsumerFactoryFn( + new ConsumerFactoryFn(topics, 10, numElements, OffsetResetStrategy.EARLIEST)) + .withKeyDeserializer(ByteArrayDeserializer.class) + .withValueDeserializer(LongDeserializer.class) + .withMaxNumRecords(numElements); + + PCollection input = p.apply(reader.withoutMetadata()).apply(Values.create()); + + addCountingAsserts(input, numElements); + p.run(); + } + + @Test + public void testUnboundedSourceWithPartiallyMatchedPattern() { + int numElements = 1000; + long numMatchedElements = numElements / 2; // Expected elements if split across 2 topics + + List topics = ImmutableList.of("test", "Test"); + + KafkaIO.Read reader = + KafkaIO.read() + .withBootstrapServers("none") + .withTopicPattern("[a-z]est") + .withConsumerFactoryFn( + new ConsumerFactoryFn(topics, 1, numElements, OffsetResetStrategy.EARLIEST)) + .withKeyDeserializer(ByteArrayDeserializer.class) + .withValueDeserializer(LongDeserializer.class) + .withMaxNumRecords(numMatchedElements); + + PCollection input = p.apply(reader.withoutMetadata()).apply(Values.create()); + + // With 1 partition per topic element to partition allocation alternates between test and Test, + // producing even elements for test and odd elements for Test. + // The pattern only matches test, so we expect even elements. + PAssert.that(input).satisfies(new AssertMultipleOf(2)); + + PAssert.thatSingleton(input.apply("Count", Count.globally())).isEqualTo(numMatchedElements); + + p.run(); + } + + @Test + public void testUnboundedSourceWithUnmatchedPattern() { + // Expect an exception when provided pattern doesn't match any Kafka topics. + thrown.expect(PipelineExecutionException.class); + thrown.expectCause(instanceOf(IllegalStateException.class)); + thrown.expectMessage( + "Could not find any partitions. Please check Kafka configuration and topic names"); + + int numElements = 1000; + + List topics = ImmutableList.of("chest", "crest", "egest", "guest", "quest", "wrest"); + + KafkaIO.Read reader = + KafkaIO.read() + .withBootstrapServers("none") + .withTopicPattern("[a-z]est") + .withConsumerFactoryFn( + new ConsumerFactoryFn(topics, 10, numElements, OffsetResetStrategy.EARLIEST)) + .withKeyDeserializer(ByteArrayDeserializer.class) + .withValueDeserializer(LongDeserializer.class) + .withMaxNumRecords(numElements); + + p.apply(reader.withoutMetadata()).apply(Values.create()); + + p.run(); + } + @Test public void testUnboundedSourceWithWrongTopic() { // Expect an exception when provided Kafka topic doesn't exist. @@ -1829,6 +1910,25 @@ public void testSourceWithExplicitPartitionsDisplayData() { assertThat(displayData, hasDisplayItem("receive.buffer.bytes", 524288)); } + @Test + public void testSourceWithPatternDisplayData() { + KafkaIO.Read read = + KafkaIO.readBytes() + .withBootstrapServers("myServer1:9092,myServer2:9092") + .withTopicPattern("[a-z]est") + .withConsumerFactoryFn( + new ConsumerFactoryFn( + Lists.newArrayList("test"), 10, 10, OffsetResetStrategy.EARLIEST)); + + DisplayData displayData = DisplayData.from(read); + + assertThat(displayData, hasDisplayItem("topicPattern", "[a-z]est")); + assertThat(displayData, hasDisplayItem("enable.auto.commit", false)); + assertThat(displayData, hasDisplayItem("bootstrap.servers", "myServer1:9092,myServer2:9092")); + assertThat(displayData, hasDisplayItem("auto.offset.reset", "latest")); + assertThat(displayData, hasDisplayItem("receive.buffer.bytes", 524288)); + } + @Test public void testSinkDisplayData() { try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) { diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOUtilsTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOUtilsTest.java index 296538301a4c2..925b8580908b9 100644 --- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOUtilsTest.java +++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOUtilsTest.java @@ -21,7 +21,7 @@ import static org.junit.Assert.assertTrue; import java.util.Map; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java index 8fdbd12212df5..6b9dde4dc9528 100644 --- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java +++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadSchemaTransformProviderTest.java @@ -28,9 +28,9 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -116,14 +116,12 @@ public void testBuildTransformWithAvroSchema() { .collect(Collectors.toList()); KafkaReadSchemaTransformProvider kafkaProvider = (KafkaReadSchemaTransformProvider) providers.get(0); - kafkaProvider - .from( - KafkaReadSchemaTransformConfiguration.builder() - .setTopic("anytopic") - .setBootstrapServers("anybootstrap") - .setSchema(AVRO_SCHEMA) - .build()) - .buildTransform(); + kafkaProvider.from( + KafkaReadSchemaTransformConfiguration.builder() + .setTopic("anytopic") + .setBootstrapServers("anybootstrap") + .setSchema(AVRO_SCHEMA) + .build()); } @Test @@ -136,20 +134,17 @@ public void testBuildTransformWithJsonSchema() throws IOException { .collect(Collectors.toList()); KafkaReadSchemaTransformProvider kafkaProvider = (KafkaReadSchemaTransformProvider) providers.get(0); - kafkaProvider - .from( - KafkaReadSchemaTransformConfiguration.builder() - .setTopic("anytopic") - .setBootstrapServers("anybootstrap") - .setFormat("JSON") - .setSchema( - new String( - ByteStreams.toByteArray( - Objects.requireNonNull( - getClass() - .getResourceAsStream("/json-schema/basic_json_schema.json"))), - StandardCharsets.UTF_8)) - .build()) - .buildTransform(); + kafkaProvider.from( + KafkaReadSchemaTransformConfiguration.builder() + .setTopic("anytopic") + .setBootstrapServers("anybootstrap") + .setFormat("JSON") + .setSchema( + new String( + ByteStreams.toByteArray( + Objects.requireNonNull( + getClass().getResourceAsStream("/json-schema/basic_json_schema.json"))), + StandardCharsets.UTF_8)) + .build()); } } diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java index 3f2d49e75fc8c..854fd5ecea691 100644 --- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java +++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java @@ -46,10 +46,10 @@ import org.apache.beam.sdk.values.PCollection.IsBounded; import org.apache.beam.sdk.values.PValue; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/WatchForKafkaTopicPartitionsTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/WatchForKafkaTopicPartitionsTest.java index 78ec57728199c..595d040bf4032 100644 --- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/WatchForKafkaTopicPartitionsTest.java +++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/WatchForKafkaTopicPartitionsTest.java @@ -23,6 +23,7 @@ import static org.mockito.Mockito.when; import java.util.Set; +import java.util.regex.Pattern; import org.apache.beam.sdk.io.kafka.KafkaMocks.PartitionGrowthMockConsumer; import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.SerializableMatcher; @@ -30,9 +31,9 @@ import org.apache.beam.sdk.testing.TestPipelineOptions; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; @@ -77,7 +78,8 @@ public void testGetAllTopicPartitions() throws Exception { new TopicPartition("topic1", 1), new TopicPartition("topic2", 0), new TopicPartition("topic2", 1)), - WatchForKafkaTopicPartitions.getAllTopicPartitions((input) -> mockConsumer, null, null)); + WatchForKafkaTopicPartitions.getAllTopicPartitions( + (input) -> mockConsumer, null, null, null)); } @Test @@ -103,7 +105,47 @@ public void testGetAllTopicPartitionsWithGivenTopics() throws Exception { new TopicPartition("topic2", 0), new TopicPartition("topic2", 1)), WatchForKafkaTopicPartitions.getAllTopicPartitions( - (input) -> mockConsumer, null, givenTopics)); + (input) -> mockConsumer, null, givenTopics, null)); + } + + @Test + public void testGetAllTopicPartitionsWithGivenPattern() throws Exception { + Consumer mockConsumer = Mockito.mock(Consumer.class); + when(mockConsumer.listTopics()) + .thenReturn( + ImmutableMap.of( + "topic1", + ImmutableList.of( + new PartitionInfo("topic1", 0, null, null, null), + new PartitionInfo("topic1", 1, null, null, null)), + "topic2", + ImmutableList.of( + new PartitionInfo("topic2", 0, null, null, null), + new PartitionInfo("topic2", 1, null, null, null)), + "topicA", + ImmutableList.of( + new PartitionInfo("topicA", 0, null, null, null), + new PartitionInfo("topicA", 1, null, null, null)), + "topicB", + ImmutableList.of( + new PartitionInfo("topicB", 0, null, null, null), + new PartitionInfo("topicB", 1, null, null, null)))); + assertEquals( + ImmutableList.of( + new TopicPartition("topic1", 0), + new TopicPartition("topic1", 1), + new TopicPartition("topic2", 0), + new TopicPartition("topic2", 1)), + WatchForKafkaTopicPartitions.getAllTopicPartitions( + (input) -> mockConsumer, null, null, Pattern.compile("topic[0-9]"))); + assertEquals( + ImmutableList.of( + new TopicPartition("topicA", 0), + new TopicPartition("topicA", 1), + new TopicPartition("topicB", 0), + new TopicPartition("topicB", 1)), + WatchForKafkaTopicPartitions.getAllTopicPartitions( + (input) -> mockConsumer, null, null, Pattern.compile("topic[A-Z]"))); } @Test @@ -120,6 +162,7 @@ public void testPartitionSingle() { null, givenTopics, null, + null, null); PCollection descriptors = p.apply(watchForKafkaTopicPartitions); @@ -145,6 +188,7 @@ public void testPartitionGrowth() { null, givenTopics, null, + null, null); PCollection descriptors = p.apply(watchForKafkaTopicPartitions); diff --git a/sdks/java/io/kinesis/OWNERS b/sdks/java/io/kinesis/OWNERS deleted file mode 100644 index 377d5939afc6c..0000000000000 --- a/sdks/java/io/kinesis/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - aromanenko-dev - - mosche diff --git a/sdks/java/io/kinesis/build.gradle b/sdks/java/io/kinesis/build.gradle index f8a9b5451b7ef..98ab21164aaa6 100644 --- a/sdks/java/io/kinesis/build.gradle +++ b/sdks/java/io/kinesis/build.gradle @@ -40,7 +40,7 @@ dependencies { implementation "com.amazonaws:amazon-kinesis-client:1.14.2" implementation "com.amazonaws:amazon-kinesis-producer:0.14.1" implementation "commons-lang:commons-lang:2.6" - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.jackson_core implementation library.java.jackson_annotations implementation library.java.jackson_databind diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/BasicKinesisProvider.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/BasicKinesisProvider.java index eb4612633e436..ada59996609e8 100644 --- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/BasicKinesisProvider.java +++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/BasicKinesisProvider.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.client.builder.AwsClientBuilder; diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisIO.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisIO.java index ab73e2de5168e..3b64a6e719477 100644 --- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisIO.java +++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisIO.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider; @@ -55,7 +55,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisReader.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisReader.java index bffe8b86132a6..a4a935eed7b99 100644 --- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisReader.java +++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisReader.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.util.NoSuchElementException; diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisReaderCheckpoint.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisReaderCheckpoint.java index 7faa2516fe976..4b4bcc3898c72 100644 --- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisReaderCheckpoint.java +++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisReaderCheckpoint.java @@ -17,15 +17,15 @@ */ package org.apache.beam.sdk.io.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.partition; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.partition; import java.io.IOException; import java.io.Serializable; import java.util.Iterator; import java.util.List; import org.apache.beam.sdk.io.UnboundedSource; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** * Checkpoint representing a total progress in a set of shards in single stream. The set of shards diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisSource.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisSource.java index c30a14f5ebf4c..e53d71ed0b813 100644 --- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisSource.java +++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisSource.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; import java.util.List; import org.apache.beam.sdk.coders.Coder; diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisTransformRegistrar.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisTransformRegistrar.java index 67359f36914db..b8e1a38c73ffc 100644 --- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisTransformRegistrar.java +++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisTransformRegistrar.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/RateLimitPolicyFactory.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/RateLimitPolicyFactory.java index 1f0b739ec6088..12e013136abca 100644 --- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/RateLimitPolicyFactory.java +++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/RateLimitPolicyFactory.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.util.BackOffUtils; import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.joda.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/RecordFilter.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/RecordFilter.java index ce8218e0f3451..2a0456e04052b 100644 --- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/RecordFilter.java +++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/RecordFilter.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; import java.util.List; diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardCheckpoint.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardCheckpoint.java index 997e7bb81ac34..b185a396d1fd3 100644 --- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardCheckpoint.java +++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardCheckpoint.java @@ -20,8 +20,8 @@ import static com.amazonaws.services.kinesis.model.ShardIteratorType.AFTER_SEQUENCE_NUMBER; import static com.amazonaws.services.kinesis.model.ShardIteratorType.AT_SEQUENCE_NUMBER; import static com.amazonaws.services.kinesis.model.ShardIteratorType.AT_TIMESTAMP; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; import com.amazonaws.services.kinesis.model.Record; diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardReadersPool.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardReadersPool.java index b9a2b5b398dea..703d10d3640e0 100644 --- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardReadersPool.java +++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardReadersPool.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.kinesis; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.util.Collection; import java.util.Comparator; @@ -37,9 +37,9 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Instant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardRecordsIterator.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardRecordsIterator.java index c80fe57ecdb30..aae179373a2c0 100644 --- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardRecordsIterator.java +++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/ShardRecordsIterator.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream; import com.amazonaws.services.kinesis.model.ExpiredIteratorException; diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/SimplifiedKinesisClient.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/SimplifiedKinesisClient.java index 7e74a70a5c733..88fcc7fcec359 100644 --- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/SimplifiedKinesisClient.java +++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/SimplifiedKinesisClient.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; @@ -53,7 +53,7 @@ import org.apache.beam.sdk.util.BackOffUtils; import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.joda.time.Instant; import org.joda.time.Minutes; diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/StartingPoint.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/StartingPoint.java index 2b7b17b8eab10..6fde16d7f3b9b 100644 --- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/StartingPoint.java +++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/StartingPoint.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream; import java.io.Serializable; diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/StaticCheckpointGenerator.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/StaticCheckpointGenerator.java index 642e6e9e84ff0..9364f98eccea2 100644 --- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/StaticCheckpointGenerator.java +++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/StaticCheckpointGenerator.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; /** Always returns the same instance of checkpoint. */ class StaticCheckpointGenerator implements CheckpointGenerator { diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/WatermarkParameters.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/WatermarkParameters.java index 1e9ca2174a797..f604dc9dc11bc 100644 --- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/WatermarkParameters.java +++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/WatermarkParameters.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.io.Serializable; diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/WatermarkPolicyFactory.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/WatermarkPolicyFactory.java index f2d47b43dbd40..62de2fe16a5e4 100644 --- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/WatermarkPolicyFactory.java +++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/WatermarkPolicyFactory.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.kinesis; import java.io.Serializable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/serde/AwsModule.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/serde/AwsModule.java index ea23b64cf3548..d8396d5da924d 100644 --- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/serde/AwsModule.java +++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/serde/AwsModule.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kinesis.serde; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; @@ -47,7 +47,7 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import java.io.IOException; import java.util.Map; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.apache.commons.lang3.reflect.FieldUtils; /** diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/AmazonKinesisMock.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/AmazonKinesisMock.java index a0a3a2f0dc770..704a5ab07ba9b 100644 --- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/AmazonKinesisMock.java +++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/AmazonKinesisMock.java @@ -19,7 +19,7 @@ import static java.lang.Integer.parseInt; import static java.lang.Math.min; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.transform; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.transform; import static org.apache.commons.lang.builder.HashCodeBuilder.reflectionHashCode; import com.amazonaws.AmazonWebServiceRequest; @@ -98,7 +98,7 @@ import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; import org.apache.commons.lang.builder.EqualsBuilder; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/DynamicCheckpointGeneratorTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/DynamicCheckpointGeneratorTest.java index 2fb5209320fb7..1426f3b521973 100644 --- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/DynamicCheckpointGeneratorTest.java +++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/DynamicCheckpointGeneratorTest.java @@ -23,7 +23,7 @@ import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream; import com.amazonaws.services.kinesis.model.Shard; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisIOIT.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisIOIT.java index 6f4a8c9d0b88f..b2ec825f7d855 100644 --- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisIOIT.java +++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisIOIT.java @@ -41,7 +41,7 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.AfterClass; diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisMockReadTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisMockReadTest.java index 818f26b550224..77cabe858f520 100644 --- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisMockReadTest.java +++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisMockReadTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream; import java.util.List; @@ -27,7 +27,7 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.DateTime; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisMockWriteTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisMockWriteTest.java index 4e189a050b889..33b0c3a096ab9 100644 --- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisMockWriteTest.java +++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisMockWriteTest.java @@ -33,8 +33,8 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisReaderCheckpointTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisReaderCheckpointTest.java index 9ce4b7015ed1c..61212fb055700 100644 --- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisReaderCheckpointTest.java +++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisReaderCheckpointTest.java @@ -22,7 +22,7 @@ import java.util.Iterator; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisServiceMock.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisServiceMock.java index 0508b0570a7fc..dcbe4224b6303 100644 --- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisServiceMock.java +++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisServiceMock.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kinesis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/RateLimitPolicyFactoryTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/RateLimitPolicyFactoryTest.java index 32536e91622ce..0d144d19a9095 100644 --- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/RateLimitPolicyFactoryTest.java +++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/RateLimitPolicyFactoryTest.java @@ -36,7 +36,7 @@ import org.apache.beam.sdk.io.kinesis.RateLimitPolicyFactory.DefaultRateLimiter; import org.apache.beam.sdk.util.BackOff; import org.apache.beam.sdk.util.Sleeper; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; import org.powermock.api.mockito.PowerMockito; diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/RecordFilterTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/RecordFilterTest.java index 429024e654d78..ad1e58c265e77 100644 --- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/RecordFilterTest.java +++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/RecordFilterTest.java @@ -21,7 +21,7 @@ import java.util.Collections; import java.util.List; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.assertj.core.api.Assertions; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/ShardReadersPoolTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/ShardReadersPoolTest.java index 5950ae455aea0..74c9446d316a2 100644 --- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/ShardReadersPoolTest.java +++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/ShardReadersPoolTest.java @@ -32,8 +32,8 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Stopwatch; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Stopwatch; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.After; diff --git a/sdks/java/io/kudu/OWNERS b/sdks/java/io/kudu/OWNERS deleted file mode 100644 index 6049011f10e84..0000000000000 --- a/sdks/java/io/kudu/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - timrobertson100 diff --git a/sdks/java/io/kudu/build.gradle b/sdks/java/io/kudu/build.gradle index 002bf0647c950..e1835a679f36c 100644 --- a/sdks/java/io/kudu/build.gradle +++ b/sdks/java/io/kudu/build.gradle @@ -44,7 +44,7 @@ test { def kudu_version = "1.11.1" dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation "org.apache.kudu:kudu-client:$kudu_version" implementation library.java.slf4j_api diff --git a/sdks/java/io/kudu/src/main/java/org/apache/beam/sdk/io/kudu/KuduIO.java b/sdks/java/io/kudu/src/main/java/org/apache/beam/sdk/io/kudu/KuduIO.java index 92682a5d17b10..6d6296733db35 100644 --- a/sdks/java/io/kudu/src/main/java/org/apache/beam/sdk/io/kudu/KuduIO.java +++ b/sdks/java/io/kudu/src/main/java/org/apache/beam/sdk/io/kudu/KuduIO.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.kudu; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.util.Collections; @@ -40,8 +40,8 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; import org.apache.kudu.Common; import org.apache.kudu.client.KuduException; import org.apache.kudu.client.KuduPredicate; diff --git a/sdks/java/io/kudu/src/main/java/org/apache/beam/sdk/io/kudu/KuduServiceImpl.java b/sdks/java/io/kudu/src/main/java/org/apache/beam/sdk/io/kudu/KuduServiceImpl.java index 07981bd4025c7..dd954bec2b55f 100644 --- a/sdks/java/io/kudu/src/main/java/org/apache/beam/sdk/io/kudu/KuduServiceImpl.java +++ b/sdks/java/io/kudu/src/main/java/org/apache/beam/sdk/io/kudu/KuduServiceImpl.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.kudu; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.IOException; import java.util.List; diff --git a/sdks/java/io/kudu/src/test/java/org/apache/beam/sdk/io/kudu/KuduTestUtils.java b/sdks/java/io/kudu/src/test/java/org/apache/beam/sdk/io/kudu/KuduTestUtils.java index 7de6a86fea3ac..d52c1a2d9c5ce 100644 --- a/sdks/java/io/kudu/src/test/java/org/apache/beam/sdk/io/kudu/KuduTestUtils.java +++ b/sdks/java/io/kudu/src/test/java/org/apache/beam/sdk/io/kudu/KuduTestUtils.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.kudu; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.Type; diff --git a/sdks/java/io/mongodb/OWNERS b/sdks/java/io/mongodb/OWNERS deleted file mode 100644 index 8b0b05531cd10..0000000000000 --- a/sdks/java/io/mongodb/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - jbonofre diff --git a/sdks/java/io/mongodb/build.gradle b/sdks/java/io/mongodb/build.gradle index 7a6a46a0ff0c7..95722a69db296 100644 --- a/sdks/java/io/mongodb/build.gradle +++ b/sdks/java/io/mongodb/build.gradle @@ -29,7 +29,7 @@ dependencies { implementation library.java.joda_time implementation library.java.mongo_java_driver implementation library.java.slf4j_api - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre testImplementation library.java.junit testImplementation project(path: ":sdks:java:io:common", configuration: "testRuntimeMigration") testImplementation project(path: ":sdks:java:testing:test-utils", configuration: "testRuntimeMigration") diff --git a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/FindQuery.java b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/FindQuery.java index e4c3507b4eb55..2131656d458ad 100644 --- a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/FindQuery.java +++ b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/FindQuery.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.mongodb; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import com.mongodb.BasicDBObject; diff --git a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java index 1c118b49fe793..1e7dd6dcde0af 100644 --- a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java +++ b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.mongodb; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import com.mongodb.DB; diff --git a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbIO.java b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbIO.java index 642c353707549..53c39e2406fd6 100644 --- a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbIO.java +++ b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbIO.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.mongodb; import static org.apache.beam.sdk.io.mongodb.FindQuery.bson2BsonDocument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import com.mongodb.BasicDBObject; @@ -60,7 +60,7 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.bson.BsonDocument; import org.bson.BsonInt32; import org.bson.BsonString; diff --git a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/SSLUtils.java b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/SSLUtils.java index 4c44953f0b840..1c46972893117 100644 --- a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/SSLUtils.java +++ b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/SSLUtils.java @@ -25,9 +25,12 @@ import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; import org.apache.beam.sdk.util.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** Utility class for registration of ssl context, and to allow all certificate requests. */ class SSLUtils { + private static final Logger LOG = LoggerFactory.getLogger(SSLUtils.class); /** static class to allow all requests. */ private static final TrustManager[] trustAllCerts = @@ -62,11 +65,15 @@ static SSLContext ignoreSSLCertificate() { ClassLoader classLoader = Preconditions.checkStateNotNull( SSLUtils.class.getClassLoader(), "SSLUtil classloader is null - boot classloader?"); - InputStream inputStream = - Preconditions.checkStateNotNull( - classLoader.getResourceAsStream("resources/.keystore"), - "resources/.keystore not found"); - ks.load(inputStream, "changeit".toCharArray()); + InputStream inputStream = classLoader.getResourceAsStream("resources/.keystore"); + if (inputStream != null) { + LOG.info("Found keystore in classpath 'resources/.keystore'. Loading..."); + ks.load(inputStream, "changeit".toCharArray()); + } else { + LOG.info( + "Unable to find keystore under 'resources/.keystore' in the classpath. " + + "Continuing with an empty keystore."); + } KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); kmf.init(ks, "changeit".toCharArray()); diff --git a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBIOIT.java b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBIOIT.java index 1294d04b44b58..65d672b61b623 100644 --- a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBIOIT.java +++ b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBIOIT.java @@ -50,7 +50,7 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.SimpleFunction; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.bson.Document; import org.junit.After; import org.junit.AfterClass; diff --git a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbIOTest.java b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbIOTest.java index 64af39a704753..d726dad5b871f 100644 --- a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbIOTest.java +++ b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbIOTest.java @@ -46,7 +46,7 @@ import org.apache.beam.sdk.transforms.SimpleFunction; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterators; import org.bson.BsonDocument; import org.bson.BsonInt32; import org.bson.BsonString; diff --git a/sdks/java/io/mqtt/OWNERS b/sdks/java/io/mqtt/OWNERS deleted file mode 100644 index 8b0b05531cd10..0000000000000 --- a/sdks/java/io/mqtt/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - jbonofre diff --git a/sdks/java/io/mqtt/build.gradle b/sdks/java/io/mqtt/build.gradle index 7137f3584e3ad..e0514fb4312b7 100644 --- a/sdks/java/io/mqtt/build.gradle +++ b/sdks/java/io/mqtt/build.gradle @@ -23,7 +23,7 @@ description = "Apache Beam :: SDKs :: Java :: IO :: MQTT" ext.summary = "IO to read and write to a MQTT broker." dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.slf4j_api implementation library.java.joda_time diff --git a/sdks/java/io/mqtt/src/main/java/org/apache/beam/sdk/io/mqtt/MqttIO.java b/sdks/java/io/mqtt/src/main/java/org/apache/beam/sdk/io/mqtt/MqttIO.java index d0c0dafc6545c..8b7f0991c2dd0 100644 --- a/sdks/java/io/mqtt/src/main/java/org/apache/beam/sdk/io/mqtt/MqttIO.java +++ b/sdks/java/io/mqtt/src/main/java/org/apache/beam/sdk/io/mqtt/MqttIO.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.mqtt; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.io.IOException; @@ -42,7 +42,7 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; import org.fusesource.mqtt.client.BlockingConnection; import org.fusesource.mqtt.client.MQTT; diff --git a/sdks/java/io/neo4j/OWNERS b/sdks/java/io/neo4j/OWNERS deleted file mode 100644 index 0ff6e82359fb6..0000000000000 --- a/sdks/java/io/neo4j/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - mcasters - diff --git a/sdks/java/io/neo4j/build.gradle b/sdks/java/io/neo4j/build.gradle index 9d5adfc32b1d0..e4bcfa157c6b3 100644 --- a/sdks/java/io/neo4j/build.gradle +++ b/sdks/java/io/neo4j/build.gradle @@ -28,7 +28,7 @@ dependencies { implementation project(path: ":sdks:java:core", configuration: "shadow") implementation "org.neo4j.driver:neo4j-java-driver:4.4.3" implementation library.java.slf4j_api - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre testImplementation library.java.junit testImplementation library.java.hamcrest testImplementation project(path: ":sdks:java:io:common", configuration: "testRuntimeMigration") diff --git a/sdks/java/io/neo4j/src/main/java/org/apache/beam/sdk/io/neo4j/Neo4jIO.java b/sdks/java/io/neo4j/src/main/java/org/apache/beam/sdk/io/neo4j/Neo4jIO.java index e83d5c585ef9f..268c6b088d896 100644 --- a/sdks/java/io/neo4j/src/main/java/org/apache/beam/sdk/io/neo4j/Neo4jIO.java +++ b/sdks/java/io/neo4j/src/main/java/org/apache/beam/sdk/io/neo4j/Neo4jIO.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.neo4j; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.io.Serializable; @@ -45,7 +45,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; @@ -814,23 +814,17 @@ private void executeReadCypherStatement( // We could actually read and write here depending on the type of transaction // we picked. As long as the Cypher statement returns values it's fine. // - TransactionWork transactionWork = + TransactionWork> transactionWork = transaction -> { Result result = transaction.run(cypher, parametersMap); - while (result.hasNext()) { - Record record = result.next(); - try { - OutputT outputT = rowMapper.mapRow(record); - processContext.output(outputT); - } catch (Exception e) { - throw new RuntimeException("error mapping Neo4j record to row", e); - } - } - - // We deliver no specific Neo4j transaction output beyond what goes to the context - // output - // - return null; + return result.list( + record -> { + try { + return rowMapper.mapRow(record); + } catch (Exception e) { + throw new RuntimeException("error mapping Neo4j record to row", e); + } + }); }; if (logCypher) { @@ -852,11 +846,13 @@ private void executeReadCypherStatement( if (driverSession.session == null) { throw new RuntimeException("neo4j session was not initialized correctly"); } else { + List outputs; if (writeTransaction) { - driverSession.session.writeTransaction(transactionWork, transactionConfig); + outputs = driverSession.session.writeTransaction(transactionWork, transactionConfig); } else { - driverSession.session.readTransaction(transactionWork, transactionConfig); + outputs = driverSession.session.readTransaction(transactionWork, transactionConfig); } + outputs.forEach(processContext::output); } } } @@ -1168,13 +1164,7 @@ private void executeCypherUnwindStatement() { // TransactionWork transactionWork = transaction -> { - Result result = transaction.run(cypher, parametersMap); - while (result.hasNext()) { - // This just consumes any output but the function basically has no output - // To be revisited based on requirements. - // - result.next(); - } + transaction.run(cypher, parametersMap).consume(); return null; }; diff --git a/sdks/java/io/neo4j/src/test/java/org/apache/beam/sdk/io/neo4j/Neo4jIOIT.java b/sdks/java/io/neo4j/src/test/java/org/apache/beam/sdk/io/neo4j/Neo4jIOIT.java index e5f606642a5fd..537c691053c13 100644 --- a/sdks/java/io/neo4j/src/test/java/org/apache/beam/sdk/io/neo4j/Neo4jIOIT.java +++ b/sdks/java/io/neo4j/src/test/java/org/apache/beam/sdk/io/neo4j/Neo4jIOIT.java @@ -37,7 +37,7 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; diff --git a/sdks/java/io/neo4j/src/test/java/org/apache/beam/sdk/io/neo4j/Neo4jTestUtil.java b/sdks/java/io/neo4j/src/test/java/org/apache/beam/sdk/io/neo4j/Neo4jTestUtil.java index adfd0664a2e9c..cee57d5d43862 100644 --- a/sdks/java/io/neo4j/src/test/java/org/apache/beam/sdk/io/neo4j/Neo4jTestUtil.java +++ b/sdks/java/io/neo4j/src/test/java/org/apache/beam/sdk/io/neo4j/Neo4jTestUtil.java @@ -32,7 +32,8 @@ public class Neo4jTestUtil { public static final String NEO4J_VERSION = "latest"; public static final String NEO4J_NETWORK_ALIAS = "neo4jcontainer"; public static final String NEO4J_USERNAME = "neo4j"; - public static final String NEO4J_PASSWORD = "abcd"; + public static final String NEO4J_PASSWORD = + "abcdefgh"; // The minimum password length is 8 characters public static final String NEO4J_DATABASE = "neo4j"; public static final String getUrl(String hostname, int port) { diff --git a/sdks/java/io/parquet/OWNERS b/sdks/java/io/parquet/OWNERS deleted file mode 100644 index 189bb00c38373..0000000000000 --- a/sdks/java/io/parquet/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lgajowy - - jbonofre - - aromanenko-dev diff --git a/sdks/java/io/parquet/build.gradle b/sdks/java/io/parquet/build.gradle index 84d10d2be175f..e7e06e9cca3cd 100644 --- a/sdks/java/io/parquet/build.gradle +++ b/sdks/java/io/parquet/build.gradle @@ -38,7 +38,7 @@ hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")} def parquet_version = "1.12.0" dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(":sdks:java:extensions:avro") implementation project(":sdks:java:io:hadoop-common") diff --git a/sdks/java/io/parquet/src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java b/sdks/java/io/parquet/src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java index 34f590e9490fc..33940aa27a5c8 100644 --- a/sdks/java/io/parquet/src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java +++ b/sdks/java/io/parquet/src/main/java/org/apache/beam/sdk/io/parquet/ParquetIO.java @@ -60,9 +60,9 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.apache.hadoop.conf.Configuration; import org.apache.parquet.HadoopReadOptions; import org.apache.parquet.ParquetReadOptions; diff --git a/sdks/java/io/pulsar/build.gradle b/sdks/java/io/pulsar/build.gradle index 73d34f71b8c32..7ffe3f22cca4a 100644 --- a/sdks/java/io/pulsar/build.gradle +++ b/sdks/java/io/pulsar/build.gradle @@ -26,7 +26,7 @@ def pulsar_version = '2.8.2' dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.slf4j_api implementation library.java.joda_time diff --git a/sdks/java/io/pulsar/src/main/java/org/apache/beam/sdk/io/pulsar/ReadFromPulsarDoFn.java b/sdks/java/io/pulsar/src/main/java/org/apache/beam/sdk/io/pulsar/ReadFromPulsarDoFn.java index fc881f33e67e1..97a8dcd6e2b45 100644 --- a/sdks/java/io/pulsar/src/main/java/org/apache/beam/sdk/io/pulsar/ReadFromPulsarDoFn.java +++ b/sdks/java/io/pulsar/src/main/java/org/apache/beam/sdk/io/pulsar/ReadFromPulsarDoFn.java @@ -29,8 +29,8 @@ import org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimator; import org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimators; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Suppliers; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Suppliers; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Message; diff --git a/sdks/java/io/rabbitmq/OWNERS b/sdks/java/io/rabbitmq/OWNERS deleted file mode 100644 index 8b0b05531cd10..0000000000000 --- a/sdks/java/io/rabbitmq/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - jbonofre diff --git a/sdks/java/io/rabbitmq/build.gradle b/sdks/java/io/rabbitmq/build.gradle index 60684caed2312..beb08f36d60da 100644 --- a/sdks/java/io/rabbitmq/build.gradle +++ b/sdks/java/io/rabbitmq/build.gradle @@ -25,7 +25,7 @@ ext.summary = "IO to read and write to a RabbitMQ broker." def qpid_version = "8.0.1" dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.joda_time implementation "com.rabbitmq:amqp-client:5.7.3" diff --git a/sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java b/sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java index cfac80cf59694..ceb82cc320a1e 100644 --- a/sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java +++ b/sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.rabbitmq; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import com.rabbitmq.client.Channel; diff --git a/sdks/java/io/rabbitmq/src/test/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIOTest.java b/sdks/java/io/rabbitmq/src/test/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIOTest.java index 2ceae333b8c16..9554c3309501b 100644 --- a/sdks/java/io/rabbitmq/src/test/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIOTest.java +++ b/sdks/java/io/rabbitmq/src/test/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIOTest.java @@ -51,7 +51,7 @@ import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; import org.apache.qpid.server.SystemLauncher; import org.apache.qpid.server.model.SystemConfig; import org.junit.AfterClass; diff --git a/sdks/java/io/redis/OWNERS b/sdks/java/io/redis/OWNERS deleted file mode 100644 index 8b0b05531cd10..0000000000000 --- a/sdks/java/io/redis/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - jbonofre diff --git a/sdks/java/io/redis/build.gradle b/sdks/java/io/redis/build.gradle index 815e6f9dea135..6957eceec9412 100644 --- a/sdks/java/io/redis/build.gradle +++ b/sdks/java/io/redis/build.gradle @@ -23,7 +23,7 @@ description = "Apache Beam :: SDKs :: Java :: IO :: Redis" ext.summary ="IO to read and write on a Redis keystore." dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation "redis.clients:jedis:4.0.1" testImplementation project(path: ":sdks:java:io:common", configuration: "testRuntimeMigration") diff --git a/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisConnectionConfiguration.java b/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisConnectionConfiguration.java index 257832b45af49..07a16cc7f2efd 100644 --- a/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisConnectionConfiguration.java +++ b/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisConnectionConfiguration.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.redis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.io.Serializable; diff --git a/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisCursor.java b/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisCursor.java index be93cae3b98d5..8b98b673f63aa 100644 --- a/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisCursor.java +++ b/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisCursor.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.redis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -27,7 +27,7 @@ import javax.annotation.Nonnull; import org.apache.beam.sdk.coders.BigEndianLongCoder; import org.apache.beam.sdk.io.range.ByteKey; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; public class RedisCursor implements Comparable, Serializable { diff --git a/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisIO.java b/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisIO.java index 46dd5566e77f6..62109a6a6660d 100644 --- a/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisIO.java +++ b/sdks/java/io/redis/src/main/java/org/apache/beam/sdk/io/redis/RedisIO.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.redis; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.util.List; diff --git a/sdks/java/io/redis/src/test/java/org/apache/beam/sdk/io/redis/RedisIOTest.java b/sdks/java/io/redis/src/test/java/org/apache/beam/sdk/io/redis/RedisIOTest.java index 10be1741cab57..5754de89ec3a4 100644 --- a/sdks/java/io/redis/src/test/java/org/apache/beam/sdk/io/redis/RedisIOTest.java +++ b/sdks/java/io/redis/src/test/java/org/apache/beam/sdk/io/redis/RedisIOTest.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.redis; import static java.util.stream.Collectors.toList; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.transform; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.transform; import static org.hamcrest.CoreMatchers.hasItems; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; @@ -45,7 +45,7 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Rule; diff --git a/sdks/java/io/rrio/build.gradle b/sdks/java/io/rrio/build.gradle new file mode 100644 index 0000000000000..d7d5c8817d065 --- /dev/null +++ b/sdks/java/io/rrio/build.gradle @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.sdk.io.requestresponse' +) + +description = "Apache Beam :: SDKS :: Java :: IO :: RequestResponseIO (RRIO)" +ext.summary = "Support to read from and write to Web APIs" + +dependencies { + implementation project(path: ":sdks:java:core", configuration: "shadow") + implementation library.java.joda_time + implementation library.java.vendored_guava_32_1_2_jre + + testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") + testImplementation library.java.junit + testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") + testRuntimeOnly library.java.slf4j_jdk14 +} \ No newline at end of file diff --git a/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/ApiIOError.java b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/ApiIOError.java new file mode 100644 index 0000000000000..5936c5dd84b02 --- /dev/null +++ b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/ApiIOError.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.io.requestresponse; + +import com.google.auto.value.AutoValue; +import org.apache.beam.sdk.schemas.AutoValueSchema; +import org.apache.beam.sdk.schemas.annotations.DefaultSchema; +import org.apache.beam.sdk.schemas.annotations.SchemaCaseFormat; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.CaseFormat; +import org.joda.time.Instant; + +/** {@link ApiIOError} is a data class for storing details about an error. */ +@SchemaCaseFormat(CaseFormat.LOWER_UNDERSCORE) +@DefaultSchema(AutoValueSchema.class) +@AutoValue +public abstract class ApiIOError { + + static Builder builder() { + return new AutoValue_ApiIOError.Builder(); + } + + /** The encoded UTF-8 string representation of the related processed element. */ + public abstract String getEncodedElementAsUtfString(); + + /** The observed timestamp of the error. */ + public abstract Instant getObservedTimestamp(); + + /** The {@link Exception} message. */ + public abstract String getMessage(); + + /** The {@link Exception} stack trace. */ + public abstract String getStackTrace(); + + @AutoValue.Builder + abstract static class Builder { + + public abstract Builder setEncodedElementAsUtfString(String value); + + public abstract Builder setObservedTimestamp(Instant value); + + public abstract Builder setMessage(String value); + + public abstract Builder setStackTrace(String value); + + abstract ApiIOError build(); + } +} diff --git a/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/CacheRead.java b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/CacheRead.java new file mode 100644 index 0000000000000..3765d25370a66 --- /dev/null +++ b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/CacheRead.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.io.requestresponse; + +import com.google.auto.value.AutoValue; +import java.util.Map; +import org.apache.beam.io.requestresponse.CacheRead.Result; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.PInput; +import org.apache.beam.sdk.values.POutput; +import org.apache.beam.sdk.values.PValue; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; + +/** + * {@link CacheRead} reads associated {@link ResponseT} types from {@link RequestT} types, if any + * exist. + */ +class CacheRead + extends PTransform, Result> { + + private static final TupleTag FAILURE_TAG = new TupleTag() {}; + + // TODO(damondouglas): remove suppress warnings after instance utilized. + @SuppressWarnings({"unused"}) + private final Configuration configuration; + + private CacheRead(Configuration configuration) { + this.configuration = configuration; + } + + /** Configuration details for {@link CacheRead}. */ + @AutoValue + abstract static class Configuration { + + static Builder builder() { + return new AutoValue_CacheRead_Configuration.Builder<>(); + } + + abstract Builder toBuilder(); + + @AutoValue.Builder + abstract static class Builder { + + abstract Configuration build(); + } + } + + @Override + public Result expand(PCollection input) { + return Result.of( + new TupleTag>() {}, PCollectionTuple.empty(input.getPipeline())); + } + + /** + * The {@link Result} of reading RequestT {@link PCollection} elements yielding ResponseT {@link + * PCollection} elements. + */ + static class Result implements POutput { + + static Result of( + TupleTag> responseTag, PCollectionTuple pct) { + return new Result<>(responseTag, pct); + } + + private final Pipeline pipeline; + private final TupleTag> responseTag; + private final PCollection> responses; + private final PCollection failures; + + private Result(TupleTag> responseTag, PCollectionTuple pct) { + this.pipeline = pct.getPipeline(); + this.responseTag = responseTag; + this.responses = pct.get(responseTag); + this.failures = pct.get(FAILURE_TAG); + } + + PCollection> getResponses() { + return responses; + } + + PCollection getFailures() { + return failures; + } + + @Override + public Pipeline getPipeline() { + return this.pipeline; + } + + @Override + public Map, PValue> expand() { + return ImmutableMap.of( + responseTag, responses, + FAILURE_TAG, failures); + } + + @Override + public void finishSpecifyingOutput( + String transformName, PInput input, PTransform transform) {} + } +} diff --git a/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/CacheWrite.java b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/CacheWrite.java new file mode 100644 index 0000000000000..25249c3e41b42 --- /dev/null +++ b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/CacheWrite.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.io.requestresponse; + +import com.google.auto.value.AutoValue; +import java.util.Map; +import org.apache.beam.io.requestresponse.CacheWrite.Result; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.PInput; +import org.apache.beam.sdk.values.POutput; +import org.apache.beam.sdk.values.PValue; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; + +/** + * {@link CacheWrite} writes associated {@link RequestT} and {@link ResponseT} pairs to a cache. + * Using {@link RequestT} and {@link ResponseT}'s {@link org.apache.beam.sdk.coders.Coder}, this + * transform writes encoded representations of this association. + */ +class CacheWrite + extends PTransform>, Result> { + + private static final TupleTag FAILURE_TAG = new TupleTag() {}; + + // TODO(damondouglas): remove suppress warnings after configuration is used. + @SuppressWarnings({"unused"}) + private final Configuration configuration; + + private CacheWrite(Configuration configuration) { + this.configuration = configuration; + } + + /** Configuration details for {@link CacheWrite}. */ + @AutoValue + abstract static class Configuration { + + static Builder builder() { + return new AutoValue_CacheWrite_Configuration.Builder<>(); + } + + abstract Builder toBuilder(); + + @AutoValue.Builder + abstract static class Builder { + + abstract Configuration build(); + } + } + + @Override + public Result expand(PCollection> input) { + return Result.of( + new TupleTag>() {}, PCollectionTuple.empty(input.getPipeline())); + } + + /** The {@link Result} of writing a request/response {@link KV} {@link PCollection}. */ + static class Result implements POutput { + + static Result of( + TupleTag> responseTag, PCollectionTuple pct) { + return new Result<>(responseTag, pct); + } + + private final Pipeline pipeline; + private final TupleTag> responseTag; + private final PCollection> responses; + private final PCollection failures; + + private Result(TupleTag> responseTag, PCollectionTuple pct) { + this.pipeline = pct.getPipeline(); + this.responseTag = responseTag; + this.responses = pct.get(responseTag); + this.failures = pct.get(FAILURE_TAG); + } + + public PCollection> getResponses() { + return responses; + } + + public PCollection getFailures() { + return failures; + } + + @Override + public Pipeline getPipeline() { + return this.pipeline; + } + + @Override + public Map, PValue> expand() { + return ImmutableMap.of( + responseTag, responses, + FAILURE_TAG, failures); + } + + @Override + public void finishSpecifyingOutput( + String transformName, PInput input, PTransform transform) {} + } +} diff --git a/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/Call.java b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/Call.java new file mode 100644 index 0000000000000..4f854ea69c7ec --- /dev/null +++ b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/Call.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.io.requestresponse; + +import com.google.auto.value.AutoValue; +import java.util.Map; +import org.apache.beam.io.requestresponse.Call.Result; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.PInput; +import org.apache.beam.sdk.values.POutput; +import org.apache.beam.sdk.values.PValue; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; + +/** + * {@link Call} transforms a {@link RequestT} {@link PCollection} into a {@link ResponseT} {@link + * PCollection} and {@link ApiIOError} {@link PCollection}, both wrapped in a {@link Result}. + */ +class Call extends PTransform, Result> { + + private static final TupleTag FAILURE_TAG = new TupleTag() {}; + + // TODO(damondouglas): remove suppress warnings when configuration utilized in future PR. + @SuppressWarnings({"unused"}) + private final Configuration configuration; + + private Call(Configuration configuration) { + this.configuration = configuration; + } + + /** Configuration details for {@link Call}. */ + @AutoValue + abstract static class Configuration { + + static Builder builder() { + return new AutoValue_Call_Configuration.Builder<>(); + } + + abstract Builder toBuilder(); + + @AutoValue.Builder + abstract static class Builder { + + abstract Configuration build(); + } + } + + @Override + public Result expand(PCollection input) { + return Result.of(new TupleTag() {}, PCollectionTuple.empty(input.getPipeline())); + } + + /** + * The {@link Result} of processing request {@link PCollection} into response {@link PCollection}. + */ + static class Result implements POutput { + + static Result of(TupleTag responseTag, PCollectionTuple pct) { + return new Result<>(responseTag, pct); + } + + private final Pipeline pipeline; + private final TupleTag responseTag; + private final PCollection responses; + private final PCollection failures; + + private Result(TupleTag responseTag, PCollectionTuple pct) { + this.pipeline = pct.getPipeline(); + this.responseTag = responseTag; + this.responses = pct.get(responseTag); + this.failures = pct.get(FAILURE_TAG); + } + + public PCollection getResponses() { + return responses; + } + + public PCollection getFailures() { + return failures; + } + + @Override + public Pipeline getPipeline() { + return this.pipeline; + } + + @Override + public Map, PValue> expand() { + return ImmutableMap.of( + responseTag, responses, + FAILURE_TAG, failures); + } + + @Override + public void finishSpecifyingOutput( + String transformName, PInput input, PTransform transform) {} + } +} diff --git a/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/CallShouldBackoff.java b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/CallShouldBackoff.java new file mode 100644 index 0000000000000..1d093f2efb126 --- /dev/null +++ b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/CallShouldBackoff.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.io.requestresponse; + +import java.io.Serializable; + +/** Informs whether a call to an API should backoff. */ +public interface CallShouldBackoff extends Serializable { + + /** Update the state of whether to backoff using information about the exception. */ + void update(UserCodeExecutionException exception); + + /** Update the state of whether to backoff using information about the response. */ + void update(ResponseT response); + + /** Report whether to backoff. */ + boolean value(); +} diff --git a/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/CallShouldBackoffBasedOnRejectionProbability.java b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/CallShouldBackoffBasedOnRejectionProbability.java new file mode 100644 index 0000000000000..62a7990d21eec --- /dev/null +++ b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/CallShouldBackoffBasedOnRejectionProbability.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.io.requestresponse; + +import org.checkerframework.checker.nullness.qual.Nullable; + +/** Reports whether to apply backoff based on https://sre.google/sre-book/handling-overload/. */ +class CallShouldBackoffBasedOnRejectionProbability + implements CallShouldBackoff { + + // Default multiplier value recommended by https://sre.google/sre-book/handling-overload/ + private static final double DEFAULT_MULTIPLIER = 2.0; + + // The threshold is the value that the rejection probability must exceed in order to report a + // value() of true. If null, then the computation relies on a random value. + private @Nullable Double threshold; + + // The multiplier drives the impact of accepts on the rejection probability. See setThreshold(double threshold) { + this.threshold = threshold; + return this; + } + + /** Update the state of whether to backoff using information about the exception. */ + @Override + public void update(UserCodeExecutionException exception) { + this.requests++; + } + + /** Update the state of whether to backoff using information about the response. */ + @Override + public void update(ResponseT response) { + this.requests++; + this.accepts++; + } + + /** Provide a threshold to evaluate backoff. */ + double getThreshold() { + if (this.threshold != null) { + return this.threshold; + } + return Math.random(); + } + + /** + * Compute the probability of API call rejection based on + * https://sre.google/sre-book/handling-overload/. + */ + double getRejectionProbability() { + double numerator = requests - multiplier * accepts; + double denominator = requests + 1; + double ratio = numerator / denominator; + return Math.max(0, ratio); + } + + /** Report whether to backoff. */ + @Override + public boolean value() { + return getRejectionProbability() > getThreshold(); + } +} diff --git a/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/Caller.java b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/Caller.java new file mode 100644 index 0000000000000..da636c8637403 --- /dev/null +++ b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/Caller.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.io.requestresponse; + +import java.io.Serializable; + +/** {@link Caller} interfaces user custom code intended for API calls. */ +public interface Caller extends Serializable { + + /** Calls a Web API with the {@link RequestT} and returns a {@link ResponseT}. */ + ResponseT call(RequestT request) throws UserCodeExecutionException; +} diff --git a/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/RequestResponseIO.java b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/RequestResponseIO.java new file mode 100644 index 0000000000000..de7d26aab4bd3 --- /dev/null +++ b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/RequestResponseIO.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.io.requestresponse; + +import com.google.auto.value.AutoValue; +import java.util.Map; +import org.apache.beam.io.requestresponse.RequestResponseIO.Result; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.PInput; +import org.apache.beam.sdk.values.POutput; +import org.apache.beam.sdk.values.PValue; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; + +/** + * {@link PTransform} for reading from and writing to Web APIs. + * + *

    {@link RequestResponseIO} is recommended for interacting with external systems that offer RPCs + * that execute relatively quickly and do not offer advance features to make RPC execution + * efficient. + * + *

    For systems that offer features for more efficient reading, for example, tracking progress of + * RPCs, support for splitting RPCs (deduct two or more RPCs which when combined return the same + * result), consider using the Apache Beam's `Splittable DoFn` interface instead. + * + *

    Basic Usage

    + * + * {@link RequestResponseIO} minimally requires implementing the {@link Caller} interface: + * + *
    {@code class MyCaller implements Caller {
    + *    public SomeResponse call(SomeRequest request) throws UserCodeExecutionException {
    + *      // calls the API submitting SomeRequest payload and returning SomeResponse
    + *    }
    + * }}
    + * + *

    Then provide {@link RequestResponseIO}'s {@link #create} method your {@link Caller} + * implementation. + * + *

    {@code  PCollection requests = ...
    + *  Result result = requests.apply(RequestResponseIO.create(new MyCaller()));
    + *  result.getResponses().apply( ... );
    + *  result.getFailures().apply( ... );
    + * }
    + */ +public class RequestResponseIO + extends PTransform, Result> { + + private static final TupleTag FAILURE_TAG = new TupleTag() {}; + + // TODO(damondouglas): remove when utilized. + @SuppressWarnings({"unused"}) + private final Configuration configuration; + + private RequestResponseIO(Configuration configuration) { + this.configuration = configuration; + } + + public static RequestResponseIO of( + Caller caller) { + return new RequestResponseIO<>( + Configuration.builder().setCaller(caller).build()); + } + + /** Configuration details for {@link RequestResponseIO}. */ + @AutoValue + abstract static class Configuration { + + static Builder builder() { + return new AutoValue_RequestResponseIO_Configuration.Builder<>(); + } + + /** + * The {@link Caller} that interfaces user custom code to process a {@link RequestT} into a + * {@link ResponseT}. + */ + abstract Caller getCaller(); + + abstract Builder toBuilder(); + + @AutoValue.Builder + abstract static class Builder { + + abstract Builder setCaller(Caller value); + + abstract Configuration build(); + } + } + + @Override + public Result expand(PCollection input) { + // TODO(damondouglas; https://github.com/apache/beam/issues?q=is%3Aissue+is%3Aopen+%5BRRIO%5D): + // expand pipeline as more dependencies develop. + return Result.of(new TupleTag() {}, PCollectionTuple.empty(input.getPipeline())); + } + + /** + * The {@link Result} of processing request {@link PCollection} into response {@link PCollection} + * using custom {@link Caller} code. + */ + public static class Result implements POutput { + + static Result of(TupleTag responseTag, PCollectionTuple pct) { + return new Result<>(responseTag, pct); + } + + private final Pipeline pipeline; + private final TupleTag responseTag; + private final PCollection responses; + private final PCollection failures; + + private Result(TupleTag responseTag, PCollectionTuple pct) { + this.pipeline = pct.getPipeline(); + this.responseTag = responseTag; + this.responses = pct.get(responseTag); + this.failures = pct.get(FAILURE_TAG); + } + + public PCollection getResponses() { + return responses; + } + + public PCollection getFailures() { + return failures; + } + + @Override + public Pipeline getPipeline() { + return this.pipeline; + } + + @Override + public Map, PValue> expand() { + return ImmutableMap.of( + responseTag, responses, + FAILURE_TAG, failures); + } + + @Override + public void finishSpecifyingOutput( + String transformName, PInput input, PTransform transform) {} + } +} diff --git a/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/SetupTeardown.java b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/SetupTeardown.java new file mode 100644 index 0000000000000..be1b03105c3dc --- /dev/null +++ b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/SetupTeardown.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.io.requestresponse; + +import java.io.Serializable; + +/** + * Provided by user and called within {@link org.apache.beam.sdk.transforms.DoFn.Setup} and @{link + * org.apache.beam.sdk.transforms.DoFn.Teardown} lifecycle methods of {@link Call}'s {@link + * org.apache.beam.sdk.transforms.DoFn}. + */ +public interface SetupTeardown extends Serializable { + + /** Called during the {@link org.apache.beam.sdk.transforms.DoFn}'s setup lifecycle method. */ + void setup() throws UserCodeExecutionException; + + /** Called during the {@link org.apache.beam.sdk.transforms.DoFn}'s teardown lifecycle method. */ + void teardown() throws UserCodeExecutionException; +} diff --git a/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/ThrottleDequeue.java b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/ThrottleDequeue.java new file mode 100644 index 0000000000000..085b13b5e1120 --- /dev/null +++ b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/ThrottleDequeue.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.io.requestresponse; + +import com.google.auto.value.AutoValue; +import java.util.Map; +import org.apache.beam.io.requestresponse.ThrottleDequeue.Result; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.PInput; +import org.apache.beam.sdk.values.POutput; +import org.apache.beam.sdk.values.PValue; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.joda.time.Instant; + +/** + * {@link ThrottleDequeue} dequeues {@link RequestT} elements at a fixed rate yielding a {@link + * Result} containing the dequeued {@link RequestT} {@link PCollection} and a {@link ApiIOError} + * {@link PCollection} of any errors. + */ +class ThrottleDequeue extends PTransform, Result> { + + private static final TupleTag FAILURE_TAG = new TupleTag() {}; + + // TODO(damondouglas): remove suppress warnings after instance utilized. + @SuppressWarnings({"unused"}) + private final Configuration configuration; + + private ThrottleDequeue(Configuration configuration) { + this.configuration = configuration; + } + + @Override + public Result expand(PCollection input) { + // TODO(damondouglas): expand in a future PR. + return new Result<>(new TupleTag() {}, PCollectionTuple.empty(input.getPipeline())); + } + + @AutoValue + abstract static class Configuration { + + @AutoValue.Builder + abstract static class Builder { + abstract Configuration build(); + } + } + + /** The {@link Result} of dequeuing {@link RequestT}s. */ + static class Result implements POutput { + + static Result of(TupleTag requestsTag, PCollectionTuple pct) { + return new Result<>(requestsTag, pct); + } + + private final Pipeline pipeline; + private final TupleTag requestsTag; + private final PCollection requests; + private final PCollection failures; + + private Result(TupleTag requestsTag, PCollectionTuple pct) { + this.pipeline = pct.getPipeline(); + this.requestsTag = requestsTag; + this.requests = pct.get(requestsTag); + this.failures = pct.get(FAILURE_TAG); + } + + @Override + public Pipeline getPipeline() { + return pipeline; + } + + @Override + public Map, PValue> expand() { + return ImmutableMap.of( + requestsTag, requests, + FAILURE_TAG, failures); + } + + @Override + public void finishSpecifyingOutput( + String transformName, PInput input, PTransform transform) {} + } +} diff --git a/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/ThrottleEnqueue.java b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/ThrottleEnqueue.java new file mode 100644 index 0000000000000..505ef86be48b3 --- /dev/null +++ b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/ThrottleEnqueue.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.io.requestresponse; + +import com.google.auto.value.AutoValue; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.TypeDescriptor; + +/** + * {@link ThrottleEnqueue} enqueues {@link RequestT} elements yielding an {@link ApiIOError} {@link + * PCollection} of any enqueue errors. + */ +class ThrottleEnqueue extends PTransform, PCollection> { + + @SuppressWarnings({"unused"}) + private final Configuration configuration; + + private ThrottleEnqueue(Configuration configuration) { + this.configuration = configuration; + } + + /** Configuration details for {@link ThrottleEnqueue}. */ + @AutoValue + abstract static class Configuration { + + static Builder builder() { + return new AutoValue_ThrottleEnqueue_Configuration.Builder<>(); + } + + abstract Builder toBuilder(); + + @AutoValue.Builder + abstract static class Builder { + + abstract Configuration build(); + } + } + + @Override + public PCollection expand(PCollection input) { + // TODO(damondouglas): expand in a future PR. + return input.getPipeline().apply(Create.empty(TypeDescriptor.of(ApiIOError.class))); + } +} diff --git a/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/ThrottleRefreshQuota.java b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/ThrottleRefreshQuota.java new file mode 100644 index 0000000000000..57e57528db4bc --- /dev/null +++ b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/ThrottleRefreshQuota.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.io.requestresponse; + +import com.google.auto.value.AutoValue; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.TypeDescriptor; +import org.joda.time.Instant; + +/** + * {@link ThrottleRefreshQuota} refreshes a quota per {@link Instant} processing events emitting any + * errors into an {@link ApiIOError} {@link PCollection}. + */ +class ThrottleRefreshQuota extends PTransform, PCollection> { + + // TODO: remove suppress warnings after configuration utilized. + @SuppressWarnings({"unused"}) + private final Configuration configuration; + + private ThrottleRefreshQuota(Configuration configuration) { + this.configuration = configuration; + } + + @Override + public PCollection expand(PCollection input) { + // TODO(damondouglas): expand in a later PR. + return input.getPipeline().apply(Create.empty(TypeDescriptor.of(ApiIOError.class))); + } + + @AutoValue + abstract static class Configuration { + + @AutoValue.Builder + abstract static class Builder { + abstract Configuration build(); + } + } +} diff --git a/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/UserCodeExecutionException.java b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/UserCodeExecutionException.java new file mode 100644 index 0000000000000..be545b6da66aa --- /dev/null +++ b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/UserCodeExecutionException.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.io.requestresponse; + +/** Base {@link Exception} for signaling errors in user custom code. */ +public class UserCodeExecutionException extends Exception { + public UserCodeExecutionException(String message) { + super(message); + } + + public UserCodeExecutionException(String message, Throwable cause) { + super(message, cause); + } + + public UserCodeExecutionException(Throwable cause) { + super(cause); + } + + public UserCodeExecutionException( + String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/UserCodeQuotaException.java b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/UserCodeQuotaException.java new file mode 100644 index 0000000000000..c513a5371da77 --- /dev/null +++ b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/UserCodeQuotaException.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.io.requestresponse; + +/** + * Extends {@link UserCodeQuotaException} to allow the user custom code to specifically signal a + * Quota or API overuse related error. + */ +public class UserCodeQuotaException extends UserCodeExecutionException { + + public UserCodeQuotaException(String message) { + super(message); + } + + public UserCodeQuotaException(String message, Throwable cause) { + super(message, cause); + } + + public UserCodeQuotaException(Throwable cause) { + super(cause); + } + + public UserCodeQuotaException( + String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/UserCodeTimeoutException.java b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/UserCodeTimeoutException.java new file mode 100644 index 0000000000000..869b8a51b73fa --- /dev/null +++ b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/UserCodeTimeoutException.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.io.requestresponse; + +/** An extension of {@link UserCodeQuotaException} to specifically signal a user code timeout. */ +public class UserCodeTimeoutException extends UserCodeExecutionException { + + public UserCodeTimeoutException(String message) { + super(message); + } + + public UserCodeTimeoutException(String message, Throwable cause) { + super(message, cause); + } + + public UserCodeTimeoutException(Throwable cause) { + super(cause); + } + + public UserCodeTimeoutException( + String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/package-info.java b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/package-info.java new file mode 100644 index 0000000000000..abaea0a58b323 --- /dev/null +++ b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Package provides Beam I/O transform support for safely reading from and writing to Web APIs. */ +package org.apache.beam.io.requestresponse; diff --git a/sdks/java/io/rrio/src/test/java/org/apache/beam/io/requestresponse/CallShouldBackoffBasedOnRejectionProbabilityTest.java b/sdks/java/io/rrio/src/test/java/org/apache/beam/io/requestresponse/CallShouldBackoffBasedOnRejectionProbabilityTest.java new file mode 100644 index 0000000000000..40aaa48c26925 --- /dev/null +++ b/sdks/java/io/rrio/src/test/java/org/apache/beam/io/requestresponse/CallShouldBackoffBasedOnRejectionProbabilityTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.io.requestresponse; + +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link CallShouldBackoffBasedOnRejectionProbability}. */ +@RunWith(JUnit4.class) +public class CallShouldBackoffBasedOnRejectionProbabilityTest { + + @Test + public void testValue() { + for (Case caze : CASES) { + CallShouldBackoffBasedOnRejectionProbability shouldBackoff = instance(); + for (boolean ar : caze.acceptRejects) { + if (ar) { + shouldBackoff.update(""); + } else { + shouldBackoff.update(new UserCodeExecutionException("")); + } + } + assertEquals(caze.toString(), caze.wantPReject, shouldBackoff.getRejectionProbability(), 0.1); + assertEquals(caze.toString(), caze.wantValue, shouldBackoff.value()); + } + } + + private static final List CASES = + Arrays.asList( + of(0, false), + of(0, false, true, true, true, true, true, true, true, true, true, true, true), + of(0, false, true), + of(0.5, false, false), + of(0.91, true, false, false, false, false, false, false, false, false, false, false)); + + private static Case of(double wantPReject, boolean wantValue, boolean... acceptRejects) { + List list = new ArrayList<>(); + for (boolean ar : acceptRejects) { + list.add(ar); + } + return new Case(list, wantPReject, wantValue); + } + + private static class Case { + private final List acceptRejects; + private final double wantPReject; + private final boolean wantValue; + + Case(List acceptRejects, double wantPReject, boolean wantValue) { + this.acceptRejects = acceptRejects; + this.wantPReject = wantPReject; + this.wantValue = wantValue; + } + + @Override + public String toString() { + return "Case{" + + "acceptRejects=" + + acceptRejects + + ", wantPReject=" + + wantPReject + + ", wantValue=" + + wantValue + + '}'; + } + } + + CallShouldBackoffBasedOnRejectionProbability instance() { + return new CallShouldBackoffBasedOnRejectionProbability().setThreshold(0.5); + } +} diff --git a/sdks/java/io/rrio/src/test/java/org/apache/beam/io/requestresponse/CallerTest.java b/sdks/java/io/rrio/src/test/java/org/apache/beam/io/requestresponse/CallerTest.java new file mode 100644 index 0000000000000..93f3de474c58d --- /dev/null +++ b/sdks/java/io/rrio/src/test/java/org/apache/beam/io/requestresponse/CallerTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.io.requestresponse; + +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import org.apache.beam.sdk.Pipeline.PipelineExecutionException; +import org.apache.beam.sdk.coders.StringUtf8Coder; +import org.apache.beam.sdk.coders.VarIntCoder; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.util.SerializableUtils; +import org.joda.time.Instant; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link Caller}. */ +@RunWith(JUnit4.class) +public class CallerTest { + + @Rule public TestPipeline pipeline = TestPipeline.create(); + + @Test + public void canSerializeImplementingClasses() { + SerializableUtils.serializeToByteArray(new CallerImpl()); + } + + @Test + public void canSerializeWhenUsedInDoFn() { + pipeline + .apply(Create.of(Instant.now())) + .apply(ParDo.of(new CallerUsingDoFn<>(new CallerImpl()))) + .setCoder(StringUtf8Coder.of()); + + pipeline.run(); + } + + @Test + public void canSignalQuotaException() { + pipeline + .apply(Create.of(1)) + .apply(ParDo.of(new CallerUsingDoFn<>(new CallerThrowsQuotaException()))) + .setCoder(VarIntCoder.of()); + + PipelineExecutionException executionException = + assertThrows(PipelineExecutionException.class, pipeline::run); + assertEquals(UserCodeQuotaException.class, executionException.getCause().getClass()); + } + + @Test + public void canSignalTimeoutException() { + pipeline + .apply(Create.of(1)) + .apply(ParDo.of(new CallerUsingDoFn<>(new CallerThrowsTimeoutException()))) + .setCoder(VarIntCoder.of()); + + PipelineExecutionException executionException = + assertThrows(PipelineExecutionException.class, pipeline::run); + assertEquals(UserCodeTimeoutException.class, executionException.getCause().getClass()); + } + + private static class CallerUsingDoFn extends DoFn { + private final Caller caller; + + private CallerUsingDoFn(Caller caller) { + this.caller = caller; + } + + @ProcessElement + public void process(@Element RequestT request, OutputReceiver receiver) + throws UserCodeExecutionException { + RequestT safeRequest = checkStateNotNull(request); + ResponseT response = caller.call(safeRequest); + receiver.output(response); + } + } + + private static class CallerImpl implements Caller { + + @Override + public String call(Instant request) throws UserCodeExecutionException { + return request.toString(); + } + } + + private static class CallerThrowsQuotaException implements Caller { + + @Override + public Integer call(Integer request) throws UserCodeExecutionException { + throw new UserCodeQuotaException("quota"); + } + } + + private static class CallerThrowsTimeoutException implements Caller { + + @Override + public Integer call(Integer request) throws UserCodeExecutionException { + throw new UserCodeTimeoutException("timeout"); + } + } +} diff --git a/sdks/java/io/rrio/src/test/java/org/apache/beam/io/requestresponse/SetupTeardownTest.java b/sdks/java/io/rrio/src/test/java/org/apache/beam/io/requestresponse/SetupTeardownTest.java new file mode 100644 index 0000000000000..eade6588955d4 --- /dev/null +++ b/sdks/java/io/rrio/src/test/java/org/apache/beam/io/requestresponse/SetupTeardownTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.io.requestresponse; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import org.apache.beam.sdk.coders.VarIntCoder; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.util.SerializableUtils; +import org.apache.beam.sdk.util.UserCodeException; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.UncheckedExecutionException; +import org.junit.Rule; +import org.junit.Test; + +public class SetupTeardownTest { + @Rule public TestPipeline pipeline = TestPipeline.create(); + + @Test + public void canSerializeImplementingClasses() { + SerializableUtils.serializeToByteArray(new SetupTeardownImpl()); + } + + @Test + public void canSerializeWhenUsedInDoFn() { + pipeline + .apply(Create.of(1)) + .apply(ParDo.of(new SetupTeardownUsingDoFn(new SetupTeardownImpl()))) + .setCoder(VarIntCoder.of()); + + pipeline.run(); + } + + @Test + public void canSignalQuotaException() { + pipeline + .apply(Create.of(1)) + .apply(ParDo.of(new SetupTeardownUsingDoFn(new ThrowsQuotaException()))) + .setCoder(VarIntCoder.of()); + + UncheckedExecutionException exception = + assertThrows(UncheckedExecutionException.class, pipeline::run); + UserCodeException userCodeException = (UserCodeException) exception.getCause(); + assertEquals(UserCodeQuotaException.class, userCodeException.getCause().getClass()); + } + + @Test + public void canSignalTimeoutException() { + pipeline + .apply(Create.of(1)) + .apply(ParDo.of(new SetupTeardownUsingDoFn(new ThrowsTimeoutException()))) + .setCoder(VarIntCoder.of()); + + UncheckedExecutionException exception = + assertThrows(UncheckedExecutionException.class, pipeline::run); + UserCodeException userCodeException = (UserCodeException) exception.getCause(); + assertEquals(UserCodeTimeoutException.class, userCodeException.getCause().getClass()); + } + + private static class SetupTeardownUsingDoFn extends DoFn { + private final SetupTeardown setupTeardown; + + private SetupTeardownUsingDoFn(SetupTeardown setupTeardown) { + this.setupTeardown = setupTeardown; + } + + @Setup + public void setup() throws UserCodeExecutionException { + setupTeardown.setup(); + } + + @Teardown + public void teardown() throws UserCodeExecutionException { + setupTeardown.teardown(); + } + + @ProcessElement + public void process() {} + } + + private static class SetupTeardownImpl implements SetupTeardown { + @Override + public void setup() throws UserCodeExecutionException {} + + @Override + public void teardown() throws UserCodeExecutionException {} + } + + private static class ThrowsQuotaException implements SetupTeardown { + + @Override + public void setup() throws UserCodeExecutionException { + throw new UserCodeQuotaException("quota"); + } + + @Override + public void teardown() throws UserCodeExecutionException {} + } + + private static class ThrowsTimeoutException implements SetupTeardown { + + @Override + public void setup() throws UserCodeExecutionException { + throw new UserCodeTimeoutException("timeout"); + } + + @Override + public void teardown() throws UserCodeExecutionException {} + } +} diff --git a/sdks/java/io/singlestore/build.gradle b/sdks/java/io/singlestore/build.gradle index 31b6a5c20e94b..796103c25c4d3 100644 --- a/sdks/java/io/singlestore/build.gradle +++ b/sdks/java/io/singlestore/build.gradle @@ -27,7 +27,7 @@ description = "Apache Beam :: SDKs :: Java :: IO :: SingleStore" ext.summary = "IO to read and write on SingleStore datasource." dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.joda_time implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.singlestore_jdbc diff --git a/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/SingleStoreDefaultUserDataMapper.java b/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/SingleStoreDefaultUserDataMapper.java index 2035ce0554f18..51db70d7497aa 100644 --- a/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/SingleStoreDefaultUserDataMapper.java +++ b/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/SingleStoreDefaultUserDataMapper.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.singlestore; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; diff --git a/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/SingleStoreIO.java b/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/SingleStoreIO.java index de2a68052789f..b8057d8796869 100644 --- a/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/SingleStoreIO.java +++ b/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/SingleStoreIO.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.singlestore; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import java.io.IOException; @@ -53,6 +53,7 @@ import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker; import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.util.Preconditions; +import org.apache.beam.sdk.util.ReleaseInfo; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionView; @@ -347,6 +348,12 @@ public DataSource getDataSource() { String connectionProperties = SingleStoreUtil.getArgumentWithDefault(getConnectionProperties(), ""); connectionProperties += (connectionProperties.isEmpty() ? "" : ";") + "allowLocalInfile=TRUE"; + connectionProperties += + String.format( + ";connectionAttributes=_connector_name:%s,_connector_version:%s,_product_version:%s", + "Apache Beam SingleStoreDB I/O", + ReleaseInfo.getReleaseInfo().getVersion(), + ReleaseInfo.getReleaseInfo().getVersion()); String username = getUsername(); String password = getPassword(); diff --git a/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/schematransform/SingleStoreSchemaTransformReadProvider.java b/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/schematransform/SingleStoreSchemaTransformReadProvider.java index 7db0dc956b35e..9511a59d472ca 100644 --- a/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/schematransform/SingleStoreSchemaTransformReadProvider.java +++ b/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/schematransform/SingleStoreSchemaTransformReadProvider.java @@ -25,11 +25,10 @@ import org.apache.beam.sdk.schemas.transforms.SchemaTransform; import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; -import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionRowTuple; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; /** * An implementation of {@link TypedSchemaTransformProvider} for SingleStoreDB read jobs configured @@ -80,33 +79,13 @@ public List outputCollectionNames() { * An implementation of {@link SchemaTransform} for SingleStoreDB read jobs configured using * {@link SingleStoreSchemaTransformReadConfiguration}. */ - private static class SingleStoreReadSchemaTransform implements SchemaTransform { + private static class SingleStoreReadSchemaTransform extends SchemaTransform { private final SingleStoreSchemaTransformReadConfiguration configuration; SingleStoreReadSchemaTransform(SingleStoreSchemaTransformReadConfiguration configuration) { this.configuration = configuration; } - /** Implements {@link SchemaTransform} buildTransform method. */ - @Override - public PTransform buildTransform() { - return new PCollectionRowTupleTransform(configuration); - } - } - - /** - * An implementation of {@link PTransform} for SingleStoreDB read jobs configured using {@link - * SingleStoreSchemaTransformReadConfiguration}. - */ - static class PCollectionRowTupleTransform - extends PTransform { - - private final SingleStoreSchemaTransformReadConfiguration configuration; - - PCollectionRowTupleTransform(SingleStoreSchemaTransformReadConfiguration configuration) { - this.configuration = configuration; - } - @Override public PCollectionRowTuple expand(PCollectionRowTuple input) { if (!input.getAll().isEmpty()) { diff --git a/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/schematransform/SingleStoreSchemaTransformWriteProvider.java b/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/schematransform/SingleStoreSchemaTransformWriteProvider.java index 19a3b383109d6..dafa10087a4a8 100644 --- a/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/schematransform/SingleStoreSchemaTransformWriteProvider.java +++ b/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/schematransform/SingleStoreSchemaTransformWriteProvider.java @@ -26,7 +26,6 @@ import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionRowTuple; @@ -84,33 +83,13 @@ public List outputCollectionNames() { * An implementation of {@link SchemaTransform} for SingleStoreDB write jobs configured using * {@link SingleStoreSchemaTransformWriteConfiguration}. */ - private static class SingleStoreWriteSchemaTransform implements SchemaTransform { + private static class SingleStoreWriteSchemaTransform extends SchemaTransform { private final SingleStoreSchemaTransformWriteConfiguration configuration; SingleStoreWriteSchemaTransform(SingleStoreSchemaTransformWriteConfiguration configuration) { this.configuration = configuration; } - /** Implements {@link SchemaTransform} buildTransform method. */ - @Override - public PTransform buildTransform() { - return new PCollectionRowTupleTransform(configuration); - } - } - - /** - * An implementation of {@link PTransform} for SingleStoreDB write jobs configured using {@link - * SingleStoreSchemaTransformWriteConfiguration}. - */ - static class PCollectionRowTupleTransform - extends PTransform { - - private final SingleStoreSchemaTransformWriteConfiguration configuration; - - PCollectionRowTupleTransform(SingleStoreSchemaTransformWriteConfiguration configuration) { - this.configuration = configuration; - } - @Override public PCollectionRowTuple expand(PCollectionRowTuple input) { if (!input.has(INPUT_TAG)) { diff --git a/sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/SingleStoreIOConnectionAttributesIT.java b/sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/SingleStoreIOConnectionAttributesIT.java new file mode 100644 index 0000000000000..6023ae00d4fd7 --- /dev/null +++ b/sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/SingleStoreIOConnectionAttributesIT.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.singlestore; + +import static org.apache.beam.sdk.io.common.IOITHelper.readIOTestPipelineOptions; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; +import javax.sql.DataSource; +import org.apache.beam.sdk.util.ReleaseInfo; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class SingleStoreIOConnectionAttributesIT { + private static String serverName; + + private static String username; + + private static String password; + + private static Integer port; + + @BeforeClass + public static void setup() throws Exception { + SingleStoreIOTestPipelineOptions options; + try { + options = readIOTestPipelineOptions(SingleStoreIOTestPipelineOptions.class); + } catch (IllegalArgumentException e) { + options = null; + } + org.junit.Assume.assumeNotNull(options); + + serverName = options.getSingleStoreServerName(); + username = options.getSingleStoreUsername(); + password = options.getSingleStorePassword(); + port = options.getSingleStorePort(); + } + + @Test + public void connectionAttributes() throws Exception { + Map attributes = new HashMap(); + attributes.put("_connector_name", "Apache Beam SingleStoreDB I/O"); + attributes.put("_connector_version", ReleaseInfo.getReleaseInfo().getVersion()); + attributes.put("_product_version", ReleaseInfo.getReleaseInfo().getVersion()); + + SingleStoreIO.DataSourceConfiguration dataSourceConfiguration = + SingleStoreIO.DataSourceConfiguration.create(serverName + ":" + port) + .withPassword(password) + .withUsername(username); + + DataSource dataSource = dataSourceConfiguration.getDataSource(); + + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement(); + ResultSet rs = + stmt.executeQuery("select * from information_schema.mv_connection_attributes"); ) { + while (rs.next()) { + String attribute = rs.getString(3); + String value = rs.getString(4); + if (attributes.containsKey(attribute)) { + assertEquals(attributes.get(attribute), value); + attributes.remove(attribute); + } + } + } + + assertTrue(attributes.isEmpty()); + } +} diff --git a/sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/SingleStoreIOSchemaTransformIT.java b/sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/SingleStoreIOSchemaTransformIT.java index 4ded4cd452a81..e1a280f1594e6 100644 --- a/sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/SingleStoreIOSchemaTransformIT.java +++ b/sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/SingleStoreIOSchemaTransformIT.java @@ -42,7 +42,6 @@ import org.apache.beam.sdk.transforms.Combine; import org.apache.beam.sdk.transforms.Count; import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.Sum; @@ -138,8 +137,6 @@ private PipelineResult runWrite() { Row configurationRow = configuration.toBeamRow(); SchemaTransform schemaTransform = provider.from(configurationRow); - PTransform pCollectionRowTupleTransform = - schemaTransform.buildTransform(); Schema.Builder schemaBuilder = new Schema.Builder(); schemaBuilder.addField("id", Schema.FieldType.INT32); @@ -166,7 +163,7 @@ private PipelineResult runWrite() { PCollectionRowTuple input = PCollectionRowTuple.of(SingleStoreSchemaTransformWriteProvider.INPUT_TAG, rows); String tag = provider.outputCollectionNames().get(0); - PCollectionRowTuple output = input.apply(pCollectionRowTupleTransform); + PCollectionRowTuple output = input.apply(schemaTransform); assertTrue(output.has(tag)); PCollection writtenRows = output @@ -194,12 +191,10 @@ private PipelineResult runRead() { Row configurationRow = configuration.toBeamRow(); SchemaTransform schemaTransform = provider.from(configurationRow); - PTransform pCollectionRowTupleTransform = - schemaTransform.buildTransform(); PCollectionRowTuple input = PCollectionRowTuple.empty(pipelineRead); String tag = provider.outputCollectionNames().get(0); - PCollectionRowTuple output = input.apply(pCollectionRowTupleTransform); + PCollectionRowTuple output = input.apply(schemaTransform); assertTrue(output.has(tag)); PCollection namesAndIds = output @@ -227,12 +222,10 @@ private PipelineResult runReadWithPartitions() { Row configurationRow = configuration.toBeamRow(); SchemaTransform schemaTransform = provider.from(configurationRow); - PTransform pCollectionRowTupleTransform = - schemaTransform.buildTransform(); PCollectionRowTuple input = PCollectionRowTuple.empty(pipelineReadWithPartitions); String tag = provider.outputCollectionNames().get(0); - PCollectionRowTuple output = input.apply(pCollectionRowTupleTransform); + PCollectionRowTuple output = input.apply(schemaTransform); assertTrue(output.has(tag)); PCollection namesAndIds = output diff --git a/sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/WriteTest.java b/sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/WriteTest.java index 68b34ff53ce96..e93249e95048f 100644 --- a/sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/WriteTest.java +++ b/sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/WriteTest.java @@ -40,8 +40,8 @@ import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.Sum; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.commons.dbcp2.DelegatingStatement; import org.junit.After; import org.junit.Before; diff --git a/sdks/java/io/snowflake/build.gradle b/sdks/java/io/snowflake/build.gradle index ddb66118241fe..377b5c2704108 100644 --- a/sdks/java/io/snowflake/build.gradle +++ b/sdks/java/io/snowflake/build.gradle @@ -25,7 +25,7 @@ enableJavaPerformanceTesting() description = "Apache Beam :: SDKs :: Java :: IO :: Snowflake" ext.summary = "IO to read and write on Snowflake." dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(path: ":sdks:java:extensions:google-cloud-platform-core") permitUnusedDeclared project(path: ":sdks:java:extensions:google-cloud-platform-core") diff --git a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/KeyPairUtils.java b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/KeyPairUtils.java index ee8098fa3e68d..280fc71576c5f 100644 --- a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/KeyPairUtils.java +++ b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/KeyPairUtils.java @@ -31,7 +31,7 @@ import javax.crypto.EncryptedPrivateKeyInfo; import javax.crypto.SecretKeyFactory; import javax.crypto.spec.PBEKeySpec; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; import org.bouncycastle.util.encoders.Base64; import org.bouncycastle.util.encoders.DecoderException; import org.bouncycastle.util.io.pem.PemObject; diff --git a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/SnowflakeIO.java b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/SnowflakeIO.java index f8ea45a0bf1d7..6fad4a89b6354 100644 --- a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/SnowflakeIO.java +++ b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/SnowflakeIO.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.snowflake; import static org.apache.beam.sdk.io.TextIO.readFiles; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import com.opencsv.CSVParser; @@ -83,9 +83,9 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; import org.joda.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/crosslanguage/SnowflakeTransformRegistrar.java b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/crosslanguage/SnowflakeTransformRegistrar.java index bd2648f7eecc9..65095da34a713 100644 --- a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/crosslanguage/SnowflakeTransformRegistrar.java +++ b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/crosslanguage/SnowflakeTransformRegistrar.java @@ -22,7 +22,7 @@ import org.apache.beam.sdk.expansion.ExternalTransformRegistrar; import org.apache.beam.sdk.io.snowflake.SnowflakeIO; import org.apache.beam.sdk.transforms.ExternalTransformBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** * Exposes {@link SnowflakeIO.Read} and {@link SnowflakeIO.Write} as an external transform for diff --git a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/data/SnowflakeTableSchema.java b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/data/SnowflakeTableSchema.java index bcb190581a93f..083697d12e163 100644 --- a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/data/SnowflakeTableSchema.java +++ b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/data/SnowflakeTableSchema.java @@ -23,7 +23,7 @@ import net.snowflake.client.jdbc.internal.fasterxml.jackson.annotation.JsonProperty; import org.apache.beam.sdk.io.snowflake.SnowflakeIO; import org.apache.beam.sdk.io.snowflake.enums.CreateDisposition; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; /** * POJO representing schema of Table in Snowflake. Used by {@link SnowflakeIO.Write} when {@link diff --git a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/services/SnowflakeBatchServiceImpl.java b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/services/SnowflakeBatchServiceImpl.java index 61fd65ccde67a..e317e44d49016 100644 --- a/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/services/SnowflakeBatchServiceImpl.java +++ b/sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/services/SnowflakeBatchServiceImpl.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.snowflake.services; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.math.BigInteger; import java.nio.charset.StandardCharsets; diff --git a/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/StreamingSnowflakeIOIT.java b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/StreamingSnowflakeIOIT.java index e2a408816015e..4920fcb688542 100644 --- a/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/StreamingSnowflakeIOIT.java +++ b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/StreamingSnowflakeIOIT.java @@ -19,8 +19,8 @@ import static org.apache.beam.sdk.io.snowflake.test.TestUtils.SnowflakeIOITPipelineOptions; import static org.apache.beam.sdk.io.snowflake.test.TestUtils.getTestRowDataMapper; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets.newHashSet; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists.newArrayList; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets.newHashSet; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; diff --git a/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/read/SnowflakeIOReadTest.java b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/read/SnowflakeIOReadTest.java index d41e2032e2879..df70fd87aac92 100644 --- a/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/read/SnowflakeIOReadTest.java +++ b/sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/read/SnowflakeIOReadTest.java @@ -24,7 +24,7 @@ import org.apache.avro.generic.GenericRecordBuilder; import org.apache.beam.sdk.Pipeline.PipelineExecutionException; import org.apache.beam.sdk.extensions.avro.coders.AvroCoder; -import org.apache.beam.sdk.io.AvroGeneratedUser; +import org.apache.beam.sdk.extensions.avro.io.AvroGeneratedUser; import org.apache.beam.sdk.io.snowflake.SnowflakeIO; import org.apache.beam.sdk.io.snowflake.services.SnowflakeServices; import org.apache.beam.sdk.io.snowflake.test.FakeSnowflakeBasicDataSource; @@ -35,7 +35,7 @@ import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Rule; diff --git a/sdks/java/io/solr/OWNERS b/sdks/java/io/solr/OWNERS deleted file mode 100644 index 6049011f10e84..0000000000000 --- a/sdks/java/io/solr/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - timrobertson100 diff --git a/sdks/java/io/solr/build.gradle b/sdks/java/io/solr/build.gradle index 301264f6566b4..1083af13e9c12 100644 --- a/sdks/java/io/solr/build.gradle +++ b/sdks/java/io/solr/build.gradle @@ -27,7 +27,7 @@ ext.summary = "IO to read and write from/to Solr." String solrVersion = "8.11.1" dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.commons_compress implementation library.java.joda_time diff --git a/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/AuthorizedSolrClient.java b/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/AuthorizedSolrClient.java index 19adae6ad5824..42391b64037a2 100644 --- a/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/AuthorizedSolrClient.java +++ b/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/AuthorizedSolrClient.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.solr; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.Closeable; import java.io.IOException; diff --git a/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/SolrIO.java b/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/SolrIO.java index 8fe4bd3a9414a..50130a8fbd1c4 100644 --- a/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/SolrIO.java +++ b/sdks/java/io/solr/src/main/java/org/apache/beam/sdk/io/solr/SolrIO.java @@ -17,9 +17,9 @@ */ package org.apache.beam.sdk.io.solr; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; import java.io.IOException; @@ -42,9 +42,9 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; diff --git a/sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/SolrIOTest.java b/sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/SolrIOTest.java index 399e5f8a3eed1..bdc76824f5be1 100644 --- a/sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/SolrIOTest.java +++ b/sdks/java/io/solr/src/test/java/org/apache/beam/sdk/io/solr/SolrIOTest.java @@ -36,7 +36,7 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.DoFnTester; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.BaseEncoding; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.BaseEncoding; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; diff --git a/sdks/java/io/sparkreceiver/2/OWNERS b/sdks/java/io/sparkreceiver/2/OWNERS deleted file mode 100644 index e60eec69932b0..0000000000000 --- a/sdks/java/io/sparkreceiver/2/OWNERS +++ /dev/null @@ -1 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners diff --git a/sdks/java/io/sparkreceiver/2/build.gradle b/sdks/java/io/sparkreceiver/2/build.gradle index b039b9ce2043d..8cfdb81d108d0 100644 --- a/sdks/java/io/sparkreceiver/2/build.gradle +++ b/sdks/java/io/sparkreceiver/2/build.gradle @@ -45,7 +45,7 @@ dependencies { implementation library.java.slf4j_api implementation library.java.spark_streaming implementation library.java.spark_core - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") compileOnly "org.scala-lang:scala-library:2.11.12" testImplementation library.java.junit diff --git a/sdks/java/io/sparkreceiver/2/src/main/java/org/apache/beam/sdk/io/sparkreceiver/ReceiverBuilder.java b/sdks/java/io/sparkreceiver/2/src/main/java/org/apache/beam/sdk/io/sparkreceiver/ReceiverBuilder.java index 5d7966ffedb76..ba2f065fd39d6 100644 --- a/sdks/java/io/sparkreceiver/2/src/main/java/org/apache/beam/sdk/io/sparkreceiver/ReceiverBuilder.java +++ b/sdks/java/io/sparkreceiver/2/src/main/java/org/apache/beam/sdk/io/sparkreceiver/ReceiverBuilder.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.sparkreceiver; import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.Serializable; import java.lang.reflect.Constructor; diff --git a/sdks/java/io/sparkreceiver/2/src/main/java/org/apache/beam/sdk/io/sparkreceiver/SparkReceiverIO.java b/sdks/java/io/sparkreceiver/2/src/main/java/org/apache/beam/sdk/io/sparkreceiver/SparkReceiverIO.java index d278bd02bdc98..d51c26154328c 100644 --- a/sdks/java/io/sparkreceiver/2/src/main/java/org/apache/beam/sdk/io/sparkreceiver/SparkReceiverIO.java +++ b/sdks/java/io/sparkreceiver/2/src/main/java/org/apache/beam/sdk/io/sparkreceiver/SparkReceiverIO.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.sparkreceiver; import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import org.apache.beam.sdk.transforms.Impulse; diff --git a/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ArrayBufferDataReceiver.java b/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ArrayBufferDataReceiver.java index 849ea0a1373e9..3db565a452ec1 100644 --- a/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ArrayBufferDataReceiver.java +++ b/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ArrayBufferDataReceiver.java @@ -19,7 +19,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.spark.storage.StorageLevel; import org.apache.spark.streaming.receiver.Receiver; import org.slf4j.Logger; diff --git a/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ByteBufferDataReceiver.java b/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ByteBufferDataReceiver.java index dcef495aa67aa..53c152be8062c 100644 --- a/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ByteBufferDataReceiver.java +++ b/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ByteBufferDataReceiver.java @@ -20,7 +20,7 @@ import java.nio.ByteBuffer; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.commons.lang3.SerializationUtils; import org.apache.spark.storage.StorageLevel; import org.apache.spark.streaming.receiver.Receiver; diff --git a/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/CustomReceiverWithOffset.java b/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/CustomReceiverWithOffset.java index 6bba7bee9af3b..084208556fee8 100644 --- a/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/CustomReceiverWithOffset.java +++ b/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/CustomReceiverWithOffset.java @@ -19,7 +19,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.spark.storage.StorageLevel; import org.apache.spark.streaming.receiver.Receiver; import org.slf4j.Logger; diff --git a/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/IteratorDataReceiver.java b/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/IteratorDataReceiver.java index 8999802542c2d..4bc51ea33f306 100644 --- a/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/IteratorDataReceiver.java +++ b/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/IteratorDataReceiver.java @@ -22,7 +22,7 @@ import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.spark.storage.StorageLevel; import org.apache.spark.streaming.receiver.Receiver; import org.slf4j.Logger; diff --git a/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/RabbitMqReceiverWithOffset.java b/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/RabbitMqReceiverWithOffset.java index 1ba295d0aba55..da8f1dde841ec 100644 --- a/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/RabbitMqReceiverWithOffset.java +++ b/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/RabbitMqReceiverWithOffset.java @@ -27,7 +27,7 @@ import java.util.Collections; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.spark.storage.StorageLevel; import org.apache.spark.streaming.receiver.Receiver; import org.slf4j.Logger; diff --git a/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/SparkReceiverIOIT.java b/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/SparkReceiverIOIT.java index 4f839fc7c5d38..90c5944602a85 100644 --- a/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/SparkReceiverIOIT.java +++ b/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/SparkReceiverIOIT.java @@ -59,7 +59,7 @@ import org.apache.beam.sdk.testutils.publishing.InfluxDBSettings; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.junit.AfterClass; diff --git a/sdks/java/io/splunk/build.gradle b/sdks/java/io/splunk/build.gradle index 8f719cdf06786..dd1b15e10dde9 100644 --- a/sdks/java/io/splunk/build.gradle +++ b/sdks/java/io/splunk/build.gradle @@ -36,7 +36,7 @@ dependencies { implementation library.java.http_core implementation library.java.joda_time implementation library.java.slf4j_api - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre testImplementation library.java.junit testImplementation group: 'org.mock-server', name: 'mockserver-junit-rule', version: '5.10.0' testImplementation group: 'org.mock-server', name: 'mockserver-client-java', version: '5.10.0' diff --git a/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/HttpEventPublisher.java b/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/HttpEventPublisher.java index 3f3ebb89cf28c..6c5537990bdfd 100644 --- a/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/HttpEventPublisher.java +++ b/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/HttpEventPublisher.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.splunk; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.GZipEncoding; @@ -48,9 +48,9 @@ import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.RequestConfig; import org.apache.http.conn.ssl.DefaultHostnameVerifier; diff --git a/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkEvent.java b/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkEvent.java index 90b2199078f38..7dd78e1754b4c 100644 --- a/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkEvent.java +++ b/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkEvent.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.splunk; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import com.google.gson.annotations.SerializedName; diff --git a/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkEventWriter.java b/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkEventWriter.java index d3d5787c8b01a..8ec2a064ee0dd 100644 --- a/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkEventWriter.java +++ b/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkEventWriter.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.splunk; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpResponseException; @@ -53,8 +53,8 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.slf4j.Logger; diff --git a/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkIO.java b/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkIO.java index fb995297d9b1c..bd1e716951d44 100644 --- a/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkIO.java +++ b/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkIO.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.splunk; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import java.util.concurrent.ThreadLocalRandom; @@ -28,7 +28,7 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -201,7 +201,6 @@ abstract Builder setEnableGzipHttpCompression( * @param batchCount for batching requests */ public Write withBatchCount(ValueProvider batchCount) { - checkNotNull(batchCount, "withBatchCount(batchCount) called with null input."); return toBuilder().setBatchCount(batchCount).build(); } @@ -211,7 +210,6 @@ public Write withBatchCount(ValueProvider batchCount) { * @param batchCount for batching requests */ public Write withBatchCount(Integer batchCount) { - checkNotNull(batchCount, "withBatchCount(batchCount) called with null input."); return toBuilder().setBatchCount(StaticValueProvider.of(batchCount)).build(); } @@ -221,7 +219,6 @@ public Write withBatchCount(Integer batchCount) { * @param parallelism for controlling the number of concurrent http client connections */ public Write withParallelism(ValueProvider parallelism) { - checkNotNull(parallelism, "withParallelism(parallelism) called with null input."); return toBuilder().setParallelism(parallelism).build(); } @@ -232,7 +229,6 @@ public Write withParallelism(ValueProvider parallelism) { * @return {@link Builder} */ public Write withParallelism(Integer parallelism) { - checkNotNull(parallelism, "withParallelism(parallelism) called with null input."); return toBuilder().setParallelism(StaticValueProvider.of(parallelism)).build(); } @@ -245,9 +241,6 @@ public Write withParallelism(Integer parallelism) { */ public Write withDisableCertificateValidation( ValueProvider disableCertificateValidation) { - checkNotNull( - disableCertificateValidation, - "withDisableCertificateValidation(disableCertificateValidation) called with null input."); return toBuilder().setDisableCertificateValidation(disableCertificateValidation).build(); } @@ -257,9 +250,6 @@ public Write withDisableCertificateValidation( * @param disableCertificateValidation for disabling certificate validation */ public Write withDisableCertificateValidation(Boolean disableCertificateValidation) { - checkNotNull( - disableCertificateValidation, - "withDisableCertificateValidation(disableCertificateValidation) called with null input."); return toBuilder() .setDisableCertificateValidation(StaticValueProvider.of(disableCertificateValidation)) .build(); @@ -273,9 +263,6 @@ public Write withDisableCertificateValidation(Boolean disableCertificateValidati * @return {@link Builder} */ public Write withRootCaCertificatePath(ValueProvider rootCaCertificatePath) { - checkNotNull( - rootCaCertificatePath, - "withRootCaCertificatePath(rootCaCertificatePath) called with null input."); return toBuilder().setRootCaCertificatePath(rootCaCertificatePath).build(); } @@ -286,9 +273,6 @@ public Write withRootCaCertificatePath(ValueProvider rootCaCertificatePa * @return {@link Builder} */ public Write withRootCaCertificatePath(String rootCaCertificatePath) { - checkNotNull( - rootCaCertificatePath, - "withRootCaCertificatePath(rootCaCertificatePath) called with null input."); return toBuilder() .setRootCaCertificatePath(StaticValueProvider.of(rootCaCertificatePath)) .build(); @@ -302,7 +286,6 @@ public Write withRootCaCertificatePath(String rootCaCertificatePath) { * @return {@link Builder} */ public Write withEnableBatchLogs(ValueProvider enableBatchLogs) { - checkNotNull(enableBatchLogs, "withEnableBatchLogs(enableBatchLogs) called with null input."); return toBuilder().setEnableBatchLogs(enableBatchLogs).build(); } @@ -313,7 +296,6 @@ public Write withEnableBatchLogs(ValueProvider enableBatchLogs) { * @return {@link Builder} */ public Write withEnableBatchLogs(Boolean enableBatchLogs) { - checkNotNull(enableBatchLogs, "withEnableBatchLogs(enableBatchLogs) called with null input."); return toBuilder().setEnableBatchLogs(StaticValueProvider.of(enableBatchLogs)).build(); } @@ -325,9 +307,6 @@ public Write withEnableBatchLogs(Boolean enableBatchLogs) { * @return {@link Builder} */ public Write withEnableGzipHttpCompression(ValueProvider enableGzipHttpCompression) { - checkNotNull( - enableGzipHttpCompression, - "withEnableGzipHttpCompression(enableGzipHttpCompression) called with null input."); return toBuilder().setEnableGzipHttpCompression(enableGzipHttpCompression).build(); } @@ -338,9 +317,6 @@ public Write withEnableGzipHttpCompression(ValueProvider enableGzipHttp * @return {@link Builder} */ public Write withEnableGzipHttpCompression(Boolean enableGzipHttpCompression) { - checkNotNull( - enableGzipHttpCompression, - "withEnableGzipHttpCompression(enableGzipHttpCompression) called with null input."); return toBuilder() .setEnableGzipHttpCompression(StaticValueProvider.of(enableGzipHttpCompression)) .build(); diff --git a/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkWriteError.java b/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkWriteError.java index ee1597da935a1..826360062398a 100644 --- a/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkWriteError.java +++ b/sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkWriteError.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.splunk; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import org.apache.beam.sdk.schemas.AutoValueSchema; diff --git a/sdks/java/io/splunk/src/test/java/org/apache/beam/sdk/io/splunk/HttpEventPublisherTest.java b/sdks/java/io/splunk/src/test/java/org/apache/beam/sdk/io/splunk/HttpEventPublisherTest.java index 0b5e34a38d97d..82374339ac95b 100644 --- a/sdks/java/io/splunk/src/test/java/org/apache/beam/sdk/io/splunk/HttpEventPublisherTest.java +++ b/sdks/java/io/splunk/src/test/java/org/apache/beam/sdk/io/splunk/HttpEventPublisherTest.java @@ -35,9 +35,9 @@ import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import javax.net.ssl.SSLHandshakeException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Resources; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Resources; import org.junit.Before; import org.junit.Test; import org.mockserver.configuration.ConfigurationProperties; diff --git a/sdks/java/io/splunk/src/test/java/org/apache/beam/sdk/io/splunk/SplunkEventWriterTest.java b/sdks/java/io/splunk/src/test/java/org/apache/beam/sdk/io/splunk/SplunkEventWriterTest.java index e31ec0ced09a8..3633844ab6d2b 100644 --- a/sdks/java/io/splunk/src/test/java/org/apache/beam/sdk/io/splunk/SplunkEventWriterTest.java +++ b/sdks/java/io/splunk/src/test/java/org/apache/beam/sdk/io/splunk/SplunkEventWriterTest.java @@ -31,8 +31,8 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/splunk/src/test/java/org/apache/beam/sdk/io/splunk/SplunkIOTest.java b/sdks/java/io/splunk/src/test/java/org/apache/beam/sdk/io/splunk/SplunkIOTest.java index f34047ff331ed..32c98513ea248 100644 --- a/sdks/java/io/splunk/src/test/java/org/apache/beam/sdk/io/splunk/SplunkIOTest.java +++ b/sdks/java/io/splunk/src/test/java/org/apache/beam/sdk/io/splunk/SplunkIOTest.java @@ -23,8 +23,8 @@ import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/synthetic/build.gradle b/sdks/java/io/synthetic/build.gradle index 07729267f260d..3098dd3042ca9 100644 --- a/sdks/java/io/synthetic/build.gradle +++ b/sdks/java/io/synthetic/build.gradle @@ -32,10 +32,10 @@ dependencies { implementation library.java.jackson_annotations implementation library.java.jackson_databind implementation library.java.slf4j_api - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") - testImplementation library.java.vendored_guava_26_0_jre + testImplementation library.java.vendored_guava_32_1_2_jre testImplementation library.java.hamcrest testImplementation library.java.junit testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") diff --git a/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/SyntheticBoundedSource.java b/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/SyntheticBoundedSource.java index 850fb6db1ff9b..798bd9c4c2c28 100644 --- a/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/SyntheticBoundedSource.java +++ b/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/SyntheticBoundedSource.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.synthetic; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.util.List; @@ -32,7 +32,7 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/SyntheticOptions.java b/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/SyntheticOptions.java index 1fb9c770a665b..ae2d49de5a584 100644 --- a/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/SyntheticOptions.java +++ b/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/SyntheticOptions.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.synthetic; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -32,8 +32,8 @@ import java.nio.ByteBuffer; import java.util.Random; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashFunction; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.HashFunction; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; import org.apache.commons.math3.distribution.ConstantRealDistribution; import org.apache.commons.math3.distribution.ExponentialDistribution; import org.apache.commons.math3.distribution.IntegerDistribution; diff --git a/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/SyntheticSourceOptions.java b/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/SyntheticSourceOptions.java index 6452ce5d26c87..cf24693164743 100644 --- a/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/SyntheticSourceOptions.java +++ b/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/SyntheticSourceOptions.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.synthetic; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; diff --git a/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/SyntheticStep.java b/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/SyntheticStep.java index 75a4c9d166c3d..d32640ffbf7d1 100644 --- a/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/SyntheticStep.java +++ b/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/SyntheticStep.java @@ -18,7 +18,7 @@ package org.apache.beam.sdk.io.synthetic; import static org.apache.beam.sdk.io.synthetic.delay.SyntheticDelay.delay; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Random; @@ -28,10 +28,10 @@ import org.apache.beam.sdk.metrics.Metrics; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.values.KV; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheBuilder; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.CacheLoader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LoadingCache; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.RateLimiter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheLoader; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.LoadingCache; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.RateLimiter; import org.joda.time.Duration; /** diff --git a/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/delay/SyntheticDelay.java b/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/delay/SyntheticDelay.java index 95247a3eec4d6..dc765dcd084ea 100644 --- a/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/delay/SyntheticDelay.java +++ b/sdks/java/io/synthetic/src/main/java/org/apache/beam/sdk/io/synthetic/delay/SyntheticDelay.java @@ -20,9 +20,9 @@ import java.util.Random; import java.util.concurrent.TimeUnit; import org.apache.beam.sdk.io.synthetic.SyntheticOptions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Stopwatch; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Stopwatch; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles; import org.joda.time.Duration; /** Utility functions used in {@link org.apache.beam.sdk.io.synthetic}. */ diff --git a/sdks/java/io/synthetic/src/test/java/org/apache/beam/sdk/io/synthetic/SyntheticStepTest.java b/sdks/java/io/synthetic/src/test/java/org/apache/beam/sdk/io/synthetic/SyntheticStepTest.java index f2eea4b9969c8..b400dd020a7ae 100644 --- a/sdks/java/io/synthetic/src/test/java/org/apache/beam/sdk/io/synthetic/SyntheticStepTest.java +++ b/sdks/java/io/synthetic/src/test/java/org/apache/beam/sdk/io/synthetic/SyntheticStepTest.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.commons.math3.distribution.ConstantRealDistribution; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/thrift/build.gradle b/sdks/java/io/thrift/build.gradle index 1b20afbceb111..47c8111a0257f 100644 --- a/sdks/java/io/thrift/build.gradle +++ b/sdks/java/io/thrift/build.gradle @@ -35,13 +35,14 @@ description = "Apache Beam :: SDKs :: Java :: IO :: Thrift" ext.summary = "IO to read and write Thrift encoded data files" dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.slf4j_api implementation "org.apache.thrift:libthrift:0.14.1" implementation project(path: ":sdks:java:core", configuration: "shadow") testImplementation library.java.junit testRuntimeOnly library.java.slf4j_jdk14 testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") + testImplementation project(path: ":sdks:java:extensions:avro", configuration: "testRuntimeMigration") } /* Removed due to lack of Thrift on Jenkins workers. diff --git a/sdks/java/io/thrift/src/main/java/org/apache/beam/sdk/io/thrift/ThriftIO.java b/sdks/java/io/thrift/src/main/java/org/apache/beam/sdk/io/thrift/ThriftIO.java index 3e5af37b460ee..f974921436290 100644 --- a/sdks/java/io/thrift/src/main/java/org/apache/beam/sdk/io/thrift/ThriftIO.java +++ b/sdks/java/io/thrift/src/main/java/org/apache/beam/sdk/io/thrift/ThriftIO.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.thrift; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import java.io.BufferedInputStream; diff --git a/sdks/java/io/thrift/src/test/java/org/apache/beam/sdk/io/thrift/ThriftIOTest.java b/sdks/java/io/thrift/src/test/java/org/apache/beam/sdk/io/thrift/ThriftIOTest.java index 0b01fb3967c44..ca2fdbef78570 100644 --- a/sdks/java/io/thrift/src/test/java/org/apache/beam/sdk/io/thrift/ThriftIOTest.java +++ b/sdks/java/io/thrift/src/test/java/org/apache/beam/sdk/io/thrift/ThriftIOTest.java @@ -30,8 +30,8 @@ import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Resources; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Resources; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TCompactProtocol; import org.apache.thrift.protocol.TJSONProtocol; diff --git a/sdks/java/io/thrift/src/test/java/org/apache/beam/sdk/io/thrift/ThriftPayloadSerializerProviderTest.java b/sdks/java/io/thrift/src/test/java/org/apache/beam/sdk/io/thrift/ThriftPayloadSerializerProviderTest.java index c19f15cbaea3c..7ba881e8942e4 100644 --- a/sdks/java/io/thrift/src/test/java/org/apache/beam/sdk/io/thrift/ThriftPayloadSerializerProviderTest.java +++ b/sdks/java/io/thrift/src/test/java/org/apache/beam/sdk/io/thrift/ThriftPayloadSerializerProviderTest.java @@ -24,8 +24,8 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.io.payloads.PayloadSerializerProvider; import org.apache.beam.sdk.values.Row; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.thrift.TDeserializer; import org.apache.thrift.TSerializer; import org.apache.thrift.protocol.TCompactProtocol; diff --git a/sdks/java/io/tika/build.gradle b/sdks/java/io/tika/build.gradle index aa6d70cdaa56f..b1c51952c830e 100644 --- a/sdks/java/io/tika/build.gradle +++ b/sdks/java/io/tika/build.gradle @@ -26,7 +26,7 @@ def tika_version = "1.26" def bndlib_version = "1.50.0" dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation "org.apache.tika:tika-core:$tika_version" testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") diff --git a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java index d71a5bd9a3639..953870fa9b29b 100644 --- a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java +++ b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java @@ -17,15 +17,15 @@ */ package org.apache.beam.sdk.io.tika; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.io.Serializable; import java.util.Arrays; import java.util.Objects; import org.apache.beam.sdk.util.SerializableThrowable; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; import org.apache.tika.metadata.Metadata; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java index 3acf9fdc59fb7..c22959407a722 100644 --- a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java +++ b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java @@ -17,8 +17,8 @@ */ package org.apache.beam.sdk.io.tika; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; import java.io.InputStream; diff --git a/sdks/java/io/xml/build.gradle b/sdks/java/io/xml/build.gradle index c165f248eb4a4..7f3b3ddcdfae0 100644 --- a/sdks/java/io/xml/build.gradle +++ b/sdks/java/io/xml/build.gradle @@ -26,7 +26,7 @@ dependencies { implementation library.java.jaxb_api implementation library.java.jaxb_impl permitUnusedDeclared library.java.jaxb_impl // BEAM-11761 - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.stax2_api implementation "javax.xml.stream:stax-api:1.0-2" diff --git a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/JAXBCoder.java b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/JAXBCoder.java index 65d774deeb1dd..d45030d948fab 100644 --- a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/JAXBCoder.java +++ b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/JAXBCoder.java @@ -33,7 +33,7 @@ import org.apache.beam.sdk.util.EmptyOnDeserializationThreadLocal; import org.apache.beam.sdk.util.VarInt; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java index c44eff677e2fd..ac2662a227aec 100644 --- a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java +++ b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.io.xml; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.google.auto.value.AutoValue; import java.io.IOException; @@ -49,7 +49,7 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; /** Transforms for reading and writing XML files using JAXB mappers. */ diff --git a/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/JAXBCoderTest.java b/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/JAXBCoderTest.java index d3444094cdf3b..b414f28aad790 100644 --- a/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/JAXBCoderTest.java +++ b/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/JAXBCoderTest.java @@ -38,7 +38,7 @@ import org.apache.beam.sdk.util.CoderUtils; import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/XmlIOTest.java b/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/XmlIOTest.java index e0a67543b7cf2..f6eb1e9a46f54 100644 --- a/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/XmlIOTest.java +++ b/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/XmlIOTest.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.transforms.Values; import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/XmlSourceTest.java b/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/XmlSourceTest.java index 98af8306bc4d6..91b29a7cd745c 100644 --- a/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/XmlSourceTest.java +++ b/sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/XmlSourceTest.java @@ -43,7 +43,7 @@ import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.Ignore; import org.junit.Rule; diff --git a/sdks/java/javadoc/OWNERS b/sdks/java/javadoc/OWNERS deleted file mode 100644 index f5a89b0d3ff2c..0000000000000 --- a/sdks/java/javadoc/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - melap diff --git a/sdks/java/maven-archetypes/examples/build.gradle b/sdks/java/maven-archetypes/examples/build.gradle index 56b4a7c842850..1edb55a10f956 100644 --- a/sdks/java/maven-archetypes/examples/build.gradle +++ b/sdks/java/maven-archetypes/examples/build.gradle @@ -72,6 +72,13 @@ task generateSources(type: Exec) { commandLine './generate-sources.sh' } +// add dependency BeamModulePlugin defined custom tasks +// they are defined only when certain flags are provided (e.g. -Prelease; -Ppublishing, etc) +def sourcesJar = project.tasks.findByName('sourcesJar') +if (sourcesJar != null) { + sourcesJar.dependsOn generateSources +} + sourceSets { main { output.dir('src', builtBy: 'generateSources') diff --git a/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle b/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle index 541c91bd6adb0..f9fabcfe19b08 100644 --- a/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle +++ b/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle @@ -71,6 +71,12 @@ task generateSources(type: Exec) { environment "HERE", "." commandLine '../examples/generate-sources.sh' } +// add dependency BeamModulePlugin defined custom tasks +// they are defined only when certain flags are provided (e.g. -Prelease; -Ppublishing, etc) +def sourcesJar = project.tasks.findByName('sourcesJar') +if (sourcesJar != null) { + sourcesJar.dependsOn generateSources +} sourceSets { main { diff --git a/sdks/java/testing/OWNERS b/sdks/java/testing/OWNERS deleted file mode 100644 index afa0fc252afe8..0000000000000 --- a/sdks/java/testing/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - echauchot - - aromanenko-dev - - lgajowy diff --git a/sdks/java/testing/expansion-service/src/test/java/org/apache/beam/sdk/testing/expansion/TestExpansionService.java b/sdks/java/testing/expansion-service/src/test/java/org/apache/beam/sdk/testing/expansion/TestExpansionService.java index 821d0e9db9ae8..b45c922eae7d1 100644 --- a/sdks/java/testing/expansion-service/src/test/java/org/apache/beam/sdk/testing/expansion/TestExpansionService.java +++ b/sdks/java/testing/expansion-service/src/test/java/org/apache/beam/sdk/testing/expansion/TestExpansionService.java @@ -56,8 +56,8 @@ import org.apache.beam.sdk.values.PCollectionView; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; /** * An {@link org.apache.beam.runners.core.construction.expansion.ExpansionService} useful for tests. diff --git a/sdks/java/testing/jpms-tests/build.gradle b/sdks/java/testing/jpms-tests/build.gradle index f781c29b8480a..6321f874c9036 100644 --- a/sdks/java/testing/jpms-tests/build.gradle +++ b/sdks/java/testing/jpms-tests/build.gradle @@ -21,7 +21,14 @@ import groovy.json.JsonOutput plugins { id 'org.apache.beam.module' } -javaVersion="1.11" + +// overwrite javaVersion before applyJavaNature +if (project.hasProperty("compileAndRunTestsWithJava17")) { + javaVersion = '1.17' +} else { + javaVersion = '1.11' +} + applyJavaNature( exportJavadoc: false, publish: false, @@ -33,30 +40,15 @@ enableJavaPerformanceTesting() description = "Apache Beam :: SDKs :: Java :: Testing :: JPMS Tests" ext.summary = "E2E test for Java 9 modules" -// Java 17 needs compileJava to add-exports and add-opens for error prone -if (project.hasProperty("compileAndRunTestsWithJava17")) { - def java17Home = project.findProperty("java17Home") - project.tasks.withType(JavaCompile) { +// direct compileJava to use specified java version. +project.tasks.compileJava { + if (project.hasProperty("compileAndRunTestsWithJava11")) { options.fork = true - options.forkOptions.javaHome = java17Home as File - options.compilerArgs += ['-Xlint:-path'] - options.compilerArgs.addAll(['--release', '17']) - // Error prone requires some packages to be exported/opened for Java 17 - // Disabling checks since this property is only used for Jenkins tests - // https://github.com/tbroyer/gradle-errorprone-plugin#jdk-16-support - options.errorprone.errorproneArgs.add("-XepDisableAllChecks") - options.forkOptions.jvmArgs += [ - "-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED", - "-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED", - "-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED" - ] + options.forkOptions.javaHome = project.findProperty("java11Home") as File + } else if (project.hasProperty("compileAndRunTestsWithJava17")) { + options.fork = true + options.forkOptions.javaHome = project.findProperty("java17Home") as File + setJava17Options(options) } } diff --git a/sdks/java/testing/load-tests/OWNERS b/sdks/java/testing/load-tests/OWNERS deleted file mode 100644 index 2a140107225d9..0000000000000 --- a/sdks/java/testing/load-tests/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lgajowy - - kkucharc diff --git a/sdks/java/testing/load-tests/build.gradle b/sdks/java/testing/load-tests/build.gradle index b957186648675..d1439bafb7488 100644 --- a/sdks/java/testing/load-tests/build.gradle +++ b/sdks/java/testing/load-tests/build.gradle @@ -77,7 +77,7 @@ dependencies { implementation library.java.aws_java_sdk_core implementation library.java.google_cloud_core implementation library.java.joda_time - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.slf4j_api gradleRun project(project.path) diff --git a/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/CombineLoadTest.java b/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/CombineLoadTest.java index 106be8186ca80..c7b0f2e9e5100 100644 --- a/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/CombineLoadTest.java +++ b/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/CombineLoadTest.java @@ -38,7 +38,7 @@ import org.apache.beam.sdk.transforms.Top; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; /** * Load test for {@link ParDo} operation. diff --git a/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/LoadTestResult.java b/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/LoadTestResult.java index 115089caa8ef2..3def60e0e52c0 100644 --- a/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/LoadTestResult.java +++ b/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/LoadTestResult.java @@ -21,7 +21,7 @@ import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.testutils.TestResult; import org.apache.beam.sdk.testutils.metrics.MetricsReader; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; /** POJO that represents load test results. */ public class LoadTestResult implements TestResult { diff --git a/sdks/java/testing/nexmark/OWNERS b/sdks/java/testing/nexmark/OWNERS deleted file mode 100644 index 8725d56ab61ba..0000000000000 --- a/sdks/java/testing/nexmark/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - echauchot - - aromanenko-dev diff --git a/sdks/java/testing/nexmark/build.gradle b/sdks/java/testing/nexmark/build.gradle index 8b989c258ed79..942dfb4c1e9c5 100644 --- a/sdks/java/testing/nexmark/build.gradle +++ b/sdks/java/testing/nexmark/build.gradle @@ -64,7 +64,7 @@ configurations { } dependencies { - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(":sdks:java:io:google-cloud-platform") implementation project(":sdks:java:extensions:avro") diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/Main.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/Main.java index 95554c0c80995..fdc6fe28f1d8a 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/Main.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/Main.java @@ -44,7 +44,7 @@ import org.apache.beam.sdk.testutils.publishing.InfluxDBPublisher; import org.apache.beam.sdk.testutils.publishing.InfluxDBPublisher.DataPoint; import org.apache.beam.sdk.testutils.publishing.InfluxDBSettings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java index fd85636308322..e94b601c685ca 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java @@ -19,8 +19,8 @@ import static org.apache.beam.sdk.nexmark.NexmarkQueryName.PORTABILITY_BATCH; import static org.apache.beam.sdk.nexmark.NexmarkUtils.PubSubMode.COMBINED; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.api.services.bigquery.model.TableFieldSchema; import com.google.api.services.bigquery.model.TableRow; @@ -101,12 +101,12 @@ import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.sdk.values.TypeDescriptors; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.serialization.ByteArraySerializer; diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkUtils.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkUtils.java index 3d8985df3fabe..2a536c4c6c7c0 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkUtils.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkUtils.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.nexmark; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.joda.JodaModule; @@ -80,10 +80,10 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; import org.joda.time.Duration; import org.joda.time.Instant; import org.slf4j.Logger; diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Auction.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Auction.java index 39bbfc9f9aa67..6c28350ec7714 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Auction.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Auction.java @@ -32,7 +32,7 @@ import org.apache.beam.sdk.nexmark.NexmarkUtils; import org.apache.beam.sdk.schemas.JavaFieldSchema; import org.apache.beam.sdk.schemas.annotations.DefaultSchema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/AuctionBid.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/AuctionBid.java index 5159278bb5725..720f35156b6cc 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/AuctionBid.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/AuctionBid.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.coders.CustomCoder; import org.apache.beam.sdk.nexmark.NexmarkUtils; import org.apache.beam.sdk.nexmark.queries.WinningBids; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.Nullable; /** Result of {@link WinningBids} transform. */ diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/BidsPerSession.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/BidsPerSession.java index e53fcb962bf69..7c55b126fa77d 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/BidsPerSession.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/BidsPerSession.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.coders.CustomCoder; import org.apache.beam.sdk.coders.VarLongCoder; import org.apache.beam.sdk.nexmark.NexmarkUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.Nullable; /** Result of query 11. */ diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/CategoryPrice.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/CategoryPrice.java index 745bec92bb781..0fcd8fc287649 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/CategoryPrice.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/CategoryPrice.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.coders.VarIntCoder; import org.apache.beam.sdk.coders.VarLongCoder; import org.apache.beam.sdk.nexmark.NexmarkUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.Nullable; /** Result of Query4. */ diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Done.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Done.java index add43e542f5f6..a07eea62fcb77 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Done.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Done.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.coders.CustomCoder; import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.nexmark.NexmarkUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.Nullable; /** Result of query 10. */ diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Event.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Event.java index d9c9a37056e15..067727fc1e36c 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Event.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Event.java @@ -26,7 +26,7 @@ import org.apache.beam.sdk.coders.VarIntCoder; import org.apache.beam.sdk.schemas.JavaFieldSchema; import org.apache.beam.sdk.schemas.annotations.DefaultSchema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.Nullable; /** diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/IdNameReserve.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/IdNameReserve.java index cd35491b245da..8efcf5b4645a8 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/IdNameReserve.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/IdNameReserve.java @@ -29,7 +29,7 @@ import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.coders.VarLongCoder; import org.apache.beam.sdk.nexmark.NexmarkUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.Nullable; /** Result type of Query8. */ diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Person.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Person.java index 49be3c1804d7b..80ca1fbc1a40c 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Person.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/Person.java @@ -32,7 +32,7 @@ import org.apache.beam.sdk.nexmark.NexmarkUtils; import org.apache.beam.sdk.schemas.JavaFieldSchema; import org.apache.beam.sdk.schemas.annotations.DefaultSchema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Instant; diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/SellerPrice.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/SellerPrice.java index 4cab647919d17..85c88abc826ff 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/SellerPrice.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/SellerPrice.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.coders.CustomCoder; import org.apache.beam.sdk.coders.VarLongCoder; import org.apache.beam.sdk.nexmark.NexmarkUtils; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Objects; import org.checkerframework.checker.nullness.qual.Nullable; /** Result of Query6. */ diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/BoundedSideInputJoin.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/BoundedSideInputJoin.java index b0c33257e32a8..09eb25390d266 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/BoundedSideInputJoin.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/BoundedSideInputJoin.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.nexmark.queries; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Map; import org.apache.beam.sdk.nexmark.NexmarkConfiguration; diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query3Model.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query3Model.java index 633954ef062b5..8146ef4519623 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query3Model.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query3Model.java @@ -29,8 +29,8 @@ import org.apache.beam.sdk.nexmark.model.NameCityStateId; import org.apache.beam.sdk.nexmark.model.Person; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; import org.joda.time.Instant; /** A direct implementation of {@link Query3}. */ diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query5.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query5.java index 77d64ab0b07be..3e40f43d0a612 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query5.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query5.java @@ -38,7 +38,7 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query6.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query6.java index 5570563f8e2e2..9443e31bfb951 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query6.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query6.java @@ -35,7 +35,7 @@ import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.joda.time.Duration; /** diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query8Model.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query8Model.java index f7b294eccb690..800e7af17fcea 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query8Model.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query8Model.java @@ -29,8 +29,8 @@ import org.apache.beam.sdk.nexmark.model.IdNameReserve; import org.apache.beam.sdk.nexmark.model.Person; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ArrayListMultimap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Multimap; import org.joda.time.Duration; import org.joda.time.Instant; diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/SessionSideInputJoin.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/SessionSideInputJoin.java index 906c715e5a6e5..9958a1efd23f5 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/SessionSideInputJoin.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/SessionSideInputJoin.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.nexmark.queries; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.Map; import org.apache.beam.sdk.nexmark.NexmarkConfiguration; diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/SessionSideInputJoinModel.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/SessionSideInputJoinModel.java index 1bdbf08c0ad1e..007a728ac0247 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/SessionSideInputJoinModel.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/SessionSideInputJoinModel.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.nexmark.queries; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import java.util.ArrayList; import java.util.Collection; @@ -31,8 +31,8 @@ import org.apache.beam.sdk.nexmark.model.Bid; import org.apache.beam.sdk.nexmark.model.Event; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Ordering; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Ordering; import org.joda.time.Instant; /** A direct implementation of {@link SessionSideInputJoin}. */ diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/WinningBids.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/WinningBids.java index c8e51815b8ba5..acb0383663c00 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/WinningBids.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/WinningBids.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.nexmark.queries; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.fasterxml.jackson.annotation.JsonCreator; import java.io.IOException; diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/sql/SqlBoundedSideInputJoin.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/sql/SqlBoundedSideInputJoin.java index 78583148cbc7e..8910075ecf893 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/sql/SqlBoundedSideInputJoin.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/sql/SqlBoundedSideInputJoin.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.nexmark.queries.sql; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import org.apache.beam.sdk.extensions.sql.SqlTransform; import org.apache.beam.sdk.extensions.sql.impl.CalciteQueryPlanner; diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery5.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery5.java index a99c01530c101..78051b5925f71 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery5.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery5.java @@ -32,7 +32,7 @@ import org.apache.beam.sdk.values.PInput; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Joiner; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner; /** * Query 5, 'Hot Items'. Which auctions have seen the most bids in the last hour (updated every diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/generator/Generator.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/generator/Generator.java index 7b51536c464e3..4174b587eea1f 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/generator/Generator.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/generator/Generator.java @@ -20,7 +20,7 @@ import static org.apache.beam.sdk.nexmark.sources.generator.model.AuctionGenerator.nextAuction; import static org.apache.beam.sdk.nexmark.sources.generator.model.BidGenerator.nextBid; import static org.apache.beam.sdk.nexmark.sources.generator.model.PersonGenerator.nextPerson; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.io.Serializable; import java.util.Iterator; diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/generator/GeneratorCheckpoint.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/generator/GeneratorCheckpoint.java index 18286b1b8499f..5cd30c4ab4ce1 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/generator/GeneratorCheckpoint.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/generator/GeneratorCheckpoint.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.nexmark.sources.generator; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects.toStringHelper; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.toStringHelper; import java.io.IOException; import java.io.InputStream; diff --git a/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/BoundedSideInputJoinTest.java b/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/BoundedSideInputJoinTest.java index 37872344b65aa..9079c5497c224 100644 --- a/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/BoundedSideInputJoinTest.java +++ b/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/BoundedSideInputJoinTest.java @@ -39,7 +39,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/SessionSideInputJoinTest.java b/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/SessionSideInputJoinTest.java index ae7beb8d94ec6..1659cccd73ecf 100644 --- a/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/SessionSideInputJoinTest.java +++ b/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/SessionSideInputJoinTest.java @@ -41,7 +41,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlBoundedSideInputJoinTest.java b/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlBoundedSideInputJoinTest.java index b005c324ef8ee..8570ba29619eb 100644 --- a/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlBoundedSideInputJoinTest.java +++ b/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlBoundedSideInputJoinTest.java @@ -43,7 +43,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionList; import org.apache.beam.sdk.values.TimestampedValue; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery2Test.java b/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery2Test.java index b7615d73a8ba6..bed58c9ce78f6 100644 --- a/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery2Test.java +++ b/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery2Test.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery3Test.java b/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery3Test.java index 984f50e62db7f..6150c8701fdf4 100644 --- a/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery3Test.java +++ b/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery3Test.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery5Test.java b/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery5Test.java index 1c5d230186e07..e040525613f81 100644 --- a/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery5Test.java +++ b/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery5Test.java @@ -28,7 +28,7 @@ import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; import org.junit.Ignore; import org.junit.Rule; diff --git a/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery7Test.java b/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery7Test.java index 8aa4256cfe7a4..b1a3924897198 100644 --- a/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery7Test.java +++ b/sdks/java/testing/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery7Test.java @@ -25,7 +25,7 @@ import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.joda.time.Instant; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/testing/test-utils/OWNERS b/sdks/java/testing/test-utils/OWNERS deleted file mode 100644 index 2a140107225d9..0000000000000 --- a/sdks/java/testing/test-utils/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - lgajowy - - kkucharc diff --git a/sdks/java/testing/test-utils/build.gradle b/sdks/java/testing/test-utils/build.gradle index 69f47b2a4aa1c..50c815dd57f7f 100644 --- a/sdks/java/testing/test-utils/build.gradle +++ b/sdks/java/testing/test-utils/build.gradle @@ -28,7 +28,7 @@ description = "Apache Beam :: SDKs :: Java :: Test Utils" dependencies { implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) implementation project(path: ":sdks:java:core", configuration: "shadow") - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.google_cloud_bigquery implementation library.java.google_code_gson implementation library.java.joda_time diff --git a/sdks/java/testing/test-utils/src/main/java/org/apache/beam/sdk/testutils/NamedTestResult.java b/sdks/java/testing/test-utils/src/main/java/org/apache/beam/sdk/testutils/NamedTestResult.java index 72200ab54f9f8..a4f40598d5a38 100644 --- a/sdks/java/testing/test-utils/src/main/java/org/apache/beam/sdk/testutils/NamedTestResult.java +++ b/sdks/java/testing/test-utils/src/main/java/org/apache/beam/sdk/testutils/NamedTestResult.java @@ -20,7 +20,7 @@ import com.google.cloud.bigquery.LegacySQLTypeName; import java.util.Map; import org.apache.beam.sdk.testutils.publishing.InfluxDBPublisher; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/sdks/java/testing/test-utils/src/main/java/org/apache/beam/sdk/testutils/metrics/MetricsReader.java b/sdks/java/testing/test-utils/src/main/java/org/apache/beam/sdk/testutils/metrics/MetricsReader.java index fe98e6400ff46..e1b8812d85550 100644 --- a/sdks/java/testing/test-utils/src/main/java/org/apache/beam/sdk/testutils/metrics/MetricsReader.java +++ b/sdks/java/testing/test-utils/src/main/java/org/apache/beam/sdk/testutils/metrics/MetricsReader.java @@ -17,7 +17,7 @@ */ package org.apache.beam.sdk.testutils.metrics; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; import java.util.Collection; import java.util.NoSuchElementException; @@ -33,9 +33,9 @@ import org.apache.beam.sdk.metrics.MetricResult; import org.apache.beam.sdk.metrics.MetricsFilter; import org.apache.beam.sdk.testutils.NamedTestResult; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; import org.joda.time.Duration; import org.slf4j.LoggerFactory; diff --git a/sdks/java/testing/test-utils/src/main/java/org/apache/beam/sdk/testutils/publishing/InfluxDBPublisher.java b/sdks/java/testing/test-utils/src/main/java/org/apache/beam/sdk/testutils/publishing/InfluxDBPublisher.java index 913c020af617a..30e72fd53dad3 100644 --- a/sdks/java/testing/test-utils/src/main/java/org/apache/beam/sdk/testutils/publishing/InfluxDBPublisher.java +++ b/sdks/java/testing/test-utils/src/main/java/org/apache/beam/sdk/testutils/publishing/InfluxDBPublisher.java @@ -22,7 +22,7 @@ import static java.util.Objects.requireNonNull; import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNoneBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; @@ -40,11 +40,11 @@ import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; import org.apache.beam.sdk.testutils.NamedTestResult; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Collections2; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Collections2; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; import org.apache.commons.compress.utils.Charsets; import org.apache.http.Header; import org.apache.http.HttpEntity; diff --git a/sdks/java/testing/test-utils/src/test/java/org/apache/beam/sdk/testutils/publishing/InfluxDBPublisherTest.java b/sdks/java/testing/test-utils/src/test/java/org/apache/beam/sdk/testutils/publishing/InfluxDBPublisherTest.java index b796af6a10630..3d410f587f944 100644 --- a/sdks/java/testing/test-utils/src/test/java/org/apache/beam/sdk/testutils/publishing/InfluxDBPublisherTest.java +++ b/sdks/java/testing/test-utils/src/test/java/org/apache/beam/sdk/testutils/publishing/InfluxDBPublisherTest.java @@ -22,8 +22,8 @@ import java.util.List; import java.util.Map; import org.apache.beam.sdk.testutils.NamedTestResult; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Test; public final class InfluxDBPublisherTest { diff --git a/sdks/java/testing/tpcds/build.gradle b/sdks/java/testing/tpcds/build.gradle index 7628d5e5ca44c..387dda8982476 100644 --- a/sdks/java/testing/tpcds/build.gradle +++ b/sdks/java/testing/tpcds/build.gradle @@ -61,12 +61,12 @@ dependencies { permitUnusedDeclared "org.immutables:value:2.8.8" implementation library.java.avro implementation library.java.joda_time - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.vendored_calcite_1_28_0 implementation library.java.commons_csv implementation library.java.slf4j_api implementation "com.googlecode.json-simple:json-simple:1.1.1" - implementation "com.alibaba:fastjson:1.2.69" + implementation library.java.jackson_databind implementation project(":sdks:java:extensions:sql") implementation project(":sdks:java:extensions:sql:zetasql") implementation project(":sdks:java:io:parquet") @@ -126,6 +126,7 @@ task run(type: JavaExec) { tpcdsArgsList.add("--sparkMaster=local[4]") // Dataset runner only systemProperty "spark.sql.shuffle.partitions", "4" + systemProperty "spark.sql.adaptive.enabled", "false" // high overhead for complex queries } mainClass = "org.apache.beam.sdk.tpcds.BeamTpcds" diff --git a/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/BeamSqlEnvRunner.java b/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/BeamSqlEnvRunner.java index 2a238cb1e9a56..4beab4bca578f 100644 --- a/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/BeamSqlEnvRunner.java +++ b/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/BeamSqlEnvRunner.java @@ -19,7 +19,7 @@ import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -30,6 +30,7 @@ import java.util.concurrent.Executors; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.extensions.sql.TableUtils; import org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv; import org.apache.beam.sdk.extensions.sql.impl.BeamSqlPipelineOptions; import org.apache.beam.sdk.extensions.sql.impl.rel.BeamSqlRelUtils; @@ -99,7 +100,7 @@ private static void registerAllTablesByBeamSqlEnv(BeamSqlEnv env, String dataSiz */ private static void registerAllTablesByInMemoryMetaStore( InMemoryMetaStore inMemoryMetaStore, String dataSize) throws Exception { - JSONObject properties = new JSONObject(); + ObjectNode properties = TableUtils.emptyProperties(); properties.put("csvformat", "InformixUnload"); Map schemaMap = TpcdsSchemas.getTpcdsSchemas(); diff --git a/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/QueryReader.java b/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/QueryReader.java index 23d98a3391ba6..4983d52a642f5 100644 --- a/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/QueryReader.java +++ b/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/QueryReader.java @@ -21,8 +21,8 @@ import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlNode; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.parser.SqlParseException; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.parser.SqlParser; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Resources; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Resources; /** * The QueryReader reads query file (the file's extension is '.sql' and content doesn't end with a diff --git a/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/SqlTransformRunner.java b/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/SqlTransformRunner.java index f8287fc3c99dd..6570b7fe81b2d 100644 --- a/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/SqlTransformRunner.java +++ b/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/SqlTransformRunner.java @@ -52,9 +52,9 @@ import org.apache.beam.sdk.values.TypeDescriptors; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlIdentifier; import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.util.SqlBasicVisitor; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Resources; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Resources; import org.apache.commons.csv.CSVFormat; import org.joda.time.Instant; import org.slf4j.Logger; @@ -283,7 +283,8 @@ public static void runUsingSqlTransform(String[] args) throws Exception { // Make an array of pipelines, each pipeline is responsible for running a corresponding query. Pipeline[] pipelines = new Pipeline[queryNames.length]; - CSVFormat csvFormat = CSVFormat.MYSQL.withDelimiter('|').withNullString(""); + CSVFormat csvFormat = + CSVFormat.MYSQL.withDelimiter('|').withTrailingDelimiter().withNullString(""); // Execute all queries, transform each result into a PCollection, write them into // the txt file and store in a GCP directory. diff --git a/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TableSchemaJSONLoader.java b/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TableSchemaJSONLoader.java index b83e0f92b1bd6..c408919a53bd8 100644 --- a/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TableSchemaJSONLoader.java +++ b/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TableSchemaJSONLoader.java @@ -22,9 +22,9 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Charsets; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Resources; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.reflect.ClassPath; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Resources; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.reflect.ClassPath; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; diff --git a/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsOptionsRegistrar.java b/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsOptionsRegistrar.java index fc596677d2681..d504ac1175067 100644 --- a/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsOptionsRegistrar.java +++ b/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsOptionsRegistrar.java @@ -20,7 +20,7 @@ import com.google.auto.service.AutoService; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsRegistrar; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; /** {@link AutoService} registrar for {@link TpcdsOptions}. */ @AutoService(PipelineOptionsRegistrar.class) diff --git a/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsSchemas.java b/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsSchemas.java index 6dc00af1f169d..a22bab1c89e63 100644 --- a/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsSchemas.java +++ b/sdks/java/testing/tpcds/src/main/java/org/apache/beam/sdk/tpcds/TpcdsSchemas.java @@ -21,7 +21,7 @@ import java.util.List; import java.util.Map; import org.apache.beam.sdk.schemas.Schema; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; public class TpcdsSchemas { /** diff --git a/sdks/java/transform-service/build.gradle b/sdks/java/transform-service/build.gradle index 94c1d85f27d5b..130a6b1d7091f 100644 --- a/sdks/java/transform-service/build.gradle +++ b/sdks/java/transform-service/build.gradle @@ -44,7 +44,7 @@ dependencies { implementation project(path: ":runners:core-construction-java") implementation project(path: ":sdks:java:fn-execution") implementation library.java.vendored_grpc_1_54_0 - implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_guava_32_1_2_jre implementation library.java.jackson_annotations implementation library.java.jackson_databind implementation library.java.jackson_dataformat_yaml diff --git a/sdks/java/transform-service/controller-container/build.gradle b/sdks/java/transform-service/controller-container/build.gradle index 75608c597ae60..db5790f7ca668 100644 --- a/sdks/java/transform-service/controller-container/build.gradle +++ b/sdks/java/transform-service/controller-container/build.gradle @@ -49,6 +49,9 @@ task copyConfigFile(type: Copy) { into "build/target" } +def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] +def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers") + docker { name containerImageName( name: project.docker_image_default_repo_prefix + "transform_service_controller", @@ -60,8 +63,10 @@ docker { // tags used by dockerTag task tags containerImageTags() files "./build" - buildx project.containerPlatforms() != [project.nativeArchitecture()] + buildx useBuildx platform(*project.containerPlatforms()) + load useBuildx && !pushContainers + push pushContainers } dockerPrepare.dependsOn goBuild @@ -85,5 +90,5 @@ if (project.rootProject.hasProperty(["docker-pull-licenses"])) { } task pushAll { - dependsOn dockerPush + dependsOn docker } diff --git a/sdks/java/transform-service/docker-compose/.env b/sdks/java/transform-service/docker-compose/.env index 5de5982cfa301..ed27b267fed37 100644 --- a/sdks/java/transform-service/docker-compose/.env +++ b/sdks/java/transform-service/docker-compose/.env @@ -12,6 +12,14 @@ BEAM_VERSION=$BEAM_VERSION CREDENTIALS_VOLUME=$CREDENTIALS_VOLUME +DEPENDENCIES_VOLUME=$DEPENDENCIES_VOLUME + +# A requirements file with either of the following +# * PyPi packages +# * Locally available packages relative to the directory provided to +# DEPENDENCIES_VOLUME. +PYTHON_REQUIREMENTS_FILE_NAME=$PYTHON_REQUIREMENTS_FILE_NAME + GOOGLE_APPLICATION_CREDENTIALS_FILE_NAME=application_default_credentials.json COMPOSE_PROJECT_NAME=apache.beam.transform.service TRANSFORM_SERVICE_PORT=$TRANSFORM_SERVICE_PORT diff --git a/sdks/java/transform-service/docker-compose/docker-compose.yml b/sdks/java/transform-service/docker-compose/docker-compose.yml index c0a28f6ae07a1..39235533b9a86 100644 --- a/sdks/java/transform-service/docker-compose/docker-compose.yml +++ b/sdks/java/transform-service/docker-compose/docker-compose.yml @@ -18,6 +18,9 @@ services: command: -port 5001 ports: - "${TRANSFORM_SERVICE_PORT}:5001" + depends_on: + - expansion-service-1 + - expansion-service-2 expansion-service-1: image: "apache/beam_java_expansion_service:${BEAM_VERSION}" restart: on-failure @@ -29,8 +32,9 @@ services: expansion-service-2: image: "apache/beam_python_expansion_service:${BEAM_VERSION}" restart: on-failure - command: -id expansion-service-2 -port 5001 + command: -id expansion-service-2 -port 5001 -requirements_file ${PYTHON_REQUIREMENTS_FILE_NAME} -dependencies_dir '/dependencies_volume' volumes: - ${CREDENTIALS_VOLUME}:/credentials_volume + - ${DEPENDENCIES_VOLUME}:/dependencies_volume environment: - GOOGLE_APPLICATION_CREDENTIALS=/credentials_volume/${GOOGLE_APPLICATION_CREDENTIALS_FILE_NAME} diff --git a/sdks/java/transform-service/launcher/build.gradle b/sdks/java/transform-service/launcher/build.gradle index f57cbaae9c0ee..0952f37109eb9 100644 --- a/sdks/java/transform-service/launcher/build.gradle +++ b/sdks/java/transform-service/launcher/build.gradle @@ -40,9 +40,14 @@ test { } dependencies { - shadow library.java.vendored_guava_26_0_jre + shadow library.java.vendored_guava_32_1_2_jre shadow library.java.slf4j_api shadow library.java.args4j + shadow library.java.error_prone_annotations + permitUnusedDeclared(library.java.error_prone_annotations) + testImplementation library.java.junit + testImplementation library.java.mockito_core + testImplementation project(path: ":sdks:java:core") } sourceSets { diff --git a/sdks/java/transform-service/launcher/src/main/java/org/apache/beam/sdk/transformservice/launcher/TransformServiceLauncher.java b/sdks/java/transform-service/launcher/src/main/java/org/apache/beam/sdk/transformservice/launcher/TransformServiceLauncher.java index bca6cc46bf949..c0a9097a762fa 100644 --- a/sdks/java/transform-service/launcher/src/main/java/org/apache/beam/sdk/transformservice/launcher/TransformServiceLauncher.java +++ b/sdks/java/transform-service/launcher/src/main/java/org/apache/beam/sdk/transformservice/launcher/TransformServiceLauncher.java @@ -17,9 +17,11 @@ */ package org.apache.beam.sdk.transformservice.launcher; +import java.io.BufferedWriter; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -28,8 +30,10 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeoutException; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files; +import java.util.stream.Stream; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.ByteStreams; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Files; import org.checkerframework.checker.nullness.qual.Nullable; import org.kohsuke.args4j.CmdLineException; import org.kohsuke.args4j.CmdLineParser; @@ -57,13 +61,13 @@ public class TransformServiceLauncher { private Map environmentVariables = new HashMap<>(); // Amount of time (in milliseconds) to wait till the Docker Compose starts up. - private static final int DEFAULT_START_WAIT_TIME = 25000; + private static final int DEFAULT_START_WAIT_TIME = 50000; private static final int STATUS_LOGGER_WAIT_TIME = 3000; @SuppressWarnings("argument") - private TransformServiceLauncher(@Nullable String projectName, int port) throws IOException { - LOG.info("Initializing the Beam Transform Service {}.", projectName); - + private TransformServiceLauncher( + @Nullable String projectName, int port, @Nullable String pythonRequirementsFile) + throws IOException { String tmpDirLocation = System.getProperty("java.io.tmpdir"); // We use Docker Compose project name as the name of the temporary directory to isolate // different transform service instances that may be running in the same machine. @@ -82,14 +86,14 @@ private TransformServiceLauncher(@Nullable String projectName, int port) throws ByteStreams.copy(getClass().getResourceAsStream("/.env"), fout); } + // Setting up the credentials directory. File credentialsDir = Paths.get(tmpDir, "credentials_dir").toFile(); - LOG.info( - "Creating a temporary directory for storing credentials: " - + credentialsDir.getAbsolutePath()); - if (credentialsDir.exists()) { LOG.info("Reusing the existing credentials directory " + credentialsDir.getAbsolutePath()); } else { + LOG.info( + "Creating a temporary directory for storing credentials: " + + credentialsDir.getAbsolutePath()); if (!credentialsDir.mkdir()) { throw new IOException( "Could not create a temporary directory for storing credentials: " @@ -115,16 +119,92 @@ private TransformServiceLauncher(@Nullable String projectName, int port) throws if (applicationDefaultCredentialsFile.exists()) { Files.copy(applicationDefaultCredentialsFile, applicationDefaultCredentialsFileCopied); } else { - throw new RuntimeException( - "Could not find the application default file: " - + applicationDefaultCredentialsFile.getAbsolutePath()); + LOG.error( + "GCP credentials will not be available for the transform service since the Google " + + "Cloud application default credentials file could not be found at the expected " + + "location {}.", + applicationDefaultFilePath); + } + } + + // Setting up the dependencies directory. + File dependenciesDir = Paths.get(tmpDir, "dependencies_dir").toFile(); + Path updatedRequirementsFilePath = Paths.get(dependenciesDir.toString(), "requirements.txt"); + if (dependenciesDir.exists()) { + LOG.info("Reusing the existing dependencies directory " + dependenciesDir.getAbsolutePath()); + } else { + LOG.info( + "Creating a temporary directory for storing dependencies: " + + dependenciesDir.getAbsolutePath()); + if (!dependenciesDir.mkdir()) { + throw new IOException( + "Could not create a temporary directory for storing dependencies: " + + dependenciesDir.getAbsolutePath()); + } + + // We create a requirements file with extra dependencies. + // If there are no extra dependencies, we just provide an empty requirements file. + File file = updatedRequirementsFilePath.toFile(); + if (!file.createNewFile()) { + throw new IOException( + "Could not create the new requirements file " + updatedRequirementsFilePath); + } + + // Updating dependencies. + if (pythonRequirementsFile != null) { + Path requirementsFilePath = Paths.get(pythonRequirementsFile); + List updatedLines = new ArrayList<>(); + + try (Stream lines = java.nio.file.Files.lines(requirementsFilePath)) { + lines.forEachOrdered( + line -> { + Path dependencyFilePath = Paths.get(line); + if (java.nio.file.Files.exists(dependencyFilePath)) { + Path fileName = dependencyFilePath.getFileName(); + if (fileName == null) { + throw new IllegalArgumentException( + "Could not determine the filename of the local artifact " + + dependencyFilePath); + } + try { + java.nio.file.Files.copy( + dependencyFilePath, + Paths.get(dependenciesDir.toString(), fileName.toString())); + } catch (IOException e) { + throw new RuntimeException(e); + } + updatedLines.add(fileName.toString()); + } else { + updatedLines.add(line); + } + }); + } + + try (BufferedWriter writer = + java.nio.file.Files.newBufferedWriter(file.toPath(), StandardCharsets.UTF_8)) { + for (String line : updatedLines) { + writer.write(line); + writer.newLine(); + } + writer.flush(); + } } } // Setting environment variables used by the docker-compose.yml file. environmentVariables.put("CREDENTIALS_VOLUME", credentialsDir.getAbsolutePath()); + environmentVariables.put("DEPENDENCIES_VOLUME", dependenciesDir.getAbsolutePath()); environmentVariables.put("TRANSFORM_SERVICE_PORT", String.valueOf(port)); + Path updatedRequirementsFileName = updatedRequirementsFilePath.getFileName(); + if (updatedRequirementsFileName == null) { + throw new IllegalArgumentException( + "Could not determine the file name of the updated requirements file " + + updatedRequirementsFilePath); + } + environmentVariables.put( + "PYTHON_REQUIREMENTS_FILE_NAME", updatedRequirementsFileName.toString()); + // Building the Docker Compose command. dockerComposeStartCommandPrefix.add("docker-compose"); dockerComposeStartCommandPrefix.add("-p"); @@ -133,34 +213,50 @@ private TransformServiceLauncher(@Nullable String projectName, int port) throws dockerComposeStartCommandPrefix.add(dockerComposeFile.getAbsolutePath()); } + /** + * Specifies the Beam version to get containers for the transform service. + * + *

    Could be a release Beam version with containers in Docker Hub or an unreleased Beam version + * for which containers are available locally. + * + * @param beamVersion a Beam version to get containers from. + */ public void setBeamVersion(String beamVersion) { environmentVariables.put("BEAM_VERSION", beamVersion); } - public void setPythonExtraPackages(String pythonExtraPackages) { - environmentVariables.put("$PYTHON_EXTRA_PACKAGES", pythonExtraPackages); - } - + /** + * Initializes a client for managing transform service instances. + * + * @param projectName project name for the transform service. + * @param port port exposed by the transform service. + * @param pythonRequirementsFile a requirements file with extra dependencies for the Python + * expansion services. + * @return an initialized client for managing the transform service. + * @throws IOException + */ public static synchronized TransformServiceLauncher forProject( - @Nullable String projectName, int port) throws IOException { + @Nullable String projectName, int port, @Nullable String pythonRequirementsFile) + throws IOException { if (projectName == null || projectName.isEmpty()) { projectName = DEFAULT_PROJECT_NAME; } if (!launchers.containsKey(projectName)) { - launchers.put(projectName, new TransformServiceLauncher(projectName, port)); + launchers.put( + projectName, new TransformServiceLauncher(projectName, port, pythonRequirementsFile)); } return launchers.get(projectName); } - private void runDockerComposeCommand(String command) throws IOException { + private void runDockerComposeCommand(List command) throws IOException { this.runDockerComposeCommand(command, null); } - private void runDockerComposeCommand(String command, @Nullable File outputOverride) + private void runDockerComposeCommand(List command, @Nullable File outputOverride) throws IOException { List shellCommand = new ArrayList<>(); shellCommand.addAll(dockerComposeStartCommandPrefix); - shellCommand.add(command); + shellCommand.addAll(command); System.out.println("Executing command: " + String.join(" ", command)); ProcessBuilder processBuilder = new ProcessBuilder(shellCommand).redirectError(ProcessBuilder.Redirect.INHERIT); @@ -184,23 +280,23 @@ private void runDockerComposeCommand(String command, @Nullable File outputOverri } public synchronized void start() throws IOException, TimeoutException { - runDockerComposeCommand("up"); + runDockerComposeCommand(ImmutableList.of("up", "-d")); } public synchronized void shutdown() throws IOException { - runDockerComposeCommand("down"); + runDockerComposeCommand(ImmutableList.of("down")); } public synchronized void status() throws IOException { - runDockerComposeCommand("ps"); + runDockerComposeCommand(ImmutableList.of("ps")); } public synchronized void waitTillUp(int timeout) throws IOException, TimeoutException { timeout = timeout <= 0 ? DEFAULT_START_WAIT_TIME : timeout; - String statusFileName = getStatus(); long startTime = System.currentTimeMillis(); while (System.currentTimeMillis() - startTime < timeout) { + String statusFileName = getStatus(); try { // We are just waiting for a local process. No need for exponential backoff. this.wait(1000); @@ -223,7 +319,8 @@ public synchronized void waitTillUp(int timeout) throws IOException, TimeoutExce private synchronized String getStatus() throws IOException { File outputOverride = File.createTempFile("output_override", null); - runDockerComposeCommand("ps", outputOverride); + outputOverride.deleteOnExit(); + runDockerComposeCommand(ImmutableList.of("ps"), outputOverride); return outputOverride.getAbsolutePath(); } @@ -235,6 +332,8 @@ private static class ArgConfig { static final String PORT_ARG_NAME = "port"; static final String BEAM_VERSION_ARG_NAME = "beam_version"; + static final String PYTHON_REQUIREMENTS_FILE_ARG_NAME = "python_requirements_file"; + @Option(name = "--" + PROJECT_NAME_ARG_NAME, usage = "Docker compose project name") private String projectName = ""; @@ -246,6 +345,11 @@ private static class ArgConfig { @Option(name = "--" + BEAM_VERSION_ARG_NAME, usage = "Beam version to use.") private String beamVersion = ""; + + @Option( + name = "--" + PYTHON_REQUIREMENTS_FILE_ARG_NAME, + usage = "Extra Python packages in the form of an requirements file.") + private String pythonRequirementsFile = ""; } public static void main(String[] args) throws IOException, TimeoutException { @@ -285,8 +389,12 @@ public static void main(String[] args) throws IOException, TimeoutException { : ("port " + Integer.toString(config.port) + "."))); System.out.println("==================================================="); + String pythonRequirementsFile = + !config.pythonRequirementsFile.isEmpty() ? config.pythonRequirementsFile : null; + TransformServiceLauncher service = - TransformServiceLauncher.forProject(config.projectName, config.port); + TransformServiceLauncher.forProject( + config.projectName, config.port, pythonRequirementsFile); if (!config.beamVersion.isEmpty()) { service.setBeamVersion(config.beamVersion); } diff --git a/sdks/java/transform-service/launcher/src/test/java/org/apache/beam/sdk/transformservice/launcher/TransformServiceLauncherTest.java b/sdks/java/transform-service/launcher/src/test/java/org/apache/beam/sdk/transformservice/launcher/TransformServiceLauncherTest.java new file mode 100644 index 0000000000000..4ef84b02061be --- /dev/null +++ b/sdks/java/transform-service/launcher/src/test/java/org/apache/beam/sdk/transformservice/launcher/TransformServiceLauncherTest.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.transformservice.launcher; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.UUID; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class TransformServiceLauncherTest { + + @Test + public void testLauncherCreatesCredentialsDir() throws IOException { + String projectName = UUID.randomUUID().toString(); + Path expectedTempDir = Paths.get(System.getProperty("java.io.tmpdir"), projectName); + File file = expectedTempDir.toFile(); + file.deleteOnExit(); + TransformServiceLauncher.forProject(projectName, 12345, null); + Path expectedCredentialsDir = Paths.get(expectedTempDir.toString(), "credentials_dir"); + assertTrue(expectedCredentialsDir.toFile().exists()); + } + + @Test + public void testLauncherCreatesDependenciesDir() throws IOException { + String projectName = UUID.randomUUID().toString(); + Path expectedTempDir = Paths.get(System.getProperty("java.io.tmpdir"), projectName); + File file = expectedTempDir.toFile(); + file.deleteOnExit(); + TransformServiceLauncher.forProject(projectName, 12345, null); + Path expectedCredentialsDir = Paths.get(expectedTempDir.toString(), "dependencies_dir"); + assertTrue(expectedCredentialsDir.toFile().exists()); + } + + @Test + public void testLauncherInstallsDependencies() throws IOException { + String projectName = UUID.randomUUID().toString(); + Path expectedTempDir = Paths.get(System.getProperty("java.io.tmpdir"), projectName); + File file = expectedTempDir.toFile(); + file.deleteOnExit(); + + File requirementsFile = + Paths.get( + System.getProperty("java.io.tmpdir"), + ("requirements" + UUID.randomUUID().toString() + ".txt")) + .toFile(); + requirementsFile.deleteOnExit(); + + try (Writer fout = + new OutputStreamWriter( + new FileOutputStream(requirementsFile.getAbsolutePath()), Charsets.UTF_8)) { + fout.write("pypipackage1\n"); + fout.write("pypipackage2\n"); + } + + TransformServiceLauncher.forProject(projectName, 12345, requirementsFile.getAbsolutePath()); + + // Confirming that the Transform Service launcher created a temporary requirements file with the + // specified set of packages. + Path expectedUpdatedRequirementsFile = + Paths.get(expectedTempDir.toString(), "dependencies_dir", "requirements.txt"); + assertTrue(expectedUpdatedRequirementsFile.toFile().exists()); + + ArrayList expectedUpdatedRequirementsFileLines = new ArrayList<>(); + try (BufferedReader bufReader = + Files.newBufferedReader(expectedUpdatedRequirementsFile, UTF_8)) { + String line = bufReader.readLine(); + while (line != null) { + expectedUpdatedRequirementsFileLines.add(line); + line = bufReader.readLine(); + } + } + + assertEquals(2, expectedUpdatedRequirementsFileLines.size()); + assertTrue(expectedUpdatedRequirementsFileLines.contains("pypipackage1")); + assertTrue(expectedUpdatedRequirementsFileLines.contains("pypipackage2")); + } + + @Test + public void testLauncherInstallsLocalDependencies() throws IOException { + String projectName = UUID.randomUUID().toString(); + Path expectedTempDir = Paths.get(System.getProperty("java.io.tmpdir"), projectName); + File file = expectedTempDir.toFile(); + file.deleteOnExit(); + + String dependency1FileName = "dep_" + UUID.randomUUID().toString(); + File dependency1 = + Paths.get(System.getProperty("java.io.tmpdir"), dependency1FileName).toFile(); + dependency1.deleteOnExit(); + try (Writer fout = + new OutputStreamWriter( + new FileOutputStream(dependency1.getAbsolutePath()), Charsets.UTF_8)) { + fout.write("tempdata\n"); + } + + String dependency2FileName = "dep_" + UUID.randomUUID().toString(); + File dependency2 = + Paths.get(System.getProperty("java.io.tmpdir"), dependency2FileName).toFile(); + dependency2.deleteOnExit(); + try (Writer fout = + new OutputStreamWriter( + new FileOutputStream(dependency2.getAbsolutePath()), Charsets.UTF_8)) { + fout.write("tempdata\n"); + } + + File requirementsFile = + Paths.get( + System.getProperty("java.io.tmpdir"), + ("requirements" + UUID.randomUUID().toString() + ".txt")) + .toFile(); + requirementsFile.deleteOnExit(); + try (Writer fout = + new OutputStreamWriter( + new FileOutputStream(requirementsFile.getAbsolutePath()), Charsets.UTF_8)) { + fout.write(dependency1.getAbsolutePath() + "\n"); + fout.write(dependency2.getAbsolutePath() + "\n"); + fout.write("pypipackage" + "\n"); + } + + TransformServiceLauncher.forProject(projectName, 12345, requirementsFile.getAbsolutePath()); + + // Confirming that the Transform Service launcher created a temporary requirements file with the + // specified set of packages. + Path expectedUpdatedRequirementsFile = + Paths.get(expectedTempDir.toString(), "dependencies_dir", "requirements.txt"); + assertTrue(expectedUpdatedRequirementsFile.toFile().exists()); + + ArrayList expectedUpdatedRequirementsFileLines = new ArrayList<>(); + try (BufferedReader bufReader = + Files.newBufferedReader(expectedUpdatedRequirementsFile, UTF_8)) { + String line = bufReader.readLine(); + while (line != null) { + expectedUpdatedRequirementsFileLines.add(line); + line = bufReader.readLine(); + } + } + + // To make local packages available to the expansion service Docker containers, the temporary + // requirements file should contain names of the local packages relative to the dependencies + // volume and local packages should have been copied to the dependencies volume. + assertEquals(3, expectedUpdatedRequirementsFileLines.size()); + assertTrue(expectedUpdatedRequirementsFileLines.contains(dependency1FileName)); + assertTrue(expectedUpdatedRequirementsFileLines.contains(dependency2FileName)); + assertTrue(expectedUpdatedRequirementsFileLines.contains("pypipackage")); + + assertTrue( + Paths.get(expectedTempDir.toString(), "dependencies_dir", dependency1FileName) + .toFile() + .exists()); + assertTrue( + Paths.get(expectedTempDir.toString(), "dependencies_dir", dependency2FileName) + .toFile() + .exists()); + } +} diff --git a/sdks/java/transform-service/src/main/java/org/apache/beam/sdk/transformservice/ExpansionService.java b/sdks/java/transform-service/src/main/java/org/apache/beam/sdk/transformservice/ExpansionService.java index 24ab73f0df819..0a2e65099e7db 100644 --- a/sdks/java/transform-service/src/main/java/org/apache/beam/sdk/transformservice/ExpansionService.java +++ b/sdks/java/transform-service/src/main/java/org/apache/beam/sdk/transformservice/ExpansionService.java @@ -17,16 +17,23 @@ */ package org.apache.beam.sdk.transformservice; +import java.io.IOException; +import java.net.Socket; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeoutException; import org.apache.beam.model.expansion.v1.ExpansionApi; +import org.apache.beam.model.expansion.v1.ExpansionApi.ExpansionResponse; import org.apache.beam.model.expansion.v1.ExpansionServiceGrpc; import org.apache.beam.model.pipeline.v1.Endpoints; import org.apache.beam.runners.core.construction.DefaultExpansionServiceClientFactory; import org.apache.beam.runners.core.construction.ExpansionServiceClientFactory; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ManagedChannelBuilder; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Throwables; import org.checkerframework.checker.nullness.qual.Nullable; public class ExpansionService extends ExpansionServiceGrpc.ExpansionServiceImplBase @@ -40,6 +47,12 @@ public class ExpansionService extends ExpansionServiceGrpc.ExpansionServiceImplB final List endpoints; + private boolean checkedAllServices = false; + + private static final long SERVICE_CHECK_TIMEOUT_MILLIS = 60000; + + private boolean disableServiceCheck = false; + ExpansionService( List endpoints, @Nullable ExpansionServiceClientFactory clientFactory) { @@ -48,10 +61,65 @@ public class ExpansionService extends ExpansionServiceGrpc.ExpansionServiceImplB clientFactory != null ? clientFactory : DEFAULT_EXPANSION_SERVICE_CLIENT_FACTORY; } + // Waits till all expansion services are ready. + private void waitForAllServicesToBeReady() throws TimeoutException { + if (disableServiceCheck) { + // Service check disabled. Just returning. + return; + } + + outer: + for (Endpoints.ApiServiceDescriptor endpoint : endpoints) { + long start = System.currentTimeMillis(); + long duration = 10; + while (System.currentTimeMillis() - start < SERVICE_CHECK_TIMEOUT_MILLIS) { + try { + String url = endpoint.getUrl(); + int portIndex = url.lastIndexOf(":"); + if (portIndex <= 0) { + throw new RuntimeException( + "Expected the endpoint to be of the form : but received " + url); + } + int port = Integer.parseInt(url.substring(portIndex + 1)); + String host = url.substring(0, portIndex); + new Socket(host, port).close(); + // Current service is up. Checking the next one. + continue outer; + } catch (IOException exn) { + try { + Thread.sleep(duration); + } catch (InterruptedException e) { + // Ignore + } + duration = (long) (duration * 1.2); + } + } + throw new TimeoutException( + "Timeout waiting for the service " + + endpoint.getUrl() + + " to startup after " + + (System.currentTimeMillis() - start) + + " milliseconds."); + } + } + + @VisibleForTesting + void disableServiceCheck() { + disableServiceCheck = true; + } + @Override public void expand( ExpansionApi.ExpansionRequest request, StreamObserver responseObserver) { + if (!checkedAllServices) { + try { + waitForAllServicesToBeReady(); + } catch (TimeoutException e) { + throw new RuntimeException(e); + } + checkedAllServices = true; + } try { responseObserver.onNext(processExpand(request)); responseObserver.onCompleted(); @@ -68,6 +136,14 @@ public void expand( public void discoverSchemaTransform( ExpansionApi.DiscoverSchemaTransformRequest request, StreamObserver responseObserver) { + if (!checkedAllServices) { + try { + waitForAllServicesToBeReady(); + } catch (TimeoutException e) { + throw new RuntimeException(e); + } + checkedAllServices = true; + } try { responseObserver.onNext(processDiscover(request)); responseObserver.onCompleted(); @@ -80,18 +156,41 @@ public void discoverSchemaTransform( } } - /*package*/ ExpansionApi.ExpansionResponse processExpand(ExpansionApi.ExpansionRequest request) { + private ExpansionApi.ExpansionResponse getAggregatedErrorResponse( + Map errorResponses) { + StringBuilder errorMessageBuilder = new StringBuilder(); + + errorMessageBuilder.append( + "Aggregated errors from " + errorResponses.size() + " expansion services." + "\n"); + for (Map.Entry entry : errorResponses.entrySet()) { + errorMessageBuilder.append( + "Error from expansion service " + + entry.getKey() + + ": " + + entry.getValue().getError() + + "\n"); + } + + return errorResponses + .values() + .iterator() + .next() + .toBuilder() + .setError(errorMessageBuilder.toString()) + .build(); + } + + ExpansionApi.ExpansionResponse processExpand(ExpansionApi.ExpansionRequest request) { // Trying out expansion services in order till one succeeds. // If all services fail, re-raises the last error. - // TODO: when all services fail, return an aggregated error with errors from all services. - ExpansionApi.ExpansionResponse lastErrorResponse = null; + Map errorResponses = new HashMap<>(); RuntimeException lastException = null; for (Endpoints.ApiServiceDescriptor endpoint : endpoints) { try { ExpansionApi.ExpansionResponse response = expansionServiceClientFactory.getExpansionServiceClient(endpoint).expand(request); if (!response.getError().isEmpty()) { - lastErrorResponse = response; + errorResponses.put(endpoint.getUrl(), response); continue; } return response; @@ -99,8 +198,11 @@ public void discoverSchemaTransform( lastException = e; } } - if (lastErrorResponse != null) { - return lastErrorResponse; + if (lastException != null) { + throw new RuntimeException("Expansion request to transform service failed.", lastException); + } + if (!errorResponses.isEmpty()) { + return getAggregatedErrorResponse(errorResponses); } else if (lastException != null) { throw new RuntimeException("Expansion request to transform service failed.", lastException); } else { diff --git a/sdks/java/transform-service/src/main/java/org/apache/beam/sdk/transformservice/TransformServiceConfig.java b/sdks/java/transform-service/src/main/java/org/apache/beam/sdk/transformservice/TransformServiceConfig.java index cf59ab69943a7..07ea5a2c3f486 100644 --- a/sdks/java/transform-service/src/main/java/org/apache/beam/sdk/transformservice/TransformServiceConfig.java +++ b/sdks/java/transform-service/src/main/java/org/apache/beam/sdk/transformservice/TransformServiceConfig.java @@ -23,7 +23,6 @@ import java.util.ArrayList; import java.util.List; -@SuppressWarnings("nullness") @AutoValue public abstract class TransformServiceConfig { public abstract List getExpansionservices(); diff --git a/sdks/java/transform-service/src/test/java/org/apache/beam/sdk/transformservice/ArtifactServiceTest.java b/sdks/java/transform-service/src/test/java/org/apache/beam/sdk/transformservice/ArtifactServiceTest.java index f88d360759acd..9105e23a5c50c 100644 --- a/sdks/java/transform-service/src/test/java/org/apache/beam/sdk/transformservice/ArtifactServiceTest.java +++ b/sdks/java/transform-service/src/test/java/org/apache/beam/sdk/transformservice/ArtifactServiceTest.java @@ -30,7 +30,7 @@ import org.apache.beam.runners.fnexecution.artifact.ArtifactRetrievalService; import org.apache.beam.sdk.transformservice.ArtifactService.ArtifactResolver; import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/sdks/java/transform-service/src/test/java/org/apache/beam/sdk/transformservice/ExpansionServiceTest.java b/sdks/java/transform-service/src/test/java/org/apache/beam/sdk/transformservice/ExpansionServiceTest.java index 298bce87f9015..9905abd1d9bae 100644 --- a/sdks/java/transform-service/src/test/java/org/apache/beam/sdk/transformservice/ExpansionServiceTest.java +++ b/sdks/java/transform-service/src/test/java/org/apache/beam/sdk/transformservice/ExpansionServiceTest.java @@ -60,6 +60,8 @@ public void setUp() throws Exception { endpoints.add(endpoint2); clientFactory = Mockito.mock(ExpansionServiceClientFactory.class); expansionService = new ExpansionService(endpoints, clientFactory); + // We do not run actual services in unit tests. + expansionService.disableServiceCheck(); } @Test @@ -131,7 +133,10 @@ public void testExpandFail() { ArgumentCaptor expansionResponseCapture = ArgumentCaptor.forClass(ExpansionResponse.class); Mockito.verify(responseObserver).onNext(expansionResponseCapture.capture()); - assertEquals("expansion error 2", expansionResponseCapture.getValue().getError()); + + // Error response should contain errors from both expansion services. + assertTrue(expansionResponseCapture.getValue().getError().contains("expansion error 1")); + assertTrue(expansionResponseCapture.getValue().getError().contains("expansion error 2")); } @Test diff --git a/sdks/python/.pylintrc b/sdks/python/.pylintrc index eedd234ed7ea4..a67f00ff2f82b 100644 --- a/sdks/python/.pylintrc +++ b/sdks/python/.pylintrc @@ -18,6 +18,7 @@ [MASTER] # Ignore auto-generated files. ignore=clients +load-plugins=pylint.extensions.no_self_use,pylint.extensions.bad_builtin [BASIC] # Regular expression which should only match the name @@ -85,11 +86,15 @@ disable = arguments-renamed, attribute-defined-outside-init, bad-builtin, - bad-continuation, broad-except, + broad-exception-raised, + c-extension-no-member, comparison-with-callable, + consider-iterating-dictionary, + consider-using-dict-items, consider-using-enumerate, consider-using-f-string, + consider-using-generator, consider-using-in, consider-using-sys-exit, consider-using-with, @@ -97,6 +102,7 @@ disable = design, fixme, global-statement, + global-variable-undefined, import-error, import-outside-toplevel, import-self, @@ -106,10 +112,11 @@ disable = keyword-arg-before-vararg, len-as-condition, locally-disabled, - locally-enabled, logging-not-lazy, missing-docstring, + modified-iterating-list, multiple-statements, + no-self-use, no-else-break, no-else-continue, no-else-raise, @@ -117,7 +124,6 @@ disable = no-member, no-name-in-module, no-self-argument, - no-self-use, no-value-for-parameter, not-callable, pointless-statement, @@ -126,26 +132,33 @@ disable = raising-format-tuple, redefined-builtin, redefined-outer-name, - redefined-variable-type, redundant-keyword-arg, - relative-import, + self-cls-assignment, similarities, simplifiable-if-statement, stop-iteration-return, super-init-not-called, + superfluous-parens, try-except-raise, undefined-variable, unexpected-keyword-arg, unidiomatic-typecheck, unnecessary-comprehension, + unnecessary-direct-lambda-call, + unnecessary-dunder-call, unnecessary-lambda, + unnecessary-lambda-assignment, unnecessary-pass, unneeded-not, + used-before-assignment, unsubscriptable-object, + unsupported-binary-operation, unspecified-encoding, #TODO(https://github.com/apache/beam/issues/21236) Enable explicit encoding unused-argument, + use-dict-literal, unused-wildcard-import, useless-object-inheritance, + useless-with-lock, #TODO(https://github.com/apache/beam/issues/28242) Enable and fix warnings wildcard-import, wrong-import-order, diff --git a/sdks/python/OWNERS b/sdks/python/OWNERS deleted file mode 100644 index 164680516a64c..0000000000000 --- a/sdks/python/OWNERS +++ /dev/null @@ -1,12 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers (all changes): - - aaltay - - tvalentyn - -reviewers (all GCP I/O connectors): - - lukecwik - - chamikaramj - - pabloem - - ihji - - johnjcasey diff --git a/sdks/python/apache_beam/coders/OWNERS b/sdks/python/apache_beam/coders/OWNERS deleted file mode 100644 index 207b624ab80b4..0000000000000 --- a/sdks/python/apache_beam/coders/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - charlesccychen diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py index d4ca99b80fb3e..7c5c8e09303d0 100644 --- a/sdks/python/apache_beam/coders/coders.py +++ b/sdks/python/apache_beam/coders/coders.py @@ -280,29 +280,6 @@ def _get_component_coders(self): # refined in user-defined Coders. return [] - def as_cloud_object(self, coders_context=None): - """For internal use only; no backwards-compatibility guarantees. - - Returns Google Cloud Dataflow API description of this coder.""" - # This is an internal detail of the Coder API and does not need to be - # refined in user-defined Coders. - - value = { - # We pass coders in the form "$" to make the - # job description JSON more readable. Data before the $ is ignored by - # the worker. - '@type': serialize_coder(self), - 'component_encodings': [ - component.as_cloud_object(coders_context) - for component in self._get_component_coders() - ], - } - - if coders_context: - value['pipeline_proto_coder_id'] = coders_context.get_id(self) - - return value - def __repr__(self): return self.__class__.__name__ @@ -493,11 +470,6 @@ def is_deterministic(self): def to_type_hint(self): return bytes - def as_cloud_object(self, coders_context=None): - return { - '@type': 'kind:bytes', - } - def __eq__(self, other): return type(self) == type(other) @@ -667,11 +639,6 @@ def is_deterministic(self): def to_type_hint(self): return int - def as_cloud_object(self, coders_context=None): - return { - '@type': 'kind:varint', - } - def __eq__(self, other): return type(self) == type(other) @@ -846,21 +813,6 @@ def is_deterministic(self): # GroupByKey operations. return False - def as_cloud_object(self, coders_context=None, is_pair_like=True): - value = super().as_cloud_object(coders_context) - # We currently use this coder in places where we cannot infer the coder to - # use for the value type in a more granular way. In places where the - # service expects a pair, it checks for the "is_pair_like" key, in which - # case we would fail without the hack below. - if is_pair_like: - value['is_pair_like'] = True - value['component_encodings'] = [ - self.as_cloud_object(coders_context, is_pair_like=False), - self.as_cloud_object(coders_context, is_pair_like=False) - ] - - return value - # We allow .key_coder() and .value_coder() to be called on PickleCoder since # we can't always infer the return values of lambdas in ParDo operations, the # result of which may be used in a GroupBykey. @@ -983,21 +935,6 @@ def as_deterministic_coder(self, step_label, error_message=None): def to_type_hint(self): return Any - def as_cloud_object(self, coders_context=None, is_pair_like=True): - value = super().as_cloud_object(coders_context) - # We currently use this coder in places where we cannot infer the coder to - # use for the value type in a more granular way. In places where the - # service expects a pair, it checks for the "is_pair_like" key, in which - # case we would fail without the hack below. - if is_pair_like: - value['is_pair_like'] = True - value['component_encodings'] = [ - self.as_cloud_object(coders_context, is_pair_like=False), - self.as_cloud_object(coders_context, is_pair_like=False) - ] - - return value - # We allow .key_coder() and .value_coder() to be called on FastPrimitivesCoder # since we can't always infer the return values of lambdas in ParDo # operations, the result of which may be used in a GroupBykey. @@ -1231,19 +1168,6 @@ def from_type_hint(cls, typehint, registry): # type: (typehints.TupleConstraint, CoderRegistry) -> TupleCoder return cls([registry.get_coder(t) for t in typehint.tuple_types]) - def as_cloud_object(self, coders_context=None): - if self.is_kv_coder(): - return { - '@type': 'kind:pair', - 'is_pair_like': True, - 'component_encodings': [ - component.as_cloud_object(coders_context) - for component in self._get_component_coders() - ], - } - - return super().as_cloud_object(coders_context) - def _get_component_coders(self): # type: () -> Tuple[Coder, ...] return self.coders() @@ -1353,15 +1277,6 @@ def as_deterministic_coder(self, step_label, error_message=None): return type(self)( self._elem_coder.as_deterministic_coder(step_label, error_message)) - def as_cloud_object(self, coders_context=None): - return { - '@type': 'kind:stream', - 'is_stream_like': True, - 'component_encodings': [ - self._elem_coder.as_cloud_object(coders_context) - ], - } - def value_coder(self): return self._elem_coder @@ -1409,11 +1324,6 @@ def __init__(self): from apache_beam.transforms import window super().__init__(window.GlobalWindow()) - def as_cloud_object(self, coders_context=None): - return { - '@type': 'kind:global_window', - } - Coder.register_structured_urn( common_urns.coders.GLOBAL_WINDOW.urn, GlobalWindowCoder) @@ -1428,11 +1338,6 @@ def is_deterministic(self): # type: () -> bool return True - def as_cloud_object(self, coders_context=None): - return { - '@type': 'kind:interval_window', - } - def __eq__(self, other): return type(self) == type(other) @@ -1466,16 +1371,6 @@ def is_deterministic(self): c.is_deterministic() for c in [self.wrapped_value_coder, self.timestamp_coder, self.window_coder]) - def as_cloud_object(self, coders_context=None): - return { - '@type': 'kind:windowed_value', - 'is_wrapper': True, - 'component_encodings': [ - component.as_cloud_object(coders_context) - for component in self._get_component_coders() - ], - } - def _get_component_coders(self): # type: () -> List[Coder] return [self.wrapped_value_coder, self.window_coder] @@ -1527,10 +1422,6 @@ def is_deterministic(self): # type: () -> bool return self.wrapped_value_coder.is_deterministic() - def as_cloud_object(self, coders_context=None): - raise NotImplementedError( - "as_cloud_object not supported for ParamWindowedValueCoder") - def __repr__(self): return 'ParamWindowedValueCoder[%s]' % self.wrapped_value_coder @@ -1577,14 +1468,6 @@ def estimate_size(self, value): def value_coder(self): return self._value_coder - def as_cloud_object(self, coders_context=None): - return { - '@type': 'kind:length_prefix', - 'component_encodings': [ - self._value_coder.as_cloud_object(coders_context) - ], - } - def _get_component_coders(self): # type: () -> Tuple[Coder, ...] return (self._value_coder, ) @@ -1680,14 +1563,6 @@ def is_deterministic(self): # type: () -> bool return self._key_coder.is_deterministic() - def as_cloud_object(self, coders_context=None): - return { - '@type': 'kind:sharded_key', - 'component_encodings': [ - self._key_coder.as_cloud_object(coders_context) - ], - } - def to_type_hint(self): from apache_beam.typehints import sharded_key_type return sharded_key_type.ShardedKeyTypeConstraint( @@ -1738,14 +1613,6 @@ def _get_component_coders(self) -> List[Coder]: def is_deterministic(self) -> bool: return self._window_coder.is_deterministic() - def as_cloud_object(self, coders_context=None): - return { - '@type': 'kind:custom_window', - 'component_encodings': [ - self._window_coder.as_cloud_object(coders_context) - ], - } - def __repr__(self): return 'TimestampPrefixingWindowCoder[%r]' % self._window_coder diff --git a/sdks/python/apache_beam/coders/coders_test.py b/sdks/python/apache_beam/coders/coders_test.py index 1d73c5977ea45..1143e9c5d8732 100644 --- a/sdks/python/apache_beam/coders/coders_test.py +++ b/sdks/python/apache_beam/coders/coders_test.py @@ -76,7 +76,7 @@ def test_proto_coder(self): ma = test_message.MessageA() mb = ma.field2.add() mb.field1 = True - ma.field1 = u'hello world' + ma.field1 = 'hello world' expected_coder = coders.ProtoCoder(ma.__class__) real_coder = coders_registry.get_coder(ma.__class__) self.assertEqual(expected_coder, real_coder) @@ -90,7 +90,7 @@ def test_deterministic_proto_coder(self): ma = test_message.MessageA() mb = ma.field2.add() mb.field1 = True - ma.field1 = u'hello world' + ma.field1 = 'hello world' expected_coder = coders.DeterministicProtoCoder(ma.__class__) real_coder = ( coders_registry.get_coder( @@ -130,7 +130,7 @@ class ProtoPlusCoderTest(unittest.TestCase): def test_proto_plus_coder(self): ma = ProtoPlusMessageA() ma.field2 = [ProtoPlusMessageB(field1=True)] - ma.field1 = u'hello world' + ma.field1 = 'hello world' expected_coder = coders.ProtoPlusCoder(ma.__class__) real_coder = coders_registry.get_coder(ma.__class__) self.assertTrue(issubclass(ma.__class__, proto.Message)) diff --git a/sdks/python/apache_beam/coders/coders_test_common.py b/sdks/python/apache_beam/coders/coders_test_common.py index 7adb06cb28701..70582e7992a6f 100644 --- a/sdks/python/apache_beam/coders/coders_test_common.py +++ b/sdks/python/apache_beam/coders/coders_test_common.py @@ -121,7 +121,7 @@ class CodersTest(unittest.TestCase): -1, 1.5, b'str\0str', - u'unicode\0\u0101', + 'unicode\0\u0101', (), (1, 2, 3), [], @@ -387,16 +387,6 @@ def test_timer_coder(self): def test_tuple_coder(self): kv_coder = coders.TupleCoder((coders.VarIntCoder(), coders.BytesCoder())) - # Verify cloud object representation - self.assertEqual({ - '@type': 'kind:pair', - 'is_pair_like': True, - 'component_encodings': [ - coders.VarIntCoder().as_cloud_object(), - coders.BytesCoder().as_cloud_object() - ], - }, - kv_coder.as_cloud_object()) # Test binary representation self.assertEqual(b'\x04abc', kv_coder.encode((4, b'abc'))) # Test unnested @@ -407,7 +397,7 @@ def test_tuple_coder(self): coders.TupleCoder((coders.PickleCoder(), coders.VarIntCoder())), coders.StrUtf8Coder(), coders.BooleanCoder())), ((1, 2), 'a', True), - ((-2, 5), u'a\u0101' * 100, False), ((300, 1), 'abc\0' * 5, True)) + ((-2, 5), 'a\u0101' * 100, False), ((300, 1), 'abc\0' * 5, True)) def test_tuple_sequence_coder(self): int_tuple_coder = coders.TupleSequenceCoder(coders.VarIntCoder()) @@ -420,17 +410,10 @@ def test_base64_pickle_coder(self): self.check_coder(coders.Base64PickleCoder(), 'a', 1, 1.5, (1, 2, 3)) def test_utf8_coder(self): - self.check_coder(coders.StrUtf8Coder(), 'a', u'ab\u00FF', u'\u0101\0') + self.check_coder(coders.StrUtf8Coder(), 'a', 'ab\u00FF', '\u0101\0') def test_iterable_coder(self): iterable_coder = coders.IterableCoder(coders.VarIntCoder()) - # Verify cloud object representation - self.assertEqual({ - '@type': 'kind:stream', - 'is_stream_like': True, - 'component_encodings': [coders.VarIntCoder().as_cloud_object()] - }, - iterable_coder.as_cloud_object()) # Test unnested self.check_coder(iterable_coder, [1], [-1, 0, 100]) # Test nested @@ -507,16 +490,6 @@ def test_windowedvalue_coder_paneinfo(self): def test_windowed_value_coder(self): coder = coders.WindowedValueCoder( coders.VarIntCoder(), coders.GlobalWindowCoder()) - # Verify cloud object representation - self.assertEqual({ - '@type': 'kind:windowed_value', - 'is_wrapper': True, - 'component_encodings': [ - coders.VarIntCoder().as_cloud_object(), - coders.GlobalWindowCoder().as_cloud_object(), - ], - }, - coder.as_cloud_object()) # Test binary representation self.assertEqual( b'\x7f\xdf;dZ\x1c\xac\t\x00\x00\x00\x01\x0f\x01', @@ -604,10 +577,10 @@ def test_proto_coder(self): ma = test_message.MessageA() mab = ma.field2.add() mab.field1 = True - ma.field1 = u'hello world' + ma.field1 = 'hello world' mb = test_message.MessageA() - mb.field1 = u'beam' + mb.field1 = 'beam' proto_coder = coders.ProtoCoder(ma.__class__) self.check_coder(proto_coder, ma) @@ -618,8 +591,6 @@ def test_proto_coder(self): def test_global_window_coder(self): coder = coders.GlobalWindowCoder() value = window.GlobalWindow() - # Verify cloud object representation - self.assertEqual({'@type': 'kind:global_window'}, coder.as_cloud_object()) # Test binary representation self.assertEqual(b'', coder.encode(value)) self.assertEqual(value, coder.decode(b'')) @@ -630,12 +601,6 @@ def test_global_window_coder(self): def test_length_prefix_coder(self): coder = coders.LengthPrefixCoder(coders.BytesCoder()) - # Verify cloud object representation - self.assertEqual({ - '@type': 'kind:length_prefix', - 'component_encodings': [coders.BytesCoder().as_cloud_object()] - }, - coder.as_cloud_object()) # Test binary representation self.assertEqual(b'\x00', coder.encode(b'')) self.assertEqual(b'\x01a', coder.encode(b'a')) @@ -666,7 +631,7 @@ def __iter__(self): # Test nested tuple observable. coder = coders.TupleCoder((coders.StrUtf8Coder(), iter_coder)) - value = (u'123', observ) + value = ('123', observ) self.assertEqual( coder.get_impl().get_estimated_size_and_observables(value)[1], [(observ, elem_coder.get_impl())]) @@ -725,12 +690,6 @@ def test_sharded_key_coder(self): for key, bytes_repr, key_coder in key_and_coders: coder = coders.ShardedKeyCoder(key_coder) - # Verify cloud object representation - self.assertEqual({ - '@type': 'kind:sharded_key', - 'component_encodings': [key_coder.as_cloud_object()] - }, - coder.as_cloud_object()) # Test str repr self.assertEqual('%s' % coder, 'ShardedKeyCoder[%s]' % key_coder) diff --git a/sdks/python/apache_beam/coders/row_coder.py b/sdks/python/apache_beam/coders/row_coder.py index 19424fa1f12b8..7765ccebc26f9 100644 --- a/sdks/python/apache_beam/coders/row_coder.py +++ b/sdks/python/apache_beam/coders/row_coder.py @@ -17,8 +17,6 @@ # pytype: skip-file -from google.protobuf import json_format - from apache_beam.coders import typecoders from apache_beam.coders.coder_impl import LogicalTypeCoderImpl from apache_beam.coders.coder_impl import RowCoderImpl @@ -91,13 +89,6 @@ def as_deterministic_coder(self, step_label, error_message=None): def to_type_hint(self): return self._type_hint - def as_cloud_object(self, coders_context=None): - value = super().as_cloud_object(coders_context) - - value['schema'] = json_format.MessageToJson(self.schema).encode('utf-8') - - return value - def __hash__(self): return hash(self.schema.SerializeToString()) diff --git a/sdks/python/apache_beam/coders/row_coder_test.py b/sdks/python/apache_beam/coders/row_coder_test.py index dbca3e7f69c95..6ac982835cb33 100644 --- a/sdks/python/apache_beam/coders/row_coder_test.py +++ b/sdks/python/apache_beam/coders/row_coder_test.py @@ -22,7 +22,6 @@ from itertools import chain import numpy as np -from google.protobuf import json_format from numpy.testing import assert_array_equal import apache_beam as beam @@ -398,16 +397,6 @@ def test_row_coder_fail_early_bad_schema(self): self.assertRaisesRegex( ValueError, "type_with_no_typeinfo", lambda: RowCoder(schema_proto)) - def test_row_coder_cloud_object_schema(self): - schema_proto = schema_pb2.Schema(id='some-cloud-object-schema') - schema_proto_json = json_format.MessageToJson(schema_proto).encode('utf-8') - - coder = RowCoder(schema_proto) - - cloud_object = coder.as_cloud_object() - - self.assertEqual(schema_proto_json, cloud_object['schema']) - def test_batch_encode_decode(self): coder = RowCoder(typing_to_runner_api(Person).row_type.schema).get_impl() seq_out = coder_impl.create_OutputStream() diff --git a/sdks/python/apache_beam/coders/slow_coders_test.py b/sdks/python/apache_beam/coders/slow_coders_test.py index fe1c707a62e52..7915116a19a34 100644 --- a/sdks/python/apache_beam/coders/slow_coders_test.py +++ b/sdks/python/apache_beam/coders/slow_coders_test.py @@ -25,6 +25,9 @@ from apache_beam.coders.coders_test_common import * +@unittest.skip( + 'Remove non-cython tests.' + 'https://github.com/apache/beam/issues/28307') class SlowCoders(unittest.TestCase): def test_using_slow_impl(self): try: diff --git a/sdks/python/apache_beam/dataframe/frame_base.py b/sdks/python/apache_beam/dataframe/frame_base.py index 24497f1de0696..4e89e473b7301 100644 --- a/sdks/python/apache_beam/dataframe/frame_base.py +++ b/sdks/python/apache_beam/dataframe/frame_base.py @@ -475,7 +475,7 @@ def wrapper(self, inplace=False, **kwargs): return wrapper -def args_to_kwargs(base_type): +def args_to_kwargs(base_type, removed_method=False, removed_args=None): """Convert all args to kwargs before calling the decorated function. When applied to a function, this decorator creates a new function @@ -484,18 +484,52 @@ def args_to_kwargs(base_type): determine the name to use for arguments that are converted to keyword arguments. - For internal use only. No backwards compatibility guarantees.""" + For internal use only. No backwards compatibility guarantees. + + Args: + base_type: The pandas type of the method that this is trying to replicate. + removed_method: Whether this method has been removed in the running + Pandas version. + removed_args: If not empty, which arguments have been dropped in the + running Pandas version. + """ def wrap(func): - arg_names = getfullargspec(unwrap(getattr(base_type, func.__name__))).args + if removed_method: + # Do no processing, let Beam function itself raise the error if called. + return func + + removed_arg_names = removed_args if removed_args is not None else [] + + # We would need to add position only arguments if they ever become a thing + # in Pandas (as of 2.1 currently they aren't). + base_arg_spec = getfullargspec(unwrap(getattr(base_type, func.__name__))) + base_arg_names = base_arg_spec.args + # Some arguments are keyword only and we still want to check against those. + all_possible_base_arg_names = base_arg_names + base_arg_spec.kwonlyargs + beam_arg_names = getfullargspec(func).args + + if not_found := (set(beam_arg_names) - set(all_possible_base_arg_names) - + set(removed_arg_names)): + raise TypeError( + f"Beam definition of {func.__name__} has arguments that are not found" + f" in the base version of the function: {not_found}") @functools.wraps(func) def wrapper(*args, **kwargs): - for name, value in zip(arg_names, args): + if len(args) > len(base_arg_names): + raise TypeError(f"{func.__name__} got too many positioned arguments.") + + for name, value in zip(base_arg_names, args): if name in kwargs: raise TypeError( "%s() got multiple values for argument '%s'" % (func.__name__, name)) kwargs[name] = value + # Still have to populate these for the Beam function signature. + if removed_args: + for name in removed_args: + if name not in kwargs: + kwargs[name] = None return func(**kwargs) return wrapper @@ -524,14 +558,22 @@ def wrapper(*args, **kwargs): f"**{BEAM_SPECIFIC!r}** for details.") -def with_docs_from(base_type, name=None): +def with_docs_from(base_type, name=None, removed_method=False): """Decorator that updates the documentation from the wrapped function to duplicate the documentation from the identically-named method in `base_type`. Any docstring on the original function will be included in the new function under a "Differences from pandas" heading. + + removed_method used in cases where a method has been removed in a later + version of Pandas. """ def wrap(func): + if removed_method: + func.__doc__ = ( + "This method has been removed in the current version of Pandas.") + return func + fn_name = name or func.__name__ orig_doc = getattr(base_type, fn_name).__doc__ if orig_doc is None: @@ -588,23 +630,39 @@ def format_section(header): return wrap -def populate_defaults(base_type): +def populate_defaults(base_type, removed_method=False, removed_args=None): """Populate default values for keyword arguments in decorated function. When applied to a function, this decorator creates a new function with default values for all keyword arguments, based on the default values for the identically-named method on `base_type`. - For internal use only. No backwards compatibility guarantees.""" + For internal use only. No backwards compatibility guarantees. + + Args: + base_type: The pandas type of the method that this is trying to replicate. + removed_method: Whether this method has been removed in the running + Pandas version. + removed_args: If not empty, which arguments have been dropped in the + running Pandas version. + """ def wrap(func): + if removed_method: + return func + base_argspec = getfullargspec(unwrap(getattr(base_type, func.__name__))) - if not base_argspec.defaults: + if not base_argspec.defaults and not base_argspec.kwonlydefaults: return func - arg_to_default = dict( - zip( - base_argspec.args[-len(base_argspec.defaults):], - base_argspec.defaults)) + arg_to_default = {} + if base_argspec.defaults: + arg_to_default.update( + zip( + base_argspec.args[-len(base_argspec.defaults):], + base_argspec.defaults)) + + if base_argspec.kwonlydefaults: + arg_to_default.update(base_argspec.kwonlydefaults) unwrapped_func = unwrap(func) # args that do not have defaults in func, but do have defaults in base @@ -613,12 +671,22 @@ def wrap(func): defaults_to_populate = set( func_argspec.args[:num_non_defaults]).intersection( arg_to_default.keys()) + if removed_args: + defaults_to_populate -= set(removed_args) + + # In pandas 2, many methods rely on the default copy=None + # to mean that copy is the value of copy_on_write. Since + # copy_on_write will always be true for Beam, just fill it + # in here. In pandas 1, the default was True anyway. + if 'copy' in arg_to_default and arg_to_default['copy'] is None: + arg_to_default['copy'] = True @functools.wraps(func) def wrapper(**kwargs): for name in defaults_to_populate: if name not in kwargs: kwargs[name] = arg_to_default[name] + return func(**kwargs) return wrapper diff --git a/sdks/python/apache_beam/dataframe/frame_base_test.py b/sdks/python/apache_beam/dataframe/frame_base_test.py index 82d5b65e1a495..0a73905339fd7 100644 --- a/sdks/python/apache_beam/dataframe/frame_base_test.py +++ b/sdks/python/apache_beam/dataframe/frame_base_test.py @@ -72,7 +72,7 @@ def add_one(frame): def test_args_to_kwargs(self): class Base(object): - def func(self, a=1, b=2, c=3): + def func(self, a=1, b=2, c=3, *, kw_only=4): pass class Proxy(object): @@ -87,20 +87,36 @@ def func(self, **kwargs): self.assertEqual(proxy.func(2, 4, 6), {'a': 2, 'b': 4, 'c': 6}) self.assertEqual(proxy.func(2, c=6), {'a': 2, 'c': 6}) self.assertEqual(proxy.func(c=6, a=2), {'a': 2, 'c': 6}) + self.assertEqual(proxy.func(2, kw_only=20), {'a': 2, 'kw_only': 20}) + with self.assertRaises(TypeError): # got too many positioned arguments + proxy.func(2, 4, 6, 8) def test_args_to_kwargs_populates_defaults(self): class Base(object): def func(self, a=1, b=2, c=3): pass + def func_removed_args(self, a): + pass + class Proxy(object): @frame_base.args_to_kwargs(Base) @frame_base.populate_defaults(Base) def func(self, a, c=1000, **kwargs): return dict(kwargs, a=a, c=c) + @frame_base.args_to_kwargs(Base, removed_method=True) + @frame_base.populate_defaults(Base, removed_method=True) + def func_removed_method(self, a, **kwargs): + return dict(kwargs, a=a) + + @frame_base.args_to_kwargs(Base, removed_args=['c']) + @frame_base.populate_defaults(Base, removed_args=['c']) + def func_removed_args(self, a, c, **kwargs): + return dict(kwargs, a=a) + proxy = Proxy() - # pylint: disable=too-many-function-args + # pylint: disable=too-many-function-args,no-value-for-parameter self.assertEqual(proxy.func(), {'a': 1, 'c': 1000}) self.assertEqual(proxy.func(100), {'a': 100, 'c': 1000}) self.assertEqual(proxy.func(2, 4, 6), {'a': 2, 'b': 4, 'c': 6}) @@ -108,6 +124,71 @@ def func(self, a, c=1000, **kwargs): self.assertEqual(proxy.func(c=6, a=2), {'a': 2, 'c': 6}) self.assertEqual(proxy.func(c=6), {'a': 1, 'c': 6}) + with self.assertRaises(TypeError): # missing 1 required positional argument + proxy.func_removed_method() + self.assertEqual(proxy.func_removed_method(12, c=100), {'a': 12, 'c': 100}) + + with self.assertRaises(TypeError): # missing 1 required positional argument + proxy.func_removed_args() + self.assertEqual(proxy.func_removed_args(12, d=100), {'a': 12, 'd': 100}) + + def test_args_to_kwargs_populates_default_handles_kw_only(self): + class Base(object): + def func(self, a, b=2, c=3, *, kw_only=4): + pass + + class ProxyUsesKwOnly(object): + @frame_base.args_to_kwargs(Base) + @frame_base.populate_defaults(Base) + def func(self, a, kw_only, **kwargs): + return dict(kwargs, a=a, kw_only=kw_only) + + proxy = ProxyUsesKwOnly() + + # pylint: disable=too-many-function-args,no-value-for-parameter + with self.assertRaises(TypeError): # missing 1 required positional argument + proxy.func() + + self.assertEqual(proxy.func(100), {'a': 100, 'kw_only': 4}) + self.assertEqual( + proxy.func(2, 4, 6, kw_only=8), { + 'a': 2, 'b': 4, 'c': 6, 'kw_only': 8 + }) + with self.assertRaises(TypeError): + proxy.func(2, 4, 6, 8) # got too many positioned arguments + + class ProxyDoesntUseKwOnly(object): + @frame_base.args_to_kwargs(Base) + @frame_base.populate_defaults(Base) + def func(self, a, **kwargs): + return dict(kwargs, a=a) + + proxy = ProxyDoesntUseKwOnly() + + # pylint: disable=too-many-function-args,no-value-for-parameter + with self.assertRaises(TypeError): # missing 1 required positional argument + proxy.func() + self.assertEqual(proxy.func(100), {'a': 100}) + self.assertEqual( + proxy.func(2, 4, 6, kw_only=8), { + 'a': 2, 'b': 4, 'c': 6, 'kw_only': 8 + }) + + def test_populate_defaults_overwrites_copy(self): + class Base(object): + def func(self, a=1, b=2, c=3, *, copy=None): + pass + + class Proxy(object): + @frame_base.args_to_kwargs(Base) + @frame_base.populate_defaults(Base) + def func(self, a, copy, **kwargs): + return dict(kwargs, a=a, copy=copy) + + proxy = Proxy() + self.assertEqual(proxy.func(), {'a': 1, 'copy': True}) + self.assertEqual(proxy.func(copy=False), {'a': 1, 'copy': False}) + if __name__ == '__main__': unittest.main() diff --git a/sdks/python/apache_beam/dataframe/frames.py b/sdks/python/apache_beam/dataframe/frames.py index 3020eecbaeb50..b7aa130fbbd8f 100644 --- a/sdks/python/apache_beam/dataframe/frames.py +++ b/sdks/python/apache_beam/dataframe/frames.py @@ -116,7 +116,6 @@ def wrapper(self, *args, **kwargs): 'quantile', 'describe', 'sem', - 'mad', 'skew', 'kurt', 'kurtosis', @@ -126,6 +125,10 @@ def wrapper(self, *args, **kwargs): 'cov', 'nunique', ] +# mad was removed in Pandas 2.0. +if PD_VERSION < (2, 0): + UNLIFTABLE_AGGREGATIONS.append('mad') + ALL_AGGREGATIONS = ( LIFTABLE_AGGREGATIONS + LIFTABLE_WITH_SUM_AGGREGATIONS + UNLIFTABLE_AGGREGATIONS) @@ -904,15 +907,17 @@ def sort_index(self, axis, **kwargs): return frame_base.DeferredFrame.wrap( expressions.ComputedExpression( 'sort_index', - lambda df: df.sort_index(axis, **kwargs), + lambda df: df.sort_index(axis=axis, **kwargs), [self._expr], requires_partition_by=partitionings.Arbitrary(), preserves_partition_by=partitionings.Arbitrary(), )) @frame_base.with_docs_from(pd.DataFrame) - @frame_base.args_to_kwargs(pd.DataFrame) - @frame_base.populate_defaults(pd.DataFrame) + @frame_base.args_to_kwargs( + pd.DataFrame, removed_args=["errors"] if PD_VERSION >= (2, 0) else None) + @frame_base.populate_defaults( + pd.DataFrame, removed_args=["errors"] if PD_VERSION >= (2, 0) else None) @frame_base.maybe_inplace def where(self, cond, other, errors, **kwargs): """where is not parallelizable when ``errors="ignore"`` is specified.""" @@ -934,16 +939,19 @@ def where(self, cond, other, errors, **kwargs): else: actual_args['other'] = other - if errors == "ignore": - # We need all data in order to ignore errors and propagate the original - # data. - requires = partitionings.Singleton( - reason=( - f"where(errors={errors!r}) is currently not parallelizable, " - "because all data must be collected on one node to determine if " - "the original data should be propagated instead.")) + # For Pandas 2.0, errors was removed as an argument. + if PD_VERSION < (2, 0): + if "errors" in kwargs and kwargs['errors'] == "ignore": + # We need all data in order to ignore errors and propagate the original + # data. + requires = partitionings.Singleton( + reason=( + f"where(errors={kwargs['errors']!r}) is currently not " + "parallelizable, because all data must be collected on one " + "node to determine if the original data should be propagated " + "instead.")) - actual_args['errors'] = errors + actual_args['errors'] = kwargs['errors'] if 'errors' in kwargs else None def where_execution(df, *args): runtime_values = { @@ -1333,12 +1341,14 @@ def keys(self): frame_base.wont_implement_method( pd.Series, 'shape', reason="non-deferred-result")) - @frame_base.with_docs_from(pd.Series) - @frame_base.args_to_kwargs(pd.Series) - @frame_base.populate_defaults(pd.Series) + @frame_base.with_docs_from(pd.Series, removed_method=PD_VERSION >= (2, 0)) + @frame_base.args_to_kwargs(pd.Series, removed_method=PD_VERSION >= (2, 0)) + @frame_base.populate_defaults(pd.Series, removed_method=PD_VERSION >= (2, 0)) def append(self, to_append, ignore_index, verify_integrity, **kwargs): """``ignore_index=True`` is not supported, because it requires generating an order-sensitive index.""" + if PD_VERSION >= (2, 0): + raise frame_base.WontImplementError('append() was removed in Pandas 2.0.') if not isinstance(to_append, DeferredSeries): raise frame_base.WontImplementError( "append() only accepts DeferredSeries instances, received " + @@ -1378,7 +1388,7 @@ def align(self, other, join, axis, level, method, **kwargs): Only the default, ``method=None``, is allowed.""" if level is not None: raise NotImplementedError('per-level align') - if method is not None: + if method is not None and method != lib.no_default: raise frame_base.WontImplementError( f"align(method={method!r}) is not supported because it is " "order sensitive. Only align(method=None) is supported.", @@ -1461,8 +1471,10 @@ def compute_idx(s): index = pd.Index([], dtype=index_dtype) proxy = self._expr.proxy().copy() proxy.index = index - proxy = proxy.append( - pd.Series([1], index=np.asarray(['0']).astype(proxy.index.dtype))) + proxy = pd.concat([ + proxy, + pd.Series([1], index=np.asarray(['0']).astype(proxy.index.dtype)) + ]) idx_func = expressions.ComputedExpression( 'idx_func', @@ -1600,14 +1612,11 @@ def mean(self, skipna, **kwargs): return self.sum(skipna=skipna, **kwargs) / size @frame_base.with_docs_from(pd.Series) - @frame_base.args_to_kwargs(pd.Series) - @frame_base.populate_defaults(pd.Series) + @frame_base.args_to_kwargs( + pd.Series, removed_args=["level"] if PD_VERSION >= (2, 0) else None) + @frame_base.populate_defaults( + pd.Series, removed_args=["level"] if PD_VERSION >= (2, 0) else None) def var(self, axis, skipna, level, ddof, **kwargs): - """Per-level aggregation is not yet supported - (https://github.com/apache/beam/issues/21829). Only the default, - ``level=None``, is allowed.""" - if level is not None: - raise NotImplementedError("per-level aggregation") if skipna is None or skipna: self = self.dropna() # pylint: disable=self-cls-assignment @@ -1675,11 +1684,11 @@ def corr(self, other, method, min_periods): requires_partition_by=partitionings.Singleton(reason=reason))) @frame_base.with_docs_from(pd.Series) - @frame_base.args_to_kwargs(pd.Series) - @frame_base.populate_defaults(pd.Series) + @frame_base.args_to_kwargs( + pd.Series, removed_args=["level"] if PD_VERSION >= (2, 0) else None) + @frame_base.populate_defaults( + pd.Series, removed_args=["level"] if PD_VERSION >= (2, 0) else None) def skew(self, axis, skipna, level, numeric_only, **kwargs): - if level is not None: - raise NotImplementedError("per-level aggregation") if skipna is None or skipna: self = self.dropna() # pylint: disable=self-cls-assignment # See the online, numerically stable formulae at @@ -1739,11 +1748,11 @@ def combine_moments(data): requires_partition_by=partitionings.Singleton())) @frame_base.with_docs_from(pd.Series) - @frame_base.args_to_kwargs(pd.Series) - @frame_base.populate_defaults(pd.Series) + @frame_base.args_to_kwargs( + pd.Series, removed_args=["level"] if PD_VERSION >= (2, 0) else None) + @frame_base.populate_defaults( + pd.Series, removed_args=["level"] if PD_VERSION >= (2, 0) else None) def kurtosis(self, axis, skipna, level, numeric_only, **kwargs): - if level is not None: - raise NotImplementedError("per-level aggregation") if skipna is None or skipna: self = self.dropna() # pylint: disable=self-cls-assignment @@ -1892,7 +1901,8 @@ def dropna(self, **kwargs): @frame_base.with_docs_from(pd.Series) @frame_base.args_to_kwargs(pd.Series) - @frame_base.populate_defaults(pd.Series) + @frame_base.populate_defaults( + pd.Series, removed_args=['inplace'] if PD_VERSION >= (2, 0) else None) @frame_base.maybe_inplace def set_axis(self, labels, **kwargs): # TODO: assigning the index is generally order-sensitive, but we could @@ -2092,7 +2102,9 @@ def axes(self): sum = _agg_method(pd.Series, 'sum') median = _agg_method(pd.Series, 'median') sem = _agg_method(pd.Series, 'sem') - mad = _agg_method(pd.Series, 'mad') + # mad was removed in Pandas 2.0. + if PD_VERSION < (2, 0): + mad = _agg_method(pd.Series, 'mad') argmax = frame_base.wont_implement_method( pd.Series, 'argmax', reason='order-sensitive') @@ -2336,8 +2348,13 @@ def value_counts( result = column.groupby(column, dropna=dropna).size() - # groupby.size() names the index, which we don't need - result.index.name = None + # Pandas 2 introduces new naming for the results. + if PD_VERSION >= (2, 0): + result.index.name = getattr(self, "name", None) + result.name = "proportion" if normalize else "count" + else: + # groupby.size() names the index, which we don't need + result.index.name = None if normalize: return result / column.length() @@ -2563,7 +2580,7 @@ def align(self, other, join, axis, copy, level, method, **kwargs): "align(copy=False) is not supported because it might be an inplace " "operation depending on the data. Please prefer the default " "align(copy=True).") - if method is not None: + if method is not None and method != lib.no_default: raise frame_base.WontImplementError( f"align(method={method!r}) is not supported because it is " "order sensitive. Only align(method=None) is supported.", @@ -2571,7 +2588,8 @@ def align(self, other, join, axis, copy, level, method, **kwargs): if kwargs: raise NotImplementedError('align(%s)' % ', '.join(kwargs.keys())) - if level is not None: + # In Pandas 2.0, all aggregations lost the level keyword. + if PD_VERSION < (2, 0) and level is not None: # Could probably get by partitioning on the used levels. requires_partition_by = partitionings.Singleton(reason=( f"align(level={level}) is not currently parallelizable. Only " @@ -2588,12 +2606,15 @@ def align(self, other, join, axis, copy, level, method, **kwargs): requires_partition_by=requires_partition_by, preserves_partition_by=partitionings.Arbitrary())) - @frame_base.with_docs_from(pd.DataFrame) - @frame_base.args_to_kwargs(pd.DataFrame) - @frame_base.populate_defaults(pd.DataFrame) + @frame_base.with_docs_from(pd.DataFrame, removed_method=PD_VERSION >= (2, 0)) + @frame_base.args_to_kwargs(pd.DataFrame, removed_method=PD_VERSION >= (2, 0)) + @frame_base.populate_defaults(pd.DataFrame, + removed_method=PD_VERSION >= (2, 0)) def append(self, other, ignore_index, verify_integrity, sort, **kwargs): """``ignore_index=True`` is not supported, because it requires generating an order-sensitive index.""" + if PD_VERSION >= (2, 0): + raise frame_base.WontImplementError('append() was removed in Pandas 2.0.') if not isinstance(other, DeferredDataFrame): raise frame_base.WontImplementError( "append() only accepts DeferredDataFrame instances, received " + @@ -2660,7 +2681,9 @@ def set_index(self, keys, **kwargs): @frame_base.with_docs_from(pd.DataFrame) @frame_base.args_to_kwargs(pd.DataFrame) - @frame_base.populate_defaults(pd.DataFrame) + @frame_base.populate_defaults( + pd.DataFrame, + removed_args=['inplace'] if PD_VERSION >= (2, 0) else None) @frame_base.maybe_inplace def set_axis(self, labels, axis, **kwargs): if axis in ('index', 0): @@ -2674,7 +2697,7 @@ def set_axis(self, labels, axis, **kwargs): return frame_base.DeferredFrame.wrap( expressions.ComputedExpression( 'set_axis', - lambda df: df.set_axis(labels, axis, **kwargs), + lambda df: df.set_axis(labels, axis=axis, **kwargs), [self._expr], requires_partition_by=partitionings.Arbitrary(), preserves_partition_by=partitionings.Arbitrary())) @@ -2955,6 +2978,8 @@ def aggregate(self, func, axis, *args, **kwargs): agg = aggregate applymap = frame_base._elementwise_method('applymap', base=pd.DataFrame) + if PD_VERSION >= (2, 1): + map = frame_base._elementwise_method('map', base=pd.DataFrame) add_prefix = frame_base._elementwise_method('add_prefix', base=pd.DataFrame) add_suffix = frame_base._elementwise_method('add_suffix', base=pd.DataFrame) @@ -3914,10 +3939,12 @@ def pivot_helper(df): std = _agg_method(pd.DataFrame, 'std') var = _agg_method(pd.DataFrame, 'var') sem = _agg_method(pd.DataFrame, 'sem') - mad = _agg_method(pd.DataFrame, 'mad') skew = _agg_method(pd.DataFrame, 'skew') kurt = _agg_method(pd.DataFrame, 'kurt') kurtosis = _agg_method(pd.DataFrame, 'kurtosis') + # mad was removed in Pandas 2.0. + if PD_VERSION < (2, 0): + mad = _agg_method(pd.DataFrame, 'mad') take = frame_base.wont_implement_method(pd.DataFrame, 'take', reason='deprecated') @@ -3987,12 +4014,18 @@ def value_counts(self, subset=None, sort=False, normalize=False, columns = subset or list(self.columns) if dropna: - dropped = self.dropna() + # Must include subset here because otherwise we spuriously drop NAs due + # to columns outside our subset. + dropped = self.dropna(subset=subset) else: dropped = self result = dropped.groupby(columns, dropna=dropna).size() + # Pandas 2 introduces new naming for the results. + if PD_VERSION >= (2,0): + result.name = "proportion" if normalize else "count" + if normalize: return result/dropped.length() else: @@ -4563,8 +4596,9 @@ def wrapper(self, *args, **kwargs): return _unliftable_agg(meth)(self, *args, **kwargs) to_group = self._ungrouped.proxy().index - is_categorical_grouping = any(to_group.get_level_values(i).is_categorical() - for i in self._grouping_indexes) + is_categorical_grouping = any( + isinstance(to_group.get_level_values(i).dtype, pd.CategoricalDtype) + for i in self._grouping_indexes) groupby_kwargs = self._kwargs group_keys = self._group_keys @@ -4616,8 +4650,9 @@ def wrapper(self, *args, **kwargs): to_group = self._ungrouped.proxy().index group_keys = self._group_keys - is_categorical_grouping = any(to_group.get_level_values(i).is_categorical() - for i in self._grouping_indexes) + is_categorical_grouping = any( + isinstance(to_group.get_level_values(i).dtype, pd.CategoricalDtype) + for i in self._grouping_indexes) groupby_kwargs = self._kwargs project = _maybe_project_func(self._projection) @@ -4670,7 +4705,10 @@ def _is_unliftable(agg_func): return _check_str_or_np_builtin(agg_func, UNLIFTABLE_AGGREGATIONS) NUMERIC_AGGREGATIONS = ['max', 'min', 'prod', 'sum', 'mean', 'median', 'std', - 'var', 'sem', 'mad', 'skew', 'kurt', 'kurtosis'] + 'var', 'sem', 'skew', 'kurt', 'kurtosis'] +# mad was removed in Pandas 2.0. +if PD_VERSION < (2, 0): + NUMERIC_AGGREGATIONS.append('mad') def _is_numeric(agg_func): return _check_str_or_np_builtin(agg_func, NUMERIC_AGGREGATIONS) @@ -4698,7 +4736,6 @@ class _DeferredGroupByCols(frame_base.DeferredFrame): idxmax = frame_base._elementwise_method('idxmax', base=DataFrameGroupBy) idxmin = frame_base._elementwise_method('idxmin', base=DataFrameGroupBy) last = frame_base._elementwise_method('last', base=DataFrameGroupBy) - mad = frame_base._elementwise_method('mad', base=DataFrameGroupBy) max = frame_base._elementwise_method('max', base=DataFrameGroupBy) mean = frame_base._elementwise_method('mean', base=DataFrameGroupBy) median = frame_base._elementwise_method('median', base=DataFrameGroupBy) @@ -4717,8 +4754,11 @@ class _DeferredGroupByCols(frame_base.DeferredFrame): DataFrameGroupBy, 'tail', explanation=_PEEK_METHOD_EXPLANATION) take = frame_base.wont_implement_method( DataFrameGroupBy, 'take', reason='deprecated') - tshift = frame_base._elementwise_method('tshift', base=DataFrameGroupBy) var = frame_base._elementwise_method('var', base=DataFrameGroupBy) + # These already deprecated methods were removed in Pandas 2.0 + if PD_VERSION < (2, 0): + mad = frame_base._elementwise_method('mad', base=DataFrameGroupBy) + tshift = frame_base._elementwise_method('tshift', base=DataFrameGroupBy) @property # type: ignore @frame_base.with_docs_from(DataFrameGroupBy) @@ -4895,9 +4935,9 @@ def __setitem__(self, index, value): class _DeferredStringMethods(frame_base.DeferredBase): - @frame_base.with_docs_from(pd.core.strings.StringMethods) - @frame_base.args_to_kwargs(pd.core.strings.StringMethods) - @frame_base.populate_defaults(pd.core.strings.StringMethods) + @frame_base.with_docs_from(pd.Series.str) + @frame_base.args_to_kwargs(pd.Series.str) + @frame_base.populate_defaults(pd.Series.str) def cat(self, others, join, **kwargs): """If defined, ``others`` must be a :class:`DeferredSeries` or a ``list`` of ``DeferredSeries``.""" @@ -4937,8 +4977,8 @@ def func(*args): requires_partition_by=requires, preserves_partition_by=partitionings.Arbitrary())) - @frame_base.with_docs_from(pd.core.strings.StringMethods) - @frame_base.args_to_kwargs(pd.core.strings.StringMethods) + @frame_base.with_docs_from(pd.Series.str) + @frame_base.args_to_kwargs(pd.Series.str) def repeat(self, repeats): """``repeats`` must be an ``int`` or a :class:`DeferredSeries`. Lists are not supported because they make this operation order-sensitive.""" @@ -4975,8 +5015,8 @@ def repeat(self, repeats): raise TypeError("str.repeat(repeats=) value must be an int or a " f"DeferredSeries (encountered {type(repeats)}).") - @frame_base.with_docs_from(pd.core.strings.StringMethods) - @frame_base.args_to_kwargs(pd.core.strings.StringMethods) + @frame_base.with_docs_from(pd.Series.str) + @frame_base.args_to_kwargs(pd.Series.str) def get_dummies(self, **kwargs): """ Series must be categorical dtype. Please cast to ``CategoricalDtype`` @@ -5058,9 +5098,9 @@ def func(s): requires_partition_by=partitionings.Arbitrary(), preserves_partition_by=partitionings.Arbitrary())) - @frame_base.with_docs_from(pd.core.strings.StringMethods) - @frame_base.args_to_kwargs(pd.core.strings.StringMethods) - @frame_base.populate_defaults(pd.core.strings.StringMethods) + @frame_base.with_docs_from(pd.Series.str) + @frame_base.args_to_kwargs(pd.Series.str) + @frame_base.populate_defaults(pd.Series.str) def split(self, **kwargs): """ Like other non-deferred methods, dtype must be CategoricalDtype. @@ -5069,9 +5109,9 @@ def split(self, **kwargs): """ return self._split_helper(rsplit=False, **kwargs) - @frame_base.with_docs_from(pd.core.strings.StringMethods) - @frame_base.args_to_kwargs(pd.core.strings.StringMethods) - @frame_base.populate_defaults(pd.core.strings.StringMethods) + @frame_base.with_docs_from(pd.Series.str) + @frame_base.args_to_kwargs(pd.Series.str) + @frame_base.populate_defaults(pd.Series.str) def rsplit(self, **kwargs): """ Like other non-deferred methods, dtype must be CategoricalDtype. @@ -5149,17 +5189,17 @@ def func(df, *args, **kwargs): return func for method in ELEMENTWISE_STRING_METHODS: - if not hasattr(pd.core.strings.StringMethods, method): + if not hasattr(pd.Series.str, method): # older versions (1.0.x) don't support some of these methods continue setattr(_DeferredStringMethods, method, frame_base._elementwise_method(make_str_func(method), name=method, - base=pd.core.strings.StringMethods)) + base=pd.Series.str)) for method in NON_ELEMENTWISE_STRING_METHODS: - if not hasattr(pd.core.strings.StringMethods, method): + if not hasattr(pd.Series.str, method): # older versions (1.0.x) don't support some of these methods continue setattr(_DeferredStringMethods, @@ -5167,7 +5207,7 @@ def func(df, *args, **kwargs): frame_base._proxy_method( make_str_func(method), name=method, - base=pd.core.strings.StringMethods, + base=pd.Series.str, requires_partition_by=partitionings.Arbitrary(), preserves_partition_by=partitionings.Singleton())) @@ -5341,11 +5381,12 @@ def func(df, *args, **kwargs): 'second', 'time', 'timetz', - 'week', 'weekday', - 'weekofyear', 'year', ] +# Pandas 2 removed these. +if PD_VERSION < (2, 0): + ELEMENTWISE_DATETIME_PROPERTIES += ['week', 'weekofyear'] for method in ELEMENTWISE_DATETIME_PROPERTIES: setattr(_DeferredDatetimeMethods, @@ -5376,6 +5417,7 @@ def func(df, *args, **kwargs): name, frame_base._elementwise_method(name, restrictions={'level': None}, base=pd.Series)) + if hasattr(pd.DataFrame, name): setattr( DeferredDataFrame, diff --git a/sdks/python/apache_beam/dataframe/frames_test.py b/sdks/python/apache_beam/dataframe/frames_test.py index a2a8ef75f8853..6e32acefc61b8 100644 --- a/sdks/python/apache_beam/dataframe/frames_test.py +++ b/sdks/python/apache_beam/dataframe/frames_test.py @@ -17,9 +17,11 @@ import re import unittest import warnings +from typing import Dict import numpy as np import pandas as pd +import pytest from parameterized import parameterized import apache_beam as beam @@ -44,6 +46,10 @@ 'str': [str(i) for i in range(100)], }) +if PD_VERSION < (2, 0): + # All these are things that are fixed in the Pandas 2 transition. + pytestmark = pytest.mark.filterwarnings("ignore::FutureWarning") + def _get_deferred_args(*args): return [ @@ -187,6 +193,9 @@ def _run_test( if expected.index.is_unique: expected = expected.sort_index() actual = actual.sort_index() + elif isinstance(expected, pd.Series): + expected = expected.sort_values() + actual = actual.sort_values() else: expected = expected.sort_values(list(expected.columns)) actual = actual.sort_values(list(actual.columns)) @@ -688,6 +697,8 @@ def test_value_counts_with_nans(self): self._run_test(lambda df: df.value_counts(), df) self._run_test(lambda df: df.value_counts(normalize=True), df) + # Ensure we don't drop rows due to nan values in unused columns. + self._run_test(lambda df: df.value_counts('num_wings'), df) if PD_VERSION >= (1, 3): # dropna=False is new in pandas 1.3 @@ -797,6 +808,7 @@ def test_loc(self): self._run_test(lambda df: df.C.loc[df.A > 10], df) self._run_test(lambda df, s: df.loc[s.loc[1:3]], df, pd.Series(dates)) + @unittest.skipIf(PD_VERSION >= (2, 0), 'append removed in Pandas 2.0') def test_append_sort(self): # yapf: disable df1 = pd.DataFrame({'int': [1, 2, 3], 'str': ['a', 'b', 'c']}, @@ -853,6 +865,8 @@ def test_corrwith_bad_axis(self): self._run_error_test(lambda df: df.corrwith(df, axis=5), df) @unittest.skipIf(PD_VERSION < (1, 2), "na_action added in pandas 1.2.0") + @pytest.mark.filterwarnings( + "ignore:The default of observed=False is deprecated:FutureWarning") def test_applymap_na_action(self): # Replicates a doctest for na_action which is incompatible with # doctest framework @@ -863,6 +877,17 @@ def test_applymap_na_action(self): # TODO: generate proxy using naive type inference on fn check_proxy=False) + @unittest.skipIf(PD_VERSION < (2, 1), "map added in 2.1.0") + def test_map_na_action(self): + # Replicates a doctest for na_action which is incompatible with + # doctest framework + df = pd.DataFrame([[pd.NA, 2.12], [3.356, 4.567]]) + self._run_test( + lambda df: df.map(lambda x: len(str(x)), na_action='ignore'), + df, + # TODO: generate proxy using naive type inference on fn + check_proxy=False) + def test_dataframe_eval_query(self): df = pd.DataFrame(np.random.randn(20, 3), columns=['a', 'b', 'c']) self._run_test(lambda df: df.eval('foo = a + b - c'), df) @@ -985,6 +1010,7 @@ def test_series_fillna_series_as_value(self): self._run_test(lambda df, df2: df.A.fillna(df2.A), df, df2) + @unittest.skipIf(PD_VERSION >= (2, 0), 'append removed in Pandas 2.0') def test_append_verify_integrity(self): df1 = pd.DataFrame({'A': range(10), 'B': range(10)}, index=range(10)) df2 = pd.DataFrame({'A': range(10), 'B': range(10)}, index=range(9, 19)) @@ -1008,8 +1034,14 @@ def test_categorical_groupby(self): df = df.set_index('B') # TODO(BEAM-11190): These aggregations can be done in index partitions, but # it will require a little more complex logic - self._run_test(lambda df: df.groupby(level=0).sum(), df, nonparallel=True) - self._run_test(lambda df: df.groupby(level=0).mean(), df, nonparallel=True) + self._run_test( + lambda df: df.groupby(level=0, observed=False).sum(), + df, + nonparallel=True) + self._run_test( + lambda df: df.groupby(level=0, observed=False).mean(), + df, + nonparallel=True) def test_astype_categorical(self): df = pd.DataFrame({'A': np.arange(6), 'B': list('aabbca')}) @@ -1632,6 +1664,30 @@ def test_pivot_no_index_provided_on_multiindex(self): # https://github.com/pandas-dev/pandas/issues/40139 ALL_GROUPING_AGGREGATIONS = sorted( set(frames.ALL_AGGREGATIONS) - set(('kurt', 'kurtosis'))) +AGGREGATIONS_WHERE_NUMERIC_ONLY_DEFAULTS_TO_TRUE_IN_PANDAS_1 = set( + frames.ALL_AGGREGATIONS) - set(( + 'nunique', + 'size', + 'count', + 'idxmin', + 'idxmax', + 'mode', + 'rank', + 'all', + 'any', + 'describe')) + + +def numeric_only_kwargs_for_pandas_2(agg_type: str) -> Dict[str, bool]: + """Get proper arguments for numeric_only. + + Behavior for numeric_only in these methods changed in Pandas 2 to default + to False instead of True, so explicitly make it True in Pandas 2.""" + if PD_VERSION >= (2, 0) and ( + agg_type in AGGREGATIONS_WHERE_NUMERIC_ONLY_DEFAULTS_TO_TRUE_IN_PANDAS_1): + return {'numeric_only': True} + else: + return {} class GroupByTest(_AbstractFrameTest): @@ -1648,8 +1704,9 @@ def test_groupby_agg(self, agg_type): self.skipTest( "https://github.com/apache/beam/issues/20967: proxy generation of " "DataFrameGroupBy.describe fails in pandas < 1.2") + kwargs = numeric_only_kwargs_for_pandas_2(agg_type) self._run_test( - lambda df: df.groupby('group').agg(agg_type), + lambda df: df.groupby('group').agg(agg_type, **kwargs), GROUPBY_DF, check_proxy=False) @@ -1659,8 +1716,10 @@ def test_groupby_with_filter(self, agg_type): self.skipTest( "https://github.com/apache/beam/issues/20967: proxy generation of " "DataFrameGroupBy.describe fails in pandas < 1.2") + kwargs = numeric_only_kwargs_for_pandas_2(agg_type) self._run_test( - lambda df: getattr(df[df.foo > 30].groupby('group'), agg_type)(), + lambda df: getattr(df[df.foo > 30].groupby('group'), agg_type) + (**kwargs), GROUPBY_DF, check_proxy=False) @@ -1671,8 +1730,9 @@ def test_groupby(self, agg_type): "https://github.com/apache/beam/issues/20967: proxy generation of " "DataFrameGroupBy.describe fails in pandas < 1.2") + kwargs = numeric_only_kwargs_for_pandas_2(agg_type) self._run_test( - lambda df: getattr(df.groupby('group'), agg_type)(), + lambda df: getattr(df.groupby('group'), agg_type)(**kwargs), GROUPBY_DF, check_proxy=False) @@ -1683,8 +1743,10 @@ def test_groupby_series(self, agg_type): "https://github.com/apache/beam/issues/20967: proxy generation of " "DataFrameGroupBy.describe fails in pandas < 1.2") + kwargs = numeric_only_kwargs_for_pandas_2(agg_type) self._run_test( - lambda df: getattr(df[df.foo > 40].groupby(df.group), agg_type)(), + lambda df: getattr(df[df.foo > 40].groupby(df.group), agg_type) + (**kwargs), GROUPBY_DF, check_proxy=False) @@ -1715,12 +1777,15 @@ def test_groupby_project_series(self, agg_type): "https://github.com/apache/beam/issues/20895: " "SeriesGroupBy.{corr, cov} do not raise the expected error.") - self._run_test(lambda df: getattr(df.groupby('group').foo, agg_type)(), df) - self._run_test(lambda df: getattr(df.groupby('group').bar, agg_type)(), df) + kwargs = numeric_only_kwargs_for_pandas_2(agg_type) self._run_test( - lambda df: getattr(df.groupby('group')['foo'], agg_type)(), df) + lambda df: getattr(df.groupby('group').foo, agg_type)(**kwargs), df) self._run_test( - lambda df: getattr(df.groupby('group')['bar'], agg_type)(), df) + lambda df: getattr(df.groupby('group').bar, agg_type)(**kwargs), df) + self._run_test( + lambda df: getattr(df.groupby('group')['foo'], agg_type)(**kwargs), df) + self._run_test( + lambda df: getattr(df.groupby('group')['bar'], agg_type)(**kwargs), df) @parameterized.expand(ALL_GROUPING_AGGREGATIONS) def test_groupby_project_dataframe(self, agg_type): @@ -1728,8 +1793,10 @@ def test_groupby_project_dataframe(self, agg_type): self.skipTest( "https://github.com/apache/beam/issues/20967: proxy generation of " "DataFrameGroupBy.describe fails in pandas < 1.2") + kwargs = numeric_only_kwargs_for_pandas_2(agg_type) self._run_test( - lambda df: getattr(df.groupby('group')[['bar', 'baz']], agg_type)(), + lambda df: getattr(df.groupby('group')[['bar', 'baz']], agg_type) + (**kwargs), GROUPBY_DF, check_proxy=False) @@ -1758,9 +1825,10 @@ def test_groupby_errors_non_existent_label(self): def test_groupby_callable(self): df = GROUPBY_DF - - self._run_test(lambda df: df.groupby(lambda x: x % 2).foo.sum(), df) - self._run_test(lambda df: df.groupby(lambda x: x % 5).median(), df) + kwargs = numeric_only_kwargs_for_pandas_2('sum') + self._run_test(lambda df: df.groupby(lambda x: x % 2).foo.sum(**kwargs), df) + kwargs = numeric_only_kwargs_for_pandas_2('median') + self._run_test(lambda df: df.groupby(lambda x: x % 5).median(**kwargs), df) def test_groupby_apply(self): df = GROUPBY_DF @@ -1788,8 +1856,8 @@ def test_groupby_apply_preserves_column_order(self): df = GROUPBY_DF self._run_test( - lambda df: df[['foo', 'group', 'bar']].groupby('group').apply( - lambda x: x), + lambda df: df[['foo', 'group', 'bar']].groupby( + 'group', group_keys=False).apply(lambda x: x), df) def test_groupby_transform(self): @@ -1815,8 +1883,9 @@ def test_groupby_transform(self): def test_groupby_pipe(self): df = GROUPBY_DF - - self._run_test(lambda df: df.groupby('group').pipe(lambda x: x.sum()), df) + kwargs = numeric_only_kwargs_for_pandas_2('sum') + self._run_test( + lambda df: df.groupby('group').pipe(lambda x: x.sum(**kwargs)), df) self._run_test( lambda df: df.groupby('group')['bool'].pipe(lambda x: x.any()), df) self._run_test( @@ -1886,6 +1955,8 @@ def test_groupby_sum_min_count(self): self._run_test(lambda df: df.groupby('group').sum(min_count=2), df) + @unittest.skipIf( + PD_VERSION >= (2, 0), "dtypes on groups is deprecated in Pandas 2.") def test_groupby_dtypes(self): self._run_test( lambda df: df.groupby('group').dtypes, GROUPBY_DF, check_proxy=False) @@ -1898,14 +1969,14 @@ def test_dataframe_groupby_series(self, agg_type): self.skipTest( "https://github.com/apache/beam/issues/20967: proxy generation of " "DataFrameGroupBy.describe fails in pandas < 1.2") + + def agg(df, group_by): + kwargs = numeric_only_kwargs_for_pandas_2(agg_type) + return df[df.foo > 40].groupby(group_by).agg(agg_type, **kwargs) + + self._run_test(lambda df: agg(df, df.group), GROUPBY_DF, check_proxy=False) self._run_test( - lambda df: df[df.foo > 40].groupby(df.group).agg(agg_type), - GROUPBY_DF, - check_proxy=False) - self._run_test( - lambda df: df[df.foo > 40].groupby(df.foo % 3).agg(agg_type), - GROUPBY_DF, - check_proxy=False) + lambda df: agg(df, df.foo % 3), GROUPBY_DF, check_proxy=False) @parameterized.expand(ALL_GROUPING_AGGREGATIONS) def test_series_groupby_series(self, agg_type): @@ -1946,6 +2017,12 @@ def test_groupby_multiindex_keep_nans(self): lambda df: df.groupby(['foo', 'bar'], dropna=False).sum(), GROUPBY_DF) +NONPARALLEL_METHODS = ['quantile', 'describe', 'median', 'sem'] +# mad was removed in pandas 2 +if PD_VERSION < (2, 0): + NONPARALLEL_METHODS.append('mad') + + class AggregationTest(_AbstractFrameTest): """Tests for global aggregation methods on DataFrame/Series.""" @@ -1955,7 +2032,7 @@ class AggregationTest(_AbstractFrameTest): def test_series_agg(self, agg_method): s = pd.Series(list(range(16))) - nonparallel = agg_method in ('quantile', 'describe', 'median', 'sem', 'mad') + nonparallel = agg_method in NONPARALLEL_METHODS # TODO(https://github.com/apache/beam/issues/20926): max and min produce # the wrong proxy @@ -1974,7 +2051,7 @@ def test_series_agg(self, agg_method): def test_series_agg_method(self, agg_method): s = pd.Series(list(range(16))) - nonparallel = agg_method in ('quantile', 'describe', 'median', 'sem', 'mad') + nonparallel = agg_method in NONPARALLEL_METHODS # TODO(https://github.com/apache/beam/issues/20926): max and min produce # the wrong proxy @@ -1990,7 +2067,7 @@ def test_series_agg_method(self, agg_method): def test_dataframe_agg(self, agg_method): df = pd.DataFrame({'A': [1, 2, 3, 4], 'B': [2, 3, 5, 7]}) - nonparallel = agg_method in ('quantile', 'describe', 'median', 'sem', 'mad') + nonparallel = agg_method in NONPARALLEL_METHODS # TODO(https://github.com/apache/beam/issues/20926): max and min produce # the wrong proxy @@ -2007,7 +2084,7 @@ def test_dataframe_agg(self, agg_method): def test_dataframe_agg_method(self, agg_method): df = pd.DataFrame({'A': [1, 2, 3, 4], 'B': [2, 3, 5, 7]}) - nonparallel = agg_method in ('quantile', 'describe', 'median', 'sem', 'mad') + nonparallel = agg_method in NONPARALLEL_METHODS # TODO(https://github.com/apache/beam/issues/20926): max and min produce # the wrong proxy @@ -2036,6 +2113,7 @@ def test_dataframe_agg_modes(self): self._run_test(lambda df: df.agg({'A': ['sum', 'mean']}), df) self._run_test(lambda df: df.agg({'A': ['sum', 'mean'], 'B': 'min'}), df) + @unittest.skipIf(PD_VERSION >= (2, 0), "level argument removed in Pandas 2") def test_series_agg_level(self): self._run_test( lambda df: df.set_index(['group', 'foo']).bar.count(level=0), @@ -2059,6 +2137,7 @@ def test_series_agg_level(self): lambda df: df.set_index(['group', 'foo']).bar.median(level=1), GROUPBY_DF) + @unittest.skipIf(PD_VERSION >= (2, 0), "level argument removed in Pandas 2") def test_dataframe_agg_level(self): self._run_test( lambda df: df.set_index(['group', 'foo']).count(level=0), GROUPBY_DF) @@ -2101,6 +2180,7 @@ def test_dataframe_agg_level(self): level=1, numeric_only=True), GROUPBY_DF) + @unittest.skipIf(PD_VERSION >= (2, 0), "level argument removed in Pandas 2") def test_series_agg_multifunc_level(self): # level= is ignored for multiple agg fns self._run_test( @@ -2123,6 +2203,7 @@ def test_series_mean_skipna(self): self._run_test(lambda df: df.two.mean(skipna=True), df) self._run_test(lambda df: df.three.mean(skipna=True), df) + @unittest.skipIf(PD_VERSION >= (2, 0), "level argument removed in Pandas 2") def test_dataframe_agg_multifunc_level(self): # level= is ignored for multiple agg fns self._run_test( @@ -2226,6 +2307,7 @@ def test_df_agg_method_invalid_kwarg_raises(self): self._run_error_test( lambda df: df.median(min_count=3, numeric_only=True), GROUPBY_DF) + @unittest.skipIf(PD_VERSION >= (2, 0), "level argument removed in Pandas 2") def test_agg_min_count(self): df = pd.DataFrame({ 'good': [1, 2, 3, np.nan], @@ -2930,7 +3012,7 @@ class DocstringTest(unittest.TestCase): (frames.DeferredDataFrame, pd.DataFrame), (frames.DeferredSeries, pd.Series), #(frames._DeferredIndex, pd.Index), - (frames._DeferredStringMethods, pd.core.strings.StringMethods), + (frames._DeferredStringMethods, pd.Series.str), ( frames._DeferredCategoricalMethods, pd.core.arrays.categorical.CategoricalAccessor), diff --git a/sdks/python/apache_beam/dataframe/pandas_doctests_test.py b/sdks/python/apache_beam/dataframe/pandas_doctests_test.py index 56eddd3cfb928..4fb05780fbec6 100644 --- a/sdks/python/apache_beam/dataframe/pandas_doctests_test.py +++ b/sdks/python/apache_beam/dataframe/pandas_doctests_test.py @@ -164,6 +164,9 @@ def test_ndframe_tests(self): ' key=lambda x: np.argsort(index_natsorted(df["time"]))\n' ')' ], + # TODO(https://github.com/apache/beam/issues/28559): Re-enable when + # bug is fixed. + 'pandas.core.generic.NDFrame.xs': ['*'], **skip_writes }) self.assertEqual(result.failed, 0) @@ -296,13 +299,19 @@ def test_dataframe_tests(self): 'pandas.core.frame.DataFrame.value_counts': [ 'df.value_counts(dropna=False)' ], + + 'pandas.core.frame.DataFrame.to_timestamp': ['*'] }, skip={ - # DataFrame construction from a dictionary and - # Series requires using the len() function, which - # is a non-deferred operation that we do not allow + # DataFrame construction from a dictionary, Series, or other + # DataFrame requires using the len() function, which is a + # non-deferred operation that we do not allow 'pandas.core.frame.DataFrame': [ 'pd.DataFrame(data=d, index=[0, 1, 2, 3])', + 'df = pd.DataFrame(data=ser, index=["a", "c"])', + 'df', + 'df2 = pd.DataFrame(data=df1, index=["a", "c"])', + 'df2', ], # s2 created with reindex 'pandas.core.frame.DataFrame.dot': [ @@ -361,15 +370,17 @@ def test_dataframe_tests(self): # actually raise NotImplementedError 'pandas.core.frame.DataFrame.pivot_table': ['*'], # Expected to raise a ValueError, but we raise NotImplementedError + # pylint: disable=line-too-long 'pandas.core.frame.DataFrame.pivot': [ "df.pivot(index='foo', columns='bar', values='baz')", "df.pivot(index='foo', columns='bar')['baz']", "df.pivot(index='foo', columns='bar', values=['baz', 'zoo'])", - # pylint: disable=line-too-long 'df.pivot(index="lev1", columns=["lev2", "lev3"],values="values")', - # pylint: disable=line-too-long - 'df.pivot(index=["lev1", "lev2"], columns=["lev3"],values="values")' + 'df.pivot(index=["lev1", "lev2"], columns=["lev3"],values="values")', + 'df.pivot(index="lev1", columns=["lev2", "lev3"], values="values")', + 'df.pivot(index=["lev1", "lev2"], columns=["lev3"], values="values")', ], + # pylint: enable=line-too-long 'pandas.core.frame.DataFrame.append': [ 'df', # pylint: disable=line-too-long @@ -511,6 +522,8 @@ def test_series_tests(self): 'ser.groupby(["a", "b", "a", np.nan]).mean()', 'ser.groupby(["a", "b", "a", np.nan], dropna=False).mean()', ], + 'pandas.core.series.Series.to_period': ['*'], + 'pandas.core.series.Series.to_timestamp': ['*'], }, skip={ # Relies on setting values with iloc @@ -535,6 +548,8 @@ def test_series_tests(self): 'pandas.core.series.Series.idxmin': ['s.idxmin()'], 'pandas.core.series.Series.idxmax': ['s.idxmax()'], 'pandas.core.series.Series.duplicated': ['*'], + # Relies on setting index. + 'pandas.core.series.Series.rename_axis': ['*'], 'pandas.core.series.Series.set_axis': ['*'], 'pandas.core.series.Series.nonzero': ['*'], 'pandas.core.series.Series.pop': ['ser'], # testing side effect @@ -710,6 +725,7 @@ def test_groupby_tests(self): 'pandas.core.groupby.groupby.GroupBy.nth': ['*'], 'pandas.core.groupby.groupby.GroupBy.cumcount': ['*'], 'pandas.core.groupby.groupby.GroupBy.resample': ['*'], + 'pandas.core.groupby.groupby.GroupBy.rolling': ['*'], }, not_implemented_ok={ 'pandas.core.groupby.groupby.GroupBy.first': ['*'], @@ -764,16 +780,21 @@ def test_groupby_tests(self): 'df.fillna(method=\'ffill\')', 'df.fillna(method="ffill")', 'df.fillna(value=values, limit=1)', + 'df.groupby("key").fillna(method="ffill")', + 'df.groupby("key").fillna(method="bfill")', + 'df.groupby("key").fillna(method="ffill", limit=1)', ], 'pandas.core.groupby.generic.SeriesGroupBy.fillna': [ 'df.fillna(method=\'ffill\')', 'df.fillna(method="ffill")', 'df.fillna(value=values, limit=1)', ], + 'pandas.core.groupby.groupby.GroupBy.tail': ['*'], }, not_implemented_ok={ 'pandas.core.groupby.generic.DataFrameGroupBy.idxmax': ['*'], 'pandas.core.groupby.generic.DataFrameGroupBy.idxmin': ['*'], + 'pandas.core.groupby.generic.DataFrameGroupBy.transform': ['*'], 'pandas.core.groupby.generic.SeriesGroupBy.transform': ['*'], 'pandas.core.groupby.generic.SeriesGroupBy.idxmax': ['*'], 'pandas.core.groupby.generic.SeriesGroupBy.idxmin': ['*'], @@ -794,14 +815,6 @@ def test_groupby_tests(self): # These examples rely on grouping by a list 'pandas.core.groupby.generic.SeriesGroupBy.aggregate': ['*'], 'pandas.core.groupby.generic.DataFrameGroupBy.aggregate': ['*'], - 'pandas.core.groupby.generic.SeriesGroupBy.transform': [ - # Dropping invalid columns during a transform is unsupported. - 'grouped.transform(lambda x: (x - x.mean()) / x.std())' - ], - 'pandas.core.groupby.generic.DataFrameGroupBy.transform': [ - # Dropping invalid columns during a transform is unsupported. - 'grouped.transform(lambda x: (x - x.mean()) / x.std())' - ], # Skipped idxmax/idxmin due an issue with the test framework 'pandas.core.groupby.generic.SeriesGroupBy.idxmin': ['s.idxmin()'], 'pandas.core.groupby.generic.SeriesGroupBy.idxmax': ['s.idxmax()'], @@ -811,7 +824,24 @@ def test_groupby_tests(self): # pylint: disable=line-too-long "df.groupby('gender', as_index=False).value_counts(normalize=True)", ], - }) + # These examples rely on grouping by a list + 'pandas.core.groupby.generic.SeriesGroupBy.fillna': ['*'], + # These examples rely on grouping by a list + 'pandas.core.groupby.generic.DataFrameGroupBy.fillna': ['*'], + # These examples rely on grouping by a list + 'pandas.core.groupby.generic.SeriesGroupBy.take': ['*'], + # These examples rely on grouping by a list + 'pandas.core.groupby.generic.DataFrameGroupBy.take': ['*'], + # Named aggregation not supported yet. + 'pandas.core.groupby.generic.NamedAgg': [ + 'df.groupby("key").agg(result_a=agg_a, result_1=agg_1)' + ], + # These examples rely on grouping by a list + 'pandas.core.groupby.generic.DataFrameGroupBy.transform': ['*'], + # These examples rely on grouping by a list + 'pandas.core.groupby.generic.SeriesGroupBy.transform': ['*'], + }, + ) self.assertEqual(result.failed, 0) def test_top_level(self): @@ -843,7 +873,6 @@ def test_top_level(self): 'pivot_table': ['*'], 'qcut': ['*'], 'reset_option': ['*'], - 'set_eng_float_format': ['*'], 'set_option': ['*'], 'to_numeric': ['*'], 'to_timedelta': ['*'], diff --git a/sdks/python/apache_beam/dataframe/pandas_top_level_functions.py b/sdks/python/apache_beam/dataframe/pandas_top_level_functions.py index 39df3f25a2e85..ce36dbeb09ad7 100644 --- a/sdks/python/apache_beam/dataframe/pandas_top_level_functions.py +++ b/sdks/python/apache_beam/dataframe/pandas_top_level_functions.py @@ -162,6 +162,7 @@ def concat( period_range = _defer_to_pandas('period_range') pivot = _call_on_first_arg('pivot') pivot_table = _call_on_first_arg('pivot_table') + set_eng_float_format = _defer_to_pandas('set_eng_float_format') show_versions = _defer_to_pandas('show_versions') test = frame_base.wont_implement_method( pd, diff --git a/sdks/python/apache_beam/dataframe/transforms_test.py b/sdks/python/apache_beam/dataframe/transforms_test.py index b824bc56c2f95..a143606cc9130 100644 --- a/sdks/python/apache_beam/dataframe/transforms_test.py +++ b/sdks/python/apache_beam/dataframe/transforms_test.py @@ -213,8 +213,8 @@ def test_batching_beam_row_input(self): with beam.Pipeline() as p: result = ( p - | beam.Create([(u'Falcon', 380.), (u'Falcon', 370.), (u'Parrot', 24.), - (u'Parrot', 26.)]) + | beam.Create([('Falcon', 380.), ('Falcon', 370.), ('Parrot', 24.), + ('Parrot', 26.)]) | beam.Map(lambda tpl: beam.Row(Animal=tpl[0], Speed=tpl[1])) | transforms.DataframeTransform( lambda df: df.groupby('Animal').mean(), include_indexes=True)) @@ -225,8 +225,8 @@ def test_batching_beam_row_to_dataframe(self): with beam.Pipeline() as p: df = convert.to_dataframe( p - | beam.Create([(u'Falcon', 380.), (u'Falcon', 370.), ( - u'Parrot', 24.), (u'Parrot', 26.)]) + | beam.Create([('Falcon', 380.), ('Falcon', 370.), ('Parrot', 24.), ( + 'Parrot', 26.)]) | beam.Map(lambda tpl: beam.Row(Animal=tpl[0], Speed=tpl[1]))) result = convert.to_pcollection( @@ -260,8 +260,8 @@ def test_unbatching_series(self): with beam.Pipeline() as p: result = ( p - | beam.Create([(u'Falcon', 380.), (u'Falcon', 370.), (u'Parrot', 24.), - (u'Parrot', 26.)]) + | beam.Create([('Falcon', 380.), ('Falcon', 370.), ('Parrot', 24.), + ('Parrot', 26.)]) | beam.Map(lambda tpl: beam.Row(Animal=tpl[0], Speed=tpl[1])) | transforms.DataframeTransform(lambda df: df.Animal)) diff --git a/sdks/python/apache_beam/examples/OWNERS b/sdks/python/apache_beam/examples/OWNERS deleted file mode 100644 index 7dce27abe1264..0000000000000 --- a/sdks/python/apache_beam/examples/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - aaltay - - charlesccychen diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes.py b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes.py index 224a2ad586c1d..ede667fd9eff9 100644 --- a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes.py +++ b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes.py @@ -68,7 +68,7 @@ def run(argv=None): parser = argparse.ArgumentParser() parser.add_argument( '--input', - default='clouddataflow-readonly:samples.weather_stations', + default='apache-beam-testing.samples.weather_stations', help=( 'Input BigQuery table to process specified as: ' 'PROJECT:DATASET.TABLE or DATASET.TABLE.')) diff --git a/sdks/python/apache_beam/examples/cookbook/bigtableio_it_test.py b/sdks/python/apache_beam/examples/cookbook/bigtableio_it_test.py index 55effaa11a7a1..98023fbc624c1 100644 --- a/sdks/python/apache_beam/examples/cookbook/bigtableio_it_test.py +++ b/sdks/python/apache_beam/examples/cookbook/bigtableio_it_test.py @@ -54,7 +54,7 @@ import google.cloud.bigtable.instance EXISTING_INSTANCES = [] # type: List[google.cloud.bigtable.instance.Instance] -LABEL_KEY = u'python-bigtable-beam' +LABEL_KEY = 'python-bigtable-beam' label_stamp = datetime.datetime.utcnow().replace(tzinfo=UTC) label_stamp_micros = _microseconds_from_datetime(label_stamp) LABELS = {LABEL_KEY: str(label_stamp_micros)} diff --git a/sdks/python/apache_beam/examples/cookbook/filters.py b/sdks/python/apache_beam/examples/cookbook/filters.py index fda07064fa0c0..daa01b0658bc2 100644 --- a/sdks/python/apache_beam/examples/cookbook/filters.py +++ b/sdks/python/apache_beam/examples/cookbook/filters.py @@ -79,7 +79,7 @@ def run(argv=None): parser.add_argument( '--input', help='BigQuery table to read from.', - default='clouddataflow-readonly:samples.weather_stations') + default='apache-beam-testing.samples.weather_stations') parser.add_argument( '--output', required=True, help='BigQuery table to write to.') parser.add_argument( diff --git a/sdks/python/apache_beam/examples/inference/README.md b/sdks/python/apache_beam/examples/inference/README.md index 326ec4b4a0968..cd92d9c127ee0 100644 --- a/sdks/python/apache_beam/examples/inference/README.md +++ b/sdks/python/apache_beam/examples/inference/README.md @@ -29,7 +29,6 @@ Some examples are also used in [our benchmarks](http://s.apache.org/beam-communi You must have the latest (possibly unreleased) `apache-beam` or greater installed from the Beam repo in order to run these pipelines, because some examples rely on the latest features that are actively in development. To install Beam, run the following from the `sdks/python` directory: ``` -pip install -r build-requirements.txt pip install -e .[gcp] ``` @@ -42,6 +41,7 @@ The RunInference API supports the Tensorflow framework. To use Tensorflow locall pip install tensorflow==2.12.0 ``` + ### PyTorch dependencies The following installation requirements are for the files used in these examples. @@ -65,6 +65,21 @@ For installation of the `torch` dependency on a distributed runner such as Dataf [PyPI dependency instructions](https://beam.apache.org/documentation/sdks/python-pipeline-dependencies/#pypi-dependencies). +### Transformers dependencies + +The following installation requirement is for the Hugging Face model handler examples. + +The RunInference API supports loading models from the Hugging Face Hub. To use it, first install `transformers`. +``` +pip install transformers==4.30.0 +``` +Additional dependicies for PyTorch and TensorFlow may need to be installed separately: +``` +pip install tensorflow==2.12.0 +pip install torch==1.10.0 +``` + + ### TensorRT dependencies The RunInference API supports TensorRT SDK for high-performance deep learning inference with NVIDIA GPUs. @@ -209,6 +224,62 @@ This writes the output to the `predictions.csv` with contents like: ``` Each line has data separated by a semicolon ";". The first item is the file name. The second item is a list of predicted instances. +--- +## Per Key Image segmentation + +[`pytorch_model_per_key_image_segmentation.py`](./pytorch_model_per_key_image_segmentation.py) contains an implementation for a RunInference pipeline that performs image segementation using multiple different trained models based on the `maskrcnn_resnet50_fpn` architecture. + +The pipeline reads images, performs basic preprocessing, passes the images to the PyTorch implementation of RunInference, and then writes predictions to a text file. + +### Dataset and model for image segmentation + +To use this transform, you need a dataset and model for image segmentation. If you've already done the previous example (Image segmentation with pytorch_image_segmentation.py you can reuse the results from some of those setup steps). + +1. Create a directory named `IMAGES_DIR`. Create or download images and put them in this directory. The directory is not required if image names in the input file `IMAGE_FILE_NAMES.txt` you create in step 2 have absolute paths. +A popular dataset is from [Coco](https://cocodataset.org/#home). Follow their instructions to download the images. +2. Create a file named `IMAGE_FILE_NAMES.txt` that contains the absolute paths of each of the images in `IMAGES_DIR` that you want to use to run image segmentation. The path to the file can be different types of URIs such as your local file system, an AWS S3 bucket, or a GCP Cloud Storage bucket. For example: +``` +/absolute/path/to/image1.jpg +/absolute/path/to/image2.jpg +``` +3. Download the [maskrcnn_resnet50_fpn](https://pytorch.org/vision/0.12/models.html#id70) and [maskrcnn_resnet50_fpn_v2](https://pytorch.org/vision/main/models/generated/torchvision.models.detection.maskrcnn_resnet50_fpn_v2.html) models from Pytorch's repository of pretrained models. These models require the torchvision library. To download this model, run the following commands from a Python shell: +``` +import torch +from torchvision.models.detection import maskrcnn_resnet50_fpn +from torchvision.models.detection import maskrcnn_resnet50_fpn_v2 +model = maskrcnn_resnet50_fpn(pretrained=True) +torch.save(model.state_dict(), 'maskrcnn_resnet50_fpn.pth') # You can replace maskrcnn_resnet50_fpn.pth with your preferred file name for your model state dictionary. +model = maskrcnn_resnet50_fpn_v2(pretrained=True) +torch.save(model.state_dict(), 'maskrcnn_resnet50_fpn_v2.pth') # You can replace maskrcnn_resnet50_fpn_v2.pth with your preferred file name for your model state dictionary. +``` +4. Note a path to an `OUTPUT` file that can be used by the pipeline to write the predictions. + +### Running `pytorch_model_per_key_image_segmentation.py` + +To run the image segmentation pipeline locally, use the following command: +```sh +python -m apache_beam.examples.inference.pytorch_model_per_key_image_segmentation \ + --input IMAGE_FILE_NAMES \ + --images_dir IMAGES_DIR \ + --output OUTPUT \ + --model_state_dict_paths MODEL_STATE_DICT1,MODEL_STATE_DICT2 +``` +`images_dir` is only needed if your `IMAGE_FILE_NAMES.txt` file contains relative paths (they will be relative from `IMAGES_DIR`). + +For example, if you've followed the naming conventions recommended above: +```sh +python -m apache_beam.examples.inference.pytorch_model_per_key_image_segmentation \ + --input IMAGE_FILE_NAMES.txt \ + --output predictions.csv \ + --model_state_dict_path 'maskrcnn_resnet50_fpn.pth,maskrcnn_resnet50_fpn_v2.pth' +``` +This writes the output to the `predictions.csv` with contents like: +``` +/Users/dannymccormick/Downloads/images/datasets_coco_raw-data_val2017_000000000139.jpg --- v1 predictions: ['chair', 'tv','potted plant'] --- v2 predictions: ['motorcycle', 'frisbee', 'couch'] +... +``` +Each image has 2 pieces of associated data - `v1 predictions` and `v2 predictions` corresponding to the version of the model that was used for segmentation. + --- ## Object Detection @@ -687,3 +758,128 @@ MilkQualityAggregation(bad_quality_measurements=6, medium_quality_measurements=4 MilkQualityAggregation(bad_quality_measurements=3, medium_quality_measurements=3, high_quality_measurements=3) MilkQualityAggregation(bad_quality_measurements=1, medium_quality_measurements=2, high_quality_measurements=1) ``` + +--- +## Language modeling with Hugging Face Hub + +[`huggingface_language_modeling.py`](./huggingface_language_modeling.py) contains an implementation for a RunInference pipeline that performs masked language modeling (that is, decoding a masked token in a sentence) using the `AutoModelForMaskedLM` architecture from Hugging Face. + +The pipeline reads sentences, performs basic preprocessing to convert the last word into a `` token, passes the masked sentence to the Hugging Face implementation of RunInference, and then writes the predictions to a text file. + +### Dataset and model for language modeling + +To use this transform, you need a dataset and model for language modeling. + +1. Choose a checkpoint to load from Hugging Face Hub, eg:[MaskedLanguageModel](https://huggingface.co/stevhliu/my_awesome_eli5_mlm_model). +2. (Optional) Create a file named `SENTENCES.txt` that contains sentences to feed into the model. The content of the file should be similar to the following example: +``` +The capital of France is Paris . +He looked up and saw the sun and stars . +... +``` + +### Running `huggingface_language_modeling.py` + +To run the language modeling pipeline locally, use the following command: +```sh +python -m apache_beam.examples.inference.huggingface_language_modeling \ + --input SENTENCES \ + --output OUTPUT \ + --model_name REPOSITORY_ID +``` +The `input` argument is optional. If none is provided, it will run the pipeline with some +example sentences. + +For example, if you've followed the naming conventions recommended above: +```sh +python -m apache_beam.examples.inference.huggingface_language_modeling \ + --input SENTENCES.txt \ + --output predictions.csv \ + --model_name "stevhliu/my_awesome_eli5_mlm_model" +``` +Or, using the default example sentences: +```sh +python -m apache_beam.examples.inference.huggingface_language_modeling \ + --output predictions.csv \ + --model_name "stevhliu/my_awesome_eli5_mlm_model" +``` + +This writes the output to the `predictions.csv` with contents like: +``` +The capital of France is Paris .;paris +He looked up and saw the sun and stars .;moon +... +``` +Each line has data separated by a semicolon ";". +The first item is the input sentence. The model masks the last word and tries to predict it; +the second item is the word that the model predicts for the mask. + +--- +## Image classifcation with Vertex AI + +[`vertex_ai_image_classification.py`](./vertex_ai_image_classification.py) contains an implementation for a RunInference pipeline that performs image classification using a model hosted on Vertex AI (based on https://cloud.google.com/vertex-ai/docs/tutorials/image-recognition-custom). + +The pipeline reads image urls, performs basic preprocessing to convert them into a List of floats, passes the masked sentence to the Vertex AI implementation of RunInference, and then writes the predictions to a text file. + +### Dataset and model for image classification + +To use this transform, you need a dataset and model hosted on Vertex AI for image classification. + +1. Train a model by following the tutorial at https://cloud.google.com/vertex-ai/docs/tutorials/image-recognition-custom +2. Create a file named `IMAGE_FILE_NAMES.txt` that contains the absolute paths of each of the images in `IMAGES_DIR` that you want to use to run image classification. The path to the file can be different types of URIs such as your local file system, an AWS S3 bucket, or a GCP Cloud Storage bucket. For example: +``` +/absolute/path/to/image1.jpg +/absolute/path/to/image2.jpg +``` + +### Running `vertex_ai_image_classification.py` + +To run the image classification pipeline locally, use the following command: +```sh +python -m apache_beam.examples.inference.vertex_ai_image_classification \ + --endpoint_id '' \ + --endpoint_project '' \ + --endpoint_region '' \ + --input 'path/to/IMAGE_FILE_NAMES.txt' \ + --output 'path/to/output/file.txt' +``` + +This writes the output to the output file with contents like: +``` +path/to/my/image: tulips (90) +path/to/my/image2: dandelions (78) +... +``` +Each line represents a prediction of the flower type along with the confidence in that prediction. + +--- +## Text classifcation with a Vertex AI LLM + +[`vertex_ai_llm_text_classification.py`](./vertex_ai_llm_text_classification.py) contains an implementation for a RunInference pipeline that performs image classification using a model hosted on Vertex AI (based on https://cloud.google.com/vertex-ai/docs/tutorials/image-recognition-custom). + +The pipeline reads image urls, performs basic preprocessing to convert them into a List of floats, passes the masked sentence to the Vertex AI implementation of RunInference, and then writes the predictions to a text file. + +### Dataset and model for image classification + +To use this transform, you need a model hosted on Vertex AI for text classification. +You can get this by tuning the text-bison model following the instructions here - +https://cloud.google.com/vertex-ai/docs/generative-ai/models/tune-models#create_a_model_tuning_job + +### Running `vertex_ai_llm_text_classification.py` + +To run the text classification pipeline locally, use the following command: +```sh +python -m apache_beam.examples.inference.vertex_ai_llm_text_classification \ + --endpoint_id '' \ + --endpoint_project '' \ + --endpoint_region '' +``` + +This writes the output to the output file with contents like: +``` +('What is 5+2?', PredictionResult(example={'prompt': 'What is 5+2?'}, inference={'content': '7', 'citationMetadata': {'citations': []}, 'safetyAttributes': {'blocked': False, 'scores': [], 'categories': []}}, model_id='6795590989097467904')) +... +``` +Each line represents a tuple containing the example, a [PredictionResult](https://beam.apache.org/releases/pydoc/2.40.0/apache_beam.ml.inference.base.html#apache_beam.ml.inference.base.PredictionResult) +object with the response from the model in the inference field, and the endpoint id representing the model id. +--- \ No newline at end of file diff --git a/sdks/python/apache_beam/examples/inference/anomaly_detection/anomaly_detection_pipeline/setup.py b/sdks/python/apache_beam/examples/inference/anomaly_detection/anomaly_detection_pipeline/setup.py index 19d2ecc4a0224..15a68228c6610 100644 --- a/sdks/python/apache_beam/examples/inference/anomaly_detection/anomaly_detection_pipeline/setup.py +++ b/sdks/python/apache_beam/examples/inference/anomaly_detection/anomaly_detection_pipeline/setup.py @@ -32,7 +32,7 @@ "apache-beam[gcp]==2.41.0", "hdbscan==0.8.28", "scikit-learn==1.0.2", - "transformers==4.21.1", + "transformers==4.30.0", "torch==1.13.1", "pandas==1.3.5", "yagmail==0.15.283", diff --git a/sdks/python/apache_beam/examples/inference/huggingface_language_modeling.py b/sdks/python/apache_beam/examples/inference/huggingface_language_modeling.py new file mode 100644 index 0000000000000..5eb57c8fc0809 --- /dev/null +++ b/sdks/python/apache_beam/examples/inference/huggingface_language_modeling.py @@ -0,0 +1,185 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A pipeline that uses RunInference to perform Language Modeling with +masked language model from Hugging Face. + +This pipeline takes sentences from a custom text file, converts the last word +of the sentence into a token, and then uses the AutoModelForMaskedLM from +Hugging Face to predict the best word for the masked token given all the words +already in the sentence. The pipeline then writes the prediction to an output +file in which users can then compare against the original sentence. +""" + +import argparse +import logging +from typing import Dict +from typing import Iterable +from typing import Iterator +from typing import Tuple + +import apache_beam as beam +import torch +from apache_beam.ml.inference.base import KeyedModelHandler +from apache_beam.ml.inference.base import PredictionResult +from apache_beam.ml.inference.base import RunInference +from apache_beam.ml.inference.huggingface_inference import HuggingFaceModelHandlerKeyedTensor +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.runners.runner import PipelineResult +from transformers import AutoModelForMaskedLM +from transformers import AutoTokenizer + + +def add_mask_to_last_word(text: str) -> Tuple[str, str]: + text_list = text.split() + return text, ' '.join(text_list[:-2] + ['', text_list[-1]]) + + +def tokenize_sentence( + text_and_mask: Tuple[str, str], + tokenizer: AutoTokenizer) -> Tuple[str, Dict[str, torch.Tensor]]: + text, masked_text = text_and_mask + tokenized_sentence = tokenizer.encode_plus(masked_text, return_tensors="pt") + + # Workaround to manually remove batch dim until we have the feature to + # add optional batching flag. + # TODO(https://github.com/apache/beam/issues/21863): Remove once optional + # batching flag added + return text, { + k: torch.squeeze(v) + for k, v in dict(tokenized_sentence).items() + } + + +def filter_empty_lines(text: str) -> Iterator[str]: + if len(text.strip()) > 0: + yield text + + +class PostProcessor(beam.DoFn): + """Processes the PredictionResult to get the predicted word. + + The logits are the output of the Model. We can get the word with the highest + probability of being a candidate replacement word by taking the argmax. + """ + def __init__(self, tokenizer: AutoTokenizer): + super().__init__() + self.tokenizer = tokenizer + + def process(self, element: Tuple[str, PredictionResult]) -> Iterable[str]: + text, prediction_result = element + inputs = prediction_result.example + logits = prediction_result.inference['logits'] + mask_token_index = torch.where( + inputs["input_ids"] == self.tokenizer.mask_token_id)[0] + predicted_token_id = logits[mask_token_index].argmax(axis=-1) + decoded_word = self.tokenizer.decode(predicted_token_id) + yield text + ';' + decoded_word + + +def parse_known_args(argv): + """Parses args for the workflow.""" + parser = argparse.ArgumentParser() + parser.add_argument( + '--input', + dest='input', + help='Path to the text file containing sentences.') + parser.add_argument( + '--output', + dest='output', + required=True, + help='Path of file in which to save the output predictions.') + parser.add_argument( + '--model_name', + dest='model_name', + required=True, + help='bert uncased model. This can be base model or large model') + parser.add_argument( + '--model_class', + dest='model_class', + default=AutoModelForMaskedLM, + help='Name of the model from Hugging Face') + parser.add_argument( + '--large_model', + action='store_true', + dest='large_model', + default=False, + help='Set to true if your model is large enough to run into memory ' + 'pressure if you load multiple copies.') + return parser.parse_known_args(argv) + + +def run( + argv=None, save_main_session=True, test_pipeline=None) -> PipelineResult: + """ + Args: + argv: Command line arguments defined for this example. + save_main_session: Used for internal testing. + test_pipeline: Used for internal testing. + """ + known_args, pipeline_args = parse_known_args(argv) + pipeline_options = PipelineOptions(pipeline_args) + pipeline_options.view_as(SetupOptions).save_main_session = save_main_session + + pipeline = test_pipeline + if not test_pipeline: + pipeline = beam.Pipeline(options=pipeline_options) + + tokenizer = AutoTokenizer.from_pretrained(known_args.model_name) + + model_handler = HuggingFaceModelHandlerKeyedTensor( + model_uri=known_args.model_name, + model_class=known_args.model_class, + framework='pt', + max_batch_size=1, + large_model=known_args.large_model) + if not known_args.input: + text = ( + pipeline | 'CreateSentences' >> beam.Create([ + 'The capital of France is Paris .', + 'It is raining cats and dogs .', + 'Today is Monday and tomorrow is Tuesday .', + 'There are 5 coconuts on this palm tree .', + 'The strongest person in the world is not famous .', + 'The secret ingredient to his wonderful life was gratitude .', + 'The biggest animal in the world is the whale .', + ])) + else: + text = ( + pipeline | 'ReadSentences' >> beam.io.ReadFromText(known_args.input)) + text_and_tokenized_text_tuple = ( + text + | 'FilterEmptyLines' >> beam.ParDo(filter_empty_lines) + | 'AddMask' >> beam.Map(add_mask_to_last_word) + | + 'TokenizeSentence' >> beam.Map(lambda x: tokenize_sentence(x, tokenizer))) + output = ( + text_and_tokenized_text_tuple + | 'RunInference' >> RunInference(KeyedModelHandler(model_handler)) + | 'ProcessOutput' >> beam.ParDo(PostProcessor(tokenizer=tokenizer))) + _ = output | "WriteOutput" >> beam.io.WriteToText( + known_args.output, shard_name_template='', append_trailing_newlines=True) + + result = pipeline.run() + result.wait_until_finish() + return result + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + run() diff --git a/sdks/python/apache_beam/examples/inference/huggingface_question_answering.py b/sdks/python/apache_beam/examples/inference/huggingface_question_answering.py new file mode 100644 index 0000000000000..9005ea5d11d73 --- /dev/null +++ b/sdks/python/apache_beam/examples/inference/huggingface_question_answering.py @@ -0,0 +1,164 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""""A pipeline that uses RunInference to perform Question Answering using the +model from Hugging Face Models Hub. + +This pipeline takes questions and context from a custom text file separated by +a semicolon. These are converted to SquadExamples by using the utility provided +by transformers.QuestionAnsweringPipeline and passed to the model handler. +We just provide the model name here because the model repository specifies the +task that it will do. The pipeline then writes the prediction to an output +file in which users can then compare against the original context. +""" + +import argparse +import logging +from typing import Iterable +from typing import Tuple + +import apache_beam as beam +from apache_beam.ml.inference.base import KeyedModelHandler +from apache_beam.ml.inference.base import PredictionResult +from apache_beam.ml.inference.base import RunInference +from apache_beam.ml.inference.huggingface_inference import HuggingFacePipelineModelHandler +from apache_beam.ml.inference.huggingface_inference import PipelineTask +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.runners.runner import PipelineResult +from transformers import QuestionAnsweringPipeline + + +class PostProcessor(beam.DoFn): + """Processes the PredictionResult to get the predicted answer. + + Hugging Face Pipeline for Question Answering returns a dictionary + with score, start and end index of answer and the answer. + """ + def process(self, result: Tuple[str, PredictionResult]) -> Iterable[str]: + text, prediction = result + predicted_answer = prediction.inference['answer'] + yield text + ';' + predicted_answer + + +def preprocess(text): + """ + preprocess separates the text into question and context + by splitting on semi-colon. + + Args: + text (str): string with question and context separated by semi-colon. + + Yields: + (str, str): yields question and context from text. + """ + if len(text.strip()) > 0: + question, context = text.split(';') + yield (question, context) + + +def create_squad_example(text): + """Creates SquadExample objects to be fed to QuestionAnsweringPipeline + supported by Hugging Face. + + Check out https://huggingface.co/docs/transformers/main_classes/pipelines#transformers.QuestionAnsweringPipeline.__call__.X #pylint: disable=line-too-long + to learn about valid input types for QuestionAnswering Pipeline. + Args: + text (Tuple[str,str]): a tuple of question and context. + """ + question, context = text + yield question, QuestionAnsweringPipeline.create_sample(question, context) + + +def parse_known_args(argv): + """Parses args for the workflow.""" + parser = argparse.ArgumentParser() + parser.add_argument( + '--input', + dest='input', + help='Path of file containing question and context separated by semicolon' + ) + parser.add_argument( + '--output', + dest='output', + required=True, + help='Path of file in which to save the output predictions.') + parser.add_argument( + '--model_name', + dest='model_name', + default="deepset/roberta-base-squad2", + help='Model repository-id from Hugging Face Models Hub.') + parser.add_argument( + '--revision', + dest='revision', + help= + 'Specific model version to use - branch name, tag name, or a commit-id.') + return parser.parse_known_args(argv) + + +def run( + argv=None, save_main_session=True, test_pipeline=None) -> PipelineResult: + """ + Args: + argv: Command line arguments defined for this example. + save_main_session: Used for internal testing. + test_pipeline: Used for internal testing. + """ + known_args, pipeline_args = parse_known_args(argv) + pipeline_options = PipelineOptions(pipeline_args) + pipeline_options.view_as(SetupOptions).save_main_session = save_main_session + + pipeline = test_pipeline + if not test_pipeline: + pipeline = beam.Pipeline(options=pipeline_options) + + model_handler = HuggingFacePipelineModelHandler( + task=PipelineTask.QuestionAnswering, + model=known_args.model_name, + load_model_args={ + 'framework': 'pt', 'revision': known_args.revision + }) + if not known_args.input: + text = ( + pipeline | 'CreateSentences' >> beam.Create([ + "What does Apache Beam do?;" + "Apache Beam enables batch and streaming data processing.", + "What is the capital of France?;The capital of France is Paris .", + "Where was beam summit?;Apache Beam Summit 2023 was in NYC.", + ])) + else: + text = ( + pipeline | 'ReadSentences' >> beam.io.ReadFromText(known_args.input)) + processed_text = ( + text + | 'PreProcess' >> beam.ParDo(preprocess) + | 'SquadExample' >> beam.ParDo(create_squad_example)) + output = ( + processed_text + | 'RunInference' >> RunInference(KeyedModelHandler(model_handler)) + | 'ProcessOutput' >> beam.ParDo(PostProcessor())) + _ = output | "WriteOutput" >> beam.io.WriteToText( + known_args.output, shard_name_template='', append_trailing_newlines=True) + + result = pipeline.run() + result.wait_until_finish() + return result + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + run() diff --git a/sdks/python/apache_beam/examples/inference/pytorch_language_modeling.py b/sdks/python/apache_beam/examples/inference/pytorch_language_modeling.py index b5fabbb1f1e45..9de10e73e11bc 100644 --- a/sdks/python/apache_beam/examples/inference/pytorch_language_modeling.py +++ b/sdks/python/apache_beam/examples/inference/pytorch_language_modeling.py @@ -118,6 +118,13 @@ def parse_known_args(argv): dest='model_state_dict_path', required=True, help="Path to the model's state_dict.") + parser.add_argument( + '--large_model', + action='store_true', + dest='large_model', + default=False, + help='Set to true if your model is large enough to run into memory ' + 'pressure if you load multiple copies.') return parser.parse_known_args(argv) @@ -166,7 +173,8 @@ def batch_elements_kwargs(self): model_handler = PytorchNoBatchModelHandler( state_dict_path=known_args.model_state_dict_path, model_class=model_class, - model_params=model_params) + model_params=model_params, + large_model=known_args.large_model) pipeline = test_pipeline if not test_pipeline: diff --git a/sdks/python/apache_beam/examples/inference/pytorch_model_per_key_image_segmentation.py b/sdks/python/apache_beam/examples/inference/pytorch_model_per_key_image_segmentation.py new file mode 100644 index 0000000000000..f0b5462d53350 --- /dev/null +++ b/sdks/python/apache_beam/examples/inference/pytorch_model_per_key_image_segmentation.py @@ -0,0 +1,311 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +A pipeline that uses RunInference API to perform image segmentation using +multiple different models. +""" + +import argparse +import io +import logging +import os +from typing import Iterable +from typing import Iterator +from typing import Optional +from typing import Tuple + +import apache_beam as beam +import torch +from apache_beam.io.filesystems import FileSystems +from apache_beam.ml.inference.base import KeyedModelHandler +from apache_beam.ml.inference.base import KeyModelMapping +from apache_beam.ml.inference.base import PredictionResult +from apache_beam.ml.inference.base import RunInference +from apache_beam.ml.inference.pytorch_inference import PytorchModelHandlerTensor +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.runners.runner import PipelineResult +from PIL import Image +from torchvision import transforms +from torchvision.models.detection import maskrcnn_resnet50_fpn +from torchvision.models.detection import maskrcnn_resnet50_fpn_v2 + +COCO_INSTANCE_CLASSES = [ + '__background__', + 'person', + 'bicycle', + 'car', + 'motorcycle', + 'airplane', + 'bus', + 'train', + 'truck', + 'boat', + 'traffic light', + 'fire hydrant', + 'N/A', + 'stop sign', + 'parking meter', + 'bench', + 'bird', + 'cat', + 'dog', + 'horse', + 'sheep', + 'cow', + 'elephant', + 'bear', + 'zebra', + 'giraffe', + 'N/A', + 'backpack', + 'umbrella', + 'N/A', + 'N/A', + 'handbag', + 'tie', + 'suitcase', + 'frisbee', + 'skis', + 'snowboard', + 'sports ball', + 'kite', + 'baseball bat', + 'baseball glove', + 'skateboard', + 'surfboard', + 'tennis racket', + 'bottle', + 'N/A', + 'wine glass', + 'cup', + 'fork', + 'knife', + 'spoon', + 'bowl', + 'banana', + 'apple', + 'sandwich', + 'orange', + 'broccoli', + 'carrot', + 'hot dog', + 'pizza', + 'donut', + 'cake', + 'chair', + 'couch', + 'potted plant', + 'bed', + 'N/A', + 'dining table', + 'N/A', + 'N/A', + 'toilet', + 'N/A', + 'tv', + 'laptop', + 'mouse', + 'remote', + 'keyboard', + 'cell phone', + 'microwave', + 'oven', + 'toaster', + 'sink', + 'refrigerator', + 'N/A', + 'book', + 'clock', + 'vase', + 'scissors', + 'teddy bear', + 'hair drier', + 'toothbrush' +] + +CLASS_ID_TO_NAME = dict(enumerate(COCO_INSTANCE_CLASSES)) + + +def read_image(image_file_name: str, + path_to_dir: Optional[str] = None) -> Tuple[str, Image.Image]: + if path_to_dir is not None: + image_file_name = os.path.join(path_to_dir, image_file_name) + with FileSystems().open(image_file_name, 'r') as file: + data = Image.open(io.BytesIO(file.read())).convert('RGB') + return image_file_name, data + + +def preprocess_image(data: Image.Image) -> torch.Tensor: + image_size = (224, 224) + transform = transforms.Compose([ + transforms.Resize(image_size), + transforms.ToTensor(), + ]) + return transform(data) + + +def filter_empty_lines(text: str) -> Iterator[str]: + if len(text.strip()) > 0: + yield text + + +class KeyExamplesForEachModelType(beam.DoFn): + """Duplicate data to run against each model type""" + def process( + self, element: Tuple[torch.Tensor, + str]) -> Iterable[Tuple[str, torch.Tensor]]: + yield 'v1', element[0] + yield 'v2', element[0] + + +class PostProcessor(beam.DoFn): + def process( + self, element: Tuple[str, PredictionResult]) -> Tuple[torch.Tensor, str]: + model, prediction_result = element + prediction_labels = prediction_result.inference['labels'] + classes = [CLASS_ID_TO_NAME[label.item()] for label in prediction_labels] + yield prediction_result.example, f'{model} predictions: {str(classes)}' + + +class FormatResults(beam.DoFn): + def process(self, element): + _, filename_prediction = element + predictions = filename_prediction['predictions'] + v1_predictions = next(p for p in predictions if 'v1 predictions' in p) + v2_predictions = next(p for p in predictions if 'v2 predictions' in p) + yield ( + f"{filename_prediction['image_names'][0]} --- " + f"{v1_predictions} --- " + f"{v2_predictions}") + + +def parse_known_args(argv): + """Parses args for the workflow.""" + parser = argparse.ArgumentParser() + parser.add_argument( + '--input', + dest='input', + required=True, + help='Path to the text file containing image names.') + parser.add_argument( + '--output', + dest='output', + required=True, + help='Path where to save output predictions.' + ' text file.') + parser.add_argument( + '--model_state_dict_paths', + dest='model_state_dict_paths', + required=True, + help="Comma separated paths to the models' state_dicts. " + "For this example, should include exactly 2 state_dicts corresponding " + "to maskrcnn_resnet50_fpn and maskrcnn_resnet50_fpn_v2 classes.") + parser.add_argument( + '--images_dir', + help='Path to the directory where images are stored.' + 'Not required if image names in the input file have absolute path.') + return parser.parse_known_args(argv) + + +def run( + argv=None, save_main_session=True, test_pipeline=None) -> PipelineResult: + """ + Args: + argv: Command line arguments defined for this example. + model_params: Parameters passed to the constructor of the model_class. + These will be used to instantiate the model object in the + RunInference API. + save_main_session: Used for internal testing. + test_pipeline: Used for internal testing. + """ + known_args, pipeline_args = parse_known_args(argv) + pipeline_options = PipelineOptions(pipeline_args) + pipeline_options.view_as(SetupOptions).save_main_session = save_main_session + + state_dicts = known_args.model_state_dict_paths.split(',') + if len(state_dicts) != 2: + raise AssertionError( + f'Expected exactly 2 state_dicts to be supplied, got {len(state_dicts)}' + ) + + mh1 = PytorchModelHandlerTensor( + state_dict_path=state_dicts[0], + model_class=maskrcnn_resnet50_fpn, + model_params={'num_classes': 91}) + mh2 = PytorchModelHandlerTensor( + state_dict_path=state_dicts[1], + model_class=maskrcnn_resnet50_fpn_v2, + model_params={'num_classes': 91}) + + # We'll use v1 and v2 as our keys to point to our model handlers. + # Note that multiple keys can also point to a single model handler, + # unlike this example. + model_handler = KeyedModelHandler( + [KeyModelMapping(['v1'], mh1), KeyModelMapping(['v2'], mh2)]) + + pipeline = test_pipeline + if not test_pipeline: + pipeline = beam.Pipeline(options=pipeline_options) + + value_filename_pair = ( + pipeline + | 'ReadImageNames' >> beam.io.ReadFromText(known_args.input) + | 'FilterEmptyLines' >> beam.ParDo(filter_empty_lines) + | 'ReadImageData' >> beam.Map( + lambda image_name: read_image( + image_file_name=image_name, path_to_dir=known_args.images_dir)) + | 'PreprocessImages' >> beam.MapTuple( + lambda file_name, data: (preprocess_image(data), file_name))) + + predictions = ( + value_filename_pair + | 'DuplicateData' >> beam.ParDo(KeyExamplesForEachModelType()) + | 'PyTorchRunInference' >> RunInference(model_handler) + | 'ProcessOutput' >> beam.ParDo(PostProcessor())) + + # We now have our set of (example, prediction) and (example, original + # filename) tuples. We can use CoGroupByKey to join them by original example, + # converting the Tensors to lists first so that Beam can compare them without + # a custom coder. If all you care about is the model used for inference, you + # can also get that from the PredictionResult returned from RunInference. + results = ({ + 'image_names': ( + value_filename_pair | beam.MapTuple( + lambda example, filename: (example.tolist(), filename))), + 'predictions': ( + predictions | beam.MapTuple( + lambda example, prediction: (example.tolist(), prediction))) + } | beam.CoGroupByKey()) + + _ = ( + results + | 'FormatResults' >> beam.ParDo(FormatResults()) + | "WriteOutput" >> beam.io.WriteToText( + known_args.output, + shard_name_template='', + append_trailing_newlines=True)) + + result = pipeline.run() + result.wait_until_finish() + return result + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + run() diff --git a/sdks/python/apache_beam/examples/inference/runinference_metrics/pipeline/options.py b/sdks/python/apache_beam/examples/inference/runinference_metrics/pipeline/options.py index b32200ed7331a..1966100430e98 100644 --- a/sdks/python/apache_beam/examples/inference/runinference_metrics/pipeline/options.py +++ b/sdks/python/apache_beam/examples/inference/runinference_metrics/pipeline/options.py @@ -61,7 +61,6 @@ def get_pipeline_options( flags = [ "--experiment=worker_accelerator=type:nvidia-tesla-p4;count:1;"\ "install-nvidia-driver", - "--experiment=use_runner_v2", ] dataflow_options.update({ "sdk_container_image": cfg.DOCKER_IMG, diff --git a/sdks/python/apache_beam/examples/inference/runinference_metrics/setup.py b/sdks/python/apache_beam/examples/inference/runinference_metrics/setup.py index d022d250be542..102699b6bec92 100644 --- a/sdks/python/apache_beam/examples/inference/runinference_metrics/setup.py +++ b/sdks/python/apache_beam/examples/inference/runinference_metrics/setup.py @@ -29,7 +29,7 @@ from setuptools import find_packages REQUIREMENTS = [ - "apache-beam[gcp]==2.41.0", "transformers==4.21.0", "torch==1.13.1" + "apache-beam[gcp]==2.41.0", "transformers==4.30.0", "torch==1.13.1" ] setuptools.setup( diff --git a/sdks/python/apache_beam/examples/inference/sklearn_mnist_classification.py b/sdks/python/apache_beam/examples/inference/sklearn_mnist_classification.py index 6f8ea929bbb6d..5392cdf7ddae4 100644 --- a/sdks/python/apache_beam/examples/inference/sklearn_mnist_classification.py +++ b/sdks/python/apache_beam/examples/inference/sklearn_mnist_classification.py @@ -77,6 +77,13 @@ def parse_known_args(argv): dest='model_path', required=True, help='Path to load the Sklearn model for Inference.') + parser.add_argument( + '--large_model', + action='store_true', + dest='large_model', + default=False, + help='Set to true if your model is large enough to run into memory ' + 'pressure if you load multiple copies.') return parser.parse_known_args(argv) @@ -103,7 +110,8 @@ def run( model_loader = KeyedModelHandler( SklearnModelHandlerNumpy( model_file_type=ModelFileType.PICKLE, - model_uri=known_args.model_path)) + model_uri=known_args.model_path, + large_model=known_args.large_model)) pipeline = test_pipeline if not test_pipeline: diff --git a/sdks/python/apache_beam/examples/inference/tensorflow_mnist_classification.py b/sdks/python/apache_beam/examples/inference/tensorflow_mnist_classification.py index 174d21b26af2a..6cf746e77cd24 100644 --- a/sdks/python/apache_beam/examples/inference/tensorflow_mnist_classification.py +++ b/sdks/python/apache_beam/examples/inference/tensorflow_mnist_classification.py @@ -70,6 +70,13 @@ def parse_known_args(argv): dest='model_path', required=True, help='Path to load the Tensorflow model for Inference.') + parser.add_argument( + '--large_model', + action='store_true', + dest='large_model', + default=False, + help='Set to true if your model is large enough to run into memory ' + 'pressure if you load multiple copies.') return parser.parse_known_args(argv) @@ -89,7 +96,9 @@ def run( # Therefore, we use KeyedModelHandler wrapper over TFModelHandlerNumpy. model_loader = KeyedModelHandler( TFModelHandlerNumpy( - model_uri=known_args.model_path, model_type=ModelType.SAVED_MODEL)) + model_uri=known_args.model_path, + model_type=ModelType.SAVED_MODEL, + large_model=known_args.large_model)) pipeline = test_pipeline if not test_pipeline: diff --git a/sdks/python/apache_beam/examples/inference/vertex_ai_image_classification.py b/sdks/python/apache_beam/examples/inference/vertex_ai_image_classification.py new file mode 100644 index 0000000000000..73126569e9884 --- /dev/null +++ b/sdks/python/apache_beam/examples/inference/vertex_ai_image_classification.py @@ -0,0 +1,174 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" A sample pipeline using the RunInference API to classify images of flowers. +This pipeline reads an already-processes representation of an image of +sunflowers and sends it to a deployed Vertex AI model endpoint, then +returns the predictions from the classifier model. The model and image +are from the Hello Image Data Vertex AI tutorial (see +https://cloud.google.com/vertex-ai/docs/tutorials/image-recognition-custom +for more information.) +""" + +import argparse +import io +import logging +from typing import Iterable +from typing import List +from typing import Tuple + +import apache_beam as beam +import tensorflow as tf +from apache_beam.io import fileio +from apache_beam.io.filesystems import FileSystems +from apache_beam.ml.inference.base import KeyedModelHandler +from apache_beam.ml.inference.base import PredictionResult +from apache_beam.ml.inference.base import RunInference +from apache_beam.ml.inference.vertex_ai_inference import VertexAIModelHandlerJSON +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.runners.runner import PipelineResult + + +def parse_known_args(argv): + """Parses args for the workflow.""" + parser = argparse.ArgumentParser() + # TODO: Update this to accept a glob of files. + parser.add_argument( + '--input', + dest='input', + type=str, + required=True, + help='File glob to read images from.') + parser.add_argument( + '--output', + dest='output', + type=str, + required=True, + help='Path to save output predictions.') + parser.add_argument( + '--endpoint_id', + dest='endpoint', + type=str, + required=True, + help='Vertex AI Endpoint resource ID to query (string).') + parser.add_argument( + '--endpoint_project', dest='project', required=True, help='GCP Project') + parser.add_argument( + '--endpoint_region', + dest='location', + type=str, + required=True, + help='GCP location for the Endpoint') + parser.add_argument( + '--endpoint_network', + dest='vpc_network', + type=str, + required=False, + help='GCP network the endpoint is peered to') + parser.add_argument( + '--experiment', + dest='experiment', + type=str, + required=False, + help='Vertex AI experiment label to apply to queries') + parser.add_argument( + '--private', + dest='private', + type=bool, + default=False, + help="True if the Vertex AI endpoint is a private endpoint") + return parser.parse_known_args(argv) + + +# Image height and width expected by the model +IMG_WIDTH = 128 + +# Column labels for the output probabilities. +COLUMNS = ['dandelion', 'daisy', 'tulips', 'sunflowers', 'roses'] + + +def read_image(image_file_name: str) -> Tuple[str, bytes]: + with FileSystems().open(image_file_name, 'r') as file: + data = io.BytesIO(file.read()).getvalue() + return image_file_name, data + + +def preprocess_image(data: bytes) -> List[float]: + """Preprocess the image, resizing it and normalizing it before + converting to a list. + """ + image = tf.io.decode_jpeg(data, channels=3) + image = tf.image.resize_with_pad(image, IMG_WIDTH, IMG_WIDTH) + image = image / 255 + return image.numpy().tolist() + + +class PostProcessor(beam.DoFn): + def process(self, element: Tuple[str, PredictionResult]) -> Iterable[str]: + img_name, prediction_result = element + prediction_vals = prediction_result.inference + index = prediction_vals.index(max(prediction_vals)) + yield img_name + ": " + str(COLUMNS[index]) + " (" + str( + max(prediction_vals)) + ")" + + +def run( + argv=None, save_main_session=True, test_pipeline=None) -> PipelineResult: + """ + Args: + argv: Command line arguments defined for this example. + save_main_session: Used for internal testing. + test_pipeline: Used for internal testing. + """ + known_args, pipeline_args = parse_known_args(argv) + pipeline_options = PipelineOptions(pipeline_args) + pipeline_options.view_as(SetupOptions).save_main_session = save_main_session + + model_handler = VertexAIModelHandlerJSON( + endpoint_id=known_args.endpoint, + project=known_args.project, + location=known_args.location, + experiment=known_args.experiment, + network=known_args.vpc_network, + private=known_args.private) + + pipeline = test_pipeline + if not test_pipeline: + pipeline = beam.Pipeline(options=pipeline_options) + + read_glob = pipeline | "Get glob" >> beam.Create([known_args.input]) + read_image_name = read_glob | "Get Image Paths" >> fileio.MatchAll() + load_image = read_image_name | "Read Image" >> beam.Map( + lambda image_name: read_image(image_name.path)) + preprocess = load_image | "Preprocess Image" >> beam.MapTuple( + lambda img_name, img: (img_name, preprocess_image(img))) + predictions = preprocess | "RunInference" >> RunInference( + KeyedModelHandler(model_handler)) + process_output = predictions | "Process Predictions" >> beam.ParDo( + PostProcessor()) + _ = process_output | "WriteOutput" >> beam.io.WriteToText( + known_args.output, shard_name_template='', append_trailing_newlines=True) + + result = pipeline.run() + result.wait_until_finish() + return result + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + run() diff --git a/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py b/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py new file mode 100644 index 0000000000000..e587ba87b91b7 --- /dev/null +++ b/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py @@ -0,0 +1,135 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" A sample pipeline using the RunInference API to classify text using an LLM. +This pipeline creates a set of prompts and sends it to a deployed Vertex AI +model endpoint, then returns the predictions from the classifier model. The +model can be generated by fine tuning the text-bison model or another similar +model (see +https://cloud.google.com/vertex-ai/docs/generative-ai/models/tune-models#supervised-fine-tuning) +""" + +import argparse +import logging + +import apache_beam as beam +from apache_beam.ml.inference.base import KeyedModelHandler +from apache_beam.ml.inference.base import RunInference +from apache_beam.ml.inference.vertex_ai_inference import VertexAIModelHandlerJSON +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.runners.runner import PipelineResult + + +def parse_known_args(argv): + """Parses args for the workflow.""" + parser = argparse.ArgumentParser() + parser.add_argument( + '--output', + dest='output', + type=str, + required=True, + help='Path to save output predictions.') + parser.add_argument( + '--endpoint_id', + dest='endpoint', + type=str, + required=True, + help='Vertex AI Endpoint resource ID to query (string).') + parser.add_argument( + '--endpoint_project', dest='project', required=True, help='GCP Project') + parser.add_argument( + '--endpoint_region', + dest='location', + type=str, + required=True, + help='GCP location for the Endpoint') + parser.add_argument( + '--endpoint_network', + dest='vpc_network', + type=str, + required=False, + help='GCP network the endpoint is peered to') + parser.add_argument( + '--experiment', + dest='experiment', + type=str, + required=False, + help='Vertex AI experiment label to apply to queries') + parser.add_argument( + '--private', + dest='private', + type=bool, + default=False, + help="True if the Vertex AI endpoint is a private endpoint") + return parser.parse_known_args(argv) + + +def run( + argv=None, save_main_session=True, test_pipeline=None) -> PipelineResult: + """ + Args: + argv: Command line arguments defined for this example. + save_main_session: Used for internal testing. + test_pipeline: Used for internal testing. + """ + known_args, pipeline_args = parse_known_args(argv) + pipeline_options = PipelineOptions(pipeline_args) + pipeline_options.view_as(SetupOptions).save_main_session = save_main_session + + model_handler = VertexAIModelHandlerJSON( + endpoint_id=known_args.endpoint, + project=known_args.project, + location=known_args.location, + experiment=known_args.experiment, + network=known_args.vpc_network, + private=known_args.private) + + pipeline = test_pipeline + if not test_pipeline: + pipeline = beam.Pipeline(options=pipeline_options) + + # For this example, use the default parameters from + # https://cloud.google.com/vertex-ai/docs/generative-ai/start/quickstarts/api-quickstart#parameter_definitions + parameters = { + "temperature": 0.2, "maxOutputTokens": 256, "topK": 40, "topP": 0.95 + } + prompts = [ + "What is 5+2?", + "Who is the president?", + "Write me a business plan for a cookie shop." + ] + + read_prompts = pipeline | "Get prompt" >> beam.Create(prompts) + preprocess = read_prompts | "Format prompt" >> beam.Map( + lambda data: (data, { + "prompt": data + })) + predictions = preprocess | "RunInference" >> RunInference( + KeyedModelHandler(model_handler), inference_args=parameters) + _ = predictions | "PrintOutput" >> beam.Map(print) + _ = predictions | "WriteOutput" >> beam.io.WriteToText( + known_args.output, shard_name_template='', append_trailing_newlines=True) + + result = pipeline.run() + result.wait_until_finish() + return result + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + run() diff --git a/sdks/python/apache_beam/examples/inference/xgboost_iris_classification.py b/sdks/python/apache_beam/examples/inference/xgboost_iris_classification.py index 59ee7868ca013..963187fd210dc 100644 --- a/sdks/python/apache_beam/examples/inference/xgboost_iris_classification.py +++ b/sdks/python/apache_beam/examples/inference/xgboost_iris_classification.py @@ -73,6 +73,13 @@ def parse_known_args(argv): dest='model_state', required=True, help='Path to the state of the XGBoost model loaded for Inference.') + parser.add_argument( + '--large_model', + action='store_true', + dest='large_model', + default=False, + help='Set to true if your model is large enough to run into memory ' + 'pressure if you load multiple copies.') group = parser.add_mutually_exclusive_group(required=True) group.add_argument('--split', action='store_true', dest='split') group.add_argument('--no_split', action='store_false', dest='split') @@ -125,7 +132,8 @@ def run( xgboost_model_handler = KeyedModelHandler( model_handler( model_class=xgboost.XGBClassifier, - model_state=known_args.model_state)) + model_state=known_args.model_state, + large_model=known_args.large_model)) input_data = load_sklearn_iris_test_data( data_type=input_data_type, split=known_args.split) diff --git a/sdks/python/apache_beam/examples/kafkataxi/README.md b/sdks/python/apache_beam/examples/kafkataxi/README.md index c4e808cad8b4d..72a8d8f85c037 100644 --- a/sdks/python/apache_beam/examples/kafkataxi/README.md +++ b/sdks/python/apache_beam/examples/kafkataxi/README.md @@ -157,9 +157,9 @@ Install Beam and dependencies and build a Beam distribution. ```sh cd beam/sdks/python -pip install -r build-requirements.txt pip install -e '.[gcp]' -python setup.py sdist +pip install -q build +python -m build --sdist ``` Run the Beam pipeline. You can either use the default Kafka topic name or specify diff --git a/sdks/python/apache_beam/examples/ml-orchestration/kfp/components/preprocessing/requirements.txt b/sdks/python/apache_beam/examples/ml-orchestration/kfp/components/preprocessing/requirements.txt index e902ead34151f..706adf9de0aa8 100644 --- a/sdks/python/apache_beam/examples/ml-orchestration/kfp/components/preprocessing/requirements.txt +++ b/sdks/python/apache_beam/examples/ml-orchestration/kfp/components/preprocessing/requirements.txt @@ -18,4 +18,4 @@ requests==2.31.0 torch==1.13.1 torchvision==0.13.0 numpy==1.22.4 -Pillow==9.3.0 +Pillow==10.0.1 diff --git a/sdks/python/apache_beam/runners/dataflow/native_io/__init__.py b/sdks/python/apache_beam/examples/ml_transform/__init__.py similarity index 100% rename from sdks/python/apache_beam/runners/dataflow/native_io/__init__.py rename to sdks/python/apache_beam/examples/ml_transform/__init__.py diff --git a/sdks/python/apache_beam/examples/ml_transform/ml_transform_basic.py b/sdks/python/apache_beam/examples/ml_transform/ml_transform_basic.py new file mode 100644 index 0000000000000..68bb38839cbec --- /dev/null +++ b/sdks/python/apache_beam/examples/ml_transform/ml_transform_basic.py @@ -0,0 +1,175 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This example demonstrates how to use MLTransform. +MLTransform is a PTransform that applies multiple data transformations on the +incoming data. + +This example computes the vocabulary on the incoming data. Then, it computes +the TF-IDF of the incoming data using the vocabulary computed in the previous +step. + +1. ComputeAndApplyVocabulary computes the vocabulary on the incoming data and + overrides the incoming data with the vocabulary indices. +2. TFIDF computes the TF-IDF of the incoming data using the vocabulary and + provides vocab_index and tf-idf weights. vocab_index is suffixed with + '_vocab_index' and tf-idf weights are suffixed with '_tfidf' to the + original column name(which is the output of ComputeAndApplyVocabulary). + +MLTransform produces artifacts, for example: ComputeAndApplyVocabulary produces +a text file that contains vocabulary which is saved in `artifact_location`. +ComputeAndApplyVocabulary outputs vocab indices associated with the saved vocab +file. This mode of MLTransform is called artifact `produce` mode. +This will be useful when the data is preprocessed before ML model training. + +The second mode of MLTransform is artifact `consume` mode. In this mode, the +transformations are applied on the incoming data using the artifacts produced +by the previous run of MLTransform. This mode will be useful when the data is +preprocessed before ML model inference. +""" + +import argparse +import logging +import tempfile + +import apache_beam as beam +from apache_beam.ml.transforms.base import MLTransform +from apache_beam.ml.transforms.tft import TFIDF +from apache_beam.ml.transforms.tft import ComputeAndApplyVocabulary +from apache_beam.ml.transforms.utils import ArtifactsFetcher + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--artifact_location', type=str, default='') + return parser.parse_known_args() + + +def preprocess_data_for_ml_training(train_data, args): + """ + Preprocess the data for ML training. This method runs a pipeline to + preprocess the data needed for ML training. It produces artifacts that can + be used for ML inference later. + """ + + with beam.Pipeline() as p: + train_data_pcoll = (p | "CreateData" >> beam.Create(train_data)) + + # When using write_artifact_location, the ComputeAndApplyVocabulary + # function generates a vocabulary file. This file, stored in + # 'write_artifact_location', contains the vocabulary of the entire dataset. + # This is considered as an artifact of ComputeAndApplyVocabulary transform. + # The indices of the vocabulary in this file are returned as + # the output of MLTransform. + transformed_data_pcoll = ( + train_data_pcoll + | 'MLTransform' >> MLTransform( + write_artifact_location=args.artifact_location, + ).with_transform(ComputeAndApplyVocabulary( + columns=['x'])).with_transform(TFIDF(columns=['x']))) + + _ = transformed_data_pcoll | beam.Map(logging.info) + # output for the element dict(x=["Let's", "go", "to", "the", "park"]) + # will be: + # Row(x=array([21, 5, 0, 2, 3]), + # x_tfidf_weight=array([0.28109303, 0.36218604, 0.36218604, 0.41972247, + # 0.5008155 ], dtype=float32), x_vocab_index=array([ 0, 2, 3, 5, 21])) + + +def preprocess_data_for_ml_inference(test_data, args): + """ + Preprocess the data for ML inference. This method runs a pipeline to + preprocess the data needed for ML inference. It consumes the artifacts + produced during the preprocessing stage for ML training. + """ + with beam.Pipeline() as p: + + test_data_pcoll = (p | beam.Create(test_data)) + # Here, the previously saved vocabulary from an MLTransform run is used by + # ComputeAndApplyVocabulary to access and apply the stored artifacts to the + # test data. + transformed_data_pcoll = ( + test_data_pcoll + | "MLTransformOnTestData" >> MLTransform( + read_artifact_location=args.artifact_location, + # ww don't need to specify transforms as they are already saved in + # in the artifacts. + )) + _ = transformed_data_pcoll | beam.Map(logging.info) + # output for dict(x=['I', 'love', 'books']) will be: + # Row(x=array([1, 4, 7]), + # x_tfidf_weight=array([0.4684884 , 0.6036434 , 0.69953746], dtype=float32) + # , x_vocab_index=array([1, 4, 7])) + + +def run(args): + """ + This example demonstrates how to use MLTransform in ML workflow. + 1. Preprocess the data for ML training. + 2. Do some ML model training. + 3. Preprocess the data for ML inference. + + training and inference on ML modes are not shown in this example. + This example only shows how to use MLTransform for preparing data for ML + training and inference. + """ + + train_data = [ + dict(x=["Let's", "go", "to", "the", "park"]), + dict(x=["I", "enjoy", "going", "to", "the", "park"]), + dict(x=["I", "enjoy", "reading", "books"]), + dict(x=["Beam", "can", "be", "fun"]), + dict(x=["The", "weather", "is", "really", "nice", "today"]), + dict(x=["I", "love", "to", "go", "to", "the", "park"]), + dict(x=["I", "love", "to", "read", "books"]), + dict(x=["I", "love", "to", "program"]), + ] + + test_data = [ + dict(x=['I', 'love', 'books']), dict(x=['I', 'love', 'Apache', 'Beam']) + ] + + # Preprocess the data for ML training. + # For the data going into the ML model training, we want to produce the + # artifacts. + preprocess_data_for_ml_training(train_data, args=args) + + # Do some ML model training here. + + # Preprocess the data for ML inference. + # For the data going into the ML model inference, we want to consume the + # artifacts produced during the stage where we preprocessed the data for ML + # training. + preprocess_data_for_ml_inference(test_data, args=args) + + # To fetch the artifacts produced in MLTransform, you can use + # ArtifactsFetcher for fetching vocab related artifacts. For + # others such as TFIDF weight, they can be accessed directly + # from the output of MLTransform. + artifacts_fetcher = ArtifactsFetcher(artifact_location=args.artifact_location) + vocab_list = artifacts_fetcher.get_vocab_list() + assert vocab_list[22] == 'Beam' + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + args, pipeline_args = parse_args() + # for this example, create a temp artifact location if not provided. + if args.artifact_location == '': + args.artifact_location = tempfile.mkdtemp() + run(args) diff --git a/sdks/python/apache_beam/examples/ml_transform/ml_transform_it_test.py b/sdks/python/apache_beam/examples/ml_transform/ml_transform_it_test.py new file mode 100644 index 0000000000000..17adb9c7c6abd --- /dev/null +++ b/sdks/python/apache_beam/examples/ml_transform/ml_transform_it_test.py @@ -0,0 +1,106 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import os +import time +import unittest +import uuid + +import pytest + +try: + from apache_beam.examples.ml_transform import vocab_tfidf_processing + from apache_beam.io.filesystems import FileSystems + from apache_beam.ml.transforms.utils import ArtifactsFetcher + from apache_beam.testing.load_tests.load_test_metrics_utils import InfluxDBMetricsPublisherOptions + from apache_beam.testing.load_tests.load_test_metrics_utils import MetricsReader + from apache_beam.testing.test_pipeline import TestPipeline +except ImportError: # pylint: disable=bare-except + raise unittest.SkipTest('tensorflow_transform is not installed.') + +_OUTPUT_GCS_BUCKET_ROOT = 'gs://temp-storage-for-end-to-end-tests/tft/' + + +def _publish_metrics(pipeline, metric_value, metrics_table, metric_name): + influx_options = InfluxDBMetricsPublisherOptions( + metrics_table, + pipeline.get_option('influx_db_name'), + pipeline.get_option('influx_hostname'), + os.getenv('INFLUXDB_USER'), + os.getenv('INFLUXDB_USER_PASSWORD'), + ) + metric_reader = MetricsReader( + project_name=pipeline.get_option('project'), + bq_table=metrics_table, + bq_dataset=pipeline.get_option('metrics_dataset'), + publish_to_bq=True, + influxdb_options=influx_options, + ) + metric_reader.publish_values([( + metric_name, + metric_value, + )]) + + +@pytest.mark.uses_tft +class LargeMovieReviewDatasetProcessTest(unittest.TestCase): + def test_process_large_movie_review_dataset(self): + input_data_dir = 'gs://apache-beam-ml/datasets/aclImdb' + artifact_location = os.path.join(_OUTPUT_GCS_BUCKET_ROOT, uuid.uuid4().hex) + output_dir = os.path.join(_OUTPUT_GCS_BUCKET_ROOT, uuid.uuid4().hex) + extra_opts = { + 'input_data_dir': input_data_dir, + 'output_dir': output_dir, + 'artifact_location': artifact_location, + } + + extra_opts['job_name'] = 'mltransform-large-movie-review-dataset-{}'.format( + uuid.uuid4().hex) + + test_pipeline = TestPipeline(is_integration_test=True) + start_time = time.time() + vocab_tfidf_processing.run( + test_pipeline.get_full_options_as_args( + **extra_opts, save_main_session=False), + ) + end_time = time.time() + metrics_table = 'ml_transform_large_movie_review_dataset_process_metrics' + _publish_metrics( + pipeline=test_pipeline, + metric_value=end_time - start_time, + metrics_table=metrics_table, + metric_name='runtime_sec') + + artifacts_fetcher = ArtifactsFetcher(artifact_location=artifact_location) + + actual_vocab_list = artifacts_fetcher.get_vocab_list() + + expected_artifact_filepath = 'gs://apache-beam-ml/testing/expected_outputs/compute_and_apply_vocab' # pylint: disable=line-too-long + + with FileSystems.open(expected_artifact_filepath, 'r') as f: + expected_vocab_list = f.readlines() + + expected_vocab_list = [ + s.decode('utf-8').rstrip('\n') for s in expected_vocab_list + ] + self.assertListEqual(actual_vocab_list, expected_vocab_list) + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + unittest.main() diff --git a/sdks/python/apache_beam/examples/ml_transform/vocab_tfidf_processing.py b/sdks/python/apache_beam/examples/ml_transform/vocab_tfidf_processing.py new file mode 100644 index 0000000000000..6ecd568924105 --- /dev/null +++ b/sdks/python/apache_beam/examples/ml_transform/vocab_tfidf_processing.py @@ -0,0 +1,190 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This example uses Large Movie Review Dataset: http://ai.stanford.edu/~amaas/data/sentiment/ # pylint:disable=line-too-long +to preprocess the input text data and generate TF-IDF scores for each word of +the input text. + +Workflow: +1. The input text is split into words using a delimiter. +2. The words are then converted to a vocabulary index using + ComputeAndApplyVocabulary. +3. The vocabulary index is then converted to TF-IDF scores using TFIDF. +4. The output of the pipeline is a Tuple of + (input_text, [(vocab_index, tfidf_score)]. + +To run this pipeline, download the Large Movie Review Dataset and +place it in a directory. Pass the directory path to --input_data_dir. +The pipeline will read the data from the directory and write the +transformed data to --output_dir. To save the artifacts, such as the +vocabulary file generated by ComputeAndApplyVocabulary will be saved to +the --artifact_location. + +In this pipeline, we only preprocess the train data(25000 samples). +""" + +import argparse +import logging +import os + +import apache_beam as beam +from apache_beam.ml.transforms.base import MLTransform +from apache_beam.ml.transforms.tft import TFIDF +from apache_beam.ml.transforms.tft import ComputeAndApplyVocabulary + +RAW_DATA_KEY = 'raw_data' +REVIEW_COLUMN = 'review' +LABEL_COLUMN = 'label' +DELIMITERS = '.,!?() ' +VOCAB_SIZE = 20000 + + +# pylint: disable=invalid-name +@beam.ptransform_fn +def Shuffle(pcoll): + """Shuffles a PCollection. Collection should not contain duplicates.""" + return ( + pcoll + | 'PairWithHash' >> beam.Map(lambda x: (hash(x), x)) + | 'GroupByHash' >> beam.GroupByKey() + | 'DropHash' >> beam.FlatMap(lambda hash_and_values: hash_and_values[1])) + + +class ReadAndShuffleData(beam.PTransform): + def __init__(self, pos_file_pattern, neg_file_pattern): + self.pos_file_pattern = pos_file_pattern + self.neg_file_pattern = neg_file_pattern + + def expand(self, pcoll): + + negative_examples = ( + pcoll + | "ReadNegativeExample" >> beam.io.ReadFromText(self.neg_file_pattern) + | 'PairWithZero' >> beam.Map(lambda review: (review, 0)) + | 'DistinctNeg' >> beam.Distinct()) + + positive_examples = ( + pcoll + | "ReadPositiveExample" >> beam.io.ReadFromText(self.pos_file_pattern) + | 'PairWithOne' >> beam.Map(lambda review: (review, 1)) + | 'DistinctPos' >> beam.Distinct()) + + all_examples = ((negative_examples, positive_examples) + | 'FlattenPColls' >> beam.Flatten()) + + shuffled_examples = (all_examples | 'Shuffle' >> Shuffle()) + + # tag with column names for MLTransform + return ( + shuffled_examples + | beam.Map( + lambda label_review: { + REVIEW_COLUMN: label_review[0], + LABEL_COLUMN: label_review[1], + RAW_DATA_KEY: label_review[0] + })) + + +def preprocess_data( + file_patterns, + pipeline_args, + artifact_location, + output_dir, + test_pipeline=None # used for testing purposes. +): + positive_pattern, negative_pattern = file_patterns + options = beam.options.pipeline_options.PipelineOptions(pipeline_args) + pipeline = test_pipeline + if not test_pipeline: + pipeline = beam.Pipeline(options=options) + data_pcoll = ( + pipeline + | + 'ReadTrainData' >> ReadAndShuffleData(positive_pattern, negative_pattern)) + ml_transform = MLTransform( + write_artifact_location=artifact_location, + ).with_transform( + ComputeAndApplyVocabulary( + top_k=VOCAB_SIZE, + frequency_threshold=10, + columns=[REVIEW_COLUMN], + split_string_by_delimiter=DELIMITERS)).with_transform( + TFIDF(columns=[REVIEW_COLUMN], vocab_size=VOCAB_SIZE)) + data_pcoll = data_pcoll | 'MLTransform' >> ml_transform + + data_pcoll = ( + data_pcoll | beam.ParDo(MapTFIDFScoreToVocab(artifact_location))) + + _ = (data_pcoll | beam.io.WriteToText(output_dir)) + + _ = data_pcoll | beam.Map(logging.info) + + result = pipeline.run() + result.wait_until_finish() + return result + + +class MapTFIDFScoreToVocab(beam.DoFn): + def __init__(self, artifact_location): + self.artifact_location = artifact_location + + def process(self, element): + index_column_name = REVIEW_COLUMN + '_vocab_index' + weight_column_name = REVIEW_COLUMN + '_tfidf_weight' + element = element.as_dict() + raw_data = element[RAW_DATA_KEY] + + vocab_index = element[index_column_name] + weights = element[weight_column_name] + + vocabs_with_weights = [(vocab_index[i], weights[i]) + for i in range(len(vocab_index))] + + return [( + raw_data, + vocabs_with_weights, + )] + + +def parse_known_args(argv): + parser = argparse.ArgumentParser() + parser.add_argument( + '--input_data_dir', help='path to directory containing input data.') + parser.add_argument( + '--artifact_location', + help='path to directory to hold artifacts such as vocab files.') + parser.add_argument( + '--output_dir', help='path to directory to hold transformed data.') + return parser.parse_known_args(argv) + + +def run(argv=None): + args, pipeline_args = parse_known_args(argv) + neg_filepatterm = os.path.join(args.input_data_dir, 'train/neg/*') + pos_filepattern = os.path.join(args.input_data_dir, 'train/pos/*') + artifact_location = args.artifact_location + + _ = preprocess_data((pos_filepattern, neg_filepatterm), + pipeline_args, + artifact_location, + output_dir=args.output_dir) + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + run() diff --git a/sdks/python/apache_beam/examples/snippets/snippets.py b/sdks/python/apache_beam/examples/snippets/snippets.py index 74e1d0a64c885..715011d302d2d 100644 --- a/sdks/python/apache_beam/examples/snippets/snippets.py +++ b/sdks/python/apache_beam/examples/snippets/snippets.py @@ -890,7 +890,7 @@ def model_bigqueryio( # [START model_bigqueryio_table_spec] # project-id:dataset_id.table_id - table_spec = 'clouddataflow-readonly:samples.weather_stations' + table_spec = 'apache-beam-testing.samples.weather_stations' # [END model_bigqueryio_table_spec] # [START model_bigqueryio_table_spec_without_project] @@ -936,7 +936,7 @@ def model_bigqueryio( pipeline | 'QueryTable' >> beam.io.ReadFromBigQuery( query='SELECT max_temperature FROM '\ - '[clouddataflow-readonly:samples.weather_stations]') + '[apache-beam-testing.samples.weather_stations]') # Each row is a dictionary where the keys are the BigQuery columns | beam.Map(lambda elem: elem['max_temperature'])) # [END model_bigqueryio_read_query] @@ -952,6 +952,14 @@ def model_bigqueryio( | beam.Map(lambda elem: elem['max_temperature'])) # [END model_bigqueryio_read_query_std_sql] + # [START model_bigqueryio_read_table_with_storage_api] + max_temperatures = ( + pipeline + | 'ReadTableWithStorageAPI' >> beam.io.ReadFromBigQuery( + table=table_spec, method=beam.io.ReadFromBigQuery.Method.DIRECT_READ) + | beam.Map(lambda elem: elem['max_temperature'])) + # [END model_bigqueryio_read_table_with_storage_api] + # [START model_bigqueryio_schema] # column_name:BIGQUERY_TYPE, ... table_schema = 'source:STRING, quote:STRING' @@ -1020,6 +1028,55 @@ def table_fn(element, fictional_characters): # [END model_bigqueryio_time_partitioning] +def model_bigqueryio_xlang( + pipeline, write_project='', write_dataset='', write_table=''): + """Examples for cross-language BigQuery sources and sinks.""" + + # to avoid a validation error(input data schema and the table schema) + # use a table that does not exist + import uuid + never_exists_table = str(uuid.uuid4()) + table_spec = 'apache-beam-testing.samples.{}'.format(never_exists_table) + + if write_project and write_dataset and write_table: + table_spec = '{}:{}.{}'.format(write_project, write_dataset, write_table) + + # [START model_bigqueryio_write_schema] + table_schema = { + 'fields': [{ + 'name': 'source', 'type': 'STRING', 'mode': 'NULLABLE' + }, { + 'name': 'quote', 'type': 'STRING', 'mode': 'REQUIRED' + }] + } + # [END model_bigqueryio_write_schema] + + quotes = pipeline | beam.Create([ + { + 'source': 'Mahatma Gandhi', 'quote': 'My life is my message.' + }, + { + 'source': 'Yoda', 'quote': "Do, or do not. There is no 'try'." + }, + ]) + + # [START model_bigqueryio_storage_write_api_with_frequency] + # The Python SDK doesn't currently support setting the number of write streams + quotes | "StorageWriteAPIWithFrequency" >> beam.io.WriteToBigQuery( + table_spec, + schema=table_schema, + method=beam.io.WriteToBigQuery.Method.STORAGE_WRITE_API, + triggering_frequency=5) + # [END model_bigqueryio_storage_write_api_with_frequency] + + # [START model_bigqueryio_write_with_storage_write_api] + quotes | "WriteTableWithStorageAPI" >> beam.io.WriteToBigQuery( + table_spec, + schema=table_schema, + method=beam.io.WriteToBigQuery.Method.STORAGE_WRITE_API) + # [END model_bigqueryio_write_with_storage_write_api] + + def model_composite_transform_example(contents, output_path): """Example of a composite transform. diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py b/sdks/python/apache_beam/examples/snippets/snippets_test.py index 0188a2814665e..ec52ca37af39b 100644 --- a/sdks/python/apache_beam/examples/snippets/snippets_test.py +++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py @@ -33,6 +33,7 @@ import mock import parameterized +import pytest import apache_beam as beam from apache_beam import WindowInto @@ -41,6 +42,9 @@ from apache_beam import typehints from apache_beam.coders.coders import ToBytesCoder from apache_beam.examples.snippets import snippets +from apache_beam.examples.snippets import snippets_examples_wordcount_debugging +from apache_beam.examples.snippets import snippets_examples_wordcount_minimal +from apache_beam.examples.snippets import snippets_examples_wordcount_wordcount from apache_beam.metrics import Metrics from apache_beam.metrics.metric import MetricsFilter from apache_beam.options.pipeline_options import GoogleCloudOptions @@ -61,10 +65,6 @@ from apache_beam.transforms.window import TimestampedValue from apache_beam.utils.windowed_value import WindowedValue -from . import snippets_examples_wordcount_debugging -from . import snippets_examples_wordcount_minimal -from . import snippets_examples_wordcount_wordcount - # Protect against environments where apitools library is not available. # pylint: disable=wrong-import-order, wrong-import-position try: @@ -614,7 +614,7 @@ def test_model_pipelines(self): snippets.model_pipelines() self.assertEqual( self.get_output(result_path), - [str(s) for s in [(u'aa', 1), (u'bb', 2), (u'cc', 3)]]) + [str(s) for s in [('aa', 1), ('bb', 2), ('cc', 3)]]) def test_model_pcollection(self): temp_path = self.create_temp_file() @@ -758,6 +758,15 @@ def test_model_bigqueryio(self): p.options.view_as(GoogleCloudOptions).temp_location = 'gs://mylocation' snippets.model_bigqueryio(p) + @pytest.mark.uses_gcp_java_expansion_service + @unittest.skipUnless( + os.environ.get('EXPANSION_PORT'), + "EXPANSION_PORT environment var is not provided.") + def test_model_bigqueryio_xlang(self): + p = TestPipeline() + p.options.view_as(GoogleCloudOptions).temp_location = 'gs://mylocation' + snippets.model_bigqueryio_xlang(p) + def _run_test_pipeline_for_options(self, fn): temp_path = self.create_temp_file('aa\nbb\ncc') result_path = temp_path + '.result' @@ -863,7 +872,7 @@ def _inner(topic=None, subscription=None): input_topic = 'projects/fake-beam-test-project/topic/intopic' input_values = [ TimestampedValue(b'a a b', 1), - TimestampedValue(u'🤷 ¯\\_(ツ)_/¯ b b '.encode('utf-8'), 12), + TimestampedValue('🤷 ¯\\_(ツ)_/¯ b b '.encode('utf-8'), 12), TimestampedValue(b'a b c c c', 20) ] output_topic = 'projects/fake-beam-test-project/topic/outtopic' diff --git a/sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform.py b/sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform.py new file mode 100644 index 0000000000000..63ce448e69d9e --- /dev/null +++ b/sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform.py @@ -0,0 +1,121 @@ +# coding=utf-8 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pytype: skip-file +# pylint: disable=reimported +# pylint:disable=line-too-long + + +def mltransform_scale_to_0_1(test=None): + # [START mltransform_scale_to_0_1] + import apache_beam as beam + from apache_beam.ml.transforms.base import MLTransform + from apache_beam.ml.transforms.tft import ScaleTo01 + import tempfile + + data = [ + { + 'x': [1, 5, 3] + }, + { + 'x': [4, 2, 8] + }, + ] + + artifact_location = tempfile.mkdtemp() + scale_to_0_1_fn = ScaleTo01(columns=['x']) + + with beam.Pipeline() as p: + transformed_data = ( + p + | beam.Create(data) + | MLTransform(write_artifact_location=artifact_location).with_transform( + scale_to_0_1_fn) + | beam.Map(print)) + # [END mltransform_scale_to_0_1] + if test: + test(transformed_data) + + +def mltransform_compute_and_apply_vocabulary(test=None): + # [START mltransform_compute_and_apply_vocabulary] + import apache_beam as beam + from apache_beam.ml.transforms.base import MLTransform + from apache_beam.ml.transforms.tft import ComputeAndApplyVocabulary + import tempfile + + artifact_location = tempfile.mkdtemp() + data = [ + { + 'x': ['I', 'love', 'Beam'] + }, + { + 'x': ['Beam', 'is', 'awesome'] + }, + ] + compute_and_apply_vocabulary_fn = ComputeAndApplyVocabulary(columns=['x']) + with beam.Pipeline() as p: + transformed_data = ( + p + | beam.Create(data) + | MLTransform(write_artifact_location=artifact_location).with_transform( + compute_and_apply_vocabulary_fn) + | beam.Map(print)) + # [END mltransform_compute_and_apply_vocabulary] + if test: + test(transformed_data) + + +def mltransform_compute_and_apply_vocabulary_with_scalar(test=None): + # [START mltransform_compute_and_apply_vocabulary_with_scalar] + import apache_beam as beam + from apache_beam.ml.transforms.base import MLTransform + from apache_beam.ml.transforms.tft import ComputeAndApplyVocabulary + import tempfile + data = [ + { + 'x': 'I' + }, + { + 'x': 'love' + }, + { + 'x': 'Beam' + }, + { + 'x': 'Beam' + }, + { + 'x': 'is' + }, + { + 'x': 'awesome' + }, + ] + artifact_location = tempfile.mkdtemp() + compute_and_apply_vocabulary_fn = ComputeAndApplyVocabulary(columns=['x']) + with beam.Pipeline() as p: + transformed_data = ( + p + | beam.Create(data) + | MLTransform(write_artifact_location=artifact_location).with_transform( + compute_and_apply_vocabulary_fn) + | beam.Map(print)) + # [END mltransform_compute_and_apply_vocabulary_with_scalar] + if test: + test(transformed_data) diff --git a/sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py b/sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py new file mode 100644 index 0000000000000..1d2197e35e4e0 --- /dev/null +++ b/sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py @@ -0,0 +1,90 @@ +# coding=utf-8 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pytype: skip-file +# pylint: disable=ungrouped-imports + +import unittest +from io import StringIO + +import mock + +from apache_beam.testing.test_pipeline import TestPipeline + +try: + # fail when tft is not installed. + import tensorflow_transform as tft # pylint: disable=unused-import + from apache_beam.examples.snippets.transforms.elementwise.mltransform import mltransform_scale_to_0_1 + from apache_beam.examples.snippets.transforms.elementwise.mltransform import mltransform_compute_and_apply_vocabulary + from apache_beam.examples.snippets.transforms.elementwise.mltransform import mltransform_compute_and_apply_vocabulary_with_non_columnar_data +except ImportError: + raise unittest.SkipTest('tensorflow_transform is not installed.') + + +def check_mltransform_compute_and_apply_vocab(): + expected = '''[START mltransform_compute_and_apply_vocab] +Row(x=array([4, 1, 0])) +Row(x=array([0, 2, 3])) + [END mltransform_compute_and_apply_vocab] '''.splitlines()[1:-1] + return expected + + +def check_mltransform_scale_to_0_1(): + expected = '''[START mltransform_scale_to_0_1] +Row(x=array([0. , 0.5714286, 0.2857143], dtype=float32), x_max=array([8.], dtype=float32), x_min=array([1.], dtype=float32)) +Row(x=array([0.42857143, 0.14285715, 1. ], dtype=float32), x_max=array([8.], dtype=float32), x_min=array([1.], dtype=float32)) + [END mltransform_scale_to_0_1] '''.splitlines()[1:-1] + return expected + + +def check_mltransform_compute_and_apply_vocabulary_with_scalar(): + expected = '''[START mltransform_compute_and_apply_vocabulary_with_scalar] +Row(x=array([4])) +Row(x=array([1])) +Row(x=array([0])) +Row(x=array([2])) +Row(x=array([3])) + [END mltransform_compute_and_apply_vocabulary_with_scalar] '''.splitlines( + )[1:-1] + return expected + + +@mock.patch('apache_beam.Pipeline', TestPipeline) +@mock.patch('sys.stdout', new_callable=StringIO) +class MLTransformStdOutTest(unittest.TestCase): + def test_mltransform_compute_and_apply_vocab(self, mock_stdout): + mltransform_compute_and_apply_vocabulary() + predicted = mock_stdout.getvalue().splitlines() + expected = check_mltransform_compute_and_apply_vocab() + self.assertEqual(predicted, expected) + + def test_mltransform_scale_to_0_1(self, mock_stdout): + mltransform_scale_to_0_1() + predicted = mock_stdout.getvalue().splitlines() + expected = check_mltransform_scale_to_0_1() + self.assertEqual(predicted, expected) + + def test_mltransform_compute_and_apply_vocab_scalar(self, mock_stdout): + mltransform_compute_and_apply_vocabulary_with_non_columnar_data() + predicted = mock_stdout.getvalue().splitlines() + expected = check_mltransform_compute_and_apply_vocabulary_with_scalar() + self.assertEqual(predicted, expected) + + +if __name__ == '__main__': + unittest.main() diff --git a/sdks/python/apache_beam/examples/wordcount_it_test.py b/sdks/python/apache_beam/examples/wordcount_it_test.py index bb5859df72514..63f661dba212e 100644 --- a/sdks/python/apache_beam/examples/wordcount_it_test.py +++ b/sdks/python/apache_beam/examples/wordcount_it_test.py @@ -45,6 +45,7 @@ class WordCountIT(unittest.TestCase): DEFAULT_CHECKSUM = '33535a832b7db6d78389759577d4ff495980b9c0' @pytest.mark.it_postcommit + @pytest.mark.it_validatescontainer def test_wordcount_it(self): self._run_wordcount_it(wordcount.run) @@ -89,11 +90,6 @@ def test_wordcount_impersonation_it(self): with auth._Credentials._credentials_lock: auth._Credentials._credentials_init = False - @pytest.mark.it_postcommit - @pytest.mark.it_validatescontainer - def test_wordcount_fnapi_it(self): - self._run_wordcount_it(wordcount.run, experiment='beam_fn_api') - @pytest.mark.it_validatescontainer def test_wordcount_it_with_prebuilt_sdk_container_local_docker(self): self._run_wordcount_it( @@ -108,14 +104,15 @@ def test_wordcount_it_with_prebuilt_sdk_container_cloud_build(self): experiment='beam_fn_api', prebuild_sdk_container_engine='cloud_build') - @pytest.mark.it_validatescontainer - def test_wordcount_it_with_use_sibling_sdk_workers(self): - self._run_wordcount_it(wordcount.run, experiment='use_sibling_sdk_workers') - def _run_wordcount_it(self, run_wordcount, **opts): test_pipeline = TestPipeline(is_integration_test=True) extra_opts = {} + if (test_pipeline.get_option('machine_type') == 't2a-standard-1' and + 'prebuild_sdk_container_engine' in opts): + # TODO(https://github.com/apache/beam/issues/28340) + pytest.skip('prebuild_sdk_container_engine not supported on ARM') + # Set extra options to the pipeline for test purpose test_output = '/'.join([ test_pipeline.get_option('output'), diff --git a/sdks/python/apache_beam/examples/wordcount_test.py b/sdks/python/apache_beam/examples/wordcount_test.py index 0b658ed5fbbb2..cea0ce368e555 100644 --- a/sdks/python/apache_beam/examples/wordcount_test.py +++ b/sdks/python/apache_beam/examples/wordcount_test.py @@ -38,7 +38,7 @@ class WordCountTest(unittest.TestCase): SAMPLE_TEXT = ( - u'a b c a b a\nacento gráfico\nJuly 30, 2018\n\n aa bb cc aa bb aa') + 'a b c a b a\nacento gráfico\nJuly 30, 2018\n\n aa bb cc aa bb aa') def test_basics(self): test_pipeline = TestPipeline(is_integration_test=True) diff --git a/sdks/python/apache_beam/internal/OWNERS b/sdks/python/apache_beam/internal/OWNERS deleted file mode 100644 index 7dce27abe1264..0000000000000 --- a/sdks/python/apache_beam/internal/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - aaltay - - charlesccychen diff --git a/sdks/python/apache_beam/internal/cloudpickle_pickler_test.py b/sdks/python/apache_beam/internal/cloudpickle_pickler_test.py index 1bbf21cfec14d..8ae93d53fd1d3 100644 --- a/sdks/python/apache_beam/internal/cloudpickle_pickler_test.py +++ b/sdks/python/apache_beam/internal/cloudpickle_pickler_test.py @@ -19,7 +19,6 @@ # pytype: skip-file -import sys import threading import types import unittest @@ -34,7 +33,7 @@ class PicklerTest(unittest.TestCase): NO_MAPPINGPROXYTYPE = not hasattr(types, "MappingProxyType") def test_basics(self): - self.assertEqual([1, 'a', (u'z', )], loads(dumps([1, 'a', (u'z', )]))) + self.assertEqual([1, 'a', ('z', )], loads(dumps([1, 'a', ('z', )]))) fun = lambda x: 'xyz-%s' % x self.assertEqual('xyz-abc', loads(dumps(fun))('abc')) @@ -106,7 +105,6 @@ def test_dump_and_load_mapping_proxy(self): types.MappingProxyType, type(loads(dumps(types.MappingProxyType({}))))) # pylint: disable=exec-used - @unittest.skipIf(sys.version_info < (3, 7), 'Python 3.7 or above only') def test_dataclass(self): exec( ''' diff --git a/sdks/python/apache_beam/internal/dill_pickler.py b/sdks/python/apache_beam/internal/dill_pickler.py index efa736c2a96fc..8a0742642dfbb 100644 --- a/sdks/python/apache_beam/internal/dill_pickler.py +++ b/sdks/python/apache_beam/internal/dill_pickler.py @@ -46,7 +46,7 @@ settings = {'dill_byref': None} -if sys.version_info >= (3, 11) and dill.__version__ == "0.3.1.1": +if sys.version_info >= (3, 10) and dill.__version__ == "0.3.1.1": # Let's make dill 0.3.1.1 support Python 3.11. # The following function is based on 'save_code' from 'dill' diff --git a/sdks/python/apache_beam/internal/gcp/auth.py b/sdks/python/apache_beam/internal/gcp/auth.py index 47c3416babd47..bab3ace4144e0 100644 --- a/sdks/python/apache_beam/internal/gcp/auth.py +++ b/sdks/python/apache_beam/internal/gcp/auth.py @@ -26,6 +26,7 @@ from apache_beam.options.pipeline_options import GoogleCloudOptions from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.utils import retry # google.auth is only available when Beam is installed with the gcp extra. try: @@ -149,8 +150,7 @@ def _get_service_credentials(pipeline_options): try: # pylint: disable=c-extension-no-member - credentials, _ = google.auth.default( - scopes=pipeline_options.view_as(GoogleCloudOptions).gcp_oauth_scopes) + credentials = _Credentials._get_credentials_with_retrys(pipeline_options) credentials = _Credentials._add_impersonation_credentials( credentials, pipeline_options) credentials = _ApitoolsCredentialsAdapter(credentials) @@ -161,10 +161,18 @@ def _get_service_credentials(pipeline_options): except Exception as e: _LOGGER.warning( 'Unable to find default credentials to use: %s\n' - 'Connecting anonymously.', + 'Connecting anonymously. This is expected if no ' + 'credentials are needed to access GCP resources.', e) return None + @staticmethod + @retry.with_exponential_backoff(num_retries=4, initial_delay_secs=2) + def _get_credentials_with_retrys(pipeline_options): + credentials, _ = google.auth.default( + scopes=pipeline_options.view_as(GoogleCloudOptions).gcp_oauth_scopes) + return credentials + @staticmethod def _add_impersonation_credentials(credentials, pipeline_options): gcs_options = pipeline_options.view_as(GoogleCloudOptions) diff --git a/sdks/python/apache_beam/internal/gcp/auth_test.py b/sdks/python/apache_beam/internal/gcp/auth_test.py new file mode 100644 index 0000000000000..98fb828875b9c --- /dev/null +++ b/sdks/python/apache_beam/internal/gcp/auth_test.py @@ -0,0 +1,135 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import logging +import unittest + +import mock + +from apache_beam.internal.gcp import auth +from apache_beam.options.pipeline_options import GoogleCloudOptions +from apache_beam.options.pipeline_options import PipelineOptions + +try: + import google.auth as gauth +except ImportError: + gauth = None + + +class MockLoggingHandler(logging.Handler): + """Mock logging handler to check for expected logs.""" + def __init__(self, *args, **kwargs): + self.reset() + logging.Handler.__init__(self, *args, **kwargs) + + def emit(self, record): + self.messages[record.levelname.lower()].append(record.getMessage()) + + def reset(self): + self.messages = { + 'debug': [], + 'info': [], + 'warning': [], + 'error': [], + 'critical': [], + } + + +@unittest.skipIf(gauth is None, 'Google Auth dependencies are not installed') +class AuthTest(unittest.TestCase): + @mock.patch('google.auth.default') + def test_auth_with_retrys(self, unused_mock_arg): + pipeline_options = PipelineOptions() + pipeline_options.view_as( + GoogleCloudOptions).impersonate_service_account = False + + credentials = ('creds', 1) + + self.is_called = False + + def side_effect(scopes=None): + if self.is_called: + return credentials + else: + self.is_called = True + raise IOError('Failed') + + google_auth_mock = mock.MagicMock() + gauth.default = google_auth_mock + google_auth_mock.side_effect = side_effect + + # _Credentials caches the actual credentials. + # This resets it for idempotent tests. + if auth._Credentials._credentials_init: + auth._Credentials._credentials_init = False + auth._Credentials._credentials = None + + returned_credentials = auth.get_service_credentials(pipeline_options) + + # _Credentials caches the actual credentials. + # This resets it for idempotent tests. + if auth._Credentials._credentials_init: + auth._Credentials._credentials_init = False + auth._Credentials._credentials = None + + self.assertEqual('creds', returned_credentials._google_auth_credentials) + + @mock.patch( + 'apache_beam.internal.gcp.auth._Credentials._get_credentials_with_retrys') + def test_auth_with_retrys_always_fail(self, unused_mock_arg): + pipeline_options = PipelineOptions() + pipeline_options.view_as( + GoogleCloudOptions).impersonate_service_account = False + + loggerHandler = MockLoggingHandler() + + auth._LOGGER.addHandler(loggerHandler) + + #Remove call to retrying method, as otherwise test takes ~10 minutes to run + def raise_(scopes=None): + raise IOError('Failed') + + retry_auth_mock = mock.MagicMock() + auth._Credentials._get_credentials_with_retrys = retry_auth_mock + retry_auth_mock.side_effect = raise_ + + # _Credentials caches the actual credentials. + # This resets it for idempotent tests. + if auth._Credentials._credentials_init: + auth._Credentials._credentials_init = False + auth._Credentials._credentials = None + + returned_credentials = auth.get_service_credentials(pipeline_options) + + self.assertEqual(None, returned_credentials) + self.assertEqual([ + 'Unable to find default credentials to use: Failed\n' + 'Connecting anonymously. This is expected if no credentials are ' + 'needed to access GCP resources.' + ], + loggerHandler.messages.get('warning')) + + # _Credentials caches the actual credentials. + # This resets it for idempotent tests. + if auth._Credentials._credentials_init: + auth._Credentials._credentials_init = False + auth._Credentials._credentials = None + + auth._LOGGER.removeHandler(loggerHandler) + + +if __name__ == '__main__': + unittest.main() diff --git a/sdks/python/apache_beam/internal/pickler_test.py b/sdks/python/apache_beam/internal/pickler_test.py index a9151cd7e1116..824c4c59c0ce8 100644 --- a/sdks/python/apache_beam/internal/pickler_test.py +++ b/sdks/python/apache_beam/internal/pickler_test.py @@ -34,7 +34,7 @@ class PicklerTest(unittest.TestCase): NO_MAPPINGPROXYTYPE = not hasattr(types, "MappingProxyType") def test_basics(self): - self.assertEqual([1, 'a', (u'z', )], loads(dumps([1, 'a', (u'z', )]))) + self.assertEqual([1, 'a', ('z', )], loads(dumps([1, 'a', ('z', )]))) fun = lambda x: 'xyz-%s' % x self.assertEqual('xyz-abc', loads(dumps(fun))('abc')) diff --git a/sdks/python/apache_beam/io/OWNERS b/sdks/python/apache_beam/io/OWNERS deleted file mode 100644 index 3f9471bf72172..0000000000000 --- a/sdks/python/apache_beam/io/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - chamikaramj - - aaltay - - udim diff --git a/sdks/python/apache_beam/io/avroio.py b/sdks/python/apache_beam/io/avroio.py index d86f59e3a4111..9225acf346e4e 100644 --- a/sdks/python/apache_beam/io/avroio.py +++ b/sdks/python/apache_beam/io/avroio.py @@ -45,7 +45,13 @@ # pytype: skip-file import os from functools import partial +from typing import Any +from typing import Callable +from typing import Dict +from typing import List +from typing import Union +import fastavro from fastavro.read import block_reader from fastavro.write import Writer @@ -54,8 +60,11 @@ from apache_beam.io import filebasedsource from apache_beam.io import iobase from apache_beam.io.filesystem import CompressionTypes +from apache_beam.io.filesystems import FileSystems from apache_beam.io.iobase import Read +from apache_beam.portability.api import schema_pb2 from apache_beam.transforms import PTransform +from apache_beam.typehints import schemas __all__ = [ 'ReadFromAvro', @@ -73,7 +82,8 @@ def __init__( file_pattern=None, min_bundle_size=0, validate=True, - use_fastavro=True): + use_fastavro=True, + as_rows=False): """Initializes :class:`ReadFromAvro`. Uses source :class:`~apache_beam.io._AvroSource` to read a set of Avro @@ -140,13 +150,26 @@ def __init__( creation time. use_fastavro (bool): This flag is left for API backwards compatibility and no longer has an effect. Do not use. + as_rows (bool): Whether to return a schema'd PCollection of Beam rows. """ super().__init__() - self._source = _create_avro_source( + self._source = _FastAvroSource( file_pattern, min_bundle_size, validate=validate) + if as_rows: + path = FileSystems.match([file_pattern], [1])[0].metadata_list[0].path + with FileSystems.open(path) as fin: + avro_schema = fastavro.reader(fin).writer_schema + beam_schema = avro_schema_to_beam_schema(avro_schema) + self._post_process = avro_dict_to_beam_row(avro_schema, beam_schema) + else: + self._post_process = None def expand(self, pvalue): - return pvalue.pipeline | Read(self._source) + records = pvalue.pipeline | Read(self._source) + if self._post_process: + return records | beam.Map(self._post_process) + else: + return records def display_data(self): return {'source_dd': self._source} @@ -184,8 +207,7 @@ def __init__( name and the value being the actual data. If False, it only returns the data. """ - source_from_file = partial( - _create_avro_source, min_bundle_size=min_bundle_size) + source_from_file = partial(_FastAvroSource, min_bundle_size=min_bundle_size) self._read_all_files = filebasedsource.ReadAllFiles( True, CompressionTypes.AUTO, @@ -280,15 +302,6 @@ def advance_file_past_next_sync_marker(f, sync_marker): data = f.read(buf_size) -def _create_avro_source(file_pattern=None, min_bundle_size=0, validate=False): - return \ - _FastAvroSource( - file_pattern=file_pattern, - min_bundle_size=min_bundle_size, - validate=validate - ) - - class _FastAvroSource(filebasedsource.FileBasedSource): """A source for reading Avro files using the `fastavro` library. @@ -338,12 +351,15 @@ def split_points_unclaimed(stop_position): yield record +_create_avro_source = _FastAvroSource + + class WriteToAvro(beam.transforms.PTransform): """A ``PTransform`` for writing avro files.""" def __init__( self, file_path_prefix, - schema, + schema=None, codec='deflate', file_name_suffix='', num_shards=0, @@ -382,9 +398,10 @@ def __init__( Returns: A WriteToAvro transform usable for writing. """ - self._sink = _create_avro_sink( + self._schema = schema + self._sink_provider = lambda avro_schema: _create_avro_sink( file_path_prefix, - schema, + avro_schema, codec, file_name_suffix, num_shards, @@ -392,7 +409,21 @@ def __init__( mime_type) def expand(self, pcoll): - return pcoll | beam.io.iobase.Write(self._sink) + if self._schema: + avro_schema = self._schema + records = pcoll + else: + try: + beam_schema = schemas.schema_from_element_type(pcoll.element_type) + except TypeError as exn: + raise ValueError( + "An explicit schema is required to write non-schema'd PCollections." + ) from exn + avro_schema = beam_schema_to_avro_schema(beam_schema) + records = pcoll | beam.Map( + beam_row_to_avro_dict(avro_schema, beam_schema)) + self._sink = self._sink_provider(avro_schema) + return records | beam.io.iobase.Write(self._sink) def display_data(self): return {'sink_dd': self._sink} @@ -406,7 +437,7 @@ def _create_avro_sink( num_shards, shard_name_template, mime_type): - if "class \'avro.schema" in str(type(schema)): + if "class 'avro.schema" in str(type(schema)): raise ValueError( 'You are using Avro IO with fastavro (default with Beam on ' 'Python 3), but supplying a schema parsed by avro-python3. ' @@ -483,3 +514,205 @@ def write_record(self, writer, value): def close(self, writer): writer.flush() self.file_handle.close() + + +AVRO_PRIMITIVES_TO_BEAM_PRIMITIVES = { + 'boolean': schema_pb2.BOOLEAN, + 'int': schema_pb2.INT32, + 'long': schema_pb2.INT64, + 'float': schema_pb2.FLOAT, + 'double': schema_pb2.DOUBLE, + 'bytes': schema_pb2.BYTES, + 'string': schema_pb2.STRING, +} + +BEAM_PRIMITIVES_TO_AVRO_PRIMITIVES = { + v: k + for k, v in AVRO_PRIMITIVES_TO_BEAM_PRIMITIVES.items() +} + +_AvroSchemaType = Union[str, List, Dict] + + +def avro_type_to_beam_type(avro_type: _AvroSchemaType) -> schema_pb2.FieldType: + if isinstance(avro_type, str): + return avro_type_to_beam_type({'type': avro_type}) + elif isinstance(avro_type, list): + # Union type + return schemas.typing_to_runner_api(Any) + type_name = avro_type['type'] + if type_name in AVRO_PRIMITIVES_TO_BEAM_PRIMITIVES: + return schema_pb2.FieldType( + atomic_type=AVRO_PRIMITIVES_TO_BEAM_PRIMITIVES[type_name]) + elif type_name in ('fixed', 'enum'): + return schema_pb2.FieldType(atomic_type=schema_pb2.STRING) + elif type_name == 'array': + return schema_pb2.FieldType( + array_type=schema_pb2.ArrayType( + element_type=avro_type_to_beam_type(avro_type['items']))) + elif type_name == 'map': + return schema_pb2.FieldType( + map_type=schema_pb2.MapType( + key_type=schema_pb2.FieldType(atomic_type=schema_pb2.STRING), + value_type=avro_type_to_beam_type(avro_type['values']))) + elif type_name == 'record': + return schema_pb2.FieldType( + row_type=schema_pb2.RowType( + schema=schema_pb2.Schema( + fields=[ + schemas.schema_field( + f['name'], avro_type_to_beam_type(f['type'])) + for f in avro_type['fields'] + ]))) + else: + raise ValueError(f'Unable to convert {avro_type} to a Beam schema.') + + +def avro_schema_to_beam_schema( + avro_schema: _AvroSchemaType) -> schema_pb2.Schema: + beam_type = avro_type_to_beam_type(avro_schema) + if isinstance(avro_schema, dict) and avro_schema['type'] == 'record': + return beam_type.row_type.schema + else: + return schema_pb2.Schema(fields=[schemas.schema_field('record', beam_type)]) + + +def avro_dict_to_beam_row( + avro_schema: _AvroSchemaType, + beam_schema: schema_pb2.Schema) -> Callable[[Any], Any]: + if isinstance(avro_schema, str): + return avro_dict_to_beam_row({'type': avro_schema}) + if avro_schema['type'] == 'record': + to_row = avro_value_to_beam_value( + schema_pb2.FieldType(row_type=schema_pb2.RowType(schema=beam_schema))) + else: + + def to_row(record): + return beam.Row(record=record) + + return beam.typehints.with_output_types( + schemas.named_tuple_from_schema(beam_schema))( + to_row) + + +def avro_value_to_beam_value( + beam_type: schema_pb2.FieldType) -> Callable[[Any], Any]: + type_info = beam_type.WhichOneof("type_info") + if type_info == "atomic_type": + return lambda value: value + elif type_info == "array_type": + element_converter = avro_value_to_beam_value( + beam_type.array_type.element_type) + return lambda value: [element_converter(e) for e in value] + elif type_info == "iterable_type": + element_converter = avro_value_to_beam_value( + beam_type.iterable_type.element_type) + return lambda value: [element_converter(e) for e in value] + elif type_info == "map_type": + if beam_type.map_type.key_type.atomic_type != schema_pb2.STRING: + raise TypeError( + f'Only strings allowd as map keys when converting from AVRO, ' + f'found {beam_type}') + value_converter = avro_value_to_beam_value(beam_type.map_type.value_type) + return lambda value: {k: value_converter(v) for (k, v) in value.items()} + elif type_info == "row_type": + converters = { + field.name: avro_value_to_beam_value(field.type) + for field in beam_type.row_type.schema.fields + } + return lambda value: beam.Row( + ** + {name: convert(value[name]) + for (name, convert) in converters.items()}) + elif type_info == "logical_type": + return lambda value: value + else: + raise ValueError(f"Unrecognized type_info: {type_info!r}") + + +def beam_schema_to_avro_schema( + beam_schema: schema_pb2.Schema) -> _AvroSchemaType: + return beam_type_to_avro_type( + schema_pb2.FieldType(row_type=schema_pb2.RowType(schema=beam_schema))) + + +def beam_type_to_avro_type(beam_type: schema_pb2.FieldType) -> _AvroSchemaType: + type_info = beam_type.WhichOneof("type_info") + if type_info == "atomic_type": + return {'type': BEAM_PRIMITIVES_TO_AVRO_PRIMITIVES[beam_type.atomic_type]} + elif type_info == "array_type": + return { + 'type': 'array', + 'items': beam_type_to_avro_type(beam_type.array_type.element_type) + } + elif type_info == "iterable_type": + return { + 'type': 'array', + 'items': beam_type_to_avro_type(beam_type.iterable_type.element_type) + } + elif type_info == "map_type": + if beam_type.map_type.key_type.atomic_type != schema_pb2.STRING: + raise TypeError( + f'Only strings allowd as map keys when converting to AVRO, ' + f'found {beam_type}') + return { + 'type': 'map', + 'values': beam_type_to_avro_type(beam_type.map_type.element_type) + } + elif type_info == "row_type": + return { + 'type': 'record', + 'name': beam_type.row_type.schema.id, + 'fields': [{ + 'name': field.name, 'type': beam_type_to_avro_type(field.type) + } for field in beam_type.row_type.schema.fields], + } + else: + raise ValueError(f"Unconvertale type: {beam_type}") + + +def beam_row_to_avro_dict( + avro_schema: _AvroSchemaType, beam_schema: schema_pb2.Schema): + if isinstance(avro_schema, str): + return beam_row_to_avro_dict({'type': avro_schema}, beam_schema) + if avro_schema['type'] == 'record': + return beam_value_to_avro_value( + schema_pb2.FieldType(row_type=schema_pb2.RowType(schema=beam_schema))) + else: + convert = beam_value_to_avro_value(beam_schema) + return lambda row: convert(row[0]) + + +def beam_value_to_avro_value( + beam_type: schema_pb2.FieldType) -> Callable[[Any], Any]: + type_info = beam_type.WhichOneof("type_info") + if type_info == "atomic_type": + return lambda value: value + elif type_info == "array_type": + element_converter = avro_value_to_beam_value( + beam_type.array_type.element_type) + return lambda value: [element_converter(e) for e in value] + elif type_info == "iterable_type": + element_converter = avro_value_to_beam_value( + beam_type.iterable_type.element_type) + return lambda value: [element_converter(e) for e in value] + elif type_info == "map_type": + if beam_type.map_type.key_type.atomic_type != schema_pb2.STRING: + raise TypeError( + f'Only strings allowd as map keys when converting from AVRO, ' + f'found {beam_type}') + value_converter = avro_value_to_beam_value(beam_type.map_type.value_type) + return lambda value: {k: value_converter(v) for (k, v) in value.items()} + elif type_info == "row_type": + converters = { + field.name: avro_value_to_beam_value(field.type) + for field in beam_type.row_type.schema.fields + } + return lambda value: { + name: convert(getattr(value, name)) + for (name, convert) in converters.items() + } + elif type_info == "logical_type": + return lambda value: value + else: + raise ValueError(f"Unrecognized type_info: {type_info!r}") diff --git a/sdks/python/apache_beam/io/avroio_test.py b/sdks/python/apache_beam/io/avroio_test.py index ba35cf5846c05..c54ac40711b1e 100644 --- a/sdks/python/apache_beam/io/avroio_test.py +++ b/sdks/python/apache_beam/io/avroio_test.py @@ -35,8 +35,8 @@ from apache_beam.io import filebasedsource from apache_beam.io import iobase from apache_beam.io import source_test_utils +from apache_beam.io.avroio import _FastAvroSource # For testing from apache_beam.io.avroio import _create_avro_sink # For testing -from apache_beam.io.avroio import _create_avro_source # For testing from apache_beam.io.filesystems import FileSystems from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that @@ -125,7 +125,7 @@ def _write_pattern(self, num_files, return_filenames=False): def _run_avro_test( self, pattern, desired_bundle_size, perform_splitting, expected_result): - source = _create_avro_source(pattern) + source = _FastAvroSource(pattern) if perform_splitting: assert desired_bundle_size @@ -146,6 +146,20 @@ def _run_avro_test( read_records = source_test_utils.read_from_source(source, None, None) self.assertCountEqual(expected_result, read_records) + def test_schema_read_write(self): + with tempfile.TemporaryDirectory() as tmp_dirname: + path = os.path.join(tmp_dirname, 'tmp_filename') + rows = [beam.Row(a=1, b=['x', 'y']), beam.Row(a=2, b=['t', 'u'])] + stable_repr = lambda row: json.dumps(row._asdict()) + with TestPipeline() as p: + _ = p | Create(rows) | avroio.WriteToAvro(path) | beam.Map(print) + with TestPipeline() as p: + readback = ( + p + | avroio.ReadFromAvro(path + '*', as_rows=True) + | beam.Map(stable_repr)) + assert_that(readback, equal_to([stable_repr(r) for r in rows])) + def test_read_without_splitting(self): file_name = self._write_data() expected_result = self.RECORDS @@ -159,7 +173,7 @@ def test_read_with_splitting(self): def test_source_display_data(self): file_name = 'some_avro_source' source = \ - _create_avro_source( + _FastAvroSource( file_name, validate=False, ) @@ -207,6 +221,7 @@ def test_sink_display_data(self): def test_write_display_data(self): file_name = 'some_avro_sink' write = avroio.WriteToAvro(file_name, self.SCHEMA) + write.expand(beam.PCollection(beam.Pipeline())) dd = DisplayData.create_from(write) expected_items = [ DisplayDataItemMatcher('schema', str(self.SCHEMA)), @@ -220,12 +235,12 @@ def test_write_display_data(self): def test_read_reentrant_without_splitting(self): file_name = self._write_data() - source = _create_avro_source(file_name) + source = _FastAvroSource(file_name) source_test_utils.assert_reentrant_reads_succeed((source, None, None)) def test_read_reantrant_with_splitting(self): file_name = self._write_data() - source = _create_avro_source(file_name) + source = _FastAvroSource(file_name) splits = [split for split in source.split(desired_bundle_size=100000)] assert len(splits) == 1 source_test_utils.assert_reentrant_reads_succeed( @@ -246,7 +261,7 @@ def test_split_points(self): sync_interval = 16000 file_name = self._write_data(count=num_records, sync_interval=sync_interval) - source = _create_avro_source(file_name) + source = _FastAvroSource(file_name) splits = [split for split in source.split(desired_bundle_size=float('inf'))] assert len(splits) == 1 @@ -306,7 +321,7 @@ def test_read_with_splitting_pattern(self): def test_dynamic_work_rebalancing_exhaustive(self): def compare_split_points(file_name): - source = _create_avro_source(file_name) + source = _FastAvroSource(file_name) splits = [ split for split in source.split(desired_bundle_size=float('inf')) ] @@ -334,7 +349,7 @@ def test_corrupted_file(self): f.write(corrupted_data) corrupted_file_name = f.name - source = _create_avro_source(corrupted_file_name) + source = _FastAvroSource(corrupted_file_name) with self.assertRaisesRegex(ValueError, r'expected sync marker'): source_test_utils.read_from_source(source, None, None) diff --git a/sdks/python/apache_beam/io/azure/integration_test/Dockerfile b/sdks/python/apache_beam/io/azure/integration_test/Dockerfile index e9ac396b8e174..257fa72cb6688 100644 --- a/sdks/python/apache_beam/io/azure/integration_test/Dockerfile +++ b/sdks/python/apache_beam/io/azure/integration_test/Dockerfile @@ -32,7 +32,7 @@ COPY sdks/python /app/sdks/python COPY model /app/model # This step should look like setupVirtualenv minus virtualenv creation. -RUN pip install --no-cache-dir tox -r sdks/python/build-requirements.txt +RUN pip install --no-cache-dir tox # Add Azurite's self-signed cert to the global CA cert store. COPY cert.pem /usr/local/share/ca-certificates/azurite.crt diff --git a/sdks/python/apache_beam/io/external/xlang_bigqueryio_it_test.py b/sdks/python/apache_beam/io/external/xlang_bigqueryio_it_test.py index fbfde550ea708..5917ca4dc7295 100644 --- a/sdks/python/apache_beam/io/external/xlang_bigqueryio_it_test.py +++ b/sdks/python/apache_beam/io/external/xlang_bigqueryio_it_test.py @@ -30,12 +30,13 @@ from hamcrest.core import assert_that as hamcrest_assert import apache_beam as beam -from apache_beam.io.external.generate_sequence import GenerateSequence from apache_beam.io.gcp.bigquery import StorageWriteToBigQuery from apache_beam.io.gcp.bigquery_tools import BigQueryWrapper from apache_beam.io.gcp.tests.bigquery_matcher import BigqueryFullResultMatcher from apache_beam.io.gcp.tests.bigquery_matcher import BigqueryFullResultStreamingMatcher +from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.testing.test_pipeline import TestPipeline +from apache_beam.transforms.periodicsequence import PeriodicImpulse from apache_beam.utils.timestamp import Timestamp # Protect against environments where bigquery library is not available. @@ -51,9 +52,9 @@ @pytest.mark.uses_gcp_java_expansion_service -@unittest.skipUnless( - os.environ.get('EXPANSION_PORT'), - "EXPANSION_PORT environment var is not provided.") +# @unittest.skipUnless( +# os.environ.get('EXPANSION_PORT'), +# "EXPANSION_PORT environment var is not provided.") class BigQueryXlangStorageWriteIT(unittest.TestCase): BIGQUERY_DATASET = 'python_xlang_storage_write' @@ -104,6 +105,7 @@ def setUp(self): self.test_pipeline = TestPipeline(is_integration_test=True) self.args = self.test_pipeline.get_full_options_as_args() self.project = self.test_pipeline.get_option('project') + self._runner = PipelineOptions(self.args).get_all_options()['runner'] self.bigquery_client = BigQueryWrapper() self.dataset_id = '%s_%s_%s' % ( @@ -244,8 +246,7 @@ def test_write_with_beam_rows(self): table=table_id, expansion_service=self.expansion_service)) hamcrest_assert(p, bq_matcher) - def run_streaming( - self, table_name, auto_sharding=False, use_at_least_once=False): + def run_streaming(self, table_name, num_streams=0, use_at_least_once=False): elements = self.ELEMENTS.copy() schema = self.ALL_TYPES_SCHEMA table_id = '{}:{}.{}'.format(self.project, self.dataset_id, table_name) @@ -260,33 +261,44 @@ def run_streaming( streaming=True, allow_unsafe_triggers=True) + auto_sharding = (num_streams == 0) with beam.Pipeline(argv=args) as p: _ = ( p - | GenerateSequence( - start=0, stop=4, expansion_service=self.expansion_service) - | beam.Map(lambda x: elements[x]) + | PeriodicImpulse(0, 4, 1) + | beam.Map(lambda t: elements[t]) | beam.io.WriteToBigQuery( table=table_id, method=beam.io.WriteToBigQuery.Method.STORAGE_WRITE_API, schema=schema, + triggering_frequency=1, with_auto_sharding=auto_sharding, + num_storage_api_streams=num_streams, use_at_least_once=use_at_least_once, expansion_service=self.expansion_service)) hamcrest_assert(p, bq_matcher) - def test_streaming(self): - table = 'streaming' + def test_streaming_with_fixed_num_streams(self): + # skip if dataflow runner is not specified + if not self._runner or "dataflowrunner" not in self._runner.lower(): + self.skipTest( + "The exactly-once route has the requirement " + "`beam:requirement:pardo:on_window_expiration:v1`, " + "which is currently only supported by the Dataflow runner") + table = 'streaming_fixed_num_streams' + self.run_streaming(table_name=table, num_streams=4) + + @unittest.skip( + "Streaming to the Storage Write API sink with autosharding is broken " + "with Dataflow Runner V2.") + def test_streaming_with_auto_sharding(self): + table = 'streaming_with_auto_sharding' self.run_streaming(table_name=table) def test_streaming_with_at_least_once(self): - table = 'streaming' + table = 'streaming_with_at_least_once' self.run_streaming(table_name=table, use_at_least_once=True) - def test_streaming_with_auto_sharding(self): - table = 'streaming_with_auto_sharding' - self.run_streaming(table_name=table, auto_sharding=True) - if __name__ == '__main__': logging.getLogger().setLevel(logging.INFO) diff --git a/sdks/python/apache_beam/io/external/xlang_jdbcio_it_test.py b/sdks/python/apache_beam/io/external/xlang_jdbcio_it_test.py index ed8745ec2ac10..54a473d1b52b8 100644 --- a/sdks/python/apache_beam/io/external/xlang_jdbcio_it_test.py +++ b/sdks/python/apache_beam/io/external/xlang_jdbcio_it_test.py @@ -17,6 +17,7 @@ # pytype: skip-file +import datetime import logging import time import typing @@ -60,7 +61,8 @@ "JdbcTestRow", [("f_id", int), ("f_float", float), ("f_char", str), ("f_varchar", str), ("f_bytes", bytes), ("f_varbytes", bytes), ("f_timestamp", Timestamp), - ("f_decimal", Decimal)], + ("f_decimal", Decimal), ("f_date", datetime.date), + ("f_time", datetime.time)], ) coders.registry.register_coder(JdbcTestRow, coders.RowCoder) @@ -132,7 +134,7 @@ def test_xlang_jdbc_write_read(self, database): "f_float DOUBLE PRECISION, " + "f_char CHAR(10), " + "f_varchar VARCHAR(10), " + f"f_bytes {binary_type[0]}, " + f"f_varbytes {binary_type[1]}, " + "f_timestamp TIMESTAMP(3), " + - "f_decimal DECIMAL(10, 2))") + "f_decimal DECIMAL(10, 2), " + "f_date DATE, " + "f_time TIME(3))") inserted_rows = [ JdbcTestRow( i, @@ -144,7 +146,11 @@ def test_xlang_jdbc_write_read(self, database): # In alignment with Java Instant which supports milli precision. Timestamp.of(seconds=round(time.time(), 3)), # Test both positive and negative numbers. - Decimal(f'{i-1}.23')) for i in range(ROW_COUNT) + Decimal(f'{i-1}.23'), + # Test both date before or after EPOCH + datetime.date(1969 + i, i % 12 + 1, i % 31 + 1), + datetime.time(i % 24, i % 60, i % 60, (i * 1000) % 1_000_000)) + for i in range(ROW_COUNT) ] expected_row = [] for row in inserted_rows: @@ -163,7 +169,9 @@ def test_xlang_jdbc_write_read(self, database): f_bytes, row.f_bytes, row.f_timestamp, - row.f_decimal)) + row.f_decimal, + row.f_date, + row.f_time)) with TestPipeline() as p: p.not_use_test_runner_api = True diff --git a/sdks/python/apache_beam/io/external/xlang_kafkaio_it_test.py b/sdks/python/apache_beam/io/external/xlang_kafkaio_it_test.py index edeb76aa0e714..a2f350e8cb7a5 100644 --- a/sdks/python/apache_beam/io/external/xlang_kafkaio_it_test.py +++ b/sdks/python/apache_beam/io/external/xlang_kafkaio_it_test.py @@ -28,6 +28,8 @@ import unittest import uuid +import pytest + import apache_beam as beam from apache_beam.coders.coders import VarIntCoder from apache_beam.io.kafka import ReadFromKafka @@ -110,11 +112,11 @@ def run_xlang_kafkaio(self, pipeline): pipeline.run(False) -@unittest.skipUnless( - os.environ.get('LOCAL_KAFKA_JAR'), - "LOCAL_KAFKA_JAR environment var is not provided.") class CrossLanguageKafkaIOTest(unittest.TestCase): - def test_kafkaio_populated_key(self): + @unittest.skipUnless( + os.environ.get('LOCAL_KAFKA_JAR'), + "LOCAL_KAFKA_JAR environment var is not provided.") + def test_local_kafkaio_populated_key(self): kafka_topic = 'xlang_kafkaio_test_populated_key_{}'.format(uuid.uuid4()) local_kafka_jar = os.environ.get('LOCAL_KAFKA_JAR') with self.local_kafka_service(local_kafka_jar) as kafka_port: @@ -126,7 +128,10 @@ def test_kafkaio_populated_key(self): self.run_kafka_write(pipeline_creator) self.run_kafka_read(pipeline_creator, b'key') - def test_kafkaio_null_key(self): + @unittest.skipUnless( + os.environ.get('LOCAL_KAFKA_JAR'), + "LOCAL_KAFKA_JAR environment var is not provided.") + def test_local_kafkaio_null_key(self): kafka_topic = 'xlang_kafkaio_test_null_key_{}'.format(uuid.uuid4()) local_kafka_jar = os.environ.get('LOCAL_KAFKA_JAR') with self.local_kafka_service(local_kafka_jar) as kafka_port: @@ -138,6 +143,44 @@ def test_kafkaio_null_key(self): self.run_kafka_write(pipeline_creator) self.run_kafka_read(pipeline_creator, None) + @pytest.mark.uses_io_expansion_service + @unittest.skipUnless( + os.environ.get('EXPANSION_PORT'), + "EXPANSION_PORT environment var is not provided.") + @unittest.skipUnless( + os.environ.get('KAFKA_BOOTSTRAP_SERVER'), + "KAFKA_BOOTSTRAP_SERVER environment var is not provided.") + def test_hosted_kafkaio_populated_key(self): + kafka_topic = 'xlang_kafkaio_test_populated_key_{}'.format(uuid.uuid4()) + bootstrap_servers = os.environ.get('KAFKA_BOOTSTRAP_SERVER') + pipeline_creator = CrossLanguageKafkaIO( + bootstrap_servers, + kafka_topic, + False, + 'localhost:%s' % os.environ.get('EXPANSION_PORT')) + + self.run_kafka_write(pipeline_creator) + self.run_kafka_read(pipeline_creator, b'key') + + @pytest.mark.uses_io_expansion_service + @unittest.skipUnless( + os.environ.get('EXPANSION_PORT'), + "EXPANSION_PORT environment var is not provided.") + @unittest.skipUnless( + os.environ.get('KAFKA_BOOTSTRAP_SERVER'), + "KAFKA_BOOTSTRAP_SERVER environment var is not provided.") + def test_hosted_kafkaio_null_key(self): + kafka_topic = 'xlang_kafkaio_test_null_key_{}'.format(uuid.uuid4()) + bootstrap_servers = os.environ.get('KAFKA_BOOTSTRAP_SERVER') + pipeline_creator = CrossLanguageKafkaIO( + bootstrap_servers, + kafka_topic, + True, + 'localhost:%s' % os.environ.get('EXPANSION_PORT')) + + self.run_kafka_write(pipeline_creator) + self.run_kafka_read(pipeline_creator, None) + def run_kafka_write(self, pipeline_creator): with TestPipeline() as pipeline: pipeline.not_use_test_runner_api = True diff --git a/sdks/python/apache_beam/io/external/xlang_parquetio_test.py b/sdks/python/apache_beam/io/external/xlang_parquetio_test.py index adcabd3e636b1..b4074d156ce7c 100644 --- a/sdks/python/apache_beam/io/external/xlang_parquetio_test.py +++ b/sdks/python/apache_beam/io/external/xlang_parquetio_test.py @@ -60,7 +60,7 @@ def test_xlang_parquetio_write(self): AvroRecord({"name": "ghi"})]) \ | beam.ExternalTransform( PARQUET_WRITE_URN, - ImplicitSchemaPayloadBuilder({'data': u'/tmp/test.parquet'}), + ImplicitSchemaPayloadBuilder({'data': '/tmp/test.parquet'}), address) except RuntimeError as e: if re.search(PARQUET_WRITE_URN, str(e)): diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py index a9297ca3d7aca..23e979b44cae0 100644 --- a/sdks/python/apache_beam/io/fileio.py +++ b/sdks/python/apache_beam/io/fileio.py @@ -266,6 +266,13 @@ class MatchContinuously(beam.PTransform): MatchContinuously is experimental. No backwards-compatibility guarantees. + + Matching continuously scales poorly, as it is stateful, and requires storing + file ids in memory. In addition, because it is memory-only, if a pipeline is + restarted, already processed files will be reprocessed. Consider an alternate + technique, such as Pub/Sub Notifications + (https://cloud.google.com/storage/docs/pubsub-notifications) + when using GCS if possible. """ def __init__( self, @@ -299,6 +306,11 @@ def __init__( self.match_upd = match_updated_files self.apply_windowing = apply_windowing self.empty_match_treatment = empty_match_treatment + _LOGGER.warning( + 'Matching Continuously is stateful, and can scale poorly. ' + 'Consider using Pub/Sub Notifications ' + '(https://cloud.google.com/storage/docs/pubsub-notifications) ' + 'if possible') def expand(self, pbegin) -> beam.PCollection[filesystem.FileMetadata]: # invoke periodic impulse diff --git a/sdks/python/apache_beam/io/filesystem_test.py b/sdks/python/apache_beam/io/filesystem_test.py index 52f0e502a2254..a4d456a366da2 100644 --- a/sdks/python/apache_beam/io/filesystem_test.py +++ b/sdks/python/apache_beam/io/filesystem_test.py @@ -26,7 +26,6 @@ import ntpath import os import posixpath -import sys import tempfile import unittest import zlib @@ -237,11 +236,7 @@ def test_match_glob(self, file_pattern, expected_object_names): expected_num_items) @parameterized.expand([ - param( - os_path=posixpath, - # re.escape does not escape forward slashes since Python 3.7 - # https://docs.python.org/3/whatsnew/3.7.html ("bpo-29995") - sep_re='\\/' if sys.version_info < (3, 7, 0) else '/'), + param(os_path=posixpath, sep_re='/'), param(os_path=ntpath, sep_re='\\\\'), ]) def test_translate_pattern(self, os_path, sep_re): diff --git a/sdks/python/apache_beam/io/gcp/big_query_query_to_table_it_test.py b/sdks/python/apache_beam/io/gcp/big_query_query_to_table_it_test.py index 501c2edee4067..e8cd888421972 100644 --- a/sdks/python/apache_beam/io/gcp/big_query_query_to_table_it_test.py +++ b/sdks/python/apache_beam/io/gcp/big_query_query_to_table_it_test.py @@ -83,7 +83,7 @@ NEW_TYPES_QUERY = ('SELECT bytes, date, time FROM [%s.%s]') DIALECT_OUTPUT_SCHEMA = ('{"fields": [{"name": "fruit","type": "STRING"}]}') DIALECT_OUTPUT_VERIFY_QUERY = ('SELECT fruit from `%s`;') -DIALECT_OUTPUT_EXPECTED = [(u'apple', ), (u'orange', )] +DIALECT_OUTPUT_EXPECTED = [('apple', ), ('orange', )] class BigQueryQueryToTableIT(unittest.TestCase): diff --git a/sdks/python/apache_beam/io/gcp/bigquery.py b/sdks/python/apache_beam/io/gcp/bigquery.py index d12dd276a1aa2..184138af75251 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery.py +++ b/sdks/python/apache_beam/io/gcp/bigquery.py @@ -371,6 +371,7 @@ def chain_after(result): from typing import Union import fastavro +from objsize import get_deep_size import apache_beam as beam from apache_beam import coders @@ -416,6 +417,7 @@ def chain_after(result): from apache_beam.transforms.util import ReshufflePerKey from apache_beam.transforms.window import GlobalWindows from apache_beam.typehints.row_type import RowTypeConstraint +from apache_beam.typehints.schemas import schema_from_element_type from apache_beam.utils import retry from apache_beam.utils.annotations import deprecated @@ -474,6 +476,13 @@ def chain_after(result): of retries. """ MAX_INSERT_RETRIES = 10000 +""" +The maximum byte size for a BigQuery legacy streaming insert payload. + +Note: The actual limit is 10MB, but we set it to 9MB to make room for request +overhead: https://cloud.google.com/bigquery/quotas#streaming_inserts +""" +MAX_INSERT_PAYLOAD_SIZE = 9 << 20 @deprecated(since='2.11.0', current="bigquery_tools.parse_table_reference") @@ -743,8 +752,17 @@ def estimate_size(self): kms_key=self.kms_key, job_labels=self._get_bq_metadata().add_additional_bq_job_labels( self.bigquery_job_labels)) - size = int(job.statistics.totalBytesProcessed) - return size + + if job.statistics.totalBytesProcessed is None: + # Some queries may not have access to `totalBytesProcessed` as a + # result of row-level security. + # > BigQuery hides sensitive statistics on all queries against + # > tables with row-level security. + # See cloud.google.com/bigquery/docs/managing-row-level-security + # and cloud.google.com/bigquery/docs/best-practices-row-level-security + return None + + return int(job.statistics.totalBytesProcessed) else: # Size estimation is best effort. We return None as we have # no access to the query that we're running. @@ -777,7 +795,8 @@ def split(self, desired_bundle_size, start_position=None, stop_position=None): if self.export_result is None: bq = bigquery_tools.BigQueryWrapper( temp_dataset_id=( - self.temp_dataset.datasetId if self.temp_dataset else None)) + self.temp_dataset.datasetId if self.temp_dataset else None), + client=bigquery_tools.BigQueryWrapper._bigquery_client(self.options)) if self.query is not None: self._setup_temporary_dataset(bq) @@ -1074,7 +1093,8 @@ def display_data(self): def estimate_size(self): # Returns the pre-filtering size of the (temporary) table being read. - bq = bigquery_tools.BigQueryWrapper() + bq = bigquery_tools.BigQueryWrapper.from_pipeline_options( + self.pipeline_options) if self.table_reference is not None: return self._get_table_size(bq, self.table_reference) elif self.query is not None and self.query.is_accessible(): @@ -1094,8 +1114,17 @@ def estimate_size(self): kms_key=self.kms_key, job_labels=self._get_bq_metadata().add_additional_bq_job_labels( self.bigquery_job_labels)) - size = int(job.statistics.totalBytesProcessed) - return size + + if job.statistics.totalBytesProcessed is None: + # Some queries may not have access to `totalBytesProcessed` as a + # result of row-level security + # > BigQuery hides sensitive statistics on all queries against + # > tables with row-level security. + # See cloud.google.com/bigquery/docs/managing-row-level-security + # and cloud.google.com/bigquery/docs/best-practices-row-level-security + return None + + return int(job.statistics.totalBytesProcessed) else: # Size estimation is best effort. We return None as we have # no access to the query that we're running. @@ -1104,7 +1133,9 @@ def estimate_size(self): def split(self, desired_bundle_size, start_position=None, stop_position=None): if self.split_result is None: bq = bigquery_tools.BigQueryWrapper( - temp_table_ref=(self.temp_table if self.temp_table else None)) + temp_table_ref=(self.temp_table if self.temp_table else None), + client=bigquery_tools.BigQueryWrapper._bigquery_client( + self.pipeline_options)) if self.query is not None: self._setup_temporary_dataset(bq) @@ -1279,7 +1310,7 @@ def __iter__(self): def __next__(self): try: return fastavro.schemaless_reader(self.bytes_reader, self.avro_schema) - except StopIteration: + except (StopIteration, EOFError): self.read_rows_response = next(self.read_rows_iterator, None) if self.read_rows_response is not None: self.bytes_reader = io.BytesIO( @@ -1325,7 +1356,8 @@ def __init__( ignore_insert_ids=False, with_batched_input=False, ignore_unknown_columns=False, - max_retries=MAX_INSERT_RETRIES): + max_retries=MAX_INSERT_RETRIES, + max_insert_payload_size=MAX_INSERT_PAYLOAD_SIZE): """Initialize a WriteToBigQuery transform. Args: @@ -1376,7 +1408,8 @@ def __init__( max_retries: The number of times that we will retry inserting a group of rows into BigQuery. By default, we retry 10000 times with exponential backoffs (effectively retry forever). - + max_insert_payload_size: The maximum byte size for a BigQuery legacy + streaming insert payload. """ self.schema = schema self.test_client = test_client @@ -1414,6 +1447,7 @@ def __init__( BigQueryWriteFn.STREAMING_API_LOGGING_FREQUENCY_SEC) self.ignore_unknown_columns = ignore_unknown_columns self._max_retries = max_retries + self._max_insert_payload_size = max_insert_payload_size def display_data(self): return { @@ -1429,6 +1463,7 @@ def display_data(self): def _reset_rows_buffer(self): self._rows_buffer = collections.defaultdict(lambda: []) + self._destination_buffer_byte_size = collections.defaultdict(lambda: 0) @staticmethod def get_table_schema(schema): @@ -1515,11 +1550,42 @@ def process(self, element, *schema_side_inputs): if not self.with_batched_input: row_and_insert_id = element[1] + row_byte_size = get_deep_size(row_and_insert_id) + + # send large rows that exceed BigQuery insert limits to DLQ + if row_byte_size >= self._max_insert_payload_size: + row_mb_size = row_byte_size / 1_000_000 + max_mb_size = self._max_insert_payload_size / 1_000_000 + error = ( + f"Received row with size {row_mb_size}MB that exceeds " + f"the maximum insert payload size set ({max_mb_size}MB).") + return [ + pvalue.TaggedOutput( + BigQueryWriteFn.FAILED_ROWS_WITH_ERRORS, + GlobalWindows.windowed_value( + (destination, row_and_insert_id[0], error))), + pvalue.TaggedOutput( + BigQueryWriteFn.FAILED_ROWS, + GlobalWindows.windowed_value( + (destination, row_and_insert_id[0]))) + ] + + # Flush current batch first if adding this row will exceed our limits + # limits: byte size; number of rows + if ((self._destination_buffer_byte_size[destination] + row_byte_size > + self._max_insert_payload_size) or + len(self._rows_buffer[destination]) >= self._max_batch_size): + flushed_batch = self._flush_batch(destination) + # After flushing our existing batch, we now buffer the current row + # for the next flush + self._rows_buffer[destination].append(row_and_insert_id) + self._destination_buffer_byte_size[destination] = row_byte_size + return flushed_batch + self._rows_buffer[destination].append(row_and_insert_id) + self._destination_buffer_byte_size[destination] += row_byte_size self._total_buffered_rows += 1 - if len(self._rows_buffer[destination]) >= self._max_batch_size: - return self._flush_batch(destination) - elif self._total_buffered_rows >= self._max_buffered_rows: + if self._total_buffered_rows >= self._max_buffered_rows: return self._flush_all_batches() else: # The input is already batched per destination, flush the rows now. @@ -1549,7 +1615,6 @@ def _flush_batch(self, destination): # Flush the current batch of rows to BigQuery. rows_and_insert_ids = self._rows_buffer[destination] table_reference = bigquery_tools.parse_table_reference(destination) - if table_reference.projectId is None: table_reference.projectId = vp.RuntimeValueProvider.get_value( 'project', str, '') @@ -1615,6 +1680,8 @@ def _flush_batch(self, destination): self._total_buffered_rows -= len(self._rows_buffer[destination]) del self._rows_buffer[destination] + if destination in self._destination_buffer_byte_size: + del self._destination_buffer_byte_size[destination] return itertools.chain([ pvalue.TaggedOutput( @@ -1657,7 +1724,8 @@ def __init__( with_auto_sharding, num_streaming_keys=DEFAULT_SHARDS_PER_DESTINATION, test_client=None, - max_retries=None): + max_retries=None, + max_insert_payload_size=MAX_INSERT_PAYLOAD_SIZE): self.table_reference = table_reference self.table_side_inputs = table_side_inputs self.schema_side_inputs = schema_side_inputs @@ -1675,6 +1743,7 @@ def __init__( self.with_auto_sharding = with_auto_sharding self._num_streaming_keys = num_streaming_keys self.max_retries = max_retries or MAX_INSERT_RETRIES + self._max_insert_payload_size = max_insert_payload_size class InsertIdPrefixFn(DoFn): def start_bundle(self): @@ -1701,7 +1770,8 @@ def expand(self, input): ignore_insert_ids=self.ignore_insert_ids, ignore_unknown_columns=self.ignore_unknown_columns, with_batched_input=self.with_auto_sharding, - max_retries=self.max_retries) + max_retries=self.max_retries, + max_insert_payload_size=self._max_insert_payload_size) def _add_random_shard(element): key = element[0] @@ -1799,8 +1869,10 @@ def __init__( # TODO(https://github.com/apache/beam/issues/20712): Switch the default # when the feature is mature. with_auto_sharding=False, + num_storage_api_streams=0, ignore_unknown_columns=False, load_job_project_id=None, + max_insert_payload_size=MAX_INSERT_PAYLOAD_SIZE, num_streaming_keys=DEFAULT_SHARDS_PER_DESTINATION, expansion_service=None): """Initialize a WriteToBigQuery transform. @@ -1947,6 +2019,9 @@ def __init__( determined number of shards to write to BigQuery. This can be used for all of FILE_LOADS, STREAMING_INSERTS, and STORAGE_WRITE_API. Only applicable to unbounded input. + num_storage_api_streams: Specifies the number of write streams that the + Storage API sink will use. This parameter is only applicable when + writing unbounded data. ignore_unknown_columns: Accept rows that contain values that do not match the schema. The unknown values are ignored. Default is False, which treats unknown values as errors. This option is only valid for @@ -1960,6 +2035,8 @@ def __init__( expansion_service: The address (host:port) of the expansion service. If no expansion service is provided, will attempt to run the default GCP expansion service. Used for STORAGE_WRITE_API method. + max_insert_payload_size: The maximum byte size for a BigQuery legacy + streaming insert payload. """ self._table = table self._dataset = dataset @@ -1987,6 +2064,7 @@ def __init__( self.use_at_least_once = use_at_least_once self.expansion_service = expansion_service self.with_auto_sharding = with_auto_sharding + self._num_storage_api_streams = num_storage_api_streams self.insert_retry_strategy = insert_retry_strategy self._validate = validate self._temp_file_format = temp_file_format or bigquery_tools.FileFormat.JSON @@ -1997,6 +2075,7 @@ def __init__( self._ignore_insert_ids = ignore_insert_ids self._ignore_unknown_columns = ignore_unknown_columns self.load_job_project_id = load_job_project_id + self._max_insert_payload_size = max_insert_payload_size self._num_streaming_keys = num_streaming_keys # Dict/schema methods were moved to bigquery_tools, but keep references @@ -2045,6 +2124,12 @@ def expand(self, pcoll): 'triggering_frequency with STREAMING_INSERTS can only be used with ' 'with_auto_sharding=True.') + if self._max_insert_payload_size > MAX_INSERT_PAYLOAD_SIZE: + raise ValueError( + 'max_insert_payload_size can only go up to ' + f'{MAX_INSERT_PAYLOAD_SIZE} bytes, as per BigQuery quota limits: ' + 'https://cloud.google.com/bigquery/quotas#streaming_inserts.') + outputs = pcoll | _StreamToBigQuery( table_reference=self.table_reference, table_side_inputs=self.table_side_inputs, @@ -2061,6 +2146,7 @@ def expand(self, pcoll): ignore_unknown_columns=self._ignore_unknown_columns, with_auto_sharding=self.with_auto_sharding, test_client=self.test_client, + max_insert_payload_size=self._max_insert_payload_size, num_streaming_keys=self._num_streaming_keys) return WriteResult( @@ -2068,6 +2154,7 @@ def expand(self, pcoll): failed_rows=outputs[BigQueryWriteFn.FAILED_ROWS], failed_rows_with_errors=outputs[ BigQueryWriteFn.FAILED_ROWS_WITH_ERRORS]) + elif method_to_use == WriteToBigQuery.Method.FILE_LOADS: if self._temp_file_format == bigquery_tools.FileFormat.AVRO: if self.schema == SCHEMA_AUTODETECT: @@ -2132,32 +2219,44 @@ def find_in_nested_dict(schema): BigQueryBatchFileLoads.DESTINATION_FILE_PAIRS], destination_copy_jobid_pairs=output[ BigQueryBatchFileLoads.DESTINATION_COPY_JOBID_PAIRS]) - else: - # Storage Write API + + elif method_to_use == WriteToBigQuery.Method.STORAGE_WRITE_API: if self.schema is None: - raise AttributeError( - "A schema is required in order to prepare rows" - "for writing with STORAGE_WRITE_API.") - if callable(self.schema): + try: + schema = schema_from_element_type(pcoll.element_type) + is_rows = True + except TypeError as exn: + raise ValueError( + "A schema is required in order to prepare rows" + "for writing with STORAGE_WRITE_API.") from exn + elif callable(self.schema): raise NotImplementedError( "Writing to dynamic destinations is not" "supported for this write method.") elif isinstance(self.schema, vp.ValueProvider): schema = self.schema.get() + is_rows = False else: schema = self.schema + is_rows = False table = bigquery_tools.get_hashable_destination(self.table_reference) # None type is not supported triggering_frequency = self.triggering_frequency or 0 # SchemaTransform expects Beam Rows, so map to Rows first + if is_rows: + input_beam_rows = pcoll + else: + input_beam_rows = ( + pcoll + | "Convert dict to Beam Row" >> beam.Map( + lambda row: bigquery_tools.beam_row_from_dict(row, schema) + ).with_output_types( + RowTypeConstraint.from_fields( + bigquery_tools.get_beam_typehints_from_tableschema(schema))) + ) output_beam_rows = ( - pcoll - | - beam.Map(lambda row: bigquery_tools.beam_row_from_dict(row, schema)). - with_output_types( - RowTypeConstraint.from_fields( - bigquery_tools.get_beam_typehints_from_tableschema(schema))) + input_beam_rows | StorageWriteToBigQuery( table=table, create_disposition=self.create_disposition, @@ -2165,25 +2264,34 @@ def find_in_nested_dict(schema): triggering_frequency=triggering_frequency, use_at_least_once=self.use_at_least_once, with_auto_sharding=self.with_auto_sharding, + num_storage_api_streams=self._num_storage_api_streams, expansion_service=self.expansion_service)) - # return back from Beam Rows to Python dict elements - failed_rows = ( - output_beam_rows[StorageWriteToBigQuery.FAILED_ROWS] - | beam.Map(lambda row: row.as_dict())) - failed_rows_with_errors = ( - output_beam_rows[StorageWriteToBigQuery.FAILED_ROWS_WITH_ERRORS] - | beam.Map( - lambda row: { - "error_message": row.error_message, - "failed_row": row.failed_row.as_dict() - })) + if is_rows: + failed_rows = output_beam_rows[StorageWriteToBigQuery.FAILED_ROWS] + failed_rows_with_errors = output_beam_rows[ + StorageWriteToBigQuery.FAILED_ROWS_WITH_ERRORS] + else: + # return back from Beam Rows to Python dict elements + failed_rows = ( + output_beam_rows[StorageWriteToBigQuery.FAILED_ROWS] + | beam.Map(lambda row: row.as_dict())) + failed_rows_with_errors = ( + output_beam_rows[StorageWriteToBigQuery.FAILED_ROWS_WITH_ERRORS] + | beam.Map( + lambda row: { + "error_message": row.error_message, + "failed_row": row.failed_row.as_dict() + })) return WriteResult( method=WriteToBigQuery.Method.STORAGE_WRITE_API, failed_rows=failed_rows, failed_rows_with_errors=failed_rows_with_errors) + else: + raise ValueError(f"Unsupported method {method_to_use}") + def display_data(self): res = {} if self.table_reference is not None and isinstance(self.table_reference, @@ -2407,7 +2515,7 @@ class StorageWriteToBigQuery(PTransform): Experimental; no backwards compatibility guarantees. """ - URN = "beam:schematransform:org.apache.beam:bigquery_storage_write:v1" + URN = "beam:schematransform:org.apache.beam:bigquery_storage_write:v2" FAILED_ROWS = "FailedRows" FAILED_ROWS_WITH_ERRORS = "FailedRowsWithErrors" @@ -2419,6 +2527,7 @@ def __init__( triggering_frequency=0, use_at_least_once=False, with_auto_sharding=False, + num_storage_api_streams=0, expansion_service=None): """Initialize a StorageWriteToBigQuery transform. @@ -2456,6 +2565,7 @@ def __init__( self._triggering_frequency = triggering_frequency self._use_at_least_once = use_at_least_once self._with_auto_sharding = with_auto_sharding + self._num_storage_api_streams = num_storage_api_streams self._expansion_service = ( expansion_service or _default_io_expansion_service()) self.schematransform_config = SchemaAwareExternalTransform.discover_config( @@ -2467,16 +2577,23 @@ def expand(self, input): expansion_service=self._expansion_service, rearrange_based_on_discovery=True, autoSharding=self._with_auto_sharding, + numStreams=self._num_storage_api_streams, createDisposition=self._create_disposition, table=self._table, triggeringFrequencySeconds=self._triggering_frequency, useAtLeastOnceSemantics=self._use_at_least_once, writeDisposition=self._write_disposition, - ) + errorHandling={ + 'output': StorageWriteToBigQuery.FAILED_ROWS_WITH_ERRORS + }) input_tag = self.schematransform_config.inputs[0] - return {input_tag: input} | external_storage_write + result = {input_tag: input} | external_storage_write + result[StorageWriteToBigQuery.FAILED_ROWS] = result[ + StorageWriteToBigQuery.FAILED_ROWS_WITH_ERRORS] | beam.Map( + lambda row_and_error: row_and_error[0]) + return result class ReadFromBigQuery(PTransform): @@ -2596,14 +2713,14 @@ def __init__( self._args = args self._kwargs = kwargs - if self.method is ReadFromBigQuery.Method.EXPORT \ + if self.method == ReadFromBigQuery.Method.EXPORT \ and self.use_native_datetime is True: raise TypeError( 'The "use_native_datetime" parameter cannot be True for EXPORT.' ' Please set the "use_native_datetime" parameter to False *OR*' ' set the "method" parameter to ReadFromBigQuery.Method.DIRECT_READ.') - if gcs_location and self.method is ReadFromBigQuery.Method.EXPORT: + if gcs_location and self.method == ReadFromBigQuery.Method.EXPORT: if not isinstance(gcs_location, (str, ValueProvider)): raise TypeError( '%s: gcs_location must be of type string' @@ -2624,14 +2741,14 @@ def __init__( } def expand(self, pcoll): - if self.method is ReadFromBigQuery.Method.EXPORT: + if self.method == ReadFromBigQuery.Method.EXPORT: output_pcollection = self._expand_export(pcoll) - elif self.method is ReadFromBigQuery.Method.DIRECT_READ: + elif self.method == ReadFromBigQuery.Method.DIRECT_READ: output_pcollection = self._expand_direct_read(pcoll) else: raise ValueError( - 'The method to read from BigQuery must be either EXPORT' + 'The method to read from BigQuery must be either EXPORT ' 'or DIRECT_READ.') return self._expand_output_type(output_pcollection) @@ -2651,12 +2768,16 @@ def _expand_output_type(self, output_pcollection): raise TypeError( '%s: table must be of type string' '; got a callable instead' % self.__class__.__name__) - return output_pcollection | bigquery_schema_tools.\ - convert_to_usertype( - bigquery_tools.BigQueryWrapper().get_table( - project_id=table_details.projectId, - dataset_id=table_details.datasetId, - table_id=table_details.tableId).schema) + return output_pcollection | bigquery_schema_tools.convert_to_usertype( + bigquery_tools.BigQueryWrapper().get_table( + project_id=table_details.projectId, + dataset_id=table_details.datasetId, + table_id=table_details.tableId).schema, + self._kwargs.get('selected_fields', None)) + else: + raise ValueError( + 'The output type from BigQuery must be either PYTHON_DICT ' + 'or BEAM_ROW.') def _expand_export(self, pcoll): # TODO(https://github.com/apache/beam/issues/20683): Make ReadFromBQ rely @@ -2707,14 +2828,14 @@ def _expand_direct_read(self, pcoll): else: project_id = pcoll.pipeline.options.view_as(GoogleCloudOptions).project + pipeline_details = {} + if temp_table_ref is not None: + pipeline_details['temp_table_ref'] = temp_table_ref + elif project_id is not None: + pipeline_details['project_id'] = project_id + pipeline_details['bigquery_dataset_labels'] = self.bigquery_dataset_labels + def _get_pipeline_details(unused_elm): - pipeline_details = {} - if temp_table_ref is not None: - pipeline_details['temp_table_ref'] = temp_table_ref - elif project_id is not None: - pipeline_details['project_id'] = project_id - pipeline_details[ - 'bigquery_dataset_labels'] = self.bigquery_dataset_labels return pipeline_details project_to_cleanup_pcoll = beam.pvalue.AsList( diff --git a/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py b/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py index 3db3d10eda689..453cd27dfdaab 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py @@ -86,7 +86,7 @@ def _generate_job_name(job_name, job_type, step_name): job_name=job_name, step_id=step_name, job_type=job_type, - random=random.randint(0, 1000)) + random=_bq_uuid()) def file_prefix_generator( @@ -657,14 +657,19 @@ def start_bundle(self): self.bq_io_metadata = create_bigquery_io_metadata(self._step_name) self.pending_jobs = [] - def process(self, element, load_job_name_prefix, *schema_side_inputs): + def process( + self, + element, + load_job_name_prefix, + pane_info=beam.DoFn.PaneInfoParam, + *schema_side_inputs): # Each load job is assumed to have files respecting these constraints: # 1. Total size of all files < 15 TB (Max size for load jobs) # 2. Total no. of files in a single load job < 10,000 # This assumption means that there will always be a single load job # triggered for each partition of files. destination = element[0] - files = element[1] + partition_key, files = element[1] if callable(self.schema): schema = self.schema(destination, *schema_side_inputs) @@ -692,8 +697,8 @@ def process(self, element, load_job_name_prefix, *schema_side_inputs): table_reference.projectId, table_reference.datasetId, table_reference.tableId)) - uid = _bq_uuid() - job_name = '%s_%s_%s' % (load_job_name_prefix, destination_hash, uid) + job_name = '%s_%s_pane%s_partition%s' % ( + load_job_name_prefix, destination_hash, pane_info.index, partition_key) _LOGGER.info('Load job has %s files. Job name is %s.', len(files), job_name) create_disposition = self.create_disposition @@ -799,8 +804,10 @@ def process(self, element): else: output_tag = PartitionFiles.SINGLE_PARTITION_TAG - for partition in partitions: - yield pvalue.TaggedOutput(output_tag, (destination, partition)) + # we also pass along the index of partition as a key, which is used + # to create a deterministic load job name + for key, partition in enumerate(partitions): + yield pvalue.TaggedOutput(output_tag, (destination, (key, partition))) class DeleteTablesFn(beam.DoFn): diff --git a/sdks/python/apache_beam/io/gcp/bigquery_file_loads_test.py b/sdks/python/apache_beam/io/gcp/bigquery_file_loads_test.py index 29d04ddd380c3..edd92f21e731a 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_file_loads_test.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_file_loads_test.py @@ -349,9 +349,9 @@ def test_partition(self): def test_partition_files_dofn_file_split(self): """Force partitions to split based on max_files""" - multiple_partitions_result = [('destination0', ['file0', 'file1']), - ('destination0', ['file2', 'file3'])] - single_partition_result = [('destination1', ['file0', 'file1'])] + multiple_partitions_result = [('destination0', (0, ['file0', 'file1'])), + ('destination0', (1, ['file2', 'file3']))] + single_partition_result = [('destination1', (0, ['file0', 'file1']))] with TestPipeline() as p: destination_file_pairs = p | beam.Create(self._ELEMENTS, reshuffle=False) partitioned_files = ( @@ -364,20 +364,22 @@ def test_partition_files_dofn_file_split(self): single_partition = partitioned_files[bqfl.PartitionFiles\ .SINGLE_PARTITION_TAG] - assert_that( - multiple_partitions, - equal_to(multiple_partitions_result), - label='CheckMultiplePartitions') - assert_that( - single_partition, - equal_to(single_partition_result), - label='CheckSinglePartition') + assert_that( + multiple_partitions, + equal_to(multiple_partitions_result), + label='CheckMultiplePartitions') + assert_that( + single_partition, + equal_to(single_partition_result), + label='CheckSinglePartition') def test_partition_files_dofn_size_split(self): """Force partitions to split based on max_partition_size""" - multiple_partitions_result = [('destination0', ['file0', 'file1', 'file2']), - ('destination0', ['file3'])] - single_partition_result = [('destination1', ['file0', 'file1'])] + multiple_partitions_result = [ + ('destination0', (0, ['file0', 'file1', + 'file2'])), ('destination0', (1, ['file3'])) + ] + single_partition_result = [('destination1', (0, ['file0', 'file1']))] with TestPipeline() as p: destination_file_pairs = p | beam.Create(self._ELEMENTS, reshuffle=False) partitioned_files = ( @@ -390,14 +392,14 @@ def test_partition_files_dofn_size_split(self): single_partition = partitioned_files[bqfl.PartitionFiles\ .SINGLE_PARTITION_TAG] - assert_that( - multiple_partitions, - equal_to(multiple_partitions_result), - label='CheckMultiplePartitions') - assert_that( - single_partition, - equal_to(single_partition_result), - label='CheckSinglePartition') + assert_that( + multiple_partitions, + equal_to(multiple_partitions_result), + label='CheckMultiplePartitions') + assert_that( + single_partition, + equal_to(single_partition_result), + label='CheckSinglePartition') class TestBigQueryFileLoads(_TestCaseWithTempDirCleanUp): @@ -584,8 +586,8 @@ def test_wait_for_load_job_completion(self, sleep_mock): bq_client.jobs.Get.side_effect = [ job_1_waiting, job_2_done, job_1_done, job_2_done ] - partition_1 = ('project:dataset.table0', ['file0']) - partition_2 = ('project:dataset.table1', ['file1']) + partition_1 = ('project:dataset.table0', (0, ['file0'])) + partition_2 = ('project:dataset.table1', (1, ['file1'])) bq_client.jobs.Insert.side_effect = [job_1, job_2] test_job_prefix = "test_job" @@ -627,8 +629,8 @@ def test_one_load_job_failed_after_waiting(self, sleep_mock): bq_client.jobs.Get.side_effect = [ job_1_waiting, job_2_done, job_1_error, job_2_done ] - partition_1 = ('project:dataset.table0', ['file0']) - partition_2 = ('project:dataset.table1', ['file1']) + partition_1 = ('project:dataset.table0', (0, ['file0'])) + partition_2 = ('project:dataset.table1', (1, ['file1'])) bq_client.jobs.Insert.side_effect = [job_1, job_2] test_job_prefix = "test_job" @@ -1041,7 +1043,7 @@ def test_bqfl_streaming_with_copy_jobs(self): # Override these parameters to induce copy jobs bqfl._DEFAULT_MAX_FILE_SIZE = 100 - bqfl._MAXIMUM_LOAD_SIZE = 400 + bqfl._MAXIMUM_LOAD_SIZE = 200 with beam.Pipeline(argv=args) as p: stream_source = ( diff --git a/sdks/python/apache_beam/io/gcp/bigquery_read_internal.py b/sdks/python/apache_beam/io/gcp/bigquery_read_internal.py index 0ca5c2e69a057..ce49cd0161df8 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_read_internal.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_read_internal.py @@ -63,7 +63,7 @@ def bigquery_export_destination_uri( - gcs_location_vp: Optional[ValueProvider], + gcs_location_vp: Union[str, Optional[ValueProvider]], temp_location: Optional[str], unique_id: str, directory_only: bool = False, @@ -75,7 +75,10 @@ def bigquery_export_destination_uri( gcs_location = None if gcs_location_vp is not None: - gcs_location = gcs_location_vp.get() + if isinstance(gcs_location_vp, ValueProvider): + gcs_location = gcs_location_vp.get() + else: + gcs_location = gcs_location_vp if gcs_location is not None: gcs_base = gcs_location @@ -144,13 +147,16 @@ def __init__(self, side_input=None): self.side_input = side_input def expand(self, input): + pipeline_options = input.pipeline.options + class PassThrough(beam.DoFn): def process(self, element): yield element class CleanUpProjects(beam.DoFn): def process(self, unused_element, unused_signal, pipeline_details): - bq = bigquery_tools.BigQueryWrapper() + bq = bigquery_tools.BigQueryWrapper.from_pipeline_options( + pipeline_options) pipeline_details = pipeline_details[0] if 'temp_table_ref' in pipeline_details.keys(): temp_table_ref = pipeline_details['temp_table_ref'] @@ -230,7 +236,8 @@ def _get_temp_dataset(self): def process(self, element: 'ReadFromBigQueryRequest') -> Iterable[BoundedSource]: bq = bigquery_tools.BigQueryWrapper( - temp_dataset_id=self._get_temp_dataset().datasetId) + temp_dataset_id=self._get_temp_dataset().datasetId, + client=bigquery_tools.BigQueryWrapper._bigquery_client(self.options)) if element.query is not None: self._setup_temporary_dataset(bq, element) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_read_it_test.py b/sdks/python/apache_beam/io/gcp/bigquery_read_it_test.py index 98a6d3831907a..589bf79437cb6 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_read_it_test.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_read_it_test.py @@ -125,9 +125,9 @@ class ReadTests(BigQueryReadIntegrationTests): }, { 'number': 2, 'str': 'def' }, { - 'number': 3, 'str': u'你好' + 'number': 3, 'str': '你好' }, { - 'number': 4, 'str': u'привет' + 'number': 4, 'str': 'привет' }] @classmethod @@ -309,14 +309,14 @@ def test_table_schema_retrieve_with_direct_read(self): class ReadUsingStorageApiTests(BigQueryReadIntegrationTests): TABLE_DATA = [{ 'number': 1, - 'string': u'你好', + 'string': '你好', 'time': '12:44:31', 'datetime': '2018-12-31 12:44:31', 'rec': None }, { 'number': 4, - 'string': u'привет', + 'string': 'привет', 'time': '12:44:31', 'datetime': '2018-12-31 12:44:31', 'rec': { @@ -425,14 +425,14 @@ def test_iobase_source(self): EXPECTED_TABLE_DATA = [ { 'number': 1, - 'string': u'你好', + 'string': '你好', 'time': datetime.time(12, 44, 31), 'datetime': '2018-12-31T12:44:31', 'rec': None, }, { 'number': 4, - 'string': u'привет', + 'string': 'привет', 'time': datetime.time(12, 44, 31), 'datetime': '2018-12-31T12:44:31', 'rec': { @@ -455,14 +455,14 @@ def test_iobase_source_with_native_datetime(self): EXPECTED_TABLE_DATA = [ { 'number': 1, - 'string': u'你好', + 'string': '你好', 'time': datetime.time(12, 44, 31), 'datetime': datetime.datetime(2018, 12, 31, 12, 44, 31), 'rec': None, }, { 'number': 4, - 'string': u'привет', + 'string': 'привет', 'time': datetime.time(12, 44, 31), 'datetime': datetime.datetime(2018, 12, 31, 12, 44, 31), 'rec': { @@ -497,7 +497,7 @@ def test_iobase_source_with_column_selection(self): def test_iobase_source_with_row_restriction(self): EXPECTED_TABLE_DATA = [{ 'number': 1, - 'string': u'你好', + 'string': '你好', 'time': datetime.time(12, 44, 31), 'datetime': datetime.datetime(2018, 12, 31, 12, 44, 31), 'rec': None @@ -513,7 +513,7 @@ def test_iobase_source_with_row_restriction(self): @pytest.mark.it_postcommit def test_iobase_source_with_column_selection_and_row_restriction(self): - EXPECTED_TABLE_DATA = [{'string': u'привет'}] + EXPECTED_TABLE_DATA = [{'string': 'привет'}] with beam.Pipeline(argv=self.args) as p: result = ( p | 'Read with BigQuery Storage API' >> beam.io.ReadFromBigQuery( @@ -523,6 +523,19 @@ def test_iobase_source_with_column_selection_and_row_restriction(self): selected_fields=['string'])) assert_that(result, equal_to(EXPECTED_TABLE_DATA)) + @pytest.mark.it_postcommit + def test_iobase_source_with_column_selection_and_row_restriction_rows(self): + with beam.Pipeline(argv=self.args) as p: + result = ( + p | 'Read with BigQuery Storage API' >> beam.io.ReadFromBigQuery( + method=beam.io.ReadFromBigQuery.Method.DIRECT_READ, + table=self.temp_table_reference, + row_restriction='number > 2', + selected_fields=['string'], + output_type='BEAM_ROW')) + assert_that( + result | beam.Map(lambda row: row.string), equal_to(['привет'])) + @pytest.mark.it_postcommit def test_iobase_source_with_very_selective_filters(self): with beam.Pipeline(argv=self.args) as p: @@ -541,14 +554,14 @@ def test_iobase_source_with_query(self): EXPECTED_TABLE_DATA = [ { 'number': 1, - 'string': u'你好', + 'string': '你好', 'time': datetime.time(12, 44, 31), 'datetime': datetime.datetime(2018, 12, 31, 12, 44, 31), 'rec': None, }, { 'number': 4, - 'string': u'привет', + 'string': 'привет', 'time': datetime.time(12, 44, 31), 'datetime': datetime.datetime(2018, 12, 31, 12, 44, 31), 'rec': { @@ -573,7 +586,7 @@ def test_iobase_source_with_query(self): @pytest.mark.it_postcommit def test_iobase_source_with_query_and_filters(self): - EXPECTED_TABLE_DATA = [{'string': u'привет'}] + EXPECTED_TABLE_DATA = [{'string': 'привет'}] query = StaticValueProvider(str, self.query) with beam.Pipeline(argv=self.args) as p: result = ( @@ -713,9 +726,9 @@ class ReadAllBQTests(BigQueryReadIntegrationTests): }, { 'number': 2, 'str': 'def' }, { - 'number': 3, 'str': u'你好' + 'number': 3, 'str': '你好' }, { - 'number': 4, 'str': u'привет' + 'number': 4, 'str': 'привет' }] TABLE_DATA_2 = [{ @@ -723,9 +736,9 @@ class ReadAllBQTests(BigQueryReadIntegrationTests): }, { 'number': 20, 'str': 'defg' }, { - 'number': 30, 'str': u'你好' + 'number': 30, 'str': '你好' }, { - 'number': 40, 'str': u'привет' + 'number': 40, 'str': 'привет' }] TABLE_DATA_3 = [{'number': 10, 'str': 'abcde', 'extra': 3}] @@ -792,10 +805,7 @@ def create_bq_schema(cls, with_extra=False): @skip(['PortableRunner', 'FlinkRunner']) @pytest.mark.it_postcommit def test_read_queries(self): - # TODO(https://github.com/apache/beam/issues/20610): Remove experiment when - # tests run on r_v2. - args = self.args + ["--experiments=use_runner_v2"] - with beam.Pipeline(argv=args) as p: + with beam.Pipeline(argv=self.args) as p: result = ( p | beam.Create([ diff --git a/sdks/python/apache_beam/io/gcp/bigquery_schema_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_schema_tools.py index 4c25aa62e0bdd..7b8a58e969780 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_schema_tools.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_schema_tools.py @@ -53,12 +53,13 @@ } -def generate_user_type_from_bq_schema(the_table_schema): +def generate_user_type_from_bq_schema(the_table_schema, selected_fields=None): #type: (bigquery.TableSchema) -> type """Convert a schema of type TableSchema into a pcollection element. Args: the_table_schema: A BQ schema of type TableSchema + selected_fields: if not None, the subset of fields to consider Returns: type: type that can be used to work with pCollections. """ @@ -67,18 +68,19 @@ def generate_user_type_from_bq_schema(the_table_schema): the_table_schema) if the_schema == {}: raise ValueError("Encountered an empty schema") - dict_of_tuples = [] - for i in range(len(the_schema['fields'])): - if the_schema['fields'][i]['type'] in BIG_QUERY_TO_PYTHON_TYPES: - typ = bq_field_to_type( - the_schema['fields'][i]['type'], the_schema['fields'][i]['mode']) + field_names_and_types = [] + for field in the_schema['fields']: + if selected_fields is not None and field['name'] not in selected_fields: + continue + if field['type'] in BIG_QUERY_TO_PYTHON_TYPES: + typ = bq_field_to_type(field['type'], field['mode']) else: raise ValueError( f"Encountered " - f"an unsupported type: {the_schema['fields'][i]['type']!r}") - # TODO svetaksundhar@: Map remaining BQ types - dict_of_tuples.append((the_schema['fields'][i]['name'], typ)) - sample_schema = beam.typehints.schemas.named_fields_to_schema(dict_of_tuples) + f"an unsupported type: {field['type']!r}") + field_names_and_types.append((field['name'], typ)) + sample_schema = beam.typehints.schemas.named_fields_to_schema( + field_names_and_types) usertype = beam.typehints.schemas.named_tuple_from_schema(sample_schema) return usertype @@ -94,8 +96,8 @@ def bq_field_to_type(field, mode): raise ValueError(f"Encountered an unsupported mode: {mode!r}") -def convert_to_usertype(table_schema): - usertype = generate_user_type_from_bq_schema(table_schema) +def convert_to_usertype(table_schema, selected_fields=None): + usertype = generate_user_type_from_bq_schema(table_schema, selected_fields) return beam.ParDo(BeamSchemaConversionDoFn(usertype)) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_schema_tools_test.py b/sdks/python/apache_beam/io/gcp/bigquery_schema_tools_test.py index dcd548d22d92f..72697e29c4d56 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_schema_tools_test.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_schema_tools_test.py @@ -53,6 +53,21 @@ def test_check_schema_conversions(self): 'count': typing.Optional[np.int64] }) + def test_check_conversion_with_selected_fields(self): + fields = [ + bigquery.TableFieldSchema(name='stn', type='STRING', mode="NULLABLE"), + bigquery.TableFieldSchema(name='temp', type='FLOAT64', mode="REPEATED"), + bigquery.TableFieldSchema(name='count', type='INTEGER', mode=None) + ] + schema = bigquery.TableSchema(fields=fields) + + usertype = bigquery_schema_tools.generate_user_type_from_bq_schema( + the_table_schema=schema, selected_fields=['stn', 'count']) + self.assertEqual( + usertype.__annotations__, { + 'stn': typing.Optional[str], 'count': typing.Optional[np.int64] + }) + def test_check_conversion_with_empty_schema(self): fields = [] schema = bigquery.TableSchema(fields=fields) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_test.py b/sdks/python/apache_beam/io/gcp/bigquery_test.py index 56a32471aaf8b..95b6c2a5fa603 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_test.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_test.py @@ -49,14 +49,12 @@ from apache_beam.io.gcp.bigquery import TableRowJsonCoder from apache_beam.io.gcp.bigquery import WriteToBigQuery from apache_beam.io.gcp.bigquery import _StreamToBigQuery -from apache_beam.io.gcp.bigquery_file_loads_test import _ELEMENTS from apache_beam.io.gcp.bigquery_read_internal import _JsonToDictCoder from apache_beam.io.gcp.bigquery_read_internal import bigquery_export_destination_uri from apache_beam.io.gcp.bigquery_tools import JSON_COMPLIANCE_ERROR from apache_beam.io.gcp.bigquery_tools import BigQueryWrapper from apache_beam.io.gcp.bigquery_tools import RetryStrategy from apache_beam.io.gcp.internal.clients import bigquery -from apache_beam.io.gcp.internal.clients.bigquery import bigquery_v2_client from apache_beam.io.gcp.pubsub import ReadFromPubSub from apache_beam.io.gcp.tests import utils from apache_beam.io.gcp.tests.bigquery_matcher import BigqueryFullResultMatcher @@ -77,12 +75,12 @@ from apache_beam.testing.util import equal_to from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display_test import DisplayDataItemMatcher -from apache_beam.utils import retry # Protect against environments where bigquery library is not available. # pylint: disable=wrong-import-order, wrong-import-position try: + from apache_beam.io.gcp.internal.clients.bigquery import bigquery_v2_client from apitools.base.py.exceptions import HttpError from google.cloud import bigquery as gcp_bigquery from google.api_core import exceptions @@ -94,6 +92,42 @@ _LOGGER = logging.getLogger(__name__) +_ELEMENTS = [ + { + 'name': 'beam', 'language': 'py' + }, + { + 'name': 'beam', 'language': 'java' + }, + { + 'name': 'beam', 'language': 'go' + }, + { + 'name': 'flink', 'language': 'java' + }, + { + 'name': 'flink', 'language': 'scala' + }, + { + 'name': 'spark', 'language': 'scala' + }, + { + 'name': 'spark', 'language': 'py' + }, + { + 'name': 'spark', 'language': 'scala' + }, + { + 'name': 'beam', 'foundation': 'apache' + }, + { + 'name': 'flink', 'foundation': 'apache' + }, + { + 'name': 'spark', 'foundation': 'apache' + }, +] + def _load_or_default(filename): try: @@ -788,6 +822,37 @@ def noop(table, **kwargs): with_auto_sharding=True, test_client=client)) + @mock.patch('google.cloud.bigquery.Client.insert_rows_json') + def test_streaming_inserts_flush_on_byte_size_limit(self, mock_insert): + mock_insert.return_value = [] + table = 'project:dataset.table' + rows = [ + { + 'columnA': 'value1' + }, + { + 'columnA': 'value2' + }, + # this very large row exceeds max size, so should be sent to DLQ + { + 'columnA': "large_string" * 100 + } + ] + with beam.Pipeline() as p: + failed_rows = ( + p + | beam.Create(rows) + | WriteToBigQuery( + table=table, + method='STREAMING_INSERTS', + create_disposition='CREATE_NEVER', + schema='columnA:STRING', + max_insert_payload_size=500)) + + expected_failed_rows = [(table, rows[2])] + assert_that(failed_rows.failed_rows, equal_to(expected_failed_rows)) + self.assertEqual(2, mock_insert.call_count) + @parameterized.expand([ param( exception_type=exceptions.Forbidden if exceptions else None, @@ -900,113 +965,297 @@ def test_copy_load_job_exception(self, exception_type, error_message): 'GCP dependencies are not installed') class BigQueryStreamingInsertsErrorHandling(unittest.TestCase): - # Using https://cloud.google.com/bigquery/docs/error-messages and - # https://googleapis.dev/python/google-api-core/latest/_modules/google - # /api_core/exceptions.html - # to determine error types and messages to try for retriables. + # Running tests with a variety of exceptions from https://googleapis.dev + # /python/google-api-core/latest/_modules/google/api_core/exceptions.html. + # Choosing some exceptions that produce reasons included in + # bigquery_tools._NON_TRANSIENT_ERRORS and some that are not @parameterized.expand([ + # reason not in _NON_TRANSIENT_ERRORS for row 1 on first attempt + # transient error retried and succeeds on second attempt, 0 rows sent to + # failed rows param( - exception_type=exceptions.Forbidden if exceptions else None, - error_reason='rateLimitExceeded'), + insert_response=[ + exceptions.TooManyRequests if exceptions else None, + None], + error_reason='Too Many Requests', # not in _NON_TRANSIENT_ERRORS + failed_rows=[]), + # reason not in _NON_TRANSIENT_ERRORS for row 1 on both attempts, sent to + # failed rows after hitting max_retries param( - exception_type=exceptions.DeadlineExceeded if exceptions else None, - error_reason='somereason'), + insert_response=[ + exceptions.InternalServerError if exceptions else None, + exceptions.InternalServerError if exceptions else None], + error_reason='Internal Server Error', # not in _NON_TRANSIENT_ERRORS + failed_rows=['value1', 'value3', 'value5']), + # reason in _NON_TRANSIENT_ERRORS for row 1 on both attempts, sent to + # failed_rows after hitting max_retries param( - exception_type=exceptions.ServiceUnavailable if exceptions else None, - error_reason='backendError'), + insert_response=[ + exceptions.Forbidden if exceptions else None, + exceptions.Forbidden if exceptions else None], + error_reason='Forbidden', # in _NON_TRANSIENT_ERRORS + failed_rows=['value1', 'value3', 'value5']), + ]) + def test_insert_rows_json_exception_retry_always( + self, insert_response, error_reason, failed_rows): + # In this test, a pipeline will always retry all caught exception types + # since RetryStrategy is not set and defaults to RETRY_ALWAYS + with mock.patch('time.sleep'): + call_counter = 0 + mock_response = mock.Mock() + mock_response.reason = error_reason + + def store_callback(table, **kwargs): + nonlocal call_counter + # raise exception if insert_response element is an exception + if insert_response[call_counter]: + exception_type = insert_response[call_counter] + call_counter += 1 + raise exception_type('some exception', response=mock_response) + # return empty list if not insert_response element, indicating + # successful call to insert_rows_json + else: + call_counter += 1 + return [] + + client = mock.Mock() + client.insert_rows_json.side_effect = store_callback + + # Using the bundle based direct runner to avoid pickling problems + # with mocks. + with beam.Pipeline(runner='BundleBasedDirectRunner') as p: + bq_write_out = ( + p + | beam.Create([{ + 'columnA': 'value1', 'columnB': 'value2' + }, { + 'columnA': 'value3', 'columnB': 'value4' + }, { + 'columnA': 'value5', 'columnB': 'value6' + }]) + # Using _StreamToBigQuery in order to be able to pass max_retries + # in order to limit run time of test with RETRY_ALWAYS + | _StreamToBigQuery( + table_reference='project:dataset.table', + table_side_inputs=[], + schema_side_inputs=[], + schema='anyschema', + batch_size=None, + triggering_frequency=None, + create_disposition='CREATE_NEVER', + write_disposition=None, + kms_key=None, + retry_strategy=RetryStrategy.RETRY_ALWAYS, + additional_bq_parameters=[], + ignore_insert_ids=False, + ignore_unknown_columns=False, + with_auto_sharding=False, + test_client=client, + max_retries=len(insert_response) - 1, + num_streaming_keys=500)) + + failed_values = ( + bq_write_out[beam_bq.BigQueryWriteFn.FAILED_ROWS] + | beam.Map(lambda x: x[1]['columnA'])) + + assert_that(failed_values, equal_to(failed_rows)) + + # Running tests with a variety of exceptions from https://googleapis.dev + # /python/google-api-core/latest/_modules/google/api_core/exceptions.html. + # Choosing some exceptions that produce reasons that are included in + # bigquery_tools._NON_TRANSIENT_ERRORS and some that are not + @parameterized.expand([ param( - exception_type=exceptions.InternalServerError if exceptions else None, - error_reason='internalError'), + # not in _NON_TRANSIENT_ERRORS + exception_type=exceptions.BadGateway if exceptions else None, + error_reason='Bad Gateway'), param( - exception_type=exceptions.InternalServerError if exceptions else None, - error_reason='backendError'), + # in _NON_TRANSIENT_ERRORS + exception_type=exceptions.Unauthorized if exceptions else None, + error_reason='Unauthorized'), ]) @mock.patch('time.sleep') @mock.patch('google.cloud.bigquery.Client.insert_rows_json') - def test_insert_all_retries_if_structured_retriable( - self, - mock_send, - unused_mock_sleep, - exception_type=None, - error_reason=None): - # In this test, a BATCH pipeline will retry the known RETRIABLE errors. + def test_insert_rows_json_exception_retry_never( + self, mock_send, unused_mock_sleep, exception_type, error_reason): + # In this test, a pipeline will never retry caught exception types + # since RetryStrategy is set to RETRY_NEVER + mock_response = mock.Mock() + mock_response.reason = error_reason mock_send.side_effect = [ - exception_type( - 'some retriable exception', errors=[{ - 'reason': error_reason - }]), - exception_type( - 'some retriable exception', errors=[{ - 'reason': error_reason - }]), - exception_type( - 'some retriable exception', errors=[{ - 'reason': error_reason - }]), - exception_type( - 'some retriable exception', errors=[{ - 'reason': error_reason - }]), + exception_type('some exception', response=mock_response) ] - with self.assertRaises(Exception) as exc: - with beam.Pipeline() as p: - _ = ( - p - | beam.Create([{ - 'columnA': 'value1' - }]) - | WriteToBigQuery( - table='project:dataset.table', - schema={ - 'fields': [{ - 'name': 'columnA', 'type': 'STRING', 'mode': 'NULLABLE' - }] - }, - create_disposition='CREATE_NEVER', - method='STREAMING_INSERTS')) - self.assertEqual(4, mock_send.call_count) - self.assertIn('some retriable exception', exc.exception.args[0]) + with beam.Pipeline(runner='BundleBasedDirectRunner') as p: + bq_write_out = ( + p + | beam.Create([{ + 'columnA': 'value1' + }, { + 'columnA': 'value2' + }]) + | WriteToBigQuery( + table='project:dataset.table', + schema={ + 'fields': [{ + 'name': 'columnA', 'type': 'STRING', 'mode': 'NULLABLE' + }] + }, + create_disposition='CREATE_NEVER', + method='STREAMING_INSERTS', + insert_retry_strategy=RetryStrategy.RETRY_NEVER)) + failed_values = ( + bq_write_out[beam_bq.BigQueryWriteFn.FAILED_ROWS_WITH_ERRORS] + | beam.Map(lambda x: x[1]['columnA'])) + + assert_that(failed_values, equal_to(['value1', 'value2'])) + + self.assertEqual(1, mock_send.call_count) - # Using https://googleapis.dev/python/google-api-core/latest/_modules/google - # /api_core/exceptions.html - # to determine error types and messages to try for retriables. + # Running tests with a variety of exceptions from https://googleapis.dev + # /python/google-api-core/latest/_modules/google/api_core/exceptions.html. + # Choosing some exceptions that produce reasons that are included in + # bigquery_tools._NON_TRANSIENT_ERRORS and some that are not @parameterized.expand([ param( - exception_type=requests.exceptions.ConnectionError, - error_message='some connection error'), + exception_type=exceptions.DeadlineExceeded if exceptions else None, + error_reason='Deadline Exceeded', # not in _NON_TRANSIENT_ERRORS + failed_values=[], + expected_call_count=2), param( - exception_type=requests.exceptions.Timeout, - error_message='some timeout error'), + exception_type=exceptions.Conflict if exceptions else None, + error_reason='Conflict', # not in _NON_TRANSIENT_ERRORS + failed_values=[], + expected_call_count=2), param( - exception_type=ConnectionError, - error_message='some py connection error'), + exception_type=exceptions.TooManyRequests if exceptions else None, + error_reason='Too Many Requests', # not in _NON_TRANSIENT_ERRORS + failed_values=[], + expected_call_count=2), + param( + exception_type=exceptions.InternalServerError if exceptions else None, + error_reason='Internal Server Error', # not in _NON_TRANSIENT_ERRORS + failed_values=[], + expected_call_count=2), param( exception_type=exceptions.BadGateway if exceptions else None, - error_message='some badgateway error'), + error_reason='Bad Gateway', # not in _NON_TRANSIENT_ERRORS + failed_values=[], + expected_call_count=2), + param( + exception_type=exceptions.ServiceUnavailable if exceptions else None, + error_reason='Service Unavailable', # not in _NON_TRANSIENT_ERRORS + failed_values=[], + expected_call_count=2), + param( + exception_type=exceptions.GatewayTimeout if exceptions else None, + error_reason='Gateway Timeout', # not in _NON_TRANSIENT_ERRORS + failed_values=[], + expected_call_count=2), + param( + exception_type=exceptions.BadRequest if exceptions else None, + error_reason='Bad Request', # in _NON_TRANSIENT_ERRORS + failed_values=['value1', 'value2'], + expected_call_count=1), + param( + exception_type=exceptions.Unauthorized if exceptions else None, + error_reason='Unauthorized', # in _NON_TRANSIENT_ERRORS + failed_values=['value1', 'value2'], + expected_call_count=1), + param( + exception_type=exceptions.Forbidden if exceptions else None, + error_reason='Forbidden', # in _NON_TRANSIENT_ERRORS + failed_values=['value1', 'value2'], + expected_call_count=1), + param( + exception_type=exceptions.NotFound if exceptions else None, + error_reason='Not Found', # in _NON_TRANSIENT_ERRORS + failed_values=['value1', 'value2'], + expected_call_count=1), + param( + exception_type=exceptions.MethodNotImplemented + if exceptions else None, + error_reason='Not Implemented', # in _NON_TRANSIENT_ERRORS + failed_values=['value1', 'value2'], + expected_call_count=1), ]) @mock.patch('time.sleep') @mock.patch('google.cloud.bigquery.Client.insert_rows_json') - def test_insert_all_retries_if_unstructured_retriable( + def test_insert_rows_json_exception_retry_on_transient_error( self, mock_send, unused_mock_sleep, - exception_type=None, - error_message=None): - # In this test, a BATCH pipeline will retry the unknown RETRIABLE errors. + exception_type, + error_reason, + failed_values, + expected_call_count): + # In this test, a pipeline will only retry caught exception types + # with reasons that are not in _NON_TRANSIENT_ERRORS since RetryStrategy is + # set to RETRY_ON_TRANSIENT_ERROR + mock_response = mock.Mock() + mock_response.reason = error_reason mock_send.side_effect = [ - exception_type(error_message), - exception_type(error_message), - exception_type(error_message), - exception_type(error_message), + exception_type('some exception', response=mock_response), + # Return no exception and no errors on 2nd call, if there is a 2nd call + [] ] + with beam.Pipeline(runner='BundleBasedDirectRunner') as p: + bq_write_out = ( + p + | beam.Create([{ + 'columnA': 'value1' + }, { + 'columnA': 'value2' + }]) + | WriteToBigQuery( + table='project:dataset.table', + schema={ + 'fields': [{ + 'name': 'columnA', 'type': 'STRING', 'mode': 'NULLABLE' + }] + }, + create_disposition='CREATE_NEVER', + method='STREAMING_INSERTS', + insert_retry_strategy=RetryStrategy.RETRY_ON_TRANSIENT_ERROR)) + failed_values_out = ( + bq_write_out[beam_bq.BigQueryWriteFn.FAILED_ROWS] + | beam.Map(lambda x: x[1]['columnA'])) + + assert_that(failed_values_out, equal_to(failed_values)) + self.assertEqual(expected_call_count, mock_send.call_count) + + # Running tests with persistent exceptions with exception types not + # caught in BigQueryWrapper._insert_all_rows but retriable by + # retry.with_exponential_backoff + @parameterized.expand([ + param( + exception_type=requests.exceptions.ConnectionError, + error_message='some connection error'), + param( + exception_type=requests.exceptions.Timeout, + error_message='some timeout error'), + ]) + @mock.patch('time.sleep') + @mock.patch('google.cloud.bigquery.Client.insert_rows_json') + def test_insert_rows_json_persistent_retriable_exception( + self, mock_send, unused_mock_sleep, exception_type, error_message): + # In this test, each insert_rows_json call will result in an exception + # and be retried with retry.with_exponential_backoff until MAX_RETRIES is + # reached + mock_send.side_effect = exception_type(error_message) + + # Expecting 1 initial call plus maximum number of retries + expected_call_count = 1 + bigquery_tools.MAX_RETRIES + with self.assertRaises(Exception) as exc: with beam.Pipeline() as p: _ = ( p | beam.Create([{ 'columnA': 'value1' + }, { + 'columnA': 'value2' }]) | WriteToBigQuery( table='project:dataset.table', @@ -1017,138 +1266,390 @@ def test_insert_all_retries_if_unstructured_retriable( }, create_disposition='CREATE_NEVER', method='STREAMING_INSERTS')) - self.assertEqual(4, mock_send.call_count) + + self.assertEqual(expected_call_count, mock_send.call_count) self.assertIn(error_message, exc.exception.args[0]) - # Using https://googleapis.dev/python/google-api-core/latest/_modules/google - # /api_core/exceptions.html - # to determine error types and messages to try for retriables. + # Running tests with intermittent exceptions with exception types not + # caught in BigQueryWrapper._insert_all_rows but retriable by + # retry.with_exponential_backoff @parameterized.expand([ param( - exception_type=retry.PermanentException, - error_args=('nonretriable', )), - param( - exception_type=exceptions.BadRequest if exceptions else None, - error_args=( - 'forbidden morbidden', [{ - 'reason': 'nonretriablereason' - }])), - param( - exception_type=exceptions.BadRequest if exceptions else None, - error_args=('BAD REQUEST!', [{ - 'reason': 'nonretriablereason' - }])), - param( - exception_type=exceptions.MethodNotAllowed if exceptions else None, - error_args=( - 'method not allowed!', [{ - 'reason': 'nonretriablereason' - }])), - param( - exception_type=exceptions.MethodNotAllowed if exceptions else None, - error_args=('method not allowed!', 'args')), - param( - exception_type=exceptions.Unknown if exceptions else None, - error_args=('unknown!', 'args')), + exception_type=requests.exceptions.ConnectionError, + error_message='some connection error'), param( - exception_type=exceptions.Aborted if exceptions else None, - error_args=('abortet!', 'abort')), + exception_type=requests.exceptions.Timeout, + error_message='some timeout error'), ]) @mock.patch('time.sleep') @mock.patch('google.cloud.bigquery.Client.insert_rows_json') - def test_insert_all_unretriable_errors( - self, mock_send, unused_mock_sleep, exception_type=None, error_args=None): - # In this test, a BATCH pipeline will retry the unknown RETRIABLE errors. + def test_insert_rows_json_intermittent_retriable_exception( + self, mock_send, unused_mock_sleep, exception_type, error_message): + # In this test, the first 2 insert_rows_json calls will result in an + # exception and be retried with retry.with_exponential_backoff. The last + # call will not raise an exception and will succeed. mock_send.side_effect = [ - exception_type(*error_args), - exception_type(*error_args), - exception_type(*error_args), - exception_type(*error_args), + exception_type(error_message), exception_type(error_message), [] ] - with self.assertRaises(Exception): - with beam.Pipeline() as p: - _ = ( + with beam.Pipeline() as p: + _ = ( + p + | beam.Create([{ + 'columnA': 'value1' + }, { + 'columnA': 'value2' + }]) + | WriteToBigQuery( + table='project:dataset.table', + schema={ + 'fields': [{ + 'name': 'columnA', 'type': 'STRING', 'mode': 'NULLABLE' + }] + }, + create_disposition='CREATE_NEVER', + method='STREAMING_INSERTS')) + + self.assertEqual(3, mock_send.call_count) + + # Running tests with a variety of error reasons from + # https://cloud.google.com/bigquery/docs/error-messages + # This covers the scenario when + # the google.cloud.bigquery.Client.insert_rows_json call returns an error list + # rather than raising an exception. + # Choosing some error reasons that are included in + # bigquery_tools._NON_TRANSIENT_ERRORS and some that are not + @parameterized.expand([ + # reason in _NON_TRANSIENT_ERRORS for row 1, sent to failed_rows + param( + insert_response=[ + [{ + 'index': 0, 'errors': [{ + 'reason': 'invalid' + }] + }], + [{ + 'index': 0, 'errors': [{ + 'reason': 'invalid' + }] + }], + ], + failed_rows=['value1']), + # reason in _NON_TRANSIENT_ERRORS for row 1 + # reason not in _NON_TRANSIENT_ERRORS for row 2 on 1st run + # row 1 sent to failed_rows + param( + insert_response=[ + [{ + 'index': 0, 'errors': [{ + 'reason': 'invalid' + }] + }, { + 'index': 1, 'errors': [{ + 'reason': 'internalError' + }] + }], + [{ + 'index': 0, 'errors': [{ + 'reason': 'invalid' + }] + }], + ], + failed_rows=['value1']), + # reason not in _NON_TRANSIENT_ERRORS for row 1 on first attempt + # transient error succeeds on second attempt, 0 rows sent to failed rows + param( + insert_response=[ + [{ + 'index': 0, 'errors': [{ + 'reason': 'internalError' + }] + }], + [], + ], + failed_rows=[]), + ]) + def test_insert_rows_json_errors_retry_always( + self, insert_response, failed_rows, unused_sleep_mock=None): + # In this test, a pipeline will always retry all errors + # since RetryStrategy is not set and defaults to RETRY_ALWAYS + with mock.patch('time.sleep'): + call_counter = 0 + + def store_callback(table, **kwargs): + nonlocal call_counter + response = insert_response[call_counter] + call_counter += 1 + return response + + client = mock.Mock() + client.insert_rows_json = mock.Mock(side_effect=store_callback) + + # Using the bundle based direct runner to avoid pickling problems + # with mocks. + with beam.Pipeline(runner='BundleBasedDirectRunner') as p: + bq_write_out = ( p | beam.Create([{ - 'columnA': 'value1' + 'columnA': 'value1', 'columnB': 'value2' + }, { + 'columnA': 'value3', 'columnB': 'value4' + }, { + 'columnA': 'value5', 'columnB': 'value6' }]) - | WriteToBigQuery( - table='project:dataset.table', - schema={ - 'fields': [{ - 'name': 'columnA', 'type': 'STRING', 'mode': 'NULLABLE' - }] - }, + # Using _StreamToBigQuery in order to be able to pass max_retries + # in order to limit run time of test with RETRY_ALWAYS + | _StreamToBigQuery( + table_reference='project:dataset.table', + table_side_inputs=[], + schema_side_inputs=[], + schema='anyschema', + batch_size=None, + triggering_frequency=None, create_disposition='CREATE_NEVER', - method='STREAMING_INSERTS')) - self.assertEqual(1, mock_send.call_count) + write_disposition=None, + kms_key=None, + retry_strategy=RetryStrategy.RETRY_ALWAYS, + additional_bq_parameters=[], + ignore_insert_ids=False, + ignore_unknown_columns=False, + with_auto_sharding=False, + test_client=client, + max_retries=len(insert_response) - 1, + num_streaming_keys=500)) + + failed_values = ( + bq_write_out[beam_bq.BigQueryWriteFn.FAILED_ROWS] + | beam.Map(lambda x: x[1]['columnA'])) - # Using https://googleapis.dev/python/google-api-core/latest/_modules/google - # /api_core/exceptions.html - # to determine error types and messages to try for retriables. + assert_that(failed_values, equal_to(failed_rows)) + + # Running tests with a variety of error reasons from + # https://cloud.google.com/bigquery/docs/error-messages + # This covers the scenario when + # the google.cloud.bigquery.Client.insert_rows_json call returns an error list + # rather than raising an exception. + # Choosing some error reasons that are included in + # bigquery_tools._NON_TRANSIENT_ERRORS and some that are not @parameterized.expand([ + # reason in _NON_TRANSIENT_ERRORS for row 1, sent to failed_rows param( - exception_type=retry.PermanentException, - error_args=('nonretriable', )), - param( - exception_type=exceptions.BadRequest if exceptions else None, - error_args=( - 'forbidden morbidden', [{ - 'reason': 'nonretriablereason' - }])), + insert_response=[ + [{ + 'index': 0, 'errors': [{ + 'reason': 'invalidQuery' + }] + }], + ], + streaming=False), + # reason not in _NON_TRANSIENT_ERRORS for row 1, sent to failed_rows param( - exception_type=exceptions.BadRequest if exceptions else None, - error_args=('BAD REQUEST!', [{ - 'reason': 'nonretriablereason' - }])), + insert_response=[ + [{ + 'index': 0, 'errors': [{ + 'reason': 'internalError' + }] + }], + ], + streaming=False), param( - exception_type=exceptions.MethodNotAllowed if exceptions else None, - error_args=( - 'method not allowed!', [{ - 'reason': 'nonretriablereason' - }])), + insert_response=[ + [{ + 'index': 0, 'errors': [{ + 'reason': 'invalid' + }] + }], + ], + streaming=True), + # reason not in _NON_TRANSIENT_ERRORS for row 1, sent to failed_rows param( - exception_type=exceptions.MethodNotAllowed if exceptions else None, - error_args=('method not allowed!', 'args')), + insert_response=[ + [{ + 'index': 0, 'errors': [{ + 'reason': 'internalError' + }] + }], + ], + streaming=True), + ]) + @mock.patch('time.sleep') + @mock.patch('google.cloud.bigquery.Client.insert_rows_json') + def test_insert_rows_json_errors_retry_never( + self, mock_send, unused_mock_sleep, insert_response, streaming): + # In this test, a pipeline will never retry errors since RetryStrategy is + # set to RETRY_NEVER + mock_send.side_effect = insert_response + opt = StandardOptions() + opt.streaming = streaming + with beam.Pipeline(runner='BundleBasedDirectRunner', options=opt) as p: + bq_write_out = ( + p + | beam.Create([{ + 'columnA': 'value1' + }, { + 'columnA': 'value2' + }]) + | WriteToBigQuery( + table='project:dataset.table', + schema={ + 'fields': [{ + 'name': 'columnA', 'type': 'STRING', 'mode': 'NULLABLE' + }] + }, + create_disposition='CREATE_NEVER', + method='STREAMING_INSERTS', + insert_retry_strategy=RetryStrategy.RETRY_NEVER)) + failed_values = ( + bq_write_out[beam_bq.BigQueryWriteFn.FAILED_ROWS_WITH_ERRORS] + | beam.Map(lambda x: x[1]['columnA'])) + + assert_that(failed_values, equal_to(['value1'])) + + self.assertEqual(1, mock_send.call_count) + + # Running tests with a variety of error reasons from + # https://cloud.google.com/bigquery/docs/error-messages + # This covers the scenario when + # the google.cloud.bigquery.Client.insert_rows_json call returns an error list + # rather than raising an exception. + # Choosing some error reasons that are included in + # bigquery_tools._NON_TRANSIENT_ERRORS and some that are not + @parameterized.expand([ + # reason in _NON_TRANSIENT_ERRORS for row 1, sent to failed_rows param( - exception_type=exceptions.Unknown if exceptions else None, - error_args=('unknown!', 'args')), + insert_response=[ + [{ + 'index': 0, 'errors': [{ + 'reason': 'invalid' + }] + }], + ], + failed_rows=['value1'], + streaming=False), + # reason not in _NON_TRANSIENT_ERRORS for row 1 on 1st attempt + # transient error succeeds on 2nd attempt, 0 rows sent to failed rows param( - exception_type=exceptions.Aborted if exceptions else None, - error_args=('abortet!', 'abort')), + insert_response=[ + [{ + 'index': 0, 'errors': [{ + 'reason': 'internalError' + }] + }], + [], + ], + failed_rows=[], + streaming=False), + # reason in _NON_TRANSIENT_ERRORS for row 1 + # reason not in _NON_TRANSIENT_ERRORS for row 2 on 1st and 2nd attempt + # all rows with errors are retried when any row has a retriable error + # row 1 sent to failed_rows after final attempt param( - exception_type=requests.exceptions.ConnectionError, - error_args=('some connection error', )), + insert_response=[ + [{ + 'index': 0, 'errors': [{ + 'reason': 'invalid' + }] + }, { + 'index': 1, 'errors': [{ + 'reason': 'internalError' + }] + }], + [ + { + 'index': 0, 'errors': [{ + 'reason': 'invalid' + }] + }, + ], + ], + failed_rows=['value1'], + streaming=False), + # reason in _NON_TRANSIENT_ERRORS for row 1, sent to failed_rows param( - exception_type=requests.exceptions.Timeout, - error_args=('some timeout error', )), + insert_response=[ + [{ + 'index': 0, 'errors': [{ + 'reason': 'invalid' + }] + }], + ], + failed_rows=['value1'], + streaming=True), + # reason not in _NON_TRANSIENT_ERRORS for row 1 on 1st attempt + # transient error succeeds on 2nd attempt, 0 rows sent to failed rows param( - exception_type=ConnectionError, - error_args=('some py connection error', )), + insert_response=[ + [{ + 'index': 0, 'errors': [{ + 'reason': 'internalError' + }] + }], + [], + ], + failed_rows=[], + streaming=True), + # reason in _NON_TRANSIENT_ERRORS for row 1 + # reason not in _NON_TRANSIENT_ERRORS for row 2 on 1st and 2nd attempt + # all rows with errors are retried when any row has a retriable error + # row 1 sent to failed_rows after final attempt param( - exception_type=exceptions.BadGateway if exceptions else None, - error_args=('some badgateway error', )), + insert_response=[ + [{ + 'index': 0, 'errors': [{ + 'reason': 'invalid' + }] + }, { + 'index': 1, 'errors': [{ + 'reason': 'internalError' + }] + }], + [ + { + 'index': 0, 'errors': [{ + 'reason': 'invalid' + }] + }, + ], + ], + failed_rows=['value1'], + streaming=True), ]) @mock.patch('time.sleep') @mock.patch('google.cloud.bigquery.Client.insert_rows_json') - def test_insert_all_unretriable_errors_streaming( - self, mock_send, unused_mock_sleep, exception_type=None, error_args=None): - # In this test, a STREAMING pipeline will retry ALL errors, and never throw - # an exception. - mock_send.side_effect = [ - exception_type(*error_args), - exception_type(*error_args), - [] # Errors thrown twice, and then succeeded - ] + def test_insert_rows_json_errors_retry_on_transient_error( + self, + mock_send, + unused_mock_sleep, + insert_response, + failed_rows, + streaming=False): + # In this test, a pipeline will only retry errors with reasons that are not + # in _NON_TRANSIENT_ERRORS since RetryStrategy is set to + # RETRY_ON_TRANSIENT_ERROR + call_counter = 0 + + def store_callback(table, **kwargs): + nonlocal call_counter + response = insert_response[call_counter] + call_counter += 1 + return response + + mock_send.side_effect = store_callback opt = StandardOptions() - opt.streaming = True + opt.streaming = streaming + + # Using the bundle based direct runner to avoid pickling problems + # with mocks. with beam.Pipeline(runner='BundleBasedDirectRunner', options=opt) as p: - _ = ( + bq_write_out = ( p | beam.Create([{ 'columnA': 'value1' + }, { + 'columnA': 'value2' + }, { + 'columnA': 'value3' }]) | WriteToBigQuery( table='project:dataset.table', @@ -1158,8 +1659,14 @@ def test_insert_all_unretriable_errors_streaming( }] }, create_disposition='CREATE_NEVER', - method='STREAMING_INSERTS')) - self.assertEqual(3, mock_send.call_count) + method='STREAMING_INSERTS', + insert_retry_strategy=RetryStrategy.RETRY_ON_TRANSIENT_ERROR)) + + failed_values = ( + bq_write_out[beam_bq.BigQueryWriteFn.FAILED_ROWS] + | beam.Map(lambda x: x[1]['columnA'])) + + assert_that(failed_values, equal_to(failed_rows)) @unittest.skipIf(HttpError is None, 'GCP dependencies are not installed') @@ -1204,6 +1711,7 @@ def test_dofn_client_process_flush_called(self): fn.start_bundle() fn.process(('project-id:dataset_id.table_id', ({'month': 1}, 'insertid1'))) fn.process(('project-id:dataset_id.table_id', ({'month': 2}, 'insertid2'))) + fn.finish_bundle() # InsertRows called as batch size is hit self.assertTrue(client.insert_rows_json.called) @@ -1467,76 +1975,6 @@ def store_callback(table, **kwargs): result) self.assertEqual(len(data1['colA_values']), 1) - @parameterized.expand([ - param(retry_strategy=RetryStrategy.RETRY_ALWAYS), - param(retry_strategy=RetryStrategy.RETRY_NEVER), - param(retry_strategy=RetryStrategy.RETRY_ON_TRANSIENT_ERROR), - ]) - def test_permanent_failure_in_some_rows_does_not_duplicate( - self, unused_sleep_mock=None, retry_strategy=None): - with mock.patch('time.sleep'): - - def store_callback(table, **kwargs): - return [ - { - 'index': 0, - 'errors': [{ - 'reason': 'invalid' - }, { - 'reason': 'its bad' - }] - }, - ] - - client = mock.Mock() - client.insert_rows_json = mock.Mock(side_effect=store_callback) - - # The expected rows to be inserted according to the insert strategy - if retry_strategy == RetryStrategy.RETRY_NEVER: - inserted_rows = ['value3', 'value5'] - else: # RETRY_ALWAYS and RETRY_ON_TRANSIENT_ERRORS should insert all rows - inserted_rows = ['value3', 'value5'] - - # Using the bundle based direct runner to avoid pickling problems - # with mocks. - with beam.Pipeline(runner='BundleBasedDirectRunner') as p: - bq_write_out = ( - p - | beam.Create([{ - 'columnA': 'value1', 'columnB': 'value2' - }, { - 'columnA': 'value3', 'columnB': 'value4' - }, { - 'columnA': 'value5', 'columnB': 'value6' - }]) - | _StreamToBigQuery( - table_reference='project:dataset.table', - table_side_inputs=[], - schema_side_inputs=[], - schema='anyschema', - batch_size=None, - triggering_frequency=None, - create_disposition='CREATE_NEVER', - write_disposition=None, - kms_key=None, - retry_strategy=retry_strategy, - additional_bq_parameters=[], - ignore_insert_ids=False, - ignore_unknown_columns=False, - with_auto_sharding=False, - test_client=client, - max_retries=10, - num_streaming_keys=500)) - - failed_values = ( - bq_write_out[beam_bq.BigQueryWriteFn.FAILED_ROWS] - | beam.Map(lambda x: x[1]['columnA'])) - - assert_that( - failed_values, - equal_to( - list({'value1', 'value3', 'value5'}.difference(inserted_rows)))) - @parameterized.expand([ param(with_auto_sharding=False), param(with_auto_sharding=True), diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py index 312e3f70c2b01..2f9420795288f 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py @@ -645,7 +645,7 @@ def wait_for_bq_job(self, job_reference, sleep_duration_sec=5, max_retries=0): retry += 1 job = self.get_job( job_reference.projectId, job_reference.jobId, job_reference.location) - logging.info('Job status: %s', job.status.state) + logging.info('Job %s status: %s', job.id, job.status.state) if job.status.state == 'DONE' and job.status.errorResult: raise RuntimeError( 'BigQuery job {} failed. Error Result: {}'.format( @@ -732,11 +732,13 @@ def _insert_all_rows( except (ClientError, GoogleAPICallError) as e: # e.code contains the numeric http status code. service_call_metric.call(e.code) - # Re-reise the exception so that we re-try appropriately. - raise + # Package exception with required fields + error = {'message': e.message, 'reason': e.response.reason} + # Add all rows to the errors list along with the error + errors = [{"index": i, "errors": [error]} for i, _ in enumerate(rows)] except HttpError as e: service_call_metric.call(e) - # Re-reise the exception so that we re-try appropriately. + # Re-raise the exception so that we re-try appropriately. raise finally: self._latency_histogram_metric.update( @@ -1491,7 +1493,19 @@ class RetryStrategy(object): RETRY_NEVER = 'RETRY_NEVER' RETRY_ON_TRANSIENT_ERROR = 'RETRY_ON_TRANSIENT_ERROR' - _NON_TRANSIENT_ERRORS = {'invalid', 'invalidQuery', 'notImplemented'} + # Values below may be found in reasons provided either in an + # error returned by a client method or by an http response as + # defined in google.api_core.exceptions + _NON_TRANSIENT_ERRORS = { + 'invalid', + 'invalidQuery', + 'notImplemented', + 'Bad Request', + 'Unauthorized', + 'Forbidden', + 'Not Found', + 'Not Implemented', + } @staticmethod def should_retry(strategy, error_message): @@ -1558,23 +1572,32 @@ def beam_row_from_dict(row: dict, schema): """ if not isinstance(schema, (bigquery.TableSchema, bigquery.TableFieldSchema)): schema = get_bq_tableschema(schema) - schema_fields = {field.name: field for field in schema.fields} beam_row = {} - for col_name, value in row.items(): - # get this column's schema field and handle struct types - field = schema_fields[col_name] - if field.type.upper() in ["RECORD", "STRUCT"]: + for field in schema.fields: + name = field.name + mode = field.mode.upper() + type = field.type.upper() + # When writing with Storage Write API via xlang, we give the Beam Row + # PCollection a hint on the schema using `with_output_types`. + # This requires that each row has all the fields in the schema. + # However, it's possible that some nullable fields don't appear in the row. + # For this case, we create the field with a `None` value + if name not in row and mode == "NULLABLE": + row[name] = None + + value = row[name] + if type in ["RECORD", "STRUCT"]: # if this is a list of records, we create a list of Beam Rows - if field.mode.upper() == "REPEATED": + if mode == "REPEATED": list_of_beam_rows = [] for record in value: list_of_beam_rows.append(beam_row_from_dict(record, field)) - beam_row[col_name] = list_of_beam_rows + beam_row[name] = list_of_beam_rows # otherwise, create a Beam Row from this record else: - beam_row[col_name] = beam_row_from_dict(value, field) + beam_row[name] = beam_row_from_dict(value, field) else: - beam_row[col_name] = value + beam_row[name] = value return apache_beam.pvalue.Row(**beam_row) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py b/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py index a3e39d8e18d1e..b4c84d589c07d 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py @@ -861,6 +861,11 @@ def test_dict_to_beam_row_all_types_nullable(self): schema = {"fields": self.get_schema_fields_with_mode("nullable")} dict_row = {k: None for k in self.DICT_ROW} + # input dict row with missing nullable fields should still yield a full + # Beam Row + del dict_row['str'] + del dict_row['bool'] + expected_beam_row = beam.Row( str=None, bool=None, diff --git a/sdks/python/apache_beam/io/gcp/bigquery_write_it_test.py b/sdks/python/apache_beam/io/gcp/bigquery_write_it_test.py index a307e06ac5b85..4b728fe7ec1f9 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_write_it_test.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_write_it_test.py @@ -127,10 +127,10 @@ def test_big_query_write(self): 'number': 2, 'str': 'def' }, { - 'number': 3, 'str': u'你好' + 'number': 3, 'str': '你好' }, { - 'number': 4, 'str': u'привет' + 'number': 4, 'str': 'привет' }, ] table_schema = { @@ -153,10 +153,10 @@ def test_big_query_write(self): 'def', ), ( 3, - u'你好', + '你好', ), ( 4, - u'привет', + 'привет', )]) ] @@ -379,7 +379,7 @@ def test_big_query_write_without_schema(self): def test_big_query_write_insert_errors_reporting(self): """ Test that errors returned by beam.io.WriteToBigQuery - contain both the failed rows amd the reason for it failing. + contain both the failed rows and the reason for it failing. """ table_name = 'python_write_table' table_id = '{}.{}'.format(self.dataset_id, table_name) @@ -454,6 +454,55 @@ def test_big_query_write_insert_errors_reporting(self): | 'ParseErrors' >> beam.Map(lambda err: (err[1], err[2])), equal_to(bq_result_errors)) + @pytest.mark.it_postcommit + def test_big_query_write_insert_non_transient_api_call_error(self): + """ + Test that non-transient GoogleAPICallError errors returned + by beam.io.WriteToBigQuery are not retried and result in + FAILED_ROWS containing both the failed rows and the reason + for failure. + """ + table_name = 'this_table_does_not_exist' + table_id = '{}.{}'.format(self.dataset_id, table_name) + + input_data = [{ + 'number': 1, + 'str': 'some_string', + }] + + table_schema = { + "fields": [{ + "name": "number", "type": "INTEGER", 'mode': 'NULLABLE' + }, { + "name": "str", "type": "STRING", 'mode': 'NULLABLE' + }] + } + + bq_result_errors = [({ + 'number': 1, + 'str': 'some_string', + }, "Not Found")] + + args = self.test_pipeline.get_full_options_as_args() + + with beam.Pipeline(argv=args) as p: + # pylint: disable=expression-not-assigned + errors = ( + p | 'create' >> beam.Create(input_data) + | 'write' >> beam.io.WriteToBigQuery( + table_id, + schema=table_schema, + method='STREAMING_INSERTS', + insert_retry_strategy='RETRY_ON_TRANSIENT_ERROR', + create_disposition=beam.io.BigQueryDisposition.CREATE_NEVER, + write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND)) + + assert_that( + errors[BigQueryWriteFn.FAILED_ROWS_WITH_ERRORS] + | + 'ParseErrors' >> beam.Map(lambda err: (err[1], err[2][0]["reason"])), + equal_to(bq_result_errors)) + @pytest.mark.it_postcommit @parameterized.expand([ param(file_format=FileFormat.AVRO), diff --git a/sdks/python/apache_beam/io/gcp/bigtableio.py b/sdks/python/apache_beam/io/gcp/bigtableio.py index eedfb8f1c811b..f8534f38ddfca 100644 --- a/sdks/python/apache_beam/io/gcp/bigtableio.py +++ b/sdks/python/apache_beam/io/gcp/bigtableio.py @@ -38,6 +38,9 @@ # pytype: skip-file import logging +import struct +from typing import Dict +from typing import List import apache_beam as beam from apache_beam.internal.metrics.metric import ServiceCallMetric @@ -48,6 +51,7 @@ from apache_beam.transforms.display import DisplayDataItem from apache_beam.transforms.external import BeamJarExpansionService from apache_beam.transforms.external import SchemaAwareExternalTransform +from apache_beam.typehints.row_type import RowTypeConstraint _LOGGER = logging.getLogger(__name__) @@ -168,34 +172,118 @@ def display_data(self): class WriteToBigTable(beam.PTransform): - """ A transform to write to the Bigtable Table. + """A transform that writes rows to a Bigtable table. - A PTransform that write a list of `DirectRow` into the Bigtable Table + Takes an input PCollection of `DirectRow` objects containing un-committed + mutations. For more information about this row object, visit + https://cloud.google.com/python/docs/reference/bigtable/latest/row#class-googlecloudbigtablerowdirectrowrowkey-tablenone + If flag `use_cross_language` is set to true, this transform will use the + multi-language transforms framework to inject the Java native write transform + into the pipeline. """ - def __init__(self, project_id=None, instance_id=None, table_id=None): - """ The PTransform to access the Bigtable Write connector - Args: - project_id(str): GCP Project of to write the Rows - instance_id(str): GCP Instance to write the Rows - table_id(str): GCP Table to write the `DirectRows` + URN = "beam:schematransform:org.apache.beam:bigtable_write:v1" + + def __init__( + self, + project_id, + instance_id, + table_id, + use_cross_language=False, + expansion_service=None): + """Initialize an WriteToBigTable transform. + + :param table_id: + The ID of the table to write to. + :param instance_id: + The ID of the instance where the table resides. + :param project_id: + The GCP project ID. + :param use_cross_language: + If set to True, will use the Java native transform via cross-language. + :param expansion_service: + The address of the expansion service in the case of using cross-language. + If no expansion service is provided, will attempt to run the default GCP + expansion service. """ super().__init__() - self.beam_options = { - 'project_id': project_id, - 'instance_id': instance_id, - 'table_id': table_id - } + self._table_id = table_id + self._instance_id = instance_id + self._project_id = project_id + self._use_cross_language = use_cross_language + if use_cross_language: + self._expansion_service = ( + expansion_service or BeamJarExpansionService( + 'sdks:java:io:google-cloud-platform:expansion-service:build')) + self.schematransform_config = ( + SchemaAwareExternalTransform.discover_config( + self._expansion_service, self.URN)) - def expand(self, pvalue): - beam_options = self.beam_options - return ( - pvalue - | beam.ParDo( - _BigTableWriteFn( - beam_options['project_id'], - beam_options['instance_id'], - beam_options['table_id']))) + def expand(self, input): + if self._use_cross_language: + external_write = SchemaAwareExternalTransform( + identifier=self.schematransform_config.identifier, + expansion_service=self._expansion_service, + rearrange_based_on_discovery=True, + tableId=self._table_id, + instanceId=self._instance_id, + projectId=self._project_id) + + return ( + input + | beam.ParDo(self._DirectRowMutationsToBeamRow()).with_output_types( + RowTypeConstraint.from_fields( + [("key", bytes), ("mutations", List[Dict[str, bytes]])])) + | external_write) + else: + return ( + input + | beam.ParDo( + _BigTableWriteFn( + self._project_id, self._instance_id, self._table_id))) + + class _DirectRowMutationsToBeamRow(beam.DoFn): + def process(self, direct_row): + args = {"key": direct_row.row_key, "mutations": []} + # start accumulating mutations in a list + for mutation in direct_row._get_mutations(): + if mutation.__contains__("set_cell"): + mutation_dict = { + "type": b'SetCell', + "family_name": mutation.set_cell.family_name.encode('utf-8'), + "column_qualifier": mutation.set_cell.column_qualifier, + "value": mutation.set_cell.value, + "timestamp_micros": struct.pack( + '>q', mutation.set_cell.timestamp_micros) + } + elif mutation.__contains__("delete_from_column"): + mutation_dict = { + "type": b'DeleteFromColumn', + "family_name": mutation.delete_from_column.family_name.encode( + 'utf-8'), + "column_qualifier": mutation.delete_from_column.column_qualifier + } + time_range = mutation.delete_from_column.time_range + if time_range.start_timestamp_micros: + mutation_dict['start_timestamp_micros'] = struct.pack( + '>q', time_range.start_timestamp_micros) + if time_range.end_timestamp_micros: + mutation_dict['end_timestamp_micros'] = struct.pack( + '>q', time_range.end_timestamp_micros) + elif mutation.__contains__("delete_from_family"): + mutation_dict = { + "type": b'DeleteFromFamily', + "family_name": mutation.delete_from_family.family_name.encode( + 'utf-8') + } + elif mutation.__contains__("delete_from_row"): + mutation_dict = {"type": b'DeleteFromRow'} + else: + raise ValueError("Unexpected mutation") + + args["mutations"].append(mutation_dict) + + yield beam.Row(**args) class ReadFromBigtable(PTransform): @@ -207,7 +295,7 @@ class ReadFromBigtable(PTransform): """ URN = "beam:schematransform:org.apache.beam:bigtable_read:v1" - def __init__(self, table_id, instance_id, project_id, expansion_service=None): + def __init__(self, project_id, instance_id, table_id, expansion_service=None): """Initialize a ReadFromBigtable transform. :param table_id: diff --git a/sdks/python/apache_beam/io/gcp/bigtableio_it_test.py b/sdks/python/apache_beam/io/gcp/bigtableio_it_test.py new file mode 100644 index 0000000000000..f61e346cff9f8 --- /dev/null +++ b/sdks/python/apache_beam/io/gcp/bigtableio_it_test.py @@ -0,0 +1,397 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Integration tests for BigTable service.""" + +import logging +import os +import secrets +import time +import unittest +# pytype: skip-file +from datetime import datetime +from datetime import timezone + +import pytest + +import apache_beam as beam +from apache_beam.io.gcp import bigtableio +from apache_beam.testing.test_pipeline import TestPipeline +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to + +_LOGGER = logging.getLogger(__name__) + +# Protect against environments where bigtable library is not available. +try: + from apitools.base.py.exceptions import HttpError + from google.cloud.bigtable import client + from google.cloud.bigtable.row_filters import TimestampRange + from google.cloud.bigtable.row import DirectRow, PartialRowData, Cell + from google.cloud.bigtable.table import Table + from google.cloud.bigtable_admin_v2.types import instance +except ImportError as e: + client = None + HttpError = None + + +@pytest.mark.uses_gcp_java_expansion_service +@pytest.mark.uses_transform_service +@unittest.skipUnless( + os.environ.get('EXPANSION_PORT'), + "EXPANSION_PORT environment var is not provided.") +@unittest.skipIf(client is None, 'Bigtable dependencies are not installed') +class TestReadFromBigTableIT(unittest.TestCase): + INSTANCE = "bt-read-tests" + TABLE_ID = "test-table" + + def setUp(self): + self.test_pipeline = TestPipeline(is_integration_test=True) + self.args = self.test_pipeline.get_full_options_as_args() + self.project = self.test_pipeline.get_option('project') + self.expansion_service = ('localhost:%s' % os.environ.get('EXPANSION_PORT')) + + instance_id = '%s-%s-%s' % ( + self.INSTANCE, str(int(time.time())), secrets.token_hex(3)) + + self.client = client.Client(admin=True, project=self.project) + # create cluster and instance + self.instance = self.client.instance( + instance_id, + display_name=self.INSTANCE, + instance_type=instance.Instance.Type.DEVELOPMENT) + cluster = self.instance.cluster("test-cluster", "us-central1-a") + operation = self.instance.create(clusters=[cluster]) + operation.result(timeout=500) + _LOGGER.info( + "Created instance [%s] in project [%s]", + self.instance.instance_id, + self.project) + + # create table inside instance + self.table = self.instance.table(self.TABLE_ID) + self.table.create() + _LOGGER.info("Created table [%s]", self.table.table_id) + + def tearDown(self): + try: + _LOGGER.info( + "Deleting table [%s] and instance [%s]", + self.table.table_id, + self.instance.instance_id) + self.table.delete() + self.instance.delete() + except HttpError: + _LOGGER.debug( + "Failed to clean up table [%s] and instance [%s]", + self.table.table_id, + self.instance.instance_id) + + def add_rows(self, num_rows, num_families, num_columns_per_family): + cells = [] + for i in range(1, num_rows + 1): + key = 'key-' + str(i) + row = DirectRow(key, self.table) + for j in range(num_families): + fam_name = 'test_col_fam_' + str(j) + # create the table's column families one time only + if i == 1: + col_fam = self.table.column_family(fam_name) + col_fam.create() + for k in range(1, num_columns_per_family + 1): + row.set_cell(fam_name, f'col-{j}-{k}', f'value-{i}-{j}-{k}') + + # after all mutations on the row are done, commit to Bigtable + row.commit() + # read the same row back from Bigtable to get the expected data + # accumulate rows in `cells` + read_row: PartialRowData = self.table.read_row(key) + cells.append(read_row.cells) + + return cells + + def test_read_xlang(self): + # create rows and retrieve expected cells + expected_cells = self.add_rows( + num_rows=5, num_families=3, num_columns_per_family=4) + + with beam.Pipeline(argv=self.args) as p: + cells = ( + p + | bigtableio.ReadFromBigtable( + project_id=self.project, + instance_id=self.instance.instance_id, + table_id=self.table.table_id, + expansion_service=self.expansion_service) + | "Extract cells" >> beam.Map(lambda row: row._cells)) + + assert_that(cells, equal_to(expected_cells)) + + +@pytest.mark.uses_gcp_java_expansion_service +@pytest.mark.uses_transform_service +@unittest.skipUnless( + os.environ.get('EXPANSION_PORT'), + "EXPANSION_PORT environment var is not provided.") +@unittest.skipIf(client is None, 'Bigtable dependencies are not installed') +class TestWriteToBigtableXlangIT(unittest.TestCase): + # These are integration tests for the cross-language write transform. + INSTANCE = "bt-write-xlang" + TABLE_ID = "test-table" + + @classmethod + def setUpClass(cls): + cls.test_pipeline = TestPipeline(is_integration_test=True) + cls.project = cls.test_pipeline.get_option('project') + cls.args = cls.test_pipeline.get_full_options_as_args() + cls.expansion_service = ('localhost:%s' % os.environ.get('EXPANSION_PORT')) + + instance_id = '%s-%s-%s' % ( + cls.INSTANCE, str(int(time.time())), secrets.token_hex(3)) + + cls.client = client.Client(admin=True, project=cls.project) + # create cluster and instance + cls.instance = cls.client.instance( + instance_id, + display_name=cls.INSTANCE, + instance_type=instance.Instance.Type.DEVELOPMENT) + cluster = cls.instance.cluster("test-cluster", "us-central1-a") + operation = cls.instance.create(clusters=[cluster]) + operation.result(timeout=500) + _LOGGER.warning( + "Created instance [%s] in project [%s]", + cls.instance.instance_id, + cls.project) + + def setUp(self): + # create table inside instance + self.table: Table = self.instance.table( + '%s-%s-%s' % + (self.TABLE_ID, str(int(time.time())), secrets.token_hex(3))) + self.table.create() + _LOGGER.info("Created table [%s]", self.table.table_id) + + def tearDown(self): + try: + _LOGGER.info("Deleting table [%s]", self.table.table_id) + self.table.delete() + except HttpError: + _LOGGER.debug("Failed to clean up table [%s]", self.table.table_id) + + @classmethod + def tearDownClass(cls): + try: + _LOGGER.info("Deleting instance [%s]", cls.instance.instance_id) + cls.instance.delete() + except HttpError: + _LOGGER.debug( + "Failed to clean up instance [%s]", cls.instance.instance_id) + + def run_pipeline(self, rows): + with beam.Pipeline(argv=self.args) as p: + _ = ( + p + | beam.Create(rows) + | bigtableio.WriteToBigTable( + project_id=self.project, + instance_id=self.instance.instance_id, + table_id=self.table.table_id, + use_cross_language=True, + expansion_service=self.expansion_service)) + + def test_set_mutation(self): + row1: DirectRow = DirectRow('key-1') + row2: DirectRow = DirectRow('key-2') + col_fam = self.table.column_family('col_fam') + col_fam.create() + # expected cells + row1_col1_cell = Cell(b'val1-1', 100_000_000) + row1_col2_cell = Cell(b'val1-2', 200_000_000) + row2_col1_cell = Cell(b'val2-1', 100_000_000) + row2_col2_cell = Cell(b'val2-2', 200_000_000) + # When setting this cell, we won't set a timestamp. We expect the timestamp + # to default to -1, and Bigtable will set it to system time at insertion. + row2_col1_no_timestamp = Cell(b'val2-2-notimestamp', time.time()) + # rows sent to write transform + row1.set_cell( + 'col_fam', b'col-1', row1_col1_cell.value, row1_col1_cell.timestamp) + row1.set_cell( + 'col_fam', b'col-2', row1_col2_cell.value, row1_col2_cell.timestamp) + row2.set_cell( + 'col_fam', b'col-1', row2_col1_cell.value, row2_col1_cell.timestamp) + row2.set_cell( + 'col_fam', b'col-2', row2_col2_cell.value, row2_col2_cell.timestamp) + # don't set a timestamp here. it should default to -1 + row2.set_cell('col_fam', b'col-no-timestamp', row2_col1_no_timestamp.value) + + self.run_pipeline([row1, row2]) + + # after write transform executes, get actual rows from table + actual_row1: PartialRowData = self.table.read_row('key-1') + actual_row2: PartialRowData = self.table.read_row('key-2') + + # check actual rows match with expected rows (value and timestamp) + self.assertEqual( + row1_col1_cell, actual_row1.find_cells('col_fam', b'col-1')[0]) + self.assertEqual( + row1_col2_cell, actual_row1.find_cells('col_fam', b'col-2')[0]) + self.assertEqual( + row2_col1_cell, actual_row2.find_cells('col_fam', b'col-1')[0]) + self.assertEqual( + row2_col2_cell, actual_row2.find_cells('col_fam', b'col-2')[0]) + + # check mutation that doesn't have a timestamp set is handled properly: + self.assertEqual( + row2_col1_no_timestamp.value, + actual_row2.find_cells('col_fam', b'col-no-timestamp')[0].value) + # Bigtable sets timestamp as insertion time, which is later than the + # time.time() we set when creating this test case + cell_timestamp = actual_row2.find_cells('col_fam', + b'col-no-timestamp')[0].timestamp + self.assertTrue( + row2_col1_no_timestamp.timestamp < cell_timestamp, + msg="Expected cell with unset timestamp to have ingestion time " + f"attached, but was {cell_timestamp}") + + def test_delete_cells_mutation(self): + col_fam = self.table.column_family('col_fam') + col_fam.create() + # write a row with two columns to the table beforehand. + write_row: DirectRow = DirectRow('key-1', self.table) + write_row.set_cell('col_fam', b'col-1', b'val-1') + write_row.set_cell('col_fam', b'col-2', b'val-2') + write_row.commit() + + # prepare a row that will delete cells in one of the columns. + delete_row: DirectRow = DirectRow('key-1') + delete_row.delete_cell('col_fam', b'col-1') + + self.run_pipeline([delete_row]) + + # after transform executes, get actual row from table + actual_row: PartialRowData = self.table.read_row('key-1') + + # we deleted all the cells in 'col-1', so this check should throw an error + with self.assertRaises(KeyError): + actual_row.find_cells('col_fam', b'col-1') + + # check the cell in col-2 is still there + col2_cells = actual_row.find_cells('col_fam', b'col-2') + self.assertEqual(1, len(col2_cells)) + self.assertEqual(b'val-2', col2_cells[0].value) + + def test_delete_cells_with_timerange_mutation(self): + col_fam = self.table.column_family('col_fam') + col_fam.create() + # write two cells in a column to the table beforehand. + write_row: DirectRow = DirectRow('key-1', self.table) + write_row.set_cell( + 'col_fam', + b'col', + b'val', + datetime.fromtimestamp(100_000_000, tz=timezone.utc)) + write_row.commit() + write_row.set_cell( + 'col_fam', + b'col', + b'new-val', + datetime.fromtimestamp(200_000_000, tz=timezone.utc)) + write_row.commit() + + # prepare a row that will delete cells within a timestamp range. + delete_row: DirectRow = DirectRow('key-1') + delete_row.delete_cell( + 'col_fam', + b'col', + time_range=TimestampRange( + start=datetime.fromtimestamp(99_999_999, tz=timezone.utc), + end=datetime.fromtimestamp(100_000_001, tz=timezone.utc))) + + self.run_pipeline([delete_row]) + + # after transform executes, get actual row from table + actual_row: PartialRowData = self.table.read_row('key-1') + + # we deleted one cell within the timestamp range. + # check the other (newer) cell still exists. + cells = actual_row.find_cells('col_fam', b'col') + self.assertEqual(1, len(cells)) + self.assertEqual(b'new-val', cells[0].value) + self.assertEqual( + datetime.fromtimestamp(200_000_000, tz=timezone.utc), + cells[0].timestamp) + + def test_delete_column_family_mutation(self): + # create two column families + col_fam = self.table.column_family('col_fam-1') + col_fam.create() + col_fam = self.table.column_family('col_fam-2') + col_fam.create() + # write a row with values in both column families to the table beforehand. + write_row: DirectRow = DirectRow('key-1', self.table) + write_row.set_cell('col_fam-1', b'col', b'val') + write_row.set_cell('col_fam-2', b'col', b'val') + write_row.commit() + + # prepare a row that will delete a column family from the row + delete_row: DirectRow = DirectRow('key-1') + delete_row.delete_cells('col_fam-1', delete_row.ALL_COLUMNS) + + self.run_pipeline([delete_row]) + + # after transform executes, get actual row from table + actual_row: PartialRowData = self.table.read_row('key-1') + + # we deleted column family 'col_fam-1', so this check should throw an error + with self.assertRaises(KeyError): + actual_row.find_cells('col_fam-1', b'col') + + # check we have one column family left with the correct cell value + self.assertEqual(1, len(actual_row.cells)) + self.assertEqual(b'val', actual_row.cell_value('col_fam-2', b'col')) + + def test_delete_row_mutation(self): + write_row1: DirectRow = DirectRow('key-1', self.table) + write_row2: DirectRow = DirectRow('key-2', self.table) + col_fam = self.table.column_family('col_fam') + col_fam.create() + # write a couple of rows to the table beforehand + write_row1.set_cell('col_fam', b'col', b'val-1') + write_row1.commit() + write_row2.set_cell('col_fam', b'col', b'val-2') + write_row2.commit() + + # prepare a row that will delete itself + delete_row: DirectRow = DirectRow('key-1') + delete_row.delete() + + self.run_pipeline([delete_row]) + + # after write transform executes, get actual rows from table + actual_row1: PartialRowData = self.table.read_row('key-1') + actual_row2: PartialRowData = self.table.read_row('key-2') + + # we deleted row with key 'key-1', check it doesn't exist anymore + # the Bigtable API doesn't throw an error here, just returns a None value + self.assertEqual(None, actual_row1) + # check row 2 exists with the correct cell value in col + self.assertEqual(b'val-2', actual_row2.cell_value('col_fam', b'col')) + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + unittest.main() diff --git a/sdks/python/apache_beam/io/gcp/bigtableio_test.py b/sdks/python/apache_beam/io/gcp/bigtableio_test.py index 012e0478adad0..f97c9bcfbd6a0 100644 --- a/sdks/python/apache_beam/io/gcp/bigtableio_test.py +++ b/sdks/python/apache_beam/io/gcp/bigtableio_test.py @@ -17,18 +17,15 @@ """Unit tests for BigTable service.""" -# pytype: skip-file import logging -import os -import secrets import string -import time import unittest import uuid +# pytype: skip-file from datetime import datetime +from datetime import timezone from random import choice -import pytest from mock import MagicMock from mock import patch @@ -38,113 +35,20 @@ from apache_beam.io.gcp import resource_identifiers from apache_beam.metrics import monitoring_infos from apache_beam.metrics.execution import MetricsEnvironment -from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to _LOGGER = logging.getLogger(__name__) # Protect against environments where bigtable library is not available. try: - from apitools.base.py.exceptions import HttpError from google.cloud.bigtable import client + from google.cloud.bigtable.row_filters import TimestampRange from google.cloud.bigtable.instance import Instance from google.cloud.bigtable.row import DirectRow, PartialRowData, Cell from google.cloud.bigtable.table import Table - from google.cloud.bigtable_admin_v2.types import instance from google.rpc.code_pb2 import OK, ALREADY_EXISTS from google.rpc.status_pb2 import Status except ImportError as e: client = None - HttpError = None - - -@pytest.mark.uses_gcp_java_expansion_service -@unittest.skipUnless( - os.environ.get('EXPANSION_PORT'), - "EXPANSION_PORT environment var is not provided.") -@unittest.skipIf(client is None, 'Bigtable dependencies are not installed') -class TestReadFromBigTable(unittest.TestCase): - INSTANCE = "bt-read-tests" - TABLE_ID = "test-table" - - def setUp(self): - self.test_pipeline = TestPipeline(is_integration_test=True) - self.args = self.test_pipeline.get_full_options_as_args() - self.project = self.test_pipeline.get_option('project') - - instance_id = '%s-%s-%s' % ( - self.INSTANCE, str(int(time.time())), secrets.token_hex(3)) - - self.client = client.Client(admin=True, project=self.project) - # create cluster and instance - self.instance = self.client.instance( - instance_id, - display_name=self.INSTANCE, - instance_type=instance.Instance.Type.DEVELOPMENT) - cluster = self.instance.cluster("test-cluster", "us-central1-a") - operation = self.instance.create(clusters=[cluster]) - operation.result(timeout=500) - _LOGGER.info( - "Created instance [%s] in project [%s]", - self.instance.instance_id, - self.project) - - # create table inside instance - self.table = self.instance.table(self.TABLE_ID) - self.table.create() - _LOGGER.info("Created table [%s]", self.table.table_id) - - def tearDown(self): - try: - _LOGGER.info( - "Deleting table [%s] and instance [%s]", - self.table.table_id, - self.instance.instance_id) - self.table.delete() - self.instance.delete() - except HttpError: - _LOGGER.debug( - "Failed to clean up table [%s] and instance [%s]", - self.table.table_id, - self.instance.instance_id) - - def add_rows(self, num_rows, num_families, num_columns_per_family): - cells = [] - for i in range(1, num_rows + 1): - key = 'key-' + str(i) - row = DirectRow(key, self.table) - for j in range(num_families): - fam_name = 'test_col_fam_' + str(j) - # create the table's column families one time only - if i == 1: - col_fam = self.table.column_family(fam_name) - col_fam.create() - for k in range(1, num_columns_per_family + 1): - row.set_cell(fam_name, f'col-{j}-{k}', f'value-{i}-{j}-{k}') - - # after all mutations on the row are done, commit to Bigtable - row.commit() - # read the same row back from Bigtable to get the expected data - # accumulate rows in `cells` - read_row: PartialRowData = self.table.read_row(key) - cells.append(read_row.cells) - - return cells - - def test_read_xlang(self): - # create rows and retrieve expected cells - expected_cells = self.add_rows( - num_rows=5, num_families=3, num_columns_per_family=4) - - with beam.Pipeline(argv=self.args) as p: - cells = ( - p - | bigtableio.ReadFromBigtable( - self.table.table_id, self.instance.instance_id, self.project) - | "Extract cells" >> beam.Map(lambda row: row._cells)) - - assert_that(cells, equal_to(expected_cells)) @unittest.skipIf(client is None, 'Bigtable dependencies are not installed') @@ -205,6 +109,151 @@ def test_beam_row_to_bigtable_row(self): bigtable_row.find_cells('family_2', b'column_qualifier')) +@unittest.skipIf(client is None, 'Bigtable dependencies are not installed') +class TestBigtableDirectRowToBeamRow(unittest.TestCase): + doFn = bigtableio.WriteToBigTable._DirectRowMutationsToBeamRow() + + def test_set_cell(self): + # create some set cell mutations + direct_row: DirectRow = DirectRow('key-1') + direct_row.set_cell( + 'col_fam', + b'col', + b'a', + datetime.fromtimestamp(100_000).replace(tzinfo=timezone.utc)) + direct_row.set_cell( + 'col_fam', + b'other-col', + b'b', + datetime.fromtimestamp(200_000).replace(tzinfo=timezone.utc)) + direct_row.set_cell( + 'other_col_fam', + b'col', + b'c', + datetime.fromtimestamp(300_000).replace(tzinfo=timezone.utc)) + + # get equivalent beam row + beam_row = next(self.doFn.process(direct_row)) + + # sort both lists of mutations for convenience + beam_row_mutations = sorted(beam_row.mutations, key=lambda m: m['value']) + bt_row_mutations = sorted( + direct_row._get_mutations(), key=lambda m: m.set_cell.value) + self.assertEqual(beam_row.key, direct_row.row_key) + self.assertEqual(len(beam_row_mutations), len(bt_row_mutations)) + + # check that the values in each beam mutation is equal to the original + # Bigtable direct row mutations + for i in range(len(beam_row_mutations)): + beam_mutation = beam_row_mutations[i] + bt_mutation = bt_row_mutations[i].set_cell + + self.assertEqual(beam_mutation['type'], b'SetCell') + self.assertEqual( + beam_mutation['family_name'].decode(), bt_mutation.family_name) + self.assertEqual( + beam_mutation['column_qualifier'], bt_mutation.column_qualifier) + self.assertEqual(beam_mutation['value'], bt_mutation.value) + self.assertEqual( + int.from_bytes(beam_mutation['timestamp_micros'], 'big'), + bt_mutation.timestamp_micros) + + def test_delete_cells(self): + # create some delete cell mutations. one with a timestamp range + direct_row: DirectRow = DirectRow('key-1') + direct_row.delete_cell('col_fam', b'col-1') + direct_row.delete_cell( + 'other_col_fam', + b'col-2', + time_range=TimestampRange( + start=datetime.fromtimestamp(10_000_000, tz=timezone.utc))) + direct_row.delete_cells( + 'another_col_fam', [b'col-3', b'col-4', b'col-5'], + time_range=TimestampRange( + start=datetime.fromtimestamp(50_000_000, tz=timezone.utc), + end=datetime.fromtimestamp(100_000_000, tz=timezone.utc))) + + # get equivalent beam row + beam_row = next(self.doFn.process(direct_row)) + + # sort both lists of mutations for convenience + beam_row_mutations = sorted( + beam_row.mutations, key=lambda m: m['column_qualifier']) + bt_row_mutations = sorted( + direct_row._get_mutations(), + key=lambda m: m.delete_from_column.column_qualifier) + self.assertEqual(beam_row.key, direct_row.row_key) + self.assertEqual(len(beam_row_mutations), len(bt_row_mutations)) + + # check that the values in each beam mutation is equal to the original + # Bigtable direct row mutations + for i in range(len(beam_row_mutations)): + beam_mutation = beam_row_mutations[i] + bt_mutation = bt_row_mutations[i].delete_from_column + print(bt_mutation) + + self.assertEqual(beam_mutation['type'], b'DeleteFromColumn') + self.assertEqual( + beam_mutation['family_name'].decode(), bt_mutation.family_name) + self.assertEqual( + beam_mutation['column_qualifier'], bt_mutation.column_qualifier) + + # check we set a timestamp range only when appropriate + if bt_mutation.time_range.start_timestamp_micros: + self.assertEqual( + int.from_bytes(beam_mutation['start_timestamp_micros'], 'big'), + bt_mutation.time_range.start_timestamp_micros) + else: + self.assertTrue('start_timestamp_micros' not in beam_mutation) + + if bt_mutation.time_range.end_timestamp_micros: + self.assertEqual( + int.from_bytes(beam_mutation['end_timestamp_micros'], 'big'), + bt_mutation.time_range.end_timestamp_micros) + else: + self.assertTrue('end_timestamp_micros' not in beam_mutation) + + def test_delete_column_family(self): + # create mutation to delete column family + direct_row: DirectRow = DirectRow('key-1') + direct_row.delete_cells('col_fam-1', direct_row.ALL_COLUMNS) + direct_row.delete_cells('col_fam-2', direct_row.ALL_COLUMNS) + + # get equivalent beam row + beam_row = next(self.doFn.process(direct_row)) + + # sort both lists of mutations for convenience + beam_row_mutations = sorted( + beam_row.mutations, key=lambda m: m['family_name']) + bt_row_mutations = sorted( + direct_row._get_mutations(), + key=lambda m: m.delete_from_column.family_name) + self.assertEqual(beam_row.key, direct_row.row_key) + self.assertEqual(len(beam_row_mutations), len(bt_row_mutations)) + + # check that the values in each beam mutation is equal to the original + # Bigtable direct row mutations + for i in range(len(beam_row_mutations)): + beam_mutation = beam_row_mutations[i] + bt_mutation = bt_row_mutations[i].delete_from_family + + self.assertEqual(beam_mutation['type'], b'DeleteFromFamily') + self.assertEqual( + beam_mutation['family_name'].decode(), bt_mutation.family_name) + + def test_delete_row(self): + # create mutation to delete the Bigtable row + direct_row: DirectRow = DirectRow('key-1') + direct_row.delete() + + # get equivalent beam row + beam_row = next(self.doFn.process(direct_row)) + self.assertEqual(beam_row.key, direct_row.row_key) + + beam_mutation = beam_row.mutations[0] + self.assertEqual(beam_mutation['type'], b'DeleteFromRow') + + @unittest.skipIf(client is None, 'Bigtable dependencies are not installed') class TestWriteBigTable(unittest.TestCase): TABLE_PREFIX = "python-test" diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio_test.py index 076a95178d83d..aac99cb8c1f0a 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio_test.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio_test.py @@ -438,7 +438,7 @@ def test_DatastoreWriteLargeEntities(self): datastore_write_fn = WriteToDatastore._DatastoreWriteFn(self._PROJECT) datastore_write_fn.start_bundle() for entity in entities: - entity.set_properties({'large': u'A' * 100000}) + entity.set_properties({'large': 'A' * 100000}) datastore_write_fn.process(entity) datastore_write_fn.finish_bundle() diff --git a/sdks/python/apache_beam/io/gcp/gcsfilesystem_integration_test.py b/sdks/python/apache_beam/io/gcp/gcsfilesystem_integration_test.py new file mode 100644 index 0000000000000..c322e4c9e1ccc --- /dev/null +++ b/sdks/python/apache_beam/io/gcp/gcsfilesystem_integration_test.py @@ -0,0 +1,139 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Integration tests for gcsfilesystem module. + +Runs tests against Google Cloud Storage service. +Instantiates a TestPipeline to get options such as GCP project name, but +doesn't actually start a Beam pipeline or test any specific runner. + +To run these tests manually: + ./gradlew :sdks:python:test-suites:dataflow:integrationTest \ + -Dtests=apache_beam.io.gcp.gcsfilesystem_integration_test:GcsFileSystemIntegrationTest # pylint: disable=line-too-long +""" + +# pytype: skip-file + +import logging +import unittest +import uuid + +import pytest + +from apache_beam.io.filesystem import BeamIOError +from apache_beam.io.filesystems import FileSystems +from apache_beam.testing.test_pipeline import TestPipeline + +try: + from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem + fs_not_available = False +except ImportError: + fs_not_available = True # type: ignore + + +@unittest.skipIf(fs_not_available, 'GCP dependencies are not installed') +class GcsFileSystemIntegrationTest(unittest.TestCase): + + INPUT_FILE = 'gs://dataflow-samples/shakespeare/kinglear.txt' + # Larger than 1MB to test maxBytesRewrittenPerCall. + # Also needs to be in a different region than the dest to take effect. + INPUT_FILE_LARGE = 'gs://apache-beam-samples-us-east1/wikipedia_edits/wiki_data-000000000000.json' # pylint: disable=line-too-long + + def setUp(self): + self.test_pipeline = TestPipeline(is_integration_test=True) + self.runner_name = type(self.test_pipeline.runner).__name__ + if self.runner_name != 'TestDataflowRunner': + # This test doesn't run a pipeline, so it doesn't make sense to try it on + # different runners. Running with TestDataflowRunner makes sense since + # it uses GoogleCloudOptions such as 'project'. + raise unittest.SkipTest('This test only runs with TestDataflowRunner.') + self.project = self.test_pipeline.get_option('project') + self.gcs_tempdir = ( + self.test_pipeline.get_option('temp_location') + '/gcs_it-' + + str(uuid.uuid4())) + self.fs = GCSFileSystem(self.test_pipeline.get_pipeline_options()) + + def tearDown(self): + FileSystems.delete([self.gcs_tempdir + '/']) + + def _verify_copy(self, src, dest): + self.assertTrue(FileSystems.exists(src), 'src does not exist: %s' % src) + self.assertTrue(FileSystems.exists(dest), 'dest does not exist: %s' % dest) + src_checksum = self.fs.checksum(src) + dest_checksum = self.fs.checksum(dest) + self.assertEqual(src_checksum, dest_checksum) + + def _verify_rename(self, src, dest): + self.assertFalse(FileSystems.exists(src), 'file %s not renamed' % src) + self.assertTrue(FileSystems.exists(dest), 'file not renamed to %s' % dest) + + def _test_copy(self, name, max_bytes_rewritten_per_call=None, src=None): + src = src or self.INPUT_FILE + dest = self.gcs_tempdir + '/%s' % name + extra_kwargs = {} + if max_bytes_rewritten_per_call is not None: + extra_kwargs['max_bytes_rewritten_per_call'] = ( + max_bytes_rewritten_per_call) + + self.fs.copy([src], [dest]) + self._verify_copy(src, dest) + + @pytest.mark.it_postcommit + def test_copy(self): + self._test_copy("test_copy") + + @pytest.mark.it_postcommit + def test_rename(self): + num_copies = 10 + srcs = [self.INPUT_FILE] * num_copies + dests = [ + self.gcs_tempdir + '/%s_%d' % (self.INPUT_FILE, i) + for i in range(num_copies) + ] + self.fs.copy(srcs, dests) + new_names = [ + self.gcs_tempdir + '/%s_%d' % ("renamed", i) for i in range(num_copies) + ] + self.fs.rename(dests, new_names) + for _old, _new in list(zip(dests, new_names)): + self._verify_rename(_old, _new) + + @pytest.mark.it_postcommit + def test_rename_error(self): + num_copies = 10 + srcs = [self.INPUT_FILE] * num_copies + dests = [ + self.gcs_tempdir + '/%s_%d' % (self.INPUT_FILE, i) + for i in range(num_copies) + ] + self.fs.copy(srcs, dests) + new_names = [ + self.gcs_tempdir + '/%s_%d' % ("renamed", i) for i in range(num_copies) + ] + # corrupt one of the destination names + bad_name = self.gcs_tempdir + '/errorbadwrong' + dests[int(num_copies / 2)] = bad_name + with self.assertRaises(BeamIOError): + self.fs.rename(dests, new_names) + for _old, _new in list(zip(dests, new_names)): + if _old != bad_name: + self._verify_rename(_old, _new) + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + unittest.main() diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py index 2010d22925937..d75af4fe6ac18 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio.py +++ b/sdks/python/apache_beam/io/gcp/gcsio.py @@ -174,7 +174,8 @@ def __init__(self, storage_client=None, pipeline_options=None): http=get_new_http(), response_encoding='utf8', additional_http_headers={ - "User-Agent": "apache-beam-%s" % apache_beam.__version__ + "User-Agent": "apache-beam/%s (GPN:Beam)" % + apache_beam.__version__ }) self.client = storage_client self._rewrite_cb = None @@ -823,7 +824,5 @@ def finish(self): self._upload_thread.join() # Check for exception since the last put() call. if self._upload_thread.last_error is not None: - raise type(self._upload_thread.last_error)( - "Error while uploading file %s: %s", - self._path, - self._upload_thread.last_error.message) # pylint: disable=raising-bad-type + e = self._upload_thread.last_error + raise RuntimeError("Error while uploading file %s" % self._path) from e diff --git a/sdks/python/apache_beam/io/gcp/gcsio_test.py b/sdks/python/apache_beam/io/gcp/gcsio_test.py index 260090461c8c6..9cc5a9e1df0ce 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio_test.py +++ b/sdks/python/apache_beam/io/gcp/gcsio_test.py @@ -33,6 +33,7 @@ # Protect against environments where apitools library is not available. # pylint: disable=wrong-import-order, wrong-import-position +import apache_beam from apache_beam.metrics import monitoring_infos from apache_beam.metrics.execution import MetricsEnvironment from apache_beam.metrics.metricbase import MetricName @@ -473,7 +474,9 @@ def test_user_agent_passed(self, get_new_http_mock, get_service_creds_mock): # soon after the GCS API is called. pass call = get_new_http_mock.return_value.request.mock_calls[-2] - self.assertIn('apache-beam-', call[2]['headers']['User-Agent']) + self.assertIn( + "apache-beam/%s (GPN:Beam)" % apache_beam.__version__, + call[2]['headers']['User-Agent']) @mock.patch('apache_beam.io.gcp.gcsio.BatchApiRequest') def test_delete_batch(self, *unused_args): diff --git a/sdks/python/apache_beam/io/gcp/healthcare/dicomclient.py b/sdks/python/apache_beam/io/gcp/healthcare/dicomclient.py index cbe951a0c7449..6efb26b7c8910 100644 --- a/sdks/python/apache_beam/io/gcp/healthcare/dicomclient.py +++ b/sdks/python/apache_beam/io/gcp/healthcare/dicomclient.py @@ -22,7 +22,10 @@ class DicomApiHttpClient: - """DICOM api client that talk to api via http request""" + """Creates a client that communicates with the GCP Healthcare API + (https://cloud.google.com/healthcare-api), over HTTP. + Note that this client strictly uses v1 of this API.""" + healthcare_base_url = "https://healthcare.googleapis.com/v1" session = None @@ -46,9 +49,9 @@ def qido_search( search_type, params=None, credential=None): - """function for searching a DICOM store""" + """Function for searching a DICOM store""" - # sending request to the REST healthcare api. + # Sends a request to the GCP HCLS API v1. api_endpoint = "{}/projects/{}/locations/{}".format( self.healthcare_base_url, project_id, region) @@ -105,7 +108,7 @@ def dicomweb_store_instance( dicom_store_id, dcm_file, credential=None): - """function for storing an instance.""" + """Function for storing a DICOM instance.""" api_endpoint = "{}/projects/{}/locations/{}".format( self.healthcare_base_url, project_id, region) diff --git a/sdks/python/apache_beam/io/gcp/healthcare/dicomio_test.py b/sdks/python/apache_beam/io/gcp/healthcare/dicomio_test.py index 342c079dd62a1..812f258576851 100644 --- a/sdks/python/apache_beam/io/gcp/healthcare/dicomio_test.py +++ b/sdks/python/apache_beam/io/gcp/healthcare/dicomio_test.py @@ -45,10 +45,10 @@ # pylint: enable=wrong-import-order, wrong-import-position -class FakeHttpClient(): - # a fake http client that talks directly to a in-memory dicom store +class MockHttpClient(): + # A mock HTTP client that talks directly to an in-memory Dicom store. def __init__(self): - # set 5 fake dicom instances + # set 5 mock dicom instances dicom_metadata = [] dicom_metadata.append({ 'PatientName': 'Albert', 'Age': 21, 'TestResult': 'Positive' @@ -69,7 +69,7 @@ def __init__(self): 'PatientName': 'Eric', 'Age': 50, 'TestResult': 'Negative' }) self.dicom_metadata = dicom_metadata - # ids for this dicom sotre + # ids for this dicom store self.project_id = 'test_project' self.region = 'test_region' self.dataset_id = 'test_dataset_id' @@ -84,7 +84,7 @@ def qido_search( search_type, params=None, credential=None): - # qido search function for this fake client + # qido search function for this mock client if project_id != self.project_id or region != self.region or \ dataset_id != self.dataset_id or dicom_store_id != self.dicom_store_id: return [], 204 @@ -166,7 +166,7 @@ def test_failed_convert(self): @unittest.skipIf(DicomSearch is None, 'GCP dependencies are not installed') class TestDicomSearch(unittest.TestCase): @patch("apache_beam.io.gcp.healthcare.dicomio.DicomApiHttpClient") - def test_successful_search(self, FakeClient): + def test_successful_search(self, MockClient): input_dict = {} input_dict['project_id'] = "test_project" input_dict['region'] = "test_region" @@ -174,11 +174,11 @@ def test_successful_search(self, FakeClient): input_dict['dicom_store_id'] = "test_dicom_store_id" input_dict['search_type'] = "instances" - fc = FakeHttpClient() - FakeClient.return_value = fc + mc = MockHttpClient() + MockClient.return_value = mc expected_dict = {} - expected_dict['result'] = fc.dicom_metadata + expected_dict['result'] = mc.dicom_metadata expected_dict['status'] = 200 expected_dict['input'] = input_dict expected_dict['success'] = True @@ -188,7 +188,7 @@ def test_successful_search(self, FakeClient): assert_that(results, equal_to([expected_dict])) @patch("apache_beam.io.gcp.healthcare.dicomio.DicomApiHttpClient") - def test_Qido_search_small_buffer_flush(self, FakeClient): + def test_Qido_search_small_buffer_flush(self, MockClient): input_dict = {} input_dict['project_id'] = "test_project" input_dict['region'] = "test_region" @@ -196,11 +196,11 @@ def test_Qido_search_small_buffer_flush(self, FakeClient): input_dict['dicom_store_id'] = "test_dicom_store_id" input_dict['search_type'] = "instances" - fc = FakeHttpClient() - FakeClient.return_value = fc + mc = MockHttpClient() + MockClient.return_value = mc expected_dict = {} - expected_dict['result'] = fc.dicom_metadata + expected_dict['result'] = mc.dicom_metadata expected_dict['status'] = 200 expected_dict['input'] = input_dict expected_dict['success'] = True @@ -210,7 +210,7 @@ def test_Qido_search_small_buffer_flush(self, FakeClient): assert_that(results, equal_to([expected_dict] * 5)) @patch("apache_beam.io.gcp.healthcare.dicomio.DicomApiHttpClient") - def test_param_dict_passing(self, FakeClient): + def test_param_dict_passing(self, MockClient): input_dict = {} input_dict = {} input_dict['project_id'] = "test_project" @@ -228,14 +228,14 @@ def test_param_dict_passing(self, FakeClient): expected_dict['input'] = input_dict expected_dict['success'] = True - fc = FakeHttpClient() - FakeClient.return_value = fc + mc = MockHttpClient() + MockClient.return_value = mc with TestPipeline() as p: results = (p | beam.Create([input_dict]) | DicomSearch()) assert_that(results, equal_to([expected_dict])) @patch("apache_beam.io.gcp.healthcare.dicomio.DicomApiHttpClient") - def test_wrong_input_type(self, FakeClient): + def test_wrong_input_type(self, MockClient): input_dict = {} input_dict['project_id'] = "test_project" input_dict['region'] = "test_region" @@ -250,14 +250,14 @@ def test_wrong_input_type(self, FakeClient): expected_invalid_dict['input'] = input_dict expected_invalid_dict['success'] = False - fc = FakeHttpClient() - FakeClient.return_value = fc + mc = MockHttpClient() + MockClient.return_value = mc with TestPipeline() as p: results = (p | beam.Create([input_dict]) | DicomSearch()) assert_that(results, equal_to([expected_invalid_dict])) @patch("apache_beam.io.gcp.healthcare.dicomio.DicomApiHttpClient") - def test_missing_parameters(self, FakeClient): + def test_missing_parameters(self, MockClient): input_dict = {} input_dict['project_id'] = "test_project" input_dict['region'] = "test_region" @@ -268,14 +268,14 @@ def test_missing_parameters(self, FakeClient): expected_invalid_dict['input'] = input_dict expected_invalid_dict['success'] = False - fc = FakeHttpClient() - FakeClient.return_value = fc + mc = MockHttpClient() + MockClient.return_value = mc with TestPipeline() as p: results = (p | beam.Create([input_dict]) | DicomSearch()) assert_that(results, equal_to([expected_invalid_dict])) @patch("apache_beam.io.gcp.healthcare.dicomio.DicomApiHttpClient") - def test_client_search_notfound(self, FakeClient): + def test_client_search_notfound(self, MockClient): input_dict = {} # search instances in a not exist store input_dict['project_id'] = "wrong_project" @@ -290,8 +290,8 @@ def test_client_search_notfound(self, FakeClient): expected_invalid_dict['input'] = input_dict expected_invalid_dict['success'] = False - fc = FakeHttpClient() - FakeClient.return_value = fc + mc = MockHttpClient() + MockClient.return_value = mc with TestPipeline() as p: results = (p | beam.Create([input_dict]) | DicomSearch()) assert_that(results, equal_to([expected_invalid_dict])) @@ -300,15 +300,15 @@ def test_client_search_notfound(self, FakeClient): @unittest.skipIf(DicomSearch is None, 'GCP dependencies are not installed') class TestDicomStoreInstance(_TestCaseWithTempDirCleanUp): @patch("apache_beam.io.gcp.healthcare.dicomio.DicomApiHttpClient") - def test_store_byte_file(self, FakeClient): + def test_store_byte_file(self, MockClient): input_dict = {} input_dict['project_id'] = "test_project" input_dict['region'] = "test_region" input_dict['dataset_id'] = "test_dataset_id" input_dict['dicom_store_id'] = "test_dicom_store_id" - fc = FakeHttpClient() - FakeClient.return_value = fc + mc = MockHttpClient() + MockClient.return_value = mc dict_input = {'PatientName': 'George', 'Age': 23, 'TestResult': 'Negative'} str_input = json.dumps(dict_input) @@ -320,18 +320,18 @@ def test_store_byte_file(self, FakeClient): | UploadToDicomStore(input_dict, 'bytes') | beam.Map(lambda x: x['success'])) assert_that(results, equal_to([True])) - self.assertTrue(dict_input in fc.dicom_metadata) + self.assertTrue(dict_input in mc.dicom_metadata) @patch("apache_beam.io.gcp.healthcare.dicomio.DicomApiHttpClient") - def test_store_byte_file_small_buffer_flush(self, FakeClient): + def test_store_byte_file_small_buffer_flush(self, MockClient): input_dict = {} input_dict['project_id'] = "test_project" input_dict['region'] = "test_region" input_dict['dataset_id'] = "test_dataset_id" input_dict['dicom_store_id'] = "test_dicom_store_id" - fc = FakeHttpClient() - FakeClient.return_value = fc + mc = MockHttpClient() + MockClient.return_value = mc dict_input_1 = { 'PatientName': 'George', 'Age': 23, 'TestResult': 'Negative' @@ -351,20 +351,20 @@ def test_store_byte_file_small_buffer_flush(self, FakeClient): | UploadToDicomStore(input_dict, 'bytes', buffer_size=1) | beam.Map(lambda x: x['success'])) assert_that(results, equal_to([True] * 3)) - self.assertTrue(dict_input_1 in fc.dicom_metadata) - self.assertTrue(dict_input_2 in fc.dicom_metadata) - self.assertTrue(dict_input_3 in fc.dicom_metadata) + self.assertTrue(dict_input_1 in mc.dicom_metadata) + self.assertTrue(dict_input_2 in mc.dicom_metadata) + self.assertTrue(dict_input_3 in mc.dicom_metadata) @patch("apache_beam.io.gcp.healthcare.dicomio.DicomApiHttpClient") - def test_store_fileio_file(self, FakeClient): + def test_store_fileio_file(self, MockClient): input_dict = {} input_dict['project_id'] = "test_project" input_dict['region'] = "test_region" input_dict['dataset_id'] = "test_dataset_id" input_dict['dicom_store_id'] = "test_dicom_store_id" - fc = FakeHttpClient() - FakeClient.return_value = fc + mc = MockHttpClient() + MockClient.return_value = mc dict_input = {'PatientName': 'George', 'Age': 23, 'TestResult': 'Negative'} str_input = json.dumps(dict_input) @@ -380,18 +380,18 @@ def test_store_fileio_file(self, FakeClient): | UploadToDicomStore(input_dict, 'fileio') | beam.Map(lambda x: x['success'])) assert_that(results, equal_to([True])) - self.assertTrue(dict_input in fc.dicom_metadata) + self.assertTrue(dict_input in mc.dicom_metadata) @patch("apache_beam.io.gcp.healthcare.dicomio.DicomApiHttpClient") - def test_store_fileio_file_small_buffer_flush(self, FakeClient): + def test_store_fileio_file_small_buffer_flush(self, MockClient): input_dict = {} input_dict['project_id'] = "test_project" input_dict['region'] = "test_region" input_dict['dataset_id'] = "test_dataset_id" input_dict['dicom_store_id'] = "test_dicom_store_id" - fc = FakeHttpClient() - FakeClient.return_value = fc + mc = MockHttpClient() + MockClient.return_value = mc temp_dir = '%s%s' % (self._new_tempdir(), os.sep) dict_input_1 = { @@ -415,12 +415,12 @@ def test_store_fileio_file_small_buffer_flush(self, FakeClient): | UploadToDicomStore(input_dict, 'fileio', buffer_size=1) | beam.Map(lambda x: x['success'])) assert_that(results, equal_to([True] * 3)) - self.assertTrue(dict_input_1 in fc.dicom_metadata) - self.assertTrue(dict_input_2 in fc.dicom_metadata) - self.assertTrue(dict_input_3 in fc.dicom_metadata) + self.assertTrue(dict_input_1 in mc.dicom_metadata) + self.assertTrue(dict_input_2 in mc.dicom_metadata) + self.assertTrue(dict_input_3 in mc.dicom_metadata) @patch("apache_beam.io.gcp.healthcare.dicomio.DicomApiHttpClient") - def test_destination_notfound(self, FakeClient): + def test_destination_notfound(self, MockClient): input_dict = {} # search instances in a not exist store input_dict['project_id'] = "wrong_project" @@ -433,15 +433,15 @@ def test_destination_notfound(self, FakeClient): expected_invalid_dict['input'] = '' expected_invalid_dict['success'] = False - fc = FakeHttpClient() - FakeClient.return_value = fc + mc = MockHttpClient() + MockClient.return_value = mc with TestPipeline() as p: results = ( p | beam.Create(['']) | UploadToDicomStore(input_dict, 'bytes')) assert_that(results, equal_to([expected_invalid_dict])) @patch("apache_beam.io.gcp.healthcare.dicomio.DicomApiHttpClient") - def test_missing_parameters(self, FakeClient): + def test_missing_parameters(self, MockClient): input_dict = {} input_dict['project_id'] = "test_project" input_dict['region'] = "test_region" @@ -452,8 +452,8 @@ def test_missing_parameters(self, FakeClient): expected_invalid_dict['input'] = input_dict expected_invalid_dict['success'] = False - fc = FakeHttpClient() - FakeClient.return_value = fc + mc = MockHttpClient() + MockClient.return_value = mc with self.assertRaisesRegex(ValueError, "Must have dataset_id in the dict."): p = TestPipeline() diff --git a/sdks/python/apache_beam/io/gcp/pubsub.py b/sdks/python/apache_beam/io/gcp/pubsub.py index 8cee8acfebb0f..af58006d6e766 100644 --- a/sdks/python/apache_beam/io/gcp/pubsub.py +++ b/sdks/python/apache_beam/io/gcp/pubsub.py @@ -39,9 +39,9 @@ from typing import Tuple from apache_beam import coders +from apache_beam.io import iobase from apache_beam.io.iobase import Read from apache_beam.io.iobase import Write -from apache_beam.runners.dataflow.native_io import iobase as dataflow_io from apache_beam.transforms import Flatten from apache_beam.transforms import Map from apache_beam.transforms import PTransform @@ -261,6 +261,7 @@ def __init__( timestamp_attribute=timestamp_attribute) def expand(self, pvalue): + # TODO(BEAM-27443): Apply a proper transform rather than Read. pcoll = pvalue.pipeline | Read(self._source) pcoll.element_type = bytes if self.with_attributes: @@ -423,7 +424,8 @@ def parse_subscription(full_subscription): return project, subscription_name -class _PubSubSource(dataflow_io.NativeSource): +# TODO(BEAM-27443): Remove (or repurpose as a proper PTransform). +class _PubSubSource(iobase.SourceBase): """Source for a Cloud Pub/Sub topic or subscription. This ``NativeSource`` is overridden by a native Pubsub implementation. @@ -460,11 +462,6 @@ def __init__( if subscription: self.project, self.subscription_name = parse_subscription(subscription) - @property - def format(self): - """Source format name required for remote execution.""" - return 'pubsub' - def display_data(self): return { 'id_label': DisplayDataItem(self.id_label, @@ -480,14 +477,15 @@ def display_data(self): label='Timestamp Attribute').drop_if_none(), } - def reader(self): - raise NotImplementedError + def default_output_coder(self): + return self.coder def is_bounded(self): return False -class _PubSubSink(dataflow_io.NativeSink): +# TODO(BEAM-27443): Remove in favor of a proper WriteToPubSub transform. +class _PubSubSink(object): """Sink for a Cloud Pub/Sub topic. This ``NativeSource`` is overridden by a native Pubsub implementation. @@ -505,14 +503,6 @@ def __init__( self.project, self.topic_name = parse_topic(topic) - @property - def format(self): - """Sink format name required for remote execution.""" - return 'pubsub' - - def writer(self): - raise NotImplementedError - class PubSubSourceDescriptor(NamedTuple): """A PubSub source descriptor for ``MultipleReadFromPubSub``` diff --git a/sdks/python/apache_beam/io/gcp/pubsub_test.py b/sdks/python/apache_beam/io/gcp/pubsub_test.py index 8e297511ce0d2..7b4a4d5c93b90 100644 --- a/sdks/python/apache_beam/io/gcp/pubsub_test.py +++ b/sdks/python/apache_beam/io/gcp/pubsub_test.py @@ -530,7 +530,7 @@ def test_read_messages_success(self, mock_pubsub): mock_pubsub.return_value.close.assert_has_calls([mock.call()]) def test_read_strings_success(self, mock_pubsub): - data = u'🤷 ¯\\_(ツ)_/¯' + data = '🤷 ¯\\_(ツ)_/¯' data_encoded = data.encode('utf-8') ack_id = 'ack_id' pull_response = test_utils.create_pull_response( @@ -552,7 +552,7 @@ def test_read_strings_success(self, mock_pubsub): mock_pubsub.return_value.close.assert_has_calls([mock.call()]) def test_read_data_success(self, mock_pubsub): - data_encoded = u'🤷 ¯\\_(ツ)_/¯'.encode('utf-8') + data_encoded = '🤷 ¯\\_(ツ)_/¯'.encode('utf-8') ack_id = 'ack_id' pull_response = test_utils.create_pull_response( [test_utils.PullResponseMessage(data_encoded, ack_id=ack_id)]) diff --git a/sdks/python/apache_beam/io/hdfs_integration_test/Dockerfile b/sdks/python/apache_beam/io/hdfs_integration_test/Dockerfile index 487d5c3487aba..ab79405633941 100644 --- a/sdks/python/apache_beam/io/hdfs_integration_test/Dockerfile +++ b/sdks/python/apache_beam/io/hdfs_integration_test/Dockerfile @@ -30,7 +30,7 @@ COPY sdks/python /app/sdks/python COPY model /app/model # This step should look like setupVirtualenv minus virtualenv creation. -RUN pip install --no-cache-dir tox -r sdks/python/build-requirements.txt +RUN pip install --no-cache-dir tox # Run wordcount, and write results to HDFS. CMD cd sdks/python && tox -e hdfs_integration_test diff --git a/sdks/python/apache_beam/io/iobase.py b/sdks/python/apache_beam/io/iobase.py index 6d75d520af55b..96f154dbe4b81 100644 --- a/sdks/python/apache_beam/io/iobase.py +++ b/sdks/python/apache_beam/io/iobase.py @@ -101,6 +101,9 @@ class SourceBase(HasDisplayData, urns.RunnerApiFn): """ urns.RunnerApiFn.register_pickle_urn(python_urns.PICKLED_SOURCE) + def default_output_coder(self): + raise NotImplementedError + def is_bounded(self): # type: () -> bool raise NotImplementedError @@ -905,7 +908,8 @@ def expand(self, pbegin): return ( pbegin | Impulse() - | core.Map(lambda _: self.source).with_output_types(BoundedSource) + | 'EmitSource' >> + core.Map(lambda _: self.source).with_output_types(BoundedSource) | SDFBoundedSourceReader(display_data)) elif isinstance(self.source, ptransform.PTransform): # The Read transform can also admit a full PTransform as an input @@ -923,11 +927,8 @@ def get_windowing(self, unused_inputs): def _infer_output_coder(self, input_type=None, input_coder=None): # type: (...) -> Optional[coders.Coder] - from apache_beam.runners.dataflow.native_io import iobase as dataflow_io - if isinstance(self.source, BoundedSource): + if isinstance(self.source, SourceBase): return self.source.default_output_coder() - elif isinstance(self.source, dataflow_io.NativeSource): - return self.source.coder else: return None @@ -941,18 +942,17 @@ def to_runner_api_parameter( self, context: PipelineContext, ) -> Tuple[str, Any]: - from apache_beam.runners.dataflow.native_io import iobase as dataflow_io - if isinstance(self.source, (BoundedSource, dataflow_io.NativeSource)): - from apache_beam.io.gcp.pubsub import _PubSubSource - if isinstance(self.source, _PubSubSource): - return ( - common_urns.composites.PUBSUB_READ.urn, - beam_runner_api_pb2.PubSubReadPayload( - topic=self.source.full_topic, - subscription=self.source.full_subscription, - timestamp_attribute=self.source.timestamp_attribute, - with_attributes=self.source.with_attributes, - id_attribute=self.source.id_label)) + from apache_beam.io.gcp.pubsub import _PubSubSource + if isinstance(self.source, _PubSubSource): + return ( + common_urns.composites.PUBSUB_READ.urn, + beam_runner_api_pb2.PubSubReadPayload( + topic=self.source.full_topic, + subscription=self.source.full_subscription, + timestamp_attribute=self.source.timestamp_attribute, + with_attributes=self.source.with_attributes, + id_attribute=self.source.id_label)) + if isinstance(self.source, BoundedSource): return ( common_urns.deprecated_primitives.READ.urn, beam_runner_api_pb2.ReadPayload( @@ -976,6 +976,7 @@ def from_runner_api_parameter( if transform.spec.urn == common_urns.composites.PUBSUB_READ.urn: assert isinstance(payload, beam_runner_api_pb2.PubSubReadPayload) # Importing locally to prevent circular dependencies. + # TODO(BEAM-27443): Remove the need for this. from apache_beam.io.gcp.pubsub import _PubSubSource source = _PubSubSource( topic=payload.topic or None, @@ -1015,6 +1016,7 @@ def _from_runner_api_parameter_pubsub_read( Read._from_runner_api_parameter_read, ) +# TODO(BEAM-27443): Remove. ptransform.PTransform.register_urn( common_urns.composites.PUBSUB_READ.urn, beam_runner_api_pb2.PubSubReadPayload, @@ -1065,10 +1067,11 @@ def display_data(self): return {'sink': self.sink.__class__, 'sink_dd': self.sink} def expand(self, pcoll): - from apache_beam.runners.dataflow.native_io import iobase as dataflow_io - if isinstance(self.sink, dataflow_io.NativeSink): - # A native sink - return pcoll | 'NativeWrite' >> dataflow_io._NativeWrite(self.sink) + # Importing locally to prevent circular dependencies. + from apache_beam.io.gcp.pubsub import _PubSubSink + if isinstance(self.sink, _PubSubSink): + # TODO(BEAM-27443): Remove the need for special casing here. + return pvalue.PDone(pcoll.pipeline) elif isinstance(self.sink, Sink): # A custom sink return pcoll | WriteImpl(self.sink) @@ -1084,6 +1087,7 @@ def to_runner_api_parameter( self, context: PipelineContext, ) -> Tuple[str, Any]: + # TODO(BEAM-27443): Remove the need for special casing here. # Importing locally to prevent circular dependencies. from apache_beam.io.gcp.pubsub import _PubSubSink if isinstance(self.sink, _PubSubSink): diff --git a/sdks/python/apache_beam/io/jdbc.py b/sdks/python/apache_beam/io/jdbc.py index f8f24ddeb8d23..903b0d1b0fef1 100644 --- a/sdks/python/apache_beam/io/jdbc.py +++ b/sdks/python/apache_beam/io/jdbc.py @@ -86,6 +86,7 @@ # pytype: skip-file +import datetime import typing import numpy as np @@ -94,7 +95,10 @@ from apache_beam.transforms.external import BeamJarExpansionService from apache_beam.transforms.external import ExternalTransform from apache_beam.transforms.external import NamedTupleBasedPayloadBuilder +from apache_beam.typehints.schemas import LogicalType +from apache_beam.typehints.schemas import MillisInstant from apache_beam.typehints.schemas import typing_to_runner_api +from apache_beam.utils.timestamp import Timestamp __all__ = [ 'WriteToJdbc', @@ -355,3 +359,99 @@ def __init__( ), expansion_service or default_io_expansion_service(classpath), ) + + +@LogicalType.register_logical_type +class JdbcDateType(LogicalType[datetime.date, MillisInstant, str]): + """ + For internal use only; no backwards-compatibility guarantees. + + Support of Legacy JdbcIO DATE logical type. Deemed to change when Java JDBCIO + has been migrated to Beam portable logical types. + """ + def __init__(self, argument=""): + pass + + @classmethod + def representation_type(cls): + # type: () -> type + return Timestamp + + @classmethod + def urn(cls): + return "beam:logical_type:javasdk_date:v1" + + @classmethod + def language_type(cls): + return datetime.date + + def to_representation_type(self, value): + # type: (datetime.date) -> Timestamp + return Timestamp.from_utc_datetime( + datetime.datetime.combine( + value, datetime.datetime.min.time(), tzinfo=datetime.timezone.utc)) + + def to_language_type(self, value): + # type: (Timestamp) -> datetime.date + + return value.to_utc_datetime().date() + + @classmethod + def argument_type(cls): + return str + + def argument(self): + return "" + + @classmethod + def _from_typing(cls, typ): + return cls() + + +@LogicalType.register_logical_type +class JdbcTimeType(LogicalType[datetime.time, MillisInstant, str]): + """ + For internal use only; no backwards-compatibility guarantees. + + Support of Legacy JdbcIO TIME logical type. . Deemed to change when Java + JDBCIO has been migrated to Beam portable logical types. + """ + def __init__(self, argument=""): + pass + + @classmethod + def representation_type(cls): + # type: () -> type + return Timestamp + + @classmethod + def urn(cls): + return "beam:logical_type:javasdk_time:v1" + + @classmethod + def language_type(cls): + return datetime.time + + def to_representation_type(self, value): + # type: (datetime.date) -> Timestamp + return Timestamp.from_utc_datetime( + datetime.datetime.combine( + datetime.datetime.utcfromtimestamp(0), + value, + tzinfo=datetime.timezone.utc)) + + def to_language_type(self, value): + # type: (Timestamp) -> datetime.date + + return value.to_utc_datetime().time() + + @classmethod + def argument_type(cls): + return str + + def argument(self): + return "" + + @classmethod + def _from_typing(cls, typ): + return cls() diff --git a/sdks/python/apache_beam/io/parquetio.py b/sdks/python/apache_beam/io/parquetio.py index dfcc1abec29a0..4696e5ae79277 100644 --- a/sdks/python/apache_beam/io/parquetio.py +++ b/sdks/python/apache_beam/io/parquetio.py @@ -31,29 +31,36 @@ # pytype: skip-file from functools import partial +from typing import Iterator -from pkg_resources import parse_version +from packaging import version from apache_beam.io import filebasedsink from apache_beam.io import filebasedsource from apache_beam.io.filesystem import CompressionTypes +from apache_beam.io.filesystems import FileSystems from apache_beam.io.iobase import RangeTracker from apache_beam.io.iobase import Read from apache_beam.io.iobase import Write +from apache_beam.portability.api import schema_pb2 from apache_beam.transforms import DoFn from apache_beam.transforms import ParDo from apache_beam.transforms import PTransform from apache_beam.transforms import window +from apache_beam.typehints import schemas try: import pyarrow as pa import pyarrow.parquet as pq + # pylint: disable=ungrouped-imports + from apache_beam.typehints import arrow_type_compatibility except ImportError: pa = None pq = None ARROW_MAJOR_VERSION = None + arrow_type_compatibility = None else: - base_pa_version = parse_version(pa.__version__).base_version + base_pa_version = version.parse(pa.__version__).base_version ARROW_MAJOR_VERSION, _, _ = map(int, base_pa_version.split('.')) __all__ = [ @@ -146,6 +153,24 @@ def _flush_buffer(self): self._record_batches_byte_size = self._record_batches_byte_size + size +class _ArrowTableToBeamRows(DoFn): + def __init__(self, beam_type): + self._beam_type = beam_type + + @DoFn.yields_batches + def process(self, element) -> Iterator[pa.Table]: + yield element + + def infer_output_type(self, input_type): + return self._beam_type + + +class _BeamRowsToArrowTable(DoFn): + @DoFn.yields_elements + def process_batch(self, element: pa.Table) -> Iterator[pa.Table]: + yield element + + class ReadFromParquetBatched(PTransform): """A :class:`~apache_beam.transforms.ptransform.PTransform` for reading Parquet files as a `PCollection` of `pyarrow.Table`. This `PTransform` is @@ -191,7 +216,7 @@ def __init__( """ super().__init__() - self._source = _create_parquet_source( + self._source = _ParquetSource( file_pattern, min_bundle_size, validate=validate, @@ -210,7 +235,12 @@ class ReadFromParquet(PTransform): Parquet files as a `PCollection` of dictionaries. This `PTransform` is currently experimental. No backward-compatibility guarantees.""" def __init__( - self, file_pattern=None, min_bundle_size=0, validate=True, columns=None): + self, + file_pattern=None, + min_bundle_size=0, + validate=True, + columns=None, + as_rows=False): """Initializes :class:`ReadFromParquet`. Uses source ``_ParquetSource`` to read a set of Parquet files defined by @@ -255,17 +285,38 @@ def __init__( columns (List[str]): list of columns that will be read from files. A column name may be a prefix of a nested field, e.g. 'a' will select 'a.b', 'a.c', and 'a.d.e' + as_rows (bool): whether to output a schema'd PCollection of Beam rows + rather than Python dictionaries. """ super().__init__() - self._source = _create_parquet_source( + self._source = _ParquetSource( file_pattern, min_bundle_size, validate=validate, columns=columns, ) + if as_rows: + if columns is None: + filter_schema = lambda schema: schema + else: + top_level_columns = set(c.split('.')[0] for c in columns) + filter_schema = lambda schema: schema_pb2.Schema( + fields=[f for f in schema.fields if f.name in top_level_columns]) + path = FileSystems.match([file_pattern], [1])[0].metadata_list[0].path + with FileSystems.open(path) as fin: + self._schema = filter_schema( + arrow_type_compatibility.beam_schema_from_arrow_schema( + pq.read_schema(fin))) + else: + self._schema = None def expand(self, pvalue): - return pvalue | Read(self._source) | ParDo(_ArrowTableToRowDictionaries()) + arrow_batches = pvalue | Read(self._source) + if self._schema is None: + return arrow_batches | ParDo(_ArrowTableToRowDictionaries()) + else: + return arrow_batches | ParDo( + _ArrowTableToBeamRows(schemas.named_tuple_from_schema(self._schema))) def display_data(self): return {'source_dd': self._source} @@ -305,9 +356,7 @@ def __init__( """ super().__init__() source_from_file = partial( - _create_parquet_source, - min_bundle_size=min_bundle_size, - columns=columns) + _ParquetSource, min_bundle_size=min_bundle_size, columns=columns) self._read_all_files = filebasedsource.ReadAllFiles( True, CompressionTypes.UNCOMPRESSED, @@ -333,17 +382,6 @@ def expand(self, pvalue): _ArrowTableToRowDictionaries(), with_filename=self._with_filename) -def _create_parquet_source( - file_pattern=None, min_bundle_size=0, validate=False, columns=None): - return \ - _ParquetSource( - file_pattern=file_pattern, - min_bundle_size=min_bundle_size, - validate=validate, - columns=columns, - ) - - class _ParquetUtils(object): @staticmethod def find_first_row_group_index(pf, start_offset): @@ -370,7 +408,8 @@ def get_number_of_row_groups(pf): class _ParquetSource(filebasedsource.FileBasedSource): """A source for reading Parquet files. """ - def __init__(self, file_pattern, min_bundle_size, validate, columns): + def __init__( + self, file_pattern, min_bundle_size=0, validate=False, columns=None): super().__init__( file_pattern=file_pattern, min_bundle_size=min_bundle_size, @@ -421,6 +460,9 @@ def split_points_unclaimed(stop_position): yield table +_create_parquet_source = _ParquetSource + + class WriteToParquet(PTransform): """A ``PTransform`` for writing parquet files. @@ -430,7 +472,7 @@ class WriteToParquet(PTransform): def __init__( self, file_path_prefix, - schema, + schema=None, row_group_buffer_size=64 * 1024 * 1024, record_batch_size=1000, codec='none', @@ -534,10 +576,19 @@ def __init__( ) def expand(self, pcoll): - return pcoll | ParDo( - _RowDictionariesToArrowTable( - self._schema, self._row_group_buffer_size, - self._record_batch_size)) | Write(self._sink) + if self._schema is None: + try: + beam_schema = schemas.schema_from_element_type(pcoll.element_type) + except TypeError as exn: + raise ValueError( + "A schema is required to write non-schema'd data.") from exn + self._sink._schema = ( + arrow_type_compatibility.arrow_schema_from_beam_schema(beam_schema)) + convert_fn = _BeamRowsToArrowTable() + else: + convert_fn = _RowDictionariesToArrowTable( + self._schema, self._row_group_buffer_size, self._record_batch_size) + return pcoll | ParDo(convert_fn) | Write(self._sink) def display_data(self): return { diff --git a/sdks/python/apache_beam/io/parquetio_test.py b/sdks/python/apache_beam/io/parquetio_test.py index df018a3a776f7..1cd5f1208cc2a 100644 --- a/sdks/python/apache_beam/io/parquetio_test.py +++ b/sdks/python/apache_beam/io/parquetio_test.py @@ -30,6 +30,7 @@ from parameterized import param from parameterized import parameterized +import apache_beam as beam from apache_beam import Create from apache_beam import Map from apache_beam.io import filebasedsource @@ -400,6 +401,21 @@ def test_sink_transform_compliant_nested_type(self): assert_that( readback, equal_to([json.dumps(r) for r in self.RECORDS_NESTED])) + def test_schema_read_write(self): + with TemporaryDirectory() as tmp_dirname: + path = os.path.join(tmp_dirname, 'tmp_filename') + rows = [beam.Row(a=1, b='x'), beam.Row(a=2, b='y')] + stable_repr = lambda row: json.dumps(row._asdict()) + with TestPipeline() as p: + _ = p | Create(rows) | WriteToParquet(path) | beam.Map(print) + with TestPipeline() as p: + # json used for stable sortability + readback = ( + p + | ReadFromParquet(path + '*', as_rows=True) + | Map(stable_repr)) + assert_that(readback, equal_to([stable_repr(r) for r in rows])) + def test_batched_read(self): with TemporaryDirectory() as tmp_dirname: path = os.path.join(tmp_dirname + "tmp_filename") diff --git a/sdks/python/apache_beam/io/range_trackers.py b/sdks/python/apache_beam/io/range_trackers.py index adadc1a7ae41e..ba56fd3f3559d 100644 --- a/sdks/python/apache_beam/io/range_trackers.py +++ b/sdks/python/apache_beam/io/range_trackers.py @@ -312,7 +312,7 @@ def __init__(self, range_tracker): Args: range_tracker (~apache_beam.io.iobase.RangeTracker): a :class:`~apache_beam.io.iobase.RangeTracker` to which all method - calls expect calls to :meth:`.try_split()` will be delegated. + calls except calls to :meth:`.try_split()` will be delegated. """ assert isinstance(range_tracker, iobase.RangeTracker) self._range_tracker = range_tracker diff --git a/sdks/python/apache_beam/metrics/OWNERS b/sdks/python/apache_beam/metrics/OWNERS deleted file mode 100644 index f951b044bbca4..0000000000000 --- a/sdks/python/apache_beam/metrics/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - pabloem - - aaltay diff --git a/sdks/python/apache_beam/metrics/cells.py b/sdks/python/apache_beam/metrics/cells.py index 259f29b2f0eae..53b6fc8495920 100644 --- a/sdks/python/apache_beam/metrics/cells.py +++ b/sdks/python/apache_beam/metrics/cells.py @@ -99,9 +99,9 @@ class CounterCell(MetricCell): Tracks the current value and delta of a counter metric. - Each cell tracks the state of a metric independently per context per bundle. - Therefore, each metric has a different cell in each bundle, cells are - aggregated by the runner. + Each cell tracks the state of an integer metric independently per context + per bundle. Therefore, each metric has a different cell in each bundle, + cells are aggregated by the runner. This class is thread safe. """ @@ -126,6 +126,7 @@ def dec(self, n=1): self.update(-n) def update(self, value): + # type: (int) -> None if cython.compiled: ivalue = value # Since We hold the GIL, no need for another lock. diff --git a/sdks/python/apache_beam/metrics/monitoring_infos.py b/sdks/python/apache_beam/metrics/monitoring_infos.py index ed7f952cf8e45..ae12c93a5718e 100644 --- a/sdks/python/apache_beam/metrics/monitoring_infos.py +++ b/sdks/python/apache_beam/metrics/monitoring_infos.py @@ -299,8 +299,13 @@ def create_monitoring_info(urn, type_urn, payload, labels=None): payload: The payload field to use in the monitoring info. labels: The label dictionary to use in the MonitoringInfo. """ - return metrics_pb2.MonitoringInfo( - urn=urn, type=type_urn, labels=labels or {}, payload=payload) + try: + return metrics_pb2.MonitoringInfo( + urn=urn, type=type_urn, labels=labels or {}, payload=payload) + except TypeError as e: + raise RuntimeError( + f'Failed to create MonitoringInfo for urn {urn} type {type} labels ' + + '{labels} and payload {payload}') from e def is_counter(monitoring_info_proto): diff --git a/sdks/python/apache_beam/ml/__init__.py b/sdks/python/apache_beam/ml/__init__.py index f319fa046f7d8..6813137f51915 100644 --- a/sdks/python/apache_beam/ml/__init__.py +++ b/sdks/python/apache_beam/ml/__init__.py @@ -16,3 +16,5 @@ # """Contains packages for supported machine learning transforms.""" + +from apache_beam.ml.transforms.base import MLTransform diff --git a/sdks/python/apache_beam/ml/gcp/naturallanguageml_test.py b/sdks/python/apache_beam/ml/gcp/naturallanguageml_test.py index bad7443d0d944..891726cb2688e 100644 --- a/sdks/python/apache_beam/ml/gcp/naturallanguageml_test.py +++ b/sdks/python/apache_beam/ml/gcp/naturallanguageml_test.py @@ -20,11 +20,7 @@ import unittest -import mock - -import apache_beam as beam from apache_beam.metrics import MetricsFilter -from apache_beam.testing.test_pipeline import TestPipeline # Protect against environments where Google Cloud Natural Language client # is not available. @@ -60,21 +56,6 @@ def test_document_source(self): self.assertFalse('content' in dict_) self.assertTrue('gcs_content_uri' in dict_) - def test_annotate_test_called(self): - with mock.patch('apache_beam.ml.gcp.naturallanguageml._AnnotateTextFn' - '._get_api_client'): - p = TestPipeline() - features = [ - naturallanguageml.language_v1.AnnotateTextRequest.Features( - extract_syntax=True) - ] - _ = ( - p | beam.Create([naturallanguageml.Document('Hello, world!')]) - | naturallanguageml.AnnotateText(features)) - result = p.run() - result.wait_until_finish() - self.assertCounterEqual(result, 'api_calls', 1) - if __name__ == '__main__': unittest.main() diff --git a/sdks/python/apache_beam/ml/inference/base.py b/sdks/python/apache_beam/ml/inference/base.py index 81204542e88a8..45c5078c13cf7 100644 --- a/sdks/python/apache_beam/ml/inference/base.py +++ b/sdks/python/apache_beam/ml/inference/base.py @@ -34,11 +34,16 @@ import threading import time import uuid +from collections import OrderedDict +from collections import defaultdict +from copy import deepcopy +from dataclasses import dataclass from typing import Any from typing import Callable from typing import Dict from typing import Generic from typing import Iterable +from typing import List from typing import Mapping from typing import NamedTuple from typing import Optional @@ -103,6 +108,12 @@ class RunInferenceDLQ(NamedTuple): failed_postprocessing: Sequence[beam.PCollection] +class _ModelLoadStats(NamedTuple): + model_tag: str + load_latency: Optional[int] + byte_size: Optional[int] + + ModelMetadata.model_id.__doc__ = """Unique identifier for the model. This can be a file path or a URL where the model can be accessed. It is used to load the model for inference.""" @@ -119,6 +130,26 @@ def _to_microseconds(time_ns: int) -> int: return int(time_ns / _NANOSECOND_TO_MICROSECOND) +@dataclass(frozen=True) +class KeyModelPathMapping(Generic[KeyT]): + """ + Dataclass for mapping 1 or more keys to 1 model path. This is used in + conjunction with a KeyedModelHandler with many model handlers to update + a set of keys' model handlers with the new path. Given + `KeyModelPathMapping(keys: ['key1', 'key2'], update_path: 'updated/path', + model_id: 'id1')`, all examples with keys `key1` or `key2` will have their + corresponding model handler's update_model function called with + 'updated/path' and their metrics will correspond with 'id1'. For more + information see the KeyedModelHandler documentation + https://beam.apache.org/releases/pydoc/current/apache_beam.ml.inference.base.html#apache_beam.ml.inference.base.KeyedModelHandler + documentation and the website section on model updates + https://beam.apache.org/documentation/sdks/python-machine-learning/#automatic-model-refresh + """ + keys: List[KeyT] + update_path: str + model_id: str = '' + + class ModelHandler(Generic[ExampleT, PredictionT, ModelT]): """Has the ability to load and apply an ML model.""" def __init__(self): @@ -188,7 +219,28 @@ def validate_inference_args(self, inference_args: Optional[Dict[str, Any]]): 'framework does not expect extra arguments on inferences.') def update_model_path(self, model_path: Optional[str] = None): - """Update the model paths produced by side inputs.""" + """ + Update the model path produced by side inputs. update_model_path should be + used when a ModelHandler represents a single model, not multiple models. + This will be true in most cases. For more information see the website + section on model updates + https://beam.apache.org/documentation/sdks/python-machine-learning/#automatic-model-refresh + """ + pass + + def update_model_paths( + self, + model: ModelT, + model_paths: Optional[Union[str, List[KeyModelPathMapping]]] = None): + """ + Update the model paths produced by side inputs. update_model_paths should + be used when updating multiple models at once (e.g. when using a + KeyedModelHandler that holds multiple models). For more information see + the KeyedModelHandler documentation + https://beam.apache.org/releases/pydoc/current/apache_beam.ml.inference.base.html#apache_beam.ml.inference.base.KeyedModelHandler + documentation and the website section on model updates + https://beam.apache.org/documentation/sdks/python-machine-learning/#automatic-model-refresh + """ pass def get_preprocess_fns(self) -> Iterable[Callable[[Any], Any]]: @@ -241,71 +293,497 @@ def share_model_across_processes(self) -> bool: https://beam.apache.org/releases/pydoc/current/apache_beam.utils.multi_process_shared.html""" return False + def override_metrics(self, metrics_namespace: str = '') -> bool: + """Returns a boolean representing whether or not a model handler will + override metrics reporting. If True, RunInference will not report any + metrics.""" + return False + + +class _ModelManager: + """ + A class for efficiently managing copies of multiple models. Will load a + single copy of each model into a multi_process_shared object and then + return a lookup key for that object. + """ + def __init__(self, mh_map: Dict[str, ModelHandler]): + """ + Args: + mh_map: A map from keys to model handlers which can be used to load a + model. + """ + self._max_models = None + # Map keys to model handlers + self._mh_map: Dict[str, ModelHandler] = mh_map + # Map keys to the last updated model path for that key + self._key_to_last_update: Dict[str, str] = defaultdict(str) + # Map key for a model to a unique tag that will persist for the life of + # that model in memory. A new tag will be generated if a model is swapped + # out of memory and reloaded. + self._tag_map: Dict[str, str] = OrderedDict() + # Map a tag to a multiprocessshared model object for that tag. Each entry + # of this map should last as long as the corresponding entry in _tag_map. + self._proxy_map: Dict[str, multi_process_shared.MultiProcessShared] = {} + + def load(self, key: str) -> _ModelLoadStats: + """ + Loads the appropriate model for the given key into memory. + Args: + key: the key associated with the model we'd like to load. + Returns: + _ModelLoadStats with tag, byte size, and latency to load the model. If + the model was already loaded, byte size/latency will be None. + """ + # Map the key for a model to a unique tag that will persist until the model + # is released. This needs to be unique between releasing/reacquiring th + # model because otherwise the ProxyManager will try to reuse the model that + # has been released and deleted. + if key in self._tag_map: + self._tag_map.move_to_end(key) + return _ModelLoadStats(self._tag_map[key], None, None) + else: + self._tag_map[key] = uuid.uuid4().hex + + tag = self._tag_map[key] + mh = self._mh_map[key] + + if self._max_models is not None and self._max_models < len(self._tag_map): + # If we're about to exceed our LRU size, release the last used model. + tag_to_remove = self._tag_map.popitem(last=False)[1] + shared_handle, model_to_remove = self._proxy_map[tag_to_remove] + shared_handle.release(model_to_remove) + del self._proxy_map[tag_to_remove] + + # Load the new model + memory_before = _get_current_process_memory_in_bytes() + start_time = _to_milliseconds(time.time_ns()) + shared_handle = multi_process_shared.MultiProcessShared( + mh.load_model, tag=tag) + model_reference = shared_handle.acquire() + self._proxy_map[tag] = (shared_handle, model_reference) + memory_after = _get_current_process_memory_in_bytes() + end_time = _to_milliseconds(time.time_ns()) + + return _ModelLoadStats( + tag, end_time - start_time, memory_after - memory_before) + + def increment_max_models(self, increment: int): + """ + Increments the number of models that this instance of a _ModelManager is + able to hold. If it is never called, no limit is imposed. + Args: + increment: the amount by which we are incrementing the number of models. + """ + if self._max_models is None: + self._max_models = 0 + self._max_models += increment + + def update_model_handler(self, key: str, model_path: str, previous_key: str): + """ + Updates the model path of this model handler and removes it from memory so + that it can be reloaded with the updated path. No-ops if no model update + needs to be applied. + Args: + key: the key associated with the model we'd like to update. + model_path: the new path to the model we'd like to load. + previous_key: the key that is associated with the old version of this + model. This will often be the same as the current key, but sometimes + we will want to keep both the old and new models to serve different + cohorts. In that case, the keys should be different. + """ + if self._key_to_last_update[key] == model_path: + return + self._key_to_last_update[key] = model_path + if key not in self._mh_map: + self._mh_map[key] = deepcopy(self._mh_map[previous_key]) + self._mh_map[key].update_model_path(model_path) + if key in self._tag_map: + tag_to_remove = self._tag_map[key] + shared_handle, model_to_remove = self._proxy_map[tag_to_remove] + shared_handle.release(model_to_remove) + del self._tag_map[key] + del self._proxy_map[tag_to_remove] + + +# Use a dataclass instead of named tuple because NamedTuples and generics don't +# mix well across the board for all versions: +# https://github.com/python/typing/issues/653 +class KeyModelMapping(Generic[KeyT, ExampleT, PredictionT, ModelT]): + """ + Dataclass for mapping 1 or more keys to 1 model handler. Given + `KeyModelMapping(['key1', 'key2'], myMh)`, all examples with keys `key1` + or `key2` will be run against the model defined by the `myMh` ModelHandler. + """ + def __init__( + self, keys: List[KeyT], mh: ModelHandler[ExampleT, PredictionT, ModelT]): + self.keys = keys + self.mh = mh + class KeyedModelHandler(Generic[KeyT, ExampleT, PredictionT, ModelT], ModelHandler[Tuple[KeyT, ExampleT], Tuple[KeyT, PredictionT], - ModelT]): - def __init__(self, unkeyed: ModelHandler[ExampleT, PredictionT, ModelT]): + Union[ModelT, _ModelManager]]): + def __init__( + self, + unkeyed: Union[ModelHandler[ExampleT, PredictionT, ModelT], + List[KeyModelMapping[KeyT, ExampleT, PredictionT, + ModelT]]], + max_models_per_worker_hint: Optional[int] = None): """A ModelHandler that takes keyed examples and returns keyed predictions. For example, if the original model is used with RunInference to take a PCollection[E] to a PCollection[P], this ModelHandler would take a PCollection[Tuple[K, E]] to a PCollection[Tuple[K, P]], making it possible - to use the key to associate the outputs with the inputs. + to use the key to associate the outputs with the inputs. KeyedModelHandler + is able to accept either a single unkeyed ModelHandler or many different + model handlers corresponding to the keys for which that ModelHandler should + be used. For example, the following configuration could be used to map keys + 1-3 to ModelHandler1 and keys 4-5 to ModelHandler2: + + k1 = ['k1', 'k2', 'k3'] + k2 = ['k4', 'k5'] + KeyedModelHandler([KeyModelMapping(k1, mh1), KeyModelMapping(k2, mh2)]) + + Note that a single copy of each of these models may all be held in memory + at the same time; be careful not to load too many large models or your + pipeline may cause Out of Memory exceptions. + + KeyedModelHandlers support Automatic Model Refresh to update your model + to a newer version without stopping your streaming pipeline. For an + overview of this feature, see + https://beam.apache.org/documentation/sdks/python-machine-learning/#automatic-model-refresh + + + To use this feature with a KeyedModelHandler that has many models per key, + you can pass in a list of KeyModelPathMapping objects to define your new + model paths. For example, passing in the side input of + + [KeyModelPathMapping(keys=['k1', 'k2'], update_path='update/path/1'), + KeyModelPathMapping(keys=['k3'], update_path='update/path/2')] + + will update the model corresponding to keys 'k1' and 'k2' with path + 'update/path/1' and the model corresponding to 'k3' with 'update/path/2'. + In order to do a side input update: (1) all restrictions mentioned in + https://beam.apache.org/documentation/sdks/python-machine-learning/#automatic-model-refresh + must be met, (2) all update_paths must be non-empty, even if they are not + being updated from their original values, and (3) the set of keys + originally defined cannot change. This means that if originally you have + defined model handlers for 'key1', 'key2', and 'key3', all 3 of those keys + must appear in your list of KeyModelPathMappings exactly once. No + additional keys can be added. + + When using many models defined per key, metrics about inference and model + loading will be gathered on an aggregate basis for all keys. These will be + reported with no prefix. Metrics will also be gathered on a per key basis. + Since some keys can share the same model, only one set of metrics will be + reported per key 'cohort'. These will be reported in the form: + `-`, where `` can be any key selected + from the cohort. When model updates occur, the metrics will be reported in + the form `--`. + + Loading multiple models at the same time can increase the risk of an out of + memory (OOM) exception. To avoid this issue, use the parameter + `max_models_per_worker_hint` to limit the number of models that are loaded + at the same time. For more information about memory management, see + `Use a keyed `ModelHandler _`. # pylint: disable=line-too-long + Args: - unkeyed: An implementation of ModelHandler that does not require keys. + unkeyed: Either (a) an implementation of ModelHandler that does not + require keys or (b) a list of KeyModelMappings mapping lists of keys to + unkeyed ModelHandlers. + max_models_per_worker_hint: A hint to the runner indicating how many + models can be held in memory at one time per worker process. For + example, if your worker has 8 GB of memory provisioned and your workers + take up 1 GB each, you should set this to 7 to allow all models to sit + in memory with some buffer. For more information about memory management, + see `Use a keyed `ModelHandler _`. # pylint: disable=line-too-long """ - if len(unkeyed.get_preprocess_fns()) or len(unkeyed.get_postprocess_fns()): - raise Exception( - 'Cannot make make an unkeyed model handler with pre or ' - 'postprocessing functions defined into a keyed model handler. All ' - 'pre/postprocessing functions must be defined on the outer model' - 'handler.') - self._unkeyed = unkeyed - self._env_vars = unkeyed._env_vars - - def load_model(self) -> ModelT: - return self._unkeyed.load_model() + self._metrics_collectors: Dict[str, _MetricsCollector] = {} + self._default_metrics_collector: _MetricsCollector = None + self._metrics_namespace = '' + self._single_model = not isinstance(unkeyed, list) + if self._single_model: + if len(unkeyed.get_preprocess_fns()) or len( + unkeyed.get_postprocess_fns()): + raise Exception( + 'Cannot make make an unkeyed model handler with pre or ' + 'postprocessing functions defined into a keyed model handler. All ' + 'pre/postprocessing functions must be defined on the outer model' + 'handler.') + self._env_vars = unkeyed._env_vars + self._unkeyed = unkeyed + return + + self._max_models_per_worker_hint = max_models_per_worker_hint + # To maintain an efficient representation, we will map all keys in a given + # KeyModelMapping to a single id (the first key in the KeyModelMapping + # list). We will then map that key to a ModelHandler. This will allow us to + # quickly look up the appropriate ModelHandler for any given key. + self._id_to_mh_map: Dict[str, ModelHandler[ExampleT, PredictionT, + ModelT]] = {} + self._key_to_id_map: Dict[str, str] = {} + for mh_tuple in unkeyed: + mh = mh_tuple.mh + keys = mh_tuple.keys + if len(mh.get_preprocess_fns()) or len(mh.get_postprocess_fns()): + raise ValueError( + 'Cannot use an unkeyed model handler with pre or ' + 'postprocessing functions defined in a keyed model handler. All ' + 'pre/postprocessing functions must be defined on the outer model' + 'handler.') + hints = mh.get_resource_hints() + if len(hints) > 0: + logging.warning( + 'mh %s defines the following resource hints, which will be' + 'ignored: %s. Resource hints are not respected when more than one ' + 'model handler is used in a KeyedModelHandler. If you would like ' + 'to specify resource hints, you can do so by overriding the ' + 'KeyedModelHandler.get_resource_hints() method.', + mh, + hints) + batch_kwargs = mh.batch_elements_kwargs() + if len(batch_kwargs) > 0: + logging.warning( + 'mh %s defines the following batching kwargs which will be ' + 'ignored %s. Batching kwargs are not respected when ' + 'more than one model handler is used in a KeyedModelHandler. If ' + 'you would like to specify resource hints, you can do so by ' + 'overriding the KeyedModelHandler.batch_elements_kwargs() method.', + hints, + batch_kwargs) + env_vars = mh._env_vars + if len(env_vars) > 0: + logging.warning( + 'mh %s defines the following _env_vars which will be ignored %s. ' + '_env_vars are not respected when more than one model handler is ' + 'used in a KeyedModelHandler. If you need env vars set at ' + 'inference time, you can do so with ' + 'a custom inference function.', + mh, + env_vars) + + if len(keys) == 0: + raise ValueError( + f'Empty list maps to model handler {mh}. All model handlers must ' + 'have one or more associated keys.') + self._id_to_mh_map[keys[0]] = mh + for key in keys: + if key in self._key_to_id_map: + raise ValueError( + f'key {key} maps to multiple model handlers. All keys must map ' + 'to exactly one model handler.') + self._key_to_id_map[key] = keys[0] + + def load_model(self) -> Union[ModelT, _ModelManager]: + if self._single_model: + return self._unkeyed.load_model() + return _ModelManager(self._id_to_mh_map) def run_inference( self, batch: Sequence[Tuple[KeyT, ExampleT]], - model: ModelT, + model: Union[ModelT, _ModelManager], inference_args: Optional[Dict[str, Any]] = None ) -> Iterable[Tuple[KeyT, PredictionT]]: - keys, unkeyed_batch = zip(*batch) - return zip( - keys, self._unkeyed.run_inference(unkeyed_batch, model, inference_args)) + if self._single_model: + keys, unkeyed_batch = zip(*batch) + return zip( + keys, + self._unkeyed.run_inference(unkeyed_batch, model, inference_args)) + + # The first time a MultiProcessShared ModelManager is used for inference + # from this process, we should increment its max model count + if self._max_models_per_worker_hint is not None: + lock = threading.Lock() + if lock.acquire(blocking=False): + model.increment_max_models(self._max_models_per_worker_hint) + self._max_models_per_worker_hint = None + + batch_by_key = defaultdict(list) + key_by_id = defaultdict(set) + for key, example in batch: + batch_by_key[key].append(example) + key_by_id[self._key_to_id_map[key]].add(key) + + predictions = [] + for id, keys in key_by_id.items(): + mh = self._id_to_mh_map[id] + loaded_model = model.load(id) + keyed_model_tag = loaded_model.model_tag + if loaded_model.byte_size is not None: + self._metrics_collectors[id].update_load_model_metrics( + loaded_model.load_latency, loaded_model.byte_size) + self._default_metrics_collector.update_load_model_metrics( + loaded_model.load_latency, loaded_model.byte_size) + keyed_model_shared_handle = multi_process_shared.MultiProcessShared( + mh.load_model, tag=keyed_model_tag) + keyed_model = keyed_model_shared_handle.acquire() + start_time = _to_microseconds(time.time_ns()) + num_bytes = 0 + num_elements = 0 + try: + for key in keys: + unkeyed_batches = batch_by_key[key] + try: + for inf in mh.run_inference(unkeyed_batches, + keyed_model, + inference_args): + predictions.append((key, inf)) + except BaseException as e: + self._metrics_collectors[id].failed_batches_counter.inc() + self._default_metrics_collector.failed_batches_counter.inc() + raise e + num_bytes += mh.get_num_bytes(unkeyed_batches) + num_elements += len(unkeyed_batches) + finally: + keyed_model_shared_handle.release(keyed_model) + end_time = _to_microseconds(time.time_ns()) + inference_latency = end_time - start_time + self._metrics_collectors[id].update( + num_elements, num_bytes, inference_latency) + self._default_metrics_collector.update( + num_elements, num_bytes, inference_latency) + + return predictions def get_num_bytes(self, batch: Sequence[Tuple[KeyT, ExampleT]]) -> int: keys, unkeyed_batch = zip(*batch) - return len(pickle.dumps(keys)) + self._unkeyed.get_num_bytes(unkeyed_batch) + batch_bytes = len(pickle.dumps(keys)) + if self._single_model: + return batch_bytes + self._unkeyed.get_num_bytes(unkeyed_batch) + + batch_by_key = defaultdict(list) + for key, examples in batch: + batch_by_key[key].append(examples) + + for key, examples in batch_by_key.items(): + mh_id = self._key_to_id_map[key] + batch_bytes += self._id_to_mh_map[mh_id].get_num_bytes(examples) + return batch_bytes def get_metrics_namespace(self) -> str: - return self._unkeyed.get_metrics_namespace() + if self._single_model: + return self._unkeyed.get_metrics_namespace() + return 'BeamML_KeyedModels' def get_resource_hints(self): - return self._unkeyed.get_resource_hints() + if self._single_model: + return self._unkeyed.get_resource_hints() + return {} def batch_elements_kwargs(self): - return self._unkeyed.batch_elements_kwargs() + if self._single_model: + return self._unkeyed.batch_elements_kwargs() + return {} def validate_inference_args(self, inference_args: Optional[Dict[str, Any]]): - return self._unkeyed.validate_inference_args(inference_args) + if self._single_model: + return self._unkeyed.validate_inference_args(inference_args) + for mh in self._id_to_mh_map.values(): + mh.validate_inference_args(inference_args) + + def update_model_paths( + self, + model: Union[ModelT, _ModelManager], + model_paths: List[KeyModelPathMapping[KeyT]] = None): + # When there are many models, the keyed model handler is responsible for + # reorganizing the model handlers into cohorts and telling the model + # manager to update every cohort's associated model handler. The model + # manager is responsible for performing the updates and tracking which + # updates have already been applied. + if model_paths is None or len(model_paths) == 0 or model is None: + return + if self._single_model: + raise RuntimeError( + 'Invalid model update: sent many model paths to ' + 'update, but KeyedModelHandler is wrapping a single ' + 'model.') + # Map cohort ids to a dictionary mapping new model paths to the keys that + # were originally in that cohort. We will use this to construct our new + # cohorts. + # cohort_path_mapping will be structured as follows: + # { + # original_cohort_id: { + # 'update/path/1': ['key1FromOriginalCohort', key2FromOriginalCohort'], + # 'update/path/2': ['key3FromOriginalCohort', key4FromOriginalCohort'], + # } + # } + cohort_path_mapping: Dict[KeyT, Dict[str, List[KeyT]]] = {} + key_modelid_mapping: Dict[KeyT, str] = {} + seen_keys = set() + for mp in model_paths: + keys = mp.keys + update_path = mp.update_path + model_id = mp.model_id + if len(update_path) == 0: + raise ValueError(f'Invalid model update, path for {keys} is empty') + for key in keys: + if key in seen_keys: + raise ValueError( + f'Invalid model update: {key} appears in multiple ' + 'update lists. A single model update must provide exactly one ' + 'updated path per key.') + seen_keys.add(key) + if key not in self._key_to_id_map: + raise ValueError( + f'Invalid model update: {key} appears in ' + 'update, but not in the original configuration.') + key_modelid_mapping[key] = model_id + cohort_id = self._key_to_id_map[key] + if cohort_id not in cohort_path_mapping: + cohort_path_mapping[cohort_id] = defaultdict(list) + cohort_path_mapping[cohort_id][update_path].append(key) + for key in self._key_to_id_map: + if key not in seen_keys: + raise ValueError( + f'Invalid model update: {key} appears in the ' + 'original configuration, but not the update.') + + # We now have our new set of cohorts. For each one, update our local model + # handler configuration and send the results to the ModelManager + for old_cohort_id, path_key_mapping in cohort_path_mapping.items(): + for updated_path, keys in path_key_mapping.items(): + cohort_id = old_cohort_id + if old_cohort_id not in keys: + # Create new cohort + cohort_id = keys[0] + for key in keys: + self._key_to_id_map[key] = cohort_id + mh = self._id_to_mh_map[old_cohort_id] + self._id_to_mh_map[cohort_id] = deepcopy(mh) + self._id_to_mh_map[cohort_id].update_model_path(updated_path) + model.update_model_handler(cohort_id, updated_path, old_cohort_id) + model_id = key_modelid_mapping[cohort_id] + self._metrics_collectors[cohort_id] = _MetricsCollector( + self._metrics_namespace, f'{cohort_id}-{model_id}-') def update_model_path(self, model_path: Optional[str] = None): - return self._unkeyed.update_model_path(model_path=model_path) + if self._single_model: + return self._unkeyed.update_model_path(model_path=model_path) + if model_path is not None: + raise RuntimeError( + 'Model updates are currently not supported for ' + + 'KeyedModelHandlers with multiple different per-key ' + + 'ModelHandlers.') - def get_preprocess_fns(self) -> Iterable[Callable[[Any], Any]]: - return self._unkeyed.get_preprocess_fns() + def share_model_across_processes(self) -> bool: + if self._single_model: + return self._unkeyed.share_model_across_processes() + return True - def get_postprocess_fns(self) -> Iterable[Callable[[Any], Any]]: - return self._unkeyed.get_postprocess_fns() + def override_metrics(self, metrics_namespace: str = '') -> bool: + if self._single_model: + return self._unkeyed.override_metrics(metrics_namespace) - def share_model_across_processes(self) -> bool: - return self._unkeyed.share_model_across_processes() + self._metrics_namespace = metrics_namespace + self._default_metrics_collector = _MetricsCollector(metrics_namespace) + for cohort_id in self._id_to_mh_map: + self._metrics_collectors[cohort_id] = _MetricsCollector( + metrics_namespace, f'{cohort_id}-') + + return True class MaybeKeyedModelHandler(Generic[KeyT, ExampleT, PredictionT, ModelT], @@ -554,7 +1032,6 @@ def __init__( self._clock = clock self._metrics_namespace = metrics_namespace self._model_metadata_pcoll = model_metadata_pcoll - self._enable_side_input_loading = self._model_metadata_pcoll is not None self._with_exception_handling = False self._watch_model_pattern = watch_model_pattern self._kwargs = kwargs @@ -562,6 +1039,15 @@ def __init__( # allow us to effectively disambiguate in multi-model settings. self._model_tag = uuid.uuid4().hex + def annotations(self): + return { + 'model_handler': str(self._model_handler), + 'model_handler_type': ( + f'{self._model_handler.__class__.__module__}' + f'.{self._model_handler.__class__.__qualname__}'), + **super().annotations() + } + def _get_model_metadata_pcoll(self, pipeline): # avoid circular imports. # pylint: disable=wrong-import-position @@ -646,12 +1132,12 @@ def expand( self._model_handler, self._clock, self._metrics_namespace, - self._enable_side_input_loading, + self._model_metadata_pcoll is not None, self._model_tag), self._inference_args, beam.pvalue.AsSingleton( self._model_metadata_pcoll, - ) if self._enable_side_input_loading else None).with_resource_hints( + ) if self._model_metadata_pcoll else None).with_resource_hints( **resource_hints) if self._with_exception_handling: @@ -740,7 +1226,9 @@ def with_exception_handling( class _MetricsCollector: - """A metrics collector that tracks ML related performance and memory usage.""" + """ + A metrics collector that tracks ML related performance and memory usage. + """ def __init__(self, namespace: str, prefix: str = ''): """ Args: @@ -787,6 +1275,10 @@ def cache_load_model_metrics(self, load_model_latency_ms, model_byte_size): self._load_model_latency_milli_secs_cache = load_model_latency_ms self._model_byte_size_cache = model_byte_size + def update_load_model_metrics(self, load_model_latency_ms, model_byte_size): + self._load_model_latency_milli_secs.update(load_model_latency_ms) + self._model_byte_size.update(model_byte_size) + def update( self, examples_count: int, @@ -825,34 +1317,47 @@ def __init__( self._side_input_path = None self._model_tag = model_tag - def _load_model(self, side_input_model_path: Optional[str] = None): + def _load_model( + self, + side_input_model_path: Optional[Union[str, + List[KeyModelPathMapping]]] = None): def load(): """Function for constructing shared LoadedModel.""" memory_before = _get_current_process_memory_in_bytes() start_time = _to_milliseconds(self._clock.time_ns()) - self._model_handler.update_model_path(side_input_model_path) + if isinstance(side_input_model_path, str): + self._model_handler.update_model_path(side_input_model_path) + else: + self._model_handler.update_model_paths( + self._model, side_input_model_path) model = self._model_handler.load_model() end_time = _to_milliseconds(self._clock.time_ns()) memory_after = _get_current_process_memory_in_bytes() load_model_latency_ms = end_time - start_time model_byte_size = memory_after - memory_before - self._metrics_collector.cache_load_model_metrics( - load_model_latency_ms, model_byte_size) + if self._metrics_collector: + self._metrics_collector.cache_load_model_metrics( + load_model_latency_ms, model_byte_size) return model # TODO(https://github.com/apache/beam/issues/21443): Investigate releasing # model. + model_tag = self._model_tag + if isinstance(side_input_model_path, str) and side_input_model_path != '': + model_tag = side_input_model_path if self._model_handler.share_model_across_processes(): model = multi_process_shared.MultiProcessShared( - load, tag=side_input_model_path or self._model_tag).acquire() + load, tag=model_tag, always_proxy=True).acquire() else: - model = self._shared_model_handle.acquire( - load, tag=side_input_model_path or self._model_tag) + model = self._shared_model_handle.acquire(load, tag=model_tag) # since shared_model_handle is shared across threads, the model path # might not get updated in the model handler # because we directly get cached weak ref model from shared cache, instead # of calling load(). For sanity check, call update_model_path again. - self._model_handler.update_model_path(side_input_model_path) + if isinstance(side_input_model_path, str): + self._model_handler.update_model_path(side_input_model_path) + else: + self._model_handler.update_model_paths(self._model, side_input_model_path) return model def get_metrics_collector(self, prefix: str = ''): @@ -864,6 +1369,8 @@ def get_metrics_collector(self, prefix: str = ''): metrics_namespace = ( self._metrics_namespace) if self._metrics_namespace else ( self._model_handler.get_metrics_namespace()) + if self._model_handler.override_metrics(metrics_namespace): + return None return _MetricsCollector(metrics_namespace, prefix=prefix) def setup(self): @@ -872,7 +1379,10 @@ def setup(self): if not self._enable_side_input_loading: self._model = self._load_model() - def update_model(self, side_input_model_path: Optional[str] = None): + def update_model( + self, + side_input_model_path: Optional[Union[str, + List[KeyModelPathMapping]]] = None): self._model = self._load_model(side_input_model_path=side_input_model_path) def _run_inference(self, batch, inference_args): @@ -881,7 +1391,8 @@ def _run_inference(self, batch, inference_args): result_generator = self._model_handler.run_inference( batch, self._model, inference_args) except BaseException as e: - self._metrics_collector.failed_batches_counter.inc() + if self._metrics_collector: + self._metrics_collector.failed_batches_counter.inc() raise e predictions = list(result_generator) @@ -889,12 +1400,18 @@ def _run_inference(self, batch, inference_args): inference_latency = end_time - start_time num_bytes = self._model_handler.get_num_bytes(batch) num_elements = len(batch) - self._metrics_collector.update(num_elements, num_bytes, inference_latency) + if self._metrics_collector: + self._metrics_collector.update(num_elements, num_bytes, inference_latency) return predictions def process( - self, batch, inference_args, si_model_metadata: Optional[ModelMetadata]): + self, + batch, + inference_args, + si_model_metadata: Optional[Union[ModelMetadata, + List[ModelMetadata], + List[KeyModelPathMapping]]]): """ When side input is enabled: The method checks if the side input model has been updated, and if so, @@ -902,23 +1419,30 @@ def process( side input is empty or the model has not been updated, the method simply runs inference on the batch of data. """ - if si_model_metadata: - if isinstance(si_model_metadata, beam.pvalue.EmptySideInput): - self.update_model(side_input_model_path=None) + if not si_model_metadata: + return self._run_inference(batch, inference_args) + + if isinstance(si_model_metadata, beam.pvalue.EmptySideInput): + self.update_model(side_input_model_path=None) + elif isinstance(si_model_metadata, List) and hasattr(si_model_metadata[0], + 'keys'): + # TODO(https://github.com/apache/beam/issues/27628): Update metrics here + self.update_model(si_model_metadata) + elif self._side_input_path != si_model_metadata.model_id: + self._side_input_path = si_model_metadata.model_id + self._metrics_collector = self.get_metrics_collector( + prefix=si_model_metadata.model_name) + with threading.Lock(): + self.update_model(si_model_metadata.model_id) return self._run_inference(batch, inference_args) - elif self._side_input_path != si_model_metadata.model_id: - self._side_input_path = si_model_metadata.model_id - self._metrics_collector = self.get_metrics_collector( - prefix=si_model_metadata.model_name) - with threading.Lock(): - self.update_model(si_model_metadata.model_id) - return self._run_inference(batch, inference_args) + return self._run_inference(batch, inference_args) def finish_bundle(self): # TODO(https://github.com/apache/beam/issues/21435): Figure out why there # is a cache. - self._metrics_collector.update_metrics_with_cache() + if self._metrics_collector: + self._metrics_collector.update_metrics_with_cache() def _is_darwin() -> bool: diff --git a/sdks/python/apache_beam/ml/inference/base_test.py b/sdks/python/apache_beam/ml/inference/base_test.py index 635c3cec4ff49..7075810ff0f09 100644 --- a/sdks/python/apache_beam/ml/inference/base_test.py +++ b/sdks/python/apache_beam/ml/inference/base_test.py @@ -27,6 +27,7 @@ from typing import Mapping from typing import Optional from typing import Sequence +from typing import Union import pytest @@ -41,6 +42,7 @@ from apache_beam.transforms import trigger from apache_beam.transforms import window from apache_beam.transforms.periodicsequence import TimestampedValue +from apache_beam.utils import multi_process_shared class FakeModel: @@ -48,6 +50,19 @@ def predict(self, example: int) -> int: return example + 1 +class FakeStatefulModel: + def __init__(self, state: int): + if state == 100: + raise Exception('Oh no') + self._state = state + + def predict(self, example: int) -> int: + return self._state + + def increment_state(self, amount: int): + self._state += amount + + class FakeModelHandler(base.ModelHandler[int, int, FakeModel]): def __init__( self, @@ -55,16 +70,22 @@ def __init__( min_batch_size=1, max_batch_size=9999, multi_process_shared=False, + state=None, + num_bytes_per_element=None, **kwargs): self._fake_clock = clock self._min_batch_size = min_batch_size self._max_batch_size = max_batch_size self._env_vars = kwargs.get('env_vars', {}) self._multi_process_shared = multi_process_shared + self._state = state + self._num_bytes_per_element = num_bytes_per_element def load_model(self): if self._fake_clock: self._fake_clock.current_time_ns += 500_000_000 # 500ms + if self._state is not None: + return FakeStatefulModel(self._state) return FakeModel() def run_inference( @@ -95,6 +116,11 @@ def batch_elements_kwargs(self): def share_model_across_processes(self): return self._multi_process_shared + def get_num_bytes(self, batch: Sequence[int]) -> int: + if self._num_bytes_per_element: + return self._num_bytes_per_element * len(batch) + return super().get_num_bytes(batch) + class FakeModelHandlerReturnsPredictionResult( base.ModelHandler[int, base.PredictionResult, FakeModel]): @@ -102,19 +128,23 @@ def __init__( self, clock=None, model_id='fake_model_id_default', - multi_process_shared=False): + multi_process_shared=False, + state=None): self.model_id = model_id self._fake_clock = clock self._env_vars = {} self._multi_process_shared = multi_process_shared + self._state = state def load_model(self): + if self._state is not None: + return FakeStatefulModel(0) return FakeModel() def run_inference( self, batch: Sequence[int], - model: FakeModel, + model: Union[FakeModel, FakeStatefulModel], inference_args=None) -> Iterable[base.PredictionResult]: multi_process_shared_loaded = "multi_process_shared" in str(type(model)) if self._multi_process_shared != multi_process_shared_loaded: @@ -127,6 +157,8 @@ def run_inference( model_id=self.model_id, example=example, inference=model.predict(example)) + if self._state is not None: + model.increment_state(1) # type: ignore[union-attr] def update_model_path(self, model_path: Optional[str] = None): self.model_id = model_path if model_path else self.model_id @@ -236,6 +268,186 @@ def test_run_inference_impl_with_keyed_examples(self): base.KeyedModelHandler(FakeModelHandler())) assert_that(actual, equal_to(expected), label='assert:inferences') + def test_run_inference_impl_with_keyed_examples_many_model_handlers(self): + with TestPipeline() as pipeline: + examples = [1, 5, 3, 10] + keyed_examples = [(i, example) for i, example in enumerate(examples)] + expected = [(i, example + 1) for i, example in enumerate(examples)] + expected[0] = (0, 200) + pcoll = pipeline | 'start' >> beam.Create(keyed_examples) + mhs = [ + base.KeyModelMapping([0], + FakeModelHandler( + state=200, multi_process_shared=True)), + base.KeyModelMapping([1, 2, 3], + FakeModelHandler(multi_process_shared=True)) + ] + actual = pcoll | base.RunInference(base.KeyedModelHandler(mhs)) + assert_that(actual, equal_to(expected), label='assert:inferences') + + def test_run_inference_impl_with_keyed_examples_many_model_handlers_metrics( + self): + pipeline = TestPipeline() + examples = [1, 5, 3, 10] + metrics_namespace = 'test_namespace' + keyed_examples = [(i, example) for i, example in enumerate(examples)] + pcoll = pipeline | 'start' >> beam.Create(keyed_examples) + mhs = [ + base.KeyModelMapping([0], + FakeModelHandler( + state=200, multi_process_shared=True)), + base.KeyModelMapping([1, 2, 3], + FakeModelHandler(multi_process_shared=True)) + ] + _ = pcoll | base.RunInference( + base.KeyedModelHandler(mhs), metrics_namespace=metrics_namespace) + result = pipeline.run() + result.wait_until_finish() + + metrics_filter = MetricsFilter().with_namespace(namespace=metrics_namespace) + metrics = result.metrics().query(metrics_filter) + assert len(metrics['counters']) != 0 + assert len(metrics['distributions']) != 0 + + metrics_filter = MetricsFilter().with_name('0-_num_inferences') + metrics = result.metrics().query(metrics_filter) + num_inferences_counter_key_0 = metrics['counters'][0] + self.assertEqual(num_inferences_counter_key_0.committed, 1) + + metrics_filter = MetricsFilter().with_name('1-_num_inferences') + metrics = result.metrics().query(metrics_filter) + num_inferences_counter_key_1 = metrics['counters'][0] + self.assertEqual(num_inferences_counter_key_1.committed, 3) + + metrics_filter = MetricsFilter().with_name('num_inferences') + metrics = result.metrics().query(metrics_filter) + num_inferences_counter_aggregate = metrics['counters'][0] + self.assertEqual(num_inferences_counter_aggregate.committed, 4) + + metrics_filter = MetricsFilter().with_name('0-_failed_batches_counter') + metrics = result.metrics().query(metrics_filter) + failed_batches_counter_key_0 = metrics['counters'] + self.assertEqual(len(failed_batches_counter_key_0), 0) + + metrics_filter = MetricsFilter().with_name('failed_batches_counter') + metrics = result.metrics().query(metrics_filter) + failed_batches_counter_aggregate = metrics['counters'] + self.assertEqual(len(failed_batches_counter_aggregate), 0) + + metrics_filter = MetricsFilter().with_name( + '0-_load_model_latency_milli_secs') + metrics = result.metrics().query(metrics_filter) + load_latency_dist_key_0 = metrics['distributions'][0] + self.assertEqual(load_latency_dist_key_0.committed.count, 1) + + metrics_filter = MetricsFilter().with_name('load_model_latency_milli_secs') + metrics = result.metrics().query(metrics_filter) + load_latency_dist_aggregate = metrics['distributions'][0] + self.assertEqual(load_latency_dist_aggregate.committed.count, 2) + + def test_run_inference_impl_with_keyed_examples_many_mhs_max_models_hint( + self): + pipeline = TestPipeline() + examples = [1, 5, 3, 10, 2, 4, 6, 8, 9, 7, 1, 5, 3, 10, 2, 4, 6, 8, 9, 7] + metrics_namespace = 'test_namespace' + keyed_examples = [(i, example) for i, example in enumerate(examples)] + pcoll = pipeline | 'start' >> beam.Create(keyed_examples) + mhs = [ + base.KeyModelMapping([0, 2, 4, 6, 8], + FakeModelHandler( + state=200, multi_process_shared=True)), + base.KeyModelMapping( + [1, 3, 5, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], + FakeModelHandler(multi_process_shared=True)) + ] + _ = pcoll | base.RunInference( + base.KeyedModelHandler(mhs, max_models_per_worker_hint=1), + metrics_namespace=metrics_namespace) + result = pipeline.run() + result.wait_until_finish() + + metrics_filter = MetricsFilter().with_namespace(namespace=metrics_namespace) + metrics = result.metrics().query(metrics_filter) + assert len(metrics['counters']) != 0 + assert len(metrics['distributions']) != 0 + + metrics_filter = MetricsFilter().with_name('load_model_latency_milli_secs') + metrics = result.metrics().query(metrics_filter) + load_latency_dist_aggregate = metrics['distributions'][0] + # We should flip back and forth between models a bit since + # max_models_per_worker_hint=1, but we shouldn't thrash forever + # since most examples belong to the second ModelMapping + self.assertGreater(load_latency_dist_aggregate.committed.count, 2) + self.assertLess(load_latency_dist_aggregate.committed.count, 12) + + def test_keyed_many_model_handlers_validation(self): + def mult_two(example: str) -> int: + return int(example) * 2 + + mhs = [ + base.KeyModelMapping( + [0], + FakeModelHandler( + state=200, + multi_process_shared=True).with_preprocess_fn(mult_two)), + base.KeyModelMapping([1, 2, 3], + FakeModelHandler(multi_process_shared=True)) + ] + with self.assertRaises(ValueError): + base.KeyedModelHandler(mhs) + + mhs = [ + base.KeyModelMapping( + [0], + FakeModelHandler( + state=200, + multi_process_shared=True).with_postprocess_fn(mult_two)), + base.KeyModelMapping([1, 2, 3], + FakeModelHandler(multi_process_shared=True)) + ] + with self.assertRaises(ValueError): + base.KeyedModelHandler(mhs) + + mhs = [ + base.KeyModelMapping([0], + FakeModelHandler( + state=200, multi_process_shared=True)), + base.KeyModelMapping([0, 1, 2, 3], + FakeModelHandler(multi_process_shared=True)) + ] + with self.assertRaises(ValueError): + base.KeyedModelHandler(mhs) + + mhs = [ + base.KeyModelMapping([], + FakeModelHandler( + state=200, multi_process_shared=True)), + base.KeyModelMapping([0, 1, 2, 3], + FakeModelHandler(multi_process_shared=True)) + ] + with self.assertRaises(ValueError): + base.KeyedModelHandler(mhs) + + def test_keyed_model_handler_get_num_bytes(self): + mh = base.KeyedModelHandler(FakeModelHandler(num_bytes_per_element=10)) + batch = [('key1', 1), ('key2', 2), ('key1', 3)] + expected = len(pickle.dumps(('key1', 'key2', 'key1'))) + 30 + actual = mh.get_num_bytes(batch) + self.assertEqual(expected, actual) + + def test_keyed_model_handler_multiple_models_get_num_bytes(self): + mhs = [ + base.KeyModelMapping(['key1'], + FakeModelHandler(num_bytes_per_element=10)), + base.KeyModelMapping(['key2'], + FakeModelHandler(num_bytes_per_element=20)) + ] + mh = base.KeyedModelHandler(mhs) + batch = [('key1', 1), ('key2', 2), ('key1', 3)] + expected = len(pickle.dumps(('key1', 'key2', 'key1'))) + 40 + actual = mh.get_num_bytes(batch) + self.assertEqual(expected, actual) + def test_run_inference_impl_with_maybe_keyed_examples(self): with TestPipeline() as pipeline: examples = [1, 5, 3, 10] @@ -851,6 +1063,202 @@ def process(self, element): assert_that(result_pcoll, equal_to(expected_result)) + def test_run_inference_side_input_in_batch_per_key_models(self): + first_ts = math.floor(time.time()) - 30 + interval = 7 + + sample_main_input_elements = ([ + ('key1', first_ts - 2), + ('key2', first_ts + 1), + ('key2', first_ts + 8), + ('key1', first_ts + 15), + ('key2', first_ts + 22), + ('key1', first_ts + 29), + ]) + + sample_side_input_elements = [ + ( + first_ts + 1, + [ + base.KeyModelPathMapping( + keys=['key1'], update_path='fake_model_id_default'), + base.KeyModelPathMapping( + keys=['key2'], update_path='fake_model_id_default') + ]), + # if model_id is empty string, we use the default model + # handler model URI. + ( + first_ts + 8, + [ + base.KeyModelPathMapping( + keys=['key1'], update_path='fake_model_id_1'), + base.KeyModelPathMapping( + keys=['key2'], update_path='fake_model_id_default') + ]), + ( + first_ts + 15, + [ + base.KeyModelPathMapping( + keys=['key1'], update_path='fake_model_id_1'), + base.KeyModelPathMapping( + keys=['key2'], update_path='fake_model_id_2') + ]), + ] + + model_handler = base.KeyedModelHandler([ + base.KeyModelMapping(['key1'], + FakeModelHandlerReturnsPredictionResult( + multi_process_shared=True, state=True)), + base.KeyModelMapping(['key2'], + FakeModelHandlerReturnsPredictionResult( + multi_process_shared=True, state=True)) + ]) + + class _EmitElement(beam.DoFn): + def process(self, element): + for e in element: + yield e + + with TestPipeline() as pipeline: + side_input = ( + pipeline + | + "CreateSideInputElements" >> beam.Create(sample_side_input_elements) + | beam.Map(lambda x: TimestampedValue(x[1], x[0])) + | beam.WindowInto( + window.FixedWindows(interval), + accumulation_mode=trigger.AccumulationMode.DISCARDING) + | beam.Map(lambda x: ('key', x)) + | beam.GroupByKey() + | beam.Map(lambda x: x[1]) + | "EmitSideInput" >> beam.ParDo(_EmitElement())) + + result_pcoll = ( + pipeline + | beam.Create(sample_main_input_elements) + | "MapTimeStamp" >> beam.Map(lambda x: TimestampedValue(x, x[1])) + | "ApplyWindow" >> beam.WindowInto(window.FixedWindows(interval)) + | "RunInference" >> base.RunInference( + model_handler, model_metadata_pcoll=side_input) + | beam.Map(lambda x: x[1])) + + expected_model_id_order = [ + 'fake_model_id_default', + 'fake_model_id_default', + 'fake_model_id_default', + 'fake_model_id_1', + 'fake_model_id_2', + 'fake_model_id_1', + ] + + expected_inferences = [ + 0, + 0, + 1, + 0, + 0, + 1, + ] + + expected_result = [ + base.PredictionResult( + example=sample_main_input_elements[i][1], + inference=expected_inferences[i], + model_id=expected_model_id_order[i]) + for i in range(len(expected_inferences)) + ] + + assert_that(result_pcoll, equal_to(expected_result)) + + def test_run_inference_side_input_in_batch_per_key_models_split_cohort(self): + first_ts = math.floor(time.time()) - 30 + interval = 7 + + sample_main_input_elements = ([ + ('key1', first_ts - 2), + ('key2', first_ts + 1), + ('key1', first_ts + 8), + ('key2', first_ts + 15), + ('key1', first_ts + 22), + ]) + + sample_side_input_elements = [ + ( + first_ts + 1, + [ + base.KeyModelPathMapping( + keys=['key1', 'key2'], update_path='fake_model_id_default') + ]), + # if model_id is empty string, we use the default model + # handler model URI. + ( + first_ts + 8, + [ + base.KeyModelPathMapping( + keys=['key1'], update_path='fake_model_id_1'), + base.KeyModelPathMapping( + keys=['key2'], update_path='fake_model_id_default') + ]), + ( + first_ts + 15, + [ + base.KeyModelPathMapping( + keys=['key1'], update_path='fake_model_id_1'), + base.KeyModelPathMapping( + keys=['key2'], update_path='fake_model_id_2') + ]), + ] + + model_handler = base.KeyedModelHandler([ + base.KeyModelMapping( + ['key1', 'key2'], + FakeModelHandlerReturnsPredictionResult(multi_process_shared=True)) + ]) + + class _EmitElement(beam.DoFn): + def process(self, element): + for e in element: + yield e + + with TestPipeline() as pipeline: + side_input = ( + pipeline + | + "CreateSideInputElements" >> beam.Create(sample_side_input_elements) + | beam.Map(lambda x: TimestampedValue(x[1], x[0])) + | beam.WindowInto( + window.FixedWindows(interval), + accumulation_mode=trigger.AccumulationMode.DISCARDING) + | beam.Map(lambda x: ('key', x)) + | beam.GroupByKey() + | beam.Map(lambda x: x[1]) + | "EmitSideInput" >> beam.ParDo(_EmitElement())) + + result_pcoll = ( + pipeline + | beam.Create(sample_main_input_elements) + | "MapTimeStamp" >> beam.Map(lambda x: TimestampedValue(x, x[1])) + | "ApplyWindow" >> beam.WindowInto(window.FixedWindows(interval)) + | "RunInference" >> base.RunInference( + model_handler, model_metadata_pcoll=side_input) + | beam.Map(lambda x: x[1])) + + expected_model_id_order = [ + 'fake_model_id_default', + 'fake_model_id_default', + 'fake_model_id_1', + 'fake_model_id_2', + 'fake_model_id_1' + ] + expected_result = [ + base.PredictionResult( + example=sample_main_input_elements[i][1], + inference=sample_main_input_elements[i][1] + 1, + model_id=expected_model_id_order[i]) for i in range(5) + ] + + assert_that(result_pcoll, equal_to(expected_result)) + def test_run_inference_side_input_in_batch_multi_process_shared(self): first_ts = math.floor(time.time()) - 30 interval = 7 @@ -964,6 +1372,169 @@ def test_child_class_without_env_vars(self): actual = pcoll | base.RunInference(FakeModelHandlerNoEnvVars()) assert_that(actual, equal_to(expected), label='assert:inferences') + def test_model_manager_loads_shared_model(self): + mhs = { + 'key1': FakeModelHandler(state=1), + 'key2': FakeModelHandler(state=2), + 'key3': FakeModelHandler(state=3) + } + mm = base._ModelManager(mh_map=mhs) + tag1 = mm.load('key1').model_tag + # Use bad_mh's load function to make sure we're actually loading the + # version already stored + bad_mh = FakeModelHandler(state=100) + model1 = multi_process_shared.MultiProcessShared( + bad_mh.load_model, tag=tag1).acquire() + self.assertEqual(1, model1.predict(10)) + + tag2 = mm.load('key2').model_tag + tag3 = mm.load('key3').model_tag + model2 = multi_process_shared.MultiProcessShared( + bad_mh.load_model, tag=tag2).acquire() + model3 = multi_process_shared.MultiProcessShared( + bad_mh.load_model, tag=tag3).acquire() + self.assertEqual(2, model2.predict(10)) + self.assertEqual(3, model3.predict(10)) + + def test_model_manager_evicts_models(self): + mh1 = FakeModelHandler(state=1) + mh2 = FakeModelHandler(state=2) + mh3 = FakeModelHandler(state=3) + mhs = {'key1': mh1, 'key2': mh2, 'key3': mh3} + mm = base._ModelManager(mh_map=mhs) + mm.increment_max_models(2) + tag1 = mm.load('key1').model_tag + sh1 = multi_process_shared.MultiProcessShared(mh1.load_model, tag=tag1) + model1 = sh1.acquire() + self.assertEqual(1, model1.predict(10)) + model1.increment_state(5) + + tag2 = mm.load('key2').model_tag + tag3 = mm.load('key3').model_tag + sh2 = multi_process_shared.MultiProcessShared(mh2.load_model, tag=tag2) + model2 = sh2.acquire() + sh3 = multi_process_shared.MultiProcessShared(mh3.load_model, tag=tag3) + model3 = sh3.acquire() + model2.increment_state(5) + model3.increment_state(5) + self.assertEqual(7, model2.predict(10)) + self.assertEqual(8, model3.predict(10)) + sh2.release(model2) + sh3.release(model3) + + # model1 should have retained a valid reference to the model until released + self.assertEqual(6, model1.predict(10)) + sh1.release(model1) + + # This should now have been garbage collected, so it now it should get + # recreated and shouldn't have the state updates + model1 = multi_process_shared.MultiProcessShared( + mh1.load_model, tag=tag1).acquire() + self.assertEqual(1, model1.predict(10)) + + # These should not get recreated, so they should have the state updates + model2 = multi_process_shared.MultiProcessShared( + mh2.load_model, tag=tag2).acquire() + self.assertEqual(7, model2.predict(10)) + model3 = multi_process_shared.MultiProcessShared( + mh3.load_model, tag=tag3).acquire() + self.assertEqual(8, model3.predict(10)) + + def test_model_manager_evicts_models_after_update(self): + mh1 = FakeModelHandler(state=1) + mhs = {'key1': mh1} + mm = base._ModelManager(mh_map=mhs) + tag1 = mm.load('key1').model_tag + sh1 = multi_process_shared.MultiProcessShared(mh1.load_model, tag=tag1) + model1 = sh1.acquire() + self.assertEqual(1, model1.predict(10)) + model1.increment_state(5) + self.assertEqual(6, model1.predict(10)) + mm.update_model_handler('key1', 'fake/path', 'key1') + self.assertEqual(6, model1.predict(10)) + sh1.release(model1) + + tag1 = mm.load('key1').model_tag + sh1 = multi_process_shared.MultiProcessShared(mh1.load_model, tag=tag1) + model1 = sh1.acquire() + self.assertEqual(1, model1.predict(10)) + model1.increment_state(5) + self.assertEqual(6, model1.predict(10)) + sh1.release(model1) + + # Shouldn't evict if path is the same as last update + mm.update_model_handler('key1', 'fake/path', 'key1') + tag1 = mm.load('key1').model_tag + sh1 = multi_process_shared.MultiProcessShared(mh1.load_model, tag=tag1) + model1 = sh1.acquire() + self.assertEqual(6, model1.predict(10)) + sh1.release(model1) + + def test_model_manager_evicts_correct_num_of_models_after_being_incremented( + self): + mh1 = FakeModelHandler(state=1) + mh2 = FakeModelHandler(state=2) + mh3 = FakeModelHandler(state=3) + mhs = {'key1': mh1, 'key2': mh2, 'key3': mh3} + mm = base._ModelManager(mh_map=mhs) + mm.increment_max_models(1) + mm.increment_max_models(1) + tag1 = mm.load('key1').model_tag + sh1 = multi_process_shared.MultiProcessShared(mh1.load_model, tag=tag1) + model1 = sh1.acquire() + self.assertEqual(1, model1.predict(10)) + model1.increment_state(5) + self.assertEqual(6, model1.predict(10)) + sh1.release(model1) + + tag2 = mm.load('key2').model_tag + tag3 = mm.load('key3').model_tag + sh2 = multi_process_shared.MultiProcessShared(mh2.load_model, tag=tag2) + model2 = sh2.acquire() + sh3 = multi_process_shared.MultiProcessShared(mh3.load_model, tag=tag3) + model3 = sh3.acquire() + model2.increment_state(5) + model3.increment_state(5) + self.assertEqual(7, model2.predict(10)) + self.assertEqual(8, model3.predict(10)) + sh2.release(model2) + sh3.release(model3) + + # This should get recreated, so it shouldn't have the state updates + model1 = multi_process_shared.MultiProcessShared( + mh1.load_model, tag=tag1).acquire() + self.assertEqual(1, model1.predict(10)) + + # These should not get recreated, so they should have the state updates + model2 = multi_process_shared.MultiProcessShared( + mh2.load_model, tag=tag2).acquire() + self.assertEqual(7, model2.predict(10)) + model3 = multi_process_shared.MultiProcessShared( + mh3.load_model, tag=tag3).acquire() + self.assertEqual(8, model3.predict(10)) + + def test_run_inference_watch_file_pattern_side_input_label(self): + pipeline = TestPipeline() + # label of the WatchPattern transform. + side_input_str = 'WatchFilePattern/ApplyGlobalWindow' + from apache_beam.ml.inference.utils import WatchFilePattern + file_pattern_side_input = ( + pipeline + | 'WatchFilePattern' >> WatchFilePattern(file_pattern='fake/path/*')) + pcoll = pipeline | 'start' >> beam.Create([1, 2, 3]) + result_pcoll = pcoll | base.RunInference( + FakeModelHandler(), model_metadata_pcoll=file_pattern_side_input) + assert side_input_str in str(result_pcoll.producer.side_inputs[0]) + + def test_run_inference_watch_file_pattern_keyword_arg_side_input_label(self): + # label of the WatchPattern transform. + side_input_str = 'WatchFilePattern/ApplyGlobalWindow' + pipeline = TestPipeline() + pcoll = pipeline | 'start' >> beam.Create([1, 2, 3]) + result_pcoll = pcoll | base.RunInference( + FakeModelHandler(), watch_model_pattern='fake/path/*') + assert side_input_str in str(result_pcoll.producer.side_inputs[0]) + if __name__ == '__main__': unittest.main() diff --git a/sdks/python/apache_beam/ml/inference/huggingface_inference.py b/sdks/python/apache_beam/ml/inference/huggingface_inference.py new file mode 100644 index 0000000000000..3ec063808ae32 --- /dev/null +++ b/sdks/python/apache_beam/ml/inference/huggingface_inference.py @@ -0,0 +1,700 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pytype: skip-file + +import logging +import sys +from collections import defaultdict +from enum import Enum +from typing import Any +from typing import Callable +from typing import Dict +from typing import Iterable +from typing import Optional +from typing import Sequence +from typing import Union + +import tensorflow as tf +import torch +from apache_beam.ml.inference import utils +from apache_beam.ml.inference.base import ModelHandler +from apache_beam.ml.inference.base import PredictionResult +from apache_beam.ml.inference.pytorch_inference import _convert_to_device +from transformers import AutoModel +from transformers import Pipeline +from transformers import TFAutoModel +from transformers import pipeline + +_LOGGER = logging.getLogger(__name__) + +__all__ = [ + "HuggingFaceModelHandlerTensor", + "HuggingFaceModelHandlerKeyedTensor", + "HuggingFacePipelineModelHandler", +] + +TensorInferenceFn = Callable[[ + Sequence[Union[torch.Tensor, tf.Tensor]], + Union[AutoModel, TFAutoModel], + str, + Optional[Dict[str, Any]], + Optional[str], +], + Iterable[PredictionResult], + ] + +KeyedTensorInferenceFn = Callable[[ + Sequence[Dict[str, Union[torch.Tensor, tf.Tensor]]], + Union[AutoModel, TFAutoModel], + str, + Optional[Dict[str, Any]], + Optional[str] +], + Iterable[PredictionResult]] + +PipelineInferenceFn = Callable[ + [Sequence[str], Pipeline, Optional[Dict[str, Any]]], + Iterable[PredictionResult]] + + +class PipelineTask(str, Enum): + """ + PipelineTask defines all the tasks supported by the Hugging Face Pipelines + listed at https://huggingface.co/docs/transformers/main_classes/pipelines. + Only these tasks can be passed to HuggingFacePipelineModelHandler. + """ + AudioClassification = 'audio-classification' + AutomaticSpeechRecognition = 'automatic-speech-recognition' + Conversational = 'conversational' + DepthEstimation = 'depth-estimation' + DocumentQuestionAnswering = 'document-question-answering' + FeatureExtraction = 'feature-extraction' + FillMask = 'fill-mask' + ImageClassification = 'image-classification' + ImageSegmentation = 'image-segmentation' + ImageToText = 'image-to-text' + MaskGeneration = 'mask-generation' + NER = 'ner' + ObjectDetection = 'object-detection' + QuestionAnswering = 'question-answering' + SentimentAnalysis = 'sentiment-analysis' + Summarization = 'summarization' + TableQuestionAnswering = 'table-question-answering' + TextClassification = 'text-classification' + TextGeneration = 'text-generation' + Text2TextGeneration = 'text2text-generation' + TextToAudio = 'text-to-audio' + TokenClassification = 'token-classification' + Translation = 'translation' + VideoClassification = 'video-classification' + VisualQuestionAnswering = 'visual-question-answering' + VQA = 'vqa' + ZeroShotAudioClassification = 'zero-shot-audio-classification' + ZeroShotClassification = 'zero-shot-classification' + ZeroShotImageClassification = 'zero-shot-image-classification' + ZeroShotObjectDetection = 'zero-shot-object-detection' + Translation_XX_to_YY = 'translation_XX_to_YY' + + +def _validate_constructor_args(model_uri, model_class): + message = ( + "Please provide both model class and model uri to load the model." + "Got params as model_uri={model_uri} and " + "model_class={model_class}.") + if not model_uri and not model_class: + raise RuntimeError( + message.format(model_uri=model_uri, model_class=model_class)) + elif not model_uri: + raise RuntimeError( + message.format(model_uri=model_uri, model_class=model_class)) + elif not model_class: + raise RuntimeError( + message.format(model_uri=model_uri, model_class=model_class)) + + +def no_gpu_available_warning(): + _LOGGER.warning( + "HuggingFaceModelHandler specified a 'GPU' device, " + "but GPUs are not available. Switching to CPU.") + + +def is_gpu_available_torch(): + if torch.cuda.is_available(): + return True + else: + no_gpu_available_warning() + return False + + +def get_device_torch(device): + if device == "GPU" and is_gpu_available_torch(): + return torch.device("cuda") + return torch.device("cpu") + + +def is_gpu_available_tensorflow(device): + gpu_devices = tf.config.list_physical_devices(device) + if len(gpu_devices) == 0: + no_gpu_available_warning() + return False + return True + + +def _validate_constructor_args_hf_pipeline(task, model): + if not task and not model: + raise RuntimeError( + 'Please provide either task or model to the ' + 'HuggingFacePipelineModelHandler. If the model already defines the ' + 'task, no need to specify the task.') + + +def _run_inference_torch_keyed_tensor( + batch: Sequence[Dict[str, torch.Tensor]], + model: AutoModel, + device, + inference_args: Dict[str, Any], + model_id: Optional[str] = None) -> Iterable[PredictionResult]: + device = get_device_torch(device) + key_to_tensor_list = defaultdict(list) + # torch.no_grad() mitigates GPU memory issues + # https://github.com/apache/beam/issues/22811 + with torch.no_grad(): + for example in batch: + for key, tensor in example.items(): + key_to_tensor_list[key].append(tensor) + key_to_batched_tensors = {} + for key in key_to_tensor_list: + batched_tensors = torch.stack(key_to_tensor_list[key]) + batched_tensors = _convert_to_device(batched_tensors, device) + key_to_batched_tensors[key] = batched_tensors + predictions = model(**key_to_batched_tensors, **inference_args) + return utils._convert_to_result(batch, predictions, model_id) + + +def _run_inference_tensorflow_keyed_tensor( + batch: Sequence[Dict[str, tf.Tensor]], + model: TFAutoModel, + device, + inference_args: Dict[str, Any], + model_id: Optional[str] = None) -> Iterable[PredictionResult]: + if device == "GPU": + is_gpu_available_tensorflow(device) + key_to_tensor_list = defaultdict(list) + for example in batch: + for key, tensor in example.items(): + key_to_tensor_list[key].append(tensor) + key_to_batched_tensors = {} + for key in key_to_tensor_list: + batched_tensors = tf.stack(key_to_tensor_list[key], axis=0) + key_to_batched_tensors[key] = batched_tensors + predictions = model(**key_to_batched_tensors, **inference_args) + return utils._convert_to_result(batch, predictions, model_id) + + +class HuggingFaceModelHandlerKeyedTensor(ModelHandler[Dict[str, + Union[tf.Tensor, + torch.Tensor]], + PredictionResult, + Union[AutoModel, + TFAutoModel]]): + def __init__( + self, + model_uri: str, + model_class: Union[AutoModel, TFAutoModel], + framework: str, + device: str = "CPU", + *, + inference_fn: Optional[Callable[..., Iterable[PredictionResult]]] = None, + load_model_args: Optional[Dict[str, Any]] = None, + inference_args: Optional[Dict[str, Any]] = None, + min_batch_size: Optional[int] = None, + max_batch_size: Optional[int] = None, + large_model: bool = False, + **kwargs): + """ + Implementation of the ModelHandler interface for HuggingFace with + Keyed Tensors for PyTorch/Tensorflow backend. + + Example Usage model:: + pcoll | RunInference(HuggingFaceModelHandlerKeyedTensor( + model_uri="bert-base-uncased", model_class=AutoModelForMaskedLM, + framework='pt')) + + Args: + model_uri (str): path to the pretrained model on the hugging face + models hub. + model_class: model class to load the repository from model_uri. + framework (str): Framework to use for the model. 'tf' for TensorFlow and + 'pt' for PyTorch. + device: For torch tensors, specify device on which you wish to + run the model. Defaults to CPU. + inference_fn: the inference function to use during RunInference. + Default is _run_inference_torch_keyed_tensor or + _run_inference_tensorflow_keyed_tensor depending on the input type. + load_model_args (Dict[str, Any]): (Optional) Keyword arguments to provide + load options while loading models from Hugging Face Hub. + Defaults to None. + inference_args (Dict[str, Any]): (Optional) Non-batchable arguments + required as inputs to the model's inference function. Unlike Tensors + in `batch`, these parameters will not be dynamically batched. + Defaults to None. + min_batch_size: the minimum batch size to use when batching inputs. + max_batch_size: the maximum batch size to use when batching inputs. + large_model: set to true if your model is large enough to run into + memory pressure if you load multiple copies. Given a model that + consumes N memory and a machine with W cores and M memory, you should + set this to True if N*W > M. + kwargs: 'env_vars' can be used to set environment variables + before loading the model. + + **Supported Versions:** HuggingFaceModelHandler supports + transformers>=4.18.0. + """ + self._model_uri = model_uri + self._model_class = model_class + self._device = device + self._inference_fn = inference_fn + self._model_config_args = load_model_args if load_model_args else {} + self._inference_args = inference_args if inference_args else {} + self._batching_kwargs = {} + self._env_vars = kwargs.get("env_vars", {}) + if min_batch_size is not None: + self._batching_kwargs["min_batch_size"] = min_batch_size + if max_batch_size is not None: + self._batching_kwargs["max_batch_size"] = max_batch_size + self._large_model = large_model + self._framework = framework + + _validate_constructor_args( + model_uri=self._model_uri, model_class=self._model_class) + + def load_model(self): + """Loads and initializes the model for processing.""" + model = self._model_class.from_pretrained( + self._model_uri, **self._model_config_args) + if self._framework == 'pt': + if self._device == "GPU" and is_gpu_available_torch: + model.to(torch.device("cuda")) + if callable(getattr(model, 'requires_grad_', None)): + model.requires_grad_(False) + return model + + def run_inference( + self, + batch: Sequence[Dict[str, Union[tf.Tensor, torch.Tensor]]], + model: Union[AutoModel, TFAutoModel], + inference_args: Optional[Dict[str, Any]] = None + ) -> Iterable[PredictionResult]: + """ + Runs inferences on a batch of Keyed Tensors and returns an Iterable of + Tensors Predictions. + + This method stacks the list of Tensors in a vectorized format to optimize + the inference call. + + Args: + batch: A sequence of Keyed Tensors. These Tensors should be batchable, + as this method will call `tf.stack()`/`torch.stack()` and pass in + batched Tensors with dimensions (batch_size, n_features, etc.) into + the model's predict() function. + model: A Tensorflow/PyTorch model. + inference_args: Non-batchable arguments required as inputs to the + model's inference function. Unlike Tensors in `batch`, + these parameters will not be dynamically batched. + Returns: + An Iterable of type PredictionResult. + """ + inference_args = {} if not inference_args else inference_args + + if self._inference_fn: + return self._inference_fn( + batch, model, self._device, inference_args, self._model_uri) + + if self._framework == "tf": + return _run_inference_tensorflow_keyed_tensor( + batch, model, self._device, inference_args, self._model_uri) + else: + return _run_inference_torch_keyed_tensor( + batch, model, self._device, inference_args, self._model_uri) + + def update_model_path(self, model_path: Optional[str] = None): + self._model_uri = model_path if model_path else self._model_uri + + def get_num_bytes( + self, batch: Sequence[Union[tf.Tensor, torch.Tensor]]) -> int: + """ + Returns: + The number of bytes of data for the Tensors batch. + """ + if self._framework == "tf": + return sum(sys.getsizeof(element) for element in batch) + else: + return sum( + (el.element_size() for tensor in batch for el in tensor.values())) + + def batch_elements_kwargs(self): + return self._batching_kwargs + + def share_model_across_processes(self) -> bool: + return self._large_model + + def get_metrics_namespace(self) -> str: + """ + Returns: + A namespace for metrics collected by the RunInference transform. + """ + return "BeamML_HuggingFaceModelHandler_KeyedTensor" + + +def _default_inference_fn_torch( + batch: Sequence[Union[tf.Tensor, torch.Tensor]], + model: Union[AutoModel, TFAutoModel], + device, + inference_args: Dict[str, Any], + model_id: Optional[str] = None) -> Iterable[PredictionResult]: + device = get_device_torch(device) + # torch.no_grad() mitigates GPU memory issues + # https://github.com/apache/beam/issues/22811 + with torch.no_grad(): + batched_tensors = torch.stack(batch) + batched_tensors = _convert_to_device(batched_tensors, device) + predictions = model(batched_tensors, **inference_args) + return utils._convert_to_result(batch, predictions, model_id) + + +def _default_inference_fn_tensorflow( + batch: Sequence[Union[tf.Tensor, torch.Tensor]], + model: Union[AutoModel, TFAutoModel], + device, + inference_args: Dict[str, Any], + model_id: Optional[str] = None) -> Iterable[PredictionResult]: + if device == "GPU": + is_gpu_available_tensorflow(device) + batched_tensors = tf.stack(batch, axis=0) + predictions = model(batched_tensors, **inference_args) + return utils._convert_to_result(batch, predictions, model_id) + + +class HuggingFaceModelHandlerTensor(ModelHandler[Union[tf.Tensor, torch.Tensor], + PredictionResult, + Union[AutoModel, + TFAutoModel]]): + def __init__( + self, + model_uri: str, + model_class: Union[AutoModel, TFAutoModel], + device: str = "CPU", + *, + inference_fn: Optional[Callable[..., Iterable[PredictionResult]]] = None, + load_model_args: Optional[Dict[str, Any]] = None, + inference_args: Optional[Dict[str, Any]] = None, + min_batch_size: Optional[int] = None, + max_batch_size: Optional[int] = None, + large_model: bool = False, + **kwargs): + """ + Implementation of the ModelHandler interface for HuggingFace with + Tensors for PyTorch/Tensorflow backend. + + Depending on the type of tensors, the model framework is determined + automatically. + + Example Usage model: + pcoll | RunInference(HuggingFaceModelHandlerTensor( + model_uri="bert-base-uncased", model_class=AutoModelForMaskedLM)) + + Args: + model_uri (str): path to the pretrained model on the hugging face + models hub. + model_class: model class to load the repository from model_uri. + device: For torch tensors, specify device on which you wish to + run the model. Defaults to CPU. + inference_fn: the inference function to use during RunInference. + Default is _run_inference_torch_keyed_tensor or + _run_inference_tensorflow_keyed_tensor depending on the input type. + load_model_args (Dict[str, Any]): (Optional) keyword arguments to provide + load options while loading models from Hugging Face Hub. + Defaults to None. + inference_args (Dict[str, Any]): (Optional) Non-batchable arguments + required as inputs to the model's inference function. Unlike Tensors + in `batch`, these parameters will not be dynamically batched. + Defaults to None. + min_batch_size: the minimum batch size to use when batching inputs. + max_batch_size: the maximum batch size to use when batching inputs. + large_model: set to true if your model is large enough to run into + memory pressure if you load multiple copies. Given a model that + consumes N memory and a machine with W cores and M memory, you should + set this to True if N*W > M. + kwargs: 'env_vars' can be used to set environment variables + before loading the model. + + **Supported Versions:** HuggingFaceModelHandler supports + transformers>=4.18.0. + """ + self._model_uri = model_uri + self._model_class = model_class + self._device = device + self._inference_fn = inference_fn + self._model_config_args = load_model_args if load_model_args else {} + self._inference_args = inference_args if inference_args else {} + self._batching_kwargs = {} + self._env_vars = kwargs.get("env_vars", {}) + if min_batch_size is not None: + self._batching_kwargs["min_batch_size"] = min_batch_size + if max_batch_size is not None: + self._batching_kwargs["max_batch_size"] = max_batch_size + self._large_model = large_model + self._framework = "" + + _validate_constructor_args( + model_uri=self._model_uri, model_class=self._model_class) + + def load_model(self): + """Loads and initializes the model for processing.""" + model = self._model_class.from_pretrained( + self._model_uri, **self._model_config_args) + if callable(getattr(model, 'requires_grad_', None)): + model.requires_grad_(False) + return model + + def run_inference( + self, + batch: Sequence[Union[tf.Tensor, torch.Tensor]], + model: Union[AutoModel, TFAutoModel], + inference_args: Optional[Dict[str, Any]] = None + ) -> Iterable[PredictionResult]: + """ + Runs inferences on a batch of Tensors and returns an Iterable of + Tensors Predictions. + + This method stacks the list of Tensors in a vectorized format to optimize + the inference call. + + Args: + batch: A sequence of Tensors. These Tensors should be batchable, as + this method will call `tf.stack()`/`torch.stack()` and pass in + batched Tensors with dimensions (batch_size, n_features, etc.) + into the model's predict() function. + model: A Tensorflow/PyTorch model. + inference_args (Dict[str, Any]): Non-batchable arguments required as + inputs to the model's inference function. Unlike Tensors in `batch`, + these parameters will not be dynamically batched. + + Returns: + An Iterable of type PredictionResult. + """ + inference_args = {} if not inference_args else inference_args + if not self._framework: + if isinstance(batch[0], tf.Tensor): + self._framework = "tf" + else: + self._framework = "pt" + + if (self._framework == "pt" and self._device == "GPU" and + is_gpu_available_torch()): + model.to(torch.device("cuda")) + + if self._inference_fn: + return self._inference_fn( + batch, model, inference_args, inference_args, self._model_uri) + + if self._framework == "tf": + return _default_inference_fn_tensorflow( + batch, model, self._device, inference_args, self._model_uri) + else: + return _default_inference_fn_torch( + batch, model, self._device, inference_args, self._model_uri) + + def update_model_path(self, model_path: Optional[str] = None): + self._model_uri = model_path if model_path else self._model_uri + + def get_num_bytes( + self, batch: Sequence[Union[tf.Tensor, torch.Tensor]]) -> int: + """ + Returns: + The number of bytes of data for the Tensors batch. + """ + if self._framework == "tf": + return sum(sys.getsizeof(element) for element in batch) + else: + return sum( + (el.element_size() for tensor in batch for el in tensor.values())) + + def batch_elements_kwargs(self): + return self._batching_kwargs + + def share_model_across_processes(self) -> bool: + return self._large_model + + def get_metrics_namespace(self) -> str: + """ + Returns: + A namespace for metrics collected by the RunInference transform. + """ + return 'BeamML_HuggingFaceModelHandler_Tensor' + + +def _convert_to_result( + batch: Iterable, + predictions: Union[Iterable, Dict[Any, Iterable]], + model_id: Optional[str] = None, +) -> Iterable[PredictionResult]: + return [ + PredictionResult(x, y, model_id) for x, y in zip(batch, [predictions]) + ] + + +def _default_pipeline_inference_fn( + batch, pipeline, inference_args) -> Iterable[PredictionResult]: + predicitons = pipeline(batch, **inference_args) + return predicitons + + +class HuggingFacePipelineModelHandler(ModelHandler[str, + PredictionResult, + Pipeline]): + def __init__( + self, + task: Union[str, PipelineTask] = "", + model: str = "", + *, + inference_fn: PipelineInferenceFn = _default_pipeline_inference_fn, + load_pipeline_args: Optional[Dict[str, Any]] = None, + inference_args: Optional[Dict[str, Any]] = None, + min_batch_size: Optional[int] = None, + max_batch_size: Optional[int] = None, + large_model: bool = False, + **kwargs): + """ + Implementation of the ModelHandler interface for Hugging Face Pipelines. + + **Note:** To specify which device to use (CPU/GPU), + use the load_pipeline_args with key-value as you would do in the usual + Hugging Face pipeline. Ex: load_pipeline_args={'device':0}) + + Example Usage model:: + pcoll | RunInference(HuggingFacePipelineModelHandler( + task="fill-mask")) + + Args: + task (str or enum.Enum): task supported by HuggingFace Pipelines. + Accepts a string task or an enum.Enum from PipelineTask. + model (str): path to the pretrained *model-id* on Hugging Face Models Hub + to use custom model for the chosen task. If the `model` already defines + the task then no need to specify the `task` parameter. + Use the *model-id* string instead of an actual model here. + Model-specific kwargs for `from_pretrained(..., **model_kwargs)` can be + specified with `model_kwargs` using `load_pipeline_args`. + + Example Usage:: + model_handler = HuggingFacePipelineModelHandler( + task="text-generation", model="meta-llama/Llama-2-7b-hf", + load_pipeline_args={'model_kwargs':{'quantization_map':config}}) + + inference_fn: the inference function to use during RunInference. + Default is _default_pipeline_inference_fn. + load_pipeline_args (Dict[str, Any]): keyword arguments to provide load + options while loading pipelines from Hugging Face. Defaults to None. + inference_args (Dict[str, Any]): Non-batchable arguments + required as inputs to the model's inference function. + Defaults to None. + min_batch_size: the minimum batch size to use when batching inputs. + max_batch_size: the maximum batch size to use when batching inputs. + large_model: set to true if your model is large enough to run into + memory pressure if you load multiple copies. Given a model that + consumes N memory and a machine with W cores and M memory, you should + set this to True if N*W > M. + kwargs: 'env_vars' can be used to set environment variables + before loading the model. + + **Supported Versions:** HuggingFacePipelineModelHandler supports + transformers>=4.18.0. + """ + self._task = task + self._model = model + self._inference_fn = inference_fn + self._load_pipeline_args = load_pipeline_args if load_pipeline_args else {} + self._inference_args = inference_args if inference_args else {} + self._batching_kwargs = {} + self._framework = "torch" + self._env_vars = kwargs.get('env_vars', {}) + if min_batch_size is not None: + self._batching_kwargs['min_batch_size'] = min_batch_size + if max_batch_size is not None: + self._batching_kwargs['max_batch_size'] = max_batch_size + self._large_model = large_model + _validate_constructor_args_hf_pipeline(self._task, self._model) + + def load_model(self): + """Loads and initializes the pipeline for processing.""" + return pipeline( + task=self._task, model=self._model, **self._load_pipeline_args) + + def run_inference( + self, + batch: Sequence[str], + pipeline: Pipeline, + inference_args: Optional[Dict[str, Any]] = None + ) -> Iterable[PredictionResult]: + """ + Runs inferences on a batch of examples passed as a string resource. + These can either be string sentences, or string path to images or + audio files. + + Args: + batch: A sequence of strings resources. + pipeline: A Hugging Face Pipeline. + inference_args: Non-batchable arguments required as inputs to the model's + inference function. + Returns: + An Iterable of type PredictionResult. + """ + inference_args = {} if not inference_args else inference_args + predictions = self._inference_fn(batch, pipeline, inference_args) + return _convert_to_result(batch, predictions) + + def update_model_path(self, model_path: Optional[str] = None): + """ + Updates the pretrained model used by the Hugging Face Pipeline task. + Make sure that the new model does the same task as initial model. + + Args: + model_path (str): (Optional) Path to the new trained model + from Hugging Face. Defaults to None. + """ + self._model = model_path if model_path else self._model + + def get_num_bytes(self, batch: Sequence[str]) -> int: + """ + Returns: + The number of bytes of input batch elements. + """ + return sum(sys.getsizeof(element) for element in batch) + + def batch_elements_kwargs(self): + return self._batching_kwargs + + def share_model_across_processes(self) -> bool: + return self._large_model + + def get_metrics_namespace(self) -> str: + """ + Returns: + A namespace for metrics collected by the RunInference transform. + """ + return 'BeamML_HuggingFacePipelineModelHandler' diff --git a/sdks/python/apache_beam/ml/inference/huggingface_inference_it_test.py b/sdks/python/apache_beam/ml/inference/huggingface_inference_it_test.py new file mode 100644 index 0000000000000..dd675d1935a75 --- /dev/null +++ b/sdks/python/apache_beam/ml/inference/huggingface_inference_it_test.py @@ -0,0 +1,148 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""End-to-End test for Hugging Face Inference""" + +import logging +import unittest +import uuid + +import pytest + +from apache_beam.io.filesystems import FileSystems +from apache_beam.testing.test_pipeline import TestPipeline + +try: + from apache_beam.examples.inference import huggingface_language_modeling + from apache_beam.examples.inference import huggingface_question_answering + from apache_beam.ml.inference import pytorch_inference_it_test +except ImportError: + raise unittest.SkipTest( + "transformers dependencies are not installed. " + "Check if transformers, torch, and tensorflow " + "is installed.") + + +@pytest.mark.uses_transformers +@pytest.mark.it_postcommit +@pytest.mark.timeout(1800) +class HuggingFaceInference(unittest.TestCase): + def test_hf_language_modeling(self): + test_pipeline = TestPipeline(is_integration_test=True) + # Path to text file containing some sentences + file_of_sentences = 'gs://apache-beam-ml/datasets/custom/hf_sentences.txt' + output_file_dir = 'gs://apache-beam-ml/testing/predictions' + output_file = '/'.join([output_file_dir, str(uuid.uuid4()), 'result.txt']) + + model_name = 'stevhliu/my_awesome_eli5_mlm_model' + + extra_opts = { + 'input': file_of_sentences, + 'output': output_file, + 'model_name': model_name, + } + huggingface_language_modeling.run( + test_pipeline.get_full_options_as_args(**extra_opts), + save_main_session=False) + + self.assertEqual(FileSystems().exists(output_file), True) + predictions = pytorch_inference_it_test.process_outputs( + filepath=output_file) + actuals_file = 'gs://apache-beam-ml/testing/expected_outputs/test_hf_run_inference_for_masked_lm_actuals.txt' # pylint: disable=line-too-long + actuals = pytorch_inference_it_test.process_outputs(filepath=actuals_file) + + predictions_dict = {} + for prediction in predictions: + text, predicted_text = prediction.split(';') + predictions_dict[text] = predicted_text.strip().lower() + + for actual in actuals: + text, actual_predicted_text = actual.split(';') + predicted_predicted_text = predictions_dict[text] + self.assertEqual(actual_predicted_text, predicted_predicted_text) + + def test_hf_language_modeling_large_model(self): + test_pipeline = TestPipeline(is_integration_test=True) + # Path to text file containing some sentences + file_of_sentences = 'gs://apache-beam-ml/datasets/custom/hf_sentences.txt' + output_file_dir = 'gs://apache-beam-ml/testing/predictions' + output_file = '/'.join([output_file_dir, str(uuid.uuid4()), 'result.txt']) + + model_name = 'stevhliu/my_awesome_eli5_mlm_model' + + extra_opts = { + 'input': file_of_sentences, + 'output': output_file, + 'model_name': model_name, + 'large_model': True, + } + huggingface_language_modeling.run( + test_pipeline.get_full_options_as_args(**extra_opts), + save_main_session=False) + + self.assertEqual(FileSystems().exists(output_file), True) + predictions = pytorch_inference_it_test.process_outputs( + filepath=output_file) + actuals_file = 'gs://apache-beam-ml/testing/expected_outputs/test_hf_run_inference_for_masked_lm_actuals.txt' # pylint: disable=line-too-long + actuals = pytorch_inference_it_test.process_outputs(filepath=actuals_file) + + predictions_dict = {} + for prediction in predictions: + text, predicted_text = prediction.split(';') + predictions_dict[text] = predicted_text.strip().lower() + + for actual in actuals: + text, actual_predicted_text = actual.split(';') + predicted_predicted_text = predictions_dict[text] + self.assertEqual(actual_predicted_text, predicted_predicted_text) + + def test_hf_pipeline(self): + test_pipeline = TestPipeline(is_integration_test=True) + # Path to text file containing some questions and context + input_file = 'gs://apache-beam-ml/datasets/custom/questions.txt' + output_file_dir = 'gs://apache-beam-ml/hf/testing/predictions' + output_file = '/'.join([output_file_dir, str(uuid.uuid4()), 'result.txt']) + extra_opts = { + 'input': input_file, + 'output': output_file, + 'revision': 'deedc3e42208524e0df3d9149d1f26aa6934f05f', + } + huggingface_question_answering.run( + test_pipeline.get_full_options_as_args(**extra_opts), + save_main_session=False) + self.assertEqual(FileSystems().exists(output_file), True) + predictions = pytorch_inference_it_test.process_outputs( + filepath=output_file) + actuals_file = ( + 'gs://apache-beam-ml/testing/expected_outputs/' + 'test_hf_pipeline_answers.txt') + actuals = pytorch_inference_it_test.process_outputs(filepath=actuals_file) + + predictions_dict = {} + for prediction in predictions: + text, predicted_text = prediction.split(';') + predictions_dict[text] = predicted_text.strip() + + for actual in actuals: + text, actual_predicted_text = actual.split(';') + predicted_predicted_text = predictions_dict[text] + self.assertEqual(actual_predicted_text, predicted_predicted_text) + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.DEBUG) + unittest.main() diff --git a/sdks/python/apache_beam/ml/inference/huggingface_inference_test.py b/sdks/python/apache_beam/ml/inference/huggingface_inference_test.py new file mode 100644 index 0000000000000..763d5ee8d36fc --- /dev/null +++ b/sdks/python/apache_beam/ml/inference/huggingface_inference_test.py @@ -0,0 +1,136 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pytype: skip-file + +import shutil +import tempfile +import unittest +from typing import Any +from typing import Dict +from typing import Iterable +from typing import Optional +from typing import Sequence +from typing import Union + +import pytest + +from apache_beam.ml.inference import utils +from apache_beam.ml.inference.base import PredictionResult +from apache_beam.ml.inference.tensorflow_inference_test import FakeTFTensorModel +from apache_beam.ml.inference.tensorflow_inference_test import _compare_tensor_prediction_result + +# pylint: disable=ungrouped-imports +try: + import tensorflow as tf + import torch + from transformers import AutoModel + from transformers import TFAutoModel + from apache_beam.ml.inference.huggingface_inference import HuggingFaceModelHandlerTensor +except ImportError: + raise unittest.SkipTest('Transformers dependencies are not installed.') + + +def fake_inference_fn_tensor( + batch: Sequence[Union[tf.Tensor, torch.Tensor]], + model: Union[AutoModel, TFAutoModel], + device, + inference_args: Dict[str, Any], + model_id: Optional[str] = None) -> Iterable[PredictionResult]: + predictions = model.predict(batch, **inference_args) + return utils._convert_to_result(batch, predictions, model_id) + + +class FakeTorchModel: + def predict(self, input: torch.Tensor): + return input + + +@pytest.mark.uses_transformers +class HuggingFaceInferenceTest(unittest.TestCase): + def setUp(self) -> None: + self.tmpdir = tempfile.mkdtemp() + + def tearDown(self) -> None: + shutil.rmtree(self.tmpdir) + + def test_predict_tensor(self): + fake_model = FakeTFTensorModel() + inference_runner = HuggingFaceModelHandlerTensor( + model_uri='unused', + model_class=TFAutoModel, + inference_fn=fake_inference_fn_tensor) + batched_examples = [tf.constant([1]), tf.constant([10]), tf.constant([100])] + expected_predictions = [ + PredictionResult(ex, pred) for ex, + pred in zip( + batched_examples, + [tf.math.multiply(n, 10) for n in batched_examples]) + ] + + inferences = inference_runner.run_inference(batched_examples, fake_model) + for actual, expected in zip(inferences, expected_predictions): + self.assertTrue(_compare_tensor_prediction_result(actual, expected)) + + def test_predict_tensor_with_inference_args(self): + fake_model = FakeTFTensorModel() + inference_runner = HuggingFaceModelHandlerTensor( + model_uri='unused', + model_class=TFAutoModel, + inference_fn=fake_inference_fn_tensor, + inference_args={"add": True}) + batched_examples = [tf.constant([1]), tf.constant([10]), tf.constant([100])] + expected_predictions = [ + PredictionResult(ex, pred) for ex, + pred in zip( + batched_examples, [ + tf.math.add(tf.math.multiply(n, 10), 10) + for n in batched_examples + ]) + ] + + inferences = inference_runner.run_inference( + batched_examples, fake_model, inference_args={"add": True}) + + for actual, expected in zip(inferences, expected_predictions): + self.assertTrue(_compare_tensor_prediction_result(actual, expected)) + + def test_framework_detection_torch(self): + fake_model = FakeTorchModel() + inference_runner = HuggingFaceModelHandlerTensor( + model_uri='unused', + model_class=TFAutoModel, + inference_fn=fake_inference_fn_tensor) + batched_examples = [torch.tensor(1), torch.tensor(10), torch.tensor(100)] + inference_runner.run_inference(batched_examples, fake_model) + self.assertEqual(inference_runner._framework, "torch") + + def test_framework_detection_tensorflow(self): + fake_model = FakeTFTensorModel() + inference_runner = HuggingFaceModelHandlerTensor( + model_uri='unused', + model_class=TFAutoModel, + inference_fn=fake_inference_fn_tensor, + inference_args={"add": True}) + batched_examples = [tf.constant([1]), tf.constant([10]), tf.constant([100])] + inference_runner.run_inference( + batched_examples, fake_model, inference_args={"add": True}) + self.assertEqual(inference_runner._framework, "tf") + + +if __name__ == '__main__': + unittest.main() diff --git a/sdks/python/apache_beam/ml/inference/huggingface_tests_requirements.txt b/sdks/python/apache_beam/ml/inference/huggingface_tests_requirements.txt new file mode 100644 index 0000000000000..adb4816cab6b0 --- /dev/null +++ b/sdks/python/apache_beam/ml/inference/huggingface_tests_requirements.txt @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +torch>=1.7.1 +transformers==4.30.0 +tensorflow>=2.12.0 \ No newline at end of file diff --git a/sdks/python/apache_beam/ml/inference/pytorch_inference.py b/sdks/python/apache_beam/ml/inference/pytorch_inference.py index 0a386bf4f2068..26e593fdd7ded 100644 --- a/sdks/python/apache_beam/ml/inference/pytorch_inference.py +++ b/sdks/python/apache_beam/ml/inference/pytorch_inference.py @@ -100,6 +100,7 @@ def _load_model( model = model_class(**model_params) # type: ignore[arg-type,misc] state_dict = torch.load(file, map_location=device, **load_model_args) model.load_state_dict(state_dict) + model.requires_grad_(False) else: file = FileSystems.open(torch_script_model_path, 'rb') model = torch.jit.load(file, map_location=device, **load_model_args) diff --git a/sdks/python/apache_beam/ml/inference/pytorch_inference_it_test.py b/sdks/python/apache_beam/ml/inference/pytorch_inference_it_test.py index 5e37772040842..2cc49be54599b 100644 --- a/sdks/python/apache_beam/ml/inference/pytorch_inference_it_test.py +++ b/sdks/python/apache_beam/ml/inference/pytorch_inference_it_test.py @@ -32,6 +32,7 @@ import torch from apache_beam.examples.inference import pytorch_image_classification from apache_beam.examples.inference import pytorch_image_segmentation + from apache_beam.examples.inference import pytorch_model_per_key_image_segmentation from apache_beam.examples.inference import pytorch_language_modeling except ImportError as e: torch = None @@ -127,6 +128,53 @@ def test_torch_run_inference_coco_maskrcnn_resnet50_fpn(self): prediction_labels = predictions_dict[filename] self.assertEqual(actual_labels, prediction_labels) + @pytest.mark.uses_pytorch + @pytest.mark.it_postcommit + @pytest.mark.timeout(1800) + def test_torch_run_inference_coco_maskrcnn_resnet50_fpn_v1_and_v2(self): + test_pipeline = TestPipeline(is_integration_test=True) + # text files containing absolute path to the coco validation data on GCS + file_of_image_names = 'gs://apache-beam-ml/testing/inputs/it_coco_validation_inputs.txt' # pylint: disable=line-too-long + output_file_dir = 'gs://apache-beam-ml/testing/predictions' + output_file = '/'.join([output_file_dir, str(uuid.uuid4()), 'result.txt']) + + model_state_dict_paths = [ + 'gs://apache-beam-ml/models/torchvision.models.detection.maskrcnn_resnet50_fpn.pth', # pylint: disable=line-too-long + 'gs://apache-beam-ml/models/torchvision.models.detection.maskrcnn_resnet50_fpn_v2.pth' # pylint: disable=line-too-long + ] + images_dir = 'gs://apache-beam-ml/datasets/coco/raw-data/val2017' + extra_opts = { + 'input': file_of_image_names, + 'output': output_file, + 'model_state_dict_paths': ','.join(model_state_dict_paths), + 'images_dir': images_dir, + } + pytorch_model_per_key_image_segmentation.run( + test_pipeline.get_full_options_as_args(**extra_opts), + save_main_session=False) + + self.assertEqual(FileSystems().exists(output_file), True) + predictions = process_outputs(filepath=output_file) + actuals_file = 'gs://apache-beam-ml/testing/expected_outputs/test_torch_run_inference_coco_maskrcnn_resnet50_fpn_v1_and_v2_actuals.txt' # pylint: disable=line-too-long + actuals = process_outputs(filepath=actuals_file) + + predictions_dict = {} + for prediction in predictions: + p = prediction.split('---') + filename = p[0] + v1predictions = p[1] + v2predictions = p[2] + predictions_dict[filename] = (v1predictions, v2predictions) + + for actual in actuals: + a = actual.split('---') + filename = a[0] + v1actuals = a[1] + v2actuals = a[2] + v1prediction_labels, v2prediction_labels = predictions_dict[filename] + self.assertEqual(v1actuals, v1prediction_labels) + self.assertEqual(v2actuals, v2prediction_labels) + @pytest.mark.uses_pytorch @pytest.mark.it_postcommit def test_torch_run_inference_bert_for_masked_lm(self): @@ -161,6 +209,41 @@ def test_torch_run_inference_bert_for_masked_lm(self): predicted_predicted_text = predictions_dict[text] self.assertEqual(actual_predicted_text, predicted_predicted_text) + @pytest.mark.uses_pytorch + @pytest.mark.it_postcommit + def test_torch_run_inference_bert_for_masked_lm_large_model(self): + test_pipeline = TestPipeline(is_integration_test=True) + # Path to text file containing some sentences + file_of_sentences = 'gs://apache-beam-ml/datasets/custom/sentences.txt' # pylint: disable=line-too-long + output_file_dir = 'gs://apache-beam-ml/testing/predictions' + output_file = '/'.join([output_file_dir, str(uuid.uuid4()), 'result.txt']) + + model_state_dict_path = 'gs://apache-beam-ml/models/huggingface.BertForMaskedLM.bert-base-uncased.pth' # pylint: disable=line-too-long + extra_opts = { + 'input': file_of_sentences, + 'output': output_file, + 'model_state_dict_path': model_state_dict_path, + 'large_model': True, + } + pytorch_language_modeling.run( + test_pipeline.get_full_options_as_args(**extra_opts), + save_main_session=False) + + self.assertEqual(FileSystems().exists(output_file), True) + predictions = process_outputs(filepath=output_file) + actuals_file = 'gs://apache-beam-ml/testing/expected_outputs/test_torch_run_inference_bert_for_masked_lm_actuals.txt' # pylint: disable=line-too-long + actuals = process_outputs(filepath=actuals_file) + + predictions_dict = {} + for prediction in predictions: + text, predicted_text = prediction.split(';') + predictions_dict[text] = predicted_text + + for actual in actuals: + text, actual_predicted_text = actual.split(';') + predicted_predicted_text = predictions_dict[text] + self.assertEqual(actual_predicted_text, predicted_predicted_text) + if __name__ == '__main__': logging.getLogger().setLevel(logging.DEBUG) diff --git a/sdks/python/apache_beam/ml/inference/sklearn_inference_it_test.py b/sdks/python/apache_beam/ml/inference/sklearn_inference_it_test.py index 73bb9341f25b2..c5480234cda69 100644 --- a/sdks/python/apache_beam/ml/inference/sklearn_inference_it_test.py +++ b/sdks/python/apache_beam/ml/inference/sklearn_inference_it_test.py @@ -85,6 +85,38 @@ def test_sklearn_mnist_classification(self): true_label, expected_prediction = expected_outputs[i].split(',') self.assertEqual(predictions_dict[true_label], expected_prediction) + def test_sklearn_mnist_classification_large_model(self): + test_pipeline = TestPipeline(is_integration_test=True) + input_file = 'gs://apache-beam-ml/testing/inputs/it_mnist_data.csv' + output_file_dir = 'gs://temp-storage-for-end-to-end-tests' + output_file = '/'.join([output_file_dir, str(uuid.uuid4()), 'result.txt']) + model_path = 'gs://apache-beam-ml/models/mnist_model_svm.pickle' + extra_opts = { + 'input': input_file, + 'output': output_file, + 'model_path': model_path, + 'large_model': True + } + sklearn_mnist_classification.run( + test_pipeline.get_full_options_as_args(**extra_opts), + save_main_session=False) + self.assertEqual(FileSystems().exists(output_file), True) + + expected_output_filepath = 'gs://apache-beam-ml/testing/expected_outputs/test_sklearn_mnist_classification_actuals.txt' # pylint: disable=line-too-long + expected_outputs = process_outputs(expected_output_filepath) + + predicted_outputs = process_outputs(output_file) + self.assertEqual(len(expected_outputs), len(predicted_outputs)) + + predictions_dict = {} + for i in range(len(predicted_outputs)): + true_label, prediction = predicted_outputs[i].split(',') + predictions_dict[true_label] = prediction + + for i in range(len(expected_outputs)): + true_label, expected_prediction = expected_outputs[i].split(',') + self.assertEqual(predictions_dict[true_label], expected_prediction) + # TODO(https://github.com/apache/beam/issues/27151) use model with sklearn 1.2 @unittest.skipIf(sys.version_info >= (3, 11, 0), "Beam#27151") def test_sklearn_regression(self): diff --git a/sdks/python/apache_beam/ml/inference/tensorflow_inference_it_test.py b/sdks/python/apache_beam/ml/inference/tensorflow_inference_it_test.py index 9c814062e6ed4..4786b7a039800 100644 --- a/sdks/python/apache_beam/ml/inference/tensorflow_inference_it_test.py +++ b/sdks/python/apache_beam/ml/inference/tensorflow_inference_it_test.py @@ -18,7 +18,6 @@ """End-to-End test for Tensorflow Inference""" import logging -import sys import unittest import uuid from pathlib import Path @@ -67,10 +66,6 @@ def clear_tf_hub_temp_dir(model_path): rmdir(local_path) -@unittest.skipIf( - sys.version_info.major == 3 and sys.version_info.minor == 7, - "Tensorflow tests on Python 3.7 with Apache Beam 2.47.0 or " - "greater are skipped since tensorflow>=2.12 doesn't support Python 3.7") @unittest.skipIf( tf is None, 'Missing dependencies. ' 'Test depends on tensorflow') @@ -107,6 +102,37 @@ def test_tf_mnist_classification(self): true_label, expected_prediction = expected_outputs[i].split(',') self.assertEqual(predictions_dict[true_label], expected_prediction) + def test_tf_mnist_classification_large_model(self): + test_pipeline = TestPipeline(is_integration_test=True) + input_file = 'gs://apache-beam-ml/testing/inputs/it_mnist_data.csv' + output_file_dir = 'gs://apache-beam-ml/testing/outputs' + output_file = '/'.join([output_file_dir, str(uuid.uuid4()), 'result.txt']) + model_path = 'gs://apache-beam-ml/models/tensorflow/mnist/' + extra_opts = { + 'input': input_file, + 'output': output_file, + 'model_path': model_path, + 'large_model': True, + } + tensorflow_mnist_classification.run( + test_pipeline.get_full_options_as_args(**extra_opts), + save_main_session=False) + self.assertEqual(FileSystems().exists(output_file), True) + + expected_output_filepath = 'gs://apache-beam-ml/testing/expected_outputs/test_sklearn_mnist_classification_actuals.txt' # pylint: disable=line-too-long + expected_outputs = process_outputs(expected_output_filepath) + predicted_outputs = process_outputs(output_file) + self.assertEqual(len(expected_outputs), len(predicted_outputs)) + + predictions_dict = {} + for i in range(len(predicted_outputs)): + true_label, prediction = predicted_outputs[i].split(',') + predictions_dict[true_label] = prediction + + for i in range(len(expected_outputs)): + true_label, expected_prediction = expected_outputs[i].split(',') + self.assertEqual(predictions_dict[true_label], expected_prediction) + def test_tf_imagenet_image_segmentation(self): test_pipeline = TestPipeline(is_integration_test=True) input_file = ( diff --git a/sdks/python/apache_beam/ml/inference/vertex_ai_inference.py b/sdks/python/apache_beam/ml/inference/vertex_ai_inference.py new file mode 100644 index 0000000000000..8c902421f6034 --- /dev/null +++ b/sdks/python/apache_beam/ml/inference/vertex_ai_inference.py @@ -0,0 +1,233 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import time +from typing import Any +from typing import Dict +from typing import Iterable +from typing import Optional +from typing import Sequence + +from google.api_core.exceptions import ServerError +from google.api_core.exceptions import TooManyRequests +from google.cloud import aiplatform + +from apache_beam.io.components.adaptive_throttler import AdaptiveThrottler +from apache_beam.metrics.metric import Metrics +from apache_beam.ml.inference import utils +from apache_beam.ml.inference.base import ModelHandler +from apache_beam.ml.inference.base import PredictionResult +from apache_beam.utils import retry + +MSEC_TO_SEC = 1000 + +LOGGER = logging.getLogger("VertexAIModelHandlerJSON") + +# pylint: disable=line-too-long + + +def _retry_on_appropriate_gcp_error(exception): + """ + Retry filter that returns True if a returned HTTP error code is 5xx or 429. + This is used to retry remote requests that fail, most notably 429 + (TooManyRequests.) + + Args: + exception: the returned exception encountered during the request/response + loop. + + Returns: + boolean indication whether or not the exception is a Server Error (5xx) or + a TooManyRequests (429) error. + """ + return isinstance(exception, (TooManyRequests, ServerError)) + + +class VertexAIModelHandlerJSON(ModelHandler[Any, + PredictionResult, + aiplatform.Endpoint]): + def __init__( + self, + endpoint_id: str, + project: str, + location: str, + experiment: Optional[str] = None, + network: Optional[str] = None, + private: bool = False, + **kwargs): + """Implementation of the ModelHandler interface for Vertex AI. + **NOTE:** This API and its implementation are under development and + do not provide backward compatibility guarantees. + Unlike other ModelHandler implementations, this does not load the model + being used onto the worker and instead makes remote queries to a + Vertex AI endpoint. In that way it functions more like a mid-pipeline + IO. Public Vertex AI endpoints have a maximum request size of 1.5 MB. + If you wish to make larger requests and use a private endpoint, provide + the Compute Engine network you wish to use and set `private=True` + + Args: + endpoint_id: the numerical ID of the Vertex AI endpoint to query + project: the GCP project name where the endpoint is deployed + location: the GCP location where the endpoint is deployed + experiment: optional. experiment label to apply to the + queries. See + https://cloud.google.com/vertex-ai/docs/experiments/intro-vertex-ai-experiments + for more information. + network: optional. the full name of the Compute Engine + network the endpoint is deployed on; used for private + endpoints. The network or subnetwork Dataflow pipeline + option must be set and match this network for pipeline + execution. + Ex: "projects/12345/global/networks/myVPC" + private: optional. if the deployed Vertex AI endpoint is + private, set to true. Requires a network to be provided + as well. + """ + + self._env_vars = kwargs.get('env_vars', {}) + + if private and network is None: + raise ValueError( + "A VPC network must be provided to use a private endpoint.") + + # TODO: support the full list of options for aiplatform.init() + # See https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform#google_cloud_aiplatform_init + aiplatform.init( + project=project, + location=location, + experiment=experiment, + network=network) + + # Check for liveness here but don't try to actually store the endpoint + # in the class yet + self.endpoint_name = endpoint_id + self.is_private = private + + _ = self._retrieve_endpoint(self.endpoint_name, self.is_private) + + # Configure AdaptiveThrottler and throttling metrics for client-side + # throttling behavior. + # See https://docs.google.com/document/d/1ePorJGZnLbNCmLD9mR7iFYOdPsyDA1rDnTpYnbdrzSU/edit?usp=sharing + # for more details. + self.throttled_secs = Metrics.counter( + VertexAIModelHandlerJSON, "cumulativeThrottlingSeconds") + self.throttler = AdaptiveThrottler( + window_ms=1, bucket_ms=1, overload_ratio=2) + + def _retrieve_endpoint( + self, endpoint_id: str, is_private: bool) -> aiplatform.Endpoint: + """Retrieves an AI Platform endpoint and queries it for liveness/deployed + models. + + Args: + endpoint_id: the numerical ID of the Vertex AI endpoint to retrieve. + is_private: a boolean indicating if the Vertex AI endpoint is a private + endpoint + Returns: + An aiplatform.Endpoint object + Raises: + ValueError: if endpoint is inactive or has no models deployed to it. + """ + if is_private: + endpoint: aiplatform.Endpoint = aiplatform.PrivateEndpoint( + endpoint_name=endpoint_id) + LOGGER.debug("Treating endpoint %s as private", endpoint_id) + else: + endpoint = aiplatform.Endpoint(endpoint_name=endpoint_id) + LOGGER.debug("Treating endpoint %s as public", endpoint_id) + + try: + mod_list = endpoint.list_models() + except Exception as e: + raise ValueError( + "Failed to contact endpoint %s, got exception: %s", endpoint_id, e) + + if len(mod_list) == 0: + raise ValueError("Endpoint %s has no models deployed to it.", endpoint_id) + + return endpoint + + def load_model(self) -> aiplatform.Endpoint: + """Loads the Endpoint object used to build and send prediction request to + Vertex AI. + """ + # Check to make sure the endpoint is still active since pipeline + # construction time + ep = self._retrieve_endpoint(self.endpoint_name, self.is_private) + return ep + + @retry.with_exponential_backoff( + num_retries=5, retry_filter=_retry_on_appropriate_gcp_error) + def get_request( + self, + batch: Sequence[Any], + model: aiplatform.Endpoint, + throttle_delay_secs: int, + inference_args: Optional[Dict[str, Any]]): + while self.throttler.throttle_request(time.time() * MSEC_TO_SEC): + LOGGER.info( + "Delaying request for %d seconds due to previous failures", + throttle_delay_secs) + time.sleep(throttle_delay_secs) + self.throttled_secs.inc(throttle_delay_secs) + + try: + req_time = time.time() + prediction = model.predict( + instances=list(batch), parameters=inference_args) + self.throttler.successful_request(req_time * MSEC_TO_SEC) + return prediction + except TooManyRequests as e: + LOGGER.warning("request was limited by the service with code %i", e.code) + raise + except Exception as e: + LOGGER.error("unexpected exception raised as part of request, got %s", e) + raise + + def run_inference( + self, + batch: Sequence[Any], + model: aiplatform.Endpoint, + inference_args: Optional[Dict[str, Any]] = None + ) -> Iterable[PredictionResult]: + """ Sends a prediction request to a Vertex AI endpoint containing batch + of inputs and matches that input with the prediction response from + the endpoint as an iterable of PredictionResults. + + Args: + batch: a sequence of any values to be passed to the Vertex AI endpoint. + Should be encoded as the model expects. + model: an aiplatform.Endpoint object configured to access the desired + model. + inference_args: any additional arguments to send as part of the + prediction request. + + Returns: + An iterable of Predictions. + """ + + # Endpoint.predict returns a Prediction type with the prediction values + # along with model metadata + prediction = self.get_request( + batch, model, throttle_delay_secs=5, inference_args=inference_args) + + return utils._convert_to_result( + batch, prediction.predictions, prediction.deployed_model_id) + + def validate_inference_args(self, inference_args: Optional[Dict[str, Any]]): + pass diff --git a/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py b/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py new file mode 100644 index 0000000000000..168ab031abb18 --- /dev/null +++ b/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py @@ -0,0 +1,87 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""End-to-End test for Vertex AI Remote Inference""" + +import logging +import unittest +import uuid + +import pytest + +from apache_beam.io.filesystems import FileSystems +from apache_beam.testing.test_pipeline import TestPipeline + +# pylint: disable=ungrouped-imports +try: + from apache_beam.examples.inference import vertex_ai_image_classification + from apache_beam.examples.inference import vertex_ai_llm_text_classification +except ImportError as e: + raise unittest.SkipTest( + "Vertex AI model handler dependencies are not installed") + +_INPUT = "gs://apache-beam-ml/testing/inputs/vertex_images/*/*.jpg" +_OUTPUT_DIR = "gs://apache-beam-ml/testing/outputs/vertex_images" +_FLOWER_ENDPOINT_ID = "5384055553544683520" +_LLM_ENDPOINT_ID = "9157860935048626176" +_ENDPOINT_PROJECT = "apache-beam-testing" +_ENDPOINT_REGION = "us-central1" +_ENDPOINT_NETWORK = "projects/844138762903/global/networks/beam-test-vpc" +# pylint: disable=line-too-long +_SUBNETWORK = "https://www.googleapis.com/compute/v1/projects/apache-beam-testing/regions/us-central1/subnetworks/beam-test-vpc" + + +class VertexAIInference(unittest.TestCase): + @pytest.mark.vertex_ai_postcommit + def test_vertex_ai_run_flower_image_classification(self): + output_file = '/'.join([_OUTPUT_DIR, str(uuid.uuid4()), 'output.txt']) + + test_pipeline = TestPipeline(is_integration_test=True) + extra_opts = { + 'input': _INPUT, + 'output': output_file, + 'endpoint_id': _FLOWER_ENDPOINT_ID, + 'endpoint_project': _ENDPOINT_PROJECT, + 'endpoint_region': _ENDPOINT_REGION, + 'endpoint_network': _ENDPOINT_NETWORK, + 'private': "True", + 'subnetwork': _SUBNETWORK, + } + vertex_ai_image_classification.run( + test_pipeline.get_full_options_as_args(**extra_opts)) + self.assertEqual(FileSystems().exists(output_file), True) + + @pytest.mark.uses_vertex_ai + @pytest.mark.it_postcommit + def test_vertex_ai_run_llm_text_classification(self): + output_file = '/'.join([_OUTPUT_DIR, str(uuid.uuid4()), 'output.txt']) + + test_pipeline = TestPipeline(is_integration_test=True) + extra_opts = { + 'output': output_file, + 'endpoint_id': _LLM_ENDPOINT_ID, + 'endpoint_project': _ENDPOINT_PROJECT, + 'endpoint_region': _ENDPOINT_REGION + } + vertex_ai_llm_text_classification.run( + test_pipeline.get_full_options_as_args(**extra_opts)) + self.assertEqual(FileSystems().exists(output_file), True) + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.DEBUG) + unittest.main() diff --git a/sdks/python/apache_beam/ml/inference/vertex_ai_inference_test.py b/sdks/python/apache_beam/ml/inference/vertex_ai_inference_test.py new file mode 100644 index 0000000000000..34c7927272d65 --- /dev/null +++ b/sdks/python/apache_beam/ml/inference/vertex_ai_inference_test.py @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pytype: skip-file + +import unittest + +try: + from apache_beam.ml.inference.vertex_ai_inference import _retry_on_appropriate_gcp_error + from apache_beam.ml.inference.vertex_ai_inference import VertexAIModelHandlerJSON + from google.api_core.exceptions import TooManyRequests +except ImportError: + raise unittest.SkipTest('VertexAI dependencies are not installed') + + +class RetryOnClientErrorTest(unittest.TestCase): + def test_retry_on_client_error_positive(self): + e = TooManyRequests(message="fake service rate limiting") + self.assertTrue(_retry_on_appropriate_gcp_error(e)) + + def test_retry_on_client_error_negative(self): + e = ValueError() + self.assertFalse(_retry_on_appropriate_gcp_error(e)) + + +class ModelHandlerArgConditions(unittest.TestCase): + def test_exception_on_private_without_network(self): + self.assertRaises( + ValueError, + VertexAIModelHandlerJSON, + endpoint_id="1", + project="testproject", + location="us-central1", + private=True) + + +if __name__ == '__main__': + unittest.main() diff --git a/sdks/python/apache_beam/ml/inference/vertex_ai_tests_requirements.txt b/sdks/python/apache_beam/ml/inference/vertex_ai_tests_requirements.txt new file mode 100644 index 0000000000000..aaf923e729926 --- /dev/null +++ b/sdks/python/apache_beam/ml/inference/vertex_ai_tests_requirements.txt @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +google-cloud-aiplatform>=1.26.0 +tensorflow>=2.12.0 \ No newline at end of file diff --git a/sdks/python/apache_beam/ml/inference/xgboost_inference_it_test.py b/sdks/python/apache_beam/ml/inference/xgboost_inference_it_test.py index e458ccab4b0a4..3db62bcc6a991 100644 --- a/sdks/python/apache_beam/ml/inference/xgboost_inference_it_test.py +++ b/sdks/python/apache_beam/ml/inference/xgboost_inference_it_test.py @@ -113,6 +113,40 @@ def test_iris_classification_numpy_single_batch(self): true_label, expected_prediction = expected_output.split(',') self.assertEqual(predictions_dict[true_label], expected_prediction) + def test_iris_classification_numpy_single_batch_large_model(self): + test_pipeline = TestPipeline(is_integration_test=True) + input_type = 'numpy' + output_file_dir = '/tmp' + output_file = '/'.join( + [output_file_dir, str(uuid.uuid4()), 'numpy_single_batch.txt']) + model_state_path = 'gs://apache-beam-ml/models/xgboost.iris_classifier.json' + extra_opts = { + 'input_type': input_type, + 'output': output_file, + 'model_state': model_state_path, + 'no_split': True, + 'large_model': True, + } + + xgboost_iris_classification.run( + test_pipeline.get_full_options_as_args(**extra_opts), + save_main_session=False) + self.assertEqual(FileSystems().exists(output_file), True) + + expected_outputs = EXPECTED_OUTPUT_SINGLE_BATCHES + + predicted_outputs = process_outputs(output_file) + self.assertEqual(len(expected_outputs), len(predicted_outputs)) + + predictions_dict = {} + for predicted_output in predicted_outputs: + true_label, prediction = predicted_output.split(',') + predictions_dict[true_label] = prediction + + for expected_output in expected_outputs: + true_label, expected_prediction = expected_output.split(',') + self.assertEqual(predictions_dict[true_label], expected_prediction) + def test_iris_classification_pandas_single_batch(self): test_pipeline = TestPipeline(is_integration_test=True) input_type = 'pandas' diff --git a/sdks/python/apache_beam/ml/transforms/__init__.py b/sdks/python/apache_beam/ml/transforms/__init__.py new file mode 100644 index 0000000000000..cce3acad34a49 --- /dev/null +++ b/sdks/python/apache_beam/ml/transforms/__init__.py @@ -0,0 +1,16 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/sdks/python/apache_beam/ml/transforms/base.py b/sdks/python/apache_beam/ml/transforms/base.py new file mode 100644 index 0000000000000..a0bc4a9061006 --- /dev/null +++ b/sdks/python/apache_beam/ml/transforms/base.py @@ -0,0 +1,269 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pytype: skip-file + +import abc +from typing import Dict +from typing import Generic +from typing import List +from typing import Optional +from typing import Sequence +from typing import TypeVar + +import apache_beam as beam +from apache_beam.metrics.metric import Metrics + +__all__ = ['MLTransform', 'ProcessHandler', 'BaseOperation'] + +TransformedDatasetT = TypeVar('TransformedDatasetT') +TransformedMetadataT = TypeVar('TransformedMetadataT') + +# Input/Output types to the MLTransform. +MLTransformOutputT = TypeVar('MLTransformOutputT') +ExampleT = TypeVar('ExampleT') + +# Input to the apply() method of BaseOperation. +OperationInputT = TypeVar('OperationInputT') +# Output of the apply() method of BaseOperation. +OperationOutputT = TypeVar('OperationOutputT') + + +class ArtifactMode(object): + PRODUCE = 'produce' + CONSUME = 'consume' + + +class BaseOperation(Generic[OperationInputT, OperationOutputT], abc.ABC): + def __init__(self, columns: List[str]) -> None: + """ + Base Opertation class data processing transformations. + Args: + columns: List of column names to apply the transformation. + """ + self.columns = columns + + @abc.abstractmethod + def apply_transform(self, data: OperationInputT, + output_column_name: str) -> Dict[str, OperationOutputT]: + """ + Define any processing logic in the apply_transform() method. + processing logics are applied on inputs and returns a transformed + output. + Args: + inputs: input data. + """ + + @abc.abstractmethod + def get_artifacts( + self, data: OperationInputT, + output_column_prefix: str) -> Optional[Dict[str, OperationOutputT]]: + """ + If the operation generates any artifacts, they can be returned from this + method. + """ + pass + + def __call__(self, data: OperationInputT, + output_column_name: str) -> Dict[str, OperationOutputT]: + """ + This method is called when the instance of the class is called. + This method will invoke the apply() method of the class. + """ + transformed_data = self.apply_transform(data, output_column_name) + artifacts = self.get_artifacts(data, output_column_name) + if artifacts: + transformed_data = {**transformed_data, **artifacts} + return transformed_data + + def get_counter(self): + """ + Returns the counter name for the operation. + """ + counter_name = self.__class__.__name__ + return Metrics.counter(MLTransform, f'BeamML_{counter_name}') + + +class ProcessHandler(Generic[ExampleT, MLTransformOutputT], abc.ABC): + """ + Only for internal use. No backwards compatibility guarantees. + """ + @abc.abstractmethod + def process_data( + self, pcoll: beam.PCollection[ExampleT] + ) -> beam.PCollection[MLTransformOutputT]: + """ + Logic to process the data. This will be the entrypoint in + beam.MLTransform to process incoming data. + """ + + @abc.abstractmethod + def append_transform(self, transform: BaseOperation): + """ + Append transforms to the ProcessHandler. + """ + + +class MLTransform(beam.PTransform[beam.PCollection[ExampleT], + beam.PCollection[MLTransformOutputT]], + Generic[ExampleT, MLTransformOutputT]): + def __init__( + self, + *, + write_artifact_location: Optional[str] = None, + read_artifact_location: Optional[str] = None, + transforms: Optional[Sequence[BaseOperation]] = None): + """ + MLTransform is a Beam PTransform that can be used to apply + transformations to the data. MLTransform is used to wrap the + data processing transforms provided by Apache Beam. MLTransform + works in two modes: write and read. In the write mode, + MLTransform will apply the transforms to the data and store the + artifacts in the write_artifact_location. In the read mode, + MLTransform will read the artifacts from the + read_artifact_location and apply the transforms to the data. The + artifact location should be a valid storage path where the artifacts + can be written to or read from. + + Note that when consuming artifacts, it is not necessary to pass the + transforms since they are inherently stored within the artifacts + themselves. + + Args: + write_artifact_location: A storage location for artifacts resulting from + MLTransform. These artifacts include transformations applied to + the dataset and generated values like min, max from ScaleTo01, + and mean, var from ScaleToZScore. Artifacts are produced and written + to this location when using `write_artifact_mode`. + Later MLTransforms can reuse produced artifacts by setting + `read_artifact_mode` instead of `write_artifact_mode`. The value + assigned to `write_artifact_location` should be a valid storage + directory that the artifacts from this transform can be written to. + If no directory exists at this location, one will be created. This will + overwrite any artifacts already in this location, so distinct locations + should be used for each instance of MLTransform. Only one of + write_artifact_location and read_artifact_location should be specified. + read_artifact_location: A storage location to read artifacts resulting + froma previous MLTransform. These artifacts include transformations + applied to the dataset and generated values like min, max from + ScaleTo01, and mean, var from ScaleToZScore. Note that when consuming + artifacts, it is not necessary to pass the transforms since they are + inherently stored within the artifacts themselves. The value assigned + to `read_artifact_location` should be a valid storage path where the + artifacts can be read from. Only one of write_artifact_location and + read_artifact_location should be specified. + transforms: A list of transforms to apply to the data. All the transforms + are applied in the order they are specified. The input of the + i-th transform is the output of the (i-1)-th transform. Multi-input + transforms are not supported yet. + """ + if transforms: + _ = [self._validate_transform(transform) for transform in transforms] + + if read_artifact_location and write_artifact_location: + raise ValueError( + 'Only one of read_artifact_location or write_artifact_location can ' + 'be specified to initialize MLTransform') + + if not read_artifact_location and not write_artifact_location: + raise ValueError( + 'Either a read_artifact_location or write_artifact_location must be ' + 'specified to initialize MLTransform') + + if read_artifact_location: + artifact_location = read_artifact_location + artifact_mode = ArtifactMode.CONSUME + else: + artifact_location = write_artifact_location # type: ignore[assignment] + artifact_mode = ArtifactMode.PRODUCE + + # avoid circular import + # pylint: disable=wrong-import-order, wrong-import-position + from apache_beam.ml.transforms.handlers import TFTProcessHandler + # TODO: When new ProcessHandlers(eg: JaxProcessHandler) are introduced, + # create a mapping between transforms and ProcessHandler since + # ProcessHandler is not exposed to the user. + process_handler: ProcessHandler = TFTProcessHandler( + artifact_location=artifact_location, + artifact_mode=artifact_mode, + transforms=transforms) # type: ignore[arg-type] + + self._process_handler = process_handler + self.transforms = transforms + self._counter = Metrics.counter( + MLTransform, f'BeamML_{self.__class__.__name__}') + + def expand( + self, pcoll: beam.PCollection[ExampleT] + ) -> beam.PCollection[MLTransformOutputT]: + """ + This is the entrypoint for the MLTransform. This method will + invoke the process_data() method of the ProcessHandler instance + to process the incoming data. + + process_data takes in a PCollection and applies the PTransforms + necessary to process the data and returns a PCollection of + transformed data. + Args: + pcoll: A PCollection of ExampleT type. + Returns: + A PCollection of MLTransformOutputT type + """ + _ = ( + pcoll.pipeline + | "MLTransformMetricsUsage" >> MLTransformMetricsUsage(self)) + return self._process_handler.process_data(pcoll) + + def with_transform(self, transform: BaseOperation): + """ + Add a transform to the MLTransform pipeline. + Args: + transform: A BaseOperation instance. + Returns: + A MLTransform instance. + """ + self._validate_transform(transform) + self._process_handler.append_transform(transform) + return self + + def _validate_transform(self, transform): + if not isinstance(transform, BaseOperation): + raise TypeError( + 'transform must be a subclass of BaseOperation. ' + 'Got: %s instead.' % type(transform)) + + +class MLTransformMetricsUsage(beam.PTransform): + def __init__(self, ml_transform: MLTransform): + self._ml_transform = ml_transform + self._ml_transform._counter.inc() + + def expand(self, pipeline): + def _increment_counters(): + # increment for MLTransform. + self._ml_transform._counter.inc() + # increment if data processing transforms are passed. + transforms = ( + self._ml_transform.transforms or + self._ml_transform._process_handler.transforms) + if transforms: + for transform in transforms: + transform.get_counter().inc() + + _ = ( + pipeline + | beam.Create([None]) + | beam.Map(lambda _: _increment_counters())) diff --git a/sdks/python/apache_beam/ml/transforms/base_test.py b/sdks/python/apache_beam/ml/transforms/base_test.py new file mode 100644 index 0000000000000..2e447964541ba --- /dev/null +++ b/sdks/python/apache_beam/ml/transforms/base_test.py @@ -0,0 +1,274 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pytype: skip-file + +import shutil +import tempfile +import typing +import unittest +from typing import List + +import numpy as np +from parameterized import param +from parameterized import parameterized + +import apache_beam as beam +from apache_beam.metrics.metric import MetricsFilter +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to + +# pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports +try: + from apache_beam.ml.transforms import base + from apache_beam.ml.transforms import tft + from apache_beam.ml.transforms.tft import TFTOperation +except ImportError: + tft = None # type: ignore + +if tft is None: + raise unittest.SkipTest('tensorflow_transform is not installed') + + +class _FakeOperation(TFTOperation): + def __init__(self, name, *args, **kwargs): + super().__init__(*args, **kwargs) + self.name = name + + def apply_transform(self, inputs, output_column_name, **kwargs): + return {output_column_name: inputs} + + +class BaseMLTransformTest(unittest.TestCase): + def setUp(self) -> None: + self.artifact_location = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.artifact_location) + + def test_ml_transform_appends_transforms_to_process_handler_correctly(self): + fake_fn_1 = _FakeOperation(name='fake_fn_1', columns=['x']) + transforms = [fake_fn_1] + ml_transform = base.MLTransform( + transforms=transforms, write_artifact_location=self.artifact_location) + ml_transform = ml_transform.with_transform( + transform=_FakeOperation(name='fake_fn_2', columns=['x'])) + + self.assertEqual(len(ml_transform._process_handler.transforms), 2) + self.assertEqual( + ml_transform._process_handler.transforms[0].name, 'fake_fn_1') + self.assertEqual( + ml_transform._process_handler.transforms[1].name, 'fake_fn_2') + + def test_ml_transform_on_dict(self): + transforms = [tft.ScaleTo01(columns=['x'])] + data = [{'x': 1}, {'x': 2}] + with beam.Pipeline() as p: + result = ( + p + | beam.Create(data) + | base.MLTransform( + write_artifact_location=self.artifact_location, + transforms=transforms)) + expected_output = [ + np.array([0.0], dtype=np.float32), + np.array([1.0], dtype=np.float32), + ] + actual_output = result | beam.Map(lambda x: x.x) + assert_that( + actual_output, equal_to(expected_output, equals_fn=np.array_equal)) + + def test_ml_transform_on_list_dict(self): + transforms = [tft.ScaleTo01(columns=['x'])] + data = [{'x': [1, 2, 3]}, {'x': [4, 5, 6]}] + with beam.Pipeline() as p: + result = ( + p + | beam.Create(data) + | base.MLTransform( + transforms=transforms, + write_artifact_location=self.artifact_location)) + expected_output = [ + np.array([0, 0.2, 0.4], dtype=np.float32), + np.array([0.6, 0.8, 1], dtype=np.float32), + ] + actual_output = result | beam.Map(lambda x: x.x) + assert_that( + actual_output, equal_to(expected_output, equals_fn=np.array_equal)) + + @parameterized.expand([ + param( + input_data=[{ + 'x': 1, + 'y': 2.0, + }], + input_types={ + 'x': int, 'y': float + }, + expected_dtype={ + 'x': typing.Sequence[np.float32], + 'y': typing.Sequence[np.float32], + }, + ), + param( + input_data=[{ + 'x': np.array([1], dtype=np.int64), + 'y': np.array([2.0], dtype=np.float32), + }], + input_types={ + 'x': np.int32, 'y': np.float32 + }, + expected_dtype={ + 'x': typing.Sequence[np.float32], + 'y': typing.Sequence[np.float32], + }, + ), + param( + input_data=[{ + 'x': [1, 2, 3], 'y': [2.0, 3.0, 4.0] + }], + input_types={ + 'x': List[int], 'y': List[float] + }, + expected_dtype={ + 'x': typing.Sequence[np.float32], + 'y': typing.Sequence[np.float32], + }, + ), + param( + input_data=[{ + 'x': [1, 2, 3], 'y': [2.0, 3.0, 4.0] + }], + input_types={ + 'x': typing.Sequence[int], + 'y': typing.Sequence[float], + }, + expected_dtype={ + 'x': typing.Sequence[np.float32], + 'y': typing.Sequence[np.float32], + }, + ), + ]) + def test_ml_transform_dict_output_pcoll_schema( + self, input_data, input_types, expected_dtype): + transforms = [tft.ScaleTo01(columns=['x'])] + with beam.Pipeline() as p: + schema_data = ( + p + | beam.Create(input_data) + | beam.Map(lambda x: beam.Row(**x)).with_output_types( + beam.row_type.RowTypeConstraint.from_fields( + list(input_types.items())))) + transformed_data = schema_data | base.MLTransform( + write_artifact_location=self.artifact_location, transforms=transforms) + for name, typ in transformed_data.element_type._fields: + if name in expected_dtype: + self.assertEqual(expected_dtype[name], typ) + + def test_ml_transform_fail_for_non_global_windows_in_produce_mode(self): + transforms = [tft.ScaleTo01(columns=['x'])] + with beam.Pipeline() as p: + with self.assertRaises(RuntimeError): + _ = ( + p + | beam.Create([{ + 'x': 1, 'y': 2.0 + }]) + | beam.WindowInto(beam.window.FixedWindows(1)) + | base.MLTransform( + transforms=transforms, + write_artifact_location=self.artifact_location, + )) + + def test_ml_transform_on_multiple_columns_single_transform(self): + transforms = [tft.ScaleTo01(columns=['x', 'y'])] + data = [{'x': [1, 2, 3], 'y': [1.0, 10.0, 20.0]}] + with beam.Pipeline() as p: + result = ( + p + | beam.Create(data) + | base.MLTransform( + transforms=transforms, + write_artifact_location=self.artifact_location)) + expected_output_x = [ + np.array([0, 0.5, 1], dtype=np.float32), + ] + expected_output_y = [np.array([0, 0.47368422, 1], dtype=np.float32)] + actual_output_x = result | beam.Map(lambda x: x.x) + actual_output_y = result | beam.Map(lambda x: x.y) + assert_that( + actual_output_x, + equal_to(expected_output_x, equals_fn=np.array_equal)) + assert_that( + actual_output_y, + equal_to(expected_output_y, equals_fn=np.array_equal), + label='y') + + def test_ml_transforms_on_multiple_columns_multiple_transforms(self): + transforms = [ + tft.ScaleTo01(columns=['x']), + tft.ComputeAndApplyVocabulary(columns=['y']) + ] + data = [{'x': [1, 2, 3], 'y': ['a', 'b', 'c']}] + with beam.Pipeline() as p: + result = ( + p + | beam.Create(data) + | base.MLTransform( + transforms=transforms, + write_artifact_location=self.artifact_location)) + expected_output_x = [ + np.array([0, 0.5, 1], dtype=np.float32), + ] + expected_output_y = [np.array([2, 1, 0])] + actual_output_x = result | beam.Map(lambda x: x.x) + actual_output_y = result | beam.Map(lambda x: x.y) + + assert_that( + actual_output_x, + equal_to(expected_output_x, equals_fn=np.array_equal)) + assert_that( + actual_output_y, + equal_to(expected_output_y, equals_fn=np.array_equal), + label='actual_output_y') + + def test_mltransform_with_counter(self): + transforms = [ + tft.ComputeAndApplyVocabulary(columns=['y']), + tft.ScaleTo01(columns=['x']) + ] + data = [{'x': [1, 2, 3], 'y': ['a', 'b', 'c']}] + with beam.Pipeline() as p: + _ = ( + p | beam.Create(data) + | base.MLTransform( + transforms=transforms, + write_artifact_location=self.artifact_location)) + scale_to_01_counter = MetricsFilter().with_name('BeamML_ScaleTo01') + vocab_counter = MetricsFilter().with_name( + 'BeamML_ComputeAndApplyVocabulary') + mltransform_counter = MetricsFilter().with_name('BeamML_MLTransform') + result = p.result + self.assertEqual( + result.metrics().query(scale_to_01_counter)['counters'][0].result, 1) + self.assertEqual( + result.metrics().query(vocab_counter)['counters'][0].result, 1) + self.assertEqual( + result.metrics().query(mltransform_counter)['counters'][0].result, 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/sdks/python/apache_beam/ml/transforms/handlers.py b/sdks/python/apache_beam/ml/transforms/handlers.py new file mode 100644 index 0000000000000..8695d5146efae --- /dev/null +++ b/sdks/python/apache_beam/ml/transforms/handlers.py @@ -0,0 +1,518 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pytype: skip-file + +import collections +import hashlib +import os +import typing +from typing import Dict +from typing import List +from typing import Optional +from typing import Sequence +from typing import Union + +import numpy as np + +import apache_beam as beam +import tensorflow as tf +import tensorflow_transform.beam as tft_beam +from apache_beam.ml.transforms.base import ArtifactMode +from apache_beam.ml.transforms.base import ProcessHandler +from apache_beam.ml.transforms.tft import _EXPECTED_TYPES +from apache_beam.ml.transforms.tft import TFTOperation +from apache_beam.typehints import native_type_compatibility +from apache_beam.typehints.row_type import RowTypeConstraint +from tensorflow_metadata.proto.v0 import schema_pb2 +from tensorflow_transform import common_types +from tensorflow_transform.beam.tft_beam_io import beam_metadata_io +from tensorflow_transform.beam.tft_beam_io import transform_fn_io +from tensorflow_transform.tf_metadata import dataset_metadata +from tensorflow_transform.tf_metadata import metadata_io +from tensorflow_transform.tf_metadata import schema_utils + +__all__ = [ + 'TFTProcessHandler', +] + +RAW_DATA_METADATA_DIR = 'raw_data_metadata' +SCHEMA_FILE = 'schema.pbtxt' +# tensorflow transform doesn't support the types other than tf.int64, +# tf.float32 and tf.string. +_default_type_to_tensor_type_map = { + int: tf.int64, + float: tf.float32, + str: tf.string, + bytes: tf.string, + np.int64: tf.int64, + np.int32: tf.int64, + np.float32: tf.float32, + np.float64: tf.float32, + np.bytes_: tf.string, + np.str_: tf.string, +} +_primitive_types_to_typing_container_type = { + int: List[int], float: List[float], str: List[str], bytes: List[bytes] +} + +tft_process_handler_input_type = typing.Union[typing.NamedTuple, + beam.Row, + Dict[str, + typing.Union[str, + float, + int, + bytes, + np.ndarray]]] +tft_process_handler_output_type = typing.Union[beam.Row, Dict[str, np.ndarray]] + + +class ConvertScalarValuesToListValues(beam.DoFn): + def process( + self, + element, + ): + hash_key, element = element + new_dict = {} + for key, value in element.items(): + if isinstance(value, + tuple(_primitive_types_to_typing_container_type.keys())): + new_dict[key] = [value] + else: + new_dict[key] = value + yield (hash_key, new_dict) + + +class ConvertNamedTupleToDict( + beam.PTransform[beam.PCollection[typing.Union[beam.Row, typing.NamedTuple]], + beam.PCollection[Dict[str, + common_types.InstanceDictType]]]): + """ + A PTransform that converts a collection of NamedTuples or Rows into a + collection of dictionaries. + """ + def expand( + self, pcoll: beam.PCollection[typing.Union[beam.Row, typing.NamedTuple]] + ) -> beam.PCollection[common_types.InstanceDictType]: + """ + Args: + pcoll: A PCollection of NamedTuples or Rows. + Returns: + A PCollection of dictionaries. + """ + if isinstance(pcoll.element_type, RowTypeConstraint): + # Row instance + return pcoll | beam.Map(lambda x: x.as_dict()) + else: + # named tuple + return pcoll | beam.Map(lambda x: x._asdict()) + + +class ComputeAndAttachHashKey(beam.DoFn): + """ + Computes and attaches a hash key to the element. + Only for internal use. No backwards compatibility guarantees. + """ + def process(self, element): + hash_object = hashlib.sha256() + for _, value in element.items(): + # handle the case where value is a list or numpy array + if isinstance(value, (list, np.ndarray)): + hash_object.update(str(list(value)).encode()) + else: # assume value is a primitive that can be turned into str + hash_object.update(str(value).encode()) + yield (hash_object.hexdigest(), element) + + +class GetMissingColumnsPColl(beam.DoFn): + """ + Returns data containing only the columns that are not + present in the schema. This is needed since TFT only outputs + columns that are transformed by any of the data processing transforms. + + Only for internal use. No backwards compatibility guarantees. + """ + def __init__(self, existing_columns): + self.existing_columns = existing_columns + + def process(self, element): + new_dict = {} + hash_key, element = element + for key, value in element.items(): + if key not in self.existing_columns: + new_dict[key] = value + yield (hash_key, new_dict) + + +class MakeHashKeyAsColumn(beam.DoFn): + """ + Extracts the hash key from the element and adds it as a column. + + Only for internal use. No backwards compatibility guarantees. + """ + def process(self, element): + hash_key, element = element + element['hash_key'] = hash_key + yield element + + +class ExtractHashAndKeyPColl(beam.DoFn): + """ + Extracts the hash key and return hashkey and element as a tuple. + + Only for internal use. No backwards compatibility guarantees. + """ + def process(self, element): + hashkey = element['hash_key'][0] + del element['hash_key'] + yield (hashkey.decode('utf-8'), element) + + +class MergeDicts(beam.DoFn): + """ + Merges the dictionaries in the PCollection. + + Only for internal use. No backwards compatibility guarantees. + """ + def process(self, element): + _, element = element + new_dict = {} + for d in element: + new_dict.update(d[0]) + yield new_dict + + +class TFTProcessHandler(ProcessHandler[tft_process_handler_input_type, + tft_process_handler_output_type]): + def __init__( + self, + *, + artifact_location: str, + transforms: Optional[Sequence[TFTOperation]] = None, + artifact_mode: str = ArtifactMode.PRODUCE): + """ + A handler class for processing data with TensorFlow Transform (TFT) + operations. + """ + self.transforms = transforms if transforms else [] + self.transformed_schema: Dict[str, type] = {} + self.artifact_location = artifact_location + self.artifact_mode = artifact_mode + if artifact_mode not in ['produce', 'consume']: + raise ValueError('artifact_mode must be either `produce` or `consume`.') + + def append_transform(self, transform): + self.transforms.append(transform) + + def _map_column_names_to_types(self, row_type): + """ + Return a dictionary of column names and types. + Args: + element_type: A type of the element. This could be a NamedTuple or a Row. + Returns: + A dictionary of column names and types. + """ + try: + if not isinstance(row_type, RowTypeConstraint): + row_type = RowTypeConstraint.from_user_type(row_type) + + inferred_types = {name: typ for name, typ in row_type._fields} + + for k, t in inferred_types.items(): + if t in _primitive_types_to_typing_container_type: + inferred_types[k] = _primitive_types_to_typing_container_type[t] + + # sometimes a numpy type can be provided as np.dtype('int64'). + # convert numpy.dtype to numpy type since both are same. + for name, typ in inferred_types.items(): + if isinstance(typ, np.dtype): + inferred_types[name] = typ.type + + return inferred_types + except: # pylint: disable=bare-except + return {} + + def _map_column_names_to_types_from_transforms(self): + column_type_mapping = {} + for transform in self.transforms: + for col in transform.columns: + if col not in column_type_mapping: + # we just need to dtype of first occurance of column in transforms. + class_name = transform.__class__.__name__ + if class_name not in _EXPECTED_TYPES: + raise KeyError( + f"Transform {class_name} is not registered with a supported " + "type. Please register the transform with a supported type " + "using register_input_dtype decorator.") + column_type_mapping[col] = _EXPECTED_TYPES[ + transform.__class__.__name__] + return column_type_mapping + + def get_raw_data_feature_spec( + self, input_types: Dict[str, type]) -> Dict[str, tf.io.VarLenFeature]: + """ + Return a DatasetMetadata object to be used with + tft_beam.AnalyzeAndTransformDataset. + Args: + input_types: A dictionary of column names and types. + Returns: + A DatasetMetadata object. + """ + raw_data_feature_spec = {} + for key, value in input_types.items(): + raw_data_feature_spec[key] = self._get_raw_data_feature_spec_per_column( + typ=value, col_name=key) + return raw_data_feature_spec + + def convert_raw_data_feature_spec_to_dataset_metadata( + self, raw_data_feature_spec) -> dataset_metadata.DatasetMetadata: + raw_data_metadata = dataset_metadata.DatasetMetadata( + schema_utils.schema_from_feature_spec(raw_data_feature_spec)) + return raw_data_metadata + + def _get_raw_data_feature_spec_per_column( + self, typ: type, col_name: str) -> tf.io.VarLenFeature: + """ + Return a FeatureSpec object to be used with + tft_beam.AnalyzeAndTransformDataset + Args: + typ: A type of the column. + col_name: A name of the column. + Returns: + A FeatureSpec object. + """ + # lets conver the builtin types to typing types for consistency. + typ = native_type_compatibility.convert_builtin_to_typing(typ) + primitive_containers_type = ( + list, + collections.abc.Sequence, + ) + is_primitive_container = ( + typing.get_origin(typ) in primitive_containers_type) + + if is_primitive_container: + dtype = typing.get_args(typ)[0] + if len(typing.get_args(typ)) > 1 or typing.get_origin(dtype) == Union: + raise RuntimeError( + f"Union type is not supported for column: {col_name}. " + f"Please pass a PCollection with valid schema for column " + f"{col_name} by passing a single type " + "in container. For example, List[int].") + elif issubclass(typ, np.generic) or typ in _default_type_to_tensor_type_map: + dtype = typ + else: + raise TypeError( + f"Unable to identify type: {typ} specified on column: {col_name}. " + f"Please provide a valid type from the following: " + f"{_default_type_to_tensor_type_map.keys()}") + return tf.io.VarLenFeature(_default_type_to_tensor_type_map[dtype]) + + def get_raw_data_metadata( + self, input_types: Dict[str, type]) -> dataset_metadata.DatasetMetadata: + raw_data_feature_spec = self.get_raw_data_feature_spec(input_types) + raw_data_feature_spec['hash_key'] = tf.io.VarLenFeature(dtype=tf.string) + return self.convert_raw_data_feature_spec_to_dataset_metadata( + raw_data_feature_spec) + + def write_transform_artifacts(self, transform_fn, location): + """ + Write transform artifacts to the given location. + Args: + transform_fn: A transform_fn object. + location: A location to write the artifacts. + Returns: + A PCollection of WriteTransformFn writing a TF transform graph. + """ + return ( + transform_fn + | 'Write Transform Artifacts' >> + transform_fn_io.WriteTransformFn(location)) + + def _fail_on_non_default_windowing(self, pcoll: beam.PCollection): + if not pcoll.windowing.is_default(): + raise RuntimeError( + "MLTransform only supports GlobalWindows when producing " + "artifacts such as min, max, variance etc over the dataset." + "Please use beam.WindowInto(beam.transforms.window.GlobalWindows()) " + "to convert your PCollection to GlobalWindow.") + + def process_data_fn( + self, inputs: Dict[str, common_types.ConsistentTensorType] + ) -> Dict[str, common_types.ConsistentTensorType]: + """ + This method is used in the AnalyzeAndTransformDataset step. It applies + the transforms to the `inputs` in sequential order on the columns + provided for a given transform. + Args: + inputs: A dictionary of column names and data. + Returns: + A dictionary of column names and transformed data. + """ + outputs = inputs.copy() + for transform in self.transforms: + columns = transform.columns + for col in columns: + intermediate_result = transform(outputs[col], output_column_name=col) + for key, value in intermediate_result.items(): + outputs[key] = value + return outputs + + def _get_transformed_data_schema( + self, + metadata: dataset_metadata.DatasetMetadata, + ): + schema = metadata._schema + transformed_types = {} + for feature in schema.feature: + name = feature.name + feature_type = feature.type + if feature_type == schema_pb2.FeatureType.FLOAT: + transformed_types[name] = typing.Sequence[np.float32] + elif feature_type == schema_pb2.FeatureType.INT: + transformed_types[name] = typing.Sequence[np.int64] # type: ignore[assignment] + else: + transformed_types[name] = typing.Sequence[bytes] # type: ignore[assignment] + return transformed_types + + def process_data( + self, raw_data: beam.PCollection[tft_process_handler_input_type] + ) -> beam.PCollection[tft_process_handler_output_type]: + """ + This method also computes the required dataset metadata for the tft + AnalyzeDataset/TransformDataset step. + + This method uses tensorflow_transform's Analyze step to produce the + artifacts and Transform step to apply the transforms on the data. + Artifacts are only produced if the artifact_mode is set to `produce`. + If artifact_mode is set to `consume`, then the artifacts are read from the + artifact_location, which was previously used to store the produced + artifacts. + """ + + if self.artifact_mode == ArtifactMode.PRODUCE: + # If we are computing artifacts, we should fail for windows other than + # default windowing since for example, for a fixed window, each window can + # be treated as a separate dataset and we might need to compute artifacts + # for each window. This is not supported yet. + self._fail_on_non_default_windowing(raw_data) + element_type = raw_data.element_type + column_type_mapping = {} + if (isinstance(element_type, RowTypeConstraint) or + native_type_compatibility.match_is_named_tuple(element_type)): + column_type_mapping = self._map_column_names_to_types( + row_type=element_type) + # convert Row or NamedTuple to Dict + raw_data = ( + raw_data + | ConvertNamedTupleToDict().with_output_types( + Dict[str, typing.Union[tuple(column_type_mapping.values())]])) # type: ignore + # AnalyzeAndTransformDataset raise type hint since this is + # schema'd PCollection and the current output type would be a + # custom type(NamedTuple) or a beam.Row type. + else: + column_type_mapping = self._map_column_names_to_types_from_transforms() + # Add hash key so TFT can output hash_key as output but as a no-op. + raw_data_metadata = self.get_raw_data_metadata( + input_types=column_type_mapping) + # Write untransformed metadata to a file so that it can be re-used + # during Transform step. + metadata_io.write_metadata( + metadata=raw_data_metadata, + path=os.path.join(self.artifact_location, RAW_DATA_METADATA_DIR)) + else: + # Read the metadata from the artifact_location. + if not os.path.exists(os.path.join( + self.artifact_location, RAW_DATA_METADATA_DIR, SCHEMA_FILE)): + raise FileNotFoundError( + "Artifacts not found at location: %s when using " + "read_artifact_location. Make sure you've run the pipeline with " + "write_artifact_location using this artifact location before " + "running with read_artifact_location set." % + os.path.join(self.artifact_location, RAW_DATA_METADATA_DIR)) + raw_data_metadata = metadata_io.read_metadata( + os.path.join(self.artifact_location, RAW_DATA_METADATA_DIR)) + + keyed_raw_data = (raw_data | beam.ParDo(ComputeAndAttachHashKey())) + + feature_set = [feature.name for feature in raw_data_metadata.schema.feature] + columns_not_in_schema_with_hash = ( + keyed_raw_data + | beam.ParDo(GetMissingColumnsPColl(feature_set))) + + # To maintain consistency by outputting numpy array all the time, + # whether a scalar value or list or np array is passed as input, + # we will convert scalar values to list values and TFT will ouput + # numpy array all the time. + keyed_raw_data = keyed_raw_data | beam.ParDo( + ConvertScalarValuesToListValues()) + + raw_data_list = (keyed_raw_data | beam.ParDo(MakeHashKeyAsColumn())) + + with tft_beam.Context(temp_dir=self.artifact_location): + data = (raw_data_list, raw_data_metadata) + if self.artifact_mode == ArtifactMode.PRODUCE: + transform_fn = ( + data + | "AnalyzeDataset" >> tft_beam.AnalyzeDataset(self.process_data_fn)) + # TODO: Remove the 'hash_key' column from the transformed + # dataset schema generated by TFT. + self.write_transform_artifacts(transform_fn, self.artifact_location) + else: + transform_fn = ( + raw_data_list.pipeline + | "ReadTransformFn" >> tft_beam.ReadTransformFn( + self.artifact_location)) + (transformed_dataset, transformed_metadata) = ( + (data, transform_fn) + | "TransformDataset" >> tft_beam.TransformDataset()) + + if isinstance(transformed_metadata, beam_metadata_io.BeamDatasetMetadata): + self.transformed_schema = self._get_transformed_data_schema( + metadata=transformed_metadata.dataset_metadata) + else: + self.transformed_schema = self._get_transformed_data_schema( + transformed_metadata) + + # We will a pass a schema'd PCollection to the next step. + # So we will use a RowTypeConstraint to create a schema'd PCollection. + # this is needed since new columns are included in the + # transformed_dataset. + del self.transformed_schema['hash_key'] + row_type = RowTypeConstraint.from_fields( + list(self.transformed_schema.items())) + + # If a non schema PCollection is passed, and one of the input columns + # is not transformed by any of the transforms, then the output will + # not have that column. So we will join the missing columns from the + # raw_data to the transformed_dataset. + transformed_dataset = ( + transformed_dataset | beam.ParDo(ExtractHashAndKeyPColl())) + + # The grouping is needed here since tensorflow transform only outputs + # columns that are transformed by any of the transforms. So we will + # join the missing columns from the raw_data to the transformed_dataset + # using the hash key. + transformed_dataset = ( + (transformed_dataset, columns_not_in_schema_with_hash) + | beam.CoGroupByKey() + | beam.ParDo(MergeDicts())) + + # The schema only contains the columns that are transformed. + transformed_dataset = ( + transformed_dataset | "ConvertToRowType" >> + beam.Map(lambda x: beam.Row(**x)).with_output_types(row_type)) + + return transformed_dataset diff --git a/sdks/python/apache_beam/ml/transforms/handlers_test.py b/sdks/python/apache_beam/ml/transforms/handlers_test.py new file mode 100644 index 0000000000000..3342ec76cae59 --- /dev/null +++ b/sdks/python/apache_beam/ml/transforms/handlers_test.py @@ -0,0 +1,592 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pytype: skip-file + +import os +import shutil +import sys +import tempfile +import typing +import unittest +from typing import List +from typing import NamedTuple +from typing import Union + +import numpy as np +from parameterized import parameterized + +import apache_beam as beam + +# pylint: disable=wrong-import-position, ungrouped-imports +try: + from apache_beam.ml.transforms import handlers + from apache_beam.ml.transforms import tft + from apache_beam.ml.transforms.tft import TFTOperation + from apache_beam.testing.util import assert_that + from apache_beam.testing.util import equal_to + import tensorflow as tf + from tensorflow_transform.tf_metadata import dataset_metadata + from tensorflow_transform.tf_metadata import schema_utils +except ImportError: + tft = None # type: ignore[assignment] + +if not tft: + raise unittest.SkipTest('tensorflow_transform is not installed.') + + +class _AddOperation(TFTOperation): + def apply_transform(self, inputs, output_column_name, **kwargs): + return {output_column_name: inputs + 1} + + +class _MultiplyOperation(TFTOperation): + def apply_transform(self, inputs, output_column_name, **kwargs): + return {output_column_name: inputs * 10} + + +class _FakeOperationWithArtifacts(TFTOperation): + def apply_transform(self, inputs, output_column_name, **kwargs): + return {output_column_name: inputs} + + def get_artifacts(self, data, col_name): + return {'artifact': tf.convert_to_tensor([1])} + + +class IntType(NamedTuple): + x: int + + +class ListIntType(NamedTuple): + x: List[int] + + +class NumpyType(NamedTuple): + x: np.int64 + + +class TFTProcessHandlerTest(unittest.TestCase): + def setUp(self) -> None: + self.artifact_location = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.artifact_location) + + @parameterized.expand([ + ({ + 'x': 1, 'y': 2 + }, ['x'], { + 'x': 20, 'y': 2 + }), + ({ + 'x': 1, 'y': 2 + }, ['x', 'y'], { + 'x': 20, 'y': 30 + }), + ]) + def test_tft_operation_preprocessing_fn( + self, inputs, columns, expected_result): + add_fn = _AddOperation(columns=columns) + mul_fn = _MultiplyOperation(columns=columns) + process_handler = handlers.TFTProcessHandler( + transforms=[add_fn, mul_fn], artifact_location=self.artifact_location) + actual_result = process_handler.process_data_fn(inputs) + self.assertDictEqual(actual_result, expected_result) + + def test_preprocessing_fn_with_artifacts(self): + process_handler = handlers.TFTProcessHandler( + transforms=[_FakeOperationWithArtifacts(columns=['x'])], + artifact_location=self.artifact_location) + inputs = {'x': [1, 2, 3]} + preprocessing_fn = process_handler.process_data_fn + actual_result = preprocessing_fn(inputs) + expected_result = {'x': [1, 2, 3], 'artifact': tf.convert_to_tensor([1])} + self.assertDictEqual(actual_result, expected_result) + + def test_input_type_from_schema_named_tuple_pcoll(self): + data = [{'x': 1}] + with beam.Pipeline() as p: + data = ( + p | beam.Create(data) + | beam.Map(lambda x: IntType(**x)).with_output_types(IntType)) + element_type = data.element_type + process_handler = handlers.TFTProcessHandler( + artifact_location=self.artifact_location) + inferred_input_type = process_handler._map_column_names_to_types( + element_type) + expected_input_type = dict(x=List[int]) + + self.assertEqual(inferred_input_type, expected_input_type) + + def test_input_type_from_schema_named_tuple_pcoll_list(self): + data = [{'x': [1, 2, 3]}, {'x': [4, 5, 6]}] + with beam.Pipeline() as p: + data = ( + p | beam.Create(data) + | beam.Map(lambda x: ListIntType(**x)).with_output_types(ListIntType)) + element_type = data.element_type + process_handler = handlers.TFTProcessHandler( + artifact_location=self.artifact_location) + inferred_input_type = process_handler._map_column_names_to_types( + element_type) + expected_input_type = dict(x=List[int]) + self.assertEqual(inferred_input_type, expected_input_type) + + def test_input_type_from_row_type_pcoll(self): + data = [{'x': 1}] + with beam.Pipeline() as p: + data = ( + p | beam.Create(data) + | beam.Map(lambda ele: beam.Row(x=int(ele['x'])))) + element_type = data.element_type + process_handler = handlers.TFTProcessHandler( + artifact_location=self.artifact_location) + inferred_input_type = process_handler._map_column_names_to_types( + element_type) + expected_input_type = dict(x=List[int]) + self.assertEqual(inferred_input_type, expected_input_type) + + def test_input_type_from_row_type_pcoll_list(self): + data = [{'x': [1, 2, 3]}, {'x': [4, 5, 6]}] + with beam.Pipeline() as p: + data = ( + p | beam.Create(data) + | beam.Map(lambda ele: beam.Row(x=list(ele['x']))).with_output_types( + beam.row_type.RowTypeConstraint.from_fields([('x', List[int])]))) + + element_type = data.element_type + process_handler = handlers.TFTProcessHandler( + artifact_location=self.artifact_location) + inferred_input_type = process_handler._map_column_names_to_types( + element_type) + expected_input_type = dict(x=List[int]) + self.assertEqual(inferred_input_type, expected_input_type) + + def test_input_type_from_named_tuple_pcoll_numpy(self): + np_data = [{ + 'x': np.array([1, 2, 3], dtype=np.int64) + }, { + 'x': np.array([4, 5, 6], dtype=np.int64) + }] + with beam.Pipeline() as p: + data = ( + p | beam.Create(np_data) + | beam.Map(lambda x: NumpyType(**x)).with_output_types(NumpyType)) + element_type = data.element_type + process_handler = handlers.TFTProcessHandler( + artifact_location=self.artifact_location) + inferred_input_type = process_handler._map_column_names_to_types( + element_type) + expected_type = dict(x=np.int64) + self.assertEqual(inferred_input_type, expected_type) + + def test_tensorflow_raw_data_metadata_primitive_types(self): + input_types = dict(x=int, y=float, k=bytes, l=str) + process_handler = handlers.TFTProcessHandler( + artifact_location=self.artifact_location) + + for col_name, typ in input_types.items(): + feature_spec = process_handler._get_raw_data_feature_spec_per_column( + typ=typ, col_name=col_name) + self.assertEqual( + handlers._default_type_to_tensor_type_map[typ], feature_spec.dtype) + self.assertIsInstance(feature_spec, tf.io.VarLenFeature) + + def test_tensorflow_raw_data_metadata_primitive_types_in_containers(self): + input_types = dict([("x", List[int]), ("y", List[float]), + ("k", List[bytes]), ("l", List[str])]) + process_handler = handlers.TFTProcessHandler( + artifact_location=self.artifact_location) + for col_name, typ in input_types.items(): + feature_spec = process_handler._get_raw_data_feature_spec_per_column( + typ=typ, col_name=col_name) + self.assertIsInstance(feature_spec, tf.io.VarLenFeature) + + @unittest.skipIf(sys.version_info < (3, 9), "not supported in python<3.9") + def test_tensorflow_raw_data_metadata_primitive_native_container_types(self): + input_types = dict([("x", list[int]), ("y", list[float]), + ("k", list[bytes]), ("l", list[str])]) + process_handler = handlers.TFTProcessHandler( + artifact_location=self.artifact_location) + for col_name, typ in input_types.items(): + feature_spec = process_handler._get_raw_data_feature_spec_per_column( + typ=typ, col_name=col_name) + self.assertIsInstance(feature_spec, tf.io.VarLenFeature) + + def test_tensorflow_raw_data_metadata_numpy_types(self): + input_types = dict(x=np.int64, y=np.float32, z=List[np.int64]) + process_handler = handlers.TFTProcessHandler( + artifact_location=self.artifact_location) + for col_name, typ in input_types.items(): + feature_spec = process_handler._get_raw_data_feature_spec_per_column( + typ=typ, col_name=col_name) + self.assertIsInstance(feature_spec, tf.io.VarLenFeature) + + def test_tensorflow_raw_data_metadata_union_type_in_single_column(self): + input_types = dict(x=Union[int, float]) + process_handler = handlers.TFTProcessHandler( + artifact_location=self.artifact_location) + with self.assertRaises(TypeError): + for col_name, typ in input_types.items(): + _ = process_handler._get_raw_data_feature_spec_per_column( + typ=typ, col_name=col_name) + + def test_tensorflow_raw_data_metadata_dtypes(self): + input_types = dict(x=np.int32, y=np.float64) + expected_dtype = dict(x=np.int64, y=np.float32) + process_handler = handlers.TFTProcessHandler( + artifact_location=self.artifact_location) + for col_name, typ in input_types.items(): + feature_spec = process_handler._get_raw_data_feature_spec_per_column( + typ=typ, col_name=col_name) + self.assertEqual(expected_dtype[col_name], feature_spec.dtype) + + def test_tft_process_handler_default_transform_types(self): + transforms = [ + tft.ScaleTo01(columns=['x']), + tft.ScaleToZScore(columns=['y']), + tft.Bucketize(columns=['z'], num_buckets=2), + tft.ComputeAndApplyVocabulary(columns=['w']) + ] + process_handler = handlers.TFTProcessHandler( + transforms=transforms, artifact_location=self.artifact_location) + column_type_mapping = ( + process_handler._map_column_names_to_types_from_transforms()) + expected_column_type_mapping = { + 'x': float, 'y': float, 'z': float, 'w': str + } + self.assertDictEqual(column_type_mapping, expected_column_type_mapping) + + expected_tft_raw_data_feature_spec = { + 'x': tf.io.VarLenFeature(tf.float32), + 'y': tf.io.VarLenFeature(tf.float32), + 'z': tf.io.VarLenFeature(tf.float32), + 'w': tf.io.VarLenFeature(tf.string) + } + actual_tft_raw_data_feature_spec = ( + process_handler.get_raw_data_feature_spec(column_type_mapping)) + self.assertDictEqual( + actual_tft_raw_data_feature_spec, expected_tft_raw_data_feature_spec) + + def test_tft_process_handler_transformed_data_schema(self): + process_handler = handlers.TFTProcessHandler( + artifact_location=self.artifact_location) + raw_data_feature_spec = { + 'x': tf.io.VarLenFeature(tf.float32), + 'y': tf.io.VarLenFeature(tf.float32), + 'z': tf.io.VarLenFeature(tf.string), + } + raw_data_metadata = dataset_metadata.DatasetMetadata( + schema_utils.schema_from_feature_spec(raw_data_feature_spec)) + + expected_transformed_data_schema = { + 'x': typing.Sequence[np.float32], + 'y': typing.Sequence[np.float32], + 'z': typing.Sequence[bytes] + } + + actual_transformed_data_schema = ( + process_handler._get_transformed_data_schema(raw_data_metadata)) + self.assertDictEqual( + actual_transformed_data_schema, expected_transformed_data_schema) + + def test_tft_process_handler_verify_artifacts(self): + with beam.Pipeline() as p: + raw_data = ( + p + | beam.Create([{ + 'x': np.array([1, 3]) + }, { + 'x': np.array([4, 6]) + }])) + process_handler = handlers.TFTProcessHandler( + transforms=[tft.ScaleTo01(columns=['x'])], + artifact_location=self.artifact_location, + ) + _ = process_handler.process_data(raw_data) + + self.assertTrue( + os.path.exists( + os.path.join( + self.artifact_location, handlers.RAW_DATA_METADATA_DIR))) + self.assertTrue( + os.path.exists( + os.path.join( + self.artifact_location, + handlers.RAW_DATA_METADATA_DIR, + handlers.SCHEMA_FILE))) + + with beam.Pipeline() as p: + raw_data = (p | beam.Create([{'x': np.array([2, 5])}])) + process_handler = handlers.TFTProcessHandler( + artifact_location=self.artifact_location, artifact_mode='consume') + transformed_data = process_handler.process_data(raw_data) + transformed_data |= beam.Map(lambda x: x.x) + + # the previous min is 1 and max is 6. So this should scale by (1, 6) + assert_that( + transformed_data, + equal_to([np.array([0.2, 0.8], dtype=np.float32)], + equals_fn=np.array_equal)) + + def test_tft_process_handler_unused_column(self): + data = [ + { + 'x': [5, 1], 'y': 2 + }, + { + 'x': [8, 4], 'y': 5 + }, + { + 'x': [9, 5], 'y': 6 + }, + { + 'x': [10, 6], 'y': 7 + }, + { + 'x': [11, 7], 'y': 8 + }, + { + 'x': [12, 8], 'y': 9 + }, + { + 'x': [13, 9], 'y': 10 + }, + { + 'x': [14, 10], 'y': 11 + }, + { + 'x': [15, 11], 'y': 12 + }, + { + 'x': [16, 12], 'y': 13 + }, + { + 'x': [17, 13], 'y': 14 + }, + { + 'x': [18, 14], 'y': 15 + }, + { + 'x': [19, 15], 'y': 16 + }, + { + 'x': [20, 16], 'y': 17 + }, + { + 'x': [21, 17], 'y': 18 + }, + { + 'x': [22, 18], 'y': 19 + }, + { + 'x': [23, 19], 'y': 20 + }, + { + 'x': [24, 20], 'y': 21 + }, + { + 'x': [25, 21], 'y': 22 + }, + ] + # pylint: disable=line-too-long + expected_data = [ + beam.Row( + x=np.array([0.16666667, 0.], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=2), + beam.Row( + x=np.array([0.29166666, 0.125], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=5), + beam.Row( + x=np.array([0.33333334, 0.16666667], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=6), + beam.Row( + x=np.array([0.375, 0.20833333], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=7), + beam.Row( + x=np.array([0.41666666, 0.25], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=8), + beam.Row( + x=np.array([0.45833334, 0.29166666], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=9), + beam.Row( + x=np.array([0.5, 0.33333334], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=10), + beam.Row( + x=np.array([0.5416667, 0.375], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=11), + beam.Row( + x=np.array([0.5833333, 0.41666666], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=12), + beam.Row( + x=np.array([0.625, 0.45833334], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=13), + beam.Row( + x=np.array([0.6666667, 0.5], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=14), + beam.Row( + x=np.array([0.7083333, 0.5416667], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=15), + beam.Row( + x=np.array([0.75, 0.5833333], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=16), + beam.Row( + x=np.array([0.7916667, 0.625], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=17), + beam.Row( + x=np.array([0.8333333, 0.6666667], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=18), + beam.Row( + x=np.array([0.875, 0.7083333], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=19), + beam.Row( + x=np.array([0.9166667, 0.75], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=20), + beam.Row( + x=np.array([0.9583333, 0.7916667], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=21), + beam.Row( + x=np.array([1., 0.8333333], dtype=np.float32), + x_max=np.array([25.], dtype=np.float32), + x_min=np.array([1.], dtype=np.float32), + y=22), + ] + # pylint: enable=line-too-long + + expected_data_x = [row.x for row in expected_data] + expected_data_y = [row.y for row in expected_data] + + scale_to_0_1_fn = tft.ScaleTo01(columns=['x']) + with beam.Pipeline() as p: + raw_data = p | beam.Create(data) + process_handler = handlers.TFTProcessHandler( + transforms=[scale_to_0_1_fn], + artifact_location=self.artifact_location, + ) + transformed_pcoll = process_handler.process_data(raw_data) + transformed_pcoll_x = transformed_pcoll | beam.Map(lambda x: x.x) + transformed_pcoll_y = transformed_pcoll | beam.Map(lambda x: x.y) + assert_that( + transformed_pcoll_x, + equal_to(expected_data_x, equals_fn=np.array_equal), + label='transformed data') + + assert_that( + transformed_pcoll_y, + equal_to(expected_data_y, equals_fn=np.array_equal), + label='unused column') + + def test_consume_mode_with_extra_columns_in_the_input(self): + with beam.Pipeline() as p: + raw_data = ( + p + | beam.Create([{ + 'x': np.array([1, 3]) + }, { + 'x': np.array([4, 6]) + }])) + process_handler = handlers.TFTProcessHandler( + transforms=[tft.ScaleTo01(columns=['x'])], + artifact_location=self.artifact_location, + ) + _ = process_handler.process_data(raw_data) + + test_data = [{ + 'x': np.array([2, 5]), 'y': np.array([1, 2]), 'z': 'fake_string' + }, + { + 'x': np.array([1, 10]), + 'y': np.array([2, 3]), + 'z': 'fake_string2' + }] + expected_test_data = [{ + 'x': np.array([0.2, 0.8], dtype=np.float32), + 'y': np.array([1, 2]), + 'z': 'fake_string' + }, + { + 'x': np.array([0., 1.8], dtype=np.float32), + 'y': np.array([2, 3]), + 'z': 'fake_string2' + }] + + expected_test_data_x = [row['x'] for row in expected_test_data] + expected_test_data_y = [row['y'] for row in expected_test_data] + expected_test_data_z = [row['z'] for row in expected_test_data] + with beam.Pipeline() as p: + raw_data = (p | beam.Create(test_data)) + process_handler = handlers.TFTProcessHandler( + artifact_location=self.artifact_location, artifact_mode='consume') + transformed_data = process_handler.process_data(raw_data) + + transformed_data_x = transformed_data | beam.Map(lambda x: x.x) + transformed_data_y = transformed_data | beam.Map(lambda x: x.y) + transformed_data_z = transformed_data | beam.Map(lambda x: x.z) + + assert_that( + transformed_data_x, + equal_to(expected_test_data_x, equals_fn=np.array_equal), + label='transformed data') + + assert_that( + transformed_data_y, + equal_to(expected_test_data_y, equals_fn=np.array_equal), + label='unused column: y') + + assert_that( + transformed_data_z, + equal_to(expected_test_data_z, equals_fn=np.array_equal), + label='unused column: z') + + +if __name__ == '__main__': + unittest.main() diff --git a/sdks/python/apache_beam/ml/transforms/tft.py b/sdks/python/apache_beam/ml/transforms/tft.py new file mode 100644 index 0000000000000..1d492642cd60e --- /dev/null +++ b/sdks/python/apache_beam/ml/transforms/tft.py @@ -0,0 +1,637 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module defines a set of data processing transforms that can be used +to perform common data transformations on a dataset. These transforms are +implemented using the TensorFlow Transform (TFT) library. The transforms +in this module are intended to be used in conjunction with the +MLTransform class, which provides a convenient interface for +applying a sequence of data processing transforms to a dataset. + +See the documentation for MLTransform for more details. + +Note: The data processing transforms defined in this module don't +perform the transformation immediately. Instead, it returns a +configured operation object, which encapsulates the details of the +transformation. The actual computation takes place later in the Apache Beam +pipeline, after all transformations are set up and the pipeline is run. +""" + +# pytype: skip-file + +import logging +from typing import Any +from typing import Dict +from typing import Iterable +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union + +import tensorflow as tf +import tensorflow_transform as tft +from apache_beam.ml.transforms.base import BaseOperation +from tensorflow_transform import analyzers +from tensorflow_transform import common_types +from tensorflow_transform import tf_utils + +__all__ = [ + 'ComputeAndApplyVocabulary', + 'ScaleToZScore', + 'ScaleTo01', + 'ApplyBuckets', + 'Bucketize', + 'TFIDF', + 'TFTOperation', + 'ScaleByMinMax', + 'NGrams', + 'BagOfWords', +] + +# Register the expected input types for each operation +# this will be used to determine schema for the tft.AnalyzeDataset +_EXPECTED_TYPES: Dict[str, Union[int, str, float]] = {} + +_LOGGER = logging.getLogger(__name__) + + +def register_input_dtype(type): + def wrapper(fn): + _EXPECTED_TYPES[fn.__name__] = type + return fn + + return wrapper + + +class TFTOperation(BaseOperation[common_types.TensorType, + common_types.TensorType]): + def __init__(self, columns: List[str]) -> None: + """ + Base Operation class for TFT data processing transformations. + Processing logic for the transformation is defined in the + apply_transform() method. If you have a custom transformation that is not + supported by the existing transforms, you can extend this class + and implement the apply_transform() method. + Args: + columns: List of column names to apply the transformation. + """ + super().__init__(columns) + if not columns: + raise RuntimeError( + "Columns are not specified. Please specify the column for the " + " op %s" % self.__class__.__name__) + + def get_artifacts(self, data: common_types.TensorType, + col_name: str) -> Dict[str, common_types.TensorType]: + """ + Returns the artifacts generated by the operation. + """ + return {} + + @tf.function + def _split_string_with_delimiter(self, data, delimiter): + """ + only applicable to string columns. + """ + data = tf.sparse.to_dense(data) + # this method acts differently compared to tf.strings.split + # this will split the string based on multiple delimiters while + # the latter will split the string based on a single delimiter. + fn = lambda data: tf.compat.v1.string_split( + data, delimiter, result_type='RaggedTensor') + # tf.compat.v1.string_split works on a single string. Use tf.map_fn + # to apply the function on each element of the input data. + data = tf.map_fn( + fn, + data, + fn_output_signature=tf.RaggedTensorSpec( + tf.TensorShape([None, None]), tf.string)) + data = data.values.to_sparse() + # the columns of the sparse tensor are suffixed with $indices, $values + # related to sparse tensor. Create a new sparse tensor by extracting + # the indices, values and dense_shape from the original sparse tensor + # to preserve the original column name. + data = tf.sparse.SparseTensor( + indices=data.indices, values=data.values, dense_shape=data.dense_shape) + # for list of string, batch dimensions becomes inverted after tf.map_fn, + # transpose the data to get the original shape. + if tf.shape(data)[1] == 1: + data = tf.sparse.transpose(data) + return data + + +@register_input_dtype(str) +class ComputeAndApplyVocabulary(TFTOperation): + def __init__( + self, + columns: List[str], + split_string_by_delimiter: Optional[str] = None, + *, + default_value: Any = -1, + top_k: Optional[int] = None, + frequency_threshold: Optional[int] = None, + num_oov_buckets: int = 0, + vocab_filename: Optional[str] = None, + name: Optional[str] = None): + """ + This function computes the vocabulary for the given columns of incoming + data. The transformation converts the input values to indices of the + vocabulary. + + Args: + columns: List of column names to apply the transformation. + split_string_by_delimiter: (Optional) A string that specifies the + delimiter to split strings. + default_value: (Optional) The value to use for out-of-vocabulary values. + top_k: (Optional) The number of most frequent tokens to keep. + frequency_threshold: (Optional) Limit the generated vocabulary only to + elements whose absolute frequency is >= to the supplied threshold. + If set to None, the full vocabulary is generated. + num_oov_buckets: Any lookup of an out-of-vocabulary token will return a + bucket ID based on its hash if `num_oov_buckets` is greater than zero. + Otherwise it is assigned the `default_value`. + vocab_filename: The file name for the vocabulary file. If not provided, + the default name would be `compute_and_apply_vocab' + NOTE in order to make your pipelines resilient to implementation + details please set `vocab_filename` when you are using + the vocab_filename on a downstream component. + """ + super().__init__(columns) + self._default_value = default_value + self._top_k = top_k + self._frequency_threshold = frequency_threshold + self._num_oov_buckets = num_oov_buckets + self._vocab_filename = vocab_filename if vocab_filename else ( + 'compute_and_apply_vocab') + self._name = name + self.split_string_by_delimiter = split_string_by_delimiter + + def apply_transform( + self, data: common_types.TensorType, + output_column_name: str) -> Dict[str, common_types.TensorType]: + + if self.split_string_by_delimiter: + data = self._split_string_with_delimiter( + data, self.split_string_by_delimiter) + + return { + output_column_name: tft.compute_and_apply_vocabulary( + x=data, + default_value=self._default_value, + top_k=self._top_k, + frequency_threshold=self._frequency_threshold, + num_oov_buckets=self._num_oov_buckets, + vocab_filename=self._vocab_filename, + name=self._name) + } + + +@register_input_dtype(float) +class ScaleToZScore(TFTOperation): + def __init__( + self, + columns: List[str], + *, + elementwise: bool = False, + name: Optional[str] = None): + """ + This function performs a scaling transformation on the specified columns of + the incoming data. It processes the input data such that it's normalized + to have a mean of 0 and a variance of 1. The transformation achieves this + by subtracting the mean from the input data and then dividing it by the + square root of the variance. + + Args: + columns: A list of column names to apply the transformation on. + elementwise: If True, the transformation is applied elementwise. + Otherwise, the transformation is applied on the entire column. + name: A name for the operation (optional). + + scale_to_z_score also outputs additional artifacts. The artifacts are + mean, which is the mean value in the column, and var, which is the + variance in the column. The artifacts are stored in the column + named with the suffix _mean and _var + respectively. + """ + super().__init__(columns) + self.elementwise = elementwise + self.name = name + + def apply_transform( + self, data: common_types.TensorType, + output_column_name: str) -> Dict[str, common_types.TensorType]: + output_dict = { + output_column_name: tft.scale_to_z_score( + x=data, elementwise=self.elementwise, name=self.name) + } + return output_dict + + def get_artifacts(self, data: common_types.TensorType, + col_name: str) -> Dict[str, common_types.TensorType]: + mean_var = tft.analyzers._mean_and_var(data) + shape = [tf.shape(data)[0], 1] + return { + col_name + '_mean': tf.broadcast_to(mean_var[0], shape), + col_name + '_var': tf.broadcast_to(mean_var[1], shape), + } + + +@register_input_dtype(float) +class ScaleTo01(TFTOperation): + def __init__( + self, + columns: List[str], + elementwise: bool = False, + name: Optional[str] = None): + """ + This function applies a scaling transformation on the given columns + of incoming data. The transformation scales the input values to the + range [0, 1] by dividing each value by the maximum value in the + column. + + Args: + columns: A list of column names to apply the transformation on. + elementwise: If True, the transformation is applied elementwise. + Otherwise, the transformation is applied on the entire column. + name: A name for the operation (optional). + + ScaleTo01 also outputs additional artifacts. The artifacts are + max, which is the maximum value in the column, and min, which is the + minimum value in the column. The artifacts are stored in the column + named with the suffix _min and _max + respectively. + + """ + super().__init__(columns) + self.elementwise = elementwise + self.name = name + + def get_artifacts(self, data: common_types.TensorType, + col_name: str) -> Dict[str, common_types.TensorType]: + shape = [tf.shape(data)[0], 1] + return { + col_name + '_min': tf.broadcast_to(tft.min(data), shape), + col_name + '_max': tf.broadcast_to(tft.max(data), shape) + } + + def apply_transform( + self, data: common_types.TensorType, + output_column_name: str) -> Dict[str, common_types.TensorType]: + output = tft.scale_to_0_1( + x=data, elementwise=self.elementwise, name=self.name) + + output_dict = {output_column_name: output} + return output_dict + + +@register_input_dtype(float) +class ApplyBuckets(TFTOperation): + def __init__( + self, + columns: List[str], + bucket_boundaries: Iterable[Union[int, float]], + name: Optional[str] = None): + """ + This functions is used to map the element to a positive index i for + which bucket_boundaries[i-1] <= element < bucket_boundaries[i], + if it exists. If input < bucket_boundaries[0], then element is + mapped to 0. If element >= bucket_boundaries[-1], then element is + mapped to len(bucket_boundaries). NaNs are mapped to + len(bucket_boundaries). + + Args: + columns: A list of column names to apply the transformation on. + bucket_boundaries: A rank 2 Tensor or list representing the bucket + boundaries sorted in ascending order. + name: (Optional) A string that specifies the name of the operation. + """ + super().__init__(columns) + self.bucket_boundaries = [bucket_boundaries] + self.name = name + + def apply_transform( + self, data: common_types.TensorType, + output_column_name: str) -> Dict[str, common_types.TensorType]: + output = { + output_column_name: tft.apply_buckets( + x=data, bucket_boundaries=self.bucket_boundaries, name=self.name) + } + return output + + +@register_input_dtype(float) +class Bucketize(TFTOperation): + def __init__( + self, + columns: List[str], + num_buckets: int, + *, + epsilon: Optional[float] = None, + elementwise: bool = False, + name: Optional[str] = None): + """ + This function applies a bucketizing transformation on the given columns + of incoming data. The transformation splits the input data range into + a set of consecutive bins/buckets, and converts the input values to + bucket IDs (integers) where each ID corresponds to a particular bin. + + Args: + columns: List of column names to apply the transformation. + num_buckets: Number of buckets to be created. + epsilon: (Optional) A float number that specifies the error tolerance + when computing quantiles, so that we guarantee that any value x will + have a quantile q such that x is in the interval + [q - epsilon, q + epsilon] (or the symmetric interval for even + num_buckets). Must be greater than 0.0. + elementwise: (Optional) A boolean that specifies whether the quantiles + should be computed on an element-wise basis. If False, the quantiles + are computed globally. + name: (Optional) A string that specifies the name of the operation. + """ + super().__init__(columns) + self.num_buckets = num_buckets + self.epsilon = epsilon + self.elementwise = elementwise + self.name = name + + def get_artifacts(self, data: common_types.TensorType, + col_name: str) -> Dict[str, common_types.TensorType]: + num_buckets = self.num_buckets + epsilon = self.epsilon + elementwise = self.elementwise + + if num_buckets < 1: + raise ValueError('Invalid num_buckets %d' % num_buckets) + + if isinstance(data, (tf.SparseTensor, tf.RaggedTensor)) and elementwise: + raise ValueError( + 'bucketize requires `x` to be dense if `elementwise=True`') + + x_values = tf_utils.get_values(data) + + if epsilon is None: + # See explanation in args documentation for epsilon. + epsilon = min(1.0 / num_buckets, 0.01) + + quantiles = analyzers.quantiles( + x_values, num_buckets, epsilon, reduce_instance_dims=not elementwise) + shape = [ + tf.shape(data)[0], num_buckets - 1 if num_buckets > 1 else num_buckets + ] + # These quantiles are used as the bucket boundaries in the later stages. + # Should we change the prefix _quantiles to _bucket_boundaries? + return {col_name + '_quantiles': tf.broadcast_to(quantiles, shape)} + + def apply_transform( + self, data: common_types.TensorType, + output_column_name: str) -> Dict[str, common_types.TensorType]: + output = { + output_column_name: tft.bucketize( + x=data, + num_buckets=self.num_buckets, + epsilon=self.epsilon, + elementwise=self.elementwise, + name=self.name) + } + return output + + +@register_input_dtype(float) +class TFIDF(TFTOperation): + def __init__( + self, + columns: List[str], + vocab_size: Optional[int] = None, + smooth: bool = True, + name: Optional[str] = None, + ): + """ + This function applies a tf-idf transformation on the given columns + of incoming data. + + TFIDF outputs two artifacts for each column: the vocabu index and + the tfidf weight. The vocabu index is a mapping from the original + vocabulary to the new vocabulary. The tfidf weight is a mapping + from the original vocabulary to the tfidf score. + + Input passed to the TFIDF is not modified and used to calculate the + required artifacts. + + Args: + columns: List of column names to apply the transformation. + vocab_size: (Optional) An integer that specifies the size of the + vocabulary. Defaults to None. + + If vocab_size is None, then the size of the vocabulary is + determined by `tft.get_num_buckets_for_transformed_feature`. + smooth: (Optional) A boolean that specifies whether to apply + smoothing to the tf-idf score. Defaults to True. + name: (Optional) A string that specifies the name of the operation. + """ + super().__init__(columns) + self.vocab_size = vocab_size + self.smooth = smooth + self.name = name + self.tfidf_weight = None + + def apply_transform( + self, data: common_types.TensorType, + output_column_name: str) -> common_types.TensorType: + + if self.vocab_size is None: + try: + _LOGGER.info( + 'vocab_size is not specified. Trying to infer vocab_size ' + 'from the input data using ' + 'tft.get_num_buckets_for_transformed_feature.') + vocab_size = tft.get_num_buckets_for_transformed_feature(data) + except RuntimeError: + raise RuntimeError( + 'vocab_size is not specified. Tried to infer vocab_size from the ' + 'input data using tft.get_num_buckets_for_transformed_feature, but ' + 'failed. Please specify vocab_size explicitly.') + else: + vocab_size = self.vocab_size + + vocab_index, tfidf_weight = tft.tfidf( + data, + vocab_size, + self.smooth, + self.name + ) + + output = { + output_column_name + '_vocab_index': vocab_index, + output_column_name + '_tfidf_weight': tfidf_weight + } + return output + + +@register_input_dtype(float) +class ScaleByMinMax(TFTOperation): + def __init__( + self, + columns: List[str], + min_value: float = 0.0, + max_value: float = 1.0, + name: Optional[str] = None): + """ + This function applies a scaling transformation on the given columns + of incoming data. The transformation scales the input values to the + range [min_value, max_value]. + + Args: + columns: A list of column names to apply the transformation on. + min_value: The minimum value of the output range. + max_value: The maximum value of the output range. + name: A name for the operation (optional). + """ + super().__init__(columns) + self.min_value = min_value + self.max_value = max_value + self.name = name + + if self.max_value <= self.min_value: + raise ValueError('max_value must be greater than min_value') + + def apply_transform( + self, data: common_types.TensorType, + output_column_name: str) -> common_types.TensorType: + + output = tft.scale_by_min_max( + x=data, output_min=self.min_value, output_max=self.max_value) + return {output_column_name: output} + + +@register_input_dtype(str) +class NGrams(TFTOperation): + def __init__( + self, + columns: List[str], + split_string_by_delimiter: Optional[str] = None, + *, + ngram_range: Tuple[int, int] = (1, 1), + ngrams_separator: Optional[str] = None, + name: Optional[str] = None): + """ + An n-gram is a contiguous sequence of n items from a given sample of text + or speech. This operation applies an n-gram transformation to + specified columns of incoming data, splitting the input data into a + set of consecutive n-grams. + + Args: + columns: A list of column names to apply the transformation on. + split_string_by_delimiter: (Optional) A string that specifies the + delimiter to split the input strings before computing ngrams. + ngram_range: A tuple of integers(inclusive) specifying the range of + n-gram sizes. + ngrams_separator: A string that will be inserted between each ngram. + name: A name for the operation (optional). + """ + super().__init__(columns) + self.ngram_range = ngram_range + self.ngrams_separator = ngrams_separator + self.name = name + self.split_string_by_delimiter = split_string_by_delimiter + + if ngram_range != (1, 1) and not ngrams_separator: + raise ValueError( + 'ngrams_separator must be specified when ngram_range is not (1, 1)') + + def apply_transform( + self, data: common_types.TensorType, + output_column_name: str) -> Dict[str, common_types.TensorType]: + if self.split_string_by_delimiter: + data = self._split_string_with_delimiter( + data, self.split_string_by_delimiter) + output = tft.ngrams(data, self.ngram_range, self.ngrams_separator) + return {output_column_name: output} + + +@register_input_dtype(str) +class BagOfWords(TFTOperation): + def __init__( + self, + columns: List[str], + split_string_by_delimiter: Optional[str] = None, + *, + ngram_range: Tuple[int, int] = (1, 1), + ngrams_separator: Optional[str] = None, + compute_word_count: bool = False, + name: Optional[str] = None, + ): + """ + Bag of words contains the unique words present in the input text. + This operation applies a bag of words transformation to specified + columns of incoming data. Also, the transformation accepts a Tuple of + integers specifying the range of n-gram sizes. The transformation + splits the input data into a set of consecutive n-grams if ngram_range + is specified. The n-grams are then converted to a bag of words. + Also, you can specify a seperator string that will be inserted between + each ngram. + + Args: + columns: A list of column names to apply the transformation on. + split_string_by_delimiter: (Optional) A string that specifies the + delimiter to split the input strings before computing ngrams. + ngram_range: A tuple of integers(inclusive) specifying the range of + n-gram sizes. + seperator: A string that will be inserted between each ngram. + compute_word_count: A boolean that specifies whether to compute + the unique word count and add it as an artifact to the output. + Note that the count will be computed over the entire dataset so + it will be the same value for all inputs. + name: A name for the operation (optional). + + Note that original order of the input may not be preserved. + """ + + self.columns = columns + self.ngram_range = ngram_range + self.ngrams_separator = ngrams_separator + self.name = name + self.split_string_by_delimiter = split_string_by_delimiter + if compute_word_count: + self.compute_word_count_fn = count_unqiue_words + else: + self.compute_word_count_fn = lambda *args, **kwargs: {} + + if ngram_range != (1, 1) and not ngrams_separator: + raise ValueError( + 'ngrams_separator must be specified when ngram_range is not (1, 1)') + + def get_artifacts(self, data: tf.SparseTensor, + col_name: str) -> Dict[str, tf.Tensor]: + return self.compute_word_count_fn(data, col_name) + + def apply_transform(self, data: tf.SparseTensor, output_col_name: str): + if self.split_string_by_delimiter: + data = self._split_string_with_delimiter( + data, self.split_string_by_delimiter) + output = tft.bag_of_words( + data, self.ngram_range, self.ngrams_separator, self.name) + return {output_col_name: output} + + +def count_unqiue_words(data: tf.SparseTensor, + output_col_name: str) -> Dict[str, tf.Tensor]: + keys, count = tft.count_per_key(data) + shape = [tf.shape(data)[0], tf.shape(keys)[0]] + return { + output_col_name + '_unique_elements': tf.broadcast_to(keys, shape), + output_col_name + '_counts': tf.broadcast_to(count, shape) + } diff --git a/sdks/python/apache_beam/ml/transforms/tft_test.py b/sdks/python/apache_beam/ml/transforms/tft_test.py new file mode 100644 index 0000000000000..41f59c868c3bd --- /dev/null +++ b/sdks/python/apache_beam/ml/transforms/tft_test.py @@ -0,0 +1,768 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pytype: skip-file + +import shutil +import tempfile +import unittest + +import numpy as np +from parameterized import parameterized + +import apache_beam as beam +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to + +# pylint: disable=wrong-import-order, wrong-import-position +try: + from apache_beam.ml.transforms import base + from apache_beam.ml.transforms import tft +except ImportError: + tft = None # type: ignore[assignment] + +if not tft: + raise unittest.SkipTest('tensorflow_transform is not installed.') + +z_score_expected = {'x_mean': 3.5, 'x_var': 2.9166666666666665} + + +def assert_z_score_artifacts(element): + element = element.as_dict() + assert 'x_mean' in element + assert 'x_var' in element + assert element['x_mean'] == z_score_expected['x_mean'] + assert element['x_var'] == z_score_expected['x_var'] + + +def assert_ScaleTo01_artifacts(element): + element = element.as_dict() + assert 'x_min' in element + assert 'x_max' in element + assert element['x_min'] == 1 + assert element['x_max'] == 6 + + +def assert_bucketize_artifacts(element): + element = element.as_dict() + assert 'x_quantiles' in element + assert np.array_equal( + element['x_quantiles'], np.array([3, 5], dtype=np.float32)) + + +class ScaleZScoreTest(unittest.TestCase): + def setUp(self) -> None: + self.artifact_location = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.artifact_location) + + def test_z_score(self): + data = [ + { + 'x': 1 + }, + { + 'x': 2 + }, + { + 'x': 3 + }, + { + 'x': 4 + }, + { + 'x': 5 + }, + { + 'x': 6 + }, + ] + + with beam.Pipeline() as p: + result = ( + p + | "Create" >> beam.Create(data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location).with_transform( + tft.ScaleToZScore(columns=['x']))) + _ = (result | beam.Map(assert_z_score_artifacts)) + + def test_z_score_list_data(self): + list_data = [{'x': [1, 2, 3]}, {'x': [4, 5, 6]}] + with beam.Pipeline() as p: + list_result = ( + p + | "listCreate" >> beam.Create(list_data) + | "listMLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location).with_transform( + tft.ScaleToZScore(columns=['x']))) + _ = (list_result | beam.Map(assert_z_score_artifacts)) + + +class ScaleTo01Test(unittest.TestCase): + def setUp(self) -> None: + self.artifact_location = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.artifact_location) + + def test_ScaleTo01_list(self): + list_data = [{'x': [1, 2, 3]}, {'x': [4, 5, 6]}] + with beam.Pipeline() as p: + list_result = ( + p + | "listCreate" >> beam.Create(list_data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location).with_transform( + tft.ScaleTo01(columns=['x']))) + _ = (list_result | beam.Map(assert_ScaleTo01_artifacts)) + + expected_output = [ + np.array([0, 0.2, 0.4], dtype=np.float32), + np.array([0.6, 0.8, 1], dtype=np.float32) + ] + actual_output = (list_result | beam.Map(lambda x: x.x)) + assert_that( + actual_output, equal_to(expected_output, equals_fn=np.array_equal)) + + def test_ScaleTo01(self): + data = [{'x': 1}, {'x': 2}, {'x': 3}, {'x': 4}, {'x': 5}, {'x': 6}] + with beam.Pipeline() as p: + result = ( + p + | "Create" >> beam.Create(data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location).with_transform( + tft.ScaleTo01(columns=['x']))) + + _ = (result | beam.Map(assert_ScaleTo01_artifacts)) + expected_output = ( + np.array([0], dtype=np.float32), + np.array([0.2], dtype=np.float32), + np.array([0.4], dtype=np.float32), + np.array([0.6], dtype=np.float32), + np.array([0.8], dtype=np.float32), + np.array([1], dtype=np.float32)) + actual_output = (result | beam.Map(lambda x: x.x)) + assert_that( + actual_output, equal_to(expected_output, equals_fn=np.array_equal)) + + +class BucketizeTest(unittest.TestCase): + def setUp(self) -> None: + self.artifact_location = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.artifact_location) + + def test_bucketize(self): + data = [{'x': 1}, {'x': 2}, {'x': 3}, {'x': 4}, {'x': 5}, {'x': 6}] + with beam.Pipeline() as p: + result = ( + p + | "Create" >> beam.Create(data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location).with_transform( + tft.Bucketize(columns=['x'], num_buckets=3))) + _ = (result | beam.Map(assert_bucketize_artifacts)) + + transformed_data = (result | beam.Map(lambda x: x.x)) + expected_data = [ + np.array([0]), + np.array([0]), + np.array([1]), + np.array([1]), + np.array([2]), + np.array([2]) + ] + assert_that( + transformed_data, equal_to(expected_data, equals_fn=np.array_equal)) + + def test_bucketize_list(self): + list_data = [{'x': [1, 2, 3]}, {'x': [4, 5, 6]}] + with beam.Pipeline() as p: + list_result = ( + p + | "Create" >> beam.Create(list_data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location).with_transform( + tft.Bucketize(columns=['x'], num_buckets=3))) + _ = (list_result | beam.Map(assert_bucketize_artifacts)) + + transformed_data = ( + list_result + | "TransformedColumnX" >> beam.Map(lambda ele: ele.x)) + expected_data = [ + np.array([0, 0, 1], dtype=np.int64), + np.array([1, 2, 2], dtype=np.int64) + ] + assert_that( + transformed_data, equal_to(expected_data, equals_fn=np.array_equal)) + + @parameterized.expand([ + (range(1, 10), [4, 7]), + (range(9, 0, -1), [4, 7]), + (range(19, 0, -1), [10]), + (range(1, 100), [25, 50, 75]), + # similar to the above but with odd number of elements + (range(1, 100, 2), [25, 51, 75]), + (range(99, 0, -1), range(10, 100, 10)) + ]) + def test_bucketize_boundaries(self, test_input, expected_boundaries): + # boundaries are outputted as artifacts for the Bucketize transform. + data = [{'x': [i]} for i in test_input] + num_buckets = len(expected_boundaries) + 1 + with beam.Pipeline() as p: + result = ( + p + | "Create" >> beam.Create(data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location).with_transform( + tft.Bucketize(columns=['x'], num_buckets=num_buckets))) + actual_boundaries = ( + result + | beam.Map(lambda x: x.as_dict()) + | beam.Map(lambda x: x['x_quantiles'])) + + def assert_boundaries(actual_boundaries): + assert np.array_equal(actual_boundaries, expected_boundaries) + + _ = (actual_boundaries | beam.Map(assert_boundaries)) + + +class ApplyBucketsTest(unittest.TestCase): + def setUp(self) -> None: + self.artifact_location = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.artifact_location) + + @parameterized.expand([ + (range(1, 100), [25, 50, 75]), + (range(1, 100, 2), [25, 51, 75]), + ]) + def test_apply_buckets(self, test_inputs, bucket_boundaries): + with beam.Pipeline() as p: + data = [{'x': [i]} for i in test_inputs] + result = ( + p + | "Create" >> beam.Create(data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location).with_transform( + tft.ApplyBuckets( + columns=['x'], bucket_boundaries=bucket_boundaries))) + expected_output = [] + bucket = 0 + for x in sorted(test_inputs): + # Increment the bucket number when crossing the boundary + if (bucket < len(bucket_boundaries) and x >= bucket_boundaries[bucket]): + bucket += 1 + expected_output.append(np.array([bucket])) + + actual_output = (result | beam.Map(lambda x: x.x)) + assert_that( + actual_output, equal_to(expected_output, equals_fn=np.array_equal)) + + +class ComputeAndApplyVocabTest(unittest.TestCase): + def setUp(self) -> None: + self.artifact_location = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.artifact_location) + + def test_compute_and_apply_vocabulary_inputs(self): + num_elements = 100 + num_instances = num_elements + 1 + input_data = [{ + 'x': '%.10i' % i, # Front-padded to facilitate lexicographic sorting. + } for i in range(num_instances)] + + expected_data = [{ + 'x': (len(input_data) - 1) - i, # Due to reverse lexicographic sorting. + } for i in range(len(input_data))] + + with beam.Pipeline() as p: + actual_data = ( + p + | "Create" >> beam.Create(input_data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location).with_transform( + tft.ComputeAndApplyVocabulary(columns=['x']))) + actual_data |= beam.Map(lambda x: x.as_dict()) + + assert_that(actual_data, equal_to(expected_data)) + + def test_compute_and_apply_vocabulary(self): + num_elements = 100 + num_instances = num_elements + 1 + input_data = [ + { + 'x': ['%.10i' % i, '%.10i' % (i + 1), '%.10i' % (i + 2)], + # Front-padded to facilitate lexicographic sorting. + } for i in range(0, num_instances, 3) + ] + + # since we have 3 elements in a single list, multiply with 3 for + # each iteration i on the expected output. + excepted_data = [ + np.array([(len(input_data) * 3 - 1) - i, + (len(input_data) * 3 - 1) - i - 1, + (len(input_data) * 3 - 1) - i - 2], + dtype=np.int64) # Front-padded to facilitate lexicographic + # sorting. + for i in range(0, len(input_data) * 3, 3) + ] + + with beam.Pipeline() as p: + result = ( + p + | "Create" >> beam.Create(input_data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location).with_transform( + tft.ComputeAndApplyVocabulary(columns=['x']))) + actual_output = (result | beam.Map(lambda x: x.x)) + assert_that( + actual_output, equal_to(excepted_data, equals_fn=np.array_equal)) + + def test_with_basic_example_list(self): + data = [{ + 'x': ['I', 'like', 'pie'], + }, { + 'x': ['yum', 'yum', 'pie'], + }] + with beam.Pipeline() as p: + result = ( + p + | "Create" >> beam.Create(data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location).with_transform( + tft.ComputeAndApplyVocabulary(columns=['x']))) + result = result | beam.Map(lambda x: x.x) + expected_result = [np.array([3, 2, 1]), np.array([0, 0, 1])] + assert_that(result, equal_to(expected_result, equals_fn=np.array_equal)) + + def test_string_split_with_single_delimiter(self): + data = [{ + 'x': 'I like pie', + }, { + 'x': 'yum yum pie' + }] + with beam.Pipeline() as p: + result = ( + p + | "Create" >> beam.Create(data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location).with_transform( + tft.ComputeAndApplyVocabulary( + columns=['x'], split_string_by_delimiter=' '))) + result = result | beam.Map(lambda x: x.x) + expected_result = [np.array([3, 2, 1]), np.array([0, 0, 1])] + assert_that(result, equal_to(expected_result, equals_fn=np.array_equal)) + + def test_string_split_with_multiple_delimiters(self): + data = [{ + 'x': 'I like pie', + }, { + 'x': 'yum;yum;pie' + }, { + 'x': 'yum yum pie' + }] + + with beam.Pipeline() as p: + result = ( + p + | "Create" >> beam.Create(data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location).with_transform( + tft.ComputeAndApplyVocabulary( + columns=['x'], split_string_by_delimiter=' ;'))) + result = result | beam.Map(lambda x: x.x) + expected_result = [ + np.array([3, 2, 1]), np.array([0, 0, 1]), np.array([0, 0, 1]) + ] + assert_that(result, equal_to(expected_result, equals_fn=np.array_equal)) + + +class TFIDIFTest(unittest.TestCase): + def setUp(self) -> None: + self.artifact_location = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.artifact_location) + + def test_tfidf_compute_vocab_size_during_runtime(self): + raw_data = [ + dict(x=["I", "like", "pie", "pie", "pie"]), + dict(x=["yum", "yum", "pie"]) + ] + with beam.Pipeline() as p: + transforms = [ + tft.ComputeAndApplyVocabulary(columns=['x']), + tft.TFIDF(columns=['x']) + ] + actual_output = ( + p + | "Create" >> beam.Create(raw_data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location, + transforms=transforms)) + actual_output |= beam.Map(lambda x: x.as_dict()) + + def equals_fn(a, b): + is_equal = True + for key, value in a.items(): + value_b = a[key] + is_equal = is_equal and np.array_equal(value, value_b) + return is_equal + + expected_output = ([{ + 'x': np.array([3, 2, 0, 0, 0]), + 'x_tfidf_weight': np.array([0.6, 0.28109303, 0.28109303], + dtype=np.float32), + 'x_vocab_index': np.array([0, 2, 3], dtype=np.int64) + }, + { + 'x': np.array([1, 1, 0]), + 'x_tfidf_weight': np.array( + [0.33333334, 0.9369768], dtype=np.float32), + 'x_vocab_index': np.array([0, 1], dtype=np.int32) + }]) + assert_that(actual_output, equal_to(expected_output, equals_fn=equals_fn)) + + +class ScaleToMinMaxTest(unittest.TestCase): + def setUp(self) -> None: + self.artifact_location = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.artifact_location) + + def test_scale_to_min_max(self): + data = [{ + 'x': 4, + }, { + 'x': 1, + }, { + 'x': 5, + }, { + 'x': 2, + }] + with beam.Pipeline() as p: + result = ( + p + | "Create" >> beam.Create(data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location).with_transform( + tft.ScaleByMinMax( + columns=['x'], + min_value=-1, + max_value=1, + ), + )) + result = result | beam.Map(lambda x: x.as_dict()) + expected_data = [{ + 'x': np.array([0.5], dtype=np.float32) + }, { + 'x': np.array([-1.0], dtype=np.float32) + }, { + 'x': np.array([1.0], dtype=np.float32) + }, { + 'x': np.array([-0.5], dtype=np.float32) + }] + assert_that(result, equal_to(expected_data)) + + def test_fail_max_value_less_than_min(self): + with self.assertRaises(ValueError): + tft.ScaleByMinMax(columns=['x'], min_value=10, max_value=0) + + +class NGramsTest(unittest.TestCase): + def setUp(self) -> None: + self.artifact_location = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.artifact_location) + + def test_ngrams_on_list_separated_words_default_args(self): + data = [{ + 'x': ['I', 'like', 'pie'], + }, { + 'x': ['yum', 'yum', 'pie'] + }] + with beam.Pipeline() as p: + result = ( + p + | "Create" >> beam.Create(data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location, + transforms=[tft.NGrams(columns=['x'])])) + result = result | beam.Map(lambda x: x.x) + expected_data = [ + np.array([b'I', b'like', b'pie'], dtype=object), + np.array([b'yum', b'yum', b'pie'], dtype=object) + ] + assert_that(result, equal_to(expected_data, equals_fn=np.array_equal)) + + def test_ngrams_on_list_separated_words(self): + data = [{ + 'x': ['I', 'like', 'pie'], + }, { + 'x': ['yum', 'yum', 'pie'] + }] + with beam.Pipeline() as p: + result = ( + p + | "Create" >> beam.Create(data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location, + transforms=[ + tft.NGrams( + columns=['x'], ngram_range=(1, 3), ngrams_separator=' ') + ])) + result = result | beam.Map(lambda x: x.x) + expected_data = [ + np.array( + [b'I', b'I like', b'I like pie', b'like', b'like pie', b'pie'], + dtype=object), + np.array( + [b'yum', b'yum yum', b'yum yum pie', b'yum', b'yum pie', b'pie'], + dtype=object) + ] + assert_that(result, equal_to(expected_data, equals_fn=np.array_equal)) + + def test_with_string_split_delimiter(self): + data = [{ + 'x': 'I like pie', + }, { + 'x': 'yum yum pie' + }] + with beam.Pipeline() as p: + result = ( + p + | "Create" >> beam.Create(data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location, + transforms=[ + tft.NGrams( + columns=['x'], + split_string_by_delimiter=' ', + ngram_range=(1, 3), + ngrams_separator=' ') + ])) + result = result | beam.Map(lambda x: x.x) + + expected_data = [ + np.array( + [b'I', b'I like', b'I like pie', b'like', b'like pie', b'pie'], + dtype=object), + np.array( + [b'yum', b'yum yum', b'yum yum pie', b'yum', b'yum pie', b'pie'], + dtype=object) + ] + assert_that(result, equal_to(expected_data, equals_fn=np.array_equal)) + + def test_with_multiple_string_delimiters(self): + data = [{ + 'x': 'I?like?pie', + }, { + 'x': 'yum yum pie' + }] + with beam.Pipeline() as p: + result = ( + p + | "Create" >> beam.Create(data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location, + transforms=[ + tft.NGrams( + columns=['x'], + split_string_by_delimiter=' ?', + ngram_range=(1, 3), + ngrams_separator=' ') + ])) + result = result | beam.Map(lambda x: x.x) + + expected_data = [ + np.array( + [b'I', b'I like', b'I like pie', b'like', b'like pie', b'pie'], + dtype=object), + np.array( + [b'yum', b'yum yum', b'yum yum pie', b'yum', b'yum pie', b'pie'], + dtype=object) + ] + assert_that(result, equal_to(expected_data, equals_fn=np.array_equal)) + + +class BagOfWordsTest(unittest.TestCase): + def setUp(self) -> None: + self.artifact_location = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.artifact_location) + + def test_bag_of_words_on_list_seperated_words_default_ngrams(self): + data = [{ + 'x': ['I', 'like', 'pie', 'pie', 'pie'], + }, { + 'x': ['yum', 'yum', 'pie'] + }] + + with beam.Pipeline() as p: + result = ( + p + | "Create" >> beam.Create(data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location, + transforms=[tft.BagOfWords(columns=['x'])])) + result = result | beam.Map(lambda x: x.x) + + expected_data = [ + np.array([b'I', b'like', b'pie'], dtype=object), + np.array([b'yum', b'pie'], dtype=object) + ] + assert_that(result, equal_to(expected_data, equals_fn=np.array_equal)) + + def test_bag_of_words_on_list_seperated_words_custom_ngrams(self): + data = [{ + 'x': ['I', 'like', 'pie', 'I', 'like', 'pie'], + }, { + 'x': ['yum', 'yum', 'pie'] + }] + with beam.Pipeline() as p: + result = ( + p + | "Create" >> beam.Create(data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location, + transforms=[ + tft.BagOfWords( + columns=['x'], ngram_range=(1, 3), ngrams_separator=' ') + ])) + result = result | beam.Map(lambda x: x.x) + + expected_data = [[ + b'I', + b'I like', + b'I like pie', + b'like', + b'like pie', + b'like pie I', + b'pie', + b'pie I', + b'pie I like' + ], [b'yum', b'yum yum', b'yum yum pie', b'yum pie', b'pie']] + assert_that(result, equal_to(expected_data, equals_fn=np.array_equal)) + + def test_bag_of_words_on_numpy_data(self): + data = [{ + 'x': np.array(['I', 'like', 'pie', 'I', 'like', 'pie'], dtype=object), + }, { + 'x': np.array(['yum', 'yum', 'pie'], dtype=object) + }] + with beam.Pipeline() as p: + result = ( + p + | "Create" >> beam.Create(data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location, + transforms=[ + tft.BagOfWords( + columns=['x'], ngram_range=(1, 3), ngrams_separator=' ') + ])) + result = result | beam.Map(lambda x: x.x) + + expected_data = [[ + b'I', + b'I like', + b'I like pie', + b'like', + b'like pie', + b'like pie I', + b'pie', + b'pie I', + b'pie I like' + ], [b'yum', b'yum yum', b'yum yum pie', b'yum pie', b'pie']] + assert_that(result, equal_to(expected_data, equals_fn=np.array_equal)) + + def test_bag_of_words_on_by_splitting_input_text(self): + data = [{'x': 'I like pie I like pie'}, {'x': 'yum yum pie'}] + with beam.Pipeline() as p: + result = ( + p + | "Create" >> beam.Create(data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location, + transforms=[ + tft.BagOfWords( + columns=['x'], + split_string_by_delimiter=' ', + ngram_range=(1, 3), + ngrams_separator=' ') + ])) + result = result | beam.Map(lambda x: x.x) + + expected_data = [[ + b'I', + b'I like', + b'I like pie', + b'like', + b'like pie', + b'like pie I', + b'pie', + b'pie I', + b'pie I like' + ], [b'yum', b'yum yum', b'yum yum pie', b'yum pie', b'pie']] + assert_that(result, equal_to(expected_data, equals_fn=np.array_equal)) + + def test_count_per_key_on_list(self): + def map_element_to_count(elements, counts): + d = {elements[i]: counts[i] for i in range(len(elements))} + return d + + data = [{ + 'x': ['I', 'like', 'pie', 'pie', 'pie'], + }, { + 'x': ['yum', 'yum', 'pie'] + }, { + 'x': ['Banana', 'Banana', 'Apple', 'Apple', 'Apple', 'Apple'] + }] + with beam.Pipeline() as p: + result = ( + p + | "Create" >> beam.Create(data) + | "MLTransform" >> base.MLTransform( + write_artifact_location=self.artifact_location, + transforms=[ + tft.BagOfWords(columns=['x'], compute_word_count=True) + ])) + + # the unique elements and counts are artifacts and will be + # stored in the result and same for all the elements in the + # PCollection. + result = result | beam.Map( + lambda x: map_element_to_count(x.x_unique_elements, x.x_counts)) + + expected_data = [{ + b'Apple': 4, b'Banana': 2, b'I': 1, b'like': 1, b'pie': 4, b'yum': 2 + }] * 3 # since there are 3 elements in input. + assert_that(result, equal_to(expected_data)) + + +if __name__ == '__main__': + unittest.main() diff --git a/sdks/python/apache_beam/ml/transforms/utils.py b/sdks/python/apache_beam/ml/transforms/utils.py new file mode 100644 index 0000000000000..19bb02c5ae1b9 --- /dev/null +++ b/sdks/python/apache_beam/ml/transforms/utils.py @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +__all__ = ['ArtifactsFetcher'] + +import typing + +import tensorflow_transform as tft + + +class ArtifactsFetcher(): + """ + Utility class used to fetch artifacts from the artifact_location passed + to the TFTProcessHandlers in MLTransform. + """ + def __init__(self, artifact_location): + self.artifact_location = artifact_location + self.transform_output = tft.TFTransformOutput(self.artifact_location) + + def get_vocab_list( + self, + vocab_filename: str = 'compute_and_apply_vocab') -> typing.List[bytes]: + """ + Returns list of vocabulary terms created during MLTransform. + """ + try: + vocab_list = self.transform_output.vocabulary_by_name(vocab_filename) + except ValueError as e: + raise ValueError( + 'Vocabulary file {} not found in artifact location'.format( + vocab_filename)) from e + return [x.decode('utf-8') for x in vocab_list] + + def get_vocab_filepath( + self, vocab_filename: str = 'compute_and_apply_vocab') -> str: + """ + Return the path to the vocabulary file created during MLTransform. + """ + return self.transform_output.vocabulary_file_by_name(vocab_filename) + + def get_vocab_size( + self, vocab_filename: str = 'compute_and_apply_vocab') -> int: + return self.transform_output.vocabulary_size_by_name(vocab_filename) diff --git a/sdks/python/apache_beam/options/OWNERS b/sdks/python/apache_beam/options/OWNERS deleted file mode 100644 index 1a1f0f0d3e722..0000000000000 --- a/sdks/python/apache_beam/options/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - pabloem - - charlesccychen diff --git a/sdks/python/apache_beam/options/pipeline_options.py b/sdks/python/apache_beam/options/pipeline_options.py index dd162fcc098c6..3fbf7eff7dd62 100644 --- a/sdks/python/apache_beam/options/pipeline_options.py +++ b/sdks/python/apache_beam/options/pipeline_options.py @@ -101,7 +101,7 @@ def add_value_provider_argument(self, *args, **kwargs): key/value form. """ # Extract the option name from positional argument ['pos_arg'] - assert args != () and len(args[0]) >= 1 + assert args and len(args[0]) >= 1 if args[0][0] != '-': option_name = args[0] if kwargs.get('nargs') is None: # make them optionally templated @@ -533,6 +533,13 @@ def _add_argparse_args(cls, parser): 'Should be a json mapping of gradle build targets to pre-built ' 'artifacts (e.g. jar files) expansion endpoints (e.g. host:port).')) + parser.add_argument( + '--use_transform_service', + default=False, + action='store_true', + help='Use the Docker-composed-based transform service when expanding ' + 'cross-language transforms.') + def additional_option_ptransform_fn(): beam.transforms.ptransform.ptransform_fn_typehints_enabled = True @@ -843,24 +850,36 @@ def _create_default_gcs_bucket(self): else: return None + # If either temp or staging location has an issue, we use the valid one for + # both locations. If both are bad we return an error. + def _handle_temp_and_staging_locations(self, validator): + temp_errors = validator.validate_gcs_path(self, 'temp_location') + staging_errors = validator.validate_gcs_path(self, 'staging_location') + if temp_errors and not staging_errors: + setattr(self, 'temp_location', getattr(self, 'staging_location')) + return [] + elif staging_errors and not temp_errors: + setattr(self, 'staging_location', getattr(self, 'temp_location')) + return [] + elif not staging_errors and not temp_errors: + return [] + # Both staging and temp locations are bad, try to use default bucket. + else: + default_bucket = self._create_default_gcs_bucket() + if default_bucket is None: + temp_errors.extend(staging_errors) + return temp_errors + else: + setattr(self, 'temp_location', default_bucket) + setattr(self, 'staging_location', default_bucket) + return [] + def validate(self, validator): errors = [] if validator.is_service_runner(): + errors.extend(self._handle_temp_and_staging_locations(validator)) errors.extend(validator.validate_cloud_options(self)) - # Validating temp_location, or adding a default if there are issues - temp_location_errors = validator.validate_gcs_path(self, 'temp_location') - if temp_location_errors: - default_bucket = self._create_default_gcs_bucket() - if default_bucket is None: - errors.extend(temp_location_errors) - else: - setattr(self, 'temp_location', default_bucket) - - if getattr(self, 'staging_location', - None) or getattr(self, 'temp_location', None) is None: - errors.extend(validator.validate_gcs_path(self, 'staging_location')) - if self.view_as(DebugOptions).dataflow_job_file: if self.view_as(GoogleCloudOptions).template_location: errors.append( @@ -1144,8 +1163,7 @@ def _add_argparse_args(cls, parser): help=( 'Number of threads per worker to use on the runner. If left ' 'unspecified, the runner will compute an appropriate number of ' - 'threads to use. Currently only enabled for DataflowRunner when ' - 'experiment \'use_runner_v2\' is enabled.')) + 'threads to use.')) def add_experiment(self, experiment): # pylint: disable=access-member-before-definition @@ -1278,11 +1296,13 @@ def _add_argparse_args(cls, parser): '--sdk_location', default='default', help=( - 'Override the default location from where the Beam SDK is ' - 'downloaded. It can be a URL, a GCS path, or a local path to an ' + 'Path to a custom Beam SDK package to install and use on the' + 'runner. It can be a URL, a GCS path, or a local path to an ' 'SDK tarball. Workflow submissions will download or copy an SDK ' - 'tarball from here. If set to the string "default", a standard ' - 'SDK location is used. If empty, no SDK is copied.')) + 'tarball from here. If set to "default", ' + 'runners will use the SDK provided in the default environment.' + 'Use this flag when running pipelines with an unreleased or ' + 'manually patched version of Beam SDK.')) parser.add_argument( '--extra_package', '--extra_packages', diff --git a/sdks/python/apache_beam/options/pipeline_options_test.py b/sdks/python/apache_beam/options/pipeline_options_test.py index f83f703e33ba4..2d85d055eafb4 100644 --- a/sdks/python/apache_beam/options/pipeline_options_test.py +++ b/sdks/python/apache_beam/options/pipeline_options_test.py @@ -24,6 +24,7 @@ import unittest import hamcrest as hc +from parameterized import parameterized from apache_beam.options.pipeline_options import DebugOptions from apache_beam.options.pipeline_options import GoogleCloudOptions @@ -32,6 +33,7 @@ from apache_beam.options.pipeline_options import TypeOptions from apache_beam.options.pipeline_options import WorkerOptions from apache_beam.options.pipeline_options import _BeamArgumentParser +from apache_beam.options.pipeline_options_validator import PipelineOptionsValidator from apache_beam.options.value_provider import RuntimeValueProvider from apache_beam.options.value_provider import StaticValueProvider from apache_beam.transforms.display import DisplayData @@ -40,6 +42,25 @@ _LOGGER = logging.getLogger(__name__) +# Mock runners to use for validations. +class MockRunners(object): + class DataflowRunner(object): + def get_default_gcp_region(self): + # Return a default so we don't have to specify --region in every test + # (unless specifically testing it). + return 'us-central1' + + +class MockGoogleCloudOptionsNoBucket(GoogleCloudOptions): + def _create_default_gcs_bucket(self): + return None + + +class MockGoogleCloudOptionsWithBucket(GoogleCloudOptions): + def _create_default_gcs_bucket(self): + return "gs://default/bucket" + + class PipelineOptionsTest(unittest.TestCase): def setUp(self): # Reset runtime options to avoid side-effects caused by other tests. @@ -52,150 +73,104 @@ def tearDown(self): RuntimeValueProvider.set_runtime_options(None) TEST_CASES = [ - { - 'flags': ['--num_workers', '5'], - 'expected': { - 'num_workers': 5, - 'mock_flag': False, - 'mock_option': None, - 'mock_multi_option': None - }, - 'display_data': [DisplayDataItemMatcher('num_workers', 5)] - }, - { - 'flags': ['--direct_num_workers', '5'], - 'expected': { - 'direct_num_workers': 5, - 'mock_flag': False, - 'mock_option': None, - 'mock_multi_option': None - }, - 'display_data': [DisplayDataItemMatcher('direct_num_workers', 5)] - }, - { - 'flags': ['--direct_running_mode', 'multi_threading'], - 'expected': { - 'direct_running_mode': 'multi_threading', - 'mock_flag': False, - 'mock_option': None, - 'mock_multi_option': None - }, - 'display_data': [ - DisplayDataItemMatcher('direct_running_mode', 'multi_threading') - ] - }, - { - 'flags': ['--direct_running_mode', 'multi_processing'], - 'expected': { - 'direct_running_mode': 'multi_processing', - 'mock_flag': False, - 'mock_option': None, - 'mock_multi_option': None - }, - 'display_data': [ - DisplayDataItemMatcher('direct_running_mode', 'multi_processing') - ] - }, - { - 'flags': [ - '--profile_cpu', '--profile_location', 'gs://bucket/', 'ignored' - ], - 'expected': { - 'profile_cpu': True, - 'profile_location': 'gs://bucket/', - 'mock_flag': False, - 'mock_option': None, - 'mock_multi_option': None - }, - 'display_data': [ - DisplayDataItemMatcher('profile_cpu', True), - DisplayDataItemMatcher('profile_location', 'gs://bucket/') - ] - }, - { - 'flags': ['--num_workers', '5', '--mock_flag'], - 'expected': { - 'num_workers': 5, - 'mock_flag': True, - 'mock_option': None, - 'mock_multi_option': None - }, - 'display_data': [ - DisplayDataItemMatcher('num_workers', 5), - DisplayDataItemMatcher('mock_flag', True) - ] - }, - { - 'flags': ['--mock_option', 'abc'], - 'expected': { - 'mock_flag': False, - 'mock_option': 'abc', - 'mock_multi_option': None - }, - 'display_data': [DisplayDataItemMatcher('mock_option', 'abc')] - }, - { - 'flags': ['--mock_option', ' abc def '], - 'expected': { - 'mock_flag': False, - 'mock_option': ' abc def ', - 'mock_multi_option': None - }, - 'display_data': [DisplayDataItemMatcher('mock_option', ' abc def ')] - }, - { - 'flags': ['--mock_option= abc xyz '], - 'expected': { - 'mock_flag': False, - 'mock_option': ' abc xyz ', - 'mock_multi_option': None - }, - 'display_data': [DisplayDataItemMatcher('mock_option', ' abc xyz ')] - }, - { - 'flags': [ - '--mock_option=gs://my bucket/my folder/my file', - '--mock_multi_option=op1', - '--mock_multi_option=op2' - ], - 'expected': { - 'mock_flag': False, - 'mock_option': 'gs://my bucket/my folder/my file', - 'mock_multi_option': ['op1', 'op2'] - }, - 'display_data': [ - DisplayDataItemMatcher( - 'mock_option', 'gs://my bucket/my folder/my file'), - DisplayDataItemMatcher('mock_multi_option', ['op1', 'op2']) - ] - }, - { - 'flags': ['--mock_multi_option=op1', '--mock_multi_option=op2'], - 'expected': { - 'mock_flag': False, - 'mock_option': None, - 'mock_multi_option': ['op1', 'op2'] - }, - 'display_data': [ - DisplayDataItemMatcher('mock_multi_option', ['op1', 'op2']) - ] - }, - { - 'flags': ['--mock_json_option={"11a": 0, "37a": 1}'], - 'expected': { - 'mock_flag': False, - 'mock_option': None, - 'mock_multi_option': None, - 'mock_json_option': { - '11a': 0, '37a': 1 - }, - }, - 'display_data': [ - DisplayDataItemMatcher('mock_json_option', { - '11a': 0, '37a': 1 - }) - ] - }, + (['--num_workers', '5'], + { + 'num_workers': 5, + 'mock_flag': False, + 'mock_option': None, + 'mock_multi_option': None + }, [DisplayDataItemMatcher('num_workers', 5)]), + (['--direct_num_workers', '5'], + { + 'direct_num_workers': 5, + 'mock_flag': False, + 'mock_option': None, + 'mock_multi_option': None + }, [DisplayDataItemMatcher('direct_num_workers', 5)]), + (['--direct_running_mode', 'multi_threading'], + { + 'direct_running_mode': 'multi_threading', + 'mock_flag': False, + 'mock_option': None, + 'mock_multi_option': None + }, [DisplayDataItemMatcher('direct_running_mode', 'multi_threading')]), + (['--direct_running_mode', 'multi_processing'], + { + 'direct_running_mode': 'multi_processing', + 'mock_flag': False, + 'mock_option': None, + 'mock_multi_option': None + }, [DisplayDataItemMatcher('direct_running_mode', 'multi_processing')]), + (['--profile_cpu', '--profile_location', 'gs://bucket/', 'ignored'], + { + 'profile_cpu': True, + 'profile_location': 'gs://bucket/', + 'mock_flag': False, + 'mock_option': None, + 'mock_multi_option': None + }, + [ + DisplayDataItemMatcher('profile_cpu', True), + DisplayDataItemMatcher('profile_location', 'gs://bucket/') + ]), + (['--num_workers', '5', '--mock_flag'], + { + 'num_workers': 5, + 'mock_flag': True, + 'mock_option': None, + 'mock_multi_option': None + }, + [ + DisplayDataItemMatcher('num_workers', 5), + DisplayDataItemMatcher('mock_flag', True) + ]), + (['--mock_option', 'abc'], { + 'mock_flag': False, 'mock_option': 'abc', 'mock_multi_option': None + }, [DisplayDataItemMatcher('mock_option', 'abc')]), + (['--mock_option', ' abc def '], + { + 'mock_flag': False, + 'mock_option': ' abc def ', + 'mock_multi_option': None + }, [DisplayDataItemMatcher('mock_option', ' abc def ')]), + (['--mock_option= abc xyz '], + { + 'mock_flag': False, + 'mock_option': ' abc xyz ', + 'mock_multi_option': None + }, [DisplayDataItemMatcher('mock_option', ' abc xyz ')]), + ([ + '--mock_option=gs://my bucket/my folder/my file', + '--mock_multi_option=op1', + '--mock_multi_option=op2' + ], + { + 'mock_flag': False, + 'mock_option': 'gs://my bucket/my folder/my file', + 'mock_multi_option': ['op1', 'op2'] + }, + [ + DisplayDataItemMatcher( + 'mock_option', 'gs://my bucket/my folder/my file'), + DisplayDataItemMatcher('mock_multi_option', ['op1', 'op2']) + ]), + (['--mock_multi_option=op1', '--mock_multi_option=op2'], + { + 'mock_flag': False, + 'mock_option': None, + 'mock_multi_option': ['op1', 'op2'] + }, [DisplayDataItemMatcher('mock_multi_option', ['op1', 'op2'])]), + (['--mock_json_option={"11a": 0, "37a": 1}'], + { + 'mock_flag': False, + 'mock_option': None, + 'mock_multi_option': None, + 'mock_json_option': { + '11a': 0, '37a': 1 + }, + }, [DisplayDataItemMatcher('mock_json_option', { + '11a': 0, '37a': 1 + })]), ] # Used for testing newly added flags. @@ -218,59 +193,59 @@ def _add_argparse_args(cls, parser): parser.add_argument( '--fake_multi_option', action='append', help='fake multi option') - def test_display_data(self): - for case in PipelineOptionsTest.TEST_CASES: - options = PipelineOptions(flags=case['flags']) - dd = DisplayData.create_from(options) - hc.assert_that(dd.items, hc.contains_inanyorder(*case['display_data'])) - - def test_get_all_options_subclass(self): - for case in PipelineOptionsTest.TEST_CASES: - options = PipelineOptionsTest.MockOptions(flags=case['flags']) - self.assertDictContainsSubset(case['expected'], options.get_all_options()) - self.assertEqual( - options.view_as(PipelineOptionsTest.MockOptions).mock_flag, - case['expected']['mock_flag']) - self.assertEqual( - options.view_as(PipelineOptionsTest.MockOptions).mock_option, - case['expected']['mock_option']) - self.assertEqual( - options.view_as(PipelineOptionsTest.MockOptions).mock_multi_option, - case['expected']['mock_multi_option']) - - def test_get_all_options(self): - for case in PipelineOptionsTest.TEST_CASES: - options = PipelineOptions(flags=case['flags']) - self.assertDictContainsSubset(case['expected'], options.get_all_options()) - self.assertEqual( - options.view_as(PipelineOptionsTest.MockOptions).mock_flag, - case['expected']['mock_flag']) - self.assertEqual( - options.view_as(PipelineOptionsTest.MockOptions).mock_option, - case['expected']['mock_option']) - self.assertEqual( - options.view_as(PipelineOptionsTest.MockOptions).mock_multi_option, - case['expected']['mock_multi_option']) - - def test_sublcalsses_of_pipeline_options_can_be_instantiated(self): - for case in PipelineOptionsTest.TEST_CASES: - mock_options = PipelineOptionsTest.MockOptions(flags=case['flags']) - self.assertEqual(mock_options.mock_flag, case['expected']['mock_flag']) - self.assertEqual( - mock_options.mock_option, case['expected']['mock_option']) - self.assertEqual( - mock_options.mock_multi_option, case['expected']['mock_multi_option']) - - def test_views_can_be_constructed_from_pipeline_option_subclasses(self): - for case in PipelineOptionsTest.TEST_CASES: - fake_options = PipelineOptionsTest.FakeOptions(flags=case['flags']) - mock_options = fake_options.view_as(PipelineOptionsTest.MockOptions) - - self.assertEqual(mock_options.mock_flag, case['expected']['mock_flag']) - self.assertEqual( - mock_options.mock_option, case['expected']['mock_option']) - self.assertEqual( - mock_options.mock_multi_option, case['expected']['mock_multi_option']) + @parameterized.expand(TEST_CASES) + def test_display_data(self, flags, _, display_data): + options = PipelineOptions(flags=flags) + dd = DisplayData.create_from(options) + hc.assert_that(dd.items, hc.contains_inanyorder(*display_data)) + + @parameterized.expand(TEST_CASES) + def test_get_all_options_subclass(self, flags, expected, _): + options = PipelineOptionsTest.MockOptions(flags=flags) + self.assertDictContainsSubset(expected, options.get_all_options()) + self.assertEqual( + options.view_as(PipelineOptionsTest.MockOptions).mock_flag, + expected['mock_flag']) + self.assertEqual( + options.view_as(PipelineOptionsTest.MockOptions).mock_option, + expected['mock_option']) + self.assertEqual( + options.view_as(PipelineOptionsTest.MockOptions).mock_multi_option, + expected['mock_multi_option']) + + @parameterized.expand(TEST_CASES) + def test_get_all_options(self, flags, expected, _): + options = PipelineOptions(flags=flags) + self.assertDictContainsSubset(expected, options.get_all_options()) + self.assertEqual( + options.view_as(PipelineOptionsTest.MockOptions).mock_flag, + expected['mock_flag']) + self.assertEqual( + options.view_as(PipelineOptionsTest.MockOptions).mock_option, + expected['mock_option']) + self.assertEqual( + options.view_as(PipelineOptionsTest.MockOptions).mock_multi_option, + expected['mock_multi_option']) + + @parameterized.expand(TEST_CASES) + def test_subclasses_of_pipeline_options_can_be_instantiated( + self, flags, expected, _): + mock_options = PipelineOptionsTest.MockOptions(flags=flags) + self.assertEqual(mock_options.mock_flag, expected['mock_flag']) + self.assertEqual(mock_options.mock_option, expected['mock_option']) + self.assertEqual( + mock_options.mock_multi_option, expected['mock_multi_option']) + + @parameterized.expand(TEST_CASES) + def test_views_can_be_constructed_from_pipeline_option_subclasses( + self, flags, expected, _): + fake_options = PipelineOptionsTest.FakeOptions(flags=flags) + mock_options = fake_options.view_as(PipelineOptionsTest.MockOptions) + + self.assertEqual(mock_options.mock_flag, expected['mock_flag']) + self.assertEqual(mock_options.mock_option, expected['mock_option']) + self.assertEqual( + mock_options.mock_multi_option, expected['mock_multi_option']) def test_views_do_not_expose_options_defined_by_other_views(self): flags = ['--mock_option=mock_value', '--fake_option=fake_value'] @@ -292,23 +267,23 @@ def test_views_do_not_expose_options_defined_by_other_views(self): PipelineOptionsTest.FakeOptions).view_as( PipelineOptionsTest.MockOptions).fake_option) - def test_from_dictionary(self): - for case in PipelineOptionsTest.TEST_CASES: - options = PipelineOptions(flags=case['flags']) - all_options_dict = options.get_all_options() - options_from_dict = PipelineOptions.from_dictionary(all_options_dict) - self.assertEqual( - options_from_dict.view_as(PipelineOptionsTest.MockOptions).mock_flag, - case['expected']['mock_flag']) - self.assertEqual( - options.view_as(PipelineOptionsTest.MockOptions).mock_option, - case['expected']['mock_option']) - self.assertEqual( - options.view_as(PipelineOptionsTest.MockOptions).mock_multi_option, - case['expected']['mock_multi_option']) - self.assertEqual( - options.view_as(PipelineOptionsTest.MockOptions).mock_json_option, - case['expected'].get('mock_json_option', {})) + @parameterized.expand(TEST_CASES) + def test_from_dictionary(self, flags, expected, _): + options = PipelineOptions(flags=flags) + all_options_dict = options.get_all_options() + options_from_dict = PipelineOptions.from_dictionary(all_options_dict) + self.assertEqual( + options_from_dict.view_as(PipelineOptionsTest.MockOptions).mock_flag, + expected['mock_flag']) + self.assertEqual( + options.view_as(PipelineOptionsTest.MockOptions).mock_option, + expected['mock_option']) + self.assertEqual( + options.view_as(PipelineOptionsTest.MockOptions).mock_multi_option, + expected['mock_multi_option']) + self.assertEqual( + options.view_as(PipelineOptionsTest.MockOptions).mock_json_option, + expected.get('mock_json_option', {})) def test_none_from_dictionary(self): class NoneDefaultOptions(PipelineOptions): @@ -703,6 +678,85 @@ def test_options_store_false_with_different_dest(self): "the dest and the flag name to the map " "_FLAG_THAT_SETS_FALSE_VALUE in PipelineOptions.py") + def test_validation_good_stg_good_temp(self): + runner = MockRunners.DataflowRunner() + options = GoogleCloudOptions([ + '--project=myproject', + '--staging_location=gs://beam/stg', + '--temp_location=gs://beam/tmp' + ]) + validator = PipelineOptionsValidator(options, runner) + errors = options._handle_temp_and_staging_locations(validator) + self.assertEqual(errors, []) + self.assertEqual( + options.get_all_options()['staging_location'], "gs://beam/stg") + self.assertEqual( + options.get_all_options()['temp_location'], "gs://beam/tmp") + + def test_validation_bad_stg_good_temp(self): + runner = MockRunners.DataflowRunner() + options = GoogleCloudOptions([ + '--project=myproject', + '--staging_location=badGSpath', + '--temp_location=gs://beam/tmp' + ]) + validator = PipelineOptionsValidator(options, runner) + errors = options._handle_temp_and_staging_locations(validator) + self.assertEqual(errors, []) + self.assertEqual( + options.get_all_options()['staging_location'], "gs://beam/tmp") + self.assertEqual( + options.get_all_options()['temp_location'], "gs://beam/tmp") + + def test_validation_good_stg_bad_temp(self): + runner = MockRunners.DataflowRunner() + options = GoogleCloudOptions([ + '--project=myproject', + '--staging_location=gs://beam/stg', + '--temp_location=badGSpath' + ]) + validator = PipelineOptionsValidator(options, runner) + errors = options._handle_temp_and_staging_locations(validator) + self.assertEqual(errors, []) + self.assertEqual( + options.get_all_options()['staging_location'], "gs://beam/stg") + self.assertEqual( + options.get_all_options()['temp_location'], "gs://beam/stg") + + def test_validation_bad_stg_bad_temp_with_default(self): + runner = MockRunners.DataflowRunner() + options = MockGoogleCloudOptionsWithBucket([ + '--project=myproject', + '--staging_location=badGSpath', + '--temp_location=badGSpath' + ]) + validator = PipelineOptionsValidator(options, runner) + errors = options._handle_temp_and_staging_locations(validator) + self.assertEqual(errors, []) + self.assertEqual( + options.get_all_options()['staging_location'], "gs://default/bucket") + self.assertEqual( + options.get_all_options()['temp_location'], "gs://default/bucket") + + def test_validation_bad_stg_bad_temp_no_default(self): + runner = MockRunners.DataflowRunner() + options = MockGoogleCloudOptionsNoBucket([ + '--project=myproject', + '--staging_location=badGSpath', + '--temp_location=badGSpath' + ]) + validator = PipelineOptionsValidator(options, runner) + errors = options._handle_temp_and_staging_locations(validator) + self.assertEqual(len(errors), 2, errors) + self.assertIn( + 'Invalid GCS path (badGSpath), given for the option: temp_location.', + errors, + errors) + self.assertIn( + 'Invalid GCS path (badGSpath), given for the option: staging_location.', + errors, + errors) + if __name__ == '__main__': logging.getLogger().setLevel(logging.INFO) diff --git a/sdks/python/apache_beam/options/pipeline_options_validator.py b/sdks/python/apache_beam/options/pipeline_options_validator.py index 3f489c0b60c2d..7c07e5a1e6c7f 100644 --- a/sdks/python/apache_beam/options/pipeline_options_validator.py +++ b/sdks/python/apache_beam/options/pipeline_options_validator.py @@ -23,6 +23,8 @@ import logging import re +import string +from urllib.parse import urlparse from apache_beam.internal import pickler from apache_beam.options.pipeline_options import DebugOptions @@ -92,6 +94,8 @@ class PipelineOptionsValidator(object): ERR_INVALID_PROJECT_ID = ( 'Invalid Project ID (%s). Please make sure you specified the Project ID, ' 'not project description.') + ERR_INVALID_ENDPOINT = ( + 'Invalid url (%s) for dataflow endpoint. Please provide a valid url.') ERR_INVALID_NOT_POSITIVE = ( 'Invalid value (%s) for option: %s. Value needs ' 'to be positive.') @@ -125,7 +129,6 @@ class PipelineOptionsValidator(object): JOB_PATTERN = '[a-z]([-a-z0-9]*[a-z0-9])?' PROJECT_ID_PATTERN = '[a-z][-a-z0-9:.]+[a-z0-9]' PROJECT_NUMBER_PATTERN = '[0-9]*' - ENDPOINT_PATTERN = r'https://[\S]*googleapis\.com[/]?' def __init__(self, options, runner): self.options = options @@ -154,11 +157,13 @@ def is_service_runner(self): dataflow_endpoint = ( self.options.view_as(GoogleCloudOptions).dataflow_endpoint) - is_service_endpoint = ( - dataflow_endpoint is not None and - self.is_full_string_match(self.ENDPOINT_PATTERN, dataflow_endpoint)) - - return is_service_runner and is_service_endpoint + if dataflow_endpoint is None: + return False + else: + endpoint_parts = urlparse(dataflow_endpoint, allow_fragments=False) + if endpoint_parts.netloc.startswith("localhost"): + return False + return is_service_runner def is_full_string_match(self, pattern, string): """Returns True if the pattern matches the whole string.""" @@ -189,7 +194,6 @@ def validate_gcs_path(self, view, arg_name): return self._validate_error(self.ERR_INVALID_GCS_BUCKET, arg, arg_name) if gcs_object is None or '\n' in gcs_object or '\r' in gcs_object: return self._validate_error(self.ERR_INVALID_GCS_OBJECT, arg, arg_name) - return [] def validate_cloud_options(self, view): @@ -231,6 +235,15 @@ def validate_cloud_options(self, view): errors.extend(self._validate_error(self.ERR_MISSING_OPTION, 'region')) else: view.region = default_region + dataflow_endpoint = view.dataflow_endpoint + if dataflow_endpoint is None: + errors.extend( + self._validate_error(self.ERR_MISSING_OPTION, dataflow_endpoint)) + else: + valid_endpoint = self.validate_endpoint_url(dataflow_endpoint) + if valid_endpoint is False: + errors.extend( + self._validate_error(self.ERR_INVALID_ENDPOINT, dataflow_endpoint)) return errors def validate_sdk_container_image_options(self, view): @@ -291,7 +304,7 @@ def validate_worker_region_zone(self, view): self._validate_error( 'Cannot use deprecated flag --zone along with worker_region or ' 'worker_zone.')) - if self.options.view_as(DebugOptions).lookup_experiment('worker_region')\ + if self.options.view_as(DebugOptions).lookup_experiment('worker_region') \ and (view.worker_region or view.worker_zone): errors.extend( self._validate_error( @@ -393,3 +406,17 @@ def validate_repeatable_argument_passed_as_list(self, view, arg_name): return self._validate_error( self.ERR_REPEATABLE_OPTIONS_NOT_SET_AS_LIST, arg, arg_name) return [] + + # Minimally validates the endpoint url. This is not a strict application + # of http://www.faqs.org/rfcs/rfc1738.html. + # If the url matches localhost, set + def validate_endpoint_url(self, endpoint_url): + url_parts = urlparse(endpoint_url, allow_fragments=False) + if not url_parts.scheme or not url_parts.netloc: + return False + if url_parts.scheme not in ['http', 'https']: + return False + if set( + url_parts.netloc) <= set(string.ascii_letters + string.digits + '-.'): + return True + return False diff --git a/sdks/python/apache_beam/options/pipeline_options_validator_test.py b/sdks/python/apache_beam/options/pipeline_options_validator_test.py index ce37688ef82be..015e3ef787f86 100644 --- a/sdks/python/apache_beam/options/pipeline_options_validator_test.py +++ b/sdks/python/apache_beam/options/pipeline_options_validator_test.py @@ -26,6 +26,7 @@ from hamcrest import contains_string from hamcrest import only_contains from hamcrest.core.base_matcher import BaseMatcher +from parameterized import parameterized from apache_beam.internal import pickler from apache_beam.options.pipeline_options import DebugOptions @@ -95,174 +96,85 @@ def test_missing_required_options(self): errors, ['project', 'staging_location', 'temp_location', 'region']), []) - def test_gcs_path(self): - def get_validator(temp_location, staging_location): + @parameterized.expand([ + (None, 'gs://foo/bar', + []), (None, None, ['staging_location', 'temp_location']), + ('gs://foo/bar', None, []), ('gs://foo/bar', 'gs://ABC/bar', + []), ('gcs:/foo/bar', 'gs://foo/bar', []), + ('gs:/foo/bar', 'gs://foo/bar', []), ('gs://ABC/bar', 'gs://foo/bar', []), + ('gs://ABC/bar', 'gs://BCD/bar', ['temp_location', 'staging_location' + ]), ('gs://foo', 'gs://foo/bar', []), + ('gs://foo/', 'gs://foo/bar', []), ('gs://foo/bar', 'gs://foo/bar', []) + ]) + def test_gcs_path(self, temp_location, staging_location, expected_error_args): + def get_validator(_temp_location, _staging_location): options = ['--project=example:example', '--job_name=job'] - if temp_location is not None: - options.append('--temp_location=' + temp_location) + if _temp_location is not None: + options.append('--temp_location=' + _temp_location) - if staging_location is not None: - options.append('--staging_location=' + staging_location) + if _staging_location is not None: + options.append('--staging_location=' + _staging_location) pipeline_options = PipelineOptions(options) runner = MockRunners.DataflowRunner() validator = PipelineOptionsValidator(pipeline_options, runner) return validator - test_cases = [ - { - 'temp_location': None, - 'staging_location': 'gs://foo/bar', - 'errors': ['temp_location'] - }, - { - 'temp_location': None, - 'staging_location': None, - 'errors': ['staging_location', 'temp_location'] - }, - { - 'temp_location': 'gs://foo/bar', - 'staging_location': None, - 'errors': [] - }, - { - 'temp_location': 'gs://foo/bar', - 'staging_location': 'gs://ABC/bar', - 'errors': ['staging_location'] - }, - { - 'temp_location': 'gcs:/foo/bar', - 'staging_location': 'gs://foo/bar', - 'errors': ['temp_location'] - }, - { - 'temp_location': 'gs:/foo/bar', - 'staging_location': 'gs://foo/bar', - 'errors': ['temp_location'] - }, - { - 'temp_location': 'gs://ABC/bar', - 'staging_location': 'gs://foo/bar', - 'errors': ['temp_location'] - }, - { - 'temp_location': 'gs://ABC/bar', - 'staging_location': 'gs://foo/bar', - 'errors': ['temp_location'] - }, - { - 'temp_location': 'gs://foo', - 'staging_location': 'gs://foo/bar', - 'errors': ['temp_location'] - }, - { - 'temp_location': 'gs://foo/', - 'staging_location': 'gs://foo/bar', - 'errors': [] - }, - { - 'temp_location': 'gs://foo/bar', - 'staging_location': 'gs://foo/bar', - 'errors': [] - }, - ] - - for case in test_cases: - errors = get_validator(case['temp_location'], - case['staging_location']).validate() - self.assertEqual( - self.check_errors_for_arguments(errors, case['errors']), []) + errors = get_validator(temp_location, staging_location).validate() + self.assertEqual( + self.check_errors_for_arguments(errors, expected_error_args), []) - def test_project(self): - def get_validator(project): + @parameterized.expand([(None, ['project']), ('12345', ['project']), + ('FOO', ['project']), ('foo:BAR', ['project']), + ('fo', ['project']), ('foo', []), ('foo:bar', [])]) + def test_project(self, project, expected_error_args): + def get_validator(_project): options = [ '--job_name=job', '--staging_location=gs://foo/bar', '--temp_location=gs://foo/bar' ] - if project is not None: - options.append('--project=' + project) + if _project is not None: + options.append('--project=' + _project) pipeline_options = PipelineOptions(options) runner = MockRunners.DataflowRunner() validator = PipelineOptionsValidator(pipeline_options, runner) return validator - test_cases = [ - { - 'project': None, 'errors': ['project'] - }, - { - 'project': '12345', 'errors': ['project'] - }, - { - 'project': 'FOO', 'errors': ['project'] - }, - { - 'project': 'foo:BAR', 'errors': ['project'] - }, - { - 'project': 'fo', 'errors': ['project'] - }, - { - 'project': 'foo', 'errors': [] - }, - { - 'project': 'foo:bar', 'errors': [] - }, - ] - - for case in test_cases: - errors = get_validator(case['project']).validate() - self.assertEqual( - self.check_errors_for_arguments(errors, case['errors']), []) + errors = get_validator(project).validate() + self.assertEqual( + self.check_errors_for_arguments(errors, expected_error_args), []) - def test_job_name(self): - def get_validator(job_name): + @parameterized.expand([(None, []), ('12345', ['job_name']), + ('FOO', ['job_name']), ('foo:bar', ['job_name']), + ('fo', []), ('foo', [])]) + def test_job_name(self, job_name, expected_error_args): + def get_validator(_job_name): options = [ '--project=example:example', '--staging_location=gs://foo/bar', '--temp_location=gs://foo/bar' ] - if job_name is not None: - options.append('--job_name=' + job_name) + if _job_name is not None: + options.append('--job_name=' + _job_name) pipeline_options = PipelineOptions(options) runner = MockRunners.DataflowRunner() validator = PipelineOptionsValidator(pipeline_options, runner) return validator - test_cases = [ - { - 'job_name': None, 'errors': [] - }, - { - 'job_name': '12345', 'errors': ['job_name'] - }, - { - 'job_name': 'FOO', 'errors': ['job_name'] - }, - { - 'job_name': 'foo:bar', 'errors': ['job_name'] - }, - { - 'job_name': 'fo', 'errors': [] - }, - { - 'job_name': 'foo', 'errors': [] - }, - ] - - for case in test_cases: - errors = get_validator(case['job_name']).validate() - self.assertEqual( - self.check_errors_for_arguments(errors, case['errors']), []) + errors = get_validator(job_name).validate() + self.assertEqual( + self.check_errors_for_arguments(errors, expected_error_args), []) - def test_num_workers(self): - def get_validator(num_workers): + @parameterized.expand([(None, []), ('1', []), ('0', ['num_workers']), + ('-1', ['num_workers'])]) + def test_num_workers(self, num_workers, expected_error_args): + def get_validator(_num_workers): options = [ '--project=example:example', '--job_name=job', @@ -270,82 +182,66 @@ def get_validator(num_workers): '--temp_location=gs://foo/bar' ] - if num_workers is not None: - options.append('--num_workers=' + num_workers) + if _num_workers is not None: + options.append('--num_workers=' + _num_workers) pipeline_options = PipelineOptions(options) runner = MockRunners.DataflowRunner() validator = PipelineOptionsValidator(pipeline_options, runner) return validator - test_cases = [ - { - 'num_workers': None, 'errors': [] - }, - { - 'num_workers': '1', 'errors': [] - }, - { - 'num_workers': '0', 'errors': ['num_workers'] - }, - { - 'num_workers': '-1', 'errors': ['num_workers'] - }, - ] - - for case in test_cases: - errors = get_validator(case['num_workers']).validate() - self.assertEqual( - self.check_errors_for_arguments(errors, case['errors']), []) - - def test_is_service_runner(self): - test_cases = [ - { - 'runner': MockRunners.OtherRunner(), - 'options': [], - 'expected': False, - }, - { - 'runner': MockRunners.OtherRunner(), - 'options': ['--dataflow_endpoint=https://dataflow.googleapis.com'], - 'expected': False, - }, - { - 'runner': MockRunners.OtherRunner(), - 'options': ['--dataflow_endpoint=https://dataflow.googleapis.com/'], - 'expected': False, - }, - { - 'runner': MockRunners.DataflowRunner(), - 'options': ['--dataflow_endpoint=https://another.service.com'], - 'expected': False, - }, - { - 'runner': MockRunners.DataflowRunner(), - 'options': ['--dataflow_endpoint=https://another.service.com/'], - 'expected': False, - }, - { - 'runner': MockRunners.DataflowRunner(), - 'options': ['--dataflow_endpoint=https://dataflow.googleapis.com'], - 'expected': True, - }, - { - 'runner': MockRunners.DataflowRunner(), - 'options': ['--dataflow_endpoint=https://dataflow.googleapis.com/'], - 'expected': True, - }, - { - 'runner': MockRunners.DataflowRunner(), - 'options': [], - 'expected': True, - }, - ] - - for case in test_cases: - validator = PipelineOptionsValidator( - PipelineOptions(case['options']), case['runner']) - self.assertEqual(validator.is_service_runner(), case['expected']) + errors = get_validator(num_workers).validate() + self.assertEqual( + self.check_errors_for_arguments(errors, expected_error_args), []) + + @parameterized.expand([ + ( + MockRunners.OtherRunner(), + [], + False, + ), + ( + MockRunners.OtherRunner(), + ['--dataflow_endpoint=https://dataflow.googleapis.com'], + False, + ), + ( + MockRunners.OtherRunner(), + ['--dataflow_endpoint=https://dataflow.googleapis.com/'], + False, + ), ( + MockRunners.DataflowRunner(), + [], + True, + ), + ( + MockRunners.DataflowRunner(), + ['--dataflow_endpoint=https://another.service.com'], + True, + ), + ( + MockRunners.DataflowRunner(), + ['--dataflow_endpoint=https://dataflow.googleapis.com'], + True, + ), + ( + MockRunners.DataflowRunner(), + ['--dataflow_endpoint=http://localhost:1000'], + False, + ), + ( + MockRunners.DataflowRunner(), + ['--dataflow_endpoint=foo: //dataflow. googleapis. com'], + True, + ), ( + MockRunners.DataflowRunner(), + [], + True, + ) + ]) + def test_is_service_runner(self, runner, options, expected): + validator = PipelineOptionsValidator(PipelineOptions(options), runner) + self.assertEqual(validator.is_service_runner(), expected) def test_dataflow_job_file_and_template_location_mutually_exclusive(self): runner = MockRunners.OtherRunner() @@ -478,20 +374,17 @@ def test_experiment_region_and_worker_region_mutually_exclusive(self): self.assertIn('experiment', errors[0]) self.assertIn('worker_region', errors[0]) - def test_experiment_region_and_worker_zone_mutually_exclusive(self): + def test_region_and_worker_zone_mutually_exclusive(self): runner = MockRunners.DataflowRunner() options = PipelineOptions([ - '--experiments', - 'worker_region=us-west1', - '--worker_zone', - 'us-east1-b', + '--worker_region=us-west1', + '--worker_zone=us-east1-b', '--project=example:example', '--temp_location=gs://foo/bar', ]) validator = PipelineOptionsValidator(options, runner) errors = validator.validate() self.assertEqual(len(errors), 1) - self.assertIn('experiment', errors[0]) self.assertIn('worker_region', errors[0]) self.assertIn('worker_zone', errors[0]) @@ -556,18 +449,31 @@ def test_zone_alias_worker_zone(self): self.assertEqual(options.view_as(WorkerOptions).worker_zone, 'us-east1-b') def test_region_optional_for_non_service_runner(self): - runner = MockRunners.DataflowRunner() + runner = MockRunners.OtherRunner() # Remove default region for this test. runner.get_default_gcp_region = lambda: None options = PipelineOptions([ '--project=example:example', '--temp_location=gs://foo/bar', - '--dataflow_endpoint=http://localhost:20281', ]) validator = PipelineOptionsValidator(options, runner) errors = validator.validate() self.assertEqual(len(errors), 0) + def test_dataflow_endpoint_is_a_url(self): + runner = MockRunners.DataflowRunner() + # Remove default region for this test. + options = PipelineOptions([ + '--project=example:example', + '--temp_location=gs://foo/bar', + '--staging_location=gs://foo/baz', + '--dataflow_endpoint=foo and bar' + ]) + validator = PipelineOptionsValidator(options, runner) + errors = validator.validate() + self.assertEqual(len(errors), 1, errors) + self.assertIn("Invalid url (foo and bar)", errors[0]) + def test_alias_sdk_container_to_worker_harness(self): runner = MockRunners.DataflowRunner() test_image = "SDK_IMAGE" @@ -641,7 +547,10 @@ def test_prebuild_sdk_container_base_allowed_if_matches_custom_image(self): errors = validator.validate() self.assertEqual(len(errors), 0) - def test_test_matcher(self): + @parameterized.expand([(None, []), (pickler.dumps(AlwaysPassMatcher()), []), + (b'abc', ['on_success_matcher']), + (pickler.dumps(object), ['on_success_matcher'])]) + def test_test_matcher(self, on_success_matcher, errors): def get_validator(matcher): options = [ '--project=example:example', @@ -656,27 +565,8 @@ def get_validator(matcher): runner = MockRunners.TestDataflowRunner() return PipelineOptionsValidator(pipeline_options, runner) - test_case = [ - { - 'on_success_matcher': None, 'errors': [] - }, - { - 'on_success_matcher': pickler.dumps(AlwaysPassMatcher()), - 'errors': [] - }, - { - 'on_success_matcher': b'abc', 'errors': ['on_success_matcher'] - }, - { - 'on_success_matcher': pickler.dumps(object), - 'errors': ['on_success_matcher'] - }, - ] - - for case in test_case: - errors = get_validator(case['on_success_matcher']).validate() - self.assertEqual( - self.check_errors_for_arguments(errors, case['errors']), []) + errors = get_validator(on_success_matcher).validate() + self.assertEqual(self.check_errors_for_arguments(errors, errors), []) def test_transform_name_mapping_without_update(self): options = [ @@ -736,195 +626,112 @@ def test_type_check_additional_unrecognized_feature(self): errors = validator.validate() self.assertTrue(errors) - def test_environment_options(self): - test_cases = [ - { - 'options': ['--environment_type=dOcKeR'], 'errors': [] - }, - { - 'options': [ - '--environment_type=dOcKeR', - '--environment_options=docker_container_image=foo' - ], - 'errors': [] - }, - { - 'options': [ - '--environment_type=dOcKeR', '--environment_config=foo' - ], - 'errors': [] - }, - { - 'options': [ - '--environment_type=dOcKeR', - '--environment_options=docker_container_image=foo', - '--environment_config=foo' - ], - 'errors': ['environment_config'] - }, - { - 'options': [ - '--environment_type=dOcKeR', - '--environment_options=process_command=foo', - '--environment_options=process_variables=foo=bar', - '--environment_options=external_service_address=foo' - ], - 'errors': [ - 'process_command', - 'process_variables', - 'external_service_address' - ] - }, - { - 'options': ['--environment_type=pRoCeSs'], - 'errors': ['process_command'] - }, - { - 'options': [ - '--environment_type=pRoCeSs', - '--environment_options=process_command=foo' - ], - 'errors': [] - }, - { - 'options': [ - '--environment_type=pRoCeSs', '--environment_config=foo' - ], - 'errors': [] - }, - { - 'options': [ - '--environment_type=pRoCeSs', - '--environment_options=process_command=foo', - '--environment_config=foo' - ], - 'errors': ['environment_config'] - }, - { - 'options': [ - '--environment_type=pRoCeSs', - '--environment_options=process_command=foo', - '--environment_options=process_variables=foo=bar', - '--environment_options=docker_container_image=foo', - '--environment_options=external_service_address=foo' - ], - 'errors': ['docker_container_image', 'external_service_address'] - }, - { - 'options': ['--environment_type=eXtErNaL'], - 'errors': ['external_service_address'] - }, - { - 'options': [ - '--environment_type=eXtErNaL', - '--environment_options=external_service_address=foo' - ], - 'errors': [] - }, - { - 'options': [ - '--environment_type=eXtErNaL', '--environment_config=foo' - ], - 'errors': [] - }, - { - 'options': [ - '--environment_type=eXtErNaL', - '--environment_options=external_service_address=foo', - '--environment_config=foo' - ], - 'errors': ['environment_config'] - }, - { - 'options': [ - '--environment_type=eXtErNaL', - '--environment_options=external_service_address=foo', - '--environment_options=process_command=foo', - '--environment_options=process_variables=foo=bar', - '--environment_options=docker_container_image=foo', - ], - 'errors': [ - 'process_command', - 'process_variables', - 'docker_container_image' - ] - }, - { - 'options': ['--environment_type=lOoPbACk'], 'errors': [] - }, - { - 'options': [ - '--environment_type=lOoPbACk', '--environment_config=foo' - ], - 'errors': ['environment_config'] - }, - { - 'options': [ - '--environment_type=lOoPbACk', - '--environment_options=docker_container_image=foo', - '--environment_options=process_command=foo', - '--environment_options=process_variables=foo=bar', - '--environment_options=external_service_address=foo', - ], - 'errors': [ - 'docker_container_image', - 'process_command', - 'process_variables', - 'external_service_address' - ] - }, - { - 'options': ['--environment_type=beam:env:foo:v1'], 'errors': [] - }, - { - 'options': [ - '--environment_type=beam:env:foo:v1', - '--environment_config=foo' - ], - 'errors': [] - }, - { - 'options': [ - '--environment_type=beam:env:foo:v1', - '--environment_options=docker_container_image=foo', - '--environment_options=process_command=foo', - '--environment_options=process_variables=foo=bar', - '--environment_options=external_service_address=foo', - ], - 'errors': [ - 'docker_container_image', - 'process_command', - 'process_variables', - 'external_service_address' - ] - }, - { - 'options': [ - '--environment_options=docker_container_image=foo', - '--environment_options=process_command=foo', - '--environment_options=process_variables=foo=bar', - '--environment_options=external_service_address=foo', - ], - 'errors': [ - 'docker_container_image', - 'process_command', - 'process_variables', - 'external_service_address' - ] - }, - ] + @parameterized.expand([ + (['--environment_type=dOcKeR'], []), + ([ + '--environment_type=dOcKeR', + '--environment_options=docker_container_image=foo' + ], []), (['--environment_type=dOcKeR', '--environment_config=foo'], []), + ([ + '--environment_type=dOcKeR', + '--environment_options=docker_container_image=foo', + '--environment_config=foo' + ], ['environment_config']), + ([ + '--environment_type=dOcKeR', + '--environment_options=process_command=foo', + '--environment_options=process_variables=foo=bar', + '--environment_options=external_service_address=foo' + ], ['process_command', 'process_variables', 'external_service_address']), + (['--environment_type=pRoCeSs'], ['process_command']), + ([ + '--environment_type=pRoCeSs', + '--environment_options=process_command=foo' + ], []), (['--environment_type=pRoCeSs', '--environment_config=foo'], []), + ([ + '--environment_type=pRoCeSs', + '--environment_options=process_command=foo', + '--environment_config=foo' + ], ['environment_config']), + ([ + '--environment_type=pRoCeSs', + '--environment_options=process_command=foo', + '--environment_options=process_variables=foo=bar', + '--environment_options=docker_container_image=foo', + '--environment_options=external_service_address=foo' + ], ['docker_container_image', 'external_service_address']), + (['--environment_type=eXtErNaL'], ['external_service_address']), + ([ + '--environment_type=eXtErNaL', + '--environment_options=external_service_address=foo' + ], []), (['--environment_type=eXtErNaL', '--environment_config=foo'], []), + ([ + '--environment_type=eXtErNaL', + '--environment_options=external_service_address=foo', + '--environment_config=foo' + ], ['environment_config']), + ([ + '--environment_type=eXtErNaL', + '--environment_options=external_service_address=foo', + '--environment_options=process_command=foo', + '--environment_options=process_variables=foo=bar', + '--environment_options=docker_container_image=foo' + ], ['process_command', 'process_variables', + 'docker_container_image']), (['--environment_type=lOoPbACk'], []), + (['--environment_type=lOoPbACk', + '--environment_config=foo'], ['environment_config']), + ([ + '--environment_type=lOoPbACk', + '--environment_options=docker_container_image=foo', + '--environment_options=process_command=foo', + '--environment_options=process_variables=foo=bar', + '--environment_options=external_service_address=foo' + ], + [ + 'docker_container_image', + 'process_command', + 'process_variables', + 'external_service_address' + ]), (['--environment_type=beam:env:foo:v1'], []), + (['--environment_type=beam:env:foo:v1', '--environment_config=foo'], []), + ([ + '--environment_type=beam:env:foo:v1', + '--environment_options=docker_container_image=foo', + '--environment_options=process_command=foo', + '--environment_options=process_variables=foo=bar', + '--environment_options=external_service_address=foo' + ], + [ + 'docker_container_image', + 'process_command', + 'process_variables', + 'external_service_address' + ]), + ([ + '--environment_options=docker_container_image=foo', + '--environment_options=process_command=foo', + '--environment_options=process_variables=foo=bar', + '--environment_options=external_service_address=foo' + ], + [ + 'docker_container_image', + 'process_command', + 'process_variables', + 'external_service_address' + ]) + ]) + def test_environment_options(self, options, expected_error_args): errors = [] - for case in test_cases: - validator = PipelineOptionsValidator( - PipelineOptions(case['options']), MockRunners.OtherRunner()) - validation_result = validator.validate() - validation_errors = self.check_errors_for_arguments( - validation_result, case['errors']) - if validation_errors: - errors.append( - 'Options "%s" had unexpected validation results: "%s"' % - (' '.join(case['options']), ' '.join(validation_errors))) - self.assertEqual(errors, []) + validator = PipelineOptionsValidator( + PipelineOptions(options), MockRunners.OtherRunner()) + validation_result = validator.validate() + validation_errors = self.check_errors_for_arguments( + validation_result, expected_error_args) + if validation_errors: + errors.append( + 'Options "%s" had unexpected validation results: "%s"' % + (' '.join(options), ' '.join(validation_errors))) + self.assertEqual(errors, [], expected_error_args) if __name__ == '__main__': diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py index e0944d81d1b27..f52616307e7bd 100644 --- a/sdks/python/apache_beam/pipeline.py +++ b/sdks/python/apache_beam/pipeline.py @@ -88,6 +88,7 @@ from apache_beam.transforms import ParDo from apache_beam.transforms import ptransform from apache_beam.transforms.display import DisplayData +from apache_beam.transforms.display import HasDisplayData from apache_beam.transforms.resources import merge_resource_hints from apache_beam.transforms.resources import resource_hints_from_options from apache_beam.transforms.sideinputs import get_sideinput_index @@ -108,7 +109,7 @@ __all__ = ['Pipeline', 'PTransformOverride'] -class Pipeline(object): +class Pipeline(HasDisplayData): """A pipeline object that manages a DAG of :class:`~apache_beam.pvalue.PValue` s and their :class:`~apache_beam.transforms.ptransform.PTransform` s. @@ -133,9 +134,12 @@ def runner_implemented_transforms(cls): common_urns.primitives.IMPULSE.urn, ]) - def __init__(self, runner=None, options=None, argv=None): - # type: (Optional[Union[str, PipelineRunner]], Optional[PipelineOptions], Optional[List[str]]) -> None - + def __init__( + self, + runner: Optional[Union[str, PipelineRunner]] = None, + options: Optional[PipelineOptions] = None, + argv: Optional[List[str]] = None, + display_data: Optional[Dict[str, Any]] = None): """Initialize a pipeline object. Args: @@ -151,6 +155,8 @@ def __init__(self, runner=None, options=None, argv=None): to be used for building a :class:`~apache_beam.options.pipeline_options.PipelineOptions` object. This will only be used if argument **options** is :data:`None`. + display_data (Dict[str, Any]): a dictionary of static data associated + with this pipeline that can be displayed when it runs. Raises: ValueError: if either the runner or options argument is not @@ -233,12 +239,13 @@ def __init__(self, runner=None, options=None, argv=None): # Records whether this pipeline contains any external transforms. self.contains_external_transforms = False + self._display_data = display_data or {} + + def display_data(self): + # type: () -> Dict[str, Any] + return self._display_data @property # type: ignore[misc] # decorated property not supported - @deprecated( - since='First stable release', - extra_message='References to .options' - ' will not be supported') def options(self): # type: () -> PipelineOptions return self._options @@ -918,7 +925,8 @@ def visit_transform(self, transform_node): proto = beam_runner_api_pb2.Pipeline( root_transform_ids=[root_transform_id], components=context.to_runner_api(), - requirements=context.requirements()) + requirements=context.requirements(), + display_data=DisplayData('', self._display_data).to_proto()) proto.components.transforms[root_transform_id].unique_name = ( root_transform_id) self.merge_compatible_environments(proto) @@ -974,7 +982,11 @@ def from_runner_api( # type: (...) -> Pipeline """For internal use only; no backwards-compatibility guarantees.""" - p = Pipeline(runner=runner, options=options) + p = Pipeline( + runner=runner, + options=options, + display_data={str(ix): d + for ix, d in enumerate(proto.display_data)}) from apache_beam.runners import pipeline_context context = pipeline_context.PipelineContext( proto.components, requirements=proto.requirements) @@ -1330,7 +1342,10 @@ def transform_to_runner_api( # Iterate over inputs and outputs by sorted key order, so that ids are # consistently generated for multiple runs of the same pipeline. - transform_spec = transform_to_runner_api(self.transform, context) + try: + transform_spec = transform_to_runner_api(self.transform, context) + except Exception as exn: + raise RuntimeError(f'Unable to translate {self.full_label}') from exn environment_id = self.environment_id transform_urn = transform_spec.urn if transform_spec else None if (not environment_id and diff --git a/sdks/python/apache_beam/pipeline_test.py b/sdks/python/apache_beam/pipeline_test.py index 18ab0f091aaff..c9ac4ce4c13dd 100644 --- a/sdks/python/apache_beam/pipeline_test.py +++ b/sdks/python/apache_beam/pipeline_test.py @@ -30,7 +30,7 @@ from apache_beam import typehints from apache_beam.coders import BytesCoder from apache_beam.io import Read -from apache_beam.metrics import Metrics +from apache_beam.io.iobase import SourceBase from apache_beam.options.pipeline_options import PortableOptions from apache_beam.pipeline import Pipeline from apache_beam.pipeline import PipelineOptions @@ -40,7 +40,6 @@ from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.pvalue import AsSingleton from apache_beam.pvalue import TaggedOutput -from apache_beam.runners.dataflow.native_io.iobase import NativeSource from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that from apache_beam.testing.util import equal_to @@ -61,39 +60,9 @@ from apache_beam.utils import windowed_value from apache_beam.utils.timestamp import MIN_TIMESTAMP -# TODO(BEAM-1555): Test is failing on the service, with FakeSource. - -class FakeSource(NativeSource): - """Fake source returning a fixed list of values.""" - class _Reader(object): - def __init__(self, vals): - self._vals = vals - self._output_counter = Metrics.counter('main', 'outputs') - - def __enter__(self): - return self - - def __exit__(self, exception_type, exception_value, traceback): - pass - - def __iter__(self): - for v in self._vals: - self._output_counter.inc() - yield v - - def __init__(self, vals): - self._vals = vals - - def reader(self): - return FakeSource._Reader(self._vals) - - -class FakeUnboundedSource(NativeSource): +class FakeUnboundedSource(SourceBase): """Fake unbounded source. Does not work at runtime""" - def reader(self): - return None - def is_bounded(self): return False @@ -259,24 +228,6 @@ def test_create_singleton_pcollection(self): pcoll = pipeline | 'label' >> Create([[1, 2, 3]]) assert_that(pcoll, equal_to([[1, 2, 3]])) - # TODO(BEAM-1555): Test is failing on the service, with FakeSource. - # @pytest.mark.it_validatesrunner - def test_metrics_in_fake_source(self): - pipeline = TestPipeline() - pcoll = pipeline | Read(FakeSource([1, 2, 3, 4, 5, 6])) - assert_that(pcoll, equal_to([1, 2, 3, 4, 5, 6])) - res = pipeline.run() - metric_results = res.metrics().query() - outputs_counter = metric_results['counters'][0] - self.assertEqual(outputs_counter.key.step, 'Read') - self.assertEqual(outputs_counter.key.metric.name, 'outputs') - self.assertEqual(outputs_counter.committed, 6) - - def test_fake_read(self): - with TestPipeline() as pipeline: - pcoll = pipeline | 'read' >> Read(FakeSource([1, 2, 3])) - assert_that(pcoll, equal_to([1, 2, 3])) - def test_visit_entire_graph(self): pipeline = Pipeline() pcoll1 = pipeline | 'pcoll' >> beam.Impulse() diff --git a/sdks/python/apache_beam/portability/OWNERS b/sdks/python/apache_beam/portability/OWNERS deleted file mode 100644 index dc488208007a8..0000000000000 --- a/sdks/python/apache_beam/portability/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - charlesccychen - - robertwb - - aaltay diff --git a/sdks/python/apache_beam/portability/python_urns.py b/sdks/python/apache_beam/portability/python_urns.py index a025709f38a5c..ed9ec6a072585 100644 --- a/sdks/python/apache_beam/portability/python_urns.py +++ b/sdks/python/apache_beam/portability/python_urns.py @@ -49,6 +49,10 @@ # the state cache, as a decimal string, e.g. '2,1000'. EMBEDDED_PYTHON_GRPC = "beam:env:embedded_python_grpc:v1" +# Invoke UserFns via a yet-to-be-started loopback external worker. +# Payload: None. +EMBEDDED_PYTHON_LOOPBACK = "beam:env:embedded_python_loopback:v1" + # Instantiate SDK harness via a command line provided in the payload. # This is different than the standard process environment in that it # starts up the SDK harness directly, rather than the bootstrapping diff --git a/sdks/python/apache_beam/pvalue.py b/sdks/python/apache_beam/pvalue.py index 2e86c9eb51c77..90882651d0b24 100644 --- a/sdks/python/apache_beam/pvalue.py +++ b/sdks/python/apache_beam/pvalue.py @@ -673,6 +673,9 @@ def __init__(self, **kwargs): def as_dict(self): return dict(self.__dict__) + # For compatibility with named tuples. + _asdict = as_dict + def __iter__(self): for _, value in self.__dict__.items(): yield value diff --git a/sdks/python/apache_beam/py.typed b/sdks/python/apache_beam/py.typed new file mode 100644 index 0000000000000..1aea47b9e7ddb --- /dev/null +++ b/sdks/python/apache_beam/py.typed @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Marker file for PEP 561. +# The apache-beam package uses inline types. diff --git a/sdks/python/apache_beam/runners/OWNERS b/sdks/python/apache_beam/runners/OWNERS deleted file mode 100644 index 1a1f0f0d3e722..0000000000000 --- a/sdks/python/apache_beam/runners/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - pabloem - - charlesccychen diff --git a/sdks/python/apache_beam/runners/common.pxd b/sdks/python/apache_beam/runners/common.pxd index d1745970d26c3..9fb44af63772c 100644 --- a/sdks/python/apache_beam/runners/common.pxd +++ b/sdks/python/apache_beam/runners/common.pxd @@ -121,6 +121,8 @@ cdef class DoFnRunner: cdef list side_inputs cdef DoFnInvoker do_fn_invoker cdef public object bundle_finalizer_param + cdef str transform_id + cdef object execution_context cpdef process(self, WindowedValue windowed_value) diff --git a/sdks/python/apache_beam/runners/common.py b/sdks/python/apache_beam/runners/common.py index 75eb85f211018..1cd0a30446634 100644 --- a/sdks/python/apache_beam/runners/common.py +++ b/sdks/python/apache_beam/runners/common.py @@ -24,6 +24,7 @@ # pytype: skip-file +import logging import sys import threading import traceback @@ -38,6 +39,7 @@ from typing import Tuple from apache_beam.coders import TupleCoder +from apache_beam.coders import coders from apache_beam.internal import util from apache_beam.options.value_provider import RuntimeValueProvider from apache_beam.portability import common_urns @@ -54,6 +56,7 @@ from apache_beam.transforms.core import RestrictionProvider from apache_beam.transforms.core import WatermarkEstimatorProvider from apache_beam.transforms.window import GlobalWindow +from apache_beam.transforms.window import GlobalWindows from apache_beam.transforms.window import TimestampedValue from apache_beam.transforms.window import WindowFn from apache_beam.typehints import typehints @@ -66,12 +69,21 @@ from apache_beam.utils.windowed_value import WindowedValue if TYPE_CHECKING: + from apache_beam.runners.worker.bundle_processor import ExecutionContext from apache_beam.transforms import sideinputs from apache_beam.transforms.core import TimerSpec from apache_beam.io.iobase import RestrictionProgress from apache_beam.iobase import RestrictionTracker from apache_beam.iobase import WatermarkEstimator +IMPULSE_VALUE_CODER_IMPL = coders.WindowedValueCoder( + coders.BytesCoder(), coders.GlobalWindowCoder()).get_impl() + +ENCODED_IMPULSE_VALUE = IMPULSE_VALUE_CODER_IMPL.encode_nested( + GlobalWindows.windowed_value(b'')) + +_LOGGER = logging.getLogger(__name__) + class NameContext(object): """Holds the name information for a step.""" @@ -753,6 +765,7 @@ def __init__(self, # Try to prepare all the arguments that can just be filled in # without any additional work. in the process function. # Also cache all the placeholders needed in the process function. + input_args = list(input_args) ( self.placeholders_for_process, self.args_for_process, @@ -1338,7 +1351,8 @@ def __init__(self, state=None, scoped_metrics_container=None, operation_name=None, - user_state_context=None # type: Optional[userstate.UserStateContext] + transform_id=None, + user_state_context=None, # type: Optional[userstate.UserStateContext] ): """Initializes a DoFnRunner. @@ -1354,6 +1368,7 @@ def __init__(self, state: handle for accessing DoFn state scoped_metrics_container: DEPRECATED operation_name: The system name assigned by the runner for this operation. + transform_id: The PTransform Id in the pipeline proto for this DoFn. user_state_context: The UserStateContext instance for the current Stateful DoFn. """ @@ -1361,8 +1376,10 @@ def __init__(self, side_inputs = list(side_inputs) self.step_name = step_name + self.transform_id = transform_id self.context = DoFnContext(step_name, state=state) self.bundle_finalizer_param = DoFn.BundleFinalizerParam() + self.execution_context = None # type: Optional[ExecutionContext] do_fn_signature = DoFnSignature(fn) @@ -1417,9 +1434,26 @@ def process(self, windowed_value): try: return self.do_fn_invoker.invoke_process(windowed_value) except BaseException as exn: - self._reraise_augmented(exn) + self._reraise_augmented(exn, windowed_value) return [] + def _maybe_sample_exception( + self, exn: BaseException, + windowed_value: Optional[WindowedValue]) -> None: + + if self.execution_context is None: + return + + output_sampler = self.execution_context.output_sampler + if output_sampler is None: + return + + output_sampler.sample_exception( + windowed_value, + exn, + self.transform_id, + self.execution_context.instruction_id) + def process_batch(self, windowed_batch): # type: (WindowedBatch) -> None try: @@ -1487,7 +1521,7 @@ def finalize(self): # type: () -> None self.bundle_finalizer_param.finalize_bundle() - def _reraise_augmented(self, exn): + def _reraise_augmented(self, exn, windowed_value=None): if getattr(exn, '_tagged_with_step', False) or not self.step_name: raise exn step_annotation = " [while running '%s']" % self.step_name @@ -1504,8 +1538,13 @@ def _reraise_augmented(self, exn): traceback.format_exception_only(type(exn), exn)[-1].strip() + step_annotation) new_exn._tagged_with_step = True - _, _, tb = sys.exc_info() - raise new_exn.with_traceback(tb) + exc_info = sys.exc_info() + _, _, tb = exc_info + + new_exn = new_exn.with_traceback(tb) + self._maybe_sample_exception(exc_info, windowed_value) + _LOGGER.exception(new_exn) + raise new_exn class OutputHandler(object): @@ -1890,6 +1929,12 @@ def validate_transform(transform_id): raise ValueError( "Incompatible input coder %s and output coder %s for transform %s" % (transform_id, input_coder, output_coder)) + elif transform_proto.spec.urn == common_urns.primitives.ASSIGN_WINDOWS.urn: + if not transform_proto.inputs: + raise ValueError("Missing input for transform: %s" % transform_proto) + elif transform_proto.spec.urn == common_urns.primitives.PAR_DO.urn: + if not transform_proto.inputs: + raise ValueError("Missing input for transform: %s" % transform_proto) for t in transform_proto.subtransforms: validate_transform(t) diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_exercise_metrics_pipeline_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_exercise_metrics_pipeline_test.py index 7bc871d870905..909c15896a264 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_exercise_metrics_pipeline_test.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_exercise_metrics_pipeline_test.py @@ -51,16 +51,6 @@ def test_metrics_it(self): dataflow_exercise_metrics_pipeline.metric_matchers()) self.assertFalse(errors, str(errors)) - @pytest.mark.it_postcommit - @pytest.mark.it_validatescontainer - @unittest.skip('https://github.com/apache/beam/issues/22605') - def test_metrics_fnapi_it(self): - result = self.run_pipeline(experiment='beam_fn_api') - errors = metric_result_matchers.verify_all( - result.metrics().all_metrics(), - dataflow_exercise_metrics_pipeline.metric_matchers()) - self.assertFalse(errors, str(errors)) - if __name__ == '__main__': unittest.main() diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_metrics_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_metrics_test.py index 06e1585ffc420..86e71f9c1ed21 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_metrics_test.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_metrics_test.py @@ -503,7 +503,6 @@ def test_translate_portable_job_step_name(self): self.ONLY_COUNTERS_LIST) pipeline_options = PipelineOptions([ - '--experiments=use_runner_v2', '--experiments=use_portable_job_submission', '--temp_location=gs://any-location/temp', '--project=dummy_project', diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py index 674d05c64ec8a..7ad6ab04be68a 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py @@ -22,26 +22,18 @@ """ # pytype: skip-file -import base64 import logging import os import threading import time -import traceback import warnings from collections import defaultdict from subprocess import DEVNULL from typing import TYPE_CHECKING from typing import List -from urllib.parse import quote -from urllib.parse import quote_from_bytes -from urllib.parse import unquote_to_bytes import apache_beam as beam from apache_beam import coders -from apache_beam import error -from apache_beam.internal import pickler -from apache_beam.internal.gcp import json_value from apache_beam.options.pipeline_options import DebugOptions from apache_beam.options.pipeline_options import GoogleCloudOptions from apache_beam.options.pipeline_options import SetupOptions @@ -50,24 +42,13 @@ from apache_beam.options.pipeline_options import TypeOptions from apache_beam.options.pipeline_options import WorkerOptions from apache_beam.portability import common_urns -from apache_beam.portability.api import beam_runner_api_pb2 -from apache_beam.pvalue import AsSideInput -from apache_beam.runners.common import DoFnSignature from apache_beam.runners.common import group_by_key_input_visitor -from apache_beam.runners.dataflow.internal import names from apache_beam.runners.dataflow.internal.clients import dataflow as dataflow_api -from apache_beam.runners.dataflow.internal.names import PropertyNames -from apache_beam.runners.dataflow.internal.names import TransformNames from apache_beam.runners.runner import PipelineResult from apache_beam.runners.runner import PipelineRunner from apache_beam.runners.runner import PipelineState -from apache_beam.runners.runner import PValueCache -from apache_beam.transforms import window -from apache_beam.transforms.display import DisplayData -from apache_beam.transforms.sideinputs import SIDE_INPUT_PREFIX from apache_beam.typehints import typehints from apache_beam.utils import processes -from apache_beam.utils import proto_utils from apache_beam.utils.interactive_utils import is_in_notebook from apache_beam.utils.plugin import BeamPlugin @@ -88,10 +69,9 @@ class DataflowRunner(PipelineRunner): """A runner that creates job graphs and submits them for remote execution. Every execution of the run() method will submit an independent job for - remote execution that consists of the nodes reachable from the passed in - node argument or entire graph if node is None. The run() method returns - after the service created the job and will not wait for the job to finish - if blocking is set to False. + remote execution that consists of the nodes reachable from the passed-in + node argument or entire graph if the node is None. The run() method returns + after the service creates the job, and the job status is reported as RUNNING. """ # A list of PTransformOverride objects to be applied before running a pipeline @@ -103,9 +83,6 @@ class DataflowRunner(PipelineRunner): # Imported here to avoid circular dependencies. # TODO: Remove the apache_beam.pipeline dependency in CreatePTransformOverride - from apache_beam.runners.dataflow.ptransform_overrides import CombineValuesPTransformOverride - from apache_beam.runners.dataflow.ptransform_overrides import CreatePTransformOverride - from apache_beam.runners.dataflow.ptransform_overrides import ReadPTransformOverride from apache_beam.runners.dataflow.ptransform_overrides import NativeReadPTransformOverride # These overrides should be applied before the proto representation of the @@ -114,32 +91,12 @@ class DataflowRunner(PipelineRunner): NativeReadPTransformOverride(), ] # type: List[PTransformOverride] - # These overrides should be applied after the proto representation of the - # graph is created. - _NON_PORTABLE_PTRANSFORM_OVERRIDES = [ - CombineValuesPTransformOverride(), - CreatePTransformOverride(), - ReadPTransformOverride(), - ] # type: List[PTransformOverride] - def __init__(self, cache=None): - # Cache of CloudWorkflowStep protos generated while the runner - # "executes" a pipeline. - self._cache = cache if cache is not None else PValueCache() - self._unique_step_id = 0 self._default_environment = None def is_fnapi_compatible(self): return False - def apply(self, transform, input, options): - _check_and_add_missing_options(options) - return super().apply(transform, input, options) - - def _get_unique_step_name(self): - self._unique_step_id += 1 - return 's%s' % self._unique_step_id - @staticmethod def poll_for_job_completion( runner, result, duration, state_update_callback=None): @@ -235,11 +192,21 @@ def rank_error(msg): # Skip empty messages. if m.messageImportance is None: continue - _LOGGER.info(message) - if str(m.messageImportance) == 'JOB_MESSAGE_ERROR': + message_importance = str(m.messageImportance) + if (message_importance == 'JOB_MESSAGE_DEBUG' or + message_importance == 'JOB_MESSAGE_DETAILED'): + _LOGGER.debug(message) + elif message_importance == 'JOB_MESSAGE_BASIC': + _LOGGER.info(message) + elif message_importance == 'JOB_MESSAGE_WARNING': + _LOGGER.warning(message) + elif message_importance == 'JOB_MESSAGE_ERROR': + _LOGGER.error(message) if rank_error(m.messageText) >= last_error_rank: last_error_rank = rank_error(m.messageText) last_error_msg = m.messageText + else: + _LOGGER.info(message) if not page_token: break @@ -262,7 +229,7 @@ def _only_element(iterable): return element @staticmethod - def side_input_visitor(is_runner_v2=False, deterministic_key_coders=True): + def side_input_visitor(deterministic_key_coders=True): # Imported here to avoid circular dependencies. # pylint: disable=wrong-import-order, wrong-import-position from apache_beam.pipeline import PipelineVisitor @@ -300,9 +267,8 @@ def visit_transform(self, transform_node): 'Unsupported access pattern for %r: %r' % (transform_node.full_label, access_pattern)) new_side_inputs.append(new_side_input) - if is_runner_v2: - transform_node.side_inputs = new_side_inputs - transform_node.transform.side_inputs = new_side_inputs + transform_node.side_inputs = new_side_inputs + transform_node.transform.side_inputs = new_side_inputs return SideInputVisitor() @@ -363,20 +329,12 @@ def _adjust_pipeline_for_dataflow_v2(self, pipeline): not pipeline._options.view_as( TypeOptions).allow_non_deterministic_key_coders)) - def _check_for_unsupported_features_on_non_portable_worker(self, pipeline): - pipeline.visit(self.combinefn_visitor()) - def run_pipeline(self, pipeline, options, pipeline_proto=None): """Remotely executes entire pipeline or parts reachable from node.""" if _is_runner_v2_disabled(options): - debug_options = options.view_as(DebugOptions) - if not debug_options.lookup_experiment('disable_runner_v2_until_v2.50'): - raise ValueError( - 'disable_runner_v2 is deprecated in Beam Python ' + - beam.version.__version__ + - ' and this execution mode will be removed in a future Beam SDK. ' - 'If needed, please use: ' - '"--experiments=disable_runner_v2_until_v2.50".') + raise ValueError( + 'Disabling Runner V2 no longer supported ' + 'using Beam Python %s.' % beam.version.__version__) # Label goog-dataflow-notebook if job is started from notebook. if is_in_notebook(): @@ -397,26 +355,14 @@ def run_pipeline(self, pipeline, options, pipeline_proto=None): 'Google Cloud Dataflow runner not available, ' 'please install apache_beam[gcp]') - if pipeline_proto or pipeline.contains_external_transforms: - if _is_runner_v2_disabled(options): - raise ValueError( - 'This pipeline contains cross language transforms, ' - 'which requires Runner V2.') - if not _is_runner_v2(options): - _LOGGER.info( - 'Automatically enabling Dataflow Runner V2 since the ' - 'pipeline used cross-language transforms.') - _add_runner_v2_missing_options(options) - - is_runner_v2 = _is_runner_v2(options) - if not is_runner_v2: - self._check_for_unsupported_features_on_non_portable_worker(pipeline) + _check_and_add_missing_options(options) # Convert all side inputs into a form acceptable to Dataflow. if pipeline: + pipeline.visit(self.combinefn_visitor()) + pipeline.visit( self.side_input_visitor( - _is_runner_v2(options), deterministic_key_coders=not options.view_as( TypeOptions).allow_non_deterministic_key_coders)) @@ -430,10 +376,6 @@ def run_pipeline(self, pipeline, options, pipeline_proto=None): "Native sinks no longer implemented; " "ignoring use_legacy_bq_sink.") - from apache_beam.runners.dataflow.ptransform_overrides import GroupIntoBatchesWithShardedKeyPTransformOverride - pipeline.replace_all( - [GroupIntoBatchesWithShardedKeyPTransformOverride(self, options)]) - if pipeline_proto: self.proto_pipeline = pipeline_proto @@ -449,7 +391,7 @@ def run_pipeline(self, pipeline, options, pipeline_proto=None): self._default_environment.container_image) else: artifacts = environments.python_sdk_dependencies(options) - if artifacts and _is_runner_v2(options): + if artifacts: _LOGGER.info( "Pipeline has additional dependencies to be installed " "in SDK worker container, consider using the SDK " @@ -501,11 +443,6 @@ def run_pipeline(self, pipeline, options, pipeline_proto=None): known_runner_urns=frozenset(), partial=True) - if not is_runner_v2: - # Performing configured PTransform overrides which should not be reflected - # in the proto representation of the graph. - pipeline.replace_all(DataflowRunner._NON_PORTABLE_PTRANSFORM_OVERRIDES) - # Add setup_options for all the BeamPlugin imports setup_options = options.view_as(SetupOptions) plugins = BeamPlugin.get_all_plugin_paths() @@ -523,16 +460,6 @@ def run_pipeline(self, pipeline, options, pipeline_proto=None): self.job = apiclient.Job(options, self.proto_pipeline) - # TODO: Consider skipping these for all use_portable_job_submission jobs. - if pipeline: - # Dataflow Runner v1 requires output type of the Flatten to be the same as - # the inputs, hence we enforce that here. Dataflow Runner v2 does not - # require this. - pipeline.visit(self.flatten_input_visitor()) - - # Trigger a traversal of all reachable nodes. - self.visit_transforms(pipeline, options) - test_options = options.view_as(TestOptions) # If it is a dry run, return without submitting the job. if test_options.dry_run: @@ -557,11 +484,6 @@ def run_pipeline(self, pipeline, options, pipeline_proto=None): result.metric_results = self._metrics return result - def _get_typehint_based_encoding(self, typehint, window_coder): - """Returns an encoding based on a typehint object.""" - return self._get_cloud_encoding( - self._get_coder(typehint, window_coder=window_coder)) - @staticmethod def _get_coder(typehint, window_coder): """Returns a coder based on a typehint object.""" @@ -570,201 +492,6 @@ def _get_coder(typehint, window_coder): coders.registry.get_coder(typehint), window_coder=window_coder) return coders.registry.get_coder(typehint) - def _get_cloud_encoding(self, coder, unused=None): - """Returns an encoding based on a coder object.""" - if not isinstance(coder, coders.Coder): - raise TypeError( - 'Coder object must inherit from coders.Coder: %s.' % str(coder)) - return coder.as_cloud_object(self.proto_context.coders) - - def _get_side_input_encoding(self, input_encoding): - """Returns an encoding for the output of a view transform. - - Args: - input_encoding: encoding of current transform's input. Side inputs need - this because the service will check that input and output types match. - - Returns: - An encoding that matches the output and input encoding. This is essential - for the View transforms introduced to produce side inputs to a ParDo. - """ - return { - '@type': 'kind:stream', - 'component_encodings': [input_encoding], - 'is_stream_like': { - 'value': True - }, - } - - def _get_encoded_output_coder( - self, transform_node, window_value=True, output_tag=None): - """Returns the cloud encoding of the coder for the output of a transform.""" - - if output_tag in transform_node.outputs: - element_type = transform_node.outputs[output_tag].element_type - elif len(transform_node.outputs) == 1: - output_tag = DataflowRunner._only_element(transform_node.outputs.keys()) - # TODO(robertwb): Handle type hints for multi-output transforms. - element_type = transform_node.outputs[output_tag].element_type - - else: - # TODO(silviuc): Remove this branch (and assert) when typehints are - # propagated everywhere. Returning an 'Any' as type hint will trigger - # usage of the fallback coder (i.e., cPickler). - element_type = typehints.Any - if window_value: - # All outputs have the same windowing. So getting the coder from an - # arbitrary window is fine. - output_tag = next(iter(transform_node.outputs.keys())) - window_coder = ( - transform_node.outputs[output_tag].windowing.windowfn. - get_window_coder()) - else: - window_coder = None - return self._get_typehint_based_encoding(element_type, window_coder) - - def get_pcoll_with_auto_sharding(self): - if not hasattr(self, '_pcoll_with_auto_sharding'): - return set() - return self._pcoll_with_auto_sharding - - def add_pcoll_with_auto_sharding(self, applied_ptransform): - if not hasattr(self, '_pcoll_with_auto_sharding'): - self.__setattr__('_pcoll_with_auto_sharding', set()) - output = DataflowRunner._only_element(applied_ptransform.outputs.keys()) - self._pcoll_with_auto_sharding.add( - applied_ptransform.outputs[output]._unique_name()) - - def _add_step(self, step_kind, step_label, transform_node, side_tags=()): - """Creates a Step object and adds it to the cache.""" - # Import here to avoid adding the dependency for local running scenarios. - # pylint: disable=wrong-import-order, wrong-import-position - from apache_beam.runners.dataflow.internal import apiclient - step = apiclient.Step(step_kind, self._get_unique_step_name()) - self.job.proto.steps.append(step.proto) - step.add_property(PropertyNames.USER_NAME, step_label) - # Cache the node/step association for the main output of the transform node. - - # External transforms may not use 'None' as an output tag. - output_tags = ([None] + - list(side_tags) if None in transform_node.outputs.keys() else - list(transform_node.outputs.keys())) - - # We have to cache output for all tags since some transforms may produce - # multiple outputs. - for output_tag in output_tags: - self._cache.cache_output(transform_node, output_tag, step) - - # Finally, we add the display data items to the pipeline step. - # If the transform contains no display data then an empty list is added. - step.add_property( - PropertyNames.DISPLAY_DATA, - [ - item.get_dict() - for item in DisplayData.create_from(transform_node.transform).items - ]) - - if transform_node.resource_hints: - step.add_property( - PropertyNames.RESOURCE_HINTS, - { - hint: quote_from_bytes(value) - for (hint, value) in transform_node.resource_hints.items() - }) - - return step - - def _add_singleton_step( - self, - label, - full_label, - tag, - input_step, - windowing_strategy, - access_pattern): - """Creates a CollectionToSingleton step used to handle ParDo side inputs.""" - # Import here to avoid adding the dependency for local running scenarios. - from apache_beam.runners.dataflow.internal import apiclient - step = apiclient.Step(TransformNames.COLLECTION_TO_SINGLETON, label) - self.job.proto.steps.append(step.proto) - step.add_property(PropertyNames.USER_NAME, full_label) - step.add_property( - PropertyNames.PARALLEL_INPUT, - { - '@type': 'OutputReference', - PropertyNames.STEP_NAME: input_step.proto.name, - PropertyNames.OUTPUT_NAME: input_step.get_output(tag) - }) - step.encoding = self._get_side_input_encoding(input_step.encoding) - - output_info = { - PropertyNames.USER_NAME: '%s.%s' % (full_label, PropertyNames.OUTPUT), - PropertyNames.ENCODING: step.encoding, - PropertyNames.OUTPUT_NAME: PropertyNames.OUT - } - if common_urns.side_inputs.MULTIMAP.urn == access_pattern: - output_info[PropertyNames.USE_INDEXED_FORMAT] = True - step.add_property(PropertyNames.OUTPUT_INFO, [output_info]) - - step.add_property( - PropertyNames.WINDOWING_STRATEGY, - self.serialize_windowing_strategy( - windowing_strategy, self._default_environment)) - return step - - def run_Impulse(self, transform_node, options): - step = self._add_step( - TransformNames.READ, transform_node.full_label, transform_node) - step.add_property(PropertyNames.FORMAT, 'impulse') - encoded_impulse_element = coders.WindowedValueCoder( - coders.BytesCoder(), - coders.coders.GlobalWindowCoder()).get_impl().encode_nested( - window.GlobalWindows.windowed_value(b'')) - if _is_runner_v2(options): - encoded_impulse_as_str = self.byte_array_to_json_string( - encoded_impulse_element) - else: - encoded_impulse_as_str = base64.b64encode(encoded_impulse_element).decode( - 'ascii') - - step.add_property(PropertyNames.IMPULSE_ELEMENT, encoded_impulse_as_str) - - step.encoding = self._get_encoded_output_coder(transform_node) - step.add_property( - PropertyNames.OUTPUT_INFO, - [{ - PropertyNames.USER_NAME: ( - '%s.%s' % (transform_node.full_label, PropertyNames.OUT)), - PropertyNames.ENCODING: step.encoding, - PropertyNames.OUTPUT_NAME: PropertyNames.OUT - }]) - - def run_Flatten(self, transform_node, options): - step = self._add_step( - TransformNames.FLATTEN, transform_node.full_label, transform_node) - inputs = [] - for one_input in transform_node.inputs: - input_step = self._cache.get_pvalue(one_input) - inputs.append({ - '@type': 'OutputReference', - PropertyNames.STEP_NAME: input_step.proto.name, - PropertyNames.OUTPUT_NAME: input_step.get_output(one_input.tag) - }) - step.add_property(PropertyNames.INPUTS, inputs) - step.encoding = self._get_encoded_output_coder(transform_node) - step.add_property( - PropertyNames.OUTPUT_INFO, - [{ - PropertyNames.USER_NAME: ( - '%s.%s' % (transform_node.full_label, PropertyNames.OUT)), - PropertyNames.ENCODING: step.encoding, - PropertyNames.OUTPUT_NAME: PropertyNames.OUT - }]) - - # TODO(srohde): Remove this after internal usages have been removed. - def apply_GroupByKey(self, transform, pcoll, options): - return transform.expand(pcoll) - def _verify_gbk_coders(self, transform, pcoll): # Infer coder of parent. # @@ -784,512 +511,6 @@ def _verify_gbk_coders(self, transform, pcoll): coders.registry.verify_deterministic( coder.key_coder(), 'GroupByKey operation "%s"' % transform.label) - def run_GroupByKey(self, transform_node, options): - input_tag = transform_node.inputs[0].tag - input_step = self._cache.get_pvalue(transform_node.inputs[0]) - - # Verify that the GBK's parent has a KV coder. - self._verify_gbk_coders(transform_node.transform, transform_node.inputs[0]) - - step = self._add_step( - TransformNames.GROUP, transform_node.full_label, transform_node) - step.add_property( - PropertyNames.PARALLEL_INPUT, - { - '@type': 'OutputReference', - PropertyNames.STEP_NAME: input_step.proto.name, - PropertyNames.OUTPUT_NAME: input_step.get_output(input_tag) - }) - step.encoding = self._get_encoded_output_coder(transform_node) - step.add_property( - PropertyNames.OUTPUT_INFO, - [{ - PropertyNames.USER_NAME: ( - '%s.%s' % (transform_node.full_label, PropertyNames.OUT)), - PropertyNames.ENCODING: step.encoding, - PropertyNames.OUTPUT_NAME: PropertyNames.OUT - }]) - windowing = transform_node.transform.get_windowing(transform_node.inputs) - step.add_property( - PropertyNames.SERIALIZED_FN, - self.serialize_windowing_strategy(windowing, self._default_environment)) - - def run_ExternalTransform(self, transform_node, options): - # Adds a dummy step to the Dataflow job description so that inputs and - # outputs are mapped correctly in the presence of external transforms. - # - # Note that Dataflow Python multi-language pipelines use Portable Job - # Submission by default, hence this step and rest of the Dataflow step - # definitions defined here are not used at Dataflow service but we have to - # maintain the mapping correctly till we can fully drop the Dataflow step - # definitions from the SDK. - - # AppliedTransform node outputs have to be updated to correctly map the - # outputs for external transforms. - transform_node.outputs = ({ - output.tag: output - for output in transform_node.outputs.values() - }) - - self.run_Impulse(transform_node, options) - - def run_ParDo(self, transform_node, options): - transform = transform_node.transform - input_tag = transform_node.inputs[0].tag - input_step = self._cache.get_pvalue(transform_node.inputs[0]) - - # Attach side inputs. - si_dict = {} - si_labels = {} - full_label_counts = defaultdict(int) - lookup_label = lambda side_pval: si_labels[side_pval] - named_inputs = transform_node.named_inputs() - label_renames = {} - for ix, side_pval in enumerate(transform_node.side_inputs): - assert isinstance(side_pval, AsSideInput) - step_name = 'SideInput-' + self._get_unique_step_name() - si_label = ((SIDE_INPUT_PREFIX + '%d-%s') % - (ix, transform_node.full_label)) - old_label = (SIDE_INPUT_PREFIX + '%d') % ix - - label_renames[old_label] = si_label - - assert old_label in named_inputs - pcollection_label = '%s.%s' % ( - side_pval.pvalue.producer.full_label.split('/')[-1], - side_pval.pvalue.tag if side_pval.pvalue.tag else 'out') - si_full_label = '%s/%s(%s.%s)' % ( - transform_node.full_label, - side_pval.__class__.__name__, - pcollection_label, - full_label_counts[pcollection_label]) - - # Count the number of times the same PCollection is a side input - # to the same ParDo. - full_label_counts[pcollection_label] += 1 - - self._add_singleton_step( - step_name, - si_full_label, - side_pval.pvalue.tag, - self._cache.get_pvalue(side_pval.pvalue), - side_pval.pvalue.windowing, - side_pval._side_input_data().access_pattern) - si_dict[si_label] = { - '@type': 'OutputReference', - PropertyNames.STEP_NAME: step_name, - PropertyNames.OUTPUT_NAME: PropertyNames.OUT - } - si_labels[side_pval] = si_label - - # Now create the step for the ParDo transform being handled. - transform_name = transform_node.full_label.rsplit('/', 1)[-1] - step = self._add_step( - TransformNames.DO, - transform_node.full_label + - ('/{}'.format(transform_name) if transform_node.side_inputs else ''), - transform_node, - transform_node.transform.output_tags) - transform_proto = self.proto_context.transforms.get_proto(transform_node) - transform_id = self.proto_context.transforms.get_id(transform_node) - is_runner_v2 = _is_runner_v2(options) - # Patch side input ids to be unique across a given pipeline. - if (label_renames and - transform_proto.spec.urn == common_urns.primitives.PAR_DO.urn): - # Patch PTransform proto. - for old, new in label_renames.items(): - transform_proto.inputs[new] = transform_proto.inputs[old] - del transform_proto.inputs[old] - - # Patch ParDo proto. - proto_type, _ = beam.PTransform._known_urns[transform_proto.spec.urn] - proto = proto_utils.parse_Bytes(transform_proto.spec.payload, proto_type) - for old, new in label_renames.items(): - proto.side_inputs[new].CopyFrom(proto.side_inputs[old]) - del proto.side_inputs[old] - transform_proto.spec.payload = proto.SerializeToString() - # We need to update the pipeline proto. - del self.proto_pipeline.components.transforms[transform_id] - ( - self.proto_pipeline.components.transforms[transform_id].CopyFrom( - transform_proto)) - # The data transmitted in SERIALIZED_FN is different depending on whether - # this is a runner v2 pipeline or not. - if is_runner_v2: - serialized_data = transform_id - else: - serialized_data = pickler.dumps( - self._pardo_fn_data(transform_node, lookup_label)) - step.add_property(PropertyNames.SERIALIZED_FN, serialized_data) - # TODO(BEAM-8882): Enable once dataflow service doesn't reject this. - # step.add_property(PropertyNames.PIPELINE_PROTO_TRANSFORM_ID, transform_id) - step.add_property( - PropertyNames.PARALLEL_INPUT, - { - '@type': 'OutputReference', - PropertyNames.STEP_NAME: input_step.proto.name, - PropertyNames.OUTPUT_NAME: input_step.get_output(input_tag) - }) - # Add side inputs if any. - step.add_property(PropertyNames.NON_PARALLEL_INPUTS, si_dict) - - # Generate description for the outputs. The output names - # will be 'None' for main output and '' for a tagged output. - outputs = [] - - all_output_tags = list(transform_proto.outputs.keys()) - - # Some external transforms require output tags to not be modified. - # So we randomly select one of the output tags as the main output and - # leave others as side outputs. Transform execution should not change - # dependending on which output tag we choose as the main output here. - # Also, some SDKs do not work correctly if output tags are modified. So for - # external transforms, we leave tags unmodified. - # - # Python SDK uses 'None' as the tag of the main output. - main_output_tag = 'None' - - step.encoding = self._get_encoded_output_coder( - transform_node, output_tag=main_output_tag) - - side_output_tags = set(all_output_tags).difference({main_output_tag}) - - # Add the main output to the description. - outputs.append({ - PropertyNames.USER_NAME: ( - '%s.%s' % (transform_node.full_label, PropertyNames.OUT)), - PropertyNames.ENCODING: step.encoding, - PropertyNames.OUTPUT_NAME: main_output_tag - }) - for side_tag in side_output_tags: - # The assumption here is that all outputs will have the same typehint - # and coder as the main output. This is certainly the case right now - # but conceivably it could change in the future. - encoding = self._get_encoded_output_coder( - transform_node, output_tag=side_tag) - outputs.append({ - PropertyNames.USER_NAME: ( - '%s.%s' % (transform_node.full_label, side_tag)), - PropertyNames.ENCODING: encoding, - PropertyNames.OUTPUT_NAME: side_tag - }) - - step.add_property(PropertyNames.OUTPUT_INFO, outputs) - - # Add the restriction encoding if we are a splittable DoFn - restriction_coder = transform.get_restriction_coder() - if restriction_coder: - step.add_property( - PropertyNames.RESTRICTION_ENCODING, - self._get_cloud_encoding(restriction_coder)) - - if options.view_as(StandardOptions).streaming: - is_stateful_dofn = (DoFnSignature(transform.dofn).is_stateful_dofn()) - if is_stateful_dofn: - step.add_property(PropertyNames.USES_KEYED_STATE, 'true') - - # Also checks whether the step allows shardable keyed states. - # TODO(BEAM-11360): remove this when migrated to portable job - # submission since we only consider supporting the property in runner - # v2. - for pcoll in transform_node.outputs.values(): - if pcoll._unique_name() in self.get_pcoll_with_auto_sharding(): - step.add_property(PropertyNames.ALLOWS_SHARDABLE_STATE, 'true') - # Currently we only allow auto-sharding to be enabled through the - # GroupIntoBatches transform. So we also add the following property - # which GroupIntoBatchesDoFn has, to allow the backend to perform - # graph optimization. - step.add_property(PropertyNames.PRESERVES_KEYS, 'true') - break - - @staticmethod - def _pardo_fn_data(transform_node, get_label): - transform = transform_node.transform - si_tags_and_types = [ # pylint: disable=protected-access - (get_label(side_pval), side_pval.__class__, side_pval._view_options()) - for side_pval in transform_node.side_inputs] - return ( - transform.fn, - transform.args, - transform.kwargs, - si_tags_and_types, - transform_node.inputs[0].windowing) - - def run_CombineValuesReplacement(self, transform_node, options): - transform = transform_node.transform.transform - input_tag = transform_node.inputs[0].tag - input_step = self._cache.get_pvalue(transform_node.inputs[0]) - step = self._add_step( - TransformNames.COMBINE, transform_node.full_label, transform_node) - transform_id = self.proto_context.transforms.get_id(transform_node.parent) - - # The data transmitted in SERIALIZED_FN is different depending on whether - # this is a runner v2 pipeline or not. - if _is_runner_v2(options): - # Fnapi pipelines send the transform ID of the CombineValues transform's - # parent composite because Dataflow expects the ID of a CombinePerKey - # transform. - serialized_data = transform_id - else: - # Combiner functions do not take deferred side-inputs (i.e. PValues) and - # therefore the code to handle extra args/kwargs is simpler than for the - # DoFn's of the ParDo transform. In the last, empty argument is where - # side inputs information would go. - serialized_data = pickler.dumps( - (transform.fn, transform.args, transform.kwargs, ())) - step.add_property(PropertyNames.SERIALIZED_FN, serialized_data) - # TODO(BEAM-8882): Enable once dataflow service doesn't reject this. - # step.add_property(PropertyNames.PIPELINE_PROTO_TRANSFORM_ID, transform_id) - step.add_property( - PropertyNames.PARALLEL_INPUT, - { - '@type': 'OutputReference', - PropertyNames.STEP_NAME: input_step.proto.name, - PropertyNames.OUTPUT_NAME: input_step.get_output(input_tag) - }) - # Note that the accumulator must not have a WindowedValue encoding, while - # the output of this step does in fact have a WindowedValue encoding. - accumulator_encoding = self._get_cloud_encoding( - transform.fn.get_accumulator_coder()) - output_encoding = self._get_encoded_output_coder(transform_node) - - step.encoding = output_encoding - step.add_property(PropertyNames.ENCODING, accumulator_encoding) - # Generate description for main output 'out.' - outputs = [] - # Add the main output to the description. - outputs.append({ - PropertyNames.USER_NAME: ( - '%s.%s' % (transform_node.full_label, PropertyNames.OUT)), - PropertyNames.ENCODING: step.encoding, - PropertyNames.OUTPUT_NAME: PropertyNames.OUT - }) - step.add_property(PropertyNames.OUTPUT_INFO, outputs) - - def run_Read(self, transform_node, options): - transform = transform_node.transform - step = self._add_step( - TransformNames.READ, transform_node.full_label, transform_node) - # TODO(mairbek): refactor if-else tree to use registerable functions. - # Initialize the source specific properties. - - standard_options = options.view_as(StandardOptions) - if not hasattr(transform.source, 'format'): - # If a format is not set, we assume the source to be a custom source. - source_dict = {} - - source_dict['spec'] = { - '@type': names.SOURCE_TYPE, - names.SERIALIZED_SOURCE_KEY: pickler.dumps(transform.source) - } - - try: - source_dict['metadata'] = { - 'estimated_size_bytes': json_value.get_typed_value_descriptor( - transform.source.estimate_size()) - } - except error.RuntimeValueProviderError: - # Size estimation is best effort, and this error is by value provider. - _LOGGER.info( - 'Could not estimate size of source %r due to ' + \ - 'RuntimeValueProviderError', transform.source) - except Exception: # pylint: disable=broad-except - # Size estimation is best effort. So we log the error and continue. - _LOGGER.info( - 'Could not estimate size of source %r due to an exception: %s', - transform.source, - traceback.format_exc()) - - step.add_property(PropertyNames.SOURCE_STEP_INPUT, source_dict) - elif transform.source.format == 'pubsub': - if not standard_options.streaming: - raise ValueError( - 'Cloud Pub/Sub is currently available for use ' - 'only in streaming pipelines.') - # Only one of topic or subscription should be set. - if transform.source.full_subscription: - step.add_property( - PropertyNames.PUBSUB_SUBSCRIPTION, - transform.source.full_subscription) - elif transform.source.full_topic: - step.add_property( - PropertyNames.PUBSUB_TOPIC, transform.source.full_topic) - if transform.source.id_label: - step.add_property( - PropertyNames.PUBSUB_ID_LABEL, transform.source.id_label) - if transform.source.with_attributes: - # Setting this property signals Dataflow runner to return full - # PubsubMessages instead of just the data part of the payload. - step.add_property(PropertyNames.PUBSUB_SERIALIZED_ATTRIBUTES_FN, '') - - if transform.source.timestamp_attribute is not None: - step.add_property( - PropertyNames.PUBSUB_TIMESTAMP_ATTRIBUTE, - transform.source.timestamp_attribute) - else: - raise ValueError( - 'Source %r has unexpected format %s.' % - (transform.source, transform.source.format)) - - if not hasattr(transform.source, 'format'): - step.add_property(PropertyNames.FORMAT, names.SOURCE_FORMAT) - else: - step.add_property(PropertyNames.FORMAT, transform.source.format) - - # Wrap coder in WindowedValueCoder: this is necessary as the encoding of a - # step should be the type of value outputted by each step. Read steps - # automatically wrap output values in a WindowedValue wrapper, if necessary. - # This is also necessary for proper encoding for size estimation. - # Using a GlobalWindowCoder as a place holder instead of the default - # PickleCoder because GlobalWindowCoder is known coder. - # TODO(robertwb): Query the collection for the windowfn to extract the - # correct coder. - coder = coders.WindowedValueCoder( - coders.registry.get_coder(transform_node.outputs[None].element_type), - coders.coders.GlobalWindowCoder()) - - step.encoding = self._get_cloud_encoding(coder) - step.add_property( - PropertyNames.OUTPUT_INFO, - [{ - PropertyNames.USER_NAME: ( - '%s.%s' % (transform_node.full_label, PropertyNames.OUT)), - PropertyNames.ENCODING: step.encoding, - PropertyNames.OUTPUT_NAME: PropertyNames.OUT - }]) - - def run__NativeWrite(self, transform_node, options): - transform = transform_node.transform - input_tag = transform_node.inputs[0].tag - input_step = self._cache.get_pvalue(transform_node.inputs[0]) - step = self._add_step( - TransformNames.WRITE, transform_node.full_label, transform_node) - # TODO(mairbek): refactor if-else tree to use registerable functions. - # Initialize the sink specific properties. - if transform.sink.format == 'pubsub': - standard_options = options.view_as(StandardOptions) - if not standard_options.streaming: - raise ValueError( - 'Cloud Pub/Sub is currently available for use ' - 'only in streaming pipelines.') - step.add_property(PropertyNames.PUBSUB_TOPIC, transform.sink.full_topic) - if transform.sink.id_label: - step.add_property( - PropertyNames.PUBSUB_ID_LABEL, transform.sink.id_label) - # Setting this property signals Dataflow runner that the PCollection - # contains PubsubMessage objects instead of just raw data. - step.add_property(PropertyNames.PUBSUB_SERIALIZED_ATTRIBUTES_FN, '') - if transform.sink.timestamp_attribute is not None: - step.add_property( - PropertyNames.PUBSUB_TIMESTAMP_ATTRIBUTE, - transform.sink.timestamp_attribute) - else: - raise ValueError( - 'Sink %r has unexpected format %s.' % - (transform.sink, transform.sink.format)) - step.add_property(PropertyNames.FORMAT, transform.sink.format) - - # Wrap coder in WindowedValueCoder: this is necessary for proper encoding - # for size estimation. Using a GlobalWindowCoder as a place holder instead - # of the default PickleCoder because GlobalWindowCoder is known coder. - # TODO(robertwb): Query the collection for the windowfn to extract the - # correct coder. - coder = coders.WindowedValueCoder( - transform.sink.coder, coders.coders.GlobalWindowCoder()) - step.encoding = self._get_cloud_encoding(coder) - step.add_property(PropertyNames.ENCODING, step.encoding) - step.add_property( - PropertyNames.PARALLEL_INPUT, - { - '@type': 'OutputReference', - PropertyNames.STEP_NAME: input_step.proto.name, - PropertyNames.OUTPUT_NAME: input_step.get_output(input_tag) - }) - - def run_TestStream(self, transform_node, options): - from apache_beam.testing.test_stream import ElementEvent - from apache_beam.testing.test_stream import ProcessingTimeEvent - from apache_beam.testing.test_stream import WatermarkEvent - standard_options = options.view_as(StandardOptions) - if not standard_options.streaming: - raise ValueError( - 'TestStream is currently available for use ' - 'only in streaming pipelines.') - - transform = transform_node.transform - step = self._add_step( - TransformNames.READ, transform_node.full_label, transform_node) - step.add_property( - PropertyNames.SERIALIZED_FN, - self.proto_context.transforms.get_id(transform_node)) - step.add_property(PropertyNames.FORMAT, 'test_stream') - test_stream_payload = beam_runner_api_pb2.TestStreamPayload() - # TestStream source doesn't do any decoding of elements, - # so we won't set test_stream_payload.coder_id. - output_coder = transform._infer_output_coder() # pylint: disable=protected-access - for event in transform._events: - new_event = test_stream_payload.events.add() - if isinstance(event, ElementEvent): - for tv in event.timestamped_values: - element = new_event.element_event.elements.add() - element.encoded_element = output_coder.encode(tv.value) - element.timestamp = tv.timestamp.micros - elif isinstance(event, ProcessingTimeEvent): - new_event.processing_time_event.advance_duration = ( - event.advance_by.micros) - elif isinstance(event, WatermarkEvent): - new_event.watermark_event.new_watermark = event.new_watermark.micros - serialized_payload = self.byte_array_to_json_string( - test_stream_payload.SerializeToString()) - step.add_property(PropertyNames.SERIALIZED_TEST_STREAM, serialized_payload) - - step.encoding = self._get_encoded_output_coder(transform_node) - step.add_property( - PropertyNames.OUTPUT_INFO, - [{ - PropertyNames.USER_NAME: ( - '%s.%s' % (transform_node.full_label, PropertyNames.OUT)), - PropertyNames.ENCODING: step.encoding, - PropertyNames.OUTPUT_NAME: PropertyNames.OUT - }]) - - # We must mark this method as not a test or else its name is a matcher for - # nosetest tests. - run_TestStream.__test__ = False # type: ignore[attr-defined] - - @classmethod - def serialize_windowing_strategy(cls, windowing, default_environment): - from apache_beam.runners import pipeline_context - context = pipeline_context.PipelineContext( - default_environment=default_environment) - windowing_proto = windowing.to_runner_api(context) - return cls.byte_array_to_json_string( - beam_runner_api_pb2.MessageWithComponents( - components=context.to_runner_api(), - windowing_strategy=windowing_proto).SerializeToString()) - - @classmethod - def deserialize_windowing_strategy(cls, serialized_data): - # Imported here to avoid circular dependencies. - # pylint: disable=wrong-import-order, wrong-import-position - from apache_beam.runners import pipeline_context - from apache_beam.transforms.core import Windowing - proto = beam_runner_api_pb2.MessageWithComponents() - proto.ParseFromString(cls.json_string_to_byte_array(serialized_data)) - return Windowing.from_runner_api( - proto.windowing_strategy, - pipeline_context.PipelineContext(proto.components)) - - @staticmethod - def byte_array_to_json_string(raw_bytes): - """Implements org.apache.beam.sdk.util.StringUtils.byteArrayToJsonString.""" - return quote(raw_bytes) - - @staticmethod - def json_string_to_byte_array(encoded_string): - """Implements org.apache.beam.sdk.util.StringUtils.jsonStringToByteArray.""" - return unquote_to_bytes(encoded_string) - def get_default_gcp_region(self): """Get a default value for Google Cloud region according to https://cloud.google.com/compute/docs/gcloud-compute/#default-properties. @@ -1348,6 +569,8 @@ def _check_and_add_missing_options(options): options.view_as( GoogleCloudOptions).dataflow_service_options = dataflow_service_options + _add_runner_v2_missing_options(options) + # Ensure that prime is specified as an experiment if specified as a dataflow # service option if 'enable_prime' in dataflow_service_options: @@ -1355,15 +578,22 @@ def _check_and_add_missing_options(options): elif debug_options.lookup_experiment('enable_prime'): dataflow_service_options.append('enable_prime') + sdk_location = options.view_as(SetupOptions).sdk_location + if 'dev' in beam.version.__version__ and sdk_location == 'default': + raise ValueError( + "You are submitting a pipeline with Apache Beam Python SDK " + f"{beam.version.__version__}. " + "When launching Dataflow jobs with an unreleased (dev) SDK, " + "please provide an SDK distribution in the --sdk_location option " + "to use a consistent SDK version at " + "pipeline submission and runtime. To ignore this error and use " + "an SDK preinstalled in the default Dataflow dev runtime environment " + "or in a custom container image, use --sdk_location=container.") + # Streaming only supports using runner v2 (aka unified worker). # Runner v2 only supports using streaming engine (aka windmill service) if options.view_as(StandardOptions).streaming: google_cloud_options = options.view_as(GoogleCloudOptions) - if _is_runner_v2_disabled(options): - raise ValueError( - 'Disabling Runner V2 no longer supported for streaming pipeline ' - 'using Beam Python %s.' % beam.version.__version__) - if (not google_cloud_options.enable_streaming_engine and (debug_options.lookup_experiment("enable_windmill_service") or debug_options.lookup_experiment("enable_streaming_engine"))): @@ -1380,29 +610,6 @@ def _check_and_add_missing_options(options): google_cloud_options.enable_streaming_engine = True debug_options.add_experiment("enable_streaming_engine") debug_options.add_experiment("enable_windmill_service") - _add_runner_v2_missing_options(debug_options) - elif (debug_options.lookup_experiment('enable_prime') or - debug_options.lookup_experiment('beam_fn_api') or - debug_options.lookup_experiment('use_unified_worker') or - debug_options.lookup_experiment('use_runner_v2') or - debug_options.lookup_experiment('use_portable_job_submission')): - if _is_runner_v2_disabled(options): - raise ValueError( - """Runner V2 both disabled and enabled: at least one of - ['enable_prime', 'beam_fn_api', 'use_unified_worker', 'use_runner_v2', - 'use_portable_job_submission'] is set and also one of - ['disable_runner_v2', 'disable_runner_v2_until_2023', - 'disable_prime_runner_v2'] is set.""") - _add_runner_v2_missing_options(debug_options) - - -def _is_runner_v2(options): - # Type: (PipelineOptions) -> bool - - """Returns true if runner v2 is enabled.""" - _check_and_add_missing_options(options) - return options.view_as(DebugOptions).lookup_experiment( - 'use_runner_v2', default=False) def _is_runner_v2_disabled(options): diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py index b00644d13fd70..6258aa8f213b9 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py @@ -19,19 +19,13 @@ # pytype: skip-file -import json import unittest -from datetime import datetime -from itertools import product import mock -from parameterized import param -from parameterized import parameterized import apache_beam as beam import apache_beam.transforms as ptransform from apache_beam.options.pipeline_options import DebugOptions -from apache_beam.options.pipeline_options import GoogleCloudOptions from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.pipeline import AppliedPTransform from apache_beam.pipeline import Pipeline @@ -45,18 +39,13 @@ from apache_beam.runners import create_runner from apache_beam.runners.dataflow.dataflow_runner import DataflowPipelineResult from apache_beam.runners.dataflow.dataflow_runner import DataflowRuntimeException -from apache_beam.runners.dataflow.dataflow_runner import PropertyNames -from apache_beam.runners.dataflow.dataflow_runner import _is_runner_v2 -from apache_beam.runners.dataflow.dataflow_runner import _is_runner_v2_disabled +from apache_beam.runners.dataflow.dataflow_runner import _check_and_add_missing_options from apache_beam.runners.dataflow.internal.clients import dataflow as dataflow_api from apache_beam.runners.runner import PipelineState from apache_beam.testing.extra_assertions import ExtraAssertionsMixin from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.transforms import combiners from apache_beam.transforms import environments -from apache_beam.transforms import window -from apache_beam.transforms.core import Windowing -from apache_beam.transforms.display import DisplayDataItem from apache_beam.typehints import typehints # Protect against environments where apitools library is not available. @@ -98,11 +87,11 @@ def process(self): class DataflowRunnerTest(unittest.TestCase, ExtraAssertionsMixin): def setUp(self): self.default_properties = [ - '--dataflow_endpoint=ignored', '--job_name=test-job', '--project=test-project', - '--staging_location=ignored', - '--temp_location=/dev/null', + '--region=us-central1' + '--staging_location=gs://beam/test', + '--temp_location=gs://beam/tmp', '--no_auth', '--dry_run=True', '--sdk_location=container' @@ -262,49 +251,6 @@ def test_remote_runner_translation(self): | 'Do' >> ptransform.FlatMap(lambda x: [(x, x)]) | ptransform.GroupByKey()) - def test_remote_runner_display_data(self): - remote_runner = DataflowRunner() - p = Pipeline( - remote_runner, options=PipelineOptions(self.default_properties)) - - now = datetime.now() - # pylint: disable=expression-not-assigned - ( - p | ptransform.Create([1, 2, 3, 4, 5]) - | 'Do' >> SpecialParDo(SpecialDoFn(), now)) - - # TODO(https://github.com/apache/beam/issues/18012) Enable runner API on - # this test. - p.run(test_runner_api=False) - job_dict = json.loads(str(remote_runner.job)) - steps = [ - step for step in job_dict['steps'] - if len(step['properties'].get('display_data', [])) > 0 - ] - step = steps[1] - disp_data = step['properties']['display_data'] - nspace = SpecialParDo.__module__ + '.' - expected_data = [{ - 'type': 'TIMESTAMP', - 'namespace': nspace + 'SpecialParDo', - 'value': DisplayDataItem._format_value(now, 'TIMESTAMP'), - 'key': 'a_time' - }, - { - 'type': 'STRING', - 'namespace': nspace + 'SpecialParDo', - 'value': nspace + 'SpecialParDo', - 'key': 'a_class', - 'shortValue': 'SpecialParDo' - }, - { - 'type': 'INTEGER', - 'namespace': nspace + 'SpecialDoFn', - 'value': 42, - 'key': 'dofn_value' - }] - self.assertUnhashableCountEqual(disp_data, expected_data) - def test_group_by_key_input_visitor_with_valid_inputs(self): p = TestPipeline() pcoll1 = PCollection(p) @@ -391,15 +337,6 @@ def test_gbk_then_flatten_input_visitor(self): self.assertEqual(flat.element_type, none_str_pc.element_type) self.assertEqual(flat.element_type, none_int_pc.element_type) - def test_serialize_windowing_strategy(self): - # This just tests the basic path; more complete tests - # are in window_test.py. - strategy = Windowing(window.FixedWindows(10)) - self.assertEqual( - strategy, - DataflowRunner.deserialize_windowing_strategy( - DataflowRunner.serialize_windowing_strategy(strategy, None))) - def test_side_input_visitor(self): p = TestPipeline() pc = p | beam.Create([]) @@ -411,8 +348,7 @@ def test_side_input_visitor(self): beam.pvalue.AsSingleton(pc), beam.pvalue.AsMultiMap(pc)) applied_transform = AppliedPTransform(None, transform, "label", {'pc': pc}) - DataflowRunner.side_input_visitor( - is_runner_v2=True).visit_transform(applied_transform) + DataflowRunner.side_input_visitor().visit_transform(applied_transform) self.assertEqual(2, len(applied_transform.side_inputs)) self.assertEqual( common_urns.side_inputs.ITERABLE.urn, @@ -504,116 +440,6 @@ def test_get_default_gcp_region_ignores_error( result = runner.get_default_gcp_region() self.assertIsNone(result) - def test_combine_values_translation(self): - runner = DataflowRunner() - - with beam.Pipeline(runner=runner, - options=PipelineOptions(self.default_properties)) as p: - ( # pylint: disable=expression-not-assigned - p - | beam.Create([('a', [1, 2]), ('b', [3, 4])]) - | beam.CombineValues(lambda v, _: sum(v))) - - job_dict = json.loads(str(runner.job)) - self.assertIn( - 'CombineValues', set(step['kind'] for step in job_dict['steps'])) - - def _find_step(self, job, step_name): - job_dict = json.loads(str(job)) - maybe_step = [ - s for s in job_dict['steps'] - if s['properties']['user_name'] == step_name - ] - self.assertTrue(maybe_step, 'Could not find step {}'.format(step_name)) - return maybe_step[0] - - def expect_correct_override(self, job, step_name, step_kind): - """Expects that a transform was correctly overriden.""" - - # If the typing information isn't being forwarded correctly, the component - # encodings here will be incorrect. - expected_output_info = [{ - "encoding": { - "@type": "kind:windowed_value", - "component_encodings": [{ - "@type": "kind:bytes" - }, { - "@type": "kind:global_window" - }], - "is_wrapper": True - }, - "output_name": "out", - "user_name": step_name + ".out" - }] - - step = self._find_step(job, step_name) - self.assertEqual(step['kind'], step_kind) - - # The display data here is forwarded because the replace transform is - # subclassed from iobase.Read. - self.assertGreater(len(step['properties']['display_data']), 0) - self.assertEqual(step['properties']['output_info'], expected_output_info) - - def test_read_create_translation(self): - runner = DataflowRunner() - - with beam.Pipeline(runner=runner, - options=PipelineOptions(self.default_properties)) as p: - # pylint: disable=expression-not-assigned - p | beam.Create([b'a', b'b', b'c']) - - self.expect_correct_override(runner.job, 'Create/Read', 'ParallelRead') - - def test_read_pubsub_translation(self): - runner = DataflowRunner() - - self.default_properties.append("--streaming") - - with beam.Pipeline(runner=runner, - options=PipelineOptions(self.default_properties)) as p: - # pylint: disable=expression-not-assigned - p | beam.io.ReadFromPubSub(topic='projects/project/topics/topic') - - self.expect_correct_override( - runner.job, 'ReadFromPubSub/Read', 'ParallelRead') - - def test_gbk_translation(self): - runner = DataflowRunner() - with beam.Pipeline(runner=runner, - options=PipelineOptions(self.default_properties)) as p: - # pylint: disable=expression-not-assigned - p | beam.Create([(1, 2)]) | beam.GroupByKey() - - expected_output_info = [{ - "encoding": { - "@type": "kind:windowed_value", - "component_encodings": [{ - "@type": "kind:pair", - "component_encodings": [{ - "@type": "kind:varint" - }, - { - "@type": "kind:stream", - "component_encodings": [{ - "@type": "kind:varint" - }], - "is_stream_like": True - }], - "is_pair_like": True - }, { - "@type": "kind:global_window" - }], - "is_wrapper": True - }, - "output_name": "out", - "user_name": "GroupByKey.out" - }] # yapf: disable - - gbk_step = self._find_step(runner.job, 'GroupByKey') - self.assertEqual(gbk_step['kind'], 'GroupByKey') - self.assertEqual( - gbk_step['properties']['output_info'], expected_output_info) - @unittest.skip( 'https://github.com/apache/beam/issues/18716: enable once ' 'CombineFnVisitor is fixed') @@ -646,43 +472,6 @@ def teardown(self, *args, **kwargs): except ValueError: self.fail('ValueError raised unexpectedly') - def _run_group_into_batches_and_get_step_properties( - self, with_sharded_key, additional_properties): - self.default_properties.append('--streaming') - for property in additional_properties: - self.default_properties.append(property) - - runner = DataflowRunner() - with beam.Pipeline(runner=runner, - options=PipelineOptions(self.default_properties)) as p: - # pylint: disable=expression-not-assigned - input = p | beam.Create([('a', 1), ('a', 1), ('b', 3), ('b', 4)]) - if with_sharded_key: - ( - input | beam.GroupIntoBatches.WithShardedKey(2) - | beam.Map(lambda key_values: (key_values[0].key, key_values[1]))) - step_name = ( - 'WithShardedKey/GroupIntoBatches/ParDo(_GroupIntoBatchesDoFn)') - else: - input | beam.GroupIntoBatches(2) - step_name = 'GroupIntoBatches/ParDo(_GroupIntoBatchesDoFn)' - - return self._find_step(runner.job, step_name)['properties'] - - def test_group_into_batches_translation(self): - properties = self._run_group_into_batches_and_get_step_properties( - True, ['--enable_streaming_engine', '--experiments=use_runner_v2']) - self.assertEqual(properties[PropertyNames.USES_KEYED_STATE], 'true') - self.assertEqual(properties[PropertyNames.ALLOWS_SHARDABLE_STATE], 'true') - self.assertEqual(properties[PropertyNames.PRESERVES_KEYS], 'true') - - def test_group_into_batches_translation_non_sharded(self): - properties = self._run_group_into_batches_and_get_step_properties( - False, ['--enable_streaming_engine', '--experiments=use_runner_v2']) - self.assertEqual(properties[PropertyNames.USES_KEYED_STATE], 'true') - self.assertNotIn(PropertyNames.ALLOWS_SHARDABLE_STATE, properties) - self.assertNotIn(PropertyNames.PRESERVES_KEYS, properties) - def test_pack_combiners(self): class PackableCombines(beam.PTransform): def annotations(self): @@ -711,141 +500,51 @@ def expand(self, pcoll): self.assertNotIn(unpacked_maximum_step_name, transform_names) self.assertIn(packed_step_name, transform_names) - @parameterized.expand([ - param(memory_hint='min_ram'), - param(memory_hint='minRam'), - ]) - def test_resource_hints_translation(self, memory_hint): - runner = DataflowRunner() - self.default_properties.append('--resource_hint=accelerator=some_gpu') - self.default_properties.append(f'--resource_hint={memory_hint}=20GB') - with beam.Pipeline(runner=runner, - options=PipelineOptions(self.default_properties)) as p: - # pylint: disable=expression-not-assigned - ( - p - | beam.Create([1]) - | 'MapWithHints' >> beam.Map(lambda x: x + 1).with_resource_hints( - min_ram='10GB', - accelerator='type:nvidia-tesla-k80;count:1;install-nvidia-drivers' - )) - - step = self._find_step(runner.job, 'MapWithHints') - self.assertEqual( - step['properties']['resource_hints'], - { - 'beam:resources:min_ram_bytes:v1': '20000000000', - 'beam:resources:accelerator:v1': \ - 'type%3Anvidia-tesla-k80%3Bcount%3A1%3Binstall-nvidia-drivers' - }) - - @parameterized.expand([ - ( - "%s_%s" % (enable_option, disable_option), - enable_option, - disable_option) - for (enable_option, - disable_option) in product([ - False, - 'enable_prime', - 'beam_fn_api', - 'use_unified_worker', - 'use_runner_v2', - 'use_portable_job_submission' - ], - [ - False, - 'disable_runner_v2', - 'disable_runner_v2_until_2023', - 'disable_prime_runner_v2' - ]) - ]) - def test_batch_is_runner_v2(self, name, enable_option, disable_option): - options = PipelineOptions( - (['--experiments=%s' % enable_option] if enable_option else []) + - (['--experiments=%s' % disable_option] if disable_option else [])) - if (enable_option and disable_option): - with self.assertRaisesRegex(ValueError, - 'Runner V2 both disabled and enabled'): - _is_runner_v2(options) - elif enable_option: - self.assertTrue(_is_runner_v2(options)) - self.assertFalse(_is_runner_v2_disabled(options)) - for expected in ['beam_fn_api', - 'use_unified_worker', - 'use_runner_v2', - 'use_portable_job_submission']: - self.assertTrue( - options.view_as(DebugOptions).lookup_experiment(expected, False)) - if enable_option == 'enable_prime': - self.assertIn( - 'enable_prime', - options.view_as(GoogleCloudOptions).dataflow_service_options) - elif disable_option: - self.assertFalse(_is_runner_v2(options)) - self.assertTrue(_is_runner_v2_disabled(options)) - else: - self.assertFalse(_is_runner_v2(options)) - - @parameterized.expand([ - ( - "%s_%s" % (enable_option, disable_option), - enable_option, - disable_option) - for (enable_option, - disable_option) in product([ - False, - 'enable_prime', - 'beam_fn_api', - 'use_unified_worker', - 'use_runner_v2', - 'use_portable_job_submission' - ], - [ - False, - 'disable_runner_v2', - 'disable_runner_v2_until_2023', - 'disable_prime_runner_v2' - ]) - ]) - def test_streaming_is_runner_v2(self, name, enable_option, disable_option): - options = PipelineOptions( - ['--streaming'] + - (['--experiments=%s' % enable_option] if enable_option else []) + - (['--experiments=%s' % disable_option] if disable_option else [])) - if disable_option: - with self.assertRaisesRegex( - ValueError, - 'Disabling Runner V2 no longer supported for streaming pipeline'): - _is_runner_v2(options) - else: - self.assertTrue(_is_runner_v2(options)) - for expected in ['beam_fn_api', - 'use_unified_worker', - 'use_runner_v2', - 'use_portable_job_submission', - 'enable_windmill_service', - 'enable_streaming_engine']: - self.assertTrue( - options.view_as(DebugOptions).lookup_experiment(expected, False)) - if enable_option == 'enable_prime': - self.assertIn( - 'enable_prime', - options.view_as(GoogleCloudOptions).dataflow_service_options) + def test_batch_is_runner_v2(self): + options = PipelineOptions(['--sdk_location=container']) + _check_and_add_missing_options(options) + for expected in ['beam_fn_api', + 'use_unified_worker', + 'use_runner_v2', + 'use_portable_job_submission']: + self.assertTrue( + options.view_as(DebugOptions).lookup_experiment(expected, False), + expected) + + def test_streaming_is_runner_v2(self): + options = PipelineOptions(['--sdk_location=container', '--streaming']) + _check_and_add_missing_options(options) + for expected in ['beam_fn_api', + 'use_unified_worker', + 'use_runner_v2', + 'use_portable_job_submission', + 'enable_windmill_service', + 'enable_streaming_engine']: + self.assertTrue( + options.view_as(DebugOptions).lookup_experiment(expected, False), + expected) def test_dataflow_service_options_enable_prime_sets_runner_v2(self): - options = PipelineOptions(['--dataflow_service_options=enable_prime']) - self.assertTrue(_is_runner_v2(options)) + options = PipelineOptions([ + '--sdk_location=container', + '--streaming', + '--dataflow_service_options=enable_prime' + ]) + _check_and_add_missing_options(options) for expected in ['beam_fn_api', 'use_unified_worker', 'use_runner_v2', 'use_portable_job_submission']: self.assertTrue( - options.view_as(DebugOptions).lookup_experiment(expected, False)) - - options = PipelineOptions( - ['--streaming', '--dataflow_service_options=enable_prime']) - self.assertTrue(_is_runner_v2(options)) + options.view_as(DebugOptions).lookup_experiment(expected, False), + expected) + + options = PipelineOptions([ + '--sdk_location=container', + '--streaming', + '--dataflow_service_options=enable_prime' + ]) + _check_and_add_missing_options(options) for expected in ['beam_fn_api', 'use_unified_worker', 'use_runner_v2', @@ -853,7 +552,8 @@ def test_dataflow_service_options_enable_prime_sets_runner_v2(self): 'enable_windmill_service', 'enable_streaming_engine']: self.assertTrue( - options.view_as(DebugOptions).lookup_experiment(expected, False)) + options.view_as(DebugOptions).lookup_experiment(expected, False), + expected) if __name__ == '__main__': diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py index 3f0bfcb395bd9..bbcf880efdf72 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py @@ -38,7 +38,7 @@ import random import string -import pkg_resources +from packaging import version import re import sys import time @@ -65,7 +65,6 @@ from apache_beam.runners.common import validate_pipeline_graph from apache_beam.runners.dataflow.internal import names from apache_beam.runners.dataflow.internal.clients import dataflow -from apache_beam.runners.dataflow.internal.names import PropertyNames from apache_beam.runners.internal import names as shared_names from apache_beam.runners.portability.stager import Stager from apache_beam.transforms import DataflowDistributionCounter @@ -83,64 +82,7 @@ _LOGGER = logging.getLogger(__name__) -_PYTHON_VERSIONS_SUPPORTED_BY_DATAFLOW = ['3.7', '3.8', '3.9', '3.10', '3.11'] - - -class Step(object): - """Wrapper for a dataflow Step protobuf.""" - def __init__(self, step_kind, step_name, additional_properties=None): - self.step_kind = step_kind - self.step_name = step_name - self.proto = dataflow.Step(kind=step_kind, name=step_name) - self.proto.properties = {} - self._additional_properties = [] - - if additional_properties is not None: - for (n, v, t) in additional_properties: - self.add_property(n, v, t) - - def add_property(self, name, value, with_type=False): - self._additional_properties.append((name, value, with_type)) - self.proto.properties.additionalProperties.append( - dataflow.Step.PropertiesValue.AdditionalProperty( - key=name, value=to_json_value(value, with_type=with_type))) - - def _get_outputs(self): - """Returns a list of all output labels for a step.""" - outputs = [] - for p in self.proto.properties.additionalProperties: - if p.key == PropertyNames.OUTPUT_INFO: - for entry in p.value.array_value.entries: - for entry_prop in entry.object_value.properties: - if entry_prop.key == PropertyNames.OUTPUT_NAME: - outputs.append(entry_prop.value.string_value) - return outputs - - def __reduce__(self): - """Reduce hook for pickling the Step class more easily.""" - return (Step, (self.step_kind, self.step_name, self._additional_properties)) - - def get_output(self, tag=None): - """Returns name if it is one of the outputs or first output if name is None. - - Args: - tag: tag of the output as a string or None if we want to get the - name of the first output. - - Returns: - The name of the output associated with the tag or the first output - if tag was None. - - Raises: - ValueError: if the tag does not exist within outputs. - """ - outputs = self._get_outputs() - if tag is None or len(outputs) == 1: - return outputs[0] - else: - if tag not in outputs: - raise ValueError('Cannot find named output: %s in %s.' % (tag, outputs)) - return tag +_PYTHON_VERSIONS_SUPPORTED_BY_DATAFLOW = ['3.8', '3.9', '3.10', '3.11'] class Environment(object): @@ -152,7 +94,6 @@ def __init__( environment_version, proto_pipeline_staged_url, proto_pipeline=None): - from apache_beam.runners.dataflow.dataflow_runner import _is_runner_v2 self.standard_options = options.view_as(StandardOptions) self.google_cloud_options = options.view_as(GoogleCloudOptions) self.worker_options = options.view_as(WorkerOptions) @@ -192,10 +133,7 @@ def __init__( if self.standard_options.streaming: job_type = 'FNAPI_STREAMING' else: - if _is_runner_v2(options): - job_type = 'FNAPI_BATCH' - else: - job_type = 'PYTHON_BATCH' + job_type = 'FNAPI_BATCH' self.proto.version.additionalProperties.extend([ dataflow.Environment.VersionValue.AdditionalProperty( key='job_type', value=to_json_value(job_type)), @@ -297,7 +235,7 @@ def __init__( container_image.capabilities.append(capability) pool.sdkHarnessContainerImages.append(container_image) - if not _is_runner_v2(options) or not pool.sdkHarnessContainerImages: + if not pool.sdkHarnessContainerImages: pool.workerHarnessContainerImage = ( get_container_image_from_options(options)) elif len(pool.sdkHarnessContainerImages) == 1: @@ -554,11 +492,7 @@ def __init__(self, options, root_staging_location=None): self._root_staging_location = ( root_staging_location or self.google_cloud_options.staging_location) - from apache_beam.runners.dataflow.dataflow_runner import _is_runner_v2 - if _is_runner_v2(options): - self.environment_version = _FNAPI_ENVIRONMENT_MAJOR_VERSION - else: - self.environment_version = _LEGACY_ENVIRONMENT_MAJOR_VERSION + self.environment_version = _FNAPI_ENVIRONMENT_MAJOR_VERSION if self.google_cloud_options.no_auth: credentials = None @@ -1182,8 +1116,7 @@ def translate_value(value, metric_update_proto): def _get_container_image_tag(): - base_version = pkg_resources.parse_version( - beam_version.__version__).base_version + base_version = version.parse(beam_version.__version__).base_version if base_version != beam_version.__version__: warnings.warn( "A non-standard version of Beam SDK detected: %s. " @@ -1202,46 +1135,31 @@ def get_container_image_from_options(pipeline_options): Returns: str: Container image for remote execution. """ - from apache_beam.runners.dataflow.dataflow_runner import _is_runner_v2_disabled worker_options = pipeline_options.view_as(WorkerOptions) if worker_options.sdk_container_image: return worker_options.sdk_container_image - is_runner_v2 = not _is_runner_v2_disabled(pipeline_options) - # Legacy and runner v2 exist in different repositories. # Set to legacy format, override if runner v2 container_repo = names.DATAFLOW_CONTAINER_IMAGE_REPOSITORY - image_name = '{repository}/python{major}{minor}'.format( + image_name = '{repository}/beam_python{major}.{minor}_sdk'.format( repository=container_repo, major=sys.version_info[0], minor=sys.version_info[1]) - if is_runner_v2: - image_name = '{repository}/beam_python{major}.{minor}_sdk'.format( - repository=container_repo, - major=sys.version_info[0], - minor=sys.version_info[1]) - - image_tag = _get_required_container_version(is_runner_v2) + image_tag = _get_required_container_version() return image_name + ':' + image_tag -def _get_required_container_version(is_runner_v2): +def _get_required_container_version(): """For internal use only; no backwards-compatibility guarantees. - Args: - is_runner_v2 (bool): True if and only if pipeline is using runner v2. - Returns: str: The tag of worker container images in GCR that corresponds to current version of the SDK. """ if 'dev' in beam_version.__version__: - if is_runner_v2: - return names.BEAM_FNAPI_CONTAINER_VERSION - else: - return names.BEAM_CONTAINER_VERSION + return names.BEAM_DEV_SDK_CONTAINER_TAG else: return _get_container_image_tag() diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py index 89ac58a727bf6..d639ad21c31c2 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py @@ -635,7 +635,7 @@ def test_pinned_worker_harness_image_tag_used_in_dev_sdk(self): '/beam_python%d.%d_sdk:%s' % ( sys.version_info[0], sys.version_info[1], - names.BEAM_FNAPI_CONTAINER_VERSION))) + names.BEAM_DEV_SDK_CONTAINER_TAG))) pipeline_options = PipelineOptions( ['--temp_location', 'gs://any-location/temp']) @@ -651,26 +651,7 @@ def test_pinned_worker_harness_image_tag_used_in_dev_sdk(self): '/beam_python%d.%d_sdk:%s' % ( sys.version_info[0], sys.version_info[1], - names.BEAM_FNAPI_CONTAINER_VERSION))) - - # batch, legacy pipeline. - pipeline_options = pipeline_options = PipelineOptions([ - '--temp_location', - 'gs://any-location/temp', - '--experiments=disable_runner_v2_until_v2.50' - ]) - env = apiclient.Environment( - [], #packages - pipeline_options, - '2.0.0', #any environment version - FAKE_PIPELINE_URL) - self.assertEqual( - env.proto.workerPools[0].workerHarnessContainerImage, - ( - names.DATAFLOW_CONTAINER_IMAGE_REPOSITORY + '/python%d%d:%s' % ( - sys.version_info[0], - sys.version_info[1], - names.BEAM_CONTAINER_VERSION))) + names.BEAM_DEV_SDK_CONTAINER_TAG))) @mock.patch( 'apache_beam.runners.dataflow.internal.apiclient.' @@ -706,23 +687,6 @@ def test_worker_harness_image_tag_matches_released_sdk_version(self): '/beam_python%d.%d_sdk:2.2.0' % (sys.version_info[0], sys.version_info[1]))) - # batch, legacy pipeline. - pipeline_options = pipeline_options = PipelineOptions([ - '--temp_location', - 'gs://any-location/temp', - '--experiments=disable_runner_v2_until_v2.50' - ]) - env = apiclient.Environment( - [], #packages - pipeline_options, - '2.0.0', #any environment version - FAKE_PIPELINE_URL) - self.assertEqual( - env.proto.workerPools[0].workerHarnessContainerImage, - ( - names.DATAFLOW_CONTAINER_IMAGE_REPOSITORY + '/python%d%d:2.2.0' % - (sys.version_info[0], sys.version_info[1]))) - @mock.patch( 'apache_beam.runners.dataflow.internal.apiclient.' 'beam_version.__version__', @@ -757,23 +721,6 @@ def test_worker_harness_image_tag_matches_base_sdk_version_of_an_rc(self): '/beam_python%d.%d_sdk:2.2.0' % (sys.version_info[0], sys.version_info[1]))) - # batch, legacy pipeline - pipeline_options = pipeline_options = PipelineOptions([ - '--temp_location', - 'gs://any-location/temp', - '--experiments=disable_runner_v2_until_v2.50' - ]) - env = apiclient.Environment( - [], #packages - pipeline_options, - '2.0.0', #any environment version - FAKE_PIPELINE_URL) - self.assertEqual( - env.proto.workerPools[0].workerHarnessContainerImage, - ( - names.DATAFLOW_CONTAINER_IMAGE_REPOSITORY + '/python%d%d:2.2.0' % - (sys.version_info[0], sys.version_info[1]))) - def test_worker_harness_override_takes_precedence_over_sdk_defaults(self): # streaming, fnapi pipeline. pipeline_options = PipelineOptions([ diff --git a/sdks/python/apache_beam/runners/dataflow/internal/names.py b/sdks/python/apache_beam/runners/dataflow/internal/names.py index 046e86f652b00..579764aeb7c16 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/names.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/names.py @@ -30,91 +30,10 @@ SOURCE_TYPE = 'CustomSourcesType' SERIALIZED_SOURCE_KEY = 'serialized_source' -# In a released SDK, container tags are selected based on the SDK version. -# Unreleased versions use container versions based on values of -# BEAM_CONTAINER_VERSION and BEAM_FNAPI_CONTAINER_VERSION (see below). - -# Update this version to the next version whenever there is a change that will -# require changes to legacy Dataflow worker execution environment. -BEAM_CONTAINER_VERSION = 'beam-master-20230601' -# Update this version to the next version whenever there is a change that +# In a released SDK, Python sdk container image is tagged with the SDK version. +# Unreleased sdks use container image tag specified below. +# Update this tag whenever there is a change that # requires changes to SDK harness container or SDK harness launcher. -BEAM_FNAPI_CONTAINER_VERSION = 'beam-master-20230422' +BEAM_DEV_SDK_CONTAINER_TAG = 'beam-master-20231009' DATAFLOW_CONTAINER_IMAGE_REPOSITORY = 'gcr.io/cloud-dataflow/v1beta3' - - -class TransformNames(object): - """For internal use only; no backwards-compatibility guarantees. - - Transform strings as they are expected in the CloudWorkflow protos. - """ - COLLECTION_TO_SINGLETON = 'CollectionToSingleton' - COMBINE = 'CombineValues' - CREATE_PCOLLECTION = 'CreateCollection' - DO = 'ParallelDo' - FLATTEN = 'Flatten' - GROUP = 'GroupByKey' - READ = 'ParallelRead' - WRITE = 'ParallelWrite' - - -class PropertyNames(object): - """For internal use only; no backwards-compatibility guarantees. - - Property strings as they are expected in the CloudWorkflow protos. - """ - # If uses_keyed_state, whether the state can be sharded. - ALLOWS_SHARDABLE_STATE = 'allows_shardable_state' - BIGQUERY_CREATE_DISPOSITION = 'create_disposition' - BIGQUERY_DATASET = 'dataset' - BIGQUERY_EXPORT_FORMAT = 'bigquery_export_format' - BIGQUERY_FLATTEN_RESULTS = 'bigquery_flatten_results' - BIGQUERY_KMS_KEY = 'bigquery_kms_key' - BIGQUERY_PROJECT = 'project' - BIGQUERY_QUERY = 'bigquery_query' - BIGQUERY_SCHEMA = 'schema' - BIGQUERY_TABLE = 'table' - BIGQUERY_USE_LEGACY_SQL = 'bigquery_use_legacy_sql' - BIGQUERY_WRITE_DISPOSITION = 'write_disposition' - DISPLAY_DATA = 'display_data' - ELEMENT = 'element' - ELEMENTS = 'elements' - ENCODING = 'encoding' - FILE_PATTERN = 'filepattern' - FILE_NAME_PREFIX = 'filename_prefix' - FILE_NAME_SUFFIX = 'filename_suffix' - FORMAT = 'format' - INPUTS = 'inputs' - IMPULSE_ELEMENT = 'impulse_element' - NON_PARALLEL_INPUTS = 'non_parallel_inputs' - NUM_SHARDS = 'num_shards' - OUT = 'out' - OUTPUT = 'output' - OUTPUT_INFO = 'output_info' - OUTPUT_NAME = 'output_name' - PARALLEL_INPUT = 'parallel_input' - PIPELINE_PROTO_TRANSFORM_ID = 'pipeline_proto_transform_id' - # If the input element is a key/value pair, then the output element(s) all - # have the same key as the input. - PRESERVES_KEYS = 'preserves_keys' - PUBSUB_ID_LABEL = 'pubsub_id_label' - PUBSUB_SERIALIZED_ATTRIBUTES_FN = 'pubsub_serialized_attributes_fn' - PUBSUB_SUBSCRIPTION = 'pubsub_subscription' - PUBSUB_TIMESTAMP_ATTRIBUTE = 'pubsub_timestamp_label' - PUBSUB_TOPIC = 'pubsub_topic' - RESOURCE_HINTS = 'resource_hints' - RESTRICTION_ENCODING = 'restriction_encoding' - SERIALIZED_FN = 'serialized_fn' - SHARD_NAME_TEMPLATE = 'shard_template' - SOURCE_STEP_INPUT = 'custom_source_step_input' - SERIALIZED_TEST_STREAM = 'serialized_test_stream' - STEP_NAME = 'step_name' - USE_INDEXED_FORMAT = 'use_indexed_format' - USER_FN = 'user_fn' - USER_NAME = 'user_name' - USES_KEYED_STATE = 'uses_keyed_state' - VALIDATE_SINK = 'validate_sink' - VALIDATE_SOURCE = 'validate_source' - VALUE = 'value' - WINDOWING_STRATEGY = 'windowing_strategy' diff --git a/sdks/python/apache_beam/runners/dataflow/native_io/iobase.py b/sdks/python/apache_beam/runners/dataflow/native_io/iobase.py deleted file mode 100644 index 3d1afe5469018..0000000000000 --- a/sdks/python/apache_beam/runners/dataflow/native_io/iobase.py +++ /dev/null @@ -1,342 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""Dataflow native sources and sinks. - -For internal use only; no backwards-compatibility guarantees. -""" - -# pytype: skip-file - -import logging -from typing import TYPE_CHECKING -from typing import Optional - -from apache_beam import pvalue -from apache_beam.io import iobase -from apache_beam.transforms import ptransform -from apache_beam.transforms.display import HasDisplayData - -if TYPE_CHECKING: - from apache_beam import coders - -_LOGGER = logging.getLogger(__name__) - - -def _dict_printable_fields(dict_object, skip_fields): - """Returns a list of strings for the interesting fields of a dict.""" - return [ - '%s=%r' % (name, value) for name, - value in dict_object.items() - # want to output value 0 but not None nor [] - if (value or value == 0) and name not in skip_fields - ] - - -_minor_fields = [ - 'coder', - 'key_coder', - 'value_coder', - 'config_bytes', - 'elements', - 'append_trailing_newlines', - 'strip_trailing_newlines', - 'compression_type' -] - - -class NativeSource(iobase.SourceBase): - """A source implemented by Dataflow service. - - This class is to be only inherited by sources natively implemented by Cloud - Dataflow service, hence should not be sub-classed by users. - - This class is deprecated and should not be used to define new sources. - """ - coder = None # type: Optional[coders.Coder] - - def reader(self): - """Returns a NativeSourceReader instance associated with this source.""" - raise NotImplementedError - - def is_bounded(self): - return True - - def __repr__(self): - return '<{name} {vals}>'.format( - name=self.__class__.__name__, - vals=', '.join(_dict_printable_fields(self.__dict__, _minor_fields))) - - -class NativeSourceReader(object): - """A reader for a source implemented by Dataflow service.""" - def __enter__(self): - """Opens everything necessary for a reader to function properly.""" - raise NotImplementedError - - def __exit__(self, exception_type, exception_value, traceback): - """Cleans up after a reader executed.""" - raise NotImplementedError - - def __iter__(self): - """Returns an iterator over all the records of the source.""" - raise NotImplementedError - - @property - def returns_windowed_values(self): - """Returns whether this reader returns windowed values.""" - return False - - def get_progress(self): - """Returns a representation of how far the reader has read. - - Returns: - A SourceReaderProgress object that gives the current progress of the - reader. - """ - - def request_dynamic_split(self, dynamic_split_request): - """Attempts to split the input in two parts. - - The two parts are named the "primary" part and the "residual" part. The - current 'NativeSourceReader' keeps processing the primary part, while the - residual part will be processed elsewhere (e.g. perhaps on a different - worker). - - The primary and residual parts, if concatenated, must represent the - same input as the current input of this 'NativeSourceReader' before this - call. - - The boundary between the primary part and the residual part is - specified in a framework-specific way using 'DynamicSplitRequest' e.g., - if the framework supports the notion of positions, it might be a - position at which the input is asked to split itself (which is not - necessarily the same position at which it *will* split itself); it - might be an approximate fraction of input, or something else. - - This function returns a 'DynamicSplitResult', which encodes, in a - framework-specific way, the information sufficient to construct a - description of the resulting primary and residual inputs. For example, it - might, again, be a position demarcating these parts, or it might be a pair - of fully-specified input descriptions, or something else. - - After a successful call to 'request_dynamic_split()', subsequent calls - should be interpreted relative to the new primary. - - Args: - dynamic_split_request: A 'DynamicSplitRequest' describing the split - request. - - Returns: - 'None' if the 'DynamicSplitRequest' cannot be honored (in that - case the input represented by this 'NativeSourceReader' stays the same), - or a 'DynamicSplitResult' describing how the input was split into a - primary and residual part. - """ - _LOGGER.debug( - 'SourceReader %r does not support dynamic splitting. Ignoring dynamic ' - 'split request: %r', - self, - dynamic_split_request) - - -class ReaderProgress(object): - """A representation of how far a NativeSourceReader has read.""" - def __init__( - self, - position=None, - percent_complete=None, - remaining_time=None, - consumed_split_points=None, - remaining_split_points=None): - - self._position = position - - if percent_complete is not None: - percent_complete = float(percent_complete) - if percent_complete < 0 or percent_complete > 1: - raise ValueError( - 'The percent_complete argument was %f. Must be in range [0, 1].' % - percent_complete) - self._percent_complete = percent_complete - - self._remaining_time = remaining_time - self._consumed_split_points = consumed_split_points - self._remaining_split_points = remaining_split_points - - @property - def position(self): - """Returns progress, represented as a ReaderPosition object.""" - return self._position - - @property - def percent_complete(self): - """Returns progress, represented as a percentage of total work. - - Progress range from 0.0 (beginning, nothing complete) to 1.0 (end of the - work range, entire WorkItem complete). - - Returns: - Progress represented as a percentage of total work. - """ - return self._percent_complete - - @property - def remaining_time(self): - """Returns progress, represented as an estimated time remaining.""" - return self._remaining_time - - @property - def consumed_split_points(self): - return self._consumed_split_points - - @property - def remaining_split_points(self): - return self._remaining_split_points - - -class ReaderPosition(object): - """A representation of position in an iteration of a 'NativeSourceReader'.""" - def __init__( - self, - end=None, - key=None, - byte_offset=None, - record_index=None, - shuffle_position=None, - concat_position=None): - """Initializes ReaderPosition. - - A ReaderPosition may get instantiated for one of these position types. Only - one of these should be specified. - - Args: - end: position is past all other positions. For example, this may be used - to represent the end position of an unbounded range. - key: position is a string key. - byte_offset: position is a byte offset. - record_index: position is a record index - shuffle_position: position is a base64 encoded shuffle position. - concat_position: position is a 'ConcatPosition'. - """ - - self.end = end - self.key = key - self.byte_offset = byte_offset - self.record_index = record_index - self.shuffle_position = shuffle_position - - if concat_position is not None: - assert isinstance(concat_position, ConcatPosition) - self.concat_position = concat_position - - -class ConcatPosition(object): - """A position that encapsulate an inner position and an index. - - This is used to represent the position of a source that encapsulate several - other sources. - """ - def __init__(self, index, position): - """Initializes ConcatPosition. - - Args: - index: index of the source currently being read. - position: inner position within the source currently being read. - """ - - if position is not None: - assert isinstance(position, ReaderPosition) - self.index = index - self.position = position - - -class DynamicSplitRequest(object): - """Specifies how 'NativeSourceReader.request_dynamic_split' should split. - """ - def __init__(self, progress): - assert isinstance(progress, ReaderProgress) - self.progress = progress - - -class DynamicSplitResult(object): - pass - - -class DynamicSplitResultWithPosition(DynamicSplitResult): - def __init__(self, stop_position): - assert isinstance(stop_position, ReaderPosition) - self.stop_position = stop_position - - -class NativeSink(HasDisplayData): - """A sink implemented by Dataflow service. - - This class is to be only inherited by sinks natively implemented by Cloud - Dataflow service, hence should not be sub-classed by users. - """ - def writer(self): - """Returns a SinkWriter for this source.""" - raise NotImplementedError - - def __repr__(self): - return '<{name} {vals}>'.format( - name=self.__class__.__name__, - vals=_dict_printable_fields(self.__dict__, _minor_fields)) - - -class NativeSinkWriter(object): - """A writer for a sink implemented by Dataflow service.""" - def __enter__(self): - """Opens everything necessary for a writer to function properly.""" - raise NotImplementedError - - def __exit__(self, exception_type, exception_value, traceback): - """Cleans up after a writer executed.""" - raise NotImplementedError - - @property - def takes_windowed_values(self): - """Returns whether this writer takes windowed values.""" - return False - - def Write(self, o): # pylint: disable=invalid-name - """Writes a record to the sink associated with this writer.""" - raise NotImplementedError - - -class _NativeWrite(ptransform.PTransform): - """A PTransform for writing to a Dataflow native sink. - - These are sinks that are implemented natively by the Dataflow service - and hence should not be updated by users. These sinks are processed - using a Dataflow native write transform. - - Applying this transform results in a ``pvalue.PDone``. - """ - def __init__(self, sink): - """Initializes a Write transform. - - Args: - sink: Sink to use for the write - """ - super().__init__() - self.sink = sink - - def expand(self, pcoll): - self._check_pcollection(pcoll) - return pvalue.PDone(pcoll.pipeline) diff --git a/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py b/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py deleted file mode 100644 index 5e72ca555b69e..0000000000000 --- a/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py +++ /dev/null @@ -1,203 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""Tests corresponding to Dataflow's iobase module.""" - -# pytype: skip-file - -import unittest - -from apache_beam import Create -from apache_beam import error -from apache_beam import pvalue -from apache_beam.runners.dataflow.native_io.iobase import ConcatPosition -from apache_beam.runners.dataflow.native_io.iobase import DynamicSplitRequest -from apache_beam.runners.dataflow.native_io.iobase import DynamicSplitResultWithPosition -from apache_beam.runners.dataflow.native_io.iobase import NativeSink -from apache_beam.runners.dataflow.native_io.iobase import NativeSinkWriter -from apache_beam.runners.dataflow.native_io.iobase import NativeSource -from apache_beam.runners.dataflow.native_io.iobase import ReaderPosition -from apache_beam.runners.dataflow.native_io.iobase import ReaderProgress -from apache_beam.runners.dataflow.native_io.iobase import _dict_printable_fields -from apache_beam.runners.dataflow.native_io.iobase import _NativeWrite -from apache_beam.testing.test_pipeline import TestPipeline - - -class TestHelperFunctions(unittest.TestCase): - def test_dict_printable_fields(self): - dict_object = { - 'key_alpha': '1', - 'key_beta': None, - 'key_charlie': [], - 'key_delta': 2.0, - 'key_echo': 'skip_me', - 'key_fox': 0 - } - skip_fields = [ - 'key_echo', - ] - self.assertEqual( - sorted(_dict_printable_fields(dict_object, skip_fields)), - ["key_alpha='1'", 'key_delta=2.0', 'key_fox=0']) - - -class TestNativeSource(unittest.TestCase): - def test_reader_method(self): - native_source = NativeSource() - self.assertRaises(NotImplementedError, native_source.reader) - - def test_repr_method(self): - class FakeSource(NativeSource): - """A fake source modeled after BigQuerySource, which inherits from - NativeSource.""" - def __init__( - self, - table=None, - dataset=None, - project=None, - query=None, - validate=False, - coder=None, - use_std_sql=False, - flatten_results=True): - self.validate = validate - - fake_source = FakeSource() - self.assertEqual(fake_source.__repr__(), '') - - -class TestReaderProgress(unittest.TestCase): - def test_out_of_bounds_percent_complete(self): - with self.assertRaises(ValueError): - ReaderProgress(percent_complete=-0.1) - with self.assertRaises(ValueError): - ReaderProgress(percent_complete=1.1) - - def test_position_property(self): - reader_progress = ReaderProgress(position=ReaderPosition()) - self.assertEqual(type(reader_progress.position), ReaderPosition) - - def test_percent_complete_property(self): - reader_progress = ReaderProgress(percent_complete=0.5) - self.assertEqual(reader_progress.percent_complete, 0.5) - - -class TestReaderPosition(unittest.TestCase): - def test_invalid_concat_position_type(self): - with self.assertRaises(AssertionError): - ReaderPosition(concat_position=1) - - def test_valid_concat_position_type(self): - ReaderPosition(concat_position=ConcatPosition(None, None)) - - -class TestConcatPosition(unittest.TestCase): - def test_invalid_position_type(self): - with self.assertRaises(AssertionError): - ConcatPosition(None, position=1) - - def test_valid_position_type(self): - ConcatPosition(None, position=ReaderPosition()) - - -class TestDynamicSplitRequest(unittest.TestCase): - def test_invalid_progress_type(self): - with self.assertRaises(AssertionError): - DynamicSplitRequest(progress=1) - - def test_valid_progress_type(self): - DynamicSplitRequest(progress=ReaderProgress()) - - -class TestDynamicSplitResultWithPosition(unittest.TestCase): - def test_invalid_stop_position_type(self): - with self.assertRaises(AssertionError): - DynamicSplitResultWithPosition(stop_position=1) - - def test_valid_stop_position_type(self): - DynamicSplitResultWithPosition(stop_position=ReaderPosition()) - - -class TestNativeSink(unittest.TestCase): - def test_writer_method(self): - native_sink = NativeSink() - self.assertRaises(NotImplementedError, native_sink.writer) - - def test_repr_method(self): - class FakeSink(NativeSink): - """A fake sink modeled after BigQuerySink, which inherits from - NativeSink.""" - def __init__( - self, - validate=False, - dataset=None, - project=None, - schema=None, - create_disposition='create', - write_disposition=None, - coder=None): - self.validate = validate - - fake_sink = FakeSink() - self.assertEqual(fake_sink.__repr__(), "") - - def test_on_direct_runner(self): - class FakeSink(NativeSink): - """A fake sink outputing a number of elements.""" - def __init__(self): - self.written_values = [] - self.writer_instance = FakeSinkWriter(self.written_values) - - def writer(self): - return self.writer_instance - - class FakeSinkWriter(NativeSinkWriter): - """A fake sink writer for testing.""" - def __init__(self, written_values): - self.written_values = written_values - - def __enter__(self): - return self - - def __exit__(self, *unused_args): - pass - - def Write(self, value): - self.written_values.append(value) - - with TestPipeline() as p: - sink = FakeSink() - p | Create(['a', 'b', 'c']) | _NativeWrite(sink) # pylint: disable=expression-not-assigned - - self.assertEqual(['a', 'b', 'c'], sorted(sink.written_values)) - - -class Test_NativeWrite(unittest.TestCase): - def setUp(self): - self.native_sink = NativeSink() - self.native_write = _NativeWrite(self.native_sink) - - def test_expand_method_pcollection_errors(self): - with self.assertRaises(error.TransformError): - self.native_write.expand(None) - with self.assertRaises(error.TransformError): - pcoll = pvalue.PCollection(pipeline=None) - self.native_write.expand(pcoll) - - -if __name__ == '__main__': - unittest.main() diff --git a/sdks/python/apache_beam/runners/dataflow/ptransform_overrides.py b/sdks/python/apache_beam/runners/dataflow/ptransform_overrides.py index 1012a7d362405..8004762f5eece 100644 --- a/sdks/python/apache_beam/runners/dataflow/ptransform_overrides.py +++ b/sdks/python/apache_beam/runners/dataflow/ptransform_overrides.py @@ -19,101 +19,9 @@ # pytype: skip-file -from apache_beam.options.pipeline_options import StandardOptions from apache_beam.pipeline import PTransformOverride -class CreatePTransformOverride(PTransformOverride): - """A ``PTransformOverride`` for ``Create`` in streaming mode.""" - def matches(self, applied_ptransform): - # Imported here to avoid circular dependencies. - # pylint: disable=wrong-import-order, wrong-import-position - from apache_beam import Create - return isinstance(applied_ptransform.transform, Create) - - def get_replacement_transform_for_applied_ptransform( - self, applied_ptransform): - # Imported here to avoid circular dependencies. - # pylint: disable=wrong-import-order, wrong-import-position - from apache_beam import PTransform - - ptransform = applied_ptransform.transform - - # Return a wrapper rather than ptransform.as_read() directly to - # ensure backwards compatibility of the pipeline structure. - class LegacyCreate(PTransform): - def expand(self, pbegin): - return pbegin | ptransform.as_read() - - return LegacyCreate().with_output_types(ptransform.get_output_type()) - - -class ReadPTransformOverride(PTransformOverride): - """A ``PTransformOverride`` for ``Read(BoundedSource)``""" - def matches(self, applied_ptransform): - from apache_beam.io import Read - from apache_beam.io.iobase import BoundedSource - # Only overrides Read(BoundedSource) transform - if (isinstance(applied_ptransform.transform, Read) and - not getattr(applied_ptransform.transform, 'override', False)): - if isinstance(applied_ptransform.transform.source, BoundedSource): - return True - return False - - def get_replacement_transform_for_applied_ptransform( - self, applied_ptransform): - - from apache_beam import pvalue - from apache_beam.io import iobase - - transform = applied_ptransform.transform - - class Read(iobase.Read): - override = True - - def expand(self, pbegin): - return pvalue.PCollection( - self.pipeline, is_bounded=self.source.is_bounded()) - - return Read(transform.source).with_output_types( - transform.get_type_hints().simple_output_type('Read')) - - -class CombineValuesPTransformOverride(PTransformOverride): - """A ``PTransformOverride`` for ``CombineValues``. - - The DataflowRunner expects that the CombineValues PTransform acts as a - primitive. So this override replaces the CombineValues with a primitive. - """ - def matches(self, applied_ptransform): - # Imported here to avoid circular dependencies. - # pylint: disable=wrong-import-order, wrong-import-position - from apache_beam import CombineValues - - if isinstance(applied_ptransform.transform, CombineValues): - self.transform = applied_ptransform.transform - return True - return False - - def get_replacement_transform(self, ptransform): - # Imported here to avoid circular dependencies. - # pylint: disable=wrong-import-order, wrong-import-position - from apache_beam import PTransform - from apache_beam.pvalue import PCollection - - # The DataflowRunner still needs access to the CombineValues members to - # generate a V1B3 proto representation, so we remember the transform from - # the matches method and forward it here. - class CombineValuesReplacement(PTransform): - def __init__(self, transform): - self.transform = transform - - def expand(self, pcoll): - return PCollection.from_(pcoll) - - return CombineValuesReplacement(self.transform) - - class NativeReadPTransformOverride(PTransformOverride): """A ``PTransformOverride`` for ``Read`` using native sources. @@ -150,37 +58,3 @@ def expand(self, pbegin): # will choose the incorrect coder for this transform. return Read(ptransform.source).with_output_types( ptransform.source.coder.to_type_hint()) - - -class GroupIntoBatchesWithShardedKeyPTransformOverride(PTransformOverride): - """A ``PTransformOverride`` for ``GroupIntoBatches.WithShardedKey``. - - This override simply returns the original transform but additionally records - the output PCollection in order to append required step properties during - graph translation. - """ - def __init__(self, dataflow_runner, options): - self.dataflow_runner = dataflow_runner - self.options = options - - def matches(self, applied_ptransform): - # Imported here to avoid circular dependencies. - # pylint: disable=wrong-import-order, wrong-import-position - from apache_beam import util - - transform = applied_ptransform.transform - - if not isinstance(transform, util.GroupIntoBatches.WithShardedKey): - return False - - # The replacement is only valid for portable Streaming Engine jobs with - # runner v2. - standard_options = self.options.view_as(StandardOptions) - if not standard_options.streaming: - return False - - self.dataflow_runner.add_pcoll_with_auto_sharding(applied_ptransform) - return True - - def get_replacement_transform_for_applied_ptransform(self, ptransform): - return ptransform.transform diff --git a/sdks/python/apache_beam/runners/dataflow/template_runner_test.py b/sdks/python/apache_beam/runners/dataflow/template_runner_test.py index 021056608f1fa..792c5cfd16553 100644 --- a/sdks/python/apache_beam/runners/dataflow/template_runner_test.py +++ b/sdks/python/apache_beam/runners/dataflow/template_runner_test.py @@ -51,17 +51,16 @@ def test_full_completion(self): dummy_dir = tempfile.mkdtemp() remote_runner = DataflowRunner() - with Pipeline(remote_runner, - options=PipelineOptions(['--dataflow_endpoint=ignored', - '--sdk_location=' + dummy_file_name, - '--job_name=test-job', - '--project=test-project', - '--staging_location=' + dummy_dir, - '--temp_location=/dev/null', - '--template_location=' + - dummy_file_name, - '--no_auth'])) as pipeline: - + options = PipelineOptions([ + '--sdk_location=' + dummy_file_name, + '--job_name=test-job', + '--project=apache-beam-testing', + '--region=us-central1', + '--staging_location=gs://apache-beam-testing-stg/stg/', + '--temp_location=gs://apache-beam-testing-temp/tmp', + '--template_location=' + dummy_file_name + ]) + with Pipeline(remote_runner, options) as pipeline: pipeline | beam.Create([1, 2, 3]) | beam.Map(lambda x: x) # pylint: disable=expression-not-assigned with open(dummy_file_name) as template_file: @@ -69,7 +68,7 @@ def test_full_completion(self): self.assertEqual( saved_job_dict['environment']['sdkPipelineOptions']['options'] ['project'], - 'test-project') + 'apache-beam-testing') self.assertEqual( saved_job_dict['environment']['sdkPipelineOptions']['options'] ['job_name'], diff --git a/sdks/python/apache_beam/runners/direct/direct_runner.py b/sdks/python/apache_beam/runners/direct/direct_runner.py index 6466bc7752b57..db53e4122bbc0 100644 --- a/sdks/python/apache_beam/runners/direct/direct_runner.py +++ b/sdks/python/apache_beam/runners/direct/direct_runner.py @@ -72,9 +72,9 @@ def is_fnapi_compatible(self): def run_pipeline(self, pipeline, options): from apache_beam.pipeline import PipelineVisitor - from apache_beam.runners.dataflow.native_io.iobase import NativeSource - from apache_beam.runners.dataflow.native_io.iobase import _NativeWrite from apache_beam.testing.test_stream import TestStream + from apache_beam.io.gcp.pubsub import ReadFromPubSub + from apache_beam.io.gcp.pubsub import WriteToPubSub class _FnApiRunnerSupportVisitor(PipelineVisitor): """Visitor determining if a Pipeline can be run on the FnApiRunner.""" @@ -83,18 +83,17 @@ def accept(self, pipeline): pipeline.visit(self) return self.supported_by_fnapi_runner + def enter_composite_transform(self, applied_ptransform): + # The FnApiRunner does not support streaming execution. + if isinstance(applied_ptransform.transform, + (ReadFromPubSub, WriteToPubSub)): + self.supported_by_fnapi_runner = False + def visit_transform(self, applied_ptransform): transform = applied_ptransform.transform # The FnApiRunner does not support streaming execution. if isinstance(transform, TestStream): self.supported_by_fnapi_runner = False - # The FnApiRunner does not support reads from NativeSources. - if (isinstance(transform, beam.io.Read) and - isinstance(transform.source, NativeSource)): - self.supported_by_fnapi_runner = False - # The FnApiRunner does not support the use of _NativeWrites. - if isinstance(transform, _NativeWrite): - self.supported_by_fnapi_runner = False if isinstance(transform, beam.ParDo): dofn = transform.dofn # The FnApiRunner does not support execution of CombineFns with diff --git a/sdks/python/apache_beam/runners/direct/transform_evaluator.py b/sdks/python/apache_beam/runners/direct/transform_evaluator.py index bfb27c4adc004..37004c7258a73 100644 --- a/sdks/python/apache_beam/runners/direct/transform_evaluator.py +++ b/sdks/python/apache_beam/runners/direct/transform_evaluator.py @@ -39,7 +39,6 @@ from apache_beam.runners import common from apache_beam.runners.common import DoFnRunner from apache_beam.runners.common import DoFnState -from apache_beam.runners.dataflow.native_io.iobase import _NativeWrite # pylint: disable=protected-access from apache_beam.runners.direct.direct_runner import _DirectReadFromPubSub from apache_beam.runners.direct.direct_runner import _GroupByKeyOnly from apache_beam.runners.direct.direct_runner import _StreamingGroupAlsoByWindow @@ -106,7 +105,6 @@ def __init__(self, evaluation_context): _GroupByKeyOnly: _GroupByKeyOnlyEvaluator, _StreamingGroupByKeyOnly: _StreamingGroupByKeyOnlyEvaluator, _StreamingGroupAlsoByWindow: _StreamingGroupAlsoByWindowEvaluator, - _NativeWrite: _NativeWriteEvaluator, _TestStream: _TestStreamEvaluator, ProcessElements: _ProcessElementsEvaluator, _WatermarkController: _WatermarkControllerEvaluator, @@ -172,11 +170,10 @@ def should_execute_serially(self, applied_ptransform): Returns: True if executor should execute applied_ptransform serially. """ - if isinstance(applied_ptransform.transform, - (_GroupByKeyOnly, - _StreamingGroupByKeyOnly, - _StreamingGroupAlsoByWindow, - _NativeWrite)): + if isinstance( + applied_ptransform.transform, + (_GroupByKeyOnly, _StreamingGroupByKeyOnly, + _StreamingGroupAlsoByWindow)): return True elif (isinstance(applied_ptransform.transform, core.ParDo) and is_stateful_dofn(applied_ptransform.transform.dofn)): @@ -1125,77 +1122,6 @@ def finish_bundle(self): return TransformResult(self, bundles, [], None, self.keyed_holds) -class _NativeWriteEvaluator(_TransformEvaluator): - """TransformEvaluator for _NativeWrite transform.""" - - ELEMENTS_TAG = _ListStateTag('elements') - - def __init__( - self, - evaluation_context, - applied_ptransform, - input_committed_bundle, - side_inputs): - assert not side_inputs - super().__init__( - evaluation_context, - applied_ptransform, - input_committed_bundle, - side_inputs) - - assert applied_ptransform.transform.sink - self._sink = applied_ptransform.transform.sink - - @property - def _is_final_bundle(self): - return ( - self._execution_context.watermarks.input_watermark == - WatermarkManager.WATERMARK_POS_INF) - - @property - def _has_already_produced_output(self): - return ( - self._execution_context.watermarks.output_watermark == - WatermarkManager.WATERMARK_POS_INF) - - def start_bundle(self): - self.global_state = self._step_context.get_keyed_state(None) - - def process_timer(self, timer_firing): - # We do not need to emit a KeyedWorkItem to process_element(). - pass - - def process_element(self, element): - self.global_state.add_state( - None, _NativeWriteEvaluator.ELEMENTS_TAG, element) - - def finish_bundle(self): - # finish_bundle will append incoming bundles in memory until all the bundles - # carrying data is processed. This is done to produce only a single output - # shard (some tests depends on this behavior). It is possible to have - # incoming empty bundles after the output is produced, these bundles will be - # ignored and would not generate additional output files. - # TODO(altay): Do not wait until the last bundle to write in a single shard. - if self._is_final_bundle: - elements = self.global_state.get_state( - None, _NativeWriteEvaluator.ELEMENTS_TAG) - if self._has_already_produced_output: - # Ignore empty bundles that arrive after the output is produced. - assert elements == [] - else: - self._sink.pipeline_options = self._evaluation_context.pipeline_options - with self._sink.writer() as writer: - for v in elements: - writer.Write(v.value) - hold = WatermarkManager.WATERMARK_POS_INF - else: - hold = WatermarkManager.WATERMARK_NEG_INF - self.global_state.set_timer( - None, '', TimeDomain.WATERMARK, WatermarkManager.WATERMARK_POS_INF) - - return TransformResult(self, [], [], None, {None: hold}) - - class _ProcessElementsEvaluator(_TransformEvaluator): """An evaluator for sdf_direct_runner.ProcessElements transform.""" diff --git a/sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/yarn.lock b/sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/yarn.lock index a6c9cc762ace2..f72e23ebd403e 100644 --- a/sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/yarn.lock +++ b/sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/yarn.lock @@ -6767,10 +6767,10 @@ mv@2.1.1: ncp "~2.0.0" rimraf "~2.4.0" -nanoid@^3.1.28: - version "3.3.2" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.2.tgz#c89622fafb4381cd221421c69ec58547a1eec557" - integrity sha512-CuHBogktKwpm5g2sRgv83jEy2ijFzBwMoYA60orPDR7ynsLijJDqgsi4RDGj3OJpy3Ieb+LYwiRmIOGyytgITA== +nanoid@^3.3.6: + version "3.3.6" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.6.tgz#443380c856d6e9f9824267d960b4236ad583ea4c" + integrity sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA== nanomatch@^1.2.9: version "1.2.13" @@ -7279,6 +7279,11 @@ picocolors@^0.2.1: resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-0.2.1.tgz#570670f793646851d1ba135996962abad587859f" integrity sha512-cMlDqaLEqfSaW8Z7N5Jw+lyIW869EzT73/F5lhtY9cLGoVxSXznfgfXMO0Z5K0o0Q2TkTXq+0KFsdnSe3jDViA== +picocolors@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" + integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== + picomatch@^2.0.4, picomatch@^2.0.5: version "2.2.2" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad" @@ -7392,13 +7397,13 @@ postcss-value-parser@^4.1.0: integrity sha512-97DXOFbQJhk71ne5/Mt6cOu6yxsSfM0QGQyl0L25Gca4yGWEGJaig7l7gbCX623VqTBNGLRLaVUCnNkcedlRSQ== postcss@^8.0.2, postcss@^8.2.15: - version "8.3.9" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.3.9.tgz#98754caa06c4ee9eb59cc48bd073bb6bd3437c31" - integrity sha512-f/ZFyAKh9Dnqytx5X62jgjhhzttjZS7hMsohcI7HEI5tjELX/HxCy3EFhsRxyzGvrzFF+82XPvCS8T9TFleVJw== + version "8.4.31" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.31.tgz#92b451050a9f914da6755af352bdc0192508656d" + integrity sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ== dependencies: - nanoid "^3.1.28" - picocolors "^0.2.1" - source-map-js "^0.6.2" + nanoid "^3.3.6" + picocolors "^1.0.0" + source-map-js "^1.0.2" prelude-ls@^1.2.1: version "1.2.1" @@ -8108,26 +8113,28 @@ schema-utils@^3.0.0, schema-utils@^3.1.0, schema-utils@^3.1.1: ajv-keywords "^3.5.2" "semver@2 || 3 || 4 || 5", semver@^5.4.1, semver@^5.5.0, semver@^5.6.0: - version "5.7.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" - integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== + version "5.7.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" + integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== -semver@7.3.5, semver@^7.2.1, semver@^7.3.5: +semver@7.3.5: version "7.3.5" resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7" integrity sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ== dependencies: lru-cache "^6.0.0" -semver@7.x, semver@^7.3.2: - version "7.3.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.2.tgz#604962b052b81ed0786aae84389ffba70ffd3938" - integrity sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ== +semver@7.x, semver@^7.2.1, semver@^7.3.2, semver@^7.3.5: + version "7.5.4" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.4.tgz#483986ec4ed38e1c6c48c34894a9182dbff68a6e" + integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== + dependencies: + lru-cache "^6.0.0" semver@^6.0.0, semver@^6.2.0, semver@^6.3.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" - integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== + version "6.3.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== send@0.17.1: version "0.17.1" @@ -8349,10 +8356,10 @@ source-list-map@^2.0.0: resolved "https://registry.yarnpkg.com/source-list-map/-/source-list-map-2.0.1.tgz#3993bd873bfc48479cca9ea3a547835c7c154b34" integrity sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw== -source-map-js@^0.6.2: - version "0.6.2" - resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-0.6.2.tgz#0bb5de631b41cfbda6cfba8bd05a80efdfd2385e" - integrity sha512-/3GptzWzu0+0MBQFrDKzw/DvvMTUORvgY6k6jd/VS6iCR4RDTKWH6v6WPwQoUO8667uQEf9Oe38DxAYWY5F/Ug== +source-map-js@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c" + integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw== source-map-resolve@^0.5.0: version "0.5.3" @@ -9461,9 +9468,9 @@ wildcard@^2.0.0: integrity sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw== word-wrap@^1.2.3, word-wrap@~1.2.3: - version "1.2.3" - resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" - integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== + version "1.2.4" + resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.4.tgz#cb4b50ec9aca570abd1f52f33cd45b6c61739a9f" + integrity sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA== wordwrap@^1.0.0: version "1.0.0" diff --git a/sdks/python/apache_beam/runners/interactive/interactive_beam.py b/sdks/python/apache_beam/runners/interactive/interactive_beam.py index 878552ef345f5..207b0f4e6451c 100644 --- a/sdks/python/apache_beam/runners/interactive/interactive_beam.py +++ b/sdks/python/apache_beam/runners/interactive/interactive_beam.py @@ -35,7 +35,6 @@ # pytype: skip-file import logging -import warnings from datetime import timedelta from typing import Dict from typing import List @@ -496,11 +495,6 @@ def cleanup( dcm = self.pipelines.pop(p, None) if dcm: dcm.pipelines.remove(p) - warnings.filterwarnings( - 'ignore', - 'options is deprecated since First stable release. References to ' - '.options will not be supported', - category=DeprecationWarning) p_flink_options = p.options.view_as(FlinkRunnerOptions) p_flink_options.flink_master = '[auto]' p_flink_options.flink_version = None diff --git a/sdks/python/apache_beam/runners/interactive/interactive_environment.py b/sdks/python/apache_beam/runners/interactive/interactive_environment.py index e508ce351ea34..0e3d0060b1a4d 100644 --- a/sdks/python/apache_beam/runners/interactive/interactive_environment.py +++ b/sdks/python/apache_beam/runners/interactive/interactive_environment.py @@ -29,7 +29,6 @@ import logging import os import tempfile -import warnings from collections.abc import Iterable from pathlib import PurePath @@ -374,11 +373,6 @@ def get_cache_manager(self, pipeline, create_if_absent=False): given pipeline. If the pipeline is absent from the environment while create_if_absent is True, creates and returns a new file based cache manager for the pipeline.""" - warnings.filterwarnings( - 'ignore', - 'options is deprecated since First stable release. References to ' - '.options will not be supported', - category=DeprecationWarning) cache_manager = self._cache_managers.get(str(id(pipeline)), None) pipeline_runner = detect_pipeline_runner(pipeline) diff --git a/sdks/python/apache_beam/runners/interactive/interactive_runner_test.py b/sdks/python/apache_beam/runners/interactive/interactive_runner_test.py index 5ffa6224edb0e..1da20fb2dfa94 100644 --- a/sdks/python/apache_beam/runners/interactive/interactive_runner_test.py +++ b/sdks/python/apache_beam/runners/interactive/interactive_runner_test.py @@ -321,7 +321,7 @@ def test_dataframes_with_grouped_index(self): Record('c', 18, 150) ] - aggregate = lambda df: df.groupby('height').mean() + aggregate = lambda df: df.groupby('height').mean(numeric_only=True) deferred_df = aggregate(to_dataframe(p | beam.Create(data))) df_expected = aggregate(pd.DataFrame(data)) diff --git a/sdks/python/apache_beam/runners/interactive/recording_manager.py b/sdks/python/apache_beam/runners/interactive/recording_manager.py index d123736233fe3..bee215717b4d1 100644 --- a/sdks/python/apache_beam/runners/interactive/recording_manager.py +++ b/sdks/python/apache_beam/runners/interactive/recording_manager.py @@ -448,11 +448,6 @@ def record(self, pcolls, max_n, max_duration): # incomplete. self._clear() - warnings.filterwarnings( - 'ignore', - 'options is deprecated since First stable release. References to ' - '.options will not be supported', - category=DeprecationWarning) cache_path = ie.current_env().options.cache_root is_remote_run = cache_path and ie.current_env( ).options.cache_root.startswith('gs://') diff --git a/sdks/python/apache_beam/runners/interactive/testing/integration/goldens/Linux/29c9237ddf4f3d5988a503069b4d3c47.png b/sdks/python/apache_beam/runners/interactive/testing/integration/goldens/Linux/29c9237ddf4f3d5988a503069b4d3c47.png index c748ea1a2d0a4..382063f75092d 100644 Binary files a/sdks/python/apache_beam/runners/interactive/testing/integration/goldens/Linux/29c9237ddf4f3d5988a503069b4d3c47.png and b/sdks/python/apache_beam/runners/interactive/testing/integration/goldens/Linux/29c9237ddf4f3d5988a503069b4d3c47.png differ diff --git a/sdks/python/apache_beam/runners/interactive/testing/integration/goldens/Linux/7a35f487b2a5f3a9b9852a8659eeb4bd.png b/sdks/python/apache_beam/runners/interactive/testing/integration/goldens/Linux/7a35f487b2a5f3a9b9852a8659eeb4bd.png index b21d3b606a37a..f3bf660dba0f5 100644 Binary files a/sdks/python/apache_beam/runners/interactive/testing/integration/goldens/Linux/7a35f487b2a5f3a9b9852a8659eeb4bd.png and b/sdks/python/apache_beam/runners/interactive/testing/integration/goldens/Linux/7a35f487b2a5f3a9b9852a8659eeb4bd.png differ diff --git a/sdks/python/apache_beam/runners/portability/expansion_service.py b/sdks/python/apache_beam/runners/portability/expansion_service.py index 39697cbca20e7..9670ac1ad7be0 100644 --- a/sdks/python/apache_beam/runners/portability/expansion_service.py +++ b/sdks/python/apache_beam/runners/portability/expansion_service.py @@ -26,7 +26,7 @@ from apache_beam.portability.api import beam_expansion_api_pb2 from apache_beam.portability.api import beam_expansion_api_pb2_grpc from apache_beam.runners import pipeline_context -from apache_beam.runners.portability import portable_runner +from apache_beam.transforms import environments from apache_beam.transforms import external from apache_beam.transforms import ptransform @@ -37,7 +37,7 @@ def __init__(self, options=None): self._options = options or beam_pipeline.PipelineOptions( environment_type=python_urns.EMBEDDED_PYTHON, sdk_location='container') self._default_environment = ( - portable_runner.PortableRunner._create_environment(self._options)) + environments.Environment.from_options(self._options)) def Expand(self, request, context=None): try: diff --git a/sdks/python/apache_beam/runners/portability/flink_runner.py b/sdks/python/apache_beam/runners/portability/flink_runner.py index efa17cd01a936..c9bf15b46e228 100644 --- a/sdks/python/apache_beam/runners/portability/flink_runner.py +++ b/sdks/python/apache_beam/runners/portability/flink_runner.py @@ -35,14 +35,20 @@ class FlinkRunner(portable_runner.PortableRunner): - def run_pipeline(self, pipeline, options): + """A runner for launching jobs on Flink, automatically starting a local + flink master if one is not given. + """ + + # Inherits run_portable_pipeline from PortableRunner. + + def default_environment(self, options): portable_options = options.view_as(pipeline_options.PortableOptions) flink_options = options.view_as(pipeline_options.FlinkRunnerOptions) if (flink_options.flink_master in MAGIC_HOST_NAMES and not portable_options.environment_type and not portable_options.output_executable_path): portable_options.environment_type = 'LOOPBACK' - return super().run_pipeline(pipeline, options) + return super().default_environment(options) def default_job_server(self, options): flink_options = options.view_as(pipeline_options.FlinkRunnerOptions) diff --git a/sdks/python/apache_beam/runners/portability/flink_uber_jar_job_server.py b/sdks/python/apache_beam/runners/portability/flink_uber_jar_job_server.py index 9a40a55c76017..3b302e334a5fa 100644 --- a/sdks/python/apache_beam/runners/portability/flink_uber_jar_job_server.py +++ b/sdks/python/apache_beam/runners/portability/flink_uber_jar_job_server.py @@ -80,8 +80,8 @@ def executable_jar(self): return job_server.JavaJarJobServer.local_jar(url) def flink_version(self): - full_version = requests.get('%s/v1/config' % - self._master_url).json()['flink-version'] + full_version = requests.get( + '%s/v1/config' % self._master_url, timeout=60).json()['flink-version'] # Only return up to minor version. return '.'.join(full_version.split('.')[:2]) diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner/execution.py b/sdks/python/apache_beam/runners/portability/fn_api_runner/execution.py index 0a59b53111376..bc60b5dd86cd1 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner/execution.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner/execution.py @@ -46,17 +46,16 @@ from typing_extensions import Protocol from apache_beam import coders -from apache_beam.coders import BytesCoder from apache_beam.coders.coder_impl import CoderImpl from apache_beam.coders.coder_impl import create_InputStream from apache_beam.coders.coder_impl import create_OutputStream -from apache_beam.coders.coders import GlobalWindowCoder from apache_beam.coders.coders import WindowedValueCoder from apache_beam.portability import common_urns from apache_beam.portability import python_urns from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.runners import pipeline_context +from apache_beam.runners.common import ENCODED_IMPULSE_VALUE from apache_beam.runners.direct.clock import RealClock from apache_beam.runners.direct.clock import TestClock from apache_beam.runners.portability.fn_api_runner import translations @@ -90,12 +89,6 @@ _LOGGER = logging.getLogger(__name__) -IMPULSE_VALUE_CODER_IMPL = WindowedValueCoder( - BytesCoder(), GlobalWindowCoder()).get_impl() - -ENCODED_IMPULSE_VALUE = IMPULSE_VALUE_CODER_IMPL.encode_nested( - GlobalWindows.windowed_value(b'')) - SAFE_WINDOW_FNS = set( window.WindowFn._known_urns.keys()) - {python_urns.PICKLED_WINDOWFN} diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py b/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py index 8d957068d08b0..098e16933b73d 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py @@ -825,7 +825,8 @@ def _execute_bundle(self, buffers_to_clean = set() known_consumers = set() - for _, buffer_id in bundle_context_manager.stage_data_outputs.items(): + for transform_id, buffer_id in ( + bundle_context_manager.stage_data_outputs.items()): for (consuming_stage_name, consuming_transform) in \ runner_execution_context.buffer_id_to_consumer_pairs.get(buffer_id, []): @@ -841,6 +842,13 @@ def _execute_bundle(self, runner_execution_context.pcoll_buffers[buffer_id] = buffer.copy() buffer = runner_execution_context.pcoll_buffers[buffer_id] + # empty buffer. Add it to the pcoll_buffer to avoid element + # duplication. + if buffer_id not in runner_execution_context.pcoll_buffers: + buffer = bundle_context_manager.get_buffer(buffer_id, transform_id) + runner_execution_context.pcoll_buffers[buffer_id] = buffer + buffers_to_clean.add(buffer_id) + # If the buffer has already been added to be consumed by # (stage, transform), then we don't need to add it again. This case # can happen whenever we flatten the same PCollection with itself. diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner_test.py b/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner_test.py index ed09bb8f2236d..9eeeaa4bb24ec 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner_test.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner_test.py @@ -1354,8 +1354,7 @@ def expand(self, pcoll): with self.create_pipeline() as p: _ = p | beam.Create([10, 20, 30]) | PackableCombines() - res = p.run() - res.wait_until_finish() + res = p.result packed_step_name_regex = ( r'.*Packed.*PackableMin.*CombinePerKey.*PackableMax.*CombinePerKey.*' + @@ -1375,6 +1374,15 @@ def expand(self, pcoll): def test_pack_combiners(self): self._test_pack_combiners(assert_using_counter_names=True) + def test_group_by_key_with_empty_pcoll_elements(self): + with self.create_pipeline() as p: + res = ( + p + | beam.Create([('test_key', 'test_value')]) + | beam.Filter(lambda x: False) + | beam.GroupByKey()) + assert_that(res, equal_to([])) + # These tests are kept in a separate group so that they are # not ran in the FnApiRunnerTestWithBundleRepeat which repeats diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner/translations.py b/sdks/python/apache_beam/runners/portability/fn_api_runner/translations.py index df3578e649740..cc1494fc7ae27 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner/translations.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner/translations.py @@ -603,7 +603,8 @@ def pipeline_from_stages( components.transforms.clear() components.pcollections.clear() - roots = set() + # order preserving but still has fast contains checking + roots = {} # type: Dict[str, Any] parents = { child: parent for parent, @@ -618,7 +619,8 @@ def copy_output_pcollections(transform): def add_parent(child, parent): if parent is None: - roots.add(child) + if child not in roots: + roots[child] = None else: if (parent not in components.transforms and parent in pipeline_proto.components.transforms): @@ -665,7 +667,7 @@ def copy_subtransforms(transform): add_parent(transform_id, stage.parent) del new_proto.root_transform_ids[:] - new_proto.root_transform_ids.extend(roots) + new_proto.root_transform_ids.extend(roots.keys()) return new_proto @@ -734,6 +736,29 @@ def optimize_pipeline( # Optimization stages. +def standard_optimize_phases(): + """Returns the basic set of phases, to be passed to optimize_pipeline, + that result in a pipeline consisting only of fused stages (with urn + beam:runner:executable_stage:v1) and, of course, those designated as known + runner urns, in topological order. + """ + return [ + annotate_downstream_side_inputs, + annotate_stateful_dofns_as_roots, + fix_side_input_pcoll_coders, + pack_combiners, + lift_combiners, + expand_sdf, + fix_flatten_coders, + # sink_flattens, + greedily_fuse, + read_to_impulse, + extract_impulse_stages, + remove_data_plane_ops, + sort_stages, + ] + + def annotate_downstream_side_inputs(stages, pipeline_context): # type: (Iterable[Stage], TransformContext) -> Iterable[Stage] @@ -1321,6 +1346,7 @@ def lifted_stages(stage): payload=transform.spec.payload), inputs=transform.inputs, outputs={'out': precombined_pcoll_id}, + annotations=transform.annotations, environment_id=transform.environment_id)) yield make_stage( @@ -1330,6 +1356,7 @@ def lifted_stages(stage): spec=beam_runner_api_pb2.FunctionSpec( urn=common_urns.primitives.GROUP_BY_KEY.urn), inputs={'in': precombined_pcoll_id}, + annotations=transform.annotations, outputs={'out': grouped_pcoll_id})) yield make_stage( @@ -1342,6 +1369,7 @@ def lifted_stages(stage): payload=transform.spec.payload), inputs={'in': grouped_pcoll_id}, outputs={'out': merged_pcoll_id}, + annotations=transform.annotations, environment_id=transform.environment_id)) yield make_stage( @@ -1354,6 +1382,7 @@ def lifted_stages(stage): payload=transform.spec.payload), inputs={'in': merged_pcoll_id}, outputs=transform.outputs, + annotations=transform.annotations, environment_id=transform.environment_id)) def unlifted_stages(stage): diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner/translations_test.py b/sdks/python/apache_beam/runners/portability/fn_api_runner/translations_test.py index 144f067900f3f..3ff2421e62651 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner/translations_test.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner/translations_test.py @@ -384,6 +384,36 @@ def expand(self, pcoll): 'multiple-small-combines/min-4-globally/CombinePerKey', optimized_stage_names) + def test_combineperkey_annotation_propagation(self): + """ + Test that the CPK component transforms inherit annotations from the + source CPK + """ + class MyCombinePerKey(beam.CombinePerKey): + def annotations(self): + return {"my_annotation": b""} + + with TestPipeline() as pipeline: + _ = pipeline | beam.Create([(1, 2)]) | MyCombinePerKey(min) + + # Verify the annotations are propagated to the split up + # CPK transforms + proto = pipeline.to_runner_api( + default_environment=environments.EmbeddedPythonEnvironment( + capabilities=environments.python_sdk_capabilities())) + optimized = translations.optimize_pipeline( + proto, + phases=[translations.lift_combiners], + known_runner_urns=frozenset(), + partial=True) + for transform_id in ['MyCombinePerKey(min)/Precombine', + 'MyCombinePerKey(min)/Group', + 'MyCombinePerKey(min)/Merge', + 'MyCombinePerKey(min)/ExtractOutputs']: + assert ( + "my_annotation" in + optimized.components.transforms[transform_id].annotations) + def test_conditionally_packed_combiners(self): class RecursiveCombine(beam.PTransform): def __init__(self, labels): diff --git a/sdks/python/apache_beam/runners/portability/portable_runner.py b/sdks/python/apache_beam/runners/portability/portable_runner.py index 2c5d4ab36093d..9ff03ec1d0614 100644 --- a/sdks/python/apache_beam/runners/portability/portable_runner.py +++ b/sdks/python/apache_beam/runners/portability/portable_runner.py @@ -19,6 +19,7 @@ # mypy: check-untyped-defs import atexit +import copy import functools import itertools import logging @@ -36,16 +37,16 @@ from apache_beam.metrics import metric from apache_beam.metrics.execution import MetricResult from apache_beam.options.pipeline_options import DebugOptions +from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.options.pipeline_options import PortableOptions -from apache_beam.options.pipeline_options import SetupOptions from apache_beam.options.pipeline_options import StandardOptions -from apache_beam.options.pipeline_options import TypeOptions from apache_beam.options.value_provider import ValueProvider from apache_beam.portability import common_urns +from apache_beam.portability import python_urns from apache_beam.portability.api import beam_artifact_api_pb2_grpc from apache_beam.portability.api import beam_job_api_pb2 +from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.runners import runner -from apache_beam.runners.common import group_by_key_input_visitor from apache_beam.runners.job import utils as job_utils from apache_beam.runners.portability import artifact_service from apache_beam.runners.portability import job_server @@ -57,9 +58,7 @@ if TYPE_CHECKING: from google.protobuf import struct_pb2 # pylint: disable=ungrouped-imports - from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.pipeline import Pipeline - from apache_beam.portability.api import beam_runner_api_pb2 __all__ = ['PortableRunner'] @@ -79,8 +78,6 @@ beam_job_api_pb2.JobState.CANCELLED, ] -ENV_TYPE_ALIASES = {'LOOPBACK': 'EXTERNAL'} - _LOGGER = logging.getLogger(__name__) @@ -268,29 +265,8 @@ def __init__(self): @staticmethod def _create_environment(options): # type: (PipelineOptions) -> environments.Environment - portable_options = options.view_as(PortableOptions) - # Do not set a Runner. Otherwise this can cause problems in Java's - # PipelineOptions, i.e. ClassNotFoundException, if the corresponding Runner - # does not exist in the Java SDK. In portability, the entry point is clearly - # defined via the JobService. - portable_options.view_as(StandardOptions).runner = None - environment_type = portable_options.environment_type - if not environment_type: - environment_urn = common_urns.environments.DOCKER.urn - elif environment_type.startswith('beam:env:'): - environment_urn = environment_type - else: - # e.g. handle LOOPBACK -> EXTERNAL - environment_type = ENV_TYPE_ALIASES.get( - environment_type, environment_type) - try: - environment_urn = getattr( - common_urns.environments, environment_type).urn - except AttributeError: - raise ValueError('Unknown environment type: %s' % environment_type) - - env_class = environments.Environment.get_env_cls_from_urn(environment_urn) - return env_class.from_options(portable_options) + return environments.Environment.from_options( + options.view_as(PortableOptions)) def default_job_server(self, options): raise NotImplementedError( @@ -322,12 +298,16 @@ def create_job_service(self, options): @staticmethod def get_proto_pipeline(pipeline, options): # type: (Pipeline, PipelineOptions) -> beam_runner_api_pb2.Pipeline - portable_options = options.view_as(PortableOptions) - proto_pipeline = pipeline.to_runner_api( - default_environment=PortableRunner._create_environment( - portable_options)) + default_environment=environments.Environment.from_options( + options.view_as(PortableOptions))) + return PortableRunner._optimize_pipeline(proto_pipeline, options) + + @staticmethod + def _optimize_pipeline( + proto_pipeline: beam_runner_api_pb2.Pipeline, + options: PipelineOptions) -> beam_runner_api_pb2.Pipeline: # TODO: https://github.com/apache/beam/issues/19493 # Eventually remove the 'pre_optimize' option alltogether and only perform # the equivalent of the 'default' case below (minus the 'lift_combiners' @@ -347,40 +327,13 @@ def get_proto_pipeline(pipeline, options): ] partial = True elif pre_optimize == 'all': - phases = [ - translations.annotate_downstream_side_inputs, - translations.annotate_stateful_dofns_as_roots, - translations.fix_side_input_pcoll_coders, - translations.pack_combiners, - translations.lift_combiners, - translations.expand_sdf, - translations.fix_flatten_coders, - # translations.sink_flattens, - translations.greedily_fuse, - translations.read_to_impulse, - translations.extract_impulse_stages, - translations.remove_data_plane_ops, - translations.sort_stages - ] + phases = translations.standard_optimize_phases() partial = False elif pre_optimize == 'all_except_fusion': # TODO(https://github.com/apache/beam/issues/19422): Delete this branch # after PortableRunner supports beam:runner:executable_stage:v1. - phases = [ - translations.annotate_downstream_side_inputs, - translations.annotate_stateful_dofns_as_roots, - translations.fix_side_input_pcoll_coders, - translations.pack_combiners, - translations.lift_combiners, - translations.expand_sdf, - translations.fix_flatten_coders, - # translations.sink_flattens, - # translations.greedily_fuse, - translations.read_to_impulse, - translations.extract_impulse_stages, - translations.remove_data_plane_ops, - translations.sort_stages - ] + phases = translations.standard_optimize_phases() + phases.remove(translations.greedily_fuse) partial = True else: phases = [] @@ -410,42 +363,24 @@ def get_proto_pipeline(pipeline, options): return proto_pipeline - def run_pipeline(self, pipeline, options): - # type: (Pipeline, PipelineOptions) -> PipelineResult + def run_portable_pipeline( + self, pipeline: beam_runner_api_pb2.Pipeline, + options: PipelineOptions) -> runner.PipelineResult: portable_options = options.view_as(PortableOptions) - # TODO: https://github.com/apache/beam/issues/19168 - # portable runner specific default - if options.view_as(SetupOptions).sdk_location == 'default': - options.view_as(SetupOptions).sdk_location = 'container' - - experiments = options.view_as(DebugOptions).experiments or [] - - # This is needed as we start a worker server if one is requested - # but none is provided. - if portable_options.environment_type == 'LOOPBACK': - use_loopback_process_worker = options.view_as( - DebugOptions).lookup_experiment('use_loopback_process_worker', False) - portable_options.environment_config, server = ( - worker_pool_main.BeamFnExternalWorkerPoolServicer.start( - state_cache_size= - sdk_worker_main._get_state_cache_size(experiments), - data_buffer_time_limit_ms= - sdk_worker_main._get_data_buffer_time_limit_ms(experiments), - use_process=use_loopback_process_worker)) - cleanup_callbacks = [functools.partial(server.stop, 1)] - else: - cleanup_callbacks = [] + # Do not set a Runner. Otherwise this can cause problems in Java's + # PipelineOptions, i.e. ClassNotFoundException, if the corresponding Runner + # does not exist in the Java SDK. In portability, the entry point is clearly + # defined via the JobService. + portable_options.view_as(StandardOptions).runner = None - pipeline.visit( - group_by_key_input_visitor( - not options.view_as(TypeOptions).allow_non_deterministic_key_coders) - ) + cleanup_callbacks = self.start_and_replace_loopback_environments( + pipeline, options) - proto_pipeline = self.get_proto_pipeline(pipeline, options) + optimized_pipeline = self._optimize_pipeline(pipeline, options) job_service_handle = self.create_job_service(options) - job_id, message_stream, state_stream = \ - job_service_handle.submit(proto_pipeline) + job_id, message_stream, state_stream = job_service_handle.submit( + optimized_pipeline) result = PipelineResult( job_service_handle.job_service, @@ -465,6 +400,32 @@ def run_pipeline(self, pipeline, options): portable_options.environment_type) return result + @staticmethod + def start_and_replace_loopback_environments(pipeline, options): + portable_options = copy.deepcopy(options.view_as(PortableOptions)) + experiments = options.view_as(DebugOptions).experiments or [] + cleanup_callbacks = [] + for env in pipeline.components.environments.values(): + if env.urn == python_urns.EMBEDDED_PYTHON_LOOPBACK: + # Start a worker and change the environment to point to that worker. + use_loopback_process_worker = options.view_as( + DebugOptions).lookup_experiment( + 'use_loopback_process_worker', False) + portable_options.environment_type = 'EXTERNAL' + portable_options.environment_config, server = ( + worker_pool_main.BeamFnExternalWorkerPoolServicer.start( + state_cache_size= + sdk_worker_main._get_state_cache_size(experiments), + data_buffer_time_limit_ms= + sdk_worker_main._get_data_buffer_time_limit_ms(experiments), + use_process=use_loopback_process_worker)) + external_env = environments.ExternalEnvironment.from_options( + portable_options).to_runner_api(None) # type: ignore + env.urn = external_env.urn + env.payload = external_env.payload + cleanup_callbacks.append(functools.partial(server.stop, 1)) + return cleanup_callbacks + class PortableMetrics(metric.MetricResults): def __init__(self, job_metrics_response): diff --git a/sdks/python/apache_beam/runners/portability/requirements_cache_it_test.py b/sdks/python/apache_beam/runners/portability/requirements_cache_it_test.py index b3f9356c197b0..08c8a36dc137b 100644 --- a/sdks/python/apache_beam/runners/portability/requirements_cache_it_test.py +++ b/sdks/python/apache_beam/runners/portability/requirements_cache_it_test.py @@ -24,8 +24,8 @@ import os import shutil import tempfile - -import pkg_resources as pkg +from importlib.metadata import PackageNotFoundError +from importlib.metadata import distribution import apache_beam as beam from apache_beam.options.pipeline_options import PipelineOptions @@ -40,8 +40,8 @@ def verify_packages_from_requirements_file_are_installed(unused_element): _PACKAGE_NOT_IN_REQUIREMENTS_FILE) for package_name in packages_to_test: try: - output = pkg.get_distribution(package_name) - except pkg.DistributionNotFound as e: # pylint: disable=unused-variable + output = distribution(package_name) + except PackageNotFoundError as e: # pylint: disable=unused-variable output = None if package_name in _PACKAGE_IN_REQUIREMENTS_FILE: assert output is not None, ('Please check if package %s is specified' diff --git a/sdks/python/apache_beam/runners/portability/spark_runner.py b/sdks/python/apache_beam/runners/portability/spark_runner.py index c06d1b9c1a2c4..480fbdecdce3b 100644 --- a/sdks/python/apache_beam/runners/portability/spark_runner.py +++ b/sdks/python/apache_beam/runners/portability/spark_runner.py @@ -37,14 +37,20 @@ class SparkRunner(portable_runner.PortableRunner): - def run_pipeline(self, pipeline, options): + """A runner for launching jobs on Spark, automatically starting a local + spark master if one is not given. + """ + + # Inherits run_portable_pipeline from PortableRunner. + + def default_environment(self, options): spark_options = options.view_as(pipeline_options.SparkRunnerOptions) portable_options = options.view_as(pipeline_options.PortableOptions) if (re.match(LOCAL_MASTER_PATTERN, spark_options.spark_master_url) and not portable_options.environment_type and not portable_options.output_executable_path): portable_options.environment_type = 'LOOPBACK' - return super().run_pipeline(pipeline, options) + return super().default_environment(options) def default_job_server(self, options): spark_options = options.view_as(pipeline_options.SparkRunnerOptions) diff --git a/sdks/python/apache_beam/runners/portability/stager.py b/sdks/python/apache_beam/runners/portability/stager.py index b31136b82db74..d59b3e32bc17b 100644 --- a/sdks/python/apache_beam/runners/portability/stager.py +++ b/sdks/python/apache_beam/runners/portability/stager.py @@ -54,14 +54,14 @@ import shutil import sys import tempfile +from importlib.metadata import distribution from typing import Callable from typing import List from typing import Optional from typing import Tuple from urllib.parse import urlparse -import pkg_resources -from pkg_resources import parse_version +from packaging import version from apache_beam.internal import pickler from apache_beam.internal.http_client import get_new_http @@ -115,12 +115,6 @@ def commit_manifest(self): """Commits manifest.""" raise NotImplementedError - @staticmethod - def get_sdk_package_name(): - """For internal use only; no backwards-compatibility guarantees. - Returns the PyPI package name to be staged.""" - return names.BEAM_PACKAGE_NAME - @staticmethod def _create_file_stage_to_artifact(local_path, staged_name): return beam_runner_api_pb2.ArtifactInformation( @@ -296,30 +290,28 @@ def create_job_resources(options, # type: PipelineOptions setup_options.extra_packages, temp_dir=temp_dir)) if hasattr(setup_options, 'sdk_location'): - - if (setup_options.sdk_location == 'default') or Stager._is_remote_path( - setup_options.sdk_location): - # If --sdk_location is not specified then the appropriate package - # will be obtained from PyPI (https://pypi.python.org) based on the - # version of the currently running SDK. If the option is - # present then no version matching is made and the exact URL or path - # is expected. - # - # Unit tests running in the 'python setup.py test' context will - # not have the sdk_location attribute present and therefore we - # will not stage SDK. - sdk_remote_location = 'pypi' if ( - setup_options.sdk_location == 'default' - ) else setup_options.sdk_location - resources.extend( - Stager._create_beam_sdk(sdk_remote_location, temp_dir)) - elif setup_options.sdk_location == 'container': - # Use the SDK that's built into the container, rather than re-staging - # it. + sdk_location = setup_options.sdk_location + if Stager._is_remote_path(sdk_location): + try: + resources.extend( + Stager._create_beam_sdk( + sdk_remote_location=setup_options.sdk_location, + temp_dir=temp_dir, + )) + except: + raise RuntimeError( + 'The --sdk_location option was used with an unsupported ' + 'type of location: %s' % sdk_location) + + elif sdk_location == 'default': + # Use default location for a runner. + pass + elif sdk_location == 'container': + # Used in the past to indicate that SDK should be used from container + # image instead of being staged. + # Equivalent to 'default' now, leaving for backwards compatibility. pass else: - # This branch is also used by internal tests running with the SDK - # built at head. if os.path.isdir(setup_options.sdk_location): sdk_path = os.path.join( setup_options.sdk_location, names.STAGED_SDK_SOURCES_FILENAME) @@ -345,7 +337,6 @@ def create_job_resources(options, # type: PipelineOptions raise RuntimeError( 'The file "%s" cannot be found. Its location was specified ' 'by the --sdk_location command-line option.' % sdk_path) - # The following artifacts are not processed by python sdk container boot # sequence in a setup mode and hence should not be skipped even if a # prebuilt sdk container image is used. @@ -603,11 +594,9 @@ def _create_extra_packages(extra_packages, temp_dir): '".tar", ".tar.gz", ".whl" or ".zip" instead of %s' % package) if os.path.basename(package).endswith('.whl'): _LOGGER.warning( - 'The .whl package "%s" is provided in --extra_package. ' - 'This functionality is not officially supported. Since wheel ' - 'packages are binary distributions, this package must be ' - 'binary-compatible with the worker environment (e.g. Python 2.7 ' - 'running on an x64 Linux host).' % package) + 'The .whl package "%s" provided in --extra_package ' + 'must be binary-compatible with the worker runtime environment.' % + package) if not os.path.isfile(package): if Stager._is_remote_path(package): @@ -713,8 +702,8 @@ def _get_platform_for_default_sdk_container(): # TODO(anandinguva): When https://github.com/pypa/pip/issues/10760 is # addressed, download wheel based on glibc version in Beam's Python # Base image - pip_version = pkg_resources.get_distribution('pip').version - if parse_version(pip_version) >= parse_version('19.3'): + pip_version = distribution('pip').version + if version.parse(pip_version) >= version.parse('19.3'): # pip can only recognize manylinux2014_x86_64 wheels # from version 19.3. return 'manylinux2014_x86_64' @@ -782,15 +771,30 @@ def _build_setup_package(setup_file, # type: str try: os.chdir(os.path.dirname(setup_file)) if build_setup_args is None: - build_setup_args = [ - Stager._get_python_executable(), - os.path.basename(setup_file), - 'sdist', - '--dist-dir', - temp_dir - ] - _LOGGER.info('Executing command: %s', build_setup_args) - processes.check_output(build_setup_args) + # if build is installed in the user env, use it to + # build the sdist else fallback to legacy setup.py sdist call. + try: + build_setup_args = [ + Stager._get_python_executable(), + '-m', + 'build', + '--sdist', + '--outdir', + temp_dir, + os.path.dirname(setup_file), + ] + _LOGGER.info('Executing command: %s', build_setup_args) + processes.check_output(build_setup_args) + except RuntimeError: + build_setup_args = [ + Stager._get_python_executable(), + os.path.basename(setup_file), + 'sdist', + '--dist-dir', + temp_dir + ] + _LOGGER.info('Executing command: %s', build_setup_args) + processes.check_output(build_setup_args) output_files = glob.glob(os.path.join(temp_dir, '*.tar.gz')) if not output_files: raise RuntimeError( @@ -824,8 +828,7 @@ def _create_beam_sdk(sdk_remote_location, temp_dir): Args: sdk_remote_location: A URL from which the file can be downloaded or a - remote file location. The SDK file can be a tarball or a wheel. Set - to 'pypi' to download and stage a wheel and source SDK from PyPi. + remote file location. The SDK file can be a tarball or a wheel. temp_dir: path to temporary location where the file should be downloaded. @@ -836,136 +839,14 @@ def _create_beam_sdk(sdk_remote_location, temp_dir): Raises: RuntimeError: if staging was not successful. """ - if sdk_remote_location == 'pypi': - sdk_local_file = Stager._download_pypi_sdk_package(temp_dir) - sdk_sources_staged_name = Stager.\ - _desired_sdk_filename_in_staging_location(sdk_local_file) - _LOGGER.info('Staging SDK sources from PyPI: %s', sdk_sources_staged_name) - staged_sdk_files = [ - Stager._create_file_stage_to_artifact( - sdk_local_file, sdk_sources_staged_name) - ] - try: - abi_suffix = 'm' if sys.version_info < (3, 8) else '' - # Stage binary distribution of the SDK, for now on a best-effort basis. - platform_tag = Stager._get_platform_for_default_sdk_container() - sdk_local_file = Stager._download_pypi_sdk_package( - temp_dir, - fetch_binary=True, - language_version_tag='%d%d' % - (sys.version_info[0], sys.version_info[1]), - abi_tag='cp%d%d%s' % - (sys.version_info[0], sys.version_info[1], abi_suffix), - platform_tag=platform_tag) - sdk_binary_staged_name = Stager.\ - _desired_sdk_filename_in_staging_location(sdk_local_file) - _LOGGER.info( - 'Staging binary distribution of the SDK from PyPI: %s', - sdk_binary_staged_name) - staged_sdk_files.append( - Stager._create_file_stage_to_artifact( - sdk_local_file, sdk_binary_staged_name)) - except RuntimeError as e: - _LOGGER.warning( - 'Failed to download requested binary distribution ' - 'of the SDK: %s', - repr(e)) - return staged_sdk_files - elif Stager._is_remote_path(sdk_remote_location): - sdk_remote_parsed = urlparse(sdk_remote_location) - sdk_remote_filename = os.path.basename(sdk_remote_parsed.path) - local_download_file = os.path.join(temp_dir, sdk_remote_filename) - Stager._download_file(sdk_remote_location, local_download_file) - staged_name = Stager._desired_sdk_filename_in_staging_location( - local_download_file) - _LOGGER.info('Staging Beam SDK from %s', sdk_remote_location) - return [ - Stager._create_file_stage_to_artifact( - local_download_file, staged_name) - ] - else: - raise RuntimeError( - 'The --sdk_location option was used with an unsupported ' - 'type of location: %s' % sdk_remote_location) - - @staticmethod - def _download_pypi_sdk_package( - temp_dir, - fetch_binary=False, - language_version_tag='39', - language_implementation_tag='cp', - abi_tag='cp39', - platform_tag='manylinux2014_x86_64'): - """Downloads SDK package from PyPI and returns path to local path.""" - package_name = Stager.get_sdk_package_name() - try: - version = pkg_resources.get_distribution(package_name).version - except pkg_resources.DistributionNotFound: - raise RuntimeError( - 'Please set --sdk_location command-line option ' - 'or install a valid {} distribution.'.format(package_name)) - cmd_args = [ - Stager._get_python_executable(), - '-m', - 'pip', - 'download', - '--dest', - temp_dir, - '%s==%s' % (package_name, version), - '--no-deps' + sdk_remote_parsed = urlparse(sdk_remote_location) + sdk_remote_filename = os.path.basename(sdk_remote_parsed.path) + local_download_file = os.path.join(temp_dir, sdk_remote_filename) + Stager._download_file(sdk_remote_location, local_download_file) + staged_name = Stager._desired_sdk_filename_in_staging_location( + local_download_file) + _LOGGER.info('Staging Beam SDK from %s', sdk_remote_location) + return [ + Stager._create_file_stage_to_artifact(local_download_file, staged_name) ] - - if fetch_binary: - _LOGGER.info('Downloading binary distribution of the SDK from PyPi') - # Get a wheel distribution for the SDK from PyPI. - cmd_args.extend([ - '--only-binary', - ':all:', - '--python-version', - language_version_tag, - '--implementation', - language_implementation_tag, - '--abi', - abi_tag, - '--platform', - platform_tag - ]) - # Example wheel: with manylinux14 tag. - # apache_beam-2.43.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl # pylint: disable=line-too-long - if platform_tag == 'manylinux2014_x86_64': - platform_tag = 'manylinux_2_17_x86_64.' + platform_tag - expected_files = [ - os.path.join( - temp_dir, - '%s-%s-%s%s-%s-%s.whl' % ( - package_name.replace('-', '_'), - version, - language_implementation_tag, - language_version_tag, - abi_tag, - platform_tag)), - ] - - else: - _LOGGER.info('Downloading source distribution of the SDK from PyPi') - cmd_args.extend(['--no-binary', ':all:']) - expected_files = [ - os.path.join(temp_dir, '%s-%s.zip' % (package_name, version)), - os.path.join(temp_dir, '%s-%s.tar.gz' % (package_name, version)) - ] - - _LOGGER.info('Executing command: %s', cmd_args) - try: - processes.check_output(cmd_args) - except processes.CalledProcessError as e: - raise RuntimeError(repr(e)) - - for sdk_file in expected_files: - if os.path.exists(sdk_file): - return sdk_file - - raise RuntimeError( - 'Failed to download a distribution for the running SDK. ' - 'Expected either one of %s to be found in the download folder.' % - (expected_files)) diff --git a/sdks/python/apache_beam/runners/portability/stager_test.py b/sdks/python/apache_beam/runners/portability/stager_test.py index bf876925ff7b9..839f7e577733e 100644 --- a/sdks/python/apache_beam/runners/portability/stager_test.py +++ b/sdks/python/apache_beam/runners/portability/stager_test.py @@ -96,64 +96,6 @@ def populate_requirements_cache( self.create_temp_file(os.path.join(cache_dir, 'abc.txt'), 'nothing') self.create_temp_file(os.path.join(cache_dir, 'def.txt'), 'nothing') - def build_fake_pip_download_command_handler(self, has_wheels): - """A stub for apache_beam.utils.processes.check_output that imitates pip. - - Args: - has_wheels: Whether pip fake should have a whl distribution of packages. - """ - def pip_fake(args): - """Fakes fetching a package from pip by creating a temporary file. - - Args: - args: a complete list of command line arguments to invoke pip. - The fake is sensitive to the order of the arguments. - Supported commands: - - 1) Download SDK sources file: - python pip -m download --dest /tmp/dir apache-beam==2.0.0 \ - --no-deps --no-binary :all: - - 2) Download SDK binary wheel file: - python pip -m download --dest /tmp/dir apache-beam==2.0.0 \ - --no-deps --no-binary :all: --python-version 27 \ - --implementation cp --abi cp27mu --platform manylinux1_x86_64 - """ - package_file = None - if len(args) >= 8: - # package_name==x.y.z - if '==' in args[6]: - distribution_name = args[6][0:args[6].find('==')] - distribution_version = args[6][args[6].find('==') + 2:] - - if args[8] == '--no-binary': - package_file = '%s-%s.zip' % ( - distribution_name, distribution_version) - elif args[8] == '--only-binary' and len(args) >= 18: - if not has_wheels: - # Imitate the case when desired wheel distribution is not in PyPI. - raise RuntimeError('No matching distribution.') - - # Per PEP-0427 in wheel filenames non-alphanumeric characters - # in distribution name are replaced with underscore. - distribution_name = distribution_name.replace('-', '_') - if args[17] == 'manylinux2014_x86_64': - args[17] = 'manylinux_2_17_x86_64.' + args[17] - package_file = '%s-%s-%s%s-%s-%s.whl' % ( - distribution_name, - distribution_version, - args[13], # implementation - args[11], # python version - args[15], # abi tag - args[17] # platform - ) - - assert package_file, 'Pip fake does not support the command: ' + str(args) - self.create_temp_file( - FileSystems.join(args[5], package_file), 'Package content.') - - return pip_fake - @mock.patch('apache_beam.runners.portability.stager.open') @mock.patch('apache_beam.runners.portability.stager.get_new_http') def test_download_file_https(self, mock_new_http, mock_open): @@ -433,38 +375,10 @@ def test_sdk_location_default(self): self.update_options(options) options.view_as(SetupOptions).sdk_location = 'default' - with mock.patch( - 'apache_beam.utils.processes.check_output', - self.build_fake_pip_download_command_handler(has_wheels=False)): - _, staged_resources = self.stager.create_and_stage_job_resources( - options, temp_dir=self.make_temp_dir(), staging_location=staging_dir) - - self.assertEqual([names.STAGED_SDK_SOURCES_FILENAME], staged_resources) - - with open(os.path.join(staging_dir, - names.STAGED_SDK_SOURCES_FILENAME)) as f: - self.assertEqual(f.read(), 'Package content.') - - def test_sdk_location_default_with_wheels(self): - staging_dir = self.make_temp_dir() - - options = PipelineOptions() - self.update_options(options) - options.view_as(SetupOptions).sdk_location = 'default' + _, staged_resources = self.stager.create_and_stage_job_resources( + options, temp_dir=self.make_temp_dir(), staging_location=staging_dir) - with mock.patch( - 'apache_beam.utils.processes.check_output', - self.build_fake_pip_download_command_handler(has_wheels=True)): - _, staged_resources = self.stager.create_and_stage_job_resources( - options, temp_dir=self.make_temp_dir(), staging_location=staging_dir) - - self.assertEqual(len(staged_resources), 2) - self.assertEqual(staged_resources[0], names.STAGED_SDK_SOURCES_FILENAME) - # Exact name depends on the version of the SDK. - self.assertTrue(staged_resources[1].endswith('whl')) - for name in staged_resources: - with open(os.path.join(staging_dir, name)) as f: - self.assertEqual(f.read(), 'Package content.') + self.assertEqual([], staged_resources) def test_sdk_location_local_directory(self): staging_dir = self.make_temp_dir() diff --git a/sdks/python/apache_beam/runners/render.py b/sdks/python/apache_beam/runners/render.py index 55e15cb856297..da153d25a4bd5 100644 --- a/sdks/python/apache_beam/runners/render.py +++ b/sdks/python/apache_beam/runners/render.py @@ -78,6 +78,8 @@ except ImportError: gcsio = None # type: ignore +_LOGGER = logging.getLogger(__name__) + # From the Beam site, circa November 2022. DEFAULT_EDGE_STYLE = 'color="#ff570b"' DEFAULT_TRANSFORM_STYLE = ( @@ -147,8 +149,10 @@ def __init__(self, pipeline, options): # Figure out at what point to stop rendering composite internals. if options.render_leaf_composite_nodes: - is_leaf = lambda name: any( - re.match(pattern, name) + is_leaf = lambda transform_id: any( + re.match( + pattern, + self.pipeline.components.transforms[transform_id].unique_name) for patterns in options.render_leaf_composite_nodes for pattern in patterns.split(',')) self.leaf_composites = set() @@ -334,7 +338,7 @@ def page_callback_data(self, layout): } def render_data(self): - logging.info("Re-rendering pipeline...") + _LOGGER.info("Re-rendering pipeline...") layout = self.layout_dot() if self.options.render_output: for path in self.options.render_output: @@ -344,10 +348,10 @@ def render_data(self): input=layout, check=False) if result.returncode: - logging.error( + _LOGGER.error( "Failed render pipeline as %r: exit %s", path, result.returncode) else: - logging.info("Rendered pipeline as %r", path) + _LOGGER.info("Rendered pipeline as %r", path) return self.page_callback_data(layout) def render_json(self): @@ -396,13 +400,41 @@ class RenderRunner(runner.PipelineRunner): # (such as counters, stage completion status, or possibly even PCollection # samples) queryable and/or displayed. This could evolve into a full Beam # UI. - def run_pipeline(self, pipeline_object, options, pipeline_proto=None): - if not pipeline_proto: - pipeline_proto = pipeline_object.to_runner_api() + def run_pipeline(self, pipeline_object, options): + return self.run_portable_pipeline(pipeline_object.to_runner_api(), options) + + def run_portable_pipeline(self, pipeline_proto, options): render_options = options.view_as(RenderOptions) + if render_options.render_port < 0 and not render_options.render_output: + raise ValueError( + 'At least one of --render_port or --render_output must be provided.') if render_options.log_proto: - logging.info(pipeline_proto) + _LOGGER.info(pipeline_proto) renderer = PipelineRenderer(pipeline_proto, render_options) + try: + subprocess.run(['dot', '-V'], capture_output=True, check=True) + except FileNotFoundError as exn: + # If dot is not available, we can at least output the raw .dot files. + dot_files = [ + output for output in render_options.render_output + if output.endswith('.dot') + ] + for output in dot_files: + with open(output, 'w') as fout: + fout.write(renderer.to_dot()) + _LOGGER.info("Wrote pipeline as %s", output) + + non_dot_files = set(render_options.render_output) - set(dot_files) + if non_dot_files: + raise RuntimeError( + "Graphviz dot executable not available " + f"for rendering non-dot output files {non_dot_files}") from exn + elif render_options.render_port >= 0: + raise RuntimeError( + "Graphviz dot executable not available for serving") from exn + + return RenderPipelineResult(None) + renderer.page() if render_options.render_port >= 0: @@ -511,17 +543,16 @@ def render_one(options): pipeline_proto = beam_runner_api_pb2.Pipeline() pipeline_proto.ParseFromString(content) - RenderRunner().run_pipeline( - None, pipeline_options.PipelineOptions(**vars(options)), pipeline_proto) + RenderRunner().run_portable_pipeline( + pipeline_proto, pipeline_options.PipelineOptions(**vars(options))) def run_server(options): class RenderBeamJob(local_job_service.BeamJob): def _invoke_runner(self): - return RenderRunner().run_pipeline( - None, - pipeline_options.PipelineOptions(**vars(options)), - self._pipeline_proto) + return RenderRunner().run_portable_pipeline( + self._pipeline_proto, + pipeline_options.PipelineOptions(**vars(options))) with tempfile.TemporaryDirectory() as staging_dir: job_servicer = local_job_service.LocalJobServicer( diff --git a/sdks/python/apache_beam/runners/render_test.py b/sdks/python/apache_beam/runners/render_test.py index 5872e003aec7c..67e7afc1c7b96 100644 --- a/sdks/python/apache_beam/runners/render_test.py +++ b/sdks/python/apache_beam/runners/render_test.py @@ -16,10 +16,13 @@ # # pytype: skip-file +import os import argparse import logging import subprocess import unittest +import tempfile +import pytest import apache_beam as beam from apache_beam.runners import render @@ -39,6 +42,16 @@ def test_basic_graph(self): self.assertIn('CustomName', dot) self.assertEqual(dot.count('->'), 2) + def test_render_config_validation(self): + p = beam.Pipeline() + _ = ( + p | beam.Impulse() | beam.Map(lambda _: 2) + | 'CustomName' >> beam.Map(lambda x: x * x)) + pipeline_proto = p.to_runner_api() + with pytest.raises(ValueError): + render.RenderRunner().run_portable_pipeline( + pipeline_proto, render.RenderOptions()) + def test_side_input(self): p = beam.Pipeline() pcoll = p | beam.Impulse() | beam.FlatMap(lambda x: [1, 2, 3]) @@ -65,11 +78,35 @@ def test_composite_collapse(self): renderer.update(toggle=[create_transform_id]) self.assertEqual(renderer.to_dot().count('->'), 1) - def test_dot_well_formed(self): + +class DotRequiringRenderingTest(unittest.TestCase): + @classmethod + def setUpClass(cls): try: subprocess.run(['dot', '-V'], capture_output=True, check=True) except FileNotFoundError: + cls._dot_installed = False + else: + cls._dot_installed = True + + def setUp(self) -> None: + if not self._dot_installed: # type: ignore[attr-defined] self.skipTest('dot executable not installed') + + def test_run_portable_pipeline(self): + p = beam.Pipeline() + _ = ( + p | beam.Impulse() | beam.Map(lambda _: 2) + | 'CustomName' >> beam.Map(lambda x: x * x)) + pipeline_proto = p.to_runner_api() + + with tempfile.TemporaryDirectory() as tmpdir: + svg_path = os.path.join(tmpdir, "my_output.svg") + render.RenderRunner().run_portable_pipeline( + pipeline_proto, render.RenderOptions(render_output=[svg_path])) + assert os.path.exists(svg_path) + + def test_dot_well_formed(self): p = beam.Pipeline() _ = p | beam.Create([1, 2, 3]) | beam.Map(lambda x: x * x) pipeline_proto = p.to_runner_api() @@ -83,6 +120,15 @@ def test_dot_well_formed(self): renderer.update(toggle=[create_transform_id]) renderer.render_data() + def test_leaf_composite_filter(self): + p = beam.Pipeline() + _ = p | beam.Create([1, 2, 3]) | beam.Map(lambda x: x * x) + dot = render.PipelineRenderer( + p.to_runner_api(), + render.RenderOptions(['--render_leaf_composite_nodes=Create' + ])).to_dot() + self.assertEqual(dot.count('->'), 1) + if __name__ == '__main__': logging.getLogger().setLevel(logging.INFO) diff --git a/sdks/python/apache_beam/runners/runner.py b/sdks/python/apache_beam/runners/runner.py index f2e09edb28eec..d037e0d42c0b6 100644 --- a/sdks/python/apache_beam/runners/runner.py +++ b/sdks/python/apache_beam/runners/runner.py @@ -21,22 +21,24 @@ import importlib import logging -import os -import shelve -import shutil -import tempfile from typing import TYPE_CHECKING +from typing import Iterable from typing import Optional +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import PortableOptions +from apache_beam.options.pipeline_options import SetupOptions from apache_beam.options.pipeline_options import StandardOptions +from apache_beam.options.pipeline_options import TypeOptions +from apache_beam.portability import common_urns +from apache_beam.portability.api import beam_runner_api_pb2 +from apache_beam.runners.common import group_by_key_input_visitor +from apache_beam.transforms import environments if TYPE_CHECKING: from apache_beam import pvalue from apache_beam import PTransform - from apache_beam.options.pipeline_options import PipelineOptions - from apache_beam.pipeline import AppliedPTransform from apache_beam.pipeline import Pipeline - from apache_beam.pipeline import PipelineVisitor __all__ = ['PipelineRunner', 'PipelineState', 'PipelineResult'] @@ -150,6 +152,24 @@ def run_async(self, transform(PBegin(p)) return p.run() + def run_portable_pipeline( + self, pipeline: beam_runner_api_pb2.Pipeline, + options: PipelineOptions) -> 'PipelineResult': + """Execute the entire pipeline. + + Runners should override this method. + """ + raise NotImplementedError + + def default_environment( + self, options: PipelineOptions) -> environments.Environment: + """Returns the default environment that should be used for this runner. + + Runners may override this method to provide alternative environments. + """ + return environments.Environment.from_options( + options.view_as(PortableOptions)) + def run_pipeline( self, pipeline, # type: Pipeline @@ -158,179 +178,64 @@ def run_pipeline( # type: (...) -> PipelineResult """Execute the entire pipeline or the sub-DAG reachable from a node. - - Runners should override this method. """ - raise NotImplementedError + pipeline.visit( + group_by_key_input_visitor( + not options.view_as(TypeOptions).allow_non_deterministic_key_coders) + ) + + # TODO: https://github.com/apache/beam/issues/19168 + # portable runner specific default + if options.view_as(SetupOptions).sdk_location == 'default': + options.view_as(SetupOptions).sdk_location = 'container' + + return self.run_portable_pipeline( + pipeline.to_runner_api( + default_environment=self.default_environment(options)), + options) def apply(self, transform, # type: PTransform input, # type: Optional[pvalue.PValue] options # type: PipelineOptions ): - """Runner callback for a pipeline.apply call. - - Args: - transform: the transform to apply. - input: transform's input (typically a PCollection). - - A concrete implementation of the Runner class may want to do custom - pipeline construction for a given transform. To override the behavior - for a transform class Xyz, implement an apply_Xyz method with this same - signature. - """ - for cls in transform.__class__.mro(): - m = getattr(self, 'apply_%s' % cls.__name__, None) - if m: - return m(transform, input, options) - raise NotImplementedError( - 'Execution of [%s] not implemented in runner %s.' % (transform, self)) - - def visit_transforms( - self, - pipeline, # type: Pipeline - options # type: PipelineOptions - ): - # type: (...) -> None - # Imported here to avoid circular dependencies. - # pylint: disable=wrong-import-order, wrong-import-position - from apache_beam.pipeline import PipelineVisitor - - class RunVisitor(PipelineVisitor): - def __init__(self, runner): - # type: (PipelineRunner) -> None - self.runner = runner - - def visit_transform(self, transform_node): - try: - self.runner.run_transform(transform_node, options) - except: - _LOGGER.error('Error while visiting %s', transform_node.full_label) - raise - - pipeline.visit(RunVisitor(self)) + # TODO(robertwb): Remove indirection once internal references are fixed. + return self.apply_PTransform(transform, input, options) def apply_PTransform(self, transform, input, options): - # The base case of apply is to call the transform's expand. + # TODO(robertwb): Remove indirection once internal references are fixed. return transform.expand(input) - def run_transform(self, - transform_node, # type: AppliedPTransform - options # type: PipelineOptions - ): - """Runner callback for a pipeline.run call. - - Args: - transform_node: transform node for the transform to run. - - A concrete implementation of the Runner class must implement run_Abc for - some class Abc in the method resolution order for every non-composite - transform Xyz in the pipeline. - """ - for cls in transform_node.transform.__class__.mro(): - m = getattr(self, 'run_%s' % cls.__name__, None) - if m: - return m(transform_node, options) - raise NotImplementedError( - 'Execution of [%s] not implemented in runner %s.' % - (transform_node.transform, self)) - def is_fnapi_compatible(self): """Whether to enable the beam_fn_api experiment by default.""" return True + def check_requirements( + self, + pipeline_proto: beam_runner_api_pb2.Pipeline, + supported_requirements: Iterable[str]): + """Check that this runner can satisfy all pipeline requirements.""" -class PValueCache(object): - """For internal use only; no backwards-compatibility guarantees. - - Local cache for arbitrary information computed for PValue objects.""" - def __init__(self, use_disk_backed_cache=False): - # Cache of values computed while a runner executes a pipeline. This is a - # dictionary of PValues and their computed values. Note that in principle - # the runner could contain PValues from several pipelines without clashes - # since a PValue is associated with one and only one pipeline. The keys of - # the dictionary are tuple of PValue instance addresses obtained using id() - # and tag names converted to strings. - - self._use_disk_backed_cache = use_disk_backed_cache - if use_disk_backed_cache: - self._tempdir = tempfile.mkdtemp() - self._cache = shelve.open(os.path.join(self._tempdir, 'shelve')) - else: - self._cache = {} - - def __del__(self): - if self._use_disk_backed_cache: - self._cache.close() - shutil.rmtree(self._tempdir) - - def __len__(self): - return len(self._cache) - - def to_cache_key(self, transform, tag): - return transform.full_label, tag - - def _ensure_pvalue_has_real_producer(self, pvalue): - """Ensure the passed-in PValue has the real_producer attribute. - - Args: - pvalue: A PValue instance whose cached value is requested. - - During the runner's execution only the results of the primitive transforms - are cached. Whenever we are looking for a PValue that is the output of a - composite transform we need to find the output of its rightmost transform - part. - """ - if not hasattr(pvalue, 'real_producer'): - real_producer = pvalue.producer - while real_producer.parts: - real_producer = real_producer.parts[-1] - pvalue.real_producer = real_producer - - def is_cached(self, pobj): - from apache_beam.pipeline import AppliedPTransform - if isinstance(pobj, AppliedPTransform): - transform = pobj - tag = None - else: - self._ensure_pvalue_has_real_producer(pobj) - transform = pobj.real_producer - tag = pobj.tag - return self.to_cache_key(transform, tag) in self._cache - - def cache_output(self, transform, tag_or_value, value=None): - if value is None: - value = tag_or_value - tag = None - else: - tag = tag_or_value - self._cache[self.to_cache_key(transform, tag)] = value - - def get_pvalue(self, pvalue): - """Gets the value associated with a PValue from the cache.""" - self._ensure_pvalue_has_real_producer(pvalue) - try: - return self._cache[self.key(pvalue)] - except KeyError: - if (pvalue.tag is not None and - self.to_cache_key(pvalue.real_producer, None) in self._cache): - # This is an undeclared, empty output of a DoFn executed - # in the local runner before this output was referenced. - return [] - else: - raise - - def get_unwindowed_pvalue(self, pvalue): - return [v.value for v in self.get_pvalue(pvalue)] - - def clear_pvalue(self, pvalue): - """Removes a PValue from the cache.""" - if self.is_cached(pvalue): - del self._cache[self.key(pvalue)] - - def key(self, pobj): - self._ensure_pvalue_has_real_producer(pobj) - return self.to_cache_key(pobj.real_producer, pobj.tag) + # Imported here to avoid circular dependencies. + # pylint: disable=wrong-import-order, wrong-import-position + from apache_beam.runners.portability.fn_api_runner import translations + supported_requirements = set(supported_requirements) + for requirement in pipeline_proto.requirements: + if requirement not in supported_requirements: + raise ValueError( + 'Unable to run pipeline with requirement: %s' % requirement) + for transform in pipeline_proto.components.transforms.values(): + if transform.spec.urn == common_urns.primitives.TEST_STREAM.urn: + if common_urns.primitives.TEST_STREAM.urn not in supported_requirements: + raise NotImplementedError(transform.spec.urn) + elif transform.spec.urn in translations.PAR_DO_URNS: + payload = beam_runner_api_pb2.ParDoPayload.FromString( + transform.spec.payload) + for timer in payload.timer_family_specs.values(): + if timer.time_domain not in ( + beam_runner_api_pb2.TimeDomain.EVENT_TIME, + beam_runner_api_pb2.TimeDomain.PROCESSING_TIME): + raise NotImplementedError(timer.time_domain) # FIXME: replace with PipelineState(str, enum.Enum) @@ -376,7 +281,7 @@ def state(self): """Return the current state of the pipeline execution.""" return self._state - def wait_until_finish(self, duration=None): + def wait_until_finish(self, duration=None): # pylint: disable=unused-argument """Waits until the pipeline finishes and returns the final status. Args: @@ -393,7 +298,8 @@ def wait_until_finish(self, duration=None): Returns: The final state of the pipeline, or :data:`None` on timeout. """ - raise NotImplementedError + if not PipelineState.is_terminal(self._state): + raise NotImplementedError() def cancel(self): """Cancels the pipeline execution. @@ -407,7 +313,7 @@ def cancel(self): Returns: The final state of the pipeline. """ - raise NotImplementedError + raise NotImplementedError() def metrics(self): """Returns :class:`~apache_beam.metrics.metric.MetricResults` object to @@ -417,7 +323,7 @@ def metrics(self): NotImplementedError: If the runner does not support this operation. """ - raise NotImplementedError + raise NotImplementedError() # pylint: disable=unused-argument def aggregated_values(self, aggregator_or_name): diff --git a/sdks/python/apache_beam/runners/trivial_runner.py b/sdks/python/apache_beam/runners/trivial_runner.py new file mode 100644 index 0000000000000..219c84f066243 --- /dev/null +++ b/sdks/python/apache_beam/runners/trivial_runner.py @@ -0,0 +1,416 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import collections +import logging +from typing import Any +from typing import Iterable +from typing import Iterator +from typing import List +from typing import TypeVar + +from apache_beam import coders +from apache_beam.coders.coder_impl import create_InputStream +from apache_beam.coders.coder_impl import create_OutputStream +from apache_beam.portability import common_urns +from apache_beam.portability.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_runner_api_pb2 +from apache_beam.runners import common +from apache_beam.runners import pipeline_context +from apache_beam.runners import runner +from apache_beam.runners.portability.fn_api_runner import translations +from apache_beam.runners.portability.fn_api_runner import worker_handlers +from apache_beam.runners.worker import bundle_processor +from apache_beam.transforms import core +from apache_beam.transforms import trigger +from apache_beam.utils import windowed_value + +T = TypeVar("T") + +_LOGGER = logging.getLogger(__name__) + + +class TrivialRunner(runner.PipelineRunner): + """A bare-bones batch Python pipeline runner illistrating how to use the + RunnerAPI and FnAPI to execute pipelines. + + Note that this runner is primarily for pedagogical purposes and is missing + several features in order to keep it as simple as possible. Where possible + pointers are provided which this should serve as a useful starting point. + """ + def run_portable_pipeline(self, pipeline, options): + # First ensure we are able to run this pipeline. + # Specifically, that it does not depend on requirements that were + # added since this runner was developed. + self.check_requirements(pipeline, self.supported_requirements()) + + # Now we optimize the pipeline, notably performing pipeline fusion, + # to turn it into a DAG where all the operations are one of + # Impulse, Flatten, GroupByKey, or beam:runner:executable_stage. + optimized_pipeline = translations.optimize_pipeline( + pipeline, + phases=translations.standard_optimize_phases(), + known_runner_urns=frozenset([ + common_urns.primitives.IMPULSE.urn, + common_urns.primitives.FLATTEN.urn, + common_urns.primitives.GROUP_BY_KEY.urn + ]), + # This boolean indicates we want fused executable_stages. + partial=False) + + # standard_optimize_phases() has a final step giving the stages in + # topological order, so now we can just walk over them and execute + # them. (This are not quite so simple if we were attempting to execute + # a streaming pipeline, but this is a trivial runner that only supports + # batch...) + execution_state = ExecutionState(optimized_pipeline) + for transform_id in optimized_pipeline.root_transform_ids: + self.execute_transform(transform_id, execution_state) + + # A more sophisticated runner may perform the execution in the background + # and return a PipelineResult that can be used to monitor/cancel the + # concurrent execution. + return runner.PipelineResult(runner.PipelineState.DONE) + + def execute_transform(self, transform_id, execution_state): + """Execute a single transform.""" + transform_proto = execution_state.optimized_pipeline.components.transforms[ + transform_id] + _LOGGER.info( + "Executing stage %s %s", transform_id, transform_proto.unique_name) + if not is_primitive_transform(transform_proto): + # A composite is simply executed by executing its parts. + for sub_transform in transform_proto.subtransforms: + self.execute_transform(sub_transform, execution_state) + + elif transform_proto.spec.urn == common_urns.primitives.IMPULSE.urn: + # An impulse has no inputs and produces a single output (which happens + # to be an empty byte string in the global window). + execution_state.set_pcollection_contents( + only_element(transform_proto.outputs.values()), + [common.ENCODED_IMPULSE_VALUE]) + + elif transform_proto.spec.urn == common_urns.primitives.FLATTEN.urn: + # The output of a flatten is simply the union of its inputs. + output_pcoll_id = only_element(transform_proto.outputs.values()) + execution_state.set_pcollection_contents( + output_pcoll_id, + sum([ + execution_state.get_pcollection_contents(pc) + for pc in transform_proto.inputs.values() + ], [])) + + elif transform_proto.spec.urn == 'beam:runner:executable_stage:v1': + # This is a collection of user DoFns. + self.execute_executable_stage(transform_proto, execution_state) + + elif transform_proto.spec.urn == common_urns.primitives.GROUP_BY_KEY.urn: + # Execute the grouping operation. + self.group_by_key_and_window( + only_element(transform_proto.inputs.values()), + only_element(transform_proto.outputs.values()), + execution_state) + + else: + raise RuntimeError( + f"Unsupported transform {transform_id}" + " of type {transform_proto.spec.urn}") + + def execute_executable_stage(self, transform_proto, execution_state): + # Stage here is like a mini pipeline, with PTransforms, PCollections, etc. + # inside of it. + stage = beam_runner_api_pb2.ExecutableStagePayload.FromString( + transform_proto.spec.payload) + if stage.side_inputs: + # To support these we would need to make the side input PCollections + # available over the state API before processing this bundle. + raise NotImplementedError() + + # This is the set of transforms that were fused together. + stage_transforms = { + id: stage.components.transforms[id] + for id in stage.transforms + } + # The executable stage has bare PCollections as its inputs and outputs. + # + # We need an operation to feed data into the bundle (runner -> SDK). + # This is done by attaching special transform that reads from the data + # channel. + input_transform = execution_state.new_id('stage_input') + input_pcoll = stage.input + stage_transforms[input_transform] = beam_runner_api_pb2.PTransform( + # Read data, encoded with the given coder, from the data channel. + spec=beam_runner_api_pb2.FunctionSpec( + urn=bundle_processor.DATA_INPUT_URN, + payload=beam_fn_api_pb2.RemoteGrpcPort( + # If we were using a cross-process data channel, we would also + # need to set the address of the data channel itself here. + coder_id=execution_state.windowed_coder_id( + stage.input)).SerializeToString()), + # Wire its "output" to the required PCollection. + outputs={'out': input_pcoll}) + # Also add operations to consume data produced by the bundle and writes + # them to the data channel (SDK -> runner). + output_ops_to_pcoll = {} + for output_pcoll in stage.outputs: + output_transform = execution_state.new_id('stage_output') + stage_transforms[output_transform] = beam_runner_api_pb2.PTransform( + # Data will be written here, with the given coder. + spec=beam_runner_api_pb2.FunctionSpec( + urn=bundle_processor.DATA_OUTPUT_URN, + payload=beam_fn_api_pb2.RemoteGrpcPort( + # Again, the grpc address itself is implicit. + coder_id=execution_state.windowed_coder_id( + output_pcoll)).SerializeToString()), + # This operation takes as input the bundle's output pcollection. + inputs={'input': output_pcoll}) + output_ops_to_pcoll[output_transform] = output_pcoll + + # Now we can create a description of what it means to process a bundle + # of this type. When processing the bundle, we simply refer to this + # descriptor by id. (In our case we only do so once, but in a more general + # runner one may want to process the same "type" of bundle of many distinct + # partitions of the input, especially in streaming where one may have + # hundreds of concurrently processed bundles per key.) + process_bundle_descriptor = beam_fn_api_pb2.ProcessBundleDescriptor( + id=execution_state.new_id('descriptor'), + transforms=stage_transforms, + pcollections=stage.components.pcollections, + coders=execution_state.optimized_pipeline.components.coders, + windowing_strategies=stage.components.windowing_strategies, + environments=stage.components.environments, + # Were timers and state supported, their endpoints would be listed here. + ) + execution_state.register_process_bundle_descriptor( + process_bundle_descriptor) + + # Now we are ready to actually execute the bundle. + process_bundle_id = execution_state.new_id('bundle') + + # First, push the all input data onto the data channel. + # Timers would be sent over this channel as well. + # In a real runner we could do this after (or concurrently with) starting + # the bundle to avoid having to hold all the data to process in memory, + # but for simplicity our bundle invocation (below) is synchronous so the + # data must be available for processing right away. + to_worker = execution_state.worker_handler.data_conn.output_stream( + process_bundle_id, input_transform) + for encoded_data in execution_state.get_pcollection_contents(input_pcoll): + to_worker.write(encoded_data) + to_worker.close() + + # Now we send a process bundle request over the control plane. + process_bundle_request = beam_fn_api_pb2.InstructionRequest( + instruction_id=process_bundle_id, + process_bundle=beam_fn_api_pb2.ProcessBundleRequest( + process_bundle_descriptor_id=process_bundle_descriptor.id)) + result_future = execution_state.worker_handler.control_conn.push( + process_bundle_request) + + # Read the results off the data channel. + # Note that if there are multiple outputs, we may get them in any order, + # possibly interleaved. + for output in execution_state.worker_handler.data_conn.input_elements( + process_bundle_id, list(output_ops_to_pcoll.keys())): + if isinstance(output, beam_fn_api_pb2.Elements.Data): + # Adds the output to the appropriate PCollection. + execution_state.set_pcollection_contents( + output_ops_to_pcoll[output.transform_id], [output.data]) + else: + # E.g. timers to set. + raise RuntimeError("Unexpected data type: %s" % output) + + # Ensure the operation completed successfully. + # This result contains things like metrics and continuation tokens as well. + result = result_future.get() + if result.error: + raise RuntimeError(result.error) + if result.process_bundle.residual_roots: + # We would need to re-schedule execution of this bundle with this data. + raise NotImplementedError('SDF continuation') + if result.process_bundle.requires_finalization: + # We would need to invoke the finalization callback, on a best effort + # basis, *after* the outputs are durably committed. + raise NotImplementedError('finalization') + if result.process_bundle.elements.data: + # These should be processed just like outputs from the data channel. + raise NotImplementedError('control-channel data') + if result.process_bundle.elements.timers: + # These should be processed just like outputs from the data channel. + raise NotImplementedError('timers') + + def group_by_key_and_window(self, input_pcoll, output_pcoll, execution_state): + """Groups the elements of input_pcoll, placing their output in output_pcoll. + """ + # Note that we are using the designated coders to decode and deeply inspect + # the elements. This could be useful for other operations as well (e.g. + # splitting the returned bundles of encoded elements into smaller chunks). + # One issue that we're sweeping under the rug here is that for a truly + # portable multi-language runner we may not be able to instanciate + # every coder in Python. This can be overcome by wrapping unknown coders as + # length-prefixing (or otherwise delimiting) variants. + + # Decode the input elements to get at their individual keys and values. + input_coder = execution_state.windowed_coder(input_pcoll) + key_coder = input_coder.key_coder() + input_elements = [] + for encoded_elements in execution_state.get_pcollection_contents( + input_pcoll): + for element in decode_all(encoded_elements, input_coder): + input_elements.append(element) + + # Now perform the actual grouping. + components = execution_state.optimized_pipeline.components + windowing = components.windowing_strategies[ + components.pcollections[input_pcoll].windowing_strategy_id] + + if (windowing.merge_status == + beam_runner_api_pb2.MergeStatus.Enum.NON_MERGING and + windowing.output_time == + beam_runner_api_pb2.OutputTime.Enum.END_OF_WINDOW): + # This is the "easy" case, show how to do it by hand. + # Note that we're grouping by encoded key, and also by the window. + grouped = collections.defaultdict(list) + for element in input_elements: + for window in element.windows: + key, value = element.value + grouped[window, key_coder.encode(key)].append(value) + output_elements = [ + windowed_value.WindowedValue( + (key_coder.decode(encoded_key), values), + window.end, [window], + trigger.BatchGlobalTriggerDriver.ONLY_FIRING) + for ((window, encoded_key), values) in grouped.items() + ] + else: + # This handles generic merging and triggering. + trigger_driver = trigger.create_trigger_driver( + execution_state.windowing_strategy(input_pcoll), True) + grouped_by_key = collections.defaultdict(list) + for element in input_elements: + key, value = element.value + grouped_by_key[key_coder.encode(key)].append(element.with_value(value)) + output_elements = [] + for encoded_key, windowed_values in grouped_by_key.items(): + for grouping in trigger_driver.process_entire_key( + key_coder.decode(encoded_key), windowed_values): + output_elements.append(grouping) + + # Store the grouped values in the output PCollection. + output_coder = execution_state.windowed_coder(output_pcoll) + execution_state.set_pcollection_contents( + output_pcoll, [encode_all(output_elements, output_coder)]) + + def supported_requirements(self) -> Iterable[str]: + # Nothing non-trivial is supported. + return [] + + +class ExecutionState: + """A helper class holding various values and context during execution.""" + def __init__(self, optimized_pipeline): + self.optimized_pipeline = optimized_pipeline + self._pcollections_to_encoded_chunks = {} + self._counter = 0 + self._process_bundle_descriptors = {} + # This emulates a connection to an SDK worker (e.g. its data, control, + # etc. channels). + # There are other variants available as well (e.g. GRPC, Docker, ...) + self.worker_handler = worker_handlers.EmbeddedWorkerHandler( + None, + state=worker_handlers.StateServicer(), + provision_info=None, + worker_manager=self) + self._windowed_coders = {} + # Populate all windowed coders before creating _pipeline_context. + for pcoll_id in self.optimized_pipeline.components.pcollections.keys(): + self.windowed_coder_id(pcoll_id) + self._pipeline_context = pipeline_context.PipelineContext( + optimized_pipeline.components) + + def register_process_bundle_descriptor( + self, process_bundle_descriptor: beam_fn_api_pb2.ProcessBundleDescriptor): + self._process_bundle_descriptors[ + process_bundle_descriptor.id] = process_bundle_descriptor + + def get_pcollection_contents(self, pcoll_id: str) -> List[bytes]: + return self._pcollections_to_encoded_chunks[pcoll_id] + + def set_pcollection_contents(self, pcoll_id: str, chunks: List[bytes]): + self._pcollections_to_encoded_chunks[pcoll_id] = chunks + + def new_id(self, prefix='') -> str: + self._counter += 1 + return f'runner_{prefix}_{self._counter}' + + def windowed_coder(self, pcollection_id: str) -> coders.Coder: + return self._pipeline_context.coders.get_by_id( + self.windowed_coder_id(pcollection_id)) + + def windowing_strategy(self, pcollection_id: str) -> core.Windowing: + return self._pipeline_context.windowing_strategies.get_by_id( + self.optimized_pipeline.components.pcollections[pcollection_id]. + windowing_strategy_id) + + def windowed_coder_id(self, pcollection_id: str) -> str: + pcoll = self.optimized_pipeline.components.pcollections[pcollection_id] + windowing = self.optimized_pipeline.components.windowing_strategies[ + pcoll.windowing_strategy_id] + return self._windowed_coder_id_from( + pcoll.coder_id, windowing.window_coder_id) + + def _windowed_coder_id_from(self, coder_id: str, window_coder_id: str) -> str: + if (coder_id, window_coder_id) not in self._windowed_coders: + windowed_coder_id = self.new_id('windowed_coder') + self._windowed_coders[coder_id, window_coder_id] = windowed_coder_id + self.optimized_pipeline.components.coders[windowed_coder_id].CopyFrom( + beam_runner_api_pb2.Coder( + spec=beam_runner_api_pb2.FunctionSpec( + urn=common_urns.coders.WINDOWED_VALUE.urn), + component_coder_ids=[coder_id, window_coder_id])) + return self._windowed_coders[coder_id, window_coder_id] + + +def is_primitive_transform(transform: beam_runner_api_pb2.PTransform) -> bool: + # pylint: disable=consider-using-ternary + return ( + # Only non-primitive (aka composite) transforms have subtransforms. + not transform.subtransforms + # If it has outputs but all of the outputs are also inputs to this + # transform, there is nothing to compute. + and not transform.outputs or + bool(set(transform.outputs.values()) - set(transform.inputs.values()))) + + +def only_element(iterable: Iterable[T]) -> T: + element, = iterable + return element + + +def decode_all(encoded_elements: bytes, coder: coders.Coder) -> Iterator[Any]: + coder_impl = coder.get_impl() + input_stream = create_InputStream(encoded_elements) + while input_stream.size() > 0: + yield coder_impl.decode_from_stream(input_stream, True) + + +def encode_all(elements: Iterator[T], coder: coders.Coder) -> bytes: + coder_impl = coder.get_impl() + output_stream = create_OutputStream() + for element in elements: + coder_impl.encode_to_stream(element, output_stream, True) + return output_stream.get() diff --git a/sdks/python/apache_beam/runners/trivial_runner_test.py b/sdks/python/apache_beam/runners/trivial_runner_test.py new file mode 100644 index 0000000000000..6acfc2fee4951 --- /dev/null +++ b/sdks/python/apache_beam/runners/trivial_runner_test.py @@ -0,0 +1,74 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import unittest + +import apache_beam as beam +from apache_beam.runners.trivial_runner import TrivialRunner +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to + + +class TrivialRunnerTest(unittest.TestCase): + def test_trivial(self): + # The most trivial pipeline, to ensure at least something is working. + # (Notably avoids the non-trivial complexity within assert_that.) + with beam.Pipeline(runner=TrivialRunner()) as p: + _ = p | beam.Impulse() + + def test_assert_that(self): + # If this fails, the other tests may be vacuous. + with self.assertRaisesRegex(Exception, 'Failed assert'): + with beam.Pipeline(runner=TrivialRunner()) as p: + assert_that(p | beam.Impulse(), equal_to(['a'])) + + def test_impulse(self): + with beam.Pipeline(runner=TrivialRunner()) as p: + assert_that(p | beam.Impulse(), equal_to([b''])) + + def test_create(self): + with beam.Pipeline(runner=TrivialRunner()) as p: + assert_that(p | beam.Create(['a', 'b']), equal_to(['a', 'b'])) + + def test_flatten(self): + with beam.Pipeline(runner=TrivialRunner()) as p: + # Disabling reshuffle here (and elsewhere) just to give a simpler graph + # for debugging. + ab = p | 'AB' >> beam.Create(['a', 'b'], reshuffle=False) + c = p | 'C' >> beam.Create(['c'], reshuffle=False) + assert_that((ab, c, c) | beam.Flatten(), equal_to(['a', 'b', 'c', 'c'])) + + def test_map(self): + with beam.Pipeline(runner=TrivialRunner()) as p: + assert_that( + p | beam.Create(['a', 'b'], reshuffle=False) | beam.Map(str.upper), + equal_to(['A', 'B'])) + + def test_gbk(self): + with beam.Pipeline(runner=TrivialRunner()) as p: + result = ( + p + | beam.Create([('a', 1), ('b', 2), ('b', 3)], reshuffle=False) + | beam.GroupByKey() + | beam.MapTuple(lambda k, vs: (k, sorted(vs)))) + assert_that(result, equal_to([('a', [1]), ('b', [2, 3])])) + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + unittest.main() diff --git a/sdks/python/apache_beam/runners/worker/bundle_processor.py b/sdks/python/apache_beam/runners/worker/bundle_processor.py index 6d814331fd6c8..7ff0ad258bc2d 100644 --- a/sdks/python/apache_beam/runners/worker/bundle_processor.py +++ b/sdks/python/apache_beam/runners/worker/bundle_processor.py @@ -27,6 +27,8 @@ import logging import random import threading +from dataclasses import dataclass +from dataclasses import field from typing import TYPE_CHECKING from typing import Any from typing import Callable @@ -66,7 +68,6 @@ from apache_beam.runners.worker import operation_specs from apache_beam.runners.worker import operations from apache_beam.runners.worker import statesampler -from apache_beam.runners.worker.data_sampler import OutputSampler from apache_beam.transforms import TimeDomain from apache_beam.transforms import core from apache_beam.transforms import environments @@ -194,8 +195,8 @@ def __init__(self, self.stop = float('inf') self.started = False - def setup(self): - super().setup() + def setup(self, data_sampler=None): + super().setup(data_sampler) # We must do this manually as we don't have a spec or spec.output_coders. self.receivers = [ operations.ConsumerSet.create( @@ -226,8 +227,13 @@ def process_encoded(self, encoded_windowed_values): if self.index == self.stop - 1: return self.index += 1 - decoded_value = self.windowed_coder_impl.decode_from_stream( - input_stream, True) + try: + decoded_value = self.windowed_coder_impl.decode_from_stream( + input_stream, True) + except Exception as exn: + raise ValueError( + "Error decoding input stream with coder " + + str(self.windowed_coder)) from exn self.output(decoded_value) def monitoring_infos(self, transform_id, tag_to_pcollection_id): @@ -827,6 +833,17 @@ def only_element(iterable): return element +def _environments_compatible(submission, runtime): + # type: (str, str) -> bool + if submission == runtime: + return True + if 'rc' in submission and runtime in submission: + # TODO(https://github.com/apache/beam/issues/28084): Loosen + # the check for RCs until RC containers install the matching version. + return True + return False + + def _verify_descriptor_created_in_a_compatible_env(process_bundle_descriptor): # type: (beam_fn_api_pb2.ProcessBundleDescriptor) -> None @@ -835,7 +852,7 @@ def _verify_descriptor_created_in_a_compatible_env(process_bundle_descriptor): env = process_bundle_descriptor.environments[t.environment_id] for c in env.capabilities: if (c.startswith(environments.SDK_VERSION_CAPABILITY_PREFIX) and - c != runtime_sdk): + not _environments_compatible(c, runtime_sdk)): raise RuntimeError( "Pipeline construction environment and pipeline runtime " "environment are not compatible. If you use a custom " @@ -897,39 +914,11 @@ def __init__(self, 'fnapi-step-%s' % self.process_bundle_descriptor.id, self.counter_factory) - if self.data_sampler: - self.add_data_sampling_operations(process_bundle_descriptor) - self.ops = self.create_execution_tree(self.process_bundle_descriptor) for op in reversed(self.ops.values()): - op.setup() + op.setup(self.data_sampler) self.splitting_lock = threading.Lock() - def add_data_sampling_operations(self, pbd): - # type: (beam_fn_api_pb2.ProcessBundleDescriptor) -> None - - """Adds a DataSamplingOperation to every PCollection. - - Implementation note: the alternative to this, is to add modify each - Operation and forward a DataSampler to manually sample when an element is - processed. This gets messy very quickly and is not future-proof as new - operation types will need to be updated. This is the cleanest way of adding - new operations to the final execution tree. - """ - coder = coders.FastPrimitivesCoder() - - for pcoll_id in pbd.pcollections: - transform_id = 'synthetic-data-sampling-transform-{}'.format(pcoll_id) - transform_proto: beam_runner_api_pb2.PTransform = pbd.transforms[ - transform_id] - transform_proto.unique_name = transform_id - transform_proto.spec.urn = SYNTHETIC_DATA_SAMPLING_URN - - coder_id = pbd.pcollections[pcoll_id].coder_id - transform_proto.spec.payload = coder.encode((pcoll_id, coder_id)) - - transform_proto.inputs['None'] = pcoll_id - def create_execution_tree( self, descriptor # type: beam_fn_api_pb2.ProcessBundleDescriptor @@ -966,6 +955,12 @@ def get_operation(transform_id): for tag, pcoll_id in descriptor.transforms[transform_id].outputs.items() } + + # Initialize transform-specific state in the Data Sampler. + if self.data_sampler: + self.data_sampler.initialize_samplers( + transform_id, descriptor, transform_factory.get_coder) + return transform_factory.create_operation( transform_id, transform_consumers) @@ -1009,7 +1004,7 @@ def process_bundle(self, instruction_id): expected_input_ops.append(op) try: - execution_context = ExecutionContext() + execution_context = ExecutionContext(instruction_id=instruction_id) self.current_instruction_id = instruction_id self.state_sampler.start() # Start all operations. @@ -1204,10 +1199,18 @@ def shutdown(self): op.teardown() -class ExecutionContext(object): - def __init__(self): - self.delayed_applications = [ - ] # type: List[Tuple[operations.DoOperation, common.SplitResultResidual]] +@dataclass +class ExecutionContext: + # Any splits to be processed later. + delayed_applications: List[Tuple[operations.DoOperation, + common.SplitResultResidual]] = field( + default_factory=list) + + # The exception sampler for the currently executing PTransform. + output_sampler: Optional[data_sampler.OutputSampler] = None + + # The current instruction being executed. + instruction_id: Optional[str] = None class BeamTransformFactory(object): @@ -1987,52 +1990,3 @@ def process(self, element): return _create_simple_pardo_operation( factory, transform_id, transform_proto, consumers, ToString()) - - -class DataSamplingOperation(operations.Operation): - """Operation that samples incoming elements.""" - - def __init__( - self, - name_context, # type: common.NameContext - counter_factory, # type: counters.CounterFactory - state_sampler, # type: statesampler.StateSampler - pcoll_id, # type: str - sample_coder, # type: coders.Coder - data_sampler, # type: data_sampler.DataSampler - ): - # type: (...) -> None - super().__init__(name_context, None, counter_factory, state_sampler) - self._coder = sample_coder # type: coders.Coder - self._pcoll_id = pcoll_id # type: str - - self._sampler: OutputSampler = data_sampler.sample_output( - self._pcoll_id, sample_coder) - - def process(self, windowed_value): - # type: (windowed_value.WindowedValue) -> None - self._sampler.sample(windowed_value) - - -@BeamTransformFactory.register_urn(SYNTHETIC_DATA_SAMPLING_URN, (bytes)) -def create_data_sampling_op( - factory, # type: BeamTransformFactory - transform_id, # type: str - transform_proto, # type: beam_runner_api_pb2.PTransform - pcoll_and_coder_id, # type: bytes - consumers, # type: Dict[str, List[operations.Operation]] -): - # Creating this operation should only occur when data sampling is enabled. - data_sampler = factory.data_sampler - assert data_sampler is not None - - coder = coders.FastPrimitivesCoder() - pcoll_id, coder_id = coder.decode(pcoll_and_coder_id) - return DataSamplingOperation( - common.NameContext(transform_proto.unique_name, transform_id), - factory.counter_factory, - factory.state_sampler, - pcoll_id, - factory.get_coder(coder_id), - data_sampler, - ) diff --git a/sdks/python/apache_beam/runners/worker/bundle_processor_test.py b/sdks/python/apache_beam/runners/worker/bundle_processor_test.py index 6b21071a87505..292b8431063c0 100644 --- a/sdks/python/apache_beam/runners/worker/bundle_processor_test.py +++ b/sdks/python/apache_beam/runners/worker/bundle_processor_test.py @@ -20,12 +20,13 @@ import unittest +import apache_beam as beam from apache_beam.coders.coders import FastPrimitivesCoder from apache_beam.portability import common_urns from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.runners import common +from apache_beam.runners.worker import bundle_processor from apache_beam.runners.worker import operations -from apache_beam.runners.worker.bundle_processor import SYNTHETIC_DATA_SAMPLING_URN from apache_beam.runners.worker.bundle_processor import BeamTransformFactory from apache_beam.runners.worker.bundle_processor import BundleProcessor from apache_beam.runners.worker.bundle_processor import DataInputOperation @@ -190,18 +191,25 @@ def element_split(frac, index): class TestOperation(operations.Operation): """Test operation that forwards its payload to consumers.""" class Spec: - def __init__(self): - self.output_coders = [FastPrimitivesCoder()] + def __init__(self, transform_proto): + self.output_coders = [ + FastPrimitivesCoder() for _ in transform_proto.outputs + ] def __init__( self, + transform_proto, name_context, counter_factory, state_sampler, consumers, payload, ): - super().__init__(name_context, self.Spec(), counter_factory, state_sampler) + super().__init__( + name_context, + self.Spec(transform_proto), + counter_factory, + state_sampler) self.payload = payload for _, consumer_ops in consumers.items(): @@ -212,8 +220,9 @@ def start(self): super().start() # Not using windowing logic, so just using simple defaults here. - self.process( - WindowedValue(self.payload, timestamp=0, windows=[GlobalWindow()])) + if self.payload: + self.process( + WindowedValue(self.payload, timestamp=0, windows=[GlobalWindow()])) def process(self, windowed_value): self.output(windowed_value) @@ -222,6 +231,7 @@ def process(self, windowed_value): @BeamTransformFactory.register_urn('beam:internal:testop:v1', bytes) def create_test_op(factory, transform_id, transform_proto, payload, consumers): return TestOperation( + transform_proto, common.NameContext(transform_proto.unique_name, transform_id), factory.counter_factory, factory.state_sampler, @@ -229,6 +239,25 @@ def create_test_op(factory, transform_id, transform_proto, payload, consumers): payload) +@BeamTransformFactory.register_urn('beam:internal:testexn:v1', bytes) +def create_exception_dofn( + factory, transform_id, transform_proto, payload, consumers): + """Returns a test DoFn that raises the given exception.""" + class RaiseException(beam.DoFn): + def __init__(self, msg): + self.msg = msg.decode() + + def process(self, _): + raise RuntimeError(self.msg) + + return bundle_processor._create_simple_pardo_operation( + factory, + transform_id, + transform_proto, + consumers, + RaiseException(payload)) + + class DataSamplingTest(unittest.TestCase): def test_disabled_by_default(self): """Test that not providing the sampler does not enable Data Sampling. @@ -241,47 +270,14 @@ def test_disabled_by_default(self): _ = BundleProcessor(descriptor, None, None) self.assertEqual(len(descriptor.transforms), 0) - def test_adds_data_sampling_operations(self): - """Test that providing the sampler creates sampling PTransforms. - - Data sampling is implemented by modifying the ProcessBundleDescriptor with - additional sampling PTransforms reading from each PCllection. - """ - data_sampler = DataSampler() - - # Data sampling samples the PCollections, which adds a PTransform to read - # from each PCollection. So add a simple PCollection here to create the - # DataSamplingOperation. - PCOLLECTION_ID = 'pc' - CODER_ID = 'c' - descriptor = beam_fn_api_pb2.ProcessBundleDescriptor() - descriptor.pcollections[PCOLLECTION_ID].unique_name = PCOLLECTION_ID - descriptor.pcollections[PCOLLECTION_ID].coder_id = CODER_ID - descriptor.coders[ - CODER_ID].spec.urn = common_urns.StandardCoders.Enum.BYTES.urn - - _ = BundleProcessor(descriptor, None, None, data_sampler=data_sampler) - - # Assert that the data sampling transform was created. - self.assertEqual(len(descriptor.transforms), 1) - sampling_transform = list(descriptor.transforms.values())[0] - - # Ensure that the data sampling transform has the correct spec and that it's - # sampling the correct PCollection. - self.assertEqual( - sampling_transform.unique_name, 'synthetic-data-sampling-transform-pc') - self.assertEqual(sampling_transform.spec.urn, SYNTHETIC_DATA_SAMPLING_URN) - self.assertEqual(sampling_transform.inputs, {'None': PCOLLECTION_ID}) - def test_can_sample(self): """Test that elements are sampled. This is a small integration test with the BundleProcessor and the - DataSampler. It ensures that the BundleProcessor correctly makes - DataSamplingOperations and samples are taken from in-flight elements. These - elements are then finally queried. + DataSampler. It ensures that samples are taken from in-flight elements. + These elements are then finally queried. """ - data_sampler = DataSampler() + data_sampler = DataSampler(sample_every_sec=0.1) descriptor = beam_fn_api_pb2.ProcessBundleDescriptor() # Create the PCollection to sample from. @@ -295,18 +291,135 @@ def test_can_sample(self): # Add a simple transform to inject an element into the data sampler. This # doesn't use the FnApi, so this uses a simple operation to forward its # payload to consumers. - test_transform = descriptor.transforms['test_transform'] + TRANSFORM_ID = 'test_transform' + test_transform = descriptor.transforms[TRANSFORM_ID] test_transform.outputs['None'] = PCOLLECTION_ID test_transform.spec.urn = 'beam:internal:testop:v1' test_transform.spec.payload = b'hello, world!' - # Create and process a fake bundle. The instruction id doesn't matter here. - processor = BundleProcessor( - descriptor, None, None, data_sampler=data_sampler) - processor.process_bundle('instruction_id') + try: + # Create and process a fake bundle. The instruction id doesn't matter + # here. + processor = BundleProcessor( + descriptor, None, None, data_sampler=data_sampler) + processor.process_bundle('instruction_id') + + samples = data_sampler.wait_for_samples([PCOLLECTION_ID]) + expected = beam_fn_api_pb2.SampleDataResponse( + element_samples={ + PCOLLECTION_ID: beam_fn_api_pb2.SampleDataResponse.ElementList( + elements=[ + beam_fn_api_pb2.SampledElement( + element=b'\rhello, world!') + ]) + }) + self.assertEqual(samples, expected) + finally: + data_sampler.stop() + + def test_can_sample_exceptions(self): + """Test that exceptions are sampled.""" + data_sampler = DataSampler(sample_every_sec=0.1) + descriptor = beam_fn_api_pb2.ProcessBundleDescriptor() - self.assertEqual( - data_sampler.samples(), {PCOLLECTION_ID: [b'\rhello, world!']}) + # Boiler plate for the DoFn. + WINDOWING_ID = 'window' + WINDOW_CODER_ID = 'cw' + window = descriptor.windowing_strategies[WINDOWING_ID] + window.window_fn.urn = common_urns.global_windows.urn + window.window_coder_id = WINDOW_CODER_ID + window.trigger.default.SetInParent() + window_coder = descriptor.coders[WINDOW_CODER_ID] + window_coder.spec.urn = common_urns.StandardCoders.Enum.GLOBAL_WINDOW.urn + + # Input collection to the exception raising DoFn. + INPUT_PCOLLECTION_ID = 'pc-in' + INPUT_CODER_ID = 'c-in' + descriptor.pcollections[ + INPUT_PCOLLECTION_ID].unique_name = INPUT_PCOLLECTION_ID + descriptor.pcollections[INPUT_PCOLLECTION_ID].coder_id = INPUT_CODER_ID + descriptor.pcollections[ + INPUT_PCOLLECTION_ID].windowing_strategy_id = WINDOWING_ID + descriptor.coders[ + INPUT_CODER_ID].spec.urn = common_urns.StandardCoders.Enum.BYTES.urn + + # Output collection to the exception raising DoFn. Because the transform + # "failed" to process the input element, we do NOT expect to see a sample in + # this PCollection. + OUTPUT_PCOLLECTION_ID = 'pc-out' + OUTPUT_CODER_ID = 'c-out' + descriptor.pcollections[ + OUTPUT_PCOLLECTION_ID].unique_name = OUTPUT_PCOLLECTION_ID + descriptor.pcollections[OUTPUT_PCOLLECTION_ID].coder_id = OUTPUT_CODER_ID + descriptor.pcollections[ + OUTPUT_PCOLLECTION_ID].windowing_strategy_id = WINDOWING_ID + descriptor.coders[ + OUTPUT_CODER_ID].spec.urn = common_urns.StandardCoders.Enum.BYTES.urn + + # Add a simple transform to inject an element into the data sampler. This + # doesn't use the FnApi, so this uses a simple operation to forward its + # payload to consumers. + TEST_OP_TRANSFORM_ID = 'test_op' + test_transform = descriptor.transforms[TEST_OP_TRANSFORM_ID] + test_transform.outputs['None'] = INPUT_PCOLLECTION_ID + test_transform.spec.urn = 'beam:internal:testop:v1' + test_transform.spec.payload = b'hello, world!' + + # Add the DoFn to create an exception to sample from. + TEST_EXCEPTION_TRANSFORM_ID = 'test_transform' + test_transform = descriptor.transforms[TEST_EXCEPTION_TRANSFORM_ID] + test_transform.inputs['0'] = INPUT_PCOLLECTION_ID + test_transform.outputs['None'] = OUTPUT_PCOLLECTION_ID + test_transform.spec.urn = 'beam:internal:testexn:v1' + test_transform.spec.payload = b'expected exception' + + try: + # Create and process a fake bundle. The instruction id doesn't matter + # here. + processor = BundleProcessor( + descriptor, None, None, data_sampler=data_sampler) + + with self.assertRaisesRegex(RuntimeError, 'expected exception'): + processor.process_bundle('instruction_id') + + # NOTE: The expected sample comes from the input PCollection. This is very + # important because there can be coder issues if the sample is put in the + # wrong PCollection. + samples = data_sampler.wait_for_samples([INPUT_PCOLLECTION_ID]) + self.assertEqual(len(samples.element_samples), 1) + + element = samples.element_samples[INPUT_PCOLLECTION_ID].elements[0] + self.assertEqual(element.element, b'\rhello, world!') + self.assertTrue(element.HasField('exception')) + + exception = element.exception + self.assertEqual(exception.instruction_id, 'instruction_id') + self.assertEqual(exception.transform_id, TEST_EXCEPTION_TRANSFORM_ID) + self.assertRegex( + exception.error, 'Traceback(\n|.)*RuntimeError: expected exception') + + finally: + data_sampler.stop() + + +class EnvironmentCompatibilityTest(unittest.TestCase): + def test_rc_environments_are_compatible_with_released_images(self): + # TODO(https://github.com/apache/beam/issues/28084): remove when + # resolved. + self.assertTrue( + bundle_processor._environments_compatible( + "beam:version:sdk_base:apache/beam_python3.5_sdk:2.1.0rc1", + "beam:version:sdk_base:apache/beam_python3.5_sdk:2.1.0")) + + def test_user_modified_sdks_need_to_be_installed_in_runtime_env(self): + self.assertFalse( + bundle_processor._environments_compatible( + "beam:version:sdk_base:apache/beam_python3.5_sdk:2.1.0-custom", + "beam:version:sdk_base:apache/beam_python3.5_sdk:2.1.0")) + self.assertTrue( + bundle_processor._environments_compatible( + "beam:version:sdk_base:apache/beam_python3.5_sdk:2.1.0-custom", + "beam:version:sdk_base:apache/beam_python3.5_sdk:2.1.0-custom")) if __name__ == '__main__': diff --git a/sdks/python/apache_beam/runners/worker/data_sampler.py b/sdks/python/apache_beam/runners/worker/data_sampler.py index 7cc8152693da4..5ca307ca1b376 100644 --- a/sdks/python/apache_beam/runners/worker/data_sampler.py +++ b/sdks/python/apache_beam/runners/worker/data_sampler.py @@ -19,43 +19,119 @@ # pytype: skip-file +from __future__ import annotations + import collections +import logging import threading import time +import traceback +from dataclasses import dataclass +from threading import Timer from typing import Any -from typing import DefaultDict from typing import Deque from typing import Dict from typing import Iterable from typing import List from typing import Optional +from typing import Tuple from typing import Union from apache_beam.coders.coder_impl import CoderImpl from apache_beam.coders.coder_impl import WindowedValueCoderImpl from apache_beam.coders.coders import Coder +from apache_beam.options.pipeline_options import DebugOptions +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.utils.windowed_value import WindowedValue +_LOGGER = logging.getLogger(__name__) + + +class SampleTimer: + """Periodic timer for sampling elements.""" + def __init__(self, timeout_secs: float, sampler: OutputSampler) -> None: + self._target_timeout_secs = timeout_secs + self._timeout_secs = min(timeout_secs, 0.5) if timeout_secs > 0 else 0.0 + self._timer = Timer(self._timeout_secs, self.sample) + self._sampler = sampler + self._sample_duration_secs = 0.0 + + def reset(self) -> None: + # For the first 30 seconds, sample every 0.5 seconds. After that, sample at + # the normal rate. + if self._sample_duration_secs >= 30.0: + self._timeout_secs = self._target_timeout_secs + self._sample_duration_secs += self._timeout_secs + + self._timer.cancel() + self._timer = Timer(self._timeout_secs, self.sample) + self._timer.start() + + def stop(self) -> None: + self._timer.cancel() + + def sample(self) -> None: + self._sampler.sample() + self.reset() + + +@dataclass +class ExceptionMetadata: + # The repr-ified Exception. + msg: str + + # The transform where the exception occured. + transform_id: str + + # The instruction when the exception occured. + instruction_id: str + + +@dataclass +class ElementSampler: + """Record class to hold sampled elements. + + This class is used as an optimization to quickly sample elements. This is a + shared reference between the Operation and the OutputSampler. + """ + + # Is true iff the `el` has been set with a sample. + has_element: bool = False + + # The sampled element. Note that `None` is a valid element and cannot be uesd + # as a sentintel to check if there is a sample. Use the `has_element` flag to + # check for this case. + el: Any = None + class OutputSampler: """Represents a way to sample an output of a PTransform. - This is configurable to only keep max_samples (see constructor) sampled - elements in memory. The first 10 elements are always sampled, then after each - sample_every_sec (see constructor). + This is configurable to only keep `max_samples` (see constructor) sampled + elements in memory. Samples are taken every `sample_every_sec`. """ def __init__( self, coder: Coder, max_samples: int = 10, - sample_every_sec: float = 30, - clock=None) -> None: + sample_every_sec: float = 5) -> None: self._samples: Deque[Any] = collections.deque(maxlen=max_samples) + self._samples_lock: threading.Lock = threading.Lock() self._coder_impl: CoderImpl = coder.get_impl() - self._sample_count: int = 0 - self._sample_every_sec: float = sample_every_sec - self._clock = clock - self._last_sample_sec: float = self.time() + self._sample_timer = SampleTimer(sample_every_sec, self) + self.element_sampler = ElementSampler() + self.element_sampler.has_element = False + self._exceptions: Deque[Tuple[Any, ExceptionMetadata]] = collections.deque( + maxlen=max_samples) + + # For testing, it's easier to disable the Timer and manually sample. + if sample_every_sec > 0: + self._sample_timer.reset() + + def stop(self) -> None: + """Stops sampling.""" + self._sample_timer.stop() def remove_windowed_value(self, el: Union[WindowedValue, Any]) -> Any: """Retrieves the value from the WindowedValue. @@ -63,39 +139,68 @@ def remove_windowed_value(self, el: Union[WindowedValue, Any]) -> Any: The Python SDK passes elements as WindowedValues, which may not match the coder for that particular PCollection. """ - if isinstance(el, WindowedValue): - return self.remove_windowed_value(el.value) + while isinstance(el, WindowedValue): + el = el.value return el - def time(self) -> float: - """Returns the current time. Used for mocking out the clock for testing.""" - return self._clock.time() if self._clock else time.time() + def flush(self, clear: bool = True) -> List[beam_fn_api_pb2.SampledElement]: + """Returns all samples and optionally clears buffer if clear is True.""" + with self._samples_lock: + # TODO(rohdesamuel): There can duplicates between the exceptions and + # samples. This happens when the OutputSampler samples during an + # exception. The fix is to create a OutputSampler per process bundle. + # Until then use a set to keep track of the elements. + seen = set(id(el) for el, _ in self._exceptions) + if isinstance(self._coder_impl, WindowedValueCoderImpl): + exceptions = [s for s in self._exceptions] + samples = [s for s in self._samples if id(s) not in seen] + else: + exceptions = [ + (self.remove_windowed_value(a), b) for a, b in self._exceptions + ] + samples = [ + self.remove_windowed_value(s) for s in self._samples + if id(s) not in seen + ] - def flush(self) -> List[bytes]: - """Returns all samples and clears buffer.""" - if isinstance(self._coder_impl, WindowedValueCoderImpl): - samples = [s for s in self._samples] - else: - samples = [self.remove_windowed_value(s) for s in self._samples] + # Encode in the nested context b/c this ensures that the SDK can decode + # the bytes with the ToStringFn. + if clear: + self._samples.clear() + self._exceptions.clear() - # Encode in the nested context b/c this ensures that the SDK can decode the - # bytes with the ToStringFn. - self._samples.clear() - return [self._coder_impl.encode_nested(s) for s in samples] + ret = [ + beam_fn_api_pb2.SampledElement( + element=self._coder_impl.encode_nested(s), + ) for s in samples + ] - def sample(self, element: Any) -> None: - """Samples the given element to an internal buffer. + ret.extend( + beam_fn_api_pb2.SampledElement( + element=self._coder_impl.encode_nested(s), + exception=beam_fn_api_pb2.SampledElement.Exception( + instruction_id=exn.instruction_id, + transform_id=exn.transform_id, + error=exn.msg)) for s, + exn in exceptions) - Samples are only taken for the first 10 elements then every - `self._sample_every_sec` second after. - """ - self._sample_count += 1 - now = self.time() - sample_diff = now - self._last_sample_sec + return ret + + def sample(self) -> None: + """Samples the given element to an internal buffer.""" + with self._samples_lock: + if self.element_sampler.has_element: + self._samples.append(self.element_sampler.el) + self.element_sampler.has_element = False - if self._sample_count <= 10 or sample_diff >= self._sample_every_sec: - self._samples.append(element) - self._last_sample_sec = now + def sample_exception( + self, el: Any, exc_info: Any, transform_id: str, + instruction_id: str) -> None: + """Adds the given exception to the samples.""" + with self._samples_lock: + err_string = ''.join(traceback.format_exception(*exc_info)) + self._exceptions.append( + (el, ExceptionMetadata(err_string, transform_id, instruction_id))) class DataSampler: @@ -103,43 +208,125 @@ class DataSampler: This class is meant to be a singleton with regard to a particular `sdk_worker.SdkHarness`. When creating the operators, individual - `OutputSampler`s are created from `DataSampler.sample_output`. This allows for - multi-threaded sampling of a PCollection across the SdkHarness. + `OutputSampler`s are created from `DataSampler.initialize_samplers`. This + allows for multi-threaded sampling of a PCollection across the SdkHarness. Samples generated during execution can then be sampled with the `samples` method. This filters samples from the given pcollection ids. """ def __init__( - self, max_samples: int = 10, sample_every_sec: float = 30) -> None: + self, + max_samples: int = 10, + sample_every_sec: float = 30, + sample_only_exceptions: bool = False, + clock=None) -> None: # Key is PCollection id. Is guarded by the _samplers_lock. self._samplers: Dict[str, OutputSampler] = {} # Bundles are processed in parallel, so new samplers may be added when the # runner queries for samples. self._samplers_lock: threading.Lock = threading.Lock() self._max_samples = max_samples - self._sample_every_sec = sample_every_sec + self._sample_every_sec = 0.0 if sample_only_exceptions else sample_every_sec + self._samplers_by_output: Dict[str, List[OutputSampler]] = {} + self._clock = clock + + _ENABLE_DATA_SAMPLING = 'enable_data_sampling' + _ENABLE_ALWAYS_ON_EXCEPTION_SAMPLING = 'enable_always_on_exception_sampling' + _DISABLE_ALWAYS_ON_EXCEPTION_SAMPLING = 'disable_always_on_exception_sampling' + + @staticmethod + def create(sdk_pipeline_options: PipelineOptions, **kwargs): + experiments = sdk_pipeline_options.view_as(DebugOptions).experiments or [] + + # When true, enables only the sampling of exceptions. + always_on_exception_sampling = ( + DataSampler._ENABLE_ALWAYS_ON_EXCEPTION_SAMPLING in experiments and + DataSampler._DISABLE_ALWAYS_ON_EXCEPTION_SAMPLING not in experiments) - def sample_output(self, pcoll_id: str, coder: Coder) -> OutputSampler: - """Create or get an OutputSampler for a pcoll_id.""" + # When true, enables the sampling of all PCollections and exceptions. + enable_data_sampling = DataSampler._ENABLE_DATA_SAMPLING in experiments + + if enable_data_sampling or always_on_exception_sampling: + sample_only_exceptions = ( + always_on_exception_sampling and not enable_data_sampling) + return DataSampler( + sample_only_exceptions=sample_only_exceptions, **kwargs) + else: + return None + + def stop(self) -> None: + """Stops all sampling, does not clear samplers in case there are outstanding + samples. + """ with self._samplers_lock: - if pcoll_id in self._samplers: - sampler = self._samplers[pcoll_id] - else: + for sampler in self._samplers.values(): + sampler.stop() + + def sampler_for_output(self, transform_id: str, + output_index: int) -> Optional[OutputSampler]: + """Returns the OutputSampler for the given output.""" + try: + with self._samplers_lock: + outputs = self._samplers_by_output[transform_id] + return outputs[output_index] + except KeyError: + _LOGGER.warning( + f'Out-of-bounds access for transform "{transform_id}" ' + + 'and output "{output_index}" OutputSampler. This may ' + + 'indicate that the transform was improperly ' + + 'initialized with the DataSampler.') + return None + + def initialize_samplers( + self, + transform_id: str, + descriptor: beam_fn_api_pb2.ProcessBundleDescriptor, + coder_factory) -> List[OutputSampler]: + """Creates the OutputSamplers for the given PTransform. + + This initializes the samplers only once per PCollection Id. Note that an + OutputSampler is created per PCollection and an ElementSampler is created + per OutputSampler. This means that multiple ProcessBundles can and will + share the same ElementSampler for a given PCollection. + """ + transform_proto = descriptor.transforms[transform_id] + with self._samplers_lock: + if transform_id in self._samplers_by_output: + return self._samplers_by_output[transform_id] + + # Initialize the samplers. + for pcoll_id in transform_proto.outputs.values(): + # Only initialize new PCollections. + if pcoll_id in self._samplers: + continue + + # Create the sampler with the corresponding coder. + coder_id = descriptor.pcollections[pcoll_id].coder_id + coder = coder_factory(coder_id) sampler = OutputSampler( coder, self._max_samples, self._sample_every_sec) self._samplers[pcoll_id] = sampler - return sampler + + # Next update the lookup table for ElementSamplers for a given PTransform. + # Operations look up the ElementSampler for an output based on the index + # of the tag in the PTransform's outputs. The following code intializes + # the array with ElementSamplers in the correct indices. + outputs = transform_proto.outputs + samplers = [self._samplers[pcoll_id] for pcoll_id in outputs.values()] + self._samplers_by_output[transform_id] = samplers + + return samplers def samples( self, pcollection_ids: Optional[Iterable[str]] = None - ) -> Dict[str, List[bytes]]: + ) -> beam_fn_api_pb2.SampleDataResponse: """Returns samples filtered PCollection ids. All samples from the given PCollections are returned. Empty lists are wildcards. """ - ret: DefaultDict[str, List[bytes]] = collections.defaultdict(lambda: []) + ret = beam_fn_api_pb2.SampleDataResponse() with self._samplers_lock: samplers = self._samplers.copy() @@ -150,6 +337,28 @@ def samples( samples = samplers[pcoll_id].flush() if samples: - ret[pcoll_id].extend(samples) + ret.element_samples[pcoll_id].elements.extend(samples) + + return ret + + def wait_for_samples( + self, pcollection_ids: List[str]) -> beam_fn_api_pb2.SampleDataResponse: + """Waits for samples to exist for the given PCollections (only testing).""" + now = time.time() + end = now + 30 + + samples = beam_fn_api_pb2.SampleDataResponse() + while now < end: + time.sleep(0.1) + now = time.time() + samples.MergeFrom(self.samples(pcollection_ids)) + + if not samples: + continue + + has_all = all( + pcoll_id in samples.element_samples for pcoll_id in pcollection_ids) + if has_all: + break - return dict(ret) + return samples diff --git a/sdks/python/apache_beam/runners/worker/data_sampler_test.py b/sdks/python/apache_beam/runners/worker/data_sampler_test.py index 6a163c83b2857..8c47315b7a9e4 100644 --- a/sdks/python/apache_beam/runners/worker/data_sampler_test.py +++ b/sdks/python/apache_beam/runners/worker/data_sampler_test.py @@ -17,159 +17,518 @@ # pytype: skip-file +import sys +import time +import traceback import unittest +from typing import Any +from typing import List +from typing import Optional from apache_beam.coders import FastPrimitivesCoder from apache_beam.coders import WindowedValueCoder -from apache_beam.coders.coders import Coder +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.runners.worker.data_sampler import DataSampler from apache_beam.runners.worker.data_sampler import OutputSampler from apache_beam.transforms.window import GlobalWindow from apache_beam.utils.windowed_value import WindowedValue +MAIN_TRANSFORM_ID = 'transform' +MAIN_PCOLLECTION_ID = 'pcoll' +PRIMITIVES_CODER = FastPrimitivesCoder() + class DataSamplerTest(unittest.TestCase): + def make_test_descriptor( + self, + outputs: Optional[List[str]] = None, + transforms: Optional[List[str]] = None + ) -> beam_fn_api_pb2.ProcessBundleDescriptor: + outputs = outputs or [MAIN_PCOLLECTION_ID] + transforms = transforms or [MAIN_TRANSFORM_ID] + + descriptor = beam_fn_api_pb2.ProcessBundleDescriptor() + for transform_id in transforms: + transform = descriptor.transforms[transform_id] + for output in outputs: + transform.outputs[output] = output + + return descriptor + + def setUp(self): + self.data_sampler = DataSampler.create( + PipelineOptions(experiments=[DataSampler._ENABLE_DATA_SAMPLING]), + sample_every_sec=0.1) + + def tearDown(self): + self.data_sampler.stop() + + def primitives_coder_factory(self, _): + return PRIMITIVES_CODER + + def gen_sample( + self, + data_sampler: DataSampler, + element: Any, + output_index: int, + transform_id: str = MAIN_TRANSFORM_ID): + """Generates a sample for the given transform's output.""" + element_sampler = self.data_sampler.sampler_for_output( + transform_id, output_index).element_sampler + element_sampler.el = element + element_sampler.has_element = True + def test_single_output(self): """Simple test for a single sample.""" - data_sampler = DataSampler() - coder = FastPrimitivesCoder() + descriptor = self.make_test_descriptor() + self.data_sampler.initialize_samplers( + MAIN_TRANSFORM_ID, descriptor, self.primitives_coder_factory) + + self.gen_sample(self.data_sampler, 'a', output_index=0) + + expected_sample = beam_fn_api_pb2.SampleDataResponse( + element_samples={ + MAIN_PCOLLECTION_ID: beam_fn_api_pb2.SampleDataResponse.ElementList( + elements=[ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested('a')) + ]) + }) + samples = self.data_sampler.wait_for_samples([MAIN_PCOLLECTION_ID]) + self.assertEqual(samples, expected_sample) + + def test_not_initialized(self): + """Tests that transforms fail gracefully if not properly initialized.""" + with self.assertLogs() as cm: + self.data_sampler.sampler_for_output(MAIN_TRANSFORM_ID, 0) + self.assertRegex(cm.output[0], 'Out-of-bounds access.*') + + def map_outputs_to_indices( + self, outputs, descriptor, transform_id=MAIN_TRANSFORM_ID): + tag_list = list(descriptor.transforms[transform_id].outputs) + return {output: tag_list.index(output) for output in outputs} + + def test_sampler_mapping(self): + """Tests that the ElementSamplers are created for the correct output.""" + # Initialize the DataSampler with the following outputs. The order here may + # get shuffled when inserting into the descriptor. + pcollection_ids = ['o0', 'o1', 'o2'] + descriptor = self.make_test_descriptor(outputs=pcollection_ids) + samplers = self.data_sampler.initialize_samplers( + MAIN_TRANSFORM_ID, descriptor, self.primitives_coder_factory) + + # Create a map from the PCollection id to the index into the transform + # output. This mirrors what happens when operators are created. The index of + # an output is where in the PTransform.outputs it is located (when the map + # is converted to a list). + outputs = self.map_outputs_to_indices(pcollection_ids, descriptor) + + # Assert that the mapping is correct, i.e. that we can go from the + # PCollection id -> output index and that this is the same as the created + # samplers. + index = outputs['o0'] + self.assertEqual( + self.data_sampler.sampler_for_output(MAIN_TRANSFORM_ID, + index).element_sampler, + samplers[index].element_sampler) - output_sampler = data_sampler.sample_output('1', coder) - output_sampler.sample('a') + index = outputs['o1'] + self.assertEqual( + self.data_sampler.sampler_for_output(MAIN_TRANSFORM_ID, + index).element_sampler, + samplers[index].element_sampler) - self.assertEqual(data_sampler.samples(), {'1': [coder.encode_nested('a')]}) + index = outputs['o2'] + self.assertEqual( + self.data_sampler.sampler_for_output(MAIN_TRANSFORM_ID, + index).element_sampler, + samplers[index].element_sampler) def test_multiple_outputs(self): """Tests that multiple PCollections have their own sampler.""" - data_sampler = DataSampler() - coder = FastPrimitivesCoder() - - data_sampler.sample_output('1', coder).sample('a') - data_sampler.sample_output('2', coder).sample('a') - - self.assertEqual( - data_sampler.samples(), { - '1': [coder.encode_nested('a')], '2': [coder.encode_nested('a')] + pcollection_ids = ['o0', 'o1', 'o2'] + descriptor = self.make_test_descriptor(outputs=pcollection_ids) + outputs = self.map_outputs_to_indices(pcollection_ids, descriptor) + + self.data_sampler.initialize_samplers( + MAIN_TRANSFORM_ID, descriptor, self.primitives_coder_factory) + + self.gen_sample(self.data_sampler, 'a', output_index=outputs['o0']) + self.gen_sample(self.data_sampler, 'b', output_index=outputs['o1']) + self.gen_sample(self.data_sampler, 'c', output_index=outputs['o2']) + + samples = self.data_sampler.wait_for_samples(['o0', 'o1', 'o2']) + expected_samples = beam_fn_api_pb2.SampleDataResponse( + element_samples={ + 'o0': beam_fn_api_pb2.SampleDataResponse.ElementList( + elements=[ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested('a')) + ]), + 'o1': beam_fn_api_pb2.SampleDataResponse.ElementList( + elements=[ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested('b')) + ]), + 'o2': beam_fn_api_pb2.SampleDataResponse.ElementList( + elements=[ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested('c')) + ]), }) + self.assertEqual(samples, expected_samples) - def gen_samples(self, data_sampler: DataSampler, coder: Coder): - data_sampler.sample_output('a', coder).sample('1') - data_sampler.sample_output('a', coder).sample('2') - data_sampler.sample_output('b', coder).sample('3') - data_sampler.sample_output('b', coder).sample('4') - data_sampler.sample_output('c', coder).sample('5') - data_sampler.sample_output('c', coder).sample('6') + def test_multiple_transforms(self): + """Test that multiple transforms with the same PCollections can be sampled. + """ + # Initialize two transform both with the same two outputs. + pcollection_ids = ['o0', 'o1'] + descriptor = self.make_test_descriptor( + outputs=pcollection_ids, transforms=['t0', 't1']) + t0_outputs = self.map_outputs_to_indices( + pcollection_ids, descriptor, transform_id='t0') + t1_outputs = self.map_outputs_to_indices( + pcollection_ids, descriptor, transform_id='t1') + + self.data_sampler.initialize_samplers( + 't0', descriptor, self.primitives_coder_factory) + + self.data_sampler.initialize_samplers( + 't1', descriptor, self.primitives_coder_factory) + + # The OutputSampler is on a different thread so we don't test the same + # PCollections to ensure that no data race occurs. + self.gen_sample( + self.data_sampler, + 'a', + output_index=t0_outputs['o0'], + transform_id='t0') + self.gen_sample( + self.data_sampler, + 'd', + output_index=t1_outputs['o1'], + transform_id='t1') + expected_samples = beam_fn_api_pb2.SampleDataResponse( + element_samples={ + 'o0': beam_fn_api_pb2.SampleDataResponse.ElementList( + elements=[ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested('a')) + ]), + 'o1': beam_fn_api_pb2.SampleDataResponse.ElementList( + elements=[ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested('d')) + ]), + }) + samples = self.data_sampler.wait_for_samples(['o0', 'o1']) + self.assertEqual(samples, expected_samples) + + self.gen_sample( + self.data_sampler, + 'b', + output_index=t0_outputs['o1'], + transform_id='t0') + self.gen_sample( + self.data_sampler, + 'c', + output_index=t1_outputs['o0'], + transform_id='t1') + expected_samples = beam_fn_api_pb2.SampleDataResponse( + element_samples={ + 'o0': beam_fn_api_pb2.SampleDataResponse.ElementList( + elements=[ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested('c')) + ]), + 'o1': beam_fn_api_pb2.SampleDataResponse.ElementList( + elements=[ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested('b')) + ]), + }) + samples = self.data_sampler.wait_for_samples(['o0', 'o1']) + self.assertEqual(samples, expected_samples) def test_sample_filters_single_pcollection_ids(self): """Tests the samples can be filtered based on a single pcollection id.""" - data_sampler = DataSampler() - coder = FastPrimitivesCoder() - - self.gen_samples(data_sampler, coder) - self.assertEqual( - data_sampler.samples(pcollection_ids=['a']), - {'a': [coder.encode_nested('1'), coder.encode_nested('2')]}) - - self.assertEqual( - data_sampler.samples(pcollection_ids=['b']), - {'b': [coder.encode_nested('3'), coder.encode_nested('4')]}) + pcollection_ids = ['o0', 'o1', 'o2'] + descriptor = self.make_test_descriptor(outputs=pcollection_ids) + outputs = self.map_outputs_to_indices(pcollection_ids, descriptor) + + self.data_sampler.initialize_samplers( + MAIN_TRANSFORM_ID, descriptor, self.primitives_coder_factory) + + self.gen_sample(self.data_sampler, 'a', output_index=outputs['o0']) + self.gen_sample(self.data_sampler, 'b', output_index=outputs['o1']) + self.gen_sample(self.data_sampler, 'c', output_index=outputs['o2']) + + samples = self.data_sampler.wait_for_samples(['o0']) + expected_samples = beam_fn_api_pb2.SampleDataResponse( + element_samples={ + 'o0': beam_fn_api_pb2.SampleDataResponse.ElementList( + elements=[ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested('a')) + ]), + }) + self.assertEqual(samples, expected_samples) + + samples = self.data_sampler.wait_for_samples(['o1']) + expected_samples = beam_fn_api_pb2.SampleDataResponse( + element_samples={ + 'o1': beam_fn_api_pb2.SampleDataResponse.ElementList( + elements=[ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested('b')) + ]), + }) + self.assertEqual(samples, expected_samples) + + samples = self.data_sampler.wait_for_samples(['o2']) + expected_samples = beam_fn_api_pb2.SampleDataResponse( + element_samples={ + 'o2': beam_fn_api_pb2.SampleDataResponse.ElementList( + elements=[ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested('c')) + ]), + }) + self.assertEqual(samples, expected_samples) def test_sample_filters_multiple_pcollection_ids(self): """Tests the samples can be filtered based on a multiple pcollection ids.""" - data_sampler = DataSampler() - coder = FastPrimitivesCoder() - - self.gen_samples(data_sampler, coder) - self.assertEqual( - data_sampler.samples(pcollection_ids=['a', 'c']), - { - 'a': [coder.encode_nested('1'), coder.encode_nested('2')], - 'c': [coder.encode_nested('5'), coder.encode_nested('6')] + pcollection_ids = ['o0', 'o1', 'o2'] + descriptor = self.make_test_descriptor(outputs=pcollection_ids) + outputs = self.map_outputs_to_indices(pcollection_ids, descriptor) + + self.data_sampler.initialize_samplers( + MAIN_TRANSFORM_ID, descriptor, self.primitives_coder_factory) + + self.gen_sample(self.data_sampler, 'a', output_index=outputs['o0']) + self.gen_sample(self.data_sampler, 'b', output_index=outputs['o1']) + self.gen_sample(self.data_sampler, 'c', output_index=outputs['o2']) + + samples = self.data_sampler.wait_for_samples(['o0', 'o2']) + expected_samples = beam_fn_api_pb2.SampleDataResponse( + element_samples={ + 'o0': beam_fn_api_pb2.SampleDataResponse.ElementList( + elements=[ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested('a')) + ]), + 'o2': beam_fn_api_pb2.SampleDataResponse.ElementList( + elements=[ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested('c')) + ]), }) + self.assertEqual(samples, expected_samples) + + def test_can_sample_exceptions(self): + """Tests that exceptions sampled can be queried by the DataSampler.""" + descriptor = self.make_test_descriptor() + self.data_sampler.initialize_samplers( + MAIN_TRANSFORM_ID, descriptor, self.primitives_coder_factory) + sampler = self.data_sampler.sampler_for_output(MAIN_TRANSFORM_ID, 0) + + exc_info = None + try: + raise Exception('test') + except Exception: + exc_info = sys.exc_info() + + sampler.sample_exception('a', exc_info, MAIN_TRANSFORM_ID, 'instid') + + samples = self.data_sampler.wait_for_samples([MAIN_PCOLLECTION_ID]) + self.assertGreater(len(samples.element_samples), 0) + + def test_create_experiments(self): + """Tests that the experiments correctly make the DataSampler.""" + enable_exception_exp = DataSampler._ENABLE_ALWAYS_ON_EXCEPTION_SAMPLING + disable_exception_exp = DataSampler._DISABLE_ALWAYS_ON_EXCEPTION_SAMPLING + enable_sampling_exp = DataSampler._ENABLE_DATA_SAMPLING + + self.assertIsNone(DataSampler.create(PipelineOptions())) + + exp = [disable_exception_exp] + self.assertIsNone(DataSampler.create(PipelineOptions(experiments=exp))) + + exp = [enable_exception_exp, disable_exception_exp] + self.assertIsNone(DataSampler.create(PipelineOptions(experiments=exp))) + + exp = [enable_exception_exp] + self.assertIsNotNone(DataSampler.create(PipelineOptions(experiments=exp))) + + exp = [enable_sampling_exp] + self.assertIsNotNone(DataSampler.create(PipelineOptions(experiments=exp))) + + exp = [enable_sampling_exp, enable_exception_exp, disable_exception_exp] + self.assertIsNotNone(DataSampler.create(PipelineOptions(experiments=exp))) + + def test_samples_all_with_both_experiments(self): + """Tests that the using both sampling experiments samples everything.""" + self.data_sampler = DataSampler.create( + PipelineOptions( + experiments=[ + DataSampler._ENABLE_DATA_SAMPLING, + DataSampler._ENABLE_ALWAYS_ON_EXCEPTION_SAMPLING + ]), + sample_every_sec=0.1) + + # Create a descriptor with one transform with two outputs, 'a' and 'b'. + descriptor = self.make_test_descriptor(outputs=['a', 'b']) + self.data_sampler.initialize_samplers( + MAIN_TRANSFORM_ID, descriptor, self.primitives_coder_factory) + + # Get the samples for the two outputs. + a_sampler = self.data_sampler.sampler_for_output(MAIN_TRANSFORM_ID, 0) + b_sampler = self.data_sampler.sampler_for_output(MAIN_TRANSFORM_ID, 1) + + # Sample an exception for the output 'a', this will show up in the final + # samples response. + exc_info = None + try: + raise Exception('test') + except Exception: + exc_info = sys.exc_info() + a_sampler.sample_exception('a', exc_info, MAIN_TRANSFORM_ID, 'instid') + + # Sample a normal element for the output 'b', this will not show up in the + # final samples response. + b_sampler.element_sampler.el = 'b' + b_sampler.element_sampler.has_element = True + + samples = self.data_sampler.wait_for_samples(['a', 'b']) + self.assertEqual(len(samples.element_samples), 2) + self.assertTrue( + samples.element_samples['a'].elements[0].HasField('exception')) + self.assertFalse( + samples.element_samples['b'].elements[0].HasField('exception')) + + def test_only_sample_exceptions(self): + """Tests that the exception sampling experiment only samples exceptions.""" + self.data_sampler = DataSampler.create( + PipelineOptions( + experiments=[DataSampler._ENABLE_ALWAYS_ON_EXCEPTION_SAMPLING]), + sample_every_sec=0.1) + + # Create a descriptor with one transform with two outputs, 'a' and 'b'. + descriptor = self.make_test_descriptor(outputs=['a', 'b']) + self.data_sampler.initialize_samplers( + MAIN_TRANSFORM_ID, descriptor, self.primitives_coder_factory) + + # Get the samples for the two outputs. + a_sampler = self.data_sampler.sampler_for_output(MAIN_TRANSFORM_ID, 0) + b_sampler = self.data_sampler.sampler_for_output(MAIN_TRANSFORM_ID, 1) + + # Sample an exception for the output 'a', this will show up in the final + # samples response. + exc_info = None + try: + raise Exception('test') + except Exception: + exc_info = sys.exc_info() + a_sampler.sample_exception('a', exc_info, MAIN_TRANSFORM_ID, 'instid') + + # Sample a normal element for the output 'b', this will not show up in the + # final samples response. + b_sampler.element_sampler.el = 'b' + b_sampler.element_sampler.has_element = True + + samples = self.data_sampler.wait_for_samples([]) + self.assertEqual(len(samples.element_samples), 1) + self.assertIsNotNone(samples.element_samples['a'].elements[0].exception) -class FakeClock: - def __init__(self): - self.clock = 0 +class OutputSamplerTest(unittest.TestCase): + def tearDown(self): + self.sampler.stop() - def time(self): - return self.clock + def wait_for_samples(self, output_sampler: OutputSampler, expected_num: int): + """Waits for the expected number of samples for the given sampler.""" + now = time.time() + end = now + 30 + while now < end: + time.sleep(0.1) + now = time.time() + samples = output_sampler.flush(clear=False) -class OutputSamplerTest(unittest.TestCase): - def setUp(self): - self.fake_clock = FakeClock() + if not samples: + continue - def control_time(self, new_time): - self.fake_clock.clock = new_time + if len(samples) == expected_num: + return samples - def test_samples_first_n(self): - """Tests that the first elements are always sampled.""" - coder = FastPrimitivesCoder() - sampler = OutputSampler(coder) + self.assertLess(now, end, 'Timed out waiting for samples') - for i in range(15): - sampler.sample(i) + def test_can_sample(self): + """Tests that the underlying timer can sample.""" + self.sampler = OutputSampler(PRIMITIVES_CODER, sample_every_sec=0.05) + element_sampler = self.sampler.element_sampler + element_sampler.el = 'a' + element_sampler.has_element = True + self.wait_for_samples(self.sampler, expected_num=1) self.assertEqual( - sampler.flush(), [coder.encode_nested(i) for i in range(10)]) + self.sampler.flush(), + [ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested('a')) + ]) def test_acts_like_circular_buffer(self): """Tests that the buffer overwrites old samples.""" - coder = FastPrimitivesCoder() - sampler = OutputSampler(coder, max_samples=2) + self.sampler = OutputSampler( + PRIMITIVES_CODER, max_samples=2, sample_every_sec=0) + element_sampler = self.sampler.element_sampler for i in range(10): - sampler.sample(i) + element_sampler.el = i + element_sampler.has_element = True + self.sampler.sample() - self.assertEqual(sampler.flush(), [coder.encode_nested(i) for i in (8, 9)]) + self.assertEqual( + self.sampler.flush(), + [ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested(i)) for i in (8, 9) + ]) - def test_samples_every_n_secs(self): - """Tests that the buffer overwrites old samples.""" - coder = FastPrimitivesCoder() - sampler = OutputSampler( - coder, max_samples=1, sample_every_sec=10, clock=self.fake_clock) + def test_samples_multiple_times(self): + """Tests that the underlying timer repeats.""" + self.sampler = OutputSampler( + PRIMITIVES_CODER, max_samples=10, sample_every_sec=0.05) # Always samples the first ten. for i in range(10): - sampler.sample(i) - self.assertEqual(sampler.flush(), [coder.encode_nested(9)]) - - # Start at t=0 - sampler.sample(10) - self.assertEqual(len(sampler.flush()), 0) - - # Still not over threshold yet. - self.control_time(9) - for i in range(100): - sampler.sample(i) - self.assertEqual(len(sampler.flush()), 0) - - # First sample after 10s. - self.control_time(10) - sampler.sample(10) - self.assertEqual(sampler.flush(), [coder.encode_nested(10)]) - - # No samples between tresholds. - self.control_time(15) - for i in range(100): - sampler.sample(i) - self.assertEqual(len(sampler.flush()), 0) - - # Second sample after 20s. - self.control_time(20) - sampler.sample(11) - self.assertEqual(sampler.flush(), [coder.encode_nested(11)]) + self.sampler.element_sampler.el = i + self.sampler.element_sampler.has_element = True + self.wait_for_samples(self.sampler, i + 1) + + self.assertEqual( + self.sampler.flush(), + [ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested(i)) for i in range(10) + ]) def test_can_sample_windowed_value(self): """Tests that values with WindowedValueCoders are sampled wholesale.""" - data_sampler = DataSampler() coder = WindowedValueCoder(FastPrimitivesCoder()) value = WindowedValue('Hello, World!', 0, [GlobalWindow()]) - data_sampler.sample_output('1', coder).sample(value) + + self.sampler = OutputSampler(coder, sample_every_sec=0) + element_sampler = self.sampler.element_sampler + element_sampler.el = value + element_sampler.has_element = True + self.sampler.sample() self.assertEqual( - data_sampler.samples(), {'1': [coder.encode_nested(value)]}) + self.sampler.flush(), + [beam_fn_api_pb2.SampledElement(element=coder.encode_nested(value))]) def test_can_sample_non_windowed_value(self): """Tests that windowed values with WindowedValueCoders sample only the @@ -179,13 +538,78 @@ def test_can_sample_non_windowed_value(self): even if the coder is not a WindowedValueCoder. In this case, the value must be retrieved from the WindowedValue to match the correct coder. """ - data_sampler = DataSampler() - coder = FastPrimitivesCoder() - data_sampler.sample_output('1', coder).sample( - WindowedValue('Hello, World!', 0, [GlobalWindow()])) + value = WindowedValue('Hello, World!', 0, [GlobalWindow()]) + + self.sampler = OutputSampler(PRIMITIVES_CODER, sample_every_sec=0) + element_sampler = self.sampler.element_sampler + element_sampler.el = value + element_sampler.has_element = True + self.sampler.sample() + + self.assertEqual( + self.sampler.flush(), + [ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested('Hello, World!')) + ]) + + def test_can_sample_exceptions(self): + """Tests that exceptions are sampled.""" + val = WindowedValue('Hello, World!', 0, [GlobalWindow()]) + exc_info = None + try: + raise Exception('test') + except Exception: + exc_info = sys.exc_info() + err_string = ''.join(traceback.format_exception(*exc_info)) + + self.sampler = OutputSampler(PRIMITIVES_CODER, sample_every_sec=0) + self.sampler.sample_exception( + el=val, exc_info=exc_info, transform_id='tid', instruction_id='instid') + + self.assertEqual( + self.sampler.flush(), + [ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested('Hello, World!'), + exception=beam_fn_api_pb2.SampledElement.Exception( + instruction_id='instid', + transform_id='tid', + error=err_string)) + ]) + + def test_can_sample_multiple_exceptions(self): + """Tests that multiple exceptions in the same PCollection are sampled.""" + exc_info = None + try: + raise Exception('test') + except Exception: + exc_info = sys.exc_info() + err_string = ''.join(traceback.format_exception(*exc_info)) + + self.sampler = OutputSampler(PRIMITIVES_CODER, sample_every_sec=0) + self.sampler.sample_exception( + el='a', exc_info=exc_info, transform_id='tid', instruction_id='instid') + + self.sampler.sample_exception( + el='b', exc_info=exc_info, transform_id='tid', instruction_id='instid') self.assertEqual( - data_sampler.samples(), {'1': [coder.encode_nested('Hello, World!')]}) + self.sampler.flush(), + [ + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested('a'), + exception=beam_fn_api_pb2.SampledElement.Exception( + instruction_id='instid', + transform_id='tid', + error=err_string)), + beam_fn_api_pb2.SampledElement( + element=PRIMITIVES_CODER.encode_nested('b'), + exception=beam_fn_api_pb2.SampledElement.Exception( + instruction_id='instid', + transform_id='tid', + error=err_string)), + ]) if __name__ == '__main__': diff --git a/sdks/python/apache_beam/runners/worker/log_handler_test.py b/sdks/python/apache_beam/runners/worker/log_handler_test.py index 4adae90edceb0..9eb9299cac398 100644 --- a/sdks/python/apache_beam/runners/worker/log_handler_test.py +++ b/sdks/python/apache_beam/runners/worker/log_handler_test.py @@ -23,17 +23,97 @@ import grpc +import apache_beam as beam +from apache_beam.coders.coders import FastPrimitivesCoder +from apache_beam.portability import common_urns from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.portability.api import beam_fn_api_pb2_grpc from apache_beam.portability.api import endpoints_pb2 +from apache_beam.runners import common from apache_beam.runners.common import NameContext +from apache_beam.runners.worker import bundle_processor from apache_beam.runners.worker import log_handler +from apache_beam.runners.worker import operations from apache_beam.runners.worker import statesampler +from apache_beam.runners.worker.bundle_processor import BeamTransformFactory +from apache_beam.runners.worker.bundle_processor import BundleProcessor +from apache_beam.transforms.window import GlobalWindow from apache_beam.utils import thread_pool_executor +from apache_beam.utils.windowed_value import WindowedValue _LOGGER = logging.getLogger(__name__) +@BeamTransformFactory.register_urn('beam:internal:testexn:v1', bytes) +def create_exception_dofn( + factory, transform_id, transform_proto, payload, consumers): + """Returns a test DoFn that raises the given exception.""" + class RaiseException(beam.DoFn): + def __init__(self, msg): + self.msg = msg.decode() + + def process(self, _): + raise RuntimeError(self.msg) + + return bundle_processor._create_simple_pardo_operation( + factory, + transform_id, + transform_proto, + consumers, + RaiseException(payload)) + + +class TestOperation(operations.Operation): + """Test operation that forwards its payload to consumers.""" + class Spec: + def __init__(self, transform_proto): + self.output_coders = [ + FastPrimitivesCoder() for _ in transform_proto.outputs + ] + + def __init__( + self, + transform_proto, + name_context, + counter_factory, + state_sampler, + consumers, + payload, + ): + super().__init__( + name_context, + self.Spec(transform_proto), + counter_factory, + state_sampler) + self.payload = payload + + for _, consumer_ops in consumers.items(): + for consumer in consumer_ops: + self.add_receiver(consumer, 0) + + def start(self): + super().start() + + # Not using windowing logic, so just using simple defaults here. + if self.payload: + self.process( + WindowedValue(self.payload, timestamp=0, windows=[GlobalWindow()])) + + def process(self, windowed_value): + self.output(windowed_value) + + +@BeamTransformFactory.register_urn('beam:internal:testop:v1', bytes) +def create_test_op(factory, transform_id, transform_proto, payload, consumers): + return TestOperation( + transform_proto, + common.NameContext(transform_proto.unique_name, transform_id), + factory.counter_factory, + factory.state_sampler, + consumers, + payload) + + class BeamFnLoggingServicer(beam_fn_api_pb2_grpc.BeamFnLoggingServicer): def __init__(self): self.log_records_received = [] @@ -153,6 +233,77 @@ def test_context(self): finally: statesampler.set_current_tracker(None) + def test_extracts_transform_id_during_exceptions(self): + """Tests that transform ids are captured during user code exceptions.""" + descriptor = beam_fn_api_pb2.ProcessBundleDescriptor() + + # Boiler plate for the DoFn. + WINDOWING_ID = 'window' + WINDOW_CODER_ID = 'cw' + window = descriptor.windowing_strategies[WINDOWING_ID] + window.window_fn.urn = common_urns.global_windows.urn + window.window_coder_id = WINDOW_CODER_ID + window.trigger.default.SetInParent() + window_coder = descriptor.coders[WINDOW_CODER_ID] + window_coder.spec.urn = common_urns.StandardCoders.Enum.GLOBAL_WINDOW.urn + + # Input collection to the exception raising DoFn. + INPUT_PCOLLECTION_ID = 'pc-in' + INPUT_CODER_ID = 'c-in' + descriptor.pcollections[ + INPUT_PCOLLECTION_ID].unique_name = INPUT_PCOLLECTION_ID + descriptor.pcollections[INPUT_PCOLLECTION_ID].coder_id = INPUT_CODER_ID + descriptor.pcollections[ + INPUT_PCOLLECTION_ID].windowing_strategy_id = WINDOWING_ID + descriptor.coders[ + INPUT_CODER_ID].spec.urn = common_urns.StandardCoders.Enum.BYTES.urn + + # Output collection to the exception raising DoFn. + OUTPUT_PCOLLECTION_ID = 'pc-out' + OUTPUT_CODER_ID = 'c-out' + descriptor.pcollections[ + OUTPUT_PCOLLECTION_ID].unique_name = OUTPUT_PCOLLECTION_ID + descriptor.pcollections[OUTPUT_PCOLLECTION_ID].coder_id = OUTPUT_CODER_ID + descriptor.pcollections[ + OUTPUT_PCOLLECTION_ID].windowing_strategy_id = WINDOWING_ID + descriptor.coders[ + OUTPUT_CODER_ID].spec.urn = common_urns.StandardCoders.Enum.BYTES.urn + + # Add a simple transform to inject an element into the fake pipeline. + TEST_OP_TRANSFORM_ID = 'test_op' + test_transform = descriptor.transforms[TEST_OP_TRANSFORM_ID] + test_transform.outputs['None'] = INPUT_PCOLLECTION_ID + test_transform.spec.urn = 'beam:internal:testop:v1' + test_transform.spec.payload = b'hello, world!' + + # Add the DoFn to create an exception. + TEST_EXCEPTION_TRANSFORM_ID = 'test_transform' + test_transform = descriptor.transforms[TEST_EXCEPTION_TRANSFORM_ID] + test_transform.inputs['0'] = INPUT_PCOLLECTION_ID + test_transform.outputs['None'] = OUTPUT_PCOLLECTION_ID + test_transform.spec.urn = 'beam:internal:testexn:v1' + test_transform.spec.payload = b'expected exception' + + # Create and process a fake bundle. The instruction id doesn't matter + # here. + processor = BundleProcessor(descriptor, None, None) + + with self.assertRaisesRegex(RuntimeError, 'expected exception'): + processor.process_bundle('instruction_id') + + self.fn_log_handler.close() + logs = [ + log for logs in self.test_logging_service.log_records_received + for log in logs.log_entries + ] + + actual_log = logs[0] + + self.assertEqual( + actual_log.severity, beam_fn_api_pb2.LogEntry.Severity.ERROR) + self.assertTrue('expected exception' in actual_log.message) + self.assertEqual(actual_log.transform_id, 'test_transform') + # Test cases. data = { diff --git a/sdks/python/apache_beam/runners/worker/operations.pxd b/sdks/python/apache_beam/runners/worker/operations.pxd index cf1f1b3fb512a..bc8197ef84f10 100644 --- a/sdks/python/apache_beam/runners/worker/operations.pxd +++ b/sdks/python/apache_beam/runners/worker/operations.pxd @@ -34,6 +34,9 @@ cdef class ConsumerSet(Receiver): cdef public step_name cdef public output_index cdef public coder + cdef public object output_sampler + cdef public object element_sampler + cdef public object execution_context cpdef update_counters_start(self, WindowedValue windowed_value) cpdef update_counters_finish(self) @@ -81,6 +84,8 @@ cdef class Operation(object): cdef readonly object scoped_process_state cdef readonly object scoped_finish_state + cdef readonly object data_sampler + cpdef start(self) cpdef process(self, WindowedValue windowed_value) cpdef finish(self) diff --git a/sdks/python/apache_beam/runners/worker/operations.py b/sdks/python/apache_beam/runners/worker/operations.py index fdde55042ecdd..bdb16a46aeea4 100644 --- a/sdks/python/apache_beam/runners/worker/operations.py +++ b/sdks/python/apache_beam/runners/worker/operations.py @@ -53,6 +53,7 @@ from apache_beam.runners.worker import opcounters from apache_beam.runners.worker import operation_specs from apache_beam.runners.worker import sideinputs +from apache_beam.runners.worker.data_sampler import DataSampler from apache_beam.transforms import sideinputs as apache_sideinputs from apache_beam.transforms import combiners from apache_beam.transforms import core @@ -69,6 +70,7 @@ from apache_beam.runners.sdf_utils import SplitResultPrimary from apache_beam.runners.sdf_utils import SplitResultResidual from apache_beam.runners.worker.bundle_processor import ExecutionContext + from apache_beam.runners.worker.data_sampler import OutputSampler from apache_beam.runners.worker.statesampler import StateSampler from apache_beam.transforms.userstate import TimerSpec @@ -123,6 +125,7 @@ def create(counter_factory, coder, producer_type_hints, producer_batch_converter, # type: Optional[BatchConverter] + output_sampler=None, # type: Optional[OutputSampler] ): # type: (...) -> ConsumerSet if len(consumers) == 1: @@ -139,7 +142,8 @@ def create(counter_factory, output_index, consumer, coder, - producer_type_hints) + producer_type_hints, + output_sampler) return GeneralPurposeConsumerSet( counter_factory, @@ -148,7 +152,8 @@ def create(counter_factory, coder, producer_type_hints, consumers, - producer_batch_converter) + producer_batch_converter, + output_sampler) def __init__(self, counter_factory, @@ -157,7 +162,8 @@ def __init__(self, consumers, coder, producer_type_hints, - producer_batch_converter + producer_batch_converter, + output_sampler ): self.opcounter = opcounters.OperationCounters( counter_factory, @@ -171,6 +177,10 @@ def __init__(self, self.output_index = output_index self.coder = coder self.consumers = consumers + self.output_sampler = output_sampler + self.element_sampler = ( + output_sampler.element_sampler if output_sampler else None) + self.execution_context = None # type: Optional[ExecutionContext] def try_split(self, fraction_of_remainder): # type: (...) -> Optional[Any] @@ -197,6 +207,20 @@ def update_counters_start(self, windowed_value): # type: (WindowedValue) -> None self.opcounter.update_from(windowed_value) + if self.execution_context is not None: + self.execution_context.output_sampler = self.output_sampler + + # The following code is optimized by inlining a function call. Because this + # is called for every element, a function call is too expensive (order of + # 100s of nanoseconds). Furthermore, a lock was purposefully not used + # between here and the DataSampler as an additional operation. The tradeoff + # is that some samples might be dropped, but it is better than the + # alternative which is double sampling the same element. + if self.element_sampler is not None: + if not self.element_sampler.has_element: + self.element_sampler.el = windowed_value + self.element_sampler.has_element = True + def update_counters_finish(self): # type: () -> None self.opcounter.update_collect() @@ -223,7 +247,8 @@ def __init__(self, output_index, consumer, # type: Operation coder, - producer_type_hints + producer_type_hints, + output_sampler ): super().__init__( counter_factory, @@ -231,7 +256,8 @@ def __init__(self, output_index, [consumer], coder, producer_type_hints, - None) + None, + output_sampler) self.consumer = consumer def receive(self, windowed_value): @@ -268,7 +294,8 @@ def __init__(self, coder, producer_type_hints, consumers, # type: List[Operation] - producer_batch_converter): + producer_batch_converter, + output_sampler): super().__init__( counter_factory, step_name, @@ -276,7 +303,8 @@ def __init__(self, consumers, coder, producer_type_hints, - producer_batch_converter) + producer_batch_converter, + output_sampler) self.producer_batch_converter = producer_batch_converter @@ -431,20 +459,30 @@ def __init__(self, # on the operation. self.setup_done = False self.step_name = None # type: Optional[str] + self.data_sampler: Optional[DataSampler] = None - def setup(self): - # type: () -> None + def setup(self, data_sampler=None): + # type: (Optional[DataSampler]) -> None """Set up operation. This must be called before any other methods of the operation.""" with self.scoped_start_state: + self.data_sampler = data_sampler self.debug_logging_enabled = logging.getLogger().isEnabledFor( logging.DEBUG) + transform_id = self.name_context.transform_id + # Everything except WorkerSideInputSource, which is not a # top-level operation, should have output_coders #TODO(pabloem): Define better what step name is used here. if getattr(self.spec, 'output_coders', None): + + def get_output_sampler(output_num): + if data_sampler is None: + return None + return data_sampler.sampler_for_output(transform_id, output_num) + self.receivers = [ ConsumerSet.create( self.counter_factory, @@ -454,7 +492,7 @@ def setup(self): coder, self._get_runtime_performance_hints(), self.get_output_batch_converter(), - ) for i, + get_output_sampler(i)) for i, coder in enumerate(self.spec.output_coders) ] self.setup_done = True @@ -465,7 +503,13 @@ def start(self): """Start operation.""" if not self.setup_done: # For legacy workers. - self.setup() + self.setup(self.data_sampler) + + # The ExecutionContext is per instruction and so cannot be set at + # initialization time. + if self.data_sampler is not None: + for receiver in self.receivers: + receiver.execution_context = self.execution_context def get_batching_preference(self): # By default operations don't support batching, require Receiver to unbatch @@ -759,7 +803,7 @@ def __init__(self, counter_factory, sampler, side_input_maps=None, - user_state_context=None + user_state_context=None, ): super(DoOperation, self).__init__(name, spec, counter_factory, sampler) self.side_input_maps = side_input_maps @@ -828,10 +872,10 @@ def _read_side_inputs(self, tags_and_types): yield apache_sideinputs.SideInputMap( view_class, view_options, sideinputs.EmulatedIterable(iterator_fn)) - def setup(self): - # type: () -> None + def setup(self, data_sampler=None): + # type: (Optional[DataSampler]) -> None with self.scoped_start_state: - super(DoOperation, self).setup() + super(DoOperation, self).setup(data_sampler) # See fn_data in dataflow_runner.py fn, args, kwargs, tags_and_types, window_fn = ( @@ -878,6 +922,7 @@ def setup(self): step_name=self.name_context.logging_name(), state=state, user_state_context=self.user_state_context, + transform_id=self.name_context.transform_id, operation_name=self.name_context.metrics_name()) self.dofn_runner.setup() @@ -885,6 +930,7 @@ def start(self): # type: () -> None with self.scoped_start_state: super(DoOperation, self).start() + self.dofn_runner.execution_context = self.execution_context self.dofn_runner.start() def get_batching_preference(self): @@ -1062,10 +1108,7 @@ def current_element_progress(self): def monitoring_infos(self, transform_id, tag_to_pcollection_id): # type: (str, Dict[str, str]) -> Dict[FrozenSet, metrics_pb2.MonitoringInfo] - def encode_progress(value): - # type: (float) -> bytes - coder = coders.IterableCoder(coders.FloatCoder()) - return coder.encode([value]) + progress_coder = coders.IterableCoder(coders.FloatCoder()) with self.lock: infos = super(SdfProcessSizedElements, @@ -1084,12 +1127,12 @@ def encode_progress(value): urn=monitoring_infos.WORK_COMPLETED_URN, type=monitoring_infos.PROGRESS_TYPE, labels=monitoring_infos.create_labels(ptransform=transform_id), - payload=encode_progress(completed)) + payload=progress_coder.encode([completed])) remaining_mi = metrics_pb2.MonitoringInfo( urn=monitoring_infos.WORK_REMAINING_URN, type=monitoring_infos.PROGRESS_TYPE, labels=monitoring_infos.create_labels(ptransform=transform_id), - payload=encode_progress(remaining)) + payload=progress_coder.encode([remaining])) infos[monitoring_infos.to_key(completed_mi)] = completed_mi infos[monitoring_infos.to_key(remaining_mi)] = remaining_mi return infos @@ -1107,11 +1150,11 @@ def __init__(self, name_context, spec, counter_factory, state_sampler): self.phased_combine_fn = ( PhasedCombineFnExecutor(self.spec.phase, fn, args, kwargs)) - def setup(self): - # type: () -> None + def setup(self, data_sampler=None): + # type: (Optional[DataSampler]) -> None with self.scoped_start_state: _LOGGER.debug('Setup called for %s', self) - super(CombineOperation, self).setup() + super(CombineOperation, self).setup(data_sampler) self.phased_combine_fn.combine_fn.setup() def process(self, o): @@ -1226,11 +1269,11 @@ def __init__( self.key_count = 0 self.table = {} - def setup(self): - # type: () -> None + def setup(self, data_sampler=None): + # type: (Optional[DataSampler]) -> None with self.scoped_start_state: _LOGGER.debug('Setup called for %s', self) - super(PGBKCVOperation, self).setup() + super(PGBKCVOperation, self).setup(data_sampler) self.combine_fn.setup() def process(self, wkv): diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker.py b/sdks/python/apache_beam/runners/worker/sdk_worker.py index f7da86234d5be..bfd6544d802b5 100644 --- a/sdks/python/apache_beam/runners/worker/sdk_worker.py +++ b/sdks/python/apache_beam/runners/worker/sdk_worker.py @@ -268,6 +268,8 @@ def get_responses(): work_request) finally: self._alive = False + if self.data_sampler: + self.data_sampler.stop() _LOGGER.info('No more requests from control plane') _LOGGER.info('SDK Harness waiting for in-flight requests to complete') @@ -378,18 +380,12 @@ def _request_sample_data(self, request): def get_samples(request): # type: (beam_fn_api_pb2.InstructionRequest) -> beam_fn_api_pb2.InstructionResponse - samples: Dict[str, List[bytes]] = {} - if self.data_sampler: + samples = beam_fn_api_pb2.SampleDataResponse() + if self.data_sampler is not None: samples = self.data_sampler.samples(request.sample_data.pcollection_ids) - sample_response = beam_fn_api_pb2.SampleDataResponse() - for pcoll_id in samples: - sample_response.element_samples[pcoll_id].elements.extend( - beam_fn_api_pb2.SampledElement(element=s) - for s in samples[pcoll_id]) - return beam_fn_api_pb2.InstructionResponse( - instruction_id=request.instruction_id, sample_data=sample_response) + instruction_id=request.instruction_id, sample_data=samples) self._execute(lambda: get_samples(request), request) diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py b/sdks/python/apache_beam/runners/worker/sdk_worker_main.py index abbcfb72382b5..1e44a998ba05f 100644 --- a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py +++ b/sdks/python/apache_beam/runners/worker/sdk_worker_main.py @@ -52,7 +52,7 @@ def _import_beam_plugins(plugins): for plugin in plugins: try: importlib.import_module(plugin) - _LOGGER.info('Imported beam-plugin %s', plugin) + _LOGGER.debug('Imported beam-plugin %s', plugin) except ImportError: try: _LOGGER.debug(( @@ -61,7 +61,7 @@ def _import_beam_plugins(plugins): plugin) module, _ = plugin.rsplit('.', 1) importlib.import_module(module) - _LOGGER.info('Imported %s for beam-plugin %s', module, plugin) + _LOGGER.debug('Imported %s for beam-plugin %s', module, plugin) except ImportError as exc: _LOGGER.warning('Failed to import beam-plugin %s', plugin, exc_info=exc) @@ -153,9 +153,7 @@ def create_harness(environment, dry_run=False): if dry_run: return - data_sampler = None - if 'enable_data_sampling' in experiments: - data_sampler = DataSampler() + data_sampler = DataSampler.create(sdk_pipeline_options) sdk_harness = SdkHarness( control_address=control_service_descriptor.url, @@ -249,7 +247,7 @@ def _get_state_cache_size(experiments): Returns: an int indicating the maximum number of megabytes to cache. - Default is 0 MB + Default is 100 MB """ for experiment in experiments: @@ -258,7 +256,7 @@ def _get_state_cache_size(experiments): return int( re.match(r'state_cache_size=(?P.*)', experiment).group('state_cache_size')) << 20 - return 0 + return 100 << 20 def _get_data_buffer_time_limit_ms(experiments): diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py index b3ad349be75fb..8570c5a7722c9 100644 --- a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py +++ b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py @@ -39,7 +39,6 @@ from apache_beam.portability.api import metrics_pb2 from apache_beam.runners.worker import sdk_worker from apache_beam.runners.worker import statecache -from apache_beam.runners.worker.data_sampler import DataSampler from apache_beam.runners.worker.sdk_worker import BundleProcessorCache from apache_beam.runners.worker.sdk_worker import GlobalCachingStateHandler from apache_beam.runners.worker.sdk_worker import SdkWorker @@ -279,18 +278,28 @@ def test_failed_bundle_processor_returns_failed_split_response(self): def test_data_sampling_response(self): # Create a data sampler with some fake sampled data. This data will be seen # in the sample response. - data_sampler = DataSampler() coder = FastPrimitivesCoder() - # Sample from two fake PCollections to test that all sampled PCollections - # are present in the response. Also adds an extra sample to test that - # filtering is forwarded to the DataSampler. - data_sampler.sample_output('pcoll_id_1', - coder).sample('hello, world from pcoll_id_1!') - data_sampler.sample_output('pcoll_id_2', - coder).sample('hello, world from pcoll_id_2!') - data_sampler.sample_output('bad_pcoll_id', - coder).sample('if present bug in filter') + class FakeDataSampler: + def samples(self, pcollection_ids): + return beam_fn_api_pb2.SampleDataResponse( + element_samples={ + 'pcoll_id_1': beam_fn_api_pb2.SampleDataResponse.ElementList( + elements=[ + beam_fn_api_pb2.SampledElement( + element=coder.encode_nested('a')) + ]), + 'pcoll_id_2': beam_fn_api_pb2.SampleDataResponse.ElementList( + elements=[ + beam_fn_api_pb2.SampledElement( + element=coder.encode_nested('b')) + ]) + }) + + def stop(self): + pass + + data_sampler = FakeDataSampler() # Create and send the fake reponse. The SdkHarness should query the # DataSampler and fill out the sample response. @@ -310,14 +319,12 @@ def test_data_sampling_response(self): 'pcoll_id_1': beam_fn_api_pb2.SampleDataResponse.ElementList( elements=[ beam_fn_api_pb2.SampledElement( - element=coder.encode_nested( - 'hello, world from pcoll_id_1!')) + element=coder.encode_nested('a')) ]), 'pcoll_id_2': beam_fn_api_pb2.SampleDataResponse.ElementList( elements=[ beam_fn_api_pb2.SampledElement( - element=coder.encode_nested( - 'hello, world from pcoll_id_2!')) + element=coder.encode_nested('b')) ]) })) diff --git a/sdks/python/apache_beam/testing/OWNERS b/sdks/python/apache_beam/testing/OWNERS deleted file mode 100644 index f951b044bbca4..0000000000000 --- a/sdks/python/apache_beam/testing/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - pabloem - - aaltay diff --git a/sdks/python/apache_beam/testing/analyzers/README.md b/sdks/python/apache_beam/testing/analyzers/README.md index 71351fe3e57e2..cc8629f9a57a7 100644 --- a/sdks/python/apache_beam/testing/analyzers/README.md +++ b/sdks/python/apache_beam/testing/analyzers/README.md @@ -19,7 +19,8 @@ # Performance alerts for Beam Python performance and load tests -## Alerts +## Alerts + Performance regressions or improvements detected with the [Change Point Analysis](https://en.wikipedia.org/wiki/Change_detection) using [edivisive](https://github.com/apache/beam/blob/0a91d139dea4276dc46176c4cdcdfce210fc50c4/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L30) analyzer are automatically filed as Beam GitHub issues with a label `perf-alert`. @@ -32,18 +33,15 @@ If a performance alert is created on a test, a GitHub issue will be created and URL, issue number along with the change point value and timestamp are exported to BigQuery. This data will be used to analyze the next change point observed on the same test to update already created GitHub issue or ignore performance alert by not creating GitHub issue to avoid duplicate issue creation. -## Config file structure -The config file defines the structure to run change point analysis on a given test. To add a test to the config file, +## Config file structure + +The yaml defines the structure to run change point analysis on a given test. To add a test config to the yaml file, please follow the below structure. -**NOTE**: The Change point analysis only supports reading the metric data from Big Query for now. +**NOTE**: The Change point analysis only supports reading the metric data from `BigQuery` only. ``` -# the test_1 must be a unique id. -test_1: - test_name: Pytorch image classification on 50k images of size 224 x 224 with resnet 152 - test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks - source: big_query +test_1: # a unique id for each test config. metrics_dataset: beam_run_inference metrics_table: torch_inference_imagenet_results_resnet152 project: apache-beam-testing @@ -54,11 +52,15 @@ test_1: num_runs_in_change_point_window: 30 # optional parameter ``` -**NOTE**: `test_target` is optional. It is used for identifying the test that was causing the regression. +#### Optional Parameters: + +These are the optional parameters that can be added to the test config in addition to the parameters mentioned above. + +- `test_target`: Identifies the test responsible for the regression. + +- `test_name`: Denotes the name of the test as stored in the BigQuery table. -**Note**: If the source is **BigQuery**, the `metrics_dataset`, `metrics_table`, `project` and `metric_name` should match with the values defined for performance/load tests. -The above example uses this [test configuration](https://github.com/apache/beam/blob/0a91d139dea4276dc46176c4cdcdfce210fc50c4/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L30) -to fill up the values required to fetch the data from source. +**Note**: The tool, by default, pulls metrics from BigQuery tables. Ensure that the values for `metrics_dataset`, `metrics_table`, `project`, and `metric_name` align with those defined for performance/load tests. The provided example utilizes this [test configuration](https://github.com/apache/beam/blob/0a91d139dea4276dc46176c4cdcdfce210fc50c4/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L30) to populate the necessary values for data retrieval. ### Different ways to avoid false positive change points @@ -73,21 +75,49 @@ Sometimes, the change point found might be way back in time and could be irrelev reported only when it was observed in the last 7 runs from the current run, setting `num_runs_in_change_point_window=7` will achieve it. -## Register a test for performance alerts +## Register a test for performance alerts + +If a new test needs to be registered for the performance alerting tool, + +- You can either add it to the config file that is already present. +- You can define your own yaml file and call the [perf_analysis.run()](https://github.com/apache/beam/blob/a46bc12a256dcaa3ae2cc9e5d6fdcaa82b59738b/sdks/python/apache_beam/testing/analyzers/perf_analysis.py#L152) method. + + +## Integrating the Perf Alert Tool with a Custom BigQuery Schema + +By default, the Perf Alert Tool retrieves metrics from the `apache-beam-testing` BigQuery projects. All performance and load tests within Beam utilize a standard [schema](https://github.com/apache/beam/blob/a7e12db9b5977c4a7b13554605c0300389a3d6da/sdks/python/apache_beam/testing/load_tests/load_test_metrics_utils.py#L70) for metrics publication. The tool inherently recognizes and operates with this schema when extracting metrics from BigQuery tables. -If a new test needs to be registered for the performance alerting tool, please add the required test parameters to the -config file. +To fetch the data from a BigQuery dataset that is not a default setting of the Apache Beam's setting, One can inherit the `MetricsFetcher` class and implement the abstract method `fetch_metric_data`. This method should return a tuple of desired metric values and timestamps of the metric values of when it was published. + +``` +from apache_beam.testing.analyzers import perf_analysis +config_file_path = +my_metric_fetcher = MyMetricsFetcher() # inherited from MetricsFetcher +perf_analysis.run(config_file_path, my_metrics_fetcher) +``` + +``Note``: The metrics and timestamps should be sorted based on the timestamps values in ascending order. + +### Configuring GitHub Parameters + +Out of the box, the performance alert tool targets the `apache/beam` repository when raising issues. If you wish to utilize this tool for another repository, you'll need to pre-set a couple of environment variables: + +- `REPO_OWNER`: Represents the owner of the repository. (e.g., `apache`) +- `REPO_NAME`: Specifies the repository name itself. (e.g., `beam`) + +Before initiating the tool, also ensure that the `GITHUB_TOKEN` is set to an authenticated GitHub token. This permits the tool to generate GitHub issues whenever performance alerts arise. ## Triage performance alert issues -All the performance/load tests metrics defined at [beam/.test-infra/jenkins](https://github.com/apache/beam/tree/master/.test-infra/jenkins) are imported to [Grafana dashboards](http://104.154.241.245/d/1/getting-started?orgId=1) for visualization. Please +All the performance/load tests metrics defined at [beam/.test-infra/jenkins](https://github.com/apache/beam/tree/master/.test-infra/jenkins) are imported to [Grafana dashboards](http://metrics.beam.apache.org/d/1/getting-started?orgId=1) for visualization. Please find the alerted test dashboard to find a spike in the metric values. For example, for the below configuration, -* test_target: `apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks` -* metric_name: `mean_load_model_latency_milli_secs` -Grafana dashboard can be found at http://104.154.241.245/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=7 +- test_target: `apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks` +- metric_name: `mean_load_model_latency_milli_secs` + +Grafana dashboard can be found at http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=7 If the dashboard for a test is not found, you can use the notebook `analyze_metric_data.ipynb` to generate a plot for the given test, metric_name. diff --git a/sdks/python/apache_beam/testing/analyzers/__init__.py b/sdks/python/apache_beam/testing/analyzers/__init__.py index cce3acad34a49..136d9f5f5d8a2 100644 --- a/sdks/python/apache_beam/testing/analyzers/__init__.py +++ b/sdks/python/apache_beam/testing/analyzers/__init__.py @@ -14,3 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +""" +Peformance alert tooling for Apache Beam. No backwards compatibility +guarantees. +""" diff --git a/sdks/python/apache_beam/testing/analyzers/constants.py b/sdks/python/apache_beam/testing/analyzers/constants.py index c0df05f61dbe5..09ab5c5959082 100644 --- a/sdks/python/apache_beam/testing/analyzers/constants.py +++ b/sdks/python/apache_beam/testing/analyzers/constants.py @@ -37,7 +37,11 @@ _DEFAULT_NUM_RUMS_IN_CHANGE_POINT_WINDOW = 14 _PERF_TEST_KEYS = { - 'test_name', 'metrics_dataset', 'metrics_table', 'project', 'metric_name' + 'test_description', + 'metrics_dataset', + 'metrics_table', + 'project', + 'metric_name' } _SCHEMA = [{ @@ -66,3 +70,6 @@ }, { 'name': _ISSUE_URL, 'field_type': 'STRING', 'mode': 'REQUIRED' }] + +_ANOMALY_MARKER = ' <---- Anomaly' +_EDGE_SEGMENT_SIZE = 3 diff --git a/sdks/python/apache_beam/testing/analyzers/github_issues_utils.py b/sdks/python/apache_beam/testing/analyzers/github_issues_utils.py index b53768edcab9e..cbbb9e5d3a2e0 100644 --- a/sdks/python/apache_beam/testing/analyzers/github_issues_utils.py +++ b/sdks/python/apache_beam/testing/analyzers/github_issues_utils.py @@ -21,9 +21,12 @@ from typing import Optional from typing import Tuple -import pandas as pd import requests +from apache_beam.testing.analyzers import constants +from apache_beam.testing.analyzers.perf_analysis_utils import MetricContainer +from apache_beam.testing.analyzers.perf_analysis_utils import TestConfigContainer + try: _GITHUB_TOKEN: Optional[str] = os.environ['GITHUB_TOKEN'] except KeyError as e: @@ -32,8 +35,8 @@ 'A Github Personal Access token is required ' 'to create Github Issues.') -_BEAM_GITHUB_REPO_OWNER = 'apache' -_BEAM_GITHUB_REPO_NAME = 'beam' +_GITHUB_REPO_OWNER = os.environ.get('REPO_OWNER', 'apache') +_GITHUB_REPO_NAME = os.environ.get('REPO_NAME', 'beam') # Adding GitHub Rest API version to the header to maintain version stability. # For more information, please look at # https://github.blog/2022-11-28-to-infinity-and-beyond-enabling-the-future-of-githubs-rest-api-with-api-versioning/ # pylint: disable=line-too-long @@ -54,10 +57,12 @@ For more information on how to triage the alerts, please look at `Triage performance alert issues` section of the [README](https://github.com/apache/beam/tree/master/sdks/python/apache_beam/testing/analyzers/README.md#triage-performance-alert-issues). """ -_METRIC_INFO_TEMPLATE = "timestamp: {}, metric_value: `{}`" +_METRIC_INFO_TEMPLATE = "timestamp: {}, metric_value: {}" _AWAITING_TRIAGE_LABEL = 'awaiting triage' _PERF_ALERT_LABEL = 'perf-alert' +_REQUEST_TIMEOUT_SECS = 60 + def create_issue( title: str, @@ -75,10 +80,10 @@ def create_issue( Tuple containing GitHub issue number and issue URL. """ url = "https://api.github.com/repos/{}/{}/issues".format( - _BEAM_GITHUB_REPO_OWNER, _BEAM_GITHUB_REPO_NAME) + _GITHUB_REPO_OWNER, _GITHUB_REPO_NAME) data = { - 'owner': _BEAM_GITHUB_REPO_OWNER, - 'repo': _BEAM_GITHUB_REPO_NAME, + 'owner': _GITHUB_REPO_OWNER, + 'repo': _GITHUB_REPO_NAME, 'title': title, 'body': description, 'labels': [_AWAITING_TRIAGE_LABEL, _PERF_ALERT_LABEL] @@ -86,7 +91,10 @@ def create_issue( if labels: data['labels'].extend(labels) # type: ignore response = requests.post( - url=url, data=json.dumps(data), headers=_HEADERS).json() + url=url, + data=json.dumps(data), + headers=_HEADERS, + timeout=_REQUEST_TIMEOUT_SECS).json() return response['number'], response['html_url'] @@ -106,73 +114,97 @@ def comment_on_issue(issue_number: int, issue, and the comment URL. """ url = 'https://api.github.com/repos/{}/{}/issues/{}'.format( - _BEAM_GITHUB_REPO_OWNER, _BEAM_GITHUB_REPO_NAME, issue_number) + _GITHUB_REPO_OWNER, _GITHUB_REPO_NAME, issue_number) open_issue_response = requests.get( url, json.dumps({ - 'owner': _BEAM_GITHUB_REPO_OWNER, - 'repo': _BEAM_GITHUB_REPO_NAME, + 'owner': _GITHUB_REPO_OWNER, + 'repo': _GITHUB_REPO_NAME, 'issue_number': issue_number }, default=str), - headers=_HEADERS).json() + headers=_HEADERS, + timeout=_REQUEST_TIMEOUT_SECS).json() if open_issue_response['state'] == 'open': data = { - 'owner': _BEAM_GITHUB_REPO_OWNER, - 'repo': _BEAM_GITHUB_REPO_NAME, + 'owner': _GITHUB_REPO_OWNER, + 'repo': _GITHUB_REPO_NAME, 'body': comment_description, issue_number: issue_number, } + response = requests.post( - open_issue_response['comments_url'], json.dumps(data), headers=_HEADERS) + open_issue_response['comments_url'], + json.dumps(data), + headers=_HEADERS, + timeout=_REQUEST_TIMEOUT_SECS) return True, response.json()['html_url'] return False, '' def add_awaiting_triage_label(issue_number: int): url = 'https://api.github.com/repos/{}/{}/issues/{}/labels'.format( - _BEAM_GITHUB_REPO_OWNER, _BEAM_GITHUB_REPO_NAME, issue_number) + _GITHUB_REPO_OWNER, _GITHUB_REPO_NAME, issue_number) requests.post( - url, json.dumps({'labels': [_AWAITING_TRIAGE_LABEL]}), headers=_HEADERS) + url, + json.dumps({'labels': [_AWAITING_TRIAGE_LABEL]}), + headers=_HEADERS, + timeout=_REQUEST_TIMEOUT_SECS) def get_issue_description( - test_name: str, - metric_name: str, - timestamps: List[pd.Timestamp], - metric_values: List, + test_config_container: TestConfigContainer, + metric_container: MetricContainer, change_point_index: int, - max_results_to_display: int = 5) -> str: + max_results_to_display: int = 5, +) -> str: """ Args: - metric_name: Metric name used for the Change Point Analysis. - timestamps: Timestamps of the metrics when they were published to the - Database. Timestamps are expected in ascending order. - metric_values: metric values for the previous runs. - change_point_index: Index for the change point. The element in the - index of the metric_values would be the change point. - max_results_to_display: Max number of results to display from the change - point index, in both directions of the change point index. + test_config_container: TestConfigContainer containing test metadata. + metric_container: MetricContainer containing metric data. + change_point_index: Index of the change point in the metric data. + max_results_to_display: Max number of results to display from the change + point index, in both directions of the change point index. Returns: str: Description used to fill the GitHub issues description. """ # TODO: Add mean and median before and after the changepoint index. - max_timestamp_index = min( - change_point_index + max_results_to_display, len(metric_values) - 1) - min_timestamp_index = max(0, change_point_index - max_results_to_display) - description = _ISSUE_DESCRIPTION_TEMPLATE.format( - test_name, metric_name) + 2 * '\n' + description = [] - runs_to_display = [ - _METRIC_INFO_TEMPLATE.format(timestamps[i].ctime(), metric_values[i]) - for i in reversed(range(min_timestamp_index, max_timestamp_index + 1)) - ] + description.append( + _ISSUE_DESCRIPTION_TEMPLATE.format( + test_config_container.test_id, test_config_container.metric_name)) + + if test_config_container.test_name: + description.append(("`test_name:` " + f'{test_config_container.test_name}')) + + if test_config_container.test_description: + description.append( + ("`Test description:` " + f'{test_config_container.test_description}')) + + description.append('```') + + runs_to_display = [] + max_timestamp_index = min( + change_point_index + max_results_to_display, + len(metric_container.values) - 1) + min_timestamp_index = max(0, change_point_index - max_results_to_display) - runs_to_display[change_point_index - min_timestamp_index] += " <---- Anomaly" - return description + '\n'.join(runs_to_display) + # run in reverse to display the most recent runs first. + for i in reversed(range(min_timestamp_index, max_timestamp_index + 1)): + row_template = _METRIC_INFO_TEMPLATE.format( + metric_container.timestamps[i].ctime(), + format(metric_container.values[i], '.2f')) + if i == change_point_index: + row_template += constants._ANOMALY_MARKER + runs_to_display.append(row_template) + + description.append(os.linesep.join(runs_to_display)) + description.append('```') + return (2 * os.linesep).join(description) def report_change_point_on_issues( diff --git a/sdks/python/apache_beam/testing/analyzers/perf_analysis.py b/sdks/python/apache_beam/testing/analyzers/perf_analysis.py index ee00e8abf4216..5802fe0414543 100644 --- a/sdks/python/apache_beam/testing/analyzers/perf_analysis.py +++ b/sdks/python/apache_beam/testing/analyzers/perf_analysis.py @@ -22,67 +22,117 @@ import argparse import logging -import os import uuid from datetime import datetime from datetime import timezone from typing import Any from typing import Dict -from typing import Optional import pandas as pd from apache_beam.testing.analyzers import constants +from apache_beam.testing.analyzers.perf_analysis_utils import BigQueryMetricsFetcher +from apache_beam.testing.analyzers.perf_analysis_utils import ChangePointConfig from apache_beam.testing.analyzers.perf_analysis_utils import GitHubIssueMetaData +from apache_beam.testing.analyzers.perf_analysis_utils import MetricsFetcher +from apache_beam.testing.analyzers.perf_analysis_utils import TestConfigContainer from apache_beam.testing.analyzers.perf_analysis_utils import create_performance_alert -from apache_beam.testing.analyzers.perf_analysis_utils import fetch_metric_data from apache_beam.testing.analyzers.perf_analysis_utils import find_latest_change_point_index from apache_beam.testing.analyzers.perf_analysis_utils import get_existing_issues_data from apache_beam.testing.analyzers.perf_analysis_utils import is_change_point_in_valid_window -from apache_beam.testing.analyzers.perf_analysis_utils import is_perf_alert +from apache_beam.testing.analyzers.perf_analysis_utils import is_sibling_change_point from apache_beam.testing.analyzers.perf_analysis_utils import publish_issue_metadata_to_big_query from apache_beam.testing.analyzers.perf_analysis_utils import read_test_config -from apache_beam.testing.analyzers.perf_analysis_utils import validate_config -from apache_beam.testing.load_tests.load_test_metrics_utils import BigQueryMetricsFetcher -def run_change_point_analysis(params, test_id, big_query_metrics_fetcher): +def get_test_config_container( + params: Dict[str, Any], + test_id: str, + metric_name: str, +) -> TestConfigContainer: """ Args: - params: Dict containing parameters to run change point analysis. - test_id: Test id for the current test. + params: Dict containing parameters to run change point analysis. + Returns: + TestConfigContainer object containing test config parameters. + """ + return TestConfigContainer( + project=params['project'], + metrics_dataset=params['metrics_dataset'], + metrics_table=params['metrics_table'], + metric_name=metric_name, + test_id=test_id, + test_description=params['test_description'], + test_name=params.get('test_name', None), + labels=params.get('labels', None), + ) + + +def get_change_point_config(params: Dict[str, Any], ) -> ChangePointConfig: + """ + Args: + params: Dict containing parameters to run change point analysis. + Returns: + ChangePointConfig object containing change point analysis parameters. + """ + return ChangePointConfig( + min_runs_between_change_points=params.get( + 'min_runs_between_change_points', + constants._DEFAULT_MIN_RUNS_BETWEEN_CHANGE_POINTS), + num_runs_in_change_point_window=params.get( + 'num_runs_in_change_point_window', + constants._DEFAULT_NUM_RUMS_IN_CHANGE_POINT_WINDOW)) + + +def run_change_point_analysis( + test_config_container: TestConfigContainer, + big_query_metrics_fetcher: MetricsFetcher, + change_point_config: ChangePointConfig = ChangePointConfig(), + save_alert_metadata: bool = False, +): + """ + Args: + test_config_container: TestConfigContainer containing test metadata for + fetching data and running change point analysis. big_query_metrics_fetcher: BigQuery metrics fetcher used to fetch data for change point analysis. + change_point_config: ChangePointConfig containing parameters to run + change point analysis. + save_alert_metadata: bool indicating if issue metadata + should be published to BigQuery table. Returns: bool indicating if a change point is observed and alerted on GitHub. """ - if not validate_config(params.keys()): - raise ValueError( - f"Please make sure all these keys {constants._PERF_TEST_KEYS} " - f"are specified for the {test_id}") + logging.info( + "Running change point analysis for test ID :%s on metric: % s" % + (test_config_container.test_id, test_config_container.metric_name)) - metric_name = params['metric_name'] - test_name = params['test_name'].replace('.', '_') + f'_{metric_name}' + # test_name will be used to query a single test from + # multiple tests in a single BQ table. Right now, the default + # assumption is that all the test have an individual BQ table + # but this might not be case for other tests(such as IO tests where + # a single BQ tables stores all the data) + test_name = test_config_container.test_name min_runs_between_change_points = ( - constants._DEFAULT_MIN_RUNS_BETWEEN_CHANGE_POINTS) - if 'min_runs_between_change_points' in params: - min_runs_between_change_points = params['min_runs_between_change_points'] + change_point_config.min_runs_between_change_points) num_runs_in_change_point_window = ( - constants._DEFAULT_NUM_RUMS_IN_CHANGE_POINT_WINDOW) - if 'num_runs_in_change_point_window' in params: - num_runs_in_change_point_window = params['num_runs_in_change_point_window'] + change_point_config.num_runs_in_change_point_window) - metric_values, timestamps = fetch_metric_data( - params=params, - big_query_metrics_fetcher=big_query_metrics_fetcher - ) + metric_container = big_query_metrics_fetcher.fetch_metric_data( + test_config=test_config_container) + metric_container.sort_by_timestamp() + + metric_values = metric_container.values + timestamps = metric_container.timestamps change_point_index = find_latest_change_point_index( metric_values=metric_values) if not change_point_index: - logging.info("Change point is not detected for the test %s" % test_name) + logging.info( + "Change point is not detected for the test ID %s" % + test_config_container.test_id) return False # since timestamps are ordered in ascending order and # num_runs_in_change_point_window refers to the latest runs, @@ -92,63 +142,84 @@ def run_change_point_analysis(params, test_id, big_query_metrics_fetcher): if not is_change_point_in_valid_window(num_runs_in_change_point_window, latest_change_point_run): logging.info( - 'Performance regression/improvement found for the test: %s. ' + 'Performance regression/improvement found for the test ID: %s. ' 'on metric %s. Since the change point run %s ' 'lies outside the num_runs_in_change_point_window distance: %s, ' 'alert is not raised.' % ( - params['test_name'], - metric_name, + test_config_container.test_id, + test_config_container.metric_name, latest_change_point_run + 1, num_runs_in_change_point_window)) return False - is_alert = True + is_valid_change_point = True last_reported_issue_number = None - issue_metadata_table_name = f'{params.get("metrics_table")}_{metric_name}' + + # create a unique table name for each test and metric combination. + # for beam load tests, metric_name and metric table are enough to + # create a unique table name. For templates/IO tests, add `test_name`. + issue_metadata_table_name = ( + f'{test_config_container.metrics_table}_{test_config_container.metric_name}' # pylint: disable=line-too-long + ) + if test_config_container.test_name: + issue_metadata_table_name = ( + f'{issue_metadata_table_name}_{test_config_container.test_name}') + existing_issue_data = get_existing_issues_data( - table_name=issue_metadata_table_name, - big_query_metrics_fetcher=big_query_metrics_fetcher) + table_name=issue_metadata_table_name) if existing_issue_data is not None: existing_issue_timestamps = existing_issue_data[ constants._CHANGE_POINT_TIMESTAMP_LABEL].tolist() last_reported_issue_number = existing_issue_data[ constants._ISSUE_NUMBER].tolist()[0] + # convert numpy.int64 to int + last_reported_issue_number = last_reported_issue_number.item() - is_alert = is_perf_alert( + is_valid_change_point = is_sibling_change_point( previous_change_point_timestamps=existing_issue_timestamps, change_point_index=change_point_index, timestamps=timestamps, - min_runs_between_change_points=min_runs_between_change_points) - logging.debug( - "Performance alert is %s for test %s" % (is_alert, params['test_name'])) - if is_alert: + min_runs_between_change_points=min_runs_between_change_points, + test_id=test_config_container.test_id) + + # for testing purposes, we don't want to create an issue even if there is + # a valid change point. This is useful when we want to test the change point + # analysis logic without creating an issue. + if is_valid_change_point and save_alert_metadata: issue_number, issue_url = create_performance_alert( - metric_name, params['test_name'], timestamps, - metric_values, change_point_index, - params.get('labels', None), - last_reported_issue_number, - test_target=params['test_target'] if 'test_target' in params else None + test_config_container=test_config_container, + metric_container=metric_container, + change_point_index=change_point_index, + existing_issue_number=last_reported_issue_number, ) issue_metadata = GitHubIssueMetaData( issue_timestamp=pd.Timestamp( datetime.now().replace(tzinfo=timezone.utc)), - test_name=test_name, - metric_name=metric_name, - test_id=uuid.uuid4().hex, + # BQ doesn't allow '.' in table name + test_id=test_config_container.test_id.replace('.', '_'), + test_name=test_name or uuid.uuid4().hex, + metric_name=test_config_container.metric_name, change_point=metric_values[change_point_index], issue_number=issue_number, issue_url=issue_url, - change_point_timestamp=timestamps[change_point_index]) - + change_point_timestamp=timestamps[change_point_index], + ) publish_issue_metadata_to_big_query( - issue_metadata=issue_metadata, table_name=issue_metadata_table_name) - - return is_alert + issue_metadata=issue_metadata, + table_name=issue_metadata_table_name, + project=test_config_container.project, + ) + return is_valid_change_point -def run(config_file_path: Optional[str] = None) -> None: +def run( + *, + config_file_path: str, + big_query_metrics_fetcher: MetricsFetcher = BigQueryMetricsFetcher(), + save_alert_metadata: bool = False, +) -> None: """ run is the entry point to run change point analysis on test metric data, which is read from config file, and if there is a performance @@ -162,16 +233,25 @@ def run(config_file_path: Optional[str] = None) -> None: defined in the config file. """ - if config_file_path is None: - config_file_path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), 'tests_config.yaml') - tests_config: Dict[str, Dict[str, Any]] = read_test_config(config_file_path) - big_query_metrics_fetcher = BigQueryMetricsFetcher() - for test_id, params in tests_config.items(): - run_change_point_analysis(params, test_id, big_query_metrics_fetcher) + # single test config can have multiple metrics so we need to + # iterate over all the metrics and run change point analysis + # for each metric. + metric_names = params['metric_name'] + if isinstance(metric_names, str): + metric_names = [metric_names] + + for metric_name in metric_names: + test_config_container = get_test_config_container( + params=params, test_id=test_id, metric_name=metric_name) + change_point_config = get_change_point_config(params) + run_change_point_analysis( + test_config_container=test_config_container, + big_query_metrics_fetcher=big_query_metrics_fetcher, + change_point_config=change_point_config, + save_alert_metadata=save_alert_metadata) if __name__ == '__main__': @@ -180,7 +260,7 @@ def run(config_file_path: Optional[str] = None) -> None: parser = argparse.ArgumentParser() parser.add_argument( '--config_file_path', - default=None, + required=True, type=str, help='Path to the config file that contains data to run the Change Point ' 'Analysis.The default file will used will be ' @@ -189,9 +269,17 @@ def run(config_file_path: Optional[str] = None) -> None: 'performance regression in the tests, ' 'please provide an .yml file in the same structure as the above ' 'mentioned file. ') + parser.add_argument( + '--save_alert_metadata', + action='store_true', + help='Save perf alert/ GH Issue metadata to BigQuery table.') known_args, unknown_args = parser.parse_known_args() if unknown_args: logging.warning('Discarding unknown arguments : %s ' % unknown_args) - run(known_args.config_file_path) + run( + config_file_path=known_args.config_file_path, + # Set this to true while running in production. + save_alert_metadata=known_args.save_alert_metadata # pylint: disable=line-too-long + ) diff --git a/sdks/python/apache_beam/testing/analyzers/perf_analysis_test.py b/sdks/python/apache_beam/testing/analyzers/perf_analysis_test.py index 000175e6388a8..4ef394d4ffab5 100644 --- a/sdks/python/apache_beam/testing/analyzers/perf_analysis_test.py +++ b/sdks/python/apache_beam/testing/analyzers/perf_analysis_test.py @@ -18,59 +18,70 @@ import logging import os +import re import unittest import mock +import numpy as np import pandas as pd # pylint: disable=ungrouped-imports try: import apache_beam.testing.analyzers.perf_analysis as analysis + from apache_beam.io.filesystems import FileSystems from apache_beam.testing.analyzers import constants + from apache_beam.testing.analyzers import github_issues_utils + from apache_beam.testing.analyzers.perf_analysis_utils import BigQueryMetricsFetcher + from apache_beam.testing.analyzers.perf_analysis_utils import MetricContainer + from apache_beam.testing.analyzers.perf_analysis_utils import TestConfigContainer from apache_beam.testing.analyzers.perf_analysis_utils import is_change_point_in_valid_window - from apache_beam.testing.analyzers.perf_analysis_utils import is_perf_alert + from apache_beam.testing.analyzers.perf_analysis_utils import is_edge_change_point + from apache_beam.testing.analyzers.perf_analysis_utils import is_sibling_change_point from apache_beam.testing.analyzers.perf_analysis_utils import e_divisive + from apache_beam.testing.analyzers.perf_analysis_utils import filter_change_points_by_median_threshold + from apache_beam.testing.analyzers.perf_analysis_utils import find_change_points + from apache_beam.testing.analyzers.perf_analysis_utils import find_latest_change_point_index from apache_beam.testing.analyzers.perf_analysis_utils import validate_config + from apache_beam.testing.load_tests import load_test_metrics_utils + except ImportError as e: - analysis = None # type: ignore + raise unittest.SkipTest('Missing dependencies to run perf analysis tests.') # mock methods. -def get_fake_data_with_no_change_point(**kwargs): +def get_fake_data_with_no_change_point(*args, **kwargs): num_samples = 20 metric_values = [1] * num_samples - timestamps = list(range(num_samples)) - return metric_values, timestamps + timestamps = [pd.Timestamp(i) for i in range(num_samples)] + return MetricContainer(metric_values, timestamps) -def get_fake_data_with_change_point(**kwargs): +def get_fake_data_with_change_point(*args, **kwargs): + # change point will be at index 13. num_samples = 20 - metric_values = [0] * (num_samples // 2) + [1] * (num_samples // 2) - timestamps = [i for i in range(num_samples)] - return metric_values, timestamps + metric_values = [0] * 12 + [3] + [4] * 7 + timestamps = [pd.Timestamp(i) for i in range(num_samples)] + return MetricContainer(metric_values, timestamps) def get_existing_issue_data(**kwargs): - # change point found at index 10. So passing 10 in the + # change point found at index 13. So passing 13 in the # existing issue data in mock method. return pd.DataFrame([{ - constants._CHANGE_POINT_TIMESTAMP_LABEL: 10, constants._ISSUE_NUMBER: 0 + constants._CHANGE_POINT_TIMESTAMP_LABEL: pd.Timestamp(13), + constants._ISSUE_NUMBER: np.array([0]) }]) -@unittest.skipIf( - analysis is None, - 'Missing dependencies. ' - 'Test dependencies are missing for the Analyzer.') class TestChangePointAnalysis(unittest.TestCase): def setUp(self) -> None: self.single_change_point_series = [0] * 10 + [1] * 10 self.multiple_change_point_series = self.single_change_point_series + [ 2 ] * 20 - self.timestamps = list(range(5)) + self.timestamps = [pd.Timestamp(i) for i in range(5)] self.params = { - 'test_name': 'fake_test', + 'test_description': 'fake_description', 'metrics_dataset': 'fake_dataset', 'metrics_table': 'fake_table', 'project': 'fake_project', @@ -102,7 +113,7 @@ def test_change_point_outside_inspection_window_is_not_a_valid_alert(self): def test_validate_config(self): test_keys = { - 'test_name', + 'test_description', 'metrics_dataset', 'metrics_table', 'project', @@ -114,44 +125,52 @@ def test_validate_config(self): def test_duplicate_change_point(self): change_point_index = 2 min_runs_between_change_points = 1 - is_alert = is_perf_alert( + is_alert = is_sibling_change_point( previous_change_point_timestamps=[self.timestamps[0]], timestamps=self.timestamps, change_point_index=change_point_index, - min_runs_between_change_points=min_runs_between_change_points) + min_runs_between_change_points=min_runs_between_change_points, + test_id=self.test_id) self.assertTrue(is_alert) def test_duplicate_change_points_are_not_valid_alerts(self): change_point_index = 2 min_runs_between_change_points = 1 - is_alert = is_perf_alert( + is_alert = is_sibling_change_point( previous_change_point_timestamps=[self.timestamps[3]], timestamps=self.timestamps, change_point_index=change_point_index, - min_runs_between_change_points=min_runs_between_change_points) + min_runs_between_change_points=min_runs_between_change_points, + test_id=self.test_id) self.assertFalse(is_alert) - is_alert = is_perf_alert( + is_alert = is_sibling_change_point( previous_change_point_timestamps=[ self.timestamps[0], self.timestamps[3] ], timestamps=self.timestamps, change_point_index=change_point_index, - min_runs_between_change_points=min_runs_between_change_points) + min_runs_between_change_points=min_runs_between_change_points, + test_id=self.test_id) self.assertFalse(is_alert) - @mock.patch( - 'apache_beam.testing.analyzers.perf_analysis.fetch_metric_data', + @mock.patch.object( + BigQueryMetricsFetcher, + 'fetch_metric_data', get_fake_data_with_no_change_point) def test_no_alerts_when_no_change_points(self): - is_alert = analysis.run_change_point_analysis( + test_config_container = analysis.get_test_config_container( params=self.params, test_id=self.test_id, - big_query_metrics_fetcher=None) + metric_name=self.params['metric_name']) + is_alert = analysis.run_change_point_analysis( + test_config_container=test_config_container, + big_query_metrics_fetcher=BigQueryMetricsFetcher()) self.assertFalse(is_alert) - @mock.patch( - 'apache_beam.testing.analyzers.perf_analysis.fetch_metric_data', + @mock.patch.object( + BigQueryMetricsFetcher, + 'fetch_metric_data', get_fake_data_with_change_point) @mock.patch( 'apache_beam.testing.analyzers.perf_analysis.get_existing_issues_data', @@ -165,14 +184,18 @@ def test_no_alerts_when_no_change_points(self): '.create_performance_alert', return_value=(0, '')) def test_alert_on_data_with_change_point(self, *args): - is_alert = analysis.run_change_point_analysis( + test_config_container = analysis.get_test_config_container( params=self.params, test_id=self.test_id, - big_query_metrics_fetcher=None) + metric_name=self.params['metric_name']) + is_alert = analysis.run_change_point_analysis( + test_config_container=test_config_container, + big_query_metrics_fetcher=BigQueryMetricsFetcher()) self.assertTrue(is_alert) - @mock.patch( - 'apache_beam.testing.analyzers.perf_analysis.fetch_metric_data', + @mock.patch.object( + BigQueryMetricsFetcher, + 'fetch_metric_data', get_fake_data_with_change_point) @mock.patch( 'apache_beam.testing.analyzers.perf_analysis.get_existing_issues_data', @@ -185,12 +208,71 @@ def test_alert_on_data_with_change_point(self, *args): 'apache_beam.testing.analyzers.perf_analysis.create_performance_alert', return_value=(0, '')) def test_alert_on_data_with_reported_change_point(self, *args): - is_alert = analysis.run_change_point_analysis( + test_config_container = analysis.get_test_config_container( params=self.params, test_id=self.test_id, - big_query_metrics_fetcher=None) + metric_name=self.params['metric_name']) + is_alert = analysis.run_change_point_analysis( + test_config_container=test_config_container, + big_query_metrics_fetcher=BigQueryMetricsFetcher()) self.assertFalse(is_alert) + def test_change_point_has_anomaly_marker_in_gh_description(self): + metric_container = get_fake_data_with_change_point() + metric_values = metric_container.values + change_point_index = find_latest_change_point_index(metric_values) + + test_config_container = TestConfigContainer( + project=self.params['project'], + metrics_dataset=self.params['metrics_dataset'], + metrics_table=self.params['metrics_table'], + metric_name=self.params['metric_name'], + test_id=self.test_id, + test_description=self.params['test_description'], + test_name=self.params.get('test_name', None), + labels=self.params.get('labels', None), + ) + + description = github_issues_utils.get_issue_description( + test_config_container=test_config_container, + metric_container=metric_container, + change_point_index=change_point_index, + max_results_to_display=( + constants._NUM_RESULTS_TO_DISPLAY_ON_ISSUE_DESCRIPTION)) + + runs_info = next(( + line for line in description.split(2 * os.linesep) + if re.match(r'timestamp: .*, metric_value: .*', line.strip())), + '') + pattern = (r'timestamp: .+ (\d{4}), metric_value: (\d+.\d+) <---- Anomaly') + match = re.search(pattern, runs_info) + self.assertTrue(match) + + def test_change_point_on_noisy_data(self): + def read_csv(path): + with FileSystems.open(path) as fp: + return pd.read_csv(fp) + + metric_data = read_csv( + 'gs://apache-beam-ml/testing/inputs/test_data_with_noise.csv') + metric_values = metric_data[load_test_metrics_utils.VALUE_LABEL].tolist() + change_points = find_change_points(metric_values) + self.assertEqual(change_points[0], 20) + + # filter the noise. + valid_points = filter_change_points_by_median_threshold( + metric_values, change_points) + self.assertEqual(len(valid_points), 0) + + def test_change_point_on_edge_segment(self): + data = [1] * 50 + [100] + change_points = find_change_points(data) + self.assertEqual(change_points, [50]) + + self.assertEqual(is_edge_change_point(change_points[0], len(data)), True) + + self.assertEqual(find_latest_change_point_index(data), None) + if __name__ == '__main__': logging.getLogger().setLevel(logging.DEBUG) diff --git a/sdks/python/apache_beam/testing/analyzers/perf_analysis_utils.py b/sdks/python/apache_beam/testing/analyzers/perf_analysis_utils.py index ec74f206ce870..11b1cc18ca56d 100644 --- a/sdks/python/apache_beam/testing/analyzers/perf_analysis_utils.py +++ b/sdks/python/apache_beam/testing/analyzers/perf_analysis_utils.py @@ -14,10 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import abc import logging from dataclasses import asdict from dataclasses import dataclass -from typing import Any +from statistics import median from typing import Dict from typing import List from typing import Optional @@ -27,16 +28,15 @@ import pandas as pd import yaml from google.api_core import exceptions +from google.cloud import bigquery from apache_beam.testing.analyzers import constants -from apache_beam.testing.analyzers import github_issues_utils from apache_beam.testing.load_tests import load_test_metrics_utils -from apache_beam.testing.load_tests.load_test_metrics_utils import BigQueryMetricsFetcher from apache_beam.testing.load_tests.load_test_metrics_utils import BigQueryMetricsPublisher from signal_processing_algorithms.energy_statistics.energy_statistics import e_divisive -@dataclass +@dataclass(frozen=True) class GitHubIssueMetaData: """ This class holds metadata that needs to be published to the @@ -53,14 +53,60 @@ class GitHubIssueMetaData: change_point: float +@dataclass +class ChangePointConfig: + """ + This class holds the change point configuration parameters. + """ + min_runs_between_change_points: int = ( + constants._DEFAULT_MIN_RUNS_BETWEEN_CHANGE_POINTS) + num_runs_in_change_point_window: int = ( + constants._DEFAULT_NUM_RUMS_IN_CHANGE_POINT_WINDOW) + + +@dataclass +class TestConfigContainer: + metric_name: str + project: str + metrics_dataset: str + metrics_table: str + test_id: str # unique id for each test config. + test_description: str + test_name: Optional[str] = None + labels: Optional[List[str]] = None + + +@dataclass +class MetricContainer: + """ + This class holds the metric values and timestamps for a given metric. + Args: + metric_values: List of metric values. + timestamps: List of pandas timestamps corresponding to the metric values. + """ + + values: List[Union[int, float]] + timestamps: List[pd.Timestamp] + + def sort_by_timestamp(self, in_place=True): + """ + Sorts the metric values and timestamps in ascending order wrt timestamps. + Args: + in_place: If True, sort the metric values and timestamps in place. + """ + timestamps, values = zip(*sorted(zip(self.timestamps, self.values))) + if not in_place: + return MetricContainer(values=values, timestamps=timestamps) + self.timestamps, self.values = zip(*sorted( + zip(self.timestamps, self.values))) + + def is_change_point_in_valid_window( num_runs_in_change_point_window: int, latest_change_point_run: int) -> bool: return num_runs_in_change_point_window > latest_change_point_run -def get_existing_issues_data( - table_name: str, big_query_metrics_fetcher: BigQueryMetricsFetcher -) -> Optional[pd.DataFrame]: +def get_existing_issues_data(table_name: str) -> Optional[pd.DataFrame]: """ Finds the most recent GitHub issue created for the test_name. If no table found with name=test_name, return (None, None) @@ -72,20 +118,26 @@ def get_existing_issues_data( LIMIT 10 """ try: - df = big_query_metrics_fetcher.fetch(query=query) + client = bigquery.Client() + query_job = client.query(query=query) + existing_issue_data = query_job.result().to_dataframe() except exceptions.NotFound: # If no table found, that means this is first performance regression # on the current test+metric. return None - return df + return existing_issue_data -def is_perf_alert( +def is_sibling_change_point( previous_change_point_timestamps: List[pd.Timestamp], change_point_index: int, timestamps: List[pd.Timestamp], - min_runs_between_change_points: int) -> bool: + min_runs_between_change_points: int, + test_id: str, +) -> bool: """ + Sibling change points are the change points that are close to each other. + Search the previous_change_point_timestamps with current observed change point sibling window and determine if it is a duplicate change point or not. @@ -104,6 +156,18 @@ def is_perf_alert( for previous_change_point_timestamp in previous_change_point_timestamps: if (sibling_change_point_min_timestamp <= previous_change_point_timestamp <= sibling_change_point_max_timestamp): + logging.info( + 'Performance regression/improvement found for the test ID: %s. ' + 'Since the change point timestamp %s ' + 'lies within the sibling change point window: %s, ' + 'alert is not raised.' % ( + test_id, + previous_change_point_timestamp.strftime('%Y-%m-%d %H:%M:%S'), + ( + sibling_change_point_min_timestamp.strftime( + '%Y-%m-%d %H:%M:%S'), + sibling_change_point_max_timestamp.strftime( + '%Y-%m-%d %H:%M:%S')))) return False return True @@ -122,31 +186,8 @@ def validate_config(keys): return constants._PERF_TEST_KEYS.issubset(keys) -def fetch_metric_data( - params: Dict[str, Any], big_query_metrics_fetcher: BigQueryMetricsFetcher -) -> Tuple[List[Union[int, float]], List[pd.Timestamp]]: - """ - Args: - params: Dict containing keys required to fetch data from a data source. - big_query_metrics_fetcher: A BigQuery metrics fetcher for fetch metrics. - Returns: - Tuple[List[Union[int, float]], List[pd.Timestamp]]: Tuple containing list - of metric_values and list of timestamps. Both are sorted in ascending - order wrt timestamps. - """ - query = f""" - SELECT * - FROM {params['project']}.{params['metrics_dataset']}.{params['metrics_table']} - WHERE CONTAINS_SUBSTR(({load_test_metrics_utils.METRICS_TYPE_LABEL}), '{params['metric_name']}') - ORDER BY {load_test_metrics_utils.SUBMIT_TIMESTAMP_LABEL} DESC - LIMIT {constants._NUM_DATA_POINTS_TO_RUN_CHANGE_POINT_ANALYSIS} - """ - metric_data: pd.DataFrame = big_query_metrics_fetcher.fetch(query=query) - metric_data.sort_values( - by=[load_test_metrics_utils.SUBMIT_TIMESTAMP_LABEL], inplace=True) - return ( - metric_data[load_test_metrics_utils.VALUE_LABEL].tolist(), - metric_data[load_test_metrics_utils.SUBMIT_TIMESTAMP_LABEL].tolist()) +def find_change_points(metric_values: List[Union[float, int]]): + return e_divisive(metric_values) def find_latest_change_point_index(metric_values: List[Union[float, int]]): @@ -156,20 +197,43 @@ def find_latest_change_point_index(metric_values: List[Union[float, int]]): Returns: int: Right most change point index observed on metric_values. """ - change_points_idx = e_divisive(metric_values) - if not change_points_idx: - return None + change_points_indices = find_change_points(metric_values) + # reduce noise in the change point analysis by filtering out + # the change points that are not significant enough. + change_points_indices = filter_change_points_by_median_threshold( + metric_values, change_points_indices) # Consider the latest change point. - change_points_idx.sort() - return change_points_idx[-1] + if not change_points_indices: + return None + change_points_indices.sort() + # Remove the change points that are at the edges of the data. + # https://github.com/apache/beam/issues/28757 + # Remove this workaround once we have a good solution to deal + # with the edge change points. + change_point_index = change_points_indices[-1] + if is_edge_change_point(change_point_index, + len(metric_values), + constants._EDGE_SEGMENT_SIZE): + logging.info( + 'The change point %s is located at the edge of the data with an edge ' + 'segment size of %s. This change point will be ignored for now, ' + 'awaiting additional data. Should the change point persist after ' + 'gathering more data, an alert will be raised.' % + (change_point_index, constants._EDGE_SEGMENT_SIZE)) + return None + return change_point_index -def publish_issue_metadata_to_big_query(issue_metadata, table_name): +def publish_issue_metadata_to_big_query( + issue_metadata, + table_name, + project=constants._BQ_PROJECT_NAME, +): """ - Published issue_metadata to BigQuery with table name=test_name. + Published issue_metadata to BigQuery with table name. """ bq_metrics_publisher = BigQueryMetricsPublisher( - project_name=constants._BQ_PROJECT_NAME, + project_name=project, dataset=constants._BQ_DATASET, table=table_name, bq_schema=constants._SCHEMA) @@ -180,37 +244,122 @@ def publish_issue_metadata_to_big_query(issue_metadata, table_name): def create_performance_alert( - metric_name: str, - test_name: str, - timestamps: List[pd.Timestamp], - metric_values: List[Union[int, float]], + test_config_container: TestConfigContainer, + metric_container: MetricContainer, change_point_index: int, - labels: List[str], existing_issue_number: Optional[int], - test_target: Optional[str] = None) -> Tuple[int, str]: +) -> Tuple[int, str]: """ Creates performance alert on GitHub issues and returns GitHub issue number and issue URL. """ + # avoid circular imports + # pylint: disable=wrong-import-order, wrong-import-position + from apache_beam.testing.analyzers import github_issues_utils + description = github_issues_utils.get_issue_description( - test_name=( - test_name if not test_target else test_name + ':' + test_target), - metric_name=metric_name, - timestamps=timestamps, - metric_values=metric_values, + test_config_container=test_config_container, + metric_container=metric_container, change_point_index=change_point_index, max_results_to_display=( constants._NUM_RESULTS_TO_DISPLAY_ON_ISSUE_DESCRIPTION)) issue_number, issue_url = github_issues_utils.report_change_point_on_issues( title=github_issues_utils._ISSUE_TITLE_TEMPLATE.format( - test_name, metric_name + test_config_container.test_id, test_config_container.metric_name ), description=description, - labels=labels, + labels=test_config_container.labels, existing_issue_number=existing_issue_number) logging.info( 'Performance regression/improvement is alerted on issue #%s. Link ' ': %s' % (issue_number, issue_url)) return issue_number, issue_url + + +def filter_change_points_by_median_threshold( + data: List[Union[int, float]], + change_points: List[int], + threshold: float = 0.05, +): + """ + Reduces the number of change points by filtering out the ones that are + not significant enough based on the relative median threshold. Default + value of threshold is 0.05. + """ + valid_change_points = [] + epsilon = 1e-10 # needed to avoid division by zero. + + for idx in change_points: + if idx == 0 or idx == len(data): + continue + + left_segment = data[:idx] + right_segment = data[idx:] + + left_value = median(left_segment) + right_value = median(right_segment) + + relative_change = abs(right_value - left_value) / (left_value + epsilon) + + if relative_change > threshold: + valid_change_points.append(idx) + return valid_change_points + + +def is_edge_change_point( + change_point_index, + data_size, + edge_segment_size=constants._EDGE_SEGMENT_SIZE): + """ + Removes the change points that are at the edges of the data. + Args: + change_point_index: Index of the change point. + data_size: Size of the data. + edge_segment_size: Size of the edge segment. + """ + return change_point_index > data_size - edge_segment_size + + +class MetricsFetcher(metaclass=abc.ABCMeta): + @abc.abstractmethod + def fetch_metric_data( + self, *, test_config: TestConfigContainer) -> MetricContainer: + """ + Define SQL query and fetch the timestamp values and metric values + from BigQuery tables. + """ + raise NotImplementedError + + +class BigQueryMetricsFetcher(MetricsFetcher): + def fetch_metric_data( + self, *, test_config: TestConfigContainer) -> MetricContainer: + """ + Args: + test_config: TestConfigContainer containing metadata required to fetch + metric data from BigQuery. + Returns: + MetricContainer containing metric values and timestamps. + """ + project = test_config.project + metrics_dataset = test_config.metrics_dataset + metrics_table = test_config.metrics_table + metric_name = test_config.metric_name + query = f""" + SELECT * + FROM {project}.{metrics_dataset}.{metrics_table} + WHERE CONTAINS_SUBSTR(({load_test_metrics_utils.METRICS_TYPE_LABEL}), '{metric_name}') + ORDER BY {load_test_metrics_utils.SUBMIT_TIMESTAMP_LABEL} DESC + LIMIT {constants._NUM_DATA_POINTS_TO_RUN_CHANGE_POINT_ANALYSIS} + """ + client = bigquery.Client() + query_job = client.query(query=query) + metric_data = query_job.result().to_dataframe() + # metric_data.sort_values( + # by=[load_test_metrics_utils.SUBMIT_TIMESTAMP_LABEL], inplace=True) + return MetricContainer( + values=metric_data[load_test_metrics_utils.VALUE_LABEL].tolist(), + timestamps=metric_data[ + load_test_metrics_utils.SUBMIT_TIMESTAMP_LABEL].tolist()) diff --git a/sdks/python/apache_beam/testing/analyzers/tests_config.yaml b/sdks/python/apache_beam/testing/analyzers/tests_config.yaml index 02e649c7586fb..2e72cd5cc301f 100644 --- a/sdks/python/apache_beam/testing/analyzers/tests_config.yaml +++ b/sdks/python/apache_beam/testing/analyzers/tests_config.yaml @@ -15,64 +15,279 @@ # limitations under the License. # -test_1: - test_name: Pytorch image classification on 50k images of size 224 x 224 with resnet 152 - test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks +# for the unique key to define a test, please use the following format: +# {test_id}-{metric_name} + +pytorch_image_classification_benchmarks-resnet152-mean_inference_batch_latency_micro_secs: + test_description: | + Pytorch image classification on 50k images of size 224 x 224 with resnet 152. + Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L63 + Test dashboard - http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=2 + test_target: metrics_dataset: beam_run_inference metrics_table: torch_inference_imagenet_results_resnet152 project: apache-beam-testing + metric_name: mean_inference_batch_latency_micro_secs + +pytorch_image_classification_benchmarks-resnet101-mean_load_model_latency_milli_secs: + test_description: | + Pytorch image classification on 50k images of size 224 x 224 with resnet 101. + Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L34 + Test dashboard - http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=7 + test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks + metrics_dataset: beam_run_inference + metrics_table: torch_inference_imagenet_results_resnet101 + project: apache-beam-testing metric_name: mean_load_model_latency_milli_secs -test_2: - test_name: Pytorch image classification on 50k images of size 224 x 224 with resnet 152 +pytorch_image_classification_benchmarks-resnet101-mean_inference_batch_latency_micro_secs: + test_description: | + Pytorch image classification on 50k images of size 224 x 224 with resnet 101. + Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L34 + Test dashboard - http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=2 test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks metrics_dataset: beam_run_inference - metrics_table: torch_inference_imagenet_results_resnet152 + metrics_table: torch_inference_imagenet_results_resnet101 project: apache-beam-testing metric_name: mean_inference_batch_latency_micro_secs -test_3: - test_name: Pytorch image classification on 50k images of size 224 x 224 with resnet 101 +pytorch_image_classification_benchmarks-resnet152-GPU-mean_inference_batch_latency_micro_secs: + test_description: | + Pytorch image classification on 50k images of size 224 x 224 with resnet 152 with Tesla T4 GPU. + Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L151 + Test dashboard - http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=7 test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks metrics_dataset: beam_run_inference metrics_table: torch_inference_imagenet_results_resnet101 project: apache-beam-testing + metric_name: mean_inference_batch_latency_micro_secs + +pytorch_image_classification_benchmarks-resnet152-GPU-mean_load_model_latency_milli_secs: + test_description: | + Pytorch image classification on 50k images of size 224 x 224 with resnet 152 with Tesla T4 GPU. + Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L151 + Test dashboard - http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=7 + test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks + metrics_dataset: beam_run_inference + metrics_table: torch_inference_imagenet_results_resnet152_tesla_t4 + project: apache-beam-testing metric_name: mean_load_model_latency_milli_secs -test_4: - test_name: Pytorch image classification on 50k images of size 224 x 224 with resnet 101 +pytorch_image_classification_benchmarks-resnet152-GPU-mean_inference_batch_latency_micro_secs: + test_description: | + Pytorch image classification on 50k images of size 224 x 224 with resnet 152 with Tesla T4 GPU. + Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L151). + Test dashboard - http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?from=now-90d&to=now&viewPanel=2 test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks metrics_dataset: beam_run_inference - metrics_table: torch_inference_imagenet_results_resnet101 + metrics_table: torch_inference_imagenet_results_resnet152_tesla_t4 project: apache-beam-testing metric_name: mean_inference_batch_latency_micro_secs -test_5: - test_name: test_cloudml_benchmark_cirteo_no_shuffle_10GB +test_cloudml_benchmark_cirteo_no_shuffle_10GB-runtime_sec: + test_description: | + TFT Criteo test on 10 GB data with no Reshuffle. + Test link - [Test link](https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/sdks/python/apache_beam/testing/benchmarks/cloudml/cloudml_benchmark_test.py#L82) metrics_dataset: beam_cloudml metrics_table: cloudml_benchmark_cirteo_no_shuffle_10GB project: apache-beam-testing metric_name: runtime_sec -test_6: - test_name: test_cloudml_benchmark_criteo_10GB +test_cloudml_benchmark_criteo_10GB-runtime_sec: + test_description: | + TFT Criteo test on 10 GB data. + Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/sdks/python/apache_beam/testing/benchmarks/cloudml/cloudml_benchmark_test.py#LL104C7-L104C41 metrics_dataset: beam_cloudml metrics_table: cloudml_benchmark_criteo_10GB project: apache-beam-testing metric_name: runtime_sec -test_7: - test_name: Pytorch image classification on 50k images of size 224 x 224 with resnet 152 with Tesla T4 GPU - test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks - metrics_dataset: beam_run_inference - metrics_table: torch_inference_imagenet_results_resnet152_tesla_t4 +# Python Combine load tests at http://metrics.beam.apache.org/d/WNzYt13Zk/combine-load-tests?orgId=1 +combine_python_batch_2gb_10_byte_records: + test_description: | + Combine Python Load Test 2 GB 10 byte records + Test link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_Combine_Python.groovy#L76C24-L76C65 + Test dashboard - http://metrics.beam.apache.org/d/WNzYt13Zk/combine-load-tests?orgId=1&from=now-90d&to=now&var-processingType=batch&var-sdk=python&viewPanel=2 + test_target: apache_beam.testing.load_tests.combine_test + metrics_dataset: load_test + metrics_table: python_dataflow_batch_combine_1 + metric_name: runtime project: apache-beam-testing - metric_name: mean_inference_batch_latency_micro_secs -test_8: - test_name: Pytorch image classification on 50k images of size 224 x 224 with resnet 152 with Tesla T4 GPU - test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks - metrics_dataset: beam_run_inference - metrics_table: torch_inference_imagenet_results_resnet152_tesla_t4 +combine_python_batch_2gb_fanout_4: + test_description: | + Combine Python Load test - 2GB Fanout 4 + Test link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_Combine_Python.groovy#L52 + Test Dashboard - http://metrics.beam.apache.org/d/WNzYt13Zk/combine-load-tests?orgId=1&from=now-90d&to=now&var-processingType=batch&var-sdk=python&viewPanel=4 + test_target: apache_beam.testing.load_tests.combine_test + metrics_dataset: load_test + metrics_table: python_dataflow_batch_combine_4 + metric_name: runtime project: apache-beam-testing - metric_name: mean_load_model_latency_milli_secs + +combine_python_batch_2gb_fanout_8: + test_description: | + Combine Python Load test - 2GB Fanout 8 + test_target: apache_beam.testing.load_tests.combine_test + metrics_dataset: load_test + metrics_table: python_dataflow_batch_combine_5 + metric_name: runtime + project: apache-beam-testing + +# Python Batch GBK load tests at http://metrics.beam.apache.org/d/UYZ-oJ3Zk/gbk-load-tests?orgId=1&var-processingType=batch&var-sdk=python&from=now-90d&to=now +gbk_python_batch_load_test_2gb_of_10B_records: + test_description: | + GroupByKey Python Load test - 2GB of 10B records + python | GBK | Small records (10B) + Test Dashboard - http://metrics.beam.apache.org/d/UYZ-oJ3Zk/gbk-load-tests?orgId=1&var-processingType=batch&var-sdk=python&from=now-90d&to=now&viewPanel=2 + Test link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_GBK_Python.groovy#L36C25-L36C72 + test_target: apache_beam.testing.load_tests.group_by_key_test + metrics_table: python_dataflow_batch_gbk_1 + metrics_dataset: load_test + metric_name: runtime + project: apache-beam-testing + +gbk_python_batch_load_test_2gb_of_100B_records: + test_description: | + GroupByKey Python Load test - 2GB of 100B records + python | GBK | Medium records (100B) + Test Dashboard - http://metrics.beam.apache.org/d/UYZ-oJ3Zk/gbk-load-tests?orgId=1&var-processingType=batch&var-sdk=python&from=now-90d&to=now&viewPanel=3 + Test link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_GBK_Python.groovy#L60 + test_target: apache_beam.testing.load_tests.group_by_key_test + metrics_table: python_dataflow_batch_gbk_2 + metrics_dataset: load_test + metric_name: runtime + project: apache-beam-testing + +gbk_python_batch_load_test_2gb_of_100KB_records: + test_description: | + GroupByKey Python Load test - 2GB of 100kB records + python | GBK | Large records (100kB) + Test Dashboard - http://metrics.beam.apache.org/d/UYZ-oJ3Zk/gbk-load-tests?orgId=1&var-processingType=batch&var-sdk=python&from=now-6M&to=now&viewPanel=4&inspect=4 + Test link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_GBK_Python.groovy#L83 + test_target: apache_beam.testing.load_tests.group_by_key_test + metrics_table: python_dataflow_batch_gbk_3 + metrics_dataset: load_test + metric_name: runtime + project: apache-beam-testing + +gbk_python_batch_load_test_fanout_4_times_with_2GB_10byte_records_total: +# this test looks little noisy. Remove this if it causes too many false +# positives. + test_description: | + GroupByKey Python Load test - fanout 4 times with 2GB 10-byte records total + python | GBK | Fanout 4 + Test Dashboard - http://metrics.beam.apache.org/d/UYZ-oJ3Zk/gbk-load-tests?orgId=1&var-processingType=batch&var-sdk=python&from=now-90d&to=now&viewPanel=5 + Test link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_GBK_Python.groovy#L106 + test_target: apache_beam.testing.load_tests.group_by_key_test + metrics_table: python_dataflow_batch_gbk_4 + metrics_dataset: load_test + metric_name: runtime + project: apache-beam-testing + + +gbk_python_batch_load_test_fanout_8_times_with_2GB_10byte_records_total: +# this test looks little noisy. Remove this if it causes too many false +# positives. + test_description: | + GroupByKey Python Load test - fanout 8 times with 2GB 10-byte records total + python | GBK | Fanout 8 + Test Dashboard - http://metrics.beam.apache.org/d/UYZ-oJ3Zk/gbk-load-tests?orgId=1&var-processingType=batch&var-sdk=python&from=now-90d&to=now&viewPanel=6 + Test link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_GBK_Python.groovy#L129 + metrics_table: python_dataflow_batch_gbk_5 + metrics_dataset: load_test + metric_name: runtime + project: apache-beam-testing + +# Python SideInput load tests at http://metrics.beam.apache.org/d/-E9aGlFGk/side-input-load-tests?orgId=1&from=now-90d&to=now + +sideinpts_python_batch_1gb_1kb_10workers_1000window_1key_percent_dict: + test_description: | + python | Side Input | 1 GB dictionary, 1% of keys, 1000 fixed windows + Test Link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_SideInput_Python.groovy#L120 + Test Dashboard - http://metrics.beam.apache.org/d/-E9aGlFGk/side-input-load-tests?orgId=1&from=now-90d&to=now&viewPanel=8 + metrics_table: python_dataflow_batch_sideinput_7 + metrics_dataset: load_test + metric_name: runtime + project: apache-beam-testing + + +sideinpts_python_batch_1gb_1kb_10workers_1000window_99key_percent_dict: + test_description: | + python | Side Input | 1 GB dictionary, 99% of keys, 1000 fixed windows + Test Link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_SideInput_Python.groovy#L133 + Test Dashboard - http://metrics.beam.apache.org/d/-E9aGlFGk/side-input-load-tests?orgId=1&from=now-90d&to=now&viewPanel=9 + metrics_table: python_dataflow_batch_sideinput_8 + metrics_dataset: load_test + metric_name: runtime + project: apache-beam-testing + +sideinpts_python_batch_10gb_1kb_10workers_1000window_first_iterable: + test_description: | + python | Side Input | 10 GB iterable, 1% of elements, 1000 fixed windows + Test Link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_SideInput_Python.groovy#L146 + Test Dashboard - http://metrics.beam.apache.org/d/-E9aGlFGk/side-input-load-tests?orgId=1&from=now-90d&to=now&viewPanel=10 + metrics_table: python_dataflow_batch_sideinput_9 + metrics_dataset: load_test + metric_name: runtime + project: apache-beam-testing + + +sideinpts_python_batch_10gb_1kb_10workers_1000window_first_iterable: + test_description: | + python | Side Input | 10 GB iterable, all elements, 1000 fixed windows + Test Link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_SideInput_Python.groovy#L159 + Test Dashboard - http://metrics.beam.apache.org/d/-E9aGlFGk/side-input-load-tests?orgId=1&from=now-90d&to=now&viewPanel=11 + metrics_table: python_dataflow_batch_sideinput_10 + metrics_dataset: load_test + metric_name: runtime + project: apache-beam-testing + +# Python CoGBK load tests at http://metrics.beam.apache.org/d/fK0U4JqWz/cogbk-load-tests?orgId=1&var-processingType=batch&var-sdk=python + +cogbk_python_batch_load_test_2GB_of_100B_records_with_a_single_key: + test_description: | + CoGroupByKey Python Load test - 2GB of 100B records with a single key + python | coGBK | 100B records with a single key + Test Link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_coGBK_Python.groovy#L32C25-L32C76 + Test Dashboard - http://metrics.beam.apache.org/d/fK0U4JqWz/cogbk-load-tests?orgId=1&var-processingType=batch&var-sdk=python&viewPanel=2 + test_target: apache_beam.testing.load_tests.co_group_by_key_test + metrics_table: python_dataflow_batch_cogbk_1 + metrics_dataset: load_test + metric_name: runtime + project: apache-beam-testing + +cogbk_python_batch_load_test_2GB_of_100B_records_with_a_multiple_key: + test_description: | + CoGroupByKey Python Load test - 2GB of 100B records with multiple keys + python | coGBK | 100B records with multiple keys + + Test Link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_coGBK_Python.groovy#L64 + + Test Dashboard - http://metrics.beam.apache.org/d/fK0U4JqWz/cogbk-load-tests?orgId=1&var-processingType=batch&var-sdk=python&viewPanel=3 + metrics_table: python_dataflow_batch_cogbk_2 + metrics_dataset: load_test + metric_name: runtime + project: apache-beam-testing + +cogbk_python_batch_load_test_reiterate_4times_10KB_values: + test_description: | + CoGroupByKey Python Load test - reiterate 4 times 10kB values + python | coGBK | reiteration 10kB value + Test Link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_coGBK_Python.groovy#L96 + Test Dashboard - http://metrics.beam.apache.org/d/fK0U4JqWz/cogbk-load-tests?orgId=1&var-processingType=batch&var-sdk=python&viewPanel=4 + metrics_table: python_dataflow_batch_cogbk_3 + metrics_dataset: load_test + metric_name: runtime + project: apache-beam-testing + +cogbk_python_batch_load_test_reiterate_4times_2MB_values: + test_description: | + CoGroupByKey Python Load test - reiterate 4 times 2 MB values + python | coGBK | reiteration 2MB value + Test Link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_coGBK_Python.groovy#L128 + Test Dashboard - http://metrics.beam.apache.org/d/fK0U4JqWz/cogbk-load-tests?orgId=1&var-processingType=batch&var-sdk=python&viewPanel=5 + metrics_table: python_dataflow_batch_cogbk_4 + metrics_dataset: load_test + metric_name: runtime + project: apache-beam-testing \ No newline at end of file diff --git a/sdks/python/apache_beam/testing/benchmarks/cloudml/cloudml_benchmark_test.py b/sdks/python/apache_beam/testing/benchmarks/cloudml/cloudml_benchmark_test.py index 3e5a640c7aa4a..ea2d93512fd4b 100644 --- a/sdks/python/apache_beam/testing/benchmarks/cloudml/cloudml_benchmark_test.py +++ b/sdks/python/apache_beam/testing/benchmarks/cloudml/cloudml_benchmark_test.py @@ -31,7 +31,6 @@ raise unittest.SkipTest('Dependencies are not installed') _INPUT_GCS_BUCKET_ROOT = 'gs://apache-beam-ml/datasets/cloudml/criteo' -_CRITEO_FEATURES_FILE = 'testdata/criteo/expected/features.tfrecord.gz' _OUTPUT_GCS_BUCKET_ROOT = 'gs://temp-storage-for-end-to-end-tests/tft/' diff --git a/sdks/python/apache_beam/testing/benchmarks/nexmark/nexmark_launcher.py b/sdks/python/apache_beam/testing/benchmarks/nexmark/nexmark_launcher.py index 2296588ec4965..bdf6f476212db 100644 --- a/sdks/python/apache_beam/testing/benchmarks/nexmark/nexmark_launcher.py +++ b/sdks/python/apache_beam/testing/benchmarks/nexmark/nexmark_launcher.py @@ -420,7 +420,8 @@ def publish_performance_influxdb(self, query_num, perf): auth = HTTPBasicAuth(user, password) try: - response = requests.post(url, params=query_str, data=payload, auth=auth) + response = requests.post( + url, params=query_str, data=payload, auth=auth, timeout=60) except requests.exceptions.RequestException as e: logging.warning('Failed to publish metrics to InfluxDB: ' + str(e)) else: diff --git a/sdks/python/apache_beam/testing/datatype_inference_test.py b/sdks/python/apache_beam/testing/datatype_inference_test.py index 445ebedb4c4ef..001752f8ab276 100644 --- a/sdks/python/apache_beam/testing/datatype_inference_test.py +++ b/sdks/python/apache_beam/testing/datatype_inference_test.py @@ -50,7 +50,7 @@ OrderedDict([ ("a", 1), ("b", 0.12345), - ("c", u"Hello World!!"), + ("c", "Hello World!!"), ("d", np.array([1, 2, 3], dtype=np.int64)), ("e", b"some bytes"), ]), @@ -61,7 +61,7 @@ ]), OrderedDict([ ("a", 100000), - ("c", u"XoXoX"), + ("c", "XoXoX"), ("d", np.array([4, 5, 6], dtype=np.int64)), ("e", b""), ]), diff --git a/sdks/python/apache_beam/testing/extra_assertions_test.py b/sdks/python/apache_beam/testing/extra_assertions_test.py index 9867c79897426..174fb54e2fa8f 100644 --- a/sdks/python/apache_beam/testing/extra_assertions_test.py +++ b/sdks/python/apache_beam/testing/extra_assertions_test.py @@ -27,8 +27,8 @@ class ExtraAssertionsMixinTest(ExtraAssertionsMixin, unittest.TestCase): def test_assert_array_count_equal_strings(self): - data1 = [u"±♠Ωℑ", u"hello", "world"] - data2 = ["hello", u"±♠Ωℑ", u"world"] + data1 = ["±♠Ωℑ", "hello", "world"] + data2 = ["hello", "±♠Ωℑ", "world"] self.assertUnhashableCountEqual(data1, data2) def test_assert_array_count_equal_mixed(self): @@ -37,7 +37,7 @@ def test_assert_array_count_equal_mixed(self): 'a': 1, 123: 1.234 }, ['d', 1], - u"±♠Ωℑ", + "±♠Ωℑ", np.zeros((3, 6)), (1, 2, 3, 'b'), 'def', @@ -55,7 +55,7 @@ def test_assert_array_count_equal_mixed(self): None, 'abc', 'def', - u"±♠Ωℑ", + "±♠Ωℑ", 100, (1, 2, 3, 'b'), np.zeros((3, 6)), diff --git a/sdks/python/apache_beam/testing/load_tests/build.gradle b/sdks/python/apache_beam/testing/load_tests/build.gradle index 144f7d12ba6c3..538d4a01bfee5 100644 --- a/sdks/python/apache_beam/testing/load_tests/build.gradle +++ b/sdks/python/apache_beam/testing/load_tests/build.gradle @@ -59,7 +59,7 @@ task run(type: Exec, dependsOn: installGcpTest) { ignoreExitValue true doLast { - if (execResult.exitValue != 0) { + if (executionResult.get().exitValue != 0) { throw new GradleException('error occurred') } } diff --git a/sdks/python/apache_beam/testing/load_tests/load_test_metrics_utils.py b/sdks/python/apache_beam/testing/load_tests/load_test_metrics_utils.py index 92a5f68351fe0..1ff46a3f7d19b 100644 --- a/sdks/python/apache_beam/testing/load_tests/load_test_metrics_utils.py +++ b/sdks/python/apache_beam/testing/load_tests/load_test_metrics_utils.py @@ -38,7 +38,6 @@ from typing import Optional from typing import Union -import pandas as pd import requests from requests.auth import HTTPBasicAuth @@ -536,7 +535,8 @@ def publish(self, results): self.options.http_auth_enabled() else None try: - response = requests.post(url, params=query_str, data=payload, auth=auth) + response = requests.post( + url, params=query_str, data=payload, auth=auth, timeout=60) except requests.exceptions.RequestException as e: _LOGGER.warning('Failed to publish metrics to InfluxDB: ' + str(e)) else: @@ -650,13 +650,3 @@ def __init__(self): def process(self, element): yield self.timestamp_val_fn( element, self.timestamp_fn(micros=int(self.time_fn() * 1000000))) - - -class BigQueryMetricsFetcher: - def __init__(self): - self.client = bigquery.Client() - - def fetch(self, query) -> pd.DataFrame: - query_job = self.client.query(query=query) - result = query_job.result() - return result.to_dataframe() diff --git a/sdks/python/apache_beam/testing/util.py b/sdks/python/apache_beam/testing/util.py index 8c91812895999..10a7a8e86f94f 100644 --- a/sdks/python/apache_beam/testing/util.py +++ b/sdks/python/apache_beam/testing/util.py @@ -33,6 +33,7 @@ from apache_beam.transforms.core import ParDo from apache_beam.transforms.core import WindowInto from apache_beam.transforms.ptransform import PTransform +from apache_beam.transforms.ptransform import ptransform_fn from apache_beam.transforms.util import CoGroupByKey __all__ = [ @@ -308,6 +309,12 @@ def default_label(self): actual | AssertThat() # pylint: disable=expression-not-assigned +@ptransform_fn +def AssertThat(pcoll, *args, **kwargs): + """Like assert_that, but as an applicable PTransform.""" + return assert_that(pcoll, *args, **kwargs) + + def open_shards(glob_pattern, mode='rt', encoding='utf-8'): """Returns a composite file of all shards matching the given glob pattern. diff --git a/sdks/python/apache_beam/tools/OWNERS b/sdks/python/apache_beam/tools/OWNERS deleted file mode 100644 index 1a1f0f0d3e722..0000000000000 --- a/sdks/python/apache_beam/tools/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - pabloem - - charlesccychen diff --git a/sdks/python/apache_beam/tools/microbenchmarks_test.py b/sdks/python/apache_beam/tools/microbenchmarks_test.py index aacf05851e3aa..9c7d0074ebd49 100644 --- a/sdks/python/apache_beam/tools/microbenchmarks_test.py +++ b/sdks/python/apache_beam/tools/microbenchmarks_test.py @@ -20,9 +20,8 @@ # pytype: skip-file import unittest - -from pkg_resources import DistributionNotFound -from pkg_resources import get_distribution +from importlib.metadata import PackageNotFoundError +from importlib.metadata import distribution from apache_beam.tools import coders_microbenchmark from apache_beam.tools import utils @@ -37,9 +36,9 @@ def test_coders_microbenchmark(self): def is_cython_installed(self): try: - get_distribution('cython') + distribution('cython') return True - except DistributionNotFound: + except PackageNotFoundError: return False def test_check_compiled(self): diff --git a/sdks/python/apache_beam/transforms/OWNERS b/sdks/python/apache_beam/transforms/OWNERS deleted file mode 100644 index 20b39cbdd96b5..0000000000000 --- a/sdks/python/apache_beam/transforms/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - pabloem - - aaltay - - charlesccychen diff --git a/sdks/python/apache_beam/transforms/combinefn_lifecycle_test.py b/sdks/python/apache_beam/transforms/combinefn_lifecycle_test.py index 6834c1d6174b4..62dbbc5fb77cf 100644 --- a/sdks/python/apache_beam/transforms/combinefn_lifecycle_test.py +++ b/sdks/python/apache_beam/transforms/combinefn_lifecycle_test.py @@ -20,12 +20,10 @@ # pytype: skip-file import unittest -from functools import wraps import pytest from parameterized import parameterized_class -from apache_beam.options.pipeline_options import DebugOptions from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.options.pipeline_options import StandardOptions from apache_beam.runners.direct import direct_runner @@ -36,39 +34,17 @@ from apache_beam.transforms.combinefn_lifecycle_pipeline import run_pardo -def skip_unless_v2(fn): - @wraps(fn) - def wrapped(*args, **kwargs): - self = args[0] - options = self.pipeline.get_pipeline_options() - standard_options = options.view_as(StandardOptions) - experiments = options.view_as(DebugOptions).experiments or [] - - if 'DataflowRunner' in standard_options.runner and \ - 'use_runner_v2' not in experiments: - self.skipTest( - 'CombineFn.setup and CombineFn.teardown are not supported. ' - 'Please use Dataflow Runner V2.') - else: - return fn(*args, **kwargs) - - return wrapped - - @pytest.mark.it_validatesrunner class CombineFnLifecycleTest(unittest.TestCase): def setUp(self): self.pipeline = TestPipeline(is_integration_test=True) - @skip_unless_v2 def test_combine(self): run_combine(self.pipeline) - @skip_unless_v2 def test_non_liftable_combine(self): run_combine(self.pipeline, lift_combiners=False) - @skip_unless_v2 def test_combining_value_state(self): if ('DataflowRunner' in self.pipeline.get_pipeline_options().view_as( StandardOptions).runner): diff --git a/sdks/python/apache_beam/transforms/combiners.py b/sdks/python/apache_beam/transforms/combiners.py index 0ebf698260150..9fea1d6a4a98f 100644 --- a/sdks/python/apache_beam/transforms/combiners.py +++ b/sdks/python/apache_beam/transforms/combiners.py @@ -317,7 +317,7 @@ def LargestPerKey(pcoll, n, key=None): @staticmethod @ptransform.ptransform_fn - def SmallestPerKey(pcoll, n, *, key=None, reverse=None): + def SmallestPerKey(pcoll, n, *, key=None): """Identifies the N least elements associated with each key.""" return pcoll | Top.PerKey(n, key, reverse=True) diff --git a/sdks/python/apache_beam/transforms/combiners_test.py b/sdks/python/apache_beam/transforms/combiners_test.py index 385b3332e0c4c..a8979239f8312 100644 --- a/sdks/python/apache_beam/transforms/combiners_test.py +++ b/sdks/python/apache_beam/transforms/combiners_test.py @@ -20,6 +20,7 @@ import itertools import random +import time import unittest import hamcrest as hc @@ -44,6 +45,7 @@ from apache_beam.transforms.core import Map from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display_test import DisplayDataItemMatcher +from apache_beam.transforms.periodicsequence import PeriodicImpulse from apache_beam.transforms.ptransform import PTransform from apache_beam.transforms.trigger import AfterAll from apache_beam.transforms.trigger import AfterCount @@ -977,5 +979,47 @@ def test_combiner_latest(self): label='assert per window') +class CombineGloballyTest(unittest.TestCase): + def test_combine_globally_for_unbounded_source_with_default(self): + # this error is logged since the below combination is ill-defined. + with self.assertLogs() as captured_logs: + with TestPipeline() as p: + _ = ( + p + | PeriodicImpulse( + start_timestamp=time.time(), + stop_timestamp=time.time() + 4, + fire_interval=1, + apply_windowing=False, + ) + | beam.Map(lambda x: ('c', 1)) + | beam.WindowInto( + window.GlobalWindows(), + trigger=trigger.Repeatedly(trigger.AfterCount(2)), + accumulation_mode=trigger.AccumulationMode.DISCARDING, + ) + | beam.combiners.Count.Globally()) + self.assertIn('unbounded collections', '\n'.join(captured_logs.output)) + + def test_combine_globally_for_unbounded_source_without_defaults(self): + # this is the supported case + with TestPipeline() as p: + _ = ( + p + | PeriodicImpulse( + start_timestamp=time.time(), + stop_timestamp=time.time() + 4, + fire_interval=1, + apply_windowing=False, + ) + | beam.Map(lambda x: 1) + | beam.WindowInto( + window.GlobalWindows(), + trigger=trigger.Repeatedly(trigger.AfterCount(2)), + accumulation_mode=trigger.AccumulationMode.DISCARDING, + ) + | beam.CombineGlobally(sum).without_defaults()) + + if __name__ == '__main__': unittest.main() diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py index 95c2617a5a08b..e980dccea7444 100644 --- a/sdks/python/apache_beam/transforms/core.py +++ b/sdks/python/apache_beam/transforms/core.py @@ -2182,6 +2182,9 @@ def expand(self, pcoll): *self._args, **self._kwargs).with_outputs( self._dead_letter_tag, main=self._main_tag, allow_unknown_tags=True) + #TODO(BEAM-18957): Fix when type inference supports tagged outputs. + result[self._main_tag].element_type = self._fn.infer_output_type( + pcoll.element_type) if self._threshold < 1.0: @@ -2244,6 +2247,104 @@ def process(self, *args, **kwargs): traceback.format_exception(*sys.exc_info())))) +# Idea adapted from https://github.com/tosun-si/asgarde. +# TODO(robertwb): Consider how this could fit into the public API. +# TODO(robertwb): Generalize to all PValue types. +class _PValueWithErrors(object): + """This wraps a PCollection such that transforms can be chained in a linear + manner while still accumulating any errors.""" + def __init__(self, pcoll, exception_handling_args, upstream_errors=()): + self._pcoll = pcoll + self._exception_handling_args = exception_handling_args + self._upstream_errors = upstream_errors + + @property + def element_type(self): + return self._pcoll.element_type + + @element_type.setter + def element_type(self, value): + self._pcoll.element_type = value + + def main_output_tag(self): + return self._exception_handling_args.get('main_tag', 'good') + + def error_output_tag(self): + return self._exception_handling_args.get('dead_letter_tag', 'bad') + + def __or__(self, transform): + return self.apply(transform) + + def apply(self, transform): + if hasattr(transform, 'with_exception_handling'): + result = self._pcoll | transform.with_exception_handling( + **self._exception_handling_args) + if result[self.main_output_tag()].element_type == typehints.Any: + result[ + self.main_output_tag()].element_type = transform.infer_output_type( + self._pcoll.element_type) + # TODO(BEAM-18957): Add support for tagged type hints. + result[self.error_output_tag()].element_type = typehints.Any + return _PValueWithErrors( + result[self.main_output_tag()], + self._exception_handling_args, + self._upstream_errors + (result[self.error_output_tag()], )) + else: + return _PValueWithErrors( + self._pcoll | transform, + self._exception_handling_args, + self._upstream_errors) + + def accumulated_errors(self): + if len(self._upstream_errors) == 1: + return self._upstream_errors[0] + else: + return self._upstream_errors | Flatten() + + def as_result(self, error_post_processing=None): + return { + self.main_output_tag(): self._pcoll, + self.error_output_tag(): self.accumulated_errors() + if error_post_processing is None else self.accumulated_errors() + | error_post_processing, + } + + +class _MaybePValueWithErrors(object): + """This is like _PValueWithErrors, but only wraps values if + exception_handling_args is non-trivial. It is useful for handling + error-catching and non-error-catching code in a uniform manner. + """ + def __init__(self, pvalue, exception_handling_args=None): + if isinstance(pvalue, _PValueWithErrors): + assert exception_handling_args is None + self._pvalue = pvalue + elif exception_handling_args is None: + self._pvalue = pvalue + else: + self._pvalue = _PValueWithErrors(pvalue, exception_handling_args) + + @property + def element_type(self): + return self._pvalue.element_type + + @element_type.setter + def element_type(self, value): + self._pvalue.element_type = value + + def __or__(self, transform): + return self.apply(transform) + + def apply(self, transform): + return _MaybePValueWithErrors(self._pvalue | transform) + + def as_result(self, error_post_processing=None): + if isinstance(self._pvalue, _PValueWithErrors): + return self._pvalue.as_result(error_post_processing) + else: + return self._pvalue + + class _SubprocessDoFn(DoFn): """Process method run in a subprocess, turning hard crashes into exceptions. """ @@ -2587,6 +2688,15 @@ def add_input_types(transform): "or CombineGlobally().as_singleton_view() to get the default " "output of the CombineFn if the input PCollection is empty.") + # log the error for this ill-defined streaming case now + if not pcoll.is_bounded and not pcoll.windowing.is_default(): + _LOGGER.error( + "When combining elements in unbounded collections with " + "the non-default windowing strategy, you must explicitly " + "specify how to define the combined result of an empty window. " + "Please use CombineGlobally().without_defaults() to output " + "an empty PCollection if the input PCollection is empty.") + def typed(transform): # TODO(robertwb): We should infer this. if combined.element_type: @@ -2598,6 +2708,11 @@ def typed(transform): def inject_default(_, combined): if combined: + if len(combined) > 1: + _LOGGER.error( + "Multiple combined values unexpectedly provided" + " for a global combine: %s", + combined) assert len(combined) == 1 return combined[0] else: @@ -3232,14 +3347,21 @@ def __init__( _expr_to_callable(expr, ix)) for (ix, expr) in enumerate(args) ] + [(name, _expr_to_callable(expr, name)) for (name, expr) in kwargs.items()] + self._exception_handling_args = None + + def with_exception_handling(self, **kwargs): + self._exception_handling_args = kwargs + return self def default_label(self): return 'ToRows(%s)' % ', '.join(name for name, _ in self._fields) def expand(self, pcoll): - return pcoll | Map( - lambda x: pvalue.Row(**{name: expr(x) - for name, expr in self._fields})) + return ( + _MaybePValueWithErrors(pcoll, self._exception_handling_args) | Map( + lambda x: pvalue.Row( + **{name: expr(x) + for name, expr in self._fields}))).as_result() def infer_output_type(self, input_type): return row_type.RowTypeConstraint.from_fields([ @@ -3430,6 +3552,9 @@ def process( new_windows = self.windowing.windowfn.assign(context) yield WindowedValue(element, context.timestamp, new_windows) + def infer_output_type(self, input_type): + return input_type + def __init__( self, windowfn, # type: typing.Union[Windowing, WindowFn] @@ -3693,7 +3818,8 @@ def _strip_output_annotations(type_hint): contains_annotation = False def visitor(t, unused_args): - if t in annotations: + if t in annotations or (hasattr(t, '__name__') and + t.__name__ == TimestampedValue.__name__): raise StopIteration try: diff --git a/sdks/python/apache_beam/transforms/display.py b/sdks/python/apache_beam/transforms/display.py index a8f60a894d6e3..0d1dd552413e2 100644 --- a/sdks/python/apache_beam/transforms/display.py +++ b/sdks/python/apache_beam/transforms/display.py @@ -45,6 +45,7 @@ from datetime import timedelta from typing import TYPE_CHECKING from typing import List +from typing import Union from apache_beam.portability import common_urns from apache_beam.portability.api import beam_runner_api_pb2 @@ -101,7 +102,8 @@ def __init__( ): # type: (...) -> None self.namespace = namespace - self.items = [] # type: List[DisplayDataItem] + self.items = [ + ] # type: List[Union[DisplayDataItem, beam_runner_api_pb2.DisplayData]] self._populate_items(display_data_dict) def _populate_items(self, display_data_dict): @@ -112,26 +114,31 @@ def _populate_items(self, display_data_dict): subcomponent_display_data = DisplayData( element._get_display_data_namespace(), element.display_data()) self.items += subcomponent_display_data.items - continue - if isinstance(element, DisplayDataItem): + elif isinstance(element, DisplayDataItem): if element.should_drop(): continue element.key = key element.namespace = self.namespace self.items.append(element) - continue - # If it's not a HasDisplayData element, - # nor a dictionary, then it's a simple value - self.items.append( - DisplayDataItem(element, namespace=self.namespace, key=key)) + elif isinstance(element, beam_runner_api_pb2.DisplayData): + self.items.append(element) + + else: + # If it's not a HasDisplayData element, + # nor a dictionary, then it's a simple value + self.items.append( + DisplayDataItem(element, namespace=self.namespace, key=key)) def to_proto(self): # type: (...) -> List[beam_runner_api_pb2.DisplayData] """Returns a List of Beam proto representation of Display data.""" def create_payload(dd): + if isinstance(dd, beam_runner_api_pb2.DisplayData): + return dd + display_data_dict = None try: display_data_dict = dd.get_dict() @@ -184,7 +191,7 @@ def create_payload(dd): dd_protos.append( beam_runner_api_pb2.DisplayData( urn=common_urns.StandardDisplayData.DisplayData.LABELLED.urn, - payload=create_payload(dd).SerializeToString())) + payload=dd_proto.SerializeToString())) return dd_protos @classmethod diff --git a/sdks/python/apache_beam/transforms/display_test.py b/sdks/python/apache_beam/transforms/display_test.py index a7605b09a3ad9..c91ad41e8d1c0 100644 --- a/sdks/python/apache_beam/transforms/display_test.py +++ b/sdks/python/apache_beam/transforms/display_test.py @@ -165,7 +165,7 @@ class MyDoFn(beam.DoFn): def display_data(self): return { 'unicode_string': 'my string', - 'unicode_literal_string': u'my literal string' + 'unicode_literal_string': 'my literal string' } fn = MyDoFn() diff --git a/sdks/python/apache_beam/transforms/environments.py b/sdks/python/apache_beam/transforms/environments.py index f6cbc7047738e..109fcb825347a 100644 --- a/sdks/python/apache_beam/transforms/environments.py +++ b/sdks/python/apache_beam/transforms/environments.py @@ -45,19 +45,18 @@ from google.protobuf import message from apache_beam import coders +from apache_beam.options.pipeline_options import PortableOptions from apache_beam.options.pipeline_options import SetupOptions from apache_beam.portability import common_urns from apache_beam.portability import python_urns from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.portability.api import endpoints_pb2 from apache_beam.runners.portability import stager -from apache_beam.runners.portability.sdk_container_builder import SdkContainerImageBuilder from apache_beam.transforms.resources import resource_hints_from_options from apache_beam.utils import proto_utils if TYPE_CHECKING: from apache_beam.options.pipeline_options import PipelineOptions - from apache_beam.options.pipeline_options import PortableOptions from apache_beam.runners.pipeline_context import PipelineContext __all__ = [ @@ -257,7 +256,26 @@ def from_options(cls, options): Args: options: The PortableOptions object. """ - raise NotImplementedError + if cls != Environment: + raise NotImplementedError + + portable_options = options.view_as(PortableOptions) + environment_type = portable_options.environment_type + if not environment_type: + environment_urn = common_urns.environments.DOCKER.urn + elif environment_type.startswith('beam:env:'): + environment_urn = environment_type + elif environment_type == 'LOOPBACK': + environment_urn = python_urns.EMBEDDED_PYTHON_LOOPBACK + else: + try: + environment_urn = getattr( + common_urns.environments, environment_type).urn + except AttributeError: + raise ValueError('Unknown environment type: %s' % environment_type) + + env_class = Environment.get_env_cls_from_urn(environment_urn) + return env_class.from_options(portable_options) # type: ignore @Environment.register_urn(common_urns.environments.DEFAULT.urn, None) @@ -337,6 +355,9 @@ def from_runner_api_parameter(payload, # type: beam_runner_api_pb2.DockerPayloa def from_options(cls, options): # type: (PortableOptions) -> DockerEnvironment if options.view_as(SetupOptions).prebuild_sdk_container_engine: + # Imported here to avoid circular dependencies. + # pylint: disable=wrong-import-order, wrong-import-position + from apache_beam.runners.portability.sdk_container_builder import SdkContainerImageBuilder prebuilt_container_image = SdkContainerImageBuilder.build_container_image( options) return cls.from_container_image( @@ -699,6 +720,27 @@ def default(cls): return cls(capabilities=python_sdk_capabilities(), artifacts=()) +@Environment.register_urn(python_urns.EMBEDDED_PYTHON_LOOPBACK, None) +class PythonLoopbackEnvironment(EmbeddedPythonEnvironment): + """Used as a stub when the loopback worker has not yet been started.""" + def to_runner_api_parameter(self, context): + # type: (PipelineContext) -> Tuple[str, None] + return python_urns.EMBEDDED_PYTHON_LOOPBACK, None + + @staticmethod + def from_runner_api_parameter(unused_payload, # type: None + capabilities, # type: Iterable[str] + artifacts, # type: Iterable[beam_runner_api_pb2.ArtifactInformation] + resource_hints, # type: Mapping[str, bytes] + context # type: PipelineContext + ): + # type: (...) -> PythonLoopbackEnvironment + return PythonLoopbackEnvironment( + capabilities=capabilities, + artifacts=artifacts, + resource_hints=resource_hints) + + @Environment.register_urn(python_urns.SUBPROCESS_SDK, bytes) class SubprocessSDKEnvironment(Environment): def __init__( diff --git a/sdks/python/apache_beam/transforms/environments_test.py b/sdks/python/apache_beam/transforms/environments_test.py index da55567a81065..25885553b2ff5 100644 --- a/sdks/python/apache_beam/transforms/environments_test.py +++ b/sdks/python/apache_beam/transforms/environments_test.py @@ -53,7 +53,7 @@ def test_environment_encoding(self): EmbeddedPythonGrpcEnvironment(), EmbeddedPythonGrpcEnvironment( state_cache_size=0, data_buffer_time_limit_ms=0), - SubprocessSDKEnvironment(command_string=u'foö')): + SubprocessSDKEnvironment(command_string='foö')): context = pipeline_context.PipelineContext() proto = environment.to_runner_api(context) reconstructed = Environment.from_runner_api(proto, context) diff --git a/sdks/python/apache_beam/transforms/external.py b/sdks/python/apache_beam/transforms/external.py index 104fc4c70142d..44bf2398a6dd4 100644 --- a/sdks/python/apache_beam/transforms/external.py +++ b/sdks/python/apache_beam/transforms/external.py @@ -23,7 +23,9 @@ import functools import glob import logging +import subprocess import threading +import uuid from collections import OrderedDict from collections import namedtuple from typing import Dict @@ -32,6 +34,7 @@ from apache_beam import pvalue from apache_beam.coders import RowCoder +from apache_beam.options.pipeline_options import CrossLanguageOptions from apache_beam.portability import common_urns from apache_beam.portability.api import beam_artifact_api_pb2_grpc from apache_beam.portability.api import beam_expansion_api_pb2 @@ -51,6 +54,7 @@ from apache_beam.typehints.typehints import Union from apache_beam.typehints.typehints import UnionConstraint from apache_beam.utils import subprocess_server +from apache_beam.utils import transform_service_launcher DEFAULT_EXPANSION_SERVICE = 'localhost:8097' @@ -181,6 +185,14 @@ def __init__(self, identifier, **kwargs): self._identifier = identifier self._kwargs = kwargs + def identifier(self): + """ + The URN referencing this SchemaTransform + + :return: str + """ + return self._identifier + def build(self): schema_proto, payload = self._get_schema_proto_and_payload(**self._kwargs) payload = external_transforms_pb2.SchemaTransformPayload( @@ -190,6 +202,56 @@ def build(self): return payload +class ExplicitSchemaTransformPayloadBuilder(SchemaTransformPayloadBuilder): + def __init__(self, identifier, schema_proto, **kwargs): + self._identifier = identifier + self._schema_proto = schema_proto + self._kwargs = kwargs + + def build(self): + def dict_to_row_recursive(field_type, py_value): + if py_value is None: + return None + type_info = field_type.WhichOneof('type_info') + if type_info == 'row_type': + return dict_to_row(field_type.row_type.schema, py_value) + elif type_info == 'array_type': + return [ + dict_to_row_recursive(field_type.array_type.element_type, value) + for value in py_value + ] + elif type_info == 'map_type': + return { + key: dict_to_row_recursive(field_type.map_type.value_type, value) + for key, + value in py_value.items() + } + else: + return py_value + + def dict_to_row(schema_proto, py_value): + row_type = named_tuple_from_schema(schema_proto) + if isinstance(py_value, dict): + extra = set(py_value.keys()) - set(row_type._fields) + if extra: + raise ValueError( + f"Unknown fields: {extra}. Valid fields: {row_type._fields}") + return row_type( + *[ + dict_to_row_recursive( + field.type, py_value.get(field.name, None)) + for field in schema_proto.fields + ]) + else: + return row_type(py_value) + + return external_transforms_pb2.SchemaTransformPayload( + identifier=self._identifier, + configuration_schema=self._schema_proto, + configuration_row=RowCoder(self._schema_proto).encode( + dict_to_row(self._schema_proto, self._kwargs))) + + class JavaClassLookupPayloadBuilder(PayloadBuilder): """ Builds a payload for directly instantiating a Java transform using a @@ -347,60 +409,50 @@ def __init__( _kwargs = kwargs if rearrange_based_on_discovery: - _kwargs = self._rearrange_kwargs(identifier) - - self._payload_builder = SchemaTransformPayloadBuilder(identifier, **_kwargs) - - def _rearrange_kwargs(self, identifier): - # discover and fetch the external SchemaTransform configuration then - # use it to build an appropriate payload - schematransform_config = SchemaAwareExternalTransform.discover_config( - self._expansion_service, identifier) - - external_config_fields = schematransform_config.configuration_schema._fields - ordered_kwargs = OrderedDict() - missing_fields = [] - - for field in external_config_fields: - if field not in self._kwargs: - missing_fields.append(field) - else: - ordered_kwargs[field] = self._kwargs[field] - - extra_fields = list(set(self._kwargs.keys()) - set(external_config_fields)) - if missing_fields: - raise ValueError( - 'Input parameters are missing the following SchemaTransform config ' - 'fields: %s' % missing_fields) - elif extra_fields: - raise ValueError( - 'Input parameters include the following extra fields that are not ' - 'found in the SchemaTransform config schema: %s' % extra_fields) + config = SchemaAwareExternalTransform.discover_config( + self._expansion_service, identifier) + self._payload_builder = ExplicitSchemaTransformPayloadBuilder( + identifier, + named_tuple_to_schema(config.configuration_schema), + **_kwargs) - return ordered_kwargs + else: + self._payload_builder = SchemaTransformPayloadBuilder( + identifier, **_kwargs) def expand(self, pcolls): # Expand the transform using the expansion service. - return pcolls | ExternalTransform( + return pcolls | self._payload_builder.identifier() >> ExternalTransform( common_urns.schematransform_based_expand.urn, self._payload_builder, self._expansion_service) - @staticmethod - def discover(expansion_service): + @classmethod + @functools.lru_cache + def discover(cls, expansion_service, ignore_errors=False): """Discover all SchemaTransforms available to the given expansion service. :return: a list of SchemaTransformsConfigs that represent the discovered SchemaTransforms. """ + return list(cls.discover_iter(expansion_service, ignore_errors)) + @staticmethod + def discover_iter(expansion_service, ignore_errors=True): with ExternalTransform.service(expansion_service) as service: discover_response = service.DiscoverSchemaTransform( beam_expansion_api_pb2.DiscoverSchemaTransformRequest()) for identifier in discover_response.schema_transform_configs: proto_config = discover_response.schema_transform_configs[identifier] - schema = named_tuple_from_schema(proto_config.config_schema) + try: + schema = named_tuple_from_schema(proto_config.config_schema) + except Exception as exn: + if ignore_errors: + logging.info("Bad schema for %s: %s", identifier, str(exn)[:250]) + continue + else: + raise yield SchemaTransformsConfig( identifier=identifier, @@ -420,7 +472,8 @@ def discover_config(expansion_service, name): are discovered """ - schematransforms = SchemaAwareExternalTransform.discover(expansion_service) + schematransforms = SchemaAwareExternalTransform.discover( + expansion_service, ignore_errors=True) matched = [] for st in schematransforms: @@ -668,7 +721,10 @@ def expand(self, pvalueish): transform=transform_proto, output_coder_requests=output_coders) - with ExternalTransform.service(self._expansion_service) as service: + expansion_service = _maybe_use_transform_service( + self._expansion_service, pipeline.options) + + with ExternalTransform.service(expansion_service) as service: response = service.Expand(request) if response.error: raise RuntimeError(response.error) @@ -878,6 +934,9 @@ def __init__( self._service_count = 0 self._append_args = append_args or [] + def is_existing_service(self): + return subprocess_server.is_service_endpoint(self._path_to_jar) + @staticmethod def _expand_jars(jar): if glob.glob(jar): @@ -973,6 +1032,76 @@ def __init__( path_to_jar, extra_args, classpath=classpath, append_args=append_args) +def _maybe_use_transform_service(provided_service=None, options=None): + # For anything other than 'JavaJarExpansionService' we just use the + # provided service. For example, string address of an already available + # service. + if not isinstance(provided_service, JavaJarExpansionService): + return provided_service + + if provided_service.is_existing_service(): + # This is an existing service supported through the 'beam_services' + # PipelineOption. + return provided_service + + def is_java_available(): + cmd = ['java', '--version'] + + try: + subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except: # pylint: disable=bare-except + return False + + return True + + def is_docker_available(): + cmd = ['docker', '--version'] + + try: + subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except: # pylint: disable=bare-except + return False + + return True + + # We try java and docker based expansion services in that order. + + java_available = is_java_available() + docker_available = is_docker_available() + + use_transform_service = options.view_as( + CrossLanguageOptions).use_transform_service + + if (java_available and provided_service and not use_transform_service): + return provided_service + elif docker_available: + if use_transform_service: + error_append = 'it was explicitly requested' + elif not java_available: + error_append = 'the Java executable is not available in the system' + else: + error_append = 'a Java expansion service was not provided.' + + project_name = str(uuid.uuid4()) + port = subprocess_server.pick_port(None)[0] + + logging.info( + 'Trying to expand the external transform using the Docker Compose ' + 'based transform service since %s. Transform service will be under ' + 'Docker Compose project name %s and will be made available at port %r.' + % (error_append, project_name, str(port))) + + from apache_beam import version as beam_version + beam_version = beam_version.__version__ + + return transform_service_launcher.TransformServiceLauncher( + project_name, port, beam_version) + else: + raise ValueError( + 'Cannot start an expansion service since neither Java nor ' + 'Docker executables are available in the system.') + + def memoize(func): cache = {} diff --git a/sdks/python/apache_beam/transforms/external_java.py b/sdks/python/apache_beam/transforms/external_java.py index f0a963864c1d1..29fc2587542bd 100644 --- a/sdks/python/apache_beam/transforms/external_java.py +++ b/sdks/python/apache_beam/transforms/external_java.py @@ -139,7 +139,7 @@ def run_pipeline(pipeline_options, expansion_service, wait_until_finish=True): | beam.Map(str) | beam.ExternalTransform( TEST_FILTER_URN, - ImplicitSchemaPayloadBuilder({'data': u'middle'}), + ImplicitSchemaPayloadBuilder({'data': 'middle'}), expansion_service) | beam.ExternalTransform(TEST_COUNT_URN, None, expansion_service) | beam.Map(lambda kv: '%s: %s' % kv)) diff --git a/sdks/python/apache_beam/transforms/external_test.py b/sdks/python/apache_beam/transforms/external_test.py index e7e4de46eb255..b0e39b7d9a5b1 100644 --- a/sdks/python/apache_beam/transforms/external_test.py +++ b/sdks/python/apache_beam/transforms/external_test.py @@ -71,10 +71,10 @@ class PayloadBase(object): values = { 'integer_example': 1, 'boolean': True, - 'string_example': u'thing', - 'list_of_strings': [u'foo', u'bar'], + 'string_example': 'thing', + 'list_of_strings': ['foo', 'bar'], 'mapping': { - u'key': 1.1 + 'key': 1.1 }, 'optional_integer': None, } @@ -182,7 +182,7 @@ def test_pipeline_generation(self): | beam.Create(['a', 'b']) | beam.ExternalTransform( 'beam:transforms:xlang:test:prefix', - ImplicitSchemaPayloadBuilder({'data': u'0'}), + ImplicitSchemaPayloadBuilder({'data': '0'}), expansion_service.ExpansionServiceServicer())) proto, _ = pipeline.to_runner_api(return_context=True) @@ -196,22 +196,21 @@ def test_pipeline_generation(self): self.assertNotEqual([], pipeline_from_proto.transforms_stack[0].parts[1].parts) self.assertEqual( - u'ExternalTransform(beam:transforms:xlang:test:prefix)/TestLabel', + 'ExternalTransform(beam:transforms:xlang:test:prefix)/TestLabel', pipeline_from_proto.transforms_stack[0].parts[1].parts[0].full_label) @unittest.skipIf(apiclient is None, 'GCP dependencies are not installed') def test_pipeline_generation_with_runner_overrides(self): pipeline_properties = [ - '--dataflow_endpoint=ignored', '--job_name=test-job', '--project=test-project', - '--staging_location=ignored', - '--temp_location=/dev/null', + '--temp_location=gs://beam/tmp', '--no_auth', '--dry_run=True', '--sdk_location=container', '--runner=DataflowRunner', - '--streaming' + '--streaming', + '--region=us-central1' ] with beam.Pipeline(options=PipelineOptions(pipeline_properties)) as p: @@ -222,7 +221,7 @@ def test_pipeline_generation_with_runner_overrides(self): 'projects/dummy-project/subscriptions/dummy-subscription') | beam.ExternalTransform( 'beam:transforms:xlang:test:prefix', - ImplicitSchemaPayloadBuilder({'data': u'0'}), + ImplicitSchemaPayloadBuilder({'data': '0'}), expansion_service.ExpansionServiceServicer())) pipeline_proto, _ = p.to_runner_api(return_context=True) @@ -294,7 +293,7 @@ def test_external_empty_spec_translation(self): pipeline = beam.Pipeline() external_transform = beam.ExternalTransform( 'beam:transforms:xlang:test:prefix', - ImplicitSchemaPayloadBuilder({'data': u'0'}), + ImplicitSchemaPayloadBuilder({'data': '0'}), expansion_service.ExpansionServiceServicer()) _ = (pipeline | beam.Create(['a', 'b']) | external_transform) pipeline.run().wait_until_finish() @@ -337,7 +336,7 @@ def test_external_transform_finder_non_leaf(self): | beam.Create(['a', 'b']) | beam.ExternalTransform( 'beam:transforms:xlang:test:prefix', - ImplicitSchemaPayloadBuilder({'data': u'0'}), + ImplicitSchemaPayloadBuilder({'data': '0'}), expansion_service.ExpansionServiceServicer()) | beam.Map(lambda x: x)) pipeline.run().wait_until_finish() @@ -351,7 +350,7 @@ def test_external_transform_finder_leaf(self): | beam.Create(['a', 'b']) | beam.ExternalTransform( 'beam:transforms:xlang:test:nooutput', - ImplicitSchemaPayloadBuilder({'data': u'0'}), + ImplicitSchemaPayloadBuilder({'data': '0'}), expansion_service.ExpansionServiceServicer())) pipeline.run().wait_until_finish() @@ -529,15 +528,19 @@ def test_rearrange_kwargs_based_on_discovery(self, mock_service): kwargs = {"int_field": 0, "str_field": "str"} transform = beam.SchemaAwareExternalTransform( - identifier=identifier, expansion_service=expansion_service, **kwargs) - ordered_kwargs = transform._rearrange_kwargs(identifier) + identifier=identifier, + expansion_service=expansion_service, + rearrange_based_on_discovery=True, + **kwargs) + payload = transform._payload_builder.build() + ordered_fields = [f.name for f in payload.configuration_schema.fields] schematransform_config = beam.SchemaAwareExternalTransform.discover_config( expansion_service, identifier) external_config_fields = schematransform_config.configuration_schema._fields self.assertNotEqual(tuple(kwargs.keys()), external_config_fields) - self.assertEqual(tuple(ordered_kwargs.keys()), external_config_fields) + self.assertEqual(tuple(ordered_fields), external_config_fields) class JavaClassLookupPayloadBuilderTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/transforms/ptransform.py b/sdks/python/apache_beam/transforms/ptransform.py index fd86ff1f93427..fcff86d4c50c1 100644 --- a/sdks/python/apache_beam/transforms/ptransform.py +++ b/sdks/python/apache_beam/transforms/ptransform.py @@ -38,11 +38,13 @@ class and wrapper class that allows lambda functions to be used as import copy import itertools +import json import logging import operator import os import sys import threading +import warnings from functools import reduce from functools import wraps from typing import TYPE_CHECKING @@ -83,6 +85,7 @@ class and wrapper class that allows lambda functions to be used as from apache_beam.typehints.trivial_inference import instance_to_type from apache_beam.typehints.typehints import validate_composite_type_param from apache_beam.utils import proto_utils +from apache_beam.utils import python_callable if TYPE_CHECKING: from apache_beam import coders @@ -95,6 +98,7 @@ class and wrapper class that allows lambda functions to be used as 'PTransform', 'ptransform_fn', 'label_from_callable', + 'annotate_yaml', ] _LOGGER = logging.getLogger(__name__) @@ -371,7 +375,10 @@ def default_label(self): return self.__class__.__name__ def annotations(self) -> Dict[str, Union[bytes, str, message.Message]]: - return {} + return { + 'python_type': # + f'{self.__class__.__module__}.{self.__class__.__qualname__}' + } def default_type_hints(self): fn_type_hints = IOTypeHints.from_callable(self.expand) @@ -1093,3 +1100,67 @@ def __ror__(self, pvalueish, _unused=None): def expand(self, pvalue): raise RuntimeError("Should never be expanded directly.") + + def __getattr__(self, attr): + transform_attr = getattr(self.transform, attr) + if callable(transform_attr): + + @wraps(transform_attr) + def wrapper(*args, **kwargs): + result = transform_attr(*args, **kwargs) + if isinstance(result, PTransform): + return _NamedPTransform(result, self.label) + else: + return result + + return wrapper + else: + return transform_attr + + +# Defined here to avoid circular import issues for Beam library transforms. +def annotate_yaml(constructor): + """Causes instances of this transform to be annotated with their yaml syntax. + + Should only be used for transforms that are fully defined by their constructor + arguments. + """ + @wraps(constructor) + def wrapper(*args, **kwargs): + transform = constructor(*args, **kwargs) + + fully_qualified_name = ( + f'{constructor.__module__}.{constructor.__qualname__}') + try: + imported_constructor = ( + python_callable.PythonCallableWithSource. + load_from_fully_qualified_name(fully_qualified_name)) + if imported_constructor != wrapper: + raise ImportError('Different object.') + except ImportError: + warnings.warn(f'Cannot import {constructor} as {fully_qualified_name}.') + return transform + + try: + config = json.dumps({ + 'constructor': fully_qualified_name, + 'args': args, + 'kwargs': kwargs, + }) + except TypeError as exn: + warnings.warn( + f'Cannot serialize arguments for {constructor} as json: {exn}') + return transform + + original_annotations = transform.annotations + transform.annotations = lambda: { + **original_annotations(), + # These override whatever may have been provided earlier. + # The outermost call is expected to be the most specific. + 'yaml_provider': 'python', + 'yaml_type': 'PyTransform', + 'yaml_args': config, + } + return transform + + return wrapper diff --git a/sdks/python/apache_beam/transforms/resources.py b/sdks/python/apache_beam/transforms/resources.py index 7bb202ab5660a..7c4160df8eddd 100644 --- a/sdks/python/apache_beam/transforms/resources.py +++ b/sdks/python/apache_beam/transforms/resources.py @@ -42,6 +42,7 @@ 'ResourceHint', 'AcceleratorHint', 'MinRamHint', + 'CpuCountHint', 'merge_resource_hints', 'parse_resource_hints', 'resource_hints_from_options', @@ -177,6 +178,21 @@ def get_merged_value( ResourceHint.register_resource_hint('minRam', MinRamHint) +class CpuCountHint(ResourceHint): + """Describes number of CPUs available in transform's execution environment.""" + urn = resource_hints.CPU_COUNT.urn + + @classmethod + def get_merged_value( + cls, outer_value, inner_value): # type: (bytes, bytes) -> bytes + return ResourceHint._use_max(outer_value, inner_value) + + +ResourceHint.register_resource_hint('cpu_count', CpuCountHint) +# Alias for interoperability with SDKs preferring camelCase. +ResourceHint.register_resource_hint('cpuCount', CpuCountHint) + + def parse_resource_hints(hints): # type: (Dict[Any, Any]) -> Dict[str, bytes] parsed_hints = {} for hint, value in hints.items(): diff --git a/sdks/python/apache_beam/transforms/resources_test.py b/sdks/python/apache_beam/transforms/resources_test.py index 939391b7adcb8..939bdcd626514 100644 --- a/sdks/python/apache_beam/transforms/resources_test.py +++ b/sdks/python/apache_beam/transforms/resources_test.py @@ -46,6 +46,11 @@ class ResourcesTest(unittest.TestCase): val='gpu', urn='beam:resources:accelerator:v1', bytestr=b'gpu'), + param( + name='cpu_count', + val='4', + urn='beam:resources:cpu_count:v1', + bytestr=b'4'), ]) def test_known_resource_hints(self, name, val, urn, bytestr): t = PTransform() @@ -56,6 +61,7 @@ def test_known_resource_hints(self, name, val, urn, bytestr): @parameterized.expand([ param(name='min_ram', val='3,500G'), param(name='accelerator', val=1), + param(name='cpu_count', val=1), param(name='unknown_hint', val=1) ]) def test_resource_hint_parsing_fails_early(self, name, val): diff --git a/sdks/python/apache_beam/transforms/timestamped_value_type_test.py b/sdks/python/apache_beam/transforms/timestamped_value_type_test.py new file mode 100644 index 0000000000000..46449bb1ef72a --- /dev/null +++ b/sdks/python/apache_beam/transforms/timestamped_value_type_test.py @@ -0,0 +1,139 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import unittest +from typing import Any +from typing import Dict +from typing import List +from typing import TypeVar + +import apache_beam as beam +from apache_beam.transforms.window import TimestampedValue +from apache_beam.typehints.decorators import TypeCheckError + +T = TypeVar("T") + + +def ConvertToTimestampedValue(plant: Dict[str, Any]) -> TimestampedValue[str]: + return TimestampedValue[str](plant["name"], plant["season"]) + + +def ConvertToTimestampedValue_1(plant: Dict[str, Any]) -> TimestampedValue: + return TimestampedValue(plant["name"], plant["season"]) + + +def ConvertToTimestampedValue_2( + plant: Dict[str, Any]) -> TimestampedValue[List[str]]: + return TimestampedValue[List[str]](plant["name"], plant["season"]) + + +def ConvertToTimestampedValue_3(plant: Dict[str, Any]) -> TimestampedValue[T]: + return TimestampedValue[T](plant["name"], plant["season"]) + + +class TypeCheckTimestampedValueTestCase(unittest.TestCase): + def setUp(self): + self.opts = beam.options.pipeline_options.PipelineOptions( + runtime_type_check=True) + self.data = [ + { + "name": "Strawberry", "season": 1585699200 + }, # April, 2020 + ] + self.data_1 = [ + { + "name": 1234, "season": 1585699200 + }, # April, 2020 + ] + self.data_2 = [ + { + "name": ["abc", "cde"], "season": 1585699200 + }, # April, 2020 + ] + self.data_3 = [ + { + "name": [123, "cde"], "season": 1585699200 + }, # April, 2020 + ] + + def test_pcoll_default_hints(self): + for fn in (ConvertToTimestampedValue, ConvertToTimestampedValue_1): + pc = beam.Map(fn) + ht = pc.default_type_hints() + assert len(ht) == 3 + assert ht.output_types[0][0] + + def test_pcoll_with_output_hints(self): + pc = beam.Map(ConvertToTimestampedValue).with_output_types(str) + ht = pc.get_type_hints() + assert len(ht) == 3 + assert ht.output_types[0][0] == str + + def test_opts_with_check(self): + with beam.Pipeline(options=self.opts) as p: + _ = ( + p + | "Garden plants" >> beam.Create(self.data) + | "With timestamps" >> beam.Map(ConvertToTimestampedValue) + | beam.Map(print)) + + def test_opts_with_check_list_str(self): + with beam.Pipeline(options=self.opts) as p: + _ = ( + p + | "Garden plants" >> beam.Create(self.data_2) + | "With timestamps" >> beam.Map(ConvertToTimestampedValue_2) + | beam.Map(print)) + + def test_opts_with_check_wrong_data(self): + with self.assertRaises(TypeCheckError): + with beam.Pipeline(options=self.opts) as p: + _ = ( + p + | "Garden plants" >> beam.Create(self.data_1) + | "With timestamps" >> beam.Map(ConvertToTimestampedValue) + | beam.Map(print)) + + def test_opts_with_check_wrong_data_list_str(self): + with self.assertRaises(TypeCheckError): + with beam.Pipeline(options=self.opts) as p: + _ = ( + p + | "Garden plants" >> beam.Create(self.data_1) + | "With timestamps" >> beam.Map(ConvertToTimestampedValue_2) + | beam.Map(print)) + + with self.assertRaises(TypeCheckError): + with beam.Pipeline(options=self.opts) as p: + _ = ( + p + | "Garden plants" >> beam.Create(self.data_3) + | "With timestamps" >> beam.Map(ConvertToTimestampedValue_2) + | beam.Map(print)) + + def test_opts_with_check_typevar(self): + with self.assertRaises(RuntimeError): + with beam.Pipeline(options=self.opts) as p: + _ = ( + p + | "Garden plants" >> beam.Create(self.data_2) + | "With timestamps" >> beam.Map(ConvertToTimestampedValue_3) + | beam.Map(print)) + + +if __name__ == '__main__': + unittest.main() diff --git a/sdks/python/apache_beam/transforms/trigger_test.py b/sdks/python/apache_beam/transforms/trigger_test.py index c8beed42c6526..06e205df61ece 100644 --- a/sdks/python/apache_beam/transforms/trigger_test.py +++ b/sdks/python/apache_beam/transforms/trigger_test.py @@ -1114,15 +1114,15 @@ def _execute( if is_order_agnostic: reshuffle_seed = random.randrange(1 << 20) keys = [ - u'original', - u'reversed', - u'reshuffled(%s)' % reshuffle_seed, - u'one-element-bundles', - u'one-element-bundles-reversed', - u'two-element-bundles' + 'original', + 'reversed', + 'reshuffled(%s)' % reshuffle_seed, + 'one-element-bundles', + 'one-element-bundles-reversed', + 'two-element-bundles' ] else: - keys = [u'key1', u'key2'] + keys = ['key1', 'key2'] # Elements are encoded as a json strings to allow other languages to # decode elements while executing the test stream. diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py index 7143844aa03b7..fb0e8e9789d8b 100644 --- a/sdks/python/apache_beam/transforms/util.py +++ b/sdks/python/apache_beam/transforms/util.py @@ -665,7 +665,8 @@ class BatchElements(PTransform): operations. For a fixed batch size, set the min and max to be equal. Elements are batched per-window and batches emitted in the window - corresponding to its contents. + corresponding to its contents. Each batch is emitted with a timestamp at + the end of their window. Args: min_batch_size: (optional) the smallest size of a batch @@ -1154,13 +1155,24 @@ def Iterables(delimiter=None): class LogElements(PTransform): """ PTransform for printing the elements of a PCollection. + + Args: + label (str): (optional) A custom label for the transform. + prefix (str): (optional) A prefix string to prepend to each logged element. + with_timestamp (bool): (optional) Whether to include element's timestamp. + with_window (bool): (optional) Whether to include element's window. + level: (optional) The logging level for the output (e.g. `logging.DEBUG`, + `logging.INFO`, `logging.WARNING`, `logging.ERROR`). If not specified, + the log is printed to stdout. """ class _LoggingFn(DoFn): - def __init__(self, prefix='', with_timestamp=False, with_window=False): + def __init__( + self, prefix='', with_timestamp=False, with_window=False, level=None): super().__init__() self.prefix = prefix self.with_timestamp = with_timestamp self.with_window = with_window + self.level = level def process( self, @@ -1177,19 +1189,38 @@ def process( log_line += ', window(start=' + window.start.to_rfc3339() log_line += ', end=' + window.end.to_rfc3339() + ')' - print(log_line) + if self.level == logging.DEBUG: + logging.debug(log_line) + elif self.level == logging.INFO: + logging.info(log_line) + elif self.level == logging.WARNING: + logging.warning(log_line) + elif self.level == logging.ERROR: + logging.error(log_line) + elif self.level == logging.CRITICAL: + logging.critical(log_line) + else: + print(log_line) + yield element def __init__( - self, label=None, prefix='', with_timestamp=False, with_window=False): + self, + label=None, + prefix='', + with_timestamp=False, + with_window=False, + level=None): super().__init__(label) self.prefix = prefix self.with_timestamp = with_timestamp self.with_window = with_window + self.level = level def expand(self, input): return input | ParDo( - self._LoggingFn(self.prefix, self.with_timestamp, self.with_window)) + self._LoggingFn( + self.prefix, self.with_timestamp, self.with_window, self.level)) class Reify(object): diff --git a/sdks/python/apache_beam/transforms/util_test.py b/sdks/python/apache_beam/transforms/util_test.py index 13b90027bde62..d8a8bacb96cdf 100644 --- a/sdks/python/apache_beam/transforms/util_test.py +++ b/sdks/python/apache_beam/transforms/util_test.py @@ -39,6 +39,7 @@ from apache_beam.metrics import MetricsFilter from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.options.pipeline_options import StandardOptions +from apache_beam.options.pipeline_options import TypeOptions from apache_beam.portability import common_urns from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.pvalue import AsList @@ -1041,6 +1042,24 @@ def test_output_typehints(self): ShardedKeyType[typehints.Tuple[int, int]], # type: ignore[misc] typehints.Iterable[str]]) + def test_runtime_type_check(self): + options = PipelineOptions() + options.view_as(TypeOptions).runtime_type_check = True + with TestPipeline(options=options) as pipeline: + collection = ( + pipeline + | beam.Create(GroupIntoBatchesTest._create_test_data()) + | util.GroupIntoBatches(GroupIntoBatchesTest.BATCH_SIZE)) + num_batches = collection | beam.combiners.Count.Globally() + assert_that( + num_batches, + equal_to([ + int( + math.ceil( + GroupIntoBatchesTest.NUM_ELEMENTS / + GroupIntoBatchesTest.BATCH_SIZE)) + ])) + def _test_runner_api_round_trip(self, transform, urn): context = pipeline_context.PipelineContext() proto = transform.to_runner_api(context) @@ -1155,6 +1174,31 @@ def test_ptransform_output(self): | util.LogElements(prefix='prefix_')) assert_that(result, equal_to(['a', 'b', 'c'])) + @pytest.fixture(scope="function") + def _capture_logs(request, caplog): + with caplog.at_level(logging.INFO): + with TestPipeline() as p: + _ = ( + p | "info" >> beam.Create(["element"]) + | "I" >> beam.LogElements(prefix='info_', level=logging.INFO)) + _ = ( + p | "warning" >> beam.Create(["element"]) + | "W" >> beam.LogElements(prefix='warning_', level=logging.WARNING)) + _ = ( + p | "error" >> beam.Create(["element"]) + | "E" >> beam.LogElements(prefix='error_', level=logging.ERROR)) + + request.captured_log = caplog.text + + @pytest.mark.usefixtures("_capture_logs") + def test_setting_level_uses_appropriate_log_channel(self): + self.assertTrue( + re.compile('INFO(.*)info_element').search(self.captured_log)) + self.assertTrue( + re.compile('WARNING(.*)warning_element').search(self.captured_log)) + self.assertTrue( + re.compile('ERROR(.*)error_element').search(self.captured_log)) + class ReifyTest(unittest.TestCase): def test_timestamp(self): diff --git a/sdks/python/apache_beam/transforms/validate_runner_xlang_test.py b/sdks/python/apache_beam/transforms/validate_runner_xlang_test.py index 4de1d884072f8..8e8e79648250b 100644 --- a/sdks/python/apache_beam/transforms/validate_runner_xlang_test.py +++ b/sdks/python/apache_beam/transforms/validate_runner_xlang_test.py @@ -94,7 +94,7 @@ def run_prefix(self, pipeline): | beam.Create(['a', 'b']).with_output_types(str) | beam.ExternalTransform( TEST_PREFIX_URN, - ImplicitSchemaPayloadBuilder({'data': u'0'}), + ImplicitSchemaPayloadBuilder({'data': '0'}), self.expansion_service)) assert_that(res, equal_to(['0a', '0b'])) diff --git a/sdks/python/apache_beam/transforms/window.py b/sdks/python/apache_beam/transforms/window.py index 3b8c3bb44e239..c76b30fb8ff76 100644 --- a/sdks/python/apache_beam/transforms/window.py +++ b/sdks/python/apache_beam/transforms/window.py @@ -52,9 +52,11 @@ import abc from functools import total_ordering from typing import Any +from typing import Generic from typing import Iterable from typing import List from typing import Optional +from typing import TypeVar from google.protobuf import duration_pb2 from google.protobuf import timestamp_pb2 @@ -278,8 +280,11 @@ def union(self, other): min(self.start, other.start), max(self.end, other.end)) +V = TypeVar("V") + + @total_ordering -class TimestampedValue(object): +class TimestampedValue(Generic[V]): """A timestamped value having a value and a timestamp. Attributes: @@ -287,7 +292,7 @@ class TimestampedValue(object): timestamp: Timestamp associated with the value as seconds since Unix epoch. """ def __init__(self, value, timestamp): - # type: (Any, TimestampTypes) -> None + # type: (V, TimestampTypes) -> None self.value = value self.timestamp = Timestamp.of(timestamp) @@ -506,8 +511,10 @@ def assign(self, context): timestamp = context.timestamp start = timestamp - ((timestamp - self.offset) % self.period) return [ - IntervalWindow(Timestamp(micros=s), Timestamp(micros=s) + self.size) - for s in range( + IntervalWindow( + (interval_start := Timestamp(micros=s)), + interval_start + self.size, + ) for s in range( start.micros, timestamp.micros - self.size.micros, -self.period.micros) diff --git a/sdks/python/apache_beam/typehints/OWNERS b/sdks/python/apache_beam/typehints/OWNERS deleted file mode 100644 index 207b624ab80b4..0000000000000 --- a/sdks/python/apache_beam/typehints/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - charlesccychen diff --git a/sdks/python/apache_beam/typehints/arrow_type_compatibility.py b/sdks/python/apache_beam/typehints/arrow_type_compatibility.py index c8e425f0e96a7..34a37a886bab5 100644 --- a/sdks/python/apache_beam/typehints/arrow_type_compatibility.py +++ b/sdks/python/apache_beam/typehints/arrow_type_compatibility.py @@ -311,9 +311,9 @@ def from_typehints(element_type, element_type = RowTypeConstraint.from_user_type(element_type) if element_type is None: raise TypeError( - "Element type must be compatible with Beam Schemas (" - "https://beam.apache.org/documentation/programming-guide/#schemas) " - "for batch type pa.Table.") + f"Element type {element_type} must be compatible with Beam Schemas " + "(https://beam.apache.org/documentation/programming-guide/#schemas)" + " for batch type pa.Table.") return PyarrowBatchConverter(element_type) diff --git a/sdks/python/apache_beam/typehints/arrow_type_compatibility_test.py b/sdks/python/apache_beam/typehints/arrow_type_compatibility_test.py index e708b151d9056..1e9ab3f27bd9c 100644 --- a/sdks/python/apache_beam/typehints/arrow_type_compatibility_test.py +++ b/sdks/python/apache_beam/typehints/arrow_type_compatibility_test.py @@ -206,7 +206,7 @@ class ArrowBatchConverterErrorsTest(unittest.TestCase): ( pa.Table, Any, - r'Element type must be compatible with Beam Schemas', + r'Element type .* must be compatible with Beam Schemas', ), ]) def test_construction_errors( diff --git a/sdks/python/apache_beam/typehints/decorators_test.py b/sdks/python/apache_beam/typehints/decorators_test.py index ba46038e472b5..3baf9fa8322fc 100644 --- a/sdks/python/apache_beam/typehints/decorators_test.py +++ b/sdks/python/apache_beam/typehints/decorators_test.py @@ -38,6 +38,7 @@ T = TypeVariable('T') # Name is 'T' so it converts to a beam type with the same name. # mypy requires that the name of the variable match, so we must ignore this. +# pylint: disable=typevar-name-mismatch T_typing = typing.TypeVar('T') # type: ignore @@ -51,14 +52,8 @@ def fn(a, b=1, *c, **d): self.assertListEqual(list(s.parameters), ['a', 'b', 'c', 'd']) def test_get_signature_builtin(self): - # Tests a builtin function for 3.7+ and fallback result for older versions. s = decorators.get_signature(list) - if sys.version_info < (3, 7): - self.assertListEqual( - list(s.parameters), - ['_', '__unknown__varargs', '__unknown__keywords']) - else: - self.assertListEqual(list(s.parameters), ['iterable']) + self.assertListEqual(list(s.parameters), ['iterable']) self.assertEqual(s.return_annotation, List[Any]) def test_from_callable_without_annotations(self): diff --git a/sdks/python/apache_beam/typehints/native_type_compatibility.py b/sdks/python/apache_beam/typehints/native_type_compatibility.py index d03d5db404538..e916f34146f17 100644 --- a/sdks/python/apache_beam/typehints/native_type_compatibility.py +++ b/sdks/python/apache_beam/typehints/native_type_compatibility.py @@ -156,10 +156,7 @@ def is_new_type(typ): return hasattr(typ, '__supertype__') -try: - _ForwardRef = typing.ForwardRef # Python 3.7+ -except AttributeError: - _ForwardRef = typing._ForwardRef +_ForwardRef = typing.ForwardRef # Python 3.7+ def is_forward_ref(typ): diff --git a/sdks/python/apache_beam/typehints/native_type_compatibility_test.py b/sdks/python/apache_beam/typehints/native_type_compatibility_test.py index 89045ae7a2516..b9280c57a393a 100644 --- a/sdks/python/apache_beam/typehints/native_type_compatibility_test.py +++ b/sdks/python/apache_beam/typehints/native_type_compatibility_test.py @@ -49,6 +49,11 @@ class _TestGeneric(typing.Generic[T]): pass +class _TestPair(typing.NamedTuple('TestTuple', [('first', T), ('second', T)]), + typing.Generic[T]): + pass + + class NativeTypeCompatibilityTest(unittest.TestCase): def test_convert_to_beam_type(self): test_cases = [ @@ -99,6 +104,8 @@ def test_convert_to_beam_type(self): typehints.List[_TestGeneric]), ('nested generic subscripted', typing.List[_TestGeneric[int]], typehints.List[_TestGeneric[int]]), + ('nested generic with any', typing.List[_TestPair[typing.Any]], + typehints.List[_TestPair[typing.Any]]), ] for test_case in test_cases: diff --git a/sdks/python/apache_beam/typehints/pandas_type_compatibility_test.py b/sdks/python/apache_beam/typehints/pandas_type_compatibility_test.py index 5a8dc72dd4b99..ff66df1ce9685 100644 --- a/sdks/python/apache_beam/typehints/pandas_type_compatibility_test.py +++ b/sdks/python/apache_beam/typehints/pandas_type_compatibility_test.py @@ -64,7 +64,7 @@ 'bar': pd.Series([i / 100 for i in range(100)], dtype='float64'), 'baz': pd.Series([str(i) for i in range(100)], dtype=pd.StringDtype()), - }).set_index(pd.Int64Index(range(123, 223), name='an_index')), + }).set_index(pd.Index(range(123, 223), dtype='int64', name='an_index')), }, { 'batch_typehint': pd.DataFrame, @@ -87,8 +87,8 @@ 'baz': pd.Series([str(i) for i in range(100)], dtype=pd.StringDtype()), }).set_index([ - pd.Int64Index(range(123, 223), name='an_index'), - pd.Int64Index(range(475, 575), name='another_index'), + pd.Index(range(123, 223), dtype='int64', name='an_index'), + pd.Index(range(475, 575), dtype='int64', name='another_index'), ]), }, { diff --git a/sdks/python/apache_beam/typehints/schema_registry.py b/sdks/python/apache_beam/typehints/schema_registry.py index 9ec7b1b65ccf4..a73e97f43f701 100644 --- a/sdks/python/apache_beam/typehints/schema_registry.py +++ b/sdks/python/apache_beam/typehints/schema_registry.py @@ -40,13 +40,18 @@ def generate_new_id(self): "schemas.") def add(self, typing, schema): - self.by_id[schema.id] = (typing, schema) + if not schema.id: + self.by_id[schema.id] = (typing, schema) def get_typing_by_id(self, unique_id): + if not unique_id: + return None result = self.by_id.get(unique_id, None) return result[0] if result is not None else None def get_schema_by_id(self, unique_id): + if not unique_id: + return None result = self.by_id.get(unique_id, None) return result[1] if result is not None else None diff --git a/sdks/python/apache_beam/typehints/schemas.py b/sdks/python/apache_beam/typehints/schemas.py index 156b877d07e25..ea836430e8e2e 100644 --- a/sdks/python/apache_beam/typehints/schemas.py +++ b/sdks/python/apache_beam/typehints/schemas.py @@ -72,6 +72,7 @@ from typing import ByteString from typing import Dict from typing import Generic +from typing import Iterable from typing import List from typing import Mapping from typing import NamedTuple @@ -87,10 +88,12 @@ from apache_beam.portability import common_urns from apache_beam.portability.api import schema_pb2 from apache_beam.typehints import row_type +from apache_beam.typehints import typehints from apache_beam.typehints.native_type_compatibility import _get_args from apache_beam.typehints.native_type_compatibility import _match_is_exactly_mapping from apache_beam.typehints.native_type_compatibility import _match_is_optional from apache_beam.typehints.native_type_compatibility import _safe_issubclass +from apache_beam.typehints.native_type_compatibility import convert_to_typing_type from apache_beam.typehints.native_type_compatibility import extract_optional_type from apache_beam.typehints.native_type_compatibility import match_is_named_tuple from apache_beam.typehints.schema_registry import SCHEMA_REGISTRY @@ -225,6 +228,15 @@ def option_from_runner_api( schema_registry=schema_registry).option_from_runner_api(option_proto) +def schema_field( + name: str, field_type: Union[schema_pb2.FieldType, + type]) -> schema_pb2.Field: + return schema_pb2.Field( + name=name, + type=field_type if isinstance(field_type, schema_pb2.FieldType) else + typing_to_runner_api(field_type)) + + class SchemaTranslation(object): def __init__(self, schema_registry: SchemaTypeRegistry = SCHEMA_REGISTRY): self.schema_registry = schema_registry @@ -273,6 +285,9 @@ def typing_to_runner_api(self, type_: type) -> schema_pb2.FieldType: if row_type_constraint is not None: return self.typing_to_runner_api(row_type_constraint) + if isinstance(type_, typehints.TypeConstraint): + type_ = convert_to_typing_type(type_) + # All concrete types (other than NamedTuple sub-classes) should map to # a supported primitive type. if type_ in PRIMITIVE_TO_ATOMIC_TYPE: @@ -307,6 +322,11 @@ def typing_to_runner_api(self, type_: type) -> schema_pb2.FieldType: return schema_pb2.FieldType( map_type=schema_pb2.MapType(key_type=key_type, value_type=value_type)) + elif _safe_issubclass(type_, Iterable) and not _safe_issubclass(type_, str): + element_type = self.typing_to_runner_api(_get_args(type_)[0]) + return schema_pb2.FieldType( + array_type=schema_pb2.ArrayType(element_type=element_type)) + try: logical_type = LogicalType.from_typing(type_) except ValueError: @@ -588,6 +608,25 @@ def named_fields_from_element_type( return named_fields_from_schema(schema_from_element_type(element_type)) +def union_schema_type(element_types): + """Returns a schema whose fields are the union of each corresponding field. + + element_types must be a set of schema-aware types whose fields have the + same naming and ordering. + """ + union_fields_and_types = [] + for field in zip(*[named_fields_from_element_type(t) for t in element_types]): + names, types = zip(*field) + name_set = set(names) + if len(name_set) != 1: + raise TypeError( + f"Could not determine schema for type hints {element_types!r}: " + f"Inconsistent names: {name_set}") + union_fields_and_types.append( + (next(iter(name_set)), typehints.Union[types])) + return named_tuple_from_schema(named_fields_to_schema(union_fields_and_types)) + + # Registry of typings for a schema by UUID class LogicalTypeRegistry(object): def __init__(self): diff --git a/sdks/python/apache_beam/typehints/schemas_test.py b/sdks/python/apache_beam/typehints/schemas_test.py index 2495f5722ad38..5d38b16d97835 100644 --- a/sdks/python/apache_beam/typehints/schemas_test.py +++ b/sdks/python/apache_beam/typehints/schemas_test.py @@ -22,6 +22,7 @@ import itertools import pickle import unittest +from typing import Any from typing import ByteString from typing import List from typing import Mapping @@ -40,8 +41,10 @@ from apache_beam.portability import common_urns from apache_beam.portability.api import schema_pb2 from apache_beam.typehints import row_type +from apache_beam.typehints import typehints from apache_beam.typehints.native_type_compatibility import match_is_named_tuple from apache_beam.typehints.schemas import SchemaTypeRegistry +from apache_beam.typehints.schemas import named_fields_from_element_type from apache_beam.typehints.schemas import named_tuple_from_schema from apache_beam.typehints.schemas import named_tuple_to_schema from apache_beam.typehints.schemas import typing_from_runner_api @@ -642,6 +645,16 @@ def test_row_type_is_callable(self): self.assertIsInstance(instance, simple_row_type.user_type) self.assertEqual(instance, (np.int64(35), 'baz')) + def test_union(self): + with_int = row_type.RowTypeConstraint.from_fields([('common', str), + ('unique', int)]) + with_any = row_type.RowTypeConstraint.from_fields([('common', str), + ('unique', Any)]) + union_type = typehints.Union[with_int, with_any] + self.assertEqual( + named_fields_from_element_type(union_type), [('common', str), + ('unique', Any)]) + class HypothesisTest(unittest.TestCase): # There is considerable variablility in runtime for this test, disable diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py index f4b350e8f0522..a880b5c70ea19 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference.py +++ b/sdks/python/apache_beam/typehints/trivial_inference.py @@ -399,7 +399,10 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): jump_multiplier = 1 last_pc = -1 + last_real_opname = opname = None while pc < end: # pylint: disable=too-many-nested-blocks + if opname not in ('PRECALL', 'CACHE'): + last_real_opname = opname start = pc instruction = ofs_table[pc] op = instruction.opcode @@ -534,13 +537,13 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): return_type = Any state.kw_names = None else: - # Handle lambdas always having an arg of 0 for CALL + # Handle comprehensions always having an arg of 0 for CALL # See https://github.com/python/cpython/issues/102403 for context. - if pop_count == 1: - while pop_count <= len(state.stack): - if isinstance(state.stack[-pop_count], Const): - break - pop_count += 1 + if (pop_count == 1 and last_real_opname == 'GET_ITER' and + len(state.stack) > 1 and isinstance(state.stack[-2], Const) and + getattr(state.stack[-2].value, '__name__', None) in ( + '', '', '', '')): + pop_count += 1 if depth <= 0 or pop_count > len(state.stack): return_type = Any elif isinstance(state.stack[-pop_count], Const): diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py b/sdks/python/apache_beam/typehints/trivial_inference_test.py index d8cc2ab19a03d..4341d11d36040 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference_test.py +++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py @@ -251,11 +251,30 @@ def testCall(self): self.assertReturnType( typehints.Tuple[int, typehints.Any], lambda: (1, f(x=1.0))) + def testCallNullaryMethod(self): + class Foo: + pass + + self.assertReturnType( + typehints.Tuple[Foo, typehints.Any], lambda x: (x, x.unknown()), [Foo]) + + def testCallNestedLambda(self): + class Foo: + pass + + self.assertReturnType( + typehints.Tuple[Foo, int], lambda x: (x, (lambda: 3)()), [Foo]) + def testClosure(self): x = 1 y = 1.0 self.assertReturnType(typehints.Tuple[int, float], lambda: (x, y)) + @unittest.skip("https://github.com/apache/beam/issues/28420") + def testLocalClosure(self): + self.assertReturnType( + typehints.Tuple[int, int], lambda x: (x, (lambda: x)()), [int]) + def testGlobals(self): self.assertReturnType(int, lambda: global_int) diff --git a/sdks/python/apache_beam/typehints/typecheck.py b/sdks/python/apache_beam/typehints/typecheck.py index 2f202a14ab7db..7e84779a0f150 100644 --- a/sdks/python/apache_beam/typehints/typecheck.py +++ b/sdks/python/apache_beam/typehints/typecheck.py @@ -31,6 +31,7 @@ from apache_beam.pvalue import TaggedOutput from apache_beam.transforms import core from apache_beam.transforms.core import DoFn +from apache_beam.transforms.window import TimestampedValue from apache_beam.transforms.window import WindowedValue from apache_beam.typehints.decorators import GeneratorWrapper from apache_beam.typehints.decorators import TypeCheckError @@ -39,6 +40,7 @@ from apache_beam.typehints.typehints import CompositeTypeHintError from apache_beam.typehints.typehints import SimpleTypeHintError from apache_beam.typehints.typehints import check_constraint +from apache_beam.typehints.typehints import normalize class AbstractDoFnWrapper(DoFn): @@ -47,6 +49,13 @@ def __init__(self, dofn): super().__init__() self.dofn = dofn + def __getattribute__(self, name): + if (name.startswith('_') or name in self.__dict__ or + hasattr(type(self), name)): + return object.__getattribute__(self, name) + else: + return getattr(self.dofn, name) + def _inspect_start_bundle(self): return self.dofn.get_function_arguments('start_bundle') @@ -146,9 +155,19 @@ def _type_check_result(self, transform_results): return transform_results def type_check_output(o): - # TODO(robertwb): Multi-output. - x = o.value if isinstance(o, (TaggedOutput, WindowedValue)) else o - self.type_check(self._output_type_hint, x, is_input=False) + if isinstance(o, TimestampedValue) and hasattr(o, "__orig_class__"): + # when a typed TimestampedValue is set, check the value type + x = o.value + # per https://stackoverflow.com/questions/57706180/, + # __orig_class__ is te safe way to obtain the actual type + # from from Generic[T], supported since Python 3.5.3 + beam_type = normalize(o.__orig_class__.__args__[0]) + self.type_check(beam_type, x, is_input=False) + else: + # TODO(robertwb): Multi-output. + x = o.value if isinstance(o, (TaggedOutput, WindowedValue)) else o + + self.type_check(self._output_type_hint, x, is_input=False) # If the return type is a generator, then we will need to interleave our # type-checking with its normal iteration so we don't deplete the diff --git a/sdks/python/apache_beam/typehints/typed_pipeline_test.py b/sdks/python/apache_beam/typehints/typed_pipeline_test.py index 9774a37ac88c3..9cb3fcdbb91db 100644 --- a/sdks/python/apache_beam/typehints/typed_pipeline_test.py +++ b/sdks/python/apache_beam/typehints/typed_pipeline_test.py @@ -64,10 +64,6 @@ def test_non_function(self): result = ['1', '10', '100'] | beam.Map(int, 16) self.assertEqual([1, 16, 256], sorted(result)) - @unittest.skipIf( - sys.version_info < (3, 7, 0), - 'Function signatures for builtins are not available in Python 3 before ' - 'version 3.7.') def test_non_function_fails(self): with self.assertRaises(typehints.TypeCheckError): [1, 2, 3] | beam.Map(str.upper) @@ -888,15 +884,7 @@ def test_pardo_wrapper_builtin_method(self): def test_pardo_wrapper_builtin_type(self): th = beam.ParDo(list).get_type_hints() - if sys.version_info < (3, 7): - self.assertEqual( - th.input_types, - ((typehints.Any, typehints.decorators._ANY_VAR_POSITIONAL), { - '__unknown__keywords': typehints.decorators._ANY_VAR_KEYWORD - })) - else: - # Python 3.7+ supports signatures for builtins like 'list'. - self.assertEqual(th.input_types, ((typehints.Any, ), {})) + self.assertEqual(th.input_types, ((typehints.Any, ), {})) self.assertEqual(th.output_types, ((typehints.Any, ), {})) diff --git a/sdks/python/apache_beam/typehints/typehints.py b/sdks/python/apache_beam/typehints/typehints.py index 7ea05f27cc874..238bf8c321d63 100644 --- a/sdks/python/apache_beam/typehints/typehints.py +++ b/sdks/python/apache_beam/typehints/typehints.py @@ -355,11 +355,12 @@ def is_typing_generic(type_param): Such objects are considered valid type parameters. - Always returns false for Python versions below 3.7. + For Python versions 3.9 and above, also permits types.GenericAlias. """ - if hasattr(typing, '_GenericAlias'): - return isinstance(type_param, typing._GenericAlias) - return False + if hasattr(types, "GenericAlias") and isinstance(type_param, + types.GenericAlias): + return True + return isinstance(type_param, typing._GenericAlias) def validate_composite_type_param(type_param, error_msg_prefix): @@ -603,6 +604,15 @@ def __getitem__(self, type_params): return Any elif len(params) == 1: return next(iter(params)) + + if len(params) > 1: + from apache_beam.typehints import schemas + try: + return schemas.union_schema_type(params) + except (TypeError, KeyError): + # Not a union of compatible schema types. + pass + return self.UnionConstraint(params) diff --git a/sdks/python/apache_beam/typehints/typehints_test.py b/sdks/python/apache_beam/typehints/typehints_test.py index 1f6b98eccbeac..a1a1913bded71 100644 --- a/sdks/python/apache_beam/typehints/typehints_test.py +++ b/sdks/python/apache_beam/typehints/typehints_test.py @@ -619,6 +619,13 @@ def test_builtin_and_type_compatibility(self): self.assertCompatible(list, typing.List) self.assertCompatible(list[int], typing.List[int]) + def test_is_typing_generic(self): + self.assertTrue(typehints.is_typing_generic(typing.List[str])) + + def test_builtin_is_typing_generic(self): + if sys.version_info >= (3, 9): + self.assertTrue(typehints.is_typing_generic(list[str])) + class KVHintTestCase(TypeHintTestCase): def test_getitem_param_must_be_tuple(self): @@ -791,6 +798,10 @@ def test_getitem_invalid_composite_type_param(self): except TypeError: self.fail("built-in composite raised TypeError unexpectedly") + def test_non_typing_generic(self): + testCase = DummyTestClass1() + self.assertFalse(typehints.is_typing_generic(testCase)) + def test_compatibility(self): hint1 = self.beam_type[typehints.List[str]] hint2 = self.beam_type[typehints.Tuple[int, int]] diff --git a/sdks/python/apache_beam/utils/OWNERS b/sdks/python/apache_beam/utils/OWNERS deleted file mode 100644 index 1a1f0f0d3e722..0000000000000 --- a/sdks/python/apache_beam/utils/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - pabloem - - charlesccychen diff --git a/sdks/python/apache_beam/utils/multi_process_shared.py b/sdks/python/apache_beam/utils/multi_process_shared.py index b9414d017f388..0c901500348c2 100644 --- a/sdks/python/apache_beam/utils/multi_process_shared.py +++ b/sdks/python/apache_beam/utils/multi_process_shared.py @@ -36,6 +36,26 @@ import fasteners +# In some python versions, there is a bug where AutoProxy doesn't handle +# the kwarg 'manager_owned'. We implement our own backup here to make sure +# we avoid this problem. More info here: +# https://stackoverflow.com/questions/46779860/multiprocessing-managers-and-custom-classes +autoproxy = multiprocessing.managers.AutoProxy # type: ignore[attr-defined] + + +def patched_autoproxy( + token, + serializer, + manager=None, + authkey=None, + exposed=None, + incref=True, + manager_owned=True): + return autoproxy(token, serializer, manager, authkey, exposed, incref) + + +multiprocessing.managers.AutoProxy = patched_autoproxy # type: ignore[attr-defined] + T = TypeVar('T') AUTH_KEY = b'mps' @@ -55,19 +75,36 @@ def singletonProxy_call__(self, *args, **kwargs): raise RuntimeError('Entry was released.') return self._SingletonProxy_entry.obj.__call__(*args, **kwargs) - def _SingletonProxy_release(self): + def singletonProxy_release(self): assert self._SingletonProxy_valid self._SingletonProxy_valid = False def __getattr__(self, name): if not self._SingletonProxy_valid: raise RuntimeError('Entry was released.') - return getattr(self._SingletonProxy_entry.obj, name) + try: + return getattr(self._SingletonProxy_entry.obj, name) + except AttributeError as e: + # Swallow AttributeError exceptions so that they are ignored when + # calculating public functions. These can occur if __getattr__ is + # overriden, for example to only support some platforms. This will mean + # that these functions will be silently unavailable to the + # MultiProcessShared object, leading to worse errors when someone tries + # to use them, but it will keep them from breaking the whole object for + # functions which are unusable anyways. + logging.info( + 'Attribute %s is unavailable as a public function because ' + 'its __getattr__ function raised the following exception ' + '%s', + name, + e) + return None def __dir__(self): # Needed for multiprocessing.managers's proxying. dir = self._SingletonProxy_entry.obj.__dir__() dir.append('singletonProxy_call__') + dir.append('singletonProxy_release') return dir @@ -92,7 +129,7 @@ def acquire(self): return _SingletonProxy(self) def release(self, proxy): - proxy._SingletonProxy_release() + proxy.singletonProxy_release() with self.lock: self.refcount -= 1 if self.refcount == 0: @@ -151,6 +188,9 @@ def __call__(self, *args, **kwargs): def __getattr__(self, name): return getattr(self._proxyObject, name) + def get_auto_proxy_object(self): + return self._proxyObject + class MultiProcessShared(Generic[T]): """MultiProcessShared is used to share a single object across processes. @@ -252,7 +292,7 @@ def acquire(self): return _AutoProxyWrapper(singleton) def release(self, obj): - self._manager.release_singleton(self._tag, obj) + self._manager.release_singleton(self._tag, obj.get_auto_proxy_object()) def _create_server(self, address_file): # We need to be able to authenticate with both the manager and the process. diff --git a/sdks/python/apache_beam/utils/multi_process_shared_test.py b/sdks/python/apache_beam/utils/multi_process_shared_test.py index b5ae90da915c9..6654ec3799ecb 100644 --- a/sdks/python/apache_beam/utils/multi_process_shared_test.py +++ b/sdks/python/apache_beam/utils/multi_process_shared_test.py @@ -19,6 +19,7 @@ import logging import threading import unittest +from typing import Any from apache_beam.utils import multi_process_shared @@ -57,6 +58,30 @@ def error(self, msg): raise RuntimeError(msg) +class CounterWithBadAttr(object): + def __init__(self, start=0): + self.running = start + self.lock = threading.Lock() + + def get(self): + return self.running + + def increment(self, value=1): + with self.lock: + self.running += value + return self.running + + def error(self, msg): + raise RuntimeError(msg) + + def __getattribute__(self, __name: str) -> Any: + if __name == 'error': + raise AttributeError('error is not actually supported on this platform') + else: + # Default behaviour + return object.__getattribute__(self, __name) + + class MultiProcessSharedTest(unittest.TestCase): @classmethod def setUpClass(cls): @@ -72,6 +97,15 @@ def test_call(self): self.assertEqual(self.shared.increment(value=10), 21) self.assertEqual(self.shared.get(), 21) + def test_call_illegal_attr(self): + shared_handle = multi_process_shared.MultiProcessShared( + CounterWithBadAttr, tag='test_call_illegal_attr', always_proxy=True) + shared = shared_handle.acquire() + + self.assertEqual(shared.get(), 0) + self.assertEqual(shared.increment(), 1) + self.assertEqual(shared.get(), 1) + def test_call_callable(self): self.assertEqual(self.sharedCallable(), 0) self.assertEqual(self.sharedCallable.increment(), 1) @@ -133,6 +167,38 @@ def test_release(self): with self.assertRaisesRegex(Exception, 'released'): counter1.get() + def test_release_always_proxy(self): + shared1 = multi_process_shared.MultiProcessShared( + Counter, tag='test_release_always_proxy', always_proxy=True) + shared2 = multi_process_shared.MultiProcessShared( + Counter, tag='test_release_always_proxy', always_proxy=True) + + counter1 = shared1.acquire() + counter2 = shared2.acquire() + self.assertEqual(counter1.increment(), 1) + self.assertEqual(counter2.increment(), 2) + + counter1again = shared1.acquire() + self.assertEqual(counter1again.increment(), 3) + + shared1.release(counter1) + shared2.release(counter2) + + with self.assertRaisesRegex(Exception, 'released'): + counter1.get() + with self.assertRaisesRegex(Exception, 'released'): + counter2.get() + + self.assertEqual(counter1again.get(), 3) + + shared1.release(counter1again) + + counter1New = shared1.acquire() + self.assertEqual(counter1New.get(), 0) + + with self.assertRaisesRegex(Exception, 'released'): + counter1.get() + if __name__ == '__main__': logging.getLogger().setLevel(logging.INFO) diff --git a/sdks/python/apache_beam/utils/python_callable.py b/sdks/python/apache_beam/utils/python_callable.py index a7de214ec926c..70aa7cb39e5c6 100644 --- a/sdks/python/apache_beam/utils/python_callable.py +++ b/sdks/python/apache_beam/utils/python_callable.py @@ -77,7 +77,7 @@ def load_from_fully_qualified_name(fully_qualified_name): return o @staticmethod - def load_from_script(source): + def load_from_script(source, method_name=None): lines = [ line for line in source.split('\n') if line.strip() and line.strip()[0] != '#' @@ -85,26 +85,27 @@ def load_from_script(source): common_indent = min(len(line) - len(line.lstrip()) for line in lines) lines = [line[common_indent:] for line in lines] - for ix, line in reversed(list(enumerate(lines))): - if line[0] != ' ': - if line.startswith('def '): - name = line[4:line.index('(')].strip() - elif line.startswith('class '): - name = line[5:line.index('(') if '(' in - line else line.index(':')].strip() - else: - name = '__python_callable__' - lines[ix] = name + ' = ' + line - break - else: - raise ValueError("Unable to identify callable from %r" % source) + if method_name is None: + for ix, line in reversed(list(enumerate(lines))): + if line[0] != ' ': + if line.startswith('def '): + method_name = line[4:line.index('(')].strip() + elif line.startswith('class '): + method_name = line[5:line.index('(') if '(' in + line else line.index(':')].strip() + else: + method_name = '__python_callable__' + lines[ix] = method_name + ' = ' + line + break + else: + raise ValueError("Unable to identify callable from %r" % source) # pylint: disable=exec-used # pylint: disable=ungrouped-imports import apache_beam as beam exec_globals = {'beam': beam} exec('\n'.join(lines), exec_globals) - return exec_globals[name] + return exec_globals[method_name] def default_label(self): src = self._source.strip() diff --git a/sdks/python/apache_beam/utils/subprocess_server.py b/sdks/python/apache_beam/utils/subprocess_server.py index f11132ca1643d..f566c3ea29146 100644 --- a/sdks/python/apache_beam/utils/subprocess_server.py +++ b/sdks/python/apache_beam/utils/subprocess_server.py @@ -169,7 +169,7 @@ def __init__(self, stub_class, path_to_jar, java_arguments, classpath=None): path_to_jar = self.make_classpath_jar(path_to_jar, classpath) super().__init__( stub_class, ['java', '-jar', path_to_jar] + list(java_arguments)) - self._existing_service = path_to_jar if _is_service_endpoint( + self._existing_service = path_to_jar if is_service_endpoint( path_to_jar) else None def start_process(self): @@ -258,7 +258,7 @@ def local_jar(cls, url, cache_dir=None): if cache_dir is None: cache_dir = cls.JAR_CACHE # TODO: Verify checksum? - if _is_service_endpoint(url): + if is_service_endpoint(url): return url elif os.path.exists(url): return url @@ -330,7 +330,8 @@ def make_classpath_jar(cls, main_jar, extra_jars, cache_dir=None): return composite_jar -def _is_service_endpoint(path): +def is_service_endpoint(path): + """Checks whether the path conforms to the 'beam_services' PipelineOption.""" return re.match(r'^[a-zA-Z0-9.-]+:\d+$', path) diff --git a/sdks/python/apache_beam/utils/transform_service_launcher.py b/sdks/python/apache_beam/utils/transform_service_launcher.py new file mode 100644 index 0000000000000..ac492513aba5c --- /dev/null +++ b/sdks/python/apache_beam/utils/transform_service_launcher.py @@ -0,0 +1,275 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +import logging +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import zipfile +from pathlib import Path + +import grpc + +from apache_beam.utils import subprocess_server + +_LOGGER = logging.getLogger(__name__) + +_COMMAND_POSSIBLE_VALUES = ['up', 'down', 'ps'] + +_EXPANSION_SERVICE_LAUNCHER_JAR = ':sdks:java:transform-service:launcher:build' + + +class TransformServiceLauncher(object): + _DEFAULT_PROJECT_NAME = 'apache.beam.transform.service' + _DEFAULT_START_WAIT_TIMEOUT = 50000 + + _launchers = {} # type: ignore + + # Maintaining a static list of launchers to prevent temporary resources + # from being created unnecessarily. + def __new__(cls, project_name, port, beam_version=None): + if project_name not in TransformServiceLauncher._launchers: + TransformServiceLauncher._launchers[project_name] = super( + TransformServiceLauncher, cls).__new__(cls) + return TransformServiceLauncher._launchers[project_name] + + def __init__(self, project_name, port, beam_version=None): + logging.info('Initializing the Beam Transform Service %s.' % project_name) + + self._project_name = project_name + self._port = port + self._address = 'localhost:' + str(self._port) + + self._launcher_lock = threading.RLock() + + self.docker_compose_command_prefix = [ + 'docker-compose', '-p', project_name, '-f', 'TODO path' + ] + + # Setting up Docker Compose configuration. + + # We use Docker Compose project name as the name of the temporary directory + # to isolate different transform service instances that may be running in + # the same machine. + + temp_dir = os.path.join(tempfile.gettempdir(), project_name) + if not os.path.exists(temp_dir): + os.mkdir(temp_dir) + + # Get the jar with configs + path_to_local_jar = subprocess_server.JavaJarServer.local_jar( + subprocess_server.JavaJarServer.path_to_beam_jar( + _EXPANSION_SERVICE_LAUNCHER_JAR)) + + with zipfile.ZipFile(path_to_local_jar) as launcher_jar: + launcher_jar.extract('docker-compose.yml', path=temp_dir) + launcher_jar.extract('.env', path=temp_dir) + + compose_file = os.path.join(temp_dir, 'docker-compose.yml') + + # Creating the credentials volume. + credentials_dir = os.path.join(temp_dir, 'credentials_dir') + if not os.path.exists(credentials_dir): + os.mkdir(credentials_dir) + + logging.info('Copying the Google Application Default Credentials file.') + + is_windows = 'windows' in os.name.lower() + application_default_path_suffix = ( + '\\gcloud\\application_default_credentials.json' if is_windows else + '.config/gcloud/application_default_credentials.json') + application_default_path_file = os.path.join( + str(Path.home()), application_default_path_suffix) + application_default_path_copied = os.path.join( + credentials_dir, 'application_default_credentials.json') + + if os.path.exists(application_default_path_file): + shutil.copyfile( + application_default_path_file, application_default_path_copied) + else: + logging.info( + 'GCP credentials will not be available for the transform service ' + 'since could not find the Google Cloud application default ' + 'credentials file at the expected location %s.' % + application_default_path_file) + + # Creating the dependencies volume. + dependencies_dir = os.path.join(temp_dir, 'dependencies_dir') + if not os.path.exists(dependencies_dir): + os.mkdir(dependencies_dir) + + self._environmental_variables = {} + self._environmental_variables['CREDENTIALS_VOLUME'] = credentials_dir + self._environmental_variables['DEPENDENCIES_VOLUME'] = dependencies_dir + self._environmental_variables['TRANSFORM_SERVICE_PORT'] = str(port) + self._environmental_variables['BEAM_VERSION'] = beam_version + + # Setting an empty requirements file + requirements_file_name = os.path.join(dependencies_dir, 'requirements.txt') + with open(requirements_file_name, 'w') as _: + pass + self._environmental_variables['PYTHON_REQUIREMENTS_FILE_NAME'] = ( + 'requirements.txt') + + self._docker_compose_start_command_prefix = [] + self._docker_compose_start_command_prefix.append('docker-compose') + self._docker_compose_start_command_prefix.append('-p') + self._docker_compose_start_command_prefix.append(project_name) + self._docker_compose_start_command_prefix.append('-f') + self._docker_compose_start_command_prefix.append(compose_file) + + def _get_channel(self): + channel_options = [("grpc.max_receive_message_length", -1), + ("grpc.max_send_message_length", -1)] + if hasattr(grpc, 'local_channel_credentials'): + # TODO: update this to support secure non-local channels. + return grpc.secure_channel( + self._address, + grpc.local_channel_credentials(), + options=channel_options) + else: + return grpc.insecure_channel(self._address, options=channel_options) + + def __enter__(self): + self.start() + self.wait_till_up(-1) + + self._channel = self._get_channel() + + from apache_beam import external + return external.ExpansionAndArtifactRetrievalStub(self._channel.__enter__()) + + def __exit__(self, *args): + self.shutdown() + self._channel.__exit__(*args) + + def _run_docker_compose_command(self, command, output_override=None): + cmd = [] + cmd.extend(self._docker_compose_start_command_prefix) + cmd.extend(command) + + myenv = os.environ.copy() + myenv.update(self._environmental_variables) + + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=myenv) + std_out, _ = process.communicate() + + if output_override: + output_override.write(std_out) + else: + print(std_out.decode(errors='backslashreplace')) + + def start(self): + with self._launcher_lock: + self._run_docker_compose_command(['up', '-d']) + + def shutdown(self): + with self._launcher_lock: + self._run_docker_compose_command(['down']) + + def status(self): + with self._launcher_lock: + self._run_docker_compose_command(['ps']) + + def wait_till_up(self, timeout_ms): + channel = self._get_channel() + + timeout_ms = ( + TransformServiceLauncher._DEFAULT_START_WAIT_TIMEOUT + if timeout_ms <= 0 else timeout_ms) + + # Waiting till the service is up. + channel_ready = grpc.channel_ready_future(channel) + wait_secs = .1 + start_time = time.time() + while True: + if (time.time() - start_time) * 1000 > timeout_ms > 0: + raise ValueError( + 'Transform service did not start in %s seconds.' % + (timeout_ms / 1000)) + try: + channel_ready.result(timeout=wait_secs) + break + except (grpc.FutureTimeoutError, grpc.RpcError): + wait_secs *= 1.2 + logging.log( + logging.WARNING if wait_secs > 1 else logging.DEBUG, + 'Waiting for the transform service to be ready at %s.', + self._address) + + logging.info('Transform service ' + self._project_name + ' started.') + + def _get_status(self): + tmp = tempfile.NamedTemporaryFile(delete=False) + self._run_docker_compose_command(['ps'], tmp) + tmp.close() + return tmp.name + + +def main(argv): + parser = argparse.ArgumentParser() + parser.add_argument('--project_name', help='Docker Compose project name.') + parser.add_argument( + '--command', + required=True, + choices=_COMMAND_POSSIBLE_VALUES, + help='Command to run. Possible values are ' + + ', '.join(_COMMAND_POSSIBLE_VALUES)) + parser.add_argument( + '--port', + type=int, + default=-1, + help='External visible port of the transform service.') + parser.add_argument( + '--beam_version', + required=True, + help='Beam version of the expansion service containers to be used.') + + known_args, _ = parser.parse_known_args(argv) + + project_name = ( + TransformServiceLauncher._DEFAULT_PROJECT_NAME + if known_args.project_name is None else known_args.project_name) + logging.info( + 'Starting the Beam Transform Service at %s.' % ( + 'the default port' if known_args.port < 0 else + (' port ' + str(known_args.port)))) + launcher = TransformServiceLauncher( + project_name, known_args.port, known_args.beam_version) + + if known_args.command == 'up': + launcher.start() + launcher.wait_till_up(-1) + elif known_args.command == 'down': + launcher.shutdown() + elif known_args.command == 'ps': + launcher.status() + else: + raise ValueError( + 'Unknown command %s possible values are %s' % + (known_args.command, ', '.join(_COMMAND_POSSIBLE_VALUES))) + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + main(sys.argv) diff --git a/sdks/python/apache_beam/version.py b/sdks/python/apache_beam/version.py index 2ac20b0e3a616..a69e3839fff3e 100644 --- a/sdks/python/apache_beam/version.py +++ b/sdks/python/apache_beam/version.py @@ -17,4 +17,4 @@ """Apache Beam SDK version information and utilities.""" -__version__ = '2.49.0.dev' +__version__ = '2.52.0.dev' diff --git a/sdks/python/apache_beam/yaml/README.md b/sdks/python/apache_beam/yaml/README.md index 5a119db64af82..3ba78784c997c 100644 --- a/sdks/python/apache_beam/yaml/README.md +++ b/sdks/python/apache_beam/yaml/README.md @@ -56,9 +56,11 @@ writes it out in json format. pipeline: transforms: - type: ReadFromCsv - path: /path/to/input*.csv + config: + path: /path/to/input*.csv - type: WriteToJson - path: /path/to/output.json + config: + path: /path/to/output.json input: ReadFromCsv ``` @@ -68,13 +70,17 @@ We can also add a transformation pipeline: transforms: - type: ReadFromCsv - path: /path/to/input*.csv - - type: PyFilter - keep: "lambda x: x.col3 > 100" + config: + path: /path/to/input*.csv + - type: Filter + config: + language: python + keep: "col3 > 100" input: ReadFromCsv - type: WriteToJson - path: /path/to/output.json - input: PyFilter + config: + path: /path/to/output.json + input: Filter ``` or two. @@ -83,16 +89,21 @@ or two. pipeline: transforms: - type: ReadFromCsv - path: /path/to/input*.csv - - type: PyFilter - keep: "lambda x: x.col3 > 100" + config: + path: /path/to/input*.csv + - type: Filter + config: + language: python + keep: "col3 > 100" input: ReadFromCsv - type: Sql name: MySqlTransform - query: "select col1, count(*) as cnt from PCOLLECTION group by col1" - input: PyFilter + config: + query: "select col1, count(*) as cnt from PCOLLECTION group by col1" + input: Filter - type: WriteToJson - path: /path/to/output.json + config: + path: /path/to/output.json input: MySqlTransform ``` @@ -105,14 +116,19 @@ pipeline: transforms: - type: ReadFromCsv - path: /path/to/input*.csv - - type: PyFilter - keep: "lambda x: x.col3 > 100" + config: + path: /path/to/input*.csv + - type: Filter + config: + language: python + keep: "col3 > 100" - type: Sql name: MySqlTransform - query: "select col1, count(*) as cnt from PCOLLECTION group by col1" + config: + query: "select col1, count(*) as cnt from PCOLLECTION group by col1" - type: WriteToJson - path: /path/to/output.json + config: + path: /path/to/output.json ``` As syntactic sugar, we can name the first and last transforms in our pipeline @@ -124,19 +140,24 @@ pipeline: source: type: ReadFromCsv - path: /path/to/input*.csv + config: + path: /path/to/input*.csv transforms: - - type: PyFilter - keep: "lambda x: x.col3 > 100" + - type: Filter + config: + language: python + keep: "col3 > 100" - type: Sql name: MySqlTransform - query: "select col1, count(*) as cnt from PCOLLECTION group by col1" + config: + query: "select col1, count(*) as cnt from PCOLLECTION group by col1" sink: type: WriteToJson - path: /path/to/output.json + config: + path: /path/to/output.json ``` Arbitrary non-linear pipelines are supported as well, though in this case @@ -147,14 +168,17 @@ Here we read two sources, join them, and write two outputs. pipeline: - type: ReadFromCsv name: ReadLeft - path: /path/to/left*.csv + config: + path: /path/to/left*.csv - type: ReadFromCsv name: ReadRight - path: /path/to/right*.csv + config: + path: /path/to/right*.csv - type: Sql - query: select left.col1, right.col2 from left join right using (col3) + config: + query: select left.col1, right.col2 from left join right using (col3) input: left: ReadLeft right: ReadRight @@ -162,17 +186,21 @@ pipeline: - type: WriteToJson name: WriteAll input: Sql - path: /path/to/all.json + config: + path: /path/to/all.json - - type: PyFilter + - type: Filter name: FilterToBig input: Sql - keep: "lambda x: x.col2 > 100" + config: + language: python + keep: "col2 > 100" - type: WriteToCsv name: WriteBig input: FilterToBig - path: /path/to/big.csv + config: + path: /path/to/big.csv ``` One can, however, nest `chains` within a non-linear pipeline. @@ -183,14 +211,17 @@ that has a single input and contains its own sink. pipeline: - type: ReadFromCsv name: ReadLeft - path: /path/to/left*.csv + config: + path: /path/to/left*.csv - type: ReadFromCsv name: ReadRight - path: /path/to/right*.csv + config: + path: /path/to/right*.csv - type: Sql - query: select left.col1, right.col2 from left join right using (col3) + config: + query: select left.col1, right.col2 from left join right using (col3) input: left: ReadLeft right: ReadRight @@ -198,21 +229,29 @@ pipeline: - type: WriteToJson name: WriteAll input: Sql - path: /path/to/all.json + config: + path: /path/to/all.json - type: chain name: ExtraProcessingForBigRows input: Sql transforms: - - type: PyFilter - keep: "lambda x: x.col2 > 100" - - type: PyFilter - keep: "lambda x: len(x.col1) > 10" - - type: PyFilter - keep: "lambda x: x.col1 > 'z'" + - type: Filter + config: + language: python + keep: "col2 > 100" + - type: Filter + config: + language: python + keep: "len(col1) > 10" + - type: Filter + config: + language: python + keep: "col1 > 'z'" sink: type: WriteToCsv - path: /path/to/big.csv + config: + path: /path/to/big.csv ``` ## Windowing @@ -230,14 +269,16 @@ pipeline: type: chain transforms: - type: ReadFromPubSub - topic: myPubSubTopic + config: + topic: myPubSubTopic - type: WindowInto windowing: type: fixed size: 60 - type: SomeAggregation - type: WriteToPubSub - topic: anotherPubSubTopic + config: + topic: anotherPubSubTopic ``` Rather than using an explicit `WindowInto` operation, one may instead tag a @@ -249,14 +290,16 @@ pipeline: type: chain transforms: - type: ReadFromPubSub - topic: myPubSubTopic + config: + topic: myPubSubTopic - type: SomeAggregation windowing: type: sliding size: 60 period: 10 - type: WriteToPubSub - topic: anotherPubSubTopic + config: + topic: anotherPubSubTopic ``` Note that the `Sql` operation itself is often a from of aggregation, and @@ -268,14 +311,17 @@ pipeline: type: chain transforms: - type: ReadFromPubSub - topic: myPubSubTopic + config: + topic: myPubSubTopic - type: Sql - query: "select col1, count(*) as c from PCOLLECTION" + config: + query: "select col1, count(*) as c from PCOLLECTION" windowing: type: sessions gap: 60 - type: WriteToPubSub - topic: anotherPubSubTopic + config: + topic: anotherPubSubTopic ``` The specified windowing is applied to all inputs, in this case resulting in @@ -285,14 +331,17 @@ a join per window. pipeline: - type: ReadFromPubSub name: ReadLeft - topic: leftTopic + config: + topic: leftTopic - type: ReadFromPubSub name: ReadRight - topic: rightTopic + config: + topic: rightTopic - type: Sql - query: select left.col1, right.col2 from left join right using (col3) + config: + query: select left.col1, right.col2 from left join right using (col3) input: left: ReadLeft right: ReadRight @@ -310,14 +359,17 @@ pipeline: type: chain transforms: - type: ReadFromPubSub - topic: myPubSubTopic + config: + topic: myPubSubTopic windowing: type: fixed size: 60 - type: Sql - query: "select col1, count(*) as c from PCOLLECTION" + config: + query: "select col1, count(*) as c from PCOLLECTION" - type: WriteToPubSub - topic: anotherPubSubTopic + config: + topic: anotherPubSubTopic ``` One can also specify windowing at the top level of a pipeline (or composite), @@ -330,11 +382,14 @@ pipeline: type: chain transforms: - type: ReadFromPubSub - topic: myPubSubTopic + config: + topic: myPubSubTopic - type: Sql - query: "select col1, count(*) as c from PCOLLECTION" + config: + query: "select col1, count(*) as c from PCOLLECTION" - type: WriteToPubSub - topic: anotherPubSubTopic + config: + topic: anotherPubSubTopic windowing: type: fixed size: 60 @@ -349,18 +404,21 @@ pipeline: source: type: ReadFromPubSub - topic: myPubSubTopic + config: + topic: myPubSubTopic windowing: type: fixed size: 10 transforms: - type: Sql - query: "select col1, count(*) as c from PCOLLECTION" + config: + query: "select col1, count(*) as c from PCOLLECTION" sink: type: WriteToCsv - path: /path/to/output.json + config: + path: /path/to/output.json windowing: type: fixed size: 300 @@ -384,16 +442,18 @@ pipeline: type: chain source: type: ReadFromCsv - path: /path/to/input*.csv + config: + path: /path/to/input*.csv transforms: - type: MyCustomTransform - args: + config: arg: whatever sink: type: WriteToJson - path: /path/to/output.json + config: + path: /path/to/output.json providers: - type: javaJar diff --git a/sdks/python/apache_beam/yaml/cache_provider_artifacts.py b/sdks/python/apache_beam/yaml/cache_provider_artifacts.py new file mode 100644 index 0000000000000..6c96dd3b0fd92 --- /dev/null +++ b/sdks/python/apache_beam/yaml/cache_provider_artifacts.py @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import time + +from apache_beam.version import __version__ as beam_version +from apache_beam.yaml import yaml_provider + + +def cache_provider_artifacts(): + providers_by_id = {} + for providers in yaml_provider.standard_providers().values(): + for provider in providers: + # Dedup for better logging. + providers_by_id[id(provider)] = provider + for provider in providers_by_id.values(): + t = time.time() + artifacts = provider.cache_artifacts() + if artifacts: + logging.info( + 'Cached %s in %0.03f seconds.', ', '.join(artifacts), time.time() - t) + if '.dev' not in beam_version: + # Also cache a base python venv for fast cloning. + t = time.time() + artifacts = yaml_provider.PypiExpansionService._create_venv_to_clone() + logging.info('Cached %s in %0.03f seconds.', artifacts, time.time() - t) + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + cache_provider_artifacts() diff --git a/sdks/python/apache_beam/yaml/json_utils.py b/sdks/python/apache_beam/yaml/json_utils.py new file mode 100644 index 0000000000000..e2cb03dc96a0b --- /dev/null +++ b/sdks/python/apache_beam/yaml/json_utils.py @@ -0,0 +1,183 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Utilities for converting between JSON and Beam Schema'd data. + +For internal use, no backward compatibility guarantees. +""" + +import json +from typing import Any +from typing import Callable +from typing import Dict + +import apache_beam as beam +from apache_beam.portability.api import schema_pb2 +from apache_beam.typehints import schemas + +JSON_ATOMIC_TYPES_TO_BEAM = { + 'boolean': schema_pb2.BOOLEAN, + 'integer': schema_pb2.INT64, + 'number': schema_pb2.DOUBLE, + 'string': schema_pb2.STRING, +} + + +def json_schema_to_beam_schema( + json_schema: Dict[str, Any]) -> schema_pb2.Schema: + """Returns a Beam schema equivalent for the given Json schema.""" + def maybe_nullable(beam_type, nullable): + if nullable: + beam_type.nullable = True + return beam_type + + json_type = json_schema.get('type', None) + if json_type != 'object': + raise ValueError('Expected object type, got {json_type}.') + if 'properties' not in json_schema: + # Technically this is a valid (vacuous) schema, but as it's not generally + # meaningful, throw an informative error instead. + # (We could add a flag to allow this degenerate case.) + raise ValueError('Missing properties for {json_schema}.') + required = set(json_schema.get('required', [])) + return schema_pb2.Schema( + fields=[ + schemas.schema_field( + name, + maybe_nullable(json_type_to_beam_type(t), name not in required)) + for (name, t) in json_schema['properties'].items() + ]) + + +def json_type_to_beam_type(json_type: Dict[str, Any]) -> schema_pb2.FieldType: + """Returns a Beam schema type for the given Json (schema) type.""" + if not isinstance(json_type, dict) or 'type' not in json_type: + raise ValueError(f'Malformed type {json_type}.') + type_name = json_type['type'] + if type_name in JSON_ATOMIC_TYPES_TO_BEAM: + return schema_pb2.FieldType( + atomic_type=JSON_ATOMIC_TYPES_TO_BEAM[type_name]) + elif type_name == 'array': + return schema_pb2.FieldType( + array_type=schema_pb2.ArrayType( + element_type=json_type_to_beam_type(json_type['items']))) + elif type_name == 'object': + if 'properties' in json_type: + return schema_pb2.FieldType( + row_type=schema_pb2.RowType( + schema=json_schema_to_beam_schema(json_type))) + elif 'additionalProperties' in json_type: + return schema_pb2.FieldType( + map_type=schema_pb2.MapType( + key_type=schema_pb2.FieldType(atomic_type=schema_pb2.STRING), + value_type=json_type_to_beam_type( + json_type['additionalProperties']))) + else: + raise ValueError( + f'Object type must have either properties or additionalProperties, ' + f'got {json_type}.') + else: + raise ValueError(f'Unable to convert {json_type} to a Beam schema.') + + +def json_to_row(beam_type: schema_pb2.FieldType) -> Callable[[Any], Any]: + """Returns a callable converting Json objects to Beam rows of the given type. + + The input to the returned callable is expected to conform to the Json schema + corresponding to this Beam type. + """ + type_info = beam_type.WhichOneof("type_info") + if type_info == "atomic_type": + return lambda value: value + elif type_info == "array_type": + element_converter = json_to_row(beam_type.array_type.element_type) + return lambda value: [element_converter(e) for e in value] + elif type_info == "iterable_type": + element_converter = json_to_row(beam_type.iterable_type.element_type) + return lambda value: [element_converter(e) for e in value] + elif type_info == "map_type": + if beam_type.map_type.key_type.atomic_type != schema_pb2.STRING: + raise TypeError( + f'Only strings allowd as map keys when converting from JSON, ' + f'found {beam_type}') + value_converter = json_to_row(beam_type.map_type.value_type) + return lambda value: {k: value_converter(v) for (k, v) in value.items()} + elif type_info == "row_type": + converters = { + field.name: json_to_row(field.type) + for field in beam_type.row_type.schema.fields + } + return lambda value: beam.Row( + ** + {name: convert(value[name]) + for (name, convert) in converters.items()}) + elif type_info == "logical_type": + return lambda value: value + else: + raise ValueError(f"Unrecognized type_info: {type_info!r}") + + +def json_parser(beam_schema: schema_pb2.Schema) -> Callable[[bytes], beam.Row]: + """Returns a callable converting Json strings to Beam rows of the given type. + + The input to the returned callable is expected to conform to the Json schema + corresponding to this Beam type. + """ + to_row = json_to_row( + schema_pb2.FieldType(row_type=schema_pb2.RowType(schema=beam_schema))) + return lambda s: to_row(json.loads(s)) + + +def row_to_json(beam_type: schema_pb2.FieldType) -> Callable[[Any], Any]: + """Returns a callable converting rows of the given type to Json objects.""" + type_info = beam_type.WhichOneof("type_info") + if type_info == "atomic_type": + return lambda value: value + elif type_info == "array_type": + element_converter = row_to_json(beam_type.array_type.element_type) + return lambda value: [element_converter(e) for e in value] + elif type_info == "iterable_type": + element_converter = row_to_json(beam_type.iterable_type.element_type) + return lambda value: [element_converter(e) for e in value] + elif type_info == "map_type": + if beam_type.map_type.key_type.atomic_type != schema_pb2.STRING: + raise TypeError( + f'Only strings allowd as map keys when converting to JSON, ' + f'found {beam_type}') + value_converter = row_to_json(beam_type.map_type.value_type) + return lambda value: {k: value_converter(v) for (k, v) in value.items()} + elif type_info == "row_type": + converters = { + field.name: row_to_json(field.type) + for field in beam_type.row_type.schema.fields + } + return lambda row: { + name: convert(getattr(row, name)) + for (name, convert) in converters.items() + } + elif type_info == "logical_type": + return lambda value: value + else: + raise ValueError(f"Unrecognized type_info: {type_info!r}") + + +def json_formater( + beam_schema: schema_pb2.Schema) -> Callable[[beam.Row], bytes]: + """Returns a callable converting rows of the given schema to Json strings.""" + convert = row_to_json( + schema_pb2.FieldType(row_type=schema_pb2.RowType(schema=beam_schema))) + return lambda row: json.dumps(convert(row), sort_keys=True).encode('utf-8') diff --git a/sdks/python/apache_beam/yaml/main.py b/sdks/python/apache_beam/yaml/main.py index 730d0f0ad0d13..e2ec8df9cfc35 100644 --- a/sdks/python/apache_beam/yaml/main.py +++ b/sdks/python/apache_beam/yaml/main.py @@ -20,8 +20,13 @@ import yaml import apache_beam as beam +from apache_beam.typehints.schemas import LogicalType +from apache_beam.typehints.schemas import MillisInstant from apache_beam.yaml import yaml_transform +# Workaround for https://github.com/apache/beam/issues/28151. +LogicalType.register_logical_type(MillisInstant) + def _configure_parser(argv): parser = argparse.ArgumentParser() @@ -46,22 +51,28 @@ def _pipeline_spec_from_args(known_args): raise ValueError( "Exactly one of pipeline_spec or pipeline_spec_file must be set.") - return yaml.load(pipeline_yaml, Loader=yaml_transform.SafeLineLoader) + return pipeline_yaml def run(argv=None): yaml_transform._LOGGER.setLevel('INFO') known_args, pipeline_args = _configure_parser(argv) - pipeline_spec = _pipeline_spec_from_args(known_args) + pipeline_yaml = _pipeline_spec_from_args(known_args) + pipeline_spec = yaml.load(pipeline_yaml, Loader=yaml_transform.SafeLineLoader) - with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( - pipeline_args, - pickle_library='cloudpickle', - **pipeline_spec.get('options', {}))) as p: + with beam.Pipeline( # linebreak for better yapf formatting + options=beam.options.pipeline_options.PipelineOptions( + pipeline_args, + pickle_library='cloudpickle', + **yaml_transform.SafeLineLoader.strip_metadata(pipeline_spec.get( + 'options', {}))), + display_data={'yaml': pipeline_yaml}) as p: print("Building pipeline...") yaml_transform.expand_pipeline(p, pipeline_spec) print("Running pipeline...") if __name__ == '__main__': + import logging + logging.getLogger().setLevel(logging.INFO) run() diff --git a/sdks/python/apache_beam/yaml/pipeline.schema.yaml b/sdks/python/apache_beam/yaml/pipeline.schema.yaml index 6aae74bcd32a5..ef0d9fe0f2621 100644 --- a/sdks/python/apache_beam/yaml/pipeline.schema.yaml +++ b/sdks/python/apache_beam/yaml/pipeline.schema.yaml @@ -60,6 +60,10 @@ $defs: type: array items: $ref: '#/$defs/transform' + extra_transforms: + type: array + items: + $ref: '#/$defs/transform' source: $ref: '#/$defs/transform' sink: @@ -100,7 +104,10 @@ $defs: - if: properties: { type: { const: composite }} then: - $ref: '#/$defs/nestedTransform' + allOf: + - $ref: '#/$defs/nestedTransform' + - properties: + extra_transforms: { not: {} } - if: properties: { type: { const: chain }} then: @@ -118,6 +125,7 @@ $defs: type: { type: string } transforms: type: object + properties: { __line__: {}} additionalProperties: type: string required: diff --git a/sdks/python/apache_beam/yaml/readme_test.py b/sdks/python/apache_beam/yaml/readme_test.py index 3d63268201489..d918d18e11dd0 100644 --- a/sdks/python/apache_beam/yaml/readme_test.py +++ b/sdks/python/apache_beam/yaml/readme_test.py @@ -32,6 +32,8 @@ import apache_beam as beam from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.typehints import trivial_inference +from apache_beam.yaml import yaml_mapping +from apache_beam.yaml import yaml_provider from apache_beam.yaml import yaml_transform @@ -84,13 +86,16 @@ def guess_name_and_type(expr): typ, = [t for t in typ.__args__ if t is not type(None)] return name, typ - output_schema = [ - guess_name_and_type(expr) for expr in m.group(1).split(',') - ] - output_element = beam.Row(**{name: typ() for name, typ in output_schema}) - return next(iter(inputs.values())) | beam.Map( - lambda _: output_element).with_output_types( - trivial_inference.instance_to_type(output_element)) + if m.group(1) == '*': + return inputs['PCOLLECTION'] | beam.Filter(lambda _: True) + else: + output_schema = [ + guess_name_and_type(expr) for expr in m.group(1).split(',') + ] + output_element = beam.Row(**{name: typ() for name, typ in output_schema}) + return next(iter(inputs.values())) | beam.Map( + lambda _: output_element).with_output_types( + trivial_inference.instance_to_type(output_element)) class FakeReadFromPubSub(beam.PTransform): @@ -121,7 +126,7 @@ def expand(self, pcoll): RENDER_DIR = None -TEST_PROVIDERS = { +TEST_TRANSFORMS = { 'Sql': FakeSql, 'ReadFromPubSub': FakeReadFromPubSub, 'WriteToPubSub': FakeWriteToPubSub, @@ -129,6 +134,12 @@ def expand(self, pcoll): } +class TestProvider(yaml_provider.InlineProvider): + def _affinity(self, other): + # Always try to choose this one. + return float('inf') + + class TestEnvironment: def __enter__(self): self.tempdir = tempfile.TemporaryDirectory() @@ -162,7 +173,7 @@ def replace_recursive(spec, transform_type, arg_name, arg_value): for (key, value) in spec.items() } if spec.get('type', None) == transform_type: - spec[arg_name] = arg_value + spec['config'][arg_name] = arg_value return spec elif isinstance(spec, list): return [ @@ -196,8 +207,14 @@ def test(self): os.path.join(RENDER_DIR, test_name + '.png') ] options['render_leaf_composite_nodes'] = ['.*'] + test_provider = TestProvider(TEST_TRANSFORMS) + test_sql_mapping_provider = yaml_mapping.SqlMappingProvider(test_provider) p = beam.Pipeline(options=PipelineOptions(**options)) - yaml_transform.expand_pipeline(p, modified_yaml, TEST_PROVIDERS) + yaml_transform.expand_pipeline( + p, + modified_yaml, + yaml_provider.merge_providers( + [test_provider, test_sql_mapping_provider])) if test_type == 'BUILD': return p.run().wait_until_finish() @@ -219,15 +236,15 @@ def parse_test_methods(markdown_lines): if code_lines: if code_lines[0].startswith('- type:'): # Treat this as a fragment of a larger pipeline. + # pylint: disable=not-an-iterable code_lines = [ 'pipeline:', ' type: chain', ' transforms:', ' - type: ReadFromCsv', - ' path: whatever', - ] + [ - ' ' + line for line in code_lines # pylint: disable=not-an-iterable - ] + ' config:', + ' path: whatever', + ] + [' ' + line for line in code_lines] if code_lines[0] == 'pipeline:': yaml_pipeline = '\n'.join(code_lines) if 'providers:' in yaml_pipeline: @@ -249,6 +266,10 @@ def createTestSuite(name, path): ReadMeTest = createTestSuite( 'ReadMeTest', os.path.join(os.path.dirname(__file__), 'README.md')) +ErrorHandlingTest = createTestSuite( + 'ErrorHandlingTest', + os.path.join(os.path.dirname(__file__), 'yaml_errors.md')) + if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--render_dir', default=None) diff --git a/sdks/python/apache_beam/yaml/standard_io.yaml b/sdks/python/apache_beam/yaml/standard_io.yaml new file mode 100644 index 0000000000000..c4748483b04bd --- /dev/null +++ b/sdks/python/apache_beam/yaml/standard_io.yaml @@ -0,0 +1,114 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This file enumerates the various IOs that are available by default as +# top-level transforms in Beam's YAML. +# +# Note that there may be redundant implementations. In these cases the specs +# should be kept in sync. +# TODO(yaml): See if this can be enforced programmatically. + +- type: renaming + transforms: + 'ReadFromBigQuery': 'ReadFromBigQuery' + 'WriteToBigQuery': 'WriteToBigQuery' + config: + mappings: + 'ReadFromBigQuery': + query: 'query' + table: 'tableSpec' + fields: 'selectedFields' + row_restriction: 'rowRestriction' + 'WriteToBigQuery': + table: 'table' + create_disposition: 'createDisposition' + write_disposition: 'writeDisposition' + error_handling: 'errorHandling' + underlying_provider: + type: beamJar + transforms: + 'ReadFromBigQuery': 'beam:schematransform:org.apache.beam:bigquery_storage_read:v1' + 'WriteToBigQuery': 'beam:schematransform:org.apache.beam:bigquery_storage_write:v2' + config: + gradle_target: 'sdks:java:extensions:sql:expansion-service:shadowJar' + +- type: python + transforms: + 'ReadFromBigQuery': 'apache_beam.yaml.yaml_io.read_from_bigquery' + # Disable until https://github.com/apache/beam/issues/28162 is resolved. + # 'WriteToBigQuery': 'apache_beam.yaml.yaml_io.write_to_bigquery' + 'ReadFromText': 'apache_beam.yaml.yaml_io.read_from_text' + 'WriteToText': 'apache_beam.yaml.yaml_io.write_to_text' + 'ReadFromPubSub': 'apache_beam.yaml.yaml_io.read_from_pubsub' + 'WriteToPubSub': 'apache_beam.yaml.yaml_io.write_to_pubsub' + +# Declared as a renaming transform to avoid exposing all +# (implementation-specific) pandas arguments and aligning with possible Java +# implementation. +# Invoking these directly as a PyTransform is still an option for anyone wanting +# to use these power-features in a language-dependent manner. +- type: renaming + transforms: + 'ReadFromCsv': 'ReadFromCsv' + 'WriteToCsv': 'WriteToCsv' + 'ReadFromJson': 'ReadFromJson' + 'WriteToJson': 'WriteToJson' + 'ReadFromParquet': 'ReadFromParquet' + 'WriteToParquet': 'WriteToParquet' + 'ReadFromAvro': 'ReadFromAvro' + 'WriteToAvro': 'WriteToAvro' + config: + mappings: + 'ReadFromCsv': + path: 'path' + 'WriteToCsv': + path: 'path' + 'ReadFromJson': + path: 'path' + 'WriteToJson': + path: 'path' + 'ReadFromParquet': + path: 'file_pattern' + 'WriteToParquet': + path: 'file_path_prefix' + 'ReadFromAvro': + path: 'file_pattern' + 'WriteToAvro': + path: 'file_path_prefix' + defaults: + 'ReadFromParquet': + as_rows: True + 'ReadFromAvro': + as_rows: True + underlying_provider: + type: python + transforms: + 'ReadFromCsv': 'apache_beam.io.ReadFromCsv' + 'WriteToCsv': 'apache_beam.io.WriteToCsv' + 'ReadFromJson': 'apache_beam.io.ReadFromJson' + 'WriteToJson': 'apache_beam.io.WriteToJson' + 'ReadFromParquet': 'apache_beam.io.ReadFromParquet' + 'WriteToParquet': 'apache_beam.io.WriteToParquet' + 'ReadFromAvro': 'apache_beam.io.ReadFromAvro' + 'WriteToAvro': 'apache_beam.io.WriteToAvro' + +- type: beamJar + transforms: + 'WriteToCsv': 'beam:schematransform:org.apache.beam:csv_write:v1' + 'WriteToJson': 'beam:schematransform:org.apache.beam:json_write:v1' + config: + gradle_target: 'sdks:java:extensions:schemaio-expansion-service:shadowJar' diff --git a/sdks/python/apache_beam/yaml/standard_providers.yaml b/sdks/python/apache_beam/yaml/standard_providers.yaml index 04d33b53d64c3..cdb4036f98c21 100644 --- a/sdks/python/apache_beam/yaml/standard_providers.yaml +++ b/sdks/python/apache_beam/yaml/standard_providers.yaml @@ -19,7 +19,8 @@ # TODO(robertwb): Perhaps auto-generate this file? - type: 'beamJar' - gradle_target: 'sdks:java:extensions:sql:expansion-service:shadowJar' - version: BEAM_VERSION + config: + gradle_target: 'sdks:java:extensions:sql:expansion-service:shadowJar' + version: BEAM_VERSION transforms: Sql: 'beam:external:java:sql:v1' diff --git a/sdks/python/apache_beam/yaml/yaml_errors.md b/sdks/python/apache_beam/yaml/yaml_errors.md new file mode 100644 index 0000000000000..e7a60f750a106 --- /dev/null +++ b/sdks/python/apache_beam/yaml/yaml_errors.md @@ -0,0 +1,196 @@ + + +# Beam YAML Error Handling + +The larger one's pipeline gets, the more common it is to encounter "exceptional" +data that is malformatted, doesn't handle the proper preconditions, or otherwise +breaks during processing. Generally any such record will cause the pipeline to +permanently fail, but often it is desirable to allow the pipeline to continue, +re-directing bad records to another path for special handling or simply +recording them for later off-line analysis. This is often called the +"dead letter queue" pattern. + +Beam YAML has special support for this pattern if the transform supports a +`error_handling` config parameter with an `output` field. For example, +the following code will write all "good" processed records to one file and +any "bad" records to a separate file. + +``` +pipeline: + transforms: + - type: ReadFromCsv + config: + path: /path/to/input*.csv + + - type: MapToFields + input: ReadFromCsv + config: + language: python + fields: + col1: col1 + # This could raise a divide-by-zero error. + ratio: col2 / col3 + error_handling: + output: my_error_output + + - type: WriteToJson + input: MapToFields + config: + path: /path/to/output.json + + - type: WriteToJson + name: WriteErrorsToJson + input: MapToFields.my_error_output + config: + path: /path/to/errors.json +``` + +Note that with `error_handling` declared, `MapToFields.my_error_output` +**must** be consumed; to ignore it will be an error. Any use is fine, e.g. +logging the bad records to stdout would be sufficient (though not recommended +for a robust pipeline). + +Some transforms allow for extra arguments in their error_handling config, e.g. +for Python functions one can give a `threshold` which limits the relative number +of records that can be bad before considering the entire pipeline a failure + +``` +pipeline: + transforms: + - type: ReadFromCsv + config: + path: /path/to/input*.csv + + - type: MapToFields + input: ReadFromCsv + config: + language: python + fields: + col1: col1 + # This could raise a divide-by-zero error. + ratio: col2 / col3 + error_handling: + output: my_error_output + # If more than 10% of records throw an error, stop the pipeline. + threshold: 0.1 + + - type: WriteToJson + input: MapToFields + config: + path: /path/to/output.json + + - type: WriteToJson + name: WriteErrorsToJson + input: MapToFields.my_error_output + config: + path: /path/to/errors.json +``` + +One can do arbitrary further processing on these failed records if desired, +e.g. + +``` +pipeline: + transforms: + - type: ReadFromCsv + config: + path: /path/to/input*.csv + + - type: MapToFields + name: ComputeRatio + input: ReadFromCsv + config: + language: python + fields: + col1: col1 + # This could raise a divide-by-zero error. + ratio: col2 / col3 + error_handling: + output: my_error_output + + - type: MapToFields + name: ComputeRatioForBadRecords + input: ComputeRatio.my_error_output + config: + language: python + fields: + col1: col1 + ratio: col2 / (col3 + 1) + error_handling: + output: still_bad + + - type: WriteToJson + # Takes as input everything from the "success" path of both transforms. + input: [ComputeRatio, ComputeRatioForBadRecords] + config: + path: /path/to/output.json + + - type: WriteToJson + name: WriteErrorsToJson + # These failed the first and the second transform. + input: ComputeRatioForBadRecords.still_bad + config: + path: /path/to/errors.json +``` + +When using the `chain` syntax, the required error consumption can happen +in an `extra_transforms` block. + +``` +pipeline: + type: chain + transforms: + - type: ReadFromCsv + config: + path: /path/to/input*.csv + + - type: MapToFields + name: SomeStep + config: + language: python + fields: + col1: col1 + # This could raise a divide-by-zero error. + ratio: col2 / col3 + error_handling: + output: errors + + - type: MapToFields + name: AnotherStep + config: + language: python + fields: + col1: col1 + # This could raise a divide-by-zero error. + inverse_ratio: 1 / ratio + error_handling: + output: errors + + - type: WriteToJson + config: + path: /path/to/output.json + + extra_transforms: + - type: WriteToJson + name: WriteErrors + input: [SomeStep.errors, AnotherStep.errors] + config: + path: /path/to/errors.json +``` diff --git a/sdks/python/apache_beam/yaml/yaml_io.py b/sdks/python/apache_beam/yaml/yaml_io.py new file mode 100644 index 0000000000000..b2bf150fa5587 --- /dev/null +++ b/sdks/python/apache_beam/yaml/yaml_io.py @@ -0,0 +1,382 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""This module contains the Python implementations for the builtin IOs. + +They are referenced from standard_io.py. + +Note that in the case that they overlap with other (likely Java) +implementations of the same transforms, the configs must be kept in sync. +""" + +import io +import os +from typing import Any +from typing import Callable +from typing import Iterable +from typing import List +from typing import Mapping +from typing import Optional +from typing import Tuple + +import fastavro +import yaml + +import apache_beam as beam +import apache_beam.io as beam_io +from apache_beam.io import ReadFromBigQuery +from apache_beam.io import WriteToBigQuery +from apache_beam.io import avroio +from apache_beam.io.gcp.bigquery import BigQueryDisposition +from apache_beam.portability.api import schema_pb2 +from apache_beam.typehints import schemas +from apache_beam.yaml import json_utils +from apache_beam.yaml import yaml_mapping +from apache_beam.yaml import yaml_provider + + +def read_from_text(path: str): + # TODO(yaml): Consider passing the filename and offset, possibly even + # by default. + return beam_io.ReadFromText(path) | beam.Map(lambda s: beam.Row(line=s)) + + +@beam.ptransform_fn +def write_to_text(pcoll, path: str): + try: + field_names = [ + name for name, + _ in schemas.named_fields_from_element_type(pcoll.element_type) + ] + except Exception as exn: + raise ValueError( + "WriteToText requires an input schema with exactly one field.") from exn + if len(field_names) != 1: + raise ValueError( + "WriteToText requires an input schema with exactly one field, got %s" % + field_names) + sole_field_name, = field_names + return pcoll | beam.Map( + lambda x: str(getattr(x, sole_field_name))) | beam.io.WriteToText(path) + + +def read_from_bigquery( + query=None, table=None, row_restriction=None, fields=None): + if query is None: + assert table is not None + else: + assert table is None and row_restriction is None and fields is None + return ReadFromBigQuery( + query=query, + table=table, + row_restriction=row_restriction, + selected_fields=fields, + method='DIRECT_READ', + output_type='BEAM_ROW') + + +def write_to_bigquery( + table, + *, + create_disposition=BigQueryDisposition.CREATE_IF_NEEDED, + write_disposition=BigQueryDisposition.WRITE_APPEND, + error_handling=None): + class WriteToBigQueryHandlingErrors(beam.PTransform): + def default_label(self): + return 'WriteToBigQuery' + + def expand(self, pcoll): + write_result = pcoll | WriteToBigQuery( + table, + method=WriteToBigQuery.Method.STORAGE_WRITE_API + if error_handling else None, + create_disposition=create_disposition, + write_disposition=write_disposition, + temp_file_format='AVRO') + if error_handling and 'output' in error_handling: + # TODO: Support error rates. + return { + 'post_write': write_result.failed_rows_with_errors + | beam.FlatMap(lambda x: None), + error_handling['output']: write_result.failed_rows_with_errors + } + else: + if write_result._method == WriteToBigQuery.Method.FILE_LOADS: + # Never returns errors, just fails. + return { + 'post_write': write_result.destination_load_jobid_pairs + | beam.FlatMap(lambda x: None) + } + else: + + # This should likely be pushed into the BQ read itself to avoid + # the possibility of silently ignoring errors. + def raise_exception(failed_row_with_error): + raise RuntimeError(failed_row_with_error.error_message) + + _ = write_result.failed_rows_with_errors | beam.Map(raise_exception) + return { + 'post_write': write_result.failed_rows_with_errors + | beam.FlatMap(lambda x: None) + } + + return WriteToBigQueryHandlingErrors() + + +def _create_parser( + format, + schema: Any) -> Tuple[schema_pb2.Schema, Callable[[bytes], beam.Row]]: + if format == 'raw': + if schema: + raise ValueError('raw format does not take a schema') + return ( + schema_pb2.Schema(fields=[schemas.schema_field('payload', bytes)]), + lambda payload: beam.Row(payload=payload)) + elif format == 'json': + beam_schema = json_utils.json_schema_to_beam_schema(schema) + return beam_schema, json_utils.json_parser(beam_schema) + elif format == 'avro': + beam_schema = avroio.avro_schema_to_beam_schema(schema) + covert_to_row = avroio.avro_dict_to_beam_row(schema, beam_schema) + return ( + beam_schema, + lambda record: covert_to_row( + fastavro.schemaless_reader(io.BytesIO(record), schema))) + else: + raise ValueError(f'Unknown format: {format}') + + +def _create_formatter( + format, schema: Any, + beam_schema: schema_pb2.Schema) -> Callable[[beam.Row], bytes]: + if format == 'raw': + if schema: + raise ValueError('raw format does not take a schema') + field_names = [field.name for field in beam_schema.fields] + if len(field_names) != 1: + raise ValueError(f'Expecting exactly one field, found {field_names}') + return lambda row: getattr(row, field_names[0]) + elif format == 'json': + return json_utils.json_formater(beam_schema) + elif format == 'avro': + avro_schema = schema or avroio.beam_schema_to_avro_schema(beam_schema) + from_row = avroio.beam_row_to_avro_dict(avro_schema, beam_schema) + + def formatter(row): + buffer = io.BytesIO() + fastavro.schemaless_writer(buffer, avro_schema, from_row(row)) + buffer.seek(0) + return buffer.read() + + return formatter + else: + raise ValueError(f'Unknown format: {format}') + + +@beam.ptransform_fn +@yaml_mapping.maybe_with_exception_handling_transform_fn +def read_from_pubsub( + root, + *, + topic: Optional[str] = None, + subscription: Optional[str] = None, + format: str, + schema: Optional[Any] = None, + attributes: Optional[Iterable[str]] = None, + attributes_map: Optional[str] = None, + id_attribute: Optional[str] = None, + timestamp_attribute: Optional[str] = None): + """Reads messages from Cloud Pub/Sub. + + Args: + topic: Cloud Pub/Sub topic in the form + "projects//topics/". If provided, subscription must be + None. + subscription: Existing Cloud Pub/Sub subscription to use in the + form "projects//subscriptions/". If not + specified, a temporary subscription will be created from the specified + topic. If provided, topic must be None. + format: The expected format of the message payload. Currently suported + formats are + + - raw: Produces records with a single `payload` field whose contents + are the raw bytes of the pubsub message. + + schema: Schema specification for the given format. + attributes: List of attribute keys whose values will be flattened into the + output message as additional fields. For example, if the format is `raw` + and attributes is `["a", "b"]` then this read will produce elements of + the form `Row(payload=..., a=..., b=...)`. + attribute_map: Name of a field in which to store the full set of attributes + associated with this message. For example, if the format is `raw` and + `attribute_map` is set to `"attrs"` then this read will produce elements + of the form `Row(payload=..., attrs=...)` where `attrs` is a Map type + of string to string. + If both `attributes` and `attribute_map` are set, the overlapping + attribute values will be present in both the flattened structure and the + attribute map. + id_attribute: The attribute on incoming Pub/Sub messages to use as a unique + record identifier. When specified, the value of this attribute (which + can be any string that uniquely identifies the record) will be used for + deduplication of messages. If not provided, we cannot guarantee + that no duplicate data will be delivered on the Pub/Sub stream. In this + case, deduplication of the stream will be strictly best effort. + timestamp_attribute: Message value to use as element timestamp. If None, + uses message publishing time as the timestamp. + + Timestamp values should be in one of two formats: + + - A numerical value representing the number of milliseconds since the + Unix epoch. + - A string in RFC 3339 format, UTC timezone. Example: + ``2015-10-29T23:41:41.123Z``. The sub-second component of the + timestamp is optional, and digits beyond the first three (i.e., time + units smaller than milliseconds) may be ignored. + """ + if topic and subscription: + raise TypeError('Only one of topic and subscription may be specified.') + elif not topic and not subscription: + raise TypeError('One of topic or subscription may be specified.') + payload_schema, parser = _create_parser(format, schema) + extra_fields: List[schema_pb2.Field] = [] + if not attributes and not attributes_map: + mapper = lambda msg: parser(msg) + else: + if isinstance(attributes, str): + attributes = [attributes] + if attributes: + extra_fields.extend( + [schemas.schema_field(attr, str) for attr in attributes]) + if attributes_map: + extra_fields.append( + schemas.schema_field(attributes_map, Mapping[str, str])) + + def mapper(msg): + values = parser(msg.data).as_dict() + if attributes: + # Should missing attributes be optional or parse errors? + for attr in attributes: + values[attr] = msg.attributes[attr] + if attributes_map: + values[attributes_map] = msg.attributes + return beam.Row(**values) + + output = ( + root + | beam.io.ReadFromPubSub( + topic=topic, + subscription=subscription, + with_attributes=bool(attributes or attributes_map), + id_label=id_attribute, + timestamp_attribute=timestamp_attribute) + | 'ParseMessage' >> beam.Map(mapper)) + output.element_type = schemas.named_tuple_from_schema( + schema_pb2.Schema(fields=list(payload_schema.fields) + extra_fields)) + return output + + +@beam.ptransform_fn +@yaml_mapping.maybe_with_exception_handling_transform_fn +def write_to_pubsub( + pcoll, + *, + topic: str, + format: str, + schema: Optional[Any] = None, + attributes: Optional[Iterable[str]] = None, + attributes_map: Optional[str] = None, + id_attribute: Optional[str] = None, + timestamp_attribute: Optional[str] = None): + """Writes messages from Cloud Pub/Sub. + + Args: + topic: Cloud Pub/Sub topic in the form "/topics//". + format: How to format the message payload. Currently suported + formats are + + - raw: Expects a message with a single field (excluding + attribute-related fields )whose contents are used as the raw bytes + of the pubsub message. + + schema: Schema specification for the given format. + attributes: List of attribute keys whose values will be pulled out as + PubSub message attributes. For example, if the format is `raw` + and attributes is `["a", "b"]` then elements of the form + `Row(any_field=..., a=..., b=...)` will result in PubSub messages whose + payload has the contents of any_field and whose attribute will be + populated with the values of `a` and `b`. + attribute_map: Name of a string-to-string map field in which to pull a set + of attributes associated with this message. For example, if the format + is `raw` and `attribute_map` is set to `"attrs"` then elements of the form + `Row(any_field=..., attrs=...)` will result in PubSub messages whose + payload has the contents of any_field and whose attribute will be + populated with the values from attrs. + If both `attributes` and `attribute_map` are set, the union of attributes + from these two sources will be used to populate the PubSub message + attributes. + id_attribute: If set, will set an attribute for each Cloud Pub/Sub message + with the given name and a unique value. This attribute can then be used + in a ReadFromPubSub PTransform to deduplicate messages. + timestamp_attribute: If set, will set an attribute for each Cloud Pub/Sub + message with the given name and the message's publish time as the value. + """ + input_schema = schemas.schema_from_element_type(pcoll.element_type) + + extra_fields: List[str] = [] + if isinstance(attributes, str): + attributes = [attributes] + if attributes: + extra_fields.extend(attributes) + if attributes_map: + extra_fields.append(attributes_map) + + def attributes_extractor(row): + if attributes_map: + attribute_values = dict(getattr(row, attributes_map)) + else: + attribute_values = {} + if attributes: + attribute_values.update({attr: getattr(row, attr) for attr in attributes}) + return attribute_values + + schema_names = set(f.name for f in input_schema.fields) + missing_attribute_names = set(extra_fields) - schema_names + if missing_attribute_names: + raise ValueError( + f'Attribute fields {missing_attribute_names} ' + f'not found in schema fields {schema_names}') + + payload_schema = schema_pb2.Schema( + fields=[ + field for field in input_schema.fields + if field.name not in extra_fields + ]) + formatter = _create_formatter(format, schema, payload_schema) + return ( + pcoll | beam.Map( + lambda row: beam.io.gcp.pubsub.PubsubMessage( + formatter(row), attributes_extractor(row))) + | beam.io.WriteToPubSub( + topic, + with_attributes=True, + id_label=id_attribute, + timestamp_attribute=timestamp_attribute)) + + +def io_providers(): + with open(os.path.join(os.path.dirname(__file__), 'standard_io.yaml')) as fin: + return yaml_provider.parse_providers(yaml.load(fin, Loader=yaml.SafeLoader)) diff --git a/sdks/python/apache_beam/yaml/yaml_io_test.py b/sdks/python/apache_beam/yaml/yaml_io_test.py new file mode 100644 index 0000000000000..7071860a7bf19 --- /dev/null +++ b/sdks/python/apache_beam/yaml/yaml_io_test.py @@ -0,0 +1,448 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import io +import json +import logging +import unittest + +import fastavro +import mock + +import apache_beam as beam +from apache_beam.io.gcp.pubsub import PubsubMessage +from apache_beam.testing.util import AssertThat +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to +from apache_beam.yaml.yaml_transform import YamlTransform + + +class FakeReadFromPubSub: + def __init__( + self, + topic, + messages, + subscription=None, + id_attribute=None, + timestamp_attribute=None): + self._topic = topic + self._subscription = subscription + self._messages = messages + self._id_attribute = id_attribute + self._timestamp_attribute = timestamp_attribute + + def __call__( + self, + *, + topic, + subscription, + with_attributes, + id_label, + timestamp_attribute): + assert topic == self._topic + assert id_label == self._id_attribute + assert timestamp_attribute == self._timestamp_attribute + assert subscription == self._subscription + if with_attributes: + data = self._messages + else: + data = [x.data for x in self._messages] + return beam.Create(data) + + +class FakeWriteToPubSub: + def __init__( + self, topic, messages, id_attribute=None, timestamp_attribute=None): + self._topic = topic + self._messages = messages + self._id_attribute = id_attribute + self._timestamp_attribute = timestamp_attribute + + def __call__(self, topic, *, with_attributes, id_label, timestamp_attribute): + assert topic == self._topic + assert with_attributes is True + assert id_label == self._id_attribute + assert timestamp_attribute == self._timestamp_attribute + return AssertThat(equal_to(self._messages)) + + +class YamlPubSubTest(unittest.TestCase): + def test_simple_read(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + with mock.patch('apache_beam.io.ReadFromPubSub', + FakeReadFromPubSub( + topic='my_topic', + messages=[PubsubMessage(b'msg1', {'attr': 'value1'}), + PubsubMessage(b'msg2', + {'attr': 'value2'})])): + result = p | YamlTransform( + ''' + type: ReadFromPubSub + config: + topic: my_topic + format: raw + ''') + assert_that( + result, + equal_to([beam.Row(payload=b'msg1'), beam.Row(payload=b'msg2')])) + + def test_read_with_attribute(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + with mock.patch('apache_beam.io.ReadFromPubSub', + FakeReadFromPubSub( + topic='my_topic', + messages=[PubsubMessage(b'msg1', {'attr': 'value1'}), + PubsubMessage(b'msg2', + {'attr': 'value2'})])): + result = p | YamlTransform( + ''' + type: ReadFromPubSub + config: + topic: my_topic + format: raw + attributes: [attr] + ''') + assert_that( + result, + equal_to([ + beam.Row(payload=b'msg1', attr='value1'), + beam.Row(payload=b'msg2', attr='value2') + ])) + + def test_read_with_attribute_map(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + with mock.patch('apache_beam.io.ReadFromPubSub', + FakeReadFromPubSub( + topic='my_topic', + messages=[PubsubMessage(b'msg1', {'attr': 'value1'}), + PubsubMessage(b'msg2', + {'attr': 'value2'})])): + result = p | YamlTransform( + ''' + type: ReadFromPubSub + config: + topic: my_topic + format: raw + attributes_map: attrMap + ''') + assert_that( + result, + equal_to([ + beam.Row(payload=b'msg1', attrMap={'attr': 'value1'}), + beam.Row(payload=b'msg2', attrMap={'attr': 'value2'}) + ])) + + def test_read_with_id_attribute(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + with mock.patch('apache_beam.io.ReadFromPubSub', + FakeReadFromPubSub( + topic='my_topic', + messages=[PubsubMessage(b'msg1', {'attr': 'value1'}), + PubsubMessage(b'msg2', {'attr': 'value2'})], + id_attribute='some_attr')): + result = p | YamlTransform( + ''' + type: ReadFromPubSub + config: + topic: my_topic + format: raw + id_attribute: some_attr + ''') + assert_that( + result, + equal_to([beam.Row(payload=b'msg1'), beam.Row(payload=b'msg2')])) + + _avro_schema = { + 'type': 'record', + 'name': 'ec', + 'fields': [{ + 'name': 'label', 'type': 'string' + }, { + 'name': 'rank', 'type': 'int' + }] + } + + def _encode_avro(self, data): + buffer = io.BytesIO() + fastavro.schemaless_writer(buffer, self._avro_schema, data) + buffer.seek(0) + return buffer.read() + + def test_read_avro(self): + + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + with mock.patch( + 'apache_beam.io.ReadFromPubSub', + FakeReadFromPubSub( + topic='my_topic', + messages=[PubsubMessage(self._encode_avro({'label': '37a', + 'rank': 1}), {}), + PubsubMessage(self._encode_avro({'label': '389a', + 'rank': 2}), {})])): + result = p | YamlTransform( + ''' + type: ReadFromPubSub + config: + topic: my_topic + format: avro + schema: %s + ''' % json.dumps(self._avro_schema)) + assert_that( + result, + equal_to( + [beam.Row(label='37a', rank=1), # linebreak + beam.Row(label='389a', rank=2)])) + + def test_read_json(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + with mock.patch('apache_beam.io.ReadFromPubSub', + FakeReadFromPubSub( + topic='my_topic', + messages=[PubsubMessage( + b'{"generator": {"x": 0, "y": 0}, "rank": 1}', + {'weierstrass': 'y^2+y=x^3-x', 'label': '37a'}) + ])): + result = p | YamlTransform( + ''' + type: ReadFromPubSub + config: + topic: my_topic + format: json + schema: + type: object + properties: + generator: + type: object + properties: + x: {type: integer} + y: {type: integer} + rank: {type: integer} + attributes: [label] + attributes_map: other + ''') + assert_that( + result, + equal_to([ + beam.Row( + generator=beam.Row(x=0, y=0), + rank=1, + label='37a', + other={ + 'label': '37a', 'weierstrass': 'y^2+y=x^3-x' + }) + ])) + + def test_read_json_with_error_handling(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + with mock.patch( + 'apache_beam.io.ReadFromPubSub', + FakeReadFromPubSub(topic='my_topic', + messages=[PubsubMessage('{"some_int": 123}', + attributes={}), + PubsubMessage('unparsable', + attributes={})])): + result = p | YamlTransform( + ''' + type: ReadFromPubSub + config: + topic: my_topic + format: json + schema: + type: object + properties: + some_int: {type: integer} + error_handling: + output: errors + ''') + assert_that( + result['good'], + equal_to([beam.Row(some_int=123)]), + label='CheckGood') + assert_that( + result['errors'] | beam.Map(lambda error: error.element), + equal_to(['unparsable']), + label='CheckErrors') + + def test_read_json_without_error_handling(self): + with self.assertRaises(Exception): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + with mock.patch( + 'apache_beam.io.ReadFromPubSub', + FakeReadFromPubSub(topic='my_topic', + messages=[PubsubMessage('{"some_int": 123}', + attributes={}), + PubsubMessage('unparsable', + attributes={})])): + _ = p | YamlTransform( + ''' + type: ReadFromPubSub + config: + topic: my_topic + format: json + schema: + type: object + properties: + some_int: {type: integer} + ''') + + def test_simple_write(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + with mock.patch('apache_beam.io.WriteToPubSub', + FakeWriteToPubSub(topic='my_topic', + messages=[PubsubMessage(b'msg1', {}), + PubsubMessage(b'msg2', {})])): + _ = ( + p | beam.Create([beam.Row(a=b'msg1'), beam.Row(a=b'msg2')]) + | YamlTransform( + ''' + type: WriteToPubSub + input: input + config: + topic: my_topic + format: raw + ''')) + + def test_write_with_attribute(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + with mock.patch('apache_beam.io.WriteToPubSub', + FakeWriteToPubSub( + topic='my_topic', + messages=[PubsubMessage(b'msg1', {'attr': 'value1'}), + PubsubMessage(b'msg2', + {'attr': 'value2'})])): + _ = ( + p | beam.Create([ + beam.Row(a=b'msg1', attr='value1'), + beam.Row(a=b'msg2', attr='value2') + ]) | YamlTransform( + ''' + type: WriteToPubSub + input: input + config: + topic: my_topic + format: raw + attributes: [attr] + ''')) + + def test_write_with_attribute_map(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + with mock.patch('apache_beam.io.WriteToPubSub', + FakeWriteToPubSub(topic='my_topic', + messages=[PubsubMessage(b'msg1', + {'a': 'b'}), + PubsubMessage(b'msg2', + {'c': 'd'})])): + _ = ( + p | beam.Create([ + beam.Row(a=b'msg1', attrMap={'a': 'b'}), + beam.Row(a=b'msg2', attrMap={'c': 'd'}) + ]) | YamlTransform( + ''' + type: WriteToPubSub + input: input + config: + topic: my_topic + format: raw + attributes_map: attrMap + ''')) + + def test_write_with_id_attribute(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + with mock.patch('apache_beam.io.WriteToPubSub', + FakeWriteToPubSub(topic='my_topic', + messages=[PubsubMessage(b'msg1', {}), + PubsubMessage(b'msg2', {})], + id_attribute='some_attr')): + _ = ( + p | beam.Create([beam.Row(a=b'msg1'), beam.Row(a=b'msg2')]) + | YamlTransform( + ''' + type: WriteToPubSub + input: input + config: + topic: my_topic + format: raw + id_attribute: some_attr + ''')) + + def test_write_avro(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + with mock.patch( + 'apache_beam.io.WriteToPubSub', + FakeWriteToPubSub( + topic='my_topic', + messages=[PubsubMessage(self._encode_avro({'label': '37a', + 'rank': 1}), {}), + PubsubMessage(self._encode_avro({'label': '389a', + 'rank': 2}), {})])): + _ = ( + p | beam.Create( + [beam.Row(label='37a', rank=1), beam.Row(label='389a', rank=2)]) + | YamlTransform( + ''' + type: WriteToPubSub + input: input + config: + topic: my_topic + format: avro + ''')) + + def test_write_json(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + with mock.patch('apache_beam.io.WriteToPubSub', + FakeWriteToPubSub( + topic='my_topic', + messages=[PubsubMessage( + b'{"generator": {"x": 0, "y": 0}, "rank": 1}', + {'weierstrass': 'y^2+y=x^3-x', 'label': '37a'}) + ])): + _ = ( + p | beam.Create([ + beam.Row( + label='37a', + generator=beam.Row(x=0, y=0), + rank=1, + other={'weierstrass': 'y^2+y=x^3-x'}) + ]) | YamlTransform( + ''' + type: WriteToPubSub + input: input + config: + topic: my_topic + format: json + attributes: [label] + attributes_map: other + ''')) + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + unittest.main() diff --git a/sdks/python/apache_beam/yaml/yaml_mapping.md b/sdks/python/apache_beam/yaml/yaml_mapping.md index 193abac610ed8..653b4abe8b89e 100644 --- a/sdks/python/apache_beam/yaml/yaml_mapping.md +++ b/sdks/python/apache_beam/yaml/yaml_mapping.md @@ -29,9 +29,10 @@ To rename fields one can write ``` - type: MapToFields - fields: - new_col1: col1 - new_col2: col2 + config: + fields: + new_col1: col1 + new_col2: col2 ``` will result in an output where each record has two fields, @@ -43,10 +44,11 @@ be retained similar to the use of `*` in an SQL select statement. For example ``` - type: MapToFields - append: true - fields: - new_col1: col1 - new_col2: col2 + config: + append: true + fields: + new_col1: col1 + new_col2: col2 ``` will output records that have `new_col1` and `new_col2` as *additional* @@ -54,12 +56,13 @@ fields. When the append field is specified, one can drop fields as well, e.g. ``` - type: MapToFields - append: true - drop: - - col3 - fields: - new_col1: col1 - new_col2: col2 + config: + append: true + drop: + - col3 + fields: + new_col1: col1 + new_col2: col2 ``` which includes all original fiels *except* col3 in addition to outputting the @@ -74,10 +77,11 @@ This requires a language specification. For example ``` - type: MapToFields - language: python - fields: - new_col: "col1.upper()" - another_col: "col2 + col3" + config: + language: python + fields: + new_col: "col1.upper()" + another_col: "col2 + col3" ``` In addition, one can provide a full Python callable that takes the row as an @@ -87,16 +91,17 @@ for acceptable formats). Thus one can write ``` - type: MapToFields - language: python - fields: - new_col: - callable: | - import re - def my_mapping(row): - if re.match("[0-9]+", row.col1) and row.col2 > 0: - return "good" - else: - return "bad" + config: + language: python + fields: + new_col: + callable: | + import re + def my_mapping(row): + if re.match("[0-9]+", row.col1) and row.col2 > 0: + return "good" + else: + return "bad" ``` Once one reaches a certain level of complexity, it may be preferable to package @@ -104,35 +109,40 @@ this up as a dependency and simply refer to it by fully qualified name, e.g. ``` - type: MapToFields - language: python - fields: - new_col: - callable: pkg.module.fn + config: + language: python + fields: + new_col: + callable: pkg.module.fn ``` Currently, in addition to Python, SQL expressions are supported as well ``` - type: MapToFields - language: sql - fields: - new_col: "UPPER(col1)" - another_col: "col2 + col3" + config: + language: sql + fields: + new_col: "UPPER(col1)" + another_col: "col2 + col3" ``` ## FlatMap Sometimes it may be desirable to emit more (or less) than one record for each input record. This can be accomplished by mapping to an iterable type and -noting that the specific field should be exploded, e.g. +following the mapping with an Explode operation, e.g. ``` - type: MapToFields - language: python - fields: - new_col: "[col1.upper(), col1.lower(), col1.title()]" - another_col: "col2 + col3" - explode: new_col + config: + language: python + fields: + new_col: "[col1.upper(), col1.lower(), col1.title()]" + another_col: "col2 + col3" +- type: Explode + config: + fields: new_col ``` will result in three output records for every input record. @@ -142,55 +152,51 @@ product over all fields should be taken. For example ``` - type: MapToFields - language: python - fields: - new_col: "[col1.upper(), col1.lower(), col1.title()]" - another_col: "[col2 - 1, col2, col2 + 1]" - explode: [new_col, another_col] - cross_product: true + config: + language: python + fields: + new_col: "[col1.upper(), col1.lower(), col1.title()]" + another_col: "[col2 - 1, col2, col2 + 1]" +- type: Explode + config: + fields: [new_col, another_col] + cross_product: true ``` will emit nine records whereas ``` - type: MapToFields - language: python - fields: - new_col: "[col1.upper(), col1.lower(), col1.title()]" - another_col: "[col2 - 1, col2, col2 + 1]" - explode: [new_col, another_col] - cross_product: false + config: + language: python + fields: + new_col: "[col1.upper(), col1.lower(), col1.title()]" + another_col: "[col2 - 1, col2, col2 + 1]" +- type: Explode + config: + fields: [new_col, another_col] + cross_product: false ``` will only emit three. -If one is only exploding existing fields, a simpler `Explode` transform may be -used instead +The `Explode` operation can be used on its own if the field in question is +already an iterable type. ``` - type: Explode - explode: [col1] + config: + fields: [col1] ``` ## Filtering Sometimes it can be desirable to only keep records that satisfy a certain -criteria. This can be accomplished by specifying a keep parameter, e.g. - -``` -- type: MapToFields - language: python - fields: - new_col: "col1.upper()" - another_col: "col2 + col3" - keep: "col2 > 0" -``` - -Like explode, there is a simpler `Filter` transform useful when no mapping is -being done +criteria. This can be accomplished with a `Filter` transform, e.g. ``` - type: Filter - language: sql - keep: "col2 > 0" + config: + language: sql + keep: "col2 > 0" ``` diff --git a/sdks/python/apache_beam/yaml/yaml_mapping.py b/sdks/python/apache_beam/yaml/yaml_mapping.py index 7f959773320c0..889f7f1ee3092 100644 --- a/sdks/python/apache_beam/yaml/yaml_mapping.py +++ b/sdks/python/apache_beam/yaml/yaml_mapping.py @@ -16,10 +16,20 @@ # """This module defines the basic MapToFields operation.""" - import itertools +from typing import Any +from typing import Callable +from typing import Collection +from typing import Dict +from typing import Iterable +from typing import Mapping +from typing import Optional +from typing import Union + +import js2py import apache_beam as beam +from apache_beam.io.filesystems import FileSystems from apache_beam.typehints import row_type from apache_beam.typehints import trivial_inference from apache_beam.typehints.schemas import named_fields_from_element_type @@ -27,46 +37,183 @@ from apache_beam.yaml import yaml_provider -def _as_callable(original_fields, expr): +def _check_mapping_arguments( + transform_name, expression=None, callable=None, name=None, path=None): + # Argument checking + if not expression and not callable and not path and not name: + raise ValueError( + f'{transform_name} must specify either "expression", "callable", ' + f'or both "path" and "name"') + if expression and callable: + raise ValueError( + f'{transform_name} cannot specify both "expression" and "callable"') + if (expression or callable) and (path or name): + raise ValueError( + f'{transform_name} cannot specify "expression" or "callable" with ' + f'"path" or "name"') + if path and not name: + raise ValueError(f'{transform_name} cannot specify "path" without "name"') + if name and not path: + raise ValueError(f'{transform_name} cannot specify "name" without "path"') + + +# js2py's JsObjectWrapper object has a self-referencing __dict__ property +# that cannot be pickled without implementing the __getstate__ and +# __setstate__ methods. +class _CustomJsObjectWrapper(js2py.base.JsObjectWrapper): + def __init__(self, js_obj): + super().__init__(js_obj.__dict__['_obj']) + + def __getstate__(self): + return self.__dict__.copy() + + def __setstate__(self, state): + self.__dict__.update(state) + + +# TODO(yaml) Consider adding optional language version parameter to support +# ECMAScript 5 and 6 +def _expand_javascript_mapping_func( + original_fields, expression=None, callable=None, path=None, name=None): + if expression: + args = ', '.join(original_fields) + js_func = f'function fn({args}) {{return ({expression})}}' + js_callable = _CustomJsObjectWrapper(js2py.eval_js(js_func)) + return lambda __row__: js_callable(*__row__._asdict().values()) + + elif callable: + js_callable = _CustomJsObjectWrapper(js2py.eval_js(callable)) + return lambda __row__: js_callable(__row__._asdict()) + + else: + if not path.endswith('.js'): + raise ValueError(f'File "{path}" is not a valid .js file.') + udf_code = FileSystems.open(path).read().decode() + js = js2py.EvalJs() + js.eval(udf_code) + js_callable = _CustomJsObjectWrapper(getattr(js, name)) + return lambda __row__: js_callable(__row__._asdict()) + + +def _expand_python_mapping_func( + original_fields, expression=None, callable=None, path=None, name=None): + if path and name: + if not path.endswith('.py'): + raise ValueError(f'File "{path}" is not a valid .py file.') + py_file = FileSystems.open(path).read().decode() + + return python_callable.PythonCallableWithSource.load_from_script( + py_file, name) + + elif expression: + # TODO(robertwb): Consider constructing a single callable that takes + # the row and returns the new row, rather than invoking (and unpacking) + # for each field individually. + source = '\n'.join(['def fn(__row__):'] + [ + f' {name} = __row__.{name}' + for name in original_fields if name in expression + ] + [' return (' + expression + ')']) + + else: + source = callable + + return python_callable.PythonCallableWithSource(source) + + +def _as_callable(original_fields, expr, transform_name, language): if expr in original_fields: return expr + + # TODO(yaml): support a type parameter + # TODO(yaml): support an imports parameter + # TODO(yaml): support a requirements parameter (possibly at a higher level) + if isinstance(expr, str): + expr = {'expression': expr} + if not isinstance(expr, dict): + raise ValueError( + f"Ambiguous expression type (perhaps missing quoting?): {expr}") + elif len(expr) != 1 and ('path' not in expr or 'name' not in expr): + raise ValueError(f"Ambiguous expression type: {list(expr.keys())}") + + _check_mapping_arguments(transform_name, **expr) + + if language == "javascript": + return _expand_javascript_mapping_func(original_fields, **expr) + elif language == "python": + return _expand_python_mapping_func(original_fields, **expr) else: - # TODO(yaml): support a type parameter - # TODO(yaml): support an imports parameter - # TODO(yaml): support a requirements parameter (possibly at a higher level) - if isinstance(expr, str): - expr = {'expression': expr} - if not isinstance(expr, dict): - raise ValueError( - f"Ambiguous expression type (perhaps missing quoting?): {expr}") - elif len(expr) != 1: - raise ValueError(f"Ambiguous expression type: {list(expr.keys())}") - if 'expression' in expr: - # TODO(robertwb): Consider constructing a single callable that takes - # the row and returns the new row, rather than invoking (and unpacking) - # for each field individually. - source = '\n'.join(['def fn(__row__):'] + [ - f' {name} = __row__.{name}' - for name in original_fields if name in expr['expression'] - ] + [' return (' + expr['expression'] + ')']) - elif 'callable' in expr: - source = expr['callable'] - else: - raise ValueError(f"Unknown expression type: {list(expr.keys())}") - return python_callable.PythonCallableWithSource(source) + raise ValueError( + f'Unknown language for mapping transform: {language}. ' + 'Supported languages are "javascript" and "python."') + + +def exception_handling_args(error_handling_spec): + if error_handling_spec: + return { + 'dead_letter_tag' if k == 'output' else k: v + for (k, v) in error_handling_spec.items() + } + else: + return None + + +def _map_errors_to_standard_format(): + # TODO(https://github.com/apache/beam/issues/24755): Switch to MapTuple. + return beam.Map( + lambda x: beam.Row(element=x[0], msg=str(x[1][1]), stack=str(x[1][2]))) + + +def maybe_with_exception_handling(inner_expand): + def expand(self, pcoll): + wrapped_pcoll = beam.core._MaybePValueWithErrors( + pcoll, self._exception_handling_args) + return inner_expand(self, wrapped_pcoll).as_result( + _map_errors_to_standard_format()) + + return expand + + +def maybe_with_exception_handling_transform_fn(transform_fn): + def expand(pcoll, error_handling=None, **kwargs): + wrapped_pcoll = beam.core._MaybePValueWithErrors( + pcoll, exception_handling_args(error_handling)) + return transform_fn(wrapped_pcoll, + **kwargs).as_result(_map_errors_to_standard_format()) + + return expand # TODO(yaml): This should be available in all environments, in which case # we choose the one that matches best. class _Explode(beam.PTransform): - def __init__(self, fields, cross_product): + def __init__( + self, + fields: Union[str, Collection[str]], + cross_product: Optional[bool] = None, + error_handling: Optional[Mapping[str, Any]] = None): + if isinstance(fields, str): + fields = [fields] + if cross_product is None: + if len(fields) > 1: + raise ValueError( + 'cross_product must be specified true or false ' + 'when exploding multiple fields') + else: + # Doesn't matter. + cross_product = True self._fields = fields self._cross_product = cross_product + # TODO(yaml): Support standard error handling argument. + self._exception_handling_args = exception_handling_args(error_handling) + @maybe_with_exception_handling def expand(self, pcoll): all_fields = [ x for x, _ in named_fields_from_element_type(pcoll.element_type) ] + for field in self._fields: + if field not in all_fields: + raise ValueError(f'Exploding unknown field "{field}"') to_explode = self._fields def explode_cross_product(base, fields): @@ -86,11 +233,13 @@ def explode_zip(base, fields): copy[field] = values[ix] yield beam.Row(**copy) - return pcoll | beam.FlatMap( - lambda row: ( - explode_cross_product if self._cross_product else explode_zip)( - {name: getattr(row, name) for name in all_fields}, # yapf break - to_explode)) + return ( + pcoll + | beam.FlatMap( + lambda row: + (explode_cross_product if self._cross_product else explode_zip) + ({name: getattr(row, name) + for name in all_fields}, to_explode))) def infer_output_type(self, input_type): return row_type.RowTypeConstraint.from_fields([( @@ -98,158 +247,182 @@ def infer_output_type(self, input_type): trivial_inference.element_type(typ) if name in self._fields else typ) for (name, typ) in named_fields_from_element_type(input_type)]) + def with_exception_handling(self, **kwargs): + # It's possible there's an error in iteration... + self._exception_handling_args = kwargs + return self -# TODO(yaml): Should Filter and Explode be distinct operations from Project? -# We'll want these per-language. -@beam.ptransform.ptransform_fn -def _PythonProjectionTransform( - pcoll, *, fields, keep=None, explode=(), cross_product=True): - original_fields = [ - name for (name, _) in named_fields_from_element_type(pcoll.element_type) - ] - if keep: - if isinstance(keep, str) and keep in original_fields: - keep_fn = lambda row: getattr(row, keep) - else: - keep_fn = _as_callable(original_fields, keep) - filtered = pcoll | beam.Filter(keep_fn) +@beam.ptransform.ptransform_fn +@maybe_with_exception_handling_transform_fn +def _PyJsFilter( + pcoll, keep: Union[str, Dict[str, str]], language: Optional[str] = None): + try: + input_schema = dict(named_fields_from_element_type(pcoll.element_type)) + except (TypeError, ValueError) as exn: + if is_expr(keep): + raise ValueError("Can only use expressions on a schema'd input.") from exn + input_schema = {} # unused + + if isinstance(keep, str) and keep in input_schema: + keep_fn = lambda row: getattr(row, keep) else: - filtered = pcoll + keep_fn = _as_callable(list(input_schema.keys()), keep, "keep", language) + return pcoll | beam.Filter(keep_fn) - if list(fields.items()) == [(name, name) for name in original_fields]: - projected = filtered - else: - projected = filtered | beam.Select( - **{ - name: _as_callable(original_fields, expr) - for (name, expr) in fields.items() - }) - - if explode: - result = projected | _Explode(explode, cross_product=cross_product) - else: - result = projected - return result +def is_expr(v): + return isinstance(v, str) or (isinstance(v, dict) and 'expression' in v) -@beam.ptransform.ptransform_fn -def MapToFields( - pcoll, - yaml_create_transform, - *, - fields, - keep=None, - explode=(), - cross_product=None, - append=False, - drop=(), - language=None, - **language_keywords): - - if isinstance(explode, str): - explode = [explode] - if cross_product is None: - if len(explode) > 1: - # TODO(robertwb): Consider if true is an OK default. - raise ValueError( - 'cross_product must be specified true or false ' - 'when exploding multiple fields') - else: - # Doesn't matter. - cross_product = True +def normalize_fields(pcoll, fields, drop=(), append=False, language='generic'): + try: + input_schema = dict(named_fields_from_element_type(pcoll.element_type)) + except (TypeError, ValueError) as exn: + if drop: + raise ValueError("Can only drop fields on a schema'd input.") from exn + if append: + raise ValueError("Can only append fields on a schema'd input.") from exn + elif any(is_expr(x) for x in fields.values()): + raise ValueError("Can only use expressions on a schema'd input.") from exn + input_schema = {} - input_schema = dict(named_fields_from_element_type(pcoll.element_type)) + if isinstance(drop, str): + drop = [drop] if drop and not append: raise ValueError("Can only drop fields if append is true.") for name in drop: if name not in input_schema: raise ValueError(f'Dropping unknown field "{name}"') - for name in explode: - if not (name in fields or (append and name in input_schema)): - raise ValueError(f'Exploding unknown field "{name}"') if append: for name in fields: if name in input_schema and name not in drop: - raise ValueError(f'Redefinition of field "{name}"') + raise ValueError( + f'Redefinition of field "{name}". ' + 'Cannot append a field that already exists in original input.') + + if language == 'generic': + for expr in fields.values(): + if not isinstance(expr, str): + raise ValueError( + "Missing language specification. " + "Must specify a language when using a map with custom logic.") + missing = set(fields.values()) - set(input_schema.keys()) + if missing: + raise ValueError( + f"Missing language specification or unknown input fields: {missing}") if append: - fields = { + return input_schema, { **{name: name for name in input_schema.keys() if name not in drop}, **fields } + else: + return input_schema, fields - if language is None: - for name, expr in fields.items(): - if not isinstance(expr, str) or expr not in input_schema: - # TODO(robertw): Could consider defaulting to SQL, or another - # lowest-common-denominator expression language. - raise ValueError("Missing language specification.") - - # We should support this for all languages. - language = "python" - - if language in ("sql", "calcite"): - selects = [f'{expr} AS {name}' for (name, expr) in fields.items()] - query = "SELECT " + ", ".join(selects) + " FROM PCOLLECTION" - if keep: - query += " WHERE " + keep - - result = pcoll | yaml_create_transform({ - 'type': 'Sql', 'query': query, **language_keywords - }) - if explode: - # TODO(yaml): Implement via unnest. - result = result | _Explode(explode, cross_product) - - return result - - elif language == 'python': - return pcoll | yaml_create_transform({ - 'type': 'PyTransform', - 'constructor': __name__ + '._PythonProjectionTransform', - 'kwargs': { - 'fields': fields, - 'keep': keep, - 'explode': explode, - 'cross_product': cross_product, - }, - **language_keywords - }) - else: - # TODO(yaml): Support javascript expressions and UDFs. - # TODO(yaml): Support java by fully qualified name. - # TODO(yaml): Maybe support java lambdas? - raise ValueError( - f'Unknown language: {language}. ' - 'Supported languages are "sql" (alias calcite) and "python."') +@beam.ptransform.ptransform_fn +@maybe_with_exception_handling_transform_fn +def _PyJsMapToFields(pcoll, language='generic', **mapping_args): + input_schema, fields = normalize_fields( + pcoll, language=language, **mapping_args) + original_fields = list(input_schema.keys()) + + return pcoll | beam.Select( + **{ + name: _as_callable(original_fields, expr, name, language) + for (name, expr) in fields.items() + }) + + +class SqlMappingProvider(yaml_provider.Provider): + def __init__(self, sql_provider=None): + if sql_provider is None: + sql_provider = yaml_provider.beam_jar( + urns={'Sql': 'beam:external:java:sql:v1'}, + gradle_target='sdks:java:extensions:sql:expansion-service:shadowJar') + self._sql_provider = sql_provider + + def available(self): + return self._sql_provider.available() + + def cache_artifacts(self): + return self._sql_provider.cache_artifacts() + + def provided_transforms(self) -> Iterable[str]: + return [ + 'Filter-sql', + 'Filter-calcite', + 'MapToFields-sql', + 'MapToFields-calcite' + ] + + def create_transform( + self, + typ: str, + args: Mapping[str, Any], + yaml_create_transform: Callable[ + [Mapping[str, Any], Iterable[beam.PCollection]], beam.PTransform] + ) -> beam.PTransform: + if typ.startswith('Filter-'): + return _SqlFilterTransform( + self._sql_provider, yaml_create_transform, **args) + if typ.startswith('MapToFields-'): + return _SqlMapToFieldsTransform( + self._sql_provider, yaml_create_transform, **args) + else: + raise NotImplementedError(typ) + + def underlying_provider(self): + return self._sql_provider + + def to_json(self): + return {'type': "SqlMappingProvider"} + + +@beam.ptransform.ptransform_fn +def _SqlFilterTransform( + pcoll, sql_provider, yaml_create_transform, keep, language): + return pcoll | sql_provider.create_transform( + 'Sql', {'query': f'SELECT * FROM PCOLLECTION WHERE {keep}'}, + yaml_create_transform) -def create_mapping_provider(): +@beam.ptransform.ptransform_fn +def _SqlMapToFieldsTransform( + pcoll, sql_provider, yaml_create_transform, **mapping_args): + _, fields = normalize_fields(pcoll, **mapping_args) + + def extract_expr(name, v): + if isinstance(v, str): + return v + elif 'expression' in v: + return v['expression'] + else: + raise ValueError("Only expressions allowed in SQL at {name}.") + + selects = [ + f'({extract_expr(name, expr)}) AS {name}' + for (name, expr) in fields.items() + ] + query = "SELECT " + ", ".join(selects) + " FROM PCOLLECTION" + return pcoll | sql_provider.create_transform( + 'Sql', {'query': query}, yaml_create_transform) + + +def create_mapping_providers(): # These are MetaInlineProviders because their expansion is in terms of other # YamlTransforms, but in a way that needs to be deferred until the input # schema is known. - return yaml_provider.MetaInlineProvider({ - 'MapToFields': MapToFields, - 'Filter': ( - lambda yaml_create_transform, - keep, - **kwargs: MapToFields( - yaml_create_transform, - keep=keep, - fields={}, - append=True, - **kwargs)), - 'Explode': ( - lambda yaml_create_transform, - explode, - **kwargs: MapToFields( - yaml_create_transform, - explode=explode, - fields={}, - append=True, - **kwargs)), - }) + return [ + yaml_provider.InlineProvider({ + 'Explode': _Explode, + 'Filter-python': _PyJsFilter, + 'Filter-javascript': _PyJsFilter, + 'MapToFields-python': _PyJsMapToFields, + 'MapToFields-javascript': _PyJsMapToFields, + 'MapToFields-generic': _PyJsMapToFields, + }), + SqlMappingProvider(), + ] diff --git a/sdks/python/apache_beam/yaml/yaml_mapping_test.py b/sdks/python/apache_beam/yaml/yaml_mapping_test.py index 3305ff8b92a53..55032aeae52e9 100644 --- a/sdks/python/apache_beam/yaml/yaml_mapping_test.py +++ b/sdks/python/apache_beam/yaml/yaml_mapping_test.py @@ -41,10 +41,11 @@ def test_basic(self): ''' type: MapToFields input: input - language: python - fields: - label: label - isogeny: "label[-1]" + config: + language: python + fields: + label: label + isogeny: "label[-1]" ''') assert_that( result, @@ -62,9 +63,10 @@ def test_drop(self): ''' type: MapToFields input: input - fields: {} - append: true - drop: [conductor] + config: + fields: {} + append: true + drop: [conductor] ''') assert_that( result, @@ -80,17 +82,18 @@ def test_filter(self): elements = p | beam.Create(DATA) result = elements | YamlTransform( ''' - type: MapToFields + type: Filter input: input - language: python - fields: - label: label - keep: "rank > 0" + config: + language: python + keep: "rank > 0" ''') assert_that( - result, equal_to([ - beam.Row(label='37a'), - beam.Row(label='389a'), + result + | beam.Map(lambda named_tuple: beam.Row(**named_tuple._asdict())), + equal_to([ + beam.Row(label='37a', conductor=37, rank=1), + beam.Row(label='389a', conductor=389, rank=2), ])) def test_explode(self): @@ -102,14 +105,19 @@ def test_explode(self): ]) result = elements | YamlTransform( ''' - type: MapToFields + type: chain input: input - language: python - append: true - fields: - range: "range(a)" - explode: [range, b] - cross_product: true + transforms: + - type: MapToFields + config: + language: python + append: true + fields: + range: "range(a)" + - type: Explode + config: + fields: [range, b] + cross_product: true ''') assert_that( result, diff --git a/sdks/python/apache_beam/yaml/yaml_provider.py b/sdks/python/apache_beam/yaml/yaml_provider.py index 209e2178b8e5e..84399cd597b2a 100644 --- a/sdks/python/apache_beam/yaml/yaml_provider.py +++ b/sdks/python/apache_beam/yaml/yaml_provider.py @@ -21,15 +21,20 @@ import collections import hashlib +import inspect import json +import logging import os import subprocess import sys +import urllib.parse import uuid from typing import Any from typing import Callable +from typing import Dict from typing import Iterable from typing import Mapping +from typing import Optional import yaml from yaml.loader import SafeLoader @@ -42,8 +47,12 @@ from apache_beam.transforms import external from apache_beam.transforms import window from apache_beam.transforms.fully_qualified_named_transform import FullyQualifiedNamedTransform +from apache_beam.typehints import native_type_compatibility from apache_beam.typehints import schemas from apache_beam.typehints import trivial_inference +from apache_beam.typehints.schemas import named_tuple_to_schema +from apache_beam.typehints.schemas import typing_from_runner_api +from apache_beam.typehints.schemas import typing_to_runner_api from apache_beam.utils import python_callable from apache_beam.utils import subprocess_server from apache_beam.version import __version__ as beam_version @@ -55,20 +64,64 @@ def available(self) -> bool: """Returns whether this provider is available to use in this environment.""" raise NotImplementedError(type(self)) + def cache_artifacts(self) -> Optional[Iterable[str]]: + raise NotImplementedError(type(self)) + def provided_transforms(self) -> Iterable[str]: """Returns a list of transform type names this provider can handle.""" raise NotImplementedError(type(self)) + def config_schema(self, type): + return None + + def requires_inputs(self, typ: str, args: Mapping[str, Any]) -> bool: + """Returns whether this transform requires inputs. + + Specifically, if this returns True and inputs are not provided than an error + will be thrown. + + This is best-effort, primarily for better and earlier error messages. + """ + return not typ.startswith('Read') + def create_transform( self, typ: str, args: Mapping[str, Any], - yaml_create_transform: Callable[[Mapping[str, Any]], beam.PTransform] + yaml_create_transform: Callable[ + [Mapping[str, Any], Iterable[beam.PCollection]], beam.PTransform] ) -> beam.PTransform: """Creates a PTransform instance for the given transform type and arguments. """ raise NotImplementedError(type(self)) + def underlying_provider(self): + """If this provider is simply a proxy to another provider, return the + provider that should actually be used for affinity checking. + """ + return self + + def affinity(self, other: "Provider"): + """Returns a value approximating how good it would be for this provider + to be used immediately following a transform from the other provider + (e.g. to encourage fusion). + """ + # TODO(yaml): This is a very rough heuristic. Consider doing better. + # E.g. we could look at the the expected environments themselves. + # Possibly, we could provide multiple expansions and have the runner itself + # choose the actual implementation based on fusion (and other) criteria. + return ( + self.underlying_provider()._affinity(other) + + other.underlying_provider()._affinity(self)) + + def _affinity(self, other: "Provider"): + if self is other or self == other: + return 100 + elif type(self) == type(other): + return 10 + else: + return 0 + def as_provider(name, provider_or_constructor): if isinstance(provider_or_constructor, Provider): @@ -85,6 +138,8 @@ def as_provider_list(name, lst): class ExternalProvider(Provider): """A Provider implemented via the cross language transform service.""" + _provider_types: Dict[str, Callable[..., Provider]] = {} + def __init__(self, urns, service): self._urns = urns self._service = service @@ -93,21 +148,39 @@ def __init__(self, urns, service): def provided_transforms(self): return self._urns.keys() - def create_transform(self, type, args, yaml_create_transform): + def schema_transforms(self): if callable(self._service): self._service = self._service() if self._schema_transforms is None: try: - self._schema_transforms = [ - config.identifier + self._schema_transforms = { + config.identifier: config for config in external.SchemaAwareExternalTransform.discover( - self._service) - ] + self._service, ignore_errors=True) + } except Exception: - self._schema_transforms = [] + # It's possible this service doesn't vend schema transforms. + self._schema_transforms = {} + return self._schema_transforms + + def config_schema(self, type): + if self._urns[type] in self.schema_transforms(): + return named_tuple_to_schema( + self.schema_transforms()[self._urns[type]].configuration_schema) + + def requires_inputs(self, typ, args): + if self._urns[typ] in self.schema_transforms(): + return bool(self.schema_transforms()[self._urns[typ]].inputs) + else: + return super().requires_inputs(typ, args) + + def create_transform(self, type, args, yaml_create_transform): + if callable(self._service): + self._service = self._service() urn = self._urns[type] - if urn in self._schema_transforms: - return external.SchemaAwareExternalTransform(urn, self._service, **args) + if urn in self.schema_transforms(): + return external.SchemaAwareExternalTransform( + urn, self._service, rearrange_based_on_discovery=True, **args) else: return type >> self.create_external_transform(urn, args) @@ -117,51 +190,102 @@ def create_external_transform(self, urn, args): external.ImplicitSchemaPayloadBuilder(args).payload(), self._service) - @staticmethod - def provider_from_spec(spec): + @classmethod + def provider_from_spec(cls, spec): + from apache_beam.yaml.yaml_transform import SafeLineLoader + for required in ('type', 'transforms'): + if required not in spec: + raise ValueError( + f'Missing {required} in provider ' + f'at line {SafeLineLoader.get_line(spec)}') urns = spec['transforms'] type = spec['type'] - if spec.get('version', None) == 'BEAM_VERSION': - spec['version'] = beam_version - if type == 'javaJar': - return ExternalJavaProvider(urns, lambda: spec['jar']) - elif type == 'mavenJar': - return ExternalJavaProvider( - urns, - lambda: subprocess_server.JavaJarServer.path_to_maven_jar( - **{ - key: value - for (key, value) in spec.items() if key in [ - 'artifact_id', - 'group_id', - 'version', - 'repository', - 'classifier', - 'appendix' - ] - })) - elif type == 'beamJar': - return ExternalJavaProvider( - urns, - lambda: subprocess_server.JavaJarServer.path_to_beam_jar( - **{ - key: value - for (key, value) in spec.items() if key in - ['gradle_target', 'version', 'appendix', 'artifact_id'] - })) - elif type == 'pythonPackage': - return ExternalPythonProvider(urns, spec['packages']) - elif type == 'remote': - return RemoteProvider(spec['address']) - elif type == 'docker': - raise NotImplementedError() + config = SafeLineLoader.strip_metadata(spec.get('config', {})) + extra_params = set(SafeLineLoader.strip_metadata(spec).keys()) - set( + ['transforms', 'type', 'config']) + if extra_params: + raise ValueError( + f'Unexpected parameters in provider of type {type} ' + f'at line {SafeLineLoader.get_line(spec)}: {extra_params}') + if config.get('version', None) == 'BEAM_VERSION': + config['version'] = beam_version + if type in cls._provider_types: + try: + return cls._provider_types[type](urns, **config) + except Exception as exn: + raise ValueError( + f'Unable to instantiate provider of type {type} ' + f'at line {SafeLineLoader.get_line(spec)}: {exn}') from exn else: - raise NotImplementedError(f'Unknown provider type: {type}') - - + raise NotImplementedError( + f'Unknown provider type: {type} ' + f'at line {SafeLineLoader.get_line(spec)}.') + + @classmethod + def register_provider_type(cls, type_name): + def apply(constructor): + cls._provider_types[type_name] = constructor + return constructor + + return apply + + +@ExternalProvider.register_provider_type('javaJar') +def java_jar(urns, jar: str): + if not os.path.exists(jar): + parsed = urllib.parse.urlparse(jar) + if not parsed.scheme or not parsed.netloc: + raise ValueError(f'Invalid path or url: {jar}') + return ExternalJavaProvider(urns, lambda: jar) + + +@ExternalProvider.register_provider_type('mavenJar') +def maven_jar( + urns, + *, + artifact_id, + group_id, + version, + repository=subprocess_server.JavaJarServer.MAVEN_CENTRAL_REPOSITORY, + classifier=None, + appendix=None): + return ExternalJavaProvider( + urns, + lambda: subprocess_server.JavaJarServer.path_to_maven_jar( + artifact_id=artifact_id, + version=version, + repository=repository, + classifier=classifier, + appendix=appendix)) + + +@ExternalProvider.register_provider_type('beamJar') +def beam_jar( + urns, + *, + gradle_target, + appendix=None, + version=beam_version, + artifact_id=None): + return ExternalJavaProvider( + urns, + lambda: subprocess_server.JavaJarServer.path_to_beam_jar( + gradle_target=gradle_target, version=version, artifact_id=artifact_id) + ) + + +@ExternalProvider.register_provider_type('docker') +def docker(urns, **config): + raise NotImplementedError() + + +@ExternalProvider.register_provider_type('remote') class RemoteProvider(ExternalProvider): _is_available = None + def __init__(self, urns, address: str): + super().__init__(urns, service=address) + def available(self): if self._is_available is None: try: @@ -172,18 +296,39 @@ def available(self): self._is_available = False return self._is_available + def cache_artifacts(self): + pass + class ExternalJavaProvider(ExternalProvider): def __init__(self, urns, jar_provider): super().__init__( urns, lambda: external.JavaJarExpansionService(jar_provider())) + self._jar_provider = jar_provider def available(self): # pylint: disable=subprocess-run-check return subprocess.run(['which', 'java'], capture_output=True).returncode == 0 + def cache_artifacts(self): + return [self._jar_provider()] + + +@ExternalProvider.register_provider_type('python') +def python(urns, packages=()): + if packages: + return ExternalPythonProvider(urns, packages) + else: + return InlineProvider({ + name: + python_callable.PythonCallableWithSource.load_from_fully_qualified_name( + constructor) + for (name, constructor) in urns.items() + }) + +@ExternalProvider.register_provider_type('pythonPackage') class ExternalPythonProvider(ExternalProvider): def __init__(self, urns, packages): super().__init__(urns, PypiExpansionService(packages)) @@ -191,6 +336,9 @@ def __init__(self, urns, packages): def available(self): return True # If we're running this script, we have Python installed. + def cache_artifacts(self): + return [self._service._venv()] + def create_external_transform(self, urn, args): # Python transforms are "registered" by fully qualified name. return external.ExternalTransform( @@ -201,6 +349,12 @@ def create_external_transform(self, urn, args): }).payload(), self._service) + def _affinity(self, other: "Provider"): + if isinstance(other, InlineProvider): + return 50 + else: + return super()._affinity(other) + # This is needed because type inference can't handle *args, **kwargs forwarding. # TODO(BEAM-24755): Add support for type inference of through kwargs calls. @@ -241,21 +395,58 @@ def fn_takes_side_inputs(fn): class InlineProvider(Provider): - def __init__(self, transform_factories): + def __init__(self, transform_factories, no_input_transforms=()): self._transform_factories = transform_factories + self._no_input_transforms = set(no_input_transforms) def available(self): return True + def cache_artifacts(self): + pass + def provided_transforms(self): return self._transform_factories.keys() + def config_schema(self, typ): + factory = self._transform_factories[typ] + if isinstance(factory, type) and issubclass(factory, beam.PTransform): + # https://bugs.python.org/issue40897 + params = dict(inspect.signature(factory.__init__).parameters) + del params['self'] + else: + params = inspect.signature(factory).parameters + + def type_of(p): + t = p.annotation + if t == p.empty: + return Any + else: + return t + + names_and_types = [ + (name, typing_to_runner_api(type_of(p))) for name, p in params.items() + ] + return schema_pb2.Schema( + fields=[ + schema_pb2.Field(name=name, type=type) for name, + type in names_and_types + ]) + def create_transform(self, type, args, yaml_create_transform): return self._transform_factories[type](**args) def to_json(self): return {'type': "InlineProvider"} + def requires_inputs(self, typ, args): + if typ in self._no_input_transforms: + return False + elif hasattr(self._transform_factories[typ], '_yaml_requires_inputs'): + return self._transform_factories[typ]._yaml_requires_inputs + else: + return super().requires_inputs(typ, args) + class MetaInlineProvider(InlineProvider): def create_transform(self, type, args, yaml_create_transform): @@ -269,7 +460,35 @@ def create_transform(self, type, args, yaml_create_transform): } +def dicts_to_rows(o): + if isinstance(o, dict): + return beam.Row(**{k: dicts_to_rows(v) for k, v in o.items()}) + elif isinstance(o, list): + return [dicts_to_rows(e) for e in o] + else: + return o + + def create_builtin_provider(): + def create(elements: Iterable[Any], reshuffle: bool = True): + """Creates a collection containing a specified set of elements. + + YAML/JSON-style mappings will be interpreted as Beam rows. For example:: + + type: Create + elements: + - {first: 0, second: {str: "foo", values: [1, 2, 3]}} + + will result in a schema of the form (int, Row(string, List[int])). + + Args: + elements: The set of elements that should belong to the PCollection. + YAML/JSON-style mappings will be interpreted as Beam rows. + reshuffle (optional): Whether to introduce a reshuffle if there is more + than one element in the collection. Defaults to True. + """ + return beam.Create(dicts_to_rows(elements), reshuffle) + def with_schema(**args): # TODO: This is preliminary. def parse_type(spec): @@ -312,7 +531,10 @@ def extract_field(x, name): # Or should this be posargs, args? # pylint: disable=dangerous-default-value - def fully_qualified_named_transform(constructor, args=(), kwargs={}): + def fully_qualified_named_transform( + constructor: str, + args: Iterable[Any] = (), + kwargs: Mapping[str, Any] = {}): with FullyQualifiedNamedTransform.with_filter('*'): return constructor >> FullyQualifiedNamedTransform( constructor, args, kwargs) @@ -359,38 +581,19 @@ def _parse_window_spec(spec): # TODO: Triggering, etc. return beam.WindowInto(window_fn) - ios = { - key: getattr(apache_beam.io, key) - for key in dir(apache_beam.io) - if key.startswith('ReadFrom') or key.startswith('WriteTo') - } - - return InlineProvider( - dict({ - 'Create': lambda elements, - reshuffle=True: beam.Create(elements, reshuffle), - 'PyMap': lambda fn: beam.Map( - python_callable.PythonCallableWithSource(fn)), - 'PyMapTuple': lambda fn: beam.MapTuple( - python_callable.PythonCallableWithSource(fn)), - 'PyFlatMap': lambda fn: beam.FlatMap( - python_callable.PythonCallableWithSource(fn)), - 'PyFlatMapTuple': lambda fn: beam.FlatMapTuple( - python_callable.PythonCallableWithSource(fn)), - 'PyFilter': lambda keep: beam.Filter( - python_callable.PythonCallableWithSource(keep)), - 'PyTransform': fully_qualified_named_transform, - 'PyToRow': lambda fields: beam.Select( - **{ - name: python_callable.PythonCallableWithSource(fn) - for (name, fn) in fields.items() - }), - 'WithSchema': with_schema, - 'Flatten': Flatten, - 'WindowInto': WindowInto, - 'GroupByKey': beam.GroupByKey, - }, - **ios)) + def log_and_return(x): + logging.info(x) + return x + + return InlineProvider({ + 'Create': create, + 'LogForTesting': lambda: beam.Map(log_and_return), + 'PyTransform': fully_qualified_named_transform, + 'WithSchemaExperimental': with_schema, + 'Flatten': Flatten, + 'WindowInto': WindowInto, + }, + no_input_transforms=('Create', )) class PypiExpansionService: @@ -403,23 +606,60 @@ def __init__(self, packages, base_python=sys.executable): self._packages = packages self._base_python = base_python - def _key(self): - return json.dumps({'binary': self._base_python, 'packages': self._packages}) + @classmethod + def _key(cls, base_python, packages): + return json.dumps({ + 'binary': base_python, 'packages': sorted(packages) + }, + sort_keys=True) + + @classmethod + def _path(cls, base_python, packages): + return os.path.join( + cls.VENV_CACHE, + hashlib.sha256(cls._key(base_python, + packages).encode('utf-8')).hexdigest()) + + @classmethod + def _create_venv_from_scratch(cls, base_python, packages): + venv = cls._path(base_python, packages) + if not os.path.exists(venv): + subprocess.run([base_python, '-m', 'venv', venv], check=True) + venv_python = os.path.join(venv, 'bin', 'python') + subprocess.run([venv_python, '-m', 'ensurepip'], check=True) + subprocess.run([venv_python, '-m', 'pip', 'install'] + packages, + check=True) + with open(venv + '-requirements.txt', 'w') as fout: + fout.write('\n'.join(packages)) + return venv - def _venv(self): - venv = os.path.join( - self.VENV_CACHE, - hashlib.sha256(self._key().encode('utf-8')).hexdigest()) + @classmethod + def _create_venv_from_clone(cls, base_python, packages): + venv = cls._path(base_python, packages) if not os.path.exists(venv): - python_binary = os.path.join(venv, 'bin', 'python') - subprocess.run([self._base_python, '-m', 'venv', venv], check=True) - subprocess.run([python_binary, '-m', 'ensurepip'], check=True) - subprocess.run([python_binary, '-m', 'pip', 'install'] + self._packages, + clonable_venv = cls._create_venv_to_clone(base_python) + clonable_python = os.path.join(clonable_venv, 'bin', 'python') + subprocess.run( + [clonable_python, '-m', 'clonevirtualenv', clonable_venv, venv], + check=True) + venv_binary = os.path.join(venv, 'bin', 'python') + subprocess.run([venv_binary, '-m', 'pip', 'install'] + packages, check=True) with open(venv + '-requirements.txt', 'w') as fout: - fout.write('\n'.join(self._packages)) + fout.write('\n'.join(packages)) return venv + @classmethod + def _create_venv_to_clone(cls, base_python): + return cls._create_venv_from_scratch( + base_python, [ + 'apache_beam[dataframe,gcp,test]==' + beam_version, + 'virtualenv-clone' + ]) + + def _venv(self): + return self._create_venv_from_clone(self._base_python, self._packages) + def __enter__(self): venv = self._venv() self._service_provider = subprocess_server.SubprocessServer( @@ -442,6 +682,70 @@ def __exit__(self, *args): self._service = None +@ExternalProvider.register_provider_type('renaming') +class RenamingProvider(Provider): + def __init__(self, transforms, mappings, underlying_provider, defaults=None): + if isinstance(underlying_provider, dict): + underlying_provider = ExternalProvider.provider_from_spec( + underlying_provider) + self._transforms = transforms + self._underlying_provider = underlying_provider + for transform in transforms.keys(): + if transform not in mappings: + raise ValueError(f'Missing transform {transform} in mappings.') + self._mappings = mappings + self._defaults = defaults or {} + + def available(self) -> bool: + return self._underlying_provider.available() + + def provided_transforms(self) -> Iterable[str]: + return self._transforms.keys() + + def config_schema(self, type): + underlying_schema = self._underlying_provider.config_schema( + self._transforms[type]) + if underlying_schema is None: + return None + underlying_schema_types = {f.name: f.type for f in underlying_schema.fields} + return schema_pb2.Schema( + fields=[ + schema_pb2.Field(name=src, type=underlying_schema_types[dest]) + for src, + dest in self._mappings[type].items() + ]) + + def requires_inputs(self, typ, args): + return self._underlying_provider.requires_inputs(typ, args) + + def create_transform( + self, + typ: str, + args: Mapping[str, Any], + yaml_create_transform: Callable[ + [Mapping[str, Any], Iterable[beam.PCollection]], beam.PTransform] + ) -> beam.PTransform: + """Creates a PTransform instance for the given transform type and arguments. + """ + mappings = self._mappings[typ] + remapped_args = { + mappings.get(key, key): value + for key, value in args.items() + } + for key, value in self._defaults.get(typ, {}).items(): + if key not in remapped_args: + remapped_args[key] = value + return self._underlying_provider.create_transform( + self._transforms[typ], remapped_args, yaml_create_transform) + + def _affinity(self, other): + raise NotImplementedError( + 'Should not be calling _affinity directly on this provider.') + + def underlying_provider(self): + return self._underlying_provider.underlying_provider() + + def parse_providers(provider_specs): providers = collections.defaultdict(list) for provider_spec in provider_specs: @@ -462,17 +766,55 @@ def merge_providers(*provider_sets): transform_type: [provider] for transform_type in provider.provided_transforms() } + elif isinstance(provider_set, list): + provider_set = merge_providers(*provider_set) for transform_type, providers in provider_set.items(): result[transform_type].extend(providers) return result def standard_providers(): - from apache_beam.yaml.yaml_mapping import create_mapping_provider + from apache_beam.yaml.yaml_mapping import create_mapping_providers + from apache_beam.yaml.yaml_io import io_providers with open(os.path.join(os.path.dirname(__file__), 'standard_providers.yaml')) as fin: standard_providers = yaml.load(fin, Loader=SafeLoader) + return merge_providers( create_builtin_provider(), - create_mapping_provider(), + create_mapping_providers(), + io_providers(), parse_providers(standard_providers)) + + +def list_providers(): + def pretty_type(field_type): + if field_type.WhichOneof('type_info') == 'row_type': + return pretty_schema(field_type.row_type.schema) + else: + t = typing_from_runner_api(field_type) + optional_base = native_type_compatibility.extract_optional_type(t) + if optional_base: + t = optional_base + suffix = '?' + else: + suffix = '' + s = str(t) + if s.startswith(' List[yaml_provider.Provider]: + """Given a set of possible providers, and a set of providers for each + adjacent transform, returns the top possible providers as ranked by + affinity to the adjacent transforms' providers. + """ + providers_by_score = collections.defaultdict(list) + for p in possible_providers: + # The sum of the affinity of the best provider + # for each adjacent transform. + providers_by_score[sum( + max(p.affinity(ap) for ap in apo) + for apo in adjacent_provider_options)].append(p) + return providers_by_score[max(providers_by_score.keys())] + + # If there are any inputs, prefer to match them. + if input_providers: + possible_providers = best_matches( + possible_providers, [[p] for p in input_providers]) + + # Without __uuid__ we can't find downstream operations. + if '__uuid__' not in spec: + return possible_providers[0] + + # Match against downstream transforms, continuing until there is no tie + # or we run out of downstream transforms. + if len(possible_providers) > 1: + adjacent_transforms = list(self.followers(spec['__uuid__'])) + while adjacent_transforms: + # This is a list of all possible providers for each adjacent transform. + adjacent_provider_options = [[ + p for p in self.providers[self._transforms_by_uuid[t]['type']] + if p.available() + ] for t in adjacent_transforms] + if any(not apo for apo in adjacent_provider_options): + # One of the transforms had no available providers. + # We will throw an error later, doesn't matter what we return. + break + # Filter down the set of possible providers to the best ones. + possible_providers = best_matches( + possible_providers, adjacent_provider_options) + # If we are down to one option, no need to go further. + if len(possible_providers) == 1: + break + # Go downstream one more step. + adjacent_transforms = sum( + [list(self.followers(t)) for t in adjacent_transforms], []) + + return possible_providers[0] + # A method on scope as providers may be scoped... - def create_ptransform(self, spec): + def create_ptransform(self, spec, input_pcolls): if 'type' not in spec: raise ValueError(f'Missing transform type: {identify_object(spec)}') @@ -212,39 +332,58 @@ def create_ptransform(self, spec): 'Unknown transform type %r at %s' % (spec['type'], identify_object(spec))) - for provider in self.providers.get(spec['type']): - if provider.available(): - break - else: + # TODO(yaml): Perhaps we can do better than a greedy choice here. + # TODO(yaml): Figure out why this is needed. + providers_by_input = {k: v for k, v in self.input_providers.items()} + input_providers = [ + providers_by_input[pcoll] for pcoll in input_pcolls + if pcoll in providers_by_input + ] + provider = self.best_provider(spec, input_providers) + + config = SafeLineLoader.strip_metadata(spec.get('config', {})) + if not isinstance(config, dict): raise ValueError( - 'No available provider for type %r at %s' % - (spec['type'], identify_object(spec))) + 'Config for transform at %s must be a mapping.' % + identify_object(spec)) + + if (not input_pcolls and not is_explicitly_empty(spec.get('input', {})) and + provider.requires_inputs(spec['type'], config)): + raise ValueError( + f'Missing inputs for transform at {identify_object(spec)}') - if 'args' in spec: - args = spec['args'] - if not isinstance(args, dict): - raise ValueError( - 'Arguments for transform at %s must be a mapping.' % - identify_object(spec)) - else: - args = { - key: value - for (key, value) in spec.items() - if key not in ('type', 'name', 'input', 'output') - } - real_args = SafeLineLoader.strip_metadata(args) try: # pylint: disable=undefined-loop-variable ptransform = provider.create_transform( - spec['type'], real_args, self.create_ptransform) + spec['type'], config, self.create_ptransform) # TODO(robertwb): Should we have a better API for adding annotations # than this? annotations = dict( yaml_type=spec['type'], - yaml_args=json.dumps(real_args), + yaml_args=json.dumps(config), yaml_provider=json.dumps(provider.to_json()), **ptransform.annotations()) ptransform.annotations = lambda: annotations + original_expand = ptransform.expand + + def recording_expand(pvalue): + result = original_expand(pvalue) + + def record_providers(pvalueish): + if isinstance(pvalueish, (tuple, list)): + for p in pvalueish: + record_providers(p) + elif isinstance(pvalueish, dict): + for p in pvalueish.values(): + record_providers(p) + elif isinstance(pvalueish, beam.PCollection): + if pvalueish not in self.input_providers: + self.input_providers[pvalueish] = provider + + record_providers(result) + return result + + ptransform.expand = recording_expand return ptransform except Exception as exn: if isinstance(exn, TypeError): @@ -262,8 +401,11 @@ def unique_name(self, spec, ptransform, strictness=0): if 'name' in spec: name = spec['name'] strictness += 1 - else: + elif 'ExternalTransform' not in ptransform.label: + # The label may have interesting information. name = ptransform.label + else: + name = spec['type'] if name in self._seen_names: if strictness >= 2: raise ValueError(f'Duplicate name at {identify_object(spec)}: {name}') @@ -288,7 +430,7 @@ def expand_leaf_transform(spec, scope): spec = normalize_inputs_outputs(spec) inputs_dict = { key: scope.get_pcollection(value) - for (key, value) in spec['input'].items() + for (key, value) in empty_if_explicitly_empty(spec['input']).items() } input_type = spec.get('input_type', 'default') if input_type == 'list': @@ -303,7 +445,7 @@ def expand_leaf_transform(spec, scope): else: inputs = inputs_dict _LOGGER.info("Expanding %s ", identify_object(spec)) - ptransform = scope.create_ptransform(spec) + ptransform = scope.create_ptransform(spec, inputs_dict.values()) try: # TODO: Move validation to construction? with FullyQualifiedNamedTransform.with_filter('*'): @@ -315,9 +457,11 @@ def expand_leaf_transform(spec, scope): # TODO: Handle (or at least reject) nested case. return outputs elif isinstance(outputs, (tuple, list)): - return {'out{ix}': pcoll for (ix, pcoll) in enumerate(outputs)} + return {f'out{ix}': pcoll for (ix, pcoll) in enumerate(outputs)} elif isinstance(outputs, beam.PCollection): return {'out': outputs} + elif outputs is None: + return {} else: raise ValueError( f'Transform {identify_object(spec)} returned an unexpected type ' @@ -328,15 +472,16 @@ def expand_composite_transform(spec, scope): spec = normalize_inputs_outputs(normalize_source_sink(spec)) inner_scope = Scope( - scope.root, { + scope.root, + { key: scope.get_pcollection(value) - for key, - value in spec['input'].items() + for (key, value) in empty_if_explicitly_empty(spec['input']).items() }, spec['transforms'], yaml_provider.merge_providers( yaml_provider.parse_providers(spec.get('providers', [])), - scope.providers)) + scope.providers), + scope.input_providers) class CompositePTransform(beam.PTransform): @staticmethod @@ -355,8 +500,7 @@ def expand(inputs): _LOGGER.info("Expanding %s ", identify_object(spec)) return ({ key: scope.get_pcollection(value) - for key, - value in spec['input'].items() + for (key, value) in empty_if_explicitly_empty(spec['input']).items() } or scope.root) | scope.unique_name(spec, None) >> CompositePTransform() @@ -365,6 +509,12 @@ def expand_chain_transform(spec, scope): def chain_as_composite(spec): + def is_not_output_of_last_transform(new_transforms, value): + return ( + ('name' in new_transforms[-1] and + value != new_transforms[-1]['name']) or + ('type' in new_transforms[-1] and value != new_transforms[-1]['type'])) + # A chain is simply a composite transform where all inputs and outputs # are implicit. spec = normalize_source_sink(spec) @@ -375,19 +525,39 @@ def chain_as_composite(spec): composite_spec = normalize_inputs_outputs(spec) new_transforms = [] for ix, transform in enumerate(composite_spec['transforms']): - if any(io in transform for io in ('input', 'output', 'input', 'output')): - raise ValueError( - f'Transform {identify_object(transform)} is part of a chain, ' - 'must have implicit inputs and outputs.') + if any(io in transform for io in ('input', 'output')): + if (ix == 0 and 'input' in transform and 'output' not in transform and + is_explicitly_empty(transform['input'])): + # This is OK as source clause sets an explicitly empty input. + pass + else: + raise ValueError( + f'Transform {identify_object(transform)} is part of a chain, ' + 'must have implicit inputs and outputs.') if ix == 0: - transform['input'] = {key: key for key in composite_spec['input'].keys()} + if is_explicitly_empty(transform.get('input', None)): + pass + elif is_explicitly_empty(composite_spec['input']): + transform['input'] = composite_spec['input'] + else: + transform['input'] = { + key: key + for key in composite_spec['input'].keys() + } else: transform['input'] = new_transforms[-1]['__uuid__'] new_transforms.append(transform) + new_transforms.extend(spec.get('extra_transforms', [])) composite_spec['transforms'] = new_transforms last_transform = new_transforms[-1]['__uuid__'] if has_explicit_outputs: + for (key, value) in composite_spec['output'].items(): + if is_not_output_of_last_transform(new_transforms, value): + raise ValueError( + f"Explicit output {identify_object(value)} of the chain transform" + f" is not an output of the last transform.") + composite_spec['output'] = { key: f'{last_transform}.{value}' for (key, value) in composite_spec['output'].items() @@ -426,6 +596,8 @@ def normalize_source_sink(spec): spec = dict(spec) spec['transforms'] = list(spec.get('transforms', [])) if 'source' in spec: + if 'input' not in spec['source']: + spec['source']['input'] = explicitly_empty() spec['transforms'].insert(0, spec.pop('source')) if 'sink' in spec: spec['transforms'].append(spec.pop('sink')) @@ -439,6 +611,13 @@ def preprocess_source_sink(spec): return spec +def tag_explicit_inputs(spec): + if 'input' in spec and not SafeLineLoader.strip_metadata(spec['input']): + return dict(spec, input=explicitly_empty()) + else: + return spec + + def normalize_inputs_outputs(spec): spec = dict(spec) @@ -462,14 +641,19 @@ def identify_object(spec): def extract_name(spec): - if 'name' in spec: - return spec['name'] - elif 'id' in spec: - return spec['id'] - elif 'type' in spec: - return spec['type'] - elif len(spec) == 1: - return extract_name(next(iter(spec.values()))) + if isinstance(spec, dict): + if 'name' in spec: + return spec['name'] + elif 'id' in spec: + return spec['id'] + elif 'type' in spec: + return spec['type'] + elif len(spec) == 1: + return extract_name(next(iter(spec.values()))) + else: + return '' + elif isinstance(spec, str): + return spec else: return '' @@ -478,7 +662,7 @@ def push_windowing_to_roots(spec): scope = LightweightScope(spec['transforms']) consumed_outputs_by_transform = collections.defaultdict(set) for transform in spec['transforms']: - for _, input_ref in transform['input'].items(): + for _, input_ref in empty_if_explicitly_empty(transform['input']).items(): try: transform_id, output = scope.get_transform_id_and_output_name(input_ref) consumed_outputs_by_transform[transform_id].add(output) @@ -487,7 +671,7 @@ def push_windowing_to_roots(spec): pass for transform in spec['transforms']: - if not transform['input'] and 'windowing' not in transform: + if is_empty(transform['input']) and 'windowing' not in transform: transform['windowing'] = spec['windowing'] transform['__consumed_outputs'] = consumed_outputs_by_transform[ transform['__uuid__']] @@ -498,6 +682,9 @@ def push_windowing_to_roots(spec): def preprocess_windowing(spec): if spec['type'] == 'WindowInto': # This is the transform where it is actually applied. + if 'windowing' in spec: + spec['config'] = spec.get('config', {}) + spec['config']['windowing'] = spec.pop('windowing') return spec elif 'windowing' not in spec: # Nothing to do. @@ -511,8 +698,8 @@ def preprocess_windowing(spec): spec = push_windowing_to_roots(spec) windowing = spec.pop('windowing') - if spec['input']: - # Apply the windowing to all inputs by wrapping it in a trasnform that + if not is_empty(spec['input']): + # Apply the windowing to all inputs by wrapping it in a transform that # first applies windowing and then applies the original transform. original_inputs = spec['input'] windowing_transforms = [{ @@ -551,7 +738,9 @@ def preprocess_windowing(spec): 'type': 'WindowInto', 'name': f'WindowInto[{out}]', 'windowing': windowing, - 'input': modified_spec['__uuid__'] + ('.' + out if out else ''), + 'input': { + 'input': modified_spec['__uuid__'] + ('.' + out if out else '') + }, '__line__': spec['__line__'], '__uuid__': SafeLineLoader.create_uuid(), } for out in consumed_outputs] @@ -620,22 +809,108 @@ def all_inputs(t): return dict(spec, transforms=new_transforms) -def preprocess(spec, verbose=False): +def ensure_transforms_have_types(spec): + if 'type' not in spec: + raise ValueError(f'Missing type specification in {identify_object(spec)}') + return spec + + +def ensure_errors_consumed(spec): + if spec['type'] == 'composite': + scope = LightweightScope(spec['transforms']) + to_handle = {} + consumed = set( + scope.get_transform_id_and_output_name(output) + for output in spec['output'].values()) + for t in spec['transforms']: + config = t.get('config', t) + if 'error_handling' in config: + if 'output' not in config['error_handling']: + raise ValueError( + f'Missing output in error_handling of {identify_object(t)}') + to_handle[t['__uuid__'], config['error_handling']['output']] = t + for _, input in empty_if_explicitly_empty(t['input']).items(): + if input not in spec['input']: + consumed.add(scope.get_transform_id_and_output_name(input)) + for error_pcoll, t in to_handle.items(): + if error_pcoll not in consumed: + raise ValueError(f'Unconsumed error output for {identify_object(t)}.') + return spec + + +def lift_config(spec): + if 'config' not in spec: + common_params = 'name', 'type', 'input', 'output', 'transforms' + return { + 'config': {k: v + for (k, v) in spec.items() if k not in common_params}, + **{ + k: v + for (k, v) in spec.items() # + if k in common_params or k in ('__line__', '__uuid__') + } + } + else: + return spec + + +def ensure_config(spec): + if 'config' not in spec: + spec['config'] = {} + return spec + + +def preprocess(spec, verbose=False, known_transforms=None): if verbose: pprint.pprint(spec) def apply(phase, spec): spec = phase(spec) - if spec['type'] in {'composite', 'chain'}: + if spec['type'] in {'composite', 'chain'} and 'transforms' in spec: spec = dict( spec, transforms=[apply(phase, t) for t in spec['transforms']]) return spec - for phase in [preprocess_source_sink, - preprocess_chain, - normalize_inputs_outputs, - preprocess_flattened_inputs, - preprocess_windowing]: + if known_transforms: + known_transforms = set(known_transforms).union(['chain', 'composite']) + + def ensure_transforms_have_providers(spec): + if known_transforms: + if spec['type'] not in known_transforms: + raise ValueError( + 'Unknown type or missing provider ' + f'for type {spec["type"]} for {identify_object(spec)}') + return spec + + def preprocess_langauges(spec): + if spec['type'] in ('Filter', 'MapToFields'): + language = spec.get('config', {}).get('language', 'generic') + new_type = spec['type'] + '-' + language + if known_transforms and new_type not in known_transforms: + if language == 'generic': + raise ValueError(f'Missing language for {identify_object(spec)}') + else: + raise ValueError( + f'Unknown language {language} for {identify_object(spec)}') + return dict(spec, type=new_type, name=spec.get('name', spec['type'])) + else: + return spec + + for phase in [ + ensure_transforms_have_types, + preprocess_langauges, + ensure_transforms_have_providers, + preprocess_source_sink, + preprocess_chain, + tag_explicit_inputs, + normalize_inputs_outputs, + preprocess_flattened_inputs, + ensure_errors_consumed, + preprocess_windowing, + # TODO(robertwb): Consider enabling this by default, or as an option. + # lift_config, + ensure_config, + ]: spec = apply(phase, spec) if verbose: print('=' * 20, phase, '=' * 20) @@ -647,14 +922,15 @@ class YamlTransform(beam.PTransform): def __init__(self, spec, providers={}): # pylint: disable=dangerous-default-value if isinstance(spec, str): spec = yaml.load(spec, Loader=SafeLineLoader) + if isinstance(providers, dict): + providers = { + key: yaml_provider.as_provider_list(key, value) + for (key, value) in providers.items() + } # TODO(BEAM-26941): Validate as a transform. - self._spec = preprocess(spec) self._providers = yaml_provider.merge_providers( - { - key: yaml_provider.as_provider_list(key, value) - for (key, value) in providers.items() - }, - yaml_provider.standard_providers()) + providers, yaml_provider.standard_providers()) + self._spec = preprocess(spec, known_transforms=self._providers.keys()) def expand(self, pcolls): if isinstance(pcolls, beam.pvalue.PBegin): @@ -667,7 +943,12 @@ def expand(self, pcolls): root = next(iter(pcolls.values())).pipeline result = expand_transform( self._spec, - Scope(root, pcolls, transforms=[], providers=self._providers)) + Scope( + root, + pcolls, + transforms=[], + providers=self._providers, + input_providers={})) if len(result) == 1: return only_element(result.values()) else: diff --git a/sdks/python/apache_beam/yaml/yaml_transform_scope_test.py b/sdks/python/apache_beam/yaml/yaml_transform_scope_test.py new file mode 100644 index 0000000000000..f00403b07e2ab --- /dev/null +++ b/sdks/python/apache_beam/yaml/yaml_transform_scope_test.py @@ -0,0 +1,322 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import collections +import logging +import unittest + +import yaml + +import apache_beam as beam +from apache_beam.yaml import yaml_provider +from apache_beam.yaml import yaml_transform +from apache_beam.yaml.yaml_transform import LightweightScope +from apache_beam.yaml.yaml_transform import SafeLineLoader +from apache_beam.yaml.yaml_transform import Scope + + +class ScopeTest(unittest.TestCase): + def get_scope_by_spec(self, p, spec): + spec = yaml.load(spec, Loader=SafeLineLoader) + + scope = Scope( + beam.pvalue.PBegin(p), {}, + spec['transforms'], + yaml_provider.standard_providers(), {}) + return scope, spec + + def test_get_pcollection_input(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + elements = p | beam.Create(range(3)) + scope = Scope( + p, {'input': elements}, + transforms=[], + providers=yaml_provider.standard_providers(), + input_providers={}) + + result = scope.get_pcollection('input') + self.assertEqual("PCollection[Create/Map(decode).None]", str(result)) + + def test_get_pcollection_output(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + spec = ''' + transforms: + - type: Create + config: + elements: [0, 1, 3, 4] + - type: LogForTesting + name: Square + input: Create + ''' + + scope, spec = self.get_scope_by_spec(p, spec) + + self.assertEqual( + "PCollection[Create/Map(decode).None]", + str(scope.get_pcollection("Create"))) + + self.assertEqual( + "PCollection[Square.None]", str(scope.get_pcollection("Square"))) + + self.assertEqual( + "PCollection[Square.None]", str(scope.get_pcollection("LogForTesting"))) + + self.assertTrue( + scope.get_pcollection("Square") == scope.get_pcollection( + "LogForTesting")) + + def test_create_ptransform(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + spec = ''' + transforms: + - type: Create + config: + elements: [1, 2, 3] + ''' + scope, spec = self.get_scope_by_spec(p, spec) + + result = scope.create_ptransform(spec['transforms'][0], []) + self.assertIsInstance(result, beam.transforms.Create) + self.assertEqual(result.label, 'Create') + + result_annotations = {**result.annotations()} + target_annotations = { + 'yaml_type': 'Create', + 'yaml_args': '{"elements": [1, 2, 3]}', + 'yaml_provider': '{"type": "InlineProvider"}' + } + + # Check if target_annotations is a subset of result_annotations + self.assertDictEqual( + result_annotations, { + **result_annotations, **target_annotations + }) + + +class TestProvider(yaml_provider.InlineProvider): + def __init__(self, transform, name): + super().__init__({ + name: lambda: beam.Map(lambda x: (x or ()) + (name, )), # or None + transform: lambda: beam.Map(lambda x: (x or ()) + (name, )), + }) + self._transform = transform + self._name = name + + def __repr__(self): + return 'TestProvider(%r, %r)' % (self._transform, self._name) + + def _affinity(self, other): + if isinstance(other, TestProvider): + # Providers are closer based on how much their names match prefixes. + affinity = 1 + for x, y in zip(self._name, other._name): + if x != y: + break + affinity *= 10 + return affinity + else: + return -1000 + + +class ProviderAffinityTest(unittest.TestCase): + @staticmethod + def create_scope(s, providers): + providers_dict = collections.defaultdict(list) + for provider in providers: + for transform_type in provider.provided_transforms(): + providers_dict[transform_type].append(provider) + spec = yaml_transform.preprocess(yaml.load(s, Loader=SafeLineLoader)) + return Scope( + None, {}, + transforms=spec['transforms'], + providers=providers_dict, + input_providers={}) + + def test_best_provider_based_on_input(self): + provider_Ax = TestProvider('A', 'xxx') + provider_Ay = TestProvider('A', 'yyy') + provider_Bx = TestProvider('B', 'xxz') + provider_By = TestProvider('B', 'yyz') + scope = self.create_scope( + ''' + type: chain + transforms: + - type: A + - type: B + ''', [provider_Ax, provider_Ay, provider_Bx, provider_By]) + self.assertEqual(scope.best_provider('B', [provider_Ax]), provider_Bx) + self.assertEqual(scope.best_provider('B', [provider_Ay]), provider_By) + + def test_best_provider_based_on_followers(self): + close_provider = TestProvider('A', 'xxy') + far_provider = TestProvider('A', 'yyy') + following_provider = TestProvider('B', 'xxx') + scope = self.create_scope( + ''' + type: chain + transforms: + - type: A + - type: B + ''', [far_provider, close_provider, following_provider]) + self.assertEqual(scope.best_provider('A', []), close_provider) + + def test_best_provider_based_on_multiple_followers(self): + close_provider = TestProvider('A', 'xxy') + provider_B = TestProvider('B', 'xxx') + # These are not quite as close as the two above. + far_provider = TestProvider('A', 'yyy') + provider_C = TestProvider('C', 'yzz') + scope = self.create_scope( + ''' + type: composite + transforms: + - type: A + - type: B + input: A + - type: C + input: A + ''', [far_provider, close_provider, provider_B, provider_C]) + self.assertEqual(scope.best_provider('A', []), close_provider) + + def test_best_provider_based_on_distant_follower(self): + providers = [ + # xxx and yyy vend both + TestProvider('A', 'xxx'), + TestProvider('A', 'yyy'), + TestProvider('B', 'xxx'), + TestProvider('B', 'yyy'), + TestProvider('C', 'xxx'), + TestProvider('C', 'yyy'), + # D and E are only provided by a single provider each. + TestProvider('D', 'xxx'), + TestProvider('E', 'yyy') + ] + + # If D is the eventual destination, pick the xxx one. + scope = self.create_scope( + ''' + type: chain + transforms: + - type: A + - type: B + - type: C + - type: D + ''', + providers) + self.assertEqual(scope.best_provider('A', []), providers[0]) + + # If instead E is the eventual destination, pick the yyy one. + scope = self.create_scope( + ''' + type: chain + transforms: + - type: A + - type: B + - type: C + - type: E + ''', + providers) + self.assertEqual(scope.best_provider('A', []), providers[1]) + + # If we have D then E, stay with xxx as long as possible to only switch once + scope = self.create_scope( + ''' + type: chain + transforms: + - type: A + - type: B + - type: C + - type: D + - type: E + ''', + providers) + self.assertEqual(scope.best_provider('A', []), providers[0]) + + +class LightweightScopeTest(unittest.TestCase): + @staticmethod + def get_spec(): + pipeline_yaml = ''' + - type: PyMap + name: Square + input: elements + fn: "lambda x: x * x" + - type: PyMap + name: PyMap + input: elements + fn: "lambda x: x * x * x" + - type: Filter + name: FilterOutBigNumbers + input: PyMap + keep: "lambda x: x<100" + ''' + return yaml.load(pipeline_yaml, Loader=SafeLineLoader) + + def test_init(self): + spec = self.get_spec() + scope = LightweightScope(spec) + self.assertEqual(len(scope._transforms_by_uuid), 3) + self.assertCountEqual( + list(scope._uuid_by_name.keys()), + ["PyMap", "Square", "Filter", "FilterOutBigNumbers"]) + + def test_get_transform_id_and_output_name(self): + spec = self.get_spec() + scope = LightweightScope(spec) + transform_id, output = scope.get_transform_id_and_output_name("Square") + self.assertEqual(transform_id, spec[0]['__uuid__']) + self.assertEqual(output, None) + + def test_get_transform_id_and_output_name_with_dot(self): + spec = self.get_spec() + scope = LightweightScope(spec) + transform_id, output = \ + scope.get_transform_id_and_output_name("Square.OutputName") + self.assertEqual(transform_id, spec[0]['__uuid__']) + self.assertEqual(output, "OutputName") + + def test_get_transform_id_by_uuid(self): + spec = self.get_spec() + scope = LightweightScope(spec) + transform_id = scope.get_transform_id(spec[0]['__uuid__']) + self.assertEqual(spec[0]['__uuid__'], transform_id) + + def test_get_transform_id_by_unique_name(self): + spec = self.get_spec() + scope = LightweightScope(spec) + transform_id = scope.get_transform_id("Square") + self.assertEqual(transform_id, spec[0]['__uuid__']) + + def test_get_transform_id_by_ambiguous_name(self): + spec = self.get_spec() + scope = LightweightScope(spec) + with self.assertRaisesRegex(ValueError, r'Ambiguous.*PyMap'): + scope.get_transform_id(scope.get_transform_id(spec[1]['name'])) + + def test_get_transform_id_by_unknown_name(self): + spec = self.get_spec() + scope = LightweightScope(spec) + with self.assertRaisesRegex(ValueError, r'Unknown.*NotExistingTransform'): + scope.get_transform_id("NotExistingTransform") + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + unittest.main() diff --git a/sdks/python/apache_beam/yaml/yaml_transform_test.py b/sdks/python/apache_beam/yaml/yaml_transform_test.py index 167e3261d4166..63f2e0e7facd0 100644 --- a/sdks/python/apache_beam/yaml/yaml_transform_test.py +++ b/sdks/python/apache_beam/yaml/yaml_transform_test.py @@ -15,6 +15,7 @@ # limitations under the License. # +import collections import glob import logging import os @@ -24,10 +25,55 @@ import apache_beam as beam from apache_beam.testing.util import assert_that from apache_beam.testing.util import equal_to +from apache_beam.utils import python_callable +from apache_beam.yaml import yaml_provider from apache_beam.yaml.yaml_transform import YamlTransform -class YamlTransformTest(unittest.TestCase): +class CreateTimestamped(beam.PTransform): + _yaml_requires_inputs = False + + def __init__(self, elements): + self._elements = elements + + def expand(self, p): + return ( + p + | beam.Create(self._elements) + | beam.Map(lambda x: beam.transforms.window.TimestampedValue(x, x))) + + +class SumGlobally(beam.PTransform): + def expand(self, pcoll): + return pcoll | beam.CombineGlobally(sum).without_defaults() + + +class SizeLimiter(beam.PTransform): + def __init__(self, limit, error_handling): + self._limit = limit + self._error_handling = error_handling + + def expand(self, pcoll): + def raise_on_big(element): + if len(element) > self._limit: + raise ValueError(element) + else: + return element + + good, bad = pcoll | beam.Map(raise_on_big).with_exception_handling() + return {'small_elements': good, self._error_handling['output']: bad} + + +TEST_PROVIDERS = { + 'CreateInts': lambda elements: beam.Create(elements), + 'CreateTimestamped': CreateTimestamped, + 'SumGlobally': SumGlobally, + 'SizeLimiter': SizeLimiter, + 'PyMap': lambda fn: beam.Map(python_callable.PythonCallableWithSource(fn)), +} + + +class YamlTransformE2ETest(unittest.TestCase): def test_composite(self): with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( pickle_library='cloudpickle')) as p: @@ -42,16 +88,19 @@ def test_composite(self): - type: PyMap name: Square input: elements - fn: "lambda x: x * x" + config: + fn: "lambda x: x * x" - type: PyMap name: Cube input: elements - fn: "lambda x: x * x * x" + config: + fn: "lambda x: x * x * x" - type: Flatten input: [Square, Cube] output: Flatten - ''') + ''', + providers=TEST_PROVIDERS) assert_that(result, equal_to([1, 4, 9, 1, 8, 27])) def test_chain_with_input(self): @@ -65,10 +114,13 @@ def test_chain_with_input(self): elements: input transforms: - type: PyMap - fn: "lambda x: x * x + x" + config: + fn: "lambda x: x * x + x" - type: PyMap - fn: "lambda x: x + 41" - ''') + config: + fn: "lambda x: x + 41" + ''', + providers=TEST_PROVIDERS) assert_that(result, equal_to([41, 43, 47, 53, 61, 71, 83, 97, 113, 131])) def test_chain_with_source_sink(self): @@ -78,15 +130,19 @@ def test_chain_with_source_sink(self): ''' type: chain source: - type: Create - elements: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + type: CreateInts + config: + elements: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] transforms: - type: PyMap - fn: "lambda x: x * x + x" + config: + fn: "lambda x: x * x + x" sink: type: PyMap - fn: "lambda x: x + 41" - ''') + config: + fn: "lambda x: x + 41" + ''', + providers=TEST_PROVIDERS) assert_that(result, equal_to([41, 43, 47, 53, 61, 71, 83, 97, 113, 131])) def test_chain_with_root(self): @@ -96,15 +152,37 @@ def test_chain_with_root(self): ''' type: chain transforms: - - type: Create - elements: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + - type: CreateInts + config: + elements: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - type: PyMap - fn: "lambda x: x * x + x" + config: + fn: "lambda x: x * x + x" - type: PyMap - fn: "lambda x: x + 41" - ''') + config: + fn: "lambda x: x + 41" + ''', + providers=TEST_PROVIDERS) assert_that(result, equal_to([41, 43, 47, 53, 61, 71, 83, 97, 113, 131])) + def create_has_schema(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + result = p | YamlTransform( + ''' + type: chain + transforms: + - type: Create + config: + elements: [{a: 1, b: 'x'}, {a: 2, b: 'y'}] + - type: MapToFields + config: + language: python + fields: + repeated: a * b + ''') | beam.Map(lambda x: x.repeated) + assert_that(result, equal_to(['x', 'yy'])) + def test_implicit_flatten(self): with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( pickle_library='cloudpickle')) as p: @@ -114,15 +192,19 @@ def test_implicit_flatten(self): transforms: - type: Create name: CreateSmall - elements: [1, 2, 3] + config: + elements: [1, 2, 3] - type: Create name: CreateBig - elements: [100, 200] + config: + elements: [100, 200] - type: PyMap input: [CreateBig, CreateSmall] - fn: "lambda x: x * x" + config: + fn: "lambda x: x * x" output: PyMap - ''') + ''', + providers=TEST_PROVIDERS) assert_that(result, equal_to([1, 4, 9, 10000, 40000])) def test_csv_to_json(self): @@ -153,9 +235,11 @@ def test_csv_to_json(self): type: chain transforms: - type: ReadFromCsv - path: %s + config: + path: %s - type: WriteToJson - path: %s + config: + path: %s num_shards: 1 ''' % (repr(input), repr(output))) @@ -174,59 +258,224 @@ def test_name_is_not_ambiguous(self): transforms: - type: Create name: Create - elements: [0, 1, 3, 4] - - type: PyFilter - name: Filter - keep: "lambda elem: elem > 2" + config: + elements: [0, 1, 3, 4] + - type: PyMap + name: PyMap + config: + fn: "lambda elem: elem * elem" input: Create - output: Filter - ''') + output: PyMap + ''', + providers=TEST_PROVIDERS) # No exception raised - assert_that(result, equal_to([3, 4])) + assert_that(result, equal_to([0, 1, 9, 16])) def test_name_is_ambiguous(self): with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( pickle_library='cloudpickle')) as p: # pylint: disable=expression-not-assigned - with self.assertRaises(ValueError): + with self.assertRaisesRegex(ValueError, r'Ambiguous.*'): p | YamlTransform( ''' type: composite transforms: - type: Create name: CreateData - elements: [0, 1, 3, 4] - - type: PyFilter - name: PyFilter - keep: "lambda elem: elem > 2" + config: + elements: [0, 1, 3, 4] + - type: PyMap + name: PyMap + config: + fn: "lambda elem: elem + 2" input: CreateData - - type: PyFilter - name: AnotherFilter - keep: "lambda elem: elem > 3" - input: PyFilter - output: AnotherFilter + - type: PyMap + name: AnotherMap + config: + fn: "lambda elem: elem + 3" + input: PyMap + output: AnotherMap + ''', + providers=TEST_PROVIDERS) + + def test_empty_inputs_throws_error(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + with self.assertRaisesRegex(ValueError, + 'Missing inputs for transform at ' + '"EmptyInputOkButYamlDoesntKnow" at line .*'): + _ = p | YamlTransform( + ''' + type: composite + transforms: + - type: PyTransform + name: EmptyInputOkButYamlDoesntKnow + config: + constructor: apache_beam.Impulse ''') + def test_empty_inputs_ok_in_source(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + # Does not throw an error like it does above. + _ = p | YamlTransform( + ''' + type: composite + source: + type: PyTransform + name: EmptyInputOkButYamlDoesntKnow + config: + constructor: apache_beam.Impulse + ''') -class CreateTimestamped(beam.PTransform): - def __init__(self, elements): - self._elements = elements + def test_empty_inputs_ok_if_explicit(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + # Does not throw an error like it does above. + _ = p | YamlTransform( + ''' + type: composite + transforms: + - type: PyTransform + name: EmptyInputOkButYamlDoesntKnow + input: {} + config: + constructor: apache_beam.Impulse + ''') - def expand(self, p): - return ( - p - | beam.Create(self._elements) - | beam.Map(lambda x: beam.transforms.window.TimestampedValue(x, x))) + def test_annotations(self): + t = LinearTransform(5, b=100) + annotations = t.annotations() + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + result = p | YamlTransform( + ''' + type: chain + transforms: + - type: Create + config: + elements: [0, 1, 2, 3] + - type: %r + config: %s + ''' % (annotations['yaml_type'], annotations['yaml_args'])) + assert_that(result, equal_to([100, 105, 110, 115])) -class SumGlobally(beam.PTransform): - def expand(self, pcoll): - return pcoll | beam.CombineGlobally(sum).without_defaults() +class ErrorHandlingTest(unittest.TestCase): + def test_error_handling_outputs(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + result = p | YamlTransform( + ''' + type: composite + transforms: + - type: Create + config: + elements: ['a', 'b', 'biiiiig'] + - type: SizeLimiter + input: Create + config: + limit: 5 + error_handling: + output: errors + - name: TrimErrors + type: PyMap + input: SizeLimiter.errors + config: + fn: "lambda x: x[1][1]" + output: + good: SizeLimiter + bad: TrimErrors + ''', + providers=TEST_PROVIDERS) + assert_that(result['good'], equal_to(['a', 'b']), label="CheckGood") + assert_that(result['bad'], equal_to(["ValueError('biiiiig')"])) + def test_must_handle_error_output(self): + with self.assertRaisesRegex(Exception, 'Unconsumed error output .*line 7'): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + _ = p | YamlTransform( + ''' + type: composite + transforms: + - type: Create + config: + elements: ['a', 'b', 'biiiiig'] + - type: SizeLimiter + input: Create + config: + limit: 5 + error_handling: + output: errors + ''', + providers=TEST_PROVIDERS) -TEST_PROVIDERS = { - 'CreateTimestamped': CreateTimestamped, 'SumGlobally': SumGlobally -} + def test_mapping_errors(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + result = p | YamlTransform( + ''' + type: composite + transforms: + - type: Create + config: + elements: [0, 1, 2, 4] + - type: PyMap + name: ToRow + input: Create + config: + fn: "lambda x: beam.Row(num=x, str='a' * x or 'bbb')" + - type: Filter + input: ToRow + config: + language: python + keep: + str[1] >= 'a' + error_handling: + output: errors + - type: MapToFields + name: MapWithErrorHandling + input: Filter + config: + language: python + fields: + num: num + inverse: float(1 / num) + error_handling: + output: errors + - type: PyMap + name: TrimErrors + input: [MapWithErrorHandling.errors, Filter.errors] + config: + fn: "lambda x: x.msg" + - type: MapToFields + name: Sum + input: MapWithErrorHandling + config: + language: python + append: True + fields: + sum: num + inverse + output: + good: Sum + bad: TrimErrors + ''', + providers=TEST_PROVIDERS) + assert_that( + result['good'], + equal_to([ + beam.Row(num=2, inverse=.5, sum=2.5), + beam.Row(num=4, inverse=.25, sum=4.25) + ]), + label="CheckGood") + assert_that( + result['bad'], + equal_to([ + "IndexError('string index out of range')", # from the filter + "ZeroDivisionError('division by zero')", # from the mapping + ]), + label='CheckErrors') class YamlWindowingTest(unittest.TestCase): @@ -238,7 +487,8 @@ def test_explicit_window_into(self): type: chain transforms: - type: CreateTimestamped - elements: [0, 1, 2, 3, 4, 5] + config: + elements: [0, 1, 2, 3, 4, 5] - type: WindowInto windowing: type: fixed @@ -256,7 +506,8 @@ def test_windowing_on_input(self): type: chain transforms: - type: CreateTimestamped - elements: [0, 1, 2, 3, 4, 5] + config: + elements: [0, 1, 2, 3, 4, 5] - type: SumGlobally windowing: type: fixed @@ -274,10 +525,12 @@ def test_windowing_multiple_inputs(self): transforms: - type: CreateTimestamped name: Create1 - elements: [0, 2, 4] + config: + elements: [0, 2, 4] - type: CreateTimestamped name: Create2 - elements: [1, 3, 5] + config: + elements: [1, 3, 5] - type: SumGlobally input: [Create1, Create2] windowing: @@ -296,7 +549,8 @@ def test_windowing_on_output(self): type: chain transforms: - type: CreateTimestamped - elements: [0, 1, 2, 3, 4, 5] + config: + elements: [0, 1, 2, 3, 4, 5] windowing: type: fixed size: 4 @@ -313,7 +567,8 @@ def test_windowing_on_outer(self): type: chain transforms: - type: CreateTimestamped - elements: [0, 1, 2, 3, 4, 5] + config: + elements: [0, 1, 2, 3, 4, 5] - type: SumGlobally windowing: type: fixed @@ -323,6 +578,149 @@ def test_windowing_on_outer(self): assert_that(result, equal_to([6, 9])) +class AnnotatingProvider(yaml_provider.InlineProvider): + """A provider that vends transforms that do nothing but record that this + provider (as identified by name) was used, along with any prior history + of the given element. + """ + def __init__(self, name, transform_names): + super().__init__({ + transform_name: lambda: beam.Map(lambda x: (x or ()) + (name, )) + for transform_name in transform_names.strip().split() + }) + self._name = name + + def __repr__(self): + return 'AnnotatingProvider(%r)' % self._name + + +class AnotherAnnProvider(AnnotatingProvider): + """A Provider that behaves exactly as AnnotatingProvider, but is not + of the same type and so is considered "more distant" for matching purposes. + """ + pass + + +class ProviderAffinityTest(unittest.TestCase): + """These tests check that for a sequence of transforms, the "closest" + proveders are chosen among multiple possible implementations. + """ + provider1 = AnnotatingProvider("provider1", "P1 A B C ") + provider2 = AnnotatingProvider("provider2", "P2 A C D") + provider3 = AnotherAnnProvider("provider3", "P3 A B ") + provider4 = AnotherAnnProvider("provider4", "P4 A B D") + + providers_dict = collections.defaultdict(list) + for provider in [provider1, provider2, provider3, provider4]: + for transform_type in provider.provided_transforms(): + providers_dict[transform_type].append(provider) + + def test_prefers_same_provider(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + result1 = p | 'Yaml1' >> YamlTransform( + ''' + type: chain + transforms: + - type: Create + config: + elements: [0] + - type: P1 + - type: A + - type: C + ''', + providers=self.providers_dict) + assert_that( + result1, + equal_to([( + # provider1 was chosen, as it is the only one vending P1 + 'provider1', + # All of the providers vend A, but since the input was produced + # by provider1, we prefer to use that again. + 'provider1', + # Similarly for C. + 'provider1')]), + label='StartWith1') + + result2 = p | 'Yaml2' >> YamlTransform( + ''' + type: chain + transforms: + - type: Create + config: + elements: [0] + - type: P2 + - type: A + - type: C + ''', + providers=self.providers_dict) + assert_that( + result2, + equal_to([( + # provider2 was necessarily chosen for P2 + 'provider2', + # Unlike above, we choose provider2 to implement A. + 'provider2', + # Likewise for C. + 'provider2')]), + label='StartWith2') + + def test_prefers_same_provider_class(self): + # Like test_prefers_same_provider, but as we cannot choose the same + # exact provider, we go with the next closest (which is of the same type) + # over an implementation from a Provider of a different type. + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + result1 = p | 'Yaml1' >> YamlTransform( + ''' + type: chain + transforms: + - type: Create + config: + elements: [0] + - type: P1 + - type: A + - type: D + - type: A + ''', + providers=self.providers_dict) + assert_that( + result1, + equal_to([('provider1', 'provider1', 'provider2', 'provider2')]), + label='StartWith1') + + result3 = p | 'Yaml2' >> YamlTransform( + ''' + type: chain + transforms: + - type: Create + config: + elements: [0] + - type: P3 + - type: A + - type: D + - type: A + ''', + providers=self.providers_dict) + assert_that( + result3, + equal_to([('provider3', 'provider3', 'provider4', 'provider4')]), + label='StartWith3') + + +@beam.transforms.ptransform.annotate_yaml +class LinearTransform(beam.PTransform): + """A transform used for testing annotate_yaml.""" + def __init__(self, a, b): + self._a = a + self._b = b + + def expand(self, pcoll): + a = self._a + b = self._b + return pcoll | beam.Map(lambda x: a * x + b) + + if __name__ == '__main__': logging.getLogger().setLevel(logging.INFO) unittest.main() diff --git a/sdks/python/apache_beam/yaml/yaml_transform_unit_test.py b/sdks/python/apache_beam/yaml/yaml_transform_unit_test.py new file mode 100644 index 0000000000000..5d5e5850fd73d --- /dev/null +++ b/sdks/python/apache_beam/yaml/yaml_transform_unit_test.py @@ -0,0 +1,996 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import logging +import unittest + +import yaml + +import apache_beam as beam +from apache_beam.yaml import YamlTransform +from apache_beam.yaml import yaml_provider +from apache_beam.yaml.yaml_provider import InlineProvider +from apache_beam.yaml.yaml_transform import SafeLineLoader +from apache_beam.yaml.yaml_transform import Scope +from apache_beam.yaml.yaml_transform import chain_as_composite +from apache_beam.yaml.yaml_transform import ensure_errors_consumed +from apache_beam.yaml.yaml_transform import ensure_transforms_have_types +from apache_beam.yaml.yaml_transform import expand_composite_transform +from apache_beam.yaml.yaml_transform import extract_name +from apache_beam.yaml.yaml_transform import identify_object +from apache_beam.yaml.yaml_transform import normalize_inputs_outputs +from apache_beam.yaml.yaml_transform import normalize_source_sink +from apache_beam.yaml.yaml_transform import only_element +from apache_beam.yaml.yaml_transform import pipeline_as_composite +from apache_beam.yaml.yaml_transform import preprocess +from apache_beam.yaml.yaml_transform import preprocess_flattened_inputs +from apache_beam.yaml.yaml_transform import preprocess_windowing +from apache_beam.yaml.yaml_transform import push_windowing_to_roots + + +class SafeLineLoaderTest(unittest.TestCase): + def test_get_line(self): + pipeline_yaml = ''' + type: composite + input: + elements: input + transforms: + - type: PyMap + name: Square + input: elements + config: + fn: "lambda x: x * x" + - type: PyMap + name: Cube + input: elements + config: + fn: "lambda x: x * x * x" + output: + Flatten + ''' + spec = yaml.load(pipeline_yaml, Loader=SafeLineLoader) + self.assertEqual(SafeLineLoader.get_line(spec['type']), 2) + self.assertEqual(SafeLineLoader.get_line(spec['input']), 4) + self.assertEqual(SafeLineLoader.get_line(spec['transforms'][0]), 6) + self.assertEqual(SafeLineLoader.get_line(spec['transforms'][0]['type']), 6) + self.assertEqual(SafeLineLoader.get_line(spec['transforms'][0]['name']), 7) + self.assertEqual(SafeLineLoader.get_line(spec['transforms'][1]), 11) + self.assertEqual(SafeLineLoader.get_line(spec['output']), 17) + self.assertEqual(SafeLineLoader.get_line(spec['transforms']), "unknown") + + def test_strip_metadata(self): + spec_yaml = ''' + transforms: + - type: PyMap + name: Square + ''' + spec = yaml.load(spec_yaml, Loader=SafeLineLoader) + stripped = SafeLineLoader.strip_metadata(spec['transforms']) + + self.assertFalse(hasattr(stripped[0], '__line__')) + self.assertFalse(hasattr(stripped[0], '__uuid__')) + + def test_strip_metadata_nothing_to_strip(self): + spec_yaml = 'prop: 123' + spec = yaml.load(spec_yaml, Loader=SafeLineLoader) + stripped = SafeLineLoader.strip_metadata(spec['prop']) + + self.assertFalse(hasattr(stripped, '__line__')) + self.assertFalse(hasattr(stripped, '__uuid__')) + + +def new_pipeline(): + return beam.Pipeline( + options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) + + +class MainTest(unittest.TestCase): + def assertYaml(self, expected, result): + result = SafeLineLoader.strip_metadata(result) + expected = yaml.load(expected, Loader=SafeLineLoader) + expected = SafeLineLoader.strip_metadata(expected) + + self.assertEqual(expected, result) + + def get_scope_by_spec(self, p, spec, inputs=None): + if inputs is None: + inputs = {} + spec = yaml.load(spec, Loader=SafeLineLoader) + + scope = Scope( + beam.pvalue.PBegin(p), + inputs, + spec['transforms'], + yaml_provider.standard_providers(), {}) + return scope, spec + + def test_pipeline_as_composite_with_type_transforms(self): + spec = ''' + type: composite + transforms: + - type: Create + config: + elements: [0,1,2] + - type: PyMap + config: + fn: 'lambda x: x*x' + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + result = pipeline_as_composite(spec) + + self.assertEqual(result['type'], 'composite') + self.assertEqual(result['name'], None) + + def test_pipeline_as_composite_with_transforms(self): + spec = ''' + transforms: + - type: Create + config: + elements: [0,1,2] + - type: PyMap + config: + fn: 'lambda x: x*x' + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + result = pipeline_as_composite(spec) + + self.assertEqual(result['type'], 'composite') + self.assertEqual(result['name'], None) + + def test_pipeline_as_composite_list(self): + spec = ''' + - type: Create + config: + elements: [0,1,2] + - type: PyMap + config: + fn: 'lambda x: x*x' + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + result = pipeline_as_composite(spec) + + expected = ''' + type: composite + name: null + transforms: + - type: Create + config: + elements: [0,1,2] + - type: PyMap + config: + fn: 'lambda x: x*x' + ''' + self.assertYaml(expected, result) + + def test_expand_composite_transform_with_name(self): + with new_pipeline() as p: + spec = ''' + type: composite + name: Custom + transforms: + - type: Create + config: + elements: [0,1,2] + output: + Create + ''' + scope, spec = self.get_scope_by_spec(p, spec) + self.assertRegex( + str(expand_composite_transform(spec, scope)['output']), + r"PCollection.*Custom/Create/Map.*") + + def test_expand_composite_transform_with_name_input(self): + with new_pipeline() as p: + spec = ''' + type: composite + input: elements + transforms: + - type: LogForTesting + input: input + output: + LogForTesting + ''' + elements = p | beam.Create(range(3)) + scope, spec = self.get_scope_by_spec(p, spec, + inputs={'elements': elements}) + self.assertRegex( + str(expand_composite_transform(spec, scope)['output']), + r"PCollection.*Composite/Map.*") + + def test_expand_composite_transform_root(self): + with new_pipeline() as p: + spec = ''' + type: composite + transforms: + - type: Create + config: + elements: [0,1,2] + output: + Create + ''' + scope, spec = self.get_scope_by_spec(p, spec) + self.assertRegex( + str(expand_composite_transform(spec, scope)['output']), + r"PCollection.*Create/Map.*") + + def test_chain_as_composite(self): + spec = ''' + type: chain + transforms: + - type: Create + config: + elements: [0,1,2] + - type: PyMap + config: + fn: 'lambda x: x*x' + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + result = chain_as_composite(spec) + + expected = f''' + type: composite + name: Chain + input: {{}} + transforms: + - type: Create + config: + elements: [0,1,2] + input: {{}} + - type: PyMap + config: + fn: 'lambda x: x*x' + input: {spec['transforms'][0]['__uuid__']} + output: {spec['transforms'][1]['__uuid__']} + ''' + self.assertYaml(expected, result) + + def test_chain_as_composite_with_wrong_output_type(self): + spec = ''' + type: chain + transforms: + - type: Create + elements: [0,1,2] + - type: PyMap + fn: 'lambda x: x*x' + output: + Create + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + with self.assertRaisesRegex(ValueError, + r"Explicit output.*of the chain transform is " + r"not an output of the last transform"): + chain_as_composite(spec) + + def test_chain_as_composite_with_wrong_output_name(self): + spec = ''' + type: chain + transforms: + - type: Create + name: elements + elements: [0,1,2] + - type: PyMap + fn: 'lambda x: x*x' + output: + elements + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + with self.assertRaisesRegex(ValueError, + r"Explicit output.*of the chain transform is " + r"not an output of the last transform"): + chain_as_composite(spec) + + def test_chain_as_composite_with_outputs_override(self): + spec = ''' + type: chain + transforms: + - type: Create + elements: [0,1,2] + - type: PyMap + fn: 'lambda x: x*x' + output: + PyMap + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + self.assertEqual( + chain_as_composite(spec)['output']['output'], + f"{spec['transforms'][1]['__uuid__']}.PyMap") + + def test_chain_as_composite_with_input(self): + spec = ''' + type: chain + input: + elements + transforms: + - type: PyMap + fn: 'lambda x: x*x' + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + self.assertEqual( + chain_as_composite(spec)['transforms'][0]['input'], {"input": "input"}) + + def test_normalize_source_sink(self): + spec = ''' + source: + type: Create + config: + elements: [0,1,2] + transforms: + - type: PyMap + config: + fn: 'lambda x: x*x' + sink: + type: PyMap + config: + fn: "lambda x: x + 41" + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + result = normalize_source_sink(spec) + + expected = ''' + transforms: + - type: Create + input: {'__explicitly_empty__': null} + config: + elements: [0,1,2] + - type: PyMap + config: + fn: 'lambda x: x*x' + - type: PyMap + config: + fn: "lambda x: x + 41" + ''' + self.assertYaml(expected, result) + + def test_normalize_source_sink_only_source(self): + spec = ''' + source: + type: Create + config: + elements: [0,1,2] + transforms: + - type: PyMap + config: + fn: 'lambda x: x*x' + + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + result = normalize_source_sink(spec) + + expected = ''' + transforms: + - type: Create + input: {'__explicitly_empty__': null} + config: + elements: [0,1,2] + - type: PyMap + config: + fn: 'lambda x: x*x' + ''' + self.assertYaml(expected, result) + + def test_normalize_source_sink_only_sink(self): + spec = ''' + transforms: + - type: PyMap + config: + fn: 'lambda x: x*x' + sink: + type: PyMap + config: + fn: "lambda x: x + 41" + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + result = normalize_source_sink(spec) + expected = ''' + transforms: + - type: PyMap + config: + fn: 'lambda x: x*x' + - type: PyMap + config: + fn: "lambda x: x + 41" + ''' + self.assertYaml(expected, result) + + def test_normalize_source_sink_no_source_no_sink(self): + spec = ''' + transforms: + - type: PyMap + config: + fn: 'lambda x: x*x' + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + result = normalize_source_sink(spec) + + expected = ''' + transforms: + - type: PyMap + config: + fn: 'lambda x: x*x' + ''' + self.assertYaml(expected, result) + + def test_preprocess_source_sink_composite(self): + spec = ''' + type: composite + source: + type: Create + config: + elements: [0,1,2] + transforms: + - type: PyMap + config: + fn: 'lambda x: x*x' + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + result = normalize_source_sink(spec) + + expected = ''' + type: composite + transforms: + - type: Create + input: {'__explicitly_empty__': null} + config: + elements: [0,1,2] + - type: PyMap + config: + fn: 'lambda x: x*x' + ''' + self.assertYaml(expected, result) + + def test_preprocess_source_sink_chain(self): + spec = ''' + type: chain + source: + type: Create + config: + elements: [0,1,2] + transforms: + - type: PyMap + config: + fn: 'lambda x: x*x' + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + result = normalize_source_sink(spec) + + expected = ''' + type: chain + transforms: + - type: Create + input: {'__explicitly_empty__': null} + config: + elements: [0,1,2] + - type: PyMap + config: + fn: 'lambda x: x*x' + ''' + self.assertYaml(expected, result) + + def test_preprocess_source_sink_other(self): + spec = ''' + - type: PyMap + fn: 'lambda x: x*x' + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + self.assertEqual(normalize_source_sink(spec), spec) + + def test_normalize_inputs_outputs(self): + spec = ''' + type: PyMap + input: [Create1, Create2] + fn: 'lambda x: x*x' + output: Squared + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + result = normalize_inputs_outputs(spec) + + expected = ''' + type: PyMap + input: + input: [Create1, Create2] + fn: 'lambda x: x*x' + output: + output: Squared + ''' + self.assertYaml(expected, result) + + def test_normalize_inputs_outputs_dict(self): + spec = ''' + type: PyMap + input: [Create1, Create2] + fn: 'lambda x: x*x' + output: + out1: Squared1 + out2: Squared2 + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + result = normalize_inputs_outputs(spec) + + expected = ''' + type: PyMap + input: + input: [Create1, Create2] + fn: 'lambda x: x*x' + output: + out1: Squared1 + out2: Squared2 + ''' + self.assertYaml(expected, result) + + def test_identify_object_with_name(self): + spec = ''' + type: PyMap + fn: 'lambda x: x*x' + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + self.assertRegex(identify_object(spec), r"PyMap.*[0-9]") + + def test_identify_object(self): + spec = ''' + argument: PyMap + fn: 'lambda x: x*x' + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + self.assertRegex(identify_object(spec), r"at.*[0-9]") + + def test_extract_name_by_type(self): + spec = ''' + type: PyMap + fn: 'lambda x: x*x' + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + self.assertEqual(extract_name(spec), "PyMap") + + def test_extract_name_by_id(self): + spec = ''' + type: PyMap + id: PyMapId + fn: 'lambda x: x*x' + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + self.assertEqual(extract_name(spec), "PyMapId") + + def test_extract_name_by_name(self): + spec = ''' + type: PyMap + name: PyMapName + fn: 'lambda x: x*x' + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + self.assertEqual(extract_name(spec), "PyMapName") + + def test_extract_name_no_name(self): + spec = ''' + transforms: + - arg: PyMap + fn: 'lambda x: x*x' + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + self.assertEqual(extract_name(spec), "") + + def test_push_windowing_to_roots(self): + spec = ''' + type: composite + transforms: + - type: Create + elements: [0,1,2] + - type: PyMap + fn: 'lambda x: x*x' + input: Create + windowing: + type: fixed + size: 2 + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + spec = normalize_inputs_outputs(spec) + spec['transforms'] = [ + normalize_inputs_outputs(t) for t in spec['transforms'] + ] + + result = push_windowing_to_roots(spec) + + expected = ''' + type: composite + transforms: + - type: Create + elements: [0,1,2] + windowing: + type: fixed + size: 2 + __consumed_outputs: + - null + input: {} + output: {} + - type: PyMap + fn: 'lambda x: x*x' + input: + input: Create + output: {} + windowing: + type: fixed + size: 2 + input: {} + output: {} + ''' + self.assertYaml(expected, result) + + def test_preprocess_windowing_custom_type(self): + spec = ''' + type: SumGlobally + input: Create + windowing: + type: fixed + size: 4 + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + spec = normalize_inputs_outputs(spec) + result = preprocess_windowing(spec) + + expected = f''' + type: composite + name: SumGlobally + input: + input: Create + transforms: + - type: SumGlobally + input: + input: {result['transforms'][1]['__uuid__']} + output: {{}} + - type: WindowInto + name: WindowInto[input] + windowing: + type: fixed + size: 4 + input: input + output: {result['transforms'][0]['__uuid__']} + ''' + self.assertYaml(expected, result) + + def test_preprocess_windowing_composite_with_windowing_outer(self): + spec = ''' + type: composite + transforms: + - type: CreateTimestamped + name: Create + elements: [0, 2, 4] + - type: SumGlobally + input: Create + windowing: + type: fixed + size: 4 + output: SumGlobally + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + spec = normalize_inputs_outputs(spec) + spec['transforms'] = [ + normalize_inputs_outputs(t) for t in spec['transforms'] + ] + + result = preprocess_windowing(spec) + + expected = ''' + type: composite + input: {} + transforms: + - type: CreateTimestamped + name: Create + elements: [0, 2, 4] + windowing: + type: fixed + size: 4 + __consumed_outputs: + - null + input: {} + output: {} + - type: SumGlobally + input: + input: Create + output: {} + output: + output: SumGlobally + ''' + self.assertYaml(expected, result) + + def test_preprocess_windowing_composite_with_windowing_on_input(self): + spec = ''' + type: composite + transforms: + - type: CreateTimestamped + name: Create + elements: [0, 2, 4] + - type: SumGlobally + input: Create + windowing: + type: fixed + size: 4 + output: SumGlobally + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + spec = normalize_inputs_outputs(spec) + spec['transforms'] = [ + normalize_inputs_outputs(t) for t in spec['transforms'] + ] + + result = preprocess_windowing(spec) + + expected = ''' + type: composite + input: {} + transforms: + - type: CreateTimestamped + name: Create + elements: [0, 2, 4] + input: {} + output: {} + - type: SumGlobally + input: + input: Create + windowing: + type: fixed + size: 4 + output: {} + output: + output: SumGlobally + ''' + self.assertYaml(expected, result) + + def test_preprocess_windowing_other_type_with_no_inputs(self): + spec = ''' + type: CreateTimestamped + name: Create + elements: [0, 2, 4] + windowing: + type: fixed + size: 4 + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + spec = normalize_inputs_outputs(spec) + result = preprocess_windowing(spec) + + expected = f''' + type: composite + name: Create + transforms: + - type: CreateTimestamped + name: Create + elements: [0, 2, 4] + input: {{}} + output: {{}} + - type: WindowInto + name: WindowInto[None] + input: + input: {result['transforms'][0]["__uuid__"]} + windowing: + type: fixed + size: 4 + output: {result['transforms'][1]["__uuid__"]} + ''' + self.maxDiff = 1e9 + + self.assertYaml(expected, result) + + def test_preprocess_flattened_inputs_implicit(self): + spec = ''' + type: composite + transforms: + - type: PyMap + fn: 'lambda x: x*x' + input: [Create1, Create2] + output: CreateTimestamped + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + spec['transforms'] = [ + normalize_inputs_outputs(t) for t in spec['transforms'] + ] + result = preprocess_flattened_inputs(spec) + + expected = f''' + type: composite + transforms: + - type: Flatten + name: PyMap-Flatten[input] + input: + input0: Create1 + input1: Create2 + - type: PyMap + fn: 'lambda x: x*x' + input: + input: {result['transforms'][0]['__uuid__']} + output: {{}} + output: CreateTimestamped + ''' + self.assertYaml(expected, result) + + def test_preprocess_flattened_inputs_explicit_flatten(self): + spec = ''' + type: composite + transforms: + - type: Flatten + input: [Create1, Create2] + - type: PyMap + fn: 'lambda x: x*x' + input: Flatten + output: CreateTimestamped + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + spec['transforms'] = [ + normalize_inputs_outputs(t) for t in spec['transforms'] + ] + result = preprocess_flattened_inputs(spec) + + expected = ''' + type: composite + transforms: + - type: Flatten + input: + input0: Create1 + input1: Create2 + output: {} + - type: PyMap + fn: 'lambda x: x*x' + input: + input: Flatten + output: {} + output: CreateTimestamped + ''' + self.assertYaml(expected, result) + + def test_ensure_transforms_have_types(self): + spec = ''' + type: PyMap + fn: 'lambda x: x*x' + input: Flatten + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + result = ensure_transforms_have_types(spec) + self.assertEqual(result, spec) + + def test_ensure_transforms_have_types_error(self): + spec = ''' + name: PyMap + fn: 'lambda x: x*x' + input: Flatten + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + with self.assertRaisesRegex(ValueError, r"Missing type .*"): + ensure_transforms_have_types(spec) + with self.assertRaisesRegex(ValueError, r"Missing type .*"): + preprocess(spec) + + def test_ensure_transforms_have_providers_error(self): + spec = ''' + type: UnknownType + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + with self.assertRaisesRegex(ValueError, + r"Unknown type or missing provider .*"): + preprocess(spec, known_transforms=['KnownType']) + + def test_ensure_errors_consumed_unconsumed(self): + spec = ''' + type: composite + transforms: + - type: Create + elements: [1,2,3] + - type: MyTransform + input: Create + error_handling: + output: errors + output: + good: MyTransform + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + spec = normalize_inputs_outputs(spec) + spec['transforms'] = [ + normalize_inputs_outputs(t) for t in spec['transforms'] + ] + with self.assertRaisesRegex(ValueError, r"Unconsumed error.*"): + ensure_errors_consumed(spec) + + def test_ensure_errors_consumed_in_transform(self): + spec = ''' + type: composite + transforms: + - type: Create + elements: [1,2,3] + - type: MyTransform + input: Create + error_handling: + output: errors + - name: SaveToFile + type: PyMap + input: MyTransform.errors + output: + good: MyTransform + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + spec = normalize_inputs_outputs(spec) + spec['transforms'] = [ + normalize_inputs_outputs(t) for t in spec['transforms'] + ] + result = ensure_errors_consumed(spec) + self.assertEqual(result, spec) + self.assertEqual(result['transforms'], spec['transforms']) + + def test_ensure_errors_consumed_no_output_in_error_handling(self): + spec = ''' + type: composite + transforms: + - type: Create + elements: [1,2,3] + - type: MyTransform + input: Create + error_handling: + arg: errors + - name: SaveToFile + type: PyMap + input: MyTransform.errors + output: + good: MyTransform + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + spec = normalize_inputs_outputs(spec) + spec['transforms'] = [ + normalize_inputs_outputs(t) for t in spec['transforms'] + ] + with self.assertRaisesRegex(ValueError, r"Missing output.*"): + ensure_errors_consumed(spec) + + def test_only_element(self): + self.assertEqual(only_element((1, )), 1) + + +class YamlTransformTest(unittest.TestCase): + def test_init_with_string(self): + provider1 = InlineProvider({"MyTransform1": lambda: beam.Map(lambda x: x)}) + provider2 = InlineProvider({"MyTransform2": lambda: beam.Map(lambda x: x)}) + + providers_dict = {"p1": [provider1], "p2": [provider2]} + + spec = ''' + type: chain + transforms: + - type: Create + elements: [1,2,3] + - type: LogForTesting + ''' + result = YamlTransform(spec, providers_dict) + self.assertIn('p1', result._providers) # check for custom providers + self.assertIn('p2', result._providers) # check for custom providers + self.assertIn( + 'LogForTesting', result._providers) # check for standard provider + self.assertEqual(result._spec['type'], "composite") # preprocessed spec + + def test_init_with_dict(self): + spec = ''' + type: chain + transforms: + - type: Create + config: + elements: [1,2,3] + - type: LogForTesting + ''' + spec = yaml.load(spec, Loader=SafeLineLoader) + result = YamlTransform(spec) + self.assertIn( + 'LogForTesting', result._providers) # check for standard provider + self.assertEqual(result._spec['type'], "composite") # preprocessed spec + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + unittest.main() diff --git a/sdks/python/apache_beam/yaml/yaml_udf_test.py b/sdks/python/apache_beam/yaml/yaml_udf_test.py new file mode 100644 index 0000000000000..5e9faa08253cd --- /dev/null +++ b/sdks/python/apache_beam/yaml/yaml_udf_test.py @@ -0,0 +1,246 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import logging +import os +import shutil +import tempfile +import unittest + +import apache_beam as beam +from apache_beam.io import localfilesystem +from apache_beam.options import pipeline_options +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to +from apache_beam.yaml.yaml_transform import YamlTransform + + +def AsRows(): + return beam.Map(lambda named_tuple: beam.Row(**named_tuple._asdict())) + + +class YamlUDFMappingTest(unittest.TestCase): + def __init__(self, method_name='runYamlMappingTest'): + super().__init__(method_name) + self.data = [ + beam.Row(label='11a', conductor=11, rank=0), + beam.Row(label='37a', conductor=37, rank=1), + beam.Row(label='389a', conductor=389, rank=2), + ] + + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + self.fs = localfilesystem.LocalFileSystem(pipeline_options) + + def tearDown(self): + shutil.rmtree(self.tmpdir) + + def test_map_to_fields_filter_inline_js(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + elements = p | beam.Create(self.data) + result = elements | YamlTransform( + ''' + type: MapToFields + input: input + config: + language: javascript + fields: + label: + callable: "function label_map(x) {return x.label + 'x'}" + conductor: + callable: "function conductor_map(x) {return x.conductor + 1}" + ''') + assert_that( + result, + equal_to([ + beam.Row(label='11ax', conductor=12), + beam.Row(label='37ax', conductor=38), + beam.Row(label='389ax', conductor=390), + ])) + + def test_map_to_fields_filter_inline_py(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + elements = p | beam.Create(self.data) + result = elements | YamlTransform( + ''' + type: MapToFields + input: input + config: + language: python + fields: + label: + callable: "lambda x: x.label + 'x'" + conductor: + callable: "lambda x: x.conductor + 1" + ''') + assert_that( + result, + equal_to([ + beam.Row(label='11ax', conductor=12), + beam.Row(label='37ax', conductor=38), + beam.Row(label='389ax', conductor=390), + ])) + + def test_filter_inline_js(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + elements = p | beam.Create(self.data) + result = elements | YamlTransform( + ''' + type: Filter + input: input + config: + language: javascript + keep: + callable: "function filter(x) {return x.rank > 0}" + ''') + assert_that( + result | AsRows(), + equal_to([ + beam.Row(label='37a', conductor=37, rank=1), + beam.Row(label='389a', conductor=389, rank=2), + ])) + + def test_filter_inline_py(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + elements = p | beam.Create(self.data) + result = elements | YamlTransform( + ''' + type: Filter + input: input + config: + language: python + keep: + callable: "lambda x: x.rank > 0" + ''') + assert_that( + result | AsRows(), + equal_to([ + beam.Row(label='37a', conductor=37, rank=1), + beam.Row(label='389a', conductor=389, rank=2), + ])) + + def test_filter_expression_js(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + elements = p | beam.Create(self.data) + result = elements | YamlTransform( + ''' + type: Filter + input: input + config: + language: javascript + keep: + expression: "label.toUpperCase().indexOf('3') == -1 && conductor" + ''') + assert_that( + result | AsRows(), + equal_to([ + beam.Row(label='11a', conductor=11, rank=0), + ])) + + def test_filter_expression_py(self): + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + elements = p | beam.Create(self.data) + result = elements | YamlTransform( + ''' + type: Filter + input: input + config: + language: python + keep: + expression: "'3' not in label" + ''') + assert_that( + result | AsRows(), + equal_to([ + beam.Row(label='11a', conductor=11, rank=0), + ])) + + def test_filter_inline_js_file(self): + data = ''' + function f(x) { + return x.rank > 0 + } + + function g(x) { + return x.rank > 1 + } + '''.replace(' ', '') + + path = os.path.join(self.tmpdir, 'udf.js') + self.fs.create(path).write(data.encode('utf8')) + + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + elements = p | beam.Create(self.data) + result = elements | YamlTransform( + f''' + type: Filter + input: input + config: + language: javascript + keep: + path: {path} + name: "f" + ''') + assert_that( + result | AsRows(), + equal_to([ + beam.Row(label='37a', conductor=37, rank=1), + beam.Row(label='389a', conductor=389, rank=2), + ])) + + def test_filter_inline_py_file(self): + data = ''' + def f(x): + return x.rank > 0 + + def g(x): + return x.rank > 1 + '''.replace(' ', '') + + path = os.path.join(self.tmpdir, 'udf.py') + self.fs.create(path).write(data.encode('utf8')) + + with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions( + pickle_library='cloudpickle')) as p: + elements = p | beam.Create(self.data) + result = elements | YamlTransform( + f''' + type: Filter + input: input + config: + language: python + keep: + path: {path} + name: "f" + ''') + assert_that( + result | AsRows(), + equal_to([ + beam.Row(label='37a', conductor=37, rank=1), + beam.Row(label='389a', conductor=389, rank=2), + ])) + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + unittest.main() diff --git a/sdks/python/build-requirements.txt b/sdks/python/build-requirements.txt deleted file mode 100644 index e8152fbc3ba12..0000000000000 --- a/sdks/python/build-requirements.txt +++ /dev/null @@ -1,28 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# TODO(https://github.com/apache/beam/issues/20051): Consider PEP-517/PEP-518 instead of this file. - -setuptools -wheel>=0.36.0 -grpcio-tools==1.53.0 -mypy-protobuf==3.4.0 -# Avoid https://github.com/pypa/virtualenv/issues/2006 -distlib==0.3.6 - -# Numpy headers -numpy>=1.14.3,<1.26 diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle index 0c2ed72dfba18..7795e77e39634 100644 --- a/sdks/python/build.gradle +++ b/sdks/python/build.gradle @@ -30,7 +30,8 @@ def buildPython = tasks.register("buildPython") { logger.info('Building Python Dependencies') exec { executable 'sh' - args '-c', ". ${envdir}/bin/activate && python setup.py build --build-base ${buildDir}" + // args '-c', ". ${envdir}/bin/activate && python setup.py build --build-base ${buildDir}" + args '-c', ". ${envdir}/bin/activate && pip install -e ." } } } @@ -46,7 +47,7 @@ def sdist = tasks.register("sdist") { // Build artifact exec { executable 'sh' - args '-c', ". ${envdir}/bin/activate && python setup.py -q sdist --formats zip,gztar --dist-dir ${buildDir}" + args '-c', ". ${envdir}/bin/activate && pip install -U build && python -m build --sdist --outdir=${buildDir}" } def collection = fileTree(buildDir){ include "**/*${project.sdk_version}*.tar.gz" exclude 'srcs/**'} @@ -96,7 +97,6 @@ platform_identifiers_map.each { platform, idsuffix -> exec { environment CIBW_BUILD: "cp${pyversion}-${idsuffix}" environment CIBW_ENVIRONMENT: "SETUPTOOLS_USE_DISTUTILS=stdlib" - environment CIBW_BEFORE_BUILD: "pip install cython numpy && pip install --upgrade setuptools" // note: sync cibuildwheel version with GitHub Action // .github/workflow/build_wheel.yml:build_wheels "Install cibuildwheel" step executable 'sh' @@ -110,6 +110,7 @@ platform_identifiers_map.each { platform, idsuffix -> } } + /*************************************************************************************************/ // Non-testing builds and analysis tasks diff --git a/sdks/python/container/Dockerfile b/sdks/python/container/Dockerfile index 83340643cf485..a49933ee6604f 100644 --- a/sdks/python/container/Dockerfile +++ b/sdks/python/container/Dockerfile @@ -40,10 +40,12 @@ RUN \ libyaml-dev \ # This is used to speed up the re-installation of the sdk. ccache \ + # Required for using Beam Python SDK on ARM machines. + libgeos-dev \ && \ rm -rf /var/lib/apt/lists/* && \ - pip install --upgrade setuptools && \ + pip install --upgrade pip setuptools wheel && \ # Install required packages for Beam Python SDK and common dependencies used by users. # use --no-deps to ensure the list includes all transitive dependencies. diff --git a/sdks/python/container/OWNERS b/sdks/python/container/OWNERS deleted file mode 100644 index 9b01cf1e5705e..0000000000000 --- a/sdks/python/container/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - herohde - - aaltay - - charlesccychen diff --git a/sdks/python/container/base_image_requirements_manual.txt b/sdks/python/container/base_image_requirements_manual.txt index be917887402b5..f2f3ea44b44c7 100644 --- a/sdks/python/container/base_image_requirements_manual.txt +++ b/sdks/python/container/base_image_requirements_manual.txt @@ -43,4 +43,4 @@ nose==1.3.7 # For Dataflow internal testing. TODO: remove this. python-snappy;python_version<"3.11" # Optimizes execution of some Beam codepaths. scipy scikit-learn -tensorflow>=2.12.0;python_version>="3.8" +build>=1.0,<2 # tool to build sdist from setup.py in stager. \ No newline at end of file diff --git a/sdks/python/container/boot.go b/sdks/python/container/boot.go index 1e70e0db1513c..ded10a44204a6 100644 --- a/sdks/python/container/boot.go +++ b/sdks/python/container/boot.go @@ -23,6 +23,7 @@ import ( "errors" "flag" "fmt" + "io" "log" "os" "os/exec" @@ -36,6 +37,7 @@ import ( "github.com/apache/beam/sdks/v2/go/container/tools" "github.com/apache/beam/sdks/v2/go/pkg/beam/artifact" + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/xlangx/expansionx" pipepb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/pipeline_v1" "github.com/apache/beam/sdks/v2/go/pkg/beam/util/execx" "github.com/apache/beam/sdks/v2/go/pkg/beam/util/grpcx" @@ -91,7 +93,11 @@ func main() { "--container_executable=/opt/apache/beam/boot", } log.Printf("Starting worker pool %v: python %v", workerPoolId, strings.Join(args, " ")) - if err := execx.Execute("python", args...); err != nil { + pythonVersion, err := expansionx.GetPythonVersion() + if err != nil { + log.Fatalf("Python SDK worker pool exited with error: %v", err) + } + if err := execx.Execute(pythonVersion, args...); err != nil { log.Fatalf("Python SDK worker pool exited with error: %v", err) } log.Print("Python SDK worker pool exited.") @@ -164,11 +170,11 @@ func launchSDKProcess() error { if err != nil { return errors.New( "failed to create a virtual environment. If running on Ubuntu systems, " + - "you might need to install `python3-venv` package. " + - "To run the SDK process in default environment instead, " + - "set the environment variable `RUN_PYTHON_SDK_IN_DEFAULT_ENVIRONMENT=1`. " + - "In custom Docker images, you can do that with an `ENV` statement. " + - fmt.Sprintf("Encountered error: %v", err)) + "you might need to install `python3-venv` package. " + + "To run the SDK process in default environment instead, " + + "set the environment variable `RUN_PYTHON_SDK_IN_DEFAULT_ENVIRONMENT=1`. " + + "In custom Docker images, you can do that with an `ENV` statement. " + + fmt.Sprintf("Encountered error: %v", err)) } cleanupFunc := func() { os.RemoveAll(venvDir) @@ -180,7 +186,10 @@ func launchSDKProcess() error { dir := filepath.Join(*semiPersistDir, "staged") files, err := artifact.Materialize(ctx, *artifactEndpoint, info.GetDependencies(), info.GetRetrievalToken(), dir) if err != nil { - return fmt.Errorf("failed to retrieve staged files: %v", err) + fmtErr := fmt.Errorf("failed to retrieve staged files: %v", err) + // Send error message to logging service before returning up the call stack + logger.Errorf(ctx, fmtErr.Error()) + return fmtErr } // TODO(herohde): the packages to install should be specified explicitly. It @@ -197,8 +206,11 @@ func launchSDKProcess() error { } } - if setupErr := installSetupPackages(fileNames, dir, requirementsFiles); setupErr != nil { - return fmt.Errorf("failed to install required packages: %v", setupErr) + if setupErr := installSetupPackages(ctx, logger, fileNames, dir, requirementsFiles); setupErr != nil { + fmtErr := fmt.Errorf("failed to install required packages: %v", setupErr) + // Send error message to logging service before returning up the call stack + logger.Errorf(ctx, fmtErr.Error()) + return fmtErr } // (3) Invoke python @@ -242,7 +254,7 @@ func launchSDKProcess() error { // have elapsed, i.e., as soon as all subprocesses have returned from Wait(). time.Sleep(5 * time.Second) if err := syscall.Kill(-pid, syscall.SIGKILL); err == nil { - logger.Printf(ctx, "Worker process %v did not respond, killed it.", pid) + logger.Warnf(ctx, "Worker process %v did not respond, killed it.", pid) } }(pid) syscall.Kill(-pid, syscall.SIGTERM) @@ -261,6 +273,7 @@ func launchSDKProcess() error { go func(workerId string) { defer wg.Done() + bufLogger := tools.NewBufferedLogger(logger) errorCount := 0 for { childPids.mu.Lock() @@ -269,7 +282,7 @@ func launchSDKProcess() error { return } logger.Printf(ctx, "Executing Python (worker %v): python %v", workerId, strings.Join(args, " ")) - cmd := StartCommandEnv(map[string]string{"WORKER_ID": workerId}, "python", args...) + cmd := StartCommandEnv(map[string]string{"WORKER_ID": workerId}, os.Stdin, bufLogger, bufLogger, "python", args...) childPids.v = append(childPids.v, cmd.Process.Pid) childPids.mu.Unlock() @@ -277,14 +290,16 @@ func launchSDKProcess() error { // Retry on fatal errors, like OOMs and segfaults, not just // DoFns throwing exceptions. errorCount += 1 + bufLogger.FlushAtError(ctx) if errorCount < 4 { - logger.Printf(ctx, "Python (worker %v) exited %v times: %v\nrestarting SDK process", + logger.Warnf(ctx, "Python (worker %v) exited %v times: %v\nrestarting SDK process", workerId, errorCount, err) } else { logger.Fatalf(ctx, "Python (worker %v) exited %v times: %v\nout of retries, failing container", workerId, errorCount, err) } } else { + bufLogger.FlushAtDebug(ctx) logger.Printf(ctx, "Python (worker %v) exited.", workerId) break } @@ -298,11 +313,11 @@ func launchSDKProcess() error { // Start a command object in a new process group with the given arguments with // additional environment variables. It attaches stdio to the child process. // Returns the process handle. -func StartCommandEnv(env map[string]string, prog string, args ...string) *exec.Cmd { +func StartCommandEnv(env map[string]string, stdin io.Reader, stdout, stderr io.Writer, prog string, args ...string) *exec.Cmd { cmd := exec.Command(prog, args...) - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr + cmd.Stdin = stdin + cmd.Stdout = stdout + cmd.Stderr = stderr if env != nil { cmd.Env = os.Environ() for k, v := range env { @@ -330,7 +345,11 @@ func setupVenv(ctx context.Context, logger *tools.Logger, baseDir, workerId stri if err := os.MkdirAll(dir, 0750); err != nil { return "", fmt.Errorf("failed to create Python venv directory: %s", err) } - if err := execx.Execute("python", "-m", "venv", "--system-site-packages", dir); err != nil { + pythonVersion, err := expansionx.GetPythonVersion() + if err != nil { + return "", err + } + if err := execx.Execute(pythonVersion, "-m", "venv", "--system-site-packages", dir); err != nil { return "", fmt.Errorf("python venv initialization failed: %s", err) } @@ -352,42 +371,42 @@ func setupAcceptableWheelSpecs() error { return fmt.Errorf("cannot get parse Python version from %s", stdoutStderr) } pyVersion := fmt.Sprintf("%s%s", pyVersions[1], pyVersions[2]) - var wheelName string - switch pyVersion { - case "36", "37": - wheelName = fmt.Sprintf("cp%s-cp%sm-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", pyVersion, pyVersion) - default: - wheelName = fmt.Sprintf("cp%s-cp%s-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", pyVersion, pyVersion) - } + wheelName := fmt.Sprintf("cp%s-cp%s-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", pyVersion, pyVersion) acceptableWhlSpecs = append(acceptableWhlSpecs, wheelName) return nil } // installSetupPackages installs Beam SDK and user dependencies. -func installSetupPackages(files []string, workDir string, requirementsFiles []string) error { - log.Printf("Installing setup packages ...") +func installSetupPackages(ctx context.Context, logger *tools.Logger, files []string, workDir string, requirementsFiles []string) error { + bufLogger := tools.NewBufferedLogger(logger) + bufLogger.Printf(ctx, "Installing setup packages ...") if err := setupAcceptableWheelSpecs(); err != nil { - log.Printf("Failed to setup acceptable wheel specs, leave it as empty: %v", err) + bufLogger.Printf(ctx, "Failed to setup acceptable wheel specs, leave it as empty: %v", err) } + pkgName := "apache-beam" + isSdkInstalled := isPackageInstalled(pkgName) + if !isSdkInstalled { + return fmt.Errorf("Apache Beam is not installed in the runtime environment. If you use a custom container image, you must install apache-beam package in the custom image using same version of Beam as in the pipeline submission environment. For more information, see: the https://beam.apache.org/documentation/runtime/environments/") + } // Install the Dataflow Python SDK and worker packages. // We install the extra requirements in case of using the beam sdk. These are ignored by pip // if the user is using an SDK that does not provide these. - if err := installSdk(files, workDir, sdkSrcFile, acceptableWhlSpecs, false); err != nil { + if err := installSdk(ctx, logger, files, workDir, sdkSrcFile, acceptableWhlSpecs, false); err != nil { return fmt.Errorf("failed to install SDK: %v", err) } // The staged files will not disappear due to restarts because workDir is a // folder that is mapped to the host (and therefore survives restarts). for _, f := range requirementsFiles { - if err := pipInstallRequirements(files, workDir, f); err != nil { + if err := pipInstallRequirements(ctx, logger, files, workDir, f); err != nil { return fmt.Errorf("failed to install requirements: %v", err) } } - if err := installExtraPackages(files, extraPackagesFile, workDir); err != nil { + if err := installExtraPackages(ctx, logger, files, extraPackagesFile, workDir); err != nil { return fmt.Errorf("failed to install extra packages: %v", err) } - if err := pipInstallPackage(files, workDir, workflowFile, false, true, nil); err != nil { + if err := pipInstallPackage(ctx, logger, files, workDir, workflowFile, false, true, nil); err != nil { return fmt.Errorf("failed to install workflow: %v", err) } @@ -430,7 +449,7 @@ func processArtifactsInSetupOnlyMode() { } files[i] = filePayload.GetPath() } - if setupErr := installSetupPackages(files, workDir, []string{requirementsFile}); setupErr != nil { + if setupErr := installSetupPackages(context.Background(), nil, files, workDir, []string{requirementsFile}); setupErr != nil { log.Fatalf("Failed to install required packages: %v", setupErr) } } diff --git a/sdks/python/container/build.gradle b/sdks/python/container/build.gradle index da54ef2ebe8a3..06b1ea918c7f8 100644 --- a/sdks/python/container/build.gradle +++ b/sdks/python/container/build.gradle @@ -20,6 +20,8 @@ plugins { id 'org.apache.beam.module' } applyGoNature() description = "Apache Beam :: SDKs :: Python :: Container" +int min_python_version=8 +int max_python_version=11 configurations { sdkSourceTarball @@ -36,23 +38,43 @@ goBuild { } tasks.register("buildAll") { - dependsOn ':sdks:python:container:py37:docker' dependsOn ':sdks:python:container:py38:docker' dependsOn ':sdks:python:container:py39:docker' dependsOn ':sdks:python:container:py310:docker' dependsOn ':sdks:python:container:py311:docker' } +for(int i=min_python_version; i<=max_python_version; ++i) { + String min_version = "3" + min_python_version + String cur = "3" + i + String prev = "3" + (i-1) + tasks.register("push" + cur) { + if (cur != min_version) { + // Enforce ordering to allow the prune step to happen between runs. + // This will ensure we don't use up too much space (especially in CI environments) + mustRunAfter(":sdks:python:container:push" + prev) + } + dependsOn ':sdks:python:container:py' + cur + ':docker' + + doLast { + if (project.hasProperty("prune-images")) { + exec { + executable("docker") + args("system", "prune", "-a", "--force") + } + } + } + } +} + tasks.register("pushAll") { - dependsOn ':sdks:python:container:py37:dockerPush' - dependsOn ':sdks:python:container:py38:dockerPush' - dependsOn ':sdks:python:container:py39:dockerPush' - dependsOn ':sdks:python:container:py310:dockerPush' - dependsOn ':sdks:python:container:py311:dockerPush' + dependsOn ':sdks:python:container:push38' + dependsOn ':sdks:python:container:push39' + dependsOn ':sdks:python:container:push310' + dependsOn ':sdks:python:container:push311' } tasks.register("generatePythonRequirementsAll") { - dependsOn ':sdks:python:container:py37:generatePythonRequirements' dependsOn ':sdks:python:container:py38:generatePythonRequirements' dependsOn ':sdks:python:container:py39:generatePythonRequirements' dependsOn ':sdks:python:container:py310:generatePythonRequirements' diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index 48c059ae5ecd4..bb706aa5c5d83 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -66,6 +66,9 @@ def copyLauncherDependencies = tasks.register("copyLauncherDependencies", Copy) } } +def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] +def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers") + docker { name containerImageName( name: project.docker_image_default_repo_prefix + "python${project.ext.pythonVersion}_sdk", @@ -80,8 +83,10 @@ docker { buildArgs(['py_version': "${project.ext.pythonVersion}", 'pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) || project.rootProject.hasProperty(["isRelease"])]) - buildx project.containerPlatforms() != [project.nativeArchitecture()] + buildx useBuildx platform(*project.containerPlatforms()) + load useBuildx && !pushContainers + push pushContainers } dockerPrepare.dependsOn copyLauncherDependencies diff --git a/sdks/python/container/license_scripts/dep_urls_py.yaml b/sdks/python/container/license_scripts/dep_urls_py.yaml index beea506ca91cd..36efb36c321c8 100644 --- a/sdks/python/container/license_scripts/dep_urls_py.yaml +++ b/sdks/python/container/license_scripts/dep_urls_py.yaml @@ -129,6 +129,8 @@ pip_dependencies: notice: "https://raw.githubusercontent.com/apache/arrow/master/NOTICE.txt" pyhamcrest: license: "https://raw.githubusercontent.com/hamcrest/PyHamcrest/master/LICENSE.txt" + pyjsparser: + license: "https://github.com/PiotrDabkowski/pyjsparser/blob/master/LICENSE" pymongo: license: "https://raw.githubusercontent.com/mongodb/mongo-python-driver/master/LICENSE" python-gflags: diff --git a/sdks/python/container/piputil.go b/sdks/python/container/piputil.go index a00e017445e30..1bafe422d4571 100644 --- a/sdks/python/container/piputil.go +++ b/sdks/python/container/piputil.go @@ -18,41 +18,76 @@ package main import ( "bufio" "bytes" + "context" "errors" "fmt" - "io/ioutil" - "log" "os" + "os/exec" "path/filepath" "strings" + "time" + "github.com/apache/beam/sdks/v2/go/container/tools" + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/xlangx/expansionx" "github.com/apache/beam/sdks/v2/go/pkg/beam/util/execx" ) +const unrecoverableURL string = "https://beam.apache.org/documentation/sdks/python-unrecoverable-errors/index.html#pip-dependency-resolution-failures" + // pipInstallRequirements installs the given requirement, if present. -func pipInstallRequirements(files []string, dir, name string) error { +func pipInstallRequirements(ctx context.Context, logger *tools.Logger, files []string, dir, name string) error { + pythonVersion, err := expansionx.GetPythonVersion() + if err != nil { + return err + } + bufLogger := tools.NewBufferedLogger(logger) for _, file := range files { if file == name { // We run the install process in two rounds in order to avoid as much // as possible PyPI downloads. In the first round the --find-links // option will make sure that only things staged in the worker will be // used without following their dependencies. - args := []string{"-m", "pip", "install", "-r", filepath.Join(dir, name), "--no-cache-dir", "--disable-pip-version-check", "--no-index", "--no-deps", "--find-links", dir} - if err := execx.Execute("python", args...); err != nil { - fmt.Println("Some packages could not be installed solely from the requirements cache. Installing packages from PyPI.") + args := []string{"-m", "pip", "install", "-q", "-r", filepath.Join(dir, name), "--no-cache-dir", "--disable-pip-version-check", "--no-index", "--no-deps", "--find-links", dir} + if err := execx.Execute(pythonVersion, args...); err != nil { + bufLogger.Printf(ctx, "Some packages could not be installed solely from the requirements cache. Installing packages from PyPI.") } // The second install round opens up the search for packages on PyPI and // also installs dependencies. The key is that if all the packages have // been installed in the first round then this command will be a no-op. - args = []string{"-m", "pip", "install", "-r", filepath.Join(dir, name), "--no-cache-dir", "--disable-pip-version-check", "--find-links", dir} - return execx.Execute("python", args...) + args = []string{"-m", "pip", "install", "-q", "-r", filepath.Join(dir, name), "--no-cache-dir", "--disable-pip-version-check", "--find-links", dir} + err := execx.ExecuteEnvWithIO(nil, os.Stdin, bufLogger, bufLogger, pythonVersion, args...) + if err != nil { + bufLogger.FlushAtError(ctx) + return fmt.Errorf("PIP failed to install dependencies, got %s. This error may be unrecoverable, see %s for more information", err, unrecoverableURL) + } + bufLogger.FlushAtDebug(ctx) + return nil } } return nil } +// isPackageInstalled checks if the given package is installed in the +// environment. +func isPackageInstalled(pkgName string) bool { + cmd := exec.Command("python", "-m", "pip", "show", pkgName) + if err := cmd.Run(); err != nil { + if _, ok := err.(*exec.ExitError); ok { + return false + } + } + return true +} + +const pipLogFlushInterval time.Duration = 15 * time.Second + // pipInstallPackage installs the given package, if present. -func pipInstallPackage(files []string, dir, name string, force, optional bool, extras []string) error { +func pipInstallPackage(ctx context.Context, logger *tools.Logger, files []string, dir, name string, force, optional bool, extras []string) error { + pythonVersion, err := expansionx.GetPythonVersion() + if err != nil { + return err + } + bufLogger := tools.NewBufferedLoggerWithFlushInterval(ctx, logger, pipLogFlushInterval) for _, file := range files { if file == name { var packageSpec = name @@ -78,17 +113,32 @@ func pipInstallPackage(files []string, dir, name string, force, optional bool, e // installed if necessary. This achieves our goal outlined above. args := []string{"-m", "pip", "install", "--no-cache-dir", "--disable-pip-version-check", "--upgrade", "--force-reinstall", "--no-deps", filepath.Join(dir, packageSpec)} - err := execx.Execute("python", args...) + err := execx.ExecuteEnvWithIO(nil, os.Stdin, bufLogger, bufLogger, pythonVersion, args...) if err != nil { - return err + bufLogger.FlushAtError(ctx) + return fmt.Errorf("PIP failed to install dependencies, got %s. This error may be unrecoverable, see %s for more information", err, unrecoverableURL) + } else { + bufLogger.FlushAtDebug(ctx) } args = []string{"-m", "pip", "install", "--no-cache-dir", "--disable-pip-version-check", filepath.Join(dir, packageSpec)} - return execx.Execute("python", args...) + err = execx.ExecuteEnvWithIO(nil, os.Stdin, bufLogger, bufLogger, pythonVersion, args...) + if err != nil { + bufLogger.FlushAtError(ctx) + return fmt.Errorf("PIP failed to install dependencies, got %s. This error may be unrecoverable, see %s for more information", err, unrecoverableURL) + } + bufLogger.FlushAtDebug(ctx) + return nil } // Case when we do not perform a forced reinstall. args := []string{"-m", "pip", "install", "--no-cache-dir", "--disable-pip-version-check", filepath.Join(dir, packageSpec)} - return execx.Execute("python", args...) + err := execx.ExecuteEnvWithIO(nil, os.Stdin, bufLogger, bufLogger, pythonVersion, args...) + if err != nil { + bufLogger.FlushAtError(ctx) + return fmt.Errorf("PIP failed to install dependencies, got %s. This error may be unrecoverable, see %s for more information", err, unrecoverableURL) + } + bufLogger.FlushAtDebug(ctx) + return nil } } if optional { @@ -99,7 +149,8 @@ func pipInstallPackage(files []string, dir, name string, force, optional bool, e // installExtraPackages installs all the packages declared in the extra // packages manifest file. -func installExtraPackages(files []string, extraPackagesFile, dir string) error { +func installExtraPackages(ctx context.Context, logger *tools.Logger, files []string, extraPackagesFile, dir string) error { + bufLogger := tools.NewBufferedLogger(logger) // First check that extra packages manifest file is present. for _, file := range files { if file != extraPackagesFile { @@ -107,7 +158,7 @@ func installExtraPackages(files []string, extraPackagesFile, dir string) error { } // Found the manifest. Install extra packages. - manifest, err := ioutil.ReadFile(filepath.Join(dir, extraPackagesFile)) + manifest, err := os.ReadFile(filepath.Join(dir, extraPackagesFile)) if err != nil { return fmt.Errorf("failed to read extra packages manifest file: %v", err) } @@ -117,8 +168,8 @@ func installExtraPackages(files []string, extraPackagesFile, dir string) error { for s.Scan() { extraPackage := s.Text() - log.Printf("Installing extra package: %s", extraPackage) - if err = pipInstallPackage(files, dir, extraPackage, true, false, nil); err != nil { + bufLogger.Printf(ctx, "Installing extra package: %s", extraPackage) + if err = pipInstallPackage(ctx, logger, files, dir, extraPackage, true, false, nil); err != nil { return fmt.Errorf("failed to install extra package %s: %v", extraPackage, err) } } @@ -127,12 +178,13 @@ func installExtraPackages(files []string, extraPackagesFile, dir string) error { return nil } -func findBeamSdkWhl(files []string, acceptableWhlSpecs []string) string { +func findBeamSdkWhl(ctx context.Context, logger *tools.Logger, files []string, acceptableWhlSpecs []string) string { + bufLogger := tools.NewBufferedLogger(logger) for _, file := range files { if strings.HasPrefix(file, "apache_beam") { for _, s := range acceptableWhlSpecs { if strings.HasSuffix(file, s) { - log.Printf("Found Apache Beam SDK wheel: %v", file) + bufLogger.Printf(ctx, "Found Apache Beam SDK wheel: %v", file) return file } } @@ -146,17 +198,17 @@ func findBeamSdkWhl(files []string, acceptableWhlSpecs []string) string { // assume that the pipleine was started with the Beam SDK found in the wheel // file, and we try to install it. If not successful, we fall back to installing // SDK from source tarball provided in sdkSrcFile. -func installSdk(files []string, workDir string, sdkSrcFile string, acceptableWhlSpecs []string, required bool) error { - sdkWhlFile := findBeamSdkWhl(files, acceptableWhlSpecs) - +func installSdk(ctx context.Context, logger *tools.Logger, files []string, workDir string, sdkSrcFile string, acceptableWhlSpecs []string, required bool) error { + sdkWhlFile := findBeamSdkWhl(ctx, logger, files, acceptableWhlSpecs) + bufLogger := tools.NewBufferedLogger(logger) if sdkWhlFile != "" { // by default, pip rejects to install wheel if same version already installed isDev := strings.Contains(sdkWhlFile, ".dev") - err := pipInstallPackage(files, workDir, sdkWhlFile, isDev, false, []string{"gcp"}) + err := pipInstallPackage(ctx, logger, files, workDir, sdkWhlFile, isDev, false, []string{"gcp"}) if err == nil { return nil } - log.Printf("Could not install Apache Beam SDK from a wheel: %v, proceeding to install SDK from source tarball.", err) + bufLogger.Printf(ctx, "Could not install Apache Beam SDK from a wheel: %v, proceeding to install SDK from source tarball.", err) } if !required { _, err := os.Stat(filepath.Join(workDir, sdkSrcFile)) @@ -164,6 +216,6 @@ func installSdk(files []string, workDir string, sdkSrcFile string, acceptableWhl return nil } } - err := pipInstallPackage(files, workDir, sdkSrcFile, false, false, []string{"gcp"}) + err := pipInstallPackage(ctx, logger, files, workDir, sdkSrcFile, false, false, []string{"gcp"}) return err } diff --git a/sdks/python/container/py310/base_image_requirements.txt b/sdks/python/container/py310/base_image_requirements.txt index d955102d4383a..82b210005feb0 100644 --- a/sdks/python/container/py310/base_image_requirements.txt +++ b/sdks/python/container/py310/base_image_requirements.txt @@ -21,136 +21,120 @@ # https://s.apache.org/beam-python-dev-wiki # Reach out to a committer if you need help. -absl-py==1.4.0 -astunparse==1.6.3 attrs==23.1.0 beautifulsoup4==4.12.2 bs4==0.0.1 -cachetools==5.3.0 -certifi==2023.5.7 +cachetools==5.3.1 +certifi==2023.7.22 cffi==1.15.1 -charset-normalizer==3.1.0 -click==8.1.3 +charset-normalizer==3.2.0 +click==8.1.7 cloudpickle==2.2.1 crcmod==1.7 -cryptography==40.0.2 -Cython==0.29.34 +cryptography==41.0.4 +Cython==0.29.36 deprecation==2.1.0 dill==0.3.1.1 -dnspython==2.3.0 -docker==6.1.2 +dnspython==2.4.2 +docker==6.1.3 docopt==0.6.2 -exceptiongroup==1.1.1 -execnet==1.9.0 -fastavro==1.7.4 -fasteners==0.18 -flatbuffers==23.5.9 +exceptiongroup==1.1.3 +execnet==2.0.2 +fastavro==1.8.3 +fasteners==0.19 freezegun==1.2.2 future==0.18.3 -gast==0.4.0 -google-api-core==2.11.0 -google-api-python-client==2.86.0 +google-api-core==2.11.1 +google-api-python-client==2.100.0 google-apitools==0.5.31 -google-auth==2.18.0 -google-auth-httplib2==0.1.0 -google-auth-oauthlib==1.0.0 -google-cloud-bigquery==3.10.0 -google-cloud-bigquery-storage==2.19.1 -google-cloud-bigtable==2.17.0 -google-cloud-core==2.3.2 -google-cloud-datastore==2.15.2 -google-cloud-dlp==3.12.1 -google-cloud-language==2.9.1 -google-cloud-profiler==4.0.0 -google-cloud-pubsub==2.17.0 -google-cloud-pubsublite==1.8.2 -google-cloud-recommendations-ai==0.10.3 -google-cloud-spanner==3.34.0 -google-cloud-videointelligence==2.11.1 -google-cloud-vision==3.4.1 +google-auth==2.23.0 +google-auth-httplib2==0.1.1 +google-cloud-aiplatform==1.33.1 +google-cloud-bigquery==3.11.4 +google-cloud-bigquery-storage==2.22.0 +google-cloud-bigtable==2.21.0 +google-cloud-core==2.3.3 +google-cloud-datastore==2.18.0 +google-cloud-dlp==3.12.3 +google-cloud-language==2.11.1 +google-cloud-profiler==4.1.0 +google-cloud-pubsub==2.18.4 +google-cloud-pubsublite==1.8.3 +google-cloud-recommendations-ai==0.10.5 +google-cloud-resource-manager==1.10.4 +google-cloud-spanner==3.40.1 +google-cloud-storage==2.11.0 +google-cloud-videointelligence==2.11.4 +google-cloud-vision==3.4.4 google-crc32c==1.5.0 -google-pasta==0.2.0 -google-resumable-media==2.5.0 -googleapis-common-protos==1.59.0 +google-resumable-media==2.6.0 +googleapis-common-protos==1.60.0 greenlet==2.0.2 grpc-google-iam-v1==0.12.6 -grpcio==1.54.2 -grpcio-status==1.54.2 +grpcio==1.58.0 +grpcio-status==1.58.0 guppy3==3.1.3 -h5py==3.8.0 -hdfs==2.7.0 +hdfs==2.7.2 httplib2==0.22.0 -hypothesis==6.75.3 +hypothesis==6.87.0 idna==3.4 iniconfig==2.0.0 -jax==0.4.10 -joblib==1.2.0 -keras==2.12.0 -libclang==16.0.0 -Markdown==3.4.3 -MarkupSafe==2.1.2 -ml-dtypes==0.1.0 -mmh3==3.1.0 -mock==5.0.2 +joblib==1.3.2 +Js2Py==0.74 +mmh3==4.0.1 +mock==5.1.0 nltk==3.8.1 nose==1.3.7 -numpy==1.23.5 +numpy==1.24.4 oauth2client==4.1.3 -oauthlib==3.2.2 objsize==0.6.1 -opt-einsum==3.3.0 -orjson==3.8.12 +orjson==3.9.7 overrides==6.5.0 packaging==23.1 pandas==1.5.3 parameterized==0.9.0 -pluggy==1.0.0 -proto-plus==1.22.2 -protobuf==4.23.0 -psycopg2-binary==2.9.6 +pluggy==1.3.0 +proto-plus==1.22.3 +protobuf==4.24.3 +psycopg2-binary==2.9.7 pyarrow==11.0.0 pyasn1==0.5.0 pyasn1-modules==0.3.0 pycparser==2.21 pydot==1.4.2 PyHamcrest==2.0.4 -pymongo==4.3.3 -PyMySQL==1.0.3 -pyparsing==3.0.9 -pytest==7.3.1 +pyjsparser==2.7.1 +pymongo==4.5.0 +PyMySQL==1.1.0 +pyparsing==3.1.1 +pytest==7.4.2 pytest-timeout==2.1.0 -pytest-xdist==3.3.0 +pytest-xdist==3.3.1 python-dateutil==2.8.2 python-snappy==0.6.1 -pytz==2023.3 -PyYAML==6.0 -regex==2023.5.5 -requests==2.30.0 -requests-mock==1.10.0 -requests-oauthlib==1.3.1 +pytz==2023.3.post1 +PyYAML==6.0.1 +regex==2023.8.8 +requests==2.31.0 +requests-mock==1.11.0 rsa==4.9 -scikit-learn==1.2.2 -scipy==1.10.1 +scikit-learn==1.3.1 +scipy==1.11.2 +Shapely==1.8.5.post1 six==1.16.0 sortedcontainers==2.4.0 -soupsieve==2.4.1 -SQLAlchemy==1.4.48 +soupsieve==2.5 +SQLAlchemy==1.4.49 sqlparse==0.4.4 -tenacity==8.2.2 -tensorboard==2.12.3 -tensorboard-data-server==0.7.0 -tensorflow==2.12.0 -tensorflow-estimator==2.12.0 -tensorflow-io-gcs-filesystem==0.32.0 -termcolor==2.3.0 +tenacity==8.2.3 testcontainers==3.7.1 -threadpoolctl==3.1.0 +threadpoolctl==3.2.0 tomli==2.0.1 -tqdm==4.65.0 -typing_extensions==4.5.0 +tqdm==4.66.1 +typing_extensions==4.8.0 +tzlocal==5.0.1 uritemplate==4.1.1 -urllib3==1.26.15 -websocket-client==1.5.1 -Werkzeug==2.3.4 -wrapt==1.14.1 +urllib3==1.26.18 +websocket-client==1.6.3 +wrapt==1.15.0 zstandard==0.21.0 diff --git a/sdks/python/container/py311/base_image_requirements.txt b/sdks/python/container/py311/base_image_requirements.txt index 755577c9ec8f8..2e5d834926bd2 100644 --- a/sdks/python/container/py311/base_image_requirements.txt +++ b/sdks/python/container/py311/base_image_requirements.txt @@ -21,130 +21,114 @@ # https://s.apache.org/beam-python-dev-wiki # Reach out to a committer if you need help. -absl-py==1.4.0 -astunparse==1.6.3 attrs==23.1.0 beautifulsoup4==4.12.2 bs4==0.0.1 -cachetools==5.3.0 -certifi==2023.5.7 +cachetools==5.3.1 +certifi==2023.7.22 cffi==1.15.1 -charset-normalizer==3.1.0 -click==8.1.3 +charset-normalizer==3.2.0 +click==8.1.7 cloudpickle==2.2.1 crcmod==1.7 -cryptography==40.0.2 -Cython==0.29.34 +cryptography==41.0.4 +Cython==0.29.36 deprecation==2.1.0 dill==0.3.1.1 -dnspython==2.3.0 -docker==6.1.2 +dnspython==2.4.2 +docker==6.1.3 docopt==0.6.2 -execnet==1.9.0 -fastavro==1.7.4 -fasteners==0.18 -flatbuffers==23.5.9 +execnet==2.0.2 +fastavro==1.8.3 +fasteners==0.19 freezegun==1.2.2 future==0.18.3 -gast==0.4.0 -google-api-core==2.11.0 +google-api-core==2.11.1 google-apitools==0.5.31 -google-auth==2.18.0 -google-auth-httplib2==0.1.0 -google-auth-oauthlib==1.0.0 -google-cloud-bigquery==3.10.0 -google-cloud-bigquery-storage==2.19.1 -google-cloud-bigtable==2.17.0 -google-cloud-core==2.3.2 -google-cloud-datastore==2.15.2 -google-cloud-dlp==3.12.1 -google-cloud-language==2.9.1 -google-cloud-pubsub==2.17.0 -google-cloud-pubsublite==1.8.2 -google-cloud-recommendations-ai==0.10.3 -google-cloud-spanner==3.34.0 -google-cloud-videointelligence==2.11.1 -google-cloud-vision==3.4.1 +google-auth==2.23.0 +google-auth-httplib2==0.1.1 +google-cloud-aiplatform==1.33.1 +google-cloud-bigquery==3.11.4 +google-cloud-bigquery-storage==2.22.0 +google-cloud-bigtable==2.21.0 +google-cloud-core==2.3.3 +google-cloud-datastore==2.18.0 +google-cloud-dlp==3.12.3 +google-cloud-language==2.11.1 +google-cloud-pubsub==2.18.4 +google-cloud-pubsublite==1.8.3 +google-cloud-recommendations-ai==0.10.5 +google-cloud-resource-manager==1.10.4 +google-cloud-spanner==3.40.1 +google-cloud-storage==2.11.0 +google-cloud-videointelligence==2.11.4 +google-cloud-vision==3.4.4 google-crc32c==1.5.0 -google-pasta==0.2.0 -google-resumable-media==2.5.0 -googleapis-common-protos==1.59.0 +google-resumable-media==2.6.0 +googleapis-common-protos==1.60.0 greenlet==2.0.2 grpc-google-iam-v1==0.12.6 -grpcio==1.54.2 -grpcio-status==1.54.2 +grpcio==1.58.0 +grpcio-status==1.58.0 guppy3==3.1.3 -h5py==3.8.0 -hdfs==2.7.0 +hdfs==2.7.2 httplib2==0.22.0 -hypothesis==6.75.3 +hypothesis==6.87.0 idna==3.4 iniconfig==2.0.0 -jax==0.4.10 -joblib==1.2.0 -keras==2.12.0 -libclang==16.0.0 -Markdown==3.4.3 -MarkupSafe==2.1.2 -ml-dtypes==0.1.0 -mmh3==3.1.0 -mock==5.0.2 +joblib==1.3.2 +Js2Py==0.74 +mmh3==4.0.1 +mock==5.1.0 nltk==3.8.1 nose==1.3.7 -numpy==1.23.5 +numpy==1.24.4 oauth2client==4.1.3 -oauthlib==3.2.2 objsize==0.6.1 -opt-einsum==3.3.0 -orjson==3.8.12 +orjson==3.9.7 overrides==6.5.0 packaging==23.1 pandas==1.5.3 parameterized==0.9.0 -pluggy==1.0.0 -proto-plus==1.22.2 -protobuf==4.23.0 -psycopg2-binary==2.9.6 +pluggy==1.3.0 +proto-plus==1.22.3 +protobuf==4.24.3 +psycopg2-binary==2.9.7 pyarrow==11.0.0 pyasn1==0.5.0 pyasn1-modules==0.3.0 pycparser==2.21 pydot==1.4.2 PyHamcrest==2.0.4 -pymongo==4.3.3 -PyMySQL==1.0.3 -pyparsing==3.0.9 -pytest==7.3.1 +pyjsparser==2.7.1 +pymongo==4.5.0 +PyMySQL==1.1.0 +pyparsing==3.1.1 +pytest==7.4.2 pytest-timeout==2.1.0 -pytest-xdist==3.3.0 +pytest-xdist==3.3.1 python-dateutil==2.8.2 -pytz==2023.3 -PyYAML==6.0 -regex==2023.5.5 -requests==2.30.0 -requests-mock==1.10.0 -requests-oauthlib==1.3.1 +pytz==2023.3.post1 +PyYAML==6.0.1 +regex==2023.8.8 +requests==2.31.0 +requests-mock==1.11.0 rsa==4.9 -scikit-learn==1.2.2 -scipy==1.10.1 +scikit-learn==1.3.1 +scipy==1.11.2 +Shapely==1.8.5.post1 six==1.16.0 sortedcontainers==2.4.0 -soupsieve==2.4.1 -SQLAlchemy==1.4.48 +soupsieve==2.5 +SQLAlchemy==1.4.49 sqlparse==0.4.4 -tenacity==8.2.2 -tensorboard==2.12.3 -tensorboard-data-server==0.7.0 -tensorflow==2.12.0 -tensorflow-estimator==2.12.0 -tensorflow-io-gcs-filesystem==0.32.0 -termcolor==2.3.0 +tenacity==8.2.3 testcontainers==3.7.1 -threadpoolctl==3.1.0 -tqdm==4.65.0 -typing_extensions==4.5.0 -urllib3==1.26.15 -websocket-client==1.5.1 -Werkzeug==2.3.4 -wrapt==1.14.1 +threadpoolctl==3.2.0 +tqdm==4.66.1 +typing_extensions==4.8.0 +tzlocal==5.0.1 +urllib3==1.26.16 +websocket-client==1.6.3 +wrapt==1.15.0 zstandard==0.21.0 diff --git a/sdks/python/container/py37/base_image_requirements.txt b/sdks/python/container/py37/base_image_requirements.txt deleted file mode 100644 index a514f4b8b1090..0000000000000 --- a/sdks/python/container/py37/base_image_requirements.txt +++ /dev/null @@ -1,135 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Autogenerated requirements file for Apache Beam py37 container image. -# Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. -# Do not edit manually, adjust ../base_image_requirements_manual.txt or -# Apache Beam's setup.py instead, and regenerate the list. -# You will need Python interpreters for all versions supported by Beam, see: -# https://s.apache.org/beam-python-dev-wiki -# Reach out to a committer if you need help. - -attrs==23.1.0 -beautifulsoup4==4.12.2 -bs4==0.0.1 -cachetools==5.3.0 -certifi==2023.5.7 -cffi==1.15.1 -charset-normalizer==3.1.0 -click==8.1.3 -cloudpickle==2.2.1 -crcmod==1.7 -cryptography==41.0.0 -Cython==0.29.34 -deprecation==2.1.0 -dill==0.3.1.1 -dnspython==2.3.0 -docker==6.1.2 -docopt==0.6.2 -exceptiongroup==1.1.1 -execnet==1.9.0 -fastavro==1.7.4 -fasteners==0.18 -freezegun==1.2.2 -future==0.18.3 -google-api-core==2.11.0 -google-api-python-client==2.86.0 -google-apitools==0.5.31 -google-auth==2.18.0 -google-auth-httplib2==0.1.0 -google-cloud-bigquery==3.10.0 -google-cloud-bigquery-storage==2.19.1 -google-cloud-bigtable==2.17.0 -google-cloud-core==2.3.2 -google-cloud-datastore==2.15.2 -google-cloud-dlp==3.12.1 -google-cloud-language==2.9.1 -google-cloud-profiler==4.0.0 -google-cloud-pubsub==2.17.0 -google-cloud-pubsublite==1.7.0 -google-cloud-recommendations-ai==0.10.3 -google-cloud-spanner==3.34.0 -google-cloud-videointelligence==2.11.1 -google-cloud-vision==3.4.1 -google-crc32c==1.5.0 -google-resumable-media==2.5.0 -googleapis-common-protos==1.59.0 -greenlet==2.0.2 -grpc-google-iam-v1==0.12.6 -grpcio==1.54.2 -grpcio-status==1.54.2 -guppy3==3.1.3 -hdfs==2.7.0 -httplib2==0.22.0 -hypothesis==6.75.3 -idna==3.4 -importlib-metadata==6.6.0 -iniconfig==2.0.0 -joblib==1.2.0 -mmh3==3.1.0 -mock==5.0.2 -nltk==3.8.1 -nose==1.3.7 -numpy==1.21.6 -oauth2client==4.1.3 -objsize==0.6.1 -orjson==3.8.12 -overrides==6.5.0 -packaging==23.1 -pandas==1.3.5 -parameterized==0.9.0 -pluggy==1.0.0 -proto-plus==1.22.2 -protobuf==4.23.0 -psycopg2-binary==2.9.6 -pyarrow==11.0.0 -pyasn1==0.5.0 -pyasn1-modules==0.3.0 -pycparser==2.21 -pydot==1.4.2 -PyHamcrest==2.0.4 -pymongo==4.3.3 -PyMySQL==1.0.3 -pyparsing==3.0.9 -pytest==7.3.1 -pytest-timeout==2.1.0 -pytest-xdist==3.3.0 -python-dateutil==2.8.2 -python-snappy==0.6.1 -pytz==2023.3 -PyYAML==6.0 -regex==2023.5.5 -requests==2.30.0 -requests-mock==1.10.0 -rsa==4.9 -scikit-learn==1.0.2 -scipy==1.7.3 -six==1.16.0 -sortedcontainers==2.4.0 -soupsieve==2.4.1 -SQLAlchemy==1.4.48 -sqlparse==0.4.4 -tenacity==8.2.2 -testcontainers==3.7.1 -threadpoolctl==3.1.0 -tomli==2.0.1 -tqdm==4.65.0 -typing_extensions==4.5.0 -uritemplate==4.1.1 -urllib3==1.26.15 -websocket-client==1.5.1 -wrapt==1.15.0 -zipp==3.15.0 -zstandard==0.21.0 diff --git a/sdks/python/container/py37/build.gradle b/sdks/python/container/py37/build.gradle deleted file mode 100644 index 547163a3514e1..0000000000000 --- a/sdks/python/container/py37/build.gradle +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -plugins { - id 'base' - id 'org.apache.beam.module' -} -applyDockerNature() -applyPythonNature() - -pythonVersion = '3.7' - -apply from: "../common.gradle" diff --git a/sdks/python/container/py38/base_image_requirements.txt b/sdks/python/container/py38/base_image_requirements.txt index fe2c698c2ece2..ed5d35fc64578 100644 --- a/sdks/python/container/py38/base_image_requirements.txt +++ b/sdks/python/container/py38/base_image_requirements.txt @@ -21,138 +21,121 @@ # https://s.apache.org/beam-python-dev-wiki # Reach out to a committer if you need help. -absl-py==1.4.0 -astunparse==1.6.3 attrs==23.1.0 +backports.zoneinfo==0.2.1 beautifulsoup4==4.12.2 bs4==0.0.1 -cachetools==5.3.0 -certifi==2023.5.7 +cachetools==5.3.1 +certifi==2023.7.22 cffi==1.15.1 -charset-normalizer==3.1.0 -click==8.1.3 +charset-normalizer==3.2.0 +click==8.1.7 cloudpickle==2.2.1 crcmod==1.7 -cryptography==40.0.2 -Cython==0.29.34 +cryptography==41.0.4 +Cython==0.29.36 deprecation==2.1.0 dill==0.3.1.1 -dnspython==2.3.0 -docker==6.1.2 +dnspython==2.4.2 +docker==6.1.3 docopt==0.6.2 -exceptiongroup==1.1.1 -execnet==1.9.0 -fastavro==1.7.4 -fasteners==0.18 -flatbuffers==23.5.9 +exceptiongroup==1.1.3 +execnet==2.0.2 +fastavro==1.8.3 +fasteners==0.19 freezegun==1.2.2 future==0.18.3 -gast==0.4.0 -google-api-core==2.11.0 -google-api-python-client==2.86.0 +google-api-core==2.11.1 +google-api-python-client==2.100.0 google-apitools==0.5.31 -google-auth==2.18.0 -google-auth-httplib2==0.1.0 -google-auth-oauthlib==1.0.0 -google-cloud-bigquery==3.10.0 -google-cloud-bigquery-storage==2.19.1 -google-cloud-bigtable==2.17.0 -google-cloud-core==2.3.2 -google-cloud-datastore==2.15.2 -google-cloud-dlp==3.12.1 -google-cloud-language==2.9.1 -google-cloud-profiler==4.0.0 -google-cloud-pubsub==2.17.0 -google-cloud-pubsublite==1.8.2 -google-cloud-recommendations-ai==0.10.3 -google-cloud-spanner==3.34.0 -google-cloud-videointelligence==2.11.1 -google-cloud-vision==3.4.1 +google-auth==2.23.0 +google-auth-httplib2==0.1.1 +google-cloud-aiplatform==1.33.1 +google-cloud-bigquery==3.11.4 +google-cloud-bigquery-storage==2.22.0 +google-cloud-bigtable==2.21.0 +google-cloud-core==2.3.3 +google-cloud-datastore==2.18.0 +google-cloud-dlp==3.12.3 +google-cloud-language==2.11.1 +google-cloud-profiler==4.1.0 +google-cloud-pubsub==2.18.4 +google-cloud-pubsublite==1.8.3 +google-cloud-recommendations-ai==0.10.5 +google-cloud-resource-manager==1.10.4 +google-cloud-spanner==3.40.1 +google-cloud-storage==2.11.0 +google-cloud-videointelligence==2.11.4 +google-cloud-vision==3.4.4 google-crc32c==1.5.0 -google-pasta==0.2.0 -google-resumable-media==2.5.0 -googleapis-common-protos==1.59.0 +google-resumable-media==2.6.0 +googleapis-common-protos==1.60.0 greenlet==2.0.2 grpc-google-iam-v1==0.12.6 -grpcio==1.54.2 -grpcio-status==1.54.2 +grpcio==1.58.0 +grpcio-status==1.58.0 guppy3==3.1.3 -h5py==3.8.0 -hdfs==2.7.0 +hdfs==2.7.2 httplib2==0.22.0 -hypothesis==6.75.3 +hypothesis==6.87.0 idna==3.4 -importlib-metadata==6.6.0 iniconfig==2.0.0 -jax==0.4.10 -joblib==1.2.0 -keras==2.12.0 -libclang==16.0.0 -Markdown==3.4.3 -MarkupSafe==2.1.2 -ml-dtypes==0.1.0 -mmh3==3.1.0 -mock==5.0.2 +joblib==1.3.2 +Js2Py==0.74 +mmh3==4.0.1 +mock==5.1.0 nltk==3.8.1 nose==1.3.7 -numpy==1.23.5 +numpy==1.24.4 oauth2client==4.1.3 -oauthlib==3.2.2 objsize==0.6.1 -opt-einsum==3.3.0 -orjson==3.8.12 +orjson==3.9.7 overrides==6.5.0 packaging==23.1 pandas==1.5.3 parameterized==0.9.0 -pluggy==1.0.0 -proto-plus==1.22.2 -protobuf==4.23.0 -psycopg2-binary==2.9.6 +pluggy==1.3.0 +proto-plus==1.22.3 +protobuf==4.24.3 +psycopg2-binary==2.9.7 pyarrow==11.0.0 pyasn1==0.5.0 pyasn1-modules==0.3.0 pycparser==2.21 pydot==1.4.2 PyHamcrest==2.0.4 -pymongo==4.3.3 -PyMySQL==1.0.3 -pyparsing==3.0.9 -pytest==7.3.1 +pyjsparser==2.7.1 +pymongo==4.5.0 +PyMySQL==1.1.0 +pyparsing==3.1.1 +pytest==7.4.2 pytest-timeout==2.1.0 -pytest-xdist==3.3.0 +pytest-xdist==3.3.1 python-dateutil==2.8.2 python-snappy==0.6.1 -pytz==2023.3 -PyYAML==6.0 -regex==2023.5.5 -requests==2.30.0 -requests-mock==1.10.0 -requests-oauthlib==1.3.1 +pytz==2023.3.post1 +PyYAML==6.0.1 +regex==2023.8.8 +requests==2.31.0 +requests-mock==1.11.0 rsa==4.9 -scikit-learn==1.2.2 +scikit-learn==1.3.1 scipy==1.10.1 +Shapely==1.8.5.post1 six==1.16.0 sortedcontainers==2.4.0 -soupsieve==2.4.1 -SQLAlchemy==1.4.48 +soupsieve==2.5 +SQLAlchemy==1.4.49 sqlparse==0.4.4 -tenacity==8.2.2 -tensorboard==2.12.3 -tensorboard-data-server==0.7.0 -tensorflow==2.12.0 -tensorflow-estimator==2.12.0 -tensorflow-io-gcs-filesystem==0.32.0 -termcolor==2.3.0 +tenacity==8.2.3 testcontainers==3.7.1 -threadpoolctl==3.1.0 +threadpoolctl==3.2.0 tomli==2.0.1 -tqdm==4.65.0 -typing_extensions==4.5.0 +tqdm==4.66.1 +typing_extensions==4.8.0 +tzlocal==5.0.1 uritemplate==4.1.1 -urllib3==1.26.15 -websocket-client==1.5.1 -Werkzeug==2.3.4 -wrapt==1.14.1 -zipp==3.15.0 +urllib3==1.26.17 +websocket-client==1.6.3 +wrapt==1.15.0 zstandard==0.21.0 diff --git a/sdks/python/container/py39/base_image_requirements.txt b/sdks/python/container/py39/base_image_requirements.txt index 01bc279860775..ff6ba0945e14c 100644 --- a/sdks/python/container/py39/base_image_requirements.txt +++ b/sdks/python/container/py39/base_image_requirements.txt @@ -21,138 +21,120 @@ # https://s.apache.org/beam-python-dev-wiki # Reach out to a committer if you need help. -absl-py==1.4.0 -astunparse==1.6.3 attrs==23.1.0 beautifulsoup4==4.12.2 bs4==0.0.1 -cachetools==5.3.0 -certifi==2023.5.7 +cachetools==5.3.1 +certifi==2023.7.22 cffi==1.15.1 -charset-normalizer==3.1.0 -click==8.1.3 +charset-normalizer==3.2.0 +click==8.1.7 cloudpickle==2.2.1 crcmod==1.7 -cryptography==40.0.2 -Cython==0.29.34 +cryptography==41.0.4 +Cython==0.29.36 deprecation==2.1.0 dill==0.3.1.1 -dnspython==2.3.0 -docker==6.1.2 +dnspython==2.4.2 +docker==6.1.3 docopt==0.6.2 -exceptiongroup==1.1.1 -execnet==1.9.0 -fastavro==1.7.4 -fasteners==0.18 -flatbuffers==23.5.9 +exceptiongroup==1.1.3 +execnet==2.0.2 +fastavro==1.8.3 +fasteners==0.19 freezegun==1.2.2 future==0.18.3 -gast==0.4.0 -google-api-core==2.11.0 -google-api-python-client==2.86.0 +google-api-core==2.11.1 +google-api-python-client==2.100.0 google-apitools==0.5.31 -google-auth==2.18.0 -google-auth-httplib2==0.1.0 -google-auth-oauthlib==1.0.0 -google-cloud-bigquery==3.10.0 -google-cloud-bigquery-storage==2.19.1 -google-cloud-bigtable==2.17.0 -google-cloud-core==2.3.2 -google-cloud-datastore==2.15.2 -google-cloud-dlp==3.12.1 -google-cloud-language==2.9.1 -google-cloud-profiler==4.0.0 -google-cloud-pubsub==2.17.0 -google-cloud-pubsublite==1.8.2 -google-cloud-recommendations-ai==0.10.3 -google-cloud-spanner==3.34.0 -google-cloud-videointelligence==2.11.1 -google-cloud-vision==3.4.1 +google-auth==2.23.0 +google-auth-httplib2==0.1.1 +google-cloud-aiplatform==1.33.1 +google-cloud-bigquery==3.11.4 +google-cloud-bigquery-storage==2.22.0 +google-cloud-bigtable==2.21.0 +google-cloud-core==2.3.3 +google-cloud-datastore==2.18.0 +google-cloud-dlp==3.12.3 +google-cloud-language==2.11.1 +google-cloud-profiler==4.1.0 +google-cloud-pubsub==2.18.4 +google-cloud-pubsublite==1.8.3 +google-cloud-recommendations-ai==0.10.5 +google-cloud-resource-manager==1.10.4 +google-cloud-spanner==3.40.1 +google-cloud-storage==2.11.0 +google-cloud-videointelligence==2.11.4 +google-cloud-vision==3.4.4 google-crc32c==1.5.0 -google-pasta==0.2.0 -google-resumable-media==2.5.0 -googleapis-common-protos==1.59.0 +google-resumable-media==2.6.0 +googleapis-common-protos==1.60.0 greenlet==2.0.2 grpc-google-iam-v1==0.12.6 -grpcio==1.54.2 -grpcio-status==1.54.2 +grpcio==1.58.0 +grpcio-status==1.58.0 guppy3==3.1.3 -h5py==3.8.0 -hdfs==2.7.0 +hdfs==2.7.2 httplib2==0.22.0 -hypothesis==6.75.3 +hypothesis==6.87.0 idna==3.4 -importlib-metadata==6.6.0 iniconfig==2.0.0 -jax==0.4.10 -joblib==1.2.0 -keras==2.12.0 -libclang==16.0.0 -Markdown==3.4.3 -MarkupSafe==2.1.2 -ml-dtypes==0.1.0 -mmh3==3.1.0 -mock==5.0.2 +joblib==1.3.2 +Js2Py==0.74 +mmh3==4.0.1 +mock==5.1.0 nltk==3.8.1 nose==1.3.7 -numpy==1.23.5 +numpy==1.24.4 oauth2client==4.1.3 -oauthlib==3.2.2 objsize==0.6.1 -opt-einsum==3.3.0 -orjson==3.8.12 +orjson==3.9.7 overrides==6.5.0 packaging==23.1 pandas==1.5.3 parameterized==0.9.0 -pluggy==1.0.0 -proto-plus==1.22.2 -protobuf==4.23.0 -psycopg2-binary==2.9.6 +pluggy==1.3.0 +proto-plus==1.22.3 +protobuf==4.24.3 +psycopg2-binary==2.9.7 pyarrow==11.0.0 pyasn1==0.5.0 pyasn1-modules==0.3.0 pycparser==2.21 pydot==1.4.2 PyHamcrest==2.0.4 -pymongo==4.3.3 -PyMySQL==1.0.3 -pyparsing==3.0.9 -pytest==7.3.1 +pyjsparser==2.7.1 +pymongo==4.5.0 +PyMySQL==1.1.0 +pyparsing==3.1.1 +pytest==7.4.2 pytest-timeout==2.1.0 -pytest-xdist==3.3.0 +pytest-xdist==3.3.1 python-dateutil==2.8.2 python-snappy==0.6.1 -pytz==2023.3 -PyYAML==6.0 -regex==2023.5.5 -requests==2.30.0 -requests-mock==1.10.0 -requests-oauthlib==1.3.1 +pytz==2023.3.post1 +PyYAML==6.0.1 +regex==2023.8.8 +requests==2.31.0 +requests-mock==1.11.0 rsa==4.9 -scikit-learn==1.2.2 -scipy==1.10.1 +scikit-learn==1.3.1 +scipy==1.11.2 +Shapely==1.8.5.post1 six==1.16.0 sortedcontainers==2.4.0 -soupsieve==2.4.1 -SQLAlchemy==1.4.48 +soupsieve==2.5 +SQLAlchemy==1.4.49 sqlparse==0.4.4 -tenacity==8.2.2 -tensorboard==2.12.3 -tensorboard-data-server==0.7.0 -tensorflow==2.12.0 -tensorflow-estimator==2.12.0 -tensorflow-io-gcs-filesystem==0.32.0 -termcolor==2.3.0 +tenacity==8.2.3 testcontainers==3.7.1 -threadpoolctl==3.1.0 +threadpoolctl==3.2.0 tomli==2.0.1 -tqdm==4.65.0 -typing_extensions==4.5.0 +tqdm==4.66.1 +typing_extensions==4.8.0 +tzlocal==5.0.1 uritemplate==4.1.1 -urllib3==1.26.15 -websocket-client==1.5.1 -Werkzeug==2.3.4 -wrapt==1.14.1 -zipp==3.15.0 +urllib3==1.26.16 +websocket-client==1.6.3 +wrapt==1.15.0 zstandard==0.21.0 diff --git a/sdks/python/container/run_generate_requirements.sh b/sdks/python/container/run_generate_requirements.sh index fd222107dba5a..6c160bc6ac9e8 100755 --- a/sdks/python/container/run_generate_requirements.sh +++ b/sdks/python/container/run_generate_requirements.sh @@ -104,4 +104,13 @@ EOT # Remove pkg_resources to guard against # https://stackoverflow.com/questions/39577984/what-is-pkg-resources-0-0-0-in-output-of-pip-freeze-command pip freeze | grep -v pkg_resources >> "$REQUIREMENTS_FILE" + +if grep -q "tensorflow==" "$REQUIREMENTS_FILE"; then + # Get the version of tensorflow from the .txt file. + TF_VERSION=$(grep -Po "tensorflow==\K[^;]+" "$REQUIREMENTS_FILE") + TF_ENTRY="tensorflow==${TF_VERSION}" + TF_AARCH64_ENTRY="tensorflow-cpu-aws==${TF_VERSION};platform_machine==\"aarch64\"" + sed -i "s/${TF_ENTRY}/${TF_ENTRY}\n${TF_AARCH64_ENTRY}/g" $REQUIREMENTS_FILE +fi + rm -rf "$ENV_PATH" diff --git a/sdks/python/container/run_validatescontainer.sh b/sdks/python/container/run_validatescontainer.sh index 1dd17c59a647a..5ee3342a1efae 100755 --- a/sdks/python/container/run_validatescontainer.sh +++ b/sdks/python/container/run_validatescontainer.sh @@ -24,19 +24,22 @@ # REGION -> Region name to use for Dataflow # # Execute from the root of the repository: -# test Python3.7 container: -# ./gradlew :sdks:python:test-suites:dataflow:py37:validatesContainer -# test Python3.8 container: +# test Python3.8 x86 container: # ./gradlew :sdks:python:test-suites:dataflow:py38:validatesContainer -# or test all supported python versions together: +# or test all supported python versions x86 containers together: # ./gradlew :sdks:python:test-suites:dataflow:validatesContainer +# +# Note: ARM test suites only run on github actions. For example, to test Python3.8 ARM containers, +# commenting `Run Python ValidatesContainer Dataflow ARM (3.8)` will trigger the test. echo "This script must be executed in the root of beam project. Please set GCS_LOCATION, PROJECT and REGION as desired." -if [[ $# != 2 ]]; then - printf "Usage: \n$> ./sdks/python/container/run_validatescontainer.sh " +if [[ $# < 2 ]]; then + printf "Usage: \n$> ./sdks/python/container/run_validatescontainer.sh " printf "\n\tpython_version: [required] Python version used for container build and run tests." printf " Sample value: 3.9" + printf "\n\tcpu_architecture: [optional] CPU architecture used for container build and run tests, default as x86." + printf " Sample value: ARM or x86" exit 1 fi @@ -52,9 +55,11 @@ REGION=${REGION:-us-central1} IMAGE_PREFIX="$(grep 'docker_image_default_repo_prefix' gradle.properties | cut -d'=' -f2)" SDK_VERSION="$(grep 'sdk_version' gradle.properties | cut -d'=' -f2)" PY_VERSION=$1 +ARCH=${3:-"x86"} IMAGE_NAME="${IMAGE_PREFIX}python${PY_VERSION}_sdk" CONTAINER_PROJECT="sdks:python:container:py${PY_VERSION//.}" # Note: we substitute away the dot in the version. PY_INTERPRETER="python${PY_VERSION}" +MACHINE_TYPE_ARGS="" XUNIT_FILE="pytest-$IMAGE_NAME.xml" @@ -67,27 +72,42 @@ command -v gcloud docker -v gcloud -v -# Verify docker image has been built. -docker images | grep "apache/$IMAGE_NAME" | grep "$SDK_VERSION" - TAG=$(date +%Y%m%d-%H%M%S%N) CONTAINER=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME PREBUILD_SDK_CONTAINER_REGISTRY_PATH=us.gcr.io/$PROJECT/$USER/prebuild_python${PY_VERSION//.}_sdk echo "Using container $CONTAINER" - -# Tag the docker container. -docker tag "apache/$IMAGE_NAME:$SDK_VERSION" "$CONTAINER:$TAG" - -# Push the container. -gcloud docker -- push $CONTAINER:$TAG +echo "Using CPU architecture $ARCH" + +if [[ "$ARCH" == "x86" ]]; then + # Verify docker image has been built. + docker images | grep "apache/$IMAGE_NAME" | grep "$SDK_VERSION" + + # Tag the docker container. + docker tag "apache/$IMAGE_NAME:$SDK_VERSION" "$CONTAINER:$TAG" + + # Push the container + gcloud docker -- push $CONTAINER:$TAG +elif [[ "$ARCH" == "ARM" ]]; then + # Reset the multi-arch Python SDK container image tag. + TAG=$MULTIARCH_TAG + MACHINE_TYPE_ARGS="--machine_type=t2a-standard-1" +else + printf "Please give a valid CPU architecture, either x86 or ARM." + exit 1 +fi function cleanup_container { # Delete the container locally and remotely - docker rmi $CONTAINER:$TAG || echo "Failed to remove container image" + docker rmi $CONTAINER:$TAG || echo "Built container image was not removed. Possibly, it was not not saved locally." for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep $PREBUILD_SDK_CONTAINER_REGISTRY_PATH) do docker rmi $image || echo "Failed to remove prebuilt sdk container image" done - gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container" + # Note: we don't delete the multi-arch containers here because this command only deletes the manifest list with the tag, + # the associated container images can't be deleted because they are not tagged. However, multi-arch containers that are + # older than 6 weeks old are deleted by stale_dataflow_prebuilt_image_cleaner.sh that runs daily. + if [[ "$ARCH" == "x86" ]]; then + gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container" + fi for digest in $(gcloud container images list-tags $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk --format="get(digest)") do gcloud container images delete $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest --force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container image" done @@ -101,11 +121,9 @@ echo ">>> Successfully built and push container $CONTAINER" cd sdks/python SDK_LOCATION=$2 -# Run ValidatesRunner tests on Google Cloud Dataflow service echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST" -pytest -o junit_suite_name=$IMAGE_NAME \ - -m="it_validatescontainer" \ - --show-capture=no \ +pytest -o log_cli=True -o log_level=Info -o junit_suite_name=$IMAGE_NAME \ + -m=it_validatescontainer \ --numprocesses=1 \ --timeout=1800 \ --junitxml=$XUNIT_FILE \ @@ -121,6 +139,7 @@ pytest -o junit_suite_name=$IMAGE_NAME \ --output=$GCS_LOCATION/output \ --sdk_location=$SDK_LOCATION \ --num_workers=1 \ + $MACHINE_TYPE_ARGS \ --docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH" echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST" diff --git a/sdks/python/expansion-service-container/Dockerfile b/sdks/python/expansion-service-container/Dockerfile index f8fe378a71ed1..c6a33cdae88e7 100644 --- a/sdks/python/expansion-service-container/Dockerfile +++ b/sdks/python/expansion-service-container/Dockerfile @@ -42,8 +42,7 @@ RUN mkdir /opt/apache/beam/beam_venv &&\ . /opt/apache/beam/beam_venv/bin/activate &&\ pip install --upgrade pip &&\ pip install --upgrade setuptools &&\ - pip install -r requirements.txt &&\ - pip install apache-beam-sdk.tar.gz[gcp,dataframe] + pip install apache-beam-sdk.tar.gz[gcp,dataframe] --constraint requirements.txt ENTRYPOINT ["/opt/apache/beam/boot"] diff --git a/sdks/python/expansion-service-container/boot.go b/sdks/python/expansion-service-container/boot.go index 171e8ef62a377..ba56b349c4eab 100644 --- a/sdks/python/expansion-service-container/boot.go +++ b/sdks/python/expansion-service-container/boot.go @@ -18,28 +18,30 @@ package main import ( + "bufio" "flag" "fmt" + "io/ioutil" "log" "os" "path/filepath" "strconv" "strings" + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/xlangx/expansionx" "github.com/apache/beam/sdks/v2/go/pkg/beam/util/execx" ) var ( - id = flag.String("id", "", "Local identifier (required)") - port = flag.Int("port", 0, "Port for the expansion service (required)") + id = flag.String("id", "", "Local identifier (required)") + port = flag.Int("port", 0, "Port for the expansion service (required)") + requirements_file = flag.String("requirements_file", "", "A requirement file with extra packages to be made available to the transforms being expanded. Path should be relative to the 'dependencies_dir'") + dependencies_dir = flag.String("dependencies_dir", "", "A directory that stores locally available extra packages.") ) const ( expansionServiceEntrypoint = "apache_beam.runners.portability.expansion_service_main" venvDirectory = "beam_venv" // This should match the venv directory name used in the Dockerfile. - requirementsFile = "requirements.txt" - beamSDKArtifact = "apache-beam-sdk.tar.gz" - beamSDKOptions = "[gcp,dataframe]" ) func main() { @@ -57,7 +59,84 @@ func main() { } } +func getLines(fileNameToRead string) ([]string, error) { + fileToRead, err := os.Open(fileNameToRead) + if err != nil { + return nil, err + } + defer fileToRead.Close() + + sc := bufio.NewScanner(fileToRead) + lines := make([]string, 0) + + // Read through 'tokens' until an EOF is encountered. + for sc.Scan() { + lines = append(lines, sc.Text()) + } + + if err := sc.Err(); err != nil { + return nil, err + } + return lines, nil +} + +func installExtraPackages(requirementsFile string) error { + extraPackages, err := getLines(requirementsFile) + if err != nil { + return err + } + + for _, extraPackage := range extraPackages { + log.Printf("Installing extra package %v", extraPackage) + // We expect 'pip' command in virtual env to be already available at the top of the PATH. + args := []string{"install", extraPackage} + if err := execx.Execute("pip", args...); err != nil { + return fmt.Errorf("Could not install the package %s: %s", extraPackage, err) + } + } + return nil +} + +func getUpdatedRequirementsFile(oldRequirementsFileName string, dependenciesDir string) (string, error) { + oldExtraPackages, err := getLines(filepath.Join(dependenciesDir, oldRequirementsFileName)) + if err != nil { + return "", err + } + var updatedExtraPackages = make([]string, 0) + for _, extraPackage := range oldExtraPackages { + // TODO update + potentialLocalFilePath := filepath.Join(dependenciesDir, extraPackage) + _, err := os.Stat(potentialLocalFilePath) + if err == nil { + // Package exists locally so using that. + extraPackage = potentialLocalFilePath + log.Printf("Using locally available extra package %v", extraPackage) + } + updatedExtraPackages = append(updatedExtraPackages, extraPackage) + } + + updatedRequirementsFile, err := ioutil.TempFile("/opt/apache/beam", "requirements*.txt") + if err != nil { + return "", err + } + + updatedRequirementsFileName := updatedRequirementsFile.Name() + + datawriter := bufio.NewWriter(updatedRequirementsFile) + for _, extraPackage := range updatedExtraPackages { + _, _ = datawriter.WriteString(extraPackage + "\n") + } + datawriter.Flush() + updatedRequirementsFile.Close() + + return updatedRequirementsFileName, nil +} + func launchExpansionServiceProcess() error { + pythonVersion, err := expansionx.GetPythonVersion() + if err != nil { + return err + } log.Printf("Starting Python expansion service ...") dir := filepath.Join("/opt/apache/beam", venvDirectory) @@ -65,8 +144,26 @@ func launchExpansionServiceProcess() error { os.Setenv("PATH", strings.Join([]string{filepath.Join(dir, "bin"), os.Getenv("PATH")}, ":")) args := []string{"-m", expansionServiceEntrypoint, "-p", strconv.Itoa(*port), "--fully_qualified_name_glob", "*"} - if err := execx.Execute("python", args...); err != nil { - return fmt.Errorf("Could not start the expansion service: %s", err) + + if *requirements_file != "" { + log.Printf("Received the requirements file %v", *requirements_file) + updatedRequirementsFileName, err := getUpdatedRequirementsFile(*requirements_file, *dependencies_dir) + if err != nil { + return err + } + defer os.Remove(updatedRequirementsFileName) + log.Printf("Updated requirements file is %v", updatedRequirementsFileName) + // Provide the requirements file to the expansion service so that packages get staged by runners. + args = append(args, "--requirements_file", updatedRequirementsFileName) + // Install packages locally so that they can be used by the expansion service during transform + // expansion if needed. + err = installExtraPackages(updatedRequirementsFileName) + if err != nil { + return err + } + } + if err := execx.Execute(pythonVersion, args...); err != nil { + return fmt.Errorf("could not start the expansion service: %s", err) } return nil diff --git a/sdks/python/expansion-service-container/build.gradle b/sdks/python/expansion-service-container/build.gradle index 17c8733da73a9..ea11f400d3804 100644 --- a/sdks/python/expansion-service-container/build.gradle +++ b/sdks/python/expansion-service-container/build.gradle @@ -57,6 +57,9 @@ def copyLicenseScripts = tasks.register("copyLicenseScripts", Copy){ into "build/target/license_scripts" } +def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] +def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers") + docker { name containerImageName( name: project.docker_image_default_repo_prefix + "python_expansion_service", @@ -68,8 +71,10 @@ docker { // tags used by dockerTag task tags containerImageTags() files "./build" - buildx project.containerPlatforms() != [project.nativeArchitecture()] + buildx useBuildx platform(*project.containerPlatforms()) + load useBuildx && !pushContainers + push pushContainers } dockerPrepare.dependsOn goBuild @@ -94,5 +99,5 @@ if (project.rootProject.hasProperty(["docker-pull-licenses"])) { } task pushAll { - dependsOn dockerPush + dependsOn docker } diff --git a/sdks/python/gen_protos.py b/sdks/python/gen_protos.py index 94d80c8d263b9..2b488af0afb5d 100644 --- a/sdks/python/gen_protos.py +++ b/sdks/python/gen_protos.py @@ -18,7 +18,7 @@ """ Generates Python proto modules and grpc stubs for Beam protos. """ - +import argparse import contextlib import glob import inspect @@ -27,9 +27,7 @@ import platform import re import shutil -import subprocess import sys -import time from collections import defaultdict from importlib import import_module @@ -60,7 +58,7 @@ NO_PROMISES_NOTICE = """ \"\"\" For internal use only; no backwards-compatibility guarantees. -Automatically generated when running setup.py sdist or build[_py]. +Automatically generated when running python -m build. \"\"\" """ @@ -321,43 +319,6 @@ def find_by_ext(root_dir, ext): if file.endswith(ext): yield clean_path(os.path.join(root, file)) - -def ensure_grpcio_exists(): - try: - from grpc_tools import protoc # pylint: disable=unused-import - except ImportError: - return _install_grpcio_tools() - - -def _install_grpcio_tools(): - """ - Though wheels are available for grpcio-tools, setup_requires uses - easy_install which doesn't understand them. This means that it is - compiled from scratch (which is expensive as it compiles the full - protoc compiler). Instead, we attempt to install a wheel in a temporary - directory and add it to the path as needed. - See https://github.com/pypa/setuptools/issues/377 - """ - install_path = os.path.join(PYTHON_SDK_ROOT, '.eggs', 'grpcio-wheels') - logging.warning('Installing grpcio-tools into %s', install_path) - start = time.time() - subprocess.check_call([ - sys.executable, - '-m', - 'pip', - 'install', - '--target', - install_path, - '--upgrade', - '-r', - os.path.join(PYTHON_SDK_ROOT, 'build-requirements.txt') - ]) - logging.warning( - 'Installing grpcio-tools took %0.2f seconds.', time.time() - start) - - return install_path - - def build_relative_import(root_path, import_path, start_file_path): tail_path = import_path.replace('.', os.path.sep) source_path = os.path.join(root_path, tail_path) @@ -511,33 +472,31 @@ def generate_proto_files(force=False): if not os.path.exists(PYTHON_OUTPUT_PATH): os.mkdir(PYTHON_OUTPUT_PATH) - grpcio_install_loc = ensure_grpcio_exists() protoc_gen_mypy = _find_protoc_gen_mypy() - with PythonPath(grpcio_install_loc): - from grpc_tools import protoc - builtin_protos = pkg_resources.resource_filename('grpc_tools', '_proto') - args = ( - [sys.executable] + # expecting to be called from command line - ['--proto_path=%s' % builtin_protos] + - ['--proto_path=%s' % d - for d in proto_dirs] + ['--python_out=%s' % PYTHON_OUTPUT_PATH] + - ['--plugin=protoc-gen-mypy=%s' % protoc_gen_mypy] + - # new version of mypy-protobuf converts None to zero default value - # and remove Optional from the param type annotation. This causes - # some mypy errors. So to mitigate and fall back to old behavior, - # use `relax_strict_optional_primitives` flag. more at - # https://github.com/nipunn1313/mypy-protobuf/tree/main#relax_strict_optional_primitives # pylint:disable=line-too-long - ['--mypy_out=relax_strict_optional_primitives:%s' % PYTHON_OUTPUT_PATH - ] + - # TODO(robertwb): Remove the prefix once it's the default. - ['--grpc_python_out=grpc_2_0:%s' % PYTHON_OUTPUT_PATH] + proto_files) - - LOG.info('Regenerating Python proto definitions (%s).' % regenerate_reason) - ret_code = protoc.main(args) - if ret_code: - raise RuntimeError( - 'Protoc returned non-zero status (see logs for details): ' - '%s' % ret_code) + from grpc_tools import protoc + builtin_protos = pkg_resources.resource_filename('grpc_tools', '_proto') + args = ( + [sys.executable] + # expecting to be called from command line + ['--proto_path=%s' % builtin_protos] + + ['--proto_path=%s' % d + for d in proto_dirs] + ['--python_out=%s' % PYTHON_OUTPUT_PATH] + + ['--plugin=protoc-gen-mypy=%s' % protoc_gen_mypy] + + # new version of mypy-protobuf converts None to zero default value + # and remove Optional from the param type annotation. This causes + # some mypy errors. So to mitigate and fall back to old behavior, + # use `relax_strict_optional_primitives` flag. more at + # https://github.com/nipunn1313/mypy-protobuf/tree/main#relax_strict_optional_primitives # pylint:disable=line-too-long + ['--mypy_out=relax_strict_optional_primitives:%s' % PYTHON_OUTPUT_PATH + ] + + # TODO(robertwb): Remove the prefix once it's the default. + ['--grpc_python_out=grpc_2_0:%s' % PYTHON_OUTPUT_PATH] + proto_files) + + LOG.info('Regenerating Python proto definitions (%s).' % regenerate_reason) + ret_code = protoc.main(args) + if ret_code: + raise RuntimeError( + 'Protoc returned non-zero status (see logs for details): ' + '%s' % ret_code) # copy resource files for path in MODEL_RESOURCES: @@ -548,7 +507,7 @@ def generate_proto_files(force=False): # force relative import paths for proto files compiled_import_re = re.compile('^from (.*) import (.*)$') for file_path in find_by_ext(PYTHON_OUTPUT_PATH, - ('_pb2.py', '_pb2_grpc.py', '_pb2.pyi')): + ('_pb2.py', '_pb2_grpc.py', '_pb2.pyi')): proto_packages.add(os.path.dirname(file_path)) lines = [] with open(file_path, encoding='utf-8') as f: @@ -566,12 +525,14 @@ def generate_proto_files(force=False): f.writelines(lines) generate_init_files_lite(PYTHON_OUTPUT_PATH) - with PythonPath(grpcio_install_loc): - for proto_package in proto_packages: - generate_urn_files(proto_package, PYTHON_OUTPUT_PATH) + for proto_package in proto_packages: + generate_urn_files(proto_package, PYTHON_OUTPUT_PATH) generate_init_files_full(PYTHON_OUTPUT_PATH) if __name__ == '__main__': - generate_proto_files(force=True) + parser = argparse.ArgumentParser() + parser.add_argument('--no-force', dest='force', action='store_false') + args = parser.parse_args() + generate_proto_files(force=args.force) diff --git a/sdks/python/mypy.ini b/sdks/python/mypy.ini index a628036d6682f..46dea481f9314 100644 --- a/sdks/python/mypy.ini +++ b/sdks/python/mypy.ini @@ -16,7 +16,7 @@ # [mypy] -python_version = 3.7 +python_version = 3.8 ignore_missing_imports = true follow_imports = normal warn_no_return = true diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml new file mode 100644 index 0000000000000..d185c45f61910 --- /dev/null +++ b/sdks/python/pyproject.toml @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# since we rely on setuptools and according to https://peps.python.org/pep-0518/#build-system-table +# this is the minimum requirements for the build system to execute. +[build-system] +requires = [ + "setuptools", + "wheel>=0.36.0", + "grpcio-tools==1.53.0", + "mypy-protobuf==3.5.0", + # Avoid https://github.com/pypa/virtualenv/issues/2006 + "distlib==0.3.7", + # Numpy headers + "numpy>=1.14.3,<1.25", # Update setup.py as well. + # having cython here will create wheels that are platform dependent. + "cython==0.29.36", +] + + +# legacy installation is needed to generate `apache_beam.portability.api` package. +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/sdks/python/pytest.ini b/sdks/python/pytest.ini index 6e93c5f96e7fa..140476b29e50f 100644 --- a/sdks/python/pytest.ini +++ b/sdks/python/pytest.ini @@ -17,6 +17,9 @@ [pytest] junit_family = xunit2 +filterwarnings = + ignore:Deprecated call to `pkg_resources.declare_namespace\('.*'\):DeprecationWarning + ignore::DeprecationWarning:google.rpc # Disable class-name-based test discovery. python_classes = @@ -30,6 +33,8 @@ markers = uses_gcp_java_expansion_service: collect Cross Language GCP Java transforms test runs uses_java_expansion_service: collect Cross Language Java transforms test runs uses_python_expansion_service: collect Cross Language Python transforms test runs + uses_io_expansion_service: collect Cross Language transform test runs (with Kafka bootstrap server) + uses_transform_service: collect Cross Language test runs that uses the Transform Service xlang_sql_expansion_service: collect for Cross Language with SQL expansion service test runs it_postcommit: collect for post-commit integration test runs it_postcommit_sickbay: collect for post-commit sickbay integration test run @@ -38,6 +43,7 @@ markers = no_sickbay_streaming: run without sickbay-streaming no_sickbay_batch: run without sickbay-batch it_validatescontainer: collect for ValidatesContainer integration test runs + it_dataflow_arm: collect for DataflowArm integration test runs examples_postcommit: collect for post-commit test examples runs sickbay_direct: run without sickbay-direct sickbay_spark: run without sickbay-spark @@ -47,15 +53,17 @@ markers = # as enabling save_main_session. no_xdist: run without pytest-xdist plugin # We run these tests with multiple major pyarrow versions (BEAM-11211) - uses_pyarrow: tests that utilize pyarrow in some way + uses_pyarrow: tests that utilize pyarrow in some way. # ML tests - uses_pytorch: tests that utilize pytorch in some way - uses_sklearn: tests that utilize scikit-learn in some way - uses_tensorflow: tests that utilize tensorflow in some way + uses_pytorch: tests that utilize pytorch in some way. + uses_sklearn: tests that utilize scikit-learn in some way. + uses_tensorflow: tests that utilize tensorflow in some way. uses_tft: tests that utilizes tensorflow transforms in some way. - uses_xgboost: tests that utilize xgboost in some way + uses_xgboost: tests that utilize xgboost in some way. uses_onnx: tests that utilizes onnx in some way. - uses_tf: tests that utilize tensorflow + uses_tf: tests that utilize tensorflow. + uses_transformers: tests that utilize transformers in some way. + vertex_ai_postcommit: vertex ai postcommits that need additional deps. # Default timeout intended for unit tests. # If certain tests need a different value, please see the docs on how to diff --git a/sdks/python/scripts/OWNERS b/sdks/python/scripts/OWNERS deleted file mode 100644 index 3a2b3498379e3..0000000000000 --- a/sdks/python/scripts/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - aaltay diff --git a/sdks/python/scripts/generate_pydoc.sh b/sdks/python/scripts/generate_pydoc.sh index 11bb131e88b73..06ad06320fcf4 100755 --- a/sdks/python/scripts/generate_pydoc.sh +++ b/sdks/python/scripts/generate_pydoc.sh @@ -131,7 +131,9 @@ release = version autoclass_content = 'both' autodoc_inherit_docstrings = False autodoc_member_order = 'bysource' -autodoc_mock_imports = ["tensorrt", "cuda", "torch", "onnxruntime", "onnx", "tensorflow", "tensorflow_hub"] +autodoc_mock_imports = ["tensorrt", "cuda", "torch", + "onnxruntime", "onnx", "tensorflow", "tensorflow_hub", + "tensorflow_transform", "tensorflow_metadata", "transformers"] # Allow a special section for documenting DataFrame API napoleon_custom_sections = ['Differences from pandas'] diff --git a/sdks/python/scripts/run_integration_test.sh b/sdks/python/scripts/run_integration_test.sh index 508d9f50421ef..5ac3627a09608 100755 --- a/sdks/python/scripts/run_integration_test.sh +++ b/sdks/python/scripts/run_integration_test.sh @@ -78,6 +78,8 @@ KMS_KEY_NAME="projects/apache-beam-testing/locations/global/keyRings/beam-it/cry SUITE="" COLLECT_MARKERS= REQUIREMENTS_FILE="" +ARCH="" +PY_VERSION="" # Default test (pytest) options. # Run WordCountIT.test_wordcount_it by default if no test options are @@ -133,16 +135,6 @@ case $key in shift # past argument shift # past value ;; - --runner_v2) - RUNNER_V2="$2" - shift # past argument - shift # past value - ;; - --disable_runner_v2) - DISABLE_RUNNER_V2="$2" - shift # past argument - shift # past value - ;; --kms_key_name) KMS_KEY_NAME="$2" shift # past argument @@ -173,6 +165,16 @@ case $key in shift # past argument shift # past value ;; + --arch) + ARCH="$2" + shift # past argument + shift # past value + ;; + --py_version) + PY_VERSION="$2" + shift # past argument + shift # past value + ;; *) # unknown option echo "Unknown option: $1" exit 1 @@ -244,21 +246,11 @@ if [[ -z $PIPELINE_OPTS ]]; then opts+=("--streaming") fi - # Add --runner_v2 if provided - if [[ "$RUNNER_V2" = true ]]; then - opts+=("--experiments=use_runner_v2") - if [[ "$STREAMING" = true ]]; then - # Dataflow Runner V2 only supports streaming engine. - opts+=("--enable_streaming_engine") - else - opts+=("--experiments=beam_fn_api") - fi - - fi + if [[ "$ARCH" == "ARM" ]]; then + opts+=("--machine_type=t2a-standard-1") - # Add --disable_runner_v2 if provided - if [[ "$DISABLE_RUNNER_V2" = true ]]; then - opts+=("--experiments=disable_runner_v2") + IMAGE_NAME="beam_python${PY_VERSION}_sdk" + opts+=("--sdk_container_image=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME:$MULTIARCH_TAG") fi if [[ ! -z "$KMS_KEY_NAME" ]]; then diff --git a/sdks/python/scripts/run_pytest.sh b/sdks/python/scripts/run_pytest.sh index 01f2318164c4f..ad35b48972b6b 100755 --- a/sdks/python/scripts/run_pytest.sh +++ b/sdks/python/scripts/run_pytest.sh @@ -42,10 +42,10 @@ echo "posargs: $posargs" # Run with pytest-xdist and without. pytest -o junit_suite_name=${envname} \ - --junitxml=pytest_${envname}.xml -m 'not no_xdist' -n 6 ${pytest_args} --pyargs ${posargs} + --junitxml=pytest_${envname}.xml -m 'not no_xdist' -n 6 --import-mode=importlib ${pytest_args} --pyargs ${posargs} status1=$? pytest -o junit_suite_name=${envname}_no_xdist \ - --junitxml=pytest_${envname}_no_xdist.xml -m 'no_xdist' ${pytest_args} --pyargs ${posargs} + --junitxml=pytest_${envname}_no_xdist.xml -m 'no_xdist' --import-mode=importlib ${pytest_args} --pyargs ${posargs} status2=$? # Exit with error if no tests were run in either suite (status code 5). diff --git a/sdks/python/scripts/run_snapshot_publish.sh b/sdks/python/scripts/run_snapshot_publish.sh index 5cdde16f53a22..6379e6f210843 100755 --- a/sdks/python/scripts/run_snapshot_publish.sh +++ b/sdks/python/scripts/run_snapshot_publish.sh @@ -18,7 +18,7 @@ BUCKET=gs://beam-python-nightly-snapshots -VERSION=$(awk '/__version__/{print $3}' $WORKSPACE/src/sdks/python/apache_beam/version.py) +VERSION=$(awk '/__version__/{print $3}' $WORKSPACE/sdks/python/apache_beam/version.py) VERSION=$(echo $VERSION | cut -c 2- | rev | cut -c 2- | rev) time=$(date +"%Y-%m-%dT%H:%M:%S") SNAPSHOT="apache-beam-$VERSION-$time.zip" @@ -28,7 +28,7 @@ DEP_SNAPSHOT_FILE_NAME="beam-py-requirements-$time.txt" # Snapshots are built by Gradle task :sdks:python:depSnapshot # and located under Gradle build directory. -cd $WORKSPACE/src/sdks/python/build +cd $WORKSPACE/sdks/python/build # Rename the file to be apache-beam-{VERSION}-{datetime}.zip for file in "apache-beam-$VERSION*.zip"; do diff --git a/sdks/python/scripts/run_tox.sh b/sdks/python/scripts/run_tox.sh index ebbacf5494eaf..ac60f26b32bad 100755 --- a/sdks/python/scripts/run_tox.sh +++ b/sdks/python/scripts/run_tox.sh @@ -53,12 +53,21 @@ if [[ "$JENKINS_HOME" != "" ]]; then export PY_COLORS=1 fi -if [[ ! -z $2 ]]; then +# Determine if the second argument is SDK_LOCATION or posargs +if [[ -f "$1" ]]; then # Check if the argument corresponds to a file SDK_LOCATION="$1" - shift; - tox -c tox.ini run --recreate -e "$TOX_ENVIRONMENT" --installpkg "$SDK_LOCATION" -- "$@" -else - tox -c tox.ini run --recreate -e "$TOX_ENVIRONMENT" + shift +fi + +# If SDK_LOCATION is identified and there are still arguments left, those are posargs. +if [[ ! -z "$SDK_LOCATION" ]]; then + if [[ $# -gt 0 ]]; then # There are posargs + tox -c tox.ini run --recreate -e "$TOX_ENVIRONMENT" --installpkg "$SDK_LOCATION" -- "$@" + else + tox -c tox.ini run --recreate -e "$TOX_ENVIRONMENT" --installpkg "$SDK_LOCATION" + fi +else # No SDK_LOCATION; all arguments are posargs + tox -c tox.ini run --recreate -e "$TOX_ENVIRONMENT" -- "$@" fi exit_code=$? diff --git a/sdks/python/scripts/run_transform_service.sh b/sdks/python/scripts/run_transform_service.sh new file mode 100755 index 0000000000000..b71d67707e06a --- /dev/null +++ b/sdks/python/scripts/run_transform_service.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +read -r -d '' USAGE <$TEMP_DIR/$FILE_BASE-java1.log 2>&1 $TEMP_DIR/$FILE_BASE-java2.log 2>&1 =1.4.3,!=1.5.0,!=1.5.1,<1.6;python_version>="3.8"', ] +def find_by_ext(root_dir, ext): + for root, _, files in os.walk(root_dir): + for file in files: + if file.endswith(ext): + yield os.path.realpath(os.path.join(root, file)) # We must generate protos after setup_requires are installed. def generate_protos_first(): try: - # pylint: disable=wrong-import-position - import gen_protos - gen_protos.generate_proto_files() - - except ImportError: - warnings.warn("Could not import gen_protos, skipping proto generation.") + # Pyproject toml build happens in isolated environemnts. In those envs, + # gen_protos is unable to get imported. so we run a subprocess call. + cwd = os.path.abspath(os.path.dirname(__file__)) + # when pip install <>.tar.gz gets called, if gen_protos.py is not available + # in the sdist,then the proto files would have already been generated. So we + # skip proto generation in that case. + if not os.path.exists(os.path.join(cwd, 'gen_protos.py')): + # make sure we already generated protos + pb2_files = list(find_by_ext(os.path.join( + cwd, 'apache_beam', 'portability', 'api'), '_pb2.py')) + if not pb2_files: + raise RuntimeError('protobuf files are not generated. ' + 'Please generate pb2 files') + + warnings.warn('Skipping proto generation as they are already generated.') + return + out = subprocess.run([ + sys.executable, + os.path.join(cwd, 'gen_protos.py'), + '--no-force' + ], capture_output=True, check=True) + print(out.stdout) + except subprocess.CalledProcessError as err: + raise RuntimeError('Could not generate protos due to error: %s', + err.stderr) def get_portability_package_data(): @@ -175,7 +199,7 @@ def get_portability_package_data(): return files -python_requires = '>=3.7' +python_requires = '>=3.8' if sys.version_info.major == 3 and sys.version_info.minor >= 12: warnings.warn( @@ -188,6 +212,27 @@ def get_portability_package_data(): # structure must exist before the call to setuptools.find_packages() # executes below. generate_protos_first() + + # generate cythonize extensions only if we are building a wheel or + # building an extension or running in editable mode. + cythonize_cmds = ('bdist_wheel', 'build_ext', 'editable_wheel') + if any(cmd in sys.argv for cmd in cythonize_cmds): + extensions = cythonize([ + 'apache_beam/**/*.pyx', + 'apache_beam/coders/coder_impl.py', + 'apache_beam/metrics/cells.py', + 'apache_beam/metrics/execution.py', + 'apache_beam/runners/common.py', + 'apache_beam/runners/worker/logger.py', + 'apache_beam/runners/worker/opcounters.py', + 'apache_beam/runners/worker/operations.py', + 'apache_beam/transforms/cy_combiners.py', + 'apache_beam/transforms/stats.py', + 'apache_beam/utils/counters.py', + 'apache_beam/utils/windowed_value.py', + ]) + else: + extensions = [] # Keep all dependencies inlined in the setup call, otherwise Dependabot won't # be able to parse it. setuptools.setup( @@ -213,61 +258,55 @@ def get_portability_package_data(): *get_portability_package_data() ] }, - ext_modules=cythonize([ - 'apache_beam/**/*.pyx', - 'apache_beam/coders/coder_impl.py', - 'apache_beam/metrics/cells.py', - 'apache_beam/metrics/execution.py', - 'apache_beam/runners/common.py', - 'apache_beam/runners/worker/logger.py', - 'apache_beam/runners/worker/opcounters.py', - 'apache_beam/runners/worker/operations.py', - 'apache_beam/transforms/cy_combiners.py', - 'apache_beam/transforms/stats.py', - 'apache_beam/utils/counters.py', - 'apache_beam/utils/windowed_value.py', - ], language_level=3), - install_requires = [ - 'crcmod>=1.7,<2.0', - 'orjson<4.0', - # Dill doesn't have forwards-compatibility guarantees within minor - # version. Pickles created with a new version of dill may not unpickle - # using older version of dill. It is best to use the same version of - # dill on client and server, therefore list of allowed versions is very - # narrow. See: https://github.com/uqfoundation/dill/issues/341. - 'dill>=0.3.1.1,<0.3.2', - # It is prudent to use the same version of pickler at job submission - # and at runtime, therefore bounds need to be tight. - # To avoid depending on an old dependency, update the minor version on - # every Beam release, see: https://github.com/apache/beam/issues/23119 - 'cloudpickle~=2.2.1', - 'fastavro>=0.23.6,<2', - 'fasteners>=0.3,<1.0', - 'grpcio>=1.33.1,!=1.48.0,<2', - 'hdfs>=2.1.0,<3.0.0', - 'httplib2>=0.8,<0.23.0', - # numpy can have breaking changes in minor versions. - # Use a strict upper bound. - 'numpy>=1.14.3,<1.25.0', # Update build-requirements.txt as well. - 'objsize>=0.6.1,<0.7.0', - 'pymongo>=3.8.0,<5.0.0', - 'proto-plus>=1.7.1,<2', - # use a tighter upper bound in protobuf dependency - # to make sure the minor version at job submission - # does not exceed the minor version at runtime. - # To avoid depending on an old dependency, update the minor version on - # every Beam release, see: https://github.com/apache/beam/issues/25590 - 'protobuf>=3.20.3,<4.24.0', - 'pydot>=1.2.0,<2', - 'python-dateutil>=2.8.0,<3', - 'pytz>=2018.3', - 'regex>=2020.6.8', - 'requests>=2.24.0,<3.0.0', - 'typing-extensions>=3.7.0', - 'zstandard>=0.18.0,<1', - # Dynamic dependencies must be specified in a separate list, otherwise - # Dependabot won't be able to parse the main list. Any dynamic - # dependencies will not receive updates from Dependabot. + ext_modules=extensions, + install_requires=[ + 'crcmod>=1.7,<2.0', + 'orjson>=3.9.7,<4', + # Dill doesn't have forwards-compatibility guarantees within minor + # version. Pickles created with a new version of dill may not unpickle + # using older version of dill. It is best to use the same version of + # dill on client and server, therefore list of allowed versions is + # very narrow. See: https://github.com/uqfoundation/dill/issues/341. + 'dill>=0.3.1.1,<0.3.2', + # It is prudent to use the same version of pickler at job submission + # and at runtime, therefore bounds need to be tight. + # To avoid depending on an old dependency, update the minor version on + # every Beam release, see: https://github.com/apache/beam/issues/23119 + 'cloudpickle~=2.2.1', + 'fastavro>=0.23.6,<2', + 'fasteners>=0.3,<1.0', + 'grpcio>=1.33.1,!=1.48.0,<2', + 'hdfs>=2.1.0,<3.0.0', + 'httplib2>=0.8,<0.23.0', + 'js2py>=0.74,<1', + # numpy can have breaking changes in minor versions. + # Use a strict upper bound. + 'numpy>=1.14.3,<1.25.0', # Update pyproject.toml as well. + 'objsize>=0.6.1,<0.7.0', + 'packaging>=22.0', + 'pymongo>=3.8.0,<5.0.0', + 'proto-plus>=1.7.1,<2', + # 1. Use a tighter upper bound in protobuf dependency to make sure + # the minor version at job submission + # does not exceed the minor version at runtime. + # To avoid depending on an old dependency, update the minor version on + # every Beam release, see: https://github.com/apache/beam/issues/25590 + + # 2. Allow latest protobuf 3 version as a courtesy to some customers. + # + # 3. Exclude protobuf 4 versions that leak memory, see: + # https://github.com/apache/beam/issues/28246 + 'protobuf>=3.20.3,<4.25.0,!=4.0.*,!=4.21.*,!=4.22.0,!=4.23.*,!=4.24.0,!=4.24.1,!=4.24.2', # pylint: disable=line-too-long + 'pydot>=1.2.0,<2', + 'python-dateutil>=2.8.0,<3', + 'pytz>=2018.3', + 'regex>=2020.6.8', + 'requests>=2.24.0,<3.0.0', + 'typing-extensions>=3.7.0', + 'zstandard>=0.18.0,<1', + # Dynamic dependencies must be specified in a separate list, otherwise + # Dependabot won't be able to parse the main list. Any dynamic + # dependencies will not receive updates from Dependabot. ] + [pyarrow_dependency], python_requires=python_requires, # BEAM-8840: Do NOT use tests_require or setup_requires. @@ -280,81 +319,83 @@ def get_portability_package_data(): 'pandas<2.0.0', ], 'test': [ - 'freezegun>=0.3.12', - 'joblib>=1.0.1', - 'mock>=1.0.1,<6.0.0', - 'pandas<2.0.0', - 'parameterized>=0.7.1,<0.10.0', - 'pyhamcrest>=1.9,!=1.10.0,<3.0.0', - 'pyyaml>=3.12,<7.0.0', - 'requests_mock>=1.7,<2.0', - 'tenacity>=8.0.0,<9', - 'pytest>=7.1.2,<8.0', - 'pytest-xdist>=2.5.0,<4', - 'pytest-timeout>=2.1.0,<3', - 'scikit-learn>=0.20.0', - 'sqlalchemy>=1.3,<2.0', - 'psycopg2-binary>=2.8.5,<3.0.0', - 'testcontainers[mysql]>=3.0.3,<4.0.0', - 'cryptography>=36.0.0', - 'hypothesis>5.0.0,<=7.0.0', + 'freezegun>=0.3.12', + 'joblib>=1.0.1', + 'mock>=1.0.1,<6.0.0', + 'pandas<2.0.0', + 'parameterized>=0.7.1,<0.10.0', + 'pyhamcrest>=1.9,!=1.10.0,<3.0.0', + 'pyyaml>=3.12,<7.0.0', + 'requests_mock>=1.7,<2.0', + 'tenacity>=8.0.0,<9', + 'pytest>=7.1.2,<8.0', + 'pytest-xdist>=2.5.0,<4', + 'pytest-timeout>=2.1.0,<3', + 'scikit-learn>=0.20.0', + 'sqlalchemy>=1.3,<2.0', + 'psycopg2-binary>=2.8.5,<3.0.0', + 'testcontainers[mysql]>=3.0.3,<4.0.0', + 'cryptography>=41.0.2', + 'hypothesis>5.0.0,<=7.0.0', ], 'gcp': [ - 'cachetools>=3.1.0,<6', - 'google-apitools>=0.5.31,<0.5.32', - # NOTE: Maintainers, please do not require google-auth>=2.x.x - # Until this issue is closed - # https://github.com/googleapis/google-cloud-python/issues/10566 - 'google-auth>=1.18.0,<3', - 'google-auth-httplib2>=0.1.0,<0.2.0', - 'google-cloud-datastore>=2.0.0,<3', - 'google-cloud-pubsub>=2.1.0,<3', - 'google-cloud-pubsublite>=1.2.0,<2', - # GCP packages required by tests - 'google-cloud-bigquery>=2.0.0,<4', - 'google-cloud-bigquery-storage>=2.6.3,<3', - 'google-cloud-core>=2.0.0,<3', - 'google-cloud-bigtable>=2.19.0,<3', - 'google-cloud-spanner>=3.0.0,<4', - # GCP Packages required by ML functionality - 'google-cloud-dlp>=3.0.0,<4', - 'google-cloud-language>=2.0,<3', - 'google-cloud-videointelligence>=2.0,<3', - 'google-cloud-vision>=2,<4', - 'google-cloud-recommendations-ai>=0.1.0,<0.11.0' + 'cachetools>=3.1.0,<6', + 'google-api-core>=2.0.0,<3', + 'google-apitools>=0.5.31,<0.5.32', + # NOTE: Maintainers, please do not require google-auth>=2.x.x + # Until this issue is closed + # https://github.com/googleapis/google-cloud-python/issues/10566 + 'google-auth>=1.18.0,<3', + 'google-auth-httplib2>=0.1.0,<0.2.0', + 'google-cloud-datastore>=2.0.0,<3', + 'google-cloud-pubsub>=2.1.0,<3', + 'google-cloud-pubsublite>=1.2.0,<2', + # GCP packages required by tests + 'google-cloud-bigquery>=2.0.0,<4', + 'google-cloud-bigquery-storage>=2.6.3,<3', + 'google-cloud-core>=2.0.0,<3', + 'google-cloud-bigtable>=2.19.0,<3', + 'google-cloud-spanner>=3.0.0,<4', + # GCP Packages required by ML functionality + 'google-cloud-dlp>=3.0.0,<4', + 'google-cloud-language>=2.0,<3', + 'google-cloud-videointelligence>=2.0,<3', + 'google-cloud-vision>=2,<4', + 'google-cloud-recommendations-ai>=0.1.0,<0.11.0', + 'google-cloud-aiplatform>=1.26.0, < 2.0' ], 'interactive': [ - 'facets-overview>=1.1.0,<2', - 'google-cloud-dataproc>=5.0.0,<6', - # IPython>=8 is not compatible with Python<=3.7 - 'ipython>=7,<8;python_version<="3.7"', - 'ipython>=8,<9;python_version>"3.7"', - 'ipykernel>=6,<7', - 'ipywidgets>=8,<9', - # Skip version 6.1.13 due to - # https://github.com/jupyter/jupyter_client/issues/637 - 'jupyter-client>=6.1.11,!=6.1.13,<8.2.1', - 'timeloop>=1.0.2,<2', - 'nbformat>=5.0.5,<6', - 'nbconvert>=6.2.0,<8', + 'facets-overview>=1.1.0,<2', + 'google-cloud-dataproc>=5.0.0,<6', + 'ipython>=8,<9', + 'ipykernel>=6,<7', + 'ipywidgets>=8,<9', + # Skip version 6.1.13 due to + # https://github.com/jupyter/jupyter_client/issues/637 + 'jupyter-client>=6.1.11,!=6.1.13,<8.2.1', + 'timeloop>=1.0.2,<2', + 'nbformat>=5.0.5,<6', + 'nbconvert>=6.2.0,<8', ] + dataframe_dependency, 'interactive_test': [ - # headless chrome based integration tests - 'needle>=0.5.0,<1', - 'chromedriver-binary>=100,<114', - # use a fixed major version of PIL for different python versions - 'pillow>=7.1.1,<10', + # headless chrome based integration tests + 'needle>=0.5.0,<1', + 'chromedriver-binary>=117,<118', + # use a fixed major version of PIL for different python versions + 'pillow>=7.1.1,<10', + # urllib 2.x is a breaking change for the headless chrome tests + 'urllib3<2,>=1.21.1' ], 'aws': ['boto3>=1.9,<2'], 'azure': [ - 'azure-storage-blob>=12.3.2,<13', - 'azure-core>=1.7.0,<2', - 'azure-identity>=1.12.0,<2', + 'azure-storage-blob>=12.3.2,<13', + 'azure-core>=1.7.0,<2', + 'azure-identity>=1.12.0,<2', ], 'dataframe': dataframe_dependency, 'dask': [ - 'dask >= 2022.6', - 'distributed >= 2022.6', + 'dask >= 2022.6', + 'distributed >= 2022.6', ], }, zip_safe=False, @@ -363,7 +404,6 @@ def get_portability_package_data(): 'Intended Audience :: End Users/Desktop', 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', diff --git a/sdks/python/test-suites/dataflow/build.gradle b/sdks/python/test-suites/dataflow/build.gradle index e0deba5c24413..b55716a42df29 100644 --- a/sdks/python/test-suites/dataflow/build.gradle +++ b/sdks/python/test-suites/dataflow/build.gradle @@ -30,13 +30,6 @@ task preCommitIT { } } -task preCommitIT_V2{ - getVersionsAsList('dataflow_precommit_it_task_py_versions').each { - dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:preCommitIT_batch_V2") - dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:preCommitIT_streaming_V2") - } -} - task mongodbioIT { getVersionsAsList('dataflow_mongodbio_it_task_py_versions').each { dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:mongodbioIT") @@ -61,18 +54,6 @@ task validatesRunnerStreamingTests { } } -task validatesRunnerBatchTestsV2 { - getVersionsAsList('dataflow_validates_runner_batch_tests_V2').each { - dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:validatesRunnerBatchTests") - } -} - -task validatesRunnerStreamingTestsV2 { - getVersionsAsList('dataflow_validates_runner_streaming_tests_V2').each { - dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:validatesRunnerStreamingTests") - } -} - task validatesContainerTests { getVersionsAsList('python_versions').each { dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:validatesContainer") @@ -91,6 +72,12 @@ task gcpCrossLanguagePostCommit { } } +task ioCrossLanguagePostCommit { + getVersionsAsList('cross_language_validates_gcp_py_versions').each { + dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:ioCrossLanguagePythonUsingJava") + } +} + task tftTests { getVersionsAsList('dataflow_cloudml_benchmark_tests_py_versions').each { dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:tftTests") diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle index 36fc75f2e4127..a713b82400e75 100644 --- a/sdks/python/test-suites/dataflow/common.gradle +++ b/sdks/python/test-suites/dataflow/common.gradle @@ -25,6 +25,15 @@ String pythonVersionSuffix = project.ext.pythonVersion ? "-py${pythonVersionNumber}" : '' +// Basic test options for ITs running on Jenkins. +def basicTestOpts = [ + "--capture=no", // print stdout instantly + "--numprocesses=8", // run tests in parallel + "--timeout=4500", // timeout of whole command execution + "--color=yes", // console color + "--log-cli-level=INFO" //log level info +] + dependencies { distTarBall project(path: ":sdks:python", configuration: "distTarBall") } @@ -69,9 +78,8 @@ def basicPytestOpts = [ "--log-cli-level=INFO", //log level ] -def preCommitIT(String runScriptsDir, String envdir, Boolean streaming, Boolean runnerV2, String pythonSuffix) { - def suffix = runnerV2 ? '_V2' : '' - suffix = streaming ? "_streaming$suffix" : "_batch$suffix" +def preCommitIT(String runScriptsDir, String envdir, Boolean streaming, String pythonSuffix) { + def suffix = streaming ? "_streaming" : "_batch" task "preCommitIT${suffix}" { dependsOn 'initializeForDataflowJob' @@ -97,9 +105,6 @@ def preCommitIT(String runScriptsDir, String envdir, Boolean streaming, Boolean if (streaming){ argMap.put("streaming", "true") - argMap.put("runner_v2", "true") - } else if (runnerV2) { - argMap.put("runner_v2", "true") } def cmdArgs = mapToArgString(argMap) @@ -111,21 +116,14 @@ def preCommitIT(String runScriptsDir, String envdir, Boolean streaming, Boolean } } -preCommitIT(runScriptsDir, envdir, false, false, pythonVersionSuffix) -preCommitIT(runScriptsDir, envdir, true, false, pythonVersionSuffix) -preCommitIT(runScriptsDir, envdir, false, true, pythonVersionSuffix) -preCommitIT(runScriptsDir, envdir, true, true, pythonVersionSuffix) +preCommitIT(runScriptsDir, envdir, false, pythonVersionSuffix) +preCommitIT(runScriptsDir, envdir, true, pythonVersionSuffix) task preCommitIT{ dependsOn preCommitIT_batch dependsOn preCommitIT_streaming } -task preCommitIT_V2{ - dependsOn preCommitIT_batch_V2 - dependsOn preCommitIT_streaming_V2 -} - task postCommitIT { dependsOn 'initializeForDataflowJob' @@ -145,6 +143,29 @@ task postCommitIT { } } +task postCommitArmIT { + def pyversion = "${project.ext.pythonVersion.replace('.', '')}" + dependsOn 'initializeForDataflowJob' + dependsOn ":sdks:python:container:py${pyversion}:docker" + + doLast { + def testOpts = basicPytestOpts + ["--numprocesses=8", "--dist=loadfile"] + def argMap = [ + "test_opts": testOpts, + "sdk_location": project.ext.sdkLocation, + "suite": "postCommitIT-df${pythonVersionSuffix}", + "collect": "it_postcommit", + "py_version": project.ext.pythonVersion, + "arch": "ARM" + ] + def cmdArgs = mapToArgString(argMap) + exec { + executable 'sh' + args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs" + } + } +} + task postCommitSickbay { dependsOn 'initializeForDataflowJob' @@ -192,7 +213,6 @@ task examples { def argMap = [ "test_opts": testOpts + ["--numprocesses=8", "--dist=loadfile"], "sdk_location": project.ext.sdkLocation, - "runner_v2": "true", "suite": "postCommitIT-df${pythonVersionSuffix}-xdist", "collect": "examples_postcommit and not no_xdist and not sickbay_dataflow" ] @@ -208,7 +228,6 @@ task examples { def argMap = [ "test_opts": testOpts, "sdk_location": project.ext.sdkLocation, - "runner_v2": "true", "suite": "postCommitIT-df${pythonVersionSuffix}-no-xdist", "collect": "examples_postcommit and no_xdist and not sickbay_dataflow" ] @@ -231,13 +250,6 @@ task validatesRunnerBatchTests { "collect": "it_validatesrunner and not no_sickbay_batch" ] - if (project.hasProperty('useRunnerV2')) { - argMap.put("runner_v2", "true") - } - - if (project.hasProperty('disableRunnerV2')) { - argMap.put("disable_runner_v2", "true") - } def cmdArgs = mapToArgString(argMap) exec { executable 'sh' @@ -258,7 +270,6 @@ task validatesRunnerStreamingTests { "sdk_location": project.ext.sdkLocation, "suite": "validatesRunnerStreamingTests-df${pythonVersionSuffix}-xdist", "collect": "it_validatesrunner and not no_sickbay_streaming and not no_xdist", - "runner_v2": "true", ] def cmdArgs = mapToArgString(argMap) @@ -276,7 +287,6 @@ task validatesRunnerStreamingTests { "sdk_location": project.ext.sdkLocation, "suite": "validatesRunnerStreamingTests-df${pythonVersionSuffix}-noxdist", "collect": "it_validatesrunner and not no_sickbay_streaming and no_xdist", - "runner_v2": "true", ] def cmdArgs = mapToArgString(argMap) @@ -370,6 +380,22 @@ task validatesContainer() { } } +task validatesContainerARM() { + def pyversion = "${project.ext.pythonVersion.replace('.', '')}" + dependsOn 'initializeForDataflowJob' + dependsOn ":sdks:python:container:py${pyversion}:docker" + def runScriptsPath = "${rootDir}/sdks/python/container/run_validatescontainer.sh" + doLast { + exec { + executable 'sh' + args '-c', ". ${envdir}/bin/activate && cd ${rootDir} && ${runScriptsPath} " + + "${project.ext.pythonVersion} " + + "${project.ext.sdkLocation} " + + "ARM" + } + } +} + def tensorRTTests = tasks.create("tensorRTtests") { dependsOn 'installGcpTest' dependsOn ':sdks:python:sdist' @@ -398,6 +424,35 @@ def tensorRTTests = tasks.create("tensorRTtests") { } } +// Vertex AI RunInference IT tests +task vertexAIInferenceTest { + dependsOn 'initializeForDataflowJob' + dependsOn ':sdks:python:sdist' + def requirementsFile = "${rootDir}/sdks/python/apache_beam/ml/inference/vertex_ai_tests_requirements.txt" + doFirst { + exec { + executable 'sh' + args '-c', ". ${envdir}/bin/activate && pip install -r $requirementsFile" + } + } + doLast { + def testOpts = basicTestOpts + def argMap = [ + "test_opts": testOpts, + "suite": "VertexAITests-df-py${pythonVersionSuffix}", + "collect": "vertex_ai_postcommit" , + "runner": "TestDataflowRunner", + "requirements_file": "$requirementsFile" + ] + def cmdArgs = mapToArgString(argMap) + exec { + executable 'sh' + args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs" + } + } +} + + task installTFTRequirements { dependsOn 'initializeForDataflowJob' doLast { @@ -439,6 +494,7 @@ task tftTests { project.tasks.register("inferencePostCommitIT") { dependsOn = [ 'tensorRTtests', + 'vertexAIInferenceTest', ] } @@ -461,6 +517,7 @@ project(":sdks:python:test-suites:xlang").ext.xlangTasks.each { taskMetadata -> "--sdk_container_image=gcr.io/apache-beam-testing/beam-sdk/beam_python${project.ext.pythonVersion}_sdk:latest", "--sdk_harness_container_image_overrides=.*java.*,gcr.io/apache-beam-testing/beam-sdk/beam_java8_sdk:latest" ], - pytestOptions: basicPytestOpts + pytestOptions: basicPytestOpts, + additionalEnvs: taskMetadata.additionalEnvs ) } diff --git a/sdks/python/test-suites/dataflow/py37/build.gradle b/sdks/python/test-suites/dataflow/py37/build.gradle deleted file mode 100644 index 9f89c61e0a208..0000000000000 --- a/sdks/python/test-suites/dataflow/py37/build.gradle +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -apply plugin: org.apache.beam.gradle.BeamModulePlugin -applyPythonNature() - -// Required to setup a Python 3 virtualenv and task names. -pythonVersion = '3.7' -apply from: "../common.gradle" diff --git a/sdks/python/test-suites/direct/common.gradle b/sdks/python/test-suites/direct/common.gradle index 27e91b4733dd5..da7aea95e1b02 100644 --- a/sdks/python/test-suites/direct/common.gradle +++ b/sdks/python/test-suites/direct/common.gradle @@ -337,13 +337,41 @@ task xgboostInferenceTest { } +// Transformers RunInference IT tests +task transformersInferenceTest { + dependsOn 'installGcpTest' + dependsOn ':sdks:python:sdist' + def requirementsFile = "${rootDir}/sdks/python/apache_beam/ml/inference/huggingface_tests_requirements.txt" + doFirst { + exec { + executable 'sh' + args '-c', ". ${envdir}/bin/activate && pip install -r $requirementsFile" + } + } + doLast { + def testOpts = basicTestOpts + def argMap = [ + "test_opts": testOpts, + "suite": "postCommitIT-direct-py${pythonVersionSuffix}", + "collect": "uses_transformers and it_postcommit" , + "runner": "TestDirectRunner" + ] + def cmdArgs = mapToArgString(argMap) + exec { + executable 'sh' + args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs" + } + } +} + // Add all the RunInference framework IT tests to this gradle task that runs on Direct Runner Post commit suite. project.tasks.register("inferencePostCommitIT") { dependsOn = [ 'torchInferenceTest', 'sklearnInferenceTest', 'tensorflowInferenceTest', - 'xgboostInferenceTest' + 'xgboostInferenceTest', + 'transformersInferenceTest', // (TODO) https://github.com/apache/beam/issues/25799 // uncomment tfx bsl tests once tfx supports protobuf 4.x // 'tfxInferenceTest', diff --git a/sdks/python/test-suites/direct/py37/build.gradle b/sdks/python/test-suites/direct/py37/build.gradle deleted file mode 100644 index bf99f72d429c6..0000000000000 --- a/sdks/python/test-suites/direct/py37/build.gradle +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -plugins { id 'org.apache.beam.module' } -applyPythonNature() - -// Required to setup a Python 3 virtualenv and task names. -pythonVersion = '3.7' -apply from: '../common.gradle' diff --git a/sdks/python/test-suites/direct/xlang/build.gradle b/sdks/python/test-suites/direct/xlang/build.gradle index 74cda691f14a3..289f5c8a0e07c 100644 --- a/sdks/python/test-suites/direct/xlang/build.gradle +++ b/sdks/python/test-suites/direct/xlang/build.gradle @@ -44,6 +44,8 @@ def cleanupTask = project.tasks.register("fnApiJobServerCleanup", Exec) { args '-c', ". ${envDir}/bin/activate && python -m apache_beam.runners.portability.local_job_service_main --pid_file ${pidFile} --stop" } +def gcpProject = project.findProperty('dataflowProject') ?: 'apache-beam-testing' + createCrossLanguageValidatesRunnerTask( startJobServer: setupTask, cleanupJobServer: cleanupTask, @@ -67,3 +69,20 @@ createCrossLanguageValidatesRunnerTask( "--endpoint localhost:${jobPort}", ], ) + +createTransformServiceTask( + startJobServer: setupTask, + cleanupJobServer: cleanupTask, + numParallelTests: 1, + collectMarker: 'uses_transform_service', + pythonPipelineOptions: [ + "--runner=TestDirectRunner", + "--project=${gcpProject}", + ], + pytestOptions: [ + "--capture=no", // print stdout instantly + "--timeout=4500", // timeout of whole command execution + "--color=yes", // console color + "--log-cli-level=INFO" //log level info + ] +) diff --git a/sdks/python/test-suites/gradle.properties b/sdks/python/test-suites/gradle.properties index 75bdf25b8d482..72fc651733db1 100644 --- a/sdks/python/test-suites/gradle.properties +++ b/sdks/python/test-suites/gradle.properties @@ -23,18 +23,13 @@ # dataflow test-suites # (TODO): https://github.com/apache/beam/issues/21971 # Add python 3.10 to dataflow test-suites -dataflow_precommit_it_task_py_versions=3.7,3.11 -dataflow_mongodbio_it_task_py_versions=3.7 -dataflow_chicago_taxi_example_task_py_versions=3.7 - -# dataflow runner v1 batch and streaming tests -# lowest and highest version supported by dataflow runner v1 -dataflow_validates_runner_batch_tests=3.7,3.9 -dataflow_validates_runner_streaming_tests=3.7,3.9 +dataflow_precommit_it_task_py_versions=3.8,3.11 +dataflow_mongodbio_it_task_py_versions=3.8 +dataflow_chicago_taxi_example_task_py_versions=3.8 # TODO: Enable following tests after making sure we have enough capacity. -dataflow_validates_runner_batch_tests_V2=3.7,3.11 -dataflow_validates_runner_streaming_tests_V2=3.7,3.11 +dataflow_validates_runner_batch_tests=3.8,3.11 +dataflow_validates_runner_streaming_tests=3.8,3.11 dataflow_examples_postcommit_py_versions=3.11 # TFX_BSL is not yet supported on Python 3.10. dataflow_cloudml_benchmark_tests_py_versions=3.9 @@ -43,14 +38,14 @@ direct_mongodbio_it_task_py_versions=3.11 # flink runner test-suites flink_validates_runner_precommit_py_versions=3.11 -flink_validates_runner_postcommit_py_versions=3.7,3.11 -flink_examples_postcommit_py_versions=3.7,3.11 +flink_validates_runner_postcommit_py_versions=3.8,3.11 +flink_examples_postcommit_py_versions=3.8,3.11 # samza runner test-suites -samza_validates_runner_postcommit_py_versions=3.7,3.11 +samza_validates_runner_postcommit_py_versions=3.8,3.11 # spark runner test-suites -spark_examples_postcommit_py_versions=3.7,3.11 +spark_examples_postcommit_py_versions=3.8,3.11 # cross language gcp io postcommit python test suites -cross_language_validates_gcp_py_versions=3.7,3.11 +cross_language_validates_gcp_py_versions=3.8,3.11 diff --git a/sdks/python/test-suites/portable/py37/build.gradle b/sdks/python/test-suites/portable/py37/build.gradle deleted file mode 100644 index f4141db2a2d8d..0000000000000 --- a/sdks/python/test-suites/portable/py37/build.gradle +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -apply plugin: org.apache.beam.gradle.BeamModulePlugin -applyPythonNature() - -addPortableWordCountTasks() - -// Required to setup a Python 3.7 virtualenv and task names. -pythonVersion = '3.7' -apply from: "../common.gradle" diff --git a/sdks/python/test-suites/tox/common.gradle b/sdks/python/test-suites/tox/common.gradle index ee183dff40649..3fdd0c0c553b8 100644 --- a/sdks/python/test-suites/tox/common.gradle +++ b/sdks/python/test-suites/tox/common.gradle @@ -29,18 +29,12 @@ test.dependsOn "testPy${pythonVersionSuffix}Cloud" // toxTask "testPy${pythonVersionSuffix}Dask", "py${pythonVersionSuffix}-dask", "${posargs}" // test.dependsOn "testPy${pythonVersionSuffix}Dask" -toxTask "testPy${pythonVersionSuffix}Cython", "py${pythonVersionSuffix}-cython", "${posargs}" -test.dependsOn "testPy${pythonVersionSuffix}Cython" toxTask "testPy38CloudCoverage", "py38-cloudcoverage", "${posargs}" test.dependsOn "testPy38CloudCoverage" project.tasks.register("preCommitPy${pythonVersionSuffix}") { - // Since codecoverage reports will always be generated for py38, - // all tests will be exercised. - if (pythonVersionSuffix.equals('38')) { - dependsOn = ["testPy38Cython"] - } else { - dependsOn = ["testPy${pythonVersionSuffix}Cloud", "testPy${pythonVersionSuffix}Cython"] - } + // Since codecoverage reports will always be generated for py38, + // all tests will be exercised. + dependsOn = ["testPy${pythonVersionSuffix}Cloud", "testPython${pythonVersionSuffix}"] } \ No newline at end of file diff --git a/sdks/python/test-suites/tox/py310/build.gradle b/sdks/python/test-suites/tox/py310/build.gradle index ea10fde831c63..f1e40a17951fc 100644 --- a/sdks/python/test-suites/tox/py310/build.gradle +++ b/sdks/python/test-suites/tox/py310/build.gradle @@ -28,5 +28,3 @@ pythonVersion = '3.10' apply from: "../common.gradle" -// TODO(https://github.com/apache/beam/issues/20051): Remove this once tox uses isolated builds. -testPy310Cython.mustRunAfter testPython310, testPy310Cloud diff --git a/sdks/python/test-suites/tox/py311/build.gradle b/sdks/python/test-suites/tox/py311/build.gradle index 1bb3766500bb9..fabf9fd4365a7 100644 --- a/sdks/python/test-suites/tox/py311/build.gradle +++ b/sdks/python/test-suites/tox/py311/build.gradle @@ -28,5 +28,3 @@ pythonVersion = '3.11' apply from: "../common.gradle" -// TODO(https://github.com/apache/beam/issues/20051): Remove this once tox uses isolated builds. -testPy311Cython.mustRunAfter testPython311, testPy311Cloud diff --git a/sdks/python/test-suites/tox/py37/build.gradle b/sdks/python/test-suites/tox/py37/build.gradle deleted file mode 100644 index 744ca67506292..0000000000000 --- a/sdks/python/test-suites/tox/py37/build.gradle +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Unit tests for Python 3.7 - */ - -plugins { id 'org.apache.beam.module' } -applyPythonNature() - -// Required to setup a Python 3 virtualenv and task names. -pythonVersion = '3.7' - -def posargs = project.findProperty("posargs") ?: "" - -task lint {} -check.dependsOn lint - -toxTask "lintPy37", "py37-lint", "${posargs}" -lint.dependsOn lintPy37 - -toxTask "mypyPy37", "py37-mypy", "${posargs}" -lint.dependsOn mypyPy37 - -apply from: "../common.gradle" - -// TODO(https://github.com/apache/beam/issues/20051): Remove this once tox uses isolated builds. -testPy37Cython.mustRunAfter testPython37, testPy37Cloud diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle index 77f4511b53d00..b1ed5f88c7c93 100644 --- a/sdks/python/test-suites/tox/py38/build.gradle +++ b/sdks/python/test-suites/tox/py38/build.gradle @@ -29,13 +29,21 @@ pythonVersion = '3.8' toxTask "formatter", "py3-yapf-check" check.dependsOn formatter +// TODO(BEAM-12000): Move tasks that aren't specific to 3.8 to Py 3.9. +def posargs = project.findProperty("posargs") ?: "" + +task lint {} +check.dependsOn lint + +toxTask "lintPy38", "py38-lint", "${posargs}" +lint.dependsOn lintPy38 + +toxTask "mypyPy38", "py38-mypy", "${posargs}" +lint.dependsOn mypyPy38 + apply from: "../common.gradle" -// TODO(https://github.com/apache/beam/issues/20051): Remove this once tox uses isolated builds. -testPy38Cython.mustRunAfter testPython38, testPy38CloudCoverage -// TODO(BEAM-12000): Move tasks that aren't specific to 3.8 to Py 3.9. -def posargs = project.findProperty("posargs") ?: "" // PyCoverage Precommit runs test suites that evaluate test coverage and compatibility of // particular dependencies. It is exercised on a single Python version. @@ -106,6 +114,10 @@ toxTask "testPy38pytorch-200", "py38-pytorch-200", "${posargs}" test.dependsOn "testPy38pytorch-200" preCommitPyCoverage.dependsOn "testPy38pytorch-200" +toxTask "testPy38tft-113", "py38-tft-113", "${posargs}" +test.dependsOn "testPy38tft-113" +preCommitPyCoverage.dependsOn "testPy38tft-113" + // TODO(https://github.com/apache/beam/issues/25796) - uncomment onnx tox task once onnx supports protobuf 4.x.x // Create a test task for each minor version of onnx // toxTask "testPy38onnx-113", "py38-onnx-113", "${posargs}" @@ -116,6 +128,19 @@ toxTask "testPy38tensorflow-212", "py38-tensorflow-212", "${posargs}" test.dependsOn "testPy38tensorflow-212" preCommitPyCoverage.dependsOn "testPy38tensorflow-212" +// Create a test task for each minor version of transformers +toxTask "testPy38transformers-428", "py38-transformers-428", "${posargs}" +test.dependsOn "testPy38transformers-428" +preCommitPyCoverage.dependsOn "testPy38transformers-428" + +toxTask "testPy38transformers-429", "py38-transformers-429", "${posargs}" +test.dependsOn "testPy38transformers-429" +preCommitPyCoverage.dependsOn "testPy38transformers-429" + +toxTask "testPy38transformers-430", "py38-transformers-430", "${posargs}" +test.dependsOn "testPy38transformers-430" +preCommitPyCoverage.dependsOn "testPy38transformers-430" + toxTask "whitespacelint", "whitespacelint", "${posargs}" task archiveFilesToLint(type: Zip) { @@ -125,7 +150,11 @@ task archiveFilesToLint(type: Zip) { from ("$rootProject.projectDir") { include "**/*.md" include "**/build.gradle" + include '**/build.gradle.kts' + exclude '**/build/**' // intermediate build directory + exclude 'website/www/site/themes/docsy/**' // fork to google/docsy exclude "**/node_modules/*" + exclude "**/.gogradle/*" } } diff --git a/sdks/python/test-suites/tox/py39/build.gradle b/sdks/python/test-suites/tox/py39/build.gradle index 380cc1486daad..5bb73b60a5d2f 100644 --- a/sdks/python/test-suites/tox/py39/build.gradle +++ b/sdks/python/test-suites/tox/py39/build.gradle @@ -27,6 +27,3 @@ applyPythonNature() pythonVersion = '3.9' apply from: "../common.gradle" - -// TODO(https://github.com/apache/beam/issues/20051): Remove this once tox uses isolated builds. -testPy39Cython.mustRunAfter testPython39, testPy39Cloud diff --git a/sdks/python/test-suites/xlang/build.gradle b/sdks/python/test-suites/xlang/build.gradle index ea407ac6f3fb5..df3ebdd1582c0 100644 --- a/sdks/python/test-suites/xlang/build.gradle +++ b/sdks/python/test-suites/xlang/build.gradle @@ -58,5 +58,18 @@ def gcpXlangCommon = new CrossLanguageTaskCommon().tap { } xlangTasks.add(gcpXlangCommon) +def ioExpansionProject = project.project(':sdks:java:io:expansion-service') + +def ioXlangCommon = new CrossLanguageTaskCommon().tap { + name = "ioCrossLanguage" + expansionProjectPath = ioExpansionProject.getPath() + collectMarker = "uses_io_expansion_service" + startJobServer = setupTask + cleanupJobServer = cleanupTask + //See .test-infra/kafka/bitnami/README.md for setup instructions + additionalEnvs = ["KAFKA_BOOTSTRAP_SERVER":project.findProperty('kafkaBootstrapServer')] +} + +xlangTasks.add(ioXlangCommon) ext.xlangTasks = xlangTasks \ No newline at end of file diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index e3ff948aff7e1..1e797d96074fb 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -17,7 +17,7 @@ [tox] # new environments will be excluded by default unless explicitly added to envlist. -envlist = py37,py38,py39,py310,py311,py37-{cloud,cython,lint,mypy,dask},py38-{cloud,cython,docs,cloudcoverage,dask},py39-{cloud,cython},py310-{cloud,cython,dask},py311-{cloud,cython,dask},whitespacelint +envlist = py38,py39,py310,py311,py38-{cloud,docs,lint,mypy,cloudcoverage,dask},py39-{cloud},py310-{cloud,dask},py311-{cloud,dask},whitespacelint toxworkdir = {toxinidir}/target/{env:ENV_NAME:.tox} [pycodestyle] @@ -31,7 +31,7 @@ select = E3 # https://github.com/apache/beam/issues/25668 pip_pre = True # allow apps that support color to use it. -passenv=TERM +passenv=TERM,CLOUDSDK_CONFIG # Set [] options for pip installation of apache-beam tarball. extras = test,dataframe # Don't warn that these commands aren't installed. @@ -44,9 +44,6 @@ allowlist_externals = curl ./codecov chmod -deps = - cython: cython==0.29.33 - -r build-requirements.txt setenv = RUN_SKIPPED_PY3_TESTS=0 # Use an isolated tmp dir for tests that get slowed down by scanning /tmp. @@ -67,45 +64,38 @@ commands_pre = bash {toxinidir}/scripts/run_tox_cleanup.sh commands_post = bash {toxinidir}/scripts/run_tox_cleanup.sh + commands = false {envname} is misconfigured -[testenv:py{37,38,39,310,311}] +[testenv:py{38,39,310,311}] commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{37,38,39,310,311}-win] +[testenv:py{38,39,310,311}-win] commands = python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" install_command = {envbindir}/python.exe {envbindir}/pip.exe install --retries 10 {opts} {packages} list_dependencies_command = {envbindir}/python.exe {envbindir}/pip.exe freeze -[testenv:py{37,38,39,310,311}-cython] -# cython tests are only expected to work in linux (2.x and 3.x) -# If we want to add other platforms in the future, it should be: -# `platform = linux2|darwin|...` -# See https://docs.python.org/2/library/sys.html#sys.platform for platform codes -platform = linux -commands = - # TODO(https://github.com/apache/beam/issues/20051): Remove this build_ext invocation once local source no longer - # shadows the installed apache_beam. - python setup.py build_ext --inplace - python apache_beam/examples/complete/autocomplete_test.py - bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" - -[testenv:py{37,38,39,310,311}-cloud] +[testenv:py{38,39,310,311}-cloud] +; extras = test,gcp,interactive,dataframe,aws,azure extras = test,gcp,interactive,dataframe,aws,azure commands = + python apache_beam/examples/complete/autocomplete_test.py bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" -[testenv:py{37,38,39,310,311}-dask] +[testenv:py{38,39,310,311}-dask] extras = test,dask commands = bash {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" + [testenv:py38-cloudcoverage] deps = pytest-cov==3.0.0 +# Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests. +setenv = platform = linux passenv = GIT_*,BUILD_*,ghprb*,CHANGE_ID,BRANCH_NAME,JENKINS_*,CODECOV_* extras = test,gcp,interactive,dataframe,aws @@ -117,15 +107,14 @@ commands = ./codecov -F python -rm codecov -[testenv:py37-lint] +[testenv:py38-lint] # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in pylint. setenv = # keep the version of pylint in sync with the 'rev' in .pre-commit-config.yaml deps = - -r build-requirements.txt - astroid<2.9,>=2.8.0 + astroid<2.17.0,>=2.15.6 pycodestyle==2.8.0 - pylint==2.11.1 + pylint==2.17.5 isort==4.2.15 flake8==4.0.1 commands = @@ -139,9 +128,8 @@ deps = commands = time {toxinidir}/scripts/run_whitespacelint.sh -[testenv:py37-mypy] +[testenv:py38-mypy] deps = - -r build-requirements.txt mypy==0.790 dask==2022.01.0 distributed==2022.01.0 @@ -161,8 +149,9 @@ deps = docutils<0.18 Jinja2==3.0.3 # TODO(https://github.com/apache/beam/issues/21587): Sphinx version is too old. torch - xgboost + xgboost<=1.7.6 datatable==1.0.0 + transformers commands = time {toxinidir}/scripts/generate_pydoc.sh @@ -170,7 +159,6 @@ commands = # Used by hdfs_integration_test.sh. Do not run this directly, as it depends on # nodes defined in hdfs_integration_test/docker-compose.yml. deps = - -r build-requirements.txt holdup==1.8.0 extras = gcp @@ -203,7 +191,6 @@ commands_pre = # Do not run this directly, as it depends on nodes defined in # azure/integration_test/docker-compose.yml. deps = - -r build-requirements.txt extras = azure passenv = REQUESTS_CA_BUNDLE @@ -283,7 +270,7 @@ extras = test commands = bash {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/spark_runner_test.py {posargs} -[testenv:py{37,38,39,310}-pyarrow-{3,4,5,6,7,8,9}] +[testenv:py{38,39,310}-pyarrow-{3,4,5,6,7,8,9}] deps = 3: pyarrow>=3,<4 4: pyarrow>=4,<5 @@ -300,7 +287,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{37,38,39,310,311}-pyarrow-{10,11}] +[testenv:py{38,39,310,311}-pyarrow-{10,11}] deps = 10: pyarrow>=10,<11 11: pyarrow>=11,<12 @@ -313,7 +300,7 @@ commands = /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{37,38,39,310,311}-pandas-{14,15}] +[testenv:py{38,39,310,311}-pandas-{14,15}] deps = 14: pandas>=1.4.3,<1.5.0 # Exclude 1.5.0 and 1.5.1 because of https://github.com/pandas-dev/pandas/issues/45725 @@ -324,15 +311,22 @@ commands = # Run all DataFrame API unit tests bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/dataframe' -[testenv:py{37,38,39,310,311}-pytorch-{19,110,111,112,113}] +[testenv:py{38,39}-tft-113] +deps = + 113: tensorflow_transform>=1.13.0,<1.14.0 +commands = + bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/ml/transforms' + +[testenv:py{38,39,310,311}-pytorch-{19,110,111,112,113}] deps = - -r build-requirements.txt 19: torch>=1.9.0,<1.10.0 110: torch>=1.10.0,<1.11.0 111: torch>=1.11.0,<1.12.0 112: torch>=1.12.0,<1.13.0 113: torch>=1.13.0,<1.14.0 extras = test,gcp +# Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests. +setenv = commands = # Log torch version for debugging /bin/sh -c "pip freeze | grep -E torch" @@ -342,9 +336,10 @@ commands = [testenv:py{38,39,310}-pytorch-200] deps = - -r build-requirements.txt 200: torch>=2.0.0,<2.1.0 extras = test,gcp +# Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests. +setenv = commands = # Log torch version for debugging /bin/sh -c "pip freeze | grep -E torch" @@ -353,7 +348,7 @@ commands = /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' # TODO(https://github.com/apache/beam/issues/25796) - uncomment onnx tox task in tox/py38/build.gradle once onnx supports protobuf 4.x.x -[testenv:py{37,38,39,310}-onnx-113] +[testenv:py{38,39,310}-onnx-113] # TODO(https://github.com/apache/beam/issues/25443) # apparently tox has problem when substitution key has single value. Change back to -onnx-{113,...} # when multiple onnx versions are tested. @@ -374,7 +369,6 @@ commands = [testenv:py{38,39,310}-tensorflow-212] deps = - -r build-requirements.txt 212: tensorflow>=2.12rc1,<2.13 extras = test,gcp commands = @@ -384,9 +378,8 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_tf {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{37,38,39,310}-xgboost-{160,170}] +[testenv:py{38,39,310}-xgboost-{160,170}] deps = - -r build-requirements.txt 160: xgboost>=1.6.0,<1.7.0 datatable==1.0.0 @@ -400,3 +393,32 @@ commands = # Run all XGBoost unit tests # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_xgboost {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' + +[testenv:py{38,39,310,311}-transformers-{428,429,430}] +deps = + 428: transformers>=4.28.0,<4.29.0 + 429: transformers>=4.29.0,<4.30.0 + 430: transformers>=4.30.0,<4.31.0 + torch>=1.9.0,<1.14.0 + tensorflow==2.12.0 +extras = test,gcp +commands = + # Log transformers and its dependencies version for debugging + /bin/sh -c "pip freeze | grep -E transformers" + /bin/sh -c "pip freeze | grep -E torch" + /bin/sh -c "pip freeze | grep -E tensorflow" + # Run all Transformers unit tests + # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. + /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_transformers {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' + +[testenv:py{38,311}-vertex-ai] +deps = + tensorflow==2.12.0 +extras = test,gcp +commands = + # Log aiplatform and its dependencies version for debugging + /bin/sh -c "pip freeze | grep -E google-cloud-aiplatform" + /bin/sh -c "pip freeze | grep -E tensorflow" + # Run all Vertex AI unit tests + # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. + /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_vertex_ai {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' diff --git a/sdks/typescript/container/Dockerfile b/sdks/typescript/container/Dockerfile index a78c5d70bb16b..5964d0479770f 100644 --- a/sdks/typescript/container/Dockerfile +++ b/sdks/typescript/container/Dockerfile @@ -31,5 +31,5 @@ COPY target/apache-beam.tgz ./apache-beam.tgz # Install dependencies and compile RUN npm install apache-beam.tgz -COPY target/launcher/${TARGETOS}_${TARGETARCH}/boot /opt/apache/beam +COPY target/launcher/${TARGETOS}_${TARGETARCH}/boot /opt/apache/beam/ ENTRYPOINT ["/opt/apache/beam/boot"] diff --git a/sdks/typescript/container/build.gradle b/sdks/typescript/container/build.gradle index 69ba3c632d16e..248415789834c 100644 --- a/sdks/typescript/container/build.gradle +++ b/sdks/typescript/container/build.gradle @@ -43,6 +43,9 @@ def copyDockerfileDependencies = tasks.register("copyDockerfileDependencies", Co } } +def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] +def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers") + docker { name containerImageName( name: project.docker_image_default_repo_prefix + "typescript_sdk", @@ -54,8 +57,10 @@ docker { // tags used by dockerTag task tags containerImageTags() files "./build" - buildx project.containerPlatforms() != [project.nativeArchitecture()] + buildx useBuildx platform(*project.containerPlatforms()) + load useBuildx && !pushContainers + push pushContainers } dockerPrepare.dependsOn goBuild @@ -78,5 +83,5 @@ if (project.rootProject.hasProperty(["docker-pull-licenses"])) { } task pushAll { - dependsOn ":sdks:typescript:container:dockerPush" + dependsOn ":sdks:typescript:container:docker" } diff --git a/sdks/typescript/package-lock.json b/sdks/typescript/package-lock.json index a8d4d6319d519..55d510ccce8c4 100644 --- a/sdks/typescript/package-lock.json +++ b/sdks/typescript/package-lock.json @@ -1,15 +1,15 @@ { "name": "apache-beam", - "version": "2.47.0-SNAPSHOT", + "version": "2.52.0-SNAPSHOT", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "apache-beam", - "version": "2.47.0-SNAPSHOT", + "version": "2.52.0-SNAPSHOT", "dependencies": { "@google-cloud/pubsub": "^2.19.4", - "@grpc/grpc-js": "^1.4.6", + "@grpc/grpc-js": "~1.4.6", "@protobuf-ts/grpc-transport": "^2.1.0", "@protobuf-ts/plugin": "^2.1.0", "bson": "^4.6.0", @@ -19,11 +19,11 @@ "fast-deep-equal": "^3.1.3", "find-git-root": "^1.0.4", "long": "^4.0.0", - "protobufjs": "^6.11.3", + "protobufjs": "~6.11.3", "queue-typescript": "^1.0.1", "serialize-closures": "^0.2.7", "ts-closure-transform": "^0.1.7", - "ttypescript": "^1.5.13", + "ttypescript": "^1.5.15", "uuid": "^8.3.2" }, "bin": { @@ -190,9 +190,9 @@ } }, "node_modules/@grpc/grpc-js": { - "version": "1.6.7", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.6.7.tgz", - "integrity": "sha512-eBM03pu9hd3VqDQG+kHahiG1x80RGkkqqRb1Pchcwqej/KkAH95gAvKs6laqaHCycYaPK+TKuNQnOz9UXYA8qw==", + "version": "1.4.6", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.4.6.tgz", + "integrity": "sha512-Byau4xiXfIixb1PnW30V/P9mkrZ05lknyNqiK+cVY9J5hj3gecxd/anwaUbAM8j834zg1x78NvAbwGnMfWEu7A==", "dependencies": { "@grpc/proto-loader": "^0.6.4", "@types/node": ">=12.12.47" @@ -1972,9 +1972,9 @@ } }, "node_modules/get-func-name": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", - "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", + "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==", "engines": { "node": "*" } @@ -2091,6 +2091,77 @@ "node": ">=10" } }, + "node_modules/google-gax/node_modules/@grpc/grpc-js": { + "version": "1.6.12", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.6.12.tgz", + "integrity": "sha512-JmvQ03OTSpVd9JTlj/K3IWHSz4Gk/JMLUTtW7Zb0KvO1LcOYGATh5cNuRYzCAeDR3O8wq+q8FZe97eO9MBrkUw==", + "dependencies": { + "@grpc/proto-loader": "^0.7.0", + "@types/node": ">=12.12.47" + }, + "engines": { + "node": "^8.13.0 || >=10.10.0" + } + }, + "node_modules/google-gax/node_modules/@grpc/grpc-js/node_modules/@grpc/proto-loader": { + "version": "0.7.7", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.7.tgz", + "integrity": "sha512-1TIeXOi8TuSCQprPItwoMymZXxWT0CPxUhkrkeCUH+D8U7QDwQ6b7SUz2MaLuWM2llT+J/TVFLmQI5KtML3BhQ==", + "dependencies": { + "@types/long": "^4.0.1", + "lodash.camelcase": "^4.3.0", + "long": "^4.0.0", + "protobufjs": "^7.0.0", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/google-gax/node_modules/@grpc/grpc-js/node_modules/protobufjs": { + "version": "7.2.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.4.tgz", + "integrity": "sha512-AT+RJgD2sH8phPmCf7OUZR8xGdcJRga4+1cOaXJ64hvcSkVhNcRHOwIxUatPH15+nj59WAGTDv3LSGZPEQbJaQ==", + "hasInstallScript": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/google-gax/node_modules/@grpc/grpc-js/node_modules/protobufjs/node_modules/long": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" + }, + "node_modules/google-gax/node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/google-gax/node_modules/protobufjs": { "version": "6.11.2", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.2.tgz", @@ -2116,6 +2187,31 @@ "pbts": "bin/pbts" } }, + "node_modules/google-gax/node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/google-gax/node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "engines": { + "node": ">=12" + } + }, "node_modules/google-p12-pem": { "version": "3.1.4", "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.1.4.tgz", @@ -3634,9 +3730,9 @@ } }, "node_modules/ttypescript": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/ttypescript/-/ttypescript-1.5.13.tgz", - "integrity": "sha512-KT/RBfGGlVJFqEI8cVvI3nMsmYcFvPSZh8bU0qX+pAwbi7/ABmYkzn7l/K8skw0xmYjVCoyaV6WLsBQxdadybQ==", + "version": "1.5.15", + "resolved": "https://registry.npmjs.org/ttypescript/-/ttypescript-1.5.15.tgz", + "integrity": "sha512-48ykDNHzFnPMnv4hYX1P8Q84TvCZyL1QlFxeuxsuZ48X2+ameBgPenvmCkHJtoOSxpoWTWi8NcgNrRnVDOmfSg==", "dependencies": { "resolve": ">=1.9.0" }, @@ -3858,9 +3954,9 @@ } }, "node_modules/word-wrap": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", - "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.4.tgz", + "integrity": "sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA==", "dev": true, "engines": { "node": ">=0.10.0" @@ -4091,9 +4187,9 @@ } }, "@grpc/grpc-js": { - "version": "1.6.7", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.6.7.tgz", - "integrity": "sha512-eBM03pu9hd3VqDQG+kHahiG1x80RGkkqqRb1Pchcwqej/KkAH95gAvKs6laqaHCycYaPK+TKuNQnOz9UXYA8qw==", + "version": "1.4.6", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.4.6.tgz", + "integrity": "sha512-Byau4xiXfIixb1PnW30V/P9mkrZ05lknyNqiK+cVY9J5hj3gecxd/anwaUbAM8j834zg1x78NvAbwGnMfWEu7A==", "requires": { "@grpc/proto-loader": "^0.6.4", "@types/node": ">=12.12.47" @@ -5399,9 +5495,9 @@ "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, "get-func-name": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", - "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=" + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", + "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==" }, "glob": { "version": "7.2.0", @@ -5485,6 +5581,65 @@ "retry-request": "^4.0.0" }, "dependencies": { + "@grpc/grpc-js": { + "version": "1.6.12", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.6.12.tgz", + "integrity": "sha512-JmvQ03OTSpVd9JTlj/K3IWHSz4Gk/JMLUTtW7Zb0KvO1LcOYGATh5cNuRYzCAeDR3O8wq+q8FZe97eO9MBrkUw==", + "requires": { + "@grpc/proto-loader": "^0.7.0", + "@types/node": ">=12.12.47" + }, + "dependencies": { + "@grpc/proto-loader": { + "version": "0.7.7", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.7.tgz", + "integrity": "sha512-1TIeXOi8TuSCQprPItwoMymZXxWT0CPxUhkrkeCUH+D8U7QDwQ6b7SUz2MaLuWM2llT+J/TVFLmQI5KtML3BhQ==", + "requires": { + "@types/long": "^4.0.1", + "lodash.camelcase": "^4.3.0", + "long": "^4.0.0", + "protobufjs": "^7.0.0", + "yargs": "^17.7.2" + } + }, + "protobufjs": { + "version": "7.2.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.4.tgz", + "integrity": "sha512-AT+RJgD2sH8phPmCf7OUZR8xGdcJRga4+1cOaXJ64hvcSkVhNcRHOwIxUatPH15+nj59WAGTDv3LSGZPEQbJaQ==", + "requires": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "dependencies": { + "long": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" + } + } + } + } + }, + "cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + } + }, "protobufjs": { "version": "6.11.2", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.2.tgz", @@ -5504,6 +5659,25 @@ "@types/node": ">=13.7.0", "long": "^4.0.0" } + }, + "yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "requires": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + } + }, + "yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==" } } }, @@ -6594,9 +6768,9 @@ } }, "ttypescript": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/ttypescript/-/ttypescript-1.5.13.tgz", - "integrity": "sha512-KT/RBfGGlVJFqEI8cVvI3nMsmYcFvPSZh8bU0qX+pAwbi7/ABmYkzn7l/K8skw0xmYjVCoyaV6WLsBQxdadybQ==", + "version": "1.5.15", + "resolved": "https://registry.npmjs.org/ttypescript/-/ttypescript-1.5.15.tgz", + "integrity": "sha512-48ykDNHzFnPMnv4hYX1P8Q84TvCZyL1QlFxeuxsuZ48X2+ameBgPenvmCkHJtoOSxpoWTWi8NcgNrRnVDOmfSg==", "requires": { "resolve": ">=1.9.0" }, @@ -6761,9 +6935,9 @@ } }, "word-wrap": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", - "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.4.tgz", + "integrity": "sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA==", "dev": true }, "wordwrap": { diff --git a/sdks/typescript/package.json b/sdks/typescript/package.json index 2894e95030f60..b582b3d5c07f2 100644 --- a/sdks/typescript/package.json +++ b/sdks/typescript/package.json @@ -1,6 +1,6 @@ { "name": "apache-beam", - "version": "2.49.0-SNAPSHOT", + "version": "2.52.0-SNAPSHOT", "devDependencies": { "@google-cloud/bigquery": "^5.12.0", "@types/mocha": "^9.0.0", @@ -36,7 +36,7 @@ }, "dependencies": { "@google-cloud/pubsub": "^2.19.4", - "@grpc/grpc-js": "^1.4.6", + "@grpc/grpc-js": "~1.4.6", "@protobuf-ts/grpc-transport": "^2.1.0", "@protobuf-ts/plugin": "^2.1.0", "bson": "^4.6.0", @@ -46,18 +46,19 @@ "fast-deep-equal": "^3.1.3", "find-git-root": "^1.0.4", "long": "^4.0.0", - "protobufjs": "^6.11.3", + "protobufjs": "~6.11.3", "queue-typescript": "^1.0.1", "serialize-closures": "^0.2.7", "ts-closure-transform": "^0.1.7", - "ttypescript": "^1.5.13", + "ttypescript": "^1.5.15", "uuid": "^8.3.2" }, "main": "./dist/src/apache_beam/index.js", "exports": { ".": "./dist/src/apache_beam/index.js", - "./transforms": "./dist/src/apache_beam/transforms/index.js", + "./io": "./dist/src/apache_beam/io/index.js", "./runners": "./dist/src/apache_beam/index.js", + "./transforms": "./dist/src/apache_beam/transforms/index.js", "./*": "./dist/src/apache_beam/*.js" } } diff --git a/sdks/typescript/src/apache_beam/index.ts b/sdks/typescript/src/apache_beam/index.ts index fc83c43ba1874..15ca4fbbf7e43 100644 --- a/sdks/typescript/src/apache_beam/index.ts +++ b/sdks/typescript/src/apache_beam/index.ts @@ -20,3 +20,4 @@ export * from "./pvalue"; export * from "./transforms"; export * from "./values"; export * from "./runners"; +export * from "./io"; diff --git a/sdks/typescript/src/apache_beam/io/index.ts b/sdks/typescript/src/apache_beam/io/index.ts new file mode 100644 index 0000000000000..fca85f2507547 --- /dev/null +++ b/sdks/typescript/src/apache_beam/io/index.ts @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// A basic subset. +export * from "./avroio"; +export * from "./bigqueryio"; +export * from "./kafka"; +export * from "./parquetio"; +export * from "./pubsub"; +export * from "./pubsublite"; +export * from "./schemaio"; + +import { requireForSerialization } from "../serialization"; +requireForSerialization("apache-beam/io", exports); diff --git a/sdks/typescript/src/apache_beam/runners/dataflow.ts b/sdks/typescript/src/apache_beam/runners/dataflow.ts index 950e630d82d9a..e7da1f7ada51a 100644 --- a/sdks/typescript/src/apache_beam/runners/dataflow.ts +++ b/sdks/typescript/src/apache_beam/runners/dataflow.ts @@ -33,6 +33,8 @@ export function dataflowRunner(runnerOptions: { options: Object = {} ): Promise { var augmentedOptions = { experiments: [] as string[], ...options }; + augmentedOptions.experiments.push("use_runner_v2"); + augmentedOptions.experiments.push("use_portable_job_submission"); augmentedOptions.experiments.push("use_sibling_sdk_workers"); return new PortableRunner( runnerOptions as any, diff --git a/sdks/typescript/src/apache_beam/transforms/index.ts b/sdks/typescript/src/apache_beam/transforms/index.ts index c7f5ff2e44c0d..bb843ad21c827 100644 --- a/sdks/typescript/src/apache_beam/transforms/index.ts +++ b/sdks/typescript/src/apache_beam/transforms/index.ts @@ -23,6 +23,7 @@ export * from "./group_and_combine"; export * from "./pardo"; export * from "./transform"; export * from "./window"; +export * from "./windowings"; export { impulse, withRowCoder } from "./internal"; import { requireForSerialization } from "../serialization"; diff --git a/sdks/typescript/tsconfig.json b/sdks/typescript/tsconfig.json index 59ccfe02b2fd4..ad7619cdd18b1 100644 --- a/sdks/typescript/tsconfig.json +++ b/sdks/typescript/tsconfig.json @@ -17,6 +17,7 @@ "after": true } ], + "declaration": true, "sourceMap": true, "outDir": "dist/", "baseUrl": ".", diff --git a/settings.gradle.kts b/settings.gradle.kts index ddc61e1406f9a..1c851da92e600 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -17,9 +17,15 @@ */ import com.gradle.enterprise.gradleplugin.internal.extension.BuildScanExtensionWithHiddenFeatures +pluginManagement { + plugins { + id("org.javacc.javacc") version "3.0.0" // enable the JavaCC parser generator + } +} + plugins { id("com.gradle.enterprise") version "3.13.2" - id("com.gradle.common-custom-user-data-gradle-plugin") version "1.10" + id("com.gradle.common-custom-user-data-gradle-plugin") version "1.11.3" } @@ -49,8 +55,15 @@ buildCache { local { isEnabled = true } - remote(gradleEnterprise.buildCache) { - isEnabled = false + remote { + url = uri("https://beam-cache.apache.org/cache/") + isAllowUntrustedServer = false + credentials { + username = System.getenv("GRADLE_ENTERPRISE_CACHE_USERNAME") + password = System.getenv("GRADLE_ENTERPRISE_CACHE_PASSWORD") + } + isEnabled = true + isPush = isCi } } @@ -90,6 +103,19 @@ include(":playground:backend:containers:scio") include(":playground:terraform") include(":playground:kafka-emulator") +include(":it:cassandra") +include(":it:common") +include(":it:conditions") +include(":it:elasticsearch") +include(":it:google-cloud-platform") +include(":it:jdbc") +include(":it:kafka") +include(":it:testcontainers") +include(":it:truthmatchers") +include(":it:mongodb") +include(":it:splunk") +include(":it:neo4j") + include(":learning:tour-of-beam:frontend") include(":runners:core-construction-java") @@ -123,6 +149,7 @@ include(":runners:flink:1.16:job-server-container") /* End Flink Runner related settings */ include(":runners:twister2") include(":runners:google-cloud-dataflow-java") +include(":runners:google-cloud-dataflow-java:arm") include(":runners:google-cloud-dataflow-java:examples") include(":runners:google-cloud-dataflow-java:examples-streaming") include(":runners:java-fn-execution") @@ -187,6 +214,7 @@ include(":sdks:java:io:amazon-web-services") include(":sdks:java:io:amazon-web-services2") include(":sdks:java:io:amqp") include(":sdks:java:io:azure") +include(":sdks:java:io:azure-cosmos") include(":sdks:java:io:cassandra") include(":sdks:java:io:clickhouse") include(":sdks:java:io:common") @@ -206,6 +234,7 @@ include(":sdks:java:io:bigquery-io-perf-tests") include(":sdks:java:io:cdap") include(":sdks:java:io:csv") include(":sdks:java:io:file-schema-transform") +include(":sdks:java:io:google-ads") include(":sdks:java:io:google-cloud-platform") include(":sdks:java:io:google-cloud-platform:expansion-service") include(":sdks:java:io:hadoop-common") @@ -215,6 +244,7 @@ include(":sdks:java:io:hbase") include(":sdks:java:io:hcatalog") include(":sdks:java:io:jdbc") include(":sdks:java:io:jms") +include(":sdks:java:io:json") include(":sdks:java:io:kafka") include(":sdks:java:io:kinesis") include(":sdks:java:io:kinesis:expansion-service") @@ -226,6 +256,7 @@ include(":sdks:java:io:parquet") include(":sdks:java:io:pulsar") include(":sdks:java:io:rabbitmq") include(":sdks:java:io:redis") +include(":sdks:java:io:rrio") include(":sdks:java:io:solr") include(":sdks:java:io:sparkreceiver:2") include(":sdks:java:io:snowflake") @@ -256,32 +287,27 @@ include(":sdks:python") include(":sdks:python:apache_beam:testing:load_tests") include(":sdks:python:apache_beam:testing:benchmarks:nexmark") include(":sdks:python:container") -include(":sdks:python:container:py37") include(":sdks:python:container:py38") include(":sdks:python:container:py39") include(":sdks:python:container:py310") include(":sdks:python:container:py311") include(":sdks:python:expansion-service-container") include(":sdks:python:test-suites:dataflow") -include(":sdks:python:test-suites:dataflow:py37") include(":sdks:python:test-suites:dataflow:py38") include(":sdks:python:test-suites:dataflow:py39") include(":sdks:python:test-suites:dataflow:py310") include(":sdks:python:test-suites:dataflow:py311") include(":sdks:python:test-suites:direct") -include(":sdks:python:test-suites:direct:py37") include(":sdks:python:test-suites:direct:py38") include(":sdks:python:test-suites:direct:py39") include(":sdks:python:test-suites:direct:py310") include(":sdks:python:test-suites:direct:py311") include(":sdks:python:test-suites:direct:xlang") -include(":sdks:python:test-suites:portable:py37") include(":sdks:python:test-suites:portable:py38") include(":sdks:python:test-suites:portable:py39") include(":sdks:python:test-suites:portable:py310") include(":sdks:python:test-suites:portable:py311") include(":sdks:python:test-suites:tox:pycommon") -include(":sdks:python:test-suites:tox:py37") include(":sdks:python:test-suites:tox:py38") include(":sdks:python:test-suites:tox:py39") include(":sdks:python:test-suites:tox:py310") @@ -292,7 +318,7 @@ include(":sdks:typescript:container") include(":vendor:bytebuddy-1_12_8") include(":vendor:grpc-1_54_0") include(":vendor:calcite-1_28_0") -include(":vendor:guava-26_0-jre") +include(":vendor:guava-32_1_2-jre") include(":website") include(":runners:google-cloud-dataflow-java:worker") include(":runners:google-cloud-dataflow-java:worker:windmill") diff --git a/vendor/guava-26_0-jre/build.gradle b/vendor/guava-26_0-jre/build.gradle deleted file mode 100644 index 66f152766783b..0000000000000 --- a/vendor/guava-26_0-jre/build.gradle +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -plugins { id 'org.apache.beam.vendor-java' } - -description = "Apache Beam :: Vendored Dependencies :: Guava 26.0-jre" - -group = "org.apache.beam" -version = "0.1" - -vendorJava( - dependencies: ["com.google.guava:guava:26.0-jre"], - relocations: [ - "com.google.common": "org.apache.beam.vendor.guava.v26_0_jre.com.google.common", - "com.google.thirdparty": "org.apache.beam.vendor.guava.v26_0_jre.com.google.thirdparty", - ], - exclusions: [ - "com/google/errorprone/**", - "com/google/j2objc/annotations/**", - "javax/annotation/**", - "org/checkerframework/**", - "org/codehaus/mojo/animal_sniffer/**", - ], - groupId: group, - artifactId: "beam-vendor-guava-26_0-jre", - version: version, -) diff --git a/vendor/guava-32_1_2-jre/build.gradle b/vendor/guava-32_1_2-jre/build.gradle new file mode 100644 index 0000000000000..9fd0b4c34ed66 --- /dev/null +++ b/vendor/guava-32_1_2-jre/build.gradle @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.vendor-java' } + +description = "Apache Beam :: Vendored Dependencies :: Guava 32.1.2-jre" + +group = "org.apache.beam" +version = "0.1" + +vendorJava( + dependencies: ["com.google.guava:guava:32.1.2-jre"], + relocations: [ + "com.google.common": "org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common", + "com.google.thirdparty": "org.apache.beam.vendor.guava.v32_1_2_jre.com.google.thirdparty", + ], + exclusions: [ + "com/google/errorprone/**", + "com/google/j2objc/annotations/**", + "javax/annotation/**", + "org/checkerframework/**", + "org/codehaus/mojo/animal_sniffer/**", + ], + groupId: group, + artifactId: "beam-vendor-guava-32_1_2-jre", + version: version, +) diff --git a/website/Dockerfile b/website/Dockerfile index 76089a8ef3d3b..0cb992422eaca 100644 --- a/website/Dockerfile +++ b/website/Dockerfile @@ -29,6 +29,8 @@ ENV DEBIAN_FRONTEND=noninteractive \ LC_CTYPE=C.UTF-8 \ LC_MESSAGES=C.UTF-8 +WORKDIR /opt/ + # Install deps being used by sh files RUN apt-get update \ && apt-get install -y --no-install-recommends \ @@ -47,9 +49,12 @@ RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - \ && apt-get update \ && apt-get install -y --no-install-recommends \ nodejs \ + npm \ && apt-get autoremove -yqq --purge \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* + +RUN npm install postcss postcss-cli autoprefixer # Install yarn RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \ @@ -60,13 +65,12 @@ RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -# Install hugo extended version v0.68.3 +# Install hugo extended version v0.117.0 RUN HUGOHOME="$(mktemp -d)" \ && export HUGOHOME \ - && curl -sL https://github.com/gohugoio/hugo/releases/download/v0.68.3/hugo_extended_0.68.3_Linux-64bit.tar.gz > "${HUGOHOME}/hugo.tar.gz" \ + && curl -sL https://github.com/gohugoio/hugo/releases/download/v0.117.0/hugo_extended_0.117.0_Linux-64bit.tar.gz > "${HUGOHOME}/hugo.tar.gz" \ && tar -xzvf "${HUGOHOME}/hugo.tar.gz" hugo \ && mv hugo /usr/local/bin/hugo \ && chmod +x /usr/local/bin/hugo \ && rm -r "${HUGOHOME}" -WORKDIR /opt/ diff --git a/website/OWNERS b/website/OWNERS deleted file mode 100644 index d056e03ebea3b..0000000000000 --- a/website/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://s.apache.org/beam-owners - -reviewers: - - melap - - timrobertson100 diff --git a/website/www/site/config.toml b/website/www/site/config.toml index 406011422bd2b..c7b0cd3412e67 100644 --- a/website/www/site/config.toml +++ b/website/www/site/config.toml @@ -104,7 +104,7 @@ github_project_repo = "https://github.com/apache/beam" [params] description = "Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number of runtimes like Apache Flink, Apache Spark, and Google Cloud Dataflow (a cloud service). Beam also brings DSL in different languages, allowing users to easily implement their data integration processes." -release_latest = "2.48.0" +release_latest = "2.51.0" # The repository and branch where the files live in Github or Colab. This is used # to serve and stage from your local branch, but publish to the master branch. # e.g. https://github.com/{{< param branch_repo >}}/path/to/notebook.ipynb diff --git a/website/www/site/content/en/blog/beam-2.33.0.md b/website/www/site/content/en/blog/beam-2.33.0.md index 8e7237591d1b0..d2afd7a79bcda 100644 --- a/website/www/site/content/en/blog/beam-2.33.0.md +++ b/website/www/site/content/en/blog/beam-2.33.0.md @@ -79,6 +79,9 @@ notes](https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527 * Spark 2.x users will need to update Spark's Jackson runtime dependencies (`spark.jackson.version`) to at least version 2.9.2, due to Beam updating its dependencies. * See a full list of open [issues that affect](https://issues.apache.org/jira/issues/?jql=project%20%3D%20BEAM%20AND%20affectedVersion%20%3D%202.33.0%20ORDER%20BY%20priority%20DESC%2C%20updated%20DESC) this version. * Go SDK jobs may produce "Failed to deduce Step from MonitoringInfo" messages following successful job execution. The messages are benign and don't indicate job failure. These are due to not yet handling PCollection metrics. +* Large Java BigQueryIO writes with the FILE_LOADS method will fail in batch mode (specifically, when copy jobs are used). + This results in the error message: `IllegalArgumentException: Attempting to access unknown side input`. + Please upgrade to a newer version (> 2.34.0) or use another write method (e.g. `STORAGE_WRITE_API`). ## List of Contributors diff --git a/website/www/site/content/en/blog/beam-2.34.0.md b/website/www/site/content/en/blog/beam-2.34.0.md index 2f335a86fd83d..288496aaa8093 100644 --- a/website/www/site/content/en/blog/beam-2.34.0.md +++ b/website/www/site/content/en/blog/beam-2.34.0.md @@ -61,6 +61,12 @@ notes](https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12319527 * Fixed error when importing the DataFrame API with pandas 1.0.x installed ([BEAM-12945](https://issues.apache.org/jira/browse/BEAM-12945)). * Fixed top.SmallestPerKey implementation in the Go SDK ([BEAM-12946](https://issues.apache.org/jira/browse/BEAM-12946)). +### Known Issues + +* Large Java BigQueryIO writes with the FILE_LOADS method will fail in batch mode (specifically, when copy jobs are used). + This results in the error message: `IllegalArgumentException: Attempting to access unknown side input`. + Please upgrade to a newer version (> 2.34.0) or use another write method (e.g. `STORAGE_WRITE_API`). + ## List of Contributors According to git shortlog, the following people contributed to the 2.34.0 release. Thank you to all contributors! diff --git a/website/www/site/content/en/blog/beam-2.47.0.md b/website/www/site/content/en/blog/beam-2.47.0.md index dd962d6824fb1..eaf99ac7122e1 100644 --- a/website/www/site/content/en/blog/beam-2.47.0.md +++ b/website/www/site/content/en/blog/beam-2.47.0.md @@ -66,6 +66,8 @@ For more information on changes in 2.47.0, check out the [detailed release notes ### Known Issues * BigQueryIO Storage API write with autoUpdateSchema may cause data corruption for Beam SDKs 2.45.0 - 2.47.0 (inclusive) ([#26789](https://github.com/apache/beam/issues/26789)) +* Long-running Python pipelines might experience a memory leak: [#28246](https://github.com/apache/beam/issues/28246). + ## List of Contributors @@ -195,4 +197,4 @@ tvalentyn xianhualiu -zhangskz \ No newline at end of file +zhangskz diff --git a/website/www/site/content/en/blog/beam-2.48.0.md b/website/www/site/content/en/blog/beam-2.48.0.md index 06bbe360f517a..11b7f88e99c87 100644 --- a/website/www/site/content/en/blog/beam-2.48.0.md +++ b/website/www/site/content/en/blog/beam-2.48.0.md @@ -63,6 +63,11 @@ For more information on changes in 2.48.0, check out the [detailed release notes * Fixed Java bootloader failing with Too Long Args due to long classpaths, with a pathing jar. (Java) ([#25582](https://github.com/apache/beam/issues/25582)). +## Known Issues + +* PubsubIO writes will throw *SizeLimitExceededException* for any message above 100 bytes, when used in batch (bounded) mode. (Java) ([#27000](https://github.com/apache/beam/issues/27000)). +* Long-running Python pipelines might experience a memory leak: [#28246](https://github.com/apache/beam/issues/28246). + ## List of Contributors @@ -198,4 +203,4 @@ liferoad mokamoka03210120 -psolomin \ No newline at end of file +psolomin diff --git a/website/www/site/content/en/blog/beam-2.49.0.md b/website/www/site/content/en/blog/beam-2.49.0.md new file mode 100644 index 0000000000000..a2e7af0e18f8f --- /dev/null +++ b/website/www/site/content/en/blog/beam-2.49.0.md @@ -0,0 +1,224 @@ +--- +title: "Apache Beam 2.49.0" +date: 2023-07-17 09:00:00 -0400 +categories: + - blog + - release +authors: + - yhu +--- + + +We are happy to present the new 2.49.0 release of Beam. +This release includes both improvements and new functionality. +See the [download page](/get-started/downloads/#2490-2023-07-17) for this release. + + + +For more information on changes in 2.49.0, check out the [detailed release notes](https://github.com/apache/beam/milestone/13). + +## I/Os + +* Support for Bigtable Change Streams added in Java `BigtableIO.ReadChangeStream` ([#27183](https://github.com/apache/beam/issues/27183)). +* Added Bigtable Read and Write cross-language transforms to Python SDK (([#26593](https://github.com/apache/beam/issues/26593)), ([#27146](https://github.com/apache/beam/issues/27146))). + +## New Features / Improvements + +* Allow prebuilding large images when using `--prebuild_sdk_container_engine=cloud_build`, like images depending on `tensorflow` or `torch` ([#27023](https://github.com/apache/beam/pull/27023)). +* Disabled `pip` cache when installing packages on the workers. This reduces the size of prebuilt Python container images ([#27035](https://github.com/apache/beam/pull/27035)). +* Select dedicated avro datum reader and writer (Java) ([#18874](https://github.com/apache/beam/issues/18874)). +* Timer API for the Go SDK (Go) ([#22737](https://github.com/apache/beam/issues/22737)). + + +## Deprecations + +* Remove Python 3.7 support. ([#26447](https://github.com/apache/beam/issues/26447)) + +## Bugfixes + +* Fixed KinesisIO `NullPointerException` when a progress check is made before the reader is started (IO) ([#23868](https://github.com/apache/beam/issues/23868)) + +### Known Issues + +* Long-running Python pipelines might experience a memory leak: [#28246](https://github.com/apache/beam/issues/28246). +* Python SDK's cross-language Bigtable sink mishandles records that don't have an explicit timestamp set: [#28632](https://github.com/apache/beam/issues/28632). To avoid this issue, set explicit timestamps for all records before writing to Bigtable. + + +## List of Contributors + +According to git shortlog, the following people contributed to the 2.49.0 release. Thank you to all contributors! + +Abzal Tuganbay + +AdalbertMemSQL + +Ahmed Abualsaud + +Ahmet Altay + +Alan Zhang + +Alexey Romanenko + +Anand Inguva + +Andrei Gurau + +Arwin Tio + +Bartosz Zablocki + +Bruno Volpato + +Burke Davison + +Byron Ellis + +Chamikara Jayalath + +Charles Rothrock + +Chris Gavin + +Claire McGinty + +Clay Johnson + +Damon + +Daniel Dopierała + +Danny McCormick + +Darkhan Nausharipov + +David Cavazos + +Dip Patel + +Dmitry Repin + +Gavin McDonald + +Jack Dingilian + +Jack McCluskey + +James Fricker + +Jan Lukavský + +Jasper Van den Bossche + +John Casey + +John Gill + +Joseph Crowley + +Kanishk Karanawat + +Katie Liu + +Kenneth Knowles + +Kyle Galloway + +Liam Miller-Cushon + +MakarkinSAkvelon + +Masato Nakamura + +Mattie Fu + +Michel Davit + +Naireen Hussain + +Nathaniel Young + +Nelson Osacky + +Nick Li + +Oleh Borysevych + +Pablo Estrada + +Reeba Qureshi + +Reuven Lax + +Ritesh Ghorse + +Robert Bradshaw + +Robert Burke + +Rouslan + +Saadat Su + +Sam Rohde + +Sam Whittle + +Sanil Jain + +Shunping Huang + +Smeet nagda + +Svetak Sundhar + +Timur Sultanov + +Udi Meiri + +Valentyn Tymofieiev + +Vlado Djerek + +WuA + +XQ Hu + +Xianhua Liu + +Xinyu Liu + +Yi Hu + +Zachary Houfek + +alexeyinkin + +bigduu + +bullet03 + +bzablocki + +jonathan-lemos + +jubebo + +magicgoody + +ruslan-ikhsan + +sultanalieva-s + +vitaly.terentyev + diff --git a/website/www/site/content/en/blog/beam-2.50.0.md b/website/www/site/content/en/blog/beam-2.50.0.md new file mode 100644 index 0000000000000..43bdecdc3ba2a --- /dev/null +++ b/website/www/site/content/en/blog/beam-2.50.0.md @@ -0,0 +1,246 @@ +--- +title: "Apache Beam 2.50.0" +date: 2023-08-30 09:00:00 -0400 +categories: + - blog + - release +authors: + - lostluck +--- + + +We are happy to present the new 2.50.0 release of Beam. +This release includes both improvements and new functionality. +See the [download page](/get-started/downloads/#2500-2023-08-30) for this release. + + + +For more information on changes in 2.50.0, check out the [detailed release notes](https://github.com/apache/beam/milestone/14). + +## Highlights + +* Spark 3.2.2 is used as default version for Spark runner ([#23804](https://github.com/apache/beam/issues/23804)). +* The Go SDK has a new default local runner, called Prism ([#24789](https://github.com/apache/beam/issues/24789)). +* All Beam released container images are now [multi-arch images](https://cloud.google.com/kubernetes-engine/docs/how-to/build-multi-arch-for-arm#what_is_a_multi-arch_image) that support both x86 and ARM CPU architectures. + +## I/Os + +* Java KafkaIO now supports picking up topics via topicPattern ([#26948](https://github.com/apache/beam/pull/26948)) +* Support for read from Cosmos DB Core SQL API ([#23604](https://github.com/apache/beam/issues/23604)) +* Upgraded to HBase 2.5.5 for HBaseIO. (Java) ([#27711](https://github.com/apache/beam/issues/19554)) +* Added support for GoogleAdsIO source (Java) ([#27681](https://github.com/apache/beam/pull/27681)). + +## New Features / Improvements + +* The Go SDK now requires Go 1.20 to build. ([#27558](https://github.com/apache/beam/issues/27558)) +* The Go SDK has a new default local runner, Prism. ([#24789](https://github.com/apache/beam/issues/24789)). + * Prism is a portable runner that executes each transform independantly, ensuring coders. + * At this point it supercedes the Go direct runner in functionality. The Go direct runner is now deprecated. + * See https://github.com/apache/beam/blob/master/sdks/go/pkg/beam/runners/prism/README.md for the goals and features of Prism. +* Hugging Face Model Handler for RunInference added to Python SDK. ([#26632](https://github.com/apache/beam/pull/26632)) +* Hugging Face Pipelines support for RunInference added to Python SDK. ([#27399](https://github.com/apache/beam/pull/27399)) +* Vertex AI Model Handler for RunInference now supports private endpoints ([#27696](https://github.com/apache/beam/pull/27696)) +* MLTransform transform added with support for common ML pre/postprocessing operations ([#26795](https://github.com/apache/beam/pull/26795)) +* Upgraded the Kryo extension for the Java SDK to Kryo 5.5.0. This brings in bug fixes, performance improvements, and serialization of Java 14 records. ([#27635](https://github.com/apache/beam/issues/27635)) +* All Beam released container images are now [multi-arch images](https://cloud.google.com/kubernetes-engine/docs/how-to/build-multi-arch-for-arm#what_is_a_multi-arch_image) that support both x86 and ARM CPU architectures. ([#27674](https://github.com/apache/beam/issues/27674)). The multi-arch container images include: + * All versions of Go, Python, Java and Typescript SDK containers. + * All versions of Flink job server containers. + * Java and Python expansion service containers. + * Transform service controller container. + * Spark3 job server container. +* Added support for batched writes to AWS SQS for improved throughput (Java, AWS 2).([#21429](https://github.com/apache/beam/issues/21429)) + +## Breaking Changes + +* Python SDK: Legacy runner support removed from Dataflow, all pipelines must use runner v2. +* Python SDK: Dataflow Runner will no longer stage Beam SDK from PyPI in the `--staging_location` at pipeline submission. Custom container images that are not based on Beam's default image must include Apache Beam installation.([#26996](https://github.com/apache/beam/issues/26996)) + +## Deprecations + +* The Go Direct Runner is now Deprecated. It remains available to reduce migration churn. + * Tests can be set back to the direct runner by overriding TestMain: `func TestMain(m *testing.M) { ptest.MainWithDefault(m, "direct") }` + * It's recommended to fix issues seen in tests using Prism, as they can also happen on any portable runner. + * Use the generic register package for your pipeline DoFns to ensure pipelines function on portable runners, like prism. + * Do not rely on closures or using package globals for DoFn configuration. They don't function on portable runners. + +## Bugfixes + +* Fixed DirectRunner bug in Python SDK where GroupByKey gets empty PCollection and fails when pipeline option `direct_num_workers!=1`.([#27373](https://github.com/apache/beam/pull/27373)) +* Fixed BigQuery I/O bug when estimating size on queries that utilize row-level security ([#27474](https://github.com/apache/beam/pull/27474)) +* Beam Python containers rely on a version of Debian/aom that has several security vulnerabilities: [CVE-2021-30474](https://nvd.nist.gov/vuln/detail/CVE-2021-30474), [CVE-2021-30475](https://nvd.nist.gov/vuln/detail/CVE-2021-30475), [CVE-2021-30473](https://nvd.nist.gov/vuln/detail/CVE-2021-30473), [CVE-2020-36133](https://nvd.nist.gov/vuln/detail/CVE-2020-36133), [CVE-2020-36131](https://nvd.nist.gov/vuln/detail/CVE-2020-36131), [CVE-2020-36130](https://nvd.nist.gov/vuln/detail/CVE-2020-36130), and [CVE-2020-36135](https://nvd.nist.gov/vuln/detail/CVE-2020-36135). + +## Known Issues + +* Long-running Python pipelines might experience a memory leak: [#28246](https://github.com/apache/beam/issues/28246). +* Python Pipelines using BigQuery IO or `orjson` dependency might experience segmentation faults or get stuck: [#28318](https://github.com/apache/beam/issues/28318). +* Python SDK's cross-language Bigtable sink mishandles records that don't have an explicit timestamp set: [#28632](https://github.com/apache/beam/issues/28632). To avoid this issue, set explicit timestamps for all records before writing to Bigtable. + + +## List of Contributors + +According to git shortlog, the following people contributed to the 2.50.0 release. Thank you to all contributors! + +Abacn + +acejune + +AdalbertMemSQL + +ahmedabu98 + +Ahmed Abualsaud + +al97 + +Aleksandr Dudko + +Alexey Romanenko + +Anand Inguva + +Andrey Devyatkin + +Anton Shalkovich + +ArjunGHUB + +Bjorn Pedersen + +BjornPrime + +Brett Morgan + +Bruno Volpato + +Buqian Zheng + +Burke Davison + +Byron Ellis + +bzablocki + +case-k + +Celeste Zeng + +Chamikara Jayalath + +Clay Johnson + +Connor Brett + +Damon + +Damon Douglas + +Dan Hansen + +Danny McCormick + +Darkhan Nausharipov + +Dip Patel + +Dmytro Sadovnychyi + +Florent Biville + +Gabriel Lacroix + +Hai Joey Tran + +Hong Liang Teoh + +Jack McCluskey + +James Fricker + +Jeff Kinard + +Jeff Zhang + +Jing + +johnjcasey + +jon esperanza + +Josef Šimánek + +Kenneth Knowles + +Laksh + +Liam Miller-Cushon + +liferoad + +magicgoody + +Mahmud Ridwan + +Manav Garg + +Marco Vela + +martin trieu + +Mattie Fu + +Michel Davit + +Moritz Mack + +mosche + +Peter Sobot + +Pranav Bhandari + +Reeba Qureshi + +Reuven Lax + +Ritesh Ghorse + +Robert Bradshaw + +Robert Burke + +RyuSA + +Saba Sathya + +Sam Whittle + +Steven Niemitz + +Steven van Rossum + +Svetak Sundhar + +Tony Tang + +Valentyn Tymofieiev + +Vitaly Terentyev + +Vlado Djerek + +Yichi Zhang + +Yi Hu + +Zechen Jiang + diff --git a/website/www/site/content/en/blog/beam-2.51.0.md b/website/www/site/content/en/blog/beam-2.51.0.md new file mode 100644 index 0000000000000..aaa4142bae625 --- /dev/null +++ b/website/www/site/content/en/blog/beam-2.51.0.md @@ -0,0 +1,210 @@ +--- +title: "Apache Beam 2.51.0" +date: 2023-10-11 09:00:00 -0400 +categories: + - blog + - release +authors: + - klk +--- + + +We are happy to present the new 2.51.0 release of Beam. +This release includes both improvements and new functionality. +See the [download page](/get-started/downloads/#2510-2023-10-03) for this release. + + + +For more information on changes in 2.51.0, check out the [detailed release notes](https://github.com/apache/beam/milestone/15). + +## New Features / Improvements + +* In Python, [RunInference](https://beam.apache.org/documentation/sdks/python-machine-learning/#why-use-the-runinference-api) now supports loading many models in the same transform using a [KeyedModelHandler](https://beam.apache.org/documentation/sdks/python-machine-learning/#use-a-keyed-modelhandler) ([#27628](https://github.com/apache/beam/issues/27628)). +* In Python, the [VertexAIModelHandlerJSON](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.inference.vertex_ai_inference.html#apache_beam.ml.inference.vertex_ai_inference.VertexAIModelHandlerJSON) now supports passing in inference_args. These will be passed through to the Vertex endpoint as parameters. +* Added support to run `mypy` on user pipelines ([#27906](https://github.com/apache/beam/issues/27906)) + + +## Breaking Changes + +* Removed fastjson library dependency for Beam SQL. Table property is changed to be based on jackson ObjectNode (Java) ([#24154](https://github.com/apache/beam/issues/24154)). +* Removed TensorFlow from Beam Python container images [PR](https://github.com/apache/beam/pull/28424). If you have been negatively affected by this change, please comment on [#20605](https://github.com/apache/beam/issues/20605). +* Removed the parameter `t reflect.Type` from `parquetio.Write`. The element type is derived from the input PCollection (Go) ([#28490](https://github.com/apache/beam/issues/28490)) +* Refactor BeamSqlSeekableTable.setUp adding a parameter joinSubsetType. [#28283](https://github.com/apache/beam/issues/28283) + + +## Bugfixes + +* Fixed exception chaining issue in GCS connector (Python) ([#26769](https://github.com/apache/beam/issues/26769#issuecomment-1700422615)). +* Fixed streaming inserts exception handling, GoogleAPICallErrors are now retried according to retry strategy and routed to failed rows where appropriate rather than causing a pipeline error (Python) ([#21080](https://github.com/apache/beam/issues/21080)). +* Fixed a bug in Python SDK's cross-language Bigtable sink that mishandled records that don't have an explicit timestamp set: [#28632](https://github.com/apache/beam/issues/28632). + + +## Security Fixes +* Python containers updated, fixing [CVE-2021-30474](https://nvd.nist.gov/vuln/detail/CVE-2021-30474), [CVE-2021-30475](https://nvd.nist.gov/vuln/detail/CVE-2021-30475), [CVE-2021-30473](https://nvd.nist.gov/vuln/detail/CVE-2021-30473), [CVE-2020-36133](https://nvd.nist.gov/vuln/detail/CVE-2020-36133), [CVE-2020-36131](https://nvd.nist.gov/vuln/detail/CVE-2020-36131), [CVE-2020-36130](https://nvd.nist.gov/vuln/detail/CVE-2020-36130), and [CVE-2020-36135](https://nvd.nist.gov/vuln/detail/CVE-2020-36135) +* Used go 1.21.1 to build, fixing [CVE-2023-39320](https://security-tracker.debian.org/tracker/CVE-2023-39320) + + +## Known Issues + +* Python pipelines using BigQuery Storage Read API must pin `fastavro` dependency to 1.8.3 + or earlier: [#28811](https://github.com/apache/beam/issues/28811) + +## List of Contributors + +According to git shortlog, the following people contributed to the 2.50.0 release. Thank you to all contributors! + +Adam Whitmore + +Ahmed Abualsaud + +Ahmet Altay + +Aleksandr Dudko + +Alexey Romanenko + +Anand Inguva + +Andrey Devyatkin + +Arvind Ram + +Arwin Tio + +BjornPrime + +Bruno Volpato + +Bulat + +Celeste Zeng + +Chamikara Jayalath + +Clay Johnson + +Damon + +Danny McCormick + +David Cavazos + +Dip Patel + +Hai Joey Tran + +Hao Xu + +Haruka Abe + +Jack Dingilian + +Jack McCluskey + +Jeff Kinard + +Jeffrey Kinard + +Joey Tran + +Johanna Öjeling + +Julien Tournay + +Kenneth Knowles + +Kerry Donny-Clark + +Mattie Fu + +Melissa Pashniak + +Michel Davit + +Moritz Mack + +Pranav Bhandari + +Rebecca Szper + +Reeba Qureshi + +Reuven Lax + +Ritesh Ghorse + +Robert Bradshaw + +Robert Burke + +Ruwann + +Ryan Tam + +Sam Rohde + +Sereana Seim + +Svetak Sundhar + +Tim Grein + +Udi Meiri + +Valentyn Tymofieiev + +Vitaly Terentyev + +Vlado Djerek + +Xinyu Liu + +Yi Hu + +Zbynek Konecny + +Zechen Jiang + +bzablocki + +caneff + +dependabot[bot] + +gDuperran + +gabry.wu + +johnjcasey + +kberezin-nshl + +kennknowles + +liferoad + +lostluck + +magicgoody + +martin trieu + +mosche + +olalamichelle + +tvalentyn + +xqhu + +Łukasz Spyra diff --git a/website/www/site/content/en/blog/beamquest.md b/website/www/site/content/en/blog/beamquest.md index eea893bf82274..dde6376b40771 100644 --- a/website/www/site/content/en/blog/beamquest.md +++ b/website/www/site/content/en/blog/beamquest.md @@ -34,6 +34,6 @@ Individuals aren’t the only ones who can benefit from completing this quest - Data Processing is a key part of AI/ML workflows. Given the recent advancements in artificial intelligence, now’s the time to jump into the world of data processing! Get started on your journey [here](https://www.cloudskillsboost.google/quests/310). -We are currently offering this quest **FREE OF CHARGE** until **July 8, 2023** for the **first 2,000** people. To obtain your badge for **FREE**, use the [Access Code](https://www.cloudskillsboost.google/catalog?qlcampaign=1h-swiss-19), create an account, and search ["Getting Started with Apache Beam"](https://www.cloudskillsboost.google/quests/310). +We are currently offering this quest **FREE OF CHARGE**. To obtain your badge for **FREE**, use the [Access Code](https://www.cloudskillsboost.google/catalog?qlcampaign=1h-swiss-19), create an account, and search ["Getting Started with Apache Beam"](https://www.cloudskillsboost.google/quests/310). If the code does not work, please email [dev@beam.apache.org](dev@beam.apache.org) to obtain a free code. PS: Once you earn your badge, please [share it on social media](https://support.google.com/qwiklabs/answer/9222527?hl=en&sjid=14905615709060962899-NA)! diff --git a/website/www/site/content/en/blog/dyi-content-discovery-platform-genai-beam.md b/website/www/site/content/en/blog/dyi-content-discovery-platform-genai-beam.md new file mode 100644 index 0000000000000..fd967e318a070 --- /dev/null +++ b/website/www/site/content/en/blog/dyi-content-discovery-platform-genai-beam.md @@ -0,0 +1,338 @@ +--- +layout: post +title: "DIY GenAI Content Discovery Platform with Apache Beam" +date: 2023-10-02 00:00:01 -0800 +categories: + - blog +authors: + - pabs + - namitasharma +--- + + +# DIY GenAI Content Discovery Platform with Apache Beam + +Your digital assets, such as documents, PDFs, spreadsheets, and presentations, contain a wealth of valuable information, but sometimes it's hard to find what you're looking for. This blog post explains how to build a DIY starter architecture, based on near real-time ingestion processing and large language models (LLMs), to extract meaningful information from your assets. The model makes the information available and discoverable through a simple natural language query. + +Building a near real-time processing pipeline for content ingestion might seem like a complex task, and it can be. To make pipeline building easier, the Apache Beam framework exposes a set of powerful constructs. These constructs remove the following complexities: interacting with multiple types of content sources and destinations, error handling, and modularity. They also maintain resiliency and scalability with minimal effort. You can use an Apache Beam streaming pipeline to complete the following tasks: + +- Connect to the many components of a solution. +- Quickly process content ingestion requests of documents. +- Make the information in the documents available a few seconds after ingestion. + +LLMs are often used to extract content and summarize information stored in many different places. Organizations can use LLMs to quickly find relevant information disseminated in multiple documents written across the years. The information might be in different formats, or the documents might be too long and complex to read and understand quickly. Use LLMs to process this content to make it easier for people to find the information that they need. + +Follow the steps in this guide to create a custom scalable solution for data extraction, content ingestion, and storage. Learn how to kickstart the development of a LLM-based solution using Google Cloud products and generative AI offerings. Google Cloud is designed to be simple to use, scalable, and flexible, so you can use it as a starting point for further expansion or experimentation. + +### High-level Flow + +In this workflow, content uptake and query interactions are completely separated. An external content owner can send documents stored in Google Docs or in a binary text format and receive a tracking ID for the ingestion request. The ingestion process gets the content of the document and creates chunks that are configurable in size. Each document chunk is used to generate embeddings. These embeddings represent the content semantics, in the form of a vector of 768 dimensions. Given the document identifier and the chunk identifier, you can store the embeddings in a Vector database for semantic matching. This process is central to contextualizing user inquiries. + +Content Discovery Platform Overview + +The query resolution process doesn't depend directly on information ingestion. The user receives relevant answers based on the content ingested until the moment of the query request. Even if the platform doesn't have any relevant content stored, the platform returns an answer stating that it doesn't have relevant content. Therefore, the query resolution process first generates embeddings from the query content and from the previously existing context, like previous exchanges with the platform, then matches these embeddings with the existing embedding vectors stored from the content. When the platform has positive matches, it retrieves the plain-text content represented by the content embeddings. Finally, by using the textual representation of the query and the textual representation of the matched content, the platform formulates a request to the LLM to provide a final answer to the original user inquiry. + +## Components of the solution + +Use the low-ops capabilities of the Google Cloud services to create a set of highly scalable features. You can separate the solution into two main components: the service layer and the content ingestion pipeline. The service layer acts as the entry point for document ingestion and user queries. It’s a simple set of REST resources exposed through Cloud Run and implemented by using [Quarkus](https://quarkus.io/) and the client libraries to access other services (Vertex AI models, Cloud Bigtable and Pub/Sub). The content ingestion pipeline includes the following components: + +* A streaming pipeline that captures user content from wherever it resides. +* A process that extracts meaning from this content as a set of multi-dimensional vectors (text embeddings). +* A storage system that simplifies context matching between knowledge content and user inquiries (a Vector Database). +* Another storage system that maps knowledge representation with the actual content, forming the aggregated context of the inquiry. +* A model capable of understanding the aggregated context and, through prompt engineering, delivering meaningful answers. +* HTTP and gRPC-based services. + +Together, these components provide a comprehensive and simple implementation for a content discovery platform. + +## Workflow Architecture + +This section explains how the different components interact. + +### Dependencies of the components + +The following diagram shows all of the components that the platform integrates with. It also shows all of the dependencies that exist between the components of the solution and the Google Cloud services. + +Content Discovery Platform Interactions + +As seen in the diagram, the context-extraction component is the central aspect in charge of retrieving the document’s content, also their semantic meaning from the embedding’s model and storing the relevant data (chunks text content, chunks embeddings, JSON-L content) in the persistent storage systems for later use. PubSub resources are the glue between the streaming pipeline and the asynchronous processing, capturing the user ingestion requests, retries from potential errors from the ingestion pipeline (like the cases on where documents have been sent for ingestion but the permission has not been granted yet, triggering a retry after some minutes) and content refresh events (periodically the pipeline will scan the ingested documents, review the latest editions and define if a content refresh should be triggered). + +The context-extraction component retrieves the content of the documents, diving it in chunks. It also computes embeddings, using the LLM interaction, from the extracted content. Then it stores the relevant data (chunks text content, chunks embeddings, JSON-L content) in the persistent storage systems for later use. Pub/Sub resources connect the streaming pipeline and the asynchronous processing, capturing the following actions: +- user ingestion requests +- retries from errors from the ingestion pipeline, such as when documents are sent for ingestion but access permissions are missing +- content refresh events (periodically the pipeline scans the ingested documents, reviews the latest editions, and decides whether to trigger a content refresh) + +Also, CloudRun plays an important role exposing the services, interacting with many Google Cloud services to resolve the user query or ingestion requests. For example, while resolving a query request the service will: +- Request the computation of embeddings from the user’s query by interacting with the embeddings model +- Find near neighbor matches from the Vertex AI Vector Search (formerly Matching Engine) using the query embeddings representation +- Retrieve the text content from BigTable for those matched vectors, using their identifier, in order contextualize a LLM prompt +- And finally create a request to the VertexAI Chat-Bison model, generating the response the system will delivery to the user’s query. + +### Google Cloud products + +This section describes the Google Cloud products and services used in the solution and what purpose they serve. + +**Cloud Build:** All container images, including services and pipelines, are built directly from source code by using Cloud Build. Using Cloud Build simplifies code distribution during the deployment of the solution. + +**CloudRun:** The solution's service entry points are deployed and automatically scaled by CloudRun. + +**Pub/Sub:** A Pub/Sub topic and subscription queue all of the ingestion requests for Google Drive or self-contained content and deliver the requests to the pipeline. + +**Dataflow:** A multi-language, streaming Apache Beam pipeline processes the ingestion requests. These requests are sent to the pipeline from the Pub/Sub subscription. The pipeline extracts content from Google Docs, Google Drive URLs, and self-contained binary encoded text content. It then produces content chunks. These chunks are sent to one of the Vertex AI foundational models for the embedding representation. The embeddings and chunks from the documents are sent to Vertex AI Vector Search and to Cloud Bigtable for indexing and rapid access. Finally, the ingested documentation is stored in Google Cloud Storage in JSON-L format, which can be used to fine-tune the Vertex AI models. By using Dataflow to run the Apache Beam streaming pipeline, you minimize the ops needed to scale resources. If you have a burst on ingestion requests, Dataflow can keep the latency less than a minute. + +**Vertex AI - Vector Search:** [Vector Search](https://cloud.google.com/vertex-ai/docs/matching-engine/overview) is a high-performance, low-latency vector database. These vector databases are often called vector similarity search or approximate nearest neighbor (ANN) services. We use a Vector Search Index to store all the ingested documents embeddings as a meaning representation. These embeddings are indexed by chunk and document id. Later on, these identifiers can be used to contextualize the user queries and enrich the requests made to a LLM by providing knowledge extracted directly from the document’s content mappings stored on BigTable (using the same chunk-document identifiers). + +**Cloud BigTable:** This storage system provides a low latency search by identifier at a predictable scale. Is a perfect fit, given the low latency of the requests resolution, for online exchanges between user queries and the platform component interactions. It used to store the content extracted from the documents since it's indexed by chunk and document identifier. Every time a user makes a request to the query service, and after the query text embeddings are resolved and matched with the existing context, the document and chunk ids are used to retrieve the document’s content that will be used as context to request an answer to the LLM in use. Also, BigTable is used to keep track of the conversational exchanges between users and the platform, furthermore enriching the context included on the requests sent to the LLMs (embeddings, summarization, chat Q&A). + +**Vertex AI - Text Embedding Model:** [Text embeddings](https://cloud.google.com/vertex-ai/docs/generative-ai/model-reference/text-embeddings) are a condensed vector (numeric) representation of a piece of text. If two pieces of text are semantically similar, their corresponding embeddings will be located close together in the embedding vector space. For more details please see [get text embeddings](https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-text-embeddings). These embeddings are directly used by the ingestion pipeline when processing the document’s content and the query service as an input to match the users query semantic with existing content indexed in Vector Search. + +**Vertex AI - Text Summarization Model:** [Text-bison](https://cloud.google.com/vertex-ai/docs/generative-ai/model-reference/text) is the name of the PaLM 2 LLM that understands, summarizes and generates text. The types of content that text-bison can create include document summaries, answers to questions, and labels that classify the provided input content. We used this LLM to summarize the previously maintained conversation with the goal of enriching the user’s queries and better embedding matching. In summary, the user does not have to include all the context of his question, we extract and summarize it from the conversation history. + +**Vertex AI - Text Chat Model:** [Chat-bison](https://cloud.google.com/vertex-ai/docs/generative-ai/model-reference/text-chat) is the PaLM 2 LLM that excels at language understanding, language generation, and conversations. This chat model is fine-tuned to conduct natural multi-turn conversations, and is ideal for text tasks about code that require back-and-forth interactions. We use this LLM to provide answers to the queries made by users of the solution, including the conversation history between both parties and enriching the model’s context with the content stored in the solution. + +### Extraction Pipeline + +The content extraction pipeline is the platform's centerpiece. It takes care of handling content ingestion requests, extracting documents content and computing embeddings from that content, to finally store the data in specialized storage systems that will be used in the query service components for rapid access. + +#### High Level View + +As previously mentioned the pipeline is implemented using Apache Beam framework and runs in streaming fashion on GCP's [Dataflow](https://cloud.google.com/dataflow) service. + +By using Apache Beam and Dataflow we can ensure minimal latency (sub minute processing times), low ops (no need to manually scale up or down the pipeline when traffic spikes occur with time, worker recycle, updates, etc.) and with high level of observability (clear and abundant performance metrics are available). + +Apache Beam Pipeline + +On a high level, the pipeline separates the extraction, computing, error handling and storage responsibilities on different components or PTransforms. As seen in the diagram, the messages are read from a PubSub subscription and immediately afterwards are included in the window definition before the content extraction. + +Each of those PTransforms can be expanded to reveal more details regarding the underlying stages for the implementation. We will dive into each in the following sections. + +The pipeline was implemented using a multi-language approach, with the main components written in the Java language (JDK version 17) and those related with the embeddings computations implemented in Python (version 3.11) since the Vertex AI API clients are available for this language. + +#### Content Extraction + +The content extraction component is in charge of reviewing the ingestion request payload and deciding (given the event properties) if it will need to retrieve the content from the event itself (self-contained content, text based document binary encoded) or retrieve it from Google Drive. + +Pipeline's Content Extraction + +In case of a self-contained document, the pipeline will extract the document id and format the document in paragraphs for later embedding processing. + +When in need of retrieval from Google Drive, the pipeline will inspect if the provided URL in the event refers to a Google Drive folder or a single file format (supported formats are Documents, Spreadsheets and Presentations). In the case of a folder, the pipeline will crawl the folder’s content recursively extracting all the files for the supported formats, in case of a single document will just return that one. + +Finally, with all the file references retrieved from the ingestion request, textual content is extracted from the files (no image support implemented for this PoC). That content will also be passed to the embedding processing stages including the document’s identifier and the content as paragraphs. + +#### Error Handling + +On every stage of the content extraction process multiple errors can be encountered, malformed ingestion requests, non-conformant URLs, lack of permissions for Drive resources, lack of permissions for File data retrieval. + +In all those cases a dedicated component will capture those potential errors and define, given the nature of the error, if the event should be retried or sent to a dead letter GCS bucket for later inspection. + +Pipeline's Error Handling + +The final errors, or those which won’t be retried, are those errors related with bad request formats (the event itself or the properties content, like malformed or wrong URLs, etc.). + +The retryable errors are those related with content access and lack of permissions. A request may have been resolved faster than the manual process of providing the right permissions to the Service Account that runs the pipeline to access the resources included in the ingestion request (Google Drive folders or files). In case of detecting a retryable error, the pipeline will hold the retry for 10 minutes before re-sending the message to the upstream PubSub topic; each error is retried at most 5 times before being sent to the dead letter GCS bucket. + +In all cases of events ending on the dead letter destination, the inspection and re-processing must be done in a manual process. + +#### Process Embeddings + +Once the content has been extracted from the request, or captured from Google Drive files, the pipeline will trigger the embeddings computation process. As previously mentioned the interactions with the Vertex AI Foundational Models API is implemented in Python language. For this reason we need to format the extracted content in Java types that have a direct translation to those existing in the Python world. Those are key-values (in Python those are 2-element tuples), Strings (available in both languages), and iterables (also available in both languages). We could have implemented coders in both languages to support custom transport types, but we opted out of that in favor of clarity and simplicity. + +Before computing the content’s embeddings we decided to introduce a Reshuffle step, making the output consistent to downstream stages, with the idea of avoiding the content extraction step being repeated in case of errors. This should avoid putting pressure on existing access quotas on Google Drive related APIs. + +The pipeline will then chunk the content in configurable sizes and also configurable overlapping, good parameters are hard to get for generic effective data extraction, so we opted to use smaller chunks with small overlapping factor as the default settings to favor diversity on the document results (at least that’s what we see from the empirical results obtained). + +

    + Embeddings Processing + Embeddings Processing +

    + +Once the embeddings vectors are retrieved from the embeddings Vertex AI LLM, we will consolidate them again avoiding repetition of this step in case of downstream errors. + +Worth to notice that this pipeline is interacting directly with Vertex AI models using the client SDKs, Apache Beam already provides supports for this interactions through the RunInference PTransform (see an example [here](https://github.com/apache/beam/blob/master/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py)). + +#### Content Storage + +Once the embeddings are computed for the content chunks extracted from the ingested documents, we need to store the vectors in a searchable storage and also the textual content that correlates with those embeddings. We will be using the embeddings vectors as a semantic match later from the query service, and the textual content that corresponds to those embeddings for LLM context as a way to improve and guide the response expectations. + +Content Storage + +With that in mind is that in mind we split the consolidated embeddings into 3 paths, one that stores the vectors into Vertex AI Vector Search (using simple REST calls), another storing the textual content into BigTable (for low latency retrieval after semantic matching) and the final one as a potential clean up of content refresh or re ingestion (more on that later). The three paths are using the ingested document identifier as the correlating data on the actions, this key is formed by the document name (in case of available), the document identifier and the chunk sequence number. The reason for using identifiers for the chunk comes behind the idea of subsequent updates. An increase in the content will generate a larger number of chunks, and upserting all the chunks will enable always fresh data; on the contrary, a decrease in content will generate a smaller chunk count for the document’s content, this number difference can be used to delete the remaining orphan indexed chunks (from content no longer existing in the latest version of the document). + +#### Content Refresh + +The last pipeline component is the simplest, at least conceptually. After the documents from Google Drive gets ingested, an external user can produce updates in them, causing the indexed content to become out of date. We implemented a simple periodic process, inside the same streaming pipeline, that will take care of the review of already ingested documents and see if there are content updates needed. We use a GenerateSequence transform to produce a periodic impulse (every 6 hours by default), that will trigger a scan on BigTable retrieving all the ingested document identifiers. Given those identifiers we can then query Google Drive for the latest update timestamp of each document and use that marker to decide if an update is needed. + +In case of needing to update the document’s content, we can simply send an ingestion request to the upstream PubSub topic and let the pipeline run its course for this new event. Since we are taking care of upserting embeddings and cleaning up those that no longer exist, we should be capable of taking care of the majority of the additions (as long those are text updates, image based content is not being processed as of now). + +

    + Content Refresh + Content Refresh + Content Refresh +

    + +This task could be performed as a separate job, possibly one that is periodically scheduled in batch form. This would result in lower costs, a separate error domain, and more predictable auto scaling behavior. However, for the purposes of this demonstration, it is simpler to have a single job. + +Next, we will be focusing on how the solution interacts with external clients for ingestion and content discovery use cases. + +## Interaction Design + +The solution aims to make the interactions for ingesting and querying the platform as simple as possible. Also, since the ingestion part may imply interacting with several services and imply retries or content refresh, we decided to make both separated and asynchronous, freeing the external users of blocking themselves while waiting for requests resolutions. + +### Example Interactions + +Once the platform is deployed in a GCP project, a simple way to interact with the services is through the use of a web client, curl is a good example. Also, since the endpoints are authenticated, a client needs to include its credentials in the request header to have its access granted. + +Here is an example of an interaction for content ingestion: + +``` +$ > curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer $(gcloud auth print-identity-token)" https:///ingest/content/gdrive -d $'{"url":"https://drive.google.com/drive/folders/somefolderid"}' | jq . + +# response from service +{ + "status": "Ingestion trace id: " +} +``` + +In this case, after the ingestion request has been sent to the PubSub topic for processing, the service will return the tracking identifier, which maps with the PubSub message identifier. Note the provided URL can be one of a Google Doc or a Google Drive folder, in the later case the ingestion process will crawl the folder’s content recursively to retrieve all the contained documents and their contents. + +Next, an example of a content query interaction, very similar to the previous one: + +``` +$ > curl -X POST \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $(gcloud auth print-identity-token)" \ + https:///query/content \ + -d $'{"text":"summarize the benefits of using VertexAI foundational models for Generative AI applications", "sessionId": ""}' \ + | jq . + +# response from service +{ + "content": "VertexAI Foundation Models are a set of pre-trained models that can be used to accelerate the development of machine learning applications. They are available for a variety of tasks, including natural language processing, computer vision, and recommendation systems.\n\nVertexAI Foundation Models can be used to improve the performance of Generative AI applications by providing a starting point for model development. They can also be used to reduce the amount of time and effort required to train a model.\n\nIn addition, VertexAI Foundation Models can be used to improve the accuracy and robustness of Generative AI applications. This is because they are trained on large datasets and are subject to rigorous quality control.\n\nOverall, VertexAI Foundation Models can be a valuable resource for developers who are building Generative AI applications. They can help to accelerate the development process, reduce the cost of development, and improve the performance and accuracy of applications.", + "previousConversationSummary": "", + "sourceLinks": [ + { + "link": "", + "distance": 0.7233397960662842 + } + ], + "citationMetadata": [ + { + "citations": [] + } + ], + "safetyAttributes": [ + { + "categories": [], + "scores": [], + "blocked": false + } + ] +} +``` + +The platform will answer the request with a textual response from the LLM and include as well information about the categorization, citation metadata and source links (if available) of the content used to generate the response (this are for example, Google Docs links of the documents previously ingested by the platform). + +When interacting with the services, a good query will generally return good results, the clearer the query the easier it will be to contextualize its meaning and more accurate information will be sent to the LLMs to retrieve answers. But having to include all the details of the query context in a phrase on every exchange with the service can be very cumbersome and difficult. For that case the platform can use a provided session identifier that will be used to store all the previous exchanges between a user and the platform. This should help the implementation to better contextualize the initial query embeddings matching and even provide more concise contextual information in the model requests. Here is an example of a contextual exchange: + +``` +$ > curl -X POST \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $(gcloud auth print-identity-token)" \ + https:///query/content \ + -d $'{"text":"summarize the benefits of using VertexAI foundational models for Generative AI applications?", "sessionId": "some-session-id"}' \ + | jq . + +# response from service +{ + "content": "VertexAI Foundational Models are a suite of pre-trained models that can be used to accelerate the development of Generative AI applications. These models are available in a variety of languages and domains, and they can be used to generate text, images, audio, and other types of content.\n\nUsing VertexAI Foundational Models can help you to:\n\n* Reduce the time and effort required to develop Generative AI applications\n* Improve the accuracy and quality of your models\n* Access the latest research and development in Generative AI\n\nVertexAI Foundational Models are a powerful tool for developers who want to create innovative and engaging Generative AI applications.", + … +} + +$ > curl -X POST \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $(gcloud auth print-identity-token)" \ + https:///query/content \ + -d $'{"text":"describe the available LLM models?", "sessionId": "some-session-id"}' \ + | jq . + +# response from service +{ + "content": "The VertexAI Foundational Models suite includes a variety of LLM models, including:\n\n* Text-to-text LLMs: These models can generate text based on a given prompt. They can be used for tasks such as summarization, translation, and question answering.\n* Image-to-text LLMs: These models can generate text based on an image. They can be used for tasks such as image captioning and description generation.\n* Audio-to-text LLMs: These models can generate text based on an audio clip. They can be used for tasks such as speech recognition and transcription.\n\nThese models are available in a variety of languages, including English, Spanish, French, German, and Japanese. They can be used to create a wide range of Generative AI applications, such as chatbots, customer service applications, and creative writing tools.", + … +} + +$ > curl -X POST \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $(gcloud auth print-identity-token)" \ + https:///query/content \ + -d $'{"text":"do rate limit apply for those LLMs?", "sessionId": "some-session-id"}' \ + | jq . + +# response from service +{ + "content": "Yes, there are rate limits for the VertexAI Foundational Models. The rate limits are based on the number of requests per second and the total number of requests per day. For more information, please see the [VertexAI Foundational Models documentation](https://cloud.google.com/vertex-ai/docs/foundational-models#rate-limits).", + … +} + +$ > curl -X POST \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $(gcloud auth print-identity-token)" \ + https:///query/content \ + -d $'{"text":"care to share the price?", "sessionId": "some-session-id"}' \ + | jq . + +# response from service +{ + "content": "The VertexAI Foundational Models are priced based on the number of requests per second and the total number of requests per day. For more information, please see the [VertexAI Foundational Models pricing page](https://cloud.google.com/vertex-ai/pricing#foundational-models).", + … +} +``` + +**Usage Tip:** in case of abruptly changing topics, sometimes is better to use a new session identifier. + +### Deployment + +As part of the platform solution, there are a set of scripts that help with the deployment of all the different components. By running the `start.sh` and setting the right parameters (GCP project, terraform state bucket and name for the platform instance) the script will take care of building the code, deploying the needed containers (service endpoint container and Dataflow python custom container), deploying all the GCP resources using Terraform and finally deploying the pipeline. There is also the possibility of modifying the pipeline’s execution by passing an extra parameter to the startup script, for example: `start.sh
    "--update"` will update the content extraction pipeline in-place. + +Also, in case of wanting to focus only on the deployment of specific components other scripts have been included to help with those specific tasks (build the solution, deploy the infrastructure, deploy the pipeline, deploy the services, etc.). + +### Solution's Notes + +This solution is designed to serve as an example for learning purposes. Many of the configuration values for the extraction pipeline and security restrictions are provided only as examples. The solution doesn't propagate the existing access control lists (ACLs) of the ingested content. As a result, all users that have access to the service endpoints have access to summarizations of the ingested content from those original documents. + +### Notes about the source code + +The source code for the content discovery platform is available in [Github](https://github.com/prodriguezdefino/content-dicovery-platform-gcp). You can run it in any Google Cloud project. The repository includes the source code for the integration services, the multi-language ingestion pipeline, and the deployment automation through Terraform. If you deploy this example, it might take up to 90 minutes to create and configure all the needed resources. The README file contains additional documentation about the deployment prerequisites and example REST interactions. diff --git a/website/www/site/content/en/blog/managing-beam-dependencies-in-java.md b/website/www/site/content/en/blog/managing-beam-dependencies-in-java.md new file mode 100644 index 0000000000000..225b3eb5981b5 --- /dev/null +++ b/website/www/site/content/en/blog/managing-beam-dependencies-in-java.md @@ -0,0 +1,122 @@ +--- +title: "Managing Beam dependencies in Java" +date: 2023-06-23 9:00:00 -0700 +categories: + - blog +authors: + - bvolpato + +--- + + +Managing your Java dependencies can be challenging, and if not done correctly, +it may cause a variety of problems, as incompatibilities may arise when using +specific and previously untested combinations. + +To make that process easier, Beam now +provides [Bill of Materials (BOM)](https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html#bill-of-materials-bom-poms) +artifacts that will help dependency management tools to select compatible +combinations. + +We hope this will make it easier for you to use Apache Beam, and have a simpler +transition when upgrading to newer versions. + + + +When bringing incompatible classes and libraries, the code is susceptible to +errors such +as `NoClassDefFoundError`, `NoSuchMethodError`, `NoSuchFieldError`, `FATAL ERROR in native method`. + +When importing Apache Beam, the recommended way is to use Bill of Materials +(BOMs). The way BOMs work is by providing hints to the dependency management +resolution tool, so when a project imports unspecified or ambiguous dependencies, +it will know what version to use. + +There are currently two BOMs provided by Beam: + +- `beam-sdks-java-bom`, which manages what dependencies of Beam will be used, so + you can specify the version only once. +- `beam-sdks-java-io-google-cloud-platform-bom`, a more comprehensive list, + which manages Beam, along with GCP client and third-party dependencies. + +Since errors are more likely to arise when using third-party dependencies, +that's the one that is recommended to use to minimize any conflicts. + +In order to use BOM, the artifact has to be imported to your Maven or Gradle +dependency configurations. For example, to +use `beam-sdks-java-io-google-cloud-platform-bom`, +the following changes have to be done (and make sure that _BEAM_VERSION_ is +replaced by a valid version): + +**Maven** + +```xml + + + + org.apache.beam + beam-sdks-java-google-cloud-platform-bom + BEAM_VERSION + pom + import + + + +``` + +**Gradle** + +``` +dependencies { + implementation(platform("org.apache.beam:beam-sdks-java-google-cloud-platform-bom:BEAM_VERSION")) +} +``` + +After importing the BOM, specific version pinning of dependencies, for example, +anything for `org.apache.beam`, `io.grpc`, `com.google.cloud` ( +including `libraries-bom`) may be removed. + +Do not entirely remove the dependencies, as they are not automatically imported +by the BOM. It is important to keep the dependency without specifying a version. +For example, in Maven: + +```xml + + org.apache.beam + beam-sdks-java-core + +``` + +Or Gradle: + +``` +implementation("org.apache.beam:beam-sdks-java-core") +``` + +For a full list of dependency versions that are managed by a specific BOM, the +Maven tool `help:effective-pom` can be used. For example: + +```shell +mvn help:effective-pom -f ~/.m2/repository/org/apache/beam/beam-sdks-java-google-cloud-platform-bom/BEAM_VERSION/beam-sdks-java-google-cloud-platform-bom-BEAM_VERSION.pom +``` + +The third-party +website [mvnrepository.com](https://mvnrepository.com/artifact/org.apache.beam/beam-sdks-java-google-cloud-platform-bom/) +can also be used to display such version information. + +We hope you find this +useful. [Feedback](https://beam.apache.org/community/contact-us/) and +contributions are always welcome! So feel free to create a GitHub issue, or open +a Pull Request if you encounter any problem when using those artifacts. diff --git a/website/www/site/content/en/case-studies/hsbc.md b/website/www/site/content/en/case-studies/hsbc.md new file mode 100644 index 0000000000000..73b693f97e101 --- /dev/null +++ b/website/www/site/content/en/case-studies/hsbc.md @@ -0,0 +1,236 @@ +--- +title: "High-Performance Quantitative Risk Analysis with Apache Beam at HSBC" +name: "HSBC" +icon: "/images/logos/powered-by/hsbc.png" +category: "study" +cardTitle: "High-Performance Quantitative Risk Analysis with Apache Beam at HSBC" +cardDescription: "HSBC finds Apache Beam to be more than a data processing framework. It is also a computational platform and a risk engine that allowed for 100x scaling and 2x faster performance of HSBC’s XVA pipelines, accelerated time-to-market by 24x, and simplified data distribution for modeling future scenarios with Monte Carlo simulations, powering quantitative risk analysis for forecasting and decision-making." +authorName: "Chup Cheng" +coauthorName: "Andrzej Golonka" +authorPosition: "VP of XVA and CCR Capital Analytics @ HSBC" +coauthorPosition: "Lead Assistant Vice President @ HSBC" +authorImg: /images/case-study/hsbc/chup_cheng.jpg +coauthorImg: /images/case-study/hsbc/andrzej_golonka.jpg +publishDate: 2023-06-20T00:12:00+00:00 +--- + +
    +
    + +
    +
    +

    + “Our data volume is huge and Apache Beam helps make it manageable. For each of the millions of trades per day, we generate a simulation of their market valuation evolution at granular level of future time increment, then by scanning multiple plausible market scenarios we aggregate this massive data into meaningful statistics. Apache Beam helps harness all of that data and makes data distribution much easier than before.” +

    +
    +
    + +
    +
    +
    + Andrzej Golonka +
    +
    + Lead Assistant Vice President @ HSBC +
    +
    +
    +
    +
    +

    + “The Apache Beam Python SDK brought math to the orchestration level by providing an easy way for HSBC’s model development team to emulate sophisticated mathematical dependencies between nodes of business logic workflow in pipelines written in Python. We used to spend at least 6 months deploying even small changes to a system of equations in production. With the new team structure driven by Apache Beam, we now can deploy changes as quickly as within 1 week.” +

    +
    +
    + +
    +
    +
    + Chup Cheng +
    +
    + VP of XVA and CCR Capital Analytics @ HSBC +
    +
    +
    +
    +
    +
    + +# High-Performance Quantitative Risk Analysis with Apache Beam at HSBC + +## Background + +[HSBC Holdings plc](https://www.hsbc.com/), the parent company of HSBC, is headquartered in London. HSBC serves customers worldwide from offices in 62 countries and territories. With assets of $2,990bn as of 31 March 2023, HSBC is one of the world’s largest banking and financial services organisations. HSBC’s [Global Banking and Markets business](https://www.gbm.hsbc.com/) provides a wide range of financial services and products to multinational corporates, governments, and financial institutions. + +HSBC’s Chup Cheng, VP of XVA and CCR Capital Analytics, and Andrzej Golonka, Lead Assistant Vice President, shared how Apache Beam serves as a computation platform and a risk engine that helps HSBC manage counterparty credit risk and XVA across their customer investment portfolios, which arises from the trillions of dollars in trading volumes between numerous of counterparties every day. Apache Beam empowered HSBC to [integrate their existing C++ HPC workloads into batch Apache Beam pipelines](https://2022.beamsummit.org/sessions/hpc-grid/), to streamline data distribution, and to improve processing performance. Apache Beam also enabled new pipelines that were not possible before and enhanced development efficiency. + +## Risk Management at Scale + +To realize the scale and value of the Apache Beam-powered data processing at HSBC, let us delve deeper into why Counterparty credit risk calculations at financial institutions require particularly extreme compute capacity. + +The value of an investment portfolio moves along with the financial markets and is impacted by a variety of external factors. To neutralize the risks and determine the capital reserves required by regulations, investment banks need to estimate risk exposure and make corresponding adjustments to the value of individual trades and portfolios. [XVA (X-Value Adjustment) model](https://en.wikipedia.org/wiki/XVA) plays a crucial role in analyzing counterparty credit risk in the financial industry and covers different valuation adjustments, such as [Credit Valuation Adjustment (CVA)](https://corporatefinanceinstitute.com/resources/knowledge/trading-investing/credit-valuation-adjustment-cva/), [Funding Valuation Adjustment (FVA)](https://finpricing.com/lib/cvaFva.html), and [Capital Valuation Adjustment (KVA)](https://www.risk.net/definition/capital-valuation-adjustment-kva#:~:text=Capital%20valuation%20adjustment%20reflects%20the,trades%20that%20are%20not%20cleared.). Calculating XVA involves complex models, multi-layered matrices, and [Monte Carlo simulations](https://en.wikipedia.org/wiki/Monte_Carlo_method) to account for risks based on plausible future scenarios. Valuation functions process multiple [stochastic differential equation (SDE)](https://en.wikipedia.org/wiki/Stochastic_differential_equation) matrices that represent probable trade values in a specific timeframe of up to possibly 70 years, and then output [MTM (mark-to-market)](https://en.wikipedia.org/wiki/Mark-to-market_accounting) matrices that represent the distribution of the current market value of a financial asset depending on future scenarios. Collapsed MTM matrices determine the vector of future risk exposure and the required XVA adjustment. + +XVA calculations require a vast amount of computational capacity due to the extensive matrix data and long time horizons involved. To calculate MTM matrix for one trade, a valuation function needs to iterate hundreds of thousands of times through multiple SDE matrices that weigh a few megabytes and contain hundreds of thousands of elements each. + +
    + + Monte Carlo Path + +
    + +Calculating XVA on a multi-counterparties portfolio involves much more complex computations in a large system of equations. A valuation function goes through hundreds of GBs of SDE matrices, generates millions of trade-level MTM matrices, aggregates them to counterparty-level matrices, and then calculates the future exposure and XVA for each counterparty. + +
    + + Calculating XVA + +
    + +The technical challenge escalates when dealing with XVA sensitives to numerous market factors. To neutralize all market risks across counterparty portfolios, investment banks need to calculate XVA sensitivity for hundreds of market factors. There are two primary ways to compute XVA sensitivity: +1. analytically through backpropagation to input +2. numerically through observing how gradients move for XVA + +To obtain XVA variance, a valuation function must iterate hundreds of billions of times through the enormous system of equations, which is an extremely compute-intensive process. + +The XVA model is indispensable for understanding counterparty credit risk in the financial industry, and its accurate computation is vital for assessing all in price of derivatives. With an extensive amount of data and complex calculations involved, efficient and timely execution of these calculations is essential for ensuring that traders can make well-informed decisions. + +## Journey to Beam + +NOLA is HSBC’s internal data infrastructure for XVA computations. Its previous version - NOLA1 - was an on-premise solution that used a 10 TB file server as media, processing several petabytes of data in a single batch, going through a huge network of interdependencies within each system of equations, then repeating the process. HSBC’s model development team was creating new statistical models and building the Quantitative library while their IT team was fetching the necessary data to the library, and both teams were working together to lay out orchestration between systems of equations. + +The 2007 to ‘08 financial crisis highlighted the need for robust and efficient computation of XVAs across the industry, and introduced additional regulations in the financial sector that required an exponentially higher amount of computations. HSBC, therefore, sought a numerical solution for calculating the XVA sensitivities for hundreds of market factors. Processing data in a single batch had become a blocker and a throughput bottleneck. The NOLA1 infrastructure and its intensive I/O utilization was not [at the time] conducive for scaling. + +HSBC engineers started looking for a new approach that would allow for scaling their data processing, maximizing throughput, and meeting critical business timelines. + +Then, HSBC’s engineering team selected Apache Beam as a risk engine for NOLA for its scalability, flexibility, and ability to process large volumes of data in parallel. They found Apache Beam to be a natural process executor for the transformational, directed acyclic graphing process of XVA computations. The [Apache Beam Python SDK](https://beam.apache.org/documentation/sdks/python/) offered a simple API to build new data pipelines in Python, while its abstraction presented a way to [reuse the prevalent analytics in C++](https://cloud.google.com/dataflow/docs/hpc-ep). The variety of Apache Beam runners offered portability, and HSBC’s engineers built the new version of their data infrastructure - NOLA2 - with Apache Beam pipelines running on [Apache Flink](/documentation/runners/flink/) and [Cloud Dataflow](/documentation/runners/dataflow/). + +## Data Distribution Easier than Before + +Apache Beam has greatly simplified data distribution for XVA calculations and allowed for handling the inter-correlated Monte Carlo simulations with distributed processing between workers. + +The Apache Beam SDK enables users to build expressive DAGs and easily create stream or batch multi-stage pipelines in which parallel pipelined stages can be brought back together using [side inputs](/documentation/programming-guide/#side-inputs) or [joins](/documentation/pipelines/design-your-pipeline/#merging-pcollections). Data movement is handled by the runner, with data expressed as PCollection objects, which are immutable parallel element collections. + +Apache Beam provides several methods for distributing C++ components: + - sideloading C++ components to custom worker container images (for example, custom Apache Beam or Cloud Dataflow containers) and then using DoFns to interact with C++ components out-of-the-box + - bundling C++ with a JAR file in Apache Beam, where the C++ elements (binaries, configuration, etc.) are extracted to the local disk during the setup/teardown process in DoFn + - including the C++ components in a [PCollection](/documentation/programming-guide/#pcollections) as a side input, which is then deployed to the local disk + +The seamless integration of Apache Beam with C++ allowed HSBC’s engineers to reuse the prevalent analytics(relying on [NAG](https://www.nag.com/) and [MKL](https://en.wikipedia.org/wiki/Math_Kernel_Library) libraries)and select between the logic distribution methods depending on the use case and deployment environment. HSBC found protobufs especially useful for data exchange when PCollections carry the calls and input data to the C++ libraries from Java or Python pipelines. Protobuf data can be shared over disk, network, or directly with the usage of tools like [pybind11](https://github.com/pybind/pybind11). + +
    + + beam DAG + +
    + +HSBC migrated their XVA calculations to a batch Apache Beam pipeline. Every day, the XVA pipeline computes over multiple thousands of billions of valuations within just 1 hour, consuming around 2 GB of external input data, processing from 10 to 20 TB of data inside the system of equations, and producing about 4 GB of output reports. Apache Beam distributes XVA calculations into a number of PCollections with tasks, performs the necessary transformations independently and in parallel, and produces map-reduced results - all in one pass. + +Apache Beam provides powerful [transforms](/documentation/programming-guide/#transforms) and orchestration capabilities that helped HSBC engineers to optimize the analytical approach to XVA sensitivities calculation and enable the numerical one, which was not possible before. Instead of iterating a valuation function through the whole system of equations, HSBC’s engineers treat the system of equations as a computation graph, breaking it down into clusters with reusable elements and iterating through the minimal computation graph. They use Apache Beam orchestration to efficiently process tens of thousands of clusters for each portfolio by calling a C++ “executable”. Apache Beam enabled HSBC to bundle multiple market factors using [KV PCollections](/documentation/programming-guide/#core-beam-transforms) to associate each input element of a PCollection with a key and calculate the XVA sensitivity for hundreds of market factors within a single Apache Beam batch pipeline. + +The Apache Beam pipeline that performs analytical and numerical XVA sensitivities calculations runs daily in two separate batches. The first batch pipeline, which runs at midnight, determines the credit line consumption and capital utilisation for traders, directly affecting their trading volume the following day. The second batch, which completes before 8 am, calculates XVA sensitivities that could impact traders' risk management and hedging strategies. The pipeline consumes about 2 GB of external market data daily, processing up to 400 TB of internal data in the system of equations, and aggregating the data into the output report of just about 4 GB. At the end of each month, the pipeline processes over 5 PB of monthly data inside the system of equations to produce a full-scope XVA sensitivities report. Apache Beam addresses data distribution and hot spot issues, assisting in managing the data involved in intricate calculations. + +Apache Beam offers HSBC all the traits of a traditional risk engine and more, enabling HSBC to scale the infrastructure and maximize throughput with distributed processing. + +
    +

    + Apache Beam makes data distribution much easier than before. The Monte Carlo method significantly increases the amount of data to be processed. Apache Beam helped us harness all of that data volume. +

    +
    +
    + +
    +
    +
    + Andrzej Golonka +
    +
    + Lead Assistant Vice President @ HSBC +
    +
    +
    +
    + +## Beam as a Platform + +Apache Beam is more than a data processing framework. It is also a computational platform that enables experimentation, accelerates time-to-market for new development, and simplifies deployment. + +Apache Beam's dataset abstraction as PCollection increased HSBC’s model development efficiency by providing a way to organize component ownership and reduce organizational dependencies and bottlenecks. The model development team now owns data pipelines: implements new systems of equations, defines the data transfer within a system of equations in a black box mode, and sends it to the IT team. The IT team fetches, controls, and orchestrates the external data required by a system of equations as a whole. + +
    +

    + In general, we used to spend at least 6 months deploying even small changes to a system of equations in production. With the new team structure driven by Apache Beam, we now can deploy changes as quickly as within 1 week. +

    +
    +
    + +
    +
    +
    + Chup Cheng +
    +
    + VP of XVA and CCR Capital Analytics @ HSBC +
    +
    +
    +
    + +By leveraging the abstractions provided by the Apache Beam unified programming model, HSBC's model development team can seamlessly create new data pipelines, choose an appropriate runner, and conduct experiments on Big Data without the underlying infrastructure. The Apache Beam model rules ensure the high quality of the experimental code and make it very easy to move the production-grade pipelines from experimentation to production. + +
    +

    + With Apache Beam, it’s easy to experiment with “What if” questions. If we want to know the impact of changing some parameters, we can write a simple Apache Beam code, run the pipeline, and have the answer within minutes. +

    +
    +
    + +
    +
    +
    + Andrzej Golonka +
    +
    + Lead Assistant Vice President @ HSBC +
    +
    +
    +
    + +One of the key advantages of Apache Beam for Monte Carlo simulations and counterparty credit risk analysis is its ability to run the same complex simulation logic in various environments, on-premise or in the cloud, with different runners. This flexibility is especially critical in situations requiring local risk analysis in different countries and compliance zones, where sensitive financial data and information cannot be transferred beyond the local perimeter. With Apache Beam, HSBC can easily switch between runners and can future-proof their data processing for any changes. HSBC runs the majority of workflows in Cloud Dataflow, benefitting from its powerful managed service and autoscaling capabilities to manage spikes of up to 18,000 vCPUs when running batch pipelines twice a day. In select countries, they also use the Apache Beam Flink runner to comply with local regulations specific to data storage and processing. + +## Results + +Apache Beam harnesses enormous volumes of financial market data and metrics, generates billions of trade valuations to scan plausible future scenarios reaching out around 70 years, and aggregates them into meaningful statistics, enabling HSBC to model their future scenarios and quantitatively account for risk in forecasting and decision-making. + +With Apache Beam, HSBC’s engineers achieved a 2x increase in data processing performance and scaled their XVA batch pipeline by 100x compared to the original solution. The Apache Beam abstraction opened up a way to implement a numerical approach to XVA sensitivity calculation in production, which was not possible before. The batch Apache Beam pipeline calculates XVA sensitivities for hundreds of market factors, processing about 400 TB of internal data every day and up to 5 PB of data once per month. + +Apache Beam portability enabled HSBC to use different runners in different regions depending on local data processing requirements and future-proof their data processing for regulatory changes. + +Apache Beam provides seamless integration and out-of-the-box interaction with highly optimized computational components in C++, which saved HSBC the effort needed to rewrite the C++ analytics accumulated for years into Python. + +The Apache Beam Python SDK brought math to the orchestration level by providing an easy way for HSBC’s model development team to build new Python pipelines. The new work structure driven by Apache Beam accelerated time-to-market by 24x, enabling HSBC’s teams to deploy changes and new models to production within just a few weeks. + +By leveraging the versatile and scalable nature of Apache Beam for computing direct acyclic graphs that process large differential equations systems and Monte Carlo simulations, financial institutions can assess and manage counterparty credit risk efficiently, even in situations that demand localized analysis and strict compliance with data security regulations. + +## Learn More + + +

    + +{{< case_study_feedback "HSBC" >}} + +
    +
    diff --git a/website/www/site/content/en/case-studies/linkedin.md b/website/www/site/content/en/case-studies/linkedin.md index 5ed8b65a55787..b16c1f162157f 100644 --- a/website/www/site/content/en/case-studies/linkedin.md +++ b/website/www/site/content/en/case-studies/linkedin.md @@ -1,8 +1,17 @@ --- -title: "Linkedin" -icon: /images/logos/powered-by/linkedin.png -hasNav: true -hasLink: "https://www.youtube.com/watch?v=rBfwjbrMJTE&list=PL4dEBWmGSIU9OkXQU2OAXmITPLhiMSPRp&index=33" +title: "Revolutionizing Real-Time Stream Processing: 4 Trillion Events Daily at LinkedIn " +name: "LinkedIn" +icon: "/images/logos/powered-by/linkedin.png" +category: "study" +cardTitle: "Revolutionizing Real-Time Stream Processing: 4 Trillion Events Daily at LinkedIn" +cardDescription: "Apache Beam serves as the backbone of LinkedIn's streaming infrastructure, handling the near real-time processing of an astounding 4 trillion events daily through 3,000+ pipelines and thus powering personalized experiences for LinkedIn’s vast network of over 950 million members worldwide. The adoption of Apache Beam brought about a series of impressive enhancements, including 2x cost optimization depending on the use case, an astounding acceleration from days to minutes in labeling abuse, and more than 6% improvement in detecting logged-in scrapping profiles." +authorName: "Bingfeng Xia" +coauthorName: "Xinyu Liu" +authorPosition: "Engineering Manager @LinkedIn" +coauthorPosition: "Senior Staff Engineer @LinkedIn" +authorImg: /images/case-study/linkedin/bingfeng-xia.jpg +coauthorImg: /images/case-study/linkedin/xinyu-liu.jpg +publishDate: 2023-08-10T00:12:00+00:00 --- +
    +
    + +
    +
    +

    + “Apache Beam empowers LinkedIn to create timely recommendations and personalized experiences by leveraging the freshest data and processing it in real-time, ultimately benefiting LinkedIn's vast network of over 950 million members worldwide.” +

    +
    +
    + +
    +
    +
    + Bingfeng Xia +
    +
    + Engineering Manager @LinkedIn +
    +
    +
    +
    +
    +
    + +# Revolutionizing Real-Time Stream Processing: 4 Trillion Events Daily at LinkedIn + +## Background + +At LinkedIn, Apache Beam plays a pivotal role in stream processing infrastructures that process over 4 trillion events daily through more than 3,000 pipelines across multiple production data centers. This robust framework empowers near real-time data processing for critical services and platforms, ranging from machine learning and notifications to anti-abuse AI modeling. With over 950 million members, ensuring that our platform is running smoothly is critical to connecting members to opportunities worldwide. + +In this case study, LinkedIn's Bingfeng Xia, Engineering Manager, and Xinyu Liu, Senior Staff Engineer, shed light on how the Apache Beam programming model's unified, portable, and user-friendly data processing framework has enabled a multitude of sophisticated use cases and revolutionized Stream Processing at LinkedIn. This technology has [optimized cost-to-serve by 2x](https://engineering.linkedin.com/blog/2023/unified-streaming-and-batch-pipelines-at-linkedin--reducing-proc) by unifying stream and batch processing through Apache Samza and Apache Spark runners, enabled real-time ML feature generation, reduced time-to-production for new pipelines from months to days, allowed for processing time-series events at over 3 million queries per second, and more. For our members, this means that we’re able to serve more accurate job recommendations, improve feed recommendations, and identify fake profiles at a faster rate, etc. + + +## LinkedIn Open-Source Ecosystem and Journey to Beam + +LinkedIn has a rich history of actively contributing to the open-source community, demonstrating its commitment by creating, managing, and utilizing various open-source software projects. The LinkedIn engineering team has [open-sourced over 75 projects](https://engineering.linkedin.com/content/engineering/en-us/open-source) across multiple categories, with several gaining widespread adoption and becoming part of [the Apache Software Foundation](https://www.apache.org/). + +To enable the ingestion and real-time processing of enormous volumes of data, LinkedIn built a custom stream processing ecosystem largely with tools developed in-house (and subsequently open-sourced). In 2010, they introduced [Apache Kafka](https://kafka.apache.org/), a pivotal Big Data ingestion backbone for LinkedIn’s real-time infrastructure. To transition from batch-oriented processing and respond to Kafka events within minutes or seconds, they built an in-house distributed event streaming framework, [Apache Samza](https://samza.apache.org/). This framework, along with Apache Spark for batch processing, formed the basis of LinkedIn’s [lambda architecture](https://en.wikipedia.org/wiki/Lambda_architecture) for data processing jobs. Over time, LinkedIn's engineering team expanded the stream processing ecosystem with more proprietary tools like [Brooklin](https://github.com/linkedin/Brooklin/), facilitating data streaming across multiple stores and messaging systems, and [Venice](https://github.com/linkedin/venice), serving as a storage system for ingesting batch and stream processing job outputs, among others. + +Though the stream processing ecosystem with Apache Samza at its core enabled large-scale stateful data processing, LinkedIn’s ever-evolving demands required higher scalability and efficiency, as well as lower latency for the streaming pipelines. The lambda architecture approach led to operational complexity and inefficiencies, because it required maintaining two different codebases and two different engines for batch and streaming data. To address these challenges, data engineers sought a higher level of stream processing abstraction and out-of-the-box support for advanced aggregations and transformations. Additionally, they needed the ability to experiment with streaming pipelines in batch mode. There was also a growing need for multi-language support within the overall Java-prevalent teams due to emerging machine learning use cases requiring Python. + +The release of [Apache Beam](/about/) in 2016 proved to be a game-changer for LinkedIn. Apache Beam offers an open-source, advanced unified programming model for both batch and Stream Processing, making it possible to create a large-scale common data infrastructure across various applications. With support for Python, Go, and Java SDKs and a rich, versatile API layer, Apache Beam provided the ideal solution for building sophisticated multi-language pipelines and running them on any engine. + +
    +

    + When we started looking at Apache Beam, we realized it was a very attractive data processing framework for LinkedIn’s demands: not only does it provide an advanced API, but it also allows for converging stream and batch processing and multi-language support. Everything we were looking for and out-of-the-box. +

    +
    +
    + +
    +
    +
    + Xinyu Liu +
    +
    + Senior Staff Engineer @LinkedIn +
    +
    +
    +
    + +Recognizing the advantages of Apache Beam's unified data processing API, advanced capabilities, and multi-language support, LinkedIn began onboarding its first use cases and developed the [Apache Samza runner for Beam](/documentation/runners/samza/) in 2018. By 2019, Apache Beam pipelines were powering several critical use cases, and the programming model and framework saw extensive adoption across LinkedIn teams. Xinyu Liu showcased the benefits of migrating to Apache Beam pipelines during [Beam Summit Europe 2019](https://www.youtube.com/watch?v=uQcpr34RUKY&t=1694s). + +
    + + scheme + +
    + +## Apache Beam Use Cases at LinkedIn + +### Unified Streaming And Batch Pipelines + +Some of the first use cases that LinkedIn migrated to Apache Beam pipelines involved both real-time computations and periodic backfilling. One example was LinkedIn's standardization process. Standardization consists of a series of pipelines that use complex AI models to map LinkedIn user inputs, such as job titles, skills, or education history, into predefined internal IDs. For example, a LinkedIn member who lists their current position as "Chief Data Scientist" has their job title standardized for relevant job recommendations. + + +LinkedIn's standardization process requires both real-time processing to reflect immediate user updates and periodic backfilling to refresh data when new AI models are introduced. Before adopting Apache Beam, running backfilling as a streaming job required over 5,000 GB-hours in memory and nearly 4,000 hours in total CPU time. This heavy load led to extended backfilling times and scaling issues, causing the backfilling pipeline to act as a "noisy neighbor" to colocated streaming pipelines and failing to meet latency and throughput requirements. Although LinkedIn engineers considered migrating the backfilling logic to a batch Spark pipeline, they abandoned the idea due to the unnecessary overhead of maintaining two different codebases. + +
    +

    + We came to the question: is it possible to only maintain one codebase but with the ability to run it as either a batch job or streaming job? The unified Apache Beam model was the solution. +

    +
    +
    + +
    +
    +
    + Bingfeng Xia +
    +
    + Engineering Manager @LinkedIn +
    +
    +
    +
    + +The Apache Beam APIs enabled LinkedIn engineers to implement business logic once within a unified Apache Beam pipeline that efficiently handles both real-time standardization and backfilling. Apache Beam offers [PipelineOptions](https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/options/PipelineOptions.html), enabling the configuration and customization of various aspects, such as the pipeline runner and runner-specific configurations. The extensibility of Apache Beam transforms allowed LinkedIn to [create a custom composite transform](https://beam.apache.org/documentation/programming-guide/#composite-transforms) to abstract away I/O differences and switch target processing on the fly based on data source type (bounded or unbounded). In addition, Apache Beam’s abstraction of the underlying infrastructure and the ability to "write once, run anywhere" empowered LinkedIn to seamlessly switch between data processing engines. Depending on the target processing type, streaming, or batch, the unified Apache Beam standardization pipeline can be deployed through the Samza cluster as a streaming job or through the Spark cluster as a batch backfilling job. + +
    + + scheme + +
    + +Hundreds of streaming Apache Beam jobs now power real-time standardization, listening to events 24/7, enriching streams with additional data from remote tables, performing necessary processing, and writing results to output databases. The batch Apache Beam backfilling job runs weekly, effectively handling 950 million member profiles at a rate of over 40,000 profiles per second. Apache Beam infers data points into sophisticated AI and machine learning models and joins complex data such as job types and work experiences, thus standardizing user data for search indexing or to run recommendation models. + +The migration of backfilling logic to a unified Apache Beam pipeline and its execution in batch mode resulted in a significant 50% improvement in memory and CPU usage efficiency (from ~5000 GB-hours and ~4000 CPU hours to ~2000 GB-hours and ~1700 CPU hours) and an impressive 94% acceleration in processing time (from 7.5 hours to 25 minutes). More details about this use case can be found on [LinkedIn’s engineering blog](https://engineering.linkedin.com/blog/2023/unified-streaming-and-batch-pipelines-at-linkedin--reducing-proc). + +### Anti-Abuse & Near Real-Time AI Modeling + +LinkedIn is firmly committed to creating a trusted environment for its members, and this dedication extends to safeguarding against various types of abuse on the platform. To achieve this, the Anti-Abuse AI Team at LinkedIn plays a crucial role in creating, deploying, and maintaining AI and deep learning models that can detect and prevent different forms of abuse, such as fake account creation, member profile scraping, automated spam, and account takeovers. + +Apache Beam fortifies LinkedIn’s internal anti-abuse platform, Chronos, enabling abuse detection and prevention in near real-time. Chronos relies on two streaming Apache Beam pipelines: the Filter pipeline and the Model pipeline. The Filter pipeline reads user activity events from Kafka, extracts relevant fields, aggregates and filters the events, and then generates filtered Kafka messages for downstream AI processing. Subsequently, the Model pipeline consumes these filtered messages, aggregates member activity within specific time windows, triggers AI scoring models, and writes the resulting abuse scores to various internal applications, services, and stores for offline processing. + +
    + + scheme + +
    + +The flexibility of Apache Beam's pluggable architecture and the availability of various I/O options seamlessly integrated the anti-abuse pipelines with Kafka and key-value stores. LinkedIn has dramatically reduced the time it takes to label abusive actions, cutting it down from 1 day to just 5 minutes and processing time-series events at an impressive rate of over 3 million queries per second. Apache Beam empowered near real-time processing, significantly bolstering LinkedIn's anti-abuse defenses. The nearline defenses are able to catch scrapers within minutes after they start to scrape and this leads to more than 6% improvement in detecting logged-in scrapping profiles. + +
    +

    + Apache Beam enabled revolutionary, phenomenal performance improvements - the anti-abuse processing accelerated from 1 day to 5 minutes. We have seen more than 6% improvement in detecting logged-in scrapping profiles. +

    +
    +
    + +
    +
    +
    + Xinyu Liu +
    +
    + Senior Staff Engineer @LinkedIn +
    +
    +
    +
    + +### Notifications Platform + +As a social media network, LinkedIn heavily relies on instant notifications to drive member engagement. To achieve this, Apache Beam and Apache Samza together power LinkedIn’s large-scale Notifications Platform that generates notification content, pinpoints the target audience, and ensures the timely and relevant distribution of content. + +The streaming Apache Beam pipelines have intricate business logic and handle enormous volumes of data in a near real-time fashion. The pipelines consume, aggregate, partition, and process events from over 950 million LinkedIn members and feed the data to downstream machine learning models. The ML models perform distributed targeting and scalable scoring on the order of millions of candidate notifications per second based on the recipient member’s historical actions and make personalized decisions for the recipient for each notification on the fly. As a result, LinkedIn members receive timely, relevant, and actionable activity-based notifications, such as connection invites, job recommendations, daily news digests, and other activities within their social network, through the right channels. + +The advanced Apache Beam API offers complex aggregation and filtering capabilities out-of-the-box, and its programming model allows for the creation of reusable components. These features enable LinkedIn to expedite development and streamline the scaling of the Notifications platform as they transition more notification use cases from Samza to Beam pipelines. + +
    +

    + LinkedIn’s user engagement is greatly driven by how timely we can send relevant notifications. Apache Beam enabled a scalable, near real-time infrastructure behind this business-critical use case. +

    +
    +
    + +
    +
    +
    + Bingfeng Xia +
    +
    + Engineering Manager @LinkedIn +
    +
    +
    +
    + +### Real-Time ML Feature Generation + +LinkedIn's core functionalities, such as job recommendations and search feed, heavily rely on ML models that consume thousands of features related to various entities like companies, job postings, and members. However, before the adoption of Apache Beam, the original offline ML feature generation pipeline suffered from a delay of 24 to 48 hours between member actions and the impact of those actions on the recommendation system. This delay resulted in missed opportunities, because the system lacked sufficient data about infrequent members and failed to capture the short-term intent and preferences of frequent members. In response to the growing demand for a scalable, real-time ML feature generation platform, LinkedIn turned to Apache Beam to address the challenge. + +Using Managed Beam as the foundation, LinkedIn developed a hosted platform for ML feature generation. The ML platform provides AI engineers with real-time features and an efficient pipeline authoring experience, all while abstracting away deployment and operational complexities. AI engineers create feature definitions and deploy them using Managed Beam. When LinkedIn members take actions on the platform, the streaming Apache Beam pipeline generates fresher machine learning features by filtering, processing, and aggregating the events emitted to Kafka in real-time and writes them to the feature store. Additionally, LinkedIn introduced other Apache Beam pipelines responsible for retrieving the data from the feature store, processing it, and feeding it into the recommendation system. + +
    + + scheme + +
    + +The powerful Apache Beam Stream Processing platform played a pivotal role in eliminating the delay between member actions and data availability, achieving an impressive end-to-end pipeline latency of just a few seconds. This significant improvement allowed LinkedIn's ML models to take advantage of up-to-date information and deliver more personalized and timely recommendations to our members, leading to significant gains in business metrics. + +### Managed Stream Processing Platform + +As LinkedIn's data infrastructure grew to encompass over 3,000 Apache Beam pipelines, catering to a diverse range of business use cases, LinkedIn's AI and data engineering teams found themselves overwhelmed with managing these streaming applications 24/7. The AI engineers encountered several technical challenges while creating new pipelines, including the intricacy of integrating multiple streaming tools and infrastructures into their frameworks, and limited knowledge of the underlying infrastructure when it came to deployment, monitoring, and operations. These challenges led to a time-consuming pipeline development cycle, often lasting one to two months. Apache Beam enabled LinkedIn to create Managed Beam, a managed Stream Processing platform that is designed to streamline and automate internal processes. This platform makes it easier and faster for teams to develop and operate sophisticated streaming applications while reducing the burden of on-call support. + +
    + + scheme + +
    + +The Apache Beam SDK empowered LinkedIn engineers to create custom workflow components as reusable sub-DAGs (Directed Acyclic Graphs) and expose them as standard PTransforms. These PTransforms serve as ready-to-use building blocks for new pipelines, significantly speeding up the authoring and testing process for LinkedIn AI engineers. By abstracting the low-level details of underlying engines and runtime environments, Apache Beam allows engineers to focus solely on business logic, further accelerating time to development. + +When the pipelines are ready for deployment, Managed Beam's central control plane comes into play, providing essential features like a deployment UI, operational dashboard, administrative tools, and automated pipeline lifecycle management. + +Apache Beam's abstraction facilitated the isolation of user code from framework evolution during build, deployment, and runtime. To ensure the separation of runner processes from user-defined functions (UDFs), Managed Beam packages the pipeline business logic and the framework logic as two separate JAR files: framework-less artifacts and framework artifacts. During pipeline execution on a YARN cluster, these pipeline artifacts run in a Samza container as two distinct processes, communicating through gRPC. This setup enabled LinkedIn to take advantage of automated framework upgrades, scalable UDF execution, log separation for easier troubleshooting, and multi-language APIs, fostering flexibility and efficiency. + +
    + + scheme + +
    + +Apache Beam also underpinned Managed Beam's autosizing controller tool, which automates hardware resource tuning and provides auto-remediation for streaming pipelines. Streaming Apache Beam pipelines self-report diagnostic information, such as metrics and key deployment logs, in the form of Kafka topics. Additionally, LinkedIn's internal monitoring tools report runtime errors, such as heartbeat failures, out-of-memory events, and processing lags. The Apache Beam diagnostics processor pipeline aggregates, repartitions, and windows these diagnostic events before passing them to the autosizing controller and writing them to Apache Pinot, LinkedIn's OLAP store for Managed Beam's operational and analytics dashboards. Based on the pre-processed and time-windowed diagnostic data, the autosizing controller generates sizing actions or restarting actions, and then forwards them to the Managed Beam control plane. The Managed Beam control plane then scales LinkedIn's streaming applications and clusters. + +
    +

    + Apache Beam helped streamline operations management and enabled fully-automated autoscaling, significantly reducing the time to onboard new applications. Previously, onboarding required a lot of manual 'trial and error' iterations and deep knowledge of the internal system and metrics. +

    +
    +
    + +
    +
    +
    + Bingfeng Xia +
    +
    + Engineering Manager @LinkedIn +
    +
    +
    +
    + +The extensibility, pluggability, portability, and abstraction of Apache Beam formed the backbone of LinkedIn's Managed Beam platform. The Managed Beam platform accelerated the time to author, test, and stabilize streaming pipelines from months to days, facilitated fast experimentation, and almost entirely eliminated operational costs for AI engineers. + +## Summary + +Apache Beam played a pivotal role in revolutionizing and scaling LinkedIn's data infrastructure. Beam's powerful streaming capabilities enable real-time processing for critical business use cases, at a scale of over 4 trillion events daily through more than 3,000 pipelines. + +The versatility of Apache Beam empowered LinkedIn’s engineering teams to optimize their data processing for various business use cases: +- Apache Beam's unified and portable framework allowed LinkedIn to consolidate streaming and batch processing into unified pipelines. These unified pipelines resulted in a 2x optimization in cost-to-serve, a 2x improvement in processing performance, and a 2x improvement in memory and CPU usage efficiency. +- LinkedIn's anti-abuse platform leveraged Apache Beam to process user activity events from Kafka in near-real-time, achieving a remarkable acceleration from days to minutes in labeling abusive actions. The nearline defenses are able to catch scrapers within minutes after they start to scrape and this leads to more than 6% improvement in detecting logged-in scrapping profiles. +- By adopting Apache Beam, LinkedIn was able to transition from an offline ML feature generation pipeline with a 24- to 48-hour delay to a real-time platform with an end-to-end pipeline latency at the millisecond or second level. +- Apache Beam’s abstraction and powerful programming model enabled LinkedIn to create a fully managed stream processing platform, thus facilitating easier authoring, testing, and deployment and accelerating time-to-production for new pipelines from months to days. + +Apache Beam boasts seamless plug-and-play capabilities, integrating smoothly with Apache Kafka, Apache Pinot, and other core technologies at LinkedIn, all while ensuring optimal performance at scale. As LinkedIn continues experimenting with new engines and tooling, the Apache Beam portability future-proofs our ecosystem against any changes in the underlying infrastructure. + +
    +

    + By enabling a scalable, near real-time infrastructure behind business-critical use cases, Apache Beam empowers LinkedIn to leverage the freshest data and process it in real-time to create timely recommendations and personalized experiences, ultimately benefiting LinkedIn's vast network of over 950 million members worldwide. +

    +
    +
    + +
    +
    +
    + Xinyu Liu +
    +
    + Senior Staff Engineer @LinkedIn +
    +
    +
    +
    + +

    + +{{< case_study_feedback "LinkedIn" >}} + +
    +
    diff --git a/website/www/site/content/en/case-studies/octo.md b/website/www/site/content/en/case-studies/octo.md new file mode 100644 index 0000000000000..9ab6fd4b00c62 --- /dev/null +++ b/website/www/site/content/en/case-studies/octo.md @@ -0,0 +1,183 @@ +--- +title: "High-Performing and Efficient Transactional Data Processing for OCTO Technology’s Clients" +name: "OCTO" +icon: "/images/logos/powered-by/octo.png" +category: "study" +cardTitle: "High-Performing and Efficient Transactional Data Processing for OCTO Technology’s Clients" +cardDescription: "With Apache Beam, OCTO accelerated the migration of one of France’s largest grocery retailers to streaming processing for transactional data. By leveraging Apache Beam's powerful transforms and robust streaming capabilities, they achieved a 5x reduction in infrastructure costs and a 4x boost in performance. The streaming Apache Beam pipelines now process over 100 million rows daily, consolidating hundreds of gigabytes of transactional data with over a terabyte of an external state in under 3 hours, a task that was not feasible without Apache Beam’s controlled aggregation." +authorName: "OCTO Technology's Data Engineering Team" +authorPosition: "Large Retail Client Project" +authorImg: /images/logos/powered-by/octo.png +publishDate: 2023-08-10T00:12:00+00:00 +--- + +
    +
    + +
    +
    +

    + “Oftentimes, I tell the clients that Apache Beam is the “Docker” of data processing. It's highly portable, runs seamlessly anywhere, unifies batch and streaming processing, and offers numerous out-of-the-box templates. Adopting Beam enables accelerated migration from batch to streaming, effortless pipeline reuse across contexts, and faster enablement of new use cases. The benefits and great performance of Apache Beam are a game-changer for many!” +

    +
    +
    + +
    +
    +
    + Godefroy Clair +
    +
    + Data Architect @ OCTO Technology +
    +
    +
    +
    +
    +
    + +# High-Performing and Efficient Transactional Data Processing for OCTO Technology’s Clients + +## Background + +[OCTO Technology](https://octo.com/), part of [Accenture](https://www.accenture.com/), stands at the forefront of technology consultancy and software engineering, specializing in new technologies and digital transformation. Since 1998, OCTO has been dedicated to crafting scalable digital solutions that drive business transformations for clients, ranging from startups to multinational corporations. OCTO leverages its deep technology expertise and a strong culture of successful innovation to help clients explore, test, and embrace emerging technologies or implement mature digital solutions at scale. + +With the powerful Apache Beam unified portable model, OCTO has unlocked the potential to empower, transform, and scale the data ecosystems of several clients, including renowned names like a famous French newspaper and one of France's largest grocery retailers. + + +In this spotlight, OCTO’s Data Architect, Godefroy Clair, and Data Engineers, Florian Bastin and Leo Babonnaud, unveil the remarkable impact of Apache Beam on the data processing of a leading French grocery retailer. The implementation led to expedited migration from batch to streaming, a 4x acceleration in transactional data processing, and a 5x improvement in infrastructure cost efficiency. + + +## High-performing transactional data processing + +OCTO’s Client, a prominent grocery and convenience store retailer with tens of thousands of stores across several countries, relies on an internal web app to empower store managers with informed purchasing decisions and effective store management. The web app provides access to crucial product details, stock quantities, pricing, promotions, and more, sourced from various internal data stores, platforms, and systems. + +Before 2022, the Client utilized an orchestration engine for orchestrating batch pipelines that consolidated and processed data from Cloud Storage files and Pub/Sub messages and wrote the output to BigQuery. However, with most source data uploaded at night, batch processing posed challenges in meeting SLAs and providing the most recent information to store managers before store opening. Moreover, incorrect or missing data uploads required cumbersome database state reverts, involving a substantial amount of transactional data and logs. The Client’s internal team dedicated significant time to maintaining massive SQL queries or manually updating the database state, resulting in high maintenance costs. + +To address these issues, the Client sought OCTO's expertise to transform their data ecosystem and migrate their core use case from batch to streaming. The objectives included faster data processing, ensuring the freshest data in the web app, simplifying pipeline and database maintenance, ensuring scalability and resilience, and efficiently handling spikes in data volumes. + +
    +

    + The Client needed to very quickly consolidate and process a huge number of files in different formats from Cloud Storage and Pub/Sub events to have the freshest info about new products, promotions, etc. in their web app every day. For all that, Apache Beam was the perfect tool. +

    +
    +
    + +
    +
    +
    + Godefroy Clair +
    +
    + Data Architect @ OCTO Technology +
    +
    +
    +
    + +Apache Beam and its unified model emerged as the perfect solution, enabling both near-real-time streaming for the Client’s core transactional data processing, as well as their batch processing for standalone use cases. Additionally, it offered the added benefit of autoscaling with the [Dataflow runner](/documentation/runners/dataflow/). With Apache Beam's [ Python SDK](/documentation/sdks/python/) and the out-of-the-box [I/O connectors](/documentation/io/connectors/), OCTO was able to reuse Python components between the existing and new batch and streaming pipelines, and leverage native optimized connectivity with Pub/Sub and Cloud Storage, expediting the migration. + +
    + + streaming pipelines + +
    + +The streaming Apache Beam pipeline behind the Client’s web app now processes product and inventory data from Pub/Sub messages and Cloud Storage files of varying sizes - from several rows to 1.7 million rows - that arrive in Cloud Storage at various times, in unpredictable order, and in various formats (such as CSV, JSON, and zip files). Apache Beam's [timely processing](/blog/timely-processing/) capabilities enable the streaming pipeline to handle that data efficiently. Its [timers](/documentation/basics/#state-and-timers) provide a way to control aggregations by waiting until all the necessary events and files come in and then processing them in the right order, while the [GroupByKey](/documentation/transforms/python/aggregation/groupbykey/) and [GroupIntoBatches](/documentation/transforms/python/aggregation/groupintobatches/) transforms allow for efficient grouping for every key and batching the input into desired size. Every day, the Apache Beam pipeline consolidates, deduplicates, enriches, and outputs the data to [Firestore](https://firebase.google.com/docs/firestore) and [Algolia](https://www.algolia.com/), processing over 100 million rows and consolidating hundreds of gigabytes of transactional data with over a terabyte of an external state within less than 3 hours. + +
    +

    + The web app requires fresh data early in the morning before the stores open. Previously, handling the entirety of the Client’s data in time was not feasible. Thanks to Apache Beam, they can now process it within just 3 hours, ensuring data availability even if the input files arrive late at night. +

    +
    +
    + +
    +
    +
    + Leo Babonnaud +
    +
    + Data Scientist @ OCTO Technology +
    +
    +
    +
    + +The Client’s specific use case posed unique challenges: the enrichment data was too large to keep in memory, and the unpredictable file order and arrival rendered timers and state API unfeasible. Being unable to leverage Apache Beam's native stateful processing, OCTO found a solution in externalizing the state of [DoFns](/documentation/programming-guide/#pardo) to a transactional [Cloud SQL](https://cloud.google.com/sql/docs/introduction) Postgres database. When processing new events and files, the Apache Beam pipeline uses streaming queries to select, insert, upsert, and delete rows with states in the Cloud SQL database. Apache Beam excels in complex state consolidation when processing files, Pub/Sub events, and logs representing the past, present, and future state of the records in the sink databases. If the incoming data is wrong and the sink data stores need to be reverted, Apache Beam processes a huge amount of logs about data movements that happened within a specific timeframe and consolidates them into states, eliminating manual efforts. + +
    +

    + The web app requires fresh data early in the morning before the stores open. Previously, handling the entirety of the Client’s data in time was not feasible. Thanks to Apache Beam, they can now process it within just 3 hours, ensuring data availability even if the input files arrive late at night. +

    +
    +
    + +
    +
    +
    + Florian Bastin +
    +
    + Lead Data Scientist @ OCTO Technology +
    +
    +
    +
    + +By leveraging Apache Beam, the Client has achieved a groundbreaking transformation in data processing, empowering their internal web app with fresh and historical data, enhancing overall operational efficiency, and meeting business requirements with improved processing latency. + +## Custom I/O and fine-grained control over SQL connections + +The Client’s specific use case demanded CRUD operations in a Cloud SQL database based on a value in a PCollection, and although the built-in [JBDC I/O connector](https://beam.apache.org/releases/pydoc/current/apache_beam.io.jdbc.html) supported reading and writing from a Cloud SQL database, it did not cater to such SQL operations. However, Apache Beam's custom I/O frameworks open the door for [creating new connectors](/documentation/io/developing-io-overview/) tailored to complex use cases, offering the same connectivity as out-of-the-box I/Os. Capitalizing on this advantage and leveraging [ParDo](/documentation/transforms/python/elementwise/pardo/) and [GroupByKey](/documentation/transforms/python/aggregation/groupbykey/) transforms, OCTO successfully developed a new Apache Beam I/O. This custom I/O seamlessly interacts with a Cloud SQL database using the [Cloud SQL Python Connector](https://pypi.org/project/cloud-sql-python-connector/), instantiating the latter as a connection object in the [DoFn.Setup](https://beam.apache.org/releases/pydoc/current/apache_beam.transforms.core.html#apache_beam.transforms.core.DoFn.setup) method. + +Moreover, Apache Beam offered OCTO fine-grained control over parallelism, enabling them to maximize worker processes' efficiency. With the Dataflow runner's potent parallelism and autoscaling capabilities, OCTO had to address the [constraints on the number of concurrent connections](https://cloud.google.com/sql/docs/quotas) imposed by Cloud SQL. To overcome this challenge, the Apache Beam DoFn.Setup method came into play, providing a means to define the maximum number of concurrent operations by specifying it within the method. OCTO also leveraged the [beam.utils.Shared](https://beam.apache.org/releases/pydoc/2.29.0/apache_beam.utils.shared.html) module to create a connection pool for the Cloud SQL database, effectively sharing it across all processes at the worker level. + +OCTO's data engineers showcased these innovative developments powered by Apache Beam at [Beam Summit 2023](https://beamsummit.org/sessions/2023/how-to-balance-power-and-control-when-using-dataflow-with-an-oltp-sql-database/). + +## Results + +Apache Beam enabled OCTO to revolutionize data processing of one of the most prominent French grocery retailers with a 5x optimization in infrastructure costs and a 4x increase in data processing performance. Apache Beam's unified model and Python SDK proved instrumental in accelerating the migration from batch to streaming processing by providing the ability to reuse components, packages, and modules across pipelines. + +Apache Beam's powerful transforms and robust streaming capabilities enabled the Client’s streaming pipeline to efficiently process over 100 million rows daily, consolidating transactional data with over a terabyte of an external state in under 3 hours, a feat that was previously unattainable. The flexibility and extensibility of Apache Beam empowered OCTO to tackle use-case-specific technical constraints, achieving the perfect balance of power and control to align with the Client’s specific business objectives. + +
    +

    + Oftentimes, I tell the clients that Apache Beam is the “Docker” of data processing. It's highly portable, runs seamlessly anywhere, unifies batch and streaming processing, and offers numerous out-of-the-box templates. Adopting Beam enables accelerated migration from batch to streaming, effortless pipeline reuse across contexts, and faster enablement of new use cases. The benefits and great performance of Apache Beam are a game-changer for many! +

    +
    +
    + +
    +
    +
    + Godefroy Clair +
    +
    + Data Architect @ OCTO Technology +
    +
    +
    +
    + +## Learn More + + +

    + +{{< case_study_feedback "OCTO" >}} + +
    +
    diff --git a/website/www/site/content/en/contribute/_index.md b/website/www/site/content/en/contribute/_index.md index 54099bb4e1daa..6a2497627a9bf 100644 --- a/website/www/site/content/en/contribute/_index.md +++ b/website/www/site/content/en/contribute/_index.md @@ -95,7 +95,7 @@ Here’s a list of things you can do to get started contributing:
    @@ -130,7 +130,7 @@ Here’s a list of things you can do to get started contributing: @@ -138,16 +138,16 @@ Here’s a list of things you can do to get started contributing: diff --git a/website/www/site/content/en/contribute/committer-guide.md b/website/www/site/content/en/contribute/committer-guide.md deleted file mode 100644 index df1ea067c6f5c..0000000000000 --- a/website/www/site/content/en/contribute/committer-guide.md +++ /dev/null @@ -1,144 +0,0 @@ ---- -title: "Beam Committer Guide" ---- - - -# Committer Guide - -This guide is for -[committers](https://www.apache.org/foundation/how-it-works.html#committers) -and covers Beam's guidelines for reviewing and merging code. - -## Pull request review objectives - -The review process aims for: - -* Review iterations should be efficient, timely and of quality (avoid tiny or out-of-context changes or huge mega-changes) -* Support efficiency of authoring (don't want to wait on a review for a tiny bit because GitHub makes it very hard to stack up reviews in sequence / don't want to have major changes blocked because of difficulty of review) -* Ease of first-time contribution (encourage to follow [contribution guildelines](/contribute/#contributing-code) - but committer may absorb some extra effort for new contributors) -* Pull requests and commit messages establish a clear history with purpose and origin of changes -* Ability to perform a granular rollback, if necessary (also see [policies](/contribute/postcommits-policies/)) - -Granularity of changes: - -* We prefer small independent, incremental PRs with descriptive, isolated commits. Each commit is a single clear change -* It is OK to keep separate commits for different logical pieces of the code, if they make reviewing and revisiting code easier -* Making commits isolated is a good practice, authors should be able to relatively easily split the PR upon reviewer's request -* Generally, every commit should compile and pass tests. -* Avoid keeping in history formatting messages such as checkstyle or spotless fixes. Squash such commits with previous one. - -## Always get to LGTM ("Looks good to me!") - -After a pull request goes through rounds of reviews and revisions, it will -become ready for merge. A reviewer signals their approval either -by GitHub "approval" or by a comment such as "Looks good to me!" (LGTM). - - - If the author of the pull request is not a committer, a committer must be - the one to approve the change. - - If the author of the pull request is a committer, approval from their chosen - reviewer is enough. A committer is trusted to choose an appropriate - reviewer, even if the reviewer is not a committer. - -Once a pull request is approved, any committer can merge it. - -Exceptions to this rule are rare and made on a case-by-case basis. A committer -may use their discretion for situations such as build breaks. In this case, you -should still seek a review on the pull request! A common acronym you may see -is "TBR" -- "to be reviewed". - -**Always go through a pull request, even if you won’t wait for the code -review.** Committers should never commit anything without going through a pull -request, even when it is an urgent fix or rollback due to build breakage. -Skipping pull request bypasses test coverage and could potentially cause the -build to fail, or fail to fix breakage. In addition, pull requests ensure that -changes are communicated properly and potential flaws or improvements can be -spotted, even after the merge happens. - -## Contributor License Agreement - -If you are merging a larger contribution, please make sure that the contributor -has an ICLA on file with the Apache Secretary. You can view the list of -committers [here](https://home.apache.org/phonebook.html?unix=committers), as -well as [ICLA-signers who aren’t yet -committers](https://home.apache.org/unlistedclas.html). - -For smaller contributions, however, this is not required. In this case, we rely -on [clause five](https://www.apache.org/licenses/LICENSE-2.0#contributions) of -the Apache License, Version 2.0, describing licensing of intentionally -submitted contributions. - -## Tests - -Before merging, please make sure that Jenkins tests pass, as visible in the -GitHub pull request. Do not merge the pull request if there are test failures. - -If the pull request contains changes that call for extra test coverage, you can -ask Jenkins to run an extended test suite. For example, if the pull request -modifies a runner, you can run the full `ValidatesRunner` suite with a comment -such as "Run Spark ValidatesRunner". You can run the examples and some IO -integration tests with "Run Java PostCommit". - -## Finishing touches - -At some point in the review process, the change to the codebase will be -complete. However, the pull request may have a collection of review-related -commits that are not meaningful to preserve in the history. The reviewer should -give the LGTM and then request that the author of the pull request rebase, -squash, split, etc, the commits, so that the history is most useful: -* Favor commits that do just one thing. The commit is the smallest unit of easy -rollback; it is easy to roll back many commits, or a whole pull request, but -harder to roll back part of a commit. -* Commit messages should be descriptive and should reference the issue number that they address. -It should later not be necessary to find a merge or first PR commit to find out what caused a change. -* Pull request descriptions should contain a link to the issue being addressed by the changes. -* `CHANGES.md` file should be updated with noteworthy changes (e.g. new features, backward -incompatible changes, dependency changes, etc.). -* Squash the "Fixup!", "Address comments" type of commits that resulted from review iterations. - -## Merging it! - -While it is preferred that authors squash commits after review is complete, -there may be situations where it is more practical for the committer to handle this -(such as when the action to be taken is obvious or the author isn't available). -The committer may use the "Squash and merge" option in Github (or modify the PR commits in other ways). -The committer is ultimately responsible and we "trust the committer's judgment"! - -After all the tests pass, there should be a green merge button at the bottom of -the pull request. There are multiple choices. Unless you want to squash commits -as part of the merge (see above) you should choose "Merge pull -request" and ensure "Create a merge commit" is selected from the drop down. -This preserves the commit history and adds a merge -commit, so be sure the commit history has been curated appropriately. - -Do _not_ use the default GitHub commit message, which looks like this: - - Merge pull request #1234 from some_user/transient_branch_name - - [BEAM-7873] Fix the foo bizzle bazzle - -Instead, pull it all into the subject line: - - Merge pull request #1234: [BEAM-7873] Fix the foo bizzle bazzle - -If you have comments to add, put them in the body of the commit message. - -## Seed jobs - -As a committer, you can now run seed jobs! These are used to update our Jenkins configuration -and can be run to test PRs modifying Groovy files before they are merged. - -To make sure you have these permissions, put up a PR adding yourself to -https://github.com/apache/beam/blob/master/.test-infra/jenkins/Committers.groovy \ No newline at end of file diff --git a/website/www/site/content/en/contribute/dependencies.md b/website/www/site/content/en/contribute/dependencies.md index 7b880e470405f..ac3a669a2d333 100644 --- a/website/www/site/content/en/contribute/dependencies.md +++ b/website/www/site/content/en/contribute/dependencies.md @@ -72,10 +72,6 @@ This will be a blocker for next major and minor version releases of Beam. For manually identified critical dependency updates, Beam community members should create blocking Issues for next release. In addition to this Beam community members may trigger patch releases for any critical dependency fixes that should be made available to users urgently. -__Dependency declarations may identify owners that are responsible for upgrading respective dependencies.__ - -Owners can be mentioned in the yaml files. Blocking Issues will be initially assigned to these owners (if available). Release manager may choose to re-assign these Issues. A dependency may have more than one declared owner and in this case the Issue will be assigned to one of the owners mentioned. - __Dependencies of Java SDK components that may cause issues to other components if leaked should be vendored.__ [Vendoring](https://www.ardanlabs.com/blog/2013/10/manage-dependencies-with-godep.html) is the process of creating copies of third party dependencies. Combined with repackaging, vendoring allows Beam components to depend on third party libraries without causing conflicts to other components. Vendoring should be done in a case-by-case basis since this can increase the total number of dependencies deployed in user's enviroment. diff --git a/website/www/site/content/en/contribute/feature-branches.md b/website/www/site/content/en/contribute/feature-branches.md deleted file mode 100644 index 24853e1621604..0000000000000 --- a/website/www/site/content/en/contribute/feature-branches.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -title: "Beam Feature Branches" ---- - - -# Feature Branches - -Some larger features are developed on a feature branch before being merged into -`master`. In particular, this is often used for initial development of new -components like SDKs or runners. - -We expect the work on a feature branch to be _incomplete_, but it must not -be lower quality. Code reviews for feature branches must have the same -standards as code reviews for `master`. Once a feature branch is ready for -merge to `master`, the set of changes will be too large to review in its -entirety. Because of this, the code reviews during development must be -thorough and trustworthy. - -## Establishing a feature branch - -If your project is large enough to need a feature branch, there should -be a discussion on the mailing list. The first step is to [engage](/contribute/#connect-with-the-beam-community) there to raise awareness -that you want to start a large project. Almost any project should be accepted --- there is no real cost to letting a feature branch exist -- but you may find -other interested contributors or gain other advice from the community. - -After the community discussion, a committer must create your feature branch. -Any committer can do create the branch through the GitHub UIs or by pushing -directly to GitHub or ASF's gitbox. - -## Developing on a feature branch - -To contribute code on a feature branch, use the same process as in the -[Contribution Guide](/contribute/contribution-guide/), but -replace `master` with the name of the feature branch. - -Since feature branches are often used for new components, you may find that -there is no committer familiar with all the details of the new language or -runner. In that case, consider asking someone else familiar with the technology -to do an initial review before looping in a committer for a final review and -merge. - -If you are working on a feature branch, you'll also want to frequently merge in -changes from `master`. This prevents the feature branch from -deviating too far from the current state of `master`. Like all changes, this -should be done via pull request. A committer may self-merge such a pull request -if there are no conflicts or test failures. If there are any conflicts or tests -that need fixing, then those should get a full review from another committer. - -## Merging into `master` - -To merge a feature branch into `master`, new components and major features -should meet the following guidelines. - -1. Have at least 2 contributors interested in maintaining it, and 1 committer - interested in supporting it -2. Provide both end-user and developer-facing documentation -3. Have at least a basic level of unit test coverage -4. Run all existing applicable integration tests with other Beam components and - create additional tests as appropriate - -### Merging a new runner into `master` - -A new runner should: - -1. Be able to handle a subset of the model that addresses a significant set of - use cases, such as ‘traditional batch’ or ‘processing time streaming’. -2. Update the capability matrix with the current status -3. Add a webpage under `documentation/runners` - -### Merging a new SDK into `master` - -A new SDK should: - -1. Provide the ability to construct graphs with all the basic building blocks - of the model (ParDo, GroupByKey, Window, Trigger, etc) -2. Begin fleshing out the common composite transforms (Count, Join, etc) and I/O - connectors (Text, Kafka, etc) -3. Have at least one runner that can execute the complete model (may be a - direct runner) -4. Provide integration tests for executing against current and future runners -5. Add a webpage under `documentation/sdks` - diff --git a/website/www/site/content/en/contribute/get-started-contributing.md b/website/www/site/content/en/contribute/get-started-contributing.md index 7d4b9193ff052..2511ce83db5b9 100644 --- a/website/www/site/content/en/contribute/get-started-contributing.md +++ b/website/www/site/content/en/contribute/get-started-contributing.md @@ -1,13 +1,7 @@ --- -title: "Beam Contribution Guide" +title: "Beam Code Contribution Guide" type: "contribute" layout: "arrow_template" -aliases: -- /contribution-guide/ -- /contribute/contribution-guide/ -- /docs/contribute/ -- /contribute/source-repository/ -- /contribute/design-principles/ --- -# Contribution guide +# Code Contribution guide - - -
    - -Apache Beam is a very welcoming and collaborative community, and there are lots of opportunities to contribute, [both code and non-code](/contribute/). You can, for example: - -- ask or answer questions on [user@beam.apache.org](/community/contact-us/) or - [stackoverflow](https://stackoverflow.com/questions/tagged/apache-beam) -- review proposed design ideas on [dev@beam.apache.org](/community/contact-us/) -- improve the documentation -- file [bug reports](https://github.com/apache/beam/issues/new/choose) -- test releases -- review [changes](https://github.com/apache/beam/pulls) -- write new examples -- improve your favorite language SDK (Java, Python, Go, etc) -- improve specific runners (Apache Flink, Apache Spark, Google - Cloud Dataflow, etc) -- improve or add IO connectors -- add new transform libraries (statistics, ML, image processing, etc) -- work on the core programming model (read more about what a Beam pipeline is and how it runs here in [Documentation](/documentation/basics/)) -- improve the developer experience (for example, Windows guides) -- add answers to the [contribution FAQ]( - https://cwiki.apache.org/confluence/display/BEAM/Contributor+FAQ) -- organize local meetups of users or contributors to Apache Beam - -Below is a tutorial for contributing code to Beam, covering our tools and typical process in -detail. - -
    - -## Get Started - -### Connect with the Beam community - -1. Consider [subscribing to the dev@beam.apache.org](/community/contact-us/) mailing list, especially if you plan to make more than one change or the change will be large. All decisions are consensus-based and happen on the public mailing list. -1. (Optionally) Join the [#beam channel of the ASF Slack](/community/contact-us/). - -### Accounts and Permissions - -- [Beam Wiki Space](https://cwiki.apache.org/confluence/display/BEAM/Apache+Beam): - Anyone has read access. If you wish to contribute changes, please create an account and request edit access on the - [dev@beam.apache.org](/community/contact-us) mailing list (include your Wiki account user ID). - -- Pull requests can only be merged by a - [Beam committer](https://home.apache.org/phonebook.html?pmc=beam). - -- [Voting on a release](https://www.apache.org/foundation/voting.html): Everyone can vote. Only - [Beam PMC](https://home.apache.org/phonebook.html?pmc=beam) members should mark their votes as binding. - -### Communication - -All communication is expected to align with the [Code of Conduct](https://www.apache.org/foundation/policies/conduct). - -Discussion about contributing code to Beam happens on the [dev@beam.apache.org mailing list](/community/contact-us/). Introduce yourself! - -Questions can be asked on the [#beam channel of the ASF Slack](/community/contact-us/). Introduce yourself! - -## Before You Begin - -### Prerequisites - -- A [GitHub](https://github.com/) account. -- A Linux, macOS, or Microsoft Windows development environment. -- Java JDK 8 installed. -- [Docker](https://www.docker.com/) installed for some tasks including building worker containers and testing changes to this website locally. -- For SDK Development: - - [Go](https://golang.org) 1.16.0 or later installed for Go SDK development. - - Python 3.x interpreters. You will need Python interpreters for all Python versions supported by Beam. - Interpreters should be installed and available in shell via `python3.x` commands. For more information, see: - Python installation tips in [Developer Wiki](https://cwiki.apache.org/confluence/display/BEAM/Python+Tips#PythonTips-InstallingPythoninterpreters). -- For large contributions, a signed [Individual Contributor License. - Agreement](https://www.apache.org/licenses/icla.pdf) (ICLA) to the Apache - Software Foundation (ASF). - -### Share Your Intent -1. Find or create an issue in the [Beam repo](https://github.com/apache/beam/issues/new/choose). - Tracking your work in an issue will avoid duplicated or conflicting work, and provide - a place for notes. Later, your pull request will be linked to the issue as well. -2. Comment ".take-issue" on the issue. This will cause the issue to be assigned to you. - When you've completed the issue, you can close it by commenting ".close-issue". - If you are a committer and would like to assign an issue to a non-committer, they must comment - on the issue first; please tag the user asking them to do so or to comment "\`.take-issue\`". - The command will be ignored if it is surrounded by `\`` markdown characters. -3. If your change is large or it is your first change, it is a good idea to - [discuss it on the dev@beam.apache.org mailing list](/community/contact-us/). -4. For large changes create a design doc - ([template](https://s.apache.org/beam-design-doc-template), - [examples](https://s.apache.org/beam-design-docs)) and email it to the [dev@beam.apache.org mailing list](/community/contact-us/). - -### Setup Your Environment - -Before you begin, check out the Wiki pages. There are many useful tips about [Git](https://cwiki.apache.org/confluence/display/BEAM/Git+Tips), [Go](https://cwiki.apache.org/confluence/display/BEAM/Go+Tips), [Gradle](https://cwiki.apache.org/confluence/display/BEAM/Gradle+Tips), [Java](https://cwiki.apache.org/confluence/display/BEAM/Java+Tips), [Python](https://cwiki.apache.org/confluence/display/BEAM/Python+Tips), etc. - -#### Configuration Options -You have two options for configuring your development environment: -- Local: - - Manually installing the [prerequisites](/contribute/#prerequisites). - - Using the automated script for Linux and macOS. -- Container-based: using a [Docker](https://www.docker.com/) image. - -##### Local: Debian-based Distribution - -###### Manual steps - -To install these in a Debian-based distribution: -1. Execute: - ``` - sudo apt-get install \ - openjdk-8-jdk \ - python-setuptools \ - python-pip \ - virtualenv \ - tox \ - docker-ce - ``` -2. On some systems, like Ubuntu 20.04, install these: - ``` - pip3 install grpcio-tools mypy-protobuf - ``` -3. If you develop in GO: - 1. Install [Go](https://golang.org/doc/install). - 2. Check BEAM repo is in: `$GOPATH/src/github.com/apache/` - 3. At the end, it should look like this: `$GOPATH/src/github.com/apache/beam` -4. Once Go is installed, install goavro: - ``` - $ export GOPATH=`pwd`/sdks/go/examples/.gogradle/project_gopath - $ go get github.com/linkedin/goavro/v2 - ``` -**Important**: gLinux users should configure their machines for sudoless Docker. - -###### Automated script for Linux and macOS - -You can install these in a Debian-based distribution for Linux or macOs using the [local-env-setup.sh](https://github.com/apache/beam/blob/master/local-env-setup.sh) script, which is part of the Beam repo. It contains: - -* pip3 packages -* go packages -* goavro -* JDK 8 -* Python -* Docker - -To install execute: -``` -./local-env-setup.sh -``` - -##### Container: Docker-based - -Alternatively, you can use the Docker based local development environment to wrap your clone of the Beam repo -into a container meeting the requirements above. - -You can start this container using the [start-build-env.sh](https://github.com/apache/beam/blob/master/start-build-env.sh) script which is part of the Beam repo. - -Execute: -``` -./start-build-env.sh -``` - -#### Development Setup {#development-setup} - -1. Check [Git workflow tips](https://cwiki.apache.org/confluence/display/BEAM/Git+Tips) if you need help with git forking, cloning, branching, committing, pull requests, and squashing commits. - -2. Make a fork of https://github.com/apache/beam repo. - -3. Clone the forked repository. You can download it anywhere you like. - ``` - $ mkdir -p ~/path/to/your/folder - $ cd ~/path/to/your/folder - $ git clone https://github.com/forked/apache/beam - $ cd beam - ``` - For **Go development**: - - We recommend putting it in your `$GOPATH` (`$HOME/go` by default on Unix systems). - - Clone the repo, and update your branch as normal: - ``` - $ git clone https://github.com/apache/beam.git - $ cd beam - $ git remote add git@github.com:/beam.git - $ git fetch --all - ``` - - Get or Update all the Go SDK dependencies: - ``` - $ go get -u ./... - ``` - -4. Check the environment was set up correctly. - - **Option 1**: validate the Go, Java, and Python environments: - - **Important**: Make sure you have activated Python development. - ``` - ./gradlew :checkSetup - ``` - **Option 2**: Run independent checks: - - For **Go development**: - ``` - export GOLANG_PROTOBUF_REGISTRATION_CONFLICT=ignore./gradlew :sdks:go:examples:wordCount - ``` - - For **Python development**: - ``` - ./gradlew :sdks:python:wordCount - ``` - - For **Java development**: - ``` - ./gradlew :examples:java:wordCount - ``` - -5. Familiarize yourself with gradle and the project structure. - - At the root of the git repository, run: - ``` - $ ./gradlew projects - ``` - Examine the available tasks in a project. For the default set of tasks, use: - ``` - $ ./gradlew tasks - ``` - For a given module, use: - ``` - $ ./gradlew -p sdks/java/io/cassandra tasks - ``` - For an exhaustive list of tasks, use: - ``` - $ ./gradlew tasks --all - ``` - -6. Make sure you can build and run tests. - - Since Beam is a large project, usually, you will want to limit testing to the particular module you are working on. Gradle will build just the necessary things to run those tests. For example: - ``` - $ ./gradlew -p sdks/go check - $ ./gradlew -p sdks/java/io/cassandra check - $ ./gradlew -p runners/flink check - ``` - -7. Now you may want to set up your preferred IDE and other aspects of your development - environment. See the Developers' wiki for tips, guides, and FAQs on: - - [IntelliJ](https://cwiki.apache.org/confluence/display/BEAM/Using+IntelliJ+IDE) - - [Java](https://cwiki.apache.org/confluence/display/BEAM/Java+Tips) - - [Python](https://cwiki.apache.org/confluence/display/BEAM/Python+Tips) - - [Go](https://cwiki.apache.org/confluence/display/BEAM/Go+Tips) - - [Website](https://cwiki.apache.org/confluence/display/BEAM/Website+Tips) - - [Gradle](https://cwiki.apache.org/confluence/display/BEAM/Gradle+Tips) - - [Jenkins](https://cwiki.apache.org/confluence/display/BEAM/Jenkins+Tips) - - [FAQ](https://cwiki.apache.org/confluence/display/BEAM/Contributor+FAQ) - - - -
    - -### Create a Pull Request - -1. Make your code change. Every source file needs to include the Apache license header. Every new dependency needs to - have an open source license [compatible](https://www.apache.org/legal/resolved.html#criteria) with Apache. - -2. Add unit tests for your change. - -3. Use descriptive commit messages that make it easy to identify changes and provide a clear history. - -4. When your change is ready to be reviewed and merged, create a pull request. - -5. Link to the issue you are addressing in your pull request. - -6. The pull request and any changes pushed to it will trigger [pre-commit - jobs](https://cwiki.apache.org/confluence/display/BEAM/Contribution+Testing+Guide#ContributionTestingGuide-Pre-commit). If a test fails and appears unrelated to your - change, you can cause tests to be re-run by adding a single line comment on your - PR: - ``` - retest this please - ``` -Pull request template has a link to a [catalog of trigger phrases](https://github.com/apache/beam/blob/master/.test-infra/jenkins/README.md) -that start various post-commit tests suites. Use these sparingly because post-commit tests consume shared development resources. - -### Review Process and Releases - -#### Get Reviewed -1. Pull requests can only be merged by a - [Beam committer](https://home.apache.org/phonebook.html?pmc=beam). - To find a committer for your area, either: - - look in the OWNERS file of the directory where you changed files, or - - look for similar code merges, or - - ask on [dev@beam.apache.org](/community/contact-us/) - - Use `R: @username` in the pull request to notify a reviewer. - -2. If you don't get any response in 3 business days, email the [dev@beam.apache.org mailing list](/community/contact-us) to ask for someone to look at your pull request. - -#### Make the Reviewer’s Job Easier - -1. Provide context for your changes in the associated issue and/or PR description. - -2. Avoid huge mega-changes. - -3. Review feedback typically leads to follow-up changes. It is easier to review follow-up changes when they are added as additional "fixup" commits to the - existing PR/branch. This allows reviewer(s) to track the incremental progress and focus on new changes, - and keeps comment threads attached to the code. - Please refrain from squashing new commits into reviewed commits before review is completed. - Because squashing reviewed and unreviewed commits often makes it harder to - see the difference between the review iterations, reviewers may ask you to unsquash new changes. - -4. After review is complete and the PR is accepted, fixup commits should be squashed (see [Git workflow tips](https://cwiki.apache.org/confluence/display/BEAM/Git+Tips)). - Beam committers [can squash](/contribute/committer-guide/#merging-it) - all commits in the PR during merge, however if a PR has a mixture of independent changes that should not be squashed, and fixup commits, - then the PR author should help squashing fixup commits to maintain a clean commit history. - -#### Apache Beam Releases - -Apache Beam makes minor releases every 6 weeks. Apache Beam has a -[calendar](https://calendar.google.com/calendar/embed?src=0p73sl034k80oob7seouanigd0%40group.calendar.google.com) for -cutting the next release branch. Your change needs to be checked into master before the release branch is cut -to make the next release. - -#### Stale Pull Requests - -The community will close stale pull requests in order to keep the project -healthy. A pull request becomes stale after its author fails to respond to -actionable comments for 60 days. Author of a closed pull request is welcome to -reopen the same pull request again in the future. - -### Troubleshooting - -If you run into any issues, check out the [contribution FAQ](https://cwiki.apache.org/confluence/display/BEAM/Contributor+FAQ) or ask on the [dev@ mailing list](/community/contact-us/) or [#beam channel of the ASF Slack](/community/contact-us/). - -If you didn't find the information you were looking for in this guide, please -[reach out to the Beam community](/community/contact-us/). - -
    - -## Find Efforts to Contribute to -A great way to contribute is to join an existing effort. If you want to get involved but don’t have a project in mind, check our [list of open starter tasks](https://s.apache.org/beam-starter-tasks). -For the most intensive efforts, check out the [roadmap](/roadmap/). - -## Additional Resources -Please see Beam developers’ [Wiki Contributor FAQ](https://cwiki.apache.org/confluence/display/BEAM/Contributor+FAQ) for more information. - -If you are contributing a ```PTransform``` to Beam, we have an extensive [PTransform Style Guide](/contribute/ptransform-style-guide). - -If you are contributing a Runner to Beam, refer to the [Runner authoring guide](/contribute/runner-guide/). - -Review [design documents](https://s.apache.org/beam-design-docs). - -You can also find out more information on the [Beam developers’ Wiki]( -https://cwiki.apache.org/confluence/display/BEAM/Apache+Beam). +For information on other ways to contribute, see the general [Beam contribution guide](https://beam.apache.org/contribute/). \ No newline at end of file diff --git a/website/www/site/content/en/contribute/ptransform-style-guide.md b/website/www/site/content/en/contribute/ptransform-style-guide.md index 606ebae488e5f..050a4bf93265b 100644 --- a/website/www/site/content/en/contribute/ptransform-style-guide.md +++ b/website/www/site/content/en/contribute/ptransform-style-guide.md @@ -30,6 +30,7 @@ Be consistent with prior art: * If there is already a similar transform in some SDK, make the API of your transform similar, so that users' experience with one of them will transfer to the other. This applies to transforms in the same-language SDK and different-language SDKs. *Exception:* pre-existing transforms that clearly violate the current style guide for the sole reason that they were developed before this guide was ratified. In this case, the style guide takes priority over consistency with the existing transform. * When there is no existing similar transform, stay within what is idiomatic within your language of choice (e.g. Java or Python). +* Please use this [design doc template](https://s.apache.org/ptransform-design-doc) if possible, when proposing new transforms. ### Exposing a PTransform vs. something else diff --git a/website/www/site/content/en/contribute/release-guide.md b/website/www/site/content/en/contribute/release-guide.md deleted file mode 100644 index 65dfeb20d3711..0000000000000 --- a/website/www/site/content/en/contribute/release-guide.md +++ /dev/null @@ -1,1319 +0,0 @@ ---- -title: "Beam Release Guide" ---- - - -# Apache Beam Release Guide - -{{< toc >}} - -## Introduction - -The Apache Beam project periodically declares and publishes releases. -A release is one or more packages of the project artifact(s) that are approved for general public distribution and use. -They may come with various degrees of caveat regarding their perceived quality and potential for change, such as “alpha”, “beta”, “incubating”, “stable”, etc. - -The Beam community treats releases with great importance. -They are a public face of the project and most users interact with the project only through the releases. Releases are signed off by the entire Beam community in a public vote. - -Each release is executed by a *Release Manager*, who is selected among the Beam committers. -This document describes the process that the Release Manager follows to perform a release. -Any changes to this process should be discussed and adopted on the [dev@ mailing list](/get-started/support/). - -Please remember that publishing software has legal consequences. -This guide complements the foundation-wide [Product Release Policy](https://www.apache.org/dev/release.html) and [Release Distribution Policy](https://www.apache.org/dev/release-distribution). - -### Overview - -Alt text - -The release process consists of several steps: - -1. Decide to release -1. Prepare for the release -1. Update base image dependencies for Python container images -1. Investigate performance regressions -1. Create a release branch -1. Verify release branch -1. Build a release candidate -1. Vote on the release candidate -1. During vote process, run validation tests -1. If necessary, fix any issues and go back to step 3. -1. Finalize the release -1. Promote the release - - -## 1. Decide to release - -Deciding to release and selecting a Release Manager is the first step of the release process. -This is a consensus-based decision of the entire community. - -Anybody can propose a release on the dev@ mailing list, giving a solid argument and nominating a committer as the Release Manager (including themselves). -There’s no formal process, no vote requirements, and no timing requirements. Any objections should be resolved by consensus before starting the release. - -In general, the community prefers to have a rotating set of 3-5 Release Managers. -Keeping a small core set of managers allows enough people to build expertise in this area and improve processes over time, without Release Managers needing to re-learn the processes for each release. -That said, if you are a committer interested in serving the community in this way, please reach out to the community on the dev@ mailing list. - -### Checklist to proceed to the next step - -1. Community agrees to release -1. Community selects a Release Manager - -********** - -## 2. Prepare for the release - -Before your first release, you should perform one-time configuration steps. - This will set up your security keys for signing the release and access to various release repositories. - -To prepare for each release, you should audit the project status in the GitHub issue tracker, and do necessary bookkeeping. -Finally, you should create a release branch from which individual release candidates will be built. - -__NOTE__: If you are using [GitHub two-factor authentication](https://help.github.com/articles/securing-your-account-with-two-factor-authentication-2fa/) and haven't configure HTTPS access, -please follow [the guide](https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/) to configure command line access. - - -### Accounts - -Please have these credentials ready at hand, you will likely need to enter them multiple times: - -* GPG pass phrase (see the next section); -* Apache ID and Password; -* GitHub ID and Password. -* DockerHub ID and Password. (You should be a member of maintainer team; email at dev@ if you are not.) -* Account to access to apache-beam-testing Google Cloud Platform project. The account must have permissions to start Cloud Build triggers. Required for Playground environment update. (E-mail to pabloem@google.com to request access) - -### One-time setup instructions - - -#### GPG Key - -You need to have a GPG key to sign the release artifacts. -Please be aware of the ASF-wide [release signing guidelines](https://www.apache.org/dev/release-signing.html). -If you don’t have a GPG key associated with your Apache account, please create one according to the guidelines. - -There are 2 ways to configure your GPG key for release, either using release automation script(which is recommended), or running all commands manually. - -##### Use preparation_before_release.sh to setup GPG -* **Script:** [preparation_before_release.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/preparation_before_release.sh) - -* **Usage** - ``` - ./beam/release/src/main/scripts/preparation_before_release.sh - ``` -* **Tasks included** - 1. Help you create a new GPG key if you want. - 1. Configure ```git user.signingkey``` with chosen pubkey. - 1. Add chosen pubkey into [dev KEYS](https://dist.apache.org/repos/dist/dev/beam/KEYS) and [release KEYS](https://dist.apache.org/repos/dist/release/beam/KEYS) - - **NOTES**: Only PMC can write into [release repo](https://dist.apache.org/repos/dist/release/beam/). - 1. Start GPG agents. - -__NOTE__: When generating the key, please make sure you choose the key type as __RSA and RSA (default)__ and key size as __4096 bit__. - -* To run the commands manually, refer to the contents of `preparation_before_release.sh`. - -##### Key ID - -* You may need your Key ID for future steps. Determine your Apache GPG Key and Key ID as follows: - - gpg --list-sigs --keyid-format LONG - - This will list your GPG keys. One of these should reflect your Apache account, for example: - - -------------------------------------------------- - pub rsa4096/845E6689845E6689 2016-02-23 - uid Nomen Nescio - sub rsa4096/BA4D50BEBA4D50BE 2016-02-23 - - Here, the key ID is the 16-digit hex string in the `pub` line: `845E6689845E6689`. - -#### Access to Apache Nexus repository - -Configure access to the [Apache Nexus repository](https://repository.apache.org/), which enables final deployment of releases to the Maven Central Repository. - -1. You log in with your Apache account. -1. Confirm you have appropriate access by finding `org.apache.beam` under `Staging Profiles`. -1. Navigate to your `Profile` (top right dropdown menu of the page). -1. Choose `User Token` from the dropdown, then click `Access User Token`. Copy a snippet of the Maven XML configuration block. -1. Insert this snippet twice into your global Maven `settings.xml` file, typically `${HOME}/.m2/settings.xml`. The end result should look like this, where `TOKEN_NAME` and `TOKEN_PASSWORD` are your secret tokens: - - - - - - apache.releases.https - TOKEN_NAME - TOKEN_PASSWORD - - - apache.snapshots.https - TOKEN_NAME - TOKEN_PASSWORD - - - - -#### Submit your GPG public key into MIT PGP Public Key Server -In order to make yourself have right permission to stage java artifacts in Apache Nexus staging repository, -please submit your GPG public key into the [Ubuntu OpenPGP Key Server](https://keyserver.ubuntu.com/). - -You will need to use an ascii-armored version of your key. -This can be obtained by running `gpg --export --armor` and copying the whole block -(including `----- PGP PUBLIC KEY BLOCK-----`). - -#### Website development setup - -Updating the Beam website requires submitting PRs to both the main `apache/beam` repo and the `apache/beam-site` repo. -The first contains reference manuals generated from SDK code, while the second updates the current release version number. - -You should already have setup a local clone of `apache/beam`. -Setting up a clone of `apache/beam-site` is similar: - - $ git clone -b release-docs https://github.com/apache/beam-site.git - $ cd beam-site - $ git remote add git@github.com:/beam-site.git - $ git fetch --all - $ git checkout -b origin/release-docs - -Further instructions on website development on `apache/beam` is [here](https://github.com/apache/beam/blob/master/website). -Background information about how the website is updated can be found in [Beam-Site Automation Reliability](https://s.apache.org/beam-site-automation). - -#### Register to PyPI - -Release manager needs to have an account with PyPI. -If you need one, [register at PyPI](https://pypi.python.org/account/register/). -You also need to be a maintainer (or an owner) of the [apache-beam](https://pypi.python.org/pypi/apache-beam) package in order to push a new release. -Ask on the mailing list for assistance. - -#### Login to DockerHub -If you are a member of the [`beammaintainers` DockerHub team](https://hub.docker.com/orgs/apache/teams/beammaintainers), run following command manually. -It will ask you to input your DockerHub ID and password if authorization info cannot be found from ~/.docker/config.json file. - -``` -docker login docker.io -``` - -After successful login, authorization info will be stored at ~/.docker/config.json file. -For example, -``` -"https://index.docker.io/v1/": { - "auth": "xxxxxx" -} -``` - -If you are not already a member of the `beammaintainers` team, please email `dev@` for help with any DockerHub related tasks. We are not able -to add more members to the DockerHub team because [the ASF has a limited number of seats available](https://infra.apache.org/docker-hub-policy.html). - -### Create a new milestone in GitHub - -When contributors resolve an issue in GitHub, they are tagging it with a release that will contain their changes. -With the release currently underway, new issues should be resolved against a subsequent future release. -Therefore, you should create a release item for this subsequent release, as follows: - -1. In GitHub, navigate to [`Issues > Milestones > New Milestone`](https://github.com/apache/beam/milestones). -1. Add a new release. Choose the next minor version number after the version currently underway, select the next release due date (generally 6 weeks from today’s date) as the `Start Date`, and choose `Create Milestone`. -1. At the end of the release, go to the same page and mark the recently released version as closed. - - -********** - - -## 3. Update base image dependencies for Python container images - -See instructions at: https://s.apache.org/beam-python-requirements-generate - -Ideally, do the update at least a week before the release cut, so that any issues -related to the update have time to surface. - -## 4. Investigate performance regressions - -Check the Beam load tests for possible performance regressions. -Measurements are available on [metrics.beam.apache.org](http://metrics.beam.apache.org). - -All Runners which publish data should be checked for the following, in both *batch* and *streaming* mode: - -- [ParDo](http://metrics.beam.apache.org/d/MOi-kf3Zk/pardo-load-tests) and [GBK](http://metrics.beam.apache.org/d/UYZ-oJ3Zk/gbk-load-test): Runtime, latency, checkpoint duration -- [Nexmark](http://metrics.beam.apache.org/d/ahudA_zGz/nexmark): Query runtime for all queries -- [IO](http://metrics.beam.apache.org/d/bnlHKP3Wz/java-io-it-tests-dataflow): Runtime - -If regressions are found, the release branch can still be created, but the regressions should be investigated and fixed as part of the release process. -The role of the release manager is to file GitHub issues for each regression with the milestone set to the to-be-released version. -The release manager oversees these just like any other issue marked with the milestone of the release. - -The mailing list should be informed to allow fixing the regressions in the course of the release. - -## 5. Create a release branch in apache/beam repository - -Attention: Only committer has permission to create release branch in apache/beam. - -Release candidates are built from a release branch. -As a final step in preparation for the release, you should create the release branch, push it to the Apache code repository, and update version information on the original branch. -The final state of the repository should match this diagram: - -Increment minor version on master branch and set Dataflow container version on release branch - -The key points to know: - -- The `master` branch has the SNAPSHOT/dev version incremented. -- The release branch has the SNAPSHOT/dev version to be released. -- The Dataflow container image should be modified to the version to be released. - -This will all be accomplished by the [cut_release_branch](https://github.com/apache/beam/actions/workflows/cut_release_branch.yml) -workflow. This workflow will also update [mass_comment.py](https://github.com/apache/beam/blob/master/release/src/main/scripts/mass_comment.py) -to contain all of the active Jenkins jobs. - -After updating the master branch, the workflow will also start a build of -[the nightly snapshot](https://ci-beam.apache.org/job/beam_Release_NightlySnapshot/) against master branch. -Some processes, including our archetype tests, rely on having a live SNAPSHOT of the current version from the `master` branch. -Once the release branch is cut, these SNAPSHOT versions are no longer found, so builds will be broken until a new snapshot is available. -The workflow starts the nightly snapshot by creating an empty PR against apache:master (which will be linked to in the logs). - -#### Use cut_release_branch.sh to cut a release branch -* **Action:** [cut_release_branch](https://github.com/apache/beam/actions/workflows/cut_release_branch.yml) (click `run workflow`) - -In order to run this workflow, you will need to provide a Jenkins username and API token. Your Jenkins username should be your Apache ID. -Your Jenkins API token can be generated by visiting https://ci-beam.apache.org/user//configure and clicking -`Add new token` in the API token section. - -* Tasks you need to do manually to __verify the SNAPSHOT build__ - 1. Check whether the Jenkins job gets triggered. If not, please comment ```Run Gradle Publish``` into the generated PR. - 1. After verifying build succeeded, you need to close PR manually. - 1. Manually update `CHANGES.md` on `master` by adding a new section for the next release ([example](https://github.com/apache/beam/commit/96ab1fb3fe07acf7f7dc9d8c829ae36890d1535c)). - - -********** - - -## 6. Verify release branch - -After the release branch is cut you need to make sure it builds and has no significant issues that would block the creation of the release candidate. -There are 2 ways to perform this verification, either running automation script(recommended), or running all commands manually. - -! Dataflow tests will fail if Dataflow worker container is not created and published by this time. (Should be done by Google) - -#### Run automation script (verify_release_build.sh) -* **Script:** [verify_release_build.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/verify_release_build.sh) - -* **Usage** - 1. Create a personal access token from your Github account. - See instruction [here](https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line). - It'll be used by the script for accessing Github API. - You only need to enable "repo" permissions to this token. - 1. Update required configurations listed in `RELEASE_BUILD_CONFIGS` in [script.config](https://github.com/apache/beam/blob/master/release/src/main/scripts/script.config) - 1. Then run - ``` - cd beam/release/src/main/scripts && ./verify_release_build.sh - ``` - 1. Trigger `beam_Release_Gradle_Build` and all Jenkins PostCommit jobs from the PR created by the previous step. - You can run [mass_comment.py](https://github.com/apache/beam/blob/master/release/src/main/scripts/mass_comment.py) to do that. - Or manually add one trigger phrase per PR comment. - See [jenkins_jobs.txt](https://github.com/apache/beam/blob/master/release/src/main/scripts/jenkins_jobs.txt) - for a full list of phrases. - -* **Tasks included in the script** - 1. Installs ```hub``` with your agreement and setup local git repo; - 1. Create a test PR against release branch; - -The [`beam_Release_Gradle_Build`](https://ci-beam.apache.org/job/beam_Release_Gradle_Build/) Jenkins job runs `./gradlew build -PisRelease`. -This only verifies that everything builds with unit tests passing. - -#### Verify the build succeeds - -* Tasks you need to do manually to __verify the build succeed__: - 1. Check the build result. - 2. If build failed, scan log will contain all failures. - 3. You should stabilize the release branch until release build succeeded. - -There are some projects that don't produce the artifacts, e.g. `beam-test-tools`, you may be able to ignore failures there. - -To triage the failures and narrow things down you may want to look at `settings.gradle.kts` and run the build only for the projects you're interested at the moment, e.g. `./gradlew :runners:java-fn-execution`. - -#### (Alternative) Run release build manually (locally) -You will need to have Python interpreters for all supported Python minor -versions to run Python tests. See Python installation tips in [Developer Wiki](https://cwiki.apache.org/confluence/display/BEAM/Python+Tips#PythonTips-InstallingPythoninterpreters). - -* **Run gradle release build** - - 1. Clean current workspace - - ``` - git clean -fdx - ./gradlew clean - ``` - - 1. Unlock the secret key - ``` - gpg --output ~/doc.sig --sign ~/.bashrc - ``` - - 1. Run build command - ``` - ./gradlew build -PisRelease --no-parallel --scan --stacktrace --continue - ``` - - To speed things up locally you might want to omit `--no-parallel`. You can also omit `--continue` - if you want build fails after the first error instead of continuing, it may be easier and faster - to find environment issues this way without having to wait until the full build completes. - - -#### Create release-blocking issues in GitHub - -The verify_release_build.sh script may include failing or flaky tests. -For each of the failing tests create a GitHub Issue with the following properties: - -* **Issue Type:** Bug - -* **Summary:** Name of failing gradle task and name of failing test (where applicable) in form of :MyGradleProject:SomeGradleTask NameOfFailedTest: Short description of failure - -* **Priority:** P1 - -* **Component:** "test-failures" - -* **Milestone:** Release number of verified release branch - -* **Description:** Description of failure - -#### Inform the mailing list - -The dev@beam.apache.org mailing list should be informed about the release branch being cut. -Alongside with this note, a list of pending issues and to-be-triaged issues should be included. -Afterwards, this list can be refined and updated by the release manager and the Beam community. - -********** - - -## 7. Triage release-blocking issues in GitHub - -There could be outstanding release-blocking issues, which should be triaged before proceeding to build a release candidate. -We track them by assigning the blocked release to the issue's milestone before the issue is resolved. - - -The release manager should triage what does and does not block a release. -The list of release-blocking issues is available at the [milestone status page](https://github.com/apache/beam/milestones). -Triage each unresolved issue with one of the following resolutions: - -* An issue should not block the release if the problem exists in the current released version or is a bug in new functionality that does not exist in the current released version. -* An issue should be a blocker if the problem is a regression between the currently released version and the release in progress and has no easy workaround. - -For all GitHub issues: - -* If the issue has been resolved and the GitHub issue was not updated, resolve it accordingly. - -For issues with type "Bug" or labeled "flaky": - -* If the issue is a known continuously failing test, it is not acceptable to defer this until the next release. - Please work with the Beam community to resolve the issue. -* If the issue is a known flaky test, make an attempt to delegate a fix. - However, if the issue may take too long to fix (to the discretion of the release manager): - * Delegate manual testing of the flaky issue to ensure no release blocking issues. - * Update the milestone to the version of the next release. - Please consider discussing this with stakeholders and the dev@ mailing list, as appropriate. - -For all other GitHub issues: - -* If the issue has not been resolved and it is acceptable to defer this until the next release, update the milestone to the new version you just created. - Please consider discussing this with stakeholders and the dev@ mailing list, as appropriate. -* If the issue has not been resolved and it is not acceptable to release until it is fixed, the release cannot proceed. - Instead, work with the Beam community to resolve the issue. - -If there is a bug found in the RC creation process/tools, those issues should be considered high priority and fixed in 7 days. - -### Review cherry-picks - -Check if there are outstanding cherry-picks into the release branch, [e.g. for `2.14.0`](https://github.com/apache/beam/pulls?utf8=%E2%9C%93&q=is%3Apr+base%3Arelease-2.14.0). -Make sure they have blocker Issues attached and are OK to get into the release by checking with community if needed. - -As the Release Manager you are empowered to accept or reject cherry-picks to the release branch. -You are encouraged to ask the following questions to be answered on each cherry-pick PR and you can choose to reject cherry-pick requests if these questions are not satisfactorily answered: - -* Is this a regression from a previous release? (If no, fix could go to a newer version.) -* Is this a new feature or related to a new feature? (If yes, fix could go to a new version.) -* Would this impact production workloads for users? (E.g. if this is a direct runner only fix it may not need to be a cherry pick.) -* What percentage of users would be impacted by this issue if it is not fixed? (E.g. If this is predicted to be a small number it may not need to be a cherry pick.) -* Would it be possible for the impacted users to skip this version? (If users could skip this version, fix could go to a newer version.) - -It is important to accept major/blocking fixes to isolated issues to make a higher quality release. -However, beyond that each cherry pick will increase the time required for the release and add more last minute code to the release branch. -Neither late releases nor not fully tested code will provide positive user value. - -__Tip__: Another tool in your toolbox is the known issues section of the release blog. -Consider adding known issues there for minor issues instead of accepting cherry picks to the release branch. - - -********** - - -## 8. Build a release candidate - -### Checklist before proceeding - -* Release Manager’s GPG key is published to `dist.apache.org`; -* Release Manager’s GPG key is configured in `git` configuration; -* Set `SIGNING_KEY` to the public key of the Manager's GPG key; -* Release Manager has `org.apache.beam` listed under `Staging Profiles` in Nexus; -* Release Manager’s Nexus User Token is configured in `settings.xml`; -* GitHub issue release item for the subsequent release has been created; -* All test failures from branch verification have associated GitHub issues; -* There are no release blocking GitHub issues; -* Release branch has been created; -* There are no open pull requests to release branch; -* Originating branch has the version information updated to the new version; -* Nightly snapshot is in progress (do revisit it continually); -* Set `JAVA_HOME` to JDK 8 (Example: `export JAVA_HOME=/example/path/to/java/jdk8`). -* Have Java 11 installed. - -The core of the release process is the build-vote-fix cycle. -Each cycle produces one release candidate. -The Release Manager repeats this cycle until the community approves one release candidate, which is then finalized. - -For this step, we recommend you using automation script to create a RC, but you still can perform all steps manually if you want. - -### Tag a chosen commit for the RC - -Release candidates are built from single commits off the release branch. -Before building, the version must be set to a non-SNAPSHOT, non-dev version. -The final state of the repository should match this diagram: - -Set version to non-SNAPSHOT, non-dev, on tagged RC commit - -- The release branch is unchanged. -- There is a commit not on the release branch with the version adjusted. -- The RC tag points to that commit. - -* **Script:** [choose_rc_commit.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/choose_rc_commit.sh) - -* **Usage** - - ./beam/release/src/main/scripts/choose_rc_commit.sh \ - --release "${RELEASE_VERSION}" \ - --rc "${RC_NUM}" \ - --commit "${COMMIT_REF}" \ - --clone \ - --push-tag - -You can do a dry run by omitting the `--push-tag` flag. Then it will only clone the repo, -adjust the version, and add the tag locally. If it looks good, run it again with `--push-tag`. -If you already have a clone that includes the `${COMMIT_REF}` then you can omit `--clone`. This -is perfectly safe since the script does not depend on the current working tree. - -See the source of the script for more details, or to run commands manually in case of a problem. - -### Run build_release_candidate GitHub Action to create a release candidate - -Note: This step is partially automated (in progress), so part of the rc creation is done by GitHub Actions and the rest is done by a script. -You don't need to wait for the action to complete to start running the script. - -* **Action** [build_release_candidate](https://github.com/apache/beam/actions/workflows/build_release_candidate.yml) (click `run workflow`) - -* **The script will:** - 1. Clone the repo at the selected RC tag. - 1. Run gradle publish to push java artifacts into Maven staging repo. - -#### Tasks you need to do manually - - 1. Publish staging artifacts - 1. Log in to the [Apache Nexus](https://repository.apache.org/#stagingRepositories) website. - 1. Navigate to Build Promotion -> Staging Repositories (in the left sidebar). - 1. Select repository `orgapachebeam-NNNN`. - 1. Click the Close button. - 1. When prompted for a description, enter “Apache Beam, version X, release candidate Y”. - 1. Review all staged artifacts on `https://repository.apache.org/content/repositories/orgapachebeam-NNNN/`. - They should contain all relevant parts for each module, including `pom.xml`, jar, test jar, javadoc, etc. - Artifact names should follow [the existing format](https://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.beam%22) in which artifact name mirrors directory structure, e.g., `beam-sdks-java-io-kafka`. - Carefully review any new artifacts. - Some additional validation should be done during the rc validation step. - -### Run build_release_candidate.sh to create a release candidate - -* **Script:** [build_release_candidate.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/build_release_candidate.sh) - -* **Usage** - - ./beam/release/src/main/scripts/build_release_candidate.sh --release "${RELEASE_VERSION}" --rc "${RC_NUM}" --github-user "${GITHUB_USER}" --java11-home "${JAVA11_HOME}" --signing-key "${SIGNING_KEY}" - -* **The script will:** - 1. Clone the repo at the selected RC tag. - 1. Stage source release into dist.apache.org dev [repo](https://dist.apache.org/repos/dist/dev/beam/). - 1. Stage, sign and hash python source distribution and wheels into dist.apache.org dev repo python dir - 1. Stage SDK docker images to [docker hub Apache organization](https://hub.docker.com/search?q=apache%2Fbeam&type=image). -Note: if you are not a member of the [`beammaintainers` DockerHub team](https://hub.docker.com/orgs/apache/teams/beammaintainers) you will need -help with this step. Please email `dev@` and ask a member of the `beammaintainers` DockerHub team for help. - 1. Create a PR to update beam-site, changes includes: - * Copy python doc into beam-site - * Copy java doc into beam-site - * **NOTE**: Do not merge this PR until after an RC has been approved (see "Finalize the Release"). - -#### Tasks you need to do manually - 1. Verify the script worked. - 1. Verify that the source and Python binaries are present in [dist.apache.org](https://dist.apache.org/repos/dist/dev/beam). - 1. Verify Docker images are published. How to find images: - 1. Visit [https://hub.docker.com/u/apache](https://hub.docker.com/search?q=apache%2Fbeam&type=image) - 2. Visit each repository and navigate to *tags* tab. - 3. Verify images are pushed with tags: ${RELEASE_VERSION}_rc{RC_NUM} - 1. Verify that third party licenses are included in Docker containers by logging in to the images. - - For Python SDK images, there should be around 80 ~ 100 dependencies. - Please note that dependencies for the SDKs with different Python versions vary. - Need to verify all Python images by replacing `${ver}` with each supported Python version `X.Y`. - ``` - docker run --rm -it --entrypoint=/bin/bash apache/beam_python${ver}_sdk:${RELEASE_VERSION}rc${RC_NUM} - ls -al /opt/apache/beam/third_party_licenses/ | wc -l - ``` - - For Java SDK images, there should be around 200 dependencies. - ``` - docker run --rm -it --entrypoint=/bin/bash apache/beam_java${ver}_sdk:${RELEASE_VERSION}rc${RC_NUM} - ls -al /opt/apache/beam/third_party_licenses/ | wc -l - ``` - -### Upload release candidate to PyPi - -* **Script:** [deploy_release_candidate_pypi.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/deploy_release_candidate_pypi.sh) - -* **Usage** - - ./release/src/main/scripts/deploy_release_candidate_pypi.sh \ - --release "${RELEASE_VERSION}" \ - --rc "${RC_NUM}" \ - --user "${GITHUB_USER}" \ - --deploy - -* **The script will:** - 1. Download python binary artifacts - 1. Deploy release candidate to PyPI - -__Attention:__ Verify that: -* The File names version include ``rc-#`` suffix -* [Download Files](https://pypi.org/project/apache-beam/#files) have: - * All wheels uploaded as artifacts - * Release source's zip published - * Signatures and hashes do not need to be uploaded - -You can do a dry run by omitting the `--deploy` flag. Then it will only download the release candidate binaries. If it looks good, rerun it with `--deploy`. - -See the source of the script for more details or to run commands manually in case of a problem. - - - -********** - - -## 9. Prepare documents - -### Propose pull requests for website updates - -Beam publishes API reference manuals for each release on the website. -For Java and Python SDKs, that’s Javadoc and PyDoc, respectively. -The final step of building the candidate is to propose website pull requests that update these manuals. - -Merge the pull requests only after finalizing the release. -To avoid invalid redirects for the 'current' version, merge these PRs in the order listed. -Once the PR is merged, the new contents will get picked up automatically and served to the Beam website, usually within an hour. -A committer can manually trigger the [beam_PostCommit_Website_Publish](https://ci-beam.apache.org/job/beam_PostCommit_Website_Publish/) task in Jenkins to avoid waiting. - -**PR 1: apache/beam-site** - -This pull request is against the `apache/beam-site` repo, on the `release-docs` branch ([example](https://github.com/apache/beam-site/pull/603)). -It is created by `build_release_candidate.sh` (see above). - -**PR 2: apache/beam** - -This pull request is against the `apache/beam` repo, on the `master` branch ([example](https://github.com/apache/beam/pull/17378)). - -* Update `CHANGES.md` to update release date and remove template. -* Update release version in `website/www/site/config.toml`. -* Add new release in `website/www/site/content/en/get-started/downloads.md`. - * Download links will not work until the release is finalized. -* Update links to prior releases to point to https://archive.apache.org (see - example PR). -* Create the Blog post: - -#### Blog post - -Use the template below to write a blog post for the release. -See [beam-2.31.0.md](https://github.com/apache/beam/commit/a32a75ed0657c122c6625aee1ace27994e7df195#diff-1e2b83a4f61dce8014a1989869b6d31eb3f80cb0d6dade42fb8df5d9407b4748) as an example. -- Copy the changes for the current release from `CHANGES.md` to the blog post and edit as necessary. -- Be sure to add yourself to [authors.yml](https://github.com/apache/beam/blob/master/website/www/site/data/authors.yml) if necessary. - -__Tip__: Use git log to find contributors to the releases. (e.g: `git fetch origin --tags; git log --pretty='%aN' ^v2.10.0 v2.11.0-RC1 | sort | uniq`). -Make sure to clean it up, as there may be duplicate or incorrect user names. - -__NOTE__: Make sure to include any breaking changes, even to `@Experimental` features, -all major features and bug fixes, and all known issues. - -**Template:** - - --- - title: "Apache Beam {$RELEASE_VERSION}" - date: YYYY-MM-DD H:MM:00 Z - categories: - - blog - - release - authors: - - {$RELEASE_MANAGER} - --- - - - We are happy to present the new {$RELEASE_VERSION} release of Beam. - This release includes both improvements and new functionality. - See the [download page](/get-started/downloads/{$DOWNLOAD_ANCHOR}) for this release. - - <{$REMOVE_FOR_VALID_SUMMARY_BREAK}!--more--> - - For more information on changes in {$RELEASE_VERSION}, check out the [detailed release notes]({$LINK_TO_GITHUB_MILESTONE}). - - ## Highlights - - * New highly anticipated feature X added to Python SDK ([#X](https://github.com/apache/beam/issues/X)). - * New highly anticipated feature Y added to Java SDK ([#Y](https://github.com/apache/beam/issues/Y)). - - {$TOPICS e.g.:} - ### I/Os - * Support for X source added (Java) ([#X](https://github.com/apache/beam/issues/X)). - {$TOPICS} - - ### New Features / Improvements - - * X feature added (Python) ([#X](https://github.com/apache/beam/issues/X)). - * Y feature added (Java) [#Y](https://github.com/apache/beam/issues/Y). - - ### Breaking Changes - - * X behavior was changed ([#X](https://github.com/apache/beam/issues/X)). - * Y behavior was changed ([#Y](https://github.com/apache/beam/issues/Y)). - - ### Deprecations - - * X behavior is deprecated and will be removed in X versions ([#X](https://github.com/apache/beam/issues/X)). - - ### Bugfixes - - * Fixed X (Python) ([#X](https://github.com/apache/beam/issues/X)). - * Fixed Y (Java) ([#Y](https://github.com/apache/beam/issues/Y)). - - ### Known Issues - - * {$KNOWN_ISSUE_1} - * {$KNOWN_ISSUE_2} - - ## List of Contributors - - According to git shortlog, the following people contributed to the {$RELEASE_VERSION} release. Thank you to all contributors! - - ${CONTRIBUTORS} - - -#### Checklist to proceed to the next step - -1. Maven artifacts deployed to the staging repository of [repository.apache.org](https://repository.apache.org/content/repositories/) -1. Source distribution deployed to the dev repository of [dist.apache.org](https://dist.apache.org/repos/dist/dev/beam/) -1. Website pull request proposed to list the [release](/get-started/downloads/), publish the [Java API reference manual](https://beam.apache.org/releases/javadoc/), and publish the [Python API reference manual](https://beam.apache.org/releases/pydoc/). -1. Docker images are published to [DockerHub](https://hub.docker.com/search?q=apache%2Fbeam&type=image) with tags: {RELEASE_VERSION}_rc{RC_NUM}. - -You can (optionally) also do additional verification by: -1. Check that Python zip file contains the `README.md`, `NOTICE`, and `LICENSE` files. -1. Check hashes (e.g. `md5sum -c *.md5` and `sha1sum -c *.sha1`. Note that signature/checksum files of Java artifacts may not contain filenames. Hence you might need to compare checksums/signatures manually or modify the files by appending the filenames.) -1. Check signatures (e.g. `gpg --verify apache-beam-1.2.3-python.zip.asc apache-beam-1.2.3-python.zip`) -1. `grep` for legal headers in each file. -1. Run all jenkins suites and include links to passing tests in the voting email. -1. Pull docker images to make sure they are pullable. -``` -docker pull {image_name} -docker pull apache/beam_python3.7_sdk:2.39.0rc1 -``` - - -********** - - -## 10. Vote and validate release candidate - -Once you have built and individually reviewed the release candidate, please share it for the community-wide review. -Please review foundation-wide [voting guidelines](https://www.apache.org/foundation/voting.html) for more information. - -Start the review-and-vote thread on the dev@ mailing list. -Here’s an email template; please adjust as you see fit. - - From: Release Manager - To: dev@beam.apache.org - Subject: [VOTE] Release 1.2.3, release candidate #3 - - Hi everyone, - Please review and vote on the release candidate #3 for the version 1.2.3, as follows: - [ ] +1, Approve the release - [ ] -1, Do not approve the release (please provide specific comments) - - - Reviewers are encouraged to test their own use cases with the release candidate, and vote +1 if - no issues are found. Only PMC member votes will count towards the final vote, but votes from all - community members is encouraged and helpful for finding regressions; you can either test your own - use cases or use cases from the validation sheet [10]. - - The complete staging area is available for your review, which includes: - * GitHub Release notes [1], - * the official Apache source release to be deployed to dist.apache.org [2], which is signed with the key with fingerprint FFFFFFFF [3], - * all artifacts to be deployed to the Maven Central Repository [4], - * source code tag "v1.2.3-RC3" [5], - * website pull request listing the release [6], the blog post [6], and publishing the API reference manual [7]. - * Java artifacts were built with Gradle GRADLE_VERSION and OpenJDK/Oracle JDK JDK_VERSION. - * Python artifacts are deployed along with the source release to the dist.apache.org [2] and PyPI[8]. - * Go artifacts and documentation are available at pkg.go.dev [9] - * Validation sheet with a tab for 1.2.3 release to help with validation [10]. - * Docker images published to Docker Hub [11]. - * PR to run tests against release branch [12]. - - The vote will be open for at least 72 hours. It is adopted by majority approval, with at least 3 PMC affirmative votes. - - For guidelines on how to try the release in your projects, check out our blog post at /blog/validate-beam-release/. - - Thanks, - Release Manager - - [1] https://github.com/apache/beam/milestone/1... - [2] https://dist.apache.org/repos/dist/dev/beam/1.2.3/ - [3] https://dist.apache.org/repos/dist/release/beam/KEYS - [4] https://repository.apache.org/content/repositories/orgapachebeam-NNNN/ - [5] https://github.com/apache/beam/tree/v1.2.3-RC3 - [6] https://github.com/apache/beam/pull/... - [7] https://github.com/apache/beam-site/pull/... - [8] https://pypi.org/project/apache-beam/1.2.3rc3/ - [9] https://pkg.go.dev/github.com/apache/beam/sdks/v2@v1.2.3-RC3/go/pkg/beam - [10] https://docs.google.com/spreadsheets/d/1qk-N5vjXvbcEk68GjbkSZTR8AGqyNUM-oLFo_ZXBpJw/edit#gid=... - [11] https://hub.docker.com/search?q=apache%2Fbeam&type=image - [12] https://github.com/apache/beam/pull/... - -If there are any issues found in the release candidate, reply on the vote thread to cancel the vote. -There’s no need to wait 72 hours. -Proceed to the `Fix Issues` step below and address the problem. -However, some issues don’t require cancellation. -For example, if an issue is found in the website pull request, just correct it on the spot and the vote can continue as-is. - -### Run validation tests -The community is responsible for performing validation, but as release manager you are expected to contribute as well. -Before accepting an RC, as a community we try to exercise most (if not all) of the tests listed in this -[spreadsheet](https://s.apache.org/beam-release-validation), and those are good validations for you to try out as release manager. -The goal of these tests is to validate that we're able to run basic pipelines from a variety of environments (not just our CI environment). - -Since there are a bunch of tests, we recommend you running some validations using an automation script. -In case of script failure, you can still run all of them manually. - -You may need to have Python interpreters for all supported Python minor -versions to run all of the tests. See Python installation tips in [Developer Wiki](https://cwiki.apache.org/confluence/display/BEAM/Python+Tips#PythonTips-InstallingPythoninterpreters). - -#### Run validations using run_rc_validation.sh -* **Script:** [run_rc_validation.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/run_rc_validation.sh) - -* **Usage** - 1. First update required configurations listed in `RC_VALIDATE_CONFIGS` in - [script.config](https://github.com/apache/beam/blob/master/release/src/main/scripts/script.config) - 1. Then run - ``` - ./beam/release/src/main/scripts/run_rc_validation.sh - ``` - -**Note:** running the validations requires the ability to do the following in your GCP account: start pipelines, -write to BigQuery, and create a cluster of machines for running containers (for x-lang validation). - -* **Tasks included** - 1. Create a PR to trigger Python validation job, including - * Python quickstart in batch and streaming mode with direct runner and Dataflow runner. - * Python Mobile Games(UserScore, HourlyTeamScore) with direct runner and Dataflow runner. - 1. Run Python Streaming MobileGames, includes - * Start a new terminal to run Java Pubsub injector. - * Start a new terminal to run Python LeaderBoard with Direct Runner. - * Start a new terminal to run Python LeaderBoard with Dataflow Runner. - * Start a new terminal to run Python GameStats with Direct Runner. - * Start a new terminal to run Python GameStats with Dataflow Runner. - 1. Multi-language pipelines validation, includes - * Running the Python quickstart example using Python portable DirectRunner. This will start a new terminal for the Java expansion service. - * Running the Java quickstart example using Python portable DirectRunner. This will start new terminals for the Python expansion service and the job server. - * Start a new terminal to run Python multi-language Java kafka validation with Dataflow Runner. - * Start a new terminal to run Python multi-language Java sql validation with Dataflow Runner. - -* **Tasks you need to do manually** - 1. Check whether validations succeed by following console output instructions. - 1. Terminate streaming jobs and java injector. - 1. Run Java quickstart (wordcount) and mobile game examples with the staged artifacts. The easiest way to do this is by running the tests on Jenkins. -Other manual validation will follow, but this will at least validate that the staged artifacts can be used. - * Log in to Jenkins. - * Go to https://ci-beam.apache.org/job/beam_PostRelease_NightlySnapshot/. - * Click "Build with Parameters". - * Set `snapshot_version` to `2.xx.0`, and set `snapshot_url` to point to the staged artifacts in Maven central (https://repository.apache.org/content/repositories/orgapachebeam-NNNN/). - * Click "Build". - 1. Sign up [spreadsheet](https://s.apache.org/beam-release-validation). - 1. Vote in the release thread. - -#### Run validations manually - -_Note_: -Prepourl and -Pver can be found in the RC vote email sent by Release Manager. - -* **Java Quickstart Validation** - - **Direct Runner** - ``` - ./gradlew :runners:direct-java:runQuickstartJavaDirect \ - -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-${KEY} \ - -Pver=${RELEASE_VERSION} - ``` - **Flink Local Runner** - ``` - ./gradlew :runners:flink:1.13:runQuickstartJavaFlinkLocal \ - -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-${KEY} \ - -Pver=${RELEASE_VERSION} - ``` - **Spark Local Runner** - ``` - ./gradlew :runners:spark:3:runQuickstartJavaSpark \ - -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-${KEY} \ - -Pver=${RELEASE_VERSION} - ``` - **Dataflow Runner** - ``` - ./gradlew :runners:google-cloud-dataflow-java:runQuickstartJavaDataflow \ - -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-${KEY} \ - -Pver=${RELEASE_VERSION} \ - -PgcpProject=${YOUR_GCP_PROJECT} \ - -PgcsBucket=${YOUR_GCP_BUCKET} - ``` -* **Java Mobile Game(UserScore, HourlyTeamScore, Leaderboard)** - - **Prerequisites** - * **Create your own BigQuery dataset** - ``` - bq mk --project_id=${YOUR_GCP_PROJECT} ${YOUR_DATASET} - ``` - * **Create your PubSub topic** - ``` - gcloud alpha pubsub topics create --project=${YOUR_GCP_PROJECT} ${YOUR_PROJECT_PUBSUB_TOPIC} - ``` - * **Setup your service account** - - Goto IAM console in your project to create a service account as `project owner`, then run - - ``` - gcloud iam service-accounts keys create ${YOUR_KEY_JSON} --iam-account ${YOUR_SERVICE_ACCOUNT_NAME}@${YOUR_PROJECT_NAME} - export GOOGLE_APPLICATION_CREDENTIALS=${PATH_TO_YOUR_KEY_JSON} - ``` - **Run** - ``` - ./gradlew :runners:google-cloud-dataflow-java:runMobileGamingJavaDataflow \ - -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-${KEY} \ - -Pver=${RELEASE_VERSION} \ - -PgcpProject=${YOUR_GCP_PROJECT} \ - -PgcsBucket=${YOUR_GCP_BUCKET} \ - -PbqDataset=${YOUR_DATASET} -PpubsubTopic=${YOUR_PROJECT_PUBSUB_TOPIC} - ``` -* **Python Quickstart(batch & streaming), MobileGame(UserScore, HourlyTeamScore)** - - Create a new PR in apache/beam. - - In comment area, type in `Run Python ReleaseCandidate` to trigger validation. - -* **Python Leaderboard & GameStats** - * **Get staging RC** `wget https://dist.apache.org/repos/dist/dev/beam/2.5.0/* ` - * **Verify the hashes** - - ``` - sha512sum -c apache-beam-2.5.0-python.zip.sha512 - sha512sum -c apache-beam-2.5.0-source-release.zip.sha512 - ``` - * **Build SDK** - - ``` - sudo apt-get install unzip - unzip apache-beam-2.5.0-source-release.zip - python setup.py sdist - ``` - * **Setup virtual environment** - - ``` - python3 -m venv beam_env - . ./beam_env/bin/activate - pip install --upgrade pip setuptools wheel - ``` - * **Install SDK** - - ``` - pip install dist/apache-beam-2.5.0.tar.gz - pip install dist/apache-beam-2.5.0.tar.gz[gcp] - ``` - * **Setup GCP** - - Please repeat following steps for every following test. - - ``` - bq rm -rf --project=${YOUR_PROJECT} ${USER}_test - bq mk --project_id=${YOUR_PROJECT} ${USER}_test - gsutil rm -rf ${YOUR_GS_STORAGE] - gsutil mb -p ${YOUR_PROJECT} ${YOUR_GS_STORAGE} - gcloud alpha pubsub topics create --project=${YOUR_PROJECT} ${YOUR_PUBSUB_TOPIC} - ``` - Setup your service account as described in ```Java Mobile Game``` section above. - - * **Produce data by using java injector:** - - Configure your ~/.m2/settings.xml as following: - ``` - - - - release-repo - - true - - - - Release 2.4.0 RC3 - Release 2.4.0 RC3 - https://repository.apache.org/content/repositories/orgapachebeam-1031/ - - - - - - ``` - __Note__: You can found the latest ```id```, ```name``` and ```url``` for one RC in the vote email thread sent out by Release Manager. - - Run - ``` - mvn archetype:generate \ - -DarchetypeGroupId=org.apache.beam \ - -DarchetypeArtifactId=beam-sdks-java-maven-archetypes-examples \ - -DarchetypeVersion=${RELEASE_VERSION} \ - -DgroupId=org.example \ - -DartifactId=word-count-beam \ - -Dversion="0.1" \ - -Dpackage=org.apache.beam.examples \ - -DinteractiveMode=false - -DarchetypeCatalog=internal - - mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.complete.game.injector.Injector \ - -Dexec.args="${YOUR_PROJECT} ${YOUR_PUBSUB_TOPIC} none" - ``` - * **Run Leaderboard with Direct Runner** - ``` - python -m apache_beam.examples.complete.game.leader_board \ - --project=${YOUR_PROJECT} \ - --topic projects/${YOUR_PROJECT}/topics/${YOUR_PUBSUB_TOPIC} \ - --dataset ${USER}_test - ``` - Inspect results: - * Check whether there is any error messages in console. - * Goto your BigQuery console and check whether your ${USER}_test has leader_board_users and leader_board_teams table. - * bq head -n 10 ${USER}_test.leader_board_users - * bq head -n 10 ${USER}_test.leader_board_teams - - * **Run Leaderboard with Dataflow Runner** - ``` - python -m apache_beam.examples.complete.game.leader_board \ - --project=${YOUR_PROJECT} \ - --region=${GCE_REGION} \ - --topic projects/${YOUR_PROJECT}/topics/${YOUR_PUBSUB_TOPIC} \ - --dataset ${USER}_test \ - --runner DataflowRunner \ - --temp_location=${YOUR_GS_BUCKET}/temp/ \ - --sdk_location dist/* - ``` - Inspect results: - * Goto your Dataflow job console and check whether there is any error. - * Goto your BigQuery console and check whether your ${USER}_test has leader_board_users and leader_board_teams table. - * bq head -n 10 ${USER}_test.leader_board_users - * bq head -n 10 ${USER}_test.leader_board_teams - - * **Run GameStats with Direct Runner** - ``` - python -m apache_beam.examples.complete.game.game_stats \ - --project=${YOUR_PROJECT} \ - --topic projects/${YOUR_PROJECT}/topics/${YOUR_PUBSUB_TOPIC} \ - --dataset ${USER}_test \ - --fixed_window_duration ${SOME_SMALL_DURATION} - ``` - Inspect results: - * Check whether there is any error messages in console. - * Goto your BigQuery console and check whether your ${USER}_test has game_stats_teams and game_stats_sessions table. - * bq head -n 10 ${USER}_test.game_stats_teams - * bq head -n 10 ${USER}_test.game_stats_sessions - - * **Run GameStats with Dataflow Runner** - ``` - python -m apache_beam.examples.complete.game.game_stats \ - --project=${YOUR_PROJECT} \ - --region=${GCE_REGION} \ - --topic projects/${YOUR_PROJECT}/topics/${YOUR_PUBSUB_TOPIC} \ - --dataset ${USER}_test \ - --runner DataflowRunner \ - --temp_location=${YOUR_GS_BUCKET}/temp/ \ - --sdk_location dist/* \ - --fixed_window_duration ${SOME_SMALL_DURATION} - ``` - Inspect results: - * Goto your Dataflow job console and check whether there is any error. - * Goto your BigQuery console and check whether your ${USER}_test has game_stats_teams and game_stats_sessions table. - * bq head -n 10 ${USER}_test.game_stats_teams - * bq head -n 10 ${USER}_test.game_stats_sessions - - -### Fix any issues - -Any issues identified during the community review and vote should be fixed in this step. -Additionally, any GitHub issues created from the initial branch verification should be fixed. - -Code changes should be proposed as standard pull requests to the `master` branch and reviewed using the normal contributing process. -Then, relevant changes should be cherry-picked into the release branch proposed as pull requests against the release branch, again reviewed and merged using the normal contributing process. - -Once all issues have been resolved, you should go back and build a new release candidate with these changes. - -### Finalize the vote - -Reply on the vote thread to close the voting once following conditions are met for the current release candidate. -* At least 72 hours has passed since the voting email. -* No release blocking issues have been identified. -* Voting thread has at least three approving PMC votes. - -Then, tally the votes in a separate email thread. -Here’s an email template; please adjust as you see fit. - - From: Release Manager - To: dev@beam.apache.org - Subject: [RESULT] [VOTE] Release 1.2.3, release candidate #3 - - I'm happy to announce that we have unanimously approved this release. - - There are XXX approving votes, XXX of which are binding: - * approver 1 - * approver 2 - * approver 3 - * approver 4 - - There are no disapproving votes. - - Thanks everyone! - -### Checklist to proceed to the next step - -1. Issues identified during vote have been resolved, with fixes committed to the release branch. -2. All issues in the current release's milestone should be closed. -3. Community votes to release the proposed candidate, with at least three approving PMC votes. - - -********** - - -## 11. Finalize the release - -Once the release candidate has been reviewed and approved by the community, the release should be finalized. -This involves the final deployment of the release candidate to the release repositories, merging of the website changes, etc. - -### Deploy artifacts to Maven Central Repository - -Use the [Apache Nexus repository manager](https://repository.apache.org/#stagingRepositories) to release the staged binary artifacts to the Maven Central repository. -In the `Staging Repositories` section, find the relevant release candidate `orgapachebeam-XXX` entry and click `Release`. -Drop all other release candidates that are not being released. - -__NOTE__: If you are using [GitHub two-factor authentication](https://help.github.com/articles/securing-your-account-with-two-factor-authentication-2fa/) and haven't configure HTTPS access, -please follow [the guide](https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/) to configure command line access. - -### Deploy Python artifacts to PyPI - -* **Script:** [deploy_pypi.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/deploy_pypi.sh) -* **Usage** -``` -./beam/release/src/main/scripts/deploy_pypi.sh -``` -* Verify that the files at https://pypi.org/project/apache-beam/#files are correct. -All wheels should be published, in addition to the zip of the release source. -(Signatures and hashes do _not_ need to be uploaded.) - -### Deploy docker images to DockerHub - -Note: if you are not a member of the [beammaintainers DockerHub team](https://hub.docker.com/orgs/apache/teams/beammaintainers), -you will need help with this step. Please email dev@ and ask a member of the beammaintainers DockerHub team for help. - -* **Script:** [publish_docker_images.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/publish_docker_images.sh) -* **Usage** -``` -./beam/release/src/main/scripts/publish_docker_images.sh -``` -* **Verify that:** - * Images are published at [DockerHub](https://hub.docker.com/search?q=apache%2Fbeam&type=image) with tags {RELEASE_VERSION} and *latest*. - * Images with *latest* tag are pointing to current release by confirming the digest of the image with *latest* tag is the same as the one with {RELEASE_VERSION} tag. - -(Optional) Clean up any unneeded local images afterward to save disk space. - -### Merge Website pull requests - -Merge all of the website pull requests -- [listing the release](/get-started/downloads/) -- publishing the [Python API reference manual](https://beam.apache.org/releases/pydoc/) and the [Java API reference manual](https://beam.apache.org/releases/javadoc/), and -- adding the release blog post. - -### Git tag - -Create and push a new signed tag for the released version by copying the tag for the final release candidate, as follows: - -``` -# Optional: unlock the signing key by signing an arbitrary file. -gpg --output ~/doc.sig --sign ~/.bashrc - -VERSION_TAG="v${RELEASE_VERSION}" -RC_TAG="${VERSION_TAG}-RC${RC_NUM}" - -# Ensure local tags are in sync. If there's a mismatch, it will tell you. -git fetch --all --tags - -# If the tag exists, a commit number is produced, otherwise there's an error. -git rev-list $RC_TAG -n 1 - -# Tag for Go SDK -git tag -s "sdks/$VERSION_TAG" "$RC_TAG" -git push https://github.com/apache/beam "sdks/$VERSION_TAG" - -# Tag for repo root. -git tag -s "$VERSION_TAG" "$RC_TAG" -git push https://github.com/apache/beam "$VERSION_TAG" -``` - -After pushing the tag, the tag should be visible on Github's [Tags](https://github.com/apache/beam/tags) page. - -### Publish release to Github - -Once the tag is uploaded, publish the release notes to Github. From the [Beam release page on Github](https://github.com/apache/beam/releases) select -"Draft a new release." Title the release "Beam ${RELEASE_VERSION} release" and set the release at the version tag created above. Use the content of the -release blog post as the body of the release notes, set this version as the latest release, and publish it. - -The release notes should now be visible on Github's [Releases](https://github.com/apache/beam/releases) page. - -### Mark the version as released in GitHub - -In GitHub, in the [milestone page](https://github.com/apache/beam/milestones), click close on the current release. - -### PMC-Only Finalization -There are a few release finalization tasks that only PMC members have permissions to do. -Ping [dev@](mailto:dev@beam.apache.org) for assistance if you need it. - -#### Deploy source release to dist.apache.org - -Copy the source release from the `dev` repository to the `release` repository at `dist.apache.org` using Subversion. - -Make sure the last release's artifacts have been copied from `dist.apache.org` to `archive.apache.org`. -This should happen automatically: [dev@ thread](https://lists.apache.org/thread.html/39c26c57c5125a7ca06c3c9315b4917b86cd0e4567b7174f4bc4d63b%40%3Cdev.beam.apache.org%3E) with context. - -#### Recordkeeping with ASF - -Use [reporter.apache.org](https://reporter.apache.org/addrelease.html?beam) to seed the information about the release into future project reports. - -### Checklist to proceed to the next step - -* Maven artifacts released and indexed in the [Maven Central Repository](https://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.beam%22) -* Source distribution available in the release repository of [dist.apache.org](https://dist.apache.org/repos/dist/release/beam/) -* Source distribution removed from the dev repository of [dist.apache.org](https://dist.apache.org/repos/dist/dev/beam/) -* Website pull request to [list the release](/get-started/downloads/) and publish the [API reference manual](https://beam.apache.org/releases/javadoc/) merged. -* The release is tagged on Github's [Tags](https://github.com/apache/beam/tags) page. -* The release notes are published on Github's [Releases](https://github.com/apache/beam/releases) page. -* Release version finalized in GitHub. -* Release version is listed at reporter.apache.org - - -********** - - -## 12. Promote the release - -Once the release has been finalized, the last step of the process is to promote the release within the project and beyond. - -### Apache mailing lists - -Announce on the dev@ mailing list that the release has been finished. - -Announce on the release on the user@ mailing list, listing major improvements and contributions. - -Announce the release on the announce@apache.org mailing list. -__NOTE__: This can only be done from `@apache.org` email address. This email has to be in plain text (no HTML tags). - -### Social media - -Tweet, post on Facebook, LinkedIn, and other platforms. -Ask other contributors to do the same. - -Also, update [the Wikipedia article on Apache Beam](https://en.wikipedia.org/wiki/Apache_Beam). - -### Checklist to declare the process completed - -1. Release announced on the user@ mailing list. -1. Blog post published, if applicable. -1. Release recorded in reporter.apache.org. -1. Release announced on social media. -1. Completion declared on the dev@ mailing list. -1. Update Wikipedia Apache Beam article. - -********** - -## 13. Update Beam Playground - -After new Beam Release is published, Beam Playgorund can be updated following the steps below: - -1. Open the [Cloud Build triggers in apache-beam-testing](https://console.cloud.google.com/cloud-build/triggers?project=apache-beam-testing) GCP project. -1. Find the trigger "Deploy-Update-Playground-environment-stg": - 1. Click on the trigger name to open its settings - 1. Change the value for _SDK_TAG variable (Advanced -> Substitution Variables) to the actual version of Beam SDK (e.g. 2.47.0) - 1. Click the Save button. The settings window should close without any errors - 1. Click the RUN button next to the trigger name - 1. Set the value for the _CONTAINER_TAG variable in format DD-MM-vXX (DD - day, MM - month, XX - version, e.g., 20-12-v01) - 1. Click the Run Trigger button - 1. Open the [Trigger History](https://console.cloud.google.com/cloud-build/builds?project=apache-beam-testing) and wait for the job completion. Ensure that the job completed successfully (Status field shows a green tick) -1. Find the trigger "Playground-CD-stable-manual-stg": - 1. Click the RUN button next to the trigger name - 1. Click the Run Trigger button (with default varaible vaues) - 1. Open the [Trigger History](https://console.cloud.google.com/cloud-build/builds?project=apache-beam-testing) and wait for the job completion. Ensure that the job completed successfully (Status field shows a green tick) - 1. Click the RUN button next to the trigger name - 1. Change values for the variables: - * _ORIGIN = PG_BEAMDOC - * _SUBDIRS = ./learning/beamdoc - 1. Click the Run Trigger button - 1. Open the [Trigger History](https://console.cloud.google.com/cloud-build/builds?project=apache-beam-testing) and wait for the job completion. Ensure that the job completed successfully (Status field shows a green tick) -1. Test updated [staging Playground](https://play-dev.beam.apache.org/) in a browser - 1. Open the menu (represented by '...' in the right top corner) and click on Versions. Validate that commit is the same for all listed containers, and the hash belongs to a [recent master branch commit](https://github.com/apache/beam/commits/master) - 1. For each of the supported SDKs (Java, Python, Go, SCIO): - * Switch to the SDK - * Make any changes to the loaded default example - * Click the Run button - * Wait for successful completion - * Click "Share My Code" to ensure that the link is generated -1. Repeat the same steps for "Deploy-Update-Playground-environment-prod" trigger as for "Deploy-Update-Playground-environment-stg" trigger -1. Repeat the same steps for "Playground-CD-stable-manual-prod" trigger as for "Playground-CD-stable-manual-stg" trigger -1. Test updated [prod Playground](https://play.beam.apache.org/) in a browser. The process is similar to the staging environment. -1. Find the trigger "Playground-CI-stable" - 1. Click on the trigger name to open its settings - 1. Set the value for the _BEAM_VERSION variable (Advanced -> Substitution Variables) to the actual version of Beam SDK (e.g., 2.47.0) - 1. Click the Save button. Click the Save button. The settings window should close without any errors - -## Improve the process - -It is important that we improve the release processes over time. -Once you’ve finished the release, please take a step back and look what areas of this process and be improved. Perhaps some part of the process can be simplified. -Perhaps parts of this guide can be clarified. - -If we have specific ideas, please start a discussion on the dev@ mailing list and/or propose a pull request to update this guide. -Thanks! diff --git a/website/www/site/content/en/contribute/runner-guide.md b/website/www/site/content/en/contribute/runner-guide.md index d64ebd2164cc3..cb3b52048606d 100644 --- a/website/www/site/content/en/contribute/runner-guide.md +++ b/website/www/site/content/en/contribute/runner-guide.md @@ -41,8 +41,7 @@ element-wise, grouping, windowing, union) rather than a specific implementation decision. The same primitive may require a very different implementation based on how the user instantiates it. For example, a `ParDo` that uses state or timers may require key partitioning, a `GroupByKey` with speculative triggering -may require a more costly or complex implementation, and `Read` is completely -different for bounded and unbounded data. +may require a more costly or complex implementation. ### What if you haven't implemented some of these features? @@ -57,6 +56,19 @@ native environment, this may look like throwing an `UnsupportedOperationException`. The Runner API RPCs will make this explicit, for cross-language portability. +### Implementing the Impulse primitive + +`Impulse` is a PTransform that takes no inputs and produces exactly one output +during the lifetime of the pipeline which should be the empty bytes in the +global window with the minimum timestamp. This has the encoded value of +`7f df 3b 64 5a 1c ac 09 00 00 00 01 0f 00` when encoded with the standard +windowed value coder. + +Though `Impulse` is generally not invoked by a user, it is the only root +primitive operation, and other root operations (like `Read`s and `Create`) +are composite operations constructed from an `Impulse` followed by a series +of (possibly Splittable) `ParDo`s. + ### Implementing the ParDo primitive The `ParDo` primitive describes element-wise transformation for a @@ -72,11 +84,27 @@ can discuss it with pseudocode. I will also often refer to the Java support code, since I know it and most of our current and future runners are Java-based. +Generally, rather than applying a series of `ParDo`s one at a time over the +entire input data set, it is more efficient to fuse several `ParDo`s together +in a single executable stage that consists of a whole series (in general, +a DAG) of mapping operations. In addition to `ParDo`s, windowing operations, +local (pre- or post-GBK) combining operations, and other mapping operations +may be fused into these stages as well. + +As DoFns may execute code in a different language, or requiring a different +environment, than the runner itself, Beam provides the ability to call these +in a cross-process way. This is the crux of the +[Beam Fn API](https://beam.apache.org/contribute/runner-guide/#writing-an-sdk-independent-runner), +for which more detail can be found below. +It is, however, perfectly acceptable for a runner to invoke this user code +in process (for simplicity or efficiency) when the environments are +compatible. + #### Bundles For correctness, a `DoFn` _should_ represent an element-wise function, but in -fact is a long-lived object that processes elements in small groups called -bundles. +most SDKS this is a long-lived object that processes elements in small groups +called bundles. Your runner decides how many elements, and which elements, to include in a bundle, and can even decide dynamically in the middle of processing that the @@ -89,66 +117,25 @@ But if your data is arriving as a stream, then you will want to terminate a bundle in order to achieve appropriate latency, so bundles may be just a few elements. -#### The DoFn Lifecycle - -While each language's SDK is free to make different decisions, the Python and -Java SDKs share an API with the following stages of a DoFn's lifecycle. - -However, if you choose to execute a DoFn directly to improve performance or -single-language simplicity, then your runner is responsible for implementing -the following sequence: - - * _Setup_ - called once per DoFn instance before anything else; this has not been - implemented in the Python SDK so the user can work around just with lazy - initialization - * _StartBundle_ - called once per bundle as initialization (actually, lazy - initialization is almost always equivalent and more efficient, but this hook - remains for simplicity for users) - * _ProcessElement_ / _OnTimer_ - called for each element and timer activation - * _FinishBundle_ - essentially "flush"; required to be called before - considering elements as actually processed - * _Teardown_ - release resources that were used across bundles; calling this - can be best effort due to failures - -#### DoFnRunner(s) - -This is a support class that has manifestations in both the Java codebase and -the Python codebase. - -**Java** - -In Java, the `beam-runners-core-java` library provides an interface -`DoFnRunner` for bundle processing, with implementations for many situations. +A bundle is the unit of commitment in Beam. If an error is encountered while +processing a bundle, all the prior outputs of that bundle (including any +modifications to state or timers) must be discarded by the runner and the +entire bundle retried. Upon successful completion of a bundle, its outputs, +together with any state/timer modifications and watermark updates, must be +committed atomically. -{{< highlight class="language-java no-toggle" >}} -interface DoFnRunner { - void startBundle(); - void processElement(WindowedValue elem); - void onTimer(String timerId, BoundedWindow window, Instant timestamp, TimeDomain timeDomain); - void finishBundle(); -} -{{< /highlight >}} - -There are some implementations and variations of this for different scenarios: - - * [`SimpleDoFnRunner`](https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java) - - not actually simple at all; implements lots of the core functionality of - `ParDo`. This is how most runners execute most `DoFns`. - * [`LateDataDroppingDoFnRunner`](https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java) - - wraps a `DoFnRunner` and drops data from expired windows so the wrapped - `DoFnRunner` doesn't get any unpleasant surprises - * [`StatefulDoFnRunner`](https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java) - - handles collecting expired state - * [`PushBackSideInputDoFnRunner`](https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunner.java) - - buffers input while waiting for side inputs to be ready - -These are all used heavily in implementations of Java runners. Invocations -via the [Fn API](#the-fn-api) may manifest as another implementation of -`DoFnRunner` even though it will be doing far more than running a `DoFn`. - -**Python** +#### The DoFn Lifecycle -See the [DoFnRunner pydoc](https://beam.apache.org/releases/pydoc/2.0.0/apache_beam.runners.html#apache_beam.runners.common.DoFnRunner). +`DoFns` in many SDKS have several methods such as `setup`, `start_bundle`, +`finish_bundle`, `teardown`, etc. in addition to the standard, +element-wise `process` calls. Generally proper invocation of +[this lifecycle](https://beam.apache.org/documentation/programming-guide/#dofn) +should be handled for you when invoking one or more +`DoFn`s from the standard bundle processors (either via the FnAPI or directly +using a BundleProcessor +([java](https://github.com/apache/beam/blob/master/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/control/ProcessBundleHandler.java) +([python](https://github.com/apache/beam/blob/release-2.49.0/sdks/python/apache_beam/runners/worker/bundle_processor.py#L852))). +SDK-independent runners should never have to worry about these details directly. #### Side Inputs @@ -160,78 +147,86 @@ it from the main input, which is processed one element at a time. The SDK/user prepares a `PCollection` adequately, the runner materializes it, and then the runner feeds it to the `DoFn`. -What you will need to implement is to inspect the materialization requested for -the side input, and prepare it appropriately, and corresponding interactions -when a `DoFn` reads the side inputs. - -The details and available support code vary by language. - -**Java** - -If you are using one of the above `DoFnRunner` classes, then the interface for -letting them request side inputs is -[`SideInputReader`](https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputReader.java). -It is a simple mapping from side input and window to a value. The `DoFnRunner` -will perform a mapping with the -[`WindowMappingFn`](https://github.com/apache/beam/blob/master/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowMappingFn.java) -to request the appropriate window so you do not worry about invoking this UDF. -When using the Fn API, it will be the SDK harness that maps windows as well. - -A simple, but not necessarily optimal approach to building a -[`SideInputReader`](https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputReader.java) -is to use a state backend. In our Java support code, this is called -[`StateInternals`](https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/StateInternals.java) -and you can build a -[`SideInputHandler`](https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/SideInputHandler.java) -that will use your `StateInternals` to materialize a `PCollection` into the -appropriate side input view and then yield the value when requested for a -particular side input and window. +Unlike main input data, which is *pushed* by the runner to the `ParDo` (generally +via the FnApi Data channel), side input data is *pulled* by the `ParDo` +from the runner (generally over the FnAPI State channel). + +A side input is accessed via a specific `access_pattern`. +There are currently two access patterns enumerated in the +`StandardSideInputTypes` proto: `beam:side_input:iterable:v1` which indicates +the runner must return all values in a PCollection corresponding to a specific +window and `beam:side_input:multimap:v1` which indicates the runner must return +all values corresponding to a specific key and window. +Being able to serve these access patterns efficiently may influence how a +runner materializes this PCollection. + +SideInputs can be detected by looking at the `side_inputs` map in the +`ParDoPayload` of `ParDo` transforms. +The `ParDo` operation itself is responsible for invoking the +`window_mapping_fn` (before invoking the runner) and `view_fn` (on the +runner-returned values), so the runner need not concern itself with these +fields. When a side input is needed but the side input has no data associated with it for a given window, elements in that window must be deferred until the side -input has some data. The aforementioned +input has some data or the watermark has advances sufficiently such that +we can be sure there will be no data for that window. The [`PushBackSideInputDoFnRunner`](https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunner.java) -is used to implement this. - -**Python** - -In Python, [`SideInputMap`](https://beam.apache.org/releases/pydoc/2.0.0/apache_beam.transforms.html#apache_beam.transforms.sideinputs.SideInputMap) maps -windows to side input values. The `WindowMappingFn` manifests as a simple -function. See -[sideinputs.py](https://github.com/apache/beam/blob/master/sdks/python/apache_beam/transforms/sideinputs.py). +is an example of implementing this. #### State and Timers _Main design document: [https://s.apache.org/beam-state](https://s.apache.org/beam-state)_ When a `ParDo` includes state and timers, its execution on your runner is usually -very different. See the full details beyond those covered here. +very different. In particular, the state must be persisted when the bundle +completes and retrieved for future bundles. Timers that are set must also be +injected into future bundles as the watermark advances sufficiently. -State and timers are partitioned per key and window. You may need or want to +State and timers are partitioned per key and window, that is, a `DoFn` +processing a given key must have a consistent view of the state and timers +across all elements that share this key. You may need or want to explicitly shuffle data to support this. +Once the watermark has passed the end of the window (plus an allowance for +allowed lateness, if any), state associated with this window can be dropped. -**Java** - -We provide -[`StatefulDoFnRunner`](https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java) -to help with state cleanup. The non-user-facing interface -[`StateInternals`](https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/StateInternals.java) -is what a runner generally implements, and then the Beam support code can use -this to implement user-facing state. +State setting and retrieval is performed on the FnAPI State channel, whereas +timer setting and firing happens on the FnAPI Data channel. #### Splittable DoFn _Main design document: [https://s.apache.org/splittable-do-fn](https://s.apache.org/splittable-do-fn)_ -Splittable `DoFn` is a generalization and combination of `ParDo` and `Read`. It -is per-element processing where each element has the capability of being "split" -in the same ways as a `BoundedSource` or `UnboundedSource`. This enables better -performance for use cases such as a `PCollection` of names of large files where -you want to read each of them. Previously they would have to be static data in -the pipeline or be read in a non-splittable manner. - -This feature is still under development, but likely to become the new primitive -for reading. It is best to be aware of it and follow developments. +Splittable `DoFn` is a generalization of `ParDo` that is useful for high-fanout +mappings that can be done in parallel. The prototypical example of such an +operation is reading from a file, where a single file name (as an input element) +can be mapped to all the elements contained in that file. +The `DoFn` is considered splittable in the sense that an element representing, +say, a single file can be split (e.g. into ranges of that file) to be processed +(e.g. read) by different workers. +The full power of this primitive is in the fact that these splits can happen +dynamically rather than just statically (i.e. ahead of time) avoiding the +problem of over- or undersplitting. + +A full explanation of Splittable `DoFn` is out of scope for this doc, but +here is a brief overview as it pertains to its execution. + +A Splittable `DoFn` can participate in the dynamic splitting protocol by +splitting within an element as well as between elements. Dynamic splitting +is triggered by the runner issuing `ProcessBundleSplitRequest` messages on +the control channel. The SDK will commit to process just a portion of the +indicated element and return a description of the remainder (i.e. the +unprocessed portion) to the runner in the `ProcessBundleSplitResponse` +to be scheduled by the runner (e.g. on a different worker or as part of a +different bundle). + +A Splittable `DoFn` can also initiate its own spitting, indicating it has +processed an element as far as it can for the moment (e.g. when tailing a file) +but more remains. These most often occur when reading unbounded sources. +In this case a set of elements representing the deferred work are passed back +in the `residual_roots` field of the `ProcessBundleResponse`. +At a future time, the runner must re-invoke these same operations with +the elements given in `residual_roots`. ### Implementing the GroupByKey (and window) primitive @@ -249,11 +244,12 @@ to group in a way that is consistent with grouping by those bytes, even if you have some special knowledge of the types involved. The elements you are processing will be key-value pairs, and you'll need to extract -the keys. For this reason, the format of key-value pairs is standardized and -shared across all SDKS. See either -[`KvCoder`](https://beam.apache.org/releases/javadoc/2.0.0/org/apache/beam/sdk/coders/KvCoder.html) +the keys. For this reason, the format of key-value pairs is +[standardized and shared](https://github.com/apache/beam/blob/release-2.49.0/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/beam_runner_api.proto#L838) +across all SDKS. See either +[`KvCoder`](https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/coders/KvCoder.html) in Java or -[`TupleCoder`](https://beam.apache.org/releases/pydoc/2.0.0/apache_beam.coders.html#apache_beam.coders.coders.TupleCoder.key_coder) +[`TupleCoder`](https://beam.apache.org/releases/pydoc/current/apache_beam.coders.coders.html#apache_beam.coders.coders.TupleCoder) in Python for documentation on the binary format. #### Window Merging @@ -266,11 +262,14 @@ grouping. #### Implementing via GroupByKeyOnly + GroupAlsoByWindow -The Java codebase includes support code for a particularly common way of +The Java and Python codebases includes support code for a particularly common way of implementing the full `GroupByKey` operation: first group the keys, and then group by window. For merging windows, this is essentially required, since merging is per key. +Often presenting the set of values in timestamp order can allow more +efficient grouping of these values into their final windows. + #### Dropping late data _Main design document: @@ -298,6 +297,8 @@ In Java, there is a lot of support code for executing triggers in the `GroupAlsoByWindow` implementations, `ReduceFnRunner` (legacy name), and `TriggerStateMachine`, which is an obvious way of implementing all triggers as an event-driven machine over elements and timers. +In Python this is supported by the +[TriggerDriver](https://github.com/apache/beam/blob/release-2.49.0/sdks/python/apache_beam/transforms/trigger.py#L1199) classes. #### TimestampCombiner @@ -321,6 +322,9 @@ To implement this primitive, you need to invoke the provided WindowFn on each element, which will return some set of windows for that element to be a part of in the output `PCollection`. +Most runners implement this by fusing these window-altering mappings in with +the `DoFns`. + **Implementation considerations** A "window" is just a second grouping key that has a "maximum timestamp". It can @@ -336,53 +340,13 @@ multiple windows". For values in the global window, you may want to use an even further compressed representation that doesn't bother including the window at all. +We provide coders with these optimizations such as +(`PARAM_WINDOWED_VALUE`)[https://github.com/apache/beam/blob/release-2.49.0/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/beam_runner_api.proto#L968] +that can be used to reduce the size of serialized data. + In the future, this primitive may be retired as it can be implemented as a ParDo if the capabilities of ParDo are enhanced to allow output to new windows. -### Implementing the Read primitive - -You implement this primitive to read data from an external system. The APIs are -carefully crafted to enable efficient parallel execution. Reading from an -`UnboundedSource` is a bit different than reading from a `BoundedSource`. - -#### Reading from an UnboundedSource - -An `UnboundedSource` is a source of potentially infinite data; you can think of -it like a stream. The capabilities are: - - * `split(int)` - your runner should call this to get the desired parallelism - * `createReader(...)` - call this to start reading elements; it is an enhanced iterator that also provides: - * watermark (for this source) which you should propagate downstream - * timestamps, which you should associate with elements read - * record identifiers, so you can dedup downstream if needed - * progress indication of its backlog - * checkpointing - * `requiresDeduping` - this indicates that there is some chance that the source - may emit duplicates; your runner should do its best to dedupe based on the - identifier attached to emitted records - -An unbounded source has a custom type of checkpoints and an associated coder for serializing them. - -#### Reading from a BoundedSource - -A `BoundedSource` is a source of data that you know is finite, such as a static -collection of log files, or a database table. The capabilities are: - - * `split(int)` - your runner should call this to get desired initial parallelism (but you can often steal work later) - * `getEstimatedSizeBytes(...)` - self explanatory - * `createReader(...)` - call this to start reading elements; it is an enhanced iterator that also provides: - * timestamps to associate with each element read - * `splitAtFraction` for dynamic splitting to enable work stealing, and other - methods to support it - see the [Beam blog post on dynamic work - rebalancing](/blog/2016/05/18/splitAtFraction-method.html) - -The `BoundedSource` does not report a watermark currently. Most of the time, reading -from a bounded source can be parallelized in ways that result in utterly out-of-order -data, so a watermark is not terribly useful. -Thus the watermark for the output `PCollection` from a bounded read should -remain at the minimum timestamp throughout reading (otherwise data might get -dropped) and advance to the maximum timestamp when all data is exhausted. - ### Implementing the Flatten primitive This one is easy - take as input a finite set of `PCollections` and outputs their @@ -399,59 +363,30 @@ fast path as an optimization. ### Special mention: the Combine composite A composite transform that is almost always treated specially by a runner is -`Combine` (per key), which applies an associative and commutative operator to +`CombinePerKey`, which applies an associative and commutative operator to the elements of a `PCollection`. This composite is not a primitive. It is implemented in terms of `ParDo` and `GroupByKey`, so your runner will work without treating it - but it does carry additional information that you probably want to use for optimizations: the associative-commutative operator, known as a `CombineFn`. +Generally runners will want to implement this via what is called +combiner lifting, where a new operation is placed before the `GroupByKey` +that does partial (within-bundle) combining, which often requires a slight +modification of what comes after the `GroupByKey` as well. +An example of this transformation can be found in the +(Python)[https://github.com/apache/beam/blob/release-2.49.0/sdks/python/apache_beam/runners/portability/fn_api_runner/translations.py#L1193] +or (go)[https://github.com/apache/beam/blob/release-2.49.0/sdks/go/pkg/beam/runners/prism/internal/handlecombine.go#L67] +implementations of this optimization. +The resulting pre- and post-`GroupByKey` operations are generally fused in with +the `ParDo`s and executed as above. + ## Working with pipelines -When you receive a pipeline from a user, you will need to translate it. This is -a tour of the APIs that you'll use to do it. - -### Traversing a pipeline - -Something you will likely do is to traverse a pipeline, probably to translate -it into primitives for your engine. The general pattern is to write a visitor -that builds a job specification as it walks the graph of `PTransforms`. - -The entry point for this in Java is -[`Pipeline.traverseTopologically`](https://beam.apache.org/releases/javadoc/2.0.0/org/apache/beam/sdk/Pipeline.html#traverseTopologically-org.apache.beam.sdk.Pipeline.PipelineVisitor-) -and -[`Pipeline.visit`](https://beam.apache.org/releases/pydoc/2.0.0/apache_beam.html#apache_beam.pipeline.Pipeline.visit) -in Python. See the generated documentation for details. - -### Altering a pipeline - -Often, the best way to keep your -translator simple will be to alter the pipeline prior to translation. Some -alterations you might perform: - - * Elaboration of a Beam primitive into a composite transform that uses - multiple runner-specific primitives - * Optimization of a Beam composite into a specialized primitive for your - runner - * Replacement of a Beam composite with a different expansion more suitable for - your runner - -The Java SDK and the "runners core construction" library (the artifact is -`beam-runners-core-construction-java` and the namespaces is -`org.apache.beam.runners.core.construction`) contain helper code for this sort -of work. In Python, support code is still under development. - -All pipeline alteration is done via -[`Pipeline.replaceAll(PTransformOverride)`](https://beam.apache.org/releases/javadoc/2.0.0/org/apache/beam/sdk/Pipeline.html#replaceAll-java.util.List-) -method. A -[`PTransformOverride`](https://github.com/apache/beam/blob/master/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverride.java) -is a pair of a -[`PTransformMatcher`](https://github.com/apache/beam/blob/master/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformMatcher.java) -to select transforms for replacement and a -[`PTransformOverrideFactory`](https://github.com/apache/beam/blob/master/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/PTransformOverrideFactory.java) -to produce the replacement. All `PTransformMatchers` that have been needed by -runners to date are provided. Examples include: matching a specific class, -matching a `ParDo` where the `DoFn` uses state or timers, etc. +When you receive a pipeline from a user, you will need to translate it. +An explanation of how Beam pipelines are represented can be found +(here)[https://docs.google.com/presentation/d/1atu-QC_mnK2SaeLhc0D78wZYgVOX1fN0H544QmBi3VA] +which compliment the (official proto declarations)[https://github.com/apache/beam/blob/master/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/beam_runner_api.proto]. ## Testing your runner @@ -555,6 +490,9 @@ All runner code should go in it's own package in `apache_beam/runners` directory Register the new runner in the `create_runner` function of `runner.py` so that the partial name is matched with the correct class to be used. +Python Runners can also be identified (e.g. when passing the runner parameter) +by their fully qualified name whether or not they live in the Beam repository. + ## Writing an SDK-independent runner There are two aspects to making your runner SDK-independent, able to run @@ -568,6 +506,9 @@ _Design documents:_ - _[https://s.apache.org/beam-fn-api-processing-a-bundle](https://s.apache.org/beam-fn-api-processing-a-bundle)_ - _[https://s.apache.org/beam-fn-api-send-and-receive-data](https://s.apache.org/beam-fn-api-send-and-receive-data)_ + - _[Overview](https://docs.google.com/presentation/d/1Cso0XP9dmj77OD9Bd53C1M3W1sPJF0ZnA20gzb2BPhE/edit#slide=id.g42e4c9aad6_0_317)_ + - _[Spec](https://github.com/apache/beam/blob/master/model/fn-execution/src/main/proto/org/apache/beam/model/fn_execution/v1/beam_fn_api.proto)_ + To run a user's pipeline, you need to be able to invoke their UDFs. The Fn API is an RPC interface for the standard UDFs of Beam, implemented using protocol buffers over gRPC. @@ -584,10 +525,10 @@ UDFs. ### The Runner API -The Runner API is an SDK-independent schema for a pipeline along with RPC -interfaces for launching a pipeline and checking the status of a job. The RPC -interfaces are still in development so for now we focus on the SDK-agnostic -representation of a pipeline. By examining a pipeline only through Runner API +The [Runner API](https://docs.google.com/presentation/d/1Cso0XP9dmj77OD9Bd53C1M3W1sPJF0ZnA20gzb2BPhE/edit#slide=id.g42e4c9aad6_1_3736) +is an SDK-independent schema for a pipeline along with RPC +interfaces for launching a pipeline and checking the status of a job. +By examining a pipeline only through Runner API interfaces, you remove your runner's dependence on the SDK for its language for pipeline analysis and job translation. @@ -602,7 +543,7 @@ You are fully welcome to _also_ use the SDK for your language, which may offer useful utility code. The language-independent definition of a pipeline is described via a protocol -buffers schema, covered below for reference. But your runner _should not_ +buffers schema, covered below for reference. But your runner _need not_ directly manipulate protobuf messages. Instead, the Beam codebase provides utilities for working with pipelines so that you don't need to be aware of whether or not the pipeline has ever been serialized or transmitted, or what @@ -660,7 +601,7 @@ sense that includes side effects, etc. {{< highlight class="no-toggle" >}} message FunctionSpec { string urn; - google.protobuf.Any parameter; + bytes payload; } {{< /highlight >}} @@ -681,27 +622,14 @@ used in a `PTransform` it describes a function from `PCollection` to `PCollectio and cannot be specific to an SDK because the runner is in charge of evaluating transforms and producing `PCollections`. -### `SdkFunctionSpec` proto - -When a `FunctionSpec` represents a UDF, in general only the SDK that serialized -it will be guaranteed to understand it. So in that case, it will always come -with an environment that can understand and execute the function. This is -represented by the `SdkFunctionSpec`. - -{{< highlight class="no-toggle" >}} -message SdkFunctionSpec { - FunctionSpec spec; - bytes environment_id; -} -{{< /highlight >}} - -In the Runner API, many objects are stored by reference. Here in the -`environment_id` is a pointer, local to the pipeline and just made up by the -SDK that serialized it, that can be dereferenced to yield the actual -environment proto. - -Thus far, an environment is expected to be a Docker container specification for -an SDK harness that can execute the specified UDF. +It goes without saying that not every environment will be able to deserialize +every function spec. For this reason `PTransform`s have an `environment_id` +parameter that indicates at least one environment that is capable of interpreting +the contained URNs. This is a reference to an environment in the environments +map of the Pipeline proto and is typically defined by a docker image (possibly +with some extra dependencies). +There may be other environments that are also capable of +doing so, and a runner is free to use them if it has this knowledge. ### Primitive transform payload protos @@ -721,7 +649,7 @@ inputs, state declarations, timer declarations, etc. {{< highlight class="no-toggle" >}} message ParDoPayload { - SdkFunctionSpec do_fn; + FunctionSpec do_fn; map side_inputs; map state_specs; map timer_specs; @@ -729,29 +657,6 @@ message ParDoPayload { } {{< /highlight >}} -#### `ReadPayload` proto - -A `Read` transform carries an `SdkFunctionSpec` for its `Source` UDF. - -{{< highlight class="no-toggle" >}} -message ReadPayload { - SdkFunctionSpec source; - ... -} -{{< /highlight >}} - -#### `WindowIntoPayload` proto - -A `Window` transform carries an `SdkFunctionSpec` for its `WindowFn` UDF. It is -part of the Fn API that the runner passes this UDF along and tells the SDK -harness to use it to assign windows (as opposed to merging). - -{{< highlight class="no-toggle" >}} -message WindowIntoPayload { - SdkFunctionSpec window_fn; - ... -} -{{< /highlight >}} #### `CombinePayload` proto @@ -764,7 +669,7 @@ a reference to this coder. {{< highlight class="no-toggle" >}} message CombinePayload { - SdkFunctionSpec combine_fn; + FunctionSpec combine_fn; string accumulator_coder_id; ... } @@ -773,9 +678,7 @@ message CombinePayload { ### `PTransform` proto A `PTransform` is a function from `PCollection` to `PCollection`. This is -represented in the proto using a FunctionSpec. Note that this is not an -`SdkFunctionSpec`, since it is the runner that observes these. They will never -be passed back to an SDK harness; they do not represent a UDF. +represented in the proto using a FunctionSpec. {{< highlight class="no-toggle" >}} message PTransform { @@ -796,6 +699,13 @@ The input and output `PCollections` are unordered and referred to by a local name. The SDK decides what this name is, since it will likely be embedded in serialized UDFs. +A runner that understands the specification of a given `PTransform` (whether +primitive or composite), as defined by its `FunctionSpec`, is free to +substitute it with another `PTransform` (or set thereof) that has identical +semantics. +This is typically how `CombinePerKey` is handled, but many other substitutions +can be done as well. + ### `PCollection` proto A `PCollection` just stores a coder, windowing strategy, and whether or not it @@ -813,83 +723,46 @@ message PCollection { ### `Coder` proto This is a very interesting proto. A coder is a parameterized function that may -only be understood by a particular SDK, hence an `SdkFunctionSpec`, but also +only be understood by a particular SDK, hence an `FunctionSpec`, but also may have component coders that fully define it. For example, a `ListCoder` is only a meta-format, while `ListCoder(VarIntCoder)` is a fully specified format. {{< highlight class="no-toggle" >}} message Coder { - SdkFunctionSpec spec; + FunctionSpec spec; repeated string component_coder_ids; } {{< /highlight >}} -## The Runner API RPCs +There are a large number of +[standard coders](https://github.com/apache/beam/blob/release-2.49.0/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/beam_runner_api.proto#L829) +understood by most, if not all, +SDKs. Using these allows for cross-language transforms. -While your language's SDK will probably insulate you from touching the Runner -API protos directly, you may need to implement adapters for your runner, to -expose it to another language. So this section covers proto that you will -possibly interact with quite directly. +## The Jobs API RPCs + +[Overview](https://docs.google.com/presentation/d/1Cso0XP9dmj77OD9Bd53C1M3W1sPJF0ZnA20gzb2BPhE/edit#slide=id.g42e4c9aad6_1_3722) +[Spec](https://github.com/apache/beam/blob/master/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_job_api.proto) -The specific manner in which the existing runner method calls will be expressed -as RPCs is not implemented as proto yet. This RPC layer is to enable, for -example, building a pipeline using the Python SDK and launching it on a runner -that is written in Java. It is expected that a small Python shim will -communicate with a Java process or service hosting the Runner API. +While your language's SDK will may insulate you from touching the Runner +API protos directly, you may need to implement adapters for your runner, to +expose it to another language. +This allows a Python SDK to invoke a Java runner or vice versa. +A typical implementation of this can be found in +[local_job_service.py](https://github.com/apache/beam/blob/release-2.48.0/sdks/python/apache_beam/runners/portability/local_job_service.py) +which is used directly to front several Python-implemented runners. The RPCs themselves will necessarily follow the existing APIs of PipelineRunner and PipelineResult, but altered to be the minimal backend channel, versus a rich and convenient API. -### `PipelineRunner.run(Pipeline)` RPC - -This will take the same form, but `PipelineOptions` will have to be serialized -to JSON (or a proto `Struct`) and passed along. +A key piece of this is the +(Artifacts API)[https://github.com/apache/beam/blob/master/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_artifact_api.proto], +which allows a Runner to fetch and deploy binary artifacts (such as jars, +pypi packages, etc.) that are listed as dependencies in the various environments, +and may have various representations. This is invoked after a pipeline +is submitted, but before it is executed. The SDK submitting a pipeline acts +as an artifact server to the runner receiving the request, and in turn the +runner then acts as an artifact server to the workers (environments) hosting +the users UDFs. -{{< highlight class="no-toggle" >}} -message RunPipelineRequest { - Pipeline pipeline; - Struct pipeline_options; -} -{{< /highlight >}} - -{{< highlight class="no-toggle" >}} -message RunPipelineResponse { - bytes pipeline_id; - - // TODO: protocol for rejecting pipelines that cannot be executed - // by this runner. May just be REJECTED job state with error message. - - // totally opaque to the SDK; for the shim to interpret - Any contents; -} -{{< /highlight >}} - -### `PipelineResult` aka "Job API" - -The two core pieces of functionality in this API today are getting the state of -a job and canceling the job. It is very much likely to evolve, for example to -be generalized to support draining a job (stop reading input and let watermarks -go to infinity). Today, verifying our test framework benefits (but does not -depend upon wholly) querying metrics over this channel. - -{{< highlight class="no-toggle" >}} -message CancelPipelineRequest { - bytes pipeline_id; - ... -} - -message GetStateRequest { - bytes pipeline_id; - ... -} - -message GetStateResponse { - JobState state; - ... -} - -enum JobState { - ... -} -{{< /highlight >}} diff --git a/website/www/site/content/en/documentation/io/built-in/google-bigquery.md b/website/www/site/content/en/documentation/io/built-in/google-bigquery.md index 61ff8c74e6dc2..26ca0baec0cf7 100644 --- a/website/www/site/content/en/documentation/io/built-in/google-bigquery.md +++ b/website/www/site/content/en/documentation/io/built-in/google-bigquery.md @@ -363,7 +363,7 @@ GitHub](https://github.com/apache/beam/blob/master/examples/java/src/main/java/o {{< /highlight >}} {{< highlight py >}} -# The SDK for Python does not support the BigQuery Storage API. +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_read_table_with_storage_api >}} {{< /highlight >}} The following code snippet reads with a query string. @@ -779,6 +779,19 @@ Starting with version 2.36.0 of the Beam SDK for Java, you can use the [BigQuery Storage Write API](https://cloud.google.com/bigquery/docs/write-api) from the BigQueryIO connector. +Also after version 2.47.0 of Beam SDK for Python, SDK supports BigQuery Storage Write API. + +{{< paragraph class="language-py" >}} +BigQuery Storage Write API for Python SDK currently has some limitations on supported data types. As this method makes use of cross-language transforms, we are limited to the types supported at the cross-language boundary. For example, `apache_beam.utils.timestamp.Timestamp` is needed to write a `TIMESTAMP` BigQuery type. Also, some types (e.g. `DATETIME`) are not supported yet. For more details, please refer to the [full type mapping](https://github.com/apache/beam/blob/0b430748cdd2e25edc553747ce018195e9cce888/sdks/python/apache_beam/io/gcp/bigquery_tools.py#L112-L123). +{{< /paragraph >}} + +{{< paragraph class="language-py" >}} +**Note:** If you want to run WriteToBigQuery with Storage Write API from the source code, you need to run `./gradlew :sdks:java:io:google-cloud-platform:expansion-service:build` to build the expansion-service jar. If you are running from a released Beam SDK, the jar will already be included. + +**Note:** Auto sharding is not currently supported for Python's Storage Write API exactly-once mode on DataflowRunner. + +{{< /paragraph >}} + #### Exactly-once semantics To write to BigQuery using the Storage Write API, set `withMethod` to @@ -795,7 +808,7 @@ BigQueryIO.writeTableRows() ); {{< /highlight >}} {{< highlight py >}} -# The SDK for Python does not support the BigQuery Storage API. +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_write_with_storage_write_api >}} {{< /highlight >}} If you want to change the behavior of BigQueryIO so that all the BigQuery sinks @@ -820,7 +833,7 @@ TableSchema schema = new TableSchema().setFields( .setMode("REQUIRED"))); {{< /highlight >}} {{< highlight py >}} -# The SDK for Python does not support the BigQuery Storage API. +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_write_schema >}} {{< /highlight >}} For streaming pipelines, you need to set two additional parameters: the number @@ -834,7 +847,7 @@ BigQueryIO.writeTableRows() ); {{< /highlight >}} {{< highlight py >}} -# The SDK for Python does not support the BigQuery Storage API. +{{< code_sample "sdks/python/apache_beam/examples/snippets/snippets.py" model_bigqueryio_storage_write_api_with_frequency >}} {{< /highlight >}} The number of streams defines the parallelism of the BigQueryIO Write transform @@ -859,14 +872,20 @@ the BigQuery service, so you should use only as many streams as needed for your use case. Triggering frequency in single-digit seconds is a good choice for most pipelines. -Currently, `STORAGE_WRITE_API` doesn’t support -[`withAutoSharding`](https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.Write.html#withAutoSharding--). -The method will be supported in a future release. +{{< paragraph class="language-java" wrap="span">}} +Similar to streaming inserts, `STORAGE_WRITE_API` supports dynamically determining +the number of parallel streams to write to BigQuery (starting 2.42.0). You can +explicitly enable this using [`withAutoSharding`](https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.Write.html#withAutoSharding--). + +***Note:*** `STORAGE_WRITE_API` will default to dynamic sharding when +`numStorageWriteApiStreams` is set to 0 or is unspecified. + +***Note:*** Auto sharding with `STORAGE_WRITE_API` is supported on Dataflow's legacy runner, but **not** on Runner V2 +{{< /paragraph >}} -When using `STORAGE_WRITE_API`, the PCollection returned by -[`WriteResult.getFailedInserts`](https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.html#getFailedInserts--) -will not contain the failed rows. If there are data validation errors, the -transform will throw a `RuntimeException`. +When using `STORAGE_WRITE_API`, the `PCollection` returned by +[`WriteResult.getFailedStorageApiInserts`](https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.html#getFailedStorageApiInserts--) +will contain the rows that failed to be written to the Storage Write API sink. #### At-least-once semantics @@ -881,10 +900,9 @@ specify the number of streams, and you can’t specify the triggering frequency. Auto sharding is not applicable for `STORAGE_API_AT_LEAST_ONCE`. -When using `STORAGE_API_AT_LEAST_ONCE`, the PCollection returned by -[`WriteResult.getFailedInserts`](https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.html#getFailedInserts--) -will not contain the failed rows. If there are data validation errors, the -transform will throw a `RuntimeException`. +When using `STORAGE_API_AT_LEAST_ONCE`, the `PCollection` returned by +[`WriteResult.getFailedStorageApiInserts`](https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/bigquery/WriteResult.html#getFailedStorageApiInserts--) +will contain the rows that failed to be written to the Storage Write API sink. #### Quotas diff --git a/website/www/site/content/en/documentation/io/connectors.md b/website/www/site/content/en/documentation/io/connectors.md index 83c53f165b512..59b8898aa2265 100644 --- a/website/www/site/content/en/documentation/io/connectors.md +++ b/website/www/site/content/en/documentation/io/connectors.md @@ -79,7 +79,7 @@ This table provides a consolidated, at-a-glance overview of the available built-
    - + - + - + - + - + @@ -798,6 +801,23 @@ This table provides a consolidated, at-a-glance overview of the available built- + + + + + + + + + + + + + + + + + + + + + +
    Improve the website
    Share a code sample or template
    Test a release candidate - Anybody can propose a release via the dev@beam.apache.org mailing list. Try Apache Beam releases in your projects, vote for release candidates, inform the community about the results and any issues found via dev@beam.apache.org. Learn more about how to validate a Beam release here. + Anybody can propose a release via the dev@beam.apache.org mailing list. Try Apache Beam releases in your projects, vote for release candidates, inform the community about the results and any issues found via dev@beam.apache.org. Learn more about how to validate a Beam release here.
    TextIOTextIO (metrics) @@ -200,7 +200,7 @@ This table provides a consolidated, at-a-glance overview of the available built-
    GcsFileSystemGcsFileSystem (metrics) @@ -522,7 +522,7 @@ This table provides a consolidated, at-a-glance overview of the available built-
    BigQueryIO (guide)BigQueryIO (guide) (metrics) @@ -545,7 +545,7 @@ This table provides a consolidated, at-a-glance overview of the available built-
    BigTableIOBigTableIO (metrics) @@ -554,12 +554,15 @@ This table provides a consolidated, at-a-glance overview of the available built- ✔ - native + native (sink) +
    + ✔ + via X-language
    Not available Not available
    DatastoreIO
    DicomIO + ✔ + native + + ✔ + native + Not availableNot available
    FlinkStreaming
    ImpulseSource @@ -918,6 +938,20 @@ This table provides a consolidated, at-a-glance overview of the available built-
    GoogleAdsIO + ✔ + native + Not availableNot availableNot available
    @@ -1096,5 +1130,21 @@ This table provides a consolidated, at-a-glance overview of the available built-

    + Cloud Bigtable (HBase based) + + ✔ + native + Not availableNot availableNot available
    diff --git a/website/www/site/content/en/documentation/ml/data-processing.md b/website/www/site/content/en/documentation/ml/data-processing.md index 9b6ec5117a55d..f6406353ea61a 100755 --- a/website/www/site/content/en/documentation/ml/data-processing.md +++ b/website/www/site/content/en/documentation/ml/data-processing.md @@ -1,5 +1,5 @@ --- -title: "Data processing" +title: "Data exploration" --- -# Data processing +# Data exploration Several types of Apache Beam data processing are applicable to AI/ML projects: - Data exploration: Learn about your data (properties, distributions, statistics) when you start to deploy your project or when the data changes. diff --git a/website/www/site/content/en/documentation/ml/multi-language-inference.md b/website/www/site/content/en/documentation/ml/multi-language-inference.md index 0d7a972e07657..1480b37ab4841 100644 --- a/website/www/site/content/en/documentation/ml/multi-language-inference.md +++ b/website/www/site/content/en/documentation/ml/multi-language-inference.md @@ -99,7 +99,7 @@ Finally, we postprocess the model predictions in the `Postprocess` DoFn. The `Po The custom Python code needs to be written in a local package and be compiled as a tarball. This package can then be used by the Java pipeline. The following example shows how to compile the Python package into a tarball: ```bash - python setup.py sdist + pip install --upgrade build && python -m build --sdist ``` In order to run this, a `setup.py` is required. The path to the tarball will be used as an argument in the pipeline options of the Java pipeline. diff --git a/website/www/site/content/en/documentation/ml/multi-model-pipelines.md b/website/www/site/content/en/documentation/ml/multi-model-pipelines.md index 569a51b8db55f..c42c8b8ae6611 100644 --- a/website/www/site/content/en/documentation/ml/multi-model-pipelines.md +++ b/website/www/site/content/en/documentation/ml/multi-model-pipelines.md @@ -95,3 +95,61 @@ captions. The solution consists of two open-source models: 2. **A caption ranking model ([CLIP](https://github.com/openai/CLIP))** that uses the image and candidate captions to rank the captions in the order in which they best describe the image. +## Use multiple differently-trained models + +You can use a `KeyedModelHandler` to load several different models into the `RunInference` transform. +Use the associated key to determine which model to use with which data. +The following example loads a model by using `config1`. That model is used for inference for all examples associated +with `key1`. It loads a second model by using `config2`. That model is used for all examples associated with `key2` and `key3`. + +``` +from apache_beam.ml.inference.base import KeyedModelHandler +keyed_model_handler = KeyedModelHandler([ + KeyModelMapping(['key1'], PytorchModelHandlerTensor()), + KeyModelMapping(['key2', 'key3'], PytorchModelHandlerTensor()) +]) +with pipeline as p: + data = p | beam.Create([ + ('key1', torch.tensor([[1,2,3],[4,5,6],...])), + ('key2', torch.tensor([[1,2,3],[4,5,6],...])), + ('key3', torch.tensor([[1,2,3],[4,5,6],...])), + ]) + predictions = data | RunInference(keyed_model_handler) +``` + +For a more detailed example, see the notebook +[Run ML inference with multiple differently-trained models](https://colab.sandbox.google.com/github/apache/beam/blob/master/examples/notebooks/beam-ml/per_key_models.ipynb). + +Loading multiple models at the same times increases the risk of out of memory errors (OOMs). By default, `KeyedModelHandler` doesn't +limit the number of models loaded into memory at the same time. If the models don't all fit into memory, +your pipeline might fail with an out of memory error. To avoid this issue, use the `max_models_per_worker_hint` parameter +to set the maximum number of models that can be loaded into memory at the same time. + +The following example loads at most two models per SDK worker process at a time. It unloads models that aren't +currently in use. + +``` +mhs = [ + KeyModelMapping(['key1'], PytorchModelHandlerTensor()), + KeyModelMapping(['key2', 'key3'], PytorchModelHandlerTensor()), + KeyModelMapping(['key4'], PytorchModelHandlerTensor()), + KeyModelMapping(['key5', 'key6', 'key7'], PytorchModelHandlerTensor()), +] +keyed_model_handler = KeyedModelHandler(mhs, max_models_per_worker_hint=2) +``` + +Runners that have multiple SDK worker processes on a given machine load at most +`max_models_per_worker_hint*` models onto the machine. + +Leave enough space for the models and any additional memory needs from other transforms. +Because the memory might not be released immediately after a model is offloaded, +leaving an additional buffer is recommended. + +**Note**: Having many models but a small `max_models_per_worker_hint` can cause _memory thrashing_, where +a large amount of execution time is used to swap models in and out of memory. To reduce the likelihood and impact +of memory thrashing, if you're using a distributed runner, insert a +[`GroupByKey`](https://beam.apache.org/documentation/transforms/python/aggregation/groupbykey/) transform before your +inference step. The `GroupByKey` transform reduces thrashing by ensuring that elements with the same key and model are +collocated on the same worker. + +For more information, see [`KeyedModelHander`](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.inference.base.html#apache_beam.ml.inference.base.KeyedModelHandler). diff --git a/website/www/site/content/en/documentation/ml/overview.md b/website/www/site/content/en/documentation/ml/overview.md index fd2cf71852588..63aaf0f86e202 100644 --- a/website/www/site/content/en/documentation/ml/overview.md +++ b/website/www/site/content/en/documentation/ml/overview.md @@ -89,7 +89,8 @@ You can use Apache Beam for data validation and preprocessing by setting up data {{< table >}} | Task | Example | | ------- | ---------------| -| I want to explore my data | [Data Processing Example](/documentation/ml/data-processing) |: +| I want to transform my data for preprocessing| [Preprocess data with MLTransform](/documentation/ml/preprocess-data) | +| I want to explore my data | [Data exploration workflow and example](/documentation/ml/data-processing) |: {{< /table >}} diff --git a/website/www/site/content/en/documentation/ml/preprocess-data.md b/website/www/site/content/en/documentation/ml/preprocess-data.md new file mode 100644 index 0000000000000..cb79afff60368 --- /dev/null +++ b/website/www/site/content/en/documentation/ml/preprocess-data.md @@ -0,0 +1,229 @@ +--- +title: "Preprocess data" +--- + + +# Preprocess data with MLTransform + +This page explains how to use the `MLTransform` class to preprocess data for machine learning (ML) +workflows. Apache Beam provides a set of data processing transforms for +preprocessing data for training and inference. The `MLTransform` class wraps the +various transforms in one class, simplifying your workflow. For a full list of +available transforms, see the [Transforms](#transforms) section on this page. + +The set of transforms currently available in the `MLTransform` class come from +the TensorFlow Transforms (TFT) library. TFT offers specialized processing +modules for machine learning tasks. + +## Why use MLTransform {#use-mltransform} + +- With `MLTransform`, you can use the same preprocessing steps for both + training and inference, which ensures consistent results. +- Use `MLTransform` to transform a single example or a batch of + examples. +- `MLTransform` can do a full pass on the dataset, which is useful when + you need to transform a single element only after analyzing the entire + dataset. For example, with `MLTransform`, you can complete the following tasks: + - Normalize an input value by using the minimum and maximum value + of the entire dataset. + - Convert `floats` to `ints` by assigning them buckets, based on + the observed data distribution. + - Convert `strings` to `ints` by generating vocabulary over the + entire dataset. + - Count the occurrences of words in all the documents to calculate + [TF-IDF](https://en.wikipedia.org/wiki/Tf%E2%80%93idf) + weights. + +## Support and limitations {#support} + +- Available in the Apache Beam Python SDK versions 2.50.0 and later. +- Supports Python 3.8 and 3.9. +- Only available for pipelines that use [default windows](/documentation/programming-guide/#single-global-window). +- Only supports one-to-one transform mapping on a single element. + +## Transforms {#transforms} + +You can use `MLTransform` to perform the following data processing transforms. +For information about the transforms, see +[Module:tft](https://www.tensorflow.org/tfx/transform/api_docs/python/tft) in the +TensorFlow documentation. + +{{< table >}} +| Transform name | Description | +| ------- | ---------------| +| ApplyBuckets | See [`tft.apply_buckets`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/apply_buckets) in the TensorFlow documentation. | +| Bucketize | See [`tft.bucketize`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/bucketize) in the TensorFlow documentation. | +| ComputeAndApplyVocabulary | See [`tft.compute_and_apply_vocabulary`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/compute_and_apply_vocabulary) in the TensorFlow documentation. | +| NGrams | See [`tft.ngrams`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/ngrams) in the TensorFlow documentation. | +| ScaleByMinMax | See [`tft.scale_by_min_max`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/scale_by_min_max) in the TensorFlow documentation. | +| ScaleTo01 | See [`tft.scale_to_0_1`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/scale_to_0_1) in the TensorFlow documentation. | +| ScaleToZScore | See [`tft.scale_to_z_score`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/scale_to_z_score) in the TensorFlow documentation. | +| TFIDF | See [`tft.tfidf`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/tfidf) in the TensorFlow documentation. |: +{{< /table >}} + +Apply the transforms on either single or multiple columns passed as a +`dict` on structured data. Keys are column names and values are lists containing +each column's data. + +## I/O requirements {#io} + +- Input to the `MLTransform` class must be in one of the following formats: + - A `dict` of `str` + - Primitive types + - List of primitive types + - NumPy arrays +- `MLTransform` outputs a Beam `Row` object with NumPy arrays. +- The output `PCollection` is a schema `PCollection`. The output schema + contains the transformed columns. + +## Artifacts {#artifacts} + +Artifacts are additional data elements created by data transformations. +Examples of artifacts are the minimum and maximum values from a `ScaleTo01` +transformation, or the mean and variance from a `ScaleToZScore` +transformation. + +In the `MLTransform` class, the `write_artifact_location` and the +`read_artifact_location` parameters determine +whether the `MLTransform` class creates artifacts or retrieves +artifacts. + +### Write mode {#write-mode} + +When you use the `write_artifact_location` parameter, the `MLTransform` class runs the +specified transformations on the dataset and then creates artifacts from these +transformations. The artifacts are stored in the location that you specify in +the `write_artifact_location` parameter or in the `MLTransform` output. + +Write mode is useful when you want to store the results of your transformations +for future use. For example, if you apply the same transformations on a +different dataset, use write mode to ensure that the transformation parameters +remain consistent. + +The following examples demonstrate how write mode works. + +- The `ComputeAndApplyVocabulary` transform generates a vocabulary file that contains the + vocabulary generated over the entire dataset. The vocabulary file is stored in + the location specified by the `write_artifact_location` parameter value. + The `ComputeAndApplyVocabulary` + transform outputs the indices of the vocabulary to the vocabulary file. +- The `ScaleToZScore` transform calculates the mean and variance over the entire dataset + and then normalizes the entire dataset using the mean and variance. The + mean and variance are outputted by the `MLTransform` operation. + When you use the `write_artifact_location` parameter, these + values are stored as a `tensorflow` graph in the location specified by + the `write_artifact_location` parameter value. You can reuse the values in read mode + to ensure that future transformations use the same mean and variance for normalization. + +### Read mode {#read-mode} + +When you use the `read_artifact_location` parameter, the `MLTransform` class expects the +artifacts to exist in the value provided in the `read_artifact_location` parameter. +In this mode, `MLTransform` retrieves the artifacts and uses them in the +transform. Because the transformations are stored in the artifacts when you use +read mode, you don't need to specify the transformations. + +### Artifact workflow {#artifact-workflow} + +The following scenario provides an example use case for artifacts. + +Before training a machine learning model, you use `MLTransform` with the +`write_artifact_location` parameter. +When you run `MLTransform`, it applies transformations that preprocess the +dataset. The transformation produces artifacts that are stored in the location +specified by the `write_artifact_location` parameter value. + +After preprocessing, you use the transformed data to train the machine learning +model. + +After training, you run inference. You use new test data and use the +`read_artifact_location` parameter. By using this setting, you ensure that the test +data undergoes the same preprocessing steps as the training data. In read +mode, running `MLTransform` fetches the transformation artifacts from the +location specified in the `read_artifact_location` parameter value. +`MLTransform` applies these artifacts to the test data. + +This workflow provides consistency in preprocessing steps for both training and +test data. This consistency ensures that the model can accurately evaluate the +test data and maintain the integrity of the model's performance. + +## Preprocess data with MLTransform {#use-mltransform} + +To use the `MLTransform` transform to preprocess data, add the following code to +your pipeline: + +``` + import apache_beam as beam + from apache_beam.ml.transforms.base import MLTransform + from apache_beam.ml.transforms.tft import + import tempfile + + data = [ + { + + }, + ] + + artifact_location = tempfile.mkdtemp() + = (columns=['x']) + + with beam.Pipeline() as p: + transformed_data = ( + p + | beam.Create(data) + | MLTransform(write_artifact_location=artifact_location).with_transform( + ) + | beam.Map(print)) +``` + +Replace the following values: + +- TRANSFORM_NAME: The name of the [transform](#transforms) to use. +- DATA: The input data to transform. +- TRANSFORM_FUNCTION_NAME: The name that you assign to your transform + function in your code. + +For more examples, see +[MLTransform for data processing](/documentation/transforms/python/elementwise/mltransform) +in the [transform catalog](/documentation/transforms/python/overview/). + +### ScaleTo01 example {#scaleto01} + +This example demonstrates how to use `MLTransform` to normalize your data +between 0 and 1 by using the minimum and maximum values from your entire +dataset. `MLTransform` uses the `ScaleTo01` transformation. + +Use the following snippet to apply `ScaleTo01` on column `x` of the input +data. + +``` +data_pcoll | MLTransform(write_artifact_location=).with_transform(ScaleTo01(columns=['x'])) +``` + +The `ScaleTo01` transformation produces two artifacts: the `min` and the `max` +of the entire dataset. For more information, see the +[Artifacts](#artifacts) section on this page. + +## Metrics {#metrics} + +When you use MLTransform, the following metrics are available. + +{{< table >}} +| Metric | Description | +| ------- | ---------------| +| Data throughput | The number of records processed per second. This metric indicates the processing capacity of the pipeline for `beam.MLTransform.` | +| Memory usage | The number of records processed per second. This metric indicates the processing capacity of the pipeline for `beam.MLTransform`. | +| Counters | Tracks the number of elements processed. Each `MLTransform` has a counter. |: +{{< /table >}} \ No newline at end of file diff --git a/website/www/site/content/en/documentation/programming-guide.md b/website/www/site/content/en/documentation/programming-guide.md index 8a135b9817ae1..564b01a7146e4 100644 --- a/website/www/site/content/en/documentation/programming-guide.md +++ b/website/www/site/content/en/documentation/programming-guide.md @@ -39,7 +39,7 @@ The Python SDK supports Python 3.7, 3.8, 3.9, 3.10, and 3.11. {{< /paragraph >}} {{< paragraph class="language-go">}} -The Go SDK supports Go v1.19+. SDK release 2.32.0 is the last experimental version. +The [Go SDK](https://pkg.go.dev/github.com/apache/beam/sdks/v2/go/pkg/beam) supports Go v1.20+. {{< /paragraph >}} {{< paragraph class="language-typescript">}} @@ -834,12 +834,18 @@ func init() { ##### 4.2.1.1. Applying ParDo {#applying-pardo} -{{< paragraph class="language-java language-py" >}} +{{< paragraph class="language-java">}} Like all Beam transforms, you apply `ParDo` by calling the `apply` method on the input `PCollection` and passing `ParDo` as an argument, as shown in the following example code: {{< /paragraph >}} +{{< paragraph class="language-py">}} +Like all Beam transforms, you apply `ParDo` by calling the `beam.ParDo` on the +input `PCollection` and passing the `DoFn` as an argument, as shown in the +following example code: +{{< /paragraph >}} + {{< paragraph class="language-go">}} `beam.ParDo` applies the passed in `DoFn` argument to the input `PCollection`, as shown in the following example code: @@ -1124,6 +1130,9 @@ words = ... {{< /highlight >}} {{< highlight go >}} + +The Go SDK cannot support anonymous functions outside of the deprecated Go Direct runner. + // words is the input PCollection of strings var words beam.PCollection = ... @@ -1170,8 +1179,8 @@ words = ... {{< /highlight >}} {{< highlight go >}} -// words is the input PCollection of strings -var words beam.PCollection = ... + +The Go SDK cannot support anonymous functions outside of the deprecated Go Direct runner. {{< code_sample "sdks/go/examples/snippets/04transforms.go" model_pardo_apply_anon >}} {{< /highlight >}} @@ -1191,7 +1200,7 @@ words = ... -> **Note:** Anonymous function DoFns may not work on distributed runners. +> **Note:** Anonymous function DoFns do not work on distributed runners. > It's recommended to use named functions and register them with `register.FunctionXxY` in > an `init()` block. @@ -1203,10 +1212,13 @@ Here is a sequence diagram that shows the lifecycle of the DoFn during the execution of the ParDo transform. The comments give useful information to pipeline developers such as the constraints that apply to the objects or particular cases such as failover or - instance reuse. They also give instantiation use cases. Two key points - to note are that (1) teardown is done on a best effort basis and thus - isn't guaranteed and (2) the number of DoFn instances is runner - dependent. + instance reuse. They also give instantiation use cases. Three key points + to note are that: + 1. Teardown is done on a best effort basis and thus + isn't guaranteed. + 2. The number of DoFn instances created at runtime is runner-dependent. + 3. For the Python SDK, the pipeline contents such as DoFn user code, + is [serialized into a bytecode](https://beam.apache.org/documentation/sdks/python-pipeline-dependencies/#pickling-and-managing-the-main-session). Therefore, `DoFn`s should not reference objects that are not serializable, such as locks. To manage a single instance of an object across multiple `DoFn` instances in the same process, use utilities in the [shared.py](https://beam.apache.org/releases/pydoc/current/apache_beam.utils.shared.html) module. ![This is a sequence diagram that shows the lifecycle of the DoFn](/images/dofn-sequence-diagram.svg) @@ -2568,9 +2580,7 @@ Timers and States are explained in more detail in the {{< paragraph class="language-go">}} **Timer and State:** -User defined State parameters can be used in a stateful DoFn. Timers aren't implemented in the Go SDK yet; -see more at [Issue 22737](https://github.com/apache/beam/issues/22737). Once implemented, user defined Timer -parameters can be used in a stateful DoFn. +User defined State and Timer parameters can be used in a stateful DoFn. Timers and States are explained in more detail in the [Timely (and Stateful) Processing with Apache Beam](/blog/2017/08/28/timely-processing.html) blog post. {{< /paragraph >}} @@ -6147,7 +6157,7 @@ _ = (p | 'Read per user' >> ReadPerUser() {{< /highlight >}} {{< highlight go >}} -{{< code_sample "sdks/go/examples/snippets/04transforms.go" state_and_timers >}} +{{< code_sample "sdks/go/examples/snippets/04transforms.go" bag_state >}} {{< /highlight >}} ### 11.2. Deferred state reads {#deferred-state-reads} @@ -6270,7 +6280,7 @@ _ = (p | 'Read per user' >> ReadPerUser() {{< /highlight >}} {{< highlight go >}} -This is not supported yet, see https://github.com/apache/beam/issues/22737. +{{< code_sample "sdks/go/examples/snippets/04transforms.go" event_time_timer >}} {{< /highlight >}} #### 11.3.2. Processing-time timers {#processing-time-timers} @@ -6322,7 +6332,7 @@ _ = (p | 'Read per user' >> ReadPerUser() {{< /highlight >}} {{< highlight go >}} -This is not supported yet, see https://github.com/apache/beam/issues/22737. +{{< code_sample "sdks/go/examples/snippets/04transforms.go" processing_time_timer >}} {{< /highlight >}} #### 11.3.3. Dynamic timer tags {#dynamic-timer-tags} @@ -6383,7 +6393,7 @@ _ = (p | 'Read per user' >> ReadPerUser() {{< /highlight >}} {{< highlight go >}} -This is not supported yet, see https://github.com/apache/beam/issues/22737. +{{< code_sample "sdks/go/examples/snippets/04transforms.go" dynamic_timer_tags >}} {{< /highlight >}} #### 11.3.4. Timer output timestamps {#timer-output-timestamps} @@ -6435,6 +6445,10 @@ perUser.apply(ParDo.of(new DoFn, OutputT>() { })); {{< /highlight >}} +{{< highlight go >}} +{{< code_sample "sdks/go/examples/snippets/04transforms.go" timer_output_timestamps_bad >}} +{{< /highlight >}} + The problem with this code is that the ParDo is buffering elements, however nothing is preventing the watermark from advancing past the timestamp of those elements, so all those elements might be dropped as late data. In order to prevent this from happening, an output timestamp needs to be set on the timer to prevent the watermark from advancing @@ -6471,7 +6485,7 @@ perUser.apply(ParDo.of(new DoFn, OutputT>() { ? Instant.now().plus(Duration.standardMinutes(1)) : new Instant(timerTimestampMs); // Setting the outputTimestamp to the minimum timestamp in the bag holds the watermark to that timestamp until the // timer fires. This allows outputting all the elements with their timestamp. - timer.withOutputTimestamp(minTimestamp.read()).set(timerToSet). + timer.withOutputTimestamp(minTimestamp.read()).s et(timerToSet). timerTimestamp.write(timerToSet.getMillis()); } @@ -6494,7 +6508,7 @@ Timer output timestamps is not yet supported in Python SDK. See https://github.c {{< /highlight >}} {{< highlight go >}} -This is not supported yet, see https://github.com/apache/beam/issues/22737. +{{< code_sample "sdks/go/examples/snippets/04transforms.go" timer_output_timestamps_good >}} {{< /highlight >}} ### 11.4. Garbage collecting state {#garbage-collecting-state} @@ -6624,7 +6638,7 @@ _ = (p | 'Read per user' >> ReadPerUser() {{< /highlight >}} {{< highlight go >}} -This is not supported yet, see https://github.com/apache/beam/issues/22737. +{{< code_sample "sdks/go/examples/snippets/04transforms.go" timer_garbage_collection >}} {{< /highlight >}} ### 11.5. State and timers examples {#state-timers-examples} @@ -6764,6 +6778,11 @@ _ = (p | 'EventsPerLinkId' >> ReadPerLinkEvents() | 'Join DoFn' >> beam.ParDo(JoinDoFn())) {{< /highlight >}} + +{{< highlight go >}} +{{< code_sample "sdks/go/examples/snippets/04transforms.go" join_dofn_example >}} +{{< /highlight >}} + #### 11.5.2. Batching RPCs {#batching-rpcs} In this example, input elements are being forwarded to an external RPC service. The RPC accepts batch requests - @@ -6830,6 +6849,12 @@ class BufferDoFn(DoFn): {{< /highlight >}} + +{{< highlight go >}} +{{< code_sample "sdks/go/examples/snippets/04transforms.go" batching_dofn_example >}} +{{< /highlight >}} + + ## 12. Splittable `DoFns` {#splittable-dofns} A Splittable `DoFn` (SDF) enables users to create modular components containing I/Os (and some advanced @@ -7664,7 +7689,7 @@ When an SDK-specific wrapper isn't available, you will have to access the cross- | beam.Create(['a', 'b']).with_output_types(unicode) | beam.ExternalTransform( TEST_PREFIX_URN, - ImplicitSchemaPayloadBuilder({'data': u'0'}), + ImplicitSchemaPayloadBuilder({'data': '0'}), )) assert_that(res, equal_to(['0a', '0b'])) ``` @@ -8065,3 +8090,82 @@ class RetrieveTimingDoFn(beam.DoFn):   def infer_output_type(self, input_type):     return input_type {{< /highlight >}} + +## 15 Transform service {#transform-service} + +The Apache Beam SDK versions 2.49.0 and later include a [Docker Compose](https://docs.docker.com/compose/) +service named _Transform service_. Use the Transform service to perform expansions of supported transforms +on Beam portable pipelines by using Docker. + +The following diagram illustrates the basic architecture of the Transform service. + +![Diagram of the Transform service architecture](/images/transform_service.png) + +To use the Transform service, Docker and Docker Compose must be available on the machine that starts the service. + +The Transform service has the following primary use cases: + +* Perform expansion of cross-language transforms without installing other language runtimes. + + The Transform service allows multi-language pipelines to use and expand cross-language transforms implemented + in other SDKs without requiring you to install runtimes for the implementation languages of those SDKs. + For example, with the Transform service, a Beam Python pipeline can use the Google Cloud Java I/O transforms and Java Kafka I/O transforms + without a local Java runtime installation. + +* Upgrade transforms without upgrading the Apache Beam SDK version. + + Use the Transform service to upgrade the Beam SDK versions of individual transforms used by Beam pipelines without upgrading the Beam version of the pipeline. + This feature is currently in development. For more details, see + [GitHub issue #27943: Upgrade transforms without upgrading the pipeline using the Transform Service](https://github.com/apache/beam/issues/27943). + +### 15.1 Use the Transform service {#transform-service-usage} + +In some cases, Apache Beam SDKs automatically start the Transform service, such as in the following scenarios: + +* The Java [`PythonExternalTransform` API](https://github.com/apache/beam/blob/master/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/PythonExternalTransform.java) automatically +starts the Transform service when a Python runtime isn't available locally, but Docker is. + +* The Apache Beam Python multi-language wrappers might automatically start the Transform service when you're using Java transforms, a Java language runtime isn't available locally, and Docker is available locally. + +To manually start a Transform service instance by using utilities provided with the Apache Beam SDKs, use the following commands. + +{{< highlight java >}} +java -jar beam-sdks-java-transform-service-launcher-.jar --port --beam_version --project_name --command up +{{< /highlight >}} + +{{< highlight py >}} +python -m apache_beam.utils.transform_service_launcher --port --beam_version --project_name --command up +{{< /highlight >}} + +{{< highlight go >}} +This feature is currently in development. +{{< /highlight >}} + +To stop the transform service, use the following commands. + +{{< highlight java >}} +java -jar beam-sdks-java-transform-service-launcher-.jar --port --beam_version --project_name --command down +{{< /highlight >}} + +{{< highlight py >}} +python -m apache_beam.utils.transform_service_launcher --port --beam_version --project_name --command down +{{< /highlight >}} + +{{< highlight go >}} +This feature is currently in development. +{{< /highlight >}} + +### 15.2 Portable transforms included in the Transform service {#transform-service-included-transforms} + +The Transform service includes portable transforms implemented in the Apache Beam Java and Python SDKs. + +The following transforms are included in the Trasnform service: + +* Java transforms: Google Cloud I/O connectors, the Kafka I/O connector, and the JDBC I/O connector + +* Python transforms: all portable transforms implemented within the Apache Beam Python SDK, such as + [RunInference](/documentation/transforms/python/elementwise/runinference/) and + [DataFrame](/documentation/dsls/dataframes/overview/) transforms. + +For a more comprehensive list of available transforms, see the +[Transform service](https://cwiki.apache.org/confluence/display/BEAM/Transform+Service) developer guide. diff --git a/website/www/site/content/en/documentation/runners/flink.md b/website/www/site/content/en/documentation/runners/flink.md index 1549857068fa4..64a6e9bade9b7 100644 --- a/website/www/site/content/en/documentation/runners/flink.md +++ b/website/www/site/content/en/documentation/runners/flink.md @@ -325,235 +325,84 @@ To find out which version of Flink is compatible with Beam please see the table - + - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - +
    Beam Version Flink Version Artifact IdSupported Beam Versions
    ≥ 2.47.0 1.16.x beam-runners-flink-1.16≥ 2.47.0
    1.15.x beam-runners-flink-1.15≥ 2.40.0
    1.14.x beam-runners-flink-1.14≥ 2.38.0
    1.13.x beam-runners-flink-1.13≥ 2.31.0
    1.12.x beam-runners-flink-1.12
    2.40.0 - 2.46.01.15.xbeam-runners-flink-1.15
    1.14.xbeam-runners-flink-1.14
    1.13.xbeam-runners-flink-1.13
    1.12.xbeam-runners-flink-1.12
    2.39.01.14.xbeam-runners-flink-1.14
    1.13.xbeam-runners-flink-1.13
    1.12.xbeam-runners-flink-1.12
    2.38.01.14.xbeam-runners-flink-1.14
    1.13.xbeam-runners-flink-1.13
    1.12.xbeam-runners-flink-1.12
    1.11.xbeam-runners-flink-1.11
    2.31.0 - 2.37.01.13.xbeam-runners-flink-1.13
    1.12.xbeam-runners-flink-1.12
    1.11.xbeam-runners-flink-1.11
    2.30.01.12.xbeam-runners-flink-1.12
    1.11.xbeam-runners-flink-1.11
    1.10.xbeam-runners-flink-1.10
    2.27.0 - 2.29.01.12.xbeam-runners-flink-1.12≥ 2.27.0
    1.11.x beam-runners-flink-1.112.25.0 - 2.38.0
    1.10.x beam-runners-flink-1.102.21.0 - 2.30.0
    1.9.x beam-runners-flink-1.92.17.0 - 2.29.0
    1.8.x beam-runners-flink-1.8
    2.25.0 - 2.26.01.11.xbeam-runners-flink-1.11
    1.10.xbeam-runners-flink-1.10
    1.9.xbeam-runners-flink-1.9
    1.8.xbeam-runners-flink-1.8
    2.21.0 - 2.24.01.10.xbeam-runners-flink-1.10
    1.9.xbeam-runners-flink-1.9
    1.8.xbeam-runners-flink-1.8
    2.17.0 - 2.20.01.9.xbeam-runners-flink-1.9
    1.8.xbeam-runners-flink-1.8
    1.7.xbeam-runners-flink-1.7
    2.13.0 - 2.16.01.8.xbeam-runners-flink-1.82.13.0 - 2.29.0
    1.7.x beam-runners-flink-1.72.10.0 - 2.20.0
    1.6.x beam-runners-flink-1.62.10.0 - 2.16.0
    1.5.x beam-runners-flink_2.112.6.0 - 2.16.0
    2.10.0 - 2.16.01.7.xbeam-runners-flink-1.7
    1.6.xbeam-runners-flink-1.6
    1.5.x1.4.x with Scala 2.11 beam-runners-flink_2.112.3.0 - 2.5.0
    2.9.01.5.xbeam-runners-flink_2.11
    2.8.0
    2.7.0
    2.6.0
    2.5.01.4.x with Scala 2.11beam-runners-flink_2.11
    2.4.0
    2.3.0
    2.2.01.3.x with Scala 2.10beam-runners-flink_2.10
    2.1.x1.3.x with Scala 2.10beam-runners-flink_2.102.1.x - 2.2.0
    2.0.0 1.2.x with Scala 2.10 beam-runners-flink_2.102.0.0
    diff --git a/website/www/site/content/en/documentation/runners/spark.md b/website/www/site/content/en/documentation/runners/spark.md index 3b166c077e0cd..29ef5c28102ad 100644 --- a/website/www/site/content/en/documentation/runners/spark.md +++ b/website/www/site/content/en/documentation/runners/spark.md @@ -67,8 +67,8 @@ the portable Runner. For more information on portability, please visit the ## Spark Runner prerequisites and setup -The Spark runner currently supports Spark's 3.1.x branch. -> **Note:** Support for Spark 2.4.x was deprecated as of Beam 2.41.0 and finally dropped with the release of Beam 2.46.0. +The Spark runner currently supports Spark's 3.2.x branch. +> **Note:** Support for Spark 2.4.x was dropped with Beam 2.46.0. {{< paragraph class="language-java" >}} You can add a dependency on the latest version of the Spark runner by adding to your pom.xml the following: @@ -243,7 +243,7 @@ See [here](/roadmap/portability/#sdk-harness-config) for details.) ### Running on Dataproc cluster (YARN backed) -To run Beam jobs written in Python, Go, and other supported languages, you can use the `SparkRunner` and `PortableRunner` as described on the Beam's [Spark Runner](/documentation/runners/spark/) page (also see [Portability Framework Roadmap](/roadmap/portability/)). +To run Beam jobs written in Python, Go, and other supported languages, you can use the `SparkRunner` and `PortableRunner` as described on the Beam's [Spark Runner](https://beam.apache.org/documentation/runners/spark/) page (also see [Portability Framework Roadmap](https://beam.apache.org/roadmap/portability/)). The following example runs a portable Beam job in Python from the Dataproc cluster's master node with Yarn backed. @@ -487,5 +487,48 @@ Provided SparkContext and StreamingListeners are not supported on the Spark port {{< /paragraph >}} ### Kubernetes - -An [example](https://github.com/cometta/python-apache-beam-spark) of configuring Spark to run Apache beam job +#### Submit beam job without job server +To submit a beam job directly on spark kubernetes cluster without spinning up an extra job server, you can do: +``` +spark-submit --master MASTER_URL \ + --conf spark.kubernetes.driver.podTemplateFile=driver_pod_template.yaml \ + --conf spark.kubernetes.executor.podTemplateFile=executor_pod_template.yaml \ + --class org.apache.beam.runners.spark.SparkPipelineRunner \ + --conf spark.kubernetes.container.image=apache/spark:v3.3.2 \ + ./wc_job.jar +``` +Similar to run the beam job on Dataproc, you can bundle the job jar like below. The example use the `PROCESS` type of [SDK harness](https://beam.apache.org/documentation/runtime/sdk-harness-config/) to execute the job by processes. +``` +python -m beam_example_wc \ + --runner=SparkRunner \ + --output_executable_path=./wc_job.jar \ + --environment_type=PROCESS \ + --environment_config='{\"command\": \"/opt/apache/beam/boot\"}' \ + --spark_version=3 +``` + +And below is an example of kubernetes executor pod template, the `initContainer` is required to download the beam SDK harness to run the beam pipelines. +``` +spec: + containers: + - name: spark-kubernetes-executor + volumeMounts: + - name: beam-data + mountPath: /opt/apache/beam/ + initContainers: + - name: init-beam + image: apache/beam_python3.7_sdk + command: + - cp + - /opt/apache/beam/boot + - /init-container/data/boot + volumeMounts: + - name: beam-data + mountPath: /init-container/data + volumes: + - name: beam-data + emptyDir: {} +``` + +#### Submit beam job with job server +An [example](https://github.com/cometta/python-apache-beam-spark) of configuring Spark to run Apache beam job with a job server. diff --git a/website/www/site/content/en/documentation/runtime/environments.md b/website/www/site/content/en/documentation/runtime/environments.md index 78c7dc32a49d3..452fb6141e63a 100644 --- a/website/www/site/content/en/documentation/runtime/environments.md +++ b/website/www/site/content/en/documentation/runtime/environments.md @@ -66,7 +66,8 @@ This `Dockerfile` uses the prebuilt Python 3.7 SDK container image [`beam_python ``` export BASE_IMAGE="apache/beam_python3.7_sdk:2.25.0" export IMAGE_NAME="myremoterepo/mybeamsdk" - export TAG="latest" + # Avoid using `latest` with custom containers to make reproducing failures easier. + export TAG="mybeamsdk-versioned-tag" # Optional - pull the base image into your local Docker daemon to ensure # you have the most up-to-date version of the base image locally. @@ -114,14 +115,13 @@ This method requires building image artifacts from Beam source. For additional i ./gradlew :sdks:java:container:java11:docker ./gradlew :sdks:java:container:java17:docker ./gradlew :sdks:go:container:docker - ./gradlew :sdks:python:container:py36:docker - ./gradlew :sdks:python:container:py37:docker ./gradlew :sdks:python:container:py38:docker ./gradlew :sdks:python:container:py39:docker ./gradlew :sdks:python:container:py310:docker + ./gradlew :sdks:python:container:py311:docker # Shortcut for building all Python SDKs - ./gradlew :sdks:python:container buildAll + ./gradlew :sdks:python:container:buildAll ``` 4. Verify the images you built were created by running `docker images`. @@ -298,6 +298,11 @@ python -m apache_beam.examples.wordcount \ {{< /runner >}} +Avoid using the tag `:latest` with your custom images. Tag your builds with a date +or a unique identifier. If something goes wrong, using this type of tag might make +it possible to revert the pipeline execution to a previously known working +configuration and allow for an inspection of changes. + ### Troubleshooting diff --git a/website/www/site/content/en/documentation/sdks/go-dependencies.md b/website/www/site/content/en/documentation/sdks/go-dependencies.md index d39f8b3884598..587fddc3d687e 100644 --- a/website/www/site/content/en/documentation/sdks/go-dependencies.md +++ b/website/www/site/content/en/documentation/sdks/go-dependencies.md @@ -26,7 +26,7 @@ This can be found at https://raw.githubusercontent.com/apache/beam/v/sdks/go.sum ``` -

    Replace `` with the major.minor.patch version of the SDK. For example, https://raw.githubusercontent.com/apache/beam/v{{< param release_latest >}}/sdks/go.sum will provide the dependencies for the {{< param release_latest >}} release.

    +

    Replace `<VERSION_NUMBER>` with the major.minor.patch version of the SDK. For example, https://raw.githubusercontent.com/apache/beam/v{{< param release_latest >}}/sdks/go.sum will provide the dependencies for the {{< param release_latest >}} release.

    **Note:** To just view direct dependencies, you can view the `go.mod` file instead, direct dependencies are listed in the initial `require` statement. This can be found at `https://raw.githubusercontent.com/apache/beam/v/sdks/go.mod` diff --git a/website/www/site/content/en/documentation/sdks/java-dependencies.md b/website/www/site/content/en/documentation/sdks/java-dependencies.md index 944ded62c9f95..890641ca16b5f 100644 --- a/website/www/site/content/en/documentation/sdks/java-dependencies.md +++ b/website/www/site/content/en/documentation/sdks/java-dependencies.md @@ -32,7 +32,7 @@ Compile and runtime dependencies for your Beam SDK version are listed in `BeamMo https://raw.githubusercontent.com/apache/beam/v/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy ``` -

    Replace `` with the major.minor.patch version of the SDK. For example, https://raw.githubusercontent.com/apache/beam/v{{< param release_latest >}}/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy will provide the dependencies for the {{< param release_latest >}} release.

    +

    Replace `<VERSION_NUMBER>` with the major.minor.patch version of the SDK. For example, https://raw.githubusercontent.com/apache/beam/v{{< param release_latest >}}/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy will provide the dependencies for the {{< param release_latest >}} release.

    2. Review the list under `project.ext.library`. diff --git a/website/www/site/content/en/documentation/sdks/python-dependencies.md b/website/www/site/content/en/documentation/sdks/python-dependencies.md index 80926e00f3637..09c56adac430d 100644 --- a/website/www/site/content/en/documentation/sdks/python-dependencies.md +++ b/website/www/site/content/en/documentation/sdks/python-dependencies.md @@ -32,7 +32,7 @@ Dependencies for your Beam SDK version are listed in `setup.py` in the Beam repo https://raw.githubusercontent.com/apache/beam/v/sdks/python/setup.py ``` -

    Replace `` with the major.minor.patch version of the SDK. For example, https://raw.githubusercontent.com/apache/beam/v{{< param release_latest >}}/sdks/python/setup.py will provide the dependencies for the {{< param release_latest >}} release.

    +

    Replace `<VERSION_NUMBER>` with the major.minor.patch version of the SDK. For example, https://raw.githubusercontent.com/apache/beam/v{{< param release_latest >}}/sdks/python/setup.py will provide the dependencies for the {{< param release_latest >}} release.

    2. Review the core dependency list under `REQUIRED_PACKAGES`. diff --git a/website/www/site/content/en/documentation/sdks/python-machine-learning.md b/website/www/site/content/en/documentation/sdks/python-machine-learning.md index 5e0cf483ff3ea..a700806f14c6a 100644 --- a/website/www/site/content/en/documentation/sdks/python-machine-learning.md +++ b/website/www/site/content/en/documentation/sdks/python-machine-learning.md @@ -197,9 +197,9 @@ For more information on resource hints, see [Resource hints](/documentation/runt This section suggests patterns and best practices that you can use to make your inference pipelines simpler, more robust, and more efficient. -### Use a keyed ModelHandler +### Use a keyed ModelHandler object -If a key is attached to the examples, wrap the `KeyedModelHandler` around the `ModelHandler` object: +If a key is attached to the examples, wrap `KeyedModelHandler` around the `ModelHandler` object: ``` from apache_beam.ml.inference.base import KeyedModelHandler @@ -213,7 +213,61 @@ with pipeline as p: predictions = data | RunInference(keyed_model_handler) ``` -If you are unsure if your data is keyed, you can also use `MaybeKeyedModelHandler`. +If you are unsure if your data is keyed, you can use `MaybeKeyedModelHandler`. + +You can also use a `KeyedModelHandler` to load several different models based on their associated key. +The following example loads a model by using `config1`. That model is used for inference for all examples associated +with `key1`. It loads a second model by using `config2`. That model is used for all examples associated with `key2` and `key3`. + +``` +from apache_beam.ml.inference.base import KeyedModelHandler +keyed_model_handler = KeyedModelHandler([ + KeyModelMapping(['key1'], PytorchModelHandlerTensor()), + KeyModelMapping(['key2', 'key3'], PytorchModelHandlerTensor()) +]) +with pipeline as p: + data = p | beam.Create([ + ('key1', torch.tensor([[1,2,3],[4,5,6],...])), + ('key2', torch.tensor([[1,2,3],[4,5,6],...])), + ('key3', torch.tensor([[1,2,3],[4,5,6],...])), + ]) + predictions = data | RunInference(keyed_model_handler) +``` + +For a more detailed example, see the notebook +[Run ML inference with multiple differently-trained models](https://colab.sandbox.google.com/github/apache/beam/blob/master/examples/notebooks/beam-ml/per_key_models.ipynb). + +Loading multiple models at the same times increases the risk of out of memory errors (OOMs). By default, `KeyedModelHandler` doesn't +limit the number of models loaded into memory at the same time. If the models don't all fit into memory, +your pipeline might fail with an out of memory error. To avoid this issue, use the `max_models_per_worker_hint` parameter +to set the maximum number of models that can be loaded into memory at the same time. + +The following example loads at most two models per SDK worker process at a time. It unloads models that aren't +currently in use. + +``` +mhs = [ + KeyModelMapping(['key1'], PytorchModelHandlerTensor()), + KeyModelMapping(['key2', 'key3'], PytorchModelHandlerTensor()), + KeyModelMapping(['key4'], PytorchModelHandlerTensor()), + KeyModelMapping(['key5', 'key6', 'key7'], PytorchModelHandlerTensor()), +] +keyed_model_handler = KeyedModelHandler(mhs, max_models_per_worker_hint=2) +``` + +Runners that have multiple SDK worker processes on a given machine load at most +`max_models_per_worker_hint*` models onto the machine. + +Leave enough space for the models and any additional memory needs from other transforms. +Because the memory might not be released immediately after a model is offloaded, +leaving an additional buffer is recommended. + +**Note**: Having many models but a small `max_models_per_worker_hint` can cause _memory thrashing_, where +a large amount of execution time is used to swap models in and out of memory. To reduce the likelihood and impact +of memory thrashing, if you're using a distributed runner, insert a +[`GroupByKey`](https://beam.apache.org/documentation/transforms/python/aggregation/groupbykey/) transform before your +inference step. The `GroupByKey` transform reduces thrashing by ensuring that elements with the same key and model are +collocated on the same worker. For more information, see [`KeyedModelHander`](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.inference.base.html#apache_beam.ml.inference.base.KeyedModelHandler). diff --git a/website/www/site/content/en/documentation/sdks/python-pipeline-dependencies.md b/website/www/site/content/en/documentation/sdks/python-pipeline-dependencies.md index d94b0e3ae84a4..c99c0b9c7cf8f 100644 --- a/website/www/site/content/en/documentation/sdks/python-pipeline-dependencies.md +++ b/website/www/site/content/en/documentation/sdks/python-pipeline-dependencies.md @@ -17,9 +17,8 @@ limitations under the License. --> # Managing Python Pipeline Dependencies -> **Note:** This page is only applicable to runners that do remote execution. +Dependency management is about specifying dependencies that your pipeline requires, and controlling which dependencies are used in production. -When you run your pipeline locally, the packages that your pipeline depends on are available because they are installed on your local machine. However, when you want to run your pipeline remotely, you must make sure these dependencies are available on the remote machines. This tutorial shows you how to make your dependencies available to the remote workers. Each section below refers to a different source that your package may have been installed from. **Note:** Remote workers used for pipeline execution typically have a standard Python distribution installation in a Debian-based container image. If your code relies only on standard Python packages, then you probably don't need to do anything on this page. @@ -67,16 +66,17 @@ If your pipeline uses packages that are not available publicly (e.g. packages th This command lists all packages that are installed on your machine, regardless of where they were installed from. -2. Run your pipeline with the following command-line option: + 1. Run your pipeline with the following command-line option: - --extra_package /path/to/package/package-name + --extra_package /path/to/package/package-name - where package-name is the package's tarball. If you have the `setup.py` for that - package then you can build the tarball with the following command: + where package-name is the package's tarball. You can build the package tarball using a command line tool called [build](https://setuptools.pypa.io/en/latest/userguide/quickstart.html#install-build). - python setup.py sdist + # Install build using pip + pip install --upgrade build + python -m build --sdist - See the [sdist documentation](https://docs.python.org/3/distutils/sourcedist.html) for more details on this command. + See the [build documentation](https://pypa-build.readthedocs.io/en/latest/index.html) for more details on this command. ## Multiple File Dependencies @@ -151,7 +151,7 @@ To use the `cloudpickle` pickler, supply the `--pickle_library=cloudpickle` pipe By default, global imports, functions, and variables defined in the main pipeline module are not saved during the serialization of a Beam job. Thus, one might encounter an unexpected `NameError` when running a `DoFn` on any remote runner. To resolve this, supply the main session content with the pipeline by setting the `--save_main_session` pipeline option. This will load the pickled state of the global namespace onto the Dataflow workers (if using `DataflowRunner`). -For example, see [Handling NameErrors](https://cloud.google.com/dataflow/docs/guides/common-errors#how-do-i-handle-nameerrors) to set the main session on the `DataflowRunner`. +For example, see [Handling NameErrors](https://cloud.google.com/dataflow/docs/guides/common-errors#name-error) to set the main session on the `DataflowRunner`. Managing the main session in Python SDK is only necessary when using `dill` pickler on any remote runner. Therefore, this issue will not occur in `DirectRunner`. @@ -160,3 +160,81 @@ Since serialization of the pipeline happens on the job submission, and deseriali To ensure this, Beam typically sets a very narrow supported version range for pickling libraries. If for whatever reason, users cannot use the version of `dill` or `cloudpickle` required by Beam, and choose to install a custom version, they must also ensure that they use the same custom version at runtime (e.g. in their custom container, or by specifying a pipeline dependency requirement). + +## Control the dependencies the pipeline uses {#control-dependencies} + +### Pipeline environments + +To run a Python pipeline on a remote runner, Apache Beam translates the pipeline into a [runner-independent representation](https://github.com/apache/beam/blob/master/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/beam_runner_api.proto) and submits it for execution. Translation happens in the **launch environment**. You can launch the pipeline from a Python virtual environment with the installed Beam SDK, or with tools like [Dataflow Flex Templates](https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates), [Notebook environments](https://cloud.google.com/dataflow/docs/guides/interactive-pipeline-development), [Apache Airflow](https://airflow.apache.org/), and more. + +The [**runtime environment**](https://beam.apache.org/documentation/runtime/environments/) is the Python environment that a runner uses during pipeline execution. This environment is where the pipeline code runs to when it performs data processing. The runtime environment includes Apache Beam and pipeline runtime dependencies. + +### Create reproducible environments {#create-reproducible-environments} + +You can use several tools to build reproducible Python environments: + +* **Use [requirements files](https://pip.pypa.io/en/stable/user_guide/#requirements-files).** After you install dependencies, generate the requirements file by using `pip freeze > requirements.txt`. To recreate an environment, install dependencies from the requirements.txt file by using `pip install -r requirements.txt`. + +* **Use [constraint files](https://pip.pypa.io/en/stable/user_guide/#constraints-files).** You can use the constraint list to restrict the installation of packages, allowing only specified versions. + +* **Use lock files.** Use dependency management tools like [PipEnv](https://pipenv.pypa.io/en/latest/), [Poetry](https://python-poetry.org/), and [pip-tools](https://github.com/jazzband/pip-tools) to specify top-level dependencies, to generate lock files of all transitive dependencies with pinned versions, and to create virtual environments from these lockfiles. + +* **Use Docker container images.** You can package the launch and runtime environment inside a Docker container image. If the image includes all necessary dependencies, then the environment only changes when a container image is rebuilt. + +Use version control for the configuration files that define the environment. + +### Make the pipeline runtime environment reproducible + +When a pipeline uses a reproducible runtime environment on a remote runner, the workers on the runner use the same dependencies each time the pipeline runs. A reproducible environment is immune to side-effects caused by releases of the pipeline's direct or transitive dependencies. It doesn’t require dependency resolution at runtime. + +You can create a reproducible runtime environment in the following ways: + +* Run your pipeline in a custom container image that has all dependencies for your pipeline. Use the `--sdk_container_image` pipeline option. + +* Supply an exhaustive list of the pipeline's dependencies in the `--requirements_file` pipeline option. Use the `--prebuild_sdk_container_engine` option to perform the runtime environment initialization sequence before the pipeline execution. If your dependencies don't change, reuse the prebuilt image by using the `--sdk_container_image` option. + +A self-contained runtime environment is usually reproducible. To check if the runtime environment is self-contained, restrict internet access to PyPI in the pipeline runtime. If you use the Dataflow Runner, see the documentation for the [`--no_use_public_ips`](https://cloud.google.com/dataflow/docs/guides/routes-firewall#turn_off_external_ip_address) pipeline option. + +If you need to recreate or upgrade the runtime environment, do so in a controlled way with visibility into changed dependencies: + +* Do not modify container images when they are in use by running pipelines. + +* Avoid using the tag `:latest` with your custom images. Tag your builds with a date or a unique identifier. If something goes wrong, using this type of tag might make it possible to revert the pipeline execution to a previously known working configuration and allow for an inspection of changes. + +* Consider storing the output of `pip freeze` or the contents of `requirements.txt` in the version control system. + +### Make the pipeline launch environment reproducible + +The launch environment runs the **production version** of the pipeline. While developing the pipeline locally, you might use a **development environment** that includes dependencies for development, such as Jupyter or Pylint. The launch environment for production pipelines might not need these additional dependencies. You can construct and maintain it separately from the development environment. + +To reduce side-effects on pipeline submissions, it is best to able to [recreate the launch environment in a reproducible manner](#create-reproducible-environments). + +[Dataflow Flex Templates](https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates) provide an example of a containerized, reproducible launch environment. + +To create reproducible installations of Beam into a clean virtual environment, use [requirements files](https://pip.pypa.io/en/stable/user_guide/#requirements-files) that list all Python dependencies included in Beam's default container images constraint files: + +``` +BEAM_VERSION=2.48.0 +PYTHON_VERSION=`python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')"` +pip install apache-beam==$BEAM_VERSION --constraint https://raw.githubusercontent.com/apache/beam/release-${BEAM_VERSION}/sdks/python/container/py${PY_VERSION}/base_image_requirements.txt +``` + +Use a constraint file to ensure that Beam dependencies in the launch environment match the versions in default Beam containers. A constraint file might also remove the need for dependency resolution at installation time. + +### Make the launch environment compatible with the runtime environment + +The launch environment translates the pipeline graph into a [runner-independent representation](https://github.com/apache/beam/blob/master/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/beam_runner_api.proto). This process involves serializing (or pickling) the code of the transforms. The serialized content is deserialized on the workers. If the runtime worker environment significantly differs from the launch environment, runtime errors might occur for the following reasons: + +* The Apache Beam version must match in the submission and runtime environments. Python major.minor versions must match as well. Otherwise, the pipeline might fail with errors like `Pipeline construction environment and pipeline runtime environment are not compatible`. On older SDK versions, the error might be reported as `SystemError: unknown opcode`. + +* Versions of `protobuf` in the submission and runtime environment need to match or be compatible. + +* Libraries used in the pipeline code might need to match. If serialized pipeline code has references to functions or modules that aren’t available on the workers, the pipeline might fail with `ModuleNotFound` or `AttributeError` exceptions on the remote runner. If you encounter such errors, make sure that the affected libraries are available on the remote worker, and check whether you need to [save the main session]( https://beam.apache.org/documentation/sdks/python-pipeline-dependencies/#pickling-and-managing-the-main-session). + +* The version of the pickling library used at submission time must match the version installed at runtime. To enforce this, Beam sets a tight bounds on the version of serializer libraries (dill and cloudpickle). You can force install a different version of `dill` or `cloudpickle` than required by Beam under the following conditions: + * You install the same version in submission and in the runtime environment. + * The chosen version works for your pipeline. + +To check whether the runtime environment matches the launch environment, inspect differences in the `pip freeze` output in both environments. Update to the latest version of Beam, because environment compatibility checks are included in newer SDK versions. + +Finally, you can use the same environment by launching the pipeline from the containerized environment that you use at runtime. [Dataflow Flex templates built from a custom container image](https://cloud.google.com/dataflow/docs/guides/templates/configuring-flex-templates#use_custom_container_images) offer this setup. In this scenario, you can recreate both launch and runtime environments in a reproducible manner. Because both containers are created from the same image, the launch and runtime environments are compatible with each other by default. diff --git a/website/www/site/content/en/documentation/sdks/python-unrecoverable-errors.md b/website/www/site/content/en/documentation/sdks/python-unrecoverable-errors.md new file mode 100644 index 0000000000000..4e5d94ce8a8db --- /dev/null +++ b/website/www/site/content/en/documentation/sdks/python-unrecoverable-errors.md @@ -0,0 +1,61 @@ +--- +type: languages +title: "Unrecoverable Errors in Beam Python" +--- + + +# Unrecoverable Errors in Beam Python + +## What is an Unrecoverable Error? + +An unrecoverable error is an issue at job start-up time that will +prevent a job from ever running successfully, usually due to some kind +of misconfiguration. Solving these issues when they occur is key to +successfully running a Beam Python pipeline. + +## Common Unrecoverable Errors + +### Job Submission/Runtime Python Version Mismatch + +If the Python version used for job submission does not match the +Python version used to build the worker container, the job will not +execute. Ensure that the Python version being used for job submission +and the container Python version match. + +### PIP Dependency Resolution Failures + +During worker start-up, dependencies are checked and installed in +the worker container before accepting work. If a pipeline requires +additional dependencies not already present in the runtime environment, +they are installed here. If there’s an issue during this process +(e.g. a dependency version cannot be found, or a worker cannot +connect to PyPI) the worker will fail and may try to restart +depending on the runner. Ensure that dependency versions provided in +your requirements.txt file exist and can be installed locally before +submitting jobs. + +### Dependency Version Mismatches + +When additional dependencies like `torch`, `transformers`, etc. are not +specified via a requirements_file or preinstalled in a custom container +then the worker might fail to deserialize (unpickle) the user code. +This can result in `ModuleNotFound` errors. If dependencies are installed +but their versions don't match the versions in submission environment, +pipeline might have `AttributeError` messages. + +Ensure that the required dependencies at runtime and in the submission +environment are the same along with their versions. For better visibility, +debug logs are added specifying the dependencies at both stages starting in +Beam 2.52.0. For more information, see: https://beam.apache.org/documentation/sdks/python-pipeline-dependencies/#control-dependencies \ No newline at end of file diff --git a/website/www/site/content/en/documentation/sdks/python.md b/website/www/site/content/en/documentation/sdks/python.md index 80c26c258d5ba..dc9f6a54d893a 100644 --- a/website/www/site/content/en/documentation/sdks/python.md +++ b/website/www/site/content/en/documentation/sdks/python.md @@ -59,3 +59,7 @@ see [Machine Learning](/documentation/sdks/python-machine-learning). ## Python multi-language pipelines quickstart Apache Beam lets you combine transforms written in any supported SDK language and use them in one multi-language pipeline. To learn how to create a multi-language pipeline using the Python SDK, see the [Python multi-language pipelines quickstart](/documentation/sdks/python-multi-language-pipelines). + +## Unrecoverable Errors in Beam Python + +Some common errors can occur during worker start-up and prevent jobs from starting. To learn about these errors and how to troubleshoot them in the Python SDK, see [Unrecoverable Errors in Beam Python](/documentation/sdks/python-unrecoverable-errors). \ No newline at end of file diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/mltransform.md b/website/www/site/content/en/documentation/transforms/python/elementwise/mltransform.md new file mode 100644 index 0000000000000..964d90f1510b5 --- /dev/null +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/mltransform.md @@ -0,0 +1,119 @@ +--- +title: "MLTransform" +--- + + +# MLTransform for data processing + +{{< localstorage language language-py >}} + + + + + + +
    + + {{< button-pydoc path="apache_beam.ml.transforms" class="MLTransform" >}} + +
    + + +Use `MLTransform` to apply common machine learning (ML) processing tasks on keyed data. Apache Beam provides ML data processing transformations that you can use with `MLTransform`. For the full list of available data +processing transformations, see the [tft.py file](https://github.com/apache/beam/blob/ab93fb1988051baac6c3b9dd1031f4d68bd9a149/sdks/python/apache_beam/ml/transforms/tft.py#L52) in GitHub. + + +To define a data processing transformation by using `MLTransform`, create instances of data processing transforms with `columns` as input parameters. The data in the specified `columns` is transformed and outputted to the `beam.Row` object. + +The following example demonstrates how to use `MLTransform` to normalize your data between 0 and 1 by using the minimum and maximum values from your entire dataset. `MLTransform` uses the `ScaleTo01` transformation. + + +``` +scale_to_z_score_transform = ScaleToZScore(columns=['x', 'y']) +with beam.Pipeline() as p: + (data | MLTransform(write_artifact_location=artifact_location).with_transform(scale_to_z_score_transform)) +``` + +In this example, `MLTransform` receives a value for `write_artifact_location`. `MLTransform` then uses this location value to write artifacts generated by the transform. To pass the data processing transform, you can use either the `with_transform` method of `MLTransform` or a list. + +``` +MLTransform(transforms=transforms, write_artifact_location=write_artifact_location) +``` + +The transforms passed to `MLTransform` are applied sequentially on the dataset. `MLTransform` expects a dictionary and returns a transformed row object with NumPy arrays. +## Examples + +The following examples demonstrate how to to create pipelines that use `MLTransform` to preprocess data. + +`MLTransform` can do a full pass on the dataset, which is useful when you need to transform a single element only after analyzing the entire dataset. +The first two examples require a full pass over the dataset to complete the data transformation. + +* For the `ComputeAndApplyVocabulary` transform, the transform needs access to all of the unique words in the dataset. +* For the `ScaleTo01` transform, the transform needs to know the minimum and maximum values in the dataset. + +### Example 1 + +This example creates a pipeline that uses `MLTransform` to scale data between 0 and 1. +The example takes a list of integers and converts them into the range of 0 to 1 using the transform `ScaleTo01`. + +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform.py" + class="notebook-skip" >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform.py" mltransform_scale_to_0_1 >}} +{{}} + +{{< paragraph class="notebook-skip" >}} +Output: +{{< /paragraph >}} +{{< highlight class="notebook-skip" >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py" mltransform_scale_to_0_1 >}} +{{< /highlight >}} + + +### Example 2 + +This example creates a pipeline that use `MLTransform` to compute vocabulary on the entire dataset and assign indices to each unique vocabulary item. +It takes a list of strings, computes vocabulary over the entire dataset, and then applies a unique index to each vocabulary item. + + +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform.py" + class="notebook-skip" >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform.py" mltransform_compute_and_apply_vocabulary >}} +{{}} + +{{< paragraph class="notebook-skip" >}} +Output: +{{< /paragraph >}} +{{< highlight class="notebook-skip" >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py" mltransform_compute_and_apply_vocab >}} +{{< /highlight >}} + + +### Example 3 + +This example creates a pipeline that uses `MLTransform` to compute vocabulary on the entire dataset and assign indices to each unique vocabulary item. This pipeline takes a single element as input instead of a list of elements. + + +{{< highlight language="py" file="sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform.py" + class="notebook-skip" >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform.py" mltransform_compute_and_apply_vocabulary_with_scalar >}} +{{}} + +{{< paragraph class="notebook-skip" >}} +Output: +{{< /paragraph >}} +{{< highlight class="notebook-skip" >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py" mltransform_compute_and_apply_vocabulary_with_scalar >}} +{{< /highlight >}} + diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/runinference.md b/website/www/site/content/en/documentation/transforms/python/elementwise/runinference.md index 39db1aa68d3e3..369b74714424d 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/runinference.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/runinference.md @@ -29,11 +29,9 @@ limitations under the License. -Uses models to do local and remote inference. A `RunInference` transform performs inference on a `PCollection` of examples using a machine learning (ML) model. The transform outputs a `PCollection` that contains the input examples and output predictions. +Uses models to do local and remote inference. A `RunInference` transform performs inference on a `PCollection` of examples using a machine learning (ML) model. The transform outputs a `PCollection` that contains the input examples and output predictions. Avaliable in Apache Beam 2.40.0 and later versions. -You must have Apache Beam 2.40.0 or later installed to run these pipelines. - -See more [RunInference API pipeline examples](https://github.com/apache/beam/tree/master/sdks/python/apache_beam/examples/inference). +For more information, read about Beam RunInference APIs on [Machine Learning with Python](https://beam.apache.org/documentation/sdks/python-machine-learning) page and explore [RunInference API pipeline examples](https://github.com/apache/beam/tree/master/sdks/python/apache_beam/examples/inference). ## Examples diff --git a/website/www/site/content/en/documentation/transforms/python/overview.md b/website/www/site/content/en/documentation/transforms/python/overview.md index d30af75352b28..666b7c95f0803 100644 --- a/website/www/site/content/en/documentation/transforms/python/overview.md +++ b/website/www/site/content/en/documentation/transforms/python/overview.md @@ -27,6 +27,7 @@ limitations under the License. KeysExtracts the key from each element in a collection of key-value pairs. KvSwapSwaps the key and value of each element in a collection of key-value pairs. MapApplies a function to every element in the input and outputs the result. + MLTransformApplies data processing transforms to the dataset. ParDoThe most-general mechanism for applying a user-defined DoFn to every element in the input collection. PartitionRoutes each input element to a specific output collection based on some partition @@ -39,6 +40,7 @@ limitations under the License. and updates the implicit timestamp associated with each input. Note that it is only safe to adjust timestamps forwards. ValuesExtracts the value from each element in a collection of key-value pairs. + ## Aggregation diff --git a/website/www/site/content/en/get-started/an-interactive-overview-of-beam.md b/website/www/site/content/en/get-started/an-interactive-overview-of-beam.md new file mode 100644 index 0000000000000..af56d31a61d6a --- /dev/null +++ b/website/www/site/content/en/get-started/an-interactive-overview-of-beam.md @@ -0,0 +1,96 @@ +--- +title: "An Interactive Overview of Beam" +--- + + + +# An Interactive Overview of Beam + +Here you can find a collection of the interactive notebooks available for Apache Beam, which are hosted in +[Colab](https://colab.research.google.com). +The notebooks allow you to interactively play with the code and see how your changes affect the pipeline. +You don't need to install anything or modify your computer in any way to use these notebooks. + +You can also [try an Apache Beam pipeline](/get-started/try-apache-beam) using the Java, Python, and Go SDKs. + +## Get started + +### Learn the basics + +In this notebook we go through the basics of what is Apache Beam and how to get started. +We learn what is a data pipeline, a PCollection, a PTransform, as well as some basic transforms like `Map`, `FlatMap`, `Filter`, `Combine`, and `GroupByKey`. + +{{< button-colab url="https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/interactive-overview/getting-started.ipynb" >}} + +### Reading and writing data + +In this notebook we go through some examples on how to read and write data to and from different data formats. +We introduce the built-in `ReadFromText` and `WriteToText` transforms. +We also see how we can read from CSV files, read from a SQLite database, write fixed-sized batches of elements, and write windows of elements. + +{{< button-colab url="https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/interactive-overview/reading-and-writing-data.ipynb" >}} + +### Windowing + +In this notebook we go through how to aggregate data based on time intervals, or in streaming pipelines. +We introduce the `GlobalWindow`, `FixedWindows`, `SlidingWindows`, and `Sessions`. + +{{< button-colab url="https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/interactive-overview/windowing.ipynb" >}} + +### DataFrames + +Beam DataFrames provide a pandas-like [DataFrame](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html) +API to declare Beam pipelines. +To learn more about Beam DataFrames, take a look at the +[Beam DataFrames overview](/documentation/dsls/dataframes/overview) page. + +{{< button-colab url="https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/interactive-overview/dataframes.ipynb" >}} + +## Transforms + +Check the [Python transform catalog](/documentation/transforms/python/overview/) +for a complete list of the available transforms. + +### Element-wise transforms + +#### Map + +Applies a simple one-to-one mapping function over each element in the collection. + +{{< button-colab url="https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/documentation/transforms/python/elementwise/map-py.ipynb" >}} + +#### FlatMap + +Applies a simple one-to-many mapping function over each element in the collection. The many elements are flattened into the resulting collection. + +{{< button-colab url="https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/documentation/transforms/python/elementwise/flatmap-py.ipynb" >}} + +#### Filter + +Given a predicate, filter out all elements that don’t satisfy that predicate. + +{{< button-colab url="https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/documentation/transforms/python/elementwise/filter-py.ipynb" >}} + +#### Partition + +Separates elements in a collection into multiple output collections. + +{{< button-colab url="https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/documentation/transforms/python/elementwise/partition-py.ipynb" >}} + +#### ParDo + +A transform for generic parallel processing. It's recommended to use `Map`, `FlatMap`, `Filter` or other more specific transforms when possible. + +{{< button-colab url="https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/documentation/transforms/python/elementwise/pardo-py.ipynb" >}} diff --git a/website/www/site/content/en/get-started/beam-overview.md b/website/www/site/content/en/get-started/beam-overview.md index 517a294b9118d..5a8fcd3b917c1 100644 --- a/website/www/site/content/en/get-started/beam-overview.md +++ b/website/www/site/content/en/get-started/beam-overview.md @@ -29,7 +29,6 @@ Beam is particularly useful for [embarrassingly parallel](https://en.wikipedia.o Learner Graph - ## Apache Beam SDKs The Beam SDKs provide a unified programming model that can represent and transform data sets of any size, whether the input is a finite data set from a batch data source, or an infinite data set from a streaming data source. The Beam SDKs use the same classes to represent both bounded and unbounded data, and the same transforms to operate on that data. You use the Beam SDK of your choice to build a program that defines your data processing pipeline. @@ -66,7 +65,7 @@ Get started using Beam for your data processing tasks. > If you already know [Apache Spark](https://spark.apache.org/), > check our [Getting started from Apache Spark](/get-started/from-spark) page. -1. Take the [Tour of Beam](/get-started/tour-of-beam) as an online interactive learning experience. +1. Take the [Tour of Beam](https://tour.beam.apache.org/) as an online interactive learning experience. 1. Follow the Quickstart for the [Java SDK](/get-started/quickstart-java), the [Python SDK](/get-started/quickstart-py), or the [Go SDK](/get-started/quickstart-go). @@ -76,6 +75,8 @@ Get started using Beam for your data processing tasks. 1. Dive into the [Documentation](/documentation/) section for in-depth concepts and reference materials for the Beam model, SDKs, and runners. +1. Dive into the [cookbook examples](https://github.com/GoogleCloudPlatform/dataflow-cookbook) for learning how to run Beam on Dataflow. + ## Contribute Beam is an Apache Software Foundation project, available under the Apache v2 license. Beam is an open source community and contributions are greatly appreciated! If you'd like to contribute, please see the [Contribute](/contribute/) section. diff --git a/website/www/site/content/en/get-started/downloads.md b/website/www/site/content/en/get-started/downloads.md index 35acb147f1100..b564a5801cd8f 100644 --- a/website/www/site/content/en/get-started/downloads.md +++ b/website/www/site/content/en/get-started/downloads.md @@ -96,10 +96,34 @@ versions denoted `0.x.y`. ## Releases +### 2.51.0 (2023-10-11) +Official [source code download](https://downloads.apache.org/beam/2.51.0/apache-beam-2.51.0-source-release.zip). +[SHA-512](https://downloads.apache.org/beam/2.51.0/apache-beam-2.51.0-source-release.zip.sha512). +[signature](https://downloads.apache.org/beam/2.51.0/apache-beam-2.51.0-source-release.zip.asc). + +[Release notes](https://github.com/apache/beam/releases/tag/v2.51.0) +[Blog post](/blog/beam-2.51.0). + +### 2.50.0 (2023-08-30) +Official [source code download](https://archive.apache.org/beam/2.50.0/apache-beam-2.50.0-source-release.zip). +[SHA-512](https://archive.apache.org/beam/2.50.0/apache-beam-2.50.0-source-release.zip.sha512). +[signature](https://archive.apache.org/beam/2.50.0/apache-beam-2.50.0-source-release.zip.asc). + +[Release notes](https://github.com/apache/beam/releases/tag/v2.50.0) +[Blog post](/blog/beam-2.50.0). + +### 2.49.0 (2023-07-17) +Official [source code download](https://archive.apache.org/dist/beam/2.49.0/apache-beam-2.49.0-source-release.zip). +[SHA-512](https://archive.apache.org/dist/beam/2.49.0/apache-beam-2.49.0-source-release.zip.sha512). +[signature](https://archive.apache.org/dist/beam/2.49.0/apache-beam-2.49.0-source-release.zip.asc). + +[Release notes](https://github.com/apache/beam/releases/tag/v2.49.0) +[Blog post](/blog/beam-2.49.0). + ### 2.48.0 (2023-05-31) -Official [source code download](https://downloads.apache.org/beam/2.48.0/apache-beam-2.48.0-source-release.zip). -[SHA-512](https://downloads.apache.org/beam/2.48.0/apache-beam-2.48.0-source-release.zip.sha512). -[signature](https://downloads.apache.org/beam/2.48.0/apache-beam-2.48.0-source-release.zip.asc). +Official [source code download](https://archive.apache.org/dist/beam/2.48.0/apache-beam-2.48.0-source-release.zip). +[SHA-512](https://archive.apache.org/dist/beam/2.48.0/apache-beam-2.48.0-source-release.zip.sha512). +[signature](https://archive.apache.org/dist/beam/2.48.0/apache-beam-2.48.0-source-release.zip.asc). [Release notes](https://github.com/apache/beam/releases/tag/v2.48.0) [Blog post](/blog/beam-2.48.0). @@ -113,9 +137,9 @@ Official [source code download](https://archive.apache.org/dist/beam/2.47.0/apac [Blog post](/blog/beam-2.47.0). ### 2.46.0 (2023-03-10) -Official [source code download](https://archive.apache.org/dist/beam/2.46.0/apache-beam-2.45.0-source-release.zip). -[SHA-512](https://archive.apache.org/dist/beam/2.45.0/apache-beam-2.46.0-source-release.zip.sha512). -[signature](https://archive.apache.org/dist/beam/2.45.0/apache-beam-2.46.0-source-release.zip.asc). +Official [source code download](https://archive.apache.org/dist/beam/2.46.0/apache-beam-2.46.0-source-release.zip). +[SHA-512](https://archive.apache.org/dist/beam/2.46.0/apache-beam-2.46.0-source-release.zip.sha512). +[signature](https://archive.apache.org/dist/beam/2.46.0/apache-beam-2.46.0-source-release.zip.asc). [Release notes](https://github.com/apache/beam/releases/tag/v2.46.0) [Blog post](/blog/beam-2.46.0). @@ -130,16 +154,16 @@ Official [source code download](https://archive.apache.org/dist/beam/2.45.0/apac ### 2.44.0 (2023-01-12) Official [source code download](https://archive.apache.org/dist/beam/2.44.0/apache-beam-2.44.0-source-release.zip). -[SHA-512](https://archive.apache.org/dist/beam/2.43.0/apache-beam-2.44.0-source-release.zip.sha512). -[signature](https://archive.apache.org/dist/beam/2.43.0/apache-beam-2.44.0-source-release.zip.asc). +[SHA-512](https://archive.apache.org/dist/beam/2.44.0/apache-beam-2.44.0-source-release.zip.sha512). +[signature](https://archive.apache.org/dist/beam/2.44.0/apache-beam-2.44.0-source-release.zip.asc). [Release notes](https://github.com/apache/beam/releases/tag/v2.44.0) [Blog post](/blog/beam-2.44.0). ### 2.43.0 (2022-11-17) -Official [source code download](https://archive.apache.org/beam/2.43.0/apache-beam-2.43.0-source-release.zip). -[SHA-512](https://archive.apache.org/beam/2.43.0/apache-beam-2.43.0-source-release.zip.sha512). -[signature](https://archive.apache.org/beam/2.43.0/apache-beam-2.43.0-source-release.zip.asc). +Official [source code download](https://archive.apache.org/dist/beam/2.43.0/apache-beam-2.43.0-source-release.zip). +[SHA-512](https://archive.apache.org/dist/beam/2.43.0/apache-beam-2.43.0-source-release.zip.sha512). +[signature](https://archive.apache.org/dist/beam/2.43.0/apache-beam-2.43.0-source-release.zip.asc). [Release notes](https://github.com/apache/beam/releases/tag/v2.43.0) [Blog post](/blog/beam-2.43.0). diff --git a/website/www/site/content/en/get-started/quickstart-go.md b/website/www/site/content/en/get-started/quickstart-go.md index c2504205aa6c5..2f0bad49659c6 100644 --- a/website/www/site/content/en/get-started/quickstart-go.md +++ b/website/www/site/content/en/get-started/quickstart-go.md @@ -25,21 +25,13 @@ If you're interested in contributing to the Apache Beam Go codebase, see the [Co ## Set up your environment -The Beam SDK for Go requires `go` version 1.19 or newer. It can be downloaded [here](https://golang.org/). Check that you have version 1.19 by running: +The Beam SDK for Go requires `go` version 1.20 or newer. It can be downloaded [here](https://golang.org/). Check what go version you have by running: {{< highlight >}} $ go version {{< /highlight >}} -## Get the SDK and the examples - -The easiest way to obtain the Apache Beam Go SDK is via `go get`: - -{{< highlight >}} -$ go get -u github.com/apache/beam/sdks/v2/go/pkg/beam -{{< /highlight >}} - -For development of the Go SDK itself, see [BUILD.md](https://github.com/apache/beam/blob/master/sdks/go/BUILD.md) for details. +If you are unfamiliar with Go, see the [Get Started With Go Tutorial](https://go.dev/doc/tutorial/getting-started). ## Run wordcount @@ -51,22 +43,18 @@ required arguments described in the examples. For example, to run `wordcount`, run: {{< runner direct >}} -$ go install github.com/apache/beam/sdks/v2/go/examples/wordcount -$ wordcount --input --output counts +$ go run github.com/apache/beam/sdks/v2/go/examples/wordcount@latest --input "gs://apache-beam-samples/shakespeare/kinglear.txt" --output counts +$ less counts {{< /runner >}} {{< runner dataflow >}} -$ go install github.com/apache/beam/sdks/v2/go/examples/wordcount -# As part of the initial setup, for non linux users - install package unix before run -$ go get -u golang.org/x/sys/unix -$ wordcount --input gs://dataflow-samples/shakespeare/kinglear.txt \ +$ go run github.com/apache/beam/sdks/v2/go/examples/wordcount@latest --input gs://dataflow-samples/shakespeare/kinglear.txt \ --output gs:///counts \ --runner dataflow \ --project your-gcp-project \ --region your-gcp-region \ --temp_location gs:///tmp/ \ - --staging_location gs:///binaries/ \ - --worker_harness_container_image=apache/beam_go_sdk:latest + --staging_location gs:///binaries/ {{< /runner >}} {{< runner spark >}} @@ -75,8 +63,7 @@ $ wordcount --input gs://dataflow-samples/shakespeare/kinglear.txt \ $ ./gradlew :runners:spark:3:job-server:runShadow -PsparkMasterUrl=spark://localhost:7077 # In a separate terminal, run: -$ go install github.com/apache/beam/sdks/v2/go/examples/wordcount -$ wordcount --input \ +$ go run github.com/apache/beam/sdks/v2/go/examples/wordcount@latest --input \ --output counts \ --runner spark \ --endpoint localhost:8099 @@ -87,6 +74,7 @@ $ wordcount --input \ * Learn more about the [Beam SDK for Go](/documentation/sdks/go/) and look through the [godoc](https://pkg.go.dev/github.com/apache/beam/sdks/v2/go/pkg/beam). * Walk through these WordCount examples in the [WordCount Example Walkthrough](/get-started/wordcount-example). +* Clone the [Beam Go starter project](https://github.com/apache/beam-starter-go). * Take a self-paced tour through our [Learning Resources](/documentation/resources/learning-resources). * Dive in to some of our favorite [Videos and Podcasts](/get-started/resources/videos-and-podcasts). * Join the Beam [users@](/community/contact-us) mailing list. diff --git a/website/www/site/content/en/get-started/quickstart-py.md b/website/www/site/content/en/get-started/quickstart-py.md index bd6fd3eaa0db0..9376606b916e2 100644 --- a/website/www/site/content/en/get-started/quickstart-py.md +++ b/website/www/site/content/en/get-started/quickstart-py.md @@ -23,7 +23,7 @@ If you're interested in contributing to the Apache Beam Python codebase, see the {{< toc >}} -The Python SDK supports Python 3.7, 3.8, 3.9 and 3.10. Beam 2.38.0 was the last release with support for Python 3.6. +The Python SDK supports Python 3.8, 3.9, 3.10 and 3.11. Beam 2.48.0 was the last release with support for Python 3.7. ## Set up your environment @@ -142,6 +142,7 @@ sequentially in the format `counts-0000-of-0001`. * Learn more about the [Beam SDK for Python](/documentation/sdks/python/) and look through the [Python SDK API reference](https://beam.apache.org/releases/pydoc). +* Get [An Interactive Overview of Beam](/get-started/an-interactive-overview-of-beam) * Walk through these WordCount examples in the [WordCount Example Walkthrough](/get-started/wordcount-example). * Take a self-paced tour through our [Learning Resources](/documentation/resources/learning-resources). * Dive in to some of our favorite [Videos and Podcasts](/get-started/resources/videos-and-podcasts). diff --git a/website/www/site/content/en/get-started/tour-of-beam.md b/website/www/site/content/en/get-started/tour-of-beam.md index 80dcb7eb21def..0deca6be0b91a 100644 --- a/website/www/site/content/en/get-started/tour-of-beam.md +++ b/website/www/site/content/en/get-started/tour-of-beam.md @@ -1,5 +1,5 @@ --- -title: "Tour of Beam" +title: "The Tour of Beam" --- -# Tour of Beam +# The Tour of Beam -Here you can find a collection of the interactive notebooks available for Apache Beam, which are hosted in -[Colab](https://colab.research.google.com). -The notebooks allow you to interactively play with the code and see how your changes affect the pipeline. -You don't need to install anything or modify your computer in any way to use these notebooks. - -You can also [try an Apache Beam pipeline](/get-started/try-apache-beam) using the Java, Python, and Go SDKs. - -## Get started - -### Learn the basics - -In this notebook we go through the basics of what is Apache Beam and how to get started. -We learn what is a data pipeline, a PCollection, a PTransform, as well as some basic transforms like `Map`, `FlatMap`, `Filter`, `Combine`, and `GroupByKey`. - -{{< button-colab url="https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/tour-of-beam/getting-started.ipynb" >}} - -### Reading and writing data - -In this notebook we go through some examples on how to read and write data to and from different data formats. -We introduce the built-in `ReadFromText` and `WriteToText` transforms. -We also see how we can read from CSV files, read from a SQLite database, write fixed-sized batches of elements, and write windows of elements. - -{{< button-colab url="https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/tour-of-beam/reading-and-writing-data.ipynb" >}} - -### Windowing - -In this notebook we go through how to aggregate data based on time intervals, or in streaming pipelines. -We introduce the `GlobalWindow`, `FixedWindows`, `SlidingWindows`, and `Sessions`. - -{{< button-colab url="https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/tour-of-beam/windowing.ipynb" >}} - -### DataFrames - -Beam DataFrames provide a pandas-like [DataFrame](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html) -API to declare Beam pipelines. -To learn more about Beam DataFrames, take a look at the -[Beam DataFrames overview](/documentation/dsls/dataframes/overview) page. - -{{< button-colab url="https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/tour-of-beam/dataframes.ipynb" >}} - -## Transforms - -Check the [Python transform catalog](/documentation/transforms/python/overview/) -for a complete list of the available transforms. - -### Element-wise transforms - -#### Map - -Applies a simple one-to-one mapping function over each element in the collection. - -{{< button-colab url="https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/documentation/transforms/python/elementwise/map-py.ipynb" >}} - -#### FlatMap - -Applies a simple one-to-many mapping function over each element in the collection. The many elements are flattened into the resulting collection. - -{{< button-colab url="https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/documentation/transforms/python/elementwise/flatmap-py.ipynb" >}} - -#### Filter - -Given a predicate, filter out all elements that don’t satisfy that predicate. - -{{< button-colab url="https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/documentation/transforms/python/elementwise/filter-py.ipynb" >}} - -#### Partition - -Separates elements in a collection into multiple output collections. - -{{< button-colab url="https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/documentation/transforms/python/elementwise/partition-py.ipynb" >}} - -#### ParDo - -A transform for generic parallel processing. It's recommended to use `Map`, `FlatMap`, `Filter` or other more specific transforms when possible. - -{{< button-colab url="https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/documentation/transforms/python/elementwise/pardo-py.ipynb" >}} +The "Tour of Beam" is an interactive way of learning to write Beam code with a sandbox, where you can write and run pipelines while walking through various concepts. Please [click here](https://tour.beam.apache.org/) to try it out. \ No newline at end of file diff --git a/website/www/site/content/en/get-started/try-beam-playground.md b/website/www/site/content/en/get-started/try-beam-playground.md index 731d54cf5d43c..cd608d0b2ef1a 100644 --- a/website/www/site/content/en/get-started/try-beam-playground.md +++ b/website/www/site/content/en/get-started/try-beam-playground.md @@ -1,5 +1,5 @@ --- -title: "Try Beam Playground (Beta)" +title: "Try Beam Playground" --- -# Try Beam Playground (Beta) +# Try Beam Playground Beam Playground is an interactive environment to try out Beam transforms and examples without having to install Apache Beam in your environment. diff --git a/website/www/site/content/en/performance/_index.md b/website/www/site/content/en/performance/_index.md new file mode 100644 index 0000000000000..f821b0f250842 --- /dev/null +++ b/website/www/site/content/en/performance/_index.md @@ -0,0 +1,40 @@ +--- +title: "Beam IO Performance" +--- + + + +# Beam IO Performance + +Various Beam pipelines measure characteristics of reading from and writing to +various IOs. + +# Available Metrics + +Various metrics were gathered using the Beam SDK +[Metrics API](/documentation/programming-guide/#metrics) +from a pipeline Job running on [Dataflow](/documentation/runners/dataflow/). + +See the [glossary](/performance/glossary) for a list of the metrics and their +definition. + +# Measured Beam IOs + +See the following pages for performance measures recorded when reading from and +writing to various Beam IOs. + +- [BigQuery](/performance/bigquery) +- [BigTable](/performance/bigtable) +- [TextIO](/performance/textio) \ No newline at end of file diff --git a/website/www/site/content/en/performance/bigquery/_index.md b/website/www/site/content/en/performance/bigquery/_index.md new file mode 100644 index 0000000000000..5d46e11809709 --- /dev/null +++ b/website/www/site/content/en/performance/bigquery/_index.md @@ -0,0 +1,50 @@ +--- +title: "BigQuery Performance" +--- + + + +# BigQuery Performance + +The following graphs show various metrics when reading from and writing to +BigQuery. See the [glossary](/performance/glossary) for definitions. + +## Read + +### What is the estimated cost to read from BigQuery? + +{{< performance_looks io="bigquery" read_or_write="read" section="test_name" >}} + +### How has various metrics changed when reading from BigQuery for different Beam SDK versions? + +{{< performance_looks io="bigquery" read_or_write="read" section="version" >}} + +### How has various metrics changed over time when reading from BigQuery? + +{{< performance_looks io="bigquery" read_or_write="read" section="date" >}} + +## Write + +### What is the estimated cost to write to BigQuery? + +{{< performance_looks io="bigquery" read_or_write="write" section="test_name" >}} + +### How has various metrics changed when writing to BigQuery for different Beam SDK versions? + +{{< performance_looks io="bigquery" read_or_write="write" section="version" >}} + +### How has various metrics changed over time when writing to BigQuery? + +{{< performance_looks io="bigquery" read_or_write="write" section="date" >}} diff --git a/website/www/site/content/en/performance/bigtable/_index.md b/website/www/site/content/en/performance/bigtable/_index.md new file mode 100644 index 0000000000000..d394528a05faf --- /dev/null +++ b/website/www/site/content/en/performance/bigtable/_index.md @@ -0,0 +1,50 @@ +--- +title: "BigTable Performance" +--- + + + +# BigTable Performance + +The following graphs show various metrics when reading from and writing to +BigTable. See the [glossary](/performance/glossary) for definitions. + +## Read + +### What is the estimated cost to read from BigTable? + +{{< performance_looks io="bigquery" read_or_write="read" section="test_name" >}} + +### How has various metrics changed when reading from BigTable for different Beam SDK versions? + +{{< performance_looks io="bigquery" read_or_write="read" section="version" >}} + +### How has various metrics changed over time when reading from BigTable? + +{{< performance_looks io="bigquery" read_or_write="read" section="date" >}} + +## Write + +### What is the estimated cost to write to BigTable? + +{{< performance_looks io="bigquery" read_or_write="write" section="test_name" >}} + +### How has various metrics changed when writing to BigTable for different Beam SDK versions? + +{{< performance_looks io="bigquery" read_or_write="write" section="version" >}} + +### How has various metrics changed over time when writing to BigTable? + +{{< performance_looks io="bigquery" read_or_write="write" section="date" >}} diff --git a/website/www/site/content/en/performance/glossary/_index.md b/website/www/site/content/en/performance/glossary/_index.md new file mode 100644 index 0000000000000..6741d107ad33f --- /dev/null +++ b/website/www/site/content/en/performance/glossary/_index.md @@ -0,0 +1,47 @@ +--- +title: "Performance Glossary" +--- + + + +# Metrics Glossary + +The metrics glossary defines various metrics presented in the visualizations, +measured using the [Beam Metrics API](/documentation/programming-guide/#metrics) +from a pipeline Job running on [Dataflow](/documentation/runners/dataflow/). + +## AvgInputThroughputBytesPerSec + +The mean input throughput of the pipeline Job measured in bytes per second. + +## AvgInputThroughputElementsPerSec + +The mean elements input throughput per second of the pipeline Job. + +## AvgOutputThroughputBytesPerSec + +The mean output throughput of the pipeline Job measured in bytes per second. + +## AvgOutputThroughputElementsPerSec + +The mean elements output throughput per second of the pipeline Job. + +## EstimatedCost + +The estimated cost of the pipeline Job. + +## RunTime + +The time it took for the pipeline Job to run measured in seconds. diff --git a/website/www/site/content/en/performance/textio/_index.md b/website/www/site/content/en/performance/textio/_index.md new file mode 100644 index 0000000000000..97c740ad057eb --- /dev/null +++ b/website/www/site/content/en/performance/textio/_index.md @@ -0,0 +1,50 @@ +--- +title: "TextIO Performance" +--- + + + +# TextIO Performance + +The following graphs show various metrics when reading from or writing to Google Cloud Storage using +TextIO. See the [glossary](/performance/glossary) for definitions. + +## Read + +### What is the estimated cost of reading from Google Cloud Storage using TextIO? + +{{< performance_looks io="textio" read_or_write="read" section="test_name" >}} + +### How has various metrics changed when reading from Google Cloud Storage using TextIO for different Beam SDK versions? + +{{< performance_looks io="textio" read_or_write="read" section="version" >}} + +### How has various metrics changed over time when reading from Google Cloud Storage using TextIO? + +{{< performance_looks io="textio" read_or_write="read" section="date" >}} + +## Write + +### What is the estimated cost of writing to Google Cloud Storage using TextIO? + +{{< performance_looks io="textio" read_or_write="write" section="test_name" >}} + +### How has various metrics changed when writing to Google Cloud Storage using TextIO for different Beam SDK versions? + +{{< performance_looks io="textio" read_or_write="write" section="version" >}} + +### How has various metrics changed over time when writing to Google Cloud Storage using TextIO? + +{{< performance_looks io="textio" read_or_write="write" section="date" >}} diff --git a/website/www/site/data/authors.yml b/website/www/site/data/authors.yml index 36931d3ee87d3..2776132cf586b 100644 --- a/website/www/site/data/authors.yml +++ b/website/www/site/data/authors.yml @@ -28,6 +28,9 @@ angoenka: anton: name: Anton Kedin email: anton@apache.org +bvolpato: + name: Bruno Volpato + email: bvolpato@google.com ccy: name: Charles Chen email: ccy@apache.org @@ -262,4 +265,13 @@ sysede: linkedin: desyse riteshghorse: name: Ritesh Ghorse - email: riteshghorse@apache.org \ No newline at end of file + email: riteshghorse@apache.org +yhu: + name: Yi Hu + email: yhu@apache.org +pabs: + name: Pablo Rodriguez Defino + email: prodriguezdefino@gmail.com +namitasharma: + name: Namita Sharma + email: namitasharma@google.com diff --git a/website/www/site/data/capability_matrix.yaml b/website/www/site/data/capability_matrix.yaml index d8455eb2edcde..dcbbca438b6e4 100644 --- a/website/www/site/data/capability_matrix.yaml +++ b/website/www/site/data/capability_matrix.yaml @@ -671,9 +671,9 @@ capability-matrix: l2: l3: "" - class: flink - l1: "Yes" + l1: "Partially" l2: - l3: "" + l3: "Support is either incomplete or broken on portable Flink Runner (#19637)" - class: spark-rdd l1: l2: @@ -714,8 +714,8 @@ capability-matrix: l2: fully supported l3: "" - class: flink - l1: "Partially" - l2: Only portable Flink Runner supports this. + l1: "No" + l2: l3: "" - class: spark-rdd l1: @@ -757,7 +757,7 @@ capability-matrix: l2: l3: "" - class: flink - l1: "Yes" + l1: "Partially" l2: l3: "" - class: spark-rdd @@ -843,8 +843,8 @@ capability-matrix: l2: Only Dataflow Runner V2 supports this. l3: "" - class: flink - l1: "Partially" - l2: Only portable Flink Runner supports this with checkpointing enabled. + l1: "No" + l2: l3: "" - class: spark-rdd l1: diff --git a/website/www/site/data/capability_matrix_snapshot.yaml b/website/www/site/data/capability_matrix_snapshot.yaml index 958f969a8384a..bdff2e814d2c1 100644 --- a/website/www/site/data/capability_matrix_snapshot.yaml +++ b/website/www/site/data/capability_matrix_snapshot.yaml @@ -689,9 +689,9 @@ capability-matrix-snapshot: l2: l3: "" - class: flink - l1: "Yes" + l1: "Partially" l2: - l3: "" + l3: "Support is either incomplete or broken on portable Flink Runner (#19637)" - class: spark l1: l2: @@ -707,8 +707,8 @@ capability-matrix-snapshot: l2: fully supported l3: "" - class: flink - l1: "Partially" - l2: Only portable Flink Runner supports this. + l1: "No" + l2: l3: "" - class: spark l1: @@ -725,7 +725,7 @@ capability-matrix-snapshot: l2: l3: "" - class: flink - l1: "Yes" + l1: "Partially" l2: l3: "" - class: spark @@ -743,7 +743,7 @@ capability-matrix-snapshot: l2: l3: "" - class: flink - l1: "No" + l1: l2: l3: "" - class: spark @@ -761,8 +761,8 @@ capability-matrix-snapshot: l2: Only Dataflow Runner V2 supports this. l3: "" - class: flink - l1: "Partially" - l2: Only portable Flink Runner supports this with checkpointing enabled. + l1: "No" + l2: l3: "" - class: spark l1: diff --git a/website/www/site/data/en/quotes.yaml b/website/www/site/data/en/quotes.yaml index c72cbe3d25abc..3a5225b3f29a4 100644 --- a/website/www/site/data/en/quotes.yaml +++ b/website/www/site/data/en/quotes.yaml @@ -11,6 +11,21 @@ # limitations under the License. #Cards with quotes will be displayed by the order listed, e.g., first card will display the first quote +- text: Apache Beam fuels LinkedIn's streaming infrastructure, processing 4 trillion events daily through 3K+ pipelines in near-real time. Beam enabled unified pipelines, yielding 2x cost savings and remarkable improvements for many use cases. + icon: icons/quote-icon.svg + logoUrl: images/logos/powered-by/linkedin.png + linkUrl: case-studies/linkedin/index.html + linkText: Learn more +- text: With Apache Beam, OCTO accelerated the migration of one of France’s largest grocery retailers to streaming processing for transactional data, achieving 5x reduced infrastructure costs and 4x improved performance. + icon: icons/quote-icon.svg + logoUrl: images/logos/powered-by/octo.png + linkUrl: case-studies/octo/index.html + linkText: Learn more +- text: HSBC leveraged Apache Beam as a computational platform and a risk engine that enabled 100x scaling, 2x faster performance, and simplified data distribution for assessing and managing XVA and counterparty credit risk at HSBC’s global scale. + icon: icons/quote-icon.svg + logoUrl: images/logos/powered-by/hsbc.png + linkUrl: case-studies/hsbc/index.html + linkText: Learn more - text: Apache Beam supports Project Shield's mission to protect freedom of speech and make the web a safer space by enabling ~2x streaming efficiency at >10,000 QPS and real-time visibility into attack data for their >3K customers. icon: icons/quote-icon.svg logoUrl: images/logos/powered-by/project_shield.png diff --git a/website/www/site/data/performance.yaml b/website/www/site/data/performance.yaml new file mode 100644 index 0000000000000..dc375811c8337 --- /dev/null +++ b/website/www/site/data/performance.yaml @@ -0,0 +1,108 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +host: https://storage.googleapis.com +path: public_looker_explores_us_a3853f40 +looks: + bigquery: + read: + folder: 30 + test_name: + - id: nwQxvsnQFdBPTk27pZYxjcGNm2rRfNJk + title: Runtime and Estimated Cost by Test Name + date: + - id: 7QKbMmgT5NgPH6RsDfVQmKMdjJysS37x + title: AvgOutputThroughputBytesPerSec by Date + - id: hSqCyCkHkh4whNZSpdVFwB8ksPWrbScb + title: AvgOutputThroughputElementsPerSec by Date + version: + - id: SghzxqjMb5QQwgZn7yHw4MnpGXBgXQft + title: AvgOutputThroughputBytesPerSec by Version + - id: QT6X4GTCnxxykSZ7tpXyBfBFCKMgdzs7 + title: AvgOutputThroughputElementsPerSec by Version + write: + folder: 31 + test_name: + - id: sHyRfwCfGCPqSwsTWXSsxfbHKH5hXjzc + title: Write BigQuery RunTime and EstimatedCost + date: + - id: p2qnxS58WdWdZTzJhXfSbkhxGyDXMYfY + title: AvgInputThroughputBytesPerSec by Date + - id: Bmg6sv8XPtcn3cGtgPtKP26Zj4cyd2XK + title: AvgInputThroughputElementsPerSec by Date + version: + - id: Q6QvktyfSFrSPq4TSDgQQyV9jttjwQJp + title: AvgInputThroughputBytesPerSec by Version + - id: ktfyZ7Th8yRFFwWgb4WSxngBpmSfz8xh + title: AvgInputThroughputElementsPerSec by Version + bigtable: + read: + folder: 32 + test_name: + - id: YQ2W3wdNnBXMgDgpzCRbmQWMHjyPvZny + title: Read BigTable RunTime and EstimatedCost + date: + - id: szvwNfPrwTtmRmMHWv3QFh6wTKxm26TF + title: AvgOutputThroughputBytesPerSec by Date + - id: ZRQw7np2mj35kQHcvgtNDwVGgP855QNF + title: AvgOutputThroughputElementsPerSec by Date + version: + - id: X7y2hQQkJnYY8ctnr2y7NnNx7xrMkjt2 + title: AvgOutputThroughputBytesPerSec by Version + - id: B3HwmTtn9nBfzwRC7s3WfQSjf7ckrfXx + title: AvgOutputThroughputElementsPerSec by Version + write: + folder: 33 + test_name: + - id: 2sC27RQwWy2MP9DXVHjbvTYSNFYpFvxj + title: Write BigTable RunTime and EstimatedCost + date: + - id: X22sDqD8krBQQ4mRXTFMmpKFvgkZ529g + title: AvgInputThroughputBytesPerSec by Date + - id: WqHVhrjQVrrjpczqgJB9s9k8fvt3Vc2d + title: AvgInputThroughputElementsPerSec by Date + version: + - id: j72shCBz6rJhQP8JwYB3JcVrhv9BWGpD + title: AvgInputThroughputBytesPerSec by Version + - id: 6DyhkfvcNfkJg53hwFnRzxqbJDdGb5t7 + title: AvgInputThroughputElementsPerSec by Version + textio: + read: + folder: 34 + test_name: + - id: SsfFgvwyMthnHjdRwBjWGNbvfNgym4wb + title: Read TextIO RunTime and EstimatedCost + date: + - id: Pr8MBG66JVgBmdQbDr6HXtRprjr3ybj6 + title: AvgOutputThroughputBytesPerSec by Date + - id: CTZYK4KVYM65Zjn6jWgDYYNBWckDYKRQ + title: AvgOutputThroughputElementsPerSec by Date + version: + - id: Dcvfh3XFZySrsmPY4Rm8NYyMg5QQRBF6 + title: AvgOutputThroughputBytesPerSec by Version + - id: dN8mTZsVZc7vGDYKJCT8S67BCXzVJT4s + title: AvgOutputThroughputElementsPerSec by Version + write: + folder: 35 + test_name: + - id: Mm2j2QPc2x4hZqYNSpZC4bDpH2fgvqvp + title: Write TextIO RunTime and EstimatedCost + date: + - id: J9VXhp3ry5zbPFFGsYNfDRypGNVNMbPV + title: AvgInputThroughputBytesPerSec by Date + - id: KwSCnyz75wMpXhGpZQ8FRp6cwt8pjhfD + title: AvgInputThroughputElementsPerSec by Date + version: + - id: VFXbPV9JGJxmNYnGsypQzH97RPDFjpPN + title: AvgInputThroughputBytesPerSec by Version + - id: fVVHhXCrHNgBG52TJsTjR8VbmWCCQnVN + title: AvgInputThroughputElementsPerSec by Version diff --git a/website/www/site/i18n/navbar/en.yaml b/website/www/site/i18n/navbar/en.yaml index d356d282671eb..aeb1988183b0b 100644 --- a/website/www/site/i18n/navbar/en.yaml +++ b/website/www/site/i18n/navbar/en.yaml @@ -48,3 +48,13 @@ translation: "About" - id: nav-connectors translation: "I/O Connectors" +- id: nav-performance-general + translation: "General" +- id: nav-performance-bigquery + translation: "BigQuery" +- id: nav-performance-bigtable + translation: "BigTable" +- id: nav-performance-textio + translation: "TextIO" +- id: nav-performance-glossary + translation: "Glossary" diff --git a/website/www/site/layouts/index.html b/website/www/site/layouts/index.html index 99254978ccd2b..4626e2dc2455b 100644 --- a/website/www/site/layouts/index.html +++ b/website/www/site/layouts/index.html @@ -53,7 +53,7 @@

    Try Beam Playground

    Beam Playground is an interactive environment to try out Beam transforms and examples without having to install Apache Beam in your environment. - You can try the Apache Beam examples at Beam Playground (Beta). + You can try the Apache Beam examples at Beam Playground.



    diff --git a/website/www/site/layouts/partials/feedback.html b/website/www/site/layouts/partials/feedback.html index cfc236032f58e..be65e540db12f 100644 --- a/website/www/site/layouts/partials/feedback.html +++ b/website/www/site/layouts/partials/feedback.html @@ -18,5 +18,5 @@

    {{ T "feedback-title" }}

    {{ T "feedback-description" }}

    - +
    diff --git a/website/www/site/layouts/partials/header.html b/website/www/site/layouts/partials/header.html index 3fa8bdc9455c3..957e3de2b1ed0 100644 --- a/website/www/site/layouts/partials/header.html +++ b/website/www/site/layouts/partials/header.html @@ -208,9 +208,9 @@
    diff --git a/website/www/site/layouts/partials/section-menu/en/contribute.html b/website/www/site/layouts/partials/section-menu/en/contribute.html index 4cf4e89f3ddd9..017151e01d5cd 100644 --- a/website/www/site/layouts/partials/section-menu/en/contribute.html +++ b/website/www/site/layouts/partials/section-menu/en/contribute.html @@ -11,7 +11,7 @@ */}}
  • Contribute
  • -
  • Code contribution guide
  • +
  • Code contribution guide
  • Get help
  • Attributes of a Beam community member
  • @@ -22,9 +22,7 @@
  • Pre-commit slowness triage
  • PTransform style guide
  • Runner authoring guide
  • -
  • Design documents
  • Dependencies guide
  • -
  • Feature branches
  • @@ -40,7 +38,5 @@ Committers
  • diff --git a/website/www/site/layouts/partials/section-menu/en/documentation.html b/website/www/site/layouts/partials/section-menu/en/documentation.html index adde851260783..2ab5bec69e989 100755 --- a/website/www/site/layouts/partials/section-menu/en/documentation.html +++ b/website/www/site/layouts/partials/section-menu/en/documentation.html @@ -232,6 +232,7 @@
  • Data processing
  • @@ -291,6 +292,7 @@
  • Keys
  • KvSwap
  • Map
  • +
  • MLTransform
  • ParDo
  • Partition
  • Regex
  • diff --git a/website/www/site/layouts/partials/section-menu/en/get-started.html b/website/www/site/layouts/partials/section-menu/en/get-started.html index f43a9e4797bb4..ec1712fd17d28 100644 --- a/website/www/site/layouts/partials/section-menu/en/get-started.html +++ b/website/www/site/layouts/partials/section-menu/en/get-started.html @@ -12,14 +12,14 @@
  • Get started
  • Beam Overview
  • -
  • Tour of Beam
  • +
  • An Interactive Overview of Beam
  • Quickstarts
  • diff --git a/website/www/site/layouts/performance/baseof.html b/website/www/site/layouts/performance/baseof.html new file mode 100644 index 0000000000000..06aa0031077fa --- /dev/null +++ b/website/www/site/layouts/performance/baseof.html @@ -0,0 +1,40 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. See accompanying LICENSE file. +*/}} + + + + + {{ partial "head.html" . }} + + +{{ partial "header.html" . }} +
    +
    + + +
    + + + +
    + {{ .Content }} + {{ partial "feedback.html" . }} +
    +
    +{{ partial "footer.html" . }} + + \ No newline at end of file diff --git a/website/www/site/layouts/shortcodes/documentation/sdks.html b/website/www/site/layouts/shortcodes/documentation/sdks.html index 57e7a35cb1304..6b0dfa1c42e45 100644 --- a/website/www/site/layouts/shortcodes/documentation/sdks.html +++ b/website/www/site/layouts/shortcodes/documentation/sdks.html @@ -12,10 +12,9 @@ {{ $data := index $.Site.Data .Site.Language.Lang }}
    - {{ range $item := $data.documentation_sdks }} -
    - {{ .name.text | markdownify }} -

    {{ .text | safeHTML }}

    -
    - {{ end }} +
    diff --git a/website/www/site/layouts/shortcodes/performance_looks.html b/website/www/site/layouts/shortcodes/performance_looks.html new file mode 100644 index 0000000000000..1a4f1d0a578fd --- /dev/null +++ b/website/www/site/layouts/shortcodes/performance_looks.html @@ -0,0 +1,30 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. See accompanying LICENSE file. +*/}} + +
    + {{ $host := .Site.Data.performance.host }} + {{ $path := .Site.Data.performance.path }} + {{ $looks := .Site.Data.performance.looks }} + {{ $io := index $looks (.Get "io") }} + {{ $rw := index $io (.Get "read_or_write") }} + {{ $section := index $rw (.Get "section") }} + {{ range $section }} + {{ $folder := index $rw "folder" }} +

    {{.title}}

    +
    + {{.title}} +
    + {{ end }} +
    diff --git a/website/www/site/static/.htaccess b/website/www/site/static/.htaccess index a2ef056a262cb..ff5e56fc37dd0 100644 --- a/website/www/site/static/.htaccess +++ b/website/www/site/static/.htaccess @@ -22,3 +22,5 @@ RewriteRule ^(.*)$ https://beam.apache.org/$1 [L,R=301] RedirectMatch permanent "/documentation/sdks/(javadoc|pydoc)(.*)" "https://beam.apache.org/releases/$1$2" RedirectMatch "/contribute/design-documents" "https://cwiki.apache.org/confluence/display/BEAM/Design+Documents" + +RedirectMatch "/contribute/release-guide" "https://github.com/apache/beam/blob/master/contributor-docs/release-guide.md" diff --git a/website/www/site/static/images/blog/dyi-cdp-genai-beam/cdp-arch.png b/website/www/site/static/images/blog/dyi-cdp-genai-beam/cdp-arch.png new file mode 100644 index 0000000000000..ef42340e905f5 Binary files /dev/null and b/website/www/site/static/images/blog/dyi-cdp-genai-beam/cdp-arch.png differ diff --git a/website/www/site/static/images/blog/dyi-cdp-genai-beam/cdp-highlevel.png b/website/www/site/static/images/blog/dyi-cdp-genai-beam/cdp-highlevel.png new file mode 100644 index 0000000000000..5f25462bbcd4e Binary files /dev/null and b/website/www/site/static/images/blog/dyi-cdp-genai-beam/cdp-highlevel.png differ diff --git a/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-1.png b/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-1.png new file mode 100644 index 0000000000000..b3f6e926b032f Binary files /dev/null and b/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-1.png differ diff --git a/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-2-extractcontent.png b/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-2-extractcontent.png new file mode 100644 index 0000000000000..6064df3455b54 Binary files /dev/null and b/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-2-extractcontent.png differ diff --git a/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-3-errorhandling.png b/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-3-errorhandling.png new file mode 100644 index 0000000000000..77829f943fd9d Binary files /dev/null and b/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-3-errorhandling.png differ diff --git a/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-4-processembeddings1.png b/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-4-processembeddings1.png new file mode 100644 index 0000000000000..b28e44a636b0d Binary files /dev/null and b/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-4-processembeddings1.png differ diff --git a/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-4-processembeddings2.png b/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-4-processembeddings2.png new file mode 100644 index 0000000000000..3649f7a563fc0 Binary files /dev/null and b/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-4-processembeddings2.png differ diff --git a/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-5-storecontent.png b/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-5-storecontent.png new file mode 100644 index 0000000000000..32edfd280a960 Binary files /dev/null and b/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-5-storecontent.png differ diff --git a/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-6-refresh1.png b/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-6-refresh1.png new file mode 100644 index 0000000000000..6683f6503e0f3 Binary files /dev/null and b/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-6-refresh1.png differ diff --git a/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-6-refresh2.png b/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-6-refresh2.png new file mode 100644 index 0000000000000..0fc1410d5bbd7 Binary files /dev/null and b/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-6-refresh2.png differ diff --git a/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-6-refresh3.png b/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-6-refresh3.png new file mode 100644 index 0000000000000..d90853fc1daa6 Binary files /dev/null and b/website/www/site/static/images/blog/dyi-cdp-genai-beam/pipeline-6-refresh3.png differ diff --git a/website/www/site/static/images/case-study/hsbc/andrzej_golonka.jpg b/website/www/site/static/images/case-study/hsbc/andrzej_golonka.jpg new file mode 100644 index 0000000000000..8e44fa6ed9497 Binary files /dev/null and b/website/www/site/static/images/case-study/hsbc/andrzej_golonka.jpg differ diff --git a/website/www/site/static/images/case-study/hsbc/chup_cheng.jpg b/website/www/site/static/images/case-study/hsbc/chup_cheng.jpg new file mode 100644 index 0000000000000..8fbb80433524e Binary files /dev/null and b/website/www/site/static/images/case-study/hsbc/chup_cheng.jpg differ diff --git a/website/www/site/static/images/case-study/hsbc/scheme-10.png b/website/www/site/static/images/case-study/hsbc/scheme-10.png new file mode 100644 index 0000000000000..941559f44b5b2 Binary files /dev/null and b/website/www/site/static/images/case-study/hsbc/scheme-10.png differ diff --git a/website/www/site/static/images/case-study/hsbc/scheme-11.png b/website/www/site/static/images/case-study/hsbc/scheme-11.png new file mode 100644 index 0000000000000..354c99b444e4b Binary files /dev/null and b/website/www/site/static/images/case-study/hsbc/scheme-11.png differ diff --git a/website/www/site/static/images/case-study/hsbc/scheme-9.png b/website/www/site/static/images/case-study/hsbc/scheme-9.png new file mode 100644 index 0000000000000..15b95c454700c Binary files /dev/null and b/website/www/site/static/images/case-study/hsbc/scheme-9.png differ diff --git a/website/www/site/static/images/case-study/linkedin/bingfeng-xia.jpg b/website/www/site/static/images/case-study/linkedin/bingfeng-xia.jpg new file mode 100644 index 0000000000000..ca07935b689b6 Binary files /dev/null and b/website/www/site/static/images/case-study/linkedin/bingfeng-xia.jpg differ diff --git a/website/www/site/static/images/case-study/linkedin/scheme-1.png b/website/www/site/static/images/case-study/linkedin/scheme-1.png new file mode 100644 index 0000000000000..535f2d5f316f8 Binary files /dev/null and b/website/www/site/static/images/case-study/linkedin/scheme-1.png differ diff --git a/website/www/site/static/images/case-study/linkedin/scheme-2.png b/website/www/site/static/images/case-study/linkedin/scheme-2.png new file mode 100644 index 0000000000000..2ab4e42810195 Binary files /dev/null and b/website/www/site/static/images/case-study/linkedin/scheme-2.png differ diff --git a/website/www/site/static/images/case-study/linkedin/scheme-3.png b/website/www/site/static/images/case-study/linkedin/scheme-3.png new file mode 100644 index 0000000000000..a7d1dd01b88ed Binary files /dev/null and b/website/www/site/static/images/case-study/linkedin/scheme-3.png differ diff --git a/website/www/site/static/images/case-study/linkedin/scheme-4.png b/website/www/site/static/images/case-study/linkedin/scheme-4.png new file mode 100644 index 0000000000000..3873b3a20b1f1 Binary files /dev/null and b/website/www/site/static/images/case-study/linkedin/scheme-4.png differ diff --git a/website/www/site/static/images/case-study/linkedin/scheme-5.png b/website/www/site/static/images/case-study/linkedin/scheme-5.png new file mode 100644 index 0000000000000..e28537a18a8b1 Binary files /dev/null and b/website/www/site/static/images/case-study/linkedin/scheme-5.png differ diff --git a/website/www/site/static/images/case-study/linkedin/scheme-6.png b/website/www/site/static/images/case-study/linkedin/scheme-6.png new file mode 100644 index 0000000000000..1dadc4c9126e5 Binary files /dev/null and b/website/www/site/static/images/case-study/linkedin/scheme-6.png differ diff --git a/website/www/site/static/images/case-study/linkedin/xinyu-liu.jpg b/website/www/site/static/images/case-study/linkedin/xinyu-liu.jpg new file mode 100644 index 0000000000000..89813af2b09df Binary files /dev/null and b/website/www/site/static/images/case-study/linkedin/xinyu-liu.jpg differ diff --git a/website/www/site/static/images/case-study/octo/florian-bastin.jpg b/website/www/site/static/images/case-study/octo/florian-bastin.jpg new file mode 100644 index 0000000000000..b401da0ce8719 Binary files /dev/null and b/website/www/site/static/images/case-study/octo/florian-bastin.jpg differ diff --git a/website/www/site/static/images/case-study/octo/godefroy-clair.png b/website/www/site/static/images/case-study/octo/godefroy-clair.png new file mode 100644 index 0000000000000..20600e7719dca Binary files /dev/null and b/website/www/site/static/images/case-study/octo/godefroy-clair.png differ diff --git a/website/www/site/static/images/case-study/octo/leo-babonnaud.jpg b/website/www/site/static/images/case-study/octo/leo-babonnaud.jpg new file mode 100644 index 0000000000000..fa0309fc5a4e5 Binary files /dev/null and b/website/www/site/static/images/case-study/octo/leo-babonnaud.jpg differ diff --git a/website/www/site/static/images/case-study/octo/scheme-14.png b/website/www/site/static/images/case-study/octo/scheme-14.png new file mode 100644 index 0000000000000..ea04c1fe7f0b1 Binary files /dev/null and b/website/www/site/static/images/case-study/octo/scheme-14.png differ diff --git a/website/www/site/static/images/college_2023_banner_desktop.png b/website/www/site/static/images/college_2023_banner_desktop.png new file mode 100644 index 0000000000000..09e0cdfe24b97 Binary files /dev/null and b/website/www/site/static/images/college_2023_banner_desktop.png differ diff --git a/website/www/site/static/images/college_2023_banner_mobile.png b/website/www/site/static/images/college_2023_banner_mobile.png new file mode 100644 index 0000000000000..e6b733a7e2ef4 Binary files /dev/null and b/website/www/site/static/images/college_2023_banner_mobile.png differ diff --git a/website/www/site/static/images/logos/powered-by/hsbc.png b/website/www/site/static/images/logos/powered-by/hsbc.png new file mode 100644 index 0000000000000..b6cc7d369e3a5 Binary files /dev/null and b/website/www/site/static/images/logos/powered-by/hsbc.png differ diff --git a/website/www/site/static/images/logos/powered-by/octo.png b/website/www/site/static/images/logos/powered-by/octo.png new file mode 100644 index 0000000000000..d71fc3b8aa80a Binary files /dev/null and b/website/www/site/static/images/logos/powered-by/octo.png differ diff --git a/website/www/site/static/images/transform_service.png b/website/www/site/static/images/transform_service.png new file mode 100644 index 0000000000000..f3763bac64eea Binary files /dev/null and b/website/www/site/static/images/transform_service.png differ